From 516952f18c83c2eb84990217c29e8ff1b27295bb Mon Sep 17 00:00:00 2001 From: Aman India Date: Thu, 23 Oct 2025 11:47:01 +0530 Subject: [PATCH 1/4] Refactor initial project structure and add basic setup files. --- noveum_customer_support_bt/README_workflow.md | 80 + noveum_customer_support_bt/api_data.json | 84 + noveum_customer_support_bt/create_dataset.py | 161 + .../create_dataset_version.py | 123 + .../agent_evaluation_results.csv | 21 + .../agent_evaluation_results.csv | 13 + .../agent_evaluation_results.csv | 9 + noveum_customer_support_bt/demo_utils.py | 1025 + .../fetch_dataset_items.py | 94 + .../final_agent_evaluation_demo.ipynb | 251 + noveum_customer_support_bt/fix_api_data_v2.py | 61 + noveum_customer_support_bt/novapilot_utils.py | 561 + .../noveum_agent_requirements.txt | 33 + noveum_customer_support_bt/noveum_docs.json | 378 + .../noveum_user_bot.ipynb | 7994 ++++++ .../noveum_vectorstore/index.faiss | Bin 0 -> 1130541 bytes .../noveum_vectorstore/index.pkl | Bin 0 -> 188134 bytes .../preprocess_filter.py | 120 + noveum_customer_support_bt/preprocess_map.py | 219 + .../preprocess_split_data.py | 119 + ...gent.llm-rag_dataset_processed_dataset.csv | 13 + ...ent.llm-rag_dataset_processed_dataset.json | 266 + ...el_execution_dataset_processed_dataset.csv | 21 + ...l_execution_dataset_processed_dataset.json | 442 + ...uery_routing_dataset_processed_dataset.csv | 21 + ...ery_routing_dataset_processed_dataset.json | 622 + ...tion_metrics_dataset_processed_dataset.csv | 22 + ...ion_metrics_dataset_processed_dataset.json | 446 + ...tion_metrics_dataset_processed_dataset.csv | 21 + ...ion_metrics_dataset_processed_dataset.json | 442 + ...tion_metrics_dataset_processed_dataset.csv | 14 + ...ion_metrics_dataset_processed_dataset.json | 298 + ...h_generation_dataset_processed_dataset.csv | 9 + ..._generation_dataset_processed_dataset.json | 178 + ...-orchestator_dataset_processed_dataset.csv | 21 + ...orchestator_dataset_processed_dataset.json | 442 + .../publish_dataset_version.py | 117 + .../split_datasets/agent.llm-rag_dataset.json | 734 + .../agent.llm_model_execution_dataset.json | 1782 ++ .../agent.query_routing_dataset.json | 1222 + .../agent.rag_evaluation_metrics_dataset.json | 998 + ...nt.routing_evaluation_metrics_dataset.json | 1602 ++ ...web_search_evaluation_metrics_dataset.json | 682 + .../agent.web_search_generation_dataset.json | 490 + .../tool-orchestator_dataset.json | 1162 + noveum_customer_support_bt/support_agent.png | Bin 0 -> 418313 bytes .../traces/combine_spans_api_compat.py | 103 + .../traces/fetch_traces_api.py | 161 + .../traces/traces/dataset.json | 20904 ++++++++++++++++ .../traces/traces/dataset_filtered.json | 20904 ++++++++++++++++ .../traces/dataset_filtered_mapped.json | 20904 ++++++++++++++++ .../traces/traces/traces_batch_001.json | 13915 ++++++++++ noveum_customer_support_bt/upload_dataset.py | 213 + noveum_customer_support_bt/upload_scores.py | 337 + 54 files changed, 100854 insertions(+) create mode 100644 noveum_customer_support_bt/README_workflow.md create mode 100644 noveum_customer_support_bt/api_data.json create mode 100644 noveum_customer_support_bt/create_dataset.py create mode 100644 noveum_customer_support_bt/create_dataset_version.py create mode 100644 noveum_customer_support_bt/demo_results/agent.query_routing_dataset/agent_evaluation_results.csv create mode 100644 noveum_customer_support_bt/demo_results/agent.rag_evaluation_metrics_dataset/agent_evaluation_results.csv create mode 100644 noveum_customer_support_bt/demo_results/agent.web_search_evaluation_metrics_dataset/agent_evaluation_results.csv create mode 100644 noveum_customer_support_bt/demo_utils.py create mode 100644 noveum_customer_support_bt/fetch_dataset_items.py create mode 100644 noveum_customer_support_bt/final_agent_evaluation_demo.ipynb create mode 100644 noveum_customer_support_bt/fix_api_data_v2.py create mode 100644 noveum_customer_support_bt/novapilot_utils.py create mode 100644 noveum_customer_support_bt/noveum_agent_requirements.txt create mode 100644 noveum_customer_support_bt/noveum_docs.json create mode 100644 noveum_customer_support_bt/noveum_user_bot.ipynb create mode 100644 noveum_customer_support_bt/noveum_vectorstore/index.faiss create mode 100644 noveum_customer_support_bt/noveum_vectorstore/index.pkl create mode 100644 noveum_customer_support_bt/preprocess_filter.py create mode 100644 noveum_customer_support_bt/preprocess_map.py create mode 100644 noveum_customer_support_bt/preprocess_split_data.py create mode 100644 noveum_customer_support_bt/processed_datasets/agent.llm-rag_dataset_processed_dataset.csv create mode 100644 noveum_customer_support_bt/processed_datasets/agent.llm-rag_dataset_processed_dataset.json create mode 100644 noveum_customer_support_bt/processed_datasets/agent.llm_model_execution_dataset_processed_dataset.csv create mode 100644 noveum_customer_support_bt/processed_datasets/agent.llm_model_execution_dataset_processed_dataset.json create mode 100644 noveum_customer_support_bt/processed_datasets/agent.query_routing_dataset_processed_dataset.csv create mode 100644 noveum_customer_support_bt/processed_datasets/agent.query_routing_dataset_processed_dataset.json create mode 100644 noveum_customer_support_bt/processed_datasets/agent.rag_evaluation_metrics_dataset_processed_dataset.csv create mode 100644 noveum_customer_support_bt/processed_datasets/agent.rag_evaluation_metrics_dataset_processed_dataset.json create mode 100644 noveum_customer_support_bt/processed_datasets/agent.routing_evaluation_metrics_dataset_processed_dataset.csv create mode 100644 noveum_customer_support_bt/processed_datasets/agent.routing_evaluation_metrics_dataset_processed_dataset.json create mode 100644 noveum_customer_support_bt/processed_datasets/agent.web_search_evaluation_metrics_dataset_processed_dataset.csv create mode 100644 noveum_customer_support_bt/processed_datasets/agent.web_search_evaluation_metrics_dataset_processed_dataset.json create mode 100644 noveum_customer_support_bt/processed_datasets/agent.web_search_generation_dataset_processed_dataset.csv create mode 100644 noveum_customer_support_bt/processed_datasets/agent.web_search_generation_dataset_processed_dataset.json create mode 100644 noveum_customer_support_bt/processed_datasets/tool-orchestator_dataset_processed_dataset.csv create mode 100644 noveum_customer_support_bt/processed_datasets/tool-orchestator_dataset_processed_dataset.json create mode 100644 noveum_customer_support_bt/publish_dataset_version.py create mode 100644 noveum_customer_support_bt/split_datasets/agent.llm-rag_dataset.json create mode 100644 noveum_customer_support_bt/split_datasets/agent.llm_model_execution_dataset.json create mode 100644 noveum_customer_support_bt/split_datasets/agent.query_routing_dataset.json create mode 100644 noveum_customer_support_bt/split_datasets/agent.rag_evaluation_metrics_dataset.json create mode 100644 noveum_customer_support_bt/split_datasets/agent.routing_evaluation_metrics_dataset.json create mode 100644 noveum_customer_support_bt/split_datasets/agent.web_search_evaluation_metrics_dataset.json create mode 100644 noveum_customer_support_bt/split_datasets/agent.web_search_generation_dataset.json create mode 100644 noveum_customer_support_bt/split_datasets/tool-orchestator_dataset.json create mode 100644 noveum_customer_support_bt/support_agent.png create mode 100644 noveum_customer_support_bt/traces/combine_spans_api_compat.py create mode 100755 noveum_customer_support_bt/traces/fetch_traces_api.py create mode 100644 noveum_customer_support_bt/traces/traces/dataset.json create mode 100644 noveum_customer_support_bt/traces/traces/dataset_filtered.json create mode 100644 noveum_customer_support_bt/traces/traces/dataset_filtered_mapped.json create mode 100644 noveum_customer_support_bt/traces/traces/traces_batch_001.json create mode 100644 noveum_customer_support_bt/upload_dataset.py create mode 100644 noveum_customer_support_bt/upload_scores.py diff --git a/noveum_customer_support_bt/README_workflow.md b/noveum_customer_support_bt/README_workflow.md new file mode 100644 index 0000000..054f833 --- /dev/null +++ b/noveum_customer_support_bt/README_workflow.md @@ -0,0 +1,80 @@ +# Noveum Platform Score Upload Workflow + +## Prerequisites +- Virtual environment activated +- .env file configured with API credentials +- Processed dataset available +- Evaluation results CSV available + +## Step-by-Step Commands + +### Step 1: Setup Environment +```bash +cd /Users/mramanindia/work/NovaEval +source .venv/bin/activate +cd noveum_customer_support_bt +``` + +### Step 2: Create Dataset +```bash +python create_dataset.py --dataset-type agent --description "Customer Support Agent Evaluation Dataset" --pretty +``` +**Note**: After this step, update your .env file with the returned dataset slug if different. + +### Step 3: Create Dataset Version +```bash +python create_dataset_version.py --pretty +``` + +### Step 4: Upload Dataset Items +```bash +python upload_dataset.py --dataset-json processed_datasets/agent.rag_evaluation_metrics_dataset_processed_dataset.json --item-type conversation +``` + +### Step 5: Publish Dataset Version +```bash +python publish_dataset_version.py --pretty +``` + +### Step 6: Upload Evaluation Scores + +#### Option A: Upload All Scores Separately +```bash +# Task Progression Scores +python upload_scores.py demo_results/agent.rag_evaluation_metrics_dataset/agent_evaluation_results.csv --item-key-col turn_id --score-col task_progression --reasoning-col task_progression_reasoning --scorer-id task_progression_scorer --scorer-version 1.0.0 + +# Context Relevancy Scores +python upload_scores.py demo_results/agent.rag_evaluation_metrics_dataset/agent_evaluation_results.csv --item-key-col turn_id --score-col context_relevancy --reasoning-col context_relevancy_reasoning --scorer-id context_relevancy_scorer --scorer-version 1.0.0 + +# Role Adherence Scores +python upload_scores.py demo_results/agent.rag_evaluation_metrics_dataset/agent_evaluation_results.csv --item-key-col turn_id --score-col role_adherence --reasoning-col role_adherence_reasoning --scorer-id role_adherence_scorer --scorer-version 1.0.0 + +# Tool Relevancy Scores +python upload_scores.py demo_results/agent.rag_evaluation_metrics_dataset/agent_evaluation_results.csv --item-key-col turn_id --score-col tool_relevancy --reasoning-col tool_relevancy_reasoning --scorer-id tool_relevancy_scorer --scorer-version 1.0.0 + +# Parameter Correctness Scores +python upload_scores.py demo_results/agent.rag_evaluation_metrics_dataset/agent_evaluation_results.csv --item-key-col turn_id --score-col parameter_correctness --reasoning-col parameter_correctness_reasoning --scorer-id parameter_correctness_scorer --scorer-version 1.0.0 +``` + +#### Option B: Test with Dry Run First +Add `--dry-run` flag to any upload command to test without actually uploading: +```bash +python upload_scores.py demo_results/agent.rag_evaluation_metrics_dataset/agent_evaluation_results.csv --item-key-col turn_id --score-col task_progression --reasoning-col task_progression_reasoning --scorer-id task_progression_scorer --scorer-version 1.0.0 --dry-run +``` + +## Environment Variables Required +Make sure your .env file contains: +- NOVEUM_API_KEY +- NOVEUM_ORG_SLUG +- NOVEUM_DATASET_SLUG +- NOVEUM_DATASET_NAME +- LATEST_VERSION +- NOVEUM_PROJECT +- NOVEUM_ENVIRONMENT +- BETA (true/false) + +## Troubleshooting +- If dataset creation fails, check if dataset already exists +- If upload fails, verify the JSON format matches expected schema +- Use --pretty flag for better formatted output +- Check API responses for specific error messages diff --git a/noveum_customer_support_bt/api_data.json b/noveum_customer_support_bt/api_data.json new file mode 100644 index 0000000..5bc3e2d --- /dev/null +++ b/noveum_customer_support_bt/api_data.json @@ -0,0 +1,84 @@ +{ + "items": [ + { + "item_key": "eda4fe22-9a2b-4b73-856b-f4f3309bf719", + "item_id": "item_1" + }, + { + "item_key": "0ffffba1-8a37-443c-8866-d53ffbfa7718", + "item_id": "item_2" + }, + { + "item_key": "f1f37bd7-0851-4659-b493-b80d3800d920", + "item_id": "item_3" + }, + { + "item_key": "43cdf081-4f01-49cd-b566-dbd1619e6cd2", + "item_id": "item_4" + }, + { + "item_key": "9a1983f4-09da-4b53-80e6-38de6878e0e7", + "item_id": "item_5" + }, + { + "item_key": "5d2517e1-220a-429d-9d59-f701bda25eed", + "item_id": "item_6" + }, + { + "item_key": "52aacb67-c361-4445-9b72-c157f79f47d6", + "item_id": "item_7" + }, + { + "item_key": "a81ca3a8-80aa-4c39-876e-8d40ea7a0aef", + "item_id": "item_8" + }, + { + "item_key": "230aad27-f3dd-4968-a45a-3c2f07ac28ed", + "item_id": "item_9" + }, + { + "item_key": "83c7dcce-3d89-4da1-8b3f-d419885d4cbc", + "item_id": "item_10" + }, + { + "item_key": "2218f641-604c-491a-9710-b51a9941b982", + "item_id": "item_11" + }, + { + "item_key": "255fd49c-84b4-4b18-887e-6308a412d535", + "item_id": "item_12" + }, + { + "item_key": "dc511122-c0b6-415c-9a49-c7b45132dd87", + "item_id": "item_13" + }, + { + "item_key": "04bebf38-a343-4563-80db-0154bef8d927", + "item_id": "item_14" + }, + { + "item_key": "5e043630-6493-42b5-beb8-79faa19bfa37", + "item_id": "item_15" + }, + { + "item_key": "7da9814d-a2e8-4c4e-b750-68b26bd5fd22", + "item_id": "item_16" + }, + { + "item_key": "16143f74-2831-4753-b33d-ce4b645093c5", + "item_id": "item_17" + }, + { + "item_key": "fc64e6cc-6739-4256-ac4a-7b80c3028233", + "item_id": "item_18" + }, + { + "item_key": "b7945c49-f584-4c70-972d-536a805d8a31", + "item_id": "item_19" + }, + { + "item_key": "f5c40ecf-36c0-45ba-9cc9-dc0329b0324b", + "item_id": "item_20" + } + ] +} \ No newline at end of file diff --git a/noveum_customer_support_bt/create_dataset.py b/noveum_customer_support_bt/create_dataset.py new file mode 100644 index 0000000..9aaf601 --- /dev/null +++ b/noveum_customer_support_bt/create_dataset.py @@ -0,0 +1,161 @@ +#!/usr/bin/env python3 +""" +Script to create a new dataset in Noveum API. +Creates a dataset with the specified name and type. +""" + +import os +import json +import requests +import argparse +from dotenv import load_dotenv +from typing import Dict, Any, Optional + +# Load environment variables +load_dotenv() + +# Get API credentials from environment +api_key = os.getenv('NOVEUM_API_KEY') +org_slug = os.getenv('NOVEUM_ORG_SLUG') +dataset_slug = os.getenv('NOVEUM_DATASET_SLUG') +dataset_name = os.getenv('NOVEUM_DATASET_SLUG') +beta_env = os.getenv('BETA', 'false').lower() == 'true' + +def validate_environment(): + """Validate that all required environment variables are set""" + required_vars = { + 'NOVEUM_API_KEY': api_key, + 'NOVEUM_ORG_SLUG': org_slug, + 'NOVEUM_DATASET_SLUG': dataset_slug, + 'NOVEUM_DATASET_NAME': dataset_name + } + + missing_vars = [var for var, value in required_vars.items() if not value] + + if missing_vars: + print(f"Error: Missing required environment variables: {', '.join(missing_vars)}") + print("Please set these variables in your .env file or environment") + return False + + return True + +def create_dataset(dataset_type: str, description: str = "", visibility: str = "org", environment: str = "") -> Optional[Dict[str, Any]]: + """Create a new dataset in Noveum API""" + + # Construct API URL based on BETA environment variable + if beta_env: + api_url = f"https://noveum.ai/api/v1/datasets" + else: + api_url = f"https://noveum.ai/api/v1/organizations/{org_slug}/datasets" + + # Prepare headers + headers = { + 'Content-Type': 'application/json', + 'Authorization': f'Bearer {api_key}', + 'Cookie': f'apiKeyCookie={api_key}' + } + + # Prepare request data + request_data = { + "name": dataset_name, + "slug": dataset_name, # Will be auto-generated by the API + "description": description, + "visibility": visibility, + "dataset_type": dataset_type, + "environment": environment + } + + print(f"Creating dataset at: {api_url}") + print(f"Organization: {org_slug}") + print(f"Dataset name: {dataset_name}") + print(f"Dataset type: {dataset_type}") + print(f"Description: {description}") + print(f"Visibility: {visibility}") + print(f"Environment: {environment}") + + try: + response = requests.post(api_url, headers=headers, json=request_data, timeout=30) + response.raise_for_status() + + data = response.json() + print("Successfully created dataset") + print(f"Response status: {response.status_code}") + + return data + + except requests.exceptions.RequestException as e: + print(f"Error creating dataset: {e}") + if hasattr(e, 'response') and e.response is not None: + print(f"Response status: {e.response.status_code}") + print(f"Response text: {e.response.text}") + return None + +def main(): + # Define valid dataset types + valid_dataset_types = ['agent', 'conversational', 'g-eval', 'custom'] + + parser = argparse.ArgumentParser(description='Create a new dataset in Noveum API') + parser.add_argument('--dataset-type', type=str, default='agent', + choices=valid_dataset_types, + help=f'Type of the dataset. Must be one of: {", ".join(valid_dataset_types)} (default: agent)') + parser.add_argument('--description', type=str, default="", + help='Description of the dataset (default: empty string)') + parser.add_argument('--visibility', type=str, default="org", + help='Visibility of the dataset (default: org)') + parser.add_argument('--environment', type=str, default="", + help='Environment for the dataset (default: empty string)') + parser.add_argument('--pretty', action='store_true', + help='Pretty print the JSON response') + parser.add_argument('--output', type=str, default="dataset_creation_response.json", + help='Output file to save the JSON response (default: dataset_creation_response.json)') + + args = parser.parse_args() + + # Print warning about dataset slug + print("āš ļø WARNING: Please update the dataset slug in your .env file after creating the dataset!") + print(" The API will return a slug that you should set as NOVEUM_DATASET_SLUG in your .env file.") + print() + + # Validate environment variables + if not validate_environment(): + return 1 + + # Create dataset + data = create_dataset( + dataset_type=args.dataset_type, + description=args.description, + visibility=args.visibility, + environment=args.environment + ) + + if data is None: + return 1 + + # Save response to file + try: + with open(args.output, 'w', encoding='utf-8') as f: + json.dump(data, f, indent=2) + print(f"\nResponse saved to: {args.output}") + except (OSError, IOError) as e: + print(f"Error saving response to file: {e}") + return 1 + + # Print the response + if args.pretty: + print("\nResponse data:") + print(json.dumps(data, indent=2)) + else: + print(f"\nResponse data: {json.dumps(data)}") + + # Extract and display the dataset slug if available + if isinstance(data, dict) and 'slug' in data: + print(f"\nšŸ“ IMPORTANT: Update your .env file with:") + print(f" NOVEUM_DATASET_SLUG={data['slug']}") + elif isinstance(data, dict) and 'data' in data and isinstance(data['data'], dict) and 'slug' in data['data']: + print(f"\nšŸ“ IMPORTANT: Update your .env file with:") + print(f" NOVEUM_DATASET_SLUG={data['data']['slug']}") + + return 0 + +if __name__ == "__main__": + exit(main()) diff --git a/noveum_customer_support_bt/create_dataset_version.py b/noveum_customer_support_bt/create_dataset_version.py new file mode 100644 index 0000000..bdc6632 --- /dev/null +++ b/noveum_customer_support_bt/create_dataset_version.py @@ -0,0 +1,123 @@ +#!/usr/bin/env python3 +""" +Script to create a new dataset version in Noveum API. +Creates a new version for the specified dataset. +""" + +import os +import json +import requests +import argparse +from dotenv import load_dotenv +from typing import Dict, Any, Optional + +# Load environment variables +load_dotenv() + +# Get API credentials from environment +api_key = os.getenv('NOVEUM_API_KEY') +org_slug = os.getenv('NOVEUM_ORG_SLUG') +dataset_slug = os.getenv('NOVEUM_DATASET_SLUG') +latest_version = os.getenv('LATEST_VERSION') +beta_env = os.getenv('BETA', 'false').lower() == 'true' + +def validate_environment(): + """Validate that all required environment variables are set""" + required_vars = { + 'NOVEUM_API_KEY': api_key, + 'NOVEUM_ORG_SLUG': org_slug, + 'NOVEUM_DATASET_SLUG': dataset_slug, + 'LATEST_VERSION': latest_version + } + + missing_vars = [var for var, value in required_vars.items() if not value] + + if missing_vars: + print(f"Error: Missing required environment variables: {', '.join(missing_vars)}") + print("Please set these variables in your .env file or environment") + return False + + return True + +def create_dataset_version(version: str) -> Optional[Dict[str, Any]]: + """Create a new dataset version in Noveum API""" + + # Construct API URL based on BETA environment variable + if beta_env: + api_url = f"https://noveum.ai/api/v1/datasets/{dataset_slug}/versions?organizationSlug={org_slug}" + else: + api_url = f"https://noveum.ai/api/v1/organizations/{org_slug}/datasets/{dataset_slug}/versions" + + # Prepare headers + headers = { + 'Content-Type': 'application/json', + 'Authorization': f'Bearer {api_key}', + 'Cookie': f'apiKeyCookie={api_key}' + } + + # Prepare request data + request_data = { + "version": version + } + + print(f"Creating dataset version at: {api_url}") + print(f"Organization: {org_slug}") + print(f"Dataset: {dataset_slug}") + print(f"Version: {version}") + + try: + response = requests.post(api_url, headers=headers, json=request_data, timeout=30) + response.raise_for_status() + + data = response.json() + print("Successfully created dataset version") + print(f"Response status: {response.status_code}") + + return data + + except requests.exceptions.RequestException as e: + print(f"Error creating dataset version: {e}") + if hasattr(e, 'response') and e.response is not None: + print(f"Response status: {e.response.status_code}") + print(f"Response text: {e.response.text}") + return None + +def main(): + parser = argparse.ArgumentParser(description='Create a new dataset version in Noveum API') + parser.add_argument('--pretty', action='store_true', + help='Pretty print the JSON response') + parser.add_argument('--output', type=str, default="dataset_version_response.json", + help='Output file to save the JSON response (default: dataset_version_response.json)') + + args = parser.parse_args() + + # Validate environment variables + if not validate_environment(): + return 1 + + # Create dataset version + data = create_dataset_version(version=latest_version) + + if data is None: + return 1 + + # Save response to file + try: + with open(args.output, 'w', encoding='utf-8') as f: + json.dump(data, f, indent=2) + print(f"\nResponse saved to: {args.output}") + except (OSError, IOError) as e: + print(f"Error saving response to file: {e}") + return 1 + + # Print the response + if args.pretty: + print("\nResponse data:") + print(json.dumps(data, indent=2)) + else: + print(f"\nResponse data: {json.dumps(data)}") + + return 0 + +if __name__ == "__main__": + exit(main()) diff --git a/noveum_customer_support_bt/demo_results/agent.query_routing_dataset/agent_evaluation_results.csv b/noveum_customer_support_bt/demo_results/agent.query_routing_dataset/agent_evaluation_results.csv new file mode 100644 index 0000000..0bca67e --- /dev/null +++ b/noveum_customer_support_bt/demo_results/agent.query_routing_dataset/agent_evaluation_results.csv @@ -0,0 +1,21 @@ +user_id,task_id,turn_id,agent_name,task_progression,context_relevancy,role_adherence,tool_relevancy,parameter_correctness,task_progression_reasoning,context_relevancy_reasoning,role_adherence_reasoning,tool_relevancy_reasoning,parameter_correctness_reasoning +,eda4fe22-9a2b-4b73-856b-f4f3309bf719,0b19befc-5dc0-4b00-9a8b-5f2f2c914f17,agent,1.0,7.5,1.0,0.0,0.0,"The agent's response is missing. It's impossible to evaluate its understanding, alignment, progress, efficiency, or proximity to completion without any content provided. Therefore, the task progression is at its minimal level.","The agent's response should provide information about the latest breakthroughs in AI. Without seeing the actual response, it's impossible to confirm alignment with role and task. However, the premise indicates that the content should be factually correct and presented in an informative manner, suggesting an appropriate tone for the given role.",The agent completely fails to adhere to its role as it provided no response and made no tool calls. There is no evidence of an attempt to answer the question about AI breakthroughs. The agent's lack of any output indicates a total disregard for the task and role instructions.,"Error: Missing required fields: ['tool_calls'], out of all required fields: ['tools_available', 'tool_calls']","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']" +,0ffffba1-8a37-443c-8866-d53ffbfa7718,30644d2a-5ab1-4a73-b4ff-cf9fa478e36b,agent,2.8,7.8,10.0,0.0,0.0,"The agent identifies its role and task, but the response is missing. A response is not provided, hindering concrete progress. The agent is beginning to understand, but further actions need to be performed for better completion.","The agent response's direct relevance and its alignment with the agent's role are satisfactory. The tone and content are appropriate, showing understanding of the task. However, achieving a higher score requires a more detailed and insightful response on current trends, implying a need for better task completion progression.","The agent perfectly adheres to its role and task. The response is consistent with expectations for the assigned role. The absence of any tool calls is acceptable as there is no requirement to make a tool call in this scenario, therefore the agent has no need to deviate from its role.","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tools_available', 'tool_calls']","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']" +,f1f37bd7-0851-4659-b493-b80d3800d920,89eac2de-b7da-4db3-bf7c-8e87c429481e,agent,1.0,7.5,10.0,0.0,0.0,"The agent's response is missing. There is no information provided regarding the assigned task. Therefore, no progress has been made, and the response is unhelpful. A score of 1 reflects this lack of response and failure to address the prompt.",The response should list the newest features in cloud computing based on the agent's task. It should be aligned with the agent's role as well as the expertise. The tone and content are appropriate for the role. Demonstrating a deeper understanding of the task requirements would be key to increase the score.,The agent perfectly adheres to its role and task. The absence of tool calls indicates that the agent knows to respond without them. The agent does not stray from the assigned task and provides an expected answer while acting appropriately as an agent.,"Error: Missing required fields: ['tool_calls'], out of all required fields: ['tools_available', 'tool_calls']","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']" +,43cdf081-4f01-49cd-b566-dbd1619e6cd2,962afebb-26d1-45ff-acf1-218163cd6bb8,agent,1.0,7.5,10.0,0.0,0.0,"The agent's response is missing. It's impossible to evaluate its understanding, alignment, progress, efficiency, or proximity to completion without any content provided. Therefore, it made no progress and is off-topic. ","The agent's response should provide real-time or recent information regarding the current status of the cryptocurrency markets. Given the task of providing market updates, the agent's response needs to be relevant and aligned with the role of an information provider. The score reflects a solid starting point but can improve through richer details.","The agent perfectly adheres to its role and task. The absence of tool calls indicates no actions were taken, which aligns with the prompt requesting an evaluation without any tool usage. The response, while missing, doesn't deviate from the intended role of reporting current crypto market status. The consistency is flawless.","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tools_available', 'tool_calls']","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']" +,9a1983f4-09da-4b53-80e6-38de6878e0e7,246cf822-4562-4d3a-8a5a-e38e0a0b02a0,agent,2.8,7.8,1.0,0.0,0.0,"The agent identifies the task correctly. However, the agent's response is missing. There is no actual response regarding the latest updates in Python programming. Therefore, the progress is minimal as it shows understanding of the task, but provides no concrete information.","The agent response's direct relevance and role alignment is good. The tone and content are appropriate. However, the score could be higher if the response also demonstrated a helpful level of task understanding by providing specific examples or sources for the latest Python programming updates to aid task completion.","The agent provided no response and made no tool calls. This is a complete failure to adhere to the role and task. The agent should have provided some information, even if it had to use a tool to gather it. The lack of any output demonstrates a fundamental misunderstanding of the prompt's requirements.","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tools_available', 'tool_calls']","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']" +,5d2517e1-220a-429d-9d59-f701bda25eed,20544f91-8828-4a0e-b253-ed1f8b95b205,agent,1.0,1.0,1.0,0.0,0.0,"The agent's response is missing. It's impossible to determine if the agent understands its role or has made any progress. Therefore, the task progression is rated as minimal, with no concrete advancement toward the goal, as there is no response to evaluate.","The agent provided no response, which is completely irrelevant and inappropriate. It fails to address the task of summarizing tech news and demonstrates no understanding of the requirements. Therefore, the score is significantly low, reflecting the complete lack of a meaningful answer.","The agent completely fails to adhere to its role. It's supposed to summarize tech news but hasn't provided any information or made any tool calls. Its response is blank, and thus it cannot fulfill its assigned task. There are significant role contradictions and inappropriate behaviors due to this complete lack of output.","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tools_available', 'tool_calls']","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']" +,52aacb67-c361-4445-9b72-c157f79f47d6,5359c861-c51b-446d-9ad2-b3d8c21ea70f,agent,1.0,7.5,1.0,0.0,0.0,"The agent's response is missing. Without a response, it's impossible to determine if the agent understands its role or the assigned task. No progress has been made, as there's nothing to evaluate for alignment, concrete advancement, efficiency, or completeness.","The agent's response should provide a list of current trends in observability tools. This shows a good understanding of the task. The information should be factual and relevant. Its appropriateness hinges on the quality of the response, specifically the trends mentioned and their relevance in current times. A detailed response will increase the score.","The agent completely fails to adhere to its assigned role or perform the task. No tool calls were made, indicating a complete lack of engagement with the prompt. There is no information provided about the current trends in observability tools, directly contradicting the instructions. The agent fails at all aspects of the evaluation.","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tools_available', 'tool_calls']","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']" +,a81ca3a8-80aa-4c39-876e-8d40ea7a0aef,acf6a99a-39e0-4f87-a07e-3a61213ab032,agent,1.0,7.5,10.0,0.0,0.0,"The agent provided no response, meaning it did not address the task. It's difficult to assess understanding or progress based on an empty reply. The agent failed to engage with the prompt. Therefore, the score is a 1.0.","The response is relevant to the task of describing recent developments in machine learning, fitting the agent's role. The tone and content are appropriate. However, the depth of the response could be increased to demonstrate a stronger understanding of the current state of the field to get a better score.","The agent perfectly adheres to its role and task. The response is consistent with the role of machine learning expert. The agent appropriately answers the prompt, delivering a relevant overview. No tool calls were made, suggesting that the agent could answer the query without external assistance, which aligns with perfect role adherence.","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tools_available', 'tool_calls']","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']" +,230aad27-f3dd-4968-a45a-3c2f07ac28ed,0b47ea07-1eda-4b6a-a183-ea9ef047e431,agent,1.0,7.8,10.0,0.0,0.0,"The agent's response is missing. There is no information to assess. Therefore, it's impossible to determine if the agent understands its role, the task, or has made any progress. The response is completely unhelpful, failing to engage with the prompt.","The agent response's relevance is good given the weather query. It's appropriately aligned with the role, providing information about the weather. The tone is suitable, indicating task understanding. To improve, it could offer a bit more detail, but overall it is a helpful start.","The agent perfectly adheres to its role and task. The agent correctly executes its role and the tool calls are appropriate for retrieving information about the weather, as intended. The response is consistent with the role and task requirements, showing no contradictions or inappropriate behaviors. The agent demonstrates flawless adherence.","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tools_available', 'tool_calls']","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']" +,83c7dcce-3d89-4da1-8b3f-d419885d4cbc,3866ddb3-b17e-47a1-8b3f-bca1e32a2ae4,agent,1.0,7.8,1.0,0.0,0.0,"The agent's response is missing. It's impossible to evaluate its understanding, alignment, progress, efficiency, or proximity to completion without any content provided. Therefore, the task progression is at the lowest possible level.","The agent response's direct relevance is confirmed, matching the specified task. The alignment with the designated role is evident, and the tone and content are appropriate. There's a slight lack of demonstrating deeper understanding, which slightly reduces the score, but overall, it's well-executed and helpful.","The agent completely fails to adhere to its role. It provides no response or tool calls, indicating a complete inability to fulfill its assigned task of providing AI news. It acts inappropriately by failing to respond, showing no understanding or attempt to interact with any available tools.","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tools_available', 'tool_calls']","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']" +,2218f641-604c-491a-9710-b51a9941b982,3ce2e35e-4518-43b4-8a76-4413da6832ea,agent,1.0,7.8,9.0,0.0,0.0,"The agent's response is missing. There is no information provided regarding the agent's understanding, alignment to the task, progress, efficiency or approach. Thus, it hasn't made any progress, and is, therefore, off-topic.","The agent response's direct relevance to the task is good, and it generally aligns with the agent's role. The tone and content are appropriate. However, without seeing the actual response, a slightly higher score is withheld as it's impossible to confirm the response's comprehensive understanding and perfect alignment with the task's requirements.","The agent perfectly adheres to its role, providing a concise response about the benefits of using Noveum Trace. The absence of tool calls indicates no deviation from the provided context. The response is consistent, focused, and free of any role contradictions or inappropriate behaviors, demonstrating a strong understanding of its assigned task.","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tools_available', 'tool_calls']","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']" +,255fd49c-84b4-4b18-887e-6308a412d535,c0c39e8b-9a42-4c18-8737-9224cd8a45c0,agent,1.0,7.8,1.0,0.0,0.0,"The provided response from the agent is missing. Since the agent has not provided any answer, the agent has made no progress on the task. The response is neither aligned with the task nor does it demonstrate any understanding of the assigned instructions.","The agent response appears helpful and relevant to the assigned task of configuring Noveum. It's aligned with the agent's role and expertise. The tone and content are appropriate. However, without the actual response, a definitive assessment of its quality and level of detail is impossible; hence, a slightly lower score.","The agent provided no response, and therefore failed to adhere to the role or address the task. It did not provide any information or utilize any tools. The lack of response demonstrates complete failure in understanding and executing the assigned instructions.","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tools_available', 'tool_calls']","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']" +,dc511122-c0b6-415c-9a49-c7b45132dd87,80987c51-ea82-4a41-97c7-ed591082b9bf,agent,1.0,7.8,10.0,0.0,0.0,"The agent's response is missing. There is no information provided regarding Noveum's monitoring capabilities. The agent has not provided any answer and has not made any progress, therefore it is off-topic and unhelpful. The agent completely failed to fulfill the task.","The agent response's relevance is good given the prompt, and the response is appropriate for the agent's designated role. The content's tone is also aligned with the task, and the response demonstrates good understanding of the task requirements. Further details could enhance the response.",The agent perfectly adheres to its role and task. The agent's response is relevant to its role and the tool calls are appropriate for this role and its task. The agent doesn't contradict its assigned task. No deviations or inconsistencies are observed.,"Error: Missing required fields: ['tool_calls'], out of all required fields: ['tools_available', 'tool_calls']","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']" +,04bebf38-a343-4563-80db-0154bef8d927,2f89c1e0-9b62-4651-914c-b71cf561c33c,agent,1.0,7.8,1.0,0.0,0.0,"The agent's response is missing. There is no attempt to answer the question, and therefore, no progress has been made. The agent has not provided any information about Noveum or agent tracing. This indicates a complete failure to engage with the task.","The agent response's relevance to the assigned task is clear, and it aligns well with the agent's role. The tone and content are appropriate. There's a solid understanding of the task requirements. The response is helpful. It could gain a higher score if it gives an answer.","The agent completely fails to fulfill its role. The agent provides no response and makes no tool calls, indicating a complete disregard for the assigned task. There's no evidence of role adherence, task completion, or relevant information retrieval. The agent is inactive, rendering the evaluation impossible.","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tools_available', 'tool_calls']","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']" +,5e043630-6493-42b5-beb8-79faa19bfa37,e30e3e61-6edd-4b62-ab77-627faa2bf741,agent,1.0,7.8,10.0,0.0,0.0,"The agent's response is missing. There is no information provided regarding which APIs are available. The agent has not addressed the question at all, failing to make any progress toward the goal. The response is unhelpful and off-topic. No attempt was made.","The agent response's relevance to the assigned task is clear, and it aligns well with the agent's specified role. The content and tone are suitable for the role. Demonstrating a more extensive understanding of available APIs in Noveum would improve the score.","The agent perfectly adheres to its role and task. The agent correctly identified that there were no tools calls, as it was a prompt and not a request for a specific action. The response is consistent with the expectations of the assigned role.","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tools_available', 'tool_calls']","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']" +,7da9814d-a2e8-4c4e-b750-68b26bd5fd22,2f513795-764f-4eb6-8dd2-108d4cdf8432,agent,1.0,7.5,1.0,0.0,0.0,"The agent's response is missing. There is no information provided about how the agent attempted to answer the question. Therefore, no progress was made toward task completion, and the response is unhelpful. The agent did not respond with any information.","The response should directly address setting up observability with Noveum, which is highly relevant to the provided task. The response is likely aligned with the agent's role and expertise. Improvement could include a more helpful response, leading to a higher score.","The agent provided no response or tool calls, therefore failing to adhere to any aspect of its role and task. It did not attempt to answer the question or make any progress towards the objective. The lack of any action completely contradicts the expectation for a helpful response.","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tools_available', 'tool_calls']","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']" +,16143f74-2831-4753-b33d-ce4b645093c5,29c7a515-2383-4b7d-8fc7-beac3f403641,agent,1.0,7.8,10.0,0.0,0.0,"The agent's response is missing. There is no information to evaluate. It fails to answer the question, and provides no helpful information or any indication of understanding the task. Therefore, the agent has made no progress.","The agent response's direct relevance is strong, fitting the task and agent role well. The content is suitable. While it efficiently addresses the query, there's room to further showcase the agent's expertise and provide a more comprehensive overview of Noveum Trace's features. A more elaborate answer could have improved the score.","The agent perfectly adheres to its role and task. The agent has not provided a response nor made any tool calls, which is the expected behavior given the prompt which instructed it to evaluate an agent's response. No deviations or contradictions were observed, maintaining complete consistency.","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tools_available', 'tool_calls']","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']" +,fc64e6cc-6739-4256-ac4a-7b80c3028233,12fee687-2a2e-401f-83c2-9c8dc7ecd853,agent,1.0,7.8,1.0,0.0,0.0,"The agent's response, or lack thereof, does not provide any information regarding Noveum's pricing plans. It's a placeholder. There is no progress made, the response is off-topic, and fails to address the core task. The agent has not understood the task at hand.","The agent response's relevance to the assigned task is clear and aligns well with the agent's role. The tone and content are appropriate. However, without the actual response, it's impossible to gauge the depth of understanding or how helpful it is in completing the task, so the score is slightly adjusted.","The agent provided no response, and made no tool calls, meaning the agent completely failed to fulfill its role or task. It did not try to answer the question about pricing plans or attempt any data analysis or retrieval. This constitutes a complete failure to adhere to the role.","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tools_available', 'tool_calls']","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']" +,b7945c49-f584-4c70-972d-536a805d8a31,9c5aec32-b6eb-4c42-93d5-3970fb118c00,agent,1.0,7.5,1.0,0.0,0.0,"The agent has not provided a response, meaning no progress has been made. The task is to provide instructions on how to integrate Noveum Trace. The lack of content indicates a complete failure to address the prompt.","The agent response would likely be directly relevant to the task if there was an actual response to evaluate. The prompt itself is designed to elicit a response about integrating Noveum Trace, aligning with the agent's expected role. Without a response, evaluating tone, content, and task understanding is impossible, but the prompt itself is well-suited.","The agent provided no response, meaning the agent failed at every aspect of its role. It did not provide a response, tool calls, or do anything to accomplish its assigned task. The agent completely ignored all instructions and failed to operate as requested. Therefore, the adherence score is very low.","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tools_available', 'tool_calls']","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']" +,f5c40ecf-36c0-45ba-9cc9-dc0329b0324b,f132d053-f82c-4e66-adf4-bd7ce68b660d,agent,2.8,7.8,10.0,0.0,0.0,"The agent seems to understand the task to define Noveum. However, without the agent's response, it's impossible to evaluate the content's accuracy or the degree of progress. Therefore, only the basic understanding of the task can be inferred, resulting in a score reflecting minimal progress with potential for improvement.","The response is relevant to the task of defining Noveum. It's well-aligned with the agent's role of providing information. The tone is appropriate. Demonstrating a slightly more detailed understanding of Noveum's function or purpose would elevate the score further, though the foundational elements of task completion are present.","The agent perfectly adheres to its role and task, maintaining consistency throughout. The absence of any tool calls implies that the agent may have responded directly with the answer to the question. This aligns well with the prompt, suggesting an exact response is expected given the lack of tool usage. No deviations from the role or inappropriate behaviors are observed.","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tools_available', 'tool_calls']","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']" diff --git a/noveum_customer_support_bt/demo_results/agent.rag_evaluation_metrics_dataset/agent_evaluation_results.csv b/noveum_customer_support_bt/demo_results/agent.rag_evaluation_metrics_dataset/agent_evaluation_results.csv new file mode 100644 index 0000000..bf749b0 --- /dev/null +++ b/noveum_customer_support_bt/demo_results/agent.rag_evaluation_metrics_dataset/agent_evaluation_results.csv @@ -0,0 +1,13 @@ +user_id,task_id,turn_id,agent_name,task_progression,context_relevancy,role_adherence,tool_relevancy,parameter_correctness,task_progression_reasoning,context_relevancy_reasoning,role_adherence_reasoning,tool_relevancy_reasoning,parameter_correctness_reasoning +,f1f37bd7-0851-4659-b493-b80d3800d920,ac166548-ffa5-47b9-8cf7-4787f270cb5a,agent,3.8,7.8,9.0,0.0,0.0,"The agent understands its role and the task. The response is aligned with the query and attempts to answer it using the provided context. While the answer mentions 'recent enhancements' it's focused on a specific platform, which limits its scope, hence some gaps. Progress is good, but not fully comprehensive.","The agent acknowledges the limitations of the provided context, which is a good start. It correctly identifies the lack of general cloud computing features while focusing on the specific information available. The response is helpful and progresses the task, even if it is a negative answer, indicating understanding.","The agent perfectly adheres to its role and task. The response is relevant, concise and accurately answers the question based on the provided RAG context. The agent correctly interprets the information and offers a focused response, without any role contradictions. The agent demonstrates excellent comprehension of its role.","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tools_available', 'tool_calls']","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']" +,52aacb67-c361-4445-9b72-c157f79f47d6,e193cd4f-cca3-4b94-95d3-a1f2446b9fc6,agent,2.8,7.8,9.0,0.0,0.0,"The agent understands the task, but the response is somewhat limited. While it acknowledges the context doesn't directly answer the question, it identifies some relevant aspects. More direct information about trends would improve the score. The agent shows minimal concrete advancement.","The agent accurately acknowledges the limitations of the provided context while still attempting to address the query. The response is relevant, aligning with the agent's task by identifying the absence of direct information and pivoting to related aspects. The tone and approach are suitable for the role.","The agent accurately fulfills its role as an evaluator, focusing on the provided agent's response and task. The response is direct and doesn't introduce irrelevant information or act outside its assigned scope. The analysis is well-structured and relevant to the provided criteria, reflecting a strong adherence to the role.","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tools_available', 'tool_calls']","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']" +,2218f641-604c-491a-9710-b51a9941b982,432e74f3-3291-411b-8d7e-85493868f8cf,agent,4.5,7.9,9.0,0.0,0.0,"The agent correctly identifies and begins to list benefits, showing a solid understanding of the task. The response is directly aligned with the prompt's requirements. The approach is efficient, quickly addressing the query's core. While not fully complete (truncated sentence), it shows substantial advancement.","The response directly answers the question about the benefits of using Noveum Trace. It's aligned with the agent's task of providing information. The answer is appropriate in tone and content. The inclusion of a numbered list indicates a solid understanding of the task. However, the response is cut off, which prevents a higher score.","The agent perfectly adheres to its role by providing a relevant answer. The response is focused on the benefits of Noveum Trace, directly answering the query. No tool calls were made, which is acceptable in this case. There are no contradictions or inappropriate behaviors, and the content is appropriate for the assigned task.","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tools_available', 'tool_calls']","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']" +,255fd49c-84b4-4b18-887e-6308a412d535,a0475dd5-ef9f-4eb2-bca4-9be72bf6bfd6,agent,4.5,7.8,9.0,0.0,0.0,"The agent understands the task and provides a relevant answer, pointing to the SDK Integration Guide. This demonstrates a strong understanding of how to configure Noveum. The response efficiently directs the user to a valuable resource, making substantial progress toward task completion.","The response is relevant and helpful, providing a clear starting point for the user. It aligns well with the agent's task by suggesting a specific resource (SDK Integration Guide). While informative, the response could be slightly improved by giving more specific details or examples to enhance user assistance.","The agent perfectly adheres to its role, providing a concise answer related to the prompt's request. The response focuses on the Noveum configuration and points to relevant documentation, which is aligned with the task. There are no deviations or contradictions observed, showcasing consistency throughout.","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tools_available', 'tool_calls']","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']" +,dc511122-c0b6-415c-9a49-c7b45132dd87,ed8ad539-f008-4f1a-beb1-12ad5793950e,agent,4.5,8.1,9.0,0.0,0.0,"The agent correctly answers the query and demonstrates a strong understanding of its task. The response is directly relevant and provides specific details about Noveum's monitoring capabilities. The information is concise, and the inclusion of frameworks shows comprehensive understanding. Excellent progress with very minor room for improvement.","The response directly answers the query about Noveum's monitoring capabilities. It correctly identifies key aspects like monitoring, tracing, and optimization. The agent's expertise is evident. The response is helpful and appropriately worded, making it well-suited to the assigned task and agent role. A slightly more detailed explanation could potentially improve the score.","The agent perfectly adheres to its role by providing a concise and relevant answer to the prompt. It directly addresses the query about Noveum's monitoring capabilities. The response is appropriate for the context, and there are no deviations or inconsistencies. The absence of tool calls reinforces the agent's role here.","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tools_available', 'tool_calls']","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']" +,04bebf38-a343-4563-80db-0154bef8d927,c7358391-9ac1-4312-a4b3-79edfa8bc0d3,agent,4.5,7.8,9.0,0.0,0.0,"The agent clearly understands the query and provides a relevant response directly answering the question. The provided information regarding Noveum's agent tracing capabilities suggests strong progress. The response is concise and appears well-informed, leading to a high progression score.","The response directly answers the question about agent tracing by providing information related to Noveum. The content aligns well with the agent's task and role as an information provider. The answer is appropriate in tone and content, offering a concise and helpful overview, thus achieving a solid score.","The agent perfectly adheres to its role by providing a concise and relevant answer to the query. The response directly addresses the question about how Noveum.ai handles agent tracing. There are no deviations, contradictions or inappropriate behaviors. The content is focused and aligns perfectly with the expected output.","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tools_available', 'tool_calls']","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']" +,5e043630-6493-42b5-beb8-79faa19bfa37,6b21de5d-456f-4501-953f-0ce5bc4fa5a5,agent,4.5,7.8,9.0,0.0,0.0,"The agent correctly identifies the limitations of the provided context and suggests appropriate next steps. The response is directly aligned with the task, displaying a strong understanding of how to address the user's query. This demonstrates significant progress with no apparent inefficiency.",The agent response is relevant and acknowledges the lack of information in the context. It offers a helpful suggestion to find the desired information. The response is well-aligned with the agent's role and tone is appropriate. A slightly higher score is awarded as the response provides a clear next step.,"The agent perfectly adheres to its role by providing a relevant answer based on the context. The agent correctly states it lacks the necessary information to provide the requested data, offering a helpful suggestion to get the information. No role contradictions or inappropriate behaviors are present. The response is consistent with the role.","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tools_available', 'tool_calls']","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']" +,7da9814d-a2e8-4c4e-b750-68b26bd5fd22,2ad4e155-7b76-4040-923d-d6b65a97a45b,agent,4.5,7.8,9.0,0.0,0.0,"The agent clearly understands the task and provides a relevant response. It directly addresses the user's query by offering actionable steps to set up observability with Noveum. The provided steps indicate substantial progress, although the response is cut off, hinting at incomplete task completion. The approach is efficient.","The agent provides a relevant answer that is well-aligned with the task of explaining how to set up observability with Noveum. The response clearly outlines the initial steps, demonstrating an understanding of the task requirements and providing helpful information. The tone is appropriate for the role of providing instructions.","The agent perfectly adheres to its role by providing a clear and concise answer to the user's query. The response is directly relevant and helpful, offering a step-by-step guide to setting up observability with Noveum. The answer is presented in a straightforward manner, consistent with the task's requirements.","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tools_available', 'tool_calls']","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']" +,16143f74-2831-4753-b33d-ce4b645093c5,6b822089-be40-457d-9fa6-0979bafdfb85,agent,4.5,8.1,9.0,0.0,0.0,"The agent correctly identifies its role and provides a relevant response, directly addressing the query about Noveum Trace's features. The response includes concrete information about the Python and TypeScript SDKs and a key feature. It is well-structured and advances substantially towards task completion.","The response is highly relevant, providing a clear and concise answer to the query about Noveum Trace features. It aligns perfectly with the agent's task and role as an information provider. The content is well-structured and demonstrates a good understanding of the task requirements, making it helpful for task completion.","The agent perfectly adheres to its role by providing a concise and relevant response to the query about Noveum Trace features. It stays focused on the task and accurately presents the information, using technical terms appropriately. The absence of tool calls further emphasizes the straightforward and task-oriented response, demonstrating excellent role adherence.","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tools_available', 'tool_calls']","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']" +,fc64e6cc-6739-4256-ac4a-7b80c3028233,08db68f6-3623-44b5-840b-41400fac7333,agent,4.1,7.8,9.0,0.0,0.0,"The agent correctly identifies the task and responds with relevant information extracted from the given context. It acknowledges the lack of explicit pricing plan details while providing related context. This shows a good understanding and efficient approach, making it almost complete.","The agent correctly identifies the lack of pricing plan information while providing a relevant piece of context, the Observability suite's free access. The response is appropriate for the task and role, although it could be improved by acknowledging the limitation more explicitly to get a higher score.",The agent perfectly adheres to its role and task. The response is consistent with the lack of information regarding pricing plans. The agent acknowledges the limitations of the provided context without exhibiting any role contradictions or inappropriate behaviors. There are no deviations from the assigned role.,"Error: Missing required fields: ['tool_calls'], out of all required fields: ['tools_available', 'tool_calls']","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']" +,b7945c49-f584-4c70-972d-536a805d8a31,6a57c92e-e473-425a-978b-19dbe976a46e,agent,4.5,8.0,9.0,0.0,0.0,"The agent correctly identifies the task and provides a relevant response, indicating a good understanding of its role. The agent offers specific information about using the Python or TypeScript SDKs, showing concrete progress toward the goal of integration. The answer is concise and points to a quick start, suggesting an efficient approach. ","The response is relevant and helpful, providing a clear starting point for integrating Noveum Trace. It correctly identifies the available SDKs. The response aligns well with the agent's role. It adequately demonstrates an understanding of the task requirements. A slightly more detailed response could have improved the score.","The agent accurately addresses the query about integrating Noveum Trace. The provided response is relevant and directly answers the question, demonstrating a clear understanding of the task. The agent remains focused and doesn't exhibit any role contradictions or inappropriate behavior. No tool calls were made, suggesting that the initial information retrieved by the RAG system was sufficient.","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tools_available', 'tool_calls']","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']" +,f5c40ecf-36c0-45ba-9cc9-dc0329b0324b,efb19490-bc9e-48dc-a365-2f839100e41a,agent,4.5,8.1,10.0,0.0,0.0,"The agent correctly identifies the task and provides a relevant answer about Noveum. The response directly addresses the query, indicating strong understanding and concrete progress. The answer is concise and informative. The approach is efficient and directly addresses the user's need. The agent has provided a complete and helpful answer.","The response is highly relevant, providing a concise and accurate definition of Noveum.ai and its function, directly addressing the query. The answer aligns well with the agent's role as an information provider. The content is suitable and helpful, showcasing a good understanding of the task requirements.","The agent perfectly adheres to its role and task. The response provides a clear and concise answer regarding Noveum.ai's function, aligning completely with the role's objective to answer a question about a technology. There are no deviations, contradictions, or inappropriate behaviors observed.","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tools_available', 'tool_calls']","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']" diff --git a/noveum_customer_support_bt/demo_results/agent.web_search_evaluation_metrics_dataset/agent_evaluation_results.csv b/noveum_customer_support_bt/demo_results/agent.web_search_evaluation_metrics_dataset/agent_evaluation_results.csv new file mode 100644 index 0000000..eb15ebe --- /dev/null +++ b/noveum_customer_support_bt/demo_results/agent.web_search_evaluation_metrics_dataset/agent_evaluation_results.csv @@ -0,0 +1,9 @@ +user_id,task_id,turn_id,agent_name,task_progression,context_relevancy,role_adherence,tool_relevancy,parameter_correctness,task_progression_reasoning,context_relevancy_reasoning,role_adherence_reasoning,tool_relevancy_reasoning,parameter_correctness_reasoning +,eda4fe22-9a2b-4b73-856b-f4f3309bf719,3b7b6838-22d4-4edf-9b00-7cb011f18a52,agent,4.2,7.8,9.0,0.0,0.0,"The agent correctly identifies that the web search results did not provide the needed information. It understands its task and provides a logical next step (checking reputable sources). The response is concise and helpful, showing good progress. Some improvement in its search query design might be possible.","The agent provides a relevant response acknowledging the limitations of the web search results. It correctly states the lack of specific information and offers a suggestion for further research. The response is appropriate in tone and content, demonstrating a good understanding of the task requirements and offering a helpful next step.","The agent perfectly adheres to its role and task. The response is relevant, and the agent accurately reports the inability to find the requested information, providing a suggestion. No inappropriate behavior or role contradictions are present, and the response stays focused on the assigned task throughout the interaction.","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tools_available', 'tool_calls']","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']" +,0ffffba1-8a37-443c-8866-d53ffbfa7718,a5d480f0-ea5b-4190-a3c3-f94e26b48573,agent,3.8,7.8,9.0,0.0,0.0,"The agent understands the task and demonstrates a basic approach by using web search. The response indicates an attempt to find information, but it also clearly shows a limitation of the search. While it did not fully succeed, it tried the expected method. There is a clear understanding of the task and a logical, albeit unsuccessful, approach. ","The agent acknowledges its inability to find specific information, which is appropriate. The response directly addresses the task and is consistent with the agent's role. It is helpful, as it provides a clear answer based on the search results. A slightly higher score could be earned with a more proactive attempt to search or suggest alternative approaches.","The agent perfectly adheres to its role by attempting to answer the user's query by searching the web. The response acknowledges the lack of results, demonstrating appropriate task execution. The agent correctly uses the tools and stays focused on gathering and presenting the requested information. There are no deviations or inappropriate behaviors.","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tools_available', 'tool_calls']","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']" +,43cdf081-4f01-49cd-b566-dbd1619e6cd2,2fd285de-68a1-4b7a-a4ed-05f539afa737,agent,4.2,7.8,9.0,0.0,0.0,"The agent correctly interprets the task and attempts to fulfill it by utilizing web search. The agent acknowledges the limitations of the search results, demonstrating understanding. Though it could provide a more informative answer based on the search, it still delivers a useful response, advancing the goal.","The response directly addresses the query about cryptocurrency market status. It acknowledges the limitations of the search results, which aligns with the agent's role. The answer is appropriate and helpful by providing a clear assessment. While it could elaborate slightly, the response is good.","The agent perfectly adheres to its role by attempting to answer the user's question about cryptocurrency markets. The response acknowledges the inability to provide a direct answer due to search result limitations. There are no role contradictions, and the focus remains on delivering information relevant to the query.","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tools_available', 'tool_calls']","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']" +,9a1983f4-09da-4b53-80e6-38de6878e0e7,48deb050-478d-486c-be3a-b8c0069d7abe,agent,2.5,7.8,9.0,0.0,0.0,"The agent understands its task to find information about Python updates. However, the response is not useful. It acknowledges a web search but then provides unrelated information. While it identifies the limitation of the search, it doesn't try to find alternative solutions or provide any actual updates.","The response accurately reflects the search results, acknowledging their lack of relevance to the query about Python updates. This aligns well with the agent's task and role as an evaluator. The response is helpful in assessing the information provided by the search. However, it could be slightly improved by suggesting future steps.","The agent perfectly adheres to its role by providing a relevant answer based on the query. It correctly identifies the lack of information on recent updates in Python programming from the search results, and doesn't stray from its task. No role contradictions or inappropriate behaviors are observed, thus maintaining a high score.","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tools_available', 'tool_calls']","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']" +,5d2517e1-220a-429d-9d59-f701bda25eed,4ecede78-e7fc-48ed-9fba-1241d6890a1d,agent,2.8,7.8,9.0,0.0,0.0,"The agent understands the task, which is to provide a summary of tech news. However, the agent's response indicates that the search results were not helpful. While this response is relevant, it did not provide the information requested. Therefore, the response made minimal progress towards completing the task.","The agent correctly identifies the lack of relevant information in the search results, directly addressing the query's core. The response aligns well with the agent's task to summarize search results. The evaluation is appropriate. Improvement could involve briefly mentioning what *was* found.",The agent perfectly adheres to its role by providing a relevant answer regarding tech news based on web search results. The agent's response is consistent with the task and demonstrates a clear understanding of the information request. There are no deviations or contradictions observed in the agent's actions.,"Error: Missing required fields: ['tool_calls'], out of all required fields: ['tools_available', 'tool_calls']","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']" +,a81ca3a8-80aa-4c39-876e-8d40ea7a0aef,91fdf43c-7170-498e-9035-1e2b11b32b48,agent,3.8,7.8,9.0,0.0,0.0,"The agent acknowledges the limitations of the provided search results and proactively offers alternative actions. The response is aligned with the task, showing a moderate understanding of the objective. However, it lacks concrete details on machine learning developments, limiting the overall progress.","The agent acknowledges the lack of information in the search results and states that they cannot answer the query. This is a relevant response. Furthermore, it hints at potentially providing an answer based on other sources, which progresses towards task completion and shows an understanding of the task requirements.",The agent perfectly adheres to its role by acknowledging the lack of direct information and explaining its limitations. The response stays focused on the task of answering a question. No role contradictions or inappropriate behaviors are present. The agent provides a clear and concise response based on the available information.,"Error: Missing required fields: ['tool_calls'], out of all required fields: ['tools_available', 'tool_calls']","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']" +,230aad27-f3dd-4968-a45a-3c2f07ac28ed,096e6592-23fa-432a-931d-8f39656c7fcc,agent,4.2,7.8,9.0,0.0,0.0,"The agent correctly interprets the prompt and its task, and provides a relevant, albeit incomplete, response. It acknowledges the web search results, demonstrating understanding. It is close to achieving the goal, but hasn't fully delivered on it yet. There are minor improvements to make, preventing a perfect score.",The response is relevant as it addresses the weather query. It explains that the search results lack specific weather information. The agent's role is well-defined as it summarizes web search findings. The tone is appropriate and indicates task comprehension. A score of 7.8 recognizes the utility while suggesting possible elaboration.,"The agent perfectly adheres to its role by providing a response that's relevant to the given query. The absence of tool calls indicates that the agent assessed the question could be answered without them, which is appropriate. The response is consistent with the task and demonstrates a good understanding of its role, providing a concise and accurate answer. ","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tools_available', 'tool_calls']","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']" +,83c7dcce-3d89-4da1-8b3f-d419885d4cbc,b56248e6-e526-4000-8c10-f5d13cf2e12d,agent,4.5,8.1,8.8,0.0,0.0,"The agent correctly identifies its role and the task. It provides a response directly addressing the query about the latest AI news. The response includes a summary of current events. The agent makes substantial progress and the approach is efficient, demonstrating a strong understanding of the prompt.","The agent's response directly addresses the query about the latest AI news. It provides a concise summary with relevant details, indicating a good understanding of the task. The format is clear and aligns well with the agent's role. Some further details or source citation could slightly improve the score.","The agent adheres perfectly to the role, demonstrating a comprehensive understanding of the task. The response is relevant, concise and informative. The agent correctly identifies key information about AI news. No role contradictions or inappropriate behaviors are observed. The provided response is well structured and relevant.","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tools_available', 'tool_calls']","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']" diff --git a/noveum_customer_support_bt/demo_utils.py b/noveum_customer_support_bt/demo_utils.py new file mode 100644 index 0000000..20904ba --- /dev/null +++ b/noveum_customer_support_bt/demo_utils.py @@ -0,0 +1,1025 @@ +import json +import os +import re +import logging +from typing import Any, Dict, List, Optional +import pandas as pd +from pathlib import Path + +# NovaEval imports +from novaeval.agents.agent_data import AgentData, ToolSchema, ToolCall, ToolResult +from novaeval.datasets.agent_dataset import AgentDataset +from novaeval.evaluators.agent_evaluator import AgentEvaluator +from novaeval.models.gemini import GeminiModel +from novaeval.scorers.agent_scorers import ( + context_relevancy_scorer, + role_adherence_scorer, + task_progression_scorer, + tool_relevancy_scorer, + parameter_correctness_scorer +) +from dotenv import load_dotenv + +load_dotenv() + +print("āœ… All imports successful!") + +def list_dataset_files(directory: str = "split_datasets") -> List[str]: + """ + List available JSON dataset files in the specified directory. + + Args: + directory: Directory to search for dataset files + + Returns: + List of JSON file names + """ + try: + if not os.path.exists(directory): + print(f"āŒ Directory {directory} does not exist") + return [] + + files = os.listdir(directory) + json_files = [f for f in files if f.endswith('.json')] + + print(f"šŸ“Š Found {len(json_files)} JSON files in {directory}/:") + for file in json_files: + print(f" - {file}") + + return json_files + except Exception as e: + print(f"āŒ Error listing files: {e}") + return [] + +print("āœ… list_dataset_files function defined!") + +def load_and_analyze_dataset(file_name: str) -> tuple[List[Dict[str, Any]], Dict[str, int]]: + """ + Load JSON dataset file and analyze span types. + + Args: + file_name: Path to the JSON dataset file + + Returns: + Tuple of (spans_data, span_types_dict) + """ + try: + with open(file_name, 'r', encoding='utf-8') as f: + spans_data = json.load(f) + + print(f"šŸ“Š Loaded {len(spans_data)} spans from {file_name}") + print("\nšŸ” Available span types:") + + # Analyze span types + span_types = {} + for span in spans_data: + span_name = span.get('name', 'unknown') + if span_name not in span_types: + span_types[span_name] = 0 + span_types[span_name] += 1 + + for span_type, count in span_types.items(): + print(f" - {span_type}: {count}") + + return spans_data, span_types + + except FileNotFoundError: + print(f"āŒ File {file_name} not found") + return [], {} + except json.JSONDecodeError as e: + print(f"āŒ Error parsing JSON: {e}") + return [], {} + except Exception as e: + print(f"āŒ Error loading dataset: {e}") + return [], {} + +print("āœ… load_and_analyze_dataset function defined!") + +def parse_tools_from_prompt(prompt: str) -> List[ToolSchema]: + """ + Parse tool definitions from LLM prompts using regex. + + Expected format: tool_name(param: type = default) -> return_type - description + """ + # Pattern to match tool signatures + pattern = r'(\w+)\(([^)]*)\)\s*->\s*(\w+)\s*-\s*(.+?)(?=\n\w+\(|$)' + matches = re.findall(pattern, prompt, re.DOTALL) + + tools = [] + for match in matches: + tool_name, params_str, return_type, description = match + + # Parse parameters + args_schema = parse_params(params_str) + + tool = ToolSchema( + name=tool_name, + description=description.strip(), + args_schema=args_schema, + return_schema={"type": return_type} + ) + tools.append(tool) + + return tools + +print("āœ… parse_tools_from_prompt function defined!") + +def parse_params(params_str: str) -> Dict[str, Any]: + """ + Parse parameter string into schema dictionary. + + Format: param_name: type = default_value + """ + if not params_str.strip(): + return {} + + # Split parameters by comma + params = [p.strip() for p in params_str.split(',') if p.strip()] + schema = {} + + for param in params: + if ':' in param: + parts = param.split(':', 1) + param_name = parts[0].strip() + type_and_default = parts[1].strip() + + # Extract type and default value + if '=' in type_and_default: + type_part, default_part = type_and_default.split('=', 1) + param_type = type_part.strip() + default_val = default_part.strip().strip('"\'') + schema[param_name] = {'type': param_type, 'default': default_val} + else: + param_type = type_and_default.strip() + schema[param_name] = {'type': param_type} + + return schema + +print("āœ… parse_params function defined!") + +def identify_span_type(span: Dict[str, Any]) -> str: + """ + Identify span type based on attributes. + """ + attributes = span.get('attributes', {}) + span_name = span.get('name', '') + + # Check for agent spans - expanded to include RAG evaluation spans + agent_span_names = [ + 'reddit_agent_run_1', 'reddit_agent_run_2', + 'agent.query_generation', 'agent.comment_generation', + 'agent.rag_evaluation_metrics', 'agent.web_search_evaluation_metrics', + 'agent.query_routing' + ] + + # Check for agent attributes or known agent span names + if (any('chain.name' == key for key in attributes.keys()) or + span_name in agent_span_names or + span_name.startswith('agent.')): + return 'agent' + + # Check for LLM attributes + if any('llm.model' == key for key in attributes.keys()): + return 'llm' + + # Check for tool attributes + tool_span_names = ['post_validation', 'email_generation_and_sending'] + if (any('tool.name' == key for key in attributes.keys()) or + span_name in tool_span_names): + return 'tool' + + print('returning unknown type for span') + print(span) + return 'unknown' + +print("āœ… identify_span_type function defined!") + +def map_span_to_agent_data(span: Dict[str, Any], count_unknowns: Optional[Dict[str, int]] = None) -> AgentData: + """ + Map a single span from file_name to AgentData format. + """ + + attributes = span.get('attributes', {}) + events = span.get('events', []) + span_type = identify_span_type(span) + + # Base mappings + data = { + 'user_id': span.get('metadata', {}).get('user_id', None), + 'task_id': span.get('trace_id'), + 'turn_id': span.get('span_id'), + 'ground_truth': None, + 'expected_tool_call': None, + 'agent_name': span_type, + 'agent_role': span_type, + 'system_prompt': "You are a helpful customer support agent", + 'metadata': None, + 'exit_status': span.get('status'), + 'tools_available': [], + 'tool_calls': [], + 'parameters_passed': {}, + 'tool_call_results': [], + 'retrieval_query': None, + 'retrieved_context': None, + 'agent_exit': False, + 'trace': None + } + + # Span-specific mappings + if span_type == 'agent': + # Agent task - handle different span types + span_name = span.get('name', '') + + if span_name.startswith('agent.rag_evaluation_metrics'): + # RAG evaluation spans + data['agent_task'] = attributes.get('input_query', 'RAG evaluation task') + data['agent_response'] = attributes.get('output_response', '') + data['retrieval_query'] = [attributes.get('input_query', '')] + data['retrieved_context'] = [[attributes.get('retrieval.context_retrieved', '')]] + elif span_name.startswith('agent.web_search_evaluation_metrics'): + # Web search evaluation spans + data['agent_task'] = attributes.get('input_query', 'Web search evaluation task') + data['agent_response'] = attributes.get('output_response', '') + data['retrieval_query'] = [attributes.get('input_query', '')] + data['retrieved_context'] = [[attributes.get('web_search.search_results', '')]] + elif span_name.startswith('agent.query_routing'): + # Query routing spans + data['agent_task'] = attributes.get('input_query', 'Query routing task') + data['agent_response'] = attributes.get('routing_decision', '') + else: + # Standard agent spans + chain_inputs = attributes.get('chain.inputs', {}) + if isinstance(chain_inputs, dict) and 'input' in chain_inputs: + data['agent_task'] = chain_inputs['input'] + elif attributes.get("agent_task"): + data['agent_task'] = attributes.get("agent_task") + else: + print('agent_task not found') + + # Agent response + finish_values = attributes.get('agent.output.finish.return_values', {}) + if isinstance(finish_values, dict) and 'output' in finish_values: + data['agent_response'] = finish_values['output'] + elif attributes.get("agent_response"): + data['agent_response'] = attributes.get("agent_response") + else: + print("agent_response is not available " + span['span_id']) + # Tool calls from agent actions - handle different span types + if span_name.startswith('agent.rag_evaluation_metrics'): + # RAG evaluation spans don't have traditional tool calls + # They have retrieval and response evaluation capabilities + data['tools_available'] = [ + ToolSchema( + name="rag_evaluation", + description="Evaluates RAG system performance", + args_schema={}, + return_schema={"type": "evaluation_result"} + ) + ] + elif span_name.startswith('agent.web_search_evaluation_metrics'): + # Web search evaluation spans + data['tools_available'] = [ + ToolSchema( + name="web_search_evaluation", + description="Evaluates web search performance", + args_schema={}, + return_schema={"type": "evaluation_result"} + ) + ] + elif span_name.startswith('agent.query_routing'): + # Query routing spans + data['tools_available'] = [ + ToolSchema( + name="query_routing", + description="Routes queries to appropriate handlers", + args_schema={}, + return_schema={"type": "routing_decision"} + ) + ] + else: + # Standard agent tool calls + tool_name = attributes.get('agent.output.action.tool') + tool_input = attributes.get('agent.output.action.tool_input') + + if tool_name: + tool_call = ToolCall( + tool_name=tool_name, + parameters={'input': tool_input} if tool_input else {}, + call_id=span['span_id'] + ) + data['tool_calls'] = [tool_call] + data['parameters_passed'] = {'input': tool_input} if tool_input else {} + + # Handle retrieval query for langchain_retriever + if tool_name == 'langchain_retriever' and tool_input: + data['retrieval_query'] = [tool_input] + + # Agent exit status + data['agent_exit'] = any(event.get('name') == 'agent_finish' for event in events) + + # Trace (dump events as JSON) + if events: + data['trace'] = json.dumps(events) + + elif span_type == 'llm': + # Agent response from LLM output + llm_input = attributes.get('llm.input.prompts', ['input is not available']) + data['agent_task'] = llm_input[0] + + llm_responses = attributes.get('llm.output.response', []) + if llm_responses: + data['agent_response'] = llm_responses[0] + else: + print("llm_response is not available") + # Parse tools from prompt + prompts = attributes.get('llm.input.prompts', []) + if prompts: + try: + tools = parse_tools_from_prompt(prompts[0]) + data['tools_available'] = tools + except Exception: + # Fallback to empty list if parsing fails + data['tools_available'] = [] + + data['parameters_passed'] = {} + + elif span_type == 'tool': + # Agent response from tool output + tool_output = attributes.get('tool.output.output') + data['agent_task'] = f"This is a simple tool call, and the tool will execute as programmed. Its name is - {attributes.get('tool.name')}" + if tool_output: + data['agent_response'] = tool_output + elif attributes.get("tool_response"): + data['agent_response'] = attributes.get("tool_response") + else: + print("tool_output is not available " + span['span_id']) + # Tool call results + tool_name = attributes.get('tool.name') + if tool_name and tool_output is not None: + tool_result = ToolResult( + call_id=span['span_id'], + result=tool_output, + success=span.get('status') == 'ok', + error_message=None if span.get('status') == 'ok' else 'Tool execution failed' + ) + data['tool_call_results'] = [tool_result] + + # Handle retrieved context for langchain_retriever + if tool_name == 'langchain_retriever': + data['retrieved_context'] = [[tool_output]] + + # Parameters from tool input + tool_input_keys = [key for key in attributes.keys() if key.startswith('tool.input.')] + tool_params = {} + for key in tool_input_keys: + param_name = key.replace('tool.input.', '') + tool_params[param_name] = attributes[key] + data['parameters_passed'] = tool_params + else: + if count_unknowns is not None and 'count' in count_unknowns: + count_unknowns['count'] += 1 + print('Spans with unknown type: ' + str(count_unknowns['count'])) + else: + print('Spans with unknown type detected') + return AgentData(**data) + + +print("āœ… map_span_to_agent_data function defined!") + +def convert_spans_to_agent_dataset(spans_data: List[Dict[str, Any]]) -> tuple[List[AgentData], List[str], AgentDataset]: + """ + Convert list of spans to AgentData objects and create AgentDataset. + + Args: + spans_data: List of span dictionaries + + Returns: + Tuple of (agent_data_list, errors, dataset) + """ + print("šŸ”„ Converting spans to AgentData objects...") + + agent_data_list = [] + errors = [] + count_unknowns = {'count': 0} + + for i, span in enumerate(spans_data): + try: + agent_data = map_span_to_agent_data(span, count_unknowns) + agent_data_list.append(agent_data) + except Exception as e: + errors.append(f"Span {i}: {str(e)}") + if len(errors) <= 5: # Show first 5 errors only + print(f"āš ļø Error processing span {i}: {e}") + + print(f"\nāœ… Successfully converted {len(agent_data_list)} spans to AgentData") + if errors: + print(f"āŒ {len(errors)} spans had errors") + + # Create AgentDataset + dataset = AgentDataset() + dataset.data = agent_data_list + + print(f"šŸ“Š AgentDataset created with {len(dataset.data)} records") + + return agent_data_list, errors, dataset + +print("āœ… convert_spans_to_agent_dataset function defined!") + +def analyze_dataset_statistics(dataset: AgentDataset) -> Dict[str, Any]: + """ + Analyze dataset statistics including agent types, tool usage, and response counts. + + Args: + dataset: AgentDataset to analyze + + Returns: + Dictionary containing statistics + """ + print("šŸ“ˆ Dataset Statistics:") + + agent_types = {} + tool_usage = {} + with_responses = 0 + with_tool_calls = 0 + with_retrieval = 0 + + for data in dataset.data: + # Agent types + if data.agent_name: + agent_types[data.agent_name] = agent_types.get(data.agent_name, 0) + 1 + + # Responses + if data.agent_response: + with_responses += 1 + + # Tool calls + if data.tool_calls: + with_tool_calls += 1 + for tool_call in data.tool_calls: + if hasattr(tool_call, 'tool_name'): + tool_usage[tool_call.tool_name] = tool_usage.get(tool_call.tool_name, 0) + 1 + + # Retrieval + if data.retrieval_query: + with_retrieval += 1 + + stats = { + 'agent_types': dict(agent_types), + 'records_with_responses': with_responses, + 'records_with_tool_calls': with_tool_calls, + 'records_with_retrieval': with_retrieval, + 'tool_usage': dict(tool_usage), + 'total_records': len(dataset.data) + } + + print(f"\nAgent Types: {dict(agent_types)}") + print(f"Records with responses: {with_responses}") + print(f"Records with tool calls: {with_tool_calls}") + print(f"Records with retrieval: {with_retrieval}") + print(f"Tool usage: {dict(tool_usage)}") + + return stats + +print("āœ… analyze_dataset_statistics function defined!") + +def setup_gemini_model(model_name: str = "gemini-2.0-flash-lite", temperature: float = 0.1, max_tokens: int = 1024) -> Optional[GeminiModel]: + """ + Setup and initialize Gemini model with API key validation. + + Args: + model_name: Gemini model name to use + temperature: Temperature for generation + max_tokens: Maximum tokens to generate + + Returns: + GeminiModel instance or None if setup failed + """ + # Check for API key + if 'GEMINI_API_KEY' not in os.environ: + print("āš ļø GEMINI_API_KEY environment variable not set!") + print("Please set it before running evaluation:") + print("export GEMINI_API_KEY='your-api-key-here'") + return None + else: + print("āœ… GEMINI_API_KEY found in environment") + + # Initialize Gemini model + try: + gemini_model = GeminiModel( + model_name=model_name, + temperature=temperature, + max_tokens=max_tokens + ) + print("āœ… Gemini model initialized") + return gemini_model + except Exception as e: + print(f"āŒ Error initializing Gemini model: {e}") + return None + +print("āœ… setup_gemini_model function defined!") + +def setup_agent_evaluator(dataset: AgentDataset, gemini_model: GeminiModel, output_dir: str = "./demo_results", + include_reasoning: bool = True, stream: bool = False) -> Optional[AgentEvaluator]: + """ + Setup AgentEvaluator with scoring functions. + + Args: + dataset: AgentDataset to evaluate + gemini_model: Initialized GeminiModel + output_dir: Directory for output files + include_reasoning: Whether to include reasoning in results + stream: Whether to stream results + + Returns: + AgentEvaluator instance or None if setup failed + """ + # Initialize scoring functions + scoring_functions = [ + task_progression_scorer, + context_relevancy_scorer, + role_adherence_scorer, + tool_relevancy_scorer, + parameter_correctness_scorer + ] + + print(f"āœ… Initialized {len(scoring_functions)} scoring functions:") + for func in scoring_functions: + print(f" - {func.__name__}") + + # Create AgentEvaluator + try: + evaluator = AgentEvaluator( + agent_dataset=dataset, + models=[gemini_model], + scoring_functions=scoring_functions, + output_dir=output_dir, + stream=stream, + include_reasoning=include_reasoning + ) + print("\nāœ… AgentEvaluator created with Gemini model and scoring functions") + return evaluator + except Exception as e: + print(f"āŒ Error creating AgentEvaluator: {e}") + return None + +print("āœ… setup_agent_evaluator function defined!") + +def run_evaluation(dataset: AgentDataset, evaluator: AgentEvaluator, sample_size: int = 25, + file_name: str = "sample_evaluation") -> Optional[pd.DataFrame]: + """ + Run agent evaluation on sample data and display results. + + Args: + dataset: AgentDataset to evaluate + evaluator: Initialized AgentEvaluator + sample_size: Number of samples to evaluate + file_name: Base name for output files + + Returns: + DataFrame with results or None if evaluation failed + """ + print("šŸš€ Running evaluation on sample data...") + + try: + # Create a smaller dataset for demo purposes + # Filter for records with agent responses or meaningful content + sample_data = [] + for data in dataset.data: + if (data.agent_response and data.agent_response.strip()) or data.agent_task: + sample_data.append(data) + if len(sample_data) >= sample_size: + break + + print(f"\nšŸ“Š Evaluating {len(sample_data)} sample records...") + + # Create a temporary dataset with just the sample data + sample_dataset = AgentDataset() + sample_dataset.data = sample_data + + # Create a new evaluator with the sample dataset + sample_evaluator = AgentEvaluator( + agent_dataset=sample_dataset, + models=evaluator.models, + scoring_functions=evaluator.scoring_functions, + output_dir=f"{evaluator.output_dir}/{file_name}", + stream=evaluator.stream, + include_reasoning=evaluator.include_reasoning + ) + + # Run the evaluation + sample_evaluator.run_all(save_every=1, file_type="csv") + + print("\nāœ… Evaluation completed!") + + # Read and display results + results_file = f"{evaluator.output_dir}/{file_name}/agent_evaluation_results.csv" + + if os.path.exists(results_file): + results_df = pd.read_csv(results_file) + print("\nšŸ“Š Results Summary:") + + # Calculate averages for each scorer + scorer_columns = [col for col in results_df.columns if col not in ['user_id', 'task_id', 'turn_id', 'agent_name'] and not col.endswith('_reasoning')] + + for col in scorer_columns: + if results_df[col].dtype in ['float64', 'int64']: + avg_score = results_df[col].mean() + print(f" - {col}: {avg_score:.2f}") + + # Show individual scores + print("\nšŸ” Individual Scores:") + for i, row in results_df.iterrows(): + print(f"\n Record {i+1} (Task: {row.get('task_id', 'N/A')}):") + for col in scorer_columns: + if pd.notna(row[col]): + print(f" - {col}: {row[col]}") + + return results_df + else: + print("āŒ Results file not found") + return None + + except Exception as e: + print(f"āŒ Error during evaluation: {e}") + print(f"Error type: {type(e).__name__}") + import traceback + traceback.print_exc() + return None + +print("āœ… run_evaluation function defined!") + +def analyze_agent_behavior_patterns(dataset: AgentDataset) -> Dict[str, Any]: + """ + Analyze agent behavior patterns including tool usage, task types, and response statistics. + + Args: + dataset: AgentDataset to analyze + + Returns: + Dictionary containing analysis results + """ + print("šŸ” Dataset Analysis:") + print("\n=== Agent Behavior Patterns ===") + + # Analyze tool usage patterns + tool_patterns = {} + task_types = {} + response_lengths = [] + + for data in dataset.data: + # Tool usage + if data.tool_calls: + for tool_call in data.tool_calls: + if hasattr(tool_call, 'tool_name'): + tool_name = tool_call.tool_name + if tool_name not in tool_patterns: + tool_patterns[tool_name] = {'count': 0, 'success_rate': 0} + tool_patterns[tool_name]['count'] += 1 + + # Task analysis + if data.agent_task: + # Simple categorization + task_lower = data.agent_task.lower() + if 'user_input' in task_lower: + task_types['user_input'] = task_types.get('user_input', 0) + 1 + elif 'exit' in task_lower: + task_types['exit_command'] = task_types.get('exit_command', 0) + 1 + else: + task_types['other'] = task_types.get('other', 0) + 1 + + # Response analysis + if data.agent_response: + response_lengths.append(len(data.agent_response)) + + print("\nšŸ“ˆ Tool Usage:") + for tool, stats in tool_patterns.items(): + print(f" - {tool}: {stats['count']} uses") + + print("\nšŸ“‹ Task Types:") + for task_type, count in task_types.items(): + print(f" - {task_type}: {count}") + + analysis_results = { + 'tool_patterns': tool_patterns, + 'task_types': task_types, + 'response_lengths': response_lengths + } + + if response_lengths: + avg_response_length = sum(response_lengths) / len(response_lengths) + print("\nšŸ“ Response Statistics:") + print(f" - Average response length: {avg_response_length:.1f} characters") + print(f" - Min response length: {min(response_lengths)}") + print(f" - Max response length: {max(response_lengths)}") + + analysis_results['avg_response_length'] = avg_response_length + analysis_results['min_response_length'] = min(response_lengths) + analysis_results['max_response_length'] = max(response_lengths) + + return analysis_results + +print("āœ… analyze_agent_behavior_patterns function defined!") + +def export_processed_dataset(dataset: AgentDataset, file_name: str = "processed_agent_dataset") -> bool: + """ + Export processed AgentDataset to JSON and CSV formats. + + Args: + dataset: AgentDataset to export + file_name: Base name for export files + + Returns: + True if export successful, False otherwise + """ + print("šŸ’¾ Exporting processed dataset...") + + success = True + + # Create directory if it doesn't exist + output_dir = Path(file_name).parent + output_dir.mkdir(parents=True, exist_ok=True) + try: + # Export to JSON + json_file = f'{file_name}.json' + dataset.export_to_json(json_file) + print(f"āœ… Exported to {json_file}") + + except Exception as e: + print(f"āŒ JSON export error: {e}") + success = False + + try: + # Export to CSV (optional) + csv_file = f'{file_name}.csv' + dataset.export_to_csv(csv_file) + print(f"āœ… Exported to {csv_file}") + + except Exception as e: + print(f"āŒ CSV export error: {e}") + success = False + + if success: + print("āœ… Export completed successfully!") + else: + print("āŒ Some exports failed") + + return success + +print("āœ… export_processed_dataset function defined!") +def setup_logging(log_level: str = "INFO", log_file: Optional[str] = None) -> None: + """ + Configure logging for the evaluation process. + + Args: + log_level: Logging level (DEBUG, INFO, WARNING, ERROR) + log_file: Optional log file path + """ + log_format = '%(asctime)s - %(levelname)s - %(name)s - %(message)s' + + if log_file: + logging.basicConfig( + level=getattr(logging, log_level.upper()), + format=log_format, + handlers=[ + logging.FileHandler(log_file), + logging.StreamHandler() + ] + ) + else: + logging.basicConfig( + level=getattr(logging, log_level.upper()), + format=log_format + ) + + print(f"āœ… Logging configured at {log_level} level") + +print("āœ… setup_logging function defined!") + +def validate_environment() -> Dict[str, bool]: + """ + Check for required environment variables and dependencies. + + Returns: + Dictionary with validation results + """ + results = { + 'gemini_api_key': 'GEMINI_API_KEY' in os.environ, + 'pandas_available': True, + 'novaeval_available': True + } + + try: + import pandas # pylint: disable=import-outside-toplevel,unused-import + except ImportError: + results['pandas_available'] = False + + try: + from novaeval.agents.agent_data import AgentData as _AgentData # pylint: disable=import-outside-toplevel,unused-import + except ImportError: + results['novaeval_available'] = False + + print("šŸ” Environment validation:") + for key, value in results.items(): + status = "āœ…" if value else "āŒ" + print(f" {status} {key}: {value}") + + return results + +print("āœ… validate_environment function defined!") + +def print_demo_summary(file_name: str, spans_count: int, dataset_count: int, + has_results: bool = False) -> None: + """ + Print a summary of the demo execution. + + Args: + file_name: Name of the processed file + spans_count: Number of spans processed + dataset_count: Number of AgentData records created + has_results: Whether evaluation results are available + """ + print("\nšŸŽ‰ Demo completed successfully!") + print("\nšŸ“‹ Summary:") + print(f" - Processed {spans_count} spans from {file_name}") + print(f" - Created {dataset_count} AgentData records") + print(" - Configured evaluation with Gemini model and 5 scorers") + if has_results: + print(" - Successfully evaluated sample data") + print(" - Exported processed dataset for future use") + +print("āœ… print_demo_summary function defined!") + + +def run_complete_agent_evaluation(selected_file: str, + sample_size: int = 25, + evaluation_name: str = "agent_evaluation", + model_name: str = "gemini-2.0-flash-lite", + temperature: float = 0.1, + max_tokens: int = 1024, + output_dir: str = "./evaluation_results") -> Dict[str, Any]: + """ + Complete agent evaluation pipeline in a single method call. + + Args: + selected_file: Path to the JSON dataset file to evaluate + sample_size: Number of samples to evaluate (default: 25) + evaluation_name: Name for the evaluation run (default: "agent_evaluation") + model_name: Gemini model to use (default: "gemini-2.0-flash-lite") + temperature: Model temperature (default: 0.1) + max_tokens: Max tokens for model (default: 1024) + output_dir: Output directory for results (default: "./evaluation_results") + + Returns: + Dictionary containing all results and status information + """ + + print("šŸš€ Starting Complete Agent Evaluation Pipeline") + print(f"šŸ“ Processing file: {selected_file}") + print("=" * 60) + + # Initialize results tracking + results = { + 'success': False, + 'file_processed': selected_file, + 'spans_loaded': 0, + 'dataset_created': False, + 'dataset_size': 0, + 'evaluation_completed': False, + 'results_df': None, + 'export_success': False, + 'errors': [] + } + + try: + # Step 1: Setup logging and validate environment + print("\nšŸ“‹ Step 1: Environment Setup") + setup_logging(log_level="INFO") + env_status = validate_environment() + + if not env_status['novaeval_available']: + results['errors'].append("NovaEval not available") + print("āŒ NovaEval not available. Please install it first.") + return results + elif not env_status['gemini_api_key']: + results['errors'].append("GEMINI_API_KEY not set") + print("āŒ GEMINI_API_KEY not set. Evaluation cannot proceed.") + return results + else: + print("āœ… Environment ready for evaluation!") + + # Step 2: Load and analyze dataset + print("\nšŸ“‹ Step 2: Loading Dataset") + spans_data, span_types = load_and_analyze_dataset(selected_file) + + if not spans_data: + results['errors'].append("Failed to load dataset") + print("āŒ Failed to load dataset") + return results + + results['spans_loaded'] = len(spans_data) + print(f"āœ… Dataset loaded: {len(spans_data)} spans") + + # Step 3: Convert to AgentDataset format + print("\nšŸ“‹ Step 3: Converting to AgentDataset Format") + agent_data_list, conversion_errors, dataset = convert_spans_to_agent_dataset(spans_data) + + if not dataset or not dataset.data: + results['errors'].append("Failed to create dataset") + results['errors'].extend(conversion_errors[:5]) # Add first 5 conversion errors + print("āŒ Failed to create dataset") + return results + + results['dataset_created'] = True + results['dataset_size'] = len(dataset.data) + print(f"āœ… AgentDataset created: {len(dataset.data)} records") + + # Step 4: Dataset analysis + print("\nšŸ“‹ Step 4: Dataset Analysis") + stats = analyze_dataset_statistics(dataset) + behavior_analysis = analyze_agent_behavior_patterns(dataset) + + # Step 5: Setup evaluation components + print("\nšŸ“‹ Step 5: Setting up Evaluation") + gemini_model = setup_gemini_model( + model_name=model_name, + temperature=temperature, + max_tokens=max_tokens + ) + + if not gemini_model: + results['errors'].append("Failed to setup Gemini model") + print("āŒ Failed to setup Gemini model") + return results + + evaluator = setup_agent_evaluator( + dataset=dataset, + gemini_model=gemini_model, + output_dir=output_dir, + include_reasoning=True, + stream=False + ) + + if not evaluator: + results['errors'].append("Failed to setup evaluator") + print("āŒ Failed to setup evaluator") + return results + + print("āœ… Evaluation components ready!") + + # Step 6: Run evaluation + print("\nšŸ“‹ Step 6: Running Evaluation") + print(f"šŸŽÆ Evaluating {sample_size} samples...") + + results_df = run_evaluation( + dataset=dataset, + evaluator=evaluator, + sample_size=sample_size, + file_name=evaluation_name + ) + + if results_df is not None: + results['evaluation_completed'] = True + results['results_df'] = results_df + print("āœ… Evaluation completed successfully!") + else: + results['errors'].append("Evaluation failed") + print("āŒ Evaluation failed") + + # Step 7: Export processed dataset + print("\nšŸ“‹ Step 7: Exporting Dataset") + export_success = export_processed_dataset( + dataset=dataset, + file_name=f"./processed_datasets/{evaluation_name}_processed_dataset" + ) + + results['export_success'] = export_success + + # Final summary + print("\n" + "=" * 60) + print("šŸŽ‰ EVALUATION PIPELINE COMPLETED!") + print(f"šŸ“Š Final Results:") + print(f" - File processed: {selected_file}") + print(f" - Spans loaded: {results['spans_loaded']}") + print(f" - Dataset size: {results['dataset_size']}") + print(f" - Evaluation completed: {results['evaluation_completed']}") + print(f" - Export successful: {results['export_success']}") + + if results['evaluation_completed']: + print(f" - Results saved to: {output_dir}/{evaluation_name}/") + + if results['errors']: + print(f" - Errors encountered: {len(results['errors'])}") + + results['success'] = results['evaluation_completed'] and results['export_success'] + + return results + + except Exception as e: + error_msg = f"Pipeline failed with error: {str(e)}" + results['errors'].append(error_msg) + print(f"āŒ {error_msg}") + import traceback + traceback.print_exc() + return results + +print("āœ… run_complete_agent_evaluation function defined!") \ No newline at end of file diff --git a/noveum_customer_support_bt/fetch_dataset_items.py b/noveum_customer_support_bt/fetch_dataset_items.py new file mode 100644 index 0000000..4a5c758 --- /dev/null +++ b/noveum_customer_support_bt/fetch_dataset_items.py @@ -0,0 +1,94 @@ +#!/usr/bin/env python3 +""" +Script to fetch dataset items from Noveum API and create api_data.json file. +This file is needed for the upload_scores.py script. +""" + +import os +import json +import requests +from dotenv import load_dotenv +from typing import Dict, Any, Optional + +# Load environment variables +load_dotenv() + +# Get API credentials from environment +api_key = os.getenv('NOVEUM_API_KEY') +org_slug = os.getenv('NOVEUM_ORG_SLUG') +dataset_slug = os.getenv('NOVEUM_DATASET_SLUG') +latest_version = os.getenv('LATEST_VERSION') +beta_env = os.getenv('BETA', 'false').lower() == 'true' + +def fetch_dataset_items() -> Optional[Dict[str, Any]]: + """Fetch dataset items from Noveum API""" + + # Construct API URL based on BETA environment variable + if beta_env: + api_url = f"https://noveum.ai/api/v1/datasets/{dataset_slug}/items?organizationSlug={org_slug}&version={latest_version}" + else: + api_url = f"https://noveum.ai/api/v1/organizations/{org_slug}/datasets/{dataset_slug}/items?version={latest_version}" + + # Prepare headers + headers = { + 'Authorization': f'Bearer {api_key}', + 'Cookie': f'apiKeyCookie={api_key}' + } + + print(f"Fetching dataset items from: {api_url}") + print(f"Organization: {org_slug}") + print(f"Dataset: {dataset_slug}") + print(f"Version: {latest_version}") + + try: + response = requests.get(api_url, headers=headers, timeout=30) + response.raise_for_status() + + data = response.json() + print(f"Successfully fetched {len(data.get('items', []))} items") + print(f"Response status: {response.status_code}") + + return data + + except requests.exceptions.RequestException as e: + print(f"Error fetching dataset items: {e}") + if hasattr(e, 'response') and e.response is not None: + print(f"Response status: {e.response.status_code}") + print(f"Response text: {e.response.text}") + return None + +def main(): + # Validate environment variables + required_vars = { + 'NOVEUM_API_KEY': api_key, + 'NOVEUM_ORG_SLUG': org_slug, + 'NOVEUM_DATASET_SLUG': dataset_slug, + 'LATEST_VERSION': latest_version + } + + missing_vars = [var for var, value in required_vars.items() if not value] + + if missing_vars: + print(f"Error: Missing required environment variables: {', '.join(missing_vars)}") + return 1 + + # Fetch dataset items + data = fetch_dataset_items() + + if data is None: + return 1 + + # Save response to api_data.json + try: + with open('api_data.json', 'w', encoding='utf-8') as f: + json.dump(data, f, indent=2) + print(f"\nDataset items saved to: api_data.json") + print(f"Total items: {len(data.get('items', []))}") + except (OSError, IOError) as e: + print(f"Error saving dataset items: {e}") + return 1 + + return 0 + +if __name__ == "__main__": + exit(main()) diff --git a/noveum_customer_support_bt/final_agent_evaluation_demo.ipynb b/noveum_customer_support_bt/final_agent_evaluation_demo.ipynb new file mode 100644 index 0000000..5b484b3 --- /dev/null +++ b/noveum_customer_support_bt/final_agent_evaluation_demo.ipynb @@ -0,0 +1,251 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# The agent as shown on Noveum.ai platform\n", + "\n", + "![Alt text](support_agent.png)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Final Agent Evaluation Demo with NovaEval\n", + "\n", + "This notebook demonstrates a streamlined approach to agent evaluation using modular utility functions:\n", + "\n", + "1. **Load agent trace data** from JSON datasets\n", + "2. **Map trace spans** to AgentData format using utility functions\n", + "3. **Create and analyze** AgentDataset\n", + "4. **Evaluate agent performance** using AgentEvaluator with Gemini model\n", + "5. **Analyze results** and export data\n", + "\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Scorers Used\n", + "\n", + "**context_relevancy_scorer** - Evaluates whether the agent response is appropriate and relevant given the agent's task and role.\n", + "\n", + "**role_adherence_scorer** - Scores whether the agent's tool calls and response adhere to its assigned role and task.\n", + "\n", + "**task_progression_scorer** - Measures whether the agent has made meaningful progress on the assigned task.\n", + "\n", + "**tool_relevancy_scorer** - Assesses how relevant and appropriate the tool call is given the available tools and the agent's context.\n", + "\n", + "**tool_correctness_scorer** - Compares actual tool calls against expected tool calls to evaluate correctness of tool usage and parameters.\n", + "\n", + "**parameter_correctness_scorer** - Validates whether correct parameters were passed to tool calls by analyzing the tool results." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 1: Import Dependencies and Utility Functions\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Import our custom utility functions\n", + "from demo_utils import (\n", + " list_dataset_files,\n", + " load_and_analyze_dataset,\n", + " convert_spans_to_agent_dataset,\n", + " analyze_dataset_statistics,\n", + " setup_gemini_model,\n", + " setup_agent_evaluator,\n", + " run_evaluation,\n", + " analyze_agent_behavior_patterns,\n", + " export_processed_dataset,\n", + " setup_logging,\n", + " validate_environment,\n", + " print_demo_summary\n", + ")\n", + "\n", + "print(\"āœ… All utility functions imported successfully!\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from dotenv import load_dotenv\n", + "\n", + "load_dotenv()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!python preprocess_filter.py ./traces/traces/dataset.json\n", + "!python preprocess_map.py ./traces/dataset_filtered.json\n", + "!python preprocess_split_data.py ./traces/dataset_filtered_mapped.json" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Force reload the demo_utils module to get the latest changes\n", + "import importlib\n", + "import sys\n", + "\n", + "# Remove the module from cache if it exists\n", + "if 'demo_utils' in sys.modules:\n", + " del sys.modules['demo_utils']\n", + "\n", + "# Import the updated module\n", + "from demo_utils import run_complete_agent_evaluation\n", + "\n", + "print(\"āœ… Module reloaded successfully!\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from demo_utils import run_complete_agent_evaluation\n", + "run_complete_agent_evaluation('./split_datasets/agent.rag_evaluation_metrics_dataset.json',\n", + "evaluation_name = \"agent.rag_evaluation_metrics_dataset\", output_dir = \"./demo_results\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Analysis of poor scores in comment generation agent." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import pandas as pd\n", + "comment_gen = pd.read_csv(\"demo_results/agent.rag_evaluation_metrics_dataset/agent_evaluation_results.csv\")\n", + "\n", + "split_size = 3\n", + "\n", + "task_progression = comment_gen.sort_values(by = 'task_progression', ascending= True).iloc[:split_size][['task_progression', 'task_progression_reasoning']]\n", + "\n", + "print(\"Task Progression:\")\n", + "print()\n", + "for idx, row in task_progression.iterrows():\n", + " print(f\"Score = {row['task_progression']}\")\n", + " print(f\"Reasoning = {row['task_progression_reasoning']}\")\n", + " print() # blank line" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Context Relevancy Analysis\n", + "context_relevancy = comment_gen.sort_values(by='context_relevancy', ascending=True).iloc[:3][['context_relevancy', 'context_relevancy_reasoning']]\n", + "\n", + "print(\"Context Relevancy Analysis:\")\n", + "print(\"=\" * 50)\n", + "for idx, row in context_relevancy.iterrows():\n", + " print(f\"Score = {row['context_relevancy']}\")\n", + " print(f\"Reasoning = {row['context_relevancy_reasoning']}\")\n", + " print()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Role Adherence Analysis\n", + "role_adherence = comment_gen.sort_values(by='role_adherence', ascending=True).iloc[:3][['role_adherence', 'role_adherence_reasoning']]\n", + "\n", + "print(\"Role Adherence Analysis:\")\n", + "print(\"=\" * 50)\n", + "for idx, row in role_adherence.iterrows():\n", + " print(f\"Score = {row['role_adherence']}\")\n", + " print(f\"Reasoning = {row['role_adherence_reasoning']}\")\n", + " print()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": ".venv", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.12" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/noveum_customer_support_bt/fix_api_data_v2.py b/noveum_customer_support_bt/fix_api_data_v2.py new file mode 100644 index 0000000..cfc4f5b --- /dev/null +++ b/noveum_customer_support_bt/fix_api_data_v2.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python3 +""" +Script to fix api_data.json by adding item_key field from turn_id found anywhere in the item. +""" + +import json + +def fix_api_data(): + """Fix api_data.json by adding item_key field""" + + # Load the api_data.json file + with open('api_data.json', 'r') as f: + data = json.load(f) + + # Process each item to add item_key + items = data.get('items', []) + fixed_items = [] + + for item in items: + # Create a copy of the item + fixed_item = item.copy() + + # Look for turn_id in any field + turn_id = None + + # Check if turn_id is directly in the item + if 'turn_id' in item: + turn_id = item['turn_id'] + else: + # Search through all string values for turn_id pattern + for key, value in item.items(): + if isinstance(value, str) and 'turn_id' in value: + try: + # Try to parse as JSON and extract turn_id + parsed = json.loads(value) + if isinstance(parsed, dict) and 'turn_id' in parsed: + turn_id = parsed['turn_id'] + break + except: + pass + + # Add item_key field + fixed_item['item_key'] = turn_id or '' + fixed_items.append(fixed_item) + + # Update the data + data['items'] = fixed_items + + # Save the fixed file + with open('api_data.json', 'w') as f: + json.dump(data, f, indent=2) + + print(f"Fixed api_data.json with {len(fixed_items)} items") + print(f"Added item_key field for each item using turn_id found in the data") + + # Show a sample of the fixed data + if fixed_items: + print(f"\nSample item_key: {fixed_items[0].get('item_key', 'NOT_FOUND')}") + +if __name__ == "__main__": + fix_api_data() diff --git a/noveum_customer_support_bt/novapilot_utils.py b/noveum_customer_support_bt/novapilot_utils.py new file mode 100644 index 0000000..f1d3d3a --- /dev/null +++ b/noveum_customer_support_bt/novapilot_utils.py @@ -0,0 +1,561 @@ +""" +NovaPilot Utilities - Agent Analysis and Evaluation Tools + +This module provides utilities for analyzing agent performance using Gemini AI +and generating comprehensive reports from evaluation data. +""" + +import os +import pandas as pd +import google.generativeai as genai +from dotenv import load_dotenv +from datetime import datetime +import json +from pathlib import Path +from typing import Dict, List, Optional, Tuple + + +# Prompt templates +SCORER_ANALYSIS_PROMPT = """ A scorer named {scorer_name} is run on different runs of the same agent/llm/tool, +the scorer gives a score and a reasoning, you will be given 25 such samples. Now the scorer may be giving different +reasonings and all of them are in natural language. I want you to highlight the key reasons for your response. +These key reasons will further be used for analysis, and improvement of the agent. Do not suggest any fixes. +Just focus on not missing out on any of the information regarding why the agent is failing. +You have to focus on the low scores only, as we have to improve them. +Some rows might show - "Missing required fields" it is a code issue, on the developer's side, so do not include it in the reasoning. + +Just to clarify, your job is not to analyze the scorers, but to analyze the agent. You are basically the representative of the scorers. + +Give the reasoning, and start with the scorer name. + +In the format - +Scorer Name: Task Progression +Reasoning: + +""" + +AGENTWISE_SUMMARY_PROMPT = """ +Different scorers are run on different runs of the same agent/llm/tool, you will be given the reasoning +for each scorer, as to why it gave poor scores. You job is to summarize the information from different +scorers into a single analysis. All the scores are of one specific part of the entire agentic workflow, +so please remove the redundancies that you get. Do not try to suggest fixes, only focus on removing +the redundant information, and keeping the important information. + +Just to clarify, your job is not to analyze the summaries, but to analyze the agent. You are basically the representative of the scorers. + +In the format - +Agent Name: query_generation +Reasoning: + +""" + +FINAL_ANALYSIS_PROMPT = """ +An agent is run, and then different scorers are run on specific parts of the agentic workflow. +So if an agent has 5 different parts (llm/tool/agent), and there are 3 scorers, then there will be a total of 15 scores, and respective reasonings. I have condensed these reasonings, in a part wise manner. +You will be given the part wise analysis, and you will also be given the entire agentic workflow, explaining how the agent is set up. + +You have to figure out why the agent is failing, you are given a bird's eye view, as in agents, a failure at step 1, may surface at step 3 in the analysis, so you will have to be aware of that. + +You have to suggest fixes to the developer in bullet points, in the format -> + +Suggested Fixes: + - fix_1: + - fix_2: + +""" + + +class NovaPilotAnalyzer: + """ + A class for analyzing agent performance using Gemini AI and generating comprehensive reports. + """ + + def __init__(self, api_key: Optional[str] = None, model_name: str = 'gemini-2.5-pro'): + """ + Initialize the NovaPilot Analyzer. + + Args: + api_key: Gemini API key. If None, will try to load from environment. + model_name: Name of the Gemini model to use. + """ + self._setup_gemini(api_key, model_name) + self.log_file: Optional[Path] = None + self.reddit_agent_doc: Optional[str] = None + + def _setup_gemini(self, api_key: Optional[str], model_name: str) -> None: + """Setup Gemini API configuration.""" + if api_key is None: + load_dotenv() + api_key = os.getenv('GEMINI_API_KEY') + + if not api_key: + raise ValueError("GEMINI_API_KEY not found in environment variables") + + genai.configure(api_key=api_key) + self.model = genai.GenerativeModel(model_name) + + def setup_logging(self, log_dir: str = 'log') -> Path: + """ + Setup logging directory and create timestamped log file. + + Args: + log_dir: Directory to store log files. + + Returns: + Path to the created log file. + """ + log_path = Path(log_dir) + log_path.mkdir(exist_ok=True) + + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + self.log_file = log_path / f"analysis_log_{timestamp}.txt" + + return self.log_file + + def load_agent_documentation(self, doc_path: str) -> str: + """ + Load agent documentation from a markdown file. + + Args: + doc_path: Path to the agent documentation file. + + Returns: + Content of the documentation file. + """ + with open(doc_path, 'r', encoding='utf-8') as f: + self.reddit_agent_doc = f.read() + return self.reddit_agent_doc + + def log_response(self, response: str, description: str) -> None: + """ + Log a response to the log file. + + Args: + response: The response text to log. + description: Description of what the response contains. + """ + if not self.log_file: + raise ValueError("Logging not setup. Call setup_logging() first.") + + with open(self.log_file, 'a', encoding='utf-8') as f: + f.write(f"\n{'='*50}\n") + f.write(f"TIMESTAMP: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}\n") + f.write(f"DESCRIPTION: {description}\n") + f.write(f"{'='*50}\n") + f.write(f"{response}\n") + f.write(f"{'='*50}\n\n") + + def process_csv_file(self, csv_path: str, max_rows: int = 25) -> Dict[str, str]: + """ + Process CSV file and extract scorer data for specified number of rows. + + Args: + csv_path: Path to the CSV file. + max_rows: Maximum number of rows to process. + + Returns: + Dictionary with scorer data strings. + """ + df = pd.read_csv(csv_path) + + # Get first max_rows rows (or all if less than max_rows) + rows_to_process = min(max_rows, len(df)) + df_subset = df.head(rows_to_process) + + # Get column names + columns = df.columns.tolist() + + # Skip first 4 columns (IDs) + # The remaining columns are split into score columns and reasoning columns + remaining_columns = columns[4:] + n_scorers = len(remaining_columns) // 2 # Each scorer has score + reasoning column + + score_columns = remaining_columns[:n_scorers] # First half are score columns + reasoning_columns = remaining_columns[n_scorers:] # Second half are reasoning columns + + scorer_data = {} + + # For each scorer + for i, scorer_name in enumerate(score_columns): + scorer_strings = [] + reasoning_col = reasoning_columns[i] + + for _, row in df_subset.iterrows(): + score = row[scorer_name] + reasoning = row[reasoning_col] + scorer_string = f"{scorer_name} score = {score} reasoning = {reasoning}" + scorer_strings.append(scorer_string) + + scorer_data[scorer_name] = "\n".join(scorer_strings) + + return scorer_data + + def analyze_scorer(self, scorer_name: str, scorer_data: str, dataset_name: str) -> str: + """ + Analyze a single scorer's data using Gemini AI. + + Args: + scorer_name: Name of the scorer. + scorer_data: Data string for the scorer. + dataset_name: Name of the dataset being analyzed. + + Returns: + Analysis response from Gemini AI. + """ + prompt = f"""{SCORER_ANALYSIS_PROMPT} + + Scorer Data: + Scorer Name: {scorer_name} + {scorer_data} + """ + + try: + response = self.model.generate_content(prompt) + response_text = response.text + response_text = "Analysis for scorer: " + scorer_name + "\n" + response_text + + # Log the response + self.log_response(response_text, f"{dataset_name} - {scorer_name} Analysis") + + return f"Scorer: {scorer_name}\n{response_text}" + + except Exception as e: + error_msg = f"Error processing {scorer_name}: {str(e)}" + self.log_response(error_msg, f"{dataset_name} - {scorer_name} Error") + raise e + + def create_dataset_summary(self, dataset_name: str, scorer_analyses: List[str]) -> str: + """ + Create a summary for a dataset based on scorer analyses. + + Args: + dataset_name: Name of the dataset. + scorer_analyses: List of scorer analysis strings. + + Returns: + Summary response from Gemini AI. + """ + combined_responses = "\n\n".join(scorer_analyses) + + summary_prompt = f"""{AGENTWISE_SUMMARY_PROMPT} + + Agent Name: {dataset_name} + Scorer Analyses: + {combined_responses} + """ + + try: + summary_response = self.model.generate_content(summary_prompt) + summary_text = summary_response.text + + # Log the summary + self.log_response(summary_text, f"{dataset_name} - Dataset Summary") + + return f"Dataset: {dataset_name}\n{summary_text}" + + except Exception as e: + error_msg = f"Error creating summary for {dataset_name}: {str(e)}" + self.log_response(error_msg, f"{dataset_name} - Summary Error") + raise e + + def process_dataset_directory(self, dataset_dir: Path, verbose: bool = True) -> Optional[str]: + """ + Process a single dataset directory. + + Args: + dataset_dir: Path to the dataset directory. + verbose: Whether to print progress messages. + + Returns: + Dataset summary string if successful, None if no CSV files found. + """ + if verbose: + print(f"\nProcessing {dataset_dir.name}...") + + # Find CSV file in the directory + csv_files = list(dataset_dir.glob('*.csv')) + if not csv_files: + if verbose: + print(f" No CSV files found in {dataset_dir.name}, skipping...") + return None + + csv_file = csv_files[0] # Take the first CSV file + if verbose: + print(f" Processing CSV: {csv_file.name}") + + # Process the CSV file + scorer_data = self.process_csv_file(csv_file) + + # Store responses for this dataset + dataset_responses = [] + + # For each scorer, make a Gemini API call + for scorer_name, scorer_string in scorer_data.items(): + if verbose: + print(f" Making Gemini call for scorer: {scorer_name}") + + try: + analysis = self.analyze_scorer(scorer_name, scorer_string, dataset_dir.name) + dataset_responses.append(analysis) + except Exception as e: + if verbose: + print(f" Error processing {scorer_name}: {str(e)}") + continue + + # Make summary call for this dataset + if verbose: + print(f" Making summary call for {dataset_dir.name}") + + try: + summary = self.create_dataset_summary(dataset_dir.name, dataset_responses) + return summary + except Exception as e: + if verbose: + print(f" Error creating summary: {str(e)}") + return None + + def process_all_datasets(self, demo_results_dir: str = 'demo_results', verbose: bool = True) -> List[str]: + """ + Process all dataset directories in the demo results folder. + + Args: + demo_results_dir: Path to the demo results directory. + verbose: Whether to print progress messages. + + Returns: + List of dataset summary strings. + """ + demo_results_path = Path(demo_results_dir) + all_summaries = [] + + # Get all dataset directories + dataset_dirs = [d for d in demo_results_path.iterdir() if d.is_dir()] + + if verbose: + print(f"Found {len(dataset_dirs)} dataset directories to process:") + for d in dataset_dirs: + print(f" - {d.name}") + + # Process each dataset directory + for dataset_dir in dataset_dirs: + summary = self.process_dataset_directory(dataset_dir, verbose) + if summary: + all_summaries.append(summary) + + if verbose: + print(f"\nCompleted processing {len(dataset_dirs)} datasets.") + + return all_summaries + + def create_final_analysis(self, dataset_summaries: List[str], agent_doc: Optional[str] = None) -> str: + """ + Create final comprehensive analysis from all dataset summaries. + + Args: + dataset_summaries: List of dataset summary strings. + agent_doc: Agent documentation string. If None, uses loaded documentation. + + Returns: + Final analysis response from Gemini AI. + """ + if agent_doc is None: + agent_doc = self.reddit_agent_doc + + if not agent_doc: + raise ValueError("No agent documentation provided. Load it first or pass as parameter.") + + # Combine all dataset summaries + combined_summaries = "\n\n".join(dataset_summaries) + + final_prompt = f"""{FINAL_ANALYSIS_PROMPT} + + Reddit Agent Documentation: + {agent_doc} + + Dataset Summaries: + {combined_summaries} + """ + + try: + final_response = self.model.generate_content(final_prompt) + final_text = final_response.text + + # Log the final analysis + self.log_response(final_text, "Final Comprehensive Analysis") + + return final_text + + except Exception as e: + error_msg = f"Error creating final analysis: {str(e)}" + self.log_response(error_msg, "Final Analysis Error") + raise e + + def run_complete_analysis(self, + demo_results_dir: str = 'demo_results', + agent_doc_path: Optional[str] = None, + log_dir: str = 'log', + verbose: bool = True) -> Tuple[str, List[str], Path]: + """ + Run the complete analysis pipeline. + + Args: + demo_results_dir: Path to the demo results directory. + agent_doc_path: Path to agent documentation file. + log_dir: Directory to store log files. + verbose: Whether to print progress messages. + + Returns: + Tuple of (final_analysis, dataset_summaries, log_file_path). + """ + # Setup logging + log_file = self.setup_logging(log_dir) + + # Load agent documentation if provided + if agent_doc_path: + self.load_agent_documentation(agent_doc_path) + + # Process all datasets + dataset_summaries = self.process_all_datasets(demo_results_dir, verbose) + + # Create final analysis + if verbose: + print("\nMaking final comprehensive analysis call...") + + final_analysis = self.create_final_analysis(dataset_summaries) + + if verbose: + print("Final analysis completed and logged!") + print(f"All responses have been logged to: {log_file}") + print("\n" + "="*50) + print("ANALYSIS COMPLETE!") + print("="*50) + print(f"Log file location: {log_file}") + print(f"Total datasets processed: {len(dataset_summaries)}") + print(f"Total summaries generated: {len(dataset_summaries)}") + print("="*50) + + return final_analysis, dataset_summaries, log_file + + +# Convenience functions for backward compatibility +def setup_gemini_analyzer(api_key: Optional[str] = None, model_name: str = 'gemini-2.5-pro') -> NovaPilotAnalyzer: + """ + Create and setup a NovaPilot Analyzer instance. + + Args: + api_key: Gemini API key. If None, will try to load from environment. + model_name: Name of the Gemini model to use. + + Returns: + Configured NovaPilotAnalyzer instance. + """ + return NovaPilotAnalyzer(api_key, model_name) + + +def run_agent_analysis(demo_results_dir: str = 'demo_results', + agent_doc_path: Optional[str] = None, + log_dir: str = 'log', + api_key: Optional[str] = None, + model_name: str = 'gemini-2.5-pro', + verbose: bool = True) -> Tuple[str, List[str], Path]: + """ + Run complete agent analysis pipeline. + + Args: + demo_results_dir: Path to the demo results directory. + agent_doc_path: Path to agent documentation file. + log_dir: Directory to store log files. + api_key: Gemini API key. If None, will try to load from environment. + model_name: Name of the Gemini model to use. + verbose: Whether to print progress messages. + + Returns: + Tuple of (final_analysis, dataset_summaries, log_file_path). + """ + analyzer = setup_gemini_analyzer(api_key, model_name) + return analyzer.run_complete_analysis(demo_results_dir, agent_doc_path, log_dir, verbose) + + +def recommend_improvements(demo_results_dir: str = 'demo_results', + agent_doc_path: str = 'reddit_agent.md', + log_dir: str = 'log', + api_key: Optional[str] = None, + model_name: str = 'gemini-2.5-pro', + verbose: bool = True) -> Tuple[str, List[str], Path]: + """ + Single function to run the complete agent analysis pipeline. + This is equivalent to running the entire complete_analysis_demo.ipynb notebook. + + This function: + 1. Sets up Gemini API and logging + 2. Loads agent documentation + 3. Processes all dataset directories + 4. Analyzes each scorer's data + 5. Creates dataset summaries + 6. Generates final comprehensive analysis with improvement recommendations + + Args: + demo_results_dir: Path to the demo results directory containing dataset folders. + agent_doc_path: Path to the agent documentation markdown file. + log_dir: Directory to store analysis log files. + api_key: Gemini API key. If None, will try to load from environment. + model_name: Name of the Gemini model to use. + verbose: Whether to print progress messages. + + Returns: + Tuple of (final_analysis, dataset_summaries, log_file_path). + - final_analysis: Complete analysis with suggested fixes + - dataset_summaries: List of summaries for each dataset + - log_file_path: Path to the generated log file + + Example: + >>> final_analysis, summaries, log_file = recommend_improvements("demo_results") + >>> print(final_analysis) # Shows suggested fixes for the agent + """ + if verbose: + print("="*60) + print("NOVAPILOT AGENT ANALYSIS - RECOMMEND IMPROVEMENTS") + print("="*60) + print("This function runs the complete analysis pipeline equivalent to") + print("running the entire complete_analysis_demo.ipynb notebook.") + print("="*60) + + # Initialize analyzer + analyzer = NovaPilotAnalyzer(api_key, model_name) + + # Setup logging + log_file = analyzer.setup_logging(log_dir) + if verbose: + print(f"Setup complete! Log file: {log_file}") + + # Load agent documentation + if os.path.exists(agent_doc_path): + agent_doc = analyzer.load_agent_documentation(agent_doc_path) + if verbose: + print(f"Agent document loaded: {len(agent_doc)} characters") + else: + if verbose: + print(f"Warning: Agent document not found at {agent_doc_path}") + agent_doc = None + + # Process all datasets + dataset_summaries = analyzer.process_all_datasets(demo_results_dir, verbose) + + # Create final comprehensive analysis + if verbose: + print("\nMaking final comprehensive analysis call...") + + final_analysis = analyzer.create_final_analysis(dataset_summaries, agent_doc) + + if verbose: + print("Final analysis completed and logged!") + print(f"All responses have been logged to: {log_file}") + print("\n" + "="*50) + print("ANALYSIS COMPLETE!") + print("="*50) + print(f"Log file location: {log_file}") + print(f"Total datasets processed: {len(dataset_summaries)}") + print(f"Total summaries generated: {len(dataset_summaries)}") + print("="*50) + + return final_analysis, dataset_summaries, log_file diff --git a/noveum_customer_support_bt/noveum_agent_requirements.txt b/noveum_customer_support_bt/noveum_agent_requirements.txt new file mode 100644 index 0000000..a72a6b8 --- /dev/null +++ b/noveum_customer_support_bt/noveum_agent_requirements.txt @@ -0,0 +1,33 @@ +# Noveum AI Agent with RAG + Web Search - Additional Requirements +# Core dependencies (already in main requirements.txt) +requests==2.32.3 +beautifulsoup4==4.12.3 +python-dotenv==1.0.1 + +# LangChain ecosystem +langchain==0.3.26 +langchain-community==0.3.18 +langchain-core==0.3.66 +langchain-openai==0.3.25 + +# Web scraping and text extraction +trafilatura>=1.6.4 +lxml>=5.3.0 + +# Vector search and embeddings +faiss-cpu==1.12.0 +# Alternative: faiss-gpu==1.12.0 # Use this if you have CUDA support + +# Web search +duckduckgo-search>=6.1.12 + +# Data processing +pandas>=2.2.3 +tqdm>=4.67.1 + +# Noveum Trace +noveum_trace>=0.3.5 + +# Jupyter notebook support +jupyter==1.0.0 +ipykernel==6.29.4 diff --git a/noveum_customer_support_bt/noveum_docs.json b/noveum_customer_support_bt/noveum_docs.json new file mode 100644 index 0000000..01778a8 --- /dev/null +++ b/noveum_customer_support_bt/noveum_docs.json @@ -0,0 +1,378 @@ +[ + { + "url": "https://noveum.ai", + "title": "AI Observability, LLM Evals & Agent Monitoring | Noveum.ai", + "content": "Monitor all your AI Agents\nimprove AI Agents today\nNoveum.ai helps you monitor, trace, and optimize your AI applications.\nNoveum.ai works with any AI framework ā€“ LangChain, CrewAI, AutoGen, custom implementations, or direct LLM calls. One dashboard shows everything.\nMonitor, Evaluate, Improve Your AI Agents\nThe control plane for AI agents.\nMonitor Everything, Miss Nothing\nOur lightweight SDKs capture every trace and span across your AI agent ecosystemā€”from simple LLM calls to complex multi-agent workflows. Get complete visibility without performance overhead.\nStart MonitoringEvaluate with 30+ Advanced Metrics\nNovaEval automatically scores every agent interaction using our comprehensive evaluation framework. Track accuracy, semantic similarity, safety, bias, and custom business metrics in real-time.\nView EvaluationsImprove Automatically with NovaPilot\nOur AI engineer analyzes performance data and automatically generates fixes for failing agents. Get detailed reports on model changes, prompt optimizations, and tool improvementsā€”all without human intervention.\nTry Auto-ImprovementEnterprise Ready\nNoveum.ai is built for enterprise-scale AI applications, with support for multi-tenant, multi-region deployments and advanced security features.\nContact Saleswith the world's favorite AI Observability Platform\nEverything You Need to Master AI Agent Operations\nNoveum.ai helps you monitor, trace, and optimize your AI applications with comprehensive observability tools designed for modern LLM workflows.\nSee Every Agent, Every Interaction, Every Decision\n30+ Metrics That Actually Matter for Business\nYour AI Engineer That Never Sleeps\n100% visibility on every AI agent\nReduce AI Incidents by 85%\nGet comprehensive AI monitoring with automated incident prevention, faster debugging, and built-in compliance tools.\nInstead of spending days investigating AI agent failures, your team gets instant insights into what went wrong and how to fix it. Detailed traces and automated analysis eliminate guesswork.\n0+\nAI FrameworksWith the world's favorite AI observability platform\nEasy integration with your AI stack\nNoveum.ai integrates seamlessly with all popular AI frameworks and providers, giving you comprehensive observability across your entire AI pipeline.\nWorks great with: LangChain, OpenAI, Anthropic, AWS Bedrock, Azure OpenAI, Google Cloud (Vertex AI), CrewAI, LangGraph, LlamaIndex, AutoGen, custom SDKs, and more\nwith the world's favorite AI observability platform\nTrusted AI monitoring tools by thousands of developers\n0+\nAI Eval Metrics0.0%\nuptime SLA0M+\ntraces processed", + "content_length": 2591, + "internal_links": [ + "https://noveum.ai/en", + "https://noveum.ai/en", + "https://noveum.ai/en/blog", + "https://noveum.ai/en/docs", + "https://noveum.ai/en/careers", + "https://noveum.ai/en/contact", + "https://noveum.ai/auth/login", + "https://noveum.ai/en", + "https://noveum.ai/en", + "https://noveum.ai/en/blog", + "https://noveum.ai/en/docs", + "https://noveum.ai/en/contact", + "https://noveum.ai/auth/login", + "https://noveum.ai/auth/login", + "https://noveum.ai/docs", + "https://noveum.ai/auth/login", + "https://noveum.ai/auth/login", + "https://noveum.ai/auth/login", + "https://noveum.ai/en", + "https://noveum.ai/en/blog", + "https://noveum.ai/en/docs", + "https://noveum.ai/en/contact", + "https://noveum.ai/en/pricing", + "https://noveum.ai/en/docs/getting-started/sdk-integration", + "https://noveum.ai/en/docs/getting-started/overview", + "https://noveum.ai/en/auth/login" + ], + "scraped_at": 1759935886.929589 + }, + { + "url": "https://noveum.ai/en", + "title": "AI Observability, LLM Evals & Agent Monitoring | Noveum.ai", + "content": "Monitor all your AI Agents\nimprove AI Agents today\nNoveum.ai helps you monitor, trace, and optimize your AI applications.\nNoveum.ai works with any AI framework ā€“ LangChain, CrewAI, AutoGen, custom implementations, or direct LLM calls. One dashboard shows everything.\nMonitor, Evaluate, Improve Your AI Agents\nThe control plane for AI agents.\nMonitor Everything, Miss Nothing\nOur lightweight SDKs capture every trace and span across your AI agent ecosystemā€”from simple LLM calls to complex multi-agent workflows. Get complete visibility without performance overhead.\nStart MonitoringEvaluate with 30+ Advanced Metrics\nNovaEval automatically scores every agent interaction using our comprehensive evaluation framework. Track accuracy, semantic similarity, safety, bias, and custom business metrics in real-time.\nView EvaluationsImprove Automatically with NovaPilot\nOur AI engineer analyzes performance data and automatically generates fixes for failing agents. Get detailed reports on model changes, prompt optimizations, and tool improvementsā€”all without human intervention.\nTry Auto-ImprovementEnterprise Ready\nNoveum.ai is built for enterprise-scale AI applications, with support for multi-tenant, multi-region deployments and advanced security features.\nContact Saleswith the world's favorite AI Observability Platform\nEverything You Need to Master AI Agent Operations\nNoveum.ai helps you monitor, trace, and optimize your AI applications with comprehensive observability tools designed for modern LLM workflows.\nSee Every Agent, Every Interaction, Every Decision\n30+ Metrics That Actually Matter for Business\nYour AI Engineer That Never Sleeps\n100% visibility on every AI agent\nReduce AI Incidents by 85%\nGet comprehensive AI monitoring with automated incident prevention, faster debugging, and built-in compliance tools.\nInstead of spending days investigating AI agent failures, your team gets instant insights into what went wrong and how to fix it. Detailed traces and automated analysis eliminate guesswork.\n0+\nAI FrameworksWith the world's favorite AI observability platform\nEasy integration with your AI stack\nNoveum.ai integrates seamlessly with all popular AI frameworks and providers, giving you comprehensive observability across your entire AI pipeline.\nWorks great with: LangChain, OpenAI, Anthropic, AWS Bedrock, Azure OpenAI, Google Cloud (Vertex AI), CrewAI, LangGraph, LlamaIndex, AutoGen, custom SDKs, and more\nwith the world's favorite AI observability platform\nTrusted AI monitoring tools by thousands of developers\n0+\nAI Eval Metrics0.0%\nuptime SLA0M+\ntraces processed", + "content_length": 2591, + "internal_links": [], + "scraped_at": 1759935887.816968 + }, + { + "url": "https://noveum.ai/en/blog", + "title": "Noveum.ai Blog | Noveum.ai", + "content": "Noveum.ai Blog\nRead the latest news & articles from Noveum.ai (prev MagicAPI Inc).\nLearn what evals for AI agents are, why they are essential for production AI, and how Noveum.ai makes running evaluations practical without slowing down your development roadmap.\nAditi Upaddhyay\n9/25/2025\nMMLU benchmark comparison of GPT-OSS (thinking modes), GPT-5, O3, and GPT-4o-mini focusing on accuracy, runtime efficiency, and practical model selection.\nShivam Gupta\n8/13/2025\nWe compared Azure o1-mini vs gpt-4o-mini on 1,000 MMLU math samples using NovaEval. Hereā€™s how we tested, what worked, what didnā€™t, and when the 15Ɨ cost premium makes sense.\nShashank Agarwal\n8/12/2025\nDiscover how Noveum.ai provides comprehensive tracing and observability for AI applications, from development debugging to production optimization.\nShashank Agarwal\n3/3/2025\nDiscover how Noveum.ai provides comprehensive tracing and observability for LLM applications, RAG systems, and multi-agent workflows with our powerful Python and TypeScript SDKs.\nShashank Agarwal\n3/2/2025", + "content_length": 1046, + "internal_links": [ + "https://noveum.ai/en/changelog", + "https://noveum.ai/en/blog/evals-for-ai-agents", + "https://noveum.ai/en/blog/evals-for-ai-agents", + "https://noveum.ai/en/blog/gpt-oss-vs-gpt-5-vs-gpt-4o-mini-mmlu-evaluation-report", + "https://noveum.ai/en/blog/gpt-oss-vs-gpt-5-vs-gpt-4o-mini-mmlu-evaluation-report", + "https://noveum.ai/en/blog/comprehensive-mmlu-evaluation-analysis-report", + "https://noveum.ai/en/blog/comprehensive-mmlu-evaluation-analysis-report", + "https://noveum.ai/en/blog/from-logs-to-intelligent-choices-inside-noveum-ais-evaluation-process", + "https://noveum.ai/en/blog/from-logs-to-intelligent-choices-inside-noveum-ais-evaluation-process", + "https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform", + "https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform", + "https://noveum.ai/en/legal/privacy-policy", + "https://noveum.ai/en/legal/terms" + ], + "scraped_at": 1759935888.8273768 + }, + { + "url": "https://noveum.ai/en/docs", + "title": "Welcome to Noveum.ai Docs | Documentation | Noveum.ai", + "content": "Welcome to Noveum.ai Docs\nComprehensive AI tracing and observability for LLM applications, RAG systems, and AI agents\nWelcome to the Noveum.ai documentation! Here you'll find everything you need to integrate, configure, and optimize your AI applications using our comprehensive tracing and observability platform.\nWe provide powerful SDKs for Python and TypeScript that enable you to trace LLM calls, RAG pipelines, and multi-agent workflows with minimal code changes. Our documentation is organized to help you get started quickly while diving into advanced observability patterns and best practices.\nšŸš€ What is Noveum.ai?\nNoveum.ai is a comprehensive AI tracing and observability platform designed specifically for modern AI applications. Unlike traditional monitoring tools, Noveum understands the unique challenges of LLM applications, RAG systems, and AI agents.\nCore Components\n-\nšŸ Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\nšŸ“˜ TypeScript SDK (\n@noveum/trace\n)- Framework integrations for Next.js, Express.js, and Hono\n- TypeScript-first design with full type safety\n- Universal compatibility (Node.js, Edge Runtime, browsers)\n-\nšŸ“Š Noveum Platform\n- Real-time dashboard for analyzing traces and performance\n- Advanced filtering and search capabilities\n- Cost analysis and optimization insights\n- Team collaboration and project management\nšŸƒ Quick Start\n1. Choose Your SDK\n2. View Your Data\nWithin minutes, you'll see comprehensive traces in the Noveum dashboard:\n- šŸ” Request/Response details and timing\n- šŸ’° Cost tracking across providers\n- šŸš€ Performance metrics (latency, throughput)\n- šŸ› Error analysis and debugging context\nšŸ“š Documentation Structure\nGetting Started\n-\nOverview Introduction to Noveum.ai's tracing and observability capabilities.\n-\nSDK Integration Step-by-step guide to integrate Python or TypeScript SDKs into your application.\n-\nTracing Concepts Understanding traces, spans, and observability best practices for AI applications.\n-\nFramework Integrations Specific guides for Next.js, Express.js, FastAPI, and other popular frameworks.\nAdvanced Usage\n-\nMulti-Agent Tracing Observe complex agent workflows and inter-agent communications.\n-\nRAG Pipeline Observability Monitor retrieval, generation, and context handling in RAG systems.\n-\nCustom Instrumentation Add custom spans and attributes for domain-specific observability.\n-\nPerformance Optimization Use tracing data to identify bottlenecks and optimize AI application performance.\nPlatform Features\n-\nDashboard Overview Navigate the Noveum platform and understand key metrics.\n-\nProjects & Environments Organize your applications and manage different deployment environments.\n-\nTeam Collaboration Share insights and collaborate on AI application observability.\n-\nAPI Reference Direct API access for custom integrations and advanced use cases.\nšŸŽÆ Use Cases\nLLM Application Monitoring\nTrack every LLM call across your application with automatic cost calculation, latency measurement, and error tracking.\nRAG System Observability\nMonitor the entire RAG pipeline from query understanding to document retrieval to answer generation.\nMulti-Agent Workflows\nObserve complex agent interactions, tool usage, and decision-making processes across distributed AI systems.\nPerformance Optimization\nIdentify slow operations, expensive API calls, and opportunities for caching or model optimization.\nšŸ”— SDK Resources\nPython SDK\n- šŸ“¦ PyPI Package: noveum-trace\n- šŸ™ GitHub Repository: Noveum/noveum-trace\n- šŸ“– API Documentation: Python SDK Docs\nTypeScript SDK\n- šŸ“¦ NPM Package: @noveum/trace\n- šŸ™ GitHub Repository: Noveum/noveum-trace-ts\n- šŸ“– API Documentation: TypeScript SDK Docs\nšŸ’” Key Benefits\n- šŸ”§ Minimal Setup: Start tracing with just a few lines of code\n- šŸŽÆ AI-Native: Purpose-built for LLM, RAG, and agent observability\n- šŸš€ Production Ready: Battle-tested at scale with intelligent sampling\n- šŸ”’ Secure: End-to-end encryption with configurable data retention\n- šŸŒ Universal: Works across frameworks, clouds, and deployment models\nšŸ¤ Community & Support\n- šŸ’¬ Discord Community: Join our Discord\n- šŸ“§ Email Support: [email protected]\n- šŸ› Bug Reports: GitHub Issues\n- šŸ“– Knowledge Base: Help Center\nReady to get started? Head to our SDK Integration Guide to begin tracing your AI applications in under 5 minutes!\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year.", + "content_length": 4652, + "internal_links": [ + "https://noveum.ai/en/docs/getting-started/tracing-concepts", + "https://noveum.ai/en/docs/getting-started/framework-integrations", + "https://noveum.ai/en/docs/advanced/multi-agent-tracing", + "https://noveum.ai/en/docs/advanced/rag-observability", + "https://noveum.ai/en/docs/advanced/custom-instrumentation", + "https://noveum.ai/en/docs/advanced/performance-optimization", + "https://noveum.ai/docs/getting-started/overview", + "https://noveum.ai/docs/getting-started/sdk-integration", + "https://noveum.ai/docs/getting-started/tracing-concepts", + "https://noveum.ai/docs/getting-started/framework-integrations", + "https://noveum.ai/docs/advanced/multi-agent-tracing", + "https://noveum.ai/docs/advanced/rag-observability", + "https://noveum.ai/docs/advanced/custom-instrumentation", + "https://noveum.ai/docs/advanced/performance-optimization", + "https://noveum.ai/docs/platform/dashboard", + "https://noveum.ai/docs/platform/projects", + "https://noveum.ai/docs/platform/teams", + "https://noveum.ai/docs/platform/api", + "https://noveum.ai/docs/getting-started/sdk-integration" + ], + "scraped_at": 1759935889.759006 + }, + { + "url": "https://noveum.ai/en/careers", + "title": "AI Observability, LLM Evals & Agent Monitoring | Noveum.ai", + "content": "Why Work at Noveum.ai?\nJoin a team that's building the future of AI while maintaining a culture of innovation, transparency, and mutual respect.\nAI Agents Economy Pioneer\nBe at the forefront of the revolutionary AI Agents Economy and shape the future of autonomous AI systems.\nCreator of API.market\nJoin the team behind API.market, already loved by thousands of developers worldwide.\n100% Remote\nWork from anywhere in the world. We believe in flexibility and work-life balance.\nIndustry Leading Compensation\nCompetitive salaries and equity packages that reflect your value and contribution.\nAI-First Culture\nWe encourage using AI tools like Cursor IDE and ChatGPT. We'll train you to use them effectively.\nNo Politics, No Toxicity\nA healthy work environment focused on building great products and supporting each other.\nOpen Positions\nJoin our growing team and help build the next generation of AI agents and developer tools.\nMonitor, evaluate, and improve AI agents in production. Build eval pipelines, debug agent workflows, and deploy/tune models alongside observability to drive reliability and quality.\nKey Requirements:\nOur Culture & Values\nWe're building more than just technologyā€”we're creating a workplace where innovation thrives and everyone can do their best work.\nDo No Harm\nWe build technology that benefits humanity and take responsibility for our impact on the world.\nBe the Good You Want to See\nWe lead by example and create positive change in our industry and communities.\nOwnership\nWe take ownership of our work, decisions, and outcomes. Everyone is empowered to make a difference.\nTransparency\nWe believe in open communication, honest feedback, and sharing knowledge across the team.\nReady to Join Our Mission?\nWe're looking for passionate individuals who want to shape the future of AI agents and build technology that makes a positive impact.", + "content_length": 1864, + "internal_links": [ + "https://noveum.ai/en/careers/apply/senior-ai-engineer", + "https://noveum.ai/en/careers/apply/fullstack-developer-ai" + ], + "scraped_at": 1759935890.6604009 + }, + { + "url": "https://noveum.ai/en/contact", + "title": "Contact us | Noveum.ai", + "content": "Blog\nChangelog\nContact\nGitHub\nCareers\nDocs\nLogin\nContact us\nWe are here to help you. Please use the form below to get in touch with us.\nName\nEmail\nMessage\nSend message", + "content_length": 167, + "internal_links": [], + "scraped_at": 1759935891.625962 + }, + { + "url": "https://noveum.ai/auth/login", + "title": "Welcome back | Noveum.ai", + "content": "Welcome back\nPlease enter your credentials to sign in.\nPassword\nOTP\nEmail\nPassword\nForgot password?\nSign in\nOr continue with\nGoogle\nGoogle\nGithub\nGithub\nLogin with passkey\nDon't have an account yet?\nCreate an account", + "content_length": 216, + "internal_links": [ + "https://noveum.ai/", + "https://noveum.ai/auth/forgot-password", + "https://noveum.ai/auth/signup" + ], + "scraped_at": 1759935892.526221 + }, + { + "url": "https://noveum.ai/docs", + "title": "Welcome to Noveum.ai Docs | Documentation | Noveum.ai", + "content": "Welcome to Noveum.ai Docs\nComprehensive AI tracing and observability for LLM applications, RAG systems, and AI agents\nWelcome to the Noveum.ai documentation! Here you'll find everything you need to integrate, configure, and optimize your AI applications using our comprehensive tracing and observability platform.\nWe provide powerful SDKs for Python and TypeScript that enable you to trace LLM calls, RAG pipelines, and multi-agent workflows with minimal code changes. Our documentation is organized to help you get started quickly while diving into advanced observability patterns and best practices.\nšŸš€ What is Noveum.ai?\nNoveum.ai is a comprehensive AI tracing and observability platform designed specifically for modern AI applications. Unlike traditional monitoring tools, Noveum understands the unique challenges of LLM applications, RAG systems, and AI agents.\nCore Components\n-\nšŸ Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\nšŸ“˜ TypeScript SDK (\n@noveum/trace\n)- Framework integrations for Next.js, Express.js, and Hono\n- TypeScript-first design with full type safety\n- Universal compatibility (Node.js, Edge Runtime, browsers)\n-\nšŸ“Š Noveum Platform\n- Real-time dashboard for analyzing traces and performance\n- Advanced filtering and search capabilities\n- Cost analysis and optimization insights\n- Team collaboration and project management\nšŸƒ Quick Start\n1. Choose Your SDK\n2. View Your Data\nWithin minutes, you'll see comprehensive traces in the Noveum dashboard:\n- šŸ” Request/Response details and timing\n- šŸ’° Cost tracking across providers\n- šŸš€ Performance metrics (latency, throughput)\n- šŸ› Error analysis and debugging context\nšŸ“š Documentation Structure\nGetting Started\n-\nOverview Introduction to Noveum.ai's tracing and observability capabilities.\n-\nSDK Integration Step-by-step guide to integrate Python or TypeScript SDKs into your application.\n-\nTracing Concepts Understanding traces, spans, and observability best practices for AI applications.\n-\nFramework Integrations Specific guides for Next.js, Express.js, FastAPI, and other popular frameworks.\nAdvanced Usage\n-\nMulti-Agent Tracing Observe complex agent workflows and inter-agent communications.\n-\nRAG Pipeline Observability Monitor retrieval, generation, and context handling in RAG systems.\n-\nCustom Instrumentation Add custom spans and attributes for domain-specific observability.\n-\nPerformance Optimization Use tracing data to identify bottlenecks and optimize AI application performance.\nPlatform Features\n-\nDashboard Overview Navigate the Noveum platform and understand key metrics.\n-\nProjects & Environments Organize your applications and manage different deployment environments.\n-\nTeam Collaboration Share insights and collaborate on AI application observability.\n-\nAPI Reference Direct API access for custom integrations and advanced use cases.\nšŸŽÆ Use Cases\nLLM Application Monitoring\nTrack every LLM call across your application with automatic cost calculation, latency measurement, and error tracking.\nRAG System Observability\nMonitor the entire RAG pipeline from query understanding to document retrieval to answer generation.\nMulti-Agent Workflows\nObserve complex agent interactions, tool usage, and decision-making processes across distributed AI systems.\nPerformance Optimization\nIdentify slow operations, expensive API calls, and opportunities for caching or model optimization.\nšŸ”— SDK Resources\nPython SDK\n- šŸ“¦ PyPI Package: noveum-trace\n- šŸ™ GitHub Repository: Noveum/noveum-trace\n- šŸ“– API Documentation: Python SDK Docs\nTypeScript SDK\n- šŸ“¦ NPM Package: @noveum/trace\n- šŸ™ GitHub Repository: Noveum/noveum-trace-ts\n- šŸ“– API Documentation: TypeScript SDK Docs\nšŸ’” Key Benefits\n- šŸ”§ Minimal Setup: Start tracing with just a few lines of code\n- šŸŽÆ AI-Native: Purpose-built for LLM, RAG, and agent observability\n- šŸš€ Production Ready: Battle-tested at scale with intelligent sampling\n- šŸ”’ Secure: End-to-end encryption with configurable data retention\n- šŸŒ Universal: Works across frameworks, clouds, and deployment models\nšŸ¤ Community & Support\n- šŸ’¬ Discord Community: Join our Discord\n- šŸ“§ Email Support: [email protected]\n- šŸ› Bug Reports: GitHub Issues\n- šŸ“– Knowledge Base: Help Center\nReady to get started? Head to our SDK Integration Guide to begin tracing your AI applications in under 5 minutes!\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year.", + "content_length": 4652, + "internal_links": [], + "scraped_at": 1759935893.7828538 + }, + { + "url": "https://noveum.ai/en/docs/getting-started/sdk-integration", + "title": "SDK Integration Guide | Documentation | Noveum.ai", + "content": "SDK Integration Guide\nIntegrate Noveum.ai tracing into your AI applications with Python or TypeScript SDKs\nThe Noveum.ai SDKs provide comprehensive tracing and observability for your AI applications with minimal code changes. Whether you're building LLM applications, RAG systems, or multi-agent workflows, our SDKs automatically capture essential metrics and traces.\nšŸš€ Quick Start\n1. Create Your Account & Get API Key\n- Sign up at noveum.ai\n- Create a project in your dashboard\n- Generate an API key from the integration page\n- Choose your SDK based on your application language\n2. Install the SDK\nRequirements: Python 3.8+\n3. Initialize the Client\nEnvironment Variables:\nšŸŽÆ Basic Usage\nTrace LLM Calls\nAlternative - Context Manager:\nTrace RAG Pipelines\nšŸ”§ Framework Integrations\nNext.js Integration\nExpress.js Integration\nFastAPI Integration (Python)\nšŸ“Š Advanced Features\nCustom Attributes & Events\nSampling Configuration\nšŸ”— What's Captured Automatically\n- šŸ“Š Performance Metrics: Latency, throughput, error rates\n- šŸ’° Cost Tracking: Token usage, API costs across providers\n- šŸ” Request/Response: Configurable capture of inputs/outputs\n- šŸ·ļø Metadata: Model names, parameters, user context\n- šŸŒŠ Context Flow: Trace relationships across services\n- šŸ› Error Details: Stack traces, error classification\nšŸ“ˆ View Your Data\nOnce integrated, visit your Noveum Dashboard to:\n- šŸ” Search & Filter traces by any attribute\n- šŸ“Š Analyze Performance trends and bottlenecks\n- šŸ’° Monitor Costs across different models and providers\n- šŸ› Debug Issues with detailed trace timelines\n- šŸ‘„ Collaborate with your team on insights\nšŸ”’ Security & Privacy\n- šŸ” Encryption: All data encrypted in transit and at rest\n- šŸŽ›ļø Configurable Capture: Control what data is collected\n- šŸ  Data Residency: Choose your data storage region\n- ā° Retention Control: Set custom data retention policies\nNext Steps\n- Tracing Concepts - Learn about traces, spans, and observability best practices\n- Framework Integrations - Deep dive into specific framework setups\n- Multi-Agent Tracing - Observe complex agent workflows\n- Dashboard Guide - Master the Noveum platform interface\nExclusive Early Access\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year.", + "content_length": 2378, + "internal_links": [], + "scraped_at": 1759935895.715991 + }, + { + "url": "https://noveum.ai/en/docs/getting-started/overview", + "title": "Noveum.ai Overview | Documentation | Noveum.ai", + "content": "Noveum.ai Overview\nComprehensive AI tracing and observability platform for LLM applications, RAG systems, and AI agents\nWelcome to Noveum.aiā€”the comprehensive tracing and observability platform built specifically for AI applications. Whether you're building LLM-powered chatbots, RAG systems, multi-agent workflows, or any AI-driven application, Noveum provides the insights you need to understand, debug, and optimize your systems.\nšŸŽÆ Why AI Applications Need Specialized Observability\nTraditional monitoring tools fall short when it comes to AI applications because they don't understand:\n- šŸ“Š AI-Specific Metrics: Token usage, model costs, prompt effectiveness\n- šŸ”€ Complex Workflows: Multi-step RAG pipelines, agent interactions, tool usage\n- šŸ§  Context Flow: How data moves through embeddings, retrievals, and generations\n- šŸ’° Cost Attribution: Which operations drive your AI spending\n- šŸŽÆ Quality Metrics: Beyond latency - understanding output quality and relevance\nNoveum.ai bridges this gap with purpose-built observability for the AI era.\nšŸš€ Core Platform Components\n1. šŸ Python SDK (noveum-trace\n)\n- Decorator-based tracing for seamless integration\n- Automatic instrumentation for LangChain, LlamaIndex, and OpenAI\n- Async-aware context propagation\n- Production-ready with intelligent sampling and batching\n2. šŸ“˜ TypeScript SDK (@noveum/trace\n)\n- Framework integrations for Next.js, Express.js, Hono\n- TypeScript-first with full type safety\n- Universal compatibility (Node.js, Edge Runtime, browsers)\n- Zero-config automatic instrumentation\n3. šŸ“Š Noveum Platform\n- Real-time dashboard with AI-specific visualizations\n- Advanced search & filtering across traces and spans\n- Cost analysis and optimization recommendations\n- Team collaboration with shared insights and alerts\nšŸ” What Noveum Traces\nLLM Operations\n- Model calls across all providers (OpenAI, Anthropic, Google, etc.)\n- Token usage and cost calculation\n- Prompt engineering effectiveness\n- Response quality metrics\nRAG Pipelines\n- Document retrieval performance and relevance\n- Embedding generation costs and latency\n- Context assembly and prompt construction\n- Answer generation with source attribution\nMulti-Agent Systems\n- Agent interactions and communication patterns\n- Tool usage and external API calls\n- Decision trees and reasoning chains\n- Workflow orchestration across agents\nCustom Operations\n- Business logic specific to your domain\n- External integrations and API calls\n- Data processing pipelines\n- User interactions and session flows\nšŸŽÆ Key Benefits\nšŸ”§ Developer Experience\n- 5-minute setup with minimal code changes\n- Intelligent defaults that work out-of-the-box\n- Rich SDKs with comprehensive documentation\n- Local development support with optional cloud sync\nšŸ“Š Production Insights\n- Real-time monitoring of AI application health\n- Performance optimization with bottleneck identification\n- Cost management with detailed spend analysis\n- Quality assurance through automated alerting\nšŸ”’ Enterprise Ready\n- Security first with end-to-end encryption\n- Compliance support for regulated industries\n- Scalable architecture handling millions of traces\n- Data sovereignty with region-specific storage\nšŸ‘„ Team Collaboration\n- Shared dashboards for cross-functional teams\n- Incident management with trace-based debugging\n- Performance baselines and regression detection\n- Knowledge sharing through trace annotations\nšŸ“ˆ Common Use Cases\nšŸ¤– LLM Application Monitoring\nTrack every aspect of your LLM-powered application:\n- Monitor response quality and user satisfaction\n- Optimize prompt engineering for better results\n- Control costs across different models and providers\n- Debug edge cases and improve error handling\nšŸ” RAG System Optimization\nUnderstand and improve your RAG pipeline:\n- Measure retrieval accuracy and relevance\n- Optimize embedding models and vector search\n- Track context utilization and prompt effectiveness\n- Debug hallucinations and improve grounding\nšŸ¤ Multi-Agent Coordination\nObserve complex agent interactions:\n- Visualize agent communication patterns\n- Track tool usage and external dependencies\n- Optimize workflow efficiency and resource usage\n- Debug coordination failures and deadlocks\nšŸš€ Performance Engineering\nOptimize your AI application performance:\n- Identify slow operations and bottlenecks\n- Right-size models for your workload\n- Implement intelligent caching strategies\n- Scale services based on actual usage patterns\nšŸŽØ Platform Features\nšŸ” Trace Explorer\n- Hierarchical visualization of complex AI workflows\n- Timeline view showing operation sequences\n- Detailed span inspection with all attributes and events\n- Cross-trace correlation for distributed operations\nšŸ’° Cost Analytics\n- Real-time cost tracking across all AI providers\n- Cost attribution by user, feature, or operation\n- Budget alerts and spending forecasts\n- Optimization recommendations for cost reduction\nšŸ“Š Performance Dashboard\n- Latency percentiles and throughput metrics\n- Error rates and failure analysis\n- Model comparison across providers and versions\n- Custom metrics and business KPIs\nšŸšØ Alerting & Monitoring\n- Intelligent alerts based on AI-specific thresholds\n- Anomaly detection for unusual patterns\n- Escalation policies for critical issues\n- Integration with Slack, PagerDuty, and more\nšŸ› ļø Integration Patterns\nIncremental Adoption\nStart small and expand coverage:\n- Single endpoint tracing for immediate value\n- Critical path instrumentation for core workflows\n- Full application coverage for comprehensive insights\n- Advanced features like custom metrics and alerts\nFramework Integration\nNative support for popular frameworks:\n- Next.js with App Router and API routes\n- Express.js and other Node.js frameworks\n- FastAPI and Flask for Python applications\n- Custom integrations for any framework\nCI/CD Integration\nEmbed observability in your development process:\n- Performance regression detection in CI\n- Trace-based testing for quality assurance\n- Deployment monitoring with rollback triggers\n- Feature flag integration for safe releases\nšŸŒŸ Getting Started\nReady to transform your AI application observability? Here's your path:\n- Quick Start - Integrate your first SDK in 5 minutes\n- Tracing Concepts - Learn the fundamentals\n- Framework Guides - Deep dive into your stack\n- Advanced Features - Unlock the full platform potential\nBuilt by developers, for developers. Noveum.ai understands that AI applications are different, and we've designed our platform from the ground up to meet their unique observability needs.\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year.", + "content_length": 6733, + "internal_links": [ + "https://noveum.ai/docs/advanced" + ], + "scraped_at": 1759935896.711361 + }, + { + "url": "https://noveum.ai/en/changelog", + "title": "AI Observability, LLM Evals & Agent Monitoring | Noveum.ai", + "content": "Changelog\nStay up to date with the latest changes in our product.\n2 months ago\n- šŸŽÆ Major Platform Evolution: NovaEval Framework - Launched comprehensive AI model evaluation framework with 20+ built-in scorers including accuracy, RAG metrics, conversational scoring, and specialized agent evaluation tools. Features production-grade deployment on Docker and Kubernetes. (GitHub | PyPI Package)\n- šŸš€ Next-Generation Traces UI - Complete redesign featuring three-pane layout with directory tree navigation, advanced filtering, real-time search, mobile responsiveness, and comprehensive keyboard shortcuts. Includes connection status monitoring and performance optimizations.\n- šŸ—„ļø ClickHouse Integration & BYOD Support - Full ClickHouse telemetry backend with connection pooling, retry mechanisms, advanced monitoring, and bring-your-own-database capabilities. Includes secure configuration management and performance optimizations.\n- šŸ“¦ Noveum Trace SDK - Production-ready Python and TypeScript SDKs for comprehensive AI application tracing. Features automatic instrumentation, multi-agent support, and seamless integration with popular frameworks. (GitHub | PyPI Package)\n- šŸ” Enhanced Security: Encrypted API key storage with support for OpenAI, AWS, Anthropic, and other major providers\n- šŸ“Š Advanced Analytics: Real-time metrics dashboard with cost tracking, error rate monitoring, and performance insights\n- šŸ”§ Developer Experience: Comprehensive integration guides, documentation updates, and SDK examples\n- šŸ—ļø Infrastructure: Kubernetes deployments, Docker optimizations, and scalable architecture improvements\n3 months ago\n- šŸŽØ Landing Page Redesign: Modern, responsive design with improved user experience and conversion optimization\n- šŸ“ˆ Enhanced Dashboard Analytics: Real-time request tracking, latency monitoring, and cost analysis\n- šŸ” Improved Logs Interface: Better search functionality, detailed trace views, and enhanced debugging capabilities\n- šŸ‘„ Team Management: Advanced member management with role-based access control and invitation system\n- šŸ›”ļø Security Improvements: Enhanced API key management and secure credential storage\n4 months ago\n- šŸ“Š Advanced Metrics Collection: Comprehensive telemetry system with custom metrics support\n- āš” Performance Optimizations: Improved database queries and response times across the platform\n- šŸ§© Model Comparison Tools: Side-by-side evaluation capabilities for AI model performance\n- šŸŒ Cross-Platform Compatibility: Enhanced browser support and mobile responsiveness\n- šŸ“ Documentation Updates: Comprehensive guides for integration and best practices\n5 months ago\n- šŸŽ‰ Platform Foundation: Initial release of Noveum.ai AI observability platform\n- šŸš€ Core Features: AI Gateway integration, basic evaluation metrics, and monitoring capabilities\n- šŸ“ˆ Evaluation Jobs: Automated model evaluation with configurable metrics and reporting\n- šŸ”Œ Provider Integrations: Support for major AI providers and custom model deployments", + "content_length": 2960, + "internal_links": [], + "scraped_at": 1759935898.441791 + }, + { + "url": "https://noveum.ai/en/blog/evals-for-ai-agents", + "title": "Evals for AI Agents: What They Are, Why They Matter, and How Noveum.ai Makes Them Practical | Noveum.ai", + "content": "Evals for AI Agents: What They Are, Why They Matter, and How Noveum.ai Makes Them Practical\nAditi Upaddhyay\n9/25/2025\nIf youā€™ve ever shipped an AI agent, youā€™ve probably felt this: it works great in your dev sandbox, but the moment it meets real users, things get messy. Suddenly, your ā€œreliableā€ agent is skipping steps, hallucinating facts, or taking 20 seconds to do something simple.\nThatā€™s where evals come in. Theyā€™re not about passing or failing in the old-school software senseā€”theyā€™re about continuously measuring whether your agent is doing the job you hired it for.\nIn this blog, weā€™ll cover:\n- What evals for AI agents actually mean.\n- Why theyā€™re a must-have (not a ā€œnice-to-haveā€).\n- How Noveum.ai helps you run evals without slowing down your roadmap.\nLetā€™s dive in.\n1. What are Evals for AI Agents?\nThink of evals as health check-ups for your AI agents. Just like you wouldnā€™t assume youā€™re healthy without getting your vitals checked, you shouldnā€™t assume your agent is ā€œgood enoughā€ without running evals.\nIn plain terms\nEvals are structured tests that tell you:\n- Did the agent do what it was supposed to?\n- How well did it do it?\n- Was it safe, accurate, and efficient?\nUnlike traditional QA, which looks for bugs, evals look for quality signalsā€”correctness, completeness, tone, safety, cost, and speed.\nTypes of evals youā€™ll hear about\n- Offline evals - Run before release, on a fixed dataset. Great for testing prompts or comparing models.\n- Online evals - Run in production on real traffic. Crucial for catching drift, edge cases, and \"silent failures.\"\n- End-to-end evals - Test an entire workflow, not just one answer.\n- Span-level evals - Check each step in the traceā€”retrieval, reasoning, tool calls.\n- Safety evals - Spot harmful, biased, or policy-violating answers.\nScoring methods\n- Exact matches for structured answers.\n- Rubrics for tone and reasoning.\n- Panel of LLMs (multiple judges instead of one) to cut down bias.\n- Cost + latency tracking side by side with accuracy.\nšŸ‘‰ In short: evals give you visibility. Without them, youā€™re just trusting your gut.\n2. Why are Evals Necessary for AI Agents?\nLetā€™s be blunt: AI agents donā€™t crash with an error message. They fail quietly.\n- A travel booking agent might ā€œforgetā€ a step in the workflow.\n- A fintech agent might miscalculate fees.\n- A support agent might confidently tell a customer something false.\nThese failures donā€™t show up in logsā€”they show up in churn, complaints, and unexpected cloud bills.\nReal-world challenges evals solve\n- Data drift: Users change how they ask questions. Models can degrade over time.\n- Messy inputs: People paste screenshots, slang, typosā€”stuff you never tested for.\n- Hidden costs: A prompt tweak that looks harmless can double your token spend.\n- Compliance needs: For sensitive domains, you need proof your system is safe.\nThe balance every team faces\nYou want your agent to be:\n- Accurate (so users trust it),\n- Fast (so they donā€™t drop off),\n- Affordable (so margins stay healthy).\nWithout evals, youā€™re flying blind on all three.\nThatā€™s why leading teams now treat evals like CI/CD for AI. Every prompt, model, or tool change gets tested against baselines before going live.\n3. How Noveum.ai Helps You Run Evals (Without the Headache)\nYou could build an eval system yourself. Many teams try. Most underestimate the complexity: trace capture, scoring frameworks, dashboards, alerts, model-judge orchestration, drift detectionā€¦the list goes on.\nNoveum.ai was built to solve this problem end-to-end.\nSee exactly what your agent is doing (NovaTrace SDK)\n- Instrument once, capture everything: inputs, outputs, tokens, latency, tool calls.\n- Visualize every trace as a flow chartā€”see where it struggled or wasted time.\n- Span-level clarity: you donā€™t just see that it failed, you see why.\nScore with fairness and depth (NovaEval)\n- Panel-of-LLMs (patent pending): multiple judges reduce bias.\n- Pre-built scorers: ExactMatch, F1, RAG relevance, Safety.\n- Custom rubrics: Define ā€œgoodā€ in your domain (finance ā‰  healthcare).\n- Mix human review where it matters most.\nTest like you deploy\n- Offline evals for prompt/model experiments.\n- Shadow or canary live traffic to catch issues before full rollout.\n- A/B test prompts, retrieval configs, or model swaps with dashboards that track accuracy + cost + latency.\nOperate with confidence\n- Set thresholds (e.g., accuracy ā‰„85%, p95 latency <2s, cost single-judge bias.\n- Production-first - Canary + shadow testing keep evals alive after launch.\n- Actionable insights - Not just chartsā€”concrete next steps.\n- Scales with your roadmap - From one agent to many, from single prompt to multi-agent orchestration.\nFinal Thoughts\nAI agents are powerful, but theyā€™re unpredictable without the right checks in place. Evals are how you turn that unpredictability into measurable, improvable performance.\nAt Noveum.ai, weā€™re not just giving you dashboards. Weā€™re giving you a system that helps your agents get better week after weekā€”automatically.\nIf youā€™re serious about shipping reliable AI agents, itā€™s time to put evals at the center of your workflow.\nšŸ‘‰ Contact our cofounder at [email protected] or book a call here: https://calendly.com/aditi-noveum\nGet Early Access to Noveum.ai Platform\nJoin the select group of AI teams optimizing their models with our data-driven platform. We're onboarding users in limited batches to ensure a premium experience.", + "content_length": 6415, + "internal_links": [], + "scraped_at": 1759935899.3275049 + }, + { + "url": "https://noveum.ai/en/blog/gpt-oss-vs-gpt-5-vs-gpt-4o-mini-mmlu-evaluation-report", + "title": "GPT-OSS vs GPT-5 vs GPT-4o-mini ā€” MMLU Benchmark Comparison (Accuracy, Runtime, Thinking Modes) | Noveum.ai", + "content": "GPT-OSS vs GPT-5 vs GPT-4o-mini ā€” MMLU Benchmark Comparison (Accuracy, Runtime, Thinking Modes)\nShivam Gupta\n8/13/2025\nGenerated on: 2025-08-13\nThis report is a plain-English summary of how seven model setups perform on the MMLU benchmark across 10 subjects (about 500 questions each). We compare:\n- accuracy (who gets the most answers right)\n- speed/runtime (how fast the models finish)\n- the impact of GPT-OSS ā€œthinking modesā€ (low, medium, high, unspecified) on results and efficiency\nUse this to quickly choose the model or mode that best fits your needsā€”fastest, most accurate, or most consistent.\nTL;DR\n- Most accurate: GPT-5 (91.38%)\n- Best speed/efficiency: GPT-OSS (low thinking mode)\n- Best balance (accuracy + runtime): O3 or GPT-OSS (medium thinking mode)\n1. Overall Model Performance\nThe evaluation tested 7 model configurations across different thinking modes, revealing significant performance variations and the impact of reasoning strategies:\n| Model | Accuracy | Questions | Performance Rank | Thinking Mode |\n|---|---|---|---|---|\n| GPT-5 (OpenAI) | 91.38% | 500 | 1st | unspecified |\n| O3 (OpenAI) | 88.60% | 500 | 2nd | unspecified |\n| GPT-OSS (unspecified) | 88.40% | 500 | 3rd | unspecified |\n| GPT-OSS (medium) | 87.20% | 500 | 4th | medium |\n| GPT-OSS (low) | 84.77% | 499 | 5th | low |\n| GPT-OSS (high) | 83.00% | 500 | 6th | high |\n| GPT-4o-mini (OpenAI) | 74.20% | 500 | 7th | unspecified |\nPerformance Insights:\n- Clear leader: GPT-5 tops accuracy (91.38%)\n- Strong contenders: O3 (88.60%) and GPT-OSS (unspecified: 88.40%)\n- Long tail: GPT-4o-mini trails at 74.20%\n- Gap: 17.18pp between first and last\n2. Subject-Wise Performance Analysis\nThe evaluation covered 10 academic subjects, revealing subject-specific strengths and weaknesses across all models:\nTop Performing Subjects\n- Elementary Mathematics: 95.71% average accuracy\n- College Physics: 94.29% average accuracy\n- Conceptual Physics: 90.57% average accuracy\nMost Challenging Subjects\n- College Chemistry: 63.63% average accuracy\n- College Mathematics: 81.43% average accuracy\n- High School Mathematics: 83.69% average accuracy\nSubject Difficulty Analysis:\n- Hardest: College Chemistry (36.37% difficulty)\n- Easiest: Elementary Mathematics (4.29% difficulty)\n- Pattern: Physics stays strong; advanced math varies by model\nSubject-Specific Performance Patterns:\nAbstract Algebra: GPT-OSS models dominate the top 2 positions, with both high and medium thinking modes achieving 92% accuracy. This suggests that GPT-OSS excels at mathematical reasoning tasks when given appropriate thinking parameters.\nCollege Chemistry: All models struggle, with GPT-5 performing best at 71.43%. The subject shows the highest variability, indicating fundamental challenges in chemical reasoning that persist across all thinking strategies.\nCollege Mathematics: O3 leads with 94% accuracy, followed closely by GPT-5. GPT-OSS models show consistent performance around 88%, suggesting good mathematical capabilities regardless of thinking mode.\nPhysics Subjects: Both college and conceptual physics show strong performance across models, with GPT-OSS low thinking mode achieving perfect scores in college physics, indicating efficient reasoning for physics problems.\nElementary Mathematics: All models perform exceptionally well (94%+), with GPT-5 leading at 98%. This suggests that basic mathematical reasoning is well-handled by all models.\n3. Thinking Mode Effects Analysis\nThe GPT-OSS model was tested across four thinking modes using an A100 40GB GPU, revealing critical insights into the relationship between reasoning depth and performance:\nThinking Mode Performance Comparison\n| Mode | Accuracy | Avg Thinking Tokens | GPU Runtime | Efficiency |\n|---|---|---|---|---|\n| Unspecified | 88.40% | 384.60 Ā± 532.24 | 135.57s | Best Balance |\n| Medium | 87.20% | 390.57 Ā± 570.66 | 138.42s | Most Efficient |\n| Low | 84.77% | 76.55 Ā± 84.97 | 57.49s | Most Runtime-Efficient |\n| High | 83.00% | 1061.48 Ā± 1311.63 | 279.54s | Least Efficient |\nKey Insights:\n- Low mode = fastest good-enough results (best runtime/accuracy tradeoff)\n- Medium mode = balanced choice for most workloads\n- Unspecified = highest accuracy within GPT-OSS, with extra runtime\n- High mode = slowest and often worse; use only for niche deep-reasoning cases\nRuntime-Performance Trade-offs:\n- Low mode: 0.68 seconds per percentage point of accuracy\n- Medium mode: 1.59 seconds per percentage point of accuracy\n- Unspecified mode: 1.53 seconds per percentage point of accuracy\n- High mode: 3.37 seconds per percentage point of accuracy\n4. Token Usage and Efficiency\nEfficiency Curves\n- Low mode: Best at 2ā€“59 tokens (~90%); noticeable drop beyond 170 tokens\n- Medium mode: Broad sweet spot 15ā€“446 tokens (~92.65%); balanced overall\n- High mode: Diminishing returns; accuracy declines past ~2500 tokens\n- Unspecified: Reliable 17ā€“428 tokens (~93.77%); steady decline with complexity\n5. Runtime vs Accuracy\nRuntime-Effectiveness Rankings\n- Low Thinking Mode: 57.49s for 84.77% accuracy\n- Runtime per percentage point: 0.68s\n- Best value proposition for time-sensitive applications\n- Medium Thinking Mode: 138.42s for 87.20% accuracy\n- Runtime per percentage point: 1.59s\n- Good balance of performance and runtime\n- Unspecified Mode: 135.57s for 88.40% accuracy\n- Runtime per percentage point: 1.53s\n- Premium performance at higher runtime cost\n- High Thinking Mode: 279.54s for 83.00% accuracy\n- Runtime per percentage point: 3.37s\n- Least efficient option\nRuntime-Performance Insights:\n- Best ROI: Low mode\n- Best balance: Medium mode\n- Premium accuracy: Unspecified\n- Avoid: High mode unless required\nHow to choose (quick guide)\n- Max accuracy: Choose GPT-5\n- Speed/cost-sensitive: Choose GPT-OSS with thinking mode = low\n- Balanced: Choose O3 or GPT-OSS with thinking mode = medium\n- Chemistry-heavy workloads: Validate on your data; consider domain-tuned models\n6. Consistency (Subject Variance)\nConsistency Rankings (Lower CV = More Consistent)\n| Model | Mean Accuracy | CV | Consistency Rank |\n|---|---|---|---|\n| GPT-5 (OpenAI) | 91.34% | 8.58% | Most Consistent |\n| GPT-OSS Medium | 87.20% | 10.16% | 2nd |\n| GPT-OSS Unspecified | 88.40% | 10.21% | 3rd |\n| O3 (OpenAI) | 88.60% | 10.49% | 4th |\n| GPT-OSS Low | 84.78% | 11.60% | 5th |\n| GPT-OSS High | 83.00% | 11.82% | 6th |\n| GPT-4o-mini | 74.20% | 19.49% | Least Consistent |\nConsistency Insights:\n- GPT-5 shows remarkable consistency across subjects despite high performance\n- GPT-4o-mini exhibits high variability, suggesting subject-specific weaknesses\n- GPT-OSS models show moderate consistency with thinking mode effects\n- O3 model shows competitive performance (88.60%) with moderate consistency, positioning it as a strong alternative to GPT-5\n7. Thinking Tokens vs Accuracy\nThinking Token vs. Accuracy Correlations\nAll GPT-OSS models show negative correlations between thinking tokens and accuracy:\n- High Mode: -0.56 (strongest negative correlation)\n- Unspecified Mode: -0.38 (moderate negative correlation)\n- Medium Mode: -0.27 (moderate negative correlation)\n- Low Mode: Insufficient data for correlation (most responses use minimal tokens)\nInterpretation:\n- More thinking tokens correlate with lower accuracy\n- This suggests that simpler, more direct reasoning lead to better performance\n- Complex reasoning may introduce errors or overthinking\n- The relationship is consistent across all thinking modes for GPT-OSS 20B\n- High thinking mode shows the strongest negative correlation, indicating the most significant performance degradation with increased complexity\n8. Outliers (College Chemistry)\nSubject Outlier: College Chemistry\n- Z-score: -2.5354 (statistically significant outlier)\n- Mean Accuracy: 63.63%\n- Outlier Type: Low performer\nAnalysis:\n- College Chemistry shows significantly lower performance than expected\n- All models struggle with this subject, suggesting fundamental challenges\n- GPT-5 performs best at 71.43%, but even this is not good\n- The subject may require specialized knowledge or reasoning patterns not well-represented in the training data\n- Thinking mode variations show minimal impact, indicating the challenge is fundamental rather than reasoning-strategy dependent\n9. Recommendations\nFor Production Use\n- Runtime-Sensitive Applications: Use GPT-OSS with low thinking mode for best speed-accuracy ratio\n- Performance-Critical Applications: Use GPT-5 for maximum accuracy and consistency\n- Balanced Applications: Use O3 for high performance with moderate runtime, or GPT-OSS with medium thinking mode\n- Enterprise Applications: Consider O3 as a cost-effective alternative to GPT-5 when 88.6% accuracy is sufficient\nFor Research and Development\n- Consistency Studies: Focus on GPT-5 model for stable performance across subjects\n- Efficiency Optimization: Study low thinking mode patterns for runtime reduction\n- Subject-Specific Tuning: Develop specialized models for challenging subjects like Chemistry\n- Thinking Mode Research: Investigate why high thinking modes show performance degradation\nFor Model Selection\n- Academic Applications: Prioritize GPT-5 for comprehensive coverage, with O3 as a strong alternative\n- Resource-Constrained Environments: Choose GPT-OSS low thinking mode\n- Real-Time Applications: Consider medium thinking mode for speed-accuracy balance\n- Enterprise Deployments: O3 offers an excellent balance of performance (88.6%) and runtime efficiency\n10. Limitations and Future Work\nCurrent Limitations\n- All models have consistent sample sizes (approximately 500 questions each)\n- Single evaluation run per model configuration\n- Focus on GPT-OSS thinking modes only\n- Limited subject coverage (10 out of 57 MMLU subjects)\n- Runtime measurements based on A100 40GB GPU performance\nFuture Research Directions\n- Extended Subject Coverage: Evaluate all 57 MMLU subjects\n- Multiple Runs: Assess model consistency across multiple evaluations\n- Thinking Mode Optimization: Develop adaptive thinking mode selection based on problem complexity\n- Runtime Analysis: Include latency and throughput metrics across different GPU configurations\n- Cross-Model Comparison: Evaluate thinking modes across different model architectures\n- Performance Degradation Study: Investigate why high thinking modes show worse performance\nConclusion\nThis comprehensive MMLU evaluation provides critical insights into model performance, efficiency, and runtime characteristics across different thinking strategies. Key takeaways include:\n- GPT-5 remains the performance leader with excellent consistency (91.38% accuracy, 8.58% CV)\n- O3 emerges as a strong competitor with 88.6% accuracy, offering enterprise-grade performance\n- GPT-OSS offers excellent runtime-performance ratios across different thinking modes\n- Low thinking mode provides the best value for GPU runtime (57.49s for 84.77% accuracy)\n- Subject difficulty varies significantly, with Chemistry being most challenging (36.37% difficulty)\n- Model consistency varies widely, with GPT-5 showing remarkable stability\n- Thinking token efficiency shows that simpler reasoning often leads to better performance\n- High thinking modes show performance degradation, challenging the assumption that more complex reasoning improves results\nKey takeaways:\n- GPT-5 leads in accuracy and consistency (91.38%, CV 8.58%)\n- O3 is a strong, fast alternative (88.6%)\n- GPT-OSS (low mode) delivers the best speed/accuracy tradeoff\n- Chemistry is hardest; validate domain-specific workloads\n- More tokens ā‰  better: excessive thinking often reduces accuracy\nPractical implication: prefer simpler, focused reasoning by default; scale up thinking depth only when problems demand it.\nReport generated from MMLU evaluation data covering 7 model configurations, 4 thinking modes, and 10 academic subjects across approximately 500 total samples per model. GPU runtime measurements based on A100 40GB GPU performance. Analysis generated on 2025-08-13.\nGet Early Access to Noveum.ai Platform\nJoin the select group of AI teams optimizing their models with our data-driven platform. We're onboarding users in limited batches to ensure a premium experience.", + "content_length": 12117, + "internal_links": [], + "scraped_at": 1759935900.21145 + }, + { + "url": "https://noveum.ai/en/blog/comprehensive-mmlu-evaluation-analysis-report", + "title": "o1-mini vs gpt-4o-mini ā€” What We Learned from 1,000 MMLU Samples | Noveum.ai", + "content": "o1-mini vs gpt-4o-mini ā€” What We Learned from 1,000 MMLU Samples\nShashank Agarwal\n8/12/2025\no1-mini vs gpt-4o-mini: MMLU math comparison\nWe compared Azure o1-mini and gpt-4o-mini on 1,000 MMLU math questions across 10 subjects using NovaEval. The goal was simple: learn when paying more for o1ā€‘mini actually makes sense.\nKey takeaways\n- o1ā€‘mini: 73.3% vs gptā€‘4oā€‘mini: 57.2% (+16.1pp, p < 0.001)\n- Similar throughput and reliability for both models\n- o1ā€‘mini costs ~15Ɨ more per request; payoff depends on the cost of a wrong answer\nHow we tested\n- Dataset: 1,000 MMLU math questions; 809 processed per model after filtering/timeouts\n- Setup: Azure OpenAI deployments, identical prompts, seed 42, 10 workers\n- Framework: NovaEval with retries, checkpointing, and consistent answer extraction\nDetailed Results\nOverall Performance Metrics\n| Metric | gpt-4o-mini | o1-mini | Difference |\n|---|---|---|---|\n| Accuracy | 57.2% | 73.3% | +16.1% |\n| Correct Answers | 463/809 | 593/809 | +130 |\n| Processing Speed | 2.10 samples/sec | 2.27 samples/sec | +0.17 |\n| Total Time | 385.8 seconds | 357.0 seconds | -28.8s |\n| Error Rate | 0.2% (2 errors) | 0.1% (1 error) | -0.1% |\n| Status | Partial errors | Partial errors | - |\nStatistical Significance Analysis\n- Z-score: 5.24 (highly significant)\n- P-value: < 0.001 (extremely significant)\n- Effect Size (Cohen's h): 0.334 (Medium effect)\n- 95% Confidence Interval: [10.0%, 22.2%]\n- Statistical Power: > 99% (highly powered study)\nThe results demonstrate statistically significant and practically meaningful performance differences between the models.\nSubject-wise Performance Analysis\nPerformance by Mathematical Domain\n| Subject | gpt-4o-mini | o1-mini | Difference | Significant* |\n|---|---|---|---|---|\n| Abstract Algebra | 36.4% | 58.2% | +21.8% | āœ“ |\n| College Mathematics | 37.3% | 67.8% | +30.5% | āœ“ |\n| College Physics | 49.2% | 71.2% | +22.0% | āœ“ |\n| College Chemistry | 52.5% | 74.6% | +22.1% | āœ“ |\n| Conceptual Physics | 80.0% | 81.0% | +1.0% | āœ— |\n| Elementary Mathematics | 80.0% | 89.0% | +9.0% | āœ— |\n| High School Chemistry | 55.9% | 76.3% | +20.4% | āœ“ |\n| High School Mathematics | 40.7% | 79.7% | +39.0% | āœ“ |\n| High School Physics | 54.2% | 74.6% | +20.4% | āœ“ |\n| High School Statistics | 55.9% | 61.0% | +5.1% | āœ— |\n- Statistical significance at p < 0.05 level\nKey Subject Insights\n- Largest Performance Gap: High School Mathematics (39.0% difference)\n- Most Consistent Performance: Elementary Mathematics (both models > 80%)\n- Significant Improvements: 7 out of 10 subjects show statistically significant gains\n- College-level Advantage: o1-mini shows particularly strong performance in college-level subjects\nConfidence and Quality Analysis\nAnswer Extraction Confidence\n| Model | Overall Confidence | Correct Answers | Incorrect Answers |\n|---|---|---|---|\n| gpt-4o-mini | 0.859 | 0.859 | 0.859 |\n| o1-mini | 0.859 | 0.859 | 0.859 |\nBoth models demonstrate identical confidence patterns, suggesting consistent answer extraction methodology across different model architectures.\nResponse Quality Metrics\n- Average Response Length:\n- gpt-4o-mini: ~800 characters\n- o1-mini: ~850 characters\n- Extraction Success Rate: 99.9% for both models\n- Processing Reliability: > 99.8% success rate\nTechnical Implementation Details\nNovaEval Integration\nThe evaluation utilized a custom NovaEval implementation with the following components:\n- Azure OpenAI Client: Custom integration supporting both models\n- Answer Extraction: Multi-pattern regex-based extraction with confidence scoring\n- Concurrent Processing: 10-worker parallel processing for efficiency\n- Error Handling: Robust retry logic and checkpoint saving\nAnswer Extraction Methodology\n# Primary extraction patterns (in order of confidence)\npatterns = [\nr'^([A-D])', # Letter at start (confidence: 1.0)\nr'\\b([A-D])\\b', # Single letter (confidence: 0.9)\nr'answer\\s*is\\s*([A-D])', # \"answer is X\" (confidence: 0.8)\nr'([A-D])\\.', # Letter with period (confidence: 0.7)\nr'\\(([A-D])\\)', # Letter in parentheses (confidence: 0.6)\n# ... additional patterns with decreasing confidence\n]\nQuality Assurance\n- Reproducibility: Fixed random seed (42) ensures consistent results\n- Validation: Cross-validation with manual spot-checks\n- Error Monitoring: Real-time error tracking and logging\n- Checkpoint System: Regular saves prevent data loss\nCost Analysis\n- o1-mini costs 15x more than gpt-4o-mini ($0.001720 vs $0.000115 per request)\n- Break-even point: ~$0.01 per incorrect answer makes o1-mini cost-effective\n- ROI threshold: Applications where errors cost ā‰„$0.01 show positive ROI\n- Performance premium: 28% relative accuracy improvement for ~1,396% cost increase\nCost Breakdown Analysis\nTotal Evaluation Costs\n| Model | Total Cost | Cost per Request | Cost per Correct Answer | Accuracy |\n|---|---|---|---|---|\n| gpt-4o-mini | $0.0928 | $0.000115 | $0.000201 | 57.2% |\n| o1-mini | $1.3912 | $0.001720 | $0.002346 | 73.3% |\n| Difference | +$1.2984 | +$0.001605 | +$0.002145 | +16.1% |\nToken Usage Economics\ngpt-4o-mini Token Analysis:\n- Total Input Tokens: 154,036 (80.3% of total)\n- Total Output Tokens: 37,782 (19.7% of total)\n- Total Tokens: 191,818\n- Input Cost Share: 37.2% ($0.0345)\n- Output Cost Share: 62.8% ($0.0583)\no1-mini Token Analysis:\nToken breakdown omitted pending verified usage data. We will update this section with non-estimated counts to avoid artifacts.\nNote: Token estimation based on 1 token ā‰ˆ 4 characters approximation\nBreak-Even Analysis\nCritical Break-Even Thresholds\nPrimary Break-Even Point: ~$0.01 per incorrect answer\n- With a per-request cost delta of $0.001605 and an accuracy delta of 0.161, the break-even error cost is: break-even ā‰ˆ 0.001605 / 0.161 ā‰ˆ $0.00997 per error (ā‰ˆ $0.01). Example (10,000 requests): +1,610 additional correct answers at an added cost of ~$16.05.\nROI and Net Benefit (10K requests)\n- Net Benefit(10K, error_cost = C) ā‰ˆ 1,610 Ɨ C - $16.05\n- Examples:\n- C = $0.01 ā†’ Net ā‰ˆ $0.05 (ā‰ˆ break-even)\n- C = $1.00 ā†’ Net ā‰ˆ $1,593.95\n- C = $10.00 ā†’ Net ā‰ˆ $16,083.95\nVolume-Based Break-Even Analysis\nScaling Economics (Cost Difference):\n| Request Volume | gpt-4o-mini Total Cost | o1-mini Total Cost | Cost Difference | Additional Correct |\n|---|---|---|---|---|\n| 100 | $0.01 | $0.17 | +$0.16 | 16 |\n| 1,000 | $0.12 | $1.72 | +$1.60 | 161 |\n| 10,000 | $1.15 | $17.20 | +$16.05 | 1,610 |\n| 100,000 | $11.50 | $172.00 | +$160.50 | 16,100 |\n| 1,000,000 | $115.00 | $1,720.00 | +$1,605.00 | 161,000 |\nUse Case Recommendations\nWhen o1-mini is Cost-Effective\nHigh-Value Applications (ROI > 100%)\n- Financial Trading Systems: Error cost $100-$10,000 per mistake\n- Medical Diagnosis Support: Error cost $1,000-$100,000 per mistake\n- Legal Document Analysis: Error cost $500-$50,000 per mistake\n- Quality Control Systems: Error cost $10-$1,000 per mistake\n- Critical Decision Support: Error cost $100-$10,000 per mistake\nMedium-Value Applications (ROI 25-100%)\n- Academic Research: Error cost $10-$100 per mistake\n- Business Intelligence: Error cost $50-$500 per mistake\n- Content Moderation: Error cost $1-$50 per mistake\n- Automated Grading: Error cost $5-$100 per mistake\nWhen gpt-4o-mini is Preferred\nCost-Sensitive Applications\n- Bulk Content Processing: High volume, low error cost\n- Development and Testing: Non-production environments\n- Exploratory Analysis: Initial data exploration\n- Non-Critical Evaluations: Low-stakes decision making\n- Budget-Constrained Projects: Limited financial resources\nSubject-Wise Cost Analysis\nCost Efficiency by Mathematical Domain\nTop Performing Subjects (o1-mini Cost Efficiency):\n| Subject | Accuracy | Cost per Request | Cost per Correct | Efficiency Score* |\n|---|---|---|---|---|\n| Elementary Mathematics | 89.0% | $0.00172 | $0.00193 | 517 |\n| Conceptual Physics | 81.0% | $0.00172 | $0.00212 | 471 |\n| High School Mathematics | 79.7% | $0.00172 | $0.00216 | 463 |\n| High School Physics | 74.6% | $0.00172 | $0.00231 | 434 |\n| College Chemistry | 74.6% | $0.00172 | $0.00231 | 434 |\n- Efficiency Score = (Accuracy / Cost per Request) Ɨ 1000\nHighest Cost Premium Subjects:\n- High School Mathematics: 39.0% accuracy improvement, highest ROI\n- College Mathematics: 30.5% accuracy improvement, strong value\n- Abstract Algebra: 21.8% accuracy improvement, solid gains\nSubject-Specific Break-Even Analysis\nPremium Justified Subjects (Error cost threshold < $1.00):\n- High School Mathematics: $0.41 per error\n- College Mathematics: $0.53 per error\n- Abstract Algebra: $0.74 per error\nPremium Questionable Subjects (Error cost threshold > $2.00):\n- Conceptual Physics: $2.15 per error\n- Elementary Mathematics: $1.89 per error\n- High School Statistics: $3.21 per error\nStatistical Validation\nHypothesis Testing\nNull Hypothesis (Hā‚€): No difference in accuracy between gpt-4o-mini and o1-mini Alternative Hypothesis (Hā‚): o1-mini has higher accuracy than gpt-4o-mini\nTest Results:\n- Z-statistic: 5.24\n- Critical value: 1.96 (Ī± = 0.05)\n- Decision: Reject Hā‚€ (5.24 > 1.96)\n- Conclusion: Strong evidence for superior o1-mini performance\nEffect Size Interpretation\nCohen's h = 0.334 indicates a Medium Effect Size:\n- Small effect: h < 0.2\n- Medium effect: 0.2 ā‰¤ h < 0.5\n- Large effect: h ā‰„ 0.5\nConfidence Intervals\nThe 95% confidence interval [10.0%, 22.2%] for the accuracy difference indicates:\n- Lower bound: o1-mini is at least 10.0% better\n- Upper bound: o1-mini could be up to 22.2% better\n- Point estimate: 16.1% improvement is most likely\nBusiness and Practical Implications\nModel Selection Guidance\nWhen to Choose o1-mini:\n- Mathematical reasoning tasks (73.3% accuracy advantage)\n- College-level problem solving (strong performance across all college subjects)\n- High-stakes applications where accuracy is paramount\n- Complex analytical tasks requiring step-by-step reasoning\nWhen to Consider gpt-4o-mini:\n- Cost-sensitive applications (if pricing differs significantly)\n- Simple mathematical tasks where 57.2% accuracy is sufficient\n- High-throughput scenarios where speed matters more than accuracy\n- General-purpose applications beyond mathematical reasoning\nPerformance-Cost Analysis\nBased on the evaluation results:\n- Performance Gain: 28% relative improvement (73.3% vs 57.2%)\n- Speed Difference: Minimal (2.27 vs 2.10 samples/sec)\n- Reliability: Comparable error rates (< 0.2% for both)\n- ROI Calculation: Depends on specific use case and pricing structure\nRisk Assessment\nLow Risk Factors:\n- Consistent performance across multiple mathematical domains\n- Statistical significance with large sample size (809 samples per model)\n- Reproducible results with proper methodology\n- Robust evaluation framework with error handling\nConsiderations:\n- Domain specificity: Results specific to mathematical reasoning\n- Sample representation: Limited to MMLU mathematical subjects\n- Model versions: Results tied to specific model deployments\nRecommendations\nImmediate Actions\n- Deploy o1-mini for mathematical applications with confidence\n- Implement A/B testing for specific use cases to validate results\n- Monitor performance in production environments\n- Establish quality metrics based on confidence scoring\nStrategic Considerations\n- Expand evaluation to other MMLU domains (science, humanities, etc.)\n- Conduct cost-benefit analysis based on actual pricing\n- Develop hybrid approaches leveraging strengths of both models\n- Create performance benchmarks for ongoing model evaluation\nQuality Assurance Framework\n- Confidence Thresholding: Flag responses with confidence < 0.7\n- Subject-specific Monitoring: Track performance by mathematical domain\n- Error Pattern Analysis: Identify and address systematic failures\n- Continuous Evaluation: Regular re-assessment with new model versions\nPrimary Findings\nThis comprehensive evaluation provides definitive evidence that o1-mini significantly outperforms gpt-4o-mini on mathematical reasoning tasks:\n- Substantial Accuracy Improvement: 16.1 percentage point gain (28% relative improvement)\n- Statistical Significance: p < 0.001 with large effect size\n- Broad Applicability: Consistent improvements across 7/10 mathematical subjects\n- Production Readiness: Reliable performance with minimal error rates\nScientific Rigor\nThe evaluation meets high standards for scientific rigor:\n- Large sample size (1,000 samples, 809 processed per model)\n- Proper statistical testing with appropriate methods\n- Reproducible methodology with documented procedures\n- Comprehensive analysis including effect sizes and confidence intervals\nBusiness Impact\nFor organizations requiring mathematical reasoning capabilities:\n- Clear model choice: o1-mini provides superior performance\n- Quantified benefits: 28% relative improvement in accuracy\n- Risk mitigation: Statistically validated results reduce deployment risk\n- Strategic advantage: Early adoption of superior reasoning capabilities\nFuture Directions\nThis evaluation establishes a gold standard methodology for model comparison and provides a foundation for future research into reasoning model capabilities. The results strongly support the adoption of o1-mini for mathematical reasoning applications while highlighting the importance of rigorous evaluation in model selection decisions\nGet Early Access to Noveum.ai Platform\nJoin the select group of AI teams optimizing their models with our data-driven platform. We're onboarding users in limited batches to ensure a premium experience.", + "content_length": 13352, + "internal_links": [], + "scraped_at": 1759935901.217552 + }, + { + "url": "https://noveum.ai/en/blog/from-logs-to-intelligent-choices-inside-noveum-ais-evaluation-process", + "title": "From Development to Production - Inside Noveum.ai's AI Observability Platform | Noveum.ai", + "content": "From Development to Production - Inside Noveum.ai's AI Observability Platform\nShashank Agarwal\n3/3/2025\nIntroduction\nEvery AI application tells a storyā€”but most developers never get to hear it. When your RAG pipeline returns irrelevant results, when your multi-agent system gets stuck in loops, or when your LLM costs suddenly spike, you're left guessing what went wrong and where.\nThat's exactly the challenge Noveum.ai solves. Rather than flying blind, our platform provides comprehensive tracing and observability specifically designed for AI applications. Whether you're building LLM-powered chatbots, RAG systems, or complex multi-agent workflows, Noveum.ai gives you the insights you need to understand, debug, and optimize your AI applications.\nThe AI Observability Challenge\nWhy Traditional Monitoring Falls Short\nTraditional application monitoring tools weren't built for AI applications. They can tell you if your API is responding, but they can't answer the questions that matter most for AI systems:\n- Why did my RAG pipeline retrieve irrelevant documents?\n- Which LLM calls are driving my costs?\n- How are my agents communicating with each other?\n- What's causing hallucinations in my responses?\n- Why is my embedding generation so slow?\nWhat Makes AI Applications Different\nAI applications have unique characteristics that require specialized observability:\n- šŸ§  Context Flow: Data flows through embeddings, retrievals, and generations\n- šŸ’° Variable Costs: Token usage creates unpredictable expenses\n- šŸ”€ Complex Workflows: Multi-step pipelines with branching logic\n- šŸ¤– Agent Interactions: Multiple AI entities coordinating tasks\n- šŸ“Š Quality Metrics: Success isn't just about uptimeā€”it's about output quality\nStep 1: SDK Integration\nEffortless Instrumentation\nAt the heart of Noveum.ai are our Python and TypeScript SDKs that integrate seamlessly into your existing codebase. With just a few lines of code, you can start capturing comprehensive traces of your AI operations.\nimport noveum_trace\n# Initialize once at startup\nnoveum_trace.init(\napi_key=\"your-api-key\",\nproject=\"customer-support-bot\",\nenvironment=\"production\"\n)\n# Trace LLM calls automatically\n@noveum_trace.trace_llm\ndef generate_response(user_question: str) -> str:\nreturn openai.chat.completions.create(\nmodel=\"gpt-4\",\nmessages=[{\"role\": \"user\", \"content\": user_question}]\n).choices[0].message.content\nWhat Gets Captured Automatically\nOnce integrated, Noveum.ai automatically captures:\n- šŸ” Request/Response Data: Inputs, outputs, and transformations\n- ā±ļø Performance Metrics: Latency, throughput, and bottlenecks\n- šŸ’° Cost Tracking: Token usage and API costs across providers\n- šŸ·ļø Rich Metadata: Model parameters, user context, and custom attributes\n- šŸŒŠ Context Flow: How data moves through your AI pipeline\n- šŸ› Error Details: Stack traces and failure analysis\nStep 2: Understanding Your AI Workflows\nRAG Pipeline Visibility\nRAG (Retrieval-Augmented Generation) systems involve multiple complex steps. Noveum.ai traces each phase, giving you complete visibility:\n@noveum_trace.trace(\"rag-pipeline\")\ndef answer_customer_question(question: str) -> str:\n# Phase 1: Query understanding\nwith noveum_trace.trace_step(\"query-analysis\") as step:\nintent = analyze_query_intent(question)\nstep.set_attribute(\"query.intent\", intent)\nstep.set_attribute(\"query.complexity\", get_complexity_score(question))\n# Phase 2: Document retrieval\nwith noveum_trace.trace_step(\"document-retrieval\") as step:\nembeddings = generate_embeddings(question)\ndocuments = vector_search(embeddings, k=5)\nstep.set_attribute(\"documents.retrieved\", len(documents))\nstep.set_attribute(\"documents.avg_similarity\", avg_similarity(documents))\nstep.set_attribute(\"retrieval.model\", \"text-embedding-ada-002\")\n# Phase 3: Answer generation\nwith noveum_trace.trace_step(\"answer-generation\") as step:\ncontext = build_context(documents)\nanswer = generate_answer_with_context(question, context)\nstep.set_attribute(\"context.length\", len(context))\nstep.set_attribute(\"answer.confidence\", calculate_confidence(answer))\nstep.set_attribute(\"generation.model\", \"gpt-4\")\nreturn answer\nMulti-Agent Coordination\nWhen multiple AI agents work together, Noveum.ai tracks their interactions and coordination:\nconst multiAgentTask = trace('customer-inquiry-resolution', async (inquiry: string) => {\n// Agent 1: Classification\nconst category = await span('classify-inquiry', async (spanInstance) => {\nspanInstance.setAttribute('agent.name', 'classifier');\nspanInstance.setAttribute('inquiry.length', inquiry.length);\nreturn await classificationAgent.categorize(inquiry);\n});\n// Agent 2: Research (if needed)\nlet context = null;\nif (category.needsResearch) {\ncontext = await span('research-context', async (spanInstance) => {\nspanInstance.setAttribute('agent.name', 'researcher');\nspanInstance.setAttribute('research.category', category.type);\nreturn await researchAgent.gatherContext(inquiry);\n});\n}\n// Agent 3: Response generation\nconst response = await span('generate-response', async (spanInstance) => {\nspanInstance.setAttribute('agent.name', 'responder');\nspanInstance.setAttribute('response.has_context', !!context);\nreturn await responseAgent.generate(inquiry, context);\n});\nreturn response;\n});\nStep 3: Real-Time Debugging and Optimization\nPerformance Bottleneck Identification\nNoveum.ai's dashboard automatically identifies performance issues:\n- šŸŒ Slow Operations: Which LLM calls or embeddings are taking too long?\n- šŸ”„ Redundant Processing: Are you generating the same embeddings multiple times?\n- šŸ“Š Resource Usage: Which operations consume the most tokens or memory?\n- šŸšØ Error Patterns: What types of failures occur most frequently?\nCost Optimization Insights\nWith detailed cost tracking, you can optimize your AI spending:\n- Provider Comparison: See actual costs across OpenAI, Anthropic, Google, etc.\n- Model Analysis: Compare performance vs. cost for different models\n- Usage Patterns: Identify expensive operations and optimize them\n- Budget Alerts: Get notified when costs exceed thresholds\nQuality Assurance\nBeyond performance, Noveum.ai helps ensure output quality:\n- Response Analysis: Track confidence scores and quality metrics\n- A/B Testing: Compare different models or prompts\n- User Feedback: Correlate user satisfaction with trace data\n- Drift Detection: Identify when model performance degrades\nReal-World Example: Customer Support Bot\nLet's walk through a real example. You've built a customer support bot using RAG that helps users with product questions. Here's how Noveum.ai provides insights:\nDevelopment Phase\nDuring development, you discover through tracing that:\n- Embedding generation takes 200ms on average\n- Vector search finds relevant documents 85% of the time\n- Answer generation costs $0.02 per query with GPT-4\nProduction Deployment\nIn production, Noveum.ai reveals:\n- Peak usage occurs during business hours, causing latency spikes\n- Certain question types consistently retrieve irrelevant documents\n- Token usage is 30% higher than expected due to verbose context\nOptimization Cycle\nBased on these insights, you:\n- Cache embeddings for common questions (reduces latency by 60%)\n- Improve vector search by fine-tuning similarity thresholds\n- Switch to GPT-3.5 for simple questions (reduces costs by 40%)\n- Implement streaming for better user experience\nContinuous Improvement\nAs your bot evolves:\n- New conversation patterns are automatically captured\n- Quality metrics help identify areas for improvement\n- Cost trends inform capacity planning\n- Error analysis guides bug fixes and feature development\nAdvanced Observability Patterns\nCustom Metrics and Attributes\nNoveum.ai allows you to add domain-specific insights:\n@noveum_trace.trace(\"content-moderation\")\ndef moderate_content(text: str, user_id: str):\n# Add business context\nnoveum_trace.set_attribute(\"user.trust_level\", get_user_trust_level(user_id))\nnoveum_trace.set_attribute(\"content.category\", classify_content_type(text))\nnoveum_trace.set_attribute(\"moderation.policy_version\", \"v2.1\")\n# Perform moderation\nresult = run_content_moderation(text)\n# Add results\nnoveum_trace.set_attribute(\"moderation.risk_score\", result.risk_score)\nnoveum_trace.set_attribute(\"moderation.action_taken\", result.action)\nreturn result\nError Tracking and Alerting\nComprehensive error handling with actionable insights:\nconst processDocument = trace('document-processing', async (documentId: string) => {\ntry {\nconst result = await span('extract-text', async () => {\nreturn await extractTextFromDocument(documentId);\n});\nreturn await span('analyze-content', async (spanInstance) => {\nspanInstance.setAttribute('document.word_count', result.wordCount);\nspanInstance.setAttribute('document.language', result.language);\nreturn await analyzeContent(result.text);\n});\n} catch (error) {\n// Rich error context for debugging\nconst currentSpan = getCurrentSpan();\ncurrentSpan.setAttribute('error.type', error.constructor.name);\ncurrentSpan.setAttribute('error.message', error.message);\ncurrentSpan.setAttribute('error.recoverable', isRecoverableError(error));\ncurrentSpan.setStatus('ERROR', error.message);\nthrow error;\n}\n});\nThe Future of AI Observability\nWhat's Coming Next\nAs AI applications become more sophisticated, observability needs to evolve:\n- šŸ¤– Agent Ecosystems: Observing complex multi-agent societies\n- šŸ§  Reasoning Chains: Tracing LLM thought processes\n- šŸ”„ Feedback Loops: Connecting user outcomes back to traces\n- šŸ“Š Quality Metrics: Advanced measures of AI output quality\n- šŸ›”ļø Safety Monitoring: Detecting harmful or biased outputs\nBuilding Observability-First AI Applications\nThe future belongs to teams who build observability into their AI applications from day one:\n- Faster Debugging: Find and fix issues before they impact users\n- Data-Driven Optimization: Make decisions based on real usage patterns\n- Proactive Monitoring: Catch problems before they become incidents\n- Continuous Improvement: Use traces to guide development priorities\nConclusion\nThe power of Noveum.ai lies in its comprehensive observability approach:\n- šŸš€ Easy Integration: Start tracing with minimal code changes\n- šŸ” Deep Insights: Understand every aspect of your AI workflows\n- šŸ“Š Actionable Analytics: Make data-driven optimization decisions\n- šŸ› ļø Developer-Friendly: Built by AI engineers, for AI engineers\nEvery API call, every embedding generation, every agent interaction becomes part of a bigger picture that helps you build better AI applications. Instead of guessing why something went wrong, you have concrete data and detailed traces to guide your decisions.\nWhether you're debugging a complex RAG pipeline, optimizing multi-agent coordination, or simply trying to reduce your AI costs, Noveum.ai provides the visibility you need to succeed.\nReady to see your AI applications like never before? Start tracing today or talk to our team about your specific observability needs. We're here to help you build AI applications that are transparent, optimized, and reliableā€”one trace at a time.\nGet Early Access to Noveum.ai Platform\nJoin the select group of AI teams optimizing their models with our data-driven platform. We're onboarding users in limited batches to ensure a premium experience.", + "content_length": 11174, + "internal_links": [], + "scraped_at": 1759935902.2391648 + }, + { + "url": "https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform", + "title": "Noveum.ai - Comprehensive AI Tracing and Observability Platform | Noveum.ai", + "content": "Noveum.ai - Comprehensive AI Tracing and Observability Platform\nShashank Agarwal\n3/2/2025\nIntroduction\nArtificial Intelligence applications are becoming increasingly complex, with multi-step workflows, RAG pipelines, and sophisticated agent systems. But how do you debug when things go wrong? How do you optimize performance? How do you understand what's happening inside your AI applications? That's where Noveum.ai comes in.\nIn this post, I'll walk you through what Noveum.ai is, how it works, and why I believe it's essential for teams building production AI applications. My name is Shashank Agarwalā€”founder of Noveum.ai, AI enthusiast, and someone who's been building and scaling large AI/ML platforms for over a decade. Let's dive in.\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, vector searches, data retrieval, agent reasoning, and more. Without proper observability, debugging becomes a nightmare, optimization is guesswork, and understanding user interactions is nearly impossible.\nNoveum.ai solves this by providing comprehensive tracing and observability specifically designed for AI applications.\nKey benefits of using Noveum.ai:\n- Complete Visibility: Trace every step of your LLM calls, RAG pipelines, and agent workflows\n- Easy Integration: Simple SDKs for Python and TypeScript with minimal code changes\n- Multi-Agent Support: Built-in support for complex multi-agent systems and workflows\n- Performance Insights: Detailed metrics on latency, token usage, costs, and success rates\n- Error Tracking: Automatic error capture and analysis for faster debugging\n- Framework Agnostic: Works with any LLM provider, framework, or architecture\nHow It Works\nStep 1: Install and Initialize the SDK\nNoveum.ai provides native SDKs for both Python and TypeScript applications. Getting started takes just minutes:\nPython SDK\npip install noveum-trace\nimport noveum_trace\n# Initialize the SDK\nnoveum_trace.init(\napi_key=\"your-noveum-api-key\",\nproject=\"my-ai-application\",\nenvironment=\"production\"\n)\nTypeScript SDK\nnpm install @noveum/trace\nimport { initializeClient } from '@noveum/trace';\nconst client = initializeClient({\napiKey: \"your-noveum-api-key\",\nproject: \"my-ai-application\",\nenvironment: \"production\",\n});\nStep 2: Add Tracing to Your Code\nWith Noveum.ai, adding comprehensive tracing to your AI applications is as simple as adding decorators or function calls:\nPython Examples\nBasic LLM Tracing:\n@noveum_trace.trace_llm\ndef call_openai(prompt: str) -> str:\nclient = openai.OpenAI()\nresponse = client.chat.completions.create(\nmodel=\"gpt-4\",\nmessages=[{\"role\": \"user\", \"content\": prompt}]\n)\nreturn response.choices[0].message.content\nMulti-Agent Workflows:\n@noveum_trace.trace_agent(agent_id=\"orchestrator\")\ndef orchestrate_workflow(task: str) -> dict:\n# Coordinate multiple agents\nresearch_result = research_agent(task)\nanalysis_result = analysis_agent(research_result)\nreturn synthesis_agent(research_result, analysis_result)\n@noveum_trace.trace_agent(agent_id=\"researcher\")\ndef research_agent(task: str) -> dict:\n# Research implementation with automatic tracing\nreturn {\"data\": \"...\", \"sources\": [...]}\nRAG Pipeline Tracing:\n@noveum_trace.trace_retrieval\ndef retrieve_documents(query: str) -> list:\n# Vector search implementation\nreturn vector_db.search(query)\n@noveum_trace.trace\ndef rag_pipeline(user_query: str) -> str:\ndocuments = retrieve_documents(user_query)\ncontext = prepare_context(documents)\nreturn generate_response(user_query, context)\nTypeScript Examples\nBasic Tracing:\nconst result = await trace('user-query-processing', async (traceInstance) => {\ntraceInstance.setAttribute('user.id', userId);\ntraceInstance.setAttribute('query.type', 'search');\n// Create spans for sub-operations\nconst embeddings = await span('generate-embeddings', async () => {\nreturn await openai.embeddings.create({\nmodel: 'text-embedding-ada-002',\ninput: userQuery,\n});\n});\nconst searchResults = await span('vector-search', async () => {\nreturn await vectorDB.search(embeddings.data[0].embedding);\n});\nreturn searchResults;\n});\nNext.js Integration:\n// app/api/chat/route.ts\nimport { withNoveumTracing } from '@noveum/trace/integrations/nextjs';\nexport const POST = withNoveumTracing(\nasync (request: NextRequest) => {\nconst { message } = await request.json();\nconst response = await processMessage(message);\nreturn NextResponse.json(response);\n},\n{\nclient,\nspanName: 'chat-completion',\ncaptureRequest: true,\n}\n);\nStep 3: Automatic Data Collection\nOnce integrated, Noveum.ai automatically captures:\n- Request/Response Data: Complete LLM prompts, responses, and parameters\n- Performance Metrics: Latency, token usage, throughput, and costs\n- Error Information: Stack traces, error messages, and failure patterns\n- Agent Interactions: Multi-agent communication and coordination patterns\n- Custom Attributes: Any additional context you want to track\nRich Metadata Example\n{\n\"trace_id\": \"trace_abc123\",\n\"span_id\": \"span_def456\",\n\"operation\": \"llm_completion\",\n\"model\": \"gpt-4\",\n\"provider\": \"openai\",\n\"duration_ms\": 1250,\n\"token_usage\": {\n\"input_tokens\": 150,\n\"output_tokens\": 75,\n\"total_tokens\": 225\n},\n\"cost\": {\n\"input_cost\": 0.0045,\n\"output_cost\": 0.0075,\n\"total_cost\": 0.012\n},\n\"attributes\": {\n\"user_id\": \"user_123\",\n\"session_id\": \"session_456\",\n\"query_type\": \"search\"\n},\n\"status\": \"success\"\n}\nStep 4: Analyze and Debug with the Dashboard\nNoveum.ai provides powerful dashboards to help you:\n- Trace Visualization: See complete request flows through your AI pipelines\n- Performance Analytics: Identify bottlenecks and optimization opportunities\n- Cost Analysis: Track spending across models, users, and features\n- Error Investigation: Quickly identify and debug issues in production\n- Agent Behavior: Understand how your multi-agent systems are performing\nStep 5: Continuous Optimization\nWith comprehensive tracing in place, you can:\n- A/B Test Models: Compare different LLMs on real production traffic\n- Optimize Prompts: See which prompts perform best for different use cases\n- Reduce Costs: Identify expensive operations and optimize them\n- Improve Quality: Monitor output quality and user satisfaction\n- Scale Confidently: Understand system behavior under load\nReal-World Example: RAG-Powered Customer Support\nImagine you're building an AI-powered customer support system with a RAG pipeline:\n- User Query: Customer asks about pricing\n- Document Retrieval: System searches knowledge base\n- Context Preparation: Relevant documents are processed\n- LLM Generation: GPT-4 generates response with context\n- Response Delivery: Answer is sent to customer\nWithout Noveum.ai: When the system gives wrong answers, you have no visibility into what went wrong. Was it poor retrieval? Bad context preparation? LLM hallucination?\nWith Noveum.ai: Every step is traced:\n@noveum_trace.trace\ndef handle_support_query(user_query: str, user_id: str) -> str:\n# Each step is automatically traced\n# Step 1: Query analysis\nquery_intent = analyze_query_intent(user_query)\n# Step 2: Document retrieval\nrelevant_docs = retrieve_documents(user_query, query_intent)\n# Step 3: Context preparation\ncontext = prepare_context(relevant_docs)\n# Step 4: LLM generation\nresponse = generate_response(user_query, context)\n# Step 5: Response validation\nvalidated_response = validate_response(response, user_query)\nreturn validated_response\nNow you can see:\n- Which queries are taking too long (retrieval vs. generation)\n- When retrieval is returning irrelevant documents\n- How much each interaction costs\n- Which responses users rate poorly\n- Complete audit trail for compliance\nAdvanced Features\nMulti-Agent System Observability\n@noveum_trace.trace_agent(agent_id=\"coordinator\")\ndef coordinate_research_project(topic: str) -> dict:\n# Assign tasks to specialist agents\nliterature_review = literature_agent(topic)\ndata_analysis = data_agent(topic)\nsynthesis = synthesis_agent(literature_review, data_analysis)\nreturn {\n\"literature\": literature_review,\n\"analysis\": data_analysis,\n\"synthesis\": synthesis\n}\nCustom Sampling and Filtering\nnoveum_trace.init(\napi_key=\"your-key\",\nproject=\"my-project\",\ntransport_config={\n\"sample_rate\": 0.1, # Sample 10% of traces\n\"capture_errors\": True, # Always capture errors\n\"capture_stack_traces\": False # Skip stack traces for performance\n}\n)\nContext Propagation\n// Automatic context propagation across async operations\nconst contextManager = getGlobalContextManager();\nawait contextManager.withSpan(span, async () => {\n// This function runs with span in context\nawait someNestedOperation();\n});\nWhy Choose Noveum.ai?\n- AI-Native Design: Built specifically for LLM applications, not generic APM tools\n- Easy Integration: Minutes to get started, not days of configuration\n- Framework Agnostic: Works with any LLM provider, vector database, or framework\n- Production Ready: Built for scale with intelligent sampling and batching\n- Privacy Focused: You control what data is captured and how it's stored\n- Open Source SDKs: Transparent, auditable, and extensible\nGetting Started\nReady to add comprehensive observability to your AI applications?\n- Sign up at noveum.ai\n- Create a project and get your API key\n- Install the SDK for your preferred language\n- Add tracing to your AI workflows\n- Explore insights in the Noveum.ai dashboard\nCheck out our integration guide for detailed setup instructions and examples.\nWrapping Up\nNoveum.ai brings the observability your AI applications deserve. No more black box debugging or guessing why your RAG pipeline failed. With comprehensive tracing, you get complete visibility into every LLM call, agent interaction, and data flow.\nThe AI landscape is complex and moving fast. With Noveum.ai, you'll have the insights you need to build, debug, and optimize AI applications with confidence.\nReady to see what's really happening inside your AI applications? Try Noveum.ai today and transform how you build and monitor AI systems.\nLet's build more reliable, observable AIā€”together.\nGet Early Access to Noveum.ai Platform\nJoin the select group of AI teams optimizing their models with our data-driven platform. We're onboarding users in limited batches to ensure a premium experience.", + "content_length": 10177, + "internal_links": [ + "https://noveum.ai/integration" + ], + "scraped_at": 1759935903.28646 + }, + { + "url": "https://noveum.ai/en/legal/privacy-policy", + "title": "Privacy Policy | Noveum.ai", + "content": "Privacy Policy\nThis is the placeholder page for your privacy policy. Edit the content/legal/privacy-policy.md\nfile to add your own content here.\nThis is the placeholder page for your privacy policy. Edit the content/legal/privacy-policy.md\nfile to add your own content here.", + "content_length": 274, + "internal_links": [], + "scraped_at": 1759935904.1824288 + }, + { + "url": "https://noveum.ai/en/legal/terms", + "title": "Terms and conditions | Noveum.ai", + "content": "Terms and conditions\nThis is the placeholder page for your terms and conditions. Edit the content/legal/terms.md\nfile to add your own content here.\nThis is the placeholder page for your terms and conditions. Edit the content/legal/terms.md\nfile to add your own content here.", + "content_length": 274, + "internal_links": [], + "scraped_at": 1759935905.058074 + }, + { + "url": "https://noveum.ai/en/docs/getting-started/tracing-concepts", + "title": "Tracing Concepts for AI Applications | Documentation | Noveum.ai", + "content": "Tracing Concepts for AI Applications\nUnderstanding traces, spans, and observability fundamentals for LLM applications, RAG systems, and AI agents\nUnderstanding the fundamentals of tracing is essential for getting the most out of Noveum.ai. This guide explains key concepts specifically in the context of AI applications, helping you design effective observability strategies for your LLM applications, RAG systems, and AI agents.\nšŸŽÆ What is Tracing?\nTracing is the practice of tracking requests as they flow through your system, creating a detailed map of what happened, when, and how long each operation took. For AI applications, tracing provides crucial insights into:\n- šŸ” Request Flow: How user queries move through your AI pipeline\n- ā±ļø Performance: Where time is spent in your AI operations\n- šŸ’° Costs: Which operations drive your AI spending\n- šŸ› Debugging: What went wrong when errors occur\n- šŸ“Š Quality: How well your AI system is performing\nšŸŒŸ Core Concepts\n1. Traces\nA trace represents a single journey through your systemā€”like a user asking a question and getting an answer. Think of it as the complete story of one request.\nTrace Characteristics:\n- šŸ†” Unique ID: Every trace has a unique identifier\n- ā° Timeline: Start and end timestamps\n- šŸŒ Distributed: Can span multiple services\n- šŸ“Š Hierarchical: Contains multiple related spans\n2. Spans\nA span represents a single operation within a trace. Each span has a clear start and end time and represents work being done.\nSpan Characteristics:\n- šŸ“› Name: Descriptive name of the operation\n- ā±ļø Duration: How long the operation took\n- šŸ‘„ Parent-Child: Spans can contain other spans\n- šŸ·ļø Attributes: Key-value metadata about the operation\n- šŸ“ Events: Point-in-time occurrences during the span\n3. Attributes\nAttributes are key-value pairs that provide context about what happened during a span. They're crucial for understanding and filtering your traces.\nCommon AI Attribute Categories:\n- šŸ¤– LLM Attributes:\nllm.model\n,llm.provider\n,llm.temperature\n- šŸ’° Cost Attributes:\nllm.tokens.input\n,llm.tokens.output\n,llm.cost\n- šŸ‘¤ User Attributes:\nuser.id\n,user.plan\n,user.location\n- šŸ“„ Content Attributes:\nprompt.length\n,response.length\n,content.type\n- šŸ” Quality Attributes:\nrelevance.score\n,confidence.level\n,accuracy.rating\n4. Events\nEvents represent things that happened at a specific point in time during a span. They're perfect for capturing important moments or milestones.\nšŸ§  AI-Specific Tracing Patterns\nRAG Pipeline Tracing\nRAG (Retrieval-Augmented Generation) systems have distinct phases that should be traced separately:\nMulti-Agent Tracing\nWhen dealing with multiple AI agents, trace their interactions and coordination:\nšŸ“Š Observability Best Practices\n1. Meaningful Span Names\nUse descriptive, consistent naming conventions:\n2. Rich Attributes\nInclude context that helps with debugging and analysis:\n3. Error Handling\nAlways capture error details:\n4. Performance Context\nInclude performance-relevant attributes:\nšŸ” Using Traces for Debugging\nCommon Debugging Scenarios\n1. Slow Response Times\n2. High Costs\n3. Quality Issues\n4. Error Patterns\nšŸŽÆ Next Steps\nNow that you understand tracing concepts, you're ready to:\n- Implement SDK Integration - Add tracing to your application\n- Explore Framework Integrations - Framework-specific guidance\n- Learn Advanced Patterns - Custom instrumentation techniques\n- Master the Dashboard - Analyze your traces effectively\nRemember: Good observability is not about collecting all possible data, but about collecting the right data that helps you understand, debug, and optimize your AI applications.\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year.", + "content_length": 3822, + "internal_links": [], + "scraped_at": 1759935906.108872 + }, + { + "url": "https://noveum.ai/en/docs/getting-started/framework-integrations", + "title": "Framework Integrations | Documentation | Noveum.ai", + "content": "Framework Integrations\nDeep-dive integration guides for Next.js, Express.js, FastAPI, Flask and other popular frameworks\nNoveum.ai provides native integrations for popular web frameworks, making it easy to add comprehensive tracing to your AI applications. This guide covers framework-specific setup, best practices, and advanced patterns.\nšŸš€ Quick Framework Overview\n| Framework | Language | Integration Type | Difficulty |\n|---|---|---|---|\n| Next.js | TypeScript | Middleware + Wrappers | ā­ Easy |\n| Express.js | TypeScript | Middleware | ā­ Easy |\n| Hono | TypeScript | Middleware + Decorators | ā­ Easy |\n| FastAPI | Python | Middleware + Decorators | ā­ā­ Moderate |\n| Flask | Python | Extensions + Decorators | ā­ā­ Moderate |\n| Django | Python | Middleware + Decorators | ā­ā­ā­ Advanced |\nšŸ“˜ TypeScript Frameworks\nNext.js Integration\nNext.js is one of the most popular frameworks for AI applications. Noveum provides seamless integration for both App Router and Pages Router.\nApp Router Setup\n1. Initialize Noveum (Root Layout)\n2. API Route Tracing\n3. Advanced API Route with Custom Tracing\nServer Actions Tracing\nExpress.js Integration\nExpress.js integration provides automatic tracing for all routes and middleware.\n1. Setup Middleware\n2. Manual Route Tracing\n3. Middleware with Custom Logic\nHono Integration\nHono is a lightweight framework perfect for edge computing and AI applications.\nšŸ Python Frameworks\nFastAPI Integration\nFastAPI is excellent for building high-performance AI APIs with automatic documentation.\n1. Setup with Middleware\n2. Traced Endpoints\n3. RAG Endpoint with Detailed Tracing\nFlask Integration\nFlask integration provides flexibility for existing applications.\n1. Setup with Extensions\n2. Traced Routes\n3. Background Task Tracing\nšŸ”§ Advanced Patterns\nEnvironment-Specific Configuration\nCustom Middleware\nError Boundary Integration\nšŸŽÆ Next Steps\nChoose your framework and dive deeper:\n- Implement Basic Integration - Start with the basics\n- Learn Tracing Concepts - Understand the fundamentals\n- Explore Advanced Patterns - Custom instrumentation\n- Master the Dashboard - Analyze your traces\nFramework not listed? Check our Custom Integration Guide or contact our team for specific framework support.\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year.", + "content_length": 2462, + "internal_links": [], + "scraped_at": 1759935907.308911 + }, + { + "url": "https://noveum.ai/en/docs/advanced/multi-agent-tracing", + "title": "Multi-Agent Tracing | Documentation | Noveum.ai", + "content": "Multi-Agent Tracing\nObserve complex agent workflows and inter-agent communications with comprehensive tracing\nMulti-agent systems represent some of the most complex AI applications, involving multiple agents that coordinate, communicate, and collaborate to achieve shared goals. Noveum.ai provides specialized tracing capabilities to help you understand and optimize these intricate workflows.\nšŸŽÆ Why Multi-Agent Tracing Matters\nMulti-agent systems introduce unique observability challenges:\n- Complex Dependencies: Agents depend on each other's outputs and decisions\n- Asynchronous Operations: Agents may operate concurrently or in parallel\n- Communication Patterns: Understanding how agents share information\n- Resource Coordination: Managing shared resources and preventing conflicts\n- Error Propagation: How failures in one agent affect the entire system\nšŸ—ļø Agent System Architecture\nAgent Types and Roles\nNoveum.ai can trace various agent patterns:\nTypeScript Multi-Agent Example\nšŸ“Š Tracing Multi-Agent Workflows\nCoordination Patterns\nSequential Agent Execution\nParallel Agent Execution\nHierarchical Agent Systems\nšŸ”— Inter-Agent Communication Tracing\nMessage Passing\nShared State Management\nšŸ“ˆ Multi-Agent Performance Analysis\nAgent Performance Metrics\nTrack key metrics for each agent:\nSystem-Wide Coordination Metrics\nšŸ”§ Best Practices for Multi-Agent Tracing\n1. Agent Identification\nAlways clearly identify agents in your traces:\n2. Communication Tracing\nTrace all inter-agent communications:\n3. Error Propagation Tracking\nMonitor how errors propagate through agent systems:\n4. Resource Coordination\nTrack shared resource usage:\nšŸŽÆ Advanced Multi-Agent Patterns\nSelf-Organizing Agent Systems\nAdaptive Agent Workflows\nšŸ“Š Monitoring and Alerts\nSet up monitoring for multi-agent systems:\nMulti-agent tracing with Noveum.ai provides the visibility needed to understand, optimize, and scale complex agent systems. By implementing comprehensive tracing across all agent interactions, communications, and coordination patterns, you can build more reliable and efficient multi-agent AI applications.\nšŸ”— Next Steps\n- RAG Pipeline Observability - Monitor retrieval and generation systems\n- Custom Instrumentation - Add domain-specific tracing\n- Performance Optimization - Optimize based on tracing insights\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year.", + "content_length": 2537, + "internal_links": [], + "scraped_at": 1759935908.445167 + }, + { + "url": "https://noveum.ai/en/docs/advanced/rag-observability", + "title": "RAG Pipeline Observability | Documentation | Noveum.ai", + "content": "RAG Pipeline Observability\nMonitor retrieval, generation, and context handling in RAG systems with comprehensive tracing\nRetrieval-Augmented Generation (RAG) systems combine the power of information retrieval with large language models to provide accurate, contextual responses. Monitoring these complex pipelines requires specialized observability to understand retrieval quality, context relevance, and generation effectiveness.\nšŸŽÆ Why RAG Observability Matters\nRAG systems introduce unique challenges that traditional monitoring can't address:\n- Retrieval Quality: Are you finding the most relevant documents?\n- Context Utilization: How effectively is retrieved context being used?\n- Generation Fidelity: Is the LLM accurately using the provided context?\n- Pipeline Performance: Where are the bottlenecks in your RAG pipeline?\n- Cost Optimization: Which components consume the most resources?\nšŸ—ļø RAG Pipeline Architecture\nCore RAG Components\nNoveum.ai can trace each stage of your RAG pipeline:\nTypeScript RAG Implementation\nšŸ“Š Tracing Retrieval Components\nVector Database Operations\nDocument Ranking and Reranking\nContext Window Management\nšŸ¤– Tracing LLM Generation\nContext-Aware Generation\nResponse Evaluation and Feedback\nšŸ“ˆ RAG Pipeline Performance Analysis\nEnd-to-End Pipeline Metrics\nCost and Resource Tracking\nšŸ”§ Best Practices for RAG Observability\n1. Comprehensive Pipeline Tracing\n2. Quality Monitoring\n3. A/B Testing for RAG Components\nšŸŽÆ Advanced RAG Patterns\nMulti-Modal RAG\nConversational RAG\nRAG observability with Noveum.ai provides the deep insights needed to build, optimize, and scale retrieval-augmented generation systems. By implementing comprehensive tracing across retrieval, context preparation, and generation stages, you can ensure your RAG pipeline delivers accurate, relevant, and cost-effective responses.\nšŸ”— Next Steps\n- Custom Instrumentation - Add domain-specific tracing\n- Multi-Agent Tracing - Observe agent workflows\n- Performance Optimization - Optimize based on tracing insights\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year.", + "content_length": 2253, + "internal_links": [], + "scraped_at": 1759935909.55895 + }, + { + "url": "https://noveum.ai/en/docs/advanced/custom-instrumentation", + "title": "Custom Instrumentation | Documentation | Noveum.ai", + "content": "Custom Instrumentation\nAdd custom spans and attributes for domain-specific observability and advanced tracing patterns\nWhile Noveum.ai's automatic instrumentation covers common AI operations, custom instrumentation allows you to add domain-specific observability, track business metrics, and create detailed traces for unique workflows. This guide covers advanced techniques for implementing custom tracing patterns.\nšŸŽÆ Why Custom Instrumentation?\nCustom instrumentation enables you to:\n- Track Business Metrics: Monitor domain-specific KPIs alongside technical metrics\n- Trace Complex Workflows: Create detailed observability for unique business logic\n- Add Context: Enrich traces with application-specific attributes\n- Monitor Custom Components: Instrument proprietary algorithms and processes\n- Optimize Performance: Track specific bottlenecks in your application\nšŸ› ļø Custom Span Creation\nBasic Custom Spans\nTypeScript Custom Instrumentation\nšŸ“Š Custom Metrics and Attributes\nBusiness Metrics Integration\nPerformance Profiling Integration\nšŸ”§ Custom Context Propagation\nThread-Safe Context Management\nAsync Context Propagation\nšŸ“ˆ Advanced Custom Patterns\nEvent-Driven Instrumentation\nCustom Sampling Strategies\nšŸ”— Integration with External Systems\nDatabase Operation Tracing\nCustom instrumentation with Noveum.ai provides the flexibility to create detailed, domain-specific observability that goes beyond standard LLM and AI operation tracing. By implementing custom spans, attributes, context propagation, and advanced patterns, you can build comprehensive monitoring tailored to your specific application needs.\nšŸ”— Next Steps\n- Performance Optimization - Use tracing insights to optimize performance\n- Multi-Agent Tracing - Observe agent workflows\n- RAG Pipeline Observability - Monitor retrieval and generation systems\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year.", + "content_length": 2056, + "internal_links": [], + "scraped_at": 1759935910.8084621 + }, + { + "url": "https://noveum.ai/en/docs/advanced/performance-optimization", + "title": "Performance Optimization | Documentation | Noveum.ai", + "content": "Performance Optimization\nUse tracing data to identify bottlenecks and optimize AI application performance\nPerformance optimization for AI applications requires understanding the unique characteristics of LLM calls, vector operations, and complex workflows. Noveum.ai's tracing data provides detailed insights to identify bottlenecks, optimize resource usage, and improve overall system performance.\nšŸŽÆ Why AI Performance Optimization Matters\nAI applications have unique performance characteristics:\n- Token-Based Costs: LLM usage is measured in tokens, making efficiency crucial\n- Variable Latency: AI operations can have unpredictable response times\n- Context Dependencies: Performance varies with input size and complexity\n- Resource Intensive: Vector operations and embeddings require significant compute\n- Cascading Effects: Slow AI components impact entire application workflows\nšŸ“Š Performance Analysis with Tracing Data\nIdentifying Performance Bottlenecks\nOptimization Implementation Strategies\nšŸš€ Advanced Optimization Techniques\nModel Selection Optimization\nResource Usage Optimization\nšŸŽÆ Performance Optimization Best Practices\n1. Establish Performance Baselines\n2. Implement Gradual Optimization\n3. Monitor Optimization Impact\nPerformance optimization for AI applications requires a systematic approach combining detailed tracing insights, strategic implementation, and continuous monitoring. By leveraging Noveum.ai's comprehensive tracing data, you can identify bottlenecks, implement targeted optimizations, and achieve significant improvements in latency, cost, and resource efficiency.\nšŸ”— Next Steps\n- Multi-Agent Tracing - Observe agent workflows\n- RAG Pipeline Observability - Monitor retrieval and generation systems\n- Custom Instrumentation - Add domain-specific tracing\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year.", + "content_length": 2025, + "internal_links": [], + "scraped_at": 1759935912.062154 + }, + { + "url": "https://noveum.ai/docs/getting-started/overview", + "title": "Noveum.ai Overview | Documentation | Noveum.ai", + "content": "Noveum.ai Overview\nComprehensive AI tracing and observability platform for LLM applications, RAG systems, and AI agents\nWelcome to Noveum.aiā€”the comprehensive tracing and observability platform built specifically for AI applications. Whether you're building LLM-powered chatbots, RAG systems, multi-agent workflows, or any AI-driven application, Noveum provides the insights you need to understand, debug, and optimize your systems.\nšŸŽÆ Why AI Applications Need Specialized Observability\nTraditional monitoring tools fall short when it comes to AI applications because they don't understand:\n- šŸ“Š AI-Specific Metrics: Token usage, model costs, prompt effectiveness\n- šŸ”€ Complex Workflows: Multi-step RAG pipelines, agent interactions, tool usage\n- šŸ§  Context Flow: How data moves through embeddings, retrievals, and generations\n- šŸ’° Cost Attribution: Which operations drive your AI spending\n- šŸŽÆ Quality Metrics: Beyond latency - understanding output quality and relevance\nNoveum.ai bridges this gap with purpose-built observability for the AI era.\nšŸš€ Core Platform Components\n1. šŸ Python SDK (noveum-trace\n)\n- Decorator-based tracing for seamless integration\n- Automatic instrumentation for LangChain, LlamaIndex, and OpenAI\n- Async-aware context propagation\n- Production-ready with intelligent sampling and batching\n2. šŸ“˜ TypeScript SDK (@noveum/trace\n)\n- Framework integrations for Next.js, Express.js, Hono\n- TypeScript-first with full type safety\n- Universal compatibility (Node.js, Edge Runtime, browsers)\n- Zero-config automatic instrumentation\n3. šŸ“Š Noveum Platform\n- Real-time dashboard with AI-specific visualizations\n- Advanced search & filtering across traces and spans\n- Cost analysis and optimization recommendations\n- Team collaboration with shared insights and alerts\nšŸ” What Noveum Traces\nLLM Operations\n- Model calls across all providers (OpenAI, Anthropic, Google, etc.)\n- Token usage and cost calculation\n- Prompt engineering effectiveness\n- Response quality metrics\nRAG Pipelines\n- Document retrieval performance and relevance\n- Embedding generation costs and latency\n- Context assembly and prompt construction\n- Answer generation with source attribution\nMulti-Agent Systems\n- Agent interactions and communication patterns\n- Tool usage and external API calls\n- Decision trees and reasoning chains\n- Workflow orchestration across agents\nCustom Operations\n- Business logic specific to your domain\n- External integrations and API calls\n- Data processing pipelines\n- User interactions and session flows\nšŸŽÆ Key Benefits\nšŸ”§ Developer Experience\n- 5-minute setup with minimal code changes\n- Intelligent defaults that work out-of-the-box\n- Rich SDKs with comprehensive documentation\n- Local development support with optional cloud sync\nšŸ“Š Production Insights\n- Real-time monitoring of AI application health\n- Performance optimization with bottleneck identification\n- Cost management with detailed spend analysis\n- Quality assurance through automated alerting\nšŸ”’ Enterprise Ready\n- Security first with end-to-end encryption\n- Compliance support for regulated industries\n- Scalable architecture handling millions of traces\n- Data sovereignty with region-specific storage\nšŸ‘„ Team Collaboration\n- Shared dashboards for cross-functional teams\n- Incident management with trace-based debugging\n- Performance baselines and regression detection\n- Knowledge sharing through trace annotations\nšŸ“ˆ Common Use Cases\nšŸ¤– LLM Application Monitoring\nTrack every aspect of your LLM-powered application:\n- Monitor response quality and user satisfaction\n- Optimize prompt engineering for better results\n- Control costs across different models and providers\n- Debug edge cases and improve error handling\nšŸ” RAG System Optimization\nUnderstand and improve your RAG pipeline:\n- Measure retrieval accuracy and relevance\n- Optimize embedding models and vector search\n- Track context utilization and prompt effectiveness\n- Debug hallucinations and improve grounding\nšŸ¤ Multi-Agent Coordination\nObserve complex agent interactions:\n- Visualize agent communication patterns\n- Track tool usage and external dependencies\n- Optimize workflow efficiency and resource usage\n- Debug coordination failures and deadlocks\nšŸš€ Performance Engineering\nOptimize your AI application performance:\n- Identify slow operations and bottlenecks\n- Right-size models for your workload\n- Implement intelligent caching strategies\n- Scale services based on actual usage patterns\nšŸŽØ Platform Features\nšŸ” Trace Explorer\n- Hierarchical visualization of complex AI workflows\n- Timeline view showing operation sequences\n- Detailed span inspection with all attributes and events\n- Cross-trace correlation for distributed operations\nšŸ’° Cost Analytics\n- Real-time cost tracking across all AI providers\n- Cost attribution by user, feature, or operation\n- Budget alerts and spending forecasts\n- Optimization recommendations for cost reduction\nšŸ“Š Performance Dashboard\n- Latency percentiles and throughput metrics\n- Error rates and failure analysis\n- Model comparison across providers and versions\n- Custom metrics and business KPIs\nšŸšØ Alerting & Monitoring\n- Intelligent alerts based on AI-specific thresholds\n- Anomaly detection for unusual patterns\n- Escalation policies for critical issues\n- Integration with Slack, PagerDuty, and more\nšŸ› ļø Integration Patterns\nIncremental Adoption\nStart small and expand coverage:\n- Single endpoint tracing for immediate value\n- Critical path instrumentation for core workflows\n- Full application coverage for comprehensive insights\n- Advanced features like custom metrics and alerts\nFramework Integration\nNative support for popular frameworks:\n- Next.js with App Router and API routes\n- Express.js and other Node.js frameworks\n- FastAPI and Flask for Python applications\n- Custom integrations for any framework\nCI/CD Integration\nEmbed observability in your development process:\n- Performance regression detection in CI\n- Trace-based testing for quality assurance\n- Deployment monitoring with rollback triggers\n- Feature flag integration for safe releases\nšŸŒŸ Getting Started\nReady to transform your AI application observability? Here's your path:\n- Quick Start - Integrate your first SDK in 5 minutes\n- Tracing Concepts - Learn the fundamentals\n- Framework Guides - Deep dive into your stack\n- Advanced Features - Unlock the full platform potential\nBuilt by developers, for developers. Noveum.ai understands that AI applications are different, and we've designed our platform from the ground up to meet their unique observability needs.\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year.", + "content_length": 6733, + "internal_links": [], + "scraped_at": 1759935913.302529 + }, + { + "url": "https://noveum.ai/docs/getting-started/sdk-integration", + "title": "SDK Integration Guide | Documentation | Noveum.ai", + "content": "SDK Integration Guide\nIntegrate Noveum.ai tracing into your AI applications with Python or TypeScript SDKs\nThe Noveum.ai SDKs provide comprehensive tracing and observability for your AI applications with minimal code changes. Whether you're building LLM applications, RAG systems, or multi-agent workflows, our SDKs automatically capture essential metrics and traces.\nšŸš€ Quick Start\n1. Create Your Account & Get API Key\n- Sign up at noveum.ai\n- Create a project in your dashboard\n- Generate an API key from the integration page\n- Choose your SDK based on your application language\n2. Install the SDK\nRequirements: Python 3.8+\n3. Initialize the Client\nEnvironment Variables:\nšŸŽÆ Basic Usage\nTrace LLM Calls\nAlternative - Context Manager:\nTrace RAG Pipelines\nšŸ”§ Framework Integrations\nNext.js Integration\nExpress.js Integration\nFastAPI Integration (Python)\nšŸ“Š Advanced Features\nCustom Attributes & Events\nSampling Configuration\nšŸ”— What's Captured Automatically\n- šŸ“Š Performance Metrics: Latency, throughput, error rates\n- šŸ’° Cost Tracking: Token usage, API costs across providers\n- šŸ” Request/Response: Configurable capture of inputs/outputs\n- šŸ·ļø Metadata: Model names, parameters, user context\n- šŸŒŠ Context Flow: Trace relationships across services\n- šŸ› Error Details: Stack traces, error classification\nšŸ“ˆ View Your Data\nOnce integrated, visit your Noveum Dashboard to:\n- šŸ” Search & Filter traces by any attribute\n- šŸ“Š Analyze Performance trends and bottlenecks\n- šŸ’° Monitor Costs across different models and providers\n- šŸ› Debug Issues with detailed trace timelines\n- šŸ‘„ Collaborate with your team on insights\nšŸ”’ Security & Privacy\n- šŸ” Encryption: All data encrypted in transit and at rest\n- šŸŽ›ļø Configurable Capture: Control what data is collected\n- šŸ  Data Residency: Choose your data storage region\n- ā° Retention Control: Set custom data retention policies\nNext Steps\n- Tracing Concepts - Learn about traces, spans, and observability best practices\n- Framework Integrations - Deep dive into specific framework setups\n- Multi-Agent Tracing - Observe complex agent workflows\n- Dashboard Guide - Master the Noveum platform interface\nExclusive Early Access\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year.", + "content_length": 2378, + "internal_links": [], + "scraped_at": 1759935914.586812 + }, + { + "url": "https://noveum.ai/docs/getting-started/tracing-concepts", + "title": "Tracing Concepts for AI Applications | Documentation | Noveum.ai", + "content": "Tracing Concepts for AI Applications\nUnderstanding traces, spans, and observability fundamentals for LLM applications, RAG systems, and AI agents\nUnderstanding the fundamentals of tracing is essential for getting the most out of Noveum.ai. This guide explains key concepts specifically in the context of AI applications, helping you design effective observability strategies for your LLM applications, RAG systems, and AI agents.\nšŸŽÆ What is Tracing?\nTracing is the practice of tracking requests as they flow through your system, creating a detailed map of what happened, when, and how long each operation took. For AI applications, tracing provides crucial insights into:\n- šŸ” Request Flow: How user queries move through your AI pipeline\n- ā±ļø Performance: Where time is spent in your AI operations\n- šŸ’° Costs: Which operations drive your AI spending\n- šŸ› Debugging: What went wrong when errors occur\n- šŸ“Š Quality: How well your AI system is performing\nšŸŒŸ Core Concepts\n1. Traces\nA trace represents a single journey through your systemā€”like a user asking a question and getting an answer. Think of it as the complete story of one request.\nTrace Characteristics:\n- šŸ†” Unique ID: Every trace has a unique identifier\n- ā° Timeline: Start and end timestamps\n- šŸŒ Distributed: Can span multiple services\n- šŸ“Š Hierarchical: Contains multiple related spans\n2. Spans\nA span represents a single operation within a trace. Each span has a clear start and end time and represents work being done.\nSpan Characteristics:\n- šŸ“› Name: Descriptive name of the operation\n- ā±ļø Duration: How long the operation took\n- šŸ‘„ Parent-Child: Spans can contain other spans\n- šŸ·ļø Attributes: Key-value metadata about the operation\n- šŸ“ Events: Point-in-time occurrences during the span\n3. Attributes\nAttributes are key-value pairs that provide context about what happened during a span. They're crucial for understanding and filtering your traces.\nCommon AI Attribute Categories:\n- šŸ¤– LLM Attributes:\nllm.model\n,llm.provider\n,llm.temperature\n- šŸ’° Cost Attributes:\nllm.tokens.input\n,llm.tokens.output\n,llm.cost\n- šŸ‘¤ User Attributes:\nuser.id\n,user.plan\n,user.location\n- šŸ“„ Content Attributes:\nprompt.length\n,response.length\n,content.type\n- šŸ” Quality Attributes:\nrelevance.score\n,confidence.level\n,accuracy.rating\n4. Events\nEvents represent things that happened at a specific point in time during a span. They're perfect for capturing important moments or milestones.\nšŸ§  AI-Specific Tracing Patterns\nRAG Pipeline Tracing\nRAG (Retrieval-Augmented Generation) systems have distinct phases that should be traced separately:\nMulti-Agent Tracing\nWhen dealing with multiple AI agents, trace their interactions and coordination:\nšŸ“Š Observability Best Practices\n1. Meaningful Span Names\nUse descriptive, consistent naming conventions:\n2. Rich Attributes\nInclude context that helps with debugging and analysis:\n3. Error Handling\nAlways capture error details:\n4. Performance Context\nInclude performance-relevant attributes:\nšŸ” Using Traces for Debugging\nCommon Debugging Scenarios\n1. Slow Response Times\n2. High Costs\n3. Quality Issues\n4. Error Patterns\nšŸŽÆ Next Steps\nNow that you understand tracing concepts, you're ready to:\n- Implement SDK Integration - Add tracing to your application\n- Explore Framework Integrations - Framework-specific guidance\n- Learn Advanced Patterns - Custom instrumentation techniques\n- Master the Dashboard - Analyze your traces effectively\nRemember: Good observability is not about collecting all possible data, but about collecting the right data that helps you understand, debug, and optimize your AI applications.\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year.", + "content_length": 3822, + "internal_links": [], + "scraped_at": 1759935915.863421 + }, + { + "url": "https://noveum.ai/docs/getting-started/framework-integrations", + "title": "Framework Integrations | Documentation | Noveum.ai", + "content": "Framework Integrations\nDeep-dive integration guides for Next.js, Express.js, FastAPI, Flask and other popular frameworks\nNoveum.ai provides native integrations for popular web frameworks, making it easy to add comprehensive tracing to your AI applications. This guide covers framework-specific setup, best practices, and advanced patterns.\nšŸš€ Quick Framework Overview\n| Framework | Language | Integration Type | Difficulty |\n|---|---|---|---|\n| Next.js | TypeScript | Middleware + Wrappers | ā­ Easy |\n| Express.js | TypeScript | Middleware | ā­ Easy |\n| Hono | TypeScript | Middleware + Decorators | ā­ Easy |\n| FastAPI | Python | Middleware + Decorators | ā­ā­ Moderate |\n| Flask | Python | Extensions + Decorators | ā­ā­ Moderate |\n| Django | Python | Middleware + Decorators | ā­ā­ā­ Advanced |\nšŸ“˜ TypeScript Frameworks\nNext.js Integration\nNext.js is one of the most popular frameworks for AI applications. Noveum provides seamless integration for both App Router and Pages Router.\nApp Router Setup\n1. Initialize Noveum (Root Layout)\n2. API Route Tracing\n3. Advanced API Route with Custom Tracing\nServer Actions Tracing\nExpress.js Integration\nExpress.js integration provides automatic tracing for all routes and middleware.\n1. Setup Middleware\n2. Manual Route Tracing\n3. Middleware with Custom Logic\nHono Integration\nHono is a lightweight framework perfect for edge computing and AI applications.\nšŸ Python Frameworks\nFastAPI Integration\nFastAPI is excellent for building high-performance AI APIs with automatic documentation.\n1. Setup with Middleware\n2. Traced Endpoints\n3. RAG Endpoint with Detailed Tracing\nFlask Integration\nFlask integration provides flexibility for existing applications.\n1. Setup with Extensions\n2. Traced Routes\n3. Background Task Tracing\nšŸ”§ Advanced Patterns\nEnvironment-Specific Configuration\nCustom Middleware\nError Boundary Integration\nšŸŽÆ Next Steps\nChoose your framework and dive deeper:\n- Implement Basic Integration - Start with the basics\n- Learn Tracing Concepts - Understand the fundamentals\n- Explore Advanced Patterns - Custom instrumentation\n- Master the Dashboard - Analyze your traces\nFramework not listed? Check our Custom Integration Guide or contact our team for specific framework support.\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year.", + "content_length": 2462, + "internal_links": [], + "scraped_at": 1759935917.23431 + }, + { + "url": "https://noveum.ai/docs/advanced/multi-agent-tracing", + "title": "Multi-Agent Tracing | Documentation | Noveum.ai", + "content": "Multi-Agent Tracing\nObserve complex agent workflows and inter-agent communications with comprehensive tracing\nMulti-agent systems represent some of the most complex AI applications, involving multiple agents that coordinate, communicate, and collaborate to achieve shared goals. Noveum.ai provides specialized tracing capabilities to help you understand and optimize these intricate workflows.\nšŸŽÆ Why Multi-Agent Tracing Matters\nMulti-agent systems introduce unique observability challenges:\n- Complex Dependencies: Agents depend on each other's outputs and decisions\n- Asynchronous Operations: Agents may operate concurrently or in parallel\n- Communication Patterns: Understanding how agents share information\n- Resource Coordination: Managing shared resources and preventing conflicts\n- Error Propagation: How failures in one agent affect the entire system\nšŸ—ļø Agent System Architecture\nAgent Types and Roles\nNoveum.ai can trace various agent patterns:\nTypeScript Multi-Agent Example\nšŸ“Š Tracing Multi-Agent Workflows\nCoordination Patterns\nSequential Agent Execution\nParallel Agent Execution\nHierarchical Agent Systems\nšŸ”— Inter-Agent Communication Tracing\nMessage Passing\nShared State Management\nšŸ“ˆ Multi-Agent Performance Analysis\nAgent Performance Metrics\nTrack key metrics for each agent:\nSystem-Wide Coordination Metrics\nšŸ”§ Best Practices for Multi-Agent Tracing\n1. Agent Identification\nAlways clearly identify agents in your traces:\n2. Communication Tracing\nTrace all inter-agent communications:\n3. Error Propagation Tracking\nMonitor how errors propagate through agent systems:\n4. Resource Coordination\nTrack shared resource usage:\nšŸŽÆ Advanced Multi-Agent Patterns\nSelf-Organizing Agent Systems\nAdaptive Agent Workflows\nšŸ“Š Monitoring and Alerts\nSet up monitoring for multi-agent systems:\nMulti-agent tracing with Noveum.ai provides the visibility needed to understand, optimize, and scale complex agent systems. By implementing comprehensive tracing across all agent interactions, communications, and coordination patterns, you can build more reliable and efficient multi-agent AI applications.\nšŸ”— Next Steps\n- RAG Pipeline Observability - Monitor retrieval and generation systems\n- Custom Instrumentation - Add domain-specific tracing\n- Performance Optimization - Optimize based on tracing insights\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year.", + "content_length": 2537, + "internal_links": [], + "scraped_at": 1759935918.6890082 + }, + { + "url": "https://noveum.ai/docs/advanced/rag-observability", + "title": "RAG Pipeline Observability | Documentation | Noveum.ai", + "content": "RAG Pipeline Observability\nMonitor retrieval, generation, and context handling in RAG systems with comprehensive tracing\nRetrieval-Augmented Generation (RAG) systems combine the power of information retrieval with large language models to provide accurate, contextual responses. Monitoring these complex pipelines requires specialized observability to understand retrieval quality, context relevance, and generation effectiveness.\nšŸŽÆ Why RAG Observability Matters\nRAG systems introduce unique challenges that traditional monitoring can't address:\n- Retrieval Quality: Are you finding the most relevant documents?\n- Context Utilization: How effectively is retrieved context being used?\n- Generation Fidelity: Is the LLM accurately using the provided context?\n- Pipeline Performance: Where are the bottlenecks in your RAG pipeline?\n- Cost Optimization: Which components consume the most resources?\nšŸ—ļø RAG Pipeline Architecture\nCore RAG Components\nNoveum.ai can trace each stage of your RAG pipeline:\nTypeScript RAG Implementation\nšŸ“Š Tracing Retrieval Components\nVector Database Operations\nDocument Ranking and Reranking\nContext Window Management\nšŸ¤– Tracing LLM Generation\nContext-Aware Generation\nResponse Evaluation and Feedback\nšŸ“ˆ RAG Pipeline Performance Analysis\nEnd-to-End Pipeline Metrics\nCost and Resource Tracking\nšŸ”§ Best Practices for RAG Observability\n1. Comprehensive Pipeline Tracing\n2. Quality Monitoring\n3. A/B Testing for RAG Components\nšŸŽÆ Advanced RAG Patterns\nMulti-Modal RAG\nConversational RAG\nRAG observability with Noveum.ai provides the deep insights needed to build, optimize, and scale retrieval-augmented generation systems. By implementing comprehensive tracing across retrieval, context preparation, and generation stages, you can ensure your RAG pipeline delivers accurate, relevant, and cost-effective responses.\nšŸ”— Next Steps\n- Custom Instrumentation - Add domain-specific tracing\n- Multi-Agent Tracing - Observe agent workflows\n- Performance Optimization - Optimize based on tracing insights\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year.", + "content_length": 2253, + "internal_links": [], + "scraped_at": 1759935920.029203 + }, + { + "url": "https://noveum.ai/docs/advanced/custom-instrumentation", + "title": "Custom Instrumentation | Documentation | Noveum.ai", + "content": "Custom Instrumentation\nAdd custom spans and attributes for domain-specific observability and advanced tracing patterns\nWhile Noveum.ai's automatic instrumentation covers common AI operations, custom instrumentation allows you to add domain-specific observability, track business metrics, and create detailed traces for unique workflows. This guide covers advanced techniques for implementing custom tracing patterns.\nšŸŽÆ Why Custom Instrumentation?\nCustom instrumentation enables you to:\n- Track Business Metrics: Monitor domain-specific KPIs alongside technical metrics\n- Trace Complex Workflows: Create detailed observability for unique business logic\n- Add Context: Enrich traces with application-specific attributes\n- Monitor Custom Components: Instrument proprietary algorithms and processes\n- Optimize Performance: Track specific bottlenecks in your application\nšŸ› ļø Custom Span Creation\nBasic Custom Spans\nTypeScript Custom Instrumentation\nšŸ“Š Custom Metrics and Attributes\nBusiness Metrics Integration\nPerformance Profiling Integration\nšŸ”§ Custom Context Propagation\nThread-Safe Context Management\nAsync Context Propagation\nšŸ“ˆ Advanced Custom Patterns\nEvent-Driven Instrumentation\nCustom Sampling Strategies\nšŸ”— Integration with External Systems\nDatabase Operation Tracing\nCustom instrumentation with Noveum.ai provides the flexibility to create detailed, domain-specific observability that goes beyond standard LLM and AI operation tracing. By implementing custom spans, attributes, context propagation, and advanced patterns, you can build comprehensive monitoring tailored to your specific application needs.\nšŸ”— Next Steps\n- Performance Optimization - Use tracing insights to optimize performance\n- Multi-Agent Tracing - Observe agent workflows\n- RAG Pipeline Observability - Monitor retrieval and generation systems\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year.", + "content_length": 2056, + "internal_links": [], + "scraped_at": 1759935921.483979 + }, + { + "url": "https://noveum.ai/docs/advanced/performance-optimization", + "title": "Performance Optimization | Documentation | Noveum.ai", + "content": "Performance Optimization\nUse tracing data to identify bottlenecks and optimize AI application performance\nPerformance optimization for AI applications requires understanding the unique characteristics of LLM calls, vector operations, and complex workflows. Noveum.ai's tracing data provides detailed insights to identify bottlenecks, optimize resource usage, and improve overall system performance.\nšŸŽÆ Why AI Performance Optimization Matters\nAI applications have unique performance characteristics:\n- Token-Based Costs: LLM usage is measured in tokens, making efficiency crucial\n- Variable Latency: AI operations can have unpredictable response times\n- Context Dependencies: Performance varies with input size and complexity\n- Resource Intensive: Vector operations and embeddings require significant compute\n- Cascading Effects: Slow AI components impact entire application workflows\nšŸ“Š Performance Analysis with Tracing Data\nIdentifying Performance Bottlenecks\nOptimization Implementation Strategies\nšŸš€ Advanced Optimization Techniques\nModel Selection Optimization\nResource Usage Optimization\nšŸŽÆ Performance Optimization Best Practices\n1. Establish Performance Baselines\n2. Implement Gradual Optimization\n3. Monitor Optimization Impact\nPerformance optimization for AI applications requires a systematic approach combining detailed tracing insights, strategic implementation, and continuous monitoring. By leveraging Noveum.ai's comprehensive tracing data, you can identify bottlenecks, implement targeted optimizations, and achieve significant improvements in latency, cost, and resource efficiency.\nšŸ”— Next Steps\n- Multi-Agent Tracing - Observe agent workflows\n- RAG Pipeline Observability - Monitor retrieval and generation systems\n- Custom Instrumentation - Add domain-specific tracing\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year.", + "content_length": 2025, + "internal_links": [], + "scraped_at": 1759935922.847022 + }, + { + "url": "https://noveum.ai/docs/platform/dashboard", + "title": "Dashboard Overview | Documentation | Noveum.ai", + "content": "Dashboard Overview\nNavigate the Noveum platform and understand key metrics for your AI applications\nDashboard Overview\nThe Noveum.ai dashboard provides comprehensive visibility into your AI application's performance, giving you real-time insights into traces, costs, and system health. Built specifically for AI workloads, it offers both high-level analytics and detailed trace inspection capabilities.\nšŸŽÆ Key Dashboard Features\nReal-Time Traces Monitoring\n- Live Trace Stream: Monitor LLM calls, RAG operations, and agent activities in real-time\n- Advanced Filtering: Filter by project, environment, status, date ranges, and custom attributes\n- Search Functionality: Quickly find specific traces using full-text search across all trace data\n- Status Indicators: Visual status badges for success, error, and pending operations\nPerformance Analytics\n- Latency Metrics: Track response times across different operations and time periods\n- Cost Analysis: Monitor spending across different LLM providers and operations\n- Throughput Monitoring: Observe request volumes and system capacity\n- Error Rate Tracking: Identify and monitor failure patterns\nInteractive Trace Inspection\n- Detailed Trace View: Expand any trace to see complete request/response data\n- Span Hierarchy: Navigate complex multi-step operations with visual span trees\n- Timing Analysis: Understand where time is spent in your AI operations\n- Context Preservation: See how data flows through embeddings, retrievals, and generations\nšŸ“Š Dashboard Components\nTraces List Interface\nThe main traces interface offers two viewing modes:\nClassic Interface\n- Tabular view of all traces with sortable columns\n- Quick filtering and search capabilities\n- Expandable detail panels for trace inspection\nThree-Pane Interface\n- Directory tree navigation for complex trace hierarchies\n- Split-pane view for simultaneous trace browsing and detail inspection\n- Advanced filtering with visual feedback\nFilter Controls\n- Environment Filter: Switch between development, staging, and production\n- Project Filter: Focus on specific applications or services\n- Status Filter: View only successful, failed, or pending operations\n- Date Range: Analyze performance over custom time periods\n- Clear Filters: Quick reset to view all traces\nConnection Status\n- Real-time Status: Monitor connection health to your trace storage\n- Error Reporting: Clear error messages when connectivity issues occur\n- Refresh Controls: Manual refresh capability for troubleshooting\nšŸ” Trace Detail Analysis\nComprehensive Trace Information\nEach trace provides detailed insights including:\n- Basic Metadata: Timestamp, duration, status, project, and environment\n- Request Context: User ID, session ID, and custom attributes\n- Response Data: Complete LLM responses, tool outputs, and generated content\n- Performance Metrics: Token usage, costs, and timing breakdowns\n- Error Details: Stack traces and error context when operations fail\nSpan Analysis\n- Operation Types: Automatic categorization of LLM calls, vector searches, tool usage\n- Attribute Inspection: View all custom attributes and metadata\n- Timing Visualization: Understand operation sequencing and bottlenecks\n- Parent-Child Relationships: Navigate complex workflow hierarchies\nFlow Visualization\n- Interactive Flow Charts: Visual representation of operation sequences\n- Dependency Mapping: See how different components interact\n- Error Path Analysis: Trace failure points through your system\nšŸŽØ Interface Customization\nLayout Options\n- Responsive Design: Optimized for desktop and mobile viewing\n- Panel Sizing: Adjustable interface panels for different screen sizes\n- Dark/Light Themes: Switch between themes for comfortable viewing\nData Display\n- Sortable Columns: Sort traces by any metric (time, duration, cost, status)\n- Configurable Views: Customize which trace attributes are displayed\n- Export Capabilities: Download trace data for external analysis\nšŸ“ˆ Getting Started with the Dashboard\nInitial Setup\n- Connect Your Applications: Ensure your AI applications are instrumented with Noveum SDKs\n- Verify Data Flow: Check the connection status indicator for successful trace ingestion\n- Explore Filters: Use environment and project filters to focus on relevant data\nBest Practices\n- Set Up Projects: Organize your applications into logical projects for better filtering\n- Use Environments: Separate development, staging, and production traces\n- Monitor Regularly: Check dashboard daily for performance trends and issues\n- Deep Dive on Errors: Use detailed trace inspection to troubleshoot failures\nPerformance Tips\n- Filter Early: Use filters to reduce data volume for faster loading\n- Time Range Selection: Limit date ranges for better performance with large datasets\n- Regular Refresh: Enable auto-refresh for monitoring live systems\nšŸ”— Integration with Other Platform Features\nThe dashboard seamlessly integrates with other Noveum platform capabilities:\n- Projects: Filter and organize traces by project structure\n- Team Collaboration: Share trace URLs with team members for collaborative debugging\n- API Access: Export trace data programmatically using the Noveum API\n- Alert Systems: Set up notifications based on dashboard metrics\nšŸ’” Advanced Features\nCustom Attributes\n- Search by Attributes: Find traces using custom metadata you've added\n- Attribute Filtering: Create complex filters using custom attributes\n- Attribute Visualization: See custom data alongside standard metrics\nBulk Operations\n- Multi-Select: Select multiple traces for batch operations\n- Bulk Export: Download multiple traces simultaneously\n- Comparative Analysis: Compare performance across multiple traces\nReal-Time Updates\n- Live Refresh: Automatic updates as new traces arrive\n- Connection Monitoring: Real-time status of your trace ingestion pipeline\n- Performance Indicators: Live metrics for system health monitoring\nReady to dive deeper? Explore Projects & Environments to organize your AI applications, or check out Team Collaboration to share insights with your team.\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year.", + "content_length": 6265, + "internal_links": [], + "scraped_at": 1759935923.99634 + }, + { + "url": "https://noveum.ai/en/careers/apply/senior-ai-engineer", + "title": "AI Observability, LLM Evals & Agent Monitoring | Noveum.ai", + "content": "EngineeringFull-time\nPython AI/ML Engineer\nRemote\nFull-time\nā‚¹12-36 LPA\n100% Remote\nJob Description\nJoin Noveum's core AI team that monitors, evaluates, and improves AI agents in production. You will design rigorous eval pipelines, build and debug agentic workflows, deploy and tune models, and close the loop with observability to drive reliability, quality, and cost/performance. This is a high-ownership role working directly with founders and customers to ship end-to-end fixes and new agent capabilities.\nKey Responsibilities\n- Design and implement production-grade AI agent architectures and tools\n- Build rigorous evaluation pipelines; define metrics, datasets, and pass/fail thresholds\n- Instrument, monitor, and debug agents using tracing/observability to improve reliability\n- Deploy, fine-tune, and optimize models (latency, cost, and accuracy)\n- Collaborate with founders and customers to scope, build, and ship new agent capabilities\n- Mentor and raise the bar on engineering quality and operational excellence\nRequirements\n- 5+ years in AI/ML engineering with hands-on model development and deployment\n- Expertise in Python with PyTorch and/or TensorFlow\n- Production experience with LLMs/GenAI (OpenAI, Anthropic, etc.)\n- Proven experience building agentic systems or complex ML pipelines\n- Strong MLOps foundations: packaging, CI/CD, containers, cloud\n- Bias for ownership: able to self-unblock, deliver end-to-end, and operate independently\nPreferred Qualifications\n- Hands-on experience designing/running evals for LLMs/agents\n- Experience building new agents and tools for real customer workflows\n- Open-source contributions or public work in AI/ML\n- Next.js familiarity is a plus but not required", + "content_length": 1715, + "internal_links": [], + "scraped_at": 1759935928.3825111 + }, + { + "url": "https://noveum.ai/en/careers/apply/fullstack-developer-ai", + "title": "AI Observability, LLM Evals & Agent Monitoring | Noveum.ai", + "content": "EngineeringFull-time\nFullStack Next.js Engineer\nRemote\nFull-time\nā‚¹12-36 LPA\n100% Remote\nJob Description\nBuild end-to-end product features using Next.js App Router, TypeScript, Tailwind CSS, and Node.js.\nKey Responsibilities\n- Own features across UI and backend with Next.js App Router\n- Implement accessible, responsive UI using Tailwind CSS and shadcn/ui\n- Design and integrate Node.js APIs with Prisma/Postgres\n- Collaborate with design/product to ship high-quality experiences\n- Write tests and ensure performance, security, and reliability\nRequirements\n- 3-6+ years building web apps with React/Next.js\n- Strong TypeScript and modern React patterns\n- Next.js (App Router), server components, SSR/ISR\n- Tailwind CSS, shadcn/ui, Radix UI\n- Node.js APIs, Prisma, Postgres\n- Git, CI, and basic Docker knowledge\nPreferred Qualifications\n- Experience with AI/LLM integrations\n- Performance optimization and accessibility mindset\n- Experience in monorepos and pnpm", + "content_length": 961, + "internal_links": [], + "scraped_at": 1759935929.383203 + }, + { + "url": "https://noveum.ai/", + "title": "AI Observability, LLM Evals & Agent Monitoring | Noveum.ai", + "content": "Monitor all your AI Agents\nimprove AI Agents today\nNoveum.ai helps you monitor, trace, and optimize your AI applications.\nNoveum.ai works with any AI framework ā€“ LangChain, CrewAI, AutoGen, custom implementations, or direct LLM calls. One dashboard shows everything.\nMonitor, Evaluate, Improve Your AI Agents\nThe control plane for AI agents.\nMonitor Everything, Miss Nothing\nOur lightweight SDKs capture every trace and span across your AI agent ecosystemā€”from simple LLM calls to complex multi-agent workflows. Get complete visibility without performance overhead.\nStart MonitoringEvaluate with 30+ Advanced Metrics\nNovaEval automatically scores every agent interaction using our comprehensive evaluation framework. Track accuracy, semantic similarity, safety, bias, and custom business metrics in real-time.\nView EvaluationsImprove Automatically with NovaPilot\nOur AI engineer analyzes performance data and automatically generates fixes for failing agents. Get detailed reports on model changes, prompt optimizations, and tool improvementsā€”all without human intervention.\nTry Auto-ImprovementEnterprise Ready\nNoveum.ai is built for enterprise-scale AI applications, with support for multi-tenant, multi-region deployments and advanced security features.\nContact Saleswith the world's favorite AI Observability Platform\nEverything You Need to Master AI Agent Operations\nNoveum.ai helps you monitor, trace, and optimize your AI applications with comprehensive observability tools designed for modern LLM workflows.\nSee Every Agent, Every Interaction, Every Decision\n30+ Metrics That Actually Matter for Business\nYour AI Engineer That Never Sleeps\n100% visibility on every AI agent\nReduce AI Incidents by 85%\nGet comprehensive AI monitoring with automated incident prevention, faster debugging, and built-in compliance tools.\nInstead of spending days investigating AI agent failures, your team gets instant insights into what went wrong and how to fix it. Detailed traces and automated analysis eliminate guesswork.\n0+\nAI FrameworksWith the world's favorite AI observability platform\nEasy integration with your AI stack\nNoveum.ai integrates seamlessly with all popular AI frameworks and providers, giving you comprehensive observability across your entire AI pipeline.\nWorks great with: LangChain, OpenAI, Anthropic, AWS Bedrock, Azure OpenAI, Google Cloud (Vertex AI), CrewAI, LangGraph, LlamaIndex, AutoGen, custom SDKs, and more\nwith the world's favorite AI observability platform\nTrusted AI monitoring tools by thousands of developers\n0+\nAI Eval Metrics0.0%\nuptime SLA0M+\ntraces processed", + "content_length": 2591, + "internal_links": [], + "scraped_at": 1759935930.515405 + }, + { + "url": "https://noveum.ai/auth/forgot-password", + "title": "Forgot your password? | Noveum.ai", + "content": "Forgot your password?\nPlease enter your email address and we will send you a link to reset your password.\nEmail\nSend link\nBack to signin", + "content_length": 136, + "internal_links": [], + "scraped_at": 1759935931.469873 + }, + { + "url": "https://noveum.ai/auth/signup", + "title": "Create an account | Noveum.ai", + "content": "Create an account\nWe are happy that you want to join us. Please fill in the form below to create your account.\nOr continue with\nAlready have an account? Sign in\nWe are happy that you want to join us. Please fill in the form below to create your account.\nOr continue with", + "content_length": 270, + "internal_links": [], + "scraped_at": 1759935932.3582091 + } +] \ No newline at end of file diff --git a/noveum_customer_support_bt/noveum_user_bot.ipynb b/noveum_customer_support_bt/noveum_user_bot.ipynb new file mode 100644 index 0000000..85fc754 --- /dev/null +++ b/noveum_customer_support_bt/noveum_user_bot.ipynb @@ -0,0 +1,7994 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "9eb0a8a7", + "metadata": {}, + "outputs": [], + "source": [ + "# Fix: Upload Dataset with Proper Environment Loading\n", + "!cd /Users/mramanindia/work/NovaEval/noveum_customer_support_bt && source .env && python upload_dataset.py --dataset-json split_datasets/agent.rag_evaluation_metrics_dataset.json --item-type conversation\n" + ] + }, + { + "cell_type": "markdown", + "id": "780ff8d0", + "metadata": {}, + "source": [ + "# Noveum AI Agent with RAG + Web Search\n", + "\n", + "An intelligent conversational agent that dynamically routes queries between **RAG (Retrieval-Augmented Generation)** for Noveum.ai-specific information and **Web Search** for external knowledge, providing comprehensive answers with full observability.\n", + "\n", + "## šŸš€ What This Agent Does\n", + "\n", + "### Core Functionality\n", + "- **Intelligent Query Routing**: Automatically determines whether to use RAG or Web Search based on query content\n", + "- **Dual Knowledge Sources**: \n", + " - **RAG Mode**: Answers questions about Noveum.ai platform using scraped documentation\n", + " - **Web Search Mode**: Handles external queries using real-time web search\n", + "- **Comprehensive Tracing**: Full observability with detailed metrics and performance tracking\n", + "- **Modular Architecture**: Clean separation of concerns for easy maintenance and extension\n", + "\n", + "### Key Capabilities\n", + "- šŸ§  **Document Intelligence**: Scrapes and indexes Noveum.ai website content for semantic search\n", + "- šŸŒ **Real-time Web Search**: Uses DuckDuckGo for current events and external knowledge\n", + "- šŸŽÆ **Smart Classification**: LLM-powered query routing with keyword fallback\n", + "- šŸ“Š **Performance Monitoring**: Detailed metrics on response quality, latency, and token usage\n", + "- šŸ”„ **Scalable Design**: Easy to extend with new data sources or routing logic\n", + "\n", + "## šŸ“‹ Prerequisites & Requirements\n", + "\n", + "### Required Environment Variables\n", + "```bash\n", + "NOVEUM_API_KEY=your_noveum_api_key_here\n", + "OPENAI_API_KEY=your_openai_api_key_here\n", + "```\n", + "\n", + "### Required Python Packages\n", + "- `requests` - HTTP requests for web scraping\n", + "- `beautifulsoup4` - HTML parsing\n", + "- `trafilatura` - Advanced text extraction\n", + "- `langchain` - LLM framework and vector operations\n", + "- `langchain-openai` - OpenAI integration\n", + "- `langchain-community` - Community tools (FAISS, DuckDuckGo)\n", + "- `noveum-trace` - Observability and tracing\n", + "- `python-dotenv` - Environment variable management\n", + "\n", + "### System Requirements\n", + "- Python 3.8+\n", + "- Internet connection for web scraping and API calls\n", + "- ~500MB disk space for vector store and scraped data\n", + "\n", + "## šŸ—ļø Architecture Overview\n", + "\n", + "### 1. **Website Scraper** (`NoveumWebsiteScraper`)\n", + "- Recursively scrapes noveum.ai website and sub-pages\n", + "- Extracts clean text content using trafilatura\n", + "- Discovers internal links automatically\n", + "- Saves scraped data to JSON for persistence\n", + "\n", + "### 2. **RAG System** (`NoveumRAGSystem`)\n", + "- Loads scraped documents and creates vector embeddings\n", + "- Uses FAISS for fast similarity search\n", + "- Generates context-aware responses using OpenAI GPT-4o-mini\n", + "- Tracks retrieval effectiveness and response quality\n", + "\n", + "### 3. **Web Search System** (`NoveumWebSearchSystem`)\n", + "- Integrates DuckDuckGo search for external queries\n", + "- Synthesizes information from multiple web sources\n", + "- Handles real-time information and current events\n", + "- Formats search results into coherent responses\n", + "\n", + "### 4. **Query Router** (`NoveumQueryRouter`)\n", + "- **Keyword-based classification**: Matches queries against predefined keyword lists\n", + "- **LLM-based classification**: Uses GPT-4o-mini for complex query analysis\n", + "- **Confidence scoring**: Evaluates routing decision quality\n", + "- **Fallback handling**: Defaults to Web Search for ambiguous queries\n", + "\n", + "### 5. **Main Agent** (`NoveumAIAgent`)\n", + "- Orchestrates all components\n", + "- Manages system initialization and data loading\n", + "- Provides unified interface for query processing\n", + "- Handles error recovery and response formatting\n", + "\n", + "## šŸŽÆ How to Use\n", + "\n", + "### Quick Start\n", + "```python\n", + "# 1. Initialize the system (first time only)\n", + "noveum_agent.initialize_system(force_scrape=True)\n", + "\n", + "# 2. Ask questions\n", + "response = noveum_agent.process_query(\"What is Noveum and what does it do?\")\n", + "noveum_agent.display_response(response)\n", + "\n", + "# 3. Or use convenience function\n", + "ask_question(\"How do I integrate Noveum Trace?\")\n", + "```\n", + "\n", + "### Advanced Usage\n", + "```python\n", + "# Run full demo with 20 test queries\n", + "demo_noveum_agent()\n", + "\n", + "# Process queries programmatically\n", + "response = noveum_agent.process_query(\"What are the latest AI news?\")\n", + "print(f\"Mode: {response['mode']}\")\n", + "print(f\"Answer: {response['answer']}\")\n", + "print(f\"Sources: {response['sources']}\")\n", + "```\n", + "\n", + "### Query Types\n", + "\n", + "#### RAG Queries (Noveum-specific)\n", + "- \"What is Noveum and what does it do?\"\n", + "- \"How do I integrate Noveum Trace?\"\n", + "- \"What are Noveum's pricing plans?\"\n", + "- \"What features does Noveum Trace offer?\"\n", + "- \"How do I set up observability with Noveum?\"\n", + "\n", + "#### Web Search Queries (External knowledge)\n", + "- \"What are the latest AI news today?\"\n", + "- \"What's the weather like today?\"\n", + "- \"Tell me about recent developments in machine learning\"\n", + "- \"What are the current trends in observability tools?\"\n", + "- \"What happened in tech news this week?\"\n", + "\n", + "## šŸ“Š Observability & Monitoring\n", + "\n", + "### Traced Operations\n", + "- **System Initialization**: Website scraping and vector store creation\n", + "- **Query Processing**: End-to-end query handling with performance metrics\n", + "- **RAG Operations**: Document retrieval, context generation, and response creation\n", + "- **Web Search Operations**: Search execution, result synthesis, and response generation\n", + "- **Query Routing**: Classification decision making and confidence scoring\n", + "\n", + "### Key Metrics Tracked\n", + "- **Performance**: Response latency, processing time, token usage\n", + "- **Quality**: Response length, source diversity, context utilization\n", + "- **Routing**: Classification confidence, keyword scores, decision rationale\n", + "- **Model Usage**: Token consumption, cost estimation, efficiency scores\n", + "- **Retrieval**: Document relevance, context quality, source effectiveness\n", + "\n", + "### Noveum Trace Integration\n", + "- All operations are automatically traced with detailed spans\n", + "- Comprehensive attribute tracking for debugging and optimization\n", + "- Real-time monitoring through Noveum.ai dashboard\n", + "- Export capabilities for further analysis\n", + "\n", + "## šŸ”§ Configuration\n", + "\n", + "### Default Settings\n", + "```python\n", + "CONFIG = {\n", + " \"noveum_base_url\": \"https://noveum.ai\",\n", + " \"max_pages_to_scrape\": 50,\n", + " \"chunk_size\": 1000,\n", + " \"chunk_overlap\": 200,\n", + " \"max_search_results\": 5,\n", + " \"rag_threshold\": 0.7,\n", + " \"noveum_docs_file\": \"noveum_docs.json\",\n", + " \"vector_store_path\": \"noveum_vectorstore\"\n", + "}\n", + "```\n", + "\n", + "### Customization Options\n", + "- **Scraping**: Adjust `max_pages_to_scrape` for more/less content\n", + "- **RAG**: Modify `chunk_size` and `chunk_overlap` for different text splitting\n", + "- **Search**: Change `max_search_results` for more/fewer sources\n", + "- **Routing**: Add keywords to `rag_keywords` or `web_keywords` lists\n", + "\n", + "## šŸšØ Error Handling\n", + "\n", + "### Common Issues\n", + "- **API Key Missing**: Ensure `NOVEUM_API_KEY` and `OPENAI_API_KEY` are set\n", + "- **Network Errors**: Check internet connection for scraping and API calls\n", + "- **Vector Store Issues**: Delete `noveum_vectorstore` folder to regenerate\n", + "- **Scraping Failures**: Set `force_scrape=True` to re-scrape website\n", + "\n", + "### Recovery Strategies\n", + "- Automatic fallback to Web Search for RAG failures\n", + "- Graceful error handling with informative messages\n", + "- Retry mechanisms for transient network issues\n", + "- Detailed error logging for debugging\n", + "\n", + "## šŸ”„ Maintenance\n", + "\n", + "### Regular Tasks\n", + "- **Update Scraped Content**: Run with `force_scrape=True` periodically\n", + "- **Monitor Performance**: Check Noveum Trace dashboard for metrics\n", + "- **Review Routing**: Analyze query classification accuracy\n", + "- **Update Keywords**: Add new terms to routing keyword lists\n", + "\n", + "### Scaling Considerations\n", + "- **Vector Store**: Can be shared across multiple agent instances\n", + "- **Scraped Data**: JSON file can be versioned and distributed\n", + "- **API Limits**: Monitor OpenAI token usage and costs\n", + "- **Performance**: Consider caching for frequently asked questions\n" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "88c54bec", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Collecting requests==2.32.3 (from -r ./noveum_agent_requirements.txt (line 3))\n", + " Using cached requests-2.32.3-py3-none-any.whl.metadata (4.6 kB)\n", + "Collecting beautifulsoup4==4.12.3 (from -r ./noveum_agent_requirements.txt (line 4))\n", + " Using cached beautifulsoup4-4.12.3-py3-none-any.whl.metadata (3.8 kB)\n", + "Collecting python-dotenv==1.0.1 (from -r ./noveum_agent_requirements.txt (line 5))\n", + " Using cached python_dotenv-1.0.1-py3-none-any.whl.metadata (23 kB)\n", + "Collecting langchain==0.3.26 (from -r ./noveum_agent_requirements.txt (line 8))\n", + " Using cached langchain-0.3.26-py3-none-any.whl.metadata (7.8 kB)\n", + "Collecting langchain-community==0.3.18 (from -r ./noveum_agent_requirements.txt (line 9))\n", + " Using cached langchain_community-0.3.18-py3-none-any.whl.metadata (2.4 kB)\n", + "Collecting langchain-core==0.3.66 (from -r ./noveum_agent_requirements.txt (line 10))\n", + " Using cached langchain_core-0.3.66-py3-none-any.whl.metadata (5.8 kB)\n", + "Collecting langchain-openai==0.3.25 (from -r ./noveum_agent_requirements.txt (line 11))\n", + " Using cached langchain_openai-0.3.25-py3-none-any.whl.metadata (2.3 kB)\n", + "Collecting trafilatura>=1.6.4 (from -r ./noveum_agent_requirements.txt (line 14))\n", + " Using cached trafilatura-2.0.0-py3-none-any.whl.metadata (12 kB)\n", + "Collecting lxml>=5.3.0 (from -r ./noveum_agent_requirements.txt (line 15))\n", + " Using cached lxml-6.0.2-cp312-cp312-macosx_10_13_universal2.whl.metadata (3.6 kB)\n", + "Collecting faiss-cpu==1.12.0 (from -r ./noveum_agent_requirements.txt (line 18))\n", + " Using cached faiss_cpu-1.12.0-cp312-cp312-macosx_14_0_arm64.whl.metadata (5.1 kB)\n", + "Collecting duckduckgo-search>=6.1.12 (from -r ./noveum_agent_requirements.txt (line 22))\n", + " Using cached duckduckgo_search-8.1.1-py3-none-any.whl.metadata (16 kB)\n", + "Requirement already satisfied: pandas>=2.2.3 in /Users/mramanindia/work/NovaEval/.venv/lib/python3.12/site-packages (from -r ./noveum_agent_requirements.txt (line 25)) (2.3.3)\n", + "Requirement already satisfied: tqdm>=4.67.1 in /Users/mramanindia/work/NovaEval/.venv/lib/python3.12/site-packages (from -r ./noveum_agent_requirements.txt (line 26)) (4.67.1)\n", + "Requirement already satisfied: noveum_trace>=0.3.5 in /Users/mramanindia/work/NovaEval/.venv/lib/python3.12/site-packages (from -r ./noveum_agent_requirements.txt (line 29)) (0.3.9)\n", + "Collecting jupyter==1.0.0 (from -r ./noveum_agent_requirements.txt (line 32))\n", + " Using cached jupyter-1.0.0-py2.py3-none-any.whl.metadata (995 bytes)\n", + "Collecting ipykernel==6.29.4 (from -r ./noveum_agent_requirements.txt (line 33))\n", + " Using cached ipykernel-6.29.4-py3-none-any.whl.metadata (6.3 kB)\n", + "Requirement already satisfied: charset-normalizer<4,>=2 in /Users/mramanindia/work/NovaEval/.venv/lib/python3.12/site-packages (from requests==2.32.3->-r ./noveum_agent_requirements.txt (line 3)) (3.4.4)\n", + "Requirement already satisfied: idna<4,>=2.5 in /Users/mramanindia/work/NovaEval/.venv/lib/python3.12/site-packages (from requests==2.32.3->-r ./noveum_agent_requirements.txt (line 3)) (3.11)\n", + "Requirement already satisfied: urllib3<3,>=1.21.1 in /Users/mramanindia/work/NovaEval/.venv/lib/python3.12/site-packages (from requests==2.32.3->-r ./noveum_agent_requirements.txt (line 3)) (2.5.0)\n", + "Requirement already satisfied: certifi>=2017.4.17 in /Users/mramanindia/work/NovaEval/.venv/lib/python3.12/site-packages (from requests==2.32.3->-r ./noveum_agent_requirements.txt (line 3)) (2025.10.5)\n", + "Collecting soupsieve>1.2 (from beautifulsoup4==4.12.3->-r ./noveum_agent_requirements.txt (line 4))\n", + " Using cached soupsieve-2.8-py3-none-any.whl.metadata (4.6 kB)\n", + "Collecting langchain-text-splitters<1.0.0,>=0.3.8 (from langchain==0.3.26->-r ./noveum_agent_requirements.txt (line 8))\n", + " Using cached langchain_text_splitters-0.3.11-py3-none-any.whl.metadata (1.8 kB)\n", + "Collecting langsmith>=0.1.17 (from langchain==0.3.26->-r ./noveum_agent_requirements.txt (line 8))\n", + " Downloading langsmith-0.4.37-py3-none-any.whl.metadata (14 kB)\n", + "Requirement already satisfied: pydantic<3.0.0,>=2.7.4 in /Users/mramanindia/work/NovaEval/.venv/lib/python3.12/site-packages (from langchain==0.3.26->-r ./noveum_agent_requirements.txt (line 8)) (2.12.3)\n", + "Collecting SQLAlchemy<3,>=1.4 (from langchain==0.3.26->-r ./noveum_agent_requirements.txt (line 8))\n", + " Using cached sqlalchemy-2.0.44-cp312-cp312-macosx_11_0_arm64.whl.metadata (9.5 kB)\n", + "Requirement already satisfied: PyYAML>=5.3 in /Users/mramanindia/work/NovaEval/.venv/lib/python3.12/site-packages (from langchain==0.3.26->-r ./noveum_agent_requirements.txt (line 8)) (6.0.3)\n", + "Collecting aiohttp<4.0.0,>=3.8.3 (from langchain-community==0.3.18->-r ./noveum_agent_requirements.txt (line 9))\n", + " Using cached aiohttp-3.13.1-cp312-cp312-macosx_11_0_arm64.whl.metadata (8.1 kB)\n", + "Requirement already satisfied: tenacity!=8.4.0,<10,>=8.1.0 in /Users/mramanindia/work/NovaEval/.venv/lib/python3.12/site-packages (from langchain-community==0.3.18->-r ./noveum_agent_requirements.txt (line 9)) (9.1.2)\n", + "Collecting dataclasses-json<0.7,>=0.5.7 (from langchain-community==0.3.18->-r ./noveum_agent_requirements.txt (line 9))\n", + " Using cached dataclasses_json-0.6.7-py3-none-any.whl.metadata (25 kB)\n", + "Requirement already satisfied: pydantic-settings<3.0.0,>=2.4.0 in /Users/mramanindia/work/NovaEval/.venv/lib/python3.12/site-packages (from langchain-community==0.3.18->-r ./noveum_agent_requirements.txt (line 9)) (2.11.0)\n", + "Collecting langsmith>=0.1.17 (from langchain==0.3.26->-r ./noveum_agent_requirements.txt (line 8))\n", + " Using cached langsmith-0.3.45-py3-none-any.whl.metadata (15 kB)\n", + "Collecting httpx-sse<1.0.0,>=0.4.0 (from langchain-community==0.3.18->-r ./noveum_agent_requirements.txt (line 9))\n", + " Using cached httpx_sse-0.4.3-py3-none-any.whl.metadata (9.7 kB)\n", + "Requirement already satisfied: numpy<3,>=1.26.2 in /Users/mramanindia/work/NovaEval/.venv/lib/python3.12/site-packages (from langchain-community==0.3.18->-r ./noveum_agent_requirements.txt (line 9)) (2.3.4)\n", + "Collecting jsonpatch<2.0,>=1.33 (from langchain-core==0.3.66->-r ./noveum_agent_requirements.txt (line 10))\n", + " Using cached jsonpatch-1.33-py2.py3-none-any.whl.metadata (3.0 kB)\n", + "Collecting packaging<25,>=23.2 (from langchain-core==0.3.66->-r ./noveum_agent_requirements.txt (line 10))\n", + " Using cached packaging-24.2-py3-none-any.whl.metadata (3.2 kB)\n", + "Requirement already satisfied: typing-extensions>=4.7 in /Users/mramanindia/work/NovaEval/.venv/lib/python3.12/site-packages (from langchain-core==0.3.66->-r ./noveum_agent_requirements.txt (line 10)) (4.15.0)\n", + "Collecting openai<2.0.0,>=1.86.0 (from langchain-openai==0.3.25->-r ./noveum_agent_requirements.txt (line 11))\n", + " Using cached openai-1.109.1-py3-none-any.whl.metadata (29 kB)\n", + "Collecting tiktoken<1,>=0.7 (from langchain-openai==0.3.25->-r ./noveum_agent_requirements.txt (line 11))\n", + " Using cached tiktoken-0.12.0-cp312-cp312-macosx_11_0_arm64.whl.metadata (6.7 kB)\n", + "Collecting notebook (from jupyter==1.0.0->-r ./noveum_agent_requirements.txt (line 32))\n", + " Using cached notebook-7.4.7-py3-none-any.whl.metadata (10 kB)\n", + "Collecting qtconsole (from jupyter==1.0.0->-r ./noveum_agent_requirements.txt (line 32))\n", + " Using cached qtconsole-5.7.0-py3-none-any.whl.metadata (5.4 kB)\n", + "Collecting jupyter-console (from jupyter==1.0.0->-r ./noveum_agent_requirements.txt (line 32))\n", + " Using cached jupyter_console-6.6.3-py3-none-any.whl.metadata (5.8 kB)\n", + "Collecting nbconvert (from jupyter==1.0.0->-r ./noveum_agent_requirements.txt (line 32))\n", + " Using cached nbconvert-7.16.6-py3-none-any.whl.metadata (8.5 kB)\n", + "Collecting ipywidgets (from jupyter==1.0.0->-r ./noveum_agent_requirements.txt (line 32))\n", + " Using cached ipywidgets-8.1.7-py3-none-any.whl.metadata (2.4 kB)\n", + "Requirement already satisfied: appnope in /Users/mramanindia/work/NovaEval/.venv/lib/python3.12/site-packages (from ipykernel==6.29.4->-r ./noveum_agent_requirements.txt (line 33)) (0.1.4)\n", + "Requirement already satisfied: comm>=0.1.1 in /Users/mramanindia/work/NovaEval/.venv/lib/python3.12/site-packages (from ipykernel==6.29.4->-r ./noveum_agent_requirements.txt (line 33)) (0.2.3)\n", + "Requirement already satisfied: debugpy>=1.6.5 in /Users/mramanindia/work/NovaEval/.venv/lib/python3.12/site-packages (from ipykernel==6.29.4->-r ./noveum_agent_requirements.txt (line 33)) (1.8.17)\n", + "Requirement already satisfied: ipython>=7.23.1 in /Users/mramanindia/work/NovaEval/.venv/lib/python3.12/site-packages (from ipykernel==6.29.4->-r ./noveum_agent_requirements.txt (line 33)) (9.6.0)\n", + "Requirement already satisfied: jupyter-client>=6.1.12 in /Users/mramanindia/work/NovaEval/.venv/lib/python3.12/site-packages (from ipykernel==6.29.4->-r ./noveum_agent_requirements.txt (line 33)) (8.6.3)\n", + "Requirement already satisfied: jupyter-core!=5.0.*,>=4.12 in /Users/mramanindia/work/NovaEval/.venv/lib/python3.12/site-packages (from ipykernel==6.29.4->-r ./noveum_agent_requirements.txt (line 33)) (5.9.1)\n", + "Requirement already satisfied: matplotlib-inline>=0.1 in /Users/mramanindia/work/NovaEval/.venv/lib/python3.12/site-packages (from ipykernel==6.29.4->-r ./noveum_agent_requirements.txt (line 33)) (0.1.7)\n", + "Requirement already satisfied: nest-asyncio in /Users/mramanindia/work/NovaEval/.venv/lib/python3.12/site-packages (from ipykernel==6.29.4->-r ./noveum_agent_requirements.txt (line 33)) (1.6.0)\n", + "Requirement already satisfied: psutil in /Users/mramanindia/work/NovaEval/.venv/lib/python3.12/site-packages (from ipykernel==6.29.4->-r ./noveum_agent_requirements.txt (line 33)) (7.1.1)\n", + "Requirement already satisfied: pyzmq>=24 in /Users/mramanindia/work/NovaEval/.venv/lib/python3.12/site-packages (from ipykernel==6.29.4->-r ./noveum_agent_requirements.txt (line 33)) (27.1.0)\n", + "Requirement already satisfied: tornado>=6.1 in /Users/mramanindia/work/NovaEval/.venv/lib/python3.12/site-packages (from ipykernel==6.29.4->-r ./noveum_agent_requirements.txt (line 33)) (6.5.2)\n", + "Requirement already satisfied: traitlets>=5.4.0 in /Users/mramanindia/work/NovaEval/.venv/lib/python3.12/site-packages (from ipykernel==6.29.4->-r ./noveum_agent_requirements.txt (line 33)) (5.14.3)\n", + "Collecting courlan>=1.3.2 (from trafilatura>=1.6.4->-r ./noveum_agent_requirements.txt (line 14))\n", + " Using cached courlan-1.3.2-py3-none-any.whl.metadata (17 kB)\n", + "Collecting htmldate>=1.9.2 (from trafilatura>=1.6.4->-r ./noveum_agent_requirements.txt (line 14))\n", + " Using cached htmldate-1.9.3-py3-none-any.whl.metadata (10 kB)\n", + "Collecting justext>=3.0.1 (from trafilatura>=1.6.4->-r ./noveum_agent_requirements.txt (line 14))\n", + " Using cached justext-3.0.2-py2.py3-none-any.whl.metadata (7.3 kB)\n", + "Requirement already satisfied: click>=8.1.8 in /Users/mramanindia/work/NovaEval/.venv/lib/python3.12/site-packages (from duckduckgo-search>=6.1.12->-r ./noveum_agent_requirements.txt (line 22)) (8.3.0)\n", + "Collecting primp>=0.15.0 (from duckduckgo-search>=6.1.12->-r ./noveum_agent_requirements.txt (line 22))\n", + " Using cached primp-0.15.0-cp38-abi3-macosx_11_0_arm64.whl.metadata (13 kB)\n", + "Requirement already satisfied: python-dateutil>=2.8.2 in /Users/mramanindia/work/NovaEval/.venv/lib/python3.12/site-packages (from pandas>=2.2.3->-r ./noveum_agent_requirements.txt (line 25)) (2.9.0.post0)\n", + "Requirement already satisfied: pytz>=2020.1 in /Users/mramanindia/work/NovaEval/.venv/lib/python3.12/site-packages (from pandas>=2.2.3->-r ./noveum_agent_requirements.txt (line 25)) (2025.2)\n", + "Requirement already satisfied: tzdata>=2022.7 in /Users/mramanindia/work/NovaEval/.venv/lib/python3.12/site-packages (from pandas>=2.2.3->-r ./noveum_agent_requirements.txt (line 25)) (2025.2)\n", + "Collecting aiohappyeyeballs>=2.5.0 (from aiohttp<4.0.0,>=3.8.3->langchain-community==0.3.18->-r ./noveum_agent_requirements.txt (line 9))\n", + " Using cached aiohappyeyeballs-2.6.1-py3-none-any.whl.metadata (5.9 kB)\n", + "Collecting aiosignal>=1.4.0 (from aiohttp<4.0.0,>=3.8.3->langchain-community==0.3.18->-r ./noveum_agent_requirements.txt (line 9))\n", + " Using cached aiosignal-1.4.0-py3-none-any.whl.metadata (3.7 kB)\n", + "Collecting attrs>=17.3.0 (from aiohttp<4.0.0,>=3.8.3->langchain-community==0.3.18->-r ./noveum_agent_requirements.txt (line 9))\n", + " Using cached attrs-25.4.0-py3-none-any.whl.metadata (10 kB)\n", + "Collecting frozenlist>=1.1.1 (from aiohttp<4.0.0,>=3.8.3->langchain-community==0.3.18->-r ./noveum_agent_requirements.txt (line 9))\n", + " Using cached frozenlist-1.8.0-cp312-cp312-macosx_11_0_arm64.whl.metadata (20 kB)\n", + "Collecting multidict<7.0,>=4.5 (from aiohttp<4.0.0,>=3.8.3->langchain-community==0.3.18->-r ./noveum_agent_requirements.txt (line 9))\n", + " Using cached multidict-6.7.0-cp312-cp312-macosx_11_0_arm64.whl.metadata (5.3 kB)\n", + "Collecting propcache>=0.2.0 (from aiohttp<4.0.0,>=3.8.3->langchain-community==0.3.18->-r ./noveum_agent_requirements.txt (line 9))\n", + " Using cached propcache-0.4.1-cp312-cp312-macosx_11_0_arm64.whl.metadata (13 kB)\n", + "Collecting yarl<2.0,>=1.17.0 (from aiohttp<4.0.0,>=3.8.3->langchain-community==0.3.18->-r ./noveum_agent_requirements.txt (line 9))\n", + " Using cached yarl-1.22.0-cp312-cp312-macosx_11_0_arm64.whl.metadata (75 kB)\n", + "Collecting babel>=2.16.0 (from courlan>=1.3.2->trafilatura>=1.6.4->-r ./noveum_agent_requirements.txt (line 14))\n", + " Using cached babel-2.17.0-py3-none-any.whl.metadata (2.0 kB)\n", + "Collecting tld>=0.13 (from courlan>=1.3.2->trafilatura>=1.6.4->-r ./noveum_agent_requirements.txt (line 14))\n", + " Using cached tld-0.13.1-py2.py3-none-any.whl.metadata (10 kB)\n", + "Collecting marshmallow<4.0.0,>=3.18.0 (from dataclasses-json<0.7,>=0.5.7->langchain-community==0.3.18->-r ./noveum_agent_requirements.txt (line 9))\n", + " Using cached marshmallow-3.26.1-py3-none-any.whl.metadata (7.3 kB)\n", + "Collecting typing-inspect<1,>=0.4.0 (from dataclasses-json<0.7,>=0.5.7->langchain-community==0.3.18->-r ./noveum_agent_requirements.txt (line 9))\n", + " Using cached typing_inspect-0.9.0-py3-none-any.whl.metadata (1.5 kB)\n", + "Collecting dateparser>=1.1.2 (from htmldate>=1.9.2->trafilatura>=1.6.4->-r ./noveum_agent_requirements.txt (line 14))\n", + " Using cached dateparser-1.2.2-py3-none-any.whl.metadata (29 kB)\n", + "Collecting lxml>=5.3.0 (from -r ./noveum_agent_requirements.txt (line 15))\n", + " Using cached lxml-5.4.0-cp312-cp312-macosx_10_9_universal2.whl.metadata (3.5 kB)\n", + "Requirement already satisfied: decorator in /Users/mramanindia/work/NovaEval/.venv/lib/python3.12/site-packages (from ipython>=7.23.1->ipykernel==6.29.4->-r ./noveum_agent_requirements.txt (line 33)) (5.2.1)\n", + "Requirement already satisfied: ipython-pygments-lexers in /Users/mramanindia/work/NovaEval/.venv/lib/python3.12/site-packages (from ipython>=7.23.1->ipykernel==6.29.4->-r ./noveum_agent_requirements.txt (line 33)) (1.1.1)\n", + "Requirement already satisfied: jedi>=0.16 in /Users/mramanindia/work/NovaEval/.venv/lib/python3.12/site-packages (from ipython>=7.23.1->ipykernel==6.29.4->-r ./noveum_agent_requirements.txt (line 33)) (0.19.2)\n", + "Requirement already satisfied: pexpect>4.3 in /Users/mramanindia/work/NovaEval/.venv/lib/python3.12/site-packages (from ipython>=7.23.1->ipykernel==6.29.4->-r ./noveum_agent_requirements.txt (line 33)) (4.9.0)\n", + "Requirement already satisfied: prompt_toolkit<3.1.0,>=3.0.41 in /Users/mramanindia/work/NovaEval/.venv/lib/python3.12/site-packages (from ipython>=7.23.1->ipykernel==6.29.4->-r ./noveum_agent_requirements.txt (line 33)) (3.0.51)\n", + "Requirement already satisfied: pygments>=2.4.0 in /Users/mramanindia/work/NovaEval/.venv/lib/python3.12/site-packages (from ipython>=7.23.1->ipykernel==6.29.4->-r ./noveum_agent_requirements.txt (line 33)) (2.19.2)\n", + "Requirement already satisfied: stack_data in /Users/mramanindia/work/NovaEval/.venv/lib/python3.12/site-packages (from ipython>=7.23.1->ipykernel==6.29.4->-r ./noveum_agent_requirements.txt (line 33)) (0.6.3)\n", + "Collecting jsonpointer>=1.9 (from jsonpatch<2.0,>=1.33->langchain-core==0.3.66->-r ./noveum_agent_requirements.txt (line 10))\n", + " Using cached jsonpointer-3.0.0-py2.py3-none-any.whl.metadata (2.3 kB)\n", + "Requirement already satisfied: platformdirs>=2.5 in /Users/mramanindia/work/NovaEval/.venv/lib/python3.12/site-packages (from jupyter-core!=5.0.*,>=4.12->ipykernel==6.29.4->-r ./noveum_agent_requirements.txt (line 33)) (4.5.0)\n", + "INFO: pip is looking at multiple versions of langchain-text-splitters to determine which version is compatible with other requirements. This could take a while.\n", + "Collecting langchain-text-splitters<1.0.0,>=0.3.8 (from langchain==0.3.26->-r ./noveum_agent_requirements.txt (line 8))\n", + " Using cached langchain_text_splitters-0.3.10-py3-none-any.whl.metadata (1.9 kB)\n", + " Using cached langchain_text_splitters-0.3.9-py3-none-any.whl.metadata (1.9 kB)\n", + " Using cached langchain_text_splitters-0.3.8-py3-none-any.whl.metadata (1.9 kB)\n", + "Requirement already satisfied: httpx<1,>=0.23.0 in /Users/mramanindia/work/NovaEval/.venv/lib/python3.12/site-packages (from langsmith>=0.1.17->langchain==0.3.26->-r ./noveum_agent_requirements.txt (line 8)) (0.28.1)\n", + "Collecting orjson<4.0.0,>=3.9.14 (from langsmith>=0.1.17->langchain==0.3.26->-r ./noveum_agent_requirements.txt (line 8))\n", + " Using cached orjson-3.11.3-cp312-cp312-macosx_15_0_arm64.whl.metadata (41 kB)\n", + "Collecting requests-toolbelt<2.0.0,>=1.0.0 (from langsmith>=0.1.17->langchain==0.3.26->-r ./noveum_agent_requirements.txt (line 8))\n", + " Using cached requests_toolbelt-1.0.0-py2.py3-none-any.whl.metadata (14 kB)\n", + "Collecting zstandard<0.24.0,>=0.23.0 (from langsmith>=0.1.17->langchain==0.3.26->-r ./noveum_agent_requirements.txt (line 8))\n", + " Using cached zstandard-0.23.0-cp312-cp312-macosx_11_0_arm64.whl.metadata (3.0 kB)\n", + "Requirement already satisfied: anyio<5,>=3.5.0 in /Users/mramanindia/work/NovaEval/.venv/lib/python3.12/site-packages (from openai<2.0.0,>=1.86.0->langchain-openai==0.3.25->-r ./noveum_agent_requirements.txt (line 11)) (4.11.0)\n", + "Collecting distro<2,>=1.7.0 (from openai<2.0.0,>=1.86.0->langchain-openai==0.3.25->-r ./noveum_agent_requirements.txt (line 11))\n", + " Using cached distro-1.9.0-py3-none-any.whl.metadata (6.8 kB)\n", + "Collecting jiter<1,>=0.4.0 (from openai<2.0.0,>=1.86.0->langchain-openai==0.3.25->-r ./noveum_agent_requirements.txt (line 11))\n", + " Using cached jiter-0.11.1-cp312-cp312-macosx_11_0_arm64.whl.metadata (5.2 kB)\n", + "Requirement already satisfied: sniffio in /Users/mramanindia/work/NovaEval/.venv/lib/python3.12/site-packages (from openai<2.0.0,>=1.86.0->langchain-openai==0.3.25->-r ./noveum_agent_requirements.txt (line 11)) (1.3.1)\n", + "Requirement already satisfied: annotated-types>=0.6.0 in /Users/mramanindia/work/NovaEval/.venv/lib/python3.12/site-packages (from pydantic<3.0.0,>=2.7.4->langchain==0.3.26->-r ./noveum_agent_requirements.txt (line 8)) (0.7.0)\n", + "Requirement already satisfied: pydantic-core==2.41.4 in /Users/mramanindia/work/NovaEval/.venv/lib/python3.12/site-packages (from pydantic<3.0.0,>=2.7.4->langchain==0.3.26->-r ./noveum_agent_requirements.txt (line 8)) (2.41.4)\n", + "Requirement already satisfied: typing-inspection>=0.4.2 in /Users/mramanindia/work/NovaEval/.venv/lib/python3.12/site-packages (from pydantic<3.0.0,>=2.7.4->langchain==0.3.26->-r ./noveum_agent_requirements.txt (line 8)) (0.4.2)\n", + "Requirement already satisfied: six>=1.5 in /Users/mramanindia/work/NovaEval/.venv/lib/python3.12/site-packages (from python-dateutil>=2.8.2->pandas>=2.2.3->-r ./noveum_agent_requirements.txt (line 25)) (1.17.0)\n", + "Requirement already satisfied: regex>=2022.1.18 in /Users/mramanindia/work/NovaEval/.venv/lib/python3.12/site-packages (from tiktoken<1,>=0.7->langchain-openai==0.3.25->-r ./noveum_agent_requirements.txt (line 11)) (2025.10.23)\n", + "Collecting widgetsnbextension~=4.0.14 (from ipywidgets->jupyter==1.0.0->-r ./noveum_agent_requirements.txt (line 32))\n", + " Using cached widgetsnbextension-4.0.14-py3-none-any.whl.metadata (1.6 kB)\n", + "Collecting jupyterlab_widgets~=3.0.15 (from ipywidgets->jupyter==1.0.0->-r ./noveum_agent_requirements.txt (line 32))\n", + " Using cached jupyterlab_widgets-3.0.15-py3-none-any.whl.metadata (20 kB)\n", + "Collecting bleach!=5.0.0 (from bleach[css]!=5.0.0->nbconvert->jupyter==1.0.0->-r ./noveum_agent_requirements.txt (line 32))\n", + " Using cached bleach-6.2.0-py3-none-any.whl.metadata (30 kB)\n", + "Collecting defusedxml (from nbconvert->jupyter==1.0.0->-r ./noveum_agent_requirements.txt (line 32))\n", + " Using cached defusedxml-0.7.1-py2.py3-none-any.whl.metadata (32 kB)\n", + "Requirement already satisfied: jinja2>=3.0 in /Users/mramanindia/work/NovaEval/.venv/lib/python3.12/site-packages (from nbconvert->jupyter==1.0.0->-r ./noveum_agent_requirements.txt (line 32)) (3.1.6)\n", + "Collecting jupyterlab-pygments (from nbconvert->jupyter==1.0.0->-r ./noveum_agent_requirements.txt (line 32))\n", + " Using cached jupyterlab_pygments-0.3.0-py3-none-any.whl.metadata (4.4 kB)\n", + "Requirement already satisfied: markupsafe>=2.0 in /Users/mramanindia/work/NovaEval/.venv/lib/python3.12/site-packages (from nbconvert->jupyter==1.0.0->-r ./noveum_agent_requirements.txt (line 32)) (3.0.3)\n", + "Collecting mistune<4,>=2.0.3 (from nbconvert->jupyter==1.0.0->-r ./noveum_agent_requirements.txt (line 32))\n", + " Using cached mistune-3.1.4-py3-none-any.whl.metadata (1.8 kB)\n", + "Collecting nbclient>=0.5.0 (from nbconvert->jupyter==1.0.0->-r ./noveum_agent_requirements.txt (line 32))\n", + " Using cached nbclient-0.10.2-py3-none-any.whl.metadata (8.3 kB)\n", + "Collecting nbformat>=5.7 (from nbconvert->jupyter==1.0.0->-r ./noveum_agent_requirements.txt (line 32))\n", + " Using cached nbformat-5.10.4-py3-none-any.whl.metadata (3.6 kB)\n", + "Collecting pandocfilters>=1.4.1 (from nbconvert->jupyter==1.0.0->-r ./noveum_agent_requirements.txt (line 32))\n", + " Using cached pandocfilters-1.5.1-py2.py3-none-any.whl.metadata (9.0 kB)\n", + "Collecting jupyter-server<3,>=2.4.0 (from notebook->jupyter==1.0.0->-r ./noveum_agent_requirements.txt (line 32))\n", + " Using cached jupyter_server-2.17.0-py3-none-any.whl.metadata (8.5 kB)\n", + "Collecting jupyterlab-server<3,>=2.27.1 (from notebook->jupyter==1.0.0->-r ./noveum_agent_requirements.txt (line 32))\n", + " Downloading jupyterlab_server-2.28.0-py3-none-any.whl.metadata (5.9 kB)\n", + "Collecting jupyterlab<4.5,>=4.4.9 (from notebook->jupyter==1.0.0->-r ./noveum_agent_requirements.txt (line 32))\n", + " Downloading jupyterlab-4.4.10-py3-none-any.whl.metadata (16 kB)\n", + "Collecting notebook-shim<0.3,>=0.2 (from notebook->jupyter==1.0.0->-r ./noveum_agent_requirements.txt (line 32))\n", + " Using cached notebook_shim-0.2.4-py3-none-any.whl.metadata (4.0 kB)\n", + "Collecting qtpy>=2.4.0 (from qtconsole->jupyter==1.0.0->-r ./noveum_agent_requirements.txt (line 32))\n", + " Using cached QtPy-2.4.3-py3-none-any.whl.metadata (12 kB)\n", + "Collecting webencodings (from bleach!=5.0.0->bleach[css]!=5.0.0->nbconvert->jupyter==1.0.0->-r ./noveum_agent_requirements.txt (line 32))\n", + " Using cached webencodings-0.5.1-py2.py3-none-any.whl.metadata (2.1 kB)\n", + "Collecting tinycss2<1.5,>=1.1.0 (from bleach[css]!=5.0.0->nbconvert->jupyter==1.0.0->-r ./noveum_agent_requirements.txt (line 32))\n", + " Using cached tinycss2-1.4.0-py3-none-any.whl.metadata (3.0 kB)\n", + "Collecting tzlocal>=0.2 (from dateparser>=1.1.2->htmldate>=1.9.2->trafilatura>=1.6.4->-r ./noveum_agent_requirements.txt (line 14))\n", + " Using cached tzlocal-5.3.1-py3-none-any.whl.metadata (7.6 kB)\n", + "Requirement already satisfied: httpcore==1.* in /Users/mramanindia/work/NovaEval/.venv/lib/python3.12/site-packages (from httpx<1,>=0.23.0->langsmith>=0.1.17->langchain==0.3.26->-r ./noveum_agent_requirements.txt (line 8)) (1.0.9)\n", + "Requirement already satisfied: h11>=0.16 in /Users/mramanindia/work/NovaEval/.venv/lib/python3.12/site-packages (from httpcore==1.*->httpx<1,>=0.23.0->langsmith>=0.1.17->langchain==0.3.26->-r ./noveum_agent_requirements.txt (line 8)) (0.16.0)\n", + "Requirement already satisfied: parso<0.9.0,>=0.8.4 in /Users/mramanindia/work/NovaEval/.venv/lib/python3.12/site-packages (from jedi>=0.16->ipython>=7.23.1->ipykernel==6.29.4->-r ./noveum_agent_requirements.txt (line 33)) (0.8.5)\n", + "Collecting argon2-cffi>=21.1 (from jupyter-server<3,>=2.4.0->notebook->jupyter==1.0.0->-r ./noveum_agent_requirements.txt (line 32))\n", + " Using cached argon2_cffi-25.1.0-py3-none-any.whl.metadata (4.1 kB)\n", + "Collecting jupyter-events>=0.11.0 (from jupyter-server<3,>=2.4.0->notebook->jupyter==1.0.0->-r ./noveum_agent_requirements.txt (line 32))\n", + " Using cached jupyter_events-0.12.0-py3-none-any.whl.metadata (5.8 kB)\n", + "Collecting jupyter-server-terminals>=0.4.4 (from jupyter-server<3,>=2.4.0->notebook->jupyter==1.0.0->-r ./noveum_agent_requirements.txt (line 32))\n", + " Using cached jupyter_server_terminals-0.5.3-py3-none-any.whl.metadata (5.6 kB)\n", + "Collecting prometheus-client>=0.9 (from jupyter-server<3,>=2.4.0->notebook->jupyter==1.0.0->-r ./noveum_agent_requirements.txt (line 32))\n", + " Using cached prometheus_client-0.23.1-py3-none-any.whl.metadata (1.9 kB)\n", + "Collecting send2trash>=1.8.2 (from jupyter-server<3,>=2.4.0->notebook->jupyter==1.0.0->-r ./noveum_agent_requirements.txt (line 32))\n", + " Using cached Send2Trash-1.8.3-py3-none-any.whl.metadata (4.0 kB)\n", + "Collecting terminado>=0.8.3 (from jupyter-server<3,>=2.4.0->notebook->jupyter==1.0.0->-r ./noveum_agent_requirements.txt (line 32))\n", + " Using cached terminado-0.18.1-py3-none-any.whl.metadata (5.8 kB)\n", + "Collecting websocket-client>=1.7 (from jupyter-server<3,>=2.4.0->notebook->jupyter==1.0.0->-r ./noveum_agent_requirements.txt (line 32))\n", + " Using cached websocket_client-1.9.0-py3-none-any.whl.metadata (8.3 kB)\n", + "Collecting async-lru>=1.0.0 (from jupyterlab<4.5,>=4.4.9->notebook->jupyter==1.0.0->-r ./noveum_agent_requirements.txt (line 32))\n", + " Using cached async_lru-2.0.5-py3-none-any.whl.metadata (4.5 kB)\n", + "Collecting jupyter-lsp>=2.0.0 (from jupyterlab<4.5,>=4.4.9->notebook->jupyter==1.0.0->-r ./noveum_agent_requirements.txt (line 32))\n", + " Using cached jupyter_lsp-2.3.0-py3-none-any.whl.metadata (1.8 kB)\n", + "Requirement already satisfied: setuptools>=41.1.0 in /Users/mramanindia/work/NovaEval/.venv/lib/python3.12/site-packages (from jupyterlab<4.5,>=4.4.9->notebook->jupyter==1.0.0->-r ./noveum_agent_requirements.txt (line 32)) (80.9.0)\n", + "Collecting json5>=0.9.0 (from jupyterlab-server<3,>=2.27.1->notebook->jupyter==1.0.0->-r ./noveum_agent_requirements.txt (line 32))\n", + " Using cached json5-0.12.1-py3-none-any.whl.metadata (36 kB)\n", + "Collecting jsonschema>=4.18.0 (from jupyterlab-server<3,>=2.27.1->notebook->jupyter==1.0.0->-r ./noveum_agent_requirements.txt (line 32))\n", + " Using cached jsonschema-4.25.1-py3-none-any.whl.metadata (7.6 kB)\n", + "Collecting lxml_html_clean (from lxml[html_clean]>=4.4.2->justext>=3.0.1->trafilatura>=1.6.4->-r ./noveum_agent_requirements.txt (line 14))\n", + " Using cached lxml_html_clean-0.4.3-py3-none-any.whl.metadata (2.3 kB)\n", + "Collecting fastjsonschema>=2.15 (from nbformat>=5.7->nbconvert->jupyter==1.0.0->-r ./noveum_agent_requirements.txt (line 32))\n", + " Using cached fastjsonschema-2.21.2-py3-none-any.whl.metadata (2.3 kB)\n", + "Requirement already satisfied: ptyprocess>=0.5 in /Users/mramanindia/work/NovaEval/.venv/lib/python3.12/site-packages (from pexpect>4.3->ipython>=7.23.1->ipykernel==6.29.4->-r ./noveum_agent_requirements.txt (line 33)) (0.7.0)\n", + "Requirement already satisfied: wcwidth in /Users/mramanindia/work/NovaEval/.venv/lib/python3.12/site-packages (from prompt_toolkit<3.1.0,>=3.0.41->ipython>=7.23.1->ipykernel==6.29.4->-r ./noveum_agent_requirements.txt (line 33)) (0.2.14)\n", + "Requirement already satisfied: mypy-extensions>=0.3.0 in /Users/mramanindia/work/NovaEval/.venv/lib/python3.12/site-packages (from typing-inspect<1,>=0.4.0->dataclasses-json<0.7,>=0.5.7->langchain-community==0.3.18->-r ./noveum_agent_requirements.txt (line 9)) (1.1.0)\n", + "Requirement already satisfied: executing>=1.2.0 in /Users/mramanindia/work/NovaEval/.venv/lib/python3.12/site-packages (from stack_data->ipython>=7.23.1->ipykernel==6.29.4->-r ./noveum_agent_requirements.txt (line 33)) (2.2.1)\n", + "Requirement already satisfied: asttokens>=2.1.0 in /Users/mramanindia/work/NovaEval/.venv/lib/python3.12/site-packages (from stack_data->ipython>=7.23.1->ipykernel==6.29.4->-r ./noveum_agent_requirements.txt (line 33)) (3.0.0)\n", + "Requirement already satisfied: pure-eval in /Users/mramanindia/work/NovaEval/.venv/lib/python3.12/site-packages (from stack_data->ipython>=7.23.1->ipykernel==6.29.4->-r ./noveum_agent_requirements.txt (line 33)) (0.2.3)\n", + "Collecting argon2-cffi-bindings (from argon2-cffi>=21.1->jupyter-server<3,>=2.4.0->notebook->jupyter==1.0.0->-r ./noveum_agent_requirements.txt (line 32))\n", + " Using cached argon2_cffi_bindings-25.1.0-cp39-abi3-macosx_11_0_arm64.whl.metadata (7.4 kB)\n", + "Collecting jsonschema-specifications>=2023.03.6 (from jsonschema>=4.18.0->jupyterlab-server<3,>=2.27.1->notebook->jupyter==1.0.0->-r ./noveum_agent_requirements.txt (line 32))\n", + " Using cached jsonschema_specifications-2025.9.1-py3-none-any.whl.metadata (2.9 kB)\n", + "Collecting referencing>=0.28.4 (from jsonschema>=4.18.0->jupyterlab-server<3,>=2.27.1->notebook->jupyter==1.0.0->-r ./noveum_agent_requirements.txt (line 32))\n", + " Using cached referencing-0.37.0-py3-none-any.whl.metadata (2.8 kB)\n", + "Collecting rpds-py>=0.7.1 (from jsonschema>=4.18.0->jupyterlab-server<3,>=2.27.1->notebook->jupyter==1.0.0->-r ./noveum_agent_requirements.txt (line 32))\n", + " Using cached rpds_py-0.27.1-cp312-cp312-macosx_11_0_arm64.whl.metadata (4.2 kB)\n", + "Collecting python-json-logger>=2.0.4 (from jupyter-events>=0.11.0->jupyter-server<3,>=2.4.0->notebook->jupyter==1.0.0->-r ./noveum_agent_requirements.txt (line 32))\n", + " Using cached python_json_logger-4.0.0-py3-none-any.whl.metadata (4.0 kB)\n", + "Collecting rfc3339-validator (from jupyter-events>=0.11.0->jupyter-server<3,>=2.4.0->notebook->jupyter==1.0.0->-r ./noveum_agent_requirements.txt (line 32))\n", + " Using cached rfc3339_validator-0.1.4-py2.py3-none-any.whl.metadata (1.5 kB)\n", + "Collecting rfc3986-validator>=0.1.1 (from jupyter-events>=0.11.0->jupyter-server<3,>=2.4.0->notebook->jupyter==1.0.0->-r ./noveum_agent_requirements.txt (line 32))\n", + " Using cached rfc3986_validator-0.1.1-py2.py3-none-any.whl.metadata (1.7 kB)\n", + "Collecting fqdn (from jsonschema[format-nongpl]>=4.18.0->jupyter-events>=0.11.0->jupyter-server<3,>=2.4.0->notebook->jupyter==1.0.0->-r ./noveum_agent_requirements.txt (line 32))\n", + " Using cached fqdn-1.5.1-py3-none-any.whl.metadata (1.4 kB)\n", + "Collecting isoduration (from jsonschema[format-nongpl]>=4.18.0->jupyter-events>=0.11.0->jupyter-server<3,>=2.4.0->notebook->jupyter==1.0.0->-r ./noveum_agent_requirements.txt (line 32))\n", + " Using cached isoduration-20.11.0-py3-none-any.whl.metadata (5.7 kB)\n", + "Collecting rfc3987-syntax>=1.1.0 (from jsonschema[format-nongpl]>=4.18.0->jupyter-events>=0.11.0->jupyter-server<3,>=2.4.0->notebook->jupyter==1.0.0->-r ./noveum_agent_requirements.txt (line 32))\n", + " Using cached rfc3987_syntax-1.1.0-py3-none-any.whl.metadata (7.7 kB)\n", + "Collecting uri-template (from jsonschema[format-nongpl]>=4.18.0->jupyter-events>=0.11.0->jupyter-server<3,>=2.4.0->notebook->jupyter==1.0.0->-r ./noveum_agent_requirements.txt (line 32))\n", + " Using cached uri_template-1.3.0-py3-none-any.whl.metadata (8.8 kB)\n", + "Collecting webcolors>=24.6.0 (from jsonschema[format-nongpl]>=4.18.0->jupyter-events>=0.11.0->jupyter-server<3,>=2.4.0->notebook->jupyter==1.0.0->-r ./noveum_agent_requirements.txt (line 32))\n", + " Using cached webcolors-24.11.1-py3-none-any.whl.metadata (2.2 kB)\n", + "Requirement already satisfied: cffi>=1.0.1 in /Users/mramanindia/work/NovaEval/.venv/lib/python3.12/site-packages (from argon2-cffi-bindings->argon2-cffi>=21.1->jupyter-server<3,>=2.4.0->notebook->jupyter==1.0.0->-r ./noveum_agent_requirements.txt (line 32)) (2.0.0)\n", + "Requirement already satisfied: pycparser in /Users/mramanindia/work/NovaEval/.venv/lib/python3.12/site-packages (from cffi>=1.0.1->argon2-cffi-bindings->argon2-cffi>=21.1->jupyter-server<3,>=2.4.0->notebook->jupyter==1.0.0->-r ./noveum_agent_requirements.txt (line 32)) (2.23)\n", + "Collecting lark>=1.2.2 (from rfc3987-syntax>=1.1.0->jsonschema[format-nongpl]>=4.18.0->jupyter-events>=0.11.0->jupyter-server<3,>=2.4.0->notebook->jupyter==1.0.0->-r ./noveum_agent_requirements.txt (line 32))\n", + " Using cached lark-1.3.0-py3-none-any.whl.metadata (1.8 kB)\n", + "Collecting arrow>=0.15.0 (from isoduration->jsonschema[format-nongpl]>=4.18.0->jupyter-events>=0.11.0->jupyter-server<3,>=2.4.0->notebook->jupyter==1.0.0->-r ./noveum_agent_requirements.txt (line 32))\n", + " Using cached arrow-1.4.0-py3-none-any.whl.metadata (7.7 kB)\n", + "Using cached requests-2.32.3-py3-none-any.whl (64 kB)\n", + "Using cached beautifulsoup4-4.12.3-py3-none-any.whl (147 kB)\n", + "Using cached python_dotenv-1.0.1-py3-none-any.whl (19 kB)\n", + "Using cached langchain-0.3.26-py3-none-any.whl (1.0 MB)\n", + "Using cached langchain_community-0.3.18-py3-none-any.whl (2.5 MB)\n", + "Using cached langchain_core-0.3.66-py3-none-any.whl (438 kB)\n", + "Using cached langchain_openai-0.3.25-py3-none-any.whl (69 kB)\n", + "Using cached faiss_cpu-1.12.0-cp312-cp312-macosx_14_0_arm64.whl (3.4 MB)\n", + "Using cached jupyter-1.0.0-py2.py3-none-any.whl (2.7 kB)\n", + "Using cached ipykernel-6.29.4-py3-none-any.whl (117 kB)\n", + "Using cached trafilatura-2.0.0-py3-none-any.whl (132 kB)\n", + "Using cached duckduckgo_search-8.1.1-py3-none-any.whl (18 kB)\n", + "Using cached aiohttp-3.13.1-cp312-cp312-macosx_11_0_arm64.whl (490 kB)\n", + "Using cached courlan-1.3.2-py3-none-any.whl (33 kB)\n", + "Using cached dataclasses_json-0.6.7-py3-none-any.whl (28 kB)\n", + "Using cached htmldate-1.9.3-py3-none-any.whl (31 kB)\n", + "Using cached lxml-5.4.0-cp312-cp312-macosx_10_9_universal2.whl (8.1 MB)\n", + "Using cached httpx_sse-0.4.3-py3-none-any.whl (9.0 kB)\n", + "Using cached jsonpatch-1.33-py2.py3-none-any.whl (12 kB)\n", + "Using cached justext-3.0.2-py2.py3-none-any.whl (837 kB)\n", + "Using cached langchain_text_splitters-0.3.8-py3-none-any.whl (32 kB)\n", + "Using cached langsmith-0.3.45-py3-none-any.whl (363 kB)\n", + "Using cached openai-1.109.1-py3-none-any.whl (948 kB)\n", + "Using cached packaging-24.2-py3-none-any.whl (65 kB)\n", + "Using cached primp-0.15.0-cp38-abi3-macosx_11_0_arm64.whl (3.0 MB)\n", + "Using cached soupsieve-2.8-py3-none-any.whl (36 kB)\n", + "Using cached sqlalchemy-2.0.44-cp312-cp312-macosx_11_0_arm64.whl (2.1 MB)\n", + "Using cached tiktoken-0.12.0-cp312-cp312-macosx_11_0_arm64.whl (994 kB)\n", + "Using cached ipywidgets-8.1.7-py3-none-any.whl (139 kB)\n", + "Using cached jupyter_console-6.6.3-py3-none-any.whl (24 kB)\n", + "Using cached nbconvert-7.16.6-py3-none-any.whl (258 kB)\n", + "Using cached notebook-7.4.7-py3-none-any.whl (14.3 MB)\n", + "Using cached qtconsole-5.7.0-py3-none-any.whl (125 kB)\n", + "Using cached aiohappyeyeballs-2.6.1-py3-none-any.whl (15 kB)\n", + "Using cached aiosignal-1.4.0-py3-none-any.whl (7.5 kB)\n", + "Using cached attrs-25.4.0-py3-none-any.whl (67 kB)\n", + "Using cached babel-2.17.0-py3-none-any.whl (10.2 MB)\n", + "Using cached bleach-6.2.0-py3-none-any.whl (163 kB)\n", + "Using cached dateparser-1.2.2-py3-none-any.whl (315 kB)\n", + "Using cached distro-1.9.0-py3-none-any.whl (20 kB)\n", + "Using cached frozenlist-1.8.0-cp312-cp312-macosx_11_0_arm64.whl (50 kB)\n", + "Using cached jiter-0.11.1-cp312-cp312-macosx_11_0_arm64.whl (315 kB)\n", + "Using cached jsonpointer-3.0.0-py2.py3-none-any.whl (7.6 kB)\n", + "Using cached jupyter_server-2.17.0-py3-none-any.whl (388 kB)\n", + "Downloading jupyterlab-4.4.10-py3-none-any.whl (12.3 MB)\n", + "\u001b[2K \u001b[90mā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”\u001b[0m \u001b[32m12.3/12.3 MB\u001b[0m \u001b[31m135.7 kB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:02\u001b[0m00:04\u001b[0m\n", + "\u001b[?25hDownloading jupyterlab_server-2.28.0-py3-none-any.whl (59 kB)\n", + "Using cached jupyterlab_widgets-3.0.15-py3-none-any.whl (216 kB)\n", + "Using cached marshmallow-3.26.1-py3-none-any.whl (50 kB)\n", + "Using cached mistune-3.1.4-py3-none-any.whl (53 kB)\n", + "Using cached multidict-6.7.0-cp312-cp312-macosx_11_0_arm64.whl (43 kB)\n", + "Using cached nbclient-0.10.2-py3-none-any.whl (25 kB)\n", + "Using cached nbformat-5.10.4-py3-none-any.whl (78 kB)\n", + "Using cached notebook_shim-0.2.4-py3-none-any.whl (13 kB)\n", + "Using cached orjson-3.11.3-cp312-cp312-macosx_15_0_arm64.whl (127 kB)\n", + "Using cached pandocfilters-1.5.1-py2.py3-none-any.whl (8.7 kB)\n", + "Using cached propcache-0.4.1-cp312-cp312-macosx_11_0_arm64.whl (47 kB)\n", + "Using cached QtPy-2.4.3-py3-none-any.whl (95 kB)\n", + "Using cached requests_toolbelt-1.0.0-py2.py3-none-any.whl (54 kB)\n", + "Using cached tld-0.13.1-py2.py3-none-any.whl (274 kB)\n", + "Using cached typing_inspect-0.9.0-py3-none-any.whl (8.8 kB)\n", + "Using cached widgetsnbextension-4.0.14-py3-none-any.whl (2.2 MB)\n", + "Using cached yarl-1.22.0-cp312-cp312-macosx_11_0_arm64.whl (94 kB)\n", + "Using cached zstandard-0.23.0-cp312-cp312-macosx_11_0_arm64.whl (633 kB)\n", + "Using cached defusedxml-0.7.1-py2.py3-none-any.whl (25 kB)\n", + "Using cached jupyterlab_pygments-0.3.0-py3-none-any.whl (15 kB)\n", + "Using cached argon2_cffi-25.1.0-py3-none-any.whl (14 kB)\n", + "Using cached async_lru-2.0.5-py3-none-any.whl (6.1 kB)\n", + "Using cached fastjsonschema-2.21.2-py3-none-any.whl (24 kB)\n", + "Using cached json5-0.12.1-py3-none-any.whl (36 kB)\n", + "Using cached jsonschema-4.25.1-py3-none-any.whl (90 kB)\n", + "Using cached jupyter_events-0.12.0-py3-none-any.whl (19 kB)\n", + "Using cached jupyter_lsp-2.3.0-py3-none-any.whl (76 kB)\n", + "Using cached jupyter_server_terminals-0.5.3-py3-none-any.whl (13 kB)\n", + "Using cached prometheus_client-0.23.1-py3-none-any.whl (61 kB)\n", + "Using cached Send2Trash-1.8.3-py3-none-any.whl (18 kB)\n", + "Using cached terminado-0.18.1-py3-none-any.whl (14 kB)\n", + "Using cached tinycss2-1.4.0-py3-none-any.whl (26 kB)\n", + "Using cached tzlocal-5.3.1-py3-none-any.whl (18 kB)\n", + "Using cached webencodings-0.5.1-py2.py3-none-any.whl (11 kB)\n", + "Using cached websocket_client-1.9.0-py3-none-any.whl (82 kB)\n", + "Using cached lxml_html_clean-0.4.3-py3-none-any.whl (14 kB)\n", + "Using cached jsonschema_specifications-2025.9.1-py3-none-any.whl (18 kB)\n", + "Using cached python_json_logger-4.0.0-py3-none-any.whl (15 kB)\n", + "Using cached referencing-0.37.0-py3-none-any.whl (26 kB)\n", + "Using cached rfc3986_validator-0.1.1-py2.py3-none-any.whl (4.2 kB)\n", + "Using cached rpds_py-0.27.1-cp312-cp312-macosx_11_0_arm64.whl (345 kB)\n", + "Using cached argon2_cffi_bindings-25.1.0-cp39-abi3-macosx_11_0_arm64.whl (31 kB)\n", + "Using cached rfc3339_validator-0.1.4-py2.py3-none-any.whl (3.5 kB)\n", + "Using cached rfc3987_syntax-1.1.0-py3-none-any.whl (8.0 kB)\n", + "Using cached webcolors-24.11.1-py3-none-any.whl (14 kB)\n", + "Using cached fqdn-1.5.1-py3-none-any.whl (9.1 kB)\n", + "Using cached isoduration-20.11.0-py3-none-any.whl (11 kB)\n", + "Using cached uri_template-1.3.0-py3-none-any.whl (11 kB)\n", + "Using cached arrow-1.4.0-py3-none-any.whl (68 kB)\n", + "Using cached lark-1.3.0-py3-none-any.whl (113 kB)\n", + "Installing collected packages: webencodings, fastjsonschema, zstandard, widgetsnbextension, websocket-client, webcolors, uri-template, tzlocal, typing-inspect, tld, tinycss2, terminado, SQLAlchemy, soupsieve, send2trash, rpds-py, rfc3986-validator, rfc3339-validator, requests, python-json-logger, python-dotenv, propcache, prometheus-client, primp, pandocfilters, packaging, orjson, multidict, mistune, lxml, lark, jupyterlab_widgets, jupyterlab-pygments, jsonpointer, json5, jiter, httpx-sse, frozenlist, fqdn, distro, defusedxml, bleach, babel, attrs, async-lru, aiohappyeyeballs, yarl, tiktoken, rfc3987-syntax, requests-toolbelt, referencing, qtpy, marshmallow, lxml_html_clean, jupyter-server-terminals, jsonpatch, faiss-cpu, duckduckgo-search, dateparser, courlan, beautifulsoup4, arrow, argon2-cffi-bindings, aiosignal, openai, langsmith, jsonschema-specifications, isoduration, ipywidgets, ipykernel, htmldate, dataclasses-json, argon2-cffi, aiohttp, qtconsole, langchain-core, justext, jupyter-console, jsonschema, trafilatura, nbformat, langchain-text-splitters, langchain-openai, nbclient, langchain, jupyter-events, nbconvert, langchain-community, jupyter-server, notebook-shim, jupyterlab-server, jupyter-lsp, jupyterlab, notebook, jupyter\n", + " Attempting uninstall: requests\n", + " Found existing installation: requests 2.32.5\n", + " Uninstalling requests-2.32.5:\n", + " Successfully uninstalled requests-2.32.5\n", + " Attempting uninstall: python-dotenv\n", + " Found existing installation: python-dotenv 1.1.1\n", + " Uninstalling python-dotenv-1.1.1:\n", + " Successfully uninstalled python-dotenv-1.1.1\n", + " Attempting uninstall: packaging\n", + " Found existing installation: packaging 25.0\n", + " Uninstalling packaging-25.0:\n", + " Successfully uninstalled packaging-25.0\n", + " Attempting uninstall: marshmallow\n", + " Found existing installation: marshmallow 4.0.1\n", + " Uninstalling marshmallow-4.0.1:\n", + " Successfully uninstalled marshmallow-4.0.1\n", + " Attempting uninstall: ipykernel\n", + " Found existing installation: ipykernel 7.0.1\n", + " Uninstalling ipykernel-7.0.1:\n", + " Successfully uninstalled ipykernel-7.0.1\n", + "Successfully installed SQLAlchemy-2.0.44 aiohappyeyeballs-2.6.1 aiohttp-3.13.1 aiosignal-1.4.0 argon2-cffi-25.1.0 argon2-cffi-bindings-25.1.0 arrow-1.4.0 async-lru-2.0.5 attrs-25.4.0 babel-2.17.0 beautifulsoup4-4.12.3 bleach-6.2.0 courlan-1.3.2 dataclasses-json-0.6.7 dateparser-1.2.2 defusedxml-0.7.1 distro-1.9.0 duckduckgo-search-8.1.1 faiss-cpu-1.12.0 fastjsonschema-2.21.2 fqdn-1.5.1 frozenlist-1.8.0 htmldate-1.9.3 httpx-sse-0.4.3 ipykernel-6.29.4 ipywidgets-8.1.7 isoduration-20.11.0 jiter-0.11.1 json5-0.12.1 jsonpatch-1.33 jsonpointer-3.0.0 jsonschema-4.25.1 jsonschema-specifications-2025.9.1 jupyter-1.0.0 jupyter-console-6.6.3 jupyter-events-0.12.0 jupyter-lsp-2.3.0 jupyter-server-2.17.0 jupyter-server-terminals-0.5.3 jupyterlab-4.4.10 jupyterlab-pygments-0.3.0 jupyterlab-server-2.28.0 jupyterlab_widgets-3.0.15 justext-3.0.2 langchain-0.3.26 langchain-community-0.3.18 langchain-core-0.3.66 langchain-openai-0.3.25 langchain-text-splitters-0.3.8 langsmith-0.3.45 lark-1.3.0 lxml-5.4.0 lxml_html_clean-0.4.3 marshmallow-3.26.1 mistune-3.1.4 multidict-6.7.0 nbclient-0.10.2 nbconvert-7.16.6 nbformat-5.10.4 notebook-7.4.7 notebook-shim-0.2.4 openai-1.109.1 orjson-3.11.3 packaging-24.2 pandocfilters-1.5.1 primp-0.15.0 prometheus-client-0.23.1 propcache-0.4.1 python-dotenv-1.0.1 python-json-logger-4.0.0 qtconsole-5.7.0 qtpy-2.4.3 referencing-0.37.0 requests-2.32.3 requests-toolbelt-1.0.0 rfc3339-validator-0.1.4 rfc3986-validator-0.1.1 rfc3987-syntax-1.1.0 rpds-py-0.27.1 send2trash-1.8.3 soupsieve-2.8 terminado-0.18.1 tiktoken-0.12.0 tinycss2-1.4.0 tld-0.13.1 trafilatura-2.0.0 typing-inspect-0.9.0 tzlocal-5.3.1 uri-template-1.3.0 webcolors-24.11.1 webencodings-0.5.1 websocket-client-1.9.0 widgetsnbextension-4.0.14 yarl-1.22.0 zstandard-0.23.0\n", + "\n", + "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m24.3.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m25.2\u001b[0m\n", + "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n" + ] + } + ], + "source": [ + "!pip3 install -r ./noveum_agent_requirements.txt\n" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "id": "e7a810bf", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "āœ… All imports loaded successfully!\n" + ] + } + ], + "source": [ + "# Cell 1: Setup & Imports\n", + "import os\n", + "import json\n", + "import time\n", + "from typing import List, Dict, Any, Optional, Tuple\n", + "from urllib.parse import urljoin, urlparse\n", + "import requests\n", + "from bs4 import BeautifulSoup\n", + "import trafilatura\n", + "\n", + "# LangChain ecosystem\n", + "from langchain_openai import OpenAIEmbeddings, ChatOpenAI\n", + "from langchain_community.vectorstores import FAISS\n", + "from langchain.text_splitter import RecursiveCharacterTextSplitter\n", + "from langchain.schema import Document\n", + "from langchain_community.tools import DuckDuckGoSearchRun\n", + "\n", + "# Noveum Trace integration\n", + "import noveum_trace\n", + "from noveum_trace.context_managers import trace_operation, trace_agent\n", + "\n", + "# Load environment variables\n", + "try:\n", + " from dotenv import load_dotenv\n", + " load_dotenv()\n", + "except ImportError:\n", + " print(\"python-dotenv not installed. Environment variables will be read from system only.\")\n", + "\n", + "print(\"āœ… All imports loaded successfully!\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2209736e", + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "\n", + "## set openai api key\n", + "## set gemini api key\n", + "## set noveum api key\n", + "## set environment\n", + "## set project'\n", + "\n", + "# These are required for the project\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c0dc8f95", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "āœ… Noveum Trace initialized and configuration loaded!\n", + "šŸ”§ Configuration: {'noveum_base_url': 'https://noveum.ai', 'max_pages_to_scrape': 50, 'chunk_size': 1000, 'chunk_overlap': 200, 'max_search_results': 5, 'rag_threshold': 0.7, 'noveum_docs_file': 'noveum_docs.json', 'vector_store_path': 'noveum_vectorstore'}\n" + ] + } + ], + "source": [ + "# Cell 2: Noveum Trace Integration & Configuration\n", + "# Initialize the Noveum Trace SDK\n", + "noveum_trace.init(\n", + " project=\"customer_support_agent\",\n", + " api_key=os.getenv(\"NOVEUM_API_KEY\"),\n", + " environment=\"dev-aman\",\n", + ")\n", + "\n", + "# Configuration\n", + "CONFIG = {\n", + " \"noveum_base_url\": \"https://noveum.ai\",\n", + " \"max_pages_to_scrape\": 50,\n", + " \"chunk_size\": 1000,\n", + " \"chunk_overlap\": 200,\n", + " \"max_search_results\": 5,\n", + " \"rag_threshold\": 0.7, # Similarity threshold for RAG retrieval\n", + " \"noveum_docs_file\": \"noveum_docs.json\",\n", + " \"vector_store_path\": \"noveum_vectorstore\"\n", + "}\n", + "\n", + "# Initialize LLM and embeddings\n", + "llm = ChatOpenAI(\n", + " model=\"gpt-4o-mini\",\n", + " temperature=0.1,\n", + " api_key=os.getenv(\"OPENAI_API_KEY\")\n", + ")\n", + "\n", + "embeddings = OpenAIEmbeddings(\n", + " model=\"text-embedding-3-small\",\n", + " api_key=os.getenv(\"OPENAI_API_KEY\")\n", + ")\n", + "\n", + "# Initialize web search tool\n", + "web_search = DuckDuckGoSearchRun()\n", + "\n", + "print(\"āœ… Noveum Trace initialized and configuration loaded!\")\n", + "print(f\"šŸ”§ Configuration: {CONFIG}\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": 50, + "id": "c765c0cc", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "āœ… Website scraper initialized!\n" + ] + } + ], + "source": [ + "# Cell 3: Website Scraper - Extract content from noveum.ai and sub-URLs\n", + "class NoveumWebsiteScraper:\n", + " def __init__(self, base_url: str, max_pages: int = 50):\n", + " self.base_url = base_url\n", + " self.max_pages = max_pages\n", + " self.scraped_urls = set()\n", + " self.scraped_content = []\n", + " self.session = requests.Session()\n", + " self.session.headers.update({\n", + " 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'\n", + " })\n", + " \n", + " def is_valid_url(self, url: str) -> bool:\n", + " \"\"\"Check if URL is valid and belongs to noveum.ai domain\"\"\"\n", + " try:\n", + " parsed = urlparse(url)\n", + " return (\n", + " parsed.netloc in ['noveum.ai', 'www.noveum.ai'] and\n", + " not any(ext in url.lower() for ext in ['.pdf', '.jpg', '.png', '.gif', '.css', '.js', '.xml', '.txt']) and\n", + " '#' not in url\n", + " )\n", + " except:\n", + " return False\n", + " \n", + " def extract_text_content(self, html_content: str, url: str) -> str:\n", + " \"\"\"Extract clean text content from HTML\"\"\"\n", + " try:\n", + " # Use trafilatura for better text extraction\n", + " extracted = trafilatura.extract(html_content)\n", + " if extracted:\n", + " return extracted.strip()\n", + " \n", + " # Fallback to BeautifulSoup\n", + " soup = BeautifulSoup(html_content, 'html.parser')\n", + " \n", + " # Remove script and style elements\n", + " for script in soup([\"script\", \"style\"]):\n", + " script.decompose()\n", + " \n", + " # Get text and clean up\n", + " text = soup.get_text()\n", + " lines = (line.strip() for line in text.splitlines())\n", + " chunks = (phrase.strip() for line in lines for phrase in line.split(\" \"))\n", + " text = ' '.join(chunk for chunk in chunks if chunk)\n", + " \n", + " return text.strip()\n", + " except Exception as e:\n", + " print(f\"Error extracting text from {url}: {e}\")\n", + " return \"\"\n", + " \n", + " def find_internal_links(self, html_content: str, current_url: str) -> List[str]:\n", + " \"\"\"Find all internal links from the current page\"\"\"\n", + " try:\n", + " soup = BeautifulSoup(html_content, 'html.parser')\n", + " links = []\n", + " \n", + " for link in soup.find_all('a', href=True):\n", + " href = link['href']\n", + " full_url = urljoin(current_url, href)\n", + " \n", + " if self.is_valid_url(full_url) and full_url not in self.scraped_urls:\n", + " links.append(full_url)\n", + " \n", + " return links\n", + " except Exception as e:\n", + " print(f\"Error finding links in {current_url}: {e}\")\n", + " return []\n", + " \n", + " def scrape_page(self, url: str) -> Optional[Dict[str, Any]]:\n", + " \"\"\"Scrape a single page and return content\"\"\"\n", + " try:\n", + " print(f\"šŸ” Scraping: {url}\")\n", + " response = self.session.get(url, timeout=10)\n", + " response.raise_for_status()\n", + " \n", + " # Extract text content\n", + " text_content = self.extract_text_content(response.text, url)\n", + " \n", + " if not text_content or len(text_content) < 100: # Skip pages with too little content\n", + " print(f\"āš ļø Skipping {url} - insufficient content\")\n", + " return None\n", + " \n", + " # Find internal links\n", + " internal_links = self.find_internal_links(response.text, url)\n", + " \n", + " page_data = {\n", + " \"url\": url,\n", + " \"title\": self.extract_title(response.text),\n", + " \"content\": text_content,\n", + " \"content_length\": len(text_content),\n", + " \"internal_links\": internal_links,\n", + " \"scraped_at\": time.time()\n", + " }\n", + " \n", + " print(f\"āœ… Scraped {url} - {len(text_content)} chars, {len(internal_links)} internal links\")\n", + " return page_data\n", + " \n", + " except Exception as e:\n", + " print(f\"āŒ Error scraping {url}: {e}\")\n", + " return None\n", + " \n", + " def extract_title(self, html_content: str) -> str:\n", + " \"\"\"Extract page title\"\"\"\n", + " try:\n", + " soup = BeautifulSoup(html_content, 'html.parser')\n", + " title_tag = soup.find('title')\n", + " return title_tag.get_text().strip() if title_tag else \"Untitled\"\n", + " except:\n", + " return \"Untitled\"\n", + " \n", + " def scrape_website(self) -> List[Dict[str, Any]]:\n", + " \"\"\"Main scraping function - scrape noveum.ai recursively\"\"\"\n", + " print(f\"šŸš€ Starting to scrape {self.base_url}\")\n", + " \n", + " urls_to_scrape = [self.base_url]\n", + " self.scraped_urls.add(self.base_url)\n", + " \n", + " with trace_operation(\"noveum_website_scraping\") as scrape_span:\n", + " scrape_span.set_attributes({\n", + " \"scraper.base_url\": self.base_url,\n", + " \"scraper.max_pages\": self.max_pages,\n", + " \"input_query\": f\"Scrape website: {self.base_url}\",\n", + " \"output_response\": f\"Scraping completed: {len(self.scraped_content)} pages scraped, {sum(page['content_length'] for page in self.scraped_content)} total characters extracted\"\n", + " })\n", + " \n", + " while urls_to_scrape and len(self.scraped_content) < self.max_pages:\n", + " current_url = urls_to_scrape.pop(0)\n", + " \n", + " # Scrape the current page\n", + " page_data = self.scrape_page(current_url)\n", + " \n", + " if page_data:\n", + " self.scraped_content.append(page_data)\n", + " \n", + " # Add new internal links to the queue\n", + " for link in page_data[\"internal_links\"]:\n", + " if link not in self.scraped_urls and len(urls_to_scrape) < 100: # Prevent infinite loops\n", + " urls_to_scrape.append(link)\n", + " self.scraped_urls.add(link)\n", + " \n", + " # Add page data to span\n", + " scrape_span.add_event(\"page_scraped\", {\n", + " \"input_query\": f\"Scrape page: {current_url}\",\n", + " \"output_response\": f\"Page scraped successfully: {page_data['content_length']} characters, {len(page_data['internal_links'])} internal links found\",\n", + " \"url\": current_url,\n", + " \"content_length\": page_data[\"content_length\"],\n", + " \"internal_links_found\": len(page_data[\"internal_links\"])\n", + " })\n", + " \n", + " # Small delay to be respectful\n", + " time.sleep(0.5)\n", + " \n", + " # Final metrics\n", + " scrape_span.set_attributes({\n", + " \"scraper.pages_scraped\": len(self.scraped_content),\n", + " \"scraper.total_urls_found\": len(self.scraped_urls),\n", + " \"scraper.total_content_length\": sum(page[\"content_length\"] for page in self.scraped_content)\n", + " })\n", + " \n", + " print(f\"āœ… Scraping complete! Scraped {len(self.scraped_content)} pages\")\n", + " return self.scraped_content\n", + " \n", + " def save_to_json(self, filename: str) -> None:\n", + " \"\"\"Save scraped content to JSON file\"\"\"\n", + " with open(filename, 'w', encoding='utf-8') as f:\n", + " json.dump(self.scraped_content, f, indent=2, ensure_ascii=False)\n", + " print(f\"šŸ’¾ Saved scraped content to {filename}\")\n", + "\n", + "# Initialize scraper\n", + "scraper = NoveumWebsiteScraper(CONFIG[\"noveum_base_url\"], CONFIG[\"max_pages_to_scrape\"])\n", + "print(\"āœ… Website scraper initialized!\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": 51, + "id": "0d0e2a7a", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "āœ… RAG system initialized!\n" + ] + } + ], + "source": [ + "# Cell 4: RAG System - Vector search and retrieval over scraped content\n", + "class NoveumRAGSystem:\n", + " def __init__(self, embeddings, llm, config):\n", + " self.embeddings = embeddings\n", + " self.llm = llm\n", + " self.config = config\n", + " self.vectorstore = None\n", + " self.documents = []\n", + " self.text_splitter = RecursiveCharacterTextSplitter(\n", + " chunk_size=config[\"chunk_size\"],\n", + " chunk_overlap=config[\"chunk_overlap\"]\n", + " )\n", + " \n", + " def load_documents_from_json(self, json_file: str) -> List[Document]:\n", + " \"\"\"Load documents from scraped JSON file\"\"\"\n", + " try:\n", + " with open(json_file, 'r', encoding='utf-8') as f:\n", + " scraped_data = json.load(f)\n", + " \n", + " documents = []\n", + " for page in scraped_data:\n", + " # Create document from page content\n", + " doc = Document(\n", + " page_content=page[\"content\"],\n", + " metadata={\n", + " \"url\": page[\"url\"],\n", + " \"title\": page[\"title\"],\n", + " \"content_length\": page[\"content_length\"],\n", + " \"scraped_at\": page[\"scraped_at\"]\n", + " }\n", + " )\n", + " documents.append(doc)\n", + " \n", + " print(f\"āœ… Loaded {len(documents)} documents from {json_file}\")\n", + " return documents\n", + " \n", + " except FileNotFoundError:\n", + " print(f\"āŒ File {json_file} not found. Please run the scraper first.\")\n", + " return []\n", + " except Exception as e:\n", + " print(f\"āŒ Error loading documents: {e}\")\n", + " return []\n", + " \n", + " def create_vectorstore(self, documents: List[Document]) -> None:\n", + " \"\"\"Create FAISS vector store from documents\"\"\"\n", + " if not documents:\n", + " print(\"āŒ No documents to create vector store\")\n", + " return\n", + " \n", + " print(\"šŸ”„ Creating vector store...\")\n", + " \n", + " # Split documents into chunks\n", + " split_docs = self.text_splitter.split_documents(documents)\n", + " print(f\"šŸ“„ Split into {len(split_docs)} chunks\")\n", + " \n", + " # Create vector store\n", + " self.vectorstore = FAISS.from_documents(split_docs, self.embeddings)\n", + " \n", + " # Save vector store\n", + " self.vectorstore.save_local(self.config[\"vector_store_path\"])\n", + " print(f\"šŸ’¾ Vector store saved to {self.config['vector_store_path']}\")\n", + " \n", + " def load_vectorstore(self) -> bool:\n", + " \"\"\"Load existing vector store from disk\"\"\"\n", + " try:\n", + " self.vectorstore = FAISS.load_local(\n", + " self.config[\"vector_store_path\"], \n", + " self.embeddings,\n", + " allow_dangerous_deserialization=True\n", + " )\n", + " print(f\"āœ… Loaded existing vector store from {self.config['vector_store_path']}\")\n", + " return True\n", + " except Exception as e:\n", + " print(f\"āŒ Error loading vector store: {e}\")\n", + " return False\n", + " \n", + " def search_relevant_docs(self, query: str, k: int = 5) -> List[Document]:\n", + " \"\"\"Search for relevant documents using similarity search\"\"\"\n", + " if not self.vectorstore:\n", + " print(\"āŒ Vector store not initialized\")\n", + " return []\n", + " \n", + " try:\n", + " # Perform similarity search\n", + " docs = self.vectorstore.similarity_search(query, k=k)\n", + " \n", + " # Filter by similarity threshold if needed\n", + " # Note: FAISS doesn't return scores by default, but we can add that if needed\n", + " \n", + " print(f\"šŸ” Found {len(docs)} relevant documents for query: '{query}'\")\n", + " return docs\n", + " \n", + " except Exception as e:\n", + " print(f\"āŒ Error searching documents: {e}\")\n", + " return []\n", + " \n", + " def retrieve_context(self, query: str, max_docs: int = 5) -> str:\n", + " \"\"\"Retrieve and format context for the query\"\"\"\n", + " relevant_docs = self.search_relevant_docs(query, max_docs)\n", + " \n", + " if not relevant_docs:\n", + " return \"No relevant information found in Noveum documentation.\"\n", + " \n", + " context_parts = []\n", + " for i, doc in enumerate(relevant_docs, 1):\n", + " context_parts.append(f\"Source {i} ({doc.metadata.get('url', 'Unknown URL')}):\\n{doc.page_content[:500]}...\")\n", + " \n", + " return \"\\n\\n\".join(context_parts)\n", + " \n", + " def generate_rag_response(self, query: str) -> Dict[str, Any]:\n", + " \"\"\"Generate response using RAG\"\"\"\n", + " with trace_agent(\n", + " agent_type=\"rag_agent\",\n", + " operation=\"llm-rag\",\n", + " capabilities=[\"document_retrieval\", \"context_generation\", \"response_generation\"],\n", + " attributes={\n", + " \"agent.id\": \"noveum_rag_agent\",\n", + " \"input_query\": query,\n", + " \"query_length\": len(query)\n", + " }\n", + " ) as rag_span:\n", + " \n", + " # Retrieve relevant context\n", + " context = self.retrieve_context(query, CONFIG[\"max_search_results\"])\n", + " \n", + " # Create prompt for RAG\n", + " rag_prompt = f\"\"\"You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n", + "\n", + "Context from Noveum documentation:\n", + "{context}\n", + "\n", + "User Question: {query}\n", + "\n", + "Instructions:\n", + "1. Answer based primarily on the provided context\n", + "2. If the context doesn't contain enough information, say so clearly\n", + "3. Be specific and cite sources when possible\n", + "4. Keep responses concise but informative\n", + "5. If the question is not related to Noveum, politely redirect to ask about Noveum\n", + "\n", + "Answer:\"\"\"\n", + "\n", + " # Extract model parameters and metadata\n", + " model_name = getattr(self.llm, 'model_name', 'unknown')\n", + " model_temperature = getattr(self.llm, 'temperature', 0.0)\n", + " model_max_tokens = getattr(self.llm, 'max_tokens', None)\n", + " model_top_p = getattr(self.llm, 'top_p', None)\n", + " model_frequency_penalty = getattr(self.llm, 'frequency_penalty', None)\n", + " model_presence_penalty = getattr(self.llm, 'presence_penalty', None)\n", + " \n", + " # Model Details Span - Track model-specific information\n", + " with trace_agent(\n", + " agent_type=\"model_details\",\n", + " operation=\"llm_model_execution\",\n", + " capabilities=[\"model_invocation\", \"parameter_tracking\", \"latency_measurement\"],\n", + " attributes={\n", + " \"agent.id\": \"noveum_model_details\",\n", + " \"input_query\": f\"Model execution for query: {query[:100]}{'...' if len(query) > 100 else ''}\",\n", + " \"query_length\": len(query)\n", + " }\n", + " ) as model_span:\n", + " \n", + " # Record start time for latency measurement\n", + " model_start_time = time.time()\n", + " \n", + " # Generate response\n", + " response = self.llm.invoke(rag_prompt)\n", + " \n", + " # Record end time and calculate latency\n", + " model_end_time = time.time()\n", + " model_latency = model_end_time - model_start_time\n", + "\n", + " if response.content:\n", + " answer = response.content\n", + " else:\n", + " answer = str(response)\n", + "\n", + " # Extract token usage metadata - Enhanced extraction\n", + " prompt_tokens = 0\n", + " completion_tokens = 0\n", + " total_tokens = 0\n", + " \n", + " # Try multiple ways to extract token usage\n", + " if hasattr(response, 'usage_metadata') and response.usage_metadata:\n", + " usage = response.usage_metadata\n", + " prompt_tokens = getattr(usage, \"input_tokens\", 0) or getattr(usage, \"prompt_tokens\", 0)\n", + " completion_tokens = getattr(usage, \"output_tokens\", 0) or getattr(usage, \"completion_tokens\", 0)\n", + " total_tokens = getattr(usage, \"total_tokens\", 0)\n", + " elif hasattr(response, 'response_metadata') and response.response_metadata:\n", + " metadata = response.response_metadata\n", + " if 'token_usage' in metadata:\n", + " token_usage = metadata['token_usage']\n", + " prompt_tokens = token_usage.get('prompt_tokens', 0)\n", + " completion_tokens = token_usage.get('completion_tokens', 0)\n", + " total_tokens = token_usage.get('total_tokens', 0)\n", + " elif hasattr(response, 'token_usage'):\n", + " token_usage = response.token_usage\n", + " prompt_tokens = getattr(token_usage, \"prompt_tokens\", 0)\n", + " completion_tokens = getattr(token_usage, \"completion_tokens\", 0)\n", + " total_tokens = getattr(token_usage, \"total_tokens\", 0)\n", + " \n", + " # If still no tokens found, try to estimate from content length\n", + " if total_tokens == 0:\n", + " # Rough estimation: ~4 characters per token for English text\n", + " estimated_prompt_tokens = len(rag_prompt) // 4\n", + " estimated_completion_tokens = len(answer) // 4\n", + " prompt_tokens = estimated_prompt_tokens\n", + " completion_tokens = estimated_completion_tokens\n", + " total_tokens = prompt_tokens + completion_tokens\n", + "\n", + " # Set model details span attributes\n", + " model_span.set_attributes({\n", + " # Input metrics\n", + " \"input_query\": f\"Model execution for query: {query[:100]}{'...' if len(query) > 100 else ''}\",\n", + " \"query_length\": len(query),\n", + " \"query_type\": \"rag_model_query\",\n", + " \n", + " # Model parameters and configuration\n", + " \"model.name\": model_name,\n", + " \"model.temperature\": model_temperature,\n", + " \"model.max_tokens\": model_max_tokens,\n", + " \"model.top_p\": model_top_p,\n", + " \"model.frequency_penalty\": model_frequency_penalty,\n", + " \"model.presence_penalty\": model_presence_penalty,\n", + " \"model.provider\": \"openai\",\n", + " \"model.type\": \"chat_completion\",\n", + " \"model.version\": \"gpt-4o-mini\",\n", + " \n", + " # Latency and performance metrics\n", + " \"model.latency_seconds\": model_latency,\n", + " \"model.latency_ms\": model_latency * 1000,\n", + " \"model.start_time\": model_start_time,\n", + " \"model.end_time\": model_end_time,\n", + " \"model.performance_tier\": \"fast\" if model_latency < 2.0 else \"medium\" if model_latency < 5.0 else \"slow\",\n", + " \n", + " # Token usage and cost metrics\n", + " \"model.prompt_tokens\": prompt_tokens,\n", + " \"model.completion_tokens\": completion_tokens,\n", + " \"model.total_tokens\": total_tokens,\n", + " \"model.tokens_per_second\": total_tokens / model_latency if model_latency > 0 else 0,\n", + " \"model.estimated_cost\": total_tokens * 0.00003, # Rough cost estimate\n", + " \"model.efficiency_score\": len(answer) / total_tokens if total_tokens > 0 else 0,\n", + " \n", + " # Response characteristics\n", + " \"model.response_length\": len(answer),\n", + " \"model.response_quality\": \"high\" if len(answer) > 200 else \"medium\" if len(answer) > 100 else \"low\",\n", + " \"model.output_response\": f\"Model Response: {answer[:200]}{'...' if len(answer) > 200 else ''}\",\n", + " \n", + " # Model configuration details\n", + " \"model.config\": {\n", + " \"name\": model_name,\n", + " \"temperature\": model_temperature,\n", + " \"max_tokens\": model_max_tokens,\n", + " \"top_p\": model_top_p,\n", + " \"frequency_penalty\": model_frequency_penalty,\n", + " \"presence_penalty\": model_presence_penalty,\n", + " \"provider\": \"openai\",\n", + " \"type\": \"chat_completion\"\n", + " }\n", + " })\n", + "\n", + " # Other Details Span - Track retrieval, response quality, and evaluation metrics\n", + " with trace_agent(\n", + " agent_type=\"other_details\",\n", + " operation=\"rag_evaluation_metrics\",\n", + " capabilities=[\"retrieval_analysis\", \"response_evaluation\", \"quality_assessment\"],\n", + " attributes={\n", + " \"agent.id\": \"noveum_other_details\",\n", + " \"input_query\": f\"Evaluation for query: {query[:100]}{'...' if len(query) > 100 else ''}\",\n", + " \"query_length\": len(query)\n", + " }\n", + " ) as rag_node:\n", + " \n", + " # Calculate additional evaluation metrics\n", + " context_length = len(context)\n", + " answer_length = len(answer)\n", + " sources_count = len(context.split(\"Source\")) - 1 if \"Source\" in context else 0\n", + " \n", + " # Set other details span attributes\n", + " rag_node.set_attributes({\n", + " # Input metrics\n", + " \"input_query\": f\"Evaluation for query: {query[:100]}{'...' if len(query) > 100 else ''}\",\n", + " \"query_length\": len(query),\n", + " \"query_type\": \"rag_evaluation_query\",\n", + " \n", + " # Retrieval metrics\n", + " \"retrieval.context_retrieved\": f\"Context: {context[:300]}{'...' if len(context) > 300 else ''}\",\n", + " \"retrieval.context_length\": context_length,\n", + " \"retrieval.sources_count\": sources_count,\n", + " \"retrieval.context_quality\": \"high\" if context_length > 500 else \"medium\" if context_length > 200 else \"low\",\n", + " \"retrieval.effectiveness\": sources_count / 5.0, # Normalized to max expected sources\n", + " \"retrieval.context_utilization\": context_length / 1000.0, # Normalized context usage\n", + " \n", + " # Prompt engineering metrics\n", + " \"prompt.complete_prompt\": rag_prompt,\n", + " \"prompt.prompt_length\": len(rag_prompt),\n", + " \"prompt.context_injection\": f\"Context injected: {context[:200]}{'...' if len(context) > 200 else ''}\",\n", + " \"prompt.instruction_following\": \"rag_optimized\",\n", + " \n", + " # Response quality metrics\n", + " \"response.answer_length\": answer_length,\n", + " \"response.answer_completeness\": \"complete\" if answer_length > 100 else \"brief\",\n", + " \"response.response_quality\": \"high\" if answer_length > 200 and sources_count > 2 else \"medium\" if answer_length > 100 else \"low\",\n", + " \"response.source_citation\": sources_count,\n", + " \"response.context_utilization\": answer_length / context_length if context_length > 0 else 0,\n", + " \"output_response\": f\"RAG Answer: {answer[:200]}{'...' if len(answer) > 200 else ''}\",\n", + " \n", + " # Evaluation metrics\n", + " \"evaluation.retrieval_effectiveness\": sources_count / 5.0,\n", + " \"evaluation.response_completeness\": \"complete\" if answer_length > 150 else \"partial\",\n", + " \"evaluation.source_diversity\": sources_count,\n", + " \"evaluation.context_relevance\": \"high\" if context_length > 500 else \"medium\" if context_length > 200 else \"low\",\n", + " \"evaluation.overall_quality\": \"high\" if answer_length > 200 and sources_count > 2 and context_length > 500 else \"medium\" if answer_length > 100 and sources_count > 1 else \"low\",\n", + " \"evaluation.ready_for_production\": True,\n", + " \n", + " # RAG-specific metrics\n", + " \"rag.retrieval_strategy\": \"semantic_similarity\",\n", + " \"rag.vector_search_results\": sources_count,\n", + " \"rag.context_synthesis\": \"multi_source\" if sources_count > 1 else \"single_source\",\n", + " \"rag.document_coverage\": sources_count / 5.0, # Normalized coverage\n", + " \"rag.information_density\": answer_length / context_length if context_length > 0 else 0\n", + " })\n", + "\n", + " # Set main RAG span attributes (simplified)\n", + " rag_span.set_attributes({\n", + " \"input_query\": query,\n", + " \"query_length\": len(query),\n", + " \"query_type\": \"rag_query\",\n", + " \"output_response\": f\"RAG Answer: {answer[:200]}{'...' if len(answer) > 200 else ''}\",\n", + " \"rag.context_length\": context_length,\n", + " \"rag.sources_count\": sources_count,\n", + " \"rag.answer_length\": answer_length,\n", + " \"rag.mode\": \"retrieval_augmented_generation\"\n", + " })\n", + "\n", + " return {\n", + " \"answer\": answer,\n", + " \"context\": context,\n", + " \"mode\": \"RAG\",\n", + " \"sources\": [doc.metadata.get('url', 'Unknown') for doc in self.search_relevant_docs(query, CONFIG[\"max_search_results\"])],\n", + " \"model_info\": {\n", + " \"name\": model_name,\n", + " \"tokens_used\": total_tokens,\n", + " \"prompt_tokens\": prompt_tokens,\n", + " \"completion_tokens\": completion_tokens,\n", + " \"latency\": model_latency\n", + " }\n", + " }\n", + "\n", + "# Initialize RAG system\n", + "rag_system = NoveumRAGSystem(embeddings, llm, CONFIG)\n", + "print(\"āœ… RAG system initialized!\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": 52, + "id": "09823cb6", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "āœ… Web search system initialized!\n" + ] + } + ], + "source": [ + "# Cell 5: Web Search Integration - DuckDuckGo search for external queries\n", + "class NoveumWebSearchSystem:\n", + " def __init__(self, web_search_tool, llm, config):\n", + " self.web_search = web_search_tool\n", + " self.llm = llm\n", + " self.config = config\n", + " \n", + " def search_web(self, query: str, max_results: int = 5) -> List[Dict[str, Any]]:\n", + " \"\"\"Perform web search and return formatted results\"\"\"\n", + " try:\n", + " # Perform web search\n", + " search_results = self.web_search.run(query)\n", + " \n", + " # Parse results (DuckDuckGo returns a string, need to parse it)\n", + " results = []\n", + " if isinstance(search_results, str):\n", + " # Split by lines and parse each result\n", + " lines = search_results.split('\\n')\n", + " for i, line in enumerate(lines[:max_results]):\n", + " if line.strip():\n", + " results.append({\n", + " \"title\": f\"Search Result {i+1}\",\n", + " \"snippet\": line.strip(),\n", + " \"url\": f\"https://duckduckgo.com/?q={query.replace(' ', '+')}\"\n", + " })\n", + " else:\n", + " # If it's already a list/dict format\n", + " results = search_results[:max_results]\n", + " \n", + " print(f\"šŸ” Found {len(results)} web search results for: '{query}'\")\n", + " return results\n", + " \n", + " except Exception as e:\n", + " print(f\"āŒ Error performing web search: {e}\")\n", + " return []\n", + " \n", + " def format_search_context(self, search_results: List[Dict[str, Any]]) -> str:\n", + " \"\"\"Format search results into context string\"\"\"\n", + " if not search_results:\n", + " return \"No search results found.\"\n", + " \n", + " context_parts = []\n", + " for i, result in enumerate(search_results, 1):\n", + " title = result.get('title', f'Result {i}')\n", + " snippet = result.get('snippet', 'No description available')\n", + " url = result.get('url', 'No URL available')\n", + " \n", + " context_parts.append(f\"Source {i} - {title}:\\n{snippet}\\nURL: {url}\")\n", + " \n", + " return \"\\n\\n\".join(context_parts)\n", + " \n", + " def generate_web_response(self, query: str) -> Dict[str, Any]:\n", + " \"\"\"Generate response using web search\"\"\"\n", + " with trace_agent(\n", + " agent_type=\"web_search_agent\",\n", + " operation=\"web_search_generation\",\n", + " capabilities=[\"web_search\", \"content_synthesis\", \"response_generation\"],\n", + " attributes={\n", + " \"agent.id\": \"noveum_web_search_agent\",\n", + " \"input_query\": query,\n", + " \"query_length\": len(query)\n", + " }\n", + " ) as web_span:\n", + " \n", + " # Perform web search\n", + " search_results = self.search_web(query, self.config[\"max_search_results\"])\n", + " \n", + " # Format context\n", + " context = self.format_search_context(search_results)\n", + " \n", + " # Create prompt for web search response\n", + " web_prompt = f\"\"\"You are a helpful assistant. Answer the user's question based on the provided web search results.\n", + "\n", + "Web Search Results:\n", + "{context}\n", + "\n", + "User Question: {query}\n", + "\n", + "Instructions:\n", + "1. Answer based on the provided web search results\n", + "2. Synthesize information from multiple sources when relevant\n", + "3. Be informative and accurate\n", + "4. If the results don't contain enough information, say so clearly\n", + "5. Keep responses concise but comprehensive\n", + "6. Cite sources when possible\n", + "\n", + "Answer:\"\"\"\n", + "\n", + " # Extract model parameters and metadata\n", + " model_name = getattr(self.llm, 'model_name', 'unknown')\n", + " model_temperature = getattr(self.llm, 'temperature', 0.0)\n", + " model_max_tokens = getattr(self.llm, 'max_tokens', None)\n", + " model_top_p = getattr(self.llm, 'top_p', None)\n", + " model_frequency_penalty = getattr(self.llm, 'frequency_penalty', None)\n", + " model_presence_penalty = getattr(self.llm, 'presence_penalty', None)\n", + "\n", + " # Model Details Span - Track model-specific information\n", + " with trace_agent(\n", + " agent_type=\"model_details\",\n", + " operation=\"llm_model_execution\",\n", + " capabilities=[\"model_invocation\", \"parameter_tracking\", \"latency_measurement\"],\n", + " attributes={\n", + " \"agent.id\": \"noveum_model_details\",\n", + " \"input_query\": f\"Model execution for query: {query[:100]}{'...' if len(query) > 100 else ''}\",\n", + " \"query_length\": len(query)\n", + " }\n", + " ) as model_span:\n", + " \n", + " # Record start time for latency measurement\n", + " model_start_time = time.time()\n", + " \n", + " # Generate response\n", + " response = self.llm.invoke(web_prompt)\n", + " \n", + " # Record end time and calculate latency\n", + " model_end_time = time.time()\n", + " model_latency = model_end_time - model_start_time\n", + "\n", + " # Handle response content extraction\n", + " if hasattr(response, 'content'):\n", + " # When response is a proper SDK object\n", + " answer = response.content\n", + " elif isinstance(response, dict):\n", + " # When response is returned as a plain dict\n", + " answer = response.get('content', '')\n", + " else:\n", + " # Fallback to string\n", + " answer = str(response)\n", + "\n", + " # Extract token usage metadata - Enhanced extraction\n", + " prompt_tokens = 0\n", + " completion_tokens = 0\n", + " total_tokens = 0\n", + " \n", + " # Try multiple ways to extract token usage\n", + " if hasattr(response, 'usage_metadata') and response.usage_metadata:\n", + " usage = response.usage_metadata\n", + " prompt_tokens = getattr(usage, \"input_tokens\", 0) or getattr(usage, \"prompt_tokens\", 0)\n", + " completion_tokens = getattr(usage, \"output_tokens\", 0) or getattr(usage, \"completion_tokens\", 0)\n", + " total_tokens = getattr(usage, \"total_tokens\", 0)\n", + " elif hasattr(response, 'response_metadata') and response.response_metadata:\n", + " metadata = response.response_metadata\n", + " if 'token_usage' in metadata:\n", + " token_usage = metadata['token_usage']\n", + " prompt_tokens = token_usage.get('prompt_tokens', 0)\n", + " completion_tokens = token_usage.get('completion_tokens', 0)\n", + " total_tokens = token_usage.get('total_tokens', 0)\n", + " elif hasattr(response, 'token_usage'):\n", + " token_usage = response.token_usage\n", + " prompt_tokens = getattr(token_usage, \"prompt_tokens\", 0)\n", + " completion_tokens = getattr(token_usage, \"completion_tokens\", 0)\n", + " total_tokens = getattr(token_usage, \"total_tokens\", 0)\n", + " \n", + " # If still no tokens found, try to estimate from content length\n", + " if total_tokens == 0:\n", + " # Rough estimation: ~4 characters per token for English text\n", + " estimated_prompt_tokens = len(web_prompt) // 4\n", + " estimated_completion_tokens = len(answer) // 4\n", + " prompt_tokens = estimated_prompt_tokens\n", + " completion_tokens = estimated_completion_tokens\n", + " total_tokens = prompt_tokens + completion_tokens\n", + "\n", + " # Set model details span attributes\n", + " model_span.set_attributes({\n", + " # Input metrics\n", + " \"input_query\": f\"Model execution for query: {query[:100]}{'...' if len(query) > 100 else ''}\",\n", + " \"query_length\": len(query),\n", + " \"query_type\": \"web_search_model_query\",\n", + " \n", + " # Model parameters and configuration\n", + " \"model.name\": model_name,\n", + " \"model.temperature\": model_temperature,\n", + " \"model.max_tokens\": model_max_tokens,\n", + " \"model.top_p\": model_top_p,\n", + " \"model.frequency_penalty\": model_frequency_penalty,\n", + " \"model.presence_penalty\": model_presence_penalty,\n", + " \"model.provider\": \"openai\",\n", + " \"model.type\": \"chat_completion\",\n", + " \"model.version\": \"gpt-4o-mini\",\n", + " \n", + " # Latency and performance metrics\n", + " \"model.latency_seconds\": model_latency,\n", + " \"model.latency_ms\": model_latency * 1000,\n", + " \"model.start_time\": model_start_time,\n", + " \"model.end_time\": model_end_time,\n", + " \"model.performance_tier\": \"fast\" if model_latency < 2.0 else \"medium\" if model_latency < 5.0 else \"slow\",\n", + " \n", + " # Token usage and cost metrics\n", + " \"model.prompt_tokens\": prompt_tokens,\n", + " \"model.completion_tokens\": completion_tokens,\n", + " \"model.total_tokens\": total_tokens,\n", + " \"model.tokens_per_second\": total_tokens / model_latency if model_latency > 0 else 0,\n", + " \"model.estimated_cost\": total_tokens * 0.00003, # Rough cost estimate\n", + " \"model.efficiency_score\": len(answer) / total_tokens if total_tokens > 0 else 0,\n", + " \n", + " # Response characteristics\n", + " \"model.response_length\": len(answer),\n", + " \"model.response_quality\": \"high\" if len(answer) > 200 else \"medium\" if len(answer) > 100 else \"low\",\n", + " \"model.output_response\": f\"Model Response: {answer[:200]}{'...' if len(answer) > 200 else ''}\",\n", + " \n", + " # Model configuration details\n", + " \"model.config\": {\n", + " \"name\": model_name,\n", + " \"temperature\": model_temperature,\n", + " \"max_tokens\": model_max_tokens,\n", + " \"top_p\": model_top_p,\n", + " \"frequency_penalty\": model_frequency_penalty,\n", + " \"presence_penalty\": model_presence_penalty,\n", + " \"provider\": \"openai\",\n", + " \"type\": \"chat_completion\"\n", + " }\n", + " })\n", + "\n", + " # Other Details Span - Track web search, response quality, and evaluation metrics\n", + " with trace_agent(\n", + " agent_type=\"other_details\",\n", + " operation=\"web_search_evaluation_metrics\",\n", + " capabilities=[\"web_search_analysis\", \"response_evaluation\", \"quality_assessment\"],\n", + " attributes={\n", + " \"agent.id\": \"noveum_other_details\",\n", + " \"input_query\": f\"Evaluation for query: {query[:100]}{'...' if len(query) > 100 else ''}\",\n", + " \"query_length\": len(query)\n", + " }\n", + " ) as other_span:\n", + " \n", + " # Calculate additional evaluation metrics\n", + " search_results_count = len(search_results)\n", + " context_length = len(context)\n", + " answer_length = len(answer or \"\")\n", + " \n", + " # Set other details span attributes\n", + " other_span.set_attributes({\n", + " # Input metrics\n", + " \"input_query\": f\"Evaluation for query: {query[:100]}{'...' if len(query) > 100 else ''}\",\n", + " \"query_length\": len(query),\n", + " \"query_type\": \"web_search_evaluation_query\",\n", + " \n", + " # Web search metrics\n", + " \"web_search.results_count\": search_results_count,\n", + " \"web_search.context_length\": context_length,\n", + " \"web_search.context_synthesized\": f\"Context from {search_results_count} web sources\",\n", + " \"web_search.search_effectiveness\": search_results_count / 5.0, # Normalized to max expected results\n", + " \"web_search.context_quality\": \"high\" if context_length > 800 else \"medium\" if context_length > 400 else \"low\",\n", + " \"web_search.source_diversity\": search_results_count,\n", + " \"web_search.information_synthesis\": \"high\" if search_results_count > 3 and answer_length > 200 else \"medium\" if search_results_count > 1 else \"low\",\n", + " \"web_search.external_knowledge_utilization\": context_length / 1000.0, # Normalized context usage\n", + " \n", + " # Prompt engineering metrics\n", + " \"prompt.complete_prompt\": web_prompt,\n", + " \"prompt.prompt_length\": len(web_prompt),\n", + " \"prompt.context_injection\": f\"Web context injected: {context[:200]}{'...' if len(context) > 200 else ''}\",\n", + " \"prompt.instruction_following\": \"web_search_optimized\",\n", + " \n", + " # Response quality metrics\n", + " \"response.answer_length\": answer_length,\n", + " \"response.answer_completeness\": \"complete\" if answer_length > 150 else \"brief\",\n", + " \"response.response_quality\": \"high\" if answer_length > 300 and search_results_count > 3 else \"medium\" if answer_length > 150 else \"low\",\n", + " \"response.source_citation\": search_results_count,\n", + " \"response.context_utilization\": answer_length / context_length if context_length > 0 else 0,\n", + " \"output_response\": f\"Web Search Answer: {answer[:200]}{'...' if len(answer or '') > 200 else ''}\" if answer else \"No answer generated\",\n", + " \n", + " # Evaluation metrics\n", + " \"evaluation.search_effectiveness\": search_results_count / 5.0,\n", + " \"evaluation.response_completeness\": \"complete\" if answer_length > 200 else \"partial\",\n", + " \"evaluation.source_diversity\": search_results_count,\n", + " \"evaluation.context_relevance\": \"high\" if context_length > 800 else \"medium\" if context_length > 400 else \"low\",\n", + " \"evaluation.overall_quality\": \"high\" if answer_length > 300 and search_results_count > 3 and context_length > 800 else \"medium\" if answer_length > 150 and search_results_count > 1 else \"low\",\n", + " \"evaluation.ready_for_production\": True,\n", + " \n", + " # Web search specific metrics\n", + " \"web_search.search_strategy\": \"duckduckgo_api\",\n", + " \"web_search.real_time_data\": True,\n", + " \"web_search.external_sources\": search_results_count,\n", + " \"web_search.information_freshness\": \"current\",\n", + " \"web_search.knowledge_synthesis\": \"multi_source\" if search_results_count > 1 else \"single_source\"\n", + " })\n", + "\n", + " # Set main Web Search span attributes (simplified)\n", + " web_span.set_attributes({\n", + " \"input_query\": query,\n", + " \"query_length\": len(query),\n", + " \"query_type\": \"web_search_query\",\n", + " \"output_response\": f\"Web Search Answer: {answer[:200]}{'...' if len(answer or '') > 200 else ''}\" if answer else \"No answer generated\",\n", + " \"web_search.results_count\": search_results_count,\n", + " \"web_search.context_length\": context_length,\n", + " \"web_search.response_length\": answer_length,\n", + " \"web_search.mode\": \"external_web_search\"\n", + " })\n", + "\n", + " return {\n", + " \"answer\": answer,\n", + " \"context\": context,\n", + " \"mode\": \"Web Search\",\n", + " \"sources\": [result.get('url', 'Unknown') for result in search_results],\n", + " \"model_info\": {\n", + " \"name\": model_name,\n", + " \"tokens_used\": total_tokens,\n", + " \"prompt_tokens\": prompt_tokens,\n", + " \"completion_tokens\": completion_tokens,\n", + " \"latency\": model_latency\n", + " }\n", + " }\n", + "\n", + "# Initialize web search system\n", + "web_search_system = NoveumWebSearchSystem(web_search, llm, CONFIG)\n", + "print(\"āœ… Web search system initialized!\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": 53, + "id": "3d47287f", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "āœ… Query router initialized!\n" + ] + } + ], + "source": [ + "# Cell 6: Query Router - Intelligent decision making between RAG and Web Search\n", + "class NoveumQueryRouter:\n", + " def __init__(self, llm, config):\n", + " self.llm = llm\n", + " self.config = config\n", + " \n", + " # Keywords that suggest RAG should be used\n", + " self.rag_keywords = [\n", + " \"noveum\", \"platform\", \"product\", \"feature\", \"api\", \"documentation\",\n", + " \"trace\", \"observability\", \"monitoring\", \"agent\", \"system\", \"tool\",\n", + " \"integration\", \"setup\", \"configuration\", \"usage\", \"guide\", \"tutorial\",\n", + " \"pricing\", \"plan\", \"subscription\", \"account\", \"dashboard\", \"metrics\"\n", + " ]\n", + " \n", + " # Keywords that suggest Web Search should be used\n", + " self.web_keywords = [\n", + " \"recent\", \"latest\", \"news\", \"update\", \"announcement\", \"release\",\n", + " \"today\", \"yesterday\", \"this week\", \"this month\", \"current\",\n", + " \"trending\", \"popular\", \"viral\", \"breaking\", \"live\", \"real-time\",\n", + " \"weather\", \"stock\", \"price\", \"market\", \"cryptocurrency\", \"bitcoin\",\n", + " \"election\", \"politics\", \"sports\", \"entertainment\", \"celebrity\"\n", + " ]\n", + " \n", + " def classify_query(self, query: str) -> str:\n", + " \"\"\"Classify query to determine whether to use RAG or Web Search\"\"\"\n", + " query_lower = query.lower()\n", + " \n", + " # Check for RAG keywords\n", + " rag_score = sum(1 for keyword in self.rag_keywords if keyword in query_lower)\n", + " \n", + " # Check for Web Search keywords\n", + " web_score = sum(1 for keyword in self.web_keywords if keyword in query_lower)\n", + " \n", + " # Check for explicit mentions of Noveum\n", + " if \"noveum\" in query_lower:\n", + " return \"RAG\"\n", + " \n", + " # If both scores are 0, use LLM-based classification\n", + " if rag_score == 0 and web_score == 0:\n", + " return self._llm_classify_query(query)\n", + " \n", + " # Return the mode with higher score\n", + " return \"RAG\" if rag_score >= web_score else \"Web Search\"\n", + " \n", + " def _llm_classify_query(self, query: str) -> str:\n", + " \"\"\"Use LLM to classify query when keyword matching is inconclusive\"\"\"\n", + " try:\n", + " classification_prompt = f\"\"\"Classify the following user query to determine the best response method:\n", + "\n", + "Query: \"{query}\"\n", + "\n", + "Choose between:\n", + "- RAG: Use when the query is about Noveum.ai platform, products, features, documentation, or internal information\n", + "- Web Search: Use when the query is about recent events, news, general knowledge, or external topics\n", + "\n", + "Respond with only \"RAG\" or \"Web Search\".\"\"\"\n", + "\n", + " # Extract model parameters for tracking\n", + " model_name = getattr(self.llm, 'model_name', 'unknown')\n", + " model_temperature = getattr(self.llm, 'temperature', 0.0)\n", + " model_max_tokens = getattr(self.llm, 'max_tokens', None)\n", + " model_top_p = getattr(self.llm, 'top_p', None)\n", + " model_frequency_penalty = getattr(self.llm, 'frequency_penalty', None)\n", + " model_presence_penalty = getattr(self.llm, 'presence_penalty', None)\n", + "\n", + " # Model Details Span for classification\n", + " with trace_agent(\n", + " agent_type=\"model_details\",\n", + " operation=\"llm_model_execution\",\n", + " capabilities=[\"model_invocation\", \"parameter_tracking\", \"latency_measurement\"],\n", + " attributes={\n", + " \"agent.id\": \"noveum_model_details\",\n", + " \"input_query\": f\"Model execution for classification: {query[:100]}{'...' if len(query) > 100 else ''}\",\n", + " \"query_length\": len(query)\n", + " }\n", + " ) as model_span:\n", + " \n", + " # Record start time for latency measurement\n", + " model_start_time = time.time()\n", + " \n", + " response = self.llm.invoke(classification_prompt)\n", + " \n", + " # Record end time and calculate latency\n", + " model_end_time = time.time()\n", + " model_latency = model_end_time - model_start_time\n", + " \n", + " if hasattr(response, 'content'):\n", + " result = response.content.strip().upper()\n", + " else:\n", + " result = str(response).strip().upper()\n", + "\n", + " # Extract token usage for classification\n", + " prompt_tokens = 0\n", + " completion_tokens = 0\n", + " total_tokens = 0\n", + " \n", + " if hasattr(response, 'usage_metadata') and response.usage_metadata:\n", + " usage = response.usage_metadata\n", + " prompt_tokens = getattr(usage, \"input_tokens\", 0) or getattr(usage, \"prompt_tokens\", 0)\n", + " completion_tokens = getattr(usage, \"output_tokens\", 0) or getattr(usage, \"completion_tokens\", 0)\n", + " total_tokens = getattr(usage, \"total_tokens\", 0)\n", + " elif hasattr(response, 'response_metadata') and response.response_metadata:\n", + " metadata = response.response_metadata\n", + " if 'token_usage' in metadata:\n", + " token_usage = metadata['token_usage']\n", + " prompt_tokens = token_usage.get('prompt_tokens', 0)\n", + " completion_tokens = token_usage.get('completion_tokens', 0)\n", + " total_tokens = token_usage.get('total_tokens', 0)\n", + " \n", + " # If still no tokens found, estimate\n", + " if total_tokens == 0:\n", + " estimated_prompt_tokens = len(classification_prompt) // 4\n", + " estimated_completion_tokens = len(result) // 4\n", + " prompt_tokens = estimated_prompt_tokens\n", + " completion_tokens = estimated_completion_tokens\n", + " total_tokens = prompt_tokens + completion_tokens\n", + "\n", + " # Set model details span attributes\n", + " model_span.set_attributes({\n", + " # Input metrics\n", + " \"input_query\": f\"Model execution for classification: {query[:100]}{'...' if len(query) > 100 else ''}\",\n", + " \"query_length\": len(query),\n", + " \"query_type\": \"classification_model_query\",\n", + " \n", + " # Model parameters and configuration\n", + " \"model.name\": model_name,\n", + " \"model.temperature\": model_temperature,\n", + " \"model.max_tokens\": model_max_tokens,\n", + " \"model.top_p\": model_top_p,\n", + " \"model.frequency_penalty\": model_frequency_penalty,\n", + " \"model.presence_penalty\": model_presence_penalty,\n", + " \"model.provider\": \"openai\",\n", + " \"model.type\": \"chat_completion\",\n", + " \"model.version\": \"gpt-4o-mini\",\n", + " \n", + " # Latency and performance metrics\n", + " \"model.latency_seconds\": model_latency,\n", + " \"model.latency_ms\": model_latency * 1000,\n", + " \"model.start_time\": model_start_time,\n", + " \"model.end_time\": model_end_time,\n", + " \"model.performance_tier\": \"fast\" if model_latency < 1.0 else \"medium\" if model_latency < 3.0 else \"slow\",\n", + " \n", + " # Token usage and cost metrics\n", + " \"model.prompt_tokens\": prompt_tokens,\n", + " \"model.completion_tokens\": completion_tokens,\n", + " \"model.total_tokens\": total_tokens,\n", + " \"model.tokens_per_second\": total_tokens / model_latency if model_latency > 0 else 0,\n", + " \"model.estimated_cost\": total_tokens * 0.00003, # Rough cost estimate\n", + " \"model.efficiency_score\": len(result) / total_tokens if total_tokens > 0 else 0,\n", + " \n", + " # Response characteristics\n", + " \"model.response_length\": len(result),\n", + " \"model.response_quality\": \"high\" if len(result) > 10 else \"medium\" if len(result) > 5 else \"low\",\n", + " \"model.output_response\": f\"Classification Result: {result}\",\n", + " \n", + " # Model configuration details\n", + " \"model.config\": {\n", + " \"name\": model_name,\n", + " \"temperature\": model_temperature,\n", + " \"max_tokens\": model_max_tokens,\n", + " \"top_p\": model_top_p,\n", + " \"frequency_penalty\": model_frequency_penalty,\n", + " \"presence_penalty\": model_presence_penalty,\n", + " \"provider\": \"openai\",\n", + " \"type\": \"chat_completion\"\n", + " }\n", + " })\n", + "\n", + " # Log classification details for debugging\n", + " print(f\"šŸ” LLM Classification - Model: {model_name}, Tokens: {total_tokens}, Result: {result}\")\n", + " \n", + " if \"RAG\" in result:\n", + " return \"RAG\"\n", + " elif \"WEB\" in result or \"SEARCH\" in result:\n", + " return \"Web Search\"\n", + " else:\n", + " # Default to Web Search if unclear\n", + " return \"Web Search\"\n", + " \n", + " except Exception as e:\n", + " print(f\"āŒ Error in LLM classification: {e}\")\n", + " # Default to Web Search on error\n", + " return \"Web Search\"\n", + " \n", + " def route_query(self, query: str) -> Tuple[str, Dict[str, Any]]:\n", + " \"\"\"Route query to appropriate system and return response\"\"\"\n", + " with trace_agent(\n", + " agent_type=\"query_router\",\n", + " operation=\"query_routing\",\n", + " capabilities=[\"query_classification\", \"routing_decision\"],\n", + " attributes={\n", + " \"agent.id\": \"noveum_query_router\",\n", + " \"input_query\": query,\n", + " \"query_length\": len(query)\n", + " }\n", + " ) as router_span:\n", + " \n", + " # Define classification prompt for tracing\n", + " classification_prompt = f\"\"\"Classify the following user query to determine the best response method:\n", + "\n", + "Query: \"{query}\"\n", + "\n", + "Choose between:\n", + "- RAG: Use when the query is about Noveum.ai platform, products, features, documentation, or internal information\n", + "- Web Search: Use when the query is about recent events, news, general knowledge, or external topics\n", + "\n", + "Respond with only \"RAG\" or \"Web Search\".\"\"\"\n", + " \n", + " # Classify the query\n", + " mode = self.classify_query(query)\n", + " \n", + " # Calculate routing evaluation metrics\n", + " query_lower = query.lower()\n", + " rag_keywords = [\"noveum\", \"platform\", \"product\", \"feature\", \"api\", \"documentation\", \"trace\", \"observability\", \"monitoring\", \"agent\", \"system\", \"tool\", \"integration\", \"setup\", \"configuration\", \"usage\", \"guide\", \"tutorial\", \"pricing\", \"plan\", \"subscription\", \"account\", \"dashboard\", \"metrics\"]\n", + " web_keywords = [\"recent\", \"latest\", \"news\", \"update\", \"announcement\", \"release\", \"today\", \"yesterday\", \"this week\", \"this month\", \"current\", \"trending\", \"popular\", \"viral\", \"breaking\", \"live\", \"real-time\", \"weather\", \"stock\", \"price\", \"market\", \"cryptocurrency\", \"bitcoin\", \"election\", \"politics\", \"sports\", \"entertainment\", \"celebrity\"]\n", + " \n", + " rag_score = sum(1 for keyword in rag_keywords if keyword in query_lower)\n", + " web_score = sum(1 for keyword in web_keywords if keyword in query_lower)\n", + " confidence_score = abs(rag_score - web_score) / max(rag_score + web_score, 1)\n", + " \n", + " # Other Details Span - Track routing analysis and decision metrics\n", + " with trace_agent(\n", + " agent_type=\"other_details\",\n", + " operation=\"routing_evaluation_metrics\",\n", + " capabilities=[\"routing_analysis\", \"decision_evaluation\", \"quality_assessment\"],\n", + " attributes={\n", + " \"agent.id\": \"noveum_other_details\",\n", + " \"input_query\": f\"Routing evaluation for query: {query[:100]}{'...' if len(query) > 100 else ''}\",\n", + " \"query_length\": len(query)\n", + " }\n", + " ) as other_span:\n", + " \n", + " # Set other details span attributes\n", + " other_span.set_attributes({\n", + " # Input metrics\n", + " \"input_query\": f\"Routing evaluation for query: {query[:100]}{'...' if len(query) > 100 else ''}\",\n", + " \"query_length\": len(query),\n", + " \"query_type\": \"routing_evaluation_query\",\n", + " \n", + " # Classification metrics\n", + " \"classification.mode\": mode,\n", + " \"classification.rag_keyword_score\": rag_score,\n", + " \"classification.web_keyword_score\": web_score,\n", + " \"classification.confidence_score\": confidence_score,\n", + " \"classification.confidence_level\": \"high\" if confidence_score > 0.5 else \"medium\" if confidence_score > 0.2 else \"low\",\n", + " \"classification.method\": \"llm_based\" if rag_score == 0 and web_score == 0 else \"keyword_based\",\n", + " \n", + " # Query analysis metrics\n", + " \"query.complexity\": \"complex\" if len(query) > 50 else \"medium\" if len(query) > 20 else \"simple\",\n", + " \"query.intent\": \"noveum_specific\" if \"noveum\" in query_lower else \"general_knowledge\" if web_score > rag_score else \"documentation\",\n", + " \"query.keyword_density\": (rag_score + web_score) / len(query.split()),\n", + " \"query.domain_affinity\": \"noveum\" if rag_score > web_score else \"general\" if web_score > rag_score else \"neutral\",\n", + " \n", + " # Routing decision metrics\n", + " \"routing.decision\": f\"Routed to {mode} based on analysis\",\n", + " \"routing.rationale\": f\"RAG score: {rag_score}, Web score: {web_score}, Confidence: {confidence_score:.2f}\",\n", + " \"routing.expected_performance\": \"high\" if confidence_score > 0.5 else \"medium\" if confidence_score > 0.2 else \"low\",\n", + " \"routing.alternative_mode\": \"Web Search\" if mode == \"RAG\" else \"RAG\",\n", + " \"routing.decision_confidence\": confidence_score,\n", + " \n", + " # Evaluation metrics\n", + " \"evaluation.routing_accuracy\": \"high\" if confidence_score > 0.5 else \"medium\" if confidence_score > 0.2 else \"low\",\n", + " \"evaluation.keyword_coverage\": (rag_score + web_score) / len(rag_keywords + web_keywords),\n", + " \"evaluation.query_understanding\": \"clear\" if confidence_score > 0.5 else \"ambiguous\" if confidence_score > 0.2 else \"unclear\",\n", + " \"evaluation.ready_for_production\": True,\n", + " \n", + " # Router-specific metrics\n", + " \"router.classification_strategy\": \"hybrid_keyword_llm\",\n", + " \"router.keyword_matching\": \"used\" if rag_score > 0 or web_score > 0 else \"bypassed\",\n", + " \"router.llm_fallback\": \"used\" if rag_score == 0 and web_score == 0 else \"not_needed\",\n", + " \"router.decision_time\": \"instant\" if rag_score > 0 or web_score > 0 else \"llm_required\",\n", + " \"output_response\": f\"Routing Decision: {mode} (Confidence: {confidence_score:.2f})\"\n", + " })\n", + "\n", + " # Set main router span attributes (simplified)\n", + " router_span.set_attributes({\n", + " \"input_query\": query,\n", + " \"query_length\": len(query),\n", + " \"query_type\": \"routing_query\",\n", + " \"output_response\": f\"Routed to {mode} for query processing\",\n", + " \"router.classification\": mode,\n", + " \"router.confidence_score\": confidence_score,\n", + " \"router.rag_keyword_score\": rag_score,\n", + " \"router.web_keyword_score\": web_score,\n", + " \"router.mode\": \"intelligent_routing\"\n", + " })\n", + " \n", + " # Route to appropriate system\n", + " if mode == \"RAG\":\n", + " print(f\"šŸ§  Routing to RAG system for: '{query}'\")\n", + " response = rag_system.generate_rag_response(query)\n", + " else:\n", + " print(f\"šŸŒ Routing to Web Search for: '{query}'\")\n", + " response = web_search_system.generate_web_response(query)\n", + " \n", + " return mode, response\n", + "\n", + "# Initialize query router\n", + "query_router = NoveumQueryRouter(llm, CONFIG)\n", + "print(\"āœ… Query router initialized!\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": 54, + "id": "a865a57f", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "āœ… Noveum AI Agent initialized!\n" + ] + } + ], + "source": [ + "# Cell 7: Main Executor - Orchestrates the complete agent workflow\n", + "class NoveumAIAgent:\n", + " def __init__(self, scraper, rag_system, web_search_system, query_router, config):\n", + " self.scraper = scraper\n", + " self.rag_system = rag_system\n", + " self.web_search_system = web_search_system\n", + " self.query_router = query_router\n", + " self.config = config\n", + " self.is_initialized = False\n", + " \n", + " def initialize_system(self, force_scrape: bool = False) -> bool:\n", + " \"\"\"Initialize the system by setting up RAG with scraped data\"\"\"\n", + " print(\"šŸš€ Initializing Noveum AI Agent...\")\n", + " \n", + " with trace_operation(\"system_initialization\") as init_span:\n", + " init_span.set_attributes({\n", + " \"system.force_scrape\": force_scrape,\n", + " \"system.config\": self.config,\n", + " \"input_query\": f\"Initialize system with force_scrape={force_scrape}\",\n", + " \"output_response\": \"System initialization: RAG system loaded, vector store ready, agent operational\"\n", + " })\n", + " \n", + " # Check if we need to scrape or if data already exists\n", + " if force_scrape or not os.path.exists(self.config[\"noveum_docs_file\"]):\n", + " print(\"šŸ“„ Scraping Noveum website...\")\n", + " \n", + " # Scrape the website\n", + " scraped_data = self.scraper.scrape_website()\n", + " \n", + " if not scraped_data:\n", + " print(\"āŒ Failed to scrape website data\")\n", + " return False\n", + " \n", + " # Save scraped data\n", + " self.scraper.save_to_json(self.config[\"noveum_docs_file\"])\n", + " \n", + " init_span.add_event(\"website_scraped\", {\n", + " \"input_query\": f\"Scrape website: {self.config['noveum_base_url']}\",\n", + " \"output_response\": f\"Website scraping completed: {len(scraped_data)} pages scraped, {sum(page['content_length'] for page in scraped_data)} total characters extracted for RAG system\",\n", + " \"pages_scraped\": len(scraped_data),\n", + " \"total_content_length\": sum(page[\"content_length\"] for page in scraped_data)\n", + " })\n", + " else:\n", + " print(\"šŸ“ Using existing scraped data...\")\n", + " \n", + " # Load documents and create/load vector store\n", + " documents = self.rag_system.load_documents_from_json(self.config[\"noveum_docs_file\"])\n", + " \n", + " if not documents:\n", + " print(\"āŒ Failed to load documents\")\n", + " return False\n", + " \n", + " # Try to load existing vector store, create if doesn't exist\n", + " if not self.rag_system.load_vectorstore():\n", + " print(\"šŸ”„ Creating new vector store...\")\n", + " self.rag_system.create_vectorstore(documents)\n", + " \n", + " self.is_initialized = True\n", + " print(\"āœ… Noveum AI Agent initialized successfully!\")\n", + " \n", + " init_span.set_attributes({\n", + " \"system.initialized\": True,\n", + " \"system.documents_loaded\": len(documents),\n", + " \"system.vectorstore_ready\": self.rag_system.vectorstore is not None\n", + " })\n", + " \n", + " return True\n", + " \n", + " def process_query(self, query: str) -> Dict[str, Any]:\n", + " \"\"\"Process a user query and return response\"\"\"\n", + " if not self.is_initialized:\n", + " print(\"āŒ System not initialized. Please run initialize_system() first.\")\n", + " return {\n", + " \"answer\": \"System not initialized. Please run initialize_system() first.\",\n", + " \"mode\": \"Error\",\n", + " \"sources\": [],\n", + " \"error\": \"System not initialized\"\n", + " }\n", + " \n", + " print(f\"\\nšŸŽÆ Processing query: '{query}'\")\n", + " \n", + " with trace_operation(\"tool-orchestator\") as process_span:\n", + " process_span.set_attributes({\n", + " \"input_query\": query,\n", + " \"query.length\": len(query)\n", + " })\n", + " \n", + " start_time = time.time()\n", + " \n", + " try:\n", + " # Route query and get response\n", + " mode, response = self.query_router.route_query(query)\n", + " \n", + " # Add processing metrics\n", + " end_time = time.time()\n", + " processing_time = end_time - start_time\n", + " \n", + " response.update({\n", + " \"processing_time\": processing_time,\n", + " \"timestamp\": time.time()\n", + " })\n", + " \n", + " # Add metrics to span\n", + " process_span.set_attributes({\n", + " \"processing.mode\": mode,\n", + " \"processing.time_seconds\": processing_time,\n", + " \"processing.response_length\": len(response.get(\"answer\", \"\")),\n", + " \"processing.sources_count\": len(response.get(\"sources\", [])),\n", + " \"output_response\": f\"Final Answer: {response.get('answer', '')[:200]}{'...' if len(response.get('answer', '')) > 200 else ''}\",\n", + " \"final_answer_mode\": mode,\n", + " \"query_processed.input_query\": query,\n", + " \"query_processed.output_response\": f\"Successfully processed query using {mode}, generated {len(response.get('answer', ''))} character response\",\n", + " \"query_processed.mode\": mode,\n", + " \"query_processed.processing_time\": processing_time,\n", + " \"query_processed.response_length\": len(response.get(\"answer\", \"\"))\n", + " })\n", + " \n", + " print(f\"āœ… Query processed in {processing_time:.2f}s using {mode}\")\n", + " return response\n", + " \n", + " except Exception as e:\n", + " error_msg = f\"Error processing query: {str(e)}\"\n", + " print(f\"āŒ {error_msg}\")\n", + " \n", + " process_span.add_event(\"query_processing_error\", {\n", + " \"error\": str(e),\n", + " \"input_query\": query,\n", + " \"output_response\": f\"I encountered an error while processing your query: {str(e)}\"\n", + " })\n", + " \n", + " return {\n", + " \"answer\": f\"I encountered an error while processing your query: {str(e)}\",\n", + " \"mode\": \"Error\",\n", + " \"sources\": [],\n", + " \"error\": str(e),\n", + " \"processing_time\": time.time() - start_time\n", + " }\n", + " \n", + " def display_response(self, response: Dict[str, Any]) -> None:\n", + " \"\"\"Display the response in a formatted way\"\"\"\n", + " print(\"\\n\" + \"=\"*80)\n", + " print(f\"šŸ¤– NOVEUM AI AGENT RESPONSE\")\n", + " print(\"=\"*80)\n", + " print(f\"šŸ“Š Mode: {response.get('mode', 'Unknown')}\")\n", + " print(f\"ā±ļø Processing Time: {response.get('processing_time', 0):.2f}s\")\n", + " print(f\"šŸ“… Timestamp: {time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(response.get('timestamp', time.time())))}\")\n", + " \n", + " if response.get('sources'):\n", + " print(f\"šŸ“š Sources ({len(response['sources'])}):\")\n", + " for i, source in enumerate(response['sources'][:3], 1): # Show first 3 sources\n", + " print(f\" {i}. {source}\")\n", + " if len(response['sources']) > 3:\n", + " print(f\" ... and {len(response['sources']) - 3} more\")\n", + " \n", + " print(\"\\nšŸ’¬ Answer:\")\n", + " print(\"-\" * 40)\n", + " print(response.get('answer', 'No answer provided'))\n", + " print(\"=\"*80)\n", + "\n", + "# Initialize the main agent\n", + "noveum_agent = NoveumAIAgent(scraper, rag_system, web_search_system, query_router, CONFIG)\n", + "print(\"āœ… Noveum AI Agent initialized!\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": 56, + "id": "4b30e6d3", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "āœ… Demo functions ready!\n", + "\n", + "šŸš€ To get started:\n", + "1. Run: demo_noveum_agent() # For a full demo\n", + "2. Run: ask_question('Your question here') # For a single question\n", + "3. Or use: noveum_agent.process_query('Your question') # For programmatic access\n" + ] + } + ], + "source": [ + "# Cell 8: Usage Examples and Demo\n", + "def demo_noveum_agent():\n", + " \"\"\"Demo function showing how to use the Noveum AI Agent\"\"\"\n", + " \n", + " print(\"šŸŽ¬ NOVEUM AI AGENT DEMO\")\n", + " print(\"=\"*50)\n", + " \n", + " # Step 1: Initialize the system\n", + " print(\"\\n1ļøāƒ£ Initializing the system...\")\n", + " success = noveum_agent.initialize_system(force_scrape=False) # Set to True to force re-scraping\n", + " \n", + " if not success:\n", + " print(\"āŒ Failed to initialize system\")\n", + " return\n", + " \n", + " # Step 2: Demo queries - 20 comprehensive test questions\n", + " demo_queries = [\n", + " # RAG Queries (Noveum-specific)\n", + " \"What is Noveum and what does it do?\", # Basic product info\n", + " \"How do I integrate Noveum Trace in my application?\", # Technical integration\n", + " \"What are Noveum's pricing plans?\", # Pricing information\n", + " \"What features does Noveum Trace offer?\", # Feature overview\n", + " \"How do I set up observability with Noveum?\", # Setup guidance\n", + " \"What APIs are available in Noveum platform?\", # API documentation\n", + " \"How does Noveum handle agent tracing?\", # Technical details\n", + " \"What monitoring capabilities does Noveum provide?\", # Capabilities\n", + " \"How do I configure Noveum for my system?\", # Configuration\n", + " \"What are the benefits of using Noveum Trace?\", # Value proposition\n", + " \n", + " # Web Search Queries (External/Recent information)\n", + " \"What are the latest AI news today?\", # Recent news\n", + " \"What's the weather like today?\", # Current weather\n", + " \"Tell me about recent developments in machine learning\", # Recent developments\n", + " \"What are the current trends in observability tools?\", # Industry trends\n", + " \"What happened in tech news this week?\", # Weekly tech news\n", + " \"What are the latest updates in Python programming?\", # Recent updates\n", + " \"What's the current status of cryptocurrency markets?\", # Market information\n", + " \"What are the newest features in cloud computing?\", # Recent features\n", + " \"What's happening in the software development world today?\", # Current events\n", + " \"What are the latest breakthroughs in artificial intelligence?\" # Recent breakthroughs\n", + " ]\n", + " \n", + " print(f\"\\n2ļøāƒ£ Running {len(demo_queries)} demo queries...\")\n", + " \n", + " for i, query in enumerate(demo_queries, 1):\n", + " print(f\"\\n--- Demo Query {i} ---\")\n", + " response = noveum_agent.process_query(query)\n", + " noveum_agent.display_response(response)\n", + " \n", + " # Small delay between queries\n", + " time.sleep(1)\n", + " \n", + " print(f\"\\nšŸŽ‰ Demo completed! Check Noveum Trace dashboard for detailed observability data.\")\n", + " print(\"šŸ’” You can now use noveum_agent.process_query('your question') for your own queries!\")\n", + "\n", + "# Interactive query function\n", + "def ask_question(question: str):\n", + " \"\"\"Convenience function to ask a single question\"\"\"\n", + " if not noveum_agent.is_initialized:\n", + " print(\"āš ļø System not initialized. Initializing now...\")\n", + " if not noveum_agent.initialize_system():\n", + " print(\"āŒ Failed to initialize system\")\n", + " return\n", + " \n", + " response = noveum_agent.process_query(question)\n", + " noveum_agent.display_response(response)\n", + " return response\n", + "\n", + "print(\"āœ… Demo functions ready!\")\n", + "print(\"\\nšŸš€ To get started:\")\n", + "print(\"1. Run: demo_noveum_agent() # For a full demo\")\n", + "print(\"2. Run: ask_question('Your question here') # For a single question\")\n", + "print(\"3. Or use: noveum_agent.process_query('Your question') # For programmatic access\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": 58, + "id": "f727a446", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "šŸŽ¬ NOVEUM AI AGENT DEMO\n", + "==================================================\n", + "\n", + "1ļøāƒ£ Initializing the system...\n", + "šŸš€ Initializing Noveum AI Agent...\n", + "šŸ“ Using existing scraped data...\n", + "āœ… Loaded 38 documents from noveum_docs.json\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-10-22 22:26:35 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_system_initialization (ID: 9af2b4b0-0c2f-4d2e-ae99-30df0d2f0fa6) - 1 spans\n", + "2025-10-22 22:26:35 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_system_initialization (ID: 9af2b4b0-0c2f-4d2e-ae99-30df0d2f0fa6) - 1 spans\n", + "2025-10-22 22:26:35 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 9af2b4b0-0c2f-4d2e-ae99-30df0d2f0fa6\n", + "2025-10-22 22:26:35 - noveum_trace.transport.http_transport - INFO - āœ… Trace 9af2b4b0-0c2f-4d2e-ae99-30df0d2f0fa6 successfully queued for export\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "āœ… Loaded existing vector store from noveum_vectorstore\n", + "āœ… Noveum AI Agent initialized successfully!\n", + "\n", + "2ļøāƒ£ Running 20 demo queries...\n", + "\n", + "--- Demo Query 1 ---\n", + "\n", + "šŸŽÆ Processing query: 'What is Noveum and what does it do?'\n", + "šŸ§  Routing to RAG system for: 'What is Noveum and what does it do?'\n", + "šŸ” Found 5 relevant documents for query: 'What is Noveum and what does it do?'\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-10-22 22:26:43 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_tool-orchestator (ID: 3ad192e5-bbd4-4236-8144-019d53512769) - 6 spans\n", + "2025-10-22 22:26:43 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_tool-orchestator (ID: 3ad192e5-bbd4-4236-8144-019d53512769) - 6 spans\n", + "2025-10-22 22:26:43 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 3ad192e5-bbd4-4236-8144-019d53512769\n", + "2025-10-22 22:26:43 - noveum_trace.transport.http_transport - INFO - āœ… Trace 3ad192e5-bbd4-4236-8144-019d53512769 successfully queued for export\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "šŸ” Found 5 relevant documents for query: 'What is Noveum and what does it do?'\n", + "āœ… Query processed in 8.68s using RAG\n", + "\n", + "================================================================================\n", + "šŸ¤– NOVEUM AI AGENT RESPONSE\n", + "================================================================================\n", + "šŸ“Š Mode: RAG\n", + "ā±ļø Processing Time: 8.68s\n", + "šŸ“… Timestamp: 2025-10-22 22:26:43\n", + "šŸ“š Sources (5):\n", + " 1. https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform\n", + " 2. https://noveum.ai/en/docs/getting-started/overview\n", + " 3. https://noveum.ai/docs/getting-started/overview\n", + " ... and 2 more\n", + "\n", + "šŸ’¬ Answer:\n", + "----------------------------------------\n", + "Noveum.ai is a comprehensive tracing and observability platform specifically designed for AI applications, including LLM-powered chatbots, RAG systems, and multi-agent workflows. It provides the necessary insights to understand, debug, and optimize these systems, addressing the complexities involved in building production AI applications, such as LLM calls, vector searches, and agent reasoning. Noveum.ai enhances visibility into these workflows, making debugging easier and optimization more effective (Source 1, Source 2).\n", + "================================================================================\n", + "\n", + "--- Demo Query 2 ---\n", + "\n", + "šŸŽÆ Processing query: 'How do I integrate Noveum Trace in my application?'\n", + "šŸ§  Routing to RAG system for: 'How do I integrate Noveum Trace in my application?'\n", + "šŸ” Found 5 relevant documents for query: 'How do I integrate Noveum Trace in my application?'\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-10-22 22:26:53 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_tool-orchestator (ID: bc60c122-f30b-4114-9f78-ee40a024b525) - 6 spans\n", + "2025-10-22 22:26:53 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_tool-orchestator (ID: bc60c122-f30b-4114-9f78-ee40a024b525) - 6 spans\n", + "2025-10-22 22:26:53 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace bc60c122-f30b-4114-9f78-ee40a024b525\n", + "2025-10-22 22:26:53 - noveum_trace.transport.http_transport - INFO - āœ… Trace bc60c122-f30b-4114-9f78-ee40a024b525 successfully queued for export\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "šŸ” Found 5 relevant documents for query: 'How do I integrate Noveum Trace in my application?'\n", + "āœ… Query processed in 8.83s using RAG\n", + "\n", + "================================================================================\n", + "šŸ¤– NOVEUM AI AGENT RESPONSE\n", + "================================================================================\n", + "šŸ“Š Mode: RAG\n", + "ā±ļø Processing Time: 8.83s\n", + "šŸ“… Timestamp: 2025-10-22 22:26:53\n", + "šŸ“š Sources (5):\n", + " 1. https://noveum.ai/en/docs\n", + " 2. https://noveum.ai/docs\n", + " 3. https://noveum.ai/docs/getting-started/sdk-integration\n", + " ... and 2 more\n", + "\n", + "šŸ’¬ Answer:\n", + "----------------------------------------\n", + "To integrate Noveum Trace into your application, you can follow these steps based on the SDK Integration Guide:\n", + "\n", + "1. **Create Your Account & Get API Key**:\n", + " - Sign up at [noveum.ai](https://noveum.ai).\n", + " - Create a project in your dashboard.\n", + " - Generate an API key.\n", + "\n", + "2. **Choose Your SDK**:\n", + " - For Python applications, use the **Python SDK** (`noveum-trace`), which offers decorator-based tracing for LLM calls, agents, and RAG pipelines.\n", + " - For TypeScript applications, use the **TypeScript SDK** (`@noveum/trace`), designed for frameworks like Next.js, Express.js, and Hono, with full type safety.\n", + "\n", + "3. **Integrate the SDK**:\n", + " - Follow the specific integration instructions for your chosen SDK. The SDKs are designed to provide comprehensive tracing and observability with minimal code changes.\n", + "\n", + "4. **Capture Metrics**:\n", + " - The SDKs automatically capture essential metrics and traces, including performance metrics, cost tracking, request/response data, and metadata.\n", + "\n", + "For detailed integration steps, refer to the SDK Integration Guide in the Noveum documentation ([Source 3](https://noveum.ai/docs/getting-started/sdk-integration)).\n", + "================================================================================\n", + "\n", + "--- Demo Query 3 ---\n", + "\n", + "šŸŽÆ Processing query: 'What are Noveum's pricing plans?'\n", + "šŸ§  Routing to RAG system for: 'What are Noveum's pricing plans?'\n", + "šŸ” Found 5 relevant documents for query: 'What are Noveum's pricing plans?'\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-10-22 22:26:58 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_tool-orchestator (ID: 29a862bf-d7aa-4cf9-963c-0a4244343bbe) - 6 spans\n", + "2025-10-22 22:26:58 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_tool-orchestator (ID: 29a862bf-d7aa-4cf9-963c-0a4244343bbe) - 6 spans\n", + "2025-10-22 22:26:58 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 29a862bf-d7aa-4cf9-963c-0a4244343bbe\n", + "2025-10-22 22:26:58 - noveum_trace.transport.http_transport - INFO - āœ… Trace 29a862bf-d7aa-4cf9-963c-0a4244343bbe successfully queued for export\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "šŸ” Found 5 relevant documents for query: 'What are Noveum's pricing plans?'\n", + "āœ… Query processed in 3.49s using RAG\n", + "\n", + "================================================================================\n", + "šŸ¤– NOVEUM AI AGENT RESPONSE\n", + "================================================================================\n", + "šŸ“Š Mode: RAG\n", + "ā±ļø Processing Time: 3.49s\n", + "šŸ“… Timestamp: 2025-10-22 22:26:58\n", + "šŸ“š Sources (5):\n", + " 1. https://noveum.ai/docs/getting-started/overview\n", + " 2. https://noveum.ai/en/docs/getting-started/overview\n", + " 3. https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform\n", + " ... and 2 more\n", + "\n", + "šŸ’¬ Answer:\n", + "----------------------------------------\n", + "The provided context does not include specific information about Noveum's pricing plans. It mentions that all users will have access to the Observability suite for free, and early users will receive free evaluation jobs and premium support for the first year (Source 1 and Source 2). However, details on ongoing pricing or additional plans are not available in the documentation. If you have further questions about Noveum, feel free to ask!\n", + "================================================================================\n", + "\n", + "--- Demo Query 4 ---\n", + "\n", + "šŸŽÆ Processing query: 'What features does Noveum Trace offer?'\n", + "šŸ§  Routing to RAG system for: 'What features does Noveum Trace offer?'\n", + "šŸ” Found 5 relevant documents for query: 'What features does Noveum Trace offer?'\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-10-22 22:27:05 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_tool-orchestator (ID: df0c6344-1a2e-4c53-bb7d-4e43b3bce1b8) - 6 spans\n", + "2025-10-22 22:27:05 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_tool-orchestator (ID: df0c6344-1a2e-4c53-bb7d-4e43b3bce1b8) - 6 spans\n", + "2025-10-22 22:27:05 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace df0c6344-1a2e-4c53-bb7d-4e43b3bce1b8\n", + "2025-10-22 22:27:05 - noveum_trace.transport.http_transport - INFO - āœ… Trace df0c6344-1a2e-4c53-bb7d-4e43b3bce1b8 successfully queued for export\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "šŸ” Found 5 relevant documents for query: 'What features does Noveum Trace offer?'\n", + "āœ… Query processed in 6.38s using RAG\n", + "\n", + "================================================================================\n", + "šŸ¤– NOVEUM AI AGENT RESPONSE\n", + "================================================================================\n", + "šŸ“Š Mode: RAG\n", + "ā±ļø Processing Time: 6.38s\n", + "šŸ“… Timestamp: 2025-10-22 22:27:05\n", + "šŸ“š Sources (5):\n", + " 1. https://noveum.ai/en/docs\n", + " 2. https://noveum.ai/docs\n", + " 3. https://noveum.ai/en/docs/getting-started/sdk-integration\n", + " ... and 2 more\n", + "\n", + "šŸ’¬ Answer:\n", + "----------------------------------------\n", + "Noveum Trace offers several key features through its Python SDK (`noveum-trace`) and TypeScript SDK (`@noveum/trace`):\n", + "\n", + "1. **Decorator-based Tracing**: It provides a simple way to trace LLM calls, agents, and RAG pipelines using decorators.\n", + "2. **Automatic Instrumentation**: The SDK automatically instruments popular AI frameworks, making it easier to integrate tracing without extensive manual setup.\n", + "3. **Context Propagation**: It supports context propagation across asynchronous operations, ensuring that trace data remains consistent throughout the execution flow.\n", + "4. **TypeScript-first Design**: The TypeScript SDK is designed with full type safety and universal compatibility across Node.js, Edge Runtime, and browsers.\n", + "5. **Framework Integrations**: It includes integrations for popular frameworks like Next.js, Express.js, and Hono.\n", + "\n", + "Additionally, the Noveum Platform features a real-time dashboard for analyzing traces and performance, allowing for collaborative insights and monitoring (Sources 1, 2, 3, 4).\n", + "================================================================================\n", + "\n", + "--- Demo Query 5 ---\n", + "\n", + "šŸŽÆ Processing query: 'How do I set up observability with Noveum?'\n", + "šŸ§  Routing to RAG system for: 'How do I set up observability with Noveum?'\n", + "šŸ” Found 5 relevant documents for query: 'How do I set up observability with Noveum?'\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-10-22 22:27:12 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_tool-orchestator (ID: 93bd37c4-4289-43c9-b34e-19eaaef610c2) - 6 spans\n", + "2025-10-22 22:27:12 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_tool-orchestator (ID: 93bd37c4-4289-43c9-b34e-19eaaef610c2) - 6 spans\n", + "2025-10-22 22:27:12 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 93bd37c4-4289-43c9-b34e-19eaaef610c2\n", + "2025-10-22 22:27:12 - noveum_trace.transport.http_transport - INFO - āœ… Trace 93bd37c4-4289-43c9-b34e-19eaaef610c2 successfully queued for export\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "šŸ” Found 5 relevant documents for query: 'How do I set up observability with Noveum?'\n", + "āœ… Query processed in 5.72s using RAG\n", + "\n", + "================================================================================\n", + "šŸ¤– NOVEUM AI AGENT RESPONSE\n", + "================================================================================\n", + "šŸ“Š Mode: RAG\n", + "ā±ļø Processing Time: 5.72s\n", + "šŸ“… Timestamp: 2025-10-22 22:27:12\n", + "šŸ“š Sources (5):\n", + " 1. https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform\n", + " 2. https://noveum.ai/en/docs/getting-started/tracing-concepts\n", + " 3. https://noveum.ai/docs/getting-started/tracing-concepts\n", + " ... and 2 more\n", + "\n", + "šŸ’¬ Answer:\n", + "----------------------------------------\n", + "To set up observability with Noveum, follow these steps:\n", + "\n", + "1. **Sign Up**: Go to [noveum.ai](https://noveum.ai) and create an account.\n", + "2. **Create a Project**: After signing up, create a project to obtain your API key.\n", + "3. **Install the SDK**: Choose your preferred programming language and install the corresponding SDK.\n", + "4. **Add Tracing**: Integrate tracing into your AI workflows using the SDK.\n", + "5. **Explore Insights**: Use the Noveum.ai dashboard to analyze the insights gathered from your traces.\n", + "\n", + "For more detailed guidance, you can explore framework-specific integrations and advanced instrumentation techniques in the Noveum documentation (Source 2 and Source 3). Remember, effective observability is about collecting the right data to help you understand, debug, and optimize your AI applications (Source 2).\n", + "================================================================================\n", + "\n", + "--- Demo Query 6 ---\n", + "\n", + "šŸŽÆ Processing query: 'What APIs are available in Noveum platform?'\n", + "šŸ§  Routing to RAG system for: 'What APIs are available in Noveum platform?'\n", + "šŸ” Found 5 relevant documents for query: 'What APIs are available in Noveum platform?'\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-10-22 22:27:18 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_tool-orchestator (ID: fee3e6d3-8da1-4ef3-8214-a7d7323d3a49) - 6 spans\n", + "2025-10-22 22:27:18 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_tool-orchestator (ID: fee3e6d3-8da1-4ef3-8214-a7d7323d3a49) - 6 spans\n", + "2025-10-22 22:27:18 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace fee3e6d3-8da1-4ef3-8214-a7d7323d3a49\n", + "2025-10-22 22:27:18 - noveum_trace.transport.http_transport - INFO - āœ… Trace fee3e6d3-8da1-4ef3-8214-a7d7323d3a49 successfully queued for export\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "šŸ” Found 5 relevant documents for query: 'What APIs are available in Noveum platform?'\n", + "āœ… Query processed in 5.00s using RAG\n", + "\n", + "================================================================================\n", + "šŸ¤– NOVEUM AI AGENT RESPONSE\n", + "================================================================================\n", + "šŸ“Š Mode: RAG\n", + "ā±ļø Processing Time: 5.00s\n", + "šŸ“… Timestamp: 2025-10-22 22:27:18\n", + "šŸ“š Sources (5):\n", + " 1. https://noveum.ai/en/docs\n", + " 2. https://noveum.ai/docs\n", + " 3. https://noveum.ai/docs/getting-started/overview\n", + " ... and 2 more\n", + "\n", + "šŸ’¬ Answer:\n", + "----------------------------------------\n", + "The provided context does not specify the APIs available in the Noveum platform. For detailed information about the APIs, I recommend checking the official Noveum documentation or the SDK Integration Guide. If you have any other questions about Noveum, feel free to ask!\n", + "================================================================================\n", + "\n", + "--- Demo Query 7 ---\n", + "\n", + "šŸŽÆ Processing query: 'How does Noveum handle agent tracing?'\n", + "šŸ§  Routing to RAG system for: 'How does Noveum handle agent tracing?'\n", + "šŸ” Found 5 relevant documents for query: 'How does Noveum handle agent tracing?'\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-10-22 22:27:23 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_tool-orchestator (ID: 3bbd0b36-c30f-4126-b62a-dd0fae1c701d) - 6 spans\n", + "2025-10-22 22:27:23 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_tool-orchestator (ID: 3bbd0b36-c30f-4126-b62a-dd0fae1c701d) - 6 spans\n", + "2025-10-22 22:27:23 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 3bbd0b36-c30f-4126-b62a-dd0fae1c701d\n", + "2025-10-22 22:27:23 - noveum_trace.transport.http_transport - INFO - āœ… Trace 3bbd0b36-c30f-4126-b62a-dd0fae1c701d successfully queued for export\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "šŸ” Found 5 relevant documents for query: 'How does Noveum handle agent tracing?'\n", + "āœ… Query processed in 4.48s using RAG\n", + "\n", + "================================================================================\n", + "šŸ¤– NOVEUM AI AGENT RESPONSE\n", + "================================================================================\n", + "šŸ“Š Mode: RAG\n", + "ā±ļø Processing Time: 4.48s\n", + "šŸ“… Timestamp: 2025-10-22 22:27:23\n", + "šŸ“š Sources (5):\n", + " 1. https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform\n", + " 2. https://noveum.ai/docs/advanced/multi-agent-tracing\n", + " 3. https://noveum.ai/en/docs/advanced/multi-agent-tracing\n", + " ... and 2 more\n", + "\n", + "šŸ’¬ Answer:\n", + "----------------------------------------\n", + "Noveum.ai handles agent tracing through its specialized Multi-Agent Tracing capabilities, which are designed to observe complex workflows and inter-agent communications within multi-agent systems. These systems often involve multiple agents that coordinate, communicate, and collaborate to achieve shared goals, presenting unique observability challenges. Noveum.ai provides comprehensive tracing to help users understand and optimize these intricate workflows, ensuring better visibility into agent interactions and performance (Source 2 and Source 3). \n", + "\n", + "For more detailed insights or specific use cases, you may want to explore the Noveum documentation further.\n", + "================================================================================\n", + "\n", + "--- Demo Query 8 ---\n", + "\n", + "šŸŽÆ Processing query: 'What monitoring capabilities does Noveum provide?'\n", + "šŸ§  Routing to RAG system for: 'What monitoring capabilities does Noveum provide?'\n", + "šŸ” Found 5 relevant documents for query: 'What monitoring capabilities does Noveum provide?'\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-10-22 22:27:31 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_tool-orchestator (ID: fb0d31fc-2465-4ecd-973c-4cd443d9befa) - 6 spans\n", + "2025-10-22 22:27:31 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_tool-orchestator (ID: fb0d31fc-2465-4ecd-973c-4cd443d9befa) - 6 spans\n", + "2025-10-22 22:27:31 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace fb0d31fc-2465-4ecd-973c-4cd443d9befa\n", + "2025-10-22 22:27:31 - noveum_trace.transport.http_transport - INFO - āœ… Trace fb0d31fc-2465-4ecd-973c-4cd443d9befa successfully queued for export\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "šŸ” Found 5 relevant documents for query: 'What monitoring capabilities does Noveum provide?'\n", + "āœ… Query processed in 6.97s using RAG\n", + "\n", + "================================================================================\n", + "šŸ¤– NOVEUM AI AGENT RESPONSE\n", + "================================================================================\n", + "šŸ“Š Mode: RAG\n", + "ā±ļø Processing Time: 6.97s\n", + "šŸ“… Timestamp: 2025-10-22 22:27:31\n", + "šŸ“š Sources (5):\n", + " 1. https://noveum.ai/en\n", + " 2. https://noveum.ai/\n", + " 3. https://noveum.ai\n", + " ... and 2 more\n", + "\n", + "šŸ’¬ Answer:\n", + "----------------------------------------\n", + "Noveum provides comprehensive monitoring capabilities for AI applications, including the ability to monitor, trace, and optimize AI agents across various frameworks such as LangChain, CrewAI, AutoGen, and custom implementations. The platform features a unified dashboard that allows users to monitor everything in their AI ecosystem, capturing every trace and span from simple LLM calls to complex multi-agent interactions. This is facilitated by lightweight SDKs that ensure no detail is missed, enabling users to evaluate and improve their AI agents effectively (Sources 1, 2, 4).\n", + "================================================================================\n", + "\n", + "--- Demo Query 9 ---\n", + "\n", + "šŸŽÆ Processing query: 'How do I configure Noveum for my system?'\n", + "šŸ§  Routing to RAG system for: 'How do I configure Noveum for my system?'\n", + "šŸ” Found 5 relevant documents for query: 'How do I configure Noveum for my system?'\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-10-22 22:27:37 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_tool-orchestator (ID: 1ae0d6c4-d1fb-4239-b536-3333187d44aa) - 6 spans\n", + "2025-10-22 22:27:37 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_tool-orchestator (ID: 1ae0d6c4-d1fb-4239-b536-3333187d44aa) - 6 spans\n", + "2025-10-22 22:27:37 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 1ae0d6c4-d1fb-4239-b536-3333187d44aa\n", + "2025-10-22 22:27:37 - noveum_trace.transport.http_transport - INFO - āœ… Trace 1ae0d6c4-d1fb-4239-b536-3333187d44aa successfully queued for export\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "šŸ” Found 5 relevant documents for query: 'How do I configure Noveum for my system?'\n", + "āœ… Query processed in 4.69s using RAG\n", + "\n", + "================================================================================\n", + "šŸ¤– NOVEUM AI AGENT RESPONSE\n", + "================================================================================\n", + "šŸ“Š Mode: RAG\n", + "ā±ļø Processing Time: 4.69s\n", + "šŸ“… Timestamp: 2025-10-22 22:27:37\n", + "šŸ“š Sources (5):\n", + " 1. https://noveum.ai/en/docs\n", + " 2. https://noveum.ai/docs\n", + " 3. https://noveum.ai/docs/getting-started/overview\n", + " ... and 2 more\n", + "\n", + "šŸ’¬ Answer:\n", + "----------------------------------------\n", + "To configure Noveum for your system, you can refer to the SDK Integration Guide available in the Noveum documentation. This guide will help you trace your AI applications with minimal code changes. The SDKs provided for Python and TypeScript are designed to facilitate this integration. \n", + "\n", + "For detailed steps and specific configurations, please check the relevant sections in the documentation. If you need further assistance, you can also reach out to the Noveum community via Discord or email support at [email protected] (Sources: Source 1, Source 5).\n", + "================================================================================\n", + "\n", + "--- Demo Query 10 ---\n", + "\n", + "šŸŽÆ Processing query: 'What are the benefits of using Noveum Trace?'\n", + "šŸ§  Routing to RAG system for: 'What are the benefits of using Noveum Trace?'\n", + "šŸ” Found 5 relevant documents for query: 'What are the benefits of using Noveum Trace?'\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-10-22 22:27:46 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_tool-orchestator (ID: 823b7927-10bb-49ef-a94e-1b914c298bfb) - 6 spans\n", + "2025-10-22 22:27:46 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_tool-orchestator (ID: 823b7927-10bb-49ef-a94e-1b914c298bfb) - 6 spans\n", + "2025-10-22 22:27:46 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 823b7927-10bb-49ef-a94e-1b914c298bfb\n", + "2025-10-22 22:27:46 - noveum_trace.transport.http_transport - INFO - āœ… Trace 823b7927-10bb-49ef-a94e-1b914c298bfb successfully queued for export\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "šŸ” Found 5 relevant documents for query: 'What are the benefits of using Noveum Trace?'\n", + "āœ… Query processed in 7.97s using RAG\n", + "\n", + "================================================================================\n", + "šŸ¤– NOVEUM AI AGENT RESPONSE\n", + "================================================================================\n", + "šŸ“Š Mode: RAG\n", + "ā±ļø Processing Time: 7.97s\n", + "šŸ“… Timestamp: 2025-10-22 22:27:46\n", + "šŸ“š Sources (5):\n", + " 1. https://noveum.ai/docs/platform/dashboard\n", + " 2. https://noveum.ai/en/docs\n", + " 3. https://noveum.ai/docs\n", + " ... and 2 more\n", + "\n", + "šŸ’¬ Answer:\n", + "----------------------------------------\n", + "Using Noveum Trace offers several benefits, including:\n", + "\n", + "1. **Real-Time Monitoring**: The Noveum platform provides a real-time dashboard for analyzing traces and performance, with live updates as new traces arrive and real-time status monitoring of your trace ingestion pipeline (Source 1).\n", + "\n", + "2. **Advanced Analysis**: Users can perform comparative analysis across multiple traces, allowing for performance comparisons and insights into system health through live metrics (Source 1).\n", + "\n", + "3. **Cost Analysis and Optimization**: Noveum Trace includes features for cost analysis and optimization recommendations, helping users manage and reduce expenses associated with AI operations (Source 4).\n", + "\n", + "4. **Collaboration Features**: The platform supports team collaboration by enabling shared insights and alerts, facilitating better communication and teamwork (Source 4).\n", + "\n", + "5. **Comprehensive Tracing**: With the Python and TypeScript SDKs, Noveum Trace offers decorator-based tracing for LLM calls, agents, and RAG pipelines, along with automatic instrumentation for popular AI frameworks (Source 2).\n", + "\n", + "6. **Detailed Metrics**: It provides detailed metrics on model calls, token usage, prompt engineering effectiveness, and response quality, which are crucial for optimizing AI applications (Source 4).\n", + "\n", + "Overall, Noveum Trace enhances the ability to monitor, analyze, and optimize AI applications effectively.\n", + "================================================================================\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/Users/mramanindia/work/NovaEval/.venv/lib/python3.12/site-packages/langchain_community/utilities/duckduckgo_search.py:63: RuntimeWarning: This package (`duckduckgo_search`) has been renamed to `ddgs`! Use `pip install ddgs` instead.\n", + " with DDGS() as ddgs:\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "--- Demo Query 11 ---\n", + "\n", + "šŸŽÆ Processing query: 'What are the latest AI news today?'\n", + "šŸŒ Routing to Web Search for: 'What are the latest AI news today?'\n", + "šŸ” Found 1 web search results for: 'What are the latest AI news today?'\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-10-22 22:27:50 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_tool-orchestator (ID: e4f3cc20-c137-4905-80fe-2d29b03c2231) - 6 spans\n", + "2025-10-22 22:27:50 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_tool-orchestator (ID: e4f3cc20-c137-4905-80fe-2d29b03c2231) - 6 spans\n", + "2025-10-22 22:27:50 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace e4f3cc20-c137-4905-80fe-2d29b03c2231\n", + "2025-10-22 22:27:50 - noveum_trace.transport.http_transport - INFO - āœ… Trace e4f3cc20-c137-4905-80fe-2d29b03c2231 successfully queued for export\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "āœ… Query processed in 3.48s using Web Search\n", + "\n", + "================================================================================\n", + "šŸ¤– NOVEUM AI AGENT RESPONSE\n", + "================================================================================\n", + "šŸ“Š Mode: Web Search\n", + "ā±ļø Processing Time: 3.48s\n", + "šŸ“… Timestamp: 2025-10-22 22:27:50\n", + "šŸ“š Sources (1):\n", + " 1. https://duckduckgo.com/?q=What+are+the+latest+AI+news+today?\n", + "\n", + "šŸ’¬ Answer:\n", + "----------------------------------------\n", + "The provided web search results do not contain specific information about the latest AI news today. They primarily focus on general news coverage from India and around the world, including politics, business, and entertainment, but do not mention any recent developments or updates in artificial intelligence.\n", + "\n", + "For the latest AI news, I recommend checking dedicated technology news websites or platforms that specialize in AI developments, as they would provide more relevant and up-to-date information.\n", + "================================================================================\n", + "\n", + "--- Demo Query 12 ---\n", + "\n", + "šŸŽÆ Processing query: 'What's the weather like today?'\n", + "šŸŒ Routing to Web Search for: 'What's the weather like today?'\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/Users/mramanindia/work/NovaEval/.venv/lib/python3.12/site-packages/langchain_community/utilities/duckduckgo_search.py:63: RuntimeWarning: This package (`duckduckgo_search`) has been renamed to `ddgs`! Use `pip install ddgs` instead.\n", + " with DDGS() as ddgs:\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "šŸ” Found 1 web search results for: 'What's the weather like today?'\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-10-22 22:27:56 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_tool-orchestator (ID: b1b13e60-5b32-44a0-9769-197a944989d0) - 6 spans\n", + "2025-10-22 22:27:56 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_tool-orchestator (ID: b1b13e60-5b32-44a0-9769-197a944989d0) - 6 spans\n", + "2025-10-22 22:27:56 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace b1b13e60-5b32-44a0-9769-197a944989d0\n", + "2025-10-22 22:27:56 - noveum_trace.transport.http_transport - INFO - āœ… Trace b1b13e60-5b32-44a0-9769-197a944989d0 successfully queued for export\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "āœ… Query processed in 4.93s using Web Search\n", + "\n", + "================================================================================\n", + "šŸ¤– NOVEUM AI AGENT RESPONSE\n", + "================================================================================\n", + "šŸ“Š Mode: Web Search\n", + "ā±ļø Processing Time: 4.93s\n", + "šŸ“… Timestamp: 2025-10-22 22:27:56\n", + "šŸ“š Sources (1):\n", + " 1. https://duckduckgo.com/?q=What's+the+weather+like+today?\n", + "\n", + "šŸ’¬ Answer:\n", + "----------------------------------------\n", + "The web search results do not provide specific information about today's weather. To find out the current weather conditions, I recommend checking a reliable weather website or app for detailed updates, including temperature, wind, and precipitation forecasts.\n", + "================================================================================\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/Users/mramanindia/work/NovaEval/.venv/lib/python3.12/site-packages/langchain_community/utilities/duckduckgo_search.py:63: RuntimeWarning: This package (`duckduckgo_search`) has been renamed to `ddgs`! Use `pip install ddgs` instead.\n", + " with DDGS() as ddgs:\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "--- Demo Query 13 ---\n", + "\n", + "šŸŽÆ Processing query: 'Tell me about recent developments in machine learning'\n", + "šŸŒ Routing to Web Search for: 'Tell me about recent developments in machine learning'\n", + "šŸ” Found 1 web search results for: 'Tell me about recent developments in machine learning'\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-10-22 22:28:03 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_tool-orchestator (ID: c5a64d08-d00f-46cf-ab3c-d8d473c10eca) - 6 spans\n", + "2025-10-22 22:28:03 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_tool-orchestator (ID: c5a64d08-d00f-46cf-ab3c-d8d473c10eca) - 6 spans\n", + "2025-10-22 22:28:03 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace c5a64d08-d00f-46cf-ab3c-d8d473c10eca\n", + "2025-10-22 22:28:03 - noveum_trace.transport.http_transport - INFO - āœ… Trace c5a64d08-d00f-46cf-ab3c-d8d473c10eca successfully queued for export\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "āœ… Query processed in 5.97s using Web Search\n", + "\n", + "================================================================================\n", + "šŸ¤– NOVEUM AI AGENT RESPONSE\n", + "================================================================================\n", + "šŸ“Š Mode: Web Search\n", + "ā±ļø Processing Time: 5.97s\n", + "šŸ“… Timestamp: 2025-10-22 22:28:03\n", + "šŸ“š Sources (1):\n", + " 1. https://duckduckgo.com/?q=Tell+me+about+recent+developments+in+machine+learning\n", + "\n", + "šŸ’¬ Answer:\n", + "----------------------------------------\n", + "The web search results did not provide specific information about recent developments in machine learning. Therefore, I cannot summarize any recent advancements based on the provided data.\n", + "\n", + "However, if you're interested in general trends in machine learning, I can mention that recent developments often include advancements in deep learning architectures, improvements in natural language processing (NLP) models, and increased applications of machine learning in various industries such as healthcare, finance, and autonomous systems. For the latest updates, I recommend checking reputable tech news sources or academic journals focused on artificial intelligence and machine learning.\n", + "================================================================================\n", + "\n", + "--- Demo Query 14 ---\n", + "\n", + "šŸŽÆ Processing query: 'What are the current trends in observability tools?'\n", + "šŸ§  Routing to RAG system for: 'What are the current trends in observability tools?'\n", + "šŸ” Found 5 relevant documents for query: 'What are the current trends in observability tools?'\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-10-22 22:28:10 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_tool-orchestator (ID: 45efae74-ee25-4004-a509-6e0ff628a0cd) - 6 spans\n", + "2025-10-22 22:28:10 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_tool-orchestator (ID: 45efae74-ee25-4004-a509-6e0ff628a0cd) - 6 spans\n", + "2025-10-22 22:28:10 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 45efae74-ee25-4004-a509-6e0ff628a0cd\n", + "2025-10-22 22:28:10 - noveum_trace.transport.http_transport - INFO - āœ… Trace 45efae74-ee25-4004-a509-6e0ff628a0cd successfully queued for export\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "šŸ” Found 5 relevant documents for query: 'What are the current trends in observability tools?'\n", + "āœ… Query processed in 5.15s using RAG\n", + "\n", + "================================================================================\n", + "šŸ¤– NOVEUM AI AGENT RESPONSE\n", + "================================================================================\n", + "šŸ“Š Mode: RAG\n", + "ā±ļø Processing Time: 5.15s\n", + "šŸ“… Timestamp: 2025-10-22 22:28:10\n", + "šŸ“š Sources (5):\n", + " 1. https://noveum.ai/en/blog/from-logs-to-intelligent-choices-inside-noveum-ais-evaluation-process\n", + " 2. https://noveum.ai/docs/getting-started/tracing-concepts\n", + " 3. https://noveum.ai/en/docs/getting-started/tracing-concepts\n", + " ... and 2 more\n", + "\n", + "šŸ’¬ Answer:\n", + "----------------------------------------\n", + "The provided context does not explicitly outline current trends in observability tools. However, it highlights the unique challenges faced by AI applications that traditional monitoring tools cannot adequately address. For instance, Noveum.ai emphasizes the importance of comprehensive tracing and collecting the right data to understand, debug, and optimize AI systems, rather than just gathering all possible data (Source 2 and Source 4).\n", + "\n", + "Additionally, the context mentions advanced practices such as comprehensive pipeline tracing, quality monitoring, and A/B testing for RAG components, which suggest a trend towards more specialized and tailored observability solutions for AI applications (Source 5).\n", + "\n", + "If you have specific questions about Noveum.ai's observability platform or its features, feel free to ask!\n", + "================================================================================\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/Users/mramanindia/work/NovaEval/.venv/lib/python3.12/site-packages/langchain_community/utilities/duckduckgo_search.py:63: RuntimeWarning: This package (`duckduckgo_search`) has been renamed to `ddgs`! Use `pip install ddgs` instead.\n", + " with DDGS() as ddgs:\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "--- Demo Query 15 ---\n", + "\n", + "šŸŽÆ Processing query: 'What happened in tech news this week?'\n", + "šŸŒ Routing to Web Search for: 'What happened in tech news this week?'\n", + "šŸ” Found 1 web search results for: 'What happened in tech news this week?'\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-10-22 22:28:14 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_tool-orchestator (ID: e30e9ba6-ffd1-4eb9-b7ae-6d19e04343d6) - 6 spans\n", + "2025-10-22 22:28:14 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_tool-orchestator (ID: e30e9ba6-ffd1-4eb9-b7ae-6d19e04343d6) - 6 spans\n", + "2025-10-22 22:28:14 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace e30e9ba6-ffd1-4eb9-b7ae-6d19e04343d6\n", + "2025-10-22 22:28:14 - noveum_trace.transport.http_transport - INFO - āœ… Trace e30e9ba6-ffd1-4eb9-b7ae-6d19e04343d6 successfully queued for export\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "āœ… Query processed in 3.22s using Web Search\n", + "\n", + "================================================================================\n", + "šŸ¤– NOVEUM AI AGENT RESPONSE\n", + "================================================================================\n", + "šŸ“Š Mode: Web Search\n", + "ā±ļø Processing Time: 3.22s\n", + "šŸ“… Timestamp: 2025-10-22 22:28:14\n", + "šŸ“š Sources (1):\n", + " 1. https://duckduckgo.com/?q=What+happened+in+tech+news+this+week?\n", + "\n", + "šŸ’¬ Answer:\n", + "----------------------------------------\n", + "The web search results do not provide specific information about recent events in tech news for this week. They primarily discuss the grammatical usage of the word \"happened\" and its variations, such as \"what happened\" and \"what's happened.\" \n", + "\n", + "To find out what happened in tech news this week, I recommend checking reliable tech news websites or platforms that aggregate current events in technology. If you have specific topics or companies in mind, I can help guide you on where to look for that information.\n", + "================================================================================\n", + "\n", + "--- Demo Query 16 ---\n", + "\n", + "šŸŽÆ Processing query: 'What are the latest updates in Python programming?'\n", + "šŸŒ Routing to Web Search for: 'What are the latest updates in Python programming?'\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/Users/mramanindia/work/NovaEval/.venv/lib/python3.12/site-packages/langchain_community/utilities/duckduckgo_search.py:63: RuntimeWarning: This package (`duckduckgo_search`) has been renamed to `ddgs`! Use `pip install ddgs` instead.\n", + " with DDGS() as ddgs:\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "šŸ” Found 1 web search results for: 'What are the latest updates in Python programming?'\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-10-22 22:28:18 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_tool-orchestator (ID: 51f4534f-23e4-4be7-8d6e-e93391d6e0e2) - 6 spans\n", + "2025-10-22 22:28:18 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_tool-orchestator (ID: 51f4534f-23e4-4be7-8d6e-e93391d6e0e2) - 6 spans\n", + "2025-10-22 22:28:18 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 51f4534f-23e4-4be7-8d6e-e93391d6e0e2\n", + "2025-10-22 22:28:18 - noveum_trace.transport.http_transport - INFO - āœ… Trace 51f4534f-23e4-4be7-8d6e-e93391d6e0e2 successfully queued for export\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "āœ… Query processed in 3.62s using Web Search\n", + "\n", + "================================================================================\n", + "šŸ¤– NOVEUM AI AGENT RESPONSE\n", + "================================================================================\n", + "šŸ“Š Mode: Web Search\n", + "ā±ļø Processing Time: 3.62s\n", + "šŸ“… Timestamp: 2025-10-22 22:28:18\n", + "šŸ“š Sources (1):\n", + " 1. https://duckduckgo.com/?q=What+are+the+latest+updates+in+Python+programming?\n", + "\n", + "šŸ’¬ Answer:\n", + "----------------------------------------\n", + "The provided web search results do not contain specific information about the latest updates in Python programming. They primarily focus on general news coverage and updates from India and around the world, without addressing programming or Python specifically.\n", + "\n", + "For the latest updates in Python programming, I recommend checking official sources such as the Python Software Foundation's website or popular programming news platforms like Real Python or Python Weekly. These sources typically provide information on new releases, features, and enhancements in the Python language.\n", + "================================================================================\n", + "\n", + "--- Demo Query 17 ---\n", + "\n", + "šŸŽÆ Processing query: 'What's the current status of cryptocurrency markets?'\n", + "šŸŒ Routing to Web Search for: 'What's the current status of cryptocurrency markets?'\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/Users/mramanindia/work/NovaEval/.venv/lib/python3.12/site-packages/langchain_community/utilities/duckduckgo_search.py:63: RuntimeWarning: This package (`duckduckgo_search`) has been renamed to `ddgs`! Use `pip install ddgs` instead.\n", + " with DDGS() as ddgs:\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "šŸ” Found 1 web search results for: 'What's the current status of cryptocurrency markets?'\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-10-22 22:28:25 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_tool-orchestator (ID: 3f7b6a62-8888-4a5d-b12c-37a3829aea49) - 6 spans\n", + "2025-10-22 22:28:25 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_tool-orchestator (ID: 3f7b6a62-8888-4a5d-b12c-37a3829aea49) - 6 spans\n", + "2025-10-22 22:28:25 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 3f7b6a62-8888-4a5d-b12c-37a3829aea49\n", + "2025-10-22 22:28:25 - noveum_trace.transport.http_transport - INFO - āœ… Trace 3f7b6a62-8888-4a5d-b12c-37a3829aea49 successfully queued for export\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "āœ… Query processed in 5.65s using Web Search\n", + "\n", + "================================================================================\n", + "šŸ¤– NOVEUM AI AGENT RESPONSE\n", + "================================================================================\n", + "šŸ“Š Mode: Web Search\n", + "ā±ļø Processing Time: 5.65s\n", + "šŸ“… Timestamp: 2025-10-22 22:28:25\n", + "šŸ“š Sources (1):\n", + " 1. https://duckduckgo.com/?q=What's+the+current+status+of+cryptocurrency+markets?\n", + "\n", + "šŸ’¬ Answer:\n", + "----------------------------------------\n", + "The provided web search results do not contain specific information regarding the current status of cryptocurrency markets. To get the latest updates on cryptocurrency prices, trends, and market analysis, I recommend checking financial news websites, cryptocurrency exchanges, or market tracking platforms like CoinMarketCap or CoinGecko. If you have any other questions or need further assistance, feel free to ask!\n", + "================================================================================\n", + "\n", + "--- Demo Query 18 ---\n", + "\n", + "šŸŽÆ Processing query: 'What are the newest features in cloud computing?'\n", + "šŸ§  Routing to RAG system for: 'What are the newest features in cloud computing?'\n", + "šŸ” Found 5 relevant documents for query: 'What are the newest features in cloud computing?'\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-10-22 22:28:31 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_tool-orchestator (ID: 343764b8-7ac9-4549-93cf-8abc11ceba6f) - 6 spans\n", + "2025-10-22 22:28:31 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_tool-orchestator (ID: 343764b8-7ac9-4549-93cf-8abc11ceba6f) - 6 spans\n", + "2025-10-22 22:28:31 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 343764b8-7ac9-4549-93cf-8abc11ceba6f\n", + "2025-10-22 22:28:31 - noveum_trace.transport.http_transport - INFO - āœ… Trace 343764b8-7ac9-4549-93cf-8abc11ceba6f successfully queued for export\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "šŸ” Found 5 relevant documents for query: 'What are the newest features in cloud computing?'\n", + "āœ… Query processed in 5.25s using RAG\n", + "\n", + "================================================================================\n", + "šŸ¤– NOVEUM AI AGENT RESPONSE\n", + "================================================================================\n", + "šŸ“Š Mode: RAG\n", + "ā±ļø Processing Time: 5.25s\n", + "šŸ“… Timestamp: 2025-10-22 22:28:31\n", + "šŸ“š Sources (5):\n", + " 1. https://noveum.ai/en/changelog\n", + " 2. https://noveum.ai/en/changelog\n", + " 3. https://noveum.ai/docs/platform/dashboard\n", + " ... and 2 more\n", + "\n", + "šŸ’¬ Answer:\n", + "----------------------------------------\n", + "The provided context does not specifically mention any new features in cloud computing as a general topic. However, it does highlight several recent enhancements related to the Noveum.ai platform, which may involve cloud computing aspects. These include:\n", + "\n", + "1. **Enhanced Dashboard Analytics**: Real-time request tracking, latency monitoring, and cost analysis (Source 1).\n", + "2. **Improved Logs Interface**: Better search functionality and enhanced debugging capabilities (Source 1).\n", + "3. **Advanced Metrics Collection**: A comprehensive telemetry system with custom metrics (Source 1).\n", + "4. **Major Platform Evolution**: The launch of the NovaEval Framework for AI model evaluation, which includes production-grade deployment on Docker and Kubernetes (Source 4).\n", + "\n", + "If you have specific questions about Noveum.ai or its features, feel free to ask!\n", + "================================================================================\n", + "\n", + "--- Demo Query 19 ---\n", + "\n", + "šŸŽÆ Processing query: 'What's happening in the software development world today?'\n", + "šŸŒ Routing to Web Search for: 'What's happening in the software development world today?'\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/Users/mramanindia/work/NovaEval/.venv/lib/python3.12/site-packages/langchain_community/utilities/duckduckgo_search.py:63: RuntimeWarning: This package (`duckduckgo_search`) has been renamed to `ddgs`! Use `pip install ddgs` instead.\n", + " with DDGS() as ddgs:\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "šŸ” Found 1 web search results for: 'What's happening in the software development world today?'\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-10-22 22:28:41 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_tool-orchestator (ID: ec7c6cae-9bbd-47f4-a8ae-2d6d9b5011ba) - 6 spans\n", + "2025-10-22 22:28:41 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_tool-orchestator (ID: ec7c6cae-9bbd-47f4-a8ae-2d6d9b5011ba) - 6 spans\n", + "2025-10-22 22:28:41 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace ec7c6cae-9bbd-47f4-a8ae-2d6d9b5011ba\n", + "2025-10-22 22:28:41 - noveum_trace.transport.http_transport - INFO - āœ… Trace ec7c6cae-9bbd-47f4-a8ae-2d6d9b5011ba successfully queued for export\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "āœ… Query processed in 8.28s using Web Search\n", + "\n", + "================================================================================\n", + "šŸ¤– NOVEUM AI AGENT RESPONSE\n", + "================================================================================\n", + "šŸ“Š Mode: Web Search\n", + "ā±ļø Processing Time: 8.28s\n", + "šŸ“… Timestamp: 2025-10-22 22:28:41\n", + "šŸ“š Sources (1):\n", + " 1. https://duckduckgo.com/?q=What's+happening+in+the+software+development+world+today?\n", + "\n", + "šŸ’¬ Answer:\n", + "----------------------------------------\n", + "The web search did not yield any specific results regarding current events in the software development world. Therefore, I cannot provide detailed information on the latest trends, technologies, or news in this field.\n", + "\n", + "However, generally speaking, the software development landscape is often influenced by several ongoing trends, such as:\n", + "\n", + "1. **Increased Adoption of AI and Machine Learning**: Many companies are integrating AI into their software solutions to enhance functionality and user experience.\n", + "\n", + "2. **Remote Work and Collaboration Tools**: The shift to remote work has led to a surge in the development and use of collaboration tools and platforms.\n", + "\n", + "3. **DevOps and Continuous Integration/Continuous Deployment (CI/CD)**: There is a growing emphasis on DevOps practices to improve collaboration between development and operations teams, leading to faster and more reliable software delivery.\n", + "\n", + "4. **Focus on Cybersecurity**: With the rise in cyber threats, there is an increasing focus on building secure software and implementing robust security practices throughout the development lifecycle.\n", + "\n", + "5. **Low-Code and No-Code Development**: These platforms are gaining popularity as they allow users to create applications with minimal coding, making software development more accessible.\n", + "\n", + "For the most accurate and up-to-date information, I recommend checking reputable tech news websites or software development blogs.\n", + "================================================================================\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/Users/mramanindia/work/NovaEval/.venv/lib/python3.12/site-packages/langchain_community/utilities/duckduckgo_search.py:63: RuntimeWarning: This package (`duckduckgo_search`) has been renamed to `ddgs`! Use `pip install ddgs` instead.\n", + " with DDGS() as ddgs:\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "--- Demo Query 20 ---\n", + "\n", + "šŸŽÆ Processing query: 'What are the latest breakthroughs in artificial intelligence?'\n", + "šŸŒ Routing to Web Search for: 'What are the latest breakthroughs in artificial intelligence?'\n", + "šŸ” Found 1 web search results for: 'What are the latest breakthroughs in artificial intelligence?'\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-10-22 22:28:44 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_tool-orchestator (ID: 6fe93abf-1413-49e8-8464-58558e25ab7b) - 6 spans\n", + "2025-10-22 22:28:44 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_tool-orchestator (ID: 6fe93abf-1413-49e8-8464-58558e25ab7b) - 6 spans\n", + "2025-10-22 22:28:44 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 6fe93abf-1413-49e8-8464-58558e25ab7b\n", + "2025-10-22 22:28:44 - noveum_trace.transport.http_transport - INFO - āœ… Trace 6fe93abf-1413-49e8-8464-58558e25ab7b successfully queued for export\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "āœ… Query processed in 2.68s using Web Search\n", + "\n", + "================================================================================\n", + "šŸ¤– NOVEUM AI AGENT RESPONSE\n", + "================================================================================\n", + "šŸ“Š Mode: Web Search\n", + "ā±ļø Processing Time: 2.68s\n", + "šŸ“… Timestamp: 2025-10-22 22:28:44\n", + "šŸ“š Sources (1):\n", + " 1. https://duckduckgo.com/?q=What+are+the+latest+breakthroughs+in+artificial+intelligence?\n", + "\n", + "šŸ’¬ Answer:\n", + "----------------------------------------\n", + "The web search did not yield specific results regarding the latest breakthroughs in artificial intelligence. However, I can provide a general overview based on knowledge up to October 2023.\n", + "\n", + "Recent breakthroughs in artificial intelligence include advancements in natural language processing (NLP), particularly with models like GPT-4, which have improved understanding and generation of human-like text. Additionally, there have been significant developments in computer vision, enabling AI systems to better interpret and analyze visual data.\n", + "\n", + "Another notable area of progress is in reinforcement learning, where AI systems are increasingly capable of learning complex tasks through trial and error, leading to applications in robotics and game playing. Furthermore, AI ethics and safety have gained attention, with researchers focusing on creating more transparent and fair AI systems.\n", + "\n", + "For the most current and specific breakthroughs, I recommend checking reputable technology news sources or academic journals that focus on AI research.\n", + "================================================================================\n", + "\n", + "šŸŽ‰ Demo completed! Check Noveum Trace dashboard for detailed observability data.\n", + "šŸ’” You can now use noveum_agent.process_query('your question') for your own queries!\n" + ] + } + ], + "source": [ + "demo_noveum_agent()" + ] + }, + { + "cell_type": "markdown", + "id": "d5b29876", + "metadata": {}, + "source": [ + "## Downloading the data set" + ] + }, + { + "cell_type": "markdown", + "id": "bcf1f2b8", + "metadata": {}, + "source": [ + "## šŸ“‹ Complete End-to-End Workflow Overview\n", + "\n", + "This notebook demonstrates a **complete end-to-end workflow** for AI agent evaluation:\n", + "\n", + "### šŸ”„ Workflow Steps:\n", + "\n", + "1. **šŸ¤– Agent Creation & Demo** - Build and test the Noveum AI agent with RAG + Web Search\n", + "2. **šŸ“Š Trace Collection** - Download traces from Noveum platform \n", + "3. **šŸ”„ Data Processing** - Combine, filter, and map trace data\n", + "4. **šŸ“ˆ Dataset Management** - Create dataset and upload items to Noveum\n", + "5. **šŸ“Š Score Upload** - Upload all evaluation metrics to the platform\n", + "\n", + "### šŸŽÆ What You'll Learn:\n", + "\n", + "- How to build a sophisticated AI agent with dual knowledge sources\n", + "- Complete observability and tracing implementation\n", + "- End-to-end evaluation pipeline from traces to scores\n", + "- Noveum platform integration for dataset and score management\n", + "\n", + "### šŸ“Š Expected Results:\n", + "\n", + "- **50 traces** with **279 spans** collected\n", + "- **12 conversation items** uploaded to dataset\n", + "- **5 evaluation metrics** successfully uploaded\n", + "- Complete observability pipeline operational\n", + "\n", + "Let's start! šŸš€\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0f918640", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/Library/Frameworks/Python.framework/Versions/3.12/lib/python3.12/pty.py:95: DeprecationWarning: This process (pid=52016) is multi-threaded, use of forkpty() may lead to deadlocks in the child.\n", + " pid, fd = os.forkpty()\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Fetching 50 traces for project: noveum-ai-agent-rag-websearch\n", + "Cleaning existing traces directory: /Users/mramanindia/work/NovaEval/noveum_customer_support_bt/traces/traces\n", + "Created traces directory: /Users/mramanindia/work/NovaEval/noveum_customer_support_bt/traces/traces\n", + "Will fetch in 1 batch(es) of up to 100 traces each\n", + "\n", + "--- Batch 1/1 ---\n", + "Fetching traces: size=50, from=0\n", + "Successfully fetched 50 traces\n", + "Saved batch 1 to: /Users/mramanindia/work/NovaEval/noveum_customer_support_bt/traces/traces/traces_batch_001.json\n", + "Batch 1 complete: 50 traces\n", + "Total fetched so far: 50/50\n", + "Reached target of 50 traces\n", + "\n", + "=== Summary ===\n", + "Total traces fetched: 50\n", + "Batches created: 1\n", + "Traces directory: /Users/mramanindia/work/NovaEval/noveum_customer_support_bt/traces/traces\n", + "Created files: traces_batch_001.json\n" + ] + } + ], + "source": [ + "!python noveum_customer_support_bt/traces/fetch_traces_api.py 50\n", + "\n", + "#. This script fetches traces for our project and saves them locally." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "641efa41", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Processing traces from: /Users/mramanindia/work/NovaEval/noveum_customer_support_bt/traces/traces\n", + "Found 1 trace files: ['traces_batch_001.json']\n", + "Processing traces_batch_001.json...\n", + "Combined 290 spans total\n", + "Saved combined spans to: /Users/mramanindia/work/NovaEval/noveum_customer_support_bt/traces/traces/dataset.json\n", + "\n", + "Sample of first span keys: ['span_id', 'trace_id', 'parent_span_id', 'name', 'start_time', 'end_time', 'duration_ms', 'status', 'status_message', 'attributes', 'events', 'links', 'trace_trace_id', 'trace_name', 'project', 'environment', 'trace_status', 'trace_status_message', 'trace_start_time', 'trace_end_time', 'trace_duration_ms', 'span_count', 'error_count', 'sdk', 'trace_attributes', 'metadata', 'created_at', 'updated_at']\n", + "Total spans: 290\n", + "\n", + "Span types distribution:\n", + " agent.llm-rag: 26\n", + " agent.llm_model_execution: 48\n", + " agent.query_routing: 48\n", + " agent.rag_evaluation_metrics: 26\n", + " agent.routing_evaluation_metrics: 48\n", + " agent.web_search_evaluation_metrics: 22\n", + " agent.web_search_generation: 22\n", + " system_initialization: 2\n", + " tool-orchestator: 48\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/Library/Frameworks/Python.framework/Versions/3.12/lib/python3.12/pty.py:95: DeprecationWarning: This process (pid=52016) is multi-threaded, use of forkpty() may lead to deadlocks in the child.\n", + " pid, fd = os.forkpty()\n" + ] + } + ], + "source": [ + "!python NovaEval/noveum_customer_support_bt/traces/combine_spans_api_compat.py" + ] + }, + { + "cell_type": "markdown", + "id": "0bddd552", + "metadata": {}, + "source": [ + "## Data Filteration and mapping" + ] + }, + { + "cell_type": "code", + "execution_count": 62, + "id": "3dfb8588", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Reading ./traces/traces/dataset.json...\n", + "Original dataset: 290 records\n", + "Filtering spans...\n", + "After filtering: 290 records\n", + "Converting tool output format...\n", + "Writing ./traces/traces/dataset_filtered.json...\n", + "Filtering complete! Output: ./traces/traces/dataset_filtered.json\n", + "\n", + "Success! Created ./traces/traces/dataset_filtered.json\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/Library/Frameworks/Python.framework/Versions/3.12/lib/python3.12/pty.py:95: DeprecationWarning: This process (pid=52016) is multi-threaded, use of forkpty() may lead to deadlocks in the child.\n", + " pid, fd = os.forkpty()\n" + ] + } + ], + "source": [ + "!python preprocess_filter.py ./traces/traces/dataset.json\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": 63, + "id": "a7484b26", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Error: File ./traces/dataset_filtered.json not found\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/Library/Frameworks/Python.framework/Versions/3.12/lib/python3.12/pty.py:95: DeprecationWarning: This process (pid=52016) is multi-threaded, use of forkpty() may lead to deadlocks in the child.\n", + " pid, fd = os.forkpty()\n" + ] + } + ], + "source": [ + "!python preprocess_map.py ./traces/dataset_filtered.json" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9bcd53eb", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Reading /Users/mramanindia/work/NovaEval/noveum_customer_support_bt/traces/traces/dataset_filtered.json...\n", + "Input dataset: 290 records\n", + "Mapping spans...\n", + "Writing /Users/mramanindia/work/NovaEval/noveum_customer_support_bt/traces/traces/dataset_filtered_mapped.json...\n", + "Mapping complete! Output: /Users/mramanindia/work/NovaEval/noveum_customer_support_bt/traces/traces/dataset_filtered_mapped.json\n", + "\n", + "Success! Created /Users/mramanindia/work/NovaEval/noveum_customer_support_bt/traces/traces/dataset_filtered_mapped.json\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/Library/Frameworks/Python.framework/Versions/3.12/lib/python3.12/pty.py:95: DeprecationWarning: This process (pid=52016) is multi-threaded, use of forkpty() may lead to deadlocks in the child.\n", + " pid, fd = os.forkpty()\n" + ] + } + ], + "source": [ + "!python preprocess_map.py NovaEval/noveum_customer_support_bt/traces/traces/dataset_filtered.json\n", + "\n" + ] + }, + { + "cell_type": "markdown", + "id": "008fd0e1", + "metadata": {}, + "source": [ + "## Running eval on the dataset" + ] + }, + { + "cell_type": "code", + "execution_count": 72, + "id": "430d60b0", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/Library/Frameworks/Python.framework/Versions/3.12/lib/python3.12/pty.py:95: DeprecationWarning: This process (pid=52016) is multi-threaded, use of forkpty() may lead to deadlocks in the child.\n", + " pid, fd = os.forkpty()\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "zsh:source:1: no such file or directory: .venv/bin/activate\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/Library/Frameworks/Python.framework/Versions/3.12/lib/python3.12/pty.py:95: DeprecationWarning: This process (pid=52016) is multi-threaded, use of forkpty() may lead to deadlocks in the child.\n", + " pid, fd = os.forkpty()\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "zsh:cd:1: no such file or directory: noveum_customer_support_bt\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/Library/Frameworks/Python.framework/Versions/3.12/lib/python3.12/pty.py:95: DeprecationWarning: This process (pid=52016) is multi-threaded, use of forkpty() may lead to deadlocks in the child.\n", + " pid, fd = os.forkpty()\n", + "/Library/Frameworks/Python.framework/Versions/3.12/lib/python3.12/pty.py:95: DeprecationWarning: This process (pid=52016) is multi-threaded, use of forkpty() may lead to deadlocks in the child.\n", + " pid, fd = os.forkpty()\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "āš ļø WARNING: Please update the dataset slug in your .env file after creating the dataset!\n", + " The API will return a slug that you should set as NOVEUM_DATASET_SLUG in your .env file.\n", + "\n", + "Creating dataset at: https://noveum.ai/api/v1/datasets\n", + "Organization: magic-api\n", + "Dataset name: customersupportagentdemo_new\n", + "Dataset type: agent\n", + "Description: Customer Support Agent Evaluation Dataset\n", + "Visibility: org\n", + "Environment: \n", + "Error creating dataset: 409 Client Error: Conflict for url: https://noveum.ai/api/v1/datasets\n", + "Response status: 409\n", + "Response text: DATASET_SLUG_EXISTS\n" + ] + } + ], + "source": [ + "# 1. Setup\n", + "!cd /Users/mramanindia/work/NovaEval\n", + "!source .venv/bin/activate\n", + "!cd noveum_customer_support_bt\n", + "\n", + "# 2. Create Dataset\n", + "!python create_dataset.py --dataset-type agent --description \"Customer Support Agent Evaluation Dataset\" --pretty\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "9f7feb8d", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Creating dataset version at: https://noveum.ai/api/v1/datasets/customersupportagentdemo-new/versions?organizationSlug=magic-api\n", + "Organization: magic-api\n", + "Dataset: customersupportagentdemo-new\n", + "Version: 0.0.2\n", + "Successfully created dataset version\n", + "Response status: 201\n", + "\n", + "Response saved to: dataset_version_response.json\n", + "\n", + "Response data:\n", + "{\n", + " \"success\": true,\n", + " \"version\": \"0.0.2\"\n", + "}\n" + ] + } + ], + "source": [ + "# 3. Create Version\n", + "!python create_dataset_version.py --pretty\n" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "d36f1da5", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-10-22 22:59:42 - noveum_trace.transport.batch_processor - INFO - šŸ”„ Batch processor background thread started (batch_size=100, timeout=5.0s)\n", + "2025-10-22 22:59:42 - noveum_trace.transport.batch_processor - INFO - Batch processor started with batch_size=100\n", + "2025-10-22 22:59:42 - noveum_trace.transport.http_transport - INFO - HTTP transport initialized for endpoint: https://api.noveum.ai/api\n", + "2025-10-22 22:59:42 - noveum_trace.core.client - INFO - Noveum Trace client initialized\n", + "2025-10-22 22:59:42,946 - INFO - novaeval.models.base - Noveum tracing initialized successfully\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "āœ… All imports successful!\n", + "āœ… list_dataset_files function defined!\n", + "āœ… load_and_analyze_dataset function defined!\n", + "āœ… parse_tools_from_prompt function defined!\n", + "āœ… parse_params function defined!\n", + "āœ… identify_span_type function defined!\n", + "āœ… map_span_to_agent_data function defined!\n", + "āœ… convert_spans_to_agent_dataset function defined!\n", + "āœ… analyze_dataset_statistics function defined!\n", + "āœ… setup_gemini_model function defined!\n", + "āœ… setup_agent_evaluator function defined!\n", + "āœ… run_evaluation function defined!\n", + "āœ… analyze_agent_behavior_patterns function defined!\n", + "āœ… export_processed_dataset function defined!\n", + "āœ… setup_logging function defined!\n", + "āœ… validate_environment function defined!\n", + "āœ… print_demo_summary function defined!\n", + "āœ… run_complete_agent_evaluation function defined!\n", + "Processing agent.llm_model_execution_dataset.json...\n", + "šŸš€ Starting Complete Agent Evaluation Pipeline\n", + "šŸ“ Processing file: split_datasets/agent.llm_model_execution_dataset.json\n", + "============================================================\n", + "\n", + "šŸ“‹ Step 1: Environment Setup\n", + "āœ… Logging configured at INFO level\n", + "šŸ” Environment validation:\n", + " āœ… gemini_api_key: True\n", + " āœ… pandas_available: True\n", + " āœ… novaeval_available: True\n", + "āœ… Environment ready for evaluation!\n", + "\n", + "šŸ“‹ Step 2: Loading Dataset\n", + "šŸ“Š Loaded 20 spans from split_datasets/agent.llm_model_execution_dataset.json\n", + "\n", + "šŸ” Available span types:\n", + " - agent.llm_model_execution: 20\n", + "āœ… Dataset loaded: 20 spans\n", + "\n", + "šŸ“‹ Step 3: Converting to AgentDataset Format\n", + "šŸ”„ Converting spans to AgentData objects...\n", + "agent_task not found\n", + "agent_response is not available ce8556f4-bbeb-46fc-8bfd-dfaedb61cb70\n", + "agent_task not found\n", + "agent_response is not available cc3073ec-c970-4821-8335-8d0fc8f37930\n", + "agent_task not found\n", + "agent_response is not available c24fae6d-6314-4342-8214-7e9905ae83db\n", + "agent_task not found\n", + "agent_response is not available 93879d8f-77c7-4fd4-881b-be500e2dcbde\n", + "agent_task not found\n", + "agent_response is not available f92643c0-44a3-4fae-9d29-45321342d12f\n", + "agent_task not found\n", + "agent_response is not available 44f379aa-10fe-4a51-847c-5288444d7843\n", + "agent_task not found\n", + "agent_response is not available e8e79f2a-2ed0-4981-943c-adcf6f7d2b15\n", + "agent_task not found\n", + "agent_response is not available e6d0eb41-5f64-44c5-8a42-a80c336de03e\n", + "agent_task not found\n", + "agent_response is not available c545cb62-5a51-4f2e-b020-b3f1d1dfe217\n", + "agent_task not found\n", + "agent_response is not available c2f21e4d-a5e6-4134-aeaf-96e2f566cf96\n", + "agent_task not found\n", + "agent_response is not available 39d99d2b-518f-439c-8758-7a01b970fd9e\n", + "agent_task not found\n", + "agent_response is not available 9e5ade74-3815-4dad-8abe-d4281d7e62da\n", + "agent_task not found\n", + "agent_response is not available 4f38a85b-0721-47a5-8a51-d0808e460f96\n", + "agent_task not found\n", + "agent_response is not available f4c03a8d-b90d-4c1a-a259-826d09e3e57a\n", + "agent_task not found\n", + "agent_response is not available 48fb40ba-a39f-4e13-9d94-18794c7e5e2b\n", + "agent_task not found\n", + "agent_response is not available d4d544a0-f052-4889-82eb-4346f2e447a7\n", + "agent_task not found\n", + "agent_response is not available 304f9e3d-9063-4661-9777-618efc49f5c1\n", + "agent_task not found\n", + "agent_response is not available e0b4ccfb-18e7-4d71-b2dd-5b807eb27513\n", + "agent_task not found\n", + "agent_response is not available f1425131-f8a2-43f7-aa17-d009f745ed06\n", + "agent_task not found\n", + "agent_response is not available f41e836c-1a82-46e0-9d51-011713874e94\n", + "\n", + "āœ… Successfully converted 20 spans to AgentData\n", + "šŸ“Š AgentDataset created with 20 records\n", + "āœ… AgentDataset created: 20 records\n", + "\n", + "šŸ“‹ Step 4: Dataset Analysis\n", + "šŸ“ˆ Dataset Statistics:\n", + "\n", + "Agent Types: {'agent': 20}\n", + "Records with responses: 0\n", + "Records with tool calls: 0\n", + "Records with retrieval: 0\n", + "Tool usage: {}\n", + "šŸ” Dataset Analysis:\n", + "\n", + "=== Agent Behavior Patterns ===\n", + "\n", + "šŸ“ˆ Tool Usage:\n", + "\n", + "šŸ“‹ Task Types:\n", + "\n", + "šŸ“‹ Step 5: Setting up Evaluation\n", + "āœ… GEMINI_API_KEY found in environment\n", + "āœ… Gemini model initialized\n", + "āœ… Initialized 5 scoring functions:\n", + " - task_progression_scorer\n", + " - context_relevancy_scorer\n", + " - role_adherence_scorer\n", + " - tool_relevancy_scorer\n", + " - parameter_correctness_scorer\n", + "\n", + "āœ… AgentEvaluator created with Gemini model and scoring functions\n", + "āœ… Evaluation components ready!\n", + "\n", + "šŸ“‹ Step 6: Running Evaluation\n", + "šŸŽÆ Evaluating 25 samples...\n", + "šŸš€ Running evaluation on sample data...\n", + "\n", + "šŸ“Š Evaluating 0 sample records...\n", + "2025-10-22 22:59:42 - INFO - novaeval.evaluators.agent_evaluator - Starting agent evaluation process\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Evaluating samples: 0it [00:00, ?it/s]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 22:59:43 - INFO - novaeval.evaluators.agent_evaluator - Saving final results\n", + "2025-10-22 22:59:43 - INFO - novaeval.evaluators.agent_evaluator - Agent evaluation completed\n", + "\n", + "āœ… Evaluation completed!\n", + "āŒ Results file not found\n", + "āŒ Evaluation failed\n", + "\n", + "šŸ“‹ Step 7: Exporting Dataset\n", + "šŸ’¾ Exporting processed dataset...\n", + "āœ… Exported to ./processed_datasets/agent.llm_model_execution_dataset_processed_dataset.json\n", + "āœ… Exported to ./processed_datasets/agent.llm_model_execution_dataset_processed_dataset.csv\n", + "āœ… Export completed successfully!\n", + "\n", + "============================================================\n", + "šŸŽ‰ EVALUATION PIPELINE COMPLETED!\n", + "šŸ“Š Final Results:\n", + " - File processed: split_datasets/agent.llm_model_execution_dataset.json\n", + " - Spans loaded: 20\n", + " - Dataset size: 20\n", + " - Evaluation completed: False\n", + " - Export successful: True\n", + " - Errors encountered: 1\n", + "Completed agent.llm_model_execution_dataset.json\n", + "\n", + "Processing agent.query_routing_dataset.json...\n", + "šŸš€ Starting Complete Agent Evaluation Pipeline\n", + "šŸ“ Processing file: split_datasets/agent.query_routing_dataset.json\n", + "============================================================\n", + "\n", + "šŸ“‹ Step 1: Environment Setup\n", + "āœ… Logging configured at INFO level\n", + "šŸ” Environment validation:\n", + " āœ… gemini_api_key: True\n", + " āœ… pandas_available: True\n", + " āœ… novaeval_available: True\n", + "āœ… Environment ready for evaluation!\n", + "\n", + "šŸ“‹ Step 2: Loading Dataset\n", + "šŸ“Š Loaded 20 spans from split_datasets/agent.query_routing_dataset.json\n", + "\n", + "šŸ” Available span types:\n", + " - agent.query_routing: 20\n", + "āœ… Dataset loaded: 20 spans\n", + "\n", + "šŸ“‹ Step 3: Converting to AgentDataset Format\n", + "šŸ”„ Converting spans to AgentData objects...\n", + "\n", + "āœ… Successfully converted 20 spans to AgentData\n", + "šŸ“Š AgentDataset created with 20 records\n", + "āœ… AgentDataset created: 20 records\n", + "\n", + "šŸ“‹ Step 4: Dataset Analysis\n", + "šŸ“ˆ Dataset Statistics:\n", + "\n", + "Agent Types: {'agent': 20}\n", + "Records with responses: 0\n", + "Records with tool calls: 0\n", + "Records with retrieval: 0\n", + "Tool usage: {}\n", + "šŸ” Dataset Analysis:\n", + "\n", + "=== Agent Behavior Patterns ===\n", + "\n", + "šŸ“ˆ Tool Usage:\n", + "\n", + "šŸ“‹ Task Types:\n", + " - other: 20\n", + "\n", + "šŸ“‹ Step 5: Setting up Evaluation\n", + "āœ… GEMINI_API_KEY found in environment\n", + "2025-10-22 22:59:43 - INFO - novaeval.models.base - Noveum tracing initialized successfully\n", + "āœ… Gemini model initialized\n", + "āœ… Initialized 5 scoring functions:\n", + " - task_progression_scorer\n", + " - context_relevancy_scorer\n", + " - role_adherence_scorer\n", + " - tool_relevancy_scorer\n", + " - parameter_correctness_scorer\n", + "\n", + "āœ… AgentEvaluator created with Gemini model and scoring functions\n", + "āœ… Evaluation components ready!\n", + "\n", + "šŸ“‹ Step 6: Running Evaluation\n", + "šŸŽÆ Evaluating 25 samples...\n", + "šŸš€ Running evaluation on sample data...\n", + "\n", + "šŸ“Š Evaluating 20 sample records...\n", + "2025-10-22 22:59:43 - INFO - novaeval.evaluators.agent_evaluator - Starting agent evaluation process\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\n", + "Evaluating samples: 0it [00:00, ?it/s]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 22:59:43 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-10-22 22:59:44 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 62c88c1e-760c-437e-a9d7-84a84e258789) - 1 spans\n", + "2025-10-22 22:59:44 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 62c88c1e-760c-437e-a9d7-84a84e258789) - 1 spans\n", + "2025-10-22 22:59:44 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 62c88c1e-760c-437e-a9d7-84a84e258789\n", + "2025-10-22 22:59:44 - noveum_trace.transport.http_transport - INFO - āœ… Trace 62c88c1e-760c-437e-a9d7-84a84e258789 successfully queued for export\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 22:59:44 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-10-22 22:59:46 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: cf8d91be-c525-4e98-b2ab-b6dead66b872) - 1 spans\n", + "2025-10-22 22:59:46 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: cf8d91be-c525-4e98-b2ab-b6dead66b872) - 1 spans\n", + "2025-10-22 22:59:46 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace cf8d91be-c525-4e98-b2ab-b6dead66b872\n", + "2025-10-22 22:59:46 - noveum_trace.transport.http_transport - INFO - āœ… Trace cf8d91be-c525-4e98-b2ab-b6dead66b872 successfully queued for export\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 22:59:46 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-10-22 22:59:47 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 087e523c-cf91-49e6-8a3f-63d7e6ea817a) - 1 spans\n", + "2025-10-22 22:59:47 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 087e523c-cf91-49e6-8a3f-63d7e6ea817a) - 1 spans\n", + "2025-10-22 22:59:47 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 087e523c-cf91-49e6-8a3f-63d7e6ea817a\n", + "2025-10-22 22:59:47 - noveum_trace.transport.http_transport - INFO - āœ… Trace 087e523c-cf91-49e6-8a3f-63d7e6ea817a successfully queued for export\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 22:59:47 - INFO - novaeval.evaluators.agent_evaluator - Saving intermediate results after 1 samples\n", + "2025-10-22 22:59:47 - INFO - novaeval.evaluators.agent_evaluator - Intermediate results saved to demo_results/agent.query_routing_dataset/agent_evaluation_results.csv\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Evaluating samples: 1it [00:04, 4.56s/it]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 22:59:47 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-10-22 22:59:48 - noveum_trace.transport.batch_processor - INFO - ā° TIMEOUT TRIGGER: Sending batch due to timeout (5.1s >= 5.0s)\n", + "2025-10-22 22:59:48 - noveum_trace.transport.batch_processor - INFO - šŸ“¤ SENDING BATCH: 3 traces via send_callback\n", + "2025-10-22 22:59:48 - noveum_trace.transport.http_transport - INFO - šŸš€ SENDING BATCH: 3 traces to https://api.noveum.ai/api/v1/traces\n", + "2025-10-22 22:59:48 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: f5d0b1e3-779c-4804-8cb6-c37743638b63) - 1 spans\n", + "2025-10-22 22:59:48 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: f5d0b1e3-779c-4804-8cb6-c37743638b63) - 1 spans\n", + "2025-10-22 22:59:48 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace f5d0b1e3-779c-4804-8cb6-c37743638b63\n", + "2025-10-22 22:59:48 - noveum_trace.transport.http_transport - INFO - āœ… Trace f5d0b1e3-779c-4804-8cb6-c37743638b63 successfully queued for export\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 22:59:48 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-10-22 22:59:49 - noveum_trace.transport.http_transport - INFO - šŸ“” HTTP RESPONSE: Status 200 from https://api.noveum.ai/api/v1/traces\n", + "2025-10-22 22:59:49 - noveum_trace.transport.http_transport - INFO - āœ… Successfully sent batch of 3 traces\n", + "2025-10-22 22:59:49 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully sent batch of 3 traces via callback\n", + "2025-10-22 22:59:50 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 9c192a1f-994d-4dba-a03d-79b70c731c6f) - 1 spans\n", + "2025-10-22 22:59:50 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 9c192a1f-994d-4dba-a03d-79b70c731c6f) - 1 spans\n", + "2025-10-22 22:59:50 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 9c192a1f-994d-4dba-a03d-79b70c731c6f\n", + "2025-10-22 22:59:50 - noveum_trace.transport.http_transport - INFO - āœ… Trace 9c192a1f-994d-4dba-a03d-79b70c731c6f successfully queued for export\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 22:59:50 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-10-22 22:59:52 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 4d902ae0-f8b9-4d47-b1d8-cfcc59d71ef9) - 1 spans\n", + "2025-10-22 22:59:52 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 4d902ae0-f8b9-4d47-b1d8-cfcc59d71ef9) - 1 spans\n", + "2025-10-22 22:59:52 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 4d902ae0-f8b9-4d47-b1d8-cfcc59d71ef9\n", + "2025-10-22 22:59:52 - noveum_trace.transport.http_transport - INFO - āœ… Trace 4d902ae0-f8b9-4d47-b1d8-cfcc59d71ef9 successfully queued for export\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 22:59:52 - INFO - novaeval.evaluators.agent_evaluator - Saving intermediate results after 2 samples\n", + "2025-10-22 22:59:52 - INFO - novaeval.evaluators.agent_evaluator - Intermediate results saved to demo_results/agent.query_routing_dataset/agent_evaluation_results.csv\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Evaluating samples: 2it [00:09, 5.04s/it]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 22:59:52 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-10-22 22:59:53 - noveum_trace.transport.batch_processor - INFO - ā° TIMEOUT TRIGGER: Sending batch due to timeout (5.4s >= 5.0s)\n", + "2025-10-22 22:59:53 - noveum_trace.transport.batch_processor - INFO - šŸ“¤ SENDING BATCH: 3 traces via send_callback\n", + "2025-10-22 22:59:53 - noveum_trace.transport.http_transport - INFO - šŸš€ SENDING BATCH: 3 traces to https://api.noveum.ai/api/v1/traces\n", + "2025-10-22 22:59:53 - noveum_trace.transport.http_transport - INFO - šŸ“” HTTP RESPONSE: Status 200 from https://api.noveum.ai/api/v1/traces\n", + "2025-10-22 22:59:53 - noveum_trace.transport.http_transport - INFO - āœ… Successfully sent batch of 3 traces\n", + "2025-10-22 22:59:53 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully sent batch of 3 traces via callback\n", + "2025-10-22 22:59:55 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: d6f4e5b6-7b0c-4cc5-a6ab-0d7f6ee82997) - 1 spans\n", + "2025-10-22 22:59:55 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: d6f4e5b6-7b0c-4cc5-a6ab-0d7f6ee82997) - 1 spans\n", + "2025-10-22 22:59:55 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace d6f4e5b6-7b0c-4cc5-a6ab-0d7f6ee82997\n", + "2025-10-22 22:59:55 - noveum_trace.transport.http_transport - INFO - āœ… Trace d6f4e5b6-7b0c-4cc5-a6ab-0d7f6ee82997 successfully queued for export\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 22:59:55 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-10-22 22:59:58 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 1791b898-a25e-406e-9946-13403e6ea2a5) - 1 spans\n", + "2025-10-22 22:59:58 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 1791b898-a25e-406e-9946-13403e6ea2a5) - 1 spans\n", + "2025-10-22 22:59:58 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 1791b898-a25e-406e-9946-13403e6ea2a5\n", + "2025-10-22 22:59:58 - noveum_trace.transport.http_transport - INFO - āœ… Trace 1791b898-a25e-406e-9946-13403e6ea2a5 successfully queued for export\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 22:59:58 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-10-22 22:59:58 - noveum_trace.transport.batch_processor - INFO - ā° TIMEOUT TRIGGER: Sending batch due to timeout (5.1s >= 5.0s)\n", + "2025-10-22 22:59:58 - noveum_trace.transport.batch_processor - INFO - šŸ“¤ SENDING BATCH: 2 traces via send_callback\n", + "2025-10-22 22:59:58 - noveum_trace.transport.http_transport - INFO - šŸš€ SENDING BATCH: 2 traces to https://api.noveum.ai/api/v1/traces\n", + "2025-10-22 22:59:58 - noveum_trace.transport.http_transport - INFO - šŸ“” HTTP RESPONSE: Status 200 from https://api.noveum.ai/api/v1/traces\n", + "2025-10-22 22:59:58 - noveum_trace.transport.http_transport - INFO - āœ… Successfully sent batch of 2 traces\n", + "2025-10-22 22:59:58 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully sent batch of 2 traces via callback\n", + "2025-10-22 22:59:59 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 93cd53ab-4aee-474b-80a1-b2e056179d42) - 1 spans\n", + "2025-10-22 22:59:59 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 93cd53ab-4aee-474b-80a1-b2e056179d42) - 1 spans\n", + "2025-10-22 22:59:59 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 93cd53ab-4aee-474b-80a1-b2e056179d42\n", + "2025-10-22 22:59:59 - noveum_trace.transport.http_transport - INFO - āœ… Trace 93cd53ab-4aee-474b-80a1-b2e056179d42 successfully queued for export\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 22:59:59 - INFO - novaeval.evaluators.agent_evaluator - Saving intermediate results after 3 samples\n", + "2025-10-22 22:59:59 - INFO - novaeval.evaluators.agent_evaluator - Intermediate results saved to demo_results/agent.query_routing_dataset/agent_evaluation_results.csv\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Evaluating samples: 3it [00:16, 5.53s/it]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 22:59:59 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-10-22 23:00:01 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: bdd95aaf-99e9-4926-ae57-f3846a34dfe2) - 1 spans\n", + "2025-10-22 23:00:01 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: bdd95aaf-99e9-4926-ae57-f3846a34dfe2) - 1 spans\n", + "2025-10-22 23:00:01 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace bdd95aaf-99e9-4926-ae57-f3846a34dfe2\n", + "2025-10-22 23:00:01 - noveum_trace.transport.http_transport - INFO - āœ… Trace bdd95aaf-99e9-4926-ae57-f3846a34dfe2 successfully queued for export\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:00:01 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-10-22 23:00:03 - noveum_trace.transport.batch_processor - INFO - ā° TIMEOUT TRIGGER: Sending batch due to timeout (5.2s >= 5.0s)\n", + "2025-10-22 23:00:03 - noveum_trace.transport.batch_processor - INFO - šŸ“¤ SENDING BATCH: 2 traces via send_callback\n", + "2025-10-22 23:00:03 - noveum_trace.transport.http_transport - INFO - šŸš€ SENDING BATCH: 2 traces to https://api.noveum.ai/api/v1/traces\n", + "2025-10-22 23:00:03 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 14a0cc75-5abe-4057-8b8e-474c095520bd) - 1 spans\n", + "2025-10-22 23:00:03 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 14a0cc75-5abe-4057-8b8e-474c095520bd) - 1 spans\n", + "2025-10-22 23:00:03 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 14a0cc75-5abe-4057-8b8e-474c095520bd\n", + "2025-10-22 23:00:03 - noveum_trace.transport.http_transport - INFO - āœ… Trace 14a0cc75-5abe-4057-8b8e-474c095520bd successfully queued for export\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:00:03 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-10-22 23:00:04 - noveum_trace.transport.http_transport - INFO - šŸ“” HTTP RESPONSE: Status 200 from https://api.noveum.ai/api/v1/traces\n", + "2025-10-22 23:00:04 - noveum_trace.transport.http_transport - INFO - āœ… Successfully sent batch of 2 traces\n", + "2025-10-22 23:00:04 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully sent batch of 2 traces via callback\n", + "2025-10-22 23:00:05 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 3cdcd252-7600-4247-8557-f67f1a66beae) - 1 spans\n", + "2025-10-22 23:00:05 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 3cdcd252-7600-4247-8557-f67f1a66beae) - 1 spans\n", + "2025-10-22 23:00:05 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 3cdcd252-7600-4247-8557-f67f1a66beae\n", + "2025-10-22 23:00:05 - noveum_trace.transport.http_transport - INFO - āœ… Trace 3cdcd252-7600-4247-8557-f67f1a66beae successfully queued for export\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:00:05 - INFO - novaeval.evaluators.agent_evaluator - Saving intermediate results after 4 samples\n", + "2025-10-22 23:00:05 - INFO - novaeval.evaluators.agent_evaluator - Intermediate results saved to demo_results/agent.query_routing_dataset/agent_evaluation_results.csv\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Evaluating samples: 4it [00:21, 5.69s/it]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:00:05 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-10-22 23:00:06 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 57555bab-ecd2-4006-a353-7d27bef7df0b) - 1 spans\n", + "2025-10-22 23:00:06 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 57555bab-ecd2-4006-a353-7d27bef7df0b) - 1 spans\n", + "2025-10-22 23:00:06 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 57555bab-ecd2-4006-a353-7d27bef7df0b\n", + "2025-10-22 23:00:06 - noveum_trace.transport.http_transport - INFO - āœ… Trace 57555bab-ecd2-4006-a353-7d27bef7df0b successfully queued for export\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:00:06 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-10-22 23:00:07 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 6b673f69-57ca-406b-8795-a74e96080081) - 1 spans\n", + "2025-10-22 23:00:07 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 6b673f69-57ca-406b-8795-a74e96080081) - 1 spans\n", + "2025-10-22 23:00:07 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 6b673f69-57ca-406b-8795-a74e96080081\n", + "2025-10-22 23:00:07 - noveum_trace.transport.http_transport - INFO - āœ… Trace 6b673f69-57ca-406b-8795-a74e96080081 successfully queued for export\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:00:07 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-10-22 23:00:08 - noveum_trace.transport.batch_processor - INFO - ā° TIMEOUT TRIGGER: Sending batch due to timeout (5.0s >= 5.0s)\n", + "2025-10-22 23:00:08 - noveum_trace.transport.batch_processor - INFO - šŸ“¤ SENDING BATCH: 4 traces via send_callback\n", + "2025-10-22 23:00:08 - noveum_trace.transport.http_transport - INFO - šŸš€ SENDING BATCH: 4 traces to https://api.noveum.ai/api/v1/traces\n", + "2025-10-22 23:00:08 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 47efa03a-15f4-4818-b062-db186df9847c) - 1 spans\n", + "2025-10-22 23:00:08 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 47efa03a-15f4-4818-b062-db186df9847c) - 1 spans\n", + "2025-10-22 23:00:08 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 47efa03a-15f4-4818-b062-db186df9847c\n", + "2025-10-22 23:00:08 - noveum_trace.transport.http_transport - INFO - āœ… Trace 47efa03a-15f4-4818-b062-db186df9847c successfully queued for export\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:00:08 - INFO - novaeval.evaluators.agent_evaluator - Saving intermediate results after 5 samples\n", + "2025-10-22 23:00:08 - INFO - novaeval.evaluators.agent_evaluator - Intermediate results saved to demo_results/agent.query_routing_dataset/agent_evaluation_results.csv\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Evaluating samples: 5it [00:25, 5.03s/it]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:00:08 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-10-22 23:00:09 - noveum_trace.transport.http_transport - INFO - šŸ“” HTTP RESPONSE: Status 200 from https://api.noveum.ai/api/v1/traces\n", + "2025-10-22 23:00:09 - noveum_trace.transport.http_transport - INFO - āœ… Successfully sent batch of 4 traces\n", + "2025-10-22 23:00:09 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully sent batch of 4 traces via callback\n", + "2025-10-22 23:00:10 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: a1e9ac36-8c30-4784-9f33-93ed2497ee97) - 1 spans\n", + "2025-10-22 23:00:10 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: a1e9ac36-8c30-4784-9f33-93ed2497ee97) - 1 spans\n", + "2025-10-22 23:00:10 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace a1e9ac36-8c30-4784-9f33-93ed2497ee97\n", + "2025-10-22 23:00:10 - noveum_trace.transport.http_transport - INFO - āœ… Trace a1e9ac36-8c30-4784-9f33-93ed2497ee97 successfully queued for export\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:00:10 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-10-22 23:00:11 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 62a690ab-2806-4619-92df-eec54d03b859) - 1 spans\n", + "2025-10-22 23:00:11 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 62a690ab-2806-4619-92df-eec54d03b859) - 1 spans\n", + "2025-10-22 23:00:11 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 62a690ab-2806-4619-92df-eec54d03b859\n", + "2025-10-22 23:00:11 - noveum_trace.transport.http_transport - INFO - āœ… Trace 62a690ab-2806-4619-92df-eec54d03b859 successfully queued for export\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:00:11 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-10-22 23:00:12 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 239c63de-f804-41f5-98af-463f42dd4089) - 1 spans\n", + "2025-10-22 23:00:12 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 239c63de-f804-41f5-98af-463f42dd4089) - 1 spans\n", + "2025-10-22 23:00:12 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 239c63de-f804-41f5-98af-463f42dd4089\n", + "2025-10-22 23:00:12 - noveum_trace.transport.http_transport - INFO - āœ… Trace 239c63de-f804-41f5-98af-463f42dd4089 successfully queued for export\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:00:12 - INFO - novaeval.evaluators.agent_evaluator - Saving intermediate results after 6 samples\n", + "2025-10-22 23:00:12 - INFO - novaeval.evaluators.agent_evaluator - Intermediate results saved to demo_results/agent.query_routing_dataset/agent_evaluation_results.csv\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Evaluating samples: 6it [00:29, 4.54s/it]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:00:12 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-10-22 23:00:13 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 4b6d4dfb-7c55-4296-9ad6-0490385e162d) - 1 spans\n", + "2025-10-22 23:00:13 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 4b6d4dfb-7c55-4296-9ad6-0490385e162d) - 1 spans\n", + "2025-10-22 23:00:13 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 4b6d4dfb-7c55-4296-9ad6-0490385e162d\n", + "2025-10-22 23:00:13 - noveum_trace.transport.http_transport - INFO - āœ… Trace 4b6d4dfb-7c55-4296-9ad6-0490385e162d successfully queued for export\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:00:13 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-10-22 23:00:14 - noveum_trace.transport.batch_processor - INFO - ā° TIMEOUT TRIGGER: Sending batch due to timeout (5.4s >= 5.0s)\n", + "2025-10-22 23:00:14 - noveum_trace.transport.batch_processor - INFO - šŸ“¤ SENDING BATCH: 5 traces via send_callback\n", + "2025-10-22 23:00:14 - noveum_trace.transport.http_transport - INFO - šŸš€ SENDING BATCH: 5 traces to https://api.noveum.ai/api/v1/traces\n", + "2025-10-22 23:00:14 - noveum_trace.transport.http_transport - INFO - šŸ“” HTTP RESPONSE: Status 200 from https://api.noveum.ai/api/v1/traces\n", + "2025-10-22 23:00:14 - noveum_trace.transport.http_transport - INFO - āœ… Successfully sent batch of 5 traces\n", + "2025-10-22 23:00:14 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully sent batch of 5 traces via callback\n", + "2025-10-22 23:00:14 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: b6ec3f3c-f9ec-4714-b99e-034d4fa7655b) - 1 spans\n", + "2025-10-22 23:00:14 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: b6ec3f3c-f9ec-4714-b99e-034d4fa7655b) - 1 spans\n", + "2025-10-22 23:00:14 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace b6ec3f3c-f9ec-4714-b99e-034d4fa7655b\n", + "2025-10-22 23:00:14 - noveum_trace.transport.http_transport - INFO - āœ… Trace b6ec3f3c-f9ec-4714-b99e-034d4fa7655b successfully queued for export\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:00:14 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-10-22 23:00:16 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: ed49c38e-e2d4-4cb2-8049-ec501c7407d7) - 1 spans\n", + "2025-10-22 23:00:16 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: ed49c38e-e2d4-4cb2-8049-ec501c7407d7) - 1 spans\n", + "2025-10-22 23:00:16 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace ed49c38e-e2d4-4cb2-8049-ec501c7407d7\n", + "2025-10-22 23:00:16 - noveum_trace.transport.http_transport - INFO - āœ… Trace ed49c38e-e2d4-4cb2-8049-ec501c7407d7 successfully queued for export\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:00:16 - INFO - novaeval.evaluators.agent_evaluator - Saving intermediate results after 7 samples\n", + "2025-10-22 23:00:16 - INFO - novaeval.evaluators.agent_evaluator - Intermediate results saved to demo_results/agent.query_routing_dataset/agent_evaluation_results.csv\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Evaluating samples: 7it [00:33, 4.32s/it]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:00:16 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-10-22 23:00:17 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 1df4cc5a-2027-401e-8a59-c0bc394f0463) - 1 spans\n", + "2025-10-22 23:00:17 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 1df4cc5a-2027-401e-8a59-c0bc394f0463) - 1 spans\n", + "2025-10-22 23:00:17 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 1df4cc5a-2027-401e-8a59-c0bc394f0463\n", + "2025-10-22 23:00:17 - noveum_trace.transport.http_transport - INFO - āœ… Trace 1df4cc5a-2027-401e-8a59-c0bc394f0463 successfully queued for export\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:00:17 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-10-22 23:00:18 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 75755b07-45c9-4ad8-a59e-f6b04685a2be) - 1 spans\n", + "2025-10-22 23:00:18 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 75755b07-45c9-4ad8-a59e-f6b04685a2be) - 1 spans\n", + "2025-10-22 23:00:18 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 75755b07-45c9-4ad8-a59e-f6b04685a2be\n", + "2025-10-22 23:00:18 - noveum_trace.transport.http_transport - INFO - āœ… Trace 75755b07-45c9-4ad8-a59e-f6b04685a2be successfully queued for export\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:00:18 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-10-22 23:00:19 - noveum_trace.transport.batch_processor - INFO - ā° TIMEOUT TRIGGER: Sending batch due to timeout (5.3s >= 5.0s)\n", + "2025-10-22 23:00:19 - noveum_trace.transport.batch_processor - INFO - šŸ“¤ SENDING BATCH: 4 traces via send_callback\n", + "2025-10-22 23:00:19 - noveum_trace.transport.http_transport - INFO - šŸš€ SENDING BATCH: 4 traces to https://api.noveum.ai/api/v1/traces\n", + "2025-10-22 23:00:19 - noveum_trace.transport.http_transport - INFO - šŸ“” HTTP RESPONSE: Status 200 from https://api.noveum.ai/api/v1/traces\n", + "2025-10-22 23:00:19 - noveum_trace.transport.http_transport - INFO - āœ… Successfully sent batch of 4 traces\n", + "2025-10-22 23:00:19 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully sent batch of 4 traces via callback\n", + "2025-10-22 23:00:20 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: dea54b30-9b78-45a1-861c-acef72c97a3f) - 1 spans\n", + "2025-10-22 23:00:20 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: dea54b30-9b78-45a1-861c-acef72c97a3f) - 1 spans\n", + "2025-10-22 23:00:20 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace dea54b30-9b78-45a1-861c-acef72c97a3f\n", + "2025-10-22 23:00:20 - noveum_trace.transport.http_transport - INFO - āœ… Trace dea54b30-9b78-45a1-861c-acef72c97a3f successfully queued for export\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:00:20 - INFO - novaeval.evaluators.agent_evaluator - Saving intermediate results after 8 samples\n", + "2025-10-22 23:00:20 - INFO - novaeval.evaluators.agent_evaluator - Intermediate results saved to demo_results/agent.query_routing_dataset/agent_evaluation_results.csv\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Evaluating samples: 8it [00:37, 4.16s/it]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:00:20 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-10-22 23:00:21 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 9cb16557-f687-4b6d-aa25-798f27e9615c) - 1 spans\n", + "2025-10-22 23:00:21 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 9cb16557-f687-4b6d-aa25-798f27e9615c) - 1 spans\n", + "2025-10-22 23:00:21 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 9cb16557-f687-4b6d-aa25-798f27e9615c\n", + "2025-10-22 23:00:21 - noveum_trace.transport.http_transport - INFO - āœ… Trace 9cb16557-f687-4b6d-aa25-798f27e9615c successfully queued for export\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:00:21 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-10-22 23:00:22 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 39999cf9-16cd-4b1f-8167-39953b328da6) - 1 spans\n", + "2025-10-22 23:00:22 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 39999cf9-16cd-4b1f-8167-39953b328da6) - 1 spans\n", + "2025-10-22 23:00:22 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 39999cf9-16cd-4b1f-8167-39953b328da6\n", + "2025-10-22 23:00:22 - noveum_trace.transport.http_transport - INFO - āœ… Trace 39999cf9-16cd-4b1f-8167-39953b328da6 successfully queued for export\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:00:22 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-10-22 23:00:24 - noveum_trace.transport.batch_processor - INFO - ā° TIMEOUT TRIGGER: Sending batch due to timeout (5.1s >= 5.0s)\n", + "2025-10-22 23:00:24 - noveum_trace.transport.batch_processor - INFO - šŸ“¤ SENDING BATCH: 3 traces via send_callback\n", + "2025-10-22 23:00:24 - noveum_trace.transport.http_transport - INFO - šŸš€ SENDING BATCH: 3 traces to https://api.noveum.ai/api/v1/traces\n", + "2025-10-22 23:00:24 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 0829f590-d837-4dee-9e65-feee419011b2) - 1 spans\n", + "2025-10-22 23:00:24 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 0829f590-d837-4dee-9e65-feee419011b2) - 1 spans\n", + "2025-10-22 23:00:24 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 0829f590-d837-4dee-9e65-feee419011b2\n", + "2025-10-22 23:00:24 - noveum_trace.transport.http_transport - INFO - āœ… Trace 0829f590-d837-4dee-9e65-feee419011b2 successfully queued for export\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:00:24 - INFO - novaeval.evaluators.agent_evaluator - Saving intermediate results after 9 samples\n", + "2025-10-22 23:00:24 - INFO - novaeval.evaluators.agent_evaluator - Intermediate results saved to demo_results/agent.query_routing_dataset/agent_evaluation_results.csv\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Evaluating samples: 9it [00:41, 4.21s/it]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:00:24 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-10-22 23:00:24 - noveum_trace.transport.http_transport - INFO - šŸ“” HTTP RESPONSE: Status 200 from https://api.noveum.ai/api/v1/traces\n", + "2025-10-22 23:00:24 - noveum_trace.transport.http_transport - INFO - āœ… Successfully sent batch of 3 traces\n", + "2025-10-22 23:00:24 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully sent batch of 3 traces via callback\n", + "2025-10-22 23:00:26 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 4e2bb2ea-fcec-476a-8505-3f15ad8243ca) - 1 spans\n", + "2025-10-22 23:00:26 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 4e2bb2ea-fcec-476a-8505-3f15ad8243ca) - 1 spans\n", + "2025-10-22 23:00:26 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 4e2bb2ea-fcec-476a-8505-3f15ad8243ca\n", + "2025-10-22 23:00:26 - noveum_trace.transport.http_transport - INFO - āœ… Trace 4e2bb2ea-fcec-476a-8505-3f15ad8243ca successfully queued for export\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:00:26 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-10-22 23:00:28 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: a091f50f-c6f6-4e99-a812-c427266b63f3) - 1 spans\n", + "2025-10-22 23:00:28 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: a091f50f-c6f6-4e99-a812-c427266b63f3) - 1 spans\n", + "2025-10-22 23:00:28 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace a091f50f-c6f6-4e99-a812-c427266b63f3\n", + "2025-10-22 23:00:28 - noveum_trace.transport.http_transport - INFO - āœ… Trace a091f50f-c6f6-4e99-a812-c427266b63f3 successfully queued for export\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:00:28 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-10-22 23:00:29 - noveum_trace.transport.batch_processor - INFO - ā° TIMEOUT TRIGGER: Sending batch due to timeout (5.4s >= 5.0s)\n", + "2025-10-22 23:00:29 - noveum_trace.transport.batch_processor - INFO - šŸ“¤ SENDING BATCH: 3 traces via send_callback\n", + "2025-10-22 23:00:29 - noveum_trace.transport.http_transport - INFO - šŸš€ SENDING BATCH: 3 traces to https://api.noveum.ai/api/v1/traces\n", + "2025-10-22 23:00:30 - noveum_trace.transport.http_transport - INFO - šŸ“” HTTP RESPONSE: Status 200 from https://api.noveum.ai/api/v1/traces\n", + "2025-10-22 23:00:30 - noveum_trace.transport.http_transport - INFO - āœ… Successfully sent batch of 3 traces\n", + "2025-10-22 23:00:30 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully sent batch of 3 traces via callback\n", + "2025-10-22 23:00:30 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 236d1ec0-daf8-46f7-be50-ea50e4a5e1c6) - 1 spans\n", + "2025-10-22 23:00:30 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 236d1ec0-daf8-46f7-be50-ea50e4a5e1c6) - 1 spans\n", + "2025-10-22 23:00:30 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 236d1ec0-daf8-46f7-be50-ea50e4a5e1c6\n", + "2025-10-22 23:00:30 - noveum_trace.transport.http_transport - INFO - āœ… Trace 236d1ec0-daf8-46f7-be50-ea50e4a5e1c6 successfully queued for export\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:00:30 - INFO - novaeval.evaluators.agent_evaluator - Saving intermediate results after 10 samples\n", + "2025-10-22 23:00:30 - INFO - novaeval.evaluators.agent_evaluator - Intermediate results saved to demo_results/agent.query_routing_dataset/agent_evaluation_results.csv\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Evaluating samples: 10it [00:47, 4.74s/it]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:00:30 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-10-22 23:00:31 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 609077f6-7021-4b16-83e5-d3ed68573359) - 1 spans\n", + "2025-10-22 23:00:31 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 609077f6-7021-4b16-83e5-d3ed68573359) - 1 spans\n", + "2025-10-22 23:00:31 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 609077f6-7021-4b16-83e5-d3ed68573359\n", + "2025-10-22 23:00:31 - noveum_trace.transport.http_transport - INFO - āœ… Trace 609077f6-7021-4b16-83e5-d3ed68573359 successfully queued for export\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:00:31 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-10-22 23:00:32 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: c2bcfc15-818d-4d23-91ef-c38392c4ce9a) - 1 spans\n", + "2025-10-22 23:00:32 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: c2bcfc15-818d-4d23-91ef-c38392c4ce9a) - 1 spans\n", + "2025-10-22 23:00:32 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace c2bcfc15-818d-4d23-91ef-c38392c4ce9a\n", + "2025-10-22 23:00:32 - noveum_trace.transport.http_transport - INFO - āœ… Trace c2bcfc15-818d-4d23-91ef-c38392c4ce9a successfully queued for export\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:00:32 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-10-22 23:00:34 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 027b030d-d0d7-49f7-a9e8-8933f7829573) - 1 spans\n", + "2025-10-22 23:00:34 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 027b030d-d0d7-49f7-a9e8-8933f7829573) - 1 spans\n", + "2025-10-22 23:00:34 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 027b030d-d0d7-49f7-a9e8-8933f7829573\n", + "2025-10-22 23:00:34 - noveum_trace.transport.http_transport - INFO - āœ… Trace 027b030d-d0d7-49f7-a9e8-8933f7829573 successfully queued for export\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:00:34 - INFO - novaeval.evaluators.agent_evaluator - Saving intermediate results after 11 samples\n", + "2025-10-22 23:00:34 - INFO - novaeval.evaluators.agent_evaluator - Intermediate results saved to demo_results/agent.query_routing_dataset/agent_evaluation_results.csv\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Evaluating samples: 11it [00:51, 4.41s/it]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:00:34 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-10-22 23:00:35 - noveum_trace.transport.batch_processor - INFO - ā° TIMEOUT TRIGGER: Sending batch due to timeout (5.2s >= 5.0s)\n", + "2025-10-22 23:00:35 - noveum_trace.transport.batch_processor - INFO - šŸ“¤ SENDING BATCH: 4 traces via send_callback\n", + "2025-10-22 23:00:35 - noveum_trace.transport.http_transport - INFO - šŸš€ SENDING BATCH: 4 traces to https://api.noveum.ai/api/v1/traces\n", + "2025-10-22 23:00:35 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 3b7a2d72-a669-4745-939e-e3eba5fb166b) - 1 spans\n", + "2025-10-22 23:00:35 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 3b7a2d72-a669-4745-939e-e3eba5fb166b) - 1 spans\n", + "2025-10-22 23:00:35 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 3b7a2d72-a669-4745-939e-e3eba5fb166b\n", + "2025-10-22 23:00:35 - noveum_trace.transport.http_transport - INFO - āœ… Trace 3b7a2d72-a669-4745-939e-e3eba5fb166b successfully queued for export\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:00:35 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-10-22 23:00:36 - noveum_trace.transport.http_transport - INFO - šŸ“” HTTP RESPONSE: Status 200 from https://api.noveum.ai/api/v1/traces\n", + "2025-10-22 23:00:36 - noveum_trace.transport.http_transport - INFO - āœ… Successfully sent batch of 4 traces\n", + "2025-10-22 23:00:36 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully sent batch of 4 traces via callback\n", + "2025-10-22 23:00:36 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 3155450e-76b4-4a55-b15b-5c369a6cea5c) - 1 spans\n", + "2025-10-22 23:00:36 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 3155450e-76b4-4a55-b15b-5c369a6cea5c) - 1 spans\n", + "2025-10-22 23:00:36 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 3155450e-76b4-4a55-b15b-5c369a6cea5c\n", + "2025-10-22 23:00:36 - noveum_trace.transport.http_transport - INFO - āœ… Trace 3155450e-76b4-4a55-b15b-5c369a6cea5c successfully queued for export\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:00:36 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-10-22 23:00:38 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 21337429-011a-4aba-a4e5-83ac5be321ce) - 1 spans\n", + "2025-10-22 23:00:38 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 21337429-011a-4aba-a4e5-83ac5be321ce) - 1 spans\n", + "2025-10-22 23:00:38 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 21337429-011a-4aba-a4e5-83ac5be321ce\n", + "2025-10-22 23:00:38 - noveum_trace.transport.http_transport - INFO - āœ… Trace 21337429-011a-4aba-a4e5-83ac5be321ce successfully queued for export\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:00:38 - INFO - novaeval.evaluators.agent_evaluator - Saving intermediate results after 12 samples\n", + "2025-10-22 23:00:38 - INFO - novaeval.evaluators.agent_evaluator - Intermediate results saved to demo_results/agent.query_routing_dataset/agent_evaluation_results.csv\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Evaluating samples: 12it [00:55, 4.40s/it]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:00:38 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-10-22 23:00:40 - noveum_trace.transport.batch_processor - INFO - ā° TIMEOUT TRIGGER: Sending batch due to timeout (5.4s >= 5.0s)\n", + "2025-10-22 23:00:40 - noveum_trace.transport.batch_processor - INFO - šŸ“¤ SENDING BATCH: 3 traces via send_callback\n", + "2025-10-22 23:00:40 - noveum_trace.transport.http_transport - INFO - šŸš€ SENDING BATCH: 3 traces to https://api.noveum.ai/api/v1/traces\n", + "2025-10-22 23:00:40 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 4ebfd611-8645-4781-a3fa-adc5413e7ef9) - 1 spans\n", + "2025-10-22 23:00:40 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 4ebfd611-8645-4781-a3fa-adc5413e7ef9) - 1 spans\n", + "2025-10-22 23:00:40 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 4ebfd611-8645-4781-a3fa-adc5413e7ef9\n", + "2025-10-22 23:00:40 - noveum_trace.transport.http_transport - INFO - āœ… Trace 4ebfd611-8645-4781-a3fa-adc5413e7ef9 successfully queued for export\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:00:40 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-10-22 23:00:41 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 622d34b2-775e-4533-9b86-f3a269f7a87d) - 1 spans\n", + "2025-10-22 23:00:41 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 622d34b2-775e-4533-9b86-f3a269f7a87d) - 1 spans\n", + "2025-10-22 23:00:41 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 622d34b2-775e-4533-9b86-f3a269f7a87d\n", + "2025-10-22 23:00:41 - noveum_trace.transport.http_transport - INFO - āœ… Trace 622d34b2-775e-4533-9b86-f3a269f7a87d successfully queued for export\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:00:41 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-10-22 23:00:43 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 094c79f8-e0c9-4ce7-b493-3ead460412da) - 1 spans\n", + "2025-10-22 23:00:43 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 094c79f8-e0c9-4ce7-b493-3ead460412da) - 1 spans\n", + "2025-10-22 23:00:43 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 094c79f8-e0c9-4ce7-b493-3ead460412da\n", + "2025-10-22 23:00:43 - noveum_trace.transport.http_transport - INFO - āœ… Trace 094c79f8-e0c9-4ce7-b493-3ead460412da successfully queued for export\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:00:43 - INFO - novaeval.evaluators.agent_evaluator - Saving intermediate results after 13 samples\n", + "2025-10-22 23:00:43 - INFO - novaeval.evaluators.agent_evaluator - Intermediate results saved to demo_results/agent.query_routing_dataset/agent_evaluation_results.csv\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Evaluating samples: 13it [01:00, 4.59s/it]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:00:43 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-10-22 23:00:43 - noveum_trace.transport.http_transport - INFO - šŸ“” HTTP RESPONSE: Status 200 from https://api.noveum.ai/api/v1/traces\n", + "2025-10-22 23:00:43 - noveum_trace.transport.http_transport - INFO - āœ… Successfully sent batch of 3 traces\n", + "2025-10-22 23:00:43 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully sent batch of 3 traces via callback\n", + "2025-10-22 23:00:45 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: ba787a5a-c805-4920-bcb6-54f73a11e9be) - 1 spans\n", + "2025-10-22 23:00:45 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: ba787a5a-c805-4920-bcb6-54f73a11e9be) - 1 spans\n", + "2025-10-22 23:00:45 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace ba787a5a-c805-4920-bcb6-54f73a11e9be\n", + "2025-10-22 23:00:45 - noveum_trace.transport.http_transport - INFO - āœ… Trace ba787a5a-c805-4920-bcb6-54f73a11e9be successfully queued for export\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:00:45 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-10-22 23:00:45 - noveum_trace.transport.batch_processor - INFO - ā° TIMEOUT TRIGGER: Sending batch due to timeout (5.5s >= 5.0s)\n", + "2025-10-22 23:00:45 - noveum_trace.transport.batch_processor - INFO - šŸ“¤ SENDING BATCH: 4 traces via send_callback\n", + "2025-10-22 23:00:45 - noveum_trace.transport.http_transport - INFO - šŸš€ SENDING BATCH: 4 traces to https://api.noveum.ai/api/v1/traces\n", + "2025-10-22 23:00:46 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: f7978f02-3999-4b3e-adc7-86d57c952fff) - 1 spans\n", + "2025-10-22 23:00:46 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: f7978f02-3999-4b3e-adc7-86d57c952fff) - 1 spans\n", + "2025-10-22 23:00:46 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace f7978f02-3999-4b3e-adc7-86d57c952fff\n", + "2025-10-22 23:00:46 - noveum_trace.transport.http_transport - INFO - āœ… Trace f7978f02-3999-4b3e-adc7-86d57c952fff successfully queued for export\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:00:46 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-10-22 23:00:47 - noveum_trace.transport.http_transport - INFO - šŸ“” HTTP RESPONSE: Status 200 from https://api.noveum.ai/api/v1/traces\n", + "2025-10-22 23:00:47 - noveum_trace.transport.http_transport - INFO - āœ… Successfully sent batch of 4 traces\n", + "2025-10-22 23:00:47 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully sent batch of 4 traces via callback\n", + "2025-10-22 23:00:47 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: f7c102f2-ae59-4acf-8a11-c8763070bc2d) - 1 spans\n", + "2025-10-22 23:00:47 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: f7c102f2-ae59-4acf-8a11-c8763070bc2d) - 1 spans\n", + "2025-10-22 23:00:47 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace f7c102f2-ae59-4acf-8a11-c8763070bc2d\n", + "2025-10-22 23:00:47 - noveum_trace.transport.http_transport - INFO - āœ… Trace f7c102f2-ae59-4acf-8a11-c8763070bc2d successfully queued for export\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:00:47 - INFO - novaeval.evaluators.agent_evaluator - Saving intermediate results after 14 samples\n", + "2025-10-22 23:00:47 - INFO - novaeval.evaluators.agent_evaluator - Intermediate results saved to demo_results/agent.query_routing_dataset/agent_evaluation_results.csv\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Evaluating samples: 14it [01:04, 4.50s/it]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:00:47 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-10-22 23:00:48 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 499bdbdf-4ddd-4197-a8e4-8fa056cf2c81) - 1 spans\n", + "2025-10-22 23:00:48 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 499bdbdf-4ddd-4197-a8e4-8fa056cf2c81) - 1 spans\n", + "2025-10-22 23:00:48 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 499bdbdf-4ddd-4197-a8e4-8fa056cf2c81\n", + "2025-10-22 23:00:48 - noveum_trace.transport.http_transport - INFO - āœ… Trace 499bdbdf-4ddd-4197-a8e4-8fa056cf2c81 successfully queued for export\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:00:48 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-10-22 23:00:50 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 9941d150-0874-4f64-8348-a5e8a777128e) - 1 spans\n", + "2025-10-22 23:00:50 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 9941d150-0874-4f64-8348-a5e8a777128e) - 1 spans\n", + "2025-10-22 23:00:50 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 9941d150-0874-4f64-8348-a5e8a777128e\n", + "2025-10-22 23:00:50 - noveum_trace.transport.http_transport - INFO - āœ… Trace 9941d150-0874-4f64-8348-a5e8a777128e successfully queued for export\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:00:50 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-10-22 23:00:51 - noveum_trace.transport.batch_processor - INFO - ā° TIMEOUT TRIGGER: Sending batch due to timeout (5.2s >= 5.0s)\n", + "2025-10-22 23:00:51 - noveum_trace.transport.batch_processor - INFO - šŸ“¤ SENDING BATCH: 4 traces via send_callback\n", + "2025-10-22 23:00:51 - noveum_trace.transport.http_transport - INFO - šŸš€ SENDING BATCH: 4 traces to https://api.noveum.ai/api/v1/traces\n", + "2025-10-22 23:00:51 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: b4820be8-ae51-49bc-97d5-6aaf3f26ccbf) - 1 spans\n", + "2025-10-22 23:00:51 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: b4820be8-ae51-49bc-97d5-6aaf3f26ccbf) - 1 spans\n", + "2025-10-22 23:00:51 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace b4820be8-ae51-49bc-97d5-6aaf3f26ccbf\n", + "2025-10-22 23:00:51 - noveum_trace.transport.http_transport - INFO - āœ… Trace b4820be8-ae51-49bc-97d5-6aaf3f26ccbf successfully queued for export\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:00:51 - INFO - novaeval.evaluators.agent_evaluator - Saving intermediate results after 15 samples\n", + "2025-10-22 23:00:51 - INFO - novaeval.evaluators.agent_evaluator - Intermediate results saved to demo_results/agent.query_routing_dataset/agent_evaluation_results.csv\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Evaluating samples: 15it [01:08, 4.18s/it]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:00:51 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-10-22 23:00:52 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 58a8930f-83c2-4717-9f71-5a34660d8fa2) - 1 spans\n", + "2025-10-22 23:00:52 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 58a8930f-83c2-4717-9f71-5a34660d8fa2) - 1 spans\n", + "2025-10-22 23:00:52 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 58a8930f-83c2-4717-9f71-5a34660d8fa2\n", + "2025-10-22 23:00:52 - noveum_trace.transport.http_transport - INFO - āœ… Trace 58a8930f-83c2-4717-9f71-5a34660d8fa2 successfully queued for export\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:00:52 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-10-22 23:00:52 - noveum_trace.transport.http_transport - INFO - šŸ“” HTTP RESPONSE: Status 200 from https://api.noveum.ai/api/v1/traces\n", + "2025-10-22 23:00:52 - noveum_trace.transport.http_transport - INFO - āœ… Successfully sent batch of 4 traces\n", + "2025-10-22 23:00:52 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully sent batch of 4 traces via callback\n", + "2025-10-22 23:00:53 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 9572f06a-e0ba-4e67-81eb-eaad1774cc15) - 1 spans\n", + "2025-10-22 23:00:53 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 9572f06a-e0ba-4e67-81eb-eaad1774cc15) - 1 spans\n", + "2025-10-22 23:00:53 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 9572f06a-e0ba-4e67-81eb-eaad1774cc15\n", + "2025-10-22 23:00:53 - noveum_trace.transport.http_transport - INFO - āœ… Trace 9572f06a-e0ba-4e67-81eb-eaad1774cc15 successfully queued for export\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:00:53 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-10-22 23:00:54 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 11aa6f5b-1e71-440f-b4d7-cfa0b1a3ae56) - 1 spans\n", + "2025-10-22 23:00:54 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 11aa6f5b-1e71-440f-b4d7-cfa0b1a3ae56) - 1 spans\n", + "2025-10-22 23:00:54 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 11aa6f5b-1e71-440f-b4d7-cfa0b1a3ae56\n", + "2025-10-22 23:00:54 - noveum_trace.transport.http_transport - INFO - āœ… Trace 11aa6f5b-1e71-440f-b4d7-cfa0b1a3ae56 successfully queued for export\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:00:54 - INFO - novaeval.evaluators.agent_evaluator - Saving intermediate results after 16 samples\n", + "2025-10-22 23:00:54 - INFO - novaeval.evaluators.agent_evaluator - Intermediate results saved to demo_results/agent.query_routing_dataset/agent_evaluation_results.csv\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Evaluating samples: 16it [01:11, 3.92s/it]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:00:54 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-10-22 23:00:56 - noveum_trace.transport.batch_processor - INFO - ā° TIMEOUT TRIGGER: Sending batch due to timeout (5.4s >= 5.0s)\n", + "2025-10-22 23:00:56 - noveum_trace.transport.batch_processor - INFO - šŸ“¤ SENDING BATCH: 4 traces via send_callback\n", + "2025-10-22 23:00:56 - noveum_trace.transport.http_transport - INFO - šŸš€ SENDING BATCH: 4 traces to https://api.noveum.ai/api/v1/traces\n", + "2025-10-22 23:00:56 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 1700f8dd-b141-4677-9778-db8440931e33) - 1 spans\n", + "2025-10-22 23:00:56 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 1700f8dd-b141-4677-9778-db8440931e33) - 1 spans\n", + "2025-10-22 23:00:56 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 1700f8dd-b141-4677-9778-db8440931e33\n", + "2025-10-22 23:00:56 - noveum_trace.transport.http_transport - INFO - āœ… Trace 1700f8dd-b141-4677-9778-db8440931e33 successfully queued for export\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:00:56 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-10-22 23:00:57 - noveum_trace.transport.http_transport - INFO - šŸ“” HTTP RESPONSE: Status 200 from https://api.noveum.ai/api/v1/traces\n", + "2025-10-22 23:00:57 - noveum_trace.transport.http_transport - INFO - āœ… Successfully sent batch of 4 traces\n", + "2025-10-22 23:00:57 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully sent batch of 4 traces via callback\n", + "2025-10-22 23:00:58 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 4ba7a1c6-56f2-40bb-bb89-bbcd9bf864b9) - 1 spans\n", + "2025-10-22 23:00:58 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 4ba7a1c6-56f2-40bb-bb89-bbcd9bf864b9) - 1 spans\n", + "2025-10-22 23:00:58 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 4ba7a1c6-56f2-40bb-bb89-bbcd9bf864b9\n", + "2025-10-22 23:00:58 - noveum_trace.transport.http_transport - INFO - āœ… Trace 4ba7a1c6-56f2-40bb-bb89-bbcd9bf864b9 successfully queued for export\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:00:58 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-10-22 23:00:59 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: c25eaffa-a215-4fb8-af0d-0d0fcf0acde9) - 1 spans\n", + "2025-10-22 23:00:59 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: c25eaffa-a215-4fb8-af0d-0d0fcf0acde9) - 1 spans\n", + "2025-10-22 23:00:59 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace c25eaffa-a215-4fb8-af0d-0d0fcf0acde9\n", + "2025-10-22 23:00:59 - noveum_trace.transport.http_transport - INFO - āœ… Trace c25eaffa-a215-4fb8-af0d-0d0fcf0acde9 successfully queued for export\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:00:59 - INFO - novaeval.evaluators.agent_evaluator - Saving intermediate results after 17 samples\n", + "2025-10-22 23:00:59 - INFO - novaeval.evaluators.agent_evaluator - Intermediate results saved to demo_results/agent.query_routing_dataset/agent_evaluation_results.csv\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Evaluating samples: 17it [01:16, 4.19s/it]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:00:59 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-10-22 23:01:00 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: b925a4e7-7af5-4039-a7ab-0fdb05e03c71) - 1 spans\n", + "2025-10-22 23:01:00 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: b925a4e7-7af5-4039-a7ab-0fdb05e03c71) - 1 spans\n", + "2025-10-22 23:01:00 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace b925a4e7-7af5-4039-a7ab-0fdb05e03c71\n", + "2025-10-22 23:01:00 - noveum_trace.transport.http_transport - INFO - āœ… Trace b925a4e7-7af5-4039-a7ab-0fdb05e03c71 successfully queued for export\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:01:00 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-10-22 23:01:01 - noveum_trace.transport.batch_processor - INFO - ā° TIMEOUT TRIGGER: Sending batch due to timeout (5.0s >= 5.0s)\n", + "2025-10-22 23:01:01 - noveum_trace.transport.batch_processor - INFO - šŸ“¤ SENDING BATCH: 4 traces via send_callback\n", + "2025-10-22 23:01:01 - noveum_trace.transport.http_transport - INFO - šŸš€ SENDING BATCH: 4 traces to https://api.noveum.ai/api/v1/traces\n", + "2025-10-22 23:01:01 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 9639d4eb-e562-4372-a019-e6eb2f867d0f) - 1 spans\n", + "2025-10-22 23:01:01 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 9639d4eb-e562-4372-a019-e6eb2f867d0f) - 1 spans\n", + "2025-10-22 23:01:01 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 9639d4eb-e562-4372-a019-e6eb2f867d0f\n", + "2025-10-22 23:01:01 - noveum_trace.transport.http_transport - INFO - āœ… Trace 9639d4eb-e562-4372-a019-e6eb2f867d0f successfully queued for export\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:01:01 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-10-22 23:01:01 - noveum_trace.transport.http_transport - INFO - šŸ“” HTTP RESPONSE: Status 200 from https://api.noveum.ai/api/v1/traces\n", + "2025-10-22 23:01:01 - noveum_trace.transport.http_transport - INFO - āœ… Successfully sent batch of 4 traces\n", + "2025-10-22 23:01:01 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully sent batch of 4 traces via callback\n", + "2025-10-22 23:01:03 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 02b5f22f-eb99-4514-9a0d-fb74709ade16) - 1 spans\n", + "2025-10-22 23:01:03 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 02b5f22f-eb99-4514-9a0d-fb74709ade16) - 1 spans\n", + "2025-10-22 23:01:03 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 02b5f22f-eb99-4514-9a0d-fb74709ade16\n", + "2025-10-22 23:01:03 - noveum_trace.transport.http_transport - INFO - āœ… Trace 02b5f22f-eb99-4514-9a0d-fb74709ade16 successfully queued for export\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:01:03 - INFO - novaeval.evaluators.agent_evaluator - Saving intermediate results after 18 samples\n", + "2025-10-22 23:01:03 - INFO - novaeval.evaluators.agent_evaluator - Intermediate results saved to demo_results/agent.query_routing_dataset/agent_evaluation_results.csv\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Evaluating samples: 18it [01:20, 4.06s/it]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:01:03 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-10-22 23:01:04 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: ed8a64df-f92b-4a38-b236-02cd5720da91) - 1 spans\n", + "2025-10-22 23:01:04 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: ed8a64df-f92b-4a38-b236-02cd5720da91) - 1 spans\n", + "2025-10-22 23:01:04 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace ed8a64df-f92b-4a38-b236-02cd5720da91\n", + "2025-10-22 23:01:04 - noveum_trace.transport.http_transport - INFO - āœ… Trace ed8a64df-f92b-4a38-b236-02cd5720da91 successfully queued for export\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:01:04 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-10-22 23:01:06 - noveum_trace.transport.batch_processor - INFO - ā° TIMEOUT TRIGGER: Sending batch due to timeout (5.3s >= 5.0s)\n", + "2025-10-22 23:01:06 - noveum_trace.transport.batch_processor - INFO - šŸ“¤ SENDING BATCH: 3 traces via send_callback\n", + "2025-10-22 23:01:06 - noveum_trace.transport.http_transport - INFO - šŸš€ SENDING BATCH: 3 traces to https://api.noveum.ai/api/v1/traces\n", + "2025-10-22 23:01:06 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 65462cbf-b85d-40b9-b50d-a71850336ebe) - 1 spans\n", + "2025-10-22 23:01:06 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 65462cbf-b85d-40b9-b50d-a71850336ebe) - 1 spans\n", + "2025-10-22 23:01:06 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 65462cbf-b85d-40b9-b50d-a71850336ebe\n", + "2025-10-22 23:01:06 - noveum_trace.transport.http_transport - INFO - āœ… Trace 65462cbf-b85d-40b9-b50d-a71850336ebe successfully queued for export\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:01:06 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-10-22 23:01:07 - noveum_trace.transport.http_transport - INFO - šŸ“” HTTP RESPONSE: Status 200 from https://api.noveum.ai/api/v1/traces\n", + "2025-10-22 23:01:07 - noveum_trace.transport.http_transport - INFO - āœ… Successfully sent batch of 3 traces\n", + "2025-10-22 23:01:07 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully sent batch of 3 traces via callback\n", + "2025-10-22 23:01:08 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: a174d9f5-fa2d-4c6d-a674-341e8382a11d) - 1 spans\n", + "2025-10-22 23:01:08 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: a174d9f5-fa2d-4c6d-a674-341e8382a11d) - 1 spans\n", + "2025-10-22 23:01:08 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace a174d9f5-fa2d-4c6d-a674-341e8382a11d\n", + "2025-10-22 23:01:08 - noveum_trace.transport.http_transport - INFO - āœ… Trace a174d9f5-fa2d-4c6d-a674-341e8382a11d successfully queued for export\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:01:08 - INFO - novaeval.evaluators.agent_evaluator - Saving intermediate results after 19 samples\n", + "2025-10-22 23:01:08 - INFO - novaeval.evaluators.agent_evaluator - Intermediate results saved to demo_results/agent.query_routing_dataset/agent_evaluation_results.csv\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Evaluating samples: 19it [01:25, 4.36s/it]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:01:08 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-10-22 23:01:09 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 672a16dd-589f-4d48-a8e4-c9e6bb5d871b) - 1 spans\n", + "2025-10-22 23:01:09 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 672a16dd-589f-4d48-a8e4-c9e6bb5d871b) - 1 spans\n", + "2025-10-22 23:01:09 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 672a16dd-589f-4d48-a8e4-c9e6bb5d871b\n", + "2025-10-22 23:01:09 - noveum_trace.transport.http_transport - INFO - āœ… Trace 672a16dd-589f-4d48-a8e4-c9e6bb5d871b successfully queued for export\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:01:09 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-10-22 23:01:11 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 89085ca6-0aa1-44f9-9d93-804b9725c2c0) - 1 spans\n", + "2025-10-22 23:01:11 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 89085ca6-0aa1-44f9-9d93-804b9725c2c0) - 1 spans\n", + "2025-10-22 23:01:11 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 89085ca6-0aa1-44f9-9d93-804b9725c2c0\n", + "2025-10-22 23:01:11 - noveum_trace.transport.http_transport - INFO - āœ… Trace 89085ca6-0aa1-44f9-9d93-804b9725c2c0 successfully queued for export\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:01:11 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-10-22 23:01:12 - noveum_trace.transport.batch_processor - INFO - ā° TIMEOUT TRIGGER: Sending batch due to timeout (5.3s >= 5.0s)\n", + "2025-10-22 23:01:12 - noveum_trace.transport.batch_processor - INFO - šŸ“¤ SENDING BATCH: 4 traces via send_callback\n", + "2025-10-22 23:01:12 - noveum_trace.transport.http_transport - INFO - šŸš€ SENDING BATCH: 4 traces to https://api.noveum.ai/api/v1/traces\n", + "2025-10-22 23:01:12 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: b7a046e6-46d4-41f7-a47b-35dae73bb923) - 1 spans\n", + "2025-10-22 23:01:12 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: b7a046e6-46d4-41f7-a47b-35dae73bb923) - 1 spans\n", + "2025-10-22 23:01:12 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace b7a046e6-46d4-41f7-a47b-35dae73bb923\n", + "2025-10-22 23:01:12 - noveum_trace.transport.http_transport - INFO - āœ… Trace b7a046e6-46d4-41f7-a47b-35dae73bb923 successfully queued for export\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:01:12 - INFO - novaeval.evaluators.agent_evaluator - Saving intermediate results after 20 samples\n", + "2025-10-22 23:01:12 - INFO - novaeval.evaluators.agent_evaluator - Intermediate results saved to demo_results/agent.query_routing_dataset/agent_evaluation_results.csv\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Evaluating samples: 20it [01:29, 4.46s/it]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:01:12 - INFO - novaeval.evaluators.agent_evaluator - Saving final results\n", + "2025-10-22 23:01:12 - INFO - novaeval.evaluators.agent_evaluator - Reloaded 20 results from CSV\n", + "2025-10-22 23:01:12 - INFO - novaeval.evaluators.agent_evaluator - Agent evaluation completed\n", + "\n", + "āœ… Evaluation completed!\n", + "\n", + "šŸ“Š Results Summary:\n", + " - task_progression: 1.27\n", + " - context_relevancy: 7.36\n", + " - role_adherence: 5.45\n", + " - tool_relevancy: 0.00\n", + " - parameter_correctness: 0.00\n", + "\n", + "šŸ” Individual Scores:\n", + "\n", + " Record 1 (Task: eda4fe22-9a2b-4b73-856b-f4f3309bf719):\n", + " - task_progression: 1.0\n", + " - context_relevancy: 7.5\n", + " - role_adherence: 1.0\n", + " - tool_relevancy: 0.0\n", + " - parameter_correctness: 0.0\n", + "\n", + " Record 2 (Task: 0ffffba1-8a37-443c-8866-d53ffbfa7718):\n", + " - task_progression: 2.8\n", + " - context_relevancy: 7.8\n", + " - role_adherence: 10.0\n", + " - tool_relevancy: 0.0\n", + " - parameter_correctness: 0.0\n", + "\n", + " Record 3 (Task: f1f37bd7-0851-4659-b493-b80d3800d920):\n", + " - task_progression: 1.0\n", + " - context_relevancy: 7.5\n", + " - role_adherence: 10.0\n", + " - tool_relevancy: 0.0\n", + " - parameter_correctness: 0.0\n", + "\n", + " Record 4 (Task: 43cdf081-4f01-49cd-b566-dbd1619e6cd2):\n", + " - task_progression: 1.0\n", + " - context_relevancy: 7.5\n", + " - role_adherence: 10.0\n", + " - tool_relevancy: 0.0\n", + " - parameter_correctness: 0.0\n", + "\n", + " Record 5 (Task: 9a1983f4-09da-4b53-80e6-38de6878e0e7):\n", + " - task_progression: 2.8\n", + " - context_relevancy: 7.8\n", + " - role_adherence: 1.0\n", + " - tool_relevancy: 0.0\n", + " - parameter_correctness: 0.0\n", + "\n", + " Record 6 (Task: 5d2517e1-220a-429d-9d59-f701bda25eed):\n", + " - task_progression: 1.0\n", + " - context_relevancy: 1.0\n", + " - role_adherence: 1.0\n", + " - tool_relevancy: 0.0\n", + " - parameter_correctness: 0.0\n", + "\n", + " Record 7 (Task: 52aacb67-c361-4445-9b72-c157f79f47d6):\n", + " - task_progression: 1.0\n", + " - context_relevancy: 7.5\n", + " - role_adherence: 1.0\n", + " - tool_relevancy: 0.0\n", + " - parameter_correctness: 0.0\n", + "\n", + " Record 8 (Task: a81ca3a8-80aa-4c39-876e-8d40ea7a0aef):\n", + " - task_progression: 1.0\n", + " - context_relevancy: 7.5\n", + " - role_adherence: 10.0\n", + " - tool_relevancy: 0.0\n", + " - parameter_correctness: 0.0\n", + "\n", + " Record 9 (Task: 230aad27-f3dd-4968-a45a-3c2f07ac28ed):\n", + " - task_progression: 1.0\n", + " - context_relevancy: 7.8\n", + " - role_adherence: 10.0\n", + " - tool_relevancy: 0.0\n", + " - parameter_correctness: 0.0\n", + "\n", + " Record 10 (Task: 83c7dcce-3d89-4da1-8b3f-d419885d4cbc):\n", + " - task_progression: 1.0\n", + " - context_relevancy: 7.8\n", + " - role_adherence: 1.0\n", + " - tool_relevancy: 0.0\n", + " - parameter_correctness: 0.0\n", + "\n", + " Record 11 (Task: 2218f641-604c-491a-9710-b51a9941b982):\n", + " - task_progression: 1.0\n", + " - context_relevancy: 7.8\n", + " - role_adherence: 9.0\n", + " - tool_relevancy: 0.0\n", + " - parameter_correctness: 0.0\n", + "\n", + " Record 12 (Task: 255fd49c-84b4-4b18-887e-6308a412d535):\n", + " - task_progression: 1.0\n", + " - context_relevancy: 7.8\n", + " - role_adherence: 1.0\n", + " - tool_relevancy: 0.0\n", + " - parameter_correctness: 0.0\n", + "\n", + " Record 13 (Task: dc511122-c0b6-415c-9a49-c7b45132dd87):\n", + " - task_progression: 1.0\n", + " - context_relevancy: 7.8\n", + " - role_adherence: 10.0\n", + " - tool_relevancy: 0.0\n", + " - parameter_correctness: 0.0\n", + "\n", + " Record 14 (Task: 04bebf38-a343-4563-80db-0154bef8d927):\n", + " - task_progression: 1.0\n", + " - context_relevancy: 7.8\n", + " - role_adherence: 1.0\n", + " - tool_relevancy: 0.0\n", + " - parameter_correctness: 0.0\n", + "\n", + " Record 15 (Task: 5e043630-6493-42b5-beb8-79faa19bfa37):\n", + " - task_progression: 1.0\n", + " - context_relevancy: 7.8\n", + " - role_adherence: 10.0\n", + " - tool_relevancy: 0.0\n", + " - parameter_correctness: 0.0\n", + "\n", + " Record 16 (Task: 7da9814d-a2e8-4c4e-b750-68b26bd5fd22):\n", + " - task_progression: 1.0\n", + " - context_relevancy: 7.5\n", + " - role_adherence: 1.0\n", + " - tool_relevancy: 0.0\n", + " - parameter_correctness: 0.0\n", + "\n", + " Record 17 (Task: 16143f74-2831-4753-b33d-ce4b645093c5):\n", + " - task_progression: 1.0\n", + " - context_relevancy: 7.8\n", + " - role_adherence: 10.0\n", + " - tool_relevancy: 0.0\n", + " - parameter_correctness: 0.0\n", + "\n", + " Record 18 (Task: fc64e6cc-6739-4256-ac4a-7b80c3028233):\n", + " - task_progression: 1.0\n", + " - context_relevancy: 7.8\n", + " - role_adherence: 1.0\n", + " - tool_relevancy: 0.0\n", + " - parameter_correctness: 0.0\n", + "\n", + " Record 19 (Task: b7945c49-f584-4c70-972d-536a805d8a31):\n", + " - task_progression: 1.0\n", + " - context_relevancy: 7.5\n", + " - role_adherence: 1.0\n", + " - tool_relevancy: 0.0\n", + " - parameter_correctness: 0.0\n", + "\n", + " Record 20 (Task: f5c40ecf-36c0-45ba-9cc9-dc0329b0324b):\n", + " - task_progression: 2.8\n", + " - context_relevancy: 7.8\n", + " - role_adherence: 10.0\n", + " - tool_relevancy: 0.0\n", + " - parameter_correctness: 0.0\n", + "āœ… Evaluation completed successfully!\n", + "\n", + "šŸ“‹ Step 7: Exporting Dataset\n", + "šŸ’¾ Exporting processed dataset...\n", + "āœ… Exported to ./processed_datasets/agent.query_routing_dataset_processed_dataset.json\n", + "āœ… Exported to ./processed_datasets/agent.query_routing_dataset_processed_dataset.csv\n", + "āœ… Export completed successfully!\n", + "\n", + "============================================================\n", + "šŸŽ‰ EVALUATION PIPELINE COMPLETED!\n", + "šŸ“Š Final Results:\n", + " - File processed: split_datasets/agent.query_routing_dataset.json\n", + " - Spans loaded: 20\n", + " - Dataset size: 20\n", + " - Evaluation completed: True\n", + " - Export successful: True\n", + " - Results saved to: ./demo_results/agent.query_routing_dataset/\n", + "Completed agent.query_routing_dataset.json\n", + "\n", + "Processing tool-orchestator_dataset.json...\n", + "šŸš€ Starting Complete Agent Evaluation Pipeline\n", + "šŸ“ Processing file: split_datasets/tool-orchestator_dataset.json\n", + "============================================================\n", + "\n", + "šŸ“‹ Step 1: Environment Setup\n", + "āœ… Logging configured at INFO level\n", + "šŸ” Environment validation:\n", + " āœ… gemini_api_key: True\n", + " āœ… pandas_available: True\n", + " āœ… novaeval_available: True\n", + "āœ… Environment ready for evaluation!\n", + "\n", + "šŸ“‹ Step 2: Loading Dataset\n", + "šŸ“Š Loaded 20 spans from split_datasets/tool-orchestator_dataset.json\n", + "\n", + "šŸ” Available span types:\n", + " - tool-orchestator: 20\n", + "āœ… Dataset loaded: 20 spans\n", + "\n", + "šŸ“‹ Step 3: Converting to AgentDataset Format\n", + "šŸ”„ Converting spans to AgentData objects...\n", + "returning unknown type for span\n", + "{'span_id': 'b6b7c1c0-3b7b-4e19-805d-a16beb7ea871', 'trace_id': 'eda4fe22-9a2b-4b73-856b-f4f3309bf719', 'parent_span_id': '', 'name': 'tool-orchestator', 'start_time': '2025-10-09 08:26:46.384000000', 'end_time': '2025-10-09 08:26:48.685000000', 'duration_ms': 2301.412, 'status': 'ok', 'status_message': '', 'attributes': {'input_query': 'What are the latest breakthroughs in artificial intelligence?', 'query.length': 61, 'processing.mode': 'Web Search', 'processing.time_seconds': 2.301326274871826, 'processing.response_length': 423, 'processing.sources_count': 1, 'output_response': 'Final Answer: The provided web search results do not contain specific information about the latest breakthroughs in artificial intelligence. To find the most recent advancements in AI, I recommend checking reputabl...', 'final_answer_mode': 'Web Search', 'query_processed.input_query': 'What are the latest breakthroughs in artificial intelligence?', 'query_processed.output_response': 'Successfully processed query using Web Search, generated 423 character response', 'query_processed.mode': 'Web Search', 'query_processed.processing_time': 2.301326274871826, 'query_processed.response_length': 423}, 'events': [], 'links': [], 'trace_trace_id': 'eda4fe22-9a2b-4b73-856b-f4f3309bf719', 'trace_name': 'auto_trace_tool-orchestator', 'project': 'noveum-ai-agent-rag-websearch', 'environment': 'development', 'trace_status': 'ok', 'trace_status_message': '', 'trace_start_time': '2025-10-09 08:26:46.384000000', 'trace_end_time': '2025-10-09 08:26:48.685000000', 'trace_duration_ms': 2301.488, 'span_count': 6, 'error_count': 0, 'sdk': {'name': 'noveum-trace-python', 'version': '0.3.8'}, 'trace_attributes': {'noveum.project': 'noveum-ai-agent-rag-websearch', 'noveum.environment': 'development', 'noveum.sdk.version': '0.3.8', 'noveum.sampling.decision': 'record'}, 'metadata': {'user_id': None, 'session_id': None, 'request_id': None, 'tags': {}, 'custom_attributes': {}}, 'created_at': '2025-10-09 08:26:49.422000000', 'updated_at': '2025-10-09 08:26:49.422000000'}\n", + "Spans with unknown type: 1\n", + "returning unknown type for span\n", + "{'span_id': '3045d971-b7ff-46cc-be11-6db7d65dfefd', 'trace_id': '0ffffba1-8a37-443c-8866-d53ffbfa7718', 'parent_span_id': '', 'name': 'tool-orchestator', 'start_time': '2025-10-09 08:26:41.850000000', 'end_time': '2025-10-09 08:26:45.377000000', 'duration_ms': 3527.75, 'status': 'ok', 'status_message': '', 'attributes': {'input_query': \"What's happening in the software development world today?\", 'query.length': 57, 'processing.mode': 'Web Search', 'processing.time_seconds': 3.527592897415161, 'processing.response_length': 795, 'processing.sources_count': 1, 'output_response': \"Final Answer: The web search did not yield any specific results regarding current events in the software development world. Therefore, I cannot provide detailed information on what's happening today in that field.\\n...\", 'final_answer_mode': 'Web Search', 'query_processed.input_query': \"What's happening in the software development world today?\", 'query_processed.output_response': 'Successfully processed query using Web Search, generated 795 character response', 'query_processed.mode': 'Web Search', 'query_processed.processing_time': 3.527592897415161, 'query_processed.response_length': 795}, 'events': [], 'links': [], 'trace_trace_id': '0ffffba1-8a37-443c-8866-d53ffbfa7718', 'trace_name': 'auto_trace_tool-orchestator', 'project': 'noveum-ai-agent-rag-websearch', 'environment': 'development', 'trace_status': 'ok', 'trace_status_message': '', 'trace_start_time': '2025-10-09 08:26:41.850000000', 'trace_end_time': '2025-10-09 08:26:45.377000000', 'trace_duration_ms': 3527.831, 'span_count': 6, 'error_count': 0, 'sdk': {'name': 'noveum-trace-python', 'version': '0.3.8'}, 'trace_attributes': {'noveum.project': 'noveum-ai-agent-rag-websearch', 'noveum.environment': 'development', 'noveum.sdk.version': '0.3.8', 'noveum.sampling.decision': 'record'}, 'metadata': {'user_id': None, 'session_id': None, 'request_id': None, 'tags': {}, 'custom_attributes': {}}, 'created_at': '2025-10-09 08:26:49.422000000', 'updated_at': '2025-10-09 08:26:49.422000000'}\n", + "Spans with unknown type: 2\n", + "returning unknown type for span\n", + "{'span_id': '42d4cb1f-17ec-433a-8087-90292c2fa9d9', 'trace_id': 'f1f37bd7-0851-4659-b493-b80d3800d920', 'parent_span_id': '', 'name': 'tool-orchestator', 'start_time': '2025-10-09 08:26:35.915000000', 'end_time': '2025-10-09 08:26:40.841000000', 'duration_ms': 4926.2519999999995, 'status': 'ok', 'status_message': '', 'attributes': {'input_query': 'What are the newest features in cloud computing?', 'query.length': 48, 'processing.mode': 'RAG', 'processing.time_seconds': 4.926196813583374, 'processing.response_length': 883, 'processing.sources_count': 5, 'output_response': 'Final Answer: The provided context does not specifically mention any new features in cloud computing as a general topic. However, it does highlight several recent enhancements and features related to the Noveum.ai ...', 'final_answer_mode': 'RAG', 'query_processed.input_query': 'What are the newest features in cloud computing?', 'query_processed.output_response': 'Successfully processed query using RAG, generated 883 character response', 'query_processed.mode': 'RAG', 'query_processed.processing_time': 4.926196813583374, 'query_processed.response_length': 883}, 'events': [], 'links': [], 'trace_trace_id': 'f1f37bd7-0851-4659-b493-b80d3800d920', 'trace_name': 'auto_trace_tool-orchestator', 'project': 'noveum-ai-agent-rag-websearch', 'environment': 'development', 'trace_status': 'ok', 'trace_status_message': '', 'trace_start_time': '2025-10-09 08:26:35.915000000', 'trace_end_time': '2025-10-09 08:26:40.841000000', 'trace_duration_ms': 4926.342, 'span_count': 6, 'error_count': 0, 'sdk': {'name': 'noveum-trace-python', 'version': '0.3.8'}, 'trace_attributes': {'noveum.project': 'noveum-ai-agent-rag-websearch', 'noveum.environment': 'development', 'noveum.sdk.version': '0.3.8', 'noveum.sampling.decision': 'record'}, 'metadata': {'user_id': None, 'session_id': None, 'request_id': None, 'tags': {}, 'custom_attributes': {}}, 'created_at': '2025-10-09 08:26:44.053000000', 'updated_at': '2025-10-09 08:26:44.053000000'}\n", + "Spans with unknown type: 3\n", + "returning unknown type for span\n", + "{'span_id': 'e22b809d-5320-40b7-b78a-e3e125538a6b', 'trace_id': '43cdf081-4f01-49cd-b566-dbd1619e6cd2', 'parent_span_id': '', 'name': 'tool-orchestator', 'start_time': '2025-10-09 08:26:32.239000000', 'end_time': '2025-10-09 08:26:34.906000000', 'duration_ms': 2667.1330000000003, 'status': 'ok', 'status_message': '', 'attributes': {'input_query': \"What's the current status of cryptocurrency markets?\", 'query.length': 52, 'processing.mode': 'Web Search', 'processing.time_seconds': 2.6669082641601562, 'processing.response_length': 371, 'processing.sources_count': 1, 'output_response': 'Final Answer: The web search results did not provide any specific information regarding the current status of cryptocurrency markets. Therefore, I cannot provide an accurate update on market conditions, trends, or ...', 'final_answer_mode': 'Web Search', 'query_processed.input_query': \"What's the current status of cryptocurrency markets?\", 'query_processed.output_response': 'Successfully processed query using Web Search, generated 371 character response', 'query_processed.mode': 'Web Search', 'query_processed.processing_time': 2.6669082641601562, 'query_processed.response_length': 371}, 'events': [], 'links': [], 'trace_trace_id': '43cdf081-4f01-49cd-b566-dbd1619e6cd2', 'trace_name': 'auto_trace_tool-orchestator', 'project': 'noveum-ai-agent-rag-websearch', 'environment': 'development', 'trace_status': 'ok', 'trace_status_message': '', 'trace_start_time': '2025-10-09 08:26:32.237000000', 'trace_end_time': '2025-10-09 08:26:34.906000000', 'trace_duration_ms': 2668.841, 'span_count': 6, 'error_count': 0, 'sdk': {'name': 'noveum-trace-python', 'version': '0.3.8'}, 'trace_attributes': {'noveum.project': 'noveum-ai-agent-rag-websearch', 'noveum.environment': 'development', 'noveum.sdk.version': '0.3.8', 'noveum.sampling.decision': 'record'}, 'metadata': {'user_id': None, 'session_id': None, 'request_id': None, 'tags': {}, 'custom_attributes': {}}, 'created_at': '2025-10-09 08:26:38.618000000', 'updated_at': '2025-10-09 08:26:38.618000000'}\n", + "Spans with unknown type: 4\n", + "returning unknown type for span\n", + "{'span_id': '813a4450-1cf8-4d43-8493-eabfa200919b', 'trace_id': '9a1983f4-09da-4b53-80e6-38de6878e0e7', 'parent_span_id': '', 'name': 'tool-orchestator', 'start_time': '2025-10-09 08:26:27.992000000', 'end_time': '2025-10-09 08:26:31.231000000', 'duration_ms': 3239.601, 'status': 'ok', 'status_message': '', 'attributes': {'input_query': 'What are the latest updates in Python programming?', 'query.length': 50, 'processing.mode': 'Web Search', 'processing.time_seconds': 3.2394051551818848, 'processing.response_length': 583, 'processing.sources_count': 1, 'output_response': 'Final Answer: The web search results do not provide specific information about the latest updates in Python programming. They focus primarily on news related to China, including geopolitics, economy, and lifestyle,...', 'final_answer_mode': 'Web Search', 'query_processed.input_query': 'What are the latest updates in Python programming?', 'query_processed.output_response': 'Successfully processed query using Web Search, generated 583 character response', 'query_processed.mode': 'Web Search', 'query_processed.processing_time': 3.2394051551818848, 'query_processed.response_length': 583}, 'events': [], 'links': [], 'trace_trace_id': '9a1983f4-09da-4b53-80e6-38de6878e0e7', 'trace_name': 'auto_trace_tool-orchestator', 'project': 'noveum-ai-agent-rag-websearch', 'environment': 'development', 'trace_status': 'ok', 'trace_status_message': '', 'trace_start_time': '2025-10-09 08:26:27.992000000', 'trace_end_time': '2025-10-09 08:26:31.231000000', 'trace_duration_ms': 3239.706, 'span_count': 6, 'error_count': 0, 'sdk': {'name': 'noveum-trace-python', 'version': '0.3.8'}, 'trace_attributes': {'noveum.project': 'noveum-ai-agent-rag-websearch', 'noveum.environment': 'development', 'noveum.sdk.version': '0.3.8', 'noveum.sampling.decision': 'record'}, 'metadata': {'user_id': None, 'session_id': None, 'request_id': None, 'tags': {}, 'custom_attributes': {}}, 'created_at': '2025-10-09 08:26:33.463000000', 'updated_at': '2025-10-09 08:26:33.463000000'}\n", + "Spans with unknown type: 5\n", + "returning unknown type for span\n", + "{'span_id': '6102352a-833e-493d-adbb-887e2af17918', 'trace_id': '5d2517e1-220a-429d-9d59-f701bda25eed', 'parent_span_id': '', 'name': 'tool-orchestator', 'start_time': '2025-10-09 08:26:23.210000000', 'end_time': '2025-10-09 08:26:26.985000000', 'duration_ms': 3775.0409999999997, 'status': 'ok', 'status_message': '', 'attributes': {'input_query': 'What happened in tech news this week?', 'query.length': 37, 'processing.mode': 'Web Search', 'processing.time_seconds': 3.774940013885498, 'processing.response_length': 501, 'processing.sources_count': 1, 'output_response': 'Final Answer: The web search results do not provide specific information about recent events in tech news for this week. They primarily discuss the grammatical usage of the phrase \"what happened\" and its variations...', 'final_answer_mode': 'Web Search', 'query_processed.input_query': 'What happened in tech news this week?', 'query_processed.output_response': 'Successfully processed query using Web Search, generated 501 character response', 'query_processed.mode': 'Web Search', 'query_processed.processing_time': 3.774940013885498, 'query_processed.response_length': 501}, 'events': [], 'links': [], 'trace_trace_id': '5d2517e1-220a-429d-9d59-f701bda25eed', 'trace_name': 'auto_trace_tool-orchestator', 'project': 'noveum-ai-agent-rag-websearch', 'environment': 'development', 'trace_status': 'ok', 'trace_status_message': '', 'trace_start_time': '2025-10-09 08:26:23.210000000', 'trace_end_time': '2025-10-09 08:26:26.985000000', 'trace_duration_ms': 3775.124, 'span_count': 6, 'error_count': 0, 'sdk': {'name': 'noveum-trace-python', 'version': '0.3.8'}, 'trace_attributes': {'noveum.project': 'noveum-ai-agent-rag-websearch', 'noveum.environment': 'development', 'noveum.sdk.version': '0.3.8', 'noveum.sampling.decision': 'record'}, 'metadata': {'user_id': None, 'session_id': None, 'request_id': None, 'tags': {}, 'custom_attributes': {}}, 'created_at': '2025-10-09 08:26:28.174000000', 'updated_at': '2025-10-09 08:26:28.174000000'}\n", + "Spans with unknown type: 6\n", + "returning unknown type for span\n", + "{'span_id': '2926418a-fe09-4df0-bf19-a527bc24a478', 'trace_id': '52aacb67-c361-4445-9b72-c157f79f47d6', 'parent_span_id': '', 'name': 'tool-orchestator', 'start_time': '2025-10-09 08:26:16.544000000', 'end_time': '2025-10-09 08:26:22.201000000', 'duration_ms': 5656.852, 'status': 'ok', 'status_message': '', 'attributes': {'input_query': 'What are the current trends in observability tools?', 'query.length': 51, 'processing.mode': 'RAG', 'processing.time_seconds': 5.656759023666382, 'processing.response_length': 1074, 'processing.sources_count': 5, 'output_response': 'Final Answer: The provided context does not explicitly outline current trends in observability tools. However, it highlights some key aspects of observability that are particularly relevant to AI applications. \\n\\n1....', 'final_answer_mode': 'RAG', 'query_processed.input_query': 'What are the current trends in observability tools?', 'query_processed.output_response': 'Successfully processed query using RAG, generated 1074 character response', 'query_processed.mode': 'RAG', 'query_processed.processing_time': 5.656759023666382, 'query_processed.response_length': 1074}, 'events': [], 'links': [], 'trace_trace_id': '52aacb67-c361-4445-9b72-c157f79f47d6', 'trace_name': 'auto_trace_tool-orchestator', 'project': 'noveum-ai-agent-rag-websearch', 'environment': 'development', 'trace_status': 'ok', 'trace_status_message': '', 'trace_start_time': '2025-10-09 08:26:16.544000000', 'trace_end_time': '2025-10-09 08:26:22.201000000', 'trace_duration_ms': 5657.012, 'span_count': 6, 'error_count': 0, 'sdk': {'name': 'noveum-trace-python', 'version': '0.3.8'}, 'trace_attributes': {'noveum.project': 'noveum-ai-agent-rag-websearch', 'noveum.environment': 'development', 'noveum.sdk.version': '0.3.8', 'noveum.sampling.decision': 'record'}, 'metadata': {'user_id': None, 'session_id': None, 'request_id': None, 'tags': {}, 'custom_attributes': {}}, 'created_at': '2025-10-09 08:26:22.904000000', 'updated_at': '2025-10-09 08:26:22.904000000'}\n", + "Spans with unknown type: 7\n", + "returning unknown type for span\n", + "{'span_id': 'afe85a1d-13c8-4476-8154-52d58ba83d52', 'trace_id': 'a81ca3a8-80aa-4c39-876e-8d40ea7a0aef', 'parent_span_id': '', 'name': 'tool-orchestator', 'start_time': '2025-10-09 08:26:08.903000000', 'end_time': '2025-10-09 08:26:15.537000000', 'duration_ms': 6633.559, 'status': 'ok', 'status_message': '', 'attributes': {'input_query': 'Tell me about recent developments in machine learning', 'query.length': 53, 'processing.mode': 'Web Search', 'processing.time_seconds': 6.63348126411438, 'processing.response_length': 1639, 'processing.sources_count': 1, 'output_response': 'Final Answer: The provided web search results do not contain specific information about recent developments in machine learning. Therefore, I cannot provide a detailed answer based on those results.\\n\\nHowever, I can...', 'final_answer_mode': 'Web Search', 'query_processed.input_query': 'Tell me about recent developments in machine learning', 'query_processed.output_response': 'Successfully processed query using Web Search, generated 1639 character response', 'query_processed.mode': 'Web Search', 'query_processed.processing_time': 6.63348126411438, 'query_processed.response_length': 1639}, 'events': [], 'links': [], 'trace_trace_id': 'a81ca3a8-80aa-4c39-876e-8d40ea7a0aef', 'trace_name': 'auto_trace_tool-orchestator', 'project': 'noveum-ai-agent-rag-websearch', 'environment': 'development', 'trace_status': 'ok', 'trace_status_message': '', 'trace_start_time': '2025-10-09 08:26:08.903000000', 'trace_end_time': '2025-10-09 08:26:15.537000000', 'trace_duration_ms': 6633.662, 'span_count': 6, 'error_count': 0, 'sdk': {'name': 'noveum-trace-python', 'version': '0.3.8'}, 'trace_attributes': {'noveum.project': 'noveum-ai-agent-rag-websearch', 'noveum.environment': 'development', 'noveum.sdk.version': '0.3.8', 'noveum.sampling.decision': 'record'}, 'metadata': {'user_id': None, 'session_id': None, 'request_id': None, 'tags': {}, 'custom_attributes': {}}, 'created_at': '2025-10-09 08:26:16.227000000', 'updated_at': '2025-10-09 08:26:16.227000000'}\n", + "Spans with unknown type: 8\n", + "returning unknown type for span\n", + "{'span_id': 'db8da393-6d72-421e-87dc-96f56b6576e5', 'trace_id': '230aad27-f3dd-4968-a45a-3c2f07ac28ed', 'parent_span_id': '', 'name': 'tool-orchestator', 'start_time': '2025-10-09 08:26:02.424000000', 'end_time': '2025-10-09 08:26:07.894000000', 'duration_ms': 5470.179, 'status': 'ok', 'status_message': '', 'attributes': {'input_query': \"What's the weather like today?\", 'query.length': 30, 'processing.mode': 'Web Search', 'processing.time_seconds': 5.469927072525024, 'processing.response_length': 376, 'processing.sources_count': 1, 'output_response': \"Final Answer: The provided web search results do not contain specific information about today's weather in any particular location. They mention weather forecasts for various places, including France and Singapore,...\", 'final_answer_mode': 'Web Search', 'query_processed.input_query': \"What's the weather like today?\", 'query_processed.output_response': 'Successfully processed query using Web Search, generated 376 character response', 'query_processed.mode': 'Web Search', 'query_processed.processing_time': 5.469927072525024, 'query_processed.response_length': 376}, 'events': [], 'links': [], 'trace_trace_id': '230aad27-f3dd-4968-a45a-3c2f07ac28ed', 'trace_name': 'auto_trace_tool-orchestator', 'project': 'noveum-ai-agent-rag-websearch', 'environment': 'development', 'trace_status': 'ok', 'trace_status_message': '', 'trace_start_time': '2025-10-09 08:26:02.424000000', 'trace_end_time': '2025-10-09 08:26:07.894000000', 'trace_duration_ms': 5470.232, 'span_count': 6, 'error_count': 0, 'sdk': {'name': 'noveum-trace-python', 'version': '0.3.8'}, 'trace_attributes': {'noveum.project': 'noveum-ai-agent-rag-websearch', 'noveum.environment': 'development', 'noveum.sdk.version': '0.3.8', 'noveum.sampling.decision': 'record'}, 'metadata': {'user_id': None, 'session_id': None, 'request_id': None, 'tags': {}, 'custom_attributes': {}}, 'created_at': '2025-10-09 08:26:08.586000000', 'updated_at': '2025-10-09 08:26:08.586000000'}\n", + "Spans with unknown type: 9\n", + "returning unknown type for span\n", + "{'span_id': 'a290202d-1b3f-431f-8728-fcb937380f8c', 'trace_id': '83c7dcce-3d89-4da1-8b3f-d419885d4cbc', 'parent_span_id': '', 'name': 'tool-orchestator', 'start_time': '2025-10-09 08:25:56.886000000', 'end_time': '2025-10-09 08:26:01.416000000', 'duration_ms': 4530.039000000001, 'status': 'ok', 'status_message': '', 'attributes': {'input_query': 'What are the latest AI news today?', 'query.length': 34, 'processing.mode': 'Web Search', 'processing.time_seconds': 4.529800176620483, 'processing.response_length': 1003, 'processing.sources_count': 1, 'output_response': 'Final Answer: The latest news in artificial intelligence includes several key updates:\\n\\n1. **AI Spending Concerns**: Jeff Bezos has expressed concerns that the current level of investment in AI resembles an \"indust...', 'final_answer_mode': 'Web Search', 'query_processed.input_query': 'What are the latest AI news today?', 'query_processed.output_response': 'Successfully processed query using Web Search, generated 1003 character response', 'query_processed.mode': 'Web Search', 'query_processed.processing_time': 4.529800176620483, 'query_processed.response_length': 1003}, 'events': [], 'links': [], 'trace_trace_id': '83c7dcce-3d89-4da1-8b3f-d419885d4cbc', 'trace_name': 'auto_trace_tool-orchestator', 'project': 'noveum-ai-agent-rag-websearch', 'environment': 'development', 'trace_status': 'ok', 'trace_status_message': '', 'trace_start_time': '2025-10-09 08:25:56.886000000', 'trace_end_time': '2025-10-09 08:26:01.416000000', 'trace_duration_ms': 4530.08, 'span_count': 6, 'error_count': 0, 'sdk': {'name': 'noveum-trace-python', 'version': '0.3.8'}, 'trace_attributes': {'noveum.project': 'noveum-ai-agent-rag-websearch', 'noveum.environment': 'development', 'noveum.sdk.version': '0.3.8', 'noveum.sampling.decision': 'record'}, 'metadata': {'user_id': None, 'session_id': None, 'request_id': None, 'tags': {}, 'custom_attributes': {}}, 'created_at': '2025-10-09 08:26:02.122000000', 'updated_at': '2025-10-09 08:26:02.122000000'}\n", + "Spans with unknown type: 10\n", + "returning unknown type for span\n", + "{'span_id': '5fbaef83-95fc-4bed-9695-9d69c37c35e4', 'trace_id': '2218f641-604c-491a-9710-b51a9941b982', 'parent_span_id': '', 'name': 'tool-orchestator', 'start_time': '2025-10-09 08:25:48.107000000', 'end_time': '2025-10-09 08:25:55.878000000', 'duration_ms': 7770.846, 'status': 'ok', 'status_message': '', 'attributes': {'input_query': 'What are the benefits of using Noveum Trace?', 'query.length': 44, 'processing.mode': 'RAG', 'processing.time_seconds': 7.770798206329346, 'processing.response_length': 1460, 'processing.sources_count': 5, 'output_response': 'Final Answer: Using Noveum Trace offers several benefits, including:\\n\\n1. **Real-Time Monitoring**: The Noveum platform provides a real-time dashboard for analyzing traces and performance, with live updates as new t...', 'final_answer_mode': 'RAG', 'query_processed.input_query': 'What are the benefits of using Noveum Trace?', 'query_processed.output_response': 'Successfully processed query using RAG, generated 1460 character response', 'query_processed.mode': 'RAG', 'query_processed.processing_time': 7.770798206329346, 'query_processed.response_length': 1460}, 'events': [], 'links': [], 'trace_trace_id': '2218f641-604c-491a-9710-b51a9941b982', 'trace_name': 'auto_trace_tool-orchestator', 'project': 'noveum-ai-agent-rag-websearch', 'environment': 'development', 'trace_status': 'ok', 'trace_status_message': '', 'trace_start_time': '2025-10-09 08:25:48.107000000', 'trace_end_time': '2025-10-09 08:25:55.878000000', 'trace_duration_ms': 7770.953, 'span_count': 6, 'error_count': 0, 'sdk': {'name': 'noveum-trace-python', 'version': '0.3.8'}, 'trace_attributes': {'noveum.project': 'noveum-ai-agent-rag-websearch', 'noveum.environment': 'development', 'noveum.sdk.version': '0.3.8', 'noveum.sampling.decision': 'record'}, 'metadata': {'user_id': None, 'session_id': None, 'request_id': None, 'tags': {}, 'custom_attributes': {}}, 'created_at': '2025-10-09 08:25:56.565000000', 'updated_at': '2025-10-09 08:25:56.565000000'}\n", + "Spans with unknown type: 11\n", + "returning unknown type for span\n", + "{'span_id': 'b3ac3924-928b-4d48-a2a7-aa749802fef9', 'trace_id': '255fd49c-84b4-4b18-887e-6308a412d535', 'parent_span_id': '', 'name': 'tool-orchestator', 'start_time': '2025-10-09 08:25:43.830000000', 'end_time': '2025-10-09 08:25:47.099000000', 'duration_ms': 3268.8160000000003, 'status': 'ok', 'status_message': '', 'attributes': {'input_query': 'How do I configure Noveum for my system?', 'query.length': 40, 'processing.mode': 'RAG', 'processing.time_seconds': 3.268723249435425, 'processing.response_length': 613, 'processing.sources_count': 5, 'output_response': 'Final Answer: To configure Noveum for your system, you can refer to the SDK Integration Guide available in the Noveum documentation. This guide will help you trace your AI applications with minimal code changes. Th...', 'final_answer_mode': 'RAG', 'query_processed.input_query': 'How do I configure Noveum for my system?', 'query_processed.output_response': 'Successfully processed query using RAG, generated 613 character response', 'query_processed.mode': 'RAG', 'query_processed.processing_time': 3.268723249435425, 'query_processed.response_length': 613}, 'events': [], 'links': [], 'trace_trace_id': '255fd49c-84b4-4b18-887e-6308a412d535', 'trace_name': 'auto_trace_tool-orchestator', 'project': 'noveum-ai-agent-rag-websearch', 'environment': 'development', 'trace_status': 'ok', 'trace_status_message': '', 'trace_start_time': '2025-10-09 08:25:43.830000000', 'trace_end_time': '2025-10-09 08:25:47.099000000', 'trace_duration_ms': 3268.862, 'span_count': 6, 'error_count': 0, 'sdk': {'name': 'noveum-trace-python', 'version': '0.3.8'}, 'trace_attributes': {'noveum.project': 'noveum-ai-agent-rag-websearch', 'noveum.environment': 'development', 'noveum.sdk.version': '0.3.8', 'noveum.sampling.decision': 'record'}, 'metadata': {'user_id': None, 'session_id': None, 'request_id': None, 'tags': {}, 'custom_attributes': {}}, 'created_at': '2025-10-09 08:25:50.304000000', 'updated_at': '2025-10-09 08:25:50.304000000'}\n", + "Spans with unknown type: 12\n", + "returning unknown type for span\n", + "{'span_id': 'b8055ffc-ee7d-46ad-b6a0-84e80f9399e2', 'trace_id': 'dc511122-c0b6-415c-9a49-c7b45132dd87', 'parent_span_id': '', 'name': 'tool-orchestator', 'start_time': '2025-10-09 08:25:38.322000000', 'end_time': '2025-10-09 08:25:42.822000000', 'duration_ms': 4500.335, 'status': 'ok', 'status_message': '', 'attributes': {'input_query': 'What monitoring capabilities does Noveum provide?', 'query.length': 49, 'processing.mode': 'RAG', 'processing.time_seconds': 4.5002899169921875, 'processing.response_length': 839, 'processing.sources_count': 5, 'output_response': 'Final Answer: Noveum provides comprehensive monitoring capabilities for AI applications, including the ability to monitor, trace, and optimize AI agents across various frameworks such as LangChain, CrewAI, AutoGen,...', 'final_answer_mode': 'RAG', 'query_processed.input_query': 'What monitoring capabilities does Noveum provide?', 'query_processed.output_response': 'Successfully processed query using RAG, generated 839 character response', 'query_processed.mode': 'RAG', 'query_processed.processing_time': 4.5002899169921875, 'query_processed.response_length': 839}, 'events': [], 'links': [], 'trace_trace_id': 'dc511122-c0b6-415c-9a49-c7b45132dd87', 'trace_name': 'auto_trace_tool-orchestator', 'project': 'noveum-ai-agent-rag-websearch', 'environment': 'development', 'trace_status': 'ok', 'trace_status_message': '', 'trace_start_time': '2025-10-09 08:25:38.322000000', 'trace_end_time': '2025-10-09 08:25:42.822000000', 'trace_duration_ms': 4500.425, 'span_count': 6, 'error_count': 0, 'sdk': {'name': 'noveum-trace-python', 'version': '0.3.8'}, 'trace_attributes': {'noveum.project': 'noveum-ai-agent-rag-websearch', 'noveum.environment': 'development', 'noveum.sdk.version': '0.3.8', 'noveum.sampling.decision': 'record'}, 'metadata': {'user_id': None, 'session_id': None, 'request_id': None, 'tags': {}, 'custom_attributes': {}}, 'created_at': '2025-10-09 08:25:45.028000000', 'updated_at': '2025-10-09 08:25:45.028000000'}\n", + "Spans with unknown type: 13\n", + "returning unknown type for span\n", + "{'span_id': '532b9e1f-5c4f-4b41-98bb-e9a1510f0f1c', 'trace_id': '04bebf38-a343-4563-80db-0154bef8d927', 'parent_span_id': '', 'name': 'tool-orchestator', 'start_time': '2025-10-09 08:25:33.693000000', 'end_time': '2025-10-09 08:25:37.313000000', 'duration_ms': 3619.5789999999997, 'status': 'ok', 'status_message': '', 'attributes': {'input_query': 'How does Noveum handle agent tracing?', 'query.length': 37, 'processing.mode': 'RAG', 'processing.time_seconds': 3.6195271015167236, 'processing.response_length': 691, 'processing.sources_count': 5, 'output_response': 'Final Answer: Noveum.ai handles agent tracing through its specialized Multi-Agent Tracing capabilities, which are designed to observe complex workflows and inter-agent communications within multi-agent systems. The...', 'final_answer_mode': 'RAG', 'query_processed.input_query': 'How does Noveum handle agent tracing?', 'query_processed.output_response': 'Successfully processed query using RAG, generated 691 character response', 'query_processed.mode': 'RAG', 'query_processed.processing_time': 3.6195271015167236, 'query_processed.response_length': 691}, 'events': [], 'links': [], 'trace_trace_id': '04bebf38-a343-4563-80db-0154bef8d927', 'trace_name': 'auto_trace_tool-orchestator', 'project': 'noveum-ai-agent-rag-websearch', 'environment': 'development', 'trace_status': 'ok', 'trace_status_message': '', 'trace_start_time': '2025-10-09 08:25:33.693000000', 'trace_end_time': '2025-10-09 08:25:37.313000000', 'trace_duration_ms': 3619.672, 'span_count': 6, 'error_count': 0, 'sdk': {'name': 'noveum-trace-python', 'version': '0.3.8'}, 'trace_attributes': {'noveum.project': 'noveum-ai-agent-rag-websearch', 'noveum.environment': 'development', 'noveum.sdk.version': '0.3.8', 'noveum.sampling.decision': 'record'}, 'metadata': {'user_id': None, 'session_id': None, 'request_id': None, 'tags': {}, 'custom_attributes': {}}, 'created_at': '2025-10-09 08:25:40.339000000', 'updated_at': '2025-10-09 08:25:40.339000000'}\n", + "Spans with unknown type: 14\n", + "returning unknown type for span\n", + "{'span_id': '14064e3f-f9c9-4bb7-b8cb-1ad52938b075', 'trace_id': '5e043630-6493-42b5-beb8-79faa19bfa37', 'parent_span_id': '', 'name': 'tool-orchestator', 'start_time': '2025-10-09 08:25:29.901000000', 'end_time': '2025-10-09 08:25:32.685000000', 'duration_ms': 2783.257, 'status': 'ok', 'status_message': '', 'attributes': {'input_query': 'What APIs are available in Noveum platform?', 'query.length': 43, 'processing.mode': 'RAG', 'processing.time_seconds': 2.783203125, 'processing.response_length': 270, 'processing.sources_count': 5, 'output_response': 'Final Answer: The provided context does not specify the APIs available in the Noveum platform. For detailed information about the APIs, I recommend checking the official Noveum documentation or the SDK Integration ...', 'final_answer_mode': 'RAG', 'query_processed.input_query': 'What APIs are available in Noveum platform?', 'query_processed.output_response': 'Successfully processed query using RAG, generated 270 character response', 'query_processed.mode': 'RAG', 'query_processed.processing_time': 2.783203125, 'query_processed.response_length': 270}, 'events': [], 'links': [], 'trace_trace_id': '5e043630-6493-42b5-beb8-79faa19bfa37', 'trace_name': 'auto_trace_tool-orchestator', 'project': 'noveum-ai-agent-rag-websearch', 'environment': 'development', 'trace_status': 'ok', 'trace_status_message': '', 'trace_start_time': '2025-10-09 08:25:29.901000000', 'trace_end_time': '2025-10-09 08:25:32.685000000', 'trace_duration_ms': 2783.326, 'span_count': 6, 'error_count': 0, 'sdk': {'name': 'noveum-trace-python', 'version': '0.3.8'}, 'trace_attributes': {'noveum.project': 'noveum-ai-agent-rag-websearch', 'noveum.environment': 'development', 'noveum.sdk.version': '0.3.8', 'noveum.sampling.decision': 'record'}, 'metadata': {'user_id': None, 'session_id': None, 'request_id': None, 'tags': {}, 'custom_attributes': {}}, 'created_at': '2025-10-09 08:25:34.918000000', 'updated_at': '2025-10-09 08:25:34.918000000'}\n", + "Spans with unknown type: 15\n", + "returning unknown type for span\n", + "{'span_id': 'a74fe8a9-2654-4b9d-ba87-dc2f86b6466d', 'trace_id': '7da9814d-a2e8-4c4e-b750-68b26bd5fd22', 'parent_span_id': '', 'name': 'tool-orchestator', 'start_time': '2025-10-09 08:25:20.586000000', 'end_time': '2025-10-09 08:25:28.897000000', 'duration_ms': 8310.113, 'status': 'ok', 'status_message': '', 'attributes': {'input_query': 'How do I set up observability with Noveum?', 'query.length': 42, 'processing.mode': 'RAG', 'processing.time_seconds': 8.31006908416748, 'processing.response_length': 951, 'processing.sources_count': 5, 'output_response': 'Final Answer: To set up observability with Noveum, follow these steps:\\n\\n1. **Sign Up**: Go to [noveum.ai](https://noveum.ai) and create an account.\\n2. **Create a Project**: Once signed in, create a new project to o...', 'final_answer_mode': 'RAG', 'query_processed.input_query': 'How do I set up observability with Noveum?', 'query_processed.output_response': 'Successfully processed query using RAG, generated 951 character response', 'query_processed.mode': 'RAG', 'query_processed.processing_time': 8.31006908416748, 'query_processed.response_length': 951}, 'events': [], 'links': [], 'trace_trace_id': '7da9814d-a2e8-4c4e-b750-68b26bd5fd22', 'trace_name': 'auto_trace_tool-orchestator', 'project': 'noveum-ai-agent-rag-websearch', 'environment': 'development', 'trace_status': 'ok', 'trace_status_message': '', 'trace_start_time': '2025-10-09 08:25:20.586000000', 'trace_end_time': '2025-10-09 08:25:28.897000000', 'trace_duration_ms': 8310.205, 'span_count': 6, 'error_count': 0, 'sdk': {'name': 'noveum-trace-python', 'version': '0.3.8'}, 'trace_attributes': {'noveum.project': 'noveum-ai-agent-rag-websearch', 'noveum.environment': 'development', 'noveum.sdk.version': '0.3.8', 'noveum.sampling.decision': 'record'}, 'metadata': {'user_id': None, 'session_id': None, 'request_id': None, 'tags': {}, 'custom_attributes': {}}, 'created_at': '2025-10-09 08:25:29.588000000', 'updated_at': '2025-10-09 08:25:29.588000000'}\n", + "Spans with unknown type: 16\n", + "returning unknown type for span\n", + "{'span_id': 'c700d0f4-f1f8-4339-aa42-a3060ae2ae11', 'trace_id': '16143f74-2831-4753-b33d-ce4b645093c5', 'parent_span_id': '', 'name': 'tool-orchestator', 'start_time': '2025-10-09 08:25:12.442000000', 'end_time': '2025-10-09 08:25:19.579000000', 'duration_ms': 7137.314, 'status': 'ok', 'status_message': '', 'attributes': {'input_query': 'What features does Noveum Trace offer?', 'query.length': 38, 'processing.mode': 'RAG', 'processing.time_seconds': 7.1372880935668945, 'processing.response_length': 1014, 'processing.sources_count': 5, 'output_response': 'Final Answer: Noveum Trace offers several key features through its Python SDK (`noveum-trace`) and TypeScript SDK (`@noveum/trace`):\\n\\n1. **Decorator-based Tracing**: It provides a simple way to trace LLM calls, age...', 'final_answer_mode': 'RAG', 'query_processed.input_query': 'What features does Noveum Trace offer?', 'query_processed.output_response': 'Successfully processed query using RAG, generated 1014 character response', 'query_processed.mode': 'RAG', 'query_processed.processing_time': 7.1372880935668945, 'query_processed.response_length': 1014}, 'events': [], 'links': [], 'trace_trace_id': '16143f74-2831-4753-b33d-ce4b645093c5', 'trace_name': 'auto_trace_tool-orchestator', 'project': 'noveum-ai-agent-rag-websearch', 'environment': 'development', 'trace_status': 'ok', 'trace_status_message': '', 'trace_start_time': '2025-10-09 08:25:12.442000000', 'trace_end_time': '2025-10-09 08:25:19.579000000', 'trace_duration_ms': 7137.402, 'span_count': 6, 'error_count': 0, 'sdk': {'name': 'noveum-trace-python', 'version': '0.3.8'}, 'trace_attributes': {'noveum.project': 'noveum-ai-agent-rag-websearch', 'noveum.environment': 'development', 'noveum.sdk.version': '0.3.8', 'noveum.sampling.decision': 'record'}, 'metadata': {'user_id': None, 'session_id': None, 'request_id': None, 'tags': {}, 'custom_attributes': {}}, 'created_at': '2025-10-09 08:25:20.290000000', 'updated_at': '2025-10-09 08:25:20.290000000'}\n", + "Spans with unknown type: 17\n", + "returning unknown type for span\n", + "{'span_id': 'f7f5981f-d840-4265-8970-d319b876366d', 'trace_id': 'fc64e6cc-6739-4256-ac4a-7b80c3028233', 'parent_span_id': '', 'name': 'tool-orchestator', 'start_time': '2025-10-09 08:25:07.932000000', 'end_time': '2025-10-09 08:25:11.438000000', 'duration_ms': 3506.0159999999996, 'status': 'ok', 'status_message': '', 'attributes': {'input_query': \"What are Noveum's pricing plans?\", 'query.length': 32, 'processing.mode': 'RAG', 'processing.time_seconds': 3.5059969425201416, 'processing.response_length': 399, 'processing.sources_count': 5, 'output_response': \"Final Answer: The provided context does not include specific information about Noveum's pricing plans. However, it mentions that all users will have access to the Observability suite for free, and early users will ...\", 'final_answer_mode': 'RAG', 'query_processed.input_query': \"What are Noveum's pricing plans?\", 'query_processed.output_response': 'Successfully processed query using RAG, generated 399 character response', 'query_processed.mode': 'RAG', 'query_processed.processing_time': 3.5059969425201416, 'query_processed.response_length': 399}, 'events': [], 'links': [], 'trace_trace_id': 'fc64e6cc-6739-4256-ac4a-7b80c3028233', 'trace_name': 'auto_trace_tool-orchestator', 'project': 'noveum-ai-agent-rag-websearch', 'environment': 'development', 'trace_status': 'ok', 'trace_status_message': '', 'trace_start_time': '2025-10-09 08:25:07.932000000', 'trace_end_time': '2025-10-09 08:25:11.438000000', 'trace_duration_ms': 3506.0989999999997, 'span_count': 6, 'error_count': 0, 'sdk': {'name': 'noveum-trace-python', 'version': '0.3.8'}, 'trace_attributes': {'noveum.project': 'noveum-ai-agent-rag-websearch', 'noveum.environment': 'development', 'noveum.sdk.version': '0.3.8', 'noveum.sampling.decision': 'record'}, 'metadata': {'user_id': None, 'session_id': None, 'request_id': None, 'tags': {}, 'custom_attributes': {}}, 'created_at': '2025-10-09 08:25:12.651000000', 'updated_at': '2025-10-09 08:25:12.651000000'}\n", + "Spans with unknown type: 18\n", + "returning unknown type for span\n", + "{'span_id': '0c7bbe2b-18af-4fdb-a5bf-4a91daa3bf50', 'trace_id': 'b7945c49-f584-4c70-972d-536a805d8a31', 'parent_span_id': '', 'name': 'tool-orchestator', 'start_time': '2025-10-09 08:24:58.518000000', 'end_time': '2025-10-09 08:25:06.920000000', 'duration_ms': 8402.378, 'status': 'ok', 'status_message': '', 'attributes': {'input_query': 'How do I integrate Noveum Trace in my application?', 'query.length': 50, 'processing.mode': 'RAG', 'processing.time_seconds': 8.402259826660156, 'processing.response_length': 1151, 'processing.sources_count': 5, 'output_response': 'Final Answer: To integrate Noveum Trace into your application, you can use either the Python SDK (`noveum-trace`) or the TypeScript SDK (`@noveum/trace`). Hereā€™s a quick start guide based on the provided documentat...', 'final_answer_mode': 'RAG', 'query_processed.input_query': 'How do I integrate Noveum Trace in my application?', 'query_processed.output_response': 'Successfully processed query using RAG, generated 1151 character response', 'query_processed.mode': 'RAG', 'query_processed.processing_time': 8.402259826660156, 'query_processed.response_length': 1151}, 'events': [], 'links': [], 'trace_trace_id': 'b7945c49-f584-4c70-972d-536a805d8a31', 'trace_name': 'auto_trace_tool-orchestator', 'project': 'noveum-ai-agent-rag-websearch', 'environment': 'development', 'trace_status': 'ok', 'trace_status_message': '', 'trace_start_time': '2025-10-09 08:24:58.518000000', 'trace_end_time': '2025-10-09 08:25:06.920000000', 'trace_duration_ms': 8402.516, 'span_count': 6, 'error_count': 0, 'sdk': {'name': 'noveum-trace-python', 'version': '0.3.8'}, 'trace_attributes': {'noveum.project': 'noveum-ai-agent-rag-websearch', 'noveum.environment': 'development', 'noveum.sdk.version': '0.3.8', 'noveum.sampling.decision': 'record'}, 'metadata': {'user_id': None, 'session_id': None, 'request_id': None, 'tags': {}, 'custom_attributes': {}}, 'created_at': '2025-10-09 08:25:07.620000000', 'updated_at': '2025-10-09 08:25:07.620000000'}\n", + "Spans with unknown type: 19\n", + "returning unknown type for span\n", + "{'span_id': 'dc869742-6797-4016-8061-37530fc7d433', 'trace_id': 'f5c40ecf-36c0-45ba-9cc9-dc0329b0324b', 'parent_span_id': '', 'name': 'tool-orchestator', 'start_time': '2025-10-09 08:24:54.009000000', 'end_time': '2025-10-09 08:24:57.511000000', 'duration_ms': 3502.0229999999997, 'status': 'ok', 'status_message': '', 'attributes': {'input_query': 'What is Noveum and what does it do?', 'query.length': 35, 'processing.mode': 'RAG', 'processing.time_seconds': 3.5019419193267822, 'processing.response_length': 611, 'processing.sources_count': 5, 'output_response': 'Final Answer: Noveum.ai is a comprehensive tracing and observability platform specifically designed for AI applications, including those powered by large language models (LLMs), retrieval-augmented generation (RAG)...', 'final_answer_mode': 'RAG', 'query_processed.input_query': 'What is Noveum and what does it do?', 'query_processed.output_response': 'Successfully processed query using RAG, generated 611 character response', 'query_processed.mode': 'RAG', 'query_processed.processing_time': 3.5019419193267822, 'query_processed.response_length': 611}, 'events': [], 'links': [], 'trace_trace_id': 'f5c40ecf-36c0-45ba-9cc9-dc0329b0324b', 'trace_name': 'auto_trace_tool-orchestator', 'project': 'noveum-ai-agent-rag-websearch', 'environment': 'development', 'trace_status': 'ok', 'trace_status_message': '', 'trace_start_time': '2025-10-09 08:24:54.009000000', 'trace_end_time': '2025-10-09 08:24:57.511000000', 'trace_duration_ms': 3502.062, 'span_count': 6, 'error_count': 0, 'sdk': {'name': 'noveum-trace-python', 'version': '0.3.8'}, 'trace_attributes': {'noveum.project': 'noveum-ai-agent-rag-websearch', 'noveum.environment': 'development', 'noveum.sdk.version': '0.3.8', 'noveum.sampling.decision': 'record'}, 'metadata': {'user_id': None, 'session_id': None, 'request_id': None, 'tags': {}, 'custom_attributes': {}}, 'created_at': '2025-10-09 08:24:59.716000000', 'updated_at': '2025-10-09 08:24:59.716000000'}\n", + "Spans with unknown type: 20\n", + "\n", + "āœ… Successfully converted 20 spans to AgentData\n", + "šŸ“Š AgentDataset created with 20 records\n", + "āœ… AgentDataset created: 20 records\n", + "\n", + "šŸ“‹ Step 4: Dataset Analysis\n", + "šŸ“ˆ Dataset Statistics:\n", + "\n", + "Agent Types: {'unknown': 20}\n", + "Records with responses: 0\n", + "Records with tool calls: 0\n", + "Records with retrieval: 0\n", + "Tool usage: {}\n", + "šŸ” Dataset Analysis:\n", + "\n", + "=== Agent Behavior Patterns ===\n", + "\n", + "šŸ“ˆ Tool Usage:\n", + "\n", + "šŸ“‹ Task Types:\n", + "\n", + "šŸ“‹ Step 5: Setting up Evaluation\n", + "āœ… GEMINI_API_KEY found in environment\n", + "2025-10-22 23:01:12 - INFO - novaeval.models.base - Noveum tracing initialized successfully\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "āœ… Gemini model initialized\n", + "āœ… Initialized 5 scoring functions:\n", + " - task_progression_scorer\n", + " - context_relevancy_scorer\n", + " - role_adherence_scorer\n", + " - tool_relevancy_scorer\n", + " - parameter_correctness_scorer\n", + "\n", + "āœ… AgentEvaluator created with Gemini model and scoring functions\n", + "āœ… Evaluation components ready!\n", + "\n", + "šŸ“‹ Step 6: Running Evaluation\n", + "šŸŽÆ Evaluating 25 samples...\n", + "šŸš€ Running evaluation on sample data...\n", + "\n", + "šŸ“Š Evaluating 0 sample records...\n", + "2025-10-22 23:01:12 - INFO - novaeval.evaluators.agent_evaluator - Starting agent evaluation process\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Evaluating samples: 0it [00:00, ?it/s]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:01:12 - INFO - novaeval.evaluators.agent_evaluator - Saving final results\n", + "2025-10-22 23:01:12 - INFO - novaeval.evaluators.agent_evaluator - Agent evaluation completed\n", + "\n", + "āœ… Evaluation completed!\n", + "āŒ Results file not found\n", + "āŒ Evaluation failed\n", + "\n", + "šŸ“‹ Step 7: Exporting Dataset\n", + "šŸ’¾ Exporting processed dataset...\n", + "āœ… Exported to ./processed_datasets/tool-orchestator_dataset_processed_dataset.json\n", + "āœ… Exported to ./processed_datasets/tool-orchestator_dataset_processed_dataset.csv\n", + "āœ… Export completed successfully!\n", + "\n", + "============================================================\n", + "šŸŽ‰ EVALUATION PIPELINE COMPLETED!\n", + "šŸ“Š Final Results:\n", + " - File processed: split_datasets/tool-orchestator_dataset.json\n", + " - Spans loaded: 20\n", + " - Dataset size: 20\n", + " - Evaluation completed: False\n", + " - Export successful: True\n", + " - Errors encountered: 1\n", + "Completed tool-orchestator_dataset.json\n", + "\n", + "Processing agent.web_search_generation_dataset.json...\n", + "šŸš€ Starting Complete Agent Evaluation Pipeline\n", + "šŸ“ Processing file: split_datasets/agent.web_search_generation_dataset.json\n", + "============================================================\n", + "\n", + "šŸ“‹ Step 1: Environment Setup\n", + "āœ… Logging configured at INFO level\n", + "šŸ” Environment validation:\n", + " āœ… gemini_api_key: True\n", + " āœ… pandas_available: True\n", + " āœ… novaeval_available: True\n", + "āœ… Environment ready for evaluation!\n", + "\n", + "šŸ“‹ Step 2: Loading Dataset\n", + "šŸ“Š Loaded 8 spans from split_datasets/agent.web_search_generation_dataset.json\n", + "\n", + "šŸ” Available span types:\n", + " - agent.web_search_generation: 8\n", + "āœ… Dataset loaded: 8 spans\n", + "\n", + "šŸ“‹ Step 3: Converting to AgentDataset Format\n", + "šŸ”„ Converting spans to AgentData objects...\n", + "agent_task not found\n", + "agent_response is not available 2aca614b-b016-4d56-abe2-43a144dc681d\n", + "agent_task not found\n", + "agent_response is not available 03e3ae4d-0107-401d-8ca0-05fc0f28fb24\n", + "agent_task not found\n", + "agent_response is not available dfdb9aaf-9925-4a5c-a0c6-2cb00f70d4d3\n", + "agent_task not found\n", + "agent_response is not available eefcfdfc-6433-459e-a134-e2647b8418e4\n", + "agent_task not found\n", + "agent_response is not available c57b4b54-b9b0-4a4c-b5cd-57a4cfe5301b\n", + "agent_task not found\n", + "agent_response is not available a062bef6-d973-489b-91df-5d40a2aef077\n", + "agent_task not found\n", + "agent_response is not available 41f82a95-620d-440c-84e6-36d6d1b9e079\n", + "agent_task not found\n", + "agent_response is not available 63f2a0d5-9016-4483-8ec0-e6aa6b27689f\n", + "\n", + "āœ… Successfully converted 8 spans to AgentData\n", + "šŸ“Š AgentDataset created with 8 records\n", + "āœ… AgentDataset created: 8 records\n", + "\n", + "šŸ“‹ Step 4: Dataset Analysis\n", + "šŸ“ˆ Dataset Statistics:\n", + "\n", + "Agent Types: {'agent': 8}\n", + "Records with responses: 0\n", + "Records with tool calls: 0\n", + "Records with retrieval: 0\n", + "Tool usage: {}\n", + "šŸ” Dataset Analysis:\n", + "\n", + "=== Agent Behavior Patterns ===\n", + "\n", + "šŸ“ˆ Tool Usage:\n", + "\n", + "šŸ“‹ Task Types:\n", + "\n", + "šŸ“‹ Step 5: Setting up Evaluation\n", + "āœ… GEMINI_API_KEY found in environment\n", + "2025-10-22 23:01:12 - INFO - novaeval.models.base - Noveum tracing initialized successfully\n", + "āœ… Gemini model initialized\n", + "āœ… Initialized 5 scoring functions:\n", + " - task_progression_scorer\n", + " - context_relevancy_scorer\n", + " - role_adherence_scorer\n", + " - tool_relevancy_scorer\n", + " - parameter_correctness_scorer\n", + "\n", + "āœ… AgentEvaluator created with Gemini model and scoring functions\n", + "āœ… Evaluation components ready!\n", + "\n", + "šŸ“‹ Step 6: Running Evaluation\n", + "šŸŽÆ Evaluating 25 samples...\n", + "šŸš€ Running evaluation on sample data...\n", + "\n", + "šŸ“Š Evaluating 0 sample records...\n", + "2025-10-22 23:01:12 - INFO - novaeval.evaluators.agent_evaluator - Starting agent evaluation process\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\n", + "Evaluating samples: 0it [00:00, ?it/s]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:01:12 - INFO - novaeval.evaluators.agent_evaluator - Saving final results\n", + "2025-10-22 23:01:12 - INFO - novaeval.evaluators.agent_evaluator - Agent evaluation completed\n", + "\n", + "āœ… Evaluation completed!\n", + "āŒ Results file not found\n", + "āŒ Evaluation failed\n", + "\n", + "šŸ“‹ Step 7: Exporting Dataset\n", + "šŸ’¾ Exporting processed dataset...\n", + "āœ… Exported to ./processed_datasets/agent.web_search_generation_dataset_processed_dataset.json\n", + "āœ… Exported to ./processed_datasets/agent.web_search_generation_dataset_processed_dataset.csv\n", + "āœ… Export completed successfully!\n", + "\n", + "============================================================\n", + "šŸŽ‰ EVALUATION PIPELINE COMPLETED!\n", + "šŸ“Š Final Results:\n", + " - File processed: split_datasets/agent.web_search_generation_dataset.json\n", + " - Spans loaded: 8\n", + " - Dataset size: 8\n", + " - Evaluation completed: False\n", + " - Export successful: True\n", + " - Errors encountered: 1\n", + "Completed agent.web_search_generation_dataset.json\n", + "\n", + "Processing agent.rag_evaluation_metrics_dataset.json...\n", + "šŸš€ Starting Complete Agent Evaluation Pipeline\n", + "šŸ“ Processing file: split_datasets/agent.rag_evaluation_metrics_dataset.json\n", + "============================================================\n", + "\n", + "šŸ“‹ Step 1: Environment Setup\n", + "āœ… Logging configured at INFO level\n", + "šŸ” Environment validation:\n", + " āœ… gemini_api_key: True\n", + " āœ… pandas_available: True\n", + " āœ… novaeval_available: True\n", + "āœ… Environment ready for evaluation!\n", + "\n", + "šŸ“‹ Step 2: Loading Dataset\n", + "šŸ“Š Loaded 12 spans from split_datasets/agent.rag_evaluation_metrics_dataset.json\n", + "\n", + "šŸ” Available span types:\n", + " - agent.rag_evaluation_metrics: 12\n", + "āœ… Dataset loaded: 12 spans\n", + "\n", + "šŸ“‹ Step 3: Converting to AgentDataset Format\n", + "šŸ”„ Converting spans to AgentData objects...\n", + "\n", + "āœ… Successfully converted 12 spans to AgentData\n", + "šŸ“Š AgentDataset created with 12 records\n", + "āœ… AgentDataset created: 12 records\n", + "\n", + "šŸ“‹ Step 4: Dataset Analysis\n", + "šŸ“ˆ Dataset Statistics:\n", + "\n", + "Agent Types: {'agent': 12}\n", + "Records with responses: 12\n", + "Records with tool calls: 0\n", + "Records with retrieval: 12\n", + "Tool usage: {}\n", + "šŸ” Dataset Analysis:\n", + "\n", + "=== Agent Behavior Patterns ===\n", + "\n", + "šŸ“ˆ Tool Usage:\n", + "\n", + "šŸ“‹ Task Types:\n", + " - other: 12\n", + "\n", + "šŸ“ Response Statistics:\n", + " - Average response length: 215.0 characters\n", + " - Min response length: 215\n", + " - Max response length: 215\n", + "\n", + "šŸ“‹ Step 5: Setting up Evaluation\n", + "āœ… GEMINI_API_KEY found in environment\n", + "2025-10-22 23:01:12 - INFO - novaeval.models.base - Noveum tracing initialized successfully\n", + "āœ… Gemini model initialized\n", + "āœ… Initialized 5 scoring functions:\n", + " - task_progression_scorer\n", + " - context_relevancy_scorer\n", + " - role_adherence_scorer\n", + " - tool_relevancy_scorer\n", + " - parameter_correctness_scorer\n", + "\n", + "āœ… AgentEvaluator created with Gemini model and scoring functions\n", + "āœ… Evaluation components ready!\n", + "\n", + "šŸ“‹ Step 6: Running Evaluation\n", + "šŸŽÆ Evaluating 25 samples...\n", + "šŸš€ Running evaluation on sample data...\n", + "\n", + "šŸ“Š Evaluating 12 sample records...\n", + "2025-10-22 23:01:12 - INFO - novaeval.evaluators.agent_evaluator - Starting agent evaluation process\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\n", + "Evaluating samples: 0it [00:00, ?it/s]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:01:12 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-10-22 23:01:12 - noveum_trace.transport.http_transport - INFO - šŸ“” HTTP RESPONSE: Status 200 from https://api.noveum.ai/api/v1/traces\n", + "2025-10-22 23:01:12 - noveum_trace.transport.http_transport - INFO - āœ… Successfully sent batch of 4 traces\n", + "2025-10-22 23:01:12 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully sent batch of 4 traces via callback\n", + "2025-10-22 23:01:13 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: eb775ca5-3cc0-47f9-b26b-0e3e770dadd5) - 1 spans\n", + "2025-10-22 23:01:13 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: eb775ca5-3cc0-47f9-b26b-0e3e770dadd5) - 1 spans\n", + "2025-10-22 23:01:13 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace eb775ca5-3cc0-47f9-b26b-0e3e770dadd5\n", + "2025-10-22 23:01:13 - noveum_trace.transport.http_transport - INFO - āœ… Trace eb775ca5-3cc0-47f9-b26b-0e3e770dadd5 successfully queued for export\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:01:13 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-10-22 23:01:16 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: cce316e9-4f3f-4d2b-bdba-b2856b9c4181) - 1 spans\n", + "2025-10-22 23:01:16 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: cce316e9-4f3f-4d2b-bdba-b2856b9c4181) - 1 spans\n", + "2025-10-22 23:01:16 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace cce316e9-4f3f-4d2b-bdba-b2856b9c4181\n", + "2025-10-22 23:01:16 - noveum_trace.transport.http_transport - INFO - āœ… Trace cce316e9-4f3f-4d2b-bdba-b2856b9c4181 successfully queued for export\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:01:16 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-10-22 23:01:17 - noveum_trace.transport.batch_processor - INFO - ā° TIMEOUT TRIGGER: Sending batch due to timeout (5.3s >= 5.0s)\n", + "2025-10-22 23:01:17 - noveum_trace.transport.batch_processor - INFO - šŸ“¤ SENDING BATCH: 3 traces via send_callback\n", + "2025-10-22 23:01:17 - noveum_trace.transport.http_transport - INFO - šŸš€ SENDING BATCH: 3 traces to https://api.noveum.ai/api/v1/traces\n", + "2025-10-22 23:01:17 - noveum_trace.transport.http_transport - INFO - šŸ“” HTTP RESPONSE: Status 200 from https://api.noveum.ai/api/v1/traces\n", + "2025-10-22 23:01:17 - noveum_trace.transport.http_transport - INFO - āœ… Successfully sent batch of 3 traces\n", + "2025-10-22 23:01:17 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully sent batch of 3 traces via callback\n", + "2025-10-22 23:01:18 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: ef7447dd-1534-4496-9aff-02d51839cd0c) - 1 spans\n", + "2025-10-22 23:01:18 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: ef7447dd-1534-4496-9aff-02d51839cd0c) - 1 spans\n", + "2025-10-22 23:01:18 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace ef7447dd-1534-4496-9aff-02d51839cd0c\n", + "2025-10-22 23:01:18 - noveum_trace.transport.http_transport - INFO - āœ… Trace ef7447dd-1534-4496-9aff-02d51839cd0c successfully queued for export\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:01:18 - INFO - novaeval.evaluators.agent_evaluator - Saving intermediate results after 1 samples\n", + "2025-10-22 23:01:18 - INFO - novaeval.evaluators.agent_evaluator - Intermediate results saved to demo_results/agent.rag_evaluation_metrics_dataset/agent_evaluation_results.csv\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Evaluating samples: 1it [00:06, 6.46s/it]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:01:18 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-10-22 23:01:20 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: d32ecc4b-d331-4e62-9ef2-ec61e6fc52bf) - 1 spans\n", + "2025-10-22 23:01:20 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: d32ecc4b-d331-4e62-9ef2-ec61e6fc52bf) - 1 spans\n", + "2025-10-22 23:01:20 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace d32ecc4b-d331-4e62-9ef2-ec61e6fc52bf\n", + "2025-10-22 23:01:20 - noveum_trace.transport.http_transport - INFO - āœ… Trace d32ecc4b-d331-4e62-9ef2-ec61e6fc52bf successfully queued for export\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:01:20 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-10-22 23:01:22 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 37646e1d-3bea-43ae-af16-71bf5ba46b9c) - 1 spans\n", + "2025-10-22 23:01:22 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 37646e1d-3bea-43ae-af16-71bf5ba46b9c) - 1 spans\n", + "2025-10-22 23:01:22 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 37646e1d-3bea-43ae-af16-71bf5ba46b9c\n", + "2025-10-22 23:01:22 - noveum_trace.transport.http_transport - INFO - āœ… Trace 37646e1d-3bea-43ae-af16-71bf5ba46b9c successfully queued for export\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:01:22 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-10-22 23:01:22 - noveum_trace.transport.batch_processor - INFO - ā° TIMEOUT TRIGGER: Sending batch due to timeout (5.2s >= 5.0s)\n", + "2025-10-22 23:01:22 - noveum_trace.transport.batch_processor - INFO - šŸ“¤ SENDING BATCH: 3 traces via send_callback\n", + "2025-10-22 23:01:22 - noveum_trace.transport.http_transport - INFO - šŸš€ SENDING BATCH: 3 traces to https://api.noveum.ai/api/v1/traces\n", + "2025-10-22 23:01:22 - noveum_trace.transport.http_transport - INFO - šŸ“” HTTP RESPONSE: Status 200 from https://api.noveum.ai/api/v1/traces\n", + "2025-10-22 23:01:22 - noveum_trace.transport.http_transport - INFO - āœ… Successfully sent batch of 3 traces\n", + "2025-10-22 23:01:22 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully sent batch of 3 traces via callback\n", + "2025-10-22 23:01:23 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 309e9e0e-8ee6-46e4-86c2-60922d034adb) - 1 spans\n", + "2025-10-22 23:01:23 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 309e9e0e-8ee6-46e4-86c2-60922d034adb) - 1 spans\n", + "2025-10-22 23:01:23 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 309e9e0e-8ee6-46e4-86c2-60922d034adb\n", + "2025-10-22 23:01:23 - noveum_trace.transport.http_transport - INFO - āœ… Trace 309e9e0e-8ee6-46e4-86c2-60922d034adb successfully queued for export\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:01:23 - INFO - novaeval.evaluators.agent_evaluator - Saving intermediate results after 2 samples\n", + "2025-10-22 23:01:23 - INFO - novaeval.evaluators.agent_evaluator - Intermediate results saved to demo_results/agent.rag_evaluation_metrics_dataset/agent_evaluation_results.csv\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Evaluating samples: 2it [00:10, 5.24s/it]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:01:23 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-10-22 23:01:24 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 7876eb8f-cae3-442a-b676-588d0e06a56b) - 1 spans\n", + "2025-10-22 23:01:24 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 7876eb8f-cae3-442a-b676-588d0e06a56b) - 1 spans\n", + "2025-10-22 23:01:24 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 7876eb8f-cae3-442a-b676-588d0e06a56b\n", + "2025-10-22 23:01:24 - noveum_trace.transport.http_transport - INFO - āœ… Trace 7876eb8f-cae3-442a-b676-588d0e06a56b successfully queued for export\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:01:24 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-10-22 23:01:26 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: a1a2f4ae-eaf9-4742-94d2-b2b650faf225) - 1 spans\n", + "2025-10-22 23:01:26 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: a1a2f4ae-eaf9-4742-94d2-b2b650faf225) - 1 spans\n", + "2025-10-22 23:01:26 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace a1a2f4ae-eaf9-4742-94d2-b2b650faf225\n", + "2025-10-22 23:01:26 - noveum_trace.transport.http_transport - INFO - āœ… Trace a1a2f4ae-eaf9-4742-94d2-b2b650faf225 successfully queued for export\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:01:26 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-10-22 23:01:27 - noveum_trace.transport.batch_processor - INFO - ā° TIMEOUT TRIGGER: Sending batch due to timeout (5.1s >= 5.0s)\n", + "2025-10-22 23:01:27 - noveum_trace.transport.batch_processor - INFO - šŸ“¤ SENDING BATCH: 3 traces via send_callback\n", + "2025-10-22 23:01:27 - noveum_trace.transport.http_transport - INFO - šŸš€ SENDING BATCH: 3 traces to https://api.noveum.ai/api/v1/traces\n", + "2025-10-22 23:01:27 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 7fd55fac-bb86-4db1-9e20-e5ce69a772fb) - 1 spans\n", + "2025-10-22 23:01:27 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 7fd55fac-bb86-4db1-9e20-e5ce69a772fb) - 1 spans\n", + "2025-10-22 23:01:27 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 7fd55fac-bb86-4db1-9e20-e5ce69a772fb\n", + "2025-10-22 23:01:27 - noveum_trace.transport.http_transport - INFO - āœ… Trace 7fd55fac-bb86-4db1-9e20-e5ce69a772fb successfully queued for export\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:01:27 - INFO - novaeval.evaluators.agent_evaluator - Saving intermediate results after 3 samples\n", + "2025-10-22 23:01:27 - INFO - novaeval.evaluators.agent_evaluator - Intermediate results saved to demo_results/agent.rag_evaluation_metrics_dataset/agent_evaluation_results.csv\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Evaluating samples: 3it [00:15, 4.93s/it]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:01:27 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-10-22 23:01:28 - noveum_trace.transport.http_transport - INFO - šŸ“” HTTP RESPONSE: Status 200 from https://api.noveum.ai/api/v1/traces\n", + "2025-10-22 23:01:28 - noveum_trace.transport.http_transport - INFO - āœ… Successfully sent batch of 3 traces\n", + "2025-10-22 23:01:28 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully sent batch of 3 traces via callback\n", + "2025-10-22 23:01:28 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 3e3b5c0c-04e1-4bf5-8ea2-c70387c8e327) - 1 spans\n", + "2025-10-22 23:01:28 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 3e3b5c0c-04e1-4bf5-8ea2-c70387c8e327) - 1 spans\n", + "2025-10-22 23:01:28 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 3e3b5c0c-04e1-4bf5-8ea2-c70387c8e327\n", + "2025-10-22 23:01:28 - noveum_trace.transport.http_transport - INFO - āœ… Trace 3e3b5c0c-04e1-4bf5-8ea2-c70387c8e327 successfully queued for export\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:01:28 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-10-22 23:01:30 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 3e9b223d-42c3-4f70-8541-e8b7c27201fc) - 1 spans\n", + "2025-10-22 23:01:30 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 3e9b223d-42c3-4f70-8541-e8b7c27201fc) - 1 spans\n", + "2025-10-22 23:01:30 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 3e9b223d-42c3-4f70-8541-e8b7c27201fc\n", + "2025-10-22 23:01:30 - noveum_trace.transport.http_transport - INFO - āœ… Trace 3e9b223d-42c3-4f70-8541-e8b7c27201fc successfully queued for export\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:01:30 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-10-22 23:01:31 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: f822ee4c-33fa-4a46-8a1c-a9fd7a749983) - 1 spans\n", + "2025-10-22 23:01:31 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: f822ee4c-33fa-4a46-8a1c-a9fd7a749983) - 1 spans\n", + "2025-10-22 23:01:31 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace f822ee4c-33fa-4a46-8a1c-a9fd7a749983\n", + "2025-10-22 23:01:31 - noveum_trace.transport.http_transport - INFO - āœ… Trace f822ee4c-33fa-4a46-8a1c-a9fd7a749983 successfully queued for export\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:01:31 - INFO - novaeval.evaluators.agent_evaluator - Saving intermediate results after 4 samples\n", + "2025-10-22 23:01:31 - INFO - novaeval.evaluators.agent_evaluator - Intermediate results saved to demo_results/agent.rag_evaluation_metrics_dataset/agent_evaluation_results.csv\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Evaluating samples: 4it [00:18, 4.36s/it]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:01:31 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-10-22 23:01:32 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: b9ec0566-acc2-4c49-8fa4-fa58362b7056) - 1 spans\n", + "2025-10-22 23:01:32 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: b9ec0566-acc2-4c49-8fa4-fa58362b7056) - 1 spans\n", + "2025-10-22 23:01:32 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace b9ec0566-acc2-4c49-8fa4-fa58362b7056\n", + "2025-10-22 23:01:32 - noveum_trace.transport.http_transport - INFO - āœ… Trace b9ec0566-acc2-4c49-8fa4-fa58362b7056 successfully queued for export\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:01:32 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-10-22 23:01:32 - noveum_trace.transport.batch_processor - INFO - ā° TIMEOUT TRIGGER: Sending batch due to timeout (5.3s >= 5.0s)\n", + "2025-10-22 23:01:32 - noveum_trace.transport.batch_processor - INFO - šŸ“¤ SENDING BATCH: 5 traces via send_callback\n", + "2025-10-22 23:01:32 - noveum_trace.transport.http_transport - INFO - šŸš€ SENDING BATCH: 5 traces to https://api.noveum.ai/api/v1/traces\n", + "2025-10-22 23:01:33 - noveum_trace.transport.http_transport - INFO - šŸ“” HTTP RESPONSE: Status 200 from https://api.noveum.ai/api/v1/traces\n", + "2025-10-22 23:01:33 - noveum_trace.transport.http_transport - INFO - āœ… Successfully sent batch of 5 traces\n", + "2025-10-22 23:01:33 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully sent batch of 5 traces via callback\n", + "2025-10-22 23:01:34 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 4a7712cd-d5db-43d4-b9f9-3bf2854fd36d) - 1 spans\n", + "2025-10-22 23:01:34 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 4a7712cd-d5db-43d4-b9f9-3bf2854fd36d) - 1 spans\n", + "2025-10-22 23:01:34 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 4a7712cd-d5db-43d4-b9f9-3bf2854fd36d\n", + "2025-10-22 23:01:34 - noveum_trace.transport.http_transport - INFO - āœ… Trace 4a7712cd-d5db-43d4-b9f9-3bf2854fd36d successfully queued for export\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:01:34 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-10-22 23:01:35 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: d3a4f8d6-9ac8-4312-982f-ed0ba66a9792) - 1 spans\n", + "2025-10-22 23:01:35 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: d3a4f8d6-9ac8-4312-982f-ed0ba66a9792) - 1 spans\n", + "2025-10-22 23:01:35 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace d3a4f8d6-9ac8-4312-982f-ed0ba66a9792\n", + "2025-10-22 23:01:35 - noveum_trace.transport.http_transport - INFO - āœ… Trace d3a4f8d6-9ac8-4312-982f-ed0ba66a9792 successfully queued for export\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:01:35 - INFO - novaeval.evaluators.agent_evaluator - Saving intermediate results after 5 samples\n", + "2025-10-22 23:01:35 - INFO - novaeval.evaluators.agent_evaluator - Intermediate results saved to demo_results/agent.rag_evaluation_metrics_dataset/agent_evaluation_results.csv\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Evaluating samples: 5it [00:22, 4.25s/it]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:01:35 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-10-22 23:01:36 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 089525d7-5531-478e-b4f3-fec4d1ac533d) - 1 spans\n", + "2025-10-22 23:01:36 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 089525d7-5531-478e-b4f3-fec4d1ac533d) - 1 spans\n", + "2025-10-22 23:01:36 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 089525d7-5531-478e-b4f3-fec4d1ac533d\n", + "2025-10-22 23:01:36 - noveum_trace.transport.http_transport - INFO - āœ… Trace 089525d7-5531-478e-b4f3-fec4d1ac533d successfully queued for export\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:01:36 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-10-22 23:01:37 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 2214e8fb-a31c-4111-8735-0c6c8057a66c) - 1 spans\n", + "2025-10-22 23:01:37 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 2214e8fb-a31c-4111-8735-0c6c8057a66c) - 1 spans\n", + "2025-10-22 23:01:37 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 2214e8fb-a31c-4111-8735-0c6c8057a66c\n", + "2025-10-22 23:01:37 - noveum_trace.transport.http_transport - INFO - āœ… Trace 2214e8fb-a31c-4111-8735-0c6c8057a66c successfully queued for export\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:01:37 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-10-22 23:01:38 - noveum_trace.transport.batch_processor - INFO - ā° TIMEOUT TRIGGER: Sending batch due to timeout (5.3s >= 5.0s)\n", + "2025-10-22 23:01:38 - noveum_trace.transport.batch_processor - INFO - šŸ“¤ SENDING BATCH: 4 traces via send_callback\n", + "2025-10-22 23:01:38 - noveum_trace.transport.http_transport - INFO - šŸš€ SENDING BATCH: 4 traces to https://api.noveum.ai/api/v1/traces\n", + "2025-10-22 23:01:38 - noveum_trace.transport.http_transport - INFO - šŸ“” HTTP RESPONSE: Status 200 from https://api.noveum.ai/api/v1/traces\n", + "2025-10-22 23:01:38 - noveum_trace.transport.http_transport - INFO - āœ… Successfully sent batch of 4 traces\n", + "2025-10-22 23:01:38 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully sent batch of 4 traces via callback\n", + "2025-10-22 23:01:38 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 4d917e77-4f61-46e5-8aed-f705e23cf42b) - 1 spans\n", + "2025-10-22 23:01:38 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 4d917e77-4f61-46e5-8aed-f705e23cf42b) - 1 spans\n", + "2025-10-22 23:01:38 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 4d917e77-4f61-46e5-8aed-f705e23cf42b\n", + "2025-10-22 23:01:38 - noveum_trace.transport.http_transport - INFO - āœ… Trace 4d917e77-4f61-46e5-8aed-f705e23cf42b successfully queued for export\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:01:38 - INFO - novaeval.evaluators.agent_evaluator - Saving intermediate results after 6 samples\n", + "2025-10-22 23:01:38 - INFO - novaeval.evaluators.agent_evaluator - Intermediate results saved to demo_results/agent.rag_evaluation_metrics_dataset/agent_evaluation_results.csv\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Evaluating samples: 6it [00:26, 4.03s/it]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:01:38 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-10-22 23:01:40 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: f08a5cf5-f699-499a-bf6f-7c5ab49bee74) - 1 spans\n", + "2025-10-22 23:01:40 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: f08a5cf5-f699-499a-bf6f-7c5ab49bee74) - 1 spans\n", + "2025-10-22 23:01:40 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace f08a5cf5-f699-499a-bf6f-7c5ab49bee74\n", + "2025-10-22 23:01:40 - noveum_trace.transport.http_transport - INFO - āœ… Trace f08a5cf5-f699-499a-bf6f-7c5ab49bee74 successfully queued for export\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:01:40 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-10-22 23:01:41 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 22fad5c7-5c05-46dc-be41-410da97ae840) - 1 spans\n", + "2025-10-22 23:01:41 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 22fad5c7-5c05-46dc-be41-410da97ae840) - 1 spans\n", + "2025-10-22 23:01:41 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 22fad5c7-5c05-46dc-be41-410da97ae840\n", + "2025-10-22 23:01:41 - noveum_trace.transport.http_transport - INFO - āœ… Trace 22fad5c7-5c05-46dc-be41-410da97ae840 successfully queued for export\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:01:41 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-10-22 23:01:42 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 81742ed5-9524-4203-aaf6-837045f5d1ce) - 1 spans\n", + "2025-10-22 23:01:42 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 81742ed5-9524-4203-aaf6-837045f5d1ce) - 1 spans\n", + "2025-10-22 23:01:42 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 81742ed5-9524-4203-aaf6-837045f5d1ce\n", + "2025-10-22 23:01:42 - noveum_trace.transport.http_transport - INFO - āœ… Trace 81742ed5-9524-4203-aaf6-837045f5d1ce successfully queued for export\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:01:42 - INFO - novaeval.evaluators.agent_evaluator - Saving intermediate results after 7 samples\n", + "2025-10-22 23:01:42 - INFO - novaeval.evaluators.agent_evaluator - Intermediate results saved to demo_results/agent.rag_evaluation_metrics_dataset/agent_evaluation_results.csv\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Evaluating samples: 7it [00:30, 3.93s/it]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:01:42 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-10-22 23:01:43 - noveum_trace.transport.batch_processor - INFO - ā° TIMEOUT TRIGGER: Sending batch due to timeout (5.4s >= 5.0s)\n", + "2025-10-22 23:01:43 - noveum_trace.transport.batch_processor - INFO - šŸ“¤ SENDING BATCH: 4 traces via send_callback\n", + "2025-10-22 23:01:43 - noveum_trace.transport.http_transport - INFO - šŸš€ SENDING BATCH: 4 traces to https://api.noveum.ai/api/v1/traces\n", + "2025-10-22 23:01:43 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 8dd29a67-8d57-4f95-b0bd-9c1fa22f55da) - 1 spans\n", + "2025-10-22 23:01:43 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 8dd29a67-8d57-4f95-b0bd-9c1fa22f55da) - 1 spans\n", + "2025-10-22 23:01:43 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 8dd29a67-8d57-4f95-b0bd-9c1fa22f55da\n", + "2025-10-22 23:01:43 - noveum_trace.transport.http_transport - INFO - āœ… Trace 8dd29a67-8d57-4f95-b0bd-9c1fa22f55da successfully queued for export\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:01:43 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-10-22 23:01:44 - noveum_trace.transport.http_transport - INFO - šŸ“” HTTP RESPONSE: Status 200 from https://api.noveum.ai/api/v1/traces\n", + "2025-10-22 23:01:44 - noveum_trace.transport.http_transport - INFO - āœ… Successfully sent batch of 4 traces\n", + "2025-10-22 23:01:44 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully sent batch of 4 traces via callback\n", + "2025-10-22 23:01:45 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: ae9c03e4-9a18-4c40-a616-34ba6df7216e) - 1 spans\n", + "2025-10-22 23:01:45 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: ae9c03e4-9a18-4c40-a616-34ba6df7216e) - 1 spans\n", + "2025-10-22 23:01:45 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace ae9c03e4-9a18-4c40-a616-34ba6df7216e\n", + "2025-10-22 23:01:45 - noveum_trace.transport.http_transport - INFO - āœ… Trace ae9c03e4-9a18-4c40-a616-34ba6df7216e successfully queued for export\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:01:45 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-10-22 23:01:46 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: aefaba52-0c8a-428c-af02-72f1145c5c99) - 1 spans\n", + "2025-10-22 23:01:46 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: aefaba52-0c8a-428c-af02-72f1145c5c99) - 1 spans\n", + "2025-10-22 23:01:46 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace aefaba52-0c8a-428c-af02-72f1145c5c99\n", + "2025-10-22 23:01:46 - noveum_trace.transport.http_transport - INFO - āœ… Trace aefaba52-0c8a-428c-af02-72f1145c5c99 successfully queued for export\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:01:46 - INFO - novaeval.evaluators.agent_evaluator - Saving intermediate results after 8 samples\n", + "2025-10-22 23:01:46 - INFO - novaeval.evaluators.agent_evaluator - Intermediate results saved to demo_results/agent.rag_evaluation_metrics_dataset/agent_evaluation_results.csv\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Evaluating samples: 8it [00:34, 3.87s/it]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:01:46 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-10-22 23:01:47 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 6d0b6d2c-d570-40cb-8d9d-c0a1187d7ec5) - 1 spans\n", + "2025-10-22 23:01:47 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 6d0b6d2c-d570-40cb-8d9d-c0a1187d7ec5) - 1 spans\n", + "2025-10-22 23:01:47 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 6d0b6d2c-d570-40cb-8d9d-c0a1187d7ec5\n", + "2025-10-22 23:01:47 - noveum_trace.transport.http_transport - INFO - āœ… Trace 6d0b6d2c-d570-40cb-8d9d-c0a1187d7ec5 successfully queued for export\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:01:47 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-10-22 23:01:49 - noveum_trace.transport.batch_processor - INFO - ā° TIMEOUT TRIGGER: Sending batch due to timeout (5.4s >= 5.0s)\n", + "2025-10-22 23:01:49 - noveum_trace.transport.batch_processor - INFO - šŸ“¤ SENDING BATCH: 4 traces via send_callback\n", + "2025-10-22 23:01:49 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: af1cb192-c53d-4c68-bcf1-63ae80a319ab) - 1 spans\n", + "2025-10-22 23:01:49 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: af1cb192-c53d-4c68-bcf1-63ae80a319ab) - 1 spans\n", + "2025-10-22 23:01:49 - noveum_trace.transport.http_transport - INFO - šŸš€ SENDING BATCH: 4 traces to https://api.noveum.ai/api/v1/traces\n", + "2025-10-22 23:01:49 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace af1cb192-c53d-4c68-bcf1-63ae80a319ab\n", + "2025-10-22 23:01:49 - noveum_trace.transport.http_transport - INFO - āœ… Trace af1cb192-c53d-4c68-bcf1-63ae80a319ab successfully queued for export\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:01:49 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-10-22 23:01:49 - noveum_trace.transport.http_transport - INFO - šŸ“” HTTP RESPONSE: Status 200 from https://api.noveum.ai/api/v1/traces\n", + "2025-10-22 23:01:49 - noveum_trace.transport.http_transport - INFO - āœ… Successfully sent batch of 4 traces\n", + "2025-10-22 23:01:49 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully sent batch of 4 traces via callback\n", + "2025-10-22 23:01:50 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 6dc7d782-6f29-481b-9738-d22d793c9d53) - 1 spans\n", + "2025-10-22 23:01:50 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 6dc7d782-6f29-481b-9738-d22d793c9d53) - 1 spans\n", + "2025-10-22 23:01:50 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 6dc7d782-6f29-481b-9738-d22d793c9d53\n", + "2025-10-22 23:01:50 - noveum_trace.transport.http_transport - INFO - āœ… Trace 6dc7d782-6f29-481b-9738-d22d793c9d53 successfully queued for export\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:01:50 - INFO - novaeval.evaluators.agent_evaluator - Saving intermediate results after 9 samples\n", + "2025-10-22 23:01:50 - INFO - novaeval.evaluators.agent_evaluator - Intermediate results saved to demo_results/agent.rag_evaluation_metrics_dataset/agent_evaluation_results.csv\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Evaluating samples: 9it [00:38, 3.94s/it]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:01:50 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-10-22 23:01:51 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 79ae16c8-3b0e-4e3c-bc10-9eae81586163) - 1 spans\n", + "2025-10-22 23:01:51 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 79ae16c8-3b0e-4e3c-bc10-9eae81586163) - 1 spans\n", + "2025-10-22 23:01:51 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 79ae16c8-3b0e-4e3c-bc10-9eae81586163\n", + "2025-10-22 23:01:51 - noveum_trace.transport.http_transport - INFO - āœ… Trace 79ae16c8-3b0e-4e3c-bc10-9eae81586163 successfully queued for export\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:01:51 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-10-22 23:01:52 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: e69630b6-5e34-4ae7-9195-929d04ef31a5) - 1 spans\n", + "2025-10-22 23:01:52 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: e69630b6-5e34-4ae7-9195-929d04ef31a5) - 1 spans\n", + "2025-10-22 23:01:52 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace e69630b6-5e34-4ae7-9195-929d04ef31a5\n", + "2025-10-22 23:01:52 - noveum_trace.transport.http_transport - INFO - āœ… Trace e69630b6-5e34-4ae7-9195-929d04ef31a5 successfully queued for export\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:01:52 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-10-22 23:01:53 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: ef939d49-86d5-4ab4-b8a3-918e10f86639) - 1 spans\n", + "2025-10-22 23:01:53 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: ef939d49-86d5-4ab4-b8a3-918e10f86639) - 1 spans\n", + "2025-10-22 23:01:53 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace ef939d49-86d5-4ab4-b8a3-918e10f86639\n", + "2025-10-22 23:01:53 - noveum_trace.transport.http_transport - INFO - āœ… Trace ef939d49-86d5-4ab4-b8a3-918e10f86639 successfully queued for export\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:01:53 - INFO - novaeval.evaluators.agent_evaluator - Saving intermediate results after 10 samples\n", + "2025-10-22 23:01:53 - INFO - novaeval.evaluators.agent_evaluator - Intermediate results saved to demo_results/agent.rag_evaluation_metrics_dataset/agent_evaluation_results.csv\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Evaluating samples: 10it [00:41, 3.77s/it]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:01:53 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-10-22 23:01:54 - noveum_trace.transport.batch_processor - INFO - ā° TIMEOUT TRIGGER: Sending batch due to timeout (5.4s >= 5.0s)\n", + "2025-10-22 23:01:54 - noveum_trace.transport.batch_processor - INFO - šŸ“¤ SENDING BATCH: 5 traces via send_callback\n", + "2025-10-22 23:01:54 - noveum_trace.transport.http_transport - INFO - šŸš€ SENDING BATCH: 5 traces to https://api.noveum.ai/api/v1/traces\n", + "2025-10-22 23:01:54 - noveum_trace.transport.http_transport - INFO - šŸ“” HTTP RESPONSE: Status 200 from https://api.noveum.ai/api/v1/traces\n", + "2025-10-22 23:01:54 - noveum_trace.transport.http_transport - INFO - āœ… Successfully sent batch of 5 traces\n", + "2025-10-22 23:01:54 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully sent batch of 5 traces via callback\n", + "2025-10-22 23:01:57 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 926cf84f-fcb5-4e44-b36b-dfa5f9c28c55) - 1 spans\n", + "2025-10-22 23:01:57 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 926cf84f-fcb5-4e44-b36b-dfa5f9c28c55) - 1 spans\n", + "2025-10-22 23:01:57 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 926cf84f-fcb5-4e44-b36b-dfa5f9c28c55\n", + "2025-10-22 23:01:57 - noveum_trace.transport.http_transport - INFO - āœ… Trace 926cf84f-fcb5-4e44-b36b-dfa5f9c28c55 successfully queued for export\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:01:57 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-10-22 23:01:58 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 5bb8255c-0f41-4735-a818-52f6115b7de1) - 1 spans\n", + "2025-10-22 23:01:58 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 5bb8255c-0f41-4735-a818-52f6115b7de1) - 1 spans\n", + "2025-10-22 23:01:58 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 5bb8255c-0f41-4735-a818-52f6115b7de1\n", + "2025-10-22 23:01:58 - noveum_trace.transport.http_transport - INFO - āœ… Trace 5bb8255c-0f41-4735-a818-52f6115b7de1 successfully queued for export\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:01:58 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-10-22 23:01:59 - noveum_trace.transport.batch_processor - INFO - ā° TIMEOUT TRIGGER: Sending batch due to timeout (5.5s >= 5.0s)\n", + "2025-10-22 23:01:59 - noveum_trace.transport.batch_processor - INFO - šŸ“¤ SENDING BATCH: 2 traces via send_callback\n", + "2025-10-22 23:01:59 - noveum_trace.transport.http_transport - INFO - šŸš€ SENDING BATCH: 2 traces to https://api.noveum.ai/api/v1/traces\n", + "2025-10-22 23:02:00 - noveum_trace.transport.http_transport - INFO - šŸ“” HTTP RESPONSE: Status 200 from https://api.noveum.ai/api/v1/traces\n", + "2025-10-22 23:02:00 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 96472f81-b98d-4453-97e8-b0000f4ac7b6) - 1 spans\n", + "2025-10-22 23:02:00 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 96472f81-b98d-4453-97e8-b0000f4ac7b6) - 1 spans\n", + "2025-10-22 23:02:00 - noveum_trace.transport.http_transport - INFO - āœ… Successfully sent batch of 2 traces\n", + "2025-10-22 23:02:00 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 96472f81-b98d-4453-97e8-b0000f4ac7b6\n", + "2025-10-22 23:02:00 - noveum_trace.transport.http_transport - INFO - āœ… Trace 96472f81-b98d-4453-97e8-b0000f4ac7b6 successfully queued for export\n", + "2025-10-22 23:02:00 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully sent batch of 2 traces via callback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:02:00 - INFO - novaeval.evaluators.agent_evaluator - Saving intermediate results after 11 samples\n", + "2025-10-22 23:02:00 - INFO - novaeval.evaluators.agent_evaluator - Intermediate results saved to demo_results/agent.rag_evaluation_metrics_dataset/agent_evaluation_results.csv\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Evaluating samples: 11it [00:47, 4.60s/it]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:02:00 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-10-22 23:02:02 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 3ec56361-a2aa-45bd-ad9e-12139cd5b25b) - 1 spans\n", + "2025-10-22 23:02:02 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 3ec56361-a2aa-45bd-ad9e-12139cd5b25b) - 1 spans\n", + "2025-10-22 23:02:02 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 3ec56361-a2aa-45bd-ad9e-12139cd5b25b\n", + "2025-10-22 23:02:02 - noveum_trace.transport.http_transport - INFO - āœ… Trace 3ec56361-a2aa-45bd-ad9e-12139cd5b25b successfully queued for export\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:02:02 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-10-22 23:02:03 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: bd7c6734-9502-43af-9227-7e3256eac18c) - 1 spans\n", + "2025-10-22 23:02:03 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: bd7c6734-9502-43af-9227-7e3256eac18c) - 1 spans\n", + "2025-10-22 23:02:03 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace bd7c6734-9502-43af-9227-7e3256eac18c\n", + "2025-10-22 23:02:03 - noveum_trace.transport.http_transport - INFO - āœ… Trace bd7c6734-9502-43af-9227-7e3256eac18c successfully queued for export\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:02:03 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-10-22 23:02:05 - noveum_trace.transport.batch_processor - INFO - ā° TIMEOUT TRIGGER: Sending batch due to timeout (5.5s >= 5.0s)\n", + "2025-10-22 23:02:05 - noveum_trace.transport.batch_processor - INFO - šŸ“¤ SENDING BATCH: 3 traces via send_callback\n", + "2025-10-22 23:02:05 - noveum_trace.transport.http_transport - INFO - šŸš€ SENDING BATCH: 3 traces to https://api.noveum.ai/api/v1/traces\n", + "2025-10-22 23:02:05 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 8c77b5d2-4e09-4e86-89db-9c4432c6a8f9) - 1 spans\n", + "2025-10-22 23:02:05 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 8c77b5d2-4e09-4e86-89db-9c4432c6a8f9) - 1 spans\n", + "2025-10-22 23:02:05 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 8c77b5d2-4e09-4e86-89db-9c4432c6a8f9\n", + "2025-10-22 23:02:05 - noveum_trace.transport.http_transport - INFO - āœ… Trace 8c77b5d2-4e09-4e86-89db-9c4432c6a8f9 successfully queued for export\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:02:05 - INFO - novaeval.evaluators.agent_evaluator - Saving intermediate results after 12 samples\n", + "2025-10-22 23:02:05 - INFO - novaeval.evaluators.agent_evaluator - Intermediate results saved to demo_results/agent.rag_evaluation_metrics_dataset/agent_evaluation_results.csv\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Evaluating samples: 12it [00:52, 4.42s/it]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:02:05 - INFO - novaeval.evaluators.agent_evaluator - Saving final results\n", + "2025-10-22 23:02:05 - INFO - novaeval.evaluators.agent_evaluator - Reloaded 12 results from CSV\n", + "2025-10-22 23:02:05 - INFO - novaeval.evaluators.agent_evaluator - Agent evaluation completed\n", + "\n", + "āœ… Evaluation completed!\n", + "\n", + "šŸ“Š Results Summary:\n", + " - task_progression: 4.27\n", + " - context_relevancy: 7.90\n", + " - role_adherence: 9.08\n", + " - tool_relevancy: 0.00\n", + " - parameter_correctness: 0.00\n", + "\n", + "šŸ” Individual Scores:\n", + "\n", + " Record 1 (Task: f1f37bd7-0851-4659-b493-b80d3800d920):\n", + " - task_progression: 3.8\n", + " - context_relevancy: 7.8\n", + " - role_adherence: 9.0\n", + " - tool_relevancy: 0.0\n", + " - parameter_correctness: 0.0\n", + "\n", + " Record 2 (Task: 52aacb67-c361-4445-9b72-c157f79f47d6):\n", + " - task_progression: 2.8\n", + " - context_relevancy: 7.8\n", + " - role_adherence: 9.0\n", + " - tool_relevancy: 0.0\n", + " - parameter_correctness: 0.0\n", + "\n", + " Record 3 (Task: 2218f641-604c-491a-9710-b51a9941b982):\n", + " - task_progression: 4.5\n", + " - context_relevancy: 7.9\n", + " - role_adherence: 9.0\n", + " - tool_relevancy: 0.0\n", + " - parameter_correctness: 0.0\n", + "\n", + " Record 4 (Task: 255fd49c-84b4-4b18-887e-6308a412d535):\n", + " - task_progression: 4.5\n", + " - context_relevancy: 7.8\n", + " - role_adherence: 9.0\n", + " - tool_relevancy: 0.0\n", + " - parameter_correctness: 0.0\n", + "\n", + " Record 5 (Task: dc511122-c0b6-415c-9a49-c7b45132dd87):\n", + " - task_progression: 4.5\n", + " - context_relevancy: 8.1\n", + " - role_adherence: 9.0\n", + " - tool_relevancy: 0.0\n", + " - parameter_correctness: 0.0\n", + "\n", + " Record 6 (Task: 04bebf38-a343-4563-80db-0154bef8d927):\n", + " - task_progression: 4.5\n", + " - context_relevancy: 7.8\n", + " - role_adherence: 9.0\n", + " - tool_relevancy: 0.0\n", + " - parameter_correctness: 0.0\n", + "\n", + " Record 7 (Task: 5e043630-6493-42b5-beb8-79faa19bfa37):\n", + " - task_progression: 4.5\n", + " - context_relevancy: 7.8\n", + " - role_adherence: 9.0\n", + " - tool_relevancy: 0.0\n", + " - parameter_correctness: 0.0\n", + "\n", + " Record 8 (Task: 7da9814d-a2e8-4c4e-b750-68b26bd5fd22):\n", + " - task_progression: 4.5\n", + " - context_relevancy: 7.8\n", + " - role_adherence: 9.0\n", + " - tool_relevancy: 0.0\n", + " - parameter_correctness: 0.0\n", + "\n", + " Record 9 (Task: 16143f74-2831-4753-b33d-ce4b645093c5):\n", + " - task_progression: 4.5\n", + " - context_relevancy: 8.1\n", + " - role_adherence: 9.0\n", + " - tool_relevancy: 0.0\n", + " - parameter_correctness: 0.0\n", + "\n", + " Record 10 (Task: fc64e6cc-6739-4256-ac4a-7b80c3028233):\n", + " - task_progression: 4.1\n", + " - context_relevancy: 7.8\n", + " - role_adherence: 9.0\n", + " - tool_relevancy: 0.0\n", + " - parameter_correctness: 0.0\n", + "\n", + " Record 11 (Task: b7945c49-f584-4c70-972d-536a805d8a31):\n", + " - task_progression: 4.5\n", + " - context_relevancy: 8.0\n", + " - role_adherence: 9.0\n", + " - tool_relevancy: 0.0\n", + " - parameter_correctness: 0.0\n", + "\n", + " Record 12 (Task: f5c40ecf-36c0-45ba-9cc9-dc0329b0324b):\n", + " - task_progression: 4.5\n", + " - context_relevancy: 8.1\n", + " - role_adherence: 10.0\n", + " - tool_relevancy: 0.0\n", + " - parameter_correctness: 0.0\n", + "āœ… Evaluation completed successfully!\n", + "\n", + "šŸ“‹ Step 7: Exporting Dataset\n", + "šŸ’¾ Exporting processed dataset...\n", + "āœ… Exported to ./processed_datasets/agent.rag_evaluation_metrics_dataset_processed_dataset.json\n", + "āœ… Exported to ./processed_datasets/agent.rag_evaluation_metrics_dataset_processed_dataset.csv\n", + "āœ… Export completed successfully!\n", + "\n", + "============================================================\n", + "šŸŽ‰ EVALUATION PIPELINE COMPLETED!\n", + "šŸ“Š Final Results:\n", + " - File processed: split_datasets/agent.rag_evaluation_metrics_dataset.json\n", + " - Spans loaded: 12\n", + " - Dataset size: 12\n", + " - Evaluation completed: True\n", + " - Export successful: True\n", + " - Results saved to: ./demo_results/agent.rag_evaluation_metrics_dataset/\n", + "Completed agent.rag_evaluation_metrics_dataset.json\n", + "\n", + "Processing agent.routing_evaluation_metrics_dataset.json...\n", + "šŸš€ Starting Complete Agent Evaluation Pipeline\n", + "šŸ“ Processing file: split_datasets/agent.routing_evaluation_metrics_dataset.json\n", + "============================================================\n", + "\n", + "šŸ“‹ Step 1: Environment Setup\n", + "āœ… Logging configured at INFO level\n", + "šŸ” Environment validation:\n", + " āœ… gemini_api_key: True\n", + " āœ… pandas_available: True\n", + " āœ… novaeval_available: True\n", + "āœ… Environment ready for evaluation!\n", + "\n", + "šŸ“‹ Step 2: Loading Dataset\n", + "šŸ“Š Loaded 20 spans from split_datasets/agent.routing_evaluation_metrics_dataset.json\n", + "\n", + "šŸ” Available span types:\n", + " - agent.routing_evaluation_metrics: 20\n", + "āœ… Dataset loaded: 20 spans\n", + "\n", + "šŸ“‹ Step 3: Converting to AgentDataset Format\n", + "šŸ”„ Converting spans to AgentData objects...\n", + "agent_task not found\n", + "agent_response is not available 95dac0c3-83fd-459c-af4e-c6a1734fd874\n", + "agent_task not found\n", + "agent_response is not available 516060a3-0f58-46ba-8a6a-4b502a376492\n", + "agent_task not found\n", + "agent_response is not available 8f8d2dc4-150f-4e43-aad3-96da4d06f586\n", + "agent_task not found\n", + "agent_response is not available cf0a915c-668b-412e-9c14-69119d059dc2\n", + "agent_task not found\n", + "agent_response is not available f75b51b8-4882-4170-bc08-2adcab250f50\n", + "agent_task not found\n", + "agent_response is not available 4180abcf-924c-45ac-8dfa-17c0bc603eff\n", + "agent_task not found\n", + "agent_response is not available 8b0022cc-91dd-4065-91ed-b77217f92bd7\n", + "agent_task not found\n", + "agent_response is not available 780df186-fe5a-4742-b937-2a142ad528c6\n", + "agent_task not found\n", + "agent_response is not available f80ae3e8-b4dd-42c6-b17b-de7805cedbb9\n", + "agent_task not found\n", + "agent_response is not available 209259cf-411d-4b97-8f7b-40b2530eec81\n", + "agent_task not found\n", + "agent_response is not available c3f3bd32-9954-443f-93e5-52aa6cf412a0\n", + "agent_task not found\n", + "agent_response is not available 940ce621-0015-45ed-adc6-c4caa2a95e81\n", + "agent_task not found\n", + "agent_response is not available ea28f049-6bc9-48b4-8386-279abe087ca8\n", + "agent_task not found\n", + "agent_response is not available 01a03da5-64df-41e3-b9c9-556e12a8edd7\n", + "agent_task not found\n", + "agent_response is not available 8d9c632b-b2e8-496d-b1b0-a89e780ef810\n", + "agent_task not found\n", + "agent_response is not available 72dfbba4-0672-4adc-a1bf-308157d831e6\n", + "agent_task not found\n", + "agent_response is not available 92c81ee2-728d-4ea6-bc52-b81295d2aea2\n", + "agent_task not found\n", + "agent_response is not available 807627b0-41d9-478f-8a8b-26540d70af68\n", + "agent_task not found\n", + "agent_response is not available bf860672-af22-44c1-b13b-0eabe8f72503\n", + "agent_task not found\n", + "agent_response is not available 5fa714f1-eeb9-4617-bc7e-67062e6d1eab\n", + "\n", + "āœ… Successfully converted 20 spans to AgentData\n", + "šŸ“Š AgentDataset created with 20 records\n", + "āœ… AgentDataset created: 20 records\n", + "\n", + "šŸ“‹ Step 4: Dataset Analysis\n", + "šŸ“ˆ Dataset Statistics:\n", + "\n", + "Agent Types: {'agent': 20}\n", + "Records with responses: 0\n", + "Records with tool calls: 0\n", + "Records with retrieval: 0\n", + "Tool usage: {}\n", + "šŸ” Dataset Analysis:\n", + "\n", + "=== Agent Behavior Patterns ===\n", + "\n", + "šŸ“ˆ Tool Usage:\n", + "\n", + "šŸ“‹ Task Types:\n", + "\n", + "šŸ“‹ Step 5: Setting up Evaluation\n", + "āœ… GEMINI_API_KEY found in environment\n", + "2025-10-22 23:02:05 - INFO - novaeval.models.base - Noveum tracing initialized successfully\n", + "āœ… Gemini model initialized\n", + "āœ… Initialized 5 scoring functions:\n", + " - task_progression_scorer\n", + " - context_relevancy_scorer\n", + " - role_adherence_scorer\n", + " - tool_relevancy_scorer\n", + " - parameter_correctness_scorer\n", + "\n", + "āœ… AgentEvaluator created with Gemini model and scoring functions\n", + "āœ… Evaluation components ready!\n", + "\n", + "šŸ“‹ Step 6: Running Evaluation\n", + "šŸŽÆ Evaluating 25 samples...\n", + "šŸš€ Running evaluation on sample data...\n", + "\n", + "šŸ“Š Evaluating 0 sample records...\n", + "2025-10-22 23:02:05 - INFO - novaeval.evaluators.agent_evaluator - Starting agent evaluation process\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\n", + "Evaluating samples: 0it [00:00, ?it/s]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:02:05 - INFO - novaeval.evaluators.agent_evaluator - Saving final results\n", + "2025-10-22 23:02:05 - INFO - novaeval.evaluators.agent_evaluator - Agent evaluation completed\n", + "\n", + "āœ… Evaluation completed!\n", + "āŒ Results file not found\n", + "āŒ Evaluation failed\n", + "\n", + "šŸ“‹ Step 7: Exporting Dataset\n", + "šŸ’¾ Exporting processed dataset...\n", + "āœ… Exported to ./processed_datasets/agent.routing_evaluation_metrics_dataset_processed_dataset.json\n", + "āœ… Exported to ./processed_datasets/agent.routing_evaluation_metrics_dataset_processed_dataset.csv\n", + "āœ… Export completed successfully!\n", + "\n", + "============================================================\n", + "šŸŽ‰ EVALUATION PIPELINE COMPLETED!\n", + "šŸ“Š Final Results:\n", + " - File processed: split_datasets/agent.routing_evaluation_metrics_dataset.json\n", + " - Spans loaded: 20\n", + " - Dataset size: 20\n", + " - Evaluation completed: False\n", + " - Export successful: True\n", + " - Errors encountered: 1\n", + "Completed agent.routing_evaluation_metrics_dataset.json\n", + "\n", + "Processing agent.llm-rag_dataset.json...\n", + "šŸš€ Starting Complete Agent Evaluation Pipeline\n", + "šŸ“ Processing file: split_datasets/agent.llm-rag_dataset.json\n", + "============================================================\n", + "\n", + "šŸ“‹ Step 1: Environment Setup\n", + "āœ… Logging configured at INFO level\n", + "šŸ” Environment validation:\n", + " āœ… gemini_api_key: True\n", + " āœ… pandas_available: True\n", + " āœ… novaeval_available: True\n", + "āœ… Environment ready for evaluation!\n", + "\n", + "šŸ“‹ Step 2: Loading Dataset\n", + "šŸ“Š Loaded 12 spans from split_datasets/agent.llm-rag_dataset.json\n", + "\n", + "šŸ” Available span types:\n", + " - agent.llm-rag: 12\n", + "āœ… Dataset loaded: 12 spans\n", + "\n", + "šŸ“‹ Step 3: Converting to AgentDataset Format\n", + "šŸ”„ Converting spans to AgentData objects...\n", + "agent_task not found\n", + "agent_response is not available ea4d61c5-b8dd-4677-b21b-2b17170e2163\n", + "agent_task not found\n", + "agent_response is not available 7505f97a-cbc0-42fe-8a72-95387d723daa\n", + "agent_task not found\n", + "agent_response is not available 8a6e18a1-babc-4166-926f-188f344d514e\n", + "agent_task not found\n", + "agent_response is not available 5bec4165-6596-43ac-930b-acfcde1cc02a\n", + "agent_task not found\n", + "agent_response is not available f9f79a07-4eda-41b8-9634-d4460114caad\n", + "agent_task not found\n", + "agent_response is not available edc52a5d-8450-4d0f-ad14-958ec4d147f4\n", + "agent_task not found\n", + "agent_response is not available 6c362c73-e063-4813-8180-d26b6a52b798\n", + "agent_task not found\n", + "agent_response is not available 1393d065-a9a4-4475-808f-2d00b3c68939\n", + "agent_task not found\n", + "agent_response is not available f419377a-fc24-4abf-8a1c-03857824e374\n", + "agent_task not found\n", + "agent_response is not available 0e58e31f-a8ab-4aee-8fc2-abcae1e21d9d\n", + "agent_task not found\n", + "agent_response is not available 48155aad-6204-4a52-8624-8fa5d6fe407f\n", + "agent_task not found\n", + "agent_response is not available 57ded2eb-0f3f-474e-8704-5ad9974ab4bb\n", + "\n", + "āœ… Successfully converted 12 spans to AgentData\n", + "šŸ“Š AgentDataset created with 12 records\n", + "āœ… AgentDataset created: 12 records\n", + "\n", + "šŸ“‹ Step 4: Dataset Analysis\n", + "šŸ“ˆ Dataset Statistics:\n", + "\n", + "Agent Types: {'agent': 12}\n", + "Records with responses: 0\n", + "Records with tool calls: 0\n", + "Records with retrieval: 0\n", + "Tool usage: {}\n", + "šŸ” Dataset Analysis:\n", + "\n", + "=== Agent Behavior Patterns ===\n", + "\n", + "šŸ“ˆ Tool Usage:\n", + "\n", + "šŸ“‹ Task Types:\n", + "\n", + "šŸ“‹ Step 5: Setting up Evaluation\n", + "āœ… GEMINI_API_KEY found in environment\n", + "2025-10-22 23:02:05 - INFO - novaeval.models.base - Noveum tracing initialized successfully\n", + "āœ… Gemini model initialized\n", + "āœ… Initialized 5 scoring functions:\n", + " - task_progression_scorer\n", + " - context_relevancy_scorer\n", + " - role_adherence_scorer\n", + " - tool_relevancy_scorer\n", + " - parameter_correctness_scorer\n", + "\n", + "āœ… AgentEvaluator created with Gemini model and scoring functions\n", + "āœ… Evaluation components ready!\n", + "\n", + "šŸ“‹ Step 6: Running Evaluation\n", + "šŸŽÆ Evaluating 25 samples...\n", + "šŸš€ Running evaluation on sample data...\n", + "\n", + "šŸ“Š Evaluating 0 sample records...\n", + "2025-10-22 23:02:05 - INFO - novaeval.evaluators.agent_evaluator - Starting agent evaluation process\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\n", + "Evaluating samples: 0it [00:00, ?it/s]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:02:05 - INFO - novaeval.evaluators.agent_evaluator - Saving final results\n", + "2025-10-22 23:02:05 - INFO - novaeval.evaluators.agent_evaluator - Agent evaluation completed\n", + "\n", + "āœ… Evaluation completed!\n", + "āŒ Results file not found\n", + "āŒ Evaluation failed\n", + "\n", + "šŸ“‹ Step 7: Exporting Dataset\n", + "šŸ’¾ Exporting processed dataset...\n", + "āœ… Exported to ./processed_datasets/agent.llm-rag_dataset_processed_dataset.json\n", + "āœ… Exported to ./processed_datasets/agent.llm-rag_dataset_processed_dataset.csv\n", + "āœ… Export completed successfully!\n", + "\n", + "============================================================\n", + "šŸŽ‰ EVALUATION PIPELINE COMPLETED!\n", + "šŸ“Š Final Results:\n", + " - File processed: split_datasets/agent.llm-rag_dataset.json\n", + " - Spans loaded: 12\n", + " - Dataset size: 12\n", + " - Evaluation completed: False\n", + " - Export successful: True\n", + " - Errors encountered: 1\n", + "Completed agent.llm-rag_dataset.json\n", + "\n", + "Processing agent.web_search_evaluation_metrics_dataset.json...\n", + "šŸš€ Starting Complete Agent Evaluation Pipeline\n", + "šŸ“ Processing file: split_datasets/agent.web_search_evaluation_metrics_dataset.json\n", + "============================================================\n", + "\n", + "šŸ“‹ Step 1: Environment Setup\n", + "āœ… Logging configured at INFO level\n", + "šŸ” Environment validation:\n", + " āœ… gemini_api_key: True\n", + " āœ… pandas_available: True\n", + " āœ… novaeval_available: True\n", + "āœ… Environment ready for evaluation!\n", + "\n", + "šŸ“‹ Step 2: Loading Dataset\n", + "šŸ“Š Loaded 8 spans from split_datasets/agent.web_search_evaluation_metrics_dataset.json\n", + "\n", + "šŸ” Available span types:\n", + " - agent.web_search_evaluation_metrics: 8\n", + "āœ… Dataset loaded: 8 spans\n", + "\n", + "šŸ“‹ Step 3: Converting to AgentDataset Format\n", + "šŸ”„ Converting spans to AgentData objects...\n", + "\n", + "āœ… Successfully converted 8 spans to AgentData\n", + "šŸ“Š AgentDataset created with 8 records\n", + "āœ… AgentDataset created: 8 records\n", + "\n", + "šŸ“‹ Step 4: Dataset Analysis\n", + "šŸ“ˆ Dataset Statistics:\n", + "\n", + "Agent Types: {'agent': 8}\n", + "Records with responses: 8\n", + "Records with tool calls: 0\n", + "Records with retrieval: 8\n", + "Tool usage: {}\n", + "šŸ” Dataset Analysis:\n", + "\n", + "=== Agent Behavior Patterns ===\n", + "\n", + "šŸ“ˆ Tool Usage:\n", + "\n", + "šŸ“‹ Task Types:\n", + " - other: 8\n", + "\n", + "šŸ“ Response Statistics:\n", + " - Average response length: 222.0 characters\n", + " - Min response length: 222\n", + " - Max response length: 222\n", + "\n", + "šŸ“‹ Step 5: Setting up Evaluation\n", + "āœ… GEMINI_API_KEY found in environment\n", + "2025-10-22 23:02:05 - INFO - novaeval.models.base - Noveum tracing initialized successfully\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "āœ… Gemini model initialized\n", + "āœ… Initialized 5 scoring functions:\n", + " - task_progression_scorer\n", + " - context_relevancy_scorer\n", + " - role_adherence_scorer\n", + " - tool_relevancy_scorer\n", + " - parameter_correctness_scorer\n", + "\n", + "āœ… AgentEvaluator created with Gemini model and scoring functions\n", + "āœ… Evaluation components ready!\n", + "\n", + "šŸ“‹ Step 6: Running Evaluation\n", + "šŸŽÆ Evaluating 25 samples...\n", + "šŸš€ Running evaluation on sample data...\n", + "\n", + "šŸ“Š Evaluating 8 sample records...\n", + "2025-10-22 23:02:05 - INFO - novaeval.evaluators.agent_evaluator - Starting agent evaluation process\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Evaluating samples: 0it [00:00, ?it/s]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:02:05 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-10-22 23:02:05 - noveum_trace.transport.http_transport - INFO - šŸ“” HTTP RESPONSE: Status 200 from https://api.noveum.ai/api/v1/traces\n", + "2025-10-22 23:02:05 - noveum_trace.transport.http_transport - INFO - āœ… Successfully sent batch of 3 traces\n", + "2025-10-22 23:02:05 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully sent batch of 3 traces via callback\n", + "2025-10-22 23:02:07 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: e7588539-fcf8-44a3-9f31-efe82ba97700) - 1 spans\n", + "2025-10-22 23:02:07 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: e7588539-fcf8-44a3-9f31-efe82ba97700) - 1 spans\n", + "2025-10-22 23:02:07 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace e7588539-fcf8-44a3-9f31-efe82ba97700\n", + "2025-10-22 23:02:07 - noveum_trace.transport.http_transport - INFO - āœ… Trace e7588539-fcf8-44a3-9f31-efe82ba97700 successfully queued for export\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:02:07 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-10-22 23:02:08 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: a47d3d33-371b-4045-9fb1-378424928ada) - 1 spans\n", + "2025-10-22 23:02:08 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: a47d3d33-371b-4045-9fb1-378424928ada) - 1 spans\n", + "2025-10-22 23:02:08 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace a47d3d33-371b-4045-9fb1-378424928ada\n", + "2025-10-22 23:02:08 - noveum_trace.transport.http_transport - INFO - āœ… Trace a47d3d33-371b-4045-9fb1-378424928ada successfully queued for export\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:02:08 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-10-22 23:02:09 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 69314585-f591-4d20-ba7c-81dcd5fa7aee) - 1 spans\n", + "2025-10-22 23:02:09 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 69314585-f591-4d20-ba7c-81dcd5fa7aee) - 1 spans\n", + "2025-10-22 23:02:09 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 69314585-f591-4d20-ba7c-81dcd5fa7aee\n", + "2025-10-22 23:02:09 - noveum_trace.transport.http_transport - INFO - āœ… Trace 69314585-f591-4d20-ba7c-81dcd5fa7aee successfully queued for export\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:02:09 - INFO - novaeval.evaluators.agent_evaluator - Saving intermediate results after 1 samples\n", + "2025-10-22 23:02:09 - INFO - novaeval.evaluators.agent_evaluator - Intermediate results saved to demo_results/agent.web_search_evaluation_metrics_dataset/agent_evaluation_results.csv\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Evaluating samples: 1it [00:04, 4.12s/it]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:02:09 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-10-22 23:02:10 - noveum_trace.transport.batch_processor - INFO - ā° TIMEOUT TRIGGER: Sending batch due to timeout (5.4s >= 5.0s)\n", + "2025-10-22 23:02:10 - noveum_trace.transport.batch_processor - INFO - šŸ“¤ SENDING BATCH: 4 traces via send_callback\n", + "2025-10-22 23:02:10 - noveum_trace.transport.http_transport - INFO - šŸš€ SENDING BATCH: 4 traces to https://api.noveum.ai/api/v1/traces\n", + "2025-10-22 23:02:10 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: a05b5d92-06f6-4fec-9673-f33c267ef809) - 1 spans\n", + "2025-10-22 23:02:10 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: a05b5d92-06f6-4fec-9673-f33c267ef809) - 1 spans\n", + "2025-10-22 23:02:10 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace a05b5d92-06f6-4fec-9673-f33c267ef809\n", + "2025-10-22 23:02:10 - noveum_trace.transport.http_transport - INFO - āœ… Trace a05b5d92-06f6-4fec-9673-f33c267ef809 successfully queued for export\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:02:10 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-10-22 23:02:11 - noveum_trace.transport.http_transport - INFO - šŸ“” HTTP RESPONSE: Status 200 from https://api.noveum.ai/api/v1/traces\n", + "2025-10-22 23:02:11 - noveum_trace.transport.http_transport - INFO - āœ… Successfully sent batch of 4 traces\n", + "2025-10-22 23:02:11 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully sent batch of 4 traces via callback\n", + "2025-10-22 23:02:12 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 032c73be-2c3f-4980-9cdf-762de61c437e) - 1 spans\n", + "2025-10-22 23:02:12 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 032c73be-2c3f-4980-9cdf-762de61c437e) - 1 spans\n", + "2025-10-22 23:02:12 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 032c73be-2c3f-4980-9cdf-762de61c437e\n", + "2025-10-22 23:02:12 - noveum_trace.transport.http_transport - INFO - āœ… Trace 032c73be-2c3f-4980-9cdf-762de61c437e successfully queued for export\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:02:12 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-10-22 23:02:13 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 4ccea784-7905-4081-b536-1a6fade9180f) - 1 spans\n", + "2025-10-22 23:02:13 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 4ccea784-7905-4081-b536-1a6fade9180f) - 1 spans\n", + "2025-10-22 23:02:13 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 4ccea784-7905-4081-b536-1a6fade9180f\n", + "2025-10-22 23:02:13 - noveum_trace.transport.http_transport - INFO - āœ… Trace 4ccea784-7905-4081-b536-1a6fade9180f successfully queued for export\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:02:13 - INFO - novaeval.evaluators.agent_evaluator - Saving intermediate results after 2 samples\n", + "2025-10-22 23:02:13 - INFO - novaeval.evaluators.agent_evaluator - Intermediate results saved to demo_results/agent.web_search_evaluation_metrics_dataset/agent_evaluation_results.csv\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Evaluating samples: 2it [00:07, 3.79s/it]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:02:13 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-10-22 23:02:14 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 4a768f59-41a4-4da7-b4fb-1530bd85f1bc) - 1 spans\n", + "2025-10-22 23:02:14 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 4a768f59-41a4-4da7-b4fb-1530bd85f1bc) - 1 spans\n", + "2025-10-22 23:02:14 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 4a768f59-41a4-4da7-b4fb-1530bd85f1bc\n", + "2025-10-22 23:02:14 - noveum_trace.transport.http_transport - INFO - āœ… Trace 4a768f59-41a4-4da7-b4fb-1530bd85f1bc successfully queued for export\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:02:14 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-10-22 23:02:15 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 490c1659-4aca-401f-bf9d-34891b18717a) - 1 spans\n", + "2025-10-22 23:02:15 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 490c1659-4aca-401f-bf9d-34891b18717a) - 1 spans\n", + "2025-10-22 23:02:15 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 490c1659-4aca-401f-bf9d-34891b18717a\n", + "2025-10-22 23:02:15 - noveum_trace.transport.http_transport - INFO - āœ… Trace 490c1659-4aca-401f-bf9d-34891b18717a successfully queued for export\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:02:15 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-10-22 23:02:16 - noveum_trace.transport.batch_processor - INFO - ā° TIMEOUT TRIGGER: Sending batch due to timeout (5.5s >= 5.0s)\n", + "2025-10-22 23:02:16 - noveum_trace.transport.batch_processor - INFO - šŸ“¤ SENDING BATCH: 5 traces via send_callback\n", + "2025-10-22 23:02:16 - noveum_trace.transport.http_transport - INFO - šŸš€ SENDING BATCH: 5 traces to https://api.noveum.ai/api/v1/traces\n", + "2025-10-22 23:02:16 - noveum_trace.transport.http_transport - INFO - šŸ“” HTTP RESPONSE: Status 200 from https://api.noveum.ai/api/v1/traces\n", + "2025-10-22 23:02:16 - noveum_trace.transport.http_transport - INFO - āœ… Successfully sent batch of 5 traces\n", + "2025-10-22 23:02:16 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully sent batch of 5 traces via callback\n", + "2025-10-22 23:02:17 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: c83bf958-6430-47bc-9407-5933e25f4892) - 1 spans\n", + "2025-10-22 23:02:17 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: c83bf958-6430-47bc-9407-5933e25f4892) - 1 spans\n", + "2025-10-22 23:02:17 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace c83bf958-6430-47bc-9407-5933e25f4892\n", + "2025-10-22 23:02:17 - noveum_trace.transport.http_transport - INFO - āœ… Trace c83bf958-6430-47bc-9407-5933e25f4892 successfully queued for export\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:02:17 - INFO - novaeval.evaluators.agent_evaluator - Saving intermediate results after 3 samples\n", + "2025-10-22 23:02:17 - INFO - novaeval.evaluators.agent_evaluator - Intermediate results saved to demo_results/agent.web_search_evaluation_metrics_dataset/agent_evaluation_results.csv\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Evaluating samples: 3it [00:11, 3.78s/it]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:02:17 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-10-22 23:02:18 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: eacc2b6e-8d42-4c12-82df-b0298723c50e) - 1 spans\n", + "2025-10-22 23:02:18 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: eacc2b6e-8d42-4c12-82df-b0298723c50e) - 1 spans\n", + "2025-10-22 23:02:18 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace eacc2b6e-8d42-4c12-82df-b0298723c50e\n", + "2025-10-22 23:02:18 - noveum_trace.transport.http_transport - INFO - āœ… Trace eacc2b6e-8d42-4c12-82df-b0298723c50e successfully queued for export\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:02:18 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-10-22 23:02:19 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 1f3f745f-2735-4197-b27d-0e54cd4fdd9b) - 1 spans\n", + "2025-10-22 23:02:19 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 1f3f745f-2735-4197-b27d-0e54cd4fdd9b) - 1 spans\n", + "2025-10-22 23:02:19 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 1f3f745f-2735-4197-b27d-0e54cd4fdd9b\n", + "2025-10-22 23:02:19 - noveum_trace.transport.http_transport - INFO - āœ… Trace 1f3f745f-2735-4197-b27d-0e54cd4fdd9b successfully queued for export\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:02:19 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-10-22 23:02:20 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 04df8130-9aa6-4c7c-b184-df6b8d827620) - 1 spans\n", + "2025-10-22 23:02:20 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 04df8130-9aa6-4c7c-b184-df6b8d827620) - 1 spans\n", + "2025-10-22 23:02:21 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 04df8130-9aa6-4c7c-b184-df6b8d827620\n", + "2025-10-22 23:02:21 - noveum_trace.transport.http_transport - INFO - āœ… Trace 04df8130-9aa6-4c7c-b184-df6b8d827620 successfully queued for export\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:02:21 - INFO - novaeval.evaluators.agent_evaluator - Saving intermediate results after 4 samples\n", + "2025-10-22 23:02:21 - INFO - novaeval.evaluators.agent_evaluator - Intermediate results saved to demo_results/agent.web_search_evaluation_metrics_dataset/agent_evaluation_results.csv\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Evaluating samples: 4it [00:15, 3.84s/it]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:02:21 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-10-22 23:02:21 - noveum_trace.transport.batch_processor - INFO - ā° TIMEOUT TRIGGER: Sending batch due to timeout (5.2s >= 5.0s)\n", + "2025-10-22 23:02:21 - noveum_trace.transport.batch_processor - INFO - šŸ“¤ SENDING BATCH: 4 traces via send_callback\n", + "2025-10-22 23:02:21 - noveum_trace.transport.http_transport - INFO - šŸš€ SENDING BATCH: 4 traces to https://api.noveum.ai/api/v1/traces\n", + "2025-10-22 23:02:22 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 25522bcf-a4e6-4f72-b6b1-be6e12c6b628) - 1 spans\n", + "2025-10-22 23:02:22 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 25522bcf-a4e6-4f72-b6b1-be6e12c6b628) - 1 spans\n", + "2025-10-22 23:02:22 - noveum_trace.transport.http_transport - INFO - šŸ“” HTTP RESPONSE: Status 200 from https://api.noveum.ai/api/v1/traces\n", + "2025-10-22 23:02:22 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 25522bcf-a4e6-4f72-b6b1-be6e12c6b628\n", + "2025-10-22 23:02:22 - noveum_trace.transport.http_transport - INFO - āœ… Successfully sent batch of 4 traces\n", + "2025-10-22 23:02:22 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully sent batch of 4 traces via callback\n", + "2025-10-22 23:02:22 - noveum_trace.transport.http_transport - INFO - āœ… Trace 25522bcf-a4e6-4f72-b6b1-be6e12c6b628 successfully queued for export\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:02:22 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-10-22 23:02:23 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 8d0257b7-8b4a-43d2-b3f8-c331f9442e9a) - 1 spans\n", + "2025-10-22 23:02:23 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 8d0257b7-8b4a-43d2-b3f8-c331f9442e9a) - 1 spans\n", + "2025-10-22 23:02:23 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 8d0257b7-8b4a-43d2-b3f8-c331f9442e9a\n", + "2025-10-22 23:02:23 - noveum_trace.transport.http_transport - INFO - āœ… Trace 8d0257b7-8b4a-43d2-b3f8-c331f9442e9a successfully queued for export\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:02:23 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-10-22 23:02:26 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: df6eff5c-8fa9-4c0a-9f8b-4fb5eb71694a) - 1 spans\n", + "2025-10-22 23:02:26 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: df6eff5c-8fa9-4c0a-9f8b-4fb5eb71694a) - 1 spans\n", + "2025-10-22 23:02:26 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace df6eff5c-8fa9-4c0a-9f8b-4fb5eb71694a\n", + "2025-10-22 23:02:26 - noveum_trace.transport.http_transport - INFO - āœ… Trace df6eff5c-8fa9-4c0a-9f8b-4fb5eb71694a successfully queued for export\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:02:26 - INFO - novaeval.evaluators.agent_evaluator - Saving intermediate results after 5 samples\n", + "2025-10-22 23:02:26 - INFO - novaeval.evaluators.agent_evaluator - Intermediate results saved to demo_results/agent.web_search_evaluation_metrics_dataset/agent_evaluation_results.csv\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Evaluating samples: 5it [00:21, 4.57s/it]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:02:26 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-10-22 23:02:27 - noveum_trace.transport.batch_processor - INFO - ā° TIMEOUT TRIGGER: Sending batch due to timeout (5.9s >= 5.0s)\n", + "2025-10-22 23:02:27 - noveum_trace.transport.batch_processor - INFO - šŸ“¤ SENDING BATCH: 3 traces via send_callback\n", + "2025-10-22 23:02:27 - noveum_trace.transport.http_transport - INFO - šŸš€ SENDING BATCH: 3 traces to https://api.noveum.ai/api/v1/traces\n", + "2025-10-22 23:02:27 - noveum_trace.transport.http_transport - INFO - šŸ“” HTTP RESPONSE: Status 200 from https://api.noveum.ai/api/v1/traces\n", + "2025-10-22 23:02:27 - noveum_trace.transport.http_transport - INFO - āœ… Successfully sent batch of 3 traces\n", + "2025-10-22 23:02:27 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully sent batch of 3 traces via callback\n", + "2025-10-22 23:02:28 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 9fca5686-3b49-4bba-961f-452fba26f5e6) - 1 spans\n", + "2025-10-22 23:02:28 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 9fca5686-3b49-4bba-961f-452fba26f5e6) - 1 spans\n", + "2025-10-22 23:02:28 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 9fca5686-3b49-4bba-961f-452fba26f5e6\n", + "2025-10-22 23:02:28 - noveum_trace.transport.http_transport - INFO - āœ… Trace 9fca5686-3b49-4bba-961f-452fba26f5e6 successfully queued for export\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:02:28 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-10-22 23:02:29 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: d7502a23-efaa-437e-ab40-138c0a1c3cc9) - 1 spans\n", + "2025-10-22 23:02:29 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: d7502a23-efaa-437e-ab40-138c0a1c3cc9) - 1 spans\n", + "2025-10-22 23:02:29 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace d7502a23-efaa-437e-ab40-138c0a1c3cc9\n", + "2025-10-22 23:02:29 - noveum_trace.transport.http_transport - INFO - āœ… Trace d7502a23-efaa-437e-ab40-138c0a1c3cc9 successfully queued for export\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:02:29 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-10-22 23:02:30 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 539d7f80-b556-441a-b288-08da522ef2ed) - 1 spans\n", + "2025-10-22 23:02:30 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 539d7f80-b556-441a-b288-08da522ef2ed) - 1 spans\n", + "2025-10-22 23:02:30 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 539d7f80-b556-441a-b288-08da522ef2ed\n", + "2025-10-22 23:02:30 - noveum_trace.transport.http_transport - INFO - āœ… Trace 539d7f80-b556-441a-b288-08da522ef2ed successfully queued for export\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:02:30 - INFO - novaeval.evaluators.agent_evaluator - Saving intermediate results after 6 samples\n", + "2025-10-22 23:02:30 - INFO - novaeval.evaluators.agent_evaluator - Intermediate results saved to demo_results/agent.web_search_evaluation_metrics_dataset/agent_evaluation_results.csv\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Evaluating samples: 6it [00:25, 4.33s/it]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:02:30 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-10-22 23:02:31 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: f4853141-1b6b-493e-94c2-ebd82a327a63) - 1 spans\n", + "2025-10-22 23:02:31 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: f4853141-1b6b-493e-94c2-ebd82a327a63) - 1 spans\n", + "2025-10-22 23:02:31 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace f4853141-1b6b-493e-94c2-ebd82a327a63\n", + "2025-10-22 23:02:31 - noveum_trace.transport.http_transport - INFO - āœ… Trace f4853141-1b6b-493e-94c2-ebd82a327a63 successfully queued for export\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:02:31 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-10-22 23:02:32 - noveum_trace.transport.batch_processor - INFO - ā° TIMEOUT TRIGGER: Sending batch due to timeout (5.5s >= 5.0s)\n", + "2025-10-22 23:02:32 - noveum_trace.transport.batch_processor - INFO - šŸ“¤ SENDING BATCH: 4 traces via send_callback\n", + "2025-10-22 23:02:32 - noveum_trace.transport.http_transport - INFO - šŸš€ SENDING BATCH: 4 traces to https://api.noveum.ai/api/v1/traces\n", + "2025-10-22 23:02:33 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: dc0ec2ed-2ab0-4e6f-9005-08c47f6be881) - 1 spans\n", + "2025-10-22 23:02:33 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: dc0ec2ed-2ab0-4e6f-9005-08c47f6be881) - 1 spans\n", + "2025-10-22 23:02:33 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace dc0ec2ed-2ab0-4e6f-9005-08c47f6be881\n", + "2025-10-22 23:02:33 - noveum_trace.transport.http_transport - INFO - āœ… Trace dc0ec2ed-2ab0-4e6f-9005-08c47f6be881 successfully queued for export\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:02:33 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-10-22 23:02:33 - noveum_trace.transport.http_transport - INFO - šŸ“” HTTP RESPONSE: Status 200 from https://api.noveum.ai/api/v1/traces\n", + "2025-10-22 23:02:33 - noveum_trace.transport.http_transport - INFO - āœ… Successfully sent batch of 4 traces\n", + "2025-10-22 23:02:33 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully sent batch of 4 traces via callback\n", + "2025-10-22 23:02:34 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 6be3cc26-2f44-4534-8c17-af00776df15d) - 1 spans\n", + "2025-10-22 23:02:34 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 6be3cc26-2f44-4534-8c17-af00776df15d) - 1 spans\n", + "2025-10-22 23:02:34 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 6be3cc26-2f44-4534-8c17-af00776df15d\n", + "2025-10-22 23:02:34 - noveum_trace.transport.http_transport - INFO - āœ… Trace 6be3cc26-2f44-4534-8c17-af00776df15d successfully queued for export\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:02:34 - INFO - novaeval.evaluators.agent_evaluator - Saving intermediate results after 7 samples\n", + "2025-10-22 23:02:34 - INFO - novaeval.evaluators.agent_evaluator - Intermediate results saved to demo_results/agent.web_search_evaluation_metrics_dataset/agent_evaluation_results.csv\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Evaluating samples: 7it [00:28, 4.12s/it]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:02:34 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-10-22 23:02:36 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: d900cab2-2ff3-4c05-b0c0-7d5ed8303c40) - 1 spans\n", + "2025-10-22 23:02:36 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: d900cab2-2ff3-4c05-b0c0-7d5ed8303c40) - 1 spans\n", + "2025-10-22 23:02:36 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace d900cab2-2ff3-4c05-b0c0-7d5ed8303c40\n", + "2025-10-22 23:02:36 - noveum_trace.transport.http_transport - INFO - āœ… Trace d900cab2-2ff3-4c05-b0c0-7d5ed8303c40 successfully queued for export\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:02:36 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-10-22 23:02:37 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 2337a5e3-f182-40a2-8172-7b973fd6f67f) - 1 spans\n", + "2025-10-22 23:02:37 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 2337a5e3-f182-40a2-8172-7b973fd6f67f) - 1 spans\n", + "2025-10-22 23:02:37 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 2337a5e3-f182-40a2-8172-7b973fd6f67f\n", + "2025-10-22 23:02:37 - noveum_trace.transport.http_transport - INFO - āœ… Trace 2337a5e3-f182-40a2-8172-7b973fd6f67f successfully queued for export\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:02:37 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-10-22 23:02:38 - noveum_trace.transport.batch_processor - INFO - ā° TIMEOUT TRIGGER: Sending batch due to timeout (5.4s >= 5.0s)\n", + "2025-10-22 23:02:38 - noveum_trace.transport.batch_processor - INFO - šŸ“¤ SENDING BATCH: 4 traces via send_callback\n", + "2025-10-22 23:02:38 - noveum_trace.transport.http_transport - INFO - šŸš€ SENDING BATCH: 4 traces to https://api.noveum.ai/api/v1/traces\n", + "2025-10-22 23:02:38 - noveum_trace.transport.http_transport - INFO - šŸ“” HTTP RESPONSE: Status 200 from https://api.noveum.ai/api/v1/traces\n", + "2025-10-22 23:02:38 - noveum_trace.transport.http_transport - INFO - āœ… Successfully sent batch of 4 traces\n", + "2025-10-22 23:02:38 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully sent batch of 4 traces via callback\n", + "2025-10-22 23:02:38 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: f49c469d-893d-4a5a-8cf0-b5112321d143) - 1 spans\n", + "2025-10-22 23:02:38 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: f49c469d-893d-4a5a-8cf0-b5112321d143) - 1 spans\n", + "2025-10-22 23:02:38 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace f49c469d-893d-4a5a-8cf0-b5112321d143\n", + "2025-10-22 23:02:38 - noveum_trace.transport.http_transport - INFO - āœ… Trace f49c469d-893d-4a5a-8cf0-b5112321d143 successfully queued for export\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:02:38 - INFO - novaeval.evaluators.agent_evaluator - Saving intermediate results after 8 samples\n", + "2025-10-22 23:02:38 - INFO - novaeval.evaluators.agent_evaluator - Intermediate results saved to demo_results/agent.web_search_evaluation_metrics_dataset/agent_evaluation_results.csv\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Evaluating samples: 8it [00:33, 4.15s/it]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-10-22 23:02:38 - INFO - novaeval.evaluators.agent_evaluator - Saving final results\n", + "2025-10-22 23:02:38 - INFO - novaeval.evaluators.agent_evaluator - Reloaded 8 results from CSV\n", + "2025-10-22 23:02:38 - INFO - novaeval.evaluators.agent_evaluator - Agent evaluation completed\n", + "\n", + "āœ… Evaluation completed!\n", + "\n", + "šŸ“Š Results Summary:\n", + " - task_progression: 3.75\n", + " - context_relevancy: 7.84\n", + " - role_adherence: 8.97\n", + " - tool_relevancy: 0.00\n", + " - parameter_correctness: 0.00\n", + "\n", + "šŸ” Individual Scores:\n", + "\n", + " Record 1 (Task: eda4fe22-9a2b-4b73-856b-f4f3309bf719):\n", + " - task_progression: 4.2\n", + " - context_relevancy: 7.8\n", + " - role_adherence: 9.0\n", + " - tool_relevancy: 0.0\n", + " - parameter_correctness: 0.0\n", + "\n", + " Record 2 (Task: 0ffffba1-8a37-443c-8866-d53ffbfa7718):\n", + " - task_progression: 3.8\n", + " - context_relevancy: 7.8\n", + " - role_adherence: 9.0\n", + " - tool_relevancy: 0.0\n", + " - parameter_correctness: 0.0\n", + "\n", + " Record 3 (Task: 43cdf081-4f01-49cd-b566-dbd1619e6cd2):\n", + " - task_progression: 4.2\n", + " - context_relevancy: 7.8\n", + " - role_adherence: 9.0\n", + " - tool_relevancy: 0.0\n", + " - parameter_correctness: 0.0\n", + "\n", + " Record 4 (Task: 9a1983f4-09da-4b53-80e6-38de6878e0e7):\n", + " - task_progression: 2.5\n", + " - context_relevancy: 7.8\n", + " - role_adherence: 9.0\n", + " - tool_relevancy: 0.0\n", + " - parameter_correctness: 0.0\n", + "\n", + " Record 5 (Task: 5d2517e1-220a-429d-9d59-f701bda25eed):\n", + " - task_progression: 2.8\n", + " - context_relevancy: 7.8\n", + " - role_adherence: 9.0\n", + " - tool_relevancy: 0.0\n", + " - parameter_correctness: 0.0\n", + "\n", + " Record 6 (Task: a81ca3a8-80aa-4c39-876e-8d40ea7a0aef):\n", + " - task_progression: 3.8\n", + " - context_relevancy: 7.8\n", + " - role_adherence: 9.0\n", + " - tool_relevancy: 0.0\n", + " - parameter_correctness: 0.0\n", + "\n", + " Record 7 (Task: 230aad27-f3dd-4968-a45a-3c2f07ac28ed):\n", + " - task_progression: 4.2\n", + " - context_relevancy: 7.8\n", + " - role_adherence: 9.0\n", + " - tool_relevancy: 0.0\n", + " - parameter_correctness: 0.0\n", + "\n", + " Record 8 (Task: 83c7dcce-3d89-4da1-8b3f-d419885d4cbc):\n", + " - task_progression: 4.5\n", + " - context_relevancy: 8.1\n", + " - role_adherence: 8.8\n", + " - tool_relevancy: 0.0\n", + " - parameter_correctness: 0.0\n", + "āœ… Evaluation completed successfully!\n", + "\n", + "šŸ“‹ Step 7: Exporting Dataset\n", + "šŸ’¾ Exporting processed dataset...\n", + "āœ… Exported to ./processed_datasets/agent.web_search_evaluation_metrics_dataset_processed_dataset.json\n", + "āœ… Exported to ./processed_datasets/agent.web_search_evaluation_metrics_dataset_processed_dataset.csv\n", + "āœ… Export completed successfully!\n", + "\n", + "============================================================\n", + "šŸŽ‰ EVALUATION PIPELINE COMPLETED!\n", + "šŸ“Š Final Results:\n", + " - File processed: split_datasets/agent.web_search_evaluation_metrics_dataset.json\n", + " - Spans loaded: 8\n", + " - Dataset size: 8\n", + " - Evaluation completed: True\n", + " - Export successful: True\n", + " - Results saved to: ./demo_results/agent.web_search_evaluation_metrics_dataset/\n", + "Completed agent.web_search_evaluation_metrics_dataset.json\n", + "\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-10-22 23:02:43 - noveum_trace.transport.batch_processor - INFO - ā° TIMEOUT TRIGGER: Sending batch due to timeout (5.1s >= 5.0s)\n", + "2025-10-22 23:02:43 - noveum_trace.transport.batch_processor - INFO - šŸ“¤ SENDING BATCH: 1 traces via send_callback\n", + "2025-10-22 23:02:43 - noveum_trace.transport.http_transport - INFO - šŸš€ SENDING BATCH: 1 traces to https://api.noveum.ai/api/v1/traces\n", + "2025-10-22 23:02:44 - noveum_trace.transport.http_transport - INFO - šŸ“” HTTP RESPONSE: Status 200 from https://api.noveum.ai/api/v1/traces\n", + "2025-10-22 23:02:44 - noveum_trace.transport.http_transport - INFO - āœ… Successfully sent batch of 1 traces\n", + "2025-10-22 23:02:44 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully sent batch of 1 traces via callback\n" + ] + } + ], + "source": [ + "from demo_utils import run_complete_agent_evaluation\n", + "import os\n", + "\n", + "# Process all JSON files in split_datasets directory\n", + "for file in os.listdir('split_datasets'):\n", + " if file.endswith('.json'):\n", + " print(f'Processing {file}...')\n", + " run_complete_agent_evaluation(\n", + " f'split_datasets/{file}', \n", + " sample_size=25, \n", + " evaluation_name=file.replace('.json', ''),\n", + " output_dir='./demo_results'\n", + " )\n", + " print(f'Completed {file}\\n')" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "1f5ca2d3", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Created api_data.json with 20 items\n", + "Sample items:\n", + " {'item_key': 'eda4fe22-9a2b-4b73-856b-f4f3309bf719', 'item_id': 'item_1'}\n", + " {'item_key': '0ffffba1-8a37-443c-8866-d53ffbfa7718', 'item_id': 'item_2'}\n", + " {'item_key': 'f1f37bd7-0851-4659-b493-b80d3800d920', 'item_id': 'item_3'}\n" + ] + } + ], + "source": [ + "\n", + "import pandas as pd\n", + "import json\n", + "\n", + "# Read the CSV file\n", + "df = pd.read_csv('demo_results/agent.query_routing_dataset/agent_evaluation_results.csv')\n", + "\n", + "# Create API data structure with all task_ids\n", + "api_data = {\n", + " 'items': [\n", + " {\n", + " 'item_key': str(row['task_id']),\n", + " 'item_id': f'item_{i+1}' # Generate unique item IDs\n", + " }\n", + " for i, row in df.iterrows()\n", + " ]\n", + "}\n", + "\n", + "# Save to JSON\n", + "with open('api_data.json', 'w') as f:\n", + " json.dump(api_data, f, indent=2)\n", + "\n", + "print('Created api_data.json with', len(api_data['items']), 'items')\n", + "print('Sample items:')\n", + "for item in api_data['items'][:3]:\n", + " print(f' {item}')\n" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "06289a7d", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Loaded environment variables\n", + "Loading API data from api_data.json...\n", + "Loaded 20 item mappings\n", + "Reading CSV from demo_results/agent.query_routing_dataset/agent_evaluation_results.csv...\n", + "Read 20 rows from CSV\n", + "Created 20 results for upload\n", + "\n", + "Uploading 20 results in 1 batches...\n", + "āœ— Batch 1/1 failed: 201\n", + " Response: {\"success\":true,\"created\":20,\"failed\":0,\"results\":[{\"result_id\":\"c92dbfca-8820-45d4-be82-a970e7e45b14\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_1\",\"scorer_id\":\"context_relevancy_scorer\",\"score\":7.5,\"passed\":1,\"metadata\":\"{\\\"details\\\":\\\"The agent's response should provide information about the latest breakthroughs in AI. Without seeing the actual response, it's impossible to confirm alignment with role and task. However, the premise indicates that the content should be factually correct and presented in an informative manner, suggesting an appropriate tone for the given role.\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:01:06.219000000\",\"updated_at\":\"2025-10-22 18:01:06.219000000\",\"scorer_name\":\"Context Relevancy Scorer\"},{\"result_id\":\"0075462f-01a1-45b4-8113-007ae6fcde5d\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_10\",\"scorer_id\":\"context_relevancy_scorer\",\"score\":7.8,\"passed\":1,\"metadata\":\"{\\\"details\\\":\\\"The agent response's direct relevance is confirmed, matching the specified task. The alignment with the designated role is evident, and the tone and content are appropriate. There's a slight lack of demonstrating deeper understanding, which slightly reduces the score, but overall, it's well-executed and helpful.\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:01:06.219000000\",\"updated_at\":\"2025-10-22 18:01:06.219000000\",\"scorer_name\":\"Context Relevancy Scorer\"},{\"result_id\":\"12ce4442-7cff-4e80-ba5b-7c9caa1cf2b0\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_11\",\"scorer_id\":\"context_relevancy_scorer\",\"score\":7.8,\"passed\":1,\"metadata\":\"{\\\"details\\\":\\\"The agent response's direct relevance to the task is good, and it generally aligns with the agent's role. The tone and content are appropriate. However, without seeing the actual response, a slightly higher score is withheld as it's impossible to confirm the response's comprehensive understanding and perfect alignment with the task's requirements.\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:01:06.219000000\",\"updated_at\":\"2025-10-22 18:01:06.219000000\",\"scorer_name\":\"Context Relevancy Scorer\"},{\"result_id\":\"f084daf2-094b-4a0d-98ee-8b67fcba5eec\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_12\",\"scorer_id\":\"context_relevancy_scorer\",\"score\":7.8,\"passed\":1,\"metadata\":\"{\\\"details\\\":\\\"The agent response appears helpful and relevant to the assigned task of configuring Noveum. It's aligned with the agent's role and expertise. The tone and content are appropriate. However, without the actual response, a definitive assessment of its quality and level of detail is impossible; hence, a slightly lower score.\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:01:06.219000000\",\"updated_at\":\"2025-10-22 18:01:06.219000000\",\"scorer_name\":\"Context Relevancy Scorer\"},{\"result_id\":\"ccdd6d5e-d882-4ea1-b957-5a99a62fbaab\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_13\",\"scorer_id\":\"context_relevancy_scorer\",\"score\":7.8,\"passed\":1,\"metadata\":\"{\\\"details\\\":\\\"The agent response's relevance is good given the prompt, and the response is appropriate for the agent's designated role. The content's tone is also aligned with the task, and the response demonstrates good understanding of the task requirements. Further details could enhance the response.\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:01:06.219000000\",\"updated_at\":\"2025-10-22 18:01:06.219000000\",\"scorer_name\":\"Context Relevancy Scorer\"},{\"result_id\":\"f530766b-4b3d-4186-a175-e9220af2bf74\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_14\",\"scorer_id\":\"context_relevancy_scorer\",\"score\":7.8,\"passed\":1,\"metadata\":\"{\\\"details\\\":\\\"The agent response's relevance to the assigned task is clear, and it aligns well with the agent's role. The tone and content are appropriate. There's a solid understanding of the task requirements. The response is helpful. It could gain a higher score if it gives an answer.\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:01:06.219000000\",\"updated_at\":\"2025-10-22 18:01:06.219000000\",\"scorer_name\":\"Context Relevancy Scorer\"},{\"result_id\":\"5cb29b73-a4f1-48b7-b215-f300fb850b51\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_15\",\"scorer_id\":\"context_relevancy_scorer\",\"score\":7.8,\"passed\":1,\"metadata\":\"{\\\"details\\\":\\\"The agent response's relevance to the assigned task is clear, and it aligns well with the agent's specified role. The content and tone are suitable for the role. Demonstrating a more extensive understanding of available APIs in Noveum would improve the score.\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:01:06.219000000\",\"updated_at\":\"2025-10-22 18:01:06.219000000\",\"scorer_name\":\"Context Relevancy Scorer\"},{\"result_id\":\"df6a1c18-72cd-4ecf-a796-2970e989358b\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_16\",\"scorer_id\":\"context_relevancy_scorer\",\"score\":7.5,\"passed\":1,\"metadata\":\"{\\\"details\\\":\\\"The response should directly address setting up observability with Noveum, which is highly relevant to the provided task. The response is likely aligned with the agent's role and expertise. Improvement could include a more helpful response, leading to a higher score.\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:01:06.219000000\",\"updated_at\":\"2025-10-22 18:01:06.219000000\",\"scorer_name\":\"Context Relevancy Scorer\"},{\"result_id\":\"eb68976c-4108-4800-af06-83e31f0ce74c\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_17\",\"scorer_id\":\"context_relevancy_scorer\",\"score\":7.8,\"passed\":1,\"metadata\":\"{\\\"details\\\":\\\"The agent response's direct relevance is strong, fitting the task and agent role well. The content is suitable. While it efficiently addresses the query, there's room to further showcase the agent's expertise and provide a more comprehensive overview of Noveum Trace's features. A more elaborate answer could have improved the score.\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:01:06.219000000\",\"updated_at\":\"2025-10-22 18:01:06.219000000\",\"scorer_name\":\"Context Relevancy Scorer\"},{\"result_id\":\"5d535892-430c-4920-8a08-3a93431b032e\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_18\",\"scorer_id\":\"context_relevancy_scorer\",\"score\":7.8,\"passed\":1,\"metadata\":\"{\\\"details\\\":\\\"The agent response's relevance to the assigned task is clear and aligns well with the agent's role. The tone and content are appropriate. However, without the actual response, it's impossible to gauge the depth of understanding or how helpful it is in completing the task, so the score is slightly adjusted.\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:01:06.219000000\",\"updated_at\":\"2025-10-22 18:01:06.219000000\",\"scorer_name\":\"Context Relevancy Scorer\"},{\"result_id\":\"55439906-553e-4155-87d2-848d453689e2\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_19\",\"scorer_id\":\"context_relevancy_scorer\",\"score\":7.5,\"passed\":1,\"metadata\":\"{\\\"details\\\":\\\"The agent response would likely be directly relevant to the task if there was an actual response to evaluate. The prompt itself is designed to elicit a response about integrating Noveum Trace, aligning with the agent's expected role. Without a response, evaluating tone, content, and task understanding is impossible, but the prompt itself is well-suited.\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:01:06.219000000\",\"updated_at\":\"2025-10-22 18:01:06.219000000\",\"scorer_name\":\"Context Relevancy Scorer\"},{\"result_id\":\"eb624892-309d-4359-bfa2-251303cc7351\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_2\",\"scorer_id\":\"context_relevancy_scorer\",\"score\":7.8,\"passed\":1,\"metadata\":\"{\\\"details\\\":\\\"The agent response's direct relevance and its alignment with the agent's role are satisfactory. The tone and content are appropriate, showing understanding of the task. However, achieving a higher score requires a more detailed and insightful response on current trends, implying a need for better task completion progression.\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:01:06.219000000\",\"updated_at\":\"2025-10-22 18:01:06.219000000\",\"scorer_name\":\"Context Relevancy Scorer\"},{\"result_id\":\"41801712-7f2a-40be-9d29-f614143c0989\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_20\",\"scorer_id\":\"context_relevancy_scorer\",\"score\":7.8,\"passed\":1,\"metadata\":\"{\\\"details\\\":\\\"The response is relevant to the task of defining Noveum. It's well-aligned with the agent's role of providing information. The tone is appropriate. Demonstrating a slightly more detailed understanding of Noveum's function or purpose would elevate the score further, though the foundational elements of task completion are present.\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:01:06.219000000\",\"updated_at\":\"2025-10-22 18:01:06.219000000\",\"scorer_name\":\"Context Relevancy Scorer\"},{\"result_id\":\"f507b5b0-399c-4eb6-befa-af29ff56a48d\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_3\",\"scorer_id\":\"context_relevancy_scorer\",\"score\":7.5,\"passed\":1,\"metadata\":\"{\\\"details\\\":\\\"The response should list the newest features in cloud computing based on the agent's task. It should be aligned with the agent's role as well as the expertise. The tone and content are appropriate for the role. Demonstrating a deeper understanding of the task requirements would be key to increase the score.\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:01:06.219000000\",\"updated_at\":\"2025-10-22 18:01:06.219000000\",\"scorer_name\":\"Context Relevancy Scorer\"},{\"result_id\":\"ff03a298-77c5-4785-8bd1-d5a19038e745\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_4\",\"scorer_id\":\"context_relevancy_scorer\",\"score\":7.5,\"passed\":1,\"metadata\":\"{\\\"details\\\":\\\"The agent's response should provide real-time or recent information regarding the current status of the cryptocurrency markets. Given the task of providing market updates, the agent's response needs to be relevant and aligned with the role of an information provider. The score reflects a solid starting point but can improve through richer details.\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:01:06.219000000\",\"updated_at\":\"2025-10-22 18:01:06.219000000\",\"scorer_name\":\"Context Relevancy Scorer\"},{\"result_id\":\"95f84560-319e-4a2c-ad3c-f0631a1bae56\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_5\",\"scorer_id\":\"context_relevancy_scorer\",\"score\":7.8,\"passed\":1,\"metadata\":\"{\\\"details\\\":\\\"The agent response's direct relevance and role alignment is good. The tone and content are appropriate. However, the score could be higher if the response also demonstrated a helpful level of task understanding by providing specific examples or sources for the latest Python programming updates to aid task completion.\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:01:06.219000000\",\"updated_at\":\"2025-10-22 18:01:06.219000000\",\"scorer_name\":\"Context Relevancy Scorer\"},{\"result_id\":\"3e2efda3-20a2-4f71-a6f8-430f50a454a6\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_6\",\"scorer_id\":\"context_relevancy_scorer\",\"score\":1,\"passed\":1,\"metadata\":\"{\\\"details\\\":\\\"The agent provided no response, which is completely irrelevant and inappropriate. It fails to address the task of summarizing tech news and demonstrates no understanding of the requirements. Therefore, the score is significantly low, reflecting the complete lack of a meaningful answer.\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:01:06.219000000\",\"updated_at\":\"2025-10-22 18:01:06.219000000\",\"scorer_name\":\"Context Relevancy Scorer\"},{\"result_id\":\"99f927fb-447b-4eaa-8413-76932289cc37\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_7\",\"scorer_id\":\"context_relevancy_scorer\",\"score\":7.5,\"passed\":1,\"metadata\":\"{\\\"details\\\":\\\"The agent's response should provide a list of current trends in observability tools. This shows a good understanding of the task. The information should be factual and relevant. Its appropriateness hinges on the quality of the response, specifically the trends mentioned and their relevance in current times. A detailed response will increase the score.\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:01:06.219000000\",\"updated_at\":\"2025-10-22 18:01:06.219000000\",\"scorer_name\":\"Context Relevancy Scorer\"},{\"result_id\":\"a0c16e26-c4d0-4765-afe0-c9a007fd1481\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_8\",\"scorer_id\":\"context_relevancy_scorer\",\"score\":7.5,\"passed\":1,\"metadata\":\"{\\\"details\\\":\\\"The response is relevant to the task of describing recent developments in machine learning, fitting the agent's role. The tone and content are appropriate. However, the depth of the response could be increased to demonstrate a stronger understanding of the current state of the field to get a better score.\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:01:06.219000000\",\"updated_at\":\"2025-10-22 18:01:06.219000000\",\"scorer_name\":\"Context Relevancy Scorer\"},{\"result_id\":\"696c0bee-158d-4134-8785-19503ec01da3\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_9\",\"scorer_id\":\"context_relevancy_scorer\",\"score\":7.8,\"passed\":1,\"metadata\":\"{\\\"details\\\":\\\"The agent response's relevance is good given the weather query. It's appropriately aligned with the role, providing information about the weather. The tone is suitable, indicating task understanding. To improve, it could offer a bit more detail, but overall it is a helpful start.\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:01:06.219000000\",\"updated_at\":\"2025-10-22 18:01:06.219000000\",\"scorer_name\":\"Context Relevancy Scorer\"}]}\n", + "\n", + "Upload complete!\n" + ] + } + ], + "source": [ + "!python upload_scores.py demo_results/agent.query_routing_dataset/agent_evaluation_results.csv --item-key-col task_id --score-col context_relevancy --reasoning-col context_relevancy_reasoning --api-data api_data.json --scorer-id context_relevancy_scorer" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "id": "cbf96327", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Loaded environment variables\n", + "Loading API data from api_data.json...\n", + "Loaded 20 item mappings\n", + "Reading CSV from demo_results/agent.query_routing_dataset/agent_evaluation_results.csv...\n", + "Read 20 rows from CSV\n", + "Created 20 results for upload\n", + "\n", + "Uploading 20 results in 1 batches...\n", + "āœ— Batch 1/1 failed: 201\n", + " Response: {\"success\":true,\"created\":20,\"failed\":0,\"results\":[{\"result_id\":\"bcc832a3-fda1-4129-8988-5ba5a30ed2f0\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_1\",\"scorer_id\":\"role_adherence_scorer\",\"score\":1,\"passed\":1,\"metadata\":\"{\\\"details\\\":\\\"The agent completely fails to adhere to its role as it provided no response and made no tool calls. There is no evidence of an attempt to answer the question about AI breakthroughs. The agent's lack of any output indicates a total disregard for the task and role instructions.\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:01:48.477000000\",\"updated_at\":\"2025-10-22 18:01:48.477000000\",\"scorer_name\":\"Role Adherence Scorer\"},{\"result_id\":\"7f5cdc48-40bb-4944-ae1d-c4e9d3ace92d\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_10\",\"scorer_id\":\"role_adherence_scorer\",\"score\":1,\"passed\":1,\"metadata\":\"{\\\"details\\\":\\\"The agent completely fails to adhere to its role. It provides no response or tool calls, indicating a complete inability to fulfill its assigned task of providing AI news. It acts inappropriately by failing to respond, showing no understanding or attempt to interact with any available tools.\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:01:48.477000000\",\"updated_at\":\"2025-10-22 18:01:48.477000000\",\"scorer_name\":\"Role Adherence Scorer\"},{\"result_id\":\"a862d4fb-240a-4682-b8f1-2f68eb386142\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_11\",\"scorer_id\":\"role_adherence_scorer\",\"score\":9,\"passed\":1,\"metadata\":\"{\\\"details\\\":\\\"The agent perfectly adheres to its role, providing a concise response about the benefits of using Noveum Trace. The absence of tool calls indicates no deviation from the provided context. The response is consistent, focused, and free of any role contradictions or inappropriate behaviors, demonstrating a strong understanding of its assigned task.\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:01:48.477000000\",\"updated_at\":\"2025-10-22 18:01:48.477000000\",\"scorer_name\":\"Role Adherence Scorer\"},{\"result_id\":\"e8604ecb-b171-45b0-aab3-1fa128c734e4\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_12\",\"scorer_id\":\"role_adherence_scorer\",\"score\":1,\"passed\":1,\"metadata\":\"{\\\"details\\\":\\\"The agent provided no response, and therefore failed to adhere to the role or address the task. It did not provide any information or utilize any tools. The lack of response demonstrates complete failure in understanding and executing the assigned instructions.\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:01:48.477000000\",\"updated_at\":\"2025-10-22 18:01:48.477000000\",\"scorer_name\":\"Role Adherence Scorer\"},{\"result_id\":\"b0ef13bb-bf66-47b9-9458-b5d52d1488fb\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_13\",\"scorer_id\":\"role_adherence_scorer\",\"score\":10,\"passed\":1,\"metadata\":\"{\\\"details\\\":\\\"The agent perfectly adheres to its role and task. The agent's response is relevant to its role and the tool calls are appropriate for this role and its task. The agent doesn't contradict its assigned task. No deviations or inconsistencies are observed.\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:01:48.477000000\",\"updated_at\":\"2025-10-22 18:01:48.477000000\",\"scorer_name\":\"Role Adherence Scorer\"},{\"result_id\":\"10dcd5c4-9fb7-4670-98a6-b858caacd938\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_14\",\"scorer_id\":\"role_adherence_scorer\",\"score\":1,\"passed\":1,\"metadata\":\"{\\\"details\\\":\\\"The agent completely fails to fulfill its role. The agent provides no response and makes no tool calls, indicating a complete disregard for the assigned task. There's no evidence of role adherence, task completion, or relevant information retrieval. The agent is inactive, rendering the evaluation impossible.\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:01:48.477000000\",\"updated_at\":\"2025-10-22 18:01:48.477000000\",\"scorer_name\":\"Role Adherence Scorer\"},{\"result_id\":\"fb9c6145-3e19-493a-808e-884484daf019\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_15\",\"scorer_id\":\"role_adherence_scorer\",\"score\":10,\"passed\":1,\"metadata\":\"{\\\"details\\\":\\\"The agent perfectly adheres to its role and task. The agent correctly identified that there were no tools calls, as it was a prompt and not a request for a specific action. The response is consistent with the expectations of the assigned role.\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:01:48.477000000\",\"updated_at\":\"2025-10-22 18:01:48.477000000\",\"scorer_name\":\"Role Adherence Scorer\"},{\"result_id\":\"0a8d4083-d2fe-4083-b161-e8fddd8f231f\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_16\",\"scorer_id\":\"role_adherence_scorer\",\"score\":1,\"passed\":1,\"metadata\":\"{\\\"details\\\":\\\"The agent provided no response or tool calls, therefore failing to adhere to any aspect of its role and task. It did not attempt to answer the question or make any progress towards the objective. The lack of any action completely contradicts the expectation for a helpful response.\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:01:48.477000000\",\"updated_at\":\"2025-10-22 18:01:48.477000000\",\"scorer_name\":\"Role Adherence Scorer\"},{\"result_id\":\"5dc8fa70-5bab-4697-81ea-745e24680c3d\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_17\",\"scorer_id\":\"role_adherence_scorer\",\"score\":10,\"passed\":1,\"metadata\":\"{\\\"details\\\":\\\"The agent perfectly adheres to its role and task. The agent has not provided a response nor made any tool calls, which is the expected behavior given the prompt which instructed it to evaluate an agent's response. No deviations or contradictions were observed, maintaining complete consistency.\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:01:48.477000000\",\"updated_at\":\"2025-10-22 18:01:48.477000000\",\"scorer_name\":\"Role Adherence Scorer\"},{\"result_id\":\"7c645c45-f5a4-41a3-ab8f-51168c02856d\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_18\",\"scorer_id\":\"role_adherence_scorer\",\"score\":1,\"passed\":1,\"metadata\":\"{\\\"details\\\":\\\"The agent provided no response, and made no tool calls, meaning the agent completely failed to fulfill its role or task. It did not try to answer the question about pricing plans or attempt any data analysis or retrieval. This constitutes a complete failure to adhere to the role.\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:01:48.477000000\",\"updated_at\":\"2025-10-22 18:01:48.477000000\",\"scorer_name\":\"Role Adherence Scorer\"},{\"result_id\":\"ca5a3ea3-7023-4a11-b12e-07bf43d91e4d\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_19\",\"scorer_id\":\"role_adherence_scorer\",\"score\":1,\"passed\":1,\"metadata\":\"{\\\"details\\\":\\\"The agent provided no response, meaning the agent failed at every aspect of its role. It did not provide a response, tool calls, or do anything to accomplish its assigned task. The agent completely ignored all instructions and failed to operate as requested. Therefore, the adherence score is very low.\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:01:48.477000000\",\"updated_at\":\"2025-10-22 18:01:48.477000000\",\"scorer_name\":\"Role Adherence Scorer\"},{\"result_id\":\"497d5631-bb51-415d-84b8-ea1d0669768c\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_2\",\"scorer_id\":\"role_adherence_scorer\",\"score\":10,\"passed\":1,\"metadata\":\"{\\\"details\\\":\\\"The agent perfectly adheres to its role and task. The response is consistent with expectations for the assigned role. The absence of any tool calls is acceptable as there is no requirement to make a tool call in this scenario, therefore the agent has no need to deviate from its role.\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:01:48.477000000\",\"updated_at\":\"2025-10-22 18:01:48.477000000\",\"scorer_name\":\"Role Adherence Scorer\"},{\"result_id\":\"2510458b-3a84-4f67-b3c6-8d55dfb2ba92\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_20\",\"scorer_id\":\"role_adherence_scorer\",\"score\":10,\"passed\":1,\"metadata\":\"{\\\"details\\\":\\\"The agent perfectly adheres to its role and task, maintaining consistency throughout. The absence of any tool calls implies that the agent may have responded directly with the answer to the question. This aligns well with the prompt, suggesting an exact response is expected given the lack of tool usage. No deviations from the role or inappropriate behaviors are observed.\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:01:48.477000000\",\"updated_at\":\"2025-10-22 18:01:48.477000000\",\"scorer_name\":\"Role Adherence Scorer\"},{\"result_id\":\"37bb1928-0b4e-4eff-9c9b-99c374df758a\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_3\",\"scorer_id\":\"role_adherence_scorer\",\"score\":10,\"passed\":1,\"metadata\":\"{\\\"details\\\":\\\"The agent perfectly adheres to its role and task. The absence of tool calls indicates that the agent knows to respond without them. The agent does not stray from the assigned task and provides an expected answer while acting appropriately as an agent.\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:01:48.477000000\",\"updated_at\":\"2025-10-22 18:01:48.477000000\",\"scorer_name\":\"Role Adherence Scorer\"},{\"result_id\":\"56b5f6b9-aa35-4c1e-9380-4f6e42d37ef6\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_4\",\"scorer_id\":\"role_adherence_scorer\",\"score\":10,\"passed\":1,\"metadata\":\"{\\\"details\\\":\\\"The agent perfectly adheres to its role and task. The absence of tool calls indicates no actions were taken, which aligns with the prompt requesting an evaluation without any tool usage. The response, while missing, doesn't deviate from the intended role of reporting current crypto market status. The consistency is flawless.\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:01:48.477000000\",\"updated_at\":\"2025-10-22 18:01:48.477000000\",\"scorer_name\":\"Role Adherence Scorer\"},{\"result_id\":\"00ffa56b-dee2-4798-adac-c9b67c04409b\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_5\",\"scorer_id\":\"role_adherence_scorer\",\"score\":1,\"passed\":1,\"metadata\":\"{\\\"details\\\":\\\"The agent provided no response and made no tool calls. This is a complete failure to adhere to the role and task. The agent should have provided some information, even if it had to use a tool to gather it. The lack of any output demonstrates a fundamental misunderstanding of the prompt's requirements.\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:01:48.477000000\",\"updated_at\":\"2025-10-22 18:01:48.477000000\",\"scorer_name\":\"Role Adherence Scorer\"},{\"result_id\":\"19e7022b-a731-4bb7-ba91-ea1f15acea19\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_6\",\"scorer_id\":\"role_adherence_scorer\",\"score\":1,\"passed\":1,\"metadata\":\"{\\\"details\\\":\\\"The agent completely fails to adhere to its role. It's supposed to summarize tech news but hasn't provided any information or made any tool calls. Its response is blank, and thus it cannot fulfill its assigned task. There are significant role contradictions and inappropriate behaviors due to this complete lack of output.\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:01:48.477000000\",\"updated_at\":\"2025-10-22 18:01:48.477000000\",\"scorer_name\":\"Role Adherence Scorer\"},{\"result_id\":\"1855df2f-4341-4e34-b02b-70b4f0844690\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_7\",\"scorer_id\":\"role_adherence_scorer\",\"score\":1,\"passed\":1,\"metadata\":\"{\\\"details\\\":\\\"The agent completely fails to adhere to its assigned role or perform the task. No tool calls were made, indicating a complete lack of engagement with the prompt. There is no information provided about the current trends in observability tools, directly contradicting the instructions. The agent fails at all aspects of the evaluation.\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:01:48.477000000\",\"updated_at\":\"2025-10-22 18:01:48.477000000\",\"scorer_name\":\"Role Adherence Scorer\"},{\"result_id\":\"13771286-0139-4c43-9287-c3290f402da1\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_8\",\"scorer_id\":\"role_adherence_scorer\",\"score\":10,\"passed\":1,\"metadata\":\"{\\\"details\\\":\\\"The agent perfectly adheres to its role and task. The response is consistent with the role of machine learning expert. The agent appropriately answers the prompt, delivering a relevant overview. No tool calls were made, suggesting that the agent could answer the query without external assistance, which aligns with perfect role adherence.\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:01:48.477000000\",\"updated_at\":\"2025-10-22 18:01:48.477000000\",\"scorer_name\":\"Role Adherence Scorer\"},{\"result_id\":\"a66d4346-5a43-4b06-a269-216bcc40f03e\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_9\",\"scorer_id\":\"role_adherence_scorer\",\"score\":10,\"passed\":1,\"metadata\":\"{\\\"details\\\":\\\"The agent perfectly adheres to its role and task. The agent correctly executes its role and the tool calls are appropriate for retrieving information about the weather, as intended. The response is consistent with the role and task requirements, showing no contradictions or inappropriate behaviors. The agent demonstrates flawless adherence.\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:01:48.477000000\",\"updated_at\":\"2025-10-22 18:01:48.477000000\",\"scorer_name\":\"Role Adherence Scorer\"}]}\n", + "\n", + "Upload complete!\n" + ] + } + ], + "source": [ + "!python upload_scores.py demo_results/agent.query_routing_dataset/agent_evaluation_results.csv --item-key-col task_id --score-col role_adherence --reasoning-col role_adherence_reasoning --api-data api_data.json --scorer-id role_adherence_scorer" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "08f948b3", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Loaded environment variables\n", + "Loading API data from api_data.json...\n", + "Loaded 20 item mappings\n", + "Reading CSV from demo_results/agent.query_routing_dataset/agent_evaluation_results.csv...\n", + "Read 20 rows from CSV\n", + "Created 20 results for upload\n", + "\n", + "Uploading 20 results in 1 batches...\n", + "āœ— Batch 1/1 failed: 201\n", + " Response: {\"success\":true,\"created\":20,\"failed\":0,\"results\":[{\"result_id\":\"35b42464-7989-4caa-90cb-5636319d4fd1\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_1\",\"scorer_id\":\"parameter_correctness_scorer\",\"score\":0,\"passed\":0,\"metadata\":\"{\\\"details\\\":\\\"Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:03:01.353000000\",\"updated_at\":\"2025-10-22 18:03:01.353000000\",\"scorer_name\":\"Parameter Correctness Scorer\"},{\"result_id\":\"28380f39-f4e4-41a3-a5c7-01241174fb78\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_10\",\"scorer_id\":\"parameter_correctness_scorer\",\"score\":0,\"passed\":0,\"metadata\":\"{\\\"details\\\":\\\"Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:03:01.353000000\",\"updated_at\":\"2025-10-22 18:03:01.353000000\",\"scorer_name\":\"Parameter Correctness Scorer\"},{\"result_id\":\"f1ea8545-c1ab-4791-958b-04d3c75cf92a\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_11\",\"scorer_id\":\"parameter_correctness_scorer\",\"score\":0,\"passed\":0,\"metadata\":\"{\\\"details\\\":\\\"Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:03:01.353000000\",\"updated_at\":\"2025-10-22 18:03:01.353000000\",\"scorer_name\":\"Parameter Correctness Scorer\"},{\"result_id\":\"db8c7724-19ca-46c6-8c6b-796f229279e4\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_12\",\"scorer_id\":\"parameter_correctness_scorer\",\"score\":0,\"passed\":0,\"metadata\":\"{\\\"details\\\":\\\"Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:03:01.353000000\",\"updated_at\":\"2025-10-22 18:03:01.353000000\",\"scorer_name\":\"Parameter Correctness Scorer\"},{\"result_id\":\"86929a3d-38c7-4bcf-8c40-820f3d9e69ef\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_13\",\"scorer_id\":\"parameter_correctness_scorer\",\"score\":0,\"passed\":0,\"metadata\":\"{\\\"details\\\":\\\"Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:03:01.353000000\",\"updated_at\":\"2025-10-22 18:03:01.353000000\",\"scorer_name\":\"Parameter Correctness Scorer\"},{\"result_id\":\"068aaa68-b454-47db-854d-c09b87d33ec9\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_14\",\"scorer_id\":\"parameter_correctness_scorer\",\"score\":0,\"passed\":0,\"metadata\":\"{\\\"details\\\":\\\"Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:03:01.353000000\",\"updated_at\":\"2025-10-22 18:03:01.353000000\",\"scorer_name\":\"Parameter Correctness Scorer\"},{\"result_id\":\"b80aac89-5007-412c-a372-d7a80703ec0f\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_15\",\"scorer_id\":\"parameter_correctness_scorer\",\"score\":0,\"passed\":0,\"metadata\":\"{\\\"details\\\":\\\"Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:03:01.353000000\",\"updated_at\":\"2025-10-22 18:03:01.353000000\",\"scorer_name\":\"Parameter Correctness Scorer\"},{\"result_id\":\"07585067-3e96-4491-8f4b-a4fa89b80493\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_16\",\"scorer_id\":\"parameter_correctness_scorer\",\"score\":0,\"passed\":0,\"metadata\":\"{\\\"details\\\":\\\"Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:03:01.353000000\",\"updated_at\":\"2025-10-22 18:03:01.353000000\",\"scorer_name\":\"Parameter Correctness Scorer\"},{\"result_id\":\"172395b1-bd1d-43dd-9359-51854355376b\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_17\",\"scorer_id\":\"parameter_correctness_scorer\",\"score\":0,\"passed\":0,\"metadata\":\"{\\\"details\\\":\\\"Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:03:01.353000000\",\"updated_at\":\"2025-10-22 18:03:01.353000000\",\"scorer_name\":\"Parameter Correctness Scorer\"},{\"result_id\":\"a54ae11d-5232-4434-b704-1a5f8b327ee7\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_18\",\"scorer_id\":\"parameter_correctness_scorer\",\"score\":0,\"passed\":0,\"metadata\":\"{\\\"details\\\":\\\"Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:03:01.353000000\",\"updated_at\":\"2025-10-22 18:03:01.353000000\",\"scorer_name\":\"Parameter Correctness Scorer\"},{\"result_id\":\"61847a0c-de20-4e36-abd0-2067bb210207\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_19\",\"scorer_id\":\"parameter_correctness_scorer\",\"score\":0,\"passed\":0,\"metadata\":\"{\\\"details\\\":\\\"Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:03:01.353000000\",\"updated_at\":\"2025-10-22 18:03:01.353000000\",\"scorer_name\":\"Parameter Correctness Scorer\"},{\"result_id\":\"70f241a7-05cc-47df-8bc3-2eb3b428d064\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_2\",\"scorer_id\":\"parameter_correctness_scorer\",\"score\":0,\"passed\":0,\"metadata\":\"{\\\"details\\\":\\\"Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:03:01.353000000\",\"updated_at\":\"2025-10-22 18:03:01.353000000\",\"scorer_name\":\"Parameter Correctness Scorer\"},{\"result_id\":\"1b5c2018-e801-4ac1-98e9-8dab56fa19ed\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_20\",\"scorer_id\":\"parameter_correctness_scorer\",\"score\":0,\"passed\":0,\"metadata\":\"{\\\"details\\\":\\\"Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:03:01.353000000\",\"updated_at\":\"2025-10-22 18:03:01.353000000\",\"scorer_name\":\"Parameter Correctness Scorer\"},{\"result_id\":\"65aa0719-a555-4fae-8203-d3f06bcd0f5e\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_3\",\"scorer_id\":\"parameter_correctness_scorer\",\"score\":0,\"passed\":0,\"metadata\":\"{\\\"details\\\":\\\"Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:03:01.353000000\",\"updated_at\":\"2025-10-22 18:03:01.353000000\",\"scorer_name\":\"Parameter Correctness Scorer\"},{\"result_id\":\"5e710451-f3d1-4d35-8ed2-8c6b54a46ccb\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_4\",\"scorer_id\":\"parameter_correctness_scorer\",\"score\":0,\"passed\":0,\"metadata\":\"{\\\"details\\\":\\\"Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:03:01.353000000\",\"updated_at\":\"2025-10-22 18:03:01.353000000\",\"scorer_name\":\"Parameter Correctness Scorer\"},{\"result_id\":\"1db2a945-e30a-43ee-b591-c98dbe004f47\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_5\",\"scorer_id\":\"parameter_correctness_scorer\",\"score\":0,\"passed\":0,\"metadata\":\"{\\\"details\\\":\\\"Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:03:01.353000000\",\"updated_at\":\"2025-10-22 18:03:01.353000000\",\"scorer_name\":\"Parameter Correctness Scorer\"},{\"result_id\":\"297fadf1-51fa-43e7-91cc-58b783550d3b\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_6\",\"scorer_id\":\"parameter_correctness_scorer\",\"score\":0,\"passed\":0,\"metadata\":\"{\\\"details\\\":\\\"Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:03:01.353000000\",\"updated_at\":\"2025-10-22 18:03:01.353000000\",\"scorer_name\":\"Parameter Correctness Scorer\"},{\"result_id\":\"f00da0b9-f3df-4055-9507-0b4b9ae31501\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_7\",\"scorer_id\":\"parameter_correctness_scorer\",\"score\":0,\"passed\":0,\"metadata\":\"{\\\"details\\\":\\\"Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:03:01.353000000\",\"updated_at\":\"2025-10-22 18:03:01.353000000\",\"scorer_name\":\"Parameter Correctness Scorer\"},{\"result_id\":\"3f9733b7-23b3-4396-84a6-5f87146cb0a2\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_8\",\"scorer_id\":\"parameter_correctness_scorer\",\"score\":0,\"passed\":0,\"metadata\":\"{\\\"details\\\":\\\"Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:03:01.353000000\",\"updated_at\":\"2025-10-22 18:03:01.353000000\",\"scorer_name\":\"Parameter Correctness Scorer\"},{\"result_id\":\"c6629883-f239-4b64-ba54-2fe517d25551\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_9\",\"scorer_id\":\"parameter_correctness_scorer\",\"score\":0,\"passed\":0,\"metadata\":\"{\\\"details\\\":\\\"Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:03:01.353000000\",\"updated_at\":\"2025-10-22 18:03:01.353000000\",\"scorer_name\":\"Parameter Correctness Scorer\"}]}\n", + "\n", + "Upload complete!\n" + ] + } + ], + "source": [ + "!python upload_scores.py demo_results/agent.query_routing_dataset/agent_evaluation_results.csv --item-key-col task_id --score-col parameter_correctness --reasoning-col parameter_correctness_reasoning --api-data api_data.json --scorer-id parameter_correctness_scorer" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "83466df7", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": ".venv", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.9" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/noveum_customer_support_bt/noveum_vectorstore/index.faiss b/noveum_customer_support_bt/noveum_vectorstore/index.faiss new file mode 100644 index 0000000000000000000000000000000000000000..83cb625da4eb1c8330cf8e5b1637edc3b48f3738 GIT binary patch literal 1130541 zcmXtA30RF=)GjGeh@_~9NSP(n*=v)CWQYtQM9Dl|^Gu0ER1`{vh)5zD&tBV*v5bjG zBr?xLX8yJN|NlOZ=U&}&zHjff-}Mgbe5Zbk`#9t_q8na@<)%aA%il?MWg; zjjqA{U9Q2*!s)O*zYDE#AB&ihihjCXv z?gl)b0rKLofzrCh0t~$@(5>A%j4v?IzBv;QS0DA2p5LdzjJCG?da5p8n%$WD4{L;< zhQ6*^Pb_Z6Hg$W+?QPDensK*zy`gpM2zF~<91LGH3^Or|j$d?5o z_VKkb<~`eq$A*l6;umJ}*om^L8U{Dshjn^Q!rJlHFtBzxX3lIX6PFCoEQQ|OKf@N@ zTNY{;y&8}Di-Y9wqr=hKeY`AUzfix}h=t5t3!w+%HJ{A>xwOnZDVJ~mDZ12}fx~Zp z0k38Cv1ES{+a8{eC9Av1!%es1*t%o+wBplZL+)gZebi3olsChKRW=eQd_$uPkaQvg(x08PX?cT%>s{kWfNB6`kaQcxTS8@+7kBlkZ&djLc6_D7&zzxbi8*62^WwM zT7ll58t}yUHzGkAakJn-lE3W7b1n}@ze9hS!uxi^1pd9wVX$}Zid|eoAZ*kIc>XPu z_NFUMJ4}b7Th_AV_a^>ori;upj)6??LVP>};qSugQl0<#9}Rg(kw5=?ceAjq(}x?) z7=@`uJLL90#`3V6BdswAN?|e1WtX7T=>X=1)RXSUlelx!r<&+tosk<&<*uhkaJsJ0 z>FkbSmU}eQH2Gqnz6p0eHA-5H@&(TmZQ*LbCsdfx88HznnkT}HD?40f?`sJ-Dp9;? z)+%T6`AaONP)CNo8Or6PUD~C@u=dIt`viRX0q0N_tB4o!m=GgfWPWxiSi~ci( z$q9!r;_clkod23&43n#!1dsB$jQS84UF<2^^}SDPJjqfcdh&|1u`FfR6dd@vmw4s< zj&+RVa$4qI#TCMAS7UjxQ!4no41|O+x-j_ZY+kT-vq*n1m@pPD3}Xgx!kjp~{-Y*J zL}A*bLE!wmH+bIZFD}k*#EYU&h(*Nn2b&$|Fg&UmYZ-DGD|>tZ(?C5u*DY9d z3E0ot`P*@y3Fk!1+|EK_zo_#eC~+IW?bE~2&nj5_%H76HU!~yW0M0h-@!-L`uRw6) z=9p$|6TK9`d;MUH76K=$`Qx!-(1RWPO0lKH8v&7{QNtl|v2|SvWvORsngnz?S zZ1>zN=hQ4KKDNdN*pT~zB~5S1Mh~@uvrl(}l9WUAkI@zp#*eZk+)uf?hMEPm@wI2<^H`oc_f%m}N( zcIbw|P+TiRl*V+RzV5*DIFAFlr%l6@T(X zAq9T*afQxb=Yj1}0eUYSt2D->zKx{)$b*MX6nCSAqf2L*H1E57Rq$OJRLV>th+ znSA`Rz6`(n2px}QK;K5hQ6+16M2$qGeg)zP?6NKwW*#4aq#0;?gA2`3=Ugo2dgB5F1VcOOpd^Ix%Eo=9aCHqG}=;So0{LxXq>Tn%&qHhSjfSSB@ zYBtQ@oD9@YboO8H%1=|Su-L`J2h_RbY?!90ZF7nFwYbBGU-|QSwPi|73@i%l21$cQ zarZ!LY1`=;sx$xl^C+e-yQnGr6$xED?h+U8fr1txVA6P=EL^446dXDVq|-oak+J6| zpw|N{NZ%bOD$X`yRC7!yYYp+4TFva_?`Yb}g1EC7sE1+N?{Q%3P+uzDQ(Wl(YXbR` z#(eh6avc40qUvqp)bA|MwgyBmx}`B)zYl_WPf3^p>Ti+Iq#=(@$i?VI-l)z(v&mEz zTp7T;BWoemgYnSQV*9&ckY^}BX;5HzM>)CWE%wQ56_78HsxR{fErG}F3y|+K;;HSs z;iAASa5Qm+!k%9I`>+?($Lkn*6r{eDIl<<#JaMRYc4#pez3d<=e;j9ZdOe54XbWD% zmWhJC`*HTm0OGBa5I(D#a31T0dI4cDZRaqiw2?Ft|GK^v)GthFm%SNXuU9{qQ8NUt ztmNFWvmgJy#T$R=RFhp^3_$W(EFp9ranLjNeA##;{>b!Nwgv5fdJ+$}mj%x)Wbn6Y zK>Y-XS=|tq_k!UEs^PR#^VGUX=c@DQxc}tUga@!HNQZ|85X_H*_WCILL z)Cb3WT`=mWMe<^rih~2;{rAqCxJ~JXAPt0z89w0IwFuv9J-GY+^-%t&76yL{)}%dY zA#0Cw%u)ABTyhoDyR65ttsP{hrw+XN8w$2{F2ZPBg5-yosyAT_JwKW&kA#QaPN32_ zQ_n0d>80pf(iAJEJOScomsR@7qVS?s)=f zr`Mv%?R4hY##s0~n*mXKFJQsiIwJbs1B~jyz{ctwb8qD+sc!`JK39FLd`H@8eV&+= zCB6@c)>x*T0`jWlWw$UJtC{Rvw`%CV-w@v~e~2Du3gP+4vpENAS;~a6WHhWCA<4rb z@qn0XaY$(i{PoS&oFCnc?z>r|b4$yA9M@ zT5QIadm3QQnQGGh(Mo#0F%A27}Qh#)T}&C2gVN{kuy&DLR2R#q%9-;%C-3?3&Zq2qt~F!w%yWlDD$+ItH) zn@7Rn)w{sYsu~nbb3?P>>locBT5*w3e#g7WLxxW7#JxM{$UQZ80<>@++t+Rz?~FtE)*g-h|?bT#vCkdAL0e!5UT;iK=mtMg{cQ`74qeygQ+ zv{{;_VAEqHPGY~NK7x<6Q*cJj>YVhJ{K7(QVEAzj`C+h+J%b^UEzrmIp2)m@7|yyo zAYGpythEzOx|~)%6_sy4|6?1TTKEB?Go;cG@$yw&PCTS}p3@)b{Lt~;Ud44tb*_57 z(j#>)5qZvndR&fCIzE-$r`aA_OhNnsUkls4!k` zze1yQ;OxNP!t(WDER-#|f1D4}`AYhk79U;{l&_ROQ(Q{xx`t-W&N~O*nuRKlSU;#H z;|tcITbE+2TyY;wXO4igQ#@QcHkk}BUqxux}0`Bl@Dv=#db8Zwy~r;zGHkOc18Az-ysYJzqX9Mz}s%Abe~`a>Kdp z9*3@1xQ3F|SyQ(@}5D+udNWaH}7L=zconO36CRu6^}qc+q=}ihD!4^!TfAhwzKB7L9gX*#mh|P z1j=tnei6evnnQTHtI#pAgZa}pGR1`{ek+Iz|1sAmU%`H1Gj3b4M5K>w#a&rD@UPfR zvyT33@{8w;{37-KWo&8qS$PgrnoTnenDM@w9NYRZJRb5<3?Fa>(|11;2B&&Rg} zw=_l*4J2U#{qMQKs<5Fdmx1N1LL{Gzj)RWitH^s;7`IM%>~ez93k7)XPE&t}$^|tP z@1wq^Ay|H1u71XFR12V74{BCG*-?33%**%ER4xdGyrkn=;vJmf<%X>Xw#U?KDO$=1 z%yo$cGkm{BJm(7-*W0F17$yAYMCslH+u1i6X(lJHgi0@+n@)swH9rV7=b5e75x8DM zX??aUI^Mbs9*~UQ3r>MS^}h_e>MMT^N>iK{TSB{Y9jkKe;q`NnaL>sj@auE7sUF6q zJ|>*<6>;TKpd2H}KPlhK2S5FwCGDqdx?1~w{stib3XX%?fb-+Ug7lTU4L-osxw;4L zg3`H>NbAmV`w)jT&k?6?*kSDX$BeM6CBG+BHdC2?QLC%yc_KxmH0lB5f2%wlboKLcdES4%%FO7x zWQS%$$Z{kPiNXJdvx$Z^xaY_1=(oa{u<#n5*Vhw?Uzql4(q+H;FGYj7Ues2V%XPU~&PW%kdvQ z#r!0s+#}9TxlDPbwKQp5kxl+hTs#`0G>!fJ*$$MCyuP&y`s7}tzDkE*6T&fe<$ghP zAdz@~1QL(q*)b>*#Xk1mu70l7 zIcACElU>wIbL>D5sbjGO2~!%H%V4QbZxL}Y7DGO@M>=;PUjkG!K|X*H=Mip`$-C|m z?bcpU9Kh6>63=1Ll1Z8?Xb($Eawz*B)~J~%aXhNI33*0T^Dojqmeac}7&P@H-*!w; z?j>GIN5y;O?FHppB<{z<`w`Q>S%LCJjishj+3D2(lQ$hC(eRX zpWRvcq?brJf^{tW#jX$cq}kUiM)MhIFgC-5&IP9}?gDzXyy5vgnpynJN19oRN~^)# zKGu~b^z6c|$EVQTc@)GizM=MyZRT7fZLSTZp~5kI8&kfG@+c5zsCg>Y`8*?kseGYk z!=B4(&c%|hb`a!0*!3@`!KSD^&oStU+%AKWKLFA$RygrJ^RZmZlpcJ4Q~tB)=*(++QtG?G)kWNIFCHqX0? z%g7I?Suq;6=`HPB9E8a~jd;b^TjE|?F>`EsSd(;jFYJHQk{>U=1rw)q59%qHQpi=fmEAt$DH066hAx4E;vgZY~e5AU{>3y4I)uQ&zI=0Ic$^C4XM?kbQ?W z=C)Jnz-hjs>biPX!0*hRh5Z(_Voe=vp;ladX>vRdZ=X5_+mhGIIh#Jf=eB`z+20Q= z?`L;tn;8UujMJfHTT2Mv+6vp3Nr>F3gT>dnNzEB!-gcTRPd(O98t58|^#g0mmn-&* z4f#2Y-jA_Cx1p?j7ECA_CiXeJz=Gm(5tdrQMi04>lhf)VWU~IaiZc7FK2dmZinr(= z=fsL_4q^H7AFOzug{1q-7q->q&D7=DMlX@}1;4j-WU~0ZS z@7E$rB!wJ@6TkiV>dT>+RC)vdoQTHLku&+uZ$qGAkO|%!_X<3$4Ta90y5Kvr9Xz?L zl^a_WV?;|m8JPDMrwnT&&yFh5*ypchz7o!eb5`B+b8XD@=X{&k@7rPeSb z;S1~tti&h}6Yx8EN%)+u&>EYc#>;(HL&nwRSkkKtFTXKWe!cKP&^_?um2E(4L*FyI z(ARV`RKNKX{CAkb;-FqQ)b15F(w`v-L->1FAky_^!`nJJL+z%5Jabb#?Vc?R+nks0 z4~5~YioLkQ#(=X+45FQk_|%Lg_y+6977J&vW}i*?{iTVdVsbdvF=9hs?HOZjA$ zrL>CNAw7Kip=;4ORO_ly_c_eb{1a^hqQJiHA{>2W7KFzYA-zWu9$C+apK+m24RBrF zLXJ-C0qR_RP3!P46PwAHm||!hXDch))#XntkKor+XRv;|g`l%xJb!sN7c}~R#MSUP z=vHzDZ{Bi}dmywb;(nt<_nFRm+mZ$7GXKH1-Vm5o-8MyU+3t>MC z9-PpV)OUROrH2~dkt>sf9!sg*)xHoPpUaw}M{_w3A z5XRp)AgbkB$(c2yCG`_O|0Yvfmapg5qt=PGCJ8wGT?VfD7mL(8IRSgzWYJ^9P8Y&x zFVkEy|AeuKwkH_flYJj-C8(FgeRNz{=%@BEa|#Dk$NE%&n=$pbKDYM))lMF<7BGG5HeBQG1lOLsK;`}20{yD7 zZ~MdH#gzH{PSd~WbGx~mnYkbK7gegd%VUeH!%W%2V5Yh^bY~A-H5ccrQKdS$8 z1FrVrvs8~~&#NIL*A)O^8m)I%vZl+og0p7?Y2SNsz1j}wsCNa59)rxEx=EcInALaV zh6OF8>XV3;Yh1?6e}MLj8VcXWoH)c44SZYSo&!eU(`u{yqp#0e$I)jDaYZn(PL%ZO z+yk0@j%439kAumyhi>I3SY$>%HyZ|0KtL}WNwA!$DM|LiF1l%*i) z9G%;2*~-_MMKAd!?DGwHwv9P-Uj2>D`C5lxoKlV*yo{yi$|82&eFXdhb6Ixas90v& zl>ZCvgv9sosH-39JYB`2ufAdRU+VJ8nu<4C8uqb;t zTM*;HY7Hs?(iq}X9sF^z5fVq@`>ppG-CNL_UDWzRH(nQGyY3`!v;xOBZzzc~b6Oc* zCf#kQ_(R$K*f@qQTPxfe_AgT(U2gYZI)7}!*^r6j+_=skQ>-{*?k z(E8IP_IA!S^d0$ESnpmadb-z_zKtc4ZKBdx@@Yc(2;u>ZEc*zyxkp`ypXG)07LwKs#BtcQRW)@Dg-cmH zZwFfOLrCYw=~@u#JPyqQPcUQi9@?RWi(%BR$SR&!K4jUDGMHJj0sMMpC&~LU#f4E{ zo67gD)8O3lmx{MEif72ja>6n0dA&>wls$!2q=jtxOjG5Xj`sZq8IO;+><()usdoio zTdMoF`Sl+z=~YdV7bkyhfjbi?h&h^7!ruRdi0u3r9(!}3`zc<*Q(@hqw^b?2xqS!h zTi8L<8+ww~El9U6yo(`Ort~_1@z^JcS>d z)F4jzi{Fz4F26JuNuTgbtzaaLmP-?FsvcCDs3kuL|9Pe2^2XQkfXHQqUXQW5?Hg4$ zrt-qkzKf82usT0f+R@}>OU0#dxwK3uZ=v+)bI)kDA>T!Ii2e<6y|$94_LT;PSy12Z zQWZ}eYuSJ>)tHU@XNXqcDpx&7U@(Ya*-={RMBKd`aIDC}JL zLU}aAoCK(q&{Goc@ps{Ea^ATxco%*I$g=?Pt6V<}#q46z?6%L8?*sBtg7gVCEb0df z$7LdUEVlc>F1#@&3vJDI17RoUamoX9UOyPqo4y03neQBp;ZD<)FxBuiBM-s4-HyWK z)EUy=e==rO&tW|~mJ<)?apjXIKO2MHKYi1L`A&x!eeYna)I(G|H<){&4JZ%VXyh0^ zZu?aHwAfDWG}Z<0tNl1-4IC4;5$b3LfYG5jNcjWG7kC2otcLbSdu+zZ2jpC_9zfXo z1=;gLk!m6WCIqodX7g~(olxZH#kbELEZoHGNco0r=)Z0i}!bc~b z#@6&XAoUoJpJW5QM1&wM$~ouoT~HQ)2Y#k<@V-shOVq`uQTBX3I}DU}c<`D9K>OkD zc{6b=_XMZ-;Xr;ERZd9TSx>wW6FHqTl9yHdD~JyTYvzl}uhZVN%3mohQW+!cTrECf z%|z0(nr!{RO-Q;7{MuPLj0N2RX$dE){jj>QQ@s?t6-{? zz90`n>pw<4(G?7I*JH?p`n-HWzOdr>8OVrr}(mn4TD6JV`ltFs_5H`$d zj1%kB<5BxM@LN0gu{XM_808k)Ya?*mlLn;iNZAXimKf|%1m^2|z^MoA;nS@P7-;nY z$qN%7wvqE+W(%d4nvw62Jd5Z(dytkqAm|Pq1d5+(-YeAZk1|kt4m(<9AY~08O~Emj zuCiu(Y_LV%10?;I6Q>*p(m~CLz+pfhkdx;?3 zh}WaVzdH$4`zFs$p6?gUt|B3A-4i&!<01|jaTU6cG8gaP{ZQJ7Q!{>`;i9$Lik+RI zZuA?ZnFHGO7|p0YESct;GZN-k@!^1ka-3FhSNke9NZZQL1FSzihlyscI5}x2M6`88 zpH`-%L#Nq|G2K+oVdNz=$J?HRZ9Y4gTCeJdALD)s$^<}uLFHW0qI4rF4@g|fyf%$f z8chDR9`SVyL>8Y_8o`J^U}Q-FTxqzAFfyMXzqKEa#!asBE#%3t;|KxnrB>X%Q*H8y zoz+}`6Q|-5y}6nzR5PoMR0GP=diV5^m&__a;je$+-c^3C za_&E**$M9Iuo+L6KNN%=?fB-~)ZSFzPON=j}7|WhP3xVIKkn9^65xDjRuC@pwj7-nkzB74p2Wa$`J5(Sun3t z)>i85i2$Pj9scy`4jA{km~b=*f{UIZ=@nWGX~(MDPRHsSYiUReUGDyA%*t=Pf;szk z(+tW`-W_a2^Pz9H7c9XEa|q&(MWwQXdWWS3#0N!JB@=RPD%sa zgQtY%OgJ8>a}rx_@x*MK?Ny$a=4+@j+tAI1yym^3$~()0X8yGIGoZJy1O`qkz{x*1 zvc{K^1@#_MHpje*2-1QIng!UxtXn}yvo)czTlrLYRq1hVGO1q3Z0vF=* z(G9D-HR(3e3`c1pBW{*K%llA%jYXBO2n$f*aSn+Cph@iuYRzh=I z<}hwR*^E4Cuuz_h@-Y6hug~LlIm&FKDVnX*cd@R4y^wqyi+P#H+L|;|@56u}kHvt1 z=3-D=6O4A+DCWO(7UNQORCyViKY&NZON9Y+Z0f-8`DkHgrY-+b-IwNgSA<34Ee-3%Y-UdNn@d5}jxrult`i4aN5SKiF_fY1 zuyMVYU}siCQnn`FD;aqduHT~{SJxR?G6ZP$fi>S%<1}Le;vVuy`HW_+#5J3MJeQgY zWA{7T$7yaN$bZpXFBZt3F!dU{VKt$V{#)XV zX^duRTBYCn-yBd{seCcz=h;ZP56L5enq?A(uyJEc+_&5ZR3_Lo#tc5S?27N7=L6M6 z<#HHW_#OMdqtEfz-=uk1B(DEy#s3&Pv7qKjNV7jJc{!!ONZFJ0@~==DGCVCt%}sDp z=}~q!xwWi;^gxI47Et+;{G;kCIrw)9s!Xjk&a&K4=`SA4{X}{7A3&soRGdz89`Ye` zq0z_((0JxXL3PzWeRD|+>F^F$k8pr5)t^H8+=j4mn<1Ree2Ig{uBYc+^VyG$1K|0D zjqvAiHSRZlgY5LjR66;DV0~TSGvYe(vGKiS``w$+d&6pWH|8kQi?ina6YKHu?TT5W zjCg4Cv_iAgX*J9K8_nLm7z{hVN8;K(=h3R}ChnJqct7F`?hZQuququ}(?2?YHGwGaf>Xk;zqS7#n{cvhcse}`e|~t0kMHlp-44t7f@L*j z%Z&$d$^%<@k7mbz#OZ!{bl4V`X$XXz!E1?>$7bvXz#rZg8z=4df$=v=;QOgnk( zO*={V6}nw@`I3E?ab3^$vc#v6{dx_~=5Ob;4)F5d4bAqfMZr*$yT`uCZCxE~iI^=0Xa zGw|`=W*8oFNz#2~*j^|3B5)=9)~cg?m{r7n3_AqVSTYRtxdGF=HjoZkzi^n%5%^H| zWKQEJ(Xe9cI`;YQ8+<(i+3!>fxztHVj;~Bdqk+CKz_2~uI#8cy6gkT6`!k)JfWG`t zw=?(}#Np&*Gx@h>RPC^+L*hTMd3M~9EE;ZbmP@B6!tGldR9#`&jWep|=zH(4h!_@wL$MCL z+){~DTh{7BF3uS{lm)*Xia*zNg>8oh;Ob2S;d|~JEO^kBN6Jgsc}yG}>Ne12TfJ4> zGJOemY;7!)+avCJ4q0BaLcp9yit=IIKjRb`*S!LlYiCcoiAL) z!+F(Zo8G%1(rzhVUHMP*wbK~h;DWB0bnJ*UjsJ}gS3Amp(0*{jeYf~>tp}aqT3oc0 z{8PU^d}7XBL2GBroGT!3g*E@Yx33&`{T8k9qsSoNeeaTh`!Yg)s$~t94faFRyBDy3 z?cUt|O=DX1{`L__^YA9Q0S07rG9``*ykL+54sT{fcGK zzlfg8HT8ud9cDsm^IL@P2K?EiW^$id3*nP?nk_sOqv;~Ccw`WKnOIx;EVbl&LL%@;6qmMFE7`^=?`S{s*x3|- zk3G-A>(jH~h~F`wpK;&W1DH<9J8j+KQCOVY5~G*hCeCOle_DsYsJ_z})eb(-a^bIh zZ)s?6+IK6P$gc0&bE6ZD`E`YD{hdJnr6`T%`z z8=+gf%B*>?tEZ3b1^mM)9&k4TPq_dsKbWwJAs4A|H7p{kDd1$B7@Y zufZ_+RrYSJ>XS8t z=g5pbb#VGjXK88s2zCp>nKS*59zgi;1je#ow-aBFXNqm)%eN7T3Gejm?>?MHr=t| zdCF!Z98ep5n%$X_0r{tG@!E^E(4f5&s@i?LS4R%|8-!OrrGe9Cd*WyVKH;>9e1Bw{ zrc2ls(y8A-Si{Mt9$5J^i$649jjLPigzmFSaoc}~kk*IGcf{jdD?K1i$LRXaWs?Ub z=;!zWXrI}12C~Og4{-W+U-W&w2-$pdPWr^NuC4>Z3chO5p1)q7kB(bUiA^zqkTN=! zk#6F@9!Ht;fDBN&c6MnhD9!D7MW3%dq9+&CbCP2NZK&4PNH}A)dmV!bev_GB9>7&w zSJeNpU-dOpT)#Qe%Y`s5LypHnWz<@wzo6br+6iszy%kvDj_2l71G*+0OlVHHGUvzA z!yt0qex;R=b=3f~WAA`>#R)KtUyiEhlotEmTM2J7+=$mouprr<-#m5-AFRDBww9S< z-`C$jr)Etpai>OUraB+uLL@H4)KLqGBYHq6SU^xyeXhgIkC6`loW*&qT)bU9Rx>iiK^{x5ss0;>L^R}dR?2^9 z3+7elR0}?6_dQm+EFLdT%hPsra)x+rpq)|$Lu4>evLBZ*_CDIR6? zI@}XdSCX%R`a8m~VdV_SjvdbRt{%bNpYI}RIja7b?>ivrRgP^yZ*gNqN8C`o0JqPm zFL$*JL*f{m@Tm!pn^X+OkLp*|d`7{rD&IrCLR;us2*=jVl6k(NN)O=o@>*iv+6;VP zwM}7=_7jGLd8Yi-Q>U#|vq~J1Ia~&<&tlW6SK$VpI+;2LgyvzNb-w64pbPlhw&N`~ z+R4)cyK;*mtF^D`^Y|URZRCp!NiKu`l&b6U0soB$@+(3=_NTbu%sBLXjK&>0y-w)|X7FVb zlIK$xh4r)SW0BS`4WgeapFmuI&!TtQaT|xWJPRW-?}!x_8c2&FI^bL$ zEXe~%(oZq5k*-`3_=bf|H4(|}n#tV0E0A8tG;U$=wbKI_Wzq+0Wp%{Y>pg*Zl+C(y z2*`_wW5ddjyf@;8t@uOUSN$mV4e-F=H3^XV@f`f^-HT5g+*0u}u!sHN!B7qLb}DMe z?o_x|K3Y((3hEVXUB50C*6bh^pFdh?ARQ-k#GMm*(X-nemF~K%>?K^U(f1Ii+RERl zJHV^jR~V>gp;6wA&Vl*;@|69`uCN6SJsIg4&h~#*#Xa{T&A}*Sm5g}#EbmaqHn^m;x&F7cbh!jbDB4| z;_Yjk5lZ(1t>=T)u>eN?QHFJ^rX^m$GilzKb0vVEwmgJA{o;{$hwa=k8EP*r#OpcJ za8bQl(s9y)VrIMXO`<{9aK7i3-lQ|;cgNL~=IdL?G0unBjTOg*+s_A-JEo)d z;y}%ola_ML3yHKZ;<*TDT|WX3a06W1Ad@`R4BWru43t+}i#cs-i&J59R1bjkd7||j zUkiwT=vrf7%}SaVlb+xEk;lG|>JEy3hqmm=J59Q%u0h|+7{X0%pTqWd(tz+U%(e`c zv;GdiH*3;CdHi`cm7p?Pk6S-ML>$J`miB1eu$$~Qq(7##{Y#lK9M&}P$4l`(GT~(# zKJdX!Mr7ZFjOm)Kg{m9pN0QS zbfN6W$wMg}k@HU+h4(9d0cE=?oGU(+?~f73^pV5D9*WJacF{G>@K2c?asONAvZX&D zz4tE|r!#@6Y#i`*ebpZ8>!P?6e^+E0p2f`nDm0YEq1lxCKzUm93U9#gbZw;~3o#%8zx_A5NJb*FLO=v^TiTr0PGFT^X&tN_)wFGlf|WH|g2Mk?^;E z1}mI90Y}!=B6;h>33TczJtIIL`~PD4;1``mLk#KTdw^g5)2xgd1ouJ{Jn; z+2-^6A~jEMPX){LAB-{;>gf#ztHr0)IV+D&_`0SttV~aAfi^phB)u>Dzh<#O{6T$K zM%m>sUVUrHFLiFo_V+y@BAVCWeV$imXUof2qMH?x*1^%Q#kkby7iG5-gb@#H5s4bp z-v@y1g)}G8kpBRy$DahP1?wjDm*h)~=<^|)2t+KM}+>c6p_>kGwu#EqzWJYu5@s@V&9{I0|4 zGlzAHZ^7)`Eokx9RYP7<@e+2^{sHo_Djze=5r*cUMu6vBJxP40sa>yGl@?GA6Jzxk zaq>$z!S6+tPo}(qHO@Xz{ii&&7&EP_$gk`RT{CM*@x7#q)0G!jSXUn4U+q2YdjEYu zy#jGgS5RSsbR6#aFVE?rvyJ?BH~QYokgCi>8B5UogT2}`o=1MTBRmhcLLBp5kfv~B z4|h=chPXof3ffE9UM41dqO-i(rfNR4@X-NH-R`j%oNidt-?(C4E`bbcw1#Ed6j@TPPt7ZjE&9=_X39jNbzb!j}G5{_-q$}y% zwL!MuHUDZi<>dWE>$26#=K$$EzjQqiRYq2xxa+%AW}Q5cbUX`IZ@PwrVINHaA-$}vDa!HYKrf%5b8UP1mJ zl;*j-SRm;P(5HMLkVZ<%zrx6Jpz?Cq!R0#qdVLFhXJ19~HG*`BT^_$n++5rkLK+R= zmFKqNwZYCOoYRRewiKeb4gi)aGsG~OBO%R3kutu*BliAy97)5(iuRYVqf<8QYFQI$ zPK)ee9wV&)AXXUMtI7Q8Rh&wXyTckhSnbL*1wVh5HWX5N9`41m8e8`kW zlfFa6)cv4l3$4p^IL&g@3=kJD7!2dKr)r6(M9R4DpnQ~ar+=Wrrr@ZZ3H7lV# zv3ABqjLuL{_QWk#^nF|H38-aL4m7)~;(-&(b8%abI1Hf2A|Ce|F6@Q{qw&$|?C#%3 zRe6MXO%tTOz;0J>B+O4=hJNNBMc(-wMqW|+ZLyF~?u3J0mvPwmNds7aAfINjCTiX! zDt4wJ%~UvXhw>^&Gf>!ee>76>@kPCI)eMMJPlB2ek^dKj1D0j5R{R%i3CA;r@#7h{ zMOKsnKNXYtifPrXB-J*yPHbyb-9?EH56Mo1jAXK0r295kL(zA zPi&Zd4L5h0$}(U5V=;3YLVvp`*ql5Wd%p>Q=`ZKv+RulDRkM7}$&c?bvi(ntz1x)s zB^b-78-KBB$8oSbZzRmSp9H_IJ_WtQ4RKS=a{MlD!ofZf;=_l!SaW3`%%|s+dcP^h z$M#1s-?KKV`+Rrwg4Sb?f@@uO=^p+P%ISB?%o$+@r5)nHX3SGq)o(St3tG){vkFDC z9-pzo^(f{$Z9sGFVOTIu%ght)(EmVZQQUPQI~P)_`P4Tay7|M#w&2`&4{o#mHMaYfi^8Wl_paAlMpUEc zf&DJwpJSgzMMfK5>D3c&c%OiEPA8#po)qEUB{1c~1lD)$5+3twJH(zn2|ZbVJ~eU@ zW;+^5daYPc=*$mx49B$EkHI~B3SSdvE!+I<1U7SDI?=SoT?uv z1Jf@7{Ts8a*5k8?{=8ABlZ+^O2ak6-usJh!Xgt>%qrya^UEM_SkuU6}!((Cd=OunT zXbwNyjF%nm-^4#H{*#UEU1X!sbY`^ZE<89=18+{rU|SZgm4{5@v~&)5uppA@UNnU5 zr3W-U>2ufa30-8f9&_p5&saHqH+7+z8(hn_l6i*J`Qz@X*mzhENxuq6=55uN2bP4e zX=6XBn!|@d8^z=9?Sbob!o+|e81T>nQa)tkl4Ey8{S(#YjEp}JQEfg#XFIGPugm_l z_{82HHIR1Gjkxd4H9&t?wWe!U=i`QL13k~3-1+-Pp5gc%qeq+KJ)3#-JFKqZ&PHdj zn&B4I-Ao1q_5H1 z8vh~8>d5dZ<5=p;JkVjGviEp@`0ybDtaGo6bxxhtdqF3z8t>)w0FPhs!Odr0Q@;dM z)uK=KdloSFl!pF4xWt;kxM2%`=lX1tYWc(W3 zRh)2{4DtV66&Akjafa(E>VZ)C(sUdeTa1C$FH^NtSDrS5p7o{Ks?pm!c(9`}*JfJF z z?)Jmk4Z^`TXc-J{pNY-ODsyZrm+<%7^`J-jDR|_&7gKNbftY6vWcRk+WUVt^3O8W2 z?J~37z7p4MP1UAM%)+s|Hlx{D3tpbW(QO5NW>Pwp5nl-Xb`Cu3`epd#GYOv`7z8_3 zwdP)%I>FQ2GkD*BxvbUnimEkCqddH|S`G2vw?=&7UV5hJ@;*W9r+TC#^%qVw?ZwAE zGMBp_m1v*6?<4Jk>d3Tj58=w4Io3TUWv2-nno;quq>pPd;+DhE35EvAdgA z@eOp#tR{)e#of!#+3%UJ;Lf_S@@wsEyw(4l=#=D(_uu`&mktv|++xPpo6|G2+b+B4 z4yh%g?Fl zfw{1B@EA66eNWZ5GVimF_TSDy8m+i!ak*`NRwzFqf{cJ?sFZh zkUdIPM)vyJdrMIvDP=?%B`G5+b)V}HWhPr>OZMKI=Q=&F=YM#9=!d>vFYa+Z=Umr& zb>G*A9gMgLS&QBa9eXQD_=hpUAy^iWDP|tni5C7wa@HbOZQk%&@~Ma7wwiK^%VN2F zgsz-QpTnOtYz7?OTn53T_tBbNWzbW77F?}48&Tb!e|+vJV|T9+2fCi(Ib)aO*TVMb zJF-kX=)4^p444DNjq;622>Te3g8N%^k{^aS$g)XU#LqXi#8K?K-&I^c?<3WeJ2Jc7 zVtQbl_u<0ltluRKgne`-T-23Rt87hX2@sEgP1J7mKidNzoof!0XU&#vFNEQEuiN7D z6i;s4y0`4uueU7D-Gm-b9>%_a1e}(}kf^b6VeQoM93x4~}5;V;l3Dob{BQu7p zvpFz~G+6EmTp4Q1rz9`<}zbUTg`>F)=2!u38%P6z1!k^ z_jQcUf`2^#s>k7Xjkgv3*tm>Qh{b*CC5vnz|FJ_SL z)k2VVf_fP}_|1v0AhNtjG_!0iX{|U^TLngDG>|E_M(BOY8VhSa6wSSR{x1$ieAAFN z!o+|foO%~Y69}DH3vOw=0iV1*jX^{8I6aRO?n|}L)C2sJiwkM6TXc6CR`D;iHRmVUxd;oL&?wh~KC``sQ`m z+6BU2X>oLrU{-B5NjR2@`|GyXcss>t_gtY_1M@zr*XNNxdn>(!hdPvKa#JJlecgDC zZ`JW4?#V{Ho4y)5cfa4KV1pk)#@Dq*K{jFGH69 z@+?cJ(*a7;XA@U9@h-nrAl8^&#T9FeAo;D1JeYKa-SnRzPYxP{dTwcqv;p=qN&?~; zrh8)>%xvh$FHdMKqxL;xq%XMb5lw}bzE=ApDt$kx>suW0Wh*M5GA+rJE4>vy7qQ~?UIr==LI7aCoQQvtQ z1~1sp6b{XuEyeK|knWqi0y??z_t6{i(e^aq*Z8l{?yzN}JhDZA@eLe1-~x+s7$@i~ zxoc7z=y>-M-kTr8u9g(SO+Hq;{AH;4aCJMYmr*2&2k*plAD6>~f;)_QNs?}WaSwZO z(n_p)Z~%9?+DtT<{2i&@h#Skbs&_8mJEAQb84WJD7mwV2Rly6SL!q3u=XzRkz&LN`dJ3Brm{Dk+V%>6>?1Xce|&N{@SCvb+T z9WNh!iuwB&vudv#=sGPyVd3NId88Fqqtbp&7bcNU%ON~`i0z97={Hx>)t}+dsX34< zO?blOIglP=&w>K+*ti@V{$LZ71++lYU*dvWUHZ=st=iAN zwLKM=F(3O<7!X&Yd>!j*_6x?I@6M|n(!;&Z+mW;g%#qnhI){_4V)OsI$VlsIi4SyTY80L-{S?-(k?Ko9 z8dW`4?T2_m-+>_CiuUVENqdcE7YgnNx|2WL1d0nX7HUE14D$8j+Pa>Sum$a_4HD$3@Snjl zR`p{I+3Dj|=yov_tG^xqH>Tfa#81-VtO0KD{lmVEEd=u6;1+lc36FtrhLb-*rD>e^ zq(I(3D;c|vejiRaS}^qvuI}DOlE1E~Uv)jCiJ88SE1Z~fUU_%9e@{MB{@$xbP5iTE z8V_H4Tx->HE6@)H%`oKVheNSh#1SMeV?Txuz~PhXh_45ba^N5&jVVU zg{0Lx@KuKUh2mz*&+D|*mkhTw)7&v$$;fvQk1obW^jTi_hTR}b|D!PaB#odgF5d-&!$>&IoSAdse`daDi*7yXp~LDD8lUkbuWE#)9u!hh1FZ75gJ!=f8e zLe=rlf2DY3qri7#qAD^mm58B)fgydaSF!i!<4pmcC})ngjcl9YRO$|GJC%DXDe8NcZUef}#& z`8M`;(@G>yfz%6vbcIIY8Q}veJg}bc$mu<_r1=zH0i8RLKT%jJCS9E-$P2N6KTg#q}pZej^;9@%j=luh`Vi!k9IcT9wC`KA=lme0r^16FXP$XEPKi+#{l^|#mVf)>_JM;D_?^i@2?^08`9#} zfv%gAzmR*K?MR!P5hvz!_a>iTkta?``U&I}xwj?`)4n!Ag$1O!1o5wU*Sk7X{v(sh0c;_P?_q4f}zqmx`DiYX@F2U3GJsOUy1kYY+_M8UTeq`Bb^7R zzPUIpnKDQ-PM9x>UeyBqau3SfKC!~B!FgXc=K(580Zxfk zxXzsjCrYLIzei_|;pPF%GL9wd#-@9SFfhF^a`*@Kos zWfbJIaX?%k8lGu~vleBDpUobs>{tZe907+9j78V%Ji?A7&4}gOG?XcFx5hh15r zubH0dg)JHnh3fJ2`C0CP#`BZWX3!*lFncs=HTC4`?+tj@J}vO(dJ9OOlnKXom!b7# za|!7dyvAS)-hE9eyzS7DpIaHgcP`2n2~WR>raP`+c>n4$;f$F~Ix~jXyPqTt7kq(s zw_3=LTmLYx^T`l2-+=4!Vi;Xx3OsAsO2`wt*q+G;@KSMW+1e)&238H?J-5W*fe&qQ zRiZ}TvEGV3BMn8vofi?W{DHOGCAhX31^qY2Vd#u{BIEH=i2ObmmyBKywf`JK_oM&} zUEUa*|3F-B^h@|P7>}=SUWBTtAoX}JTF)G;6(_KrE_;cN=^-(^VQwnhe?9 z8*7#0eijUr)4VGj0NSXFXQ`W@j7J2~RtIzB<;Er-;MgLH>$ zNTc*XY^)}`bZCOpe_Qk8yGL={9VJp~b)kA4jK4l^lJmxx$SRv#%E2F&!;UBOarlrk zP}DXBhpnz9*O!jv4Z1$ne0TK3;Wa06(}F;n!FpAgCwE4dfL)rH^z*{(N=-ih-eBsv zq0nr!D|Bn|0kh+(a+4urFeNA%R{Tivj+@#D_FvmA9ICg=OR065t=CqO7mSDUu}4~B zZNHjQcajbNU9|zjK_04h#j|bSSd+F7kj|Y`e}UVby*TRqP1tfa7?$6u4&NP*K+^%U zSl2$8=pWQk6vp1skQmNGvX#-%jG`8-|-DwyPeCVVLO@LH6w8~$yVNN zbpSTg=OyS&VEyxHs+NVVOCRR#v`(02_2e5Ho)J&iz0ov2wgK0B_NIAiAtr|&yeyJ(xaqN0Bt$2R7v5hsQ3%~qn(K?6CZ z?p|2fr6=#0uonGvLxfY8I5s#t1F0TZ?8&E5>&H2-YvP#Xz4FD=P(H|PoSe9E zCx7IW$_6{U$1KwIsvaNrJ0)hl87VhBvsZNu2`vI}SPzc>cz03enF;UHJi$HSD=2Iz zX`PPrIu$jtHJ9eCnarefe8H%8C7p%paru36E%vHf9Z1h!uc1Dd$w%*LOZTV3?&kV@ zu~8EtPJTk`ZX@CBA$xGUU@txV8zKEWw%ET8mfpR~j@4Zwx}Dii^ZX{{IRw_@JC4$4 zB+Z_(s#P1Y0S@)(d5I8^a_}nw)W!(+)U*D z@XliGw?;B!0fSEatofA7=LGcyA2rXIx9-1}oy<&t5fl57kG2wQ8-3nuGJOx@S6`@6 zBU_BfYR)I0XwUadcIScaje#&wF4~a=Ar}VV>jwLwYuXm=zMgh+c|d7yc zM}Y&y=_nm+{bX$4S_-?dLEv<_XcQ^0tXL&p52t>b z_!j&3I;v_|WM@4E!WMpFpr2)268{y~XgLw3a zl~i1}^3DTn^&%SvSEejsepRLwcf>@#mL>SaEDuBkkFBo4Zlv<1R8uFkAw z>nHd`&stE=<9eq;&G|YT;8Ms<%=b=$rcFXYVfCs+t)}jr{g9Jf9SGm#=zqaT&xKRF z#)|Vz>$2kya&V??GsUwS+G{-JT)Tqnt3Qf|CT&N+05fw*^<7bO^jx65Bt_cR?kN<6%o_-eE`2-e&r#$aAme+AX8~}Ap1o$?HMaS75a^S}+Y>qAc{>^W+1gBty*$~(A3ihM#wTD}`#iJ9B zR@B$B&VBfu_0NEKo&B9;iNjMGDV&u$r}}CS>?s90YvDNi5F>ntL7QrG@BWKLcK6*# zxGfal5HE^;pL*kiZ{e_euRkMwf^QOjqFd+JV#Br$jJOCjo;I>{R5vK7PM_Dz<(&3| zFDdR%7^fpA4(f$}h8i<*au)5y6Gj+PJ{ zzY}y=tAo8gW~+7Kx=z*j-NYGE)vrrZ6<%$@PR#ySh)Z9M(?ov%imJCCTGNDE?DAsrLBKg4W6VZ?=-Oe~`XMk9RLQDn_l^sOkLl z7>u#|igYfF^ba%F=-{CgL%hBG9=2O|4K&|Q=20KWy^r@HSPhh<;fN0&iYsmQV$sg$ zK=r0EdGAP?YMq99n)P0E6_$0r#Ek4wC~VH0*b4Zx;~K(fZWz9dkw!tnOf)Xgk!2Z6 zx%q?{c;)9cR5cy{^*2*`cfIFDu=6s6#mD@xY-vrUkLmheL#>B(WRiISmObxJx???# zZ$|U~r?r>8h8;uI193~51N9DXG3uab&~yc^iSmTG$4{Zc*U>dzYsL)j4&~2Q;H2>b zA;fT>_%(T~3@^F`q`k2GSye{-#nwKri}ZX*r|*Yv=#g_@6NFRfV-bO`^0f5(o7X^^0-tExqSDM%cQE_+B-}Bj z8J)#V40PXvZqePv)nm)pN}DX)Q~t|))@%!Y=XPf?*~FebikhU>23f3*mb8mTf9NB8 zRO-jH$qyjdH3OSAsm5EqXboOI^jWukhcrt2Z`hWPQ)8W!-V(W2(jec@856!NJ$a9I-~d%)@6#c1+=2RIpbMAk38;_>0Z zaY#KWex-T~>T&rk$Ur#%%)^qGJz=AzRM!O4Og<8~tjV(+ze5nFD=h-Vcc?s;)roMP znPm>7>sZqPp9Gz$=E}i2aHUOCNq&&<^S=s>LDvZ6m-xC)q3ATgfuDNggAdOi18K5C zWAS%CdiVb== z?rxZNUq}ACwhHTdmttb>ZG1325IRlmC@%k$@75zjtjbrTQ%9}q*khn!;UYjSr z3>(Dj&T*68+iwGTHjJRpa|)W{_;JlFRC-mZWOk~wyzUOGK>G2u)!;dTh0XY9nO-x$?lMQu)a zHw4LBj8)A^|Ujq*z;Vpk(^QHAbM*oEUY@O+6mOG(vXXUBGs_52<~+DG!A-@mfN z-kZR9wY&0){9q49PMTdqz6nq4s)bELB+Fu3pj(GuNax2|>^D`INj|>?(7CGT3AHBY zHc_yob|$X&A^q}ou~2oO({m-Ny)TR%3e+30)8l3Zewqv^*Z2pWsPJ})?M=y_Z5RK& zy~hZ@iPvcU{Mka~$zW$>dqMsj-u$XjF^Dqw-s=G$79)-t{TMUz48*dJn1U zPic@TcnfzoUnP3>ZUuBca2qof-vRj;Ri_ne z){R)K9z*$s6R0yHOoD~R^Fd`3bW9|W-vzz!W}y0zc#Ie1r!XH2BcyYr znSaqp*$44r0$w@zMV${4A2Z61w3*kwiJ|rm)T5tp@8$~_y);7*e^4*&Q2LsU{cFYv zPssV<2`BjDgky_wQbgtM@EzpwIs zj5H_t;`({*8=CQ8S65DcQqWn5ku_>dbv-Iy>$KHpl-S__HzZ^f_e-cz2lzaL-r7SyjmgH;}jd+gJ3@hhf#kgumr9(|47Zh|I6?{W^M}=1{9^j2s59FV0={^5w$~rN{k&?1K)c>#@Lj1#(FT>tP_dwC> z(@f!p(kYJ4{fQGEW90eODl-zaUa+9gXzi$944=1T;G}vBfbuEGj$6TUPbTG(x99CT z%^?l_2#Z4KyTZoFpt6==JDLHzs)r;!h-*q~LH^;_u=z_f*l!=AFqe^r;^f`cJ_2zn z>(U{We0g^f+WIUU8sZ||Ti41X{U!n_^A9m}VauZ3DsZ3jJp6sCm@OAXbTSUJjEv;Bz^DjqG`Ie`)PvXmevZqQ>Dj=Wkx`ua0-?!1{%PiTk5 z*HymKTV)5Naiym}-LtC3CC!r2HF3~TUqQHt+dsP_WhE6l{afT*%5jddZXFgO`7?2_ z#|~jz_msi_^6;iG*(9+-pKtxIDPeq`@^BiJiRK@kLTA3V;u^nC-hjV?hBB2GQEowf zaZt3$Fq6+(W)TN|g%c;fikk<%lWu#A+twg1`dJT@E*jQDGRompPQqPRbdaXqBj9<* zW1>&{rG)E=m~vxAUfk4_igR4oFADE{AIv-wSHk?JUO-t3)l6ea`3m88RiV5R?T_-M zlp7{#<~PknlZjKo@4wDWor8Ph)y!hvFQ(S_f0;>;Tbj2)4}Bz@fP&C|kl215MAG*+ zs6IuCV_&X(5sYljn~lzZ1A7K5FROGWZr)c++-wSz1BfSj@l5G9uVpm<|Lh}`@6tXO zQr;1R#9JD|boe@D6l*c6xuCORt-l`!?^1V8ITupxF!BVdCK-8m=6EQSDL$YaTxnHd zIK+%ojUs7k&C97ll%2(5`N%%_$f*h8^LHejr*cTjFrEmyUTt2p7|Q1etG50D!ej`3 zRLrg>ZKd4%rOMd_oevU@C_TrNZzBz+A)#+U!QrZ1JR_;BV7oBjJG5=U~%VA-0| z4e)YAe<1$F(_LK|WmJ{(|10PJSI+;BtepQ}IsdiDFimuws! zfp?zx@rfZX@xPBJ@w9&f9`N2r8nyn6ePhpH;hIn4XK^z=s8O7w~qwH^G0UN1$h$9=vVAIP`B`UB3BsNVvT6;>>L=p1brA^p8*Aa~Iml z>pxo1eM?^E@qwjgg<8odNnkj;j-nbf)}pNGo~r@*DV^?aU^w zkJa2;VS=hAXg#7VF;`sr*PIMqPUPRa5 zcqH16naug65IEiC2jmAF#08I+$s>X1aCrZxqMAzr6npFQQSsMcZSi8NJ!h)p(L4_P zFd?8j%A&4Z|3WQ(wO_dQyWSs&8&yU6eH_M4K0ZhLxLO_`G72^>S`X%u=KtSm$?we` z4OilIc)#&?(X8$uRJG?CI|^@CUjY|7zh*P~9M`ya(t%PZGoaewUsqP=Ip5pk+vufO zf4Tu4{Lxc-Eqx8s7wtp!Lp>vg_a6s4rd$yJp19$es>>?gcWI5`c+fmZ)vT8Gl^;vG zi^Jw8vg=Q8pk6H#Ui*@roMP|JovZn=3!SGhwFbIAUVr*(*!Q`yESfP8Op|EtSf@pz zZjZa*@iLwD9F>CxJN97J1iJU-l%812s*$*SaD#X6`=3SPJ(^W=<1&9;W1Z^BJi1Oc z3afLord9dPIq$L7Z<>|jmdfb|hP=yx2iCXo{S1WdM%(!^!*u9s-%a(s=yojuwl#f_ z7qsaiYPuu9c z)rax7JHWi&hiPA)!qXSQxYM6Q!21Hxq}xewc3G~aJrQjSo-#TcL3K`h_77j#egeWx znX$5?938zw8U~L*swNQ1UI&wxg6Y;UmQ`Td`3z)WIm^8U^l-5)rN_Gy#v}b>i>Ofo_$hgL{ zy=ZOt$h)gBJNn7|w0z9`#hCy#2fk?9#oJ zq&3JX-{;6P^+Pf2ZIoQ_zNxI{Qnv!%9?vR)M^{G3Zu#x;_TPbEHKMchUK1yi3U|S@ zs8z^5)7&gN`?tG?Fv2N3`M8^0USgu1eRwK-wD82F%+t8+9etlP^dk5icrPyWr~9Qh z%f>OoLvX^0Owl81CpHc@Qtzq#)~=Sk*6cF%bWPdCX|e1Xq${Z}WpS06;2K*7#Ff1E zb60t_pCKchV_^pN{L)!RNjxO(cRIxtma28uSnx=67``2w^qEt^S%g3AMOX^bS;?A1 z9OUA0S@iwDQtgx$k!;cCtGH*zN6b9qj_Ul0&%GBV5r4@wN(dDeq@x!>Qm$7&fU@57mK%M0XPbc1_&lec^If(GH zOiQoLugqP7#LE!05Af}|82+%Ku1r367G7*#fybBI@_9`cg3YgRc=p&`VT$(o`6Q9% zXfD4*w9~9RVU3Dom$tIz27Bn9WAE2d&2-{7r*YNe(A?;)!ag`#qbX2LVEyiw*sflY z*kQPx^h$P;vHSPq?e9)Fzn-N`Kg(dvr&gkH%`C{9+JoDUc?I--;zZ5X^2fFNg!NS+ zsjz{x`DKXeeG87>7cmcdRNzpR{4bh}Iix~&#Px9!Ni^qUjs6C#{_-?FVR}5& z(};&GZ6nn_lg8t^J}xkz@D?iF)Hn)pSCjsX>H_X@0!sYKqT(@^MdeXhj6b8^wL*ELL~x>hVN5x!yz zu0D8%YO9%S)#`#+lzspR8|A2Ye>n8?Jp`u2Q!VSknumkn?3c|*I*W94919ur7oS@7 zg08i#<(T0*#FGZXaJ7w8J-M^PBP=LfO*76v4(AHZOEqXKg&q#LRrdeNqppeIlE9)xpoDIB^y;>J^h)v z^`0OX`wzmBguM)}EX4CP$8K5KN7mxXHppj={Cl(3vSzDitn9EIFF3D}9)+K9)3;*W zy6QO?E_{m@O%ssx948OJ6`v~(Yh9P)hVQrGM1xxVeB?MST@N@L{}M`9k%rTd9>WE? zVc4-wLrL5~n6A&MzfoaUuZGhIGxoE-bWiF-52c{mlZwMa9o=}pbsI5!Nt(Da_mB9p z&6W|SiiUbO@Mic0rfN*}I(Zow_vsSm?vG*cwh%gajn(=*3l(!0ZfAK@i-hIOojCQ{ za-j9G+^uaS=>`bC(~A#1x0G4V4^X`)EXRJw)hq8p#`GuJBSF8lbbea1j?rM6yO;Xu z6&xQ@R^gALYrVuz&${yWgZ69p4=iD%f4TB$N&|#UzAaXrY>3peu+Lzn=<~{s+g~s6 z4t3lwyf&OwK8&3-(%~I?oIv7XZZ+Z*dumn8?4LQ%by|Y3CM-0_gS)3!;hnr;*m-Ue z`LrCmhvGv_KPzBPtSgWnfGYWOD*A5fkzZic$&S|@qz5mjSgV>Bi|c%21j&w!)B)J%dTR=ps!lu1KxPiLyTSh1&MPYzD*qH{cSFZdvHv@ zQgqnW2&zU6WVPN+Q6945+H`~Rlva`B&y2yYEgDr|4{21;nHI-})NLSi=9~rcdPv+u z{&_T0JWBoxslSzOhbJry&Xxg;N^8!rJdpTuB@*5u)d-N5QMJO#k~(5m-+0Bns5Bh) z8Ia%S)UQDM35)%2Da?@#YiB~KO99$hRIlJP^1!I{=ex)vAg>0>*C<~*!gLB0oYxYL zYha|UmADz+Qxb;~js^+82H?(db%gR6F1Lb| z7Z(rBQkl|jkFGpG^2zu+s0(?KA#gLK4&?zx-2V1lwPz65Cz;ME1xY)=b(0!6Z0Qy4 z(z-OSu*n-OWe=dV1l1Iu6PqDUG}wT{N37Ro#M?3R%PyK(uR@q_TriM6q%&9rFK2cG z;yXb(2XDD-frfr?!Ph(DwEr%wu3G|1M-(Pr(!8H+gruW2q~nz*1^Z`7*q}uRXfkKN zcvO_Z1COPMVH?~;*rTyP8XKp4|H4-GFH&p54xMXg6;BWjBjpev8rpNkKje4CgXR8UwV4@>G_ zByY=UpOoLg!L2f}Z$=={nJ}tN;(3M#n!OUI!tw?2yd(`mTCFXTrluOs!$zq+u+yiP zjMj{M+h12YL`zwRFt$F-p8l4gFqZNecHlKy+B_8mND#WMHidmy3XTgsJ=5`XB)Xs6S}x8_J0 zkHUNMe6vLJdfh=kFB>U4!d`ZOA%m7m;x4wkz5{=;<`xu9c4E)I8bk6qKS}R}#2cy) zMdyiM^C$<=dc15V6^DdJ>tUa(QQ}shJEr|C#dgtx&LpZL$0Gb9|LLRf4_tRt&r^D) zjgEW8>k|e`@=euhZ|RIn(Ag$gI7Y;4yp~SEk^`YQ)`5}#b>y|) zz1FH6i|~qiAs-f{yDKaelqU(& zC$)CvIU&t72O}F@V`Ciz`?7upuQ?=%oeJxKlsiyOT8tCBncVS5PsZr)f@ z&VjGoE&=Hmru+=$m|Eq%pT2m+h}Y=sw+Y%u&^q^n-qqZ&;9fjy5@b*3J4U@GlFwr* z&!OI^!0j0gujAkvKk)w0>p<5{S(UZC*2s>u$r({Tq^q~uui-t9&^+RwVE$O2E4|xD z+XNRLi^LC|uZU_kc;{`Hy*%9W`1pJ*uC6184O&XQo(0Rb~qdw@BoQ#y|a>8#xYlFe9TAd8aMC55_EK8mWs@+8Plx`q z!`RGk&qdm^B`SMCsyRV*q+zy)Fb>YLnA1SMh35a;BxAt6XZZ7>K^}1rGbySDJ7PkW zN8{w@$T$83W9ycJvQyBjwN~LCk{(idv`{*r&#e=J@`s9SWMGVz<+B|c(y$upTdus# z^P>eoSSNL+S;^^(T)=ycm6ozomANBj!RYteh4wp5;W@-j@RWh+-Qe6^BjQN|@6TuT zxXR&N{#tU%mht`Yhe-JZxGwYuU#n>Rbg&Ot)_(~7+}&jRhYoy9>~~?Bx18Df>hZ%v z&%n%DDOlt3G02&%FHhDAlsla#}fH763@eRYLuW$C!4-D}qFjw|;+kqa(AE##-owfM)BiMX#yK6b3?%jb67#h%zc zVOwvVgRZ?k;yuFFdwU&t0AB@{n~%evBTf0MqFuZ&##*FKH~~}cX0iDF?PVLI9x^G^ zlJ`4$6&s(o<~^s|$kyxr2xR-2NSP+ZSaG$!eViRG%`ejyILNKU|2Ln+%T*KUUe`^k z${fo{xN&V=`Sr#)cxq@%*Z+d~TG7n902_2my9{v~qWFKGW=eKuBt9++0rj|;B`1N- z1Z!u|{mSmoz%G9OnC`gy%);iU3>@wv>u=qHq0et(q2ocEX>~Q)H11FeCdJY(Ts3uM(O=WfZ-NU=PU9si(r)<~0X8hcl zRkB6jrtH@IUHGTjFANTBD4p6?msAI$KF!Pf)Oa7xIbtLak zeg1JAMp%<*D0kZIz~&t<35Q)vq0j!; z=o%A4?-h%CdYZ{ObPw0s1-?@E)~SkGRyD12!a)A`wM2xR@x;B${y`qyvt?uAICg$W zjMm$36MOfc0r&Tc<#((s`6S;q(yO);H@2xKy9c*p`=V|M$J7s+;^k@ZwP#1(*Qu-Q zdUyrD(rU19YxxzHI0nfMFT+s(?_HMgc_FV|P{`DKt36raU02ql`w6xjkRVI-1>`kt z0|m*!ymo6>Z54JMQZAo_c{HoOD@zrnORV_Brh0t%v1I(uza}sG_Z;(@((g{HZo(5k zH_7dDgxGZWFg$ws3^xRC;G;%H;0rv54+h@Fny=rnPy2#^YAx^VNeeJe$*Mr(rEEzb$s z2i)+o4g5Dqhd0$(fK#7tg=Yzkuvx9EU>It~SNg1AN5<~2SWnL%2-;*_e&LOs)Oh=e zLa;7(I2ONXZL@DR2M5Q3Lae$F-Vqf$GT! zjBU0-=e>2&$80+;UUwO)I7euV>(!Lgmu|tk+DY)U$_$q28^LnpT%=)WG}ODRD`U5O z0;*Zzy$7N3qX3x}`Vo_}YH?bVyd4q_%W{W+j~U%>``;=|GSwjApqEcbHCb-;5!SY_ zr2TCzPi|ereG{|AnC}L#{#*=ey1*ZYd#t6$%#^BzsD}uX;-PlNDyD7!S>#y0g#SK$ zgjR3r@SQt4gH=r{*=b2F9&S+v(-!OzRG;EtMkKyE-5eg>+QyPXH;OK8TFHQ-M1?zS z=MK8B#eH9v5U`2;7-}nP-pB;ixApF>RQQ4~JdqFACs0W_-LieT7 zD+k(lZ0OdAf3WEXJ?>S9-iGm@aDL&>Lio>rJzUMM!?bSBNarP~w?X~=`io)eIkMfG zZ<;nS@3jk8PU1GHvABLtl=Ma z%!MB>4g=LA?3=e$Jx?CekWY`bmsFn+)!z$W@kpS)<)h#3q`C>=@YpH2Oq}A;1=A>2t(u$n_jYc$3E~Ph3;c<;TrY!GSN^-5mo2_YqGoW z8C%c8TAvY+_F@H^|2m7*cd~E7PTYKPw2b{x6VG(%BJF2~;Nu_pa3$SSTGo359e(%4 zK6`)A-q!=d47u&uE@}2)6TeU`h~=8-@w=}5MYC{Mcw#$NIxp=aA9k;-O`*$Tn z^V~GN9b%}7^j_dib&4DR>afOx4${5Es(|90H`5nLpDibGnM)_wbhst1e|VV<%W4Pf z+897}{HXRBx4*Vx#Ch=L@+@xfAru!c(&KHnmEhN|y+L#G4c&KjKMwW$OOKz45;KrRp zaJc+6GuSXeOz!3^-z~ce^lz$$__8CF_MB46e(8h*_i&PkX5;H*K!mXB=QFC;6ZG)t>aHxH}m_zq$r~LrO)EUe-v9+9;#F4ZSTpREQ-H){4 zb531nb#F#P-0o7PYxuzoWBz)uz?WW|nEQbOrD0%by?oq#Fp+(z5{u27ha%Mu`+DOk zq~6}Ep~s0`p8cS}x|_D~pyr@>j&K4WPu|N&+f>x%qVQo@G~=>Z_;{5VIpHlX`qvic z4H`-No&xVfJM&KZZBgOVrO29UZ#BXHJ;ItjO?b(}-MI3~mWs3Y>%300;$#OT9e~td zptxm-O|kZpMe7QTo?q$+&v$HQNA4N`X({=m{5U+(ZNl##){}&-%tLNdd?b_kQTC$q zG+{P@%df|p;`)hnPk==&<&?2QAf@Ye*jrwM>#zNShev9m)r_8~|9S)aW644F=Hnk7 zU`>+);?_$eM!1cHIil5@ElSr_{5aMb%!0f=poLy@{8~_#Dg35up_(gWXD&_FK6*J` z-rjPWkv@Z5lQ7K?y%0tkiu$LM(7L73*#**jNx|9)4e21LUSEfFYZ83%Fq3N%!%=CD zH^bfWVbCSDKAh2TI%&wKf_hv|e_sWYi~2zKcO`=A4JYN;3I9G<@W-z|ILS8$ch%SD z#MNj?pWT?Yps`k)b5T>}@BvsZi_zNBP}b;Lhh6w+#!n`_Mbe`RTSeLDUVL_1Lwu34 zhV~^&@t^Fk;66Q1E%gRzjvOGI5`m-bczk?S_T0H0?-^3Yy5HK2g`12Zf5FYXO@)8N%kHC~ z>|RZwH+U6x3n~(6zb;@fWjX44cX?jHlmCph3M)I?>hdk!TVRX6dqLq4X)Dapage03 zX>Y>u+K}ncC*EDvk#@R4dr36|*?*d0(er)c+JGf&+Uz5YbPPT-UoL|-n*zO#*yL;~ z3GbEmh4Q~gAl9t6*2Uxj)b9BbPan~6>K93zA`}-)XlTyq-|$PPp+Fkh+tI!!{P(B{ zCryLx?qrCA1!tk+Y{;Xm^?3lKOQHCiF3&i)SoJt4{kHOPxv<`~L&R3U1FANxYQ_og zRyVxpTr?3I8$fLJpTdJ$GW9-}4y1BRwKnqs3%NIZ+A9mK=#p>XOz zS6E%+9jLzg{k^)pZMIW9S-64|9twXi3#fZ_J188Wa|Jxrgip@b1?9bZt@Q({6NO7i znhpq4#U+1~$j8C~oHy zHGRn!CNW3*3j!|qgQ{urAEZP3LXJ+7=HBsp=r`~d#yh0q=d4ah+E9?DV~y?x!=9ee zP;K&+yko=mAmIjiSUo6A3&16ta^Un4Ey9)43a|0+t{X6*EC@+|0^x(8en9FWAa9PP z`b?7ctHAR`bIrM1+lkt}4-&LGLGu~$6U-a*P@9~!fjJvYQMyE=FE&NOX_3ERpqTo7 ztioLQy~>Gn?ls}p*_11OchISeBMLUL4ztpMc$!x|TA#1_=Alvgzl`ot&}!~+AT9y_ z)6KBiKYjQgj;=eN>Mo2MQA(jj)5u7n63RW#Y0%W(X=`e0@1%?nB{a}7+CwzB=Q&0t zZ7R|pZ@aWLG~eg=qff=Xzk9~>eAoSrMg5njJ&LBFt4@w)p5APuVR^B__k3%6_uVyoze?-%-bBK4y9 z{UefXyY>`SKTvNgt^@LWc%=0uOz+%K=9a7>{Ts>hHHTJ4fqBm*f@)0=KLBYE ztk@B(uo_;}uFWay5NTcqfwT}%T?^9voUl>*-aoD?e;MDGV|j4{AzYJ0Z7+KWk4we_ z^Bt5{D^klWLz&xDWx&R!-v$>~9LHS8*(-lwGUim901C2YAH|K+@CHV}F!x z<(cmXi+S;7KpY{HW|pctRM?4i=ch2jC(0vs0j)c}D?bmKbQ}K9{fSuGXFXDe0ukHe zncB;7UL!~s+cNT1gh7EgGB_V8KcX6&zzzS7QLlraQ|LL{+3h%u1yj%87s?}2{(!`l znii*g1aY0JTQT&(YfrVu?_S;{9tab^zF0`o;N(G%h~eW7YWwS_ROMGz)$8y+Gi`ao z09&9MCLgk!JZNL`vvYB2pBg-=@*O&Nc#SzlQR39XP;D?}jFcULh5KRZ>0RVmv@r8V zf2wD5C~UKybj4bjx6YU=Tp|30B8&YAf3po68!cV-l{>vyjMpDnH!w+!a(_ zvg^|3RJZ$}!8Q#)y3ZU&)_%!oERy`XApglp>i}goY5;T6H zGBuimRJl`Rr&(f9S{P|xUzPiy;ziQfVt8paPWl4>wtX)Gv+fGzwf{9w0pen)-*mPl z>_d}rBP3-fKpH~nCrD1~D2tmVqQX7GJfQqRPJDfoeEJ*0gSSB1n1AbeO^^;n;uJ9| zdLL?4?H@Oz6oRd{JTn}iOpuWWu#$Y+3F?AFKcr~X43i?O4=h;mtuNnZ#H|z zRmvsCh^fif@yoUt*2;LicHQ+xT;W&I#-)WBrGq1}*Di=^eyYEyfpngQBxQ7Cs~ z^}uG5v=mxK<_W?cG3fMQ@^)5|vO4)@z%a^IUgK8BC{T4u_`?aCq{3CIA3>TE4+ZH+ z;&ry5Nn};69XauVyp1`K=7&C0kHc2m#zOs!yg2oe6QkS|$rr-KLPO=hVaMOcsIn|} zvH_e)_=WF3d4kG2$wwjaKcjWSCKEiMq?aAidQ0`0kH-uk4hqAOH7bP5QEyHflTn_+ zDP!foO)gb=Sd~Ndyx18CcVXp<5Z0)3U67X&Q0W-*3_$rP&b)CA+RyUE>uz@hr*D|}P?%=VU^1mbN}I7xa1$aArfcbBxJyBP5(r+iE8 zxA^hs3j9}bmArCI;&Bh5cXM!6_DlIGWnXRBx19Y-{}I3K%(5$Jfl6De+=*~R)3e`v zP#Gxc86@sS@~`l>?HtkujWmSY*y?92E}!MZ{m*{I;gfc-dIrZdy4~r_xDK!B8CCkq ztAo-$Tl5ZK+daS7J(on7YDM3p&mi4eeiFBM&47|a!BB6`Qtd~xjd&rjCLfkzEzPub z<;yRvWJ9>BIaf2nGqNl~Zu35mPD5;AP{4i|>os2!dSM$}u)7QOqs&FJr?Z`;N@Q9 zq_*I<5Khxzr_)USY0yA;91sV$D@Wsc;BsQ~2h8=N4!2k}1EsS!24&N8BQnxp(}uo+ zK1Z4k+0Sy~j=&i>3AaK{Ff(}or!`|E?0ay_ zs4To3Wd-+Kn(#TincRARv-tsC zaGZL6G7ng-!AC|1wHEvU{wS2tv#f5c@T!}st z?t;%dGv0sVW^MCK12*A7XKtq-50-hIrS0?&=hn=3Y15(-1yhFIb zyszwjXq#;M`3*ZY@V7Q5uZ^Ty6grJ8#H}Cy;Et66Hrp_UA4?b~4}ZR;`C^%d9>?k2 z>6{Owt(Kx^qfM|eu&&g#AFTEbwob2ys(xwCP`mqPJn8Ml>-{@`3X9Kp0*$pcocG_s^&@Cf7Hb1gz zeh=q%q!W5Oc${p|h)WBI9h zZ@RXlCieT~AgJGPW8X-8oL)-L2kM7c-TLxw-CV@ZY#VuK@3^z9)y;!yWdXNIyhdZ*qRqQt%45Eq!kKe>F}1Hj+4ey5XMb1uaHk{<$BE|%^lx+YuVs3`u;CI1gXdP zQh$NHvX6pQeiNql%21EX>)xHfAd#MB)m;yoj$FoRta!L@m^fqP0==!a!~DJ*I}Qs> zKw1N;`Pwqx|1>&lqvgn`w)|P-Wb8k&E?-;u5`WWq!h~_sqf-zT`1FC{ee20r4$;tS z{A#eex=entwBxG&PX|8|iNQ}XGIbqHIM`8)@a-qdy&l3xGZS8Ca9^2m@0XyxfEGpV z84-dy2%^xJZ6;1mlvZtTs z!_Lji;Pk_dSiWcq>x$o&S=6w)x zYU~5guU0`YJHQs}488&*zvM8t04KD6P$>S?T+7nH6WU&|g+@M;aB;#5aq!b}&8Y|U z%ChU&p051;02u5cp_vcYzG3bu90#18{2*z-U`D9oA5JMcx7?umpG)T=d-SM0B z|6`&!sV=8}kzI+fyLE}4!*9u-jdv}=?KDXe*0vm%_SR~RKvioz!K>dl0<-YW> zxI*7ZI!}10`bAcA^Jk0lGG&}@KT)gODbntJ;CP}|JkKrEt~fA~SA?4Im~?~W%8bxgdMZc$!P`aXyL|x{Mv|rp5Dgp4|l0~gTuit^3|vYICx$)IKP^n z!X|t`c zP2Ygud@~)ldTe0L8pSc{Jv44UUFid1652*#raG?d9{d3bU&IRiaHJaHJbx>W%C85V zMjNr`cEjbrXD3uW0O7vEJfu3)Q2jH)7*O~b^TmUn2ii~)j=~J~9M`=w;s&E*G=sP` zCp_UzKhFTKVk4e5w+sC4Jq^QZjAFzIg!6XV-4m{{^S0EZ`Au+|bCHO38wzt%!!$N6 zL$wAiBOsyKGPdMeZ+ZKC310M-pyoqb1AQHA&^#a=+OgR{TmUTxl?(3&-Yk-Gbw9Hf z+D{LwQx84Y>^JWPCY9gme4@HCtN9@Q#HUbXHNOtsmbin$t_E{UAn@&V)aknm6?ZMs z9i;tgr7P)YU{q{|R~$A&?Ygx%@g`(vc?pBW!LTJM0iX71LFcv5v%}g4up0W?!N>Wg zW?w*(AkM|B*SL0QpAK5v`!ku(yBMH45WOqsibYX}H6bO7xmvfx=A}5c$g!#~s8_-J zAn;hX5^dr0ZP|_HZNSf28nLftG!pgzaS6t>|52sicAXystIc-dob>0K0`qfV?>!vW zY4v1}?h&9cPIrwSg!s8Yo498Nrg$YPqITFMtUfbt~1)k+Oc=I;W(FD_WwY4pGPop zX>BB&5Y$haHG{$PT-jP24}>$hxU9$yA2*yVUrt@XsTUaafFQ2ZjMJ};gayziP9kAk6<7J4 z{tSdOGTp5hNt-g&*HLt?Vc|$WB);Z;r|09UB}cSMGyF2zn@wDUN2mT0DHq}dX&W4z zT1%!}sL%Jr9f#XaPqBOOT)26#r98DHjWBj9oVjH$ZwV&? z-Y=m?)(zGx)PYaa8p=Bpdcf@=8*pvqG^Bdh5H_IVJk5*sI3=_N=zY1QT|MYA`{B2f zeSVt{gsB{K{!o3!fbZ3L%qFovTWEhqkapvNhfF2;091N7y!UrOJ+39Narn?|iA9beTBnuXQknO-mO z7F{#2k2l~snpH4r+Ed6(+9H$b`~%WW_`}6qj`8ozWZ#AdnV-${Lq2#P6tPsXr4p z*8OAM%pBm^^EmP}8J>Gr{v=MS#fRqn6W`;~NQcy6Q$kaq_M>%Zw4){uE(GkG7%y$aZw*d|Kk0*HM9My;-MRe^L#cE=IM~a@`6f*M z6y%*2VsyrKjcM#_;(>RNGUu6=dQ&sPm!5wa(jJtiX!5+R@+Xk{%N#sEzaT9#o0A^o z>G@%T#>J|+y<+5vG(~!Qgz7!lxdUYl4=aAIOpgsSMH03la5q9##L>(ubK)p z4{$m^3rH_<`W$ik^i-waVb_rQa(r=!NGy&0ZJ_@&`zI5YnfWWlwz<#i(wD(y)XZ+D_4?e9+SXv>shsZgxn3Pr)NV`I-6` z_CuGXy~1IMC-yHL#mVF0jdg*VOWRk&^j|IHo}ZnOcoi}z@1s3}b~UETthc>^)>e`Z z7le^jn!1i<0iu-=|4}j(6P7s2U&p?Jk?9adb*k!#k^Wbjk_S6qhGPy}L}2qfu&SHA z==SChD&IR~btg`>p?U=~r<;iRfejUg@EZD!VB(->kfCR%G#>HCAXK`MJRMsTau&>f z4nfklnz#2BAn7X1-7^haTjs;^>E`@(MJWBfRparjSSYO7nVpO`9vs8DJ|p0|cMcLy zpiZN0YW%!!$zSGpC<&-nQ1#o)HetBcWD*ibO4>Ul9?v2?ReMFX8pyH+jRaS?s8L>a zc+`b@X_XBy*QX3qYusX_)xcVJZ1#7}SJwWP77ecX0(YB`tvr&# zS>iPwI>#T#%drL*gK?SO5!Tn`7T9Y!q#k5PeDh&mWCdxck)&D2D=iLgJPnkl?ld|a6h7Q+ z_9^>zXsq~ndYm}?c{r%OCvC4h6p$9@%k=2{-KUR*!ndJ0fodxy!vhqD*^k@S~o(nc#eJ#e_e@gp2 z5J*>G{djAn9tOL`50p37&N9-1r`;UTr)z5JluM->#i)F`KF#YPwhAc{yWfR~q^ut7 zcgj}!-scr7vvgFw1L`$|%bfBML3s#LCLxqop9T41(Wo@^(l>(xt23(Bo3t1^WcLsv z%W9(HGJ3suQq!=?_YmHamuN>Ay$;FyfU3)>Lz2Kf)dY!4fxG~czE-+KOSlDvk^Vqj zr&Yd5>8F{c!#U}8M!m<9mKtE;$S2I-FC&Y*B5_w^(RI3|T;Y9*UEFYvMdd7E)stKp z&wDcrlhxIoZ2m3Nkel`8)L>R&X;_P~N?$Y?EO*8`6QX&F@er}}E_^d?AITSt{q zoI2=4diEr4KSklMD5@co}{7q@oQb>=26x#V`3e+xx^Y3E~xBj@2!JW zU*=Mismn?GAbCOf?|8ec#NbJEK2i!J55WIDbCZpaUlku7w3b_^$Fe^)_29D3bVhY6 z$)`wzmNnJbl)pv75+q+Plt=yDZ~_b)IS=yZA}a4k>%{8%z^XTkkk%R98m?6S1SWMiU3JM(c&&!zRk}vA6DYzIXE} zKx+W%eRFCym+_SmdZeR#}kxP<26Z#TsqZ!daU^$N z^@4iZj4OX*>-AH+Ah(S2qk2gBhuUvWSr6Nrl0qJB3e{aTjCW6Gq*tNe1uM`a>Wz&K z)_Mju7o>Yc)>2(=G-F*APmxzak50OrG>xEsfY%APNYk0A+=naQb@0)#S!nT5RH;b*PtFjHz$b1Z1a7R(L`4bXfYg*N<&LaY>wEN3KVQ0A`kIkJxxAg~J2vR)M`#r?fHJ}mwH9nnSshTB7xg8>^S247zO&Zl z+3=rv7q0RJ%1hX>Q9I!Nocrh#>nA9$Qn@CkN9#~FY>%$Lt8;u8#9k*1XS((`h{KWv zeTJaCxT@!f-&ux#rs)3Al9Bdhl!0njnx0jD4eFG0@bVr^J}C*+I=6h70+bV~9FmcD z*Ss1PPaJa#rk;OBdF~&=&sXFTt&lzg>`jZre)9`3HpvEP?vgSCm5Tsr8`4wPKxq~7 z1%fygmG3Dz{U78_)RPuvja6n1lsh3$n8Oubw6^TZUifus= zM#ezbuLg2L^VV{^tYGE;P;_<8V?9TQflsI%^j`lPDlBsG-@fW{)7f#jcvOU(yeI^k zfB2~#Fk+#+ot%K*-8^7-;Q(&nG=iVIugi79o4_Tnb$GB9J%f3UrL-Nk62Cf3<>8SZ zG4IS1j9$Nj-x_Hut#eJmsa-ej_pcMiCO2WvvmE&4i-kB~U0eFTuMMY1tV2Y_3%pmJ z3Im7jfcM6=<*SnBoW@K)vqh%_Eq2i+gYo3Pmxn|P58cloaco7ts0;23&a;~yYoQ~ z5nQ-j#Us0IWa`OGT$UOK_cjmX{XFe~_J-4#rRiWtc=LG{gxf5aJGK9?hNlB8-g*wJ zPt$6CM{a`D&$FTY%vzX|Kb8Nw;3IACRhQ1Q_F>(mOsMVQ%Gbn<0*jf4w0);}!on{z zAiL9M%y8>~rayeSTIWmAlli@2O=P9`iSK(CirDwRk=9SXyM92F<%dF@dGYkRsW_*> z723mJ_{*lTG~umFxLE&VZ$CDWyRCPjpKmm`x7!SdbEENeMRj?`F-TawYayRp@|IrJoMrFzE8)Do zF%<2uChv@xk0%GZurb>uq*o4+i(UGI-5VEKajzQAoOl@>qHD;3j(*TSH6CYT0=xO& zMLg#45@@~Q;K;ccx3GqM(Y64Jp6B3){K-6gWp7BdjpMZDRD(JAeB5o=wV@mnyw`}L ziYe;2F#hvdiO?=6WYt=jazO;oNAsylDpZ`Orj5|7R6J?l^RffKS7> zg8uqdvUOxVxyfmmtYhAT7mpnSz0+d&+jDn>V^<^CFxrscZJfpIHa*HNJ^KJgB)Rg% zpX$lF2CY1=x9QF5cZtFv*^tw^$+*VZnElL)t2G(y(T*E$(pA06@<-Iiytr$E`V_xz zx-WtQbKqR=EL>39l;3h1DmE7%#pP!v@{hp>VblRV?%QPp9}y$@e%9T@gUN8+0(LEoIEl> zS_U5xJ!c%n@XjXE?c-vk9sr&2M5wf#$Egn3z+e5u$3E9?vu-Wy)!c2;Yk z7&n@)`8iN)8SE^}$ycoy(SR?jT?=VWaw%)hAI_`GP3+&HqgW-TPRYY#Z=B?{mA&|p z4oXa7T2vvSqKszu~Zr6Zx`C0jJuXp?Vp^KP*~@8#aZe3aVdr?tYMFeBT^F_yp5)=3!pz+iDN^8RDW%PNm?xs)t93?(!#7{?=&<1#$+vizucJzdj$)^LZp2{`ng!5 z!}!>e-B22ug41(8iC#BTnI`u=Qq7`SlrayTa06+tnBm{EQWG;8jTT1G{dl(UIk(MZ z=fGUtSr*3#hv12AJ^0w_9!vaDrm$U%j%X+e8|6`>M$qj!-5V(C74YiGGGVX-66UGb zVr`EMjr)7L-ppH<_tfpkdw=>&b1#(#QnvD`u|wGXt64DH&<<36Q}0SqFkC(@tH+yX zbiocq?U|mgr=&H+Nxd(^MO=b3R=9S{T+*BcahyDUxr;nB+(HsJ0Bo$L_F9b_3Bz!> zZM-iP#;j|vIo6Rp6YG00OOf-Ky9#|#8MtGjs~Am(~;^N_iia9ej5fo!V}<$Uq3lD_9k&jsYux92lFoU1-Hd|V2@2; z;j^>omI-tXMUEPa=!BKx_{2+^tJPZLT-ybBTE7ipNeRxn+!gj~tZ~?nk-XNhooe54 zMe94*<@S_Wtg07Y=ZrYkFkTnt*LM8Ny2=s(hoU4 z=U~^@|Jb09&>&5r_MzKjL7I6OO09c*g z0}o_$5~{`xKQsl>3);5pcj8RD7%?a(7Rri`Xx(aW06D!akWP@}O@gsnA8#aFVB`2! zQBo(AF8GNCs(l!`hN0^9u3tm2B4@6={<#j2MxlPbhrVSE#DNb>LG?r0=x6Xdp#v&y zvNEq0AGP5fbi!r)(d%O(>DwA49>x2U&)^B$&)W1MDZ<|R~%j@Ye#2k?rR4r*&S&F1{P^@p7mAhDn(SGyC zKZf(8$9m)BCa2J3<0MS<>kp$NwulM>NhN8yZ{?>nW___v9@;fG-h_+^s-+A)un9>9b7&%q$n z9@}j#&@^$riA!%d$QDLTptIdamb%xN&$Khc9R~X~<4?M=E3TcGcFY9T4|2e|bZy0a zx@Vc)W+Z-9^8tmGG)~^7#dbKdJDE{U!_Su zFE%161DBj#!&;9o6PEeYvh0S~$vt0cVOTXMsr1~y&1G0eE=HxV9_+H?g#VIo8HvAv zxF?%9hN-bOY}*U}=(|Amd?R7GD3sNkVlh!*qBknkVUH;qN&T-fRm?&)Y-0i-n~tpfzuB(Y^-fwZ=dSjnf4 z*xF_XZXddu_$nK=3~P!sUr}>iJ4QMdJVtEc>A0Qr@FCdvX&!6%C5qN)Ee77#gJzl( ztn6wiN!uf7VC{JsuOZyR4cod%rHg{^)JD<>@L*ROoQ(3v($E&1I01i+FTwqqJrKU~ zBL4g0uW*~@xLb6#3y?$2y94nE8Z=8+{RaWrX<+N{8n)jxlGI}w)!!#pca;TeT|xEL zl?}Cd8&h4%OVVNPz+ zYAm!_rA_ePsiRcq`*C4EEhCMruns@94CW7q{$U?uucPuL)c3N^-AK52htBQH-H7XF z+i>C)&z6RJap2|=U~u-axOvkChrK?|X#JS`nMsb~^9gUxF9X_ptGiza#DCqb2xx*nOJuJSvor0Eg zlNA5)L_yc2j`)JwFR$5vg+{#9x5wDjccaqgK)k9vgJx37HdyXE3(gJyhM#hyfP9CW zs-4b(o%sWgW2`||D_UzS*fQChxak(s>qO&oV_*fH+eZdeN82CN@Um;;J=SmdRj9 zjh!0hS2k?RgZ}UTK#Tb?#P=h3-~9D#(W!Jn8Vxo!J1U6tvi@5C7CT!kmJ6)^(*1z) zpi%g;Dy>F5CF{T7q)jVu$@F&GZk;!g|9OThi(DCLSD<|t)O%8`NBDjNd~?zjTH7?^ z?=)6?OHN(h3lgDLP8{KVV^KQlC1H0YRyMdqb05gb6X3VX8%RBjpN5+&yy8^X%*2~` zcj6V6YNF58jn+B^SYTkw$uQ=NF;9xBDVwKkXS5gcb@*aIT#NmalQiQ$&!KZvtT>bA zpu8gKN2zo)`5i&p3yHT#$4pi0Eog0_@kw_<{NPD7$wD(;B4&{aI-WY^x_5Tjmfy%+LA?vZ^zA^c2!!IbQ9kCupQ|0k+dw37GkRx zYyhe=Y<;00Ub34b-Z~eP9`Qx;3fdOOE^8gThU1k)H#CgzC&cJx*z0_q^zLq`6wBGvDP2@gDlLuVHMM;{Btppd zd(h}2jZ;N9?ETmf zuh~?J@6jQ6eH(qJ)iqSm7%|~>Tk^G?Q1R+N=zR4k+gt5*WKoYiepHR3G%zRZI?YLucCL;4hY-WlIh1GXrn&c zVbGG3|6pp2)-9{k9yO}!d-5!rDer8A%2;M>8V5tYgS6x)8Tn~&sxTtlYbq#5VaF`G zfcR9G)-g$tZee3*wt|ZJbriOOT5HO5w4bJQ(yCf9^&2QE8~Av7jI^7G@e0vVke$xF|ih;DdXZ+?{5a!yMwO!ww#pH(}eGdGk zdsHh9DM>SdTO;XQ!8+BIh6RP2pK@z+HgpeMY*+}|ex52DVaj*=b}?quGt4mlD%v_s zM>eS!+*uf=Rk~$fMGbi{e~JjqrF$v0Xaw{(5TEf?9UTSbMapxaN1icdX9m#J-viuy zdWf=Vr&-<^KS@2rDc8ZG$Pa`w8OoExnU4Fldwpwjr89_2C&V%SX0@A69Q^)AJsjLMEOOy}7>`rw#ww!W_ z417MI9q+L%Ul=uJivO{Ct2JV%{VY_PgYq}gp(v9n&G+-$1C8?H)Wf*&nYY;Rd85K1 zP35;#rCx^W9m?x|;<`b&27)vxqz`Gql_n#f4bgq* z+%xlYjP#FqGy18*JxM*x=ZN1L(*H=>TKNWXe$siM{Q&Z|3iB1tYQGp!e$wip_;ukD zZ2FJ@<4vk@J>6`|YG07X@WW$49+X?Y6r>%QsvXrMp6P{n_}WQE`H6PQyJXh%UoQS` zmyfdz(->vMuq?H{xb0~RpC9)cs|sb?j&EgGQo509%v-~y&FPwdCbDWhqMC*9iUJOg|8$aee_E;W zLv*hNT&LPfc>VPO#3xq6y+dQ!nw=%kRrZoIk4!YgfH~Gc>5237Nrl<@@b)AB8k82(nlRhs*2o>$7*GPQ-nnx8wi@;yq@X@+maI@M6FdcAMQ+JQCOwnsE3j4;e7CUNbX4eXXX$!)@etM*;C0YAV z9~|yAg12r`3;uLiDK^YGq2?%7wCOHgGi%C8`@&?sCUm~u#q_w7bhs)+uT8lNd?r8YdNqQDILi4wgxO?_QtbgSvb{^}<@3@$<)Z_Y?o%#`n z)oRHT+Xu^#TW);m>oie6%$Pq~XUWxEnZGAnw>Vx5Z8Qy6oJ_-hLDR(Y-UV=FX*zEk zk$@Gy%h_DIm(SuJ4S9S6TX~jSj}%~m>cARIm6_CWpqx^ z2P^(PMnag!P+l-KMm%%31fSI%d3<6p{&nQg`Dr*`Za9p~S7x%db~GGqunv1~Si-Kw zKLSm=n)3X{Aoj4NKV+2Vqeailpf|CZr1fX+&{!VgUTCtpsf;^t*0ag>)_iq!Lq6@D zJwG+Dt~9h6BU5s{_~_#=VE_Jk{0-f=Ie12icyZf>*1QaM{nM9vX;FCG!j=2h8KG)P zjCNel_uTQ4<=>)NWQP#Q{CgZ7SXUw~>yzBS#bDe<(D5$oOL z$gwr>&*vyOmbDquCf3?j}$sUZ>j}7s+fR24?;(&u?82|V)P%TP>d9`^|GXrU1+7zgcB-I)3U2Qw| ztKJJHPd4Xtf`{1sscz4{|sXmV~>n7i9o6SONKV#oUMdOax6n2&F*QNUnWE;PO(BSI~k^exKCocUg z-f!P2rbJZ-!W{Xf&rUY`)-ZP9K{8%&H52CBeuBSm7#jUdl2_cj$S3y4n8Li+`Eh*2 z^QodubB#1_eh*LUJ^`v})^N75q&-8Kwg+t9Yanvv0`{?zD}1XOXRbk4h|Y_X2PWxL z&A%2c?A}0F`T|z>*kx7!R9kdDWrsqs&Gn|{Ucp1~IOPce$-{8tyh-rfah#-{gI{s8 z*qUi+@cUx3Q&cD1+^-LBJM1-1slJaNT9bh`hc1bG1>;~w_Fhqv+1b<7m`=%ix`WlR z2VoJx(6;|5RloAmfIPHbJBn(L?)|a<6Hsk)sxQ2?y)z8)--hnzekk0;XG={8BPXG* zr!((Wm(H&u?5B0X&yP4){kU#%1FkT%qJ|5nV_{r*d+MJ$@<4(s46$p;d-{dLwrSPm zB?~io=;t7azBWXDek4WY@l;Tsq1MmdK2dhn$;MgaORl)*@rcW*EGxYop8J`ObN+RQ zH3u719jn8X2!1VoJU+dcBdCAm*em9+$?Aj{a;OFvopzG8F+Q|MZCGi*09mqf6AbHb z!8;utDF{~~G{is_r*D!yR)q7G#!a~0l1o6C%*%@Bfc3&eDF0SRDjd$){#YC|tf%@3 zU%zaQ%bB6#29}m~24?);f?I;Bq5gf4zv9Y~@IgC!;8*r8zy%^M+wzIb0q>u-1jQw^ zxA3y&9n{XMfnFebxE()BzfWsl#!x z!b#p?N=@nU&R=_B$v>pNg}_D6Q0>8y7G9uuI(1Gx_M!MH1|*k4+yQUJmD%kSj_UoSORg}N3|~5T(+-Qmb|dK< zFx(G3rv8E~!aKCOUQ1q_-bFTS)fDL8EcvoN-WomtuH7zTOPm@?ngdX;vg+5JIE|4{ z*k%rdyS$IH5#AVLCF*xe@|;)a6skULc4`zXGJB4`gL;a&M;n5f&M%<4Lx;wrFuT zh#Tg((Y1GLs4wpV&5s?K>I%ERcws=9q1rFZg8mh;j*gr- z6;>vu!M9O$2n$DZDA`W?x|EaFVZQag{BvSWEY0Dc>w?OSG_lrhznGcNkh;Jcy2=8n|mygnwUL5yfBD zz_e*Sz#>9a)dOima2l+U4OaA#wC=LW|scmR%6ot+?jkV4z zTJeI%ZSlsv)$;fg`tHg|2QTiPtGE!9UX0GG4{H8%KN-q#`aOC3?Gf0t-esh*;&aD` zTID@g7a}y@`}fmQ zAM>)vHCQe1ibip^!X5*cW=h||`BBB9@O)db^1x)-yzNgm#BM*ko?65rE9!INbV0{q z*{K0Ydm@w`xlYgTAw5RAwjIno@?5k2d8#m|d<*j~)4iA46tI&H$*^c^0q*;51EhIX z9RSq`FY(+h$WuVO)xKhvM|&<-9*0#UPcp(aO~u-)NSLXu*&+%yYA8?9YMI&6BrKR} zCKj1ZfrGvExzEX$_^^b|>uA2cYF!A6#excVLH$F%$X$|Vf>P&4xOI`YFgB{qLt?uz z*MJ1g(FXdO4QWrsXPcw=&xGUZ>Y;Gm?-A}9pTbBJajGk5V!45QTo8sW>jqcEZs;-h z0^Brh%j-39;^eQeb(2U|lDkIn8&}$rxEKgKmF~)3R=1(jV*K!dWc=oTRn-;dKl-O3 zoz8Dpj)RZYSK;3YbpAhn1KI~cx?4KyMncQDWuo2cJJ@G(mgwGNr1)kxmY#w2SW7*J z>yzD7Zwk_dEXbll+?kS#e~vX1^!cFYTrO63a@VN-P&}u!)bCJdNT2RU_kb1$%`yleTt@v+uf4-~Rd~o^N8y7aS=MmkzVfu%BW-u=bUtXClNw=ul^dzsU`3OhJ z*S3=6U)jQDdr3DGpeEa&Fnk@7))cDNZ`@e}vGeQ8R_>9E^qk^vPCAXhNj(Z(bv${! z+XBc(DE|q+b^1Zl$5=@5r0Z*!ttFjr#yW0Viax8eu>JO$QhW6=G{|A1{*~oG^@uAE zJOiptObBj|HzSIwFh2f9d!;==dG4`8zX`&9HplCf^4R35_#yijQsrL$|9Us3+A1O1LYig}2n#l-#-u1xjZxu0z@X-1}o z{C2j&$Ybq#L5^$@3+1k^HfScu_+-Lg5TonxjU+Ob~i?rSf7^v%YGk_dBn=33-jW zTGDTLXMY>1)|ogE`s~{;j&8I;M{gh1BjhW8v$TnWC1D{F_Th>j@t(Ktx2fv?ukD)S z_}Ax|$hC%j2LFLt7dkc)Z;?MJ0Huv5KlD)AgzEVKD!oypo2PU=x;QO^FE>|`J{yFz zW*X&z7U#Lcz=l~Yv8y+b_wuAXfsXMI&s@(Fjo*yIvNOjNW@!|kD<45ToPF>5Eb zr0Hqx-YX7(H9Ln^VJ+b}8rs~43EQ^9xUXS^)z!FXjRdyobv32?F#l~Apc)XTYTID) z`dFpoku(kDL|WrNx~Br+n5V+5Md6{4+0}--{#XisU-VU4S@|!~WVm2TFF&8vAJorc z`BT`uAr5z8J^sM|y&w)l(%_2kNWWZX=Qrw#+jK1k=?PeuZNTqEAA_VLEmS{i4{eVoJL-|gB!I*utF*6C9(*n7E0gg&wJA638mL4V}PMC>-qZR6EOS2 z7Rq{lBjFfk7=L0@&yioJeF?2S8595B5TqS(Zp%~DJ2zpawV~2#xVXm_PZ%9os2s5Av%+jTO}`yM22Y%!&^tx<*fI=NBVD#Pd+x?Oo&Mp}+h-645$ruu_; zKtr|2&+Vcz7SMyb>hfcE4r+8{J~m>qv8h|Hh81 z^8TONzJs4*Ia0h;p*IvZem`54Jw*5VqG>TIQF&+Ov2kUh9pB}VDabo(NT;!pW7h*|F75e} z51=yA=bv5Fc*Q{9{hGmx%kc8W@hl-YQ$xJQNPF`I9rg;!8>HraBTgQZa<$j!xz-bB z1TTWN5shI-qiQl_Ogy8pG0OdLVM-_?oh3KD4#sJ1&4GLaQjUTrpB-mCzP99U#o08b zG|EvQpz@IlkE$6s#;w5b7=L@V^yt~ACXOnvg7Hor_~xQuL3uDVJClgy9hC+^>(h~-dV%-`W)EBhimz%E zyj9!@8WwlSArv*8I`*FFteIbNb((qr$OI{|44LApds<3qwN zWdDr>hsJlQ?t3Y|gomTraPm}~_>_3O8dNTk@JwfpAY8|xy9$AFRruO&Cs5rp+Y6QC zt%sq?s8tSt0Uz-;3}BSq!n*c$xLAL$(#NPg&wxkkku)4}&JBfqocdAq1-Lr50;(HO zHAY%f<;YdJ?LJK`8`;?6w930EUtpwX*qCO8gn3EWe99qhMS)a(gHPPv6VDyPljE)v z*0e;*7Maqzq@9tlNcfzbhW|b7#Hn|%`;{yt{RX7*m7b?;F@`;8*O_|DkutIIjIwAU zExY3W-!0@Jv!*;P`6PsmY%ja}>v5;4H`%4<5s=#Z1sn-~2RiW$;Ar|9m^7*c@6_CY z`F>8a{p-4XV)08B(O`w#b>Kfi?}LwJoEwJ4F01(1z28Mp z>MfBRCE3o&FBQ6D3rS+Cz)6-KR!DxJT#B0TdHOXatrxsq9E9*X0s7UsgC+f1u-VO%(ZQxW zpMCuoifltppN*roPU2_vyYqx`Hj?JVf^HaK6VDWEe66YcePbfnJ+Z^+qv?Ev?Q$H_ zWDnFUU4YH#Jl}n;hO*k8QGGn?+$?!u@4d5Eg^j^$3}tuJ}-HpBm5t@qK8e|QV>6MC$Zo;UyZ zBM;}VKFtgU)aACZg?Lu~EYlgDkHh}kNFFB!cN%01$_qL7>tSI&uO9zoc2Rh4rac+y zeT5_bV!&3ap%0}h_qpx2E%0whH9_M>m%KKd<|#K_-X%H=Xs^a4W?a>>l{H8372W82 zo#G3bCJzLJ%}EiXC!SY%!FEL+0jga=bC>BqyNVv;{H49mHbHfRJ==_u^d2_5&POPY zxa|C3O*QTk)(v%PO#{;pC*V@qcgSgcTD!oYx%SYRqwqmE@pbhjtB?1PumKh)=`+fI zHo|W{hOF;|Wzrv)F5Q3{T)?jvC#&^w9o^H`T<i3vswF6Xbtk|?)jLCx}AoQ^=V*Y z@E`WA_yr4QRj}QOx^n86k(#9Pnv!Y+OR61XIYYjPc3JToXZL&3GMD?haG4Ai1MX+~S##MgSDYDV6T zIjE&LFonmlOY|Wwe6Rf5x>VF%(u3kLm~UK5_a;3(O#Isb)tYR5Ru^(ZYV+^q-TBeJ zOL6@A2GX_gq@;X7i}{`KQt}!>^N_UO(!sYYuY0H*SI0Mk*b+M&99bJY`?r<< z*d;l{cq;GqHwh`yVr_#yp2Va;&1x~(qE zDZGt?9<9?57R%@P?}_XBV*k|^TFOnl+mfF7&~9h-U30{!oJvN#A&C#+wr3sw*dvg4 zJ;-qCoZ9?lb|P;3>PBZ`b&{>3>!VdU^4Zgy;;e#nF>32`vG>V8!nFg=;U{|Hj1jHD z-2A2X(^!V1ERNxny=~~;n5isu!%+Ef_9e<+57d5c1(fT2Vy?ZyY)$Nvb=ag%H$LB` zr#xKQ8%Hj`DpYLgho0SUh|il%hQ}T?r0vWEX#V{eHeX{!9$!bk^QV1u#*Gq1tL8#0 zk3C2+1l>=~==B(An{rhc=JcU8alp~-^dYeQdHhw!Nlw}m4*JD`+r8;oz6tBZymPg9udCMlzyIvu zR%kn|nb~<*;eHs$9B>g@M{8MZby8zqbP%gQF@|QT&Oq~mrY$zhjFbq>w`tBYkE9CH zGeYqgaV})hc|vUCNLXoJ3kjRhxqgtO`O@$1X$BbR!O<7i+LQbmow>IYX?>vbdq;eb zYRwf-hyC0EHHMvq5h=y0Ca`;%Ok=_ZnIReG3|L${Qry zl?`8qvdvdAfbtKH_Wg+qQWvs(n*=DDNc*5vu2lG{A^yVacR%9u{F!3X%Rol@i%~tp zhO|IVngr9*&x(IPJgRa+U2j?&e<-%<3Le`$l(xazi>nL8p~S^_tCy3yj?xQ)xK{kW zQFtcor#ZB^5Fsr7?15_z)n#a|E9Z0T@aWzDLCqa^*#zw^!lhqGwTX$RYf~*n;IJ;+ zgu>Ug?`|N@_yvTSS`~w}6`??5&`fbH&uGxFMU|!_U4>MC{Pu1=S+Db3_+i>e8f;mo z*$A6qQ($v`ee*JS(E1jmIOC>adYp8E3vmSZc+yYtC-yNtp>aEAE|0%N7SOH*Z}oTs zwhYi>zun!X!ePZ_X#s)Qnac@<(fMl=ij#;US+GrwCQ&*T+dn@ruNabFU%kVkFJQL%gH$Z`xzhdPR`_^CWv_ za*g_mR{UJYE#i8c{jg$rF*D!vowZ8;fhxYFxnSTLXQt|bcs!G8LKaOlmP49Y3y-0X zvE$13TKyHrnB~;~rQeYBG>MpUAOiK4(!SJ9s&S?1&jhc= zA(v*W^@A&J7ZtW5{YV-IUjFZ;&^f$GVF+wr)fE*U5Vv7!8l7`Bsy;i!U1SHBN#H9^%XbYTmcP!X3NiV%~Br^ zRGjo+R4F6<1of*A#1G;O%($wdb6*+?!WprpRtMR0{t%(J+>|d(z9{6@jZEbrX<_2c zjqu{gVA4Hrn7qH1kxqe`(*of4x*1?`DoN9x?pdf6`PzBZgafJ`s{F#Xua~hEyke;= zybAX!Tnr>>YBtm-T|+s}NH4Ih!7s#7?=*bZv8@d8UkV-XS;CF^W~7nxf$*5IhB*-Z z#STVhWnpsYRzZ3a2~)`rr3%t>q)VcJbhXA{`ZqXVat3!B76M%pdbW846kksG##ht+ zhV5 z)#=&HkNfKJ<>6~l@y@>?0YKPNrKNr59##6RO4qs0uqB-uk5=uL0qJABo}5jg8Zj#MYT(vfp|;&iRrDa*DszhT89(AXR7Px7_JmbE4eubv+!T1!Q3I%<;9AQ z++d47aQofj`sUxzq|h7*>mXvp1ynxu>62QrZAyRWzj~KgGa(1YWp72sa5#Ken11NOx7r%WvWBft zd2Hn?Nb}33Po}8<7lZ|9@MRKf^Q@RX+wRC|&0yif5O|QfM-c8}Y|;U3Q18Z!I2@d` zT`;y}H`{1GQj|8(=he1t!|QQf$gj4-Eq@k6ZqG)Nv__RD+W0yjNrRz^8R=PA9$p<% ze<(kd3XMmGV`j4pDrfQg=_{F2(}Q6goxR?y)eYGAdOZ2I9zfbz{hKiCKR7XWnw&f1 zte`PuUfLLowB}SmE68KgZeaa-F!FDzj$ro?Pd@F6l_Z`ZT$_os?o7R(@SN_&DAqnU zG7*IDic|5%DHCjM>KxuB_8=eHk@(ZN zikmJhIe_oAjo{wQ^_VbjKk1<)g~uB5Mq2XeQt54VeX0ZadMX)?jT*{GKY-11MaXwPTTni|D9RNtC2R~IwgPw_xG;NoL{23|Y=r}PUu z*U?l%8Uenac2{17eRgPqwGK8BQ#O7Eqx-RfJOU7Zg3<((oA@K74SWBuHd0)*t$xnd zOb@=m$V01GllSZk?~@9U^iq{~CvOSsUp8e*6TkS}r%Ia?-QSDSc`Ws)`~=NG)fnt6 z8^taDSW_=a3*?=J@)@1)yMXG8ke&kXEjij*1)Xt%b|~sC--6_oA=+u5*c~-c>2glI zz>mhZ2J#7tm-(oiRZz653|5bCM*V@aBEa%J)^2>85&q!e%5zwA#|W`x=n^2G3Qqr$ zMbWv}NL(WRUZ`J%OXTUQdOGCIkoZaUL{QZ@@g37mIfCRhfc!a-Cz03N*ziZpUKBsz zc-wO;+QjtK+|GE%EPFKIgy)RgTf)a0vw<*AqqGb4eBjS>XPj7nKT|rC^kY>We;m_T z5|%4YL*>boRwSGjZ9eYDuJ6+nzN`KZ`2EfR@+L})VZldTQ2L#Efz+EQp;|o4ltxxP zlh7B{xaw17QtvFeO39JR1*##p2{0`-J|YFE(fTK1}@cBMJ>V+w}}>aS_$ zI0b6RPr~9U5kg@`YvW7sJWHPsEo(x(1XDpC52o(BhW*c*@{yZ@Q1zgebdEyLnvQ~W z1JzeQ>W!4+mm{eJRM>Xm=X)RNM>?zSpNU+>eSAe)nFaecxAC7LHgg ze$_bx^qJy{TR%y$);RWATh*T+56;rp#-Zwms(u*Ny2e1)O69SjKCHrhrK7NwQ4*ec zc8rmRR%2&RhQ{FoO_(Odt6o*l)Qz6uB>jxa-z{@9#`~?(2xso$w4j5k?zFTfl6W1- zn+k=csyA4+G1K|Mo~fvEf7hCYK)uRL>NkSY;|K2EfklSrknkRk<-S7VP@wv(sza&) zpfy6m6wQj|eyRt+jp&Ylgo(j}l+p#bfoKP#+9w;sich~GW2ZMgxBg3`G%{fmQ+`b4 zT&;&s#LWCCn7`VQlUL`Y%|&y~L7{YBQhBiEz4by=8mrr1BTl?ZIm(f|2^84&uIi6g zYFcx39pX(!+6SiWO;NlhcWElL#B;>&4oLlG?eUkOu%8E5&aCR$_HAy0b(dtRJ~}9k zT5IrJmE(k=zZm%xiqjwJRYWs-FC(mG)GI^vL)^@$r;ZeF(uHND=WkKYO-7}Iy1a|T zZkY{8qg2C;oey#5PHRrS62tbrQT=n}6{>n14mM}OdRL(8iD?=yoTusp?0p()EB3V% zF0WgGackNiy!lFuc<>4Q>AYFv4oSlMTr5PHHGurdA0X&WPc%M*n4dORRxIr<*!CQS2v;z0adjkKquQG@a*9p2hPH?*y;mbf3n(J7_ugAZ8D2kEsowvurDK?rQu_ zILKY-(m0sM>0UsWZ;jFUUPrE8Gg?!~?ENpn&4u&PYP*Td?=^tiwfl;JZ~W0={C576*P2uL0KcZ}XZ5jA;7M4zL#;rnWj|ZnTw&>R%H0tmS_MiF=uEsMUeoltg zV(u8Qdaw&hK9z`D+Ra!ntfx#HnZa`I*57+Wj>qvJocCiI0*d5;%RP+e-4Os^)bmh1<=%t&qep!FE&T_h@Y zIPgV>ej>#JBXhGc&X-GvqG~`f!>sc+wO*Elx4(OF6MbWT{Jw-~r?rA%vKD`+mPU^L^4o$s=Iy6++F0K=R%wV3PJ67cGMM&^xaC|$$YvaG(7J>>dLiY72$UAoyy>@rQte|XWd^|_QNr7IG zS`p-N3bOm0U?D%(^74s;C5<0SvhS;Tu&5=Q(64S?Oz&IECD>^T4?cbjqxAMd#ltf! z(u>Z6{(mk=OS2?&OUMHM{q-O+@v+G1Vj$@AWR}iTk(=(WQR`8eNB2BzHH4t0w^{ke z@v_|GA&YF41)j@S3(6g^JZmQ@?&!NrBcqnok~wWQ$(*RQu)pCvbhgk1OC^!$HQ~e<4hI>|t~bnI3D5!FiU_{(A~@Xt)z|@4N%!eb>?6^C0-U zH;{`A7Vu})zhLPnCvLI&7Tl`0i7#^h3wDDiVtl_?cD8v1ruw|X;2uUY{Xi(~EoUOU zXYS>mXaUAw_jTX^vg9hWo81x?Vp%Md)z7P8A7g#AG~?tG*_FWEms9)G?BL(52t-9wzJYYEjYyxs7{f_Li23QsLhg#MetI+Kb7gB%)8?1%Zj}TBXcD+6p^XtA9sy_Gozty^o zivWw-lbO{IU7i!P5!BjS7F-n-v=@M1PAzo}t*g~Unwu>zsCxo}y;CvRyCYPtsVV*G z9v8oyP2hY%gZ`63kZMe8yvBhWFFlTydl4#48$x7g2H3YdtBqd09-JF=NB>7HsxZeS zsV?3+XDB~BUqaV^fo@~ZK=gx92u?b|3x4Nl{dby6KZ|flb;8}7x0JWeZQyw?9ni!h z3d%Ze#aosJlKu|UjyR$npMiNZd&1d8|3uz$M>=2YI_9tS=I$f(rEAJZ;TCg2q%}xE znnz}-`)Q5)*IF2S#$J})BJFNcg@uwZPTGf{Ho-bTq>bwWAs_XyV6vn1ahU>n@zy-M zPZ#DiZiRFXw*#kV&%tG+nOx+)S`_x$hDN5J2upmK*Yc}SIBcV&_+eD{RG_i3)YzWj zeXcPx`ZxqCl5B*reotZIwv(kF(Bq}FuT`MaZ>_(f+H5=PS|k$x`$@JgEx9 z;{8uRR&zJZ>hl)t6uHQ(=v#sGn}jN`o&OI(W9+A8$|oae(3^!F73eNX9G^_ z#XRY`@q*;;RL`~0Dt9kBpS%FRhSv$_!m)M21xDyg z=ew6??`M{-XK8oHNc3Lb0Q_cbk@Wc(>|_d2u5CDBHBwz5;hD>#>F-(m(XEhLavt)& z*l=%c9J*T_#<=^FWn9rik(Dr6I&2Ems9ZSvstmL1H)e#ZiaW4;-xHvG=FSy^xZ%Iw z#8EZRCY;*H%X>@~1&{ZGT92r`_BpOag zAgsXP))vz3gB!TtwdD#!3Pua?{4-TUc>)T%Ehlt6Plpwdt<#D}T>67KE9%I_cKK*o`-q4iyaOXHEeE#| zi1fE?@uA*eoa8ErTd~61n#KEfku;7f%qnwVD+r??=<+@Aae2v{2i;=EP1ESU9@=}- zvmSU3md0J3@qKl6SdWVI<@xDKW8yiU2 zb_esq+Oqgn$KVz{M`&WIClz*7WL;8t#tM@+RPjMhRCTUixAZ!R`+dJK|E;vw=4y`a z)Td3jxE6yuM5BrRMkHPaXP+EYd`tC#K9Ao*RP9~h^`|{oHSFJ!&f;jTM|0Md#N`+r z8ibw$-Z0-8eMP+P76@{>0QtSHDg7jZ*F~aB&HA!zXH6dX=DWx~-w4xg&>kA=A1G{) zg>jqFyT)M*+95HmA`^4J+{g4I)mh%37_b;pMi?=PJC!`pI=Ag0gQkr|i|+pk!gGzP zU8^NF7;k+F?b)N06lHb|755<3KhQO*c$u!RjXKs8EW4jaF9%m{ zvcrnIT+judt6wlJdMh~OQ!cFarv2p1Ar&qot+R$WPnI|JmkPK2ZzY4p+$RuqwjpFi zeGn=y-DVx7bq(fJAJT6@7?kdG!Qd^maB;O9g&#b15C`LVUAcV-s9MqdFe%4rZsNG<2@t6=kauma!modJulHO&K9lq>PwRyF%aE~FzN9LD9yG3yUMYI z>8G*0a2ii&s4G3kj7QQt#Gl7CDldag4-r>&hu}BHSWveM^R8#3u}Z8VRc#Qzf#>1p zqWoh%dTCI2z0bhXU+I`$QXAc#-p0tp(Rj1oQlNTNYeXD!LaVsBu(|>F>tzRV2M5UX zn$@am!g9iK<~+_7y%$;tH+mM_$G!yp49&R8g~%Vzgui<~mwQ{rFyb3#w`z$X z91+iYt--kOH{sc^?^t>z2E7KPyCl>KL&8v2Vw0!limn5HFw)Rk8aqFG_dntHqP?VC z=IMhBq;uoUDz5a&tRU@sSm`EVS(b%<{hLdr(L(m=VCA&>GCsr}^4=c=(#BA*wwW~E zVgtFoN27P=aOvIIgnK*G@%g%S5c$=0tJw9XAm>O4lmOGvP;CcUri zpyw7IDJ??za}tQRS(-^IdIk0sq|12WuKLnjvkvk*jRd2n#h5o>AJ46E1Jf?F){^f4 z!gcAM5~AMEB3&mT_BAs`cFco<&WFW$|<$DB&Ke z`58_8%82JU)eKq{M8o3sURcpuWIE0M%pB&J@mtMwq-Ek6h;r-(IYGuqJjkg=py2jF zEKCco(zArD1Ow)nf{;bEnX~(ZiZZhgBwM;$6Fni%4Uu`181J_kdn!1##!0 za$+M%{sFR9^wTOG5%+yCC#}dy=V9UGwIbDLD)SCBW#n@(?~fG*xoMTYsq%q@``rK0 z0pYr06wf+8xC-}4UxLzIPKtgy>OS7xK9u^QBA&uiZz+a zQv&ICksw3S^7K)r_$;b>N1!=#sz>4bdt`>&iEg~$v5lnIf?J#WNPH@z{>BR$AM<+2 zfwVm*Um>ThCBL_G5Rxxt#3O1g(dS(OD|>DN4mYo;5_DSX$7PqIPo2l#>M#Yn+R+FbIFY#j^w$-&2wLsW<`s##prw687P-v z(Uqy-emxONM;(FqO$X3<+$AV;uMXu+zq8}ff0%Q{4Q=r8_M{zNIg_6Q^4UOfM5;qb zm=h$um(%$`V{2&B8Yj{mXK|OwH=r#l2x|gv0-(3vam!WD9Z=PEOlvyBu^sktXHA?iUgswvwMsw62@tRFYklAX3i|iD-H_m4-lR|u8O^}hr1x%42=rt z?8Y4nS!&lCmO3zoGF(dz8pZUebLB*SP zn1B0;PV2L7r2&xVI`kt zqnl19P+ha^McWwl47A=en{bQ2Gm*3kVdxBu+S~<{$0M(S6fftTS9D(Nw$7ZePGm

h!lve_|TV>|fefEL5{P&4{~H93g+wo|}fQ2ivddSa;Y)SU=nY)mmDN`-MJB+Vj8m zwPgID9DerrPOyCYf)NhkZNy2~Fe3nIe$;Vs`~KE6FNN?W{U}@v4#azjUUK2WA@cH( z29f~96CAGz*K32g#kggnS#M)H*8;fcbst;2`KbLIAHlDcZiJQNm}E{@S(CxjY0Ubt z{m@D{dVQJ9iJ6L{E+j+L%H6bXE%0ULZ=o9<4WHxn<&qB6_nv!EG_O4fdY3E~?YmDx z-#boJPn(ebEwh$+ahrX?K=|Zt`=_zvbYF2jw~q46Lmj?mbz9l{uARg@V?Go^k)_my zXMLCOAK#6n^Uc-ZH1!I2o#-deem|t0+roiw{C*i+H}%3Fzn))MAE}lpAnIFgMaXL1h95oMP8fT-zyWwUB88-MJZ1(!8eYcCAN3zNS);0&; zF+CZpr+e^#_KP`jlfs?!?s)`a-N$0*C#jkp zVwS6p#i(u}_IEIBzE)r6S{1w3pnFu4s&9m0E6-p*he6o&lD_N+kD&YCC@AhXQ!0#= zFGv;!A7kLo@+4fyE`Uo*rcu02yv68T!Qr|WSI3-m&4smzIHYPyZfF-;#YvYo3*a~E zLH%t)IRZnhG{b8_nXCUVumjX?T@mRS$nPju}iJvQsf>gmVTyvSdUF&g6+L-<9n zk;JX-_>;-RXGW*`%T+;h_4Oppan7HAW2(C3){#bLF5szsLvPS{& zEyg@=CI1*5!45B5V&Rzs(C712@_*l<&7eh9pXD8-A->heL6IS_!cY&hJb|}~Xd|01 zi_>(iX(AV9JS3ePh@?OG&GH0H+Pn_YPw9=MjqsJu(oY#DE_V-E;}wNXM}S>`lq}G?)Y{G`i?7M zhrd1(VZK3xr5mD2Dm^+nIejyz&} z7=Lx)rmA1CIXD#CXFJHjgZr|&8|SfQlMCI0-a3#Mm;^z~CXtWWigyi5&@9=Vt2sRQ z(~M7}teCoUCmOx31+%8)X&qu?PC6q9T@)OB$7_!mH+%n`;6eqq6p%pSTz>jEButU zZd_ycy2DH{;P`m-+nxsfV-4VfV>{A>uo{*o}b4ifz zB5$>duB~?F743#YrbC<<^KXINANz*Gg?o2 zJtO|Ajl+8|GCI$4`WX zosE2PWero~3NOeNL#8!D1E&gFUwxb~em4vA{RgKl$H{FrU4Z!kK@&4jWe5&)>Gs4tvcskyOH=s`>3$1Y&Jj9s?hE0*JTH0`4c3r6k}_eKMd|CD#>-fQxWNcZVJJ6VZ{}pfU_M`fHCj4VI0{H;t z>!D)Ode&gbIMJ;BJ*|F(N3wUoQWr>=X&FQ-!IZ0RU7diMbG9-!BLr}7pmbI9wntHDp7Q{W$C z%E>2l${zF`VmtDqb5Yl_F9f~+2?6cD;r*<+=+v$d*XfN$@_Y22Zv6O`BZ730%bOZG zBWgMnEG>eo!&)KLAOzKgZh@P-Z{ws*d&&2QF{+EI97MiXqqK{- zABN>!L*ggQbf_;4$81Bw0HaI;xmLdavvk#ARV`l|1QSGTEW|>^Mie-+Mp4A>?(W3C zwg{LY79s|S9S9aGXVzdM*e!N))7SBpt-o$o zV_*k#UbyjVX1y%H41L5e&HTS8xqJqd4Sh?Ey&ekq1Oko{_|5oE=yN4@wrDS{yf+%II@}+Ew9` zODU`YFadk5EZhlRbJoi}GYEQ1S_`!z_#rZVR5`*+qW%U>UT1{n>aM8 zE-)#PqRXdI%!y9CdAgt*4m~Uh!zYVi@7ZF@iUMrB9(aNx3;rNzKk;qxRZbXEPC|=| z#mkzSK2aCC&mgiUGw_ltO}?uVhJP0$e(yAzgpvJ*~>U)@TC3Gea5JL0DtVT@phZTgo4Lufko=5zCyFBD{T~d^P|)SnpnE#) zsXRzcE&PZI-@v|G_1hZ_omHp6RYKoe(JKr-%KH6j|G5m!NHM<`p{~c%yQ(1!ouZ(} zaGa-@?+_k2FUqoh#7H$K;RVBk(%sPx41DF#O<^=;Zdt=yR%lc%xcR8j(RwoOA-8WA zBzhk!f?VJgL%XS*^Sa1Yxl0n}m%&L^n;hF1bxwBaFU6>W!Iar=5Sxq(`Ymf5=w;*8 zsd^26K5()$`ZKrjsK$qo(aWeIdH3mfy1$^VoMd}At0$YrMAMkr`E<7N`+3%zErw%_ z7p38)`Y6Mpt0z@rOTmricgZWOw2|;q&_*?k-_Fo8ynA&8hRzpbKa3&ZFB`p;l#ni$r9nEjhClZ4W=vTMEv(#<#eVJA**3zLlL~wD~{tAT+ix<^jLqs`zV&9Bj<7I$Oa;)>b zicN2tT+h;~nrP8$o^t(L2bmV~fY5X5RwCX_?%c@gZ;Sh`%GA!{dFV3Va5Qj$M6CsS5w8J_8?E&t4yz- ztvHPb`8URTy%*3r^fIEE1ST+Y4P}=;Ouj9&i`NfzF_^5fjrZb$BfRypqVZg3m#Y;T zgG|4~)LF^Mz~t_AQ;=Icq#kF-NpPXSykb7=^{Io!*?5BD|FuWvcGBQKyygvp2Bp@^ z?^4cr85U$V8u^mZCCL~oHGEdv3M{jhEImvb?bqJrsL8pqG>`kmlhFD-$Yj;%A4*IP zF}MrO@`Q|*F_~OV=YbNMNFh&P_#kV-xBdc|tMO3A%hJGuxTkMV8Zxf!^zhp;ifmpT z{w;-o1u89M75RA1p@W_JFnlYa-x>Kr7Uv@SfJfR*+w4D6>fIpdpKEO9<8|P0T3hNc zBZt+ro3FWZoX9=;$!)2jpQZy;jcSh1Y^9-o300xqO>kwl5HUatm5K z1bgbbmDgd@YUs}ezbl6YdHQgDCZ3gm{yCr?sbjVf8+}x2$~^*SKTWa)X+sX@JF1R;p3o8W1^?)5mv{@4qbI zn7;RU+Nsw3wX272_BoknpG)E%H(pZMI|sZE`z8nE>Y=N?t&MR7QSfor!^JOBR}Yr>0;^<|%jvAEvVi)zhG;p2FZu60T- zx${9HEvlnw;Waz`%WgWCbb85!QatqDi(b0vay(#ce7sZW{uW5H?FY;cqi!nYkm)ie&bG5JLJA6ky_D;fHf!m7d{V$1jRP0BW`s4ns zH%csPA5S%XTIx&F2Q!XErP2=4_k-od#faQ;*H3TX)bk_tUjO`J%!}+Y@#!_5*DHvU zkNE1$tIIj+@d56=-B!OFx><)@?!h(Uhw#wY5&Ba9SPs8ERN@>YYE@ghKNQ0~aE)+o zn0gu=`Q#5?@n^*3uK$3%RfxEA$v&U)R|!FP}F?zEFir{1T?ClfW!ljl6czNLemI64XM z;Vrd}Gfp0+3V*BXpn{e3f@Y=3>h@Wcd$NTxkA4v0IakoAmnXPVjl(>6?g05<;t%ru zwUs`NI>QeNAExUa9`m+@(sHKL96i#%m>CQ9?wxNroAObt&wiHB7ucWl89$ofOOqXL zSx?2<^RBb4G!HC=UH2r^Ea*}n%>$&*5wz}qzHHTpi*Q?>~X z<=)5Bxxkdkz@Hv`|H4_CkYhjRJTpU%ZDON)EcMa$pXTVXSIf!3#kKTN=O1J{b+%6I zx|dyN&*g`48&&4fK-Bf=WQ>`|6<=xn61sxF#NJj|C(iIKNoDpH@vVIA6?;1^#J@b+ z^nQ~S^{K>Ash{?7;rWwt%K=}Cad2`6-RRmP-Qo8aM+^E{Bv#=VvSd(z92SG@zgLh-9lZ~|Qvuhyzwv||UJ`38^iCIvV-ePw z9-4RFw8ArcBRBEbeog3e!O3dcsq>bLC9CP}U-FC6(M}S0&Cd?E!#N&K-da|P=%jG& z9NvyjR60rq9~)hB;NBlR$*sOcxAFMEfp~_wL9Yc|?@&qZG~pCoEV)YzZ*+t9`Y&Z* zsvJJHA({HnKE&N$hf!wn9JI58qkc7`G5HmZ66Yo#A$%VD{1oJ8Ehp*+J!IdI-bA1LRg{b83ZZk8{bY;#8NSXo2Qa_363r}DjVZx6AIiRW zqP&)t$%7tv>obeWa?RBD;z8m18uP$YqEhg@!Z*I1{{{ow_0s6OYVFlu0&B;U9lA?{ zn?Z9!^v@!trF@#hvZCH;wV+un-S7E9M0H+HjiyiHmjmm`+uxT^f6)T_bVTz8`@A|h z)=^%JSU?+h9AeLQ9YuwUC+KxuT#LK4heu(PcdViBto_yW$$3fySFI_qUc#(T>3yB_ z`R3QDZ{lFtq|9Ri=5pogHVm91;Er@%eU6I`D5}AI(kV`GH_v)%*rD-^@6)-1!@2I9 z&0_kTQW9J)=5#C0|9NNU#a{wt^}~6%ddmdf*&$4%KdYl!m0!g6(QkQR-m)U9@^)dq zv+z2wa-M=C3Lwb?z|*^Hb1#iTVBp}Myx0w%+ZC@_{`>E za%8UuRNkYQyfDdD8hnS|u$uGqPWobS2-$rZ(f&eDdik-R0B=~JZwNfdQ+GGUGnaKV z&XbPpu1ABT=Zbe+187 zG`hw?UNvTo8Z~LCYUvP6m0sl4)oO2{6?^bb!0X4TYwa1lescln9v^xdHivKixIj;f zZQ<76-Kbb@e?7m{1k~_Q`5~<|S^QQ*FXdDDp6}uc3oo;8#R8&RpKa9bvb_#JT83i6 zaNqyho?^n_w`}+MC;{`SRe1+$Hh87clgO)@>2#YJGQw>wrOd1@fdRtIy}e^|F>_4? z4gQq%8wF9$JqJ+7AE-j$B7I}z2rl_0j?OIFz_spuCvYXP{0cc7yNuIkOXYx5YjM%@na zGJGuE&i3LaIZNt@!Px(~q~@Qm#vAQx`j1>!x3az$ir(Nlr>U8e35lF?ZwPhjo=tCG zGaVYnR2`ggk|?wD;qsv=j)|Z%NR16VD0mDY%l_!r5WiFyuaBu0nH-7 zMamN%sAYxb3~sc12`$9n8zsgMlQnl%Qov|hR-vJKIWUF3#P-P2)-y*dfxeSB+Hc|l zZ>}h4P32VmK27U9j-a=#fwqe|x$kTRPO&t!OJW?0`X^LGoh;*jZ%eSlm?{PXs7C2V zJhx~BwDC;_?sMX}pKP@Je`kt9_P1h)-iaJP_x?ThI%g74Zmn;G#a-xS&f-k zS^90VNX!EPhmygi)NONdeXi&{?w|h7x2ff|_+|Hp(SO{z+A4aR*qNaF(Zlb=9gn`+ z)KZZQPw7{*D0I_T0Ud13>~~Pq%XE^P2ISN&+Iw-(dgu*cm+vThxoq!KvRToD?4w7~ zzp3>(b^m6*->|ZJY~yP1j@ozxf^%GXS!ymyuDz1;H5sCSdA#LLQ*JmZzifBls@kw~ zGFx-R)37QRRkbNDYQxf+)Z%^@&Yx^APY2YbCykd7w2O6f+kp&CMwPD(XRl$Ur0;&b zi|6lOm3$_X24 z>N^>|DhE8A#BZkOZhb1r3`fr_e7Sev1ZDWH6|DdP ze>_#C{ie{>oM8-YD|b|U$ocz@flj|h=tFY*nFg&gmRuhkR?vO~Pb3Ol4uk&bZt9=G zN2XtW=N8S!=bLvHHAtsojpKWj;~3|Sp4)1j@#wk$23r?y@5(C! zyTku>G<+ssANLXPfaK96mj?Hln#$_4y?Lcee5SvK zKU5LB>Wh?n&-v=DGBSt5BI7j#FrB~DFH2s*1K~SbNHZ>@XA?q8stxwxyy=67@rFjr zs)*0Es8Rn!Dwlt$O5I<9pvMWGkm1L;T;?cxp8W*<&L+vMbDQAo(&Q1~BaUh8!V{Od zNO&^cMlQF^{M}o%NOjix7J%n%V)$^>0Zy#al)GIHq1Y*SSJlKH@Q|+=`2+QAo|7jW zp2I~ySgFP`H-k+i$9WjfKsXlXEu5jiX)?Ou9<>bnU(Oga6A_LD>yibTyw_|?8FS*Jxp;c+ppz1neb|<-OWFd7tU$Q`d zlHqp$l9E4l2m!y8(!q@3w7%^kbpb)Ez^U zS&(5^!24pyYOTz;fAw4{R<1ii(AA8*!uU@DztLK2Z9@H6jF*~c|3<;@3-lki`f*>p z`mjO4dnto+Mpu_T6GI(e45Zn=VwLf&;7=Yjs)zBK%J=+q1)fo}=lO$grZDP>UTt61}~eslP4-QknP88REAft zf37UyO@KZ1sm*Czt4i-}a7~+x%lOPAZ9^G4NF?Wc4gAh9Si=qN>hSy}7gb@`fePN7 zhF|(k=uNrjU{$WRybq)I`T3x~*1W&->0)9KHXqv;)v^MI=u7<#;6E3G!2}okZ{}(xYD+gN;_`1%gHrl_u_C)5GFffYAk6 zI;_~R>1s{&3yhpdnsqxy8>#2VkIH0M(0-ON+0I$hx?fW8+^R^%UP88oJh7-o-jd~$ zp$n}>6JVZ&SGk?O@b(O!!ZF3Fqju(-+Ti9L#)I>RnmV>rSlC{faRn9JX6nVK!|$AI za=N_p_ZhvxUB>6(E?17z(9lh)&APMd&C2tH^UuOo_XP#<99&P6jfq?Jj`5zNgL(9Z zCF91^k_M-+0RcEx|_Ld_6N4>)Op1S)u}_Xyf&c!3PmT$g?0Z3%&Feyk5{(0DV zbwb`rf0`x|`h!}u-%o$Hy&xQ$rVWc^Xl^xO@F;_ks#Etl0{tO^8-G`}Q*De+&&m|d zIQMgVPG9dYI>yyj&~|iaYc0`b{6gcu40j^`k4gSNpY@k(f1d@MUa63EGtNaoqY-1T ztJif4Ta3FC82(&wMmXe{J_j9iv1#gf>u*)aL^Zxb2r zUWk`(2vvKn{&ebB43EG5AN|2IL(?3bWT^vnWv-ByoPBH#ABX6f?D-;?D!h#5@{g9W zSL2H6$A9H?j{ow?T0YV8PHF@N^s6SP7br~lJ#{Rc!I@tQ=~kklh_tC}N%(r4D`&4O z=O=zwt42HOPrv?&W3FALZTI@}!<&!79RJdcD7MRAOYZ6E#$ly;sA2D}@u-B^GKYI( znwi#7wQIjX#I^cH50|dwjX8?Tmm{j^#Ub8!cB>_qUvh@r-;GxHZ*3yArcd&w{g-K7 zZAW>jN?Wee;*WYUu)N+e?*aS&nM0rc?BPRq2h*qo8{KoYt*kn{FPG|5oGiy&WcKv3 zTq<97eu#HQ?pRPj4*M{Vk|Tm?=Bc@|^od4Xyj>XA{p%t(_NuD~O)JH2XU?cQCF7|{ zkx}|XR8C#DV;z0sS~LCA{SMErxPwdXnvpfWTJx*RB4J^2$b%+y=>95^_h$!wj(0$1 zKN%=j=B!D=jdXWQylDOM7v5zxMfPrURAqj8WVxL(pD;GgXuFvf9NMirpKYpFjqI+^ zB=+V?txr(fpB7cXbvxfkNat>Ma!E6WV-qLnb3?Pq+V$7dg;K|O(a98XyC5>gwO`fN zaV_+(YL6AZOZ|P_Llm{7vh%@~*qi(>Z=cX$r26wzQ1x28Z+LZm|He4=v1Jnq zJo;FTX!(ZigM?YVtU(@%Ut@z3BF;ogfVZt{pMDk(0ZBr z6Yit4G3^TevM+=3H4)jlUJ`&69A zRXQTNdc0v^f@M*Wj$Ge6*s=uAc5l`WydYmI+$Ufom-tSWm`lRvP0!KtX5PBN+Fh1H z(_3=aV2iwet32>$1>3~W)dkYR~2Nn8a;bWRt$m)8&VE8%+-|O<9Qk8&5TB6_A%^6xF~C4m@61 zPVeu*{_Wn1y~s)2N)^;!``w|zHG8S9*MhXc;URcN*`d)2^ng9L3|}DQek@X7_Iy-# z%k86Har1?|_M3JEbdx!&wc_qRzbvQ|y=(16`ZhdL6_1Of>+3CI`Y<&e*6=n+u!y)0v<8)30g+_`vo1g!)z!!-vaT2W=(hj`tQUAe)}d zQ2y;kQTNCBG}cIBoP1R{i(7-N0zFE(?t04}4Qf;4{Hr+rc@pJ$oysFS?dA5nQr=rLXWelsllH4_+X6$%K`VVgc|0{o?VpR zn`2zdrx^I6h>p0MU60I+)Y&KX;CZ%(fx91gX~UY_Y|=5S>EAQ9?y~G0&DPfw^n@B| zcy93)d-Mc{EZjypXBEOZ&z5%iW{E+SuZzd;eF^w!jktY_dj?G?@TdC#TBIF{ikJZ+sGV2MPM~0Ksy8JS1 z!gsFXDAl&g(poPR+QGe=etE13d-M*GyP7P@!f*7jICm(@awhSfZ;t=M$<$xmmPS;j z>69$7uf>V++#n#o zp3MR=9fcS=@e&u5poyQQke=eY;6{L?!Y zu+RR5T=r@Rc_g{%*B7rb)Sb@yx$nX=bip5B&0XZASr`=_sf zY5r9JM9tfIND(S6!xOjRpE(52wAlN0=)$ zwv4Z8cCNSn(*2s{S&!HBv~v+@c+_!&_n}P(>}ArWg$5tEdbtpu<+c&e2u+sWJ(x3o zV!yRMzu~b$frUxb?aq7#b|};U-#dZ$;s$*?{Ett? zwPj#$GJ4Ch+wQWc|5VYJp7TU%ozkLuIeYSaVJBNH>rHPm?ttSzaqlg?ptaHny1>*Z zP3gE+VLS@Y4RiO0ciC_=^oYk9`m!I@tnq+4jovJtJ9Jk6Hnakd z{8Tfsza8dBuE@l@?+$zO)t60mW+M-|ZvG?<-d3ZNa%R;Uu!;+euBU-{`v0+4u5TJg zwcjr^V-O8i|5nY)PgAemUel2y`wiCOyo0&3>k{&-mq#!0{f_-%iI1i!iho{*mPG@X zui5I3gWiZW(~HyR_>%IIia%pguj4rx&X#@e^I3Ukb%jl$e zqUiF1664iVvvs0_b5DzN`MfkV3pB+8RbX@q?;TK)HugHe1Iu1O9bBcu{g*QIBKZzG zE$}niW-q}VQOp-0C?N`8HHNSgah9?lvoYc}w__lsI6~~%NjqDbx z#UTfAZ%!}pMgac`nZ@DmEBM%Ccl{yq8jT%q!_YBYD(@BOn<&2K_mo{;)v!WCspq{) z>49b2@U;(ZxN5-@hVymqx^wZYdwpuSalg@2rf29;gCGf=Ku_94_@ciQ>YAauQ11>j zv2JU|u?Y2{^QUy+`$3&)z=V}FwBBoIayvS;F`G7C;&9(Sve?H1WbkgsgX&`IxHIZ# zv0#~V)(TVK1l$+EM7l8Ho@iEj3+~HZiTVg9OJF-H7W`yH()Div=;K%PC;97c<@&>mo+`gSOcysdU0v;BX4DGs{;D5x1f6K@& z-#-cpJ;$e-?-vQJ3&?icH`BPO^JsG8#SDB*t~EaaJ@Z+ZUW3n({vBeuZQBfX9UN!8 z7S(?$lo)46&_PzEP)bN-iDs#=rWSjULqJEuA1YZT*9bv5>Hi|dwY9Y}J z>Zf~8GfxUWQ9)C3wmdsj;i?CrPYQdjoplB9w;JXB zd4+m!i(sQMP*=kAW8m1<61A+>2FICW84TmP9g~DdZ+kB1S&6~z5;|6(ekfnxnzZDJ zqehQX*{k+=zqE_y_Ll@Oo+A_b5$acpv{8E8rHxtt|K{>Mb+LRiMxVj+g<9Ht+9zt= zZ!VfmYN+8?eBsNKnaBEd+DYhMx$=kV7C7iS-}1|kUw4pyhtdrG){VMvvBFQN z(nZ1qIE&W?m*J!fUAfZMWwi8lL8G-5>X`k1UF0S%rMbB~@)d9#dR&1kjILngvC$LO zTL*(^(B))s@IF4`++5)t8Javx*BDJu_)Z!0f{Vu58~;w#inp-Ts{7oM_~*2+?S2?p zf~!pb9F>I?riOtf3=Kuanuli9_Mc~St*BQ9k6Y#Jr3$(Qe$JVX7Q-{xvG3{A#sT#1 zK_mgkb(=L&W?n2QJ@U}|B{mv9jt*S!!|ijPVsHbzdKHe`T~Nd8`8p3d$o&f)wN{QF zlBJ2@U-@UH#ccG5Cy zc8ner7bh)|B(LLgA%__bfQ>>KPXa|E6C}ZnU70_oInOx*TsfKjLD>tET%si2)OK>=galtgo zZUli>tYxcY;Zhc#^c&b&SOPBz@T!6i;YTZ9vv&_7Xe`1utP1$6fZICWFNDnbL-VS_ zx5k*9iu1_48aykS6nQK9?0O>Zq~_Dp92V31re&lVhu#@0;6X`uw39vGzZQ$HVvj+* zGt$%!G%i&?odjJ!gONQbnD5jNdYT6S!2)Qh#x2>z<%`>ie?v_Wk_URzI@v%95);kMz3;L}vdX}JF;5klk!_Upc!5Y;WURM}hHZh_kne{TdcHP4hWHjl$ zscpo|Liu!|3N>}Wo`wXRht~Y&3v5V+SEr+CLiTtukbzxU+&I)@fXl@v}vF=85ce~hMk>D z5U`iJ23rNVLPR+o1P3KSC+3&G@3!LJ+w$omVU7x!B-Shi{%skXti)y23m)-(vA}oe ze7jSX$pe93*2pfYVqS+Fz_zhs#r7~ZJd#I-Q$owW1pUS#tNZiCbMf%>eGFb(^7Y+q zddzTCmL`FAQN7}F>GdD&8NF^rZXuB2XydbumwZ#XyulX>@+bAAO+IO`&?mW%c>lbl zg!kh8r-S*+zH%mS<1zWC=ze*vsFz0It2lD9xy@s&J$b_YS`wNCfEHm`zEUeYYWO;kGIqIKp*~BjvRtWJ*xNPWIw zIDIU=Ma}c`mNuzHrT^l4qHNe1+|w|~H+k$|>h^Up<*xtNGBN)&)oDvPj@ogG!O|1@mqm2o;Z_Bkm{VOtvcf@R=GwXeM@~8~%b6$vvl|HGN zjnNBlmvV(7D`n!T%6ej@alGq#aaljVli(*8xyQdkoQZpP^R0TvPv7Qb?>w)}`Re7j z%jzrF8uFz4_PRty4$hbq!kf#SW6Xo~>$|fw=T=8KFXJVZ2(6hl@9mCcmtQK!%h+$t z)an)vy4Bd#;`eR`nJ1*btb1n~W1KRkR%NzHSkHgM%E_EbN@owe&fVMf*N006Sn+$V zkLTLLiw#tX&7RU!>~VAV*<*HZ6r$TqbCauEEF!np1>~0__jp{lGwd?3wm3PluAZ3x zH`_OSk^H-66P2(=(Nw2m()|2Ws}=NUs+azit(!XA`Z1NgSB76~_U3obYf0JiiHbbG zhpVm|Y37jKiv33^qu1~dyHpMuve5U(mg!WqZYnk0+ZgZN`OQ10rtr1mmDIH1Z-nom zwOES+az~wRWRn`DD?cT5b67!%@7GC-`pCAGo9ptIYVrUtSN$-lpuQnC>p>~^+2z0j z5q5Q+zPDnfwRpx1OY3u^q;uj^-qf=y7bu%7M$V4mvJ*RUu3zi*qhIeu_Q1~C!G0^H ze*MT@vf0q#&;p{VgAWbs6USqB#>wp6`p{S0S5hecm%8MYM>cP}jy~q9K{H%pMeP0d zTP*m5YBTo?RJp|@n;<=D@*-fG@Dqq6)~+tV_9Qx5WM z>Lqcm^6R^ndQXM9mVhhA`A9zxPIA~pX00&a)ONd*JU6|at{ZSltgTmE4-b!^xzm8L zZ#Rovzufp;WDo8nJ5!xud#K*y(lW5nGj3ADO?Q5sOYXm$L{m=HmU@|;u5|D`)^mV* z(z&G!!+pJp%?8n!XWy}>Y7KFz|9dgAXgeBVb4xikbJP=mInt<9JN8m|<~4q>oZ=Zo zs~Uvr!)tTvz`vu-@#O9YZiI8^CU;MnI-;COg4V5mYY8v*gg1SLrYvSl<3G%0FFeOm zXh@8*Uy;I-4p-yW=iXWauK3d{7YCMg-ilEPj{3RNA7wWqjl{7H);ov39>U|HykT}B zx$Wk3-e(&_6Ot-2`i%ND$fxl)^zzkOtNq_aR6FlA^mY)xoB2e=wRe|SQqJ(qUiRWn z8GrsMC<5vgx_NXEqXZsh?LEPkXP_pjxZ*$UkY>x09KLBTM|n8uWsQdDn?)rjkDWolL|HeX2)VQ(>EVIB-@DxxFNb#}QxD#$ zd2zmHsM@1T$#1@F5M6d}*O(Ve@K|U0zOb8aAG(F3j{1p(r)|0B$dZKmqv7HCWcrH* zS>pi)sV!|3S8EZg`F9hQW8OIhET!V_zL0$(JNf8r9+_szpgvDM<-s8~;`!uLqW(5N zgO$=?WAo=1$nUT>b#u(Y%g1$+_0DdybRKa+l_)tt{pp>=cmH7wym zrpaLXy?c?4YA{_qeEeVXqsKuKSj*kMIvwI^j{IwcyF@LhI~U#%=0c|4v(sbR1Yqtz zbHh`bfP4CTzxAkINnZ9sy~dLJ58s{OM3C}^2gY|P%y8bw3sk2qI&<8^m6zO;di~d z-kAD<>mZ98T(3MWx|+#dc2~9zO6fyQ7k#!G?f`e{Pft!_^O%3MtXiP#w(*eRW^MWb z^?*6-qWgH~X7BL_rDcX>&u_gL^PpqwRmv_0wulrl#rF!-D4&RS6?X@3qpNR{$-d!GS!M25&NFI+>UpA$YEol8b$pnO z=9j!6md-kf@qFMGGZ&Fvp{X3b|AH`cm$QzI2LGweaoBsO{cdG?0c)*8TUD~O4|S)_ z3s>vilRL26oB;8HUUBwgRkf+V^jS=|0;QZ0kAsLu(qoHdz%3pBV>310eQx%!&D zk;L!Vd=F|s8Lk74srpM#SsFH*!o$@Bx$({rf{vk1@;U(z89FCfwQ@E6%H1rldFW~z ziq5gc-~_E3IiLD>ol9pA?pF9dZZ|qnfm^LOS8ASlQlyM7AqSM&Dlm8YK=L`u8Mldg z>x}K}xyP1^^xwObncHL5qs-5YsyIDqPaGdNj=p8|q(DD;i_{rg7YS-b7 ze55vUpQn52XoL&r$hnMe;XR*NKSH0PmmlNvw}g8mdpW1-PC_s7&DnXh=l#LZ5(Bw; zW@BL2Id!j4dp&H@Dy(xKUHa-8OVeIX@*3vaKw3X`J`a3O7TgiSyP{48qMS9i~VTDr)XZbYd_AVDsRC3)~yaq;))9%==;Y)Om9f= zCSTkip!@E)!grqyV5bV@dHChmJo|ZoT(^1@;cxl+{dDv|8}8~jLqQ+wFB#WF!xDuo z(1voz$Ytc{Sy;DUyN(+V?hfU5Kq?bI9xAd-d9M2a^8y8MR~TJp zYOeFu{Mz)*-dFCr!^0^Q9$krsZTibrJP&u{au*BEMYfr?o>IDO5{C+VSO!h8<65;e zpB>cQig^RBI!RzILo?GEx36@6XI=>oP)QSBh(6PwD)4p|KlqgC$91D(sY-)l4DC%j z&lWQHX>BrLI)C0igg)av7|;sz^evuQ?&n39rVJK$7CIB~TY%RH$K%*OQ3^dIUpR$g zZ|-AM`1n}#$4E*nVWTfS+Q;)pZ#EokJ+;e&YAmsrW1dY`NA}s!u`Lm*)}H@xjGAON{bfvg57X-soWyPYj#{>V7*FT^L}IPVX=qTf z&m|K4Fy6PcE>G$IW{b>!cn=ahL_$a-xonpgy-Jk8D(p3X^fcplgmV^SSL{%gyqD9? zsr3}JBNZx~u1u}9elm~^)_lZ$AX7YTMc+T$33Vq2PMK=fTcWmU>8!?Ce(Hv(peH14 zRnXMR_^_^ye;L?AM&}!D-(SN~EN;?XuA1A=_!{-!npEq_au3)oTONZ;8Xij)&-g2v zUF@niCd^@@k)N*FL%|ry+I$rjKcQ&;Skn>gzskm{}i% z`XXo}fq)+TF>;tY+s5jGRYRJd}H11{u0T+_NtLoFV zpGGSx_$U> zv7u-aQLyb+n)nM^?NuXT=CarAs?ra8dx4`Fy4TF9@k`qIWKW|jw81RYGeu1F6Rtn( zR!eRo6jl3k1KNLXK*Y9{(Yz5CpmOrEYExQfM+>W*YF@J z;FU!f#QP25A?ro>Z z@D}FG=xKsx)!iKDTEHP1{3yakSW!DC`M)&*SsEm^shfQMJ)Dm`9F(PJc0P8H(0jC| z{~W$P>9^@ghIXfe&y_gx@F_3eT^d}ooWSAg&Y_9cYAs3|U(E2|jJi~93S5B?2{v49 zFMOSB1(bxj?WRySS92nf6 z${+C^D#JFmHMx~!8YNB+h2_zIOcvPIRksTU*6hL9@x-1 zE2}Y>W_)&PTA0{5AWbapeaiZ~Nj-vg6`3!-@xH_~0+%7@SggR0M)Rn?%X<>$-}3ry z6&@2;0@;sNZ+!oWpdoq0&0)(KI!eN)h%bH{$z&s@&VjYk9s7(f{I3dQz6pLY%X<%- ztgv4v6y$BUBReTeSc>-_pD|op_M8A*BtcyG`{@PgHPwq zAgK0mH>kBmRW+>`fsZ;32QdTqk<&&iXiAldL!d_1lsl5+V(wBHLeR;Cu9_C<_$VXrVkBBMpmcbm-+PVReWH>bphP6{;PI}4Zi<;{ZP~{f7WVp zu~xDFXjB09Aj{+|FAFRY=RNFo04)^@_T@=7xQ~oSF4(t_fpeVu@KNz()bXrr&nIp( z&krprooEd*y^U-z`djFHHK<}W!nvxtcYgCU%Bi0(K4P>FH|!f|g$_VHEMlAk2R@sv z98WbN=oo9Kj_0!Kpl-rS)92QP&3+1*v`4k;;zQFqe1%V`q@V>g@LV9fQyJ=k$r?nh z^!MBj*=^<4dFX1Vobv5Ot64KL*tMhOMP)p)$z~EgNBbHNZR+x3spVFac|bc#a}I~s zE)dWj?D``Z`zd2@h&tUkEZe9oeGILtwytoK_B)e=$@xlTeYDf7i7#&;`VC?gkA z@YC=IKe_6D>>G5wJV9SFw4X)~n{3tRjl*jA>vkHNUcgHr56L0Wlj6erO{TxO{EMx~ zcm}Gs7cE>kGEC*p?}ByfY&@ynT&5L4L$bl(4Aqwxo(^JY5CKdO21B>ct3mL+vR3hv z!uU{w(OU-ZW#ErB_Wot$8D|ANHXlfC#Zd)|%9&pd@ca17$k$@5W8&5blW8Dt6y+~r z@3~`6Y_ftT6aFY@FtWRVd!CA>SuwZpSyjMiV+eVifS%$=?1`52rK;GM*n~d+*1XxL z5%Q|?eEw)%%XZgCc&}YsZFDH~t)vf2=zH_+R^+{O$H%IS?}I0k$QOY>j%?Ns*_ePH z(T4kD_6(#W`)1R_n6=1ba;#BgGEbYHr@8I+#R43Wm4!9*#QiA9fw#{~WbIdtjCVGC2R)#&GLHe;^5psw zE{1GHe%LELD(^eKItXcweFRxS( z_)dx3M1z;4mzsyXK7n?$Y|A)j1-wv&W=7Hg$I9Aui&ooRM$51{dFb-W>3G)>i6uuI zbjiz1vwP2#A0FDs+@1$$hFd5{It}5DJ2vuCk1%<&z$^-B*pJ6IxUJSaiBKgklUyMz z(ml*i%=J$aE-MuE&eoB9!W;8}E2-RPU`46hr1PkyRk`H}H<{VclOuW`;yTHxv=Gl1 zq}}y(wyJLSEi3Btz&u*S9ME7ixTUYTaP%; zmn)Q$@xcxC>L_xe2QMn>JJ0?J75qi4{O`HIIq<{w+h}g!d^z0xhS<{REf+qNS1+>+mYI`1xyrwz zVt9w0l(zMUD1WHFh{~rpZpaASlk-20t~(&dE{aQ95;D?2q7)%pz4u(JjO_8*d+$9m zlTw;AjUuEa6iMT~=Z3PSP_nnojL2T!x&8Hh>g{>%J@=g7IPde^+XkV_MJ(8|j)}5< zGBm*oqP+4uQK6qr@hk!wuIJlh+Z(u_`sIG;u=hQsxe0-NY^I?xxjW4V2 zXWECV2j1EhFSb=?v4!Vu3;&Z(!Q$&SywsPT7Z2&n2hTVlwk+u^)_r<_@q2qntGS3X zZaK(X3x>;GiM2t`#vioFma^WuU>k8L z8n}AEeLQw)sjU3ejyvr$gM(}#Pb!X}IV8gZd04~-aWJ1k=XB<|OUKT=!D)FuK72V+ z>w2h(>{YV~;Y~-ZUB{Vw{qBf{7rXFMBTsqk(nqjw5$W>rh!>>KtpM-EkFog5Z=m}@ z^7000#z^9^43Gzl%=e2SfO^s!BjXm{|fFJG?&I|gFEhOk=B^KS>(t}pE=XNH8Q%{ zJbtt+N=({S%=r9^;z!Xj!0bw#p{e;=X#7N(Z|C)3`WVb@{1(R=Y!_!3Z^ zTXR#{U`PlOws99*OQ~Jok9(LMlXX*_>>o;$K6z}(x%S~WV!Bw{J;(X>}8?DBUp)qBGY7X+o?G?QqxC2z{5A-vP zyZ#m<{?=lrhV7AkeFwt+*12%RCy&wf39H`&?Y~T+YB*EW!J@D#qLgf{o@#rS{ zGJm#ajgO=n;un(EK+Bx^!tAsbdnUU}i@zfK^BWy-|F{pL&4F#b>(A42b^zrb4AvQf zU(Y+qXB7^b#5#-c?#dC;{!CBsJ~)-~d=^B+zrgPMF2bmn!{ol^rkYq~jqAau(mX>Vf&noB)So_H$+8lAf`27&&`Vt*G=Yg?)H(vQE6)5+phT@Pg20A7; zk_~CUoJBLoNy<-oySg3321)E=HBe+5MZ=NNOEgoum4I=lK)LZV^@<(c_%vq+(Pmz6 zp7-x7tkkQHgJ;Wc8WzyPrIo02u-~+x7!c7;_(f3Yg6h13u=?q)2d)Gre*feF1w$jUf#Q#?qIyZHBW&C`6#9J_h)2ulyo={`z&WokU%dMc zT70cdJ*^#B@%pIxf0dW~rp|R}TdX*J3IuiaBVeU4R`4T8MpD zSESM2Rf=QlEHmZ>5BFozy`DhzNt`p0D;_v;*$qxlT@7yin~HUVYVaF7e#7mWHz4{1r~MIqjKwD`M9))t+$Jp-`VF_nnGH&{6Wfi1-I;Tdu94H+;uL9}K2;dN z3t~{^!pBW*Szu-+-r9eiQSGC;2Yp7yFD*wG+B0HF`=i(@BofXR(>~3lq0U5o#r6Mn zK;l}Y`^ye1b~0!iBMiS7VpzuGHp_D~*1Qw|4qs0r$z0{^(_D2rI~0toLkx$pZU)gzJ)L*hnErT2Z6zU%w4DbLO6FDchq zka-|Xk4qN+rL=%`Z`$*<{*(BJ0Y|ajd^5#yII&%^*0!u3?HQCMMmN4L2%BKV>Ir;C z$|ym4Nb1?pK8zi^GQt^*$pSQ=!m#rOb2>B2M3eHoR~5f|{Z6O%HwMbDpS^*&TSnD> ziJ$%3$w6oLV{CS)*!XKQ#GaN4pJj_7Jy2=QPL_S-=Qo?N=%o!eIBm!YW58G6kX!8q z*k)sl#Bb~=Iw)MxUVXm~oAt@n(mjy$6b3Zv3R4!AppVHDp*ZKYt|P8_JrOC_faZ=c z-AWGYQG|q-O6RILVuryd&15Hgbq^NV$%HHISF-aOo(^%>7({e|;iYkkYsJZBpCH9F z3QBek!#fZD!UeUTeRofc)Yo9fv|VtbjX8{VkB1h=!eN^6D23gO@Fr)PF@1mb`8XQ3 z4v-;}o-5rf_|6#WYwkeeXsn)jLVJGscbxY+MT`~K+1}Aoqw8(J=4@&$m*|d1(mR-N zx18yf8(=+e7fCvRjY_|UIcY24cDDpcx)JY?|Mord1nVRQi9_ppf$|R9gMyIq4$m2` zLE=eh-6jefjj1h($8)Mb^Mgjg+i-u!1%!w9VS9%%XOAaNU}6vi7grRB;tpzJ1}cr8>bc~K~S?S z+`6WVoOSpR5GRthbd*Z(DQ&W>sI5@rR_l59&<%*}6jHTLNvVsF@EOL9zX7ff_k!WY zlM3H?4)yWU^BS=yO_Bw15~R`l|D-{spV?D5V)FqOJ9t50zzq0&QU?y|m&4|mPQ2Oj zFqpn-lpu|Wr8DBOYm2@Vb2FY6?21~8N+fPa!c1r~uP#tMi0EdoDW__3!ei~EpDmH@ z!#vwRg2R0q3exegVd7-G_MwpN8uSW!_q(k01duj@fVDuB(a(R^gyztswF@X?7RhVNFw!v^-Eu*G^o>xFkq6WF1_i(!YsHl}dX<4GE$ zT!56U_dq!=2=}qL?KQ=1NS+70|LF6J-3ydI;M+UQ!qJU~D^H1E!<<;ao9Qfhc_dIx zkak*t5ld!s;%s@~j42PWo`F>VGCzN>AT5kttt@zZQ#W2|?TzFy(LKLFlgt}PrR6j4 zEyrtlzu5VGR~Y5J;(OUmyIV_GDv6UJboMN+#?juW5fA&5O*~Tzos#k~YrX~T2_FaA zAuBoMJEwfW3Nszj1KWXouH3a&DBMxr2`=|&#uE&BDxZb>|IC*MS-EIhTo)GqIzm`` zR_Sq|F~Gx@#gs4EE-lBLhV_{>lupBzV|p^z4Fj0UN2*)qtM5*CpJSBQ&K1#tzC78RY!+^Y0LZtxde`_!lE zdK7;_oQ0%!K>7dKuS!AH^~&4S-WwMtui|MAoMm`p>3LD@-t<%t{1@MP>c z7VJ&u@y)Fe`JKNhey24ZD`+g5WpQ`OBMtyxX2_jNc0j?%{>ODuTM+pY?WMd0R4#o0{NKGhzmI5h#zLuZ%4#r^xh=6)Cpz341_Km@^(-)Hv91* zh@2aOL2ok!VS)6TtS@ZRx)Y`>2I_$*T_KciC!H*OG@VgD$dEMQWWrioP8x*vas_cI zxd6iFxbQ^iDR$POS19}B!iBEZa{KR@7&B^~X5%PdRKBIW))D0yA%DkZ>OC+)n1e@y zmS{-#Dc`5HaNMpu4yU=hgnlu`tmWs)FBA&WSXey%pjZ(@`+|}0RXw$v`)CcX)xfUr z&r#26CaOC2x&0W)M`~!zSz)>j?JZ)>RBo=+cr)$#3)pb9mXW^U}!#cq8#H(0cHW4;rc7 z3yl3Z91Dj2#EBDHa@B{?9c&=HI&=aR_wTdX1M#NV5YT}WH!;FMsrqx*Y>U|4e*c(X z?JiY%hWbEA7>dLhRI3$Y=ITSJynlQ`IZGXCCLe4p)+n4LZz3GN_H?HDhGoCK80C*P zU~K@>JjB%(gQ35lH4mSYBT~P7K<$cX@oOP`1q=b^OsjoTs(^%#WW>*cd`FeLmrP+w7dzke}K z?>S0KeH4-Ytr7lN;*0XQ6;dyRbiy;O#b1WhlTy7`Abq7#{(<5N)Dz{AcV18*v6?K> z7K$@bt4X&RNu>kmp46Y6AXSe@=|aLY>Iwc}!45O4^b2856)){hWsG~{Gr6+f%3M8IE_iY709vzkLpq{I^ zT(cQr#4JX!5yTBhoF%r64$>%1L)eX^Er99<6t^nBM7;$hKc;#lpEew13;bC+Uo^6g|V!RquuRwT8-ruy?DE;Nc;8F4QB@uAukGbO&!`Az6V? zkJ*GhmpHnV9x;9pNi`7{c#3%e10naD2qB(a))z|f?tt(X@k7B2x`*7kxEZlg2 zde1q;Ir-Y2H}?SfcOacEX4E=FUUMF(ejNP`PWqWhe4r1}eYUG!7o$EW<_51K&u_;F z<5>$j7wy#7RY2ZOEL}8S_0=?#3nC@$43aMpO3Msx6@Sorv9Kw@NOh|9 zoRqF|LChU-i`B^5TBV7IzmWRWsy{4OEvc@0t=SY8F>d?_VKk>Jd0dS+`mQ(ih6GSf zaKZu2s(5dt{@S4Xv3(Y^WD zrS_V7wV&hO<0f)owQlnCm%6;^%_iJ%hK;P;Jpo@Neq%Sib$MxWUnpn$S^Uv-tk|{< z|2tzu`%ZS3CSn!svABw79v>jz<*&u5seb&`Re?Kso-oh};=z?s%*n$V6NdYM{rJ7O zBykva?A3`UW^WS5oGqYVd(@tdPX$)p4}*sFfqTC?%ZsHwB#lu%9rqE`|9ctiW;uVa zW9wOOAn@vSwj}Wygp59dzjx^H>0KWS6Y+)ie(EV!>~xTA>oiBVJ_EV+*f1JrZPtCL z37lV?m$O4#h^zB&i12<}_;-iBc*d#(4^=xr-#a_-%2xO~{TyD{ZY?Xd*$~>Ri9E2`P_CiPq;d!ONOUcv*+X z#{7X86VB~pM)QU{yW_9=+u+RIXkIh(<3WvGZjUUN7h)-rwiTXucGiEs(@k$mXuqdBnDO+}Cs& zFMgQLm$Y(|H=pl@&c+wCMMEp#)dK2o`OJa32fQHnLn~f(AWW*y_IVux>D{aG!H3>q zoih{hlc_go^E@Hua3VT9s0-K94SBV#W2K?}5UF?fJkIje;gesFh1cnQxVryzvl@I* z%XOUMAo|vM1kR0rLH~y5;YsOlEG*avS1&Ze7qfEdY)=|XKTG*EDxdA%@)W?~5}TT; z&zp>BjZ{O7avTr7_C@_)bdK4c;rMmN68><@c40)%Xi_d?fRUYiQyd3>XfLv#o#yk6 zlcpi98CX0&C)SVo38T(0k!#=VVAh@`@Gdt3&g4kgG~*(?{!Guk{1f8v+*oMt>J7Kv zRFk)tj|a`BdfL3{E7{v0v$Sbrzc6~c4hbu?G&al)UyI8>kHOGKe$ex~E2SL5_EMzslp%6e)-JdEV_X=}wk3u6^m&Gavc z(Cy}UR5^bqqX9Nzb>!z2=F)Gx0r$`g!C@Wy^OCqJsQzw+<~#c9=Aza*4a5977CALU ziZ$%g`9Qx9fbA3O%V8DUpwGaL(rS>a7_iY5_O8-m{^QNC({sJHBp;;8mz#DYp+$K% z*1E1I`vfnLCI9L`lZ>7`tYs3+8N42gH#C+XE=^@~QVzqMfI@h6aJEf#Rq^yB(7733mSW$FI%1a>`S9e~|osiwAIJN;J|ucx>m*;=r?} za))*ZA7FJAR1(>vR`NRPL1`pagX(txi^3hW7OFD;(;0qe1*l6V9zZ(2?3_aA2oqX?H;_SPl!5kCS?;DV^eQpJ$&r}zgn&7U$yBf%b*2cyr*-|*4jUiMm_C9WoF)%@Y` z+@^5LQ6uR-kajYGz56$aFCAdR*VJ5rvEPp(@p(?_wHU3!MEhHZP;m>b34Uo@q%E1& z5(xic$*e|lr{^zhwyy*9EwqzkZ*XMQ^`+tih3SvC-)F^^ewwD;GO)&&k&-Y0yvv)T z`@|Pw(f)Z=@w`UQmbY-I#sAQL$QF7n=v|P9U^%oGkJ;Efj4%`~F6At9{QW$}xV@NMN5U;K>&9bG>1?e5M9WoS>x|(pszXPjf zY2S1qk8>>sJ8bmBF+oj`)&&V0HBsH}!H9A6EV5A@dF79;+}!rQm@w57{(4mimuN$7 z(EF^od_R%tIJQF4BjQuUO5(~iruf6=^fvgfx~mI6jKrJ66$Vj$3`H*k3;ylJQ_>8l zLB+qmQ!h|lKx<3;W-JsQefy$Mr+hfkA`st>HUrW`82+XhNsEBN5a=3%=ov*1hPq4Gah+WUP-k-|ukK*#KizirFuvz#zb zIy?x4Yol%>VH>kDu8I8{_JtF^&EVy$8MU+x?FJ_;jAQl0^21=@JYRE?11f9N%$#OZRm)p9XpEjgx{#NB56n*zIHA+ z_NgsNH;9sHiNvFvEo|~t;RLVzYK;1xJHfJLHKB6#$lEYMd{GtC!l1J__|PJxv4PTw z@0zt%V_|8f-c>wGa{vO)WUG?Fmb$yaA*sdDjsywe_!(x24|^hl9$MQw`sN zso@_%^F+7V$vBUE%FJIT@HCU&HT|;& z@aPU!+j^S~FWrSrGETF1%~HhWb)n+^sV7K$0`zzCyF(NCd&g!NT1yWvo_s0@uO#IP zh(4e2$mPy*_(^)q(%+u%7@G~P4)AU&w}*}>v2~Mpv-%SfCWFG+&|Z5L zPEn0d6Wt;e|DR?`$929;@9JKQ;iR3gU-%6mEdXEcHN-jn>HCkkDI)Zdz8GZFK-S10 zk4JMZaEpi7n2kW18rG(|vNdxKYKZ5NVuaVy{b7Pkg>D^IwhsvZ^2t{)Kz<5$KNXe+GRkPi1f9iVpi z*O21>6BSRXu@a}kZ_6~a>?!aOG#5c!$_GH8;wU`W`V@2>F$deccSGWIShjR#mA1_|{0#F? z#ba6{WQ0+|xluzG(iEak@LQF`;P`5iX5RKL__Eboq4@IFhZlls1x(Bnk+dnCds`^@ zd|j#X^g_i+aG89Pkq_m>FI8S)!jH}TXj(3khp6J%*d2}Jq3Kzua9?TqQ_Gu(j(U%k ze<59^FNtrF=75xMn7PyxNXsgoMCF&3(|7w_wrR0lcogh@QjZ^tc7hGNJ9AI&tGqJo zymy$i?h;Viinxcb>N$lI{)u5u0-T+r&+id*Eg+s}#!KL`qk$)h8be_`u0QfU8}TGDf03@APS%IO9@+k6E4^R}Q@$}3QHOd6vq zW{Oj$omj|2pO?eb52wh_1qk9SB+iD_(}3UD^Ib$coDc(`CX(M{ykIx$x#K7jCo$3+ zjPx`tb8?i*w|yM+ANkQnv{u!WM}(j{!BE%j8jvR?J?N=4HIUB-(tO(7kox@Npc2xX z8Km{I2$P2(<+#wfRFC%1Wk?*YFpjkBF#NehV&sZ;N)Paz_Z;aur^V*LTAb#NEgXWG z=1DSp@7)trEyeXYiRt^zcvk0J&CE%*Q0L5jXjp8DPnLKA;e_(w7#M7hd9nSZ!mA(U zmmm-5d66b9m}N^Zp#BGqc^LHU+z83ji@pC%!n-*?HQ?Y5q$#yE+n$8GW`{NRdynG8 z=j8kTVD@d|YS#@oxvvr5bNzLW!W&b+z$z`P{Fz!qr59){gdq;3S&Fs93yk8!CgnH8 z827sHrntF^xynr}^Xkk`kFiIEqlAs@SLOpC4-0fJ!CZSNKj~7Ek8J6H13+_N)DsaD zBk0n`R;s%79^%5DnRg>!`9Y8dLDDf)cSDE^Z!^C?rCRC-QEV%LV#G*OYe*NsXR}wL zOVJAIcbE$05pB*@vhyiXhF z*PJIdY%di@`^NOdO*$Rn?A!^;v#34@Q~rp?PJBw|e3;D@<3=49vmQ4B)z2C7)}C*q zcXqAb9KqDc{^TP&lNMPG{9(+GiC-{*@WrzqS$F@oZBfcAw-5UQScojzHWV?Wz% zLE?6>*4pzHfpMbN%29B#peHB&iOLU=c3}yH+nLp%d~v&|CiL0r3%VtbAZgb@LHZXj zecPiY%$HhgCynB@4c(e@s$(S20ybNiT>URWjCt1y9s6w4JfCu_%DFGpj?gNvPUYRDl*b%>=4KP&9fP#;38t}}Z^5G*~C1wRhg z<>SJ(Nb;j-c&?iy-9@^5s8CvTPk}CZgG_whl%A0<35LVQ??qcW>y~4Q3M-C>_ZL`J@mh6=>`} z?Gw-LKhr4vLRv@M+a0TLfK?dBu+(c-eCey2kZjdRae#*SiV+8r7rX_vI_+UK7cYh} zLkA#fd_i7KUNs!fiMNH)qc>mw(~y3ziV^j#U~XG$ta&9CpU1T3qkJ>b(s8y>xI}tY z)W6zVk~RX$34Zy0%PReBlDQqMsn??V56Z8&cWog^!->uF(tv))>c2NYrNhd+GH~6Z zKw2+4`{7%ERJ}Cnw~~kWmy>L@QxFdEVTZ|gFU9$#qu}GAEf9Xl5;~R9-bm|%K;hf=X+PM1 z)$@V$KJ#7ojCxajadk6Y^155$dDB>|DC#Fjvr;T$fOrZ>56joX(vbXARlK*cOx1g1 z4R3p>J{%|?xuQWAE%|vdDJ~mH8-UX8#A_PrAt-HzUnd99p2}_MS@}&G!ZoKt@x4&tP{IPrysG!Aj}Ev0$1qy?aNkD1g1cnT}!0*cv1M&1Ml>&FV} zB?9?6rKgdwlwvg-$TQ%uk#>ywt*qPEZoK;^`d)KM8?EYR6K4tX9WDniCPT|r^*Q-- zyfL8%Q}s#S9En#H_pmWRd-2k^6X3eJDJcI&y#Z;|-%;_9>H&ct?iH5)X4pEpood zl9f-O{uvq%2`3-yM;KUN+uAuD*M_*mAv;U#$+~Ltiy6km&%q5Z_oG_}KmIJODc`;D zD^A~k4RZRAlXJZq%GhJQ@#^HJnx(!q`Q`m|w$PiCqVD@fP~qN^x2&8c(nj0y)Ab_I zHYWw2o;$?)Y^pBL%Jy=@KnJMt;DU($U?JDjzS9S8e!|14!>jH?KZCPNGPJLfv%xLO z1g93vmiuow@c2HXyk3X5mF2^NS*@qQFU=hy|2;bjr`OKJlP;ZQl5S_{H@Yi-Q8EOE_h}~2 zB_$wT6U(z6u<$pwnA__!*d4USF%C{yg9iDb*`j@-pJf`iX94_lY7m|ei^gzg8(B1P67Jg64bFeD;oEu-=MQV> z$Qs+PK$~1^X!x>&eERS&8oZ8!z2^tZ_$ODfczY{&vmp*vq-f-Q_k}#?$yT|3S|{mq z^#=>;x|%gJ9mC%TU0{uxzr%rljp#Gw%(P_yhU@;sUUrVy`|LZs?Vr!P{v9g!*avBv z<=2yw+b1B|zxKmEwKTH;RZ9#)!l^8wJ zMpk?Ijh>gRBk$2WgX_*|vFyzq4Xd=~)e7U_Sz4@%>Ag}>)7VJ%?|B%H3>d&ZHuxv5 zKsH;QNxb~fT^qh{Ic*dXpyDPAf9{aOCa;2ojeT=MdKkc}PRk)>k}*8+tk07JY-k)k z;qJt?^2V5K?Uld`93OK}un84FbB1Wc5AgRe$R_12*&v<&(3IW-q#TtF7l&d{lLP4Z zZWG%Xl?T6?`|_fJ4}jt=pQq1-!nEh&$H{WsMCbahe0o?sA2LnF2Wb5f+Jx~kAAg|u zLTSbw&DwkCaLieIT)kih*U4@KYu@YgUctY?w%KpgpV38*aEg-dVHIFxJDE?ReVFLm zd**3fPY%ob4IB5ohkHjqLtt4NZ0yp6KlM0@`M#%E8@mdTo3IZ18ERSN(t5IPU?@KI zXqyvOtvx-BqRS&!+!u#`_(1x31AO2a!H#b;lS5B+l&V%k%@|y*+ngQQbsj0Mn6=Cf zRxfx4lso)<$`a81Jc8%NTvoXZ{!@A*ce;nSVz%(gQ98n7#5U1(yg9%3Jptnq>p^YH zcKq?de3-LiKQ#5XhTltdq>&89Ujd`punC7Tuu~hU@;{=)0EOf7hUMoU7uTC***Ka;0*IHR@WhkxkGeS;Whz4u^h(+qg_$C6hU z(?ns~OsLk_8Y!1Vw^weWMbSy{8h;hOo3(~sE9~)LWo_9i-3D6 zP_Lm2PHq1ln#87H+cWbZ6Fjvpn2H_rx=N>Z-g3!c3*PRrFHoI>gINK3E~+i5rd*cS z&VwbdP348BQ{|M%FDhO@m<8AAJ!6%(n~b(%dG%0Qw;Awu(?WUN^*PS&Z_Cg88_t(% zZiB)+x))nNb0b`?(GL1ts^uU`CUWUf(LsW%kkcRobX4)#J|Kw%@?7?c0HNjEnndYuB)FV&b!89VgLG2 zFK;vdVBZt>cWuvamk)pkWix4yqjqqg&aq|_mN6sShgdU7NA{q7z&@IGVaHo<7r&Qo z#@B8gaCepi()!Tv`f#h+IfR|dq3rq=&?FwzK52DaY}g&IFrOc~`w}UK<)1SbsKz5S ziE{*A_4tfwOMbF3s}E}__u(-;a!c5SD!!EldSX#K^2)|Xu|GZQe_*iSONW@F%SM<9 z!Xwebu$ay-8(t>o>}bXbUtq+J*>EZIGOR9ogM@_`NX|1JHg4f-Q?kZUQ%%h z)t#1d0u6m*SnbS~3WM0;dUt>@09AYxf0T6z7IbZJ3|OOGGI%%I*RL*4Os^)_)$hj5 zdl~Vjw!8Sz7rn*($XKeKfpRwKvB}?mv$vbdp$5H=Kpcw|$G(Y_GvVr58igC}!j16V zhe9Ac(zFbntUdOL-f1Zfgu2jxCk;P_Q!h1^_n$^Ht$09s?Q= zqgJWCTMl}G%j@P1oN)n6Zej;V2h78qjUe-2)AcOl723KSggdivv+FK z8*CA6lOx&BJ&ok&1;If1A^)2{Oq%&sy7-mW#3v!yIhD=siW>=?!L#I%n0CSxQ&vxP z>2znG$mz44>aIQane8CWdj$Yt9qi6Z2F+ONff*T~U4<2&{3RVa+wY;eod7P_PDazS zarAk-R_MiYySEbg-CTv@+KdzD@j=0MY=6}aY_Sq@?;en6ejA{@hfx(r6X$WtE95i1 zVg49*IQFTDh#pC2EAAhrQP*7CDh~!83xT7ZPeD|&4`dAO!NZ*oVCco}yl%x4Hch8C zKjqLA2b|vpca~=Z<)5TA0KzKDi6^w?R)VlW$41W{fO>yH$i$KDyB#myc4vxLBjMvHe zh<0P?-DIzZs4#%W1fetO9E}$t=>Fxqw*RMgQoGTM?mr)i8~Edj9>T}`h?cY}ueO%^Wg4!LU(Dksn z%fdv7XciC*4E!sd$UErsr=}J+}KXUSSTD znorZJyjOWf>x`^&pTcF)%=8xVR2O#Pk(sMCh(53vd^O?X z$-NAyGh!d3wZKhFZzIhIhzGShJ1}^2xhc^2<*d9+rAdj4i)$*Cu!N$EmoS1c4#Y9-PyC|lQD8ZJkOyqz1?sA#!V0^Qo zIsPi~hkeoANV8v8_t(&TtFSroWH%t4$0!z}<-NL?r2m&K)r`de_ijkshI?9%5`@8^ z^62xHCrH`{yITwcrEy2l9=`R?T@fjDwo|_O!KU029C0NCD0e|=35u~KPQm7zbFldH zF-G}ZrHQY4tl&OZ*J8&Ht*J(Bmirq3BU2gGpG%w8)Ao}Av0(+hWdpyqlMxPrT=38<_e`lpZ$p@-FTRly%~qrwL@T5U3z}f za&J`*bUFVJc3B;Rn;Tw%`~4PZbF>H_&h>|X12*x8y) z_q-$6cKj~*Q|BzZ=s8a6tvyW`W}_`m_C&&WL0S*bjcdhMXEu~KFIe(RmLrg~3sRnd zSF$@an`opZEhIYy9fT6MZ-O{U;UEuQJO@bYbHaxzZSA>e8%z#!#CC2W;>vp_RedY` zRodEWwhIp4JBZhw+gdjL@R8!|!Is88BK`=%0d40){rVSyYEaEfl3ozRJ#1Tuk0hN- zxU?0C>qXYe52)s%_?&nc*8g#o<*|nR!umSW-XlPqtL=e==cHr%G8H$7m<~oZkyKx2 z!Mj}+jGOTW%&sTmfCdeD-==+F`rGPU>4B%~zTv}6I)l820k7t>RCfKPhh3(asBt0b zMOafY4AdNHyj6H(Ymmga(*zuHlcVnE8)`hvu(%Odxg@+UYbow3cHE;zI1mo8Q6Hu9 zSBg978dKm_Y$2<&KL$E)TLeQ|XcZ69XX}yI+l>{!S}A_OIyDZeSOW1nTT^k3_CxL< z+Woi-7urT>-NTj%$_W)OB;SK|Mp(n(xBVq)2IY+zX-V7?Ye9axKGJ-UG!yE~w~~)5 zPUU>sOwU@R?m^FzDcU*D8%v5640bsO-yUWQGryG(u(K0TUXVW9$)=qM!|xl{o2N(c;UiVB__MDvzr4lgcI1O1Sy41{WlpgPKX3 z)p}#;V1HKB(GZ-sNL)N49+Ymp6+1-e%#T6xMO8Jg8@yTZrAzRraMs9Z7g8Sq_q(j4 zIX|NQlr7~#HzcnnXbo`3k4rdF_JX!&UW4DA675^7GE_Q{YD=g*Qt=%*$B(f12@ISb zCOU6>2W@IK=ES9ZW0$=^8k=h6gO)U(ywG>9s696YRu$}r2bRI$-g-YI>1L2d&xg^$ znaUF&VI7!wc9FzIVzXNwBkYzlZ=b;edLEE;J&T=wUh`n{1LFOz&?vV|q(7SIOdO?t zF0?ytLfhm9iW~TKaY~rpu7%xC_Gk8OZiy*vZQ;~eGiDkx9w?VI*2^1km9M1Hm5MqDuw{|5YXDg0@sp}0U%M}P94enRDl;(F4iyut31OnFJXfP${6 z@EXaZ;kaF4+MmN`;cCk!j4%?+6Y6rx3r1^Sx z6Hkd}W*^1z`{6L(=&B&UE9y^ktIC_m zm&sFYgC7=^>N>FSL05k2!CUf7k@I=_@T0LD1}5Y(q&#B(`Q zFYN*9IcQW5fba>dF82i0M^XB7%djS-|4s?wFgV`24)5*o1*tD3Rqug%We~KWiOcXl z``yacxQd1IU9*`nUKObsUF%JBYE<`{LruUx4~+N-MF@nOQ(R2_TOonrIFJ z`8J{SE%`u<;bx$;RY*lmd1`eu)6s9h+`p`mgaOz%>z^PDP_^&U@k5G+{1$ng5n9r3 zV0t?k>4;;t$KIHHqaG60%5QDDVD`6qoUlw1FK9>`C~ZrcGlu#e<=E-dN}>7pVt!KMb>}-yNQ5gSFmoR+*PjDIlZ_TanLfiT( zcabzSagGsm;>3&Go=*-tj3+J)$J$N4V=kR(UgTASPbe;cl{?qskgaE6pXIq@=*c5w9q(flji4}`daGAXg)UBgyj8lNPAK5 z^^&0cVJ(_;M8ZPV*CAh1O_EnarH9NGS*t!0kOv^W?uXwm9@SDWM02%%IeDCwq}Muf z@>;|{@9@a)!K$~_?Lk2uY+7o?<0Gy!s%upI{qDdB@%W%C)o?pzkg=XLmI*4|sQbt*k;eCC2i)u`(4s2cjd@t5i^ z3&I5?U(TlQ)u;YX65E=838+tu_UC+odYM)FgnT)WMu3{Ni&>UQEOw=H+tLhMaq34C z#x#PP7ouqsa}(rdwD0P@pQ@Ke8d+$_vuHXVNL4z$s_&-T-51WMd;s$MtmZgh)o)Th zgfFx45o!z>C;CEm-B(J#v;Xv|U!eO_={XnWrPUmjHY#%-jY{YH@%yST%m@P@uBIjV z>%)pS&@|*O82QhFxMgpZSA;#SZ(<`Ob58vUHBU9KoQURTs%)IfU=H?^P$fmvTsXVZzCVe0jr?fJ^SCzO9tYc6U%1x=6Py&!vnCCqnQqWSDu0FJ$F__7m;=(OQD=$&~C zFAN>Aa=te+Ic|=zW_GZky&XT8;*VqMb9v=uPxjgN2P<#p!8_Sb(OjC{6KbtBl7rUY zg2k^hFtTnO8`rHmx4X6(R?u;V@Ydq9u&Q4m z1X$_dBo|${=+PW{=w;%Yk$X{d#ZDS|_{%&rMTe&MRV<~Yy%v2}i)@PzDJUIMab)wR zm_6{0=0cD9^4+MWB5-6V-yX7Gqw&tr)H?SRHw+AB7s`6@gFT9wbP1LH*OuatZ5muZ zq>{$8Squzz;djlJYd*c+3`_rQVqSwoNt}~|F zg;#rW5Y4NZ$b%_{fFGO6jI1wsC!F2`Pag#Cvug42Et|=U`RlRfcSBi&zGGhc?;lG{ zTZUpuEKBbcf&*T!WPbm7fXXWy%keT~lqs#nTI_Am4Z2@VgeHEgdEB&}Sh}gDTu@4H zY?`;^Nij*dp?ZvX+Nc(&n9}^PCp~{dpO>3gYM^L+JgcnhBrnxk1m7ndfZpa7ysYsm zusdbVUwrE?pUx=5cE8%=k9WG#KcFcun0toKO1X)&j?!RH2DS_Rg~#Su@#68s*OpVU zxZYC8sP+;v7VH6sHEF6&c+8>qY);T4h|%rM1C9;kg{}>;`YC#M`^9dU$@OF}zf6s# z*=-!BnU7RIFsi(c?ANOw-;mr-y7_m5l9L&DIkpHbufE8p{aUeMyfeS!z6`#$XIiJh@$_2U=K5UN;nkD(@fd||Yti1$ecQ`37K5ctp%>T`R%3Rv3XtZ+ zn>==xetTbG4SMdX;?ET}xzQ=`$a3JfCQXO6AF_~YTY4Wrq3$4yB2FH7{fp6t@G%NfD6u3&j}b``!{PCE|e^X9VET^+#ULu*L+ z`HYS8nV^|dmX2QF2!HJ(SyVTe(!{7*M>mk z#9z27z7H>%Fs*9di~BAWb!@U=x63u*y4ylF8(dAE&#jj^b=N8EoqJ9MSZ$VvjQjHA zf6H)+R+ke7py9fukd?CZGcauGf#0H z1=ZUA(VjP`A+Me2gh3q!vD&ZgwN&d!p4sfIH}%b3;=$w`3SZc=o{d3x*=qHkPlu%qS0} zQ%D4Ky!TGKZ;BaA-TFX$ZTVCE7phOU1%<{k#w$Bqgs;2j!t*Vf$EV@{S*#A?IPfJVg&!VU>?o;7I-}!Bj+~hjsNt9(LL+S z(AqZW60rja8~9w0D6DSLNql+h&5G-N#DcsAEWt5ACKq=VMeA)bB{dL#M;(Vlv#6d= z8ehU=7vVITVhg-y`*so{i9~w{#!d#IULi#R(=U{lBxSz;m30bm6`X1Q)vx0 zWoRLea-IitADD8;hVL8FS>>zd+PKH6hMgxS?H7a#f^tXpx$Ynr&h0D{jrzc~x*zb{ zk_D`{K^AnfeS~8troxD>bEU$YkF`^IN&E=uJ}VfEYK?^LrC}PgE3;^A`pSt(&3JZZ zZT7_X5O%uILVD}(WyCKqDjcM3Tgeh&9PsIp(Q=gYYYa&*K$^QCe#0fVM`%y!zp47njGl`a<+$+s?;9REz8p<| z2Lj<4Ol38A_Z7u#PWV=pf6f!OVph&q_HnL`q+DjCCA745oN`|fcSx20ZPpzV z#xD~UXW@dECqU)#t*n#GuJHtz@$Hv*lyXxLwhH1DR)3?hyf@_;s5%I>_Q4@0w|2>MSJZkvv5F;MJm(eCN>`yLCa`C}7%_gDZ2&xSj-DNXz z|2XF-h0*wM+DxjS`nzo1CGKBE=XzTt=FYmoQ?zsJ+Q2ir8fak7T^%m`JU{k(FD_VLl+sjRZrdVEG~AXGcTI8=1*tLLc(PDP}-F1R{9~~ zH5M;)1(jEn3#?*)U0InqL2lHv=7#G&YX2wcy5nm8-gqfV11%a7g^bdW=$_}0os~V4 zk?l)n_NtUpT8c7KAtAEWJxG- zq?)JwpQX|>K>dkJHM_7@Wi)7J4Ui6Ev)J2p2;W)1nN(}#)HhU~?ywg3zSscS<5Kac z#W_^3OB`y>Z_g}&(3U+I%?oL_XhFXR_rh~n%eX~a(gje^wl>eQ&StdUGPS2CTrsVI zT^<|Jx_(uf3~4=O|F_>TDmq=J?sy9cC2lbL^c9|o>C9u(yO&w?^GQ|t~&;Y=!RE?slBi7Lc(UkQctC+SwV77KL3mn)Lm{u+Tpb3 zzcpig8n>ss2HLS-miw`${b6`}U^qPY)w1u?!m-t0dbYK56tlU11{FVyXqthZ-JGxG|9K>9n<#=Q~;+o%(Aj5?6o>3&p}Y4Uu{dkI4n(2ZrL-Z;ya98|jU; zP`2%ALHNc=V6C%8=6RU#8I4Y|4Pk|-G}zLIHw0-bT(W)wlBNaXO?mjuCRq7&7>?}I z2T7Mo;#v_gk74a)vq?ub6e3aT0Q$NcRs@Sz0A-{|sj8+>80Kb=Q??WG9s8;7+UwZ#UrUBq8!OPK!W zFVI{np9H-dY*N~n=3^{U9k77>FqZ3PEad})DJLSabm(aG7#e{eCWmsGA5>b1Je4pT z-VNK%=nty?NbiZnHzVbZgecD0$tuiW_3Rd`yKN!A{JWr8LeJv1>$x4ryKIGDdY4F> zRxppGC8*Y3|H-zhHLf||kW&qc+o8@}&25R%G^GWAw5Vo5?>VS2GR(jppY`7aK|vmP zc$|eR<`qH^ghEut3MQiZAFofUrk0%X<7=o(D#` z^^(*JKq8L5&v6y0!g9C|^uH2#sAi)bLnNn&Pkc+$a%EN`lxSv$INvD}ALQeq}$K{bJ1v8{#_0s7-s&}volTS+syjUt^n6Ba!xEi+#zMpLmeaq=P8tYqdG<(S73vjK5*#Qy$z!s2&a&l_y#Z3@5HHNAiL~C-s0PyJiVo zs1*$NG)Iwim#DB=qk10S4o+vyCzXQoYNY+d?AzB^@$NjfCi4~Q?Y823CfQ2G$!ea@ zyL<9zBL_?#xtKhm4a`n_jN~hTGz-3eu7fuc3>EIPJ2_=Y844r4Q9$|x6@N9Fy9fJB zGUZD5(fbv&opD8cQO{7GjDq(|da2iQ(!5Ch1-Wi9gnyPmS&Hf{JkmI~D(@gX*M4_0 zXM}6SJ8w{b!Fw7*HBcCy9<&VCC1xp}RNetz?iBol(>^}Mq{ByJk>P(pIg$4CCV#@U zCggX$)cQ%%dZ_yD!RKyDJEG1bCr0B2=~IWRwN!b7bN_1aF(U!JjubLqULy8x*$KqC zO3!23S|^y- za>5wA+bazgg>F#&O`gq&r}%jQ{Y-i!{btIO(;OEZ*Vf@3Lk1CkISR^JfUr`MXJ*9f zp!~v?FJqu(j5+=Nkk%{@(n}9A%1o-XeV-w_kuU^QA1~M6q@lbG)(l(72p5!|L#3hU zJz(Oo>VmYS^ln>(V_P_~ke;)V^m3KHQP~IOX3Bd&Z1G-nes+P;oYMXF$1&!`ZK1cJ zH4OXsgZ3e@z=Gs+^i0MvvHq-wydAoOQMSbQ#&wt2X9TC*MAK}68*F0z;dP7)`PU07 z2P0kM#Z|sl+<&B?{6Oh!t~|cV)kgOvk3~5HT{G#LOtHLQ2qXQdRck^xrfnIMgnQ`x zGScyEJUy#&Vd2&)A3|QY>RO4fsE1R?(_VsKfsVXx*cX)-q4iNaMtrFCdGH>#JzlT0 zxx!yoE5n;xPHPUbSt}&XjDxqPtF^+PYxf~#e_Ux2@_yc_0hYwsCsdvUAL1maFc zI!`FSMHvqe=gQs<>ch%an^EC*=d?6I{)5H63c!iu4}i08LrL`xw6@qR+Kioy)Zs zVat86>mtI`;xUsu6 zcpd3jmDf@Js`{A@P7JEbq$q<{W2Ic$gffOqLH>#pe`=MtBQF7~o(+d~x`$ZH^4CD` zqdXO!cR!$2x{qo_s=P$bR^vFj1_nbeFS~pDg?Bj#>+5P3vCw|bS;5z)heur6{ zGzRy}Cz)%XS1>q4hYuZWi|glfWRV*SSaq9Mc%w^wjuXzJyV(x-+BkwcPt$_i!=p&o z0ZGL=a(co#wy0@SuwL1L4_Gk>)%%r=ug34pYb1@{d|I6Ksb=f#z4H#ox{B_Oj879<0UEVx&1msw-{Qi$Pp_^_N)E-BC9GS3`EFnZmh^p`Gk&wNH(&OxxAysz4`Ae$ zg#PXC3L3k54IY~}08>uZm2do;^StBTWZdQnxLxB5?{b%kV%z4j^*{%{Z81G3`R+9w z+BXGa$C&W#XWl_&xAn5pDhDcl?Z7Gad76{APGRDMK=xu;H8@B2;GF$+Q=EEr36A|( zrya1O43@j$0cJhbm6I4sh|lmR8k>6z^6F-*Q#7Dn^&592mh!0z4;7}YDM z*VFeBo#5b{I($~Yp6JnO9Td%<%nfVRzsNJY)@LmTnexw{1u2G5&J(8__|d&+krbC0AW&CaIUW z2thdP$oS)yp-}yTJ@3qH<71MciEld={$7TIvD6=rA>8wL*F~3sddJoPtE{ci&|HV|JNCM)b9yqI(@aV zvKLT|X-7G?ytGo%d%MH z#SMk7x36e27nuOfDZYFAkVP%NhCdFT6(Ri};`2ZQp1mm@&jM5HiRF4)@#Iwl*{tO- zJaT0McK=lh8#+&his&2K(CcRTvxpxHq+g(m*seJN)Zfr>Y8)O2*Mrb@U8R?~ z)UMUJttr&RiwhrC!tmn9aCUDj+}V-9=o%%}0(_h);Lf~EZLWKm=yTbEcQR_pQyzXs zV+RLF4$6m)FIzF1Ps0BPBBahT)@>=h>D6DbBx)t@z2qaqhtqk5(=*?xb1rgtlu+~@|F{>vA;dykgz!x$ENr)dUV?Z6!({}Xqc zTtMnEIdcrhNxu)^)*Z8{4%f-Av-(Rb-HL*&sW!6qg8}S5?S+2+k|ADL)*1|OmvnIn zmapwK{N518YCpKjMosczNns7RUAt(}X0DxlVpoel)0V&iGfz43&Ji5GBLX}NTzsA{i4O>qo5=9;-Aa>#mBs>+MBL(oest zABLY#^I1VkmZ07T!%g(eSy*4HKg4fcb$MXUMo{=c?*j`5%mb=*BwW#si$1Sv5o;~| z!irxOz=Qcmfp{4`;)C#r`59ILM)KX;F+6O8lZJYQ&fn-Or}{UL*42JtzYGlqm(CEH z3GLxpwZ@Wa0lK{E0#imMO0O#;fN+exozX=0sDBRaYHT2GGlc*C`0<~!pR>#DC*c~0 z7GmjwL?Dcn&U-d#JEab1efRBwmU}(19yuHZ)PqQC3EihV@JsX@z~89a zd`kQZB;LSo#)U}NqM;g8oWc~IMEv-n(LKM4U2$9ugSsTDx|QM94{-kgOU(cDh%nn; z6MyWHSifx_49dKWf!^l|X6!d-3R^?;lLf6iOaJas#U;cY@NYqVPC81hI~JaiSg#}# zS~k?jtu~ZN4RME=S8Jh{Imf!!lkt5*rf}`3gJ;qz8R-%B=xmloz3&=_UC?*Nc}=?` zYmw@XP1zl&){%RDJOPU)8KS~C;#b&)D^)FtFQ2VAVY}!zdKf%-X(+uv-N0^X?}2cW z=3xtrwWIyNrX_1kMlQgoX+21P(6hqS+tCdy!FW$8+wjc}oX(7s_bLCdrSl(5YUje| zqNXgX`B9*HluDo7jy(R%S)w7m6r*&024r`6Arz!mDBB$89@%@dqvrl8rpsJZs zlL~O9*di2meNH?|_^rk2e@uAb$scMwSo5!=Pqp%ojPOVf&3dLZq)@yy<8dgcetdJ` zDb{w zS3otz1`eVuzOXukE_9Z&>733Le)+I#^C8XK8QbyGRU4pwz#*Rn&|EUYX(TNS{S&%l zAKh>u+@ksMfXcPwfa+Qho^jH5tT-Ts^zcL+S^W-7Yt;@XaVt*OBX(bO!XXn(vB88H zT9LgQ)OCDx8;*$$eYM2Nj5M0MUJYq)&Dd8K@VNVZ*8JZpoSC_W`Y2OtI?G7bEN!hk z1ZLmUk^Axn;>zElLiM@oA<}55Jc9$BXuB@l3uYDB;eYL0%F?_nsP(>tDek8F0llfm z;c-h{N&UkWcVBTdlGM{kT8GmgBtL?*p5So051fz7hB_TR&@y}{#Lr5Bn7^)kbHqc; zc^3`DSM0`^g^cew0wY&BV56AT%=Fj|)Lp$D55~WR4rdOqHgh*({`l5(Z<&GEI-+R- z`2_aDWItHVIpS0HX&-U!6Z}wGshPx6v?qI|g8%aN_{?QJW;JgM(;Jk)-SwMsva2H| z)hR`#Q^)(ggtZ6PAk718v2LX;*V6#aFVlXfdm~P+!%vi7U)E&v#aG3oWHdfPZ;YcUhjT^^oMZ% zBrcj1rF5_IPC!4`C@rIJsw1lPC??D^h2_I`gR4#hHBSD%?{TDBhfU+3LRvCc^RDI$ zeCOXorS-mMolshUr%%z73fsu5pz(bKdQWIlS`C&Yzf*XsRr>asOFinp=j?Y=4z9m%pWt($S<7 zkZKCvEVssLi#)aT-on&(BN8`bmub;J>&KKnBkU5|E~Qmi^QXceRR7a!IO#IJqtsN@ zEvFj6joVKF^(Fa`+_2E{k#>99RlPaG0_pKbm&znoLs)`DE?6RN`8l}>vWaT`ja24!Rrm|#|h_nx86<)e}Fg}ZXXSY%CZth8lGFPJT6*0 z?n8QwmTHi`ukOVA=<3nF9X()WkBg)Wefg_Vn{dD<+OJ2(YX}dCgA;_Rv1y~7fwZdl zl(~!5Ggt|?o($yizS*Gk8R;vbYh6ZXU=J3pKD0pb`Mdb&c1z4k%oTC9k1C!~cq#&( z-X|YuBUcB!g}U3C$c5fENLVC(9P9yf-AMJo=6z|X{H2DtL)9J-221K)M!b|CwqYx) z-M=y4o|`C=I-=?oJYWC1wpU&~;)s95c^!GBRWtHuHp&Z9{hnt^Prl?0Ir%502Lx$( zoY=>Z6DLE~i!wp=MjZTEV>~7l$S1+OR-N(Ho1UoZ?ZO9rs;gka%D=?rUzzXxKk2PSJSayvj{VF{s{GE<1!kPqRZtHpoCmMYF4~KuN(+=PCQr(VcQvGU3Y3nS|8x)g z*H#Pt8+;Q_oqve@?N+?mtFNqK^IUC@JCnqrarSa#el3_Y>l6f~wU?t(o5^lVlYlY? zAYO#Evv;HEu_0P>k7F$It{v6y5KtaIewG#S`3l%uXA^jIsw-SBwZadz0vYLM%7IL% zwvyF4kw!UyYQ6|FtNLKc^?sbZEmOYS`db1>pKxwcqS9)g3-QJu$6WsiK3T{~Jq3pp(s?VX`(J0S2Y4un}|7Ixzx?-adc6_j& zo6>Sf_@doEAi0VQ)NAdBS7!>#2q)PJZA~7q-dqgLnuwi@b_3~=f_4#&!RyLx^1Ty4 zc->}h!OyhH8z1_3USSxV?{gBOHM1aK>k_2#Gc32Q>U)*L(C^^3o(I)@Q!epWL)u51 zu=JnS-g+xenM&tH5Esik90Cb)CY4*1+} zFmL={W2MVAEsiyjqn)cq_U{$eQEq`Wc2{8@=K&K`HCCQNMA(2;GzpEqXO~{_-1*knjat>l?&Dm1Jbkw zNDk~!`KtAw+=;MD0^K_aE z`|K+~bIL;ckHPTO?%eG78&KI4)rGL9dx6Lg2*MjqT!RlQ{z2bInV-bft08H8r2(}~&;HajvAu*>uN=~;tBR$%DYsLC_RSc&*5E0GG%f)Qsq=_ zQ`U=mC5u1I+vQ2*h$XnJ<^?ggcL zLHS*(b1|U9C-LUOcu@WAy?wQY@Jsf3Xn?ApDaX`M4uSh07Euj0kSh0}jDwyph{VMe zojLhj=GJHf5SG$fdy)p+=yP@SIH5b-8}8HDmt!g;(RTcBL7EHBY3Hdd7lUhE1=l`P zA#BPaD1PaqxSJ8y6exX8nGx&Kaz~ZUCSHIs1|P`-Syqk5|3pJE(Ii`_c_4o&xMuEXw8yvHsi1s+_$A{~8#n4nM zQ;9%&2F~C4gp^;Q$>!TYc@Fh=JNeXk0oyX65>zI5+U~ekX%VM0Pk=He$bG+s#=M{{ z<-(kLlFe(Qr*ak_>TQwN(U_fDuncQAJgoeimiSAU&Z1}cJMKry!I8KEXx)J34e9ss zW@rd(Zf8mJ{aZ^Ji!A;94$1psi}%Tz64r!|?8c?n%36|YhEpad6^>oc44^#`Tse7Y z@)b#fG8aMFGp1}X#C?QA%dCz=LP2Nj6#NW+-Ma?z%RSV$9}dB5?!iTSe<)lV0lpnh z;iacTq>tNJ+2*?~U&re4yt9$=qI)m&7&ZqF$B%+lcBN3?NGn42b%6Je>igJ#N8yrh z1;@hPW9aeEELVR5PVm2p2h!Sb(>kwk48%gqdaJn0{dlk(ehNSS=>Ux{wvg$k5-{!R zMc~3%ZY>#&yNbGFKOG0&#@d}E;o>N-)Q*;8{2LH zPm|^JKD&fnd=sW!XV1gwOvFs|fu(czVc~RRIVL0=?2nabr0p;qzoEBuIiJL?++GfI z9b3V-kGgWzpe}rzXB&9CusTAG6&UMhz{O214%?rEWBS$vH|U4I9X{gNd$gyr$v@aJ zzJ@%te5f2}!YDUbD1(OO(R1u0Q73mJ-i?lhEo*Gxnfp#{L8ttt5?6(%MPy(Fee2;Q>*+X*VWJ?+Zu1hsd#dR?^}23;5?=1LlNof!F{& zaGGc#7kDjVuBPtjwZuU7jaaQM_!vC>PMfsl7M9KR&bM;bGu`sQUft>sdGZyq>*D*R}e1YVBJGj?kBcuADv!u_0`MeF{)betwsk>08k&d+07<1|m z=D%B?T}lnYkjQ|uZZdCC*U)-~q_K1=ZMl&3i0Nl&PDxDa;CU&$kSHNeFV>!1#u zak98;9bRxLnfTQ~QhjpOH-_i)s5#c zTT3&(a;25(8K5=79!ERD{7-k$u|%IA$UVRf1LFnNxfoUbH`{V>t#F-|36**mQJ8u1 zDYvU>*1WWo@FfiC=TPy^Sa$f+jO$G)h2=;3U|XGzGIc2Rc8oRqmT!dABeG7daB%++ zfaB+0g?_giOO5;l`@7i5N+^M0m$(s*DyJ_bw(TLV>%^CRECLfwr|MBmwKgj>)# z?Aw&|-~&5t;O6m=+J(yp!v%hCy#=aA%oMr$6~d}@Z&YI!W(O2Di#ykKSmLjTNcbVT zmDQzQx-Wu`W=K=-r(jkT2lQTG7#@x0{TlL=6V>?75Y+A}It~3h0r>A;S%tIILyYhe zeKd>zhZ{n46c)F2j9DG0_q9r zcwiMS-`Y?0E3Urz7r))t=R=)z`OrU+w2#)^g2L%B@XoiE+?2JH{8l_J-Oz%sYIP3x zO#6-RQ<_nY1h8vn37q&xES>9)FD1>fZ`Ws1UdsD1hbPVT>KTm5tfNCs-HS6$#(R)g#g$W8j`HYS)#fwy5D9jEMORhCi zJ;Fv7kHV=now$>0eSX8%TYhitAQ%6)T74Fbpfjuk=f7fm@@EPfzi>&~sGZn?0j6Jtm ztf-jIE)JoaRv%Rd>K#GY%@Vs0h6b~*K-b~bc+&NWBJxcxTr;b~|6Kbm`u1Lnw_@f9 z;&_M+){%Wj*Wv%{S5uEpgpm1uivOieA43d2ST2Z<#OuMcwH~y86#WhRw#FijNm4J1 znU@x|vwc@rCU(D$*M0XvtG&lyZfy@vYYb_*WmtbmtAcSR0y@0fja27Cuk$sa-XRR| z;)>sig9wk?z_TTrNgs5B^=EUGE~(NiTT32jSFSgQZVin|e|%;E!Cf%@jgh2z6#XY3 zfxb=C2#>ze9BzVp`kA7#`585C4F47mURgSP%OJ=6@)?)KjW0~A=G$zOB_}MxNAMi0 zy~`K#vnO+f1q!PP`+TV0IdLtoxI@oiWUPa!ootct4`1x&I6|9`{Z8+|8n{B-yPu8i zZl6*(!(;s-!LX-!6<1FpZC;vHAPTS^WOUpPP3!ovj8-|Kv2d02H!Gb|#9CDzp!+z> zP+iZr?WV#(XB<4WjD)1TX}p_hElGMr5)MjQYvK2yK&#uQ1rI$whuzx$7ORi`hRsWV zYQw+jp=(3BhwZ{+cCBX(PB?;21J^O?UATXKCrmkH2#W8h=Ok$@nujb1@axHHo&2gj z$u;19?h(F?t%ln0hCrAm3Dcpk&pP~Xy_c$S)}%)VcEy-)``{K-kgw4Fc zY&u7~cd+JXPzSL1*h?W7pJ&g-i1f%nW$M~ackA}1kX_lk-{K`?F&qEp) z-pz}||8{vmen&moed89Eb0M8k&4`Fz>1tiMt#ce{gC9cSOo@Imqj4dPiPpC_WKO?%QbJ;4` zo!1k;FSnq1*Fn;%K)jC2mkzlff#Z2J2f5fiqzQi9-wQsujKO)<}lXe>tT& zv#}DX?$GR4b!J9qq!8BNAoJHgiqo&?EJ9;nfyCdq{%kX)%e4ocyK4^}zKJuw&V`3j z0jP9@(xfY8EWFBmMw(`*922Djgtg+GZ=C9JB+buhjT8saxe?RlC_3{j!h8qbn(&h~ znc;(^^CY!@7TVnR=oGBkBY-dS2q*U>mE^``0$D|iD)w? z3>rV$p@}l=CwRlPp!HKV1^&lEg~Ha%oyf@3@Xrrl zt98T=Blp9s4q-rCsd$sEsCdh4buFdW5EHnye>(QuyX@zA7;th9lD^Qo>MwqjZN%JsIAtOKWN=iD(pJkc`Ezhr1fd@)Smxn=%_pg zm|W~lePGAU`Bv>yO9M&#!M)QOj%bv2sSnJV4jGWmi_aJcg; z;<*I)S-21VXzxM7SJa_BqT^OO$#Fj=oh@lD)axqs{Gr#i3PE#Dn6MV+>_`JuU!*04 z3+=CrTRL-9-=sO1;mlZY9p0D`50D4$!>j2Y5xSZ^IB?Vhrs`f{71ak4r_ec?LF~di zS57_wsi)AZb8YTkcZjB=!6l?#S6Z9AQ%CrC@(fd2K9?&Sp)p{ zozCa^sLO}DI9K`D@6+l++EstP&GkE@>jC?x#~En^ygtwsrn=cxUz65YQE0EfY3sSp84EI;UnbyNZ#?G|4rt&_y6M|@zzPR3KV5DBVRNjHfR4-Wuo zWd6G5Ur;*vv%xdgpy@r4eyR?qUgsZr975tBSnO#bpH0bNOINu{heN?gIK;?PD=#Yu zGcn`qKBW3Y(u^24zbVw~J42*ZoY#I@PzILWdT_dCp!`JK?Pn!P=OMiZe->{m0)kI6 z(#m-A&m$!Lh52^T5I24&vpej7x%Xc&s#)6CsteHP3GzNbH3f$b4`Y_wHBgQAe^m8D zb%{4>6tHJQw&Ht{rBTUYG`?eF?{P!J^(y+|k~YtBfP07=iK^@;m;t=c%=v(Q%44 z=fOsxzNqqGs}{9aS_#Qx3#FZ@7LYs+YG*n@(XUOgu383?CXj?@O!ayHOgbB)YXT~b ztMVkuW`yD`%I$zW161}cV#@nafAY3Xg~-Wn3=yO36t2pUeaE#*dyw}h{~70_*4ey8 zHti3VM;bJV`Y~HjAEFw=n%`zp`DfB5)Mw{~%kp%k2{>V@@)0a(*-lj6h2}}l(XPXb z?&lf#7buI}29G~{V#M3HC{dq0!uF~xGa#Tl>Eym>mS+Z~OOBIIkU-u+P(9Ju^)SDc z0V;g1TX+w6PA#nO?uFz#eduTp^o`Nt{q1>3^Qm+!`MhwT{899Kn~CIuLG|&?KVw1Z z!IoT?I~;l^luxH@57c!p-%6S?ydkQrsEp2O$-O^Ske-CMv0;Mv8I-nNIV)9|gx!RF zN1}1#nfc;lc~_*&LgCoa_B}Qeh7Tew76X)D$ai1T6z>Ch96`NEIxB_75~s2l(!32M zVJs@G&>+IUN*gaqTuGjLqA1SG0m6j>@%Io@UUOQLZmK7tjk`G~okG6d4&vIoA?3_f zHP)bMps;`1p(+z4z2igNDwN(P4bIm2CzD_8Pdu8;(wl^_SBs~SpYlQFk(B0D-t*M( zz$%QVYz{`&zE-eh5$!!leNJ4h^ceIVoj~=~5C&)b)@JqWMH!kupW0~y_Ut+cGQ*3M z9#ZSTDAy&udPi%xGhFo^k`An@OQS8hKzg!D=TPQ_q}jOgTi0D%OO-p2<{&KGU6r>e zJgjN95h70=0F|p#UVvF`{o%*+U5K%#P}j0Gs2p`yBOjnTrMzP}iq}WcRuhB78;ray zWi}b&Vc8|>!6?#L*O4+VtQ+zYh_|r!*zZ7Dr)(_tpwc+SMNNdK=TF!+@FNmNqsm&# zURxn)6z;UJ4yQhcR`!U_*#JUM=UfNf%eA!s;+-&TC@yyxFQ#PAsCw@c zb+3X!!w!HuHMqwpLJO!^8OoVKMrr6tE zUz)U>$RFx#N58`$SAFu7o~;tV^3E3weI6qNv>m~~c|AsNzlx6AMqt7k59#3k8hmQk zqj@bvr*)2!=8UU(b)TDx`vW$?#JY9hvqcc}$jIeAGF)T_?`+wB!g)M4&Qn;HS#sS= z@v!7TZ~pxKIE=L^0^7H91bv2dPgsF>H@wB-L>p<-q>Icp$cCQtVxeB3rJNOb119(k zlYe5Y<&Ga;wO?KqipW}a{6T$F*6Penv@n^@GQKp$h;~fcxYUTX~2%zbbHh+XgJc;|WF;YT%%bzFa*q3$9r%hoEDIyk1~3^lt7VU;Dig*WADfJ~vs7+ukBCa~nkGw%~6+ zw~#aAE{eF4P%$&kAL4GC@fz<>;mSc3`}@%TYk%6wn?vclnm*C|S;M;`>|`$y z(?1DbEf*s6-D|bJns;YcL$=)!Nbh@Gd+5_OSUJ-e>Hl#2tY%zO@fV*g{sO-+LDRn1 z5!S)&qNsMzS*XvU8san;xZE`v43B=J{?lc>E$gHA&e0mLM{ZpAUUx<{2~C)hw6NXG zT%0}%3yTt#X3>y4zuSom+TVlsdmPdERvI64H3(PzG~(~di(uZ4?O;D+Esk5003UmK zwkzqC2sfgFIbjdh&oGuV=zjc#rWHPYejH>MZ;qCW461XgKRngwC%Px-NIBJrXVJ5e zh2@!5bD4GculUsEHl9tHtO+^yR*dTZ6PNY7i$j-Xsd>PrbpKx0)>iW3%?F_Rq-)1M zpyuRv(hnTqx)>H2M2G_|wlRfC2fD?IulZ+@dQeVX(pTO&U%}+8so>c;gJsl-*J>)3 z;dGN7*wwEIr#~=dOKmx8$TChf$y-D`!l07L_@&GPoAl^}>b19;E(5|Asrrh>%v8_3 zDCsO7=e|K}KYd(bmJ2VF&!bxN$(L)$?#~Ws)qE=q%G#SQ2kPAs`<9J|9($jP6SXT* z`)3jCi$J*LIt0Eiv=QIOM??L?cJlVS?S$>4wKvwKW0(Kl;{7RKp<@|+fAg{s4a{rE zvnem|egN0}@N2}WSJC<5LOAcI!)czi;jzEq&pg^sVgD>3JjRV>^QD7hYoxL9kq4*n zyBnq>)h^Zl223ehK>G;ZhY_C=HSK(kii^)`BlQFJb{xdL9_48R!)EbG74;zJ^dG`j z6yrAf$+TdNK^T2AFQ6}DBE>^52N}ogo1{G zC9K)+V>X7)ps8I6{|&F9A&j6i)abb%{U4YoO(eY!{KvP*29pJ@_*#rl(ht%6(!Ol9 z*TSn=dR)yh%^~fJwMdq|8pTi9rZF|w^zVZ0?%km0(rK9drjERnQG>g$9|JFr#-O=( zgnTtC9E>9Wil~EOYL3Ly{$-eL=Us)7!FO88wo~0C;XJO@rM=h3o6`H*0AWi()Qz63 z%>@VfbUN^XW-*|bUXNG4HI%f5>{7-mQRBrsc+lZG4q597kl3DIaIwZ3kK*V%-4z&o zs)6hmWlWrW3W+hcGWEEReP4_dWYW!PXcgKKu>5Ey-y1wW32J69#F(f8!3U99=h>2{K^4o1B8fw#I( zU|(!9o89^d{upvg)ZQ|f=B_%6bZV(R8)qmhf1MD%-~WiV%a4m=-OsU<%Q;|R^Bk>? z>r28MmfbuUmxd0)x0Ti4$B^NW=&#FryRU?>BWq#LZxp}I`Qfk*m2m0sda-#`ZCT^? z5u|bQYo=SU?}>4o`il`i!|TCz{M*(78b?!3bt;m}SKyb&QxxZj{Q>o5Oy|c`D=w0> z13UMhAJ02-NF4LeQuTt42UgSAqgg6Hp`~?^Zt3~#Ze~}7ZE|DTcb0SAwMsi|sQy7+ zi&o)cZ_7iX&}(&YhI1JhP1xiy6<=ySE9>_YoXTJuNdJmjvZ*x zn1A_@hSoz6{hv5Nwqq@s{P}}mVw6<6g0!2Uer3TgpHaX4#56ZUN&UuhE)HgXR~mEs zl}>WWfeXxr_6;M9z@j2ENt%arO|H^8H0S!?ME#}T&D0Eyf-?7T+oSXB^B+i=t0kw zCz(TgJGE|T)8LJ$yX&{u`*0AXdcR=K&#!1nYpLFsZ#?h7psSBjTpJ+=JxmhndM4fP zK>bU52Nr7h)R6g_h!%Qs^TG#O(tTKSqXGV%JPe0h-Nr9>qLChYrj6#pLoyFsz*|CmiONmv+Y575y1i->kYd_A7Yt~Reh+`Jk|BUaUPt4sTETBD=*KKmTfHQ<&8 zr zrye3q;ww$YKBc9nx(SRQ(UAKr3EC%wNDMz@Jp z^C-@4I-z)&1>6f&dPn&F9V`ZT4OAWl3M+2mcK1^d)8CwM=(8AnvIQR+q(S0Sq&30J zw->SVrV0o?m99ze+kvp*9lYN%Tk|rx7ZQFjQ^Rz?tjEGCpPqAguY>DD>Iic;SMo%Q zIQbb|ZmkbDLTf>Cxeo8(=Fh_9oGQE>KlL&auCfs|vhb&i50X|Ew63f=osd3seor7R zj--QN(*5IA96-E7*j51B#Zs)SSEiybPZj znS;l4U!}ca=*&+V(rc1*Gu#hKQR^*IL@Ml>5Fu49#G1cC<-fu=(4KDjtAO+g?(WnT z2nVRAEG20hxV+K<^V^(gukV+@b5<6Si&*u=|Iva@v0V5Kwa zEgmXJ%j2xTA*{}lH8^s+hq@j{euT7^qr97A5AM;;M90#5*sa=T@`M=}baWrx|6q;e zHwAG4RPP?>CzcpCVJv=j4&7(S>0IL@tAT3`ERO4t-eT_CR?=IYA25Rg` zW5LY}M@!NvRj(a2F%&$V^C5jmJ>oH2P#C{7v!`rOJts{>L{-Una{o2 zX>Rh2f_R1c?<(o5cr5US^>X8GAozKY|J#=!`?t9X`l4W1k@ zi=F=K4l9H1RACGGL(#g1A@*i5tXKM>im^3%`m#Fr($^+C@*gK#8}m zJd*GP=YFX!E-yU|^cwM{jJ|tyaO9+MxsFw;mNWqACmXOVOU8wEDgVbI^JX|RB^yYG zF^A?cK2(2{H6#ntN=*6rz6IfOmCoxb-*?liRTV#ueUYQQFI2QLfe(oq!ig2^^1S!z z+F;v=i6SgBob+`YPM(x09sT^95VgviROutaIdP$$q0-H;X}=q{eft>`{XPs$Rv7sCVQqo#=A{G!Hrq)-=y)1f{>W zV0Ccjzjv%)q-D6m1?7c_C(zd<5<4`iD@enUKPeUi&5Xs<>4=n*h$CUUnNBTVs@=}e ztOP~- z5naZw5#)DK6z`kV3&Uir%xRBe$DmnB^a@abGj>_2TQ z?7g)F$jec$4V5KsJAwSWxU}~;{-vzjX^~b?eS`91w9fQd{-Aktqbeu*y30(G{-9jz zB9P94TSm1cMuLT zxgxzMr|TdMX(Gqy)82xs={uq=@shkfkiS&(2reG<%xOTOhPYm7b>$Zd)OaXs0LtBv zyfjeu#!3Hj8k>A~`IY$B_8}@CPkBW_R~L?$yi1UNki;Rl_jxY)kGFy{BJr<%J1yy0 zPIJd9tJxssDe}-$J0LAXTt@ePlU{Dpqli563r1LuNuD-*@*xK{M&Asn=D_)DsD?5! zpn6u=bYzxulxC&!^kHH;#)#r?+=25b_aYXw#LbE-L(yv2^@ik|gZ zGK7(SMdckya{%$9(%;zo$rwR=hrK#(N6J?;t5yVbnU~TT+E_&~h{{3Ag9t~Oi-ISo zmHxnE_baIf=-#M1b=CXeMLi>}(rA~nlE5$MBa7T-Br}5?<-V(-csX}1kX~nmA&lyP zbY~wynuw`Bu^r!{sn1R@HG0v=o+)L&8zd}y*dfykthSxsMjlvLG?-)_f|Np4#X3- zbUsGf7FU?m_KJV9;3g$Z?1zOAw!V5j!Fv7T|SXq#vHQLsW$Nt@-^cN_7o}5~nQ-;G| zrguc8<0w;w#QyEDj*voO`l-t&RUS${d0~~F?7aIhaoq5dGwvuo0uQQZ0%;MIZ7||OLD?u+zxjwahYzNEk?+9!1HPE< zYRF?Nqv2}3dEgQ01>ctZfRV;0!Pm=`r+9};y>E_ka>gEP-)0{(SsI8=Z>;2S+h)h>d$ z;W0e*)IzRpP*-LqY=MODw9l1sM}G9vO7IwT34hoggZ#IBvDcu^;=-p8w?mX~_Rn94 zqZfPO7Ru&ar`>0rEp2%Er&qZC_8r_3-A?-a+>F!Xj^O^y?PY(rQ@mXJ3V!?wWApq% z;o|5fcpGuB-6wv$!c0QT2S)t<(Ln8UvjXUM(~9RE{Q)c9o65xU zYItqOe>lZ+6yFkX11_G>!E5JRa5auyu4^!Pb8T5aY@7UOdlS`JT%A&iMwvUnZj?FC40qCU|37dg#996s`<<;w`OMzgPJk7~ z#}OB%f$lfN%R83wFSgy}-${|UZYI!kO(r-=e;S%K`htG>O?X~o0q=eZ;pOip8^vC! zTED=&bTQrY3V!{y3lePpGQG=lF!g&PG;b6S7el&N%`06W{P;Q#%KxnwF_-`qt*rP# zyHII!T;HuYw>NKlq%|5^yUSP8>qx3KAitIYzo-ks_8gO*rXA!ke|>P> zULSLe^m*L{p0cmK4mfuGAq*md(A+u#-~Z|rP?e-6}@&s(+Sw`#1%^uFHm)2|fvXUTA$IeMX_@oAp!Sc_E0B6s&W z_OM>8Jovr0W@mXXOfAqs---3OvEfaq>)f5y=<|)y9Enz=0;#q?X^V>vF`8$YcXTUqL zM4q-W;KtTz+EJwuuz%T2NGgb`TFb!bPk6594Y+aC1z${0fKJ0&$kj1>z~gWV4)))P zHnE3=&cR@~eEt;MpST%fD=kE@M-on4oJ?mcH0DpIpA{8m9M<`ipwrS#}!xE)4^rgG>5#=@g($hDP|r1-oj;hlX!r`EcjQpUezItbLq_6+$~^qz4Z4? zaWkY2jLL7v*I4|8@e0t+EsKu%Rr!)fW$%9G0O$UDX z=?HD(>ZPRVitdB2i!W~xd>Vs& z*Aw>Kfa^W2c$3+`L|DK_Og-HUd@??2|IuDe)vwn_RrmdF?qTUhS4Csz1ggP+ym?3M&w=6YH=Nj^|!G+kqhYn%!N*PPvsr7F@oDcKr zBM;cjkuIzIqRaMH)Ccu>x31<@HF#oSDKrXxCBpJdREc9b0)xm}~pI4b`bmaOT30luv%r|!gGXeQEoZzT*|js_9B z@?P7TkaK=I()vmJ*}btbJr6TvVwO1BVJzJ0xlt09iSWRh{4ed*N&P3oXQtw)jDfJI zy(=ev66xdH@+H1rKs|(iAJ3Hv^8y@RiPur@I|wi@ZNPO(%WZM5+fA6!qy;Q$P^_I);vp}dSc?nNa)EF}Gvoaf_;)Rd zp4)t^))uFI*@F2sk~Muzn<3RIv^?TPV|a(egILq~t487Z0dG26blxs)9m8(;z5i`g z*g<@W+ou$O>4JMecrOV{=xoMU+FSqT0M#?px}1-9>d`F<-t(~M!~liwZ1KeiuxvC5 z2+MJKOb!rEf~ptaCjRWxF9*IdZ4+^N7##3+gx#A5vmq|ORIl^ZDHAnYrhEWL%iC<| zen(zWv6H^L9S(uTb$D<8ejv{KLut|}W)V9J7J95g;vJ|Lyb}AHn(`rDBT+xoUQ*q_ zA){5;xcWru%@<L44Ql96&VA%wfBgXfdz2< zdv!_gj~C8+qTAp0NVp8IG=u5freLbkG_ZXTD_%yqsP)B?;kEfNmzRP*lYalAmhc{^ zpP<L&+uzfGhChL0a34X+!RLH51U>!Z=NqUG5R-Yg-<~E0)+D#yHQ)QWbk4^_zk81 zV!&gyo}BvrTUKhqS=KLG;u;S(HojyCR*%20UFh*gP_3hh^K+qao3H~$HaF+Jrdi^z z6>H_-?A=J%q`k3bDe;mXf@wLeeOF$3>j8GN@2KjV(U>sxsV84l?1NvgjaGdIx9Kdg z+$+^oO(S8G@VrT9)KnV*ho9a9;zleyAH?SR(szxqSy*%2Nm1wcLk(Mfm|hcziOagF zwUQ?iY~Wql7}X2>V#rzOe77;--LjQ+Jp9g0 z)zd`3epoM-&b7fhIOwM}o^R{`Zz@9JM0lPyC}ty_2j`Bf136yo*_HPEh{cToPuaDj zy|DQ0It|s4;=C$Mj))Zia#yF|dHXzuOwnl|IK)Qg5=q{psvK!h#eNJ+52Z3TTbR z7VrYc9%dL=yam<;Hb&B2s>TTuJrw@1dvi>m4K>xJu25r{Q({8T zVjoo4iSvt}SFLqPtAXf#bD+j4=P~>_uuZf*k_?4MZKa3*Y~tL(Xhh%D7ES*OIiFfe zizkNC?@M>h#hy7BTKN-5qvC|nc`!Z07)W~xx;7+S(ey1XWexh6V!_)kShglo@w=R_ ze}H%@#H~U*Qd78Q3WmHouT|GZzpLgPUM5)pja{~>9>Fw`bCC2LBTdVcXV@4p9+gJC zR6~PtCVjQq4xltISQ<8wb>nNucEL4xu*EAN{m(1Q8pyqsy0YuE)7WCRFZ@|jgmnya zfjE@W=fbD5eCXrm3Hgn~)S5^&M|&%SfOHq>i!$->pBC;_yaCVBMPf7hU{C+iOs((h zUv#GQ>b{`(J2nkIM(l>I2+=_NKZ+->g}b{y`dgEvuiAMQTjl0 zzD9l(mZ{ID&l1ngyh+#nM!j#TqS1mDqLK6_UVB;;=U!6p3AcxzK{hN% z^Yz+8pqfOR;LDJ_tR0ZP(5U$!yyI_^g3#)mA5IUw4p(+Z;&$)%qTp>7+PxWwq)!-e zkT$aX1sL?1g7+$ByV132Nt1%+aDSn6vcj~q(~+wFV2JTBuE|NIta2+5uV9sSb>712 zhS@3aASW5G)e`=5@(fV%%a6Be(*u2iwiI!%Z_AaWxmJerWC~ z|8>%g)Bk}E=XN3KH!bxBG^qX=OaC>HPp5YW;v+Q&jI=D&IzC%emc@u`JEn5o%d1Iy z*s@P%HKpC%`I5Lx{e_jnHgYiGLW=dfU=VVOYxU&Td=r^)lc{3bzy4bw%-MrBy* zNecDjGq%TdF1#3=2dW2bX3xkb&Bn5x>ca5m2Oz+sA6z&(P0gRku!(`DqcXs_vk4}i z?txKwmc@aAAic(a+s3hzeYx`3O!>2hhMVMTvo@N-^STPJ#Pxb(ak^(;q;bnT6|3-_ z?K9Fq6)Y{mnJ-A|#_fkyK<2{13>UUnb0+d0W@^ay(({2YP-om!Ft%3}>)fmq9%o5uw>73jJLDb{^T8HhU zMcuGO+yTkL>8`|C^GXDHGk7)FiSI4w31@mf#m<%qgh^3&WLX?4+}c?%m31>6%5@@I zP+!gzs@{nQaZ5^V?)EngS_He|mYGJ}WN9@ydSePGOzjz|f%ZMtX78+QQKj{=t{gzp zso*%fv%+p9ZWKmz-xSpv5MH3FyKCnU0{I}q)o{AL+LAI0D2*z@TF2kG5s#t%b=ohI zdgkA$D~vR&nonV{^niAc>pMYu3!j;d6U6QOhiy9`9$_>F{zFq+qj;J$BYa;GrMv_7 znHIvpbP7<;fs50Q0dW=($1IfX0uOA0^&DT1`H_^;(eLgVRPyq}LX4${&!{0~WnMfrRmLMP6YQ{t}iE?py(- z>DzUE!6s)|asT$)VN_{tPWZ>qjPAyozZs7-W}S-2 zOw~ttWrd)c#*TN7FnX<2zVg4^U?5y(41j*NLQa znfuBQlkQ;d?jCGJelAm*mwHIVZ+4d#ckfgFk(n+SLHyfDiW4WvBayFlcu>XNgl8~0 z^s$CA0#H7@@U1=f{8`AzlS#rIEdOT!qxXj^PsKXtAkIx2#MF4L9-zv~M0R+*CbIG( zDjhT{ZnyUA?hazAeo<9cM%RvZZSIf{NEH{3zMyk)pWvOgyMa6r7OtW1$wN~aVZZ1- z@wKotj6?Fan!JadHTUW{BIycBUF6nUGk}TvFdu`4g4h z6x3*d3L6O5$dBA24M6wu>A!#xrF#Y82ip~3#a__aq*fnyGs1Kze0!Mk!b9-+mL7R7 z9Zt2vC^J(W#wZIQKhXeVp#gZ>N8u2+X#76ugqHj}ql^t-x7vuM71_A=bU7pbqztk? zWfb<@=G--iEzbbLMy+Gl8O)74Zx)2XS*wd}sK$a}mhb9pwZ??~T=6LRBgZt0kY0lS=XgWsibt0QMG~XUgLc?o?r| z(p~eL=tA9bYmxjO&m6yz_VZ0s`H)y$oS=1Goy91R1k%0CD9o3UPLvh5vQTkw|I)z_ zmbV$FUA!gOc72)P>gPr{gOs6ZJtFhPwu0{(S_4V^rSeG`YH=Uy#)lwfjH_ed#+G!tgQU!rGR>(_GWZKp_QIM3 z-v_EGY!@5?A0`czq%*pN1uTFr_QM!uLQ02`jvG$-S*95OQ@G)L3xzhj9 zb`QG{|4Nllk%kA-)>>63!`*b{`&;#pcsZN=iSpAzd6Mb#=EBC5rVtiAh@QV#4X;}n z%ZIlDsxlK*^W9B*AmyJ-+jln{8Q)duKSr9Lz6Y$Wp_+%wM~eW%H&ZU|J z+gtejMjLK^HWLnhZj0+<4nb_Iwcu4a9^EGtWZRwF2n{P*a(cEL+F$jOX{S%1C}@pN z&r+EytV0hwkO6m^$>J}U*yKYULA{TAk6zr=_BSlCKFTaa1U!G+NqWUC1OElf`MQS_ zgwDEwa>Vj65EbYr4kj>(mvq>+VL@0P9Shfc+d})1H*v%$A21&*pqBOqJ22o1G_iUK zGkW&n?^=&T_crBh-J#_eDt%>BOLrO6U^kq4o{z8ZS@2r#vxLsKesa@cx(BB0vSw-L z+VYh}OU;T-Q_;$ICDJYI+VA7^WpOZR`-2O)!P$5=YoVc>7nX;8K1O2nB4b&5yo*d+ z8P0W2ZhfGdplOU$S6?C4l9!8!|gks}Xoc;|3V}rq~$0JA? z=EPrp-GeUE$74p<7wFe^FgMfC7i@diGlQqC_@%{m+Muy?Tt17PPOx%1JHd z6N@_Hz436o-Z~b-#?-~KFOS8$*avKW{60yaE$H7c1iNymQ(ZVe(B<$ zAs1u@;``%TyjL;}l6w8bcT+yHG|$uUE~^Ybxb3J~OB%l{-S zdb=NB%Wn_dAoDkjrTN>;t7)^a;mGE4ZqxTTtE@UYd^jkIHhe|L9khq@^v0lf^CRZv zI%&P@SAz`QyV@Z?Zo)73BHUy8lj`myhS>z+TI=&PH&dj;1RHRovu~(|L9_WLz7H5I z)V!!V8QU^Rrd)KCrMXKmX+^1KRR3yR@7)Tn)_`hT9B-z_ulGK{Dm#Q=>-*U;_+0n_XBzgG zdMBpiuDcO@g!TYtYUh8KRvv#o>quj99%)!-TY>zS*VY;y@}?V)Fn_M}0pze{C$t7DqVxjUfY zW@Gl%`65&Ec*b=Y=r1i|OCww1l2_;8+HATu!!Y=8xsfDn5SP2q8E*QQSoZ~bGJZo4 zlwS_Upji=^>D?6d^+vN9xBKJB`Z1!@1Xt-a=y`~+17yJujTj^k_H{Flav>2%$A=EUVEhT^X)k11BOZJPrl59Do ztro61^^j4MhVlV$5`)6)QQn*>k{4xbQ=VsNhx!Lgs~2mqc6Uek_O2!nj^K!w_cY#z z*5TN-=g=$eJ5+OYK@DMXMEk8;M>`YwDx?`c`Kl+^h0TM5o(Y(1vqY||{{_$X-h?ei zy7ByrAE3Mj?VnK=ES!HB^L*FYeDTg(jQT(pHCzGuz602-NHfV?PNBhHXG!b9GR-%! zK0WCy-VtGRADw|5VfP9M!+1v2eXKn`1A2S4K~*z)w$E`AOb{JAeuq@=^{_S5Q_xuW z@$_(+pY#HJCLTZslVem%@7%uAd4)c=_oC`OS`Vq`F~^O@r)rF=Yo;|tbE_e;$=T~H zV1+(m{!&>|?kx#>F>iK`nhUOKr>u1!o@Dd_2;)WE>x&rgYAWw}Pm*Vbjm1f&y|DJl zmHH2H4j{nm_2&ratfDjUrY z0>VK~HKQdg;q-I0hO#VplDt^cM?MeGLb6S7=~=IbFpsz{71r84+eZDijj~!-@b0t$ zE?ue*ga=4Py8Q^OTT+~=Zv1j;p@Pl<8FA@`U$A@pc*Dj zexxN#6a$*3VDG(6;pN7w;*9HE$W8OdXkAlT>a@V^VmB0~wkuViAYm*k?yW^ydrU1c zCak_E^xpjfI+dO9sJir7djvanTqd06r)!4ySOogpo}lxOjr`ueVHj=G5DBMwn2j!U zFV*1zD?;(xD%z9&)Hz1ikD0;iHBScaqjj3bqIDy|>dH5WcoZ+zziz$VA}-x~whi|p}^b0CiVX`=WBGses(-s%VMrYrzf?+EjyXLnW4S=_HD3FA>`-3{&S zM$Pz4ZVfBKikaEcw=5%SBzEgJS5^+HT{T~EuRW!i?{hKkSanV~i?2fVL0Rux(0;QU z7RMf8#Q)G;rzR&}6e$gx0^uawGan2z=P(zRt1)KZzhNoWn$5|tgsm}>S@&2gHVr+` z;xBGvBOk7V%zdVk=AZS?7$FYKDnysim&_Ng(pk>0#IDKaGJR!&pc-P2AKUT3g_+R1 zP9qAAY-EdnO=Q^hIIm!z%O>)q8PX(`_N;6qv`I&a$Pfo8_-YYLP2l51K> zooyqvac2-hXOmUp&Ggr=ba=l#SkBOmI?03&0$IiWVD`|Z@s zZ+8fPM_(2V^qpi}uZdD!@9jAjcrW)c?D;egh(pnDZ+|gqiw_hwyUJ>BZzAizAE-18 z6a>^FtUX6-N9VI>2l75WgRAt@@vS=2F=MMV8EK4@w%pD(pnax0??|Xx_oz?4Y|?^$ zq?;~Nmi8FjO&%fX&ngTgO-6Ir6a9VWq2gS-#2~h_b0JvA?Iz86LECxxP7#wso}*;H zT7R5q@I%n`i~NRVEVX2#n3!KKz0YrE8EF$?5${vi#;6gN_T z;W%z63NN)&nh!|FapG@5SOZE&>>YDfgNx}o?oM4~UhrNP^@+aUTyPc5G!H@XlfmGF zNSc%Mtd5Krup0=Eu~Xa?m|Q;%e)j$+dLw=J-It5N$ZlA>MRTP3$MPjn*kHVuJo)+y zQ?*d=)<9gY)WOO;L#1i){@7R~4XUs~oNCxfzIV@MfToI@RUE^OE&5>IVNN5U*d+7k)uNZ)_b z&^Tc1PC5_w@^SdKJ4RMMp>rB`lxq#F7*J1Qe9}fF&j7@?@Ow!vjy*-^3=v`cEzo|yud7VI3s@L4Nh`7&Oa83%e6E%Rhy{qdmUpM z^aauixbERIpx(`med#T1@)OwubVM(Fus}^ z1M6SMh$uNvkbk2yGA^-kqs^#pQZ(;+S3sv8$Dr~_6eHcjO-61c4VeL*Gm^#p{_SP8 z+KynAI!8FIkCXwgtD)1m=G^I-Bd2lmy^HAihS&vMX>9NMf!w5`4sB?0MAZphe?#&F z8al6-bkp)NlImIGtCOYHUhDJmFr&{EGmkZrK?6dtsNpZBa9CgOA+mFBlDGv|9C{{b z-5B*UikhfAX%()=^`bq6x2{$EBm##X6~x78(6O=XwB`)+$Z&^J^XZ^)n)IA@reSAB z{LM*6Xnyuy1E+pF(%Q|C1L(X++*)7xQgD6{#K~hI=_I8m;Er=O;uIr6{Q|^|YE6;6 z4AnI4%kndk{NH0{cI63>-s3ACUd26uUw||oqj`qFYC=Qn1;o8>Cs+IgHSgAK65N_Z zJQp#)x(Vu07Y~9bDbd5Jqk4n`?X{S42pWEwYp7{Yx z`+Ob<8?|*!bU5)CG-M5VZr9bA+;tFad#fpf`HTKS<=lD_ z+HO)_0S5S3D4(k^Ln^E$?L+$5mj&F}2c+?!_OfkAx`NS~!u2Uf(002Q^|U?Gwc>O4 zR$}6g0DNMxg^|XC+(j>_W{a4~RejQ9 z{TZ}vag)W_--nYc{84}Da3H;+S!#`$-Lty* zGwYYCZ?~{9+mvU7C58i#uqNA~a5w4wM7r*IU=i9v`D+;WYCC!ubYx14k|sxoxDH6y z4CYUBHSPap2=ZvcWBD<~VMsVB2uJz#FX_-~-X}Kq=px?YNUBg8-7qu)T2yd_L8R}! zv|k?IuJWDmqdBMS0{r$~6DdjitFYkt+fy`0wh`TAfaJ%Qw#fP5o9p5wrE>JF8) zPaYzVJRA)z&9e;;?tphiv$@hqGg@}TC(aExaTKgQcA09`2Fd3Oz(^#1VWdTH276A5xgvgbq^hMO5DXL)6mp4Su4tDFGYHt#>0-DeOnkWXbkLP&D->B_CrBfpz;l@ z{L=_8`dv^t4{=Z!BP_=>GjH&@oR!XB%zLbolgUQd% zhu#m;N#h-cd$Zp#RUeeYXz6`q(61=rOOKOox{!OCbdKou{Ws~*I=t22ZyMz>qPsRn z;@xb*DDq4rfieQF;OWLZ@US&H^r#Ps zyUaPwFOWy=LOcZ%cT^(bAZf%1^lDRyDno#_wHRq3-tp{jAWTe`&cwM{?xTP>a78m89!MwK6<>KZ7*a8uH>m z8c?b^ApeVMUS|3`ldmw4C5vw}g=@rlfaT#p*vcpeV#L>+ay-)ihMd+wWpi+;x`U+b zt16!(ZUW_9=sj@W%r1OsStD5AD?{NAkX|P5Y{QhcP+EFa|3GYga5QuWE2Nx(r(8@W zonMX9T4BR~p(<+=ia$vwz`ft|8Tlmi^>IL@eMy%w%8>;9o>HK^Y1s zEd^$JpFn5YMM3$FhP1uPc(~6;d)f4UTVZZhNb~t0s{G>M-4>ks4>FxBx$?KlKW#dk zi5{cus{ASC73}54e!OhTDEOJRf}UsYfRxh_W{nW!PZ;?|Jk#)+%I_%CQGTf$Rb~=c zja99MdB+ZtG9`EtvVrcJ+9QZp1+AqB$c&+!^aks{yqB66LEc^%oV6CIxw?$B5PsZb z2wRQZ1kEq6++>Zv#y%0fXV6}HAzT^@rLRXFpQ9SiVEbED4jwF!#8p zB6Jc*rDustHSqql00UlCakNmB;P1%Z;wOsjwgV|q;eeXit{s3 zVKU`QTGyjGIJfi>5*JBb{U9VSO4(Z^SR6b`UU`HdO$0wYjAY z6{PV4;R(D8UQv}%!3!>b?bigJV9oOAkQb1M^j#@{ndQ9 z|8_4>Hi?eQ*CMR}%sX_IG}jwJxdcn@dXteJU_;u{9$>c`kng^WlsQ&q3_~7V(o$_8 zWe9-7_0Yfcvs=^YC)kiW&h$IOIr&(jJP6f-@^hfP@)T2jM!7ktZ1>%lucS51IQ0w? zuCWmrbAf7!GL*-x)x3qUH_8!5j{EZcbTeRZlAoOTJsuwF2?EwpGWc%7?>&N7seQvuMoh>ndO7Gv1tEd}GdI zSG47wR<___Lx$qfZ!a)MoMerg&xR#MyLbo==MBdUmg|3-h|XT?wQHUBi6D9=%-b&) z-S0Zev8PVqt5&D9?srpR{*~46`84N73)01r{rb>zR%=;!%$WO6>j-a0CyKfw|00iB z#4WqfS$rSHOMW;W={@nyyMa7AYdoGySt?RBmb`tVB&Zb+pjT7?WuwYaMA-ABTHzx9 zRWD&;FNEy88Ue$J(Hjg_9BGCK2sJW!($-(?rbm9F*Vo?b_Q z)(>f{GV0kFY?ZSGyr&t+fBu=Mwd>6FioAK#v&TjJaw~ozWFKA{Xd~|4JBiEoxM0A> z?_!u^6r0w!Iq3Ij!EQq#-MJl->H*FUcIF}ZbiYlzUfEIqouSWaiK~8G%W~8Grs_bh9z9C)kk^$Z zhS9js*mSMX>qgP0_k#AoHI%5dSS6A|LIR@F-DFCe<>+C+DZo z-p5{5WA9av$y%qZk#`GkV4uqF_-o}Q<~O1ikIC2y-J|}pam_c1=eO3vH%R3yTe78cU+O=DO``$~OXzF30hDrB6`*1R1bj>qP#1F>}@q-wTOVk_OR z^09O&?0%sO&JQ1JD*Q6h*uz3Do;;ozLwgx_G>(-`PK9DUchvYtLqP)*Ih*&z%O{Rv z>t;=0SN#N-Y*rwoPj}6r4xw0J;UJryb%vTZ?rV=tZzc^=Epc$@H2gjySm+)Y%U90~ zMXGhegJi;%jPgUz_rryieljLu&g ztxIG5CPJf{5C$0WIf)u<@cuWueo6;I*AL@U|Enu~$5sQnE}Ta9DyBMJcN^7W5BwK# zn-MO-kUt@4|MIfDvGg=lV{_!kkNbJ??`868kg2r1cw1wxlg)b0kHDT2CdeDJ=i%$n z$1pVe9^8FYA86j-&%yWD?~(`1Nz;X}5nb@klobje#qeldiC<#~cW57Wk4y-Ne2Akg z=gal4O7T(D5+wZL&vqSwSFJih{lE5f_OTz&FZu{K7Tjhbd0MPFZ8~O+I!({oZ!-K%J5veB;`u+D&C|f$9YlZw%sJtlLZT=rU|D zp_68A?KXVi1t;{LXp8NR4w1ein_$hy>ma4@02UtXA-`^_A-&pnTYQSvEkN2=80AkaNu=rQ)Oo zorh}m-NwM&>d&D^?X?=;fx|?-))TT7FGLJGtkq4T@1<6+#K6t{uydJZ_K+V5V0)p4 zRQ#AcuO9#B-&s=6V#h;%qI-e^);ee-iC5&2=ut>DPvcvNRM%ANJ8)Z5OWti^Q?7X6 zUd$`p9=Kl&YuOgk+W`pwA2KbR+wxIJJHdfnEI)*sN;7L(Hb1H|PqrE&kX%t^z z^zmGXUzAlb5%hoz_!dO#O z9Bj078~#YYj8p&HM%R54<`+9~;$ON?=Lak|+91QV7t~x}zgtF}aEOm7FJu9+2S96= zNj2PB9*#e$`cF`u%f{Oh@pWWNzHDENpm8$7RTyR6MR7J|gq>)fb@-NgW8wZnOKkRa zAN*{fg&$f|pgyS54T*O$(D~_E7Cf>qFB>&UP#vYF`$!}VVGld@r?cy>;Jm3HVGo3W%}nTH*(;VsWt zng4K@X5*Gxazpde5PiHG8qL0jxvj!s!doNRxokVC{-!=a!bA3HXgW3=a|`)?CO>`) zMQaCJm^XC+k}iZ5Hx8$yCxYl%@^7z>YKbGT##}#~rdbNXV~@hPCic?g&lG5Q zm-m$#(- zXjikCw05K$EV;qH0ray&K->pvjBSQSuowSo$;P|FS;K-2V)gj0RM*>q`Vs%z*%(&b z3ByObxzO(u#_0Ot`-tkKNuxA6#p4)h2f014Ial1-Jt2k0kf$LW!Ep_G(0sVz$2VQ6 zj}GCdp-wovds|k&`X(fn{yV&@Od5ti3aH ztTvGDd)>Hpyr^z`6INUpBO82qf@cD~xkI=KerXYn>slYdjDl)lkh-0Ex4JCSc;LI7 z4#H=2J9yvBolhvsBOPW6b*}f31tzqo(1uo6HtrIb?XQELtQIPNMA`xA=h8asCElNN z90;G44uR)qhr(tVgC>iE@q3?pg6cwJ7g=3qU)ltP2hWMDVYhM3bqoG>yb;a4Q5I=^ zMzsg0KV87X@f~rjb{E4B8G=5aPrWyt{VvNw@?D@ydygq?w5{%Sh}-)S>i;|^sAr+l z^P+h3xg*kefIUh8x;7wQ#oW3-8EJNGQ+k^jnjB!iEB3%N%{Lm~9q5?qfvz6ixI=x~ zzpL^rTu;ry&`oY~=iqfv)X7gg+to(KFW*D=&sgzq@6AYKH0Q(z7!Wy{oBS!qoHO-# z;*Fc2nU{~#k4K9kFTZH#Bow)woY01b&`erwC{b%dJsid&!|Joc2kQ&c;Yi~bY5B(# zuZgE^NJE^Dpf#-_%WeKo6>B?QR3iG{l+@Gq9rR2}f5o<{t(*)9$FL?E3h-n^W}$n(PB?G~!{v3ufXt*Rlq z9{KHk6r)}OszL3))s666PAK|~*d$|@^c9sJrSPze6V*sGw9a+}Ylj0M{WiFrc)yUv z?OlMLom(m2N_u-F^PaX|^$xS_m<;4SQQ--B8@%TBPkUj-Nc8AlO)7n#Nqe)`ibtkX z`~(`?->lNLgkOTV7S0a0r2QY0NmsALI^!4Nq^vNF!X45As@}<`9S6EbAkT}Wzq40d zc!Uby$P24=5p7C+xbg&aE!1b@@Q~|S++~hn^}PdEII46AX-~Dr+DTc4KzOK8b4gl6 zj3`}$mwNeT5BO)v>B%Bg+O)~I7OKapd~zRsV+~=p@*nW=O%8ELf2hCd9s19%&&k75 zF1wkfcT7;cBRiZ5$AIm*jCg{c!yb&gOEio$o**7Zl{Jv>pf$7y;u}WVP5C1+G)I8t zMIUf_RTE>^M6m~V_lr9BOgUYXP+EmFC=2?=SeIjUi9i1nN}v6=bGP_$>WEOBPk9HZ z%%nvtbF_axnJdmE9aA+IGzVDSGy_RnVcNy3Vpn||^2vsrxL!lK3ES6WH@+R7hvZ*4 zd1NFk1X_DW^TA!#hOyrZ==_{T&tb~q(d5@Nh)bV~96HNZc_N?jTjAH*R6&>_l=d3F z=99*}W;K5Kw3#H2LH9*hB8`Q#-Wv_&9^|`&@y6`;LTP@gaYkMUNMmRSXYhRe{;0ep zafOK8iub81xH}cno>~^hF;uz(JFEf9O32(#MpXA#tcxbtXt^?=s81ltz>YhuIt7cz1+WX@dc+qw;^hT< z`EPYB5>}JZr9SkJb}^q&^4Akbve0CwdM<8@pVw{{2M}8QhWA-!xafub`X> zK7(lesUqtLYq;zm|BgZ4!2bwuf1 zq}&Cx4vl!D^WDMNHD7%ErjL}(fb#7N547gWldJirxvk=|PAd;$Bl=!P@fziMjC`e* z<^T*W3`kSXW`xfe5Wim4x2jD+d;u1dJe9{lZ{LU7QLWeFKDXPd1{moxZ1RboSJSIY zX9+$+2divUm`GYdoC)nfnVh%ssEqo6lfGd)`|pF3WBM@i`|MEL6{N-M;{HQPu)9Qy zwr3Km^riC5Hy^hWQ|Ua})K1RaYMnp4vb}|hSEcu3&BJ!n(P#8A@o8jTMxF-~@3_G4 z3M)J|@+)zhr=)R1+=07DSgiaVOK(kQDuu6O9n%xp@!e5fD3{Pu&w%1&wT?<3_W3>* zm7gJ;lGg@oAzwchjIS;sujs^y8_0i^vJoF0A#_JIPHUia1uFifxyx1_`N7@hJa&y4 ztkT^Mbqd>PY-~pJ!sNqjaAbQ%xIlhThpTK(VLfG$NO+I8JUTO#lTh~3Wve%xaobBr z_Pb^$scvBO@>Pl_={klfeSnmENz#_2ll!9bZj?8%HRrsjcH+hN0YWH$G2}BS{8qkB ztjU=M--pEG`d1Sb7YXWRApTQYN)%Y65a#J4c^F2yAdufg^4k#M`5Vpknn?1^Y}w91 zREXvh0#E{0lFB?f(H-uVdk8vgn=7S;p~Z@T2)t7UO{=en3Su8K@T;dQE}qI zgHXuarK9pOm9HpYE0r(17w1ix)fXT`kS}H17k<<9I{!}Tb5Nd}{5mVG%m!7r ze~z7oorCqHe%CXMG%}Dsz|rfInTyeD4e3o(oJE-c-`V{>UFS=n%mxVaP-%jDaRsRG zgyx1*j)tV~S$or`NWaI{e{n$acUtl*n09k35;qFUl?3I2NO}s3G2kS}IpT&o<-P@f~d?VKskm^+^4jYX7U4Ip@Es?3{WT4ww8A4Rt(tJQ9uN!U%y z#mauhvS?dVsaJ9fr)_0$=394OgYLz5{R(uB!!B(~X)OB2Mu0`^eeihXE!6vbX&1#m z-CmCynj6Tp9B-Uo5Cgxbc=Py&cG&&!e_*t{8P{*{1CAJ?=y&cZlwbP9jyLq>_v>sF zK>;K2YdGx})Ye$;URjBQR!4(DL`ymRRTel_7~zfw@93W1LlAPW6oUfp|tfeYo6@Q)z5m45TezbCcN@%E*@x>g3b-Yr9qd`822GXPMr1&2erP8 zjosG^b?qa%)zDrXlLD>@3*__42iRK%&=hUk3*BeC%l7R|aMc+@d0(50 z@495+)ebgVo638rKF_Ek4R7u31r{wLVD;}~_~_|){Bp1a_rqH@A#**dG3S1$A-`W; z4cm6rlv(lnG5>57o~&`3>c>Yswy|do<_}~&U$lXWohh>7#2ohM=~J=sZBrbk7l5t* zY~~y3d$(3|;$%(3XyC6c`Qo}wWZ=@>BKwyk4h|VinScR~o;3-5o{W-P8va#tC&#kJ zQq2+72)r2}p~X&njJi@t=ijc!UFH3SC_jYsS<>kz?L7p0(a<*sGe$PW0^9reTe}Na zZ?plkw6XHnI8*J>9;LW)ZFl%kQHxIPWAFRtpMTRiCJ%$qF;MN_w} zaslZjtB?LH`u!q4$LEMBoZ-j)hCjwyl~PB$9ExE%-ijI1#-c}Oz6 z_4$qTbJ+a&o~ZZ0S|)T(V_Q1f(m5JA!hNbPUnwde;Sk5TwN~=xZTi0YMjrc;qbn0@ z*-25}5;L~-Qd+^1MutdV!5v7Y0nHF<*Z;7OA({y7jKOKPn69y~MG-v*7#HzhG;SDJs8q!_Ct- z$`{wn`RvE5WUmuFxnF8)sS~wD(i*`Mm$llIg|)E5bIN8L{1E-ZG`wx=xp4C~oujGg z%08L2lQ&`vD5Hsu{)*mz*zcs9dd+ee`**7bg+w?rnn(i#5Ud^h$&oRD?onud; z`u)G1^f}cET)5T+XBKzk^?&K|U%g(7#w**hM(Nwxn2T?~C9R9x^mz=pC7Q`&S&=Xx zItC+(8_Gj_>fwuJy@h6eGOW3H5&l>n&|D7k;@P=+{Biz6y2jVo__a_yudY?>+5H65 zz1{J^@!e9*x59(1Egq>}QS}6Nr#B$2J73gSPoAxJNVC%MJ1h=y;%c*?6 zr0d0UtFAosQ4p>_+(@pf_eu36?^LrpSDa8_JCd8ane!7v|FR(iUy6w0UE+D3i=@vI zCpU+R;*3C6VUP{#b*HuEaC4G1Z2uC?bf72RtN9w-rv~Az`>!#x=PooDvX<_n3Wj?t z{CV8eWnAHRJxfpNG{XgMeU8L~0Uf1l!bk9bwG|vLA7b?T(AsCYJoikBc71QLHYfK1 zVGH4F2e~LaT9fIpm@}Q-sOECHUJ(-K{EwvTj_30G;-sV@l2VEi8EIjy|t&`xqg3mHSl@vbI&>Nah~Vii!8@mK5>k| z!F_V*%(?ptSMmPTWO?V1t{mVpKy0t5q_z5Y1tid!%ug-E;JeML@?hfR#q!IxG5leE zDy;fclXEvZQ+|UTVPH)*yhpf(I1Fx8Z_B;YZm}hyrvzanr0vGPZaHhKw_OI= zm&(9+?QdLqzaPX0)uF#HVKuUH@N#Y}1oDq8vHDNgbeQ(w+1d{Zjz@_5Q%%|2Ei;{| z*2Rd>mu$h>D5i39T7&0!zhNKr&#WdNy4L2Gx{pP#wjCj0db)6?wet>7Uoom{Hf7ln zY#pZq1N3%cfX`QqEUk7-*GgA2(7HA@inu8*zjUJgGCbwMd5;BY1mfv7JS(v~zg|dZ z9#KBdEodN(&OF72ecQpa*#2nRx}O|4wgY}}`34i((0Xl0M`$rP8%Pg`{!iUm==0rR zkXwuQU85DtI&1*q5{Mqt76{`(>6lR`+Htx+IkKdacsqdeh}B0?)qvv35N~&OBJ(?# zPWpsPj1R%)kFBe?n{-kY<~mhAW!9rtR^G zeR>A?ReW%nukx2_tN~VA8jY^?yQ#R#)LEhYe8othI(Ifn-U!v+HkKvjeel-@I_EFb zUbdRpL*{u^Q?U~MtH%hX6*6^uE9^kxGy3~E_Vz(LpfBvPvKO|`puHaR)+s&^X?tvW zk6XrEaY1fsF1CGt4&Ki#)9gEDEa&g(4`t7EkoXM#J<_1!pRoUIIE@=9hH!ji0gjF- zl5p_=G&EXMr43CM&&R^?NzOD^Olh0#y)Pr^*LHSZ#+X4QQwjY;`-8Wa|E zY*LGlj`4))Wpwt#S9+C_bW@&J80z$Z(w66m2RGSVIf=fX*IrpOkC%pOL;&~K$9~__x#b0U%CP*cj2NB zt?P=t1m|zBCl5V@cxAP!acoyFhmoGdEsZYW`r|q9#IXi1KY57$-CvDYGycC~rLBSH zTe$Wg1Ia5URn^mtC|CTE#}wyj)OB6)4dypU7ubUQOt~kLu7WOy=Zi^yVxWD-Cit^; z9txe;ta}H4AWj7vyEJgT5`cd9KZ$4qI%{XAIc$l%#|+M2B0e8a&BBdHx#67i@{UkA zGW|yuxE#q96IX6wy%tzX3&${^G2)-CUo>UULTM}lPeyi@$3KSCdw3p#Vy~&6SEh6= zVT(6@FSkb9uq@b_&*xHKAbyx5DTwmiv2UQVP?D!RtE3I>xK5R%i?Q5SV?iiF4Ft zM*M$_5hrWNQ$euHPR-WA6-XGxb8-(LaUE#7=pku&XkPk7gtyrTicc4{DsU!^sqmZC z_1}b~^Wa-$AYYd}6)CqlX$s{bv{ZX=+s0T{Hl4uqPsJ+^a8_Dt)S#_MSSghMAT7(4 zE^}{|qN%u1lW*|e%v5d@j)Kw`2QTcUpKpObuUg8VS2Mt??S93HQft0et8|R=6oe~0 zWLrI@Wq6GL8|D4@^SybHI!i~I>sPX)^!}UrzN*omhkxR39|vaA;~ExB)w0%eN09~@ z&KjRh$I(r@V}~h(f1#UTP{4a!-gRiVY^&B{>+dz7eA4Z4N5qND1<+vJLB&5Th|Vx1 zEsLfr8c8)~;u>k%`Xb6V<=Qr5hhb(>3vlwygVu9t&!oSJY>NFfPFSo_Ue#QG1ycS? z$^ki$_H9<0*7wW?K2HCjAiWDW2AcC_JD%bPy=$bk&OyZYUJ&%|45k;?l548j$mrA? zqW|13Qt{W`F{`<4MF(8;+600R&4mNb0+f~_-h$VggEUkVAZtVu7ukW9&Jo%*V>;E6 zACNZZhk7g`j8DU&w@t|x{8aNx{rjct+nEz8hXr{m(XQYj(mazVUEh6{o-WtlHAO?7 zOJOD`?Ms-7ca7>Od{Ono97iLuS$Q)=j9{wK@@NzkkPwZxlab@S)F0 zpC#%4FnqGFmUw_`qb;=aXW8OHzf7dqVbr=5;yPcP8Bb>t-`Jx#f;`x2nzwKCZpmsb z>2)M*u8KeTCmGkk6@0cG5<}OvNAiiRxbbfdXDqmRz4rZ zo=~G{&I9U62E}r{;iMwpAUr3Gm;%HXnB;N1fa0xj zbeW3^3zZI0ct-jS&xc*aevR9JuWv0l;?YEIKRS|y9bLg4lU%U4@e=1ZCT65x4LN;= zw$G3AjCu`>=8RuixJ`5_UQ(cRjM9?C&L{I!1Lh0cwp*2k29 zEDgm^5Pu?ZJ8{1we$Hr)gcnHhlB5@_`ZmrnZ^3zxr)I~d^+I>U0DL3+0daQ~mMg8< zJTeBUhoXK*G@6=-{7nk0@_a73Dj)D)mq_XrNT8mB;(mW1b6xx4yaQS|R^!#) zrjqXe%cy?`q@$VYQ;oN6gx}YW;KTfS%b=bK;#WGIgXVXRk>4aQ>!8K~iPaau{5_Wx zJ^}Se2qU*cegkt>K79)5oKxie4hrhwux{E$p!!4Wr#>QowwW;OrXbD{cJjO+3>NF# zYe`SfW~aN>A`QO}ntApT!&^NR>SqaGu=|Ods?MaHiD}i?$3FK)#eWw}BvP*eOPm`D zia%F)M*W%YyJoxy|N5di*m04K!np<*4}gqt`Dz)$`efbQ%4@T84aDQhcoM%T{&S4 z^uBJVb&s{CK1?1cuBGuPu2p^xRNZZJGnB-o%1>kWzH4yPcxU1+MxPl0q~Qhe6HuR2 z%=kB6^#kzS#Ft{rG&6b8XBDQNoq{&k2e4X60Z^9E1?Swnr5&`q0VnKL{ue(T+@^3y zPHRhh{j9DJ#5FMM)qPZ-MV>+HGbLUqO-+2m#{`r1rT!UtTh)uu%uegT`#(L6R3{qM z-yr?2`4=+_F6(_0)HCizb1zkWQT{RN@$$nqT~+TxtFVVO32BJVoO;7R9#~8F1xnAG z{R-1e|It$HZ*oUVJw1B<=~u;LH6}-hHe+v*FNkCF}BJ#kgsEO zZ63vo;rZ}##pOcz!aAwdIrS4YF5qUz!F*#%W|dbWuG4zIx01x=Y)tSf z_?YAd8G_#L?U_Ir`~s=}jMPV#%{GZK$s*?9(f7!V(|8*_K{U#T`ake z5$&C}(3MZtRzS&UUG|XH+DCVsDeu*(#pWMw#|^T+v9r~_CNz_Qc_UB9f5rw(wo zeYZXK$#2R{m+7)`Zwj#Y$fj2UFj2hYSdXw%nO zmi%>+sei2a@6+3H&9$cdchY8NGVUh&PjzOwcV^@Jqt`@@u(Qym#(LZr7zs)5`)Mxg z%$Js>*TFfwA$JLwj4SnCi1PF&NY}zl$7!rqhZt<*bqqed7y*%wJM&YI_4tNOi4Z+u zJ~rZ6aA&}D{<6hZ8nAH+|+8hpJWnapiCG5A3jg_CVtlOe9jiZn? zp?yU{hFbDxYii2sEe}I29X;uNA_U+5_=F8UOqZW_{sgBZ8MJQKh_i+tF|+k<&D9U3 zV)m^JSkW_F4jh+_!{7hG^@f5i47mz*PEO)Kwsqx)cLbrDC#%*5-1~z-%-$|O-Rc4F z>=%oFCY!le3q3jaYy`J2UaW02T_T838);50wHd#m33i2hIJ^xRhJWOw0ok78rf`f|^j?)>uDFg`f16n~$2!RUUa zgTn<@yW?ee;+!Bm*ha$Ly;^OZ6V^C;=N%}$xEp_Vih~M$FG)Eq7L|-+sxE_enc=ao zhJsDyKdq? zK$<7$Y0{SeroB$o2!kp`uj>P7&vbX%KcCCBJv;JD9a&O^0L!>-}{XmywDd$u9JYU7y}FP@kGn*NUP=` zY3*MijKj&V7el;DHU48>yDI$9nKw&(`MM7_zF7k_ZpydIfO9v?J}ZYh6Bdf(0p@&i z6E|$LQ=e~{YJh|}RdYSJs5kCi_*nc13FiG=4}pV2ANg#go3u;IXFZ<=!kZ1vfeUW42*aeXcXNEvI?oJ(6ds*vpzdpHuub zazU@&(hr=((_Jx?T?=3`x1p>tZ^iFNVD{K4n%->I7#L`5< zl+&7odQSHb@9d62toKHkSbH~H(K?jg)AvQfW$5Sj4ZeHW!soNL za`dHQXnf_5hUN*U+-WX#o$AWcOK%F|LZ9KV__r!gfH)Kv-L#P7n)YN@3)kUy(+vD- zy9oR4e}Jv_^wA__nDE#ZC2+6w6n1ZQ4!5Ky;D`Yi;c$&^+}YPqPMflrN9k zFsEGX@Vy7-t&azL`_-6wDGet1J%OCwW;l0!4@nD0`7?(Ck!*RBrH}anaiNpgvQ^t5 z>0K>OxW+KM6;Q0J>by_>MjZ3!I@7(m0tkzN?u8`}9F4`s187}-4yJA%0)z|Dps*$H zWcGFw{dEw;~${ z4UC2*lNLf*Ofi0(Pv=?u+KTEvF1nVmLEimYj+qfm{~nKj57Y&&m%=D-VbhXK?U+A4 zXm4MWt2y_W!*Q0`3DN_-_>2*cnfhYSUa;k=)Ii57~<9jE; ze;jMBM4--G|WXQ0#S z5kS1ppBD_Sicy`9Ihg-_JXrZ9K)&rx%!>KV{OYzw;c**1Cv2n7(v?Tg-bR1+8P9!e zii*R&Mw|mx8^sU9xLcU1+~MpdNekiqjaINuuB7>E2~&38tn;+v zq2izU&yKKlQWr{pca$nOTU`2$h1C~I(tJRg3%wKGDc;4Q|Ar`Sr}PW0{k6gM51eGp z*?Z|py+2<4Fq&;@b^|>V_kzN~S~{zs_o?|x7oxXin7lIX3>)LvM6PJy!k2uGg|RWa z1jbo)73p}Thv0*7@6_b z5N(yOq2&ugm9exmvGX`FNf8vF;W zIXnfWMM$d&;(4WeRGna-(4;DTLt}!-`dq$?j0Dr>U!h6cepq^m_PvPgCW|^s#V>fc zWFI~oc@iK-2PU=l(@slu1&ULZj#V1KVsBsk9C!x_FJZ%si&|65P8@F3#lYyPm{fQg zTgKc&;yk5G#qxt~v2Ke)%<1SO@!x}$bY4e+;!A}`aBGY^-_)-O80Pnf`t53CIGrWr zHK>k?A-1|l`&k9lkcFLGV3I>Bwv0~2J;zP}VG_G7hbS!o3YU{COfcZrf7%CrI+Acn z5LXipU&2jGtknI$RC<58b-{!xydrIdXR-p-XG=AQge77`e>$J8%W=$`>;>s@OBszD ziV|WV%zpu-wS8MKX!$yE_WDfb_uwfOPOJ`BkNe7$P6epIgY9+5Vr!&`1*)aM{pt=^<4Qun<+wEohb4RhV=XP=P%3C0v>P&fuf2_L_ zF4cp*J9pyJFIR-ppUtQ3#$MHivEcdjkn|Pm*Ka1ttH8D9lSCGspZ4mGt2mc#O>*n8c3LHb4M4Nf^E zC*<5y+6x~o?#MSUeTt-UNLSetpDaZ?^2kL_m14`(HAsE{`d_WhUHn@qu4P3XIVVpc zNDl+Wx2lGFe?s^gUIX?SEC%K2j9i~-*UY}mC}-I5q$`Z##xni(Lzi3L%Co@8y-j%S zgr@v;x3%5W{1|WPB&Rx8facIS;!WR81vp>xbm~cvQe5v9f;w`Rp9cdZGQ=s^i@KPvk zb@|#1!X-;i*uf+A@5MnMmmO1@m2eHtooGOw@im*|HxZZQ=wSbA{lRzEGUXRpXjU?i zzoMW2E8JEM6P^LrVf+?$3Dv>8y)LL+Fq>%4Z?kLKdv z(_v6JF&xN0Vw1n8h+7Y#@@Ws>__5HrSA##yF-SX(VmeNjuPVnJ&LEBOR3H zBrF%{v07|YHU!8kC@cZ$86f4O(w$;w+F%;%R?=fe5OX{XYmZN&_b6wB!w4fz8cNpE z?FGaicxbz=(i(ViR#T~P;?mU|pnKHb!bH}nUMZvdL*hoIxy6^FAU^-dN6nFd2>kNx zBTO~0CtP|C3%8{}<3BnIlW^3=P|^DnC|*M1SI|E&0QTFxa#rIZpFtcMgR@@#WTpcJ z_B6>=@f7BmiJR~5$4&jxs^T?ii&Lq-o};{xtMCH$tU1Oy z+Ib;qA&5=@=K5Je$KluLtlSOs-b#!nG<%pv#qQOvWBf(qtr(qvf-sL$oU3Y0^&ga; z)2rAkY7D6aijQowGZG?v%7{<1wWKpy!WDD07EjpUsiESDv$5hn_IF4jC|^Q;8Okr% zlU6krJ|meaJkW#%c-odA7L^l5b#F#(D7S z)(enyr22hO*km4eA3pBr2C5%J{R+I;qk(9+?;`rn`iq8n(QIY4R=i*R_i&_3k7LUJ z5MF4AcjWMQTBh<~WbS<+&A}+|wMyTO=pThGHhpGqg6<;alqPt=HF0E8Z)vl8HA|1x z;P}PHNcMg*ZFS9`THu6>4z$kmUb)yd_Ze-raJBhjLkBi@@TPZKA zp?P8(u_qo+8mRT4GjkvBDS$Vrj|tntfyPU9Li;(MnM|DgSoj&fg)c=f@x-@M(w;DEY`Iw1eg>obhgZ9Q0ck_Dzi^DW*cwO!C~Ref z8O}aN?*z>SZ0pk)VorEe`76csq1m@^YOSg0(#ZmZJFQVHP8EgzMKCOWx*+bSc*Zlj z5BT%Bt!UF{F?t_Ah{Z;y zPfBiU_B^VOur}W>s9y)2cl2fvK4bCNm(7CmSK%6FH!)UzRFID3$4{9G%6kzJ7RU&T zGy}&^B5yn$hHPGm#0{i1W>)#-5lia>=?}uzjl%eC+Z+cfC3`{k#{145Rb1 zg2%vVlPjt(1k_u@)3ckg9!YKG$KJ2#dQExX$CGeC`=+!))(J>!!?Gb3K)p58OBySK z`t(7gSvjOr3{d$_;$R?et@>w54mq#>uXMrAi}@Csi+y#>{41oFdz_yLb58?w&xk88>6V554Yfw)oCfOhT7 zXzJk)f*rMgS83zHD{it;GyX8@wUNi!#>j{AC$-kV$*{w0+PO%0`#ldwE*nBzod%@I zQB%HM_0YwJ=}~m$)nb{nXq$la6JqVKDjY6r^5{NWFX?xApad zxI9OBH03I3H`+JeWURC{@5sB8!Qj3683TB@M>!rCM~Z7=R5V}n%_1t^z_p>&_j5$2<@X`_PptUU zu@|f@AI@hiroA-t({aMXp1fdde|ieMk!8;qj>F*tjPA9uV8X*w&?_ikTY2LKjr}s~ z99tV#ef)zgth#(Nv^h&FxuU7zn2n95Rl}H`eI$LJcwVb1R4$$b3u?a8j?SpXbH@(k zKf|ZNZKEIP+Ke>a7Jq)S-)f$G`<>9U_=58;Yyi`oZ}^(tU7lg(CM&kpgwjJ(@#C-h z@@M)=G~4eYO$v931NG|4+Li^dI4~Pcf+P5~_R0Ks%y0Pf)C$+V-qM}M1sRLoxW|zj zV$J&pIL%=fdeuIT&9>cyt;@EHyasFG;vElemDNR*jUA8AjwMOQ_Mb&T&IakkR^jcS zKuu5yVwdYhxX8bUT%mK3#UxCZ>dQPc7#59~2h3qThL|_!MfaX!T(7oJ_d!Y+wAvs^Wo9s2WG7rVma0DEj`V#>AI`S^N_zBD)$C=cmxhiBuqi|6@muxMon zM)%DJH`89yxK$28b`#Q{7%@B@ zf~?1*XTeb!GS`qhF+D!sH5H6!WMWCnJ)mz51^UGUB;6COP4kyIjZNU=2TxS#zG zm@;p&P@i}Ba|Mpv4*ExRXI}dUaW8Djqk238^C6e<(R3r;)@eBR9QqG~ia()#Nh*%~ z)E<6z_2N5>0_C-f_u#-`3%16`OymcwXWj==q_0~A4%imRO;6V46mw3w0@mBpWHW<$ zg63FD`OI7B)!~{2&b+;=KE@14R`&%eckMrI#$M)e{N{`)94o8ItRBqFRao}Vk`CXv@{>!esGq|IA0FAW3Mpx3lT`UBn@p=LT!g| zu=<^i8>5ZnAKS&8>a)P0X$hpnwvz+$_OmZv>&cgXE%|88Ao(?!o|DxWj|MB^HJ0v` zSm*pcN%<{&d~M{4uPY#^cy?8+X`IaSOf~6z>WA2RD+zu2+|}F;dd@E1IRhrabiUa7 z5rS$4t^chC{n_Ji+oX?J$I3#g+?jT*0FRfy(wu*A4iD(L!HLX7d{>#w=(7;Nbi-A? zN1Ss{t;RsdbND^hhmXCxSn69Nto-p5A5QQ98|Ov%?n@te(KZOpcFg6+KXg`eB@X$X zN6K%o@$lfI-|oddGdGDNwFq-h_LR}t8Qt|u8uO5#p15n;G3XLb``~|BDZkKp2H10y zOu09hmmcpT=Up&Ehq7seP4|VB%OB?ZuZawF?9B`hFJ%iX^yI3KOJS*T6YZ4CL&0dO z1gaC*e!)RF+a3^0YYdd*;)S5=v6&~{;HJ(s_?PaJrBSnV=@hjCtu{c3_TcCg6vR z9=Q0BC*;xom5WE+fv#P3IK=`gD)wmnUY&>1z;obTC%j;6%y5O5yfUJLG`QRm7VV!% z_v(&>WjL>8Bri;Q0&(A?z{98~_zpMV>k>Oknn%Ks27H}+JU6R+43z7_c;YbWA9W1W z*c{)cYes!(APIl4MtFU$>UdsZe~QOX*sy3RFQ~51&oXCRP;&;-*d@hAtm}RnR*t4U z`rf9)J^T+SXMk`S>HZ+s*ikM$+M3^P6^{e{`eK&{iP~Aq8gSPTbG`?A@FmCV^8%eD1nmlm%z3kog>g|Bb=-c&eUmD?AV&cR+MGhjn_Z(b67SyW%-DHGBf z)1u_*fM1R>!bqUGg)i<0==pAYq`IRsT_R!TK`|GTeI_$_XmpN8f? zdLY$4L~NJfHlvfuReAg5O*P-_&7TfD&3O&o(_sFm<5?EzeUP}QE~nq;Up*T^nzMm0 zn0FfXoQuY|@3g;6zvt{Iy&q54tg3fa2Nf06N1^A(Yd^G<#zkw{^1siCClYZ|Y%~16 zVKdGPCQoGb9on@H=9;skMQ8i}s(9pLM_QL$`It4|G8hwHFurJJG{P(&B>uGZUJK z{55!BbbDwUa~8FKEE&xqeCRcp6AvPB7C-A54w)f^v=8!1q?jnKXXhXEmOG4|V`xA) z9{BMUU;n&>)56v(?3C?G`!lLjP-B~F>!EqnzzWO8KEjkvC#cqsI>*PnVjCRqf*Lc` zH4K2E=~g6Wqh3lp;q!uDr%)@)cug;VU_xn{iJ{Va_BbW(9ImKxrr zdM*LtL72ASA`s?C!WXpf{EY=Jnu@v|Y>Bf@i^yG83S%*HS0i)lX;mpllqvmTq}ZGqwM`zbs^AKz_Axh7%~-YVQyTn3tJJH?r(Qf8j{ zQSmE#se1?}bl;Cb?HuJj|EKu9Q-Fl?Cs6T;&O=MI{Ta?KEUgf~R$nFl{sKccdq~PN zx#LnC9M#RjvQudVxiuca-GX;`cz7EqJ35fYPHW%}8VkRk-AEUa{xLV?go{8J#JiR3 zVWcD2mrYwmVXxzG)GtBnV$*>3-fBSc-dGjCm7^_SmqkPApR5N%y)$VYsG&S`Y^88~ z>@BI*Mj>lJr@NtCI%GB+{dNnBTr60muCaXF zs0Od4t%O-k?m_33bY0z+aLuNV3=BWa*5uTXo@c_;JZmfBKj9Xeqm1fM246|nR<`NM zY0j|eQGM;{hk;a!zVh0fx0p~7iT7K?h&O-o!E$5@q^~%GZbeqizeBOOeW|(FH|z!T zD_e#Yt^>7A3==i&yD&~VM{F~@EGednj}_KYzGt8lJ)b8XhI5*?l$BNsP+{7Nf$zmo zZ!4vF3aD-*{XUX5BYgWTEWh4oPbb#}HK)YcQt3LXRUWi-p!8op19J?|!G)6X5cl}A zX4iZ@@bYgfDd(}n!wwL$@hDSz(ei+v^yxEF)hLFH`Hm}gjz*U`M^NSPA23E$AL?Eh z_4gOm9*pmk<~a}di2%~CoH;attBY?4-~PONRF6qe@Kb}{2O9G9JG5uPd_7rqstLB( z(+qz)E`tTrc7WmJk@$T3V~ERWib2+o*w9@@DsI|>>XD#(yBjBMsv#^CMX@IYakB7V z{tgwVxm?@^*F!hrim#QzG{_PbHEhNROMo;ftb5&DkWK)_-!v{f-}MMwd}@RgTP)~Z zhNmNc2$cuf3$sAs9>@^PGdLqC7m@M^DgR{B@)hFt-2IT?x(7e5zoW(_X4!0K6iabv zTmyPXivbR>C7k|>?vpOzr~zS=uf|IM$d{WJW9tMj?dk4AVZ%!ue5^AIPrGlWcPl@k z<-lT~yvK6idu*)JD@J+7h@T*>9fvxLW3;5(8DRvpruQN5o}s;NygLcvKQ`lrg;x2P zY4^kNT9f74;&xZ@H$6``F*B2-FTpA|R1#0YyGm0qJ>rJsXPE!(Q3}VzFXz4J^YOLF z2|SE6Mse4|jw$WgV$UW)_aTU5aoX9LFfzL?BmIEO={y6YX68V8RXmoBNdK=rs)8wN~LL{&Z`At$S_<f)4NVTw{wNa; zh=TVYfP4;`jkqS1=B0VmywvS1$a8YiWmJnNt8flxUDwPpy)BIMf8yA3>I2i5>dZQU zbbrb_iR~>@F=N(z?$qHJx|=yj!Wt;d$W@+K_zqt~^Jb~?2=gVq1KwhU^lBI_vWw0# z(j8Uzs^ajpo~0~1dWFsjY^=N^BzG9i5BYi_)t(sMo6ce$ZJ{OIqVfS0Clw}5-|0<(( z5A`Me42GP{XEm;O77b03MbWc%aNtJ>oK4=TyoSOO%_awp*2>oel%IGvpu5uNiYK(e zQzsztC@g8LLmniaJVb4&d|djxpVUWa$ek`N2lu*VEQZ$h=a1}%$x+4H=M$r#!^3=3 z8o}|cxrqFB2Cqg~qWb--i-#0so}Pf@kD%Y!JZvfVAkHn(=PXuW-$Zk&0th|RSZI0LN+ER|U63X)s zX5g2^67V=O1kbzaD$d8qZysp-zJL*T$~i@yps!)J>PJB6ft}j)j+t8J%ZV=p^-c;V zyf(z4-<%ZJt9jx+`YW|}=chp1@`FGg7O&mfPWmy64KQdSyLJtwo=XnR&ts-A@nK2T=NIq{LR%ZZU1 zrP0oZ-&Hk49``9IUu<-2JoSAFwMt9SS{gY0dOvB&O6{s~E49RTT;ZFkQ9dZGth7(i z(kp_rB2r(9fmI%RpY%nLE@oxM^qitN6iSPAXflG#qCu%5U8E=F7N$8tTE=bnds z!zMHG%b+l-a`7`T^WP{c(h#ltJ94UBNqsVk*Hy$FrsU}(30JGDegM*EGx9i+d;oK7 zKbHqCdJRoRbw=V&M)QmmGnE@K{AVMiSc}#hlTh`}l(r_WQh5Lk_xZ`fLD5Kf&xjWw zIk_RY&^|0m?Bu39}4ji!YmN5WO|81ZmA za-pPL#nQ$$-2B&A)b{O3aZDi3)P?-kN3F3qLwgFWM$1hzfN~DHwM&DKLv)D$MuJJ* z{^GpjG%ax)WSkEG@@ah68CxU|&eUgr|5%4Bjr`It3#oSlo$s|3Us*q#+R+G=rd_U$ z7eCh?L5eFVd>}szy-r$!edqbud<&h$exU}s7iv|$(S7^C&DM!aGBfpI7wWy45+V$XF^Hv07f~Brq!Q`WM+p{ zpArY`VEu+XM2bDtSGQxTALG5VSew$xRXFyFg7&4C3Kn6m;$$T4z=$&i=?(3ntTz}O zvX^N;-C$)?a+L2x@>oFetkQtwe^B)alvXDH4oXiGhcm0J!^)e0@8^N!Azi9`?)?^Z z*!h?1Pi6u6$e5v8k9lC9SHz=v}8eP|t{u z&WM7LBhT2N?biBHde!&JDHF5NByNTz{=F9W(8kqUo3+ z=AV4=nm64s>cLJDzOK*UjFBHM{kfMu#1t6}@*;&-@wGOR!-P8Y|{okuN?}X!usA zAiUM-58U48z$Tw^;Ufb(qwDHYO?bm}k^MbetjJuCztWBP=mZCO+_VDtjU8C$;!7ca zSJOVx8G-NoV@0i^#h?kOk6s&1;o8LnZLiDVcK6PgUF?Ve`bTPJ*ZY`}b zCtw*|-4P|aq)+Cjk`3YJt9P(?U4NkYfr2HIS(^cN;D2l)H%mQ(O<%Z5)@cq?_da{i zS+>3g>C%jou;l0sTx?KR(!9y1OKM1uTX(v}@*8;Bu@rKryb<@0)!@r*r|@4MuORW; zH$h(-7S))0-uL8tf>Th0mf(-WTC{U{06TScIL#;i3^Cww78<_0Up&&>V57NGz90St z`##H(*BfrZx;frB40>SpfmIk2AH!azJ%REXo-*uW9hsS%0z6)y)BNGel;vgyBGdwG@F`ZBh*P$KtB2-+PzBnQ2vw?N0 zfZT_zF?dodnN~0#*3$c4-8GwJ+5Y=5ZLco><4}kuHYc&>brXz=+N`<9@8Ha{A2ELQ z1lc3(J-l<8fw!-mq8y5V4bSX&gX;Eh>{4@S6Kx0XcJ)b{n_z6%XB^OZi;TIE1A9Hn z(Y_!WL;R=Um7*+kX?zZ+gvGM(<8$deg1T_^jRWqjbcg&``h5898oVT&-W!_Um394S z$OG+ufJ0e*S>NLx&dEEd4H^@HQ6rY~XV3Oxm~k7vIy_I@o)V>UUuG?xFKalJY1$a9 z<{P%wgvTvr^Hh4y*eY+IG)^&*tA<{}rKNuS!iFB4>Vp?7c_DIAHet2p_7Lhg950lP zlhf)mexOqdx3~0=_biS>PD(Smv9k>h&HTYSY|55&En#=4nYvGYzh@ELZs3nAGU*vN z%~^198B%?*OG{1J*nl5Z_epul?{uGvf66Un^ptRJm%Iud#NGt@U8H$qH_Q#CN7J7$ z{XYZgk$4P~*5^1kd2$`gPqe~*uM=>olP>t#hGF9oCOpLd73*Ld!2Z^1#SP33;`8Kr zEO^p(RWmGbb2nVG`v{xK@`c;pr&V=#to{keO|LDfUhu{WPb_^9iw|pt9bQ?-h1;0E z#1WALVA=3Bc)B1N9uH2yTTxk<(7Qb+oDn@^i?N!)E1)_?x5@6Bw=T2~<#j9WwJ{k^ zCC6cIdNiKAc^oox8}ih%^>~qeYYZ*g3{fYK0&p>*eO;UZU_{r^m5_$7RHnLVTD@i#Z*X|8smP;NKG-_udoByYWsoqzi z?#B=KuiakOwrxFOHe?&#xslJxrVE(b(O2b{!0{lDw3$YKBC1>&UD5~e^&hMpa!^k0 zUrBY^fpV}mpE0Tr|B+%Ps9x~2|4fcen{gez-f268n8GV24C1@Kz5w(7i^Y+! zMC>qpm25m>9k$zWn4S68nT2)e1twD-!>aVR(9$(cR5a@#G#83d<*-hnADtiXuJE3H z33Nfz@_Ib!cR3qmFq2nbwH_NUbd)~3S`t1LW0BWsq;aab)2KR6ZZRFQu5UuK2j!Z@ zx8K4n;^}mw&Dg)-6Mpq<#dSwo0nIzy9M_iD{!>%kx17ApQoeh4iO%t~z?n_WWbuL> zsB*x+T}yc)uRES>@(6EvP=ATm^Y;AjwW>dgSEkBsxoklur{AF*H04xdTzpFgziXqk zggxNjE(;V6e9^4|l&@%0HaxN*l;y!^VTb_cXM*+FJ%2tzd;8 zroy5F&xCW2Xt=SWh_wj42un9;fUt%u&D4Brfaqep6Ut9)7o(=pI;OKp%&b!kP(2CC zFO}m!G37tCyD)sqJ)t=5oIxV=I&vJ24SR&|O?=^tmE~ch!#SvHx)Pn~tPK_0PANYH z;i!iA7wc=Az<*u3^Mqw6({!+P3OG$9(=sVqy_o@$-sY5tvBJyK5Wzo|p{9bYc{`9$xbU&zSllVgt zmk|fN#HcHr&!GLyNp~P=D6Pr|-=&{KI_()Z-lhoe-N_)mHXoE$e(h`~!?*as8TX-_ zxJGswbXUy1sfB9J1~8GHpE)`DzjBz_d6Jp^$+O#fshjho+~yg#q>AUHn>PS+Bp7F3)4biYn}@ZLog&Fw zOcx3ZsU9^utkT#bTF;R*I0lUa=Az-XVeGs47p=c8x@$xR4Hf)sqmVb%n z>LqwTeGXo&N`NpxIz8_wCpa8J;u@IIXg7v0z5&FQV%px7u+%3VoH7#`=_#e(Fs=O& zO^sa^5Ku1)zkPjMrTMot%Te<|dOZR=>R;4MT9}XiuXJgxS^yIMF~$GwOH07+&?c~M z5vq8ZbaZQ=*kOLHt!kd}d$76u?A8rPUm?{sP;TLwH6f_xYSD~d5V67y2A#f)=jR+_ zW1Ei<*_nT}{pbA$#Q#W`g*zgBn76YbFZ=cz_r+JJ@3Xfq5klcjyQ5dw#Z9@QUWo@I zKIWvErB2~J4b>uV5dYFyVOPQ4NIZ6F3r-v;;PP|96jl+h;sWb#g7Q(t0tTerq37K6X^idH#w_Sn5gvmC+LHu`J}gV9KQUp2CJR+B0m|(%D2X1 z^R4BCU+)T%?_XpmzTAP1C->mwQ-#bs!xQ&WFTLqGJASn8brIa6hN?SQ-^x!si1o!i zhr95oD-s94yeS^dc8~*2BSplS3E0AN_~ZM7xNA>jL(Ptn_h<>G zGrggQMHf!7(yAI%wcvTb2&RP9mFA}kG^8CF-JjCuA}Y!P(|x(5dJj}F4-Kg!b$r?i} z`zXzX3P&hM@aC%-(6r42$`yT1K9`Y35^Q%zBu@ux=nTzU??0;3alGrhLK5O*vTSJ(T9j&XNIQQXri&q+KEs%{RSI<4J5 zw>QfxZw%?aQbTisRO76q-dEV-Uy~~zWA(ZM=u71|Q+Y$q%H%NOZWAgtvy6(6fyEop@ zCW%NwMUoYg^qlLEO=Lzk**kknQX!FqjHql0QBu#jPGl7+6)Cgqk(tdK?{)g){rrBP zJmWs+T-SF!_kGS1OEmPIqO@Ptm=)G+U$V_(^3dZrzVtA&t)7k7&(m{*Z!Ki6ok!A1 z?=#Z)a`?41Sbn)!U6Y154;1IC+9#ce6K2qUhU=#SaS(J)t^t=D4FLKKP}(W1(+f0O zHCtqF8?3M$=ES|xD&ADsr?5$XTm)%;cc>h26;hlkHL708FNv+;17S`upEj?h^Ftc(iX-#1l~Gf`q2>zWq`ll)=7?1f?_;CSww!89dv*F4u)A8$ zD2GAffGS)doh`|40eL17`er)yOg?KLUkm33=2!juoVXdlj-L^SySC>)2kCMeKY7_= zLEeC^8e#*~1JKv&q2dux8OGQy^U>8TT|@c+M>W&q_6@d))K=x%S{qjY`F$RI;R_?J z&rUQRi_Z2!{V?q}GNj5Af=SE-G#`Svv}{7j$BTmF#VRI}kUs zRqfYu#rH}dU;1?oXuNQC*8uqY#*Akq+)(;f`t|jOMZYq!m9dd*y`T*`&NS!V9U44ysQC=|dUvO$P|W8F3Dh4@TmBp)>?#bJXuYs%q~*g&VH#LeI~Bjz`j& z^7_M4&Hi!qA{{ z@vHGuaqXE2k9k&JrD>brYa*$Blpj(xzzV;GD2~RM{W(>&uegsgCZKs&KtJEJCUV0tyW$R%!{tz*6>>z(us`tiN>~uaP$sS z{S|nvp@z7F^zjPHGuEazN$`}FCWp~D{tLL}gi=ifajJXmnY348MpmMT(5uPm8pxwC z@)P?o!_E{{zM@-4ms78^(cdP**AwHw%4DY+JDiwYsn&#znYR`Cly%_53FM>mk@^8i zb7(22QrV2kNd)ng_R8Z1DrewBExmBy{%i1a(AqT88%TYYzRT1F2#0ZvVLsAMWUwg3 z9_l|Gg0yyG_R6P>vOrPSJXrM`X_ZQ$aN^FPT*~1F3VI#49Xy?M;yO*)pBaj4h2hCf zxO~o4ApfGV+!L6Vd8doiS+0?EJ&+UAT5+P5xC3t;F;QLtsP2$5DfXy$ig@g_7xIS{ zAoV@m{qTU<7>z>9Qah>E%WNv0*FM(=NW0?KsToSgtE^Wj&9AcIu8;IouB$YP>M^l5 zV=}ZjSwT5fXF(oOQeFbxdM9Aa$d+i|z>?ofj>f^+49sn&C~VYFhQU;hNj=NrHf&*} z#TeycsPx3DXe}cj&znV9aevQ7DkEk44|=FR0?X1ky0+6oWydOeGCkr)y-=UL$QH3C z(}nc4iBy^^S#JqaCJf7;Rl`lzH}S{TWVky0p&A#uERBH+VQ0uA*Fx{dKLzO#>Z1b? z@T3oWX_~?g6xx!8HkX9MjCf4tqM~-+Y)!`%c`&`zRJ=d2lzpn#jg9VSB1-!ofX{H9 zkq%`kPsc#{!3-dbP+2Z244)V^8c8Fd>ose>r28OT)+d9!%q1;lHInjK_N=TkbU#xd z6t_@Dp&@UTPIZXnb!4Kpi6jnS%KK1%ibuUW(4CWcq*E$8HLE`ykK6dC=f*dXN~0P2 zzEIg55)N?kIvSI;_kpq{p!(1d#wc9JL0v=P;huchxS<9c-}bZSTKfc8u)Q|FvY`d- z0r(i`$O&Lg^T6Ckm%rZbEl-Bn$e24D@XR(BFfZ=zQ3LMdVLC%R-{7st8Cy>0kkKZr z!$)$hb3LBvXoX=%(%DZZSLu1mj$P`|lE<%F51LtN*r09;o=@3Ba*t6E~~L zz6)-^cqz`{DMsV%myBL(C&=Qd zf1zxn2ag_k6CIXzmOf<*rK4{xeEGm#JM~{8Iz`RkPK_n_1<%tyN&Q2ge?{c=_rYE| zjp4L+FaDw5Q`~fAiu65`q^OWof}Ik)^WSIr?t6#Nz|wnj?tdDpRP-!S?5;4@s5Uyi@a`mjHL7eUUO&NxxG z4flS&SRUROgDDI2pfJmV({;mFGh^8NyOSJl)Xrn(t{u4g%Mmet|1LCJcSXFt5r_AO zP2>068Ld|w#(i>yUDGFub?F7T=VT6Ex3A6b4|@e&A3PNnwyouVj_qaJs~XwuiwS#l z{3UKl9)KSCd73@@-C%!Aiq<7!8npkjS%&=8Xlbl)wA)!Y(liv;WlsZRyVGF$W1e&< ze+yn8jrh?Cl--tm#I3WR&Ab7Oga#3;68o23k?QGmt%HRZLgp7PJivtYEuMd}<~h*=Mv(PpJ5OCI$=s9M+d z?k?5)65qC!(|go~Q=Qv_$Cv|rUw|=RTXTr;7~{aJIkcs7C&r?0R5e*TYqO?&QJjeT zl+354T@nZLL-~mZHsbaR1hu~7B7$VxzrHwqVmD6fp}p)khZ`+9Eq=`1Cxcr5#c8el zWpwT`9;1mKwup_3=<; zA8gv?EmQAL`f(MGq&~)_7kuEv&0ro->o%i$g}Ya4g5~oOn(0Fmk>-Tf?iIGZs>Sgq z+i=|rdxX1j6S*#ZFkcFmd{muYyl~_j;oISh#y&R#opdAVo{0`}RbMA;Gx4zYdF|@B z?ZA1QTi;hxQIx{m3U6WCKC9u5&|xalwCFM_URAP8x@Q+m)NPtN-=inXsPCU_Jpn=Zr6o9C;Q3RVP4WD;+6QfAqzfE z+6Xmk(ESwQ#prV>3%(Ek4yyOGo#?r`*e9^Jj~g^ULHEg~u0++-wXU^7+YWbG{Yx!y z?#?>0X#Wt%yIu?E8o9y|x<;70b36JyTMHku8?$4N_QLCfllc5!&SHJhDzUfEIdJri zl#lb9h(TBS;I=cl(D0%&6nfo;N5`k4vz{7z)KyOX*NV@+^Bz}fB4N;#Y&>VHhgX}_;RSKG zVYmGOW9%5=i9DtsIJbr)Mg`>lUm>jv4s zhe7-vS4lX8E#v;8!dAtB(`gSHS}R8D%|~shfQsig(7L0qAnZf^YSp;Hm(izBtO4gZyFsC9~Z1F0>o zajN-c%(>s3J6SmKisxReck3s>XPn2da|8MEWpB}KToZOD0)g;_?(z0Tzo|X&=HhKI z$G;doJ%5Ru`^WGe<^QUd2~*^X*rO~fHwt^cUW)Di)Ifd8%H5%{zw?|IJ;ytT+!x zj!weMk6REY++h)ItIInPz;oVYV26~0sQB5Td?xksZG}A={_YneJqE@H=Ob|&Dx9Hy z#0AC|amx+|-tch|jG<>o$5~z#9^T=EA^&0Elp(OBX}KUSMVd1tjFiONQt7vWM`AQI z4o(>9ArnfF>P4e^E!aoEv3Knx;R?(;+R|fIYF9!1gkSWQ!pm{Lp?<+d#UXr>_Ofuy z;gWC|IvlP+80ZItDM*?Go3_0p`gdP1mHy0aybY4G^@v9U1nEV&efe^sdUxKy1VMU) z4O{sLXzux#+|iJ2GMfb?LX}o|_10T++3&reTEO$ybP1DpDjc9^KN1Q5vyjF`bMaYF z|BLe@?jwD!tUI@*RGQNyQIC`M(RN5_LN&gP?fQC7kp5$(9|p?F*GJ+a@8d}I&GeVG zWzDrmaGd=E_M~YO?lYi)3|#aYNDEP2R5HawuJtwC1+IqWBr!x9*6f4Z-(JTZOZH-{$>_BQdwl zF~w0zmui#y_2f3QTPPjP2(ys%qpY5=7F4fL{Sb$2*Nl1iS(L44!rwKDgVsZ9l5RBR zr!p&4zhV7L2~}8qz-^y6W3>RO#&JmaSNAt|_L6i868?#3uT)|4akk=GJYQ6gE$mzy z{)F{`9g%TJI#4K%%}AdsViqqIdDkOx`p{M~D7+>oT`c@6qH))YSV7}v-a{zgB;8JH zgzL`_s(RhKb*XG=QfGW_x)P}N1Zifpd|pD=QUUZgs-GX}2izLU&wc%b=7*V7HGb?* zZ(gGN8rla8;>2fi%BP(uo+hHzIV-N_rlsmCLm!=73WcXYKbef zfs3Y7{cdKKUp^q=mHhF!6u)0-$Wy94z*fg2i1+9o!Wt&XN7;)zfk&z|3|%vhc{pD~ zYaqTkcR#4j zOx*2Ls3BbeHPY+JEB_70e1n-H_sm<(JkL6$4}T)@r+oS-4USy>%9Jj5E-8c`tCO|Q z*Qc@OZElNOB~I*c^-`qQ^Fg;CE6>M?%Yn3*rqjW2px%T=Go2w~KqtO^bS|UX!Yj+S z3+h+mr!Pw9X$rlDLfqss_PgM$*csQc3Xf>51@#Q<4&2Cyld3S9I07z|ouKE@w8SmV z@n%#8D7+*;oc4Egh~T5*$(vHP0qdF_CfgOKUXZb-+lGaA)9zmSO2>T&? z^&du>u*$b8O}ADD@Mdsx)^}0|_QW+EmEN$=?JUWwVs*NQYmF!bT1RO4kIvEj@f64p z%E+JR8Tl#7ChTx|wa$uXd0ziJxZk~zpdLWQ(TYdsubPc&9di%dA>K*=HGlM3j5HCu zF<_XWc?RMh(ievX=|mjkwHv9g#1mH!r5|8lzy{?h(CT)ksMhWpJ&)ZNj^-Ii^4>iA zcM;k;dg3EDEoxTe5k@-;s`V;O#`~G!+1B+n>52E!^&VZrP3>zakI#oDpJ7HJJyj3z z=R5Nht|)!Qj9soO9>g@g`)uG5TYhYr0V*HCLkzy&%p#5LwWNZ)djatB*2=>_GJ(b_2!yDhn?G!D1<_zs&*J<7W8 z+gQb~1J;Zu?G~DT^~hb=ZTXZ1mzi_&^SJ7{1%IDzqO?Mq!bF8ht?w3Kr?U>q_otE9 z6y(1|bnZ`8vy8^7G6bZ#<0E)5(mb()L$5Q+M#w)feB*MSsdZ9%=HSFqI5=^G{_c-H~9JICt2K+eM&Pb zti-?X$8z$SjPQw5R-6`BP}!Rn1pvG8&SdH#88 z7_g>Pgzj#Ntmyz+S3_*HybB)sa8|Q@w72p(NS zJB_^MGeQ0gNrQs&e}rwI@Sf@k{=R=i|NlI6UY0A4EeiqSVJ+z#PIw|Imtl!?Z!pa% zTkAnjTPe*4CqwpV zsR#MwUeV}(dARadf_h$IBqY5)&5Bwy!wDWiN>_r?9!alV$X|S8lnV&T*2Gt{4XAj) zs`m#Z&cFl*13tI;93ai0>X!PkE08Y|0sES9^3lrUL&#q(Dh{mTIj{O$;VShAP_NPF z8FR{!NK^$v*2CjK81F%vm(v=Fq6S+T#e4Hgm2CuzZ!z2rRl`Y^|;c0m*0D;Jb_Ubf;|#ral`v2s-~#d2dgCuClx=)LN{5Z^iTU(HBRsI+4np_|~F$k1q zr)yAtm-g81SCv<)>-@L8UApRd-fxrme%m$e@UFAMgZA-NdTO!}&V*kPNPyfj}!xe@uy808Oua#}6rM0nQ&-AR{YuSNZkauGqfASyqkxZUb@S9p0d9@b^Q zLggLIi!ZBd5Tqd(VXd~=VhKur&^5BZkF#La2#J*cX;z-%oaSEncR@V@0f~sDMaYLw zWt0gkEa0Tg7=5mW^tr-hg&$N0H7JX`sOChHo>m%&^n9cAtnv%=Ofq2*Wvb_5ui)q= zW1!=vDISzNB586q)x5jPX~^3(k(8?P+T@9p-eMrVVm9->?n{myJabDwbGWf2FkvAVQ7zmxXX zO2@P-S8?vz{d{=904(2ahUd1egplMs{1p_6;}^U~(eDT@`uz!BFS3*-LrSq*&9*R* z?p=J~)&qPNXt;H1Gd97wTyi$gAa!lpue7&ZzQ z+d1+77X8F}|C)<|tH<#}!oN2 zsSC%-!gpKn(Ys8@KeKo9O7X=289hil1QEdU8k(W1YKRYvy-u~ zphY;odkwie(vdrnzYD$TDkCdmpnUfxcx~AdKb~>L8Ba=RO!qPPVk}M?@Eqpch?8Sm z&^?q(NAT#n_h8?g7}?^>Zx~!Yl3%#$#;=ZO!C!Riqs_Ch=0)BfJl1p>7EJyu%4sj3 zrPF6KAB&+dcWonn@Jkooc%qJ^HG%(n?t)K-bidZHQRuUPeiu$0Aq^tDWt+HKvg;Im zzT?`DWBST(J8ZW)@tBEBbY~8tdu{F7oiccjC-L4W1YtjlJkT`*pKS z_$ygQZqS_Ot7g`hw+~jwSsCGoaUko=TFOcmw8!d8OGNj$AJE$RE1e&hB2Hehl%D#w za-GQmoEq2$Q(|wT(cUNc_-7G&{BszqxSJ?Cw(pCyp4jAJM7q} zUsl|0B_}22z^BD}{A^Sj6oxs;sK0HQL+U;D@%>~q2AT3;wLGWxNhq7n&a6MASvZK! zlS?t*%&OlE8;LHae+|Hg_{AE;U=+p2J_AK3o zlke(+&P6A<`u$Q>O?J5AjTM&T=xo9lNbg~<4=&)I@g1X!UMAw4^JB1o#Wr}s+t z_2XHATP$2$7d8~Z@Bp*YpIf)^_ zw9ni5of0;N!>gprB5QJIC|h_y`z&iEzqG?zqjyt4BJszRmb>5wQP^>>b^OG(Y^vRP zK)nbnt@I&2^BeuNZNn8d^wYluRL6Wk^FF*;Y&^C#uOX+oCu+xukMQGgA+kw#p?1o0 znwKK{)}y)XS(vHnfqCl3u-9}3;KH*lp<}$R%-yhp)>2QtwmiY~FU*v~--l|}iGKF`JO^2!Z$0jS9^LDxj z+4P*kwsn*FecRG1t|H9E#yY#@&b%nPo)uzVqu=nXyMe51`JEA0$@1)S#dTO#iwU>- zllhohm*MG?hBEI|lH3=w3Um8kXWi!4l=(p6&k=rH!id-CwHn%}3t@%@MTeoFHhebhBLxhhEO*-Zdv-Y`yEk z=I<<=l++KdcZf&gO;)SdU(G%Zk-vO&-DJM`O9_ZL2Ta@#U|cKsXP_ZlxlPo7oL> z0m38F33cVC#X-{J+bA)qaWJahC!EH}t>@Tiq~9Ya^pk|wc(Ea1bgJW#x0pU@MN961tBVDh(Eu$gchZg+JAi=hK0)g#XPSRX@< zRBB98u7Sn5R6LZxiHk?_OU=rNXPyh;IFF5pPDaDeDKHva(ORBC2ghxA;_GSXa3vQn z_peLqw+(dmd{+1m3dfV$ZvpBN^zRfy-+xuC->N6KpKBwwvc_VfVXTOw^Aw+D{ej2| zTPVdt*kMBqJdAEBdhOgQh_A6m#UYrz*px4w{tl?tG41?USbSz~6<+Rw2+>trAMbTs zi^Do@6ogqC(odZF1s@GKF8n*$s{iC_`c2p6n-f=DM7RRaf9uJZ)cva8X%F+}pw`%F zTRwCh(H_=s4Z?@Nw&B!9y@7NM`&heyIK9$BT3c^nL7z|*d8b3uTb;4%_ZzCW8F2>T z>c8~k%O~L?qi(Qb^9%gasg8_!5dc)vqTQr}glR@n>7Wa`(~xji`yyaDKY3*v=`O(> zy*3Hram|X4H${xKq3TgoJw$WLWYjE}=H801Y!+8_z3NXHjPD3Qn5OZ*eMZcxeNfU^ z;cMVXG15{;zI0oP)E83mZDI{8PW7cNTj)UhIkn;rj+Uy%+2Oh2kdc^=-t%w3;*1Lz z;$S3{)*!8ky*fC9dq2u$T%u(6$=3vF3>f*2&fhwo%L?C_3&WwdY+uZOY1d1`G&8z; z@?IOe!2F^gc&Ec1anY%vIH*$#5@L5FX%beMTw2Njq%kWSB==iz` z)4Lar5e{1Lm|UA1SN5T^3XTEkZ~WZYg*)rq#B%pTushgC)~eSCNZX0yp>z%%%}vq9 zw|FJKm0aQx4okN<$hpr#Xx*ly_Z{f~kvCro;xA4Zg|lim;eG?{xxRfN5FcQRFDt}= zYZHj?C8{y)m`(Rj^Eq(uPJQ`%v6~E-@sKp=eqs3hnK-G_Lup}6{oZro*$Z;BuM zoDnbjzMY|PtZI$_CP%T{k$XUCRq7u$q*@nDwb>2y9b!Xo9Y~|IY?W45SmoDkkm4|* zbZWzp{=A;wR(Sv}3DSa$dzwP;@@Aq%{dUB|%h2MIktBSP%}==TF-5hY-oIn8GI=jp zOo$USSK!%xmEwL8Fk=kX?bA}t0S*Xrk)(y`9M`pc*|RilN@7FO4!_6`d}Y_>N5GSC zBRXGmI4+uToV`_OA9&CurVK*v& zPHj|qI@O}WNq)pxr@A+smWOKM#*^3xvgko-BlTjjrBQC9HqZ@4} zeS4G9yaCO*RNBPw+&WM?<)@K`lfM(BHyN!PuSvfj_#HYAJL>JkRW}3Ke{1`K{~mqX z7s*iZsObAcgM{xMYa-}(p81tHak`=O@Vx|m4jh7(2|qQ*z4X!GZwQj#fwU_<)qFsI z?M;Ymlc#hJB=)@~lrBAYHHax6QD<2nP`tH-a)an53F%aqa>ijNEol@m?=2v#S9c)4 zO}ySh%?)_ywPOE0OA@Wx*M;b17ny;^UDEYQ@({AY@-*^T9hmZ=+dr3ThX>?|m=}Kq zeZC;fmH(c8VSE4CNy1>H-orkgT+(Z`Lj}OtY|knlr}by6?W3{Qas#|wD)DoxKvX^W zwBf+C(3v;cdm|$#S+kCG$`ByW3MNCw1I-a0iwa@nLrBxFL*hKbt0FD&fKb}6=(HVI zp4WYSDNNXPki0{>mb47y+a{vo-PY?{@!)Z3jJQTvX5Jv*TLN=h(SDOJ`gK}8>oDqRj;R{jOj+AwNbT{!X8g?wKfVDp@`%DeCVNBdGesMIJwR-2xWRQ{;FO?*{O zqI{Opu8Dnnv0)ue$OFeP$~Yj?Q%_Rui0g$O@afP*AYP(9-i$e6G*h)f-kz=cGXYyU z_7t|9&hS0BK-Am%3F_QGjpXCx$^8!nH&N|0D%2fTf{Sz{G> z60#;|Ao(rS8Rv;Je{*3wn=;x(mW*&saR4JfgOz*QK)b7^yngQwkz^L}v zyrN6mH&r(U_ ztcGlVQyUoYZYT0p3<*Oed0UOjYHnriV5AW#kFex~11h5-jp7B*-=ATmRY)U^1k$r| z;xBKwGs;xf>J@;bK_qDo(z!*<$aIfJ^`M%c-I0gk)ATc}W4<3#V6h_5Y)@*j|Mo4#ya)+ z9#)7|uWLM0aXcThu!Y>ihCuq<3I5b$O%v6M28zlr%GkZVH1?%`5p1h<|kgdzT+Um7&cxEM&(n(H{559ufDhM?K^B z(4#>Q7`1bWC~V*ldHTH>X(@%dI5c}Ryq=bcv_^!B7d1<74COQ*K-VUI7u(|d!?83s z!K8tHgR_4y%E_BxwD(s*>nE&Jk1*12lxv=Z)%Nw^(0OyE-+<-cS9f#LH>qQET@m2RY3l`D^Usoar$|8R$?d`jh!%kRE|0DAsKzs*4;-vUZ2DvuG^ z%YwY@EF_HcP#TeHB7MF64SZYc1FjesLSD0`pq!QXN>A2JuvBZvz6RbSJiUVR2J1p` zwt*yH%*nT?%#b+pgGTjcvhR9~eGm;DnwQqAVd8sScTy!#A9B!w%2;yTG?25rT9B7_^>B`Z5s+Yk>5QXm>RXNOuztdjsWK zc=Jbl@*anoZQE2(_0_|7F{mC@xsl37-gocBsoscVIbR&K1&uT73Bn8@Y-hW2hXd&i z>48Y`!{z;SCjGs;k_vJvGM2770!IaZ#7JHu*~K{Luk_|X-1I5GMqX`)+# zd>@Q5IIUfGRLiWs)IrixbWhb{p*Tcc^Sr@pQRR6<4*0V1-8T!uOvnYC;#05%!qw?Y@<{?P=gCbdPd{XWPQX!Qiv_#OouNR`*0JXE*sQwFu*_Z{pFhU%}>M zF}&Ow4UrC3eAL@*kdpL}ZJ84zQl@zctxcXVI;+or2H(?^x0}zqtvH82S#N23y*~dD z{S)c>xzna0%&eO$EUC;8jrzo5-td#+RIhSecz-4jj~a#NrwqfItsBb#+v6})*A{xw z@6+v~8%j;;dYm*QLbIggn>IYHiTqXfFS~kpKgQikXGc6*%A_sEoc_PY^;nkHf9xn; z+j+;QqgK=lO8|+jH?ZGptmyB-IQYLp_ zixH7iaA&L|^d1rf?FMX;-tYnTghgW3gGdSWQ^TZIp0`MACgS!}h0Q<^%oUV(h zzuyjP2~i_9$Pla{W+dE|FYg^^c{Liy7N?rwlQH#0;;1le-f*dQS+NdOf4LTEj-*r4 zUe@*QO9-Pq$`;;#i;dc@kep=U&}< zJs2YI1!*?a^^!@&HkjnvUpm--7l0^T}}h{=`(;_iBM_H$=dRJ`?1f zA>E~I;CfAuJM*zD%?o?)-7dcxU1!^iH_PZ1V=#9{f4bH;Vr@Y@jYp49P0Z0o^tmlU zoo9fv=?K^`-;itG`-4kMcg>4~9kH!n3trytGtJ`RK3mAj62lbPDHVq<@aN1KJqeM>6XeS<9RnX?6&DU;HnV@?C1dH<1~ zk?esAJL-A|$h{YXMPoM|Ve8lr5*`He7E`wK!Rdd+yYSXXH~>LLR{TpVOIfYF4CB7) zaqGve)jGo!-F*B!Bp-U&MquH~F|2`C9Zd*s#;m4If$Eztjo(bY<%P6{sQI@WmiM4% zxbFC(=L|D!vwk}B>_O)rvDR|1M=(ZOlnaGRE{1clG^kk2zkU(v_00DAZbq1g7wbL; zHhZzGpFWxD=LYn-Fjh=7T>yi3oI{se2KZZh6DR!Yjdj3>wYA=bPd`X;BuK)qo$GK) z!FEnhQea%@QQkmT52;W1ew$1=FxX65KR$_3@iy>givto~&{-2d1l1&udfSQ<-lcjE zu;zMA`by@uS!{M(!?XU~2Y1~F7=Gw2T+AH|hnnkR(;jZ3LDUhu&2u@up-6ossHb=*cRjR< z-pX%OzJQCtU3fiBDQjPCIqF=m&hHlVfs^-Y!g>~kAHzDynG5E!y+7U&Rv18;e;U5& zxs_$$QLTC5Q&9D9pJKpOJzt0(2U9i#Va?~&vG%})a`tCB=kE>On_BFo@CFI<;kVUB z*cS0gsOx_1I#oWs{uTNs^p!RreHBLF@KoA+Zc?kNYj{*@On5&=T042lUQ_I45alHO zT3*3OhaE`Q2K0TBdJw*a4v^td_wmc(L(pcoB~Q9K4_761!>_yFc#zov)jRfSoeu6^_OLi5k+4}0X$@qn z3xgD1@~wcZl_@HhArh#Yd$ZV58Gl zq`E@-EKoe;VE;t@9q%j+$NtubS;vo);lO|kX#B^Duin>0`b2qx-GR|`%^&a=omIAD zMtj(|!InGyYArMS`@*bXYt@I+f%ccN7(5+&4qeJ=zHqzwcW9m1PMRf|OY@ESte>2Y z3UiFku9Kx6SE-*I}Y- zIY95{M?P6|fB$OY$Aqn5ICCmJQ)7rH??oW3C;lqkLHzBnK8x04Ak9-gcF+%2ngOY% znEu~?XnWpYDz4pd?1SPO`FZa|JpI-~9?HGKh+A+`Ksw~acL16@2=mx4=ocW(mm zlA!q)>8;-*@fF(lN?@D(Ysp6c!a+a4Ks=bB&$l!U6Zia#A3G=o`zKu2{`qtP{`2oi z{n3N3z3>B<(EXyO!-jx`O(1g%+6Y&g)t3XuUt+#Z=E27PT%&N8bQdFiC-y9R1^?y@ zf>7r-V)4jNu*l7czmCtOJps=P!>b1%dc{K=;7rf&{&*)kzcr9I98Rk`L$hCTN@sAV zX&d3ZE$wYYa}{b}2NC}~ahdNQ%#K`xdQGll-^2{4`JDEB+pdd*=jgXA7q%DIpsY3& z!g4M_a(Hdl^v?l2mev*&&eK84@`KBN;IgPMIIP+SelMHLO#Plhn^*y)EzxAf-cBPd z2Fqzr={e#?SAnoWkVavb9~nx*4Fd^O@U40Er^$5nr&Fkd$P;{Z7$9AI3# zwwi+#n~C?TWXC)Fd?$N?9RoQKqRVoAwj z;)?dEN@&RwJSV@!1Cn0-Ov>FdnpGi6y#y5)8WId=wcy$tI zF+;Se6OUKNglq1%u#l-;tHa0xi$xRjZVHEF=aL7q-=g72+Cj*KN<8k~5eKISpwl!H z{<9(l9_bHJIH~<)k^xhVHb|p4<=RbkiWq4-2;AF7Hh*cZRhl8hZ!qD8jwI|ss&mqn z=A1AP1~?B?^~3^)#ITa56;*RO;a3*zpS@g;2xz7B8Sn9OFLQjo4#(IT;`^i}nhKlg z+L$57;mnK+tkc7uqOQ27`X9<3)t1jkHRTuSnX|>SY`DL!v(O!V2Af^h;OY3Ic;xY1 zJUiS@Sd|^X2D(8&bLvt0=`6%N&m}GLTKo(iiPN5DFx^9!sn;z;Z_g3pj!8SE3q)L~ z6Kps(7g|qhT!n9>PtvItLHAI1c#-vwvHOGI<*YoR#?Wm4d`=i734efa2Z)OV^)w8h zqX(;9`tYLXuK-*Aq&KA1j`$nww+(jiklk*KEkxTW@ep<<85d0DdfIxz$%-ZtSI^-mBU7;$>NP#jj) z>2myys(C84r}?mK9bHSouNnm?LLs6B(3%wei+=r zYnCgpPJ)Z1bw=t(p)jC<*LI}^Shp2!K%R{Jq_1{SRzIwEAP<%HT_5ZyN!!9i(+W76 zGmsHy3+1&47ZhKLNBV^x6E?kL>spTE%&ms@k@f<#rhV8~7HtChK2H7+h?DW91G4JN z=!_@QLW3V?v8z#w6rL)b2ZSqJKPwvdPRT|!C&aH(X;l5cYsAQu#$5S5fU3B;o8uHg%AV>Tl8l$)LDSX(zg7 zoZPw>dRV68MB5mVFnSH?)jOatUtxN67uwO0c$TE)`)a>L%e_YA`E}qPchM@YSJzJemVRr+cf!p(g#9a- zPt-;H6Z9MuR}UQD4V^QuVsXZCcu?WQNy{>&yNTNc`BLJzBUPTKlj8_b{^jnZ_efrX z{7)f%-uo1zi>&#gtQ=NjiV<;_WXIBylx9WUHNO>paq<|<_uYC>T$n?@seQV5f>A9Y z@8!<&t{8J#6RGk8npZ|R1m}EO!pySk=-95VB-|4cR9d~|#<2+60R8r(QwsjGLf|>`dcY2vkEr z{EX@~);k)|c@ay1u!60x(BWGwCn!%@)!P-{8{q-8p}&=}lCLqOf4czXB0M|tACfL$ z*DQaN9^4`b%UIxZPp0yZ@za`+E^!y{lfH}B@m(3^HDcKeM+{FpNItrmRQ#oInQ$Gx zTQ4L}UjmfhP!`dc>Y}Bj+Q+^&dyxDUTN+%AK;re0N*&dYe>6z?7%nd$Ag-o+3?8 z?+F9j^B5GotwH);d^YDYs+uD%Q{EXVC!jp278+i4 zATPgBX=IgyXm^`c=agZ1bj;Z+25Ej(c{9?ToH8Er1LQBwmk6g#g+ROn3l9bX%?~@~ zRRqHV?WM|nKAhH-cTCP;#rOT}g_0Ad z^1^XrNj#u*Ui!4%t>kUne5fr42<46b9;UrriX4IbIQ$LtC){*Y-U%)9->Ym-=)NDO zw7)z%JWfM6jwRa#A}ohYE|xZB~3EL@hgkWB;g#VK4wMVBcWTn zCOlLBs;2AaJFxC)4$vGZyl1UD4+rvP+7CZ#bHY_z<5@u**o#$;IU;Dz1!Z1B`5yI| zzsK!mj7@}tOR;t5yP!UcGGnA#6_jTJ@j1*5YC_qAh5A0WGxo3`PbvtD)0IEa z{HtG;VNz}YgoWgR21sG#BBzJe;I5zVq0+h3Q$X3CAk7RNKRy>T45E}rq5RZLlHP!u zTXqnK%n>Rdiy7h!QLbH-Hxl7qePHu-Z=SKxO6gsV%7>^&kn$I%G*@)EAt(I~^uK7+ z=`E|Qy&YC1T|$-75Y7SNu+p_E6F{mtJY7>?My)DfDvKd)hTk~i5IXm6@0326-X)rm z4wsaNP~ZErh8@2tTq2+UoIaDDZR$^FzJvyPNb8ZjZ(?VlwGouTKbNZDxxW$=q(=#lYM1G|WCbY$S(Qz#aP;r$j{LCqO)qVU%}&N>Cw z;b(e$7sR{T@k!?h<88_F)>8VP_EVzgE1H_7sWFnSQ<<00i#$X=^(wf;-vk%hL-2g? zKK{L2PhOtg29o|QVGAyPK)apaVdUe(@U-3#eyQXF_%2PvkRDTT-%De$YE6Bp8Q~(W zEqddH{Y-XTH9&MrZ^uubN(PhDR_VSDqge7%bBqmd#J9c91johe#pCvIa4#bo)4Ntw zy>|Ojx?eAMhCFH0Qzjj6AY0)c7?Iis3I-l!{Ui2*-ksWfRKsq}FnkbXyKTgkiRLiI zWd_U2tBG$O)ZzCklX1yX2RffkK-%SWbe|HzXEg}~!^M&KujB;uHlX`X#3{U9mWur~ z*I~iT+Op80JwCV8$ZwY&d4AziO~A1=u>Yrv{O(&vW-NIu#`<1kZZnYgvs35E9ez*2Z*mdL zJJlQVjz^-6!(6%L>MA*Ty^Y*f5QGiPTp%kif_?Tf;5JJ=Wt$q~py8P-Nb?0L-+J-a z$?iB<{}AhEIU6cA+Vcqgt&n@ykZ=Dz9rTYj;-3yLlbt^rLKU@_5)87l5w%r#zmNoCA-g9pO->DgW%(14nKg%GfMHWY zu)F(P=4$gEyR2y~1Kladel&|4K5@Y8w0pSq?jB6oG*>3Qrr&1AKR~Jz5qof~?B;DJ zwHfhPmNbIDZ0e_p%h5vlNN>?1^Q>soq>gl^{T1vy_i6{YwG&;+Gc?n61R9y>u&$9l za)em}oVvR!@9Y@~5#16czw8Ib`+u;$I#-eEhAkiC13J1Jd3@i6xT?Gb`J%OQ!Ns}K zG{}y7H9x?*w0MBf^Eiqj7JPLE-J4c9SZ3Jii{_^{3IE=Cg6d1w%*zpe-aDmZ&P0eC ze;m)d*A$IUW<@5>*8 z`~D~Cy5nm8-Z;@#WTX-jm6Z_EJsY< zn|MWRUui$y3mqP;l`&4kp|$x2G5&BheoKEcSgah$H>Qlkj_sRc#1T9Bs>M@QC$p9u z>(httd-h8#4avvTkAhLve$S_-a&^)_@wUS!SW`4c4DqW_rU1hP312;ed>#*G9}=cq~C+cVjXrY&<)E97DLyrR@g6Pfuw6?9x+#-<%h#E zW8oS)=l2gTq*^0q$mqV6CEjfIOu54=I#-Bx1m-eR) z!m0k2vQI~ER{Y;h%<@TMOFJ!r|9(b5!MM3F(m0lf#jv_&rF9!&z&V*Cysh#$NUFQ)xK+B$@`^z!C!X4xU=!m zs`~af>CZRS9f`LuIPnKAUD?&ZQ>qqm@8s@C^$lq?^e|%BP{Hr*1nLz$AJd#4|Mwl& zh9&XG^O4bKP%ST!lW50q_lRzUTHi?6PtTB|2r3+bGMlmb!gIv=Pf`SByMkmJ&Wn?3mgM!KJ| zNMwO+C7o-!tXMp}V$Z_cZ-8wxL-6;NI4zBhG}4`qXT~>`{_$P;&m#wKd~z{S`B}n#qo|SMK3N;FB8}a#iD^QX-9!EdFnpjBQPO{cfzl zo$1yh?{`gZXVaN5dK|w$nKE&zQ>tJ7Ib#PX%%Of2T_S60|D1kBbA1v_UsQ)?^%CKX zktet~ti-(EL1=7VTlV!IA*)^ZhT-jXf#%yBF|@%I zG{F`!ZSGF4=6i_8NPZ`6na20>TzIKzC{yEV$+clEk#L7_RF@O(Gxx(r81=jr^+_;P zU0)c0{rF<289~n;OWLKr1hh0nwzMuM@+iN zuVdQEHAN4q>Ot|L(T8-nk+C1C4q1M{AsEJ!@VCoR_kipG}mC+wFOuB zdpl@5oR2vO<10r)Ug8P78ImGmf)0bzW)mDCo?zgWTrKGUIq~3p`MjHx+|@f2(pM&m z`bS?f8Y`@3(O14QvBgJUYbb1__1vNOaq1Yx(!s^^<1mx^?Kd)s7J-FooBzLw#|5 za2<91xZlV}>zjTY2x}ncN_*j}x1RrkJTr{-^J*OX+wWmPhwEZ|C zqCZd{;$qt&;tXl+0l!lj)dah;u@K7)HshQ)8^tw@a2W|J;QZZggptF6=9Up=vc`H1 zk>(7Q&QQIk-ovUe(v@@>p408DZ83RfIBd-`DWbUn5gZ#<(p_Z|7UjV;mtHk zSL|F+O*RP{3{CFE!=8u?JkE_|kwbxNPnVfhIG0trR7>0?Nmt>{1#59@kG4vu@$So` z!EEzRHOA~`#4c@Xb6py&u(B>` zI|p8anF#AguChyHkhmOg#XJ9PfGs>*bA!Itn9;jBCw@WdQMPXHbugg4ZG%39fYK+v z>Fx0tJ==6;<3PMTu&+?PM*2}pKU4k7QchmM&DRd%RLA!4u}*cU-Fz<=UMwRXPvqO$ zF2+}Wt0eV-(t~Wg_fc?YvJKfG=!6gj3th}aC zJ@i+%8TAio*!l~hPG%VU;BXfI=GBz>O^31-Iy12q?P;~A=!WJ=%;?U{dkmiRG4;Q95l;l`365!})g74~`` zEr!@zGpo4Lpl_UpFbcaz+@!J37Dq=-6ihEXoAe}_tlET6KL$X*HyhykN^6CQaPas$ zSobwvxH$ZQyqDL&tn)<-v}vfgOT>3PB?{u!!D0EE9q=_HJvADvL#BaQlZ~LT;ls*E zOl(tvW!nccrLS>?&Rcom+*;AsQ3IuRW-_>&r=amB&KA_O$HlVV z4XSjFs@=;!7Rl?|?+dzLuCO_AnvHbmI7TeV$id9mF}N(1G(x)Az{;4z|-NFBj{!?{Nilbeo)CQ&9#+IK_TAU?! z-Yy6mFu7S;NKkdY%uoKWT4Cwyvkv8)puC|B&YI`xNOSnbrqJ5rA@)~C1zuLtx z^Y;Pm+l}X4mj|U2hPI=_Ec9e#S$7t&QVZ(-O>pnY4v9c24Ytd=11^K>SDg z$GP2r>PQf-C{4xQT6KVc;##OQc93p9R(C%HOGEl$@4f~+`H4G{r_iYWuyzj9=>B}F zGz(Cl$Qh`ouoV8Ad{7XsLDwydM316Rq$}nF^*5C4e~XsC^!b=n3rY8ZWlj6Zi9dU4 zX^d6v0{J9dG`P3eLC^3IcQd7Llwd0NrT=L@pzpaPQ9U&Sv&+rp{h}P~e57Z#;_iL4 zcY8(bWufNUaqA)S=r3K#EAxTZTreX&5+^J-M$$~$Tly~)j|nv|^*sF5{DZCcdE(HG zjOItI{h6&aAQCo{&l(M@U)81i4#Jn$bTNA3B*kM{#Lsxodn5d~RR}iamsw6&eVJI& z0d}X>lm+8HviY|Yn9q{7SZ9)!u6sBT?*QQ@bWKPhObF6c1cfM%1w!6FF~ySSxSVIyTQKY8JLnhXEa|>w$u|_Os|^P7gh*ONTwfWdZEm_n zQ@#9#ApeZIo~K|`NC`Cc@nui_TfnSqY3THQFMD$Qu2A(ZBHBvQ3F6bCpEN(Uc-^tK zX%@{# z(T?>wRvjDu-6xFwXOizY24My*MYL@np|Xojzb&f#p~iU~P)`HZxzevp@iBQcB>#(< zLA3XMh8v?E2c?r=E~B-3d@W47`w++*fz28h_O0{kG4`P0*NW?~V0oM*~Uhjd2$@uS7~%KNDNfxXch z>|t6HNfW}ZPhF9;LY1EFHQqyz&!nvC4N#szW4;AdPf$H$p4SB8^L?UuHl5MZW0x4$ zx>)$wmSXDtB%q!XPkLR#X$_Yv9WTfSRpm%_VR;(*d{!?m2cCC3rXj5==ihDtJ03;g zmWzKV=X!{QQAl18!sB)TjTQTMw++NqUJ|Mnqcb`o)hTA{jF9B>@bCPNIK7(>eXpOU zrAz{cwxNAw9nKPVmcapET6@0b8&0}%O$^T8CrGpLiIaybOc5$eQLkT9`I1F9jC6;a3SSxFtEvgDLfS9}<5kPe7QFPT1*4{>Tf757DKL zRNN1grU}x{;PIic)N}dE=2F~4eT3Ai1+qUp@3F4*ruvl(SakWc3}@gO?eZY=Fd1~1T3R0#5P3SSj| zaOHuPE-)f(TN9q8xTErx>b{$NnjlE0A$b^02Lsx_tmqW$8HQRKW2iqjpY_Re$2VVw zg6fM)%cSx~3Ul$TRl3s6;=E%djNLIw`PeG${k)_xsL#Gv|0^inGIVBRRQmTA?Jc2v z3-K!Xu)8?-J|GXeh%ctqlJqmF@}x(5(`lVyGliFudKIaU#qi=C;;;1+AWuv+U?w*> z+B4GBcWrD?p9IQos`S0mFLzrXfuEJFs{HiL5ZbpPGDzWdHf0t-9+azm zm1>4Ey{*NE0#)@*vMtoikE7HZoC`I z0+=T~Wn^pFxGDIzL94`1v&w5YMRlb8V*AG?oWf#^5DDx5Od~Mb9l$%o? z)L(fsJp1qtSUoR>ZMUsZW#3fqe0!aJNO>!UB<4Z8Yv;7&<5eyviK_*TfyfWAqelp? zl3()G4*P2?Dn9LoiYrGUenq@!_!Dc@m;DR>Gaq)pJF!}HTSmAt|-J0lwIoEz*;e;RHzPKf98T%R8zjAO*Y%B9l zPUFMyBsRQsh-uEpiR18-^_^57FRrT3r~f`L9(wk~r%AQt(VwYU`YIHb<|MKjo6Pz0 zCeOrKlbbNM=s)_IJuk3sD_itRM_hkg^joqC;sflsj}F!JREfneQ(%6LrtlzhGI#FP5qMlkN1mh24Yp zBaHzT(DNCSW0teaF9y)A={ln4Z9iVu=OuQcwaRDm4Y22n`*8YXCrzK`)Dy-}h5e2S zwtsv&I?ndz7Z>lscfUL0d&3kKnb#TrT&yc!nD>;A=N*&N4Vm2h^dc^f)d(6x_U+pa zSZE(AwWBVxZw7y9@7v#E`FA~Tns*UT9YU~q^8(UGPS@zBtmE^IVqu5XMld;(uMK=X zQQB=%BX!0uP zzq+QZd_NmzYW9iLOP7U5p9Fk*%vd(ry9ZwfeaHSA)?!{v5bwY5xk#*aj%AkD$NsB> zu%y)zadq=!@%Dl@+}(dt`)HDjoalN6V=qPXCfg3dxZK)s>2?W>Y`7NNt(c78%)f&n z*uoL-7_p&WYaIT#Y4+s9U+6n;PkC#4iN=V|l~wcG^41WU)a3|l+_+3!I`>!;ts4oe zm$l=wcb7s8?X~{euc3r@LDDH?j2w_ZfcN@RB9=Xk#NG|urTG(S~S!Dg9%==g)b#3@_&~04UsRp;XOy~Uij;4M5+Tr15oz?rnR

ox(`utBN^#41=XW4Kii7an6WYO&E<@SCt=c{liI&^N@1}1ek_~0 zU7E2)n5D1Fb*3aUx(_h&+JUd!+en*pA!uOJQSS2Zr-64hIn_U3n{Fzb9*P0#Ax787 z-4`dpGr1m$?>O@x?&GrlJYK}RNA9Bc{02jO%h}DLj|Gio*U8~qK|89iq?+dIo1V}1 zo#dfvS;l>whr9PU@>7QZBCmFU(l6hk%d{Z~jUHh9)HDpacnj1#r>qR5&-_o+oAMCt z*PF;I9rlYM0~gAB#XGT}c@hjSNWrmkyCzG2gEYyy3W>VC(XewO#?NjgqwPNfn7p96 zZUZ@@j*Lj}ExHlXOEHhZ$e(&+w%SZTYeh(~v+=IV+uaRRj8nRD=*W-D& zEVvWNc>C18{P8>|VLaXx;%6UWst@cYG?NW>oTJ3~7Au_dk-kHY2AW$*bpXffHe2ebxlpH86aKD(_ClU9l^;yqfZ6=6IDM`yWWLy?dLLFVI05YQSD^lvfoJq2^*>&n z6a-~UL)qk|jrjMDQzg}+%(&b#`>ftbJgq$r11E(`T*`jU%o7Xkjb-_(9Ig5tReKLQlrYt=Yqr=zxI+{Ep!FSCH8uw)cDw)$ zuf;?1*6E;2`yACw*eLsSI1XD!uERI`r^2!_Q%U!X2M>j)>w^c)@6+0!YW(MzL0rwn zX>EJC-l7iBGizwGjXgrudqGv{__Ta|X=F;YR zAkz3yjqK!4^XKrb%btQsdNIs7HVD_zdNx(-3S$(8_FMlIGj*53$I)}-@+One$It-p zzYk{A%bdm#?-evsy?}AI!qIEGA2{|lJYt+oG z2+{~hJdZZ_-(l(F+H#FthKiR1X5I$sck0&&_|Rk@bhu3CICa^G^PY4>!WB)$6I(cD z(oEv!R8V-a*TDjT*Wy zRQ;~-&8po)QB;(vX+dj)lP8z~@raD>^oA+!ozTVu*F-k~su83f!q;OOp?=LC;BexF zxY@B9#-3=VxEh9?c42h?(5YnwTU!F?-vcKoKPSA@nrAy!adoi&4%}Rtk4rDw z!m7zy@srl`e|x%d8*~SIOBsG;!w$plpyCb-n<0`xplPPmfMPc7bQ5bo~6*g_v zktGHe(r@NLRJwwC5SD-6$Vqq6_e~Sz)k!tklmj(zz0oz1)vOF3FG+Frn*JQtW^Klr zneRp1H8ZJioZ~vUg$3`RJ&PN997W=5>@wiF9eKP|-4YI8At zWEtAl@&dvQAnifgVM7&G8gF?CqgQ7tUe#W@d|wcLs{7T34_=1f|3tDY9c(yh0Z!a2 zGk&aL)k#(?96kw1J2J%$2a=aU)SGxgnjWbKRV@j^JJ-oE|B=RTCWu!VVT?wsF}n{d zjg0EwsYd7-Js|y&CY~m3g{CJXs1|lr-Sb1w3%GFjGx1b>N1r-D&~N=hS2ZrnD|O_d zPern>lMyp(775FAzM$f|j358BSetqdpV&hBDGI++|T;d37`8%vyKe>TNG1 zp2X@tCqT_3@sd3IbG&T4&Jm+NHCA<-r7(arog{rOI$ZAre=bJi%@&(L)ytVX)d&}h zfb+Gm1D@2Nh;@PhBDE{5>SVWU}k;oHWQ8 z*h}Sq4zUsM8*}zgE#Z^-AIMoav_wFrHAAsih z@2YXW7_?2J`0mh~Velu@ir0SA0qy2k%GRIH62>4H_1_J{iya{K#4s8sb8<O;)zl zSi3meA9r&p6r{mryhVG$pu>F23 z-ednZp|p$QzPfy!mVBD*q5aAnHiu}*gTdU~^O|AkEP7QNNO&6{^YybB)g2pp_zdB= z3%e8A4aOVXXJd9daPme#HIJ30dc6MfVx%!f`$f@sAjDFT7vM9R9Am2YJl|!Cp+^N! z-KzQk>IH>wFy-`2QC?iY#%xcazH6o-O@JRqf2h(&>&^NDVX1~VRA~`Rtks=UZ)0>V zQ=U6Fj`U}HKB3Je7}@PU{ytzY5}u6&!Wv0^h1sUn@pvmUM!E@RIyXnUXAE9gSKHi) ztNy}a7o3p%GNbDRs#!)DNZNTW{%hMGl}=YZqWo>cOLodTDUQIg$6^KbD^}~_hV;4G zv|H{tqlptI>__KcaZtCVDd~t#?De4Lbd6gusY@$x9AAMnZ}k61kT_BCCfe@^KiQPl3CDjsEzJCt`i!KrOZ-i&G?wa~R zJpbf0*YHWz_SsRn|?DOrWyb%TtN2@%de&xj~KRh(%688FaoqER&ZLU}6&gBV6oDQwt(w^7E zd&e8aRPi`@FQA&K()`4Ec<(|}xD(?d$jd1_V3fg7pUqJ?foZqSqFJ-LK=TLWQ`vl@ zMw~Pn)#RqEY^MqI9zT)vgM1pl9SC!9aARvpxcI|geRm0nV5r=!!%XACuU9h4=HER5Qic8bMpM- zk#Zbu*N7}oI^$V}J}>m#285G9dPYp%xSc7^Y3ua|hjgr?^c`&5Li>@H>VbY`@2Xys z5iOB0E}L>PHn7M;IQEX9tZsx1Gj6B)j5Ou}R#|AO>AUDRBc6h9AFMceFPbkubXl1z z_O`pK{owgsH1CoPZ5p*g>Iw4PBPC%nx~8>M_>C41Vgz|0sQoDo)%!SXHblxkfOHb7 ze1~ufT3#E5cOnI*p17&9Lm)3k7?guIrP;JmTbvK~v zW?;G{55=Pyw6?kDUYONA7Kl5M{1RHv&KC6pJu&lP95|P^;Oc$G8;r)Bv|XZ+SHAer zZM8PQuN26Wk+;28m9fz{@_A2AU=p{ZypYbSG>O{aF@fd=#B_4#S%r*FulW<&;NkK+;G1$dara?ag^3vk(Qyrq z`5W><*@OvITW{^tzo{Xa3%HrbSwU}3jiNV*Ufttp`Ml&_=Cl-8PW-^}@RLxGfIQ*M*PD32gK zoT26x4SP1>lz;Q0^WE^i=`^4*g4nZ`t|tQS;w|6%NLmgl_o({(dj9tq<+KW;$bU?i z|HrR5^a9YWwk@;&G!IgoX+Q2wj_~624UFD>9_P9zs{e-r9uu%t39aqwxEHr% z{ez)vw&Re}vAD-*C?56bjwPeMWpAn##Y4O8)Gpun4$CuY%7>|zj9vpFD;D60;tg=} zd@4ltErOK#IZ#q>8&umdOh(_ekwe4>mh$ZyJ7lcOyI!XK(%RRNP5;C}>zt;t@s~)h zKCf>GYW#GjVPtL?78ESPdYauVtH}{~W?(MsX{?0Su$K(2o(;V%?nD28JEF(h8ZvI| z4m><%wf6R#-Z-r&nqFgtrf1q|zxEE|r^^qaX=N?GC^1!dtm)`_jroX9(ZS$&j`D5gE=W-Z#@gAERnW;^@)1JT1e}#el-06&H13v5caDF9d8!uZQ$V0M+fP2?i zaQE3PO&{%nEuB!l_<9^44jw4`biax3*0rZxI|FB#G>4XpGvVXNS2)hr0gqKws(E2< z?IJ|^QhTi*28v&UIJgWPgloEWk-twziP>GO#lHEirGef{ZQ<8B+{VrvY5bV)!f6`1 zE}SrPt0wmROjdHM12amo!o-*xV%)`1T63o}{Dz?(zUy@ZzYR8pd^*RYWM>I`ZQqGC zD%*sPBbJC8h8qDcWecY@pI}e9pQL}ogrhFJ6Rv`<_6=mP_cAb_-57lrz7{v$?qIpJ z4nClCEx#Q92h~_a_8lV6v+r0^uMUr2aY|J0P)#1$@&g@jIP)$!)nv$uFSzVy6C9RQ zOWqyQSbMiaI~o0BD@-uyA;;CoVjHJD5`8aOsn3Lq;nM}pA@+M^$X9nxg?E1EV0M={ zKCVVrsM$LTcgM`9_g=!ouD-yBa~OUouwb=D`e-lIug0_fxx&mEmU8C>ArimuV>4g0 z;!*E6;FP>jS=hl7v{NG>awDB*5?%)FFOAHmdc}1O{Ar9c&|+j)2y9&pZ7(0;FFI_M z?LHrX=6hE|a^Jepp=Sf=Si3unF6zWfD)v)7`fwU|xou?vl&!ChKVQGZx1(ILukA{J zuVu8y`kP=?vusnF^>TegOL;!CBfPBCf}hT7&^hDDaH|ntrgg_DhplDZoUv#g7AEN5 zaQM&P%-u4zYW(!(m`M5^QCN4Jq}sysx8vd6y5o3y?hF`szny${)|68{piM7bscUu? zuh6rP!)CUVhtCfXSxs8WM*ah3a^@$X{@|e(d|_(3JsbUz)?F^_D^D9m;}|iRH~&3A zZn(D%rvKW)9;U{CxHXsW?c0IV&%`0)4Pfrn4Yv+$quxU<*J&k3JL>SDu{ZJaqtq-3 zmk29Lkg!G4e6ga*MXFYKr~mGw)tRmQUAUF6^jv6 zV_Lt}2Zh41Q`L;Q?%*b>f25a73O;QAP$W+a#8EFTd3fuNvW`7@KfONk*&BPv*!8Rm ztIpqUM%d;k0#-S554jC|_RzV%9^JF^m+NtF+df*)8GoPwtwH|%`xpG)aR>;<_#KA? zrs{;P@si(j{3VU0e6ZdIEKci68$Ac!eD7a8aVLUjj|t|~m&~u(ZPzP(I?Cy#O(oS1 zf9|jWs9v#rXS6t%mW1O!+`!O;awPnL{Knag=7LlI@b*bFxDVR!m2GV0wcG2k@u^No zxPd31#^T_Q=V8Kt{ZL1Lr`SF}A3H_s#swibD`F})i=K>&f;m4vPYZ2d zTgYv(+rebgM1`O5+(B15d+w7)VQpY{r#++z`ZCqq>le7HIb$o2y@V6r*FxTvJh1=t z2|}Os<7q3RMgC~o-?%=VS8B9C^fdM2o3{Cg2(1oR*hctET=7X*26mvm+2ZitnctA< zY>J}ZHB@8O*lZm)>A8UWG+P9ZcXr|}mKez9zpvmkgB{q(rinX+yi92oRxrpDt zs4wC${%&>otiVMECYs1G7L6hI-){Vn(uyliYw|En%_&o3n%iVEj@W*f+ia%Xq@)M` z-bh1SV5i<+qv~_^@K8xuje)^U_=pqc>UH>gqz7?dO|IsBb7vDip)EaoGJTMA2(@Nk z^%Di%H}%+2=zD3mcJBCSAl?mu-uLkq5%FiHBYZZ2JK)Yr1# zCP)~EPOip0yk9&^ThV~O)2o1;8y@1jvs=XJv`gw*z`t!>*t)<qnx>B8 zb=rHzP+ay3XT76#;>?R0BweH-j6mW|Z2Nkbn7WYmka$<3S$6LjoH@9cDQqXaR~(Do zJgdn@$;q&}b2!rVYx%2|@Z#n zK=sS7K@xAqk0l*6n%iCaLHO_rmvs6MCIwjH-MVMcEnqR$+R<5QGg^k+OJqGM&@R_G z0*XtiKebK|IC|H(!}i_&i(J=24(hQH2AiHB{iX|FlSiV~!xjwmrhsS7Bo;n>n>PM# zpjcV!q|yO+Iew*d zkZ?tl*RsB}+aHJ>UR}b+qefI=c4eSDkOsgGlWJn%yDvaoj>{UF%7k|faN5TpPIHWk zGut@QncEAp8P%mm>C<_&ZeZQD|LAo=qHCw9$SDhaPO<3D(_`0PkkUr1xi zo)u&QVXycwdw|CEq@5&vC5Vfl+Sg{1uu!x3g^~OjUrj2Ea#}M_<9WCg=Nwv&O0V`D z=)~y$H38#tk@x^AABD5?%Wi6kn}uc15~!Wu6>2S^{CdG|G5qHWoH8Q^NvGiFGaTJM zozV2$Y{J`pwuO@iB|b~JtR=pMr)Sfk;qRI7^f9fUJ~9f%2Ct~n9W+Py$~sva-me*~ zvAG0C3WqbgM?AO1RlPTKS!)Z#x6JO+5|Mc4II21!{O5C?kJP-@zeu%iz^M;t{$2vj zGwV5PFM5rz;?KfHVNO;JoJ%=YXt#!nck!BanW$-R#|`~X0F9-Vyn`fM5XUN3;q=md z>b+Uq(|e-l=5VxZFdcs>BTNg5Zzo#IR;=w`j_O*^@46r?c9=^VV@Q4Sh7n%~@&uUY zQA6%&bsmY^@$TF(Fr?>5Eqe|H@;iL0`3gL~^#~FV@!w16T;#ldu&t>N8uq%NjlR2| zc4n&7(EGzR`aWdSu{@YCvo>RWOECCBE}gM))AKy5AB7Rcc&oS z_yVJTC*9!6qsL5^q{UqqENsP@Jch0J9f#$I?vsAe;cMe+Xo)-7ohj?U*y`q0Ou@3d{J;gn@#$2*#g& z2VB2vjxgdE`rhgkk`@Ey7h?LYQ`n8`uWu9!e)ucAgrc74Xnf03j)iM^=eFRa*9KgkN9_S9nk<;oCkvU zgamvO9*SxV$eRiB8n9{G9o5G;Chm!vTdm?0^0y+ewHLJX>5HWCfpjg{1x6zIB0)WX zpT3$&CY`GM@n8SZ+I_cID?KgDKR?0JUAnmYyBl~}AS|?b1%~V5$@8ZW=Etx_{l-Ia zY8H~W!9ul!D7m%Ly0v0zMm=m zsC3GJ)f3hD;MuyB!t~5hcIJFIv`>1Y{To`KS`-^ApW_z9!{ruz~ftW zvD(cOOzUw#)LN3qJ_n9x-J55)-tBNyL)ioBSjWPro9g_wQhYXO1##pD^dC?isU8LO zCGq*gY|`H_xTpk(KUhHA0YTb|(RDNOtGs{h^>CNIt6Vu_t-=ef!gzW=E%`*sWa=Y& zwFSz1XfN(dO5*_IhwuElL;RSF`1G$ahCa)O=Su>}b50YK2XGs`!%W?;-=4Z! z@*s@75s)U(>Qtt%iZ5rGn!APFhT}K0_ElQ>;ImL9Oy>$C4s38!{!u1Rs}3c%(%Izx zgMr3Y`5pLux;|Hbn(pt$RDMMB&T5q93G)2fBdwZ~H>#<9JvkCDruqQo8nCjKv(hx+ zKOj)d2p)s_OLyXL*O4m!A--Krd5yDtvTm?k8tWxf|J|+14D4&ux#>Swu%dH~Y1}&^ zX#kwtEVD|#T&m~>bUzx^7hmn8!Ml5BP@Yct2l6~h=VR{?*)%S_QH`mc?rl+1KLbbJ zuZ8x?A$QNXZuc(5+$U`q-5)yrZYv0j zk#ZlQ`0wdiN0nCr)hB6dOQn@rt5XfgJEuSczj{*Xg698BjKXWKKjEj-C$Zv-J1TuhoT*+130p6_lFp|wNF_~cA~SYPV%3T z+)fMw@}ppOYAAeb-xMj=BK_;58FiQ`ji#ZDh?n|05U=b(2pP?ecN~*VURfv(rF;o^ z3w`N$vKP`A@+W>1uvh6yLD~s=pVZ4HABfZs?1<4R()mp|WlvIJ)bc~4aZmYkE!W+p z&HUyBgq>XJp<8DMg0sIV(!DeCoGKeYosrSt()b3^m{K?Wj!lx_5I<9D$$rU(@2&%|Q7R z?ww`>0pofpf5B6m^i_VJGAcS7n6e+rE!cMpYxDp1rkX@Q~6BDT|W*jmN>xaxo=cHqNxl#i!*88X7cxx`^!AO+ML5NztTWuRrYE$8)R|&Ka|h+z?Ww6DT4eHSUGNnkq@ldZoe74@@E{{dRp`Mp|zpc_>DNkt`DD;mBgCXY0X#I zWw3)~&Ma^GN?h4`6#D6XgNU9!IIbnFpC`2uHpLgqhn|J->OJ6U*+JB}-4y>~4rBHn zOFqKjl~^^YrARy%j<#!Bg6EoDxc$U%Ub@l;=kBnRdh2G&W$$0%(W765`^L7+sgD6` z)%h`aw+Vr#?Lu+GvF1=cVYYm+$58&QJi~^@yw?2pWfb3b_#?h~aSr$1ZOtDHF_i&y z7T9r%>#)@8A9HV3!TLX(jsCVlxZ-OoJZfsldwxsCl1T>O*07yya<>##ZZ+ehEo(z& zfsV8vsRIsCHzDLqDtfmul#_Ptz>7O{G494R{@ue+9?WpyX-A4+e0Y2A=~+v3dz%Zo zmQENoNV4!<8}Zyn7yf~siwK>Nscjse0?waDg573Q*~B;X7u#vqqp3a9AZWm|Y)`vOYcEgN@O}Ine1U}sVFC;~^ zfDhvmVXu*kJar}x9tKNE_pIG^c&=3U=g=kwSB~9)=Aq-WpF|#HAKUK60n2Wo-yT;+ ze~0}k=d@}}=o+*4lmv@uX+1FHOj~K~Qlt%=QYsF8`Gw)m*Px)!GQLS0g5_^(;?>i1 zhIh-Kcp02Q<2FoFnO6s2o3)WTcPvGI_uCqWv4{AhunG9avy!!J4>%@!8Eb2}1HuaG zLe_OVF>&W&G|FAW%*<9`AKEXlyBsYi794=mr(3yp-f@;xIbI&#O8cV)(Vl@V7Srzx z1o|vKSwEV4&|2`=tGW2M(_9(1f3$42d;>h)*@wN*yNLUjoyNU~R&$o*B$uD9#GHP# z9}spPvJr!rkHQ+bF#8GZm(q-G_pQ z653lfG)y4y9MHN7sA2e=~(Y&I`9>F5aJuaHS;FO zoE7H0a^hXhi(=D)4I0xLX+0UV@&{7g2wSU9 zSljIdt64t`w=H^(>FZj{ofR`#+k~<5^VI~T@#od{6$sx~li_jxmrytDnf6HgF#hX(21ajqr=9c73Jg9^hg!~m*}j0f z*#F^sn$LXzy>^Q1Sthc&VIp*%aECpg`4Fbl+JF5f-bnKd>8CP8Qd9yI*_lhKGq5f5 zlb&lVvfoz}s6IpbIrU*ZaA;%6Ph^jR^dU{;f#-Gbi{TF#GbBeum<`}zfA+F5uXXq& z=W$5m!F!DA$^1jlP+xL6dz7=x_&rAwR=}g?cDSan20z&JIi3ug$B(ZH=05jV0#Ebi z8(w+K!slyoX#2Lj;;s>Io^c0<*XRi**I(j*)Ih0lrRRmc%y4urIIVKwRFi~PTcJVY z>U?*HvC_8eIZ&TU>SyL%xdv8_)#W#~SFkgg(=oGPovK}#Z*-Wc>op&C90&V&^O+}g z)tK=R-wK#Y>tnaoN&^33+n`{-Tn}l- zC$^&m;%QU9wCn=)WCJ#Da*RCK)Q_v{p>fu9*F)#N7sDk;V=PB5dBG~3 z3TdyJ1ZnWO3>0T{cpVAtlJ{3(B-MnNSYQhuEj;0dM;J`SxEZNz6KU{eqErxO z3j1?hHvH63E?U-GPFqahNu!a>x_*f2KUQHO&yAkKkF>AFY(3*Nq3znpdr#(KNcet! z%6GEnQO-LU)TkQI+&Kc~O|}%lO>AXD2M?MfJvtlRj2pMR4H7*<`(PZuxMKs-`}0ZFZwOb{4xsvEr0Ee|b!I=k|UEk;|wSoiLUH;@~e!O0&ZC7uM=F*HfFlv8Hd^T`eVhzChNpfuQgZ@unsQ#s$|nwWZ;Egt8m)mYk0?ma$T*Z z;suO;+X9Jas{TAV{2aEuW5|srOcS#2Op!J7J*!h|B+LD8kDxzs_vL2TzlS~VZd;#I zEg@+=B+hef5i$oJHqn8*?z-~75es1M8Tqb;_x-b9GZtTjlCs`i4r_kzZ0O6Z~ET(M3R3}nt1pcblCL~=Wk6alF z+xzWit~ZyWM~MSowQ4U0K5q~G5_6O;g~#6o=EWxA(u_h;en*y!|~zA z>3pzVst=jf)e5T9rQ)j9aaiA*&ir~{4e!spF{&HUvOQ}1r#yuPa~?Av_s$~O{}Efd zW)pgPhM|XUCk^oi_5N=*e+I&qJ*|LhPFlxp$|ijdq%UE^v4d2bx|*q_Z5Z)4ly4}^ zBD}`b+pe^Jy9=5RYtKmw3M>u96HVyM#A*@jcCj8Kp2E$!*V*3guN2n;y@o5!Ucclf zyc_AqYb3X4RKMKsdT-`2XRZ)#EqK|Ajr6|RXf-*TnM_UA_V1C1#3SUN-8k_skZuqc z9Q36fT8VCfA=rBNP7F$3>q>p3bcPJCejckQyjOZyL*v9ou)2yTG3P}w;btft?VSb5 zH4fsb@H(0Vn;Lw;qi{(*j-UP=hek8T0L?8bo=DMnOS%T7?WLL7Z(%jrRjafYX)>6) zXrX+Yc^8xh`%feT)hDYqJBMxWXC+C%P?AOg2pC+F*UO4-e)>SJtMAgZ6NiKuFSI1hu&)7b?5~W@T;e}U><&vJ ziFZJ7P3qnulp@E^BKQ@B>#K<`>x-1&x^k3Fs6IQXG_QB`Fxmk1{)LV#j3vV_;2BQO zc3pVWXXDu@v2+>Ecvu9z-cq*4J-GLuPmeida-Vzs;m7#Iy;Qp85z@CQG zWJGRodtaP;)SJPLballFzGgQDdTExJiF4{QU(Xi64+dw`t*wW$xbpU+_dN1nPu}&S z9=BiPL<^3!(#C_LAL#ODXXEYg4*N)|F>k$q2jxeVGU-{ihIFuHCFL|L2RL_`Qn~Vz z^QY|6tS5L!?Tt*K(T=6`l&<}((MKL|&B&+w>sRa|Kb#81e$0wyPo2P(Cwjn7p^OTyTIs;W-wNx?&`P|j_%$+~)8ODozn=W7^jw^2dx_Un zyU(?~lC2mo0q5v$tKmkU)7vr~2|AqM4HPsO1rME`rK|IQwXo4uRa5(0`X<~XzvH=e z(`~(|L4+F@YuuU6Zrjb*25eKNZhm(7g!*qu=|dLNgOAY`!`Eis@Ht}k<0||P?;jgZ z^k09PuiGytgB|dY1f66t-1^|aPlQPsnz8PAr*H^6n8uzQ8MHZ{$&CQH{BzThFTPBh_Z94*ULPGa5Jnm`&o%0G(M z7vK#Gw5x)iBGjDf`E8-We@2}UdV-b)+>8-*>@5Wr;RmO_~s|40S zG{QNp(Ph1~(W&r===n$sFhi}Y6=!}|nYuK+S-SKMexnPq@rm$4!sHgv5DIkwPN-@H z?<#PKsVxo|qD9iB@v88J6_%Ri%Ms?5u&xR|LBoHTe$T@4F4H!v?&T(O#GSPY^{$}l zm0#tv*24?_DP#|F*G_karsYN#9Hj9(&ZSmL_vA3~X;3Z!55^bTJXD2jic8!N8J~eH zgok&(A^a}}i^B_M2xxp79hYgX*1Dy7QX`qblLiZorlU7{EhX%p&B#e8Xl!)?p5h(8 z7~}25v=em*nTeSTLeHRg>sUJ6xXq)7_fTcKm6kJNCL=eIvPOrzS^m0Cn!k?RQC-1b z67;635Nox-pVIt~{?g0|_#}Ns&4XO-$0F)J&&3MeO^q(hW@I=F4+8&$GrJspX}sqS z>g0-dggV!^cpX@n2JLO~8DuNja2C}Wqa%?Y;oRDujrG_2J9zQMy=?D36q=GPSUUyZ zr3^-1UUFIN|Lw}~U&facG9*!>Lj>X9EH9gMp+YwC2HQpLJzp8#n^Mc&5_XA;1K_u< zdAA%Cvs_oJlCNgb^s0rC3H>DGD-3S6B4g6+dZ&q;l|#h80fpd)ZFT3zf#Q3?IJ}E& zD~dI~t;Xe9!0%UH2hQHd+^jUc&F_D@Six0)Rqq-P z*zlv@@uh~(X>PNHYHeR<)LSVDtW)rKZ1^McVQqGt)>uHN6MUO>^85>8Li2l6V#XF+ zFPWg-6|}M{?s+~PoX0;u-KOJxI$Pm2`DH$YU88KY!{Z(bwPZD#Ic#(_h0IJ0|SCH{f7vG+s+YPr^)?LYfhW#zz zzuD+Qli{G2M2=(sSlkgN`?5eoD&#%JH;ARzZgJagD|t}M7V@LRHE>*SMh_F}-RM?} z!QpEI@a$cmSLng(Tp*$Z`1PFnIX_0gKZv(qnyQ0^;|Vyfwg-+Q=qi(wSk<_8@?n>2 z42>oluBxdbA09}D7RJBZXm~w?sWh^>3-o^u4ZP8F@;@Z^nLiAMh>dS=C{tH2Hn&pH z>vDMa4%Bh-=e+LXTTEe`jU5>RZNVgLeck06dz9xm`tL@82izg*x`r zTYDYIDND|7rl7@@i*F2thj)`@mhNWgEnza=pUMXrMh*#GpfcOlktPF1zw)N8*9m>W zGX^w6ruUwvt?-pIFPD*X!>nTK*{jG37G!x{WRu3@S_Z#P6|Fx9@SJI0IwbW*n#tOL zcZ@8XI!|Z!7pX_}ak){1i>|6y)Ro4k5?tO>VIb&(fE?cy_fLx)i=Qz zQi5}!m7+`zNSsw>!_y(j`God zgZYBXZP32C`;`!6E;rynjYEi2Ol=iKPoVm83>~5(}BM+6Q?!Pb7f{jvt zJGFrSwdl^y+i|wa4^N$YNq=#3#BN?Oz*e^EQI~$F+sH1-SNY@Q-<)~FhWwMp=>3&! zb$G2PdTNC}in2R*zP3r-9EXF7`E_IFpxij1Qo= z`%NjpDHjc_AFQ4FucbnTI#by#&9GO2mkj^mr>6OR=FUN^N-T@vextmsH>+L7pHWoc zO(a)2lwB;TT3D|=zMdzaZjQZi@k|DIa>6%PI(KxH`jv0Kh&VP$fAStH92@shL*9kZ z^~_#!f2B~GS+oo7YhGTa-wG$|tiR&G!u9;@`DD&!J6rsghq%BSHyzk$q^>t?zo;o& z=yzLcQ_RSXgbrmGi8JPwZ-)1Ik9JG1hdnHfF7|y^e||N z-dgw|*GzYjg}xN$mi`O$!4Ij*W7QmrzPFD*gfLQbFI6b<4&+37UXyUAHvQ3k6>R|9W)qcWoJ_|fPbUBww zi7!Dhoqy4)&lgql=yPO~GLG^cZ>CHCX+TM#9k@`T+pNlr*XdmwYuhV3%(`M7pVF)S zr>OYbP5j^1wmj{_ax$MiC21pVi*uEqZ+ghO*^BAqUE8T@bDVQs;k9^PZ#kQ5;=btU zhNir5_)v+@(mtM}vwEdv@m@UguQQc8*q6L55$aN2PwC$xlk08s(&#O{VO|@3eL#ZM zY4H_0S~MqDcMa$5_2+WNJUe}<<5*p-QWZJ1#X9zSHcQ2K&Z&QXYpt^nPh#ug?gI59 z%zumAYo(=feJVCDw=8_%5;B%q-2cyW3;NlNf$@7}t~G16E%h0cU9agmS8wf7lq$zx zO22Rx`*6k7q`}Sln)>AY=euyxK{sh$_X|2BS3QA$)4`oi(1}I6CDuiLZJ-Eq#@U_S zC9p*HK9NUt?0KDCo<5|j(}j*M7)7ri{Nsg6EGsPON*0{_-r;U2T*{P7nL{*})eXMq0@S31rXyF*g=N=FCP zFZzNAa!%#1VTI(@F0uT`UCOTeo>=?;=_d~@-!4LJs>|~4tUTyNbA4(>9WL{rFeT=S zr;ius0J9u`GZkd1E-hq_Sm0!{`qVeqOpdSjM7{2Eh~}g;rs=l_tER7W%8?CT@!_!! zS$$n0?G<+jC_;gaK2Xi{#k6bkUG!}%=4FV&Sh$?aQxW2W_p!IhEvCPsW{YQ$>)e~PD^2mso#}*LDq1Mf{CF+PS{~RJGl?j$zztrO=h0d9A>hxPqG~BZ!Z^k(u7$eP?`oy}< zXD`Lq>#YM$SJ1#m9Z|=MIb6rpBOB=oqZg2zT326AU4*%rjq{&(i6Rb%NGCOv(pp^m zm+{s1`PXxQaRU2m&l7V}PEynD+tLgU0Mod;WsfCX-7x*3D|_eFgSHIi&gZA-#9YPY zit_P{-W7RrY(^5Yoko?eYr!08gPombEH*r3oiL@P1n1CiSw~tfTNP?kKPl9lv%S8P z4vaDVA)0?$L*NmyZ_FdQ*27)g3!1L#=h;c7R))O0Ypw9#X=T=7MEodq{`GlPvc@Xa z2HAIgJeNARL!i|&#)C>vn!|ftPs=Jp{?VB`J1Nj6f{$f?A{usxVds^-bjGO868*=m z>lBcgH)dMemY>OgU!FjJ+taX{NBB>X{Ius#Q|;7hv>5E@C}WZxbfaSDdB6iHRhj=d zp`=yq`I|(+1z+%epAKBB;XMWh)2dDx++<^6a%%NS)N0sOzT6pL)?bWX(1{i=E=8xi zj}SRh3kci`W4-mPZTI7&uP7d5~K_Jy2gw8+%n; zy6S8GTP}1C=3W1-&BC^-t`lf&xMshEX9D=Cuy&L$Jd=T^V#`;YQ`of(_S_aS!j^XgRfH>tk@ zUQ_nrKB7SK9{w+*I+q{)GvL8MKlyV^1iLo*Mq=z9gB6xOZyQt6pH=MGJ(n()>mtn= zH$d!&_`)dRCYg}Sw@7o=`H)qN|zpSd~4N9YUg%Sw7P3c#d5V$ zhi|Xt-Y1qRqdzdO>d2AN>U*UUdfl7$sPp4=dR(-!EWe{Fu0G7ykNGg>rzW%v<-Hb6 zIScG&=mhJ8^vdkA9`8YbXKuNbm%u|xDcze^jmV=;cYm)w{YM5%WrLJWM%QFv3+jf7 zk9wiNzZ^gFh#EKMpfVhjbZ{R(dgG`gpH^e=8k_#R7vm&5pZ`S#+xS?P`d8p6x9n=) z`#2T1Z#j?q_7_~=lD1d8PwoX9a-%12L}utWRke?eT&pzqdG>KnAuR*pFKLzJ8}j=wEP65s@NWU9OFKWS4IRvY{?IDUI?F%2vc2?r9W z=;m5F&?c{*{VLk%Aqw&F(0Q7KGjK^7{;o8+fYFSE&oW$Su#yeF%&IbxJK3MNPD;Re zOW*EuY*dJV<{;n%16!rhwZ~>Bh$gFb$3euto!R zv+3B8Fuwb1sW3H}DWc6hSca{BLIs!R(8DKNpx1`6@M|mm`+De7g@;l3(Ua9)*C5tq z<}&zH8GSr|+FXl!r_!oisYNt2q8;>Y6xD4sQ@)AY$(G7T1;&##?zi>J;!Lc0Y&2E( ziEc)_>xr}O16#Bh@vsdQ9n_JaMW{mBATBWS3U_|Ck+(aRQP2W7S2Z@vD;TWZG%8Jx z#r_+pXYySWtjAPpZMj%gl261QGFlM&j8XFe!&dh;xT8bP=R|#Ykikdu*-`p$0B`|o z6;8_v*y|sW0s3K(E8Tzdm4*ynN9A|j;erz%lV6e2*cYuem8tJbC6*PTqR*qaqNklS zYq5Q>q%O(rl!ID9sC9vB5O_iE&B#lBkB*VSMOV9Msy<{X=j<3EvCg{J=34|jWY33f3%qU8ST{491Y!GF+{%Fp1TUh?g|apr#toNDxhM12wdM9ti#ht;1e z;0i(SsOp!>(B6y)9(iahjYwM~UbU@84%2+O)WK&|C2Wh)Fs2X87<9S$8w7q{;`5Ck zlCkdV2=gP0)Qh7dHdhJSTY(R$>VjI98Q)7tgQ3&C&RWybThYX>n{aM{w-`{ki2Pe- zvzc3lZo~SPkTu4{kXbX+BhW&k@b9V;+C>b#m?Dg~a4&eAZsb@)=6B&02|h{=jkpSL z6)%i0cs%hMG5M_i_uo&zD?(PRXS#AMAQlIy)=;8X2x2VTwP9$Z4(K83tH}pGCt?m ztu4HVa_ZyTZWFu~<4+!uR!~Eu%2!{v5Nb+8XNus0{f$-?)poyA=n)HgOO~5o(qOsq zmBMIbzl6RT*l&1-<_B(~_|6OQp57>ECp-GrH=Mpa+(76b*?Z>z)b}f4>ge5zBC1Zm za&l^J7U&_mu(%*kr|(0;jCWELyZ}7to24lW z-JeL%jlyV7@EXT-oE&#C(9}`rk;~SI>1*VbyGIqgsx~?l{!R^VHw)b5 zLf;nnaD1U{hOgoEhYG{Zo-r#J9LbZNb}(uQYc*EDS7dpbqeT}|<3LA=brd1)LkJ#< z?eDPay)%QpX1l8}cMLx&dpEsAZmm~kX~u-#b_TP+5hb*D$$_*F`^CJUzlz*HyXb9k zOQ?3y0D(R+Swt3}z+(hJBc`v}^qhA6SIA^Tboh25gUtm0%B%Gf?iOVQM!ZCCjZmh} zkWGO*moxM38!^eSjFHg=enn%;EySsvA=I2q3muZG-fB|~>B{W6iCJEREsJ~1lZR?oY$ zgui;vMt^*#4;5Xh=K)*OM*@9C@K99Z#vj#o#6yEi(4rG0FpfIatt8;5)Q?LAvV8UW zoG$pg9qD6+>_Uz)g?kN}C`|9JKVOlzd<{2T%efPmDqwBa8u)mAqJ|xc&@{g>1WwD! zOyG5RcsqOGTnx^pTg5Jk|3sD#`yOh+{WCa%jHdbXsFvwDbyrv6^W`d2k(oA>I^;4V z4-z-dr8D%ANPd@#4DZ3);97WBu5cjMe?Et_IcxNdPKi$t@Q?J~LJO?M`J8TYw+l71xWZ^A><|koI+Q*5be)*OSM}PrHa2TBca*( zQ5$dd;`su`T=M?*{@UZ$I{37Fvex0k8vYX+%8t<2=6)1bxZsb##Q$+C<`q}JH0dEfhD3jL#AY)(=(DPKja_3>mGxo5z#vbb|7m#9codVzSaVH3WvPFqrO9}W zE-?9_sXGmfR?aIQGyE0-8%4}u8UJf)9C;^CNKa;XEA*nfva8ilAg`5K z`U$?Qdixhtt0ybI;&-}mv3!7!-Ii*D+KiHJdTt z@?E_X_&l3ituKyjseaZRq&&2fZ8M9?Z_UqGkX;h)6&M*JOa?XhWGOb-39rn0=yL_X z%p=22qn^A88rfpJ*xebe7#N@}NwfHO!c75>rA;j&^QMcbQyF||xEy@bna6BhCaPux zg3o&q?niELa6|pC=fg#(u-k<51no+GUFY!QFDpzoPgfV#p{~=OQQyC{ zdNa|TtdR?7(VLC(_NN+jqt`+?p!7USx;}>=_4m@pUasYKk!koDoO98sE%r`mq-SlI z#Y-<-p$E?t z+g?`v+ft-f@Y3G9ipwFDCTU?GB4-yo$1N%}q<<$4S$BQwz~&nNO)IK{2mD8^ZS&}p zXM!x_PF2*u=X%Qzt@gGp ztLNeU&SD=L=ppw$vr9r0Epq%$mq+zX-{{v9I2Wp;b4I97cz1U7plmdAY7P1P+%%#* z&8YtM6r7UxnkGD1L`8-!BJYBe)z33~X#Wt4cJ{tv4gRxUA3og7+_Rn-5l+*4V=wz@ z=Q&S}L!4Ou5)HMyqM7H@xKyb*d@a#m$lReI+3J!7S#XnZidZ$>bvV0Ji*j|Ne&x_M<-{0jXGf94=_uLfw zO)m@Z*Scg4J}8&#Q*%>MLlvwTFK;{-&T3mXUGUBg_N|^9@0eQEyDoKUNyi*Gd#J2_ zeE%WgXVlP3d1Rr6Oc_)0jHqo{8PRA9x6G)*A?~?Ek%=SdQ&5krny55v6BqvCrwY!$ z9FSmNNuJo4P18Z_r{GbBNAwEd#i^e-x$beO4!D(5Z#?Hj6J|$A)VLg9CV?=1;D^7|$2IjoiSMME*V3bW zJmkpX<@NC%&#X?bi|EyZ=JK%xn^fzzo>F=%9)EouS8a8T7j>8>OUqj#+x^O_eaj8( z`*tXyhv?dWiCDv(qCrFq^~}A7kK2ymi+jgW_TbqJjGzT2n`(n!iS=8{l&OXFRgO&e7P=Ns0}arq@?sN_>(;b*yx9y%hH=0*=6fpIkfBGeiWa$UxYPW z!W+teAisT$fj4*Q@%Z9%;)JT zouu3T@4%1z(#(m$gvhTY1lEDZFPlu}=VFe|r)oQJMsu4kdg@|Nvb+vw%%OC5&MEyz zSC`!uTmuGJ>9J=dnSN+{qlHes?5;|$Z9$iQM$y5|nKWtt8GdnlnEtqUET3()R5a~9 zSdPc@REHcNlRwT$=yeu*3N0_C>lO&Ip1K)8U$9Ty`D)T&FK3)zpl{x~s*1-3QsmbJ z5!25@6gX^eO*lK5fNhoz>DhJQ%4PCSauImI95P>Kn)n_2fl3Zsqg%|k(w+R@3=WE6 zjc;dRUDCT-mTEhK>g!=&>K=8B3iMfGbspZ4 ze2}wDDN$AXH0q#xP8zLm9<~4NVQNPF#oh>)i-pj4+fKsFH8@2Fr~TqjOG0_d%+icC()(SG^SuJI_2_x?^}g5s z0D1Q)tlCVx1)lTfHm)}5QLQ!!T$cl1tdu^`%D4Q2=y~r%V9ZoerF>q>GrOU7vL7wM z$L!XBjQn~ymA9Vqw4$bQ&B`=*-!88C*jd<=-p9wI@-gO0hV8`npLXupS7oyNaJ)?V zh&-vvsR=SK@SJCr3v5a3$EGi<{=LBVPJIP&cF#?ex@g(u5`557m*a$ottf5#Dh8s z_Nipgue+xR+?Vos*^_$g-pD=LRk75*U0NF+-E*j#__O^HV;llJX;~zFH2Mh7T0iG1 z_4+cfPPujuQ>Y2ZykGFz9oF7XE(J@5ij?s!)}-P$)Y>-dGj>_X86Z=ET!Ol zqv+n?`OInORa1v?bbr}7tHH13cFQeyPktn))vGD_{(m%Q;$g}xSVaB#Xh+RDI6;rh zQS;Jp#+TcC%Z_6kc=>J@37w*h_Bq_=25Nl*^&hleKnp0-7o+DnOYjU&DY1)-x3nts zE)AWLlb)v}iAUKJvTAF5nG(8vY&p3)e<S+_~v)?ohIq^6PYk7rVWrfXd!-&C(!h*!c~Y-Md3% zyy(WY>;L6kFDlAh%Z#POw{UgGKMri4C}k?ryWFTA)|Ycn)tnL{y+O5j1g+?a` z=ni0`s|M#A%}1DDMxRr|_sgx&6EfF}?F6ou?LX|u>WAY!`f8h56=Y!iXJyu|>G!aBmrj^sBJmy`M2px~b>Fn)}#lVAfbWpOvx3U#+z0r|N@E9H&v$(p8& zFxmk)X5kqlj?%IPlHEPu89l?&Tgbz1ZDhHM|5@v*b~^O)uXLkRs+5mFfA%t*X8JUJ z+^Oxnwah|ZZOth=9M4qo?W^m4T`SSfD)V{o=f2#-&PSI!vR*~Jtj;|reNrQbZ{?Uu zPx#OFp5ogVoR9jqmX7hrEjwNbVf2i3XBB7uaD0=Pl)bg}!lPQMM2^yyjt|cUOnCR+ zI@h~Z~s#lkKCLZly2QBBrTOYp!K0Bb-YHR2NUEynMj+s6{K!cb%6^19z zdfn%t3*Ry7hOn-x?Tz#MZDwsF7n{X=s@y>ZO+={;2D8y3(84_7-3WT!y$0rQCIJ)u zS6~l_jFp|#<$YlUeZ?;uwBw8GN@#FC9k{Z?;#+&P(YLaH{`>}SmF=>%-0V;fGj7W( z8=NmTwv{%022R5MyPFvGZGpbvo8>!aVMNfTe=0p<8t)mgT|u{UMAQy$Jf^w91=Z?H zQC{_Nm)=rfEtl<>Mi`&o6yH#x7DR&mjsU~e##c;R<0uV3HK9e1TUa~{q*T!HuWK2MZV>C8h8~! zMr$RPSWQ*qIt%z0T3NFZ1=cF0LN2@#InJFE*BV#A`*rPkq<1MfvqCZh)8*fR%?!U- zV(}b&iwei_41PcA(y=&OeLcBN4@W{Dajm2Kt?)E5=e+$!AE=A64uND+>vrhRWcgsOP@diPLmelBEO^_qs$>2X#Vz4 z!&7uEx~haGWX!YaLkswY@+sAPPt!D*iP>A~??iO&7Mjpg7qFc(SEra94hkV<4 zPjAAzs8wlV9Xko#%tqf>{u?ae5rprx)ja!jqb#q`C$661YXz>S0u_+A)peApL3L|I zZCd!ymzx&}mMQ0p>ywj?QH`bFfUzH~@Dggkcoz+v<=!>B$>PvDzMuk)Rv(*N@1eRj0I(?$*+FkUTA9Zle7#y#`USA8hk{p57a zu{2ukcik!6y-^_e+*V&tjLXtkSQCRiSy*NCa=?8r+4Iadqi6KD1-WIuH7Npm0X>`~ zpc#$75Jr>0Cs6-tLkQSKzDna1VW2)0YD7M~OzB9ckI!dJ6X|flq3E zr&5N?-w*NZ)lCTxiu{0&dE^ z2|eVw6E~?`-*WO;KyhyORI6V%6AYHp(Tcm_H}+9R1sm(=Pmh)Hz;9zZ;C-uWv@!g- z@kj!AWPMU1Oy#V#hsrO=WQyWf}Prbqrv^QDfm(2 z-?YgJCe(;zJ=R~MUyUc_m2=OEzXQ+WK9aKht$n}S;Ol$;$1A~QGC&SZ$EAz;tWltHnW;J6h z9(&N>AACU+yvrtnkG4F|8%L&>;br-5x79S#d!x}?Mxzltr+ByRgmT}u()_pbNOP2V z>+C{J95LRKqyEM*JW&>RW4&qKpSd(K!bvV#Ta-|T3Oy;@eizi}6A?RlHva!;gX8=x zxF&-y40lj;*L>8>F-)P~^_bQj8NDLmVT9kllia93_S1(RXt~OVH$Lw|;|A|%Xk7NJ z(vsonq49H*;WW%IcmA=>ct{Gf8A`v(Z4m~`&79tjPlne>kl+FZuGLE`o-+PjY)d*S z@~x?%W_`)d$Qo4gqK^a*Y-#3b2h1r+$X~F3T}uXDQ&48S@LXJdHBzzRL8#}%k9a6LZwa6Jtz zo=?yEGEWrs?P7%vri8OU1JWk?AS;^1;dKjB(KDgc==UtcZI*u41aaKN>ePYiMhRZ{x&IXB0dL z0k8ZGC&Vu7$VOvanckY91=N|3tBwB@nWyv1RvQXhfCr||DK5IP_AkDJb2TWT=lX6Z z&Q~Ar>$+f`E{M&^d+GSsoCbFo+L6i^TTZ=(E#)FZb79PT7`j579r(oj$ua#_u+!=S z%IkYM=<}cITH|f}+9jEhmuP5f;aE8pUaE$wH=(w{b!GCS3LBe=S=}EqydS<7Si#U~ zGGpo--e+@N)E(S13)lbDn`iW_#eK|EfqBltag*&BE`<;0mkFnU;i(EfMqIaFg#K!x@MW1B%E_t-kv}@O; zQ|SB+;Jt(7da8>YlYCAX%tu~B>F+-Y!|B$#bD*>5fe$Kcx0poQ`}rB)yw%REIW>8( z+hjY?mIW=)ul!%df_$v)Sn~S5#k!vZ#WY&Ru?ufNF6GMt6$8*hzV24}>U_LI<};EW2G{~TF*qeMs&fftO=RLD&* zzxAlgmQ7jt4eHohaktPZ=i>!@3OA{;pWqV|I0)HHI2V}un4gGXUe_#@puq&RDH%Rl z^|31W_&+lE4!(mH$U}>_EVW<`XkJ=hyaN^^YMk84k?o?v8~eC@%aZh_PA?j6f6Qc# z3ONncT;_n>?;2lEEJi~r;mj7)Y~qUR!t};~(%A*>A351t29~FHeAr9?H}&Vu?Kz@R zISKDb$RT*gv62d&h>kak5XgT;df7ywoZHFSr%#Yu_P-{pR^Ud}zgi}Iy@N7*4ZKo2 zAIwTeHbTgHIjqe`a%mIErB2|SsxzS`f8g?WKM-^#*ByL;kmnIHX@ak0%u!ahVeSEb z)6%BL9Q=;0$x^KQc3wn&yq!?5S?imAqo|mT@45^Jx1G}j8Ql+y$v-Qooiyy+7{DW zpZ^{%FU3GOA+l(D9=|?2Y#+mag#R`c=Y3^S1NJt=Lb#7?BRpo1q>*S}Q`@m6l^F zETyt;HTC0#&fIk8F5XiuFSRe{s%_>x6$?8pRGuXk@t4upxNX&vdf$;s+FYaLjU@bT zqBVQSKvinQJ+XUfs4o2O73sIfcuI#dWb>@BbZs8aJL@^fu$s=&ZtiyK)a4@$j{7PG zs@%Hbn$O~w<1Bq3x;pm`U!%J@J(chqv}+%GqcqA+TM_o`df&8O_IZlzH*O5vQOIimCC;qqcc0Dr!8Ojq}tNe_LW z<8yzgj(skQ^9@LbKCC3=^2vOp*hdaNvtBnow1FFp?uK^)9;)h}29ctz1D!SUjNcH(-e=b>~2RCc-M3s1$oo;r2#Jd7E z(Uo&QEv5TU7f0r0mk(rpnv}YV6Fu*$IU}3OI*XDmAD`GrQ$KZQUE(vrJ893Ao8+^7 zryf)MrILsXaF+|DGPwxob6E^mC1LpHV|+8@5rc z!QOAv*5K@|9C<>A}68=aAPg4cSoTdNW9!K1Mxf;CnDh-W(#CFxz@W|>RdQ64fbg$tp#&~%_%V#vG-FK@? zy`FS4Wwc)Na~Sn2zEZ5-AI0yQWYd^i>hdQ@H|PZUVBm~lGqu1~0S z3cb;a6Q=a!kr5uMtS%utx@z4y_gq@?$x+vvy;_VA0dhs*_HuLYTeQ-Byt?qB8NYBp zMH}Wg@#N0iG*z{hLqGy@-8Jv;!suN!%mN zc+eO3=Vh%?>X>M{6P~$$Yd-1|O@+QS7KfI3i-`jY=qanG(t}3vDt>lt3fWRix6H(= za_3{LriOyjT;=OlSGeT)=9b}>k@`fx+x+bi_Oh(emIwXqCr%!7)3^rBh}=bCp?ZJq zM-wbTor5{*;P!yHT8HWBt+_n(?P!Tw=eNIbc1nwqw90!uJ*vG3Isbw0BRAmjXCZl*I86qUYaqD>Q`}+9bCRpuWErbpr5a9 zblO(WF11QssgO$l70D?Fs-<)~rKtQ{=NRR1id9FhvFi4w1NR$tn!n|0#lS9lR5N*h z?&oPLX;+L+!r47FM;*3YzTQb55(~-k>JKnuoR`IWR zdMg`Ga^9++TzF_X>XV4InYj%@(;$in&Am-ERYFRqDk z6BiM%O8;H-UG=_tLHuyZ&-XJ1a8ar%Hx;#$$#1sMrHmC4HO9yDca`t`n(6?%?!xBT zI%R(^TsG=8SbppEScDJ6dn9q=bh&}c`2NPc8hr^~D6B8M--0>T3aB^Ypc@ zI%VttJ-Ay@U87V3y}5TM9s9T+xi)uV{%0fp-Tq2Dq4Ruf>oL|qZ#?*qPmY)=tE|J` zg*NMDc56JL-pGCYb;kX(|G{vUpK4fd=>l@&m}V54xk!MkaIKkSaM$Cyt7^5lDg6%o z#X4Y;>3=R&a*OKdRfyiSUdzh@XR(*lXsUkUB^S%^R(bB-!FN^)I#U&U4Kqu(#=Di5^HI^zaf>p;iFM=dZsVVs0lBq6Hk}X=iWWyu$uE}Kmi*K zzEYtJ*0Nk5uP@Iph4sECa_q^a<7XdWa0lbMyg6Sf>JiYM&UAO7`8|uFZ@l^43X9C1 zJ2m}>OEeFw-;%=JH;7ku!Ftc!4P-xdB#Xp-7W5_miAhyDK2N~f#zP3)Z;isfP1SBT z(j8pJ(&BYV%466B%*jsr-aCmdV6To3EA8db;04OevAa)kdH=(4%;~Od8zaToM(}}x&k-z`4$B!iE&qV)C{$@w{*uJf~t$6*_wB#fjOa z;l0#xyQtOTGu8q9QY~Hzd%A~hV}toOMi%D5-C`MB%%~M@@Fsu%3KDqFa|)&LsB44K z9}!|=%6T<>b$JQSDpeYG=6nc)EIp)&fI5PPE>|!~Z*06>(+pto2M(Lc@ zXcW6Ek7(DOH*D&ybM9=?x6ljzJUfpb*!2cA>7ByI&Xwiuf7Y_;tM{*0a*Ln}y2{G{ zeX4&6;Ph7VIJZl@%$Z_^7F4ExQ0v+yezmo5iw&kHIKnTHuNOq(UbZ2BDHBJ1u0;W( zFM(G)<<%R-+4MQE2Ab-hs`K~|12@UPXG<=$&0Tu0?x?X=s?u7f3V6p89HhV*8kj49 zFY4gj@eJ)pSI-|cbwfqg6w{VmbEvg{n&C2>)iKp*OrwLi@u8BYFC^BAhc}4QY1U{_ zxlxj}SY%GC{XHjbbXwtry=w7CW^g`XPHEqf9bCRgb)DJhi|8}%0ec?3f?l=P;30k= zxW($T*M;6MyeEw2@pAgal_MWefs`%mJGrPdYmUCl(zU2Z@O(~ z%#Xd=C)IBs@pldc-!Z( zl>9ZD#P3Sy*{M1@tiZCx}cvHE-)oz)SU}bs>I^XSWR>7#_iVNnj%b zM|tXolG;Bf>rwOk_|m+)$|=8_Ho6A2DBuOeiYbdl(RqD2Gu`&tId9j2Z*ZW5-_Tz@ zo)I*vb$|Lr_9%W-lsz$%`=7I8yDL4#r3?=p($!wp995TgFSTW0CMWyfH|xeTeGhTp zv+1<+=^0wx8|Q9&W|uuJ>kY>$_$A;}lrrOLdv{qn{0hFecrWaxwd679uBTla(T-53 zGRWa5bx3>6&@ zBj`kiwx%7i735cszTn1dz-?!`v)4g-f6qhXj@7|^=h4vPYH<6y5?CtK-&!0Kvy|GO zZVx^7k=CzxNtg@M;}RTX`dD0>SWQCjfH#kbhGRCVjb)>(f9o_b^B}edUE=aNis{%o zMWx-e3&0y^Y3}1vWEk6*Bigd_ItAgJcWC)6ZN7Kt6`nKTjc8NmH?MW@wA}sP#bCNR zJiV9kDZ+RGa1ej8@`Dx+7?6XL~aTj`PiIrp@9%ZJNjL~hGj z3;YUa8y}`ZZ(KCEZu*=&hacibr$goXCcb*S>dlx7>X-hVCXvZ7&JEVg2}ICZFB5fDFbTJ^O$&e+DY>mT;IJWi$}&Emnk%&hR0pz9*Dh^<3La7Ae(rDdGlLVdJS6&qKhA2*Zsj}ixbsEm z-j9-mx=`0jLOb3sNYDsWVM80rS0I^i9nmB?!(!?R`qTn1%X8hj7~QI#+>~TEW9ZvY z>E@b&fz`6GYe{XZnCo_j2j(eZg)h;j=JpROB$iJ;#=EOF;aY<#nwlds2V1)wRgtq2 zEJa?lXRFgHHgjsc!spq`2;LX>eVnikY-$VZZ}fVAAKv*KMRsKLW1jF+3ZJWwm#~-5 z-DXhNrj04tznk$##-nA`DR_bEtXxFZ9&O~`ZD&$=kB*$pHIwX%KA`25^XO`iO6ljL z{PB#QpENl@#H3#;BJ4I#>Vc$$G(eu&{ykY8A zDr8eeV|=2^pVJoTBwC#uprI*vXVfZ=t5uiV1lWqXCrV57wv36%O;a~)P)Em~v)t{t z!rYTGoP0~KraRlqFvh^tx)g7AX~Zi})@5L=6}u)vpNGnes zX5f>_0@AOXn`|(V;gzgcd?Tq@lirpVU!vLMBtwI9h>L{?p{*2tkXgyl(`_F0R*%-xA>A@Y};IikRhpdV}1x=i-P|W zR||X(K$c`N__{u@gDM#RCQCDz8h{V9VqPrh1A-?J#y=JJU8`R7n9631zkZj}#>YaB zFg%4A8h-%$SO!v{U%LQcCLLO~o=1eg<@bJP=+?k+h6iW#G?jDiNYr{Gfgd?nv)Tlo zqT$;V#sYm>l1vr={m1`S#gg${RbPc=^)$vPzxVbre$V6-7XO|X_}-e=3OK5&Ep(IV zUAL(7-<{a_@NRJ>gz<7?iVx)7&z39XM#eYdd*z&2c^2kUfY%AVVI4cXANu+vo%>c> zq!bO7=I6`m1Xf!U1$dV*_X64-zIp~%=(rx4+G6OLJ)+0kqPn%eKZB=Lqrs;P&nwg{ z!P`*h5Gz%ycpL9HlvB_!{PomMHlKrhglw1Q;GA2Nv*xBwrEzJvjsT8}^-WR<{HDMq z{9loUj9iT1=>@QihctU?JT^7|+)CSDEpNg2)!Fzy1gy{ltS8L6i9Xl77`j{SJ^YNJ zPZTsh`lTf9Ka(zBzs23_;T_v0FDRe&OO}>{rWeR`%sOcJG{ecl^lJBd)hTM<1^!#> zg(&BIQ{Z~)*YAy@DD8sBSPTk#VOhAt+x!mrI*e_xpZs0BFll_zt6od6Ryi2@klG&_ zFKnw`Mt#RwhClDdexq~fPJQzm&4qIcI*3OL>c}C@`k*$PB-Tn*+_;9La`xiAhx^ij z`f(il_&KfrjQ1PIdMMfIt9CRkK)cIg4`yVk;4mwA(qfIpxh_q72Q^)Y2R@<_lc|f*b7UQwtMXkUk%cvQhWS8A5mB|gS_x~zlGjl;(k;+V7 z2rPZfO@H;}%MbRE!GJj<&xjH4XQw-*cQU>%-QbDID52{_QG9RjcI`MP*Hz%yG6esl zY27V`*GDGM3ts0nrxd*|?tf^=vyb1SxhE1;D|?jb9ped1KQ{YaiCRA`t-SKIHn}3d#n*u*->v=V8>fuT5a2F0 zIXOHRCAH~C=o5=ii}7T%%k&S$WJ8a)!f0gpID*$C@EkYJ)yV>1$;ei%YdUX6Jr$#b zJFC^p?LP#(0go+F&G<`g=EP(x&^ZzrH+?;LS*7K!pf*e`FX0gxy&}GzszP12+L(L* znm3X$?+koas7vHSm3h~aG_m`cyE0kH42MuUvGgE)Xggh2_Gl^GR@@f_PTA70s@G`9 zVpo}SVGm&b0a~)OJJs@a=CV6G@>JU$+_zjb=d6;!zV-n;YISctCRa4)6WHtZe~zv@ zpvEqYYpH}}MP?!qDWdK@&nZz#WMq%*tq<97WS5bWvdXA?o@1pDQb|^1?~#!$ z-*fuw^Kr*{&v~BTy6<~V?`YJgdt_cbGZtGP&{>u<^?8lC&0!wx1>I=PY|vb6jeqAw zVs=Ak{wlvIE?&16yVdI~`_^bGzn}gJ?;Gj!0KfWt-I^10KToMh!-2y6)D-Adavg^z z>gaqWFUWHIB7VK{9h|lYjC)pqk*jt>dC`FpG2FSy4cZjAC!NiGg*(M zX$R}}v$oqB;AeK0&b_Z9pU3yXA9+slMDR!LpM~QgphcOeQ}PasF71%>emL-_;SA^6 z+CxzJJ(y{F5g#3{0pm#*xz&CQNiNpVr`A_k|L?3wNiRkh=QXmd??H@v5hEWRj)Vz= zH_NWS_v(h~cf#Fs9^&b{0_kE&7Q>2C(va9!6V9;kn)J-y!g-@IFR|git--cTl%`JR$i^0&X<>IZO zm%Lv!O?*5upP%fxTlD*RLUvdk#^yiEhKdb)1dU6W49SGlff0OA!_TgLPb|hi|5iv% zpIvgy3gGlwF>vS#+5XxYJp5w=3^?6a(rabEur^R^c?muCEeBgoZ`wQmJ{pFOl_4n) znKsrQ)7OQH-Yp-9CVP$wd3A|4y+buVzGgTKKX;wcdU7M1I%se|j6J{ZD+e{C8tu~< zyrVX&T4aW{Qz1&Xl1CR=3Ma2PN$+EedQ8#PpL!BXUFdnn&d#!PlWRcp!C&bG;mVLx z5OnDwcCA=|thx){^6D<_S02@tUpD5)Dm0wxR%Xv@$2H|Yf$EGKrys|-y0v(Z@iTZ; zZp)l+0S|t;U7nwPM^`N9e5)x((RIl-3}t`W!R3x}epffXXx}3Ye)ADu9r!`DJQI8O z4O8~VII2y<=k3vkOH@9}gb!vmVx1Fdt~X`0z~yv~HnD>xg>_48A;Qtnu>V!E(c>_0$V=_RbRxvSV)WEzlfuq=09? zbFh1WC$`x+Lr49Fv6I(A-IhzZi|JD&JmB_^k`xwlZ{zv!;9_s6<2q4V?+=qNPp!jj zZ%6W&%dqa<`Yh1EN-pm_fK&bQ2{+1NeS6@i#vjAkvFmaBPDh|Rg57(E17RXO_mVr@@H>Tc~!8@Ck1x4)Lp^q~^SizN1GRR&O8mgJ|7=|y;~1Fk6f63* zb`s6?{jsA+maUK8)_trt69*QIV}||B<+;3ksLB(?=2S0!ph-{K}W7PtZ2hnGQJ;{T0}TZ8J~5k{+Jm}?U3cOl7Qr#o+j_$MNG0I?MakI8&gM-BH=kd$&T=Xg45As_J+1jk1Trj*p&AG88JfY95fYD~h z2{+B*>b-q%+vy&9Tt5I*>-e?xV6DP~BlQ+(2=l>i#Rwj{xf!l5)d%V~=s`MB?_v+) zlTk2n&R@LRrIGw+)Q*klWFW6^kr)fn&d4f(dpFM73utq~f`b?|!?+ zPOc^&4Sb7EbF3uw7nlu?6d&*If_mBQ)%w7cDm@@$X%-M?pu%=1uckED8h1S!p;`!_ZLb0=7GY#C4Qj{>`XYj}^d zFWCFC`?x2;f&Un4%Slr}O89Uzr?YyfwprEfag6YR_nBHB=TvV1zMIcrw0>RmnyUxx z>|A88YQC^#+ekHDe&FI;LEjDKGbh55o5!?mc7D*h9)2V8rqLM;`st*V+=S=NzM|dW zPPk^KH4-<8g%v|_u>2|FN6!O?Zn;Qv#$TSIz1l2WfbC=-oa%FneP-?=->!r><238h zz!a&LCDjT}vka4KJzA-n;RX-yYD$(!9JcTb3+X!vLpR6c?zokeS|#*CBP{rCD(;~r3erLNryv}N@4?S{HHKgIBJB2OqFQxKz5bfcz08Jqhy7SLWG)c5@d@@H=p3mX z;^^>>jQR=3(Y~LebRz!rl%hg5COrL|8T@9Z99nb=Mn7-{+pMm-k;zdwBgU1v-icAoG}&q_Rt+VD{8X1)$D{@bIAdr=1#cSr`}WVYm{BVo>GaPs-8Ti9-pB+drc!_#1| zl@5DEl5ifJ&Rbs5hLM2K6R<=uGp`8wq#WpM_6x?9Ll-_v2S2%oP{+ z+sWOtOjx5;yRhhQC-LaVXGWZlcM>)r)wFQ(nagb_U&ion=jmBGI~ZJl7bDCDHD=O} z=49Sk1VTS38<&7+I^h8;!?NtMqHcpSNO5i~iW?$>vPLRv^tK6z_I> znL&JU21wT+@je`A5``0f?WB8c21>#c^!Mqvk2urZS6 zDIK4fJ3^A_0kjG<)z-ugpm2-uQRg&g2OQ{Cjpw{QjgG@sgYA}!_@KELzQ`O4E55u} z;}UJY)`Hcgq43CNJAVCnmsxwSW0BrB6b@4D+(*677ifQg*6{s_6;67Q#uon8M~mKh z!mEdoq}NGn-?2(-Gn&UreM5c$TFClJ`+)nYS9mV3A7vSZ@X@%TBz{8o$2D1iADu1r z&s~y^QaT3T_a2CWOMc^_>E39Z?xf}t0zNh2Ef+gU#Y1^v%{3$RJs@9aI=vyNW zXnlBD${x~e`>@Z{Sd7ou3OoLDrt28SNE^ZLQTeI{$(Njjox`pw?Bj&%a$&_C?f6F# z;!e_YoN!}ar3WHh5c|hn0o9iZo7=}-Ag#QGyux`bv}hvVG&q8BmXq*Rg%xSOMX)G- zEhH?e$_ckbrgbmj)$@KO?in0=fL(uE!{2!|#2@{k;1Sdw=o*3eCG%oYFkCG91yu9s znD7oZO*ln!6^nevFit%Sae~fIZ|bjfC;2%&T}bgtl$ULlj^(YIX_U6d6UCFkJ-oi8 z9$;IKHdH+aE6OaXpT>h%qf+qd5w43)Y$TH}okhRwC?HM3i6@xjFk`qmEEUxIy$$z> zge2Opi+aI)j0^XjbO9fiH>UF#hVY6F%@qfT>7#2ikN9Mv-a~c7tZG@%Z$X7f>(3A8 z#$fr(=Rg{_Qp?4UE@#;dPl$Wbwldz}ul8Qd9~|d?J9FgXP3$h6&qEp%0v8=%&l-n(UhJbOZBPOzVm*}VhZAA8!76Ti{(pO?jl z@fM_6&I0*cPP{JabT*fyla;n*q;nM)fvP`8do$T$!w%9)w%n}4WF+23(zi|$c;iXRARF4hhm#{oRb63exG;ZSfR!Hlpz3^(h@_dz89y{%k zu=afnDd~e*aMS|QD$|6j1=6^rKjTD5$|zL*NH_wcrv1|rcM9rz7H9baHu~CN=%oGd zWcop9`s$2OzP%ycLwP4@q9pv*{L#Oo^o7DAY&>)i(3%4IJQ#Vq8g~A8gArz+!n5k_ zEEw?<6Q$2#jpuJ#OB+sghq;^R`QicAeA77xO}9*d<8!@$xRPqE8wM_FrRtZy+YZPB zKro$I8T5H2;dU{ss&$LBUm%bU2C6~1`ozdeU8cr5JGMIM#2Um;xx^`YI9t0}jZruz z1j38jE2##ZRm~&$Is9n69mwl*;$t}D))9J#48oLSA;Kf*Fit7F!boQ#)sLVoftd`^ zRAQvULdBKJC#c-xu8$>GT2gsM!ceBLwP}4OciG=y@0$+Q{rGbjNqb|+!3e@YPaW}? zAkU-pRi>(Mr6-bH_OdIV##U-x!Yfq=w6F9TZ0r6FNCz_Ijg{sjOvJ-ML117SQt34= z7PQ1KHL~DLtu8#;?aw#WEY}u`JeFyF6~ZrP>qaiF z4a)0Z{MQ_UyVu59{clpH?}0~8wMC`Fw(mR!4`cg*@-!-=Dc@nnZk|kFi7^fN>wWu( zW4~hZ_AF2tPrSiW)^^%1pgH8T=>8naBQmqrzeSsS|8P$DPk1{QcVU`W; z_q|}gP(GP_r>K2l7*30RhRVk&oQU1BiYeY5EEmZ<$i^!l%phV=O+n*=*qT#F18uEb zYgH#-N_!ChzChw)DEPjH_@E#A+Oi6#{7I0WXVg11rlX`ET_ojWY|Wl8OzAJX0i!9) z@De-RGZ^VA9c33V4(NBtrEyw6Yh(PS^TF9uY9Xx)+G?&r-K9+x7KkpBJ;Lwx*C_iLg`Rr7f&3d# zrp1z~|EBpH!$=SEfdx;n;@oFJT1oDyl7z}Hdo36Ol#^lkj&-OwPx)cW6hP^x!LMj< z+aI5Wi_2*0|K$*Nq$BZ3qWI=kN1C+Z^jWi^1)ZCDqWC3y8L>gcbkT~eHCvGMotDNA z9{c{`mahYm>Kusck?;$P>>X7m0EAbPawA#&<_tkTLj>2W&&Zn~aTXAtf%0=|Jx=J6 z_TQAI(`~N8uk~|~DG z{4FlOJ6v03i3^a&#jk@38F?2z?!TKFuemE!mZ2rCmQ+iidSizLBKb?5a5W(eS_FHl zwATEf%zVcA6VS71x;XczHD`xfP_Q741e6)`4Mx)_i*8TcOV3iI7iXwkxA4U`?a13}DX)I0T@`p;qz6>x zO=fq(h}N4Z>uZ5Wy2dkF2Szy!s%(HX5hs7ErThm{uTKTa;wrgW5O)%D=?s$71wPJ(NTuWo((z6Mw>aQEQ&XJQdiP?A>ovuTvyv? z>n5aZS17Ji+5YbEcB0R>ZQvBqlXrIB3S*id1aRm_@dca#0U90s@OAMmNT7c~DJoH0bN z6Kh&@4SlLXnPx?gE3=sF;9T`c4lY=n`ALvT@(ba*m!Ed-3(1!>{?@Obf5tkptMSH zi%RTy;Z;F%V+%!lTfiGjd&p^Z5m)%Xz}6WV@?CuS=}yJP;p&OD$S%}ib$ z2P?aOc>CirNGqxeG45;d%FDjkK(D{N`v z=n@9Uc)^sY6|zRPO^_Qfjt7icuDf&hoQ&}ZgpYGKYd=((D2C)Di%nK;ykFip9v)#W z8~Np;QjmDM8wafh>&a`KdP)!4FK=Y4`MA{Tvd%`> z$cxz@#X_f=^21txm}cJ%PbW{rz*cpjoq7GT zG2owY1)emUfQ&L&}4v zS2u>2Dp!b#$Isy_>DiS|nDNAn$3L&a6CKRCf!=*hHSk4xF+I>2*W#xnogfz0A^hbE-9&*87j{Oj$IZwRBf? z!uxs7<6mAfsOGau=uOyxXHFV(svYbwwvpTxH;CIkFp$&>qQo~Go7x=5g3|hO%%Ls1 zLH50AJ)@|fJXQbjqZRQmw@ZmgKWC3K;-+%K0(_cBdE&|2u<)B9J__BBlM~KXj)~?6 z-ntFr)xykW!((e;l<#@0QN3KZcsV`uzWf57PiqYEsl#Q2?_n{=*M+4Ot-^xnG@!m< z{lB#5)C+LD__a3W@&uf|w@45!FB2=~x`>N)&behy;S>B~bUbcodx zZ?V#En|RpbhMHSh3{~J{Y;BpD=g6-m->2sSCvn0u$x*WOAA(Oj?L{TlLkrJx2TJ}owf-fMj`S@~)AJ45>fqZH%w0cs2-+tJO<70-X z{^y?K=kjLZQ`9_z{=IgRxI>2Q&DEAzjHY`{IeOZfunAAnsm}amL3Bs1?gP8Ayc>>u zWy+1SCc#!{D%CjlC#@F~^X>qRO;GiGo zSJthwd2gI~+niHBscT2VF|62I6MSrTqrwrw7dTqcOty_2)V8E3pKrUNF6j8>e{}R=bV>)jE8f zzd-c}*o0WYrSNT)@4eIarck_iEqMg@rZcgqcR|%^N$*!^Z$AbV9}ynOlLd6o@+2F& zu6QJFkgb~B$C@E6!PhwweP`C^wFa9hev=PVQDKnI$EJxan$%FT{{sUUv#_bLAiuR?S8mu^+|< zTC=Tb$AqfOO@q8qec!big-Qo;;!NqVU>LMa^3nC{wh7E;I`d{>M$-0Lj93tQ3(8k6 z0qRFQQ2q)IO=>TU-ZqB&JFmgIf(}R+gZ7hNLvq;;ptZ(;A_u%!QIFQ;l5TLls7elU zfPAPM)dnX|?ZOowExS^MKRCA)sjgV(pON%jTC&msjA~fS+_egc1F`GLEL8PESPH{; z)q-K~Zwul}_Ta)V{ByOnw9T6j#AlV-AdB{ipxzZfOm|_nrWxstzHpSz8gi-rSdfOQ z)FX-0e`CXWRrr%8dJx}yEp~Wo!Y|QtT59cFy^Q7cCT5B&k+>39M1I7ALCHet9O5Z> zy5$=lta=w3&I=KW8^(DTi@=;_`1yCNP`YAA#8wvn@+qS^RveE#G$U}^_};L<`x>L3 zf!Yp(pw-X`;OhNGtsfF6!LP>?L2=TI_t#_OBQu%Pd zc9>x5DG7^pL;6_~2h0OM?=f29WvMhxjA00_tZj}aubg=P^q2U0|8I;+st;7BXfnGQ zE?#OZsXlezn+*q%V2LaJ8o)rqI8@s3*ii=gkAGrVTq&F!VJc6&yo-4;O=R`v{g~pL zmAAujid7*bc+&Z)`O7iJunOtMZuFi-P-p5W+-y(JIiw$T`<2?c2P|tp@?n_WI+$>uqXxz-!c(?YWW*vKV zda%;MsI-ZzT*6l5)MJXn=RL0r)aSTy>OMUD{;0w>Otw#e-qnY|5(gWkJ_FyM23%p7 z@&t~z596=Ik%Dck!UqN)6NI^HZdrImIz0Yy67}P4;K}ZKc<#-) zns)IU;Pn*p(t%eXd+tHzQ{)PE55jo!-@jOoiUPDXZbDp?0-2^-__^F6ll%aqegV=- zIAzd%;*wN+o)E@sdNq<$to)ID1YWQ^s(cSRRg91k;awQ5Hz<8UI+3YGRHQk|y!WM}9@c@-?>G-M%+|3FV)9bAyE!;+T z%l`~9J}k}iOiw~f^I z=oC*rE6=6RT}g8u&1#Cna5SQZ+`6DO{BArG-3EQ6zS#hsY?9bVjT!s<`IaDGrF;*h z>QRPeco3B~u`eqotn7`yhx~xCu{yS&_BLxWdkYe$;hHY~8mbL&HowN&45*JrZ+GE_ z7>0?{7c-lX9k4#Mm$u{4@1VSp@_P%vt;DIpwvxD0=cIX#gxyN#v-5f@RLuy|M+!mWp zFGJ#|N^b8mA{%Z+UIJCCgu|?~z!>k3>5eoWTo&9|X-6D1>6W;6w6!E}hg5TdIF={d zw^X>sI?kQJ$ZIg_QQhEsD@ACU4x68;$(62Lto!0ZeNNoDS`=6{2f`9eoVty2+H*{4 z-BwTfW0&V?x{$rT_Mze-R1{C(dm=r(}vBmMCS z$O}P!VwoU4T{$6E2L;5Tvy*KJ^|Bw)~{{9%()C{DK&^B623u+~VN9qeM^h3z+HIo%DQ$Adkdt zH?Bp`_m3G}1M~7br6G+$HFIA0mll$)8^hGR5q>b@8c}QTI(*-^C;V+cT1@N~A!t5X z&dy?ySlU+kN%BFlnxScVY|&7}-M%(-?{G4vM2^6^QOQW!Qeik7=-!&R<)8Red=+<_ z?|{=kdUB<8h{G$npS&TQEZE6keIBg~81oN{Ap|mqN-Wj;PcDR;w zsE)WEvft@}O{gyaV~w8n7+y7I8>R#kWA( zMfEz6SEf8?A&}QnbAg24G`~k!je_Yi(#epUZoiI{0f3WcqNEz9u_Wn87h^wTQ?T0Q z0FARsfHV~J3@?Ha6_-S@+fzn8CPF5i*S620^EG~#U~-w2ToZI3NCz^yZY_CYrZBK? z%v~|rp}oqN6pzcW_uV0?NgLVZRcpCGKBV;=4h2?wNS|(C#HpA+eKKMGV~lE&EXV^2 zm17WIC|nWCu7oJf0#`%ZBVD^7{UntZIe7I3&O2l#=vr`c>1BK$*Iui%JbgcsHbvqp z$e`a#dly_s!cURAZ94WY@DqPe_h!VKkW*Z$G7?n2f9#0kDmTO8MelG=>@Hkh=Afm| z1HwJU1Mp%{JW!scu9LoFGm>tBn5s|3;F6bwO*N3P8_7?A(j`h)td|4P*>{kr;ax#< zF|3kfr4`u7&x zbd;wc?@V>CL(*J=w6!+3<}!spU_Ii7u8vhO`~Iw-IAd=mAJ4d~B}^heW}*BA>vAX= zf9g&%(rc`lZz;SwU5xa7Qe{iz-34hRkX8H2)?Yl7W@A$g>oUq86^1F@L-;fYl+N#G z`cAD4kY1+zPhS%Ev*l$+1Z80=`vIql`>Jm>4d0h%5GFv=?cH&z;bxVciKSLP_%6*v zl3qdbS4cS=sN8_O8I9#cr8Xl?#9`?HLH%BtZGG<2T&m1ttKJq+e3aEU0BQZ8`iQ&Q z2N!}NA;w8ey;2Ph+i5CwJ8_wY>KIOip4L(A$hC)7h!~G|l)crGihJRU3;wpME@@2# z@e&`|>Kmh8!=qXC`B0$S{oPwDp1{6C^D{}C0PD6(t+7;jgwoflmuOuo`SXTnCc6`N z8f-UKL(&RRUUM3oHtDLE>DPj|YYWxhN?n~&3O}t5A?XLDt;ipZlxv%BmK#j-aC*rD z*nGM|_S z;y_4=yf2mxDqzH$NZN;L>L`}4{DT*>XTrg&eSz{;Fm|brYTZ<}MmmVucv)1A`|syL zNSuU}2QlScJmSL{)s>vrylG{g+AJ&_;(`jaS5kX`${;-Bmcr=YBULuVsjq2{8VkyS zfOHhJn^2XlZuNwG*AY-zEnSaRVHIV1oG@Mc|6h(P7XUjy<``SqZ={qVM3D;iffg~6GY@!OGbivROJ zr0`6)f9OHbuh#{!v^gj3k7GwT$&%Hd$R~Ui%I}g#MapYI=?V)s7w-Dkhd0%w@oU=; z;ILU6;qveK*s8N3dw;hrpI7^p=Fc&E-m%I(*!-m(8ve`_^Vgfpx7nsxb#*8np1KN* zYK)M3-&dnOj2!ss>N_&;HY{PocphfoJc0Q#74+hqGj}opn}aq$W4jK54Xh)de0dK6 zr7rk>gpXwFEM(5qIojD%Ot^0PaA+w?VAw1RzV}phzOZQ>ek5ZItKT{t7bLu8tBxE6 z`hRV&!%o~R>p6~g`+-;UZQyAaOKFxhk(-@u3;44I+v|ErG+?%9kakP#_lZWMbrS9r zEmInzG)b<~-|c48+BrV`XWz>3Y^+p8{9*j?S7)`at&r#W$ieC30|N{ZnxG ztqnf6oX6HXEr4lv>+vpeCVca`*D#@V4w^KqD%CtZ(AaU)6jS;8?F2S{lqb|Sn~U39 z(^*ubJS8sfDzoVvF4=Xpu6~E#og4PO1gzK*YQ7AVcBN0@&&xJ650`;n&wmZFmAkJP z@QLrL^IEB$xHw|Ki_Z)e&5R0l{^e(3@YZCx)8q$xcr``VeECl7ltX2!s(&&!yYhQrs4M`Zt~U~7p-m219J9YG|_K7 z((mpjU{^XG+i&@xc~^G=AHK39uzS<-hetU7?{8yVM)!=*v^aw&|4_!6{6mcXuL`SL zXsv6UxDl@A?+5QShoH!A8~sl2N59Q4z)KTd`RE{lu8!;F_g~E++RsG$FES6eTAl#U zV;6Lf?r(&**Z-mT`v9G5brl(Ri(!B7UcCR25mI;Rr>;etVIpBGozp!kOoWfG#i?Gn zc0+G$@7tH3n-C>wej#kgbF@6a45?4F{y%?#d&Os6@9t_(kbKeZUkB+^)KeCpDTX6m zvS4c6zTh~cSY0QlTGkYpT*I(i-tu_KWY+I-Q<*;FE!6s6h(7+Q7=5Fa9C5Wi&KlZ6 z%_YpO=)oFfn&^&iybP|+m#Ak3iC25|A+&l9UTaoCbySbjT*+UzvoP-QXnc7mL>`K6 z$^!zd=zOVAINRh9biTDpbZhD^TeP_={+Yjn!A~yB*4vh|Y7294_2NLNakCe%ZLHxv z&K+b~UV%W@46A?ULC(ut*fKB_$v*Ofog2zsq4D68AB7j!=d-;!Z>(N*12gIx$CeJC z4>!KH#j*bkc#m#5Fe4|9&SauJFrST3f5x3IRp706YmgfrGlz|xApISkdAh6?{`Yqm z)K6}~U8ZLv^{g&w7VQJMX0WFGLpk(6av$mSLSASo_q1Fi&P}-j1J*5rZcWdE?zx`a zxGhOD(&`hqrX?f9cf`q69B@i6Kat&kEA+D8ENK4t$toX3`IOys)?Fxcw=#qtZtJ9~ z(fPNJ;G=*JFnGcXe9iu_ePLHLnavq@JGzZ8>gWvm&R^i})-wp!`S`iUHh36P9jM0H z@3+sO>S}#7cjzyv4)9u|?zm^pQ%G#{LG!aF-P`mji0i+m--5mtf*EpfK zZzkT>3xP?r2iNP9`(d=(LRsTRG**vDrG1vPyx-&nr(+IA^0oaM!kqNh+~w3Wv8)-F zck%$1z5A)%RV5Tw9qGV2W$%%BhkLO3H4kGh>T^{CVFM0CG(o%EqZ+dA=fpzC-G!--9XMwSoRl+v`U@D4fi9_l1AvfjA_^m;b1}R9^nh_@~92 zF|%<9gu0tb&Cf{wBdZ0E{3b<}TWR2yeiKhtsk|2X_q)0LkT@Bwc1!|Q52>~RlJJi3 zthS`N&a82x7T2HLg}*2fn0BX~Al?G%TTW|=J$Luw>YA*d%mv~%v^SWJi+kL{U7^in zqq;SP(U}mb{_euBYs6E|7-XHGv-KMT!}aTNoBdUA{J4L@s=*aZO*qT1KU#xJ{X=*I zlc(Um(2AQ(IDkI+HRX!zvC#2PoGzu5_AuUF0ONdZsGkkwD#z<|=E5f2@pUkGPP9>d zEiGp!=voHq6W))9^GDmmr`!(;b8%4NI2P>}fj0X?fL;SsOS-LFV)@W-ui2BJe3<>_ z0FGU;7tR!CqvN!l(r`gzK6t_;`g0@xZ%rN4UNGlXhX!L;k6pN@r75SrWnYRL%1x!~ zWS)H+jdsH|`h74D)z~|<{*0e|`qMoQ`&Iqn!yA*3@K=2{B<|mW(?5NJWIsAfVY4SE z?gttJ#62!$wNlf?@2TEs;dUC#JbdJBBPKiC*^19rbdk}6DsbxP4@heZ-jUCRnaAbK zA09@seuu@fxa(JZbl(D~*U&!WDjr?hS|01s2B~(~6OB#t+Paa8l}?ufQP>BcZ_1f)kd}{ZAJBywg=iyu*e^ z{3pB?UBn9ukBak04~w|EHX_$}A2eL$h0VfFU{Sl9?BxOJs_=Xgd&#bRsUi&)--e&V z9_xr_)#pR3|9i!cOko!F0!(sA#k&)Ji!H`|kZK9py|>zKgE;q}vk%UvzrwiKu56s; zBe8MUlgi!*2-qQ->N<*IT_XtTV1xA@?Gb++YD%hGr5E6@Llic3ttT#4OGjEer1@53 zWKY9CVacpSMz7_s%1W@^(%SN5a4aayF?n%;O`LH^y+%Z~(UViIzoFOK%1=H8u&`+i zj0rDc6U#2ansxrzO8>kVxp^C&u5JYf^@}kxZnrLf?Q4A9sVXD>6lsGuz_w>)tnm6K zFgioe+m6gB9P^9SI?;D;f(bV3#Jf#7aH@L)x#(D|Xc$WOTLkRD)eD@& zvc1FQ@QC-=t-n6DI&F;;lE>?4?J?fY6A0H;AL(jt+k`a+O~ts;7hp%(P8@YmgR}0@ zZwIYs^TYq0$M_MeF7-A?tFLOk(xc9vT;lo zeYyOZ0n%rnb9EzK(dhb|cm#IuSPWgG=c}~QAomY%d3_s}Kzh?SL>sTSngK_(mVNH|Ct~$~mbe zEYb}V$MAI2X{b8zEX&1EARQtIUj%V43kmLqUE(6pzRMB9{zVwP^ro)sh+g)?fzwU>a_?uKlp>~X>Wz~PZ|r- z%wY0lwWe$IH_~PI*!lD+Kp24!PFLfdx*I4?$MK^sVo=f_89qK52lew(d@g&RFVqlM zi3!P1sW%tESc9|pp{1{?Z^*biSf0v0DH^WIhNkrUS^LXQa@p>9AiW@8Y;265H*}Vp z#KI&UGbHg5RdEFX5{dGd6M-*mvI53dr`5cgD zLed>b>n=KZpMh~(y3y;7gW`t`E2{$OU!_s`laC#x@*v|b8$qMd?NND=ZY_c#yT2u; zng!BZ@UHGN*PZeXqdI05*pTNmEmC99zN?iCYYV$bz2=2Fs%Bu@&V;FP6bcHiL*K?j;qHXR`2Ox8SlrDD=GC5ptuL)sIHrwUI0A@Asekgt zMk_DA%F&Pbg`T-gv{m@cNar#&mxSMJ&1yO`E@BH?w)Y-k>IS%zyF!rH&|WPX1^?E> zK%W7{Ncva#6!b55N8)tW*zTyX8@3Mm+lAp?-6d9Zq)d4#h4&cQW`*#YXDapfeMJ4& z1C%C!MIBF6YD0RD@??TfIRUwA&f@%M!FsMo>~ty4%)Bg zA*_x60;Jb~a2`$G{8fC$ss0t_qHWJsbbe%xaHD%R)b(5qH{`R5R|@|*EjVF`pk5G4 z6VWeQjPMt2<}MUptu->Co*#Ow7otu+og+i*K|Bn1ddD+A_mkN4%zHd>@`i@yp51%c zg0E_~iIKKrON*LF-*L5pu0?k@#()!FV?+A=Wj5{4*QP}ch6}>UuU92+OXtk{?k6k` zVFOoZ5$|z!f&U&if<=Fhia)uAEma`<7Iz15z9|@0{ zW!Wmc`M4AIx%EXTJ@iyt2s1tPpsr;zAZ?9H2Mh6KbwYSas+x=IZQC+&=|rKB&hh=*TO}KNHuZ#%BuD zT2#g;FQ7RJz(bpdp-ro}N*<%N5ScB%u`G|{Kv<_dJ&zKGjQoFPhM>56sectt*s63t zsB2VQF@0u{Xk&YsJjP(9)0lVUMD(Z@r#=I`&3uq_CBL7PAgJycd0Sp(2%TY?-5Dt> z;A%aq45$a!EF)1Zi-jKE%tZjb)%~vC@1%JPw_*n@{@trGqy}BaQeGO^mIf{dkycjC&b-sNqdH>roE#)_r*HZGq1aGgt zBkJy60HkS@7vhx3aJ9zu#+_lx_b5+#>8X_nvTn)mJ=`Z~jA&HjK2EK*S>;TUG`*aW zdlzTq4$~<-r!kWLZJzo3vK54`wbK@?a$w~52_sLz@@HY#HA}I1J&+EDMLtNKJ*FI2uqt+(gMCIXGu6CNrbZd#3GJlJD~vKN${u{+s=tSZ z{GXQcp~@VJ@T&Lt3h$?<}azFr*@&|v2L<)yJ<=< z0QIdPfH9~s< z{LG~15;h`bDLTq9c-$r?&Rue z^`CFkb_~`7!bKfrHoVN)k#8~1(@nd(g>XQ>k~24MYfU}-21xs>oCIoG(LTv;&E>pH zmz6KTqrXj6uW8%D$E<<7d4ftcrsbG`qhMQBS zh;ww`ZP#F95xuL1yl`&_v@o3_o1_+M)AIcIcl}fBy^oo=`*SP0U3`J!G3^%_Z2-3e z&am@|&!FF`jbJ~G%P$r4xc9c9XjUggu1ll)AJ=D~%RLkRH|7j@u6e?|%%UM}Lrb2! zc{l8I61e1FDXwdE91=Bqpq+&uM(2m{Zoz@*7H=p^bOXS%evX*YZxugPbp==q?jvb@ z&>(6O25)pK7oz{yJ)=3YRTXBLwReDMzAhzvnYPE7ai+GVzs7SU^34~ zi@|x!sb`4Je&tGh_%vEz&l_0pdRrc3aRD+T>hOkP$6)6yI@_gjoM_u6S2T49!SkJb zV0K0U_Vtg0(2Mt>VA*24ax(|V{aPWjN4~+*S;OSE#V5swy^qA9kF9vyeil4F-B0c< zSqrga*J!4PXrNVtQT+3e7pP~}h?!q80uSSS^&K$6wp{zU0moyl&%m>nou!LkEAVcG z@=x=9G{=)<>aoWd>HYvO>~xSdb#rmz$749|zc{gAytOReH5tP<+wq2;tHFG6FTQwv zcYyhq#HEY*?C_8vY+tya70&TOx^6MX^D&%rTgK~ky@bEt9l_^OUa-T`Tz;ykm6Mm>pty2WHv)@ej5rEbe|&=Jc|W=H>bLkSH`rJ=2yb;Vz)DN-)d_-pEyCTIMe}L*5@J36+H{-Dl-m8 z)937wUcQ;IDc}mO3LYy>^6TTC>x7J<+44IFRo@L1&ntY+U94AkFUki1d3Rv|gO4GW_cB5A_ z=c~2hQf)K3SLO_K{%6D@>N7~6_W<84bd&dsc3_iKe;6`rCb#TtA*p8A;R7kK{NzM_ zE0jTb>rVVsMh-l*Gz5cfPa(Bpr(B%47S^m8BB@T~qt8`!2Y=m%U6ymv>vm&qFlPmx z+4=$JK8}*73Tr^u;MFk1sRyg|^thm2fvV*(y8WjkF;hECR!zFgKI|PM^GNRxY>|&N zeysk}Rj&QSb=0TwcZL9FV9%d7Y9n6lW{`Oa@$@^5IC^#n9HujrJN2B12kY8!_e@9m z{lstK_vWkrq;64)_B|(X2m;}nRDs39il^X@Me-Ll%I$dBctxa-o8mpVL|O#!OV5V zHq0^jg>Xm<%}v8#MU&pR+cq5icKsF0yY+;WA=Pt7g)Un_l}W7!6s-ziUARiD3qx{o)+#PhThA?`R9X_78AO0LRlJg&b0q3imcmD6O&8e8l-SPb zxxWo~zt{u{OJe;OLcoPQyuPNlBDU7e(?W=m&9 zn8~w4vqj#rSlHO681!3tD=g4`80R8OXWeCq|JBLdcuK=F3TMNU>udPzjQ-f%ryZx> zXOHikMKQiU_}3W@vg{#@j2ZzAO7=+C6%BB}7Gp{Ct37hJDJJc$%69e&WgG6(zIlb8 zabM>#e8-i6e8u!tjCz@EH>uFhuDV_>C^IC!xCsu6I>Cc2#bD&jsYj~uHhJT5tj3s{I&=OiY%YGeTSrpAF{(*^%p+Wz-zpJWc02$R ze5^6sC4wc3j?&EVqxgKFx%`>+sj@fDv@Arc({zvEq}lkH&cex)XW`Z<2VR!D8{MbR zN5e=uSJE+(#{EY9nI}8+aFQ9^H+?h%bsjs;@zf^gdRu>ttFPhn`J(ee-|wfimR_4q35!U`$@t?ZPzj9P+?ZDKBE*qFv3+v>kc)y)MjP5O@OeC z(f5;12*W4U)3M9Y5#sBlk<4aYJ$cRg1*lp~Yiy#U`qdhp-@}Nj*(t9>Q1{LloVJ7R zNd`-~$~aql&aFb6jN73je&=_xTasqlh9>#33|!)1*!5b(P7^Z_f#}byQz03N! z4q>)u-@{tz>pFkNZK&ew2}=_{>H_+v;qb1&R6W$a^9XnLqTk~dUj@Q5etY9_Y`^j~ z%)8_bv|cbarzTijnTA!9hM{J1fAv1d-*zF>cgg{rZ9fA|14hHx|5-ZoxSX~xj2lui zmLVjm3?ZbDdiUCfGAkJ~lX*xokD);&NkmB!B1w|UP-m~5%#zGw$~A{f88iRZ?(_TO ze(vSg`<}D+TF>)+)_dM_mX8oOE45{A%XUIzcZql3Sn>@utLciWZ^|C<@;jT|csP>B ziZ@PC8r3P#f2+Mzdxmd^pT?x&u8ew~tGXfv64hNY*`T<_g7!%H0Q4E?Bw8h&W^3Hm z3ZwZ^%yidy^0k_>r0BUAw>S-&7&XAwu0zSQ8{kq~y8k0(yxR3&z_gz)aqYFk5Mo)z zl^c(O98Sj! zv-LvpTbaEce_CLu`q$}k&Q?q@yvbo!V~iUF_BE{Fi|#>OT8K-67zu98 ziogqxwgc7tN%OsCyxD~&a(neAXw|}&kLwz&BTv;G{5eh@nNdZ033KTGox`g{5ETM7&GDupSB zFXE%>i%{*Is%g?n)G!!GOM%!0Rj(=5^`P%-_p;p1#MEf4mGW0{6`j-i3cXx*mt*b8 zyJ_zqkaj*4AxB2=f=gdnA$^{s_CEj8X7ZgYbp7F3K|IY@e0&XSb~-74$AWwM5Pdrl z6pJctzvuE@`4+tEcn;`TVXL`?+_$SQYt{G>yZLf595?QkJ>*9RPHafmne0IIPZ`}K z=7qZ%qkSV*v7zhIBSrmBNkB{pu3x>enbQeio9n>4pV#5PKZs3w8;HRvbXxiYEVV_m&fZ1n*j(pC#jnxm0N;l!e!VvXBB#U=P-$Q>ur z8{&;@L2D%rhVMq|>Akk@7?Ip$bp=*{3;BZ7aReRPAQa=T5WjW0AB3 z`@O6O+qFlm>dR)N-_7L9#cLFUL#HP@Ag|th{I#MUoV1xLRWEzL+6{xvZ8+6F?%gy* zORHTEyv??vVlMI@jgA57_mEzPhxnx6%QKFgSX0{lIuFFJEbQJl)el%{S{NIVmMM}u zUZcKiC_Vn%gpnR+;qm?}@bAe4jkpI@EqxnzK-3I2<5e;aX^J(C^WLzMqG>p9$t<`( zta`=qT1Drp-h^o(m!YPfPX&+Cc@tMgilH4Wxr>hjpKn%<#BpNsg-uE`MWun~pndU} zids}Urq1>Hm?msc-Qz$^3gk6Fj4O$I7_lqZe5VZCunNl6&TQu8&O9=Xu60i}fPCOZt zt$3Ys9oxIo!h7#TvryF%;>d{j=d%GY4Lzd91X!I#fHPF1p0F;x!o4(U_Y>qWNNGaCMJ$4w6nZ0AgWiWyB=l4 zQ?lo~eR#IcAb8`{kiS`;BN75*;oDVbm~UeUl%+ZO4HB12%782~B@k)$Lo<*D+M?8^ z@FTvi%!t@u!84=}u%r4Kai&c@pnk+j?T*9#iOpbsd@E48NEwEnojwlqXLaJF<=XI7 zJ#j{h{lw2l;Mw9Gkp8LxsNY%kaUGCXpw*}Z9Ff);U9Z>Vim$wC@1iWR5P~l?t>~3m zVKt~1^_jKlCXT;oCZuS%dgEC;- z`84b!M*{V)+CQz$t9CH%pp|@G&59}ScX-jAQ&yH|uUX0uh8K}o0|Lvhsy$#G^|!;t zb3+s(q8-NJjH3ovrEIv4GAq=YJCg0Xf0>c?!tm-XmER(m=!X|k!Zq&a{fe`A!Dz%Zo=9t=5^O+6td1h(P^gI}Zh-nfd{j5>GHOl(%VN*%Xi zPpZ8rL7WU!qe!(|F}4`idKfp*MrMcQWgBYdr1S z9Vd2tKH&Q0LNNx-yRf6p51q<9YIa968s5BtlThq&$;*>bcF+;uiR_C&vzHp6v_N^q zsU~r7$E6c*r5u~9`YTh*L_EFY;k^!Dv(G2FeUSLbs$3t=2E-X*( z^Z(qniP0JhgSdUTF{Xe~et>Z$`GR}{Rez}r;*mE=`8Lul5h$y&8PU0Fwn_8VO5}Hj zPQ6IK zKVI>bpt%Fk97JsSIt9pgE9RE{O@nDR_e0H|fND(6PCS96MSR{_E2+(M#lB^in94$w zm%w1~Y*gz*451@-L$$Z+oN2Fd)&dvOg|TcY8l%d+XLcSD`)@dL@;s!ujG7$_wO$YA zc7b|>Vl?tv?zXHckDA*<^bb#9N^huN1syA=IWI)aGNsIa4Q_D;BtON@ zQ`@UK1yDvpnh#SQ(EYncUl+t^DnFo_!4RV={gt$qNOLx%e2KK~tWnd;jLwOvnU9xS zBOdl^FT2*(pSO-3A3BRjc&Ag3 zt9{TfEeFj_7BlsCZ|%R*wE(t!Qiob#cYTmHD`&2_NYBvjQ`dmcd)-Tn%m?glR0F0< zV_jyPo?OykJ6~S316v+gArBtEfmhbek=>0pkT>iC=Kmjz`(_OL*No!k=`Jkqz%Ps) zW-agRi>2#i_2I+m-FRzYT|VV&9TE1+QdYN}&GWqXiNJbW_|;8zPM7}LNW;rntZ1VR zX4M*n`?j4GHy>_?{kHY-Q2Bi5@!U?jF6zR^Jx)a5DWgRSJsU{xP8E1i>)-k+=D0QH zHR;|>x4eG=y#}z`8jrHnKSR&ECUV3^kEI)h`$yWk13R zPIhwh-uklA=`W(rsd-R#W+gY>e4JUX=*#QL&A3Q}!No)i_(s={o}W^g)+Z3(Ja{dv z+~%S4nLW_=N30wl{{+|cyvR#_{Y1?qUeexz@JXi6t+Vm@q#0G2!_ny%C z&l))(#g2y@Vbb2KsWe+>#Eky41bL|ou9@2$^{bo^?Zgxq*JKUvWqkvNuWyD27TU`Z zo!)`xkn-#<18T?!{oWY<+ZWCiH{);Zm+(&YHp}mQD)F3dhuPWoPoejN5@@sBMdFu@ zFnN9o-Fv@W3mH(8UtNA4Yd6{rRc6$bfn8Ti`Wd>HbT>X`S1`@>3b9>44L(RuLYE^t z{^!&Vr@tMIdDGXqv@cT_x(o$%e!q1NVDhDg>~Azh7RC%gsuAGl*5Kt{gLsz#8}RXi zE$W=*+^;hr_?$K9+T3OrzC6bQ&5%diH9@yJdU8yaZZao$KKz~eTGjnYmwtM(<*4Hr zb-1g1YTpdjJ~WZtmU8;s`!>*;z{nXJQFH%>qdP{xzTBPIAx+}1{O;MW`sq^-pT6fguNSlrRjmaXh!$3NsQkh z2KKGYOU_vF3pIwql#Vk+bkkJb+OjH~UQdR^T*K6HE_|JTb-DfQZ)P*sT!PnU$iHv| zy{A25W8Pkd?R$)1bDWt;!7OUz*Hx_GV~UB8qcu9784cYyjvrXNhkuXn3+lMYjL z15WRvXF;9B*jZcAvC9ge+L7z7yW;OV)x`1MBVg$(dYY_bN3OpcqP&Ka2E^-c*gqLFM^KQ>tNM>GwIQ?KR>WAgSXj~1}An+;#TbfazHU-$3`bk^1=SBr12>6xC-1|s z_)xiV4tX5ir);-8;(pmn;0%Ns6~Vy8boUxFCe^^bOU z@?Cmf$3j_@(1`Yn?u+PcB_DjO#2W|C<7(fJo%)Ew-WBMkC$-`&dkxSH(&KQm-FWz1 z-h>-0NM&a775-S5DGaAqmpz9kN32y!ED?0Pc(r&5(s>JqhwsFtspH}Oy)g80 z+s;>)Fs$Jo2fNmem4}iK!WaU0ZTH{}5C)(c%@yqJ-^0dMX~AYa^R|Ho#-2 z=2!fq^v`x*43Oty(Zdfoy_N@5?mHUax4te^PlP`(k~wZMQ2F~-;s2=^jCRrUt{!?r zgEuZP)RykiX7eA!QVw6d;vy)2TxRnKw;8{L_79`*J%F6}{jEqHcUHEvDaPVo3CjOb z`Jd;I<=E|siDkXH5kk<+TzREjrhWy_t~&}lR+J) zsz<~4$9Uvhq=+y(inM?DTXzC0E$qaVpQYzCge^ujnOC zbQDV$({lx$tVLhfw;J^pC|#nu#+6!i?s)AIyiD*R?e=5y27hA3zpUVN!=`X~R5w8k z!N-Qg0_hVR|E~oWG}mz5^(*)xsvm4H-UL;LG_T-}9?#Pmc_}eMEb&n(6g~7|hz^{ko)pn@%9viAhp@7Fws!Np4a{~K zf{$LDfDUP#ee4^Eq?PQU!6l*iA$f=;@lhk`VizN3?er7=)d#ZqpVw3GH4wW)_u`4X zZWuBkjCEh4A!$4>U7U@iJEC!1p(wvMl{LIQ3XYBKDfbTDPu$apH!z9QNO#zlN!J*8 zG+r0e;CqucT9jWgp2E6(?$?)4S~nCQHtQmlc82){iA6Q8GvY$LKoS>=_hlJrfU_ad*aKrRVCFPP%lX0x{7~a>3Tq0aPg>6$M`EhM{yDB z==}rq%IA@P8Ou?30)hB|O|v>f`at)-vi~eDe3=HUzk&QzCs;SG$zNdG^YP1`*UT?) zh9rHXHFFRbS6tEtnkJJM<_q#9`n^Ru;nS8WPDz`-f!O4|7ErW6BVJSWgNE^*{NjrK zNPRDdHfarKov%U@%N)gLSZRHnSd;i2#m>soY~FrGeXVp(QXj~EVSnj$lW{4&WK;{9 z@-tdjC*l^^#=Aq-H~Ma2Zmgn@|Hp#* z2NLf+1@Kw~rvLpwVwo)R7cp(HDGn>(V? zXr-GXf`&Scnaz53fYni{o&0|1NNEl~4E9@qtf^9E8$9)t6M4!Xe9F zTN+r0P5eGrte?A#kqhvnp@-Nl(uHQj;(H6r^;dLc`n9 zwQs7Yi8s_SvhmXmWy7HQK+HfM@f=9ci7$?$)0`Ng7$uLc7a@J3dv-6ukA15$+INWR zS1pJ19EqQFCmqv}bO39tUySLuD|1>;L2Fi_i+k+K*v+y)d86kg@n+8!{Mc!`(kWR_ z8fghE3v+EkpzV}6@7WB6M}Qc(xg;)O#P>=Y=zEgc(A;M&@v1pL@6eWY z%+^!fBgmt%bK84Zw!02a&2x|&yPt;dYeoar5>O_g<1>(>R}ZK9nW1(1-GX~f4P@j2 zIONa~T+y|a&iPk43(GDB(hT$)xQTpjHTVU-gr={1QU+Pit_-RNE4n7K{3266@n9RC z^gRl9-rFFkKR{P+mM-i?SKi|0P-fMDo{O-r47Ln-2Ir^v>z?#C1c!u%+{VKm2A{Zv z)|!DR40ulYrIqZNHAYbW(CGc(lG}VF&w$9E@lYOLlYB536?6LSxQQQUe_*}1gd?_I zIKTa5P+sOxHU1k|jL!!=r+a(W5=ni>GNa5qx`ug|%7zvB!F1&S{_n0QZWuTN z)8e1PkAd@yHbnIZ67_Vj3x`&i2W&hTLrDEI; zJBG3u(#vw_xBw{hPBp)%R#y<01oy=THZ@NHWO+{zo_yET4dUv5*R8WKjk ziec`71gy6DCR(nzfW1H564y%*d+(_W78Zh@D^m`YpI8$IY=)GfzNkkRB9)!_3B>L? z+GC(vM2{hRuu;YZ{B?Yo(i8ArIS`V*HxviPUx1G0U4`0z@(Nl*>k3Bj&!&4&IUZqp z$%h#667g_t9{>J^;!Nd7Vn9l|I(9*QqRq+Z$4IL|`Lj#4T>P}?1YG-b5UEe3V%;x8 z69vr}G-58McgloU9kLrA(D!5x?+O88F4Ao?PJD;@cPBu2n~_2>(b>N1CucV`<=YYd|l@v^%mNXw8bq0rn0v6w+hNt#L~Gq^G$CguM?XKOOf&evi>1( zXW1FtyWzL^{4^C*e~!PLjl|Gc+~pTgrbW^-LHmrfcXD!ERmz?jG-I%4sy^GFZ_6kL z3E~%Rnw2wB55lj|l{#WfJZJI^O77QDa|tBf#ac6-G0O0G`ri>^o~3x+HHuh#3pk&# z;KV5C&oi(z6sSDCxAMa2j>n#v2ldrT;)HE<~^|I>0ux~WaMQ) z><^t8L_C!puFJoWU=3NE2FFjG<#uoFT0`D)IwH&)fEx< zb1AKbFC2N?8h5QNWRKer$9}IbiRD!utC)SXj=c>u&r#Y67ep-{)T4KHMu*xkAgT}P z`+cE2!YghZk9?k?GOVPZN26&$PAcE2y;d^}%4m!{TKjeUCe~l&sxmUov(nT)vbGNY zfb@v8bT|+z;q;`$Z1eQ9xYOtvQjUaieLMttl6)0C5Wj6ZM}2#PDJ~~o<)gOtpcxij zub(fpHv5}OHKQ>4Z!cT-(vfmOCaPRYYlR7Q9}CLiNb7+KcMb5B-$a!y*@E=m>U<%& z&IA~@_>A_UFzVEHr1^?Qz0YXgM059-G-uihPT57`jC`Y0GpTPs7Hd=| ztlzA9k~B{-lw<6$B$jw@3Jf3L2YOsRuRL5VbU26}q04p0<{wrwK}I^NW`x383(--& z0hLixhW-|`zmjG=j5r9Ch7((Aow*bu?`Jv?YP%~$%*a-q@P6L%|`%&)k0jeLx=*%{}z5H@wB%^tb*8Y4l zkcYtJxIk9($v<}NP7zb=dg?$6*0;b}(!7CXRz8j`>DeOf%PMo4m%*-1(Ll2)Q1$oW zbaS{i_{prxrdj-&}PorazK_fef8LJA|kRP8|<-Yfs+7o(jJag-= zj%F{M@+;NidhPG(JK))9kFc;{x-z{}g4PcMipm(x7;$BDCu}=q6lTnr4u|hfVzoNd zgqEYaLZ#D-L9fm@b|<`}cC*1p_I$uJxE`<(KbqRmahl-V16Js|=osA24#x8i$vA53 zDb2CV4=k}BE+emAK;lJCYl)OYD*Q%f^w&{cSLm$TCz>hIJ&iI1F&$dexWCPBT{>KlfL8261r|s5xhPPLva6LHrf^ zldbb#ubrwnBFkr9BB;4V+54Lq7u8qI?cmwR(Uj#n$%_$P@NeT*oP0%+j|<9l>}TJH zjIs&vFnYG-`Z!oy766?Fz0!znkTjZVp|-ZhqqCY>0Qm)J=hTX0Zth@Rk)>z5TxV0p z_CTt6ZRO|3+EFKah)c=f-IwTi{r2ne;O^<5=8!853$o)&6Zfu$fLN& zZUHu)S_NoUExe~qWRwL&yAmDEpZBW#PWAIqH?GfJq(1}ks7~c|+Fy*n|AGx)m|o#a zb22VsonRBbWa>lGhA>8Z29(J}r^zFrPH;3Tf2MwAYR;{ARB1ZpVVLh+AZ+(FMl~-{ z*}`-heQ%jqNX@)RlYugkqMBJ zWUlFhjT$$gdnmpW9je{OrEa_M+kz4DV5K;blUa!it4*R)^g!mCcU;#jsV(?~oq*bb zh0y2xJUq0g8FW1MN*HdtfR7S7^6~HVw4=72WzcU8hYvvPmvkLGOOv3@vxTrZ^*FeX zrSI@9UnvSRP3e9~^VyB9kq}#{E-aiFiYEJ}@c5pQxIes~eB^l*epi}MxFycQ?f`rWtrWVXx@n@IdT0rfar-&%;E@b1jxPmn#SLjiTMbq14t zPjO}V9{B#UnjD*V2Fi^+xW%!DNNWV!eav`~G4Mds2XGDkvCD3y;QNcn^wbRs82+8^q{dA5xNNLPo2d4LtoPU;`G7!;vm$oH&vAPwuWU|0VJ2z zg0~4m#FUPM-H*4@n!ORvZY98y_>GwC?J6tvqTV+g&Zm0UlpY@Qu;G^*!gA_cY_#o` zcx3(_GkqK|tgf*v4-P@^4pBOTet_0pp5m(o{j#GTCBuiZ0?e8?kn0M!v2=@;+;-{@ zpc(-u8$Glwankkon9Ktlm$7TPr*QcBU)t+O%YbT$FY#i0OZ)-3F7GufuW}KF!UlN0 zXg}KT^Ofg28}MzD8sx$T^?J}LFd-vGEhaT;#^&`{FDcjy_x`>^eR zD9{x)z;VCp%NeP4q_bgrh$(W#aGPN$;tQbY+C#by*Z@=9JIduLIWQ{p6y3A!3Whw2 z!|u`d;C{qt=^8x^9g;1iF6Nkgd7}>>maL(HLn5nD`8$lCP2cOcag?d?xj;3^hZTj% zu!2tX9rp+9o9!3YJT#0~r@Z=W=omqJ0S)@}!r+vA_>n$J+I{hoo<)uM*S>l()^{)j z#_z!Jv`D;X)dt791&Sw2{Grw}`o7iT=}>vPEpO6gq^f01s+)q%``Lh1-X^+Vs}pX_ zu@ng@FS0k8xoEeawv<&oETsLu--2`t{-mvxk!~loP9Y4b=BdsuQ7^rs?@=9qwcd%C zYE@aP`t>tApw$D8+zjyFH3lN(;SKzomV ze458EypmZPYGzgEeHZ@&jL2czbHm_#*#Yp*qUQ@=8;m#CRb}tC9)^pxU%{5g`W%ie z;dE-8YX4kMV-e87!XR=yJuC$Ih47sCs^_}&pxq> z0Xw)M-IsTjPXP5&Q=xi`>I+=+y27mJztHb>CoKNFORhA0495%ZVf~WkmG=|dhw)DBCGzm!?rf%M51>DTexGA-zy1#J zjlIb{ORcrA0rPqH=uy&&?h~VQ>dVNjf;3p}^zOm^CYo@&FQs5Qx)EOmc9JxdGmfOQ$`kX8?HxXY2CC!7r#-Z1C;#xBLGXNQQreo>5ggsunkk}g>YsyC@71?h|kP8o!Swqv2j#qAIo z(FnTN426y>L$U63gz?tXkZK9n4|0Qu{Naq&N2h$dT1h0tr=G#|M`tT^+pIhWO7ym~ zbG}P35$?l{x;MeJ)di?+&=mijOI03%`De_KGypc^Idlyf38Z_ld;Bd%zN-3+uZT;7 z=xe6vU)vgATimbcRim44yV)e20O9qU&F)@Vl7C>+$Zxv1iJf3=S~VDro583-0S+1; z1iNbYf=3BQAhW|=bnUYQ>nEPlNKeJg5^vmRu^qQ~-65^rBj~m083kvcn}IWbx@Rcy z#3#H4YuOsz2;JciwiUh@<5^p-t2q!z|KR)nMCKKjP@&iN-IhZ(F(!E? zyv~SQXIb%_l(mdJfe}l=j=J%>HRYeN;L!xwSgDHA47hWvp(HH?@+eeHN1nvm$5_MH zGiRaYKA(!$T$eW+>U}!`b;`qVmYWgJv)T#jH58Z7u@OfsgXPw1kobYo9oOUu-&r_& zxhGH!iTS=s_`oj@bB(QFxIqyvxBjauPM4@WX?56H_VH$cX7Q;x%07 zb_n~M1>huT1l6shA$m}uXll@=!iV*~S;85*&kAWdR~+o0e?c9Wbf;&glIN>lK=N(n zvARZ$w?Uhko#j`*XG#a5-HG$rch2b}?XB*U-B88aU^#Uz)oBgc!(j||2yG0rY919c zy*dKb6Odl<2WOHoG2KvV9{}iaeavlnU-lxNp1)YOv4S06SalU9vjd@v;cjS9Z$1!v za`HMxwI}kdnz9An^!={$zeJmvnN0CiLx&8-RbsB;Eg+9z4yAq-9;Es)<60L^y{9|v zb5^XT`Lbf&4W)y1v$dY^>BD^Xspn%Hh=FWL$QiHm{>G@3E$--Xi^}@JG@qp%`UJ+i`gLMha;|70AqO1zne36DBX5wfF0G0dcq_ zJ{I;b##i)d`Xd*4drxnsyptFSJ!$0Yq7KVQ~b5?^JL zf0BldPGbinU4J3P&Q+v9?J&q;({YtICfhgosHUs&mo> zJXpz03^(z@VKc|VfznrEo_R3rvuH%DoQN~W??$Rq(R*frVjc|5`lI-bp2cRv$uH@i z`I2;^7)hhVyrNc;xPVn@kehXHT6La2Wt|we{1#ANQp|$-(Jg?o4KZRJN!bgZ_q?V3 zZP^}(fB2XZPnDSjWjWT?vIOp-Yp#oo5ft~>X|-fGgEP3SdmU6gf1~a&WYwm#wBJvNv3mpY zJhsN=H*&w;Y3j(#3nHMT^AqXJj%uB9TRkM-rTZdnW2Q3?5dWLe^EK$1(q;}~SW!(D z@V60?ma1%`DISU`eTbz0Lgg5$E4X6oR>2wlJ)Ttfr_!?0t~K#*%fpnb>DrBVTh!07 zZYDQ~1IEBzy8dLF?G9p|mRw6MLbdkN&z4gz6x&ipGnIW+59dABp~}!Z)0*?r^nqf& zuAkC3#E9-#bLl~C{5!gbklRpI!^G~oiu%;=(?v%N&#C`7y@vd3 zdt4OC&!|_F{y9>m+{nequ!8JM_Ji;10!6z89GOd-D;m_v> ziG>Q;yrM+W_WTH#5Eq4Ylj4YF_mb~e@=~8SqDC)9+!WUfzoN5srO@$}Qa4r4M;D zUy7s*-I70i^afpinz3pnbWLUNaHKU6d%GXk$j8+@fl;5cDlwJtV2)rkTM@*r;*R?r zoUgk`y&Hf_OkMe*iCjlnTW2%f1CnjtGR19EbQ(Rs=7ad{e3@oXyFhvRThDQlvJ~$2 zdaCs3KlD!pLXg({W>%X?hBCul2%4 z!xkYOi^}E9*nc&U7RVC4Mo4*2OR8%F?F$Wnatrhx-JQ3zK8h|`kgakCX$O+Y!_Vh! zIB6e}-=oriAgvHQXU=49zx5??IFhbtp&jVHUv`1avZOtzoUQbAYxjdnW+{<(QJkg8WO{Q6L#{ zEE0oM=;i&0`=GJ~%?udL3MaoRG@W7wN@*`El_Ft51(Z!yGhlPsy4d* ztSi*~%HQ2Us+=;&Ynz%|NaAjK=F=P~@$U z`HGQlN_AeOo%~*WiPBD}O3wwSyewMIHsX{^iMRKvx}y0)OZ7V4o;9qhvW7Mk&cUxw24j^8F8iHYQAa0o=$_J)nJ3&` zj4GQfc6_5dS++wcZzc9a-_p}q7Lbdb@^(U{6XsAkDMHdr1}R(cdyo6cY=IgKM? z?QuiwBvkVzI!?tsNO=LtkA<3}s2OBR?W;`HB6(;$ zHs<~f#eA^9CQHyPPozGkj1`lF#%GHwJc+U?5J%EH_LiDq2+|KpSyOudbV2Vm>+sT| z8WntUz|&M~YH$!~zqpz`5I;Mqj7JOsisRLO^_+c3d5X9?nL!>ecEP1xrgDd z&%~2C5jg5`n9%!l1$%n66U*N^$d9M#+61pY@N;Y+ZhvtfZFkUjs!eNRgKhKicI#Lz za^Y7x4+bMmenTyzqv$<9Q7d)*D`Jb2p%2{)c>3OIJZFvU zlo!l`p3%KezZ`)E9qY?Omlw*5Pxs5FW`=V6i_thhZx1`rJ&;|!`W_~wH=fLT5HC#{kGfK|KmN?iE29&@5A$w_HtY|5t++-SBG z-D_XPdg-_E%gGE~y!m3Dn;eQgPh1m?HlD^U5o5qPUr){mu;rmS7saLJK6vWCA1wd= z0E~98!!@y0p8nDiug%)RbH_V^a5x}e)$9UWp&AT~eS^7mXNoT;?K$lM&zsNqtIMlQCb^9{mX>)WEETR-`fG^^U3 zS@OivGH4NGj9U|)(s8WN)ob!dyS-^2zB8%AU%z)}lb4Nz(yBpxg=x76%=E^PZ_lu? z&jdblYJoUXd$uq%rSErltp!6qMM2>4Cc2kxVr0UvT98_|p;$0uAg6u8m`jW0?5Bph zgK_B)(cmJ+wsD94q2XBbu``Q2bwNCScO3_}v&W{_=w6e29S^>+RHktcXzWuAA5Vu1 z@%lF2c(zZ~FTWZ%fc}00k1km!3-)wC?^cy?!@jyQ#N{~lt>q5SVyJM1_Z43Ik4V?c_h9kF zo!D#e1)%lk0YUBg_2`?}_Q5ouT7*uv&mnKZNi?t5Q9f{O&uPDKnSl>n?s-d$dNc`z zLue#zL$1AHBIKc$} zPCVCmE3+?(fVXyim39F204z4XjE`@|=&0|2>K;xSHQ*0dzQd?F7l5<^hj76U|Ca>T z^*it`qY|Xc5eqr9@lX29cs2Xto{Yg4zCm;6rZDSIFHV2vuSN&K$peqH{RZ-2`v?t&CBdO-Q7}Gk7O3;c=)8)zes&7-!^$zJQ+rJ> zaTQt=8q@yn!mQ7>;8WTSPOUJ*?u^S{yg7ty)|aOAU8R5dI@GV`!An}tg$I=l_>zr9 zVs=y((%NzQS$tl9DvX<#ghuiCGC#Z}^}U~V)Z&vav$2c3x9$@7IEFHN=XdZa;ScGI zBX4MI%lEt-#ecMmhV$2&!u?}irAOb_S*p(ut#t$UGi`XC7EW+0#Xw}=%@_N&26Cz& zxtH$6;TYQ(CtUSM%U0dww3hz-;K|uMxc(*hIeHZLp7(>+Ybk!7TbtguzpOid1O7}r zsO#k072Y2nE#nfynOdKLanT~WM2{N>*5{k@_M%IJHG(vejlFx6^z;xTEy&&qL9ljV z3#=Zn4PzhvVWfl7qNk6feZ$keD#>a-Ve&!c5Ge0+6)k#xfa(DaK+f0-6JBS-s(~-? z(B)6KqfaJp>k=!Ty|CmPPsG4v zS5)RaUI)1o2VjOQiVK|nT`{GNN>(nc!;l{HTvQl~g-1RV!#1Gl3_L>-sfcHoA zpp)-$k!xHZca%SYz|40@-oiZkH>Bt1_7>zZeCdM@&~jEP=Fn$WCdr9ve`q#(L+Jvz zPPvN8zhfHq#j0l}0o6S0qx&U~c%^C28%8kYo7baiV@0rFHbvy$7E%m4l+>vTPBb~s}OYe#+)w|)cy~c9t#trDvKMHex7qf?x zj)O5JE8XA0 zzuBQH@sN7vETcLQeTpVSpSGQ0MnD>x^oj%b+~07d_G|hcYayB>{a{;)4vFm0mE@nY01;khl_>8O>t>H{(OzgqJ1Z4)TlpNxynEl1VI zO3R6VfG*4hS_^pCU^7gwwg48S`(VQ4Ig<1iTD&u5t0&CEsOFY@V2KdTZ0qrtn=R#l zxyg+D7!DWzLOLE@+-Q(PJ~foFk?V9Vn-(E?E_xk2jbXtlKuo3Hi^XTrJ)kWXqWjq0 zaMSn&^+*-@dwp*huOH8QR<)#Fv6ib`jq#|r3y_ZJ=y+x8=y_^CMV;;o$fx?Uk@@tb z&fiIx)ze=_)@ls*oc5vm-D7`h;FCFZ)w#j8o9<9oeD8(}*>yv4YZ@;`(3u{E;|;w8M!sS}I;Gb{-1T zhn>-)j(dUo66zTnSESw))W7(`vkM!%EKsPLEj`hhudX*hZvD_1-G254v+YfBFWsAH zg;`5RwWjKXDZd(hHc5235{99o>h5^u}rrJ@fOu zR|R{03ZeV!w27f}+KJ>XY#n_j)X8?5Bn`%(OIF-p7XnltO2gsg!5WfuSnKH8Um6Wv zi6P~6AwJ857e`pZ`^hvPSZ0p?jt|+}mli62p!v#fIOa2X>I8q7rVZkT(~m;ntV{US z{=AM}2S`8A_IY>3<)YAc2hjRR^`1Ppog`LNtRrtubbvouc9=cS8Rx951PMPLSLg`s zaX0EkWScpVmqOX7(QN&q1fkewh1q^$^`^qv?rDW?>3jbNDQP->eNYFD^f`Og_7m`I zb&RSZF}Ax-yz&?WCcVs<^(6zXUZx-X^NRxORVUzk!zZvVrmG~DqkI3J1Lce4yFBT3 z0et9vO=Sr<9_0^97wm`jZ{IP`|0Y7~PR5+Fg`}R%8c|S%5A$v#ckj#q(oG;{CLTMi zw23Jm_}F$kkY?cQ4z{x6v7t%Da>C@-YE z$(0`0ACO;>@7yApVs=f{`9VD6v0TB(9hHPxls&Z#I^P`_cW)yRH!A z*RU;Ujc)P8rjYTbGOrx@2}zs5Cw>O#QoahFdq_~$vHNTavqua7{lOA|}TuCM7;SiIp08R8gI9Y$;Z5NMWwd-HeGdq1P+lfTbe z?zA4l=`%0&8hc(5Yl3&BUAH;K|qMGWi;e zoBNBk8k9#f1#{l7%tX$;-%*lJG3sd`-z3g@3$;HF0<(bwnOVjy(es46PUU;z75LHS z4O9P3UX5xEW@W^o=aUT(wCaZPF8EM%o;l~ywF4dVFk@FITDNtA^@$OKx`ZS37^5B& z#LbL2!jXEF(SFFU_I(7^j>TQS#N;zSwcfo;XGjg|_`kR~%4jKCM4`O)vwhC;U6u7ZhL6F#}~*Bwl0x z*1X0GGb1U3&Bq?MIy1%iU%5cbR<#_t7BG7P8`z^ ziPOa5Z^oQ77)SV}1ND%2xp^;iFb|Q$Kd^i7WEj!elX!fvpfzIW11`hB62_h!??V3S zPuaX!qdsKc)|!w9g`&#C#CzCflohAffO<1*;X+2Y>?z}$F}v^mv}>6Ta+FOv&ZB!M zu3fl~S}2vwV;8PIN# z0sN^}l@p7IL2EOz`#h>GMSPm#biC)(0&MN*nFHyLiaVT?&X6|Jy3P@I_a0WfqgY-< z$7bR*p984QL$S8fT=hGfY}bl_t^ariW${DhLlIpT%jM(oPlP< z%XC9GY=n1tfJ%qxJusuIHKz=TMThm}rR5)VJ%XR=djB}jNb|&pEf&PFH-Yjf)rB#x z{MSQL-^#d}rGi+R(QAq~G=}qCIgG1}{?50QR;B~goi|a$OlgFQ5C7Zq4=PPdccQrq zw)u7m%WD0PrR$FC>3ic^N;D)^(~&BBGSCN+c1|_?+h$ zMG=wA5ZQZX^fiCa>Gk`=D^#C*?>W!&UiaQ}T$Hz9rg!V0^4a9&w4_Thf8q-*=@hJ$ zH-R;(JLHs9{7bFVR zgSzKyBK<#U@h$3lsLIWvhD)*c zRS4dJazU9S5@%tZzXyQkTI^lfU#m2W?<-BtpMi&RsCHQADgS^nIG`LB zDU*MeW17?jxZ!j!Xrp6~bDtV9%1A-+)Bm}Flg3frORF@=t_6+dts1UXdT?x??MONd z$tS>|TgQR)x~59M4)&sRb^|(hL3)s~6(czPdoQiB%XhVpJno<2K5j zvN3-2z*zkFpa<;8ic#~xY@J`Q*vfH&vYckge zN!tKXn3lQij32 zIdMGb;d5FmraRv|AQv-RO8oK6QJ!7151%&)1<%_Xap0o!82PlX?Db_5lzV<4ENKXj zUgbjG&rZ;=Mq5$;%TWG!^FX%8*$?vi>|_Dv8Q@2YpEOQ#;`%0+@olFV+;oN3p}giT zk9pI2!wqQfPA7dhVew4!;r#{m&W791-)22t#{LK#8%NuCTYLdZVY`RCP;7X zXujyeUsz;G>kruf#^T+la9`6)IDW%LVK^uO?qAvq+Adboeuh22w!9y6Evg|h{u!hF zzULzBKp1T75()JhfUDE zd!(k@oDJ}zU>kI6+FXY3@4^}d=gGt{%zL_xxGP2F$txclLE9Lt^I$FpR9 z@9%h32RX-u8_m-Zto!FVWTrjF!qKfH&ACux^Xd!Z#{Zn*ateLHWu{%vQ3OB*flod?arOmoZp)Vw*#N!;Cg)n|-o*+8b;w3FjC zw13G|{i=CB{^=*ix7#b0oH&4k>egVQ!6ad@^`jV)_ei^I+bABByNmvR1GL}XOIG7G zxnJM4jII}Br=+lE7RSNm^na|(i*k_eO+;vRx!7>cowwfKNHVrr)j3@26;8_nZst^5 zuwD0qIAJkC>Ytv1RBxJs9VeOUr!z<6MEmUp?5gt{jm?P=)nl5xpQoT@%Wzmbt`j|{ zpo$>xon_UFi^ko zJ}+;>`rjR38nct9h|_Fa?_>#sH=v;|!^=l%@pk5k=-&S&?pe;*@7n{gEsqDi%snt> zZV(Ud90D|!tV@c8OiOwM!{95$nRYn($~b5g+zlE&=p+>;G^}08uv2wD zZt_^9KH+n2+!7vc1)778jv--?{5tQxAiU)bYcJ+@rL<1hjq`HU=dE}#t0CsC97(l2 z5qs_uTEY<+ImCy*SlmwbJLbb3gaLn3rwt@W`e6NqKUrmXM>)_-m)7dI2|IT*h4Xv6 z@hQvZc{5lr{#PW2VKFB!}O%PYj~M_qWnhmqWO z;{`gOHNJ#TsKMTIoy5QSrRreBZnkOSST&%Su)}WY~-VZaozr7 z?Z+DM!cqf1>T_+`eMyNl58KIKf9{UQyk`|t{rssBdzh^FL&}BdCWRBf zd&y3zd-3b^ZF1!PwvzB1=^Ehpr=?Z%S$Z%F=Xc+yup4t67qV|X#}g-bwgLAv*WcF@gk@0pxIZ{8XeJvU`9^DdM*;mFG(NHb?_Ylny(3#o>LqqI zGY}LO&~ZR}UW;_DP_V;=dNEk2aR`ZCBow~(A36$*|ILFsYp>%o!{<<6M#1%Ox;Sd@^_pEHH@t$vN zS*vUHc$=8b?uxUoEx!Vt9Zre1ydmzpF$oEqrNib;@MpfIpgF=C^VYEju7`wsVJ`Lg z9weSneaZ-mrrxowfuM|i#2{rz1KZNQ7r3+wsw<$=x zh~qlgVsPFkJk!BbPVe>#yw;9~wdwU>!?hetu9c$ccThiv>Llmu$4t+qaw(ORC8SX9hz}VSeL|H`%jGO!j2EW?ENLO_xu0hgjJZaJ{ zq-&IY0y|*C2d9|sc2^!3Zh^N)#iK#08R^ZroW>c`hW^38<|a5ROpE3t_h9Qg0g8i& zm)7Ex9ku0`J}G#)-Er;878UHY`E9%~uMk}{w=iN_XP(i~N6`4c9|B2Cw!%0fqCQ6C0pMMpGa>5$7A)QGE`VUsg^B(gQ$T?e==X zRHW;aq=|tr7gzY$GklQ<1=e?wmbitaU1h3Pc~|Go&@f_%;#NVJ2})xT2cgq~1H|Kd z80mU8OV0}bIRAnM+J3TT0_7Slo-p?vtD(ef4TN?CLD-2$y&MG9ik9$P)_YTn5#DJ6 zwvRwFH&?cH+^HPmU`#z0r1S&nt`r>6egX!JjTT97JXw#68B$>%>1!4e{R8(WzgKmO zr1{l2Gt$gRI8Rz^6})QXAxX>Ol-zLf_wpHNHm;65eLqS=*AGLdzNfl4q9MJ6go#z> zs5J8^M_-v$u!hmkvkx5xl82e7DTz8DvWEm=vCURoREyRoqB^`=V;-YE){rh|$?Nsy z@?#H$sg(~?+U|MFt60%6oiJpOCeAcV{Jwn+XD@HXhZ%OjWe4jb@gH;9kcBU=jS?Tm z_F<%Z@yC?QTGw@-2s3sI!*!1own)Mt*t1+{vlj0GI&ZxGK1LlIX}-YV?s_2a1I0GA z74OTV73aVdn1L?$Oit-a_}oNN%H}}Rykl$N+Kh#A;%jy@S07^P~uoGrKa_M(KI%TjMNKI)e1M{BJZv z!bd#bc{@}5OnysHEg+3`&Z>%bbj&X>*tP`vOsDrY^|FCH8V<})rhNqV!x5hhMtCI& zZz!+w;^gH}@@x2c;eJs5)z=~y$lq#mo44h4|4c-Au7Z5`tG`hec?Ukgh66V3wRd6jpYQg}lQ-nUS$xL%-D_&xld8~n3ritC0Q5C`1%(@cv4)LuEWUJN%CIQgU_lspUxQNO;fpgbi@xBZI!4z5MggZSlVPsP>RV_Vlj z)K3R6o!C}+B{6Yjw&u%)wYYS|Nl+S`ysjX9gGu#r*v(U$iQ7(NcHDaLBJB(5pKmPs zqa_@<97=up&N`v`!}^JhoLj4l4ii-)KRZ&UTf>2pUWBjy6D^ znGxp0467g{9U)GY)P@<4odkJZj67OH+MLkSs`{EX`!U=+Wy;ehm~isd+Fhe#(WF~G zmIWOpeVd2Gi`;ElIq8WRQt@Zb{_8^F*d^18(Ee!()dM|4G-m@lH7paqT+pYvUgxf3 zz70BLrc*sxa+TSL(wXYXeSN0?2dW zsztY;X>3DYYqlr751Fb}_!P_9(fm6Q=DCsv|A$JOy>0MA{K#^K850ModLqqf#sf`e zQa#N@<##roPeGbv5#NsXJnvJV(;TBkHD~wL-Vb0z+8LE`NjjcJVKmJTj=a`_6K^Vy zs->Q1K1*xL!L&{bjS~w>u?Na`fUuqX%t7M&kK*)wS_d~{E6lmP3CXJpRa?(W2$yz! zfUO%!f%GhH?A>3ksz}G*%}!7cn@G~`xU|M~@)>$?CF}t-a(N7-K^XN1)z0Re1-ts; zfX3GH!pCOXO?5Uw?7JeZ@~ksP)BEkruIjTDmr`cHGD7H?c&go`U86;uX=mZNWv=Qq z<%6+#wuc~Hr;g7T2X^C>0kDH@ER@cL*eNYp*W>ryC_h5-`B>6D4?jG8k0lo;Q@%ip zk&=g|Op36;6)6MJ&^ajlf{D6iB9^6B>9_paW}36bdX!BzlPZhb6qBy>DWh{im4T>h zCOxesy#!A?K7=jzgIT@@X@EML1aYjk%&QZUuVjQ@g17>=_X$IT@k~T<||%A6zs;MNzXRg8GawH)^T?7`5T--@15lTn8iu!XitxI)B0|L1F2Gw;Nz; z!ab$&7-6!)3V8B-S(WcH@K1xQDHq_|kAJXzKXrF;X| zZF~ue#iCmErOLR;<0&2oHHPXORPL#CfXYsl$E0(@`}Or;UrSqx=5IIdr=Qzx+ar4l^!6-AW7IXMim|BSG)Z(%zF33e4Em0Uk0kdXdMj{6 z>&E;m=z+=PzR+*}WIk47+_Ub0=mxNw z-p4ZQJfGKCG62tg4?q{+NO?|U!zVR74(IQ^g+|k+!-q2on9SR%>)}DVHqifyu8d9G zMtkTQLs-IM8B)=S`4x4C-OYor*+LEfm&@fmT9ZNJ7^D3dIuuVBCi49QT;x>ET4-Hi zhaakyA&nD8I1G@M8KwBfVw@bj@B%h^J{R2^CgY*o8z_IYao=$K9z;Cdg1)zg!|bhk zeBP%WIH6z*Zg3sK^V3W<^&i-(a~ERCFWBVksOAFi(mk3MxBAJrc|w~qN>`TO83w_- zx3jiRSD;Vbp|W?6Bk(3-ia4i%|54X} zCxcVf(E3PEq~oI*ZSZ(#0_QV6cyGEY)}zyG5dv zPe)F5g8TGCup-MC$1i!#yjEDrG_y?ie)C^Kc>{Bp9&06UUa^-pgptw3L9*-VTWBA@ zmk0TH<7%^1nH0DK>aN%b&%EiG_ES!h&VhYh;l(>_e25wT-32ZDB_|0E1H z-Vm+-nDf#;HRyS@V>rtHD2({ifYZ)&+${q5zeOfIDX?zU`1~v^)~*|Tkr};t4wfI9 zuvI<(1NvN^(R8_d_gs(nXzmA8$EvO|-FX8JN;g5Lyx!cx(uhsf=<~E&?=fT+#|zK= zrH|eqS<`C`)HoRfGa8xkPG862yiaxcRee)XeU#{zBqE-g@oB{qpv|3DSh6UXr;G}g zYi>Oe9VafveXGJH&8^s45C_dK-^Y$)bU6O4z{Dh9ncpH>(m5iX7Yu4$7xtdLgoYDm zV)O8ZjOr14EzpxaUc^HE2MJ)f=ah)_+OL`56@&4$qLJnY`$YQ7<^~Q-OY8SfIkgdJ zY$VOU%s4n-?9iCWRx8EBDu6xP+QA#ZNGFa8`I1y;IisW!igq@x$W zci>pD;aeRU7QF`E-rol8#VI-d>pRk3^`Ty!cg%BaH-01c1o)cKJ`j{~`W4YSiBWUl z*|j`;HP?);(Ng>GcPO_06UqDB+#!lv-lg{}T#?2GRL@SWhFJ1t14u7t-ecT8Ja3ty zy}4}^Qk~||x$^89I)p>-1vhRg4_?^`yH?QN?6ob?t6P8ElR1K0w?8Ak@2ZPEGrhEZ zKYqZr2XDZq{$ZG~e_>S}Ra9QWw+&0+OX4Bq#eSS>3F1qN#JN*VWRvx$H7zH&vSY6Y zh>aJJt=nYI4`jPa+ndJxwb5+ID_+Wtg2o`#3cUIeCL5e>A(xsIVcPbi@ZeAb?)a+% z3>{g_y1I|%G$;JPOj?_3g%xc7ehalzqM+~5V;XNiQ+mJu0%Xy9`c&KUuE9XCFB^+f zy8go054kWS(UA6iy~+|FHstfqo`!lmC&;XSW}GleyVS%-#@q`)^RCZnJiEz9we67R zMqWQUP;8#EUUaMT15|Git^nZ^vz%wcpRKI}r-;{8&wq~X3|VW9_(Y4dg6c%hv7M*t zP-E&?3!cnAj!^A5OdoqyWG;>2x8kqB$icFzX3|frsltk)RXgz1>|_~T{1G?VJjC;{ zPUvyZ8_e@Zi~JT=3bUn7!W~AnDXm|aa=+gFWVEpxcRjfu^NN2$b?@!`WbN&&@3imi zyOpi`tkVJb*r^h!Cb9g^ZS^yvRmVqgRev)MxjYH>{@f~utj@%h={u<}|1gW#)!1ac z2Mj*AO5;dox@X_3)7w8a0z!BT8@$ z?eBE6-c6)B=WYf1()Di%aZh!4@wAydv8jPJTn11Hd1Q`j%7QL&s7h9Qbk+&OPF2Oq+Jn$pYlpQg4&$lg6LHhEG+bymh!egbOWneV zYoT`!CwbpK5nuiNE1+6?47QI@HG#wvka6&96)suT_yz9*YVr$-39S3hj}YkQ2s93O zYg7a~@ZSPl9-m2EV<~SZOOyo%pZ|eY9mhiL;;XE>whdABs&b0yJ1OoYw>u}S$5>|a!vJXo1jrkCq6a5f=y1ffl*B-;b_`dulw+J zXnJB9+U+E%Lxb@$Hwyf@onn350Q)p$u38%!B1xeV?r3H#YZ@6GzT93noN4P~7}asPmmxcK7|ZO9=5B+e4Az1*t) zf5x+(j4&2WQ}t2tMxbnj#EFpc7?_&R@THru_RwC8I7E^@+?DK?iLG!hQg-F_k)8FyWD|<;8peb4TALu!C#f(X(P~q8y zskhX8!;Gn(XdKc-F5wO`rRc%78u~W3mOjChATU5 zg}d54FsJzhsLcF;KDo6uZ->Rfm#G10VYwcR|Mrrdjk?3Tecocx!`etaO>=ULtsC5% zIhp>4{2}#r9{R!h`kNp8vns?5FlUk}DB%YG%imr+4uXe)T1=fT&^`&+5{=`d}OkqG} z`-Mo@iER?b^Z$M}XQW@n5QDaG{7?HTJ-j;n1ncR06W7<;MJ;n-{MQ-yCoolMsBGK1 z-q`p*UnHyt-uJQSTA2WYe@PmHUsyT)G@ILgC~?6=Tt)BK&{*(2?E@4~sIg|j6Rv2C z{}!O`V{>hbkr&jQ@uRzGe^l)T4QVb->zq5JHJ9ZyvSa-8b9-7ngT z4aY}fa&B`OS9wxyacj>`-sua9*NLy`GirkB%SroEFwxIJ8~ecvI|aQaKQK+UdrRwH z_zz%Zi@U@89$B;}MF45o&)T3c3+{3)21z$d&)Awuo73K2jIY}iiD6AYW0a$_()D0J z{w|PSz#%zDX?*UM~)4j>1QuD4q zV{-~{)`}*u?23sTy1NEXUjGQ`xY8yw3kmz!!ny&lsdP6EF*t@FyGMia1#@3Gu*V1g zY1>Y+gjB7ipkp(2yi9Rhs6KkxW*Cs4(M~gV2J>pkKyx7Icaby_d%CnKCojc%PTvM2 zf-R9SMvQy97D&56{c%?8<&;L?*fUU2oygY(>7e+Sj>nbWziUv;>UPV+!>wCmpj$F~ z?Ul_)i@8(p5!O|(Wmh85Yeh7R*^-W=@qzFi-#O;NvkN7VxwM`d3mU5tigzKuLl_(} zxIpuHNRUpVy@W@y&yCt~;zeHar3Q@Y;?6H0`J((AC(Q$-bH&uYGZZEW@(eJ$#0!=G z+vJ;!9WEpz{Tq_?Ub1$x4S2}w^USjCI3&Hzl@2GLBh}Xh0~9uXLef=0eXX!Vagb2s zl(7H1hUOJA8W_m2J4VvJbjC=%j!Sdnl{b}zyTXAqKE#J5V4u3Tm~PWzAnr%)(>OtM zrSUauEa^>f(XeJIlBZO9nUn8?flZQFx6La#`48c@$pF3mHY?AFm6@}6V90Gox{KcH zD}pa+u|T|}^cz$+?}y5d?HpTL<(=r-dGLgO!q7%y$m{NkU%~8q1~y(n*0w% z*&ahnlr|>5oT8!gR=5KO)$cRE-dBLM9QDf7D(;MwE%8jlUb1A7E`K!mA%3zvgUWM| z9v6gPRbKy}-XCnRt{iUNw&c_&;`FdB*o!fsabkaGHdNkJ=}>I3-9#RnFdU+3oy6~3 zlYlh2hIowb@jn-u$+v}l*(3b>Wjmg0mqObwI^wve^gK`A5Kx~{*S$m>3Fk|_lGSn#%wcfrEAyGncF zjrG&C1L9LPXC2+x8taCV{1xf=I^1e=1yg=}YG?-RY!nCnlV}~Qk`auwfqQBAMeJ;J zzDf^EqlutA)v-g7;AU}3d5kK(OXtQ&ca#2j#U32m#i%E%@RrVza*J}HTtYru8>KW1 zJ8f|r>HMqaO7X6$E6N#|tJ^MoF(!>YaJB^NBl)aoRb3#D4?!2ND{sQ zX+JvBLe#5B9+2rdodfdNRddpqn*w<)P>6Da}CpA}-e`T%x+i20iv6Y5yv%OUJ76 zM8qX5KFpRsncW_^@qQp*iOQcUpH-0+4vLG3!=%a?ewl_N)i|SkKoCYSb)NKl!f|zL zj7!s_EG7hx+3ys2mqNtsS~Y>j0!$rgf4r2zjPMspOOp>6M13O!)$q2Sr&OX zeXjU`e3EeSHHXu~%s6>ob}{S_pHbooCVpw?WLg&$CvM|h)be1m7g+&lj#yx%c2$SeVPx^p`MJ!iS{t<3F2fo zMmYs(j@Fzq20=9<4yW5=e2EXJ-|KbO1ypv`_4H6i8cP<;?=?) zP-kjO*(^O6N!QW+%t!QWM=D-->7z0-m{oZQNax|}kVRP)^R5Hsc9hBNqTVye>25oP z&FbdI0{@ml*@;?GWre!_^)Y#6RliHB1#l_)iOW?24Jd*Husf-pvD0I`H8K}vE5 z>{z-Uh!>D@5k^=|c~fg3y{hJkZ#n74SN*!Eau6V$O&P@uq>PL*KMRc5bGu4ot9m63 z%#-yd=Qz{)0Dr$&QU-7hXnuj_fDtDNmA$B}k+LL4m>@{gXn#NKfCZHvtaqVJ$arno@XHjW_XBa^lO|HiK;W|cKA{A$l_u~p{ zW|!oG8jI-etH}d)QT&W)0~bJH^PfO9E!5bqf4g5$t|rJI!}HRP%$nAfDmv>6N^_CM z*QjGFz9alo{+M<7;KWJiup1^%HH4eQQ|2m5xk_IWACZP?p=yD=;b!W=rCiPXiW!e`6i%p2pNOJ=cA$FC9KN;GTB%+0oMza+<#g!I^QiZiLY~ z`g#l+%+u4}%+aDF>mn0BKBjY!ulq5strv+WD-OVafsSJ9 zURyroOix)}@>tBdKSAORcRsQ1b7 zdmD-&4@b$BJ-(wyi!d3ssH=Q_`W~9<9$_P6$7zPlaN*|vf&5l@mVIBKkL}Z6i|7uV zAHHJ*M)e=C@a*qud<9)Mr(^S!RU@F%I+%6(&Sbah^RsSeZN*Ll>!RtQ!E$Bt4i?qD zfwsrcQ{w6TOyL{Yoz~21i5)8^$n*40YC@ex+;r#}jraDhvf&H~PR$~PeMNULY|>X6 z=h=zH|V1r7T=_`Jv$V@!zWglaMqab-@OPtudIcR2L{R6pf7DR$HAjE&u~>j zGP`=a9S^wn7xR-Q%9bskz=gl3v328lXjt%BOrBuD1MB^Su-U(`+)03QP=<_(oFbPE z_vN`YhO_c@+cC2DdmR1v0jr%(`P+Lr$&XsdsmpO8E}Wu{K^x+c?TmKUiR#aczV@LPDvjNPm0~pv;JUa zJi8V*xZ{X4KA^5CJ-}W1cRwJKJ?v$-ljSUYMheci50JgR6KD;>&*1pVNf<73;NsR3 z-0M38Cr){WUY4sR)smp&z^UY#{CM>MIhwihC-(hd)hBmV3mCdO9iBF^=Tk;s5uuL5 zdE1#A<(;1&nO};Pj7WQkt3KrbbMk|vUKa9_=X~%o(a3knYp{4}8=0869Sc%)`477H z*x&hJ&Y%XD@b?H)j7S?pHE1Ab46F&mM;yTjLpS{UyQXZAZzW%xnl6XrZ$j@6HnMlO z;n?fH6EO3u5m;Y83IiAQl|L*eqJ2-=lh5ou2GW|FLE%fuBkpB`&IR$I{ww9>jt|uD zs=lbI*Mv#!0iB~T;C6T1IBm5sso8)Rqy)>i{ja0_-sAB1!zws?ZWNE7GLlCaX1m?- z?ZfCR3-WNLewr&%TsETS{s-tV%302O@5O)r&A`5%1{m1rtoW5y2!>a)#OH^-7}cGK zifKd7;N_xE+hUy6Wf*MV?3tad@#WQv%UNkbJ%3K^MW&(@dBrx8#*)4q=K*O|JXo zwB~Y0f1K-57w9^L&9zYWx&0{qQB#P^b?ftg74h1`ch>W>mO;2?duLwvX)PI6=qnv} zSW6lc@K_edx9>0jZvO_THu;eJT6{+QU2!2b6;%CN1(TLOehzK~MZ+82wHlosVR&KC zVDy?Cg@ncIh0!XE;0QAwoQ@|hZ|=mKr0j*r{XKcSn5kL$WxO2uvl^#s=5w8{ z!Kwdti<3k5N%toi#0mdlZeTm0ITOsOR78gs3x!#*wg%sO^a!l!>cG!UEQQ+L9wRI5 zfrLl$p4VG6-}6*ZouTnmde=m6D#TrG$~Ve7m@w%$)z)RHa6vq1A*m0*#c!=_c%~Sr ze>h<_e6#gM)1jWy(W|CE(H-x-Wfn82lim=(su-rvQ9y7ZJI$M45YP)beaXb;e z<^5!*9g=X+xsN#9AQpfB?Fb6P2@5sN-rUl_oravokeweF2Ct9DqSIn~x&C5nsqmu5 zF;i)ByemIV?@7^dU~Gq8Fx1ot;_DV@x?0iuE7X4h%cpbqElp(Xyt%x>xrV}EjeqxM zG&fV_mj}Cnm2cD#?+K?y-n{>64W1ZYh`kfiVeYyX(({Tw25;;Qg!9<{20gFT`#tME z{9#tw+Tl@uM~6FIcIEX}cNfGd5K(O| zQ+-4Ak7*YUYo{ORg@i%UWv~ZOe@pM}=kU&`E!eO9683Da3Enn7z|L9Zi{>x%!`3)7yNIzlA(hR<>Y@R;mDp>vF*&7jLwrQOr)C97B4-D9s4du;uE+% z+Kv0ixS=un@pcp9!RkRzyw-FjSh+Uly}R9k_IiH=X%aj)(GR1Z{?b03`3Xq>AoYD` z(k*QL#Z*R^&t}Arz~QN$sz0?I4_s#>eJnLEO&Y=&_Xzi*(0o)i6LXuMAO8;^d`3&O z+Hf+*$2D5y`M<$)*0p5jr?&8SfSqvh)93L)dj(w=e%Y`QRPE65p!=94<~g@DzU%P> zry7|fag!i?U1SxbO7olPFji4XAsAX&?wDE zdW7n0hiE(^6AW5upyuQROtaIMUpCa_Z-&;xg8c}Nn}eT zl!?*dBUzSC3H-(u2POX#7yU(@)6nKWo$O=(9ymzGL;+9XlZbt36K`KZkVs`=(v zdf+5>IC@slzopUx3+*=Zmf^&$H)*}2tY(6?F~HeR4htVwD{MT%6(4lT1*%8Qa06@N zj%#>qY75%?;SH$%&h)M!F2CBWjcK=C1a0jiCubcHWo1jyVfT7WDQ(Z`??5;u62CW> zL(b$Y{Z)mt8v|*d$>8^_Wr%tUVh%X6v!p$b zA@#1-cxq2jdd}@_qSC31t`$4=86c=m@!SkMb>7-mYqo)#M`wOQKTBxt&&pYL+*uHI zQ-)p*d~3YL?#DZ8Olt08Gra!6>w!PT*4W`dniXyvH{+x`SaRnwO)2dyMe`&UoB8pK z{`Zk;0=;(`a?+QAK0|x=Uo9NH`4U`yWvaa~r5Fh#@%mx|nH1Lyc4zIvZ;R64c=dA8 z-ng3_w}$S|>0g$UCJA=tViV|aYMs0=s00ZspmcC!-gnzhh*|y^gE~)UO0NwYIaDe= z=K6IT)zKrEJYgG#TNJ~Q`DZcU+Df53fZ|HUvE*Y&OL_odlO(SJ#KFR5ffGzSV!}VU z?GscFxJ;j(Wirm;HrEU|;SQ4CM4C?|>;}X65262Rtr}C!uWDn^^MZ$l`VK=K@6Z_A zOVVnO9w$Fw<>P=TL-xJ7G z&yqd@st0y)ZxoB2_eI<4<^i}_$CAI@_l$kZTM98D2k}^*4va6lCCDpK|Ik`7D{@%x zA2kRk1d`toB@IKM-n|5^;!D181=w6X11o+Fg1!$YqRH@ASUPwwVMRVzuWSGZw?u%_ zAweE#NE#W`XA<8-^43hY;HteO?q&B9!$~)70bHHTmi_rjKFmqHK71pGu7^JNI;-v zE*gv0g|Apj>2a~OA|2)z_XBl&;$7?${icdnh`;3dG+HC#{yjmSjVtV2&?JlLzFy1r z8dc{-cm0&Vgs8|@uz2bTAiXCu_R(JEE;>AH(K$x`PfI?AO}M1T&ot-+mKz^1#ZAP; z@TJ8&P`=9A_P5f6jPw#Hj#oTOei2$Py#fI(M@WTx>U(9+JmF%6qlUDLmd+<t$u*QF!T2fn5FLv^)g<{}r14eSK@eYP zM^i(M-y@syqw7w__zLoY&bI2jt(7&Ow|!)KT>@pV8+6qo0uj zuP1?;-`D!iugUaa{Q1$-&wy%!FoT})9o$hyKT1LI z;1D+b_8v|z>Wnu;pr9Iqq__iMu!8pGq0B(>wd%LNZl0ii|D{R1D4DhdK6W)H9^V5r z$Eb3Q><`sBVYZ-JP(38c@59BO8pTJbv~lgE)u4Q}-|rL+X)`#~=7aJRK)#&QIM8v9 zK|siHDD$vJ@~2gNL*9&Z1Y(_CwH4mGQ%(V-31pqFZjAmdYSX^A{%X#+d zx?A;q;y-utJFK2_dqzH;vXU$@@VWzCs};XlXSTNcg24)SwS^#Q~&q)({@jK5aCg$sURLaEl(N+yqUs4Pj%Zk06dl@)|;o6^$=p zfDib`WJC7Lv+TCsBgi|p0{?#NQNg>FBK4MrG&$y)CkUs-iJ-7o=dH82Gs;Y+IQ77|#b(lq_M+OI)qrrd znp8R~|3@WVTS1jpB`zedauX`EpOA-HOk=o(QAR=fMHf~M(dCUyEtEbGL7iWs{;GxI z!24|NvcoU&Z|_o7GkEI1M*QoN+uF01AITft1j@Ro*QWw$B}Ta%Tk|1O5Pt)WF?JZ0 zhUX2B!uN|YV3*+re|!5e@(EN&FM!5Yc-AjrpPrjYooAE5s!o~W z`;&(OS#cc82{Ru8}MG zDSFp(R~U9MiUX=Oq}pNc6SgTIt;SoD#)e6?#xv46uw?QEt}yP@o(o#iv2dhG8d6W- z9^Lioo>^JR#IM zfpW5O*@~0aH#mbjJL!fn;TVmJcs=~AmgY*HnblkAca=ji%0>lwb5L1hQ20*`d2!9e zTn^dW(@5LC0WWPbPOlt^3znG!VFIUXMB*RTIJFPZwXh>st&#MU!X$9mZ6trzC=w(0 zHrJ5G1LCx*42W_!q*~VW+T$lx4=TS#*U3nCNW&(V@j=v34dp77_svs$3n~jD%}tqF zQ%3xP`YtVT#`oi>a!K-K&(>GZFT-MIgozT{Ops2v?R;J@|j`k$hu-Q*qVO9R+T?Z;_E~H#z8MIns!AUDsc^c|Br6ZW)f70rN zt6KOr`y~=LYslkLMz{?E7smqWO7sdS0#&z4zkFHt56Kt7y^q6jMq(rQ_%f3%J$W0H z&b%R>0%;!{VsTH9e?XO+`D|#(RSry8j-<)8Pv^Um&$xk9gDT(ASQk!#&hPtS(9U(r zYcTb9x+V?jS5{tnP+VemQw9!`AO~1_~9-2j&r!j6$ z?ZgSIVEngq5m~txZ;deIAwP6s##Sr-=5ICb@Y4rj?>2Zay(c`5?ZA7D8v(PwUlm1f zyyU`BgCTdSF<)`XioZ|nDSk~Dj!m=+ST*l&(BtqZK4-!TjIpi9XPlWKTxjj-tY1IH z=z{a&?B3gAd5t94Y1stEckS4er6XuwY0F(StKz zY`C4?e~>geL?%tvlTA0-Ngfc&jvqP-F=ZR%kIdq1pPdGL&^l|`vsG9A=t(EYEpLPM z-HrIV12uSy&->v?uUX>e&h6}B>3q5Em@O2}yMbo|F2VkX{!sgJ9-BF+2JTN^hsWE8 zYeSx2hM-lg@XxkNapP_iZRyuxyu9^VTr+VJj0&3p?QU;`@FC`eyat?Yll$qjD=wwQ0^xFE57mb+W~Evk$O;_cix<&&G2D9nNc1)`qz4)kTkK z{iTJa1v<6}l5{=jSx;9i9efbSpSG5rUl(Hj9X(m{I0jvR(f)(nN_vFSI=IIj_`;;d zka^oiUcY-AjNI(d?MqE;UsPB6j{lB(CchVFPDG$4fYt~Pn$9CCZ(-7oQmTnCvFLu5 z*pbj4E?$kmIseRnY64%ZEXFm960p{bW_-dstu}x8O7^XHQ=YWLj~TXo2{j)ELkP@X+gUvRL52@IQxt=y<#kZx5&~ z4*z_BG;iG8cD*cWRSaW_Lz#KsG3>{`M$kI?74+U2gmg@N_w)wb?|2NRCI)lUNj3Sr z%uVQN&{sO!nQ@P+E&`XOV&?5Qpn4Vio2P=qoG-8~;e{G^>^g4=u5B0y4R?0|od;Xx zHpg6{dA$Q#JbDXs4RB{&3%ofXAI~=q!&BW?%6XZ!!1D5DC|z+v^_Yw>jsU%=&Ft_` zf9UY#8@#oD3^NC9;4P-~LWgb*dFJ^*>2iJ#&{zqYYZ=_E6#BH<VX1J#XYa^PW>JSIZUb3V>a3q6*q9U!AeJIMJBenNIc zJ*a&D5>%hwPaQ(fWLogfa~(PLEDP!#1$q5P)BT22d^Ei_QjhQ&FGk{ko?Y>(OKsep z(;gNLegM?FeAfEL)YEa|&YymoaSJX(;qRlkuiZZMx2^%dp8>}GR{_h<4wd0UBH0H2 z>T*n27My+UfL$&~hD_zHc6{vVt2q8!I}B-K!@GeqAC_Ujsm?XjU(nXdn)YB?jR6I; z=AY$0_!Owm%dLx`z;u^PzI7aF{L$m;NxHAljGJtK%%Wa>GyCWZm4I_$jeAlw12? z%Q;5;>qS7DgRY$V7YIva^Fue8_kbCiQGQlz?T1Y;X#RNAuBA1rn?%ELyCM9_$<{L5 zI2U63y2<*J$H?==?qi15`Jg*mXn<7@Iik# z_b=AcTHt-LXx?sx#d1%&V7lQ&**i8a1xHOXv^($1A2!W{tY1sHeJumN!*ZBBX;ntJ z+MQ(&t0uiF#z3lGBk1K4EON__((&3!tJ%Id&$&HR^CtgnXba;O^yM_)FfVB+pJwz4 zZst#u2I2L2YWuG+|9v7Pj`2e0vGMYbnLbyT^3o#@0-XLZ!UIvX?<-q+Uze*|JF+wv z`kIa9Mmk2E#zYQHJ^w$Ft~;*B?~7NmD;W_DqevzH+;iUJJkN8lxVqO)Ib_11qcgAvOy<+9hJtNdjg-H3 zA?2uap5+KRfZ&S1AuYiWTMn&;_ZMIn}AW5VesXT4l}O#UfD6fz?J^HNQAveL4g++{lRY zIOQr%tbB>ntP0?0(JP#{;jwnn)IV%{dMYaHI6nLaY#z(lnr+=t;lZ(!yO>FkADF-1 zDcO@Kp{_sLYq%s_=CZ7qIFR@_Z7$}l&B1P+no5Opqek`TCoO%ktkRVejzjaJR>axi z;MQ)rX!FxlKKQej6aLAk^t}VkKc~8fl6MpFRZV+boaT@9y_<_iOKYk*$0tW(U`>yO zjPML-P6cr{+)MgPI5(XyTAl<{-%wHiGE=eYa%!U9byvhtZzwt_8%&)#MoC zY_Y1KrzGBDNzE;B_JgKkp~EW(?XVGJsV5LJeXvm6`exY!Y|#G{hMlkx^~RaNz`@;s zIElw*?1C<9A|dPdbGY5|IQu$e9}?%|r)C!!{f<=iUAxU2M){B4?}k*(6=90zuk{|b zWtcl&8$|CAT^oZFFWlGsE$KsJ)Pq6!=A8JM|2#Go_jm==?PT|;^1u&xN7f6UoM#@W((WDPyT0^MaW+wGzgOKtB4(a(aIuEktC07$wY-f7K{zo*fig!-mr~xA*?&BFZJNVr99(%A9F<{zxpqSwC zL(S00+go$HWosnPz_GV9+;!&(@yD!CgcknMKC#p1q{B4jRH z(EVX-hB3yy`NT?E{fB$n7-HJ2v?_hK>s1-jJmAfCUh3YEYE_Myo`;0UH9bBdVFhuR z5i{~G1gd!!V-g5u>&;|P;7R)awTt|=!9}jz_67Trrxe|IX*TQp)nqZv{r^J_Lx+3s;lAwE&e(eWy(^V#{>B1;j zuhX918E>!Mmi~om-whA??^jwB2xpYWls_FVLytjCk#H3icl>b(ViXHGuKO%acAg>M z8aP%^{m2d7GI3d4W0?@;tX0^r@F2V}8!zs-p)?4+qq~WZJX24Mi8zXhZ#J_|pO@hs z(++~Ti*mYJl@938I6{zygrw$&P=A;oYhHAOzMr~_9!`2yG3<#RNI5R!&mYqeH^~b( zM^XL7g2&rLSTVIBt@-;w%>(?{5TGzvxK_`FCMm7pjvdjBA#gEM^V`VJIc&^Y1pVqf_7?Ge3WWPlVWab;E@ z()GZl$$N#v;?;UDM!bi&-tNF5eHl=ApnO#r&3})f zeJaY+!AeuYJeJgx25p@9wmYYhG%M-l8eGj+Mg1Ol^X(O__fiMiV|XOa81h9l8~+AB z)?Y^WZbo|PBkuaI1rTNl;&|$l*=c{p9hYMwthoB~saaR4Vr*HuPxHd~2n;>5iuUUL zD7FmCWXU(Jd2l*?i~cTB>0EB3txNbi4TCGMzZV%uqt!q%-EEL4?ibF^z87fT1OW`|^{0Yl+e~tI0EBT^8 zq+CJbL@beYNITleL|QwDa7Fni@Q?iCvTWWq=#k0s!ee8pGz9U38Z+DSu!SHEOZuk> z37b+Dx>_h*#UFm9{iTW1{m*um^S4Gr#?IMy*NJDKLW8 zu5Wq6UbLh=(faSx@T^U)@_y$RjE?1`HL5g2-svtp)5ntP_5xZNY!ZZ*pc|>re|6Pm z%Kblu-z zgL(~5fy$#?r)$u$w+5SypGos$&JTUcuF^8&yjsA{kB7JOvq%m9Gv+;k&pwhME*PvP* zCrKVpntn<_6TK7Aty2WeO91vWEg}!;%AOXxg6Dk?L9y2kJ@bPRH&Ncr0P-trz0O(O z)95b#dr?#N{vCzUuM3oymyhR!p#Rwmb1RJ`}N*k}yLDHFYuWfL= zaj4Q3oU}30I3b|i076cT6$5IN!EE}bjChkN9iy~!PG%FC+`FIB#_YwGYl1WlBP|TX zSEynT9A5yWhuIE#PI;mCUuZisOX(Cmt8FF~huh8Et-K`IMwKGvI;O070&h~=3i4)b z&%yIRd4pdz%;#$w?1wrXY`NFhwXnLxAD0(=!$_9_SUqqb^nY4QX%{VdGS+H)lu$8W zzm1-aPv0-nPc?zP-e=jqUf-2J6BF&%Y0W+bh@$*6%0r{Vqgf%j#E&}U8%*V?40lQQ zi_dQ5Xb5lcv_T_&FKIoCxbhI`I4$WrFlh7}>3mw!M_l>ooXovI9$Gt-zWv|kN&Wje zy^uT`%WL@n5B4%toiFm*qT2<@KuSB<`d)cAxRq?7O>`|(QdKsYpEh+p8E3RP#)%@-O`C~>n%1A$n zC%YDD=H5OEg!vdz*pT9$y50?-Z82djio5&x+FqVbr^b!ux&9R3Ar-I5LRht1pQ| zAb(^9T)g_6Lh?a8wwr#}M=A>f| z;L@qtic1BJi9H_m0AfsNPurOutf9+i;W_v{BmYP`(4LWRhg*|6$z2OFHJ8E-WhFhw zCqIRsNAIliDRhrWSWFzTj2Vo(PIG0*o1A=yH@+mRxfAPp2%L~Q2$~>0!l8MvAZ?{hoN$J8<#$1IE&Ntk5XKl(am065dtTJ%7JD}8IDK*84BOI|khfTe zE4M8{!VKu#DUZI>TMS(joZ$W3c=-2Xi-z)*kv12kA((!0xftgajkgY`kq&VZgwasA zc^vWM3&wnUgX)zP9Iiusmmf-xR{1Q7o5E+}q!6KM$1Z%PB=03`I-Wz)8{*CT^+IVB zU*CpOCp5!_bSrxQsRa}lp!g%ryXNO|BN4K)hSGpybn!Vgj~HLk5r@iiSpMLVAilwb z5pPlTY5G2uN;e9tjEm&;Ev530rDrTb^*M|l#pBl@rd(kL{gD(`(4lplhISq<$TL>y z%9np@!AvL|Ti%hH`wVG&pzmb53+u478OjJ@Eyo+0O7FGVE@kp{hFxwXw?`CdKdFKlHO%Di@4$d zAYYH>uZ^U&_A*)wSWBANSQ^us^ooO24}kIkU2Kn53* zg18JD|LA}%24pd%6ErR2s(ciA4^TL5cXI=~0X5~$j}u%-H>;koXw=I|s(vQT4|lv; z2!yASbPPOix=(p5O~}V~KsrZxOg3e1Q40Bft?K8^9+HZLJ2Lgu4Mv&{2>bDpVJDok ze>)f~(3PZVwaSOM&&~tlTGgi%7JIG(X+(aHKVYP>29Sws+N2yX}u^^T(oWMX5tU})~u)_^gHax&&RX}rPZkyOFT86dNjU3 zeNrT>1Ja><^H@=Zfus*n`H;GgULeJT{%sq6`!-JXPIyZ}|gR_js-6x`*<_lF$n*LK6oCfx0YzcXxhzv>$&wfj;QP*r%Z6*^kwBX>m=%1PWS5agO;0c=iNq@pm4lv3B(yYI;`J$jAe< z*LaUE9Qq|Qnk$~{bO{H}8Y=IdDndH9yelumn*k*-wsU)~@_6o`3V2|x&u?1HU@xnm zXVbck<^>DWAZG0~9H*(yckRC;zm!I^&|j{2y23)v`O%4mr>$o{OQ;vocNQHRfK7}3 z;pXE^&R+Nl=Xv+#y{ESkL-DjI|5_p5wR@<=Cc#K!m2Quo!_!0aG!|a%RsF%7#|tsx zYd_iU1nJ&oqa@Y2Tp0fd^lUNIVK}YXdT0dARrSURklRE=K3w$L9^*IsLv%-F6ZxX7!VgpH!ERhPbjNFZ$LAP1+Ga2~4 z@gja^;vwu;YdYb}0NLO^y&w3@jDIlChD#0;!Si@1ztXEdH-6O}jvvZ|Zr>hb>$i?j z5*naY<3?X+Ibazb$h7wz&|}%@3Qmr*(5rBw+W{dU9QJYyREt2)8#2 z)KL7uYhXJ*C@W5hl%7{jqA2~J85$BJNVo~4DqPeG!!WH#%{xmsWtUY%WhS{Bw zS!+&$>#k?Gto*um$1q!@yn$(jTX=`DkH9bgJ0on6Th0)UMuGfW9E^6hmhw!Whe$cW z`mdOdS(Bz==L4STz26Rl-S+WW{j=e{z}syjZjIR_MHN%_jR4;;$=+kb=+wu6bwUO^lt zO)j3{HreNK%CiH!&BI9gHn|x+Q~HdjM`zL8onw^aK=YGwrENazu)Mj7J=J42+?73K z`SC#EdfFNr4fIzzh4mNcs{8`|mn&KM@j84cwu5>hzA(SXIJsiQ4lJOxcnRmQ-ilP7 z-t{@$>qF1KvnI0*6IZ~y-j?*PmXRbLTu?Yi zId_0jO)C83r{*UJ!Zgf%7z);Z4~X6arV`HdftY_Cv8?EzXku4Qrr+5GS;4!tiq~4! zD26{?18FQ#!f;ao%;<1XaUE{V(hB{58rm!Go=`YW*o%#vF3Zr}jpgUhK{)bttms$k zm7qFOIExKW|511hy($K29A^fJ`tSE>g0ppGo4f9s{wrQ-PQAOPZJX4GrB()OdrvqU+Kl$K=`GZ; zG;c`v3XckX#J8PGh2qxqlspaPy;#sEiFCmhAT7j+%Y~Zb&pp4u-x1c(+4ON0W)$5% zgFzl^SUW2ZuJV954jZ?&hl@S?YgN59EY($9EH|Z`K)2PjC&&ztf4puoiXEzaJbSXQ zq&yU*J{g$$F&V#~HN;M`pESO)R^f@TYEToek7|nniGx8M-`5e~-k2-HcnwAug^+m`a+ch_uj7Xdf6T31>yEPOEYDml9R~ zTB;F2+{Tnf{nvMw!e~kQ0kw6Xf;Ps8)}Ux2ON=Afx0tT{<-2mNsc*X6H^@}Fix+q{ zy^f&z!od4Kv7J>GQVl6gaG7CyjgQH6z-ns;D<0I+dF1E&Ihy;XBOv-sGu-g~CglHm z4fi{lD!i4%34*i~SNdhBYdGn}X+U|&er`9CglR1OjbJRmJIPatHZx zYb2yJ9{}}-)A}4+9)qWm#6`~08sk@~K)OK?PEm|z(H?TNe@>^vLUCgB@9I4J?{1*{ zBEAR^y_Ra!y@=Y*Lparm`0vkB{G=1D=1=o&=T|XP_Y-SYxsMV4K-yDl(a^&V?P6&^ zoz;_Y$dQ@w++ho@+d2^1_CKgezHSK4Z~Eh~v*EZPBO6FhaMHa><1pe-L3u34oSDPl zJnx95i#Fk9&pi0`^ah3(Uly%`%|w7_8?n_n{18W{#1Boub}PTkxGe2(BD|jssrNnvFJ&=&ZLBQY-7p z;=}v!zb2P7mTB#)^pu+O$#>^rk@sGMU?!liIxN{qh+t>5E^19VP_{8$~?|K||lP%{cIc^rXt zTW>PT1AIWw3zQZzFh2#E8>ebWYhlE}4Wv18&}PC`?c44iyxJ~}AUwe3K1=a{eN#Sg z_-h>aUQ!(QfzlkmJ-=1)jEbf8j7*qtsyb30C=Q^l$y@T+3uD>GpM9%v!|`Pl){To{ z3lDr&7|+LVdyk}Vkp5P+CrCH4YX&F5dBYJna+#h{d>a9Ey2n7_pvyq`jY@-%mKM8D z=<-%aH=|)IU!eOU-S~;A8l&-X$}OmD{*_U^OS(RcyVR6BohoIomY!E$NgN(>6}lh0 zsrVj=Z>XM+;Fh1ONXr;9((7>O(>){(LjRK1Ff(B*C@mY~e-Nf7%}0IzHn=dSIV7LU zWrXE0(X@tKV82+i$i64n$|MUb{< zN8Qc|$~n?Yo{F!ipLYZuK0HLjs763}MD14Ot+;5C;aywX4v-ObS*)3iseCl*{{k=7}8A_-=16uFa6!cp4pd? zcm*3Z{fkbg#sg_*ac6lr{;YjeSicyB#N&ee1dyL4@8~av#-suHE)DSl>L%}ksnckE z%`NSf=LW~zanL`ct(ZEkrr6ZVOU&5)5lFKG45s(*$v@Jy$&V-EGFlIu<{s|eqqWLC z)@asTHDZcG$>WIC$10I@1S?%s9ZB=4+{Zy%jnty;A8+kGyk43$V=j4CV7-2Uf{Vo+ps@%(u^+nP#sQgL&;jUst0b-=KX;`{S8M303;}yk04gN8(ra<}=dRU}t*>9=nYcFPcUod1FR> zIjWZ(KpwoxU)=0=N=rR17<$Ezu-aJY8RjGPGRSZICzNidnnTsk@N#-1$X{u`x$Z#K zuT%In>P|YnZ?qZvb!{jK=W(K6JR=`1HpZ=FisOh2k>(QzPue3UG>8{8-{McoH?a1r zsf~W-MtvG9q&ae?o*+=ah&UmFdKsI*w8=g+Xk5U2eRpByEd#OWa6hCvb{P%t80kVr zUKxo?1@$3eGc@3xJGBJr4WiPr8BK23$V8btr%eUmD5#)>Tb3mL%d3u=mr9W5K4)8qT z5?b#JC@iCx$?z}!s&+J_%bDtxsNUkteb+$6f2V7A)f1udRp&=zM?w99J1Ln5>5ELy zJm&MXh{X;_gdYDUE6<9mW~i4b7ah()@?c6&Xu@=x$V!*4lKP&!;Z;{uT7kSjkoJ-^ zhm3k+?0upoZ!_vVVf#O^FV`Lx*ndVU z7{$>dzFiXYD;gxK&u6&K{~KDZIfp713Oki{R$7XBym-yPMv^zj;TH6)bMi(pZAc`` zMD4>rA$nRoBVMBT z9Rlhjz}~(+<$K3OLHgK5mOZV~O~g-}dPXcR{}7~(d?u8(B^-q-prM|p3lcU0;f>1E zD$N;jyBfU{I|>O``OYh+=$S|WsB4gKL#4r2r*`Ai2PPb?#l82RVGbY0BJnXdo-&L% zzoBOM1JH0BRvDk!`iQZ zsQ5=qc?RnuofVE#PSUz}chZ$67sa%PLvA*G2XB+&qWaAf*4#(pSlScs4((@ctD#;C zQ&>wJDaaS8ev*oxB&-qz#cv?k?ha|V-l+2Qz$pjb>0S`78r%)H?zMr0plGFIIMs+u ziF`u%ZlL-Vf_%Hu=$g@Kca<*DjJw?yKjTCgn;F1oxcG?Cox`ZV_z2?LZ(w6Tx8w(Z zU1V=|Ea91FHN1Ae9x`C%R=lvKF?%&@3+)j+hT8^Bg^@?Cc=^1^(s$!kc+qJH{JHxd z9;#Ua!MRUyxwjAZY}!b!s+fb{bKXOQ&3o~Fd8k}{v3=4QG@qW%tlZ1rIVzf#>_e2>gWuOI&CnBmX&m7m5n{$t@*JP?{HZ$9*i7uAMaq9^~ez&L;o@_H^U+qV7w=J}H+Up!_@ogV8p`PhhhnMgx z)sZLZUB|R5Hn?fmEeshuRBmaR4x0jE<-9`;c;xc8%q+X1R54n%%uAN{$i%D$JLQzO z_o3D0>o_kh2!5H);Wy@;!-REzv^M4W=xbYtFPwQ5!fxc?kCu_x{p1aZo)8FL+yBCa zBq!m0b^^0AJ`cku=Ah1HQ#4IoE$N)NwsRjzF_SYxHuH5U4RCIsDbl;-H|9S&3Ey|w ziDygQWaIrt+$|=FO>DH0xy-YIE)`o~+v7Z-6|7iZw`bzWh*7jQ$SFK~U?X(2oy7N< z^@UxFY~=77_3+F{TYPdl1PeA##O(fqd35*sU{5@s@nS13c`<=Mrad`p6!(=O9lFT) z!mD_DHBb-Y8>~y&jis&K#oaZ>pn)vcGuXP?aY*;QfODJnVI7ZblMA+SFx`|Q1{yyXNAmqJG|3NqZ3FSp11`T$ zp>L4h-lur{M9s)SlI~MJ^Jv2x%&iTtX4S-pV;ah|UvDs@dj@Vx=*a2brQOGFg3b$L z8k%7$AH$E99ET>W_P|g2K2(k`gy{~6tj#k!*}bp~3!4lPZuAZ6up%bwR@z-qn9?6|Y_PYr_miMrU&OD%8lYW!?BjpXat=Wqy&yoez z8tC`E&$h394h~BL(8wZ4&50`|r{*|P(0HtLS>?fK+^qkG&(O#EmKg37i!ZxZqTc#8oN7d@ z?R<@5v<(;TnS>*c9>q!j;=s68SB&zl#=8&P&DQmpjSdf<;pWJtnB}y^#k*uJn>1>s ztlMxgFKD4BGiz0fNNy_^&g@NlHxGetrRi|ZTSt2L)#W>M9XaJCr+P>s{Ls8&TeyfT%%ytak2}tiK9)XUI?`M=IPS>_KSbp}4fnX~ z!fnzM;AZn?(%t1awp|h_wzTZY16I}o!XNB7V!d>Gx=h5c4rIL@bfrFYmQ;86?@4`% z`#HYCsg$tQiV?dXd}`x&0E@vHrPY}A!W2j8~nx_Ow~pF9>P zpKrp6qfm`~a%pE=Nb7e}41r<}{|q~bzwKl2%%(nsTPJ~H$CJm+MXw1@S+!ZW6^}`a z%`N4yWrxrvY!4>p)WY!_cgsCzyl5SYrNnpR*y`C~P95sa7q1j;Ppd~R~8%;Cwshx;G^3(@YxmhH3tW z!U$I{&AX$HQ1^kO{JtQ8Rh0c11;70J^ zdm5BHn~f@N2HTrSns>~7GeyRgMPQB(?e9`6gGKzzW1-C^;@C`mK4R_%APmM+Ex+Ja zooomX^nsRJYNNvZ)Y&U>*{oQgeA3p}HkXGjOjW%~;#K<2P7B{;HY2@37*v}lT4k_h z5X^QO7BUaVrAT-s&hIXO)aNb8bKTbV9aBST2naZzqXoAgc;iF`CN!aU?n2+=-taY& zcnt1s$;Qz;2Z8U#-txq8A4xeVXOBt79(ALMqed!j!9MSdu>4gZgeK`q#R>1O+CtAJ zy;;b%kAxK^jIc;X#YLlW$v$yle5-E6J<#0ij-a}QhCZET=ar9PzWyTEp;;u4&kcvz zsRL!d^cPI&g!l1<*m>myQ1e#)st%C0Aw9T2^Yu$(xw)4XeSA$}K+mW<(6 zi#vl^X#Qo3u`3v9 z5~TB}SZfz~AV%e4n z^sV<4r4?1Ik@OSs*5#_X9W-;dsFPpcEO;#G)<4pZ2K*3>3#KcNHo)uQ*hFIfQ9 z4nNRw6XvwLi?Q9#$;bblg#1S~8mcQvdR7n~T_k;ugb9T0btL5=60U=X<5yI?MAs&6 zse$#cxyh65ti^6UQ&xH07c8CY3gQ-Q=9Z<|zIF=jG5Aq4O1#78Z5;T<2tO^wA2$~C z5|j^I=^f$&4aJBN=W}h?5wW<d2;+;9`x@j*yhL}@jp z71$0^1!a=+LxZ>?BuFKG|nk%M4w_%G0DvjXk<5yqx*s>?jI*qzC$r|ae~o5?oy`k~_ZsO=|V{{}O8pks9)F9XZX8gt@i zNOKOlip^qjEu- zRYSEYv|+7yjp&-VeQ+Nj9|lIAZRJ9CQk!An38de|`h88Q20WN6$gFFeFWY>2X74t=TS|e1)8uYK3%N@NY3n zd1$Z=dcjC{G1uMSL5=ab?+o(VdZ;abD3m6p+)E)%2%m0u5x3`?x-=U+2Xp=;W1`m+ zMtVxzbJ)q`cyFb@l8-f-fuxTa;X4u@;mIk}Aiqu~D88qB1a<9Ce}_ua5`07hC$PCW z0!XW(LCPg4ZJh~r&qJXoN6U!+m0uU6kC8Nx2(PBYW4iX0G+q(CFBeHOC=bAjA}xqB zTHsKN1*&GDAaR|Db1T$P9K?hJqe*k_fk9UW;N;RI{Pm7%SAPd=IL6`lwOdSilPZjN zZ-V4^xs&!bewpuyd4HoJVcj|?e4EI~xACVRmy^%%)lgqT%@_arB^>kYx)WEwRk;fa z=T)prPMu;j$3R*~svM!70sC~!R4C4*`cN865GFCIL0s2xAnCwO^u5*vR34u6CgDUJ zPz^HjH2iFKG?<7j_@$9CJksnJ{*5jGX+0nx4-2oXXM5`%gI?b}IprZc9+3zwWAxD!APdWsui#$$8EUF?W0KzYL$$em=sht~)uzp(8%Cdddjg-E+Vor5o2@Rte`u)fehs6h=@_04cuO2iIG1eMkbO=c&g5gYrI!&s}PB z;zuMMD#$CdQ*SzQ$|FX;ffe?i4~@U}g`QS3wI&PJGR0x`dEFr@cEI_5#7%3-lXc}QYV^eW z1}4&V^Bv;Sk?f%PIgRRHk*6n4-3Wv!e76zE@|pqQHDMBT@-ybEuI8y;6Odnkvy~iH z_dB3;6lsF*SkS`1>SrpZ+VZjNOQT(=Jmf#k06C&TBD)n_pA(*{9vWe@)C>#XD=tpA zsPdYNJo?LV$ASd)td!oz|9b6#!z&6fcv>~pmyr`DM=0G1q$#+6i{g}uvURZXWfqWL zz{SHBNUBTX>cMa*uc_*F$hq&^k{^s?g{Af2`olDQ|FA+s{Q|-t%ZqgHReh7}{`o@r z)604NT`n%}Noz0$0nMFg*YPyWNXe`*j16fjM5L)m-uAsn=)gSfo3zVgw8DQodnxOJ5u?(}*->6A2) zUC{{X7(RPz5#`<;P<^GEL7hndbwxY#;Ywr6+&9k^Uz6rMrL+O{W?#eQ!)H|-lxC-I zAY5fe{S%CGkn~f1w5R9F{|`(pGH6Pjg|BmKwjXK=Ha`LCwF~O0tNsu5a3aam7${sr@{>~G^Rij?{KfsoT;&4gp~&%BB4})q zZds@~XutgidE^?Lj+07*{4L$5mJEzPPfAaW$jp85TY8}hgTA=_u5Kp(e2Z; zp6&&cOlglEyGan?_JBQ!35L5v7ePwbg=m?40(Mc%R@K+%u{sTyW6NVAHSa1^tlurW z_t^sPviHG2-Wuxun$H*i`zI!C+9GKDFi-Crn(z0cf7>9tn_dx{b}q&`9RuXiDN}H1 z@&egmr>#0(Iwo0)OnO)F-#TB|K6?`Xs6R#K)xU{r-2n#n`D*3#vo z9>3VaOgJVTg6uK0hvbPf_@U88uxZ{yy8rft9#dm+TKa8>vWbvgPu&+^uf#&uw}a5X ziKFy%T!S<=eDHaX?9#Xt!>3lV!p;M@eSB{jHGLFRyla9jfA;`7zdR6p6?ErMl-BcG zV8-|ynCo1Zn|J@ta{RId%`+4iKT}s>om*Z?vKj9~OHushK?#t9Y%Gcw! zr8KzrNc8-E0#?`VEax<6%CIA7!(p34NbhYh@T8oxcNH zwdyLH4cd!WD>5)Q>VrVH{aCTS0mP;Fap%d|*k{x$(aAMOP8%DErZb*FQPwqfD(NJ~ z{G$Cz`xwJ-+FRE$aFT3yYXTk(iNQNY_XOpD^!L0kmhUaZYe%MFQC1(yldf{P{%8nI zEYh5rw->gx@xZei-$U&6-ymlN^V?VEg86=3(Ro}so}qOu%$6F+Tan-4W2d$}ctu^= zGv_H})Bf1q*R7E7GZIEMK7_Bw*M=E}i00i_;IBs|>Yn)3jP;t*k5MrGT}Qr$F?M{sx8~qj+$?J{aKR&$oGR zmoJYu1)p(eDUbA}vsEpK`!8L)d&flV()%bZGiWC-ejcvkENj;d#@co3@fSlTuxI-| zYD&&*Lc^p|;S^d;#)fXk;*NJgoAZ`=zsu7kKGczoQXBBg8(yHQ>wdZJGW`1jJiKHh zn{Z<3#?E@G+xPvf(*89412w&wI@z2L<%7QnOxeK z!s)&=3I{rTDHRd=&E%pkdze$V9~9#(X`JHBJS(&~cv~=hYB!SiHvA2AZ=l9OdBMXS z=)02fwPjebr|f&jggesSOnyV#@`G`+*u3$+tl{jDa!I*9K7VhER}-p%#qRT1ZD1yZ zwipBBeB+p*N3k%f)l|}X_{$ZA?5>$EV?K zZ#L*#7~j}&2lV=Q7ib(j_hu*FJ5K=BKCR!@7}n^%0Gk929R2ISZTU+!$XTjhnC=Pdv-`diVAC)X-hy-8765Uf{g(I#T7B!WzO+ zR53jVV}bGud5hgVvx!TJ(|^e@$>$2KZw&m*ye1gm__bJa*_EGIXvkO5{tFZ*G1xp% zM*SFuge9WxuX1dbcmo$6X-0XeCojx6!>Jxwg&fK^KeOk?-qpC{=3<<;;FtDH!43Ih z$YXKIG#85xl`x&!cKBb`W{t{q!VPh6)DNOdM}OmtvAnYRM;W(iVwiO$??o5az4^rVOc#} z9AnY)=0RDhwhAr!vTNMb9@h#pPcZDtr!_5;D!(Que6Yz z<{9DGN<9d){El-rm?7Oay}Q$f8MJ(X+4Ri&U(4m%qakgz6bJF@U_a5RAVWj*CPMa{ z6VL5tgW@mZOi8>;9D7>+8_d`NFMS!b_&l3geJ=Jm`kqzOdSWM?+HzCNdBS7E7%;KQ zM#U$To4BIgU}!hx0qp+yQVcPejs2$Y0-e5dw2xosgVF&lCoMliWFIDvf)n}Ny2b=0-l`^$at$bDyS?-?u_rS3+;8}X(r5p4!m2by2LJTn4M z^y&*SI}^mlL%WgiP3Z|u`(I~a+|W1Ln28@WV`}`Z!u_F9REHI-VgK)b@b_(X(gve3 zu1(Di9~B7teP+1K67yVlv)!V#s27?cehxhko*9*F zNqKZ1cCUY;?#?ot z@pw55yww~}&Xh2sn+xfUaxuLARao)P2nsvjsEWml22nto3Y#qGM9*gZn67RKCL8}^ zDpwW^nh3Qn6(Vspr`i>CUfR#hk9gxWD;Pe5&8>3{=4V_GNA4%%my#81OtiPB7Xu;i?`-Y5e`ZnT2F$Gzl+zaL;;Zgm*7 zW*C0TY$aF7el*?`MtCl-la`<{Z=)w*hNi@+RFhU;gRX7^0i!V9?`>%x+sgDDJ%# zxq@AOUxvHe+QI6`_SB2nkNexzNrHGz60b4mtUu_{tPTHt z$V{HMsxKcMcqvE|$fz(&p5DWnuj>01LkC4`DriqL!bz5^aaMUD-+;F~WOJ`G_e;m*M)(8&T;M6{{a<7qM>-Gufo)2hv-Y;mY&~wyJ&r&Bq3X zVfdk zSg@kXAjOL^(RVqVh`$LntslVgO9vU@4Qb&|q;V{{!lTS4DdI~vJ<4Srs-Xr-<08$w z=GL^Dcyn_rbu13;^cei>?h|NI1n-B@`zB`piQ*eA2%ENw`U9L~$-2v$F1@W7VHWA# zfpSH=e5Ips?QlyNvHXYPDzRw4Cp+<|0Uz#K8|x29)DUK9^41r?x-|`D?De%+d?Ol& z*NHR3(P>(DAijgr_7fptYaf~F^wWj30-T)5HEOJ+ouqY-5*PX%RGPK*o#8;oXrJw) zJ$haqflBRIzBoG=d^*tn;A>lpl-n9X_$AtFK0>XRP5GIE0pggomAsMpOw~VHHG2hZ zet6RN=OHkl-V>zX!%FQ#3{2?>ch)=g(VWAjC{|IFv@Ko|ow zFYKW)!(JBVP-*jweT;IDV1@Nl zK6SOjMtY0=?Se@Y(SB!Fnz<~_u1C34mVCkG6>B~H7R(>vD5~H8m@;b39kJ(M8juFy zZYBc=Z(pl?6cjUIR-6u`vtiky$3WKxrR8T`v*Wj>ZGo&QN12N4u*_oZ;4#-zXzsNI z!(W5KhSia06>p-KUz!;I@)On3ehTs^;x?&loR zoVt*v!a-B^vtIr~SlXzM(EQv8xvzga!f2|XKMf>dD_<~(=H%uqL3K{r@`=KHBo2b% z^?NG}5kA+Q6c6Flt~NN>ypAMq$omI$;Z(0m*Mj27Ye$}A;OiQ4O$S>^bIwTb@C^~p z;M)BXi|-u^h56w?+{$U3TGCu%*cVG~NWB*FaboDiW8gP5M)3xcC(!gd*O)(le@X<{ zN_-lf#zqb=rTrj&K*Ghk-0Rg>b~Rz6xY^W#6K`_oO4|SD?>tc4ptStp%m46tksh4X z{0CEW4g=vIymSszxkdcqCX`P-H@hWio^{$cBU~`SEFGvW#ic&GaEIMt?C@ngVaE$} zHD1igE6Yo!{RQQU*!^=_6>pJl0`ie)+=jl@AuLw-#$Q2tK+{UnT3wZT zRo;X6gDGuGIKfVa`(yLc147L`=@3B}g$KT_Vga^XD6FJ&@i1DWgtWS>r_hb)oV5dqM{5s4^EB@)?}LL}=u(;q(GOZVR2%$)Px?(a7P#`DsU z^Z={fbBc~?RgzvK9cm&83!rXm8=!UO1UZdVb}zK#QyKM%^5*0%y7L*Mo68?AL5%7ET3F8mnmgRy^d6GmWncRKVWfAFu0e4! zT5ei}?s@*WufbF`2cmn47mh#Pgr^+K6RM_~ZhX(aIZg)B2#Mupp+Ym}hdBT7m#`{G zfQM@@X-7J}1;s06PUl(t@%La#_j4(KNgRo!NwB2D2~4x!j7ndupLZLq?@!ZG7NaHI zi*xtP1LX&%E$_y*_WZ!=?!3*qTnOUziYz#35hR@8q*0XiWo_&JB%kl7qxyrExo3cS zUXUkX#20LR)xJQP1_nNl1EswQ-GIP7)|D*vvsB|!dwm?Pj;xNxPi_kGlw#hZ z5qwnY3wDHmV$|P~a1xbXv)L1^DdyUU6Lz{KdDZIP*^B9oUApH5Fqdq53)<9n8rbEC=IPhLr;1{$Lmy`{V+^tR9iYh`^qh7driBexE3qA6M*%5_kl~>8fp9H>=Ez1cLUX; z)_Zbwt}8#IG8W>XT})+5UQ3pVg2nTIcnl|PFp}Q~JreUfSxTkdb0A!hU#!#)%7i1Yji}crC+UH5#MBwb-~vl*=7ZxWZ_S@v+TTRC!f-qy_hV-5V>~pdg*P8RefC zWlzE=ISRYpbB1lL8?&>24zM`~uK?Am@)@+pb2UMlRZwOjR31s%T;Z+Kh2)>N0M(FK z?_5*W5xa6K1}VRgGzPu_X%Ae=dC6N!UrSRSn&_l*7HHrz8+*<$#J2MSD!nIZK=yi1 z1tZ_9CHx~DU#_+4$JxD7XUb?gG19NFsa+^8j)-E)2N7<`C!Oh8`@XXnWl(}R5Slc( zgygp=pRLWeU9QiRhgX?xtEk$}Uc84QkE_%keA}AmKicFJ%sM z4-3UztIW)RxJ*)?@R=VPkvA$=Yk@<&`-8$6%92z*B}lKp;-|$bQ?2ySkV&~C{rC9) zp5V5f582^CX8e5pZt`hLe&t#cp0k**ua*8|Dhr)+uq&9i+fF&^a-cDz^1bB42v61! zKODirbz#aE0L>qg{-?g|T$vvbcM6MhZ;e{@l$rBLseV9YD9v{1t@>7)C8Fw z-Y^;iR~f?2ZcBhZm$19G%4G%d5vuHldJzuLK4Y7LF5`!{?Rm1ZH*2t^Dt>ZylE+Pt zLY4D}xb^E?SnRHc>7)n4Oi$B(LEU7v=e9CtnIA;9q5D+~M$4@ZAWddkO1E+iKZZ?V zO3z7f$#x3(&^^#oNso{0IU6QLJ;%)jTuyux0zIP7i0hdF5PS2JxRWvvm;D|n2TqF? z^q#_hb}z`#c+m6p|6uUQV`!tl3CFpIVr&~1X<2Wp3^Q#fOP8IJrGI~; zSJ~{!G1U*N##29Rk~L^go}_?aIN=!yC!!9s{dLcXI@v)O5z>;M3~4C7wfh050_=Ex zN;kZ|vKm~|dw>U{TB|>j=94!tA1+S?ghRo(WY8Pd0Fpgx$kdQcz}IxrO^?_nuZr$^pZ{C7E+IMw$yTCD}ql zW7Yi#ZpI^xm!j$UI`pi`7MxsGBr=NbR$ilp)?C`?S7%=72l3kd4UqG>kEHb$vmb5d zo=%4`JLV?-XE2z~-?8L=A4bUZ$Q*2#zm5;>(^Q5Oca%42&xQK-Bk=NOZ{a#bi_Nn# z=$xt%yl(jz^l(3-eOPo&*6v^?S#@4xdks{{JVDkc$FC)-pm9I{?o?-(V=j10pp zi%db+2)n{NN$-`@bt{8z>b~j4i1qP3bS0<%pqjtl+GaEjztSC?E~xd2$t+v z%YCLfjLEPPGO(w^U-;d)3D?CG!l4f> z zIa!-T^ybgdXX8oS|8^Dcv$3_jT;z?GsU}cu+FR(908pjyqxPn$zFgg=udJW$iL~}` z`{r%gXvk1(&|y7X9B3(K;uyYuY!7~N#vHg@^cWkS@!<5{x;eieX)pB*!S)-wQN1*h z)F1qG%uLEYV<2PmUGXR{jTvQI%MHI%SoI;V(5kix-(lb;J=~kfepxN$)H6H8T>I); zn=#d4?!ZUTtK<~g{tOffFMcm;!V*Gu1Jw&O-EmvcweqnJZy5a?=caFj3bQio8^4@W zpG#{KV{BAD1lD0sPI#!jcE1{ghM+U%M=q^fMe^*B7dW)t6N3-0Y_Xd`W4IwRY{`^9zQH zZbr^BvcM5qn)em;`Wwo?vMX*2hg9X`>%I~V60$Mhw!3_`AEaLFK^@gQA7?jBYufvArG5SsR|vZu==sQ=`%Mc3)w}OJE@*9sZPZ>3+9+gjfoQO0y}}fE zaM^N6J*c@d&;-(qAL6Kl19D~YO)ZTPsNNy`cWe3a@;`|8+{@m1bcE9$@8C+yF|4oC zNf37ClCTBm6?-zmLl(VxzMSE1D!;|Fz|4Dt<$=w=!KZkF%&xe>%AWow?6dXg{Ij}z zl4FQ;a_uf-ZRc`Ucu8CM8CfmALN+D52x}#eN7KURnBH$V@37RJ(VWVB+f>TPT2fsZ z$*}lSusBptUXC}#_}_cjEYo}(zhf`vw(JSp>grhD{O-&=td@@E6$edd&S%Gs!e9PP z@yPw=(oSnY+;yCu5ietZKl!0W!Z}4hn#SWwGM> zoh4{o{t{>&@o~}>7;`2JXS&vq%fkooJGQsA>gQ7n#>0+yH)+=2LiY2lV0{W(6F2&3 z`=_^%pEf%o^{u=*V47tpl1HF}5TWo)>B1T7V&s&YK7MDJe+Uh-cG?|I6&L z&d9OU<9Bh?v~b!JuBvz&(-D{W48h%W&sVRs-}tNLQFv`}360Ls{_bhE+9}q%kgx&* z!>!YK9_AO zzD(!Am5{b6!gjAy>3-2*K{c3EJ=zyrSl?vE3+T)<(l7PX@53C=Tt+y-hA#SoO?M0e z%aT0immG-1J)HVN=G{z%{!#RN?TpqsS`VDKCIl8b48vh{=d%dgWw?K~6a71#BV?cl zG>6oKBhht;ju9X5kf4stE;$E@t3cr;%_ZE}Kdh2ZyH4wX<)52#!bCW7|2P`v+i=of z!fsapuD;&~X?;b_M@5XVkB=^XSJ}72H#TBQS4_6A!-#W|sL$x<+mQHA)HWK&RNwnX zZ%5KQu+(%2)qD7e^}=VPMZINchQV+#sxQ z`HMUaZuilM@RlniaT=R7b~F-~sd`d+6f?%^@d@$8iYI|E5WVah@~JL`Ox5Y+^j%n_ zq3>G#+i~h85$|awY2LNO$J*gOwCBT53z?jkk6)8wq1U2SIG|@aEb61rl}5Y}w^yj2 z5oZzaZe?{^ZNmv0_b}2@NlE7>;`1UM629PoWJfTZ@CK|x^Mu_y1HR5D4WDn;V!b1Q zm^i73G}G=%oTvANab)V36aCwzKl0XtcI zP*8uX@h~&TA{ag4KvMUBH8hv@K!4*x2f7cEzLzJR1m}xteFMhx@veK}eZTWqHS7SJ@p>z9 zeQQi9+YENv0C`|+ThbI)arnl=Nz?8>VYEKr6*UJRI$uM=NNilMirhAvYLjZnujfOc z8j>_mq@{LYrwM>jX%5mO^ayqy$^Sv~q`9E`4Gba7ryY_u zV|6pOYH#Ib0dsZbG_OFt%5v}Z5Et@ZR(dm~-)4lG^1G7;(7d*g+Sji9&ivbg)>KPg z1XAbc;*pqd;?m6*bT)y85teAkL*l{+eR(|o2rkO>;Do0T?%76-h5r|K8%9mDld7I- z=Wk=r-ZI58q~k9#&qopHuyLT!|7phzJJ~4 ze_A|;Hmv1@bn?nM&}C^n8XCR9$pa@RQU5_{<5Nue3AM&lE0rA6&?p-e*MRPjeUew?JaQKp&F+(Qy8keryLlv1QiC*n6Y==mGId%PIyP}2l5?Q!=x@K z{gqe{oMh0wz9iknljCYg#ZTntaQ6onOiU_*#}+B7*056oonvHFjZ^KzWz%hpG&GP7 z&=C)7OZ-nWrO^~eD7{KJj^x|5$|D?3TB`IHyA}}0zOOE4TiS5K!PaGB~d$f%;k_i@*r6$x(o}BR!hSmKH z&9nL|?Z9aLVSwKW(7L-(7I98-2h4B}f}@$oxWecK0Xa<7o*IwJ7pO--@jlfeBOV6A zJGQ9LIt_V59dWN4;Sn16y{pV72&WZi0c9ULsttCqtd+tnopHUrP=of{s`YM%F1KYG zqpU+bcza!8j;_PfWa#nljE+1G`4@9O!E`E8Mue9&W96uX`aA*Z$QfzgmHwA%M@u>w z|E3&#x1wz=YDaRWHIs2Nh;BT2K!?g9VMZ0STsI+xZrP^k5lZwqkE%08$x zl3K^15l0o)Q6|wJi0gpz5=K~s)HjmGhGWjuld1=(UdTKApq$KKoL+f=)@!}eZo>DU z8D&__AT7yD_DWl+e7x!r`9d1Lsmdrg3N9%$@1BCDbDIF&W2GL)X>?SMRsAe`l--EdF> z`B2K^6fRZUV$1&=DrMxyu-gP@NGRTwME({m>(vmJsaqKRE}N1Z4Ep`;L_%?G$_O^$ z>q(tJ`TB_l#ndAaNL;LZBVKTRg+GFiDeaV?eVw52T44$v^EpbKeMXH@`oKi|W7}9z z-T(^UT&77J>DK_Ke0auA6@+VjrUheQSy$fM{S;~Yy^ulsJ0u!U!k3K`mG*>Vbsbb@ z2Hfj9I=~49(c~(4A@mAHG zTjP>Dl^lIK<{cw0gBL9|@MG>bpu9kNHNu#4tlkkr@n?P{)^vKVdRb5&Ac*G}X%R^} zsj`QajwD?V(FR9w!AJub81fy-LsL#Mi1#mB3ZIB?aYlv6njnBjAT&;8P>+}NnGUKEmf6A6cWzrr+TXf1xthKw& zZu+bfUnjMqS~y7cc^u{}-2^{3M=8%NnlJ5-gjp)bU>*IsfX<_kyv2WvGBvcGY)m@w zq#!@1>V%n;6tl-OQW*JfxolvjP-{E3#9C!-iW7zU({MsIJDgO`ln)LmzJQZ{KdFP|2s<>ubAUGiZbRM@L|6*Us$d$$PWn0W*OxVs%AL7FXcCZ zUW#E*N|!R+6COqUu4*{g}!1dNdXE2VL`2qZlS&U!w)o;_xfHFje0ec zXUi6gaQ|bX*V%i}*wRqWnNRzB9~=ympD*Ao!>hs1tpV6#umx^2-@&$RYr^wfFRFFH z!HwJL4&3T0UYxLygjx3{gDadLyPrEK`w47DVTdddbFGqBh`1IXR9ANt!{y7Xr_cPYA>z}O* ztAzm7E6{v0i&+gN4&Q`HQ-{Hj&4*xIksGe=RGqKZ+o`SPafH|GHxE58-$iGxhaCg{ zz_7!=MCXV7Sy*UasBN?n2YK9vxF`GgQ1`0R=w~phZ|AnzHjDu4-OHbE~dETjQd=MOzu4SzZ2k zpDNVakDZW7bzuW-oR>(dM=V|KE`yq#r+bEnaT+7&U)upDboS9Avvla(g@M_vv5?n5 zBlVV&4*Ajm%r7RvcRgQ0bI!_VJVd>vzYKd6+r?cG9h`zJltk1u!+b7?L`r~7^UCpUz=s1+~Z+Y;O z&eo-R!DhV+fqKpLq<0+V9>~OzyG(fB#Pu?x$9ZP--w2Ui+!g~Lu7}a`e9~xJP&2wkI@1h_!`6^f) z9?ESlAH#8$&ah(AVA$R28XHoh2G?)9R;oEo$sU1JPYM%Y&&t}I<^wNQ(UYsDMnkxN zGuiH-JMZ zeXe5lc3KQt*jz?=?T0phR;@0m{MV{hL}XuHS+Xnv_Czl#eu&H#-SGgnmQbC20co6F7k&}JJo z*?(i;(XIFJtVt%DIpd3NYIys~vD5W1ns=s2TPwQ+r((lAZ>(wgAJ|__#PEv^K|wCFXfLNaByVZOmtT9S zTYwcTtJWfXZW_Zvp0(s(riU}tXKnYtLQk)W(5TljoVorJQ<(JW$1*&6*ah6%wwBF* z_fR+tgm1Wg+HVZyJrdxm{!U*cv{P?k7SZsFNOIh9bIH_*Ea~!FIC;L zcb0qb+|z7m;j#y7H(oDv^<5d&AX9inbI$8dUxUQW(B+y|TG1KbR)L=UpUE&meTo)2 z+hN!C0&UO9aJrqNp#ZX`Nkn8QPL+L(NSw}yHUl0S|js+SsGkVrbHvBqLLtG0* zmtC-PX>ZaA!Ju&1By7D%eR!IAdw+$ZFKwV<&HG3g$lg5Z0gsKWRJ|k-#$$d?1AcXZ zwN~}D(Jl{3Jju)tNKJ5VIjnr4BThRBz0P_d=>}D2tVNZBFm*8E98UX{>eb@sV#i?I z6B|q`TaS_T{9xPTy-4+nS2};Btgp3DuivwGJJ@bFBu9Pt|h@S1HLZefbJbrau>Z|T@gZ2PeB<*5^ z32@Re9=!^VVarSYn#>inC#XwTBy5rwPIs*2y0c?_QPpL@&@R|#XdT$>LHiCpjm8_F zi}8oO5Zlbn2@}m_T+CruJtRQYG~Aoqjp_6RtaNrl(uZ3AzopF2(O4TbZ;zz0^TDhY zo?52I1JOv*xcGrvSH!)QT-K_3x{`~Dn`BMPAFSm0Lu@dTz7I>W5 zzXfSf+Nb^xF8pMzukFql^f?J@xi)7Dy7)5E zms+pX7%Wf7AilJa79UNd@&`*!SJ##-_ho7wCqMrph|ff?;+uG~+gW_u%Y{6Hu^jyI zBA$r+goOE$YMr?A0X*NnM{M1ElhK+3%{fs0YiB=TlGX$We+BU`X_;g}J+33{z~JVy zA*jYBB<%|?YONE5SCaaJb$zje&3+)j_niYDJ7J(axg!eN)*mj2JCue2;!up?kJ!Pg zwK?f}t~g5ho#nwF=|0mLf_ejQT|1=vT(1S{nYV^Lee21bu-d$Ld@R0pEEM!}g*igM za4Q~*swrw^b;7e{--JW#5;XTu(RFr6CQtO3RYXR!TNQhO>PqPJ#F8R;f?Yk_dGTYGNZgv+Uefh){Ep;-+y25jMCSy^|q_CVZ6dOVcPtyz=SyNRUt zL#xAgA$H_&Ade>gJsb@Vv9wp{nx-=6j)~HHNF2`MGdDr+fm)@*pyat1s5v10h!z{_ zL3oy}Jk`rsoE`fL$P0e zA!%jpJGw`Za1q+>_vKxk4fxyLEqJY}?RbM@Q{-Ty^J=Z}pobx+I%TxRjQkGx8bm^~ z-V=02yY4~zIgO}}MrvPo^X9|p`(V0$M!FB2jW+}7U)p2lobtk)@Lx`#bAJd+pjQU@ z2a90#$FPFVtu^PW56Mr$vL4Gozru(s?rM=41*QGd6&4}!7>@jJd*yplZ>Tvy(!A(Y z?JO?d*%8QFVtCemrZl2$^S!M7#&BGJ=U%0+CQXcVok$uOhI}?tcqhnnA?YYg-@K@D zUU`o@xVUyBc>UFv&*5#*HRudD?u!$hMd{pLT4UxUBOqSVE$dMsTCOwZQS&VMwr{({k`PmHZdwf-o_>Oo<*#6U z=Tt`A3O4i3Vg81eNSqG6eLE{26+QbFB~g8&dB;JTFPC;A;SO5Une6XZRTBs43|n2N z3zZy7ID%@fVvnT@(#k;f2(D(~K(B|Kuw6*njt#I~saw(3ONRBk5BalB=~k@B5UyqR zoUm1=H0x~Pu5>CtN$37i-GDfF6jBTaCn*j51Q!q@Gi_1ii!BF~zyc#}j8GX#?CacQ?>` z62|-zHy7T6K6dpb^%Nd>iHy8F{IF+0zDE$(gIQ=Xs(LC6d;uy?A*_1obmySw$K^wIU4Snn+XAl?SS}3Te?r^C^rDqm_p-U#(YB~drTj(Qn%9i z3~#plkZ7nXvQdy0pe=Kzv|k7d?llLVuYo*c|$4Tt-kd7PWSkJ zXbx`{=YW3ODvWr6_BTn@(l{%1sKRB(^CzH!?iW*B)Hjj#-%hQmOD$~$?@l==9RcKx zarW>b;>Gt3Qu!Uycu4C9q}jx@z?u+umF_1g{mk@&t4_@C5mUp@3e*Gv)wJ@)5W95%j%-y= zM{BO~8KfKnUmg1bg}TRJ?%xfnPwoIpKhy75WwA0>j}H2 zJ{OYI!V&%;)z^7p zwzVfGUSeGbtQDO+$AiMooV8Zuo9J2dzt>0$Edc5v`nRWII26;~wxvk;B$TcUa&Jf1 z+P@NCX4#K~b_bKOs%9V74 zrwPq(+|sC8RGFrJA?>Zz?QxP>s1H(JiBwZ^!-wNK#e*tyB@ZHEN+zqihnWZ8>69*| zn&#y*yx{0;Abzi+@>pR>_tPtXt8^dbL9EC1GL@Yu?84MdI?9#GwC8(Yr5y1xjO@7= z@5U7{<(a9T*tsQ>vAqbvD&N`>hPD-|zXAd}qnObevO65X*^yU)@*tu7j`I0_-g=mA z(_CKo?Z~eLOoCw3QmmNcq_P9pH=!IzV`@~cdc-Xa9{UBN($vcLQYIisON%)pPax@9 zZKDZ|F(bIS!cLXZAYn5a)_SSD6pam$bexVfv)JtMgAo?f`m_@_9=%ciMWej5%86%H zdkhPYdnHj`$;g93#e)dZ;#DIgou)E2HYg&6G>oC796}NoL6?g2f_^TJ-fzR~ue#xV z{i@PpLl|-A_M`#+A7JX;6FA8A9z63(Q(1&~{q%dl2JOPzw5*IU7ujIW% zt+(g2VNRns<;&!a>T=2&k#K{&(i=!WN_w+bJdpl}-bHjS=7&zC9~SU(dN%3jky1^| z<_~b|cRjGT+|AGZ7%l$QkJWztIFw(%`W6zrs>y}T(=>e5XlQZ%rp90emrjn84aE>cAsAHIOZqRfmt;je-6hw&cCTZJVy(bW45yZ0|rx zbD+i!>T_@IT)-cXaAP}$Md{{`od{jqIN{j=HTlY`%h9c>0nTao8T0CW!p3uJ$yI(Q zWx|f$a#-IPven1k7ivT=-{P72dF0%uj9!P0JeDy-$0o9U0BLwSyKCRQXdHGZMUJri0;j@V(S+^=ZP$;^ z7oq0}r~1ZWvy2Y%rXFz=4ufg!Yw@?UT1Y3Ef}a{Rl~>yQ(WP!!j%inV^7zjEH;xYr?I-ZhR9j42zhg-@E9*-czz+NV1(3yp=YI3Wt323uF zUe5ZnkL~<75A}EF>lR*oLi?AtMonW^zI{#yIpJIoPRvS{7ylT^=M&q5(Sf!|WKsXBsC9;#N2~CNL565%ZOYeZ%+aRBU_3YY zJZ!Pp$*298iADQAvlT;1Fubg~?7UzSjL!@cDHbKTd*^QGXzK%g=1oU?%Q_QK!(t62G*XnZWb*x|OrKT;NV>w{=O`-a4jov%qM9(N}f4UDI6s$%yhe5Ol+c@qo z{|43KSLwe0t_Llt*3QfJb;2NY#Slj0fetPUc<%AZ_-K=pFf92Ef3JD4=*-RHN4uAx z)+^}f9jF~wf~`K*VI{$*MT7jl;`Q{ZmDl*V*aNF|Z!Zr`uPd$NQ=wt9j%Pn~5&oy@ z0j)ic`nQ%Jru}I~b)r65Yp;8?J{a;{H;9#2f5HZ13Ex`O=Vpbk@L1d+RkLFL?cOr$ zQv=@UrvU`jy@-47tm7MuId^~f5I=_u$G8Xo;iHsNMq`3w`pv;fc9YZc6Oigij!Nqz z3vTv6ll49M?oo4L)rr|~a{n%%e&m-+_o97NoG|Z{3KZ#_~(!u_OQc9e!imv@Ab5$_Wt30`03>;{2qQ>KC0Lw zu0GDjD{WR`S#~P4Xxg3+?j8XZbsmX`SsG3lEZ;`Gg`GuGHzIHp_@aR%u7O4!dvk1l z0baMKbF;1=qPYsgiU5g(hxlw<4O!CYh9FMC5rMflc@ouu<660{adYjvq4kik7Y3eN ziS2ZD7_xW++py0I2rtlWR|DSP{xY1c^9J1$o9ezLxUkY{TCQro(dLGdxQU+g)8c>M z=VQ-J9@4Pa2&Qmlg;hInyEUFAKd3Hy4yh$uXK&y`FSU{%o{!>RLjQxB9qO^6_n*My z5s65z;iKl3h?+kaLA%^6xNe&&raz%<@^44vrV9xNle9G=mjdr;BVW=zmO0`$z+F?E zQ{^(@MI9*X5hH>&)W>wcFuLA@&}+_B^u++&JhhOAwQ2^}x110TFS}CzC9pcLHS)r< zQo_1He0bS@Y}tGSw5Df|#=pC*xEljk-C!MUiy3hX-*I&dX1?{4uLgX8gX@p5rMen~ z_p2Z}(^!^d_~G$Jd9d%Eqa=R7;=X{qF@vTte<x3l!!1rm#=(~#LTB-?o zY+;a~YtH#=XZwGwIItZ?mBOP_?&aL+DK+-3o;g4Se zCzPT8;>B>eWiFiW7*)BxW70pUpUL`PoYA^jOZeP$4rj)9v=$A#1l1Qm({2g6o;nFP zJcC$hvvNUv4bPgrN-{n)46^q166a+$5qPLPWOhr|HT5m zmOa>Dg7kh!_%3u^?Bs)jziidBV0=(F3Z_yvYj{Z@aTf1c;R=my+=zPx#*FU^|LV7r zdv+&6#wlZ7yEp`|+jd9lYr;DloOh-UsL%9_^aaCS5AfQtOX7;p5iRuzY`4ybn67(} zG?gUISO3Ot-XQ$^;w)!+Ooz^TD=@Vk>GXafx~T18Nb^GLv_QA>UnzRWjR)P>EI3B@ z?HaY}gSnl4i$Qxo;e!oZD!Dmuf~Aa@l?0tUPvh+QQSih*TswA@IrMlOK=@q-=EJ`L z)h?=34-tw)mTC@D?oF0s9g9_pRK;O9$JS@RJuS|o@)T7T6<7Gx@$E{&1JKyWo+-> zaoBN7hSDBzI-&sQhiuT1{srPsP4I|*j5r^v?W-=mi`S!P?c2=8q`v&GU8v%E@vU?* zk4xw!^~cz9y~U2~RCofqzWRXP3meE0gF;!tj_W|$T1&sfh+kOW5o3Tpm)D3a$0N^Y zNTu6|({XXBBR}=UT-CX_ZD>SirmPa`a|j=Fbu&+5zfu3RC$oo3aXuQ|#{8!x9Vfcb z9{884oI=t*Qhlacv%j4aC24l*8AHWKq;=N__lI8W2_M3V1K{f8Vdyn016CaCBdB&H z_4}oe#npJGdtdZO$|4P)2i83b7~v)i`f#1IiG`RqfWiMYkz3@Y89a_h1bp zjrgEDyMc5xqk3di@0GYYzi9-7A36)e)^3LeXerV*R+XK$`GLm$9Vkt|{>~R9%;v-a z`1WTCOFE-3C(<4uR3muRneKOfaZGeOtOe4qI`RbCJ#TBF{@e3F{ZHOvGB|eWLcI7# z>{_^zkxrsL`{%*BTZ6#7le>IlIaFZ~vgPqw{oP}Lv;mTK0;MmC_Rr_^dPeoGdS7;q zHDts!V(HgR>XEHLYosHt2Bne6AD~=W1H3vI^LpW<)p`lr5>v1r>4-nUCIT$TuG}5QOop zn?Y!~=9LNUeJ3P}P*os=*LwmzYto9he_Vz%25I-r3u&z2^Q94#H0mp+ zMtp>&>wICo;XWV_$4Nt1)=b7oXMU&qM=@%yA6^c-4CE2S?wg*Z2_CT8`h9t`s*ALQ zv*O9?C2UIXQl*mx^$xdQ;Sc1CKyeD;2-h@D0Md?FdZtKyrmVfN9eJ`eHa7UKj{LCe zm9Ijx>GN<^wN_55(?*(M*7s?D0A^cK^y?8hG%ZxR_WVLcpnAdZ`bgc{&7Lt zo$9_6sBbypnNIm(^4U0IkTpJ^corK1E#J^@jU59a7eES zLrvF1+pkYprx)pBV^1T#X2DQsT4e}QjY7(_bZFapJ&>PKb0K0^uE3Km+Nf~@=|dpA z7SAtjqQBRYC$uCkGnKYAyYe=@E%?{?9k6Jn9jFY$_)xYWeMWuV40oMu3@*=(=!i3@ z)~~_u+4mtO_ms*;fcz8AzkOZ1ulshO_r*7scPsOcWU~x-Hf}2@y-WE7P=9C@Hho<= z6NtA3@io4lzMgQtD%XzHBduY|@~<~kUVvFQi$T)iNLo#h&qTr`?X6qLvdzE4`KT>O zeFs+K-@&YF3vrGYeYbddEUQ(|S*qWy7oLZNy~>{gc>*|K9xBLpiP`lt#qrnHs5~+0 zKyeF>Domhp2g9Co@5Q5(dCHF|JmJ;*{)FCJm*B}sAC(4D8dZ=_hm2SK;luM>$~hd6 z<{P{lToZ4jX5gZ*A<%Es8us##J>1^Y2tTH0py_2ZooH$X#BD(HCQi19!wFrFlb_I| z@3$Ij$xjNW2s6AC>j{(zkf%GORUS5>;Y7Tg_*wN0;h_s7Z4WTvI8vseOWgIIp5?3~ z{4yGI@@r6cNeHmuMP2C%34soNPdL!6MC-xf)$YFgg51O^4-ud z95n6}wkdTZoDIV&df`Z#TT+b*&*{h5JDbIfbci5b2WMwofuehRL3B%@T1rXWR&kti zDEC*ctKsaw~M(=SLrq=SL9ErXI{qqVa7l64h&(NHg6CP`+ z-uwmOC)CKZQ$7Y;UHnAvYeDb-NpY|)v+*A4=cZioD!mV^w)>%a58DS+!3oP=SMmt? zGUZWdt>%+9_Qj%mR!XBY$B?}^#%(wf-@$*mUC{2)QP?*z1rKljpjC52xrfYsx{=kJ zVuQ?hFtl2E3e#N|V}Qdgh3_nK?syzs?=T}ig{QltVYTN$g%c3lL&rQ0go_qU&$2ZO z?qcoYx~R0O!ULbyli92F7O3(o%5T7F+6?y5%SWipgz`6Dw%=c~!5r{bZ$d4U%uJ^s!a%L}4wrV)nAFhHY4W6h{`05R^j-#pg5^ z(B(7jH8kK5ypBv0?_4BIB9Hz2_t=LMuj8h~+VH-UDeT?*1nK`N!;zr8(MLCDm3u0DK=J~3KW{W)b-GsNw0@%* zuHN<(Twl!=G*^mWw1=jLROS*7J3Mj=E`F|QwkV&GHw5P&HPGSZ6^txQgYK8-$ffIE zVcDbt3_sK!U-UX6Xx(x0{-$7fiK**k$!7i8vO1fI?_QBMJP7pEGyKfARP!U1RvJEa zEot`$V%iK3X6b!R%!sPbcP;$Q-W6WJn68#8lj6j6f;dc4c8uh&f%pnGt^J5j5!-bN z8^}k1-)MnH-BQ`V*sFptL${^>J<7p+8F8|_KYbxm=7?GQt})6$bx$*=!^!L%g+18* z$OUMBq$803p}H~j8*M(MqYOZdThhJK zH|EdljSgPX_)jwqRSrYBh0<_9ITz$_uV9K-x_tKGlnIOR-tn}~J(V{l-FFHu*z85} z|CE(o0>!CoUR+Z7HD%4WvD1q_l(VEU<@alpXMt5$SIir>o272}$udTsV=6~d{Y_&; z<)0{XP`nGpt?FXIsBui?qKAi`6vbcWqRL-Q`ma~oOAuFb$_ZfSyhk{E{RO1D!}acY z!XM&c@}$2?BTLE?V2GmyC+tG$NH_DOp}rU~sUrj|Sj$c3yMqIH++{1Lp}{kq&SByLUK*Mvns0Z6yKQ@MiyEKc zyirH_yKE=O`P*Ri&MMsA zqFC|E8anF5f!m4ca)ebT9-NzkA|ywAJU0=%gZBy34uG zHMtnGvp?sv0`W%o`bal-#Lh909j6z zzf>TC7ET7MJ2fyrG=e33zrwfDeoKGrEChAUyj~4%)4~~H`#`?D;Zr!2j2fc?FX@v| zO$NUVfVV#VWP7g~8d_^kbtX-67s$?od%$Vk0b~yk!c6)uCZ|qIo;|+sQgku?x2jUm#du)`c2T_% zvSr`0@7XINNh!(_N{J#ONq5d5+1F%=D6+5FW%oPN?~nJBx9)xJ%$#$+%el`pW1Hu) z?u!5!X$yq?Zl9PTv_>1kygT=epnqUklK``w2RPF5!df zwM3Qo{k>nZk3Mt7sS`Fhe50AXWBUhvnm6X(Z@z-y+%|&p5b~18$fz@=xW%lRtaH5& zJWPD1u^K;^Pxjv;O{wQPZlXS<7%$h>yi^^l4IK?DDzl;f_891~GlY9z-6CJ>c0-l> z!}t2i`AxP!zU@$LZQ_F_j{;e=-et7oZKcW6?(#%y5Kj%)vb0U@aP|^9_r(1R+Vs<4 zojfDCzow@f_N5S)kK0Pm}2{GaXw7*?wU zm(@Hi4$bL@CULa4jCl%r9heP+m;F zPg{p_!{o!kHkujZ7 z@n2{D&AzXUu=C=r9`2T1OLVy2WMet3)-d+u6`k>$o{rzf)#X8xrt!pMv4n4Ru#e6S zSa&1>Kect_xsUX4$gGcW*r%pez4yf8u9ET}_SPAC=aVx^sV#%-0YT`#cg zpEfwB?=+#V@nBs8T)8y^D5kvjgFEo)en(Ci!zsU@p2Z^^Qs)HZK3XePycW+4fWLpV z=svFlPRh38s@6ZeJc(C(kKqC1o}h)HG5_V)6g%8pfE@-$%H=qV&wAgB6P82p#6yHz zvCSys~%CI=Ow^MNZ=$G_+quMkO`gi9I}(zwcjYCtwl z8VuJXuZc@XzQgctJAi6WsvH^>)tOuG>yF>2(fMI@*Ma-}Cs22MN3JlZcI5Cp9j_vB z=R{xHOL-%ln=l6FzIlwi*Dq$%Ih0jT9N_API9G@~^H z^P4x=C_5c_EV>>)zNIU?@u(rG{`r>8Z`iDR8wmeeQqI+37lz)%i(Y3ja^(j&T|E|G z9%za^j=G9qp=C-&~R3pq8eW53}Gq{0*Do?9`Y6oui5!O(c!PK1lm zTH+RP@yJp5%69io5X3vIp2cW<=ayVLlSKL7RC7Vo8RD z+;?}or21gh!;K~3C0x5{Dfe{mBvOq3V$I;%%qr(UXxeeOR2;R_$VJUlcKOp=687T6 zMWgaMnT{6*tt-UQF?Ye*!${0qXoa)7dGgsyYSG`zaLbBLm>1caC)XV$yZ_eb#2ebR zs|JZZ8PDO%HETh&2R&=$h=AXR*z7CSczJX&CY)adgLmr_{$}91_VdC0tOF#EQyJGR4;e4j=0Z7W4biGgQrC`mj+#Vg0LJ>#@H}7U(!{V`gsD`yFi#y06B= z-u@%uYys_&W_F3z;4vFeV#gJy6YkL7VVQZ1_BEi-R~mwyPqM*?sO2ya)8J9@K1SD< zgkyN)I{A!E5$HLrF`u*T1*n>>)8Y(xKk}F`X&BL)rir&gF9G{?772 zQeR0JEmgfy9pSDc?cj6VXkoI<2s_eVt3lOVfjC&)`)ZG*M`Y>8GqCctt!UM2uy&JE zBDT4ILe(Z1{%9hJW3lTlH~zJF0P*WG*V*4^$*1!LQ2tX6*O!*V+oFzhQ@k8~1U8ko zkV_i)^5{>o_^shaHokeD$WCvKgkkcT=@I;Mz@2YfTgtW;9+WB;o182p)e5}XcZ@h< z1FD}7lIUCk(TNAWUP*X#j@7&rN?LBaxaeGuv|Jqi`_NWyD`=te8fbpvyF6Tl(yoV+uIC+=wwq|bg&vG)~55T zqsJ>w1*LHa8{t*X8Hl&tpc(l_qIWFqle8=b8vESW92?XR^ODO&7Z(FO;XV(QHc~Zj z|FS>*eK>^n@Dfg0S%fhkAi}7TR2X#27vSGmtz1 zdK}-OV$NpMS=%w2>d0m~YqW!RuXFt|f3#*}|Kl2^QAV$<;yWaL$fw|CNSa#8Nar)c zG*xqaZ~sz_*PpjY7{*CgBH@bm!20J9oqHYJ*G&e(c2w9m3i23X66H!4nXo<|+CH6( zd%Nt>EDM@X`(+*yRP(rSWGzkEvKrFpZ6rLAb>)YI9*~nrXFu&}A%C8`uO&Z0Yg7nB zKCcGyT1Xg=A6~i(${lR(SDkOQ%0RcGJn$U0iQV_P2E>-cv&h9 zsPyoS7G05a0P*N*%`cr@IR433;?HtHwTn*;_4x5uPJ+A#quLRd_e_OrnwyONpNFis zkaw$h1?$*Ig|Vo7nc^E&r^JsCzNC#X^;)QOp13tWRW|K-AHL>qhQ6?g5swQu*Xg3o z*>I+3+Ei&&xc}D=_q8_V^>po}L)<$gjYavk7M|Li0*Z;)y(Lz8A#o{Oj~7ho$t!Es z5Txn(n3m1>vj;P9aR8k?`oj%6=M59nK4*x!IR}`jWo_Pj+YYEv9E1IOo)Bd}XHZPt zS$~%f&|z8uPz|^$PEon&J2P6_{^%xD+G5)7_EOKWi1fE7Yp@}dO}ZFB_klZN;+1}C zUP!NlQ(Dk#o;0OBCYECH`MGLts5f~tQ{I@?4fRa7Rrwdf4EeSAD$2 zYTT@yWov6A*8TUos`(L?!|yLfOmW7tiaOl(Z3pFJMA@=1ao21nDqJFr=ae7pvAKzS zJw%{WU=HdJug$;CYKpY3c+JLKYdR~Gkron!{doP{MxdO~5QanFynk?ehzXzE)L)R! zk%UK3xV#%9?xQ(({zZIctw146@vO48tWk!lpfZS530holvnk5li&kNSMv0j zODON2B7Ij~aSvL|zaSJ>D9ol>hmF4gzE3+wPU5ggD3R{iZPqv6OKS06&Ak8c<6^Ak91HQ|S3NtzBIZ^(|PkUr2on7#1Jk{|# zQCTZYs2VkMsjUP(eu7z!LY=P=bl2(W96B6kA=P+oLIbP|2 zDm*9r097Z{XVVY{F%={7mZJ3jdL$gzoM==NOEQN@<#R{}Ak`sT{XPhkPo!LslRuY~*VJD*#!gzalZy_nMEjz~8uFa*MDrc*X`f++Q@-%)Fd4}I;nVr73ezH2 z&Z5JY7XjfOk|z_oHumJjmqNsP|?pVfDug}!0QVh+1t1yv~mSw7MN7n_? zsBmEYL`bTUL30m;DV2su7@$dP)=Q0DliP8XiktFOeC?`R$}g#$0P;`5;Isyh|9g$3 zQ#30NJ!TX;5R|FiSkO>4trwF`oapi;*7S)Ptb@LnMn!D;496>%$Ps z1>!`^d*vmlzE%)l8pu(0AxJzaRX>DkRq0}i-DXC0q_AF4KZuQZ97jEbXQ+BuE*{Qo zO*1Q%>oPTRl~{Y*N27TA65WqNpQq(Px;(2*uqa5?lPb4Xw`tFBE*?StWD51AkhHuG zuhsDwsNV3*S}T~UY3gIa`(76*$5Vj#2wv@~E0(7n&m+$QVbAjwFQMX=u)+wYVntq# z`8Cag-6OB7{xXpE)sl||!Y)SoNHhDf3vX&}4HUB~pE~S)GOLppQ+0jn?P+dwd9P8u z2EumbX<=v&9VEPlZ}u0Jrsso3HB-GD(W=8vM)|Ix{xuSZ@h&crU^)B%ES`A}sb`1! zYZi*1St+Qzi)Wp!(CTZ6Mrj76#Zq(UG3v|Vn3g-(#0CyPK7lV!b794!yK#CSG%%RK zPBdBt@wZ}8`8S1+q)GEqbHAv*yLj6#6UhHzZTgN)^|(m?!Lo`7)o0ZxZ$LFAD39dL zM(Ls?!BR_pN_lCZSS!zu85VU^Z-=j}R)(8j_tVnYnd+xd?o*H5LVCTR=P>7GG4dU3 z^}SAF=Gr(xnio|KbbhR6uy3m5JrZ9{PM1s zVB9x>u3^J7&8qX{o{n-vaTmGu)+(8reF7k?H9TM6Uh1~r0}Jgl;pTxcve@P}d~dTD z-qcJ%mp&gM&qo(@r;SDny?xMhTfF#pv%54J(VN+Y9YME=rZkSz@Hc)f?O&+R=U3jr zhEdG~eID5OJJhn5x&aXD#E9{T1&5Lo;g@rpJAh+;5L}i~BKZh5>w(vyO zsd6^XFC5F4P5!5i44sYnr`ll$+m3j0^7y=QPG)kUPM)kBYc4Aey~o?1m!nabp#4p% z^V?15O0V@Z@TFW0LGBHJPMgGcFJeThmJ9MR9xa9&@p%Wu9hW43F?wN^c@ok}dOiL_3N9TU!#$!pl+`Nf?`I=9GtJFHN z4*9erWw&%;pT@+R!gLN z1j~c%E{c5)N8s{1U#YvS9>maIM3XZNv0qt~*lJ8haz08qXV=<+BifAHF=JI#SH> zL%VkJjh_MUTltcuP8clHtO^;eFJ3maXD8r7L7|`UvoE_l}iktw*nt zJ20kX0POr5&2_aG)Ob1G93>ZK7aqSI?Zb3ab#kG7s~RziWt?SJTgT`OLd47&o@KexLd-y;s87^Nnjp*S3{G0 zefe+HBs7ib%tl0ba~dPBvCWd^F&J%pp5hRa!L#YotMhJF)dhrht7PNi?K8#lhCktIQ&pknP3+~)lY7iRmS*P9xQFoGA& zwBtjfPEGi|$?*BjT68#nl--C4XH zYT6cHa6lNAHwM)B4TLn=f_KQeMRlvo=jpBDQFN~Hwe(A>?(lZg&rDbEFs3Z10W=?Q z$dsV-Wttkh3|ZP;9$9fgL;1~hYHd^3fmFA#V3%RXW6wvrsxjD&aN~*QbT)4_<9Oz!CYb%CQhZn%4<+roLX1%AE6xvL7V46d!`Y7F~F ztYsat2BB}w$Cw+pUgbGx*LyHpYw7HRIACQ2EDtl3MlYsHN6!LW9<~9BX-_@h^L9vU zizE8SX-#wO72g2mFVcJ9wqIlZAg~^+?XLsLamKRwndGY1>NSkRUGE-i9BZvbJTO%d zUoeaEZ02Wp2B_}Eg|to>6Q6^BNu6>0JGtYelJS9PYwMwo(Qb~J=#(VrpM zslJBRneeR;RqfIHn7g|*JyZ3A(fIgtxAS<$zcVcKNru}s+fvQdkrrBaPBjl7^8Q2m zJV}_NAxv_Hu^!&;oMnQ&Z> z_sY5q|E${b+n@j9uocsUQLk#U^`-0Vd3K7X@|_z5neD)KX>*`PKIvzjx{UG^!EgYr zw-tX-fGp^vJy*3bd4x4OmA6s43QqX#fH!`0&tQBZtIP_+%UPSjsXPOnbq0y#=C0!E zv}owCtseYrz8RC7=P}}9?7A`*Paf-!4-zkEy=OjPl;g1K^&k;;D+(#sG0MjbHcSXa z_mmzu3XpMfB4M*8bh1xF4(x=sUT%FhTVP_fq2E)q%U{>vsvKcr9=} zmv{p$cQ_(lpU#qMBGaBWmcB_x*sIk~X-!r`YKaq8*j#bNo1{1Y{9;dfHk2c5Eu}|z zHFvUZ#-jz*CNr{J4=c9%!J*|gIw$Rxpm<8cJII}8hP`a}F%`=y zuG^6c4>$iogQqufv)5Btaq1$HcA|Y|N6XX?KeREjwRFtgMf2~2sp~IkjAt1@)3IL# ztr6WTdR%<2HWRHY+OiV={VX!Bxg_n1RR5ebD=R;DQwEteW0d1)70?R5|5t&!gSX<> zFLfkoHw<#$NI9^X#|B%2V=X;UnulVEKhuxnknpZ-%AD8W?w&$tvg|_AFJS!ZAUc;k zgy;4vA#z$9#i>}iXsmRx`lEf|vQ_Dos&(kHpBRz--Nr_A@M1&CXX%kqX#KGR6UA>k0vnD;VNrR=iiKfdI$@3jA8TE z{?Kq@I^?Dgr1*X(oYlqYw@kF%b4{4a$APDYW7w|{yzS6}Uobb4v|fZ=!I}r&qcJ)C z0a%ZVz+GeAdDE;?eDFGpgfKlOZ~Nk3+SXh zrcwMu+#*i|Zv^5zej;hL>-ato#q(@yxybDcBmFCtzW!Y8y=LyaFPKKps1%>F6+e1L z$i+|7fYy|Uq~v0|FD07Hk5TwCzJXBs$uKwoi~l?VRWn(Emf~umn_POnvEqHH#<$Sk zvC60PYWxdJ@=7Rv5h672FC%@WcvGmc;>`9y+J#Yl!)4kdo^k_?daZ==`q@6CN_NFXI?t9uJJm60f>01;uGjgV$)@&FL%?F1193 zpT94bBgKYdu^E&W^XOZPkp>q-T(0AhDIqMOou5<~PIV~*a_a%r5q~mGmvq^AK{{So z#gC=?UpsKcRTt8-!J@qdANJ(}OV-wrbrzh~5-tfEBcL>eZHeO;c`G@;Jes_W3FI(O zq@3eacR*_cN(ZUmM^4LtRMQzyY~#;jlM+Dj$^I7(U?m4bh*@L2yl5WF{8*3Y()rQW zQ-`9_(WeiNxqrNV@@wzNlu!FGzxAk5#Y z5v1+l!qvUn5B|FuaS_zfnTTmOC&B#6!MH1;6A$sf&KBypYG~c)cXwdK-wf<9cPr>N z|L$6J(_fI*){_6xl72$+VJxuc1);P|icd5qq#Tqw1FLi8Qq?3KW-Bfz|Z+OxSO@- zx!RSeuzmitY)17>I-2&ZORrS(;0=$`{pGEmFw(Rt&ndSivpIjQkYd2~8a6^zFaK8G zX9@#SeCkVw@QYA+uAfx-MEQ#3;ehfRCTB!q;rio@{3lX8#JSgbqF4V!s@*#1<i3_1q^NFx|L<6d0+9UpkHU8*Xhj*p>qEkNv z3+0`HE*({RS4)1H<`SX&754huiBTVcEAHr{byONkT4sM?^9J&gP|r9+Q0zG6jPNK}qH(4}f!lCY;_6W?!CY zsP{p=2wk+gRs;P%b`<6BdNcA}OzCFAP9EOz6O73Y;J-@@NneI2UeeIoN#BZXLg5nm zYk9D?Kaj>{s&+iWPm%Y#iv}|n3exIIljD&UeFXUgZk*W*2`f?X)ZnCN1z}VbXP#@&4wY6UU!>VQXDI3Gt!TA!I|j8IkE7a$;HdH~#7$F~ zY0ObA=~S)iGx!?rrFnt+IARW+z2&vTT-6ZtZ9bCe?KlPp+x2CwKX%846I(<1 z*i^V$ctYem8z>BOB`*cu8l!6xYp>I@8}|EdG|GI6r>6nuR$0_Ey&!q3?S`{`X8S=pabrO1m&8VYI<*m}o)OEf6fmmX zj7Nrkgs1gpKoBnD=yTZG)%zA z!T_MY29oE7NNubjUn>@toMF_*qgaOGu$ZmN7m-IfCiWX9A!#Wj-GZcNG=vv;am7g_ zO$Y@`_fbD5Rzn<)XKG*3t}r~pe!ICz#f7oKDG*t%t1v-QtqO~{%j8YpL(}5_)Ht=8;ND2uPD9*Jy{9Okz8KQ=$%|PjZoxBoPc-C{q3PV+V4Ay+ zQQwPE4NxEE0i&29)jvJ|HXf9YP`*!b24M!Lo@HM5DF;QN8{MC<^)UR_sX=`YHMbhB2cIj7#AB+ttTuP|q97x4dh z7%InZC(qedKA*tk@AY=fxAYzS8}$NA&F{hhCo6n7)Kl?;2wK{o`n6$PVJ&HIrO7lZ zr^v4hr9X%ZHL7R!vpIb)rL$KM4oKCnI2XQ7GsWwhE8(y5fP#8#j5HwJ_L#&9KCU9o zHj_QvybPK~Op((AwV2rPl}Iiu1nQBZeNs|rRd zug@^vT6rZ&W5-1kUNW!j$7tHN7LtBrK@(TQkihP&yfJE2Pek<`>s2^2?87idrbEsC zT^V^~pjw6JZoQS}C2d%fFYa?ks2(Tj87Oq~MT#F#9g|^j5EM{iD@^h}l2U?HJqmVq!N|H7P z>WMO{2cfh+XW;;9Hy_p#P{du8C(<~@Ah5OO&Z4EJX))LOAY{gaw|C3oyZsUk!)-vMUU`el&G%gu8x32Wesx3#& zGvK4&%mg*Qi+Wx%7`}=R^-t4%O|CHTH}w^_+K5EzQ^&ks4N3hu?E|UHFUv8oy_OFe zFKUBdr@6?jy(jT~ERxo9smM!!2UIs{=UcP+R!o_S@?)_Y@#2^&ksBb zju+d*%sZa)Lw!$Kz9=2`%q!By-yr+#lIQ2tlXF2IO>I7C={nkHx9NVcH#5b6YILuleF}X4TVL95xd^?Q=D_D%>d!PB z3(fW4u@GZ{0fuK-$hR?Yi1yUF^eYY;9sVeeBulUkuEkdj87MnUc0-FeJ=uMZKWuJj zPG>p&#yNo(wOi&^LSUDBqIv6cV4iLy29=h&=v%f|W0GesWkR=C-lD9qFK<$?TvDx2 zZid488C^u#``!30P=^(*k% zlVE6A>j2VvXb#L8BHR9r5nqC5N~&R%+t9W#9Txj8gss|{kC#4q;sz5F{PD6jwn$%s zt*5!d`yR9x+v(#_Ez3cMcm9W2SFf_K10KTpIn*l^4q49+4%i~c z-q3-{x6?)cww{#p??F@S!8cUAVebD1ga3JdUT5kSwtvrY9FbXs>NWHJ4w3lr1d%kRZb7>Zpm}Bq;mhG=6 z3ye)c{&bP_KAMjQSczu*e(&zG?D-8=YTgMc|DBU{g3hm_Os zEs)M|YIhO}j1xr6YdUXx^A)YaxsTNifNB?fa{55GN6Q%D0w&XUHB=`!!@!1*3^|P{ zS3be5sR^ncaAcA>)s`POPddpxq?=saat-9P>A^p5=|k~I2bJfx^$SGjr%|xvC+)qr zr+X*NjYd+cRv^xgx_7bw@z$k$k5 zAa3pbLal?wwEs3qxu6LgRRihT;`X3uoTPh=5#Hic{SZDYyM~yVJB9bJH-)RQrhG|6 zOFCnh)=7?Eb{@KKY$t=p^$^|k_MwVZK&{U(7uLa^xczXycY!1fe>3~5-Ur9goEuOdkZMO(%i7PVcBJN8Q<;0Q2!1W6Gwpvh zg81j_U}Mx`pxmMN)Q701OPRukF=Z}l{Q#RZ#C3EISj#<|`Jw8EKyT;*Xj1SO6Aq{2 z$VDG50q=#?n~_C)|m~Eu$Xq6I)N$|$1m%Ry8^!86Vu~JIsy;cR-k@c zx(C_bnrEF^it^`iv8OtWu%wb1K&*|DJUMpHqTP^2DwGn zipRk|cQ;bLv3vWDLV19%x)%TR zcsVK!sFno9O>=n5TX_0rFZkK_MqRpRvFv?I_Vz|lj7w~x@p28u*Nvp;_UMl&u45<1hnvZi+XW&r#Z;P%ZCQmK6nol#sV&l>Mo1y(0 zQ>=(yPxF|Cq%nBKtWMxIuz_6R;Uyg((%GUuu550bm!eshWmULx<9Y?kr&&O;6r`u9 zPHSKXzb=yWn$oW_yjQYlVBQ&mLU)Lv0gjUN2oNqnM#)z3@9BEkymc=$o_S9^F?^xo zjp-k(r25(EKF9_QpSWhdwd2dD4~LuatzcZ-bwM1;-ikyt?a~kxzRZ8=1D0hYVDb|~ z^gd9jcu;hh9Ss}mokgV={@rI#65E!}G;@}(`s~CGlTYB%LvwNV{2OrF%MBVd9EUZn zGe{@ifO88zllHDDdra5~O>ehQwMO;69BA(1QRX_)^mY;uFCgKBh#m79{d%|1)H`p; z>z10(vvnWXx@~mNVznQdTJ?4d8XsYQ2CBRX_D-O+e@Ps)9j4Cu1r&FldQu=^Gbc?V zut_Yg?7T@sIVAOle$%een+dx@nsd_ISTQS`&0fA6o@IW8W%--Mw(dHTcnjALZO3*L zbm6vUn{dY!+S|4~hF$Z?(;WIS67L0n(kVrb+8z2S^LA0afScb~cmrJB8(W`X*&iJANPQyXrCSKUjx6z;aNvrD{UO zd}5Dkbf(;7oIQ9E)EN{5l+$p!##cHE`Zt>}Ob>nqP6HYn*tWBfD_eeo9=&>MNn=p% zy#&%ktjprdT8ayKfR(&~`BCQLu?i{Xd9R#I#C6xhg78bbro&dcHDn^BUZ^X{i&WJi zc@t=B&_~Y7I0D;zk2AN<#}yU{;&<4xdMjSOPUnmMX@ET@G>0E_-wW{zJW9>OEwR2V zZe&eCTrb}x7Qnl3!IW;x_by{aDP@8@5Z*B0V zP>s&f8h1u0-I+G(l2E#I%0WjY4T3Ze>`3=fJ9KN!N&oQ=w*83DF5^porA@aZ`5f4$ z=|DL1EAL@JEL`9Ru;}tyM)(e-bBRmWqo3shM*ayN7Br;%ILHV;@LShnpqk>3GWGeH z;-`ZA6!UYt&YC+VVC(t%oIDh9kPc5ce;7AZ1VMP`72?08I!NA2;KxZgHTo9l=1szH z-S%lO%W@2TbWclbis`x;jA96c5rQ-x{MqqE;g`0!UW>0ad3I>Js@4i$V)MomJWQ%cyr+zqV3))Oe7408|6ko8UcdayU9VNBPr%JMW>gamCjLlEQr6^l&iXYlCC%G zcpNP#2XP>MUotq=1S*1QpItM$|9K9b6-Ju)MNlbIT=Fh%4zx)Bg)_dI^FVsOmR(v$ zdZRNC#!2$zNO>flICkUv!ykgri6-!F&!(!}P}q(}9@IC8Cw)84!NapIrf#6dmjy8ke5`4sK7q3eJ&I+%3t%XS6K(~!1P zJq=!Vei7Ea{6#$4aF~%+hE`|pf#QmX z5=YC`8}$`-s(uFaIlT!>YdLVuwOL5~uY4CQ+l^KJd%=#0f^bJr{Q>n<1ob{Z;U48V zBVASH?P&}^Iv!ud)fS!8?qg3^Lpsmc8QK*QPIe4r@|}(#{mS%KCIM~DET~_^ygV-f z;SN%Lu-2jZxXEolnmXo-lQG|vJ`kk+=vj~oI_n`pkY+>T7np8#ALug_Pmm66FUU7B z;&hyFE{`bhOLDF$7a-T*bW2Ahmz)%V)pWuvasU3EA>*4`ZDt44_C!$vfC$1 zpuP<6U10-vbAnKL3gT8qnopy&HT4;gVglhKTFcCFNup-c03dxx8gnd8{t=Hm6YP|J z)+ipMH502ht^>jt`kez)o{BzS^%p_KTKQY$e^f7lG(OC0){$`E5poQI;EvlGB<_Q& ztG2<+hm9cj<$gxq7YJ98unqEJB9X8}^>J9MGXXI0^91sS&#C7WPTV#gM>lGUM?ssE>?`Tslo>D*R2J4ThU|0Rn_AmjVF|UV9PhJrhZ*jY>?KF zEjSxP{jSqk>)ApV;#^wqYp8lkst;BXyo#xO|JHQ~QlEnpzCx0}Aw*X*1l9M6zx7h6 zJSLnFS64N~Lsn~<>Vv$C`=mTNP>)b?pVH`gq~(7I< zN6o-^^wStgbBZR9hamOUU{wDx>isM!^=Q#3j} zTM+liFUea4`76pnH}KhY3KuQji~7!8Ie9Utc6}A~#SX%c41ZYZ(48yJaO-yp8>ec8 z>LGoJY=HC{F>P=?!pBND?%xUOzI}p|+WPRGA&r8rl=H@`C` z56cF7bL&U7r1y;j%Q5dX~$T|akS^FOvcHKB4mf3WUxiWzR+lKPa5MS zoE4IcCu`M~wySHZ*YZpG>+#0mr()jao%ph!E(8wf<1llpBaa()7=nLzViUbq z>^%C)fEU)B)>@V?zlSYP#L52QyP&K=6q@%pkowhAS$vO=EM@djocJP2t}fQ%)qY37 z<6t-N>T?U7_DL;Q`f4}pl-HE_^ zughrMv7)`_3t`yb3UF5ksoCKtMcG}fX80CpJ!Go^bg#;ie%gTv^>DxcMeKCx5R7~L zAFNoD$_zjKMqyzion!xif5=rd39rP|P&>YM=YDZx(;<0zPd21Z9H8Cr`T-}uOGCfL zIy}C|Y;NMwMTW)QVd^!db6UxST7CJ_Q`ziWbVq4So8J6e_YkJtsU!FM_myAT&l5ie z@1}FGdxFilSUJ<`sWw?S%kpL?nT_pv(WY^Mc2BAq1|3a+UR@#}{Ko+d-_npbPk08M z9QyH+eir=2wV5oOp7nj-EFE?n3d3f8Hhgf4PVj#0RDAj0Qua)@CC^)EEBl^o2Y*j* z#33msVb0lFxI5oluA^rK-~BoUOL!|ZqWip4L(}ldX%3aXb!1uYOY}LI0X8pwfmv@K zesjb!dHmQ_C{Fu%yheK+Oe)@hYHfH8`o67PU-k(Z$0JVdkb7!%qP+}G;0yc4Qv36; zw&>n05M{Jy`ipwJdcFEm#kw)uAx9l`#MnV*{KnUdEcWMT42v6#qkhcL`a@lL@AW8| z*27~202&wVn`6NX8o0_+`yb=kMQ>qygC)?-;V9ltDhJKb2HFN)=$x?!i#4vd zYx1(M+48e)T~Kv6pw0o9of5}ci-%wtuP1U%Cu5uWBl#8kQ(Bcr0c|HBz21z0&t{a`PGmtMbPQ;EL`mDjc16rypnOLzOs0Q%APlkMM&=`5~ z^eZuRd;+9e)rFN;CUC+dpzA={{F`y^*Nm-qXH!sX+A!1K#=Z*P9=~ z+j~9aqT)hmpBRZ5BRAFW2c6YjM4r6r#AZ>C}i5mv{+C}uU>>rq42 zs`TbX_dbY2EpKAQ!;!GNXOHE=zo#`MVHJ8i?3CA{=W0)_Z6v!7bVr?ap#UjwU_;h7Al$-a zkptj)LmnJfi>sR5YA}|MITa{R+7E_s<8WSi*cx)K)D!JSox*-+d$Yk4=E;~Z!!WvS zlqSb6_hqWpdN_bfwSjkbL5szQ zK)ovs!1>|F z@a@-T^2+%=oUX%(v(#GSy6zdGw&gHMc`nmC4dFZd1Cid(8^+T;ei^lhN0$3ksP$Te_rS5s&(uPxl2AYU~3oVGJM0RE<#{a(lxz3PbR&t_#$$oy&F@?F7nUx<`FE zqp|Rrr~2ZIoyB>IlQ%@QLeGu^7~u<;o;j`f3l8l%hb{pXNO;V{=>9Ehj~4vM`5U5} zLuYu|&XJov?~L_kegM)jV6q|vNE5Iw18<9vD2}T>H)52tkhwKeVKo?LGR5D#WPcZI zS^gNG)?R^x`|`1SMR;4Hd$1aTCgc*U+d5ohPdvw0!!Jo!UQEp5LmY zYEdW+wD5CloTxt%+hp&%O1}3>x-XQ@vg^v|X~mm9z*H&XJ_2V!?0zcD#1=geGwQYC31H zkCr%vhqSfEoeOW^=F$DctQNKT`ixR+TvA;=Iu@zofR}Z>HD5nm)+&9MWZ{eTXT%Y< z|3sIBJofQ)EBK-53v*r$!injpH41O$dpqE^zK-&?dv9r4-4KiDE7-<=Zm>fw2g2+L z7vR|>M4kS*us5S4pX{BFy}HzqgvWw3B={PaA=MSO8sGacA#t_~R-e&Y zHp@Q26sM?lB@CBKt*)V}&mLW$!_+|~NNa?xJ6I_E$E^J0Rrx@92WS0vi11s@H^L$hL(mFN_##UPfL>4dmZl|eV*Q4e)oOOxvuZ}-S;_nsQz~W)s^;` ze-bXgc#UxVR1Rrxpm~$IFU;s%@=(Z_(^6py?Ws~zZoIW7dqPn&p`X1Ewz!u7;h^|1 z_7<&U4Nf)3`|-cz4UUSx^q%(fJq7G~*>>y^=M8h)Hs^7AHQ}L;izIAiN~7PPcf59H z(05zgocSERBRDrd3LiO-RW%N}_Z|wuAoT1dXw7Cr%Un}Au;3{WZz)`WlQVpd&9q_|6So(LW2(nWz6Uui5AKF(iz2Om&|#{1OJ6usi_V~>aV413%YN( zxo(^IHOa49pBmmgjI=gLo&(%+4#KO;MZ`5T!JFO@Adf~G$puJ%E3Lpuzsr;Ly3#$o z5TsS;nYk`V_$n0cs~UaR=mSLC{OWdjnJunp8!OU}UDZ5WJ_^^*xuK=G(GW+-;mtDz z)q|Yis$~Ja))W5z5FuJec>6I@kcSb}7h>rIE4Wj#83{Kz=~1^%y@-d8Efi{Ojat{> zA3`(WZdEO%lgM`@3x_2YIQEDcC+`Y2H!YPXkZL|0zSqa&FYIJZ-$GV2(M<&C^?`wH zq8Q}_q}zlD?G}vWrTN_^_t^bQh7dApn}#$BBi`avx2*8a2)AWn^|arQr>NSYb7v)+ z;hG8!>^ixv@Ej#cH(=?f@677z7*0G5QQd}#nM>+$#rgH8jlkKdpBVK&a~Rn!XKGv= zIOflT6>SZG_y}IVz06cLKw1g^i%$@w&pG)KY-gt{mtPzxmpxB|0V|_1s@pyMUJ*cl z*JnR|EkoslsZZhkw?w2m)hdrbng_-Y-3-?P*MNzO9wu32A$eN4&`?)|Ii%287I}g^ ziyQe?AiNSMGa3V7HER@QElWQwV!3Ia;@w%x8*}^0X-HTJ zu4acZW6oYd>xxb99jo?il(9&{PifmQ11V=<&oEW|>@f+H2PaNO%DPZ#k{}0qhbuK4 zn|*JIR2!Oww_KH%VMVR}(^A%jrw3c(Any-&d09Hp`T==c3@#2-Jfu~*OzSsGSf^$? z;Aea^7^JVjn7{cN@_Hho3u*&W+6$*?=A86~%3TEUI?`MV!emt1hjI%+bt{hPS3#@2 z8rWIyIntPb^bU}QhIt1L0bgmuhU|DNsE=TK>XB|QCZ=Gm(gj+Dkym!d1JyVqzbKzr zw&i8lH=)8g@(VJr<43LXEaZ~}WqvHagP)-M3{-~pc_O`QD(|`d?VAKs@^>kXrnnxQ z2LX`ZpxnS)UUcgNVFfWrUJlGt#?%co?w{<^&M>8$Flf3pG`;62 zsW%023{(D2>41;8jyS?dJl?Vst`F#7Nngg~ru8W}yyDCoyAF-ia z^0yR8Hwx12Y|XG!K4IAVgf-(M2tMeGrg<8T-rRxZ*76M&Y zc|w&HfbyIp11$N^czukbvjvEo1ZAa+YDkb4XDZ7u^3~$QkJfIg&Z)QLuHa_)!O{qH zKf6eUjmi^|wr2T1Lz$ZMM|aL*X1n#+Z^>0yy!2)@_OA~21&ioK+9V5Cg_(@9F!J}2 zIQQSXT9rdkUIHp3B5sD1v~Q4c`bBk4Lp}`%1I5=vZ^SL9t$6dJW3^@_4JLU&Zp-kp1Do>nU-iY)|U0K?( zH}&^8p!|Te`*^VEd0KgKq|A;tx7x}A*Onpqcceam?iXtC(34>~-Ma3^))yRrcoCHk zQ8{i}txQI~LffeIO0oUYZ;jH~ot|AI|F;(jqu@bEFuuKG4`FL1ke*Yz0y4{b1Lcic z%HsHhr9bKC%qR=F0Cr}*S&Y9wQU)dHdiY9zFJ(`ESmyP&K%A>l`3m_k$|INJO`ifh zGCB$=>)?c~ytw5?_|;@HQ=F>u%9%+cIAN>UeaeQ@AB8U>^ZIlkPbn$CslMiT^ZrOR zEh!^M%9#cE;cAU5mgNHFccA>(@5p{Y9+7ggpR9A6c)lvxRAThVIpPVtS8830$2>s^Y zf+^BTEFAw?@ySJ;c)nxim++<*%H+GHYcT9L}OWi@QBQ-QaStNd|d#eNW!pa;bdK zQIG%ZaGlM3{7#&CI)D$Svzqje8?x9w-^H6@8`U`A%+sqm+SZ1Yxx8nO$9sqe9V4as zd<-AQlM626@+NO#OBfdw_FKiybuA#S1?}bSehgmkix(&6^^^m`P9gm)EQs$XJ&)#L zj@SZ|dl%DOZN6v4#HHaUHcX@A2ROy zULObdZvrRRbdd+n&4cW+-MAeOX=vT#lx|mH&e|~eH`Gwh4Ic)3ozLT~(b-^@u~Dq( zu?kIp#;ZD_{gYm>&rOeOYDeg4(=(F6@>LB!!?G1GAJv?136n@;=XtuGfN7;-Ybq1M zYBEOkIwJ;K&*Sz_HPYzC0SFs;AKO{o5$2y7U|gSVGAUOVmwJ6;s(=38>dGC5FUB48 zoJ(n|k#zPTm&+#y@&)l1;p9|jem>G!z7CxY>+^LWHILy>gPZuSz=tn7;mXzTta9zb zKP2YM;9G6MXJ4=^YSCOy%6beNw8O;bo<}uvrt8AnkZ{bm{fh@oZDh-`Pg7B?W~tH|Fjm9cGZ$sM@|IGbK&AKil^&g& zFpQrb*N!hgNqZ>wz9~AKILdy_^W;Cw$6`a{Q&^He9aSA1tg{}=+C_u+y2TLwatJ@v ze4_j?;tI_}3$`ZqtJr$UjVnA@eAQNDby_Ch6%?Z8msYg6m88w6^NFqbV}&ipSyIn! zfmyBp#e$DfjQW~UFGKr={Y1a*H)*f;!L0k%N?2WYKTPu0m3y6gbN9itUuRH=wDapF zU%%f*zdM!FIK-g-f3SGPOy0Uv93S%j4M?k+qORT$>ATN>aI!8CE^wMZMt{e0`-Sq! z+cQ|%TOW@bT*9)<2@u!Ef_t>;&;7fa@JHVU@(+tWWSuS#q3O1xIH%$vu8rL!vc&@K zeYXN>U8`%SWbAtE`DY_^nfVTO)NIf1Zf(a)EKKoKxdU(GYalDUO{GbR4W}B1Rjwz1 z?h6lZSta9IG~(CniZEf{Cg|+cSekWxr=4UU1uJ^g=7edqcfkub@mwq?JcC6u1UlI^ zl(T#{NxClk*gjXQ>XXI>FRfd$u(Nj{Yr#L%Ph)V}-{bO(yax0Ky`Qw?mfmCKoV7Z% z-&GV3E>8u^-I1D#-){+X4&ml+ePq_fWK_7!KOfdOCM<+YCl}$%eLqFS=F3!b6R~8h zA-{dT3s28zftNS#5aXJ*0PWi8q;+pX2g*@4O}hrCjyP&xhkjy}y`ymm-vBqKfzMrY z_AcEQgcY47UHFlmYpjL%@4K{n|Le!!PNF^JPEkJ9WI8l^XfKFkP+>QH9#{3{*KP## z`ud8g^};)~U|7+0Xjsq<2(#st=ov_#qp%c-_jnD~nP;tRL;qhYW-a(Z_+Lx@nF@0H znhf4zTqs&KAHuC~dZ-!~Ti<5EE`JNiw_U+(E@)wojW5>R`Iq=Rl%I3&hI2fkv==^f zM9hsyh{1vAF;h=U|aR?I5sv6?HT>@a%j)vH~P+!fxP@MM9 z#E!h*{2|&I7Z;sSo9Iaou#k2Qzx1>$3E7u#9HY1VMyQxkrDwr|enZx`^#p_NE{gZ59gny{o{%+Bu| z8z0aF;;+nM=O@zMRZVnwhWC4M;i@D5I1n&)ZZdNmUIkv4I*ZOuyP()@uG`j2Ik=*@ zzD!BvXgH!X{^&MZP(Ptd^jL5kbOEeZ7ib92VQ$(`s=LXO=G4u{dJ_;=%YO&F0^$W0 zxV#^0JSGpXxrA$I&DqZHzeKN~I0#vp$V`7)%g;SKk>>c1wbjvK%Zfih%(9NMW8)0` zd+RiO{nUmNmulIRQ%Lj5KL0#{q_x-^!>#ZpcZPJHJYL;5Ht*XOTP-Nl5{|GY9bDjR z*Oqd1_HsyBd`3KGS2RNnqu3VKQznJ3BCRo6CU&WZw5D3B4Ne?`x(oNfp6))xvmDEJ z+sHn9AF@AF5ecK&`XOU*N6iD8%9~FR={?DRpRLBayN2`G(_doI*KZnW6)rE+Sv#tx zNMmV;Z`jNov%r1uQbzwjis@^hSD%Kms83VSkC*}lOC~Urrd4P#;JS9n89IYC(}R95 zQ<7HDP`%Okk1)DtWWBPG<`}7N<;)!)#itTyuG=P*)$g7szM8hi=C@Yh#5${3yNbI= znghRN901}84Pm*+?R*#49ySuh7clWvj0~hR;ZpO2;sPLzsMeBab^5KP`jh_luc5SG z5nQR-#0dMPMOQjYq;50jyN~w0o79hZGN)Rb#hkYz4r+=s>(g`2lV5iu9j$$I`v(5A zKSLX~q!@18zeX9<6f{3KnKhW?4qspW0Mb}Wi;JD#mkOF^p&LA(^zt5Yc=Z@~qc@E` zxzv;YyQ_gDoz7qJJ&gKCyZ@)1XtmOu_}CEaI`_ffEqf4F9%JUF^|8dwm?;iO*cHeE zZr*h5A9s%Ranq4)cIflrKME8tV9=X9C@_hHt{;m~@uix>Tdo>m3%5{>#b&v?^r$gF zagTP~&z6j89WT$cRvd}z_P&&tPJY+^7*hgoe~gi&okZDyNH8w>4MX)t5ymeO#JO6J zM+NNcJx=xaSk(a3xv~;hnVrNDQ5WE0+-;#b?GDsi+D=Bx1?eB8URK-!s(uQttzz{y zcoI$+alPZ8K_?p}s%bS9zG$si~SvcXHWnqy0i6(=rK{e-;^ zmjU&Q?DCxQ+yrw;I842jEs{cOqtX)87aIM$4U}es7yf_M=Lphc;?t?FV(PXm=)K}G z`~CMUCP!VvQHxzUjSI-@5a(P2)1H=~yRbR@v)_)tSl1p2bCEQ;s$t&H*afG~%Ykoo zwllq&JE3LDd<|)P@ZI$o>GMS8@6`~z@(*@?J{Oec3_fB@>*6KWkF5>oDurgD?pgL` zn6bFM=OpPlQ@L|pvY_?j3Tw^Jy~ufer4gh0ATHaA#IZ1assq0}_#pMoD|UAMSxgT| zgM?wzL6AqM`9<=Gq^Wd}G%6e7-;k{--36&H+&FnD!Cmcu?va; zj5l3?)`vWCwoYB78pY9hnMk+`E=|JVZ@DvO`^F>96y`p^B{Y*zLv)v{_ zRyN^3R)?eF9)W8!{5Vt|6$W%?` zmq$Q-4!6ue8iaU3^OosinXQFvKDBkV9_>B$DTJgn<`uuE!lA7hthjx^Y7DMhI-s`&S-cXmM>_YfOI{Qe~{##VPn4=Y=&iu zQ23y*ept~U7!nbuNy?3Y153X%v9>O+Z_|su|2_{!`nKec97%s#aq=%pkD*zHF&`Y< zSdwOvRm+DX@ei5Jdt#*GnyrnkRKHlWE(&Uk7 z)hR^jBmDiax9t4Sc35%XIhqA*70OeR=AnCCiOQEPeNJb<1?iUtIX%F6m zIq+wVhUF~SA#xWLvg1DIF>O{C#b0RLejiQ_yeG(0qw@6Xo(?^DDY`u0h(?JWH2$y3 z#F$xj==(mJubdnOL!Uj+(tT>b(|#$dKD($q0Pb$xuX-E_518uVXpFy7DksuBtJ@FLO ze5e{N$$u+)w3vfCgHzbFoKtMYfl184^Aud)>?6r%E1tl-wjP|;kC6ru#POVZLUVgr zSIQ5%Dg7nLPvCsN0Z_iQDW}g>UYNZ%re}Xmx4=&RkJ3ScdIfeo$w0zSNjxp@4|$;> zKZzzy7h~_(SRg&8=~-&3@Q%^kYt`D5w`G)s&inaw<>iwW(s^S{ zwQf`TRB7h1C8^NotDeGpM(e~jH#)C1ZED5o+Jbxu5;h_AxjZ_ikbLWUAbrFBi9Ccs z24}>8@mY-K3x;qb^b5Ho4nNRkgulFOfF6*RQaYWHwqm4jl|Mq#EQ0z%tBW>#>6u-E zGz6b5>I>55K>UsO4*wtw3DeZs7Y+vPNZ)n83~LM50j)8&mq;GDJ9xL*okN-mY%XYk zyf=Ps+8O3A>d(o?Nb>jMVnQ3(dAu1Xu9386In=A@<39_Wd-x+|M2zwjK{<-bVUTbi z$7fjOxbI0t(f}~hSkiulg^+%|8UEUKNeo(Oi?#jr1>qhNw!_&Oz4^mwccH@GoVU9( zT-7P_URR~*c&`J?uHTgUIud%c_>8Z3b9v;@eTufx-z=98ZQF`1w|cat~WAXPtG>-Dx1sva31Kp7#`OHIerYHvHoY78EJ5P_~s zZlcoL!)<7P*DvG1Z@@4xUm66nAZL@ zc^X%mV}Ec>y3UBx1?6No^1oaS#5ke$&TA}K@&uF?_1|3|l*grBgr3>v{N?vnNInUu zR|vC5Q5I4I)i@pLyIIm5jOJ29eWLhNdBke|GdGO`$_|zG!?c0UsPaqO@Im;hOF0{E zGoU)xB=1Uivc$O1dDiQL?h+1IpsKSmvohS2W;X~a0LogbH3nT5U%$5i z@>;Z?*+EvLQKr(uKzXcCwLPa|EY#|6uDpe)=64!PnI7%wKpLLrOb|yh$~4#vO$%Ap zPG8MCWi)h_NAwJck60kCy&T1?R(n-@G#Vd$&v2P~Y&VkUWmOk^@!69uZ0+FPL|khHdyV@-+N=|J>edm;K+UDfTobNr$Vr~$E!{Tfsl1~)(_f7 zUyc356V*NKcy$jWzk&s4$6$vOognZ6?K`peB~l(Jh7Jn@lYk1W9T6#1MxEBat2W&3 zhT>kuuUgU!oG=o*oh=YlGvsrcK>k1N@Zr%yTwB;sQa%C7chWj4F9s9mOyZQKVsp!1 zDziiCb?LrrAMdw)Fn_T938Va^+BD79>CUR z>w$Vl=|H4962zHmZUpIj^0@5<`A}$n>o0UQ83vS7=P2#$zqBDIP6D6rrpjkNAqm0;{B|y0OF9$Z8xPVO@O>uHt{8FI@%veTZ_d;ZTJPF4x}me!oHG2F!OOR^;;}u;^9b| zuR1?zur?J2lmq?||M0eRH$&jxGIp#-1`}R$G+P~3@^e@8*}mI$eCQ;EZ9Q&bEC06g zMb&2*lUxIqx5|LIKD+3Qi#KTT)Q$EScIOxGj)vn$y5hYw+EZ}j0d$-60{#}!`vvpY zIri$$iJ@Y%Qw zVG;cYmU$SnSFvH}?^1yDcdpm0ja;4gl1=n+5jN}pgQySvdAVDAInaF){(ABhYoCw8 z>A{=$;R-#O7;sQKBYqHkx@#p1Y;17&`~$Mrq~UVjiXvR(o~(&JWdt@b5Kfgg;G>M^ zU}(r#R`%~A(h<7++z}go%~fBPyB!c}&h7QblP;XXkA0{Oe6B8+1sYq@Gbd&wU+UDS92DwdBL^V2lK)5 zoX#2H4vpmXz_aioZVF~p{wru)c!V;PH5J(!sv*3PwwvYW)#ay3*K#NQA^iBPS74q$ z7%zrp!;t`EFtFXq=>BQXni#C>e;qvU`|^hA>tucUPQO`lsitn`EQlU641@0{$x@vi zNWB83Rm(M(#tq^9^TRNqA_;wdAHsW;?c~Ld7Thf372FSP!NaUO$Z_x5fYXi(u+!BK z_kS>x8aq0xQvVi&T5Ir^n+?@`3hlxEo6$N;wN}d)UW2C5bMTsLG=5qMlf4aL92fpi$jV#uI)4M=)&g>}ccHE*`pN`2d zTg%hq-U)*xuCjLINqqOD3TTa4yXXMyI%2(~-^b;x>|})JzgV?ZV34&R3~y0eM)>IP z6R|h3a9kg`yl@qCaykVix=X3f^m)ajx-iQ4Gp@Qak+qMk!H?;Tl6TB2m~-Y4pwG-P zX^_MN-35fS>I1YMY(P0Z>Kf9RdZP#*Y_2B1=odvR_YB_#l0g*P@r*Mor1FQJj9C1s%GC*8Z7jT6UU!OzMkU zD(taO-#XaCX&3(^e&F)L+Vq{$zv6k*`&hPh82mieMlSQ%2y_iO>yQ?2HJt~9Q*3U< z1pdsRj-WMX4bvOq;)>h2DQ^TA@2F%aUd93jmTFhEY9cqdJ|hirSJ0et>R&nUzA<51 zPkEzbUD+-AD#EBhcExQv&kDWAGG1CjM1U`EAGwmNHH&et&GjwPanHg=oaR+)3gj1Qpi4#`y zwe+0h>(BE=>#_tHykau!pz~3aW6PlNk|v=3OZ^FI-RW~7+4Yhn4&bW(rWAIB12%=2 z>GcPwA9+&fFR(Qn19b1Q=~@G{ZeWb69|$L5ukUgEujn^(q6 z{vKH{(ET~CZ?yr6>i=Nl58Y*Vj)cnr$<`uig*ncT&%=^q_xXo&hd}*r#V=ll?bwUg z&86O4Iz#u$MB;;0IHt~gez{;A=rv1&9ShS$1E*>99r_7SyjM~5i9K>0T8-yp$xF;B zT>ymrK)1@tT(jj@X5wDT!D)@~rOq0bberD&&jIap59Tg@=$ZGv*ZFh+hYS;d;1nla=1ZQYQcYA zSO{sJCpFzXerQRzVBbrx*y#imMv?8{QS3TF{0i@@=F`feQ%Dv zruTi03D(zKct#(C9?={M&OOZuk8x92z=-t>IMJ1t?6)@er{F**JLn^JfLhwCST!b~)EllyLGfLE(fIQTaVmc?g6pHcUT zZ)-~8C&G-%;_1!KtY%w#x9`^@a# z{93E9xT=?xns=;O=7aC|_hQ|HheD~J2E9;EQcvM<>zQ1w_r1#fK-?sserzIVhm>K` z?JRLE)|Tqv4o#ItTFbZ~$Bz&$2x)bJ5l=vn^65r#c@m1-O$N2zzxStC?YMy{0RDr@-UNc%=1`Mv?oW zW?3P8|2$f#d32%ozixC~4fT)HJK4JT1l6J_n3W69L$^S`^&Rla<~G=B$p)5~b)Avs zP<5zy77oms3#4)I{y0~>*=sb=b@}vMXCPh?g!w4L`ihSEwz$!^0GHm6RP_wAFSX`M zGZp0&u_x_@i~9KwwZ!`(U{Xue%bx~Y5@rkHXeK+hLh3udz0OW}+A9w}*1wHbyCdPW z!4(+1zB!|N=Q%pp&`d9x9SxlXq+KBJ;5npmlLU=hkj4S(8M$Gj z4H7nhKF**z>MH+nX$^y)pVPXp*D~8lzp-X-I&1KwE*RXnuRZ4X6HK?6$z>mg0bv}3 zOfu&*Ch@5qB4HqI3Frp(GUoEP$IcK(C8AfWN^!yKC3AZBTBx-l9{}fcx+CE}o;#AD zsoh!CR0p0>IaTU=KZV^#3h}*Pb95Rt4{f~tis!`H8|~eElk%5gkMkmQT52!Nk@V711KE(wD>V2z9X$W5NEzy1BAt5 zV(@O5Rxtxf(<<)bx826^_ICN`up|O@#x6v{KH~6!s6M}b{#d0|U}IW{(jaiv&kwxk zccs0Pa|QVg+*7B%lzA4aw(#uc6VN830LJZzVC|gFk!Ez_YHm-4=wM37E;RP1GbT3n zR^u0)r>!RZ^#jsn3R9E@gwA+SkWS$=C&Fu4Ce8P5ApZf~Gg`}z8SZS~iaNZH`$so3 zy%t*1=0Ni%LN8hJvu=w-W?ByW(XJDe^~wbDTs*e)hSKC9T$^*ME7<|M$l|+sNWO>8 zXxt~h=ney#f424Z1#}I5s%iqupL{^_7}Z=xx}3LOKT$(-!^poO-5Wa0pN@|M_rujA zhH_hceY_uRoXc&AYauS5t>JbA1{s&%|FvOoc`k4Y6(hDVFBxOg=5yjq(nb z6%YxuCgS#MdN8b#LYD$_|HWp{4_CScy8rHn>OO`}0`f=)fwU+2tn2K1Qa9$;>K!z0F-d7C z%2cKc${2vOA@xOL+!S(3*nO!FPNTj8)d&#Qu!pl+^U3A({A1n#rT1CGRf9+v_O1&a%yT4-nJ$P&#FE^2By2{dk(IYr`NoLnE@~X~Ozj$>uqq-YMUa0Z z|Ik_5Y^%*Me@>uGM8#BBOigIqb=Y`m@%iENFFr3Lv!yeCd^MjrMGOo9k>RY^1R!+s^8p{ zuAucpT2IQrE&=5`Vq@9|4fz)Nxc&?n+v>4UYujPHAN6>M;!&k-AU4K9^&V5&{GPLa9J@a*t2*l=pAQEXu0i+b58*`YV9~yKuiFjNX871k!r=r@sd|9)3TfKM z^p0!3OI%qYh~mAjEXmABf&X{aW&-quU-aiz7YSL(|kcG@$Zay1i9H4bMC zP^0p_+d_5+^egTiTn+c#gVGwYMEU zzkSXf8$*%W@hVJNR-g2p89uq2DM;(1cZ)58v@1K3m8rBAs;p|phmS}e8EG8k-Cnbx zQEB2ImsF(Olxb~F3;GPLhtF7m%od!qj9h)lv|5w;KTj8w0n1EZOF?}}xm6$HxLnNu zj|nih;t90e@mQ?^t{Zb1$+vUcpR4iQtO6Y4(O2nxikkHOIn zV>s!DYW#{wX~k4;5pGtm&s^gHI4OFy(kB{&C0XLQO>YhLv2@LE3=`(tRAZEn&rNKi zw*~o)HF(+JsQCObf^v#w9 zv%;UL#bQWwE7vuirOK<4hNZ0C?XAk}fwF0w8siVj3msTk4-b{bGsT^?BNCxOK(yxe zvw6H{{udbZWGvim{D)26_YmV;rttfnj%!l8euQP%l7FTgxV!x-xqW{S)Gl@5ds7GU zy}trPhi<=d+mtP6U(p`Erj@{v+|z7H_zpPaumxvy4dI7p24S}jw2$BnKbiBi3}U}- zl)s{~!8CCdKK@GU{dFxm=jciEPIjC{rZcCrgRtM~BIaGb15$J9@MdwJwEo|n<+?9* zAhiEFEOOLp9u*n#on}a9#>~U%sRQ_enN#E@iyM&lxLDk&IEodigJI_E@2uVa7f{>D z8~z5_;ayV~d2!Zf@TX_b)b$-qkAm5g2Y5bvrI@NUlzjtYH2>t!7j!LnW4~672~5d@ zewZ-=je`5(SL+I)>s$<}c&)N0;}!*QF69E;sp!P(RBG|C=T4;g0PXMp+#gRCol~@;MKPtd1?@KfXW>U>@xz-G`O;2ijy$LLfq%_BM!u&4Ps{9Q%o+pU={uOy$q zFk!{r1~dgHt3bIETk)`%o6vm7IO#goK{WZI$Ne4+;It;PUHDd9cP3YMrt^Kjtc(S{ z?ZvYR;E}H1MH};;khF4@*q_&*PZ)e1CWSTRw^xtBM8|A66%fY?zh|?P zyPKg7y^q|oVVK7I*F;Y1fR5S*aBp%X`d@fkU0-xfx=(W%QdEzh{O*dO=^vqR@DT9p zV2$RTGGO+-nsV^8uJlUtGZ2Yud5`oExx8}^p!(Aom&D`#wSO@9>|nmcZY5tHwjRHi zH|5W&?n;r^fUPmIl;@4CWUuGn;h>?R{9{?F`7dlUR6f3sJt{$yRI(fGR^Nl#_O~f zeLn*NOA zoZ+uWK5} zNjdJT&o%i``!(viysudf>>cPV3u&L4G9-SJ*8xcR4h# zucT|D#lTK{!+{@QnG*p`d;^*I)`s6rvXo=&>&S*-^?1JT47A_(0H)A6T(|Gr$wLli z{Eqi!>{4zC@4x5Z#DLZGJ$xct>QM{+$uAeJ?DctWc>?{h;d0ew{6pWlUhltA{ugE{ z$NCJXvtQRS*StPr#KW^}Q^P38_k9nG>{sLX2S+7+ChcqU2Q7Y0fMsS(KA!y(Y8tNQ z|5noehK&N)_XOG(+_;&f-@!po9unp_)Q z+EF+uOwtNr*^E1a{tXPm&3NF-C@?cg(_Z`%ivw2b$%)TqsWlZ9sW-86r-Raza!$of z`VDKy%}#eT#!n~1$-#v}UGqyxOL>#t?fxth`Br&IhE3Mt>w?T|+dIoEQyqWfGowDJ?z3a*pD|?UTy8R`8=my2&FJ@7{C!)jY3W;y z0YT?VFg?mevt>^^puUD4fmhW*)tU#By%1`HAZK? z1t#I-Kl7z=i8VI7-%75ipbRXmfz;1+RG21CVQaBtcM{NNNzxPe(qS8$KJY}&mOcB0 zgJ~!6>T@^{H-W0Rw4Y-H@egh;)v#5@d*IKRbGZG%as0hkLb*u?B(BE8KRkKHneNzi z_BYYqEE^&c*I=Y}1FYTarhPK?1m1r3l}#8t2(>F?uto<<89XhE6|JnxzZLI<2PsJ2 zsQKOZ)kG3ssD6TBRW~84;t0cYWB81&AF%zi5S(7V6D{Zb7MC98191~09M+DXVgf5F z955)+0Bz}A?9sWISbBCSpZBnrpuR<)oLZc8k7oE80G+C9)mTE;LcgdJ*!)Ky5mjss z`nkvO#*}UFBl;{$D{3HE`!rB>3KRBEft98wNRQpcQOO;FJ?z1aOpoB%j{8wtvIlzq z{YCd}4Tn9gv@>#|<)1UXaM+h+X76XqmHO`g|b+ihD zA9npL_BVQlAI!QV=|QcpS2~b}WKpXwvNuk{`KDYKF?04JnIm>#CJAt zzojCr7q5}=o5eQrN7ANpa>YTWYWhq8NSTm@FB>kXx zRx}OT%P&VBW-o^Xh}U$kz-dp$QO)Rc&!3-1 zG}DAV4MNg0p!lR)#$Pd{C>E?6J;d{QSHLHlq2ia?_E9iD(hdh3H^ooqUxSzRY_RFD z14z?C+b$OgOaB9K*g#k^4>$a{3_qtj!K$-`NScPX@HE5t)%O+mO1dU#xCWAVS@jSq zJZ(~SozeKjFPAdaAA;6J^*0bFih(BW7;yp`JaCXMLq7ra5ObY+2+3c;2eUIw^%M1u z`b>;2k4MFo?zvs~EF%lKGWe=EoP1p20Q{~R21nPtSH291->3%alRmCNd!`RiIvt)q zy6JWysw?>hU6^#A_VoUK0*Hq|>9W;D_33_XQSZljNT+>aXbpk-90*h4rdDHpb^KUTBcPh{tuy=NmxZO^No3A=ojiHJk+;dMb(p^m9f9;4S ztlr?OjPwe7P@V|?b?(E{hRz4#8}@o;8S_r4!KXN$2AV^B_uwSVb1r4HE}XCfJ|*qI zyqvnMG@%x6?qDcOw3}7E;VrXVMqY*bP#66v3$VQ~6$n?s@1Bk%? z!s(q#OA5kTB<@A!PxcNx3BguVWk||dY#TdX6hyQ}n+}cS%gBz(Hw*G)K-!ge^=hYd zJGfPO)B4qiuix51LgQZ=;yXr|jhigI(Id%3D&O~L=t8AU@kAFdw=hRnK|hD&p9T3r zL7X9eX4m8ijh8~?Ia<5Obme`KG?&u(e2`z2Al)ooU7oW=5y>>a3xG5$Ty`+$eZFTR z-79NN@2-@YUsjqKh_9&Dmc!0~dhBZ8ZuM`PrQcm-9iKaJpz1bka4=r*7w?#)rYmA4P54bQ)<0n+$D-a|GvGLowJVs+c-G}Z>Kd=#x^|H&Me1yH<-2w(qKf?|_ynrDcQ*p`Q8)}^?fB1^D zPV#6(3hAg9sB~A(~FQ6G&&HD$YK0_O|LjM)?N$;b`8zW6J8=Fi0K@!1|pn{v98Z`Tlpprr3D$rrd0zNxGVLw$;|%IgShi=9-Rb-kUk9S5Y{W?tV~ z$TfK$;`x_)oH&DeJBqwnAyb@q<;MV2dBci*CABrMMHvK4K=nr>Y*2N`v?4AE!BQv=esGb0)Ob zTEY%VdK#1mnE%5AsgKa1aiu1%LjtQmV=%OeNCNUTtXbrKruy*xU|($cVg*F|EfsH^ zinPNm?;+_zL0q35X>tI`cgU09v&6sY^T<1#(ohCLn!OiX&2`1`u}=lziW_MMadZDP z2oO628V^*(&+S3{7umBe8R zv#7UYfG~$qU10P;CmBI!RMMKUkwc!Set`{<2SwLLj|9~d)af6~j#cJ?iAJ6(5q)12;t4i}Xj=NkeYSrW^-oKA6&Y`fCD#bPery`C6!4^=iXaFe4gBONXJ# zYib51(>yg%+Kin^&_l(?locqy#;A5cWp+C*ya$?d($Vilqv#x{E88JuRXL<@(RN0m z@`#Llrlbr&5Z9v`M~^+JsPr<`mK)`yNI4iPJR3G(H)+vVNSvpow>TN6VI2R(mpQ?H)7Br!6=&&)B{?jQJsvN@QQ)1oVZeuo<{RxWBAo! z52jeX6;HBtxZaS5)pJf>m})fwE+1Hev{pzOU96k+kx@NkhnRtEpviuCm7Iw)zZ!wk zvXl=|2B*hEiq@cW*I*!j38d}d!o6CG<5BSf`5N~0(O@V}D+bba%EwX8@m`R2#C2b2 z|EeL6sXo?oa4_N?6&b=gS_4kTiEN~Uu}1X(%^eIsgOoSZciA!58R4ZMtXABsq5A^j zbLBOvwL{^Bnc(=OROvd6(nmqx_R@I)XO$lYr3=xkx zT<#}doe4dndTR!J+K6v(FI=;p0ZSh>!iZ;A;H8fcfg9Z*b!06_N`HjSnx}%&)g(w)Qlwdt}3_xMKVdHZ0DR518WyV=g%Gh~Mtq zr1ov>F3KS}CKrp&EN0E#z1H!(%sW~7g*L}chRd+9>pRayE!KP9=&O~ycW->UHWT|6czf)N zuZ6RD5+pwk<2dv;MCa$=?}$i|%g(?Q+wXAwax3}H^*m14nuwdGwFF_j1GGkZ+B%MMXnP+F4dEjlzC#x90mVeF-o5P779JokGs3<&PS-7;U| znu#4`onQVy*T!#O|4%yKIgUHN?34HMo8Y6ten8g&j_W>St#6xf)zLy&)Ay=4QgIr` zxY0T8bRPq)ueNAJpsaD<9G>l|E9JT~@FpW!Y%wckSz9A8Ec%$pjtqc@w5Pp=p9RK+ z*5omzR{X_~l{{^1SIT)$;X|)y+AfCkV0ZjfsQ0H#97sG(^%jJ7H^%c>_U`z>zX$J| z{uNHws>9zn$DxhQ5IOX*8`=gA;b#}Uqm;QOpEuN9JbbePceSh~JuK)M+EJaj>Y=ms zy7LiB?xShlPdKJmHe1{OH9nj346h!k!;@n=^4T>i@oVBi`ffN%PO>=#U)vgD=W5H!?g8I@n!Q=iZ2bPRvxg)pjrc{g_jYPJmJhx+89Th~tXUZ#?I zNKSj3jrIN8px!`p+395`?mROAbyg2%wSTyxPJX1|y)Qxhs+#QD#X9nEzW@lU)k3;| zuP;0GEeGlgUN_EGuCjV9RvqoajqcNa+Oey7x_LEhIH4y>(|n;`^R8Px`Ejg^r2fXz zZDW|@ZZBDDq=ZR14du>}Q*p>cOPRH`8t-#+4N{-Lb=s?=ppy;1ugQdUr3>Nc)M;oM zGfp#hTtoOr-4>(=Wm~6ClCTr#^I>Y>DWIOhij4*^r;|Ik z(mf6?4z!1S$XrSD3RLH?_En)s3#0vdk~WJx`^9p&c|*F!A0q9-8=QOM6g>A@$?9hX z@t48L@XU;1ADJqtm%(n(XVOYTaL#c@IsH}zRE)e?)hplhW=Ti;i+FR{3N6hiPbz51 zH#RAPZ(Do7jh8X-={b|MHiR`jv4M$LK?=0D^e{)g%XgSXRXh!dS2ggq; z5&dpAud4AHr;9~`NiL|d*z}2F)z{O$9(zCH(ZGmNC)^-J)*S0si&$M&gR31yTS@szilf} zz0aaD{|YcW&B9Qc2Jy4pOb&DnV+g{41aPA&{j`YE@139UoL|E>EPLC z#^RrmKPN4~L;kIW;fqgdZC6AoZ2*Lc$S)RXZ&h|zYmoo5<1HK(wij)3+QATUWG%P-J zUG*p&(XEXxHTEF!1-r7P6@T0MAZ|FOBRyRjaNiEiRh?AvQ~@#a(eKPJ|!NpmEo{;$TlGSW~3=_+Km#C7kB`keVHLT*4qpb^>mohMm%IQgJOG5 zJ&p~sKaqCmNEkjAsh`-?*NM1vL=9C3u(c)m$`Z5mBR^y6< z7LsaLd#(3QamTu@EO5w&=qoxHIv|a3L11M}Q^>!OhkFjv*`&>LwTE78WUbdFVESSw zW>Pzd&Z)k}!rS~{rYFsj>Q)j5!@9{=!QewP!mHP4elP|44z`rjBF&h3Z{l4^I!2@~ z-jCF0vR-QoykEKksfQsuCxV_=?*xnXHR5_+#q55{4InO&%gZX!(8P%FV5H(0(WE3E z4(BamjP|J9`(qicXR25;F$YPHAYrQL4)Yl49SB%&sj++86RKS|l*AW#iZ@9gh=Q~; zo~wFVN!uxQ@*my*zWl#J;mGQ1iAOa#wof4Z9AW*D0+!zE7mlyE4`I>nc&=zU(zx+k zRy|hjdL-e>UZi=IFMSv)eDI$UjHJzko6B6(O#$g(djwV85Z80Rm9!W3^HMCopNWx4 zBM7sPF~yU%fp>8Bh6T{1>1R+pbG~Z~(l~MHqa=t!zXY>o)M$z*T8OSXI0p0v%1Es%LiOpyI2Zv%cb@ zg;T(OXB11x$b}e9kx<<1k-1vbscQk#raD&f=!f${o8;Y5OEm(S7i!^U3w=pFswqjR z1Ef9h*`8RuW9SEI7p5Zh9H+U&4fGjT8)~XF+>MuYu+{7%c>QvQ>H*xgKa2&PK1zD| z5fUE2Zr_chsm3CWQS03KCBClfB2)cN%XxRwJl&sMWOSWc!Ue>ki-}+D6`zYymVMcR z&uOF+JdrqFQcWv;3O6S1go5L>rHg|x?2F&1xwBHM$!HV+rarXifc zK9c4tfa$j;vyc}KobU%@O1FBR+jJdUud9zMT*4L4aC)uho`a>H-Ni)MS+kbR!vK1o zG>$3#;B?Clv!+}FF|(VvTYW4WQNO(;F5`rylGcqa>6$LubT-C-^;=-Lc_J$-s4JH( ztRY4n-V4X_tTD-P0GLzm;pws&=NB!&!Euv7@y)qb8QNFF1^lzw#45gJDo&v_*F&ch zLnLtrcX}ViiHp(b6F~T=7BFR31%~`H<*u~~V8ZM9_~&&G+LP)qC=E{>E2zGCvkD8b z_4WplZR4z2tFu8+AH($yH*jFOC0Aa^y76&QckykdMZ}Ms^Vsl(L|r?7M!F0M&orhj zWtA_W>lfoEQC?}|!!5@hCTmDdCN4Py6 z^t}AB_5xd_(}8pqJ5tdVnsoG{HK~WTE2>v{ppJ_QkaQ-fIemAwsf?4M+Ah}K@b>gd z_?7n@cGW#CY;q1mxBN@spKKx{Z}=Yqx};} zFP#I$VWjbqu2(1z+3$l1{}vH~H?DNYejnE0&ys`i`srUrILZw^?80w)A6dG2gqG@E zBs9~d`RJ@Auc7_n4nX53Pho&xESt%MS&vx27jK|Bgm;@i5dH?hH@97=X_Ey?XOhOi zuG={F>B^eH?p`{oNi(=TEOjsy7}w#Z21Wc!6D3P6Xw- z^a}<<@cBtlZ^YNx#EP+ay%}V9~xL{CMFeX{TLCI0MR;?-)J+Xnf?e z{?Hl>2hw*iL0vn{}<{Tk0(0W+W)xZ>rqKp;Fp zg}ub}`1tlfAgt!?=Nt1g`}E`{ub<-BP7mUNRaF|8cop~mE5*mvSHj@9S3o*edwKj* zM)L^AcIIKk?_)r9j{5HmQDKk6uH6a~z+=-MP0v~Lz~~byY*4zJ$BO<+Uox6|h4D;R zV=fwnw-Ka4JCX;(8N1fONY_N_>5c4%I|6w_&rxmL6UUlM!fGw`H+V10()^i%V$5?> z{`tiqF!5d`UWeTzUfUu_KVib^eXMZq4oHbB6{IC;?Pk(BB4JEvbc?nZ1obTRN=aoj zW_h%6ZB*L-zR6#t{6cO$=?l}RpJE+DzF<(sWexEqX^}M1$g(<6<)tvst_8d9Xc$ zneua=7T1$*XD%vl!KjC^Z~8nmy8jV6U6~8&y-wbVrc7odP@QOe`erG=k)tRSL`CZs0kgPy~MFO z^FeVt)jcEpV#-^Q-qjKpL-)$fTGE{Ab8%2h2T558ji(4iR5YB8^#p?%n}Ehk`N((T zn|aEcgX&|dd9d({C;m9W=za6`nvWA#kHi3F6pH)!7t38poUd{mk+|2FQyptb7YNE# zs&rs@n}r(EjbfWmDf_dc8Tq1GP;EJ}>^q!vSpe%(r5@|l^{B{de-b7_1wD!>D z80~{iwO{5a#EJ$vxI6xh>N%<}cTRc=Ukzt6|E;yq28|})dsh&?YgCp*e1Mj9TRXVzEQfb!>p>WrziSoqZkqN|5V z%EkEiUWPQ^UxmF9-CLt)t|5$4nSj!^z>>ZQRnMets&YE2NgV1oOz8!%Ge}UpRMi`+ z%hux@hbyW-kTN+9cy6doF_HJ8hNCL#F~`0%u;yxLGpHUCxh1{~0RD`*{Xa$Yu_C$?QsO^L_V4X~A| zu~hs*JcESWNLn6A&kE%Ok{1l%D(}kibOzG%NOcA&g}RJ1FjA%iZ{ma}alJU8J5A_( z_^0v_wKklv0bSD)82Ljj<&YZXV<`{Co*&8?^)HgX(~^I#^6a-mZx9yQvtRuXC@X=H z;q%3mU6YC1YJlo7)eEE#AR^?UP`-fpn2}yX>K7#3*XCZ^&JG-|g*1jLZJuL0T6r`^ znSjz%g60&rlK;4HWIFlJa$yu)hUEPPaT{eh-Nmbi=LBg_(p3McxmIhA&7-`ulo64y z&;Vt%qL$w$B)))c!MRE|!JcmW@jE?$OL+o|3*CSL59d|&9rZiu=Fu40E{C-%I)cRY zG~Sv(Sr$^B3o4iS_x3pu*K0Lx+EFIuB1xYMs$FPy?E-1)IM!gS1t;u>cL$le21!|( zBwZ{H_3f4Uma>{|}>9&G!Z#^|@wgp-dElxv9rW~Qo_s&JKj zS5;2qH9m{c&lJ}v&4Q$vwUj3?rTf=a+gIh;$UjmI09P5Gx_-(zx#C*FGvx)f3eS{R zBK~BQS>`K`s&uX4tV2M$S$u69iK8q_G?bOId3PKs3%sK|A|LBsQRV+tUMKB$Ft@G+ z;IpkAl)QW=l&7BGrBYn_y+ganP?zsH+fmvLnu97ABn>XKM#~kZs|;OvHr@j1du7v? z`GNB;Ygv!YXn1$9Bus>MUF?D8iGAwzoV|X!1PA%h`A(F*5}yF+c}eRd)i_j*Q!jA! zxs->o@e%E{r@90xp4X866iu4N;qN}4F!v}ui+pJ>ULCa#F8X-j;JZOE|Hp8+-1rRJ zQ$0;{bEzZu9qA_P)@&~AR~MtZ)6=|vx*J67hI5$Tp_z2wvJV$pb>@TqEeEeAzhSu5 zEtuZ>F-uzhRD^UGEA_LI=pMM&aP*5Ub{|$Lf)7RWBb|-p$E7h?aj+Z)+kJrt{WnwY zR)f>OW9`>;Ci?eqZfG-F(|=2xc!UY8Uq3Itv!WE{O?(Y)xBtUkW*ubU!doAL^BMkM>XUo zpDfYw@-6W)Yd)rIPZRVrI!AVj%s4iMhrgYV*^!&@{fIAYrCBfV%=N_sJ(|ms%#Qr% zmlRewyS7|dYL7HO>_F^E95FTnGrK(iYn^XMW0f`D8Os}~4W;+$Z1Bk#!LqC@_|yE$ zcynnQY(Mu7SoB)3+SG=7PHVw^FL&ne_xkg5cXi~%FxtnZw~c4;;agBXaTBx+*e0Ia z8S%;}(gokr#Ef`%aeLPqsn@KfbZhM-d$?yI_f5lW%ii#J#x`bja4m6=Igj~s6GJ*Y z!@OZ}qV!{t$lUk?7MAXTTa|C{e0EK)=C1ZoV;MX_;C*8s-o0*hd3bFFT>P>v1oe zW7zChGLCcKCi6S&(ij~ym2ndXNxEkFj-H>u3Xm%cUFD%8_rSdQdc zxX`4g40Mn<=E7L~o1cet-I~JL(OA9ox5jo|EEy90z3&rX$h)RR!7j5L?!B7s*Dh(q zFB~}o8Fy(AI2H1qn0wF7t45AsGZ#WB@b*TOu&&Pvw%JuGP@0z2RF2X z&SQOe-Q?le&nN|!Ug*uvOxK0be(7L$2-!7@=`bg|pC)`*0X&PlkD>J3kE+K}mk;3H zId4Gq^24Xq`Gj`Yv6cHd*1lgL9{YG#v*~0{b6%Z{~uMIOVgEC+2O!^bsn;K#9hQ{C=tc9zi0~qxdh@cg)ptBoyT*>&h_lDVFz*O_yDyg0ny_x3Cz(z3F3%>->T#N1vCQ3&j~W$%^qx?!JPV@w55eyX zG+6eR&Zv(YhUG^NYj3+1iZ6?M@W4&_Fvjx$4$vPeXZ7gM9S&q-k0pJ0_UKo5Y~OLb za%HrP>VHRI1>$bkQL^|}9?p(&;8R>qT!+blVBOUUKfnKi`eLq)^bJB zS1~eK4a8<1hte^22*I*E|7Y1QFaIAqLr~c6R zd018FM2;T0T|;A(^|!9lQhnk2>6t)d$FDVL zzMnqUK5_=}-t%z8mk^jXyA&__ald~kJoER4jWN+w)3&hN^)W14lEPI#Q+-M5S^RD? zo6ekUBsa#?<_(Uw<-{xb;ln!7Gbg8Hn!cr^et~Z<$-BE7%1uAc;^)f2;JQ->N-}dr z7g!C8BE9j)fOBlzl05w7yipEa-&#(aJwP@+dI5SSACXks@|91DHt9o4nVK|G92=E_ zmk<60o0y4Imknip$8vcxsR!Sj(U^^y@{YL6etb*&lOyzis$yjx%v#it11N z>llviPr3@i7&yM2?oY|K=i8SC$Q?_}G^?FoBJ~LkhoL$P((cEzzvicOkxd$QsPGfH#)SG=Exjmx~)@fpMc!O{6J3%Tl-; zy-WM4x`cC~o;aYT0azLu@%cVsxHV!c{PTBWd!5$E21c~ELF_Fyxhx%3zf9LJ0###A z^ZT%BcdkMEzBQ^ijrc&Qepu~10ah#;3elluaDBKhr~bveW+vQeSu1?pI|>@<6|fHa zGuV_9|FP%yj^yk6UK4~HS~U+whf1N8PID`8(PQmgX2ROu{?P5hLYbOK&ySB!R=NOp z3>huX#?tehvGH(k!4)99(Bv$?C<1!9qR)ZNxWqFasD=ftwcOLcuu7K@-pHV%P59!H zVbCJI4E|QT0aFu;z-`_cFz8|?(-W?U<1gnh^EOd*9$Oq9i0#aiKI{_{+t%RGw$+rr z;E{p5G_`j(hbJu_;OkX~U||T|vqbpX)?uInvmd{Q1oaJy!x?{lOBR45pAV^(0$_NpW^P0Q<#Q6ursKU(iix!RVXespNur0 zNPH&N)0ySOLF~wy-T1TBO&~61#JiZkRB9CGY`mL+Pg)G;^x2%*`T}8Q)%Y4ayuz?A zeZ*S*gH`>xJlOzMkGs!aBKOYT3*O#aptk!%Xm8{Jgw4X}nbVJ16D`HPlcDln zn?dUns&NK%U9e@JtYnpm+2M>g2wM zK4r(j_QpZner`HY9}89cj~{tUL+d0}UvfglUm)Ft8*(GC|C2*_+^i`ljfjL3IMczA zrzSnZ>H5A%I*k`^*haW$NgB4E(o#U!%2p06r?YrXxT-_qMKnwQ#R%VIhTTzY-iY>b zO}CS@KCtydF3>t6X(MReURUZJ9Zfy<6V~8ukN7qD_(%?iKM9okT3#aFyq15bs6E~q z;*OmTIcSvrb348jNPBC857idbTdY@GJI?Mn0_jE^=3R|1uP9f&1T+WyOx#DMIq*{o z)Al%9K|0eBekN@vodkFop92lR!y!OvHO=|#4RZUNB<-#m|KX0a=}LFQ6ql}i z;PfHHPosq6r)pw(#YBv#WzPG!+@t$uE{fnEpHTJTuEIp5wWu2N{=B0=b<3j)niB3< zNrUz8M9cRYp7?PF)(Y+i+dBSbe!=!o(YOs)9PLzZoR~i$mCj&oEgu;i1;P?g>v{HG zcldAQDlj?Fh5DCt)5?u_d+|td?cxA_f4;G788KUs_J!NKOnCLH$530_6>m~D=~C;a zX!ZI4qxEHJ1|yMh7%WWc^G`!|3er3{bbY2~3}q5jH*}v?1`J=MfVTdstr zzhB~mXC&g zKej%HKWw+6_x^dKK^yk zVEAF&ij!_v*M@tYOprVVySBUoUwk>91y5j-^c;)n+6qZS!O>N829ViPLE0Dh=egn7 zVK)SM0`bmb2*z~Hz-t!oknj{QhtM})kv)*iZAkUj=kR4kNNZKSo8y|=$& zJ=;yzsx?>o&+hweg=N~pKI6cuSFl**X(H_SGl<28C@+bSW^O7UnlX_XYy75 zfxd45P~Ry`kfibW?9g98YcDLfbiz5dmYSd`#n|KySKgQWh8d8?V5CuquSUz`X32sy zGl(HQmHt8HZAO0>C?E81AbM|~q&SB)9~;Dnbw01O9vk8tgWlejlKMvQn~$gte=A-_ z^3rf=O%3c8@qqTqZK!$NyE@W*!=M;s&gYj2bJGWS{*M#s{jRJ-%dJS-2B@|)O|$fY z_>p}-v5^t(=aUx_q^np;;9Al=%M?Be#WAF*;YWX6(sZ*$udq()=g`jO1L^&5!t9a* zkQVn;zC7ejH?G$1`UO+rU+Y8_&d?lir9H?ii1H(Cv{@l-VM6j&(plwVt*8UN==>Pc z589TVgR9oR^^y>#d@{P-5eZfYz`5=Yv1w*^BE{l*iXV@aX z2V;J3M6(f=q;s7_+rY6veIWk(P=H<2sss5`mHBAXd$)(&3(XkeB@!kB;RKkUI*~`~ zN;p#im0x8Q-|7GUldtBtt!a0DXV9Dc*Isd`cYB@6SGe*@$e*Lpl>kuVaUn zR+ObMjK4OJ`nsu>FoV+?@VwMRjPwOv$4`aR`1|n(I5EEoTq#M?nyy$QNFPbkzx@7y z5mkPS=2FHs(O_0)Z&9RQjgtq^9Ju#@6}T1y@hC4{@keDPK$?<~UL?(^Be$iVR{j;% z#qY%9+hfE(|NTPcC7l~N0L@dC#~}X?^m+I*$r{LqF!E!Z>Qojty@9gEODO&5%gEp3 zz-h5s;t}O1aYyabDkIU{tV8=;A2j9WfeXd&%{P?C$|J7;Nt!ck)MaNN{h}r8Q@Mhg zQMdlyxDsRyB zy}i<2NIFkT`6MfN(Hh8$kQX~I&f13{jhFm?ButyNmQfbTsMZ+ehOE$_2No=N0Yh&@ zume3(h3B+%B<(AXj9b7|-44|0DG1MDNY@}w(&ANhykp1{M&1F5_qgfG{xq*m;X_A| z$}_GkJcK&a0=eQZf;^OX-LRZd?_jU; zNYXEx80Ds5(|slQUOtJG2MP!N0+dc%w)B-Ctx%<}>3VscGql$e^+1pRRzcIrhmi76 zt;($2j$5J1jEMVCWm<$~q_1jlm1}P1HDSx>r7-%#4bk*y&nliFd`996Eon$TZI&rh zT5Yd$f2F5U;jPNJ(hMGH$v0~wYTwh)|0~Y-Brn9syLwXIK{_B;EUjm!w2{WL-((nH zYd@--v_<{a{6lyUn-%+pGPN}3KKm~t%mFo5^zV|eR#HDyX#w(ysC0Ix@g@qdQF&{n zHxfU-_ng>vz2?b`Eb{Jtg7A>#4ZkZWw_z>QY*5$lDTD-XV81Y2Lq1l_KYS9&i}RX(MnM~C@)@i>PUwhT#pDS+((nBmWdk^93Oi-Ry`Kscs;o|q>nS%7V zP?;z3rIyCVUadXOC>yNOYvHNENcfpgdIVG^?b&>t^1dqT0+UZ$v}!%>4|u3@DI^Z4 z@_z^J4ad0~9*S*mM}w*xy=MD7%};p?l?~GGa+S5ZZ)vGG0#xR;&M^pv_|}nsTAT%? z6;-bMj*hD868TzC&L#)niJ_kAC`ogWj#@)moS9tq<}h<>y&pc-@fW=VFA378k}^9q zSeFhS4{{h`5nR(Gk^lTm98^;hk0Rk4-0VR6X_p(yfw?YRW#{BiIsEPq1+M;}xQ=pW z@-41aI9Gf80gV087MfIZ;dYhB#gmf$@ax}Mv~Rf@X43Z=)qfPx^JyQT=a-{knH?uX zd|SvzJ^gXMQJ!$waafo&9f1AE^b&6(>Y!VX({TKfC)EFV5@y*rY6ngZX6;RTz@O!P zrDnks?U&g{=vh2-S-X2{c=s}izY2@MC2@`Av-ZJaK6%OKUmwCH=UHg-BNiIY+pG$pJQT`i_m@Lz$~xDh;hxGq_y{bw3t>Mjqi4o9s6HrXCI`B zwNrZ|*BsTn3f7kuk2v?Si)9-JjSy>Hufj{1%ybS6!ySEF$|Ki-2in`<{?ud<&}AFe zEKJS&G0X>PjIzYJCO>?A8ZOT2CDW?)mhDZ?W7Ie^Y@?^apElii?0-|RuwEQ?9$k}f z?D!b!RfO~S5vyUzoER!l} zjehnM*UYbqUSD)&qlVVAysJYGIpZO&WYZ>4ov z+KwOn^$RD}Wb*uI+OKzXYd*Nf6IAb28r^_*+-M`;`dY~L6YIzk*8gC5*g|}6F`TQ+q+VfsF-DP@eDa9RT z+}*T+ED!DgUt(QBt;fL0_gLMEENDD(C=Lw>#d7OVps~qv>(?;s)@)2%Ss!h@op|EP z1UU35l%;t$kWu5-G1pPqTGgj^nV<21-(H}az)5R#vHiptd=_Je)%uL#wijo~2JvU% z^OXZIcllwoXyXMpa`)iw!G_u)F%wznm38dh%SPDw%rA&9oQH!ax0J6Qy+gVl(U-0F zoa-9hmZd!c|%qi10+a~-*Pr5{uPozMC?P0Z`sUPE=yhll-u$WxK% zHn9+BPPID4cQJKISDyXrye3Y#^WnEn!uu(Ka1ru=|xvqyVGl7X!~h! z=D}d3`N29Z4rwl#eTD*4+9&g@og7Ths;1ssU)76i4xGoVgcDGh zNJjGlcaz;!zp(HI_WbwBo;ZL1bo?+SMFtp@GOAO4Wd3IiwBLa*Zn=QvwK`bG_y}&8 zd!MDycYyTyoG=}(Uap6l&5yJVvQMLdYdE`fW(B$~4F|Q>3ydP6S>!Qr`1B7J?3YmD zya8>Rwc^xs5_`16n7iYE#xJG?n!{$@jcSf~OFHK$VsUGEw9!v7&af4R26mH8ik`u$ zZr;4u?g&13aRR0VZj#jBxGypg7M8E#!Cz-eg~xuE9pL!uj{MRjpfTuF^;KiE6|1Y) zm2c1Wk;dB=huaHOh*g z)|8@A$CKi6nxRzOkg(^w>Ssyy#s8HD5zen?saT6$Jd=Y|i<@`zhRJs@MA$n4`m|rSLb!iW6>BVJYzerj4|K-Zz(vXNQ{s zjgig%m(IAxjd|8{kcOozfqDkRuKtFYyJh&Ten+{m7VRmOeOgrR>de->iUX>TeEW;u zKz+}M6BGwy;MswcVMgG`UiR|xGvRCV+(dB!W!~yomQXGMVnV4^ziZ<0A zD=vpCZ|d;Mzs8bq8%#xgg$F>~OFxehqr&2G_Q?@KVO;2unk?G#DXO08)A%IDiDS6= zX>D#S&cWo?M)GM%6V(@NLFjC0TR8*m`-OQNscS?SdI3$M~x3F)*O*X$gAE-8=OIbW_yRb{B>kAmu2$y>u$tP{Y z32PxZMxQCHm=hAhceFXcOt)Ez8H=uII{ns1!f4Pb9*XOxTJf1SLtu0FziLhd^)P9u zu8hVc)1LK_mz}*CVKEL1s1DQWTXNz-c-{OLdXGLqc;q1#z4sH}le|2i#tc+-gFa>5 zal)FTINEwEK0CY%iEn|p2C2WK-iHai)vx`6Fk9X?DA1g&*Ogzr9EVMc#hkKX@J?DnWGjV$K^=`}Q%`!3&TlQGoqy9WQ2-)4i~K1IS&nS8AcHlKcmJ+9G* zJ0p99dW8?5;Vix=Ba03ra*du)GX#b;Sbldgj&EO|ihH&fN@q_; z?Jr(e)q_^mWaKGB(o3(@>y`Eqo5XVP-f03&p1I0jk9$CF=YfLsApi5Yfi$*^!7R@O z_;_-CP8=gHPI`;x(==6_d~ZxXQjK^jogA1xSRRwTvD&&;oaPye^~Z_hP7%zbUJxVQ zDwM_}eFJ+Mnuv@3?=f_?uU6?B_t*ZA)S@wvp2e2ew}V&s3{bVbEn_$MrX}K;-vha2 zc0C^49&qR^52m@%5;m_qS5RP?|<@?Up^}vZr}FaKCg>m|v&{ug8*3w5uzp962FgM|6Q2 zbn~I=^Xa#(V1Hx^hWMsy)tnH%f!nHDn7LuB;(65@7;G^^5>E-zBBTi(B8`>Ce_ED1F}ark6udT&N;`MjiCP0X>w|b7 zr`9hPaU{rBi!O8k|vY(b!jt_@iynY)qjOw-lGz&)Kss8jFd9OXF=7uy8kk%8AdKR-G zCGAkvV&oHjPIZ9OZujOX9va?zsAL(LPrzG$0Srn{g<_vW*mcr&9GMq`21oM+>3!nG zICfRF{d@ILJkq=(y;gg5<^|E=^%k7e+EZyyIX^@P=ao+Z+m?c@t9?!AR8NJR z`wj7OS|pP1V@fw1J~pFDi~6MfNB(9XY25vSdI(jYFYS;*_g|idnS3zsF*5^+XF%z+ z-5ycW_e=xyJCqK6hnhlmBR@WMc3nAPjT`^PV_AAND{0(uqaeM3KW<0C`EY0O_s>9@_}R+ndPvr=O8NU#$CZ9mzjI$(t_NhZVrRWIe8GNO>R6oZ-+svo`8Z zeJn^H<8(kT#fQYMML_jdWFBV&m;Av>VKSTRG|qA z-KeqJK8QY}5T4GuMSFe?0JgccaOigqjs_Je|BOEuZGe7%qETtWF^_DhuFbg8=slvH zaMAm7itBjwyWhnTdfdyf)IfP_Ts7{v!XbrC8p0ej9Q1)TO+8A!`64cvW5d?h-i3_6YN_W%ZKSjjK-j)EJBLC8L0S#`j+^) znf!KJ160F!DN2jIZXTn1v`@pKe#b;ZkKRE3o*NwXWW*sbs3+Z5@81O2kZ1W@#uqD1 z1$GlwVsY6Er2d4`0ljfeaXfi>Es*{|Uw=AJZPQ)6P?inkpJ7H)6Zt4T7S{ zW-Az;_zjy)$YpANPg@1@#v|{d@0rcGaK9<1UcgSrHKgel1J$9@QHtj^#KS7f;1BFg zIrSnVPm8x$23B(oQF^edwnysd>gNum%n23NChS=ts1Mkkhi$98 z;-#``Fmo8;#Qt|k8iIwr4}&=;GjP`9Vc68Kk`1#s4w;n)*p9yY*qU`c$rHb3-QU>b zoUSh*?(Q<7{H@7Ar z^Cq6u*oMk~D&L}W;65xJm`uLRTWK^cX;{TOTE+X6$Kc%pUFq0%i&k+VWm%kF!^*?j zW5b3eETNMrC%jhk0~5^C=FDSrj0y7=a$Ma?IM6lcNmGxycNEbE}ad&mG`Na)Lc@L8A7nGggaEo8)jv5KdNjPZ+P~LKJt#}dF z>7DWb%G*K3BZ<`x>_@^~B)^KJJ8;$RDCnZUNkera7P`#Wv>6ovFTQ5LzGsak^%7Q} zYpQex@pLUr&^@WJw2E&?)55)FE7_A)Ni6W}4Z>?zR5>8^HG39tr;5v)rn;fxu3JN$ zxWZM+m6-A&it{^%uVXQmhcRO70mXkp;cN4{OQ6N4t)j!mep2a^8#iqw;i-nSuqWXL zP`)8lrfR*!LD03bc6$p%)~6`Q`cz#KuL9+s+QBtOD?VZe+rDDv8Twkv-2`b6^zn&? zC`TXV^Wb3HNvOQmd7sg!K9l-|QQt`Qekt)0u(nTM@=n1DBPeq$COm6|3d<=I^9=s_ zK|?-Psyu`^7L@)pm^&TRnvpiGs>SF` zeoZ<}b2s^z$_ug5_lLqGtgw#4n+wxbmZfwokmmwBzg6TVe<}}3JRB(9oh&H-ny&O3 zcs>4!DzF2Iy`CVblMOUkcn6IO|Qp zic?~J$jhB&d+;Q?Vpf~0dQ-XaES@Uk3%`m*F7;6PTg5}ezIGzb<|+?=(hz1?N~&R{ zWu@27x3qu4M97v)@npL$TzSYjA&ev2*mzDFf%ymcyVgJ4hh;;evQm&2ima@gtl4)W2WzVceAFKgrO0eem^ldI}R z;NtbKna{Ny`LE6ke%>aC(a+TDF+0YY&Dr}9_b<`M;VWA4HK9K8_?}WcyLJ>`(JcU< z%}o`7J6rLkA3^x&cEjzLZa`ZDJ$!3jD!B1qSQ6Y)3@xmII?Yqr&EzseWJc0RDdXPR-><~5Et5uw^*`} zPP?#t%mY^9+g*+dkH#R#wN|> zr1;a=>^EJDQ5^_6yHB8cS6zkax@>L$LmGr;g-kSVe!bWP*C~=X+&phO$akY7lED{?}+!ix(EmfUp zj-DI^v4~uI%uJ59sLuztNx*QMn_#xRvF!e%hP1ABk>)*?_E+j8)`Xsd(lldf)oG{f zxXK%Mjp!uDSL?%*&*$U1t97|o-Ef@Il=gw~i{RY6rmS2?=h3DP<#Bda{BJ>^tT>wq zUOvarb=*`m^!f_)K3M&sDa}o5aG7N)MOb%EH6_{=bJZ6zHlQE03@gO<5h)Nbey*l_ z^e*gjwKlg(H$^p$ADahrszWF~RtY02O=x^RVpP|jNWBi>?_K1+1`cf4I|n|ra6NW$ zeZy}2W4!0mmmV$3BtGcdfzvt&RiEZPX9LxlT5Hr@_DOTz<~F`;pMzt%NO0=(7bnGc zm3{;7qwn|xNN%$QH)rc}o#xHO==%>fG%l9nV#3SYwZNrKpNkhe4>KAs+B;;@8A@eMXa9?{^9Ac~B%k-f* z+dmqp?)eaxD9l-3OS+Bh!kZ81hDV)x^Er>=p+R^}Uar3kRXx*dq^dm{1O8hd!&N_? zKIf(#7F16<`|04)Q$|wtUfi;8+R%b4c*}Vprgy(5_C)!^yaz^nRM!iPdKBotkg(_y zP97TpE}qxL!L1+gL9<%C2Wsd$sXPoxuOZuROCVf`fFGre`LT`Gy#KX7@c3ya-nZMX z@v5uC2eq^3j!pICarT0(y5kSO3TNQ_E3d@yJp-h|7IsJitqbAbK1uV*I;_47E0?Z; z>EYAS3Uf?4~2VyVG+PW2!!ElWa$XS8nO@xHIx!<*?$*9>PYUYP@1H6!H17v~kO z8D_J!6^Up5z6;#P>6{LYNQ-!;W+ZL?R&Q@-{ zIunAM48yNCJ>;6CS9E^sEA?6Q?CljWc$y}|Z3f8)<+s3lS|3hpEHl$<;qEoJH8n?_ zXG4cJ1Je%2;bZtF_|u{@sy>~U;v=h@xxwF!O=RN3jp9MGjj9%8%U+H7e{?UA>CXK~ zeF&fL2EyRPYJ5u%6In6hq|6!@%fA}FWyBH8`05+x+OZqheP4-{>n1{pd0kF_kZ@Vh zlP+@j8%wM)W-yfAXv2#}-w;&Gm>o0+X#VkgVi|eGP8jy{Gi=IB#@|kTS(MQ%pxTH0 zmwm;T+h?%S?l*i*ufr7&P|qt47VlQH(IhA8^U@oKS)Zr$oMhq?9#AqG>GeREE}Naa z50{oT!Vy^k$huvHRX(|RYDWt-Zbm%J&z&+>y^h4$obUuA{NFL^2k`c4$c?W~h2LW% z(J*ZcH(1E1e}}}T5SpwBd?LBtjDI6eFwyx7@B*nKzP&u7#9b{n^% z-ityY{zi|%Yw_=(iClNxaz?nuA`k2)j*XL1-YsEw-aoC~#)i15e4sQ;dkQAI^3bq{ zAt*kKMbLzPU&WfZlxayPKu%zgX!-0AY}iyFe6HE>inFaH;SBu!ZVFBQ9wMCULfH3O zP)#tx3Sn%Ls#*57DfBGArg#eow~%oCf0E8Rtg2=U<93S$2o|;|CI)b3jfvQ(sMsA~ z7sh7?*kS<&Dqt&OVZfO+f}moeVxm~s-Cg$`?jQI0o-45To|(1Y_qW#BXGVRyR)dDh z60)Vs73<#>5h{D7wRC7G*6HVGWhv#(4lU2pRVpI?{9Mm1hvc)Iu2WniPmw3nNwtp@ z4>D`%4v&w)6Y|I@6AJ5nGL(#8On6jJ)_52tDmn{!=}s?M&F&K;@07Sdfg=ZWQ0h^A z2J=wKeMh+LwQU^M0dM;>@|1N_(}kt=eMYRaSM?TzzbEGzojLE6T5@ZJ*VO6Wa&fu2 z2S3SImd5y;;nu+=wBPWWY!f_=gNH2P?se@YJXF42(3qcXs;GPRsY905{T2R=i=J99 z+>iKCWt}K8@2$c;cd)%n)l?*J!ja;=Gdm#R0le0aS z4Pm2mZ7VtI8pXPcW&Tx-Ka#`VYC5E21KD%gV{6LElCsaychq@O1;bXl(68@wp%jT) z0ggrs3EG0OhQ%+4uA~Lt^&GvtFi#E$=he&dl3}Z!r@~d!#_Q>tB|C>6T}^Mx`d|+5 zxln@!+VOU(sO`~J&zrx);(a(rT*aRyXk)(8>e8LtD*LjWh}pG#aep%%IM>1CspZmy zF?=XA#pJy49t~f%+I-94$ZrGW{SLe6qC;&BPox=Ndx$NcP7>xxml^G4O(uy! z?sQ30&^VsrH`DmCoc1$-yX9Fbz%R(lD1I_NKbIeBBjHWl`$GeY9B@&T-nG=q53BIf z?5%Z=FPCX}@@wnFwnfb|sp#>;G;#sH|Hf)L*p3&Cxh$~uB3JY*v8GZudAu&Fx5mw; zi(cL6xEN-A^T8cstx9=qPB3_fn&!?Fy`3`H=%8L$!R z?#>>G*gBw)D}3;8eX<$QhUdS`rLX(%BE!4TZ!0)4u0fq(qh&ng#$oZ{KpT!OQ9!SJ z`%ipsal#rAz8bl=LjMYF!?SAT5GMzA)9^;r8gHn{O)F*Zd|Xt)o=Lqw99DJmuA+l~ zV@+Q0ieF{*sWY*(BC&_@Oy2pbkdzZf5IjhXJGM@MP1Vo8NBG(KHgZ<2d7SNKf8lgF zpVhlvn8}4K&N!+)!7PpV`W0Kox6v2O99WHh<#^timS4iTYKJ;A znBL5#DtoQ6gl6i{rz!Z{uZT?8;we7O!h0FWLCk+#a{bzhFexL(srF6~00&Fpj&UG&$!Jvj~bfp#RMbhS7 z>fiEW8ro&zllI>76mr5#v2ffyg*l+x-*W0eudAX-(`tG;zL)b6N36sS&ZUtrwCg|$ zN2Nq7_^txOD(~HRw|w1Q5jyh@_6XRiE;{U^Hz(!`aBxVRWfK@_2inm`oXBe@ z-P=#czUL(+_?^K7;?blk^xuVx{N_YXX|w}eq=qMtB5*nXee;X{`aY*pdmPoPQ%(e~ zP#*Zc$iThTWk`j>x?1`96g~c_+S}|P!HWp|EL`iA5vyMg7M_20i((a?Ta5?Y4lXIu zS8nIoi!bw&OGAy%5Lg7y+E=5drPvb>D5ZBaq1V9wZLZ3QMFt-zujN_jBBjt zF~lX0Ke9xD1BHptISZ2%&iLbj6F<_PS%H?}HPZ=xm(>R-Ba4Z4v;7q6Xqx*rlAq_w zp4Ar_d@IRP>Q*A-o&p-A+{$$|b*Kcxa!^b?MlHhceDi~w0>sqF2BvlphV{YIJo)ZI z!W^pwwk!Gc=L>wLX|ysmKgI!ODkz~vs!>u9T{zeg^=dKQsONfBEA0z>q@Wz{y;q>u zBC9A@1>Ys-Cz9rtIfL3aioP_9Rp3kRe03%xrWCwK7+tzvXFMeYY%*~{ORt*q`3e*2 zhOAx=^vnXzl$Brk2?sk{U1-P?!u9$4$m52$$m+dMJ^%4u<*1OK>u%o0w|dT??F*PT zY;DW_mzRi!y?64W-F6IK7Vu{2jIE%jCK(Zibw&)c@*hMIO?{{U;dgl+_FQwj-yh$5ChX=8q+J(Ub8wAGJ`txX^kq zr_L@!&+-%@!@-6tqx0dNAi2h)=c&>fb&nOXi+IC(W?0isG{PEPVhMDz9rf(9pXx*& zV)Q34Cm+bf7Pw#a=%?9d;u?mQWj(vi`fNte6y}}c?WR!VSr7TS1oqR8JfxPN{A~P= z9=2I#$q`pZ4$SDsLH6Y|Y7g=ozL(L5(FO9YwP2ma#D`l6tVS7~J+)!s&6^_4`Z97} z8V*IyTFHJRp$3gdZmUL%z4?RVQMP-Pk5AT%gm-S|b&aQ4Po&0DR{uh@RxzMDgVpqAo6`bGNfa+;}68 z*Q0NT^J!alusqJlslhV@78Y?!=hD7#yi+}AEVvF#iJHZ``2F}sJssta!-J{hy_Y6# zbTcZ*s81OFLhvb?Huxc(Nm`uMC&QTHOx^ytK(sQw2C#!-jcS#=wpQ=o^*7rm~FXL(jwXZAft>vle6^a)Mv0T#>4{BXO`P}3){ zf@>|fo`;DG^wAi+%;=qpiBpd$uV3M0opGHpem&#sEplktM!gtt$x?T7WoYer^jY=^ zc)LdK3!~NPKZ+4pSo}@6rV!t#iv|%mgiLSi!;5UnrAQlsj`NGUr8II*7mJ%t**MOM zdYQqB#uKvoaMg~58HOO#?E*CiICLpH-u7n45({|mo=t>ai1N#pXgo+o1_v;-RYGsF zyaIh0M(4l;)XN80J&G0AMl@20eGBSA#Oz!XFBbG@8Js1-5TbE#v!%d<`gB@-)DXMZV!Q52&}5kwVy3Khv9n~zskx6pIa+b&%XJHSjC^fG)>y!^_Z8U1aIXfZsEkTw`9$7O+Os#B(Vy1n$0FuB z5*S;&ds9aHWov5stLW?BV)Sz)uIq6iTXTtV5PF!XY3pg3%P#*NvS!d{MiD3hIV?FZ0{2fi!5k{lXWQP100UO#c`MK^M`mdi#D)0>Lfs;Ktg%ylj2TWwb5zI{_Awk+k^k8XPG zxe=~j^gCqf*|U|7X}Loe9ad7UzU)dF7ZR=M6VhncMXATHK20|^mDaQGd=_8aedY7? zpO$ZT@~Z#pE#&^)eCcJIFlxBB6Hg4UEFC%&;ER8I>Cq8(+;7ALJ;8o7PdzbGci*yv zp5ScFIKSU~!8L~-aC^DFgFRJ`wk@H5`7fgamk;yK%yjBh!Bfs_`;Tm+e$w~;wV4LQ zP{~jxPul`w;P-`kMxNt*^R0^%8F}QNZF9t~`Y)B|gG~Ayu||Ds)_}c7EYvmhN-lFd z*qS>bzg`ykTl5YU6g2)OSKPYLBe|ycA)#B9lhy1nA$9TsO0H%%lF%p zDO*ejRq9Gzc9;{%)!o5DuZ}9uX^rH{kj8rVt7BZxCzIz4sHq|bZBvJT%;QVfwvbtq z9OqM14P6xDDMWo@8|r;wN9oL)Iy&c37pfW1fs=5?*RzU^t=Vh~igMqk=qty+P=OJP z)Y)yLXk*XQbnm5=2HDT!d!77r#jSa@MNcQ^!>+2=p6+`5sXqGmf(3kGS_4gMlCz$F z|L;Nk@%KROH?}_)tv{F7yN)4TQ{)S2s*gVEMyW@gb&sm`sN?XjV%oVjIx(`SUXk%W zZd0)kzVomTJ~w|PP7hzqo-yqys{Isx++iVIT(^&3&cNqU^GnEFqn}g$*_B1_Uw-=0 znT|4j`*W^UGn`#~uX0Ru8u^yJN^;gtxp=6fZs4_zQvAOQ+)rL8@r^V0bko&GW#i-L z&d`+LZaP=wtgJZ6X1i1)4%L+&ZM5Ul_Hua3YGQVgjco6^mntv6Ew+DOAV1ZaDw>bK z#WOs*%2L?VRkkap_5H`{>wD~d=~a=2b$&v=ov=sE_YZ95ZpxR{oOX8(ow2T0H@n_a zwx3FCGvEJCykOp&N^9+M%MVi(pI>pA%rj;D^VPw3=Snk2f5)Yh`_)}syKN{XEqTQK z+%nY0#QFMHctxJ*ZzG3XJZ^HIC$tTbcy8*qX!Z!-)lPXju|n8800v=8Jv`nOP9zOM`k=h zw``@Eg9{G^(8`o+ymNmG`L&XT2D)rkeXtkAre%G&#fmiUnJ`ga`#Ic-amgyt*C={R zfV8W6NcAjcr=ewfPAz*m$*G-Qp0H5LA7j~h?nc#R`XP3jQAgo9_`$N?y34-S-17Ws zfjPrG^)Wxg^EENC&xv`o_;2qe%bTDA`fTH-;@*};+`nQXpXrhziX=b5*k7}6r{Owv z{%T#CZMA2s@+!M)Zt0&mmZm>X_Gq`eH#uX^8kai-pC7iattMUT=ke!tn;L*oAAVpD-I z*RuGrYC6^?g1bKK$Mu}ro4;kR`61S|`%)P9Ba_eAYgjJ$YNIjN5mEpUJ`hS_qJ=&t(X+4G#^d@Eiz%uX-8ww3Vw@_q3Y9Mt#(*^U{a z@eCR{NxI$wTDYRK-d7iUO$^>b2}kx4G@oky$VD^qH0JOvi&gk_CvCK%>%)7Z(w52k z-b;(zyS#^9btsNM&0ViD>nzv9i?70*PNsMG&PT;^mj(2mEB897{{&5vmpYD=@H*pf zVvMhkxU=sxkE}CIy@`vIL(6r9x2@$(o_jQ6L{F>xm0Auh1ue%rrhz}Kqtjf3f8sk; zrFe1OYsv}sEb$+~FRY?SS?YB6)5$(0zg7N~7Ji1N3$(Pk}_voj~m3*&0u3y%Ff(3qfk zgnT0t`%9w=%VXh{pz)0a{GN8lRgivT?^D%#BRJ8ci?ki{m;>J&P_yp@Q1!H$)U|## zd`??Pk9wA3-EeF>m#;rcVlLGC#L-lHri%{Gca+zycM*5&0#z~Bag3ajPWw+;-AeZ+ z%qQKyzFAu;d+_xUK_+*&bCcs->_Q7^bYt7ZOJZZyy3}1<6om^_k3ZUW4Gq{aOnj`n zSWNT}rp`?ovsIpiPgr>FENm)+*h5|s@|dQUSk4Xa597Wg=hMCUf4JnJs^a92(H!sd zK&(3C&aI+?)!@wAv_^%IPv1B&z!b)qJ@#V10RPYS>X?6Fe(X4a+>T%$fz2EEDX)SD zPot`J3+dx)4~gS{?uqIV_IgK~lH$-}Kk_}cgI75`vAWz@LWLH7Fk`2|b{_~@Cv*Jm z%aKppa`}r1|nd^viUFrIVF52akA{ptCRA&i_CXScoP4P|}s!P+W|(af0QMdePvRlF>(g`0m} zL6|$E&+6@s4Gg{FTj@n4o=N_5u12N1-Naajk}vkbxYy|!kEz7u_8jkc9yVH#p7|*tbU~l~CM`}rJcRUMZo^+vbxSUgC_! zPvPYAZ7H3rUO8*N4=sL1jfXF#8<*=z#H9Kd){tL(4-#M=eIUYCeyp2c-mH+s3zPi# zYKZ2LLOT^0Q9{GDhsSkVJN=+C_k@0tbzCno?)Fsey7goHq-{C);&tre)G@je8DECOrB}dK;S2M0w zGhRcNqL!%wIomOGUhm28YH~-62y@d|7i(z_VZ+@`7Y1=l*IfGEmVC0@$${XRv#P@( z5Bkt@J41iex?!OLz9wRAob;65O$gUU%zXeeFQvt{M)1T`3oUvu(2SSh!>CE>6V{!-RL@zb-$eY?ZEEH;7;BRFWK)zHjSDpOXmzbA3xVRewn3{dWHRU zP+L%rzjqlPr@+XRFXe`c4QMA(ugUt(dldX%6)$^D7$58X>lvq(>1Jv_zJI+OU+Hny zlKyQmp{`ISKX>j=X5ab$2)r%9d!pBrW^8<+f43VZ$JL73tJTW(%jj6D2i?zigeUnt z|Y|T17G9Qou{Oz=(VT`Ns{5&x|i$|Fs1^EDt<8%|>G}W^uOQ9&R+G zoNUpDi{PRg>AKxpBUMH6*x2LUX|3oiuzte(R!SrkT(lZqj;Le9`9cz@=)SX5 ziQ4jzi;c~*>XK*DC@CXQf3LZYQ*vD3FS!$y;nk)KarRbVF~Zu4j;f^be)@AWj9nJL z@IXz0+BU$)rE%7{UM8i+8TWK(0S@l3JNb60E z+Lh2Nu$sI|-gTG!tJR`>wL{ptbsU8(e5b$?#>>^uN7wk1_k6_gb&pM+HBe8kR~Q4~ z9`5)ZjXB}4l^0DuWR3UFwrui=+wJ&8G3RLj&MNadx<^@-Jh7tRp>aP2F5#$=^9a2U zwc%J5t{3mb-Z|Sc^iM9X_mL4}H1)(QkFK^~2--+tD?7;8fb%A|_`xR!+Li5^f=4J* zw^XjS4>|ST=!t4_$&LHJs}5btqelK7%&2z=-pC`uj#IC;9e3P~@jQ2X%ExGljFA8 zYr6&YrO|fOkeauEPJcfq}SQ5s`_8nphm!1 zve2oMwd~}Yw>8?zPQh#wshihWehbLxx6dRFB)+)~KVN zHe?h1Sbfs;qZD#cVBGwC$72EBphoA;agndn)yT6Q2s|m@-Y6wcwJRd84&2F8oU5Cd zQ@!gH(eO1!-^^nCvC*Fa#+L;6gxe(K(RxTwL4D>p8J_0Rxtr?+)wf%#=Wc;M!EPFVD#O}4RPn=NCA7Qu2WnI= zA3sNcfN!mdd8B}+Lvugi=W7_gNS5v$30g>pbTAv%*tKD2Ry|?3A0B5#FG^r7xO(nn zf*+|XRbr`D|ATSpUov`J=>HWoelI3w$NsaMH&By?LkKL8#hmCtsU0WpXXQh+v)+^| zs)*k4ZIUpoTYF|S?uB<(pzDSa-Adn4dzas0XsI$Rc(~?zg*ZVEbTHLz@LVKmoUI$V znu{OXNnrD=dS~XS5%i{92hkP%;|j0hxpc)7s_Nj19`JFIvM?vs-oezo=w&3M_j%0v zRTozG#|*Th?ofC(VOY>{b19A928B-OVtt>8y<%kxGCdOS5qTwi)Wp7k9>*isxa{{u z$}qlRP4pfl=2SGQw}{bm73j;r+x`fnIjD8Wtf_C=E{vE35A4~nJ#@y&r5Z9a!G|w zxXS3Y%I4eN@{4}164#QY7rjvfnp(xQA!XV45_-;5s6tCFwm!!AB;OeRnT|Dhs=z|@ ztlKpK21idcFYb%8RL9-U;#c&Wz`QH1|9VS`oKr4~=}?zJcXyf|ln3I|^i0hCRYB#z zsDr3N+*asNHEYu@_e>8^P475Y)N{H(6K3w=u&@8o5WnJJfdoQtnZN4g3@tOYnTWqx zQlSq{WeQi}Plxv--UcyhA|2dfpE6#u&sO9g?;BiR8@(|4jk%<@cjs9V z2hzS+m>G-d(a@YVN!BS*rB#9QskFKC8PRq}5Rd4wi2GRP@u7AXDZNQ|`OABuTK6ZX zEO>CR{BR|Z_i$m6gTwh{#u005`9%6RC(@Fv%TfQNBeXZxMV6|$R*uQ>jNb>`w60Nj zFZAU!E_m>f%Co1twaD!X^3Y#fVH4n}pExcQHD5*Y_obbAV*ZIVBg|8}_lw;HTtMAWkjCeg{K>hH%$Pi`Xgpxe8ghoYuwf;9l#$QK9BP==F-PRPG|Q zGd+Wz>?tE6(&y2>VWU*Q^(A~WS08Rtw7z*>-nln|K9tim?cield6S)d(XPC#V!>w= zt6y=9V*`FNtO@*jIrYjnNc0|Bgirb9k`JfO=Za%C^N!L5Bz{Nk$$y&rUi(OXoxkw2 z3$3)rsWx?3tSJRSohY8q^dHEu0QZ$yQ z{cT1G!@I~8(Aw>O&cWD7s=872k1) z&#o^kXJe1)`(BUCyogRq`-?jp1F76%DZb<^XRZ1;gz``JSBJkGqQhTYWuJu>N~rRc zf9nyJueXZowUIZ(&52%odhvcWt8Qc9SB|!OmDG>8r@q`}1V#Req53b*iBC=aEu*jxz!97I%Km71{!-gTW%_pJ z+>XszO)sg_f|hXNhr)99iE(;Z;(d-t&!N@ywS3Gcov|)@YSG!e!D+wvxZg)zdb~hK zZ7-p5Z8Nw0C7~&w|F@mX`r%!Cysvk2BED0#F_23&{OE31dl-iYxry`b?%pmdiy%6_W|8yy+%v)&;LT?j{zsM;zm|{&sYo2IW~jtydcV$=A+HLWp1dg z1Kplcpqn4}!x>{c)?DJsyVq0TnHl<3^TyJy_EWCg1b7<9UHQD8yl_Y7*6& zFrRAovkiQq!WGK%wt+HpbNC$Ix%;Wa-2L?>7m87dF?FeGq#I?QdYmqPO;(%Sr|Qr? zb0n@o$XmT=Kov&Zs$FweQ?6_udGwLmTDe*IK-c}0^3F|ToL2mt{70Np`SZ>a2i6sm zCfCSjH>EAWGamu@6R2TJCSqjnvMwQn18ybH3BHPRi zprM-&i!R=wj5(#!-p$1?@4OQ0D-cgKqDMa2e$!S9;#z$QJf((au9CkGuVBOo-M=}N zYkr=tmp;Fx@>@@F;n)4FWBZ55)fI9hf2G8~sRCC+pHDdIAmdA9`AdILav=s)0#)Pz50?{ce=#lMc3)7&q8ZfPHy<~ zTQ&OOE}f$N>7pfyKDjN|`-b8B)0-~yET6)b4hgBeble*Ged(1fjYPcC>buqWic=2x z!n%Z?yIv-vugL{&QsWEPxkINRy5F1TI=5pI&G>R(jIO+%F%Q;}jc(A;I_E5Ta4z%n zCu`LZ`=1IiM$kZ(Z}Ung`l~+oj1wh3q0e|{8*Ta6POtfwMir<0r<*>hB5M`Cfc4%d z>t4=Ir5aAqw{Rxt-fxSvi6^(>i|Jk1Vru&d+SuGlcde9+B5% z@|_;lxVrO5cFxmO9N9Hdwks4Y=6+g9AF)r#h!>l9eXyfEyX8CKXE^U@Dvj#6gEkB< zB(bqAL9cP`Fn;W5tMx-C?G-UWV~unkFjiX1tqr{NzcD)W z>wC36cs*r*=FE}Hl6hh_2mPjQc5X9h7t1|0^zoysp_z-wX-i>gzqAC+jtk|S6}Qo= zgS+B03i?aQSB-Ab zq;CzW+v1f9o{*&nE1xbl8Y5?X=}%5Hi)`@jA#zc9hv1#*+J)$#>mO^tu=4SU4>qx% z-DLs4?ro#tk$RK+cf?6g#A|=PSJ|C!wvHE#WSIWsR#-qEsl$XY%jn9E`fkUT@_S=P z9la`>thDkhH}bEkDQ{Hxg4{ky3CSc#z7RasV>~&%r4V$== zZiS9#%&G1-s|R~bEvBt@MWo)8q>O(XCfU7JGoF=9Tm*kV%`cLNa?WAc7dUh~4@$$k zH~X5AOe+=9j;Jd1xVcTJvO^%}pZuKn^(`jDk`_tiGaY#1 zsrAdB;%i8(f|e2HjaSW?MmIKhp{j4*sh8zTXlRpO5?)@9JYJOZTU!dZ(nC1y&n1Oe z6laS)BHvOw7_ma0S0D0UocRSVpoEYC(&zMTONp07w8ghSzu9%#I$`Qvif_7+gF3m% z2U}OtraGHc&l(|S%yN<^&IOw`8|UHF((nqyC?ja}AwU zqt@gUCPtt&yz<=;S!dZcHoS_FDQFp{g z3;0vH@BPmjv#6ZVBMej za^}oZ+Nob%Xx29o6cR#SHg|>H!&-XU=)JV$bS*Y4l^VI5obx0y=AUQmswNS0;3roG zN3-$2d|f4DK1H7vcd6#*UnE@82!3ZcS9F>eXktV3&$EWU4d0=`S>Uhdyt~0wo-nnq z@f*v}d1Dpwi0rB?wVwLEo{`V0RQO!W_zguUVQFr#NE$=y$fZ|7eewKJ27k%uiuSt1 z^zFu5wbPd6;;qdk3u4{y3fF1f3jTIi^!k^Fz?E|D{#XibIh%I%f8fz*{9k&|A6zndjAb0r&nb@OeMYmH%!RNfX#slN^YR2n8-Zi(l{u)wWLnFA+_*v5U z%%ALMC}PAqHmta1+5)cCxjIix_F{O2irV*G_YAPfs>6cT09yY zR$p(eH0&V25z2GOlq{Atyz%~bF>&c}9u18Gm;B}TaShbNx%X(>i{VtP=^kG0cM!3i zjVm7tpeK%bC456axn7+XOc-QETnRbsB6Yc6gMW{6HEV^~4l&#yU-e5WBgb-bm{$++{5anJXe3`vYwmIAYF+_< z7SMjafAc$~Mpl>GYPAv=zX#?*M2v7y!L*IR3}BTa@(128MUBG7`+}owRKGV#G-**g zK6RK?tFX0*mFa4X*U2pALOmid2KD>?M(FQMimTS19b%3Xn2e4FEXBMY<)d|L%Xw#V z>N5AcQ|KEzIj!9^J$K{{OWDKcsP%6rs&d{Fb<{A`smvsLIr=@#I^dz1(>~nn$;m^|M%D;;I-5n-xtrk5!_El}{_f z{^J_tl_sx|U()%`A#z*dt&yMf@7)r@d{g*YTdMH%5gTn9ZtKGS`?fGR1MIq8fL#Sw8gjg9QjNETFHd&F#?yjiT;!h1!%m9}f*v z4R0QbgQN1H&xATpoo<DjH)@Ck<^NP>kov2X%^P-r(9!1(L>Ly)_Qr~M6EIIObCG-WT z@@_{q+5qn3oU3f~qp=m}$F?Z&jI*vd<%fA@`n)Mrxz2Acq3uSa=zZ?d##^QP-XElP zY_mM#bTO)9RSp;yX+b^5&?~*;W=%fQTXAOb&+10z0ioP1biPGS3*uipcCRdl&$m-x zYWlM{2X}tHjB8ij81J$Z?@gcvH+={&twJ42%Wss<$`j+2;~FfqV&g(HHih^V_^-Y` z!&#!oAjH@@%Jb?#RkF-(+LU+*by`oKY6`)cuElRnYWFvY&{xKkR9e7RAZJz(QfQPw(@CyZgZu$ zzMSU}CDqI?n&s_mJe|SX8uhEGEqIbBAq_W!n`mJCRmAaiGQAV$wcW((_2)#H)lPcb z#(Ip{;p$E+O|GF9Y0IUa$7lKNmzz3jV6|=ILB>4Ni*}K! zRLzpoHN^?F^KW5%+0+N8zB|fN+cHdzNBA3o`i}~ot1qGVrYEFKFJ!^+ye8+A>1Ui% ztrsbYfJ-1g%dc2BC0Apm!00dRcd%Ajht4WUM~pG zVwG<(^VPmRXKD0!ypIGfmlem!Z4YcDV%5|@0`q<5nC4SCjV)J(b*ZD1H32pTIZt z{`V?{x`a_%3UD-ydk<#1b4WmEwB^}n-g)FFCI8oq;r-mNUU}?Wa#5fk#HGEHOw9|P zR>q^O;5lVj5A{P9KlbfYi!QHC!F?yIF5Sn`iaB$IsmZ#}*~fJUmXtsLv&A#iu^7#) zU1tiTwnBf#F{>w$-sGkSdRwA>Z4Ze!L2u=Z@=LFy(5oQ-U$;%~33Go)xo_QUef1@X z!42}k`!igB`)FbK%J6Ia@~(zAQCo%>4OOT!#qt~BV#A*qj9#i}zvC~Tn75UN9SfvJ zd$SoGpr+O5GxCSKpTs^#L+@a$9TXVU@>CoWk-usgF5!y*VolE#>*7mue~(lqr;Jyk zr%J}VOrK)$2WT1QamMKr?5G3#q=3W{Rn@&tI!u^%qQOCMN7758W?{_7Jm+J5=ZEt3bU@Ax78u^rUZJP zG?A?guI5tP`cSiWwPjFn3>-QL%dE^03^ z{K!Jp`{W{VajqwgLf?5tHSFIPoKEI>+%|Vr4-c!$IDB!d3YiH6lW@z z?b=4a6{~2X#Yy9Oj5%RvyTN+#%2vAhvlon$M(LPeIsLWk2{LnWd*3P5)89cJEP0+c zm0cr#hLn}K7qz{=L&r{3V(?veY3sL*l782u{q^s&L+`vYtYHW3edYr>1sv1Kt*dgI zHMR65$77V4GKsvZdFhBFLA?F6ag$y}OY;R$5c&K_wLnDO*0E{(5 zWSq@=*SC&rpIn>^WEPT$ZT^z)48^xAAd5{r!DT$t#CpegnuL9JT;`sm{bL8J_Wx`- zPORa+%b%#4tD14&+#{&ujGwAuo=jzv_cq1Uo;S~cYTX28MUNz0~Bhi@_ahkRMk^DNd z+(OzIR9;QAL~t4Q)y|vxan!6#PEM@BpZ4D5-G?e@eBO(nI|-}{-D%W}uJ$ZJb7LK~ z(`z9bp6f4>zsfhDAqB1tvJ`I7M`w2Yph}dER#*PudnHrWNXJ_v)Yy5)p*v-{pWQ%q za&9bE7oDSS?0H3RTuR9~_bZE~2O7%x?Ut#lOa4&hsrWwgpbO;Qs8yCW<(*uG^Cgv& zSR-Bg%|UWlHeR;fTA71;MDVaHrM2tul{#?7WQwv5(Q#8krB}7n^t@Uk4H$m~8a9DD z#Qn73_qBe0hd1@K-UtwBmDvtLmmTGycJX5E?ScMSsm;Zc+acRVcKFObKjq{oa;c%5md!Zdz_R z7sdMnT^&wQu2iKuKFX%677e6%qh{-W&7N9M`G;t0@452iGiTrzRTX1tTaiWp77zC^oeoEPabbGj0AMsDgE%7{lCk&2J2>U$D)sRB^!>{Ea`oDL`QUECxMcQ0@CbOz z*|URjuN90qH8COA&h_GoWunCI_shxTU!6bB5?aAwKiX<+JS7qxn(4Ryg8A*^e|$0| z0y^j?Kdp;breIDAOQWgv|63?`ma0P`cK59XKj!2&@76+3 zn$Y}(!NOtfV%c7-<@Ylt$euOB7#c-YbH8T%zKLy)4l7K^W%eI#qs@H0FOr9GqBSQ4 z9ON_YvmqvT(yUf^Pintl;d{ioBekKP8a-6lZr#ClE){8Vxuw*2!3G5%w!)LO|Mr#Y z7~ZMsj`y;>dk4|P$x-y+?>hDkIO2il=0k%$waLR5`xVtIyH+6<#DUqj#Fu=nbxO9p z@YkO7@Yr99nl*@BYuV`cGd?QBq`;=>)XkYp{P4;Ozb#Gio^UeGA}KofoOl#If?RLR z$nw<+m#fLAdDA#(*=V|Sw*eo$Ur`o))l%0Q+QzUDf`h5|OvOczK|JhWI-t0q&oA@@}U1+_d z2lRB0fEQ__ftls<8n0KOReVLk+6!dC+Ie{D*uiW#rSR;2a#FA5bk6QB&4_4i5nJ~0 zvlSiqV#_3|w5^OBndbq252zy}_lzbVzQ*u=uKuFCno-S@I*+?U$Pt>9wu*Eg)9WPKe_rk;l0}$N!iI4AAf{opc~pIpq4vk$Z4<#m^+} za-p5^Ingz;Dd$_Tk-!qVWDy%i{>0x+jaAmQIr)nxKGU7NNfmnbigwl5L9f>A6ypk5 zbh`ajg1!jb`@1AOR3I1Gc*e*)19(RPcWcCd^XW5tbA{G5v-G7&&cC$7%S!NK1)pN? zwRu(f!~=#oMaM^T7|g_TW7pH~!v$rlUsh%KEn}WlbgGsIBtraj$eo#SFHkaSm=6*9N==1N#Mz99=QrM7Oh(~P)55>_x&xR(~2*V%T?4dzs;h0 zmD_Bt34P)gv-@y1pT6ej4DI5O^b9sS-7I_@MNgk)Sd4r;FH@7SRl>ZZ*L(C7`sUoo z}VZ?vB4e4iG1dRyP;8e{b<8!A%s zCvr@Q4~EIOwug%x^YT1_jTv*D#mk2xy!6ZYbp?0~eBYIO9bRRP_>fmt>pT?ik6+Nn zlke5oCU6g3YQR$q@`TTI?k}+pv~HEm;)({YqZ#=te1F&Fx1yNt|Khr(^WM8`{1Y)i z^QM%fD<0bwSSBmKI#nK|9db>U7xigw;QB`R%<bNQ!$0Ebu34)aq@edka@iQkzvUp%AO&@P% za*0dzc>PdUX<`l62cI-0aJ*hW^bjXz92ck$Xrgh?L@XRnDy?ej%>p2yjJ;OaJ)_QNaQs2V@KMX_H@cq2kW54l2MZT-^inB{Q7 zVe7pbm9_tdsq#SHYa++Xjs!kY9j-3+==Ef}VS0u>T9EIwy4gz;E802zfd$-wy-I(Z z`ISo|8%xACC**5O0ZFA0JB{gm({VDjn}>dwRyb>&8=T6fkZ-DYwxt&QJq@nzDh(9$eQYUo(N?yA9amPB-C!yBczgs z))8`;{`+>15PM=(y-xz{$_bwBE#}(LbP+h?E2Bm=eF!jF8U5#E4tgGOl1~h=pnhD- z2hMjkeGQFVmX_6ShBcM-YLM`CkELcGvr~yR`vp8goF8}tEa*nVv(L4dI;)JQk8Tv) zmrxVa%J-}2VcXRTJ_Y{2LGWS==2zD(94Ug!;0!Bar&FdDlqTm;7w9VO7noXH&zrJV z<=%1t{O?Do@x-oSyYRe|+0{FxhZz~9wcau z2D{R_ZbC<=t%!^0_nB|FK2t^qEBS8_0ZAJ?mSiL{n3*tE@i1*US-cus8n<9TQwucPETyeMOyq^YTlzZrg++;I}>vCe90r4dXuYJlr2SmWa+ z3osMC{bwU^FDgCmF7JzeB#O5>YQb0)>I0l1)rfnPaf%PjH6g3!Fx+E2>T>mcwC3#4 zth(pZ-tO2t%SDfzHN^Vr(r(fJMK|mHLr#Xf-FlRXU@&A>orIp0KyJyS_qWie${EVk z2!?%4kKp*6RMob0_5aay-ElR4|37I;Mp9BpMnq+l>Ymq$Qp(;Wdv8AW9xWmw6-uRK z79phWc^%0tD>AY|_RJtG@KdyCUm8Q)Rb4(fMV~*-Jum*o{ngTX)bQSnEcpdP~I$22fHlZ51(V9pj-nD%3eOkehovZF&l7!2fTMV567 z{%Kn^SkrtA`Ccp_ynDwA@p^0a;wJmQz?{=$zbUPzWp)@U! zPJ<7V6Wj(4x`>C0XR+7gy^(Y%>B?4BT9xul(nGZ*VKKW1>qzt3GR0+&9;KmL(~%n| z!WGw^oN$&A2cV4+o&CFV42G}1j-;c>d#-;6z+4oLwxHas=nY>X(W)w5yTaY_)tSR zp`bhh2|I!O6dWopLv_8XHt$coc|hCFw4>*(eG~YslC`{lTZ}ee%X%RNq4J zkLOyYJ%eXdVo_L+AOiaVj{Mp!jXu43>sd(KvjP}r!7ZK*tK8bFp*`8CLNb7|m-+nXdFNt?2 zK&96tB+smPUPD}o6-~>LybhRG2kBKi84tc2!oRdl#v2Eop!Yg!C>gm=TQbs~&-^=$ z7e0x=$ptOB&yacO6j7nQvM>R=AHN4_$*YCeq*XYrCY`BrIMn0kWctiK#F3AGAz0$K zTtUAF?QLtz)q~$L`XfGR)4-=P8djKD%k@^CoSi@nJ9$g{B6J+sKRwM>zAA^jQ$^@E z(?AX?+l}k3Xz$UFp>RyleMIGxS%+Q%7x_1cRzCGUJ z<_i^Y)o&4coX=-dy4uO#1Bat$iX*oF{$9Ja`E!_QzY!+h%Y;R!kzE59vhSbizKDYZ zX`kaWx_>GIw`jkC^})rs>hTv?{&6M${&|it57K}z-iU+Wrm!d78}P4tcc?kz=G77D z^|0FENi{1?5Iz7;?5m zW9u5kg6D_IfY!62%>RTe^LK@2%LmD8ucI{fV>*k4rzIxSXSl8<_qAcJ17Y8~5BTJ= zm29_)P7G^n%X8@L`q34=A#&M1ly{!P=w-d+~S9nYdu zW&*oqIS_YuYlK~CZ{PUIT*llp1x;rk_*(5BraW{6o8Hkmzk+qJOKfxgxaD%pJ2e0* zJ9dQ|clG&RPfIi?sYF^2Iq-`;=-1q%N&h+mMr?YZt&>;?i#uhB-?vA|l)Gt|xG)b) zAL^i*N4wR<_)Ki1I;kfEFBssy4v}($aV9=DcuvnbKPR2CcH%5+J?0$p6U?HzNep<2 zf4dgad>V3XbPVXU<=EKQln+0pBOmuqVuh6B-LBmm4fJ;6RA$5JH8oUUJSFfT1YR=c z#&hfP7p+&rvgAF$+qla7NGCppcvWv}Te*2&Q*3lO3NR#G7_N-xG><5YN}*BmM?vp_ zv)@0$Sxwf0#m^3!3k^E)q3yqm)m|F z9&j+C@h{@r8a%+EBP@aT*z%ki3;2Z5CvjuQAvV(VepM|lJa-!Yxn02P4~<~p4|?Xy z?pAX9)_Z_!BjCU1ciGQ6CenIv9JJq6r>Z834gIuDxFK|yJ%!h$^L72EhDfSWp8Vtp z7~cCU3fh(yPgoQO+>u$zUw?jTI?(9r)y{ttHhabZB4%s(C-gUC`l@lXP-vtmXEZ7&CN(>|=Hs>ymdcdAgS$Uvdtw)|(_L=CQ?m~u_tFC(%)x%4wBM&Zk(|(Rd4a>ubO`WLLYEh3? zle-g_OXw&RHqhBz&*6D{18!E&U+CH%$Jl|!oca}lFF#{Z9};lotHqMmSM?E7*mun9 zq^vP35>GADl?H2U(e|V+UzXg0ceXyj;%W|nB~I;FM{@(sGP`|X>sn2=9+ah}^}>4V zx`=s0`iK|lL0BB-E47=V#K}}!@ZOlDrM?HF8$W<>7uFo^BdPvi+=o28yCN53`>ug4 zFSfE}jhpkkD>}o<4Hja?p(widE<$#gHysO~*OJtmeAL>(+Vb^u4nnVN=$&v8bGB_` zT?2n0^$9<5>>${W$yKk-rt8g-ey5*mAGiFg-D*1)8$Pxbv%Wn?95EFRwA6>eQ<62* z!_dK|CZ8%3;kZY8h4qZ;5?#hWhR^N?z~-hUe_Ut-_ZH;94xR4&(8MC_=}-iw&Lz0~ zBRvOT$#Jx;@ff;&Tq~$gaQ^k~7*JH38-Low32W(VV?=oBc{sVRnV{=KhknPPS8@-2 zqJ0}_JU5CR?cGJToIvOK|82kt`?%_PtBEaSh}T1$uJ;)Tw_$5R5Zp`&#LImvaKLXz z88E1>%t~*`>9xdpTNJP650SXmMDEqZz<$H`>iuPpBSUaicr-J09?7r1H|7)Q4)liS+Y?<6ZQvKqPM-70+$%k7#1LDhU z-z(E`^XHxD_o^E4*G9rNQ}7w^30suWJ?8O;wF+;kHjp^3s=www9KZw7-GcB*Bii$Hk;M_FqhJiY|xBmKy?qTMj)!WN}uxpb_DkT z-vu>gxBMx1#-%5gj7@_Yd2iuz-xWX@$A?|pfJRs6!GPt?GG|9aLD!J8yIUftzP%N9 z3-u#ga@&)^;!I`%+ooZCC#u}fEz)1r$*vvg9vj4v$ICD<~dw-`hSA8FFXNxwnq7|m?4ndDRUnJh-Ki*8o zM-ON(=(|6l`t|y#^;PSBE4IG}^%VH_nF;%yBDC`c^hD}=+~&6$vs&*F#6x^{N+pJ; z`eL!+MUR__#X$WktS0V3g@4oB;&9)_|1|k^jsW2nryAn3l7tQ8_q-G6r7c)4?XwygY=L0a8OxuxU;b*mR;z` z_8-|m^}P~^-&h-;G>w`o!a+W7z-5+1(H`NXxYBbe`)0ch2>%J6eyU!8C^|D`R7erq z8}$|4{d$0NhLwyittWl^d|+F9hM>|a^ZWSI*@wM=deh@c_wD$+{TlqXs{w+uI}Y!n z3-$lF;>{-akT8({8afI-`gEvTYZ^BjUjK=PYL3BC52kDDg3oJAR&$F=o8(7kP~Tr; zNuL0y-spZte`tHAm3UOV3x|wph@Uf?%bMfDu-&R}_`)$nz8~ZXXEQFq*NWc|zAs-q zeB~rWL@k&zu9)uet>@v~&RU*pbq2;w?;?gZeZyihrh62uZ^OYkn_Z;O$(%Fd*yH|< z;ck$(^n0fVF}G`Qw;TO=(t8h3{qfkLQqa7h$8(mwGm$dKK;mrroc38N>{xkGlZ0#Tw;{| zAi7V$obPl9+oHU64u2Hl?Dcd&3od0 z0bLI?F?+$rEb6W>NHlnN0DBy~2@l@2r_T*cCGj`XdJwnKxy}9QUiYnrQjM!yw^yn+ zVD8;Q^qN#jc+g!cj3iAgJ8lVuos(SQ#t=OgJ=#oCoe>|ll%KQ?ng=sGl0NH;iZiL_ z;quZZNO&iYdmI%39}{54G(W|5l^Nj3#C3ldsV#jKRjZm+LiLYsm;k%7FDDBkt)E-=(U5i!^0nVj$rH>xV;f%{=xY~8T)~0to z-fr1vR3QAbA3_()y%<;LHnW_Q0r5sO*KSs_()l#th&MoMNnT_= zS{KKIx~9b)IzrSND`|b?5B1;~@HUKxiMJ0^@0s!|bFHMppx7I0N%z)O-$Um#3wS-F zpP;#i4H2~0^-eC7mpsMo=6AvBSu+`RW+}~C3`hMMY{2hB_}RTRZnZ6i0LL#L*PSOS zeT-XPSjc#zLZ*`@Fg9(9aKCy@>u~KTYoBB)$m;-cAiHr(S5oaDX<%l&Fh+E(ZVB6L z9^uzkiNFtZgfAH>q9U(1)wnyV_t#CzMDmfS*5uB{%Q%+q^(EW~#lzHVNSFo6Z+y<$ z2gIRtEe&OLdZq`xmY}-D-93^yaX6#?7t~W&IBp|TnswcdT%1sHLXc(v`Wbv;w?yuo zmJOTdrQju<=A5{cQ_XSu9?%#FGdHroe#F_nkKxEoA4&ZPr?CMB?q09ZpRyUX`jr|M=a{hZ+KmH z111hVio_K_btMgTY*5ue(DL<~8wbWzc|jUG+xa?+xVsRukDP z`BKhj7kV}f@l7wgD}wYs7?sBY&68$bqdYY}_;1pE^3r$3gyF}bbE1PZk*`>J=?lCN z?Qu63S6Ut6p2 zn)~Bu+?%}xjFxU=qyZUeU;d*(6ua^*o^+`JS}Zs!NDnA|uRN#1E)Ugbt7^n*3Ads5 z_H&FpfgpVfgjwW?_4$j?r-H5@8cf|UNQ2w7&1 zAHBum>gUk%V_PH~7Ua9wao3ET*c*mwe2jE7;c<7-K4~MqSUnogrJZHwL0X{qlH)#j zX@*^^38ejy{3*Qpa*OzQJ}Q4t90n0}-NCG0K2Q${(s4NDe5#;zK*g<8yGXqTtN)pB zdTpM0k+C5je~8O!^6Ade+I5Y*$Oo?j@>7Di3WILGQU6EsH%iAqnSjIOwf-h`*Y0m0?UfVJWCiu}@5% zs&lFI!}|a)Bp!lm)9L|n7PCHBE+`jLc?5H~l%d^=u^ypfD12)k57b{`eSLv%pXM{A z<;b&XD1*Y%v(eb;UNIxQqw&NuH4iGgpm_lD0_ajo_d;65DUHO5R}`L)VM=cbnlGT-1*u7 zk69P`T)VX-93)-Ri?}FO%`=;FzM0BsAhk~_>)fPmRgN<02Awt5eY3WS&_}f{@Y2Q$ zwFmYaDE}^%Ug+sCfV|ozAgmRg6Vq@_`+h(kNJ}^hIrX=~&+)G?I_ooj>~feH811Uc zH*)>&qlk!v*jBHAd@GRt$H6Zf%XmN?PbIUE0D z=URC~T}#r~?{g@(ST2V3iXflV5Giv|nncphCXZkD9wTWcykpV>C?f;%SgNkD>{1qG ziQ5@vf?Q$Gvc|iyjgJG8#?}V<=qt|x3f~DI1Z6ow`7O#)fbtGdI8NLKHxq|)oBiME z-tjWZI@(p?Zs|q3_q@FU$ZCc#zBHT7N({pz{kvk{TAX^jE6g}l7pRt)qt;Fmo(jt1 zfPRLNXM_SH2TqtHNT(BD(>Y6vZ!pRm;YDj5+2UI+nIz6n<&f3@MYWntSrdijVQs zFCADNS_j))NQYm;$CB6B4ne<-Froa6mikL+2)Y)Dr13EK(r|1!;f&JkVA6Ri+^%=M zN^21|Y3!eNfhm4_kn&`(DKzJl%?QFqMwrh@PiU$B%m}kU;gp&q&y+*r-J|oAqXfahnE@&v2Bi^{UNi0Z1P=Te z$4FahR8B;i6S~CN!=mhy@Z8lHsXnO>>Tt`M>7q%@B=GZ|z*Mfh>()N@!@L&!($ken z9}$-j9+eYc1t~qH^cnN$dJ(7>uzsaGQoU8_kN>8XV7pfJNlRLBwGQ-eNqMxU#`~HO z{38tNuW;ikXNygvXHkuy$3DkDB>z&aDtGx|9*MW3uEE)M{*p94VZcV9jE=JGpH*BJ z=x?MADB6VXJ$s|^H#1E|-c`!zcaUzBDmRr14{U2aWrqh}sJbrlChUC?$k}naRbGQQ zhI@}&4wOx47J34fuc@x3-_brbn+bP6?8J$4D`BT|dv2NpxTA&xL?l1QtGlm>{@F>~ z<=Gv`<)cBj|6MTMx)mp__=C$$Or+O<_w3?-*~0xzuq>m`v)csMlaJh|v*D(#`4aP4 zylIzotaP%+r8WI|%HZ$f+F7tL?f7k%Pa zv+nBxgon{6X7Mlz7Ct)!C_`v;DmjA84R+RAIkM#>xK zy?H7Rk%R64zn)<#=>RynIWGWKx;}-{j@Hoe)s6Z+QWJyz^wvyM;ZjWw&d@rGigCmG5k7wK~tZ(CkOdrlX8{gty> z`trbkabUE<9~SFBW_{%>d8b;XrZ6>1#+kgrBm4Cwt&^Ct+*4gYmz{pV^rYEx$R|I- zfWg4qFXn0uX>A0}m)v2n4P1*GVP???+?Y!D2ers$q21`dot%k$!F3bqoVphO7~9JG zCY~reErfqf8py|OJHUYrWBHpUTT!iDv#!;+`k6?277o<|F4L(kubuoPcGQwkJ7o*3 zbOikN;ScT^?}w{hTHv&Zt6~v-&iT~afoqKm@Mx}9o^h`YJKNBn?I~umVA*lpJ)O?y z*lEvbK4s4hPV&R1NcmKwCmq)86>CT6$}Y@}WnRj{!qmEqYCyi8)lgRR?w&()uNl$m zv-|NIO(fMEZkFd&y};T)!*t7k0>o_kde3!Tpt`km;6XtheYgu+D8t2G>)I|vljJcFXTG*)c_ivWgE78a{3tz zUvrPe-fzhp=+g5hHh#x|uZ`fh)eL#MR~Z}Y)tL`{?;_nEt$}x`^z6}n@fiDv&IGus zi$~91g&r%K$N@JesWo9#BZ6v<>vqr7KCETV%W_`eflwv`f3M-ct=_}Vjz0Y6bx%xa zIvTDW?twXFI=t@*+M_$gtg5fh4RI9B`f9MGMR)M-b_~>5I=l4YgbPCTW|Nl5Y-!DW zcs6Xb)K+UI(-sHg_j?K0Cn*}Xzc+zlA8Sy}zs#X}&@9#;jT8Dt5w5JlQO%m-?m^ki zc=I$q+VhPVa&Jq{)c^ER%{$=;?&@8LR}=Q5m)mZk)9oR?-F%twhW0)!H&JVZGxh|i zIg>T_K4tw5Yyy)Vli&?Oan{dgVCSwc%Rid)E9(q+Mv4stHd;V<@*VnTHx`lGf)rLk z=*Z6eTDlwVU(i<0xo`!kKKS->A35OSA^!V!9eHVeS8?wzeU2fPAk8!C_1jT39_npP ztpnNAZ}~LWy=mS(VCx5eb~ArFgbW$N{#onGaf8OfFtcBx<=7?|XLw87Y+!GZwz`#! zE_9Hurx{B9(P?=5W{l>}+X8qMyafL{yp633zQBxvhV#ERXS8KWjbKAeI;>qb5|-C) zjOO=*>KnZG%}5d!X%5UZfDdUy zRnTjzHNw-5^>AD-fxZFT<=+N}HE)(!U}L9l+<$3z^zLy1n)y+l;qX#8iC?1C+Gj|$ z2`159cw*gXm{(wjSGUK?;_v50LjApvd90c&%5#>R(weFF!TQhLc;=mtc;bl@@44>> zyc%u7sm8edd@s5;Ym>YboX9q176`YBv8>&xzWn)w2W;?*%WU!-Z^-`9lB>D_i*`Wx z$4s|`;90-M@FuK~samJkgf(O8^NstiilojJNd1_zP3Il5>XFp9V^p8P!dG38YE5CO zB!0r8yAxPk+fH5>djQu3chejd`>L?Vc=KD>p-UW-eG%#PvGt;Da?qs^pgzU6 zM=G=?n;hla?Q`Vbg>T?!u0}d0AI14khT_0nL$DkEwd!l-Kedv0hgUd1XG2LZ8BU6Z zJ8!#lg^h%zV)9fQPJIcV_88!}e%G)pem!ffsbm8#cL6nLZ(|+_`kwfDhn`1$AytfQ zum|7oxX7H2-Nc5*^qI-0BzRC63fE8F!?eY^a$3kfSZ)%*R=SSklSbL1;Vcid{CSVr zdU(UT&@HGq(DLIk{=jaY;w-o@5plJd38cPG|^^;b-;v%zp78PMk^4GtY)(?}12| zB23@MLIq6U(g{QQv_sqWHn5{c8ngOh z3^Y$L_x5n?^fnhhAFnU%hR=iW*}GwNmr+RXgW0-cU{srnNI#?aL(bZt0mOMZ(?aa! zSdTbp+V3Z9%IGAYkdGp~R<$6%m!4syL$nv^yz4tJ=}z7^Cm8V>U(qmLWdGQUiibWQ z33l(1uoUU*XcSfk=jnVO>S;K%Wg}Cye!FZ0s=E4e(M|C+u9+e{S`10W?+g_I&{zLlQ7}gh;qjCLu`j`D6|;TjDG(l+AqEZ#Jw;jd=Yy#?6JrE zTa9I4qwi>4p)ai~!nAa)a4jYnsdwPIwj1C2y^J`ZA)M_RLHIWwHnrl$~B4#8pkJ%cv{2kaUwEe3iG0_lX?_mc;QhWSh8r=CIxan);mb=zlx{i*k(7 z@?A2no#!S-cRCC8Ud)$PKWrekaVU;7-=g#ewwe+SXZoHK$)2adcaT0;dgJHobZJ&p z1lO}V!g%uLqyx1rr~2aiXnG#T_6V6-^pMTX(osJvW8TgMrK4h=UJ>(NSAb2;STqhl zM_6V~_2t4_tsM{abwSvOlV>{PzMvN@A}m21K7NW(zbYNWTTbjCe8-%U%_n^nq!rlu zsM~_(L%!d6A3L{-#l5r63epvt0cYc3;G?IYxZ-ejfB2>AmOUdf6|(Lc$xdYh&~TiN zWF{`0xEx5&O5!>oZNs;JYAoMw?g{l@_rTe?<8ef*IWRlfh${_49DoV+TWSWE>_Orx z8ULrDRJxkDMyT(f8#NG?>eFj^K7`FM1&wX`}v?kkGPH&H{iO3rjmFZDxLNTH9sexQC$tFBMCR)=D9HZc&M!)&IQX?8!>chIe2?+q&Zqa z7%&_voF~Bd+s15+QzIBqIFJ#hXUAL&#g**~wMw6o-i0Zv`@>y5%G`%rqBA|CX{5QC z!YsU*-vCH^qD{^EY<=vMD$Un=6$jEBxMBM#k$J5ElWaDj(UKE5aZ)!yTAEGYT7a$( z+5wFT&J3Lf#lP+%)i#@*91iKxJtQ_81ibYC=n-=ZqK?+Yraqpa>OAY=4siQ*4eN?V zGNn^In%!RvG%u*BcOS_k0P!4<{=vdC@3l(rke(J6f8N9Kd*^X@{X!u91;*h%+Bc zviP5dK=a56Co%B1f%t2C8{H~OVbA11S~tBa94XkkT#&ZY46h6P+a*((^CcQG_BgZG zv*OX^)f=GtB270D_l;Yx{18vJk5I2I2s63r3pKB?cj^J@68urVNX$4e9jHc`M|oXI z_ydGTNWM;M`O1_jEKs#U9!=E;;c9bE+8hW^fcytEcDg|Qrdy@SNiX5{@^f@%KqA=K zj|AdGVAX5EiiQnIzeUjZeE8}1Ey4UoEaBxWg-68C?Um<(p|6`N{eE&}gBEg(Q=(A* zhx8*0_?oDpdSp>2=pGRF>9DHx0m6eeFvg*VeD1qiOZdkA7$+-TMzvlO==#N?AAMOO zWvQ-TUNGeg$9D;4v-UgTY;hePHtY?=cJlynUmRaf6jy;YqMR-9E@9I0R6_wUX4BESwRJN1@R!hZ^b zxxMhn!R<&GFG$mD9D%uZ+mvRYzb(;ki&E#G6;rHRQF#?p|iHiJd9VyL!TduuC)# zq~}w1ZYmUpe<>^h;!H{YUZea_LZd5S|6c|Sxo5_y$CY*^O?es!Gmtb3I3~A8r4wka zk+@Hgc7uvK#u(bI2D*Q!uRMoP8tK@XI)tf%v_lOm6dwZBCjVv71FY)4z+VQJ6sA!v z#=^;m?d5t4kNr+FDQGU9L*Vt zN2|P4+aoK0_?{_VIJDIk$8>)IQMZ1J!KK;s4DWA(@LiHtMf$(CHQhI1@uWGNpGnU- zbkBxr-UXy<<$!0o~yQRhfge*8A=K~2o0=d*kP<=fR5$MUwcZ_}Op za!0Oy=Xd>k7&>-t(!J%HPHrm`5kSH6Q4Z2c?xA2-n`O;0t0ihIb55ym&H z@_3|~mG8kDS6?IPQy^>wgSl6#IFsg$x!$b7A6jS>1|roGf9knSX-n9GM$&Xkb$LBw zBXk`$7$3O=C_Teejp+3<;#AW(DR3iJ_u7EeV`5L=@=H9qUTH)rsRo*v2*niUOT#9=I`XCV3cgFty77JtqpK8i$oALi|y ztfd?aW~|?=#wExjRCxu;7BqUj!m&TLXDSOMe1Y~EA(UH0pwhzYRvr^ACuRzjRedcw z2HJ|+(liNC)oY(!Ejj5*@u%(uXg==Y$(gY=d54EHl}^=?#sR7U=#lUnJe#(ail0k+ z^dk*=SmhJKZ*&l&906!t@a-BsQ+(lR4ds!Hd_T5Xxf&1dx(Brm^~AK*`$_*4fx2du zrI2S6euI3vOcwEMB8{MfZzplN2ef;)oJM=7zuiaD_dsg^l%)Y-rl@zuKwLZd zADj#90#DqW_}-zBf_w|3-xKMZ-0*k(VQ}klZ`A{ucN^T9%Ap2bs*bmwJs}TdqOvO- z@4sJAeJS5f8B1+xJg2s#oLqEnRX=BUzdhv7w^8o*k}!HXd(!n4d9beJC3d0G9;aG5 zaLU4D9@ufJUs|Utpwq@2J|QFg?*ZY{qr?oIbbJ(H%^^y|-}jurcxFrS@JUjmq%S|&zC~eZ*C7dX zb&TPq?kqU}uMOUDvc(~tEqJp3OZIW}8`h#*s?al!Qin8h&Hhi)zI;AFw#S?tc zBDp>9mFFZ&joct@`%>A{?I4EE$k3#wr$M~!2yj}^la*An!j!24Fvs&fTk(B@%nH!( z@i%nE)hUtq?&EPdQu+x!7rkfqk_O3wj)`Cx)tY~7z85$DYsj>t16W>x7S{KQgqiCP zf%T6_*mlI8jjwqD$3OZ4f9S0Fm!o1Z;?8C?zxa`5IZT!hYw1YuIq`H|jisw;HL0`E zKwi}_d=T|PQ`7De+?hI_KkHE&ZZSiy#^rbG6{N6An73#J7WO%fZ==It^RD?YH*^C3 z5osjVwdfX|0AZPd4hcHA^KoD4_%0utb`Fy?Zt=VAW)`ELBWTWK)*4ea-{5@XH@?{G zDO+CO19Qfg!-L!|++)QFc(qDLdYJFTdd;@t0k~YbJIcSs$rJfaV^%9a{!^O;@st18wAma3@LYM0<{>$Q2u|Vb;e~e6rF7Cf9o| ze8YBvU)*l+p3_iv?KDP)#0SxN$KK3g)*%+4H-fe9?Zq4W7Gs7@x;!=M2Tq<{gR3>$ zHDWsU8{J8&dAw9UPC5o>qVdfjS!7So82mjJs?mlq|L-@kVAdE`CoB?P)V_^a<076q zVjWjnsZAWZIDm;*Fk zc{GmBid~wo)(tI;^Wp5v!S7hiIztbJ}%n z>^RLA2IX~zOl=3O)u~X>m~nz*2aM^sL6i8a5!*xO{`lCeQ*{j2&T$#_xuf(gJHZ#t zNr#h95%vzTk~6b);OZ3Ge{tpz^fVqOy^Zu_)6V)lv%d}66&T90_-62`*?nRBQNz)}b1wKE#uC>d4CSd2+OVE9P>_OH#kXoFU;@pU%ulnOcN(YexzHX7-S~ z_9<4%BckZ?gPhFo-@(N`fUlldN2BIcz4w!xMX42+gkRTl5o4Wa8*pDMrgSDW;-A(k`#6*o}L-OLmGX=jAQ1I`F(** z3bBQnImhtil9pKH@&)s|=&{9D@8g`@B=D?tNBr8gkv^Bcr5W*gmvCBeSuDS`SZ+@} z&GtR9#%sg(W!nrm1sz>FGgTX5KiWxSXdnYi8p*x6hhgDLGk)#MW{gSLm1A)&l+l{= z36+na*Wb^=iO$FQHD3$RYB#IgG!jbZT*XxH7^L6Bu3i_y)B8 znE$+l{?5g~ZZ4d9oE6M^g6zLOICP9IocLQqex6qb)K9YDk^!jx7=I%eECUS1zV4Zv z#sqUSPhvswLZo?AI0b5s2ye9UFSg??wpP~cmk6DX-p4^P_R^;LJG?Z=80!pW*sZo5 z_ec-Nie&+O{=7>9=tD*Ws^|)NoEL$AC7{>^Mp>OrI5KGT&az3}-{ZMN&ereMc z82@N4=5@FMJys3DyfM33Y|A}Bm>>#Yw1ySU+(Be)ZR1mg7rlG7hrAB`<*dH*w9B{wztl1dzqUDtLzWH$3paWuQF4uv>x38BldId z@;*jPw7o27jnOdbfQD*VwwzpBM4KgRG_Db_=wKN>zVL_rBE(^p+Vc29o zf8<^?i;D*J+He0_^RsVTO7n}eMUytGVO7nqp7^gu5d6X*%S1&JY&KN9j6JM2EYz zHM40CXv(%daBqBFiStS{zGbdedSpXEYu^7&Hq7!4ku-NOW>Xs6_Wy~Q8%D#yIjH!b z69@2UGg~08XMx?0qrRa%&V5n?lKm&a;y;J5Q^U{NU0-iOi;|ks6I$beZ?>XdGYk0F zF@sfKZ$dRTi@zLYj)b$AH25mpM*G$}Ufz#S4jS@LzGk>xlg2(}?1!ag7hr!iBkY;% zEvdH9M%;o2Lmc?ckfRvCT$k7TPS2uz^aW`xwUcQ-WjQ`a7ur`%^;OmT{o0RL>%r*j zFm2&}xaaB#U(YzQMfdY^+;kes*h~M!>?``bqf4&vSmCPmu`!l(9dhP~`=W7^0)(NYn`h?%T7YXYoy}*+_Z))3=JJZiQVHFqN z&l4Xr-rSOQ-0NyU7XKA+aqB(F8C~5vFcm(J*8JkXMpOzt2JLUzv=xo zAKGsu{)*KoeKR1X7FTsluPsUQ$>Qu?jP$AE7^yh)+~N0_Z*~+(6X2`Y7Q98bW}tNL z+ld{3cp7Lu)HO@u0udG7NG{jy#c#g9$le?pfsrllX-IF8Hc)*qNtAi6z$3SGV5E) zmiCzGq%+g|O@O$KTL!F!mvKXRowD7acw)-9R7Sl@95hE1-|O%e%HjYZT6o`?xZyUrp|_aBazUUG}F*YxSCNI4ln`Ux#6;xfd+V87!Xu=_}~kIQ<;< zc>SJ=r_Lm3Kf9;1`m4?hg#ikGss5l%%j&57kLq>On}V=|`h5y`z1jwk zFN^}C&@<5E&;bqgHtRUfj4y5-gv531LDWo`9UA9W(RJrSwf>|rf>3vA z6Zo|HwoqQdDAXIi-^mo+7H88QS6zjNNSaUKxR!X22kbZxq^k($Ht-fD8-RL6^a|Jx zZd2QV!V>Z<+O_V+d}Qoj&Fi^Uy(CC~2!&VY4$pvy&ii0@Qdh-IoOH5O`h>6?B9=}A z<%y!`ehI<>d8$oE&FYD?|D13&zK=6kT&ed`E2y?eyPC*W^z5x>JC37sK)W$rYxLnRQon+#6C>+`LUC22d1-Lv-7;0<bCL%42E{@4^D71KUI%?ok7MNL*t27Mkn|5>$ZX>FIfN@gsZ>UhVl!dV%c37vh<#C8JsTmMm=P8OSzg??X<_97~J!WApJ@jY&Puu z5DW?rsV|{CsHvRU)RJBN>_hmF$6Ccs2jWj~=yG42Gl^E5L1)(*$(0W_!mqRL28QfENO(qjs3Jb2!V>alV!7@W4UG{J>MelA4^IQ(rqX_#dI74BaF;X=;*#U| zzUOnMdZLF#7p%K@ABN?Z0Ijo@=7%zo{}}01QD{387HogRW`6COL;V87uV;bcxn6l0 zaPJD;$K2Qo6Y6~@jk$@H-hT#kZG!x$TvySD=01QE7qKb7HZTigf26F0Hy`u6%HL6b zqT*V7L@zt4JD*@2b7`i+5I8W>4Ngo={W_X6QW>wyC!^f@<8qF^Xr7dMwKm`|Cf%djo-rCXnh%P zOK0pgT#w{&py_^VuEs*+#wLZsAUf+8K5g9$`u%9a)gR(`jB`8-3;T6O;x+WwZWciS ztp(*K8l~%%4kKMD)cljr! zKd3xz)!gsjQ;G?ck&?cz!ZY$#ptLICY?Y29kAVvRlvdoE>&(gj!lsf8p!brl{^vmH ztQp(PIgJfd*s7du!Gk1u1cl*Mbr_wsgtDEdcxuu@Bt6W@&&s10t(5*nmBA?9rmu@- zrH?V(_m?JYv|p7U9zDxbE)6@vl;7PFc!_*v4PM#O4`1;8V7vA(t54^Ndgb^ac?so- zK-)aF>NNs`7GdPTgJRpew<5vIAF}eU!_M6`x$+3Y(g|o2HDRtH&byh)91i@4RO3il zi&PqhI9&O7o)nUd@0Yl#yoV7+Gs@@0Dxck=yg`I$wu9>R$$DrJGznkomWjLqO;z1d z4GQu$;%aJtApEV;WTYd3JPMp#vX>Dj<0s!Oj66MSIHM-aY`RTUKYvZDI70P0=@^X~ zhw?2d2Xwx%l#ynpK3f39-=KK=>W$rC_`5SGELFZ};KLgj_4ybQzo7b>&FU!S-6dt3 zNVyN~`PEk#$dqOxeIqG%lGVSaF!vGN6<)BeTh<85K1lm|sD5OQRyL?K5an4K@(?02 z@}u(fBDe7s)-mxmylj^T-uK%9Wp5taj_gx;9#YOh^I}5hl)b^{Wv{i?`3z{@QR(|E zhgT|>A}tXvX#AY=FhLoi$|fn(7^!rp_TCkHH5cT^_K`1$Ve028XNA)H*}k(xRx*Z6Ej$2nW%XgS4>&QQ=nW1 zNKdKp(tNBIswc_oRb|yf3$_F0Onj7CZD!#8mh^B6<%0P_afHfRR9@x!*pX3o&DO8& z4}=+2*)e4mFf8&B8!(Sy(dFSl{vX_Q;_=S5PN=f;?Kh$!y}l`IdpC{mc0DB4WEiNP z)>yYMLh@{^|7ts+Tuq!S$pxx+#nob(sTn6tDJIWeuRS%)fRkRAb_Ht{zbNmaGCE-~ z_Yo)#4qj4Ut(VFT@r}h-u_dsFcyf@=LO)$o8fDh!%3D$fBX*jm=1?XHG>2FMBN){x zBhSH(&XjCMqcAA@^bF>H{=za6H-L4oMo`pYGo(gF^QNOtuxYg&c*{4Vd7nw9nxcA4 zu3CE!t)iM~qqBU$mOi)M{L&mA4e5f~jYq)zm?f`lcL{Qbgn?0qDG=m6P^9*o3co5x zVA|K3yu^;q@a?ok`XmqK?w%FG_trCt z%IqmKc*7>-PUh=_tnoOXrSZvaEA@jmK$j_z?7iz-NQZtvUk9tGRJpBZE$IEYiL6-A z566D&go75hz=IhE^6~UVa@F||v~Cb319rH{-06s(hs<%*p}sWE6!3p~iZ9=OlO>(^ z7ZGj6o};^HW1Y8A09l*h2DeiyiW)6+P&$1xuu`b@$@E9{BRRAGTa6s6}=$NAY7*Q zdm~;Im`n3x^qeaD8YsT|LuJWpI-5C>-&yA*lb+X;uXBG0$KZ*0|4A<>{L~7q=JpXu zQJJu0ftTEBsVjTx-+<5c7E0G)_2u`A^<@04WLQ`G1=btWiH|x|DCqid`>J?(j8P#15NQ+ z=`?9Se5?HY&r?*`&6bzrYGBJZkHKz>7u0;IFCYI6mS+OJ)q3Ku%7?JH@E>*$uZ9Z; zILk`wdQ?A6Wy8Jq>E2ae9MMjP)u?$AKZIWv-{*IbeT7!4%){ttw9c&uC58FRWk>p7@eb-H~4&$&d=7=GZW$luJ> zdb)I-MbD6HYbwoS&Y*R#MA!iJuvJZul>SwaGkAz50jp%Qqeu_0Djb zTgcJ$WdrM&YQy5@z=7m+=vcH*`?-<1?DC_SWlYtP>bkO~CE@(HTLrBljPP{>7Xuej zZc$M~&`1Gc;OotYV zxpnCCHHX&VJi-NUeK6*a*I&UcGlL>z%RytUSHpS?OCSepp*3 z?=+Rgb!_CQE4#5`j1{ym{~t-$9gx%a##>Z0h!RRAN=AfI?|BYcWk$BFkWE&0#!D)48$Z9v0m2-w0l(^H{!2rvfI~H(^El4J3TpoLeu&7RT>zgU`1H zvA>Pq3mRWWoWWgM_;TuhTo_327`et^>`^0Jd*J~y3)?SL&4#_QaRIG?JkOG@b0zup35w@r>yg=b zBC0lz-S?U;N~C?$f7n*x4%Ht2y04?C)^`PuZ+)9tHq3;EKLvj8HXhV>ss2b#^rz>1 z@^I%gUCl7@N>S!dJLpRLOF-H)rP`#EKNwR)(GT2IGkHZ2o}5IL@&v*WL7ljdU8v z2CT93GTC>R5f_TyFMf+GUlVSd?au6TKe0sK4xQIGX6JXlK)<&C!JfHG(Ei(244C$c z@S;@Q89oNb42+kv3k_ss#Q=pPxN}D|rdsNP`Rmj8eY__uEgplXy04WR(k|lE4>xdg zu)PR+0CHI7J5bm_mA1OHZ_Vwd1JN>l5wvf*M$qwr<`eci*Qe(< z>+#@;MOghu#YXzJI9r_~$}Mf+E>=!n}|>NV0p)#Q!0Yt+1oj_d(GZQlnT zJI$_|v*>~_%{x~!xTkGM7|;g^UqJDNp{*MhI@V);by~vQZZ{BJ1}k1>ZfA-GX(sgd z=*aI5(C5=12B6PkCrLF0+DCC3!edZ4lfsU|zalpg*z=1pz%R_lxx4i7*(}s}zWQrl zrDM8V#Nhj3q41`{i9f2XVQy&)KxUo>-}U#Av=Y`n6Q{XzF$q!vR$y#3&Oll~OdIxA5SHQ2J{d4{QVTVPjQCN$v2Q2hcf}|S)q1xN6Tc%bpyTrt zHII@o22L4{#hI4_F(TJTwm8~eeqB=zsk?gt=_YtN^Z`zK@>4u7i($!$S5=>~g{Cg@ z=j${qccXh0T3-m8{}A_#J_DK6nj+N~1TU!s)88mUj@OXO23~@5SB@#};M{4iu$y@p zJDt7=W6jpd5PDYgow*lR*lSwr3)C>_ASYJPz8B?&5Vc_d4$>?E>IGJ##X4Bl(Te!O z3ce;7$`*ywRIRWE&sJ)w7TBhjZH3}Y!WkglD0?k?Drm0p+L!03YHs$v!}#aj6Qu#M zrgs6J|6E9OzKZRj{XF7m{W|I&cysA0c(>P+FASCqc=|N>cOpR99R05*J{)cNZYX8F?GPCFcZx$zGb9I>H6K} z*MwD?a-V0oe9s}Ic@cDOZ2b9Zpz4=+8ScN^%T^6~gCm1XaD|yKw+gP!Zr1zZ^}cI& z=s0sKE^Hi#H!ima!gSeWdM%)uX2yAgQPm!eou)9SnYzNs+| z%M4bUMq=o3g@gFqY>Vh_VFvWHJFQRf70>qU0i+jlBW`U${d4hn{)7-zTcB_waMkuI z?7XzPmeO!Q+`v9fyaf;QZ;KYytbl3~Ns~kN5^Fi6(n16bag*(2g1sk`bw+f_tvx~_oS~t zdLLU<+rTR3)`gIZo#DCg5*HV{V~U}ZxW44N2!F5u{OV+g;f`Zr469z;J^l`_qcmTqw-p$WAURc?frA&rABFhlvn+Mv=JwLCX{|9i~#a5 z>YS1AgdfwZE@;eQ`i0%h$X{Z{tb2^co`nV#!f$OU_FHmBkiHP4U63$YkS39rmwkeU z*0uRqGZV;;EdlaOP=8cCrN7CW-9q>1UDRJMkg!kG9Ak}M&pL^F+ATo-OZZKUhNpIx z{MYO{lJqusqzvQKn{d;?3O2UPtKwIsy-KIvL8=3x`~-cE(!;o{;|9XKBhYs0dNKWi zqte-+@P9|yJSg`Gl0k0UF~0RM%t(FZRoRfOP0(33JecMrSd%f@? zAbqbT9_Q0GY|LZR4zlMW4~cgfU$h#E3l_4jLlZ%H$Br{!ds4ktaj^1{uQzqa;LSJS z;-Su@b))faq`6r9=MdrCTIIhvaVJvkqQVH`ZYa4rkUKtaEYB{z0pz>TyJsFf|Lm=P z2K)NwLooXIqPbA9L{P!$X3!yW&^w@hA+M70|m>dY$>bEk3MG ztT*3qW+y%09fArQNmC%pj}f0zb(uQGs|UT=p%$fV)?h!sP(oYhKAK&PXRy+>G0rU{nTbi$9Sbae-62^mL{nqS6hc@7Q z$qBtUg}4=}jPPR9+lO(?e$AVpwKJd*QY14a?@Fuv^zYl9&0U zC9D(F$IP%imDY0gQQm>A5YOp)9F&d&^3qi}7@E_J5&VV~nS=EAe&iW^i7g!pH;Texq)D+OQeC7e*stBEJ*; zM&%#)cit88IA{y*Sz4bHZ{#`qF9g#4#P`dq`cTilKe~D7OY+{zm#O~-;vmwFrkpfB z3|g2*IZqhjp%re3oUgEkH1ZnafKw`y!nVD&q;+be!iVI}AH0-bB5yC!n&%?*ULNrh z<~3Xlx)lX%l5UA0ZGe=I2*N?`WONJ%UyKCuB2sxfr5&jcKwQ&8dH|!G0lqj~ z1-fn|pUF-ys>cbF82MCoX^Md3^*E^S7#2DXhCGsJ`>L}{KcT~VZOetr+kAPK=cB;< zxJMr898Owal1E~SpOgk9FN>r9&$-l`-_Nn9nmMg9$ ztk=U8t!yOqASbNQ)Q)_KO0ze663&9$8o`uppYs%td^m7lv*PD^=)3DHw0LpSOZmh{ zZOmbxXM%?M4^<6za5*RHcRY=jFCBSFlVjL?%yl?+W&|#+5yeTt|rg!_%N^VIp$99DVTQ-<`+ z)GjR^7|~P~I&Q>3(Ofzix%0xIj!KU(;sccFl9;_)q~_ct2~t7ms0U`;4U2au>&I>!xWoQxTIgKhH3z?$9|rDWnZWrIf*_y9(~u3Lh^M&^(FmWL)j)% z{U2L+f_mo=BQ68N38b+G%EF-BVi=H@5XDdWGR5hiFV93#`dMW$KsBwUT%0|5v${&N z`7GYkVP54W!Xgd)%vho!jfV!mwkypK!6H-|Ec|aV+kYdc$`4T1P^EvUeu45LHq)dgGrrv%$m3yUza3Rr zNBOfl26@YTpjzYPvy@jJTyeWR}?S(Ce`QqspSWGg0tT1C;G(=83iFTLixsU6AVfO@g-o5<-?6xgjb9RIW z+F=Dmnva3Df0HoEevydTY{!?a>4KqsX5!@=y|LN$82LEwy(TS57jG`2y*IW*YIp1Q zrRNn(h2POk{4{Mfc9;;ySAXpfrDII^+gf#{scUoEqcs`g|9-=^d$OT}SsO1;pD}Xp zr8eAs`3}5#&_hn!@hjKSBLVfC45ag>sZysr)2r{~A=1ctx*R@v6SDaSpg4UuE-o?V zGf#Ko3H1D;^|eWOe$g#_o7Yy1?Yy|%8eSk)7SMU)Don`G;<7n;c0I_3}Ki{xmo2Fj0n;d%M zS%+@#y?JZ5wshY-6AiDsV|4%Sh(5+lbeSx6}vR7cY@>KGo!DFY}>d+*Nj%_P(lX-5PaRF!r5XisL+M$u>(&#Jznth2ujv z=@B>sGkwmnS#+OZ@x;G4BVsybX0y=a%uVdNKT19d@sicsE*5lM+L(_zGUdE2M%_J) z3BLYfN`6b;-K(LrcIip?Tkm5*8#6xU$P=yQ&p%p+y!6;{RVQ&n$5IsBLlEalNMOi-T{Y(IYW=3z&i|lfns1SsWm>X zea`@%WK5>#WK8AOvr!V;kA@|~=VN|Ryl@My#-HvOD;Khx?4qfOoVc?lfA#AojJE8} z?|-88K;P7rwxx@?r>=`ExRb%2_D$uf+n%w`&2%CB*>N1&;+)r^z#Cv(^EpJ;4HxqR zR>7`Tk^DwuFWGyQ5&e7u^bo1Au)H3)m#$#JB|dPz@q8>yHs&-h@K3K3ztirzwujCL zT7!u4+5*N(GYUl^p>UcP6eG0r*p-e2hp0J^iD@Vzalg` zNAF%eJpf*`?w-NucCz7cOJ4ic6ES*v27CD5NqK2~5~lWcmZ>SL@#KI?EC}l@!#zU4 ze_36&f6o@yc}5PTb==9CX8p+fRM#1X-dQdBrKQ7{-5o?XpA;x+ze{7aw-fcOKCLlk zsQOo{>Zr|>$Dod-#^%^a10L|5yxhE<_)i+~!q?|;@iYg8Pt4DN-X%PcCr;{g;q*E9 zF|Mw*``X($__G!NvYVc7zv9Vgjxc*lO|d_57RGGe0d{r#Xs`1P+-3Pl-2T^#zmAE5 zb=j@B`-~9j)3R=!4todn9=;RlHCD1Uom$9wwGL`W7nt(A8QbBH%MDi2$VqOiaY`u6 z8hE?_qJKUWQI`e8=X%4#*7`i~H-gpRFx6W!NV^HmXcfohp^P_Ka$c+OC;L!m3@$kb z3ESI>{`YTdt{NmG{T=w4J^`m10hsXNJtI7j-j1U{;pmZ<|G~9S)3LdIGLCG02TmSZ z2~_h?JfI7Vjp{2~R*%9(4!%4!#fX2JX2i3HJ|evA4}*R$!hI*x<=xXyVPCzPvT4>r zPT#A29aDq1`l-k7S01N2wctsmW2LwIYFu&75(pdN#c3zn6Rr<5KYv`zh^i$Q#(O}m zuTx~m%#W~TNt>#FJD;(Z{ojZ4i@yW;?`PX!QABG_V~A!G?;u?xY)Ux`&L1u^!f_cf zw-6fL-OfH+O zs=|MsvCdQ$4>&D6&J0GY!Fv38J1gmDOnZccv`>c7$%-LAaqraf@7hir^Dd0ou70p`g`cErWFB*@X-`5I zwk=m*jEvO6%voV@+H*EtOFnmrGZcCyjj7L@Kr!7rSo*d(UVlCTw1ISg-FF^wm>DBn zmDL{s4nJ6ek8Fp5n%jm4t_h2~BY^Og>Z1m#x%&`dB`*{l0X3f4LjxHdPt^ds(b`*- zEp0`8-h$Sb;pp$2h3|Hx3gR^=y5C7okDrT6yP06{QezS9)Ql^AK$t0rC*+3}bD+t! z<}&u$WegkaBouEwi!Vc;2Ia80uDAC75Nm00tghy_#|WY3`@!>lSnp$f(Q4aYvECyG zA{=VadNT`2M+9r@pACd_X|?dieLdp7IA+-NGb}kBhR@;yWJpL7Z*=}MhO=bi*yB~> zp?VrWSDo$U}S4Unv-kjE6*hO`CR2#LX8dqbMG6&?-Z(aDf8+#$> z*gWv`d5C%rfpqQrA%FG}yf@ws)OGZFQLJrpvJz9joP|0~J#cBKSK61Ezk#raPwOku zY0pQb^AlT|ltBDMeNK3V|AyP6WwsCKwp)u%V+9^hY$pkqWoz1l;x=XX5?+Q%MYK+fITH%*ILy))y`~8Um(pG}LLlT#>zczh= zdJ!K_YKx=;wCie`L)3yY4EOkgFQ#=zbw0H>&^~Ur2C}_PEX2dx4!p2)Ga%iBzj{;$ zh54$6i9_T8_m-0Q9N%WNV2aP22ep@UF7$q^6JPJqK@vC8JnY1gx4#2v3#l;ch*u{W zch(n_&Y}8dgiTf4rF+jsPN*zl-=}^D)pvO_8gbHbNF2%tPvxImR=BlZ4^TD#C?t=m znj&v0EuIylO)3U8p`qchZE#BPTT9RhMbKfta{^^D=v%Uj2GhayO z7>Z*5 zk706M_CbX=@TPeRthKhn>E0e_WE)HOPL81evP5(nSq61N>q*srALmZfs`0YgdjeAr zn+vKjrO~ls<}lITy*o~*?1zCH2gx5b+u`B;u1I(!sSg-wc{H`DRN4}*)V2fCUt0U@ zH8^jQ8Is2FBL0v)dmTf?j-=slg8Wec{!w!RS0FTOAWJ z7C#ZY_tQf=@AndpW{n!?vUN(V8cbFTQz+g6fhMbgMpX_w14F2bglzp=a57I3R!iy^kzNY_VtL9q7g zoRnvP>$gWE@dJ*Y-V((hSH=&Oc@fTH^=@gPYHX!z93wvAs-OG49z(ox8C0D(q&z_4 zJ>_Y5nNu-&fFrC^Vjm58C)sGFWMw64cG7HY!V z%KblXGxAV;N@GjeV4@ASS+b6q>&@0A#Uy0w zy@s<^I#O}OlFXr;c!ghjd=0t|bPuXa@;={0 zGtC}G9)O=bWI)`u6G)FT>dQQuJFf7B#!!&1#rN^s@<@Xq;UUYKAJ55~q4EfdGsyD^ z!=`hE;zp%G)m$XC>dIe^?g&HHA41YZd_?D!Vy*QCAbv*gt2>nMLDHGn>Svh9w%A|Q zuj<+f-(<$(aY(*g>*M^PsxL{asv1SYc1GS3mUfzqN}nEZ&p=;0(<)3L&c~CK`Kfw4 z^;yG7wz~alOyY}p z()Fx-OB{?Yi00%kVSJ&zq#l4Dqh?{d4>2$=ueYiVLAe2v-{ZvV*n$#Psxu7O*#nRB zYs_V(4xcdO2<;_vosDl429!tOqIZD(tZ zKCQ20oad~Kvi}LA^UFoGebK1$9);^)+ZN&Vgl1mv<89fTEzKqImltVt@-lS)sXN*9 zGuBE2L4?gjr9&9`Nvi9Ir03s?czSP@FuRJg&4&*_@}of9%TD#O!J{`zfw+~C{t{X9 zEx5`L4(rAW<*x$#YRVA_rvzzUVf`##lpZ+`J$G%uF2?;xt9?*f8I*oh*~^2NLe|>t zsM3I342%^Mx4cA$pC|IV7?%JY8;xrj16>3B+FYPfe)xoGPnh#*t@1jka*?o)oZIv< zg9i{r*c|}my-|AU$vxqvKwKo1ZXj+X?y`{N?I|-^g-XjRUA;{+!_PJYe-|Gt{-YXk&|a8+g)MR@1k!r+u3|mq`9N#zA!ljXeSoqZ2phZy zPi?&eT0hzw@XJ{dQ*jR|7vO{wUKcE9s!U33uGk^SOM%kys`o=|+eLU9*AJjRPQjRpk?<`vCAYKH5L`;@v?&dw=d`j4dq@kQ$DCwFxP??>sPZ)O znkr*c`JrgNx=vLMk`EKwql4JBormf9939--Bpz>C9;Li!a31+$X?EchT5Zl{)5{0p z-9kf_XT_mL#!eV>JdzL74gux=E;OivA69%Ko;r^=8t2g(amTf)Pn`bJ^KgF$P%c(k zmFcC|Fu*Q>nRv-Fo9ccgl7H6B?nd`DR;H>`*nuwXD3651zf}5!*1LFloR2>Gk{HlyHFm&kZ=>`>pN)) zMVVBAMrDV_wbo#pdQBu{L!fX~C%&Jgj1Z&$8RCC=W{k88ZY~R{$`nYaGU7?_oqYig zbjbq3AZ&f|n}wUi|$C701vx@{+#Sgx!)U#P`LAf-;!37;m&%`ty9?FljPeIg+=`SlFqM0bNgYA` zI9#aQWx)=P9t}I7rYwQ{(S5Hz#rae>!SZ-w77%`@Y!oKN1S;LgNEa%91uvY;aU)*G z^1Q|@tiLg>&%6LH(;k1BgX-}npFiQ6zNTVLUs^kMy9r-hx(VLY`U4XTEAilmZDOn5 zD|EZq77qrNv*!9IMU#uVICNVJ{(a|Tk+_92u9z6KSyv3EX|%Sa_YB--zFIb&^#Iql zZ^&q%GW%sPf?3o4tp2tM!sK%;R_+nwWnOrbv8}&Z z|5yLv(du>4>Fs$ansgeL$4%E%+ft53z8qg{(?_=+jrh_}xp@Db9%dc*KzlhNd;jqf z&O5mq)?aEWH;>PTmG1%RyqQS3$mF0R{Pa8q@0{%li-x<(5AEWGuHG2huYC?qEBOe& z|MS6s*|YfX^E;t2JzuRmDhw~bEW@jpUt!mTQcdTeUEai||JTwGJua{D3fo@l^rZaDERmxjRP z-x-kBc^fNyJ%rzzU61SkahEIKBcA(E30o|WK>HvmUiMqT?wB^hRFidb+>*NTY48>q zwCx_Axtt>#Of{hS3B~%eTFIirTk-s$Me5vmpA%g$<0-u(5OfawZ+c?Z=U!Z=|2Pbl z&T{g(hCIe7Q0^H&K^+Tw)}gigzL&7>(Fft#n@H?_(~zHAk?tNYQC6-_aG8v{TS@1^A*+D>lHSUR6qRMisv}iCmhR;{zChvx5O0}6FK^cv1}H! zPWJfdB>kgD%Zm>TRt7@E05iH^pNo)KYyzHVcVDLtq$+u^&fvSb_t(;ueQ|f zl?Ud3Ua-$6{=m(UI`XToD`-7kc)xf1dEs9>*+}1cszTo1p55-*Sr8(!B6X?xXqBkF9yc z^k&$n%TSoqvkk{v_s}taEXMYV%bSyGAgJ!)-;dR@IQNrAzgsPC|Jp|0?s1#7D2iY< zHvhrnPFXoo&cDI#!&%(DekVjIpU{btRnf?*rc*zhT0}Hmd%-Zg!$~Dm`Mvy)Or_AjpyrX=f~V&eg@mrS(+};(|H$ z-0J6RvHcY7n-@4m_PJIQ4j9Lx^U&$6V~-NtWj$J+-#U;bpV5Iff1HocZO5K4c^R!+{WA|!Lx)O}W{t&ts}P_*=3C>p;u-&9Xe8RpHq~d~ z*SzZ(0Au*~to<0}yaX5Sb;Ip{Izo)mBTP6H1=?eEQ3TB9P5d^%^}koJ&I}zs_P8I6 z&rI{m`g{Sb2E|}fQnGM&30Gq;3De}SdZ~0DVTfGR#FmGDeJ<2>U)5hM-RhV!cmEJM z)bu-^D>9SMW1qsr;xiblmjl-y`UJwU?H9qB&7eayt%?Bd7Vw_N{3>z&v4VPTaTWd_wyTaTy`hX8{t$<>TLV{I*5TNH%_P;1w*I3Zcp^L#+f;uG zs(ue!Oyt*!I;ry&6*uWQuoin!VaKfvBOud6!mm~D$rCi>L*5(`16R32Kye*z6i{8V zkQzKs-&!u(wHosM>|jv(QoK5v_Is&3NigPT=i*VlL^ppM-p1C*#Qp{9rjMSMy=(9p-A)2;<9PID*uKu_j05dndP8(1KeHo`Rdia z*l$mN8CorjO`i6HnUyt?Z#zAseYB!f4HNDj0yQVuX};7WW>wetw_R5mxyuU!iW7vw z=nwyNXpguk8mFG3xQDZRyvh_h2fY)4xB#{!cbC+=xU+>b4;k_wb}~&8{ikh!pm8ah zEfIgj+Zxx|#Nuu;q4YPlZ=F$vAG7A!$mOZ-IBFO@_ts$}^!T_3HoYifgw1l#fliu- z!I}l<`jsg{hAcF%yF)-I8oZ|o2G5~cLNJKSIk~d+kq(-$8hqwE%+h0 z44EavCGhBJ?K_yfnfGHT{c{1f~aUK%>dHPzZ7;TL}YZNLYXn5(*h zo+Ek+;*P3*-kbc5y~BQJZccj<1WZPw^&9ztQ^(M9eGlI8wmt+k=m8IJ#?tYp0%;|X zYlahMn$fyqms#S#Ny4p8G=A-A1nT(O50}}6vGsYePG5fHYFHH~zt|S8y|v-4IMUe< zs4vu51L+M${3r?+uch-y!}(vcfiP0WYVVPjnMM1VcgKZeZh_0DZQS))6PWfyE4*9Z z#~%m(Q+z9K#~xK&s4@8*3zhpLaLlR~pn8aUUvV~nF*sd2dw(mY`Mw=^oqQZq;w`Qv z91`E#SV78#GNd|#r%s)T+jg+Gt2(M)sKVQS+n!_clu^+3>>4B-gwP*j<%E92<;k># zOySJ1s3#CIM3>!sHX5%-S7Sb_Rw8jTQy52e3umk>BxxEbcX=l^ZGWc?UUC+W=6gZM z9{o7!7-($roH#oT(}&h!=Uc93B|$!1X%W&0Y}Y;u8PxtEHcrZrEZd8@fO=FWpIITc6}Dn(EQwQ*xQ~v7thcET zs~eD`w611Y)Hvw#D-`Yay~*4E=e-)|Dvv?v}*tm8Y@cdgyIM@&f!jJ?NR-vy6I|6W?>36vrMdehUthj5y5`DBf98&|jJy@W`8<*Z~Pw*x2*+ zq{7ffa|2lA{%SzD%DXzQW!48?5>}Thzr@{i&R}tmy<(}R4ibm)bq?1UVX$6ul$qn)7vj*@yv*w7)d())`1!Vt{U%SC`o_{EOi&K^tem2F+J+SpQRe zTUMW!*i2VGSJtb(7K`awy!4?7z*@H9fdbo@ zxKPx8v@n-&2ew=)g?Vc&fbc^o+_ne|l!K3N!$BX1g5L5AOQAiGT0w?!W zB-~})hHb?c*Ag|`|KuyZ2t%JoC@e&pQ%2)U+LG1*w;Kbs77f7Xw>ei_Li!o1wP-~1 z8YPq$*zN11ye3O?9fV5fHer|YZg$ec@UIi{_QhXJMr+T7TKO)mb&;{HUljX95CZX8n{OXxhZs?s-`NQI-Ofp-9Eir2mlg77{M!OurY$ z8_g{f!wvm#^0HHmv?F?*HYLr`o>e<^RK)hGt!hg1;_MfUn|?=)H({c?pz#(_mBvV# zkzKqf;M3qTxb?st*|VEWcPBlz3MC=GY0X=_d%8RtCA$IL++ zDPIEe84GBBGa3C{IJr0B`F=ZqxEm?Y(5lRddWlg+#8sX^I+$uLP^7uuptp)%A!%kW zIv=cTa8>0TnnkoWE#(yCTbn|`jX)%fmE?)BUiF%A%c&Nx8_s%5;?Y%LuPfJPp2)l6g8KVqL^`xBe z&{W3ibxIeT$PP@Hu1BfKFC1IT7YUERsKU+ zBO{y=q~{>s&5j?fo`}a9w}ap3?UA@%)qGX2PM)@qDIH3g6uuc&%4l5G81slGemG)T zEZ#jq`l!H;llDd$3-aXoxIFb3l8>ePM8)i1rYjHzYu$9ZG4j`-`Q8X4hmX;Ad3;BZ zeiF5e4LRXABmUBiDLDz{dxnF`A4%8c5e{;~YgXO*rl4^}${2u_(WK+MD?A4B;aB^UJMKse;3 zyfWodQt5Wm9&+}$flBkT0RvL;32sM~Stxy~yeVZ#NS+u@upxpn2%!3csT2BP{oGYv z_vk%Bt!KJ8?E9VGDK@8je5bW@>h{Nftu~X_|HTw;uT7oEHf;~43s-|sWiO^_rYchd_f3~I#j(1ScbIVXH;(y_(SC*- zd5_xsj8(k0#HK>*|1%Rlw6l-~ZiRyCf>{JU(kO3#W|I@Dyqr8WQuZezcjdv%%u=9Q zQFy_OP4xM`G%KD|EeM+C<}%V^!Y{8rDzCV|+XZO#`jSR{|I-H(#mz~dvD_sX$&)IK zL(diW1dStX29;{OT9>L@7Rl?wCeT!D|UmaDeZ zA`fM(G_@qX#VIcrghe8pdsF|dW=ivtkB1vy7l`Va_u=KZ5u($TbTK(D0h-f0CbP!7 z;r{%CaMEZk=q_2t&ezNY^ITKeEy`Bb(d{Jt?QcV?#J(^(#amum<4t)mJu6qSK5zD* zzidqMMp!#Z58Ag7xn`$5SeuHra)kK-oITu#NBcEJ$G}};T}oSiVk6}lvDeTz+?8MR zs|CZGt%jnqebDlOoBa8zp*-p{i9g-mQM=_zH1hIDc;qvjHxJ#7aW-$nru5dbc)X*wi+QtT{qgF6p~S{w~QEt-puVwx4=|9uezla9*0ZFe z{R^nEC<2-#>A>pk0T`t-1(Fvwm%W@@e@uiovwAe{}0b) z^#W7>5YDqga*v*>==gRM9$=Pdka zs{z3g-Kgn(dL&%wm7(7{P=VR(hM{_w#5$IO>6KiAYU-v<|mp5 zyU68MyO5cf%H*uuK=aK_?$%Mq!Ir!KlX1J21C1BIpWXoCHkQI`?MZORABrP&zT+X^ z2I9H55zjg`K)PMOiC2et$q1jX?A5Tgvg?iiWVe#}a@(gz*!#t1100va=FPuh`4)E>^{ta^ka82t>|Mo*ViX0Be!_8cb8*6El%#V+ zsxe7(#`7wSaM7MPn#({z$AdkyW^>?m*-md;O*`#ESW*8IXX#D(`w&2ks49yqN&+8ac-SK73 z=h8c7{TYo7OK~d)oobDEttQ>E&7|#`O207t@u?fkY`Y(ijo*q1UOU-+*TZ7(n~@mm z#ZRK?gZGP zCfq!@vADZ#0{%*jVKaKK($)-FtR3lLEM0nUl1@rXt5$nNsU;4LyN{W#>kTngc12EeBD!$>tC zt4~`AgfEZq!s^OWdwX>`|wN9Jyp1w^?2Sn!D8`eO@MQIfUs&rEK5G9r(vv zSEhG8&2AMH;jY=KjCu&u47A`~Gk_l%bB2Ae-wV^yxg>rdEFYloQZ8%$SvzT}J1?_} z0K=_oLDgldnZ08${Y%;Or#p? zX~CwF<^ieJAn^Ql7Nt`zTHP?>sxJ=~yULP^|KQS~$FOKmZTTj1BaU6Jh2aOcdO2?K zr}{nv310Smc497iJ$IFnZFkE@x;pZ8r<=I!`6gPo@v3~X?Oz< z&G6d2C3myca5WCSeq~6)QvT!5Tj3b!0C#Ku0$=BmkU3=rQq9Rz`Tsv(`hIcO-(5a# zy9ygU9wnYS4#eJX7UGPW#aMFuJzO)pNu1{(7c9@hYX%uG<4QxEW8MV9ru>Gt(R9!8 z+Y~{(q^=n%Hfo7$x&vW#Zk}@<%=HKZ;z1bFI|hG_HG@=An-AL74-uKJOsVxJu$4Uj2i$m~-}o7?*hpG*%v9WKp5;0pI^D z0@a69TktQ%98Uc#h5cTev{ZN6U-fsxy}!%)qBH=D2sOkHQ`gG6GrnQFd3J(05XbnnlQ$=7f%+b(H=x34A@s1#(u^=~ zfy7-3qcK9)1`BCDCE^vrwjk!1_6wJ0#{=;$FC~wik8G$}J1LB9 zJ7}n|35&-kiGmRm(BQm=j&BnZgHBmiuk||Y zezq?>&S?nMGe=?RwU1yPoX)7%X})&TKBJCUD@X_QW)H@aZ~L*=ykzXyBnLi+9fQ@w z|KlT$Ic3U4k>kd-0%u0lT-bj`ZtF`%CnW@uc6= zv)TKxi3#nK{$K^Z4{5JS_!-2W^>X8?CO>Ym;7w1{vwjYnsK(4N$~(16f6)AhdQk^x zo_-+xp80R-D!Yp&Qeo`4J;|CwzMG+F^9)cpG-lXHsIcjW8CG@p-P(&~?rdAUkduPM zh3uT?7+g~HjCE;d&bLIm@D8*V{pv(F@bRc64>X^k`VO1JWcpn?n8vl1#805K&B9-e zc(VpcG~Wi|X`&_jo8c*^Wxc?N$TmPdsSU{(hY9)FAkORY8J6>u4$-W7V2N2d>6%*Q zbun?OGb4WVq`C9Rd-)fJK^boFzaaZOF2UjBbD>swA`X7`Md>wE{lCDV2NDm97a1qu z51UUl+EA{ESeoZ{e;M1nBND#FddMSVy2<=qAJMkOX)rr|Tsy#|I^j9J>$xKxe%*PA zv3sKE8TV(<)b|Y1^@xkFT==fze~~l~OL}gBS2E6tTIs{{(xdm_4@hUM*dLzNm{0h0 zNUO$~=7;p>5>YsYW8FEre6%g?YcT!2wVZ}8o+ z`#}1a)(79E^a`$bFVZ&JYCu|ZEv~NbBYV_|LegGH=MMFqr$SL|H#VN`L3a+_O3xhR z5&y(u#jdBc*STQ3zU4CF9SrOig$+^$Lu$fzq@I!APjN8+%#rvRi61a^jh_fh_t0p^ zl_Q-8&ggB(H~9DB)DPI*J5Lb4VRiR8Eaz-|^_nH3XdQf|AxIN?HawpN27dj( zcAgVY_1mtkwEjTarjb@*-lK~%1Yrvta~unb$BE}T)dA|#`X)-V>OZNc@QYLZDGi0x zkDTyFkZzMlBO9asD1BAGV$sA>xHzMuoO#ev&^2RlWCyOe#b|)D?2+7v56kJUB|W09 zk8uA!`o}c@)zhShkj582mY&v@mA?Xfm7^Bb0Q!woGROxF@x)2F# zNJrHttSrO}ki$yc?h;RHAWPQ_HKmJ@G#ugSV)Usv0I`p?Xdk|p_KMs9(;KXSv7IU* ztVv_8DgBjqcHw5xH@b_`bnHS-W9--{T%`3S)k4MCH+RRv1ej+z@;EY01;2X$habO8edyI|j`Zchb55c?nVfNiF3Aw8>e+ zFlKqnDo&=k1rLkSyyW;yXkdG)N?%hwNgIP_5FdRIe=Ha-iSv>813t?M(0G4@FfAS< zv)d*yx9fxPxlUiapH+Yjk~8qC^)gz|^BfRYW4~;B*|C8c{~t-$9ar=B#;MFABFRdG zL==^}=Q;My-h1y6U$VDUl!%P%LWm?;b&F-)9(*2)UEqD=RD7Q-OuNo zKmW}pcufWVlCy%dsXV#sNnjuCtmZ86dreQ#-Nz@@(JH&Mv^KcMI?nDm8}6ETttf*t zgY5ZKmTI&~fN!IX)H0b}ww6u}C{(EPi^=k9(`mOCK#@~D|R-3vu z+M81U>>}Vh-_Jf>f3+{e@L1Zd+X)f>hYaqhQX9M(^(xCXs!#PD(}lr-sY!Rm*cK^l zG)%JxE!q6u)R|4;jhxUYZ^WCJ1!B&TKr-A6Z73QITuRlg->vYN;z{@e1)acH16p&g zivqUNRVV0)P<&39wv@mv=KU3Fi%ni|E%z)5-zJTY0Ou&^0ZwSzkftW}kjL8>W~>i1 z@N7=5S63LVfEriu3uJr%dPHLFxyQg*zStrbydJ6GnGOG0>OOC4aK>u9#v0ds&{OS2 z&q85TAo&J|`i>)To9=KkSb#5hx$`s3`7zb^+8Ty`%upCF(gv+@jRa&ihTYcvekz#b%~S0lF0;~fVsQZDz>R`4V4 zcl;D|bHy_1;@XbjH4LAL3NP{*t#0xPHkbij0e~wFEX!SwoNTn>v;__tIfy_;Vfvkru?Xm1;Nb-_{CxIE zS+)9ZehI!NWIDp=k~}fjEH{hfkVOMlbM>Fqt+nlMik_WrQ?2tAu~%JcmImr_cbzD) zFUnxBrGZ)x?r_rvFH9~`claOy+$CIF;1HRU4@`ZfuD{zLOzi{rbr-_iUP@QUl&|&|H9ogrZ z^^<4LHOs0=^bLLAQyA+zlX_3HNMuZUK7e(Br`p8CuYYJ$niHlwQ;Bo!Yf&jDr6O+WVj~J*bw84on@YO+6T2 z0o`D(PTW>4D14*NwF~{m>ySCyu&I}E%w3C-~9gRE@&o<$Cw+nGE*mLo89{41I zfSs6^W7eK`MhfU8c#9Vb+K@l*VBS^sfr4%ZhR@+crTxX&OSL8Rxyj9Rhx28OR|+y1 zw5nPOld(~+`gwKg%#&)QXLi2Z6Z@HuPeiWt8CvN$A@?%589`4A=ypQRlxe-wPC(;O zHa}g3_sDH@OL8b&Rr^Vt%o+8HKxK@`dxd5>c96O1Ctf<8UcJW z`a;!ezn7sS#E7kn#r=rF0&A^-&B_M%LSby^<(gC_M=XJx;l(S#n@to$0w;-VIaV95 zEszZeWIlXmQ3-?jh6|xZ?+JK6?1}PTK(mO|ealkDf2;V&_+r}RY$+=$FtAd&wModz zCAGiR_Y_A1sVE_kH;QXC8o8c z{Ebc#G$(v}N33_G!9HGd?xq!-ONF=kvfZUe##>p2$KF>*cbv4eafnb}_?#$i>IAjE zMJbbIQSkdPQ9F1s*Z6A>J=wv`JA45?oAY`CydKZU^cY;sPO+1NjL)sS`fHXKyfFm# zdgLZO`1B!m-+N6#mkIog$&(n~9R6&L!KnojA1}w;QsL7lmS-fSk0i)y68)AOHS{y#(#>Vd=4wr%)ygX zkCG{bHBhc8QyF?#mdl6`z=o`B1m22^29)B2sO*F_qsp!i3E3O9sMkYn9B`bTJ$ok> zIMtQ>bB-)>+0pntx-`wj%(-gStB{=9D_l&9@gsO2UgNkV@WcIA1l_0}JI>%M-3qd~ zM&z0Fc4~Jzw4yX4TY)Z|PDXR#`ULvPWQhb#7-+J_KfcY$4*Mm5H%-kb;IqWKYVbiO zFjXQ)P?!sovl4O+gQF&|gFY<`tT;<-w?pjhR#cgs*S@N|d{xPo;GGyeM#xoaYz}f| z!$C#^8NZv;|Dz~-%Rw(1vXK*xY!~n^@J0{BfY2V8<9-(CW(6MA0c~8h^Yh+xt)3tI zb{b-|j=^#^-m>_d@$ArR6K&b{!~AYW=1#^x47YD4moJ}0aZ`U#+T-iuaHBW{E~U61 zk1c01uBp2hv?c$(O6pk9HqMT-qtdf)HvgspcYUPq!T{baq z?WrGo&!%3}6RCE{f4WidUHrH#UH{V_t*1od5%Neflbq7cI7Lh&n=a&WUH4^jAcToJ)SJp#aWK->$$)w^33jqnH@HarSBVS>G`q4^}G-V{jySixeoWTXY`&*@o)a|wb51N zXjgBpLfXwK3%?lS1d)&(+Ib zS#)TpRqQ&^MknVSz@sNkqWrbbiuZZliP@BX4~HWl=NZU<>evpU>1*A%&_VOzNm zpF^M8<49FM;n|L)n&{9QvoFUXWT} z7uowG!_Mvstv^1B`_`FEIUXrXRP0*4qwYNp|5#4%z85DC(|fS@Sp1P_ys35 zu$NcbF60Au`l+UyUs9tc%jC#nUOL;+-qicRRk5$+Mb&1&UMf1dk)E}1rKq@b2Gtu{ z731=jUDy9s}8ik@MhMev$fx#(gWI8+#pLtg)bm$O_@b)cpA^3}XE zVIA3prIX#-2%hfWUalXoUbVriLDDt03+z zIB&U|{(|1VTu1fK{84p-<3!uTGiB^>N7PJ|%H6vJPy6hKaZDzMHS_uCts&O;(S7); z^AJXV)5R;>>1NyRyl47FE?T0dL`~E4w|(f#Cbpm-WMi)ey5Qa_vfJ7{_$<3F`eHwQ z7&MDFKRM0gU3SoP|0~=&@|wt%J&u|+=_g12y28un{$SLn&eouo={cRR_dvPj@D6M2 z+wJU8B!}MrJCF&_2RJ0Q)7>9z}T?AD7uJGRUc5wErn)7LU zR6X13O)F1$=o$s@SvNmvrV92brz1ZN*HQ1c=qs0ZTKz{op^A&WrL|-vw@J-SaJSCOeO3Qc}+wWY@iJ;2VVFNETdjMbIYlI`RLr(a>}g9kP_?l)0Qtp_dX}teD65+ zZMg8YsT|e07(H|Rt+LPBMz3C#)_;>4@Zz~uZwc>92^^80LE9c(U=Fz6qBdF*?-v@>4osn>DMM22ggoz8Aukb0Ol4je}2XWyM_ zi5hvPQJ0c@{!;%xvw+!WY0Bj=avx`}x6QEAss~{&pkT|z-Q7`p%`o4CdF+p_l+pXI zIB3mj)pA#=r+!_{SNhqBsmOQG(YDDZ7I{@Fi9 znI5duypxKk-j#YLTIgQYnHrp}rhQ&R$E>ZD!Aa-8PpMv17kGJc$OkL9i1HjLC$G0ztmiBoi29!`E-gOHPOBDEv~O?j7>4Jh%O+FD zuy_FtBF+Z{@|^8L8=L{&TXWW#&bVfNfisG|F0bNh1q+iIS7ZARSi4A?Hm0uOWr_OW zCeIi0y#lGcq}c(}M-2WEm=6my6`iSp&misE%ZLkeRZxp4&T)7kKTaD#uD{j`@EjMy z9$$&i4^g?ROL-X8R;xDI>uDWAScH`4*%#aK*rI7_=wxp;y;lFs6uI215~KFGS>#-< za1Gq@Ul?CoXd_VzDzr-<{uTL_+dX|`9jKS`=A+*&7!$SHG?I941NEpyA^`_Qrv@Wc zOnzGpETcQ`3+nFmz8YOeD~8?=I%28J8B=2{#eZ((;N08P!goKYN_8)t$9)^m{8UDx zS9I*~`SfUnRe*cRp;iQYjBjgk%=SP*zwkQb0Y*E$v6lk9%+4j10ypxrDyQH z-oQCDbHw-ZtFR97rhlzn9uJmt7CsZjEY8}6{VexJP1ZTr;hz6D%hk{o*|kmIB7Co^ zjsEm9gqI9mOtn&6xqqorn9pZaL8gq>VQyuT57j z7dU|CVqeW#%?dyZr}2%@uY96&u*Un#`b~>iW8S5*J?^hKY568@IC<(m(>%lx*Fy62 zjgbc5X#ejF0@E=#IH>aHOksa;I70`RI?&)!@@bu2xOPqDpHH`m?CWfGmzl-&)|uH= zrGQ}!9ZTp-GVj@H(|dusP$84jM28Ju1#nLpZE+H za+zy|GeW-?(gpq&*XuScVBhUg3@wEozaA9&J&xnB*L{l!M|S`2jk$9d2X48M!SK3U zRxu@J&bj0PtWyH#+$JoA?o4_A= z|6O4{SZazpdtcypk-H>+5ZtC2EjO0PC~yK9nKeb-w{Ti zd`!+XHKQ;N=};@D24B-dM_>98vBC1H`8zK3r<5GN+)keUewYWYt|!g)_3hP)lf&y# znT-A1Ev&DIKDUtj-rGuQN&#(q$c-1AiKn5z@>+nubYaN><+AENwSSCU1KJZhQocB4 zCnG<-Q5jS7G1e37y48wW<83o$vC#y;Nz~IK0qmE+L^b2fKv^l^wK^!0G0xptI0)^+ zU2OMqxsH~s9vC0r9-QZ)M#twx&pm^NykdFKvJ`sY1LiiC!BaZk;W8QjGX)h z@U^;JoqMr*bgrHSxUaXwV1(M;)1hVX8h)G!tYYhxzr%~6yf(+%y>(?HiF0LDQwGy7%WzuJvvHo zgf!~|{baZa{?kSpPifQt7Q^$Smd*(D7Qs7$Q*ZHr?)mv+6CdT%dIn)kTyuCbXX{r8 zn&`PgeKI^1RgK#r;Bl#csnz_Y@eY9;fH42&T6kW|42$szPeQc%b9kAuu1;pdg~rPw zli*GD^ApzH_-*zWi*u*udGcS6vJ$$2_D?Dzu6)WZgU+Q`DlhVZ@9F>@VVC7S@NX6J zGlu>>{V6K>Z4`vP5Wttvarn2-RI1QHXW=ETRiFD}LyPC8QgcsmL$g=`e?)65{t%`% zk*`=^SH8-?f6n-~P4p2>gCf_LWHF-PD9Is(sni8H&MWb*o@qJ94Z3K+*mM-;vtPS+OB_In$dhJ2=_k>pg%aMMP7!UlH8TV?k9pVPk&zH%CixtW%iL_-bN@ zb=GYC;>ctL=;I)G77G8*R>JGj@rjQGbd8xCQ9tFeLXMCLj;7oXYwB+UqUc$b^Cqte zGJYrU{So1QqjMG(!50LY+$dL>%KRgT9T(i2D*FDN!TF{u>$XR2Byf~=W%D-Pfz`dc zZ0hvGkQ2hY9rkHB842u9FqojA)kVG6tp&6yw|w56eEdFIZ2EVVzJ)q7vIv4Fl<){N ztI$4rKDV&M-|SQ>P#3OwSs2axB)^}&UcC-YtkIIL6)Hzg(t!rwsSaH=KIrS5L3;R< zFV^>SR*|2tkDNJbIvXu`X!2b~w!$$VJ_)Rk^}ipnL6`%hN%g*xZRi^I4l-KC)XUhV z`M5?b)3qy;X~+Fo_WH7$JpI>`=h9JxaqHf-cLKYI5cq==2S4TeBax>=2iRagV(2iA z_I;p`x#?3bt`u@B6B;stSHGyueWtBf@1q}DfFs5$%4SU-8(qwWdcMJZP%#Wl=e~cl zlkrH#U+?H$ok|83wHj=0oLolMj%mQpPS2o8MRs$C@H^DAbWsHlug<$X#yL4oLiJ#J z740H#j`G#VC-!6HeHz{k_dDOPMx+Ol@fpEG{&1C@&BR08qm5q4;tc#By*iIjO9XyY zMvJ@}cF5xB{xj3)4)|!+C8AV?Ydx9Qd=SG_NflhdR>F6S;~8^}_9x_QMrRZBgwm^0 zEYNhukMOSz2UW*@^^wJ$4JujeF!G$-a$1MCSswi75j+RmFt_%L`wQ&c!95eL1ii%Y zVQh3S^oHEMe;;jqm=`&RALF&OY)CwxyzU8aS|KYx1^=2p%UZJu3E#o=ix8XlrSWoR_1;+5JR|#2b^61bL+(mPmir@1TVduK>(ZZ0%^2SwAML9v?ZDabFUxYf92tzPl)Lw7Dv&3s(aX;T zV$Gy@W2=W3-$(|7ksDYqUi&IAR&}%OoveHj<1oHS99uDkOnzHBa0QnPil>QJvKhZ9 zO)i@~BGlS)pb+RqWzF_opqI(KPwU8zWa`LZ^`8c8{E2AQu#7NsTH=P`nKUZ9bfZAV zE09A1a}$lGfgkf=_z3bCn_nV>Vc-BE=N8G~hX~$)r|kJGj2A-o$|gH`ai)pMoVcES zc{Z4J)%hhuUvr1`zO<@h6_XdzyQW)J>|0-veO6UjaqJC@132_zgEjE&bVA<9duzTj zxf3C$Qb|X*TGW)-tW472Ff#3+`BmIR=#72C)Z4#VDJCD{Z;b*pa%)DGOfw2I^6g6^ zPub7hwm~_)wSH%n5H$^TU@y^s8tbnO=ACj`#v3AP%lWZyvT)2~JHRi4f2PO9+$-rS z`gF3m^=Fv)xNHGGYkfrJZ4ky^ts{B*)MYY!t>(K8)TORR6Xpwy-`R~ULDzapWZffTw&n&Gk)6R~P&T(;+eh246 z<$6o4{btE+N49X`sJy!D*ROnhY+38aH7efuPNq(>9|8``USpMl)mwLcI47Q@FzmeiH~ zEvnqV1DIP+ozttlxN^x(mwB0@t2Sz;@0W7YnR$1hW?XgOUB{@vsJFbTX=T-A^dXf! z=YECxpzW8x6Xq+cK2A(a;qWyUjrF$9y;6-f#X~CZzR4}i?Vtr;7gJo0zB+$jJm-HR zT3XNfRH5KyvDdeRHJ5iqInpVa@8k^3y!Nn=zWHG--~8sO-^b==Z!wcl ztJGD z7jGuLquDzb5PFO;4*1~v>TuWD)_-4pX!H8xJR(mIUO#Sy+VQj*b)8mFE-vWHZIM-_ z-<-;qj|K3I$ouU6cmtmQ@RH$c#*3;=3(`yM6a5CyxfIvGL@4%1&fn`cJ*oSd{{5&f z|9ws4eSU5-q1$g2w!~M5*e;h?S597SrwYdeQ{?v3%BT57;T&5)x~y>{-~sjDd!F|B ztzq;!C-$F%wSCFOcJAbHMTw{94prdpW?F=uP}V6z&TUsB7{+Zkg-$KKm5 zMU#@SPCX1ZSi6kPt;aV`WULqIN;pFs{r3AY?&0e4|0>G}?VE&syfE zD_0$*w_np)J>8;An(0-U!@Vo;Z=V(GVVVDO4&BaHEvq(a?cPVgBT0#yu_0hE+Z8P= zuRnL?z&Q&o4?H?BuuO+~*XO3WE@Axwn}ZBy6Aiz7hAsUe2Mk5m8wJ}v0Glb^Tj#xY30Ih*Ox2ctR8sAooj?OHD5>n z+hp||=3X2fvXqyWwh>{mTSTWJRyMT=ER+lX-BuG?Z$>{|r^&s(p^yGzEvxGEn=^UK z)(adkt|nK=kF&u2mx{sns$FC-1AVHn6y!r0bq_l@0E{;Ow^AntE^(#o6GV z@uAJBPf9kOIB2fC=bT3`t6C7a1|D5ohCN+M=y}*jAYfcJdHD?Ulwpym(^>lUm`cFd z2uq1WIb^d!?)vkl1m4PJWWMgj)kU?LCbaefcT6Gec>#Q3s~f!@qrp#l$iv-~iu);l z-CC^22F$0*)oSRlxGLzM#|Gns+!!M77kA`$k(!@2{%O{dfX8$`DW(-)f>t1!|Xkac>ToHR5a<4h)CLxwK*etEO;PJMF-NCoyaAsF5v8W4l(Ay zvhF`;!u$X$8e3isUc4)1t+8n|3rxLq$M4S%ekaxV*uM&v1N>Q6L!XSGZl z-?Pur1s-e(HVG#G>X&j&!$#YNJk|7&G?j5G7ICtRl-|L6?3fZ zW#^Go)m=whirCVS4E}&`I5^)5wO@LQ8dDAfFIz^m(M>WIAZ=)gs4L&Ro`eJX+X)K}|!+Vv6Vitf26FsHP- ztv|41MV4+z=y||mEwYa=ucDRjAL_I1AN4=rtVg?l6~F-=?IAQUo?i|qLH{|YU@NzL z66-8i1TLq}L%uVvgHUhK26dHXaVa)^)K57|;E~+C@e~idw2|gNo~8oE?PXwrfX-6$ zpB+~BiaIdrQ&g%rOpeG|Qd6QAIJF)Fhxl0wJDJ!&kBogYm%(rJ_q~_tJ8*BBnABxQ z77jkDzER9tHi|~%T*WGIhBZASpaB^51r8=v>f8bXuR&)- zK`YG6Kw>ZK~u6ds$@VN(KhXzn@-N&-jJ95au#-b*quIB^ciYEyD zAl8KzWz$E8+WaDDGXAvoy13&oon}U1pQ-&B=#lT-J6Qp5Yb(4jixqw7jzdjtv|pRr zu|cL*q4ntWf|?TAQIyzkBdgRrq%Q38kj-76DDWAV{p1ggRtj3Vtymj!o_^ge%HV4~ z{7)QLs=AX+jTD+OBU8B^6z$vGho-uuUJdRpqYfqT{AW333pa1+(`qR}8}UWuW$lx) zS3U2T4;Z)#n*6+)I>OxZ@2P7{$*J!-&mrgx;f}qDfRU)9-V8ht1JAGLlpV`BdBqH) znZ=A*ZaT4lBQA0DD4mM_s^AZd_GjpG%2R6smA;Gp&KpMvV6!%L2;HJkpN#PdXalx! zIBG4I1AB{9ctF5$URL#-xOBL>4vd(u>|4y`h)svGG~)CbC9-r3G(6rbw=DCfD)_8Z z=7O(naeb?R*HL;#WH0EWzXG@;!5IV`p{kAMsg?B~8;;bEDMW&!3EGEj9HT{Mfgs?- z5O@+R*UGdryv6xCIvd??nR{`##`?o2giBM4r)K-Bp|56e!>1$Yp?9KaUShLVmB=N` zS~h&K6SXs)(@t#{fpgBY(M}jYLnA?>l%ba8J}8gh(|PiM>=IfQ*kdCTdII}Xt7y}A zQwB_tpVlttsH43V`c<@W8>*L8&MzyLEF&+Zy9-mVC&SZC{aNmM-~PQJd?-?J#A&%+AY*qJnNnmR)NrSl(+6)6)-W3ZJo~v@PIN}(J#bbdJ7w1 zT9^gifxu;iv8meo8Z+<}o&e8Q|J^_*!{5`CVGWG_)C+O1E%1{+Z@Nq1Fw66f!F;4g zw7?pIM}T?oMZidK#X{@+XAuf>pp6eHIHo#dU96}*-uZeF16L%p8l|LNhL+EuKhN$# zV+Uq^@5Hp1*3s^Pd~)VC_gjDQR2_r&xWH|g)23K|}aYNlmE8||isuWQIM z@6)rq>qD>GC~#yH>^W-Xzp|DG!@TAJlw5}Z_BXT zYJ4tj-WUV^>kkiFlA}*QWB3g@;N)t852DyNC#nD8s|-CUQPY-O-p`;h3Q@-%RR}z; zB2(>UW&iQ?^WsUhHl&sID!)ZR!_db0AE*+t0o0U)#sF5>%Q^eI3g|u5`2y;>G}-z- z_9%FBvRYQT89d@bgYgO&$d7}(!EqfKy32a~$r9rSpg*buTRUsRHKi`LF}!0mEj;ug z>eO$i*!AuT71t7dI9BvtHyXVY&Oc5^815usC~v=Pg`YVB?RCbCCkv}O4cuc5&R1O3 zIdWF~e>^tO>+cnCJjm#rSu6LbN|k-3eT$~T_}8^V9pO8h@RDLyIx%esdg(erGZT71 z^i4mk%(cSP;cJ`e_~--?``*!FGJ__+j?$Y+BNVhzmN$fk(T@T)lhIT)!cM9VUAJJL zui_e*32ih}l(Wgjz1Mvb*B-Wp-U$~6o9$6K+)GM$3F^JhANaAK4|hFk^Z+l)?MYiR z-6USGz}Gx&%vt!=O%`}Oqt)R7KSEPnp{rLTtwW!)sUd@j;QCzzt-_W4t(HxTzwv&# zNgQgUh4K8*s+hBIf_BQxhWq%cWqPtG@ z33$lJ6$rYE##BEkxLmA&KQ_1+H2hB?nSRqoudH8Q7MhVAcvXhibXE#=$p+7N|LP5` z(~ow(D9A?h8%+j$rr|$h@!pxdVa_YD6nm?b&mGJWNmIo*x+Ly8PO}{Dl!HS4v{uK4 z7el`X%HI!)OQX4giRSz8_9qxTz~?qxGJcrRPpW#KXY5sWK$gdYu9HXl6{Nnq@|b*u z%e<^?){cyiMqZ(8ofk0PL!JyDY4kBU4)BoPfzk?{3ZD}Xe5t91|2#{;Hy%~|J$feC zYBU5e0XffHDzfYvyxx7Wrn7+le6G%?6gSxk_1+L@H95!iCq*yk*XUVMV|pWT zx@hF;ddTp>Ydi{koLKKzAC=PQM?l0B6)OSTBrMv~fJaS1!#8gx|4L z>sHoqtkK%K+ss9l-QP=VqqB2(l!mS=iMnjT(bL!PM*qtBeGI)SzHG0kbNN)( z$QY3`R#g6>Kaf4#G4o+^3xQr!^)9>d@zJHpVCvo~9VPw^jdR0la)Y7ICz=|;xjAd0 zwZb@Peg^`t5&B#u4mt_G9Q4Z>@)L(>KGfRne8X&eLhxNGQa=#&J8mtyn|S>oNL(V~ApZKVMx zOM&C;@l54hVCttVZXa+G=U}?8g2z6f23>t_1qP%3z8T%i%d0J-r@{NwefLWSV+Gbp zb?Wy~L9;4kO~?}-T1_q!V%teHD}d*Das~>_CG?k*MlPxjwV4DQ?93)_G9Czdjo}AN zV z#;3B;Zne1qL3^3}1G#C08hE}BLtDvt5B6r|aL7rl=naC8GkTK3lWgSs*x72I~_pZ8nJ< z6k7DKFnPfGDLvu6-$Eyh6ShvLvpjsR>f6Y!xC_A_h^sg9GB84#95B?YjzB(2M+aUM zZR=f7s~RQK#Ew4DHUGr5+cBy(=Q2HKutr2|$&Fh4lj$F-&>uBS{z-#x`%uu_#g;gm zGRR;*BERj1p6g+OSCGIg1${!jR&4{m?gyq{RM7BbbhXJSCifc0$RrJqbI{yN#=8n= zaFa(PdrcsW1zb9b`en>Dxiz@hPXH$l%-yeqdkSr;R5f3%$l^7iw6bHP!C#32ZX< z)vc;Qlj8pDj*G!f0gQ3Ud|ir~`bQROrC+z4)kwz;*hgiP$XQ{Xf;I~}5Wb(gOo|Y) zVISryS5+a~Flsk+J`Y{e&+0bAU6vkvRBUWGlTK7$OrxLX)+?MU>DRfN>-?XhDYm&Q zZ{U17=ebIHp?^&^FK;~ETboC|OP)$e{%?ep*_!Iqg-bZ*)BcVIh9HmeWH4D=h9xf{|^t1_my}CNlyM4!?%-LQt|LO858zR zj4QZ}HpDj3xxBmS!)Aa@Ld{>=OQ$;Kq{i>-OFUT2r;1y}uIqDn{-EQc^5X*9a=EcCo6<;Lh=?)Q zEJRFuKKAJu4IOw|lq(*ipQLA#ZwCBPnYKO}V^&}4y%eSMzf}1@*=tL-eB`|E5$B7q zs&7|6%moHb&{vntqQ|=~tL3wI(}jq9dPa^LbfS;9>}hot>4WRbfwg{8{JDxUu4G?X za%C%hW58K%@3E5Gf4@ezO5~D-w)|9%DMjV=+nu@8w-l;T-G$C%F4afy8OX~I^ZDGF zY|>_c$$yYO+WL1WH(i#&Lyv2D7IwlZ_K8voau#)s&O1#Gxi z(T4ikXFpxqehEK4A&LJ(;`42KnT>4PHE_(*^E5x{Tgmp@96`Yz6ll;UmWnixzM9hRg23t>i&RoD=?$FjS$;~AHoc&>Dsthap^qn>1y3Qm;rccdJGI6ts#eA%-FmgF&aBp0A3fYsK6wyNFF*H{SBKQ5?Xy17@JYL9RCHxJTW_uy z8W&3gYfY3p7U$M?i#k)gTEXl%y`4B%vpEl*IZ30w)S0Sf6wkcX@EzN8;S5_4KfC zSu4h;dt{#@cLi8Qtos$JzIBl>xUr;r0sZpxBbt4-8{e9^hxT{aB!U`c)7>|ua1WnE z%Hy&+!(!FS6zq0Cx!Yhn$3aQG8a>mTr}#$BXyvDym1gUmeqTsocK zzFEa}zT-TkgPvkwzIl|GQbgZ!>uUKoX$L?0-H5)9NX7n4d^T;}@ zaNb$Mej0MqR(IPQ$r%%bT;>%)Yx}RGzok3sip55VPZ!QpuFj?T+$aw#){!FXEaFF% z?vq*M=5l3PakaLQ>r zeXZpJai)im4)@jqPk+$O1Dmz!?|viF#J{VK^8CNo>~(%4&srBCYyK`q`$EFRGU-Iw`jqiCnvY9W8CJOVz;VEU0Vw zv3_p7Ht#W=WIIE~wChQ2Jo8J`1r7E%MY+6h^O4(L`orsKvSiH!9OWgXC9Z4rA>|R88W2r+0t{;wAWtP0e?1d5|Y9-Hx*} zesDy$dR%@JK1VyTPhsw8ZGgSLH}(h>=%1u?rJ1_UuiupVs7_F88BsRh)DK3i#2H|zrr{0R5nBDSq;A%Vs0w>Xn*iY=hUmx}ZA)OiZr zU}{?|nY+mn_3()K8fEKkC!T%3B4+;yrB;QLEXrk>-gT=xqjo9d@>27@jBDW7=F1F6 zQwhh}V(!-}7Vv;-^~6Ds-F=a;W@a9_XvQiAr&FIlMYwRa;_{eVWowP=7T|bM2~8u- z8s%M?$W5;g=89uZS?T%q%!zl`^V)!DVd@vWN$X1a$imfD(AMw;L2uub%;L}`ssQs%7SJvfbEO4q9rXuO5G|a{aBxzgop`lZlyTykdGuKf-ItndejM5i?}yn#I=f zE7q}U=&J`Uohc%p)?m|XiMOf%ci*YM2REqa_a~}rpLe3yT?Fb%>?w6mY#RSo{X8*@ z3fbCb@s0Z@tJ=LkmC#E(xRs+kVvqeHpGBCuwnRkV5{bB<(r67_6YY7|kXNj9Q%1LJ zy3|yEe{hUmO|qk+I3Ky>%Jbru+pWx7_)KV-S5C&*ngR*M|xG-Q$`iMOKak{TFo_1FS(Pzx5CT;u+cKJ*DgMsHb5B7WcbuOB!>m~Zkf1k zE<>kbecD*QM+`MsCO-XH#K3YvE+=W7*+31sRLizkQCWkIn9_KdvXiKO!1x%$P z4ej*pD~0%P>1#N{@hHzem0LC++ep;KIXOpeZx%!Hl+d5M9^p^ed=O5FAmb>L4ujZO^DL37%Fn#>aB8U!{a{akV~(s=^_DbuD2plPwo^B~jq(vShwL zs^=g9ZERgxei7MqUr1OxF*edFZq(SVj0Q{b@`sMgiO+1SNN8;{KhRSJ#fV-xCHhNm zTAxmG)CTHdcZa9N)v`iU%1!I5(W3RX+Hi0Ar0f#CtE)cxNYI3OUx=+}bR!p~ee=@8 zKGa4HZR2SNuL*cKK2hCSG&oXv&h+XuMs{jMtd%%6+T zerBD}&PPIL3$u2fF&nJA@{VQnk@f8JZ9#7q9$=$c!8z*cvQ7f^D$vhr@8fy6zc!sS zd*A2KwS(z!+EbBodKS2{9qtG3#KH5*aB`K6g!!Ss1;^-IBC9&5`{}1AG~KqVY{o?B z3t8j3m5TjZZJB8kZsrHqGtj#CSszp5dfQ}3L5^G>FnsWZq`~1vfx%g7J4KHf4 zQS~0ZmcN~h;%}ZpT>8EttB&U5UVLat3P>57bv@hfwWOf2@#4?Grm}3V^6mH$ln;}&-Zuw6uQhmgziSary2kik4Kev+zM$=CH57VEz>|u6gKG zr!Vz}2xLT7_&a5^dPoj00*d<(aPIxBfF{?h1Ur0V*PCoAPozlu50+$YE zWHi!bPv9j9USRLK2NW=Z-P+_)BIb(GfWVs@lu#dKbM*{%d``c_lQ;XEFNeY_oO*4~BYHjO>h& zy9MIsELcaXZr^}YympD_+xt+X0q3dUp)1@p`2y-SkS3IiSL07AqeF%7w_8*>OzOU&g=t&l%Z z;n-7Z^}Ph@yL%nw3~fo!0c?EGf$es>RpD}S{igMq$USLQ+H7`sIF?7gIcu^FS*iO- zO0vz)#l7|*Z*#=C>E$&r#{z$-Mh5@O(ll5*5%#hoW1d9$rb&E!;%pw#`u4Ofve(WA!i zi_PjC!=J(XjaJYd653wD>+tjs@8Fq+2ylWj^=-)(WA%UUtjwA_v!kWH*bLpX%dEXb z)+C!=e_=5`>iX6Le4|DH8~tniugMLHj45G77AnS^-OAZ=dP?Mv!r!fmJbk&L{2V!l zz~cnHDt>u15$FNJdRc)PrbmpYA%o+{mDuF7<%;KHW%66&on5LV6LL#svPjerk93VQ{y{)rtKX?wd2-FHx~kZ4hsos#UYMH| zbupMtM{49EXoO6o3-)*L)SkQ--nf$T>@knPD+c4y=g9&&1j8SYHO52#c6e*DPHFPn zi*-XJ_zhlrJ?%`vXFhwTF>-Ti`}nQZ=N-;P+A)S|zdRY#eO(53?lVP1xQyfpt6b=} z@|K&=Z=~!K57W5fL#alkk6eGyb9VYFbZY+7mTDn^+~%pDp5yAM8*DC4eS5d00locX z$x(ylw?9Sb1)jGI44KKx+Fs%h^nwTXVsiC6!ZVk((WO3imEV5d=JF$ZSn`dlBcqqi zP=6=6S{7d2N`CKh;qxk6T{pS2j3|1B4o2tT{q}Jb(+Ovwj%vZ)+v>}1eIDXx?{eD; z_WFwLTWVUdh|ZtkLd)8Wu%twhH4Fzb5>|-|t03&&%AU{~|6twurdWkiQJ*=WKr9zJ}oZv zg|8f`BA0Z?!8gOO56Sp2F*sotzu6mXJ=S<1UqASp>b-K%9R@h)FH=19m~P$ljIM?F z%29jE@VlMqc0FJ1>-~_D+C<5|iH)Vpu5I*P7T|x%Nd}}Ykz=~u=FJ!EbiJ3`_+jnG zH0Or1^sGAEGWS|-xwa6_uQ?V*%bXV>Pi>_e4aubw7To4WgU)c``R|zrPaIZ@ovT_` zTslIn$~TcNyB+j}@bkR!{6Tdk$0qXH)>EdP{zEA-liAn1kGgK>!x65V3BRAKA`2g! z;LkHxILHyJp0jqa*O@0raj8#%T+a*GQ)v`W>Uu)_5<>p@dBTd<<2q+??N=LFHCsci zJZ<%=xb56%uoq2OwN!b%_osY8HX7@p2e0WQKL4F3+&`66DXofGdabQ3J>I_($7cuX zCKYbuv#Kyzc$_`e9J-5JZ!D^Ra};5&b%p=Yblu@tes3I6N{F;H5YeC{74NxE(oQAq zz4tEdp%Nt`qmYnNN{J@%p8J%R(jwZbZ$nFIYH0rM_+2dr4W5>a-wW_^eK)qNxHSwqumc~T z%Y#uZ;*ri1H{BaT{ilm`a~ZDp8pwy+{>D+ZPsMQiy40g%;pT<8Xq0^%sAn1VChOd` zCf`waG-6~i zUU+(bv#9;;6NKKGB2*7i?&WR$tAw{E{P@5ca^ueJ=sW%xIA`>SP9JJjLuJ{>%O)rao&Xv$qV#=}C*xJGG9S^ehL5?ze`f zCp_e`$D4)ek2+Y~D-Pa!#A9jh9avMsG^?7{lGy_q;L`E+(QEn-wE5f>61Q&^i~rM+ z1qS2g*VVJ-l|FX3zVS|IweURT)wIReo<^#U2zPEmozCeRsx?{5y(#Utz5z~LdxqD0 zAJ$m8n#vX9n#nfKM_}uYnqu+8SNJq-j^M>fnNnIyUD$55>#d)eT@v>?O>tc^_zF5v1DSL8I5hpp?gypMCHV zzfo_u!bcluAME`Mt_F35v9`UW@3>or~|Lthhei+YN3o4Eh~3rTPgYsX4?9?I#Wo4h=Zfjo5W7hlcZ0?{-^*bxzJd?@Z66fqYSG!j&K~Dj$B|0hc;Q( zKuwzwvU&SQ&|`LUG3*86*VSd(A=3{4aSvwpzYQDWimmKpq`G#0*)4n#2e|`*|-qXIT^FkP5s;FJL88VC)aMuq(BB?Iz z%Q*KrQq7CCw04X-x4==Sh0ENvn$B&5k>ZB!+nzy<<4M)K>~h|BxVv#DCJ#=51rrW3 z>fIdDGN>@};~8VAaO6~NdZ%RQbQm@Hk2rB{GXD8pP4+pYBZo}80@3x3RPlU`6}Rw+ zcTZVV-5M?iePg5%XrJS;wD*-B5XOm>kLoB5#xDnrkqs-ZGSZJCs-BnPI5f+gCJ29& zuj3SdPC5)p@2Q#v;vYen#CyHzEGo5gFl6{AQBa-9x#MTR>EP*_q`|qwQQw8e(N(H* z$ggP#q_ISbc_b2FgVKwA9mXQn8&i6Na9^XXZO?9Od<%rtITUZSJ5~UMql~ypo~nHo zO@Bma>34+jyGfIRB+QVFD~_V?*zdS;r-jn4giU@(wWhE_)8XAhp<;Em?_dmXU?thS zTHI^;3asN6f;CpQN2d>*bcCCXj5QN=JDc;?pQ79?mz>5U;f?w6{tFrPDf{6UtgTn^ zQnNCC98%57{gL)S`jXM#OI0U*Uwg>+e_m02ZWb4>RF{PDO6M`JmN__L?|Z1|vKdJi zfXRwoF!iY?<+7C|j0S_8i#c`NYH^yHk><+8j`OAtyBV#{H4|X+4CRFmBIZrg-i7h6a+fBkm7> zCmcS9!0Rp*u&ZbQwrp1pH-b`-&2^H?n{5EX4jf#%ijkg2r2|g)b5Pn8RQx-o{X^ac z;e4%l*mbK0w>e=an^iWc=){yN-s&JjoUg~@M=GPEBt{J)1RPA z&(shW1L-zCF*(=W^27tYcOeT1dtvr}H!;5GH56T{CTBlhg35D|SCT(?1U?FA%_t|z z^GLgobFfAGZs7EOA|oDz4jJub=9W+3p56>Td8`LnYY_KqV$Kgo)WyR-JCW)JES4+* z%2`!C5Vqs?XnOa5bpepx1iQ~Etn5s89@QWauk_XBgHpzW);IvWwy{Bb--bwffL}AH z6vxAQ$srC~phQ1f{9XAUJ_@XX#Pdj8E~@`*jH(U@^Vr-87cjWNZKUh1(nw3k<>cJ& zSsgYtYAr}}VpM$xLH-iodSpRKH8&uy1Wgy`G0Ua#)PuD+e|wJE`ksJ_za~6(+YA)6LGYcmN^855-<9*N zj;g)~g^Ruuwqms(YvlbNt*Gvopu@3~a5!Qc6c{`P(uthS{sK>R`-n8}i5ki?Oquu@ zNk@=hwBe*HwfdT)2x)!Mf6O+qHKqW<@97bCtr2U-ZXsQC8`{k;R+ZsLTmR99=k$~ByDQ(vm%9toR-VZ(>BH4k^7v0DnG z--}iYyYrq0yQsR*OS}Sry`I%Iquz##=9-HwbU+EP1yK(}k8rB-+ z!<6I{B(Kj%*AovWixK-bYO4Q?$4w86;pw?FEU2*qNGr3Jw63lB_Z^Thc#Ql7J2Zq7 znzl}UV)`B@Zg^@TCmv<=8F9}bjC|<|h>UUNk9D6TajC|4>~fYFdPC_B7_?t#hm}qM z;xa9L4x(vx<#Dh=X#_#%RMmGT*|DT)j}zXTaq4aA`%^4#SeT~TuWhiTSt4Q1Pj=Sp z2r8U;n)?awcUvGxha>5Ip}3H=8l1U;a4;eRlg%BKc7yM~-T4c>S}K0f+(iqW9SeZ8 zAk?dvuDl0pedIqJ{K|~=e_2+=H3@@DRIf3Ll{@hiDE;W3M*A&P)I-CZ)-r6w38r|W z>pOjV2JO30KI_J2Jy4oU#f5N&_AP#h(RG(;>t*iLgKdeg8Uv3M>FbsIeco7 z(hQ_!(g zRe8Iwe*2)?p)C09cLpimK>CU>b{Smio1nZ4kXJ*xetycRRCy0I7Z3{XO;$8R(kjY( zx_?-E>QukfHlV-ipddZZ0!j>MP7nZ{-VH-z_YF|{+ahqA*Mj(n_ESI8lEp^vQJ#n9 z4xhD)69?eIh_)Ki!HjaHw4zY;uI4~$?nCt;7d}3%C49~%tV7aJ?ET*!oaPBtwM99{ z4(|`4((5-bywe=n+J#5m4IwVd@aph*u!qO7-@V>3l}TdU}=_k=IJTv zdx$^Y!+%}>!RoR0YW_o?t0k4EbvSMY;dfgQ{_aADBR?7SiBue~yfpb4#o==Bt4OB2 zu8Q%jqG#nz=dhFqZYH^>das5NDJ>=WLKQQunvD@x zGdd^y#Pf2Z25ky+c^sQ9NepXs31+Jv>JCf<|=I$M^t%Z zr7P*WG~Z+*)lr(_Mx}GL)qd6Gu1=SPVa_3%)!Cx*>dr@LuO*o%!tbVF1#iO^Cj~JB zZF4BqC7scD4r}S6&uDhTiVV(+P4}Ilu-8c--en56$j7m_G>%`KR0Any+E-_HpuxXg&o6tf zJmRfQv=3QUZ8cj2#m{v+Woq8|>x&^%%b`;H7L1oo#T~aKj%~3XYPoeq!j-DoW9Yr6 z@ao?PVby*Ge()PjUTG^*ylG}PpvvEr=NhP-gPJjX!`$po9y(xsY&Nr9xDq$-@IcZq z*uB*?M!n5adFL?zFGvp>a>5%e<(rd!!c^1$*lXP=!RGh~>&0nEa|pg}rx$za zZ&l^h2w$leX28zv2Suj0DclX1NVT_$v{XseJb-!?$R`5LgZcSdjis6;)$P==YED!s zE+frmg{*ax)&!>6GwH1cCyNBtie_`{Wa#@Q0e@;9Bh^0^_E@F)#CvI4zh6z9g8{Du)9GlgH|6`=H79X?=YEcIhwHse_U>S-k?ZAZAx#y=ZMu~?5| zN)M9$Y^>r;zF;`cLcB;doxr0lid63k;$t{6usN;eP)+mYbhPOD)&c5ww{EPMA{& zq^rpD=AcJf1~gr=SIsa{&BsV@Lg?LG!dPP<-z^p1D$b<&h|&tmo5Hz}mdX>;Om33g z80&|Gmylj?gdL}~Xh|a|Ud7puE3_VmjZoupW-!urystT~vs&Jj!RERn%(!MOSKZRY!txWj!m*ErKN7^DHlo7KmcVRdtQj`WeY zNJv68=l**_E#CMay%wAzZ&)>lbqde;rsat~iCm{D^VD>(9l)%Fj64 zc?I;C6o@~auCwh)=6sq(4S7B_Pz)=kHPSw;V026{7(PXwpB2d+^3K5R?G5E$qc~{Q z*+c&4KVK|_MC>`T(7pBHMRH!8nY?D5DvmA5$F5zYP|}`*yJ`(YilKahZTX6|{=Bo7 zA?PkCg`tTl>iYQD-Sv4cbAz)#BGGliZQSqk61>v9d8%~5oGd5a{drRipl24G{qDet z7k%XAUmMWlTpIgS(L@G5yaPF|+v!29x^QS-3*P-zAuIIUDW7O)je==QfnvkI)-&Y= z)gQ2X?5=1tZHKU4pUS4i(7G8>SF#VfOoO3~Yis)8OkQjoCtS*s@p5iAEC}q1u49ss z@|NS!!yZEuZ1||of3QdMNGx(*kG+ZlVDj8mU}Reln~bR@i>IGqZ(Ikno6ASR;%Q&u zc6}FFJ7E<}>cp`50u$OFasV8u=Ec+YPDP7&H=OR(3roiY@uhbvfd07z~EgRiW4{vYT}N_W{-4QXRnI$NQww!AYFXU2ZB^Ao?y1$If1x z=`(kj?q@G{Zr%k=+pP=HGOZ5$srygpbV*^smG#i)XCvO+xv}ik?+x54StRPL>c!P{ z87WEYLZN{1ap3+mxFkzGWen*w^B3Xx;O~=q5bldPr6MXGROi3!z0& z+C0%*j~>m7C)eg9b-PH1fH(+E*drF6?t~r&Mey^;OA&Obre;)UKdxfg>6#5s3e=Np zY{GJ!>KS2Li@%!ZPlsS_%csyge;_;#Nk;Q||G~o55x8f0V|jMba_;swov9qmk9fo= z9^$m|e=I%b4?A_~66*Nv!0ohVZp82Ea#ZGXLEq19zK&LLW>e@no}8?7v@Pu_w}xvp zHM1@Q9ZN%XAT|Rc#ZC5@q%S8JPL{D3PKm(_t|9dj(9dz7r2`kG)%k>r4EA4%)7V+* z;)e2tY?k|Bap>q(@HTxV?)3lUK09N){B1oQpS{=O$5n4c$|y_JqxAv_2k?5`v*P&_ znlpIo$}4mA;Cfw8pxT9X8(W}paSgt3-FaLxE?j+v-??%ZFaOe$gXc8jRJ(AXrZIdS zswdpeXkq1-4s2xCV~pY?A12ZMJw=|9zL#f3ZpM`^2WX9mM)FzOc6{k=4L@*zJkb3g zXbN}vnKNS(_(-pj{JJ` z1EhYHC9|C6lYh5>>YTeQ`~ceY%ABEzEo90LV^z!W;?`WW8})_O9iIwc7G= zzjn~!>3mtE?;1IwdvorUc7w&gI4E;K0vb_MB|^ zb3YWf95mFN(%URA-W-ei*8AA2^MfhoqpAO{()xJuOx0dW(LWs8I27(@hH%66HKj{h zJE%6pPky`pO0>9pUl3kkA5DRby{|AiNigT?VGt(YH;AgQPVs7q&fc>#}q{biF zb6W%_yvN1QO9UNL@a=;I#SSBv&Q;t1JKG0po?r%BGKsKBb5wioaAzK&qZJF+bw}br z^jSY0sz=(&Huf##haewVO>2v<%wCKGVs!cKVk5cCSPwM|oOs)ndXo(Xp3P!x7z87wd z8i@t8XThoS+u@znMtqwxh?VasL&aD5?a!gY4#GhojzPkF^jtQGfBI3NeP6RoL)?hR zPJn!0b2j@mGfY@I9isS+k?}8D!R$k6sz11&g*^~9=8&Ec1A{FA6DA<_8XDZnuZls9 zp84?lZ~r zFf@=h$6ZC-DcUo2>>+HIe-sr?9A6ki%QE{xk+Y8M`#lw@Ur^}^&x%tJFvAXrzoF(= z26XOl>B3i3ISXl%jGF;CaVE^!^PROk7s~7ROGD}ls-xMsJt+!R&WW3}?*~PSm)^VC zuF#nfzJYLIk2`+fWCAB3wmtOv+JjTJxjE17Tf_GkDpr37BPWk}t1qf@$~j+^Ii+a17LW5q7Fv@dJI=s=4i5IBD6{E!aIev zMTZ{yfwYQDWpO|n3JIH_OMy9(uEeyxrJ!oU{)(@3jK7N%3pTBxmg?3+>0Mgu%SU|& zqgY9WC-oX7fx^b`hN)OI+ZV>w%h&GyGecf+GQ|enmSe3?o8kD8n)sl`E<2-T7~w5r$vMW1T(o30(;!o)|arp3!^)scD;zs9Cw^zEB0e8Uhe^_krL z=M~{rUKMus-+5AT6}F;h$-nixL8~N9RCxiyd~BZ`1C=WscvsydD7GbSQui#Q>&J2A z!Kfzaopv3mwDZly`_WAVu^N}?onFg4+LJ&ZNQd*Wjn>e;&0mTS*pSID*qJ_7aHNKx zupbnr+2L=dbQ5L_+=noJ8ytGO1?Gny5{90%9>d4I_~e(3(vUgC(QLp76P(si11mhX z;(_k#ut9z#P)y{9fJAXGhYSBx$HeDZ19ARI@fv^vsvtkm%2S|+Qk}kC*4VnjI8~sM#cAb#)u&6(9JhmvdXQYRKG=_Nkxty)giN|VFykJD0ACMN8 zW1ge9QD}*ztI&4(Xg0a*38swN2|j=O^LjVGYwl;(;#3z{ct(dedDH}^jc}4tM;)tr z=xVYju3xhVDjs-?cUA_Zr~Qy>07(Z3!V})fC>lwt0bK{PH@8H}3j}SB0pePvqau{F zyG^y=Oif2d=jJZ&TFH=iZF%wQd%|}50~Ts&D1SN)!0{og8EJjK(!~KvQukuXs~AxF z)OUU>-cnbGD{ZTMuKrVJXja$}^E)nuJy|A7zkt#;giD+_nbE&=FQq+hY|uaw?$erW$#rbvWI7 zrH1@BTzMa_^2Gk>y#V5TX1wDPY-v7U^*gHmpj_kBF;8&C*#j`R+yw^K&?O&3dyPbP zg9sfBDC{Dgh7TXKgIV7;ialN80?5hxd0 z_1($ut}|8tQ)c$oynA8>Z?fJp^2CgE9xJFmT2Rax)j9aS3nwn~p#F)&MBOeYyu?Tk$=>+|f_yyA*|QwTJBitI+&Jk1t@6<{yWpw6Qv`7%Q@xA|$6Vn*dEjV; zZ$KJU9{V;K)?H`_<;AVVU(;($aZ9tvb4Z$;IK{Y1J5oP_&zcVS^z%GMIF9AThd}wM zzTaCyzn3O_el0UlHI;RB6_6e!J}s=m4brq?*^RB_Z9?gD_2@a+wJ`nKe+-@+CVU$Q ziNg=#vCl^s@gbY-*i$Uu;3`N{G15qMK8LVbVJ2Zhyl{OalH^5iz}n!E?BRnWNS>LwA1_AzT_O0Z z)lP*A%#GVXacK+Q@xCEfbAoC!{xH(CaBN9mP`;XS&K1`Y58wxyHCjykp?n$A+$QJz zvv2AcptOSW@*(YpDIY1H{Mw_X`4;b^83^8C16ae{aiIE9)hZoR5Dv+U_UVa7M*^=N+Z0F6>I zM9HiA+##TmB!9ukbC6D>XJd!8z}I{e(5ympH|Ut^4TxB~ThKg@a&4u25qkPIBhQ`( zEisb+>sqmky|f3LLwn)4y`jW9K^pgjTe z-Ba$aC!O_O#g{zx7^;nnY}KNjGUGaV-`2Yn&N9j$T!?RqE(;gHC*NwI=2^t`EU`fr zQZ1>OqBwW>l!o*hqZpIdUJNIu91+jv+Njbvf zT|nN7p7H*xbTa9rBJ4h`GwE#|F|H_?=3(nFet#A`XwA7tKsL>R8gk3_D1b1fSbAoYRx*3TR2+HZou@e7I5>r-E+Yto~|RalAf3@D-^C|&63REKc%rk$8A$VMtDA;^n$^B6r2F(c zW<%4u4~YXd(feNOX(nNhiMQxJ!^SC+yd2W(F~|3vDUx1+NhfWT?ndHq&}-iXs7Cp- zvLIB~X#U{`tMmImg(rMZ)>f@=dJ{%eO>1m6(JVn+{=Lk3-dt{n~JaqmaCJShho*?CHxF9vXa zR8uKZFN2p^B;70BuiabJ84`ZZ1?P<)@L73Rd8YUuCI$z=m~q`DG^mX?Gy3zoB@tLL zBn?w2Hf8Vpv1W%Hc6!Ay*iEzffR6`Yyi*|*d(j?_y=i}rhF97Bxf7t!(Ge>4G~jji z((@$uECIP~$Pch#<4#K38_xbfk!=+$f>I+z*C>Fxtz>ZuIOPrj~MIe9ab?W!lA zGqJ?*3Ru#D+?JF`XYwuxV5moU)xGL%0L#1mP%^55`uI6C5id&A8$aV3kw zZjOd{@5{~oAY~{10TcYtRQ~A0iPn+|iQvIH_EwGfsSWVeDw~-8- z7J>u+#mF(HchNe-3cboKaTU!!nvclh;o-EdQhs;d?80a~ol}i}G4SBO^w*20bFAgh zgc7!7^c5H$(}A0mhk(YqCbux{NoxddNBe`%*;#o(ej1yKv%G4_XHWFx={!&Ejfi56 zbLu(!@|%#SvOW~Z9C_rEj^qk4{Dole?vim&>aw4V0~)^@ss zGi@z+i-Z>PMQL{|bUX|bHviOyd&aFY%My>{blD98Fm+i0EKO03q5v+hBpfrrQAmfOy9(lq~54&MiA|7@e@sxyYk zYjK4tPoV3dl>Xefc)_G%5zYLrx;^qKEcnW;h}FI9mia z@qE}#?A)mbkF~er!+wtD3wEyn&xm&XeU~FzmAA2JN1%IFw=Lp@Hpk?MG+eL9MaJ+vOeviBGg)Pp~GdX{22MB3?GXL?KR zW$b`@Jlb!w_REj;+-E>Fv>=~EHH7f79=6mrL!E2k+@!b&E)Sg~T`X$x33sL=)em3z zzLfGFjD96;;7@uC-}PyMJP}T70_aH`U6RaX>o@fIT~NQMuBuB&Q0hncOTG^R8yq$zGBO{E>IdAC!17U2g)-y z@4t-VwgKMTE$4aT#>4CsSJ_|d0Il8+hL@x4rFs8v!hAYCPqW;Z6}p!(zq4O3adM=b zwMa)eZ(P8${USBjPHcu?_j?Kh#KyQTe40UHxc!BJ<3D5A9d=oB@cL@8`{ZfiZ{1jK zew+^PqIaNg>@VnXdzUC_*H%)kiUIYC1l1g`s~4p0u;w#FhvcC@-Jd_~xEH(5$^pV6 zMt#T5PcH!%3tHQA2d(jM(S#F@K)uLmjB?8d&G-j|Wm;9sjqVJU^A0ZMw$~bBr^QS8 zU-+sp1XbQ2)u#LHZ|ATNC(LBt@)&4hvPndm?bROn-A+2z?hof0E`Y6;*797bR`nqC zzW<-5#hCA)citSv7PY`Xw`xP)MLixhAzY3M>x8>XkIA1|+j7)-J-oV!FM>h*qMj=| z)c7?T=WS>EDxEmhJ9s{hSGdX}&1|K?(=B+YqqXd{wj=hoG~wMRB1~)$ExWzF&*(bT z`H1Guxfn9ho8Q`D#@pvLl_$$O!acK0>bX_CYsZan^7~?>9HHOY#dzUlZ`OZXdpYgG zJ7~Vv5yrF%=dTV#F)v!@FQTx5>AL<9D;MP9j?6!x-{zO-@oX%&ukpJ-Q%0e_~++T%a6z%CZz zX;giF*wL1EiF4#9zkdK%z^lX^{h#8 zaEq%tkNG{lVVe1UHjD+!`0}86p{Aq7A+DeJ`S!KL4lfjEa*&s!u2x6m`O z9Q6_s!12XxFxZyF%Af1QthHTWXH+0kf2g>K^8Eht)|nu@xvHVOJin22d@+o#S+6Sz z6QREKdK`G{5PSAy1MEmWgWH}Skny*dPgtBvH2_JJa$x2071SGd@A z>s*!%&#`@R3H8<~coX;t24&7s{URwwVn}OmxnXrrd3oqlF~?yTtSBz0-j7D%KUlVF zT7&N?ISKR5r=yCO-4_!`V2$~T@)k1Z#Y&8+{YulQK>)4i^B;bE5Tvmd;}i~{iivLs z*DM)O8}-|))`-+T3QKboAAilbjb$#BfIRM4b2RKvC?}19DLP*t6HOL zA#O95rhPKOslz!`J@9x*QzRaPO|M!bemjZv0uR8?^EdH1bAr@q+i;F$EKHwx5$U>x zfA!}`wSdhoJfNPi;AsK1vDey6Rbw(vzROAWiN(a2o-%*+2vJ8ikTY&O zaKn2hP|_{~GTo9zHv@CwVPYy=Y%aUiJv$QU7@+!vG>IHEV;y|+%D~xQc2gep!9ZIN zx7VAh$++zfikA-CvKKa(*zcr z#`K9JIMouQuL_5VgswcY*(AK)u@&kZze{~FN#$Cd8(V8S5sBZ>DDyrP4r$JRZ`#c& zEd>(CfR|ek5JxjSrz1P8@r6DP*MP7{JoK}~pW%tPx-=CXz84_r0!Ufy4ewSz!4pf1 zl|JH=e9sU*?AHEz6bv!7XTYWEOBrFc=;`H;>UgBfaAW9tyig*6>P&uWybAn#N2+rG z50h}ZUrs*QXRDxNV2y>dkuU^v7mVbGA7?@MQZH@9`A#^@<|eGZvJf9=ui|Q#dWUs>SSWTVPO?HW+by|{{stKs{Jf>OM$43<0)Ri{$Of~fze--usuXfpw3k?(5v1ao^G)D8dtZj_|-EziBV=vPe)d z`lYWUsE33)FNMjW&SPXBPkkgV(Gp%`--NdE)s4+nxUqB2R*k}if%mNWmf!YxA|W4> zJRXAUmB%QSYzNt6k9_M?p>!AOosVLKnWQ^P1U~#NsOR|SQ$-k`myO519H;lXGchP2 z0Bnyx(KL?R%Q}@bs;X(*`@`{PR$aK<+g4KFX^$?7Qn8ivol0+G$&_cL1x&&0Q$I;P zi%n-81=4=FqIe!yi$@yLZv5Aynz&%@1@_~bx4bj68BFTeS98+LQRNeAOgW6-7ulij zy;joSI$lsNQN@b1htgm0a9;#;bO~UTTTM85)|(mEC_a^J@rze1)VT!N>JBn$?sfHD zTK7tGKJWV^*uB0M(%&-DxLoynWV5GGdNf>=qX|17HV8+0c#*E_4|ZR+3O;fz{@d3G z2ty=(bCt16wqU4jQzXqTiQgI3mBwn~37~$Jsi8ITxMz|&UwNjyGas6CRCy4tv?O7m z!bPQxac7SmRodIEVK?b_vLhpmku3+Wf+IEOYlFrb^M@h#*s+venjX&{YU8TS*3=7h zP){6t;Sx$aLyt-j1cdoc56n2*5RyObZQ_>kPMEa0MU{qj+v(}!5 z?ZeGEajWRHWigu+mkFfV1@Rn^9)kmRFW9Xeks3O#`djziVSTXoH5=lVquGkL0~$2L zlHqn$98Z3MG{yu@It+h&A)kXAq3?M+qQ&NW#JA0X@R}{zx>jj9c-#9hcG+JB-Q0Ugs&86zv?C|ZhKEot-AA04W=eTqp#;3NNnZ|rJ5&N>Q;erJp4ecm$9#Ce1{W?b>- z@)hTSya+VRj}e6X_}k4D7frcD+Vd8WzhvZ3*qgv{%1iJ@f4q?V5QKWw<>WOO)eWs5 zHvzNK^~Lz!n<22vcA#rvW=8a$RG(3tj>#w|*se8s-7XeLUKbkKokfZ_k`E=lJ(9tZ zSFHWdn>fGOJyE|hIEOs0P#)vd`(?N}@t+`!qem}yvl{{RSj)jb#pu^hgj=AqC=G4| z*DHPs()8l_&%YXnp2!-kZq3I#`Cu-yqctPTMZseaw)kI?cyZT2sPCp)m(+VIw?N)p z%yb^i2@`lv=X`Nav*`_%IN1zFg=G;AnM#GV^gT+$ z;(~2j{FyL@Q4dSfSK9QT0Sd2}andf#*s~GoT(DP48g`!ug0KKK99ao7ZE63<2Gw9n zuCBEFWqRFun<{-k*UE`Bisi=E2_ zeTOh99t=Hgx4^pPfjIMbA|sp=H18nYJ5Q3==h?#>bLBIrzCig#$`Kt$AM2Ec!RJ$h z!Fdpa6&4Aov=8;SyLLdcD*xFdBNe~dW<%%KcNzIrPTqqN=R?lEAV%?090v_FYgqm2 zmy!4Z5-d_w-!u9d{xUmVr4jZg(_RtdZmT+i$**j1afhLj^tm9c;FJgA=Rj0iVbVH3 zAUwp}$(B_!mnHtWg7OIAGed#=u25X~;Y4H=mJlwxkq3i~p*n(kRq0rqqcZ_ahkb!X zQ+h*cTyN+gccbE1!adFXx&JZJT0;4|^P>`R#_a>JtF%2@nfHddGXu1w-=N=ICTNDi z>!l@X>?{rVh*B$FWZRCd8D+#zgd5>V7n)nG+5zOz*wubhCC#OfYDbb@fZjTH#Fl68 zQTe@>9`i{{&s_!5{z6OGaL}0`G`taNBr0|~71-E`_1#13Ay+=OlsN!bD z?Ly72y1D0(4|=F-lP?(%DpcMGHzC;3lJwF(;(@+E*o3oFPO=Za&v9x*HY%S_wIhko zh4RE|{;+%cU@)=p2lB+277!|Eo(1Pz4rq_1#+QiOM-iY!# zQO!ZLYJNa9nsd*s4fRw#y!}NG7a4H+Jz8|HgrIJ;A3JHf5v55Ab6E27Ft~0)dvqIQ z;Xdnqq_d4B^#!)v))MJhV!ET5(wImb%G_rEsG4u7nZ)Glvp_pw7Z3*n$B8N&>RUeGP!}+Y*#gh`xLMXn~&%uX>O=6hWum` zZezQFHO`G=q&IMcJV-N$NF=W<2={UM90Ppg(~A7VTSj^o74MQ(#M-Sv3_LwkOWH@1 zxS&96-2WRl9H|644UnC?p+^SNoM}YAb7Kn@xBSjXL+QYaZ^aBmC#rkQ6JCIrom5#R@eOGyE4w zb?F8)-x8#q#riZiKDb~nJRaZxO7}P1{2B;1w4_&oJd-=kid0+~%{SE?6VArmz(V)g zRorJ~eu=GaI1cu__T^{FPoeW50i?m?)zBz5`&w)G`hlJYx$^^jI$YHrXm9|?kMI$D zKMCuMBvCQ(4Po#pO;0afn)m$>q;-(Al_35D)h84?>3OO})f`3nm$R}5_40hUb*2VC zS(XWk`)J+_#w%I^;i*>XkfUpTkTjp9oM;r!6OJIo842IjtPLA|3?-dnrFlAMH&Fjl zpZ$SXCksW)>7QV^ZjPE;Yw4Ph=0SwL0u|Si2bG5!&lKykmjUrO5MPpqH&r!-v!-A7Qnn`M$E&=g7=IpD%X~u^k z85&038{M;~l1`69nzbQn>|{W25>#J41bLbnme@6ivT6^pmV2-`khFxyKJ;Mqw0i{o z3|?=lhjYeE<@vL(G#ZIaS&Jq*W{CLZQ)O+?850T>CDYwl61^D zjw9}@#PX;0@%^gFyzR{M>}BXM>2+=xw5g#d!zw;QnfDgpQI5FRvlyDBc|*P8>bU%~ zC;C{Pg0wz#@9$ItdHwQycvsp9I_|TAOTnMnmY$R4p{ZA}q26COaPu@g-uRr2AGZ-F z3w!C56~#kV^@rMtv+-y5T2jaDE_ToE4)6XnhSXQpu*kimJmhx-7Q8KB<_}-840|2^ zcTXC->G2i1j?j~-uL6X*Lw(-2P78K9sWUWe{2Xup4dLPMU3liTKdeqKQFUx}4p+P{ zYp>VcMQdYk*Y>TGqSZh02-E7XmcJYf#b1=W-`aGoVQm3|NI#NNSza~WtCSAx4QIbx3+o7t#0 zv<}O|4ak1$upiDZuyAiM4w&Cd#@qZxAKfB`ntd|Xq?SDOBS_^$?D!lg6J{B~*fp&s z#SXTNza(sL^ptKsbzs)E`S`%(k!U;cv9_0ABk+0s4qJZzjm>XP!@8f_@bo$YV!!U= zPrclMYLq4Yj)CrFVK~7w3znL_!py&yuxkr9L3Iqr6F0%xj1X=8T}`B_g{)r=^7dbP zmubKzG&KGv<}NyjwDTmI#2(Vl9eIJBZ*x;Tl=I}2p}si2jlg-}!TnQd4K2IcSo-J! z>I78RoS2!w)p4|2>hj!Uhd9+VwEhILS!gS^;O#(g`O#J!x#!F)^i4#|F=p9w_HW}V zS9_w)!OlJP#EGWm_{6mW$4q~L{^9>&%Eu`XbfZM&7fyC?;g^!{qFy5xSy6PJPfD4@ zla9SY^M}*fjQTA_{ap#z$jcZ$wpxsDW=F_1(_8YJ^Q|cMdU!i)H6LNq3hNJ<2WQuG zx1Rju$NXRcEGUSQ}L2l<-4#3}Pei2A4e!O)|xw4XFc5`Mt2Q60GJ{a||L zWCv_nei+^zpUeJ8XMXX)Q_X`>v~Tjp2!M?CJfor`zj$^HT-8~rxz=tksJd?&-j}I7 zH-6$KzxfZvW}_$YJrh@O>Rai$bSZCTd=6t;o5H#$-q=wF@`G_Fq5NqKj$2uar?72M zJ8K}Wt(OCS{}$tet`0zb&o*_eBllaG(7h^saQ-`AcIh#WC#MF1$=aUW=xq)TAHEIh zpVF1if8T2&-bR7p$hJ}%Xshstuu%307zMKq57E}?#kj?m??^o$tcNC{iVO97Ro?fF zXb6NC;^~@de1fNrJW#EsoHM2rmzi8biZv`98^hlpKAbZ(pb48lI8b)tZv}lHtXUU^ zorfHS^llqraGnFVyz&K0zqOKtM>6NY4kVm}UVd?CQ@v2^U7X0g4t`WM&6yuUOO7WZ8Qv4_4wY688V*KiZtkYY#AMxQ~#RUmv3gx74{rCCr+ z>n%MhcHsxg_JVz%xp;n=4tLG9LmR7(^2xfr82nU6b7$=*ZTjPd+WdqDn0u_X`#jsx zl5h!H)_I1yGyLI0#DN@Ds}V)%SQ8Ec^*nP7dM!-))BT@-QEbTh8uIm~Xifbg<=}Ph zHu^QZFLv&U=6379VeP~+DBN39)`V74VMw!-t?JrvON5#Hnspip*Ld|enw+bdo9V=f=_3`uZSdb^h%@+4e{2{2%;oge&5;Bf4s!>MV#ugTMp_lPS zLGhCQ4@|1AuK}&iLN%R3xXwRB(EQqXolLrN4PFoJ#|iVrtZnbnty!K{al};@+MiW> zO1R{e;=hgdP}8=DBs{?HYfs{$oloFE_9%=!v=j(y#IgAgq2K;O;tvbZG^`F(H?S=> zfY*9|pV^g$sJ~|vNA6;_9k+fOjMV%5MALalIK-%aVCdLl_O$&_NjMKVW8VmhkGSAj zU#k?^>N6mYvAoZR_Zl6j=%4Bgq*D>gyr*lj4%L;U>NUKy)NGndIl#ad&*y5HJEwL zPpY^)t5+bh&o{-xj$Nrv9XRosbS#P$SNo+>pBk|Ki(Y~?(?G9&1375|oHipCyOpQY zbv_cy?>|KIul-Ez42$mv&UXHoBJu&a3scip%J_fZ`(9^LVsBti$8>b!3GhcA#_u^@N6S0#?VH zbHYgC^4E;GK=m9OZr4{T4v1+Tjwc3o1~L4g9Q5KEQvN8HlhEDOh*KZojCNrvhpO-K z;fKzhYP}z&I8k9aT8%ivh)X1K6DOR2z@Z&Etr!kLnctx0m`MEZq%n3Kv4Nj?X~7?u zjAOPs5rh|qMDKQ9Nbw`wEmm4kyj=VdKYyAj6`tt0jo>xL+(5CwM6SQkSK~eSEUf=l zmlIbr>On@wgSILrNk-B1vRKh$&f9AFi%?Q2aN^pbC?g4t^rOPjtlE7ms4SnOBgw6mIsO z3TN!hAZ?+98Eeb5W^J4~%^UjkJ*YII!fuV4r^TmzIIQJ#bg#J{jwjCN1(WVDrJdpI zMrd<$1l7qpxOSnthWc8R7QYii4o#Hlc4bIC17-I+%c0}u3&M7oqnCrSq8wx0PpLhY zr0a2KtJyH;$_&ih?1a>>k~EXJo-|xbT2EeFL}w(p{Dqydx=K4}jFZ}EltwHxxquJp zJN*UYj-mB$E=iBFxuL%pc@a_LOd7;}o(F30)V{=x)MunM@RxB@{>wNOsn%2tXh?h0 z8in)Ry}7v5B}DOscze*3yh{t>e^*+Y7s7A+F7|H8V@4XdN(+#GVVZwUCFu_QI_*8U zmlrFH!TA%fqks5TaX6)>B&-%|^ZMerR>|}Z$0*IBli!%>;t239Z6Ms@W1)NHThJ>$ zMdzs3K)a0$##hqeZY>@#$H_aX)-1#{i)b8jz#aFLZ3EJ!?DZfcM)k^8vcayex)x!> zAC39Nd}BV~<`xVeF-&o@m{@Mat{wY@(N9yM{y+ydb8c(F&O39T1c_eGpryS8&z&9Q zzlaQ^8h6#1s>4T5nJfq|l~&-!lOI;?4{1zE*u>g2kK#{F;^|rks2^5PZ46Qz$DD4} zLcVT4?bmDJ)wvOt-6!lkUnxH4jnz!8w+E&)=&MOOeH#;2jz!Yk`)kokKL;RjkbC7jlgj1_O$fJM_Y|+O9<`3JXO*>p_71{!rK8eJ#PdqeyV6>bw|uOf_A?DGw5u*U z-_67itt-h7^kLMWFysN9J5TAT` zLh_mlZ+J@&PmJucRy6#PB4EP>=(Y6(JZ|p@1#ty67{n4!3va?F}Dy@HNU4*1L)3YWWk=8`3G#st5HmaJv8UviU zJ%RL~9vqq0ND?;U(|t<>&5J3GWNzFM$zRClC$pyD zT|v0U`&yiVK>_wK{eB9#Xa>OAH|Bi9ucx^ENw!dWV)ML!svahcffKsc5aGRnu%Wsf z7_|?ScM39_D#$0n(4bs4WxzUc)h!V9B96nGPBn#_-znvhWP9geg_H8r$CtR(rGU;I z)aSzJCsN&r-koM3^*>?SE7w8OYF26Tljq~n!2T)QoSsvqB}wy%sg8Bz8qif7tronxoNGqEG>2RRF!F5LrL|>11ZO*gDKpcUzrm7xU z{?YS5eISU-kh}ubTPa={vc;A5Th1ER0+l!Wa&c$X&xym(%*-49T(`vR*<)Z`s2dv0 zwE+4&jBKj|0(~CgpqhvKOnD1vrT!5ciq~4*HS)! z%F}wHC2N;jN`CMo8=;|dv5aHL4~^jUrS8+aPqPwOerXO-dt*zkBc zDxSS-OJ`Nly(3D4(^&Zn zcyKgMkjGNF1E@MApA5}j_h;9h*(p6s`BoUc$399B=I8kybprZ4y6^cfga>Nqyu1c5 z(qmDA=AHeOGIAgNrksv;%GmKU$-vXprG>fkTC+tJwR;GA5*{7&0 zWgm=k3QlWQl@U_DQB`ZCADRD*<02r#Lel5qgd6*mH{g_+5SBEQIaUYQzP=t1V(F^- z3zz@sOde$&p19AZf%K}%WO(PhtyK;nR0fq7 zR0b*^BMyV!{!TI>r6;TV-bgA9qcj|00Vq!}Fx8YRKTlk&r7RJN=Yc#gk_V%_?-kJe zfIJ>r7@5hkGi9!ho^CK}b}djjl=}N`lf8uDwFO98R73ue?vuHIOCsofs4Fh2Ycb^; zJDJtxBvpMA_2MMVYJMB350LUNuIhk%og~dg z=eaF_oxWY!PqXf-hA3MsL*gdNE^Tmli*s!MSVKaO{=QWismx|l%*C&I# z`W`8ZV4(|tR@E_arzrZ60Sapkx}6iB-|P{uUoV8fmyV*feM=xeM0jqA^__}cv-6B3 z=`MUX;g9mmFf+!O6aH|*cjB6o03&GpH@6ewNHDUYRI?FYt(;@P7~Hl#HrtoC?JS|XeoT`7kE_kS+RZxO!$%?C^! zt_!MhHAkWJkMhCS&o#wvJ!IEOk;q7cNa81)cP>WtBcpZTO;#!G{}QPF;Q1~MxHil{ z(wj(q$u*_dS;n2uTYD+Fbec=l~gsW68)j1}i@RoO5)1$_~O2iUB72g-^*s~jC2{XSrA z&orp%7Xx>k+8{M9?61)hoi#PUHDQ5eR=!aw7j6` zk?>`eMuvorfE6=Md4Bj_2)+PZ+pUQ8T96JKdUk`NL?@XOvkwm@TcD}W8NPP)LuS~f zgIv1gC`-`Yj_3Wi!LS|`FnHJ|P407R-tPQa@c(oUXCF0@{$edVYVj5>pY4DiO-71$ zuckoPXSv!Id0fs~vQ8f8)A0mMFPj~~K6Qu}b1ntr zvWcx(WqK`oHr1K8fAxhut!yjX>DS_`x7QXfb5GFo_U4+YR@LOM!(Q;tz7?B1rI6K* z+|GARv*OhYN8!3d_2i^b3wo|M0}3;j;_)ZfVPJ>Hc<${uDGn`T4}#9)*mxH?xM-04 zYZfksj$v}ebWfT7aU{|>G4;R#OxxrjGv{bUUB7lR$a_2V`xL>?Z%gLSrrLwg8$G$~ zU3(lgD2ekMb)iSPMn1ByDSzAd7hAsHf+FAJxOQku-Z9J`CmLSE2%D)8Hs}x1+qc>6 z@ORiHHV(RXI0^0a&fu?{_2_#36=vL}_jdnYBkuX!#Ik@Md{_G|ptGwjx3#zAf9U>= znHQ{Od;8-!sZIk~@7M@!}$xD!t(ww1Ie@^u4xw_|+^X#XmRQ;kTs0VZ--uR}u3ze4Ae zxW9I@9D1pXSWtH?#5vBD&a;=ewrMn&_Td-!M%R^xYM+PhQ-dXq6OP_Jg6}HY$ht|b zMfV>y8T~$M?r|FXr?~Q&{4%D0ya{(7_=!`4FF{hP58_5;E&jrD1lPIy6w1IB0JLA7FS4L|o#WH>Tb@(Da ztS}%>c!Y&IwfU~eSFzsmZPI0~8`OIH4lnsQbLtxq$5x_+?@P9^cR2WrTEy4v43LMe zTmZE`l{0EnZ;Y2uJe88$mb_!?b4*^&HYn4ZiIkLGU1Wsp}=f^ySGM)e6A z{e2i!^DZ3fP46PgSs@*IMDk-@cVhh=YZ*IyA1-*V2jBdG__Id?{@MID+S+fB-STU3 z>IuwCG{=p{^RNZ#VBZiU`FFqypn1yA^;gOdmq!EDN#2xuVH%H@HmV0f)uP{!uZ(Jr zn=!g)%!_M{os!gEf`i*BF>$kv7-o0`c5S}|)OVWVWv$_qPa?M7GfYG44z_-Sc<(Wr zVgB^JK=Z{;Uz_5Tx`>Crh4Ez*Gcmeniq<1*2d8Uctq+H_FaMbH9(U%8Z;vN4RVP$S znDp@&Zi(87clWfA7auUbtwdi^kMgUp(o|pKrI`0Zm&5{Oynt$_U%TQ}jN4~S)gF_Y>3c?Sb9d3Y6KTMVV za-*@qoj6eaMt#EFhaASg%Y#Jl#cJ~QNk^*5Jh8gZGIUuG2GjEAh?-5$F*7|KDS~^- z-2;oUeTA#01%`@|sZFGFwP+{{D1c61Q`s%^u1IT*Ll*R={&f)vuvM&>egQ)t8uIa3 zlacBIXSXs`xFR=Ov?dHU#+%zXk6iZ%8mTCic|CTesna_1x0KmeV}o#)mU7VVo`Ou4f`m z8uUf#O=eZO35G0+LEGlu)YD&t8Po8ni)pSjM(qpF8#wmdQMhKbK~#@x%1_0V!kFEs zwDVJ+u9j>xxtRxIpk`H(=^LZ)CY^_vtaSaQ%D#m4T3+j;UNdaYKtgA zYayL~E=R&7bSB?;kiI+LVf)_o>4z6=TS5$&p3TQI`}1W=V^f&it`FAOUj|Pr_h~HM z??FQG4t8?d9n}MzFp^V^0D#c=nrEu(eAZj~W8J#ZhseikK$@Hjzu z0OPnLQqO8~3R?;Dd6jGluf)mIqcLLdOuX~69Vg7JpDl(`Il2((r>GG#O#pEE|(^fTlYddI5j zJNVppw(#FZcIBWQ-&0tVA6R$~h#$o*lh2U<`5`v&ie`orUSZg3H(2<4h&-$$iG;^_YTb)y0}KIbm=(ie6rW+3J^-6XCbD~59)n)8-kZ3Ok6pudOI z1E0Z6FO=5lCOTezou~LWanowT?L39kK>eT{vY?2m{%u-eCHqzUM>Sk5XdSf&C)5*H z4(_7-#Ffw3+X`tO;QXg2Y*{%@X%*(=Cm>rOU0(GGoXoE zQKw_VJxd6?sywF1`L4wM%n*MeTq2GZIddBpufeI{`~be|z%_M5&uddYCG zIloIh&1(;Pt>1}#Yo?*9E!6|-7H(GcA;Qlvr3ZLxr#s+P|Av~MYo`v|v`QlpKhpVP zwczgTmiT3ejwI~GW$&^ystyQ)p~mw8xVq189G=@oUWz+{ZJuq!MF)&!p-wD>c^81f zE~-P>+G(`Xj2iWnpP(tDvTm*YKoy5aoRfplfZo5eEd;r zg9gjGiftuPs5Alfwn+GQ8Iu;a<(cdY}rq$~@ zl=OfBS3JLIayEP6)sT}X!cy6f#fq;Qx}LhGY||tW`;>Zd(nIoOz;2_MAElipl$%(zASF~rmse~&+ltKJPJ9=VTwd-T`#Zq|r(PTxd* zzC-mb|6{aNWFP-e7z8gs(wVeROSoI#65^2sa9GxmTj$j0V~YY^2}hYY^cxfg(-`o_ z`P;%{-#x__NFE0x*G(YbbR3r1O%v-|Bw*s$HP}RNryyMe9XuvO%w!W9OCO+hV&or? zJR+1q__|N&MKt^S9S>bI zl|8GqsjBNygRFRk&OFh8&d{VM8_9nFafn90 z>?HJ`QjNQwpU$rtS#qjB$Q|L0Uw?kk)J*ut6wXs^!mp+EIQa-D*wskZEt`l-_#(g)%;_AS14vjcB5n79Uqgu@{cSj5rj~B& z!Xa6wk+d8m>=1;DU>tBsVIbjzrNVrne82#+qi`qS3A4W$0B36i(0A||=w81IkVlf# z4>;qg9X+e#$i-U@b;}+LT01)GU+^;IpER{Tbb^PkdMPdwp7;X)+x!?h=8Z%0OCqW5 zQEiCnj;en<=21S&?zb99r)Y_Lxb?S1+N<;q$k3hVuwGs=_PJz18YmsA1!SrI6XroK zNF0siuSh?gz~BprQ>|8rn>UpvZKiZT`Jw?x8c*p}3>z9DcsbR*Rhou8s-StQd;>3p zdm`yDc5v(!F>BEvxavC!sNOMX^;V(0!mHWmV1G#_ny#&ho_%M*`qvgfIsm&2p)**= z9s=?w|0Hpgh0w$1OfA1Sh&jfpoVFtiz;h=>cZH<1n<@x0zW-_7FkQH-+`LZ>(=GV?mh- zS6ub`y(uR=WWEuD`Q9f-Vf#Wsn7>i!4^&!Zd$%xc+LgV6IEwLqY2&FkGIH^ zYCqN9h(Vq0In^LOO8o9jo>D&0O9aY1c*nk(_mhOzkoCB{V&} zg=$HUm0T;=-15vJ99WB_7pT^zvf(*XpvEf`BwS?A{2v32OW|MF;L&t;E%Ew)*PBD5 zJ)NOv(SAnzEDqJ~Pg;3A@#afLeF)@3pqI-NW;@=Tb^Ce>M|Un}Vb;;A=48yo60kn$ z#{Fz`@}}HA!N}txd2O`wpIeoSRi>{Jl(X=I=|(u&v6+^*1Ski9&*satgt>D3n+(vuyprlrD^zYo z+{hGuPv7m&XCI9RuSY3h_IEL+SbK8P3aIm74^pq7pG_!4)k@V8zjERf%Gg$+@+R!? zSav-x7S#Av9-uM{eR{{Gs$0^@NSPm!w$PGi0lj(ND%(NIkC1d7kQPwagOdSG=<_b% z^P6-o$}v6agRi8Gbr`KH&b+V%FV_3Q$TzDj&{d7qy?&1NEuAme4Lz_Z@u=2#b|O+v zq4Fenzoi*pecgj?{0}G#8mu&`AYSKN3Q`HHD}|~>%6DMmMqfO7w=?m0BU$DY%_vut zlwWFzqhLnEaX4mgB;{!vz|Q|8JXto2va>CyvQm}N{57jCDTCuGpWL$Y0g`r=$GSF` zhd-YXgs&RXl8ola2>ZpK8xa^gP*3;`S%{q+eAQmEyu@|bui}qJ=`+GiSpMZQBdsHq z=OO)|G!eLa%?7G*u5~x$lppfi{qD1IngHCi^|i_YK=u5muV#|?3O8-fVned^Ir$}2 z*f8YTZzRlRq@|Ug0`jA<>fI&c*0BnE1Zg%-f5r*T#&hyaOlcbHRT1QKbG4+Yfv^hx z4yXpib1Fl^j@VqDHE^Wd%LbHY63e1g4$4S7A!VLO+Ju=js!cjG2?;YW&hewd5b>$9 zR8zh1mh#?$GF3sDrt)s$=2ShT>^aXNXelahN!}4D7giYsZ{Am5D4lQUe*mok8t|qU{guk%dM*1{Eaka`VtFkiJ^@osf zQ>>hS1gG3sQsq09?rPv=0P7Y~j^@Q-M+-VbE%AF^nKV;QH{#;9&e#D*T-z8+}wsd#%*}J_>j_tc($yg z@)XJkRAsw${pgG}yWT*02fIbo=3WhxfpW^c${lAkyX(aR`4*=9o{q~sthZJlbiNt$ zY4^C{A+`F z>i)^N^R)@KcDEIy2YrMoQf#Wdk3|VroFuR(Xuyqx~(0rmUxlNyE799#eVcrH{6_F+x_6-O?4B=2>s<}dx{HGxp5_5xzGd*u4ai^R2f;BMj8|S>g@{YL z`JsOFoaN8i(n(WY2EKR&3Ga`ARi9!^YWo!iM;_4Jd|HS{ZL-;)xohPbohvxld72#8 zyo>xDlg;i`yGPGZcA<9<#6#A)cKl)47WtLVxcJ!Y2^`Jp#c7VrJRk(yReOm=4Gdw( zpY7VsKRV-*R;$r9KNMbk?8=7xX~HX4ILYaS1K9OhSH#{r?Lm6o!9W*BDRe@`c=e}2QsQ6nX{_{W{k zocIQsdV9zXiFdVBKS*PM>4hb5l6RLU<7!avJ)`|59U7*iasGVH1`$*O%38klw580DfO|_~nl4!O_iJdcCLff_mJB zO#6$dsd*g_weHOWcQ=$-%}%ljhIM&Mrw!2kaT9qe(TFd;oX?_WpGDIaZ8&`{Ca)bR zx4qkjcCwjvc-S`j{4Lt~9t`PsB&|K`F=`#{XN;KHtT89tkYmUE=W62jNqagufq$I0 z2X^fkjB!5)Fy|h%xc32HC^TP*oo1c^s%fDao(wIn_vEox>3JH@^*EwIF18CSXO$i1 zxYD>mFJ~C)Rg88zznMQ7^GU44_+|d>xQ8uOyEgxT8e(*&f?2d3;E(>uJDhpEk|SykhJEcKbElT zaX;8Y$18iT8#ZyP=UAu$ISgX;e(6s!iw(Y7fkb0Ni z#kV?KW}EJkPiAVh?FKu*@Gu)rV-rnx81mQu3Xy6Zt7lyV!U|bCe+c?TSaPrT>*4*r z584UZL2|BJ7g>GH9?+}NP&{4#0!6JB<>@t?r zBiO1W3*!1GfXCnCqz%?#QSfrv;#ebCY+FO_Zqkp3ly<;Fn>+EeC$&_~YYDr-$#*Xj zjsamPSJ>G5*+#Z&LYPb{sU~_woq=n;?c`3w0j zBX+o8YouI&2k>`HM?SgRK(6*JZbnzpHqS!1((&bYSG{KIr)07hp%i(A{)i`mHcoFIe^X(F`QKQE3E{{oINNkBL>{jx z^>=E;QJapOxQEUYe}-PmceC}q!*H5ICn!4Ci_c(g3M=65kLySr#gj(Yl{4Nf#=%o+ zLh)38LAWkTx}<53mIZ)i154rE_$-KoL^P!DH+v5Yms>|fAgz(AMO<{=Rq+sw*+Dui zx~*vhN7(tl|4I6{7&;(RalUr#;$G7I(n_G(7B`1x!_C6RP?6%F7j(KaJgq(y6HM-l z-Z|febnz-9< zs~A_^SkgRTfaaU19A8J=8hubchM3`hu2DmYS1( zFY;*ZaP7#(^2(kFae3||(AnJ&n{{gg^w~UeY7c%opb}GK-6Yi)9(5b(x^3GnAPff8 zm!Dp3$y;n2!IA^*kv;=oMqk0-nhG5ITVE|q(bFRn>H?EJ(EuVTJ@tm9!xJVLb z193mI`5CNukX>w32iABex%%{4fY0d+*=xtlWa6xYu=!T3c+%(#EL!pz32&jn2@UJ? z`zHA6H>UISGqr@Pve#4_ehe>(D_uA9ZDUsph1&){Bj8=Rfh1ih6^Gn5UySSN-j@MC zTS?1vUht-73Clb-QE`T-?C@Tc9kJ%s%N8pBfv{5zF|I$IX%rU$CVT*FkLb!`Cw0R2 z0nIpJD5I}J{qkmcrQC16=CQMgCdK z7VfN##O+M+wBIgdHMSn5wMho*6+W)|8q&b4VM2e(Iy~Y4`(6_?SH59E7wJBJ6B_L8 zu5_JHb8WTs6>c3-L*6;8i=^%0-0@7|($|E2%3G$^3u)iPH0gmQL4h#GG)xql>&d&* zPvhe3O*pA*A9?aXJ2<{9jP>4d8TraVsJ9_YbRSZi2f7@TXAkXUX+8gSRlH%Gkxg}z zz^wZ8B@WmD$JaT1*077d!BC?u{> z`VYyYC|s58U1EWHK-l!p2O1ymaBUy1DX5NZZ)vO6&edxJs9L*oI1EOd`vjY=L_&l6 zkMdsZn6KUIa82ZP?y0aiX{ABFy!1a=N4XmfjrhHwT({4m9KCpW{tmtk;vfTxU$83#tK@@U;$ z!EN7CnC~I*q46u`ec(3Xld<9gc=Edx^Pew}Yd5{Yr%&2(+IRdibv;+TPhN#loikN$ z6L&Zuc_Li4a5B*Uhu05M@L|X`eCo5Adh!I4w&OPoQ`8tZ=`yI-{gm<~Fmml)4PiK# z2mD2v4?EOv7AI{Ege6#ha5Z}B9K_ttr@?CK93ZbP2%lm3%DKGy>`>z2j5j`dA=;&&apHoRi2}>Mf!GfaYC)F()o(RNPD*7&X>2UngR0aq(S#WM*W_!ZLARzZi@Li zyHV-(RxBCFLvrFlNIvu#3p?$Dkxkv8Wx9^4V&)`;X2{mV-@9kR&Z0VB9HvO z*!rSAVaaMbj~0+PO+1RYBOXmYilM)rz&rm&g6f1d*k&PV4>g-torS$AJtX_DuTWaW z!)Bb)G|+p4H~J*zVa%BQD0*B(=LsD|QppO1<)nvuDG$!b18QH#(|JeKUv9rnqFzuG zkVj{Edvk#HQDG?*Wv0NK!IyCH$LFAQ(zUM-km^qPYStlVC0vgSR=SqGEe#{BeV>uG zLekL6H^7s-_Qa82;n8F#PWqUU#;ej6gkfUMqrE^LmkoSbM`UiKdYi`xtT`Z%LvPQw&(M6T9OvFrdmr>SqvW>>7e?R{);D{SE5<= z+*|oh@#tDA#w@$aoIjox8Pg18xT`Ucj#qmMp4rp2q`8oC03d&>bP3M?@&$Xmvcpzy zu2G#O!M?)TNWNMSma+l2Lvdf>J4QGRl^>d_te|R-6qeICC`bA!$h&EB!=q7gbrn|{ z{33k#RkeSlB~bZo%Dfalf;5Q(PuuIDbQ$>ywa=6}_2dmJ-m0EvLyg+;nd?@G%;zhV zr-bwEw^CMg7_ar-#VEr9>J^xE>zTdf`-AcGe~fe% ztg7({1FfQ2=>-O5;lBm(rc8UX3YB*7STg|g>#fI^^`{UwoYGL;DV+)HuW!JiZLgw1 z%x3EQiKz6EcU)&4Ht#ysrz?%Q5vObgN#mlU+dB9(DhiYbx0`hjEHn2Anunx$!Q_h% zLEmCGD9@?*m+~+T>0TCF>4D@0;bNOOy088XJbc$f9=p2;v*tx0^V{J{I$29~4jIM& zkzOD#Y3K{4y)HBIlvTc)`b|Ug0@AQ-x>K%(@Pl|G5vrY>A-?Q>BoxL}`(OaX59GTK zBdrx9U#X=WLS;{&v}XIW+d=Ux<%skQ%0TTRO$L&u71T?D`?r)emer}sHm=n8f-)YYb=A^mQ?_#yJ~o>qT5Md<^3pJP zN@SHUB#lEHQwD?|lpWDonE7-z^geyXmAHw{QB1m%zwXf8Rng=dAoOCY~jT(s|e!Y-zi@rzKmrBcc&TWXV4=x8*8b{tQ z9K(h<0_9i4c^x5beoB=dA#cZ29zgj6E*pL#Pvx5G&y-Knxu9IO8+<^J1}9yAFK_PH zyZF`Z7m!b3(xi@Dq_LNjgNf|au0URjsqs%5`40Eez5djHl>e9^<(^gk|5Gh@skEia znC=Dk1zvX|_??+ll~a>XQ(i=)6ZA*S{Bs`K-)P2bho-?G|K1{QBc0>!UZ`ab zi@5E310Hj$56@s-c-Y2DJeqPJr^N5U|FkjCD*7!3gfkqqdkwpF=r(?tV98tf(OI5X zUc#pDi=n0U5i#P*7>EpLfZqOE9DhF!e#hqHtI8PJV*g4$+i4`xW&^lO= zl?e-<1>%Z;j$+=Q!*H>@7Jom%K_rE|f?Km5W4fLV{}DR?utFc!j7x=+Mz6%&?(}Sd z?_XH*$5@6stpVFqrl~>qHwQS)h7#XMU?pa-bkR2aHtZ<87)EDaxCUbMxUTZKvk6{C zA2bOnzy{XuMdh6iGVh@q&al*x)rPOtN?&)_UTQAopjocw_AlAITG4PYCly}WjN;~< zIJemE32GhepOCM5wh~Vbx+cbY4n@!VkKlsyFYK9LUv{>xPWM6VlpRV6@Y$?Rc(&XD z1}D>*G||KOUcE>@-7_83xbFUI2pjqg;d#wnxbc2}p!JsVF^}25J1ZD{K94pHpmEXJ zWg)frx#9~L_HhGTF5V*_+sws%EJZf9?}`Nvtu=pF+-C!KpFo5DNvy;u4-yMz$?An} z^8K)zu*b0+3YuHU^zX$mFY6iRJv;$lhIN;;ZoJv_O>9Tb7`9E%86r#n!bf^vH?1vh zV1LANCu@kkCGaf0V<5881sETH61;57M1pA_*&|wqZ`E^`h0XP0cD$Zczf1GL2biP@ zA9qRfuOAcXdfVg~-)$Jx$`%~_EKvVa6u#Q@63lWBu*Hk^%BJ@F;aM9E-;r|{kA7bP zhvv1CH=RpFtwMWwto%#Y$VQ%WwC8yIy(tm*g=~=P(^iNY`fcT_7dh}PT}Qq?WG*vU z4whpd{MpY3f6#eUXWSRiwW`U~ishKpNDoi?9>cAsRlYXQ{D-5`7C*TxBdeySef7lWs=q5fHr zJ5Nt@CHPL&TpR2^3e$@6F?nMYZ>{?uk9yWh2DkF#t)H04`-j|NZ}a)!gB1+5oWN)fa#7YmaIj9sUp5g~J@*w}{nrf^I=S)Z&hbcljrkAl zX?}(BR@ZTj&j73yG)fxvH{va)uRt+nrx+jpM#OBqD3&%_Ci=fVg*|QmvPbC? z=nTT6s)k@`K)))Cs2h}tpZf*Kfv=sZ*NjEP$6849Wd5}Z*-q9&mKSeDzbn1i#G(=6 zNvX=BS)QfAVy?SUKe-<3YY`lMB;~O*N@ywQd-L2-5aE8-b$kyKHS>vD@ z@}YA}=HlThEq*mX!Vz)O*BM-+?Nt9^?Lup?+24p+2PI&w!VJ~3`07O$%%=0j_Jvr= z2WSPokGNp8&m>hV(yTCyO$+ZM^)Kn;HlOPBvIkGz@O>)YLPt8hbp>=T>B4D@{L_RT z_$i&z#CM&0IfOk)AEC{!`~?w2`{~55nsSXMk{;wZD-j%8fSg zF_s*^Ui8xXUKxzN{a#dZe5Ui@{@1qd@_gwK2JLEjL(7qJ4~-RTQ=oV#%s z?=|d#)f$DZ>{EIJoL1D06Grks!x~{|EBc;)UUzv553;giWlB zY3+FJPS^0f`(2>x;?!DQ(at&_SC88dDQuLM@Pk{t??u<#B`PnD#`~38a4yLdv@de5 z({*?~YYTi%ufyg$oyWni+sJ;l&xKF2H+0u+%oXOSIw4LG=ZZb~eRl(N_o&5boJbhK z4u0Q+cmE}0X+jFronDUDoJU~c!!=U%%jn&@SUa?tv@7|DpKAWoY>IIciVHt(>Lmxd zu9Vpp6G`J>AA{EYi&Xnf-}gxp#LL0Pw8a?t?j=Gq-P(|1a@BWe>qtkTZ$ ztj!A^Zo!3%ob%F3BQZ8WM-qSFs#Y7sg6`XqaF#fs2c8*p4wqyVfd5D{9Ob_ezQ*1F zhmwKd6`0K3zY4-pD^0~Pf7-ulJb%yx^!VzC2Qy3f(7at5hu#MVZc`_ zx|j9=q?B$$OShW2v?h|S0XzCMJ_#_Y!z((*r#n9AUHGAhibk z$iwSM7>1AA6rrjQqyDS$yG??2Co6!rv)165lzQAEb&>ec#t@#r=ph@XUd4q);Yj#^ zsUc^?dY{gGwa*QOH%LE+*XG}Gs4b-_k>Y_!{jK%hcZ?ah#U)0qo7m)f7H7 zmRmynV0TU-Qcq|J7v;Sdlf{gpBg`+J&TDXOCDTf4Vk5shUBfoc#}3veocayFoc^gW zlc(nlMyffOqb4^av$Idc3lV+c*gU#F^~h=By1H`Y>zcBSUlYyE7yn`Kv!~+4 zEPKAUd8P0UjIYvB*9Yl9JCEr|d;r9o=xe(TXVls++;`8$sg`k=6|zdNBZ&hW3{G-roh!?o_Z*{`G*cnGL%y+4w8DkeBcgNZUxl0P0t3wB2vY3Coc5 z5fBzaHH-S3c$F*cyjXlx)_=br2t#4~{W?6ck*T~~+zL&eEJa!uNj-w3^>{|b3#4^G z`W)FfC<0qQiO;J@?}#pmkw93F0r6|$h+_*uSjPo=gf5E2G5#^w zBGr*k={{PFu-&VTdbS5%RT{Dvhql4*JGEusBm02x1lp90lDi8U2;v1zkLXbJwwxut zn>I(csqs%A-3y6$p(2p1y|N(b%2J`R0w`7=+T`e)R8_db8R zZ^ev0hH|Tq38q&36VxMGm#=RKk5^$#%n~G=g*)zvG9oKMX`iaGKdEfSOa4vL+F5ra z+$~h=qWXw6PrR+6`QU}(lh|x}eM}3Jg<`IvHm% z;wCL&4^YjJ7W|AL zT2G*UJ5G9EjF0b5?`&(x1DxLB`i=Wg;S1?;W|r%~LKoHKvudS+Hl;UYu>?3#ewm#u zKftz|u7)PoBf#RjCi0X0lXuqkPnz$|5|OSzY>TzmkS&hJ@vy+usq%W5PJgY&3P-28MCr za)?8cri*biAiZ9tu}@mLfztZq1617t=?t77o~0h>@@-7v(UdIa#Vc|;H< zb=9U6X_DDwRQ2$ie?szULgBG~sD?Y_e~0*6j{I?(-dy2Vsrx1N80kHm?(I1FTuE!i z2?Mw&zXubq>{mVlPBh=nRKF9>YSJ@*5pTbTZm;KY!Zbl%7|45aFNfwz14>#C(umb$ z>@CL0+W~1AHq`PFkf))!H-=96;|1}b?Cn1T%B*yOGzZf46fZ&@w>DVQVg=IYA!&No z|Newk;c1iUEu@p{2kda8p45v=hJ%>{Krgg6dA#Y+JwILHrRcG2Kae)jG`7zr+?xb_ zkF-V7vS?PQi|xEmu$Y*i*yYGSp}73ipkefxbC5J7;pSraFDnN|dp<AmBQ4MdKHdQ& zUXrv%st;IF=`m2aJv)9SDjiI|y-E+4S%nFu!w7>F&M6&4&q=vys4pb_8Hm#p(i75Z3C=6V>%yG@gOGA(Lfd%jwm) z>i3R`+n}j^JD{w9g)B0cDKQyJC#Y)+RnwE=kHXFv(n!8G^6e~Rem)@+6lW2x;na~e zu+Or(xW@D8Y&4k4}1z+dxp+*=Tb%I6Y4ib>%QkURk4JfPysuK9F+ ziT_4Hx((K*#bTBt5L!;dt-D9DSz=YDTswa*CUd+=Ux+p1iLIFWBHI0pv5&WY)s zEtH=H!gEGgOMU%}aBQd8sCOHbmLy&gx^XSV2+Lt`Blt8BwrXMnLP^s_Lq?w%-t@*; z(vZeT{ZNJfglBSSV0G}bUB_x#RIBQ9!g%fRlq(w2M^wK}vDj-_DkIG;vmHAiaTY9X^jkwe2h!M5)k}ZZwRk?F9Eb;n zXCR%qo6=hlUU8)Xl+U0s$ey+#DnsJMDTdT*pIIuM51XGb3P|5+GQK0sU*wL8cL}Rt ztYXjvrn?98T5=aj*V7%v-w3PNEr?$j*AwAw_3_(Agv&qe9JKTOk;O(HYK)K(qsx^3 zR`re#53>Qg`!n19YvE*Ocoi>w8g@baqxV*kz7tJ5>7ci*J5&30>0b#@9-~p(rvGbq z)!TTWAQW#GuLR1{fHW}*tKk^mXb)jgzT#1(i6Fbeg^~UcUz7h$exXKxvmyJ;vjViO-XnGJSr_t68YAp$Gh00MV1ES~W^x?;yP3-ERI#^=l z3Mx;bOaxWm5BG0|!yJro6us9#Wd#lP(|J)P1CjiSmUtXU4~T_D)$!J>iP-$fZlnxB z(?0qdC@p`axtFATLy$*PI)UwU{2xcx9oJ(Q#ZgK|NV1hAlE^6ayXVSYDJx_qp==ph zAtNJ|5*iYsL}?)Ex#yatVN0_2-XrU^-gEoo{k*8(^W1ySIp1-9&vWlvK{`-*Nk*FR zc*`fPx!>UM<#v z@{Ot&x~O{;d%CX_l#VAI05k_dJ!&AVrQXduHvWDijBMIeDm@i?@i}RP$6`QWwF&?=9fm@8$yD!*Z~%UI8G~RB3Rihdxh*KSB=ttM0f8HoCV~TR z@1VlD!ZBN*!_ZFhJkH>R5lr>#lx}h|-mdxv=sx+JGcFq~vuA#Vp6-Dn)a$j*x^*9_ z&C|H^aDVNTXE1-FjbLLlyYaaWrLc~^3w{)Q68DWNz>e3>p-a;|%~-Ko!8Z+efO`^V$d z_kr{*;9A_V=o@7BN?=xRKjYno+4#k!6%TGPpV8;>YNfd>%0G?K^{{I0Z&+Uc9bR^D zfspV&+U{g-odk4@PF&t~qnNo{WPr<%1t(eDRJ?Ic!P8%B3=jF-m zcDm}_Nn^;dEYC^?X?EVq0^q@*k)B^ z*2N{9)`lJG8iU<~W`Td~U)IU-Bbu2+OIl-0yApxX3$CKm(`vlN_xWI4sZ^v)Y$N}9 z)Z}9Z)yF>XzGL*F4nRMrHP(mgMpt3ptRm6ii5}4U;j6#Z(4l7ssQ%woDDKo4?rY(6 zhEo$YFFCyHLXlEjOJ2NMUw-o+55@*{;laRn>^_|-61m%j_QO37Hs`v_tP%&-sK+Oy zdE&^vCh)^-8=SE=MT76{xUrhI0DkJxKS*1tzN?AfPSgk8LZh z-p&Jquzs>2qPy5|>OIcrWG`F)strL_)uloCB{BQYM6_%^S;Z7bKkmSOx!TJ8KI&?zsoPPu7oS!3xm7IsLosIdqlfz}` z1v^f8g_o|y(gXOIbh-mIQ8bV?&DrV=pXupMa`HCP77YZ)K9m? z(3!pE?xA`7(8a@I@0XqIWaCn7)G!WOZ<;Lwo3GC;s~@3iMqb{v5B9XGjU5cPW8?Y& z@3y|yhQ$12$*o56kGl^D69Z5DogJY~2`NGEPxa)VHRfm?*;cxhtQ8@D6GXtn+A0n* z%DyB9?GQ3wl{RO3BQ>_l1zPCwUeo^gOR3GAgaENsLH8#M2=SH$B}b+P+L zW0b>IgX#6D7;#|%EN-ZeyG8|J%pFgpoFtr>O?&*h@_>6CdFG9J3M*LGdQCays(3Xe z0-~$W6LJ?@4;BfS3l@n zk-~|v%qw)5uq$&TwMD(1*K+?nJ2=|>|}U7=`hlo@qEt<;M-x7?w7j}JloV$KHRVqug>bsMlUXAs`jS7 z|4sP01BT2C0)^4RExtfO%Rg+&_wCwg@-M8O4@{kqfF=evabid)j(^#S*L2uNXF_ko z^!6)c-xE)vdHOP0bbbq`&%o~t4|%+c{X)ycfNeJy|kaT8`+B25Bws(`akJiAUbLjc^>P&v} z-;R`@I#o}hj`i4(YEAIg5@+5&bUv<>tAVfugRQQ@e^2+KiYM^^WZ!b)l%p!=ab|~W zxHH=tnl;~vwC=d=z;2O!NRO*Ldpx>2Up5dV)quohw}5J1tLs!Gx-M-g%MXO89DrS~ z0j^ygCyZhZWc5E8;$3thw(-i9R8x}p2PZZgz?Y;&}1l9LO$2J%i*nUEr7dMj%`g8_pfX50idEqWN%K|M`J1 zYIm0EJq!q^;4<~lOs_x3FW0=WB&8~7hX%_1KP_ba)Xn0TYpT`}(%9bddfe^z9TB8i zaYvV-&^FqR&p6)(T3GF7=ePIb3jhAA^$XS)Rbx#b&g69y+q2Xy4>9k15ZA{sgmalt zhqYFHuaOB7DgS`~@7pa^X8yp67hhZD^9JqNdz z8wU-0lgI)AxArtmuQL@4f^bfyy8IUZmMP1`H3lPVW|P9`$KL!3R`N}fI$%69gD zg7=3T@*6$!a|!Qsucp-DMzQPQsTE_CThKOc6P%B20hgS1!kUq1aOf>VZhy0i?#;Qe z3NN7~X+OUDaRDob1>w*eHu(5dB*i0|`N!X7yGB(4iW`nwYXrFu1Q1W4)#zi`ea(4x z@bnkVD^JwH3%d%8)_d>D-%lyS-rwp1)v$1Ots`rmKLMn57{x>qk0Pu01lmmj*y52Wja${X|5=fr@C z-K1}a8tPi&?`vzBzT-4@h+ZI{eQnK~2k#Tb`6Hp#DJh62Fy`h`(t5L5_n>!>Ic zSkR{;CzcIFtn%pq=I7U@c+**{_jbVDh7P>nwlL^1YP%#of`r9D91N!#ECS+4c+@Zu zB49h-cGKtoJ#9(hYO2Kr)saej-pM*7>`SeGNG=&q|N zNh=VquRy95h2cWw9qA!PI!T%(eie^CSCOG!)5SdM5!IQz0#%Jg)>^~sO#X}AvJuVS zrefugM4Y%j9f^B^KCcnafR4V5Bfevb=Lq}p+=%xIBSeI=lcYS+HrUsZA-}pyU7zbn zb&P5r-~8_Z=`@|**xJx6>lW^K>a6|9s0GC1Ks6*=SXqF@$hvGu%o4H0q!0`zbdgO0 z&x%cx3XwQiH#uUn;Pmc>(wC>+Q*STpmclrAer_woruBgnN8I4=a-RxqNqUn#^9n|38RF@vc3E;XTLK^FJ zCce~{q|d3YBGJdDns(Tj&Y}%ljHH=J-!6t9E^pbYqkhtR)x3&0JGxxOi5B}QP8Wgr z4nE&(mK*QBi;f5yhd27I!n4Qb;iu}G#e<{6WP^uEu>P|b`kv{hG#3yrD9%RZ1yqh6 zn)pH04^W+HwC+ObWrf?lHdp6|_v8tMsWs2jK5C?&6E`~XnuQP9gl8VobHzHs#xeN% zLIYV+yB4;}WKgQ^K zNV=LSUTijS7?!0TWNA}vb(YPaR>Xj?N$R~n51o=D@k#4g-L~YOnE8*+CwsaVVjFD~ zq_xO1?9n{;{R6ABO{CRmM;>f76q*h(2IbHCZ@vm`#@(T3QBP^glWeKpO0@g&14;XG zT6-klh0gsuRd4~t-kEfiMqUc+XE&2=qYq)>pEbmhd&Lv26c+VsAXl|$2mgkq!_Zsn z>HW2Ruz-0gKO%o;Tqi%53I_jG=kF?~w-*X0*Fe%t@N;=n zY1yJDzhaOE+u$BtsAL0*?-jmN+{0`e77@69ai zd-{yXTmJwF3v?loci>vL-AEdXz3FVOG&fd_NMg6|O{(yMRAWeff%Ml1Gk)w^kQ)yszR&Rz0d%kWOXwQ)^4Y5ca!i45;<`Z)rE>XT;dP zx6q+a7`ocdVUJ(=f!>>ULV1oQ^{o|;h@W+C=}0T$Mc+)Sk;8)45a##TiZf%o@ST%^1s+%V&NgpE#( zRn1Y=$oVrr80mb4-H>d#7!_at&aQ(>BM@J4${U>PFhblOXf4Tmis{~GabI2ts?Sk; zC{O)?G#M(Mol@=u%Y(y(L+K18|H2h!lHcdStvjIN5;?32-g~u&mG8Yo&+4yYiRL9p ze1ZoGG87)Dc*3ZILiJtH?TR&A-!((%ouTtYBl98U%Tt){-3xOEwBSu2zJ=W$iMoLu zZ54iMH)bsb{gXLLn*rThM_LGeq-6kk09ZdRAA5f}B38VuuQURZw*^`!DDpleOnV%J zqVpy&{!Iju?j(i)gxkHO)`1m!W#Y*#Mo=`F?FTPyK;%?{*=D#88d zXNA(6M;2~|U-x}sNL2|lV~-2c_A<2SGLVlH3e!$`R;3<|G4(aR;@dcDu6hLB*SHFm zw{MD^wPCy?5AoB-J|9Q0cE3h)@@}AdM#R%V`ic9tJ0wQ--BOVsggaQ4asfzpgThkM zJ@~QnR9riJvWPicS(5%zb;9e-<(&A3aO4?OuSD-7t(qsvLo>xGw+AK*@?pfy=E}E{ zcmGIuUV!P%)a5lA@Ch-ZQL5vk6w!1|7O@8^JuQ^^Ma^|>_Fk5PK%PWol+|JxLZ zBa!kQNe@Gn^?5q-MVg6`BYE&Jq?*Ec(`P8G#1qGCfqaEVx);-YF2acceoWUh0H19- zgQOXOydS9kirduF;FxAA)!0Z^KzVW+_@0~w<5s1zfcb-Xt@u67o@tz?Cz(?*OvonCpdfTGp_r}C25)pj6MC*N4Mzqa8%xw;=w=eU9Y1U zRQM(80XmUCXUUd7ko15gKab=Exat>@CSYOqqrjlty&`W^UaxBIj<;$C2*OLC8fT=} znEmXXSf$*M8H7w@P78bmX&DUdoK5{UW6s^P1?f)Z#gKTG52?0;#nro5u?9_tgrnbf z6InN@CO7mpk(>Wcf!-T0p}pTDXMLOIq;Ucn@n5j5DN20%r_t=28@4*)c1iS3C}gs`;WvHGDcKt*@Rzq zIEm?9bU>bz+c_I5zXl(G_6Q-JLVYXhpO&H0NUGoJxxzrw^{C%hnJb?~-n~L=C!KDG zvnrhwge&-<1-;+l7stqZO0}jNPR-VQ^XVLeoW_j25|qyMMC;c1YAitgWb$TrfIfp% zn{=+YEC0ItqnZOo&2|^0%?0)C1nC9xM0<29F9U&e1%_8K1g{9<@+vo@ zT}0NEGBDbnf~1L5zf|FYAb$&_tw{5wJ=(p?s7jEb?aNo=fhV)Ge6qLG0Yxo zb*?F0vNuA#(et4FpO3Ke)lBG}3Xt!3Om_|2$kJVHL`a^A?C%d?du*U?>828N49jE# ztv2F=gO#~;c?d6AOV4(+=?w1*Ye}o5r?~B3b39$`3NHM!j<3?UfGJx|W$}-NP&nWf zCdIEs&wy%t^SW!8@MSGl>hVPcjGx6Cn19CoEA_c~gBZy0_>Mm>TSL~a@%-$XX&CUc zFVyL|QvObV4(}%J2a9Fh`1XP~;*a+cxPSIE_ORcJqZ600B)_`Sc6Ax{+*-ss`c#*D zhbFLpX|cFrNgA{2af036HWr@E?1BFthGJRcGa~KQJqUmAB;I7jL3~Pc=#a_9m0C@) zN-KXYYB-WKSY7l^+ahhg{nb7jEz zcbJ$9n61rre_4SeB5?E71lI~64yMd@O2Y@vSlr2v4a!xMBoi) zxwQ2mF=WMKUDV4|*0T_?O91Eh3MxyZi=S0)h+9SOIC14{dHks(x?~%P^@nzWPko`` z{R5~!zY(X{;h&;duw6Y$9=)o^FQlGof;>vQtpDZmay=TfqWl9 zd(TYLr`U{=uR5J(hqXFD>mi?lv5c?TM+E#l$1-rd?4Oh&Hfw$4;*=7~&rN7!*iD+` zHu<{RMKh9EH86eFjP*$Qh0ufv9kHRSK5^RYvw zWa}Sof$1l7{CiGkJilx+dS5b?F@5vl#hb?ReY>L-^C~xL&7;jsFgtb#FW!2Y#it}l zii=FT&{c-Iq(LX!QS43q9uQLAN7A^3#e>Pb<(R+d=i;p$3}3{{+#Pr$$7k3hW+O|s zzpwHLN-PHR4-YCqrF$3H(y_Z?|H`$pXS*j5exj=U-t;!B5;|H^UaFX&$K7R!0lk>2 z53RtFr`3%VQ7>&}NT?U@UE7C`1Nxq4-g0LC#7euu2TN5$RA1^k_~hLSX0@=R4DPfJ z=e(TAJ-d|(BZphUxL^wR+g~6aZ#<31kLrnrI`SurccW)A!`fyw<&oEC(M_)i?iH+( zr}iC?m#oKW_q>|$?41z;1J1IGTdGKdkeaxrK?zV@!-%L2P-D;_){?#}b=&b6R^~bb z)g^2Er!4Ox1jf; z0i0?d>FqcOh@8l4(b+WxuU6x>&L;5hVNI~<`U!q?t;Jj8C{*>aXZn4lYsu*JRKlR$ z`0jTl+VAf^CX7lI4QI!o(ZzX$DgVUEp(WzxKsw(vwm!R1dm5(Ix~dDEzYB7s*T4pP zKVYzXhR$K$A2{=2Gh4CIKxR+5hr=^Bl#Q^FXcRr z)1SF_+#c=9tu_+x+d!SZF1%AgW<~6db$ks8U-bFl=l$uNp?qvpRvp#cRNfgjpDJy3 zm9fkk`h38i55jW73AR6@D=&L(44;YfBAm@d#H=k$zs3=uyoZ8SO=bMep@cnaW!dvf zFlVh!?stvknPqiwdPqP1=2;k&HSQ^A?a?T2T^W8`Ocwt z=-Y6&%Om7~(#MclM*Emv%TpMlQEiBkttW948y9YRcN4s&cNNVWG{m77tI1hc-@%>0 z9q99HJ%qZnlZ5|_IE81{Sc{?a&H0PLr_tQC2K0W=L3Xk&6Imrg1B$RU#U3 z!hdMK^)IUY-?FwUCyb}DRY$59pnOvI5?7M4*_c-KCB;^Lw%$+gBP0Xy6^-dJx~wuI zY#GBTX2NGcd!W2_E-$5;%~*}cGkf94vkd!&gFHLjk&UZXABi(4_kH=B0Y`C|-3Mmc z{ji|ihrx}3+Xoios`y;lU+q&>)Svh6b%UNstO{pZt|4BK#GQMw(!%FJ@y0v@Jz2ln zKYZ~fj`dztq>Bp5XPc~k>In0}H+4OJc~9SaylW@TPHw`$8@Vc%MC7z#;_IGM@ZxxD zzNF@VLggdz3pf6C6DUvl_m|Hxu=*jKwEdB8#fW-b<^1Bs7GlcQt^9L%7}gB@$KE`v zRqcnWaM)0{FW5~PB{skQ z!G^4uBu($ukTC&^G0r9*j%S`#cn>rN@LOamX-vA^vzr6)AL{R~DTilP6FzkQZx8#H zyvY6lGINe<>}y`^BCe)$?XUHQ2Er8#kNvB_jN*xR(ebMpA9`_vcB=ataeUBDAg;t2 zmXC^c{?d-D9?z~>(Q_(VbxF7ZOEeEP_DTjPMRMCW-s zo$6|iICV1~@)P3G>RtnWFDhKsKQCAnFFQ2c2Z~3{K6K*?y=gDc^`4A$6T5ukD^?ko ztaCiwl5g{^3UOg?Fx#Q6szu7Jx1i>)ct1B<4+u|z@)|y*)`FOTA0n!u7ZzQp!i}n0 z^Ad~8H0Swfl{6W$9S-7t*Sg4#k7S<7g58R30>P7Vk=8*cOqxm3toZ0gIn}2#FL^r_ z2+J_EhBK$NlB5;EXy-}d&p)b8rR{4j-afhr9fCrz8tr$MeySPwG`5tMTRP(JrC+oo z$A&P)zgodDh#Im*JZ{y5>U_WOdSj}z7^8UNo&Iga0HYhs?xMtNADcKSy|}X3LnLjD z&9|<=v&Y9_1M_*16F5Y>ZnzmqBPmV7sb=6rt1Il))T20SkB;)|JyYD@dc#vtdJ_*G zV};Ad;q&2@Ne4Md(gWf|m?IQqZx;ENF2mZ4w?O>O|8%79lJ8W7_yx8V^?OoIgH>13 z@b#nxm}Qs;U!8A5Rl_e@T8}(o@V*mVXH|H;(Vw+u7qTia{_L7p*rn${R9f+B=}WLF zDHhXDXrvkUQazh1?m?Jy5RFdTZ!4m?+3{CPB%hJ1ID_U3q`5T0O3LLF zB+br7cx}T@-+iEOTQfdqttP7Oy+E9OMv(T$PF}?{kEgnK6I)B2Faz>yeCQNQ^=HPb zNR4!@x`s1x8;}n`8lQ~so+ElKGoqZVD`_2wlME$cDwO-zg1ZmtOgmi?kQRX8;$3Xy z_wMRiELeVR3Ho2Q0L3joW(0xmgAorc%hxQ= zOx9V|IuE2VdCIapM)i#A>#t*r7ffW|Ji?H)6H?q!;WOnLVaFjHygHG-Q?De+V<6=X zj)M(ahQ}iEo^#RTp}?Za@!-{@KYY(2Z)e+vU+8rgsh;F)3sYI*aYn@DCy>@iW4XT$ z(DM`vV347aG#k|_cTh=vKJNM+agd%pBHV#X;XS4H{S~GiIH;*O6EAbIR8M-`{RLDr z*e~R_AkBi_otWr3aU{~*fjk|O|CM2ECKA5jjpkK&=?n+X2G6Cly(8eyHftQ(u>;O& z>H#ipEy*{nk}i$x!GgXoAe}-U`ZLCb&0yp!kn{%p{5Kj3ms4-kZLj#{9>j+SZo?zT zu88qhyhY)FX~2eOlJ0G(HEGsLUJ5^hjSLP8(xi;E4A*SObMi&#HlqX9itZ#8#vf=O zDHLWky4FqBPESJ8O5k0k8YAC?Nf(amIz@4Gnrkl-TIPV#j`;~q<=$qg8u6{@`^}Ox zvOmj?y@GTd9qA1qPJ*N^3A!3VCmH23M(hcO$D_lYpPPrn!SL@?hvDMq`Kdr_rjZ8G zR{8an{L%pBTXC58B-|G5#6v9HB&@|$zz9SKv}(`rp5aTn`< zD-lkNK8xe^M*#UTd==Lk>-0S-)S6PD?&OIK%}y z@+4qU?Qk^gyTXQ5$kiTZ6-xRz-UjY1kK1gS?rzPb`856mgQ zvD|1sKHJ@eO}B0#$tzLK1=1PdEy>5VW~G^mFB_ zuC#;uWADJ@Py3-qj1kN~yc;MssPtBh$u+vpW)^Ol2BL2ro@Lk@i34>(GuL7K&8vd) zUttoD8Z@k8u1cTeJp6{_RhaVdOoYRh)v) zuDsBw7b2FPJjck($O5+m6*2ay&#+|eSETiq(T@|9&ZK&-D~aQw%k3xZ3cn0q$M-O* zW%56QJkJcKu!eG0P;Y_fUE2(&y6a)CPaU?#wJ(ytAs^G2(|B~VoIRwf57i?h?vh(G ze+bnlp&kWzaY`Y8J~*VAVOdAUwnbi1$Eg9MT{_JvGe0dq_D2q;)I& z1^HSe-vir!P2hxgg76EQ+j|1(A))#nb$`~71zCfs&$1PoxZTv^zFyUlZ%68xh)ds6 zz)(A>^fqr+DM|SCYQZL$SL3EBF|cd>1TAaEYoYieduJVd(qb%pywe?BJ&@K`L+`)+ zbd>u#>PwKPpyy?W)MS(cSn%u&Dh(Xz@Bobqp0SB5-z#2Y&t{&+)IR#^+SuvrM}=Lu z$!du+X+EI*)Y(mIz^TW@=y#~cnV@tq{4)&#@>Y28+dZA-w=$tL4`Beb-Y{9t^f%flwqSDXW*I90oe3uNISC9It)w$}`L{?1` z<^Ew}w#8N;4X2~~vh5wNsT|Upb%{aJL2#>MHq!Vw^{jA1pIkwl0MlyBM!!W{!P#lK z;#l%cjcGrzej?7Mk4D$S6)kpCPo)S#+BlO&qBEIplq;S<<>~3VK)obZJaHW4cGxMD zM|&1u4&;d#c}t`}GpZ@g$wl z*{FwX`Ro!PyslU)!e8Z4sV*j=>Zx?}2~oXPOqzUE#f+YfAENwl2fd((h_T?u#0*- z%Ja&pVzS*R|EMM*5bct z!qlT^QO}e55mRyUqkhU43c?MkyfgJ=6)yuO2O#z91=TJp-A4Z0nLIr9r#(GS>8`V! zy`7=w-FR$x=PzvOUlVmXbpDDv(+$ENc=aDb(B4I&!^=9k`+ubB1~lG`e`9XI1h z=#H%%UD_S99vebn6KnWrZNo!-3c<2x3y2#A7>3`Nok_eLx1)w^>TAlU`u`Lq`G#^s z@4>tUonO+j%_+Q>)DGU?+lQ0ee1%9)M;@Lw2-m-{kgdDc=5ud0;nNNdm##ih%;?)` z_;PIvT=+N(tTrWK_2&)H>9HAHJHM2l>b(ieMn!0@HxjYLsR3U3YWerHPyr^}TEn{m$O26Bk|Kt4u15Uc8Jg&mix;LM^y_&BaHkIpob^FtQP>&=g0 zLDjCJTGTk$*>JtOCtCcB2NjLZ28*5efjYJj%Y^{c3@J*ez_{jjW1YRReYbk0~Wly z$0%obaf?cPCb+O=1Y0Oua(m2rk4y)yMSw5jswMrb)oN(eTOEo z8B=}bq?wO!N<&wvV}0OCqvrh8OGo~#vLhz=u0YCRKC?s*RSi)d@lEqT!N!^WwVz{UEvxU%{iZg<;@CmnX9!m_kh~9b|VaeMnaLM_zS{E(g>@e&( zxSe#pVJZ2TZZg8}A-2mnh8O;`g6pO$@m9-A&^R>+irfXJ1}(?k{SH!Hl%wf@Z-VZL z53gT_9Q%EY<|KAwRdA)v(+=Pr`}7gcaztc zGPW+CQf-f}cZ@y1y{@y&u3VXyvl+ZUGTS60X?_VRb5-o_%cem znYRX2zx&^p!{HIpti`0S;_0myEbUz*PGiMA2W!ie`vd7*f-u-zyA#Fw2kM3Ap-H8A zgeQ$~rSyauPweF9Id)w|pXVz6^sZr3sN!J8R*d=LOnHVce>@~UtHt$K#BsV8402hf z8}DC9wCglix>r31UlXnfmFFM2SLdHUKL^LA7j#k7$FR@)0pCB>kXITdU_E+gnQ|T% z-a7*S8Mw=ku;JLtrHaCC9OIM>Dqj`;^jLBnY=WYIa+L7QkG~&sl&xPJsW2b!Z>u6# zPdme?J~03AGHr-wWxm+Vn6KNwxEQ>Xot@uaP~FHz-JZgED|;!!r}JeFhK%YFDSpCg zQ!5z!wFz(BJVI<+)Iqj6n1FQ~m!RwDyW)}E3anXQfXZ8^DMOI%A@{zzOSqf?`MtK3%oBgkJi_3_c+J1Z0(P#$8StH9z*PSBD;O?Ed;SMh?|5uX1{9wzmJi96pQe<>V`n?B~K`sOWWwquJoHe;VQh6}2BsC}oZXwxhm z6C={$w8ebzcsLARnDoSEx2fkld^iuKcOD7bMD@}8adUj4*xAq!J&p9_mH0Atan@0_ zj=GpelPC|1byq)_z^G8vCH!rLwcYFUNY7(H+{KA&G2o(+!Y90RB^HQx@kyl}Y4e7!G=Ss61p#aE;)4G}gGT-By_XIzSI7Tg7k56@Vhk3M`2x6ZXc>Mo7nbl}8` zNL<5+JNfsE^Py#SCh3vyNSMQFwweO1o({*hw@$R)!gKn`^_W->=~NbLIrh zV@@zyQ|6XP?_P{{--Cz88Bo>B6U&2o-%b3&W?z%H!oAKKD|8T^W6mj5^ z1=MgkCtm)TC|ZoG!n<#Lq{hYkwB4?9+wS@$7F2l~* zNp&yR=NH7t&fz-32fTdDi5JqI!_}hhqW+4;NEm{h-=&Dg&1tV#mo_rs>^1cKum?#~ zRs77;XbqaJz5)~jN%@KKJGw)O7{U}T80gw_`yMXP-5>?1_BGN!I^soPd-5SBn}mrE z-9v=%D!{r{)iKwgktCkO(p5W{TUs=o^}9jLYS#%o#>WEHzpna)V_5KHEgyqH-y z`>BhNi(>o1b~h_Hy7ehsEA6Q5^4_F0jhwnHQ@mW*nv*UjjB1ZR3Mz4ffu3;jVK0T1 z>}`ukm7`Agb}WR~H$&0I!U+5IZY+N-880U8ZYej-p9k%rHUG5H7%4tT`Gnp-hthvn z!MQLS+_xOViS>>!;s&G`$z+pSpm2B3Zcl|#BJzAb>TGwQ$7>^eIMfU$jM{}|#pd8> zt;g>K7-9B;2>G~~hi*syd7Z-GyO+CS5;Rn}!gV<{pvSZ@Xz=v`lExP*=5vA{;>hG? zvc8QakcP*(54*0%!StX)(dzF}+~yn1-~L&w@Q@vBeVI`{!K1t+ z8nYwj4```%xIdaaM|Vkc<8yw@VN^?kI0z#C8OozuC#(F?ZPm9x$IW+e>4-a!yzQKp z8q!5;FfSRqloT++4K^mA?xN9XVMAM>GLV*LeD39{&QvT?^$t`qw(1-R8H!YCYZ-oq3xidSS9B& z5VuRhaY32_8$8R#W~X{q_=F~lPQd+!s15CA0b}|P;l$UpZ^v=EuO)`wD5JfRR>6a( zOBnGdc)Z?-rICFgZqihx=aeo|T8Z?qsq`J%iYx7TcG`DA{OkNTF^o;wbQ?k%=V9KB zda@ntXQX8$#Z}I3S`7#%G2AAK^1B*-s#!?fL1$ad8|#$d`&UGF?uEv)U*YrIC^p~l zB)hUY0Br*93c@DnHqaEiM>XWyk`z$V^c|+oBG3{uGB<^QM^Q!^*Jlx*v9p#oGCvFh? zgPW2rzJQB=w`1w;d^u@U`F`RAoLuVslbTv% z`geSNv!>z|ARVit*t2l+ahP0fBrIQ1gRA+Fzf~RwC_m)k6LYZh+{f7Kj9^JgbJ2N3 zGFx-^Dk{$~-gkg_lU`X8-eBKHM;ZA;7!a1AsTw0+1YJt{seB+@DJ6dW?cB9yA`bSm zC13Fdly4wy09FmRD;=kNV}-Vu`u?QW`CS{?v|L}_nR$)4FC0l5kw?uVA7~1h3ugdf zDJBN3Wpw|FTqI6IR#&HWpl7*ujq=c`{2@Ip67{+x}RPTVifb^ZQ4pa=O zH=}d~thqZ>aT~Vl^pUAv$f3T48sRoS)VDuBSN{ftc*Oq$Ykz380b)^<19{vWnT9!~StGXe+4LtqmyjuBUjPPLX(zR%Ifg9Cj|s(u8G zse;4#y+v%}@d>Iw#hw*Oob%&@2sgj0DUKiwji)>uKn!li=ZE|N(n+w)_6pA4b(*lS zGXM0u8Rchd)pL=fYJXEWLVmp~x9qt`Q@&04BGS+rtv~;-PH@HN6%H$`SvjysEWOr( zFL}FKGti}rQH>4(^((O1m6>?);aPSkU?bS~ILSzNifPS!kp9jn52VR*C#|9$}kS)72^J)S8%gVTq6&}j;t@$fqh2|tH?>aH%5x5B=10)HbEoJpd&AjN%V}c(%=~f4x!S-idzWlu(d|}evj#< z+rR7?W`8fz6yNMQxY>DpxifQ`W5|E%>PXcWO1WPQRI6fSJoOOUEQD1C_n_sD%NVss zCz>aFBYA54wZ@ct*=+@?F$h1V2c+Y8jl(0L)~{R4XHXcrZ&(SWDHwSv%E#)G#wSSg zNzy>L;!~W`0BOH`c=qQTaaSUsQ&`(XB4vw%kSf@-KG`Tsq};M z2x~~{^ax{UAK^Nu_fZdG9(oL)r1}d;>nkXilm=AZh28#n29LBT6%;S=wMjHkFH%s> zi)72=VqS~gNM1{=Cy&{l!c>e2vy~4<9sxVQ_cRIw7NtGM9Oo%vF&^DI4%xJFSEmi$xw7B70Aa( z!e+&HNIFH(yjk7oMs(jDXi(7H+0~~Wbl>PL0vsnX(qrU*sDDniZC3jWm;P;(sp0sGevp z^$wJ7*JhhsW#rchf9A5+mqsA*7p|+in!ML=yb^i{-Wu8~%_oR2SjOf-aMZ9|#!5`b*Lx&uA)ylxl=nyzKHs`b3o~X_Q9!`cj2z$ zQPBTn1Ee{?F_zw?=--4+sdH4?i_`TbBy{DNFIygwL+LT`c$D&3x53*OCGzz+ZJ`X@?f;4 zK-{JJOVkshJ&z8xuc%GMN2bmf@Ys;WaKttgZ~yE`*tePb3SS`Iu{lyN0&E9eWr|b( z)o&)L2P{dmQ7-Ofv~J9L)DzX8mc1NYHS)8pYn2(m>S{$$WPWm+PFjVCkLeU|rXU%Vm9`$70&E$aM{#sM-w& z)cK}svd(@IuQa94Qww&OFhK8Tw?3|aj*-SD%+NEte88tmC!mzQe3_`<^% zi*GH)T02W&RW)n;{?Qq)rMU8E-c4k-dk}V+JVp%6%+Q{fUJ=`N50j?e{W$AvCz~WE zO6!l$1kDS-y+{=?h4%8SZ4m^PIm<%}E}%tOI2PV8GBjWbMdEl{*%jnLY z(84uOyqbTUpPR6OM52 z-)}GCBdU7HCztBWwXIr;0oFON{7s6s>hlvYcZ-I-oug&%*RS!yJ72jpXBxK{*$NVB zh4Mu66x1CxgU!$0Lspa1Vo~x%X4m#1P)+e89qeWBzFb({bepO_(PP$r=pX4NZDJ2G zRa2*8dZA}}6HeC_zY;ftF2O^)ICDjAk0%*ObQ+(QwN!b+fBx);SxtJd^*Jfxzgd4kZyfF0=#!5-ZVy6(B|8P>AYO=B z4G(vWM8cZf8-u&c!{ zr2k{{#NTXSW(PU(_gZJCZRZ%_6yB>gR2mnolW}7mD=_kJ+X`G_k+BWrNz#vALtn#$ zm)o>SYi;C%tcJ87_FJ$vXd|Kyt%IQx9^lWP1-NEMDx{mPV`EQsmCN6>5ek39{hjF< zne95lAatJ;tU1*f4DCJY>Ilyuj`mJ8m<>=;&;-^5PZLqmd2-g27?%6susFZA9v>(o znD(+eZ?tnOJJ#a@8g0sj1{;%*at=q(e*WoydqLs2Y}WEsTN!iseQv4t0%#pnZ9-w+ zFFJ)U0j-b2?n<6`*5W1H-P(lT`FBP)a_w|Dx#SDptJevuKFveIZ|T?kHKW|%RL2yn zqYzMcA#=M~OILzA*`~H5-{iFuX)fIGu|H7$$iXwBSo@PXqRgiQzxmD@BEyg2J{gXk zpYyuBV zoaR1yr^L<2eH1>*;Tt0Gr12o>+;$hcSM92#&&s(ohqM1ly6(81zb>wnN)#%EhLwtt zvijb0D_PlOkL;bjXCfptlqe%o2vJe0?>(0aKNLcC$d-|jZ1SAj^T+eT^Q-UY-gD1+ zkMq5sdlhaI*I-Y&7t?EG76cvI$~y!d#Z-fvvh?3tUPgNjXs$@LMEBb51?}Kcyfdna z+%s|#@A9Dz5EqGW4|~gc_Ya60?d$VY17AG8*+c{ew2@Q;@IIowR9Jj>P6U>u#lXZD zb20t=Cb;7sz=*df|HgrRa9y-tl?ufc8zA{>2OeG{62oiM&NY?4p!R?kbsYk_5`xRG*` zFvS5*e0c^mJ|64q$Q5^+T&%%~%Ww-tK|8^5YULvwtBH zqWtI_iU%;@juvk^JfppnqZ!pF^Xj@pxJ)NJx9Wyn{f97BzwPzRp%dw*VhepacmbXJ zPr2LW-ZfZjX21g$??;uZ#Bsvyw?65I0EL}MYe8}SDApPt(#$Gu%&x`P;Z#3VJ4x6m z;)!^R@{)SGdL3L|Hh*BOA|@!4b1Jh=(;3>@M0(u_k$1J znP}6^h?6$bHXY~26>jbQz<_d-I|aQ%>l7D!S2s>%7SOv;nKQA3&LU2Uvc`>8AG4pP zT!3Tm9r&ObIV@n@Skb1J3l^c4_ARetL5FJL$h^&Pu8T3>5%&*gT|bL$@?$Z%+D`lx zwjYOVi^J{4D;VJ@Bi;i0hf9r3#S*Ee;rpj-taxn0OGoEkA6XiPvl3s?5b6WYPkdAGMSWo}aj zs(!ESXsh(4Jlb~-e4QRmcw^6Nwb$Wu=d`G*A>!B0gb(Ro2aKq}o(TC`fK zb*+=~S#hr*-Gm1wm#%@eg$ zgAn?<3%ghT7Vr>F*NvE~8Q zIgMXwWf+~435_SzkYgXUhY>ON*|;$UBI9a(B%P~t4Ui^bq+gU?ft8c=B;hY!`Qd{{ zt`0}HPl>8VpwfH+7OanAHttDSo11dd8nEj>KS7*`gc(42kEBb$DafDYw%$W|9FLD| zqj2!*6gr=+nFtv7Sdivm^xp}`7xK%aon*&1ztT#1HuvFwqZaz8AHpPDtyGk zR)#F*JjVPc?ibUu0`u? zffsH?WGj56-zR+EfXgq~NYbsaA1?)dYf~BB;TUN{W2`Z0v;<)A`HBvwXYsOF zZzS&kosA1c5gm9Owe>kRxy_KcL6Sxw&2tNhlTjC6v6jv3Vg9##m1|;*=QaFyq&Ft} zd+^am7ppuWE*%HD7cUB`4nqy6=@NLNF7uv%QjQQfdXSIZ1xWv^8ZcovrGo-vkaKE)c`H0=u zbQPVJEa5G!&g0Vy2GF-_1}c1sFx@XmOHm(u4;*M|L^W;+#KSOj-fyjTY%i{K^s*iX zO8YU=|4QHCxCNo0lk}aPs63~<6<^uUo0AUD?pCd}G-XJ6AFHJt!CJPST>VKgVar@D zu(QogWXom;I}@kTUg}~P7EXKL$|s2uhj>=)jJx~w?GJ(Qgt+3C`|0F`P+XJF{xS36 z((t+;<)jZBy+FtS-Q3{9YoZ|IRVx7ajtx0Fb>aYY@%4~Hw%NkZ53e*MEopz>E@#YW{Ro|pUB(feqmZ;D68{S5 zxd%x@i1B~)WZ(YvCB=(8L-%=i4 zReVR@;xENEgcmRFqs@ySMCZzQ@%s}VBIOz6C7+~c_L91fLeT$aJ$|ae2l#g+AE-@) z&8K&Pj^D=-eN37a*#A5>VLGS`!ox)jYD3+;-tpzgk9oSSYvTj=Pu~c zJu6gGXxhUDtWCc&m1~Aed&7*&>*beTbUw;P2TZluE@pBK?{03*ub6BBr7cEuHX~o> zS{2XE$2KYNhAKxJ>@^q4`@UYh8D|ymSN?!-VhWU2G$yRrCVtGlj+B$)TlrQosxO_R z(aM&)+nr&JPy1KR=lg^2>|B8^b4oO+@;nr`?AV?ch4QqkC!}c>C+!E1?w7$KVjuaM z*P!(Ammw|iZok<~;rPXiV>rm>JSQO$NPh|MJHOR`gT%vqd0M+5R6Ig@n^7+Sh%>A7 zrSjCOUawj<;KctJZTCh{9nzY)s{Vn{@kwJ>*X0PyfNr38kLnc6%CZJSgMlg^lk#evSm$@G2MRCHyZ(DbJ%k4sAMGN@0}F+n+kQTRH5I2>{F)C+ zODms^V{Z-y(pT*Bm9^9-h=R5r19)UJQ{4TsHJ@?H8z`SOge#md2GtsbzRuN>R$&Q8 z>_FuKc{(<*_a~9o;sEGPIf4E?F2T!T8mv2@70>8ur@SGbb~K$0D7cTAcbZpuKEu$h zg0vP4>^)D_B0F-gGbijs-fk3-*F>eMR?mtA@&xF2<^?l&_E|$d6o})`mq^-MOZ_A*#Y}lo zARWz1EB1rpj~^@2aK28m`Wa6AOnrw^Xm{flF4~@k#8o1usBWhDQX@uQU&KeJ!mxv( zP=4E#6ZYWC3FVJGv7)yN| zeJex6;))53`WLwUw7HCZ9mwLo_rh+*VM^1A-&R(vrbRng-mSgz(R|H6It$J$7`=6? z@hRgw(s?f0`U3-+q$S zn1h1qKug+&tG7QM!17mkfm*t-vEf$Wkf^>`Sb09));qx3WI%H4=61HXLjv? z3HoLl=hH3e4!zqN7vD3CK1*C)xk2^yvQur$8Tlt>WoAQtEF%t=HmLl7fF~D$)`IxQ zT8`@I#h+>qA$fGJaGN|S-A%U_mk+SuR6n3_lJo$(Vq(k>wmV;iGt^5`{uvY(su(KG zI5{SU_@qR0!zlyxe{EE~5S|d{0W0XPIqJ^{>QxA}o(pSg8O4-QZD^?913TUBu~$#p zA^8IFId?yjXC{(AhsR46s#;=%*&3z!dLQgh8u1cP&j+ZFqETMFUtR&u9q~e(_smkb z%m}Zm^gZ#J@|A)#nH-UzC2rrRd=U~4g8qP6jJ!8G=NcMzW z{R++f#tUOQUx0cdpgbk@7a?s=Oa8Uz0?Pl>tWWbPgk@bA`911O{$V36=Ly0#)#GFX zziuXddr#?g(x_v2EyLA3+HMA$R|e!&yJD=Lksuy`b`LYe_umbnnr8w&(5YXQ7fQR| z>fwr~hlk97?`?jv5lUQjrNUeL@+=K`IMQ`1VM@~oW|DSZLp9E6 zoxn2SJSZ<;KX4~#XB@4H75P@x(_*Ay$=Ca;T#-{O+p_rm+ZcH%E%oEr{`$L=USp(H zMfR;hl6vftdT&5^ibLykpmltY#8Hy^2^cnY1X9fc<+!{(WgSxgQS}pO4Hk37-Ku^` zyNc)ztyJI4ov>Df)?1AUk0LPB?X~E-g=6DNe~=crTBA%y?SbpSmwM3s898}i{+sr@ zZmK79YqXJX9jv4@oXxiW%7=DmX3AlgCy2Gv=Y!thMVdB_X0U!&0kmm!R?KX20$b
y!93x0}h_sQE=k=Ls_nuN$SF zWX)la(b>#Ou%6(=ehPfb$kYrb6*va4p_@5YYx8x*1$0S-U2h{y@0#O%e)6`NBgiW~ ztilr5Mnj{>?LeIIDU(|i*i3;L#Ko1%XyGdSe#)o31ZNpe)8=@)HC>ztzBC@shKrE- zMfkJN^nJX$g!YQEC8E^y_)~;A599CB$SbQ=e42%ZMkEXS^p%z-MRp3?^-~DCnB@}_ zDn`<0pJ?$Ysv7P6wL^I~|09sgsd%A3V6a^(ZT1$0b7#y*=;QT7VZ0DJYR%%6<>mF? zGK+ZOzY+qqstgBWuf>Kh@8NCr=y`>8gq&&k1NSx0qwYTndCjHp>S<>g6y95a@7ZXg@n|rrXutF_a>Zd~_zi0zk)>%=O@)0L1jf!| zcuAng#mzVQ&0NXM5KQgT<90Qr(fny2rxVsn;pZ?9anNUvyNdC7hl#ygT5yrib-0#; zgT~sD;rfwdZdlW*`*36XW(xi+F$a*(E0khriSY{ND%UJUV~goSTSBPPzgVmBZOmZ! z5kd-7uDbJ__VV=yxfJ+yo{aKxrkW)6p*F zkmn97%+-(+$C!-6|E9%J!BbCF@P&5z$EJ!4dS|l(#!Az+hgAwpM>lSpZU~Wdv;;QG4 zaM+CDW?sdZIjHS>9q4yvc@>y*QC3YZPHrMit~GO}wUuy9(X8DH`CsBZOy8@!v4^ra zu1ujLoID{PW=*Tfa3uVk&Yhj7e5Szn9ge9z^&To{l66W{5_qa6gEN(L^@?WC`F=$k z;hx%=F)z#Ng^Lljw5dgCtkuka`nArl+LnKn#U$`OdL3Owb>h@nRN$o!s=DnYqLbLGf_1?B}&Am{SSgxiOp)cf?g2b`QIHNIUweU(n7o zqrg$L`{Es{9oL$SC!7Cnm)2Gd*zd@g4|>h7ThJ0PaD|$W-$CvyQdFbH+3Zu5#4&{V znDt=2C_>((20eFh%oNOee18Ztn=#x!@OT6Iw4{ti-5UL{ATN7?VWgQeKzp*h*|3%A zyNu4bum8Y$RsGp9Q9pAI!^?S3y}p(tYQWxK8cL^redKtD$(Hk7HgLNivt;V@L6np; zj%&6a#53onlG~aXa+%?ypI=$Y)jmh^)XR%lUD?5LTU_+U&37pEZMty(Ya!PcouvwU z%;&q5mY2WRlQWAcHTyw5{V}*RU3$7iM!nl$2`wZ=-;-NpQ?IweX{Rf1n7Wi=gEFjl zN4}xxDL>Tq`3dy=+HZaqFkjkyt|-sHSZrBc^o5AdJ4fDHah|JPs3UFCW@$SIdp+(z z75y0ZlGr>N%&k-UsF`zHSwB4V*X|lW_G@WV8yRQx){H>)S3lxlZA#1Z7fTzp(Du0BCbPphiMkU$ zy)O#KT#MK_bw0BfWZ3(wA_3InNz_ zb+MQZI@3lcsX2u6(f&(H@~t8z`1_Fk>~Q^!TD{|z`1r3Z4=MSLLfjs)V?b|d_jU(O zN^U0)x)&7o|MgN~|3M3!4{*Tq+|udda5lAN>g1sN67_w`4IVsigT(QRgp83CRo_>e zdcryDz=wG_DfEembjn6f7ALFO$ABi?*rwfh-FMS5ZgFQFdz`E%2X`oj-&^#Nt3$qW z%ArubI(xD?Z{C@=sb?09)2j>%1@{M%1~Ut(4Aj^_NlWHKd(JXJHr2FsalE;96R zC+*}^SH`(^pqry?aSz5ZHh;H!>n`_S5FJxlECkZHkH$kwr887dDx@bOIp=? z4@J~MJ$#AvIzOZdqkgDxqYRpfYZ2Q0up!%4h3N9cA?*6%8pVv8O!iaOlJmLuoE#ZV zaes5`@A+DA#`A$PX;(8`Z`PdyuD8@fVqb`M8~!wiq?YWzrI8Mc2-n-5ZYI+MuP&}4 z{I_x&e~Rq)Y@34ema%LCSqlUIUdxFj0gd|>} z^Df2oxJAwN(y%XTQut|3SonyrevCRW>p)4X%5lZtZi#lX0wU+iIyw@9`NwJC7Vn?dNyd%j$$C9&FBUdyTdnc=0TIMAXnN zDzudTJtEclNwX+^@k6#-A}Qmlw_F-~L&Xo9rqRz7T1X4;D$QkbNC3}VIbW`-v`r*` z{wy4RwpBP5v3=qsIv-w9rhFKqtM#oQosYDFK5Q`SuD5rLQ;%x}^N33IrNiL++F;T8p5QAfY1=7*wk#E~78dFi-_ zyyyj;e9C@(myrtV&tdN6Bz33^y^Z9^wz(;2>wje5wSjbRIhf)PCR@?J%5U`yg>}%^ z#)RkrIG88WeT;g{LAMj_u;K1>XM@>Y_~f53Yz(- zuKGBlmiFWR>?igKKYmQ6R$N+;Hg!BWsiI8(Q%-N3{+(}qs?X49j&-|4I4^Dbd|IBV z@?%eXIRY;e&RlI#afgpnz=HLJ&-aR{dXb%vq>ASyn_&H(DvxGEbjsEi_?}wQxxp%m zFHnOLj%TNS2M22Oqk#5X0~WYY^slDMcjHQm&Wj$t^DC>@(h96&_<^wbyp7IJc|p)D zPW^L0L{+H7_BZB8r=455yJwu}IAE!aT;*tznAe~;kr>UpDzwB%b!?QNOM*ZMc*_{Dw6{5#fx+!{7wd=Ar< zo@DgE0l$~Ry2ws7M)9P5yEtS13WlHX<@uEqJU~1TyDYF*3Yx3Znr*T8Y%7xGpB>k% zPCz~D%SU5H{IEa5{&ySN{=J!qYk5%gEwhNG=3dF;@<+1SGiVuIjyg;J!#xFh!?I*; z9ewxyBw_pDrHY8_!hL`55=md|rAyC8G_vSK&U_Ti_`6io^EjCr0z=3a_2!XF{prxv zXyNwsI$OUj5FNi}=Mnew%IL=S0_UYN+c}WB;wGWvYP`cQ-j+M3j_y)RzN#`Cdp?3A zbK<#xJ?)uxFH>)OFHqY%)+Xm=HhlhtgMKyOgP3;o5@i;%;{^MfGNrzkESoZlpb=_P z?cM~>;-IasRcz}60vscrcS;p!UKijPkzd%3y2I{=$0=o3OS&w^v+X{*SdV{PC}#W5 z79+Oh&|o{u!CMtQ!Fk$$%TddvJz-+wpA*W~|Bzae6iHwJzI?QXH6?f;8Q(`wXya2J zw{D~F$BJr&sWSjWuhZ&`)$ed5=O}QEFgg z{fa8dsh0~#kNpKzHU7frZT7oR7VKdwP@lB4@j$^4ZSORk~=os ztK4#XYSWu`J?d-!;YY}&XFtpFN=0-;#!R-cZlK71yE$#xTjksBGPgM7rNF`(EGb*m zAI{&qj^WiI<8Y5aTZaEB@TQKL>nY(y;$7d5oVvLs8|IU-6S?oJP{V0F^TJ||^H?l8 z9qq?pV&VG3k!D_SkiJc;oAZ@@|4U@Ay@kl{bBKsP*o6GX)E7zp)>7ck>_)SUmQbWq zFrRt7$m*u?OoYr441JX6r$ArfF&~#M6A7g|SSH~UFt7v@mleQNij37b9y6$j7CR3zv5td1<55Y%`|2 z2A^q%=T)d(?U4js;xkjTvFW+Ys?{xV%lhaP$D0;tEP3xNti4qNz1mfet63xD9?t@V zS`toeEgC(@U=D&Wsj!od8g)epL#H!-mQ&v4hlWjJ$9cynBPvlj&q`)6llCdRg<@9U z=eUu-DPqAwvqpIT@Hg^!waV~{H7I%xJ6?H9ouV@7T2)DIm*a$~%RLXa(7Vt5=%ZsZ zT}^mfqhD;InJeuiYEeYHR5m`w0j0kxRq`64=lEUUQ51K$F{_fDIrQ&phOek=kCqZL z4NZM|K+NRJWkIVUUBEl22MHs67MB;+PGJp_+a*ZKPp7Xl^3bRGm36 z59{h>e8}qkC5gQz<~95YPJC%H9tFLt1>V?3;7(4eQdll6IZ)gBd+6dG}K3?g6HYn8i0^B3@Iol-xMQYsX+ z_*MNRzTaph0$2G^!1~4nelVV{pd0GJf1#rLC_D~&fs39xeycf8^1s_$c@1tY;YZ?H zRZZWA)DfsBZgJ)T4QX^+p=LQ|Oc7m;$7Fe6#=ry8)JD|5WGm{NQXRS(FU|6!%yG9U zdGt|^`qzX;6kevG$NYTI2zL9roHi~xWY$fBNs#mA3UIK3_kw{=DZ`nr>vlrldQ;8T z&jq+o7;l@De38Kilsf&n73`k~RuZn)_R+^P&RORR{U;~-O}B8}`Q7_JaU36wA9D1D zyQY3Qs%H&pyvKO=-S-KEJ=5udah%>QpIkC8zueX^p9Y^%dXFIMw%l`tVS;KQxpdjr zR}+!x34S3=uFvZG;LceY)zsDH=igX1dd+6vP3?oNmG1!y$1O`!5rbnCj)T0boMJGA z@eZrq@SY}P3mN-^uN8Bq%N@&Tc$oli3jDWzd^3Xr=aiAv9{G_+*Gg>o27V@OE7p{e zq3Ik|FgxC7RaQ?eVKrPpPB|Qu(e2QdCkge+K1Yt283LHUILE(frG2N?)yVSlyGt(a zZc`V>RnmmfLS$!c z&aqw1u&jLlZgfp@?%kK*9|}6$6!u+&X8X(FcnaN8Q+hQSsyij;iCmOH#)7Ef&*fllE0LzAnsu8BOijv6C`76PdFyVg6vT#k;id znH;;Kw{TKt7+I6OyhFst3bqU`Ab63<1U%wgYX~DD;AcE;YBWOwY2)+> zj2RDl{VvY4kleGq6l3Ouc~pos`EwA34tJB#3c5Rdu7LL$y|a|PS(EqJ=t|XM1ksYwb3{&)l&eo4`qq3j84AW?JMyJWm0$LAj** z7WDCAbNtFMieW5~VO=qkEFHi>6%xp9I__c3^IJWuKFeaVJLW=E z?a>X@w@d*--4hr?VUA!)dvkJZZR_3GX*V zHnE^)d3F(E$0xglO~@*SPBVOyFZYinv<%qSZt5waz@RV%4Yj-Axy7)3N` zE<1L-YQg&Gkvm?o$A4!z_3c;=oU>7ED^Vp8y(GYb0-tZ``=6Icn9K@mY=O7xY9XOu z79TSkAm~fhj0ucqg-!`*KAq{dUZs`u){$S<3Gkq$UE>;zOv;&aopeCwm2B7ma~*QR zdj^8)_afWnB<*hJutLjp_vLFTBJR57-RQl986~9+8)ov2a*5uKV?HIKg2$7sO>1TR zWMt7Fd};h_JbzzXurHWD6k-||&hB?_@vVmw*)Y%GE2(UH@viGp%p~e?l7 zjK9}RC?M}<^Vc(NS0xUK9n2g5_~=Ppyanc{0`pO2*wD=6;Nu+qqAZQCdmrntinq^i zpluU23ghpAyIvBGKWj#UER{7Y-nhua0*{dJAmy4j!HS$^^xX2Sx{c~rZmj5jb~Bqf zG3wq6$0f&aenUo+F&kDsz1vCDgX;ca4psYHM1n6BW*`bZV7!?o<;x)Bd0+w>>|9Vg zTn{9}c;Em=ZCN9~l+yTa_NF%iJu4#B3c_sJ=!eXxu^S9n+hRE0)I`mMMGCVcayc`S zV^;fP240)f7ZqkRqrq%2OC$F@s35ItH-U3+n;8$swXAFAaym9>3TD?0td5s*$%Nyx z`Bm`~T(0h4GN0F?MK$E*_FOjqVM5-vV1AewbSGRHo`&`)^sh|mSxQ59;B_&E?+Co2 zFe|lUy|7k8ICYV$#@^GVJ$VROPhciUb}fIHz10!x4WQ5H>jjM2E^5n(-94IU z!vM(p*2mY28jc~8Gwg9)tD9bEt-;y#gl}zR;)|V>K6*d*FFJ)idlZ+s^eGBIQA9XC z51~n6dHMPMg0kd#XKh>TFAta&&YOxKqsm`qtG{)6YyZ|$%%531uIoYhhc}l+f7Fuq zdak2lAFQHr1uq_Wvb(+-Ur48w%q9Qbs>oeG2djT={?hif%eipJA^h#g3<_9PU0z?a zlnS_(pnJo2{Lj&K$JO|SaVsj3rV+A|RVl0Pd5&zMVQtnR(%ea3e@_kGV{udWBlCdLc-{w~pCKu#;T|i|?)$JY>dCbRTITN9=rwp{9@ESKonh z;fofsoX+OF;z{S$jWht4dycF|ViV5k9Fgy#!+GZJcKk!pa10(i2KIH>1SvP_@FJJ( zJaAcwmOfklX*W==s8b1}YZ}1S(Oty!pqAYHeq*H9i}9Tc;F?2}Tzj$v_pQ3YBFE(6 z$q5HFj}s>g`5;|b(Y(Wc-xglz?;vmG*OrY^_p>cGkHfsN`4H;(0dD6;q$R(eWlU^V=#-j62K4yb|3${KAQEI)QK3meTLP7~wxJ5&hR) zKw2L*x}Lypu?I*s$3}E+C_CG)=Fh7IYlmje=kZq$Vn*>uPV0aMHfK;L_pWAoP%Lx{ zZY`T<4V7AxQB<$hq;J<)er>@6 zzdJZ3sS|r*VYEkbG};+WZ)>(1@Vq4~E?aMsyazIb~`_)RmzHb4;N7}&jH}k00J95f}pjhy*&%G2L zU{duEFr0Q5M^3t=>GiTY^yx(REYu5t-(`lN>Dh?i`{p9|_dgEAHF(XT8SPax6Y$j_ z*+0_&O6K_BoCcx%@Vil1>wIUtKpe=LY|su{-dFyZg;+oFFH1ku4(Jz)ti4MTN8p+Z zkD&YFM*Qe@>fx1jgt_rBc(!CR-|KFPm;2R$gMXXzb<27S%DLR>wMk|ML}}_|7l~^v zb|7(=j2tss5+?Di1;#RMP&w}L>L5zyl(8varty?d&E&Y6CnbGu)%Cu%qQT@1wWh==ZG6ewKG?D5}*9SwI}8tgM`|n;dsa~JbBm#CzE99&MnwnC=1Xo}FA;?FdU8u8ZyZ_Gg6C zxN=7e{`UG|Sl_z^%&TKAW5!fKWvvEa7rOx*uA~9wP^5f{5;fo6g)3*?()!0aWvtvt zwWkX^XuhK`Cc487_8~MJ60dqm(hT^_*a%kCSpf$PJ5YS1Fanab-?A3L)LU-wzRg;o ze5v}83cG9mnaHwN=7{6(${{iNA&i{V7tPyDWQ3JsfF% zh}Taobr!=S3)Jh;W9KEL&(w}}a_4JC=!4<|>L)>b1wH?*hn4HhIIXQpj~G@9gf^p2 zV~yN1@<_q;RQjDMZE<|*!74479%d{_=V~Iq>ynoJ1%!WmL7_gI)wLSve4B;q?RLY@ zc5UU73uZWa<~ZrPelLvaoC;lPEx^3+rtmZL8*?4^L%xpIZL;JqBMpT)o;e~Tt*6wNuQAB3JDg4~7sQ8beZpK! zu^lCdXPMG&o_p)^-}TxGinoShjrCnRL)wEV&Bc}N2U8?7aR# zP@nNu?+F&hxgc>cw4mos&5iHUwVuW0n>H&wBM2XHb7e9UO?|<+(92EQYNCDj>_b|&q;O=)6- zjP^i!Y{N-=YfL`w*Q%e5*iu2Wybo)fiy~IP31icJ>;4U3wTvp91kHTyKAr_J>>z zBMqtxiUV`nm&>M@{Q}j;Z8fLy%-rjOG&{J@+{;RxC-YM0CqQ*2^uie`uR<7s5$?TU z|CnT`pSVWpARt`P{0v>Bun(QL{bt3}9N;7aB8~n{=}FCw z$_lYFIun~mX%*iH!eY_$Uv>WQ?R{ZW?+TDt1f^e3tUd2UdS04Tvyzr85(IfLe&x(; zu_~gwB+MqCWWvT6v_h53{d?wA^_*JA6N_L}{)x01j(@sGggVZZgyVudAEXUhiaOy3 zl-~ft0_~|DAxOD|2m6h3Ht%n5|$g|57J>oi6>SyDYy zPFLWg-1|T}T9EezQ;PvW|IQ7EW#e3)0*6l=gpkjcpgYY_()Eg{{zD{rW~8{XX1fwF zzQcRS{btQ)jx|Tpex&P-kzMS`-aM@)0T~K%N&mJ!>RH(|5vgU_I_Xu?3R;0GCV~r1}=5Up2&wTIEBO zCRl#0w;*jx-1!abjTdn7U{9sZpx?c{tl5VBIC%MbuJn@9nKl0uF{Pa;7qIMP3%=)4 z1?K7e&?^1wI;lELxD^Ve(-tGeUyyEKCkL6};lEx&`5(poMH^kP+wWDlt;Po+4+yog z87R#Wv1J(=I3Lg|&q+E_sv0>z;Ta?DWzY66M$hK+@qpJ2HfXZ}!`*eH?zAc&z(&l! z&BAK9;*x~T>bf{#EmJyL`Eljh4|>f3JM+ilk^go^UY+)Kt`0Mwt^mcKU2l^H?za(2 z!q?)1#7Hr^W|;Cb%piOs<*2J9i~#k1r>A!%uNtbfAN9sBE%h#(JTMC1XW2`ok*I%> zJP7I98iM?dFsxP!DTeB2l(s?gq=I@Lk00o&r8U4?y~#k>0@rBIEz(~=+5;7Cn3cAH z3C-P+JS9_od1m`FrhEh8B6Nvut)*P2k{4y9eSqpz&1Qhsz?F9=-3lisG_S%^#eeim zRJhWueWDNQg$vjVyP#l6Q*b<92)p+TfuTbh%CW!pfM#fG_Qe0bmOK?cdj16VTv`SX z^IdpC?K~Lf6b7#M3Ixq$IB8w9Zr6r9U<58|a0?FIp}ob<)A!V7dxgU}Thd&&(p)F4 z$4^=J=vuph4c)a%x{E0vMRlOK4|@&FWu%YA_o!l;P937}K_-LJjgdZ6@PZZXS4DbH z#gaak-G44cz~BqYqrvO6lnw3P-JS0WUkFfEs03PO+x+;ALyT|OpWf_U2RV?_1iM@fiSCOFm$#C4>gR}& z2J7KL{kP!l(Gp&i>LBrjCeYLjKHV?Y5{JOwuZP67fTtq3Zj4Yf!y3{`U5mmq#g$tB zc^%=yiXcI=N6o@IDa0 zs^JQjbtVUXjItMMMq^d4rgXhG4mQklmGQ|YikooN$=;l>MDaFp^%Kqfu6@{@77vs^ z&@OB6QRqEig9`gq-To`zR<|ej}T9c=F#uT=G4z(u^ zd;%Ye?!hqY>dM!!K0OPSN5sYN>5LW6qd?arsco~V2QVF;3TW0Y$o|Ha5-s5z|&kG8~{EzW_$ zlGrZKk!ER(G${UQcNJ*{q;1_UfZ`BHzV?KcX2ODI4xD_R7&<8qwrewCqiJ_FBL35pZ97QwM>9+b=q%Tz!W^mQ7G#iCmZ;kkw?Ow{?Ak6^r z*j!yn>#Fk0^s^fGnGID9VMFINjOI6lf7=xftK8uue0b}wknpd#e@0utj3MZCF9A-maM;zTA^+fFCl@rmg8fEq!(RW|r-sIz zLF=e;oDgIt>lc5*uJ!djBPTG!az&$p{o1P8LJwW3-iRM7yQM7;q9e!WDjttJrtYYGv*md=~(mG zWVY(UH2Aryjhs>DAxG98Cj);6qUohk>=ZqZ`+k@X_cpecfmd3<;jcZU1*d0#96RuZ zgYKjMkN}N}$CBD~Z`i5`2oBZZ)xPTUC#?peZPRm@Jv9>+Eq#vFgD*nsHB01Uy9X*( z(oo+Y8#QLqv*ufDHopZox!DoV51PTne?i!C9Ookz#)9>djkv5vN0w=3k1zVTaJqYf ze{paH$5sbq$*FMo)q9h${MUjfS=pp~vo*uRUsC`#&BB{S#n|}8XzB54j9eW53Dd{qIUTbgBN_tOA$b7{ZnYqcTRQeyjQ6Yy~BHS%Pf6ZC046%K8?g1c!B z{?WMrJ{S){o;O@}a4<)TJ*S^z9py6I?3|8=C$D3TlZ@o?6;si3zJu&H$`FV5H<8)D z_sBmHR&t~35RrCsEJXZhE2)Oia?)Z^8u=0|IGse+v;k#7^j;_%Po^1LoR<-IR$|moS zL$dS0al}(m-((~ob^@ih@e#Belm!D1j{>S)ab;fu&ZKjfPR@A}LrQ1pt;nKqZRq$ccG#ly0FRk3|n_7nx)l{ zsu#6&->zo$&O8UU`U1W%c$ro^YA$biMMryRTYpf!8)I+}$_Fh&Zjh2Xb>Aj@vHXOf z^+NsRb@(UZ5WY)21vmZm`F|6Ar9oN`?tCu->M$E`OuV)BVca4PpIQlCOFW2*Xa zQ2qu+`Gt+HH^Ag(BDOJil@l9OAYHF)Yi!2%`V@<0j)wePdVpx;VTqUSIm`5*&b+?K z12n18fRElYPJYx|l6G+QFu7&gQu^-DMjHS9j7^?KV_$Rx%bof7!I+*c4%#i6#+1Sl zypIbT)#o!T=?u4-BS2nQA`2altG?9qJ2zFWNla+$COeIGmlyh?|){rm%HJ=mhtX^QnAs>z42HgCF|IA8*NbRH+) zxxQj;{oWwu3|6##kyfXBKi=ocR-_zZ!&XKjVP6FHbn1^?|E}QSKX>EVu+c#ME)|xI zh@U9~w}IjYd9B?cTx^(%ryVr77zMzyHytJM0sdT-z}{xn;tP^3!Z+>_QVS1VX^+SKtMeA+gXJ-cZSq1vDLzOKlbXi;#PyGQ3VUSt%8Qu# zkWP=hvKLoe+Xh~pe?md@2=M#V9gLu(y#8?$r}t!s+co6}cAN*QJ;hrv>-2eCv7;kj z`Y;mKJr2Nnulmczt8L_*X&-=igsEB^dgdDz|479&$5^a9>VOmm=9i~PP zmb(qjSxLKk8-oQ#j=4a{mPzXxtoM`C3BDD?t!_%u`(IsViriKVgqnS@8Xplb~yt zC#dl_iktfWP;o`aX1@h-JENY3HLG2?XU*C0^nHoOS+^Q`1kAwkV>hyCt>0LL!|j`Ebxjc(d&h{B}GiRBz_eo@@h+4_5Kv_zUBK zYJv9FKZTy89|!9gap#%$SMGJifb5M7{XMDvu_y>*kd#d~_LF9Ant^=6Of~I@@{9S*UzsE~)2m z-o>MsHcv+?U6NXNXH{=s+1CcI;Y9SJ=YAUWUk{sB6o`Vlk<2e(9#eR^WMK#X({Lg` zJ9;6QkI2J)j*a-P5(jD8;h(nl{kw23y(X@H@r7M{OwR&F4wv38bvgd9)6#pur+hp7 zJ+Gl4o)Dyg*c^v;eEj>K(!T#@ApVrQY2~=?kR@O9CR`}YEyGgwZ&M}fT+$Fp zBZ}k_2FWMJ@H3t+qUgV^n6~G;hH6LneY(KhMn8i8Cca~oV|HPcu~fLQyk6y6cJC{S?;&VF`n$?fBKndLXXFG8ISSccl81Mhmx5S1)r?@ul_4 zd$wWdv!pTMcn6%`PG8Da>9FkP0Qg#cqtGcjA}IHA{QGoJdh1htQ|+ke(?~ea>|x>eUcNpS2JzfIbpZn*|r5&`y5%PHHal*+V*m}(m{Nmjil%^ma!l|8lLl~>W>&$7#6?arL za;U=8F_lB0?ctkfW2Gn8F5k{bTgX;lcQV37^wAqeI&`nLd{71S_(6NacCrJvi_t8^ zA{dTZW`p~ulW@GRKU~QvW7N;k@WM5u-VrsA|I+j>AHhEBq{D`P42W~ku!Q!zTRTD0 zH7fndNiR@dmqPexA6zYJ$ntmb08fu9jRvakW^QPUq{Za5ubEJ^BO99@swGL&z^%l6 zjAD#b7gF=W6ZF5`V~UTiTQw=)Hu}hpxM~Yuaou<;`baN4i!gg)`?uqM=u}tsE9TL-G$w zlZn3V*5jp~A=)3qZ$Yk3H#zCpEhw98jMIZB;egU@NSrH3FQb<(ooSw|Cq3sUv4KTS zD#!4v?jfO#*ojmx;GUL))xK^fZ7_q?ICn>DW3>tLN-i_vZb(X-jxXou!D_Jqsn?w7 z8h9(T;~}Ljk$3{M5hl=HM;D%KHRP0g^!EEDlt!JrJcK2Ex~A4g-hL*iUVRX96qFt` z{_97Y_5halGy?J?U^=6QbZpgxTTc3}(U~*T+V-F~5l$a%fL@t4*wH4Z?DVElhc!9Zhc|c{z>Jmz$T6-!`j9y7(a|&5<@Q z2nZ9@Izag&$^(!;K*CwY?MS*;umDBy&Ot@07NMVIA|o&6L>?G*jKknaX#FZb*X52c<-T1N*54bw zij&@BRM$v;Sdv!&(*1(2Q*$)6sXTr<8|2?Uh9l@q z6Y_b)3s=~%M)X`?qrK2-=xZ40_eCg8K>7iG`}}16U7b{J>3UAV!{iH~{GFYM;KJih->c1FeM*IGqUfi|0dBkTqs^nn>@_ozrWiyOlBj zx;q^SgOyH$X7gh-BY`9NF$}+W7V{VOhl_{K5*N}QJLa$P@t(ih<#uUsJ@pW5POL2! z$IoXrpAxiGpH(_T<*#twR?@sL#jr*}swPH9}0C_JQq|$glX+~cKI@G;ms+y^PMhN z`YR^y1J=D*kCV@*8OIYLuCi*E0Zw*bfEV-1*$ac|qEy#L%_VSF-}g$hGi&G8O6T&jcDC4mLloTk^8`q9 z5jMCeUcgDm+|abcT3XLs{NTS6s2}C~VL@VkSU;L!q`(H>T9Rf9eEWnDq&RWKMaRtj z)Et5|{#zveQ$AJUIPP4V32KheMjinM=XR2$)f7f5TqW+d7SxmUvxZ3cBS@Ple}{V(o^zxg$2azYps=aV z9MVjC7UPUj`;>;{7Ywh1(m1NespdfWPE)()Aji^q%B~OacK%`N@t+u~SBRufP}Mrs zDNsENii33S{7FOe1VQgzP+F9FThcYatQB|E>jdQ%|J>D+7BM`+2%i0d1& zkp3U*Twf({(`&I};#DVuPZs=SeiWUbqA$H7dnsJw#1|@7Z2i5HpyEaI9u-d{J%e3q zH0B<|W7zKghrl;C8A<0zr7Z{Np1^yy2Iv4?1@S4KoHL4ia}R!_vuM&*`fe>8WKRai2W^s)4CVYIZBL}+B z98U+8o7f3Wyk4N{P11a%%d3;7`VDo% zjtR;WBi>0PP8GSicW^_u7BoNku36{wNL>s2w{;&7)~UELH48j_um)Q*U^K5}`;s5Y@b|cGQQe8Q;D=?}Wl$r4a~kG=y_%?!ibe z!04Nt;<%nLzc(X35M%5*Ysd#H96{1vjMk*3*$0IW4i)>Bf-fzC}aam`3yEDt$wG z4d}Ck(!*+QLwpP?ow*lcj+^}dQw|MVn5kC$K!Xen|V2?c}CbQDmnv&*el4bxZ#~O;Gnl(zo zK~%Yd6ak?J#(?HHbfq&;M}8KK0#EpXr4fB5m;eBN$$0mQDc#IlEe>|)ddOlUkF zQoSPJWMU93OEbh2Pivl0{Q&;---dok)3CwcyKJ#-I1ZRe_e^KBkp^C^r9*H7erL3X ztM@p5^cci$c_oG)qWg5a?Psm~T5^XyW4Hx98SDAbQjWAMhOP~o%6=oZ;K(CR;Pt8y z!XoxUw*%HNGw6$;*Wjne?fA;kJ@HBmJs00F7Q-F0;bq1Ger|A6KGY%`?&j*t(Brew zcD*ZoMq}Bhz(sT{zlk1t4Oy?AHPB?aJ|D8$P7bLx0@b?ALQkStoP_!7hRV3Dh3vJa z185uvqy3*++-Fm7d02lVyvi!T__cL#`&o0&U7tI7=0yv>WK! z;e9^!&9htVokueBZtw*msXlL+GhUv1@eA<72B|yYDXPz2>pMu+JW(C88&{V>0iF51 zXIHSc3-Et-4}o5n);aK~HsEJxxLve_pBcM~UpqgL)1Rx}=WX6g@Yt^5vz9hz%{rZc zzcwFW)ffli7*!M3zgjQXet(9?QdY}5xA#EOy$ts7k!=bpFH~&9H=GeAcQj>VP%l5guh)L!P5>TyeF z54JnIB{n)Z6e&j9dSx1!u)RCBE?NWa#v9A#Js%4nqf)B1Xy)vmBU7hrWAquE-Y@Na zKJBUdbu}9O*I&@*%PX<8HtH>$gUnmA7}xv1 z$9^N%^J!Cd!nK5TGT6WcTHm~Yr*8em37NaFR`)H~$lM0sdwteY4rRZh1lsfL96O}9 zlZQn#grzxo;I?fz8#t>CcG(yRIz86oH}3}QNTQ@Ut#>$qdS?qUcwxJJ)d+Qa2D%Q(dbdtGfTp9F_c9h~L|K0F3`FILy{ zbXv7zhwyLHBPIJJ|1@1_M6}q+E03W#*zXyeC-N=}PK-x;70T{H-Coe5S(Dd)W>~uZ6)R0q)Np z#)s;MvPH?xwDw`pw4)=;J?JV&wa}N5Pjuz%&B^H0HUgT&o`Bbb7s&5lEx?G*c+Bnq zc)REaQmvwMwiW)4o2zxSqH~DbbmV!Zn^^Y<3vNA*_KJD2l9ydhW;aszqSveOKsiN< z7f_wajX{ktW_2Qcx7-WEGkxUF%TDlN%1dn0$c(MJ-9!ZV)#JwhoWM^1B@|S);PG|n z`)H$Zsd5&xx}z-IK>OMLY6KTv59ZX5c%v7SW+r#5Fl9?*C@wfRhEMyoKwh^s-~sDK zN~#AKnfe_!8yU!DeU6~#gG4cL+#)P8=!aB)BFcLgJl^$MJPFq2ZVk(E7uQGopr(?r zLOyNrm=QkFGyWqmA-s>wGUf7*MLJeXdyi{#Rr-UndcN=^27W zhsz>v>v&jS?!s$l?gO(L=3Fdxz$b<=(tmt?_*mE#PW2zl{Mx&6dN2O{%MnPMT2ltq z9my%i_xL8wD zqR2@WD#lWA+R|Rma`QoRr-1c0jR4f7jg33>ib z6`6uKTexkb{ebNI&vgfEV}n>n{_DE&D*P0{4zdw z7GxY)j-}Qc;aaCQpt#?3a~d98;Q@q?BF^7Lvu>_E=Fi^^*>Q~ss}sO7B@vr6-UMHs z1TyMdm`C^aD_-#|oe7fmQB!?5rte~OX#SfKW}(V!?Ykq`(L=Z3nNbM0SvMAPvX_eZ zMz3&tRkG8f*5U5Dt$FD2w{X#FIg~`! z!G-56L3=UA=o_2Bjm4L)z(L-sFjBy(@If)zI%z$v3rtf2jLPxM*MnrHtf7Q%eYFq%R? zmja}7kTL0ZehhTON$0 zp$KmawNET($>ugK_}H#3dF9#w@uJ&Ixng7<5bn#h6E=umkDkG&5jv7yLwc;S{CC7& zCffJrQ{VKV9JhsIb_I+u7=w5D(!DR4g5nH@?^?+tO}n$yG6$*uYBJ@wA>l_;zOsJ+ z((8Es&rVvZEzmCNDXAYIc-K21OjH~tGED~I!7qu7a0)l9^`cs@%@sZ-&-*1P=1A+o zpWCj3-UB;Hu@X81uc5)Cx;*H&0o@1a1h}NFm@AlPt z6jxC#GwM0)d6~;Wv)9RcV_WfjdJbG^;i39W(En_g-NPx*`vMi$Q4Ub)<$fdT;TUlT27PS}xBKp8_r}Hn;iFJ^y=n~2 zvOhC8CC{^ZQ}!~Yd3{Z8Q4S6X(jKaBQFnr;F#I!#4~_S6{BL&+xv$VsF1tQh(w~W6 z=y{y*bg_4+4HEZYj}v)#X=`;p@Y7YLC*j%tj#V1&SxXN|_*WG#qu??mj0e&t8k6Ol zHR;PY$P>qnVDGS&K$wGsLojcv4rw4APQ6yuXBW5Bg3+i{VkYv~@jFOvSR{AMx?;5aQ~cu*7)?)#zX_4~x|*eK*z15-;Q# z!q3`WxHPPUk(PUq>XFXk2vodHXUqPB3v}+o*~|vGaraYZJ@&ZDGcHZF0Qw9;_$cT- zMDJ@`#N~DmDA&`Fyg+J=esh@8f}~&hPk%!;`Z%2tLHdAt6cS?pVyAh>nBr5?07&tN z_g;PwV);%>{X~3l1(sdE&ZzIfA}$8zpZx?>cR)2JsBTFkuNPEL*x45{CllWyn4>1-06)j|Cs*3Yv7!ZSwKAQexJc5sp% zevHMQ50_)Zs3Le``I-?9puw8~p!#Cp)FSL|T&pTa#MNoK_XXHiwgcj{Dm}hKW2^lz zWxAmJan*x!&AP!Ki=L#xHe=v|^-!rB!Fsxl0^)Kpzg<3g*FuQ;1oyg4ZCVGW>pVZ-1CvRacwd^Us|1$mW1X# zClJP^AzXY2q%W|0;#BmATcU6Q*89)lrPg(z-js%XoOFb+#K}N?oJPJyaXRZa-Gvih zaMA$cT0%9Rx!ImKJvRYdw>}ZXS@Lo;LfDDnRrT%HSVy|n7AlXV`AzwerWYXc($5wYs8I~J_JjHA-LkvtxBYjp$t+TRrZC1*u)=~SGUrOW-2UI6jF6ZJW9 zVLkFlmua8z&RlskrJG29VY}JmSmyw`m!hku5@ozAmo|LEN8UU1#{DpWtdUvW2K@CR__Tk+JueUG znZVE{_gH$h!>mpJjjY~+C*Zc0_H*_6jD$IY^b}XuXl~jcjvdZY*vO1NY{nL;mR#R$ z9q7M$qRGiF*D5dexj0;Td@(-vHCX$UYRIpn(xQYJY{q+IR9sEEOAxke)!&u2@B3~U zqnN;^jq5bFl}EMOz!gwI=V^j89m|S*CelL&$sUpPJo1FcY2@pm zuBRQS`30?y`f!g>n$X5?J87?mQKW>w3C9#77m5q;vmhnN!ayt)}qi zutgudZ0!Z)Mb&f9JmpbMSe?5Ssekz6fKH6&Kf<&06*P>R=x7phS3|y+`g0aijDdI@ z98V5L^3;qp8AC+Mw;G9Ye~5;@&GUXMlL0>M&M}N_)dpkCO)E#G9-~n{%MNJbfN6=#Yu8 zFS;pBtWk3a(gdOmc3@cI6V`r0ZKSm!;k44k${TR?9^}D@6WzG#=^x*=RbgY)=M)^| z6NZ^Jwu+%UY^BmXy*wn%lI{R$74l=B1>vbAAHb;J={g(Wn@d?r!|{jTC#rcG?qG3( zG%(FB2E)^RDT zR&RwLvQHuK>JE~f8T(s2{;ews=Tt3d6qnBUQvj8;mt0opAvQ8~vF2vZJs=#$h#8J7 zK{vW8w_DoK-n+zCR9oZ)(^&By4>lnqMsd34cay=SscR{ZBt~`~#_RtsL$jKjtMHmM z1_b|EM>;Q#)k^P7`*bcLZaarGBS7;1RsMQ-vr|Yo2rY*Fr)ox$P6HL^fu9C~n%A}a zO8XFy*RM-kiB)S^192%m8%^^(@`I2SY6Zl*N~ft=2PluDG(}izflA9z{z! zes#IhnA9(fG^CT7?NH9OgmHXsTR)|vaqhvJ8t(=RL~-*)aQ?v&;>7m6n$}B8eaZAE z4dm2wuqZi0LzqndAq+}SA6IiNH9KRE=Cx$RX-;zEDDvTHnw*{^2zPWv_tqKW{@3qX z#gC+~1=R%1-rNnzlOpLx+)#l~_{$V&MyK=-QXL^-IVer8xQe)ucTX%4r>_WP-v#UgqWo6*|rkxS%>GFNBA799P(eapwLs zhyJ8IH2MA%P`;h?9A0Vfg7d6jGKw9NC*q~2JE&PLB#nQOwiC=5&69-E(lmcV(o^&d zSV#Q+#hR{A(JCz1hy#=H*)Eoz1 zoV-YUJRfTGqq7d1A5b$vAUzGF_r&KJ=Ry3d|tQEX6g57i?lt`w%mla#kZ(lc0Vz)00YXl(3@nH3tY6A}g%a;m12!#@08sSEK5 zxTbMipd#A(9G@$ehp&W%{>}C zDlW5o#)WW>z9XiiX|$(;ddp4@jl^5AhEFLkz#6`drJ3DiUTkJ2*N3dfj~T5XGPo1p z-5DW%<{xNh>_hKqEw|dv=BF~ZqqUo%!Vnv*jw%B$WxI36JVO#{hgDY@a9AgbaLc!a0Fj|>{42pAxA*< z`nWoOCCFfUZlHW3dmRL_z_k}X&fLP=4gQNQ+*{+TqCH|o`X%NX$KkyB23Y%JEKQy2sRBK7+lPJAtl~XI5yj z=FhHtY;Y2c8oYuP1xe`Q(+4v1zeB9i5?)x;knb=VBs&|e#nhrEJT0>UPjT(bPT2Uv zUANuMxE1GCn&PAQ1H8%4Bs_0k zLt2Myf_Fn`UyaNd(PuX8zmeHM(w)mJpmGOR*j*PhOx!dJ@C(c^*AshV-FV-M{_>o= zIaGU`g==D7i7(PaKAYtwM_2BFQln3xuC1_WFE;t%a#A@5G>x+R`Gy6uM>U$dv)#PzHQJx=z?WyR~!+?!rw9 zN5jb>t zktCXiY}R(IYz%%Cy|J&u9*wYlrIqPi%y(}pgJvg)PriX^2g)A+mfyt!8`{h3G4Po& z-f%PKDLvQQMA`9-qR%=+&^nN5J{-|8QP# zXI|Ib2#Va=({)~hkMWJc3Qv>_QE>{ zd$`9!=HoNiAc*D%?&gwenZGJ1#BRmgAUS4A75;bzxv<#IjC;AAhlW@Q5pmsNoO^Rr zF`pK&gx1$Y&WulhwVl%$#gg0k?xyp#j^bw5f%3!SPP}18GL}3(2~<~d<&dW=a&{}} zZXODDuDPi2Vq2w=bc&e+>xwL8zHeGu^-+qOKdnOR%e-pon{R2s`&frV-lT9a1aDjgi-AE%%##gfzGo`zpnZkH~DVCn=zgW_vG7jCurk3 zl-KiZ1V)3`^84Z+PA&A10cOqMYT*DJLwnw5+qa~8b>I_yw*yvgp}P75;~Y$-o4X~i z%!n0)ogi#y!unVbeCex$Nzu!Z@Ck6oK z2}Z|Ts=TmuMbYw?_dweJqz!vEWVAfdnf5vLrDuT(N5XHTx!@Ulk6j2HBRvzga?gbN zK(!4^g9gBsnf{6cgmL*ZZJ*g4c<9fg?A+`Y_|JH>Bn}|&YR8FpSmdmkxTL6sToX7^ z?23=UWo`~=8JC4ieE`zk&7|Ax_K+5P9jVX6D)$J@MAspdb008sXu|6hj)Bjn9fZC^ zg!ZAGn{49|#x|M^rWr=O&@JA{t$r+_obM6-zV$i9g(U{&<3ao&LJLZSuUi*%w=19; z(8px-)d>4mLe(^3mdKgioL@J&tfBhmfyKHo-C?I7?nA;==#}n*Ln`S!qsmY*+N>o{ z@JYpciZS6U`p&}^QmJ^O~s$Z7;*q(Gnrif1nA-pePc@?RcH0$O699?-_P1_ew zGL%vhn#{9O(m;2wU75#9JTs5Xif5=~CZ&XkP$5GD$sD4)*G@b{C^A*bJWoH*kfivn zegE(w-Fwd2d#&{y);VWyHs$Abpcoa^w4GJT zLY>m78~vSC%=}yP&)${TGW?gqb-WqX2+vs@$3GQA@O^-dBwZ$;@;L7Et)=h)+wS|s zVrFZzRUO}2o0EMrYKt1XEL+R@SjRD(@YnrZ*-mU3*@ORG)<#+ly{+kxn*>j0K0)G1 z)_v_z#f5TZlqDaZ(w?m!ng@~H%}^87LJ;2z!d7~BbpU71dNL$;2dww4EqCSQYIDzb zkdzN#eBOxvXiF2yV0lgSws9=|#&~4|TkuxXWbL1cjBH@C*9; z8p3h!u1w7>v(U;lgF|3w&HzxklQrum?hD+2oy*PC|M#H!m-Q;1LL+bLkEcfAwx}IC zU3qcVP*8kF^N0VHHId(Q&OxM)HJdi%sV;X2`*_+aQ`P+3!0$?umBh6bnB;x;qdwURC*>e9E*N7z;l(UNL(weoWsE>cemmRh4q4t zqgi+UFpH%3O4BMv@KEz9&@0&kWRAr8Io?3^jJd_Z_^aC~Fd14z?*W-99L4`~H&D*U z!p8jt>Ku$XAChyof@Q=@X5Qo=G;}uLlrQjnUpW$f@g&FL=&`OI>0T2Y{(XVC*tJ^4 zNe~WVky9%?T4p2L&Kd{Th7`kK?=0+Gde>b$qbi$tS7z@Y3>A^7qVvz45bD%P^Ralo zcxhoL8$8T~%)OP^y1SKBK7#Nc|2XZ&$I0CYgU8YD7UP=POcH;HZqy@8b!x{A%DUjK zR0BL>(GA8#c4tc0 z8p$R;)YIDcm}zHcf$+AMq9r^P76HfLYWW%@eFIH=+5ycWT9>6mP40T67^z$U;&6V?Cl0cr8uNa%e>LGX zl19*JnnZ8MY*hye4+>?G=KJ1-imN$kMQ9h;R5M}LNc=q`n1@%4;ycUhs_`hE1>y#N zb7(J2YCKRnvF4_+ZXN^ELx1E1l^Ssh4@NZ>`(%YCY_^VJkt?-_7KZKh* zy`bWj>V1rn7LcLdnZR>x2(PDMkM7O*cHcFiyef?g64w4J$ZsI|3_ic{P$VzK<8vGH zZxuszK3DnMS0&?cam8q88`VlSdw8Bv97xBs;4T#t(PQv39hXp!YKRYnL&_oL)#Q%v zf#gXKk>8rfZqIh%4I7 z&7V+XQ+%Rb>m|EZ)mD7Z2#0xApdk#eJPPE;$xl6ls)*aFM%mLJJ=8g%I4}}Am&XF} zGN?IKF(F<@gR%^fd;S3R3!dTDrgb#ky7fcSLP$P|6W(%z{iSe8j({%7x%l9r1K6(h z;SIn4z&!Ku;>(&a8NTsp7a;{?Fx^VVl=y?ck^b^Tj zlm31IMNa$ZxU}~h>1@68J@~zV=Ct-qsPgWrMi})sNK;5SPS2-MO=7CUEb+ITz^yDK z`2t^Y#oLbeYnb)H{73N$^!#y!?fJoV{-ANWIrTe#Gj+X_ zyrNwKm=x@&L&r;Qa1enW~Y40sC?HkTOPiEe?kN4=Vz8DsI+dX&zG=qc|!S z)MpkvJj;~drF>x>_CFw98;uo_?UXmyP@ZWNCy-Z=^tU>W=5gc!pgNU=Dg3wh0gNw8 z0xMq=KGLt*zCFxguo{|WoDoSbWTSQfqiS(^nEyzod2HS4^(00|jMvV#aN~4tMmiibQd*Ffw&djTa9zrNL0Sxz?_B9? zK;Db`ofdAY9+9-8mOKVf{wa?MtpbiR`m9iVKcwQJ@&`ckO}>5rkbf0aPm(ZQ^Yq&< zmESu5=IrEv^c_u!#Sw99tp{njG$cPRKKL|(JLO?2x8;iS_DI(amVxntIG9u4O<0$8 zl0Ak@huPofLFACXL3!7`=Q~rc>Kdea>+0nkVLDE^P>ZWM`8xPLBMpzF)1?~wv-2sc zHpTjGT(tlGU66-E!YDP*tV#Esg62)}s_KI=>KQ0pk<<@hqxiRn_BOrA11MLW}Z7 z-0P<$KVoqL2{V!E6$mGkR$;0Z2*U;CHy+!6opjZ3pgIwa=7hP=F5Azjzla7U|KR@0 z4uqpSw3dO*w9TEmQNJlrsB0n2ryh$V5`Ssj0uD2odFKK~mMzjq_=2Zn+Vsa}oE|mxTLBei~a1c`bapZzm1V3Ftf8ye7?r z!g$9(>Xni%@hTk-R9oC6Dp}Nuc+IN6Pl1URXT@MAdhdyJ@}DN=*yf>!rg@VCsE5CR zI2(xjq4LKWReLJmnd%$2Iq!m(ayJX=t7R+w=~doP@uuA3cn1y-?W^=R;rvq_W=)8) z0qT)y<}_)El(TrLED@=Pp?E@2FBD0SBjt$VPWaZOF{d2lgdO6^ko)3A;9BCH4C+Vs z<%GGCv?JWf5vZN*M)r~c(lcv^EuLnn$Zxqza)ShozOH2M) zlnwr^yRlS`g3>VL5d5u?!fd=|9xHI~-`_K2n7LF}Jo4XU2Q zuzl|^-^T*Ue*@_^;)pFGGucHbk2$XCdZsv{UtnXTn3H~~hy8t{&#BLxvvAb)L*5B` zUO6B|kQP(lBQHDbVKh?xY7s0OcQ6fA5QTT`vG1C)p%l3pps* zmUk*{Ah&qE#sw~|$RBQC3%sVmhDHNvPX$2xfVD8feX?AvoSr!8d_H?)^AgssUy5`txK-E! zi$_O*->oj#=4n0dZ@EP?(P1{w^`c(oH<)d96dQ*A%|GyT+)`bSN0}ywFLMTnbE9j? znL$zVNwYdoHoa0i{aG4r*x!gdxme?A*BtopatYk>90|NlGS6{X2!08sqR4A5Y%WZI zCExbpH+p~DHpiY%%*lj{>xRk1-QU@QY6DsEI~Uz!)A3z#a|qnuP@ag_lTkKhIChK~ z8q@PrT|FG2pdeDScK9Sr=@+S?G4a5cAHivkD{Kxtz_!fE!JwMm{Bu1?@4Pw5(69Be z#VI}B_Chdk@naIcGjr!RN-~*uathkLe9b0Sc*~}D=AeJ>6lq~JU(k2q-9B2ICANoj zx6cv>*4F@Cx2(T<946vA-M9|diGqBH5x>Xz$`uZC1LT@Bw7-Qj}7L4zD z8&3OgM<=hpQE#1xJQBhIWm|rv#8HloJq%GJLS^Wmp=@4YTbUaDl3iSGOtA`a&lpJ0 z7^j?r6u*3QzxuC8ym|v(HH_w0vPQGw%`N3-&qDaLXgtfhvRC+@Zh<|fX2Yd^hVoNE zG*`Km)8-hfv^xR&hjoXbxg&6dzp;!CZX;*flrXvmRAW)GXz|`go}N1vCyvV%LA|O4 zT^oEj`ZxcbL%zso4NQ!T=PH*@8@^_5R*mL8@(x4zg%xn&p(PrJi%%vHpyrJG=eSYI*1$vIZhx?aD zLucFf^lZ^CaP~Kp#>f0+(B)d9Lr@so{=0`f;W-@gU;NFpik=H+|7f^6q6Vh+)00Pd z8@zkjh%aj$i%AaiaEnV@zW4n>#>a6UJ#dJ;zwXy4`yr$cMh7SJW6Ac~lzw$Z!@`Zi=~hP`YL|*tV_J&z=Y4S5 z+BI1DbsP9PuciF3#1%Pb$lJ}Lcexsi(lNGjdjAB?+Kd?K>NXfDk6Alw#^b+kU=>rG zd7R-3xaHhP4jAztj{TMda|0U4s8v3E?6kXh%tM2TW15N^jS4WNXrjD(+lMR6I}tjU zs~jZ!Lbkx!DxRSnQxYbblp5^#1DEVWw-PnW-q_hq{E1sM4n}|N}F*c7ptaC zg>##2@teO5_*I7iVWP+{-bwv{wHSBVK>B$-Mf$vW5l!nZo4JEtyBnyr?=A1vv^-YpgS9l~u!t7$Ye={Zq z3XTQw&!ItLz;mtW@u$D^TU-s}r+=Ut3s;yctH++jXsE}n$F`A_8}vC1q&$s7HHNU< zm!crqk?yspz{Gckto}ex^er^!2J8QUEpt4ey7g{mSCYbt|J)Nj>{{>_Pp!~5u$5Fb zll0CM^Itq-#3NYU^@-?w#6_M6JcPe}8W0zp1A`1&4>8sO)@J#sc%WbE2()k;AYF~V zg75KF^h{q0cQ$+|HXOajh+FaM0Czda_7i>%{fc#MhH>lZGjPDFtFXZ91^eCRv>-kO zcY8PNJn$kA9`fI9=zN8{u!r3q2>Sr;@tYHou8&{LeUINRn@H~Pip>TCSi3R{%oF#E z`evyrHnLz*u^;Lh;3CXA4=1o0yu{i|cOT>D$*un56|?HEjWx ztTN$6bfD;D))D>ax$o^=Hsi5HBe9QF2WZjuEL*Vn4>QQ9#S04R${{=vR-K9l|I>SP z-y>W*S5&`<_--js-m#)vS>RoF2Y5!=Gs0uq>a8=pIeZLmg?`q?BzLBJ{-5y4DRXqb z?TJ&Z4x!@E=^5*pnupS9lYzJvp1ePbgdu|R3`h$}{O%!N^fN~7w2AJ|p41|(6aY`6 z7*bBF7~}8A17fUw_L-m>Tgb5e0qy$J9+I15aMJtzpzwvT0BMf+8`IXjpS1@LIqECI zpT-Kp2Mu8-#MHIrbZpW-TOczbfz2GyjZ@ttX&&CX@&^XFp3^G4QX0l}%6{OqhE`a+ zVs_-^SIGVGk4~q=Z??j3h94CMDIF8;i_x1KuY1=lOO-lbiT_ovNeqj9yxRTLW8u2*s zIc2cYlUk}>>09p@63=Io<|5pR$CzUuQQ;$T8N~UU%LNHrF{$DXVd55H-MStSR&mlp z(Dl_Spm<@_^IW#JzYQZim+s$o3cHsxp+_F=Sa{2wsrp_r?Ex0K90I-5O|VzHBBgPq zUeldo$>?6zwPJ{byB@lf?5?|`eULV{b=yDj2ywVhJ748AjFp_v36YV_}|L|6p z4K&0Hg6b6>y`%jQyyo-ju9HPbQ8{VuacC{KDXb>!Y|cr~$o56vW0u<};a^#US(^-G#}S{z73;zB-HWG4 z_#|A7f>cf-)g7b!5TSV|MXz?lm9~Qrqo1S^y2!K)GZDY59H{M zXM{%)#1b|=bTb+!K;;?FsQhpPCLD`U4fN$ z-8khDE28(~du%r2c7saE^N+$+=e?0Ikrlag#@R6fgL8IYy8-V5HZC zc1$8#{6388_e#T%M?&Aw7cjtP6k(E~Q2C&5)r#%)wI;6nil2^EupEakNLm$$C%Ez! z18nZ=a_C@?2|D+x=&AT;USJB;9r0da4R5nzJKY;S0}Tsl&$u&LRPU|j{tup-HdbvV zVGF(U*jCbaL2)Y8J529y2?vHhW1WJnKI9- zBhO8}ELODXq~r8s0b6kNoiE~k(JNf_z8$uBzl+`%^ybko2v_!7NW}+ny{MnswLpiB zbbYE$$%AwhN?($9Q+)Q@p%OD*JS5F+3+V&gh~okw&TyI_jAr*Y z)su4<`Eun~E)H{*r2CPsi|Y-eHCXz$7Tb#^OGnQu@ND^J@Xvh)pX2MI@>l-L!zs74 z5Wj08^L*bzhIH8oG#^TX(;Uz}qhpIW=^r3Ghq0*7Jlkra>gqk6FW&Apg8C`Vl{Y5c zbwyN6IVHZh*vn$OuF6+~@-7!GcLM2qM&nWX8GTa(q(7%MNypgYuGWn~;jhv(irojR@8Dx6B z!>bS0z$@#UihGdqSCS4T?A#0CetGcZ{3z(ccj$1nesELb153PdbUb1DB_K>?=Khgj z`K=joTvItZ?-;09#eZ$UJfn_~KD~%lv0C!Jon*XSM@~AHcQQN3C~j;(kO5Z9&V0XR zJSPo6ezP04u}UF44F(lMqYwvSZ2p&qbON0Fx{2QXM>g}hR5ioqIj=*SA5gKSTFIu^ zkiIpQq-Vh3nJriGnjPGdkv?H&bGj@4gt^Yq=q{6Dxu!@!-& zW9aZPDW@kUerpS@+7+^1mYW!PY^2Y~uI8iR*@w&SPP5D-$(K)Oku~QA1|r?H?eocw8x0y-;8h*za;FzETccl ztApqJJ#7E!R9F&w3`pYv;kr;cLwZcsJ!~Z!9{-894pUiR$sz3Xu>ht&YmfC`+)(&K z^_YSmLjHwz&-QBL|Iq#Dg4NU`N!31EcZVr$Lb?EmFEQl|K=J0ixL}UZX#&!}jI;vj zk@>WT;2f^7xm^amZ&Q;&|YQl>VnSG<#E+J5u{CE?CyEs zw|F_QvF#bvDq-mr>@Xk$8hZA@0j67F$jbn{dibyyaG2g}b!FH!c^^J<2yq|3jQaZvG>$QLH>s7rxx0k zw1tr0E?TAc%>C;^Si3Pwm#{ZMjYWo;Gt%6EZN&f`hWMr4(Y%9L(vuEAzED%SIT;9# zg^DA2FL}i}pV4(uo}OUgR!0aE=fdba3xPb7!W5x;PK23QCOZImCNT4QsHHvx7ERV8 z|MFc_PyYp^8L`)}L$H14Wl_BAIg*#+^m|Y`ROKUiByHT>`+|BYg8CASa)WB=5Kg_Y zRPiJdE&<^$b_!0^#k`<2UwLzgT2+A5KLWyCB;AWW5)wHSIs(+X`kM(So;!6Y1IJNZO52KZ_}CG3dxyu$a<9&{)xW3+F+GPgPDZ@=d6E zMWh>Hm2Wd$p3%=(eCN0FuaviGSUL4N#w;q*RPLhnm`B%xb9Na(HHOM}P;Mf5X)qXm zhxB)I^3r9@{hKx1iVsKSNt8w-%>?s2hXQ$IjcrAqpgxPJxY8a7PpEe^f%#R}lGHn7 zgvF@pIRD7M*=DKUs{VzmTfQcd_TVDDCWvi8~neB|v|LxlWU{bX*41N7HfD zfK>xI_4*`#7m2SjcED%P;_OE`IfVJ1{KJ0h>}TsnAoWLBV55FIEk`&@J))M}>{cqO zIUzkmJqk;?1nY3ZNYY%jC3z9*%h^(&$4zN+S>*K*uQZHgMuSF6)t@8|BhIXiDrQud zK)!`=DFt|rBUH7D5Mjd{LHS{eo!J`Fi%9(h+){t2Ak8Z1{M091uXq?#&noomI*NA? zBkn@V8=N`f8#@>j17XV}#b@Vjr0c&R@i0HO=$?uXqqqygUpQ8p2lGN-D=egXT0@%Q z6_Cfo&a|J??}`aXJvJbJMcO46-vq_6icRZ<^IUH}q=eoxemEAA-fd=DEh3hR*Yowe|7$h+{S3ImYv7*anz5c+410_D4)-VTt5(2%}V-j3Eb ztF3&X;s8PXrT7(d#tou=*i3%qnS%&1TEbO-y2bb+rhKKRZzD+HPMpXP4b7 zw@l5knmB;Uo zK2+H9$frJ1&ksn4a5Yx#)k39%l!w*z2$Zi4GW2)<=6Xr_e+_XRC*6Q5c5j{!5|w|( z(0A!QC9eRgyJVc&Z#DI7Vu82;sc+~`*TF2hgex2b>SdtnfyUpdR5&f;61#HJ%0kti z{)Gi>rs;Nd9AMMmim_*DO%ShQC|ddg3PudY?x)kh_Fz|Lzj`g(ezpOwxs<}Lwmg!3 zW5;n=7Cn<$^Gv**mxzb3I1JC}!tiz{o!p{&w zC%M%LsFkIMtyhkNvX;B~OY6hzOLUaD{~zshJE1neT2vvzw0^Xva0CAB>l)3XGxM>i zoe3On*Hs38Pr^{sV`A_+W8U~k3pry^ZP_t%DLtq1S>(-Y#y%H~f)kY~Vw=fxy!gHW zAM(bPKDP&kUvq(r?>A)cUt$RUX&3p74hc~G!c)$rwGC`OFNYV}FrHKgMbemw{OG21 z@hj>EULOAyI%V&W`J zAQ`Z(SEY#DbQK?M^wwI(j^J@8=E(;`(!hN04y?Lc3kJ3xpkjj6PYq%5`HPy}zuv;9 ziGnl1Oze$#_DLDJ`F9mi-%$79b=y@Z(Mh4Q`6b!Pm}5y;4M=YMa=#JqX&c)sHZ89lNG zerlrO;S1JC;}#dd_#bw(UbNg3>XDJt!;Ea-#jo^>`pW1>nG7>8)iQt?_y8) zpk0%BTh|6Ky5=rEhG+2iLt0I(5$(sM-g#v8U}@(Z@!@a|FBrB1vn$@fNWO-RxKr2t z;pIm-_~B8!rQL#QpN&~gaDBGTs4J%&gsmf%%jK`e$zbXmw#=dXE*s0mn29U-5hqu; z6OqMEU0%tnXx)pJjXLvnbWgK>YCYbA_E_j=G*{$CRin|#&6GRy;aAj986Vt9voeL& z?wfQ7#|IbVl+1kmWFDcVnv*8WT<}-vM;u-Am!`vTKR9*qs;2+U$y}?)B*k4WyKf75 z=k-BaUkBG^7O43{_c+|#@Z`v5JU_ale3|cwwi#3T>>aDb@EZZ}#zg?l4|aY2LeQLX zn*kAc+~*hm<=l)U5;?_Z4IX8ok+V>P2Gz{l|J?|@D=0)S@HobAJ zLrdA}pWPza{x9~&?>|k{r~gFNilty|vw>Yc(@osz_%GFBEnHnM3aEC(FtnG;V{60A z@C`uY5mb*Doz{lWjvJ(MS`IVyk@nrJS*zAF`IbH&+&`W6Y}y_{ak`81S|5aoD>Ino z{SUbAdyI;=@SPBj*1S7bE$vGf++Lnpew>X;IxHIXZYwW-y1+VRH>Gz5r1n+EQyBcc zIgGy$!wB!hy=~vH_s+F2ciMWM_O&+++x|j|8xMN39Tc{= z`iItMojDl^kNE4mjaaTzdqEfs(P_!L-w(1&luw66LW|R9bofAZB7T2cfhva9@63R{ z!!xJsW!K*hX6E z`gT6FyhQK!noQ+6p&q!@{+s4?LPHQ$JahUx-O#%3+yvEDt-o3y1bQB-xZzNEk}^G8}q;p1}Jai~*}B zIY{%Na$F9>BZ{N2X4_pjP_s?WdhQ});|>AkBu+a&9Cufqf`RQ;0L7LcdGG`O_$`o2 zu37LsQykIxA7kFx$_t$T*$5xc#A;Ha^Hgk+aE%q^`|^gx2gUFXouqa|89l4L6Z+Mo z_jQu(k!pm|_~cmaf0~TYg6uDwzk)&dc*3V6aI%g)9-3(e!*9Gp`nSwGzebl&RJ%M$ zKN4Qdy#dxA`%5I=$$!i~uQ~(a zt(!^Wa=E%*J#M-Vz~P;qq#WlzOd6v}!9A@1BORt3-zHv1O@Nf>0pj682`c};4H`$! z^dxI~Y$$?NH7+P;>anay+QYJ22A*#71<}X^RE|w(5vFnk9e6U*anOC>Jd9)=b=(W* zzpx1f+fiZT>T&6~ZJedZxI}BKcyPJ!=@RgxURCt)TD)1?WDz~`EI@W$Im_)nHu_&J zm|w6K_RTvBv9I!BZ<8u9s?R@C;i;ZsGOWso1L8q=w@VMcMAy?2c9TBYfJ-~9gRq$= zu-5GBP%r5{D(o#Q>;}1Eu~1USSoXI#kBaw(&E9~uvfWr`df)GRN;6J)2827p@F2&n z!>-`nr=2BXJ3EoKMyS}1xDyE<&s~9c`{#(hN0&f+#~`4w@Pvq0pmLh<4ptj+^jI>M z5ho(?E_`0)s)@^}Ei$LyW2NTX;Jwcf;g~dp@1gZEGpxsB``&TrY~d@OPi)9j-+tDR ze&G-2)#KBx%*3gS1Cg)^zaQ`xPe&jcgg?Zs`;_!FX&3(QkTb$7+zx*G zi0qi8=NH1c#P4{c54|Jg9|6ORE#Trc zkC&T)w3zs>EFHIwsMhHq!aPXXSOYQFB4GT2WPG@gqpI&oYyg}b84ttGfsE z@GaQ>d%4O5iq|3ZY^%>Hp8Rx6M8Xz!a9n*E9^4aZJ~a{^*Dk?I?|n+!;-rgw>0|ac2{GrAaA!^!O7JmkElI zB#c#hRRryNLiyA{eBEIxiPzahkFGGUHNCHTe3;ZEq@Pr^M>)z=%t!~)el!)hGO2<5710#b&&%DeXesBAbSC`yWhF@? z;&yE>mH(V@Ne~Z82i^vZ7fivpns%UgUVV;qn0C*U<9N%&nO}ICjTc_bqug8r$9Hvu zeJzJ_;}*Sokd|VE+X_>3{7}}?mRlH_%OfsTjWE+8Gs<{TXhqdwmN@&K%^*J4>ai0=Ozc@s9Z0U86i+I1E`rUvlV z*Ei#eICpGb@4>lFrk`*c?q*-ycY|Hqtr)&QUx!B;{ZYh&ZLrpiCbIeH-lTa4GQy^F zq+6i6hd*|1x*jO@j7|B2&6fORZCy72`3MmamJ7cgoE84b8=;T)Ef!okf^cXkCoTnr zU2%wI0~c*VXJp6J{z@A=~1X! zQ6&ID!;YU;F_CnwvQfVh&@&#xMh`a9m=X$`+Q8EtuNTxZ$!$uSHWnTJbmdK^%q*?5uN#{4`vz+&VKkZXhS7VIXU%RtTrvLC>ac_mwxHsBe1NZ85_~LmF zFs=VIX?($3+%h?(GzyX)#}@wEP-&QSqa<TQPNM4{4ikh-IgIN%i{FOG!8 zcq08AqjSNn+!`H@2lq2%Y5iX?x=yL|;jg;V7HoY+GC$?BjFViKT7f{{PYcL(xy3@!Rt1%jkX8Wb;!l+LRD3&i6`Fv@*;S2n)rLiQg#-ZCSWV2hVk>*E`f0fJK`g4Ufje4g* zPH2wEDoJ*K+xI8wkJVgVgVG6vTY~r#{ ziOROR8a~~UUMH!!ghinaGTHtvX{J0F)#sDumK-3pdIF!HnvTkYQ-0!z&W{wPLdltZ z3R}eY125^EnhW61DE(mZ*ns6RL#AHU7j zkY2^~i5@U~QDb(p{uHM0FkO2R_bvPgsc}{?DC{#@(R(fAZP|!$8^WDo+EDXpFm+OA zT*%JC1E2AD{_aU=U2s|XFqOB^E7qBlci@Wi$X|2P0g5M}&tQ6Q%Y2%KbPG;g*@X{S zkcs7MegSzP@TRr#Hg!Hmnq?FI-g7`{Zuf)Z)+wA)I0#OoyYp5pPAcssmfhbCq^HG5 zuXA7@E(Fc9u!>0{yia!@*sex%f6o=>v7`iuUs0Wt&$hv%TmA-<>n*h&i&Eg><-f6T z@?WHhV^zMR_m_G|*v(W7m9`}u6cgF)@dK9Y8Z|Yok@wMQ-QkPka3^af-P4}2`c;oa=9%2={?ob$;t1^h@c@v9<#WdMgyM8d z%#EP+rL6@}-BZu2JAWNT&nlkyAKrD^PC1-`6AGFj^*`d4!Bzp_qr?TfJRs3G^w949K}L8@gZPFjk!Wuf$0 z6Vm*T+5J8Jq1(acq@5-4U02|?GvKUSQ#M844T-ZE^+L3in-tsIO4qaa)=|*@%RwYu zfg$Vx&b#C+>3cZLI6+kC(O#)1j%#=1|ACHw97VhBr%BVql5RPO0ZBHn#=?|SAB?%K z%oobjSr~2=7aoP^9U9^@H~>ILh7dw=dDK4e1d!e9@w`Z2*2>cCQAtTx=>*kI_})A z#>Nzu5q4>b&!MT+Fq|`b4WoEM)zV9#dIpYzGuSPe2ZSNy0s3>rQG=I`3yL+Yr$ ziWr(OnD^Y?4nJK?z?f?W?z3a#XpNbX>ilfZXnn@di@OFf~@H6Y9B!qv}KNN-*N& zg|R|!1H|r_qP7O`5eJaGkJ9IGo7UQ{Q=UM4kcu?dg#Fuq^fJ;pHRNfKFjnC) z{P|r74J`i07WNFI=gKdTXQSSc=`lumseRJ_boT#RZq#W@np1b>lM5%`Wy%LMz2d-? zzt}gg2lU>l{LfhGXa7*VhvX@gFTt55&+yN0UvN*GLVjbMP`ut5hpOvU7{JKu0Ud)= zJmCH=GbAqn)UOh%M~>RoIV|I0SdEetZF7Ry#%RK*cPbhuO3>Qtn$owN( z&DlDl_OooQ*N{%E@U#U?-4%y9+p|PP-$HyGGEvfZ_{3K&c*LR-HpVGKv(w})_L|ZY z3c})WWLhxhKl0}PyrA`#9HgN6K)a0l%*3hPAxvjzXG zPQa+tpSV2q4$yd{d1e~+JvdA5>NiA=Y4C%UnLdQ5%h$o<;d5N{W(z;M+M4FvAFP&K zgZHJ=@!^BTU|e&8uiZKr*il-?B;*N>UsMXojc&pEp{?Ng!^dEr*&f^64i~SN^o0e3 zkIKD=Z{j2VE4%SES6OS}cl4-sh3;Fa_w0BEw^*j}bLo?$kBx==`ezSxwYGtrCH;7} zY4$*Wq~W>Mn(pykaZBUA@Y_3Hv^irA9gPvZNRK8@w#s$BGJd_`oqMa!qxXrGGZXECwaVP-n@6dP6Bhz)3V6!T7Wg*6Qp z(S6Xfns(1Kp@nZO3z%{i^UiF7i>0r`g~tPAhaQor*|iP+e#-HbX&GiexxkVVTcF<{ z7a4VVh|H`UBCUdF!6&a-qCx6%>Aorj)Nvc6R>Qg$dUCkuLf!W++HSxv2LeDZp_6=? zvH~dP=zZW5m{?q(xo8im(^IhBBR~8x*-P%PIu9{t!*NoL13$WZgZ5O%YRMYV5z=|Ce?F3etENr8<#!8HbMWukZqoLq|XB|I#iBV5|gmO z)AQ`&uXFILd^pTGd5G4>*5~V5td^-^7P6wRublX55C1di3QU{7ic_uu)q(7sRG-&h z#$m<6FA%iCfsd-aUe%f$w|X}`uRbZ3bWg#{UbA7AOEcMfWoI_JNj7BH+9vwGHRisJ zhNI0*TlAWlh(T*iG54mWq~qXwTO(1&Y#dN-@@lsz^*edP$DFUsxC@Q1oq;g3(PCD^ zX&5(?*4y{f=krS6VB5pPaEbAJ-pTVT#`Suk`R{~1Z(;HQ7v-JrN%hVLIPZZ@Va>q6 z+Lm_=s3&QjHEs`K~vhpV=Fv;w+0@c4TFGQFExZC zVhaDwPS)7-!cadPytshvUeQ%^$Bp*iG5!J`&DSB-6<^wSxNuqqNI46&;s%J0T`!@+ znP;`)_!(ZU%aN@^4&Z89C+ps9L)oFx2%dZOEG%x?U56c&3CTG0w!Vx!RGLj#B{wdm zb>vqk!hGlE{G|RSOg8?Xwy1oB^eOot%*is6O&xpkl0#!uedFeL2J&8?`*`0Y1cSHb zVCy~llJXQN*4Qy13YxeiqS5Ef}XFFzL`jtzphtW#dOM6S&TxCF5 z#mDb9LBdn^SH?&rEXKmn+d#2`t;3?^>y_=b6dQ1Kvf~v2`Vey12ZFo;g~Hzvp?|@; z2UX;wI`Zp1^!dLgvtYeZHJm+~E1zWUgIcGp6dnT23v4Qg<1zDB!r%I4V7Ooz3{R@3 z34K(EbUdyJc`DLEi`eBgCi2zH?byHctq#*lQbTd|&nr~>>u}}u4)o4M545}9i65D5 z#McbWM$7O9{LRS@l8z-er9WW}X7y<~#@j~H^}@Y@JwP=67n@k7;b&jOH^|vgM;SCN4+OV zHCEw5eadStVn(eUQq@-Ep>fcCK^5i&y+d2QJ#gsfWguJy;nGC@@cbkRANZ#=8?!T- zoQ0NwwRqdS3ygk7!ZOW)<1T#1$xc{WR-#?~v%f6K?}C>G+X2-S?FqD!)mj)L+T!NGk0aF(6d!ZPWnoQN_LE*{xPbPyeH#N!EIWxe z!O!t>zrol&{-`GX*%?(M@~`7rczEDBNUyz){pmB3H#!^1{(dSMuN9AOu3N$Q!3v~$ z#leaD7;z7rGFpZc4~<7>dJaMH(~zLK;PdDfBP`&G_o^~(D@e z`@&w0AHP;{xjE&JpmU08JGvubqcn^^#>!`zN_CyFwYF2;fC#x=3!h#)p*gnon#zAT zSyl=b%?+WwO%pixG8ySy{N<2#ocKzI=@eTvf9~aWLviHm1UyLlB2YX8VLv_7(OO=; zv4@`Vm_Sbn7Y z`e_NN?8}>m!Y`mMr(`Zpt9!X=Gv zQx0~*L2GFJ+z35!{$xjf%1@8~@cf5Qn@?*zdH#pxvo_J`C-z3=mam^Pzlb3$#9Ir_(&Du3IPW$9IqWOP5zIWn{iR*WO-7fAv?Sfuo)7cc4#bZ*&(;J;L*msuDdwABC;fRG z3Cn4VvA=<|sIa-Y8~^mV2QT~o3sk%Ed9@RK-X_4Se;&L&a#@G#cDJJGy^>;}d4bWz z-$m=izA8uLtMIzqwQQnLoYisA8L{K0l_VUN#Ifx0u2>|Fg_1+X;Iz6q5?3P49cb_W zfrl0dleGb`n6N2F#LEv?eq9%GfsAGge8{;fyxuA z8RE*n*yFRc6sPLABWA}*B<%*Nef^MXPy4k>Dh(!xE7;Fl29mUqRQy-t)Iod>qV-D? zTbw((dM6V0vt~2tf!zXH52^b;R`fd!HXXgd=rg!+sh8%$i%zoYdRuhKcMw5d?~ybV z)%G}~`en0+bd=R@_4sqk6g)@o68$`$$^53LLCpN$K=pvln>h%j^Fj@3XW#2HO7^$i zgbn>7epe4$IJcCFUzDyP?2x4QVa2T|(ciWq?k}0k ztFBx`(p%WvcC*rt*!W!(SW>Sm!_81CP9(hs>uS0J@id$H@EI;TFjwge-YF~|N5u8Z zCcmQe9=v;TP{j4WO#HM9e*I2`j}GoS-FIfyKKx>^n(>!u@Ld+^tU@##t`$Q1rXKRJfsZIq5^D zIML><2_5%{@-foHDp~O;v@iJ#q`4;@%rANssu`9?9c1 zLk|}U6{DQs7d;45dBWp?Xgc7c&YR3^UJtF748nndddz)QTRyYeMYMerCl{E|b7lqG zHA=I^AAE&JJQfj#Jx1bFkzeuwi63E{$6}$hb{#hpX_aLxOZ8(2``chjfPtWMfQRKl z&APJjFvhGHiVGV6X->8><1cZy!8hzQ`7&fyURQk0S!O$~IGt)lbR6W$3Fl=%ujyjk zqdw3B_d)EuJWIqjuZyadh_|$)S(Tq8{}u|Q z88Ei`6htf<%YV>bJC|qJ@+f^%SUSC(@SkN4Z*MkJIu!TNUdl(Nb>>+ij~HoA(w7`x z@7%$nyB&uMde(TOs~6ZDoK4vEN2mXY`^EMwLw?HWm%#p)Xb!J2()QA#c{I(rrB(h{@tiJSJ*v;Zqp8QBD#H&HH`u3r2dZoFW4RL}-5^r?-WJCXjddsACfxZd zFma6|fA~c!drbw>^VtUFV<2Ek52?I3c@}ojwgKz+crd?fc8~OF0X!{eB)ZbNdc?6n zapgUXeq+5pGBI6kUH_Wuz(LYe^v0$vRYZR^xPlG<7T;W26jTd{Z}vq(2$?WFQO_N;@kL z`J$me)|mapo?EuX)4eBxid#Ru=1`cBf;!!N)Qh-jqu3C!hBWe2Q1xHO$zG>l6oyc( zYssgRS3Re>(*?O*jQpw6V=6vO)fjONqho`?&uiIwp`(!cATZ-eAF7iWwslxN zHgNDUpx6qEDS0C+ezWf_F{*qNG){j`oHYhXr@-psV5JEe{Z45vE%764ewm_TO#Uen zT~e266wXkMfzqOS%X_f42mU80-je(ztX)uvbi8b($Eluxd>UKRph64`T#Xy1p1_-v zFW|G6t--f@s+gYckF{>CSN#~mxEOeH^gK{5;)DfD)HP_m4I2T`9yo6gBMMC***P7$d)bSh;e?#uUmq6oCeIH3ai*0df ziqG%=#>&uPm;D9YI(~}oAEe{%HJfzWn=~j>J}blQzJ~lDd-wPX5FSCDCIabq zaMP34dRm&Jp?M;VPXN-~OnH#*3zh=)qXhYFN!kR-`vBnvF6~>S$*#^6q;oN1w=s}k z=fzvE;-nT~N}mFGN6n+&Wk}Z!^Ym=_llj$HZg+$Hv;VB=uvUq`#$HqUeD$IK1T-U zi)Cj@u+)_H58L~ee1Q*w}e&aNy;Nicw0-% z*f)u|dJOvajl)up^+IWKm6dfp(gy1qEr%=rbpVAGTdTZP{>m0SZ;$%*9%x7xD&4_c zJlBF@q6_7&IjFPX4p64aw02QKH}fvGpHfFEt-5$>6in@LMv%_L=?Pxg?VJWE+fX@% z_!n-)7YF1b%_UR040$^AsXGxfU0We>h#*{3*)V$b-pK-1#0&HNIY#AIo)?yuUQ?;Z!p!I^m zNd8`}Hyq7rM*Dz%5%UkX6Er49nnU4*P~IlwNik4ChZl) zZZ<&rMCE5d94aZRLzj9%8p>diJcounHj;lNj6YkQmr+gJ$SB@OLg8%8UKu)@)&kc4=k!@c-V*>{ni1(4UywuQ^-4BnzC4=;M z#L58me%?#Y`N@?fhWti=ojCQ*8C%$8Xs`Car@bg>AF;jJMRhwp5PxHi!tAPebZ9e< z&bds68#}JSGWY&quyUBZA5Qzxb#v!OQW~Py_(;~(ybU_sJP55*4P>(A|KP@h&shI` zbD2HjF6wn8-RWc|i_>%P_m97#?Vxqy;=Kku$J3bKP525qXUFmfXCDgRSX1s$u$><} zpC()yUcmb)_4#d+2e8W3oYyzZN3%wC`2@GGEPc@h^j!D@GEKv!@QuQ<27{z6|H+bO z^@H#r9Zc&RAttzm^1ZT|jYU*`6S_O+%X=Mw1t z?c1OHv2fybxUtel2ARgfoOegjt@S_6zxU?^!`xwq#w6xScXct=F4FZ^IvPg-C9h#3KNzFt+lHX>jj6QdgN zJAu|{^sKd{Il!{8DAg3U%|;!4YoYHxR`m}1=of_h z%97wm|C<`SxjIN=;}*XjA~cvRUED`-iv}O)taEEfw{Iv%ej17vr5AAVH&=+w9E5dW z?ZvOKls8^pNYCfBgL+!Q){VPRs`cZZ%%ZcZgdP-VXk)Y?tyS5i>x9&uI(zcD< z-Q+u4d~XM=XLaQJ^n=2i&PDVbQI0_?1Ho2r2vh4lq+xfl^IkHXm~|0@S1`y z^dKliLa&~6QLWRs%=_?UVsCEC7g7e-KBiUYlKL%^kMA6s_9gj?s0;seBH z=Hq{rp?R`YpC8%!8;e|0SJeQBya3*0)?*lH@&>2o-2}ogNf;yu3+Q@&LRr7>7_fP@ z^gjEEHQT&i^Wf|V^v-;QW@kg;gC-p#lhR;s|50eQ&w=lqIg?+ztO3GX7#5_@M&9Tl z|HgC_m)0i>Vf+UpOp)>|Lucx6Tj)Hj) zABp^aRX{wdJ>&PNnm>E>Tu)rqj=7o4#Bx0egxzv?mMeZ;-+=fa0={=k#yxbVEzN$S5z==t_K9dV7}UhReY-h8w94m@FH1tIg@ zq3$b37=P735>CkMR~uRVoCrL2U_BOWWPJVnY@AW96}R&p;rEYa@>uy{yw!RdDh#7> z$*)b`gXe{KL6{8Gch%V1+pb0R&%eo-OwV{6RF|2t)-Z$5 z6dSKU*5*3qK)%~lF*JXq-SjX9)arfWet5LX&TKa z*CeR=XDbzdR@okc8Rs)#R(BJ=epo76Ha3RAEmKw9u<$T5{wJv(TDUC}*Dh<3@Dv;h zY~_^LSfD;p7{RBv_hkDw+3{yjUAU{Msnl8Yh@ENEN78lSm`?pLJ7EzTe;UV%f4&65 z0XVU?6q0|wWlz#RiY0?qV#dmDaH+{0I9GfE6T3%qsy*&uLHARxE!Pkp%k4u^OPnJ< z6|6+U23+O3N#oh98%+3h9cbO~OhF8z>%$YXdSS@0=1^;w5irNjpvK~`bqzXH920Yn zB#5$pVOVF{X&CpSN*Iqe(fV8M1FCGSTi4}@3D4J9E936;peaYBuu6=k-O`P zdFhBHIyDtX*P`wx?(dCpwI!6Uc(ZmW_SWR%k?>kbSOQdk zNMq1;k6Q~t0SB@_ghUG zG>Y8Z-c{*H=2O>MUeX<_@mVp46~*evpa3TrxIUAx*$~!m^yS2vcwYJ zETrY&cYu{796;&|I38?{G%tkRN#aQOD$+VJEJD6x)NfGg=uA51C#Kdp$@Vp5lHO1C zH*7l+PxE<3>1>EzI0Fesk?L0UFAgcQmxfgV>n*65!W zCbjhj8Z$pxRSO25y97FmMr$hV*Wqt^wlS${4N?!Gx2H8H4#Rb&4Ek)X1*Ct(`$@)1 zW3&7QK{);X3a#p`Pd7K?lvpD{*95di;>Gt!M!FsomuFD_7^t2`(pvodqV6#A#u>J5 z=T`D9yRoZzrWkM7mW56Dg^jy!LBdhhtIFFbjf_-}g#Bkx)wtb(os4t{Bg|v=&bsto zCAc9r3O9bN@P3}x4fCVuEE&-V_J^5(_iTU}!~d~mGrut6DO@l;5!6_WKRqQ~Qvv0} zXA8PsIVvy%NG~(uLHX7|!xgvXN39e!Thi};KmG&KfHK`akxfnP!f&jkj4RFrMg_K3 zJ*CZF_*!ut4>)2ihi=-3O*}YL^`fi0NUJdD)RpaEo!A?RAHe?SBAoNCBX6lZU@2qW zn+9{WF2}pCApcN9{tn3?kMIs7ThGT6bmn;Ug;lJTeuC;H{^EO0PIIFvrvct$UTp{RRWfIwEuZ&umUdZLiX_Zt&p(!9+VqVW5zrDAF1pH|OuVEe zo)NcNzeE2jI?ufzrigkI$;U~0UsPDC@P#x8yAg7Tk(XpH?hip+9}cGseDGJ8E`L1C zf;7h&r2a&DPY5`&4TkK};iSKSum_a(R6TX9=|HfH+khYFoM-jk4JS_IVRg=c#pG2= zhe^V2$TMsP&E9qPg5!BeUxOf1BUsw95qP+!u$tY9B>cL=9P0GbR4zJE?Q6Z)U4}JR zSMs#I?$9qc3~xBCWoxeL@sUZpvAk&@d7|S0QS~|bY8kZhis<7v11o(}Nq?P(jFqQ> zum`B`d4?<(twx>`q|xBPY}#|t&RXsnSAdZ>P9otIINWq5pQnSwg`E5bd6r5I&A%l5 zDZNtml7|=sp*nRT=S8K$XNURh~&x-cWEAVbJI;fzuJK8tk3&28X7ewHRePJ^#=qM((% zBj{|I1?N_$;*`@)T=||!X-mbxIa?v)!2%o>6onm1%q7hk9CEFz>Ptghh*|#}HN-{u z#>$*@)!#yTUu=D^4aYtl2q#&O?i=vZIj$j{gOAxt47<4tf_99-mebRLbODeKWKF89 zf%*jo>V6XB;nbLLo55+-$He_C3oYD`vMZTPb$ z5Ip-RP+t^{)pii19VBTkXnXFu>I3qCCxA3C54+hM3)8+(&-8}I!7Y`}_Hc{LLq@x7+HqDy>9twLk#gXYiYb(}8#o(VFKUjom6xhuT*XEpk z!jvBia!Q5-K1Jz6F>|>K2;-$*t}Ab1%@0vR=|=K0f_O~(CgU|z+KfDeHhj}ge7v!M z{aEu&yZ^8Yjv4WkIQXuXY7wYTwPz|a@k3}ak`KgpmY-qH6-PPs zdr15al)n`1o)d?Er|lJ_`9;*UeB~2l*wzTr0?veUo0WzXR~vdl-Gh;!c)rZ-rBHd4 zvB^B3ECwdsibC?$!v9gE(%0y6VheN{wjW|&gjH*jncIDUFk9UG*Nkdu9uS789<9#o zel$FS(XSH4hJ6jV(vioS-V|4V?*{U{lyd-6JY-SY6I*{1xM21EYW+XsKoSta*SgYJB3~p4D@N)?3l6S9AXXIt-+fPViLel$1+cj-D zc?gV2v6T@ag-E!>4`Yq+(8m%Q$6&hUiHW9n!AB~i6HJ{ z4t{glN?Q)QFiYjNT!Ago8KQ|L6mNKjUVl!>r`FGs2I zRA*UK12}b5Q;Z2;1LQMssgZ%ab~r<$@VUG^OZhJ1pblu${)A9IcyPCNz}`3JNA4D3 z&Wkz9|8vSwaM{dxLS?5(9xl~>aFYK8K^YNHT|m;>r`5h^&3g+v^QRkExx&cSzkzU^ zk%s}{t54a>9Hp6r^7${??;|}N2{YnqD_uf7Py%{cC4w?A?S1Vo!lv!oCAx1l*ABPF z;r|X(Uf2g!E~~gDgw^LBfrs!(-K{|Up~;zj5DY%+a^hOFF>i-ALR_l-3Hg4AzZH(; zy@cX<@)NM6$r7Z@2UBmRV%s^XKv@U6%>2WY_a)6u-sK{Y232ccv>{d>JI(i|`($lI z#LDxO3BG6hFViy~ls!>ir?e)cc?IJocFZ840BEf_>1w7ts`Ag3*S;{NwaL?HC=12( zk=|l#N;A6V63}mWRb_Nq@&-tF4}=$NNtUO$`79nc`Wk63-rGWX;Bc<$vV3@jxM}MK zl))h35>R%+63(|^lq-`D2*%xWO!?a*tAxk2;}BglL-7TY-$BY`m|D-+*&QThJB+vu z2(xI{*>yl(6DRC!%!vzu=9p6^h3ATWIr)7-`n;MS-&942#>>5sd<_QA?nt?Kq4w9| z-7w_D23520eDO|HnXh|N1{(d-NW~9IYpcv~U6cV(eJaf@DHqp9cj_g)Ge>jc3tT#m zVRt@{@|rTF90Mo|Al-Nhj7pNK-&5rhDqr?;wZVVm>+?3VC-CJ>r=#)s=8#(Zn`VCF z2atBk6bwUl)Au*R)K4|}y4MG>(SpHJGh&pq2r%T~1{X9N8~jGo%@tznlU#V1UI^3M zjOO@c5;N&Vzmw~+lkK}*sA)B*5HBXyhqUiy;5U4xSXTcI7SNeoi(ah8IzbKP6rJt( z`M>7e!`TenI_(!p7wezSu|Hgtr$~?Zz_@Z`(S19g!RssggdU$<~EfOpKa``gz zHXQi&5el#PVcx?*BI9ikZarCxuK$sUG8~9!p8bN-g8O1}r|pU*SZ`$znD#Z6Tkp4`XR^KJs2awyA3YQPYjImyxZhOJ zIMAZsW;UkhZ}E3zZNB(d8(w?L9!Px({Ap|&bhU55m%QG@-|imBhjyPNZG7q3$Ke|O zWM^kSWXNBn|Iv3eCCGui?G}Ya02yJ*(nj}^h{%=`6p15=s-fu$u z&vK}2_TOuqWYkJqxw!<|2lbNc>Xx&CLCa_Z2mj$5=L52 z1&7Jk@PAo9*_ulccr#&}7+A-Y#~HsA56{kqS6hz(eV(d4xxgk!{+wup?sLQVyk-xX z@Ax@z zuYhOAK0xC+17%V6HlRPay(Av5CD};oHSOn&5bY@1|DaRIXK%fs`s~2BnS6J}Lp-~? ziCp?98lGfaf_kIUfz}xRH^xA^cy7j}%|F2Rgh0tV?1moBZF$G$`l5A26-;z{D*V_< zP5847=r#Hg6d!Ry^`}?U#ysv(GtHlnP1wiIPEwzvZ^H+yn2WxWN0+IP5&& z0rRso6tw2p#UdR2n>RsP3&{NWNl^bY%QYWR^+3HLTd@5AI=|R23;wL_#c!S(1*2gC z&eL^~FV<&3)$ArR#)9^)%rAhUjvDHxS2(yelaDuN;psDdc*(w|(ye0(--UJLn$OAHY3DCYrv!xA4`xRnWbAA8trKeu+*2 z_UL^N?7GEM4K9SdLuM>hM8aaLVSL*6I38ki91WJ8VU|PF2#fZEchg7~Fmwh)mBy*Q zr!{y1FT==-^**L*6se!659xQId48H!^InPxwNhn`aQfb`^^86TD<-WF8jn5jbVDZQ zznQ8qPotAGR;>Z2x`nx46CtctA-J?#E#EGv$JREXGiV;}K!r1EjcScB<+T1VqP{(0 z_i)*?a~ycGeMmJ556@PJE@Nji#TNlXKcG>lhR<$M3UQCp@kVq_=+mMV-E*;3ejBI5 zBX8H_QKhD`XQNGU?(ssYu&aZA9@}5qm7m#SsOCxz(hQ*eU_L_6`UX7r{WN|nawUv! ze1b5f3g?(;q*0)*!YoN+!c9MV%TCt1vcvoz%vIA^Cf{i(*S&V;zPyMP-+ckoYwktD zWm-!=PS=bhj1$3m#&hp_EAJ~DV(oh6D=uWGkDtOJ?R#*q(W6;|=_gP(;jxx5hxX=~ z0w*85#1_$i1oetctv#QxvkEPj4ke6RA@r7V{JF6Yn)NmU#fLOb9A%pB-L6|3&|i!& zzG1$my^{gZ8jG>s&!ImbD2F+hh*-VrykkFqJil!dD@#0p$J5p_;$|rPx?JHBwC&%K z-(OZks_QaH`BJ1XX-VDQl0E|-?(fRyl-Iz+BDFBsGZ?7H6)tG94;paYS^F@g^AGml zr&PsJK>V$tH4)oiNSLITLgQ~(q&|oGk<(hrhf8$ilG>f|)~@z=YF7h=mAux7IMpAl ze4`sEoef6;7PN&V(t3lIoIu zU9=OT3~$0plR@%+lnGYOTnGEhs9+BqEbA?ruKUNFyc2MlaWAC$ zV7|b&ZCAoVXqTEDHNON zC-a>tIh#1W!L+my!OHYHrqj@{)J@)oFi{APkRkk%2&|9i}6-9RtWpD7+X{c0R@1h*3 z=jK<>z1Xwf`)JW#+;A%a9bF^v!-_phqrvjuZa`}){M-v@9yg%n&?fTmsSN1UtPU#N z_8niT-7g!!%$8-e2i*W!D|j1_CW1a)2TQ^PMm<(kc6t_%sMsfbS|x$c;1R^T7d7g= z`q-u5?qVz1ZPX_q{V8hf{jU54j5W`KQ<=@+ddX;5_jISyG^jj)Tl@Ob-EkG`(7JXt z*OB()#CP~-+f#JxcZjsq3=C~IM`0h^dm@q1&jNiuqk4lOMs@L0s-4{RP|Jw-`LMcH zXg#bIKXt0BSbNh1Qhq%`kAdfDuY*A7?QtBeqozS8UoH4HUW+4rUJ9i}=(7d&11PSg z@ruQt)8Kr1W95fH@r{>jGF?|~?l=6SXzymuNl#(I*NcL*veK_i=_^$?qyxF?lSM{d zfV==aJv>yl9Ni7?#wEdkUT>5Z5cR|FL9_oREAD_#sn-kby4AzHTLEH}!4`4i;7%ak zL^bF4a0JZ!y%i6>xC_q@-a+zI{Pz?K{xWkghy#uInS4*awD}C?Fu7l~p0S(ik1rDP z&|=m}O;4{ASg|S_nojbgdcF(FuXuSma>7@%2)ooVxhx0B18En0+abd06yt+!&YbWa zn+-WE_hpAO@<`ZZ`4mAuPh2l~qXyp-ngH3=V1RP&bd7#-WxBVM4hV5?>Wae=~Zg}dVP%ipk2?q=S|%TkX*cAh!AaOEkSI)6pj$1Y=pA>#Zd zx=(3^gY(7 zCqMqt5N9y*4nSTGXkIZ(YpLoP>GP4a7EG<1gLBFwxO1Q<=+8Ss+QLjWnAZ!b-ob>< zxW2q@722+NE3CTKmZU3KWEFzx=GpAWQYSXZVw)h|*H9gaSqAMfmhy4JY%TB5LMpz$ zMH$cTiV>JwQNJD^{S0`^v2yo5!HlOT?dP^m|nXRu?8ed@`X%pt& zyA#ebILt@`(zp{~{?ByM6KhcQ5n+ZbUA+a_lzB*)C6oU*hOW^Hlx8OV!~9rhMi?Tw zcRP9GW+>i|T1%R6DGaO=2fwFmhCkcJF;$;4TaJ>7gGlRNQn$6n{Au zt{wH~-)C(pqHLfVO9|(Zath-ARG{^Roa6iPzwfP~*^u^Xt$=z@^H=Q=o67Q%@4pi;XOm%UHH;8vFawb{aCP7%!Vn1|G((0RacPW?_=?3Sug zmQ>jjsD1@;CG;9Joid(5knpva(j{88CPQf7AMyi8`V0v-k=7U#j*aIHGm64*YRsF)DvV+6xBRIN*?u{c+R1G~8UC4+=l-##srMA z<4mQE#PzhjVt*;=$ceF#anXr)Yn`s0_GcR-T;#MSa4+Q$>uO`nLk+#C%KdJls! z`j(8ixLV^-c0#;0fWCeY$rEWvSHQIm@oIjm=Zm}$=I&dC$3Ha* z=D*nF&-#4#gvA6AM~Pg=qP!nkM+^%7Irj~&-%^jcLX8r#6_l(F4c{RA(!`{01abjGzH z9r@`!X@>!jQz)h5#GpzW|z!hP;{tT^xr=yeL0 z6hBw%#pltfLV5X*@l29`6AD|8-0fN2dz3HAds}t^;X9}Aq0Zd>Vm{r=NZFcHI#cCt zb%GKx_cGm+w!#M}E7KAOs9s79hXr&_%8GmX;_ni78t+)z(R)ZAkhwq^7G)*DjIusBcm0_tFPz56^Qjus zC=IK)cUFTlBA`tsX5Kie`Vc-(sDXL;wxlz*0_AnpU%Q^>$gdqe$0&agl)F&2@K%tX zN8$;1xpbq_Xev9vQ7yY7@g}H@=A--zk5A6zkCq#W+s8}6Xht?+@DQZDlyubsNciAe zr235XtvBfxPTYZ!UtM9_$Md-At%K6aK)O!#0%o0Tid65^@6BjEJOuSR<;?3rc}T_k zOLO+F3DrW16*b2N>?i!-*(lO@|u$c~RAU(9bbZP!vj9)unbaAxg2_F6U3Y~%QBWEWZ z)^@@`KAq0d8X@a?c9ZJ8MqikX(WmaB8=d12_uEhodUXUtT^xAR{`qj?!%QejSqiPk zJrdUz&y({;#PQ`bqOj|XHCUOkPYet8NA`^c`Z_x@ zKS49-l`DGNT-L76UW8y@C9aI91sg}K1@)QwzuKd(iJ=Tx-$nf_r|aX-GvgrTQb*15 z8T!&G-V!Fq$6+tSTjKS_0KW8b5NSp%Mt+0k`c)a?EPn#2-D$sh{shN5ZsgXt zf_S&Q8W388_L!qRPzK~Sf;;_p@dm#wVeGI`(qK64Kl6z}=&ft;YwmD3x~v}m;Ib8+ zJx>Z}@;q&HGvrvu`f|+UWbM@&9c0H5325Qt19~M^kUB$GZjRf{%Ll~B83Qb2WIH3c zn7fWSmiOh4{7%4TIukTMO zEL!uZy__6xDmT1ptJa&ZrZbA}v?#*7JBcEr-XeKI?1IdppRoC`=j_ji$)u?m6t+Md zWA=(29y?V=pBf6c%T7Vcu%XcD@GAtnZBTpeT%6-{4|9V%;;8=FFn#JxQSW;dI-~_@ z*A6hl@*}i=M39eM!$ag_-9)IrJeukK7%#UaH4%N6N5HM(0`Pm?NWPxF8E<%8z)puJ zpss_XItq7MkL2z_=TPGj1T3RSvn3pZaJ5 z_Fsq3#coLRjP6d4&~?BztP@}_qdg08QovS5>nDs}H{laP3$*lJIMK2Of9rG!+U~yv zMwYRfK(95J@#rA#xcL$aOIBg3`5_$mZ#MsXyR|$>XG*pT1^z5vPhOa_5M!tIeBICCkz`gF{|TPU{4hdpxB5 zo?f#i9b2=s==E^^VsF0oCt|jRx%QazQ8w_h1E=?b#|t1H!+d4 z212cIgMVR~>_(w5cZV0uemM?J+<&l^H|NMluZ?(ly)TGqJ$a*b<3-NLJJ>C{p)A#E(4+R zKUqli0j0wmt2IK8d4o8nT@XY4-J#PmTRF}*6>0qV(0o7MGaDhi=BJ~{?UQKWd>$V7 zZq$5=v*F*88gTunYdGD@fQQAklo8QpvMPN8`vmps`D5LN@_&0j!;%(VU>I6Rnj5T*3pMQ0G=G>8Q=rIn@J2b;HiSDv#)@s!kuyk4> z+iY{1&Nv>WcmroW(UHwcET!tJ#A3kbOZQ;*ymPRA&N8f5xs7nEwjdnB%k#Q(H7^q_ zBlv;bc=qPyVB%*7xiBLadW05vS2?=^y`Cp+8Gt*|w}Ij(EA3^iLD6fWc#_@O1ztOi z*~_VC@$Qd6j7`+A>0U9QeO$kqx2;~$bMBY4+NCBddHU;kF(siwOW2DOg%hMWn&46s zJxMqxBBD0{tvB&ZIu7db3a*S;g)6Q^!}h3?-iyn6aauRIB0UA`4@hBu-N#^s$63Oo z*Fe3A7kr#)O*bR0nLIstsi3v=o|XDebeQfaJudGT4R@TZ?q%Xju}e2asJYX=m6rey9@9Pi0q(4vNt|t>_z*AcXvh05ufYw5=R$*j-!=OaJL2uKE-ZdtFKx`yLs&bn zwwGSgA@sCJ6ZHq!$qV(nLXLl1cs9Nf^+SxbvFS(Wv`v8D!);*EgjiVKb~a`=`U*5h zFf6W(rt;xjc)Y#mHy!Db?(%{TE83V?@kS7o;_2g+H!i zzTJAfddp8L+*BIurk$~vYe8os|Dt_S_YFsbXggHctaMT6*b4mCq5x`4`GGx8m9a89 zE7HvefUuR>Wq{%tP`qZ`ZVL_`Q6HA{cVc%hOyKBi$+j*tmxo=_NFz*A7{mg?YVp?X z%yH1X>-eEYHXi(4%S-%9g-iclpyd-oIqc*&_PN52Ydr32iAUhlZ#~&N`yZW2I0_Pi zwF-NY`bOm4X$ba@)3N-~K6&u>DnaWb+CTD^wt5@T$nQ4({ADFc>k8@}Nj1djwXot! zZLIfw2_xM>9R7@{xm9C~aOQH8zYDwOzh7+a-x`;8x&rfU&I0KYNw~x4Ga+Q$FesSv zlK9q$|FhkJq(x!Q<(<%I-E;PMTyIo8U7vp@Og^doBT|rfp<0)A_(|uQc=QDQsD4{LY^-TgkrINJS_afc?g-HAXonI^mg?FlE3A52Q_zWWtq5P7Tu#Gh? z_7@B7Hj_8{#%xYj!C7NVZZ#rFxH`0Fbx(~4(&CJ`kWU%EM_jtRRbe*#yyU`jX^+gL zjwx(^;?ZiH7%`y^?7H1Skhfw_13zN1?Ovw5mPW6x{PeM(>MeXEcA$CWYoWYIsdHYn z2MB3bhhOkHh~_`fX(NhCVX|$upxQpF)`t8Vkp2YvH&3`@P5QJuZ|Qu7Rc!uGlQbb( z`(f-0i2pYgBZ@{M@sw;nP!~xHqtbPxSI~HUFE#(99V_6-z9+EfQzzV5!%N{lkmlnb zYP7=>uh+@c87rXA={7KXl~Z;998XU~nrnP#Q3t6Xkj4Xw-)(HJ;llsUqrKS&oN_k` zI{YjVF3rb_h88=4ya^+`5wmj}@ES&4=yx#9Xy5TA8XBK`o%$H28unNG#c55s(o}g5 z7ovJC;iR;<_=r(2%K@v}N$LSnP+Ua#P5X`=p1~9Qj+8a&S#Qzyt?~|7va}V|Y9!iE zI|= zXDBRESU9M2t!mG&^nH5hVe(&nkopr>_d5+uvkX!BJmO73xWxQ*9U8YYam$wtg?S`_@Hb8v~Cv~kc`EdescN(o~vRe1ec;qIJhaLpNG%#=uko8Uy|XTy z|5-rz+zmIb8w$!FO}bqu4r`CmwNGZJC;t)Zz13R3_Q_We8JSrxDasi#1B=+%{Vp=eqM z=vID-y_?rX=_SIk9MWRRIOq3THq@pkkhTC*!v|W@-kMZ%7x89QQ#t;MQMHaBzb&#Z zUIm-D_N3{u1g)!}tVh)_hMOP3FH^<Z+&{>7!t+(y>ry$p<(Wd510V7zp|Uqaf2^ zH;^8n*PMeV`@7KD2X}GKWqWuMuuc$%BF!uQF6sw&r)8_mKoAyDKGPY4GCU<^GVJZ; z2AnbouJnqUAB7f=pX0~IFC&fOvo_#7s$6ssY zDT9EU6-~(_*X6NZQRKh=GRpo)8?C_@!$9uSzk^iVNm!t~B&MzXgi2=+pFmo9SCvU& zq-YK22CTu6Wmkk_v^BFen2UQeEjiT+(%PurVJIo(+$X{s;m@B*Svn6qC3L>HsgybTEh=VJ<5qj^R~A6TGFN(m48ys zNqK=WpC}HH4%PtiDN??{{6seF(zPRhHUyM6+u^KbN)zrHcLQp4bW*;oS_4q71L&Ke z)=nFHx&=SJcOlxmsVU67Hh_$#{a_Q`7Ug>M5#%p{GEqTUv+ScUArzAA%5m77nwt1l0{e27O6KSbg@PP!KRZLK5jmgq_1S4o)^ z;Ycjy0e#>)#^B)*hLZBa>bj*|oh$#Ku3^&tKpbjw0V%Hp;x6=lFY))UVu<|Tc1&Nn z0t?p;XVZq7<1P0U;%d7M(A~KZ60el9Hi;+1*y46l_{)J~ zn#ww^DPa1@jc=Q}9!5^l<)c#mgPU~U?aR-{M4N{VWiR*tF$t!8p}i>{I@1^Zqc(DS zUmRE7jct4O7*a-E6;`{Cv1I*ktWL&9ly|>jbj%!X^Kb~5ZYyOo*JpuEm#%!)t=odG zkKgNg4-MzuLF>Yk7_;ax#BUrZCq|y9-`Q-%fTq9T)$;?`@1zaf_8bgVW83qJdb^ot zz5^C!f5MbeedVM37c{*FAA{IKyTzehT1}lnIoPm72d@xx$}_P^8B|4 zY*vcPduIj|JnkyKH=IoSe07#@;)`LM<5BTc z>_zAL?d8cotz`bfp|B~m9jyHt2&XR(l1n@{!kwp9yxE~0&`z&0->tP~FRr%co#S$O zABU;3bDW!8{iGM_owx!<_q*_<%1SoFzXl(jeE?h9UT1k*w?gxVW2Hme=Q!i}E9_tU z3^pHe6kVr}0{s~ig>{`}u&>7~Ugtvzz6{Dj`V8%<=uB{!rQv@cra}I~yV#~s7Xq&x zgnK=Q;U>Cg$GWgH_tnqAe-oQ=?F)D5-97;ieBXsVOv1dczVnv!dfB3NJ&o<(X?(&S zV@7j;UVArT`H6<|+kX*kj-ie8J6fdPmk%xKL*vaB9}5HURB4D@HE|?3UDC&k<^Gz1 zum;}GO@o}M26S&@KUt9c9`iDn@t$Ei+SM&~;imJ`WQVxV-bb&w%H5_z<;wm+vd_UJ zTz;#8EJ`(!e-qCM%Xq=2T;2en(^<;ACqU~aXVzMZcmMk*9Pc>F?`aKSM}BLJUSI;W zerRQ!1;KeK=+U+XMvQ%jQ!8i>w~nbe`JWf3wT4>k7;ohg!56xPBFzD;Yi)r8Hu&LD zi`8=crV)@c;T9fgoXMX(V)E5yikHyre%U=eyg+J{}@%Ax2+HYhYw_9K! z>xxv3M_`edc|#W+xjDD}u!_bVi*-d~UUk-1E~xhysP6FF*Bgwk52#K>P_(0%`}&^v z)%G2z`TTaB^NVeoVvV7uxV@nR@BU^iG@46$`Gics+n%?f;p+s9c)b;xu31Cbo4dB4 zM|+-nS}>i}1?YadBd2;2ZNvVuC##-<)!Bj2;pTZ%fefZBV;o zl-zP@wdx!B!n}s;-GhAjur%1XcZi_Rk*Y3d|mYijO|K>5LW?jIq zu_G@(kqCI(9uJ1^5o;{XV2b?}d}x>kSH^jXe-qPj>Vd1c;cOB=UGWC>{#yX9>zZJ@ zCkBw!Yb^WN#FJap(&8njFF<|9LnC9wf~0l0^>|O9YlPoCS10#o~Wec;ekcDS9=N`8unFQ_LanrL~Sh z@>f$?yTnJ_PWa_}%k{I0QLEQ3x99?%@&0QE5NfDI>Em35T3e`Q&$ZtX54aOIy}1A zo8QmusrU%o>Q<)x;*8+jG1E}6b>7%E(-*Oo^4qS$t zt*Y}z-kzE!g(q;|8WS`*@{G-Y8_F95rhhW^*s}{JiWhB+mW({!~^XQ}p2m98Q zzUzMg@uIB$!kAYdHBp`p%Er>)v7GH5Ud1z`roI-WUs~c;x$NmFY(bm4(tKl=m;3PT zs;-jw44`L2p)}1k@)f+?^O{YlRSTLp{s$Azr$gA3$85%WEgEcZsOkdy8byj_PhHuG zYI-o{w1;>&ZZuj?s>$CRc_(yyHWSV=H1v}bM)8|o4f)^vT2zBgk+e?R;9>yGEKkN2 zy}zBZd=RYl`I9J@)Z2i1{!y&@il*c(s!NA+32>G&<@n5hBD}X7q(Q#uezXrR9exrn z_ql-cZjYe4tr!{M(EvelwPNcG3W`TuaOODC~x={5J3b#oMF0O4L?iIG>) zrqXvS-Zl>b!+QFH@Qwa-|NrV&DRfPaa=6D++5(~-zK$5mHQF6ObUToB(*yplig*`)s1exDN%hSmIu3fkL%v`)5Zat??$*!|24z?uZ( zK8=>G$<$$e{=8yJ=ekDsl5`!KDD4C0ltK4Q|INTB|BX|62&I$ia@~G|fwVc_+9U7h4MRr zfoU4zU0#@7UrzRjQCzR;jq1ch7+&o_+;E6^)ceasiI1ZXtK(kAdL-tntUZ zb9j8jIfVs2ae$M;m{>h}1t?GYYeEM07;DJC_fJ!Gjl|{IgM8aV#=^Hq{#E7--UhSx zb9g0}qmiu0BAu4~N(*s)QwLmr*jSP#!==;bg!QAlYP>u@X)}6^+^gn@6*^63kGD6L z$9G%835UJeb~@oeT7%!W=*rgJL-5jzdziiY3Ml=h-{IXpHKFh18$kY^mxaZ{rno5@ zss)%awOJMZuFt35J*y=NFKW!F`~m3=?K>kBuB7Gr=}7()S(6&P=A1WFZ`OFFvKId& zwRmrqmCU*FDx1cF#3y3E<#!EfoaWm(I(H?cHYZPj2WxhgXC7{cqbY~L;HNGx2pNF$ zGP^1t%G4U%-;@sI)p&=QwYm2~2fUJ=A=F&YY2JlutcE0wMbb%mc&st+RXYK13{A$) zHubrxt680gaMCwc;gXGECAHCbdMBHo4&aWLf* zNjyc_;0`p^&qvBL{7;85taj7~B#(YJ6T6vVmZH}|6wd(L6R$*vy zcMAP^Bew9JILj1O4a2m>t~Blrnveh4@yJJek@ANoVBVjqdLW;{ZsnP?q$X>j7wz@X ztwnFRyuq5c?yifGJA3j|GcJ-RO-8~Iz_Y*5;Kxt&$<9;S27e#f38hzU%dEJ{ShSYX zyZv>b`6Z5PFXw-I52W4dy4kjNH6hl94p9=ru6Wc{Z+-C$+S$c+vD9+8Xae@>?n+z-=G8C;v)Q zPI*S;46e^rP9!{$hhE7Z8PI+U&(?@um%IdV3EcdL;>{-yInb=8(mGt?)I^e}Le0)V z)DJAunr~rvVMGl{*QWFUo0!*?dDiu$T5qS>PZ;?#pe*Emq;_+v$qj<=fQ7pTbE+2@ zy(|*4eO>VL%tBl|=?Y62c0`R$^LN++uDhu@k|sb6T}Q6gJ11$NpseJsIATN0Hqp5E zahP*vD5D&!>O{UfHxp>xwaSC^Dt?X*4W1)qG)OeM2c)@3>m^lYq8y`*ZMzZ($D}I> z#DT7={^85#p=f@48~E(VB8I9^^93uKPUKUrG=>BD`%!akCDvQoyUK^x>6(6u^06q) z-UezO$+M$}Sx;z{6{Icb`W2K9S-PvP2pSY79S44QCk5F(^ktO*P zI`1yNo!p)khVgYnkhoLx-s&Qnotwxg!)ea$0N(!jI-EK*p56A@$q0`SQ+q1Ae!L0# z=G{QkSBdD?YG0L)Q67mfOM4wv195aCo4)J1c%J%|k!CX6rRxOwWc5=^+zI4ggqO)X zRkuRrhVh$b!gS#$DUWM11{VtD=_vmy{3>4uc^(M@uvmC>3IocrJmZv9+QgzSFeI-E zv^L65Yl|{NA;*wJ<(==0D>8tLS+fv#3SnS?Z8Qq_F5=rR>3kHECY4zA>NC3#7u%c5}ja>D;*mD8x7 za7mEg1BW&Jk^Hf&Y}y?9htt_tWAs_mmRo@IT-CFnK8nhgK=sY^+t7DXyBnhGuR&OU zl^b@u6;8Y#q8(QmjFf$$$DOfS@&a)GW*Cz8!K+>=*dmy7y)7%Mc#yKAQ2vdFp9AuC za?Y8XR3pn+DeYfF^T&)M`cb|$6qKicYKN(HB`?7~(%y2yD~`rJn>~EHo@(0`=?{-R zk?K09PoP-mebwFYYKE$PbkTHX3oiB`PyCck9FR%adCU4XUM{|!JEWx=gS(?;UAk6c7jVY!jSZcQU6D*C`ihp3?PU*v*@!vY}!8w@4nat@!jYw34;cb z`Xg+v-XG@GaTOBhLY+xHd6%48eDCMYK+jdC;9WmUIA+w4U)#C?37fKOUjrz5;}8D* zi}2XBu0nZH(n$DIw>OU6{~H_CpXY92#jAP{R7X(b%s@_K6{=69yqn5_JuEwEI|e*w zRKr5`oN5{Dq&~qtsF{hvZ6e}@Wew%g$-{-x>Mw4sxWcT9=A=9ICfLxU>*3keEHz)M z-ays9@(B8gt2N3$=i9ze>%_VY(htl$QF(c4EW)ma5;Li>?$U)IzsK-1c@x9QyUC0`FUxK(_w~+dMxWUa)__ea3 z++Lf!i=G_V=bunnW5;y^PCZ4OE?(frIxei!wk;Y}3u!Qr@`D4m{@X_L`+X*BTw^Ru z{m@Bq3+d$w(d^h(Ej^!V-~#3pHzQ81t}sR3wQE&YI@_%p?yg}^c_tJ*ZhVAQXG~RZ z0w}w%o!)gYs);A3-T{_8sH6NT-l9G5sP}^R=gi?<&-y~Y(jlsDao!kRFl$+Woe~x! zX{l&G@}csvO?ca7ci%bIU*&aiH;%YgbBpg+Hc=9726 z(sAXbMS5GrZnL8?U|d7l|5+Cv{cf;G=&)Pl{k*0!9x4qbP9naX%r4~J#?na}(M~$U ztCs=g;_G`?-F z%YWI7qMqy8SlxA`7C@<}0f#*H&szo8R`TURV*wRJ5P9MZa z{^yKGk~@hm1AMvphy6mFb;swnQ<$HN3B1(Vfjg(vlTq36Ftt@zoY`-voYkZZ2d3AQ zvqIhgjQjxwr}o48`WM;nKjGMqzQg!AV<`4IN%tJ~4g_C16SZ+;2bupofPZ=a23G{r zUM6(To=xlSZk`u=p!wjJ5PR|^){jrYt!4l5&=D;mJUt&X{c518Ss>ia`7L6OoAHW~ zEAiaE-P%^RuduOiXC4^w1J925q)|WJ=C|QScConph!@N+naz^kc=NqEE)Zk+0H3X? zD~neSkUOq~z=jS6{BXf8ZN{lLV(N$rylv70GAq90twq^rHqeacthpmJHoL^f&U-}{ z)5~z^%tgQO47_>-GWu<42`I3P9wx)hnCUGQ!c>p$b0kVbq#_!7D5I^EYJ%5j9% zXxYp1z6je45cg+2PThDCU%xX01U!Q!sr@WrXo0|g8wVLsP4^Lv|Uqjscelq@DkcmLw z38_4yEAvwX>SnBkMXL??X!!-+&D@WzeU`|>1r1@b%?sAcawN^sB5?ZNoyT4532__! zWx(;rf?Wyb*4f2uaG1Tg(>O`JADjCa@ymcQGP*m;&KJ&z+SX4*HQ^{t+67^+A-3## z$0)EmwO1BSTCcfN^bDHbB3Zfy8k}{R+|5ub=pP=mLJ56$@7Wck#%|5EkLE zSerMu1s{}o3wK=6s0*r3c|{1NTHJ0vOuFNRFz#mDONGuF>=lf^oqdcnQ+ ztyrCjx|}c}3;geASq)F3HJpWYHlBdnCg*WOhxK4TqoFK*dR{d7mV?zAe_&IrpMpWp zuCS)-N$B?DB6e%~4e5IMu{$k!3U9}^44oko>S?igrycBT(;r}+Jwp=Su=OV=-aj{t z{djgsI6U#-GmLZaQo9hXT1Toe7Vm0{<=GBEb-`^li8%GsPwZ!FElxHr!DaW_iSodw zNO+*Vu7c2iP(yfTy;C&FSwZz~fR2tG*`ak6%vRHsyZT+l;j4$>j*%-sJLm=Nv+fL? z%NC$h-d3ui26Qi%4rX~|Lgc}&;<@b`q`AlBQLDLnpCr4PuyN)-k)1(fI(SC?o5v3O zhh6cF8&)Cl0IYwvoy`nt zfV%JZh&>rYphFW!IMD7oYRw12w|onx#x~;AWI3sJJDFzfdaP==rQC z^`A{R;SBnZ`puU2t;L8F`RAerkl(A9FnEy*n!qtyRYQ9+3dNQae4ot z0Lrryh=cvO;*Y@&hmq=@cZ&?fUngH-w@4ix@@g>XIj$1WwvQZj_#&+1lqTqw9kII-2oDCfzu3Iz|4veBQAKsGfKGNPl_Mtuai$+ZuW#IZ?e8 z1Mw)L;~MDvyC!6LIEnLpYVwYQt(e~W|M<`2QJT`1Vc0f$KSqSnJ>>mvLg24pc;WM1 z)sZktyDSP`4wP@V2O!NIs5%=w@QJV(e?g9aZ!M#|4{NO}+VLsY<0N4iMbiw?FK9fC zX(K4E`nhT_U*UB}Th{v;5*Nu2WeY`vj;2z__b(&t$rFw2Fep_I*#ox2p3ufX_`&w+10-pY z$k=ckidW{5=G_FPXMQ_sftOD^xxQa*N%JXUEJp!xh>U*LixbxIr{zQ9-;vn&&^f## zkFd!vt};Ex-%#lN1xxF?@$$fNP%bK}F#oq6lXhOW`Q9-cj|ZB=`hMZ~=XP@({=B#5 zPG~VZIPfW?Ta6*^`G=RcU%`MOQ(3biZ?aDxaFb!$Iee563c|bQ;YWL3bI>#B?Ya*R zIkZ)^%QZF)W&4nQaPRsSRM&X+(@H!$F$|5~Y+(7TYs(I&(}6ex!|DFe$Ghsu`GY56 z;IA6cDLhT_4K}u4KwNPZr{6s!8qRfO{)2YO>bm>HxBML}N!v``DH<>DT{q@2LmJ_Q zw7&fD`5e*n_;%8b5TIJcj)V8$>}wmu`@uH6^=1Jbk8Od(SNbxwzcFuey&jN8!lPdy zw0A|I(ii51pV+YEj+*)FQedV1Cv5pbN01g~Zz*Xi7WL{0s?M@a1EiY28jrFx?;GC3 z4SDs^a`ig=)dKi4&pv!-+gx_kEf)M6+)~^lsaBv}KiW$wXaXay2IUd1exNnXeD@fYri_-0!8iCNlBUsEb$E{z)p#uz zV~x^Kv&?WtT*utzAFkqfnirrk@kLhKQQ@8*M?I2*NOg#W4`{ij4!p}V1&<{XI-G9L zX*_tT$$arCCLV3?rD;M(q$2T$W>1j?P>qYZU0l$+CD+6ZiPjE@(v>xv9uW$o$)DY@ zu6rDirUPlE_D<*wP9uRN)0UwIXzIT4wcF0-t-=7bd{sYZZ#AHuE8vw(b!Jl`i2 zIt?viq|bax);nmrq7Q8AY$dHXxpC5OrMLW>(?*dPunn)oAJowNs-LXu`&VGmER)Wb zxQJ7-rb(Z@QF!9*0~kMGn<(%fsn!W}&HM8SAB%DB@=J`ak;YAXMV{xV{GP=*Ew=A} zPcwOy9w&^jk#|yL*PNqD!-0ISY*1)Jnnm|Y`ftWuvl=+L=UH@e_zc8FO!>jW$~7AA z;d7PW63sp~mgL!x)|VQ8_FBmAH6Kk$bM8XeS8Cs;t&R7p- z7K8XScYCB7fESyk8}Xg??7z)OUIvM8VA0C~xP5*aZuPsSCI5pjqhpY?PI%cBW1aFt zKwb*_X%pSYdSpQI*6yQBV6ZBbk>>!b~5hdlfFz8*WnL zgNV}^EEy}|dy)$an;ao7hP=hg(;LXUIjgF4ihR39Wezpwc>|;HbJ0##E%=2X9b;kI z5?G%WfF~Nou-e1o1!)evT8yYXUw8c!LHGmmC*pN_0V{ib8-EQe(JCENo|x8&_RKiQ zdalY>^$Hvgs>cZObkhA`4pNSybF8hzyn*ICdPN}l*Cc#!bTbrAGT>*1B_e4XIR2^* zW3Cz?;T=emfjquO@l-bLLzY>wOxy|HD~b|3@I&hs;Nh`dVQ9fl_VG8}SKenJCLZd6 zAN{Vw-uJI@Ze|1!H^Y=+4ubpxC+sS|1?3m}zFG_F`YBtGZVjNh*@?uvQ?G5 zJE!80Uu2>3$F;w-cvf9?>Za^4d4hgNX ze>*R^KCK$xGvE=kz1Nn920X0d_3Jw$xbj?geY2|k6^&hVwNHh&VYA_k^>|MB(~x&T z^}2*1l`q+0=gr{l7eqd8D;wQ5fIruks_d#TME4R@Yv(X~%+>W`L|LlhTH=}IyjtT~5NWep5GRNnOB24L zXdFK<)kNN)J$c5P=Lz+GdvlJ#@Dn>!1_z}z%7YS)Fz4hR&DSxf#af*xG#+&fJRR*!%Db%5x3ryCB8-m(_;>({36H~VC3st)PLBleLI@5V8i1~;WlT@2%xN_dNnZlML(oI z0K7e&DTr&>(kuP(j@MnbF)Z;l}f47awMeJ`q0r}6XamrGvhT+}}EAm^qwAY?7+-XeTp(9?Pzf0e~1>o62 zgH(?=<4s*u*+k`qFrRN?hH)23YYB>Lj`Zk)x8Isj_B7;_<=7oBZ?<=BDeii3ochsD zO!30uW-FoQpz-kPzO!uF;3g2>g;{28sd7Gfpe*7V=Gxg33@kTm`u6D$hoVBkrg9Zf z&60+6;U72rVVxR|V3f_}H1~Km>-2Xz_t#V|nWPUE;a}J$3w>FqRSxXSH%IEV38g=F z!Isn;kJhT3rLgIDHVflgF!(yggHta?qx55YT~~4PYPiA#F0wMETHJ|>Tdp2^2;{k_ zH=xT^hM{`lk0X9E>d(QWA@7I>N6|IT1*%KJs-K)`7pHne(s-1MhP*W+Er)+^?%~XL zv>)2bJj&!v`0>{fRXO`0-Ak+bki=&|JV_pSKK}kw8!2lluOdq3x09-mNY|{ToG8@m z`MJ>jo5SoG@dZ#lfJO6vLg%W!>aDRg=AmqK(+qy5g_(*AkvYpQzUq$fi0yZ_Nm#8U$V^&z01u?No>K2P<^#E`K$8uF!ZO1F_D zzowyVqg7lNJ8ZVfL8_-F9ZO4rvQ$-nZJxzo@yM|=kkswR6`Pu07_Y9bjahig?|LCq7@Bv4N(TX{Unlzj9Tjpl~UOF>+MK5jAe znUlnBV?gzJpl`iwh%5g>--q0atg<5~-v^{0ochJQ;FPVn^wk3{q~%h-c%mQ-viuZA z-g%Bjl!M5=C1Ine7556OHInD z=NKes|9&$AP=UzDR$2{Q@u8Y8?Y$(fE80Vqv~6z zj7W3J=dWu4g_VDS`iemHgVe)-c8lp8lZwe$cX@9h8aAyj+s4+AJ>1Os@lo6P zk;+Y)-xVS5&5vxynzIj!3lXvCJ$0_U86O1itxuqx`9|4uSXbVld=J_OS;}D7YP{78 zbKY{qLAd4A9l9^BWIHB?;9ab}W%&WLU2xUCdC`7+Ss$?dpdZC zL#~Ut?}tJ(bxKof2P@+S$g)6pk-9M)PYsQLiiy6wO!PpyW_OxvuuY*oJNJYFjSFqO z?KEaTm%!mc{XzeM8zk(tl73>7r1xO62h;_FQ#J9vel0rhMOTdtk_+cc|FRXZ)?_!g zN?(Twep_JiVFOIjv4fMFi)k;k=IAzigG}~ar+rEI-gX`N|z}Q(Inf{bSn%Tcl zD}w3gg9bHHICf*DJh^!W zj?RsS@HI)aZq4ys=5L`J5doFk^yRXS)c0O+RWz$T1o}K$Dx7;v#S!<~5uT!u{w{FL@BOXq+Ux$#28^;j)ww zRcqMzQ4l0`&qN!$Vn*`}F@wADcbTj3SG!5*YITSO+O_19kL!WgP800-{-eL#^RKEEk?M7Ex!X` zSAmHe2S#9NHU3XV5Cxq8j11x~EZy2A=na(nyWDo(n#O8~!V40Mh!KUV8#Ro&_i zzXNASf5O)%I-qFN6qamkg*PpBiX&|;Wj}u{;inF9*eFKVfW!+JdHNjgnDPL&&!`La zB0oXm$zI&^=Uq5nG_(pIKTZe9c3-l@B8N}dw^dC%ZvTXBOV|m-ORR297Z`ZeRMjAh z4?YiW^CrNex2F6}w|&fVrxQ|rLAx(*w5N#^pW#ys<0B^ELjB&j%X0-*Uf)LiQWHaG z25D!XZGzhp8lcuISStSaDzxU&f5-D*F$t(cXVRP<8Ld&*wQTM+uzgvZx9@Gm|L(FD z#B*}SCOa8)C=y7Y(CX-8_cL94OOIW8vU=KSH81j4>&+UYx5M~VPhaBJ_Wb!Y;_^es z<=!jn`K#MTtVxF~AWU;r7xcaub$2(*`LGoV-cASN8$sAnoGq*AwctumCf&`3uMq)g za4M8O*Lj097}~*>~p#lAh0Nz#4^N;)Y)$pw>|NW9@ZDJf_V#J{pK`ar5hz@{eH# zc6`?Y&tE(UeJk_H|9yuxLzYy%S3iFZ@vS}%Ssw_i>*(RpZou2DYa^Sj+^XqX(^M`_ zr#)s%1|#VYezQ6QQ+KVzqH488{xyzH6MOO9sAXdownyi)xvSt z%_;CJeko+1nHlXmM=87PXW?c88P0Iza4W~8*h{j#K%B&sn!<0z(UZCsDneQ%@lvfnaEjx z{h`&t@mS4Z6-3<7BMnUlYh4d|{v|AJqYGn3KGh6Zb`+<*(-*y-wdU*ghw$RuTMEBm zduTX1A9s{PhkCJkk^ixk)6Tp5e{>`-Fcth)P9-0)2k+N;gci~6T+QL3pB8*3WyRR| zcr<)p3+7JG(zc$XBk8))k+%O)9Ps0;H%xoC4(}Kl@lDr>qI$_Twy)PGjEkF{FKlkhf$V@zNG`ExdQwIc>eIjoj<` zn4#b4iRisQ5e9BCfZW#YNfTCKfCt@&)_p6Z+7uI>oW+qh95u?%nT1WH{ZY)JU!7#^ za?FY9_>!2`#1z^Lw3p+4$AaRA@ADQDr(#y#^2g%Oo#oi9=Oa}3p4I6jcK^=~o7=aL zW6g z-Rcg!wEalPblfV&{aYkcw>%{eP!8d}=W&`Rbnfw1P;Cmr1Cq8wmJ6L(y)YM-dVdz` zy`Q|a(|o-c4km*)uscSZXB8+Kt8$XY}$;hkXdu!USvtCX4JkgK% zr40|SZGvki2uX7RHr_EnK2`=_>LM3LI?FkxHRR&M>v8izJ4qPhK4qbd){`F3h`(y% z$es)hOCI3>j}-TpUf*%a#)DAbB1nwp$7o%18P!OYcOLe2Bi4N0)qSm(0|XtJ&OV;= z;-hP%ip;WyKyy~*{gjt&SjSV6M_@ZoX~4eEW_-BV6hHjFgAdbf<;`c$g{6b7e4DqH zsp|^MIVFb8v`2$>rL?~KIB8-M3mNebE?7;F`)s=a`7N!|ccaGR;MzDRS#N8!R$WJJ zPwH##Zz^ccl@I1+z1u51@lPus|9(t2tm5na>OM}*XAotWBQPw*hX-?7h z%~LF}wUbLNE!g#@2gEA;j%IyzVUcf&crX@#{HMIHzZS?hBJqz@97CGOqB~h|5t{`V z_&`g14&*_Z@^@@X2(G)g30b>=*t>rgo^2GVQT5?!>(6wDk6?;#Ou~2If6HsB8s)Qo zwU&fyc5PfH(Eov*d0oC|`R{D1aV)=O#Rz{`^3a}>zJXQoHXI-E9712b7kiekLc+Ir z;6VEgJ@FI&buWYzofvkH_KSLEZjXH?8`1B-;;kfa!rx0VW^*L%XVFyodMKT`iPaxA zLD-u-(CWvutK!<@cI&}z&`xn|yDiMv6a!Sp&|=a0?Ax*}`G5sbsb`BmWi`3-+G;Hb zmq?FSz6&QLxRS@K!*z$~Fr&A(Sj6rd+9=ayswO4X7iFbZR1pph${zF? zVtewV3sBd}3;fG|f}is@e0X#LwrhVLH|mW=@_Y20?)>!b3_&``ez9X<2`An32Y4jpaP;Zn%|n7pF!YAm1Ot2p3g3h@SjB)}Hj{@nN%AG3bA^BrzV-tlYmp{YF8h-_OVXnOI=-fs~yvS0ARYP^X5q(&FszV_|4}IlQgVD-7P}pH0M)4fjqEV(V|B1 zZ#nRdH4K1Wo6d9Zi;2y{#FW*8fxHAG-BbP+NT>K%`Wz^5J6C!Iq?1726;oJC47C4) zThc0#<~W=D8v8Y6HxyntLHQ|EE9>^c+=`~sG^rbqzhZ@dbAUWH;-}pZC-amZ!?2n0 zRT;DV)*d)e+?%j94Vz$sNRHe_H8BZGVjF^f%n?SOSgkQr`6AW(HhArMgS8DAd8sGl z;n(BfKT*nOLRnxo8{u_caViqd8P%@XxYSnA8W1N0Lt*$K(lukb-T5An-{M-c0YG^O z6pmVM?#QYB_=g`lvUqbYPTln@o3b0rn$wv$DIIn!#ayWSRpfFU~e?>!WZY3Vc@sXtR;{5?d^NB96@b#%BFG zq&x{^=j_Cq$tR)FdCCqwl9gx2GkP+cd?Gyd?ufrDbOmu34|>xG9QOBQl=-u}cG(Yi z?&!K39xGJ3$@@%urP+B(3zX#)_L<_7z){Cwuxn$=fqMn3NGdtu^e}XC(8w+ywFl*mKw!Xs{}*if7;16aCKm zNL*!y_Mvl>wm@vWJ!PVuptAq-nfC&uW+CnI86N2PZ1o<&~oi{Y!Avk;OPEG8r*&^sj5mE+) zCv)o{@hgTVZh+-s_DZ+38$e6aw&a>Tz5zy8+Xov`U3Bfyql=C)`#yhumPH1 zBu&y*(@R0BbEt8pn9VT_hQEGOQ1!UTzg3Ncu&lB=on6)gPhO&ZqOKUIjH+@7D8EcK z#Gkxb2+!7c=1X*Psxa9ndN<4qFp||59!LL=$x35Y7KNF%9y?qTyIxnBk^a0Wh z^3!kN@3^gERI|s7JVN%9lqjWXRlPQqB`ND^h=0i=Epnx-EOzgDgj?oUWfZZiU@H>W zi=~!31l2Qo_}xIAkp}Jqbfc^Mi^`F^#}y*=@nC>Q3+l&NY02-o*SVPi#9xduB>oQQ z$u*`9lGa16c;o}0-*K3ba0*D524ys z*4$3{p{txb-Hm-&n95W(c;xX!aWRUiN5lZ#M*Ox+qfECGThG74$YTNd9SwPks^|Uj z)Q~m|65S8e*{7ZtF=J0NB%jX~l+Od=UsQhU#E~nEyb;ixGL<=L9t7ztTOJlK(zA7e zvLf8`pQjTOe<0re^{^yv;z-udtuL;kGcS6se+iRd$J5~^MzWoeb= zVL-Q4f_x(Vy+qWz)&i(kgQV>m)jOC;->)6Ki1uBZcScZONcH64yYo(5P+rEUH{p(1 zoAFS`I6S=mZWSJ`#Pq-?FJo|b6zw-RyBiC%Uo9q0`N61OvrF6G7v$?er^zr-x|@1A zQcHQ6JVL0Tyj6vpS%3ct${wmF;M&bzRXC+=2w&Xos&WAJ2+{A$E>zf5eLc;9w#;kn zTG8lFC+cZYKOi(uJWSsY6ODUnd+5;ju9~=1={)%}PTHV(|DqM?nidrEGq#UF@@^vDko~z1GD$By;Q}ms_%*)i{N`)6+&p}kbc9h>v z195@49I^}CoP*%hi1A3d6$tN0{e>#crQQc+qzp*Wt%QqD0?Gf}K{X#A=FfzEwmC?B zSV{dFF{Q^PMp;|+kb(FViSu#Ryski*jsCwjZ~Np0?&{$v6a0)-j~S_F$LTrn^~yM{ z%8Z2@{e|-P#7US(djprP?F=bHc5-(EM@`BD8$LSjG_J7vA%fHFdBfth&>+N!rySb_ zJ~PV2`ze0trq^ChSXzg7?w<>(33cT`CmkMoaxEV084Tn4*z=CGM`Y@j2v(Y=frKu3 zFm5!R(=@BSywLcYh>iMBy3cAZLp|#9IuR1_rjPi zGJE8_r;ymbJJ4ga<9}O9qs_0ud}{*T_nm>C>eQvP-U`Ks5#FeYwC9P>yGy@m)gZ=p z89V>_he)z>#S(cDMxDK#-7)1cz(0Li(m6XzqIa5cd=l z;s&DG%2qPzRW_F0J%OuU7eoG+I`Xo`LtK5tSC*Z3k)Qj~=UoB%@CoZ;qF!$_&ue@# zWb#%XWbeqYd)RXCwhO`c9`#WT(|GQU9MPn|{tS(J(DzDhmtkr*+Qa$hMAY1f!Amd>#*|C^oaWU$DPw z9n6Ndmo%U1_uzcxlPE7s0_XTrM)L=wBQ9dDT@Jczo5a0OdkYx_=nrwwdYvZLrGkAta2-B8d4wayTTF`7z~!O$v;}kY(4Tgqy>^#& z?C4nsW9!Zl6%Cv4gEQA+WiXw4KxbPkOc&G%W$cy(pU}cky01w^qiZi%YC{tkb?A}8 zBOg1mErxuwK;jv8)hiRIwnXP6{V;FE9?jTZhfv|YVf|p1nAi)QyOw~jdjT6=atYjO zxbX%do3UZDP`pN7zTv=XJV>?`!FTr~;hVl^>CAzJq5ttKxH4-gk6%0neHUMcHcaSW?fB=neImG6fA>{n)2kGPEyUu=iLEvY+eRd(!H|5zvD#l-uW`I`-SZM?0Q)7 zX`eQAW2C~k7{As6gX>#^v+jI%AM%>u7moQa4Wz<;ZeUl!6_@HUK=&qu{A-BD--gNj zFA=ig-B3oa4}@p#b8P`03J74MLhpgX6RkZ2lHTQCHiB}Q_ z5LYfm=cxF+F}W8XG@f`!D!htAK(x0dP=149AU5RGb9f!RC|yNE(1;8BS38+g)DRRe^*l z={I>jR&23WYXN29X>8QH`;gjjD-x&j+^;=A)ko3+pj^L^-d_(I7#Yg?|Mh}T$&t); z&>1+mdOeUPVcq77WuUW@ByN%AtsS`Q6nAcJ5UtU?rMrTc4~M|KETpw!xd$#lN?8<=wh?a?VbQzq?$_?NmARvl zktVX_1!?T6QAch(YJ|L8eg{U1sr>!rm#RjvXx?8zSOS`3?hpAmzPN#Oo1Vd4TLQY4 zcnIfKix3tW%KYRFgsEg^KGIfk6w6y-g$?i4MB+MLxx)aH?T=t(Y6LTF*-_9>toP`n z(oA;XH{Cn$aJOm>iQna_({EV46GiCPx(1KG_e3P`%)|l=g@86aDl?7DdumCI~;W@^Ez^3`%F(Yn!fv>bn|0(5#t^8~;iapZyG^e|Ydq zFa})hCX4E>X2fAUwWblg&zvEgH2-kT`+IoJuN9}-mE~p4@Li!lnl5wU2`jI%ur0w@ z|Mn%M$MO#~o`b?s!44gMc@yms+GCaW(!`0l^7Cgfz3fan*O-y+1I-s3XLmrOJP2_X zmdy3Us|z~G1HW5Bl7R>Kd4#gg{riHisTUBwalCyaU}qxHd-+P#7}{2OnGdJ#F^jo7 zFed2_q{LeAy6G!u&06C5*UK4=PbmGM--GfTZkvu%z0nz_+pEi@Svi8{87e;6@Y0YR z2)dQ5%}a;^)96#EIk;E&&{>>*D}rDGt4G(G1lK6X)*pW!R&-sBEUi8d%yWaVXF+IE zSOD~UB(8^&AKm53Z3}T#&~zTWr!kEGy&F=8*Ar&tY3hBE6&o@0QwPW^>q_%AO`!67 zZF%X!B(}0P^{t67+-+I_&}TQyeg{n0Z-gbU`pOZHR`Kf__f_d)z>^+KbI@AS`>>z| z&1CtdR#YSOUGQ5ck#q^DZsjV^S*UpxN8A@r`2Gf@l|a|YD^u%8zsJEE@|t*U?Ge`L z=S}fRe}|UzM@w_1#?NyNC27PM26D-9CL3%$QQk!pV| zUVm&YnL4+Io<}%?x)&=^X(g?fHmR3@x|2RtaYx{dbc_p~27YuO0bNg3jh7AVpiNEK z2tKw&}#^1t%IyFh39+RHzg)7gmLTExwNtj`)jUOYumgYjxcXw5% z}hS@=x79vZcMMQ1M6gplf6P}O{JaSY3i>@Jj!kS{^< zts3H7G`793)|+1sZzf1TAlJE@2pB=kCnHpe7Ae|!$ABj=zWD*jSfIQ zv+DJ!4xmA&Ef6|nvsOKC^ur6_^I|5BUtr6}BzEGwdNrinQ1y7ig-EHs3yR(?g9B*? zHF@poYt}>$f(Z{cfZ5el7;BROg`HJu+d zZw;diC{tUTx{|lns9Zy9CFzT%%-8gqSo7WoP2ODrpQIA^geTXO4&kcC)0pW*Esn9w zM&d;|F0ch3+uj){N5~Sxi=Y|3kJhchTt`~_IPpKTd~4wIZSA=0aXps znj?)9<0I|>X$Sb%p!cCVEPZ=MROW=hb@O;eK40mxs(+-MA^p4bawi@_(gQ{{M$dZ% zSJw2!2IC@t=1-0{vf-cVY{Y`^L6irk2xD71?}t3J#aC-yQ8Na~Z)p|(PB5=6eKT{> z^&x#H?ClnJAKU4a6JwZ5hnE;^l&JVdT-db$MwfW<$~XF)-jmby0*#x#*V-6eCwAv5 zPm+IxKpQunWOoY6_NQY}=u&N1yXuf)z6ah%Oo0;TT!@YOB-n40s`@A0V5D5mZ{7lXsw}m{6gZcIJ`bN4$R_?XfJQN&59hBXpQ|xVZ7;R&)U! zAHhLueQ8wjMyNHT^@8BFjZ~(>+#6jawtIy3cgpz{F$5K>Y3x(4=W#P61VnX z#4!q&jC7lkXJX`=Rjq-G_gi<$&YbcSjsJ}%(Xub2Y$oXa8F8gt6I~!mJ9T9j-@n7_ z=JwSn>OuUcWz{*YiBp_uhNX`F!5%d+)hu*&_*6Z;oqyhH#iuFPYL! z=~a2k2@_5{fu9;!5e`^!C)x*=u1`mrRD51J6x0l=ecUt1LZi z;Bw>YDzCtzz8XDaWKBI852T^N_j?Ag!x6aF=BAkBWzI=&3HTSJd<9z@G?EbyfYN6p z!bg$PjAYY@fN)R6t_a!s4PR00k@M|6ESTxR@$q3i@ zy5;o!aGf6fqj?4^ywq1({aX*LzZsU5>}9lu;=>y!PMCl+CU_nAACQ+<{*{xC(vjzr zrM)iD{*gU3#gCMIXymmBLsRh5h&sG(v&~2vj;ViBwj=VB{|KrFUbf5*oITphF1a3@ zjw@>G<&a*Es~8W>G2vDKSk`DPRbPJT90bm*NYmIRp}$uwfO!ZbPKQhFBXp|w>;4Ib z@vAgYm}61aPDdGrj`S#V3X$T|MY`@Q;jDDr88Ax!RpBuRKPtxI`@IuTzv@=^euo97 z?*$!?1zOcW(pGF@UwY2!R+O0V*nxO)ph%8S0FzJCfO=7omP6u89CK)#B>f~v#|fi% z{|TA{rHPPo05InBfv|{?PEdYKl0E}nbPOooA{~Si4>p$tH$EU~97b7^7~RQ>u;Zq% zOlrx^2XyCU=Mx2S1C&{|;p9!gX;(L>UFygaT|A}5Oj~J{?+?mn()n>>bW>K4_7+H| z!TULb@ng#_q)Cef=|SOp)gH)CkPmR?#F03<*b}Mu@k*b{g0zxUJV`taNtaylT%EB< zJuH%^#-QqL;!Gsp0bPC#1LeQSKPi6>q#re-;O|hjau1RQ1v&=h7PYj%Fhi;I1L;>F z&jHND0t+7#&y{-#!Xi-hqkQX?&g%tzJ}3EXeNDOQE%de?{;ck@%;CE@avK+%obc0loJ4DU{H9e5I%pa4QKC-myT&g zjC5y3ZZzr0RxR0fXNA^DShqsu3CbgK(gE3K3kJc5i9%^^@cLL)QpTY41ggw%d>PlV zE<30ncIoOw#u6`i(Eh7A`=N3yip&-pNAw*eN9Z&lY8sZKbLU zl~Yo_Cy1vc^)3*X3c@Eyn%S55FY#L-tdfL8R_ zmSHPNhd|Mvj@W7STTwdBP1N~mjxJm6IOQLLaxXM9qWccKB7kx)ZB)S=W_G6-Euse?Ggtz()aHt!hzNvKbRm>Sz$t><&+N%Qn+7% z@suSf55h-&^5%w1${Fcl!j+Ze4>#f%qX$6vt&Pvh$|3z+;dNJy^r`SD=FQKdrTrc# zjXo_tkBWk#iJjnLNt`h0IUe~`U+%Q)Bz%a?uJ~K$YU$u*RSBFQWWZ{_*Fc#PT>4BI z(1ZJ8)V_J-0hS0aD_=%>g>@YALs0IGgk7+^6YUGyjpMnt!&NP_q5vl~7eLoY>$?di zMkgvBr5rpdr}SqY`K)sENLYsf@zH{?7wMW9`4#vyJ6dOH-%Ll@l^_mOc?^0kwW!F% zh%b@y7I`3lufjR#OQ2Kvj$9zNi`PV;|*LXCqG`1Trd;f{4;}@@q z#wunV`O)##7}{u@nDyZ%&P!X*tJ(C1wdY2Np~He%;=u3lWb1a^Tdh8SGsjMD4sfDp zw+3K~Ef>J%!+i1RUJ{5n&zz*=*C4)=AuqNYf?fLm5wEAVkOwzD#-4xt;PaoOnEhZn z%uAz;Ft!qJJ9`o~@~jWKJS$$cuo*TosftgY9KwUa=6qJ!M#u^GhqdQ7@Fr*6u>R0c z!f-3TcY>pwwyF_YU$`Jj8m2;@I`d>%bam-CqKiCp(_Q|wDZ%xv4r0T@YbvgvO#ZtX%ov=@QH&@i@%?r>h9+T z0bLv390x=Qzy(Dr{JBmLL5_Kz|}S6c`uchW>)5% zy6uK5jq)($Y%aTB`wJf4`Ar;H++Y4`_e{{aMESdMtgfyAO|u5j-sI(&vVIJBJL{oO zw~fNLeMiig-CjPrvsio@&<7TtekSG&_=)5DQjW7X4jW4e!;R^?<)h=kxmGKF_ydX| zLq@@vPA%c})FioUt3BqoGSfwTjukb+_TZMMW5wc?)uEtEf}Cb&z|ZgXqcL`83$zmW z;-RNKD0-fAzF%j#EdHSQ(Zoa6a4E8gUtgNCEkzLOy9CpMLpy?P@ZPYYXfg#YgR7PnJAwm)x@ z<`;T{Ein5jRt%esp+2oy$=vRo=92aD{0-KN%URmDo^qG2Hq!ba)i+#g=Z7D+tmm}O zII3k``G$G~j$KKA| zy@?H&7u>^JpDW1&=V!r>~E}i~V-OxN;a9KJf*9+GQ;N{d^;W(wD){b!}Ko z!(Oae>yM&dY<(@=qaIAN3*y7Zn8KE)_rxQA13~Ku%i?}eUxuRZZ42HyEk?9(S_31! z+w$hYXKnv6ez|sCX{^(u{!k6Vr z0BQSYHTZB3N$V#Lc5oA;+swsrZ?|LpA5!f3XU;awjAMG0KIxKI3`57mRp3rqUmeYd zHs;U zDSjc`-gXRO+YWrZDI8)N7GT44dk$Ck1Fb8&<-SYgU(Uk-`mQ*)Mo;a*l0w`!$4$H8 z@(`zYc)*Asca5Sq(5}2 zwmb|}{Uz>qgQm@1qfH-9E@dK>I>g>%^%Jpw0MUV$|x2 zYaxAb6wkeV2sBq>QBxP;ercloI`*dS<+yk7Vq_hzc+_x@Q+Cn`eV$poN#O&wZytc_ ztoPD0Ld&?*IKkk-)QtZkgzZl#*_phVTYg^z=tcZ0QG7G22VQC2nh%H{3)l{ zt=$^rMBah#C;#F3#34x7n?t>&J5uEu>-Ivx*YO)!L6;`1eRF+qcwL1LjF<@TZ@U2c!#Bjlzl%I|W^jpy!5h?2`xd&h0Nq`bLIcD!?bVz;8Fa7qXJKCq+K${rG32L)a)du))(BhQctHpI z$KWx4AA416r1-aIAo0jgv6l9?qxs>Hf4{?~lTLW!O<(!ftqqS4@{z>bVt!`zidrMA zLi71uC1IZYKlbw7eM!@P+@!97b&vfm+*_;?@2ua!sZ+-l*3!8Bab)8y;M%pO+~V>d z?GFoh`ka8&S6v_?hH&|2B{^c+dlt94K72`S$ge-`j?}L>!)dDcHpN)g3K(o?z)9!P zUh{Ruar*8=aoSZMQypoV3SIeHxB_Vwa6B{=<5tJQ_24D==lKYs^uZ;Q<1l-dtGYI< zlwqXiT(tJMAQK1Mg2}|S&^bPz_LTgFW|j-lFLZ=#Mf-iKbM!W|;v;f(5Z7inOS$b0 zji)!2Hqpa$hj+D-q&J!3f6{zFT8xSKtuD*lN^r&RfjqF^KuK68=Xy8*{S2(SUj(I# z9v065!Z++ybE|HC=0;X`TWwC)D_2(bfUFHg?2-w+Q~vG&(iD${`TSfwHl-Epop1uD zHoHP~a05s0vhIK(8@C*qTlAC=`$VTz3%lPdC8(xdy1T z;}4%jx;3*pLV4s#P&_T-9x>u)_H;))7CiJr@%1n|Ri3XfiT7`JTo49}ps_|so`8|& zgaMtI%b+(a=~yelt;!~`IUoxj2TdT}=!L%m)?t`UB<3x%k!4XgVbKyjBpri|jINWu ziN>3*FVO0(y+#^JyqeHRPHi?AZq z*8^!#7}?kqmiTT4J3|RCMm%SP-E7_E`x@~$&^)r@8=WfjC}As{GA+Wne@J_lB(lM~ z-$TO3VWNH0Yf4ul@jQ^ehZZjBN)yA~sxNe}#VIW7<%%C3Tk_d`S}5Go;jp1x;mz3> z!8j{AhqUW9ApgVm{G<0roBRXcz)N_(-*J{U$ea&N*$0bPZiHpMHzVO=PRF1$>X|Q0 z^&0sc-g9^Y4jFPA8&6MEUJJUtV1g~K0;Gd9!Y@IX2{e9AoFHcJIwk(boz2lpt;V`_ zDF*UYgu8RJZlUvV&*A=@dO`fO8L#F^kWUn(DY0_RgTk_H7U|Oy;_dlCfO}q&<|~Kk zeo5ketHnCEOz=Qs$(e(L&p)Btj$Dy zZ0=x-#O<7PETj4XlSv(5%X33XJqpck=))=6QxS(1Gs1X`O7jA$U(U|WlZ%V@R{a0_ zYa7MY7Vb!WMxHNFH~4b~>-eZ2y@R}mB){T9zD%fj?AB!{C*8}pXVvF~*9y;A|NC>1 zYLGis{($-?-vIvZ%m=$)1=Ujt-D6Spn>^HnlO|NXEZ#P#!;_|RzTuaX@_j7c`H6VE z$%J>flaH~@FJkTHf62=r5RNhHrRh2wdS|85fTR<3#1}}KU(z}Yy-JIKIGC`YmhAG~ zjDPxe5PA-8g2dm_ZR~!H{Df#yCzKIq;ZFain0aM1+8&62t?z0ptu3gIvFER=*vF~~ z_Om$*isPuq1#yMa6{tKm^@MgW%NM3y%OMWVz*DCE1dS0%lUL{(r3oxcno=)VNm_g5 z-=SUo{<@BiuXO2uiddzwCn-y4%nQFnS73#zVZst54F&c31y0;c==>wwx;fE zmOlKIMjA>qy*5k?j;l+4aT{!(c#-an%LdZ_%%*I&7JJbiYCO+n^chK601{p>^6APu zOTrwkZLi7A3&z6n!l6j|Q6u~k53e`ExqmI?xUGG$ve#MVnZW7T9msafSYr*VWba;^f=Ffq%f@><&o2QQRK%1>UcB zWrVvTv$$rq@}Gp4IZC6F=aQ%E4n^ugcGYJKU*c;F%S(x9uLzmA9Oiag+EY2yY+gPTDaTTE2RvBmD}6pI&I?rAEpVi#;RlA;{pY znpffsA^Ouk3bzL>!47wN!pp-BoOCpNKYf&ycCd39k*mR9w}VKW0JjgE7QH-Ifyza$ zH0lA@JJ$fJ8#Pa$>XJAd*)~8M>eGl`TmC$>;SJm1KyZbif z>N@111V$bdp?hc%#{{WbwHYw)&IejuE0(oYeJ7N=f%SODDZxnt+YJp&#HXioHNRA zDZfd$)LhqXa}!~`cnwoI5ostfFT(?E|F+lOp1+I<-Da{S>q>N81E-5%&!-qi_er>R z-Nf9lnDT%qBfNHBV}^UOn9}W(1ra74$1$5u>sIP-1mYxh9P#X94p0V{L+h`yCguB~ z%FIimm)Buhzb~Yh`m;UdlgK|bK%Du@wXFzGvhbsrOHx?$+qGbo!>u3)koF}dM-G#6rhrBwRKF9dD z*He+OtAbmv1?*yqcg`nXMaoiS+taJDL9#7UE(8}n-QkzLrOdFJ!4#G#?cF@s4XDO+ zq|XJdH`?I=_Qhj8ViINefq_Z~0Ii3Z>9SpME9s>+7(F8pc7F;07t=qWJM|LDW8JP?xv9UPj-bnc8zK*o3x1cpe z@(qGAQ2cePH>=Yr7w4u95=wiLPf)oOG@kmAxkl|*9!F5#hjb39ybIxl${3Nd1NeG$ z5iDNr;iCKn={ZQ6DpkfspIv~arVW6w7e@PQjJSeD8)p#*Ws*)b zpiGk1ESLOS1gfmWYR7Aw^WSDh=a42x3WUl73BR{i^7NhxuXQGqj;J0}995x7NV|v$i3W1_Cw-(|*C@AOl;KF_vz3>8 z-@1ju7aiqK;`v|`uCUNK$CG`0)sU0-!sA8$_%)-s%G+>(*$U}pq*1-N5nm;OvEF@L zppD0+k2-SFEOfoQApHJtPS~w7Kxc>WE;6PdS?F6ibIM(iG=%Ci)En;1XKY*nzc+ML zIWu8>PDS29`Iz#hS20SK@+#8f;Q@uc5 zNO^Wn-beVqXjbvF)uzvcNev50HOL;Wt|aN$${))1%jYw~9-QRq3-b+=fOG}?cH9Xn zZ&UTRV}2;899s1#^`U67rN1EDs^G&Wbqta61mfca`fe*5$XB4^SK@IDc)JWg4LFBN z&*>c;fiC808f8=!-X~;1GdWZ*Pms=0d9LDAnRUlZ65m6uxDJx|kGFXISk&tA6v+!? z+tYv8bcgT?4lM3p30@vchOEC1Jk0MXt~8E;pCjzisU$!g>~a@QW-JrIJ8N@;XDN`f z&>8>LdBG+QS|(3s{0EO0wuh%vxV&6D3_dm4FE+Y5alL}NyyxbJtXb#<@LWC3C1+7N zjQO@1YIG~td7G^g9{ZbP^uhC(n%tIuOdW=c^Q@$YtS`Hlw&bq1fq27XBZlmcLEoqa zn0%-%58k;AH;zh!MqZSw!EC-D+Z@NO8-l0$Wx&ji8MtG?ThZcmD_(7vn_P9>h1cv> zP3p=k33kX1N@vXD@$d85QDYOFXdMBqr&WUAF&1cgwgj_gtcQUozQf+M$}*+#WcXKS z8XnxWA1>^6!DaJqqvvJDylp>=wY{kqO1EOmI@@{r`PS0gb}=5lRiAe=GiLbwDvtYS zfq#dd5hgo6;EU4g=v4Eq8n4`TznQ##vj_T^>B|PUtKn^*0NmH~64E?q-#%Ofzk4J2 z#@lb8L12f9c^`VJD*xI#l_z|67qR_o$$^Vv+3zE@c+G&x+_7X8(slA(9b2Pb+HNcl zrFXMu3R!jGEgU;^vOL+amqx$GZuDGRSiPwt-Teiuq&;pPynKc>ZUJ)eDr+9wKLo7b zne(rw9^&FL7tm^ICwA7`QF@#GM*SYIbAET|BhumotTe92)z3cS*-v2FZZHHTe{Vr_g zJpv7W?Uqk}ePmS^j+C|Z4?w}UPdKWw9-Is`XMJlq!z|B4Ty!XrR~_RC->RMgi_~AD zxN}Y3Z%90RGO&j=RtYTO(J-)C{05z!?+CjEe__WRCkSvn%JKI(@o-HOxW6O@IuXj4j>A41QVThp2Jr!AC-uSx2GkvnG6@#Y^mCZ!Jf@GvLP` zWWtK`ow$ssB5kuS(|S%3&qog6p(A?AboXhn;N^d`r>Z?Gp8S!`tveLv=-m^B?uOF) zmm$n6sDhmZJ@cA6ov(24g&o}k<;k5jUjLfV{4$XW1k1-RUJX~Oa}3>rMj?s z&oSu>Y07$)U`g2q?96&{iy6Cweq=8CoNR}KE|qG%uJ?quR<)3KDP{B04CO21KcdQ- zbHMWUXeHU-vgz?q+_T<-C*NI-8I^Xy(#%#!eFk3LO(gvdym`A*(=QK!PDVGWw|()$ z+UFv1m<_+4djZ!@&}Su0d*T=KE)}15nciCxo)MO`=ev$RN9r#g{xD3|`SeJaKf*;; zz1v)>ak$|CxSrczr+V|RwiS*{8jrq~$F)xe>Cm*EFN|Av70(*J!7-tyacX}{UhCKp zFt3$|Cz{2nYZGdX9xqJ84Xd_GyA9r`dT!xMJx+5eU$tBU$F9?TA+a_2pL3P@$Ncly zc0qL<*u)Lzu5T=#t~|~<`RqaSr6%&?w0pwGEC4O^$q1?5ePmD{Sod>I%4?bb>L4)o|Nl58m}+vTo#@GonGmDdNw>Ongv2 zR*tZF1Ycd7^S&WVuxX#^@cT%JjPAagJ$be~=jk(FPFRa$z8PoZDntA=$A(iah>|Ps zf#!l2Jgg!Y4DLm9{{?%Tmq55D|MS{SwK7$z@zY$f-yP23q8=ll)Wv|S_0Hox=-m(} zF{V+xR5cLXk0alu1F9Lgo%9A2zWOiPf|jXOxKFoP@XB~KT-!1MXSc{heG7lw?XwwZ zURe9d+u_aZJIwV~d%3ru47*SkH@#~M2)XeWM;&RcP0RI!z9C;U#T~?*^7Hevw0X=w zh-kNvkJ+1z#LZIm1Jwh~;V9|pR0ExsWO3aZjt<`^AkBm5WaOdN3!Xl!#7mrY{57Vj z{>EAj8cQfM<&M$xjB|~(=xSS)PwgMYcGW+|s7CQjOk?(7dNSO2p9A_mW^%g?->~Mq zIb!^oE~1b9ZW!^TG6Xfe!nUqEOV?9|{nv&-6^oS^bNmWZb635Ko+SPg`x0sIonc3W z>IGVB892~ZH!RQw_HBxlN0tpoYrkIXHN3^Dd2OYtzruA0_j=j!jjmacQuYPnt1uw$ z7XgkP<<>eY@P6W7AbkPDf0R*;50F3~Gdn7brTSdTn`=2h zxB$~V8sOH~+r+o3Zk&FH>O52fLUA48nCQ0kysn4)3Y;yL^1UT9fOHIa@LNE5h@^9J z#K6X?UvZ4?9nOr^gP3aj6;43>+#r}ddIMa{IwI(NICOrxAa2!YtkCc8S(ZD$0iV+7 z0He8+Cvq<7E?9fZ18Y*y;bE4D=3<%l?#5YR6iJp&-O?lL%Z{AK+jZEW3zRt&9~P1iH-NSgLW6j$}ze*k~ofgPWu3) zOJLg>JA8h!o;ZARFP`cI==tIpoDR3bYSkm)9=+$2)(@ypsh6M8=kGBaH+}Bc_7G4n z;obE$rNg7?q$Q?ezrVc+yDo``7Q^H$pB=Q$p0de}I9=Zf#=LB7pp1Pgh1R5lByA-o z4*Cx^pWBXZEn}f1u{z9oGnqfzYA3ImHdFP4Mmwb>OSKgVFi;o~%xwzy|jBR=qi z54H+^EFa|ssNUejn|SqjFFEPpb$sx03L3X;fwQl@!}YIv@%Y$zK!1z39u-m#493xh zYXs?I`Stcq)}w)ijaJQaxb{?%G%!3WBT?r%afxF9oM%RvQ+UyaESD^8C z822v{6@Ql(|APsi_liP#KB~WEu1;Z3-?CXa+%iZFl$>)r_uNrVKB3Al_m1z2D}yaSrBGny@*8kHs4IB@g?^tLc&{7 z^OiGV-akUJ`g zWm~qT}m(L&*;)@SKzx;JbFe4+PXkXC>fr5llS8WgXX&Pq0!0^v6!UIRKF zCU`~*>LLExIFR<{&I8N*Mbsa2A)THjEPk4SLG$-14%W^eZUgpP^!b99%fzV+J;>W1 zE;=4vPRE!Dox)<&Ix?GfVL-SEkH%!d51a1Ts>VKu+})G!DjcVJof9YF5ZBYXl(Hpo z=}#1`RdY!ilx0{&5kD+(X(C%d;lK6l?>~AE5_yQU05=|SxHG&>=Y&#` zdRO?VJFKN#nt4EwrWVSFjW#St!X8jMUvc|!dqcLl&oI7YgO~C(VtHJFcDDIrw60uJ z;gTee#p~ZLXYN-=%e`rSs5J76xTE0zsXrs_&aU=fjti#)P@RfP+X`Vum0qfkVYTlR zN&3~rX7MbwK7#5CNE-q9GxQiSP^UbF>#a1}o9s7`7Z9X_B>4+Qo(RY@k~Xc%%_dku z^g(wKWo^LC9t@{`*XMmFY(~}hnn4bdE@Dat_~k!&DbvPPpHeyVBlX#L-B)SIw#NOVaf0c%>_( z!Yrz12={hnjek_(AO0rmh|dJ+CnSxaG`YOjzO(Ye>T^QzV`L38B#woJXPS!sInX244i-%S}kv$;Rw;W?_CE z@oKf5RL>@?Msg7--l92EdKyTx$}xr;bi^T&_>qOq(WrLL;_|p&6&fU=uQh-5Ul^Wx zGOa?-9C}uZliq{4$$ogt=eO!fB<&7auLL{wM^7bE#^ zq`DM?&99RW@mE}}Q(Ce-bg#-52!EplX=b>1xfSu+G?+Ts5J=z4Q$-)p@ccQ_t2<%G zh9|HwC4edZ4VvGQxT?7%u3=6!XM@kkNl1Q&k67MCXWunV8$0u^P<={k?m`|#hTK>S zbra0t`MjP;V?g5m3SVLLUnX>B`7~Fpq#94J;X&v$-W9w~_mh-ifUevZY_kSIYI3UX zX2%3}+&)yJtVA{{iXyN4Lr@I^T?^1>NT+zPVLKb?l<%#P+z*sDq<>VMX{2>vgZX)t z7YJGpG2AN`TOKY1(yD^I8Xmizj+1I90{I%&YWFCrTN5M>7mb#8Q`p9=V}L0?kY{rt z=Qi!>e(k^reqztaimXOqn)2BN=R(<`Dd*VciRZQ7eYyf^SN8Yycg#pX2gGHRb8HsG zk4p21N!!K)&A;y513O%h*nqMhDc-yMW4<$Lvb(dkA?YYiK81aC-3uxkQFTsO%WY_% z(M`+kkmei6FIIT(Sw2=kcV6PftPbFB(36p-(P&+CKf*WTp^K47wJIMS3d8a3s&LZE zkowDy`hFE&``HGIYW&60A>W-SA7a$IsPwxTy$jRqYZ1I2{sgM#r3%6jB+aBW3-NFu z+Kt`Ass3c9 zR-HK2m)v}=jwC+i#APCGa%JUpC@-tWm9JCt;JvjQKC^uY${*6)gVPcxPFWQ1oMeiN z7v^D~-qGm0))&{L7c?g{JE;-aQcv->{cDaqd>;+p{a7N7>h5MB8@ZgM1 zNH~WzPM&7Z=jB&qd(MgbFetDgH-LX5tl0Jz%gyR>T|7!`%lI*kBXtAoMF}QZ{$;K z1!+M^c+M!h6Su`^eB$s!$$TWi#mx^IepO zR&{x`(KbzG9;6-Vxv714a%V6j?SYMd_|tyM3z2du46Qt`LLZY>6$c`%x&EO{rgFXp zRo^r6r11B(57M~>VJK2AB*;&bKlq7lkJG+Ew^{({OC;?lsfSgz>ioV|Ecxr9k~BRd zFF|=oHAX$ju6#^V{f#YNAE1n9swlmxL(8a8QNzrN)@`Knq;g-=fj}A(6$alHlWb{LvB~U86kXA|sEDCvpa2bnUu)?bj3d zG4&edYrC`siE&J28k9G(7R9vpT%Iwitf0@*KY}zERJlU;Jk>p?(cF^HvLilQ0F=iu z(o+~kd!l82ZOby!`@omq68AZ~QC`&oFQ0bQr8Is@@3k93d!L+F`j$i47Wl7Uibi=a zlsIcb`9AVwobpA&A462@N7)!7Jt7tN?>agaPGzizC%bl1KDgPo z$TLB_zYoTlrs|ZI8GkreP!5aa&5`C>BWwh!JK6JbXF)h5cDKkDt#;oA>VuquK83n8 z{b2E!-d{28z-yq4I%iDtrGjupm+fohF%ta+d^PrdO)I3#5=BaW!Z!g@H+{V0?J zr)*3|>DmsIcUF8ydO%cU9#f?5`K>LK3;bbL0}+UOA?k8Rpqzx6^-hGdp@;EmquNN> zCz3{DAKTU?eEls{y%8rc}eqjPe~O(gLW?<_V_UY`aB2Mj?Pvmm?>7Hgwm zieYQnc)t`WR`YqNUro5SeKo!72-ucWwPeF<2W8?g3dATF!ZfnC=tXFkj9<&vwFWHcDVFm%m~(27EFs88$2o z;EeOXqjy!BXraG-(?PH_wO{DJ^T=yJMe?G$nM6XO#~*+m_zsEv_2L9KLd_H&qUyeZqFu<)xL>&!W9r`BqHNsUvGY55+zoFJO;u z@p6@~KKk@4!6WAv%CDc>bNhrku$Rr{2d;0UF(ktrc}VOE;b7%S@9E5Qmv)_dfc@fp zeE4d()-$D%>|V1G;f(_tn>h3Je;m-DunWIcW34=Lz6@G5k8~+ZT@S}*exiK{9^>^3 ze}K*h$%`#y*GfBKzey=pw%Q2!UPU}2(oh};%2>|n@gX&@h}f0k@NUl`TsZHZ?oRSw zrq;6O(IQxUdn`Xj&jhX4e~i0Maa=_EV=Zs|0F18H&Ye{{}!Xl>Pil7Hb{mUosvE?)lu!>}o&Y^yWqZ_s49DCCD{-F*!4v9g( z`IF#VaAn?vn@Nj7VMy4eYVW(JVu!AEx&Msn*l5=lplizcD${UxD1ATBJen;%bRQ4Z%R@UaOHRi_ zKZokF+Q=Z8)$0pk(_X&HHvkU$eV|%y1OqN!VhM$_nM;dUb?vw%?uDS9gS;{EqWc4P z0KL9IKf^u6?{M2+Lw0h=PT4!KKkROk3#tBjjE+xO{Q;|d{qRG zdB%9rkrj|}?GrOuku4Lxaa`8ZUre|(oYPaeyote97*?w}FK)Yc@ zuK1KkH!+X}vE|GCCG`+LcVIc#@Ms571>|jRs#331cP@gyJ9RUTC8_U8a17sxA#O)<(W&W&XT1f6QDDmEp zGaB92)_tu?9Pn1W*wPwr-B`n)eWG{3wwfqntq}W--3?JaU!h&+Jg8O8iGTf^22^|0 zLwk@g1{{*>N=twG{(tHiN%bl3RBj2oLL~NT&|hTNh=tUV3v`}NH=tIhO>)f_%8KpW zxRG zzd-Tk!CbFyCwao)EQ^Xa!iCOb!P|1YJg}lPr}DcD-tWf%TzBpm2Hx$A!hfl}HKhf# zEbS&9tQig`_J3xTpH-7_>EXES_;_BHs>c^yIgB9#Yw{YN!Dtgu8~^)lFMmxBP=scBSDStmK=)I@E~9rFYsZ&eFCzxZsv;*+BD;t)se2>LYB}c?p!4tf#A1KHNr)t zacPS>`U>iSoYQ&dHPR!r&(wTLcm?ra-vIFkgUN9?a%~Iw`1JvJx-l7-%x{lt&OTIg z4mz)MuzjBfyGl((dYdv-9BZrhtXMR| zj=kOBaQ*Fq@L!Vdfq>%byqQ@9L@m_m%+o`~!U#R2d7|}e$FGcDg*}!w;;I)&=W*f( zq%rb8cRF%a+ru6X=AVuq7T%U6cz5nTUbgNq+qU2(5Z>u>KlDZFMpFtx+#Bd`^h7Amc~iv> zm%ka0RBJ$EN0{6|4(WCU2``n-Rr83)tAEx_w5Po?su{4zPDWg5zZ=n4@l=?*uKKnv zF!a_Q#kC@H(Publ76mt=hv3}@f8m_k&%WzgjI5@^<6b);qpcN;bWeijMo7CbtRa?R~X9F3JvWN2p>|G-&q4vNvDn8lBjx+;Fv z*^G5<$;(EIwq+w%u>%2&OQIG<}Svh+qMIiYFE}MDNv!I6jM3;KhA6ba28z}uC2DVQG z`W@71W{i_AAAk_+EZn%fi=1{S1&9+#TiQvb_mnnSbfuk8*R9s`UdlCy>=ah9P6yKF zA>lKO8G8+SK8y#0!c2wlJcn|;V|jJilSavcI0@3*ECkY^azmY`klOSk%(q<+n}Vmn zm&{7Ax7vH~jqAj%7jK2h-X4N9BHo&kgpRa7QCdJ9o*vp0wR&HXxE%>Iq0uZ;pneds z)^Dh$YI4G3ZNe`br1P+~9Uj4<-j;%NJggo+5wCu{%ytZT4L$l4DLnzCjUaeM4W6Ih zh|fRU79!{&`(?)ds=twZ9FVTTpr@H)QsNDI@x zl=ywCJqvz2nI$id1nLRWPIGYEg6W(%oA%c<J30jel+8 zhvYHQJ^!LEnOjJuojxa7TG3DC%a-_f>aQJ_~o3 z&z5`Hd(rs1Da`+!N?3bF>2aWIfQPTHQ@v!n*o-;_t4`=EorX4}Tv^Z6{g|pp>RT3A zEr|F#76}KmjQCGEH6m*OjT5)+%zS+N~niPMRZS=-+YZc7>&!B$x+_KjayXD5QioyFD>T&8V!H$90kSqMKyP8$`^;d z--L-7`jje+uuZw5&E)W-|3E~PM zEh;*_aI4S{xwKEkil_b+$D{cd#92ss2bBMx{`wZEzFvBV^1TVPXS+d9B#$LN7_Db} zCYj@)9Zk{iY6cdIJ%lk|a8yPMY)J1~UpLJYkDShD>iUVZ@pO?7o}>565%1vjAA?<% z>}}5pC-8muw%D}93<VQyuOL!vGI{!RZg*$(YAx$t82xG;ctNM_6VF#Si>n}epqrFcZHsZ)a`;m>DwhdROk*`A(=_?>x8{Pb!KKlx^?+YRgftlNk9@IDaz%3ESD>(mXsm=?wa$rVG-4 ztoHjHB+a8WC<+JFW78j}K;+CY40(4#5EjVw6RU})>0Jp^<^yFQN>>P_+es%2e_dy+ z7Ggk}a3W!C3r-pY8q5T7KKUX<%y8lR!Bs??&?A(+a^aj~6B+(zD#m%t(yj3bMCDuF z8>TAH2>IKKD0^UnFb5BZEYOkeQ@&5DXBVzK4yUoZZ249TvlgEvzi?TQ#=`4k_lhNP zmB=GKuE^AGCeRvQ)knt<1(aD$Mb*dtcOE18NFA*?yL_xE7Vd7sRBbNR`7v$PIkX(9 zG151jJccCx#KpJo0nH!GSa*=PBm*zytVHFTs0JAIruf`A4SlC)Fv4Q-YHAzUQkV^d z@#ww44~chy)`L4dsH?IU7+p3LFAn~NmBUmSNPEV0>;!7we@yED#G7Jua7Rwu z#0UeW%IB`OxWexB`NuXGcd5`blmj7QC=zE-uYM9!eNs?)|D=8IS=!(_^1+(xI)#(u zO@!@tS7+*PSoCK-qx#W;R|F%CLtJ_}5c+Ot!Xswnh_r7XQCku#e$S_SPcKA4*d`~^ z5mhNOENAhpXKUo|xY8&yhmAt3)0?qc>r6p4MOwZ-DoiB3C)E6^9#(#Ow`Z<+J}!lQ z3-(j|!hS^fE1is#eYm``KL~ddTS^|?fHW;VAG|LM53HD{g->(_>Ki2PfI8naTGQT0 z{+zdL<|@fc>+&`?f=y|`R5M-WREv=?J>wRTJ{0#|ZKTh-64rTe0ql31r=yHPD861i zy%L5+JO|1ZwUWNqak8t2MmdT&_Fr8rUl54$Xal4ygml6)t=?aTlu4=V6-Zy{lz*W4 z1j+x&fy6WD zzWE}MM@OYQD03A>HLVFFrZJiuLEM1ESz`0Z5S`LAgxyHm0;q35ajWu6lr13nF_rlU z%1{~Mp0o00q?1|HLJfjUYVe2dt>E92V~nsA*G3#u*$d^h2Fio5QlC^jG$_x7G7@bG zJx5k$;6@gfwFy2QZwGPCSU0*CbN*oI{+Guesjb3FO~_bh?;gm_lB27N|UqK7$i}?I%8{2C==uRo2BQ z=fvDlZ}R*t8DTtYPVYrK`Q01H+lhtq#;RORN3|dhrk_Ug1wv_A!Zf_&)Dpu^I#uuo ztry$s8H&`W+S&=nR4s^^!)`PEtc?|#i1-UBr&jr}^j=U|Wv$sXFJjEtVWP$iNAkEj zak#VxWkUj}COF}M&O6BuDSwvA1C+hxa$<4<3pzJj*LUPU+!sMx3jfT-O9sW-p96&-%NSqn~k`^l&11)%s70x|37xquL{4F+#BAr z-7M+wG5oZ7Gmbc21A?k_l}5rF*R}HICyw@$rTHsxQrZUo`jWtCo+qkT3gMw&qnQ1& zCb(~?KeQSfj|=t>L5J?0`2Os5;)ruS=+gnUGf8Q{DsRA$A-&-K@6NLDRyRr4D4&ig z1NGnQtH-dMzs1;Q+FRIkshBO;e-*+;rs5ykt9P>FV__t|LAbT6SQ2e3+nF>&H`^3e5eesrA+9yarkkxSBe z>q!APzxf1dcOVH1m$t??#|rRVcoX?m%Z4r88_7gp1G&=l9{T)iDSf;U1_f`0@l6VJ zp^KVHcxA$O#g#*x5$CPqMsmyOuK2t5W;lH>mRAlt$&TEoeWLA$$zq>OsJ(R*?=U_E zJFo#fto~g=^TnKQ-eZ5eIm?qbGH_#JD=^zoQ_``J#-?jMNgt0*&BrG9_4(ShUFE$x zcIX`+&K%S0Lds+(wVrZC*mUWB_cIPm+K5|cOp^8OUSj!=Xs{U4Sk8^}=euqXk@bVN zGvo4ftk$qn#klNVKMQpH35%TiS4+q?^6|}u{B!DE3@gjTi_7i{hYNRcXw5cg{NXWPf8dEmFOsqK zc~fp4{~qscUB+m<6q^{>&1T3Z0oC}rHJP~8ZW_P-Adg2h?&=kYPKS5jQ;D^Y4aSdkqCtB+5;pJMh0X4p!IeBi zuCsiAG<5DEYuq`Fqa$_skk=O0n8I`8H*pHmz}r`ivpZG&GBWO*7M-~5f2 zFD`;hXRPqW=wjO2ldh$+t$ecn1WS#10?_On8jyqm2L;YX0k6Bs> zejORXAH*aGBYH-Yav4_`Im+_u8Sv-NANc7tfiDUljx=Xr^X!yZ(Elg&JslzEm2Y77 zkvE{ScpV%olCXH>S$O@4o_qNx#NKh~&?F!lZk6lEvgu)_@M_-!G_`hu2QM5U`an;SZ&(+aKbeg!rnQy%6I`_WyJu;s z9_4~{mvEQm0XVllmK_{27+jhj#4)EDBh?&6=+XK>v%4}L@fs-qRh|OHQG<>fB;gaJ z{Eow~P1m_3QQ#-jq+k*$!->gZlr#uV3d&oto0moH+2_ZBKASbKc0>OlteTP)f(SitW^ zW>w8^#_q1@UHS>N$N!>NTpw8fwJ}#XP+=PY^^I4`J?-z{+fI}5XW}Nl@J=O;?=u1^ z&sm4RF5*EDU{dGm+&bEwhuKj7H4Y{M}ljU{0u|9wj<1CIBX>((@d+?&

rxN@7GKY-_ldY+{(^(60R6*NZUKTr0TF8@0w#x_t6XYQ2jlPp>M^ZzAZpLRav%HV(mCv-u zVQ!XRL=M2wgO(}OyZG4nGW25)u2^)IR0ju|fPhF8Qo zNYy#}kTmL%bXd9W+yb32mmWIIJF6@o(--e{PueyPq~Xe2!!8wCX6@9ShpU>u1=eZ_|eUv_ofFF zu*DY`NYM8dlNkW3WQAXY_P6l=UJO64kDTF~#WSJ=G;*dG zyt5>EHK|N@GUpKLlz#tl(#UUUiAO&k-8h!!7PZ$-J(7&qwRYR}kHNiSc8+bBz#$66a>< zC9!z=M|jRwDxt$EWpt3qyefDl(6=+othfI3X6QBv&tdWn=-|VF7y!JLz%)YM!1?^K7R2sOq`!O>%U`TE z^GqrJTWHT?I0ar2Wx_QnO1tu*^; zSsLBAAR!;nciQh_gYVPV92A%@Wo_P>MEm3HI$^Bw(_D9ghrCqN$|fg+UiPhgbRG3t zmLWDc77=4l4dTyjW^iUHrv5IkO+Lg`9!N$#)4c-qgwcoKO5U)(3;EgpXMxvM39lCr zG`B)-!CyL8SI`vd@u_*rxGCvD1r-haycp5g> z2#m4dT+j(!<(l$GEWlIrzv6(X+*lveTLb@1Z_(#{cxL6Miv&l(GuT^9p5xzgqktyT z-&Z#0_k$>+*F?;1(YsR{=@JYKBuHEEIB<#wGN6}Z67v8mo&Z3Wv z+wnx#A9am>tP)p@x{q2QY_E(HEv#!9{zDA@eTYV0+)PGe82$F)TTMb&j-$_CvqeLo*DZAU7>)k4wqYozV9Tk7IR+dnb(M4m$5{FiLuiDNEsfET-;^?k0o9 zMvs|sHoh2|%Tjk^1*5eUunl~CLIKxk?KdkUJHVJ-<#w$L8*gE?7QLi^!2)wl8J$8= zhYT#VG1^WA{YfPE`2o_@n3-?PvHBMCP}HuNpO7V`jV#tuqSnY7;v`Wsoa}RspnEJv zn}airjl;Kw~FQp}VAzZ+NE6jy7+@FxU zqKJ652+!fC6L1fCNOyv^7a5NCQM<7M`4Jh+|Nh)l4oK~;;I#?b9{R0{5Bwm!t3_6M zp!q+?wPf;N5Z+B7FiFCc4INc#vpg&bpra(=sAz^j_v-HP0bBTF^a$eFFiXElkeB5$AX ziu`E;n+#@h{UZ#WN2Z31Mng@iz0(u8?#4~VgTgn)8(*wUu3~)0BfmY=e)SYRObqhv z)%-kl+`f?OI$otC{Z_DJ^)yv+VieX3tti(G8z_@buIGQVM)8^U$nwg4R!vh2$^sQy zN%t}bS?%sBE1kE~E2giIb1(c9yHjuT>T5PSEoeMnSo53PCHd$i=hs|r&}3O^dW4Rd zwTX*_6q9*x?B)AyHgWkv!{x|&Si^31C0aGMgRJY=fs#tBw%lm;hHhoE*H=;^$l=-* zQPa7T4E*ay%ci8MYNIOa;D?fl^NJZ7%(Two&GXv2w>ysZZ|4 z-Wv^C$&V`|$!3wO?!Mv)Utcj<)fzWTW{(ZiDf>nd?onP}@l##@vxAZc2WpHP&zyUZ zs=C$Ed-M5o?Sp%j^OQ`H(JfT%%ioCVAIPI~MAl=w>G*!G(KUfFldl_`;rc0EwEvG_ zo!`@$?CzJ*$GV>3c33}Q*+tTynuW`+crWl?>l8J__cu?i=OQ*2^Awi8eq!>C>D={a zDs}2IU*GPOP1bb2Zms5CL3ixDMBG_>fWv16@tHi1bgD-gdE=iu-bwLJ%NJ2X?`JR7 zVa>qQ2~n-Y_u5ZX>u!m3EN64IIIN(#F82TNRXk5v%{TrlsYhbpu`u64dgI6z+J62L zu6S;dKAvKszt^Vmhxdi`$ENwnjAcjLSQJCIdY9u%fzOEf6?$>c{~Tn| z>N%Ngr|^gt`E}9EJaj2X34LMBR4y`Zh!xi(bI0awv(r)?){XNc2{`9`21_s#6@@F<+ezI%KFK2&l zN4t-@YSb-FeO^p9>d{EAcfH9UE$vm+qx;F>QBH~9ku~~1_APZ~4u?f8l>^!o&}WnU zW#E>E^2d^B^4{}~PyU)fm`5%jT$(GobfFX7u+~mDTRla3kmrGeY(0pKDU>>=>%pCOR z7PcoS&-E6%(~g|VG4miLmhqs&+lFz~yfY-mho6dRR3d%6L=V%f`BgOfgr2S5EbcY< zXX=nqe{{WFZN21HcD-vvg2KJ&iszhlfxEU6bIezQ^2%m?^DBcF8>WPt-k@&pKJxrI zg&BRvUD`C2&b4P-oo-g)hTYE!i&H~=(v(eesadES)a zbfejDMm_Q0M>W_#sH*ZkigkocdWHr zhieUNn^t*vbN;;O2sh6j$QeJ+aZ-sA`q`(g^lkBaRjQJ$9=IXIV4$2DyPT)r>8NMk zYovR9YN?8s+bV0F$|+wZ+Hm*HMRfPgOAUUCz1EZ5ykCS^FgK5>b+Q(6W8g5|UelZ> zyq~E9{~i_9kC&kz4VLPSZpS&aZV9fI_JN|Sr1IH+7wE^USPt0KL3iD-S4Z8)yXh`n zr1zmJ(u_%;dVj^Ogv*+2e~EP%S#Ql$(w<;-}5h z80V2a+85HP15-p!uUb0WGiRB-9=_lFQUrX^fqHCNz^C>U;{|i)5yoG1So4e$#y6BD zM?U8>`|Pu7ty7NNx^U%cS=e&&*EbbzagZVRQ)zf}H!gCmq#n2|K%KO&tZ^Um@%t8h z(mu&|(KRpj*>_8sI`*>vBZf4oi(U?wxEK7aSz6@H23()Ax2w)x95!O4ZrGy0H$ zSv+cSIjtL zWWHN#>A}zC!tHDTzSqMZDzRarN_s>3*mPmq(H-ac*iJvWcQo|%pEiUVr?9FIDZ_52 zru@|mwgFq*$>rn(%y&Eix9Ci-eAMx4FKO=c&qH5IOBq2J7x|=VV{I_`cu+%q)HzJW zX0G8G4Q5$xe(EVB!zBm%ISExbTHE`?@~`{nECGA$aQ*KH_$)9+9Pj>KtQb9*F70rW zBV$|Zzg54{WJ<#QcM>tzD_doj-&2}nR80}jA!8Sn+yz5+dLd(->+54WoCK( zs(uI6c=HT$xmror54tZtKWzd`_`)_d9`XD5VmhRbC)Lw&3``^ZOpSfL8Q%?U;=z7L zIBsGm%6F@j#$0o3@GDh#%wKA_`xsBY@s~pC>=I7l6{Y$6qfN7D$ooCOq-F}dO2H2^ zNe4K=g| zHMQx_pQ6vQQ_T&$YeY%;7W*p#=lHb45({t$8Nmb^kuXP{?zx<*j`SkKDd1@WHjp_s zxSLk zA2=Jx$p>4a-r~9a)@t%jt7BRB{@0?icqBh;!>O{9%+^9Pt>Wsl$6;`zm zwaQr={OzD>4ZKLuW4z~>lkS1_{ZDN83y#}C(2iMi^Q%!PR%G#F=qH0cW-ir*?7O%P z*1hSpBRp$drpPs7z>g>ft}(QZ81T9qPpdy4wO&?2H_06?7U0SrntJP{GFrrNRgVun z1jdew5C2X0UAp=!T@8ABlFNBi!gFn2^23T4ORII$H2R4K{>;I9t<6Qu>50mtVqSi` zY$sp--kVXQDhJkeDw;WuCqJ!88%A6aDacnkv>c(T7hAT=G-8goGrng40%9K&JO8FN}l97nX?FxdKFSwGBYI2E`QXu16T0WYe$Jq>^72LY=@ z@yXR0eISjliJ!JXjBU8hXjq!iB!@0OdAq{c$mBD*v_6czppN3bd4I(QJBkmVYB)?C zu`cG{KAO)u&f*cs6+7A%q}b|iG;CFI?zU;I==|NE_YV-dlmAV+wrL-9_9ZdXn}vPG zKmojvz@xNNfurk`8^4Xx^G8{y-WsC&`EOKw$dTsNUP%Mi&%+qBr?0pE8LZ&oy6q^v z|004eveYVYj!KRGkAYVLT!XnRO3+MMeVA)*5~sLsq;+MNi0N@T<&v^3Rgb^3@)NsmA`VcZJlUe{v#sE zc^J8R+@aV^56fA{>~i>`2fS6^HJVr&T?ws3#?v%OdSy-Te2>79YW&+ma+c3#XiZ}1 zJ$=xz4lf@$O=PbXz?FuzBjB6yGX^7lQJ?CZ&2ek{)xS`GL2Nh*oS02DHBuj)-L2LO~1cvwMdLe$SvV5siFNTDjM8TLa!Lk;E9$4 z!gUt*r?r_3{k4zl2lbMt|2$GP$Id6W8`(5)9({FE;Jm`+>K2CPA?PR?-`JJGTSk-9 zmJ`+Ghp78%S9&79UgDRFZq z^oeRxT{Y`tE zuF;PX|5tFulxeRSWn`i=s4 zC5{fTqt0DtsX>kX`TqJf3R;{DzXvU!WZ9?c5w5vzi_WLj$?nMv4XnI-d==YH=Vy39 zf%#RyECIbv$SaJFF`AL2a~TG{8NMRt$hj2uDBc~{V;6ujzN_ZlXT*^kC=;ou52WaRr)`w@Y$K`?&T3=w?4X!bc&A2D( z@i&^0H;8f`TNpjUZU3CNqHn~lZDDHm4ZOcsHiUq4z{u^?BxxuQUtLf}%?nXQgRfC) zf$Efhb|G1RPz;T0R2ur&PO9#`E$~vp_VyVv8VGgFr+TatRf;M`O~?Y5F8QLL_1EWE z%lzL89vBeGe|@)*@nlXnw}R86sLz_(()hyvIwlF+J2peFrZP61PR(ELUfq1iJaf+3 zqmZX)w;Lf;?N~Cw=abRnlcM*FZ3Q!h!IHMv*E^e>N99W;D0mrqvXZj6-1y18{Q@Yc zt*5}Bs(X`RxULWtJyU4vUlW?`F#n!^h{wNO%!|+VvtYbs_~ZO)QAmm~8OEC6{mS@w z!xv}q-Sof58w?*9{SAFQ3L5LS1s_=0S%*|H8}KPSuSulR7p;-Hnn-8T6Q_4&;#^lsYL-3c||2x>m~n@1Wqwp z%L+X$hCO-9@U_TeM({Z7LjkO&W*zSe^aMkDB5T{s-xilO_-gvf`nJv#LbgO3F4{A6 zH^Zx&Iky}@=3Fgpv?x4dr*-Vo4dkCvPQQ7r1;z)@(;u{C{A_J$*0BeNFI=Ea@3-vI zfXw-j35eYz3(FaI{mF24v9f!uMniV)nw{Y})T?iyn4@YuQJ8=I_!~~NIW!- z`qY~)pivD!^PATa*!r8C@8o6p1k9&^9?vb4gF^{8Zh>at_YdE&@x8m!OB>H>fhXX} zH6|+Pc0yiB$RY)Fy0vL)Ecy01PxHMBXk-Yuw*$sYDb%wCpXdA+`fKEwl=JR>E-2>cgVhq!HraUOiP7hF*sNu>#`b6gI=Q- zYZeIMfxsCB?7X@2(cLmqA&@Sg9#9_?#% z!t5G;bY@FO>!VHCk#QyxbgWn~t&8#AggnQxZG2urE+J-k9aqK9i!42iIu{KF+gsp^ z4d?3B*#E<;a1E;L+QfJm1x!=$%Uq|3JEQmZLiZa_=nKrCs4JHR>O|zJmToz*tENB> zM6)t})8KeT!&k=&!*R&rL}RQu9kl$1LM|hK18To>D!4G&9FKfcoALMhYoVdLNaPh( z)FuH_HF}kyGYOgue&dNS<9X3`Gu>bBLH)j!7ES*9C`^8sp5MvjFZBF#DYFbjB1CZfY-pr4Mz@m5!IACBm}+GO^que-UPzWHp^0<<2knxds*I=y zKUAASXTB52_$YHrBGz9Ts*n#UlVO>h4*bE$RN!fDk$R>!HB{aB2^YtdEer zBD<}}$jz}{0KOwDD#&EPKbPj>ox^Gqa1|Nu62pcT45`=WWUJuD5cWwMDk~4xwo!pwG(><)rqTc--daJbheeycJ62{`D_)AA-oqBtogH`SbGMyog|YtBnO7<7Ic26yeKS(l zpHhW9d@rkKtNi4D$rWkwY)46F-Szuq54@w0SF>+R>0kIWKMY<#{d(rt(OrMh^|!h8 zFqbF}Ivq(hBg3fTrCh$t=eH*Jd~bO!bfGdCPat|Fw zksB+^kmh;xo#;5Vr`I)#8Tm#v`PD$)@Wa|v<1*<alXo+x&9(CO z`-iNvU%KePU#|r2h1Z-}r)#|O)%~QKoWI>w?)RFbuihxaWsVQke}*~g+CGKoP4Syt zZ&fK%2iAfsZ(Bc2FK@<0J$GHsPW$?C$iNIf-gJ_jt5#CZrZeS`+Y`BJ4-bidlc#H} z=K0%~(f!axD!O|`-TW;pj0FYdX|DH{Tq;qg%6sSq<%Tt|7WZEwdo1g%YYvFw~8@`R-{EgPOZ%S#?D_(_5$iUl)+;zk>dHwAiYUa6x-3z$Ojjhu7 zJ>D;$+`5$X_#T75`|9@}=9B%E<*IlKThTc2A-&9lXG@pOGqokrAAECFQ3|ZzS=8f~ z)+&cPuuIi4+QV%QSFVssriN{L+3UD}wdIkto(f~jg}zuRXZFQn0QLndTCyaq9@7M0 z^Pf{4%Uz@r_dkiC7ngYN9}hF9+@@MVj7u?jGH57I?Kyzgocv1H0_$_vYB*c9c`{{` z9pgGJd)7BnfgTIRdG_HkOUm$*W>_<(-F`j#ceof|Mv~Y2$>eV-LI|83p^aB^{jJ3EMK2pOjm!bxrE-O9^h(iPjgrUcKpT?DgO-cT$O<~5^;z}ic070g+n8q5E~-o9M8;l! zmMn1KGPlW7oC@4;uJdop%Ma)~IS(o&Fc*4fosXjCrq&#M$WET5(i;6Q-&)tJ%`?W) zxzcXByN88({`Z9%6h1}o{yV~dhRr8-3(*Pr@|&^Za}E1aW>Rq)d1RFIz4npr_g=7xt!`GhU(QG;oAMyS8QY9X*L`M_0f1qUQ5v(~#TWDZ#O?jDNCK z{Aje9${c^CN*t~%%Q$4JV^|~WL}Cv0b5ee}_f}=;;E-LqdpUE^iCYw1lnkfnsSQ>s z+BA_&9inH{<;szI=%jRIc&bh10kUnQa6Ucij^$-QHd@fUj@)+Uymfq&-CWDhRc>wj zlMQBXNLj&82G8N2w~x`CU3pab_|nIZ9H<_ zCXHS+yhstP{l(5YV+hzxThelqhg)v_XI=;V?4v@jh-f@tjQZ7PY;JX_Cf(AQfKmEK z?+>EH{jnO?Er5II)4KBc*V1hJa-77qav|ReEdA1X(xL^xj!1g{_8uiI#`^V*D(dn5 zepAX77HLh}NMI;GK6`*0j@TsEf9%RD(!HhO70ioeR*nSj?qTKW9Ttkg9#80}H9+G& z+2Dei`vapqb@Q`L>HMEy!yDq?^gyYv6{6mQpQ&+kN@~=Q1YVnK8`YG+8G36iOJ_GV5h|f7T`VwITuBOJ z$JGOiseNf!7vMxV{r3UiBUefX7eo{4Lk-Thk~gLIGFVD|r{ADmtsYx2?$rLj(;R-a ztf>c{T&0RIdSph;*HrJp0s``-eGRB;G>nRC>>;FrOgk~xvD(=Wn z#Vh1B{f_z1r=rRqQaCpQ$2GK>C~*HI_5YD9MyD2~remL5!3*G~{Cv7b2`Ze@MjzdQ z90k30>e@ju?%X`W{n2xuKIj4LYrDXWz?FQt!Yy9(?HII5MQz3v3lHYEwlaHAi`W8t4D!DxNk@< zdL7ENyj7Ge7_^FC*$?6&_sYpvM}Kk2ggxM+Sj#}CHC$!dVCB@lhJG0kLKr7=E!C{tw9@V+- zuc865t$5Js-F&<7;j{tm9OVxWPYvzE=l^^a?)gS=a@k}Cz7ce(xM>qWs88#_*IRg9 zXDukHobI*LMs8SG&a%1JRt8V;iG=IG-0Qskz-a}2$sZrvYiK~|?2A00<`ROwqyg|F zYFZsiyfa9Q{+&}>50@rUTbtwR#U^~D5E=vC;wH2 zzF0JLE3Mg*Y^~j8F0cAoTu$v1n+DA%ZR*7sUB+2j5%Xs}Msm|Kvh$DZ>RQ}Z?4$mb zDlLfTJWuW#PG@kSW&Xz;^6^_QdQ-nGm9E-G_6><3XfM&ux{W7v-2g2xk{0d0LEGl$ zW5a9EyGGv%V1Nq7`?KH);o^|2pd+|#?>2P%(`i0+tv;PSrKmy19LD&`bEV_ar!~c- z-J!y0>}EB7Q@i~$*sjh6sy*=vq5pYa2aAM{qSnzTt*?$wQ3j`-#!Tcsj|GVufim7M zF%5cxcOOsnjd0AXc1)Ycp;M0&^a->@6KKIqstHeeosZI>x|F7+(io*PGJs(~DjEfK@-~)d}pkGJ8J(dlmRneS93rtG>M# z1*gUGlxNuA-ESE;-jL6F^WZ{CtB_Ly7sZ2Lsr01WDr!-#953x0Lr2(A_gDel1iS)f zLPI-qsgy}N_|RR%g;tLa7dFL;InB8 zpwpQQXByDq0OxyK*EB-as zQctqL2c|i--(fzZ%9T9L(4PctPgfFa$f7@gQH744^x#)rHZ@o+E|P#7y8guJV!|Y> z!F#(jeLHQFr7JP-+~vBXL~l{a1TO^-MMvMCP|&|z1baA+$n7A{*_4&g;d;`@rQEq& zUET6QDvx~^!Uq=qvKUP?@O32#pFjhhJPe+}+d8qq&R!Qz(V17Z;Dy%^w4qvGD3B@* z`o@?~t}=6p0v=I$Ud*|N`!KW{W8U~t-s$jhNp$g30-nLTr{M9pPp)0m!J6_H zx@5N)FMR6zUV^t1ALbNseKCSzSFx%aJ>vV z5k$WWRyX}(v<*!S%PozsgC`g3I!j%~p)(zP6+zOkG5g*~phe%BP?ec3)-JJY)XMHx z)PQlb1UQNRL}fSrP8to5z9FOKQ2Sy)>=^;fCzCCFjUGxDCZ6G;-krEXMsAK76K{M4 zj~`e`bUYHzzz=aKd_Cc3%J_QJoPusAcs?#1lT(gq=VO5{Am|#^W7z|Nx&fy~GdNBR zI$fK(VE;65v6&Ns_u}{`RcKGcywYgN56uq1v$|^VR2GlVY0zB5+tEAEvlLT(yW;N- za%|0pw53>mJvdu2eV`3n6C58YWD#^?W|T1FYdj2e1fSh>jGvZ%FW~LqQy*KQ<&2M} zKT)%MQTxU>qtE;D>RJ(WdaMH(e`kF1QfEMMqaiVar7*Ah6_c;^3Ey7IUjo9-{76r~WAC?Z4@QTLodM1+WvB}+nfB5Srt z$Qlts%F=>_Qr&YVLUt)4YmY6mZ`mS#XZpQ=yr1_$UH4ow=X|&Ox@MrDFj^f zC(TP5?1;$gcUY+WM)%qgXg*uZ*3cesSGKRvlBNa6=FKJLM@r)`^29>vK;l5Aa+$T^ zogt>n9(p$(J z|1V`VJ5l+;{F3%uVLo9R&|Hb=iDpQAB45_9Bi%fOQLcbW&yjAE4s*AXCLMuP3)=Oa z+;L)tn^0r;Sic6R+Cl0`yjW9%mB+4W5)U>~*%EXvJfR)*x|LM^g}4_~Hk9_wNsfOZ zkaZ|jdQ4UuJqjszB8_0l2x~CLcOp~s(7-hjRIido!Emz)U|talgg>a8oFPb8kWZXL zJXRZRwx$8$tO&Loq;dlw+=Sd_JxS9#$xWrzID0z*NDtE-hPe_SvvBh$gp@-{m*I0K z8>N{r;c*%DX#Xnjes`|3RJqyqf2W|Jp&8$MFA=ix50js6$cUf#q12h=NqdmSFv2Y( zLxH%5*?gObi#^McvKrDpW!WXu<}lJAO!30VL7kO8rFY=^fXXMRR;n1e2DRi==KDlo9=6l&PS7 zLLFM~o#ZEKio9u*8QDgP?}Ltt!i`c4tw--qd6sL(eLllTJ0bgEBA#?_$w{vxTIjGh z1AM5z+Cdln-GcO+d~dx5Xuh>7|Dm}R=H1G0(xCqo@2Ol(DEwAAK}p+AoHP*d61?zoh1a zt+;U-RqpUD!-y|8Ie-Vh>A|8+3(vvc*y5DS)yar2&*qmw#JZQ~rCC?jsBv zuk=CcEkQi)nq1LWt*tnkJe;&+GbkE!UQ>2u*5}q`~nXl?|!}vLrotYucAE<1WastKoRsI_ie=0rT`ais8o8zMe^*5bOKb>^% z=_<}5f5P1Z4v3kpZ)Yz$RfG08c%}RV^xsQoO%O&CKajqcRa%*P0Vo?0TW8Hs{){Ou zq-<0WUJ2q^*7>U)S2~0+Ow%XRhrKR33JYC20R6vsZ>>jsA53{|HQIag4fCJSik>~+ znVnI&AAUwOP&yT-mp76LIdNiK{nJR9H0soJ)|;k$|NB*#}jhh zva0^PXY^3vpJa*-b4^`|6E&253evSeJ)m?Ys&yhito*rFX%otTs&pLTvm|ZJMtt{G zbtJacae>D(n^EpHO=g;I(@+-aN;y1t+p?FQ&vB5-$51YWq=A5VLl6#&*!VN-mH%oa zE=QVIO%8O$9$&g~vvFhiJmW>0>puI?>qi*BvUDQk)J#K%{u6Qe>MLNkD;cbo^^&E9 zJJ9}fG=@?}CJGZV?SPexi@y(7-hO3He>GtHawr$;p5k?%ro7;LXTH3V5Gx;dmLFFP zg`3Z}@Yr|zG!Z78jlFPyySu(+W=XcVFEk2{1y^Vv%zOh0i+_m2Ycrwr_HTRCL9FoJq&@?V&T5DeBWhrA_Hoqxa2WkZ zG|*Wu31nsIH;Dk!$=29t6Q;m&JXHgz0ArPpTmuKIrPf4dqy9oJ%PRA=sgIuK`g znrHVgOTgvH?ZEg}P3an$ilzGdJI;0={%wO%KK*ngUm?>{46V6Fcb6okX6d`Z`J6!t> zl=N=Fa)d5=ba@Br{v&P1bKrEok83}OH4W0t2;0vxCs%@1Wj&E@<1Ov& z7RT9qrDw11K}n@Qi|Xzn4e5QG%E2L=>W=jqozF~bZ9?-aYcct29~neiD72fEEV;i3 zyPWzajDOyc-s4R$aFP~gx{idLFY9GN?fY1k-$({u9w2i>IMDq_|1FkSOwUt)4?2Oy zMQ_B;#A5dGpEU+qeuGtZPtY`QG}nKd2n)}7@l#IqnBkOBV%?P1+|>Le2A1k^v;66( zYD?A0DW@&s`{Fw|Y0W-P<7b%_Gars-GR+--l&R+R1L$ z8mXaE67XF^cfRDzAk=AC8=ZT`(Q zbGdH+aYk#2J;vPt!XQ>QZXK9^94vGn2jP(RA4TkCR&`(cmo8}SbzPyX;EzU)+w@sm z)!#0rA$yq4fO`Dn{&c6k!`I_|YsPKA8d3%T-pW(R+;_?tZI_t?^SuRLpMcsZ;;_Had^t)@oxXB58 zSKV4>_H;&de}~%-#HsmL(RpKg>CvJu_a6~0t}hIO^v(aF{uT}uw|elXV|PUQv?Y@I z86r$1>(QnUKes9nm+x51V_F?z=RSXC#y}n%(j#!4bPbZ9G-okk#(4c#CsA_h1SkwkwR z(2y6eOwyzq9TOh!&QeVmXKyKV;fB5Hix+|VSTXb~Q#uaeTq)an^He* zp)Agp>gX3Zd}zpLWFDirudJueY3K$OS z%ANX7g7j18wA8=Ccjq!e^B`}}S%u%v#cIAT^+Bo+Os+Les&!3Y!oX}?3E2MFDya5_ zfAhNFG`%0Ey2RKPhqdOO43rOv8WtUN%fI-Zo%l7IbUBjc8S z$Kp?2kT4pL+1&)XFRA)`N$pVnaik%i@i7%DcWOl1>0Ahm+zWJl*W|YOXr6V2m0I?s z`Fk#$dk4w9gC6ubln-=`p=aOPLnb|kI_}g2N&UzO`?Z0QR*>+^U3kXrfCK~T;rqX; zdacB04-y|RozI8h`r?;Bn1xsVoMf>rTA|^caBh}#5Qn>@G4JtFJUYfn^)|P9SBMc? zXQ6M~*{HLsk@)^AK^t-U7WnQkm((M2OHKpAo^-XgGPhMU$?bUX?N6OEv zpzy%BN3@z3KBWCDTzEVME7opD;u9=b+e#1zu+nt}jMhpJ z=8BTu2Qq_~ z4m_`mE4R*@1%we|VaL%xoQ1&$UyEb?PcqUmtaQi~cKNjtCbPOsch-G$IvpZ}eg|%~ znfA8(Xr;~lxPh(oU&&@1sv$|+X-*ZlbRnH5{STbc+&sFD#r-)ibp12LmYirH4S)q_ z+H3RZxdFxd)Du$ilymVt>ifAYB(5gEog4{i(|3qd`}d)*ZjvaNa31siIg%cva|fe4 zaqs8nu~*j!pwHBjZmiOZgbREXJvU_<_!tShh&RUzXQy6ZdNYE5FMEy-+m0x{1_vE& z)jCl>!S!=T1=ST?nd&CXpcM}-FyhW9>Ir-Ng4EmSd&8Qvhc2V~gec1w#G4l(IXRX% zEei5R?m;un4tlxxBTy|O&5K-K>yIWgtru9|O=TYMdMMq72@AJM>Lc;>?RYF6lLzNq zv$B=;m~=!3-qAaO-rZ7#ot}$Sy{SKwaNkQ*zOqYj<37EL*R$+Wk^ z)2{$MGUz+uzZywel6!QSj^BsP!JVr@U|!xpB>oemH)Zl_7fxKn-i`f;SK|Ak(tP~b z0vuZwp>!H9zp)w9oHw%}XT~9EVrKip6qesG=AQ2D)cx|YQ?k)D)dYM$)4MLGdV=qq zE-YuU165O5{NciftO!z(%MPd6QObj5OmyU0mMsU-33!gbp@G|mm>Wx=Dn*+7`6~i*PMll z*lb}Qb`D81$zJsjK=RHBhwD2tqiLHd&Z8_K# zq|tYlw*2Vh1Mu$iIQVGVTiUZ8+Ju+;VfhYyPPNDq=pAU{F$`>4E`;T6pq|5bV`o9y zX*U#>Lowp-P9$Ctx1U4-c^U|Pev&Q60`Tv9M{6}Ljs<({)r7*jtC2-n!xJPS!4M?+Wugg4^*`UXI{2_vd4tJ2_v zJIcF(Qy(3!G#_DZHqEKHd~yxVtE=K6ZI3TG=(@l_=}*#$uaw7w;CCHJZyW;BotV?6 zK991TAa(tn<>A^>QR$1@Uo#ZW%2nHEveF-8T)+N_7sip@>6wJd!hcIFe(chklTSj@ zYTD?OyV&cRKO?@Dd0nz0uSG+RZ`)7o`!kji_=w$ZE8%;PBcpmkkIRoCsQW!ZHH*n}8c2JbQ^kQ56Hanf z8?pb+X{pZPim{*K(kecx3@L_lv(I4czn(Co{#HqNjHMH4kLvPm5J%rvkayvvNyz7| zW~74!aW840i$dv4+dWByZ==DXe<>r~#}v0MJeQB858%quQY~r!Y|={1_V#i3c-g$_ zehL4<4CtA);Me$ozB`N9`cd&AChU8l>K9{2w&#j_6n8&(p9!>P^6S(z{OVVSZ&_29 zAL>>|l0Jj4BZ9Su^j0&a3G7$ZlGd+pWs|1|;tQsHeO2GD7+U2~Rb3U7enZ~VghaTz1M$>`7W z>D>yL8s3cCKk)_UVXaW<8{#vr+tPtYM|{QDde!MXD{C2Qp;fv7%xV}5h1(H*F96jm z_R7jq83Jkk#+d%5pY#mvB+1M3?^d;NxK5iYEv|6%a_9=Qx>%bl{B>Sj8<%^X6Lfvy zJ&$V6+lVcReMk2+Ly#7cW1XC^vQ|Gvx>8I{2_jzcBD}o}9`z0i@@$;=gYdg4c}{)O zR~;c|{6Fo{S=*8Lm3VL&#AeWboox=PIS|B^?Bhc&v^h24YkXJqe4R^w*IjA@cR|iL z3((DMCY5#!Y_gfo+AR_LtnDaQDPal=$vaDRk6sSzs{H-C*$!YndMprs2*N3sg}3WU z#SejL6S2Z#4em_z1JX?@uYk(I<%~2SX{MTzyu0#YP&W4%sJc|Wzh!U)kennqHuFwcq=K-wJ%Kk2>_fjlixjdG=L2#etRIZM1di@tX+ z|0HPLSwywJNID)O4mfbvo|DDL7U8&jPD`Z;1!*?9uWk)anw)gvVCuOuB6qYURD3_6 zQGK@Xcw5EmOs$u~RB!JgLg7qwO*f$YL8D7&Y0)+D*!I!POk76tn&{eN75msi7fBZY zagp)_IOAG>B;CiT#@W*}C!9KPCoXS0772IYyU#{u>oXp%AJ^xmM-CwARc1@iM^y%N zg2(#_Hx5=Peqe^Z7pj~9wiFWQFFTKvGXQB{rZNrNg}sGcL`^Wx?ZXq+cR`00deXt) zR#J_qI%1^z73U}q0@^@xBpigi6)!L(Gyw?%A#Uk>q>P7p{Q=#l2|s$c2~y@ndDJTH zxzE86G2M<6R=|Vls~K^h!aEI}_l-w&^rUYx2l7aoiZe|l@fi}%?fUMDo8)DwB;IpYJXEn*&yH2?f~^My$AHfL@14-G$F0MM)yN2 z=}O8?9yTMI_G|bSiICF;$eU=WW^pBLbzu4|h3Tw{ zLBeSy-bBh4S*W)gBfi#N=|mZ&TMm%71Uq|6seJ50pR=&BXCLnEeM>|0BWEU-q4N~_ zKL71poEjd-4q3Fxre0#C1sPpmGj@tA%Upzl=2%1AD(zQAkY?$nvH>7`smhp0<7o+} z7-2qK_S-8;YHp#^=$d(_c{6 z^YE}IzdQ)7>z)Sk+ZxIbvxD!i7fMeP_6X`}* zWs|L>wTGKWYXf;bMwuY!_x+*PR&LyzMp}JiSHfrN>qw+*mukxlot%40r2)wAYo8W1 zR62$pMqPl0bvB@#+@SsOsk+>AZvbyTJsF10I)Rq;7vo*O#=NVz2#a5vuxFQ70UKgK z=dPG==}CLqH28}xdauWa1J1%9?Mm^x!)VzmaS_h)-KzC2ptCWi_Tfvub>LGs?*TX3 z8gzRtR}A_&T&~<{Li+%~M3*Tm1YHLP?b?RFyn^u1>1A;JT00)TGzyZMTfjg} zWXG?pmamNG@aB`_L}65O82h^ktRHlU_r11GbkMGV%!M`Oxq8MjwdWgbJ7pq2xSrk# zAN2xy*KR5gO>kjXo+U7=D;_X%6`f0v_zO06jHOr8((%@-FEpNe=w{u3*WBKg=iffd z7U-Ia@Pi%XvTJ#F`ik^XsTUv9vIH{8Pk z>D$EGNtWFE#}BMIvySv^`2u(UxC?fpJ>d-WADu6s%rz;#72Rz zVVNJKema1gH~rJLY!xO?q^{PQ8qqT(=Y}&?Ljm{JvON*C@KY?E5lC}AbZl*&YB>O_ zuU?7$zMsGsMe&TTi5W4*a)Itv*`l{0#JGL~hx?K8bX*I@S6>t-s$GXWL+Q-ffR55W z_!riCTY;Ge+SBu9Hc~fgzxX))2o|KDMymy3nB3|hD~>F}lofy3SBFNj)L{wN8?;i} zJZBcy%-e@IANufZe-FZZ`!uZn@Ro)?AF{(+NOo|rq<_P)bUI)lD|6~#-)psbd3X?=^=Tq&&fK8-73Wu53s-8p@!!KjVdeoZ z8U3&xrykYN{lbldo2q`sAmTFK59*<%HAdUGQlwgj9!(-~lyjn7`YRi(7U=Mu7c8)s zjsdq`e+r)$Gy>3E1B`&%T=FCJp!8|-a^!@dEj_1n*DxE zd@9$nBO}Hk^&a*KTnMvPXkfvxdw6c$SC`CuUuhdRk()jb!|37&ew;EEr??e#HnF3m z`DW1%x2T^(3z>{%JL|m$lED`be5D8RI%n25%TnSt9M-H~l?fnw@+ws|Y7HdX8HwcH=#u_Zlv} zz;6LVHU1Mu@L@Y{vLZTHxNXl%wANbjy?!IOb=C);^`0a9xC{p8yB8t1(*d|y>~ z^Wp8!-{NPVq3|g53tHxOf*En{yve>e*k$n+lNU~<-q|c4SKkED#kE-bAs>P2TKnon zKe}gQJ|vk5s%O9%?b)m&;{{vl#PQ5moU&#uqdJiFUm421n|6cE>4#eSY&3D~hJ@R4 zMW!X3>)IY%Ue1Is-uC>(u~ovvz?xtC&;y-5U&NW-YoT>OA)XzXt1*490|9MsLdBhx zkW!-_JhW-dEA0A7g#%7klW?S?6>s1+32036XL%jCFm(#HY`dNxZ#E1=`gFv$mdWsD z!7AIv#sxSKqNH~82L z!=m+|-JhnMYE=+F$f0A`(Vh-_HQy&b7KADIVE!Y1>rN+RC#@mC$SPOnL@v53_G}ioPTI3K{Rm!rqK5Y+U0O1l-ey?7Oc;ac5mQ zIklC-Z1y?i3~cms<@7&k;+TVUPij8#%#ilD!)Ty}cnXPUK)WFm@0;r)VKUxaw+Yo} z5jJzZ`TK!-kgIC10VY3|ry6b`-j-j#TT_$5hu^<_Ij+^Mxn zNCx^WuwIjla_1d99d|%HwYUlocQ27w>Rw|L^Y$>R?dVs==14%W=a>ra_dm^{M{=9Kj{6aVe^b{^MzSoB4cQ2iP zkf%MjZZ1Tm4J5o?2H(4#QS~AZo&F0nX7=g@JxlZYJ7m1R12legn=_ITR*LNgy*Oc> zX4A9do__v_M`jSg6^O$w3@;)+j>U#Y2T6Cze0QkqH7 zn#tyqn@O5u-j3c)_@>E#Ej`X>$IaS;M+3@*ol`U1Hux=gopr4G8Coa${7AU#w-ak! z@P}{48#NmaeFMT@xbJ0i5vO}lRtl)q&cJidLq&mXs=kDQ@^`_EdYnX<5St=dW_=F)R z{$i0A{;}7ef>2#^zFjLx*F?YFBjkq(F_7Yr1vK9{r&|*s?1X;b9c2f7&S=iz=IU_x zUOxs51Ji+YBc6Or`%*WIr9N6ny*2^wxfsbsUA=fgtxERG>lJ=kctz-Db(0^pudwuy zGf?e`u>+s8q4#EBaiovXeg2XuEuhBbJIzked~1mJ6i$NY?Y){Fy;}1>`e!u6!8kC> zN|Lsa24hncXW+VCdhny?6;PUSiS=-WYe2Qi=$@E;c_gGZI){5|%!3O1+o*H98+%b? z!(vSRsxW6Qz9o(J80$>yi@k;x0F4_u)+?lYJ%Jr_BJt5+PeHhh5Br{GyDX+b+>TUX zxw|1Y{MrUP*7KH%<1Y=Ji}Qzg$q{X8tA3_FuFk1PBxzirxnr`=X5!>h&6Br{VCgMO zZavOWOTDI@)a*xA=C`k6>uSTEmBtmS&dqE5Ky4)@Otm+nk}&7sOHIuXZ^1|JC77T#LkLw>+q~d@<0m zCh+%tj_a=YT5`v8TdC%!d-tY1`|cD`klsboza!~n_A1*310I>mg=3~y`LE|coFVz& zYE=G-`iyj9j0o&9L(+9b+eUG)#A7~g|8F;3=+$41Qybi6EgsR^4dF{oIQ1_)oI3^; zKLkpBUbOfqDDE$*)tDD%oyGfG%UHx1Jzo9c5S)1+A4vb8;)3katHu7jCsq1`)<~_l z)E~Nm9dfM)I!CS1cWZA(c!@9dN{BChX^3xGlRPO2f0={s|IZ`&0hp$B=X8Cjc5nmn zkrmZ{7xJfjAi$TrIAM+Q)4L0@nB532iM!DH@ggiMyAAC;M+rN_Fi>+%-iG(?d=Uw= zfV4W0j#4})iz7V6mAd__v|}q@I>+1pAg=oI7Wx<4aniR)+7FkXGLa?zWm?s@%XIP? zVYxIb*v`nmz@U%MwOvXc;MVV3u&&hxMsqA^yoBL%a7}tK+V}BB!eVH>_9Ku_SAJVi zpNlVT>vG*FYb5;v$IIKRdJ^O(HRNHmrHv?uh=M^E7|r_yoYZUqt@{q`-Q*IOaJ! zV?wa?+F5w1^)*J?h$&7d9jxY`P0Pt=ho=v~*2e3_nsh&|FjeVj<;Rt0U)Fj$m^FVS zHk(ATgyk+(W3&%m1d2b0?HGilF<>=iL@P35g&O;*)4Le)8Y(SBnFG|O=b;pzHoU$b zw%WJ?`BCjvSV|hg7ZoQe%vWoW;ybJg7xMa#6_5Vbz*>8|f%cRytWL^C?73RJU!Q@* z$=bNxF7Skpfdw5jaOaQ@+BfS2G?z#@2kg3R3&eX$Q!&+-& zvN`LE_Yp?i4dILQR9*x9A3AdCCBD5@HfZKGt-?~pf6B*rG(_U^Ug)z)KsOc*GeVjG zBt3xi(*DZ3$*&GEl)bd1XJGGZyVP8SYVJp^UjWsM`|`~EJ3#&sybJD&^lddc-5>dF zTTXe4@~2SkW;#&4Dx4IFy&cHET+LRVyyn?kqAb_B${&ztW;o**kbYF$r#!97VW|Ed z;|g&I2qURyTjO@e04yKTo$hH3`Lwx8Be1TSD2RWs1MO%@dK#`Z91HY2%<=U}K^X`m z?;`yA*4L80AYD+M(OM}VuI+pawZwx!nhJ<}l=cFZanTw9>0w6e#oG^|vzpEJLY{S| z%8&5X)>Ww1ROMaumRE?~&6gu-J>_A*Hbg?uVN==nOQLvSMSk#VYq;=Pmxo@}Q+y=S z{^T;si;y%B_472Q)`YwSbGUCnSwR!|&c6mHy{U8p+YGLvkI6~Uu5AzHqptvAr^*@_ zd3*9VC5$i{)!*$o;eaED3)D$6ljEOMtr^?%=q~=4@mcV+50oYAOUf}+z5v7>SZ-KD ze4jX4aS)6ecN4OY_vc0NSFuKVXT19ID8R62I@|aNM)tR)T&xFB9ihnlk8+!PTBnOm zrRw?dPxb)$0(hI!T@s%0H#crmt&CJYG<)eUSA2GTBkY?N0bNcxNz(F?`V1Smy#&(W zs5tlTD0+9J_;M8vo~rW(o_?gWAQ#p|;sp(92OvM9CCr1lJr9esVgCu0|EbJy)&7>U zXnu31c#ODG`>l&TT)G=Bu3OF3ls?K}c7{5NS6HX~lR{}%;yffjrskVb9znYPA^B5% zAn&FA%q??I0cisEaoZK>)y_#g3Z27gev}^s!U;*)jG70`I^xDv26xmrm8G?KqDdOq zlM$9H4W=Dv^g*O~E<-gZ%0E(mq5j7y^KvCWCRDCTnwp(Rtx4L6&aoS}o-OV^9w-mP z*rJ{|UQgiNm1k9M#fiJMlxwj|cXdJeAJT`yZ})vvxN>$_rcm1X#GMv!WW*DuQ}Kuq z$E#X^%6hkyw3y@&>T&w;d^8VWhJl z|7de0p8#&VehSjoDpS-@*2UfWgFN6*?^|1a)eu+UuIs7zz`Y$>mCu4eRfEK$<`20s~V#B2$wLbE7B}cK=}lyJd1L0 z-Z963J^N~}bcP_T1cf92869OE^d;p(fsFWy>WTIP-A2#p=X_*FHamdwUCOL#$3dCi)?M#`P4sn zI?SI}Og+J_q%?-!Ck}|+A)Wa@+Gl|N%qK-#!2AXJGU3NQHqdYsME)I%=Tn!6(~a85 z!k8}LGHNh>es9VfjLO00jmNS!`hoCdv8~)|Gnjn%aCsndHP)Hz!#dnqjQije{~Uyc&XaQ-ilDuBBl*K-DJFJivg_s77~I*Ke}2;r1MDaA*@>&McT>)j zF2%!Kzco0_rX3q&(h*yS_T|&qR6cR$K={XZ%Xi1NK;qRkqV8X7zIW45|cUk$IAyu|JMCc@A3K)yD_9=yGJ$()&s zA)xhgEsNiWh3)BmvUE#v*ysZU_J4^qUbTkW8*eN)t+|-FCLFTjALB6Ft=J~Mp}cJ~ z4gd4w())$6-1;GqdpRbtSe=_9p&}d$bPH&0H^>!XxuBmLE;AyHpwkHpes5zfFl7=g zT93l0QA=gF99PJ-84toI506j|{yh3VwC~>^x&2T{wSzSGe0b7g_AYDzdUVUd{H0-R z%>omt=KOLaCmDXp7;_Bj%U_Ro$mhFTO2^*=L~7JGX1f{t{A};@Y7Wy3VgHuyCihcoy z@Th?SOU}Im#X+uke~kw0d_#FrdLb_S{fYL|=pow}9anV-%SVNXvHE|t?KA$0sLxkG z)w0n~AEdeA&fPNjr4N6RdWE-eGnEZKM?y^Q0N&x=L>W7?jNJ>E!Kb=6!R@jCuyJe; z2AyldL#_|!GE>?9;=QbQQ;Ge{zG|sn+1aS4P#AL>LoL5#&xxJE>qhBm-zU1m(3uxd zZ)YJiy)z$&l$_7}@BKP#(e0?%U>QoavjQ7@IEW(yQsK!`10J70P3|x1O0`0JRu&yX zsso_@kOg~J;`ICi*Qhbi)mUXWpITsSiJCi)!zGOYa&0eHVMy=v+IXSp3M|lDhl5(r zXWK&mg4I2L-Xla$?A`PlY8qMb`N06Oy3V4;+YGU)t1<86W-D(+59W2IX=$(QBiJKk z4Ib0E2ODfIA=NJocdswI2JB&vI}ZbXfu2E?tC(ZdF6cPzI;;&jf>#<0kP46J-z4=5 zZ#tRI^|mnP@l(S^ZLemSmQI=_rz7v)zX;!2)#J~jN6>dubF*iR@s{DS^MQI)sx_Mutr9_(&CiDGRXKaPd59TjFW9LG^)P$w5=|# zbsi{h_;^WL7d8KQ*3Lnr#zL4#8SoUiynrC6r`XTFgc@9%}CZCRlYOFJQBn7hwpA;~%(dX0)G_T*`<-odl-s3p_qVEOf zm)wDhq6u=vhml~maD@EZ;1zSGz1O--SV@}izw8KacRuLCCbWBd34NkX#GaT~96Hk- zBY!XAqZ{l%lcZtbS~gX_4mt-{<0r{_M?i6dd}?qAHAeJ4daee4x&6hXAyVVPhI1M# zFnv0wIsG!0>-CjXe=I&|AMMYx7kvJ@^K~=F@bX)<`E`m>L87mpKTkhZA|A; znk~FfHStJnTlRuAGCV6DSlDp-47T6RThbiKl94UshNg7Z?&nMhYtWiEPqi1iCadVV zEPMK1ke=DeJjCui*5!L*ED0kv<9WA0v~^z0Z)>f%ey%POhNE+l4!>CLDNkE$mOC6? zQq69Vgx!LAR&M#xP`>$b4*fsV*`^m_u&K@#co+5qiYE-E{^_P_VFY16FLP_FxoP1IWfMFZ&>FTG>fzd>Er zBI!Szb14FwH(Z44bdRE4STt9?8r*q*)%^XQP|AJ>T|=4!Hly51e#-rYTP7Swi=0Gl zv%hT^^*#&i3~-ZPjm-%YaJM;-t!jWU zgsn_8;$Ihd;L@NCklooGSDN&MtD0S?z2L66?Q~w@b2h6%Gs5;}K%5GneZ~nlU48lC zU`bU!MR<(?`fnUa-xHDMV7~*Fqn+<>L0rzlFC@dan7)cvf#y%MYjHLFQ$7*vMy+8J z9=_CMX(&7SyC3H`YO|C567W4Y5l`!`hY7TA>YI(m-00f}dVjVUu68{oY}Sr}=)^`q zb&oegZZhH$_;{%|Y;;e>8$S79ZDYcTV;Sic>c=3u2JNln;w3g0(s!6gkaZJ7fiwvc zM+o9->^i24%wMV_b)3s+4;b3RqbN+|Zu-rr_85BoVpA+%D?Ub@Yw6&@=z099C*T)- z?>T+KUVL4VgrrX-jf>WLTUBqH=C{DX!XH=ddjz`;!qo4G>mzrwI(KF;g_qh(ZTU(g z|0@1-r)Ma(cWulApE}77cm8SjyA{IBHT0fe(r0%5K|=|#LuB6xhWuYjCvAbj9w0qP z*k~rE6?PHCk<81nbyaPUrUTM7(r5C*Dt$Pc_Dox+NAEZg=8lVf!A4#F%{l}(M&I9K zL=xpX#HF0@gRput4qN+6L-j3mLr=5%mx~FH-Z1)oCKsE^TKe8HFMp{Z3{`c?BLDt} zmRX||*NaZ+D}ZVdjhiMwMtLq9l-rsw(5nq0KJjb>3xH6|Ir#V3E@>6K8Hnp$RsGsK zbd+W$8_+SzT)v5HgMHi$q~d8A(gzHKBg9M3M3H^9o^1Fw2aY@)CKqQKXn$yr<84`k z$puGU>oxp@lYR%uK(7IORp&NR>A?KOv6@az(}mKj*Dsxd+TJx~^VB0O#p@IJe@kaWb*$1lTO|47V zvQcudcN^I06~sy3;Ol_-K>dSzwbf-Gw}Y_qU=sSJn}X`Q{$Z_fDVr-B$DV~2>>{#* zx{|awR9fv~guD15exXF~vBaUbnBpVSy+GsRgqfT$Tj4KZ+X!{f*x_EO=<(EAo(f7t zrA?39c9$t$v$IKe3x}tT_*A#kKs_kFEUhm$WNy=x?`^<;dR#@CTUR5SwE&KBP&Q)| zr}`wlPUn(4e-?yISVw1#cBl6ZaPHu4e?t0tO=;JBXKXf zubu6BKhm0CDDJ>Rd~A?(27ImC01j*PAT!07)0mNZT#O8GKsVjBZ1l1#81iZjIIRsL zowS!_tVsgZQ>wlRyOG8U-=fFUwX4g2wErP-uqH&zgt-^?Qw>MruEi^GbI^9S!|^&$ z-(!*IKILm5#%~vPe-S~rpNIw}aWK&EA@tgW&?&tdr}_udwMf{6i<=H-?zXg_utBPr zAJUxmoD)czn7w+uQT(u}1gd@Qw4x?j(k_DPfbZ3= zG2L$PppihujWizo&&N;s8m7iV*v@D?IO*>VOl@nZp}CRuQkz$41f{1FCS(zwttQ=E zE*@!`la83miL2oH=oLtw4}AxI@|$(5OXIb782JoFYmND@I%D?LE#kF$N7kS|o^*Ra*})X0 zdEux*q~<_mP5CY6GOT&gmMbkqnhEMPoQ?Cm?jYeg^|u3Be_O!sxV?kU?m9r)74KWE z!5$Bb6y}2RZJyzEfOv}5%9_q8eJBX);C)I8BQ57j9vBIa;B}=D8s582+G;u@T*1z% z=~bS>GUFQ~FN2j<=JI#IKOpZb?iStB&|1hdx|bm3Rdb=VG4(GHCMfJe>Iv107#yFi zG@|@_%pBJS=T+$@(vYO{UIO8}Q2snVsHxIFjQB;X^zP=OR3IFNf%|e{fKe_!(j+kQ zk22(y9(iF)jrLLs?l#a7d!jS6RF6nn8fYx!JC6YA3^)~JhnXg$!R132PPHz@8xx)y zor8mY6wWV#TKo2DOc}?lgXhtqu$+d$QFt@u@9NJ+0Mk3qaGRh_21S&Q^~1oep^{jE{n)GKxrC_PB|2>ckh zL(Fm1@G(YLHA-XByc1Tg6Qn;ylEW-P*M%m1{i%MsaLRJv>w-(DaFetk%*`o6TbCz5 zUQgu)DpwI!4|~$Rya&2A+qJ+#lNZ!c5RL))RPCggYCPoyVBOzZM)QrN7uoan8^{OV z6{?n7ULFUL(e32ps|i3@3lEOz3F;fM$?Y<#>>%X@idPTp;H!ZH5=VC>4S+Lfulv;K zZcsKg3C*{M;FXk}Fl_x;Zf}1Tl*S=^ugWn9Ye3JTDX7mq9AAWu_Z>!`$v<&f{(bCi zS*$cUedQhZA)17M zA+S!*l)S|$Y*?-A0dZ^Y>^)U4kzdJRKsh;hUpMl)LRg(6i^*M^q+tT`W4u3U|zqvfp7=~`MZpvFaRM*2t9mDuRf04c*E?ekqwmVrHWnp0llz$k~ne&vQD z)WeXxYFAKR@{`|1R665-JjKc1Na|mq@=?;pXutD0&R?*WJZg&K3Qn3Bnv^67!W-u8 zbv}D?W|+7YU4RGHG*h3aAueHW${(@cl`$}@(Q3ZewhJRaf%LnKGC?-*U|lw2)LSI| zMf>+X(UQ*Mm>P=6l0vq%wYy;Wv)?W#GniQlfFdCqQQ)B$(C+J#=7 zH^ILi)x???DM(%#2~$X?Wx(eAXFzzz%ZhU;BPbL~vk!Urm^^<4kXJ=L$8e@H6;tP6 z?q~Ug(HtU;Lo*=s0gfna&+bR=hkA{NF_mZ0da&c0=5XcFEOP5|(%y`)f~!o`pzm!# z`3&SQw&djdLF@1noBi%Z=awyjI@2^jo*W(~GV+t5NcG6BkFJLMi>>&wfU}HpQ?1e> z_189=j@|Q#sf5=g-B~IRLwnk zB|&+BY26Bd@^(ofxCWFHRO8hTj#Aq*>Cb{%T0d(_~vmGrXSrN~5$sWp3oZ z^_Bj@krNtls&7GhlBwQ1oM*6z$^=k^xiLGzK2 zH7*v89a#+vvW+owtQ|k2vm1Xd+l;eM24cO+LKfmc&)7~%!s+sibRW7AeG*+^+{*{B zAvOk_ciX|F)t?3Z4t{*tnlB#K4fFQXd-09paf{1EcyT6}pYm+N2U}i*!kY$ieacjH z2pd43VJ>^#?0y2E6~04zj=QP*me;xc(T5kdt_C#bBAR=>eme z0*#9&I{wk+Q`Wi3!v<^MW$t}Euv`zLPMC2%;Hm4N+A;7bd_U$atj@z;>M)DO+kx&K z-rc(nt)ARqZwFq#Eg>I?i|hdV&Om)$WuNxSN2N63FPkEaa}<(JbvIQ@6kb>7^40fVA6JZOGXX4UQ} zRJMN)p?*%nWp51(dl@E|mp{Se%q6lQe+Qhrb%xz}u^o-Z8_MvN^nL!>*KqIWQH=Og zO-}qX0;*+C*LWX%j8B^Pf-4z}-s5WmHF`fGeSIw3A-E>?diw+JOwVQ);((_Q(3SP( zIpF*IXVE6RzLX0#;oD0l@<_53FJF8HjDAnS)EeC*EM20O~TrL}pp z?t|F&i#EiQgOTP)Tlcj_9*pXOEgmg})+0@2({2yN6q8b_wK&$@>x#?@*v#lNI9)IM z-936f`P*C2=gYi!+8b)hT6s{v8Qy+&3~6rVo$^O$x?l$`_Pq*r!`sTcOGC(; zzC(APFdi7N4K5#CAtUPag%;OOW9p6HwDo&5>UN?1+RfVIyK$eiw1(3C(Lo3~eTp5b zy^U{(Z3OeL+=YRghcJ&ITkJ$0vu4*Y{5rWFOKew(QRfQbO4!i>oy+u&r6XKOTfk{fkuv{r(})wV2`9 z6_Ag5Ukl?F_hL!gYw^M$$`JZTWBC+a z-pa2FIBpLFk52+=t-0v}Gx2LnH)uv*I8yJ^y=l1T*GB9@+7Fk>3$`-)UGNMQczdch zA8fFm&CTdeWA}z8_AYc@%K+(XZ6NoQ=*y{*8R*(77A)eA!Yj{US^mWmOy(!R#kPR? zkG>(*Dt5ov5`QPm(7Lp=z*Ro=8+{=+bYl<{q zKy@bnkE5#$sH$ngSRjZ>i--jl1}Q3cX9c?my9ZnJ4C)-Hiyd#KaXxgj0PF+>#X_W3ITt7vDE%Wogcq=U9o5x%Iip zKR0kNcmw$rEqIC^o%?0HMXH!3uIL~O)?K9Mbu8ih`+=P55pR1iX>R(U8dEmKZNPbF zM)9eY^W-giLmskbxTJi5;TKDAJ!y~No(FN#qa$Mcm<3p5*c&PTM7;lYDBS*DJPWVO zy&9F`c5Z-9Vb+qcLOyR%$OxZkjsGxA-O^KLn{ipyBm;HQKjZ44n-s_8yxFx?l5$9_ ze%%_jU+u&f-FOAxXB^TvRfS^vl`Ckz83YfVE!M4mq0#p=loSK;@FhK0=xZupAL|KE zR{1miM<>|1$4xjtvyKPtor|j8R*s?b%f1*$%5P4X2G?1zngi{sS5C6cy=~yr_!+hy zc?nwFn4vI8#UB1P+k=m1x&eI-eja=nTKlc!-=EdSNuH4)sz%5$@*=Qywd9m}1sL43 zt^DB^CsOo|V5NRTXwv?QNZvda)|U3?x>>uyyru;g3!U*4>R8O`R} zd2;$JUh@4Qq)(|O!|Dy^G{^XU;CG;WfVWxGMSevh)0{j2{{PNtE>=`vN^}l5tjl6g zR((KCw_B*{fN)CoxKW=U>phKAE?387Y4I%CW$#|JPT8(Gs-+EOgB!@Xjp+;k`;9{9 zaBJSkJRNFwZVhH8#hO|Y#m#d<&9PLRw%Eg6uBV>gss3ki%SjuBKS;bMH|(*MieH}0 z>I9Ly`a$R2?zAp*m?SRaQ|7({g^!8LXc;YfIb2|p4b#$A8_;fq+||3w^gIZV$ZKZQ8IM6|LSjFxRbz|!I`_;huVnga|qm<)-v@8PCn*RX$$ zGWPJtLfGj;=N5l#!&~Pxlbwy?fU^TYxGZgk&5(Al?SSSLSNPA8`foqdUejA~ZRrj0 zTf7gW+irk*Pj+b)zl^=-2bp`8;M=wlaHC@@P~2~}J{|Wh^#Q_1k-W-Ovu2hP=FQmw z*ODy>t5c!*g(GOyI10W$3uRQdFuPGa#WI0SdOHIo?W3mZa8$2_=-li#Bg{e-SKSB0 z*rEM*;e~M|@3dw#+`P6}q*%Vgt=Ywbv9|EwyW$@Y$@;EwSlE~oWU)#LM-ha z4M%&fV9l=m74u;ZW*Ym_>n_k)-i_qGn(yIy3a#aQnT@BiwuohS+u$5JKk>}NGf23j zacS6&otxl}7oLVN#RVUEIFg1Uye-f^YwjnTb!x#!cW%imR)>h!t~2D);ZJ~YU#=cU z=RQ7t0sk9TN78#pk2RM64m!yrPCfXP4?QWyZQ!s&J|hgq@a;k9R6k45JVV1zHu9i# z7k2SKXKCPs zh3i(+8m<<)T;b!n*_DFk9O-(nJo*;sKXQ;XSE5e88)*2n9uNC%$OtO|7qt<-bLKOp zSH}-H3Ntsx!&%QzH3t5w^A99FrLYVP=Y9m6o=5P)+*df_&@rWjB;g$%?;MXb->@_9 zzLs*G5$2FqO~BFE8wn4jf3OKCuA*FKRCCz<3YP-{*T~$_Haw5kfh#RM*q|Pa>3d&W z!Qy3$^KfLExukr# zNcbS!PMlV`&z`Ox52TmiM8+X-s5hGQ+-JpMs@Cx2z1OI?j$(jHFZUi+A4iG%FyMzR z+@WCko z-Tv;VDR&oilfk#BcetY_@k<$0Y{?M226sZ@9_)Jb30~e@gZKO2b)_faMQn#^4fmp@ zk0ku7o-gC@|Bx^qNSkO(m#o)hELkUyrWwPoAuWM02MLE@_U1aIf$DIowdy*%w5dLy z`rmn>c#!f&*6CUUkM^u$%bvZ&fFSA=q|$ zJ)Rvu2&esi3&hvpP#MRE9k{AA0MvUwR8(#~jW;`4QoNh0`M}cJx1mol?K9lt0&I2D zFjYg8BSJ6R5SqK+L*i39A2<*G(jMX~$5_e1+m544!$WX+^ID`i6~RNlV&U&d;_4V! z(>->ZB@19WY8ezna z=dA5$I=9>VCN93%1n4^i;iI6>5It^e5?9(iqF7Hw@&XrY_MXX<79{=3%U3mIBhz+J zU7TQ4qmY{P2RqI_%oLxJ20)sB_&j+oL^l7VrFtShxC+6yZZWERXp)==b58vaDDQxB zOiEXa%*wA?zJ8E*8N9}C~ z9htbiD=h5(RNTHXk@vh=gOiqoX5GgT#?e7QmuMZ?9$V~kWD5EuFH*PwYgf(WZ`vlbblm|2d*}^ zajxARvFg<+aqjID9G_j6&prJbi1*#7&WQ`_lSjG&%$bQdtpw06!I4-jVOnwqW+&T!ycA+yn**`eh&jW6`UxylLv5LD1 zgTH_~-;8&Llmhh?s1_=;P0#fL((`h^z9|g0%4Hck2Ux2<5v=~aXW+G(_H(5*j)Xaa z^b}WPv@q)fhYw^cY-Gk?)?-m`=V9>P>P0)xV(YqdMFvlqT#ncPnYGVCBtJy`$2} zb?!eGitD<2J;tYH-ALzFX_;I1YOSX5=0KC4c!l=dBQHvMxt3peS_@X(+l*8{yfCCA zqyCSW^!6PziXY$2H1dIld@t3fAJQBH@i=rlHW0~EGtz8Gx?V(v8}b_B5%m^2x{d9k z&q;@gJ3|(;7shotX&O~mLh%HCsmT;RKP~OeXD4KV>J?F5il6>lAb!a>Y~gVk9*s3X z)l2yK%|-PwNDtQI-LIYk!WLEA?B~+kg7AXo-<34I8#|Id4Ym#XPbluans^e3_bCsP z#EA3{Ow~APKu)~Ly0$t4%FEOD;k@=)`2LcY(!?6ohagQL{)26}@yIh~H%=Gn+K_Ns zX=3FKxcUt8V8n@DT-EfipV8IW82{}84hYzYSv5C{!Q1Sm(mWnMl6pz^fwT(wv2TL# zRFV&1RPQuSBm8hVTWL7{xMaNQr{Ol1EJy=W?_wZ4-%aa&D&xefR!2yKhteFJfrZy+ zaF*=k=-;C7cU?(1r*c7~ zxO93|K2*?Na@ia9v*8yPYVO?31;Sz6JG~oAt(#CC+fA)#?_J_6$}RGO>8yCC4;z=6 zs5o8o+iDe|X9iQ$ezc!S^np?R(K)p$)B1H!A;kj_hHwdq;9p`8{Hx1B-i5g_^h zYJWYn>2V|+gcd{os+^Ie)98#|;^+VM1J$o<_oGTEuV0V064Pl}6LIk#RQ){igOI(U zH4yJAou+ynpgfM!6dP>?DlK!RQ7jZUI12ZA*W*fKQoS(JkZ!8CLowG9#_?Hg<|-Xc z=U?B^_#4d=#myGL*+&P76YaQ;cCwc0k{L|s$EoIE!MRKgVKVuLjqsNC8>K#!>YcHt zvs*IaG&dPBf_!+o=4SU{ggbRb7u!tn@JESO@gwPLK{)||>s^sNDUxo)b!7+zm1ami zI;DS*@(2mbL1}WuRm6?F%aK=tJQpfWc6R)2t->$GqeFKXz;mPRKyd`}mD-sbn@WmH zbst6f6v9V0>T$y92ZOlk$B^a{Whpl0vSg8D70CkUjcf%KmEHvKG! zKQ}Q%|FejS&{4RB*V-dY$i0mD`uD3DOt*{KPYE?=@-I-+#C2 zks-wvvg0OK_pqHCoTatWHDy7}Pb6)n$uxPaQJRY81{L>EK62tpVP-Nxc{?OMgSGn( zS2cttCPA20rr~uWH$oFRMbpuFHD=gQf0M2;t6lcqr_IOwi+0ofjd=x9W zZ)rZ(3~DUR9VYT(b91>iaxH$%w1K$rj(mqBLduLPXlD{YpV?M!wh!dTv$o<P%P8AWaYf6sx~+6-h^ayQw-Z??Kl7;9mGEAp0s47Zc+rQ_NAFp&A49$1*QiMJd0 z2U~dC;=7`qVrj-@=9x^-Oj@ji)xTzezmG0V4(Kk2M~1-#uN;Sm4S;rD z>G&>dBX;nkJunFv{24rURs-@oKzSWX02IJ@W<6UhHV6Rq((oO*0@M3)?xJ5Qd3!)iBk38dT0Xl`*xH9pb_%40q3%|*7WW`Q+Yy3Y@V=E}y zg;wRxm>scA4hZmIT?f6zZ)R!mB<}}XleGh0mMoVPUz-15>0+s)Zhed;5!FVd8JGQiFh?&kCWW4icEI0L#CKc9hltW@~ z@_Bixpg*?qye}wM!8+gydYj%uoy-dunb`r1XV&08nT~R@{cLp|_$cI)7!au|o1~h7 zYjz#EEaWH3kS|E%gsp+L(ktABn-+|OW4{7;>vGy}HL|Y!Ur=rMo}Z5tZ(N+ZQMNRt zb1RB0(KA`{{za*h-lrLrOt%|x7Gy`5V9T=pVt8a7wD!`%IsC+KTdV=EK~s6#fL?OB z>1|vbxlXnXxXj-NI76S%m*8SQzuQ@7Lur_Dl+GG22fv^#aMN!J&-cvY_9=GqgI7C# zs<^fsS#aO&zY2Z+C#ko*Zc?Z{5zv*pWu6w+k?XacD;mSxvL4vWd8bC$zthSLF6MY! z%do&y@xP$Z^gX4I081ZWey3&Ftq}N(M1Q!G_#FCYSxNiQ*CN*I1g=a1qe7>@(*W+JMwxK#!%#CN8`K!UsEjQae6+{FJg~K^!Jg= zosH#TixBwfMCaEHnks*U3C$2X@BNndc07j)rrI zzSa0MDXc$Ba%6n6*I8(U6|gtC3ykq@hHB2IhAg7%vywAXQt1rO3`TRw9fEeCp7|kM z@7Yg&DeTA_WuC)Vg~x#MN-i7poW%v&NN9*6_@l;V2C1M9G_@gMEJ zLKmJ?et>dyx56`sw+Pid3U4bLxP;28U#~DBrMZd=TT_%EEB*V)i{-7@i$NphQAgV6 zG^iH0D;N&Hjc386q+E7Bbd;P#`(8~-Z2*+pusEzQY?`r3aey!>eWC3c*q(1FKg7-i zw!nWTBPDSFc~=Kcyu;%BX5gZt7IJ0ic(FYt5re&)v3YVfE)D=X587OM1=>M6t-Yl> z6U)8#YQ}pGqL>GOxw945D;Nde%-RbB=e^p;4qmdA&qfwuI*@u8DWYz13~&8w5ygC` zSQXTO)4Z@Fp?SCuzlaU_uSAfS3wk@`Qw|v5ISkSWCmW%1nlMY;3~a`4nO@OQe)G`c zx-iW-MiBQQ;VO7!^v6LJjbv`c1~JmSB~J~wh`FUcn4CWuCK&&&#@G3w6SR-~e5)~* zry5IPzFQD>5C+W<4drY2AFb!icRUZdrT2t$(nxv9v!}ZDYCKRpWl}ns;(ZSv7T18s z&iYvJ>n#fk>xYiU|HP0W6D{$M_VTayFuklN45u@Ph#T;d=OwT!oyb(K+%9uN;%@Dh zknfBzPivL`na;8G7Oh;?i4k6|c+$yMP71rhwuHL&g(5Jk5gsdVE&j+H?HPx`A}rjV6OM7pUv{q0 ziF8GlNJ*tV*8EPZ+?$c5&B|;e zDIdTvvkOyswSfezZaRyUl|waId>~IZP3f=+#&4a`zspCI~agw;oXP_7Q+i~6D?sU)Clz> z9^2_QJ6Q`F&m5SC^Qv|0*U+Kx?blQw{>0phdU&xs6^UzQW0w$c%8FAwp|D=i=V&4` zX)gzdN3g$i7!Nj?0zHzugUpavH^U34o-r$LA^vte1NA*~vCP{@;V6F3+CVwK1vVz- zsPACJ`H-Bo4JfOr+k6eiN#3x#p4}@V)sZr(!KR@=&yO=Ql}~vCqX!f zxlS$cSWy#c>pvE54$6apUgy!S;E|hlMrAtju1rrF2&Lhv!meTu1Ut3Ue94C@m~o)Fv@?3nD75kyh1S zR^oG}4LA?NtD_`;qx=LvoVOU}I*eq*X-HZX-lwip8j(>A3i2glcR)I8o*kj7={P{~ znxNWZD^qNcumS!^Z?0-fI3)|hf66OsnBb!aggJapU>nx3$PSN0Y!{WD5*~#g#EO(% zY?^mGPz~ap(8qAdVFyculasi0L`D5=GI3Hok`vk=x;Wd&*sMa*G zUJmK14ip~TlewB-3G)?KbJB{?D#$`J-hTxCow1OIl#b+kignd^6wd;21HbFp1LIBm zOR5WTCNL3|wpN%x?-RC(vvE-ICPp|fO3Y(`_!@h9w?L{7BwvH%)8I$yTIdq8l(;Af z=KBvrHMe#ZRGx*tMJUczX<&p6j25;Q?(+}9}$ql@(* zv^Wy=L;m2$vy(NNEN^x&;TP9KaI+7sStjL6{Mk04i>&UdI_} z0U7Ld8hBO{!s}_+)wvPh;j<2uSEX^m?y$>({05TG;B!qqk-QY&m1W9HN@+h+^0~_2 zzRMqrOG-z9ZA1&%s4$aJ97xAB{AXauO@f?3L;N>nEcip z_8_1$Cy&JyM<4O`#=Qy4s(r}^`zc5sgDD&&j$ov@nbKW^_Zpf9*4KEYnm?h&ruak~ z<{>*()>3@W2#5LkAbl8Gehlb3$xpq4%FqX@M%l}Z z+C^z1EAs&L3tr(ii#i%d$393}2+1dL!dqTDsQ|9XVbDG~3!fI+gLRlUt(hys>n7vG zk9DgtCCM{=h+|J!ls^p)B)cj+z^#KOsCvW@6JOkHG8xKCDv)qad#d0D>?ycK{&*qY zD^F&f3OO`N=^!?y8gSwYNp%9`Cxz0HN@ICs+`yp75cbA16BZg*AbD%j-)|t-DUm*x z&VD1Et&`c6KMrgPoiJE=cU2>d`WvJvB%GkZ2})!tuBl;e40GfBR{M_luwwVtLA zw5A15a7~suae7bupZ^Us9{5l33Un*G#`cx*YJXr_Y(o9czqCzDGMntx8DIMkz{1p8 z_;}D$B#(e|owqSnBL@SMFm6y0BfJ&|L;i=QfqN7;Yq8)uQyL>Lg7yHQ*UT%tz?9#m zd|_>po|3Mm^ToqkDQ~W!JkuynAg>_lU-dbf=ivu{>QoY@@V{OMa92@0H1?^_2ini& zSMrNtbYMHFaFaY6lGeu4;b}0^WIFktt%C3y3`|F8u2*b?+2JSM8aVdHo#k!eM)GNT zws?U0wBq%`4s%$qRRf?>Q-p*t8jH-YtLXA}m*v*I3&*Al?BZz_F|&vRUjXMitB1bq-bZPv|<^4Q(pS16A`xCx;dm*9NpQ&3zSn$Z)^TI-n0kw!UI*cVqZ?mtItb#tcZ0V@Jx<=7cR`2dJ3aL$7uu1x^Vh#MmikRQks#Lw&djTFft`ckQPJbJ6HMCChtZ4PO~nm z9+9-8mOKVf{wa?MEdq}*daY1=Kd7`&`2(Q&CSTtV$iE7zCrOyDd0FyX<#)Azb8)gq zdXFaG?5MaO)}6H6ek4CGK6@L(!{W6nx8=&rj!4%HmO;A&aWJR8o3JWsC%X=s4gtUB zK=`1EpuB5BrXBUFZbGV8b-i39OvfqNHMp9Sp94QJ((p(+U8=Fa%1lwUDb_o3(fZdf zK^_hXqtrYzbLTyR=1uXc>Vq-r87N$l)DK{!d4PI_LiPC?JNDD6n33)V>ZerWt&3MT z(t2TVBLf6sD1P?d3e+!QGGrE#W}~y-Te;9|M;E8#!lzIx2ty#q z>kJUDpz6_xLPPPyBu61gr;4=9LvCv$YXJE&pk5eX7Jh;ygyxI(*}9x^GM%tY zxMmEHs-Es-?q{=#4LE%labI0YxR2zgvBjYG!rOTVX@GV>@6ld2ZzL4PI|Na$lyr$l z!BC*u;`JkvMUBw+tm@Ykm|%8640JkLt&{(mo1jf$cTH3C1E_<)fjAq8`=Pw-oT@#Q z?@aZLZ8AIKm8@t%eYJF@KRt>YDBhG?9Uj6F&)!OZ6VAV^#;oxXO@Ml2nwjR!k#ZKV z6vZO-FceP+>V+cdaiknk+zBP-rkrw+6LyFfgPw>tL1Dx@Y1EJI%?WcQX-9aNAy6C8 zh45)F3^Y&1n8ayrs^?3+B1S%@S}&@4B+aAZ$sPRj6$WTk4}v@*C%#g8mUVUB1=JVd zbe-MQ{B4>+`H@hr-Cz1p>r~W+lcv^EuLnmJZxqza)Mi@L(2{=^MFYQRRljq7Xff*$ zejb$W^>u2`lk;CN>VL89(2;1Ec^+E%7|NT0`^2)41#EV*235~taN(3-c)t0%X5yjT0y zkU!kO=6Otm4aWUxeHUQIz%Ur*yAgF8Zsd!m(DUD}&eBkK3k-l7IBsh%Zq#`f?8toq z*M?4KK|Bs)x12)XR*!LM+z-0{IG^^rUBO=XBw|pVwfvF(M?7$TIL1yZz%~nNuqf+= z?5wXjk62)UtdlnecWA{w*QpOfW_H6#nYY-7CT}5Z{W7HQg8TQHVcw`v+IOx!+Pu`` zzLr}x6YK+kt`~L6OCZ4L7&Zu=$UpN#xV1`;M>O0me$4DAE{>`p{T4*X7meyb(e!fd z^jG_FLy|Ff>THE)T{GbG+of>deFSiuWS(I^ABOC1C~`e!LG-=du(Tu*OJdKXbw)=% zA>%Y$iX1Fs<9@MuRkda5-z;?7atJ@>HHDz02J+-C9U0N22*->zLPL6fs#AA+$jJ%U zgbkg9DSg5+w6tBS=l7~ zXynH42-}TtV-JkBRHBU`to&*73YJu>KFwb<4VI z#$qf+R*&mQod~!Eo5bI-K60h~Ou6;`Nxaf0N^V%BMQb{gxy4ya+2@rFpH~$szv*#t zYvCc#?ROBW{#yejv0dbwm2E`_<4m~mw5EJA#gPXY(}Abc7m3s_qwvbhofx*hl6~#& zp`r0|nn#R@JqF{(9Fn*4tl-IC3G?dSgto7{LW{GznY+VZoIbP&V_X@OUzgD9r2%`= zeg_lp(j!?Z#yh zd*cqgYY@e+ogc;WqMOTT_j~Yl@i=z=T7vLB+YGx-O@}Le^ySx_D6VoV!{#_E?{E^5 z2Gjnlvxea?UqczS&_?<-$!By8sK%mV(d<(bd3M$qoG>;^Ea*`s=-S}(v5EX|2KgfI zbub}(7gxD-R{uTwuzD2ldi@B5WUs{IIknI*xv7i~Z;Ayi*3z?9REKp3;D|k$%wp+(BqOVNGYV@z5R1$Ne-=-HyZ;Nq(< z4UhZE1y^f`HVf9W9e=yZlkP*|)|-j^eC})E;u{4whW&x5eRSkeZiA1m8t~;Uw_v>e z9NgO3mM45V$oN>!qxuh$Pxjnq1vhf(|5KR5J-Wx7;VcVE>`;5d2V8TX(z9|)*p4Y_ z+T>P++Rux6VbsE8emuFOHl>fQXmD?%aJt`?2X{!t%F)fmq1U}|Ii3Ab{&PF{xP(!D zSmMfzbL8#((d3V*C>U)mr}y2h2}|1~UAqiK%462bit$}PH?YzvE<8s64cvDzmi>l( z$1x@GFe|X0j9Bf>$4q;K$GdAVcC>}KW1NGFawo_~54^d;ypzGRxXMAoFJxOiuo7 zK$s|Q{c6t$}J54p{YLv2@P$qHB%u zk$#|xDYJ|Vkd`;H7{!D;+=~Vtz8xZt?8i}iM-l#v2Z{@3+4kj+Tzh~~T@UzdJcJ(x z15o%)*n{ozHn>f&*$+RejJUmh~luXjO+Et-~>Ieu^g{mX3WLw^6 z)k&CiCY+~6O$Wja_|RlJ5^u`y>0h{Q4GSv}MPP60PwW!5DwHO59#ZqVyAkn%DH)fm=hy%jmh4s@?Q1txsd zXLb9#qt87PUVHs)*gCU2RJEk@0P<5<-oMA9Ylmk1&CAB<6VyVgnu-6|5O2MC#)wC- zs?!T$ceJxS8FU!`de8aeNgak}UHVB^gP-7YVl_R}m%?52 z3&nVT7 z*M`3D-d^n5VIQph3~sxkW09_pU&{J~C0FZ9ZvT!2Ky3(HbskJ&lSEykR23VUvp7!> z25Z$d5GKK>sUVD&EXKsLPk6B5Ufh&?7#)Irx%I~o;s+^QdMrjSld(wHjR_I1;=@xq zlPBvF%qbj$hTS_54|S1I)+J(7jt*3Iyn&zJR)Fr2SkTGJhvj2Bun~LQ!E(!fRC%E| zqEms;G)Qp;s#m^f&n4`n8zZ@m4X-(bu^yk#LoC+i+PEb!Y20X-<5Cw$w}?w;54vef z+DNa(p3tfLaip4NEh4jp;)`!CX0iY`Gc~W5BJbnb2e-vqOEbLDuo+xwd>Y zZE*-a_r0TiG#+0(0(&)X1I=tNuz5@VvD#@hcutP49K>U358Ej4J-e^^eS~WlbL-|3 z-z@{mJC=K&_DIm(3GNXc8R0P<^=1bjjvR;k!QZr-lI`f8|5to*#spm+xZ~8uhf#6p z^tAO%%|pSo$v|8SFFqYZ!Vp1u2Bd`~{^~B@^f5&3v@Q!UD1uPME{GzAf+Sa-moyuonwWlcznYm4PK0n6QiO{@!HV0u;9Vfbf3Z}%wq2l z$gr_T9&RSpHGf>#0bHlF`nCRz;#Ihj#TUvyR+8fGA z5hqBKMPr6{9iUv{8+dcDx;a|DDEm&=LFZ@UX5qI$Pk0^o69!3PjI}gSdkGZ_6(`2zIx0p2dSUTrsyTWS1&ZeC7oD7b(Al>iBT=d5w z5~=PO<%bBqeoFLcHB@OkSY%K^8lk=1pJpU>?JWkXKVBsdgIW7pRF4gX(vUPSj9!Pt z@oKKYL9YW6E}_cd^M5sJ?qCN!*F`ntMmkx}(rwI*cI)FK-`ybuSv=|a6qT-kH?YFs6KC@?~ z`Qo^40ywz$g@^^~l^+4cK?iB`!;XP$ZJroojzKx#PSY~obrg} z?rwrzqm6in0R`mwN8;*CFCr!aOExfHF;8< zLkAbsN0%O@-4y@K2}%LoVV@M%aGRAo=-%i#XmF3tj5~Lp>b<2*`s}W;X>2PATWHOr zt)%yY;#8`4nBLbC4h((8+AVA>qwiUYTh_~XZ92;%q)!Oo^k14e_ssd7ItEhd$ct02 zij_9)syY35;8q;<@P~Mk`wmxsYK6@{?WOgCUOegz;YyO3RD2NAgZinRa;mYBu20n| zd62e3=}Yo%{LrvUaWeP|1V`^=iqHPi+WfRPg`~Ny;ZVOW#Bo6oqd!d$MzbfI_2jI@ zK3w^gOM_h{>3*c^;yQzE<+Z*o#g5#`(!u>2yjl?rzFDu}+b&&H{>pbn2<4U*cI}(EL0YEa(glI0#Cea~n-N@ z$63fx*N=mW)vllQn0v%g(x;cOa*LL{Z#%iGLt9QdmbWuH$S7{C--6m$CGB{U`;|3U@d{YjoRL0ZMl+q2|H7@icOYp>@)2f|bOmT}19SJ;%6vb&L4%+@%41aH zV|+$8j4iQ+7On2F9+sOKd2FQD$WA7s;MM1=Zsak<^z~($Cz~U9Q|kLr&%=#!T57j+ zmiP5E+_Iw!R~}J$EY**w8TACot1yKflgi#w{cmO;^WtHc{zOK&i9dGl!}A9JlvfA$ zPy1NX*;H7%EnJAsRVgNXLW9-wEN3gz8u9m5JE(-!+wgQ0?iibJaT$q)lK<+#DFPWCgG> ztr^uSVd*t&(~tHub?=4!8g7F@Zv*khkt3qt5n9*k%CJ*%B0jTUO=IHf zpiz7j^5v1zoq{-!x%{ypJe-CG`7P)Zq35+}nwbX~e+{rrRUw(A!(ABlZ}47Y*NL{Kk9P+x*kZcr^9#;MuM6i*`I z5)l4kyM?jUG0!QurMx*rtj1b+YiRx%^~3OV zl>w4JgMg!2@apbFdbTN&c4O4fVoF;KIC=rhrZf{YR5-YgBD+HA%Q5uaj6U=e<1oFrl>(c9j`YfXKT5BLYq2AGWHl(VCq~0MTEJjtw zw~k&;H%j$V^)Fnz&~BllcLM1}F<@bwhWeLC+`*_X0X0{eRBN*44$Fc1Xw_WRZ*_l8 zy*|l5hU2@mo$$>)Fa23Y24TKC|D0r%{wi`9Qh$U68TYBya)h(gBWli#?x&)f6Vfx( zqp*}qu?{DUB+XSzk{6-AoHg}%x+qO9b3MM`wFcqLV8AG;`jfE`vqWtJBoluzr73&%kG$7q%lxV;+V6uznNrT!-p z-!Zc(`vmcX!a5{;XU%*X@b=t7VE_^yL+Y2OqUO0{K>04Hw*%xMG^8(;w?oo^$_FYA z5X4`KUom6s0P2VN@oTT_#Ug{HT=l1$jmu@qSGxNcOTsA9j1{EwTdSU3dW)HlLE9rnW1>`D#gCJ<*49|yyz ze*;+VIU=MU3B^X7EJ~HgA$8ua&fx# zH7xM|ghS>8^E(47#plsY$;<5KO$KVv=X3$So_rg-u=fx& z_?_6<$d&&tn1z;4W-{waZ?^Sk?gfu7r8srUB)KXvo0%Tn2kWlbN~@2x8H=XpQJC*r~gKj!?__T4 zS`SA3d4$j56-+FQ7xj$O@ugbx$n;>DU55C4B!lM+-ihg@A7BJu$A&%Bbt}C33KZ8zw|DFEf^_O?X-vwWA)Suy+HbaNNnM*e`ef=hLtqzkEce(tDHC)fE3DW8| zxGpzC%|E)w(WMKX8qtW~ifSw0-f~Clw5dE`=V~$ZP9S{fEP&<*JH37*XwG<(exZ26 z`!^1EapZI6nSfXM0uaX<)7&j}3w_W*n)q~}`t2>>wRMrxn^$2U>RnJSiJza}byG35 z!#gb6MF(e_CGeoi%`9N&KQ{elDqiaG8lv4kGpaig_$3@_TBKpOKC}*TMKp7NXhGu~ zt)cqGPrvOzXH0#W8WSsWmraxFPq)W5K21c9P6w`Pi*kr+=^O-`yur^$y33iz@>!qS zYvpeHhMc}bdn%45W&xHas9^fXZn(xS_Y~=xD`SwDjX!dv)i&hsqUZQk)*)oR$Y+!m2c8`Q$T3{@SGC zEqum@pcQw<%4NL?gImjUD^9SH@kfMlPg{BEYc^|_Zb54Xr1o9gGq~_qQy7=Mi4oq3 z$J{6NAZ z{{E3M%W`Th2!kPNe{%KT2Xu&)F9(N1v$N-_@qy|@{4H6DDuz}cjey?6Pftl;xBnZ= zFUBr{mdjHie1m!}z)Mc-wv{P7kJxP`{qm=X zr}B*8?zpUDiROK18~m?FFE)B?e}2|lmo1r@qz!3xR~zee6{*%FVGOp+=>v;TYh+pd z7)`8GADMgW9>$L@hUG3>_}UA5(X-EC$)>DE!cfAOq0sZ&c;5TqCTRR318F`~j?2M# zRB;sk+5QL){Mj!3Uw4*UVh#i4Bu>j5igD#g_b@t`l7EMajbroAATw}Sne0YPrgJNi#c2Ya6h@REn1AX*py-spRq#9u~ zJ~>AFU6U4^ll~+6C)5rZNBDFUPSxp%hy9FT=$(&9Kg;Wx>#FmKYL~~?42L(f?ts;o zzOsuw-81fT0H+#83a8pzIn^8f*S$Vmc$X&5M_oXb3&i7^j9bsyx7Ft$q-7&XTrSt> z>G6h<0QMhsB;`0St8a?+a~@;evO_TC#CGvMVmzco^%I5jC8+!_88DWf=}Fdf-H;2b z|8z#_r^C*N*AR^z)9|d#4@86dpmJ<{v$ZNm(4HqFeGa6%ZXatY(yq|nD&4u9|8gk|pFmvZL)|OaRDG*SNeB(g2w-=0dsT0AaTwG z*z)ccB$!u?VD(0hhIjgtdMru|~jc(2M_s z3VVz0IYQRjEs$TwQ1&&;M8$i912$ld^e)Vf*8BZRX~YT7fN)3XALO`g@HKq&(oPb# zvy=PR2^HI655wWh#cS{}X{P9XY$@z&y8vh`e0S(OP&rL_2Wt#Cc3(P%5ho(?E__?< zs)UJdMf#={KA2x` z3GCgsSj{PrrV$gIexXbDF(54>Xl}Gr6X0ST3(J<;p~BZGb!M{kQh!ETOUziB4*SmC z$E7#7(|ex-57-O4AM6#7IUg>@{=z%GXpN9>C=52Vl*^;uYhz!Ikt%nHGx*$`Iz0Ae zK9Ei#&WjYAnirwvnjHw~;q+{73~GjE!-@V4k+g;8#ZMD)yGu9(#N1?E-$nyzG4Z|V z5N;b*Rjq>v^B`s8AJ}v=6voX<#=`j=Reeul{ovGyT|j(*G=BVisIFWzbSt+0RjhJ> z;&m9^ZEJFhCqLU9k+6jw99vh0EbInc<#GnK|;^XyNs zJ)b&A!zT6Q%@xf_1I<*N0)1-NkWOR$39FZY!ya>a&Afm;d;S%P%LK(p62>aMDi-X0 zLHSfq{M^}460fsM-8;dYmb6~=$wsQh`dsO@4{z?VQ=`4Oai1|z=S2>L1kV;z4eJVn zvwd)Q_z@w|E+=`iiSDJSrLeHWhnG99zu%%R*|2PgJALSl1IZrH3x zSJG09a9d$YH9r(Jx8`R0Ch}_mh66I$tK|Z?J)ty>g)T{Vkllg?PR*K{Q&r6oUR-Vdb(dX00`n16FC2R)Op{ z{}ZH*1nDjy9m-}N8v`T1-2w6dOxI(HthIpd{~LG_HrW7;0bD!SiL$AIyyfj^Tp8nr zP4%8$Y}fEBPQy6%!z~Uv*lxp+c{Qu?NK1HF$eSnP&%Nd8Al@!@{qJurAuV;=VG zj2mf2dfr{1H1R2=BT2{6y;#FMh@w6GW~SNlEirbo(~IM{XvB3~dgqqXa7g(^`b<~G zk08y`A&PWCxtV)Bj1j?zkGiKi&}S6e$|^ED?$B`J7~zz4uIHZxLU# z$jT@wQb^lus(U_%%w(0V$liO8tl#JK`@;*j?sK2#oX_XI?(>{ef2@44;|83mojzX6mEjqFzw#gP)F z=OAf&330$PMqV5Vi}6TWKBLcq(!xJgIKHTd5lim*gwc6Qr5Bg4$)sM)0>Tw(`92ut zE;?6ms!s=4pOqm<%Yt)|flxJ}c(CoCFWN)zt{~wR5?6J<_JfG`-huWE<^M z)OpF6=AR+^%wJ*uJyu+qkpidFT44CQ`81bL82Nl@vEdWEuWu@etFdo%ZD~^)gfu>a z{Hy%Wy(d>#W7%~d99wcs*^`5UDA z!*{kHSpR<(vRY76aGX>diBq8UV?Ur8#oZR>vQiUghV&|?jq`-T0kv4Bp)XT-n5NCdJqzAKQdC{=3I2d}>A4p2wrohK z9pTPE?Gh_Lm^`5+E?`IDR{t@0{CXxdynIIaFjcqEWlMWb-hnI5BY(|F2PmF^ZoTQb zEh|3_=@uNfvK8+&KMN~DegSzP=tpbgZEAUlG|MLZn~`^l;w!?8^@u_ zC{&+c+M`Q2PkXcx>xbYd%W%%O*Nc>V=r4HN8)ToS%{Wuljin{ z()BE+VK_AXl!}Ba(1+c^xu@GpdLIrnjTPT?Oyr{@hqc?!|AXfL+K496IizW}kZwuE z8S!=yQrm)44#qmHJTH`|t8ElXTTX=HIM>BcH^PC&rTjT06n?l zDXMw8=C9hneGF8tBKpSm=AELO;JcHt7*Sl!WBQgTT4QFI`g}HPgr4F(m8tN*eO62A z4WulPdMutYscVWy1?gO%I%ISlnX>N|+U;yAa^^Od&ByM;tSn>p{8katzZvyBDBW+^ zt2R*91giG!zuQ8Sr@N6bwBYlbskr}&2{u1Kd&3;FsnU0pFT$X?-$BhgsFmnH_Ce2_g7@_|R%uHbQxD~n zOD0}t$_Lar>%x`4*fX~ybls`^&nU{XzboED@)XLKpnusT{P%YNcqIFh-xw_vueZd$ z>U(CsNzYS+9qagn&9-Hk$(q>TW_d2Te zhVo6?TJl=fxaC=IR9QG3Q#NpPm9PKaXEu!+Ny?tN^3>(ly-Dki5QO8X&YiA{G?EGN zI<498J{TX@90e+iAK|_MBeEKCi>?pZ&0XthuYmoSrfmuJ+d2{nY)np3Y>ph`v1D;-S9jZ?^hm-{$S@^Sf z+`Ye_9I{{`?-FV(?;0<|M-RNPe(yH4?$TZOSsDiM3twZw?{@O0=U1FRz#Nu${^$`~ z{+t;FjKwwQ4`?+<4TS!q0C;yuWyyr1$V~&m4JJKp7k9 znxWZg_7c1Jc7n^nQ8+Al5uU%-kI$j~C%j#xpz%PbjGN4?ZX=kycp?rhHRLYV8j&=m zv6P{d?b>a{e?hSrp7ay{TXGHPdZblWGImd$CgXbakt0pMv%3~|ApFb)@VrC&e!SSq z4+hnvF`oo=1B>Bx`4qf!YbluiIl@;*_6By4)-hT90LKKBL!xCVtnb?Z9^bhSR#{E4 z(Unl~Jg_^=?|o42-hT=2@m~eCiaSW%1z*wgcL!(}N!fFoT-<7#%=6PG$o_V<<>!A1 z(7K)-91HBh+xj^J{gFobL7H|kt#NCu?(nx?jA)c=1d3`v4hS7*WskT z4TaXWiL4u~#g{J)DeG*{_w}zOmsS>H=UQh)hWk4Cu(&Pg{C5<0*z9K(le5`yePhJJ0rWlSQB9M_S>QNe3!CA46pL~7)4vRTJ9sEOMK;~#R&wz!K0 z4^FW79gaB2$4!Qx=_9j#^pSNJO@nt{(}YRVf6`;sK2XOsN%{?I9d+d3&I_vEchY_X zeoC1EIFt41h(6UXS}AvG&dXBb!AI7!nyzo zbhnA_FHQLX%fV=O$sWD@ci_Sh3oN{3E9p4++TK_g)Eo^|oBX$XxcZsA*x!n;%(xD< zigO{j<_Iy(+z+GtX}$eHdVFsA3v9f9AO@Pw<1IRW#;7h&G#`&Rb4Rl`7*KS)6ZJdq z)jk1Q1lvHhdiK2ej2e>0S>t}|KXmwo`1sBQc_sOyW}?MCL1QbeF4v(wJR;%Ys}Q(< zG#F-dd8#2C5x)E{%lzZaOO_16-b*jDT`O8^uDKiX=BA&((_t-AU-4z#2MgEbfK;=f z8`VoRZ+#jS&OFkM;<@~HRgFaUNx>jmC+o&^BiYPy2roQ$6qeR$SA`uvViU3N6+O8o zy}W?1N^V?6>&OS~fO+j5c&6SaOf-G3y;?Cu_Ah%6v+|5(oi?3#S^7xT-?;fzHF=}k zO}y#37#D3jh7A++B-JUaot z_lug+?i>TcDn4eH84{kdp&7%Fuoz30TmhOZh#VL$pRa7HrMZDNu1@^h3_V!9zdtPW znkf|i4p}l3*50ZlAJv>+=%~j}nN5TB#=qg{!9w{UYY*t=)Kz#0G%m2|auko4w-Uzd z)dZvY%VBVQ4b75!B}m8Pn#B)A@{+6UOo*9$<{ync%U@PuT3ON(4ElMNdVejhywHrE zndpd47h3QG(~WtEcLCaln(!By%_JR5Zc2N=yv<(Iy4ruJw+<2hzht4^ZnHg)4-*dr zs&kRQDz1X;@)#Q~Edi=;VYpo4Nax~$q9X?}*nK^BPF{=`XM|$eGdqZRl3t*9p{3lm zvN`T9@`NvS+7PC`0RP`Pa%4X{N#_eUygP!ZbqcL*)9}NNT};)vLnj>`T=N#Z*4d%P zOfFhDOWx}`R;p_iZj@6NyNRi~+okHQHR+?F-TX={TKEd>brK-`=NTYe1>t5bzjuBo z2_N{okc|Zy)<>cKOkLi%=mewxBVm~)<**yyp4kG+@0MwUe)g1Q=Ud_F-cCS0#U9^W z$#fSi79L$6Xt&lnjfy+RmYB;}oj6>WK;MU*D}byHo3VApEwRpG9L~P96E96^#Je9T z2EF6}Zqp|n3|tQ|op>X8;z=6RUGzd5*uOU9Og_xd6>+25YaCxvbD|pZajpqLn zb}u8q+O~ywvFI_L>Cqe8#T?Y6J<3%*B8MK%!~Nd*kfy(u{p&W2TOM7*#y^yd>&Bpa zhe$BJwF0SMvGX-+xe9{R3Ig#1_f@FK4UcY7TKvs0f_F8&dc;|(PJ zn-MQzpyd^+!4~KfLhI*->4@W*&3X199sa%Z9PT%d)_Cgt5i6!`#xEUp`SHB9uycT! zjPkePvldN-Ee}rO;Nb0=w%y86)v`~IOG5woPV79;2&#Jn8U@@1>I+`Ovl1f~Pry$_ z07;!cV9A1c?1E*9Iu@p0NJdp(pY67=Ljfa^a9O=gtj2_kg8HjqMb36D zU5n;Pt&@y;SGwQ&4IS=Qfbr{KbUxclR3^`ZioT9mFVm??^QgXVxMMHAy5Cc}J*zL* zoOkBho2x|ff7x)p(i7?5xH5Smb`3m-xvLuRf1Y2^q5Kt~e;qm6&xtj&yoN7R{RnS< zF-NO@oU{>;CW2P6sYtqoYW*8)-nzRQR}JBk{AxNITz74fGeV7k_>3zIrkclb=MUoM zee@l5d^jA>Gn9Q6zE%HM{m&bOY}F{> zHJ6=|V**Cw=>euy8m&pS)p+N}dA#z-P3F*gy)*o6Nw+oS1N%n<@gvT)Hv`g;c&*ea6Zt*Ziq=DJhNHYSY|EX8s<&}UaW*OpKYZQ^ieGHTi5)Dl z?2HenI-#B+t{lqlAJtWyTE!g^+cS~08zglfgw%W5&y`YXFhN|wewJ2~q>ZHFzdx?c z#D|5ne(4U!{DVO|k+7fH_|px$muWquc6->>zsa!a;0Z>*gBzE5X-+(8AuBI5Mz`}W zVxiY-Bn?HqJsPQh+4MfmI{ITsm%%^9Gejd(ZgQg@y#Js;i{eTWOE<)+NCDrr` zZgd+id)jY8^Fd*9b+*02W_2uf^q?0fZ^5>O8uD7NYM^8L`cmT|K3`m)+bOvt` z9D_rmdK8dfQF;$vJxLW&J%ZLI>=t5`hhSQZZ7Y{mA4@s#`fI1 z45S;8dKx#M)CM=s_fJz%Zb_nyRm$@IOxy8WMzfGmW z4W-LTA2P*>b}!B7xCfMvk!E!h6^}yGvJXJomy^ck&(Vk0%(%J^@X0mRcsVvuvAnb31`({)V z^cmo3o2prRcMOcIc^ht*m;h-`wlZU=xNhJtCU>bl`;Z zaz>XaV)TUb*gHX2lE)N3mXxaF(Q^iVN^{EirG2YDch%oiZ24cF*y2zfRWA{5X-Tsx zKS};=36N&MEx)rNEMOG>PJ8W~nQG6&^(UHovmwo7*6Qa!;7uGz%F$ng9zuCOzVbo}(6Pm{E|0_?=N^)DnIPQN?lhzI)9tq4T%BE_t+!x}7QQ3T5QF5A zweV^U;RRIT)ZAZM<$o2=Rn@EK?_9Vy`4CiQ3=rW27nJbJjA3%o(o0oiNcxkJ zPX@w3p|rE|kWb7f;h&ncN3m^V%;`D~)V%f3ae$K8eORTN4|)+dZ4?{ALP#S|2G#!t zuFh5ZMPUf_x|V!8dDVQ)HS0X`a!!Ec{_h@?Aw64lkXb#0= zhiW{rqdr&s?Dkp*?shPjMcbgtzvgtUVB}Ag9#iwfRF4tYFgi9=`&nF|vt&3@4gymj zbfZ3rV37lBFz?=nfaX@voRT-H%P(~=6~ilrL#?#O#91SebP5FBUZgZ3qn|0wr6qoZ z%}@8KIVS(K2HldDYZT5s1~@7#9H#4ju=pMI1XnP@RLeU$ro}zqkTs zA2_G2Zt@e=xv%=$9Qs#wfuxBindNpjK{}jrn5R&haaAba-Qmw+sM97Nh`*sQ^(oMG zsN6@A&thBM>fqy>fAPnXf#jL}s(6F`rgJcs!;-&fl^4Ed?amYikw)kD<7!Idk}d36 z=o-{L+)0u*Al&K!HCH7u(k_f}70*l(PnW&K>6J*z*VTeq)8xMe>vXX_Vh3t)O%Q zQ{xj`?l0SzEajAC0A+@ZekMq-q2f{U$^2(dElKkuGxf|RohMUuuUU3dvphH&RWCYa z499;L!Xf_*1Z4LP%E$L3KdM5M5kU|G?bqK zc|PiV)&zuIAp(AH+q|chI8ybz$4ZUwK3cud0jLss6;(6VQ9WX1v>B zlTeymWo6wmTcNq}D!BY#J5X4$#$nrW!%EH)6JyPs>^1qh8aE23euT4E4mYQKc@l8HdKxw{sr0cWj@(RW64x5 zL!J&hn@_Vrk}$D9S2-l{V?pub zL#VKG!!%1y$4B4EuHgJA9|?=AYJzkYYL2dkBS}AyGF7I0j^aJyRn{(Gj2a7-LGwim z-w4a?IYMbFg)8csRFVzh7N>-p-Q$oxYavhxSL0tEW6851@OYDMts&gH=zf`~paaiNe8Jw&l7G|8R=&a^1gJGv0~& zf836z{A==H3j-dnvyS!_Y{tuO_$eej<$My707CA%(jXG8SnB23ej(G5n@JRXG2 zA+n=&)|Mo&^Wjaob-;0v_NvQJz3}q$6g*D?>olUumO>g;qKUk`=#KDOZ^4`9?c&FZcMBJr^H>&d#c!CF z!y4CG+{!2yE$f)X=4>vW%BSFhT!3|Q9DjCTopkg~WjAL!$>DQ;6pX7~gTJ{mfVF>5>sr&6bJ2AF_Ko-6m_OwTTn+6k zM_H_g`LB;+`x@-WQze7_2q-94$Q^tILQwCIcAdBES#dqs+sHvU8cRs_S_eA3{00(!DXKT@6q59TDtF6^$nAXUeVwC=oyH~%!SrR z6fvF9BcdM|4EiYI_8w&y_dE04yY7^&$vXkSG4G0$*RXn(NRAI{W#54Y(d5^u*r;oJ*2O3p*u(|OGQV;vr7V1nzPe1jH& zqi7GIjqq{%6$of`3%}^yL8=$n;(eyT`K=Dsu@NV%;*l{NyLxS-o?nL}j&$Pw-o2&U z_;5Zy(^|S$%RbuUKs@@Jw~)58L+Kb}SfZ7u!Y$V3h-3o|w$QpQ*0O73eLipQFok;;q~X1kYo-$_+FTJDz!dn|B}8J4S&qZebvU041lY!^&* z)D<7r)z!T0kc4r^yd`0VnAag2=-iQNQe!BW$-~u}5GI6c-}%kp?f&Ir{zqE7z-T|! zRt6@tx>|*I>3fC&@rpEA9#5RoLH7Q!2!oEam&8wu`ar%pc8vX{tWPX!E_Doc;`3c? zf$9?SM%GpM$@;arOYB`YZm(O56`l!2XF%YmJaa!n{bvH!VJlv!7|jjofBX`F*84_Us5=Z#fechS7D&FZJJm`}r+`Fd3-t zs<5@MLzAlazf7M-&v+bBwE(fdO!5zV_Jl9+2Ya^A;WdNm!E8Q9L|(bC&2E+jx$S3& z5dpNXPUsNg?v{LRkByA9fo5#WWf;Dcp0Vw94^*G8wDFRpX*A};!klFkb!cn-wF(Sd04eiA$N<0%jhK;g!_5clI1d$9YR zSUxlq(?h$%#rpH%+^Is0=@HJU_IQWdbU)?BXBxs|xoZS!iF3sJyig=;z%{N>8uy0X z!SCl4pn1cyd21P+9~RE-gHuN~hU)u_fi>#_>ROua*nmwx924_1qeaEQ05qF<8YVsY zB}~SfYQ1e!f$CrAZ^~h$uP#o>CzUV0^-uc#iJ)PS=95t1z zap$TZ+eA6TJljUxzQJ=^(Kt$T;RP583)5dy!rTsx6^(#mkK&u&*W;GlQxlVF^(EAzg#E$L5VN%I5(7 z@;ELcj<>49U0sm?)MsF0-VSLV3)+k_mc`lruy^|%@nM#seD=%@@t3ytSlrL!S? z>1-q%MXFoXzc{?2k&M};WuyV&(P;2VY;xLT3%b?$m>OlHOyzw_t8k^)7KjdIwznx5>LK`G1B!IvnrkX$58b&lGftIfjwaC)w3)jaR>R9BjUcJRawC z$K2JlmyDFRmSAI2=GOo?0Fs^?~)l=HTOJ69i<35@7 z*tdIE{^_Fnu=v{IwBUl|(|s@eCIBez3=c z=evM>mCPDq&lmoft6fuu!NjE0kjo$+UY9=|`bHffHtNd1X)Oz_Fv z3B&j4aME8u*aJ#?s-8O5UcR|nwkyNd<-nC^0gijmtaFlDBqpp1_QGL@T&7h zwxL9ykBv>jXASz3Cpr$W+KQ8}mZL&1i+;Uk&Zuj&#*SgM3Pv`41t}#XIv6l`fZ3E>I1scD* zUNpX?kTU0;ARSsY2g7a7VEe0mq{3dTWd{dx_iKXXO!zW?HMD5d40N{7g>&l?aQbOy zu6&RG?iFIl{2h>9z63{(T8-__)RHu2aLCnM)t8325cmIUrXeoEm$tP?SN+YW<6_I( ztvL4MKsd>I_SlLS&v6au9K6p$G2q%h7`0~tHlMW@NEZOSN;lhe-JcP-Xz+_o(MrG0Gnx?%261FLof;sS1;oR~!APKKRZ%3e*<`6SeIG zX$MJ~3!KhfQGGxjunY2Xa=-XWBT>gE|6K(C)Hb`}h^EOUJ<;{Eg zHd6eqC4Rv4JvW4SHxuZb@+oH!$<1qX>Q7MDn(*N;_9#9iVr}DL`Tjie@#i3+*JMHc z#_~ga@uJ=X=o5AXiKDo}KlMAemOf^#KJ+YNf~76fUtiXJfK$3{8FK{TJP z^r4uu$_0eUicYSf*Ra~#)k5h;@-l*WO#3qZ1ykCLJcKqVDiQBT=CSV^zG~BsxZs2_ z4~c_sX{i>0>QsC7Lk7NGbPCA_;%n83`9kvkjVQ|q zAs;YaVVs~$0=mZZk^y;7eh1253X?qGf}3>42-w0|m89%W*>5GacQ|Ld!f ze6{et7pC+zx)g2)&yi`c{z*WUCYiIVGZ1ErYyTQjEiDAX5Y?kqnca7rObmY>Betg2 z;YvpyYj91JR3-uWUdlOuDIThQw>P%@DsaiVv?~2S`#>xZ=3!j#YP`@kmUO2HQa@;F z*O`rknV{xD;aH38U+7!68sX<~)-~sLzRuW7)VJ@Et_ex+7wpnFaq1)f7<7(>XRp!CaB*%DxGV2`_H4iMUBp4{(5`KvP(FBA_cp-ZIP%O}1(@|@zViQ^aui%S zXQ5EpX>13VDnID&eO^#T1XLFgyYXR_@7eICHtqS-ovU17Y|Ba@9B1TVKzJEY_A*~- zCZT-(leVd(hr?j@<{CK>VS}ns*QkKk0GeTC}Ux2Cq(asq!b}`(exVASCZ46wi~NfaUd< zBV|6Aa4i9y<|hDU9q2OW4^!TkG&gyd3qTrF&3(bv^# z84t>yD6dmmlhL?>$#Mr~=#vLD*PL`UQyx|M=Z~eInbO+iX*86D;@+_yVq$zlI_EQB zU{j(pIxTqvB)kX03$}c}ySVmf3r2bwYcJg1PI=&HuIlpH=nvwWeS4q`1__sdvJ)0v z+=NlCOg_LDljd9SSD9->hndGAyjr^A3nagTl*uqPpX=wfla%c+;x-`6qE%-jfV?L9 zB|38ALZC6`lu6;aQ(m0>z94;G#g8w41q;VjosfJD`p)Y}xp=vD1p5H1*>~A7|o7+s1IjpVlgPR|aGETfidSBz)8^ z629K5#*fn)wFb+F$-WcD$!u36Zrk9T#>uV{y)S+c?YC#c=TG^NQ#p?B=sS(g?x)Me z!M*I``=gqTBl6Mpg%$m*0@B9M5qlopK+6-Wx$g6Icw>x>?4`d8Khr%UMLw4BEGeWK^M}O&c&|N%4j7~F$F`GPObkC{0*Qh%%;bIS{8&v^|i|fkjZ%yR&7r{I%X^~vk zH52@DtmLmjznIXU1Osj?4x)XLg?Q+Mydz(fU2uHtuhH08CgH&7$%uMVo^) zc>beSyw%`juz3nxZ}e`c-n0&H|2m4(x$yJW{_;n;0d5_q;d4{EaJ?~qF|uoQ{&)Uh za9Ey!R&H?+~-Rc)w%-jBu29T5iT)gOG{DrSRVhq&?I}$Q^xO z;7YR=+8=Gt!1aJWv|sKsW-xx0^q`q9?rX(5uGHaahHa#(fur*lNdrSKe#j{tsODi^ zdJHyi*Pj;}hC!<|Kk?^54c02vK)zhK8_RYyhT0wNA*4?Zp8j~4%czT3?e%nU?o^5& zdwgdNJQd4g=RyCY$6%134utGoC{tgLl+#0uX@C0wRk!R>`}sUP zWjjoM`wM-d!f7sBA@6q!{%mN@r@VLI&mU(a+nomiTPCmcMX`XZIL`#+qvQ-^BH^B!}5dyqw(a2zudw26ZPev z2jM{H4Gnwlh1dEU(C$RR}?f8q<)p{ssPBg!5 zBT?1-KE2lXKF1mR8aeX*ru8*h)1%Nv;~?usjl(tVgR#P|9;o@9RAZ3bcCD1rxWLUO zv+!G=K=ci)!Ka#C!&v++Ug)gnKkoR6t>d(s)TO78{tv$f+VeLrJD~Yq7f^Hku<#Dc zbu$*x1eh>$GGL|ytEnY|-3|{N2%|kKyLaQvI`b{X$wzmIm+sfXv`1b{)sg;ad)p_&n;`f#HSzsuf<(3D9yzM_P615&n?)} zHV|t4uE{Kpg@M<=k(|z%$Im*B`&XV}gefwco}D~tvYOp)J{#2aMDWSHRoWAvno#Eg z_C||vRl+B_-^_?|1JJbI|5Q*POI5>V%{H;}qqix?2^HNtB|~Vf46Occ2HgD^rfINi zB3v7}o%?t;2V*L8APj3t}$fu7ZH-iy}`4Ffeto0nK{dJX3OYRC_K9Ibj9 zmK&#Hx3hCV@y2tHh3wxGe>%sv*nPi-Qw`I)1bdPA2L5eofhU&lkTZJgaK%5Z?psLL z7m={($Wp1Wi!h2UI?#<17vY?>rqb1OFy3kP4j$7!PgH|^$ePyD(o;()#oz>-zw4QBqdHi*2e%mOYMRt1%k#Cc+G-9`mN$ADX#%)Hz zez4a0=wT6FMr#6=u(1BLyKb#v>~pJASlPZOe^)Y*85kDg%i??5rQ2#kiQROd@ua%> zBZ6JVOB?+Kgq@X`{9*)s#}F#oUFLXvdp~-P#|Q}9aMR%(ILKm;$F83(6|TV!muwC7 z4?VNBL{M!4;kTS{@r-a`SNO9b-nc40iru_?07oQ8u)80t0rd+g{KN(wxh7awnp&PB z+|Z-in+MFQuVlS%f*PxLmvzxGHYoq|4PS71x)o9{?bFzvH{@+b(sRT2zOh&l&(4Na zLjN5`8p0f*qj!vWBOc!ynH3ZoRs-TCu6TiJU9MT@g*_Iw#cLz$VAH;htH!C#p{a!P zZlH8RBj>*Ov#>3CU)YNMcdU~gTdjfvAIn9L;`c!Mf+-Am&}J+4yZQ-hm~^3@|0k$^ z!M{~Y*j+c6HU4q}iO>0tva1UF7-1xip8pualD@F?(k56q{uCaaYt1jMcSK|Ju2@pf zSkgSPr1OcO-ZQP{P&q8o6q}Ysz^v6KlI8|nquQ~3Bi}$^jTF^WqGPLnOutbyCZy87 zA`@%!ZfDm+CvU|z2g70cgm7qee>3UF zzFf^&kUk0J^Jp<#OD&#(J7d) zh(7L2X8S$aT-RXJ-nU^33mJk;WHq zWYTvHl{GZPH&7!tn6I5<$_`Gpl)i!CP;N?EZ!&5-<{5Lp3GL(t%_E_7-b!CXP~)mT zk2HrEImZ_+RJP;%yN#rAhEKlzuyKN(cyekijQDktDZZC44v2tJ2Hfe4Mh0e`k!-g+ z(wsx=REgiMYAPKoRUJ`1<4bKM)rO z9ZkCUd~)h8-T!~T!0OLj{%1of-AfunKO2K%bxSk~ zABd~**iu`m^!ktf%f!Zm-3>2gjm z?Up{A`Uq_ox8Oshmyp~H2tzP;eKC@D0zHG{v=78g$mphp>kT#{^@LdIG!7;PtP$ju z;K!XOK-VTZw%h|bNshcR-6x{>hBTwn%?fM8U%Oh|r@8~ar+cso)4`!xZ>IE>svE_f zs!yQ23yj~Yf!jGFB=s1Yx$J;l173pur5us^`Zl=bOv52odr8Ob&nNCd{~O+dG?Gv_ zc+$iG)tJ|g7z1M-@2Jua>X_uI_~pU1IgP*aQGDoXcTVHNoI4Ff8VhtY^1%VCa3WKbpEPqzIa_4v0&{*N`T%)5mDiY8HIw@fAuZ0&$n1UV&8!3wRUkD3!7aaCM&EtVZ;`4QV_OwG3wp@Q0s1~8extfq@ zA0kQ13aViwUj)}1&g8z!o`L18ro4TZYfNbnrByEssKx0TQTfizjoR?osa(_ZSr2Lc zeKvj89jdpFV7~VdqaM)=k%d*f)uGvL>~_Y2ypbhW^+?=KpJA)CxgZYw{6^1y=n zU*RK<@it^LYu&&N8ef_BXFNvts6oDB3|zE*fjseOl@=U6^C3{ri};f(S=#t_g18;; z?QUA-v2`O`%Hh_H2-j#&Mbq5{(_6PvH7T|`wS#l3mqV?3J(X4m(h8!ZX>URL41B7` zqrv!p8p0b^r>F)LS%yP0?}TCdtVstXBGnb)`7FY=OjJI6Pe*sK&d3Ds3zqWj*gi=0 z4hub8c;MJIsOFK*S(1)tPi!TSHeqxQss|PCYkCJ}iPIn3;DWQUu(OYmwo8X5Qt|zu z+g})IN9Bv)_V`%Bnru+oHu$!q-1#jHZU$Zyio<7oTbZxkLtKmp&0UdnN~bk$57_*T zvj`*ZV?^XB&ESw+!f69Bsm2vJnb!nJH}Y4^8YrXiFevSS#0wBK#Yj7}d^?g?1snZO ztp8C@+;4J(H8hXGnKBv@i}pf;Ya1}}=}jb#lCul9qtene7jli;MAF2x#=@**j#IVGTsIXe;;Az@zK=ndgzL|P-oOF!Usq!KEONVOv`;J#y2}1VN zW=k&aWYmYOZG(7J8A?LxdCF+!BY9Tz4sgZIOE!zM!Es{tOm}g*XH6tO2PfaT2&y%R zj_Zl4j;g&{582_~oUp%O;4b4TELGRDdZr1ix)Beg3E<*My7!#)=Y#UL#5pfv%)Dyc zz57u7GsHpTSvU(R?-Ar}z;VX}X7V`#D(_8U#Ano(E0A;}Hgdj5?>!>ArWHWW?1ebU zF9^uTs@f$jb{%O&0_I`S2S_W6ALXlI-^f|0atWndPVXwwsyXqhj77?isAiKf^v-VV z(5yZjJCX3P=is!N$V`7l>7OHVr>mCBH54Ga<+sA82tD)Da*{C>u+``q2d@T=v!!(E6 z=sajG+l%+HUFLI@H)xfHC2vRPV2y;sGJW}JK?|(l&BkWfJp2&t0h&lUlJWk&_u%vO zQ0BhF4U~82ec6gNvf5Aau-nA#wgC2KTB)D z#KDG=>YeVZYzEb1_rtj`ds$;*Ltgt41L`w8)l$opFC^>(U0$s9@l3@n**}#pkd#Fz ze~9EiL20g-gd){v8s~^3(Dnifn=N-S*KNI4eU{&UYc}*L21z{Z4#OTAq>bgUr40ikH|b@AlC0WC0^jPWovk?&`l@8&or0 zc|7L)s3w2kC5A1Heos2ifN#kTBpo`Oveqsd(hgLY50EsS$~}2(PO3~0{Z>4uKDf^m{>9Jg%uf8b6v;b_X4Xp~Y9f=RI=P~AyM2OuMOE)DX*62W zSOe)8PCvtcw3avNJ@QNz^260hVC6=8v{Dw;;mR@Q_^vY$-e{FyB+t!yZmCXL#{YYp zxSZ-}0yNn50LX6wWk#@MiH7B__W`Owq&mUZ0S(AEMFQ0=PJ9_Byk;N5rQ0YkA`Dfv zqVP}E6ZN6WSv2H>aAkgxQ2)OW?!W_UG*cP~2H%QSy@uqk3f#M2*Ak~f?~?f#*=-6_ z+V1lW(&co1Dtjq#aMu(p{EBo9N+$}+VUWBuy0Ae|@J|P$^X;L0(hi{U#@}-W!NbI6 z(5x#ChUZ2Q&*Ci3>Ns)so&*KAr8^xSb7H zNw;%E%>Mhw(9(_bpKEwOoH*T$@ojB7k{J8*S3i{g>y!H#ogiN z+$wXh=6O>`K6bx8kN?>acV#(o`XklPJ>EUS<%15eIZa!^-bHPtOXJ64^2WuYYcp#e z-C-aP(HR2Yvl8Km))V{lS+tMV7-{a_U8-Y^KR*w{Pu{}zw2#B)N+UV+c_uD$Y04W6 z%7wzWbD$u81+<)WPn0fOC>M^|%va4`jooH%z>n#vVx+GZfAhQ%q`2oo*~me1&*f?| z$jeSkLZPClrNKW0AK^o{@!>8=MTGC_93+d^UhT=ye7n=}ZXSPwSB8 zOVHelg^eo3ucUgU`!?WnG4m(! zNG|4-MT@k$%jFrd7t)8nN81sP*`K#F;7$;OlPwU(*L%*gCd`x}=Z3@0`{!Wyh~dyF z>p6mTEYw`E0R0_qWA-F>96KlzX3xAX>VEr$c8Mdk;e+epvm>;B#7Hk0#;3^&-Ds%4 zI)v$cpDbfznu-Cd*TRjwBwp)Sh@G;gqjjGU=85L=OQ0d77!)xz9{ap^ z;EIej%ay@T1us@8ipgl#RUa^c_8*qKf zU;G~AAy4<}mRlJTgvpJ^aMzI+QPXiGu(VuF;L&#cRmn~?6dlpT>ZFL7Qj?Dw@Q6`; z!!+H!KtEIOh23)}Nsr_({@9=ZEPBr2G}rK^(L1cax+eE(agouu!Q$Q>_??!)y!KWL z`PncX9HN#8N4GAx;q4+ASacf?1lWsYrw}lC-(TaMdKEt9xgd=*x;i|-4ufOS#M?%O zxSho5-n$shpD=vUluuc7TubkTQ_XbvE7E25sW-sTY?H>vBMj3X9Kt=*vkCL@yz&U7c^dC zEJyDgA>IqmoKf;Dw-!xBR_!=<8rJp2b@W&>00 zDaR9RNQoV%_ky^>nW*2=k~iJ?fc;H9t)=${sspXpprd%mVX$1P7s|&N=Hk+_-QYCy zj;b*L>&Eg_HQN8pE&(h;b@`AI@_2!p(8kzU(i{ji#|{52(PTDS1Pk_f!2G8Z(b)An zYjxdUK6qispVj@0nAnFmikK{pzAMA-A?EV<&H=c8aw2m2x+YssSB7j2hdARkGzM2# z`7^G@iA^$fm1%WIr71$8qa4gpaG_NvaRHH>!!6BkZ_vD96M?F}$}c zbPBYR6FVm$-9MI_rsC~-qlL%fBs9Kx7Hc_Pg1eo!Yu<0MYNa{ZtyINPHZU$Ui@ zTpLnP{z}@$zQ*iC$9r$Yii~Q!dEpYUDr+gC&eFcG-{yhBYbTG|yh7Jp{!aM>D_eAh z5!gV|*q}~^F}|HJg|{l*ir1~faAnp+a8B@7wJgo*4@1?{)C=O5BY8{R6q+A8`?^yB zP;bNEn`vk}gfi4lL)oB*dOT!uGW?|HZi-I61Bb0qxa8d!N$)Ki>Et8TF^*kdDJJ|p zAXgZ=$$>>9faVkr3_OeUdR7(~0fqJRfa+9EZTyJvGm`fxH36!9=DMxp{t>;9%WY-+eN4v^q85>n!K)|e%EYdQc&Nv>ccmwA>tR|Zmm`T-F(RqN6 z1NLL)!i%ukKM?DFj3wNvDF}zKWML1k#$~G6T7EElD|`8L81b{6T$Yv%Js0JA{&MU9 z^m-n%b1?2niUq|_7TOYRt=tzv@gytT3m(Zv>?wW!bnCkhZi?2h*&d;wt*Bqc+ZNAh zoqMrX8(_SiC%xDz{GvW+343v>aDd${jWNJjPZG|FwISPp=9_pX35WK44yB`mab0N$ z#0Q=ATyeiIr+Jg>l6GVL!MoXC*YWtF;|0Q_7eKvC<@+-4;q)T!`A{dEyI6dwY(y+ zGH@rrhfGcDUe~}^=M;>rHJek7VYGEU>GQx%@jUO85ez>34eAfk=M78jVav$VA|v6c z_S>w3;^%_9*y{R6F}6hk_S)P79PiD>#Rs0jF`Icj<=#X3`!t2=5ZH1Hj?O!$8CUoL zNb?B7Ibr0~P(B~-r==RioHR$Jq0n_?Z-^?3WdrOS1@X7|XVpxozE3lXfe0@bIU(o* z5D$Zq?m_C4dz!s9|H8eHP!Sr?4rz|r$?sS3nDu77e503CxT!SQb!#KBpaGqU{FC-Y zJunJug;=A)W~GZ3P56LcTO0=+zwg-R+E+ku<_|g+a`FNe$k{$rOh!yozsiGVxGl4G-}%0gC@UL9>Sj za>UuM?9&HpuIYG3OFRO_m3p#Y=07@Kl<$W)3zLN%-u+0ePr0SkU~4 z_76OzmEJZqr0)`b{Irmybp`c~q#EM%T3Aze5NWvXPp9#|@j)3ESPl<00`9G^YNLm#9OOl~c#A8-5u^+0Q zuFpRaCZE;*q1=Rcp-PuJ|DbbCI`#qmpn+JBz8&;v{YE5i_9Q&j^v_R+e(6nkXu5^! z5Ao02kaU-=s%fNq)!Zt4LTBY2$Jf`sE4ad$HPbx#)ImMm-Ra{s#B!?(a_jP^>fvKXPv4`x;I#!=Y=7YZP?CQ z)8u&L8vNaXyVx$kT@r3-Nh`zQYky$E&sOaFlR4t_B1d|bFJ4gn%ENn{iBo3Fj$I7k zz{fD8S|y$w&S@NBN>Vf4xVKi2?v>;th)Zg5njcB8fp@*_Fw7`UOI*TL4HadZ2(mcL zpjw4EG!o#%{T_T|tv5)02Uwf~#377uf;903G<=Z=R5w+c@A?2=k@T=rWX3H<$A}Jg z?OE+}lYz82BQE5AllP0_l3fb3;YV>ho;~RV#<=fhsnI8@aANcn6WDvxS&+A4|M|Sb zJgXF@yp~3nt>0yiW}6IXyQhwXgDU&@Xc}&a%A^ z$MZaSo%DUs)2ur8uQXP@UR4`2=sUi#1$E?&DLOpgpf|5kd|m`pSjZuJnBp)E^(9^N zFLBPhE>ImyqcPR0^mK36kJ?q=Z;>yYAjl`;%WOkX-b3ji;v`1gA?W`2;XOxz_?xZo z<*nT~u)A#Fby{3}(3W+zUs8pSwnyCs%@dt_cL8eVgkZn)1cmv;-8JBHJ1g9}un6b= zIE{qoj5rX2mQ>^2IyZ-9<$>hAhT!RxAJDw;n})n+F3qWWFMMjWo$ZgBDW>)+2C5Cf z1rvnwVx)(J!tj7jK1%CCLFPi18xzm2p7rC(XDBRESU9vxjVjNt^nKEzEb?Fdk@^!i z4$OyU83w3)9`U9iTw=X-TS8K=o{T&$BQ7Al_#W3hJ1_2EAJ3@$*u%fn#s_O`0>6QmuhdTG|cUM!ZL zd#7b$yD&edNLN zn5wJXPNa_}X_bxzla+5N*DhmAI}QQ;AwiICw+~1U&}%Nj!_=;H_Q5UmFR_7#-VuU0 z6lq+sGIt={nw6rG^YQ%Vbw_kIPR-6N5-C;10` z#mC~vVl(p4^jt--EwB0P8t%G(QDGe@uk>Nk5m0(J$aNMF7b#!H+Xb9Z{sPY3bP{=6 zgR!+)2vFX~^WB_f-H(kWX;(b|%@0V6YLqv+vUodg*nFDw=^^oP6^D0=x4`27JEc=J z%47LWwgbxOp#5rE=XBt%mgW#LW;e#BhNrRrx5eUwVHWt0JjX0C76@N7q{%h5EoqPP zpO!!x)$^}zq&RWEfL)l`M^OKY*+xx(JT%SMG^7jyu77Ao9=SH(o241kF-%3 zh8p;AuR+dIaVKGc@{*Vs{vMUiAU=V_XWdjLg&Ra0xHvcr$J{R!EkhbJt6B>%CEbiu ztsu>f>OD>wsi6MH=RaJ4e5%Sy;a>PlBs~Z*5J>v&3ax|JN8&{6>vjr>`&G^-NLvZQ z1@u_lT&w&LWrSjp4#P8>b15TmW0(!$8}13pTrgu-b-A}@ zU{&oD`u_mR#sp<7SdjS{eQ#XXQr<&3f+>0xY)7hJVV2#L#(E==$0lF0OXWfEZTEMW zzGOX8U$A)(*Pzln$&K&fxzrx$yEp+@ z%l_D)XdGK^vzWDAVxY1&KEr1Pl13uGuFJ~?+2VUkhD!TWW+9ZWBCSI_JC;dPdqKPn zGPg|hzRCFV4{hLkOFhbo#&LVAV_MRt8kK)i&PjQJ5uYj!k`C4Y@hMWi!Fq{IdUn;C z{FyH(Z??x#%akVEJMkLmxH~9cR;2+b*8%JtrRGk%DZd3jow5uqUsf0OJhp)hq5WW^ zUgc(Y>@Ub)0%f9tvS_g=S)czJzlywtXH{mfV4)9^rxPk?qVd!EJB%PL?uOO2t$|_M z3qW3q|6X)}5xyejMw~n>#I`uCxX*)loe^h>9gAbA7wS;HnMm(d0|={-ts0;*I8?d0 z-z|N4@M$@o$SOzTJWje62ktTvw+i$m@vEdvig08T`da+UQ z%|y3)BeYi+TC*0*>cPu^wb18KePLg=RAbw)5ij;EN2hbKVsYn6@EBvvx9RIkmlGl2 zL+iOlbo+!U$2I)cv@T-s+CrG4oyf-ZnMdEj&|2@0w0FL9g68M!I&wqS8yx#$3D5a& zxV#Z`2uxcU$(BYnxTeho_R!}VtUF&D-W+u0`-gUAN$o~r)|4RZ<=#b3to;zi9&XP^ zy*e#CFZf`pcc!pyxC@Rv-otbkbd&$}sKsZPjm0bV_KU|~6S3l`Dfo_zWDf>f%E0yO zggSR2wPTs+|8oZ9_S-22%+tuw&&x1uLNz|Vg|0jd*YVQZ>T=!Dj-tuw?A%#b^pHLa z$M}`w*U6XI+9?frKYu;GVg>NV<2SJJE!M&0`M3GsdXf0hXRz$uw=v}Wt%O&NI`iCq z4e`u;H(HB#Qq1^~3un@Q!=R^Ovd6+JcsVZ!eK*y`kNhD!7*5+cmsRJ>s?TC)(pPcM zbbF~gteI5faQ0Xj^o{-iZ_m)#GS4&u`;^c*VY}h-!`5=-tQMl~?g?-)EL=NbOLv}Z zw?(d)_E_x6;<;l)`a$fTQ9q{`-N(( z<$5x-Ais~xXd-xVby<+v7|DkM@p!N0C`W##L1?)FTJ zTR&z{-u)InCElj*IG$kWEK`)1W{AIi??Rsko8dX_=TCp9nqDD0Ih)AMF@MlE-&F z!FVZN+<@uJ6Tl>~jl5@c9^9w;${Wd7)ce86&|cW{W^Y(Hei|NKJsvV$Hp2Yw^sG?N zi`aJf80qx;A>3XO3oAEA!Ivk8*oqY>=o&Hjb^;u+evV}p$(T5?38&fv-o#bwzulhi zeqD?UQmSKt^8=WqzmxT`xXDh}J_Xe4Vm9k3LxyYcomZN6Og|54u)H&>y1Gj_%9LvZ z!FS0CFsHLNo_jd(Lwg!%FTLFWQ)r#&n}a#9s_jrnES`YbOQOW1tUf$wa(7x17dD_S>Ao zU-+eB@xKPV=go`adX4%rpt+&AG~xn2$|^+vyaQ~0;9%CJeRtY_wHs!Mckp1&DYoCe zsW39`$ZT?&$rhJai0w&vpzvhhCvRyQxCl#5eiP2YVe;_Rt+M!Eq^P#d3C>=lH8~*` za+^(SN#g_&buAz;yq~yB-whtS9icI~^Pa8n?u_ewzG#~bZY>E{MYY3&A%2Y^R4e+5 z@?a0?)7%mgCnmrMO>4N@_9)g|X`t?%rT(hJsh&hz>z$C^@RevWt>D)BDV)ZQHOTWu1N$WT`dA=4 z^!*QnEE2Axdd%p5KORGEJ)eF z=`-NI6bG(Ma_NQHho6k5GN{_TxrQ zjXD6+^AK}@gqwuHrls4g!E{y2502{Ar#T`68;Lq7xq#B2F&q74IfC8kS zbA#n)v`@D^Kv8rAgAUlpKEFD^__}R)Wob=Lyd~q;Yy!rcfmdk}*t&o$t9}WGMD69> zQp~WjmJdK;45qEfz=}6_#Co4aVqH{y-fASRWl8_7AsoX+XCDAz3ct3mk^F=!Vfn9S zaCqbr@ZRMKv)-l%>R(X&cNm6nlev!a;vIW@ee5r57dwL$TX}%OWB)&Ip**bwozGl_ zlK!q}uyZo+TgONy7C*#T`u=;}`OVbJg;l(CcghdNMVd{2Yx3W;7<) zdQ%d%A9qJ_3_O2j%uj^w2jX}3Uk4``ZHZ!=n~przZ6NEk@f#3_0dW9((lQ>8SWo9B zcjiHX^EjICy$S*E)*XOj03-= zH({_#2AtS`QCK;g#_!$lz74Fh}eZj|Hn{q$n9dxEn4I0l0sy2{#1%pQF$*Gm= z(8ZTcGz53t`oQK;+b5)}IeLR!v8?XC4sgkya@uDE63dQ-mORyCe-{l4latkEX|sGjlQ zJPTIe)R~bklceXg?dxhJaf2k@Ry{@QD`+43#M@$HNf@KKW}fL8)Z3B#_KL=`a(5W~ zOu5PGMefAkGkZ#hziH4Q?-HXv1awS?^6+%_I;I@`7uVvqqdE9)+RE=vSteFqEJwm? z_P3q8oFDf!SB>8w-Apx>LTOWRcr)zT-k5*yK2}qtJq=}rx{3q%(&SILGkqh-E_#ID zz2HepD|}hhOzf^TADWzYQST2lH?T%GK`QJC3$P{Kld7;y#%&ymhjk;+wGpjhb}}RE z`wO9EH=*_I&Wgk6zh_VzOW*O|d5@Df9Dz9lno8165Heh=RX9VMhIRYm%Bd!$i+(F; zPHW5Ce{P7Cr6HPBi)@%-cAp(H*pxfi#hz8X8Hvkwr6TdHRCPc#>G`&D2oCO5U1@AV zy@~W*w5Epk{*3#|u6;U5`&R4GIU3<=JkcS`P()9Dt@$1DQpCPBlZ17Q##cMfbSq4X zSgCqdP%j9Fl0!K3aaX=@{4>?#TBW&Ce{BKcCsofv>B(7lNHI~o)c&FLd z37c}+z&&ozyKgLcmf@7aG(h|K4sgF|B3m<1D!!sU!o%TdD6QRXQUFt3y>V5qr6OVG z3lGxWxXr6ChJOCRSeTcf-bT8=T+$+-w58Io&i5CAakJV$wVPh0Ta0^tw!3>&l(HTnP8o9msLAw4|%S%PV2wrpmggyuQrl&3nz_?cIVq;QHx!9$B1ZS zT_7!?{hWQovs>Ot+S6oP6(+e??$C@ax(?4R4R}vF)98y?TYhj}3@W{wCFY(O(eg6& z-+J*aH36yqpfs>O=+;~-gKsy1d7c5F?px`cMtw6dWmmM=(X4_M-I^o`@0qGG(r%=4 z3UI-#HF)TIfVTKGgJ-#QgAH-bm;5Q zj?;a!b(crLIX!#6x6T~=*L5pg{?nKIMuH?wLwn_(0arI$@I8x+N1 zbtg9l<=4U+Jz{+x)CR*J@~`Z~{@(a=<58UV$ds^C@j!1zn4`%JzQ>jow&1rf zdJ^YmXiMX+1Nm7P`rT4?7*+>y`Uli;oC~B0M6Z`gP}Y1l|J>w@;(N`W(w1;qho`f5w$pF5=b>e z-m<&r_|N)O<7H?X_(j{rG>I@ZhY`n$rF{(;y$|GPpTk4a9W7oS#S^o1h4Knr+UJu; z-6sYNHs>oGtdV*XGY2~Gd#ztW1jljf3iZxpKfCk{o*FO;vv#qrNB0st(5Ly9b@8PbC*a# z8VM<9sM6KBbk0&Q_p=awxG7N9f*X3ItML=tPd$T&v%Vr_Hwu#=i_XV*Up`$+^(9C* zLPSLz5>7LV;yQ5Fc(ImrC?mZFNnf^Ww`?KqDsKohUfBBUV<3I2z4)z^QT++Ur>pL{ zBJqUsn!>33B&hiqS8-5ze%x}#NlQH?==z27u4*2YzaSog!@4GX>a#TH7-I=7?`~8c z4JZd8oNt8+qfAGHAaSNq)b4m5-1HWQs3{V{i+dH%7jI?bKNaYAjii^5I9NO? znFCEfIB`+2NhqBn4%g;X2jnG9RJ~DNT&n6p>JJ$X@8{d$xN{qkxPy_Ofp^R2v3;K| ziG4R)N{g4yNMj>sH`yzuy#EZXcFf`-8gs}q-#}c^Uu8zLJ|G@`2PPu%5F0Q!I+rpn zNtz6*pRmxX_c30&6xY33iOOdXCbD{z*OER4`dv*}Y)>E$i=?eYwMF6ZCo_&weju}|RC~MGAe+%M3Y}QdC!|kRf2%x0<#jSHWhq#BPXeWZiT5ab(&vO@K%NswyQ_Q!s8)pH zHmg<>x=JT?LOB)wZ_z`WEu|@HFC^1yz09Gj1IWYC4$w-Tx1!W7UvYEkC4v>~w4>Y!7ZmAZ?KZ44=Y@WtbE~z64-??$K&B{x$i>9B@ zGl!xz=a&MBE$` z0F>=&DFcMS^}3QUSgZ0zs#mtj!v_c_=?ofUNtq%{@M?iXzwKNLWkk*t3 z3QZ|z(ZdIuCPF%$3rDq#U)!t~r1?PQ3(9-`V5Xqvi@dGMd@(R>o=~1wasRBu5TMKn zDRbfE_Y~$y-4^?_Hdpf)Wf8)xRw7W3Dh;QlJe?8dfbzDamzl~A6WZHg&fgU>sqb&} zD($PZ36eJfl?jjrpfTtLf9o|s@;qGWqWmjURd%iNI3&E{ii_60`2~bG=GSbTgB4mz(fO&)m5FroF69y9WICyKrpfg4nat36}k&btd61ywli?tkamaY~K)j z4wF6Q{Lb&e)_)ylm6oB|qasMyxeGgu+eVw`4dR`>=zIE((eST$8>9Dv9Z#RL@b1l} zeprUMHK4oL@pL-Y@@voI5=P=_ucPSrp@ez*(X#_zzcRgyhT81&JGIV*MUK-}&@a@f!p@OEVq*w;A%_w6=9^u0v-?sy({ZQ70d4!({12U&9MTFTj))WE*9 z{`APRwa~`VSn~0^U>ogG+V@5z9^W(p8|J+bKkKYP@2@r4tpWL{msSI=FT9MS?K|=% z=HyZIR^b8j#eDnorRY1`RD7|X&#q+ZbC-;OvPw!>exqpZn&JpZTu~}c zgwpq+k7$qRfu{WJ#AmqK?i9X?k3{ox4_W;2>e8V9RAf`miY_+g^luZizi#>Q%h7k? z(K9Wqx;YHDzxaoA&-_G^9uM+gL}#ZQg=?GZv-`%r{CL=l+;__ipj8J4xb$VcsJycr zEOTSz7t4oub=i09OwY+=ULMB}83chzn^CC7YpldchxewVEIIY@0VR!l0K zCF`^}iUTv(Vi#6hUfH3MYcsX%T+<6|@S^Li{R`&2$zGy<@7U_+T;=Wyo(C!SyJ3Dr>+o(Q=pR+sK($HG5| ziMs2g#oaKb#-D0QT{B_&OzudWp`-`gD*#DY(FT-@m7R4n8?;c#~D5Sd&$EF?o0q z?ist1?$w|6{@n`sjm*)o?=aZ~8VaY^w7%wp72h<)7_4_W^1#b>Fn!-ZNLt*IcY&RV zy%wO}*6TcJ)MKFf7vXK6iK7==%CcEo;MMMUh&giuyII-ups-9vb1eJbXux!L%)(f0 z3$E%4d#_+taU2$~3-I6bb8OfgOJ2)oj`rZkJ9u^JJbAz6HW)V7md{;b21AEY4*9L0 z%x~OO(!5snFI^M5?K9^=^JlZG+mDJdCU>EKd?3DFJ4mQ|r_Yv8{;Mx7clYNm7h>^s zT)3RMVX9pDsWsc&S%-CRWXdbvep0w0*Qag5&mJy(i)}j`YZoL4Pe6@5^)DMb--X}q z@Dk>Y(W;-pt|Wh$zW+AX*E=l|SNLdZ4fraqE`5(c&lpU37>v2=nxP!O84-KA_ zW54Uhl+%`bs@lj&%M-~X%f;;ueyFg)e`GRmIUt0M>Rk(|CqUuX#%b|_Y9Iez>A;U( zOcC?O^^tUsk}!<7f3A<#s~7ODO_;nnw-sOcaSx8JSE-p5Uzh#9ITBhq=b(d8J?THv z23EZb1^PX#vC)!;IeZiKn$?%H;^=wxs9(4`XA(|1UnJfyU5oAMxs8I~-MCm?OHO{= zL>8Eb$ZyViO#NPbY_3?JW+)F|4`iM*{?k(Jz}rdhU{-83>6))A8x|}(N%Ja8j!W&E zfDUr{DI=M%(wEH}R|rRnPGOouDBfxR9}FAvkZI^VxMwL-VB+&m^3>P}jC^z%)*VV0 zenE4%*+T(hGU*!ct(ayyOy=YVi&2g}#H{$Onr8=QE8O8m-aSNJ8ykiDeC~w1_$DwH zIuv(>r)jG|GyRS*d&nf!5zzg^UjG`>-)p*17_DmhOUF2j{B5Y}oU0!42+9#lD_>B( z{6&}Z+vOea2GZipKzvZkE!U_w$9+RGpzv0Es*eTueE%4}&?D1>YDOD){1dJ&iQ*Gq z^rUA7U091Dtzf%eH&`%CmuFsnfP{@|z9n57>TaJb_bfXFwS3;f#24qWhxvL?Jx+au zZA=I8)kg<&g*CCc@o4tDhIC75#fhWv(%s&W+CCN3XMfUk!;O7MV#%?NqHe9{I5p2j z2F-hdI)~fI-rwr*5&hfC>8E!Z97*$yvBH7R4(uoktyj=afEGoV_~x|7GreA!6)4j#d$<_p+Aa0ePp-^1cgM#9ytHdOQa@~mbP zBODg0J_xf2^OCT@q7Ki~dBYm!f55rRBVd!aH}|qz$TLjrI7BXX-69%mo6-D@)C3-z$!d4emzB3aiX!cOMt#PQ zZyX_rC)m@}MbNvPo`G3#A(!S54;t^zC5&UAd}}K%mV~dGUu6^Vi`!l}zVWsgU&BVu zS+yUoZD}kgUo3~4g9-bHpRx;TOX6#sP<^{>-F_dAbh;_7hwE~MnU%Nw@kf0Ne)z4Y z-0L4qeBB&_L(L`qo?Kop9N$b{0>{H<@Q)*!Fv4P>o@ZlP@5UJ+0eF00oH&*$pxv88 zu)OXD=rq(LK}yFfo;T+nP?##I zwpr$tY1(E>oq7Lb9VA^l_I&t7m~@L0R3{?y1Fqsp7%+? zF|{Z0ekZ+&hn>-P;6AoAgaR{(kIvpuPh>AqmblyNpV2(D-3= zc{Ch(rXzpfxC+ltZNWyE#U>U1(h#@8v$L%rt(}E@ns)*_WG#i318VY_342t3u+yEj zK-etr^syptd#L?xmW8A>WNEe?I(!-nFZZ}&i=j@u-P>cZ>VpMNev*QpSP`3kDvY$1 zv8;CFJX`4L&gM=Sg>0%f9)5EKFCU50`W)SbcY7Q2pLJ_-rGxt1cI8y_g6@mOod_3Y zv+D4HnL6^rXgB_k_J&rtO4zFy?Em)P9JfRC>GNkijUHE#2|t_JVB-9>rZ4oG9h z2+x4(SQ<=k#^nUsN8>_c;s!s(wMf{LOWFgyIv+=+MG5m2j%yDZnaTZbBcQ{Pf3Ufg z4-!{Gcgwv{bJiL7@#7v&EyyDtuz??@dqCAkMC&p^SSYBcSbbf0eyVGsX4<(7NtnZw z)>wPk8LvckRdWZO#^~UMpBdEufALJGbl9-qw|M;6Mh=~y4GQzNnj@i?77X*2>5~?2#|h(+^b=h8Var_8 zdx1`-n^ZW4lR8pg7hv7yu`K597F77q^S@O{bBcHR#N(V*cC6otv$$#U294r`_9cbj zo=oSmmKUQ(FtA+`6lRGRNhs3ma?$T3t#dXw1zs&zVzr~CFy+6^+T;bBp^fQP(A&~i zTk+-1mvD;kpKUeG+%B&D+UO z+G!M)jIO)D^J}9kSWIWIQY~Tkf5XAw>v=Be7EOAcV_-Y{0+8NjquS9Px|RX$s$Rn~O*xvDYuOH5@EZN||(G=T%o z9%!{@h0;Qdya+t0(B*52O)%8Y6-iqm=>&d^@^s=TL3M(r|0v5I_5r4!iIaAN6UE9; zsTgo#1@3pdtv*+A7W3IP4czuOl~hB@+mVMT5>yXJSfFrMT3LC&J=pR1&2oC;H^#MoB&6t*M!!AkjFB=X9*rbuF~{h z6XmVMdRmi{G-@&@-$eHx!GG4NM%v&QPJXl#$)j+?DYZCxD9@}CAD(J96-Z-N;Rk7S z$yQfqZ&#&(En7v;%2bDX)q5$f$1cr>(0DcBb7vhbz-SsFgD z=VR);QhJ(Q&N4x&ZTxq%jPyk_^1N}Z@(U2}B@^~FQ=Sb}4Fu1{ zsjz%`J!1?GsiUrw&dS-(hAij?en}@)-(kh;N33Cm3v_QBr~OYi7@RJTLaH&P#o=M~ zhWyg0KSI^*g`f5wuKDXodtDVCA^Aw!h_uUmQ20c;gr~KOp&pH)9yZnP8@!je#+a+| zpnFi-Q7YX`nw7M29_!Tgd{sS?-@!u$`rLEIL?B#Lm@Ve4v<6k{$}3nMuLcwKHUVi( zI=|^0P(1;0BO^_Snl<6jzVsUI-gShz92&1)BiPhSsQS7JLjxRfsP8iJwGl`-MPAlN z_`+!QicjJ9MDr%}m=(-ts~GSJH^DE-`~kw>`uH7=U)} z_f$rJm7jCq`2M|uK2Lac>4cfD(==O~Q_atJkxPI7fMurfU~&2)=>!wXHC{6E^@4o0 zypkD&4GVr~?c9cnv8|#M25_orq}~(c-%yRC(h}oqB(vMj{^D2Je=yEKizx%baAe00;vIp`Bn)tZ0_>s>1sn={fQ<%AN`aaTw=IFNRo*+!5xtfE7d5|>vwbIx~niA-9 zRh~lLWhE+JGW)L^UXJPm@y9oS(odwfkh}~dZ;voJ7wXompgOpV;XOORXOCL5fVB-ul$|Ex3K+ggTT7#<{%PmiC#o`kM8uF6#-Gzaw z3*{w9v)pCm@vHKJVMAw-jvU66mLR_(taly4s9(C0aJdRwsdtsGL)Wy^ShwbQ85HIO z&X?Mv%Af+SaXBE`SA4OW0bZTmAl<&2RG8wI3)K^-{ zw{XXk@d_h=dV@I92A6-kNb_YR45!v*H1?qSn!KN&dh=BNB-Ag5bWJ40$9l?l<4U1p zsyjyh*$x(Wjkt5>Jo2{Pu=rXi)qssScECmcp)fNzWDBhL++Isw2&v|oHSO6EZvT!s zzLP+_BK-L+9FS#@d(Kmr+Yavqq+?-BotgM*-5od!v8?^eDD3s~08-8(GEDYU=4>jP z1WeMB76j6Xg8U5b9me>*HfhYb=OL;A9hBYg3d;UOST9e0EF}Xsr0Obv56b_1H)~Ix zq$ytfwu8N$G#3o}l;Egd)ui*`>9CH@#kuZ(nm*G^s+^5_7uo6!$`2{tLE=Q5wWa`T z%)PC-wj~yx{I?x`zi%K(i!fC$)`71yaS3;@<-jdiQS&g6_EEZnDKAPILj3&Noh`N; zgU2F%K+~1ils|)v+KycH%CS_YcpudKdTuyKV>JiKL*wTr%YbwZdf!3OG%AAhb&wio zPWn~N8JFEhDLn&2B7*T@^_@yLOP7PUU}|1AQYHc;`e&iSI_gPAI-I>;`M*WKEb)~kORR)sZ;9WJr3WqeLTNIa^O!3i^O2<%7QucF2`ERB&(s28^Y_D%O zIkHa-wyA$fkWY}x4-B8Fr}TzUm}+%=4I?hmD$WUWSW@MSl_%@@+l%zablCBvg6&FP zuDA}SobQVrinMHZOd5u_ora$4TWA8WII`@F>$GMtlXB@RDvx2L?|?EiW^qSP{(c_` z+G)d-zh>m?RDMe5$2yQV8xAVhBD@09x2g^Wd88`sFyWOiY=}4w1F|+&Vb-6w)y0l# z_QE3=1!Xm$SsRJU)6r+)<*W!gqreU*gA@lkWI>HkQ&3qX^$4$Hr$x$21--YzA9`l5 zhP>Bm7ZL{nWpWzI$Hb(f&mybTmnRSJ3jx;)#M_C}c=JWE_@IRjDj!`ov!bK*i&c|2VM)YtP; zc@ZZ+3Y0Me;XaaA6kBVQ1Nl?5ZZllIqH`#G`_$x1TNk72zsKTA*EN_K_eA@1Ni~)* zXC?Z0MRD_UZ@}xA3*6t{iw9nLuYKTl88*1z#r*L0SZ7HW{(NLlX=fD0^SeyvfA*42 zj!uTf-NLYblofkvc3r!>*IHTIdOZx^+XFrfF~Da2(_r|MK&%|y1(vq%$8TSL0$tnJ zltBY3pv|l_7}D(s{1rE$;n(*3WEAbU^zFOwaO)2CaV~tv8eOnFX^QJ-HsK?i?9yBv zrO&rCe4zC`wSgU79*pllXmC!mNKAi3=R%Kri|KPC+4#dJAlK80_5wHo^mADMAP%3L zr}bWWTk*!@{(NfT2*jLqFxKUXrjLA$Ek4)bXHPbfJ$hV2>qX7*dHed@!;j8wy19~t zW-b(TjiM~^E_OQq2zFoB6J@XV%65;tg8K7dZ43^d6pl|9>C0=`d922T3s|vhH&%aC zgKg_MgFUk_fxCT9(h(I#aM67;d^H}UeKagxZtv*H7u=b~50nd>RI**%-khPi8?}~C zDyh$()eeC18Vx=ehInbWsp{Bg+H?f$4 zTj23^6P8vS5l0%H6p=AC<(7#D;N$nLsIiLz^Sl1cmew4svG@ny9v{%wT7Ls<-1Wq= z&R21DwMt0q_YYdU{0C0gOnKpS3!d6(6aD)N(w~>{KfWLO&eG#{XZqrwNzq96Dx2u)*b=Q__Bf26e@sjF z#yah8$9CPn$F3Lk=HKSEm1_PqcZ=xx)SWo4ub)_KA1t=&JIM!MqcC90aaP$l6du|& zjN`Ei=Pv>twY+Hrp5@n;x%IUB}Ley4GBdWW5B ze_|Ue5ytI2ioIO-!q(*nHG3>g<&MqY@y5J?a^>r9EFASd;tdctU=c=h&miVEOO*D4wv#OL*?uj`SMX zn7bbO9t2sm+L~9l?JQM)n;vQoWA55XkEH?Fq}pCSV0dF_uwty7a&(BS=AH(<`?aQ? z(dDVGD}>JsZ|T*h16wk>F8@@gA&y^3=dw7Pz^0cGZ0Ym*YE1a3*EfLb8o#yp4~CpL zEGF)*#r2FB_dh!k?mT}f%El#$lYPcjz3;6P73h9npV#Sc!smK4l17`_LYs;d+}%ru zuxK_TJkTDfy@ti}k8r+bb^f-W3lFN@QquMEtMnYL!;E>j^I=1&u`}cJSx93py6oE^ z({20m+T(!HnBmClm+^|nd`=i8CuRh|#W&r#UZD<8a*2W9PVMEBdhXE8>>@1t7cF$H zW>VHP0Qc@2EqhOKl+@eUrlO^s_Gm2jzMaloQtYICD>rW4<`$0jv6XXf_L1vnCcyhl z9e#Dj3UR4C3fZN2Tox9~YTGY^RM$#Pzwi3ea!w&W8B-0L-*CmLg|{^{=JLP*Pu4Oz zS{yrTM$dsa6gJBcZQgilhB>bjbYJYfTmsMcZIRj5wb?#=h)ueTVdpn4;dONL;nB{= z5K8O)Qm1#4E5trQbOLFm%ewq%8nGh zgM|hkfqD{(ZTD+?rZ&Tg$LP7-e~+<|;VST6byQQgu{W#tu>sd9&ysY1&?s~xuRC=+ zTj6BG2f=7OV6_;%=@}lvB&4~+VL4ym!h~~J!rH>;R&liL|6cJX)gFTej2Cp@a^XdH z`RmjcINtaM&YBVe{xOc6=2%93?7^dFHou@Y5sAoCK8BOz_R;JZV)$; zddz``>6j|K2UVYyn}e|Koh`h!wI1Iyrk||+>Xb-uI0xOA(OxGxi}32Rnn<-nICcv) z(@F{d!?4)aj_a*H34=`!prEsE8}ZIncuzb@&rR!q^^4t@@VXYqwY2y9p@Zl?NRNLS zH~@93Kg}iHmWnrSZ?+c}b^?9-gka><%&*0X}sau?A&?*=CIiF7kgd3 z1=8<{_iO{Uu2n9!^?V@=|F*)*&1T7?Z}rsgFDSC@de zL!OPaGW#jq&^F>C{galiM;={XgNL=H zGo!!Q@pjtUQ1iG2tPTtT!hBd`Q%BmJDPnf3>!U;VNibYsC&zwDhmy<#npUhSe|oHw z%*gC7sIRm}$(b<4zp>(Ls;SBRbOUp~Z)GdCD}JvWd3}JqkYOaL_F?q7JuGX}F&y^X zLpwRP6teXVFuP}0#SvIIJq8S(*+40+{oYg&plS*q&2f^X52$AC;Ni59vfG4huxe=s z!d_$9=ywe?G5ySl8^GMeO`iGh8urs)33tsG@jvE1Ks+pIKC!|iP7p>)rFp^|SxCBm zgq^qD@BW($#Bo5F1c%x-=QR>`K&YdR!hC4f+Zn$7a=@+a9s_ALX!M(OPo@sKZwHaw zcenUzd{0Y!!4Fjb0K?zauHsmwLr6dHjTe>y@htc)c9TX$ld0w}LuGs=BVDKY;?;~B zjM{@QCs@%wLpHeybAD-Zo3|w%bdxvQnagJu5xh_23820s4^>wz{&7~U>DvvqzHKA( z7TI&ETSk2dO)XN<_G%>~{E~}Ib>xJ4HL(A{?&L@2iE~$M!Kvpqnc7mHD|~o+CJTsf zv2;i#J_|SGL9wl6Q~>R@M|z{^Oqpik$JdY??kh=~vH1BNiL3OW`HdWr)6o1V0jU@NrO%W_sl$_#aDW9#`YHg>eZDN~R<-B~gh; z)Y)s3S!S6d88Q!>_@NRtyoAXZ&03#jjNtc1n5tSz`ZLop4 zmVH!w2jiQY1N|O-moyuCTCad@vIVi!7oghEl7DC{>fL5~wvB-H4AMsr7ab04X4Q@z zb|Kag)czLwC`hyp#9yO|an->VP#k70={Ru&JjOFlkw9Jy zq>ZqCMs;G1rto!mo~EK!3tVEb2oB+792GD}X(6a}%U^7W-7>=@M$1IqTp zpps?S$9@RA99avFZ8X<>|9&51e!S4=_u4JV6Pe=u&zEcAiMRGZeuSUaa!!5-IjCX( z8E(;(dOJ&(p;swG#A}mpX{ElbWHQM&Obc-@dNAH|C`cc7&_}JTeE5M`9DqcIe7~xZYVr@ z1Bmg#(1U=Neqwx!6XLjS91=gn@>=pxP#)dvT{CW2kpRIukwAQh#A+%7LCpmQTzPTikPEQ&=1rjf zSD6K;`dx)ThYdM#vi5J~0PIm%q#-uNKig_T$`&Vn^BG+?b$APEPMXPHi(EnNd&#mQ zbj@`}l@}2ul_@qET3`06|m#`n)2${sLfP9N<>n(I=ZGv^J) zU>flf&P=-ee@#kd>HIpZE$)D-x72x1W`P;@31UnAOW5aQB({FMMfrv>+jU;pUzd!$ zSgMRM=$WtLX!7#NDh5mLbr?OmHiv=wPT*KHN%4+0hpyF7dUMFWm< z7Mk#0Xuv*S-iVFAmWj#rHUcp-rc`tmS5{sYb3A?m=_8ac$q|EEYOtT@Sfo79x(uXy zEQ)KyUpkWdSx~l7xe8v6rq2@E_C)e^P3!#CTHjLGh9KCIhb4c23K4q^97n9C3MPfVV&>)xQHjy~4;}t<( zpfp}69io1Lg|k|t{a9C~cP^SK=5dREi(@Kf6=xk?&=5w>n+`tVV^OjF{G>*bw1TPG zLQ2Ic{5!C|r1wI~wm^KvK0gb_)?qzSWz`zqzgg(LL-5sP2M~wwxbzt9!^0`!-JddDBrEtBz8CCDode*Pc|$|)yFu;#xOJO2t4k%3XfbflEKUBBJ~cCzoUG;6^WI>)_J-dbD;(! z|H7$dIVxLd)vSkdInbO)@ZX2=#9{|Yd`msF5wj1SrG8l8vgu_it#JmkO3XpUZ%V&t zb|EOcV22KLy;{AE=#|_}qv|vGoEy02p2q_Z_R-oklFA=@({)ZNi;+$M&Cb+Zh0#7j z?&bEl!(b3DoOKWsOBI-XXUb!{Jq<+SckI3=nMH0}p)}jsecurc&63#C=cpn52IJdl zRr9iK6%T>t5^Vpwn}X&^^7a-JB)x&w!#1%)YZ6c$dzbqsG-sC&rddk|(vvixSpd>& z3aShK%(y8hy%1FQjF?a=kB&P0mn{#-LdE`FS9fEi$>6!(o;@y2hq{kUxXM3j_M~)- zw2{$#0JYgCDXTgQ%1}VOCojjX0s4Kkk6tB+y%_bU$e;Cx)!bMI4n?+?)T=-<5X^Yt z4vUPAsdHC3ORM&qxQ?_oNvM8PbE3SK#ZaxAjmj3dKfzSed zFlilbgP;4ZDovu}f+nN;%9e}X5-ZN1Ev<%19r0q){NM^FQ&H?oEB`Dx(>! z7;ZHNdN!_(l=C=c9O4HCNh9xp>4+mj&8YsBn$s-mPL&RM|BGcRL$`1^j*8tXBjU6} zzh759L;IA(NF%TpeIJT4g`ggn4j&q0llI|!{*)n5=5v(p-*|xa&uGJmf2-J;vJ1_F zvQ%zkr2T^GmZ^Cx&4kf1?lU8;hscX`fAo$Ah+VRAGC`~I2sOuZ*hhIXuAI>-5 zpjn?NFUo{{=ldYd*sE%u=6kSe+!!EsW5fcYvbhz%)2KdETCmq^qwuv1lngz+io(L28$kGdFx&CBEY=TUc07E}(}7Dd zhjE#`9gnzUBfp*&nBi|Db?e@Pl(05%sYP`e-8c?4-w>lCvyom`7S~?}rv}||3EBM% zmi;M$to8$h^^sFx)T9&Y6jYaOJ8a_<_S*39p+jXI53N{D_l>?$FpB4U?*-4ZhhhDg ziTK@GpREe$1nW;7lR@6PXeJjDCl5WixlVJIf4<3^i|6Ht0SK+v9Y*`%R#zg(AZ{u@T; z5|2qmwWtnW&n(o zj>N2j6OhoQlbkW_w8*pCjzf!+F|GC|Sa5bBESc3$v?Jb6OB(}4X5IO}fBP~0Km|;= z>?GGEE>-I-oo0W-!Q(?@-H#KPc?_|f^)(sY(_R#1x54fMd-EN(q)Q#&iTrctMBSK7 zNXs;4RIk$a>lr$?SLnCRSk}2WNp6~7fh%@oxHLav%oB7LfaaP5_nfv3ho?mHH;w_I z8D<0_Kl?M7VSrX<;q2gHM12%nwxm#2sYF0Lu&ZiyugCYHR z|LQ@w`)N(tVAB~~cw;_%t$EkwOz=ed9+~XkL~IYrD0cL{4zv!^ zZ(E4w;J9HJL)U#fnN1M1=aBOy31528g0cq{aC&_s=;r&G)$-Vmi;g#z`TE0n=$rH4 zJO2?p_f5u5bUnfQ4np-eOmU*?kCO^S~Q7z`g!=g+#S_1=FA%* zEz6#WCl5?Ggl>mxF@t1Cd`Gl&Phzxx@Z>=#sQwTM{pELye3sU|Gf+RKt)9?^7Y)?m zS2}g&R;2-W?)w**u{E7N*jI?5lb?dd!VO3_peQ?@Jz0|hs{&4kH|-4%zLa6q zwy{#PlfLsEG=lc2fvm6lNbKuk#FP%iZu=`P4X7h~nk3>K8-M<*RXYDY%8v~>KUePg zUY&P*62odG9S4huy3%xc8$PFz0o2jAlyjUWNx#%x+{}M27L|75#n&20(k7t3mX(f6 zAU)|U)@ZU7^as>}{E(xVwf;FcPPB*k1EJbce%+yKs|cjy7tYS1n9|*t(XnH2z#63e zX5NW`=&9F(3(JpG=N%+zr7U>ASG=#;k?RcltPSq`3aMWO^*A&&b^`i0)ba1f3hLX* z6jL4QB|CaX>JF;m5W0taH96nD7GE^tD#WxY!W#v5S>J|@v?-ob@nFYHyp-9QuWC1y zsrUXpsya|ziN2v4{=Ng_)T5Af@}R~os-65*x`uS@pr}_C0ry@zK;7<3dCiJ!uuI-A z=y*V9bX%!EOBj(#@h|pPo zT9nJ3oL++uIbSSD=Q!0q(tF9&1CLlqa<16C;w+RIUJ{kX;c73H*0H#y+rivjPqcjY zOq)KOyrt7fJ}>yD_GyOHfXk%HrclF-h5ge1Qs?JU-$)3XhNKj^TKkvl=OKPYC6u;OJrP@cNg zLCOn#cwvOZ@T*^0Uv5cj+ZyH#p1`T@81*=L)I&{Da5vOgSCi^|XuWtq%Z~KCha@p4 z{5Xp^+Dz4&@*d6C(+n#c?$y#-Nb|C8ocaXcA9)5t7aAgY3A3L)6iSkZP@UE1d*grO zj*+1BL1`z1uPDG7>3&Fh#=}>5;Ppd(sM$Fm!{$UuyTB-X^WY4;ve2XFvzW?dV^Y!c zm=89kXFlY!2tH?4D=~=fOMdSABG$2HTc`}}1@XnJz9_ zNcG60KSd!p=HR@&blvosOza(Z75b0bCr1673Rm=P4c4LO!;2kmA<_H;ch_M#gm>6V~J^7 zS*N}!!f4-CRDByaXA&QMmag6FTU$nLd!S9}9>Qkmy=M6yGeOl{amGXp>q+-G^n8g_ zhosw^(R2J*@Jlt9iVgBTij*hu_z#~&C-$4A)%F3$2P2>$qdw#({iltMcI31MJhdWG zLt2Te+P%b~3%6iw^l7!9=(~chpJ`ky;y$G*pF&kL3vZMtZRNV#Q=G}aSX$UMpkBs7 zaiwVHP0w`iRms#b-6`u)^|ueLG$QrDctO6YQ66JuTp$XweerpRM%34Cd_OK%Jum2Q zH0Fo)iOLyW<*9@Ns5GV7@_~}-5N9=i0rd+83+vn6@kxtCU}|NoCH==GyEg;*IsUUg zj(rWEfyL|qa7^wi*LMrTv-9-i!`rP;XT2jTEj81>48&PrFWyrBq-#hwz`d6f?D+W+ zBQ}KNtkn+OZEmd6M=^ZeE!3T7#jkkul9k*R*Jf!z^>)uFUD%;xMRnCp+Aj&@LEVyo zYKE0Z1*7}rFm?Q}ely*-GP?*=FPWW72CwFbMqds=$1Q%cz`M7I7#E0ioK&Y>aK-#H z5QITec?9V(abg_U9(s>nt7~J{=uto(DT!N<)`XKTBh??gAKPEfX|x$$)HHgaEHzz^7E~D#p9A@-$pA*%z&bOYfk(HrsdTxD1IFt z6y)+?`E6{;I?5=Y>a1u9&G6iIXk$MNl2c#p2l}`0-5*=!r|+LwN}qR*xYn(a%xU5R z?sG3HEhonDXLNi_vB%(JZK&23O44~^pin{Yhq^^qfOr_xe%oYp#GcM)s_KMlmpRQ| zLY`k;rdJFR#IWQW>7e#_c-}1Zoj)D};|9dNh#MfqAZ- zc(B}EjCX3N@rYV1==gwYQ79hjJ**SnpS=}`TUp23uSDFOLSl)9NVSXotLu^1_zChk zAcmvkYbVAmJV9%Fkm|!qDn7&&ecH*;Zu#uZ#JVzf6M|yUhqvRI*q-JHbX!*Vor`H1FNX6(FVq2SA&9>oG-?CPcJ#(9DA+5(YVU}yoGdFpLkyeVY zCc&T>u0|7LOybu%`Uhajq`p#l^^AH=;nZ&}P|d13;ff;`Xa>Wyy*DAlV-FU&4g}(K z9Q(sil3vj3I>V~ftD)qG`&)=&^1iyk7H$b+R~wai3hE)bR)-mV_PgqquLM zT*WG!JdP`WCrwa{!rYxJSnriw|2X`*r6YImbyg6w3Hv2A<&5;|iWT5ZwVv#hG06Z6)Ovd{WJV{W(3Ed{rL}e*YE3`WUi%C6n9FBXKN{HVNWttWWo0 zvzT2&&TE~ADihXRH3vS&UJzE|BC5PV3@JPv2BDdMu^@kA{?m2%>GeCHscwa!Y%d!a zhauIHFl_Hi`}L@b-PF1??RpU?r-Dx(T`g%NU4vJfJpBQB-8cw+iw(Hk?o4?h92FBQ z7NB)ur166GL@cvEhVIVm1^I;NeV4u?-@#Fm&QX7#R4l7{SCD@wh84uIbEHJa3;1QVnWVPm(TbXikDha&A}ULdx;@r{*g3 zO<#n>SVG)llCL!e6-Kr z1GMKn=FZLY6VkQhb@Jdwi*j1Hebb@9;RyMLtN$yRQG9% zDjgK}2-hdxj54fTpt%9#gWA(9R8wYpVy$X2DLVbFYwj5AF)@ z%U9q)^BQpdP<5p2FYL3M(K(E!eQM5WmQ%%#tDpOVsv+`lIjdbXeQrM-#`ZtRyl%7x zT1!DaFM>8_X;nrx`eFr?5vq8EG7;#76{(qxP`Q+NkeF>MrY40UaUPNvxKIwl-YX^0 zIhi%YUhxlbQ|Al?@DXnvb{>lc1<^#NjaFdThDP(9PLly~9i!Vp3E z9%5Gf2S&Z6IC^U{oLc$}D3gl(q)H&Kkd*nTS4IKx4;HXgRJnj&5B^##ukxru*EW<9 z4q!^{Ye4fYP4R*(c<(#%{JnzT81FL}n>8tCPg;CY`z2`2q{`FOYgjnD6U;a}h?h)s zLSlb8tFktdUr`39&)8eu73=M9X)0%&Lt<8>%nQi}x?ooOE0kwyas2@sQDbqHJk$~| zeY~LNLK@j5O0gxt&xoM6V)tlUj8|zn33iLxZ&0k-&pKe>eXiSHgbGrxa5< zJUf7KQx2weC#Kq}$C&i%$xb$%!ATSnGQ3HY_`T z1jzgGV6AT8S$GOHafhnvIpxG^p=OGoO2eV@jHdBK4sphR1)yT%fO^7h1VCtLo()whm0U zcQp2#;tI`g?;@|>DDvxXgeUH2@MD~wniD{Bzv<9DdnRhGrGT2fss5umDtV3;U%t;& z`H1pf0;76Y^^KIPIn`5DrYp#ZP;(XadWtE9(o65b=YaAIsQRSiWUD)y%cfqrG_U)N zQ?*eTvi6{}Uf4{iyQmwC`u7q@b3qelCFcFJ1mvIV%HyNQAF_J!u28bR|8q5PaOh@{+& z6F&FkU!U|>oC+#ilfR?i{RL?Hg|2rMR|U%j)l70z)3<6JiGN!`@~T8A-Z>nI&wyq^YNia9 zxx-!V&o<)f{m9!f_jPSanxUcJW0V0YbAQ4eCu#%vAet_Z*El~(2bvpkT0hqGbuuR0GA0)rOqnjq54Pd+Hrwz^{u$zMsuVlK zak@6Nr964&Dl9s@8yf!n25u37FlO>B?vuHUJ$rBf-gj>)-=E81?Xwn(C6|7~hUp_^ z%bq)-_B2nVJAER(C!YKg2OHXzqo1>eO+89JdG{oq%iPL}ciuA^_we}F zW-#?fec7%@3sEuFlIfOoVg2G9c-F_WESu(5qpm%|-p&HPZZAZaI*sIeoh#bwqfVef z?oE6?pRNV}Vk!Sbz9nt#A-DXV4WmNq$#Em-^Y4juWZ|$(apI1n^e7r69I6e&TH#;u zBsDBYK~7cG@IN9@TM^Jn}tVJ5&o+WEzX$aD8Sy$pBjo%fa~X+j+w-gXI#il?{I` zfrh{RVV6~FzJ7Tf*`?B5Sn2kb=iB>%Md!Pi_qaX=hi`(*qwUbK{XG8T;u`5>QXhU) zjD+lT3(>2@5zuZUY@YU5{QftVKmGofmD>eEb~lcuq4mY(uy0UL{{*)DUJ2F59u|&Y zTJwv}Y49QI6XcC_lnM9j`{7x>0C6ETSHfZbHy0EW{@bBRE~fLvAv|@nd@k(cN!bK&w_dN z40uzY_FVmqfnkD}pJ2vkR_`Y#Xiuqirh5b!@{H-X@khlSWVTDO^Wj)%pY^q>CnK*t zr2W%H_5M@mp9ZJfE^@BUHF(|eDIBhzjUP5AXsK@C+Kpi-ZI;97?ayF%n6Yf!J5;?M z_0JNBa(Ctri>@M_8x~yg60;}#g}aky$OUuKam~L+P|d{|+@5a5q!y|0OXGmS21ii+ z4)rlp>-u4HTTXkREy-LWgZ#1?)iS5LK+T~;K=mg=uaA^6y^W;N?_x;KT$)C8 zESo=!gUD;+m`Pi6G3Q$h?D!SP#`fNZ(Oz?r))#ltXSVICUuMK1JnV5Kbo-pZhCB%a z{cf&&P}km2!mX;bpu@y)KJTsrp7z*@!5iG76qh=IRWD6uL*zw`Sd;@0cUZZZ5Fy zGj1Vi7UXxiiqunBXJ1;{jv5 zzh(8|NvC@_O79d~HtVwTBRR#ZhIHTwGCA`*)*00q&sB_)M{}w{LC8+{7a(xXT*lLz z43bJ?m2UBIaqv4Z6(3q1fQMcmpwf0TH(Qv9sFeO8gzi<^9K z<)=N&Bx#Enws@ba39f$rGI$?8^f|)oP$sQ>rX%W)JcqOgq>-hf?Y#3ihrSEm{dqOW z{q+$ibiU)PdjGbb16O)sy=w$t8TDFh9Yvo5ISs%ep}(P4p#e^b8O#P7*z)3?W8}qU z^)#e0V#3cYnl#(RIHRK%UYX`0ns}9{*VYgpsMjVf+borr&2D){E;XqqU+uanX3Xox zjW-NJsta_tjs=e-e|5gBqRBuct(Gt7`9wC^3uyi3h(6^nVSAevZ19bI)*+#>He{-o z+EZ=#lvl#_Zz1#H)|~Vb^#9Ov8$KOjfo=a|R_6sI(=)#tOnIaIyvkQgd#631OZPd9 zOV(^(XpQ8zka56P?i!Clz#y@ml#3@tx+acFs zVcigM)At6}3)_a|_p+DmRxo=<&rKok-O{y}>J8AHKAQ4XU$w66QRfk~HyO}3QU?Zy z8}M3%n{lZ}V|oU_XXSg?>hcUacMJ09t*G?+nU|mNq~`&_r3m3Z#T$n$ZXsKyrNYr1 zUpOYqqJwygoa!(R+mew1?foIGtaN2orjQgnEvw}mRwqe+3B&Oa9D47EnE-P z-ps0BQ+t-KxsCa;3fJ4L5OMQQ;VH0@V-~jMn?@(0VwF0fe>7A(E}zc~kOmPOLFv=L zX-`16-8z`!xfC82DR!C7GjCpomqRbZ-8OsiQ*({1>vSbe`EK>Reypo~H9oo^8aL(M zV5#{``3`swb>mbkjCvOKU4G5zJYeStM5=%Ktb9MlIe5UURvpCZ6^78Eg$rv|lR-b5 zx_rN-9TGs`{v$ zo=cAn_oOCW=i@Yok|<3*z3d8a)0;9eL`~FLph2h4L!V@$6-I^i!WdITi?} z*F2c=${9yOsdhGC>mLoIVOW{4_!&&cKT@1sS(CXM?Nu5j)ca6fQqQ~wF>ar@U}T|q z4U=|`l{cmq(kwkmD6ToT@r*{-))orB5hWx+3o+u(`a;giYVz$ZHJe7a z=hSXY%0ELiIwk+^eP6eAyC#HM2R z@C?Pg__Svf5T_zBvf>;@TEQ-rKL^q}B(LEQtt^Q-i|Btp)6Uw`Mo!-~oQ>EEY`NVh z@GqZ&Urj=Vo=rF_ioUHOorJ}!{KW2>E|gOuNV9KfQ+7OtL0vPEyagIN-ct2WURfY0 zH?f}2Y7jeI6I;4o5=(EN7Y~y*039!@R+<8FKkD(ONdf4am&M{cPU3?u9AHm6jpiq3 z_X6Sp)l;JM&M6$R?H?3edB%trD6=eK zV7g~?Pg7Q-{1r~Dso^!2MTvIR`^&YF?=-%7eTbWDNTrQ>-j9&9OglUJE}P!UK?Zt1 z(1g+VIW{hB#EIRM4hv#*7W=RTkoRkxOd{B;iOVtYV^?}!sw?X`whgK^qFgE_(Q`_c zdXV=ws9vRysup@3XwNDCuns#P!PY0CNV);Ukdi#4O23*|wGp(w;-QZXsAKBY?k_HG zKbMgf(0>=>mrEorFdO zyNCr{u+|t`UVpDtdjWRecJrG2FeL8 z>YPcV*~dBbQQDO$u!41eYDD9lxecVtS^V{>9qjk~MjR}tcEd%d9ZwY)Yz6711`~RoI zc&~ZbYHM?BuQLd0x7#j0HP^zAji*(5nlkBd#YgC+SQAJe8K(Ch}N?geQ# z=osx|q}7Zx4Cie=LNyo!S>E??Q`l07KpQz$Br68QH>to5SMOWJ>y$^pWbuBu79J#g z$66^?7Vo}AGnz9XWfzTF7d0EW6H>`2mvBAr@t7JljrOL2rq!^1aN~n?Zrs~M^)st8 z>JRz3XF$t)3{(6slHq0(vOUtkOR^9;m(l zERa9IKclAfZzrUg5~n7Uh1vDl5>;!i6+d)4a8fP`%Zxrckps?ab5*sQhtz zn|&%{f@kY?YCV8vM{0(o<}f%jwL6f8LUF=&Y&N`?q&;FA|GWW}#pv~5=+x&(nVvrX zFknA2gE4pU9+mTvvkpiBV7^*A6m z7ZjIo4gICK0%%^t(j(@=((TVx9%f!W={f6@^x^qY1GKcX2kIM<_0b7R>jq1Q%rYd; z!7KYu0j)DV-?t4@eO6Ydf&6;AE8T;or8a5j1GOfoc#(Pp>hI-{nlCUhdZZY#?>rFa zA=O=4#(|?qIwy#UP|X@+I^PCkgJW2KTLmMg0yVd3RAA1jhsE>xog~f81hKHNd=QW1 z@5-OB<5zml?UdI*^94?f%@@q+fo<;F5TDnVr1kKb?gPB6nJZQvID#~fV5=;$Ffud@ zj@|8uG|xegBwbiHBM(vzH<2H|T4-qwBuLiK7?b0?iA7-=b%UZTqKu6_j4xeU~jVY`2i^?-9iPT=^W;KB~P`z2#g%&y1k^ zldK6$qgl#)Ts*rsZhd0HNyjO7hKjLe%c*xdvxRkyHELF>Sdw@FX!Zm%IzH1-&XfH< zb`!)|{PgWq!9LkR6RUW6X5mJl{t#-O;(D=>;$wO?%6vR) zS1M>8mqs&yDi)$yC23Uvm~YzwG$$h8P7zm^m@C#}yQ~i4hLX-$SOYXF>lyc>=X>N- z!;;KztbI;(473SVJi{rYE2f6c`%lB2J>9`^c2*TH5yt@axFA+z3@--CWTA+2SPWa3A3ta7bQEM{w1%PoA z?W*?v#Av5s{?2jS!{-=~M!V3w3`|2K;BxMD(uezC5K)U0S5(zC@rQ=yqVS(lGfeXM z54`D`2Achf?lXo%^GRQ-@)G51Bral#oOe|K5qGkdkeOWMh2 zr%gmewJ83thdvucv-!+kqhU%TiHBF$lx(RHZ+o!??=tx@Gx&RqS<-iJOBPI$qU;~^ z54-?1$7Dct=7*2(w&Shn9*i}Xw1Y>}^yJME)nuPJz9P`hg}=ULA>qw&W;e68jE=XJ z7kk_CmlMZAM(J+NbbovJ^z1)o|1gQg29>ZD%dX)3yc>@kYGo2X9xA~ z8`1mQ32op+sP6FvJVrN=qpkk9JX;-w10Veqc}G65f_^Pzvl{e_uUHSfRk0Eu&3MJy z|A>;+BR^uFPJvipd8&r0?<%#9!wEc;_h&@^J0Bn0quGoU8&EYOQ(K z{J-c~QGl;bzQI-&2K>i>9wM)CAg)ZT3v{j`&Uiadpm~*7;tIfTt?-BTtk`((9%P;w z#F`gf5U~OE`1)Us#jBIkolod^frZBw4XvTJvb&@F-PW3w25pDexxZlGpi)uU{WPTg zbeEpp#^BQKe_{6SP&k|y32tRxqU7&PG#av5c5df{=92^jrd`5QK8J+$yl{C>HkA`9 zJIWIp51H3(scdw2DkO(VEd5zCW|0S&I;B`=Qg?e5@`79l9_yI}44 zL#Q?%V0N4NxPGH0zOCLH(;SBKzP}9Q$eu6R_Xm5ht6nTDS~(bJ)tw3sHimL-V^3b5 ze+ld!>3~z)$#@I`u>Fuq{Pg4?TE`o~;@kq<7TKEX1=F)+ChQhBn@(ZsZ)gpenO9xd zSSQ~_H>ou&+G=AJT55SsoCt643BJ$=XU)4dk z?QP_%mcHN`Y|iQT(c$wpZ4Z~3Xwk}qSEd}%SUOwq%kMHU@8xtkzFJLAJq0CBuOOw_ zS9rDZ1L}02gcF?Y_^iEi<%?5V?CNR;v%js<9J-T*V@|Z?X3@dI?VCRaou_AiPVFp? z=^DU?kuABg<3C3Ghtyjx^nTE__cx?^fb);jQ9P(49s68ihR-tLXWDZdapt(t4xR$P zcUj1;tLWYqtDnNnoZB#MP!LwjMZ8|^AY3%;h*^`~sUC&3raJQA=k{t%!ES#oFtb@C zru!eq2mXVA^hcuLoO%}T1+?U^ifUlcWPkBr!}hY?0A29^ zR$uBT?15%A+ETx-5K1SM?%W(TQTUvS)%>)3fWP{GK&l-GU-lL=CP$#*2o03X^^>!0 zcHyn1Ti6`bdUqvzN zyE-1A+Jb*QHepPo!K#Ltm(O3NM{?I6XLOiK_f#odz;MGcsz(pa;ayAd(49W$+3h}# zckyNJJNt4v2RlS#PP2 z*za*1y!!AQcW1mqS_^nsk3M@YH|FF+0Ac%pG*4+7JwGZ=d6H<4yTIdBEH++WseP5a zS#Ioq15FAfsJ^5gXA?(tmsth{LVEXxM)|cQ=@tJyIgUSk_z0DrtULUcy=C3u&)JUh z-NH@G)HX*BvOB8w5`(6BNyl^DnOn!l*aC*YuX{`Qk`~7Jd1Hoj?$L`6TdT+SJlcn~ zyWD23|3WpSmHb4~En!_4#B;hIg2o2NVfXJ>NOgyadK#|kwOUk9q4&>MBwsgmaBfqFz=YKb+S9dMuubrswlS z>!w{eogb(Fma6xaesv!+j8RWQq(uUdp7NAzJ)pJYv8F2|`5pJLXsnKr_i{R?p_&zk zrVNE&ed>_6+NY5YaO!jZ;f0OVqwAKa_M};KZ8`fc-7l``6uz;}2u;+Yvlw^27S3|> zlvLO3?QwdRM*0uDF}hRL`PiC15~U-`)E?1&r<%ahrA=Y-#bj()=QML%m&s;p*8{PD z+H28mqOBxt;)+QgeW}ai`h`I~6MeX`t-VNY-e0Qzxf{0$nvZwF+(lU~O3S;{&SiUl zoPxZCpMYvyW|s7mORpY*0=G3R$o4fp9C(Y4XBl^IlLv=l+*p4H2kBMVNDd5mkNLyx zfV4@}`5BMyJG0TO#GJ32UyMHnq-*-kK8B>veC3PUpe^`-Sv#lklnOs_vdIon=YlN~ zf1v7(kM4sY(eU%fh#_>z=RqH2W&!ZOb>0mp)XTKGEf6xqe zb}^82KHO~TM^Fqn%k8N!j{ioiR4J&IS@Q@7ZaZ`oQVrm{BmPpiRT=*H=D>kI1GRV; z2{UG7B7M&|rXkTRjtKJ?Wk(t$2aoa!BxO z2AjP)$eCg!S8KD$uqCJU!o*X@klqt$UnT7m=$l7@k=Y#OVRY`RWP|=4SoX&+;AKWL z&_)u-^MRNY!dgDWB`IH(J|XQpvyR`2c}M0{vDI{JN6-FtgW1{&Mw}o?3-RH@@zir$ zk(gD|d1I{UPbB|^Ju_|O*^CX`AkzV|PcO&aou7g7iuFV2Zc@6paX{P%@O4Pn(0YUD zNVB;YbWfaR17P~V&YXIX{BHm!O<>h>0sOUQKxw3D*EK$NXR<()=_rJ8f5b>$6Hot=v3_h8efDx|vo#A}HiN$? zc4Ef!enFSpBY@Tk>ilejlO5yXfK4K58_dA&SF%~QWjK4bx*`9UTTgCHsZF{y6^SX3 zdX?YGUMCt(yNHc@B_jO{NQ0_8qFtk8Z2j^)R<20akY~7%Zeirn6wqznL^_pvBe50t z-w=yE-S^V>_5CIJ6;KZ<4w1x&%qzYr?^1gR&zm2KtojZ{ena`AAv)M>!P3TS1o1nX z*?ghrnI+-kaDVWc*OE88ehiuyb%!I{dn4%^Lh~aKJ?{i>wtSD`RywD4jMhRDmy;g& zQ11_iR_vFSSVB&4J_d?!DF28z;jJiBBr@^_&5dnII8duEM@L36(q>NDCWt*59fu%I zRdt{xPgOm^DI>wdeUr4rBg{5v7@Rw34s%j16+dg1y+M$BE@1upTZt*3YlyYr`r=(x zEV*;@PSW4DoOsY>#2I^+yu)>9<}y=5IweNs6~nB48x*sP^88*vo`8SOo?zEqnv1(} z`s&}9eYy-f-#HA0ZEDG1vA4OePo1Hz?R z7wUC%{bwS{Um5WpmS%orq%+L+$y4&0EtIn&@ovZZlxgnKwNHDHIMRjq8rE+-0Ms8M zHT(~-e9UPUtf!buipdJzR8~E2bE7K#$)||cHuFNZn#j6Q~V4{uk1R6vGWJ| z$W>iqu_0fXZT|>D`5R(JtMNU}};@geHL3>(C;u&qg zo{oqEQmC(YR@IH;y5798fjy9CX%^n>=&~mBCJ^6gMxGuAgZ8iGM#HCztXq?i^i#wR zY9##!w1SZzM=JfO@;Bl>ICjlgUa!`=igAm(I)meh9X8rsgnxRc!MP_Q(CZC@^f?0~SYlK_{aFY~jQ-raXo8S18TxvTF?S z{t_UKVc+VS3(8&ad)hjcTNNJ)-4m7gy2mF~>x`J1K2LkX_J7f-%8|qfy!zE(*8TNE zRIiaV&I}Y!`nXyXpFa?adx;(S;oCkK=zCea7rQa1;!Q$jn+9K7fMU1VyG^i~CEc^- z{%+{1XTd1PYB%-!i4iWLtmWtc%3U^SS-GEb$TmE_)K<*2zlM~Lk=|1joZrrxUAL1o z&%oC7y*>I}Mz15v4KE1gA;eacClkm+7K_ov73|kK18Fk&j8^gV+AVkJ9y4@b*^z!{ z6N(Sxy7Hw!T+is7u)wX{Wqw`&zFq2qbi7sdtLpY&En`Tz@(LQS|4(tdAn(z{9J(iV zU6iO;j&ckVSHi#z#+)(9BiPjl>L#;y=ho=(2XRK_HhhJ~#!RP9e&#~oB({i|j* z5tkbiPhL@3M382K%9{=EbyIb#vcE`}{2SW-i2;@G2krmMNLN|Id#Rwfb;_{wqK3v^ z5_9sj3lGvr518g{buIOt_!YZSBGg=hDW<31kPCJgV7&qUng<;u{`~qF z700Ljyban0Z`JGuS7jYkdUL+r)6P=)A2Bi~ui>OI+BjoV<<&^NiH-V=$4*9;GII0* zAdRfbe2>H;P+5=uKS7LuYyP_7rvJVou?^%8KSs0AK0uiVXkVEA^TzV!%^{R2yThpHJ3DjhsGr#U9BO2IWweNp+T-Tsa2Sx<8e!@{_yl&q?@oT5l zNH{;_9IW&)m8wpaevmGUf%U&pb~@uivrN=U8O6OHg^6EveDLJ4a-lT*#FEk2$k0z^ z5TN|ZW*KkPsu>qCg~~dLYXor)(oBF6D{7TD+6Lv}TGI_c+Qf+a(P{01DxEOD)k&6S zglUMM809pyc-Mf%pRT6nZTPU>Jz}ij|8aEPaXoHfT&W~d5+NgzM9V1kJI^6xNA^ky z*(*Cc8cHIfC?jN~j6~yio`YmGr0l(CS=kwv`<(u`pL;L$d(U{D@A|#(c_F$>eMuT! zOZbWzLl8*=seF=fPmk+fo}#X&rTS&xP0kBFoerR~%DWTSLbQHkzQs)=sE*bCD?H>w z-(6ShK%9__lw+dHsWQb&Ec)^;=)W_G@S~88_jZHluS0=616|9W*9*22vkZfo?XSk_ zTI{QR8j^-#lx30jXi$~iowc3lj2fO%x(rl3lP{H2m$W|KsmvzmZ-KK;yVzT^E`|G2x=EylUYre{SEQ zw2e^fQux9|&i1V%*X#;q@1ia;@?AXav&0ph0!fz{a`GXJ#s||T1;Wb8ZICmqKy=SL zgOtewVU2cJE&7hh#g7jN7zo`;KcH*xBjDJX@wg#3u=1!SNXHT()^_JJH;u=DZ;o(n z3B6CJ)^1^Vemp<^Wdcm?(MRs@R17$FJ?{8q4FAlR@*n&D6Lnu3@+FrCXK6ZI zz-_avrgDb9K1D=D@(VKUD`yJ-`39!5;^xZlV_O^?}XNW1K5C%55!w|2Zjcwz-PGPWR<85ijuW>Im&Y&t>x7 z$(tA$xdAdaH-WbY{@|EuJ-PD=iEkVna>`uJvub|TrCse@xY=efuc%Rk7rJx<&lVeT z{%q1$OMhwDyo0!T(ghfQus&ubWQf)g*U>0tG}PNrpX-(g?VE`@Y+s(a%yW!I&2&>p zyT21pu9_lqx}C*`Zh3Hv?w7%e=Sb_L<^ki`Eby?KApQo`5;O;lnrg`DztO~~joU|; z2uN>#1;53XplxCVU)ys(ycso7KK-mGV`6n>;WWml==+M3tLSXI-)UNd@b7T=FVY-B zWVNA_WSsaW0!HNE_-p<7!j&h*^4HsQX0$nrv`;*k_mg%{*WuE~y8LU~qmVkqkByFR zDYHt~qUT#JHnX@5iM^M~1NZOZ%G@u|D?brKb35}{$&_jQmk4fC8Wxmw@2A;W;?0^uDh*vEev@F-1JHui4(IeCU2WS2^Rqn%wnKE$-=c0aMNn#$UtR-~jiA(s9Bk z%nBGMO^go7Zc_%pIqgvK!Fs6Fxs=SP9-#H71^j!5o^Y|(DBM%G99?RjV_VJ3)Ev=Q zYasV}-xO{3(b;@g4r9WjJ8a$8KC+G8VUPhW@lKDOAY=n+p8E_2RQD8j7fq+^v>wns zHdz0L1z2`?jF&=tNUPHm<*GP|G&d}1w}r>vT0v{MMqKN07v`13@P5OR#B;Z`_`{?% zp5I*rdW-tN!jlmwt2Y7~vz)!?E-qZzN#<9sVN>SnbNX|>e|ifH+VBj=F1Mt6dg{wb zTasbgGs)9SdqDBPj`DPTpa>aIso8x?pHuA$*F&8p^%`1s7^nGs^ft?j?*uDs^rafJ zc2z1%)(Mui2ffO^!WdJ*a0nza3W(PYL6$gk9u zpO22D{yc@Aj&5Cz3ovoWHd*bAG1h$@!Dvn8y3=2^^{0NskN#bt`+jWnF2-VLXX%xwW&bv|{&C~e3A1Z77a*-XvZiwmS!%)@3vtNnGuBRj6iQ;cQ zzk8dgAsHq_gteNJUUXm9u{Rpil&0FD#Tn4B zaRPROc(mNLmK6`oz@~TKYOk{HF#onK5SEFksS$Xfx2}BQu?q;(dCzsuaQtsIL0X8Z zy{oawTn25JAo`s;jC7BTjNV7@r93qmX1RS}rt=FR_SS!d>o1v_6U~)0M^mKQM^-Z& zXpU@RRu6pOGeN|BU9a7jw*?oTG$h|KLP5GGJTIMyjv#LLSb#4Pm|IGs1 zgyKBsMDg>Tfuwqe6LpS(!YZeQt>OIc{-k-_h=(Ub4IdAtbEy@4h%@8FRm7|Mve(!z zK@&qN)kPxNF$RIvv0I>*AI5yV!&!ckq$1 zF&nXbBy_KH533D*iK*AaLFqZ2O@pMazA5S3<)nvx0`)%7^(E;OB(9c5VMrF6amjWeplN!9mA`z1$@FVVP6zbxuGM8Q)ZdUvw%!5ms! zJh9&xD=nnkzSkJS1+;Ue9+}-sc z9JZS;_;E9!9wQ&oleAzlY}S|n^#iZflTuru%h=(zSfl3p~oCfrK+Ho^n>Cl9blwNV@))DGcwpt)@J4c)zCk@N+=C4YbCvr*{{i zz7_iQPve^nooPS)NrS`*wTH78jmF~88ddc@=#2oI<|n|%a4Zj=-%Q5Y?t-+^f8zAs z4T3%wFPr8w;wQeuAeDE#V0J{wEOi`kQgpQ=eb7 zlT=?wbuQMm-APr6*Q|`J#N9HUxN5Vb!*FpzLw-ZN)lTSH-5g{?C{B-8nKB#=v1`B$Z z3He^?71fta^(WPgx*KUvn|D{bpt!XD^Ek(=~*ZP}9;zFsvoIStkc(ma}+ZXQy3N=+vewimm|WSwe~Fb^h}W)c1yD_?-! zcJ2Y?TSzCN@?G{Zj!bC;s!v!O=)(uCN>CaZ7u)2z4{B!0YOQR^ss6Q;W8j>}zwk!+ zOB}d3guO882PYCf;^5W?ac$szp}ezOt2!{~#1t^LafYnY8rWscOfOB@p1iRAT2=WNS(_eJ;hSK@5fOa*DKD^&^jR1FOJl9NBd9NQ0Fq@jS+R&V~;Un zR*4sDm1!#DW9UB4Z(Tuo`0B>C=wtW*g6HqTc4aLzls8Gz?X1_7O;wn%==DuTKTml7 zgZ8+c{2d#s)?V=uobDEilA5% zaHe{1vt}dKTDcDn9I6dxY8|HIFSEN|w(h>BMV@0lO~Vi-Z(XAEAgU!mzg4%YbeZxV)!lqK<#X;-KTK&1UFVC8G71gt z3%k0*3tYO!iS4!HKxGV+jS8v(T%LY`IPd`y-w+m+Fv2kPsFH6jRwi>Xt8|)A{aK!TrkS@o_ckMCdyA4;^nvfc$ zJS6GFS#aDgOEel4hPBLo1NA%P#h-*tiQD1mn!{+CvJ*n-xnJt>oGJGXfH+afsJ`Lk zDaHwlL1_v1?}J2#F}s-3N|XhI>Vr>#Gm$hV>i+tKl$*o5#)z~pQ2y^DcKGlO&$}Kb z{I5m1jUjLF;3%q}SGu_8m%)UChE+A(_4^hP8mB|K>SL|aPo}$8VXfO;H1`G#2haN- zSdsHLeAeTmrbozJ)U~+5M!u?m)izz(gi8)w+?>U3IU3>{_jUMr*LO5dHsO_THbeXq zCcUv89Km(CZ|!Ap{prd*XKjOm;9soC@5Zu=#}J5F-cJt5c9MHSN}==qQ({Q?GI?O* zdGH_j2rJ*1p>??%{}>rt;-f$(BCar=iaf0|OlS%l@yi z``3J7``sR*%WPzUSu&&;n)9bCEVVm4(qUt1e?h+wExTv34c+$PxK1_shRY>5s#PPT zF{8_#90=dL4rZA3lw0?k^N5G(7~GO6zDhB6)gAd#9Cj?F-n)-^Vr{t?;Ky97oBUyNBNw`7Ctd(A6?8%t3|ro zP5p;QzMR5obqZkb{Q=y$nc9hv?9kiv2pbx>7xK7|)QM;(k4~-0uME$l&kTm! z;bE{~X$SZuF5|06dJpCFiE`S6evpXxiE&o8K&psAEd?S2$ zunQlK8bIg6&VwI+XTXq|t>vGJb?6;dO$ObwW$`I7(#vkGG#$21uH4)h>&M&S&TYgw znSP?zp))zX{&?^UTkqgki;G&L9TqbEw*xv^eb%UX^@|-3=Z3oG$_ySAaCY$ zlC-vRX$Kp=a@c1G9~jBMAE0v+(j#zsNE7ZpFB}{A{-^qb=SKVUpB|=?`VcfaPIC3w za<+JPHO>~Tk#h#W(e|^c1;d)O5UbKY!oH40IK`+KmU{gXrHkszf0IXJ-{NTOZ|K4& zHcwT10e$KFtm4Ky_CuXP z8?a@qTJohiK%ME&Vb1fqn3L6nhpo=W!abLf`V7X;IZo@l67#hOFn+xj2PTf@w)JRi z=bB+F8-GSUpfy}}0spo;0kfmb@z{@aDA75G{sa9u?H%O^j-Y=yh-Lb;F0HES4ArL?s|^;hbwbNZ1@6e63k?uG$#MH>p*)r9h|J@fytls zFfynyCLd~sah>R^!QlN6^)dx)`#DzC|GC9A_{0&n=sGDFd01DDDviet-CFW1&9=e* zs6KMrk=ppS`aL>ptr}ck8O<^`U%{mv_T$U;X8gp^R3IFHn-z~?ifv2&$z&kERHu!& zxh7KH>$r-kUV5JR4$i$K9r0inw=RCh9_e}EPEQAj-P2Gu{9-Nr^V;*Agp1Ez)=!Lyn<2hicIT7JmTKF-KL@JMy}yi5JOM_ly3tufy`;IJ zj{Lb;V7tG`aOzkq=(8{eZYPc5FCWIhUbCOt9@BPU^9$wB>>v5M?ylTrdL8Vjtw;4e z6A>K|!0@GztQ zh9>m`!PnUs7xoT;A2Id8mpHmFJqtrP!1s9VN5uzwoxJ6D%X3<+*MKlNUg4OeHNu}q zUuj;Lo<+i3asGrOBp=$vo*4GwieHYjeX7RB?Vk<=88%h;)$0s{rnVIk2f|^|m(@6I zsF(UVsBC^(Ou61dHhXp!2lc1(f#c%f-i1Tjw`HM7>!Q|&+5KuGbuESwN7)Mcd&(D@ z!I-$W8ifJgUv9A%rXJWTp(Ec}au>qBM=4&?ScYE`eOUlLxw-?cd%UD~$~Hh6D^Gsh zSK$#f9lA|nkGy_ugV1m8#ok&ubH6h#jA~KgzgSst0o+sC^AF4I^|ICT(H%ZD>OT+>R|h&r`npfp3o!Y z6IkBsBekFNuyRBh5Q;uVKqO34 z`^*~kPl69^4pq&+oQEUmw}bgtI$}ZaQnsk=5yDV^AU+gHw+CbIsA*~+A<8`!{(0%~ z`qm?aXQd}wv0)b~U7&Q!(cLGh4u?VBm~@3((5Wv%igg)eWh6;-n1R$!!sM~5X6u>NyyCHs+)!@aw?2K)4MZw&!9&UwXIrpcZJxoMqopbbe=#k8F9PA{OS*8C3tu zft*)@IbocRq&2`P*Y_Z43zP#T&{$w|a4&Wsjm~Fg$MJn>TeLb>7k+Px7XK#Sg@rZm zvsy{?UfDStvH8d?Fzt3z>gTsrT86Ni6E^Yn3$C#ZK?8*vFX=W6tv!joe!8CSp%{oq z&M-zMl~7E=UilhIDx(Ewgx?N_Ncx?tw{np<}Xn7i)N$31!+CpxZMitHujUr zj|;?xV?SA&8P)jv_tkiMhtaBMuzB@*y!c`Qkgnu39`W91r2PIOj_vShFYUr^LfO!9 zeEO$(3Xds!S|K|Hzh-X&UEot_3$%5(qxc>af1Yakn>6VncoTSv_9&0E(S5PDeMcbd z!hsIeQCe(7RW z!I>dBxS+>cG@ExHmDVJD{X-*E*gJF z5Wk6a(e{$|RZIF$cvN_?$ir(`g!x0HHw9`*L*vF8i;;86tcTtOoA9|qoFuIR2R3@*iFLPt_#Zn?ruWrU zJZ6L|uxy<*>61N-FhNd))!_Eu38Z=F3<>`Y32!o#rU7lQ3sw4?xEH+M`tfJk=TKpv zmt76)-82@5|0{$*W>R%+!hG^GN02lrY+x73TIcqMEHbL4)I9Fsy__7+${>@?~g$-okw_UUzyfr?-n}NKL(n$E$>NuUX>80k$3ClRu1;0P{Al|LK&5ZiQVfbAasgH_oLY>%qt14@)Hq}&R`!s00WL* zN5V0t_*VJLrz;YbM?vdxwej!mN~cnymhB%$M`0#rJ?M$3Gl z`iFU={k1K+Xe4Qo5dj^$UjZ8GQqZH;3|tiJ?ILMR_$KZvb<3=p0k>6p=1Y z{6W*!tn2tzl01sG8J#6Z_zhGW;?VIp(5?s+-6kc0@^~L_v_|48_90^!9-FBzBWK=Y zn!@UG{*??QToQXL_ld8an(&7UTf&KTBd|lA`eMrf3qflqNCUAe!|8ke5z~-(NKkDu z-znKZdyAF~(R!B?hJCNaz1}_o;(KksCAGQk`hzH@SdfPrCOvbvvtKQ>LSfW3U&a@W z9|el%%NMj4groS|vo?=7V89QJt*`c94!Efg+uU*(^{nDI(JXZ3-A13o9!^$M4KU5=z-@Qh_0$nIXiF229%P8y<$@5vKTt+yn<;sqYY z`*6bFQB@ho(rKlP{I7<*w3xZ)y&w&%FjKoM`G+7pmdhLUmsYQU{FgDOI+xYwr~}o^ znR~M^_;Km1csZ4m)@dWW{7>N58JF-=uO+a0$yuhZvp9!pk=~<8_$p|A*znXZrZl+9 zOybvT=)4PeUfat|=`5!7BF#y85gzwnU7-F$s(DmgWZ&=??C;#0@|r+4r!g1iUDJfp zM2&8FK|7a0_^h@kJoWDhx2HLA<;B%H^P;~n{(uh>?`TM~io34CbY6!S`#PgHC*6RT zXE&w(n9UULt!_39AzKh_ox?v13UVAHs-pwsEFR{37SIT)VW7ggRt zo=0@6yPNf8bPwQpdOtH^`t_B@+{owyVbMR84dDg9Be;9jFi=>0ae^IgeBS~yyAeP!d~w5NZ*i~VKQTA9c2(bww{+qPV~3vK zN4;Pn6$aA2!R*S)+vc zbT;xuMtY3RkB{JiDK8a&NSEmwU{jj{q@R(0ll0-&yE;|g`Oc#?NM47fCA#4adS-uf z)+f>n^iB(UyXVTLmk2vAD=jE35^W^m2vSW*#XE=nHxUokC5(OsrHi(bhWd(BQ-U-d z#t%2a$72g%!N)M=^MP;{)Hs8SbJ1*Ld*aOV@VCxb%~QL^RhlFs@;Fd^iL#-O#mLKv z+K3_gQh7#|b)*OCNXlb0`P$a(Q0!Sjy`wTO-ip3&B8#rX%sO3GoZ* zeD5a8hjFSsS~GLXGAhArlmSqE(c1Kqgm--Zx%o`blAhJvW{H!_e&yV{7hZ)WlReLoIEsP z$sHgMDXa9{922HLe}JkzL0N++YjP5u{Sw&Uui;v%3#z^KVs__PQRKW3S9h-ulyzx0 zzTcxAH{+zr_mHp?4>X=j`~ILRhnUpzAfuW`%GI=lmpSAWg~~gWb|ud%RE>_XYAO{D zI0wyv+94dBcpsISAmu`;mVvM$M_~u~9Iev6|0>tuEyoB@I)rdoVFvy-cp*qn0rj(( zvoQwejeeqdBZvGMBmE%h=Y_&q$~_o)4@uqzA2dCn{GrmbKwh7Pd^Vw+V+3*bTlf;u zluxVES3Eu0i?ASpojvTTu%2puBcq&-5l4aAe}w@GTSYKa2~gBRnF zeSp`Gg7oWWS#G0Sis zKdX^5>T%M{qNeS6Xt9}*e~6=Xq;rA;qVZdvC)}Epfuwa&5PnKS7mMF#eh5(()<~DDb41-mUyuL8$7%! z4YnEB^7NT8$VRMjKh?|!JagZ{){LFpvO#xjwtN)d|E@J}Ydu!}Ew3iWYwzR5A**5Z zmO*gYvyWVy*j{j5^L}+`@d*kJs*W{BWH+7mL?F;ag~&GHj35p9a`h2E_~J90K7l(Bc17U0|uJE zhVqG5-P=JIt>NGZeDZ0NZ0B7B<%2wVpKdQjnd>v`JA0T6PdW;rs}|FJM_D*z%VXB) zR!{9RonVk|N3kbHLKC|ZsP|_!#QK}_DbWvchVd&?$7B^_M9p0|!$e&Mq0=|Zuum4hpjB#4*El+jh=dG!6O|LT+CJw$PVYt-Yy1`0hf6keZ%ux9 zbB?=7KYF)bVxGpsa*Rys+7)^mp2qzBM)-T56OFmLh<~ykQi5)4GyC_GeY#D;mC?r; zTX;#(exm*lJAR4IJy>UDD^<-+yl=)=42y&v+wED-uA%OAU&*);mkcr<%mFz=O{GWqdNwXNxQGSFuOpHPLih ze~eDr4b*!u@5XNIT+#{p!WyiZvViKgCeNB1fDX$w{7_eCY*Cg6H)F`J()-eDbZ$*+ zL-&CDKZXmQw^=LidzfiY?{#o%E<#2>#H>m7;^Gh;e(qDTnEPZTU%YxPT+`#wn4Vi6 zmcLJobEzi#PWdN(PR!8E?>Jsg_W!`Hnl6MJo;k2DJcb2oxms)NT2_N+bv5JEH*#)q zAtw6ki&4?P*^(PEV(bS;wA#55a?Zbp2Y&0ZLQjVW6*GCkW;3I4NYx+NLGEzcFQ5JQ z-3`O&T!PLYmZ0v+aP~a%7jCjIW%ZjLfC|^^So=o;4)zX)*j1xtK~XY%|K6M5idqMg zyQOHle3-8ANh*9AG6R&~cCxU-y%T zH@nL(`%a4!*`A<1W5n;Z%?IiOucX(ZtQL3TShg&BaIX2yd5+CA~aqs^e@FTLt=1&tsN}=)fNah zw4eAP)hnQSW!{bEveyS6F{@i+$eNT!n#P(_J;6V^&t4~_Ow-P;y=)!Q4BaftsGmkb zrxA7KhHG;r;WiBaz`1KmWBKR72vFFtV*Wr#8EpptJ#WVgKXrkD-&*i>RtL3&G1#`E zrqs9l14~}-)Ob7gMXFIa-;DFtd!IAHLfmcR3m?mOVKklf=I_&(KPjhtcJyYU`pv3j zJoN3Og~a)0P(E@yarc*D-=|Vb$Fi*mPP^ z4)qai?zmsj{fqsF%JQ{T??B2=*4pMF4pgh(*}5S=u~TA&;s8*%_|epzmF#Lwykn@XXqza)(k2VSDLirh z4|Fvw#%=bwSifm&`Pa!`QqPK|_Wx;VzSL(DZKeUFZm)yplqXJ^-H_f#x(2zV7K6ppCjhOzP<6G&em3=Z zZB9JFRozg(f&c1G%rRsm(6#VU`d0XI>^n@zb!Nv$SLYdNwV_@`Cg|HOS9=5vmcM2i zyL!0XKNsgjU(qN$EhwMHKTqtA&qub=-l42@$qiExwQ4Hvok?eeCvTFlya`gzqN#8Nq=Hqd;1Pk#^$^*KLA}lj?xd7=c?_^1FJD0MktP5bYB7y8H-6{#q}dTG-3Q zTg>?^%WCe+N6%zRYY;!7VZc@`t&h@0Y^H5Cd-6z6tTq@1Z5t$G;Y$OB{dnYEJZaAL z*s5s+8@)OY&Tg(P!_r;>aUi39lje4QID5)T__Cr^)t+jz={^hE|6%u!V)T0eMfiO&?DZSmk)$I#El<_#tb7lj2AvWf-4;mVbM5vaKK%a3 zEM``?1O5&FCAMCBCW8O{MdA_O=G!}LxL8lBeahK&9V3k!WB#sFsP=U3QGMZ=I}<17 zhrx^wF|yvn6jAS?op$mtS9x*L1|aT*E+5MATj>$dS=WWVlBa=mFsswUo9?U7m7W_L zgUiNjI$z>4J+E_9?CH3cEs8!P6z>i0RB3ll^C-*XzZQQ8XAdeeS?g`xUZH$q;<1w;QOof%qES%IN+u!k{e#{np;$f30fP>NQt-4F9>F5`+cf#`K}QZeDM>@rfl|wh3Wpf(~o*Ue@PZlVXJ0i)w={ zr1CtCvVzrI#E09PNRQ=fJ1uVqHGIzs;vwuG;hOU*JzVt*5TA<*iw!vNnYqFZo}AVe zo5yLXw}*hLchV)W+h7}B3bK@Eqv)NGgkg$%{rp!zn_ITrzDKM$>UJ3hdsidB^N@7? zeGIxgU!&&y(djRa-_nhfcOq=k2>;o(Rk)z`E<64&j4-sq3fH0XfLSipp%wpnI9BOU zObGf0&pi_b@wC)?c!|BPIbZheqk&2~w`cVY9UyIn-^(`=2492?8EF{&ZzU|(9Arv| z<(!`dJa?IVUy9iNPY2bnc(tfLSGxUHlo8zYO9ZM37Ub3F~x@*LI&h5ohYqv;T1#IYwJ9RxWLcq{Wm5 zVdRl$4`-rNNgg9^&^AxGgxOKOxVMR(++@{|*BCK_kFa#WXU!-Nv2o^4Om8D`0Mgj; znnfX!|G?1sUHP;2bT&EhOT?Efr3taOVHdQ`Ux7F09zxS~Nu8*lwD;TZWTa&v#%397 zlb4|UrREOZ=TxiW0`6pieZ?UK{ z7VGXj0ufKzR@KhAM`zi94_*qVp~pfu(tEX#YJ@2+Qhu`Tag8KhhH7oEKB9Lbldq=z z;JwVhgYE*@5c&>RCt7F~kJImTrNB znH)V#_M*~Vx6@m3!aHakxCw5{u_XSUiuZ^2qQX7$U5s!Dt`=>j^AsCM|JetjWY>Wz zEl>JVkY^`s8b$NJLVQ=i+M1M*j#;dFTM*VO|3Gy$7Y9wL(2%y)(!ZI)f8q?hRvC&b z=3k-tWQYWJTmy;w1DL~dEi~(A0Fm~O;ZBhO+Y>Zd=|6m<7o&0nLHk8o zZkC`v!^h93U`bRXn40keso$7s%1bu=$t%L{|5*14spLJ@f%2sWb=O1lz4Q2>KB?@} z*jd=2L49b_G#+519h+zHna!CU2}&!Q_@)zX9Tb#VFv4SPYS9xoGpQ*jOk#!s^#x%X zk`~iYonh2~27I~37IL%cc|ocdbq&%B%qM?~;-Z{iE4FGVw_sO7&m(aeuD-Ea`Ale- zatoUI*5lK2_rSZ%F=+QE7rRerhqSJOvH+RB`7oX<%0S{Cpsa#X&uDa48dTxn#2XyS z+8!3s+Y6CuO63|t`SR$b=0F^+I0#q!t&*=IrCNRaLgr7+Djw3XmneL`~oc2od zuWxr$*$veNBb|n%S%exxd&?js9_MK~~mYZaKc+LO^*3!YWoivJvm~%~>jq zMp+OLhaqVUB#+NJB?iLk*Bg{Cf!pcH^qoVR%D_bYzrj_Wk+Lg%BnNQHLFn&Wk^W!C zzR?*hPyXx6tpAjP;;{0GgMqYwv|soVbIU8$JeWx!J;y*knY@flJ=2D#%xq3sNE0BA zWCNdVQM!teH_&_@xsFkOL%f=ZeN5_cRTn-tya*?(VSHj7cCg#4C4ACS)?P z8uY7Qwf|I0N>^(s@1yKxrck=6LT@!|@ogF5ZbPoJv44Hff#P!dJbtzO9?iD`V&}I| zz7A-Q@sHeujqskQojY>M_$ygwrzK=Ntov$e4^y~r;ViWX%(fHWN=e1K&guL;fU zd1A_x9?DmXJ-xO7eI}^vZ<1|q>bW>ZYr(9o-$KG@XH?l0^)nDgRApT!_IJj{h0$7D zyrA+tM*IV6?UYARnDQ|60O_o+s+NVYSWLKjiCKgol7)9l-7F0C??4)}JE|P6AlpI` z#%QCX=^6H&4kBlN8?Ne;_EaHn%RtQplM|D&{98!+t z7m=`y`1c=^HZ6ETc06xs=Ry831Skih?}0xs@)JVk6JxZ~Rc3&%O8c{%&6a%cDh3*F z2hw7~p283s4<0!fKV(L4JC7^wk08mHSX-NARrKYVyrKS&n;e2-3p% zO-@Ula;2`)pw&P7loL0FDlI9m3y+JN{-4w2VItIo@heXd)l)p{+#|RIR z_*!8H%vqWXjk2oAyPvE$VXRQKrE13^a1C>qb_73Yzi zK5mz0^SNzM-PfECo#Z9={5J!c{EXuUjbOF)ljz+_PVPo^`^yA+|JVwr>d^k8gc}Zn zc%zCG_?9|{&ehwCK6ApLdfy7XeZrnwzxIPUo7W)yIZtlBOkDZCToxa^4l7osu`6Zi zFgMnkKdXHV$~LFMw-qk@$aeB%*0GrFazt!QisX44kBPwtww~AZUGPSAuf8e*& zkMxeq?HHUSup+_({4-ken~!_RvWx`r;YT){d9V$SHm%7!818^;JMwX>zJ)wvbUo*_ zPAlH!i;c!}iM6bAe!N_{+Y%kD=1KqE|FlbgmWbrit9blq42Jfq0ksx1WMivE3mU)X zMCoq6ZMeUjJJ?fRn_Wkq^Duy%&|7eJmX&PObttUMevCPxBQR-ZfQ-5vB6YLhVC()~ z+M-L=T>st=vGvnDO^KTizhcyatFhA9LD;R~7EL{Q2fJGGW_oSh9)FtkPD~eFLmeQp z&n3;|hV9YgVlAi>cMz`RcZdE1^X2mQtuXcVaFM@17HCW|WY2xHo8=?wWW|Zx4I4Gu z*Zbs<1}$-F;X&4FhNTEj&AK?7<-)Ju#c-oB3a3tuVys`WzTNDw` zEoRQb8b=;N*n)0s#8Oj6bCVtyqwxLY_R_@QEFKDZgLDnKE_)nKt!)6e1Ft~iy;jn0 z)_qjh89M1Q?*3DRI}I1hnW>d*;SUp8-%nRs=F$B|0sElA{YmiseKCxE_Xbn%tyAOE zZd}z^yC}kf#}2iEj}6=Kf)kJ6)YY!Md)NR$zsITe(KY`+NSgVA{dD%?m(15=>BDfo zwDB2sy8kBDar-{pb!0Ez@_tP3RCnMd?P9Qt`Bu2)Jr3RsD#mAprTD&QclS->ip18| z;W+q!5vQM%mBlOB!d^KT**IEU+tCMQTWk2UB?mSQ36?dE`0`gPUSo3UU>UN-RF-XS zClBn)65pNDU~kucY};^phg@+>-YV!7r1fhlk7yZ8)0{;-@DhHS_)3k336C+|%hG0; z;i8mqZKDc1UYuYkb2pgqq;ZWT{W;J0dlzf#Tk+E~gWWG!WwG$QahA(||t zv+Pz>z~R0jX#V@8X3e-2nuJ$VG)tKyI4|CUBi*lw;z{3x>V*>Lp>i-Z}gFRD?=dq9(`9=w>xwCGEzI%vA4WBU^fPuOvcfL zcZKQjG6*zzh4go_TaP(F;}Et93uTW>?d0IOKU`^Tx&MvLter81go@KW=C;00+?fGVAyA!&!BD$@Z`9WktkU z$V#sOp57ZeYRxnb^$xPRN$;sw;(;)aM_D?+@X;;!3H<#(O;;Y5^YZ;kC6$y&QizCT ztF%0G#+Jwyk~RBIcGtd^M0+TbD6&MPM9VX02qhw0wyb5}_nr8j>G%5mabNddp6B_@ zXU;kA<$UHd15_WHjC*}>-sat~_(ywG__q=+X(_Kc%dqFIDo^9C%GYdYYB4CRT(F>2 z%bZ4uJ%0>uRg%fWJu8GXLHn7qEVtjEm>A4cQymBzAFo0SlA$&&{=P1Ig* zx(;ie3dEJG9HqzK4lw!VH8}pMout~3io5#Au0S})&OZDhtms_9(|_%G%%7F`m}c}SO7f@55ngQCxFI`4p&;qPBTM!`)9^z z6}5t|J>7@DYMuqno7BxkWNR@0EKO|{buT91qIxUkO|do4SSWzD?!;)YFlhv2bj z_?Y(Zt`$f*cuUhI_%IUB;JIKsd7`fwfAcLKsRq#V`Zu9J#zp>qbedT&)aM=dUIO)* ztdAz}oc5iZ-M$k)vGhXTs{AmG>5%E1@D}|~q(j`tPl7lZb3b2#M-7)z-Y#bK1N!54 zSWI~Ro@UnRG%~J6#ESYu=9R5xa zFO$BBjC;kXa+c}^lh+t%owqfYq4vWe!0ID5G;1imL^WBFXd(VPo{7XW+|BTw=&^J+ z{B6+cr2f}Pg{Ep3k z>+q)aehT6rl@G9Ox-Ql1FhSY_#~iJJAEsVqmB&+I|B!ZImg~x`om*i_)MqufY~1y_ zA}p;Q>4y|`J*tsPMmj?2DkPkSS>;ApyJa0n?tD<#F763+XB?*Y|Eo#jax~~@rTt>{ zLVL)d5W}WtE9_#U2e^XbSif`IU{$^$(wqtNJB8qQ)f*=@FjG1}(wHP^J{Hu`6G&S@ z1Dk?8;z%&mttJT@!7IUx`(E!MKkv|Gy5tvKUi9O{-Br2q!ZQdGf4AY+M$X9D!42ip z+5_a5+B<;oi1w$wRfU~|r|7)z9dV4W{E?ppO`n}$G-lX&xEWG@C=IA3%+@64^%oRR z-nQu{rnu@&0|Qn(EDQ-(NtfP4(zECoJp)LiVRUjWNjc0dFKb2YnOxcLL^+uqD?VZV zWcGd4kgEFF;@1^Tw-zzNWca#s4`h9eVs4eDl5`Q_S~Kj~Zv*~Heu9e|CgCuncsA@+ zDkyGCy0j5DcD{!J?c#v&0f_^VzAs$vRu{STPN*^P8olS@lO3*7>E}5WZNzn-lk7`v z8%df))3cuey>l;di}N6@;z6oy_9Ll)5xz@xed2$9G^!gCX5wBy6Fyg`5htD_Y}n)R zzQt!uD6@vWLEf~t%@d*OGIm@5iw^h(-i{uU_=GLK)m0QWb;FP|Gk`Q0Bi;tmMM{4m zkGTz%$6JBru4=0GH3#Dl2{o4Hzk{IK&wUh!+4z0c4SGMTt{_aV;;vzi>(S)49#6Vt zhDxh%y8IYPmk{5)VeRYn5`-U0KjWQ!fq4{bsrZUC9uP<4z-B&JqY>rUR%5O>H`^r= zN|tUT{g|(fKJ2IIY+J4+PKB#R#J6sXK>NJ8xEFGi5zot}HQTXctxtQ2AYUg=+79K5 zGqfJJJ)YWrhbdn`_rkk$(p9m?K8aTaaVsN@F3Q%`+ zp4ic62q&%*V?Dz`#mi{7FH@XAniZ#xjE0vLe)uEbnY$)aDr6T~0T|C%l) zy(O` zTS%CWNq0w)HhiwpN!uYJ-)+KS-#@d~NAg&~Ra%4WCUC=+fy!sFvUQt~u$<8~WW0V3 z<=H1exT{ffHMe;f`gGp`J{x-~ZK6?H?8^35Jm8J1_Defs*!Q%)jOhKmiohnVMpgpt)9PK2BCpJNf9WLaMDxG&v|Q;Xg-@yp%of z&?y_X=0qw_t696tAEH+$F!v)hdC_NUsrN+k+epSU2$5d~N z>!(@3^oD~J2GKrQZRN%5vXz1zw1PC+5Cyg6+e0~W3 z_UwSyM!q6l+81^P6cEOlkiJRPm~QQb_LmK$$rgLQC1{yAxR}-*)OyDZI!{KGUnfOt zscJk#uQ$?vL%Sh*oH#%?LGZNQ3AVaJ3iiTeAipOnZ#Lkh6>zgtM`7R_ic4C~z~y1XWV_6z!q7>Jq)iz4 zaC)}V5`6}|Aph${Iw%rIH)ErZy_JrmXF2A0HnTI|K4TA%F9Y(iBK__`Pe>!!ZBR1yut$C#k<2KJpaM8+=SZc@t@bNJf~?ao=R+O9|r|GV(W= zUTd;8b6ia*l{r*zAwaza&C&~TkP@{(^$P^WLsA|y`-HDm_e&flNz(xFnb5nlSae@k zqD`bdc1hziiW5FNSxEh~Ld8AUX=W`(o=BSgs)v;4aQ>_gR~o))ZZNFovr+k_P7ZCb zegtspwP=)vpt@iYy{9S+@*v&8KK{7PuFpw_vukxlT|n*?Xcga3qKqY zhz@^gf1*7t=)9@js^(aux*nW78|jKQ+utSG$@chByBkqSG?UrxO54qw}6^x zcMNV3k3HKoj)aOI#4TwTFN=*QU6k@AQmB^*$oNTAtcI5~zMOY91$kvz6~hhT!nTBv{|lU6RKK%1J@KM^NvH z`Z=$wbVrL?v(cvD2x+Wx*9!e^6S7JW0;BT*VWc zFy_6JB;Mn-)_78_uT@yenj824;goon8qJPHSFo0y1DVp!%3G4pQ#k}l@2k;RGX-Iq zP(2jNb!{~|gNyDal?sz+t^=XpiE&7CD=7wA$`|6~I-(xm48_9?sh?33m9HfZ7KPpRfy(c) zW`*qWiXshpTp(RfcrX@kmrp^}^SHOUl97+3zF;_Hq?ik3ZsCdvVaTMlbLisKAS?c*9jZeiID4%vP_GT#qb&Kjq}4$G zkUy?am<`m|gY;TYiI0!rrEh;g?_DEKzJXWX+{*VojbqfWAnclkr~mFJEbpLnxrV%e zhBPpKE7z&gdvqkm^^>4ba^1PD`(Y>hC($pQ`#UeD^py=i*WV zOg)fGy`pDuYvNksm0+$sg@$~4&bt%1JKQIl#07}_$rAhZYLiiULVs^|7|Ea$;L-$k%E_Ml z$-Z*hmjbLE6vj=uo)Zxr?f8uQ$Ds6EA6d}JQzrZQ;*~--{?5XP-&#{swk)Lwt5GL3 zZ+aEMt9`WIGV=}Vef0%AzLX~}om;^demBwFweN)AWHhJWV1DE8<4c|CJpAPve6#Qb zV>1VU+g1naHywt_v=>Fzyd>~RIR_bCjp0ilEe_~45?3_%taY=c^+yj&z{IiA{Tl5b z6MwirAL|^;I;@!qd3!J6jm$HmWp7(dsVL=6186-_SvFjmeqLibtUXWb z+KE4K7$g^z+e4$IHSopSQ!YAwkwu?qAx9fLf{zXDvBd5**7h^!cZWFAnl4=^Su;Ym zTx`j$_m0QD4x3<_U=xmkq zXGBVq*!-5gOj|AG3$^yQ}m zvmpFsHE#6#B_wR&n3uOrt6~#-up{?r+YWuMo|Hiy7lHfQv2ZNBFLbf1A>*3%lOv+r z@PbZfFpl=@K2e?eTPtH(MC2wo;Q5Q*z3L%rYGO1LXPkS!E581hf_ASgWcMLkgv*o$ z{FH768_{b6bWO04$%$W)VhZ(J*U^MTGC5(@U|GZOsiu+VCeYh<1|2tUhe`b#;P)mE zG5z;pQQCR7m^#pzZ@ghFO-s&;*a;?hdB6_pJ)9EJ)O+(AKbpUZeuX*XK7c`xGaqlQ zE3XV$C28z3+NG;_oZLMxwZehU#IoWvHo2hH1sv3&7#-h+s+L%EjY!U4cqhq`_Avgk_y7{T?aS0zVA}GWGBLlKkGr0zEPZ< zI1vrGqgHSAR?)=ian=2n*agTqS|3mMgq!Zp!a4WD^UU)8YQ7)n0drfl*E*87*?1!Y zI+6xFd2$e6e!MyOTCavm4?1U)&N8o&QHtq#0o*g^DR`T=!JdCSVb+XnrshmssRrMd zmx$MG8*$rkWB$waEPfwv%b4kSxNq75vpiFwIL(FcXx@c;KXr!3AzKKSnh6>!j?a(7 zu8(LgF5bhhn?}RJE>7rF*oU8}9t%UYA+WyhPb_MjMb990WzA{pVf;H|?CHRGpjS7R z8el5_O^Ox0+YaNmeNAOrVsoy0cO6W)(~X{uX(8ju4n}z=qZS2ePU&`l^6~&&+1gdQ+s|VO(|W4BQ`ZNRuHG_q zPcvcLs8rnkwo4S3rQyr@yGf&75;^1MVzkdp7IyoWHffp-&Nw2W``X%k*t;=kmwKIk zV=X+pdIL6XE*6C=r!W=g#9HRusjwIhBu(RiyN=+9h8wY3z*j6TYeQIZ5SL#bAna28 zWWi!R(pi$3*G<(Ztgvt708VL*rA7D{AS5$AITZTIHlv@bm*n|T?>9!Q5@lQ<7^na(Z3ABdz+ zI&$}*NP2#^LObw^8MgSspzv`W}Td zJeu79~wN%?07m8lKTLLiqf1m9Z{c(fer@x_!2!*tP~{O6wSojsU-ft;FEA>jYu1 zFvvIvoqrC579YL~l?Tx^_afto zjV!eNfpi^d@w}gyR=yN&&jftgqm3po@(Bi?_>P7}8`-m~L-@*Hftc3yI4&649STmS zGN+o6Q1CVhyPCSoKey^~%2BAup}ur*9${W{oIUh4QoPW1bbU$nA^-G0tZ-P`jq6Bx zd=3m|Pg8h-S8Cry>*@n}V@o%xM_PLmHHZ^$h>Y25p>fZ4l0V#og!fdl7J~3fbi4Ew z0=-Vb%N}&%)uQXd`@(ui+b|p!%$-f=!q<|Y`u~HAjl*!v_;9@Vp)PkDP*YAk`wbnI z9m7W9$L0F{&vE;Ma%gB-&U!EWh2HegfttqzFwUTNMRBh$8KPHK*ZYStw1I}gnuIU`?;iPjOFd@rJ}^*^LBpx7OXMIV@4I+xCV z)%k;jx7yaa37E3(sU~$D!29doK>UiU>a|1PC9^PMY$Kc%e;Cc`zeTq~TMVv7XQ_SN zz(@7jgRACJ|MEeU!W_s-vf#wUv`%O(Q*n4^9#3(2A(mgh3}*B0!pB}aVa}1*yzZI0 zkiFYU4%;*rV-IG6#?Fy<^6e*f1WXWbQjVk1u-jl?_M92C^un6P4JGBXc)oBdENr1K zLuvoQ9RZIp^Y3{^IK}Dqck&*Il-7&Uv`z$5{2Kj+-aQ~+>qz}p71Qkh zn3|uem@Vz@!#_6Mj~R92aL3*4n3c4JO`>NgJ}KTB!YZ+*P5r9-{%LK+bqX3o=N*rr zTEIdWoYI8t{h>wTXrOwdwE@Q zrC|AED{jBBfRUcjl0L*Gt#*T_`E?l9htBt*-`sTft%^UooYs@0Ck-Tic)(gaZPeTy za)>pX_>JC8v&4q@5tq%X2B%{eu^ZEqaY?W#|3940&nvE(qEY#x>eladcbp#|4AqJz zYH!+);n^djg{m>7GqdVvf?kn^6*YFwqZ~uR6Q;M#6C5Y*5mz#2a?&LfPe(}_6pfOW zqr$nlUGCz-^9>~ZF3`V`#)kEF&z6lHgYbF7pYZHyGJYCgjEa{rsxu#S-Cx}c)YOcCMy1Y1=3%buwo-*{nq6^p@}fa zVkf4TxZ=N6`B1cM7n1hDh^0k9I-j*oD`F2WJ;GlG15oAW#m0s}BCpA4<-=ya|J*J9_r?-nIJ+1@x zY$Wyg5cxj8Qq$N`2MNQmc1Cq=T)}9m)3i1!-A;2(__3o(L#1uF&DOMu6UCozVO+s| z%0sI~ZOJU95#+D@HhA;G zdl=y?S=X8S*sAU=_}-wVXu576@s@{p*0u){&IrZ~ui(mGqqC$v|v%XGNA zHU-;H$%l{8-T0s}-BgTJ{9skJ$Aa*cy&IFRyaK1(f}hq2NZw77UV>IDe`*y5kS~LN z7Db}uUXkK<@&#M8C)ykFV|k}>LbIo+cunCC)gKT~G0IE$^^W#oT{ReHeQ8vMWvceR z=rj2+zr-`EntV&nagWaf`L4dpd2DC=~GpUs*d7TXa&^|ZtQE%-Pb-u@-RR+$yKg; z9cwD0^?UQ!2~j|p&wD1{z$X!l=^c?BLg_ib*QbGgn<-ow7x!2=O~1!Ny`vd62*UY*?`vpKzxz6{lPe{zwU?PIsQF5UNo9l3&oW+@Ls1Y zP!1|Ss=^S`_Ka!}dxv^4uSX3y`3k}ZeI8oP47}%TgA(^_@==YX@)@?(_rtG&^O3Z; zq#EGyhf9DwDA1hZt9{ED;S}8MHXaFEt8mic^e|Oxpz>tSeIq2CLE;A7YE!d{14ti= zPyJVm)3M(e;Tn<`1@epNHekKb+}j}L7}coK5L6pd@p{4*2}@ehxqP>Lgng%>7 z*ei}FtpTL7xZ^}SR9L%c!~?v%vL4~|0JgKv7saW<>sYep#zT#q_jo4lb=e>KjhO*O zAB57gbOz{4h1HyLil2M)XO*rQQe_G5}MPp?0cFE`tSbGP@V9IzlgnTs?gHCB}$ zOvku8t<5mts^*{h3y=3p?qJ`LYD%YRSB)A3CQr)p2)9u68yZ@U1LxYa;hf<#p}a@m z;kBsFLMR3U82Mq92aLQwC!PZGI?4kmPlI{kyNQp!G4g^)8d1d?R_!!rhbyY{7X79m z-mDAl*>s;&&-0Y_VU(LN>0>HH`@B*3qx2jueKuD0ShVD6nDV9z zXB|*JjFEo@i{D!@(an&PF9PyJKzYX}EjLk~kJb3`O$1v_0rFeySo`~|J)Nma{t8r2 zV}?fxOlyV^T5Th=?7d4vJqwzzLPnnzVKKMJdmG7q7QL{^%`a#^+k!9eQj7L$%EtwB z-y!)xLAilc-*QlgK7u?HBW%I7&ua5d^!UrV_X1v|yB>a=nTw>6K=D(8<}w)4vy{te z;V>i43{@NhUtYubuOB>K=d7Xqx6Yx$swX}Cgv;r5VB2Ugt;^$Jo>-gHcp&e23@E>d zJ;~F=2P1tBI6YUUegqJ6S83Wh)!Ci z$;g`^<%v*!GjsRd+-?q&KQ}19uI``u3+$J{ zWw6YSLDFp$2Q!|~v^ftrPhN_2lj_&J>k$Nb;mue%?U6xP&Pt{nMy0KkcOo6G_3>}P zQv+hEJcQDP(G@DnCnp4tEc#4VCvli&6O}RR4{UzZCOLk{D?>TKCXeC{3braqvIdJet* z`WiQtUqte+B56ykhV+cWbY9x|9UvaZa1lyAur~E;u8K};&H(8N)jwf-Ga7Khe>LBn zdY(Xf0(6>2itRz4peQvSD&wvQ@~&Ei>J3y4T5m3&8>RIY& z@>Fk$e&@34XKIuuAq^(*)gce14+cfJ3*rb5(zwi_aZmW_I)~B!YbmF2Ky)RNz69O7 zden;yWJ+hy{1gA=tGuDVyNmQ^u;ge>P<@LUev^^CBca-0Z^oJ76G3xaBw4gMFYbSY4u zl~E5E$>Yd!`v}5(JJm-Ml)3F;4)p3Bn$%i!m3|dyHW zY|H*t`Y5jHMd4|m1k|^Jz9DOnv?29t`?2yNH8gvT^*QNE?qutsdQjS5@0w9xYN5h? zrg}5Q1I`NS3uqNTtA6U!IU!0ra_W6B>JzBGum^c@q<$mz?QtSc^#bfqGD-N)#_HKX z+W}=r_ykJlQ;!TOAK=e&JD5E9hnl;*jZ1fu5A%cU{{;E29pL!37LxDQo=!c5N-yP& zZsV%7&Io6&`m{iGLVcwoM#{M7pM-;7 zGhR`44%(y}akgfw$*GQynq2`__pUAyt~YyJ4H<;TVLXY?Lu z@>Sg6Yb3W9naZT1N!#H*DMu|tBR>jdzPkHg@3 z*5c6PRS@=L5w4sV4VIM|*e=NrgV(gg_CFEV)cP&Fn@z;x+jbW=>^Fg8EqR#1c?C^#{sK4l-d?;LGSaD66cKr)3xnIUSUgWZag)^Yl z>-Nm^oC9|+wv%gHc9#8ZXF!LYHTa1}M({W>1)kMQ#wJ$hpiyuRd-BbQVvvZ-h71Mk z%k|}gx8;!XxuYy}-;J+VPQgx|jp5m<#~3=diEPoZuO`~A2R~fe93IZSsqJ5P9KIP7 zDD&cmApN(bIly0!YqPX{>tOVj?Hc#`H8g&OS+e=$Un1dF0Ma;t;)*>YZZiKz#keNP zO`43L_hb8v=KA|i3U`-j*qip<-ulcImj*SJHdDT%>!BD~F-MR7?u8iJc!AVry=1S? z^~i1CV>>KaU?JCgR;Tl4n@IhAT~r>xsonIx@2o;NPh9KNX)=5Y|BNRc>hpwKuSKJL zBmTw5M_zs1n~(bFhvTPJgYc*;ZV?6}3$k(0vg(pM2f*$d zHgfF0NIp&DAxGAUgR~~4ILiY;#OlhPUF~u19}9jmWdf(gnPRV5Ps-O3`1{jVxp+?y`E`)GB&~{?KNkBlPb05s$}J=f)!^;oebuAn<3BN8IcdaJV!@G%@O& zx7X|}i`G_?7Y+USRBy&`cn6ic;(6kCX7Bb8Dej!=3)y*&+`)_97+P2aMJ z9b6`j-o^AvYl&+~R`Q-pI>aCB4-^xKelbVovaoU(z&z|CMeSq#cx>|`@htMKrsau^ z813u>1E2JOd6)Z%hV&ldIL_MXSufs`_m@ zxh3BpdRp|jkg)3+$3`4Kgp?1=XQnrQcKZU*{o}$n zb#T(2RQd8*FduF@MNW_1#VhRgvk^@`;IX}36DfBq3K{&M~I7U3FeE%3VpZP&F=p1emecx(y-i zc(jJqizsHV;BVpCOd+bD` zaM7FgDyBKlYckZDC+CF1aMNe3di54;NE2)N-cIn#ZjF6AB*UrAHKA5xfEm_5vDX25 zuQPm|9Adf&;#MAE*9{`!(W+=|(NI_J>GX~@?p0ToBsYK)R~*5sqYYH-_>WV)KFiZe^xhZU`>WaaKw)OWn8tGB?Rt!`mg=_|WYB(AYK)0Dtx%ga9ceD)?i)K$ z-AA?l*O9mpe#IZboUHm%aoyUx57Fi2aTrm}i2rwg9i2s$f>f{2ozA$UdW1g-3YL1A^sPOYZV!o!fg z#|Q}D2)YN9woy+-4`r*T~mU$hza z7D!u2&z1ey*C&Q(vCf4pa6c)C1E8U?fFAUIVblLQ$YGzZvV~Tw!P)Y;u|d7FlSm$Rer7Z8o=*HKL_G<_IE}-97E4h31_8F;XrNrzA~U#i>Ci&Fv52j zzSWX@_^uGg`=lV@worUSyeI~J_Q8kWH$Y12U`F}`-){efZN1)zjfq_uaS>{q8_2Q= zy=kAEVEB;BIn4)OS>j6=rz58i?~j%KhD@AZKy&ew5r$xDyISrqR&K)YXZy%*1KUZ$ z9a;KQ2h=r`zL}Tb9lA#7pihVY)VMIRyDq=CbDmWB>p3o8k>Y97y>x4Em8iJ-XhTFT1w^NS|OrZit5PO|EkF z%A=YhY`=(p)i>a%wyov%um6~h`vxGMgUZmQIDc{|>FY9hl6DxUt!sd}S6ky%ucJu1 zfyG2-Yl$z#;QdX|(e)ABog3s~^Y(x|b!riIt9OU;@n%(>DBLaku}*0uF=c8E`Iw%Q zlm3Jie(NYtjOiJGK8W;yR+5Gz zJ}4JgT~qPGt`|W0rZN7|lr&YOhH9GmEWZY;y)Ls_jZr9U&YbQ7e9lP?VKg_`u$qxZ zLBdQlJfkBY9a_cfPMe2Uf0d%j=`G*>Fr{~+oiBlny8)~?;f;@0)lvGG?(Yqll{b`0 zb!lz-3t!S5(KxkDd*pMv$o`{GpsInmmF#DSVx1@D09R@)L2gUDz$#TPm+d$e2pFFS5h`(6Gi$+M_hcy5Fl&8f?Cxb<7XN9p) z?Qsk&Si6IMqdGkB?T#nbUBb(ju|RrOD81A3tG6VN!2k5CCjXKLqT1JzF61pHwdSwB zMroARp}NG?7pp6MD$ieXQhHZIbImAsfpnN4oI+3YP<)-IMRm`lDe$S*4V7l5yo3M# z%)sQCZ73GEaA>=I*fyq*xOQSSTif6m?tAjvW5Iv4Cg_fr2rzEUDxzknvB5FcP)piH zqwimV6=mK`yXPT{cRYlx>~*=z%TD0#IbZwgV1`C%|BZ?HID505(pw_;>H*02ZjIZ& zuI9z9wxfy7Nwn)(AC+#O`)5m)?)i`_VX@&Jk#@WeH-syiAlVnr{wYD@56NI>*bUjB zw5rc<7$1jJlj8S&4?#69zmKXZTK~$!o3HvoY(1&&3Fa7oB5n!Kt2Z@S5T+|F0>pQy zJXZZv8+hiixtH>3$KeCvtj z#V0@-2Wrg!9!3w}KZ0V9XYyNcmz~3~_&&(7Z7a3YhnI)}H#A5!h;L^Z;k?Ti$uE?< zPp>@(y>sbY-v>JKUug(7@+iZdxp(m4+@bV-dUp@$K}hQQK^XTdb2BTu}UAV^Cgc_&sjZ!ivQ+K_zlb4Gqj>*7)m z`gWkT7+bC(#f4oDyaNq}zXBJRSlpvSXQdvrkl8k6?mGQy;P#>z!tEfw*wBtsy)nwe zs@$CRek78&%4;_CgHT?HI1eaSm2ZK92Oi>4ccI2rg z?+G=g)~->o(vtdmo}^#Ctq>{?bo#ADHTMOZ{efx&c6GQ_g`dVFpJ)b;I#uQEc6O;v z{%nVs_3l0+{3c!t#@FWxlqZ8-VO<3Ib9g%|ob~QF4?iWW#B~OL8TAh!X|ye0Q){_M z^S9@3nzaSO5EvZLml3Zs>jzD+0X_RwHAA``KOD%Sy?v)D9+ltC4gh%u@qWMs{2lOA z&^4sWKczutPSKa7EikFBCv=V+49cSs&LepgRDR)P>}^bGA0qnsxB$fm?qH_kJ0KsU z^0aEqH9}XY&rtuu4pfW?ljzLrC7^l}VFCT&iWrBlSB#A6pCQ zINue&-?$EayU_ZoBPS?6f0?C+o-|ya4vJIg_m!5c^099s=AzBp)|_i?c=v-_l%~O1 z3r#usD;(Kw8T_|tzo@-riQ)}Dz4m+UuH~Ir!$qsKXE%ib`CZW4Ks{xuL*g-hCVwyU zG_QpeM@Z=$gVg&VUfhmXk9<|}LE>XZy(4XA>38AZxCzzhXH1R1h%xkDDDel?QnJ$5 zZ1P`IPIyW_2(`C*bVB=$MOgcv0}yr!)d%{#+!AY6YlYM|LFygApKMPdjjMC&LkRMU zxag!8SN@iI3{d0x9Z>UDzfC{brkf4Qs|CNBjswnSXxBw&h&<~JTJi`uVulY>8j5fh z)*tSnydNXYNxs-NuS;`NKHkxhlb;k63o*{bQmXq={k2{Rb&xn%OWp;>wp}Snm%-W0 zIT$rAUPF43>a;!1dN`O(dauu^mw@+H8v}QKO`KJod~h>)dvKSk`uRwE_)Yu!1Yz89 zw&h$TC{3+COI}dWeN^!c^*gG(YRM1}q)CZb`SLy@xD%}f9_b+4bu!B%{U(Mk4aOh?2evwhDCco4|HseWKpFN3(8Qx6o+EG*Kfp6L5A zL-4ikHtLgk^W4)-IC)DA<(XJJp{{1fQ#({VdSTZzM%>Hxj-tK8`i9`RYiCGbe_^e? z;u+P1cJ!7;%2#@*-T`S`={%UuT{pR`Svj!|?St1t5H8}5FYS6x)RP_Kn@rlq9*3gLHk zp}Z2!kMgC|H%!tjX_bq{(`SSCEH9?w(5~frX1@40Q{(&pp2?`T2Rv%_)kne!I1@Yw zc6N!RbqyDg@>A?>I*==01mimKHWLp)`o0m$%PO6T@drzYn`;C00mM_iEllY)_tj^q zpH-pyyEM-Q)bH4Y#9JD|boe%N0_!-Ty`WgJPCrh9M_D^geJ-TjVdM!^PBQZDtZ7Ct zQ+z;uaHUm+!AMh1If|sIHLqrmqTbnNd@^nTR@m7SKL0?{d8!XdJ&dP^Cg5raOBTCOP<=aSuX-G54>$07O`i5XIGKTPHAoY0? zL5-h!NJ=x%xeAw%`VB1l#e2>E!Yx2GA^l79@le4w;-?&Jwy6jISE?_SulQJ0#Qyv9 z3W*~*^@oXJ&q55dIXzvHZslFl!PIMy4_7^1e`W<=Qc5wRj7R=Eacw z&w-a2Ed{qgTO2#06^1`+0J|2p!hz2#JONt2x9o-0zJ~Hw3F&=5E8cvX37q6BtA4kJHSinrXJhZh_N+kexJO7K;Hf9GxrSUPiyQKw0Y_`XCv^=Q7Ob z%F~ZnNd4*tVnhGh^2N#&5mubX==-=W@H&(~o(0I`*$09WSE*m-Ma$cVO zS;%I6arM?;__F6l+&9fv^i6PKWldA@G3{qlw!l)-_2u*C^lnAZYVv;GQDI;gCmge@ z^IW4BNcRPQwz%?8|JN|RxE1ehmm~HEXTZ@vWBHnM>v8{s%UF3d7Ei{^;@iIu0*gRX zyglY6c-t6=YDsm#e^y&~bWSV7?aDB^MGZOm{9l~r*G8TiaaYr-I0Sx(n91DS#ZcbY1gwf2VCJqb5I^}FZuB;Vu~|i8*vZdY;|3@3T+cOd{Nf6{>)DAv zzC2xiEqpKN8u;PD7ND`AfBp{iH;)3{E5Beuyg4ih^u)m}Ut&YOnUZn{|LmBI^!w7{ zdbPa4EvJLbzapM=&J_l23go-gP<;708RMJkb5_J?J)jYvetapuMiXhbXeP7$WXkU> zD?*wNH2l(0CYpC(=G|M!N3$DA>zH`y?cW=_-aLb9Ts7&PoVl8R!f_(K&s}FRj!c^k z5eYYuz9T6gS-1P2aM7@u&~=5K9J!|psJQx@oA57_Y-L~m*3jf;g1@n;nx$i z#;)xms1`PkzqoZ8G1&2Hu zO1~HTVcs8!T2DK9GUhFuoi>s8o46Z>8g=6TT5ZDA{yXKNh+1+Lzl%Yy>(LpOdKhfd zjMIE-E05O0r-${qVXxhI<-$7l8@BR}^SAL12inN{=DwnMdTpFvn1)L~Y=rd>I@6r4 z5#=$fR9rD{nJbs(r$p$PFz95ohlO-JqTOLN7P11`;h|@#@b2wW+&ex6x-`>cwqYM} z!O~Z7ru7mwCnO)P(fie9B{^`9_Hm%P<132pYy5|=Vp`8Vu)kw|UO?BDu=>ye2ys}$ z%Fb@UuK`yy7VVN%yYPs0;FOMFy|tITNAGLT3F!(g!W&8B0w4I9 z`AVDQ+lX>J0iq@r-~i1#dam7%Q4Vu8pIdZO*;3CSQR3l%l+@g2bBB3;<^j*fM1CwC)1nPdK-j2-2rwrswI0>ci^^Ofqcf| zT%@t7&^c`_XVr?8R8PF%b+&BucmuZ?5h5H-cVWQW z7nMye~!_dNuQGFwuPPLXwAtdoR$jB*rm4mL!E3F8)q;F*nEp}6`xIOxAZ_*xoC z&wb_aev~(e?-OJ_qs`bosUrlN?55w@r*%oZ!In>{BfSmC&ldLQ6Z`q9_^@8-c|frR z6_Xls4v}WI;QJiR^19R>s<|5SnR4YX_{HVOt}CcVK5RXnSuqVLH$=#-I^3@L3ZU3R z`QEmep6!MIMQ!tl>@|gldl>L>3-xj2AlfUQ7U54XP6gjWD;XE%fC^`y#t(zptIhcC zX1Um|%!Ct`D-5g37srWalJJmK3o92?+X~N-upbubZiRrZ`{2KdT71}b(tf#%WtxqE zfX!QQt(OawJoA8WcXkRKtHZvhM8NZD|M445{^GFf&E%}?6iB)GP0hPJvP2gyoN;8C z&hxA4eU9@$N%O@%#p?3cNbfg(87xmXJCB{SvYGZ}u;5#VQ*H99G2U2j!yMjjRyZS5 z;*YcOD>`BQ+}&*1j$QJLSx1F8FrmX2Ou4y;%`!a$s^*&4Z_JzCA1{}u{gcmX?&E}! zBDTpWZgq4yy!Lw!7atnRKAYEx=+(OwuK?8;b82}TADs#pI@KNs!XKeiKT=_dIJ!GO zcZvNTJS$2u@`xVNHLB`z$+CJ>+b3}6TxfR~At2Vg5gw|7ZGK-N&$HHF)lVnldKjDo{?N&CYME$%@V3-Y1&4@13}$ z6Av9~TmV`!CX1&>s<=V@y4|_KRXeHbB)Y{~k5T{K!&Zwe1Rd)@7}6E>$=@XHHG*OG zn`Nb*9&<=YQTPm#Og72^&RxLvQw;kaH5v}<)C9_R_GJx5^5{DC>L-4U6$QgD3(?`*s3cg_?c>E^#GOFHQ(9XuO|HLw8z+f zfU)ee>Lx4j8VbLmfh^y9SS&Yh!v6(zM8bP`=sFgwJz33SFTQ4VU+D3|k$U`B$vI5o zGa+zc1L;5OFj{_(#3q;5!s6T^Y+;-?GxWI%#4&`Y)$r%phDaER?>66IbZtRn_E6(r zPy30CcHK_eXeEwoW+4eP^Xv_1@AC^53O~d(Zzovw>J<>i@Fmx?u*;b12<{{OR>@nvOX1=Q~yZ^ibr8(93#sMca_l;9vM4x1Q&?-=@ zu=2sensE?WtA!-J#ppXcvez?(ZD{jx3VSoR1pSBq6*fB;iEduB@2ZtV(ruEoo+PYb zR2$&T)VRI9J z_?Rcox(ub0VuYj25H#nY_;G9Q6kfV}5FDJF=59s(L8T+|O788WIzQgf~G>@I3v~H?zl%U*}>iTVd&%>pjI+C1#{ezGu9ct642tO?WP1 zoGYNhmjhi-;R>Dz?F>DvAF#aZH=vbWOK5VrhNN)|;%%H(opIt>_Qb80ByG*-TPA|h z?{U27iVi%;uMo`!KV+|xGk|akKQyjMnDQ5Y92B^sXfzT(;TOXoB#xHL_FPdls5nte zdJyIfNXHdcmyk|OVWk5qP`CMOHE&G$g~PoTBk5ojKUCb&IID%iQaJaZTqtd!_~=u& zSQb|7A=}6PfdtwcKQ!H6>Ko)h-IhgFIB}#!J<2I7Hs+rJy4Ed){Jn`tF%(qia=J}J zuJl|==PeYs93UPiZ%_&0b|cWa^tsY#hPY|}aH!K>Hjij;I&U-uX@~9kaUVkT0?&MV*x`*x?2I8?M zZ$WY9TPI_<(PR}&H+aQJLom1N8}VTJOxbF}RLs%MW8FGDCLE~2l}?`ebQE^}_+1m~ z9{@9Z-9Y>FRGPc)Fu$-3C=J?h_$WSRD?QI$(o$|Wt`5Ezdvo#{I4U$8Of>z#D0ME9 z|A5B}`vBFfhVGB<(Uy}A$h%_&`=ve*vydD%Bpeh(OW+(6O+V z%=ETJ(urVYwVkcJ7XeotuOo2@AGzlwI&{_LRAW4GN>lI@(So=r?@a3-g1i9S8*45H z?v6xHQ3s!FY{ma$2Y~zz4_dnr=zh4@`B^xc_W_s0AwYT$d2e z_;qHe5G@j9(D~_bxVdR5x?6_OtI`b3T zIG_u^_O32zu0ot{vkjCdApMPzUlu~!f`>r(K=~Pni;T|Vm>xgL7w9pc)oal8uIg z%*c}n3URXadn7GPc-Th%_aav)zSInVi=UK%9c3iY_wSq^4+h{vHzl%SrT}0`Z_`=wv#Fdc6%N z&4EfgDC{KMrrO*L6kFyT*OE_p(Dr{MU3Xkh-y3hzA`J~fipbvTbDm>mX3Hi;_TJl9 z5s4_J(y%F`B9-c%=OCNNYS=>d%HI5*)9(*2)aP?Q_nz}S@AbL&993tWdWUd*i}+Wx ztKzyzvy zG0Gv8db$CS|G_T~rSPDkz8tZwn!;g5S^-E$;q>^k&~mOLpHa z5TpG}>UMBQEqT`BQ^Exk_uv?%qxrl!+`E zU`~0_B-v|GG9xbxx*1M<*wWR?n^2ZisC+b1Ukl1Z1ZiPZ`Y6F7NaDnF&`&xpbSJ}b zU)2n(AL)%b&aoAmm-01Kp3P^I2{*mvqqMV3sy>dc{UjJJybFD&UdKs4H?SJ#4+`o% zByWxvKZO$)e4;GC6=vQDLdw>J@^TM~BE)U;AL7aKB|6I1cy{^)+&W<)J{xIQp{LBhZ}KpcvqytT@+n923M%A4c3*Q*&}0_%}%MmbCuAa6#R zbd^w=i~KPDYiP~mx3!Wv=96_%)3&j86TFag9E*N&ku|ASSG^AdemoL=0$fDDCe`Sf zfDK~)3lA|SWow0&q5J`QX1!1~fUWAd^IO9-m~qUN|DZKsjmv4+V-lN&D$k z4N~rc(9TSf9_Ig==P9fckMqoRq)`>F;Js~yikm&tTYqHBXLR7CrI0ch^6`(gLyJb@ z=j_e8Wq)S@@so&V8Y51|Sxt5^l|$~W9IUjEPWhBS-d_Z11)zM8@Gwohwrd9DvBhVr zbR^6aDm$h8gDE{g`3QF2x(e3)OcdHnBf_gsy0G}6oOo5#&e+GuZ%N`GR@T9TJkDCU zZ{eyLdDH;XcT(vi%IQEioc5iK%2CJ(g>9Z7Ha+Baa8<>qJtc&G0B?GNvyi(%urYQqc6R0jG=!2zy8)U1F5E64q=4(p)MN#ttK5bu_1F zbvd~LkN0eCt1<~eUL0QaJ;})5amq~u=`YIl;(+uCQ?GFwYzmIXZwND{GRo34#ov2g z?^9fo?zqX>hPg{JAC)-641CPUk*Muzhm#W z^|;5nYm|r4y2C%6_#cZlEXXApDf`n%%PIav@}9((e}&?ZA+(>A@?{erq_UEOjbvq{ zby+MTLHSG4kE*ZafZr*oJhkGunomp=|60YtbGRkjTsvR zjjLAn`n(v}{4E9!b%)PJWso`74mQM?z=>loaDeYRsDJ7b`>~-9JR7?K{v0sie*Wua z%RjXD;;>+})(1X4z6JM9>?)gW--v_O2eOjrRAv}oi}&7Z#r>P!W{z2j(75cAZfTo9 zmh*QDd;5FOYCy96B*-Q1qjK4W8Wd23%jBRXEL{%-?ZXl5xVdB4%x zxKBadl2n*{Zx2LQvEcPT-|}4eu)ZjpmW@B(zraU#cH?$;TAOH@sjR;t2`ArkmA5FH zu3hyNqw7W!pDobx>UWlT_ky5n!+vd(VESYiv1GP}n|^l4`dp#kPM4n78#EHZIJMi{Zxd!O@dYerpp92|h3B zb7knxHu5>G_4=(r3wb}gnEe>MA7~{a@ELX$rnR$`?%BU^u=62!Z<&!>gZ5Wh5w(_m zEP0Kuh9di&<{+20(UbmPGts>7DClF-3~%hS=2^w9WbB?}9<{+(zPD@*qbkMYq=VJv z_fzHUPWXJFKNvbC0)5^&q3*GtRG;Vgyph~HzmZ(=+e2LP`w5d4hp^j?hl>+stL5*f zsM|g)7T@2m2T^Hjz%0;$dsg3!r>l*S%dU3P*?oNji!Ewn+fZxmo|B#YHfFFWoY7mp zd#sZ*K2lxVrc=IXvfe{3owgSWZ>(42O3!7URAY{#Zv7SEgQL+0tHO)OuSjFd8oWP+ zvweNosy9CPb8S0_+202PH}-|^r)J~zd+m6HJddqM$HV^ieLZ8WR`QyeOL(hB7V==z zUGT`ONc371hmRIK!8dOv3aTB9e0qR8*1n2k8+x%n2Ug+1U)H?!*~@s~qLFOebsI#u zE#-k<|LMN89L;Ud>Whho4@sNE-*`W;l?({!1xLGW7oV?mq&ZxJieZc(&n*#j z?rfRICz!CJ7XP@jha6LIgU+#BWRdQ^b)N1Kk>T=#SuLn(y9er&oW{pO8KYcqB`~B7DH#_;@9k)~H-Svg`ZpAX_UAzOX+l+#N z&1XQG%MGe;TmE!nUAen@Juxi(1Y5X2NcS-HAkaL+f@PspKmFzLL6=3n^Zq)j8?0(! zPQAU&vvJp2SZkg;?mg5GK98>=hb^ticLayyq0L;nUjE89Onyt(Gmqt^0F-ymgI6bJ z!XdxIKtH2(>-sRgyKgni+avJysrtBOSs`IYQ~9%2FpTIimC@M2$C;k|<)|Avx;E|Y zf3&9I+h*MSXbnEi9P3OKAO|`Dclg#uNR`hWJ?|uIF;&&?mG{owTFja1G=7E zmzKw{LqQ018s7>gWOf#_8h4ZRPkBrF9A4dhyBtz8nmZfO9xj19*p-lbII7SbyU;q` zp=^QNF(^nkA~^~kS+vv+BupVp2}i;OUFk!5erah<2zX;6oVusuJ$%E0To>cnt0qYE zgV((OF*$6UT##xnr7*jYxr9|DA}eCN>5x-O?ZfbdI4IF0xF z?IlcEFW&Ttz(J#*!qclh*t45C{>n(xS)>L_Z|jjrHH}5rpR0M*-X3Yf37>U+uA5@Y zNJo(jPjvH~7h*C5@b)hp=v)m&(yi{Y_P+L_Lwg<1Z&-{=Dz6i>-40`$VMe6)Tj0Ax zb}VN9Wr&;h9;i2|mIlGOT{Dns8y@u9Bvk);9p4QKFSF>`$Qnl~!NFgV6@Bl_#>uaz zZD3zg4q@m=;s3P_C=4eoMfF;b)gyT6ep{ileTS+~+STExUp!1+%`DVW-EqQ4?BNgv zk1Hog+i&6Mwze8<+r;?3`=cfC6dq`mjhllu!kzcq*x*6AaA)`c-XLeYrut;{fY~x@ zM^&6Q!$UUASf_)|6D8G->~lR7Hoxj9Uk`MH&M_x3Y1i(GTK$#LOw#e>6VC&ja2E1X z5D4G+gx;3C{)nxZ`elTqo`=jT!{7rnM!mUA5-wxWKLeg|K!cSZEtujK>D&SB&X6}7 z>Wo&rt*Wx$-yqEYkPdA&H6)C##K)efCf^;3(X|bY zB%b;WRBJfNrYC;=na%Ip2V!8o1n4mH0mjVPk92;xd}|`kbut9PblhU?B5U8fi+-)% z16^kh&4KJVr6;ucc1QGhwFudKdrthsvoEg&sug@$uNi-}?h>|&IxaRwPk@w>ag2Bq z|8-1d9(}Sv@mk)}G*Fz|BHx&=IYjsLShbP96P#(RYa!JctKxMS#`;ZSeis2QyLLk3 zAA3|^Gllh=BKmt$jmzL8aqxBX8pXe$-b>sGO|0Gs{L~Fk&o%%$CM4~0p}Ml?hciPV zV(lKqm5_b866VAe!Jtn^>ACFXsCrIu@u*w0PD@r7!nM12{a{0W?eKBDx8{O~Dz(8L zufBnvsi{WTsZ*S(=7+El2@5f8#6rS|jt~M45LCyQ>oI$z`Ipp#Y`kL-?*6YBM|LSh zT^CbM=SlPY69|K-XX0qh+CVVa(V8#lJCk#>Q&?yeryHK)E)Qp#s(<6aa63+OrSyk( zeV!4gvEco--(nAzCE~fM7qu2`JYZh-Y3+1!lJ35}9^ZMamhR%O`?~%uA_!xqDja3> zI@}R#DM{CW_0~|d`#K$R;)Za;%ZG6L#}XtiN7euGU2`P9%5@FsDz2_*f$NQ~W9)Qm zxvhRE62{=z54Cyx#M@x;(7IyGr(Yjjp?gSIXg53Egu`oR%8R2y6d%CvBM z6r*a8t|t_!=9$t{Wo@Ep%)a8#V?$)nx@>>9*jfhXGrJ3iMN^J z_`2=-KvdwX3Z9)%>odFiDg?*p4%0r}7$RYoJ}CU8c>>Z@^nB}5W_$57Y>K#wb{jP@ z*J(BOk2|I24K7^l59!{Sf^b~vL`WU78sq;Y$%JzT^7u z^ZS9ch&VjB6iIs{u8+bW@{a09vAa)CT(x=^q?MnBzg@lf_yP45E(5#Y3-0;osJGKl z^G#57t#q`YUKP|Uv>vG?-ZX746`ns_SV^`T+X55DdSRRaN5#9I|9Oc{SH|O~DX#K& z+E(aq@CEuBI_Q*kqj_L{zr1Cy(tNhS&YKaR;jD2lE3oHQgguxCuat3~W1;SqIKr?o zQopza)U^$N(4M0Tz7O#HzAUuM9>#1~XfX zg2`1|fn%1FSaW|fs(Q(+QUX5)H;HWlQ%IWb8Rg9l zc(ckUh2s4QwdRA<;Q&VZQHHiR&hoeFde5pvL9=RzAGGGoV1e3iD6(~ z^9to0t+1HZ=r`Drp|lXX2M&Yl2lvCK5r=ZMb6419tqR!HGl8DVoAW0I#yqBD4I#qa zrPb9`@(Z5F~WQ?h_Usq3#_Sny^t~esP{JclLV;ZgJ z+gBHvQB$sdE|IQ_a4sAgS%>33UJ2LO(!HxV9rr9b2~P}YPoBnA#PQJCss}(0d#w5S zn*reu9cwhK{*Urv;`3WSF0$_dt*wIzvLOiV0G< z^(61dNkb_fk@JtH!n+l}fV^Erohv+*?+z2jbeDre?~6?hw$U-G;qccw zi`|?%7KW}&NAlr7-US~V?S$lMne&aq)T8ZrYKRs8Fz+fT|McoDJr8BtPoml5*?nsTC62hac3Q~n<(J@Q1-4;s}C*!rJ> z>-Ub~ncWe(vcf4)GxG-{Plbkt1HftV2{q43qf>odQ65%i?yZN;3FeaCmouks91wm` zAC{7LIe?en)a2(|*Jpcr92Ma%m3jAPMlA11Dcjq{35n|<^~-HsYW|D7+fk~Ko>(se zbvD2E0eu!yPNF0I0ZxxT2s#(EOztg7mjdA|#8~~q#6~-y%gD7#4+-NRu6)PaJS6Vd zksd+BT|)2f49`FIz2SW8eL$bbid$N7C(9}dJEiJB-`3Vb;T~ZlsvZyD;E5`GA&uW| z2z}?UcJU3Ebt)1a-gMHDmQ=We?X^EZI#&5(raQ#Y{=-o4o@*!x?{rnH>Q-<8`7q&Y zyoi%t!m)nOD|9mX4cbeKp8vC^y}~Oeh|yEqiA!I5K)YjRQhdK#f$2(%t6EnY;9r#; ztf2R9pk9G^x;IzV1o1fB8n-;Rqh1W@@AjPfuEO(>#}br(u$LSCdBpo7;eDV1;^^;! zIE7pE>;}r;5LSp^K|86omx-|-Xf7`|uE>WLKHR6X>=1{mvJO-Il#BU+bySThJ%Vrg zR0Hq3Z6)D_Y|=3VjmHhu5}IeJJ`&_<0qYmSp@!zrpzd|1vaQpzS5@GeU*uLG4}c5z z>r0wR)S zcPm!9g?1U=s8J3gh<_x_C*@xfl?PEe^341%I;s!iZ7-2-Qw_<-0QCgFy&(vco~QQ; z(*K}1&-3{LNppb1p7aIcNJ;)znAhy9v>Z0~EP!9HZqOc_myvXhAYNh@{I`i~i)%oz zV;}zYbQE40;DM!9=PPP`y5SQcv(3@(*ClrQa|Yg=`-t*|QTRSdPZAgLZw-zJ(k`r< z%LyIzGUWWV#YW5WFx@6X;lJt&TJ!HNtc(mG|I{8yTg&?|%XGH>U37bvbb<$E1xQ&K zD$W1*ZZ)p*nSk1{%6H*@y#sU)%T{o3eaeO}FlPQ64T#I8YovF9>R6I4)CLdoRD4YS zxQQ0&)K6?bd67}Q3&K=9x$+DnK4D!O6bZr&R5hx&hO$VZ@+ZQZ3SU5am=S-&pI)~; z3mHen*V>kjyg2?e(%7(e$JS%x)c!d4(+X^Qrw1qeBb_xDHs`d&+B0o{@Lh2YJe|0U zsWC0J>cOc8QE`^_R#T;48R-W>++2b4i!HVbn@jeRyaxGTW9VD-TO%&Uj7}!3_ukD& zeiW!bVD!{uf-(p}e9VX+aQ0_sd8EHLP__Yg>o$S2vmPp4CW>hv!DG7qKs<}2S9p5H zL{@EgmTp&iWqEyGFulJ>7)6IDy+JucG@~2|DH}!d_^KYUYxxl*4ihVyoyQh!a$sA1 zQ>2_0+5L-*xB`@(FzZi*9+z%As{9*ejtA8-byGT> z6jS$(hI9J%lKLKejlThTY7gRPrLB=LMP)sN9a;D-VjL7rZp*qfp|w4pROd6=&cTOv z_nG2o;&=EoWe=!qp;4(Grz}ThfVgk4jaJB1y4`Hw}v9rr%DX80JE2PIQ&w z26R8L-+BCV_@nrg)tGi1~a#(_->T_mR% zPmmKb&jbC9*-q>5X?Sn$7}7?D7r&*oA>7&Q>05Q)Yb;RJgyXjM;`X7>?1lRy;r!((?4ed zu@Cm?I@9;A9d@;qbvw?b&wk3D1ZLyH>RsSUj+4A-V#FVHNW&U~JIW_@QC^HPmiv|j zv#Gux)R@EjejCK24$Xk;wZy#vwBO2o2S|CJgG&yV2u++}=&TM;hknGF}C*1)9LA+(12P;A-LUlu3&W4GPY_{3d0_E_@+(&pu(%e2Kj_g}QE^=`1(ux5a~3eE9o+Ea9#(^Bp;sstxmcZN!{j*2M} z&*0awu^8|iAQexG6UR?MR;$IDJew|l4SU9xeef4_E~s#9+KZ1cx4)ZgI=!>(yJe?* zvE2`6*@l5@&@vd%>=?R~e$93Lx`e-rHH3~&j>E%IJ2CA>cZhy!D?2o4FU?N&S9JqU zF&CI??0>j6DoslnpN+oTHqrB84*W?9$1W@AJCg@f7~zF5ZtBiM3ogK~VH5G$zJ9QE zWh35yV@oJIbrSE4TQ1G&onBR3E}5ml6<<_rQXi zIqcTw=`yrwS9$&5ah%mC94BPl9^oSVqe&nmI8 zM$MU@d2Rmjg|m#>vq7ZwILnVs2*j^vI$~G<$KqkPZRj#&0T4IJw-&3|$IyM4+OmuM z;9EyNo|;MgT%r+2vF|-^VECesR8!56)!HLw4~_ObdgcWicvS~$KDrYw8cM2FwjsS3 zh{wQw(;oD>&>J6Jq zW}I_SjQ?|j`C4y;p?${45dT~qy$(8`>x23J)g*B|pOiWrzR_L;`3s{!)gEyV-YNMh znq+z6JC`+BH?y4d!bezo_72dT6TYND<)0Y}`{4KLH`p6&A^A@X>XvWMW&8I6@e4DE zic&oZy}S(Nuf6(c7*Oe1^(^p@$O}x zVMyL+oP7I%mi*j?-+8+NtK|9v^*i=XkK}4@(tJsSf7Vs(Mz7##9wDM&kkQ%Y}MGUYX_FUuE=ZQDvZR~)73 zfq#mN+*iXCJdeVGw1*ZOu00(|6}^^eAIa z@8g8~QjIh90RPme5oxenbdB*)qXXi$hJ%^<8EHc~PPAhV*P6)RRc8`5Y*JbPCXe@L zCtBx;JL7)iT+cnC?-plI^WFpbs_bwiE&%$uM!gJ5TKaT{r;_ z^=rckk5Tb(_A*0=^{FRk=SK!@AcR28bD`MURcUjnVFrDWQK|L*Bth>hu$FbZvN)SfK9tpp3dxsKn zd3PMGA9Ei2k8LcUhUb$ONrM|FJIiO?eL&T(;ei;8Tx1MKhK_*hd0X*z-Xe8QjC2Y5 z`1A$RYf|yIcUfgY8cI?vDUFBTWg$pf0@l`Up)?U@R+=kFi(v850NN*c5^l+@%Bf#i z`_4v^#x9q18YsO@bqZz&FJq5t%dmlqjnXLC*fxt%-!Q@}d}?$FsORD7lAciF^MdKc zuVOhjEeTg2u^nH=z?ZF_pjD5qmP~or@S=7w^BVCKY#<1$2X+>Hip= z8xF{A&y?PXx!{7Mf166GaYj0oz4I_(CtqD)367oM_M!R2)eSt$Zk-kzs@=e#4W_X7 zoq;^O<2ozxnJiBYACAUN4=~aO*w=Ii5YI5fqC}YM>ddcAZYwt(e9lN;@ERGq3N3xp zp#YV>pW5Rc`hAH<hG1#mKCk(mFHr1IZXfAn^9qpj=y{mYCaU{D@dZ*-%vaGCZUsosbi%QV$H*Y0O% z`Th~m2=`;go!1q-Kspq<*}uZXYdtufuhwE&5!15_BAly@bS|hcC-|HzKGbT$sBWu7 z!}qoL_;q=nSJP94(g~{m>$w{ckLKZ=Dz$i7z**+wbA=hbu0z*p3knOLnCFsK2tlR& z8ZO;IKJ6IcVJUXJB1peAAzl3${+wL^Insi=2K@rNakY5hDdd;K(98V*~h=G^6&CLo>$$PK>DhO>bgk>J%io}me`_F2~&EU zMeZWc)ru?MU|}#qDvl2HTMdV9XNi{b0Vr%&=T74Rb+^AouLC2IJd@U|X&{guV$Vkp z#p4r?;h2Z9@OW@bB>g2W(>`9}@=i^S=KzP+ip!W+?b|RUx>)%-)}tEjOMS_k>m4!1 z{qEb4v|9oIm-AqT#oIXiV_C|EaGjk)u;V9hV5D{wTl=@~}am2`HD;$Hkx#RI8N zVDL&GPCg%KJmJUuTeR-Eo;>7{4yKb&W00LW;`byV4~zu~jW}s*xEmb|&C zf|R8iD4jvRUfkT+M-sMRZKL6WJQe<{yprjEtSY;FyaBxmg3$Xn+uh3 zbJsuY+k`VfJ{+13J&uIOKsdw6pPE(mLW! z^LM$~+GQlu=^$x}@RNy1hn z?T4f#k#rDFZTVXD4;FSUr0aXFl`q{a`fp5Uv`jEW76(G?=l$yZfp`fO){-^@(rrTV zmk+#?l?~85!!5INAdi4rV<#QyXrSJQK~}WyWo{{uE(hX$R`7G5;vCAAvN5}o zu}tW84#=x3f2wo=b}ySR$O}Max_3froQUfdg7VqYFVKkv*RP{+5J=Ney_rL3K~Lp> zkhF=?mx6FoqZ~vd{3lJ?j&cQK%rDv`R2~2PcN?#75%_-mrix5V z{&5{?Ny@#tu14|{NWCCPSLhU;5k8>81G~kv23r4D z8fiX-S3vU)6xS8@fX@ZAmJvfk)g3$(R26*-yLkwuTGSSCR0rv1Jb&Jyrib`IMM`E z|Erm?y_-~8CeQ9Fy}vCBxbCTVSj(6&1J1-hV#I6GZrnDZ@{q!21A%-X<(DyRe`amU zDaQl(JH^TD$Nb?+&nsVpEy`)_Fxwxb#S4M1o0Gqg``v4iHaRcy7I=G-&#%Z6r|h%$Pf5NKA2`OcO}*df$P**Y2dKU&n6;NO$ZDK0U*x~Gf=XpAD0BP7&cu(+ z{jxO|C^v=ow=IBpk@a8H3>}^|=foXQZ@V7%I^oOd+H@^j`Kmdv0Y&cxq zSteE;i#AiYLZ5mZBR$;Uvq=WzIaTB7alq;|Z_MYHgu}*nC!uleHhh426*=Z*fA%q; z6I^(8N-NiEOy2>&q5Vxt*-e9k80=X^eI8Er>c&eN=OQ=y2jO#l!I^wuLF`Ovk!Z&Q z2XX1Y=@hOiru%wc*W&L+O~K!u%V1)SW7vHt6 zM%&2L&28wu_|s@K?Fd*^&V-ive&UOFwcPR9&RBoWI=u7YD61lC$>VKL;*FCv<=Dep z={dI$zU=u9gg`ys$MBf8->s*-G_x7Jn*V~)aq;TU8@ku?E6EKF?eJogo!qWPBp+wp zOrFVGg?Ub$@f456(6fPJedTbNmG&50hB~vMKNmsu7N7A~v7xkmH-W{|`qk^7RhD}@ zo`;3Y4e-hke~IaPa7ScK+UNe0b~1M~qu27C?aMJ}UX&c4y_4>(Y9t*!^4o7N>MWNA zo1t?hCtlssTypw8jQKU-FbjRp;?oYwR7Cc5^d3Yq%NJ82TSx3L@f) zSouB~^@E#h2lGSWw`Ux>te>Jj6Zdtr6F;*ausps5-W;=)2j6@|dJWw-az|ae%>Vey zbFR-`aqLN5KJsKZH1Y3}Q`NZ)XbjO}rQ#KQyM-MbX;sI6h_> z*!0>=dOTelR?lAuM8)39#b6<9E;33p#Y>ob=3=DoPNRP(*_%Oi0OT41I3HCbjL z=VQ#0ws0pxZ_&L1!$b~0f-i5Q$#^Km&J20{8BN|sn zKGo}NZc6_W&m)T`@-F=hR3Aaf%SQ6nrhjl?@=&<`-z?l0G=y+LM>rFKshgit4fKUJ zvj?H>_ZA$Qz#KCmYLiVeY!0Gg+&dAzlP&&JMn>i zYx1Biyjp9mQaY*%SOR$N}DdcVQbE#-ML< zHSoQ#1ozgS!_MRx@I!B^5w?W^f6^TaV#Dz1d89F4iw<8AT2+}Ta}Hc)g|6w^7wtuVGiA$JSmryL>j(InzOwn6<*MVYS&DpCU9g zsY&bJ?LWSg z<+c${a<3)9>5Aw!At;d}bC|!{R*%X@ezt~(hD{IPkWbB4pGiYDOlFK5lQbVj& zCxM>7Z4dqSkI)p(pZqlylgm7L>53Bc`FLDs=5g4WxWT|SjeQHy+~LuRy(XEoEj;X z?wcp?#S~LNRF~W5bmR1YHGcT4QC;pYK{w=jWc4?Y|qu0JB?93K)j(C258>;m(RjNxg+Byou>;d-S3G<k(QF`UU4;6D%b)??|Sp0w>K$Hz$1&FA@Mg7=JBP$ z>tT&_#|!4agv*pbS&(xE`xrh)I5ivQ(0xeid})kz33MNGu;)gmxcpmzFFM_qXgQA7 zec|RP`gH{3uxxR+^;~#R;}i693WmND^D%zPOmW?}68Byi$EenY(|wK&?NXS*K4ZS| ziiyxH(($h};vLs{LjPCB+C=saiT}7)#9XW~qOQ)ev#C_r^x6G2)o)!`60jQoCN_g@ zRa+3BWx(RP!&tlO-3U)pkT4bXx(w3!&3J>vk=$t7Sh)Ig5E2%nL(wxjzey+~)Tf z97$_dH{E?raUlGdcMVC8sq=%^(HoTx5+?NBQQwI*)@K^jdwY z4?7^7z`cJ$u*KwG3Xk}@y`%V$*p`y83+MO@VZEH%YKo`AQd)`G<(uHtnA)nwMd-6B z6}-F2@&{=lBS~|LEqZwIr{_0dliI71bRcYc7ml;j^!S`s+h{HSc3NDe{o1Qm?r56T zU50JAr^W~tZqw-b`4Mu>vII2yR)~*XyfNYOAcZA_IW-mE;o;9)@z|4H;?|~>w6|VU zx$(*?y4C@tE9VMFdm9+nIt!^**oYk_oYQ?=ZIAlXJ*D-aNzhVoU(%WX%o>BMza18& zHTcJ1I~cL!6`bvMP0Wi9g8b`|ETnTZjQB4|F0SjwJH7sgKjzf|!YD{Mw45Ex*U6RD z4&&tQHz0mWnArN%S(1*xaUa~|u$uj-rnf`OgUP7yqNmd{q4dShFJoE9b-AL|n_yA7 z&7{e_H_i+D+;ejBa4}hs&nT`ec`__YvR4iVBJul zUfp=sagu&6af2ki#t1`k#DDq~7@_*U_`+Z`b2tR*cP^K|>2A&F!A|BjCVo50iZ2x6 z(Yuc5WAhD;+pbf!uI3uj<2DfPG?BzXKz++OauVt2CX)0FxLjVu8VTBC86H7I<}}6c zNIDE}Z`uarC3JLLN%O{9R|yjfmuKMdp^HIbbY`H0A+%Nj`30cOo=N*1)<~PPMcYg< z)6ZB^O>i|=$z>)ypmi{;2>Xmlr=Kw5WI<4fkrAZboTs!H%loxXl1Af$ zjrhC1_=IZKgtHENgwj|FPmUSg!9MjQ`73Wpnp`yglLB>~eieTc)1Yg*y>8NqdFlUJiX!$@bqt{fcLH-WrOI*@)L zu86~B z=Tf*4_8RjUNym_euFPEzZGw=_-vnU;Hn6)7O**be@_YDvdUHuw1~i9Yk$4_zc*HC3 z0nN(B@Ff9;veP9y-hJqLR+Ri7v~FgDj$bzrSNF>$@5Cpqh*y2XlEtzlTdNLOc1wjiCP^tM#5`&FDt9!SGa&+pM5m8+F^L-J|R&9-TU9#B4W zRn{){@1M2wU2Dpn&{&dIq_NQ>zu_dP7Zq*@g>9>f8}g{7ZScj_1)v{XillSFy7Fo? z*;3BBwS5YU^>(QFRP}@?vRUE>UR*p)TUZFkO%&tqE z`GhdX0_*Il#4cPkL-Lt~(T`}3dLVg3zS^^c^7I%!_dc_|(v(lQl??B6EwyVk51?D* zNi-`?#A>T!p;EzFF?aqUwnmO5&OgFP6X|G7m0khr9Ub{RAdM`@?+fMC+Euq@!+q}& z*SE(8h1=k(xu`ERzkZjwfPf|tUxuy@{YMjTI?<((i6 zz{r2m{=*is()$Fgm)W&erX{t0Qa@sk57Ms#`qs*c1p$)HzDZ4dAWgUFvyz z^01aj+@`!3Cm$&e9why(-(E-k#7OUBjin<l*w-4c2b<&hqnCn|7(rNh>zSV9ZY2#Qu`Sd)98cQ3J$N62+%Q2n&ko1@&&7^at z`@(2k2~+c#(!Xa6Poc`18vnU3y0?YlYH(ejI$Po{i^0VZ%fHbF$aInG?lBVXBsui+b!@I*2msv`y;?^^9 zN;?5*Lml}*cvLDda8MX==)4Ngv-V#$Cm#ML6#Hg;mXy|?u%a1!rx>g`PK{|q- z8S1P9@>4WU$W;B6A5T<1k1&k(n>AL%J)G&)E3Zn{XbZ-WUh>nr>q{nJM7cZ;p4<@>kB z17%A1q01}EZ+mK`D~9PPdxFjO0|>W6U`WnOL3t{Cofb>_GC)!Vd-FO=$_t3A>UoswjmB1QjPQH4>5@DG>BT*S_lt8Wmz1Q@boNon#NS=%wL_q} z+Xhd159zw274=GW@7XBco?ACGBR(#u$Z8YTpV0`5=^3blq#eIgeq}9B42=QuS*TeA z3i92gc?UD%FGl%-%El=398bOO1oNWxwK-NXOyxvt5?xqagHnz10w8P@l)Z4unmJ`Q zgxk*i;G4NH{rn@(ra=yr!vx8JGj}W94IS5+Ao*}quj|D(hy}X(aNPeFByoqp&>_qRGK+hiSRX#x@El7LM zA7s>{Qq>9N721hK+l0#6Dci=w?S|*x8{CunBmybF66A9!mkvPEn^db7+#)dsssEAc zf>92J@p2@8+< zi#D*%E(?%;$LrppHKc|-29J%4VVOlNZrRfpuce)U@I?oh^Bybi@Gf60DjO!;Iu4Y) z%u^QCq-U8AwUL*SE;2sx4X*nz0Y7^z;^Vb35SwTQFK0(V$JN82Ucf&#FUd(emKzLK z3w3;j%;)yk{aL$_n=$83A!PYIMZe{HgyYCnbd0C0tTL0WS`C7lKX$NPpKrjrR{wBa ztCe`*Ry?xv1)5jKQfxtMU4DtQ!`urTLbqpWmL*R7w5$S3ImKut1EwA8^{U<2w3)LiL^Zu zA?Eb?!?su&X$y?%%4y9uKxdaP@Q|Lf(KopSWBhVptA(4~VVelMT2|&6GicAc;V#f? z#9PtLv?i>Z)tcran15c-SU#K35pK%GQhGIoF*~nA@MkZ$ZL5=0taf6+H`-fbl9WsI zy>Lg#bJ(u^#(}?S53&6ITp;f#yJtEb9q#E?vUz-&RXz@3=8q$9H_}SRyw}11I=b?JoU%5ql*U$e`66Q}QDcNu?|F`OLegX; zYh%e)gvmM#LPUuPi7XLAvSw`4d!A#>mMw$HGBH^ulVvbj29xi1`Y&&}@44qZ&u_c; zo;x${H62Q{)$dAU^%m@L5tFc=_}U7d<+@kR^HZXE(=pO(&|Mxg=%ZNGWIcV^cRk5Z@fRNZQ*t$ds*@X!UVFO8=<>?|Qw6>(`paa!j;*bg+u- znVLb784LMA|4O>S`t|a%Q&YKlp|5OrDUl9UJ48O7PP%zfZ`tF+V=Lx^eLR2UehE$G z!UL@|zC%X1bkU`6zvaqhJJj6UhiRw7Mymdy7iAzvK<%l)56_9V=}Yu7hvm9zJrtCjz zSYloBDb?mzbt5g0dwhtL8_w;e3SQsSqt{!d*Xb0I75&ikEjx^SL7A7j(~O4&${e%* z&dIV~$X*eB?-=Iny;?d8_g#11qHU*E(0%vhbE}z|6xqy|j+Pej^d?E-`wqT8o+Wzs-Dx%B zIzKv_vP$-g7qchwj0iVgyZLWkX@_gbTRY-<^aw7TvyytR>PF9pq|3F#egiJwCSVDn z$92Cv!95GCIVEI7hUw<|M0ciHmR$g z=O^$U-*08y!X5NOa2rBBNUPs1@^U^+wu!Ux{J*OXbiS^9p5LZhsnHzW_60q;T%RxQ z$9;p=p)xPDflLgtmjiD6$kk`A6%pZs=}61Z$$8^?SgNACZvx{$qT~k;$ z?f6$a>f)Nr8~u7=&;EmqIbkk%!|!W8;i)%VbIU|2D9|6!W0sA6(#sW z4==l6=0p@sYbbxHR$rH0`bUi}2-JN&UFqSX&vfI!SE9|y4m$JF86LjNjStW6!*$zs zW2}Xa!8;>xE?m3+Oa6GrP4)@Fo)`f)DSlOvx|W0eUT=4h9+j(6(5Mr_f9C`ASBgr8 z2beR?MWPS1TX01^r)-oAntqe<|J1na;|-oEoGY7J#aKkFOCJK)(C~59EdSUi%fkGV z3TsN|dIs>hycvwo*kGr@f_fpXDIoj;&Cx-W;S;Sd+ToqAcTyF2N24EsQ^WPS*D0o# z=DCVLF!3QvYoOiyR!zRN*|q}SIA!rV3RN}a;c$ZdsSDJJ9X84 zl1KBk_q%mWf;U&adylY2I^#eNJEm6D&d-C$%)_8|bwtXF!(8_u9?1SIlAF8Q$ej+2 z%4@*PV?&RoV#%|4*4#^BvP;n&^SSKnna=awn`>|@`15P^V9h2*&kEob`$zTSwbi!s zx^X-847ZnR!X*zj9Q7vkgD`c0Ib`q#?t4y9J;RnLe~&-)c-LD3Z`JnbLNFtBB z&(-lcai$kg6IPnhXE__pdK%lC9`)=e$KxK6(zk^Q7%fbX<8xDk4BD3^pC>dabl9I{P}nfN_Rg%Q=%dXyIjhJlUJzjevLH7 z!NtWdh0#Pe>aPW_eMa>h>oWS8IV{%-%w^1r0OsKxD31v^$M+`vVg5}ieHvK}E*O2( zdQ@8l9VKrJ+)pJdJA;G8mZ*5Y;o z&v1+3K<9o2s{}YeN8WEF+s{2lC4cPYtA)7NA$uI{dTlS!M|$nB{<0|P0{i{4nr6@b zSwNf6{m4c<0?&x1M6}oadQX&}BE8g}?eJ2AZK+QZo|)S-k>k4kNTq&n`GMzOV%y77 z0USc@T(q27lZ?8})4&dC=ApPahN@e=DWLLtQ9ro{r->cxHhYm=fBQ5IEh@2giYwrz zaXod$KkX-k76>d}p~UV8qcM9Nw-9~$NY&nm2=pSzAD;1CZl3*+T^x43rQn`X@Z zzh`t7A@|yt8dvZZ&@pxl9=B|**^$pWc*$vY8RfdixA!nYzwr?rML4F_^=u~^JK0vB zOBhOr-#60s_R$8{sF%+%#=MZ}*97d3w!vi>;~XrXEnjAIGI?ygLDgv#gQsail9TXE zt)=m^0$-YXFV}7DlYc}0v7Fk=?bQD2-{8`0f)17FnO}@%+upF29P4u#ShkI)#fPG9 zgY;4FdU{W2BE{GIlItg*#TbLguC^;bJ^qNHV+j3+_v-$nPJHr_|KPgayqVYORKpCj zzKX-X(xzXa3HkHPH=J+t9fvkpOe3Clw0vq&yBsf?=8oWqjsK|JB&L*ye{xyUU_Re` z4NX)v74)O2d8+W_F8LR)A=BeHCU;%#DGO_c)8#(GaIP7LWyI5?jIl^?sWe{eWoA{n z-f|xWHQOPKrpPOf5Pvpz*Q@@pA%mUpL^6K@o|z86%~)G)6TgP4kDV`KE9cNb*WMI^ z{j_Fw{aqP<^Yihu1Z<=yPWvglNh=*+a~~b(It$Mjd02)9XYhA}_Yw3xKl5q`&yhml zYYLe%i&r&qkxg@p82Zl&tk>E$B?sS+!oG;c8yRg!Q`7N&v;V~T(QOF&OaVJ` z;FSp&3Gee9HFcYcQ%=z6)HVb?t#Dj2M^!4{T#cOpuNd>)=goD_{#C$7{B*@6BfY+?=cp0Oa>;G zuT#Re8`PxeGvv5q9UHF+UC%>)*vMxBF4#{f$P$8!qSmQMVNIFSR~$-*--@x#aKREi$|pW4}Qg2|upjz4Zm`r7+-z z69NC|H~abg)2Hn@SZhXZAmJzZqTOXG%`}{8fY!S&DG zo!!fPbKtdP+j~tk^dP)!EQ9}bqqUNJt9IAGEE=+6t%3(r=t+7Z!Z_&2OM~0Az2Qn; zUUiBvnS=4v(5(K!*TcENcXMUMF<+yeO4#mKyK+u}-eYJ^LQPX_coclVQ+T%N!su%7 zx`O@@m?r@rp}|$$wCg@zS2l$&zB=J8*FT922ApqRpWeP$$M9Zq8J-_CUJU*~8+~B3GrS!EpN+1S6$Y$E zwvx!k*Z!}nFGskXF!>129k>wK{GMtyOXSx-eW8|QUqr^TOd;G;>BZ`KVIO|=+cJaq z;FzV1j7dW`Q0F$8!2I{PcJ;d|i5S7~l*)TjJRvJ!>$GKT{Df`dM{8&U2MyiL>5p)~ z;q~#HRpL#+CqD8xp1+ZYxU~IB<2l*r+`(@5O>SXoT%-vPHZ_hMzyh35o!hivtPg#i zy_Z{jVHNP58a{y0XM`RVosKURKMXs|$lxSAAV2*t4LWHRfA#cB>K`@8a0YL=F9?2} zr=D6Ydbn5OHuD?FqopIb<sPcXceKyUKiQ*I0|U`3uO$2gA^s3VD-grKz<+F9>BQ_-6C z-q8X*C4fKB7gs2CZ$^36fgU%$g)pB+3&1n}$428AjA&~AK|lwhhN0_Mz2LahC)hqN z$LItLFhyPHVk=!NIbv5r2s?XLP{Twb71wf-z!<~Rz}J>Ey%SEGDRSpG(5qmU0Raeo8cF%2EV)tJdxRXNN7O? hJ~G*e!GB>g#R2uQjJBtIn-?ZqR}G^Cy_Vnht)}~C=lf9$tZKS@dU|(yx_7E)ht-0{H}l@C z%;~JmA|KsT>(Rzw2@HycU3OZmd3l#Guozpm_zNN|OAZmX{*kf6#sQWe;TVU*5SBw5 zmK~0eBW!=Z=lA>Fd$Y2-x>kUg-e`iI?ykCz-~Ij0`+UziZ+`XP{2#yaPkn{{@4@?~ z$?RY_PA0Qghx5b3^=z^_uBP+hay4IMRepMy9nKfW4s9~Dhi^T+@WT)P z>ca}^ui9H)ebwIcm6P=0_-h_~?P0b`(qxs`55H=? zm_B^)jpNnoXt{g&@@#%DTOU@F2`gMpR#Tq4yLThHHCSegd&yuj6i*6|zZBh^ESH>| zy{vMJy`D}E#;g08{u}LI{qQmxCP%CFB6Ev!dudy_JW6I!GF;4gKsY6PQ%dHHqDg+TGEV9jHdvvz5POl|}a^%ZVdGJCXT% z6&+;@zV0xY4YLS>Sd6nItycC|$)Ze^qE_vhJIH43Vzigu(>-Z)Gg~bt!=;Wq(PdG> z@f~thI+x>UIfSSENw^J+CXgtUBvOgiOVPacU*ECF#@TE+fi5%mLS2&6OEtQ~ZFkwL z;hMj|NXv|c^CoS~WST6DyOzl)6PFDp$>i8&oIR3izgKv8E)uZFo*7IuiI=q71c=5rX zFa2~h$)@S@)n&HgSD$*gGyaD0H;>=-=?CvFU30XU++(A4?|Oi28}w`SPCt#4q?yL8 zZle(={Va`pwfeBzAM(>t?cum$LUCvO^~l5VyT;!>{s#T!o5ugj_}$|_H+~ONWBlg} zQ8E7J@BeE^hku^$HOht)kVb4}wt#?>>G3Bx;4&RaIvQzjk^!tYzeyB48&|NNbTD%@?a>1Q8t0(`*`{GiC=Fr;6}8JX!@Q6y$;g(`pXa!X}b# zCf?y4$v>T&=zY9~WE=_JgPS?%YUR!X!Ne)Y{>XLTH4!O?vCN{EV2QG1oGhckdNN(v z2XLvsE?z?MCQ-|>@RDPh<@!k9Vz)R>MXW_fq~1G88i=h zN24r}6hR7IK}5kt(LQfo8dMqn+YPP&=t`wj@IpPNv!`Nw1eA_O?ID`6{MZvlD$|?4rZ)LbfWWL%uMnX!mYBO zWljh?wID00w(sCrs|BUD=%f~8tKYG+?Wj~DHH<%9Y?+PGY`k# zg5((g#TUolT8M-3w;AWx8_B3XXtd*Dtp(?II^DR}>W|`DZ8WN7y-v3_*yjBAJ(lzT z0i6G&Z(AiOwL)L8x!z2XG;!vJliLu`#BV!dgQGE~zXXFAnWx!ceSk1KaH_%>J4P{> zTAi3$lt_W!Zp^qL!SoqL%cE?T>P47LQX}`WEO5^6`!j}ZFzvgsU+v-A{eCZ!kHwQB8|%37XB7VW>VSG#$! z;yD458A4wf9@?WN>af+Rck2Ci+-vsh>`BU=w7V_-+3EKC$*7$*xA&yw_vFF5w|u-E%ubP-NXKOU2GIy5xeq^3>__vHsvr?R_~pUBh;x zosJr5+{tQvq{pBiC#^v*PP_fiu#G1+YH!;OAAX$lc(R?YPCIR;DaO+<9n|7hquY!7 zNoNqJz2>md9n@Nlbld5ADL7q!;WVe~@?bhYkmvO+MYif$ezcJ~!vK5FSNo`U+WMz| zSDw|2#J)CfAvQ-uM{YMU4uUw*EW`PDUqtp}G9+R_Xo9p$anFt-NGPp?$#Czs#q%yy zD=%aTp-GG1C~e^a+c>)}rp>cRHQgV}(anzauSlLM0+C#af{pmD#0M=KqA)~=`xGxd zOzyIh#dBT3x zkpD89lIXKA{r;HL$YFGy2tZQlU9LCXieFK-PODUi!D2P@x_k4zWpr?~$```>>z8V^ zT4diyN@pA`6D7}5poyQLCr0^$=UCmo(yjoxn?@IchX}L*mj7YE4<$ck;4@|U&*GBOdF3&Qwm^h4{(@UC(cW75w+ zOf2_c$(bbNVK-&$k);f0E^RT8{ovJ?^0Qz3+|Rxly`nGX<@pJJu$&>`#uM`VOfn05 zU!+D!L=7h>P)lHH-+i9+gy646FVCivyYS2+k>;P1wcm=xU-FBL)+8(zN>LG#7j=Tq z`k}S$?o==&)Fr0CmqPB!Rb%#uq2@E}kYy!icRu|Sfl8p7Xs0qm>#YxCBZ$g{m~0o& zkpxr}nF}KOIs54|^>%%<2*f}+j@UP7=PUu$YGzrhH|$`Bwn?rgwPxJwW4NcIX0J2q z4}0y-wpDsJ=%xd7(-_^fB~X4Bf$}Husg>C93x-lrT5+`~gov{-PfNCOmoNliRm8}^ z$DGvk$ux3(vmpqJJo&FPd+Xmip&>Z8zvK??a(-}mlChjCbXM_;ysY{fr*`di?4sq; z{w}@q{CqZNJ%##-M-yn!Q@YbxsH$lsxD95TAU<6G@(i%hVrc>Z6L1yKO*=0l58R8= z0}vd}*aQ=_02IXy7mNbm`Wu1db4)}mw1-0kmC$4&IfrzStz{qTQwkYg3dt9?qxJWFUqRPDGb;X0HhY z2c88UkbTbqed@18ACVM`_9fFQ^=fowJf8z%`It05j8BSZe!53YjLsVFKz#v4(( z#?G~Lwe-yraK*N<3HNhtp|px!+GVL<{M^HzUt%xh5Ac6~5DTCX@&Zus=iMb{RKaAHb` zYqttRSI|DlP}KahCk%#t7za2A=~xg3md&?q%+gAHrZlP5NBt35g0z=l={83l{QgEK z&f3k|uvcsM>W%i6xwCkzH2JTQCi`e%1is7zZ9(sWvQ)GJ7>7gtA-UZ0Jqt#-z^TFi5Gf_^MTc_5VSjI& zP-c5+8eqY89=24hu*GB`;bohj0@s|(;$RzX`o(&7S)SZ-KxQXaUq)C3(lijGNK1gN z$>=y5%vS)qXW0-ub4gsEMEsSJkkv|1EYWiww|BNA8Fjlg1V%S*4!~}-8pxDEqg#vX z&1Suqf%s|+hg*{5__31YrzA=Kgg?7-H3w(sBI3p5o?ugKdhP@XacyWL+vCC%jqVaW z^T1s@L#O=|th+Y5Hvux`P?NX3l`rFdgVWuiIc14$KN=C@fh7vWC5}n)9Fq&uyH_^! zE^!+S*`S+kiG^YYKs`GVa@nRmg$-ZI?B!qruR0jJen@BxgUARt;NAE58zt0pP-4aL4GCvp_M&R#6W#+L1O7puaO07_YqfeGY! z?%WkZMp^**Ohlt3ZGh^Rq@Pe=Cf_Qj%RiHN4wsIcHy|;1)aXS>x!4%7jE4r-lRoF)BX7i>b> z9B$_!-+Zhb_?Nz#qL6p)j#O?0S*tL1$@jPvWv_$Mv&sPRN^t$mXcn$oB}ACjj(d`! zDn3*t;zGz}yIOAf!_UeffD4z12+pwLgB7xlWpBOpD_9Y{cfBv+Yk<5TYtQ@XNQ>TxP3;ClMJwSq%gUy?5*G5}?VH6qHa)E` zX(6$kZhhOSYuS9aTMu^UFGe3m=st_i8%_NE55GXAiwn8zXRGzmu5&bUt!enc*U;TW zHbxkM!e3gvKmi(C0pe=!M*JdT_Wlxb;P*|6S z(BF-o#SoZgu?yBm1YR5>-_|7No>G$|nQu{lNW`A>@H5fA!1XNV+ANK!0sz~KKpHNN zF%6LDb~_e}Mb#{HWu^h))%5{m_XBT|8Deg3T5~= zGHZp)n592>mhfqhx}!!LtE1IuBYcv6El%nn&&k_0>gj0I?sf-TM##VUSmE;nU+LVL zCvB^2OK~cnT^|^`Q<1V8*#E|IiL2{2{imZ3&*t~3TOr`^EXsJ7=q*9UD^N+cutXy% zz_}pcA>jByu2A}MZYOxs)43j;n*p^UxZK`LF}vo3e<7FLbY#>HZv1x`Nd8)~AfCDB zObgzVharn9kY}Cs<-#q5=fWCibFAJ8aW~6pkpm+XG&>nD(Zzx<6G$eiQjPYA8`xj% zGrdW}HBRCB0?25&R$+>QDx*b)dL=%>DvSH{Q_FKkufh3dWrZwSxsFDLgIo47PpFwj^)Sp!HO|_LP8STRN-`-X~|BjMz9T8Tah6V zT4=-tvgHj?QDxa<;?3xPq@7~8lNp-L+jEfD437$%IAmfEc@lv&!gRQ(D?H0kJQ`g? z8qN=oaR8wgXd4n@k@j|IfgB?;mdEP{q|4{iwZvrNN@DgbT$?{ajue?tVtSj4m+8qn zK!)o(a$AZLQALFHbX00pDe7$H_C)vFt_(8NKPFiIdo;s2^4(B*9G*@F-C|iO8CYz>b_{qVPN$&z37k z?mQmVO>!JuD<|9RQ8r(v7e8ka)hq7&mA>JuQ5~pQp9f;69V*{)1y|OKC0l*t>NR^D zz?~IoeSNEHEU$0`DjZqKQBrqoioh3L-wbyEGrZTBcH zPNpjeTsu?tyATREKb#SIHz#XXw}I|bFv;zM7Xvc$VC->3-EFq0R)i|zofu?%^U~3U zl;Z(mUtMeTp}n#6kG;T7ai$n}5u`BkD)1XzpO+=L4LVcS43b)R)a^Io;joSl?GM`V zpj+?6{bpy-9CnkeKH9e3KO5BNTjUlVo+cCi0GaUQZ(Q*D5c;md*LzpKV$?$Sh(^ukmaSQ0bVO z)cbbnQ1VQanHVb&`yeO~1WXXlkf>JYB>>^oW2xyOk4?#+j0LFfB#ORv1!hCKzWzl? zRBB-6S5(!;W>R0McTZ*9P~eljZYUckl=Q($7>h^QaaYkRp=oakQqjpTX9y$v;OzU< z4ll=(BR$v_3c$(zMBJ*I_4$|JR9EO3BDL!k!AqXtN<+1As*ocQNu)qYMeG-Sc9MsL zB%SY+l%efnA=}Lg+K7i<0;z+OLv1g~_yLmH(H~mPGC+58qtN>U+A;|uLL;V>Sb{0_ zTYQ7M2te*1Jj<3zs8ec>TJ4zf(01HP8@0GkX;a*2k1{OtX3`ucTju${{aE?(HQ;Lg z#J`)(76(n|bBnQv`i>IY5>X9mot%6MQ_@jlOv$9Dx6=#=beZ{%j%!Y50|&`U3Tkvz zeztUP_M7#_2vk#+4%vNR;=R_00+Ea#_G($03`SeK|I3fDB+=Axm(VTmx#IqgN<9Hz z`S2Hlk@)-X`Ynva2VW58v4UDAC2xOocUD|vqH@KVr^o|>3op!ZA@YjxwIr1x5zUl= z00SJNieErL&+K{QZBF1&rj08^mG1(puI6iWuKC4lc=JX2c~s2dtlT7o$+lCNjXZ(v z^pCS>xz{HTUvFeFq>HQB>9enm#xU;II?b$K?*N}|ZtdoOaK>(8H;yl}8*utezgNDPz`gB*Id9IgLRMRNO4P@esW_d@Aif8IB74%xtvk2< zHsn8g4$#S*^t!*~gOz=ESLK!(UJyQ*VXSE*E3U3I|AQZ`2RcgqVW;QzTTdbq&Z*Ac zPcwrA5o+B-FyxTE$Jy$G6|<4cKR)XKM%kb~8gw~;G=ZE)15%fr8U+u{Y|u>yO@WxV z4&XP!0la71F#>k;M+MW$7T>4+d?3s)*>k5CVUZIZ^IJ!m9@{%E)j)A~<>sq=Sm zrT*hGOe+YKr!Xzy>k2@EV&lH0!x;bs+2l`QT2EnGA>WC2?%H%%`1=WfX*D`v(9<^7 z<*+fJJgZMBQa^1YO@{rf(W|wRQ(#*E;IY!=DNGAU&B^4&Q<#>-rG+J*!nC~Q_2mQ8 zsx_O*pf?=FNm|QDCyfvygHDg0GJ~YwY1aEmZ@bX*w;wA>p2D=`?1*um!n73T&C48u z$~}o`b1{LZFs(l+m{!*74TuAd2w#DZXpOp(1N9^x)CaXbC63*0x?Sh?-#k_hJcVgJ zg=w7z8G8!TdJ5B`U$Acw+G>wF^-Qm&|e(uH;SyT{a&f9UH%7U_Tgul}Ku71rwMM9q>t z3CNF&vX#rdLja$(%w3jA(bxjN?>pyvfHK+8O%(uYm}GK}4s&%*J}z_aRDqJLr-BMC z!AG^;DFxQAj5E^McHai#L3IV?<(~*Q^Cs8)+ETNYu{S_IE&a)DZ~$2U8{O9Pp80vl zl+e3fUV?Tnue3f{_UM2HAJc=uKU#Y1eGz4SrdYvq5Vf*o-1(n#2`sgfcMUmw|^D zVdX&ZhQ;!gzbrD3HCKuv;Ef;qO1JXQq8-Nq7kCF%yLpjYURCAG#rBFyFzSNTYj9cF z7y3jP8@|C|xDB;!bpR^Q2SItojb;EeSf;+S@}})9ZTGTH2as2d4i_T;r3s*~xL41* zz%u%2GusAX{Ntc}f1k|6ACQ^Yk{*AC0*x=TU4-`S%*tgjzMrq!FmDy{f()|?A}XCd zvO3hiib4eo*#${>7oeZ_`ow*zPeOLMLY&RO^eF|&y9Ftz6Gad*hKQw%czKylO8V25 z{;A*iC%^eAwT!o+FS}G!X%fK<%mc|Z0JHoBM&a+mFI!Sy#l=<=@HcX{c@6*t%QhAR zWM(K&p_%eEYd*|GwJ5qVuV8i!zk!=mh75qg$-A&Y!>N!lYX0Eq`K=#2S=oI{Q$2z( zQp+L@0P#Rkr!}IYDsMHO~4)f%mCd(>>j z{aO>`MjM1_k~If}B!}6sRcAEIDUkMm`dA5c6PT&K_=(;(JF`!H9^z&Y33Xno;T=+; z<$EER%C*hb#3@uQs7H<8;I_9v^|!gfSN0hDG6e|!!*1|ZnRZ`Z{6lb`u}A9YAO#1X zY9s_L8{l-Wa$HGyETA`C;%MPOm$tyj=PW7bfv0O#6!TSvP^eXn@j0lZP)?p#9nr^u zIxQ`#T=^Fk7(cESocjt9;Y?zlL@hePPF98&BLQz>(q=Kjlt8-=r!;TO%*qKy?#E-?1_{(0r2=$uyZ;w}Y7yg? zijFdPH04+)yC|J-!l-$8w7(MvC^1mx6{_we#O6^Bk2=a|NK9NFiH!tFI!t`A$T8@3VHk@qaOQ{1`x~*X1=~i3 zmu7XHu=!kJuUUQhFsQutL^Hk+TWIY!k>GAR+ADy=&DF$!vjS-AA5yM2=iJZ)fGA}z zA-=7Iq+AAWkNm>tO({s$JDcu#kg=u`4&gB!lr^IPVLqUTJSu?4=8Z7)3P`yqG;zxe z_HA%p&_tAwjwh;XISTEUe1(FkE_M}?wFxw~C_K53xJrd$yPnOc5=iqh{kXKxQcTXm z4FwLiEbLNhGO!rWvN}40L7&!*?YP|oKGAAoe+&}yd%B(4sFl`7jjfEwYhMMlJQyDT z-}v3HeTDx0o|{`z<)3)lSKQ7EvSWfTl-pSVN$hm8@Y?8YkgTjw zUvUmR#O=;vo8|6m?o?Po!KqPf!752l?JV{vxSB2OkG7py^p~^6JOX*mzKvp#t-fQnPNZQ_Q1aiz=$~*7q)sKY`|(Z2-bY1j-h0fs!_L^WEjN3U zyX1E914j3S_E_Z}2?#6{DRoZBHH+Px+Jem1rXm+}eq6IMdkN#{D3O*s^%JiBq5oEy zoY=|I^$?Y7WCm3H(erxLOMBGp4#;RAaH!eEIzoTM&0cRrAb8Lnj9MqL>91AJDQkYm zWX+SleVnC>M_5OU5CD-v&i&?S$FVRIsES_euG)Y|!fIF*W0e9bc81cd<$f<*0(WcV zVwn6WC2~^Z<_+I`-CXF`v(YU|I**zZYfxCI#&XvrTS2akxcpjBrX|&h46@Kcq2stP zL@5c3pn$0#+5o{ z5gu~DbFROmxs(>MvggtwS_R)s?auB8L5`HF?F!%f021B z+K_Vb3v);(lv1mVmJFktifT7Y1zBTSYm!3E|7_@@{YakBM({?_6|>NnrJ-`_3$T$u z^Ofejy&Qku(02CnB1Rb+zJ<7|*Om^EW~;Uv@}Q$x;*hsfro1Sa;s_y%>lJ|wPJz#D zp)gHH!qqP5cugKhdDbkcseIn6746$AQOMivX?UUeIBryB*iadr#I)3{%habCjo2`qsd~X>8BxO9c^_XFVf2YU^=9v%)y>!8E>!;NhLE z=4pDjGZ*qans3;o(bw^4Ju}%1i6}IyrG{~-r52~|&1Ta0w!nE~Pi`omu@Y>q`;Y_| zYo(EeNgqy5Jqj$`S9EL%<+|al>=hg*Oy=`CVo6jk3oc;0y>=5MeOFwN9o>IgiBml$$vI|3BxERiJko$A=w_tby=7Y}--bLwk#|RcP42P#?1ghC*aLSq zm`;}brH@PSJ?y#OvN~x7x-A%ECaej9JMP)|2HJ5D^>N~lj#p&^>&l!;T7T0y+Vtt; zKk9uVQt@V!$#=<{m2Q||MrcJ}KU-67h|wu{evEPX&e0@!Xy@rkEQl$GUkqFN@5mu5@O5-6kLU;so0uvffw`#A;5|6=xwJMee-9?|E;7{!M-PiTCMM)(^yVXGYh7{X?3i&WE2ZBi`=)DM?iuSrK?B7xB$;Kso9>s3ndgCBnisp>TKj{a zkv?D9is@Gu^4%7uDL2W?-z$#DV}3HE)G~-A6egbeqk5vXh46WPt$`0j3gvZZS1hpM zK_FTzDQhVMCg0F^1Q7|+h=wNNgL%iZYvOqXLo8@vG+_oEoT&8@Zotdrva)DK%u2@O z@e%tlTOqL!?_h309`Ym_5A&!8`Mg8+jA0XQ2fgh9^dR?~amETXge`AGBR|RdlEvvm z0Gv7a9*t@`352j4vEF3=oEdWI)S1@RpwVcn&Y-SVALFH8t1$?*O*7JF)=nATvK4Z9 z-#I0WOqO@>zHl%eMFgTlIS1Rdfo4}>IYuA8eZwU8uaf;Iz$F5CU$8=?$~K)RPft0@ zs~p7Rm@{~9nu@UY=yC?d;}&_V)Tn7aTPx64)?U}t%+j%G$T>U0Td2bp*U~!1^&*rL zw=to8pSD7DrMZoMTwGnR{G=@#JO0Jb{oLn;PAK1VJ47w8E?vJPp1zmP%|EMHNBn5X zqLBcY40LAIA2zd!7f9spvmbyQV2~r)Q@V7wXDAv89cJkSw0$DY$9h>}5=zujM6?va4x3WF5#XrC7bOWKpt`^P8t%ct&+@ARJS^gD5l*}U^A z9kJbsUHcF)ZzeIZexsLgps_JSBn30bVVp`pIQ`C*4f+ulpbCK)>FY%<0UM{wo&x}M z*S7!BJDpBto|+E%69Ywhr zIg4a?mgP2z#ufzfc6gV|bEy%)rd-cBB`kT%>scBpM>8DJE)tE-L_amGqtlusyANoR z{?%?4|E`vx+eI4O2i!Jy4rPQ4%n@=fkxUOc&@~KTBq$?}1?rqH#3udewJ9w;EU&5U+7Qw<=xu0Im9gl2pG4mW}>HtWDJ~*>%Cf98)mIx zqq7}l`arSM55DbG7!m0P*H=uCp+97Q+fVY zM~~4I=_-1uJd|B&bBBz?T{yT}Y2c-i(t$<@QM>5SL)4MbXIx&HRtV(H+PQ}Kiz|Te zriBgXlNoMi981xCNG-?BaL$YkT_RJvX-`#plh$!)yE%uYep;D(A#z-!8ZS>cIdBhA`WFqEhM<&tOx-8Yq`t zEiqYMiv(kDfMy^~1Ccp4^+^|ZGN<8h+>vkl2xl{-Va_21klcAXNHYoyhR)AI)ltAO z8_$&!K(`FKL*DzcaGx9#`JYFQa zLj+h=aUmy5vXn=7QMw@*{868pN%OIOnVZW>Qy;&fqJ9;fW3g{@$JtgVO1LV(%VlC-)u^JJ66FqV<=)Kdly$b+{ZXgaZS}T8)*n8< zd=L;>XZpwgmA0AWYY6`maD@Np%DFUMU~qS)kGKSq2VxcJOsYwQ&7}iBvmuN_hX4Pks6d-Hz<=sVP*b1M=+$W$kL+}$TqwA4^W9Ev~++I zW?i|&rf;rq@Auih|%Ny-|-0N=62Pxj>A$um(PHOZaa3 zV4JFRoSHt#?}Z9=KxQv*Ty!HyBQN;)jJJRG7dKq$TP2P_kmcu>D^1OqB3^Htv30-Og`u;gpW$KC^oSHh3Ye2vf#?9*_2UkWR~QiJ^aC^ ziHRwMo}01umN4A3bP|4?X^x}NF`Q~N*>m6eg&cV0;)pLhF?n*o4>WkhrRK|`RuA7C zby&dzKOx?Jys*8WQt)X17yubIR>hSdbCrc4`kX|R;O%ji@QoeS7*KOVYvb0l%W?4R;O9GMQ7$cl6EugH=4b;JL=JRU{r6? z2ANr|%sm?PsORmpPobud71Z<~KGlgHfV8`;y#?OF9P4D_=4rB&cRx7L<~c6o6ZVw4 z=KE}3_?97M6L*o5nQAU z_DD*~%}&%l*a%{G2}e5stwg{&Bbp}E{jLg3EF4Z~i>M{!>;j0y)reHiQjlqowW{op z+J`j%S51d9Y8hBIIhkfcp?x*=7!|JcpIVh4SD~&H-8DsDTLkd!H-8?1Wfk*pzxfNH zJY*Ry@3Lg>MZr<-NB{o<>c9^LIpK$dA(&I3s~R=wMyfH0>}hmiv}95Ca+-)MPs_jo zAa;DP+S+TljFF6D5~}K}8U^rMegPWEmk_EExE{`Hlo8Zl*$90yiKTcsWa=~Vh52Y@ zfc+(-WinSJ#qbF|$rU2XBmh>4Tgw)^QI2{NtKdmK_3fM-${kXYIdb+hjb+jfMPodC zYzamm%$dG&L+|H0QT?UpKJWV((lnMBQef9e+kKhBpb))1SxQ#faJK;Zsq0EY2JnDi zDbCh%6^Ta#XFnkhzoaHI)nkQu!BjrU2|IJagu(hEA6mFlx%v$%Hs%Rt?hvNUGAhnC=0s zX1CGbY7_9KThG}|HsVVCu_eT_Q?z^>Byf+vh*NL0(}hEjVV?& z`^Z=@r;=*)A*jG5>_mF^=j&-YgF-kM5}$AiKXn9?4;S)>4TD>*!X7BI7qM%vQboL; zMbzk&hmofV#jxTkfSu2`dSgtYd@Bx|J!vnt#v}`hDG-aub(u^@PsJhS2Zr)Iq@D2lA8kfn75evVBZ7W(cP?@N&_o`- zv8;Y4ue?@5^qMt5j!zCax{zx%5jSMLFPZ4KF+!SQLynF4HPy2W5QAhQSKQ13gKr$i_a<|~ zE22Snnml9QH==RAWMZfx>Jx^yB={x=&Fl(O=$@BVa4F;lIgTR z&CU<$50yLq)Q@AsyU6^m-<$(=1(4)bKN4d6nz0qV;>9&W@Gw|_aDW?~wLti5tGJ1A zZfnL7QZ*>OS7_xIGgYE!%%CyKTCv{z>anXh6E$HN58!JS^)uL?v?d+G7zSor_dKJ7+DYp zPS}l}gS+L=xB+sh6|J$o5+~Lb0ga6ivuF8uAr_o$5V={108n&#n$|9+VUDWWl`JbC zlU1pSVs@+JGNjfzmWpAtq`7#lLm*LT`*#%=^^sOjlPpo<9Z+LHxlRgMhFR^ z0UfcBeU{MRYz>SgIfCKgnh83){&f`fuJj)KLp<97sy&FI07$@$d2Ud0&!E+wv|=+i zFu`Td@F0mQE|<-$9EnQ+y#3~{=xe4EmU1wci$GYQhWP7k`__drDu%Fab zCyR+%mKKqmk)Lg@h$zG(+=IR`;3ZkkDg390>|K7=bd-M+e3(0Jm?J{I@FN!DRA5nKod9-5pGDt5R2=xB}iE?PS%1DiSa}>g^x@ z6|dK?Ngy z5L5cLm-MkxTrn2XJ!UXdENRzsnK7kxT2vHUYc`Kwx80h>p!p7spdgR*gHoErXvTA* z#%D^CE|VN+I?|?QS6vl54J8^5lVO3_?a`uee%pRV-i`!_mmLHjO}m*q zS=tzC7pf_EOazD!OX7?s~VtpETWq5dT<^~8(r<0dn3Rr=zaK16HzGaHx7r=PSTzK>t1SjgLj zg{2Upu8uId?}|Ivi+I_lEJ9TGuC`j+%9h$OCnK$VYRF!#W6!Xk=6B(^SS^r!$;yS6 z%<6J+Y?V7vTHZWE*(>;QXS(RkwUO z>>`Jtt`qQ+X-vqTI(wII<0PIVEU%cOkAXp20Er%r$ySSdADbmRh04|G_;PqnhQD%j8GPI-f}oo2)n zFcpjGZH2TU0ID1-*Oh*q1Q*X1l;=QwGRtMVeX9crtH^PS{)*?3174MmC7Ts;^}K>2 z+6if5pEtIoy;ct#!H#WTWR>%%{S z>Xs3mTARbH5f3^w(z?AibDz4PM;h%;qdj6UEy7^SN_gx1tY70&3!J}GHly+(f;+|) zL=D(|t141xOXu#Q2i{T?QNTSE7@2kGSzd>!%v@-88W3J0VGWR&&kVn>n^mrD{5A!M zFK^fS1R^^iCA@1ZeTE3K!^s*~KgYqIDXP<2r=Mo6UYvFMbdsa?Fz$^gPD(qitVV~q zRyx?4EAraU7U%WgcAzRpLe*q)nb9H!?>RW4ClX-2d&^jVZHK?K=IU5Ai4PB_>sa)^ zwqOTlm5@$C03BYxeJ8%PzaQOOM*3?z{Kemj=k_qVdGm#r)gfy*CSZ8i!l52-xU=Wo z3cO~r8gSbsgdeGO?cz=1c*_eXaX>F$_Bfz}um7=8m{#(~zKY8Jm`>`4(e?GwDyj4? z*P9LsXQgw5>3G^ zGPsxPbsZ`?c&^c`?cpHbLl5e#1It+tUW1zPa~vF|U8@xVbV~+a5QXBN3FT>l;9w#~ zJNL(PVTY8;7tV-SmCwgH&7pV+QS2^(8*>rtOlybk`I8PQPAl_MXueJc}3VH?3H*yH0zd&(`9L-N%FCm>C7({|x@Q5R&HgMx`eEWGRYi*GFo zYuNIWy@EE@Y=Mb;Bz(lDf>GEE2aBT>bx8MMa+4SC05@qP&+95WNDNNhM$W9lt`D+o z2-rTZcN@gq$?C=Z;ed&S!)~{g4*PV%OSkmZzf;J+*FLAezx%b%>yP)`+!7-{f>-;5 zZ(rpNf9h{&ToC5k@EGa&k&sQ}?-Soc=>y;{>nX-^B``|0tAhES%dy_2?(I9i4dS@uh(b; z4PpR6f5;H@W_`3J1b*%OLf~IR2q?<)1a8|&@otz^>=I0bWC*n!s$HtL6%LumqkwklD(=K4#?#vwHV90pCG z8oYkH+O0nmfjY6FC4p*1*(#nPfOLBd71*V4N&gX-lF%Y`hpSv%U8}ZX+pPzizP{oC z2`3gAUU`M}q30Lmo;uBHr+La_iuG-LPOn~VoqqZAk^%b;!@PM5Z#eC<{N@3ol%+%9 z4E(%j2{HcjgrPAlROsUNDPMH7)9h56e7jmJxI4wU*QkoH8%{rpXvb~u=1snD4p0hr z3Q$9GtOUO};sd2(Ue1*6H&$~*B;M$u5?Fg*Rp26}GJE?cpRMFu-A1dI;*=(X5vrZx z!(fg3!#HgvP1>!{)oHX1`1|$q^LNeo`<=QomAwHQE#rA_dXNnkNntiRh9fJAKNj-Y zil{aS3JFDmp-RYw$d4vfjm?}k@qU__C{7-ekq|TV8_>Gf90G}9R4-j7MLq%IJ?Mw* z^`T6CegY?Zc82{5GY-fqF(ymz2pUHq`iL+G9bw&ay}?@y8>=#Dc5sTcax>|re~beS;1=YYftZ|6BTX!W+-y;Y(j&M-@v0_{@sv_q73Ho2YKcGr zK{G_=<|k@Hj+Cn6i6v>6o5nglXUY~|x54{pU#V^%QRugd%~xL)(t!7o*z^u2LP4?p zu`{~B0edIgQ-NW$3g0B`ilspl>r06OCq?H^Qr~IN?~Rk&sO2NChIlb76CP>)J$m1n zL5X%_#Ec($qV%#h6>J$T6#8v?tr3e55T?-_HW_yNAM?r3xTkRA%M@EK(h%9-0e{60iOSC25Y|Aw(F^Zm{ zppoTbd{V`cuYcnPSq0B;Rbmh&X~OE@48c6eH#osBpFoj55>|&yL4pW7zXJ(f zyiocKaZ!f zxX#s77A+W`#YZ_Dh)AVG3z#oq1veEbxLqO>LH1mX$~#qf0wten@7_VN4K{dS$ptJo zgt7P+8xwl1KH95#xrlghOYY({do4Vr=+^ZMStdKk0Y*O}krd z_49jus4m`oh8a9OBsfHMQ2NzYPmifL>vha39@OaetL>Ja^a2?LIyCuBVi{-I{b@p} zy`)Q*=1Xy_%b>s>{T$==pj*#cS-01yZ4DWF?eCpm&isAk%$LhPRy2>G>@wlPjDm`3#hr^~@8P--;Ig#W~(^n~Ug=Hpd@`wEj_kaY{BDI20j zv+@JV1pd8dm+1ZZ(`B-tY`whjf(#tQ0<4D%eaZBdDeYgXsyBNAyi8>vb_Q=-W*<<@ zosc1dQF4~bI)p+>T2&B$FnvPUj!GcjGPmE+H-ZTpnTs9vYd3;*dcvJ*$l_P zm<$gs{t7&6!GtirwAdq)Mkt$xzz}^h9KsJqJD=>=p1JS=W&a=z%M}`sc;H5Q)w?x+ z<;h)pR!P-}${^4UbVXl9^)?S~wK}CIH>#bs9-b4RAmJ#fvm{8Z-L7}*LyVMkM0Ir9 z0TeSLkk}t(O<;DIFQ-Hf{1|5SLBH3wqdQ1!b(p-EPFtrtFtulFM2RE~E-K~FWpcc2nEI^GR4@uQG^ zT;TglU0!vgm!c4ON(;}Q< zk>!SklDZ7aI8E-CwWJ;AOSf(?(A(=vo@GDu8%ld4XJ<(`pCxKKwN9^kUx`-2`BD&6*uwmN@7{cp?v`}_dS?Eq z8a%0Fzw$iCL_Xao{&p-8=z3|Hsgk^&ODSSzE<0#J1fETRikznCu;9{-i{uSjG)2}) zp0~t+!uk+dI2L1h{=u8cxl`{vG~(grJZ|Ulpt-fQ8-0|78V~~+GA>amyuk-t8d3-n zhk1@ylwkSpB+E*N8s3nmB;T_W!iXPL_6?8s0&y0f#pKsPDn>g{IZwRuQ9jD>S@~6~ z3*O~X3yzza9I_zo-_Z5^=fnf$LY&MR@eJGTa{5OZBSIed2-S8^f4bOgc$#xXg6rs+ zt>o0yYt?#3w^`oEzd$EdA{;$DwW)XC$jQweT*~IXr}dA z0urQ?Fo~ztpxUw5syE`%aMXvJsDbQ^wt_CdbAI{qpPDoD1Z~y^g_mWzqd$<3`KK1+rHX8d zf1b!!`h?J`MfLP|!{wLPc#1qhy(#L#Q#dDkfDd}gT*_9iuCTQHBI#!jJMA?<`n?S! zQx8b5RrR>YKOrFCZbY7~reW?_yV5Pif_xC9Y?tM_-T4;xDCf*2$(3TJ1}H}&S%K%u zn=vCQX>SDooCkRF=uR%0z00BKPOePY7~{&D=19dvLpoF}S~yVN<7rQp)+kGo{MKfV zQuA;wkGj)~tNf-$_Z&A#z_>f0S}Ia4UPKX@s$^vXmA+d~W8tB!aGSnVY7W58JZsb7zm1+Am@|G5!d>N$t;J$iMw9G98m~3C=@Fb#bZ@V z?8xI3Vbpk=;_^e+*UXxAQ1PeweQWd~?=~#f)^Q@-$+B{3ae7`t)a&{x1U75bp5;-- zBH__4!s3LbLaJwT%SO0cmm7IL3JTDT}78^eYmn6ZXYQ8tMouGtyfc~0ueyi z$_`x!`NeS2DXCDt=`YhbiqqJQ;zqUIY`4VryqMA;0c_Rap+GAElh)f@XscxCgfuXZ z^9!<8yv{Z7$H)Yu5b`!$#>r%sm&c{o@p4C0B|=b=jj~o>P|y1$3w2yl2tEUrTcRRA z1N&{Hc1}-W@hGSL{L570tiW4Hn7Q_120MfRNN6k#sja}^S-wq!-U|I1FyUUmOOu5* z0IXhZfNxXpc1YcJ5=eeaY`lGbvGFS=Hr@f7!2aD46Sxg0>3IyNn&%;KU*6>WF4gJO zc@YmpTosL<=6X!G89B0*6ubr@8?mN|+ThA=!ncd z0Ht4yst9wpeEKJ_g)>0P?7&+%a&pJ?O%y#N#HbEWVYPb`07_PioyTMXk8X({dkW>a zcU0W7x!9aFU9@I%R~%z(N-Q56(i%4N83)Y_ z6ca+3h%rJH(F>|6al+Qv{JFJySXf0XBhzt|NK2Kj(a97xg>|4D2LG27i`A226nz2% zaVHUE2T2NLV62={Nx;MQAvz>O2}sU5EmZEL2M;fB;1A?6rSSi{8_o*sC zvH$=9ta>C!3Yohv`B=QG>$6dyScqnjqYX9?7-ca~@x+!3-m8RStu|Gmh-jn#hY4!v-ReJ^rnVejUa82I%tsp}!U{m643h5wH zlawNF3u6TFsJDkiRW+)1;#_t*4$2*UNomsxAvalE@3x>)Fd&38r5;v(u>hfJ)^7k} z3Q$!LADF{C3RsKk2tZQ;T6HIMU>5|==w8mRQ|yYcPVjl;+mu$}ixr#~YcNt%fMu@s zB@4Ove#vJDu6$Io3VQ$nMJGbm$^ZMngl(20o!CQWhHo0jnCa5GvXJmYb8T$7Bo?j+(OeQ>0+r{zbD$JtV zB?k8KBm-spxML~c;Gj?nIKqFAh&XOX`_8uWY3Wlcn-JAnmugF=!{mgLEbQq6c%{X- zrXjHF!a>Unk(LXNLSVS%5)FBj+YY|f>MAcO?`TOwFAtEFK@I?%P88TGB`CZRUE*!4 zx)f_(AmE`68<^uc8(I+Ja=r$f3In`xd%-=t5RnOMuRZ{~`7EtZ$67o{*2<&Hwuk5O99Jz+YuJEyjg3E=+ z0?cDOZeHgop@IZNjamrUV8ZY=2Zc4HoPqVdm4om{d}F%MROR8{{LG(%&-lObw~W7T zJ^qU?j=%L&K)1hP{LSNceR}+DFRlj<-aBeDW+J6wNQ1FR0IKT4crcV*jyA7%vZU4N zZbLHv^jGrCZx9Upn;ut>`DreFvbT|evR3|02X@Qp(O*<;LQbl6%l4SvE-89?Iqs#X zRnUv67^lxqEA9%)eA}5@&FLHOT|S3r)T5@VFDTT{m*^3Y)AL126q1MU#DyqI;m7=l1()*IlL<)mkv5k zq9)en)|8`+xOx5UH-Ad0S{@#zgxJ6vAq45@yJ1$NI`Zpxqb{JfXZ#`Zkk;Hk$pDN% zOm;5TJJtHp5xM*5dw?s~YIU;_2@%mu6Ooa-q1pu3=*D&R(ZZ}$%tU}J*8H{Xs-`=9 zvflo!&pH*41YXD7y z3*d$Y!EK(v2cllBFJDJ`67tl$ZH>OoPd;t+vQUPyph&DeE~7`bn#Zc2v!MqWEHoEys6 zAvVo<{nwPcrlza5U-!9gh6!)KH#*YR) zdX)kssE@!9w3uB+te;-UDI=Y0qy&+Q+ilnXTp{;f`$ql!-LHL<{y0U<{2j#1m%$dY zR$mWkO0(_$ihezBo4Q_#JU~Rp2wJ1;E-<)(gXO+!HVoh;Mf~4DOh|GxJis?8=Z@E{aoAzY?s_uJ1X9X66sU}=V$L9*{gA1A(E zqj0i8&DAqM$d%>Kxh+7*ZuEUoAFyCpwY6m9C}}949$&E9H}CYwo^~k;HXVmJY|G~Q zErNe!qQb!wFu#-nqRZPx3?&5k5Q=gotha*1u9Mz0K=zrW`;Y3Xu+dP0n!cN;Pgj)B znO?O9^g3;y?mnONDV^}@yG1Q0L7@8gS+b)&ND?M355O%Q!LsV~P_Ldp5hW z6o8y|-PISHVBSuoST(0Qo+J3E+OWGz~Y=<4y z=4=X~BF2Z{A_C*hK9&8FF*Z{&@NfXa@`M+d(;)Ou+2XKvE#*%s@4}Az=0fU7Y@S4G zm!3v~;?#a06cwsg+es-zXOE&VeNnC%R$-Q{&)s%I(yDBLTQd65BGiUCoF!&z!!+sC zKq7QgTF1~&qDPf_H;YI0;h^14YlJbjtJJ^i{9@+6K+L?uw}(yla>LJzveC;bWZcb7 zOKme@0qJ?#k0c9es_g=LwMKAQH~*xi(-}*lu!BmKxA76$vw-R=G{3Vnl_m71aY+dy z3vVhG0Xqcqxb(QwtX0x(CoR}*1;k@F&FEUSN0Fi|o{c}bE_jPuzYEt^tnGj&<+5@w zd<*=5umb`6u>sZN_r9Opwx6J;=nTx7GmIxkr3d~~q*a@LFIo%O%e^bGUB&0r&wV;~wiLu}6!C+TQVuaUFJMpTiuYILGX zXsgR;x<=Zq*ShUNt+QpdeE0dqiKaEEmd-q2t?LQfJ7{df5x)Y)RujwoYzNAwm4Vjj zK>$a7X7Y((#Fnfe>Vxw05#biFS~=j9ub(&>Y3bU8v+&d~fve%HsWhW#{92@zo!^ z@cv8DcM;wb1XVTynv*7^JH?HvEU#VQh94OCJAPnb_jqg%!!LE~#vQe4Uw0(-89%W6 z?#Yt>l;8KA?pFdOe*br#ytk)&!@H{G^%)xyjnZumsqmt` z@0K*FpI@5%CennH{&Ltx&f=pk_onI=kCpM+I7t&bS&~@l3#Z-zv9Y0)nq%9W^O@iR zj&U`uP7y>G(69*rR_H1Ag^`(SW>O;$_C@>0#4rz4$bKN}NQ@2Tk&d$o0+z+0ag)p- z1VSm?5ey3#Rjs#QN0}3yHb8bs7e_nqwfJq9QQJiA?H{#z-FjYy;HzxsXQA7>G-Iao zg#{r|)Vj8Afl@FT-YD|icxZv<@)FQmzD!=EeT&xDW2kh@#BJiIB_OwA*(}qCdn~-m z=oj}YGss=F7>P_C!lO#wfHp;~t!apil6RF`To*E%zJa`HcrzM4S-;fm<4xL2_ylog z=JwIP)+Su%JL+O!3dOr_)L4;(t4ju?yFfq1(6c{~FC3f2uggE$41w~SB67NIN-CDI zWlm~4`m{25qjX;3?+wnxI_JnO+%PcTOH#XEZ-l#OXsZUQcH)gDoi1=9E$)A2?4-_Y zTyRtg4cmvBjSIYbFd%@^=r)_cC>lglwsDcY^UIBY#-*nQbpAze0`I#@W5W>gjxMr& zjef7iHeIwmt9NSc{2z^a%l@Ifi1yvp9=g6J*AVqI&>Jo*WxjJ)1oD!F>iAq6piU+gtVAe*IFf-s3Uu zeMYb3#8kmVOh=JgePpxM)~ozW$`^GSvZU>zcCUG%5_tXOXSsh5Bt49u*a)<|@+)-1 zLk;3ESmSL93)S96^33{%-)Fif00RDPkOd$kL{W2TJDbm9fyoc~wQ9M5l^OsgQT0Cl z%4))3$`yUiUS#6IuG`JX-=K%&kkh&=VvOGY@gF1CUAP^tji6zYxxV70-!)0UaAK9b zLZtM|J3*)|!BIHNs`T62>fLju;5~$S+ZWKLhmS zl^LNrHD@ZwD+f5H^cxY2DNRX$%lwtwje_i5eOi}E^1J(Ybg_P3ryo8e`{ZK^C2e^~ zzv3lHgNyY`4869x(0*)4N{# za8Th(v2p2_`W-k;RUSE4v_kLOmua@Px*-VOFe(c3@z8GC5> z#&%?%TWpoydPSFG`PVituU{hFSM9g+E3~60F6msV(cOS+K02SS>233@A;n7^l{2G; z_NWly7j}{8K^wd0A-y*xvT!PPTM?Jp*6PJ0xqq*)^n+S|L_&hMtHGdC zxKhzKAWd+V35rSkhOAr{;}Bvu4)dBbK$T!DI_ zK*BwQgHUWJmL-G4&?!NA;>3#l)ZtLtj@i|Ul*f1<>&=p(Xh#hHeWECfY-@hgOx2;$ z9bklF1m`v&x2{J4F4u-t(+H=jUtPyF8YX}1R-_zG>~nN9hqFW~K`tUp)AS1FQJS->OK z|A-B6x!Ol=G~-pmPUq!jggC+849A-Ex3sqxXf#-sgdG2E!+WaXkx}I7 zRkvrEEA2kxsPLu;Hn27+Ea_? znBouF$+ENURQ+R|^o@qP``BdJBOwRqhexC%$w2#Lu33n9-qJN@LiU^d742&p9Wps3 ziS6^VCvR<-D^F8eyY*2iRh)XCP~N^@MhEDtt^14Ya>y!g-P@D}(*ouqLnJ^0%9-$U z+qe(uSYPtq~o*Yu!Pw*TXSQx5vcYIluJyQ5hw!5y|$j1U~H@ zyJ_O_f<=rVv9I%iX1F6p{w1ou5Mj)tlWJy6C?9WEAmD8doaYR)>>oiyganLybI*yK z7DZteNpmx_^i&s*3$##5>vOw-LQcQk=Xw=alOb{vnnEzofGCf+c}`74WSRI}vH)^( zV<1lR6T(-2i1dkzJB06#c1r3v4^DyuGv5Uv;``QYlgqAHx#c3ECd0Y()NSChcC+ve zPyJf1>9$JOgav=Zk5jV#2#;Cw_eJlO6>DqW7AY(j?2HTWhUK0vx44qM2NfH<*%N!P zAK+zZi}yC^dRMwK{~gjF`&In~rEs1XuGC_=kCLxD1fI^=W_rat1t^L2z^IF^7dR7MlRH41@t-9I&PM zA=Hi5NTD6qcgAKB2@XI4IsE#mm0H>`;08T%ygJer$?h$=nO84tS z_i2I#GSwc+^i;!)Hu(7l@>6)w29cFt@)s!JVZf-;jbu2us*MGd137GYY}V4ioq~=I zi(e8(%ohCe1aj1}8j51&XSG2cHhqNlD_Vec85?LqY;oA|Brzr`3J~UoNcGiYT>tWs zWWb5*3A;||`|Ikw<_^7D++xAb5Oc-Zagp(o=1|Jk>O!Z?3!fe{$4@D!s)oxDCB{@eX}t=9)% z-mYyIhY!v#T%3dQr0$%^xQx|Ra*RACkp(w#XUFA=a!O0mdx>?ZcT8$CN<2~vTR~Gt zqUC&Q!+(53L@A5FnNX}%f{UL#%q#Sku*ju_LA*0Ufmu!vK3kZt0+JG*;!p_zzTgVT z8a2C1RH|g1*!=1hc%4|4Hp$hacyNr!VitP-kp(>oIwc51Nsx+xWI`29e3dOI*0eLB z1@;0jhRt#eNcN(D9!>#l3yWxxU>x-V51r34A_3}3D9RsJ>bc}hk+@W6vt&L16-jd& zg~aU?hY}{{=?Dt3S;+i`2eJY;Lv|V;trz4}5gO%h@D1}fmRIb~?c(Hc0n0v|i_j8& zkr*lgq;LR(dMMq@@V<%W*Ot?@WeMSljJ&jMhwYnvSP_E-pyNA~?SP)|Du;ctxDKCIiDMcS!j;$k*F>JWf#seO=^TOMj6fMMqPp$Et*=!Nu8!*oqoSJ9F6$h zDg9w4=a(Lenr>#d-`RVmJtic1C7BAuDO*qNq?F~eS}o|&8RhC9I#eK3Qa*4AKERq9 zwMABWX|kjw&)Tpkh*(iS1N+%?3A=PGoB(s@*XW4wCif7t!69OCaBMXD>0u8 z#sR{DEM%!YkutB$)9}TKyw71G9Zk77%tVCb3EVw}pWBrss1@7eFk2<27=@xQcbB6@ z8-S!B3&wzn9v3Lqhr4YTk{&n zyhvjiaJGUP3k|9&c8(XDKfHUz;AFiwKV%0REmgvS8Qlzf-yjB_+KjfnypIp$Lgid; zxXGH^_>v6)Wv5k`MC^_X1qbffyZo$`{z(9vO?Z;{ELV*EtcTUSM%w=<75-9xMboPz$pJ z;1>9*@gng@Al`h>4W=moGjhetM>6=5D3$PIBcq^-teiYAb#=4^<0ujdp3UX$PG#Z^GU zEwY%-Alz>0CKCv=?7rWxAf<8~1}QQn#^g_Nut+0(KXPx@6QkH%@w)4Fl#)Xiv49}U z9Zc&!5**7yd8{Z|pd#3xX zUFe6GXoX{`K50ikd8o|eMx~*Dddc{#>`O=~Bss+1IhZfZ?Jv>gA(N?m+kX*DN>>;s zCQQ1y8yC7!6y@Rz8&-@ z6pW=u@o711r+t(G3`S0UQq71ikvd`~7%l3m;O6JK20o>>0}_Y~nP980 zN>VqSND!j@8&2Pg(0Z}xxOkTNQl}gDXxMJX?OKcGT0^Sc`sr}Mz{FY~Nz-ok27@hB z`6tgWSNN}mVn(kid}$~=dV((ARq`~a5eY z6GXsR<@Nl}#vLmdfK;6?_+&gSsLn){6nklp&}MG2MN`a>cwSbn18j4|i;9%Vz(?kd zVxrr)oUHHg32)}p%{s%R2c~n75shpmbgAp}gE;MF%}#g3ZwI3-_W6;5eIC5}=w?1@ z0u@tXu#8vpSfbP6P*klT>9*u&ayR1{iM{nA1 z0;M4{8i^GNQmiKx$3tI)y&KW3!BRnMr*QKVBiEWAqx${)%Ei#{=Tk3!Lj8Wam=b@a zZ`^ME)4betoqui!dE~#N0@_y~3UHN0U<^$-)i`pP7lZ|EjRs(~Er^|B^-rZZX}&K9 zfVNb$vz$j`sS%o+(fnC2NXw;>~uTKZ?T|8 z3_zM+J6l=L$8KJ&vBzPY>3n{KSA)x3-1Nc=H_ff#vNfnt4Jl>F&|MrMmhpZfg41lo zOjZ($R*-aGt@ZBf4WX4z>cXp4q*=1!5VO}&t~^ET9D4MXj@bgdYS{Ub90K3_e>AnO zL^#Q_|DV0Lfwe5l>ia;B16)TKk@g4?WL*k-x*F=fy5Ap#n&x#?b$2mc-9=UPOb8~W z`*Ci)SM@&1@4naFHQghjWMq+X4Ks$ei9CiG0@je&76v51ik!%!I5tv9BF9E{631ks zBpyUeD-ywu5|ZD4t-bfz=id8ny&02C7YqXW)w$>Fv(Nrmd#(TaueE0MkI#P)_G+nW zSCw&v>FVg$X@6K*GZ9|^B~}FiDc~Hjl1tTDq6>_+^&x!e7;JA#zk{7 z=rjtd?vOkT7szlnr=rGuIO=z&-DY>P8cq7crKq|0G5!72y+5n}tl75jvTbj(PX47o zqGpW7@SXWq+)rUu4D%u1OrRq?W_yz#9RkYHkWuh!6S=oK|vFFy}kmGMjoAQHX(Y*WVANlK07cg{$@5;@A7VqIK z9;>2&G5HNn4uBGHu6peX71bd)-dlL*2YJwC$%mr+h#|%1af;_az+)x~{v>7QT@UKD zUi|c5AV;0VGN%*3lEbB;5d{-4nz9@PYq2H>*tL1233De|6F>T~(VX;HLm`g3946YthD)4_L24T=gb-GaAlE#b7XO6eFYz z5vIz%LuGX$j0UZ7dpd6oSLvXC{_OJRZyDn7?x&xe(GnX0A#K$~(U@?72*l7^>gVbc`jeZ7P;lZ%Hg z?us|HzgQ$wj18fj9GQxw@pVw2gQIzYWps7jVmgCKPL4AAggt@Y;tk{`ybf>HAy36~ zO*)awm93iF091UEih0NGC1N_TQ*{`M|CnOOPaf3rJGTVKov)1_?B2OwJbdH0pihh$$o`~7d7gBf3(yHr1?x176BKL;KnhB%ih@BPRR z)SjVz)9k;$`2$;_3EpP6yPU?RBH+Mg5?v+m#=|$is+mZO2;C)Hg8ATtmEg3JcR&5k z$3cSZ<1gNfX+k80mFyo0BK68dNszdV<-tORo3D^f0%STL^m;u4)>}mZpM2OEH74ZO zX>=x|KE7BSu;bP6x4(3D0rGbcAWz_sA>Tj$iJ$weU;acT5}G6r@Wq0$|e#FX89}M{q!e(Nu9VUm4&yf z(gTxz-JlGkWE5f|T(DJ2GpxclkG&_pU)l9^8FMO}oR2=Xbe2oYQKP6?(z)Y#9>q*N zNbS7)?ooRZNZx+q#Y;1~&ueKGYCgJ$|OORSr7p>ymEYCt1r0n@;bVbhsPQ zdix->csFS6`uUq0cbUzN?UTC_=j^K&JwGuU{F!KEu(2Tiqx2+ab+kC76kxuTn=N$N z)VBxF@sPwA9qJli&Ol1JK*+zII&x{9xH^9hk5nU$(Ya?#bh$!%C(6qtv#4pUU;}yZ zE5eIxA?kn*{_j1JZCsi*^rvQO<{C+dP}V-MvK;*E?yYgjP8iOa!Vpo&MP9`54WbJ2^d_EyTlt=`m2ci@q zb+I?Ge)PhRTa5JGg!hPdwgC;nW#;Uq^ni3zo`qXW5V|0(dq>rj%(fe9C{nR z%tml4d>Hmw92%bb2>+Lc1i6p4@9rnDUxj@ZvuA3kCUNsKDhEU{d}ocH&&Q$K(v6Gw z=rfFRSl3Hq$ShgojqvmD%^1d3A3KKe#yAoWd4KwZW32DxQH&4x_~FTAL0=AXG>m^3 z!jEhcF5~{FMF30_G8kn}a0gMdc0L?%2z1ENHR?{Aqg9s1&z)Vk{7ng$Y!Pl;aq*kJ ztfN&fIgY)X(ZDPvdD#B+)iYF9w_>=j(5|=Nf6vyDFypffjJN2J`t+=X7m6S9)x`p}c0YLH;ybVV2& zcDxQMA*sSW4N-sh(;%iSFN{A>5X<5edso6)l57hwJ8RRh|6{MaB;G1X*pD81_x|F~ zRJm|i<%W_=a2Qe+8E^GNk-vf)L6A3txJM&qm*+DDmg0`4jlyF%B+YQAL!ori7l5*2 z==54W{@I^*`mI*8KUrmx{_@#{#}6PpH2P1*NYxD8oW+_L!ZA|ry%O78amwLwG-r_3 zEGx_44ny>vnE|d)DB8-np%rX|Wqq)H?E&#P{b7*8czz-sz ziSJQmguV_GU|p40DaLYzth|D4Jil1rShB9JMQ)`#A;ewG)~hyr`a(9sJ}o?dqBwp^ z^%h5RZ-5Ab&W6ZX^CbIMseG8+nPqybn5_e@e#lt)eDM$|m~Z%8hubOk9^gS{#~6T} zyAw3HY&Mg18Bx@bC~dMxXEn3a(VHcd3CbddZs;yFgJKMW;^8Z#EhD`U*o)nfjlB(<*>Cw+;QIxRSCPh> zN&yL;1O+H$RpQ+Nv&Ji9Sy$@F-%Xo)G*ewC3Y+%YEzFK~d)gRxXT!#H-W|-ly<$Eb zug0#4JQb^;J;rT0$2dGA_Sq>qJH(@pe_ud$%9{zBc3PLNkgx@{QE^wD{ zOwO#?4dV<9%5>{Y5w2&>MqjB~1HmcBiZP0KG-%nBFX3wD)nqQupgQhzT43Y>T%rh% zu|kZXY!FH9ihE50T|_^)_?hjpyu}-dU!v7!XB|!g?U1YSz)Hs%>(bM6w42R+i?YFw z3eKl6`VSfaBHs@wYU(hVVT&e$Ndh$LL*h#D)#y~z;m3p9%UFU5AWsb1)`FJv8zaZi zK2_rE!my6c6!t?bepI`b$4%stK$%N%xtd5ND9qqJ{`*fiIbeexM#yqng4p>Wsj|j+} z;hx5S#u!f&FBi17TIFQE{BXKg)bc1e?u_ffW?J8US&;&{3L^Oqzy_rueF^bsPVFS3 zAbd3mQTm?m37#v94cNCb&HT8$7yiPEC-WUGr!QiWLi?zHG5ReD{5PsS5U zqasTPydQV^9U|{%%~c=me|~nUqL{WPYqXfFTdZ`p5EHM}-r4FIv&8FQuA3}BFqw*O znyL_%1elI1Rx_jN_R5@o)@X_0Hvk7v)K`po2$L)sQQJ0Bk=0IvGAZAIZJ<~1)3Od( zk~*$fJt^I^REerm?k-y^hZYZSc-j`MB#{?tl_@ZTZaTLdf3238^+4emrZibd-|m6D zc5?n9rKq+5sp~+Cjsf$)kBIccFT)xjlIDcZCV_SEzQCN2B9Mu}nw5r_vd@-?f<~?( zQ<|TVaxPh)%_SA#5zEq#PkC#+t5WYGd`R(U(ugprdrV3h(GvILDXtRR*aycAo#F1f z4i1xW!5~J-mh%Ee4JLBG)}39V>E$xH1UgG*Kt4Kzv`1M{agQwdGIZi>!2(zu-?6GH zfSSVHbs1NAy6pbt-5IFnrq;qoB*;P#X+Waf_B=SgBUx$R@R+vBf!KOL!jyzAHM}WRpk*3RHSL|`Vz9p1{Rpy&`vJf3r|;&(QxN& z{wq3i>|yeYT$=?*ukw`OHLzP*c^@tuzC6fYl%sEthk)Z}Zh@4~SVSZ_pwD5VbnUV( z(T%*>B~UN5fxy0dg637W5U(ubP@(si1z;ld!uvF0;fp^aG?X*)yahA~#swKRCV)pc z1?gwUo>5>e%e_x{D2{Q9abju^c$d9ehAy0c<~uj3E3FRj#(dbNk~~ z14Ae^5&nIKC#mNt%SHQ{r+2~g)4#83_lIS%+7hG;iaefw`s&S_uijkqVwplAEh2EH zxg2hzH^1NkFW4>kjw;H6(VDQ!5J3`4udqa=ImyM8Gf~ND3qcI&j0Uned_X*?a)Y4a z!%nIY&dog`Ey7hUsqPuW@G7dF#jOIIcsKM?n#S_1a>o~kME8lHX$?SA1p^4#RN!Wn?UbLejArHR5R$j6$0aCx#7_X21` zgziDHF&|KWhM6-S%o~Fy*#^7AIne2>Xf}#Y2ao%#7!}i1Oy$2jyEu8yFqPjgBgvY2 zfkaU(y5vz%g&F`rDFbNw0qw`m!BQ}g9R8_=XPV8Utsr7x10YRl^>e!r-WW;<#(`%Y z&BE{v+;PAlS~|3IL!tK;shuy(%XTKAE+Cg!I(a{W03giNJQW5cthKwKgJHi*MPO60 z#LSqOoXl^j|r4ongOlY#oH}8QL0H*xr4N_YyvKD z{^Ac3`L=in>YplD8KDgtu6ib+V`vOAkCFp>v`p|7+|mOmqqg-SWnL+81YHYmgPi1s zXcL?>R&0$?jj@}I@g{G)#k*J&NRMp;I!$ZamOg{bL$ETh~^1T9sSkqHt^+2`x}WT+&YKV8!7e&x3YQ5PW%S z)0y8e1m)!<6m9}wM%F1*?#RO55FH^-J8mz7iE>z!9q;Hd{mJN@hGR$%k@&H~g~d!I z7{@aj=L-I(*knb=QBu7SiRhB=IdNb!k#T%zha!Np;pe1IVc#?Q`yu4RT8$BROUJG0 zMJm&xF=O})7c~%xr@|aBO)jT1kwHSGn34kw6jf16!ztBE^9Dq1;Wk8{a9q3KRXo7h z;OVV1MgeaU6N7nwHsPdiwE6-gIup{8kBSy)ygJPmMDodewi;slg)@tZN53K$+Et6P z|9u>&-0tH&9sv>$+CqxuT%Jt36Gm|~ZMP@gS4C})-_&^u(fMh^%L6C|_{y=%c3Z=}?RaW8ca7JoPADgFZX0byqBR$O6WHR|~4n1V}xw?_`{k9+|D-P|- z4JCV=#w;9h2MJb~>q4*)O3XUR_FVn8-;JM@VJN4?52D?JExDfP8Qc>9>kIX3PZR2C zPC^N%=w7pE=knoQ214FEXnRt8!mzwS=;k#>8odo?i^anMP9eB7JyK32FNJLf^9C6n zqu`gs-&Xzl!@9_OilU^>)&lj6M0a<`NAwJ7XE_~Y!DZ?q>;Vhy+d-#9)piM~qfnv| zO9(@EQlN3A*K36@ILoyvJdxe@wTstZP7&83uN22T0+mB>Ji|lkPSlyW0vbC-tEORj zZj&T!&b!Sv8Oo5P9g-DH@IjB7%~7K@9*m1_d(s+oRu!&)QObaO+Eqjl{~!9#`>wBS zy8jV{tJ<5=jy!#!`3QH9#H~=S{A^E%?W3*2NUH2^8Toag>QYmtFjFc}FFde=SuiC? zJ`&4^>eC`D!9(bV)LMnxS6nno~Op!Oz0f!Yp2xS}8iq{y_Kyr;oc zV0lqW`wUQpG-xP&(L;x8FL;3o3aJrEE@u!hPsS*v zfNL2@>_3~bENoUL1EIg!ZVwZIOu!IFv|C`X<`%-s&MAvOw;#6Z=&=SY^1>srADn7p z@xcHMiqpGP3!6x|z;(HmxaJwE0)cHW=zs9-|6R%3lbvui?r_lYdqB^Ed( zVcs0?dmE^{44AGRJlIcN523ynD}8^9pMiKdSU+5NRq^^qSm5A_Pgm}L8kF6eheP#XynCL{hQYTKoS25(Z`yj*2$<(LI;OiMeeX|OUEI->zwXX^ zgT}Dvt0Xp}rI@yQq%2zzFkf?a0rNR-4O`^@_wSa&*%_73;`mo?!QbO>bDsspna2o{ z_{L3f=lLP3^TSM%^;2)B`g!~R5G$TNcLxKtP!hev^sQC=$U#6$JG(q5E{LwPww z$4C)KS>F!ABpjcpuc_Geo_4q;Pp#y5hGjwUfq$s3K$>hwGiS_v!Gi zwEWDv2;T6lwP3>$-$oh$59f^+!kz1f;bsievE3_Tj{~))_$0ni#FgXBzI9GYu+k1t z-Cm@RP|b@g-_MmF_((M>dgG2xnzY3rhkq4!qF;KUh|VMAV+=heyRuDSmC;!3e(Yxm z`S7Z=j2LOP2!NVn z6OC4Fpszo>1o;oSH5@VFU}XJIm24uIBTvd8R-$v)OuDh0Z04WIV!*IV1Qf<#nZ&iA zCIoGq934cAdQJL`Ba$9)8XHZamYw)Zk~pnZKe^dNE&@D+&qn+F)9PKrdbgH-Vt1Kp zRz~twt}Qw2)!f$Z43D?|GUCXl+wH~CzyGwaY(@P+vItqE#rFu=U{59u*`Btm7eO6K zkiJ5r^Vq}`5LMnmr$QLyIucAcT+7>I%96J{XUV1d(;D#AfaRwz5a&+<5KmL`z1rfg zvM7$whu%L70mTF?chomF1vj*-H+Ru*z4Ck%0a;%q#FPvrUEaC5J5ujmPXwZf z-xSaa1Unz1q%gn>ISLX8zIpDN9PkJLF0w?5lM+G$fF!izn(>lN`c;#@1ZGs6#L$VD z1tudCY#Y2M?432Ik(HC%pdZt{c;9}^tp_OMJ%~xyG6FU|3WVhp!%aim;3P7a)UJ8z z?R=7%1wxH$6_ZOR!F}qWWO;OAoKWqpO>oI*JR0&(i-PvF8L#s`hYkH)`3yYi8!^O91 z=j}|1MqTX& zqs7`LG8U18i2{VK#K_`xn3geX#57Cs$41VH4rsy4Ixq}dR#_HL0`I|^VP=(hmtJ!+ zm<5{LLJ{zYR>87xvMxLGB?Vgosl`-U_9+!0|Ip(32*?TazspSCN|0j=GmV+GY{4O- zGyzdkdBj96>45XgH1XeivOB%66n`-URFcUM+(^U%q-NO$z-W^o8IXOt*Cp;}JemP>#K6`K>xzG&OrH*c6GZB^)DKJyMv%<1)|6 z^IJx>a=u#JpV8vV$<-O68o@LYdQgZXr?6Mw1aDZNJrC$o{05wf+74iMGNl;a4JX`IK8=fO@^h0}~lvnJJ$X zzQJeFbCZ!IU_4tJaElMTaYIEgLAIBJZ)?}~RldjE&my>F{#87(W%}qCvm0YcLl+!M zspR_AbiSrfGQ%ee_XSu2Kee>0S!tmd{BSc``OKOTjYB*qZoK;khi~8?B5+d4J87PA zk>$la%VZ7;(9M`qJfSPbz+oPwk6I|0RJ0dUhB_j^71Ei^>X;CjWniC$Dy+9`k{U@sKM*uI6NR96DME*k)EqR59uzuhw7(dn*KD1XqcCEt zHP=yODl+aEaISk)lMPDj_!Z?>i{6y9*3y817seH!G_th@lH5hyuEA##m>%F&KJDr#b$1PYdD5ZLV>DgOQAkv60VI#WNng^KbW#Qr~F!-kykyg3?h@yn$mEak~jD0$fH6iD5Q%Yw;O!6%b4TA{YMt3477y}Q-lsUGBuSSlTjyXBF`ncX4{1dT4d{ge zl`)AWI(D5_WlH+4I1y!3@C#7^&Rt01Sgg?c&g4BweBI^;NxI>78QaPp1Lq8PZ zA-W`@Pl?>p~qN=H*RWO;2he6L2Fa3pyeeNzs@L=BO&9TBC+{8{Ix%G(~Fz^NE1>LbwLfP)O1tc=Q+k0<}9}>9|ctiesc|W%P^uEuro}v&5 zY9q69A8LsG=E{LPDuUr;dmM<;@*cSUdiY@Mwa+ZsTRh`4^(5U8&1Hvh6NnSt9zmhG z&T3Xff<>3UxOX@i?_G8^HvA&w)`8mU6I)VwZz&E{aM7|~1TVmCap@A#fxzy-hSKfm zU$M)faWOYv!Ne+NuSN%r-Yf@%XpW(BeNwEHJPd2&6;yeQrYpv`i-9r~DxNGhX#g4h z1+9=QPA=wQS!~}RrcfnPXmXVqciq`M5@3YiYNo{Z@fHXXu$VyKcmdv`l`onO=3W_- zCNfxz6qYDrGHpDO0$BEJ!><^tZp>CDeWR9X{qF^sc8a1{1`j{LsFJK~I z<6IK`#ioMP`K}O4risN87P5z|Ir|q*tx7u;?e1zq1BL&ck(l;haO-<; zNElmI?m_()bRvKNQOb<6Sw?f%Ij#sXhvwPODzM7G6}Vw+vRl5o$^aJ7a3&`QMXIb* zR%OKG@&&HlaZ88q$pgiQ`1u0OmCL~=Iu_cc)7y?Bt!!_Ha3>BM?u%Tyz&|tO!)`$v zpCj8q24X}!cW={AnHXAdS)U^7#J1L(BtJs=f(GYhpb!CSJZFdk(vG7|}D6>L1OxO%@@xW;I-u#<3Rum1;_=wpW z`j%$J`lA>DGom$NX#)8?%Vnz>Hxww4xc&73=d~B-YcoM5A#vT-CRx%SO#1yknn%%V zstywdemYzA+PrXfS@M3^_3!DyDMf)|*+L|U08PN`vj&7f7f;j4 zTq&?k8^jbxcGA9uc4#tf62pQ3l4m&+V&UKl+sH#9{2to^F&vHKD&z&=Alkl0yhcn@ zeb$h)C%iD_xZRx$2PE#9jrtT{nG8FPQBh1AtpTMe$KxKADOXRt>*Xqa^vv3Luf5{# zc<&;}rw(j@Rpk~^K-V|NGyH~KTCNfKjb9z&vv&T-U;PcmXMN*Wf7Mr2A3-2+!1KS_ z&xWnC4=MO=71vnHJqS<+$3NIRSk0E@6=E31u)Ktl=-7^qQ?zl=g$!gw{7qC}Mga z!|SO>_1pT(_8SB0ZtunEvw8WK-j1`m?!hyTsJ(sZou|&M-TZqxq5ogKe$#9Mwd2c{ z20ELLh&dc_*9AOKWFS;O0dRYH4o7V47WE%y|8q37rHi1NtQ6$^uH|PZ_7{f_eW4Jd091_;s{6w4rUBVdLoJ-+rx@g2G^`Z zujS8HG_(w+jbsCRib`3=ypVPu@{*VCA{#&t76S!Pa&mj7%p`QlfZ#VmE!9k!Qin6w z_`9Ee^l8vuz9tHYT8p)A8+DUhqu4>eIpkO>MjTb$)@X@iv;YS=(T<`4>A)bC-YA0Rr`Y$6m` z%MUU3MGAf`5iw5bOl{9ZX$6*rAjQ{c(ut^LHX3NXZ`9I;vx?aIP0`6=JEH_?j={EY zNs~jDOohgsv1RmU-l5i-)qK5nyghIelUeXQ3Aj0hQQeU+T`ZM(U$p3X*?Aw8g zug(@6#h;z47vB&XaN;HbsxXkllUDX)%W0Q>X$ncox5-aC@h9(w6a&IPY8UJeWlj!% z?j25T_xXbVs18eGE94K;X-He4oKx*Wbg-rirVA46r*x3k4*9BLy63!|Kqf#ISy_eJ z1s;rem8Bo9T^Q@c$1vZLOaNN%BDQP`Dsz;Mnk1K41>6u8L)5=KZ%=ep;PqwKN`J9k z8}hs_uLkK1f^4-K9G%zP6YPcePyFb8d?E=R*=gt?S^60YI%uUh!v`-&*GMsk3A&orCkKQRXbLE5n}kkt z&}$DSlXhc1CaS$Vm_ps|%({(1e?IRIDMvZ(u7;n!es-brPr3EIJ2I8}aB)VAiZBEG z)3h7oESjn;3d{ zFR_^7(P`0Sp{Rh4QT32Lqn#i`{*n{|W9^r_j6^gf1XMD45GYE7vN$NiB0B18$PS!bTbCMNIf&{=p~#WCPEkbo+yK^Sq&(YC6*aF?NGBF z6$xX!DyKJtqlfq2)xANe0(6B;ChLg{P-IH^`_nCNeSQR*hfCbAcq`o}4oHmISIhrU^?Gy^f0l z!%;d+BEj?McHqjJDQp`6bYU)GiCVV@YI?f6a?l%($!^wdwA+-3AyH2uIWuZ>=KK%& zx>}tzSl>s^E@wW3i{XhJG+Yl9>Y5aAzA4VMobLt`R!(%+Wu}LFdm3vtJ)Q|=mNLJl zIkYd}R#Wasw$sKEcr9>9T9KBuc> z_5nY+2kK<<4wH3%_hi44I!A-%-F`?5n>}aq;e6N}Od8Ya7)~I2LD?I6jcIc>AS!#( z>-AS5?2neq`q6iv9rR381IDfd6v72DgcmG3FD2XggR&3zZNN$WA`i&ngA=lYl^J^S;u#6; z3VCDl&$Zc&Prp%2vKB7X_q{S7zyOJ{EH&k-XJ2AVJBY=zc6sOMr=<|*^J3niQqr0? zoIR&w{6Rd%9p)R)UY87}2wBkkq#BYw*Meg*(#oq)mc_=ay`5cryZ&-Wi4QWJ3H-W( z#|2*RDyD)+z=!p>Yi~CijmrPgRG0>C&BD#!u3z7s&Gw2nM9ch+x_RowgnYaH`M>r_ zXXDUXHgg*uUwY=HLm1ziUUsF-$g=dPQa7_Wcsk@KS?cEUlj`erFJ{3|XxMB~WshHV zu7|h@-vOkUQo-1ZS02A|kD*rQ&-{8dk&e{2nbsfu_w!*+TZhGr4zU!oovtC%B@P1s zVi~^YQA(x_LX_J+7KrjWmzz>{i)C2C@ln5tMaX`W9eV@k!6IDPs%1ak z618K~;oTdPqO{kUH>N|Y$u#RUCNAoz`Zk)5N1Z``1p@teDdq0{d;0sSdmqz(-gkXP z?)(PVpS%&1)}H0Ra_2v`+#v_{%VUJg@iWrLwZv@}qCsegP!tm7ucnL#=xb5O%cY3TWkfBzoUezpT2ib+(qFYyq)J%!06!4-lcd} z_{opQt(@(|C}r|j$7{S@MJ3UyFzapG#3Y(Kbf%G+EP5SN6*J8xLTKcp8)T9Vn0ckgu-86YO2cSi)UF=9zI$|dAj6`BLNEs5Ob1+d zlcX6

a;kN=ot6_ir}1Dy~w-e``-!+v*8l*O8Sp6^Jke8WdzNPgqvB+d77tB>fD zJARSoo-Hpo)SuTRjo~5Xt7X}w4HRF~eag+I9c9|x3_TB9q+-5$nzXD@OhMyN`jVBp z44;-v{5<7$XILZ2)5Zg@CI7gYmASt01NuNX)bnsU*bj@z(8sNvXcbf*J0 zbf-s=*e33d{(RUT&08y~!>7s({iuJsffP0Ei-ym65`0Veh{O{kHT)E32;QsToMHhE11SWsj|0RzMgpt$_T|K zOx>=!jm%`p*K_^A@EkRU^1R3jN3IO2I!(zF<>P~yhaLfcF3Oj=!}0VJ zE^{Dd>)O_ah(5KLcti5agZSjZH%TJk7=r`PjSWsDL7Izct^5k(yCk~+etBMwB?y5@ zvCPLSTYY=;#;Zqnalw69QqV`w=Y)Bk5o{>pieE05vaR;A+?U&>DnYtnJyX;D0T{M1 z{MDwk#{DdZmy5r}M(^yfIU5aoaL@WJkY&svQhVaBYZUx2Eqd))vsewx_$z0ZK0nK? zn#}vla{mb#yUJ0oWvhWU+q(?KU{$THX{7o==rSlIxf>)ZA&%vG)ZetC;)QvmB#G6M za~Gf9tJ_wzxfCACX7SZ*TSksw^n+aU@_$PaN}3s$5{m2*OFHV-#D>_)!xKl%7Q|I$ znJuY>QlSuyLfe9mlN2F2FQMvD!4()1LLOoXfknkNu=J^BU|(fIVY1rrIA2Zxj^!!2rrwYudyc)g358W=Ouz z@wnd{&*t-Sb5&XS`BGdw>Yl;aIvU??RFr3#TKx@c=rI=8uYPUtLhVlf_OHp}`b|1? zdUNUENDERW%HwMEZ7Yji3*7R&_-e659=5Jah_%}lZ!G7tT!fA7le=2Oh=>&MzKz)yTJu+WJsh}JR`i&|fs}q`5(OEBun^nn7$Yovf^b%J zam#6f#g&~kZX`v~6j|KjL;dL!v}@yPiz-gJfMut0MdW3D{L{))cU-0{<|pH6C@0v` zrcYjfs87ayGS2-&8|A75iAI@bG>-p@F;<#ij3smnghR?tnH`}>zo&6>&t`ld`iYVo z#qkZ)4y-gR3$z#mDi3zO=FHeG3V14N>Q?GBw81Y zWn5fNfW^5yQ#;ujc>x_x<4Aronc9U1#)p|Te{}3unD*4B8cGDQOe{tkzT9_eFi-Xu zl{SrUTm}ANxF&OPsz#L1Q%eT3Obj+YWr3j1$0(S6&v8t+SytJJ6|>B^^!oX7YH_Vm zW@;|1V)Sf$UZ!E@o!zJa^=6ZKk4#>T=2Y~wE-8x@7DlS`ahD7kz20;U9{eAd67AkE z>F=lRDGLVX*NUw9MWc-v$MT67tK5>+ASRBHEBF=jtK1yf1bq;j8b$O9G-u^N$PkHI z42-JZq%4@0sP_C#j(Pi6N{;a%HWvG%VjaPAl^Uss#CI(>Lwu^SZNBI12NZdeYS1|$ zU;BZt7rua6rpnQ7b5&Mfwj)yZO8gD8#_?f;|CC!N+DtaD{uKfBP%q8=IL(~W3o8Ol zPB4vMsoA9+8a`QGlgKP91+;njExmK(3SKs68L8nhh!x61^yo*)>ub zp7;SY=50lh*=RY)F_N@S3&?llA%x@~dX~*_Qo7xcPXER@=@NHKUE)hKBvPYEN5F1Q z$?{eXYLmOz89EYWJdg_ma`~zh2G+FnfM91y2Z<+URH_!_)Q@bxOeF|%}(fXzU zaZt0|KvnREBgrNI*amFbMy*N*9VF4^lo%8dV$_~=A(4jdSz;tIIK;n0P3!!N1C{x3*3Z;)(Z9Vs0c3>feM-ZIPYlO?nbdcAggzh!$5;_%R@v1l&ji}V!3j+Yfpw=|g8B%>TN<+J+Q>#O zjrgANUL5@hW(FaK2f)8O?yiVzl9HQI5)8|gNvDEV5doG`g}}r}o5|o)Y!pJ|$vm>B zFr?$1koW*YLyTFxCK;xZNl&kO2?`ivSJqh=8HWUs|EbT&X%nWN ztF7y2UFzP)Vsl|49ZyXm0`L6!@B44R%Km2q&%8Zvwp(PqZw#g*5Yus|Pf_SL@hrp9 zVAAVPhUh;l{^q||3bK3urT%{E-hZY4yzlyog!#K%e}c!5>b2F)l2aaP*@{pyGy>nW zbFY=$LLK#=jk6TiwC;=!K(b>f0-|wiJhw1wLt!#Y!JMJ&E{A9G2hGEhK>Dyo5$@=dao(NAzDt&RwoQ;dxsx=hNnbTy^ zCX6{JVIit5RX{0gVcH+=j|m)2d#x%-4F9-$h@6`g56M@ecwDi1W)358AwMtTIkX0i z&dxf7e%eL3+L-3ePlY7bqpU8~M#r2vurfO+17>uLC}CY+t8sBsKcWfs7&-DevIVUo@n5Ea;8ZZ{GsY&m#vQp4<@u9rk7YP8m@;1hf z!(q{!kJ}@tP6IKYU@?*dr`ecwp*~S|Yg+VINz=bxuAoQXc?OmGL1_U|a*e3gfMH6h z?w-1u9-~%&;cEgSfA0_dpRYwk{BQ>F)ZkpT zuY51cl{*-dI-LuubwiQrtSewJAXA}~wqA}JZAB_%CCJI{yDJ}~Oo?Vd@&S7_TLfTq ziaf@FT}{Y?Uj%1%7@h~^3K9z{A1YI^6<6LsqeHk0PJ_6l(X-wP7vCq+20(#M z^c02@{gI?W4FUGZnGeFyks1#}p0M(%suASok6{#{dmJ@Na&31rOpX>Ra9dhz)FG-A zt$ZZJ1>zL7liC91$w~5IP=s=J!RRIV;tWVn;DmXdq#gk!Emws-mP+=eWk-{D4|Z|8 zDlS}WhF}YIZVUiq-iND(moR7+pUS7|rhvXP&7zfRr+~=h2W47yt`+BB70D+4UAKdA zB?YE-Eo?fi|NN2X%hEu#%JQg!$C=)=>0oSihO^O}`hnmBByF3_C^Xt14`FCi`K#L< z51MPT;48;XJQ)9H{r%Lv|3&|S5Vb68{xR2|{1Ig02>D9-Y-QMQC!-l(Ca@bZ78ceC z6p-JCD>8_N*Nmf7`OTo?e3J8D>s6YN5bUdHkrQaolwHsEyS6ndGaCvBJFSNLi+JDU zc+A@roE^>3{JI`v6<17$!(0Z*GG3NH$wx`*1yjLMDzt133^fYp4H(!><2E46wniH-WJ;W9C#(Rpezh<~S>-r@RLzV)YmD&o}X zj?@fZzFY_)HKQDo1gXLVWb8RcwC${~;rTq^#K$^a-5|)c(ez( z3XL&k-AZJsO$S_a-X2VQMYGWv_R*e4WLg?_NclEV^{9TQHG$Hy0#AR{zfx+?|CglK zQ}_OF{b!9}`7N$z$s|+eu`l`fW>vPZ?RzobzT}}asbR2nYK~T>Q(E4 z>UpFrTa6^C5M1?;(l%V)0oT1ye|c}bKfXrX(3`$qUuCUrUsDST1!_>`1#@p2FmN_s z0?%=1rwSzGNKz)DV7)%B;0WqFJHL{Qz`pyt{lhI^vVV9P2&sJ<3znPI4iWg(3?2&G zrt4)giH3WWtz#ss+N&QcGsObH->m7^4ltl0s_fZe!f6$9%^;6{tnj4kq%$e#w=7WEL_U#U9bd z*3)Zlld6F+iD7kI^}dCmg>*XUC6t;`3<3D?$32sA_*Tvl+;}w|_bjRqNw!JcOuDl< z>9-ZLLV1eqN-#8v2n$@EFru^bc# zeD&vj7hiyQrZb9CY!{TTi5$1289JVBY5itLtgQ&0uK-$mWYU;@Z49@2Wmw{m9sn`bBM5dCUN^!l1x$cox`g9{dMYs5rx45QL%Niq`ZIJ;A_&!QK&b=z|C<_(%$2)IRbI5kVgj>Pw#`7y0 zLm0R`(lL6Yc;D@$P1t>KyxozfEE+a>+(X_M3%+F67GcvOOBaMO@aZU->@@C*o_)eB zr-Yj#|Fu!m+^b)U==>}98V@E1s=4=*?wdScZuN3w@-kKgt;Vv_TMz~fuH5tLU{a@O z0pFKmKudMXJ{m8+VoSYdI5Gn5g%q|ZEC!?)Q5GZ)U>qNYmhP! zjFK}>)qHHSxU#!%@>&EBDDxc7n`8=fUaT%yLY=AdVGny$ZJ1D~dpH_RhOK7NZjL%D zD$LiPU8el+CR3i|(PN;mF>A3#*sPe3F=C|gKy8-|BlWRyIB%fTG^lOF{WnDlQ86o@ zld2(xNm_=}aP!NgXx#%FvndvOl(LuEgz6wVlzV%+cX+~yh0?(?fn9;{36U3(BNST}@RojE8?eX664)^B(V&%-UmzTtm=md(vhvCfJSER9FUjC^G;xm<;74AnW zFN!)ja)_V^7I3l-0<#Le%}z@8jn2v+fn390p^^(05??hFx_^r(Qiw3vrIv@z`pB^? z$}s-{e4idYwDICzDUo|t@iPsmRJ)_%t^_!5gZT{LCAWfFW)jRdVd=U)|&;+vG9MCK44=>=(Qbkb~bH<2oB2}~L4VQDhLGSiB)uiS)b<57J zMyv(M0yswJP*+I-YS<+S(yD`R9SbV3nZLqCc_5!-DaZyAoAjZtV3H<)m^mF>2=_+) z1F00a^ce}VXvx_tc$Bn%K^I~z{*cC+qf(}%h<=u_aXH5!8oirq6Nmt>h)t}9qG&MOp_wUFF$Hy4{~;ubThlNkv?BkK;8}L7g1dcgP@s$yMQD38Lag05 zl1s{<4?(5)d36{(Gm}fepZ@VmjCvkMeoDHMT)EIn_+T)OBH3C-41Qt{A5Rp{^f4v4 zW8aX}F;9`Y$ULN zTwGl(RqIV!OncOY!84sJvKpB(DB8pnHQSB;WKQQM{n4zq%J2D>v&$5DIpdta$A>98 zh0E$JnS97i3ayanWUen%8%G+ZOLd#a^6=O8y(zL*!<_D6gG)Bprqe>$I~>o{VD#RZ zrW~sx4wq~@Jf}BH$%LGAo3W9LxijGhc)Qw4lY?KN<6c;Jqt8Yx0sMIOZUTCOP z^vp2T!R%pPrM=4sB8ZXzzAOVa_Dh|ecsAV&^|=qH;qd8$dB^i7NNb)k0K?XVsdX9` zCLi<~bCcVGaS1>~2%7R5F4iSHlPf4Fpa` z#_#q_X)M}sQxGe3HR+uJ3t-OCWUw2fQJ6-?zDp<pu^-j|3RS^*#Kz{vEETH8(vwaH2)fMY ztUF^aP~GO4A+-iVO5kinbZ=PA(wZvBxjd064`A)`wTqXp*xC&@)eL%Da^S|e?c9t< z!UzZ27!l3MNY&H%3@7)HNUeEm(il&NfVjp%DaAGjB?;ojjxLtHYf#BF#ibcHR0I_}=R z>-I~~IY8>hN?4K>jV*9_OgQ>T#iXztD^B;KSGk0UXH_oCqQq?Xg9XcSKlv}Sg;`x! z&(&YTrXu>x^ehPg?HkWM?t-QiXF=g}WJLZV#-vA08zF>R?iw0N8LT(2`xw)E${Dwm zz_au#S>8+)s8GsyPb$i{hivQ6&efm2&`Uy5r$*aNdms&(bv+D_#Q+^<8)_@tZmZM9 zW*Ls#Z2cDI%A`<`=CA|xWilL%XYCbrqIKr2|LEPbb@2~IE}eOA-k;B!O%iUk+j#0H zy~b#+vF|tA?P)RZk2|ZN)36-ub%{&Ya*GzT`-H?M;UJ1pOPldpc`f9keGJR!dt&O+ zdw=@C?^6ITni(8+lE5i+o@3FaQM4-T zgp@|+RlriI2NSoJJ_7#}>e=^6mazB9(tM1jtIZ~z1?LR{@fLWn62{>naGyOwfy*K4 z)LcaO1m`8SVZ7Lx9FC7SFmG^IAd!UAHAn_B>5^OXSOCwXwCs5zNUk*DVn#siG%YUSv ze3tC{*;C9M=_)& z7bK5Y9hSGoYNuM0P@F_v;s7e(3!Pr&jr{KI`LKsx+U&Hb;eaSYIQ1KYCIuYEc=6`F zRc^pbXBS1+%{YC+2hb+(6Q8jhT!;asqOd@?{8+Y84}M@KP5xmXQ|0NK>}-r77K!#T z)5JzZlL(NDJqKYn+!ApuJM!4Dh1I3jmS1bvg}TS4$`E0rG%0QrRSlYI)hqf{8h_b2 zxyA$(=q=$;f+bEwk+Hcs2qPl!Xa^`yIS60M3NP3|OMuLHAEkE$d6En`iJc?-m<8uO zRWTLpSFy>?Xr%qYuz-e>R;z^q(HJ%Rn46Q1vWU%to!-?>zrKvWPy9{5VU)wbEm( zjiL_fWIc69GBP@=MMh|Lq zqf0(S5{k}9&7d;aqiL7)>Z`EMzgY^kd$;xXQ}!u*>L9-f=)?=S9LcAr1>^c9~sj56M zhH@_EA7uZ?nwByJV@uOTL|L&_Qy;*H){O6ueyXmrp~O-Dv{T#tW;6QXj0i#$|Hp&6o@g2A1%F9Y>5@@Z;$tT|&7XTp}%(gr+sj z4=Qxdtm9BTx$;tdA7T&nC+#_|pH90wUXAp+b9O=V+dTrz;)$7&C@CH)x-_1u(vM0HkclJqbQ#=ZDu9Uxyfc@bkIHeA zddL0-l_!m!7lkUhmudbl)%N!Gw~X9WyP!WKFyQ|>F7`RGjCW)UE@DRM4Pc*o%JZgd z8O{@)v`i{n4*B>cF=Bk1{ImxK+PaRePH-yEogB}IOZ3u>DJ6MsytnCdH^dQlY9|a{wiv zHzVAzK+h@K{mHaDp7qDARg-OgcA@i65jyXF`n@_bl^~-qU*u&);e;;=`W8yVq|=wp zZg@ga&81DtiEAK12w=9uGUuVnTY(J`tXazz_Leu_&{2rMw9++}#~z%48jtII`OE~==Pa7Q@PmfgeieundRVe+xKW0nP4=?_B9J>(o6(u zmGOEl#vsVsbb)XR^1D*KZ6Q70vLykHIUQZ~I!Q1KiNSS~tmLW6Wp<6xkL)h78OjLa$aKg| zKZItWnx2l7B?zqy15wJ|kk-(vswP)c{6a_*iIP+$d3kDEDc`(21H?+SOhh?H$g^{> zG~_qCoW{+0e>54j8q+pa!MoETSyqRWX=B(PH%T7dYc42~1Q@PcrOiw4JWohQ3bS}63Qonr_3v%Ozk z(c~Ca=tRJ7W3YzJvt(X|pq-VL@&^O)B+vPvJz+%;CnF)AO%gMVC_&xoP6oZscr>1O zSJMw2m&^Lmvu8v+6Il?OBMB0y#AHHGf*_YMi^pY{`H9H=xc|!U{+5u=f6ht0zNwmc zSq&@poW0x=d^-VlZ6bi+3BM3Xk~&?S6}ZaO%#uAMs4IByvp4#?;?r*wlhP}|fxJI9 z00C40f}CWqr92lju*xHeB6oRh=g?=!TwV@*(jqMsmJNsMmw-e%V8>^qw?5!#gMnar zVd8g`!}}^ogJJ+`Z)XDjZOaL9f`-B5<>!TXGn!ArpLPiiCkpWG+S`prqw;?=70Iq| z=S-Wo>t)&R`ghbnbR^Xnqv`GX=l>f1K#dix<^0#vA76S#sUB!2qxCGkjL2$L1U_7S zU8G689eE&+c~X6q&zja43Jsgpw9g(_{4cr@Y7k$!{Nk0zuiPVSS)C6wl*hkbO{DYm zZ7Fq?>=|ip*Rw_9)F99?7b5y$0DN4~X{Gr(Us;>cH5F47t;;%MMhl+m9eF)+uJa96 z)goNjs%1ao3OsqR36`LE?YP_P3>qB-8igJP(1-iec4OWnqLM7fqvmimB;g~ah`aYt ze?N8a!}`zruCK_Q-$3rf(a9El;@rG9M@%|rMp+7TbN=Qbr}fKY4$k9eq>PKzag%=( z$=*W^O{=K<)g1akGmCb}Ekp!f$!&|FOshPC9k22B;>{^;1KCm<3(Hup zH|)@vMrN;=6H$dAY?jJ_tVG#o17^3CZjvM1C)}X%{sPNIVOPcSM zhVz7vU*rmA%gaet&qER=>HouAabR1J>qEjdFgbF0BbgH%bMVY8s2$5_l za@xX4dJca@0U1pU+aRj)Z**n|n+OXZ(i$Fe?lZ2Hc2?2^%P|a*NMEszcCu`n{i0PA zREcU#=5n1jTkuQ?S8tHhu0Ni1htqX@kUv{)=trGXtfO&Ivo9Jl*Dy@Z0$vf{vWYHU zw;Yh$#{G%w@iEkoFC>HLuYS`n${_kDh>JDtoDvwH7)EwLNkAWYzgR+9Nc`G#Eys=& zS87wc4_kS|gIAR-AYj(wE3oBEag=Qnf{b$!e*rx5^~1f_b(AIK(uOEt#R;7>JTZ2T ze`tRSJOD;8JzF*uQPm*vvD)|D!|~oC512{jn(YatN#pnm-y`(gOF}46lE#EOE<ub8&pr^~)ICNs6xl*K0HudhQG^)7kY{;-2uvKap}!%m9L^lO&A*4WXW%kRi7w9JQCT#$A2~P5OoZ;MO&J&D zc*V-`i5Odk3R||qsXXcW^y zb3U822JOypC3fnaQh43_r2c;D-k;NdNL#-wUVazx0?g|G^b?OB{OHTa!Z?1+bfBY# zuO7B^wt2!~P`eH*j zz_=&(jXejuK#5|_)G+yBOZ8ICJKp#Ze$UF*B(ID;Ko^Lls3z#QM%JB8DEI!=*Sxet z;5{bSjTj!{?8Ry;)(1-^pJDpuMp{2?8b0T;@8;B!sOEA#|x6yZCT{*TpGf zIZaD?nsqG7aC0uLm0w|emt>bG_6T7@Nvq7qE6UIsl@#WUSC8)EeEYD3sE=M;OxrVj zU84N@<#H+8Y7g{(Jto<2ISCfDNQ&hLoX`im0AKo1Dx)35K_OUpHp7j6*?x04>~;)m z>7hDxNjXEN`xezUJF`K1G@5tDtHAQ7&Mtj^7U`qOq=KJk28(k42^qVJuB0PC18o*b zBC?j)hcqU|jD*ax3a^!xa+taF$^tvC26;W+s9{xBZen?I zGr1Jtdaig6X~fx-}rlav8M>BcbY{#tG;v z7!5R#6YO2+E>c<4!LW{{pnM5c9e6Akn3V)CDc_43m&_G!i<6upUA_`~yz@stx%0$$rkG!E&*p;14y)xC<2s2A-N@C>w3e%q2R6{pnCB;l8M)o~$ zyfv6bAZlrw#-{O&tH3`Dr(`ZmBNbmu2AfR8H9loQdf0r7g3b3_vdPqL>QlugGYY+a zzMNWIYmu3b3+BhtXM3!4cBA$)CviomIVC{3JEsz2f8K8lTk}z)IcZT=d^G8_=c`oM zzfy{`d!Nu7q5{^_$}Z^Po$kxXBrB|H5eVb^jqY77BaCG$OUAbV#jhE%tIV#HSkDR=Obj z0WIDsT35xnc0EEbd;y(I*>K$8s;s_jM>qpX78i~aW#r{Q-o#99w6_FY( zk7?GXFhZ@3m4n*kE=GlpM1f>5Bb13`MG6C0?qT1;&xj|Ls^>U6Gb&Y!<3>6a`7vrG z!X9#L(rbB zO%kIy?azpB>o%IRCN-<)^A;!WWZ0NbTFp_b)0%bHm<@lUlo+R`EJ_ZVsab}Q)+%gM z+zz5 zX2pD&0dfo**>eh|aMfeZ#6YqRQEq9ES&$v2mMjD~bi5peRE*TdkGKf0rK!^{e21FW zKjT1Umcsg(dM>)Pmq!4GHy}qG0(od8AnunnHNu&%geHc!t<;cI&QJhLB1x4t0Uz#k zN9n2eVlJl8q_9_a{Unf&@VliUtBkgNeHghk;#T5=CbjZ{=0sTG0l+V=avyLzZ8Cpd zN#8D4CN0G82?JK@a@ZGXGYR6NdI523O5eVx(4*s>{U{nNHKg&mgF#TqZ9@JT8_ML& zUT0op{Y2h-p3=}G#oGA@aRAkvCXoHK$aU8Bvo2kawADmcNT&$$+R# z4Sgbpj?{-(oLJ1!kx*fVWM88yAH=w6!(Kzzk10q^(+iUmjR5XUwa0FgLU%lC+)`G` z_adnx7btkSmiZL8_D|bXpiVk z^xnm0cTk~EJZx>OY%Waga8Y#)xS=*-Zk z0YDpkZt7(ExJ*`5to^>9pM)t)__h zugewmX!tnvJzM&Xupdk!N|ElKx|ANHN`K*NLb98^U+n#*2*ID$n1v$7?P*D)5O~Rt zoa+wKnbV%FI-z1d4hvGt-;{roDOT0zGJLy?!jqhktrK)Gz+{9+%I_f1QJ@s6-X0ou zsI8JK-Rh^roG4V8>YMzBsPmQYMc4A$z)`+eq|~~hDDeT=&Cb5DniDRBQJrZLcofS( zUP4C(@-fO3X9gr6uvc9u+9CjyQ?Lm%glec3IV@;B49|m7c&cVwK2)YwD}KCz7KdcF zc*#pPdR`#|Abj78+^C`zjs)&yOhd#?^gdFWB?QtVWIhNV$NZBob3j|NGCEg|zJU^< zdz>>fctv)1gcm%K2JJ-6qDsihM?%aWj!-+MEg+uUBT&h**#(rB*o!kDC4uARb+S?f zSTyUG9!te}(z2t;y9bt}5U#k^42*ttZj77Q+znTaZSm|4P$@!B@q#!}o zS5YA+Fr0U)D}!zAxg^1?(`u-{h`UXm#=K48*U|LMuj?^ZaV_u5VJ?FNbT7*vW#EDD zhdml(8|!hZaD<~n5h&Rs_))ctlSGXIH|l?6Pz$SdstgQ{G}x*Rpa$Lvvd@e8SL&#y zAuE3bK5Mm}OLXna64Q^b(|J~E!0SbMdVL*ho4iB_%6tHjhmswMFPjMro|K&TtP#l1 zJr{{m&le<$)ng2j9}lH4mF!>Ep1&xaS28d6;(f`g-j@dKZW1otes|hwk><1Ljlmsf zI4Q=n=CIM4GzXLEXfW$c1}nnl*Uv6oes-k_-am-Csp|{?~H3JbLC7{W_-kyf}-=(WOvtl{pVyMTl6fuE(g-55Fm@ z(*MtY_H|OFKg;p9+6;ATUf;dT_BX{f1P#Vcm>?+C+ZEENS=&n@wzlRSx6z4{$RqAQ zqv9P>XJ&>3 zZmCU%BLU|h^cA}EI%T40mm?qz8!ad8!P8Xpx=S%*HL0pOI+VtrblC^5?-ER)Q$(jC zNun4Z4lWWrSOsQ(+0?P15(MG}ArJNoCnOCrN3yY0tip>19mjqd(d6?M$SNwT(lFMG z*QDelth&976}!>0O6U#N9pFP0c?D|NG#(Hd{!Ei#=B(gem z$;8$jjyn`kAp_*R+3HWmlTp7pTP^zdZYg>0{ignY>fZmR|Ge+|ikSGEX~9qQ@G<>H zxQG@@xi8p*KCLUWp&cWg2u(|_XBJ>NKePw5e0XwqM|JqhJ$Na3ghOJ-Izdj*nEd%5 zT(sM#Ztr%i&$t)79x)r^V_%lwIopJ-j$f~)qhjA zdYn97gH4&`eQO{D7e%D%X_tod{b{92>D1nlA0-Afz#dCSgv7M4iG2e)30fq*EKAn3 z-(c?|yQRPj`Q@z?uh-xaO2t+xr=yG#QZ7|-$+Y;f=`}y6XLF5~_YFGCA!x5v=5SJD zV&bavUQnK+%w{Vyo5djre99pEOa6n9db~cR{JoqkWlz+(K-9v^vN^EC+d9t#qlYs^ z=L?6C1aUYTj^8-6f|~klkt^W{UM9>})j`Tu7?|;ds@z9jdlPghNwF}P3*~ncrJ4}9 zlYPq<=~b}G1b|wpK@{fa$|hB}Gw%$hs<}ixAS6f^2{IlK2-%qzz2S5|?oN8m6;=0l z&MrZeKEc-E6F6?HR^K)@?0$yhG(ITy{HXAaR37FO3}TTtZg59sX0KpmPA&DTHUWvz zOv#d=EsY}XBX5mx#v`l0Zky#6slksMdZjN5-M`q?j^mQ7N`Gi_gdbkZOA8?MGwW7% zhL>x_0I9G+=aJp!N4%k0B;wb0y;>gY;nZi7@htW+MX}!~EEgRL&*6h~^2?|V=ce&# zb*R(Q4c8P>D=$FoQ?`-vaL=$OOJA6k>j+1X_fMxc6*vwLP!Ga!jVeIhG<-j*zri3y z(Oya
R;78)TXC{wPmOmQ@zGjRm66PQOYTkctZJusSYJV_o{!M?k*{VHA@IG5>n z-f>zm_02?^Unvczr@mRt>ADinS-l0m2gj&$y8VYFJY4667JVX`167vvS{eQy54xtGHA$Lp zk~Bpz92DaFO**3{1*(V8|60_Jh5iRzM@_PEv1(NR?%5^HKQvwCJwHS&jvV5-Z7Tb< zEEM*tuqU7Spo1v1HzaXqSG(hp44b4))FNPAIJ*Ig!dT}XI;WIOrFljJ57~-Kojhsm z5*C@(Zfn%CTSruz2wt`zsi%3E#U+8IzxCWYJ^p6YaoJ~dLf^Ru4rM?+>&^6E&RZ)U z9)gm+v2gl=Sg~rAJCd&4C=I$ym(D8^Wu_b3ijk;KFk9}!0=Bc!7lN-(0pD3|ANvB| zH1GuvO3H9>9o2X z9paoB(JTxSPAb64NL%lrPT5@ZJd>@DcY`K$AdkH+wK8Ur`8_ZXwk9$2ifP2*@yArv zNGni4Cfv#=tx@`-YXFnCJg=13zuFI)&6QQV;WFMG6<7-vSV|y}MlMZ5?C<6T_`~_2 zIUZ4>XEtpT2;H7o7Ka&msrx-j$j%1C=Bg*)5C35do^SZbJLh-4@o_eg%a!-?=rGF3 z|3m@;j#0^lQWqvK8+vtc4Mnd{upxNXcM1#_vU0`rszJOb(?jXR6*l6kv=b|gvch3N zRr;jI2D(&OhBZ$$We7fNiFXAVA+Um!>wwUhO@Ti_CTk9E)Z_cdhm;LueeNF;5$T!_ zAH>trtngDKXs>*9dA;PbA=!IUi`uN*%`EFS}+=b6kn`K&a=%j)i7dKQ9#^7t0Q^^Bn*t z&(Jvz<+5m+^c;G3*-#U?4d2*FRK1Kz6Ux3?e+m^)L~T_ME@N(5P${quIqs2+!mYyL zqNBpTkgy>qA&tZA%LMu%!X=I^L;%ToByWN#j0_CJ7b`V}3}dV`lC*i>rT2uwkXc$M zkqk?QcNM+EK*s3fU<*aP(FZL%2Z2 zXh04eQpD!2F=)d(eH&w}u*$>xU7Dz$nR!FYw?a~Kq|oC{ny|m=VE};;Zg*wDlhVBI zi?al`XP73gOy>qiR=&?wbQ-fF(^n)QQMXJqhapWmpzDUEeJ67vSJ5M2w5WiTx2c0j z^TEQq>fO{3%&q#MO$kf~C89CO;OLd@pZVD8k4bj;$VtC+Ly92_PGbleC?b}~H$uMl za2WXfvrUqC)Tv&D5#>jNjUJse<@}(V?PK9Am?WJgBAFupt)rGlCwMH%%Gt?8l4#v%_H>4WQD-v4 z&Ot}PIYiO1ZkuAJlj*!an@(HP;c5lJfBo!YHIobD%6)h_1Q+o~|h!Mo>Kp%CiqW95tpvi`N3FNcY={AixcIv(xmDVC9C zn6^~z=;Y+Z+%`XeA*bmy_pO0oVS(%`b&=>>dF*UBw0NGbS;mb>?Ui(~%o`+mOZ7zg zDwaRrN=E;9swU&ifBHfZnMB9cak45iqkHDQCvFq*Vj1dmBDCZhEq zltIPe3EeQYAw3-Cfh~q8!>YoHi!fCb7alLXVx8!-V;8EnfChkO+yP`FrVM-F*j9Ex zR6f&^7LddD?w9tKKGD77E^gG6#7>z%EPSQJ!(=RQDj5-a)>?C&C{u$BxW+_{LzDw+ zd(!Jd}*~O12~+J6Gnq604|y9hvbH8%v)2Hdmr{j^VMR>-+XrY^50u5@{>M- zHk(OH%)K@xE*Esi7!4ke*`O+>Fm{^C!HL|dXmcBuuxp;4G%Ub+<-w(uVTBm-#Fo!V zZ8uda%0t!JF7a18qLWlK8*E9FHt2I25=S1d5sgw6ETN9{Z3mLIOoBBnSJ$>S1JTVX_9FdR)`=?j*{wmI#~bxfx4#Hn}(Pwp@LtAX%w z@J(-hg9snLfSPbMw2xSS!CdF4akY?OeEZtP>o2p~ya%k${X;mUiFapbuLYEZO#lj}fLi1iB}Twmxu}=%p8Y_Q{WUvdB4N3H&As5Oq8Ut zHqAK|5+J&7TWF@AvJ|(7|x~InO)<1HecA0dCQY0-Xdk$>Fd! zQ_BFO0UUtU?Xf<7e>S~Q#gW)%94{`#G7$_A=fY1(&l-{OQ)XF!EWVcqh$+0zQb+I8raKz%hMhaX zla5v$MXGjI*q(M{QV$UvG0i$fv~M~iSguU1yWV%y6{Bu~$OUBHo^4Pib6PZblL-)( zcLEOVJuYU{9?fyQ7dYl&+Tg$KyU@pKG>ZiUyr9-TAQ>$lGOZZlaCq_+tLWeU_#4t z8uffmgS2gFz;X+on80BLbViap=h^wfm&^AMQOc&R`J#ve^g^qaQAOFfk$N4-$V6jWIsY z`Fm5y9Pm7(P0f~Js+>(vKbZSE5W+eytVRELXvE7s%_Q`ZO zbwoJ#I$3tGgW(GHXo&dqrEo-Yo*F!PAZGA0oR^p%Jk4>LtfCPT9V_akhFjGaOI#AV z#PA*0ZlZz?Y7j3-n$}`Zc~~DH?AS<&g~I*lb+o@*yhv;^*OpARCCjl)k~YjVatCvX zaw3+gl+w9$wl$q@YcxJ2>l889egz5)!|St-`>Qzu;e!bi&F}sgesKoqc zF(rn2%8+{a+}K?OO@LMb!e-@!3?k0Y8+C31!_ze~zqPSPyXuLC4Pmjw(z^N25uG)i(5l`xX zrM(QlWDoD*2h$(N(V$eUwairUGhwyBS;U~%>4Vcg;u}>4(64wb-!cGgX9V}a7wfhd zpQh}dYjLCJbQf)7;W)XzMfnrMSn^;7*y!y>10S+HOh}ueM;b-xRPExhgoXM75=R2Q zqw!!PvpEyAZ~q-|7%NV<_=Abt`&<{Q&CFYn;3sR)pm1?eIs#X_N!VPU|C{)D+8u># zYCUez=Aog%mY$(Od?h^axYUh$6gJgT^#CS_KN}kAC^($M46Kjy?NtQYc7gKkvS2tF zYGsXre@1KLvj~J4fpWh%JBIj8*AmEwVUN!GNs7W#420b9;dBzcsas#F{-W6^=(Zfl zIZK^jH7IO21fUz7erKXKF}4(ArI2A!E~9EfwiOkv(`7hZa`{3kmVgt6q4bDxZS>}D z3o+KpM#j1dRJ1rEb#&+U<^`;N!51-JMUGTV9Ygj9j!M^Lsvf>O(2ayz-c=NiKqZC3Ga$4kiA3;*H#xSz2#RY^~dUdwq*2 zR!*Lm)ob{u=~+7y`!nI6(9{{u@L*`AmgteYr4L!pEGyed;w8a)!3mXt$P^n08Y<#Y zG*yOqlR0r1m?|7LO1D#}sKL{20UDiMto}L*s?0bG7&bo@Bke&Rk7&%E|Dt3Uw>kn7 zsJ@StYk1cI)NzXR_HuYcf;JB+_k-HD$Z%Od%co z_bR!Su%@?Rhd+oN{(7ESS0`4pJ1T(ov@PkpBo*sO;PRwPL}8mXS=xifek z4wQA2Tji1qoPlIkgwyb>V^1JRo~Cy@$|F3wV8aluOk%ZSJ6{@pgr&~SsYe~Ge{z^x zv)&A*T_pizn6~bXBla2OLQkAOZT#L2MqQx34-Pbtils2S!OTHnYdcrrQvbS`ty|5# z9^$j8UG{#D8f62!+Yk$e^@lURhL}&Hk_uk4ex$MzC8!nRG=e~NViR=R9%m>I=bkPt zT6M5}_myCrs^s^xB9R(5=Goc;cVVPXL27;KL!c~4!EZo|_fU2Zdn&F9!`lHNuhh5f zLW2g3E*J|M!thL7C%%G0_No&U>b=9uO(M0AN`B^ldN?PRB-}@6N(laf36K2a(+-SQ zD16%qXPfR$o5`3gr*|-!1WZGNX3rMev<#Q=?2Tvh_9T}SJ=TpdJvT0OENjBDDz4OW` zfFTpu*Og{f@-Fz%XBgqQtv=y>AWWb1uF3I_13=!#> zST>hLfJUOy-d1VMqPy#gX=Luy>v&!WTxjQ5sq;6hZZG=zsa;NC*+cSIAO_Ux2!ory zbX8IGDLfHRY>doM3qWe<&dx;d)`%W7ppRhAu;$B4nvhy^IkbsRrDCaa1@SAYaAn)m z(7LjP6vFw@g{aZLyt_U)bDJ6uWMCVw1tM!?oXIV^>+lS#=5FlP&nySHKXfecZJA^4 zDE60?v!tS0~nkcZp;Daminr1fqtc{XK za1Hb<{hoW0cw`5E0U{Mw^E#9jguaZ6DTS-GrPf#`lPc%S7`H+i=D2jRl8F`anNlo~ zEM+T6R0m8X8jbb&&@irBWEgpCcF0{dx6Y5$fbIqr9*(2w1tZtnbd5iB<7-~~J5RAq zC`8ckt;jF}8nBB!=xAJM-~kTJENK|;Ds=G8Ko9i?7y%({x1;_t%;^~Nf*1#mD~se1 z7p7;QC?4szO$$*K!d)SRIA8eqs^<19XhGgAe`i<@QM3UiIO z-n_xvWR`m|^F}XiuJvv)Z}PUA6`nC~_6E&cyger370g?`QFEO)Y2M}?G)>-N^LFoH z^A7L0x!!x+tn{8X@ARHEtGwsTYVUco#`~3dm-kz9gZF#$ZtqX#J>H+qd%eGzwccxH zo%av(KJQ;U!WCg*KCPGzK6GYY$Mr8wGpKKwFNs>^L9AR8K2ousx_@wj_CS@04 zO7;?_#UsqfIAOo+C(O$Igah&r;h;Q9xKB?w7NKL&D?8DVmpIV;RXG2a)4j+-9{LqE-P!U!$%Lt!w9`H|3TYt9Kh mGv>!abEo-<(3W9-Dm>4ap9%F5^K+r(HO~tr$IQ`_@&5wUJQMf; literal 0 HcmV?d00001 diff --git a/noveum_customer_support_bt/preprocess_filter.py b/noveum_customer_support_bt/preprocess_filter.py new file mode 100644 index 0000000..3f38da9 --- /dev/null +++ b/noveum_customer_support_bt/preprocess_filter.py @@ -0,0 +1,120 @@ +#!/usr/bin/env python3 +""" +Filtering script that removes metadata and container spans from dataset. + +This script filters out spans that are not needed for agent evaluation: +- api_selection spans +- reddit_agent_run_1 and reddit_agent_run_2 spans + +Usage: python preprocess_filter.py +Output: _filtered.json +""" + +import json +import sys +import os +from typing import Dict, Any + + +def should_keep_span(span: Dict[str, Any]) -> bool: + """ + Determine if a span should be kept based on its name. + Filters out metadata and container spans. + """ + span_name = span.get('name', '') + + # Remove these span types + excluded_spans = { + 'api_selection', + 'reddit_agent_run_1', + 'reddit_agent_run_2' + } + + return span_name not in excluded_spans + + +def convert_tool_output_to_string(span: Dict[str, Any]) -> Dict[str, Any]: + """ + Convert tool.output.output from JSON array to concatenated string format. + """ + attributes = span.get('attributes', {}) + tool_output = attributes.get('tool.output.output') + + if tool_output and isinstance(tool_output, list): + # Convert list of objects to concatenated string format + output_strings = [] + for item in tool_output: + if isinstance(item, dict): + # Convert dict to string format like "{'url': '...', 'content': '...'}" + item_str = str(item).replace("'", "'") # Ensure single quotes + output_strings.append(item_str) + else: + output_strings.append(str(item)) + + # Join all items with space + attributes['tool.output.output'] = ' '.join(output_strings) + span['attributes'] = attributes + + return span + + +def filter_dataset(input_file: str) -> str: + """ + Filter the dataset by removing unwanted spans. + + Args: + input_file: Path to input JSON file + + Returns: + Path to output file + """ + # Read input file + print(f"Reading {input_file}...") + with open(input_file, 'r') as f: + data = json.load(f) + + print(f"Original dataset: {len(data)} records") + + # Filter spans + print("Filtering spans...") + filtered_data = [span for span in data if should_keep_span(span)] + print(f"After filtering: {len(filtered_data)} records") + + # Convert tool.output.output from JSON array to string format + print("Converting tool output format...") + filtered_data = [convert_tool_output_to_string(span) for span in filtered_data] + + # Generate output filename + base_name = os.path.splitext(input_file)[0] + output_file = f"{base_name}_filtered.json" + + # Write output file + print(f"Writing {output_file}...") + with open(output_file, 'w') as f: + json.dump(filtered_data, f, indent=2) + + print(f"Filtering complete! Output: {output_file}") + return output_file + + +def main(): + if len(sys.argv) != 2: + print("Usage: python preprocess_filter.py ") + sys.exit(1) + + input_file = sys.argv[1] + + if not os.path.exists(input_file): + print(f"Error: File {input_file} not found") + sys.exit(1) + + try: + output_file = filter_dataset(input_file) + print(f"\nSuccess! Created {output_file}") + except Exception as e: + print(f"Error: {e}") + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/noveum_customer_support_bt/preprocess_map.py b/noveum_customer_support_bt/preprocess_map.py new file mode 100644 index 0000000..196b7d0 --- /dev/null +++ b/noveum_customer_support_bt/preprocess_map.py @@ -0,0 +1,219 @@ +#!/usr/bin/env python3 +""" +Mapping script that adds standardized fields to dataset spans. + +This script adds evaluation-ready fields to different span types: +- Agent spans get agent_task and agent_response fields +- Tool/validation spans get tool_response field + +Usage: python preprocess_map.py +Output: _mapped.json +""" + +import json +import sys +import os +from typing import Dict, Any + + +def add_agent_comment_generation_fields(span: Dict[str, Any]) -> Dict[str, Any]: + """ + Add agent_task and agent_response fields for agent.comment_generation spans. + + agent_task -> agent operation is + (agent.operation) +\n + api_title - (api_title) + (events[0].attributes.subreddit) + (events[0].attributes.post_title) + agent_response -> (events[0].attributes.comment) + """ + attributes = span.get('attributes', {}) + events = span.get('events', []) + + # Build agent_task + agent_operation = attributes.get('agent.operation', '') + api_title = attributes.get('api_title', '') + + agent_task_parts = [f"agent operation is {agent_operation}"] + if api_title: + agent_task_parts.append(f"api_title - {api_title}") + + # Add subreddit and post_title from events[0].attributes + if events and len(events) > 0: + event_attrs = events[0].get('attributes', {}) + subreddit = event_attrs.get('subreddit', '') + post_title = event_attrs.get('post_title', '') + + if subreddit: + agent_task_parts.append(f"subreddit is - {subreddit}") + if post_title: + agent_task_parts.append(f"post title is - {post_title}") + + agent_task = '\n'.join(agent_task_parts) + attributes['agent_task'] = agent_task + + # Build agent_response from events[0].attributes.comment + if events and len(events) > 0: + event_attrs = events[0].get('attributes', {}) + comment = event_attrs.get('comment', '') + attributes['agent_response'] = comment + else: + attributes['agent_response'] = '' + + span['attributes'] = attributes + return span + + +def add_agent_query_generation_fields(span: Dict[str, Any]) -> Dict[str, Any]: + """ + Add agent_task and agent_response fields for agent.query_generation spans. + + agent_task -> api title - (api.title) + api description - (api.description) + agent_response -> (concatenate query_generation.queries, it has a list of strings) + """ + attributes = span.get('attributes', {}) + + # Build agent_task + api_title = attributes.get('api.title', '') + api_source = attributes.get('api.source', '') + api_url = attributes.get('api.url', '') + api_description = attributes.get('api.description', 'not available') # Default to 'not available' if not found + + agent_task_parts = [] + if api_title: + agent_task_parts.append(f"api title - {api_title}") + if api_source: + agent_task_parts.append(f"api source - {api_source}") + if api_url: + agent_task_parts.append(f"api url - {api_url}") + agent_task_parts.append(f"api description - {api_description}") + + attributes['agent_task'] = '\n'.join(agent_task_parts) + + # Build agent_response from query_generation.queries + queries = attributes.get('query_generation.queries', []) + if isinstance(queries, list) and queries: + # Concatenate all queries with newlines + agent_response = '\n'.join(queries) + elif isinstance(queries, list) and not queries: + # If list is empty, use stringified version of events + events = span.get('events', []) + agent_response = json.dumps(events) + else: + agent_response = str(queries) if queries else '' + + attributes['agent_response'] = agent_response + + span['attributes'] = attributes + return span + + +def add_email_generation_fields(span: Dict[str, Any]) -> Dict[str, Any]: + """ + Add tool_response field for email_generation_and_sending spans. + + tool_response -> (string value of events key's value, it is a list of jsons) + """ + attributes = span.get('attributes', {}) + events = span.get('events', []) + + # Convert events list to JSON string + if events: + attributes['tool_response'] = json.dumps(events) + else: + attributes['tool_response'] = '[]' + + span['attributes'] = attributes + return span + + +def add_post_validation_fields(span: Dict[str, Any]) -> Dict[str, Any]: + """ + Add tool_response field for post_validation spans. + + tool_response -> (string value of events key's value, it is a list of jsons) + """ + attributes = span.get('attributes', {}) + events = span.get('events', []) + + # Convert events list to JSON string + if events: + attributes['tool_response'] = json.dumps(events) + else: + attributes['tool_response'] = '[]' + + span['attributes'] = attributes + return span + + +def process_span(span: Dict[str, Any]) -> Dict[str, Any]: + """ + Process a single span by adding appropriate mapped fields. + """ + span_name = span.get('name', '') + + # Process different span types + if span_name == 'agent.comment_generation': + span = add_agent_comment_generation_fields(span) + elif span_name == 'agent.query_generation': + span = add_agent_query_generation_fields(span) + elif span_name == 'email_generation_and_sending': + span = add_email_generation_fields(span) + elif span_name == 'post_validation': + span = add_post_validation_fields(span) + # Tool call spans (tool:tavily_search_results_json) don't need changes + + return span + + +def map_dataset(input_file: str) -> str: + """ + Map the dataset by adding standardized fields to spans. + + Args: + input_file: Path to input JSON file + + Returns: + Path to output file + """ + # Read input file + print(f"Reading {input_file}...") + with open(input_file, 'r') as f: + data = json.load(f) + + print(f"Input dataset: {len(data)} records") + + # Map spans + print("Mapping spans...") + mapped_data = [process_span(span) for span in data] + + # Generate output filename + base_name = os.path.splitext(input_file)[0] + output_file = f"{base_name}_mapped.json" + + # Write output file + print(f"Writing {output_file}...") + with open(output_file, 'w') as f: + json.dump(mapped_data, f, indent=2) + + print(f"Mapping complete! Output: {output_file}") + return output_file + + +def main(): + if len(sys.argv) != 2: + print("Usage: python preprocess_map.py ") + sys.exit(1) + + input_file = sys.argv[1] + + if not os.path.exists(input_file): + print(f"Error: File {input_file} not found") + sys.exit(1) + + try: + output_file = map_dataset(input_file) + print(f"\nSuccess! Created {output_file}") + except Exception as e: + print(f"Error: {e}") + sys.exit(1) + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/noveum_customer_support_bt/preprocess_split_data.py b/noveum_customer_support_bt/preprocess_split_data.py new file mode 100644 index 0000000..e4caf20 --- /dev/null +++ b/noveum_customer_support_bt/preprocess_split_data.py @@ -0,0 +1,119 @@ +#!/usr/bin/env python3 +""" +Script to split dataset_filtered_mapped.json by span name into separate files. + +Usage: + python preprocess_split_data.py [input_file] [output_dir] + +If no arguments provided, uses: + - input_file: dataset_filtered_mapped.json + - output_dir: split_datasets +""" + +import json +import os +import sys +from collections import defaultdict +from pathlib import Path + + +def create_name_to_filename_mapping(): + """Create mapping from span names to desired filenames.""" + return { + 'agent.comment_generation': 'agent_comment_gen_dataset.json', + 'agent.query_generation': 'agent_query_gen_dataset.json', + 'email_generation_and_sending': 'email_gen_send_dataset.json', + 'post_validation': 'post_validation_dataset.json', + 'tool:tavily_search_results_json:tavily_search_results_json': 'tavily_search_results_dataset.json', + } + + +def sanitize_filename(name): + """Convert span name to a safe filename.""" + # Replace problematic characters with underscores + safe_name = name.replace(':', '_').replace('/', '_').replace('\\', '_').replace('*', '_').replace('?', '_').replace('"', '_').replace('<', '_').replace('>', '_').replace('|', '_') + return f"{safe_name}_dataset.json" + + +def split_dataset_by_name(input_file, output_dir): + """ + Split dataset by span name into separate files. + + Args: + input_file (str): Path to input JSON file + output_dir (str): Path to output directory + """ + # Create output directory if it doesn't exist + Path(output_dir).mkdir(parents=True, exist_ok=True) + + # Load the dataset + print(f"Loading dataset from {input_file}...") + with open(input_file, 'r') as f: + data = json.load(f) + + print(f"Loaded {len(data)} objects") + + # Group data by name + grouped_data = defaultdict(list) + for obj in data: + name = obj.get('name', 'unknown') + grouped_data[name].append(obj) + + print(f"Found {len(grouped_data)} unique span names") + + # Create name to filename mapping + name_mapping = create_name_to_filename_mapping() + + # Write separate files for each name + for name, objects in grouped_data.items(): + # Determine filename + if name in name_mapping: + filename = name_mapping[name] + print(f"Using hardcoded mapping: {name} -> {filename}") + else: + filename = sanitize_filename(name) + print(f"Using sanitized name: {name} -> {filename}") + + output_path = os.path.join(output_dir, filename) + + # Write the file + with open(output_path, 'w', encoding='utf-8') as f: + json.dump(objects, f, indent=2, ensure_ascii=False) + + print(f" Wrote {len(objects)} objects to {output_path}") + + print(f"\nSplit complete! Created {len(grouped_data)} files in {output_dir}") + + +def main(): + """Main function to handle command line arguments.""" + # Check for help + if len(sys.argv) > 1 and sys.argv[1] in ['-h', '--help', 'help']: + print(__doc__) + sys.exit(0) + + # Default values + input_file = "dataset_filtered_mapped.json" + output_dir = "split_datasets" + + # Parse command line arguments + if len(sys.argv) > 1: + input_file = sys.argv[1] + if len(sys.argv) > 2: + output_dir = sys.argv[2] + + # Check if input file exists + if not os.path.exists(input_file): + print(f"Error: Input file '{input_file}' not found!") + sys.exit(1) + + print(f"Input file: {input_file}") + print(f"Output directory: {output_dir}") + print() + + # Split the dataset + split_dataset_by_name(input_file, output_dir) + + +if __name__ == "__main__": + main() diff --git a/noveum_customer_support_bt/processed_datasets/agent.llm-rag_dataset_processed_dataset.csv b/noveum_customer_support_bt/processed_datasets/agent.llm-rag_dataset_processed_dataset.csv new file mode 100644 index 0000000..92b1c61 --- /dev/null +++ b/noveum_customer_support_bt/processed_datasets/agent.llm-rag_dataset_processed_dataset.csv @@ -0,0 +1,13 @@ +user_id,task_id,turn_id,ground_truth,expected_tool_call,agent_name,agent_role,agent_task,system_prompt,agent_response,trace,tools_available,tool_calls,parameters_passed,tool_call_results,retrieval_query,retrieved_context,exit_status,agent_exit,metadata +,f1f37bd7-0851-4659-b493-b80d3800d920,ea4d61c5-b8dd-4677-b21b-2b17170e2163,,,agent,agent,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, +,52aacb67-c361-4445-9b72-c157f79f47d6,7505f97a-cbc0-42fe-8a72-95387d723daa,,,agent,agent,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, +,2218f641-604c-491a-9710-b51a9941b982,8a6e18a1-babc-4166-926f-188f344d514e,,,agent,agent,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, +,255fd49c-84b4-4b18-887e-6308a412d535,5bec4165-6596-43ac-930b-acfcde1cc02a,,,agent,agent,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, +,dc511122-c0b6-415c-9a49-c7b45132dd87,f9f79a07-4eda-41b8-9634-d4460114caad,,,agent,agent,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, +,04bebf38-a343-4563-80db-0154bef8d927,edc52a5d-8450-4d0f-ad14-958ec4d147f4,,,agent,agent,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, +,5e043630-6493-42b5-beb8-79faa19bfa37,6c362c73-e063-4813-8180-d26b6a52b798,,,agent,agent,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, +,7da9814d-a2e8-4c4e-b750-68b26bd5fd22,1393d065-a9a4-4475-808f-2d00b3c68939,,,agent,agent,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, +,16143f74-2831-4753-b33d-ce4b645093c5,f419377a-fc24-4abf-8a1c-03857824e374,,,agent,agent,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, +,fc64e6cc-6739-4256-ac4a-7b80c3028233,0e58e31f-a8ab-4aee-8fc2-abcae1e21d9d,,,agent,agent,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, +,b7945c49-f584-4c70-972d-536a805d8a31,48155aad-6204-4a52-8624-8fa5d6fe407f,,,agent,agent,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, +,f5c40ecf-36c0-45ba-9cc9-dc0329b0324b,57ded2eb-0f3f-474e-8704-5ad9974ab4bb,,,agent,agent,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, diff --git a/noveum_customer_support_bt/processed_datasets/agent.llm-rag_dataset_processed_dataset.json b/noveum_customer_support_bt/processed_datasets/agent.llm-rag_dataset_processed_dataset.json new file mode 100644 index 0000000..31d416f --- /dev/null +++ b/noveum_customer_support_bt/processed_datasets/agent.llm-rag_dataset_processed_dataset.json @@ -0,0 +1,266 @@ +[ + { + "user_id": null, + "task_id": "f1f37bd7-0851-4659-b493-b80d3800d920", + "turn_id": "ea4d61c5-b8dd-4677-b21b-2b17170e2163", + "ground_truth": null, + "expected_tool_call": null, + "agent_name": "agent", + "agent_role": "agent", + "agent_task": null, + "system_prompt": "You are a helpful customer support agent", + "agent_response": null, + "trace": null, + "tools_available": [], + "tool_calls": [], + "parameters_passed": {}, + "tool_call_results": [], + "retrieval_query": null, + "retrieved_context": null, + "exit_status": "ok", + "agent_exit": false, + "metadata": null + }, + { + "user_id": null, + "task_id": "52aacb67-c361-4445-9b72-c157f79f47d6", + "turn_id": "7505f97a-cbc0-42fe-8a72-95387d723daa", + "ground_truth": null, + "expected_tool_call": null, + "agent_name": "agent", + "agent_role": "agent", + "agent_task": null, + "system_prompt": "You are a helpful customer support agent", + "agent_response": null, + "trace": null, + "tools_available": [], + "tool_calls": [], + "parameters_passed": {}, + "tool_call_results": [], + "retrieval_query": null, + "retrieved_context": null, + "exit_status": "ok", + "agent_exit": false, + "metadata": null + }, + { + "user_id": null, + "task_id": "2218f641-604c-491a-9710-b51a9941b982", + "turn_id": "8a6e18a1-babc-4166-926f-188f344d514e", + "ground_truth": null, + "expected_tool_call": null, + "agent_name": "agent", + "agent_role": "agent", + "agent_task": null, + "system_prompt": "You are a helpful customer support agent", + "agent_response": null, + "trace": null, + "tools_available": [], + "tool_calls": [], + "parameters_passed": {}, + "tool_call_results": [], + "retrieval_query": null, + "retrieved_context": null, + "exit_status": "ok", + "agent_exit": false, + "metadata": null + }, + { + "user_id": null, + "task_id": "255fd49c-84b4-4b18-887e-6308a412d535", + "turn_id": "5bec4165-6596-43ac-930b-acfcde1cc02a", + "ground_truth": null, + "expected_tool_call": null, + "agent_name": "agent", + "agent_role": "agent", + "agent_task": null, + "system_prompt": "You are a helpful customer support agent", + "agent_response": null, + "trace": null, + "tools_available": [], + "tool_calls": [], + "parameters_passed": {}, + "tool_call_results": [], + "retrieval_query": null, + "retrieved_context": null, + "exit_status": "ok", + "agent_exit": false, + "metadata": null + }, + { + "user_id": null, + "task_id": "dc511122-c0b6-415c-9a49-c7b45132dd87", + "turn_id": "f9f79a07-4eda-41b8-9634-d4460114caad", + "ground_truth": null, + "expected_tool_call": null, + "agent_name": "agent", + "agent_role": "agent", + "agent_task": null, + "system_prompt": "You are a helpful customer support agent", + "agent_response": null, + "trace": null, + "tools_available": [], + "tool_calls": [], + "parameters_passed": {}, + "tool_call_results": [], + "retrieval_query": null, + "retrieved_context": null, + "exit_status": "ok", + "agent_exit": false, + "metadata": null + }, + { + "user_id": null, + "task_id": "04bebf38-a343-4563-80db-0154bef8d927", + "turn_id": "edc52a5d-8450-4d0f-ad14-958ec4d147f4", + "ground_truth": null, + "expected_tool_call": null, + "agent_name": "agent", + "agent_role": "agent", + "agent_task": null, + "system_prompt": "You are a helpful customer support agent", + "agent_response": null, + "trace": null, + "tools_available": [], + "tool_calls": [], + "parameters_passed": {}, + "tool_call_results": [], + "retrieval_query": null, + "retrieved_context": null, + "exit_status": "ok", + "agent_exit": false, + "metadata": null + }, + { + "user_id": null, + "task_id": "5e043630-6493-42b5-beb8-79faa19bfa37", + "turn_id": "6c362c73-e063-4813-8180-d26b6a52b798", + "ground_truth": null, + "expected_tool_call": null, + "agent_name": "agent", + "agent_role": "agent", + "agent_task": null, + "system_prompt": "You are a helpful customer support agent", + "agent_response": null, + "trace": null, + "tools_available": [], + "tool_calls": [], + "parameters_passed": {}, + "tool_call_results": [], + "retrieval_query": null, + "retrieved_context": null, + "exit_status": "ok", + "agent_exit": false, + "metadata": null + }, + { + "user_id": null, + "task_id": "7da9814d-a2e8-4c4e-b750-68b26bd5fd22", + "turn_id": "1393d065-a9a4-4475-808f-2d00b3c68939", + "ground_truth": null, + "expected_tool_call": null, + "agent_name": "agent", + "agent_role": "agent", + "agent_task": null, + "system_prompt": "You are a helpful customer support agent", + "agent_response": null, + "trace": null, + "tools_available": [], + "tool_calls": [], + "parameters_passed": {}, + "tool_call_results": [], + "retrieval_query": null, + "retrieved_context": null, + "exit_status": "ok", + "agent_exit": false, + "metadata": null + }, + { + "user_id": null, + "task_id": "16143f74-2831-4753-b33d-ce4b645093c5", + "turn_id": "f419377a-fc24-4abf-8a1c-03857824e374", + "ground_truth": null, + "expected_tool_call": null, + "agent_name": "agent", + "agent_role": "agent", + "agent_task": null, + "system_prompt": "You are a helpful customer support agent", + "agent_response": null, + "trace": null, + "tools_available": [], + "tool_calls": [], + "parameters_passed": {}, + "tool_call_results": [], + "retrieval_query": null, + "retrieved_context": null, + "exit_status": "ok", + "agent_exit": false, + "metadata": null + }, + { + "user_id": null, + "task_id": "fc64e6cc-6739-4256-ac4a-7b80c3028233", + "turn_id": "0e58e31f-a8ab-4aee-8fc2-abcae1e21d9d", + "ground_truth": null, + "expected_tool_call": null, + "agent_name": "agent", + "agent_role": "agent", + "agent_task": null, + "system_prompt": "You are a helpful customer support agent", + "agent_response": null, + "trace": null, + "tools_available": [], + "tool_calls": [], + "parameters_passed": {}, + "tool_call_results": [], + "retrieval_query": null, + "retrieved_context": null, + "exit_status": "ok", + "agent_exit": false, + "metadata": null + }, + { + "user_id": null, + "task_id": "b7945c49-f584-4c70-972d-536a805d8a31", + "turn_id": "48155aad-6204-4a52-8624-8fa5d6fe407f", + "ground_truth": null, + "expected_tool_call": null, + "agent_name": "agent", + "agent_role": "agent", + "agent_task": null, + "system_prompt": "You are a helpful customer support agent", + "agent_response": null, + "trace": null, + "tools_available": [], + "tool_calls": [], + "parameters_passed": {}, + "tool_call_results": [], + "retrieval_query": null, + "retrieved_context": null, + "exit_status": "ok", + "agent_exit": false, + "metadata": null + }, + { + "user_id": null, + "task_id": "f5c40ecf-36c0-45ba-9cc9-dc0329b0324b", + "turn_id": "57ded2eb-0f3f-474e-8704-5ad9974ab4bb", + "ground_truth": null, + "expected_tool_call": null, + "agent_name": "agent", + "agent_role": "agent", + "agent_task": null, + "system_prompt": "You are a helpful customer support agent", + "agent_response": null, + "trace": null, + "tools_available": [], + "tool_calls": [], + "parameters_passed": {}, + "tool_call_results": [], + "retrieval_query": null, + "retrieved_context": null, + "exit_status": "ok", + "agent_exit": false, + "metadata": null + } +] \ No newline at end of file diff --git a/noveum_customer_support_bt/processed_datasets/agent.llm_model_execution_dataset_processed_dataset.csv b/noveum_customer_support_bt/processed_datasets/agent.llm_model_execution_dataset_processed_dataset.csv new file mode 100644 index 0000000..eae155f --- /dev/null +++ b/noveum_customer_support_bt/processed_datasets/agent.llm_model_execution_dataset_processed_dataset.csv @@ -0,0 +1,21 @@ +user_id,task_id,turn_id,ground_truth,expected_tool_call,agent_name,agent_role,agent_task,system_prompt,agent_response,trace,tools_available,tool_calls,parameters_passed,tool_call_results,retrieval_query,retrieved_context,exit_status,agent_exit,metadata +,eda4fe22-9a2b-4b73-856b-f4f3309bf719,ce8556f4-bbeb-46fc-8bfd-dfaedb61cb70,,,agent,agent,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, +,0ffffba1-8a37-443c-8866-d53ffbfa7718,cc3073ec-c970-4821-8335-8d0fc8f37930,,,agent,agent,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, +,f1f37bd7-0851-4659-b493-b80d3800d920,c24fae6d-6314-4342-8214-7e9905ae83db,,,agent,agent,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, +,43cdf081-4f01-49cd-b566-dbd1619e6cd2,93879d8f-77c7-4fd4-881b-be500e2dcbde,,,agent,agent,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, +,9a1983f4-09da-4b53-80e6-38de6878e0e7,f92643c0-44a3-4fae-9d29-45321342d12f,,,agent,agent,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, +,5d2517e1-220a-429d-9d59-f701bda25eed,44f379aa-10fe-4a51-847c-5288444d7843,,,agent,agent,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, +,52aacb67-c361-4445-9b72-c157f79f47d6,e8e79f2a-2ed0-4981-943c-adcf6f7d2b15,,,agent,agent,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, +,a81ca3a8-80aa-4c39-876e-8d40ea7a0aef,e6d0eb41-5f64-44c5-8a42-a80c336de03e,,,agent,agent,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, +,230aad27-f3dd-4968-a45a-3c2f07ac28ed,c545cb62-5a51-4f2e-b020-b3f1d1dfe217,,,agent,agent,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, +,83c7dcce-3d89-4da1-8b3f-d419885d4cbc,c2f21e4d-a5e6-4134-aeaf-96e2f566cf96,,,agent,agent,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, +,2218f641-604c-491a-9710-b51a9941b982,39d99d2b-518f-439c-8758-7a01b970fd9e,,,agent,agent,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, +,255fd49c-84b4-4b18-887e-6308a412d535,9e5ade74-3815-4dad-8abe-d4281d7e62da,,,agent,agent,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, +,dc511122-c0b6-415c-9a49-c7b45132dd87,4f38a85b-0721-47a5-8a51-d0808e460f96,,,agent,agent,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, +,04bebf38-a343-4563-80db-0154bef8d927,f4c03a8d-b90d-4c1a-a259-826d09e3e57a,,,agent,agent,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, +,5e043630-6493-42b5-beb8-79faa19bfa37,48fb40ba-a39f-4e13-9d94-18794c7e5e2b,,,agent,agent,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, +,7da9814d-a2e8-4c4e-b750-68b26bd5fd22,d4d544a0-f052-4889-82eb-4346f2e447a7,,,agent,agent,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, +,16143f74-2831-4753-b33d-ce4b645093c5,304f9e3d-9063-4661-9777-618efc49f5c1,,,agent,agent,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, +,fc64e6cc-6739-4256-ac4a-7b80c3028233,e0b4ccfb-18e7-4d71-b2dd-5b807eb27513,,,agent,agent,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, +,b7945c49-f584-4c70-972d-536a805d8a31,f1425131-f8a2-43f7-aa17-d009f745ed06,,,agent,agent,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, +,f5c40ecf-36c0-45ba-9cc9-dc0329b0324b,f41e836c-1a82-46e0-9d51-011713874e94,,,agent,agent,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, diff --git a/noveum_customer_support_bt/processed_datasets/agent.llm_model_execution_dataset_processed_dataset.json b/noveum_customer_support_bt/processed_datasets/agent.llm_model_execution_dataset_processed_dataset.json new file mode 100644 index 0000000..6fa4b1c --- /dev/null +++ b/noveum_customer_support_bt/processed_datasets/agent.llm_model_execution_dataset_processed_dataset.json @@ -0,0 +1,442 @@ +[ + { + "user_id": null, + "task_id": "eda4fe22-9a2b-4b73-856b-f4f3309bf719", + "turn_id": "ce8556f4-bbeb-46fc-8bfd-dfaedb61cb70", + "ground_truth": null, + "expected_tool_call": null, + "agent_name": "agent", + "agent_role": "agent", + "agent_task": null, + "system_prompt": "You are a helpful customer support agent", + "agent_response": null, + "trace": null, + "tools_available": [], + "tool_calls": [], + "parameters_passed": {}, + "tool_call_results": [], + "retrieval_query": null, + "retrieved_context": null, + "exit_status": "ok", + "agent_exit": false, + "metadata": null + }, + { + "user_id": null, + "task_id": "0ffffba1-8a37-443c-8866-d53ffbfa7718", + "turn_id": "cc3073ec-c970-4821-8335-8d0fc8f37930", + "ground_truth": null, + "expected_tool_call": null, + "agent_name": "agent", + "agent_role": "agent", + "agent_task": null, + "system_prompt": "You are a helpful customer support agent", + "agent_response": null, + "trace": null, + "tools_available": [], + "tool_calls": [], + "parameters_passed": {}, + "tool_call_results": [], + "retrieval_query": null, + "retrieved_context": null, + "exit_status": "ok", + "agent_exit": false, + "metadata": null + }, + { + "user_id": null, + "task_id": "f1f37bd7-0851-4659-b493-b80d3800d920", + "turn_id": "c24fae6d-6314-4342-8214-7e9905ae83db", + "ground_truth": null, + "expected_tool_call": null, + "agent_name": "agent", + "agent_role": "agent", + "agent_task": null, + "system_prompt": "You are a helpful customer support agent", + "agent_response": null, + "trace": null, + "tools_available": [], + "tool_calls": [], + "parameters_passed": {}, + "tool_call_results": [], + "retrieval_query": null, + "retrieved_context": null, + "exit_status": "ok", + "agent_exit": false, + "metadata": null + }, + { + "user_id": null, + "task_id": "43cdf081-4f01-49cd-b566-dbd1619e6cd2", + "turn_id": "93879d8f-77c7-4fd4-881b-be500e2dcbde", + "ground_truth": null, + "expected_tool_call": null, + "agent_name": "agent", + "agent_role": "agent", + "agent_task": null, + "system_prompt": "You are a helpful customer support agent", + "agent_response": null, + "trace": null, + "tools_available": [], + "tool_calls": [], + "parameters_passed": {}, + "tool_call_results": [], + "retrieval_query": null, + "retrieved_context": null, + "exit_status": "ok", + "agent_exit": false, + "metadata": null + }, + { + "user_id": null, + "task_id": "9a1983f4-09da-4b53-80e6-38de6878e0e7", + "turn_id": "f92643c0-44a3-4fae-9d29-45321342d12f", + "ground_truth": null, + "expected_tool_call": null, + "agent_name": "agent", + "agent_role": "agent", + "agent_task": null, + "system_prompt": "You are a helpful customer support agent", + "agent_response": null, + "trace": null, + "tools_available": [], + "tool_calls": [], + "parameters_passed": {}, + "tool_call_results": [], + "retrieval_query": null, + "retrieved_context": null, + "exit_status": "ok", + "agent_exit": false, + "metadata": null + }, + { + "user_id": null, + "task_id": "5d2517e1-220a-429d-9d59-f701bda25eed", + "turn_id": "44f379aa-10fe-4a51-847c-5288444d7843", + "ground_truth": null, + "expected_tool_call": null, + "agent_name": "agent", + "agent_role": "agent", + "agent_task": null, + "system_prompt": "You are a helpful customer support agent", + "agent_response": null, + "trace": null, + "tools_available": [], + "tool_calls": [], + "parameters_passed": {}, + "tool_call_results": [], + "retrieval_query": null, + "retrieved_context": null, + "exit_status": "ok", + "agent_exit": false, + "metadata": null + }, + { + "user_id": null, + "task_id": "52aacb67-c361-4445-9b72-c157f79f47d6", + "turn_id": "e8e79f2a-2ed0-4981-943c-adcf6f7d2b15", + "ground_truth": null, + "expected_tool_call": null, + "agent_name": "agent", + "agent_role": "agent", + "agent_task": null, + "system_prompt": "You are a helpful customer support agent", + "agent_response": null, + "trace": null, + "tools_available": [], + "tool_calls": [], + "parameters_passed": {}, + "tool_call_results": [], + "retrieval_query": null, + "retrieved_context": null, + "exit_status": "ok", + "agent_exit": false, + "metadata": null + }, + { + "user_id": null, + "task_id": "a81ca3a8-80aa-4c39-876e-8d40ea7a0aef", + "turn_id": "e6d0eb41-5f64-44c5-8a42-a80c336de03e", + "ground_truth": null, + "expected_tool_call": null, + "agent_name": "agent", + "agent_role": "agent", + "agent_task": null, + "system_prompt": "You are a helpful customer support agent", + "agent_response": null, + "trace": null, + "tools_available": [], + "tool_calls": [], + "parameters_passed": {}, + "tool_call_results": [], + "retrieval_query": null, + "retrieved_context": null, + "exit_status": "ok", + "agent_exit": false, + "metadata": null + }, + { + "user_id": null, + "task_id": "230aad27-f3dd-4968-a45a-3c2f07ac28ed", + "turn_id": "c545cb62-5a51-4f2e-b020-b3f1d1dfe217", + "ground_truth": null, + "expected_tool_call": null, + "agent_name": "agent", + "agent_role": "agent", + "agent_task": null, + "system_prompt": "You are a helpful customer support agent", + "agent_response": null, + "trace": null, + "tools_available": [], + "tool_calls": [], + "parameters_passed": {}, + "tool_call_results": [], + "retrieval_query": null, + "retrieved_context": null, + "exit_status": "ok", + "agent_exit": false, + "metadata": null + }, + { + "user_id": null, + "task_id": "83c7dcce-3d89-4da1-8b3f-d419885d4cbc", + "turn_id": "c2f21e4d-a5e6-4134-aeaf-96e2f566cf96", + "ground_truth": null, + "expected_tool_call": null, + "agent_name": "agent", + "agent_role": "agent", + "agent_task": null, + "system_prompt": "You are a helpful customer support agent", + "agent_response": null, + "trace": null, + "tools_available": [], + "tool_calls": [], + "parameters_passed": {}, + "tool_call_results": [], + "retrieval_query": null, + "retrieved_context": null, + "exit_status": "ok", + "agent_exit": false, + "metadata": null + }, + { + "user_id": null, + "task_id": "2218f641-604c-491a-9710-b51a9941b982", + "turn_id": "39d99d2b-518f-439c-8758-7a01b970fd9e", + "ground_truth": null, + "expected_tool_call": null, + "agent_name": "agent", + "agent_role": "agent", + "agent_task": null, + "system_prompt": "You are a helpful customer support agent", + "agent_response": null, + "trace": null, + "tools_available": [], + "tool_calls": [], + "parameters_passed": {}, + "tool_call_results": [], + "retrieval_query": null, + "retrieved_context": null, + "exit_status": "ok", + "agent_exit": false, + "metadata": null + }, + { + "user_id": null, + "task_id": "255fd49c-84b4-4b18-887e-6308a412d535", + "turn_id": "9e5ade74-3815-4dad-8abe-d4281d7e62da", + "ground_truth": null, + "expected_tool_call": null, + "agent_name": "agent", + "agent_role": "agent", + "agent_task": null, + "system_prompt": "You are a helpful customer support agent", + "agent_response": null, + "trace": null, + "tools_available": [], + "tool_calls": [], + "parameters_passed": {}, + "tool_call_results": [], + "retrieval_query": null, + "retrieved_context": null, + "exit_status": "ok", + "agent_exit": false, + "metadata": null + }, + { + "user_id": null, + "task_id": "dc511122-c0b6-415c-9a49-c7b45132dd87", + "turn_id": "4f38a85b-0721-47a5-8a51-d0808e460f96", + "ground_truth": null, + "expected_tool_call": null, + "agent_name": "agent", + "agent_role": "agent", + "agent_task": null, + "system_prompt": "You are a helpful customer support agent", + "agent_response": null, + "trace": null, + "tools_available": [], + "tool_calls": [], + "parameters_passed": {}, + "tool_call_results": [], + "retrieval_query": null, + "retrieved_context": null, + "exit_status": "ok", + "agent_exit": false, + "metadata": null + }, + { + "user_id": null, + "task_id": "04bebf38-a343-4563-80db-0154bef8d927", + "turn_id": "f4c03a8d-b90d-4c1a-a259-826d09e3e57a", + "ground_truth": null, + "expected_tool_call": null, + "agent_name": "agent", + "agent_role": "agent", + "agent_task": null, + "system_prompt": "You are a helpful customer support agent", + "agent_response": null, + "trace": null, + "tools_available": [], + "tool_calls": [], + "parameters_passed": {}, + "tool_call_results": [], + "retrieval_query": null, + "retrieved_context": null, + "exit_status": "ok", + "agent_exit": false, + "metadata": null + }, + { + "user_id": null, + "task_id": "5e043630-6493-42b5-beb8-79faa19bfa37", + "turn_id": "48fb40ba-a39f-4e13-9d94-18794c7e5e2b", + "ground_truth": null, + "expected_tool_call": null, + "agent_name": "agent", + "agent_role": "agent", + "agent_task": null, + "system_prompt": "You are a helpful customer support agent", + "agent_response": null, + "trace": null, + "tools_available": [], + "tool_calls": [], + "parameters_passed": {}, + "tool_call_results": [], + "retrieval_query": null, + "retrieved_context": null, + "exit_status": "ok", + "agent_exit": false, + "metadata": null + }, + { + "user_id": null, + "task_id": "7da9814d-a2e8-4c4e-b750-68b26bd5fd22", + "turn_id": "d4d544a0-f052-4889-82eb-4346f2e447a7", + "ground_truth": null, + "expected_tool_call": null, + "agent_name": "agent", + "agent_role": "agent", + "agent_task": null, + "system_prompt": "You are a helpful customer support agent", + "agent_response": null, + "trace": null, + "tools_available": [], + "tool_calls": [], + "parameters_passed": {}, + "tool_call_results": [], + "retrieval_query": null, + "retrieved_context": null, + "exit_status": "ok", + "agent_exit": false, + "metadata": null + }, + { + "user_id": null, + "task_id": "16143f74-2831-4753-b33d-ce4b645093c5", + "turn_id": "304f9e3d-9063-4661-9777-618efc49f5c1", + "ground_truth": null, + "expected_tool_call": null, + "agent_name": "agent", + "agent_role": "agent", + "agent_task": null, + "system_prompt": "You are a helpful customer support agent", + "agent_response": null, + "trace": null, + "tools_available": [], + "tool_calls": [], + "parameters_passed": {}, + "tool_call_results": [], + "retrieval_query": null, + "retrieved_context": null, + "exit_status": "ok", + "agent_exit": false, + "metadata": null + }, + { + "user_id": null, + "task_id": "fc64e6cc-6739-4256-ac4a-7b80c3028233", + "turn_id": "e0b4ccfb-18e7-4d71-b2dd-5b807eb27513", + "ground_truth": null, + "expected_tool_call": null, + "agent_name": "agent", + "agent_role": "agent", + "agent_task": null, + "system_prompt": "You are a helpful customer support agent", + "agent_response": null, + "trace": null, + "tools_available": [], + "tool_calls": [], + "parameters_passed": {}, + "tool_call_results": [], + "retrieval_query": null, + "retrieved_context": null, + "exit_status": "ok", + "agent_exit": false, + "metadata": null + }, + { + "user_id": null, + "task_id": "b7945c49-f584-4c70-972d-536a805d8a31", + "turn_id": "f1425131-f8a2-43f7-aa17-d009f745ed06", + "ground_truth": null, + "expected_tool_call": null, + "agent_name": "agent", + "agent_role": "agent", + "agent_task": null, + "system_prompt": "You are a helpful customer support agent", + "agent_response": null, + "trace": null, + "tools_available": [], + "tool_calls": [], + "parameters_passed": {}, + "tool_call_results": [], + "retrieval_query": null, + "retrieved_context": null, + "exit_status": "ok", + "agent_exit": false, + "metadata": null + }, + { + "user_id": null, + "task_id": "f5c40ecf-36c0-45ba-9cc9-dc0329b0324b", + "turn_id": "f41e836c-1a82-46e0-9d51-011713874e94", + "ground_truth": null, + "expected_tool_call": null, + "agent_name": "agent", + "agent_role": "agent", + "agent_task": null, + "system_prompt": "You are a helpful customer support agent", + "agent_response": null, + "trace": null, + "tools_available": [], + "tool_calls": [], + "parameters_passed": {}, + "tool_call_results": [], + "retrieval_query": null, + "retrieved_context": null, + "exit_status": "ok", + "agent_exit": false, + "metadata": null + } +] \ No newline at end of file diff --git a/noveum_customer_support_bt/processed_datasets/agent.query_routing_dataset_processed_dataset.csv b/noveum_customer_support_bt/processed_datasets/agent.query_routing_dataset_processed_dataset.csv new file mode 100644 index 0000000..8578b36 --- /dev/null +++ b/noveum_customer_support_bt/processed_datasets/agent.query_routing_dataset_processed_dataset.csv @@ -0,0 +1,21 @@ +user_id,task_id,turn_id,ground_truth,expected_tool_call,agent_name,agent_role,agent_task,system_prompt,agent_response,trace,tools_available,tool_calls,parameters_passed,tool_call_results,retrieval_query,retrieved_context,exit_status,agent_exit,metadata +,eda4fe22-9a2b-4b73-856b-f4f3309bf719,0b19befc-5dc0-4b00-9a8b-5f2f2c914f17,,,agent,agent,What are the latest breakthroughs in artificial intelligence?,You are a helpful customer support agent,,,"[{""name"": ""query_routing"", ""description"": ""Routes queries to appropriate handlers"", ""args_schema"": {}, ""return_schema"": {""type"": ""routing_decision""}}]",[],{},[],,,ok,False, +,0ffffba1-8a37-443c-8866-d53ffbfa7718,30644d2a-5ab1-4a73-b4ff-cf9fa478e36b,,,agent,agent,What's happening in the software development world today?,You are a helpful customer support agent,,,"[{""name"": ""query_routing"", ""description"": ""Routes queries to appropriate handlers"", ""args_schema"": {}, ""return_schema"": {""type"": ""routing_decision""}}]",[],{},[],,,ok,False, +,f1f37bd7-0851-4659-b493-b80d3800d920,89eac2de-b7da-4db3-bf7c-8e87c429481e,,,agent,agent,What are the newest features in cloud computing?,You are a helpful customer support agent,,,"[{""name"": ""query_routing"", ""description"": ""Routes queries to appropriate handlers"", ""args_schema"": {}, ""return_schema"": {""type"": ""routing_decision""}}]",[],{},[],,,ok,False, +,43cdf081-4f01-49cd-b566-dbd1619e6cd2,962afebb-26d1-45ff-acf1-218163cd6bb8,,,agent,agent,What's the current status of cryptocurrency markets?,You are a helpful customer support agent,,,"[{""name"": ""query_routing"", ""description"": ""Routes queries to appropriate handlers"", ""args_schema"": {}, ""return_schema"": {""type"": ""routing_decision""}}]",[],{},[],,,ok,False, +,9a1983f4-09da-4b53-80e6-38de6878e0e7,246cf822-4562-4d3a-8a5a-e38e0a0b02a0,,,agent,agent,What are the latest updates in Python programming?,You are a helpful customer support agent,,,"[{""name"": ""query_routing"", ""description"": ""Routes queries to appropriate handlers"", ""args_schema"": {}, ""return_schema"": {""type"": ""routing_decision""}}]",[],{},[],,,ok,False, +,5d2517e1-220a-429d-9d59-f701bda25eed,20544f91-8828-4a0e-b253-ed1f8b95b205,,,agent,agent,What happened in tech news this week?,You are a helpful customer support agent,,,"[{""name"": ""query_routing"", ""description"": ""Routes queries to appropriate handlers"", ""args_schema"": {}, ""return_schema"": {""type"": ""routing_decision""}}]",[],{},[],,,ok,False, +,52aacb67-c361-4445-9b72-c157f79f47d6,5359c861-c51b-446d-9ad2-b3d8c21ea70f,,,agent,agent,What are the current trends in observability tools?,You are a helpful customer support agent,,,"[{""name"": ""query_routing"", ""description"": ""Routes queries to appropriate handlers"", ""args_schema"": {}, ""return_schema"": {""type"": ""routing_decision""}}]",[],{},[],,,ok,False, +,a81ca3a8-80aa-4c39-876e-8d40ea7a0aef,acf6a99a-39e0-4f87-a07e-3a61213ab032,,,agent,agent,Tell me about recent developments in machine learning,You are a helpful customer support agent,,,"[{""name"": ""query_routing"", ""description"": ""Routes queries to appropriate handlers"", ""args_schema"": {}, ""return_schema"": {""type"": ""routing_decision""}}]",[],{},[],,,ok,False, +,230aad27-f3dd-4968-a45a-3c2f07ac28ed,0b47ea07-1eda-4b6a-a183-ea9ef047e431,,,agent,agent,What's the weather like today?,You are a helpful customer support agent,,,"[{""name"": ""query_routing"", ""description"": ""Routes queries to appropriate handlers"", ""args_schema"": {}, ""return_schema"": {""type"": ""routing_decision""}}]",[],{},[],,,ok,False, +,83c7dcce-3d89-4da1-8b3f-d419885d4cbc,3866ddb3-b17e-47a1-8b3f-bca1e32a2ae4,,,agent,agent,What are the latest AI news today?,You are a helpful customer support agent,,,"[{""name"": ""query_routing"", ""description"": ""Routes queries to appropriate handlers"", ""args_schema"": {}, ""return_schema"": {""type"": ""routing_decision""}}]",[],{},[],,,ok,False, +,2218f641-604c-491a-9710-b51a9941b982,3ce2e35e-4518-43b4-8a76-4413da6832ea,,,agent,agent,What are the benefits of using Noveum Trace?,You are a helpful customer support agent,,,"[{""name"": ""query_routing"", ""description"": ""Routes queries to appropriate handlers"", ""args_schema"": {}, ""return_schema"": {""type"": ""routing_decision""}}]",[],{},[],,,ok,False, +,255fd49c-84b4-4b18-887e-6308a412d535,c0c39e8b-9a42-4c18-8737-9224cd8a45c0,,,agent,agent,How do I configure Noveum for my system?,You are a helpful customer support agent,,,"[{""name"": ""query_routing"", ""description"": ""Routes queries to appropriate handlers"", ""args_schema"": {}, ""return_schema"": {""type"": ""routing_decision""}}]",[],{},[],,,ok,False, +,dc511122-c0b6-415c-9a49-c7b45132dd87,80987c51-ea82-4a41-97c7-ed591082b9bf,,,agent,agent,What monitoring capabilities does Noveum provide?,You are a helpful customer support agent,,,"[{""name"": ""query_routing"", ""description"": ""Routes queries to appropriate handlers"", ""args_schema"": {}, ""return_schema"": {""type"": ""routing_decision""}}]",[],{},[],,,ok,False, +,04bebf38-a343-4563-80db-0154bef8d927,2f89c1e0-9b62-4651-914c-b71cf561c33c,,,agent,agent,How does Noveum handle agent tracing?,You are a helpful customer support agent,,,"[{""name"": ""query_routing"", ""description"": ""Routes queries to appropriate handlers"", ""args_schema"": {}, ""return_schema"": {""type"": ""routing_decision""}}]",[],{},[],,,ok,False, +,5e043630-6493-42b5-beb8-79faa19bfa37,e30e3e61-6edd-4b62-ab77-627faa2bf741,,,agent,agent,What APIs are available in Noveum platform?,You are a helpful customer support agent,,,"[{""name"": ""query_routing"", ""description"": ""Routes queries to appropriate handlers"", ""args_schema"": {}, ""return_schema"": {""type"": ""routing_decision""}}]",[],{},[],,,ok,False, +,7da9814d-a2e8-4c4e-b750-68b26bd5fd22,2f513795-764f-4eb6-8dd2-108d4cdf8432,,,agent,agent,How do I set up observability with Noveum?,You are a helpful customer support agent,,,"[{""name"": ""query_routing"", ""description"": ""Routes queries to appropriate handlers"", ""args_schema"": {}, ""return_schema"": {""type"": ""routing_decision""}}]",[],{},[],,,ok,False, +,16143f74-2831-4753-b33d-ce4b645093c5,29c7a515-2383-4b7d-8fc7-beac3f403641,,,agent,agent,What features does Noveum Trace offer?,You are a helpful customer support agent,,,"[{""name"": ""query_routing"", ""description"": ""Routes queries to appropriate handlers"", ""args_schema"": {}, ""return_schema"": {""type"": ""routing_decision""}}]",[],{},[],,,ok,False, +,fc64e6cc-6739-4256-ac4a-7b80c3028233,12fee687-2a2e-401f-83c2-9c8dc7ecd853,,,agent,agent,What are Noveum's pricing plans?,You are a helpful customer support agent,,,"[{""name"": ""query_routing"", ""description"": ""Routes queries to appropriate handlers"", ""args_schema"": {}, ""return_schema"": {""type"": ""routing_decision""}}]",[],{},[],,,ok,False, +,b7945c49-f584-4c70-972d-536a805d8a31,9c5aec32-b6eb-4c42-93d5-3970fb118c00,,,agent,agent,How do I integrate Noveum Trace in my application?,You are a helpful customer support agent,,,"[{""name"": ""query_routing"", ""description"": ""Routes queries to appropriate handlers"", ""args_schema"": {}, ""return_schema"": {""type"": ""routing_decision""}}]",[],{},[],,,ok,False, +,f5c40ecf-36c0-45ba-9cc9-dc0329b0324b,f132d053-f82c-4e66-adf4-bd7ce68b660d,,,agent,agent,What is Noveum and what does it do?,You are a helpful customer support agent,,,"[{""name"": ""query_routing"", ""description"": ""Routes queries to appropriate handlers"", ""args_schema"": {}, ""return_schema"": {""type"": ""routing_decision""}}]",[],{},[],,,ok,False, diff --git a/noveum_customer_support_bt/processed_datasets/agent.query_routing_dataset_processed_dataset.json b/noveum_customer_support_bt/processed_datasets/agent.query_routing_dataset_processed_dataset.json new file mode 100644 index 0000000..3bca440 --- /dev/null +++ b/noveum_customer_support_bt/processed_datasets/agent.query_routing_dataset_processed_dataset.json @@ -0,0 +1,622 @@ +[ + { + "user_id": null, + "task_id": "eda4fe22-9a2b-4b73-856b-f4f3309bf719", + "turn_id": "0b19befc-5dc0-4b00-9a8b-5f2f2c914f17", + "ground_truth": null, + "expected_tool_call": null, + "agent_name": "agent", + "agent_role": "agent", + "agent_task": "What are the latest breakthroughs in artificial intelligence?", + "system_prompt": "You are a helpful customer support agent", + "agent_response": "", + "trace": null, + "tools_available": [ + { + "name": "query_routing", + "description": "Routes queries to appropriate handlers", + "args_schema": {}, + "return_schema": { + "type": "routing_decision" + } + } + ], + "tool_calls": [], + "parameters_passed": {}, + "tool_call_results": [], + "retrieval_query": null, + "retrieved_context": null, + "exit_status": "ok", + "agent_exit": false, + "metadata": null + }, + { + "user_id": null, + "task_id": "0ffffba1-8a37-443c-8866-d53ffbfa7718", + "turn_id": "30644d2a-5ab1-4a73-b4ff-cf9fa478e36b", + "ground_truth": null, + "expected_tool_call": null, + "agent_name": "agent", + "agent_role": "agent", + "agent_task": "What's happening in the software development world today?", + "system_prompt": "You are a helpful customer support agent", + "agent_response": "", + "trace": null, + "tools_available": [ + { + "name": "query_routing", + "description": "Routes queries to appropriate handlers", + "args_schema": {}, + "return_schema": { + "type": "routing_decision" + } + } + ], + "tool_calls": [], + "parameters_passed": {}, + "tool_call_results": [], + "retrieval_query": null, + "retrieved_context": null, + "exit_status": "ok", + "agent_exit": false, + "metadata": null + }, + { + "user_id": null, + "task_id": "f1f37bd7-0851-4659-b493-b80d3800d920", + "turn_id": "89eac2de-b7da-4db3-bf7c-8e87c429481e", + "ground_truth": null, + "expected_tool_call": null, + "agent_name": "agent", + "agent_role": "agent", + "agent_task": "What are the newest features in cloud computing?", + "system_prompt": "You are a helpful customer support agent", + "agent_response": "", + "trace": null, + "tools_available": [ + { + "name": "query_routing", + "description": "Routes queries to appropriate handlers", + "args_schema": {}, + "return_schema": { + "type": "routing_decision" + } + } + ], + "tool_calls": [], + "parameters_passed": {}, + "tool_call_results": [], + "retrieval_query": null, + "retrieved_context": null, + "exit_status": "ok", + "agent_exit": false, + "metadata": null + }, + { + "user_id": null, + "task_id": "43cdf081-4f01-49cd-b566-dbd1619e6cd2", + "turn_id": "962afebb-26d1-45ff-acf1-218163cd6bb8", + "ground_truth": null, + "expected_tool_call": null, + "agent_name": "agent", + "agent_role": "agent", + "agent_task": "What's the current status of cryptocurrency markets?", + "system_prompt": "You are a helpful customer support agent", + "agent_response": "", + "trace": null, + "tools_available": [ + { + "name": "query_routing", + "description": "Routes queries to appropriate handlers", + "args_schema": {}, + "return_schema": { + "type": "routing_decision" + } + } + ], + "tool_calls": [], + "parameters_passed": {}, + "tool_call_results": [], + "retrieval_query": null, + "retrieved_context": null, + "exit_status": "ok", + "agent_exit": false, + "metadata": null + }, + { + "user_id": null, + "task_id": "9a1983f4-09da-4b53-80e6-38de6878e0e7", + "turn_id": "246cf822-4562-4d3a-8a5a-e38e0a0b02a0", + "ground_truth": null, + "expected_tool_call": null, + "agent_name": "agent", + "agent_role": "agent", + "agent_task": "What are the latest updates in Python programming?", + "system_prompt": "You are a helpful customer support agent", + "agent_response": "", + "trace": null, + "tools_available": [ + { + "name": "query_routing", + "description": "Routes queries to appropriate handlers", + "args_schema": {}, + "return_schema": { + "type": "routing_decision" + } + } + ], + "tool_calls": [], + "parameters_passed": {}, + "tool_call_results": [], + "retrieval_query": null, + "retrieved_context": null, + "exit_status": "ok", + "agent_exit": false, + "metadata": null + }, + { + "user_id": null, + "task_id": "5d2517e1-220a-429d-9d59-f701bda25eed", + "turn_id": "20544f91-8828-4a0e-b253-ed1f8b95b205", + "ground_truth": null, + "expected_tool_call": null, + "agent_name": "agent", + "agent_role": "agent", + "agent_task": "What happened in tech news this week?", + "system_prompt": "You are a helpful customer support agent", + "agent_response": "", + "trace": null, + "tools_available": [ + { + "name": "query_routing", + "description": "Routes queries to appropriate handlers", + "args_schema": {}, + "return_schema": { + "type": "routing_decision" + } + } + ], + "tool_calls": [], + "parameters_passed": {}, + "tool_call_results": [], + "retrieval_query": null, + "retrieved_context": null, + "exit_status": "ok", + "agent_exit": false, + "metadata": null + }, + { + "user_id": null, + "task_id": "52aacb67-c361-4445-9b72-c157f79f47d6", + "turn_id": "5359c861-c51b-446d-9ad2-b3d8c21ea70f", + "ground_truth": null, + "expected_tool_call": null, + "agent_name": "agent", + "agent_role": "agent", + "agent_task": "What are the current trends in observability tools?", + "system_prompt": "You are a helpful customer support agent", + "agent_response": "", + "trace": null, + "tools_available": [ + { + "name": "query_routing", + "description": "Routes queries to appropriate handlers", + "args_schema": {}, + "return_schema": { + "type": "routing_decision" + } + } + ], + "tool_calls": [], + "parameters_passed": {}, + "tool_call_results": [], + "retrieval_query": null, + "retrieved_context": null, + "exit_status": "ok", + "agent_exit": false, + "metadata": null + }, + { + "user_id": null, + "task_id": "a81ca3a8-80aa-4c39-876e-8d40ea7a0aef", + "turn_id": "acf6a99a-39e0-4f87-a07e-3a61213ab032", + "ground_truth": null, + "expected_tool_call": null, + "agent_name": "agent", + "agent_role": "agent", + "agent_task": "Tell me about recent developments in machine learning", + "system_prompt": "You are a helpful customer support agent", + "agent_response": "", + "trace": null, + "tools_available": [ + { + "name": "query_routing", + "description": "Routes queries to appropriate handlers", + "args_schema": {}, + "return_schema": { + "type": "routing_decision" + } + } + ], + "tool_calls": [], + "parameters_passed": {}, + "tool_call_results": [], + "retrieval_query": null, + "retrieved_context": null, + "exit_status": "ok", + "agent_exit": false, + "metadata": null + }, + { + "user_id": null, + "task_id": "230aad27-f3dd-4968-a45a-3c2f07ac28ed", + "turn_id": "0b47ea07-1eda-4b6a-a183-ea9ef047e431", + "ground_truth": null, + "expected_tool_call": null, + "agent_name": "agent", + "agent_role": "agent", + "agent_task": "What's the weather like today?", + "system_prompt": "You are a helpful customer support agent", + "agent_response": "", + "trace": null, + "tools_available": [ + { + "name": "query_routing", + "description": "Routes queries to appropriate handlers", + "args_schema": {}, + "return_schema": { + "type": "routing_decision" + } + } + ], + "tool_calls": [], + "parameters_passed": {}, + "tool_call_results": [], + "retrieval_query": null, + "retrieved_context": null, + "exit_status": "ok", + "agent_exit": false, + "metadata": null + }, + { + "user_id": null, + "task_id": "83c7dcce-3d89-4da1-8b3f-d419885d4cbc", + "turn_id": "3866ddb3-b17e-47a1-8b3f-bca1e32a2ae4", + "ground_truth": null, + "expected_tool_call": null, + "agent_name": "agent", + "agent_role": "agent", + "agent_task": "What are the latest AI news today?", + "system_prompt": "You are a helpful customer support agent", + "agent_response": "", + "trace": null, + "tools_available": [ + { + "name": "query_routing", + "description": "Routes queries to appropriate handlers", + "args_schema": {}, + "return_schema": { + "type": "routing_decision" + } + } + ], + "tool_calls": [], + "parameters_passed": {}, + "tool_call_results": [], + "retrieval_query": null, + "retrieved_context": null, + "exit_status": "ok", + "agent_exit": false, + "metadata": null + }, + { + "user_id": null, + "task_id": "2218f641-604c-491a-9710-b51a9941b982", + "turn_id": "3ce2e35e-4518-43b4-8a76-4413da6832ea", + "ground_truth": null, + "expected_tool_call": null, + "agent_name": "agent", + "agent_role": "agent", + "agent_task": "What are the benefits of using Noveum Trace?", + "system_prompt": "You are a helpful customer support agent", + "agent_response": "", + "trace": null, + "tools_available": [ + { + "name": "query_routing", + "description": "Routes queries to appropriate handlers", + "args_schema": {}, + "return_schema": { + "type": "routing_decision" + } + } + ], + "tool_calls": [], + "parameters_passed": {}, + "tool_call_results": [], + "retrieval_query": null, + "retrieved_context": null, + "exit_status": "ok", + "agent_exit": false, + "metadata": null + }, + { + "user_id": null, + "task_id": "255fd49c-84b4-4b18-887e-6308a412d535", + "turn_id": "c0c39e8b-9a42-4c18-8737-9224cd8a45c0", + "ground_truth": null, + "expected_tool_call": null, + "agent_name": "agent", + "agent_role": "agent", + "agent_task": "How do I configure Noveum for my system?", + "system_prompt": "You are a helpful customer support agent", + "agent_response": "", + "trace": null, + "tools_available": [ + { + "name": "query_routing", + "description": "Routes queries to appropriate handlers", + "args_schema": {}, + "return_schema": { + "type": "routing_decision" + } + } + ], + "tool_calls": [], + "parameters_passed": {}, + "tool_call_results": [], + "retrieval_query": null, + "retrieved_context": null, + "exit_status": "ok", + "agent_exit": false, + "metadata": null + }, + { + "user_id": null, + "task_id": "dc511122-c0b6-415c-9a49-c7b45132dd87", + "turn_id": "80987c51-ea82-4a41-97c7-ed591082b9bf", + "ground_truth": null, + "expected_tool_call": null, + "agent_name": "agent", + "agent_role": "agent", + "agent_task": "What monitoring capabilities does Noveum provide?", + "system_prompt": "You are a helpful customer support agent", + "agent_response": "", + "trace": null, + "tools_available": [ + { + "name": "query_routing", + "description": "Routes queries to appropriate handlers", + "args_schema": {}, + "return_schema": { + "type": "routing_decision" + } + } + ], + "tool_calls": [], + "parameters_passed": {}, + "tool_call_results": [], + "retrieval_query": null, + "retrieved_context": null, + "exit_status": "ok", + "agent_exit": false, + "metadata": null + }, + { + "user_id": null, + "task_id": "04bebf38-a343-4563-80db-0154bef8d927", + "turn_id": "2f89c1e0-9b62-4651-914c-b71cf561c33c", + "ground_truth": null, + "expected_tool_call": null, + "agent_name": "agent", + "agent_role": "agent", + "agent_task": "How does Noveum handle agent tracing?", + "system_prompt": "You are a helpful customer support agent", + "agent_response": "", + "trace": null, + "tools_available": [ + { + "name": "query_routing", + "description": "Routes queries to appropriate handlers", + "args_schema": {}, + "return_schema": { + "type": "routing_decision" + } + } + ], + "tool_calls": [], + "parameters_passed": {}, + "tool_call_results": [], + "retrieval_query": null, + "retrieved_context": null, + "exit_status": "ok", + "agent_exit": false, + "metadata": null + }, + { + "user_id": null, + "task_id": "5e043630-6493-42b5-beb8-79faa19bfa37", + "turn_id": "e30e3e61-6edd-4b62-ab77-627faa2bf741", + "ground_truth": null, + "expected_tool_call": null, + "agent_name": "agent", + "agent_role": "agent", + "agent_task": "What APIs are available in Noveum platform?", + "system_prompt": "You are a helpful customer support agent", + "agent_response": "", + "trace": null, + "tools_available": [ + { + "name": "query_routing", + "description": "Routes queries to appropriate handlers", + "args_schema": {}, + "return_schema": { + "type": "routing_decision" + } + } + ], + "tool_calls": [], + "parameters_passed": {}, + "tool_call_results": [], + "retrieval_query": null, + "retrieved_context": null, + "exit_status": "ok", + "agent_exit": false, + "metadata": null + }, + { + "user_id": null, + "task_id": "7da9814d-a2e8-4c4e-b750-68b26bd5fd22", + "turn_id": "2f513795-764f-4eb6-8dd2-108d4cdf8432", + "ground_truth": null, + "expected_tool_call": null, + "agent_name": "agent", + "agent_role": "agent", + "agent_task": "How do I set up observability with Noveum?", + "system_prompt": "You are a helpful customer support agent", + "agent_response": "", + "trace": null, + "tools_available": [ + { + "name": "query_routing", + "description": "Routes queries to appropriate handlers", + "args_schema": {}, + "return_schema": { + "type": "routing_decision" + } + } + ], + "tool_calls": [], + "parameters_passed": {}, + "tool_call_results": [], + "retrieval_query": null, + "retrieved_context": null, + "exit_status": "ok", + "agent_exit": false, + "metadata": null + }, + { + "user_id": null, + "task_id": "16143f74-2831-4753-b33d-ce4b645093c5", + "turn_id": "29c7a515-2383-4b7d-8fc7-beac3f403641", + "ground_truth": null, + "expected_tool_call": null, + "agent_name": "agent", + "agent_role": "agent", + "agent_task": "What features does Noveum Trace offer?", + "system_prompt": "You are a helpful customer support agent", + "agent_response": "", + "trace": null, + "tools_available": [ + { + "name": "query_routing", + "description": "Routes queries to appropriate handlers", + "args_schema": {}, + "return_schema": { + "type": "routing_decision" + } + } + ], + "tool_calls": [], + "parameters_passed": {}, + "tool_call_results": [], + "retrieval_query": null, + "retrieved_context": null, + "exit_status": "ok", + "agent_exit": false, + "metadata": null + }, + { + "user_id": null, + "task_id": "fc64e6cc-6739-4256-ac4a-7b80c3028233", + "turn_id": "12fee687-2a2e-401f-83c2-9c8dc7ecd853", + "ground_truth": null, + "expected_tool_call": null, + "agent_name": "agent", + "agent_role": "agent", + "agent_task": "What are Noveum's pricing plans?", + "system_prompt": "You are a helpful customer support agent", + "agent_response": "", + "trace": null, + "tools_available": [ + { + "name": "query_routing", + "description": "Routes queries to appropriate handlers", + "args_schema": {}, + "return_schema": { + "type": "routing_decision" + } + } + ], + "tool_calls": [], + "parameters_passed": {}, + "tool_call_results": [], + "retrieval_query": null, + "retrieved_context": null, + "exit_status": "ok", + "agent_exit": false, + "metadata": null + }, + { + "user_id": null, + "task_id": "b7945c49-f584-4c70-972d-536a805d8a31", + "turn_id": "9c5aec32-b6eb-4c42-93d5-3970fb118c00", + "ground_truth": null, + "expected_tool_call": null, + "agent_name": "agent", + "agent_role": "agent", + "agent_task": "How do I integrate Noveum Trace in my application?", + "system_prompt": "You are a helpful customer support agent", + "agent_response": "", + "trace": null, + "tools_available": [ + { + "name": "query_routing", + "description": "Routes queries to appropriate handlers", + "args_schema": {}, + "return_schema": { + "type": "routing_decision" + } + } + ], + "tool_calls": [], + "parameters_passed": {}, + "tool_call_results": [], + "retrieval_query": null, + "retrieved_context": null, + "exit_status": "ok", + "agent_exit": false, + "metadata": null + }, + { + "user_id": null, + "task_id": "f5c40ecf-36c0-45ba-9cc9-dc0329b0324b", + "turn_id": "f132d053-f82c-4e66-adf4-bd7ce68b660d", + "ground_truth": null, + "expected_tool_call": null, + "agent_name": "agent", + "agent_role": "agent", + "agent_task": "What is Noveum and what does it do?", + "system_prompt": "You are a helpful customer support agent", + "agent_response": "", + "trace": null, + "tools_available": [ + { + "name": "query_routing", + "description": "Routes queries to appropriate handlers", + "args_schema": {}, + "return_schema": { + "type": "routing_decision" + } + } + ], + "tool_calls": [], + "parameters_passed": {}, + "tool_call_results": [], + "retrieval_query": null, + "retrieved_context": null, + "exit_status": "ok", + "agent_exit": false, + "metadata": null + } +] \ No newline at end of file diff --git a/noveum_customer_support_bt/processed_datasets/agent.rag_evaluation_metrics_dataset_processed_dataset.csv b/noveum_customer_support_bt/processed_datasets/agent.rag_evaluation_metrics_dataset_processed_dataset.csv new file mode 100644 index 0000000..cf8d2b1 --- /dev/null +++ b/noveum_customer_support_bt/processed_datasets/agent.rag_evaluation_metrics_dataset_processed_dataset.csv @@ -0,0 +1,22 @@ +user_id,task_id,turn_id,ground_truth,expected_tool_call,agent_name,agent_role,agent_task,system_prompt,agent_response,trace,tools_available,tool_calls,parameters_passed,tool_call_results,retrieval_query,retrieved_context,exit_status,agent_exit,metadata +,f1f37bd7-0851-4659-b493-b80d3800d920,ac166548-ffa5-47b9-8cf7-4787f270cb5a,,,agent,agent,Evaluation for query: What are the newest features in cloud computing?,You are a helpful customer support agent,"RAG Answer: The provided context does not specifically mention any new features in cloud computing as a general topic. However, it does highlight several recent enhancements and features related to the Noveum.ai ...",,"[{""name"": ""rag_evaluation"", ""description"": ""Evaluates RAG system performance"", ""args_schema"": {}, ""return_schema"": {""type"": ""evaluation_result""}}]",[],{},[],"[""Evaluation for query: What are the newest features in cloud computing?""]","[[""Context: Source 1 (https://noveum.ai/en/changelog):\n- \ud83d\udcc8 Enhanced Dashboard Analytics: Real-time request tracking, latency monitoring, and cost analysis\n- \ud83d\udd0d Improved Logs Interface: Better search functionality, detailed trace views, and enhanced debugging capabilities\n- \ud83d\udc65 Team Management: Advanced member mana...""]]",ok,False, +,52aacb67-c361-4445-9b72-c157f79f47d6,e193cd4f-cca3-4b94-95d3-a1f2446b9fc6,,,agent,agent,Evaluation for query: What are the current trends in observability tools?,You are a helpful customer support agent,"RAG Answer: The provided context does not explicitly outline current trends in observability tools. However, it highlights some key aspects of observability that are particularly relevant to AI applications. + +1....",,"[{""name"": ""rag_evaluation"", ""description"": ""Evaluates RAG system performance"", ""args_schema"": {}, ""return_schema"": {""type"": ""evaluation_result""}}]",[],{},[],"[""Evaluation for query: What are the current trends in observability tools?""]","[[""Context: Source 1 (https://noveum.ai/en/blog/from-logs-to-intelligent-choices-inside-noveum-ais-evaluation-process):\nFrom Development to Production - Inside Noveum.ai's AI Observability Platform\nShashank Agarwal\n3/3/2025\nIntroduction\nEvery AI application tells a story\u2014but most developers never get to hear it...""]]",ok,False, +,2218f641-604c-491a-9710-b51a9941b982,432e74f3-3291-411b-8d7e-85493868f8cf,,,agent,agent,Evaluation for query: What are the benefits of using Noveum Trace?,You are a helpful customer support agent,"RAG Answer: Using Noveum Trace offers several benefits, including: + +1. **Real-Time Monitoring**: The Noveum platform provides a real-time dashboard for analyzing traces and performance, with live updates as new t...",,"[{""name"": ""rag_evaluation"", ""description"": ""Evaluates RAG system performance"", ""args_schema"": {}, ""return_schema"": {""type"": ""evaluation_result""}}]",[],{},[],"[""Evaluation for query: What are the benefits of using Noveum Trace?""]","[[""Context: Source 1 (https://noveum.ai/docs/platform/dashboard):\n- Bulk Export: Download multiple traces simultaneously\n- Comparative Analysis: Compare performance across multiple traces\nReal-Time Updates\n- Live Refresh: Automatic updates as new traces arrive\n- Connection Monitoring: Real-time status of your t...""]]",ok,False, +,255fd49c-84b4-4b18-887e-6308a412d535,a0475dd5-ef9f-4eb2-bca4-9be72bf6bfd6,,,agent,agent,Evaluation for query: How do I configure Noveum for my system?,You are a helpful customer support agent,"RAG Answer: To configure Noveum for your system, you can refer to the SDK Integration Guide available in the Noveum documentation. This guide will help you trace your AI applications with minimal code changes. Th...",,"[{""name"": ""rag_evaluation"", ""description"": ""Evaluates RAG system performance"", ""args_schema"": {}, ""return_schema"": {""type"": ""evaluation_result""}}]",[],{},[],"[""Evaluation for query: How do I configure Noveum for my system?""]","[[""Context: Source 1 (https://noveum.ai/en/docs):\nWelcome to Noveum.ai Docs\nComprehensive AI tracing and observability for LLM applications, RAG systems, and AI agents\nWelcome to the Noveum.ai documentation! Here you'll find everything you need to integrate, configure, and optimize your AI applications using ou...""]]",ok,False, +,dc511122-c0b6-415c-9a49-c7b45132dd87,ed8ad539-f008-4f1a-beb1-12ad5793950e,,,agent,agent,Evaluation for query: What monitoring capabilities does Noveum provide?,You are a helpful customer support agent,"RAG Answer: Noveum provides comprehensive monitoring capabilities for AI applications, including the ability to monitor, trace, and optimize AI agents across various frameworks such as LangChain, CrewAI, AutoGen,...",,"[{""name"": ""rag_evaluation"", ""description"": ""Evaluates RAG system performance"", ""args_schema"": {}, ""return_schema"": {""type"": ""evaluation_result""}}]",[],{},[],"[""Evaluation for query: What monitoring capabilities does Noveum provide?""]","[[""Context: Source 1 (https://noveum.ai/en):\nMonitor all your AI Agents\nimprove AI Agents today\nNoveum.ai helps you monitor, trace, and optimize your AI applications.\nNoveum.ai works with any AI framework \u2013 LangChain, CrewAI, AutoGen, custom implementations, or direct LLM calls. One dashboard shows everything.\n...""]]",ok,False, +,04bebf38-a343-4563-80db-0154bef8d927,c7358391-9ac1-4312-a4b3-79edfa8bc0d3,,,agent,agent,Evaluation for query: How does Noveum handle agent tracing?,You are a helpful customer support agent,"RAG Answer: Noveum.ai handles agent tracing through its specialized Multi-Agent Tracing capabilities, which are designed to observe complex workflows and inter-agent communications within multi-agent systems. The...",,"[{""name"": ""rag_evaluation"", ""description"": ""Evaluates RAG system performance"", ""args_schema"": {}, ""return_schema"": {""type"": ""evaluation_result""}}]",[],{},[],"[""Evaluation for query: How does Noveum handle agent tracing?""]","[[""Context: Source 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, vector searches, data retrieval, agent reasoning, and more. Without proper observability, debugging ...""]]",ok,False, +,5e043630-6493-42b5-beb8-79faa19bfa37,6b21de5d-456f-4501-953f-0ce5bc4fa5a5,,,agent,agent,Evaluation for query: What APIs are available in Noveum platform?,You are a helpful customer support agent,"RAG Answer: The provided context does not specify the APIs available in the Noveum platform. For detailed information about the APIs, I recommend checking the official Noveum documentation or the SDK Integration ...",,"[{""name"": ""rag_evaluation"", ""description"": ""Evaluates RAG system performance"", ""args_schema"": {}, ""return_schema"": {""type"": ""evaluation_result""}}]",[],{},[],"[""Evaluation for query: What APIs are available in Noveum platform?""]","[[""Context: Source 1 (https://noveum.ai/en/docs):\n- \ud83c\udf0d Universal: Works across frameworks, clouds, and deployment models\n\ud83e\udd1d Community & Support\n- \ud83d\udcac Discord Community: Join our Discord\n- \ud83d\udce7 Email Support: [email protected]\n- \ud83d\udc1b Bug Reports: GitHub Issues\n- \ud83d\udcd6 Knowledge Base: Help Center\nReady to get started? Head to ...""]]",ok,False, +,7da9814d-a2e8-4c4e-b750-68b26bd5fd22,2ad4e155-7b76-4040-923d-d6b65a97a45b,,,agent,agent,Evaluation for query: How do I set up observability with Noveum?,You are a helpful customer support agent,"RAG Answer: To set up observability with Noveum, follow these steps: + +1. **Sign Up**: Go to [noveum.ai](https://noveum.ai) and create an account. +2. **Create a Project**: Once signed in, create a new project to o...",,"[{""name"": ""rag_evaluation"", ""description"": ""Evaluates RAG system performance"", ""args_schema"": {}, ""return_schema"": {""type"": ""evaluation_result""}}]",[],{},[],"[""Evaluation for query: How do I set up observability with Noveum?""]","[[""Context: Source 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\n- Production Ready: Built for scale with intelligent sampling and batching\n- Privacy Focused: You control what data is captured and how it's stored\n- Open Source SDKs: Transparent, auditable, and extensible\nGetting ...""]]",ok,False, +,16143f74-2831-4753-b33d-ce4b645093c5,6b822089-be40-457d-9fa6-0979bafdfb85,,,agent,agent,Evaluation for query: What features does Noveum Trace offer?,You are a helpful customer support agent,"RAG Answer: Noveum Trace offers several key features through its Python SDK (`noveum-trace`) and TypeScript SDK (`@noveum/trace`): + +1. **Decorator-based Tracing**: It provides a simple way to trace LLM calls, age...",,"[{""name"": ""rag_evaluation"", ""description"": ""Evaluates RAG system performance"", ""args_schema"": {}, ""return_schema"": {""type"": ""evaluation_result""}}]",[],{},[],"[""Evaluation for query: What features does Noveum Trace offer?""]","[[""Context: Source 1 (https://noveum.ai/en/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\n\ud83d\udcd8 TypeScript SDK (\n@noveum/trace\n)- Framework ...""]]",ok,False, +,fc64e6cc-6739-4256-ac4a-7b80c3028233,08db68f6-3623-44b5-840b-41400fac7333,,,agent,agent,Evaluation for query: What are Noveum's pricing plans?,You are a helpful customer support agent,"RAG Answer: The provided context does not include specific information about Noveum's pricing plans. However, it mentions that all users will have access to the Observability suite for free, and early users will ...",,"[{""name"": ""rag_evaluation"", ""description"": ""Evaluates RAG system performance"", ""args_schema"": {}, ""return_schema"": {""type"": ""evaluation_result""}}]",[],{},[],"[""Evaluation for query: What are Noveum's pricing plans?""]","[[""Context: Source 1 (https://noveum.ai/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....""]]",ok,False, +,b7945c49-f584-4c70-972d-536a805d8a31,6a57c92e-e473-425a-978b-19dbe976a46e,,,agent,agent,Evaluation for query: How do I integrate Noveum Trace in my application?,You are a helpful customer support agent,"RAG Answer: To integrate Noveum Trace into your application, you can use either the Python SDK (`noveum-trace`) or the TypeScript SDK (`@noveum/trace`). Hereā€™s a quick start guide based on the provided documentat...",,"[{""name"": ""rag_evaluation"", ""description"": ""Evaluates RAG system performance"", ""args_schema"": {}, ""return_schema"": {""type"": ""evaluation_result""}}]",[],{},[],"[""Evaluation for query: How do I integrate Noveum Trace in my application?""]","[[""Context: Source 1 (https://noveum.ai/en/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\n\ud83d\udcd8 TypeScript SDK (\n@noveum/trace\n)- Framework ...""]]",ok,False, +,f5c40ecf-36c0-45ba-9cc9-dc0329b0324b,efb19490-bc9e-48dc-a365-2f839100e41a,,,agent,agent,Evaluation for query: What is Noveum and what does it do?,You are a helpful customer support agent,"RAG Answer: Noveum.ai is a comprehensive tracing and observability platform specifically designed for AI applications, including those powered by large language models (LLMs), retrieval-augmented generation (RAG)...",,"[{""name"": ""rag_evaluation"", ""description"": ""Evaluates RAG system performance"", ""args_schema"": {}, ""return_schema"": {""type"": ""evaluation_result""}}]",[],{},[],"[""Evaluation for query: What is Noveum and what does it do?""]","[[""Context: Source 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, vector searches, data retrieval, agent reasoning, and more. Without proper observability, debugging ...""]]",ok,False, diff --git a/noveum_customer_support_bt/processed_datasets/agent.rag_evaluation_metrics_dataset_processed_dataset.json b/noveum_customer_support_bt/processed_datasets/agent.rag_evaluation_metrics_dataset_processed_dataset.json new file mode 100644 index 0000000..78aad41 --- /dev/null +++ b/noveum_customer_support_bt/processed_datasets/agent.rag_evaluation_metrics_dataset_processed_dataset.json @@ -0,0 +1,446 @@ +[ + { + "user_id": null, + "task_id": "f1f37bd7-0851-4659-b493-b80d3800d920", + "turn_id": "ac166548-ffa5-47b9-8cf7-4787f270cb5a", + "ground_truth": null, + "expected_tool_call": null, + "agent_name": "agent", + "agent_role": "agent", + "agent_task": "Evaluation for query: What are the newest features in cloud computing?", + "system_prompt": "You are a helpful customer support agent", + "agent_response": "RAG Answer: The provided context does not specifically mention any new features in cloud computing as a general topic. However, it does highlight several recent enhancements and features related to the Noveum.ai ...", + "trace": null, + "tools_available": [ + { + "name": "rag_evaluation", + "description": "Evaluates RAG system performance", + "args_schema": {}, + "return_schema": { + "type": "evaluation_result" + } + } + ], + "tool_calls": [], + "parameters_passed": {}, + "tool_call_results": [], + "retrieval_query": [ + "Evaluation for query: What are the newest features in cloud computing?" + ], + "retrieved_context": [ + [ + "Context: Source 1 (https://noveum.ai/en/changelog):\n- šŸ“ˆ Enhanced Dashboard Analytics: Real-time request tracking, latency monitoring, and cost analysis\n- šŸ” Improved Logs Interface: Better search functionality, detailed trace views, and enhanced debugging capabilities\n- šŸ‘„ Team Management: Advanced member mana..." + ] + ], + "exit_status": "ok", + "agent_exit": false, + "metadata": null + }, + { + "user_id": null, + "task_id": "52aacb67-c361-4445-9b72-c157f79f47d6", + "turn_id": "e193cd4f-cca3-4b94-95d3-a1f2446b9fc6", + "ground_truth": null, + "expected_tool_call": null, + "agent_name": "agent", + "agent_role": "agent", + "agent_task": "Evaluation for query: What are the current trends in observability tools?", + "system_prompt": "You are a helpful customer support agent", + "agent_response": "RAG Answer: The provided context does not explicitly outline current trends in observability tools. However, it highlights some key aspects of observability that are particularly relevant to AI applications. \n\n1....", + "trace": null, + "tools_available": [ + { + "name": "rag_evaluation", + "description": "Evaluates RAG system performance", + "args_schema": {}, + "return_schema": { + "type": "evaluation_result" + } + } + ], + "tool_calls": [], + "parameters_passed": {}, + "tool_call_results": [], + "retrieval_query": [ + "Evaluation for query: What are the current trends in observability tools?" + ], + "retrieved_context": [ + [ + "Context: Source 1 (https://noveum.ai/en/blog/from-logs-to-intelligent-choices-inside-noveum-ais-evaluation-process):\nFrom Development to Production - Inside Noveum.ai's AI Observability Platform\nShashank Agarwal\n3/3/2025\nIntroduction\nEvery AI application tells a storyā€”but most developers never get to hear it..." + ] + ], + "exit_status": "ok", + "agent_exit": false, + "metadata": null + }, + { + "user_id": null, + "task_id": "2218f641-604c-491a-9710-b51a9941b982", + "turn_id": "432e74f3-3291-411b-8d7e-85493868f8cf", + "ground_truth": null, + "expected_tool_call": null, + "agent_name": "agent", + "agent_role": "agent", + "agent_task": "Evaluation for query: What are the benefits of using Noveum Trace?", + "system_prompt": "You are a helpful customer support agent", + "agent_response": "RAG Answer: Using Noveum Trace offers several benefits, including:\n\n1. **Real-Time Monitoring**: The Noveum platform provides a real-time dashboard for analyzing traces and performance, with live updates as new t...", + "trace": null, + "tools_available": [ + { + "name": "rag_evaluation", + "description": "Evaluates RAG system performance", + "args_schema": {}, + "return_schema": { + "type": "evaluation_result" + } + } + ], + "tool_calls": [], + "parameters_passed": {}, + "tool_call_results": [], + "retrieval_query": [ + "Evaluation for query: What are the benefits of using Noveum Trace?" + ], + "retrieved_context": [ + [ + "Context: Source 1 (https://noveum.ai/docs/platform/dashboard):\n- Bulk Export: Download multiple traces simultaneously\n- Comparative Analysis: Compare performance across multiple traces\nReal-Time Updates\n- Live Refresh: Automatic updates as new traces arrive\n- Connection Monitoring: Real-time status of your t..." + ] + ], + "exit_status": "ok", + "agent_exit": false, + "metadata": null + }, + { + "user_id": null, + "task_id": "255fd49c-84b4-4b18-887e-6308a412d535", + "turn_id": "a0475dd5-ef9f-4eb2-bca4-9be72bf6bfd6", + "ground_truth": null, + "expected_tool_call": null, + "agent_name": "agent", + "agent_role": "agent", + "agent_task": "Evaluation for query: How do I configure Noveum for my system?", + "system_prompt": "You are a helpful customer support agent", + "agent_response": "RAG Answer: To configure Noveum for your system, you can refer to the SDK Integration Guide available in the Noveum documentation. This guide will help you trace your AI applications with minimal code changes. Th...", + "trace": null, + "tools_available": [ + { + "name": "rag_evaluation", + "description": "Evaluates RAG system performance", + "args_schema": {}, + "return_schema": { + "type": "evaluation_result" + } + } + ], + "tool_calls": [], + "parameters_passed": {}, + "tool_call_results": [], + "retrieval_query": [ + "Evaluation for query: How do I configure Noveum for my system?" + ], + "retrieved_context": [ + [ + "Context: Source 1 (https://noveum.ai/en/docs):\nWelcome to Noveum.ai Docs\nComprehensive AI tracing and observability for LLM applications, RAG systems, and AI agents\nWelcome to the Noveum.ai documentation! Here you'll find everything you need to integrate, configure, and optimize your AI applications using ou..." + ] + ], + "exit_status": "ok", + "agent_exit": false, + "metadata": null + }, + { + "user_id": null, + "task_id": "dc511122-c0b6-415c-9a49-c7b45132dd87", + "turn_id": "ed8ad539-f008-4f1a-beb1-12ad5793950e", + "ground_truth": null, + "expected_tool_call": null, + "agent_name": "agent", + "agent_role": "agent", + "agent_task": "Evaluation for query: What monitoring capabilities does Noveum provide?", + "system_prompt": "You are a helpful customer support agent", + "agent_response": "RAG Answer: Noveum provides comprehensive monitoring capabilities for AI applications, including the ability to monitor, trace, and optimize AI agents across various frameworks such as LangChain, CrewAI, AutoGen,...", + "trace": null, + "tools_available": [ + { + "name": "rag_evaluation", + "description": "Evaluates RAG system performance", + "args_schema": {}, + "return_schema": { + "type": "evaluation_result" + } + } + ], + "tool_calls": [], + "parameters_passed": {}, + "tool_call_results": [], + "retrieval_query": [ + "Evaluation for query: What monitoring capabilities does Noveum provide?" + ], + "retrieved_context": [ + [ + "Context: Source 1 (https://noveum.ai/en):\nMonitor all your AI Agents\nimprove AI Agents today\nNoveum.ai helps you monitor, trace, and optimize your AI applications.\nNoveum.ai works with any AI framework ā€“ LangChain, CrewAI, AutoGen, custom implementations, or direct LLM calls. One dashboard shows everything.\n..." + ] + ], + "exit_status": "ok", + "agent_exit": false, + "metadata": null + }, + { + "user_id": null, + "task_id": "04bebf38-a343-4563-80db-0154bef8d927", + "turn_id": "c7358391-9ac1-4312-a4b3-79edfa8bc0d3", + "ground_truth": null, + "expected_tool_call": null, + "agent_name": "agent", + "agent_role": "agent", + "agent_task": "Evaluation for query: How does Noveum handle agent tracing?", + "system_prompt": "You are a helpful customer support agent", + "agent_response": "RAG Answer: Noveum.ai handles agent tracing through its specialized Multi-Agent Tracing capabilities, which are designed to observe complex workflows and inter-agent communications within multi-agent systems. The...", + "trace": null, + "tools_available": [ + { + "name": "rag_evaluation", + "description": "Evaluates RAG system performance", + "args_schema": {}, + "return_schema": { + "type": "evaluation_result" + } + } + ], + "tool_calls": [], + "parameters_passed": {}, + "tool_call_results": [], + "retrieval_query": [ + "Evaluation for query: How does Noveum handle agent tracing?" + ], + "retrieved_context": [ + [ + "Context: Source 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, vector searches, data retrieval, agent reasoning, and more. Without proper observability, debugging ..." + ] + ], + "exit_status": "ok", + "agent_exit": false, + "metadata": null + }, + { + "user_id": null, + "task_id": "5e043630-6493-42b5-beb8-79faa19bfa37", + "turn_id": "6b21de5d-456f-4501-953f-0ce5bc4fa5a5", + "ground_truth": null, + "expected_tool_call": null, + "agent_name": "agent", + "agent_role": "agent", + "agent_task": "Evaluation for query: What APIs are available in Noveum platform?", + "system_prompt": "You are a helpful customer support agent", + "agent_response": "RAG Answer: The provided context does not specify the APIs available in the Noveum platform. For detailed information about the APIs, I recommend checking the official Noveum documentation or the SDK Integration ...", + "trace": null, + "tools_available": [ + { + "name": "rag_evaluation", + "description": "Evaluates RAG system performance", + "args_schema": {}, + "return_schema": { + "type": "evaluation_result" + } + } + ], + "tool_calls": [], + "parameters_passed": {}, + "tool_call_results": [], + "retrieval_query": [ + "Evaluation for query: What APIs are available in Noveum platform?" + ], + "retrieved_context": [ + [ + "Context: Source 1 (https://noveum.ai/en/docs):\n- šŸŒ Universal: Works across frameworks, clouds, and deployment models\nšŸ¤ Community & Support\n- šŸ’¬ Discord Community: Join our Discord\n- šŸ“§ Email Support: [email protected]\n- šŸ› Bug Reports: GitHub Issues\n- šŸ“– Knowledge Base: Help Center\nReady to get started? Head to ..." + ] + ], + "exit_status": "ok", + "agent_exit": false, + "metadata": null + }, + { + "user_id": null, + "task_id": "7da9814d-a2e8-4c4e-b750-68b26bd5fd22", + "turn_id": "2ad4e155-7b76-4040-923d-d6b65a97a45b", + "ground_truth": null, + "expected_tool_call": null, + "agent_name": "agent", + "agent_role": "agent", + "agent_task": "Evaluation for query: How do I set up observability with Noveum?", + "system_prompt": "You are a helpful customer support agent", + "agent_response": "RAG Answer: To set up observability with Noveum, follow these steps:\n\n1. **Sign Up**: Go to [noveum.ai](https://noveum.ai) and create an account.\n2. **Create a Project**: Once signed in, create a new project to o...", + "trace": null, + "tools_available": [ + { + "name": "rag_evaluation", + "description": "Evaluates RAG system performance", + "args_schema": {}, + "return_schema": { + "type": "evaluation_result" + } + } + ], + "tool_calls": [], + "parameters_passed": {}, + "tool_call_results": [], + "retrieval_query": [ + "Evaluation for query: How do I set up observability with Noveum?" + ], + "retrieved_context": [ + [ + "Context: Source 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\n- Production Ready: Built for scale with intelligent sampling and batching\n- Privacy Focused: You control what data is captured and how it's stored\n- Open Source SDKs: Transparent, auditable, and extensible\nGetting ..." + ] + ], + "exit_status": "ok", + "agent_exit": false, + "metadata": null + }, + { + "user_id": null, + "task_id": "16143f74-2831-4753-b33d-ce4b645093c5", + "turn_id": "6b822089-be40-457d-9fa6-0979bafdfb85", + "ground_truth": null, + "expected_tool_call": null, + "agent_name": "agent", + "agent_role": "agent", + "agent_task": "Evaluation for query: What features does Noveum Trace offer?", + "system_prompt": "You are a helpful customer support agent", + "agent_response": "RAG Answer: Noveum Trace offers several key features through its Python SDK (`noveum-trace`) and TypeScript SDK (`@noveum/trace`):\n\n1. **Decorator-based Tracing**: It provides a simple way to trace LLM calls, age...", + "trace": null, + "tools_available": [ + { + "name": "rag_evaluation", + "description": "Evaluates RAG system performance", + "args_schema": {}, + "return_schema": { + "type": "evaluation_result" + } + } + ], + "tool_calls": [], + "parameters_passed": {}, + "tool_call_results": [], + "retrieval_query": [ + "Evaluation for query: What features does Noveum Trace offer?" + ], + "retrieved_context": [ + [ + "Context: Source 1 (https://noveum.ai/en/docs):\nCore Components\n-\nšŸ Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\nšŸ“˜ TypeScript SDK (\n@noveum/trace\n)- Framework ..." + ] + ], + "exit_status": "ok", + "agent_exit": false, + "metadata": null + }, + { + "user_id": null, + "task_id": "fc64e6cc-6739-4256-ac4a-7b80c3028233", + "turn_id": "08db68f6-3623-44b5-840b-41400fac7333", + "ground_truth": null, + "expected_tool_call": null, + "agent_name": "agent", + "agent_role": "agent", + "agent_task": "Evaluation for query: What are Noveum's pricing plans?", + "system_prompt": "You are a helpful customer support agent", + "agent_response": "RAG Answer: The provided context does not include specific information about Noveum's pricing plans. However, it mentions that all users will have access to the Observability suite for free, and early users will ...", + "trace": null, + "tools_available": [ + { + "name": "rag_evaluation", + "description": "Evaluates RAG system performance", + "args_schema": {}, + "return_schema": { + "type": "evaluation_result" + } + } + ], + "tool_calls": [], + "parameters_passed": {}, + "tool_call_results": [], + "retrieval_query": [ + "Evaluation for query: What are Noveum's pricing plans?" + ], + "retrieved_context": [ + [ + "Context: Source 1 (https://noveum.ai/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year...." + ] + ], + "exit_status": "ok", + "agent_exit": false, + "metadata": null + }, + { + "user_id": null, + "task_id": "b7945c49-f584-4c70-972d-536a805d8a31", + "turn_id": "6a57c92e-e473-425a-978b-19dbe976a46e", + "ground_truth": null, + "expected_tool_call": null, + "agent_name": "agent", + "agent_role": "agent", + "agent_task": "Evaluation for query: How do I integrate Noveum Trace in my application?", + "system_prompt": "You are a helpful customer support agent", + "agent_response": "RAG Answer: To integrate Noveum Trace into your application, you can use either the Python SDK (`noveum-trace`) or the TypeScript SDK (`@noveum/trace`). Hereā€™s a quick start guide based on the provided documentat...", + "trace": null, + "tools_available": [ + { + "name": "rag_evaluation", + "description": "Evaluates RAG system performance", + "args_schema": {}, + "return_schema": { + "type": "evaluation_result" + } + } + ], + "tool_calls": [], + "parameters_passed": {}, + "tool_call_results": [], + "retrieval_query": [ + "Evaluation for query: How do I integrate Noveum Trace in my application?" + ], + "retrieved_context": [ + [ + "Context: Source 1 (https://noveum.ai/en/docs):\nCore Components\n-\nšŸ Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\nšŸ“˜ TypeScript SDK (\n@noveum/trace\n)- Framework ..." + ] + ], + "exit_status": "ok", + "agent_exit": false, + "metadata": null + }, + { + "user_id": null, + "task_id": "f5c40ecf-36c0-45ba-9cc9-dc0329b0324b", + "turn_id": "efb19490-bc9e-48dc-a365-2f839100e41a", + "ground_truth": null, + "expected_tool_call": null, + "agent_name": "agent", + "agent_role": "agent", + "agent_task": "Evaluation for query: What is Noveum and what does it do?", + "system_prompt": "You are a helpful customer support agent", + "agent_response": "RAG Answer: Noveum.ai is a comprehensive tracing and observability platform specifically designed for AI applications, including those powered by large language models (LLMs), retrieval-augmented generation (RAG)...", + "trace": null, + "tools_available": [ + { + "name": "rag_evaluation", + "description": "Evaluates RAG system performance", + "args_schema": {}, + "return_schema": { + "type": "evaluation_result" + } + } + ], + "tool_calls": [], + "parameters_passed": {}, + "tool_call_results": [], + "retrieval_query": [ + "Evaluation for query: What is Noveum and what does it do?" + ], + "retrieved_context": [ + [ + "Context: Source 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, vector searches, data retrieval, agent reasoning, and more. Without proper observability, debugging ..." + ] + ], + "exit_status": "ok", + "agent_exit": false, + "metadata": null + } +] \ No newline at end of file diff --git a/noveum_customer_support_bt/processed_datasets/agent.routing_evaluation_metrics_dataset_processed_dataset.csv b/noveum_customer_support_bt/processed_datasets/agent.routing_evaluation_metrics_dataset_processed_dataset.csv new file mode 100644 index 0000000..42daeec --- /dev/null +++ b/noveum_customer_support_bt/processed_datasets/agent.routing_evaluation_metrics_dataset_processed_dataset.csv @@ -0,0 +1,21 @@ +user_id,task_id,turn_id,ground_truth,expected_tool_call,agent_name,agent_role,agent_task,system_prompt,agent_response,trace,tools_available,tool_calls,parameters_passed,tool_call_results,retrieval_query,retrieved_context,exit_status,agent_exit,metadata +,eda4fe22-9a2b-4b73-856b-f4f3309bf719,95dac0c3-83fd-459c-af4e-c6a1734fd874,,,agent,agent,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, +,0ffffba1-8a37-443c-8866-d53ffbfa7718,516060a3-0f58-46ba-8a6a-4b502a376492,,,agent,agent,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, +,f1f37bd7-0851-4659-b493-b80d3800d920,8f8d2dc4-150f-4e43-aad3-96da4d06f586,,,agent,agent,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, +,43cdf081-4f01-49cd-b566-dbd1619e6cd2,cf0a915c-668b-412e-9c14-69119d059dc2,,,agent,agent,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, +,9a1983f4-09da-4b53-80e6-38de6878e0e7,f75b51b8-4882-4170-bc08-2adcab250f50,,,agent,agent,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, +,5d2517e1-220a-429d-9d59-f701bda25eed,4180abcf-924c-45ac-8dfa-17c0bc603eff,,,agent,agent,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, +,52aacb67-c361-4445-9b72-c157f79f47d6,8b0022cc-91dd-4065-91ed-b77217f92bd7,,,agent,agent,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, +,a81ca3a8-80aa-4c39-876e-8d40ea7a0aef,780df186-fe5a-4742-b937-2a142ad528c6,,,agent,agent,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, +,230aad27-f3dd-4968-a45a-3c2f07ac28ed,f80ae3e8-b4dd-42c6-b17b-de7805cedbb9,,,agent,agent,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, +,83c7dcce-3d89-4da1-8b3f-d419885d4cbc,209259cf-411d-4b97-8f7b-40b2530eec81,,,agent,agent,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, +,2218f641-604c-491a-9710-b51a9941b982,c3f3bd32-9954-443f-93e5-52aa6cf412a0,,,agent,agent,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, +,255fd49c-84b4-4b18-887e-6308a412d535,940ce621-0015-45ed-adc6-c4caa2a95e81,,,agent,agent,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, +,dc511122-c0b6-415c-9a49-c7b45132dd87,ea28f049-6bc9-48b4-8386-279abe087ca8,,,agent,agent,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, +,04bebf38-a343-4563-80db-0154bef8d927,01a03da5-64df-41e3-b9c9-556e12a8edd7,,,agent,agent,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, +,5e043630-6493-42b5-beb8-79faa19bfa37,8d9c632b-b2e8-496d-b1b0-a89e780ef810,,,agent,agent,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, +,7da9814d-a2e8-4c4e-b750-68b26bd5fd22,72dfbba4-0672-4adc-a1bf-308157d831e6,,,agent,agent,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, +,16143f74-2831-4753-b33d-ce4b645093c5,92c81ee2-728d-4ea6-bc52-b81295d2aea2,,,agent,agent,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, +,fc64e6cc-6739-4256-ac4a-7b80c3028233,807627b0-41d9-478f-8a8b-26540d70af68,,,agent,agent,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, +,b7945c49-f584-4c70-972d-536a805d8a31,bf860672-af22-44c1-b13b-0eabe8f72503,,,agent,agent,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, +,f5c40ecf-36c0-45ba-9cc9-dc0329b0324b,5fa714f1-eeb9-4617-bc7e-67062e6d1eab,,,agent,agent,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, diff --git a/noveum_customer_support_bt/processed_datasets/agent.routing_evaluation_metrics_dataset_processed_dataset.json b/noveum_customer_support_bt/processed_datasets/agent.routing_evaluation_metrics_dataset_processed_dataset.json new file mode 100644 index 0000000..295df2d --- /dev/null +++ b/noveum_customer_support_bt/processed_datasets/agent.routing_evaluation_metrics_dataset_processed_dataset.json @@ -0,0 +1,442 @@ +[ + { + "user_id": null, + "task_id": "eda4fe22-9a2b-4b73-856b-f4f3309bf719", + "turn_id": "95dac0c3-83fd-459c-af4e-c6a1734fd874", + "ground_truth": null, + "expected_tool_call": null, + "agent_name": "agent", + "agent_role": "agent", + "agent_task": null, + "system_prompt": "You are a helpful customer support agent", + "agent_response": null, + "trace": null, + "tools_available": [], + "tool_calls": [], + "parameters_passed": {}, + "tool_call_results": [], + "retrieval_query": null, + "retrieved_context": null, + "exit_status": "ok", + "agent_exit": false, + "metadata": null + }, + { + "user_id": null, + "task_id": "0ffffba1-8a37-443c-8866-d53ffbfa7718", + "turn_id": "516060a3-0f58-46ba-8a6a-4b502a376492", + "ground_truth": null, + "expected_tool_call": null, + "agent_name": "agent", + "agent_role": "agent", + "agent_task": null, + "system_prompt": "You are a helpful customer support agent", + "agent_response": null, + "trace": null, + "tools_available": [], + "tool_calls": [], + "parameters_passed": {}, + "tool_call_results": [], + "retrieval_query": null, + "retrieved_context": null, + "exit_status": "ok", + "agent_exit": false, + "metadata": null + }, + { + "user_id": null, + "task_id": "f1f37bd7-0851-4659-b493-b80d3800d920", + "turn_id": "8f8d2dc4-150f-4e43-aad3-96da4d06f586", + "ground_truth": null, + "expected_tool_call": null, + "agent_name": "agent", + "agent_role": "agent", + "agent_task": null, + "system_prompt": "You are a helpful customer support agent", + "agent_response": null, + "trace": null, + "tools_available": [], + "tool_calls": [], + "parameters_passed": {}, + "tool_call_results": [], + "retrieval_query": null, + "retrieved_context": null, + "exit_status": "ok", + "agent_exit": false, + "metadata": null + }, + { + "user_id": null, + "task_id": "43cdf081-4f01-49cd-b566-dbd1619e6cd2", + "turn_id": "cf0a915c-668b-412e-9c14-69119d059dc2", + "ground_truth": null, + "expected_tool_call": null, + "agent_name": "agent", + "agent_role": "agent", + "agent_task": null, + "system_prompt": "You are a helpful customer support agent", + "agent_response": null, + "trace": null, + "tools_available": [], + "tool_calls": [], + "parameters_passed": {}, + "tool_call_results": [], + "retrieval_query": null, + "retrieved_context": null, + "exit_status": "ok", + "agent_exit": false, + "metadata": null + }, + { + "user_id": null, + "task_id": "9a1983f4-09da-4b53-80e6-38de6878e0e7", + "turn_id": "f75b51b8-4882-4170-bc08-2adcab250f50", + "ground_truth": null, + "expected_tool_call": null, + "agent_name": "agent", + "agent_role": "agent", + "agent_task": null, + "system_prompt": "You are a helpful customer support agent", + "agent_response": null, + "trace": null, + "tools_available": [], + "tool_calls": [], + "parameters_passed": {}, + "tool_call_results": [], + "retrieval_query": null, + "retrieved_context": null, + "exit_status": "ok", + "agent_exit": false, + "metadata": null + }, + { + "user_id": null, + "task_id": "5d2517e1-220a-429d-9d59-f701bda25eed", + "turn_id": "4180abcf-924c-45ac-8dfa-17c0bc603eff", + "ground_truth": null, + "expected_tool_call": null, + "agent_name": "agent", + "agent_role": "agent", + "agent_task": null, + "system_prompt": "You are a helpful customer support agent", + "agent_response": null, + "trace": null, + "tools_available": [], + "tool_calls": [], + "parameters_passed": {}, + "tool_call_results": [], + "retrieval_query": null, + "retrieved_context": null, + "exit_status": "ok", + "agent_exit": false, + "metadata": null + }, + { + "user_id": null, + "task_id": "52aacb67-c361-4445-9b72-c157f79f47d6", + "turn_id": "8b0022cc-91dd-4065-91ed-b77217f92bd7", + "ground_truth": null, + "expected_tool_call": null, + "agent_name": "agent", + "agent_role": "agent", + "agent_task": null, + "system_prompt": "You are a helpful customer support agent", + "agent_response": null, + "trace": null, + "tools_available": [], + "tool_calls": [], + "parameters_passed": {}, + "tool_call_results": [], + "retrieval_query": null, + "retrieved_context": null, + "exit_status": "ok", + "agent_exit": false, + "metadata": null + }, + { + "user_id": null, + "task_id": "a81ca3a8-80aa-4c39-876e-8d40ea7a0aef", + "turn_id": "780df186-fe5a-4742-b937-2a142ad528c6", + "ground_truth": null, + "expected_tool_call": null, + "agent_name": "agent", + "agent_role": "agent", + "agent_task": null, + "system_prompt": "You are a helpful customer support agent", + "agent_response": null, + "trace": null, + "tools_available": [], + "tool_calls": [], + "parameters_passed": {}, + "tool_call_results": [], + "retrieval_query": null, + "retrieved_context": null, + "exit_status": "ok", + "agent_exit": false, + "metadata": null + }, + { + "user_id": null, + "task_id": "230aad27-f3dd-4968-a45a-3c2f07ac28ed", + "turn_id": "f80ae3e8-b4dd-42c6-b17b-de7805cedbb9", + "ground_truth": null, + "expected_tool_call": null, + "agent_name": "agent", + "agent_role": "agent", + "agent_task": null, + "system_prompt": "You are a helpful customer support agent", + "agent_response": null, + "trace": null, + "tools_available": [], + "tool_calls": [], + "parameters_passed": {}, + "tool_call_results": [], + "retrieval_query": null, + "retrieved_context": null, + "exit_status": "ok", + "agent_exit": false, + "metadata": null + }, + { + "user_id": null, + "task_id": "83c7dcce-3d89-4da1-8b3f-d419885d4cbc", + "turn_id": "209259cf-411d-4b97-8f7b-40b2530eec81", + "ground_truth": null, + "expected_tool_call": null, + "agent_name": "agent", + "agent_role": "agent", + "agent_task": null, + "system_prompt": "You are a helpful customer support agent", + "agent_response": null, + "trace": null, + "tools_available": [], + "tool_calls": [], + "parameters_passed": {}, + "tool_call_results": [], + "retrieval_query": null, + "retrieved_context": null, + "exit_status": "ok", + "agent_exit": false, + "metadata": null + }, + { + "user_id": null, + "task_id": "2218f641-604c-491a-9710-b51a9941b982", + "turn_id": "c3f3bd32-9954-443f-93e5-52aa6cf412a0", + "ground_truth": null, + "expected_tool_call": null, + "agent_name": "agent", + "agent_role": "agent", + "agent_task": null, + "system_prompt": "You are a helpful customer support agent", + "agent_response": null, + "trace": null, + "tools_available": [], + "tool_calls": [], + "parameters_passed": {}, + "tool_call_results": [], + "retrieval_query": null, + "retrieved_context": null, + "exit_status": "ok", + "agent_exit": false, + "metadata": null + }, + { + "user_id": null, + "task_id": "255fd49c-84b4-4b18-887e-6308a412d535", + "turn_id": "940ce621-0015-45ed-adc6-c4caa2a95e81", + "ground_truth": null, + "expected_tool_call": null, + "agent_name": "agent", + "agent_role": "agent", + "agent_task": null, + "system_prompt": "You are a helpful customer support agent", + "agent_response": null, + "trace": null, + "tools_available": [], + "tool_calls": [], + "parameters_passed": {}, + "tool_call_results": [], + "retrieval_query": null, + "retrieved_context": null, + "exit_status": "ok", + "agent_exit": false, + "metadata": null + }, + { + "user_id": null, + "task_id": "dc511122-c0b6-415c-9a49-c7b45132dd87", + "turn_id": "ea28f049-6bc9-48b4-8386-279abe087ca8", + "ground_truth": null, + "expected_tool_call": null, + "agent_name": "agent", + "agent_role": "agent", + "agent_task": null, + "system_prompt": "You are a helpful customer support agent", + "agent_response": null, + "trace": null, + "tools_available": [], + "tool_calls": [], + "parameters_passed": {}, + "tool_call_results": [], + "retrieval_query": null, + "retrieved_context": null, + "exit_status": "ok", + "agent_exit": false, + "metadata": null + }, + { + "user_id": null, + "task_id": "04bebf38-a343-4563-80db-0154bef8d927", + "turn_id": "01a03da5-64df-41e3-b9c9-556e12a8edd7", + "ground_truth": null, + "expected_tool_call": null, + "agent_name": "agent", + "agent_role": "agent", + "agent_task": null, + "system_prompt": "You are a helpful customer support agent", + "agent_response": null, + "trace": null, + "tools_available": [], + "tool_calls": [], + "parameters_passed": {}, + "tool_call_results": [], + "retrieval_query": null, + "retrieved_context": null, + "exit_status": "ok", + "agent_exit": false, + "metadata": null + }, + { + "user_id": null, + "task_id": "5e043630-6493-42b5-beb8-79faa19bfa37", + "turn_id": "8d9c632b-b2e8-496d-b1b0-a89e780ef810", + "ground_truth": null, + "expected_tool_call": null, + "agent_name": "agent", + "agent_role": "agent", + "agent_task": null, + "system_prompt": "You are a helpful customer support agent", + "agent_response": null, + "trace": null, + "tools_available": [], + "tool_calls": [], + "parameters_passed": {}, + "tool_call_results": [], + "retrieval_query": null, + "retrieved_context": null, + "exit_status": "ok", + "agent_exit": false, + "metadata": null + }, + { + "user_id": null, + "task_id": "7da9814d-a2e8-4c4e-b750-68b26bd5fd22", + "turn_id": "72dfbba4-0672-4adc-a1bf-308157d831e6", + "ground_truth": null, + "expected_tool_call": null, + "agent_name": "agent", + "agent_role": "agent", + "agent_task": null, + "system_prompt": "You are a helpful customer support agent", + "agent_response": null, + "trace": null, + "tools_available": [], + "tool_calls": [], + "parameters_passed": {}, + "tool_call_results": [], + "retrieval_query": null, + "retrieved_context": null, + "exit_status": "ok", + "agent_exit": false, + "metadata": null + }, + { + "user_id": null, + "task_id": "16143f74-2831-4753-b33d-ce4b645093c5", + "turn_id": "92c81ee2-728d-4ea6-bc52-b81295d2aea2", + "ground_truth": null, + "expected_tool_call": null, + "agent_name": "agent", + "agent_role": "agent", + "agent_task": null, + "system_prompt": "You are a helpful customer support agent", + "agent_response": null, + "trace": null, + "tools_available": [], + "tool_calls": [], + "parameters_passed": {}, + "tool_call_results": [], + "retrieval_query": null, + "retrieved_context": null, + "exit_status": "ok", + "agent_exit": false, + "metadata": null + }, + { + "user_id": null, + "task_id": "fc64e6cc-6739-4256-ac4a-7b80c3028233", + "turn_id": "807627b0-41d9-478f-8a8b-26540d70af68", + "ground_truth": null, + "expected_tool_call": null, + "agent_name": "agent", + "agent_role": "agent", + "agent_task": null, + "system_prompt": "You are a helpful customer support agent", + "agent_response": null, + "trace": null, + "tools_available": [], + "tool_calls": [], + "parameters_passed": {}, + "tool_call_results": [], + "retrieval_query": null, + "retrieved_context": null, + "exit_status": "ok", + "agent_exit": false, + "metadata": null + }, + { + "user_id": null, + "task_id": "b7945c49-f584-4c70-972d-536a805d8a31", + "turn_id": "bf860672-af22-44c1-b13b-0eabe8f72503", + "ground_truth": null, + "expected_tool_call": null, + "agent_name": "agent", + "agent_role": "agent", + "agent_task": null, + "system_prompt": "You are a helpful customer support agent", + "agent_response": null, + "trace": null, + "tools_available": [], + "tool_calls": [], + "parameters_passed": {}, + "tool_call_results": [], + "retrieval_query": null, + "retrieved_context": null, + "exit_status": "ok", + "agent_exit": false, + "metadata": null + }, + { + "user_id": null, + "task_id": "f5c40ecf-36c0-45ba-9cc9-dc0329b0324b", + "turn_id": "5fa714f1-eeb9-4617-bc7e-67062e6d1eab", + "ground_truth": null, + "expected_tool_call": null, + "agent_name": "agent", + "agent_role": "agent", + "agent_task": null, + "system_prompt": "You are a helpful customer support agent", + "agent_response": null, + "trace": null, + "tools_available": [], + "tool_calls": [], + "parameters_passed": {}, + "tool_call_results": [], + "retrieval_query": null, + "retrieved_context": null, + "exit_status": "ok", + "agent_exit": false, + "metadata": null + } +] \ No newline at end of file diff --git a/noveum_customer_support_bt/processed_datasets/agent.web_search_evaluation_metrics_dataset_processed_dataset.csv b/noveum_customer_support_bt/processed_datasets/agent.web_search_evaluation_metrics_dataset_processed_dataset.csv new file mode 100644 index 0000000..d415212 --- /dev/null +++ b/noveum_customer_support_bt/processed_datasets/agent.web_search_evaluation_metrics_dataset_processed_dataset.csv @@ -0,0 +1,14 @@ +user_id,task_id,turn_id,ground_truth,expected_tool_call,agent_name,agent_role,agent_task,system_prompt,agent_response,trace,tools_available,tool_calls,parameters_passed,tool_call_results,retrieval_query,retrieved_context,exit_status,agent_exit,metadata +,eda4fe22-9a2b-4b73-856b-f4f3309bf719,3b7b6838-22d4-4edf-9b00-7cb011f18a52,,,agent,agent,Evaluation for query: What are the latest breakthroughs in artificial intelligence?,You are a helpful customer support agent,"Web Search Answer: The provided web search results do not contain specific information about the latest breakthroughs in artificial intelligence. To find the most recent advancements in AI, I recommend checking reputabl...",,"[{""name"": ""web_search_evaluation"", ""description"": ""Evaluates web search performance"", ""args_schema"": {}, ""return_schema"": {""type"": ""evaluation_result""}}]",[],{},[],"[""Evaluation for query: What are the latest breakthroughs in artificial intelligence?""]","[[""""]]",ok,False, +,0ffffba1-8a37-443c-8866-d53ffbfa7718,a5d480f0-ea5b-4190-a3c3-f94e26b48573,,,agent,agent,Evaluation for query: What's happening in the software development world today?,You are a helpful customer support agent,"Web Search Answer: The web search did not yield any specific results regarding current events in the software development world. Therefore, I cannot provide detailed information on what's happening today in that field. +...",,"[{""name"": ""web_search_evaluation"", ""description"": ""Evaluates web search performance"", ""args_schema"": {}, ""return_schema"": {""type"": ""evaluation_result""}}]",[],{},[],"[""Evaluation for query: What's happening in the software development world today?""]","[[""""]]",ok,False, +,43cdf081-4f01-49cd-b566-dbd1619e6cd2,2fd285de-68a1-4b7a-a4ed-05f539afa737,,,agent,agent,Evaluation for query: What's the current status of cryptocurrency markets?,You are a helpful customer support agent,"Web Search Answer: The web search results did not provide any specific information regarding the current status of cryptocurrency markets. Therefore, I cannot provide an accurate update on market conditions, trends, or ...",,"[{""name"": ""web_search_evaluation"", ""description"": ""Evaluates web search performance"", ""args_schema"": {}, ""return_schema"": {""type"": ""evaluation_result""}}]",[],{},[],"[""Evaluation for query: What's the current status of cryptocurrency markets?""]","[[""""]]",ok,False, +,9a1983f4-09da-4b53-80e6-38de6878e0e7,48deb050-478d-486c-be3a-b8c0069d7abe,,,agent,agent,Evaluation for query: What are the latest updates in Python programming?,You are a helpful customer support agent,"Web Search Answer: The web search results do not provide specific information about the latest updates in Python programming. They focus primarily on news related to China, including geopolitics, economy, and lifestyle,...",,"[{""name"": ""web_search_evaluation"", ""description"": ""Evaluates web search performance"", ""args_schema"": {}, ""return_schema"": {""type"": ""evaluation_result""}}]",[],{},[],"[""Evaluation for query: What are the latest updates in Python programming?""]","[[""""]]",ok,False, +,5d2517e1-220a-429d-9d59-f701bda25eed,4ecede78-e7fc-48ed-9fba-1241d6890a1d,,,agent,agent,Evaluation for query: What happened in tech news this week?,You are a helpful customer support agent,"Web Search Answer: The web search results do not provide specific information about recent events in tech news for this week. They primarily discuss the grammatical usage of the phrase ""what happened"" and its variations...",,"[{""name"": ""web_search_evaluation"", ""description"": ""Evaluates web search performance"", ""args_schema"": {}, ""return_schema"": {""type"": ""evaluation_result""}}]",[],{},[],"[""Evaluation for query: What happened in tech news this week?""]","[[""""]]",ok,False, +,a81ca3a8-80aa-4c39-876e-8d40ea7a0aef,91fdf43c-7170-498e-9035-1e2b11b32b48,,,agent,agent,Evaluation for query: Tell me about recent developments in machine learning,You are a helpful customer support agent,"Web Search Answer: The provided web search results do not contain specific information about recent developments in machine learning. Therefore, I cannot provide a detailed answer based on those results. + +However, I can...",,"[{""name"": ""web_search_evaluation"", ""description"": ""Evaluates web search performance"", ""args_schema"": {}, ""return_schema"": {""type"": ""evaluation_result""}}]",[],{},[],"[""Evaluation for query: Tell me about recent developments in machine learning""]","[[""""]]",ok,False, +,230aad27-f3dd-4968-a45a-3c2f07ac28ed,096e6592-23fa-432a-931d-8f39656c7fcc,,,agent,agent,Evaluation for query: What's the weather like today?,You are a helpful customer support agent,"Web Search Answer: The provided web search results do not contain specific information about today's weather in any particular location. They mention weather forecasts for various places, including France and Singapore,...",,"[{""name"": ""web_search_evaluation"", ""description"": ""Evaluates web search performance"", ""args_schema"": {}, ""return_schema"": {""type"": ""evaluation_result""}}]",[],{},[],"[""Evaluation for query: What's the weather like today?""]","[[""""]]",ok,False, +,83c7dcce-3d89-4da1-8b3f-d419885d4cbc,b56248e6-e526-4000-8c10-f5d13cf2e12d,,,agent,agent,Evaluation for query: What are the latest AI news today?,You are a helpful customer support agent,"Web Search Answer: The latest news in artificial intelligence includes several key updates: + +1. **AI Spending Concerns**: Jeff Bezos has expressed concerns that the current level of investment in AI resembles an ""indust...",,"[{""name"": ""web_search_evaluation"", ""description"": ""Evaluates web search performance"", ""args_schema"": {}, ""return_schema"": {""type"": ""evaluation_result""}}]",[],{},[],"[""Evaluation for query: What are the latest AI news today?""]","[[""""]]",ok,False, diff --git a/noveum_customer_support_bt/processed_datasets/agent.web_search_evaluation_metrics_dataset_processed_dataset.json b/noveum_customer_support_bt/processed_datasets/agent.web_search_evaluation_metrics_dataset_processed_dataset.json new file mode 100644 index 0000000..ada83b2 --- /dev/null +++ b/noveum_customer_support_bt/processed_datasets/agent.web_search_evaluation_metrics_dataset_processed_dataset.json @@ -0,0 +1,298 @@ +[ + { + "user_id": null, + "task_id": "eda4fe22-9a2b-4b73-856b-f4f3309bf719", + "turn_id": "3b7b6838-22d4-4edf-9b00-7cb011f18a52", + "ground_truth": null, + "expected_tool_call": null, + "agent_name": "agent", + "agent_role": "agent", + "agent_task": "Evaluation for query: What are the latest breakthroughs in artificial intelligence?", + "system_prompt": "You are a helpful customer support agent", + "agent_response": "Web Search Answer: The provided web search results do not contain specific information about the latest breakthroughs in artificial intelligence. To find the most recent advancements in AI, I recommend checking reputabl...", + "trace": null, + "tools_available": [ + { + "name": "web_search_evaluation", + "description": "Evaluates web search performance", + "args_schema": {}, + "return_schema": { + "type": "evaluation_result" + } + } + ], + "tool_calls": [], + "parameters_passed": {}, + "tool_call_results": [], + "retrieval_query": [ + "Evaluation for query: What are the latest breakthroughs in artificial intelligence?" + ], + "retrieved_context": [ + [ + "" + ] + ], + "exit_status": "ok", + "agent_exit": false, + "metadata": null + }, + { + "user_id": null, + "task_id": "0ffffba1-8a37-443c-8866-d53ffbfa7718", + "turn_id": "a5d480f0-ea5b-4190-a3c3-f94e26b48573", + "ground_truth": null, + "expected_tool_call": null, + "agent_name": "agent", + "agent_role": "agent", + "agent_task": "Evaluation for query: What's happening in the software development world today?", + "system_prompt": "You are a helpful customer support agent", + "agent_response": "Web Search Answer: The web search did not yield any specific results regarding current events in the software development world. Therefore, I cannot provide detailed information on what's happening today in that field.\n...", + "trace": null, + "tools_available": [ + { + "name": "web_search_evaluation", + "description": "Evaluates web search performance", + "args_schema": {}, + "return_schema": { + "type": "evaluation_result" + } + } + ], + "tool_calls": [], + "parameters_passed": {}, + "tool_call_results": [], + "retrieval_query": [ + "Evaluation for query: What's happening in the software development world today?" + ], + "retrieved_context": [ + [ + "" + ] + ], + "exit_status": "ok", + "agent_exit": false, + "metadata": null + }, + { + "user_id": null, + "task_id": "43cdf081-4f01-49cd-b566-dbd1619e6cd2", + "turn_id": "2fd285de-68a1-4b7a-a4ed-05f539afa737", + "ground_truth": null, + "expected_tool_call": null, + "agent_name": "agent", + "agent_role": "agent", + "agent_task": "Evaluation for query: What's the current status of cryptocurrency markets?", + "system_prompt": "You are a helpful customer support agent", + "agent_response": "Web Search Answer: The web search results did not provide any specific information regarding the current status of cryptocurrency markets. Therefore, I cannot provide an accurate update on market conditions, trends, or ...", + "trace": null, + "tools_available": [ + { + "name": "web_search_evaluation", + "description": "Evaluates web search performance", + "args_schema": {}, + "return_schema": { + "type": "evaluation_result" + } + } + ], + "tool_calls": [], + "parameters_passed": {}, + "tool_call_results": [], + "retrieval_query": [ + "Evaluation for query: What's the current status of cryptocurrency markets?" + ], + "retrieved_context": [ + [ + "" + ] + ], + "exit_status": "ok", + "agent_exit": false, + "metadata": null + }, + { + "user_id": null, + "task_id": "9a1983f4-09da-4b53-80e6-38de6878e0e7", + "turn_id": "48deb050-478d-486c-be3a-b8c0069d7abe", + "ground_truth": null, + "expected_tool_call": null, + "agent_name": "agent", + "agent_role": "agent", + "agent_task": "Evaluation for query: What are the latest updates in Python programming?", + "system_prompt": "You are a helpful customer support agent", + "agent_response": "Web Search Answer: The web search results do not provide specific information about the latest updates in Python programming. They focus primarily on news related to China, including geopolitics, economy, and lifestyle,...", + "trace": null, + "tools_available": [ + { + "name": "web_search_evaluation", + "description": "Evaluates web search performance", + "args_schema": {}, + "return_schema": { + "type": "evaluation_result" + } + } + ], + "tool_calls": [], + "parameters_passed": {}, + "tool_call_results": [], + "retrieval_query": [ + "Evaluation for query: What are the latest updates in Python programming?" + ], + "retrieved_context": [ + [ + "" + ] + ], + "exit_status": "ok", + "agent_exit": false, + "metadata": null + }, + { + "user_id": null, + "task_id": "5d2517e1-220a-429d-9d59-f701bda25eed", + "turn_id": "4ecede78-e7fc-48ed-9fba-1241d6890a1d", + "ground_truth": null, + "expected_tool_call": null, + "agent_name": "agent", + "agent_role": "agent", + "agent_task": "Evaluation for query: What happened in tech news this week?", + "system_prompt": "You are a helpful customer support agent", + "agent_response": "Web Search Answer: The web search results do not provide specific information about recent events in tech news for this week. They primarily discuss the grammatical usage of the phrase \"what happened\" and its variations...", + "trace": null, + "tools_available": [ + { + "name": "web_search_evaluation", + "description": "Evaluates web search performance", + "args_schema": {}, + "return_schema": { + "type": "evaluation_result" + } + } + ], + "tool_calls": [], + "parameters_passed": {}, + "tool_call_results": [], + "retrieval_query": [ + "Evaluation for query: What happened in tech news this week?" + ], + "retrieved_context": [ + [ + "" + ] + ], + "exit_status": "ok", + "agent_exit": false, + "metadata": null + }, + { + "user_id": null, + "task_id": "a81ca3a8-80aa-4c39-876e-8d40ea7a0aef", + "turn_id": "91fdf43c-7170-498e-9035-1e2b11b32b48", + "ground_truth": null, + "expected_tool_call": null, + "agent_name": "agent", + "agent_role": "agent", + "agent_task": "Evaluation for query: Tell me about recent developments in machine learning", + "system_prompt": "You are a helpful customer support agent", + "agent_response": "Web Search Answer: The provided web search results do not contain specific information about recent developments in machine learning. Therefore, I cannot provide a detailed answer based on those results.\n\nHowever, I can...", + "trace": null, + "tools_available": [ + { + "name": "web_search_evaluation", + "description": "Evaluates web search performance", + "args_schema": {}, + "return_schema": { + "type": "evaluation_result" + } + } + ], + "tool_calls": [], + "parameters_passed": {}, + "tool_call_results": [], + "retrieval_query": [ + "Evaluation for query: Tell me about recent developments in machine learning" + ], + "retrieved_context": [ + [ + "" + ] + ], + "exit_status": "ok", + "agent_exit": false, + "metadata": null + }, + { + "user_id": null, + "task_id": "230aad27-f3dd-4968-a45a-3c2f07ac28ed", + "turn_id": "096e6592-23fa-432a-931d-8f39656c7fcc", + "ground_truth": null, + "expected_tool_call": null, + "agent_name": "agent", + "agent_role": "agent", + "agent_task": "Evaluation for query: What's the weather like today?", + "system_prompt": "You are a helpful customer support agent", + "agent_response": "Web Search Answer: The provided web search results do not contain specific information about today's weather in any particular location. They mention weather forecasts for various places, including France and Singapore,...", + "trace": null, + "tools_available": [ + { + "name": "web_search_evaluation", + "description": "Evaluates web search performance", + "args_schema": {}, + "return_schema": { + "type": "evaluation_result" + } + } + ], + "tool_calls": [], + "parameters_passed": {}, + "tool_call_results": [], + "retrieval_query": [ + "Evaluation for query: What's the weather like today?" + ], + "retrieved_context": [ + [ + "" + ] + ], + "exit_status": "ok", + "agent_exit": false, + "metadata": null + }, + { + "user_id": null, + "task_id": "83c7dcce-3d89-4da1-8b3f-d419885d4cbc", + "turn_id": "b56248e6-e526-4000-8c10-f5d13cf2e12d", + "ground_truth": null, + "expected_tool_call": null, + "agent_name": "agent", + "agent_role": "agent", + "agent_task": "Evaluation for query: What are the latest AI news today?", + "system_prompt": "You are a helpful customer support agent", + "agent_response": "Web Search Answer: The latest news in artificial intelligence includes several key updates:\n\n1. **AI Spending Concerns**: Jeff Bezos has expressed concerns that the current level of investment in AI resembles an \"indust...", + "trace": null, + "tools_available": [ + { + "name": "web_search_evaluation", + "description": "Evaluates web search performance", + "args_schema": {}, + "return_schema": { + "type": "evaluation_result" + } + } + ], + "tool_calls": [], + "parameters_passed": {}, + "tool_call_results": [], + "retrieval_query": [ + "Evaluation for query: What are the latest AI news today?" + ], + "retrieved_context": [ + [ + "" + ] + ], + "exit_status": "ok", + "agent_exit": false, + "metadata": null + } +] \ No newline at end of file diff --git a/noveum_customer_support_bt/processed_datasets/agent.web_search_generation_dataset_processed_dataset.csv b/noveum_customer_support_bt/processed_datasets/agent.web_search_generation_dataset_processed_dataset.csv new file mode 100644 index 0000000..f3c8e35 --- /dev/null +++ b/noveum_customer_support_bt/processed_datasets/agent.web_search_generation_dataset_processed_dataset.csv @@ -0,0 +1,9 @@ +user_id,task_id,turn_id,ground_truth,expected_tool_call,agent_name,agent_role,agent_task,system_prompt,agent_response,trace,tools_available,tool_calls,parameters_passed,tool_call_results,retrieval_query,retrieved_context,exit_status,agent_exit,metadata +,eda4fe22-9a2b-4b73-856b-f4f3309bf719,2aca614b-b016-4d56-abe2-43a144dc681d,,,agent,agent,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, +,0ffffba1-8a37-443c-8866-d53ffbfa7718,03e3ae4d-0107-401d-8ca0-05fc0f28fb24,,,agent,agent,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, +,43cdf081-4f01-49cd-b566-dbd1619e6cd2,dfdb9aaf-9925-4a5c-a0c6-2cb00f70d4d3,,,agent,agent,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, +,9a1983f4-09da-4b53-80e6-38de6878e0e7,eefcfdfc-6433-459e-a134-e2647b8418e4,,,agent,agent,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, +,5d2517e1-220a-429d-9d59-f701bda25eed,c57b4b54-b9b0-4a4c-b5cd-57a4cfe5301b,,,agent,agent,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, +,a81ca3a8-80aa-4c39-876e-8d40ea7a0aef,a062bef6-d973-489b-91df-5d40a2aef077,,,agent,agent,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, +,230aad27-f3dd-4968-a45a-3c2f07ac28ed,41f82a95-620d-440c-84e6-36d6d1b9e079,,,agent,agent,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, +,83c7dcce-3d89-4da1-8b3f-d419885d4cbc,63f2a0d5-9016-4483-8ec0-e6aa6b27689f,,,agent,agent,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, diff --git a/noveum_customer_support_bt/processed_datasets/agent.web_search_generation_dataset_processed_dataset.json b/noveum_customer_support_bt/processed_datasets/agent.web_search_generation_dataset_processed_dataset.json new file mode 100644 index 0000000..105a46c --- /dev/null +++ b/noveum_customer_support_bt/processed_datasets/agent.web_search_generation_dataset_processed_dataset.json @@ -0,0 +1,178 @@ +[ + { + "user_id": null, + "task_id": "eda4fe22-9a2b-4b73-856b-f4f3309bf719", + "turn_id": "2aca614b-b016-4d56-abe2-43a144dc681d", + "ground_truth": null, + "expected_tool_call": null, + "agent_name": "agent", + "agent_role": "agent", + "agent_task": null, + "system_prompt": "You are a helpful customer support agent", + "agent_response": null, + "trace": null, + "tools_available": [], + "tool_calls": [], + "parameters_passed": {}, + "tool_call_results": [], + "retrieval_query": null, + "retrieved_context": null, + "exit_status": "ok", + "agent_exit": false, + "metadata": null + }, + { + "user_id": null, + "task_id": "0ffffba1-8a37-443c-8866-d53ffbfa7718", + "turn_id": "03e3ae4d-0107-401d-8ca0-05fc0f28fb24", + "ground_truth": null, + "expected_tool_call": null, + "agent_name": "agent", + "agent_role": "agent", + "agent_task": null, + "system_prompt": "You are a helpful customer support agent", + "agent_response": null, + "trace": null, + "tools_available": [], + "tool_calls": [], + "parameters_passed": {}, + "tool_call_results": [], + "retrieval_query": null, + "retrieved_context": null, + "exit_status": "ok", + "agent_exit": false, + "metadata": null + }, + { + "user_id": null, + "task_id": "43cdf081-4f01-49cd-b566-dbd1619e6cd2", + "turn_id": "dfdb9aaf-9925-4a5c-a0c6-2cb00f70d4d3", + "ground_truth": null, + "expected_tool_call": null, + "agent_name": "agent", + "agent_role": "agent", + "agent_task": null, + "system_prompt": "You are a helpful customer support agent", + "agent_response": null, + "trace": null, + "tools_available": [], + "tool_calls": [], + "parameters_passed": {}, + "tool_call_results": [], + "retrieval_query": null, + "retrieved_context": null, + "exit_status": "ok", + "agent_exit": false, + "metadata": null + }, + { + "user_id": null, + "task_id": "9a1983f4-09da-4b53-80e6-38de6878e0e7", + "turn_id": "eefcfdfc-6433-459e-a134-e2647b8418e4", + "ground_truth": null, + "expected_tool_call": null, + "agent_name": "agent", + "agent_role": "agent", + "agent_task": null, + "system_prompt": "You are a helpful customer support agent", + "agent_response": null, + "trace": null, + "tools_available": [], + "tool_calls": [], + "parameters_passed": {}, + "tool_call_results": [], + "retrieval_query": null, + "retrieved_context": null, + "exit_status": "ok", + "agent_exit": false, + "metadata": null + }, + { + "user_id": null, + "task_id": "5d2517e1-220a-429d-9d59-f701bda25eed", + "turn_id": "c57b4b54-b9b0-4a4c-b5cd-57a4cfe5301b", + "ground_truth": null, + "expected_tool_call": null, + "agent_name": "agent", + "agent_role": "agent", + "agent_task": null, + "system_prompt": "You are a helpful customer support agent", + "agent_response": null, + "trace": null, + "tools_available": [], + "tool_calls": [], + "parameters_passed": {}, + "tool_call_results": [], + "retrieval_query": null, + "retrieved_context": null, + "exit_status": "ok", + "agent_exit": false, + "metadata": null + }, + { + "user_id": null, + "task_id": "a81ca3a8-80aa-4c39-876e-8d40ea7a0aef", + "turn_id": "a062bef6-d973-489b-91df-5d40a2aef077", + "ground_truth": null, + "expected_tool_call": null, + "agent_name": "agent", + "agent_role": "agent", + "agent_task": null, + "system_prompt": "You are a helpful customer support agent", + "agent_response": null, + "trace": null, + "tools_available": [], + "tool_calls": [], + "parameters_passed": {}, + "tool_call_results": [], + "retrieval_query": null, + "retrieved_context": null, + "exit_status": "ok", + "agent_exit": false, + "metadata": null + }, + { + "user_id": null, + "task_id": "230aad27-f3dd-4968-a45a-3c2f07ac28ed", + "turn_id": "41f82a95-620d-440c-84e6-36d6d1b9e079", + "ground_truth": null, + "expected_tool_call": null, + "agent_name": "agent", + "agent_role": "agent", + "agent_task": null, + "system_prompt": "You are a helpful customer support agent", + "agent_response": null, + "trace": null, + "tools_available": [], + "tool_calls": [], + "parameters_passed": {}, + "tool_call_results": [], + "retrieval_query": null, + "retrieved_context": null, + "exit_status": "ok", + "agent_exit": false, + "metadata": null + }, + { + "user_id": null, + "task_id": "83c7dcce-3d89-4da1-8b3f-d419885d4cbc", + "turn_id": "63f2a0d5-9016-4483-8ec0-e6aa6b27689f", + "ground_truth": null, + "expected_tool_call": null, + "agent_name": "agent", + "agent_role": "agent", + "agent_task": null, + "system_prompt": "You are a helpful customer support agent", + "agent_response": null, + "trace": null, + "tools_available": [], + "tool_calls": [], + "parameters_passed": {}, + "tool_call_results": [], + "retrieval_query": null, + "retrieved_context": null, + "exit_status": "ok", + "agent_exit": false, + "metadata": null + } +] \ No newline at end of file diff --git a/noveum_customer_support_bt/processed_datasets/tool-orchestator_dataset_processed_dataset.csv b/noveum_customer_support_bt/processed_datasets/tool-orchestator_dataset_processed_dataset.csv new file mode 100644 index 0000000..a057016 --- /dev/null +++ b/noveum_customer_support_bt/processed_datasets/tool-orchestator_dataset_processed_dataset.csv @@ -0,0 +1,21 @@ +user_id,task_id,turn_id,ground_truth,expected_tool_call,agent_name,agent_role,agent_task,system_prompt,agent_response,trace,tools_available,tool_calls,parameters_passed,tool_call_results,retrieval_query,retrieved_context,exit_status,agent_exit,metadata +,eda4fe22-9a2b-4b73-856b-f4f3309bf719,b6b7c1c0-3b7b-4e19-805d-a16beb7ea871,,,unknown,unknown,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, +,0ffffba1-8a37-443c-8866-d53ffbfa7718,3045d971-b7ff-46cc-be11-6db7d65dfefd,,,unknown,unknown,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, +,f1f37bd7-0851-4659-b493-b80d3800d920,42d4cb1f-17ec-433a-8087-90292c2fa9d9,,,unknown,unknown,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, +,43cdf081-4f01-49cd-b566-dbd1619e6cd2,e22b809d-5320-40b7-b78a-e3e125538a6b,,,unknown,unknown,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, +,9a1983f4-09da-4b53-80e6-38de6878e0e7,813a4450-1cf8-4d43-8493-eabfa200919b,,,unknown,unknown,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, +,5d2517e1-220a-429d-9d59-f701bda25eed,6102352a-833e-493d-adbb-887e2af17918,,,unknown,unknown,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, +,52aacb67-c361-4445-9b72-c157f79f47d6,2926418a-fe09-4df0-bf19-a527bc24a478,,,unknown,unknown,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, +,a81ca3a8-80aa-4c39-876e-8d40ea7a0aef,afe85a1d-13c8-4476-8154-52d58ba83d52,,,unknown,unknown,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, +,230aad27-f3dd-4968-a45a-3c2f07ac28ed,db8da393-6d72-421e-87dc-96f56b6576e5,,,unknown,unknown,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, +,83c7dcce-3d89-4da1-8b3f-d419885d4cbc,a290202d-1b3f-431f-8728-fcb937380f8c,,,unknown,unknown,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, +,2218f641-604c-491a-9710-b51a9941b982,5fbaef83-95fc-4bed-9695-9d69c37c35e4,,,unknown,unknown,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, +,255fd49c-84b4-4b18-887e-6308a412d535,b3ac3924-928b-4d48-a2a7-aa749802fef9,,,unknown,unknown,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, +,dc511122-c0b6-415c-9a49-c7b45132dd87,b8055ffc-ee7d-46ad-b6a0-84e80f9399e2,,,unknown,unknown,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, +,04bebf38-a343-4563-80db-0154bef8d927,532b9e1f-5c4f-4b41-98bb-e9a1510f0f1c,,,unknown,unknown,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, +,5e043630-6493-42b5-beb8-79faa19bfa37,14064e3f-f9c9-4bb7-b8cb-1ad52938b075,,,unknown,unknown,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, +,7da9814d-a2e8-4c4e-b750-68b26bd5fd22,a74fe8a9-2654-4b9d-ba87-dc2f86b6466d,,,unknown,unknown,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, +,16143f74-2831-4753-b33d-ce4b645093c5,c700d0f4-f1f8-4339-aa42-a3060ae2ae11,,,unknown,unknown,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, +,fc64e6cc-6739-4256-ac4a-7b80c3028233,f7f5981f-d840-4265-8970-d319b876366d,,,unknown,unknown,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, +,b7945c49-f584-4c70-972d-536a805d8a31,0c7bbe2b-18af-4fdb-a5bf-4a91daa3bf50,,,unknown,unknown,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, +,f5c40ecf-36c0-45ba-9cc9-dc0329b0324b,dc869742-6797-4016-8061-37530fc7d433,,,unknown,unknown,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, diff --git a/noveum_customer_support_bt/processed_datasets/tool-orchestator_dataset_processed_dataset.json b/noveum_customer_support_bt/processed_datasets/tool-orchestator_dataset_processed_dataset.json new file mode 100644 index 0000000..5e81efd --- /dev/null +++ b/noveum_customer_support_bt/processed_datasets/tool-orchestator_dataset_processed_dataset.json @@ -0,0 +1,442 @@ +[ + { + "user_id": null, + "task_id": "eda4fe22-9a2b-4b73-856b-f4f3309bf719", + "turn_id": "b6b7c1c0-3b7b-4e19-805d-a16beb7ea871", + "ground_truth": null, + "expected_tool_call": null, + "agent_name": "unknown", + "agent_role": "unknown", + "agent_task": null, + "system_prompt": "You are a helpful customer support agent", + "agent_response": null, + "trace": null, + "tools_available": [], + "tool_calls": [], + "parameters_passed": {}, + "tool_call_results": [], + "retrieval_query": null, + "retrieved_context": null, + "exit_status": "ok", + "agent_exit": false, + "metadata": null + }, + { + "user_id": null, + "task_id": "0ffffba1-8a37-443c-8866-d53ffbfa7718", + "turn_id": "3045d971-b7ff-46cc-be11-6db7d65dfefd", + "ground_truth": null, + "expected_tool_call": null, + "agent_name": "unknown", + "agent_role": "unknown", + "agent_task": null, + "system_prompt": "You are a helpful customer support agent", + "agent_response": null, + "trace": null, + "tools_available": [], + "tool_calls": [], + "parameters_passed": {}, + "tool_call_results": [], + "retrieval_query": null, + "retrieved_context": null, + "exit_status": "ok", + "agent_exit": false, + "metadata": null + }, + { + "user_id": null, + "task_id": "f1f37bd7-0851-4659-b493-b80d3800d920", + "turn_id": "42d4cb1f-17ec-433a-8087-90292c2fa9d9", + "ground_truth": null, + "expected_tool_call": null, + "agent_name": "unknown", + "agent_role": "unknown", + "agent_task": null, + "system_prompt": "You are a helpful customer support agent", + "agent_response": null, + "trace": null, + "tools_available": [], + "tool_calls": [], + "parameters_passed": {}, + "tool_call_results": [], + "retrieval_query": null, + "retrieved_context": null, + "exit_status": "ok", + "agent_exit": false, + "metadata": null + }, + { + "user_id": null, + "task_id": "43cdf081-4f01-49cd-b566-dbd1619e6cd2", + "turn_id": "e22b809d-5320-40b7-b78a-e3e125538a6b", + "ground_truth": null, + "expected_tool_call": null, + "agent_name": "unknown", + "agent_role": "unknown", + "agent_task": null, + "system_prompt": "You are a helpful customer support agent", + "agent_response": null, + "trace": null, + "tools_available": [], + "tool_calls": [], + "parameters_passed": {}, + "tool_call_results": [], + "retrieval_query": null, + "retrieved_context": null, + "exit_status": "ok", + "agent_exit": false, + "metadata": null + }, + { + "user_id": null, + "task_id": "9a1983f4-09da-4b53-80e6-38de6878e0e7", + "turn_id": "813a4450-1cf8-4d43-8493-eabfa200919b", + "ground_truth": null, + "expected_tool_call": null, + "agent_name": "unknown", + "agent_role": "unknown", + "agent_task": null, + "system_prompt": "You are a helpful customer support agent", + "agent_response": null, + "trace": null, + "tools_available": [], + "tool_calls": [], + "parameters_passed": {}, + "tool_call_results": [], + "retrieval_query": null, + "retrieved_context": null, + "exit_status": "ok", + "agent_exit": false, + "metadata": null + }, + { + "user_id": null, + "task_id": "5d2517e1-220a-429d-9d59-f701bda25eed", + "turn_id": "6102352a-833e-493d-adbb-887e2af17918", + "ground_truth": null, + "expected_tool_call": null, + "agent_name": "unknown", + "agent_role": "unknown", + "agent_task": null, + "system_prompt": "You are a helpful customer support agent", + "agent_response": null, + "trace": null, + "tools_available": [], + "tool_calls": [], + "parameters_passed": {}, + "tool_call_results": [], + "retrieval_query": null, + "retrieved_context": null, + "exit_status": "ok", + "agent_exit": false, + "metadata": null + }, + { + "user_id": null, + "task_id": "52aacb67-c361-4445-9b72-c157f79f47d6", + "turn_id": "2926418a-fe09-4df0-bf19-a527bc24a478", + "ground_truth": null, + "expected_tool_call": null, + "agent_name": "unknown", + "agent_role": "unknown", + "agent_task": null, + "system_prompt": "You are a helpful customer support agent", + "agent_response": null, + "trace": null, + "tools_available": [], + "tool_calls": [], + "parameters_passed": {}, + "tool_call_results": [], + "retrieval_query": null, + "retrieved_context": null, + "exit_status": "ok", + "agent_exit": false, + "metadata": null + }, + { + "user_id": null, + "task_id": "a81ca3a8-80aa-4c39-876e-8d40ea7a0aef", + "turn_id": "afe85a1d-13c8-4476-8154-52d58ba83d52", + "ground_truth": null, + "expected_tool_call": null, + "agent_name": "unknown", + "agent_role": "unknown", + "agent_task": null, + "system_prompt": "You are a helpful customer support agent", + "agent_response": null, + "trace": null, + "tools_available": [], + "tool_calls": [], + "parameters_passed": {}, + "tool_call_results": [], + "retrieval_query": null, + "retrieved_context": null, + "exit_status": "ok", + "agent_exit": false, + "metadata": null + }, + { + "user_id": null, + "task_id": "230aad27-f3dd-4968-a45a-3c2f07ac28ed", + "turn_id": "db8da393-6d72-421e-87dc-96f56b6576e5", + "ground_truth": null, + "expected_tool_call": null, + "agent_name": "unknown", + "agent_role": "unknown", + "agent_task": null, + "system_prompt": "You are a helpful customer support agent", + "agent_response": null, + "trace": null, + "tools_available": [], + "tool_calls": [], + "parameters_passed": {}, + "tool_call_results": [], + "retrieval_query": null, + "retrieved_context": null, + "exit_status": "ok", + "agent_exit": false, + "metadata": null + }, + { + "user_id": null, + "task_id": "83c7dcce-3d89-4da1-8b3f-d419885d4cbc", + "turn_id": "a290202d-1b3f-431f-8728-fcb937380f8c", + "ground_truth": null, + "expected_tool_call": null, + "agent_name": "unknown", + "agent_role": "unknown", + "agent_task": null, + "system_prompt": "You are a helpful customer support agent", + "agent_response": null, + "trace": null, + "tools_available": [], + "tool_calls": [], + "parameters_passed": {}, + "tool_call_results": [], + "retrieval_query": null, + "retrieved_context": null, + "exit_status": "ok", + "agent_exit": false, + "metadata": null + }, + { + "user_id": null, + "task_id": "2218f641-604c-491a-9710-b51a9941b982", + "turn_id": "5fbaef83-95fc-4bed-9695-9d69c37c35e4", + "ground_truth": null, + "expected_tool_call": null, + "agent_name": "unknown", + "agent_role": "unknown", + "agent_task": null, + "system_prompt": "You are a helpful customer support agent", + "agent_response": null, + "trace": null, + "tools_available": [], + "tool_calls": [], + "parameters_passed": {}, + "tool_call_results": [], + "retrieval_query": null, + "retrieved_context": null, + "exit_status": "ok", + "agent_exit": false, + "metadata": null + }, + { + "user_id": null, + "task_id": "255fd49c-84b4-4b18-887e-6308a412d535", + "turn_id": "b3ac3924-928b-4d48-a2a7-aa749802fef9", + "ground_truth": null, + "expected_tool_call": null, + "agent_name": "unknown", + "agent_role": "unknown", + "agent_task": null, + "system_prompt": "You are a helpful customer support agent", + "agent_response": null, + "trace": null, + "tools_available": [], + "tool_calls": [], + "parameters_passed": {}, + "tool_call_results": [], + "retrieval_query": null, + "retrieved_context": null, + "exit_status": "ok", + "agent_exit": false, + "metadata": null + }, + { + "user_id": null, + "task_id": "dc511122-c0b6-415c-9a49-c7b45132dd87", + "turn_id": "b8055ffc-ee7d-46ad-b6a0-84e80f9399e2", + "ground_truth": null, + "expected_tool_call": null, + "agent_name": "unknown", + "agent_role": "unknown", + "agent_task": null, + "system_prompt": "You are a helpful customer support agent", + "agent_response": null, + "trace": null, + "tools_available": [], + "tool_calls": [], + "parameters_passed": {}, + "tool_call_results": [], + "retrieval_query": null, + "retrieved_context": null, + "exit_status": "ok", + "agent_exit": false, + "metadata": null + }, + { + "user_id": null, + "task_id": "04bebf38-a343-4563-80db-0154bef8d927", + "turn_id": "532b9e1f-5c4f-4b41-98bb-e9a1510f0f1c", + "ground_truth": null, + "expected_tool_call": null, + "agent_name": "unknown", + "agent_role": "unknown", + "agent_task": null, + "system_prompt": "You are a helpful customer support agent", + "agent_response": null, + "trace": null, + "tools_available": [], + "tool_calls": [], + "parameters_passed": {}, + "tool_call_results": [], + "retrieval_query": null, + "retrieved_context": null, + "exit_status": "ok", + "agent_exit": false, + "metadata": null + }, + { + "user_id": null, + "task_id": "5e043630-6493-42b5-beb8-79faa19bfa37", + "turn_id": "14064e3f-f9c9-4bb7-b8cb-1ad52938b075", + "ground_truth": null, + "expected_tool_call": null, + "agent_name": "unknown", + "agent_role": "unknown", + "agent_task": null, + "system_prompt": "You are a helpful customer support agent", + "agent_response": null, + "trace": null, + "tools_available": [], + "tool_calls": [], + "parameters_passed": {}, + "tool_call_results": [], + "retrieval_query": null, + "retrieved_context": null, + "exit_status": "ok", + "agent_exit": false, + "metadata": null + }, + { + "user_id": null, + "task_id": "7da9814d-a2e8-4c4e-b750-68b26bd5fd22", + "turn_id": "a74fe8a9-2654-4b9d-ba87-dc2f86b6466d", + "ground_truth": null, + "expected_tool_call": null, + "agent_name": "unknown", + "agent_role": "unknown", + "agent_task": null, + "system_prompt": "You are a helpful customer support agent", + "agent_response": null, + "trace": null, + "tools_available": [], + "tool_calls": [], + "parameters_passed": {}, + "tool_call_results": [], + "retrieval_query": null, + "retrieved_context": null, + "exit_status": "ok", + "agent_exit": false, + "metadata": null + }, + { + "user_id": null, + "task_id": "16143f74-2831-4753-b33d-ce4b645093c5", + "turn_id": "c700d0f4-f1f8-4339-aa42-a3060ae2ae11", + "ground_truth": null, + "expected_tool_call": null, + "agent_name": "unknown", + "agent_role": "unknown", + "agent_task": null, + "system_prompt": "You are a helpful customer support agent", + "agent_response": null, + "trace": null, + "tools_available": [], + "tool_calls": [], + "parameters_passed": {}, + "tool_call_results": [], + "retrieval_query": null, + "retrieved_context": null, + "exit_status": "ok", + "agent_exit": false, + "metadata": null + }, + { + "user_id": null, + "task_id": "fc64e6cc-6739-4256-ac4a-7b80c3028233", + "turn_id": "f7f5981f-d840-4265-8970-d319b876366d", + "ground_truth": null, + "expected_tool_call": null, + "agent_name": "unknown", + "agent_role": "unknown", + "agent_task": null, + "system_prompt": "You are a helpful customer support agent", + "agent_response": null, + "trace": null, + "tools_available": [], + "tool_calls": [], + "parameters_passed": {}, + "tool_call_results": [], + "retrieval_query": null, + "retrieved_context": null, + "exit_status": "ok", + "agent_exit": false, + "metadata": null + }, + { + "user_id": null, + "task_id": "b7945c49-f584-4c70-972d-536a805d8a31", + "turn_id": "0c7bbe2b-18af-4fdb-a5bf-4a91daa3bf50", + "ground_truth": null, + "expected_tool_call": null, + "agent_name": "unknown", + "agent_role": "unknown", + "agent_task": null, + "system_prompt": "You are a helpful customer support agent", + "agent_response": null, + "trace": null, + "tools_available": [], + "tool_calls": [], + "parameters_passed": {}, + "tool_call_results": [], + "retrieval_query": null, + "retrieved_context": null, + "exit_status": "ok", + "agent_exit": false, + "metadata": null + }, + { + "user_id": null, + "task_id": "f5c40ecf-36c0-45ba-9cc9-dc0329b0324b", + "turn_id": "dc869742-6797-4016-8061-37530fc7d433", + "ground_truth": null, + "expected_tool_call": null, + "agent_name": "unknown", + "agent_role": "unknown", + "agent_task": null, + "system_prompt": "You are a helpful customer support agent", + "agent_response": null, + "trace": null, + "tools_available": [], + "tool_calls": [], + "parameters_passed": {}, + "tool_call_results": [], + "retrieval_query": null, + "retrieved_context": null, + "exit_status": "ok", + "agent_exit": false, + "metadata": null + } +] \ No newline at end of file diff --git a/noveum_customer_support_bt/publish_dataset_version.py b/noveum_customer_support_bt/publish_dataset_version.py new file mode 100644 index 0000000..37cf008 --- /dev/null +++ b/noveum_customer_support_bt/publish_dataset_version.py @@ -0,0 +1,117 @@ +#!/usr/bin/env python3 +""" +Script to publish a dataset version in Noveum API. +Publishes the specified version of the dataset. +""" + +import os +import json +import requests +import argparse +from dotenv import load_dotenv +from typing import Dict, Any, Optional + +# Load environment variables +load_dotenv() + +# Get API credentials from environment +api_key = os.getenv('NOVEUM_API_KEY') +org_slug = os.getenv('NOVEUM_ORG_SLUG') +dataset_slug = os.getenv('NOVEUM_DATASET_SLUG') +latest_version = os.getenv('LATEST_VERSION') +beta_env = os.getenv('BETA', 'false').lower() == 'true' + +def validate_environment(): + """Validate that all required environment variables are set""" + required_vars = { + 'NOVEUM_API_KEY': api_key, + 'NOVEUM_ORG_SLUG': org_slug, + 'NOVEUM_DATASET_SLUG': dataset_slug, + 'LATEST_VERSION': latest_version + } + + missing_vars = [var for var, value in required_vars.items() if not value] + + if missing_vars: + print(f"Error: Missing required environment variables: {', '.join(missing_vars)}") + print("Please set these variables in your .env file or environment") + return False + + return True + +def publish_dataset_version(version: str) -> Optional[Dict[str, Any]]: + """Publish a dataset version in Noveum API""" + + # Construct API URL based on BETA environment variable + if beta_env: + api_url = f"https://noveum.ai/api/v1/datasets/{dataset_slug}/versions/{version}/publish?organizationSlug={org_slug}" + else: + api_url = f"https://noveum.ai/api/v1/organizations/{org_slug}/datasets/{dataset_slug}/versions/{version}/publish" + + # Prepare headers + headers = { + 'Authorization': f'Bearer {api_key}', + 'Cookie': f'apiKeyCookie={api_key}' + } + + print(f"Publishing dataset version at: {api_url}") + print(f"Organization: {org_slug}") + print(f"Dataset: {dataset_slug}") + print(f"Version: {version}") + + try: + response = requests.post(api_url, headers=headers, timeout=30) + response.raise_for_status() + + data = response.json() + print("Successfully published dataset version") + print(f"Response status: {response.status_code}") + + return data + + except requests.exceptions.RequestException as e: + print(f"Error publishing dataset version: {e}") + if hasattr(e, 'response') and e.response is not None: + print(f"Response status: {e.response.status_code}") + print(f"Response text: {e.response.text}") + return None + +def main(): + parser = argparse.ArgumentParser(description='Publish a dataset version in Noveum API') + parser.add_argument('--pretty', action='store_true', + help='Pretty print the JSON response') + parser.add_argument('--output', type=str, default="dataset_publish_response.json", + help='Output file to save the JSON response (default: dataset_publish_response.json)') + + args = parser.parse_args() + + # Validate environment variables + if not validate_environment(): + return 1 + + # Publish dataset version + data = publish_dataset_version(version=latest_version) + + if data is None: + return 1 + + # Save response to file + try: + with open(args.output, 'w', encoding='utf-8') as f: + json.dump(data, f, indent=2) + print(f"\nResponse saved to: {args.output}") + except (OSError, IOError) as e: + print(f"Error saving response to file: {e}") + return 1 + + # Print the response + if args.pretty: + print("\nResponse data:") + print(json.dumps(data, indent=2)) + else: + print(f"\nResponse data: {json.dumps(data)}") + + return 0 + +if __name__ == "__main__": + exit(main()) diff --git a/noveum_customer_support_bt/split_datasets/agent.llm-rag_dataset.json b/noveum_customer_support_bt/split_datasets/agent.llm-rag_dataset.json new file mode 100644 index 0000000..5f77f7d --- /dev/null +++ b/noveum_customer_support_bt/split_datasets/agent.llm-rag_dataset.json @@ -0,0 +1,734 @@ +[ + { + "span_id": "ea4d61c5-b8dd-4677-b21b-2b17170e2163", + "trace_id": "f1f37bd7-0851-4659-b493-b80d3800d920", + "parent_span_id": "89eac2de-b7da-4db3-bf7c-8e87c429481e", + "name": "agent.llm-rag", + "start_time": "2025-10-09 08:26:35.915000000", + "end_time": "2025-10-09 08:26:40.841000000", + "duration_ms": 4925.434, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "rag_agent", + "agent.operation": "llm-rag", + "agent.capabilities": [ + "document_retrieval", + "context_generation", + "response_generation" + ], + "agent.id": "noveum_rag_agent", + "input_query": "What are the newest features in cloud computing?", + "query_length": 48, + "query_type": "rag_query", + "output_response": "RAG Answer: The provided context does not specifically mention any new features in cloud computing as a general topic. However, it does highlight several recent enhancements and features related to the Noveum.ai ...", + "rag.context_length": 2579, + "rag.sources_count": 5, + "rag.answer_length": 883, + "rag.mode": "retrieval_augmented_generation" + }, + "events": [], + "links": [], + "trace_trace_id": "f1f37bd7-0851-4659-b493-b80d3800d920", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-09 08:26:35.915000000", + "trace_end_time": "2025-10-09 08:26:40.841000000", + "trace_duration_ms": 4926.342, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-09 08:26:44.053000000", + "updated_at": "2025-10-09 08:26:44.053000000" + }, + { + "span_id": "7505f97a-cbc0-42fe-8a72-95387d723daa", + "trace_id": "52aacb67-c361-4445-9b72-c157f79f47d6", + "parent_span_id": "5359c861-c51b-446d-9ad2-b3d8c21ea70f", + "name": "agent.llm-rag", + "start_time": "2025-10-09 08:26:16.545000000", + "end_time": "2025-10-09 08:26:22.201000000", + "duration_ms": 5656.22, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "rag_agent", + "agent.operation": "llm-rag", + "agent.capabilities": [ + "document_retrieval", + "context_generation", + "response_generation" + ], + "agent.id": "noveum_rag_agent", + "input_query": "What are the current trends in observability tools?", + "query_length": 51, + "query_type": "rag_query", + "output_response": "RAG Answer: The provided context does not explicitly outline current trends in observability tools. However, it highlights some key aspects of observability that are particularly relevant to AI applications. \n\n1....", + "rag.context_length": 2943, + "rag.sources_count": 5, + "rag.answer_length": 1074, + "rag.mode": "retrieval_augmented_generation" + }, + "events": [], + "links": [], + "trace_trace_id": "52aacb67-c361-4445-9b72-c157f79f47d6", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-09 08:26:16.544000000", + "trace_end_time": "2025-10-09 08:26:22.201000000", + "trace_duration_ms": 5657.012, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-09 08:26:22.904000000", + "updated_at": "2025-10-09 08:26:22.904000000" + }, + { + "span_id": "8a6e18a1-babc-4166-926f-188f344d514e", + "trace_id": "2218f641-604c-491a-9710-b51a9941b982", + "parent_span_id": "3ce2e35e-4518-43b4-8a76-4413da6832ea", + "name": "agent.llm-rag", + "start_time": "2025-10-09 08:25:48.107000000", + "end_time": "2025-10-09 08:25:55.878000000", + "duration_ms": 7770.424, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "rag_agent", + "agent.operation": "llm-rag", + "agent.capabilities": [ + "document_retrieval", + "context_generation", + "response_generation" + ], + "agent.id": "noveum_rag_agent", + "input_query": "What are the benefits of using Noveum Trace?", + "query_length": 44, + "query_type": "rag_query", + "output_response": "RAG Answer: Using Noveum Trace offers several benefits, including:\n\n1. **Real-Time Monitoring**: The Noveum platform provides a real-time dashboard for analyzing traces and performance, with live updates as new t...", + "rag.context_length": 2773, + "rag.sources_count": 5, + "rag.answer_length": 1460, + "rag.mode": "retrieval_augmented_generation" + }, + "events": [], + "links": [], + "trace_trace_id": "2218f641-604c-491a-9710-b51a9941b982", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-09 08:25:48.107000000", + "trace_end_time": "2025-10-09 08:25:55.878000000", + "trace_duration_ms": 7770.953, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-09 08:25:56.565000000", + "updated_at": "2025-10-09 08:25:56.565000000" + }, + { + "span_id": "5bec4165-6596-43ac-930b-acfcde1cc02a", + "trace_id": "255fd49c-84b4-4b18-887e-6308a412d535", + "parent_span_id": "c0c39e8b-9a42-4c18-8737-9224cd8a45c0", + "name": "agent.llm-rag", + "start_time": "2025-10-09 08:25:43.830000000", + "end_time": "2025-10-09 08:25:47.099000000", + "duration_ms": 3268.575, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "rag_agent", + "agent.operation": "llm-rag", + "agent.capabilities": [ + "document_retrieval", + "context_generation", + "response_generation" + ], + "agent.id": "noveum_rag_agent", + "input_query": "How do I configure Noveum for my system?", + "query_length": 40, + "query_type": "rag_query", + "output_response": "RAG Answer: To configure Noveum for your system, you can refer to the SDK Integration Guide available in the Noveum documentation. This guide will help you trace your AI applications with minimal code changes. Th...", + "rag.context_length": 2237, + "rag.sources_count": 5, + "rag.answer_length": 613, + "rag.mode": "retrieval_augmented_generation" + }, + "events": [], + "links": [], + "trace_trace_id": "255fd49c-84b4-4b18-887e-6308a412d535", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-09 08:25:43.830000000", + "trace_end_time": "2025-10-09 08:25:47.099000000", + "trace_duration_ms": 3268.862, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-09 08:25:50.304000000", + "updated_at": "2025-10-09 08:25:50.304000000" + }, + { + "span_id": "f9f79a07-4eda-41b8-9634-d4460114caad", + "trace_id": "dc511122-c0b6-415c-9a49-c7b45132dd87", + "parent_span_id": "80987c51-ea82-4a41-97c7-ed591082b9bf", + "name": "agent.llm-rag", + "start_time": "2025-10-09 08:25:38.322000000", + "end_time": "2025-10-09 08:25:42.822000000", + "duration_ms": 4500.0109999999995, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "rag_agent", + "agent.operation": "llm-rag", + "agent.capabilities": [ + "document_retrieval", + "context_generation", + "response_generation" + ], + "agent.id": "noveum_rag_agent", + "input_query": "What monitoring capabilities does Noveum provide?", + "query_length": 49, + "query_type": "rag_query", + "output_response": "RAG Answer: Noveum provides comprehensive monitoring capabilities for AI applications, including the ability to monitor, trace, and optimize AI agents across various frameworks such as LangChain, CrewAI, AutoGen,...", + "rag.context_length": 2740, + "rag.sources_count": 5, + "rag.answer_length": 839, + "rag.mode": "retrieval_augmented_generation" + }, + "events": [], + "links": [], + "trace_trace_id": "dc511122-c0b6-415c-9a49-c7b45132dd87", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-09 08:25:38.322000000", + "trace_end_time": "2025-10-09 08:25:42.822000000", + "trace_duration_ms": 4500.425, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-09 08:25:45.028000000", + "updated_at": "2025-10-09 08:25:45.028000000" + }, + { + "span_id": "edc52a5d-8450-4d0f-ad14-958ec4d147f4", + "trace_id": "04bebf38-a343-4563-80db-0154bef8d927", + "parent_span_id": "2f89c1e0-9b62-4651-914c-b71cf561c33c", + "name": "agent.llm-rag", + "start_time": "2025-10-09 08:25:33.693000000", + "end_time": "2025-10-09 08:25:37.313000000", + "duration_ms": 3619.1870000000004, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "rag_agent", + "agent.operation": "llm-rag", + "agent.capabilities": [ + "document_retrieval", + "context_generation", + "response_generation" + ], + "agent.id": "noveum_rag_agent", + "input_query": "How does Noveum handle agent tracing?", + "query_length": 37, + "query_type": "rag_query", + "output_response": "RAG Answer: Noveum.ai handles agent tracing through its specialized Multi-Agent Tracing capabilities, which are designed to observe complex workflows and inter-agent communications within multi-agent systems. The...", + "rag.context_length": 2736, + "rag.sources_count": 5, + "rag.answer_length": 691, + "rag.mode": "retrieval_augmented_generation" + }, + "events": [], + "links": [], + "trace_trace_id": "04bebf38-a343-4563-80db-0154bef8d927", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-09 08:25:33.693000000", + "trace_end_time": "2025-10-09 08:25:37.313000000", + "trace_duration_ms": 3619.672, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-09 08:25:40.339000000", + "updated_at": "2025-10-09 08:25:40.339000000" + }, + { + "span_id": "6c362c73-e063-4813-8180-d26b6a52b798", + "trace_id": "5e043630-6493-42b5-beb8-79faa19bfa37", + "parent_span_id": "e30e3e61-6edd-4b62-ab77-627faa2bf741", + "name": "agent.llm-rag", + "start_time": "2025-10-09 08:25:29.902000000", + "end_time": "2025-10-09 08:25:32.684000000", + "duration_ms": 2782.976, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "rag_agent", + "agent.operation": "llm-rag", + "agent.capabilities": [ + "document_retrieval", + "context_generation", + "response_generation" + ], + "agent.id": "noveum_rag_agent", + "input_query": "What APIs are available in Noveum platform?", + "query_length": 43, + "query_type": "rag_query", + "output_response": "RAG Answer: The provided context does not specify the APIs available in the Noveum platform. For detailed information about the APIs, I recommend checking the official Noveum documentation or the SDK Integration ...", + "rag.context_length": 2284, + "rag.sources_count": 5, + "rag.answer_length": 270, + "rag.mode": "retrieval_augmented_generation" + }, + "events": [], + "links": [], + "trace_trace_id": "5e043630-6493-42b5-beb8-79faa19bfa37", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-09 08:25:29.901000000", + "trace_end_time": "2025-10-09 08:25:32.685000000", + "trace_duration_ms": 2783.326, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-09 08:25:34.918000000", + "updated_at": "2025-10-09 08:25:34.918000000" + }, + { + "span_id": "1393d065-a9a4-4475-808f-2d00b3c68939", + "trace_id": "7da9814d-a2e8-4c4e-b750-68b26bd5fd22", + "parent_span_id": "2f513795-764f-4eb6-8dd2-108d4cdf8432", + "name": "agent.llm-rag", + "start_time": "2025-10-09 08:25:20.587000000", + "end_time": "2025-10-09 08:25:28.897000000", + "duration_ms": 8309.805, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "rag_agent", + "agent.operation": "llm-rag", + "agent.capabilities": [ + "document_retrieval", + "context_generation", + "response_generation" + ], + "agent.id": "noveum_rag_agent", + "input_query": "How do I set up observability with Noveum?", + "query_length": 42, + "query_type": "rag_query", + "output_response": "RAG Answer: To set up observability with Noveum, follow these steps:\n\n1. **Sign Up**: Go to [noveum.ai](https://noveum.ai) and create an account.\n2. **Create a Project**: Once signed in, create a new project to o...", + "rag.context_length": 2870, + "rag.sources_count": 6, + "rag.answer_length": 951, + "rag.mode": "retrieval_augmented_generation" + }, + "events": [], + "links": [], + "trace_trace_id": "7da9814d-a2e8-4c4e-b750-68b26bd5fd22", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-09 08:25:20.586000000", + "trace_end_time": "2025-10-09 08:25:28.897000000", + "trace_duration_ms": 8310.205, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-09 08:25:29.588000000", + "updated_at": "2025-10-09 08:25:29.588000000" + }, + { + "span_id": "f419377a-fc24-4abf-8a1c-03857824e374", + "trace_id": "16143f74-2831-4753-b33d-ce4b645093c5", + "parent_span_id": "29c7a515-2383-4b7d-8fc7-beac3f403641", + "name": "agent.llm-rag", + "start_time": "2025-10-09 08:25:12.442000000", + "end_time": "2025-10-09 08:25:19.579000000", + "duration_ms": 7137.042, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "rag_agent", + "agent.operation": "llm-rag", + "agent.capabilities": [ + "document_retrieval", + "context_generation", + "response_generation" + ], + "agent.id": "noveum_rag_agent", + "input_query": "What features does Noveum Trace offer?", + "query_length": 38, + "query_type": "rag_query", + "output_response": "RAG Answer: Noveum Trace offers several key features through its Python SDK (`noveum-trace`) and TypeScript SDK (`@noveum/trace`):\n\n1. **Decorator-based Tracing**: It provides a simple way to trace LLM calls, age...", + "rag.context_length": 2787, + "rag.sources_count": 5, + "rag.answer_length": 1014, + "rag.mode": "retrieval_augmented_generation" + }, + "events": [], + "links": [], + "trace_trace_id": "16143f74-2831-4753-b33d-ce4b645093c5", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-09 08:25:12.442000000", + "trace_end_time": "2025-10-09 08:25:19.579000000", + "trace_duration_ms": 7137.402, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-09 08:25:20.290000000", + "updated_at": "2025-10-09 08:25:20.290000000" + }, + { + "span_id": "0e58e31f-a8ab-4aee-8fc2-abcae1e21d9d", + "trace_id": "fc64e6cc-6739-4256-ac4a-7b80c3028233", + "parent_span_id": "12fee687-2a2e-401f-83c2-9c8dc7ecd853", + "name": "agent.llm-rag", + "start_time": "2025-10-09 08:25:07.932000000", + "end_time": "2025-10-09 08:25:11.438000000", + "duration_ms": 3505.745, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "rag_agent", + "agent.operation": "llm-rag", + "agent.capabilities": [ + "document_retrieval", + "context_generation", + "response_generation" + ], + "agent.id": "noveum_rag_agent", + "input_query": "What are Noveum's pricing plans?", + "query_length": 32, + "query_type": "rag_query", + "output_response": "RAG Answer: The provided context does not include specific information about Noveum's pricing plans. However, it mentions that all users will have access to the Observability suite for free, and early users will ...", + "rag.context_length": 2105, + "rag.sources_count": 5, + "rag.answer_length": 399, + "rag.mode": "retrieval_augmented_generation" + }, + "events": [], + "links": [], + "trace_trace_id": "fc64e6cc-6739-4256-ac4a-7b80c3028233", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-09 08:25:07.932000000", + "trace_end_time": "2025-10-09 08:25:11.438000000", + "trace_duration_ms": 3506.0989999999997, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-09 08:25:12.651000000", + "updated_at": "2025-10-09 08:25:12.651000000" + }, + { + "span_id": "48155aad-6204-4a52-8624-8fa5d6fe407f", + "trace_id": "b7945c49-f584-4c70-972d-536a805d8a31", + "parent_span_id": "9c5aec32-b6eb-4c42-93d5-3970fb118c00", + "name": "agent.llm-rag", + "start_time": "2025-10-09 08:24:58.518000000", + "end_time": "2025-10-09 08:25:06.920000000", + "duration_ms": 8401.725, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "rag_agent", + "agent.operation": "llm-rag", + "agent.capabilities": [ + "document_retrieval", + "context_generation", + "response_generation" + ], + "agent.id": "noveum_rag_agent", + "input_query": "How do I integrate Noveum Trace in my application?", + "query_length": 50, + "query_type": "rag_query", + "output_response": "RAG Answer: To integrate Noveum Trace into your application, you can use either the Python SDK (`noveum-trace`) or the TypeScript SDK (`@noveum/trace`). Hereā€™s a quick start guide based on the provided documentat...", + "rag.context_length": 2803, + "rag.sources_count": 5, + "rag.answer_length": 1151, + "rag.mode": "retrieval_augmented_generation" + }, + "events": [], + "links": [], + "trace_trace_id": "b7945c49-f584-4c70-972d-536a805d8a31", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-09 08:24:58.518000000", + "trace_end_time": "2025-10-09 08:25:06.920000000", + "trace_duration_ms": 8402.516, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-09 08:25:07.620000000", + "updated_at": "2025-10-09 08:25:07.620000000" + }, + { + "span_id": "57ded2eb-0f3f-474e-8704-5ad9974ab4bb", + "trace_id": "f5c40ecf-36c0-45ba-9cc9-dc0329b0324b", + "parent_span_id": "f132d053-f82c-4e66-adf4-bd7ce68b660d", + "name": "agent.llm-rag", + "start_time": "2025-10-09 08:24:54.009000000", + "end_time": "2025-10-09 08:24:57.510000000", + "duration_ms": 3501.569, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "rag_agent", + "agent.operation": "llm-rag", + "agent.capabilities": [ + "document_retrieval", + "context_generation", + "response_generation" + ], + "agent.id": "noveum_rag_agent", + "input_query": "What is Noveum and what does it do?", + "query_length": 35, + "query_type": "rag_query", + "output_response": "RAG Answer: Noveum.ai is a comprehensive tracing and observability platform specifically designed for AI applications, including those powered by large language models (LLMs), retrieval-augmented generation (RAG)...", + "rag.context_length": 2334, + "rag.sources_count": 5, + "rag.answer_length": 611, + "rag.mode": "retrieval_augmented_generation" + }, + "events": [], + "links": [], + "trace_trace_id": "f5c40ecf-36c0-45ba-9cc9-dc0329b0324b", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-09 08:24:54.009000000", + "trace_end_time": "2025-10-09 08:24:57.511000000", + "trace_duration_ms": 3502.062, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-09 08:24:59.716000000", + "updated_at": "2025-10-09 08:24:59.716000000" + } +] \ No newline at end of file diff --git a/noveum_customer_support_bt/split_datasets/agent.llm_model_execution_dataset.json b/noveum_customer_support_bt/split_datasets/agent.llm_model_execution_dataset.json new file mode 100644 index 0000000..1ca42e5 --- /dev/null +++ b/noveum_customer_support_bt/split_datasets/agent.llm_model_execution_dataset.json @@ -0,0 +1,1782 @@ +[ + { + "span_id": "ce8556f4-bbeb-46fc-8bfd-dfaedb61cb70", + "trace_id": "eda4fe22-9a2b-4b73-856b-f4f3309bf719", + "parent_span_id": "2aca614b-b016-4d56-abe2-43a144dc681d", + "name": "agent.llm_model_execution", + "start_time": "2025-10-09 08:26:46.681000000", + "end_time": "2025-10-09 08:26:48.685000000", + "duration_ms": 2004.578, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What are the latest breakthroughs in artificial intelligence?", + "query_length": 61, + "query_type": "web_search_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 2.004547119140625, + "model.latency_ms": 2004.547119140625, + "model.start_time": 1759998406.681066, + "model.end_time": 1759998408.6856132, + "model.performance_tier": "medium", + "model.prompt_tokens": 303, + "model.completion_tokens": 105, + "model.total_tokens": 408, + "model.tokens_per_second": 203.53724594656313, + "model.estimated_cost": 0.012240000000000001, + "model.efficiency_score": 1.036764705882353, + "model.response_length": 423, + "model.response_quality": "high", + "model.output_response": "Model Response: The provided web search results do not contain specific information about the latest breakthroughs in artificial intelligence. To find the most recent advancements in AI, I recommend checking reputabl...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "eda4fe22-9a2b-4b73-856b-f4f3309bf719", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-09 08:26:46.384000000", + "trace_end_time": "2025-10-09 08:26:48.685000000", + "trace_duration_ms": 2301.488, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-09 08:26:49.422000000", + "updated_at": "2025-10-09 08:26:49.422000000" + }, + { + "span_id": "cc3073ec-c970-4821-8335-8d0fc8f37930", + "trace_id": "0ffffba1-8a37-443c-8866-d53ffbfa7718", + "parent_span_id": "03e3ae4d-0107-401d-8ca0-05fc0f28fb24", + "name": "agent.llm_model_execution", + "start_time": "2025-10-09 08:26:42.090000000", + "end_time": "2025-10-09 08:26:45.377000000", + "duration_ms": 3287.441, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What's happening in the software development world today?", + "query_length": 57, + "query_type": "web_search_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 3.2873661518096924, + "model.latency_ms": 3287.3661518096924, + "model.start_time": 1759998402.09009, + "model.end_time": 1759998405.3774562, + "model.performance_tier": "medium", + "model.prompt_tokens": 165, + "model.completion_tokens": 198, + "model.total_tokens": 363, + "model.tokens_per_second": 110.4227467330248, + "model.estimated_cost": 0.01089, + "model.efficiency_score": 2.190082644628099, + "model.response_length": 795, + "model.response_quality": "high", + "model.output_response": "Model Response: The web search did not yield any specific results regarding current events in the software development world. Therefore, I cannot provide detailed information on what's happening today in that field.\n...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "0ffffba1-8a37-443c-8866-d53ffbfa7718", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-09 08:26:41.850000000", + "trace_end_time": "2025-10-09 08:26:45.377000000", + "trace_duration_ms": 3527.831, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-09 08:26:49.422000000", + "updated_at": "2025-10-09 08:26:49.422000000" + }, + { + "span_id": "c24fae6d-6314-4342-8214-7e9905ae83db", + "trace_id": "f1f37bd7-0851-4659-b493-b80d3800d920", + "parent_span_id": "ea4d61c5-b8dd-4677-b21b-2b17170e2163", + "name": "agent.llm_model_execution", + "start_time": "2025-10-09 08:26:36.433000000", + "end_time": "2025-10-09 08:26:40.446000000", + "duration_ms": 4013.1009999999997, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What are the newest features in cloud computing?", + "query_length": 48, + "query_type": "rag_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 4.012948989868164, + "model.latency_ms": 4012.948989868164, + "model.start_time": 1759998396.433433, + "model.end_time": 1759998400.446382, + "model.performance_tier": "medium", + "model.prompt_tokens": 780, + "model.completion_tokens": 220, + "model.total_tokens": 1000, + "model.tokens_per_second": 249.1932996219951, + "model.estimated_cost": 0.030000000000000002, + "model.efficiency_score": 0.883, + "model.response_length": 883, + "model.response_quality": "high", + "model.output_response": "Model Response: The provided context does not specifically mention any new features in cloud computing as a general topic. However, it does highlight several recent enhancements and features related to the Noveum.ai ...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "f1f37bd7-0851-4659-b493-b80d3800d920", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-09 08:26:35.915000000", + "trace_end_time": "2025-10-09 08:26:40.841000000", + "trace_duration_ms": 4926.342, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-09 08:26:44.053000000", + "updated_at": "2025-10-09 08:26:44.053000000" + }, + { + "span_id": "93879d8f-77c7-4fd4-881b-be500e2dcbde", + "trace_id": "43cdf081-4f01-49cd-b566-dbd1619e6cd2", + "parent_span_id": "dfdb9aaf-9925-4a5c-a0c6-2cb00f70d4d3", + "name": "agent.llm_model_execution", + "start_time": "2025-10-09 08:26:32.526000000", + "end_time": "2025-10-09 08:26:34.906000000", + "duration_ms": 2379.957, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What's the current status of cryptocurrency markets?", + "query_length": 52, + "query_type": "web_search_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 2.3798890113830566, + "model.latency_ms": 2379.8890113830566, + "model.start_time": 1759998392.526488, + "model.end_time": 1759998394.906377, + "model.performance_tier": "medium", + "model.prompt_tokens": 162, + "model.completion_tokens": 92, + "model.total_tokens": 254, + "model.tokens_per_second": 106.72766620002568, + "model.estimated_cost": 0.00762, + "model.efficiency_score": 1.4606299212598426, + "model.response_length": 371, + "model.response_quality": "high", + "model.output_response": "Model Response: The web search results did not provide any specific information regarding the current status of cryptocurrency markets. Therefore, I cannot provide an accurate update on market conditions, trends, or ...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "43cdf081-4f01-49cd-b566-dbd1619e6cd2", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-09 08:26:32.237000000", + "trace_end_time": "2025-10-09 08:26:34.906000000", + "trace_duration_ms": 2668.841, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-09 08:26:38.618000000", + "updated_at": "2025-10-09 08:26:38.618000000" + }, + { + "span_id": "f92643c0-44a3-4fae-9d29-45321342d12f", + "trace_id": "9a1983f4-09da-4b53-80e6-38de6878e0e7", + "parent_span_id": "eefcfdfc-6433-459e-a134-e2647b8418e4", + "name": "agent.llm_model_execution", + "start_time": "2025-10-09 08:26:28.250000000", + "end_time": "2025-10-09 08:26:31.231000000", + "duration_ms": 2981.5170000000003, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What are the latest updates in Python programming?", + "query_length": 50, + "query_type": "web_search_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 2.981428861618042, + "model.latency_ms": 2981.428861618042, + "model.start_time": 1759998388.250087, + "model.end_time": 1759998391.231516, + "model.performance_tier": "medium", + "model.prompt_tokens": 285, + "model.completion_tokens": 145, + "model.total_tokens": 430, + "model.tokens_per_second": 144.22614791708833, + "model.estimated_cost": 0.0129, + "model.efficiency_score": 1.3558139534883722, + "model.response_length": 583, + "model.response_quality": "high", + "model.output_response": "Model Response: The web search results do not provide specific information about the latest updates in Python programming. They focus primarily on news related to China, including geopolitics, economy, and lifestyle,...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "9a1983f4-09da-4b53-80e6-38de6878e0e7", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-09 08:26:27.992000000", + "trace_end_time": "2025-10-09 08:26:31.231000000", + "trace_duration_ms": 3239.706, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-09 08:26:33.463000000", + "updated_at": "2025-10-09 08:26:33.463000000" + }, + { + "span_id": "44f379aa-10fe-4a51-847c-5288444d7843", + "trace_id": "5d2517e1-220a-429d-9d59-f701bda25eed", + "parent_span_id": "c57b4b54-b9b0-4a4c-b5cd-57a4cfe5301b", + "name": "agent.llm_model_execution", + "start_time": "2025-10-09 08:26:23.486000000", + "end_time": "2025-10-09 08:26:26.985000000", + "duration_ms": 3498.348, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What happened in tech news this week?", + "query_length": 37, + "query_type": "web_search_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 3.4982941150665283, + "model.latency_ms": 3498.2941150665283, + "model.start_time": 1759998383.486767, + "model.end_time": 1759998386.9850612, + "model.performance_tier": "medium", + "model.prompt_tokens": 280, + "model.completion_tokens": 125, + "model.total_tokens": 405, + "model.tokens_per_second": 115.77071186088594, + "model.estimated_cost": 0.012150000000000001, + "model.efficiency_score": 1.237037037037037, + "model.response_length": 501, + "model.response_quality": "high", + "model.output_response": "Model Response: The web search results do not provide specific information about recent events in tech news for this week. They primarily discuss the grammatical usage of the phrase \"what happened\" and its variations...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "5d2517e1-220a-429d-9d59-f701bda25eed", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-09 08:26:23.210000000", + "trace_end_time": "2025-10-09 08:26:26.985000000", + "trace_duration_ms": 3775.124, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-09 08:26:28.174000000", + "updated_at": "2025-10-09 08:26:28.174000000" + }, + { + "span_id": "e8e79f2a-2ed0-4981-943c-adcf6f7d2b15", + "trace_id": "52aacb67-c361-4445-9b72-c157f79f47d6", + "parent_span_id": "7505f97a-cbc0-42fe-8a72-95387d723daa", + "name": "agent.llm_model_execution", + "start_time": "2025-10-09 08:26:17.058000000", + "end_time": "2025-10-09 08:26:21.807000000", + "duration_ms": 4748.643, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What are the current trends in observability tools?", + "query_length": 51, + "query_type": "rag_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 4.748546838760376, + "model.latency_ms": 4748.546838760376, + "model.start_time": 1759998377.058629, + "model.end_time": 1759998381.8071759, + "model.performance_tier": "medium", + "model.prompt_tokens": 872, + "model.completion_tokens": 268, + "model.total_tokens": 1140, + "model.tokens_per_second": 240.07344535272622, + "model.estimated_cost": 0.0342, + "model.efficiency_score": 0.9421052631578948, + "model.response_length": 1074, + "model.response_quality": "high", + "model.output_response": "Model Response: The provided context does not explicitly outline current trends in observability tools. However, it highlights some key aspects of observability that are particularly relevant to AI applications. \n\n1....", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "52aacb67-c361-4445-9b72-c157f79f47d6", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-09 08:26:16.544000000", + "trace_end_time": "2025-10-09 08:26:22.201000000", + "trace_duration_ms": 5657.012, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-09 08:26:22.904000000", + "updated_at": "2025-10-09 08:26:22.904000000" + }, + { + "span_id": "e6d0eb41-5f64-44c5-8a42-a80c336de03e", + "trace_id": "a81ca3a8-80aa-4c39-876e-8d40ea7a0aef", + "parent_span_id": "a062bef6-d973-489b-91df-5d40a2aef077", + "name": "agent.llm_model_execution", + "start_time": "2025-10-09 08:26:09.227000000", + "end_time": "2025-10-09 08:26:15.537000000", + "duration_ms": 6309.442, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: Tell me about recent developments in machine learning", + "query_length": 53, + "query_type": "web_search_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 6.309402227401733, + "model.latency_ms": 6309.402227401733, + "model.start_time": 1759998369.2277088, + "model.end_time": 1759998375.537111, + "model.performance_tier": "slow", + "model.prompt_tokens": 309, + "model.completion_tokens": 409, + "model.total_tokens": 718, + "model.tokens_per_second": 113.79841926731602, + "model.estimated_cost": 0.02154, + "model.efficiency_score": 2.2827298050139277, + "model.response_length": 1639, + "model.response_quality": "high", + "model.output_response": "Model Response: The provided web search results do not contain specific information about recent developments in machine learning. Therefore, I cannot provide a detailed answer based on those results.\n\nHowever, I can...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "a81ca3a8-80aa-4c39-876e-8d40ea7a0aef", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-09 08:26:08.903000000", + "trace_end_time": "2025-10-09 08:26:15.537000000", + "trace_duration_ms": 6633.662, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-09 08:26:16.227000000", + "updated_at": "2025-10-09 08:26:16.227000000" + }, + { + "span_id": "c545cb62-5a51-4f2e-b020-b3f1d1dfe217", + "trace_id": "230aad27-f3dd-4968-a45a-3c2f07ac28ed", + "parent_span_id": "41f82a95-620d-440c-84e6-36d6d1b9e079", + "name": "agent.llm_model_execution", + "start_time": "2025-10-09 08:26:04.977000000", + "end_time": "2025-10-09 08:26:07.893000000", + "duration_ms": 2916.6910000000003, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What's the weather like today?", + "query_length": 30, + "query_type": "web_search_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 2.9165940284729004, + "model.latency_ms": 2916.5940284729004, + "model.start_time": 1759998364.977324, + "model.end_time": 1759998367.893918, + "model.performance_tier": "medium", + "model.prompt_tokens": 343, + "model.completion_tokens": 94, + "model.total_tokens": 437, + "model.tokens_per_second": 149.83230293069235, + "model.estimated_cost": 0.01311, + "model.efficiency_score": 0.8604118993135011, + "model.response_length": 376, + "model.response_quality": "high", + "model.output_response": "Model Response: The provided web search results do not contain specific information about today's weather in any particular location. They mention weather forecasts for various places, including France and Singapore,...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "230aad27-f3dd-4968-a45a-3c2f07ac28ed", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-09 08:26:02.424000000", + "trace_end_time": "2025-10-09 08:26:07.894000000", + "trace_duration_ms": 5470.232, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-09 08:26:08.586000000", + "updated_at": "2025-10-09 08:26:08.586000000" + }, + { + "span_id": "c2f21e4d-a5e6-4134-aeaf-96e2f566cf96", + "trace_id": "83c7dcce-3d89-4da1-8b3f-d419885d4cbc", + "parent_span_id": "63f2a0d5-9016-4483-8ec0-e6aa6b27689f", + "name": "agent.llm_model_execution", + "start_time": "2025-10-09 08:25:57.553000000", + "end_time": "2025-10-09 08:26:01.415000000", + "duration_ms": 3862.011, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What are the latest AI news today?", + "query_length": 34, + "query_type": "web_search_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 3.8618290424346924, + "model.latency_ms": 3861.8290424346924, + "model.start_time": 1759998357.553477, + "model.end_time": 1759998361.415306, + "model.performance_tier": "medium", + "model.prompt_tokens": 365, + "model.completion_tokens": 250, + "model.total_tokens": 615, + "model.tokens_per_second": 159.25096456685014, + "model.estimated_cost": 0.01845, + "model.efficiency_score": 1.6308943089430894, + "model.response_length": 1003, + "model.response_quality": "high", + "model.output_response": "Model Response: The latest news in artificial intelligence includes several key updates:\n\n1. **AI Spending Concerns**: Jeff Bezos has expressed concerns that the current level of investment in AI resembles an \"indust...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "83c7dcce-3d89-4da1-8b3f-d419885d4cbc", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-09 08:25:56.886000000", + "trace_end_time": "2025-10-09 08:26:01.416000000", + "trace_duration_ms": 4530.08, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-09 08:26:02.122000000", + "updated_at": "2025-10-09 08:26:02.122000000" + }, + { + "span_id": "39d99d2b-518f-439c-8758-7a01b970fd9e", + "trace_id": "2218f641-604c-491a-9710-b51a9941b982", + "parent_span_id": "8a6e18a1-babc-4166-926f-188f344d514e", + "name": "agent.llm_model_execution", + "start_time": "2025-10-09 08:25:48.496000000", + "end_time": "2025-10-09 08:25:55.406000000", + "duration_ms": 6910.508, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What are the benefits of using Noveum Trace?", + "query_length": 44, + "query_type": "rag_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 6.910433053970337, + "model.latency_ms": 6910.433053970337, + "model.start_time": 1759998348.4961321, + "model.end_time": 1759998355.4065652, + "model.performance_tier": "slow", + "model.prompt_tokens": 828, + "model.completion_tokens": 365, + "model.total_tokens": 1193, + "model.tokens_per_second": 172.63751644545212, + "model.estimated_cost": 0.03579, + "model.efficiency_score": 1.2238055322715842, + "model.response_length": 1460, + "model.response_quality": "high", + "model.output_response": "Model Response: Using Noveum Trace offers several benefits, including:\n\n1. **Real-Time Monitoring**: The Noveum platform provides a real-time dashboard for analyzing traces and performance, with live updates as new t...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "2218f641-604c-491a-9710-b51a9941b982", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-09 08:25:48.107000000", + "trace_end_time": "2025-10-09 08:25:55.878000000", + "trace_duration_ms": 7770.953, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-09 08:25:56.565000000", + "updated_at": "2025-10-09 08:25:56.565000000" + }, + { + "span_id": "9e5ade74-3815-4dad-8abe-d4281d7e62da", + "trace_id": "255fd49c-84b4-4b18-887e-6308a412d535", + "parent_span_id": "5bec4165-6596-43ac-930b-acfcde1cc02a", + "name": "agent.llm_model_execution", + "start_time": "2025-10-09 08:25:44.242000000", + "end_time": "2025-10-09 08:25:46.618000000", + "duration_ms": 2375.3199999999997, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: How do I configure Noveum for my system?", + "query_length": 40, + "query_type": "rag_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 2.3752830028533936, + "model.latency_ms": 2375.2830028533936, + "model.start_time": 1759998344.2429838, + "model.end_time": 1759998346.6182668, + "model.performance_tier": "medium", + "model.prompt_tokens": 693, + "model.completion_tokens": 153, + "model.total_tokens": 846, + "model.tokens_per_second": 356.1680856486205, + "model.estimated_cost": 0.02538, + "model.efficiency_score": 0.7245862884160756, + "model.response_length": 613, + "model.response_quality": "high", + "model.output_response": "Model Response: To configure Noveum for your system, you can refer to the SDK Integration Guide available in the Noveum documentation. This guide will help you trace your AI applications with minimal code changes. Th...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "255fd49c-84b4-4b18-887e-6308a412d535", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-09 08:25:43.830000000", + "trace_end_time": "2025-10-09 08:25:47.099000000", + "trace_duration_ms": 3268.862, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-09 08:25:50.304000000", + "updated_at": "2025-10-09 08:25:50.304000000" + }, + { + "span_id": "4f38a85b-0721-47a5-8a51-d0808e460f96", + "trace_id": "dc511122-c0b6-415c-9a49-c7b45132dd87", + "parent_span_id": "f9f79a07-4eda-41b8-9634-d4460114caad", + "name": "agent.llm_model_execution", + "start_time": "2025-10-09 08:25:38.732000000", + "end_time": "2025-10-09 08:25:42.276000000", + "duration_ms": 3544.302, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What monitoring capabilities does Noveum provide?", + "query_length": 49, + "query_type": "rag_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 3.5442299842834473, + "model.latency_ms": 3544.2299842834473, + "model.start_time": 1759998338.732058, + "model.end_time": 1759998342.276288, + "model.performance_tier": "medium", + "model.prompt_tokens": 821, + "model.completion_tokens": 209, + "model.total_tokens": 1030, + "model.tokens_per_second": 290.613195127697, + "model.estimated_cost": 0.0309, + "model.efficiency_score": 0.8145631067961165, + "model.response_length": 839, + "model.response_quality": "high", + "model.output_response": "Model Response: Noveum provides comprehensive monitoring capabilities for AI applications, including the ability to monitor, trace, and optimize AI agents across various frameworks such as LangChain, CrewAI, AutoGen,...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "dc511122-c0b6-415c-9a49-c7b45132dd87", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-09 08:25:38.322000000", + "trace_end_time": "2025-10-09 08:25:42.822000000", + "trace_duration_ms": 4500.425, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-09 08:25:45.028000000", + "updated_at": "2025-10-09 08:25:45.028000000" + }, + { + "span_id": "f4c03a8d-b90d-4c1a-a259-826d09e3e57a", + "trace_id": "04bebf38-a343-4563-80db-0154bef8d927", + "parent_span_id": "edc52a5d-8450-4d0f-ad14-958ec4d147f4", + "name": "agent.llm_model_execution", + "start_time": "2025-10-09 08:25:34.043000000", + "end_time": "2025-10-09 08:25:36.950000000", + "duration_ms": 2907.183, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: How does Noveum handle agent tracing?", + "query_length": 37, + "query_type": "rag_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 2.906853199005127, + "model.latency_ms": 2906.853199005127, + "model.start_time": 1759998334.043113, + "model.end_time": 1759998336.9499662, + "model.performance_tier": "medium", + "model.prompt_tokens": 817, + "model.completion_tokens": 172, + "model.total_tokens": 989, + "model.tokens_per_second": 340.2304596387895, + "model.estimated_cost": 0.029670000000000002, + "model.efficiency_score": 0.698685540950455, + "model.response_length": 691, + "model.response_quality": "high", + "model.output_response": "Model Response: Noveum.ai handles agent tracing through its specialized Multi-Agent Tracing capabilities, which are designed to observe complex workflows and inter-agent communications within multi-agent systems. The...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "04bebf38-a343-4563-80db-0154bef8d927", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-09 08:25:33.693000000", + "trace_end_time": "2025-10-09 08:25:37.313000000", + "trace_duration_ms": 3619.672, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-09 08:25:40.339000000", + "updated_at": "2025-10-09 08:25:40.339000000" + }, + { + "span_id": "48fb40ba-a39f-4e13-9d94-18794c7e5e2b", + "trace_id": "5e043630-6493-42b5-beb8-79faa19bfa37", + "parent_span_id": "6c362c73-e063-4813-8180-d26b6a52b798", + "name": "agent.llm_model_execution", + "start_time": "2025-10-09 08:25:30.278000000", + "end_time": "2025-10-09 08:25:32.086000000", + "duration_ms": 1807.8110000000001, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What APIs are available in Noveum platform?", + "query_length": 43, + "query_type": "rag_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 1.8077418804168701, + "model.latency_ms": 1807.7418804168701, + "model.start_time": 1759998330.278618, + "model.end_time": 1759998332.08636, + "model.performance_tier": "fast", + "model.prompt_tokens": 705, + "model.completion_tokens": 67, + "model.total_tokens": 772, + "model.tokens_per_second": 427.05211864758854, + "model.estimated_cost": 0.02316, + "model.efficiency_score": 0.34974093264248707, + "model.response_length": 270, + "model.response_quality": "high", + "model.output_response": "Model Response: The provided context does not specify the APIs available in the Noveum platform. For detailed information about the APIs, I recommend checking the official Noveum documentation or the SDK Integration ...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "5e043630-6493-42b5-beb8-79faa19bfa37", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-09 08:25:29.901000000", + "trace_end_time": "2025-10-09 08:25:32.685000000", + "trace_duration_ms": 2783.326, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-09 08:25:34.918000000", + "updated_at": "2025-10-09 08:25:34.918000000" + }, + { + "span_id": "d4d544a0-f052-4889-82eb-4346f2e447a7", + "trace_id": "7da9814d-a2e8-4c4e-b750-68b26bd5fd22", + "parent_span_id": "1393d065-a9a4-4475-808f-2d00b3c68939", + "name": "agent.llm_model_execution", + "start_time": "2025-10-09 08:25:21.471000000", + "end_time": "2025-10-09 08:25:28.496000000", + "duration_ms": 7024.773999999999, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: How do I set up observability with Noveum?", + "query_length": 42, + "query_type": "rag_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 7.024708986282349, + "model.latency_ms": 7024.708986282349, + "model.start_time": 1759998321.471663, + "model.end_time": 1759998328.496372, + "model.performance_tier": "slow", + "model.prompt_tokens": 852, + "model.completion_tokens": 237, + "model.total_tokens": 1089, + "model.tokens_per_second": 155.02421554068192, + "model.estimated_cost": 0.03267, + "model.efficiency_score": 0.8732782369146006, + "model.response_length": 951, + "model.response_quality": "high", + "model.output_response": "Model Response: To set up observability with Noveum, follow these steps:\n\n1. **Sign Up**: Go to [noveum.ai](https://noveum.ai) and create an account.\n2. **Create a Project**: Once signed in, create a new project to o...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "7da9814d-a2e8-4c4e-b750-68b26bd5fd22", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-09 08:25:20.586000000", + "trace_end_time": "2025-10-09 08:25:28.897000000", + "trace_duration_ms": 8310.205, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-09 08:25:29.588000000", + "updated_at": "2025-10-09 08:25:29.588000000" + }, + { + "span_id": "304f9e3d-9063-4661-9777-618efc49f5c1", + "trace_id": "16143f74-2831-4753-b33d-ce4b645093c5", + "parent_span_id": "f419377a-fc24-4abf-8a1c-03857824e374", + "name": "agent.llm_model_execution", + "start_time": "2025-10-09 08:25:12.892000000", + "end_time": "2025-10-09 08:25:19.220000000", + "duration_ms": 6327.166, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What features does Noveum Trace offer?", + "query_length": 38, + "query_type": "rag_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 6.327087163925171, + "model.latency_ms": 6327.087163925171, + "model.start_time": 1759998312.892979, + "model.end_time": 1759998319.220066, + "model.performance_tier": "slow", + "model.prompt_tokens": 830, + "model.completion_tokens": 253, + "model.total_tokens": 1083, + "model.tokens_per_second": 171.16881306375637, + "model.estimated_cost": 0.03249, + "model.efficiency_score": 0.9362880886426593, + "model.response_length": 1014, + "model.response_quality": "high", + "model.output_response": "Model Response: Noveum Trace offers several key features through its Python SDK (`noveum-trace`) and TypeScript SDK (`@noveum/trace`):\n\n1. **Decorator-based Tracing**: It provides a simple way to trace LLM calls, age...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "16143f74-2831-4753-b33d-ce4b645093c5", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-09 08:25:12.442000000", + "trace_end_time": "2025-10-09 08:25:19.579000000", + "trace_duration_ms": 7137.402, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-09 08:25:20.290000000", + "updated_at": "2025-10-09 08:25:20.290000000" + }, + { + "span_id": "e0b4ccfb-18e7-4d71-b2dd-5b807eb27513", + "trace_id": "fc64e6cc-6739-4256-ac4a-7b80c3028233", + "parent_span_id": "0e58e31f-a8ab-4aee-8fc2-abcae1e21d9d", + "name": "agent.llm_model_execution", + "start_time": "2025-10-09 08:25:08.383000000", + "end_time": "2025-10-09 08:25:10.938000000", + "duration_ms": 2555.3149999999996, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What are Noveum's pricing plans?", + "query_length": 32, + "query_type": "rag_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 2.5551459789276123, + "model.latency_ms": 2555.1459789276123, + "model.start_time": 1759998308.383137, + "model.end_time": 1759998310.938283, + "model.performance_tier": "medium", + "model.prompt_tokens": 658, + "model.completion_tokens": 99, + "model.total_tokens": 757, + "model.tokens_per_second": 296.264873413499, + "model.estimated_cost": 0.02271, + "model.efficiency_score": 0.5270805812417437, + "model.response_length": 399, + "model.response_quality": "high", + "model.output_response": "Model Response: The provided context does not include specific information about Noveum's pricing plans. However, it mentions that all users will have access to the Observability suite for free, and early users will ...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "fc64e6cc-6739-4256-ac4a-7b80c3028233", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-09 08:25:07.932000000", + "trace_end_time": "2025-10-09 08:25:11.438000000", + "trace_duration_ms": 3506.0989999999997, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-09 08:25:12.651000000", + "updated_at": "2025-10-09 08:25:12.651000000" + }, + { + "span_id": "f1425131-f8a2-43f7-aa17-d009f745ed06", + "trace_id": "b7945c49-f584-4c70-972d-536a805d8a31", + "parent_span_id": "48155aad-6204-4a52-8624-8fa5d6fe407f", + "name": "agent.llm_model_execution", + "start_time": "2025-10-09 08:24:58.886000000", + "end_time": "2025-10-09 08:25:05.872000000", + "duration_ms": 6986.124, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: How do I integrate Noveum Trace in my application?", + "query_length": 50, + "query_type": "rag_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 6.985715866088867, + "model.latency_ms": 6985.715866088867, + "model.start_time": 1759998298.886451, + "model.end_time": 1759998305.8721669, + "model.performance_tier": "slow", + "model.prompt_tokens": 837, + "model.completion_tokens": 287, + "model.total_tokens": 1124, + "model.tokens_per_second": 160.89975910075202, + "model.estimated_cost": 0.03372, + "model.efficiency_score": 1.0240213523131672, + "model.response_length": 1151, + "model.response_quality": "high", + "model.output_response": "Model Response: To integrate Noveum Trace into your application, you can use either the Python SDK (`noveum-trace`) or the TypeScript SDK (`@noveum/trace`). Hereā€™s a quick start guide based on the provided documentat...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "b7945c49-f584-4c70-972d-536a805d8a31", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-09 08:24:58.518000000", + "trace_end_time": "2025-10-09 08:25:06.920000000", + "trace_duration_ms": 8402.516, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-09 08:25:07.620000000", + "updated_at": "2025-10-09 08:25:07.620000000" + }, + { + "span_id": "f41e836c-1a82-46e0-9d51-011713874e94", + "trace_id": "f5c40ecf-36c0-45ba-9cc9-dc0329b0324b", + "parent_span_id": "57ded2eb-0f3f-474e-8704-5ad9974ab4bb", + "name": "agent.llm_model_execution", + "start_time": "2025-10-09 08:24:54.382000000", + "end_time": "2025-10-09 08:24:56.965000000", + "duration_ms": 2583.161, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What is Noveum and what does it do?", + "query_length": 35, + "query_type": "rag_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 2.5830719470977783, + "model.latency_ms": 2583.0719470977783, + "model.start_time": 1759998294.382122, + "model.end_time": 1759998296.965194, + "model.performance_tier": "medium", + "model.prompt_tokens": 716, + "model.completion_tokens": 152, + "model.total_tokens": 868, + "model.tokens_per_second": 336.03400051448244, + "model.estimated_cost": 0.02604, + "model.efficiency_score": 0.7039170506912442, + "model.response_length": 611, + "model.response_quality": "high", + "model.output_response": "Model Response: Noveum.ai is a comprehensive tracing and observability platform specifically designed for AI applications, including those powered by large language models (LLMs), retrieval-augmented generation (RAG)...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "f5c40ecf-36c0-45ba-9cc9-dc0329b0324b", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-09 08:24:54.009000000", + "trace_end_time": "2025-10-09 08:24:57.511000000", + "trace_duration_ms": 3502.062, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-09 08:24:59.716000000", + "updated_at": "2025-10-09 08:24:59.716000000" + } +] \ No newline at end of file diff --git a/noveum_customer_support_bt/split_datasets/agent.query_routing_dataset.json b/noveum_customer_support_bt/split_datasets/agent.query_routing_dataset.json new file mode 100644 index 0000000..a2b0f8a --- /dev/null +++ b/noveum_customer_support_bt/split_datasets/agent.query_routing_dataset.json @@ -0,0 +1,1222 @@ +[ + { + "span_id": "0b19befc-5dc0-4b00-9a8b-5f2f2c914f17", + "trace_id": "eda4fe22-9a2b-4b73-856b-f4f3309bf719", + "parent_span_id": "b6b7c1c0-3b7b-4e19-805d-a16beb7ea871", + "name": "agent.query_routing", + "start_time": "2025-10-09 08:26:46.384000000", + "end_time": "2025-10-09 08:26:48.685000000", + "duration_ms": 2301.289, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What are the latest breakthroughs in artificial intelligence?", + "query_length": 61, + "query_type": "routing_query", + "output_response": "Routed to Web Search for query processing", + "router.classification": "Web Search", + "router.confidence_score": 1, + "router.rag_keyword_score": 0, + "router.web_keyword_score": 1, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "eda4fe22-9a2b-4b73-856b-f4f3309bf719", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-09 08:26:46.384000000", + "trace_end_time": "2025-10-09 08:26:48.685000000", + "trace_duration_ms": 2301.488, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-09 08:26:49.422000000", + "updated_at": "2025-10-09 08:26:49.422000000" + }, + { + "span_id": "30644d2a-5ab1-4a73-b4ff-cf9fa478e36b", + "trace_id": "0ffffba1-8a37-443c-8866-d53ffbfa7718", + "parent_span_id": "3045d971-b7ff-46cc-be11-6db7d65dfefd", + "name": "agent.query_routing", + "start_time": "2025-10-09 08:26:41.850000000", + "end_time": "2025-10-09 08:26:45.377000000", + "duration_ms": 3527.556, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What's happening in the software development world today?", + "query_length": 57, + "query_type": "routing_query", + "output_response": "Routed to Web Search for query processing", + "router.classification": "Web Search", + "router.confidence_score": 1, + "router.rag_keyword_score": 0, + "router.web_keyword_score": 1, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "0ffffba1-8a37-443c-8866-d53ffbfa7718", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-09 08:26:41.850000000", + "trace_end_time": "2025-10-09 08:26:45.377000000", + "trace_duration_ms": 3527.831, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-09 08:26:49.422000000", + "updated_at": "2025-10-09 08:26:49.422000000" + }, + { + "span_id": "89eac2de-b7da-4db3-bf7c-8e87c429481e", + "trace_id": "f1f37bd7-0851-4659-b493-b80d3800d920", + "parent_span_id": "42d4cb1f-17ec-433a-8087-90292c2fa9d9", + "name": "agent.query_routing", + "start_time": "2025-10-09 08:26:35.915000000", + "end_time": "2025-10-09 08:26:40.841000000", + "duration_ms": 4926.115000000001, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What are the newest features in cloud computing?", + "query_length": 48, + "query_type": "routing_query", + "output_response": "Routed to RAG for query processing", + "router.classification": "RAG", + "router.confidence_score": 1, + "router.rag_keyword_score": 1, + "router.web_keyword_score": 0, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "f1f37bd7-0851-4659-b493-b80d3800d920", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-09 08:26:35.915000000", + "trace_end_time": "2025-10-09 08:26:40.841000000", + "trace_duration_ms": 4926.342, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-09 08:26:44.053000000", + "updated_at": "2025-10-09 08:26:44.053000000" + }, + { + "span_id": "962afebb-26d1-45ff-acf1-218163cd6bb8", + "trace_id": "43cdf081-4f01-49cd-b566-dbd1619e6cd2", + "parent_span_id": "e22b809d-5320-40b7-b78a-e3e125538a6b", + "name": "agent.query_routing", + "start_time": "2025-10-09 08:26:32.240000000", + "end_time": "2025-10-09 08:26:34.906000000", + "duration_ms": 2666.057, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What's the current status of cryptocurrency markets?", + "query_length": 52, + "query_type": "routing_query", + "output_response": "Routed to Web Search for query processing", + "router.classification": "Web Search", + "router.confidence_score": 1, + "router.rag_keyword_score": 0, + "router.web_keyword_score": 3, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "43cdf081-4f01-49cd-b566-dbd1619e6cd2", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-09 08:26:32.237000000", + "trace_end_time": "2025-10-09 08:26:34.906000000", + "trace_duration_ms": 2668.841, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-09 08:26:38.618000000", + "updated_at": "2025-10-09 08:26:38.618000000" + }, + { + "span_id": "246cf822-4562-4d3a-8a5a-e38e0a0b02a0", + "trace_id": "9a1983f4-09da-4b53-80e6-38de6878e0e7", + "parent_span_id": "813a4450-1cf8-4d43-8493-eabfa200919b", + "name": "agent.query_routing", + "start_time": "2025-10-09 08:26:27.992000000", + "end_time": "2025-10-09 08:26:31.231000000", + "duration_ms": 3239.343, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What are the latest updates in Python programming?", + "query_length": 50, + "query_type": "routing_query", + "output_response": "Routed to Web Search for query processing", + "router.classification": "Web Search", + "router.confidence_score": 1, + "router.rag_keyword_score": 0, + "router.web_keyword_score": 2, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "9a1983f4-09da-4b53-80e6-38de6878e0e7", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-09 08:26:27.992000000", + "trace_end_time": "2025-10-09 08:26:31.231000000", + "trace_duration_ms": 3239.706, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-09 08:26:33.463000000", + "updated_at": "2025-10-09 08:26:33.463000000" + }, + { + "span_id": "20544f91-8828-4a0e-b253-ed1f8b95b205", + "trace_id": "5d2517e1-220a-429d-9d59-f701bda25eed", + "parent_span_id": "6102352a-833e-493d-adbb-887e2af17918", + "name": "agent.query_routing", + "start_time": "2025-10-09 08:26:23.210000000", + "end_time": "2025-10-09 08:26:26.985000000", + "duration_ms": 3774.906, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What happened in tech news this week?", + "query_length": 37, + "query_type": "routing_query", + "output_response": "Routed to Web Search for query processing", + "router.classification": "Web Search", + "router.confidence_score": 1, + "router.rag_keyword_score": 0, + "router.web_keyword_score": 2, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "5d2517e1-220a-429d-9d59-f701bda25eed", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-09 08:26:23.210000000", + "trace_end_time": "2025-10-09 08:26:26.985000000", + "trace_duration_ms": 3775.124, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-09 08:26:28.174000000", + "updated_at": "2025-10-09 08:26:28.174000000" + }, + { + "span_id": "5359c861-c51b-446d-9ad2-b3d8c21ea70f", + "trace_id": "52aacb67-c361-4445-9b72-c157f79f47d6", + "parent_span_id": "2926418a-fe09-4df0-bf19-a527bc24a478", + "name": "agent.query_routing", + "start_time": "2025-10-09 08:26:16.544000000", + "end_time": "2025-10-09 08:26:22.201000000", + "duration_ms": 5656.666, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What are the current trends in observability tools?", + "query_length": 51, + "query_type": "routing_query", + "output_response": "Routed to RAG for query processing", + "router.classification": "RAG", + "router.confidence_score": 0.3333333333333333, + "router.rag_keyword_score": 2, + "router.web_keyword_score": 1, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "52aacb67-c361-4445-9b72-c157f79f47d6", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-09 08:26:16.544000000", + "trace_end_time": "2025-10-09 08:26:22.201000000", + "trace_duration_ms": 5657.012, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-09 08:26:22.904000000", + "updated_at": "2025-10-09 08:26:22.904000000" + }, + { + "span_id": "acf6a99a-39e0-4f87-a07e-3a61213ab032", + "trace_id": "a81ca3a8-80aa-4c39-876e-8d40ea7a0aef", + "parent_span_id": "afe85a1d-13c8-4476-8154-52d58ba83d52", + "name": "agent.query_routing", + "start_time": "2025-10-09 08:26:08.903000000", + "end_time": "2025-10-09 08:26:15.537000000", + "duration_ms": 6633.4220000000005, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "Tell me about recent developments in machine learning", + "query_length": 53, + "query_type": "routing_query", + "output_response": "Routed to Web Search for query processing", + "router.classification": "Web Search", + "router.confidence_score": 1, + "router.rag_keyword_score": 0, + "router.web_keyword_score": 1, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "a81ca3a8-80aa-4c39-876e-8d40ea7a0aef", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-09 08:26:08.903000000", + "trace_end_time": "2025-10-09 08:26:15.537000000", + "trace_duration_ms": 6633.662, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-09 08:26:16.227000000", + "updated_at": "2025-10-09 08:26:16.227000000" + }, + { + "span_id": "0b47ea07-1eda-4b6a-a183-ea9ef047e431", + "trace_id": "230aad27-f3dd-4968-a45a-3c2f07ac28ed", + "parent_span_id": "db8da393-6d72-421e-87dc-96f56b6576e5", + "name": "agent.query_routing", + "start_time": "2025-10-09 08:26:02.424000000", + "end_time": "2025-10-09 08:26:07.894000000", + "duration_ms": 5469.894, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What's the weather like today?", + "query_length": 30, + "query_type": "routing_query", + "output_response": "Routed to Web Search for query processing", + "router.classification": "Web Search", + "router.confidence_score": 1, + "router.rag_keyword_score": 0, + "router.web_keyword_score": 2, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "230aad27-f3dd-4968-a45a-3c2f07ac28ed", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-09 08:26:02.424000000", + "trace_end_time": "2025-10-09 08:26:07.894000000", + "trace_duration_ms": 5470.232, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-09 08:26:08.586000000", + "updated_at": "2025-10-09 08:26:08.586000000" + }, + { + "span_id": "3866ddb3-b17e-47a1-8b3f-bca1e32a2ae4", + "trace_id": "83c7dcce-3d89-4da1-8b3f-d419885d4cbc", + "parent_span_id": "a290202d-1b3f-431f-8728-fcb937380f8c", + "name": "agent.query_routing", + "start_time": "2025-10-09 08:25:56.886000000", + "end_time": "2025-10-09 08:26:01.415000000", + "duration_ms": 4529.776, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What are the latest AI news today?", + "query_length": 34, + "query_type": "routing_query", + "output_response": "Routed to Web Search for query processing", + "router.classification": "Web Search", + "router.confidence_score": 1, + "router.rag_keyword_score": 0, + "router.web_keyword_score": 3, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "83c7dcce-3d89-4da1-8b3f-d419885d4cbc", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-09 08:25:56.886000000", + "trace_end_time": "2025-10-09 08:26:01.416000000", + "trace_duration_ms": 4530.08, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-09 08:26:02.122000000", + "updated_at": "2025-10-09 08:26:02.122000000" + }, + { + "span_id": "3ce2e35e-4518-43b4-8a76-4413da6832ea", + "trace_id": "2218f641-604c-491a-9710-b51a9941b982", + "parent_span_id": "5fbaef83-95fc-4bed-9695-9d69c37c35e4", + "name": "agent.query_routing", + "start_time": "2025-10-09 08:25:48.107000000", + "end_time": "2025-10-09 08:25:55.878000000", + "duration_ms": 7770.734, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What are the benefits of using Noveum Trace?", + "query_length": 44, + "query_type": "routing_query", + "output_response": "Routed to RAG for query processing", + "router.classification": "RAG", + "router.confidence_score": 1, + "router.rag_keyword_score": 2, + "router.web_keyword_score": 0, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "2218f641-604c-491a-9710-b51a9941b982", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-09 08:25:48.107000000", + "trace_end_time": "2025-10-09 08:25:55.878000000", + "trace_duration_ms": 7770.953, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-09 08:25:56.565000000", + "updated_at": "2025-10-09 08:25:56.565000000" + }, + { + "span_id": "c0c39e8b-9a42-4c18-8737-9224cd8a45c0", + "trace_id": "255fd49c-84b4-4b18-887e-6308a412d535", + "parent_span_id": "b3ac3924-928b-4d48-a2a7-aa749802fef9", + "name": "agent.query_routing", + "start_time": "2025-10-09 08:25:43.830000000", + "end_time": "2025-10-09 08:25:47.099000000", + "duration_ms": 3268.6980000000003, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "How do I configure Noveum for my system?", + "query_length": 40, + "query_type": "routing_query", + "output_response": "Routed to RAG for query processing", + "router.classification": "RAG", + "router.confidence_score": 1, + "router.rag_keyword_score": 2, + "router.web_keyword_score": 0, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "255fd49c-84b4-4b18-887e-6308a412d535", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-09 08:25:43.830000000", + "trace_end_time": "2025-10-09 08:25:47.099000000", + "trace_duration_ms": 3268.862, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-09 08:25:50.304000000", + "updated_at": "2025-10-09 08:25:50.304000000" + }, + { + "span_id": "80987c51-ea82-4a41-97c7-ed591082b9bf", + "trace_id": "dc511122-c0b6-415c-9a49-c7b45132dd87", + "parent_span_id": "b8055ffc-ee7d-46ad-b6a0-84e80f9399e2", + "name": "agent.query_routing", + "start_time": "2025-10-09 08:25:38.322000000", + "end_time": "2025-10-09 08:25:42.822000000", + "duration_ms": 4500.249, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What monitoring capabilities does Noveum provide?", + "query_length": 49, + "query_type": "routing_query", + "output_response": "Routed to RAG for query processing", + "router.classification": "RAG", + "router.confidence_score": 1, + "router.rag_keyword_score": 2, + "router.web_keyword_score": 0, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "dc511122-c0b6-415c-9a49-c7b45132dd87", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-09 08:25:38.322000000", + "trace_end_time": "2025-10-09 08:25:42.822000000", + "trace_duration_ms": 4500.425, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-09 08:25:45.028000000", + "updated_at": "2025-10-09 08:25:45.028000000" + }, + { + "span_id": "2f89c1e0-9b62-4651-914c-b71cf561c33c", + "trace_id": "04bebf38-a343-4563-80db-0154bef8d927", + "parent_span_id": "532b9e1f-5c4f-4b41-98bb-e9a1510f0f1c", + "name": "agent.query_routing", + "start_time": "2025-10-09 08:25:33.693000000", + "end_time": "2025-10-09 08:25:37.313000000", + "duration_ms": 3619.46, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "How does Noveum handle agent tracing?", + "query_length": 37, + "query_type": "routing_query", + "output_response": "Routed to RAG for query processing", + "router.classification": "RAG", + "router.confidence_score": 1, + "router.rag_keyword_score": 2, + "router.web_keyword_score": 0, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "04bebf38-a343-4563-80db-0154bef8d927", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-09 08:25:33.693000000", + "trace_end_time": "2025-10-09 08:25:37.313000000", + "trace_duration_ms": 3619.672, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-09 08:25:40.339000000", + "updated_at": "2025-10-09 08:25:40.339000000" + }, + { + "span_id": "e30e3e61-6edd-4b62-ab77-627faa2bf741", + "trace_id": "5e043630-6493-42b5-beb8-79faa19bfa37", + "parent_span_id": "14064e3f-f9c9-4bb7-b8cb-1ad52938b075", + "name": "agent.query_routing", + "start_time": "2025-10-09 08:25:29.901000000", + "end_time": "2025-10-09 08:25:32.685000000", + "duration_ms": 2783.166, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What APIs are available in Noveum platform?", + "query_length": 43, + "query_type": "routing_query", + "output_response": "Routed to RAG for query processing", + "router.classification": "RAG", + "router.confidence_score": 1, + "router.rag_keyword_score": 3, + "router.web_keyword_score": 0, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "5e043630-6493-42b5-beb8-79faa19bfa37", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-09 08:25:29.901000000", + "trace_end_time": "2025-10-09 08:25:32.685000000", + "trace_duration_ms": 2783.326, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-09 08:25:34.918000000", + "updated_at": "2025-10-09 08:25:34.918000000" + }, + { + "span_id": "2f513795-764f-4eb6-8dd2-108d4cdf8432", + "trace_id": "7da9814d-a2e8-4c4e-b750-68b26bd5fd22", + "parent_span_id": "a74fe8a9-2654-4b9d-ba87-dc2f86b6466d", + "name": "agent.query_routing", + "start_time": "2025-10-09 08:25:20.587000000", + "end_time": "2025-10-09 08:25:28.897000000", + "duration_ms": 8310.025, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "How do I set up observability with Noveum?", + "query_length": 42, + "query_type": "routing_query", + "output_response": "Routed to RAG for query processing", + "router.classification": "RAG", + "router.confidence_score": 1, + "router.rag_keyword_score": 2, + "router.web_keyword_score": 0, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "7da9814d-a2e8-4c4e-b750-68b26bd5fd22", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-09 08:25:20.586000000", + "trace_end_time": "2025-10-09 08:25:28.897000000", + "trace_duration_ms": 8310.205, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-09 08:25:29.588000000", + "updated_at": "2025-10-09 08:25:29.588000000" + }, + { + "span_id": "29c7a515-2383-4b7d-8fc7-beac3f403641", + "trace_id": "16143f74-2831-4753-b33d-ce4b645093c5", + "parent_span_id": "c700d0f4-f1f8-4339-aa42-a3060ae2ae11", + "name": "agent.query_routing", + "start_time": "2025-10-09 08:25:12.442000000", + "end_time": "2025-10-09 08:25:19.579000000", + "duration_ms": 7137.247, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What features does Noveum Trace offer?", + "query_length": 38, + "query_type": "routing_query", + "output_response": "Routed to RAG for query processing", + "router.classification": "RAG", + "router.confidence_score": 1, + "router.rag_keyword_score": 3, + "router.web_keyword_score": 0, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "16143f74-2831-4753-b33d-ce4b645093c5", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-09 08:25:12.442000000", + "trace_end_time": "2025-10-09 08:25:19.579000000", + "trace_duration_ms": 7137.402, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-09 08:25:20.290000000", + "updated_at": "2025-10-09 08:25:20.290000000" + }, + { + "span_id": "12fee687-2a2e-401f-83c2-9c8dc7ecd853", + "trace_id": "fc64e6cc-6739-4256-ac4a-7b80c3028233", + "parent_span_id": "f7f5981f-d840-4265-8970-d319b876366d", + "name": "agent.query_routing", + "start_time": "2025-10-09 08:25:07.932000000", + "end_time": "2025-10-09 08:25:11.438000000", + "duration_ms": 3505.9610000000002, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What are Noveum's pricing plans?", + "query_length": 32, + "query_type": "routing_query", + "output_response": "Routed to RAG for query processing", + "router.classification": "RAG", + "router.confidence_score": 1, + "router.rag_keyword_score": 3, + "router.web_keyword_score": 0, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "fc64e6cc-6739-4256-ac4a-7b80c3028233", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-09 08:25:07.932000000", + "trace_end_time": "2025-10-09 08:25:11.438000000", + "trace_duration_ms": 3506.0989999999997, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-09 08:25:12.651000000", + "updated_at": "2025-10-09 08:25:12.651000000" + }, + { + "span_id": "9c5aec32-b6eb-4c42-93d5-3970fb118c00", + "trace_id": "b7945c49-f584-4c70-972d-536a805d8a31", + "parent_span_id": "0c7bbe2b-18af-4fdb-a5bf-4a91daa3bf50", + "name": "agent.query_routing", + "start_time": "2025-10-09 08:24:58.518000000", + "end_time": "2025-10-09 08:25:06.920000000", + "duration_ms": 8402.101999999999, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "How do I integrate Noveum Trace in my application?", + "query_length": 50, + "query_type": "routing_query", + "output_response": "Routed to RAG for query processing", + "router.classification": "RAG", + "router.confidence_score": 1, + "router.rag_keyword_score": 2, + "router.web_keyword_score": 0, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "b7945c49-f584-4c70-972d-536a805d8a31", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-09 08:24:58.518000000", + "trace_end_time": "2025-10-09 08:25:06.920000000", + "trace_duration_ms": 8402.516, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-09 08:25:07.620000000", + "updated_at": "2025-10-09 08:25:07.620000000" + }, + { + "span_id": "f132d053-f82c-4e66-adf4-bd7ce68b660d", + "trace_id": "f5c40ecf-36c0-45ba-9cc9-dc0329b0324b", + "parent_span_id": "dc869742-6797-4016-8061-37530fc7d433", + "name": "agent.query_routing", + "start_time": "2025-10-09 08:24:54.009000000", + "end_time": "2025-10-09 08:24:57.511000000", + "duration_ms": 3501.8790000000004, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What is Noveum and what does it do?", + "query_length": 35, + "query_type": "routing_query", + "output_response": "Routed to RAG for query processing", + "router.classification": "RAG", + "router.confidence_score": 1, + "router.rag_keyword_score": 1, + "router.web_keyword_score": 0, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "f5c40ecf-36c0-45ba-9cc9-dc0329b0324b", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-09 08:24:54.009000000", + "trace_end_time": "2025-10-09 08:24:57.511000000", + "trace_duration_ms": 3502.062, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-09 08:24:59.716000000", + "updated_at": "2025-10-09 08:24:59.716000000" + } +] \ No newline at end of file diff --git a/noveum_customer_support_bt/split_datasets/agent.rag_evaluation_metrics_dataset.json b/noveum_customer_support_bt/split_datasets/agent.rag_evaluation_metrics_dataset.json new file mode 100644 index 0000000..b8091d7 --- /dev/null +++ b/noveum_customer_support_bt/split_datasets/agent.rag_evaluation_metrics_dataset.json @@ -0,0 +1,998 @@ +[ + { + "span_id": "ac166548-ffa5-47b9-8cf7-4787f270cb5a", + "trace_id": "f1f37bd7-0851-4659-b493-b80d3800d920", + "parent_span_id": "ea4d61c5-b8dd-4677-b21b-2b17170e2163", + "name": "agent.rag_evaluation_metrics", + "start_time": "2025-10-09 08:26:40.446000000", + "end_time": "2025-10-09 08:26:40.446000000", + "duration_ms": 0.08399999999999999, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "rag_evaluation_metrics", + "agent.capabilities": [ + "retrieval_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What are the newest features in cloud computing?", + "query_length": 48, + "query_type": "rag_evaluation_query", + "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/changelog):\n- šŸ“ˆ Enhanced Dashboard Analytics: Real-time request tracking, latency monitoring, and cost analysis\n- šŸ” Improved Logs Interface: Better search functionality, detailed trace views, and enhanced debugging capabilities\n- šŸ‘„ Team Management: Advanced member mana...", + "retrieval.context_length": 2579, + "retrieval.sources_count": 5, + "retrieval.context_quality": "high", + "retrieval.effectiveness": 1, + "retrieval.context_utilization": 2.579, + "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/changelog):\n- šŸ“ˆ Enhanced Dashboard Analytics: Real-time request tracking, latency monitoring, and cost analysis\n- šŸ” Improved Logs Interface: Better search functionality, detailed trace views, and enhanced debugging capabilities\n- šŸ‘„ Team Management: Advanced member management with role-based access control and invitation system\n- šŸ›”ļø Security Improvements: Enhanced API key management and secure credential storage\n4 months ago\n- šŸ“Š Advanced Metrics Collection: Comprehensive telemetry system with custom metrics ...\n\nSource 2 (https://noveum.ai/en/changelog):\n- šŸ“ Documentation Updates: Comprehensive guides for integration and best practices\n5 months ago\n- šŸŽ‰ Platform Foundation: Initial release of Noveum.ai AI observability platform\n- šŸš€ Core Features: AI Gateway integration, basic evaluation metrics, and monitoring capabilities\n- šŸ“ˆ Evaluation Jobs: Automated model evaluation with configurable metrics and reporting\n- šŸ”Œ Provider Integrations: Support for major AI providers and custom model deployments...\n\nSource 3 (https://noveum.ai/docs/platform/dashboard):\n- Bulk Export: Download multiple traces simultaneously\n- Comparative Analysis: Compare performance across multiple traces\nReal-Time Updates\n- Live Refresh: Automatic updates as new traces arrive\n- Connection Monitoring: Real-time status of your trace ingestion pipeline\n- Performance Indicators: Live metrics for system health monitoring\nReady to dive deeper? Explore Projects & Environments to organize your AI applications, or check out Team Collaboration to share insights with your team.\nGet Earl...\n\nSource 4 (https://noveum.ai/en/changelog):\nChangelog\nStay up to date with the latest changes in our product.\n2 months ago\n- šŸŽÆ Major Platform Evolution: NovaEval Framework - Launched comprehensive AI model evaluation framework with 20+ built-in scorers including accuracy, RAG metrics, conversational scoring, and specialized agent evaluation tools. Features production-grade deployment on Docker and Kubernetes. (GitHub | PyPI Package)\n- šŸš€ Next-Generation Traces UI - Complete redesign featuring three-pane layout with directory tree navigatio...\n\nSource 5 (https://noveum.ai/en/blog):\n8/12/2025\nDiscover how Noveum.ai provides comprehensive tracing and observability for AI applications, from development debugging to production optimization.\nShashank Agarwal\n3/3/2025\nDiscover how Noveum.ai provides comprehensive tracing and observability for LLM applications, RAG systems, and multi-agent workflows with our powerful Python and TypeScript SDKs.\nShashank Agarwal\n3/2/2025...\n\nUser Question: What are the newest features in cloud computing?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", + "prompt.prompt_length": 3123, + "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/changelog):\n- šŸ“ˆ Enhanced Dashboard Analytics: Real-time request tracking, latency monitoring, and cost analysis\n- šŸ” Improved Logs Interface: Better search functionality,...", + "prompt.instruction_following": "rag_optimized", + "response.answer_length": 883, + "response.answer_completeness": "complete", + "response.response_quality": "high", + "response.source_citation": 5, + "response.context_utilization": 0.3423807677394339, + "output_response": "RAG Answer: The provided context does not specifically mention any new features in cloud computing as a general topic. However, it does highlight several recent enhancements and features related to the Noveum.ai ...", + "evaluation.retrieval_effectiveness": 1, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 5, + "evaluation.context_relevance": "high", + "evaluation.overall_quality": "high", + "evaluation.ready_for_production": true, + "rag.retrieval_strategy": "semantic_similarity", + "rag.vector_search_results": 5, + "rag.context_synthesis": "multi_source", + "rag.document_coverage": 1, + "rag.information_density": 0.3423807677394339 + }, + "events": [], + "links": [], + "trace_trace_id": "f1f37bd7-0851-4659-b493-b80d3800d920", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-09 08:26:35.915000000", + "trace_end_time": "2025-10-09 08:26:40.841000000", + "trace_duration_ms": 4926.342, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-09 08:26:44.053000000", + "updated_at": "2025-10-09 08:26:44.053000000" + }, + { + "span_id": "e193cd4f-cca3-4b94-95d3-a1f2446b9fc6", + "trace_id": "52aacb67-c361-4445-9b72-c157f79f47d6", + "parent_span_id": "7505f97a-cbc0-42fe-8a72-95387d723daa", + "name": "agent.rag_evaluation_metrics", + "start_time": "2025-10-09 08:26:21.807000000", + "end_time": "2025-10-09 08:26:21.807000000", + "duration_ms": 0.063, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "rag_evaluation_metrics", + "agent.capabilities": [ + "retrieval_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What are the current trends in observability tools?", + "query_length": 51, + "query_type": "rag_evaluation_query", + "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/blog/from-logs-to-intelligent-choices-inside-noveum-ais-evaluation-process):\nFrom Development to Production - Inside Noveum.ai's AI Observability Platform\nShashank Agarwal\n3/3/2025\nIntroduction\nEvery AI application tells a storyā€”but most developers never get to hear it...", + "retrieval.context_length": 2943, + "retrieval.sources_count": 5, + "retrieval.context_quality": "high", + "retrieval.effectiveness": 1, + "retrieval.context_utilization": 2.943, + "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/blog/from-logs-to-intelligent-choices-inside-noveum-ais-evaluation-process):\nFrom Development to Production - Inside Noveum.ai's AI Observability Platform\nShashank Agarwal\n3/3/2025\nIntroduction\nEvery AI application tells a storyā€”but most developers never get to hear it. When your RAG pipeline returns irrelevant results, when your multi-agent system gets stuck in loops, or when your LLM costs suddenly spike, you're left guessing what went wrong and where.\nThat's exactly the challenge Noveum.ai solves. Rather than flying blind, our platform provides comprehensive tracing a...\n\nSource 2 (https://noveum.ai/docs/getting-started/tracing-concepts):\n- Explore Framework Integrations - Framework-specific guidance\n- Learn Advanced Patterns - Custom instrumentation techniques\n- Master the Dashboard - Analyze your traces effectively\nRemember: Good observability is not about collecting all possible data, but about collecting the right data that helps you understand, debug, and optimize your AI applications.\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to O...\n\nSource 3 (https://noveum.ai/en/docs/getting-started/tracing-concepts):\n- Explore Framework Integrations - Framework-specific guidance\n- Learn Advanced Patterns - Custom instrumentation techniques\n- Master the Dashboard - Analyze your traces effectively\nRemember: Good observability is not about collecting all possible data, but about collecting the right data that helps you understand, debug, and optimize your AI applications.\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to O...\n\nSource 4 (https://noveum.ai/en/blog/from-logs-to-intelligent-choices-inside-noveum-ais-evaluation-process):\nThe AI Observability Challenge\nWhy Traditional Monitoring Falls Short\nTraditional application monitoring tools weren't built for AI applications. They can tell you if your API is responding, but they can't answer the questions that matter most for AI systems:\n- Why did my RAG pipeline retrieve irrelevant documents?\n- Which LLM calls are driving my costs?\n- How are my agents communicating with each other?\n- What's causing hallucinations in my responses?\n- Why is my embedding generation so slow?\nW...\n\nSource 5 (https://noveum.ai/en/docs/advanced/rag-observability):\nšŸ”§ Best Practices for RAG Observability\n1. Comprehensive Pipeline Tracing\n2. Quality Monitoring\n3. A/B Testing for RAG Components\nšŸŽÆ Advanced RAG Patterns\nMulti-Modal RAG\nConversational RAG\nRAG observability with Noveum.ai provides the deep insights needed to build, optimize, and scale retrieval-augmented generation systems. By implementing comprehensive tracing across retrieval, context preparation, and generation stages, you can ensure your RAG pipeline delivers accurate, relevant, and cost-effe...\n\nUser Question: What are the current trends in observability tools?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", + "prompt.prompt_length": 3490, + "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/blog/from-logs-to-intelligent-choices-inside-noveum-ais-evaluation-process):\nFrom Development to Production - Inside Noveum.ai's AI Observability Platform\nShashank Agarw...", + "prompt.instruction_following": "rag_optimized", + "response.answer_length": 1074, + "response.answer_completeness": "complete", + "response.response_quality": "high", + "response.source_citation": 5, + "response.context_utilization": 0.36493374108053006, + "output_response": "RAG Answer: The provided context does not explicitly outline current trends in observability tools. However, it highlights some key aspects of observability that are particularly relevant to AI applications. \n\n1....", + "evaluation.retrieval_effectiveness": 1, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 5, + "evaluation.context_relevance": "high", + "evaluation.overall_quality": "high", + "evaluation.ready_for_production": true, + "rag.retrieval_strategy": "semantic_similarity", + "rag.vector_search_results": 5, + "rag.context_synthesis": "multi_source", + "rag.document_coverage": 1, + "rag.information_density": 0.36493374108053006 + }, + "events": [], + "links": [], + "trace_trace_id": "52aacb67-c361-4445-9b72-c157f79f47d6", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-09 08:26:16.544000000", + "trace_end_time": "2025-10-09 08:26:22.201000000", + "trace_duration_ms": 5657.012, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-09 08:26:22.904000000", + "updated_at": "2025-10-09 08:26:22.904000000" + }, + { + "span_id": "432e74f3-3291-411b-8d7e-85493868f8cf", + "trace_id": "2218f641-604c-491a-9710-b51a9941b982", + "parent_span_id": "8a6e18a1-babc-4166-926f-188f344d514e", + "name": "agent.rag_evaluation_metrics", + "start_time": "2025-10-09 08:25:55.406000000", + "end_time": "2025-10-09 08:25:55.406000000", + "duration_ms": 0.056, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "rag_evaluation_metrics", + "agent.capabilities": [ + "retrieval_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What are the benefits of using Noveum Trace?", + "query_length": 44, + "query_type": "rag_evaluation_query", + "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/docs/platform/dashboard):\n- Bulk Export: Download multiple traces simultaneously\n- Comparative Analysis: Compare performance across multiple traces\nReal-Time Updates\n- Live Refresh: Automatic updates as new traces arrive\n- Connection Monitoring: Real-time status of your t...", + "retrieval.context_length": 2773, + "retrieval.sources_count": 5, + "retrieval.context_quality": "high", + "retrieval.effectiveness": 1, + "retrieval.context_utilization": 2.773, + "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/docs/platform/dashboard):\n- Bulk Export: Download multiple traces simultaneously\n- Comparative Analysis: Compare performance across multiple traces\nReal-Time Updates\n- Live Refresh: Automatic updates as new traces arrive\n- Connection Monitoring: Real-time status of your trace ingestion pipeline\n- Performance Indicators: Live metrics for system health monitoring\nReady to dive deeper? Explore Projects & Environments to organize your AI applications, or check out Team Collaboration to share insights with your team.\nGet Earl...\n\nSource 2 (https://noveum.ai/en/docs):\nCore Components\n-\nšŸ Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\nšŸ“˜ TypeScript SDK (\n@noveum/trace\n)- Framework integrations for Next.js, Express.js, and Hono\n- TypeScript-first design with full type safety\n- Universal compatibility (Node.js, Edge Runtime, browsers)\n-\nšŸ“Š Noveum Platform\n- Real-time dashboard for analyzing traces and performance\n- Ad...\n\nSource 3 (https://noveum.ai/docs):\nCore Components\n-\nšŸ Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\nšŸ“˜ TypeScript SDK (\n@noveum/trace\n)- Framework integrations for Next.js, Express.js, and Hono\n- TypeScript-first design with full type safety\n- Universal compatibility (Node.js, Edge Runtime, browsers)\n-\nšŸ“Š Noveum Platform\n- Real-time dashboard for analyzing traces and performance\n- Ad...\n\nSource 4 (https://noveum.ai/en/docs/getting-started/overview):\n- Advanced search & filtering across traces and spans\n- Cost analysis and optimization recommendations\n- Team collaboration with shared insights and alerts\nšŸ” What Noveum Traces\nLLM Operations\n- Model calls across all providers (OpenAI, Anthropic, Google, etc.)\n- Token usage and cost calculation\n- Prompt engineering effectiveness\n- Response quality metrics\nRAG Pipelines\n- Document retrieval performance and relevance\n- Embedding generation costs and latency\n- Context assembly and prompt constructi...\n\nSource 5 (https://noveum.ai/docs/getting-started/overview):\n- Advanced search & filtering across traces and spans\n- Cost analysis and optimization recommendations\n- Team collaboration with shared insights and alerts\nšŸ” What Noveum Traces\nLLM Operations\n- Model calls across all providers (OpenAI, Anthropic, Google, etc.)\n- Token usage and cost calculation\n- Prompt engineering effectiveness\n- Response quality metrics\nRAG Pipelines\n- Document retrieval performance and relevance\n- Embedding generation costs and latency\n- Context assembly and prompt constructi...\n\nUser Question: What are the benefits of using Noveum Trace?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", + "prompt.prompt_length": 3313, + "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/docs/platform/dashboard):\n- Bulk Export: Download multiple traces simultaneously\n- Comparative Analysis: Compare performance across multiple traces\nReal-Time Updates\n- Live...", + "prompt.instruction_following": "rag_optimized", + "response.answer_length": 1460, + "response.answer_completeness": "complete", + "response.response_quality": "high", + "response.source_citation": 5, + "response.context_utilization": 0.5265055896141363, + "output_response": "RAG Answer: Using Noveum Trace offers several benefits, including:\n\n1. **Real-Time Monitoring**: The Noveum platform provides a real-time dashboard for analyzing traces and performance, with live updates as new t...", + "evaluation.retrieval_effectiveness": 1, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 5, + "evaluation.context_relevance": "high", + "evaluation.overall_quality": "high", + "evaluation.ready_for_production": true, + "rag.retrieval_strategy": "semantic_similarity", + "rag.vector_search_results": 5, + "rag.context_synthesis": "multi_source", + "rag.document_coverage": 1, + "rag.information_density": 0.5265055896141363 + }, + "events": [], + "links": [], + "trace_trace_id": "2218f641-604c-491a-9710-b51a9941b982", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-09 08:25:48.107000000", + "trace_end_time": "2025-10-09 08:25:55.878000000", + "trace_duration_ms": 7770.953, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-09 08:25:56.565000000", + "updated_at": "2025-10-09 08:25:56.565000000" + }, + { + "span_id": "a0475dd5-ef9f-4eb2-bca4-9be72bf6bfd6", + "trace_id": "255fd49c-84b4-4b18-887e-6308a412d535", + "parent_span_id": "5bec4165-6596-43ac-930b-acfcde1cc02a", + "name": "agent.rag_evaluation_metrics", + "start_time": "2025-10-09 08:25:46.618000000", + "end_time": "2025-10-09 08:25:46.618000000", + "duration_ms": 0.013, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "rag_evaluation_metrics", + "agent.capabilities": [ + "retrieval_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: How do I configure Noveum for my system?", + "query_length": 40, + "query_type": "rag_evaluation_query", + "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/docs):\nWelcome to Noveum.ai Docs\nComprehensive AI tracing and observability for LLM applications, RAG systems, and AI agents\nWelcome to the Noveum.ai documentation! Here you'll find everything you need to integrate, configure, and optimize your AI applications using ou...", + "retrieval.context_length": 2237, + "retrieval.sources_count": 5, + "retrieval.context_quality": "high", + "retrieval.effectiveness": 1, + "retrieval.context_utilization": 2.237, + "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/docs):\nWelcome to Noveum.ai Docs\nComprehensive AI tracing and observability for LLM applications, RAG systems, and AI agents\nWelcome to the Noveum.ai documentation! Here you'll find everything you need to integrate, configure, and optimize your AI applications using our comprehensive tracing and observability platform.\nWe provide powerful SDKs for Python and TypeScript that enable you to trace LLM calls, RAG pipelines, and multi-agent workflows with minimal code changes. Our documentation is organized ...\n\nSource 2 (https://noveum.ai/docs):\nWelcome to Noveum.ai Docs\nComprehensive AI tracing and observability for LLM applications, RAG systems, and AI agents\nWelcome to the Noveum.ai documentation! Here you'll find everything you need to integrate, configure, and optimize your AI applications using our comprehensive tracing and observability platform.\nWe provide powerful SDKs for Python and TypeScript that enable you to trace LLM calls, RAG pipelines, and multi-agent workflows with minimal code changes. Our documentation is organized ...\n\nSource 3 (https://noveum.ai/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nSource 4 (https://noveum.ai/en/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nSource 5 (https://noveum.ai/en/docs):\n- šŸŒ Universal: Works across frameworks, clouds, and deployment models\nšŸ¤ Community & Support\n- šŸ’¬ Discord Community: Join our Discord\n- šŸ“§ Email Support: [email protected]\n- šŸ› Bug Reports: GitHub Issues\n- šŸ“– Knowledge Base: Help Center\nReady to get started? Head to our SDK Integration Guide to begin tracing your AI applications in under 5 minutes!\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability ...\n\nUser Question: How do I configure Noveum for my system?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", + "prompt.prompt_length": 2773, + "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/docs):\nWelcome to Noveum.ai Docs\nComprehensive AI tracing and observability for LLM applications, RAG systems, and AI agents\nWelcome to the Noveum.ai documentation! Here...", + "prompt.instruction_following": "rag_optimized", + "response.answer_length": 613, + "response.answer_completeness": "complete", + "response.response_quality": "high", + "response.source_citation": 5, + "response.context_utilization": 0.27402771569065715, + "output_response": "RAG Answer: To configure Noveum for your system, you can refer to the SDK Integration Guide available in the Noveum documentation. This guide will help you trace your AI applications with minimal code changes. Th...", + "evaluation.retrieval_effectiveness": 1, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 5, + "evaluation.context_relevance": "high", + "evaluation.overall_quality": "high", + "evaluation.ready_for_production": true, + "rag.retrieval_strategy": "semantic_similarity", + "rag.vector_search_results": 5, + "rag.context_synthesis": "multi_source", + "rag.document_coverage": 1, + "rag.information_density": 0.27402771569065715 + }, + "events": [], + "links": [], + "trace_trace_id": "255fd49c-84b4-4b18-887e-6308a412d535", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-09 08:25:43.830000000", + "trace_end_time": "2025-10-09 08:25:47.099000000", + "trace_duration_ms": 3268.862, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-09 08:25:50.304000000", + "updated_at": "2025-10-09 08:25:50.304000000" + }, + { + "span_id": "ed8ad539-f008-4f1a-beb1-12ad5793950e", + "trace_id": "dc511122-c0b6-415c-9a49-c7b45132dd87", + "parent_span_id": "f9f79a07-4eda-41b8-9634-d4460114caad", + "name": "agent.rag_evaluation_metrics", + "start_time": "2025-10-09 08:25:42.276000000", + "end_time": "2025-10-09 08:25:42.276000000", + "duration_ms": 0.041, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "rag_evaluation_metrics", + "agent.capabilities": [ + "retrieval_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What monitoring capabilities does Noveum provide?", + "query_length": 49, + "query_type": "rag_evaluation_query", + "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en):\nMonitor all your AI Agents\nimprove AI Agents today\nNoveum.ai helps you monitor, trace, and optimize your AI applications.\nNoveum.ai works with any AI framework ā€“ LangChain, CrewAI, AutoGen, custom implementations, or direct LLM calls. One dashboard shows everything.\n...", + "retrieval.context_length": 2740, + "retrieval.sources_count": 5, + "retrieval.context_quality": "high", + "retrieval.effectiveness": 1, + "retrieval.context_utilization": 2.74, + "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en):\nMonitor all your AI Agents\nimprove AI Agents today\nNoveum.ai helps you monitor, trace, and optimize your AI applications.\nNoveum.ai works with any AI framework ā€“ LangChain, CrewAI, AutoGen, custom implementations, or direct LLM calls. One dashboard shows everything.\nMonitor, Evaluate, Improve Your AI Agents\nThe control plane for AI agents.\nMonitor Everything, Miss Nothing\nOur lightweight SDKs capture every trace and span across your AI agent ecosystemā€”from simple LLM calls to complex multi-agent...\n\nSource 2 (https://noveum.ai/):\nMonitor all your AI Agents\nimprove AI Agents today\nNoveum.ai helps you monitor, trace, and optimize your AI applications.\nNoveum.ai works with any AI framework ā€“ LangChain, CrewAI, AutoGen, custom implementations, or direct LLM calls. One dashboard shows everything.\nMonitor, Evaluate, Improve Your AI Agents\nThe control plane for AI agents.\nMonitor Everything, Miss Nothing\nOur lightweight SDKs capture every trace and span across your AI agent ecosystemā€”from simple LLM calls to complex multi-agent...\n\nSource 3 (https://noveum.ai):\nMonitor all your AI Agents\nimprove AI Agents today\nNoveum.ai helps you monitor, trace, and optimize your AI applications.\nNoveum.ai works with any AI framework ā€“ LangChain, CrewAI, AutoGen, custom implementations, or direct LLM calls. One dashboard shows everything.\nMonitor, Evaluate, Improve Your AI Agents\nThe control plane for AI agents.\nMonitor Everything, Miss Nothing\nOur lightweight SDKs capture every trace and span across your AI agent ecosystemā€”from simple LLM calls to complex multi-agent...\n\nSource 4 (https://noveum.ai/en/docs/getting-started/overview):\nNoveum.ai Overview\nComprehensive AI tracing and observability platform for LLM applications, RAG systems, and AI agents\nWelcome to Noveum.aiā€”the comprehensive tracing and observability platform built specifically for AI applications. Whether you're building LLM-powered chatbots, RAG systems, multi-agent workflows, or any AI-driven application, Noveum provides the insights you need to understand, debug, and optimize your systems.\nšŸŽÆ Why AI Applications Need Specialized Observability\nTraditional mo...\n\nSource 5 (https://noveum.ai/docs/getting-started/overview):\nNoveum.ai Overview\nComprehensive AI tracing and observability platform for LLM applications, RAG systems, and AI agents\nWelcome to Noveum.aiā€”the comprehensive tracing and observability platform built specifically for AI applications. Whether you're building LLM-powered chatbots, RAG systems, multi-agent workflows, or any AI-driven application, Noveum provides the insights you need to understand, debug, and optimize your systems.\nšŸŽÆ Why AI Applications Need Specialized Observability\nTraditional mo...\n\nUser Question: What monitoring capabilities does Noveum provide?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", + "prompt.prompt_length": 3285, + "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en):\nMonitor all your AI Agents\nimprove AI Agents today\nNoveum.ai helps you monitor, trace, and optimize your AI applications.\nNoveum.ai works with any AI framework ā€“ LangC...", + "prompt.instruction_following": "rag_optimized", + "response.answer_length": 839, + "response.answer_completeness": "complete", + "response.response_quality": "high", + "response.source_citation": 5, + "response.context_utilization": 0.30620437956204377, + "output_response": "RAG Answer: Noveum provides comprehensive monitoring capabilities for AI applications, including the ability to monitor, trace, and optimize AI agents across various frameworks such as LangChain, CrewAI, AutoGen,...", + "evaluation.retrieval_effectiveness": 1, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 5, + "evaluation.context_relevance": "high", + "evaluation.overall_quality": "high", + "evaluation.ready_for_production": true, + "rag.retrieval_strategy": "semantic_similarity", + "rag.vector_search_results": 5, + "rag.context_synthesis": "multi_source", + "rag.document_coverage": 1, + "rag.information_density": 0.30620437956204377 + }, + "events": [], + "links": [], + "trace_trace_id": "dc511122-c0b6-415c-9a49-c7b45132dd87", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-09 08:25:38.322000000", + "trace_end_time": "2025-10-09 08:25:42.822000000", + "trace_duration_ms": 4500.425, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-09 08:25:45.028000000", + "updated_at": "2025-10-09 08:25:45.028000000" + }, + { + "span_id": "c7358391-9ac1-4312-a4b3-79edfa8bc0d3", + "trace_id": "04bebf38-a343-4563-80db-0154bef8d927", + "parent_span_id": "edc52a5d-8450-4d0f-ad14-958ec4d147f4", + "name": "agent.rag_evaluation_metrics", + "start_time": "2025-10-09 08:25:36.952000000", + "end_time": "2025-10-09 08:25:36.952000000", + "duration_ms": 0.18100000000000002, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "rag_evaluation_metrics", + "agent.capabilities": [ + "retrieval_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: How does Noveum handle agent tracing?", + "query_length": 37, + "query_type": "rag_evaluation_query", + "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, vector searches, data retrieval, agent reasoning, and more. Without proper observability, debugging ...", + "retrieval.context_length": 2736, + "retrieval.sources_count": 5, + "retrieval.context_quality": "high", + "retrieval.effectiveness": 1, + "retrieval.context_utilization": 2.736, + "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, vector searches, data retrieval, agent reasoning, and more. Without proper observability, debugging becomes a nightmare, optimization is guesswork, and understanding user interactions is nearly impossible.\nNoveum.ai solves this by providing comprehensive tracing and observability specifically designed for AI applications.\nKey benefits of using Noveum.ai:\n- Complete Visibility: Trace...\n\nSource 2 (https://noveum.ai/docs/advanced/multi-agent-tracing):\nMulti-Agent Tracing\nObserve complex agent workflows and inter-agent communications with comprehensive tracing\nMulti-agent systems represent some of the most complex AI applications, involving multiple agents that coordinate, communicate, and collaborate to achieve shared goals. Noveum.ai provides specialized tracing capabilities to help you understand and optimize these intricate workflows.\nšŸŽÆ Why Multi-Agent Tracing Matters\nMulti-agent systems introduce unique observability challenges:\n- Complex...\n\nSource 3 (https://noveum.ai/en/docs/advanced/multi-agent-tracing):\nMulti-Agent Tracing\nObserve complex agent workflows and inter-agent communications with comprehensive tracing\nMulti-agent systems represent some of the most complex AI applications, involving multiple agents that coordinate, communicate, and collaborate to achieve shared goals. Noveum.ai provides specialized tracing capabilities to help you understand and optimize these intricate workflows.\nšŸŽÆ Why Multi-Agent Tracing Matters\nMulti-agent systems introduce unique observability challenges:\n- Complex...\n\nSource 4 (https://noveum.ai/en/docs/advanced/performance-optimization):\nšŸ”— Next Steps\n- Multi-Agent Tracing - Observe agent workflows\n- RAG Pipeline Observability - Monitor retrieval and generation systems\n- Custom Instrumentation - Add domain-specific tracing\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nSource 5 (https://noveum.ai/docs/advanced/performance-optimization):\nšŸ”— Next Steps\n- Multi-Agent Tracing - Observe agent workflows\n- RAG Pipeline Observability - Monitor retrieval and generation systems\n- Custom Instrumentation - Add domain-specific tracing\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nUser Question: How does Noveum handle agent tracing?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", + "prompt.prompt_length": 3269, + "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, ...", + "prompt.instruction_following": "rag_optimized", + "response.answer_length": 691, + "response.answer_completeness": "complete", + "response.response_quality": "high", + "response.source_citation": 5, + "response.context_utilization": 0.25255847953216376, + "output_response": "RAG Answer: Noveum.ai handles agent tracing through its specialized Multi-Agent Tracing capabilities, which are designed to observe complex workflows and inter-agent communications within multi-agent systems. The...", + "evaluation.retrieval_effectiveness": 1, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 5, + "evaluation.context_relevance": "high", + "evaluation.overall_quality": "high", + "evaluation.ready_for_production": true, + "rag.retrieval_strategy": "semantic_similarity", + "rag.vector_search_results": 5, + "rag.context_synthesis": "multi_source", + "rag.document_coverage": 1, + "rag.information_density": 0.25255847953216376 + }, + "events": [], + "links": [], + "trace_trace_id": "04bebf38-a343-4563-80db-0154bef8d927", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-09 08:25:33.693000000", + "trace_end_time": "2025-10-09 08:25:37.313000000", + "trace_duration_ms": 3619.672, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-09 08:25:40.339000000", + "updated_at": "2025-10-09 08:25:40.339000000" + }, + { + "span_id": "6b21de5d-456f-4501-953f-0ce5bc4fa5a5", + "trace_id": "5e043630-6493-42b5-beb8-79faa19bfa37", + "parent_span_id": "6c362c73-e063-4813-8180-d26b6a52b798", + "name": "agent.rag_evaluation_metrics", + "start_time": "2025-10-09 08:25:32.086000000", + "end_time": "2025-10-09 08:25:32.086000000", + "duration_ms": 0.04, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "rag_evaluation_metrics", + "agent.capabilities": [ + "retrieval_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What APIs are available in Noveum platform?", + "query_length": 43, + "query_type": "rag_evaluation_query", + "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/docs):\n- šŸŒ Universal: Works across frameworks, clouds, and deployment models\nšŸ¤ Community & Support\n- šŸ’¬ Discord Community: Join our Discord\n- šŸ“§ Email Support: [email protected]\n- šŸ› Bug Reports: GitHub Issues\n- šŸ“– Knowledge Base: Help Center\nReady to get started? Head to ...", + "retrieval.context_length": 2284, + "retrieval.sources_count": 5, + "retrieval.context_quality": "high", + "retrieval.effectiveness": 1, + "retrieval.context_utilization": 2.284, + "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/docs):\n- šŸŒ Universal: Works across frameworks, clouds, and deployment models\nšŸ¤ Community & Support\n- šŸ’¬ Discord Community: Join our Discord\n- šŸ“§ Email Support: [email protected]\n- šŸ› Bug Reports: GitHub Issues\n- šŸ“– Knowledge Base: Help Center\nReady to get started? Head to our SDK Integration Guide to begin tracing your AI applications in under 5 minutes!\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability ...\n\nSource 2 (https://noveum.ai/docs):\n- šŸŒ Universal: Works across frameworks, clouds, and deployment models\nšŸ¤ Community & Support\n- šŸ’¬ Discord Community: Join our Discord\n- šŸ“§ Email Support: [email protected]\n- šŸ› Bug Reports: GitHub Issues\n- šŸ“– Knowledge Base: Help Center\nReady to get started? Head to our SDK Integration Guide to begin tracing your AI applications in under 5 minutes!\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability ...\n\nSource 3 (https://noveum.ai/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nSource 4 (https://noveum.ai/en/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nSource 5 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, vector searches, data retrieval, agent reasoning, and more. Without proper observability, debugging becomes a nightmare, optimization is guesswork, and understanding user interactions is nearly impossible.\nNoveum.ai solves this by providing comprehensive tracing and observability specifically designed for AI applications.\nKey benefits of using Noveum.ai:\n- Complete Visibility: Trace...\n\nUser Question: What APIs are available in Noveum platform?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", + "prompt.prompt_length": 2823, + "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/docs):\n- šŸŒ Universal: Works across frameworks, clouds, and deployment models\nšŸ¤ Community & Support\n- šŸ’¬ Discord Community: Join our Discord\n- šŸ“§ Email Support: [email prot...", + "prompt.instruction_following": "rag_optimized", + "response.answer_length": 270, + "response.answer_completeness": "complete", + "response.response_quality": "high", + "response.source_citation": 5, + "response.context_utilization": 0.11821366024518389, + "output_response": "RAG Answer: The provided context does not specify the APIs available in the Noveum platform. For detailed information about the APIs, I recommend checking the official Noveum documentation or the SDK Integration ...", + "evaluation.retrieval_effectiveness": 1, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 5, + "evaluation.context_relevance": "high", + "evaluation.overall_quality": "high", + "evaluation.ready_for_production": true, + "rag.retrieval_strategy": "semantic_similarity", + "rag.vector_search_results": 5, + "rag.context_synthesis": "multi_source", + "rag.document_coverage": 1, + "rag.information_density": 0.11821366024518389 + }, + "events": [], + "links": [], + "trace_trace_id": "5e043630-6493-42b5-beb8-79faa19bfa37", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-09 08:25:29.901000000", + "trace_end_time": "2025-10-09 08:25:32.685000000", + "trace_duration_ms": 2783.326, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-09 08:25:34.918000000", + "updated_at": "2025-10-09 08:25:34.918000000" + }, + { + "span_id": "2ad4e155-7b76-4040-923d-d6b65a97a45b", + "trace_id": "7da9814d-a2e8-4c4e-b750-68b26bd5fd22", + "parent_span_id": "1393d065-a9a4-4475-808f-2d00b3c68939", + "name": "agent.rag_evaluation_metrics", + "start_time": "2025-10-09 08:25:28.496000000", + "end_time": "2025-10-09 08:25:28.496000000", + "duration_ms": 0.039, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "rag_evaluation_metrics", + "agent.capabilities": [ + "retrieval_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: How do I set up observability with Noveum?", + "query_length": 42, + "query_type": "rag_evaluation_query", + "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\n- Production Ready: Built for scale with intelligent sampling and batching\n- Privacy Focused: You control what data is captured and how it's stored\n- Open Source SDKs: Transparent, auditable, and extensible\nGetting ...", + "retrieval.context_length": 2870, + "retrieval.sources_count": 6, + "retrieval.context_quality": "high", + "retrieval.effectiveness": 1.2, + "retrieval.context_utilization": 2.87, + "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\n- Production Ready: Built for scale with intelligent sampling and batching\n- Privacy Focused: You control what data is captured and how it's stored\n- Open Source SDKs: Transparent, auditable, and extensible\nGetting Started\nReady to add comprehensive observability to your AI applications?\n- Sign up at noveum.ai\n- Create a project and get your API key\n- Install the SDK for your preferred language\n- Add tracing to your AI workflows\n- Explore insights in the Noveum.ai dashboard\nCheck out our integra...\n\nSource 2 (https://noveum.ai/en/docs/getting-started/tracing-concepts):\n- Explore Framework Integrations - Framework-specific guidance\n- Learn Advanced Patterns - Custom instrumentation techniques\n- Master the Dashboard - Analyze your traces effectively\nRemember: Good observability is not about collecting all possible data, but about collecting the right data that helps you understand, debug, and optimize your AI applications.\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to O...\n\nSource 3 (https://noveum.ai/docs/getting-started/tracing-concepts):\n- Explore Framework Integrations - Framework-specific guidance\n- Learn Advanced Patterns - Custom instrumentation techniques\n- Master the Dashboard - Analyze your traces effectively\nRemember: Good observability is not about collecting all possible data, but about collecting the right data that helps you understand, debug, and optimize your AI applications.\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to O...\n\nSource 4 (https://noveum.ai/en/docs/getting-started/overview):\nNoveum.ai Overview\nComprehensive AI tracing and observability platform for LLM applications, RAG systems, and AI agents\nWelcome to Noveum.aiā€”the comprehensive tracing and observability platform built specifically for AI applications. Whether you're building LLM-powered chatbots, RAG systems, multi-agent workflows, or any AI-driven application, Noveum provides the insights you need to understand, debug, and optimize your systems.\nšŸŽÆ Why AI Applications Need Specialized Observability\nTraditional mo...\n\nSource 5 (https://noveum.ai/docs/getting-started/overview):\nNoveum.ai Overview\nComprehensive AI tracing and observability platform for LLM applications, RAG systems, and AI agents\nWelcome to Noveum.aiā€”the comprehensive tracing and observability platform built specifically for AI applications. Whether you're building LLM-powered chatbots, RAG systems, multi-agent workflows, or any AI-driven application, Noveum provides the insights you need to understand, debug, and optimize your systems.\nšŸŽÆ Why AI Applications Need Specialized Observability\nTraditional mo...\n\nUser Question: How do I set up observability with Noveum?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", + "prompt.prompt_length": 3408, + "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\n- Production Ready: Built for scale with intelligent sampling and batching\n- Privacy Focused: You control what data...", + "prompt.instruction_following": "rag_optimized", + "response.answer_length": 951, + "response.answer_completeness": "complete", + "response.response_quality": "high", + "response.source_citation": 6, + "response.context_utilization": 0.3313588850174216, + "output_response": "RAG Answer: To set up observability with Noveum, follow these steps:\n\n1. **Sign Up**: Go to [noveum.ai](https://noveum.ai) and create an account.\n2. **Create a Project**: Once signed in, create a new project to o...", + "evaluation.retrieval_effectiveness": 1.2, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 6, + "evaluation.context_relevance": "high", + "evaluation.overall_quality": "high", + "evaluation.ready_for_production": true, + "rag.retrieval_strategy": "semantic_similarity", + "rag.vector_search_results": 6, + "rag.context_synthesis": "multi_source", + "rag.document_coverage": 1.2, + "rag.information_density": 0.3313588850174216 + }, + "events": [], + "links": [], + "trace_trace_id": "7da9814d-a2e8-4c4e-b750-68b26bd5fd22", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-09 08:25:20.586000000", + "trace_end_time": "2025-10-09 08:25:28.897000000", + "trace_duration_ms": 8310.205, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-09 08:25:29.588000000", + "updated_at": "2025-10-09 08:25:29.588000000" + }, + { + "span_id": "6b822089-be40-457d-9fa6-0979bafdfb85", + "trace_id": "16143f74-2831-4753-b33d-ce4b645093c5", + "parent_span_id": "f419377a-fc24-4abf-8a1c-03857824e374", + "name": "agent.rag_evaluation_metrics", + "start_time": "2025-10-09 08:25:19.220000000", + "end_time": "2025-10-09 08:25:19.220000000", + "duration_ms": 0.053, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "rag_evaluation_metrics", + "agent.capabilities": [ + "retrieval_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What features does Noveum Trace offer?", + "query_length": 38, + "query_type": "rag_evaluation_query", + "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/docs):\nCore Components\n-\nšŸ Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\nšŸ“˜ TypeScript SDK (\n@noveum/trace\n)- Framework ...", + "retrieval.context_length": 2787, + "retrieval.sources_count": 5, + "retrieval.context_quality": "high", + "retrieval.effectiveness": 1, + "retrieval.context_utilization": 2.787, + "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/docs):\nCore Components\n-\nšŸ Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\nšŸ“˜ TypeScript SDK (\n@noveum/trace\n)- Framework integrations for Next.js, Express.js, and Hono\n- TypeScript-first design with full type safety\n- Universal compatibility (Node.js, Edge Runtime, browsers)\n-\nšŸ“Š Noveum Platform\n- Real-time dashboard for analyzing traces and performance\n- Ad...\n\nSource 2 (https://noveum.ai/docs):\nCore Components\n-\nšŸ Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\nšŸ“˜ TypeScript SDK (\n@noveum/trace\n)- Framework integrations for Next.js, Express.js, and Hono\n- TypeScript-first design with full type safety\n- Universal compatibility (Node.js, Edge Runtime, browsers)\n-\nšŸ“Š Noveum Platform\n- Real-time dashboard for analyzing traces and performance\n- Ad...\n\nSource 3 (https://noveum.ai/en/docs/getting-started/sdk-integration):\n- šŸ‘„ Collaborate with your team on insights\nšŸ”’ Security & Privacy\n- šŸ” Encryption: All data encrypted in transit and at rest\n- šŸŽ›ļø Configurable Capture: Control what data is collected\n- šŸ  Data Residency: Choose your data storage region\n- ā° Retention Control: Set custom data retention policies\nNext Steps\n- Tracing Concepts - Learn about traces, spans, and observability best practices\n- Framework Integrations - Deep dive into specific framework setups\n- Multi-Agent Tracing - Observe complex agent work...\n\nSource 4 (https://noveum.ai/docs/getting-started/sdk-integration):\n- šŸ‘„ Collaborate with your team on insights\nšŸ”’ Security & Privacy\n- šŸ” Encryption: All data encrypted in transit and at rest\n- šŸŽ›ļø Configurable Capture: Control what data is collected\n- šŸ  Data Residency: Choose your data storage region\n- ā° Retention Control: Set custom data retention policies\nNext Steps\n- Tracing Concepts - Learn about traces, spans, and observability best practices\n- Framework Integrations - Deep dive into specific framework setups\n- Multi-Agent Tracing - Observe complex agent work...\n\nSource 5 (https://noveum.ai/docs/platform/dashboard):\n- Bulk Export: Download multiple traces simultaneously\n- Comparative Analysis: Compare performance across multiple traces\nReal-Time Updates\n- Live Refresh: Automatic updates as new traces arrive\n- Connection Monitoring: Real-time status of your trace ingestion pipeline\n- Performance Indicators: Live metrics for system health monitoring\nReady to dive deeper? Explore Projects & Environments to organize your AI applications, or check out Team Collaboration to share insights with your team.\nGet Earl...\n\nUser Question: What features does Noveum Trace offer?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", + "prompt.prompt_length": 3321, + "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/docs):\nCore Components\n-\nšŸ Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frame...", + "prompt.instruction_following": "rag_optimized", + "response.answer_length": 1014, + "response.answer_completeness": "complete", + "response.response_quality": "high", + "response.source_citation": 5, + "response.context_utilization": 0.3638320775026911, + "output_response": "RAG Answer: Noveum Trace offers several key features through its Python SDK (`noveum-trace`) and TypeScript SDK (`@noveum/trace`):\n\n1. **Decorator-based Tracing**: It provides a simple way to trace LLM calls, age...", + "evaluation.retrieval_effectiveness": 1, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 5, + "evaluation.context_relevance": "high", + "evaluation.overall_quality": "high", + "evaluation.ready_for_production": true, + "rag.retrieval_strategy": "semantic_similarity", + "rag.vector_search_results": 5, + "rag.context_synthesis": "multi_source", + "rag.document_coverage": 1, + "rag.information_density": 0.3638320775026911 + }, + "events": [], + "links": [], + "trace_trace_id": "16143f74-2831-4753-b33d-ce4b645093c5", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-09 08:25:12.442000000", + "trace_end_time": "2025-10-09 08:25:19.579000000", + "trace_duration_ms": 7137.402, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-09 08:25:20.290000000", + "updated_at": "2025-10-09 08:25:20.290000000" + }, + { + "span_id": "08db68f6-3623-44b5-840b-41400fac7333", + "trace_id": "fc64e6cc-6739-4256-ac4a-7b80c3028233", + "parent_span_id": "0e58e31f-a8ab-4aee-8fc2-abcae1e21d9d", + "name": "agent.rag_evaluation_metrics", + "start_time": "2025-10-09 08:25:10.938000000", + "end_time": "2025-10-09 08:25:10.938000000", + "duration_ms": 0.043000000000000003, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "rag_evaluation_metrics", + "agent.capabilities": [ + "retrieval_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What are Noveum's pricing plans?", + "query_length": 32, + "query_type": "rag_evaluation_query", + "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....", + "retrieval.context_length": 2105, + "retrieval.sources_count": 5, + "retrieval.context_quality": "high", + "retrieval.effectiveness": 1, + "retrieval.context_utilization": 2.105, + "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nSource 2 (https://noveum.ai/en/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nSource 3 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nThe AI landscape is complex and moving fast. With Noveum.ai, you'll have the insights you need to build, debug, and optimize AI applications with confidence.\nReady to see what's really happening inside your AI applications? Try Noveum.ai today and transform how you build and monitor AI systems.\nLet's build more reliable, observable AIā€”together.\nGet Early Access to Noveum.ai Platform\nJoin the select group of AI teams optimizing their models with our data-driven platform. We're onboarding users in...\n\nSource 4 (https://noveum.ai/en/blog/from-logs-to-intelligent-choices-inside-noveum-ais-evaluation-process):\nGet Early Access to Noveum.ai Platform\nJoin the select group of AI teams optimizing their models with our data-driven platform. We're onboarding users in limited batches to ensure a premium experience....\n\nSource 5 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, vector searches, data retrieval, agent reasoning, and more. Without proper observability, debugging becomes a nightmare, optimization is guesswork, and understanding user interactions is nearly impossible.\nNoveum.ai solves this by providing comprehensive tracing and observability specifically designed for AI applications.\nKey benefits of using Noveum.ai:\n- Complete Visibility: Trace...\n\nUser Question: What are Noveum's pricing plans?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", + "prompt.prompt_length": 2633, + "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to ...", + "prompt.instruction_following": "rag_optimized", + "response.answer_length": 399, + "response.answer_completeness": "complete", + "response.response_quality": "high", + "response.source_citation": 5, + "response.context_utilization": 0.18954869358669835, + "output_response": "RAG Answer: The provided context does not include specific information about Noveum's pricing plans. However, it mentions that all users will have access to the Observability suite for free, and early users will ...", + "evaluation.retrieval_effectiveness": 1, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 5, + "evaluation.context_relevance": "high", + "evaluation.overall_quality": "high", + "evaluation.ready_for_production": true, + "rag.retrieval_strategy": "semantic_similarity", + "rag.vector_search_results": 5, + "rag.context_synthesis": "multi_source", + "rag.document_coverage": 1, + "rag.information_density": 0.18954869358669835 + }, + "events": [], + "links": [], + "trace_trace_id": "fc64e6cc-6739-4256-ac4a-7b80c3028233", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-09 08:25:07.932000000", + "trace_end_time": "2025-10-09 08:25:11.438000000", + "trace_duration_ms": 3506.0989999999997, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-09 08:25:12.651000000", + "updated_at": "2025-10-09 08:25:12.651000000" + }, + { + "span_id": "6a57c92e-e473-425a-978b-19dbe976a46e", + "trace_id": "b7945c49-f584-4c70-972d-536a805d8a31", + "parent_span_id": "48155aad-6204-4a52-8624-8fa5d6fe407f", + "name": "agent.rag_evaluation_metrics", + "start_time": "2025-10-09 08:25:05.873000000", + "end_time": "2025-10-09 08:25:05.873000000", + "duration_ms": 0.229, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "rag_evaluation_metrics", + "agent.capabilities": [ + "retrieval_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: How do I integrate Noveum Trace in my application?", + "query_length": 50, + "query_type": "rag_evaluation_query", + "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/docs):\nCore Components\n-\nšŸ Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\nšŸ“˜ TypeScript SDK (\n@noveum/trace\n)- Framework ...", + "retrieval.context_length": 2803, + "retrieval.sources_count": 5, + "retrieval.context_quality": "high", + "retrieval.effectiveness": 1, + "retrieval.context_utilization": 2.803, + "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/docs):\nCore Components\n-\nšŸ Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\nšŸ“˜ TypeScript SDK (\n@noveum/trace\n)- Framework integrations for Next.js, Express.js, and Hono\n- TypeScript-first design with full type safety\n- Universal compatibility (Node.js, Edge Runtime, browsers)\n-\nšŸ“Š Noveum Platform\n- Real-time dashboard for analyzing traces and performance\n- Ad...\n\nSource 2 (https://noveum.ai/docs):\nCore Components\n-\nšŸ Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\nšŸ“˜ TypeScript SDK (\n@noveum/trace\n)- Framework integrations for Next.js, Express.js, and Hono\n- TypeScript-first design with full type safety\n- Universal compatibility (Node.js, Edge Runtime, browsers)\n-\nšŸ“Š Noveum Platform\n- Real-time dashboard for analyzing traces and performance\n- Ad...\n\nSource 3 (https://noveum.ai/docs/getting-started/sdk-integration):\nSDK Integration Guide\nIntegrate Noveum.ai tracing into your AI applications with Python or TypeScript SDKs\nThe Noveum.ai SDKs provide comprehensive tracing and observability for your AI applications with minimal code changes. Whether you're building LLM applications, RAG systems, or multi-agent workflows, our SDKs automatically capture essential metrics and traces.\nšŸš€ Quick Start\n1. Create Your Account & Get API Key\n- Sign up at noveum.ai\n- Create a project in your dashboard\n- Generate an API key...\n\nSource 4 (https://noveum.ai/en/docs/getting-started/sdk-integration):\nSDK Integration Guide\nIntegrate Noveum.ai tracing into your AI applications with Python or TypeScript SDKs\nThe Noveum.ai SDKs provide comprehensive tracing and observability for your AI applications with minimal code changes. Whether you're building LLM applications, RAG systems, or multi-agent workflows, our SDKs automatically capture essential metrics and traces.\nšŸš€ Quick Start\n1. Create Your Account & Get API Key\n- Sign up at noveum.ai\n- Create a project in your dashboard\n- Generate an API key...\n\nSource 5 (https://noveum.ai/en/docs/getting-started/sdk-integration):\nšŸ”§ Framework Integrations\nNext.js Integration\nExpress.js Integration\nFastAPI Integration (Python)\nšŸ“Š Advanced Features\nCustom Attributes & Events\nSampling Configuration\nšŸ”— What's Captured Automatically\n- šŸ“Š Performance Metrics: Latency, throughput, error rates\n- šŸ’° Cost Tracking: Token usage, API costs across providers\n- šŸ” Request/Response: Configurable capture of inputs/outputs\n- šŸ·ļø Metadata: Model names, parameters, user context\n- šŸŒŠ Context Flow: Trace relationships across services\n- šŸ› Error Detail...\n\nUser Question: How do I integrate Noveum Trace in my application?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", + "prompt.prompt_length": 3349, + "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/docs):\nCore Components\n-\nšŸ Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frame...", + "prompt.instruction_following": "rag_optimized", + "response.answer_length": 1151, + "response.answer_completeness": "complete", + "response.response_quality": "high", + "response.source_citation": 5, + "response.context_utilization": 0.410631466286122, + "output_response": "RAG Answer: To integrate Noveum Trace into your application, you can use either the Python SDK (`noveum-trace`) or the TypeScript SDK (`@noveum/trace`). Hereā€™s a quick start guide based on the provided documentat...", + "evaluation.retrieval_effectiveness": 1, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 5, + "evaluation.context_relevance": "high", + "evaluation.overall_quality": "high", + "evaluation.ready_for_production": true, + "rag.retrieval_strategy": "semantic_similarity", + "rag.vector_search_results": 5, + "rag.context_synthesis": "multi_source", + "rag.document_coverage": 1, + "rag.information_density": 0.410631466286122 + }, + "events": [], + "links": [], + "trace_trace_id": "b7945c49-f584-4c70-972d-536a805d8a31", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-09 08:24:58.518000000", + "trace_end_time": "2025-10-09 08:25:06.920000000", + "trace_duration_ms": 8402.516, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-09 08:25:07.620000000", + "updated_at": "2025-10-09 08:25:07.620000000" + }, + { + "span_id": "efb19490-bc9e-48dc-a365-2f839100e41a", + "trace_id": "f5c40ecf-36c0-45ba-9cc9-dc0329b0324b", + "parent_span_id": "57ded2eb-0f3f-474e-8704-5ad9974ab4bb", + "name": "agent.rag_evaluation_metrics", + "start_time": "2025-10-09 08:24:56.965000000", + "end_time": "2025-10-09 08:24:56.965000000", + "duration_ms": 0.048, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "rag_evaluation_metrics", + "agent.capabilities": [ + "retrieval_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What is Noveum and what does it do?", + "query_length": 35, + "query_type": "rag_evaluation_query", + "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, vector searches, data retrieval, agent reasoning, and more. Without proper observability, debugging ...", + "retrieval.context_length": 2334, + "retrieval.sources_count": 5, + "retrieval.context_quality": "high", + "retrieval.effectiveness": 1, + "retrieval.context_utilization": 2.334, + "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, vector searches, data retrieval, agent reasoning, and more. Without proper observability, debugging becomes a nightmare, optimization is guesswork, and understanding user interactions is nearly impossible.\nNoveum.ai solves this by providing comprehensive tracing and observability specifically designed for AI applications.\nKey benefits of using Noveum.ai:\n- Complete Visibility: Trace...\n\nSource 2 (https://noveum.ai/en/docs/getting-started/overview):\nNoveum.ai Overview\nComprehensive AI tracing and observability platform for LLM applications, RAG systems, and AI agents\nWelcome to Noveum.aiā€”the comprehensive tracing and observability platform built specifically for AI applications. Whether you're building LLM-powered chatbots, RAG systems, multi-agent workflows, or any AI-driven application, Noveum provides the insights you need to understand, debug, and optimize your systems.\nšŸŽÆ Why AI Applications Need Specialized Observability\nTraditional mo...\n\nSource 3 (https://noveum.ai/docs/getting-started/overview):\nNoveum.ai Overview\nComprehensive AI tracing and observability platform for LLM applications, RAG systems, and AI agents\nWelcome to Noveum.aiā€”the comprehensive tracing and observability platform built specifically for AI applications. Whether you're building LLM-powered chatbots, RAG systems, multi-agent workflows, or any AI-driven application, Noveum provides the insights you need to understand, debug, and optimize your systems.\nšŸŽÆ Why AI Applications Need Specialized Observability\nTraditional mo...\n\nSource 4 (https://noveum.ai/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nSource 5 (https://noveum.ai/en/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nUser Question: What is Noveum and what does it do?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", + "prompt.prompt_length": 2865, + "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, ...", + "prompt.instruction_following": "rag_optimized", + "response.answer_length": 611, + "response.answer_completeness": "complete", + "response.response_quality": "high", + "response.source_citation": 5, + "response.context_utilization": 0.26178234790059984, + "output_response": "RAG Answer: Noveum.ai is a comprehensive tracing and observability platform specifically designed for AI applications, including those powered by large language models (LLMs), retrieval-augmented generation (RAG)...", + "evaluation.retrieval_effectiveness": 1, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 5, + "evaluation.context_relevance": "high", + "evaluation.overall_quality": "high", + "evaluation.ready_for_production": true, + "rag.retrieval_strategy": "semantic_similarity", + "rag.vector_search_results": 5, + "rag.context_synthesis": "multi_source", + "rag.document_coverage": 1, + "rag.information_density": 0.26178234790059984 + }, + "events": [], + "links": [], + "trace_trace_id": "f5c40ecf-36c0-45ba-9cc9-dc0329b0324b", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-09 08:24:54.009000000", + "trace_end_time": "2025-10-09 08:24:57.511000000", + "trace_duration_ms": 3502.062, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-09 08:24:59.716000000", + "updated_at": "2025-10-09 08:24:59.716000000" + } +] \ No newline at end of file diff --git a/noveum_customer_support_bt/split_datasets/agent.routing_evaluation_metrics_dataset.json b/noveum_customer_support_bt/split_datasets/agent.routing_evaluation_metrics_dataset.json new file mode 100644 index 0000000..d52d6a8 --- /dev/null +++ b/noveum_customer_support_bt/split_datasets/agent.routing_evaluation_metrics_dataset.json @@ -0,0 +1,1602 @@ +[ + { + "span_id": "95dac0c3-83fd-459c-af4e-c6a1734fd874", + "trace_id": "eda4fe22-9a2b-4b73-856b-f4f3309bf719", + "parent_span_id": "0b19befc-5dc0-4b00-9a8b-5f2f2c914f17", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-09 08:26:46.384000000", + "end_time": "2025-10-09 08:26:46.384000000", + "duration_ms": 0.041, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What are the latest breakthroughs in artificial intelligence?", + "query_length": 61, + "query_type": "routing_evaluation_query", + "classification.mode": "Web Search", + "classification.rag_keyword_score": 0, + "classification.web_keyword_score": 1, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "complex", + "query.intent": "general_knowledge", + "query.keyword_density": 0.125, + "query.domain_affinity": "general", + "routing.decision": "Routed to Web Search based on analysis", + "routing.rationale": "RAG score: 0, Web score: 1, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "RAG", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.019230769230769232, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: Web Search (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "eda4fe22-9a2b-4b73-856b-f4f3309bf719", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-09 08:26:46.384000000", + "trace_end_time": "2025-10-09 08:26:48.685000000", + "trace_duration_ms": 2301.488, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-09 08:26:49.422000000", + "updated_at": "2025-10-09 08:26:49.422000000" + }, + { + "span_id": "516060a3-0f58-46ba-8a6a-4b502a376492", + "trace_id": "0ffffba1-8a37-443c-8866-d53ffbfa7718", + "parent_span_id": "30644d2a-5ab1-4a73-b4ff-cf9fa478e36b", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-09 08:26:41.850000000", + "end_time": "2025-10-09 08:26:41.850000000", + "duration_ms": 0.055, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What's happening in the software development world today?", + "query_length": 57, + "query_type": "routing_evaluation_query", + "classification.mode": "Web Search", + "classification.rag_keyword_score": 0, + "classification.web_keyword_score": 1, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "complex", + "query.intent": "general_knowledge", + "query.keyword_density": 0.125, + "query.domain_affinity": "general", + "routing.decision": "Routed to Web Search based on analysis", + "routing.rationale": "RAG score: 0, Web score: 1, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "RAG", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.019230769230769232, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: Web Search (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "0ffffba1-8a37-443c-8866-d53ffbfa7718", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-09 08:26:41.850000000", + "trace_end_time": "2025-10-09 08:26:45.377000000", + "trace_duration_ms": 3527.831, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-09 08:26:49.422000000", + "updated_at": "2025-10-09 08:26:49.422000000" + }, + { + "span_id": "8f8d2dc4-150f-4e43-aad3-96da4d06f586", + "trace_id": "f1f37bd7-0851-4659-b493-b80d3800d920", + "parent_span_id": "89eac2de-b7da-4db3-bf7c-8e87c429481e", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-09 08:26:35.915000000", + "end_time": "2025-10-09 08:26:35.915000000", + "duration_ms": 0.062, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What are the newest features in cloud computing?", + "query_length": 48, + "query_type": "routing_evaluation_query", + "classification.mode": "RAG", + "classification.rag_keyword_score": 1, + "classification.web_keyword_score": 0, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "documentation", + "query.keyword_density": 0.125, + "query.domain_affinity": "noveum", + "routing.decision": "Routed to RAG based on analysis", + "routing.rationale": "RAG score: 1, Web score: 0, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "Web Search", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.019230769230769232, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: RAG (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "f1f37bd7-0851-4659-b493-b80d3800d920", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-09 08:26:35.915000000", + "trace_end_time": "2025-10-09 08:26:40.841000000", + "trace_duration_ms": 4926.342, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-09 08:26:44.053000000", + "updated_at": "2025-10-09 08:26:44.053000000" + }, + { + "span_id": "cf0a915c-668b-412e-9c14-69119d059dc2", + "trace_id": "43cdf081-4f01-49cd-b566-dbd1619e6cd2", + "parent_span_id": "962afebb-26d1-45ff-acf1-218163cd6bb8", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-09 08:26:32.240000000", + "end_time": "2025-10-09 08:26:32.240000000", + "duration_ms": 0.05, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What's the current status of cryptocurrency markets?", + "query_length": 52, + "query_type": "routing_evaluation_query", + "classification.mode": "Web Search", + "classification.rag_keyword_score": 0, + "classification.web_keyword_score": 3, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "complex", + "query.intent": "general_knowledge", + "query.keyword_density": 0.42857142857142855, + "query.domain_affinity": "general", + "routing.decision": "Routed to Web Search based on analysis", + "routing.rationale": "RAG score: 0, Web score: 3, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "RAG", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.057692307692307696, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: Web Search (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "43cdf081-4f01-49cd-b566-dbd1619e6cd2", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-09 08:26:32.237000000", + "trace_end_time": "2025-10-09 08:26:34.906000000", + "trace_duration_ms": 2668.841, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-09 08:26:38.618000000", + "updated_at": "2025-10-09 08:26:38.618000000" + }, + { + "span_id": "f75b51b8-4882-4170-bc08-2adcab250f50", + "trace_id": "9a1983f4-09da-4b53-80e6-38de6878e0e7", + "parent_span_id": "246cf822-4562-4d3a-8a5a-e38e0a0b02a0", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-09 08:26:27.992000000", + "end_time": "2025-10-09 08:26:27.992000000", + "duration_ms": 0.060000000000000005, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What are the latest updates in Python programming?", + "query_length": 50, + "query_type": "routing_evaluation_query", + "classification.mode": "Web Search", + "classification.rag_keyword_score": 0, + "classification.web_keyword_score": 2, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "general_knowledge", + "query.keyword_density": 0.25, + "query.domain_affinity": "general", + "routing.decision": "Routed to Web Search based on analysis", + "routing.rationale": "RAG score: 0, Web score: 2, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "RAG", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.038461538461538464, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: Web Search (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "9a1983f4-09da-4b53-80e6-38de6878e0e7", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-09 08:26:27.992000000", + "trace_end_time": "2025-10-09 08:26:31.231000000", + "trace_duration_ms": 3239.706, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-09 08:26:33.463000000", + "updated_at": "2025-10-09 08:26:33.463000000" + }, + { + "span_id": "4180abcf-924c-45ac-8dfa-17c0bc603eff", + "trace_id": "5d2517e1-220a-429d-9d59-f701bda25eed", + "parent_span_id": "20544f91-8828-4a0e-b253-ed1f8b95b205", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-09 08:26:23.210000000", + "end_time": "2025-10-09 08:26:23.210000000", + "duration_ms": 0.044, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What happened in tech news this week?", + "query_length": 37, + "query_type": "routing_evaluation_query", + "classification.mode": "Web Search", + "classification.rag_keyword_score": 0, + "classification.web_keyword_score": 2, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "general_knowledge", + "query.keyword_density": 0.2857142857142857, + "query.domain_affinity": "general", + "routing.decision": "Routed to Web Search based on analysis", + "routing.rationale": "RAG score: 0, Web score: 2, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "RAG", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.038461538461538464, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: Web Search (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "5d2517e1-220a-429d-9d59-f701bda25eed", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-09 08:26:23.210000000", + "trace_end_time": "2025-10-09 08:26:26.985000000", + "trace_duration_ms": 3775.124, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-09 08:26:28.174000000", + "updated_at": "2025-10-09 08:26:28.174000000" + }, + { + "span_id": "8b0022cc-91dd-4065-91ed-b77217f92bd7", + "trace_id": "52aacb67-c361-4445-9b72-c157f79f47d6", + "parent_span_id": "5359c861-c51b-446d-9ad2-b3d8c21ea70f", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-09 08:26:16.544000000", + "end_time": "2025-10-09 08:26:16.544000000", + "duration_ms": 0.087, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What are the current trends in observability tools?", + "query_length": 51, + "query_type": "routing_evaluation_query", + "classification.mode": "RAG", + "classification.rag_keyword_score": 2, + "classification.web_keyword_score": 1, + "classification.confidence_score": 0.3333333333333333, + "classification.confidence_level": "medium", + "classification.method": "keyword_based", + "query.complexity": "complex", + "query.intent": "documentation", + "query.keyword_density": 0.375, + "query.domain_affinity": "noveum", + "routing.decision": "Routed to RAG based on analysis", + "routing.rationale": "RAG score: 2, Web score: 1, Confidence: 0.33", + "routing.expected_performance": "medium", + "routing.alternative_mode": "Web Search", + "routing.decision_confidence": 0.3333333333333333, + "evaluation.routing_accuracy": "medium", + "evaluation.keyword_coverage": 0.057692307692307696, + "evaluation.query_understanding": "ambiguous", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: RAG (Confidence: 0.33)" + }, + "events": [], + "links": [], + "trace_trace_id": "52aacb67-c361-4445-9b72-c157f79f47d6", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-09 08:26:16.544000000", + "trace_end_time": "2025-10-09 08:26:22.201000000", + "trace_duration_ms": 5657.012, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-09 08:26:22.904000000", + "updated_at": "2025-10-09 08:26:22.904000000" + }, + { + "span_id": "780df186-fe5a-4742-b937-2a142ad528c6", + "trace_id": "a81ca3a8-80aa-4c39-876e-8d40ea7a0aef", + "parent_span_id": "acf6a99a-39e0-4f87-a07e-3a61213ab032", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-09 08:26:08.903000000", + "end_time": "2025-10-09 08:26:08.903000000", + "duration_ms": 0.06499999999999999, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: Tell me about recent developments in machine learning", + "query_length": 53, + "query_type": "routing_evaluation_query", + "classification.mode": "Web Search", + "classification.rag_keyword_score": 0, + "classification.web_keyword_score": 1, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "complex", + "query.intent": "general_knowledge", + "query.keyword_density": 0.125, + "query.domain_affinity": "general", + "routing.decision": "Routed to Web Search based on analysis", + "routing.rationale": "RAG score: 0, Web score: 1, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "RAG", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.019230769230769232, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: Web Search (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "a81ca3a8-80aa-4c39-876e-8d40ea7a0aef", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-09 08:26:08.903000000", + "trace_end_time": "2025-10-09 08:26:15.537000000", + "trace_duration_ms": 6633.662, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-09 08:26:16.227000000", + "updated_at": "2025-10-09 08:26:16.227000000" + }, + { + "span_id": "f80ae3e8-b4dd-42c6-b17b-de7805cedbb9", + "trace_id": "230aad27-f3dd-4968-a45a-3c2f07ac28ed", + "parent_span_id": "0b47ea07-1eda-4b6a-a183-ea9ef047e431", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-09 08:26:02.424000000", + "end_time": "2025-10-09 08:26:02.424000000", + "duration_ms": 0.025, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What's the weather like today?", + "query_length": 30, + "query_type": "routing_evaluation_query", + "classification.mode": "Web Search", + "classification.rag_keyword_score": 0, + "classification.web_keyword_score": 2, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "general_knowledge", + "query.keyword_density": 0.4, + "query.domain_affinity": "general", + "routing.decision": "Routed to Web Search based on analysis", + "routing.rationale": "RAG score: 0, Web score: 2, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "RAG", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.038461538461538464, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: Web Search (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "230aad27-f3dd-4968-a45a-3c2f07ac28ed", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-09 08:26:02.424000000", + "trace_end_time": "2025-10-09 08:26:07.894000000", + "trace_duration_ms": 5470.232, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-09 08:26:08.586000000", + "updated_at": "2025-10-09 08:26:08.586000000" + }, + { + "span_id": "209259cf-411d-4b97-8f7b-40b2530eec81", + "trace_id": "83c7dcce-3d89-4da1-8b3f-d419885d4cbc", + "parent_span_id": "3866ddb3-b17e-47a1-8b3f-bca1e32a2ae4", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-09 08:25:56.886000000", + "end_time": "2025-10-09 08:25:56.886000000", + "duration_ms": 0.017, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What are the latest AI news today?", + "query_length": 34, + "query_type": "routing_evaluation_query", + "classification.mode": "Web Search", + "classification.rag_keyword_score": 0, + "classification.web_keyword_score": 3, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "general_knowledge", + "query.keyword_density": 0.42857142857142855, + "query.domain_affinity": "general", + "routing.decision": "Routed to Web Search based on analysis", + "routing.rationale": "RAG score: 0, Web score: 3, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "RAG", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.057692307692307696, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: Web Search (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "83c7dcce-3d89-4da1-8b3f-d419885d4cbc", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-09 08:25:56.886000000", + "trace_end_time": "2025-10-09 08:26:01.416000000", + "trace_duration_ms": 4530.08, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-09 08:26:02.122000000", + "updated_at": "2025-10-09 08:26:02.122000000" + }, + { + "span_id": "c3f3bd32-9954-443f-93e5-52aa6cf412a0", + "trace_id": "2218f641-604c-491a-9710-b51a9941b982", + "parent_span_id": "3ce2e35e-4518-43b4-8a76-4413da6832ea", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-09 08:25:48.107000000", + "end_time": "2025-10-09 08:25:48.107000000", + "duration_ms": 0.066, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What are the benefits of using Noveum Trace?", + "query_length": 44, + "query_type": "routing_evaluation_query", + "classification.mode": "RAG", + "classification.rag_keyword_score": 2, + "classification.web_keyword_score": 0, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "noveum_specific", + "query.keyword_density": 0.25, + "query.domain_affinity": "noveum", + "routing.decision": "Routed to RAG based on analysis", + "routing.rationale": "RAG score: 2, Web score: 0, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "Web Search", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.038461538461538464, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: RAG (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "2218f641-604c-491a-9710-b51a9941b982", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-09 08:25:48.107000000", + "trace_end_time": "2025-10-09 08:25:55.878000000", + "trace_duration_ms": 7770.953, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-09 08:25:56.565000000", + "updated_at": "2025-10-09 08:25:56.565000000" + }, + { + "span_id": "940ce621-0015-45ed-adc6-c4caa2a95e81", + "trace_id": "255fd49c-84b4-4b18-887e-6308a412d535", + "parent_span_id": "c0c39e8b-9a42-4c18-8737-9224cd8a45c0", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-09 08:25:43.830000000", + "end_time": "2025-10-09 08:25:43.830000000", + "duration_ms": 0.022, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: How do I configure Noveum for my system?", + "query_length": 40, + "query_type": "routing_evaluation_query", + "classification.mode": "RAG", + "classification.rag_keyword_score": 2, + "classification.web_keyword_score": 0, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "noveum_specific", + "query.keyword_density": 0.25, + "query.domain_affinity": "noveum", + "routing.decision": "Routed to RAG based on analysis", + "routing.rationale": "RAG score: 2, Web score: 0, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "Web Search", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.038461538461538464, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: RAG (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "255fd49c-84b4-4b18-887e-6308a412d535", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-09 08:25:43.830000000", + "trace_end_time": "2025-10-09 08:25:47.099000000", + "trace_duration_ms": 3268.862, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-09 08:25:50.304000000", + "updated_at": "2025-10-09 08:25:50.304000000" + }, + { + "span_id": "ea28f049-6bc9-48b4-8386-279abe087ca8", + "trace_id": "dc511122-c0b6-415c-9a49-c7b45132dd87", + "parent_span_id": "80987c51-ea82-4a41-97c7-ed591082b9bf", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-09 08:25:38.322000000", + "end_time": "2025-10-09 08:25:38.322000000", + "duration_ms": 0.041, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What monitoring capabilities does Noveum provide?", + "query_length": 49, + "query_type": "routing_evaluation_query", + "classification.mode": "RAG", + "classification.rag_keyword_score": 2, + "classification.web_keyword_score": 0, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "noveum_specific", + "query.keyword_density": 0.3333333333333333, + "query.domain_affinity": "noveum", + "routing.decision": "Routed to RAG based on analysis", + "routing.rationale": "RAG score: 2, Web score: 0, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "Web Search", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.038461538461538464, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: RAG (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "dc511122-c0b6-415c-9a49-c7b45132dd87", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-09 08:25:38.322000000", + "trace_end_time": "2025-10-09 08:25:42.822000000", + "trace_duration_ms": 4500.425, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-09 08:25:45.028000000", + "updated_at": "2025-10-09 08:25:45.028000000" + }, + { + "span_id": "01a03da5-64df-41e3-b9c9-556e12a8edd7", + "trace_id": "04bebf38-a343-4563-80db-0154bef8d927", + "parent_span_id": "2f89c1e0-9b62-4651-914c-b71cf561c33c", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-09 08:25:33.693000000", + "end_time": "2025-10-09 08:25:33.693000000", + "duration_ms": 0.044, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: How does Noveum handle agent tracing?", + "query_length": 37, + "query_type": "routing_evaluation_query", + "classification.mode": "RAG", + "classification.rag_keyword_score": 2, + "classification.web_keyword_score": 0, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "noveum_specific", + "query.keyword_density": 0.3333333333333333, + "query.domain_affinity": "noveum", + "routing.decision": "Routed to RAG based on analysis", + "routing.rationale": "RAG score: 2, Web score: 0, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "Web Search", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.038461538461538464, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: RAG (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "04bebf38-a343-4563-80db-0154bef8d927", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-09 08:25:33.693000000", + "trace_end_time": "2025-10-09 08:25:37.313000000", + "trace_duration_ms": 3619.672, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-09 08:25:40.339000000", + "updated_at": "2025-10-09 08:25:40.339000000" + }, + { + "span_id": "8d9c632b-b2e8-496d-b1b0-a89e780ef810", + "trace_id": "5e043630-6493-42b5-beb8-79faa19bfa37", + "parent_span_id": "e30e3e61-6edd-4b62-ab77-627faa2bf741", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-09 08:25:29.901000000", + "end_time": "2025-10-09 08:25:29.901000000", + "duration_ms": 0.028, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What APIs are available in Noveum platform?", + "query_length": 43, + "query_type": "routing_evaluation_query", + "classification.mode": "RAG", + "classification.rag_keyword_score": 3, + "classification.web_keyword_score": 0, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "noveum_specific", + "query.keyword_density": 0.42857142857142855, + "query.domain_affinity": "noveum", + "routing.decision": "Routed to RAG based on analysis", + "routing.rationale": "RAG score: 3, Web score: 0, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "Web Search", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.057692307692307696, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: RAG (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "5e043630-6493-42b5-beb8-79faa19bfa37", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-09 08:25:29.901000000", + "trace_end_time": "2025-10-09 08:25:32.685000000", + "trace_duration_ms": 2783.326, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-09 08:25:34.918000000", + "updated_at": "2025-10-09 08:25:34.918000000" + }, + { + "span_id": "72dfbba4-0672-4adc-a1bf-308157d831e6", + "trace_id": "7da9814d-a2e8-4c4e-b750-68b26bd5fd22", + "parent_span_id": "2f513795-764f-4eb6-8dd2-108d4cdf8432", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-09 08:25:20.587000000", + "end_time": "2025-10-09 08:25:20.587000000", + "duration_ms": 0.041999999999999996, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: How do I set up observability with Noveum?", + "query_length": 42, + "query_type": "routing_evaluation_query", + "classification.mode": "RAG", + "classification.rag_keyword_score": 2, + "classification.web_keyword_score": 0, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "noveum_specific", + "query.keyword_density": 0.25, + "query.domain_affinity": "noveum", + "routing.decision": "Routed to RAG based on analysis", + "routing.rationale": "RAG score: 2, Web score: 0, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "Web Search", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.038461538461538464, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: RAG (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "7da9814d-a2e8-4c4e-b750-68b26bd5fd22", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-09 08:25:20.586000000", + "trace_end_time": "2025-10-09 08:25:28.897000000", + "trace_duration_ms": 8310.205, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-09 08:25:29.588000000", + "updated_at": "2025-10-09 08:25:29.588000000" + }, + { + "span_id": "92c81ee2-728d-4ea6-bc52-b81295d2aea2", + "trace_id": "16143f74-2831-4753-b33d-ce4b645093c5", + "parent_span_id": "29c7a515-2383-4b7d-8fc7-beac3f403641", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-09 08:25:12.442000000", + "end_time": "2025-10-09 08:25:12.442000000", + "duration_ms": 0.043000000000000003, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What features does Noveum Trace offer?", + "query_length": 38, + "query_type": "routing_evaluation_query", + "classification.mode": "RAG", + "classification.rag_keyword_score": 3, + "classification.web_keyword_score": 0, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "noveum_specific", + "query.keyword_density": 0.5, + "query.domain_affinity": "noveum", + "routing.decision": "Routed to RAG based on analysis", + "routing.rationale": "RAG score: 3, Web score: 0, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "Web Search", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.057692307692307696, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: RAG (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "16143f74-2831-4753-b33d-ce4b645093c5", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-09 08:25:12.442000000", + "trace_end_time": "2025-10-09 08:25:19.579000000", + "trace_duration_ms": 7137.402, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-09 08:25:20.290000000", + "updated_at": "2025-10-09 08:25:20.290000000" + }, + { + "span_id": "807627b0-41d9-478f-8a8b-26540d70af68", + "trace_id": "fc64e6cc-6739-4256-ac4a-7b80c3028233", + "parent_span_id": "12fee687-2a2e-401f-83c2-9c8dc7ecd853", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-09 08:25:07.932000000", + "end_time": "2025-10-09 08:25:07.932000000", + "duration_ms": 0.043000000000000003, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What are Noveum's pricing plans?", + "query_length": 32, + "query_type": "routing_evaluation_query", + "classification.mode": "RAG", + "classification.rag_keyword_score": 3, + "classification.web_keyword_score": 0, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "noveum_specific", + "query.keyword_density": 0.6, + "query.domain_affinity": "noveum", + "routing.decision": "Routed to RAG based on analysis", + "routing.rationale": "RAG score: 3, Web score: 0, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "Web Search", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.057692307692307696, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: RAG (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "fc64e6cc-6739-4256-ac4a-7b80c3028233", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-09 08:25:07.932000000", + "trace_end_time": "2025-10-09 08:25:11.438000000", + "trace_duration_ms": 3506.0989999999997, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-09 08:25:12.651000000", + "updated_at": "2025-10-09 08:25:12.651000000" + }, + { + "span_id": "bf860672-af22-44c1-b13b-0eabe8f72503", + "trace_id": "b7945c49-f584-4c70-972d-536a805d8a31", + "parent_span_id": "9c5aec32-b6eb-4c42-93d5-3970fb118c00", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-09 08:24:58.518000000", + "end_time": "2025-10-09 08:24:58.518000000", + "duration_ms": 0.054, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: How do I integrate Noveum Trace in my application?", + "query_length": 50, + "query_type": "routing_evaluation_query", + "classification.mode": "RAG", + "classification.rag_keyword_score": 2, + "classification.web_keyword_score": 0, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "noveum_specific", + "query.keyword_density": 0.2222222222222222, + "query.domain_affinity": "noveum", + "routing.decision": "Routed to RAG based on analysis", + "routing.rationale": "RAG score: 2, Web score: 0, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "Web Search", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.038461538461538464, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: RAG (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "b7945c49-f584-4c70-972d-536a805d8a31", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-09 08:24:58.518000000", + "trace_end_time": "2025-10-09 08:25:06.920000000", + "trace_duration_ms": 8402.516, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-09 08:25:07.620000000", + "updated_at": "2025-10-09 08:25:07.620000000" + }, + { + "span_id": "5fa714f1-eeb9-4617-bc7e-67062e6d1eab", + "trace_id": "f5c40ecf-36c0-45ba-9cc9-dc0329b0324b", + "parent_span_id": "f132d053-f82c-4e66-adf4-bd7ce68b660d", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-09 08:24:54.009000000", + "end_time": "2025-10-09 08:24:54.009000000", + "duration_ms": 0.022, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What is Noveum and what does it do?", + "query_length": 35, + "query_type": "routing_evaluation_query", + "classification.mode": "RAG", + "classification.rag_keyword_score": 1, + "classification.web_keyword_score": 0, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "noveum_specific", + "query.keyword_density": 0.125, + "query.domain_affinity": "noveum", + "routing.decision": "Routed to RAG based on analysis", + "routing.rationale": "RAG score: 1, Web score: 0, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "Web Search", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.019230769230769232, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: RAG (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "f5c40ecf-36c0-45ba-9cc9-dc0329b0324b", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-09 08:24:54.009000000", + "trace_end_time": "2025-10-09 08:24:57.511000000", + "trace_duration_ms": 3502.062, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-09 08:24:59.716000000", + "updated_at": "2025-10-09 08:24:59.716000000" + } +] \ No newline at end of file diff --git a/noveum_customer_support_bt/split_datasets/agent.web_search_evaluation_metrics_dataset.json b/noveum_customer_support_bt/split_datasets/agent.web_search_evaluation_metrics_dataset.json new file mode 100644 index 0000000..d4f2542 --- /dev/null +++ b/noveum_customer_support_bt/split_datasets/agent.web_search_evaluation_metrics_dataset.json @@ -0,0 +1,682 @@ +[ + { + "span_id": "3b7b6838-22d4-4edf-9b00-7cb011f18a52", + "trace_id": "eda4fe22-9a2b-4b73-856b-f4f3309bf719", + "parent_span_id": "2aca614b-b016-4d56-abe2-43a144dc681d", + "name": "agent.web_search_evaluation_metrics", + "start_time": "2025-10-09 08:26:48.685000000", + "end_time": "2025-10-09 08:26:48.685000000", + "duration_ms": 0.01, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "web_search_evaluation_metrics", + "agent.capabilities": [ + "web_search_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What are the latest breakthroughs in artificial intelligence?", + "query_length": 61, + "query_type": "web_search_evaluation_query", + "web_search.results_count": 1, + "web_search.context_length": 708, + "web_search.context_synthesized": "Context from 1 web sources", + "web_search.search_effectiveness": 0.2, + "web_search.context_quality": "medium", + "web_search.source_diversity": 1, + "web_search.information_synthesis": "low", + "web_search.external_knowledge_utilization": 0.708, + "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nJul 23, 2014 Ā· ꌉꈑäøŖäŗŗēš„ē†č§£ļ¼Œlatest čæ™äøŖčƍå¼ŗ调ēš„ę˜Æꗶ闓åŗåˆ—äøŠęœ€é čæ‘ēŽ°åœØļ¼Œč€Œ newest å¼ŗ调ēš„ę˜Æꖰꗧēš„é—®é¢˜ć€‚ęƔ如čÆ“ļ¼Œč‹¹ęžœēš„ newest äŗ§å“ļ¼ŒęÆ«ę— ē–‘é—®ę˜Æ Mac Proļ¼Œčæ™ę˜ÆåŽ»å¹“ę‰å‡ŗēŽ°ēš„äøœ ā€¦ Breaking news from Hong Kong, China and Asia including geopolitics, diplomacy, economy, technology, lifestyle, entertainment and sport. Latest China news, opinions and analysis, covering Xi Jinping, Beijing's relations with Taiwan and China's tensions with the US. Latest Singapore news, including the city stateā€™s relationships with Malaysia and Mahathir, China and Xi Jinping, and the rest of Southeast Asia. Horse racing news, fixtures, race reports, features and analysis.\nURL: https://duckduckgo.com/?q=What+are+the+latest+breakthroughs+in+artificial+intelligence?\n\nUser Question: What are the latest breakthroughs in artificial intelligence?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", + "prompt.prompt_length": 1214, + "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nJul 23, 2014 Ā· ꌉꈑäøŖäŗŗēš„ē†č§£ļ¼Œlatest čæ™äøŖčƍå¼ŗ调ēš„ę˜Æꗶ闓åŗåˆ—äøŠęœ€é čæ‘ēŽ°åœØļ¼Œč€Œ newest å¼ŗ调ēš„ę˜Æꖰꗧēš„é—®é¢˜ć€‚ęƔ如čÆ“ļ¼Œč‹¹ęžœēš„ newest äŗ§å“ļ¼ŒęÆ«ę— ē–‘é—®ę˜Æ Mac Proļ¼Œčæ™ę˜ÆåŽ»å¹“ę‰å‡ŗēŽ°ēš„äøœ ā€¦ Breaking news from Hong Kong, China and Asia including geopol...", + "prompt.instruction_following": "web_search_optimized", + "response.answer_length": 423, + "response.answer_completeness": "complete", + "response.response_quality": "medium", + "response.source_citation": 1, + "response.context_utilization": 0.597457627118644, + "output_response": "Web Search Answer: The provided web search results do not contain specific information about the latest breakthroughs in artificial intelligence. To find the most recent advancements in AI, I recommend checking reputabl...", + "evaluation.search_effectiveness": 0.2, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 1, + "evaluation.context_relevance": "medium", + "evaluation.overall_quality": "low", + "evaluation.ready_for_production": true, + "web_search.search_strategy": "duckduckgo_api", + "web_search.real_time_data": true, + "web_search.external_sources": 1, + "web_search.information_freshness": "current", + "web_search.knowledge_synthesis": "single_source" + }, + "events": [], + "links": [], + "trace_trace_id": "eda4fe22-9a2b-4b73-856b-f4f3309bf719", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-09 08:26:46.384000000", + "trace_end_time": "2025-10-09 08:26:48.685000000", + "trace_duration_ms": 2301.488, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-09 08:26:49.422000000", + "updated_at": "2025-10-09 08:26:49.422000000" + }, + { + "span_id": "a5d480f0-ea5b-4190-a3c3-f94e26b48573", + "trace_id": "0ffffba1-8a37-443c-8866-d53ffbfa7718", + "parent_span_id": "03e3ae4d-0107-401d-8ca0-05fc0f28fb24", + "name": "agent.web_search_evaluation_metrics", + "start_time": "2025-10-09 08:26:45.377000000", + "end_time": "2025-10-09 08:26:45.377000000", + "duration_ms": 0.025, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "web_search_evaluation_metrics", + "agent.capabilities": [ + "web_search_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What's happening in the software development world today?", + "query_length": 57, + "query_type": "web_search_evaluation_query", + "web_search.results_count": 1, + "web_search.context_length": 159, + "web_search.context_synthesized": "Context from 1 web sources", + "web_search.search_effectiveness": 0.2, + "web_search.context_quality": "low", + "web_search.source_diversity": 1, + "web_search.information_synthesis": "low", + "web_search.external_knowledge_utilization": 0.159, + "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What's+happening+in+the+software+development+world+today?\n\nUser Question: What's happening in the software development world today?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", + "prompt.prompt_length": 661, + "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What's+happening+in+the+software+development+world+today?", + "prompt.instruction_following": "web_search_optimized", + "response.answer_length": 795, + "response.answer_completeness": "complete", + "response.response_quality": "medium", + "response.source_citation": 1, + "response.context_utilization": 5, + "output_response": "Web Search Answer: The web search did not yield any specific results regarding current events in the software development world. Therefore, I cannot provide detailed information on what's happening today in that field.\n...", + "evaluation.search_effectiveness": 0.2, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 1, + "evaluation.context_relevance": "low", + "evaluation.overall_quality": "low", + "evaluation.ready_for_production": true, + "web_search.search_strategy": "duckduckgo_api", + "web_search.real_time_data": true, + "web_search.external_sources": 1, + "web_search.information_freshness": "current", + "web_search.knowledge_synthesis": "single_source" + }, + "events": [], + "links": [], + "trace_trace_id": "0ffffba1-8a37-443c-8866-d53ffbfa7718", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-09 08:26:41.850000000", + "trace_end_time": "2025-10-09 08:26:45.377000000", + "trace_duration_ms": 3527.831, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-09 08:26:49.422000000", + "updated_at": "2025-10-09 08:26:49.422000000" + }, + { + "span_id": "2fd285de-68a1-4b7a-a4ed-05f539afa737", + "trace_id": "43cdf081-4f01-49cd-b566-dbd1619e6cd2", + "parent_span_id": "dfdb9aaf-9925-4a5c-a0c6-2cb00f70d4d3", + "name": "agent.web_search_evaluation_metrics", + "start_time": "2025-10-09 08:26:34.906000000", + "end_time": "2025-10-09 08:26:34.906000000", + "duration_ms": 0.019, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "web_search_evaluation_metrics", + "agent.capabilities": [ + "web_search_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What's the current status of cryptocurrency markets?", + "query_length": 52, + "query_type": "web_search_evaluation_query", + "web_search.results_count": 1, + "web_search.context_length": 154, + "web_search.context_synthesized": "Context from 1 web sources", + "web_search.search_effectiveness": 0.2, + "web_search.context_quality": "low", + "web_search.source_diversity": 1, + "web_search.information_synthesis": "low", + "web_search.external_knowledge_utilization": 0.154, + "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What's+the+current+status+of+cryptocurrency+markets?\n\nUser Question: What's the current status of cryptocurrency markets?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", + "prompt.prompt_length": 651, + "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What's+the+current+status+of+cryptocurrency+markets?", + "prompt.instruction_following": "web_search_optimized", + "response.answer_length": 371, + "response.answer_completeness": "complete", + "response.response_quality": "medium", + "response.source_citation": 1, + "response.context_utilization": 2.409090909090909, + "output_response": "Web Search Answer: The web search results did not provide any specific information regarding the current status of cryptocurrency markets. Therefore, I cannot provide an accurate update on market conditions, trends, or ...", + "evaluation.search_effectiveness": 0.2, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 1, + "evaluation.context_relevance": "low", + "evaluation.overall_quality": "low", + "evaluation.ready_for_production": true, + "web_search.search_strategy": "duckduckgo_api", + "web_search.real_time_data": true, + "web_search.external_sources": 1, + "web_search.information_freshness": "current", + "web_search.knowledge_synthesis": "single_source" + }, + "events": [], + "links": [], + "trace_trace_id": "43cdf081-4f01-49cd-b566-dbd1619e6cd2", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-09 08:26:32.237000000", + "trace_end_time": "2025-10-09 08:26:34.906000000", + "trace_duration_ms": 2668.841, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-09 08:26:38.618000000", + "updated_at": "2025-10-09 08:26:38.618000000" + }, + { + "span_id": "48deb050-478d-486c-be3a-b8c0069d7abe", + "trace_id": "9a1983f4-09da-4b53-80e6-38de6878e0e7", + "parent_span_id": "eefcfdfc-6433-459e-a134-e2647b8418e4", + "name": "agent.web_search_evaluation_metrics", + "start_time": "2025-10-09 08:26:31.231000000", + "end_time": "2025-10-09 08:26:31.231000000", + "duration_ms": 0.022, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "web_search_evaluation_metrics", + "agent.capabilities": [ + "web_search_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What are the latest updates in Python programming?", + "query_length": 50, + "query_type": "web_search_evaluation_query", + "web_search.results_count": 1, + "web_search.context_length": 646, + "web_search.context_synthesized": "Context from 1 web sources", + "web_search.search_effectiveness": 0.2, + "web_search.context_quality": "medium", + "web_search.source_diversity": 1, + "web_search.information_synthesis": "low", + "web_search.external_knowledge_utilization": 0.646, + "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nNews, opinions, infographics and videos on China issues: geopolitics, diplomacy, economy, technology and lifestyle. Latest China news, opinions and analysis, covering Xi Jinping, Beijing's relations with Taiwan and China's tensions with the US. Breaking news from Hong Kong, China and Asia including geopolitics, diplomacy, economy, technology, lifestyle, entertainment and sport. Horse racing news, fixtures, race reports, features and analysis. All the latest news, opinions and analysis on Hong Kong, China, Asia and around the world\nURL: https://duckduckgo.com/?q=What+are+the+latest+updates+in+Python+programming?\n\nUser Question: What are the latest updates in Python programming?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", + "prompt.prompt_length": 1141, + "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nNews, opinions, infographics and videos on China issues: geopolitics, diplomacy, economy, technology and lifestyle. Latest China news, opinions and analysis, covering Xi Ji...", + "prompt.instruction_following": "web_search_optimized", + "response.answer_length": 583, + "response.answer_completeness": "complete", + "response.response_quality": "medium", + "response.source_citation": 1, + "response.context_utilization": 0.9024767801857585, + "output_response": "Web Search Answer: The web search results do not provide specific information about the latest updates in Python programming. They focus primarily on news related to China, including geopolitics, economy, and lifestyle,...", + "evaluation.search_effectiveness": 0.2, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 1, + "evaluation.context_relevance": "medium", + "evaluation.overall_quality": "low", + "evaluation.ready_for_production": true, + "web_search.search_strategy": "duckduckgo_api", + "web_search.real_time_data": true, + "web_search.external_sources": 1, + "web_search.information_freshness": "current", + "web_search.knowledge_synthesis": "single_source" + }, + "events": [], + "links": [], + "trace_trace_id": "9a1983f4-09da-4b53-80e6-38de6878e0e7", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-09 08:26:27.992000000", + "trace_end_time": "2025-10-09 08:26:31.231000000", + "trace_duration_ms": 3239.706, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-09 08:26:33.463000000", + "updated_at": "2025-10-09 08:26:33.463000000" + }, + { + "span_id": "4ecede78-e7fc-48ed-9fba-1241d6890a1d", + "trace_id": "5d2517e1-220a-429d-9d59-f701bda25eed", + "parent_span_id": "c57b4b54-b9b0-4a4c-b5cd-57a4cfe5301b", + "name": "agent.web_search_evaluation_metrics", + "start_time": "2025-10-09 08:26:26.985000000", + "end_time": "2025-10-09 08:26:26.985000000", + "duration_ms": 0.016, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "web_search_evaluation_metrics", + "agent.capabilities": [ + "web_search_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What happened in tech news this week?", + "query_length": 37, + "query_type": "web_search_evaluation_query", + "web_search.results_count": 1, + "web_search.context_length": 640, + "web_search.context_synthesized": "Context from 1 web sources", + "web_search.search_effectiveness": 0.2, + "web_search.context_quality": "medium", + "web_search.source_diversity": 1, + "web_search.information_synthesis": "low", + "web_search.external_knowledge_utilization": 0.64, + "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nhappenedēš„ē”Øę³•å¦‚äø‹ļ¼š 1.happenedę˜Æ äøåŠē‰©åŠØčƍ,作ā€œå‘ē”Ÿā€č§£ę—¶,åÆęŒ‡å¶ē„¶ēš„äŗ‹ę•…,也åÆęŒ‡ęœ‰č®”åˆ’ęˆ–ę— č®”åˆ’å‘ē”Ÿēš„äŗ‹ć€‚äø€čˆ¬ä»„äŗ‹ä»¶ć€ēŽÆå¢ƒć€å½¢åŠæē­‰ä½œäø»čÆ­,č”Øē¤ŗā€œęŸäŗŗ发ē”ŸęŸäŗ‹ā€ę—¶,锻仄介čƍto引å‡ŗā€œ ā€¦ what happenedčæ˜ę˜Æwhat's happened\"What happened\"和\"What's happened\"都ē”ØäŗŽčƢ问čæ‡åŽ»å‘ē”Ÿäŗ†ä»€ä¹ˆäŗ‹ęƒ…ļ¼Œäø»č¦ēš„åŒŗ别åœØäŗŽčÆ­ę³•å’Œä½æē”Øę–¹å¼ć€‚\"What happened\"ę˜Æäø€äøŖčæ‡åŽ»ę—¶ę€ēš„ē®€å• ꎄäø‹ę„č®©ęˆ‘ä»¬ēœ‹äø‹what happen和what happenedēš„ē”Øę³•åŒŗ别ļ¼š 1.čÆę€§ļ¼šwhat happenę˜Æē”Ø原形åŠØčƍhappenꞄꈐēš„ēŽ°åœØę—¶ēŸ­čÆ­ļ¼Œč€Œwhat happenedę˜Æē”ØåŠØčƍhappenēš„čæ‡åŽ»å¼happenedꞄꈐēš„čæ‡åŽ» ā€¦ what happened?和what's happening?ēš„åŒŗ别ę˜Æ什么ļ¼Ÿ \"What happen?\"č”Øē¤ŗ发ē”Ÿäŗ†ä»€ä¹ˆļ¼Œē”ØäŗŽčƢ问发ē”Ÿäŗ†ä»€ä¹ˆäŗ‹ęƒ…ļ¼Œé€šåøøē”ØäŗŽčæ‡åŽ»ēš„äŗ‹ä»¶ęˆ–ēŖå‘ęƒ…å†µć€‚ happened\"ę˜Æäø€äøŖå®Œę•“ēš„叄子ļ¼Œē®€å•čæ‡åŽ»ę—¶ę„č”Øē¤ŗčæ‡åŽ»å‘ēš„äŗ‹ęƒ…怂 它ę˜Æäø€äøŖē‹¬ēš„叄子ļ¼Œäøéœ€č¦å…¶ä»–ę–‡ęœ¬ęˆ–äøŠäø‹ę„ę”Æ\"What's happened\"\"what has happened\"ēš„ē¼©å½¢å¼ļ¼Œä½æē”ØēŽ°åœØå®Œęˆč”Øē¤ŗ去发ē”Ÿ ā€¦\nURL: https://duckduckgo.com/?q=What+happened+in+tech+news+this+week?\n\nUser Question: What happened in tech news this week?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", + "prompt.prompt_length": 1122, + "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nhappenedēš„ē”Øę³•å¦‚äø‹ļ¼š 1.happenedę˜Æ äøåŠē‰©åŠØčƍ,作ā€œå‘ē”Ÿā€č§£ę—¶,åÆęŒ‡å¶ē„¶ēš„äŗ‹ę•…,也åÆęŒ‡ęœ‰č®”åˆ’ęˆ–ę— č®”åˆ’å‘ē”Ÿēš„äŗ‹ć€‚äø€čˆ¬ä»„äŗ‹ä»¶ć€ēŽÆå¢ƒć€å½¢åŠæē­‰ä½œäø»čÆ­,č”Øē¤ŗā€œęŸäŗŗ发ē”ŸęŸäŗ‹ā€ę—¶,锻仄介čƍto引å‡ŗā€œ ā€¦ what happenedčæ˜ę˜Æwhat's happened\"What happened\"和\"What's happened\"都ē”ØäŗŽčƢ问...", + "prompt.instruction_following": "web_search_optimized", + "response.answer_length": 501, + "response.answer_completeness": "complete", + "response.response_quality": "medium", + "response.source_citation": 1, + "response.context_utilization": 0.7828125, + "output_response": "Web Search Answer: The web search results do not provide specific information about recent events in tech news for this week. They primarily discuss the grammatical usage of the phrase \"what happened\" and its variations...", + "evaluation.search_effectiveness": 0.2, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 1, + "evaluation.context_relevance": "medium", + "evaluation.overall_quality": "low", + "evaluation.ready_for_production": true, + "web_search.search_strategy": "duckduckgo_api", + "web_search.real_time_data": true, + "web_search.external_sources": 1, + "web_search.information_freshness": "current", + "web_search.knowledge_synthesis": "single_source" + }, + "events": [], + "links": [], + "trace_trace_id": "5d2517e1-220a-429d-9d59-f701bda25eed", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-09 08:26:23.210000000", + "trace_end_time": "2025-10-09 08:26:26.985000000", + "trace_duration_ms": 3775.124, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-09 08:26:28.174000000", + "updated_at": "2025-10-09 08:26:28.174000000" + }, + { + "span_id": "91fdf43c-7170-498e-9035-1e2b11b32b48", + "trace_id": "a81ca3a8-80aa-4c39-876e-8d40ea7a0aef", + "parent_span_id": "a062bef6-d973-489b-91df-5d40a2aef077", + "name": "agent.web_search_evaluation_metrics", + "start_time": "2025-10-09 08:26:15.537000000", + "end_time": "2025-10-09 08:26:15.537000000", + "duration_ms": 0.009000000000000001, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "web_search_evaluation_metrics", + "agent.capabilities": [ + "web_search_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: Tell me about recent developments in machine learning", + "query_length": 53, + "query_type": "web_search_evaluation_query", + "web_search.results_count": 1, + "web_search.context_length": 738, + "web_search.context_synthesized": "Context from 1 web sources", + "web_search.search_effectiveness": 0.2, + "web_search.context_quality": "medium", + "web_search.source_diversity": 1, + "web_search.information_synthesis": "low", + "web_search.external_knowledge_utilization": 0.738, + "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\näø€ć€Say和Tellēš„ē”Øę³• ęˆ‘ä»¬å…ˆå°†say态tell态speak态talkčæ™å››äøŖčÆåˆ†ęˆäø¤ē»„ļ¼Œåˆ†åˆ«äøŗļ¼š ē¬¬ā‘ ē»„ļ¼šsay & tellļ¼Œčæ™äø¤äøŖčƍ之后åÆ仄č”Øē¤ŗ谈čƝēš„ ā€œē»†čŠ‚ā€ć€‚ ē¬¬ā‘”ē»„ļ¼šspeak & talkļ¼Œäøčƒ½č”Øē¤ŗ谈čƝēš„ā€œē»†čŠ‚ā€ć€‚ 而ā€œ ā€¦ tell sb to do sth 和 tell sb do sth 都č”Øē¤ŗ告čÆ‰ęŸäŗŗåšęŸäŗ‹ļ¼Œtell sb to do sth å¼ŗč°ƒå‘ŠčÆ‰ęŸäŗŗåŗ”čƄ做什么具体åŠØä½œęˆ–č”Œäøŗļ¼Œč€Œ tell sb do sth å¼ŗč°ƒå‘ŠčÆ‰ęŸäŗŗę­£åœØčæ›č”Œēš„å…·ä½“č”ŒåŠØꈖēŠ¶ę€ć€‚ 它们åœØé‡Šä¹‰ć€ē”Øę³•ć€ä½æ ā€¦ Feb 3, 2024 Ā· \"tell sb sth\" ē”Øę„č”Øē¤ŗ告čÆ‰ęŸäŗŗäø€äøŖē›“ꎄēš„äæ”ęÆꈖäŗ‹å®žļ¼Œč€Œ \"tell sb about sth\" ꌇēš„ę˜Æå‘ęŸäŗŗ叙čæ°ęˆ–ꏏčæ°å…³äŗŽęŸäŗ‹ēš„čƦē»†ęƒ…å†µć€‚ ē®€å•ę„čÆ“ļ¼Œå‰č€…ꛓē›“ꎄļ¼ŒåŽč€…ꛓ倾向äŗŽęä¾›å…³äŗŽęŸäŗ‹ēš„čƒŒę™Æꈖ ā€¦ tell apart 和 tell fromēš„åŒŗ别åŒŗ别ļ¼šäø€ć€ę„ę€äøåŒć€‚ tell from 分č¾Øļ¼Œč¾Øåˆ«ć€å›ŗ定ē”Øę³•ęœ‰tell A from Bć€‘ä¾‹å„ļ¼šI can tell from your face you are lying ( to me). Aug 23, 2023 Ā· tellēš„čæ‡åŽ»å¼å’Œčæ‡åŽ»åˆ†čƍ都ę˜Ætold怂 čƻ音ļ¼šč‹±[tel]ļ¼Œē¾Ž[tel]怂 释义ļ¼š vt.告čƉļ¼ŒčÆ“ļ¼›č¾Ø别ļ¼›å©å’ļ¼›ę–­å®šć€‚ vi.č®²čæ°ļ¼›å‘Šå‘ļ¼Œę³„åƆļ¼›čÆ†åˆ«ć€‚ 变形ļ¼ščæ‡åŽ»å¼toldļ¼Œčæ‡åŽ»åˆ†čƍtoldļ¼ŒēŽ°åœØ分čƍtellingļ¼Œ ā€¦\nURL: https://duckduckgo.com/?q=Tell+me+about+recent+developments+in+machine+learning\n\nUser Question: Tell me about recent developments in machine learning\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", + "prompt.prompt_length": 1236, + "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\näø€ć€Say和Tellēš„ē”Øę³• ęˆ‘ä»¬å…ˆå°†say态tell态speak态talkčæ™å››äøŖčÆåˆ†ęˆäø¤ē»„ļ¼Œåˆ†åˆ«äøŗļ¼š ē¬¬ā‘ ē»„ļ¼šsay & tellļ¼Œčæ™äø¤äøŖčƍ之后åÆ仄č”Øē¤ŗ谈čƝēš„ ā€œē»†čŠ‚ā€ć€‚ ē¬¬ā‘”ē»„ļ¼šspeak & talkļ¼Œäøčƒ½č”Øē¤ŗ谈čƝēš„ā€œē»†čŠ‚ā€ć€‚ 而ā€œ ā€¦ tell sb to do sth 和 tell sb do sth 都č”Øē¤ŗ告čÆ‰ęŸäŗŗåšęŸäŗ‹ļ¼Œtell ...", + "prompt.instruction_following": "web_search_optimized", + "response.answer_length": 1639, + "response.answer_completeness": "complete", + "response.response_quality": "medium", + "response.source_citation": 1, + "response.context_utilization": 2.220867208672087, + "output_response": "Web Search Answer: The provided web search results do not contain specific information about recent developments in machine learning. Therefore, I cannot provide a detailed answer based on those results.\n\nHowever, I can...", + "evaluation.search_effectiveness": 0.2, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 1, + "evaluation.context_relevance": "medium", + "evaluation.overall_quality": "low", + "evaluation.ready_for_production": true, + "web_search.search_strategy": "duckduckgo_api", + "web_search.real_time_data": true, + "web_search.external_sources": 1, + "web_search.information_freshness": "current", + "web_search.knowledge_synthesis": "single_source" + }, + "events": [], + "links": [], + "trace_trace_id": "a81ca3a8-80aa-4c39-876e-8d40ea7a0aef", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-09 08:26:08.903000000", + "trace_end_time": "2025-10-09 08:26:15.537000000", + "trace_duration_ms": 6633.662, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-09 08:26:16.227000000", + "updated_at": "2025-10-09 08:26:16.227000000" + }, + { + "span_id": "096e6592-23fa-432a-931d-8f39656c7fcc", + "trace_id": "230aad27-f3dd-4968-a45a-3c2f07ac28ed", + "parent_span_id": "41f82a95-620d-440c-84e6-36d6d1b9e079", + "name": "agent.web_search_evaluation_metrics", + "start_time": "2025-10-09 08:26:07.894000000", + "end_time": "2025-10-09 08:26:07.894000000", + "duration_ms": 0.041, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "web_search_evaluation_metrics", + "agent.capabilities": [ + "web_search_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What's the weather like today?", + "query_length": 30, + "query_type": "web_search_evaluation_query", + "web_search.results_count": 1, + "web_search.context_length": 899, + "web_search.context_synthesized": "Context from 1 web sources", + "web_search.search_effectiveness": 0.2, + "web_search.context_quality": "high", + "web_search.source_diversity": 1, + "web_search.information_synthesis": "low", + "web_search.external_knowledge_utilization": 0.899, + "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nLog in to WhatsApp Web for simple, reliable and private messaging on your desktop. Send and receive messages and files with ease, all for free. 4 days ago Ā· Latest weather forecast for France for today's, hourly weather forecast, including today's temperatures in France, wind, rain and more. 2 days ago Ā· Get the latest hourly weather updates for United States today. Detailed forecast including temperature, wind, rain, snow, and UV index. Stay informed about today's weather ā€¦ 2 days ago Ā· Latest weather forecast for Singapore for today's, hourly weather forecast, including today's temperatures in Singapore, wind, rain and more. Aug 25, 2025 Ā· Sing and learn about the weather with the Witty Bunch! šŸŒŸThis fun, repetitive song helps children explore different kinds of weather while practising the que...\nURL: https://duckduckgo.com/?q=What's+the+weather+like+today?\n\nUser Question: What's the weather like today?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", + "prompt.prompt_length": 1374, + "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nLog in to WhatsApp Web for simple, reliable and private messaging on your desktop. Send and receive messages and files with ease, all for free. 4 days ago Ā· Latest weather ...", + "prompt.instruction_following": "web_search_optimized", + "response.answer_length": 376, + "response.answer_completeness": "complete", + "response.response_quality": "medium", + "response.source_citation": 1, + "response.context_utilization": 0.41824249165739713, + "output_response": "Web Search Answer: The provided web search results do not contain specific information about today's weather in any particular location. They mention weather forecasts for various places, including France and Singapore,...", + "evaluation.search_effectiveness": 0.2, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 1, + "evaluation.context_relevance": "high", + "evaluation.overall_quality": "low", + "evaluation.ready_for_production": true, + "web_search.search_strategy": "duckduckgo_api", + "web_search.real_time_data": true, + "web_search.external_sources": 1, + "web_search.information_freshness": "current", + "web_search.knowledge_synthesis": "single_source" + }, + "events": [], + "links": [], + "trace_trace_id": "230aad27-f3dd-4968-a45a-3c2f07ac28ed", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-09 08:26:02.424000000", + "trace_end_time": "2025-10-09 08:26:07.894000000", + "trace_duration_ms": 5470.232, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-09 08:26:08.586000000", + "updated_at": "2025-10-09 08:26:08.586000000" + }, + { + "span_id": "b56248e6-e526-4000-8c10-f5d13cf2e12d", + "trace_id": "83c7dcce-3d89-4da1-8b3f-d419885d4cbc", + "parent_span_id": "63f2a0d5-9016-4483-8ec0-e6aa6b27689f", + "name": "agent.web_search_evaluation_metrics", + "start_time": "2025-10-09 08:26:01.415000000", + "end_time": "2025-10-09 08:26:01.415000000", + "duration_ms": 0.095, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "web_search_evaluation_metrics", + "agent.capabilities": [ + "web_search_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What are the latest AI news today?", + "query_length": 34, + "query_type": "web_search_evaluation_query", + "web_search.results_count": 1, + "web_search.context_length": 981, + "web_search.context_synthesized": "Context from 1 web sources", + "web_search.search_effectiveness": 0.2, + "web_search.context_quality": "high", + "web_search.source_diversity": 1, + "web_search.information_synthesis": "low", + "web_search.external_knowledge_utilization": 0.981, + "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\n1 day ago Ā· Artificial Intelligence: Read latest updates on AI like Google AI, ChatGPT, Google Lamda, Bard chatbot and more along with latest news as AI technology advances and makes ā€¦ 4 days ago Ā· Get the latest AI news, live updates, and expert insights on artificial intelligence, machine learning, deep learning, and more at xix.ai. Handpicked daily for developers, ā€¦ 5 days ago Ā· Jeff Bezos warns that current AI spending resembles an \"industrial bubble,\" where both good and bad ideas receive funding, potentially leading to lost investments. However, he ā€¦ 6 days ago Ā· AI Magazine highlights this weekā€™s top stories, from Microsoft leadership changes to new language and video models from OpenAI and Anthropic 2 days ago Ā· Explore the latest artificial intelligence news with Reuters - from AI breakthroughs and technology trends to regulation, ethics, business and global impact.\nURL: https://duckduckgo.com/?q=What+are+the+latest+AI+news+today?\n\nUser Question: What are the latest AI news today?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", + "prompt.prompt_length": 1460, + "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\n1 day ago Ā· Artificial Intelligence: Read latest updates on AI like Google AI, ChatGPT, Google Lamda, Bard chatbot and more along with latest news as AI technology advances...", + "prompt.instruction_following": "web_search_optimized", + "response.answer_length": 1003, + "response.answer_completeness": "complete", + "response.response_quality": "medium", + "response.source_citation": 1, + "response.context_utilization": 1.0224260958205913, + "output_response": "Web Search Answer: The latest news in artificial intelligence includes several key updates:\n\n1. **AI Spending Concerns**: Jeff Bezos has expressed concerns that the current level of investment in AI resembles an \"indust...", + "evaluation.search_effectiveness": 0.2, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 1, + "evaluation.context_relevance": "high", + "evaluation.overall_quality": "low", + "evaluation.ready_for_production": true, + "web_search.search_strategy": "duckduckgo_api", + "web_search.real_time_data": true, + "web_search.external_sources": 1, + "web_search.information_freshness": "current", + "web_search.knowledge_synthesis": "single_source" + }, + "events": [], + "links": [], + "trace_trace_id": "83c7dcce-3d89-4da1-8b3f-d419885d4cbc", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-09 08:25:56.886000000", + "trace_end_time": "2025-10-09 08:26:01.416000000", + "trace_duration_ms": 4530.08, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-09 08:26:02.122000000", + "updated_at": "2025-10-09 08:26:02.122000000" + } +] \ No newline at end of file diff --git a/noveum_customer_support_bt/split_datasets/agent.web_search_generation_dataset.json b/noveum_customer_support_bt/split_datasets/agent.web_search_generation_dataset.json new file mode 100644 index 0000000..faa3fd1 --- /dev/null +++ b/noveum_customer_support_bt/split_datasets/agent.web_search_generation_dataset.json @@ -0,0 +1,490 @@ +[ + { + "span_id": "2aca614b-b016-4d56-abe2-43a144dc681d", + "trace_id": "eda4fe22-9a2b-4b73-856b-f4f3309bf719", + "parent_span_id": "0b19befc-5dc0-4b00-9a8b-5f2f2c914f17", + "name": "agent.web_search_generation", + "start_time": "2025-10-09 08:26:46.384000000", + "end_time": "2025-10-09 08:26:48.685000000", + "duration_ms": 2301.089, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "web_search_agent", + "agent.operation": "web_search_generation", + "agent.capabilities": [ + "web_search", + "content_synthesis", + "response_generation" + ], + "agent.id": "noveum_web_search_agent", + "input_query": "What are the latest breakthroughs in artificial intelligence?", + "query_length": 61, + "query_type": "web_search_query", + "output_response": "Web Search Answer: The provided web search results do not contain specific information about the latest breakthroughs in artificial intelligence. To find the most recent advancements in AI, I recommend checking reputabl...", + "web_search.results_count": 1, + "web_search.context_length": 708, + "web_search.response_length": 423, + "web_search.mode": "external_web_search" + }, + "events": [], + "links": [], + "trace_trace_id": "eda4fe22-9a2b-4b73-856b-f4f3309bf719", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-09 08:26:46.384000000", + "trace_end_time": "2025-10-09 08:26:48.685000000", + "trace_duration_ms": 2301.488, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-09 08:26:49.422000000", + "updated_at": "2025-10-09 08:26:49.422000000" + }, + { + "span_id": "03e3ae4d-0107-401d-8ca0-05fc0f28fb24", + "trace_id": "0ffffba1-8a37-443c-8866-d53ffbfa7718", + "parent_span_id": "30644d2a-5ab1-4a73-b4ff-cf9fa478e36b", + "name": "agent.web_search_generation", + "start_time": "2025-10-09 08:26:41.850000000", + "end_time": "2025-10-09 08:26:45.377000000", + "duration_ms": 3527.3050000000003, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "web_search_agent", + "agent.operation": "web_search_generation", + "agent.capabilities": [ + "web_search", + "content_synthesis", + "response_generation" + ], + "agent.id": "noveum_web_search_agent", + "input_query": "What's happening in the software development world today?", + "query_length": 57, + "query_type": "web_search_query", + "output_response": "Web Search Answer: The web search did not yield any specific results regarding current events in the software development world. Therefore, I cannot provide detailed information on what's happening today in that field.\n...", + "web_search.results_count": 1, + "web_search.context_length": 159, + "web_search.response_length": 795, + "web_search.mode": "external_web_search" + }, + "events": [], + "links": [], + "trace_trace_id": "0ffffba1-8a37-443c-8866-d53ffbfa7718", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-09 08:26:41.850000000", + "trace_end_time": "2025-10-09 08:26:45.377000000", + "trace_duration_ms": 3527.831, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-09 08:26:49.422000000", + "updated_at": "2025-10-09 08:26:49.422000000" + }, + { + "span_id": "dfdb9aaf-9925-4a5c-a0c6-2cb00f70d4d3", + "trace_id": "43cdf081-4f01-49cd-b566-dbd1619e6cd2", + "parent_span_id": "962afebb-26d1-45ff-acf1-218163cd6bb8", + "name": "agent.web_search_generation", + "start_time": "2025-10-09 08:26:32.240000000", + "end_time": "2025-10-09 08:26:34.906000000", + "duration_ms": 2665.69, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "web_search_agent", + "agent.operation": "web_search_generation", + "agent.capabilities": [ + "web_search", + "content_synthesis", + "response_generation" + ], + "agent.id": "noveum_web_search_agent", + "input_query": "What's the current status of cryptocurrency markets?", + "query_length": 52, + "query_type": "web_search_query", + "output_response": "Web Search Answer: The web search results did not provide any specific information regarding the current status of cryptocurrency markets. Therefore, I cannot provide an accurate update on market conditions, trends, or ...", + "web_search.results_count": 1, + "web_search.context_length": 154, + "web_search.response_length": 371, + "web_search.mode": "external_web_search" + }, + "events": [], + "links": [], + "trace_trace_id": "43cdf081-4f01-49cd-b566-dbd1619e6cd2", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-09 08:26:32.237000000", + "trace_end_time": "2025-10-09 08:26:34.906000000", + "trace_duration_ms": 2668.841, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-09 08:26:38.618000000", + "updated_at": "2025-10-09 08:26:38.618000000" + }, + { + "span_id": "eefcfdfc-6433-459e-a134-e2647b8418e4", + "trace_id": "9a1983f4-09da-4b53-80e6-38de6878e0e7", + "parent_span_id": "246cf822-4562-4d3a-8a5a-e38e0a0b02a0", + "name": "agent.web_search_generation", + "start_time": "2025-10-09 08:26:27.992000000", + "end_time": "2025-10-09 08:26:31.231000000", + "duration_ms": 3239.058, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "web_search_agent", + "agent.operation": "web_search_generation", + "agent.capabilities": [ + "web_search", + "content_synthesis", + "response_generation" + ], + "agent.id": "noveum_web_search_agent", + "input_query": "What are the latest updates in Python programming?", + "query_length": 50, + "query_type": "web_search_query", + "output_response": "Web Search Answer: The web search results do not provide specific information about the latest updates in Python programming. They focus primarily on news related to China, including geopolitics, economy, and lifestyle,...", + "web_search.results_count": 1, + "web_search.context_length": 646, + "web_search.response_length": 583, + "web_search.mode": "external_web_search" + }, + "events": [], + "links": [], + "trace_trace_id": "9a1983f4-09da-4b53-80e6-38de6878e0e7", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-09 08:26:27.992000000", + "trace_end_time": "2025-10-09 08:26:31.231000000", + "trace_duration_ms": 3239.706, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-09 08:26:33.463000000", + "updated_at": "2025-10-09 08:26:33.463000000" + }, + { + "span_id": "c57b4b54-b9b0-4a4c-b5cd-57a4cfe5301b", + "trace_id": "5d2517e1-220a-429d-9d59-f701bda25eed", + "parent_span_id": "20544f91-8828-4a0e-b253-ed1f8b95b205", + "name": "agent.web_search_generation", + "start_time": "2025-10-09 08:26:23.210000000", + "end_time": "2025-10-09 08:26:26.985000000", + "duration_ms": 3774.711, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "web_search_agent", + "agent.operation": "web_search_generation", + "agent.capabilities": [ + "web_search", + "content_synthesis", + "response_generation" + ], + "agent.id": "noveum_web_search_agent", + "input_query": "What happened in tech news this week?", + "query_length": 37, + "query_type": "web_search_query", + "output_response": "Web Search Answer: The web search results do not provide specific information about recent events in tech news for this week. They primarily discuss the grammatical usage of the phrase \"what happened\" and its variations...", + "web_search.results_count": 1, + "web_search.context_length": 640, + "web_search.response_length": 501, + "web_search.mode": "external_web_search" + }, + "events": [], + "links": [], + "trace_trace_id": "5d2517e1-220a-429d-9d59-f701bda25eed", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-09 08:26:23.210000000", + "trace_end_time": "2025-10-09 08:26:26.985000000", + "trace_duration_ms": 3775.124, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-09 08:26:28.174000000", + "updated_at": "2025-10-09 08:26:28.174000000" + }, + { + "span_id": "a062bef6-d973-489b-91df-5d40a2aef077", + "trace_id": "a81ca3a8-80aa-4c39-876e-8d40ea7a0aef", + "parent_span_id": "acf6a99a-39e0-4f87-a07e-3a61213ab032", + "name": "agent.web_search_generation", + "start_time": "2025-10-09 08:26:08.904000000", + "end_time": "2025-10-09 08:26:15.537000000", + "duration_ms": 6633.127, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "web_search_agent", + "agent.operation": "web_search_generation", + "agent.capabilities": [ + "web_search", + "content_synthesis", + "response_generation" + ], + "agent.id": "noveum_web_search_agent", + "input_query": "Tell me about recent developments in machine learning", + "query_length": 53, + "query_type": "web_search_query", + "output_response": "Web Search Answer: The provided web search results do not contain specific information about recent developments in machine learning. Therefore, I cannot provide a detailed answer based on those results.\n\nHowever, I can...", + "web_search.results_count": 1, + "web_search.context_length": 738, + "web_search.response_length": 1639, + "web_search.mode": "external_web_search" + }, + "events": [], + "links": [], + "trace_trace_id": "a81ca3a8-80aa-4c39-876e-8d40ea7a0aef", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-09 08:26:08.903000000", + "trace_end_time": "2025-10-09 08:26:15.537000000", + "trace_duration_ms": 6633.662, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-09 08:26:16.227000000", + "updated_at": "2025-10-09 08:26:16.227000000" + }, + { + "span_id": "41f82a95-620d-440c-84e6-36d6d1b9e079", + "trace_id": "230aad27-f3dd-4968-a45a-3c2f07ac28ed", + "parent_span_id": "0b47ea07-1eda-4b6a-a183-ea9ef047e431", + "name": "agent.web_search_generation", + "start_time": "2025-10-09 08:26:02.424000000", + "end_time": "2025-10-09 08:26:07.894000000", + "duration_ms": 5469.763000000001, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "web_search_agent", + "agent.operation": "web_search_generation", + "agent.capabilities": [ + "web_search", + "content_synthesis", + "response_generation" + ], + "agent.id": "noveum_web_search_agent", + "input_query": "What's the weather like today?", + "query_length": 30, + "query_type": "web_search_query", + "output_response": "Web Search Answer: The provided web search results do not contain specific information about today's weather in any particular location. They mention weather forecasts for various places, including France and Singapore,...", + "web_search.results_count": 1, + "web_search.context_length": 899, + "web_search.response_length": 376, + "web_search.mode": "external_web_search" + }, + "events": [], + "links": [], + "trace_trace_id": "230aad27-f3dd-4968-a45a-3c2f07ac28ed", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-09 08:26:02.424000000", + "trace_end_time": "2025-10-09 08:26:07.894000000", + "trace_duration_ms": 5470.232, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-09 08:26:08.586000000", + "updated_at": "2025-10-09 08:26:08.586000000" + }, + { + "span_id": "63f2a0d5-9016-4483-8ec0-e6aa6b27689f", + "trace_id": "83c7dcce-3d89-4da1-8b3f-d419885d4cbc", + "parent_span_id": "3866ddb3-b17e-47a1-8b3f-bca1e32a2ae4", + "name": "agent.web_search_generation", + "start_time": "2025-10-09 08:25:56.886000000", + "end_time": "2025-10-09 08:26:01.415000000", + "duration_ms": 4529.669, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "web_search_agent", + "agent.operation": "web_search_generation", + "agent.capabilities": [ + "web_search", + "content_synthesis", + "response_generation" + ], + "agent.id": "noveum_web_search_agent", + "input_query": "What are the latest AI news today?", + "query_length": 34, + "query_type": "web_search_query", + "output_response": "Web Search Answer: The latest news in artificial intelligence includes several key updates:\n\n1. **AI Spending Concerns**: Jeff Bezos has expressed concerns that the current level of investment in AI resembles an \"indust...", + "web_search.results_count": 1, + "web_search.context_length": 981, + "web_search.response_length": 1003, + "web_search.mode": "external_web_search" + }, + "events": [], + "links": [], + "trace_trace_id": "83c7dcce-3d89-4da1-8b3f-d419885d4cbc", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-09 08:25:56.886000000", + "trace_end_time": "2025-10-09 08:26:01.416000000", + "trace_duration_ms": 4530.08, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-09 08:26:02.122000000", + "updated_at": "2025-10-09 08:26:02.122000000" + } +] \ No newline at end of file diff --git a/noveum_customer_support_bt/split_datasets/tool-orchestator_dataset.json b/noveum_customer_support_bt/split_datasets/tool-orchestator_dataset.json new file mode 100644 index 0000000..6855e0f --- /dev/null +++ b/noveum_customer_support_bt/split_datasets/tool-orchestator_dataset.json @@ -0,0 +1,1162 @@ +[ + { + "span_id": "b6b7c1c0-3b7b-4e19-805d-a16beb7ea871", + "trace_id": "eda4fe22-9a2b-4b73-856b-f4f3309bf719", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-09 08:26:46.384000000", + "end_time": "2025-10-09 08:26:48.685000000", + "duration_ms": 2301.412, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What are the latest breakthroughs in artificial intelligence?", + "query.length": 61, + "processing.mode": "Web Search", + "processing.time_seconds": 2.301326274871826, + "processing.response_length": 423, + "processing.sources_count": 1, + "output_response": "Final Answer: The provided web search results do not contain specific information about the latest breakthroughs in artificial intelligence. To find the most recent advancements in AI, I recommend checking reputabl...", + "final_answer_mode": "Web Search", + "query_processed.input_query": "What are the latest breakthroughs in artificial intelligence?", + "query_processed.output_response": "Successfully processed query using Web Search, generated 423 character response", + "query_processed.mode": "Web Search", + "query_processed.processing_time": 2.301326274871826, + "query_processed.response_length": 423 + }, + "events": [], + "links": [], + "trace_trace_id": "eda4fe22-9a2b-4b73-856b-f4f3309bf719", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-09 08:26:46.384000000", + "trace_end_time": "2025-10-09 08:26:48.685000000", + "trace_duration_ms": 2301.488, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-09 08:26:49.422000000", + "updated_at": "2025-10-09 08:26:49.422000000" + }, + { + "span_id": "3045d971-b7ff-46cc-be11-6db7d65dfefd", + "trace_id": "0ffffba1-8a37-443c-8866-d53ffbfa7718", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-09 08:26:41.850000000", + "end_time": "2025-10-09 08:26:45.377000000", + "duration_ms": 3527.75, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What's happening in the software development world today?", + "query.length": 57, + "processing.mode": "Web Search", + "processing.time_seconds": 3.527592897415161, + "processing.response_length": 795, + "processing.sources_count": 1, + "output_response": "Final Answer: The web search did not yield any specific results regarding current events in the software development world. Therefore, I cannot provide detailed information on what's happening today in that field.\n...", + "final_answer_mode": "Web Search", + "query_processed.input_query": "What's happening in the software development world today?", + "query_processed.output_response": "Successfully processed query using Web Search, generated 795 character response", + "query_processed.mode": "Web Search", + "query_processed.processing_time": 3.527592897415161, + "query_processed.response_length": 795 + }, + "events": [], + "links": [], + "trace_trace_id": "0ffffba1-8a37-443c-8866-d53ffbfa7718", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-09 08:26:41.850000000", + "trace_end_time": "2025-10-09 08:26:45.377000000", + "trace_duration_ms": 3527.831, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-09 08:26:49.422000000", + "updated_at": "2025-10-09 08:26:49.422000000" + }, + { + "span_id": "42d4cb1f-17ec-433a-8087-90292c2fa9d9", + "trace_id": "f1f37bd7-0851-4659-b493-b80d3800d920", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-09 08:26:35.915000000", + "end_time": "2025-10-09 08:26:40.841000000", + "duration_ms": 4926.2519999999995, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What are the newest features in cloud computing?", + "query.length": 48, + "processing.mode": "RAG", + "processing.time_seconds": 4.926196813583374, + "processing.response_length": 883, + "processing.sources_count": 5, + "output_response": "Final Answer: The provided context does not specifically mention any new features in cloud computing as a general topic. However, it does highlight several recent enhancements and features related to the Noveum.ai ...", + "final_answer_mode": "RAG", + "query_processed.input_query": "What are the newest features in cloud computing?", + "query_processed.output_response": "Successfully processed query using RAG, generated 883 character response", + "query_processed.mode": "RAG", + "query_processed.processing_time": 4.926196813583374, + "query_processed.response_length": 883 + }, + "events": [], + "links": [], + "trace_trace_id": "f1f37bd7-0851-4659-b493-b80d3800d920", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-09 08:26:35.915000000", + "trace_end_time": "2025-10-09 08:26:40.841000000", + "trace_duration_ms": 4926.342, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-09 08:26:44.053000000", + "updated_at": "2025-10-09 08:26:44.053000000" + }, + { + "span_id": "e22b809d-5320-40b7-b78a-e3e125538a6b", + "trace_id": "43cdf081-4f01-49cd-b566-dbd1619e6cd2", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-09 08:26:32.239000000", + "end_time": "2025-10-09 08:26:34.906000000", + "duration_ms": 2667.1330000000003, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What's the current status of cryptocurrency markets?", + "query.length": 52, + "processing.mode": "Web Search", + "processing.time_seconds": 2.6669082641601562, + "processing.response_length": 371, + "processing.sources_count": 1, + "output_response": "Final Answer: The web search results did not provide any specific information regarding the current status of cryptocurrency markets. Therefore, I cannot provide an accurate update on market conditions, trends, or ...", + "final_answer_mode": "Web Search", + "query_processed.input_query": "What's the current status of cryptocurrency markets?", + "query_processed.output_response": "Successfully processed query using Web Search, generated 371 character response", + "query_processed.mode": "Web Search", + "query_processed.processing_time": 2.6669082641601562, + "query_processed.response_length": 371 + }, + "events": [], + "links": [], + "trace_trace_id": "43cdf081-4f01-49cd-b566-dbd1619e6cd2", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-09 08:26:32.237000000", + "trace_end_time": "2025-10-09 08:26:34.906000000", + "trace_duration_ms": 2668.841, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-09 08:26:38.618000000", + "updated_at": "2025-10-09 08:26:38.618000000" + }, + { + "span_id": "813a4450-1cf8-4d43-8493-eabfa200919b", + "trace_id": "9a1983f4-09da-4b53-80e6-38de6878e0e7", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-09 08:26:27.992000000", + "end_time": "2025-10-09 08:26:31.231000000", + "duration_ms": 3239.601, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What are the latest updates in Python programming?", + "query.length": 50, + "processing.mode": "Web Search", + "processing.time_seconds": 3.2394051551818848, + "processing.response_length": 583, + "processing.sources_count": 1, + "output_response": "Final Answer: The web search results do not provide specific information about the latest updates in Python programming. They focus primarily on news related to China, including geopolitics, economy, and lifestyle,...", + "final_answer_mode": "Web Search", + "query_processed.input_query": "What are the latest updates in Python programming?", + "query_processed.output_response": "Successfully processed query using Web Search, generated 583 character response", + "query_processed.mode": "Web Search", + "query_processed.processing_time": 3.2394051551818848, + "query_processed.response_length": 583 + }, + "events": [], + "links": [], + "trace_trace_id": "9a1983f4-09da-4b53-80e6-38de6878e0e7", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-09 08:26:27.992000000", + "trace_end_time": "2025-10-09 08:26:31.231000000", + "trace_duration_ms": 3239.706, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-09 08:26:33.463000000", + "updated_at": "2025-10-09 08:26:33.463000000" + }, + { + "span_id": "6102352a-833e-493d-adbb-887e2af17918", + "trace_id": "5d2517e1-220a-429d-9d59-f701bda25eed", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-09 08:26:23.210000000", + "end_time": "2025-10-09 08:26:26.985000000", + "duration_ms": 3775.0409999999997, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What happened in tech news this week?", + "query.length": 37, + "processing.mode": "Web Search", + "processing.time_seconds": 3.774940013885498, + "processing.response_length": 501, + "processing.sources_count": 1, + "output_response": "Final Answer: The web search results do not provide specific information about recent events in tech news for this week. They primarily discuss the grammatical usage of the phrase \"what happened\" and its variations...", + "final_answer_mode": "Web Search", + "query_processed.input_query": "What happened in tech news this week?", + "query_processed.output_response": "Successfully processed query using Web Search, generated 501 character response", + "query_processed.mode": "Web Search", + "query_processed.processing_time": 3.774940013885498, + "query_processed.response_length": 501 + }, + "events": [], + "links": [], + "trace_trace_id": "5d2517e1-220a-429d-9d59-f701bda25eed", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-09 08:26:23.210000000", + "trace_end_time": "2025-10-09 08:26:26.985000000", + "trace_duration_ms": 3775.124, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-09 08:26:28.174000000", + "updated_at": "2025-10-09 08:26:28.174000000" + }, + { + "span_id": "2926418a-fe09-4df0-bf19-a527bc24a478", + "trace_id": "52aacb67-c361-4445-9b72-c157f79f47d6", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-09 08:26:16.544000000", + "end_time": "2025-10-09 08:26:22.201000000", + "duration_ms": 5656.852, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What are the current trends in observability tools?", + "query.length": 51, + "processing.mode": "RAG", + "processing.time_seconds": 5.656759023666382, + "processing.response_length": 1074, + "processing.sources_count": 5, + "output_response": "Final Answer: The provided context does not explicitly outline current trends in observability tools. However, it highlights some key aspects of observability that are particularly relevant to AI applications. \n\n1....", + "final_answer_mode": "RAG", + "query_processed.input_query": "What are the current trends in observability tools?", + "query_processed.output_response": "Successfully processed query using RAG, generated 1074 character response", + "query_processed.mode": "RAG", + "query_processed.processing_time": 5.656759023666382, + "query_processed.response_length": 1074 + }, + "events": [], + "links": [], + "trace_trace_id": "52aacb67-c361-4445-9b72-c157f79f47d6", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-09 08:26:16.544000000", + "trace_end_time": "2025-10-09 08:26:22.201000000", + "trace_duration_ms": 5657.012, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-09 08:26:22.904000000", + "updated_at": "2025-10-09 08:26:22.904000000" + }, + { + "span_id": "afe85a1d-13c8-4476-8154-52d58ba83d52", + "trace_id": "a81ca3a8-80aa-4c39-876e-8d40ea7a0aef", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-09 08:26:08.903000000", + "end_time": "2025-10-09 08:26:15.537000000", + "duration_ms": 6633.559, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "Tell me about recent developments in machine learning", + "query.length": 53, + "processing.mode": "Web Search", + "processing.time_seconds": 6.63348126411438, + "processing.response_length": 1639, + "processing.sources_count": 1, + "output_response": "Final Answer: The provided web search results do not contain specific information about recent developments in machine learning. Therefore, I cannot provide a detailed answer based on those results.\n\nHowever, I can...", + "final_answer_mode": "Web Search", + "query_processed.input_query": "Tell me about recent developments in machine learning", + "query_processed.output_response": "Successfully processed query using Web Search, generated 1639 character response", + "query_processed.mode": "Web Search", + "query_processed.processing_time": 6.63348126411438, + "query_processed.response_length": 1639 + }, + "events": [], + "links": [], + "trace_trace_id": "a81ca3a8-80aa-4c39-876e-8d40ea7a0aef", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-09 08:26:08.903000000", + "trace_end_time": "2025-10-09 08:26:15.537000000", + "trace_duration_ms": 6633.662, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-09 08:26:16.227000000", + "updated_at": "2025-10-09 08:26:16.227000000" + }, + { + "span_id": "db8da393-6d72-421e-87dc-96f56b6576e5", + "trace_id": "230aad27-f3dd-4968-a45a-3c2f07ac28ed", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-09 08:26:02.424000000", + "end_time": "2025-10-09 08:26:07.894000000", + "duration_ms": 5470.179, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What's the weather like today?", + "query.length": 30, + "processing.mode": "Web Search", + "processing.time_seconds": 5.469927072525024, + "processing.response_length": 376, + "processing.sources_count": 1, + "output_response": "Final Answer: The provided web search results do not contain specific information about today's weather in any particular location. They mention weather forecasts for various places, including France and Singapore,...", + "final_answer_mode": "Web Search", + "query_processed.input_query": "What's the weather like today?", + "query_processed.output_response": "Successfully processed query using Web Search, generated 376 character response", + "query_processed.mode": "Web Search", + "query_processed.processing_time": 5.469927072525024, + "query_processed.response_length": 376 + }, + "events": [], + "links": [], + "trace_trace_id": "230aad27-f3dd-4968-a45a-3c2f07ac28ed", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-09 08:26:02.424000000", + "trace_end_time": "2025-10-09 08:26:07.894000000", + "trace_duration_ms": 5470.232, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-09 08:26:08.586000000", + "updated_at": "2025-10-09 08:26:08.586000000" + }, + { + "span_id": "a290202d-1b3f-431f-8728-fcb937380f8c", + "trace_id": "83c7dcce-3d89-4da1-8b3f-d419885d4cbc", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-09 08:25:56.886000000", + "end_time": "2025-10-09 08:26:01.416000000", + "duration_ms": 4530.039000000001, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What are the latest AI news today?", + "query.length": 34, + "processing.mode": "Web Search", + "processing.time_seconds": 4.529800176620483, + "processing.response_length": 1003, + "processing.sources_count": 1, + "output_response": "Final Answer: The latest news in artificial intelligence includes several key updates:\n\n1. **AI Spending Concerns**: Jeff Bezos has expressed concerns that the current level of investment in AI resembles an \"indust...", + "final_answer_mode": "Web Search", + "query_processed.input_query": "What are the latest AI news today?", + "query_processed.output_response": "Successfully processed query using Web Search, generated 1003 character response", + "query_processed.mode": "Web Search", + "query_processed.processing_time": 4.529800176620483, + "query_processed.response_length": 1003 + }, + "events": [], + "links": [], + "trace_trace_id": "83c7dcce-3d89-4da1-8b3f-d419885d4cbc", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-09 08:25:56.886000000", + "trace_end_time": "2025-10-09 08:26:01.416000000", + "trace_duration_ms": 4530.08, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-09 08:26:02.122000000", + "updated_at": "2025-10-09 08:26:02.122000000" + }, + { + "span_id": "5fbaef83-95fc-4bed-9695-9d69c37c35e4", + "trace_id": "2218f641-604c-491a-9710-b51a9941b982", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-09 08:25:48.107000000", + "end_time": "2025-10-09 08:25:55.878000000", + "duration_ms": 7770.846, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What are the benefits of using Noveum Trace?", + "query.length": 44, + "processing.mode": "RAG", + "processing.time_seconds": 7.770798206329346, + "processing.response_length": 1460, + "processing.sources_count": 5, + "output_response": "Final Answer: Using Noveum Trace offers several benefits, including:\n\n1. **Real-Time Monitoring**: The Noveum platform provides a real-time dashboard for analyzing traces and performance, with live updates as new t...", + "final_answer_mode": "RAG", + "query_processed.input_query": "What are the benefits of using Noveum Trace?", + "query_processed.output_response": "Successfully processed query using RAG, generated 1460 character response", + "query_processed.mode": "RAG", + "query_processed.processing_time": 7.770798206329346, + "query_processed.response_length": 1460 + }, + "events": [], + "links": [], + "trace_trace_id": "2218f641-604c-491a-9710-b51a9941b982", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-09 08:25:48.107000000", + "trace_end_time": "2025-10-09 08:25:55.878000000", + "trace_duration_ms": 7770.953, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-09 08:25:56.565000000", + "updated_at": "2025-10-09 08:25:56.565000000" + }, + { + "span_id": "b3ac3924-928b-4d48-a2a7-aa749802fef9", + "trace_id": "255fd49c-84b4-4b18-887e-6308a412d535", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-09 08:25:43.830000000", + "end_time": "2025-10-09 08:25:47.099000000", + "duration_ms": 3268.8160000000003, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "How do I configure Noveum for my system?", + "query.length": 40, + "processing.mode": "RAG", + "processing.time_seconds": 3.268723249435425, + "processing.response_length": 613, + "processing.sources_count": 5, + "output_response": "Final Answer: To configure Noveum for your system, you can refer to the SDK Integration Guide available in the Noveum documentation. This guide will help you trace your AI applications with minimal code changes. Th...", + "final_answer_mode": "RAG", + "query_processed.input_query": "How do I configure Noveum for my system?", + "query_processed.output_response": "Successfully processed query using RAG, generated 613 character response", + "query_processed.mode": "RAG", + "query_processed.processing_time": 3.268723249435425, + "query_processed.response_length": 613 + }, + "events": [], + "links": [], + "trace_trace_id": "255fd49c-84b4-4b18-887e-6308a412d535", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-09 08:25:43.830000000", + "trace_end_time": "2025-10-09 08:25:47.099000000", + "trace_duration_ms": 3268.862, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-09 08:25:50.304000000", + "updated_at": "2025-10-09 08:25:50.304000000" + }, + { + "span_id": "b8055ffc-ee7d-46ad-b6a0-84e80f9399e2", + "trace_id": "dc511122-c0b6-415c-9a49-c7b45132dd87", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-09 08:25:38.322000000", + "end_time": "2025-10-09 08:25:42.822000000", + "duration_ms": 4500.335, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What monitoring capabilities does Noveum provide?", + "query.length": 49, + "processing.mode": "RAG", + "processing.time_seconds": 4.5002899169921875, + "processing.response_length": 839, + "processing.sources_count": 5, + "output_response": "Final Answer: Noveum provides comprehensive monitoring capabilities for AI applications, including the ability to monitor, trace, and optimize AI agents across various frameworks such as LangChain, CrewAI, AutoGen,...", + "final_answer_mode": "RAG", + "query_processed.input_query": "What monitoring capabilities does Noveum provide?", + "query_processed.output_response": "Successfully processed query using RAG, generated 839 character response", + "query_processed.mode": "RAG", + "query_processed.processing_time": 4.5002899169921875, + "query_processed.response_length": 839 + }, + "events": [], + "links": [], + "trace_trace_id": "dc511122-c0b6-415c-9a49-c7b45132dd87", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-09 08:25:38.322000000", + "trace_end_time": "2025-10-09 08:25:42.822000000", + "trace_duration_ms": 4500.425, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-09 08:25:45.028000000", + "updated_at": "2025-10-09 08:25:45.028000000" + }, + { + "span_id": "532b9e1f-5c4f-4b41-98bb-e9a1510f0f1c", + "trace_id": "04bebf38-a343-4563-80db-0154bef8d927", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-09 08:25:33.693000000", + "end_time": "2025-10-09 08:25:37.313000000", + "duration_ms": 3619.5789999999997, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "How does Noveum handle agent tracing?", + "query.length": 37, + "processing.mode": "RAG", + "processing.time_seconds": 3.6195271015167236, + "processing.response_length": 691, + "processing.sources_count": 5, + "output_response": "Final Answer: Noveum.ai handles agent tracing through its specialized Multi-Agent Tracing capabilities, which are designed to observe complex workflows and inter-agent communications within multi-agent systems. The...", + "final_answer_mode": "RAG", + "query_processed.input_query": "How does Noveum handle agent tracing?", + "query_processed.output_response": "Successfully processed query using RAG, generated 691 character response", + "query_processed.mode": "RAG", + "query_processed.processing_time": 3.6195271015167236, + "query_processed.response_length": 691 + }, + "events": [], + "links": [], + "trace_trace_id": "04bebf38-a343-4563-80db-0154bef8d927", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-09 08:25:33.693000000", + "trace_end_time": "2025-10-09 08:25:37.313000000", + "trace_duration_ms": 3619.672, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-09 08:25:40.339000000", + "updated_at": "2025-10-09 08:25:40.339000000" + }, + { + "span_id": "14064e3f-f9c9-4bb7-b8cb-1ad52938b075", + "trace_id": "5e043630-6493-42b5-beb8-79faa19bfa37", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-09 08:25:29.901000000", + "end_time": "2025-10-09 08:25:32.685000000", + "duration_ms": 2783.257, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What APIs are available in Noveum platform?", + "query.length": 43, + "processing.mode": "RAG", + "processing.time_seconds": 2.783203125, + "processing.response_length": 270, + "processing.sources_count": 5, + "output_response": "Final Answer: The provided context does not specify the APIs available in the Noveum platform. For detailed information about the APIs, I recommend checking the official Noveum documentation or the SDK Integration ...", + "final_answer_mode": "RAG", + "query_processed.input_query": "What APIs are available in Noveum platform?", + "query_processed.output_response": "Successfully processed query using RAG, generated 270 character response", + "query_processed.mode": "RAG", + "query_processed.processing_time": 2.783203125, + "query_processed.response_length": 270 + }, + "events": [], + "links": [], + "trace_trace_id": "5e043630-6493-42b5-beb8-79faa19bfa37", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-09 08:25:29.901000000", + "trace_end_time": "2025-10-09 08:25:32.685000000", + "trace_duration_ms": 2783.326, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-09 08:25:34.918000000", + "updated_at": "2025-10-09 08:25:34.918000000" + }, + { + "span_id": "a74fe8a9-2654-4b9d-ba87-dc2f86b6466d", + "trace_id": "7da9814d-a2e8-4c4e-b750-68b26bd5fd22", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-09 08:25:20.586000000", + "end_time": "2025-10-09 08:25:28.897000000", + "duration_ms": 8310.113, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "How do I set up observability with Noveum?", + "query.length": 42, + "processing.mode": "RAG", + "processing.time_seconds": 8.31006908416748, + "processing.response_length": 951, + "processing.sources_count": 5, + "output_response": "Final Answer: To set up observability with Noveum, follow these steps:\n\n1. **Sign Up**: Go to [noveum.ai](https://noveum.ai) and create an account.\n2. **Create a Project**: Once signed in, create a new project to o...", + "final_answer_mode": "RAG", + "query_processed.input_query": "How do I set up observability with Noveum?", + "query_processed.output_response": "Successfully processed query using RAG, generated 951 character response", + "query_processed.mode": "RAG", + "query_processed.processing_time": 8.31006908416748, + "query_processed.response_length": 951 + }, + "events": [], + "links": [], + "trace_trace_id": "7da9814d-a2e8-4c4e-b750-68b26bd5fd22", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-09 08:25:20.586000000", + "trace_end_time": "2025-10-09 08:25:28.897000000", + "trace_duration_ms": 8310.205, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-09 08:25:29.588000000", + "updated_at": "2025-10-09 08:25:29.588000000" + }, + { + "span_id": "c700d0f4-f1f8-4339-aa42-a3060ae2ae11", + "trace_id": "16143f74-2831-4753-b33d-ce4b645093c5", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-09 08:25:12.442000000", + "end_time": "2025-10-09 08:25:19.579000000", + "duration_ms": 7137.314, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What features does Noveum Trace offer?", + "query.length": 38, + "processing.mode": "RAG", + "processing.time_seconds": 7.1372880935668945, + "processing.response_length": 1014, + "processing.sources_count": 5, + "output_response": "Final Answer: Noveum Trace offers several key features through its Python SDK (`noveum-trace`) and TypeScript SDK (`@noveum/trace`):\n\n1. **Decorator-based Tracing**: It provides a simple way to trace LLM calls, age...", + "final_answer_mode": "RAG", + "query_processed.input_query": "What features does Noveum Trace offer?", + "query_processed.output_response": "Successfully processed query using RAG, generated 1014 character response", + "query_processed.mode": "RAG", + "query_processed.processing_time": 7.1372880935668945, + "query_processed.response_length": 1014 + }, + "events": [], + "links": [], + "trace_trace_id": "16143f74-2831-4753-b33d-ce4b645093c5", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-09 08:25:12.442000000", + "trace_end_time": "2025-10-09 08:25:19.579000000", + "trace_duration_ms": 7137.402, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-09 08:25:20.290000000", + "updated_at": "2025-10-09 08:25:20.290000000" + }, + { + "span_id": "f7f5981f-d840-4265-8970-d319b876366d", + "trace_id": "fc64e6cc-6739-4256-ac4a-7b80c3028233", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-09 08:25:07.932000000", + "end_time": "2025-10-09 08:25:11.438000000", + "duration_ms": 3506.0159999999996, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What are Noveum's pricing plans?", + "query.length": 32, + "processing.mode": "RAG", + "processing.time_seconds": 3.5059969425201416, + "processing.response_length": 399, + "processing.sources_count": 5, + "output_response": "Final Answer: The provided context does not include specific information about Noveum's pricing plans. However, it mentions that all users will have access to the Observability suite for free, and early users will ...", + "final_answer_mode": "RAG", + "query_processed.input_query": "What are Noveum's pricing plans?", + "query_processed.output_response": "Successfully processed query using RAG, generated 399 character response", + "query_processed.mode": "RAG", + "query_processed.processing_time": 3.5059969425201416, + "query_processed.response_length": 399 + }, + "events": [], + "links": [], + "trace_trace_id": "fc64e6cc-6739-4256-ac4a-7b80c3028233", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-09 08:25:07.932000000", + "trace_end_time": "2025-10-09 08:25:11.438000000", + "trace_duration_ms": 3506.0989999999997, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-09 08:25:12.651000000", + "updated_at": "2025-10-09 08:25:12.651000000" + }, + { + "span_id": "0c7bbe2b-18af-4fdb-a5bf-4a91daa3bf50", + "trace_id": "b7945c49-f584-4c70-972d-536a805d8a31", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-09 08:24:58.518000000", + "end_time": "2025-10-09 08:25:06.920000000", + "duration_ms": 8402.378, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "How do I integrate Noveum Trace in my application?", + "query.length": 50, + "processing.mode": "RAG", + "processing.time_seconds": 8.402259826660156, + "processing.response_length": 1151, + "processing.sources_count": 5, + "output_response": "Final Answer: To integrate Noveum Trace into your application, you can use either the Python SDK (`noveum-trace`) or the TypeScript SDK (`@noveum/trace`). Hereā€™s a quick start guide based on the provided documentat...", + "final_answer_mode": "RAG", + "query_processed.input_query": "How do I integrate Noveum Trace in my application?", + "query_processed.output_response": "Successfully processed query using RAG, generated 1151 character response", + "query_processed.mode": "RAG", + "query_processed.processing_time": 8.402259826660156, + "query_processed.response_length": 1151 + }, + "events": [], + "links": [], + "trace_trace_id": "b7945c49-f584-4c70-972d-536a805d8a31", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-09 08:24:58.518000000", + "trace_end_time": "2025-10-09 08:25:06.920000000", + "trace_duration_ms": 8402.516, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-09 08:25:07.620000000", + "updated_at": "2025-10-09 08:25:07.620000000" + }, + { + "span_id": "dc869742-6797-4016-8061-37530fc7d433", + "trace_id": "f5c40ecf-36c0-45ba-9cc9-dc0329b0324b", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-09 08:24:54.009000000", + "end_time": "2025-10-09 08:24:57.511000000", + "duration_ms": 3502.0229999999997, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What is Noveum and what does it do?", + "query.length": 35, + "processing.mode": "RAG", + "processing.time_seconds": 3.5019419193267822, + "processing.response_length": 611, + "processing.sources_count": 5, + "output_response": "Final Answer: Noveum.ai is a comprehensive tracing and observability platform specifically designed for AI applications, including those powered by large language models (LLMs), retrieval-augmented generation (RAG)...", + "final_answer_mode": "RAG", + "query_processed.input_query": "What is Noveum and what does it do?", + "query_processed.output_response": "Successfully processed query using RAG, generated 611 character response", + "query_processed.mode": "RAG", + "query_processed.processing_time": 3.5019419193267822, + "query_processed.response_length": 611 + }, + "events": [], + "links": [], + "trace_trace_id": "f5c40ecf-36c0-45ba-9cc9-dc0329b0324b", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-09 08:24:54.009000000", + "trace_end_time": "2025-10-09 08:24:57.511000000", + "trace_duration_ms": 3502.062, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-09 08:24:59.716000000", + "updated_at": "2025-10-09 08:24:59.716000000" + } +] \ No newline at end of file diff --git a/noveum_customer_support_bt/support_agent.png b/noveum_customer_support_bt/support_agent.png new file mode 100644 index 0000000000000000000000000000000000000000..dec6399f20350e7a9c3d8b2a07a6539f475d55d4 GIT binary patch literal 418313 zcmeFZcT`jD_6CUJCpN%O6a^%RB8XIhfIt8h1Oe$t4?UqIfYL)15m2ep4NZ*nPUs~l zD4~VWAygFzB@`j_Iw#+Ee`RLYy7$gnv)0TXH)|z1WtC@wRw$;?=L-2J0tXREoKzLT^aR|N9>CvVnFR2rskR{T0DvwCV7U7B7JXh&nDm_f) zjX&?oEq{h?k|(BPo$iJjw~IVjEB{*o_t5)?4BW-Kzb;&H*8L@M1=nZCaV!Gse5GF61hP^`7K=t1TEb!fc%}KHmQ5&DJfVZ5@E8IdsCpa{$(T}g{8dx9zAvPY0?Yn!vi;5A(0{LNb0dCL?tWQ!S0>+ z;a;-ML%7+^Z~AZO*T0|r^CIHGb#7u5Bc)oEzimeO!bscYdmToCHxl3Sd_B7BUnl`( zz1;88ByK2wQP%UOURU@$-jFIb{YbyIIsMFz|51xksq_2SED9fQ#YUg!dh__sxik9T zqv9UN?igK(rl?X6Z1inj$U121H_5GtbeuIEC6n)WNi_B%G~DR zi5-@kf5tFJQCBHfg8bS7Ih*fFuieP0lX3!~n@ELmP-{!qJJZu{`PC$Yp zcDO)mR#(1UbG3Z7b?h7P^052)4VFpbozrmJUIxLpLEq@C&)j=sAI|YZzUC^3sU)CO zW*Rweb0cbX>vhg+$>v+ljE`i0cJsW*M7~L~c_=%;yhzjSIWl&ni7ny7<-RQPbok$17zQ^{E>A5v!jCyjT$KJf6l3|L*Bz5 z0!2&C)Q|~yE zg|dkm$Xs!Ef?r^H$J359$5fd#T<+ewQvZ5<%6nO387OaeTbIp?A6FH>{Zc&@b<3z* zv%8q?PLo3Z@L0+`ZuMLVMSL3Te*H@xIJb1L)3rEftzqJR2ioE@&c@bnMyuMq81 zYF}JSPsayqSPJBxtp6II&5Zt{`kdA5q~RCF=U0orl~){@+fO zDJ}hJ_d0UtM$nV{mx6h?^PZf(`S5bvQ(Jl6SA|L)?}eU#%bBGkQk8^nIJI%kaQD4! zP?pcUmLIkFQ91XD%q1A+y2OP@MZS-0^Oq~=pf80VclM~cUo{C!R}#sT>haw8V>I~0 zQ$`!>>}2}Z)*2(8#E2{#F{GqHM1pmKkkr78GS6HS*Nym z!~aIXdyTg4nH#QR#*c(~l48ALqu-arX1rI>li*Lhtfj{D_{j%NV`Y-E!)LQ>cWrsS z%G@D+7su=1aG7Wm<)WJE(cRD z1kaCE+P*b40VEDoj5h& zP{Z@I%cTV^lN82bYCp@u70PmNede}m zMiy6YMp4tFCi7$531>^Yi%z10mYA<>!NosiS{g&ohhAl#QaF%O9O&s5ZG9g4)1Za5 z1$4^>V|gpDR|cb}3~haCqcANvJ=ltmkkz=xE6-~kH^__EII>(Qzh8LWfQ=$iFgDO5 z@lD^V?t8`H2KGaxaf5Lt354Az>$_E<>UJHeaH{ipmhduGaUn;+0PJU@av^b{VUu*M zp9{<>!M?I5e{!+5G965;`%>4sm(ECN9frTt$Rc={ei9m!9PcJ@cYa%Ded z_ZsEir+Z_qkS$3n+1F*wdE09D^B!``VH>)0~g1-3zT#mM6ZR_tWHol+QBBcbcd8 zDUwUdb-eIJQtXA;Z>o&#pJ!grEVs)|)8k9nrr!TT>&Cmfz1n>3z#{xqA~{)nZJuY3 z=jbzi-s^#H18I2{JMRzF0<_pgocv9yUkVrrcyn*C6tjB1Yj`a1nAq3YZ{M~Jo z(EDK_#TZSxO@7z;y`=n6xg@NS7}YiTej+6<8OxWHnu7K<`Xtnzn3|4+phjmT^BOCj zjLLs^>p=5Ql#7x)rEg~3aB1rgQrHtZ$MIe3UD&St?v-7xY~H7Gq8bRatx2bQ{X%$- zg20)(JFwI8ilP`)_Cgm*oU8j+*QglBhcwB0RMc+jA8s<_)5IdQN?MZK>hPnS=7bQJ zIlgWeGs_Cgl~;I$KaTbFKI`xZ)K&H%er7-3xw-af=sV_WiRnR+k&$27x4h34Go6>d z#)d0I@6|5NO=c~qnTIko{xRhTvSUC8-V5;~-1^H2) z`a*W)K;^s2W!O&`BSmmX$YMJSf;~_U3H+YE8A`cY64>;x=I5evQ)FdIgk%D8ufn)t zcPsISkVk?+LYC=UQ);#RL}KakxX6-4el-VDgmmN)W~<`+?CIcwA|#sm(vbm%=xb#)StfYpSmA4Q>uA7C9Gr zBu$7&nXT3IjYwuMk35?y;4AsVOx8Ee>##5K4TqD|w96ze!L{J9>h~5^R@%$_F!6Y7 zuHJ3+&y_1v!wnXDUw%KFU9DcotwcyXNvHF@6*T+g!ksyEu-XU9v- z&yZDcS-h~=u#VqRkTHK^;kR;kPme;}6mc>K9dxdiB(A)e;4HB%@v#s=VUFQ=)1+#G zoQ@px@bnyboaGWkTYy!(J=;tT!PXNX>n8V#?_w4V^UAz~8?1=i~E*E$#P3DwKoo zpT6X<3ZRR3Fe6pCh6deTVEYW+sT1sUe*jx2fVcb!j{n|PK5>VR{;%Ip($R%E(4G26 z8%^Mw_KE@Cv_5}-)5nI=F#vyE0^Z&qPySbHhMte~|FwPU7qE}+p`N0uD)6mm;|ha0 zyV<+Amkupn1AaLB{E4v}9Ubd!+WUm6F3)dZ{1FFzBX=W>r!qD!PQq5UF4iz%Z>Q(9 zap)l4GQg%2%-!mSw-eIYO~za9=3gyjfNk2(A~$dR)x;ekchg8i`-Y;6E9{1(u&A); zP5E;-Zrp&l+QMaYm6ZR{9r#P`roFrSa~Tm4FE1})FL7ZPS38k=($dl*qGBRqVnRR* zAvct>yOp<)v)iq|2l=mYlwfW)t`5)L9bBAm(8jg0cJXkRyLpo~(SN`GwiD*<@INy- zyZz&^fCChv?Gd>rEGqKfV*_0wv|nYk9lT*kVF6HNsVY6x z_dc;S4l7N2JhZiT1vpDyUblF-&z}ABRek-16(;du#ow>J&*=PGSy}OnW%1hnQg6UO zK8)4UD$xUn?XKqMB>&3mAEyj&Pu8y%7_MF>QJ)B}>z9L&q5J;vo_B5?Bm;3GBO`F5H{`%HFf6t|9S=nK- zP&F4dn-2zhbbU}%j=FXuW3lmYtgVbTkH3#U$I8Qk-I22CUYkijJv)bq{KvVd-FJh% z)3DCOXRVQJf*F3JP~`(jULOd@N4w|}SbwX~x=pma>&AT9{Jr`K@8Al=m>O>7C{EB6 zC+5@e^n+M?yy~r)I4xYF)h%TezzGLChgZv;d|w^x=y6HA&e(4xLuD*PIAts1N}6_c z5{03E%9-zNEWlbTdS!?fojq2VPk8JJ+&)=P_33yKb=;JRQ|6K!9M9x5+gi-&ksf#< zBeIM>TO&o8+=aeKM*Z%YvJN{RICV&ghD}ZSuLpNizXpeW!Xu}OE607*4R;qQT(-Tx zV7=2>Pe16g3MJp^La6*T#G}J8K+;C#)xQQ=3E7 zX{IC!rE?!2%?PkzmXIckTBK+4 z+zI-ynuzV*Fqe++;Yl-LTt4k9a=sWma;V}xWNn8&lPv9amA0w`2YvU$MQ{lL^r!11 zw!iw8baDkAWKze~J(ovWocEU>jFh}-3ohCY^+I1b+J++W4< zW(2OKJx!4p@T~d0-Jcy1ZxO)8O5WR^3XBF(L*_DK4|dnvs(oBsChFvT8afgL+M48j zOc6&0*-z8hfQvyC_3JsCjC=ghxRx-q?z$XVA>H5X+*RQ_Q~twxk}f1xIrETp*sLCO zZUmy0E+$8F)4>S0nCnc9DDkR@c?5{MkOO*u>3ev4_0(pJMoP7Ev|$y}VOMP!D@kPG z(+CF5I*CV^Bt&u4hyuh=3C2xcMr__S8EIG6j z7qRDL`J1xq7AWntVz!YS4e=|{&o|=kqV6%Ke8LNu7v^s4taZkaA~Qs9(=6I44G{C+ zLlTzu+DTYD4Bl`4YkiiiWh9ap0N`oZ{52H;9Z;DKX~AAh6zvvA_;+DZ-`Tg<2pd6=;qQQc#VXVKb+(^P%nAz67AXnXV}1|I>2&wzqRimx1z>eVzV_k)Vv5w21Vu^ z{!=sE3E~aMf+?KRj%L8}c`!9O)p?f?DU>1?aGcST^DI=@81hODx7 z!W-A;&E@Jl_cl`=NmAacObrxa&4$V77=o}x+1*i5^~5f*7YC&=dQ4nD<5b4Bzpz3@ zLXbn;(JP}_7_%u{WH%OPm-v0ImrWcVlNYBFdLdxsc?NQ8e^ow?^XyxzbY`w7B3yQ& z$6UtCk*l0^n+fG!=e}S&6j8>V!YPKd-SD@e_rs8uK!oSWbY@3$w*AHDhaRqa)tYFM zF^DsRU~(iHxbiKAzy$ggrxJKL+cfhMUY@8jreI^?<9Ce_QbnM3m*wDDYO}Xp(jl4fXvT)8Y~rDe4S)D3-j;do zV(`)F46mbi?L)+tcTb^FX6g(v!w{)Ah=HGHz919lj>apjVjD)E*?LJYSO=K-uq}9 zekB^+pQRy?CIb~Y@2-cxWm}0ht*+zsOD^e0ZROS?Kj`XK*uU4t;;nCJpe6b$)fpz* zOg?4Eu5lL(jxUaqtyoBo@_|PUhXuKY3w^g0pW8C>g)!kpPV44q-kxw?x-7X0c?AbuSF0-OC_?5&cLXCTSz}6g9dLMev7II z!9n=7Xx>vPDYDL#?39XdMV3wrw<}bwu``ul?3Yanq~!$#vNJ5MY{>e2=%xk*ubqb$ zENe3IlNVbzs+`W=%Q~8KNs)B8M)q2++qAlMsBj#8Jbiq}3J_&7ER5s?gWzU`qtq&c+hDW0MxPMo_oMaDT zMX*eR!pQehIL@0m9hSc6hbN(zsl1Az8&-{3GB8{Wp#C4X#ic%;j* z-(LD128NWFRT`wMOiKY_LR+!McBCSq-U}}k{lVCdF{#LWYzep2b6vO5B%$Y&ZP-bO zq-VKZC+~2D{mt_p>5Y!KOq=?iZ+|-Cu1{UpEpoiwo+wveI1rgLjh|*_H)45KkVvQ` zX9;V(k|YlOXnZz2nn;3r=GHnUyo_y04IT=gW3lu56JkYd?h?nv)8_m8RrcD+~+^++ZFPFdm#%Q@g^XLg$)F zmtkIs3}1=h(Z-3DI)id`1!ke(?KVu2bSIj_z#im3V}B5IX@3&SSoBBT>W>6Mm6My( zPO(MPSJ#dmc**)~#~V)3iVKssQwB9!HKi*0kp^yRIgnAw@U2yaJHX7dy^r77gBtxR zughO*(m{lo*SYs04aB`mkX4(NLd~Izb(>V)&B6&7aE^N?CPnJ~w9mCcf#@puGsI+toofklYfAx3T4o|lY(ZfZ zuTF8n9QSriB?$;l#SEt&R3G0E`4@2MFEDkKbgJ0rS~C8p|Cd)tP=WKuYWqo}cUJ}O#^rRyKyB4FT3fQoB;QsyR01Mb$U#qI zx%%qY1#x)i2k|xVaEXO!zJBpF$H}r=_x9MuZQE{$$REc?a7w8r3PM3Z1pfO6y;tXC zvLZDvnFP13x2)q7_Z3y@IZs*&)AWl%~MkV5cJW?I0xoZcyZIGEB~2*>a_A^ z3&e#4zz*P49d4U5g8|q#0&5s^q3~{Tf;~nBr2`;1__2?&-~+3@)LlbXk{7{Bu{T zM%wK9Y^POvG)RtB4!*k%%kn5-6|?%KPpWWG2hRJ;i7^)5r|$t|SQFs|gsCLqfx4u%G`)! z)wb1ALCfZ_D!J=JbEDgjADJURKil^<_bs<-W|Au@lC(wFu-Bnrdvx$!OmS`$L#B*X zzegHlLs63@l$8;~pUojNLn-xb-iN7Btf&9d^Pj600FpGWus6=#{R<~zp<@QI;&x0Q zJ{fo84jBM|WmNidLy90g;eOdGOOc(aP7Wa$E`bI_h&vvG$EEzi; zylyXdP0%77dvCokZuEiHREwB5T=RX~w~mJE);xb$gsNL2u~@jRjMvV5-CvD;VL6maaN+wa2AUd!)@7~qO#E!Qs~O$LKRI(@2bN_3Q@#H*OXoxvieD{%z*qj`F3={N$Fgk3|q1N6b?N44dK<)m*rNJ({a*Ls(D zbSOGWnA#5K^52msM8)&B28nHg^re#k^BS+@AvgCV*W~ zi(GtGNp>io;Et}clQ88PgZRR*+Y`QFBjQsP3s|%Dd6UX~bxy)(p{DhAj9w>4N#jK_ zNhGCFL$IX`Ew?_;4$gl!F=c>V9{i4@fe4qTUDsTaKuV+Oa5-Kp^#VAfC$JTV%*fJd z@yR3?F_Dn(F7waG5nNm6%{yO($di^htEWD%PKyKZ3ppg;v?RNi&oK|UIPuftbM3Z_ za0u&kRnXBJ=W`~{DVsg8^NEZcl8EOLMA-YG3ROjZordHg$SptPZRVOc-EQ`hkp_;XvfslJ7N69~DZ%D|#_} zHFEBxSKmQVa6&qzPr@=K|5?8N1@PD_sVx}CyvV{$g5#FYeEA=pxW;{0_RvzVgm%3F zBlPpL_iy)lD*LVOIdjZcJ;9iqy{dM#$bqf-D^v!W~dDr*z&5Pa*6d0=WE-FWWuYSiN=_gnnFv`(hzVmf@R|WCz zETl)haWoHDmJwG!3gOWq4&UF+ejCd01UMja|DClUXWlW0h{;m_MK`_bZuU^~ zGY?C_t>17RNg^!GmSYdLz3Iv*yOa~QKW}D)COb<_-bfOLGJ$<17~_TX)J7|u`KS3b zdw=xiNY8LiJgp)KmM{>Cp)zmg&9lG@AZhj~3GF(4Rfpj%lg8G((Tmgl*VhxBc}?jH z4Oz73G?#A!Zr=MSZ3pTu&v<`6sdv)j21a;afYMGT;qY1JF-B7exbRgQU#;* zV&cK-K$-uD-{}0nMJm9VwO?ptE`pa3W+K?cU!%6{<4YMgfcZm}txH0$A&k=Ge1p2u zs=s8Bx0X#1{SU~ZLbdDtEXw{dCPf z{*IAo)3{G(WF~o2lfYf|0U4AFt}(VTSt`(=Ab1SBQ^meJdfLagm2mV|rB{Z&C<#rT za~CHuu^9!Ybe`U;B-}1>_R;F?>V9=Rk`(DdgjR>58`-VX{mJ)?Ga2j);Ten@*`pFM z3SC$g%xkBQQzgV1NuoL9JuQXJC-&^6r=_CNrV=if0Bh6b09a9s0%$BnBfyQ^t&)yK z4qqIrJY1IwjQf>!3yt3Q?9BAuvErCd6o~3YnJ*7qv*Atp+g;A2pZJJ|8J+{V(yf27!-Lgj9?PnhX*}R^3BP zw2SjW5qdBSuHnRiyGccW^hTJ*8&G|-Ri2nXykk9 zoN>8rqWfN^Z5kU0YIYMALJG&oLHL{NYJ}Qmzma`X4mra5ynTJO9LTJQ zrO-0y;HP-kbkfB__<8;%|2YHfP|CeHHv;>mP?YD+X4vOk(kqwxrbaTUxa@i6D=VGZ zwbF}@02PU>NrQOkD)#Qg=^J1VN5h;te8jw)d?*;cLa>ZcE zK4~Z&6lokdcz|CGd~MP5yQaFIQrG`T&R4q6i&dju)~t9TX}bB!(^ zE|q;q5QdJ5W)Cj)&LfBIIg8bvl=lelVPxdDsw!$BNpe=}FRSy6o7DAkG&$3~1U@1x+`+ zXZz7YBBd5LsL6Jn@a6sQ@+}atdY(@B=H;qcLv!$P^<3|&3*s(Yp8~@?*WL*tLI|X@2>@^@=6E zTy6LD;mCiTm;Q10%jr4+zTkf&hy5ROX}-p`~qm?ChO<%oZrXN@tdb$#u-|LtuYkhxWjQ+6EA2dyk0?laS{p?-mh zbi*n#UBuiyxAs9mdz!2v5T~Mno0O0TWgy;Z?2>JU9^qMmL2Q4HW*7?Ry=Glu1JLF? zTIu8Ex43npD2FWjY6*7}1`13Cx~Mq+eH$Q$Y>}n*SlW$TxHRrSy>8(KJlAepOf~@M zn=+@gBmnD(@B0{#CQ9^8b1XKieU&QfE9~cNQFGD44d;jA2|V(PjVaYLLktm18-4xB zwYpj7T#pWRfrO&4*_$-h_7th=%YxcBx~OF~cnn_s{ml>wMo8X(6Vs}RWX+<==yL5qg5so1X*;L?RD3Ehm$mD((6?lMKpKY0vau^3@2a?LrRsoCR zh=U_M*Is~TyxZKbIcdA@>$(j_IwWW6l0J4~OmcE%LLQtXU@9|0f@H{g36I_jE0$G{ zzr#x2ZF`TQS-ZssCkobkuD)O^0rEkykDBP6(us>+21^3ITS#4iBS=e1`}NGNCDa2p z7X@_&C-tKB90;-ygz&RRmo}v5OZy>G_S#Yq`sLACablKw(q+^B(MGqeCDMpPA`rf+ zv2pg3cYpu9MVqXO3G2h+wd~1O65p+A;_q38ssh_D2>`T~u7Sd>CDd>Md2iD(+#hZ_ z5ja2Rv{djJx4l&B7lyOwPL*bnO|@;0HD0R!Lf-AkZ28Oh2rI<0lK9?1qU6Y28Qw3!uzrKg7wMMWt?~^cOK;fWv8%i`GFr<-d zHJ91ZA)Mz{=cWRW<*ID?4GQ1b0^C`~*-Ntk&IO3Hu4J7hfWq?otxZdfE}MV{v=CLa z6k6_3>TSK`IXyJzH=GzipFP|BFzzV-L*EUE8e8Tv{hfVh;va`!X5BBW?CQplH}*){ zMJBK!Y3}jpve2z|m5L_Y`n(TX#bqoLygw8F>@EUSL{yL@%9D`h1UNW)2=IfnDiA5a zg@~nxw}Fn)aO|!k0Uxyw7#HC}1w!0WldT3A7Cq}daJ}4k!Vn8?4@5|smRU;^Fd{PK z-rCQlD;ee<^oM{lWb_D!89bfAg}dG8!yH3$Bh!Q^etYfv<$k2Q zY?9U$>hry|{hPXvxvrf)5_?*OoM82jtOcJr+L_+|?6n#s<~B4=%cn;lZ$=fGR0z=>xa{*&W6LG&e&WFYsIi=S7S@FvGNZU_LEAd?Fp12mk6A)@BHE+wYrZwcO(W| zjG!H+ltth}WQ%KVHx{2%G&q+3%#BUd^2tu>o5m-;bPL0C47Bg>qwC zj#r$;O~L~WJ_MO0#PxNA%K{0T@ibXt5Y}+j!uu}7j2X0-(#iWsapKP6Kwjg~ALChmW+K{Rrp%gQV-My%ZTJ?Xei$0px<+<_nYcy`edZz2 z910{@gIz{|zOa2u+~Crm6+=U!hgGI^YEwr|9q+CbK#O}oo)3kp6l{H@_He4=}%oF zn;AgQ1#&d6LuLVUFCE8S=$ot1vEc<|F!Q&zRb`xBA9}Q%Njfcd9tQIJA-piQYG_>U zUfT)YTHOyG+3sl5nFeOx`RdDqagIHd@qWGZ!oIvLY*U2`XMq6GG+=JEt@4eygsA`y zfhGq6h@*gcJTS-kn=K=F)zkhye-1~xmfTv*1|ZfEG+EN466NZ`w%1*5;p>n!6JD#! zqoV1z29|o?SZZ7%%2NSe>Iti`$uBXY+ym>~v$eQXo zg2i97u}gj6C$@GowzaWqNiUD8OkLuw>oSoa(*F2RCNfr0VOKfnAMn1G%G%0w^=Q&H zmZnuf3w??7p8(WnPEQ25n+{|Ikv{k+yX`Jnm!?ZY8g69aH=M!;he;L(daQy*f-Nc$ zFC6lhJ*OEnb=t`!Ced#=WN^zmZ|71&Iff2oJQ1Zb7el_gE zq(Jmh5_>TeD?ifHweX{nA(L zRL~<~2X=99!vO5w%_;Ze5peX;kH+B^IeY0W6_7tPC9zLCB~2)0;zx{%Y4BL|RN$Yk zY9)|8=pm>SUXh6=p^)pb_Kbj)waRWR3@eY?Fm=IQMn4O4Ab4tv zZw;=avuWfqHwK!McemTym^|mKYSa)!LW&kbx<& z5+S8z@d>OpB94Roj^hRHwk5cAW~TtTOMsL?IY9Pf9}bErLcJ+`(FR|BF#O}8LReR~ zKy<+Fcf00*VAyXE|~Lg~Hb7i*q|CUjof zE+@}JARvk5&%+7bni))!O`0EFhQ3D|5VsD+7vodA|)P=#__(v0V-@ zo=HwrI{QdrxLFChXqw1+*VCs91D6eQPD}HYnTc;Hu|M!Q=}#p)WroY|rRXq}vdmZA zsHE$kPCC70H3DJ+`q*Ud8{#wMx~{t~wpHsf=4hpz1-M6&17FYbKx10fK}nHu{dFzM zTRvIUSsg6+^OYXG?X7TOK2Dj+W0@d8EnzTmabtJ(BE_{rFQwko)XwfegmJ{*DZ@sR z2x6AlI;zw_)mJwLT*>*A-X)X6M|4V;laXCMp*>AN8o@y1)tBZef?rhFQNC5Qfqo-r;Sb&O5uGe*&_`UmMv*5nxNx3ZfiZT9E z(H4fKcKI|#0NIT;Yn@yUD(w-?t=W}&XF%XEmiax(AEwp3HK%kWiJm7Xiiu=^$NCa* z_%$86r2if-qBe4N!xh#EaGiUz7ZEqq2R4Ig;mqyLQ$Fne!cr>03}^UyHf5*e?SLa- zncpfzL{@^pNj<9x?hD1Kb=$e5lY8&3bWj=_q1ZDS7V2dz^S%-_m8h-zkx(CCtHNXN_dn5YkzA z$kx=AVMD+vgQulRkdplC_P_#MW!YcaO16^?N{4Dqtd@3l|12G_;OmjYk`{eVP6X@@ zkx935Dp=-+2Idk7*}e2PwxlV>)C9<$QX4-I`LoP>e(4cNsRXALaWAnQH-V$8EZVy; zD9tvw9M#^xb-M$;Tu%&mF*->ENyCo(P@BVI)B`OTc1wVMS0Zc*HzBsOW6@iR9AfGl zF%QREF`9}TBoItk%J7i6L4Y{vKUOCd0#IXdd+?doN4#*yj_<#t&WKF`gzD{1tu^NV zx^)oGp8$iQYQ%Q=U+Ls8KDbePwlpYx93DY={4Z7Y7oS|BF)ptY9{w8wk^dO{4?V+w zJp5k^>;K`Bs98L{cep*l4bC@AaA{iQrwlno95G)NG~3>aZ_6Hw!KCK;aI+SidkUYBJP_q3*=!(*I6 zTkP%5O7fBG`as_Nz15mSUWb9wbiaM5mT@B-y5S9w`LPy(3s=afe^d$p3jV)h7^>gj zlK;l60tuHPBiwFPSfNRyq)$Vtbb04*tHT*8u_@s_2LG2He;iX^IOyB6vTTFbQi?_a zssUH0PBI`W9&ptFGaN*(s2=gR5-G6qTRxAK0&2>hr zY9+g<>sJas;XC8`F;A~a=>r1nf7A;60N_j9@|ahfmI?L!EfoxY$z9QRVdQ6>Hnk5X z0I(kbv-C6#qAS0@!~x(jciQo9mnbAc-`2N4xMcize7cy1E%>1I?lCJ<2R!PD#cx0& zPNNcHZhJ<%`UsE=py35PZ3sk#Tgzi5)Nw5)5Cl-p)>jqHkW=bjqp#9bCFULR{IJPX zKoHEpd?-Xa5~`NM#gwTGpN#~11|~rFlD!GGo-&jqp1&#-2|zfb3d?-IJqNE{TG+lO zzq?)Uda$EfR2#iC7I?$w>3$b(ZpBoQaGq6|2KbE1pQHWOjlKRZ3^>~DH3hOO#=AO* z$$GEAjbuQI`ksNSMjbbVN>>Qfq1oH_UA!5>d|hGy*1$X&2o*N3x$HQJwR71>)<}_v z$AcOzfKmz!kmwTyjQz5*EYh@Re)_`oUt*KKs||Mp57xqHRTQL3U#*SgsujP*_Ao^P zEkXv2NmlTR^(&y%{aqEXl_H`FK&aWU3?KDR0e0VUfY?QdcszafuWE>`)LmNHL54V@ z;b6yxYhf%frqde`*EIa4^*=64S{knv@adANM6`8^$YH?8cf{=oH2;*Y&Y3svsjuRi z0EpAkNmLsWGzKaaeANLdQLoKVq0Te&z~vB8;y(~^vLq~igK}K3u_7+oz-+_dzZQ}M zWE`){5F1mStfz_R$gTaUV#R_gDYss(0!~;4NTt8~E0fpG`hIPir5*7AE z2OtjX8#H*V)5IA6Vwx90pr1JiIp1qDO4EI;7eww|=iI+*TwddcAv#*v2Y_GnRnXv4 zbWf`7`V8B;+v<_)%J}a5JuaZ!<8O6UODh3T?F$$v1soi?5lV1z|ELVOz&UOT?@GQU zx4FN#L~;WT`}r}{I-^pONEyQ0VMESey?}}%U=1Z@=abUKyjGfH^wOnSG6<8t#AV%j zgUuMdVrVt@Xz7C>zrGUUwpBVlD&;buSP^}IoqW7!=1BEh99RJED0NiUyb{XePB@^G zXh2b4cO9H3{7{jd?e+H!cQBWHGhg&k)jnUjjf!;}oLP3J7s#v&B&vEHXNL(wSphCf zJxMcLi}Ol3$*M)=`=!vP)uX{C*!D^zP#3@{b6pO;zg3iFI?VC8o_w*&@alf~fmL_c zi}k#f$_s3ZfZBg;SH)FZoD@Tlndu#_bl|U^@Udy*zbX_(#OQr~!hN2sF@#zX4h{n{ z)foZnxRL!GK0?VBU`LT@WsK}<&Ew*_AoEJK$L$FMhC9z?eFTqMZUKsa8c|}+{oLBx z?~aXBmQyOve2Rag@J*WH-#0^TR_)3+RyH8Hx)$WX;k*@8Yx(sOX+fX5DKps(t*<0& z{95~HoM(+{&rm24w7{7}U5}jlg8fkx?EHgJjrQVICcb5~OnJO@!j9_q_2^ zx!OUczo{b?Pc!5j-4{nR$s%Ugaek0@&b||sR^rW|7~%U6vm_BiWuP+V^2%}xQ%WMh zAtnf?$+SV7EJ;5v-bWj!3<QBT-1``#V-3FV7DG=cvZ<`rYd`Eq!w7 z(l14W0+Re|z`HlMw7rk~{61=DyQ}i+eDcmE+1!hf@wi)?XKMFBDzI#*p+#7FZ%|F@ z4`zxTjlX=OTc>}PKkL|zkscUREkW10wzgoXq3yPjWNG5CM89(K^yp<9Fh1-KtFy5Y z9H?Pv8w2z$=5qv7p0O%EC1M$)C36VhM&>Ac5aiXZ^Gcmb0s`vp+$%a*=yaOb_;pFa zYU$StlIdW$v3SNGAUk4uW#E;~@&_flA?pDC;XPHKT<;EK$}3E(ouccinvQA!C4yDJ za3Pe>&i$6$*0YKyR(WQ%p9GMgj$0wQNFHq3W;4}UFkCo{o~CcUxi?|< zOi>3xH@e#6Z3P}{l(SH%u0}>qNtCUzOXYJ7k%sISdP^MK%|U&)3UVyYi}6k14XfO4YQipV=dAz;Ip~h#tCn!omKtcl=fM#N&`L*WE(z^rZQ5n03APpZ5r>uDGXy5H*!gm>086c%6QYlzw%La$(9dC@LTE z7=`#Emy$E&;zW#U^bt`m$yl^Y57M48bgIbMO)U&vV35Zny(woOrUr4WBJiF6Qeumh z2YU@5J?r5-6d==5PP)4XvCOD6z-mdRsE#6v?@!%}#^L-^dY;#q&+Y#Dz5^LWW7lT^ z;KU!@oqP}D-h-$}?Ze#bTXE{FUdDcPRRh=>wAHzJc9P>(8(iG>NeP46sKeJD~DExjN9I>&8j7H*GRax1#Y` zPOGpn^#(xnv~l6yzs^ttMb7O{UovjkN?Mr-{~OuwzB2$GlqimP_pgVJ={hy?Knu>& zI(zeS1@Ny<D8JRJ#@qFW{R5RuL|*PRv)!U8H+-KwYjA3lNk zRz^AH%o7Da%(mmXV%KDdok=C{fS}%cC)3v!AN5U?Xh&=v9{(7?nWv3I=A!I=70)BO z;`!qs)PY-KK=K8sHGt7jOfd8K?RwAULG&E0vKdh78&)Uk10_~3=Kg+x92n2FlfW%1#wYJL#v`^d{ z@LQFuZU-v36$v?-8BvW&dCv;!1^{HitMvi+-wr@cu{sc%HmvtyO{liU4YnwnMH5~V zcU73U-Y;_k;P*5>Ia~w(%69#;RV=ujyt#<#uUY)6GTw{FFsYWqFMU#0~kVA<~8T zG`|g-0a^h7o#URSL!ed8V=Jzat2pXApdQ$mRnQCoY);ja*W^oz$kQ~C?T|sD0X$U` zYGTnCt!8@CcWzLWNWCiP$+nsZ$k;=uS8dTl$ZhYX0_zahzE3EYB9w{I)b}X6z~dv| zk`Pq7RN#A{Ab#it;G97?ld8-x(C+4 zQSW(qSFsra6y8htJT>*Ia+nb7qPA+7{$)b|H4Dh>JabFx=Hz*nA1~%Kij?-7dGdd- z_nuKrb?v$^O)NBp2b6B1C{YOtu6vfX?(4p;Ywn1|gA)B%xgGPo#SA`S z86|M6Km82k*nkag^i$9{i*ZbOtjwltUeEbaLzcP8p32ZPZsh@r5dwXOuhpy+N%0H0 zo!s^A_DR?hcem!I7yL53U{r9}JH03nc9oxN-g={9cSZDftrAc2FYMmRAH~i^%Irj3 zJFKqvWo%Cp)672n`AzOadMKbU=vKQg-KN=vc_eG1_#1Y@d3rbQc!Sz~6`=T=^~^zf z@Z^-js+ZHRa7MjigSGAx!T2r)HAN*lK+Ll}J=vd1Z>ud-Rc=odQp|>(of3mIj3>&( zVaV-03JWh^!ACB|fBm9f1r!LKLCV-?Ug0qGV0*Tmxira_Qe>egl|SFAV1CFBtpLO` zn^eAj##PR4*@r;M0!#xirBy(Q3tpW(rcM{Pw&zi|_7_3bS|?At2%k24&MzriMBqm2 ze7AnJED96>3xG?sT$e%W?_sjzS>>`2eY8<`YZe@SICwl5Llv??(0Pa7@NH2Q4BM(v z_GtPEU~O!T=KDeOS8D<}Dqc6&b%2VbiUB=28mNG{v~F+2qd@NZ zw84O70M6N|aiX^CDFxEkQbid%OI;KtUaP3ZrQ_OZIgq8!&#w!~6Tvh3m-Yh+uJCq> z`HtO)EHskK)hYTFkEbDMLw%vY4;c3~@H|ZvaCvWuS68BJZ^WcX!h)gdZ+k2UzC14I zx!)56uPfE-Opmlp5iqi=s0Ey07wCTFTr7rZ+q#o=ve(K7VFiNFE#ZJx*^~c~4q}6s z@HCOSopo8#m|bL7xP)?^td2(U8P3L_CjAUVeOXiEmj)JoI2B0kS)`imG5ZctxRTvn z3mkDEiVVsDy!V8)j{-CkbTjy}YyMQR9`QMhY^2E-y#d)!>Xrv67BWrmCzV07)_Jfv z^??B10G{W`A{a>T%vBpSV351EU+x8kTZSKSkub>uD)RJJmEI7!RCp1O3eAgnX`l$` z&$nqLS^>hL4QFszKI0-_%ERmEW+EkBi`Z5lk&7Jh^e;ckcsD1fyH;`+l?_P!l0bOH zAZ1)jH?wy+A)D&bUhiD_qdcHlpmDp^3TiHpUX#9ao!Naf;|%O9UpLfBiks+ojKj(p zKxOt@0ZE9_+<#)veXn?CH$Om(Q(;wJNxO(IFLTv zmC8q4(MGo2EC}Z9cO{-6lKLH)hNHc_ZN$*(;O#A)fdG-_q?dsEtpN9RH%MCmjBc`Q zs!4An04S=l%K9*{pZX63pjXQtAwhV(0BJsdo@&bf)?lr5iF$goedYLVE0IUV6t0n( zgn(bVn>7RtjBQ&&je{Y1Z67`kn`zh`X}<0%rN zJJ4uhk1y3&Gu+mxa0Pt$c^xQUg0MXM0+JXs*PKhCGi*A8Y7#=w%YEEsH|tJQqT)S%i&UR(s#7HH~0ML^C)Z> zAvj^B+Dy|#sTF;fw~8noIH_O={t%FdvsI1{0{Q@?aeI`s8sceyGLE1|yEYh7#puq&v;opTFXAzw z{bKR&Ll#M^?dc*|Bm|&4-aa}~K*Ru+wU|+ynESUJn??o?4g(Zk(GlnXiz%S3gUj&{ z8`e0OAFS4=t;~+-U@apuzOpRBzldNS8n1Rjde;czhgOG}w!c``F+v@tsx}sgopDa6bWz92Ju4Pw|F1bt={ODTRI>aJBk50Xh5+TY~A+a8MRrCONkBbMiG#>iLl@`u4O_T3#XZVy-s(V6C~;aUdVy_p2go z72h*#04GoE#{y;`Kp{3N-<8z?Y%PoLUvku7uC^r$*YidJZC?j#?M+uaf4PX*$ZdJB z-)cLD53uuzPu4_DC`I`7mObCD{SYXCRVcIApRZ;>7kRTI+^l0CeUey=p9>$99Lp1% zRT^k)-y~n_7f$n5-t%<>YY;Fh5Y0aKnT2cuC>z6ge?%_YFh}w;AN7|OKF3%iDK18a z*yV{OqkXz^=AY$g1ik-x%PYNlD2czwy)Jn`6!+No>wB4?=bWu;oox@EN5W3?=F|0& zZLLKp7wLjgH4LaOlU(v2@;N?mN_hYLD1bXkrePAJ*t8^9Ge440r_;Su#N{~zP5$Wq zy%m%isf2-6@-KOO!vD;$fsW z1}-LO*>Me=DuTJ1=yp(p&gYclc~s0V5D6EusoUeeyhlH;D4Of~Hi^MbM_f|M6=*adf zG*RwmZEBajh3_PO3c~J<&gr;#wP$k5zXw(0oO$m3y^>xxyJ4oI?ZQ!X@+i^{rO0eo z(R0_gwOGwOG9jqbefhMxoo4##d9yYEwk4yJjNUJai~%*Lg(*%zWaon9s@DS)P94Y( zAnvD{Sb>7kPfNpv0)zP*w2_+B5Mwp_&58;Sz+D*aZ!OzLw($oA!QE8N?DQg|+FFsB zd$Iahr!1IS0-ya?4{*;~$O5TIR+%LoWc|LzG8Qg%9yX4N|1h%fS>&&(p?t={3cq5v zn6^$x*lN{y(Y&y}3*RuDY}qAl$Zo?bl4qd^uD&RVVGsr)nr5K+0So1?%9WCZg&6&! zx>QhgrYIGLBS()$Z*^W2S-?lr$&5uw{m0wZlupl(OpzuM98 z?Ry-LS}yuBf*Y7QfmY>P#I@f<47tV3>6R4D#wng9SRq4Bwimy&TMC)@5=-rI{Mz_8 z>cf{GIS}bcfztE4pI-mY;cxM*eC?~35ujlpe*#Hz}5mV^S`c+ zl`Ha@)Ps@;Ft1}XKo~6id zM!E~Is+sLt5)$vJnFA;+tUH0fXtq5WF!w`gw(w;GrTnre@LP4KP^6mTZM;$v zd|=o|DYaE@DITGUDDcpV6?d~W&Vii;F0LR6xwLQNQIZ(>x=JbAZUkrLh@k-rDC*wU zw*p<;qY3fy2dJYY+rlTG@&Vj_e*snrY?5SpKpCFeTRjB6dvtv-?xw<}9G#*HbgkUp z8a8nOUu{$<7@9x)9z<4>aJy=^&ahb45AU7kbN(S!HnDKF38zmm5SUMy-@BtzsB%zS z0joF!0`iVHO{h_+OIu+vI+VHRwEq5U@#tdTtW~$%D(Gncc1@uIG^Ga!1AIe)LLusA z5LpmR+IfAA@U(Zw@Y2XE$r$1^v;jgBh5@k zkv6E7uwZXY-n^HaUC{nyij=oGKmW1sodU6;RFPD-Y%zn1e2r51l9&7}PVj|leWVoe zntN{5i}qj9;%>RWv85gtZC`gjty1}q3%3;CS9gUKG_BT8f=rI3h+3R;Z`EvnPtWLe zYdv^a^H}T%eNTBkZVOVwz4Zy!`B7;{05nW0XZ6hO(Yw#9_@Dsv6kTee1o&T?loBYU zl=(=t3n=nUOK?92nDvxS0`q*;OH-63(i)PtlBr240t_^=R6#tR$Taa6bS=9T$QMTGkaj0NP{oyu9&k;EF9c||lvQexO zjyaG0gD6O|8^yxg z$LiJq!@nM`!(wkaISPfLEbUEzl*rH)4Mo?d7gY!PnGZ9%ZW(F=j{dU>hY1#MsyCv2 z4X1|{cn*CRT#ElcC>>E2o!afa)9Hpp0es0t30!g9HYm9l*m(QCVe7!NEY^l1k^O{3 zE~sKBy|($>Q?AY3j7o%<)E0QIJOmv5DHFfbr`a%ExELPzIy+sE zCkyH-_3^f3NR@6uI9l&Du#xVn#J$M-PV6jOQ*kcxWIYzqbJ*u$B zC^6WkIbdr|@r3>^0HOdR%mu)@_Y-u3tD{(J_p6~3@z>=-B3utk?FTY5gzd*=aa7lM zRz#ui3P*{8Qe=FYZBqo@Eb#zm4Io3;TZaG|{k8=L7_9YwR)~3mt#)Gs3#CzYS%22B zhrWw{qNIfv{%AlE{&IiQ7}y~|!tj2R`jxgfPH<-o(wt`ef~xRA!L%v${&XvuL}8dd zp3^_&xfaxGE$F%E$cIos!6DJ^Yx+U(;e0(--w5<=frErpITQ_t>Tf7(H_rdfPwO91 z9~k)B&KE>54kx^49#c9yeQhzcT+TOq5BkXDFh|iO@vhUmPEY>2gK&sxYx*_)+m(h=Ldq-N@ z`G#wNflsl)IuBd@`j~I#WM;JO!L6>JHyWyhqlWs! zIcIhFe%C;xb$T=FcHSh0g=&>abbPR0tLY zj#7otJ)iX7VD8V_xUN?X4Fa5gh7q|`z2Nrjz8Rxd@72De7KW7bC*p>p{4Wuge6oNnm0&K zb#7iU2VfB*oJ#toj`g)lxPAJqJqyAXO?7D|TjQ_RHuzrgK0A&GXAyb|C{?W%FH zJxR=ES(1t@g1F?WO~kVvsiUueTJr<|&eVWB z?NO;^OoFke_@Jc!x4#MfJ55&wVcUSvuPE!kz$_Zl+GIjXLB`mKzwFhbC9et{pn-8b z{fwux_@7N!0UJstU-d}$i;Ip81qj`lM&NBKm(Ak+q2k6FFn|e$FLWDR#`AXo|NkbS zfegb~*7~1&b)t+Oq(=|^Ht@0I`4YJ3y;09l3U0#J_7vCYA6lm#OvTE(zy#;Dw>MNn zXv=|h?eQu%Fb>@AJMS1&{#gfd)MCZj9-YxC2e|c7Xi!P+9eFO|ILeTU8heNav>wqx zu=x4ml|5!H%iw-ov;jK3V)~^+r&KQl)M86`g(}JYT2+8Md?0qJ)h7Oj8#36%(GthF&HXdn;#w3ZB{QTWqvVrs4SIU>l?9TK z8Tc1W9-u=Q-cF_D*bPK^xpyRM!==NaK^uUr z`ckp6u>~jf!@-U^@lPRkXx)vqZ>0{tJ`jX4;-uRSD1*v!g7X3E9elyLzrMJohE?r0TD&p&YVYMVB}V+FCH=dTo8*hGXUji#FZ{KdKnf+SGg4>q zaz*MVq$m57!VAJCXDp^m-Jsy&V>ViLWx! z0b{<2{IMaMi7~)2`6`SUdDTKk0%+b1w@r;Y=~CF%$7msp=0JFuK$<_(lV@4OVsJ1j zm$$&Lr27bv@B8Mg8-$;HeR5+o^L}Dm5%+PHWwKSI>J!EuanCg!+WZ+b5X^T~l4{^$6bM$!hx=(EUA*WUC@Id(8D3pTqsT z07NlBX4tDhG4P+;#SS8(eZgV|j^0prj3?bQjAh0Ih9zr#ErAr+WU>I(J*=y+Pr9<8 zeRSh|U~u-}>U1V*_YP1hVl5%U%R1filI_ti)+eqN`;`bV2@>7soX#ShyZ*(I< zx&}xLldJhn3RhD_j=&Fq*RwkpkID%n;r2k$-?_$CqKI-|o~*~(r_}k`G^Vh7EIo$W zsHpIXS1cfmt;AT$h^dSxNYAD7kC!SyRmzGF%) zYLtodeS=4ftaWwmRh3vot_}cof(a0L%&>?c038Iw@SQuW3~D`f`uawLbB^9{=pM^X zhoz7;#6Z%eiht*iRLg_8?%wUeyD?%ys8}f|Ji&MTQ)3LkZK&KtzMz0{J7shgaQR14 zVXnvg7WxD4rN~C{(o5hZOIV@`HyJ&B+t%r)wO3?vwA+ymWE3J&u;Xp}=-EEp$~Sl0 zF|--D;>8KcDiMTRSGv9LY9eZkU+H*vrsAvi|NiJeHX4AFv|H0(`!~4%VtDROC)0mWr~h)Ur|t z6`V8qhM$0qaji*9nOJ>$90~{pAV2DSJ3c;8&9VmVyTq|8%Re@2F)7q!gUY zt^VuG-&c{p5H)j~An?l-*#Ie<@nYJ4IQfk_0pZT3-Y51yzmxrZ@w#q`^@|Y#5^4p4 z`@8bf56u6eHy!~7(&9ye@3wz_7uZZK+bdbp+{;3;dRg}F1f`-i;s38Q54dS$2~L1o zu$cLez~)oS{v790yG;zHoa}X}_Qm2Gf3^+uU86~gY%c*WMqTnjxvh2)c%B;5pmxui?0Q0Ht^6s{3_e_xc5AaLj`hf1( zaD`jL~-85G-T-Z?`lze?=euXrg)oJQH({_rp6e8GwQtj)B)EZv;Ks10I)rbR%*{q-w)WM9rBAF%D^|MC2H>5lUdM3OFfKV_k}Ve zED-*gG7lqM3e^Ad*BN%#3z>4H`N!% zk}Tnog4>~&KTpN%UVwxlF^fxC)$oP|;HFuP=DZZo1Mi+Wra8f39&zJ@W=oikBC42>k&yJjKwWSE}3||4Q**rCHJ;AwE!0xV1`ClhY zzIeWyQ6yiHzjjmDZDTSa8j3U*6|xFXPpQKV6r1|9^)_s3H(zfKcEi4^K~tlQ8$ z@q~Gl!6Jyx$F6<0*Q6&(uS1R<6sA^+8sx=XUWhKNN!9FEq3`r`J2sqLI+NPZsB*db zCW}AtV6S=W*1=a6wbvt-hY~8dU!#2u#@>OTyd_uXP4OkUW}m|-*`pJ@()RlwFm0Jv zrU={ZN#eWd%f(xZ`%{Wf&jOnk0iUMOt$uHr>{*)ZU%RVP0q(0KbNVJjTycvrky9d& zo{wU~Up{tW+=;?#A((#Me(B>4{ZjsUhaT-Cx^n zd%hQ>nsG`T+y`7Y3|C^0&M#R}?U~q*R!@}I0|YVAs+`qt^*-?MbOl4>ZR!gK?263G z+eTks_LX-xNMWyOBU0!neUyLR{m!%O>qi>?zns?Z({QM{<#j{(>nGWd!_kpF7y$t*qZOZ>59f1C zUsn&-DVpdvOxDTh7kd8^y2^Tda8c86U;VG0jU%A~!zy?i>_W2G%p~hXh6`u3)r~F1 zBBO;#Q3StY*^Ez+Tlu3oa&&_lT$gw;D>+}+w{@QbnAJUj%b-`43$ZrgCE^xL?o-n zMx?`3J+B+oKLIvr^ljl{C17AB+Kk0MZB-ZRk9OSi-cw{9@$VgBvi~OPJ)B&SGvLeY znbaa$(^1eK`{9HNIZ||^B>oEU0|1k-{$yTLyhB=VYmP~ZVNHL5*QFlRr*4%PM#S{p^o#<_J?~e4Vv$E-(++#Fm-0Fo?U4`x1bCxT4o*2vY6^daO;FH@%3C*#yi%lsqLP z)o>h_RvMs0!3okM(kwvO7X9iUbzMbxjF#xVdIetBEfAvpEn(}-mwqkYX`yPSeik7; zFKtc#El{g!)TsFR#kKKrOvk;>AWm`I*Th=qtn!qFpn3KAg zCiPG?NqCHvR8gFNFgLHlu{!edb1b(`a7P@kIR_ncguA~`)D1Z=udi5QHps?ERcFV& z%KAF7eg851q%@|)8DJhR_o@ZW9&L>iR)sWSbBmha3 z@t+iF16(r=3+c#q)tm^%>r1^ek~R*L$1PiqZE`53zQ~S0;?`39K zy`?0fdf`($7}2C28Rhq1KKAix2o>@Y3+#+h%|~w_!5WtsXqo4zgb44C+8r>t??G>$ zpLVlhKm(fEm0Y7dao)Lxg{BZ<4&ygPH@p2_JXrjd(P@LxmS}UIX{Av4sdk#uqyO?9 z9iK*&17I@5Z`_Fq5}*{KjfWyrVol*X$0NxGC)zA<7y7_Y7K|`i{M7GI>Rf zLQE=!c_z`MJffX6-IUFGUzXy_V1|qcrFMP~)n;K^Fb{Z^ z-)P#{ijj|wW8cuSU)Xpux6CRjKY!ho1V;x>ZEn8zJsf>y#+%uf1I{P=0W<}!5z$`lLYhh&(A8xCBF~_Pu{=K zNZnF2*hF8%^53f1CN%?G|p(afS>AWhqxQ24=O3|4kvprPdRhbNL zL}x6Y!nP=HyWzH|x%r1TmRqQ?ZN*Ax-=GQKn%|1I+(AQGhyd_%M#pSYcwl|(D#vMG5Bh|@rK$LPe`Bh*qPeRPuFkU?;)){!iFCEBp(OCPHU4!U%B97WX>xYgP4;M%&|bqC zrLGU3Pjuvp(#F3ts$o$=<7Rovsk*0oafq@mrntaTy2!M*NlQyRt;(*JD=l}uxUQG{ z(FOn6uG^r0;z*SKjA-C>61G>@yxO1M5Sq)64efURDQStx=%pxWl_I3LmY4Z=oP6JM zHs;q1XI8uzO3%=2Ku;ZLaE33sk{gujP^s_qUc7Dj`CW&hB{%c^QKg5yiKv8oj8dPj zS4Vv(`lE;bBil^_K!6VQ?SDzOnMi(7rC8lb`ml}ed|>6_yy>&Rj=H5TPcNQkE&iGC zKAlPuMD~68WnbPXp*x#u5&A4yx?niH6Wk#oQ)iOMS#hiB z%}`6*^m)}N^D;D=zSiuE-xSL$daoV8H$^vJ9%In8b_ z$_rE9uC0cQgr56{F$d^6u1v)L0v%8iQKy7kO%LfE?)JiL75cX_7M42bZCEdF4lub= z5Qtk8sCR1!HHlo#aURJSp>!^ko z#r}j%R)27H-aRK?vm*FvaqHW6@jnO8?|Huoungu9qCZpqfY$-?%yK-FT89i4pLXc` z`z}&;nLg$qVzAcU<;s%JIdZ)cfOg2ujg5Ab5v?C2_ei$<@Felpt7tt8@9n{Ku!7G($%6*ar zwtGIGO5(MYac8-lXz{EOLgC0pSzGGyZCa!Q!+%UfHA(%{cOCf(LK=I{iPmQ$LZ`-L zHBrj7+;R><1h2~JQRdx@zJ$lU-mk$GE zUWi?Oz_(RMUPY_@N5lSJJ!JKPTc?-)HHDG?j&>0cXSa>+! z-CrebIEqmd>ZZ^CwjRU0QZ9WZL+#0y&*y;0UYschFE&W1KiqP*NUL?bIYZiO(P;5M z#>$ox|2$>Azr*^+wiCzx$%%I_pOT(f4m&SEs)W?JoQhJ9=!n|iqyE!m zLB3b{AkTB8_hf3$@B~;n8t!zw6e_L5-Wuus>*QgTqrZ4@_1R^?*P=9nM5H5kR+=MT z@WXsi)Xqe2iwMsyd%N}#qxrpw7H%4AoQ}m~(&~q=Xf$m(Jbw^;A|;B^&y}kFJ%|D|W`5Wzf#Gf+U)}R2SP}C{ zMUW5{;+SSK@0iwkw^i?zwo+ac_pfZ(e;h@$_)!%1;SKX2qbTYH5p@fBDM?Oz zZX*HC1jo3@@I>47(+m6>dh}f2dclO&(0@J)IsJ3!m*6)xz>uPe@}*xjmIB9lJhdgQ z_50o!GrECF12$5j`BcZG;zWyHWuHiXcg-J5Szs^8E7D%D&6^6?4l6xTXrcd<_`IAW z^E_O#%4I;PNAjU&rb-qloQ70s9ivW9$2H6Bk0{|n9yg3?Jq1o%P4ny4(814u@*U0x z*^s&1O1C8ncDCe{+8?KZV#)Q(y^M|{nhcM|r&7|AYuu42vc>P`_93DU&r&-QxSJq7 zpL~zTF?c1Q*Yt+^z>5;t(AeFAcMYRc2VcSdkEtbx$A@HFqLk+Lfbf;0k+Ci8+sZTs ztVXUGG*7Vs`qfUqOo|Ak$gqk7G^WrOm!o)iP0|`F?Xm9j(ICh{8wkO3^)sSNU_Fz* zpBRIyIF#cbdF{qE49vC*S+{fynL5z=BRNw&-%v7(s~{J6(p(n0Tc+-Xm%FW?GXcES zyR+nCy`NHN3ccbV)0HSR0AR2W_9<$eIwK{ATW!_7ed&Fz_zi_xrvmSyfy6Hcu<{(B zlfbUzcN1^kd^PDaV_}Xhoy+^>$TUAz7%45^7Mc1Sv7o6E?v1c3sdID8Q2>0JOrshQ z2?04$TtnsA;~3AR1U>`$Osy?Z@7d||i-S1>t@A=wX5bpFV_>@vjn}x!YY30^ab|qT zAA^l{PmYp@BY>K)@NI+BN4px=`qVbj))61vPU79lOB2A+vx9Azn|^^=!ad00<6Dt4 z*oG>=y~TO;swJ<0Na4T^CVhw41)IIqL)Yb{#lEM>%@ZOd;$^ zdh71!{G5+E_dZ)_UmFT#CYvLxE_L{}idApTyY+3IF>zKsh48?X$(0qIfy{_IH*eko zc~*xZ(yTz`(G!<4=1#C1=4Xt9y}UC>4iWa~^{U&cKHH|z-u76-3@_Kzi|u24X;5c_ zTOGibw_U3HF`0i4a5rIt1sm7CsEYHZUdkQkE}XVn02KjvLsP&yCoap6xK7(knr@-+ zOL|}1%zRv@Zm{96LLTGVY%#?jV$hjP-W!!FUGGGV{o9WVfOk%BIjVH3!rtI%VUk$5 z$;^c*j~b(Qbi6A29?Oq34<&0%XDln1yY{xjZ}@wgVL{q!xB?x>WS>LDU`Binemot< z&5pz$F_pkhf>jH5LtpvV<6a$l(;z4Jc^0ZNRxcsHz-|CZ^LL$GSb-=ET%l&hH z=2)=f9v+Y_sz0vX3JOK(6dF1eXs)w8JC;00yG4V^rOaOr=!i;)bWWN~$L0ak8Hz-u%UutIwEU8tt3!*(LCrc7g}wJTtR zQ=B!GReA#kkGspwu)qh_(Bnxrg*w(Z=3o__Up|BIoj%O6=|ryyC!EAyJsfRZyE(j| zk)hN}zK2hU7rNuZQcf^4tw7DVdSD6mEPXuI-IMVPQ9d16U+zTA1v2LnSn4@{SGD@} z(4O%F%R`p$tvAU%*2Y$L6>IiUkY5XdmyUVZW@1dB7ZB(**JAcFdii{fI_>PK7VBX1y12c!6OC|=*>Q&#PWpFv1taI3v8#A3H(~*U3 zr;ET1h!N3Jx#1of-lbAhhr6lz>!AOVa=l&^CDZf~`<7;k)XF5QFWJ#;J9M@s6w-5_ zFQPm44o&+>$-$ijgKd;Fx?BiM>G2V0lc+(BJ4LQ$4F>>`MeP8-@nO;ns|9bCuCeG` zI{+IQ&ECg$aZ1++troe~4Cd=aEDsf!L*sRmG3;``#>5ZP=5gASf&H#$*&L2j;I)xpYS2{KICnZ+^I9?Ef zxsV2WD26z%haPd#&Fc}boaWvcZ{KRkNVvnH%J4ME*#booT!0Y1m=E;JefPY^^;Sl4 z3VsBm^}bgzl97(O<5OQIyzGdnIqOLL?I*s}osP*^lVfW8`&GxLQvEx|KfXgj9CT+} zpMihoM;{UxIrL9@&Cr@73kYy7J<_HePF+dtkeUY5524i5*0D|%w6gyDXU66TPF=FJ z&e_Yb&8rA|lHR%B7s#}&jAM+y;B_|T*lS%f+OG;%p%imw1A9)`u?O&pX6_yaTl|`} z(M}oyoAssv*DK6OZI=dt5l6_6h@27VE&MSN2f+$Z@GEBr=^i#aix(wr8oR-Qe9V6B z1_blf^g{?$;w(pCFSCTl=-0B|h17a=u%}QZo$S?+^zb4T$+9c535=R<6T`Fr=>_2Q zQ8npaM6!@X!N_ro)tM>cO!yl!<0T8{w}$CJ^hLM>bf8ZG5*L9Iyx-rtJ-5Jm4Pk$- zx5jl{K7r3zkcP^hWSmQs-eKJYH@#(?)=b^;8S4zo3#&flV zj8nUlXe@|gB}X1JO%1DIypf9iC(>HKz4UqxZcm>`jnEk?iOFY0bUH)7(-mbFF=I_f@;N9OOSs<^+u;4#Xh&#I4h zOdIYViK0X4t0^8%%JaNOk`%;hJL$yo=#7)aaH@PX0}-37DZ;u;5z)Miae7%()=`rR4-TiQ{mA)H7}9xn9h~P%zkURY9}G63m&4y6*;@l189+ z0#5e~iU9MqGaLmWTJ+eK5A|Upr^OFNi~dG{PhUd}8(d23#3IM-l!@L(5a5_4pMI7N zylsw*An^M3>gmoPO(X+iSnpC=!&Xs($`i#`;ov$24vr2S)~d;n0VM({E-u!SwZt}? z@vo2392B*CJy#=oT)M{(wTTuhvGQQYm>&c~kC!R>8%6DW*Z8Lc#bYlAVO55>b3s{_bANSD_Ejkw@H0Da4AKgYNz4~}y zXz8@s4yJB2QSPSvVLQ`td0g*qYlAVwPssoxb2c6`Pp|q)z2dlUC{Ig(&=ItS=fj7c zJ%cl02xeOmOyb98G`}PFSsrK@BTyAYi)ZskGXmz#toEE$k2yk^#9f51gAt?V4DV-=`+XoE%IE+je=*gcLg!M7Q7g6%rpB;(pwxC03R5bd#Xk z5NZ(|cJI9X_##l1e*il9=YmxBNdf1q!Ipn2X{yghJio?=JGDQ!%vl_f_&K^=_bqEJ z8>SzQ3cA!Uma@yC=EaB{u+TPRGx2$m#2(hQ79D8M_kG5B^}xvva@7pM{*upsQ5)L* z4dOytu7wcmoF5UOx7rUhp(j2x_;8Wh8Hy>^b6^AB&1R)@&~O*86k2w*2YC?e>LJ-N z$3;7cl@&s)m!JMs1vH*q@Zy`4a^dcCqn5CdQVV3)S5!PNqWyB&)=w_Lu+>dyuwoSb z;0(R`f-^G7ggt#DH;5bv=Q-5{mG>9!nhW)8iq1ey=W|AQ-t$(y@3@C~dAY-Y-+6P3 zx2+-~#BC{^#3O&Ki=HPz4B48C@?nwq_D#Y7qI{WO|1^cS_T>Jxhb1VT(Rt1IR2!&5HF(c>c2A;GN9tli;>diG z&)FShy#-NlghDB$f{AX=HR}D?^%d&t%A2*+XZ)SrS0t?M&&pX4L*s1miGm6r8?3}Q z;YW9e(tRI2Lc0@YjEg3l`fl^QF#mv!Xnr&2*do*YcsMCBHQ8nUmy2 zccXn>+M#d$GZc9&6-GP|%&$oFf_`Lyt?fO1MLZT%~AP(+(cZPIkwmoD0k65vp|l!bg^{n1Ggf zcDcNW81M#kZ|ET19%o6`e{N5)Q>v18Dqr$!bXzg&(iD_yQxUU|vb37w0q;lt7EP>c zSk%~#B6j!m^RTGh%!nG@RKA6O*_0@EJ=4CU5A|8ii0^jho+8pQKJ{Ea&XV^iUAa_7 z9Byt4@kMTTS)nZVQ1d?$r925HCNfwJ{-wO}g_3CN-YHYwvDqqt7aT`g-F)#R*o)fn zHAn`;l%)b*oH@-TSQe}sBRC!wuA*e8?s=xS%U@+i9P9^wW9(XYtrYDzZam2|a#BXg z?vo>AZb^j-s*`*m2vAoCu*1w9y$^}HW;^F>77SSx_ta$j)`mVmr{Aq7$KAWNa3fy8 zinPOkGn!IWMp?&<&695zS9G{^*rv(+nD=#ot!sP1|nNkiaDa@nn}XovIyO>DVQE_YnkT{3d%>Nu^U z8V#Xd#d2mFqD6QFOC1NCEt{n-xFMBi0=>@Rku$&O_n=)rQV>c64_v28>k_GV%a z46)6RyP?x;dyeoA`0DS*^%DVjeBk@-@1%dU4HzPaYLn_Q4f*w9JFyh5r?o^4-%lP| zrrC{8S_R(vxni(UP|40HRjZjE|Did${FH-#mvQkH=5#tNMrS(TmEw!?Wh23UDf%7; zMJmx5156N~miYtvp{+uzO}K!=P7ai`S?F#(xx>aA%BeRLm8IQ0Po%hxl%%-k969bD ztZHl5O($Ft8_;Bl31n%aU&M_)P){ZDwiIs7={kL!sU-Ic!py~9aahhp^EJuQ~bu?N>Z2~Vw0i6MO(5%%a=+Ylu{5N9thzt)YlBHi@`~y!dvK+VYu%GBV{n&!TAekcLqaF=Oy5qH&t>5eG|gonu`$d9ORb0wXmU438t< zX9L*#jS1-)X*G01thg?u(!>-z9B7?`AhH+#p-N`+3fz&6*)>y=KZXKZyzKKu(dPz= zMA)$KFCR&>vl{#3jVNv z=e~;#=agKeF^zqF5KG;B_8zH)Gfl4(`Xn1vTw5zqY21a;pz*2&}*HcuID-%Vo$;d2hKd+=UbxeBXjB!}9Y{{nK@n{C@jnBI(7B z@owyASE(mv32<1^o7Fh4L0ElYN|qWMGo0ms!c~u7@$IRdpV#Tuv-8gud!o3@2>%fb zr@@b0nx+f4{v5d_By3SvUpPHGQaCFDVDDT^QXpSZ<6Z`za?RlNytOeCgVN>4ff@oU zh0iF{$>Rc38od4zIm`|1vo7rx^JJ-uM|RW3#8<0lxW6JSTExlARQ8)FLT`@1znY<+bFU#l-7 z=HZ9-b%XQ-^^WkcxY~o+0OF@q*VKtmbtnyu%8bBDR_Hohd3-jE*8-LE^S%0I z?_4A5%K@v_FS03gyRC8Ov$BS_g)cuI>Lqvepe$g%&9al{c+;a@LWjc69Ko_Nfo>G znyiM37iXe9qLh=DH!8b#85g=Wyua+;w?L5$6Kd<77krzaIHivhtlc%3n~oP3c!0@c za<`$RXNAQLc7N$S<<(r)c>oq?KhHDyW21MPSIXiVr2PKiU;KWU2FNnp1hb8SLFMD0 zV`Z8qDk|UH>Di&e8UN_by8-JxvN}fnkh{2e-QT6jE^>K5y#PE5TAc@9%YTQP!J!qRV;@Nvpr#|EFbp8mK~~B-O0X{jsFmgv&mk9kzp~cktu zw)o{530MlvfA!^sX`;RO-1hY?K#f^neAn(Ig5Tbv)*D+PGLzH}@7e`ij5bYip+ON% zBjp9$y;4z{iD6?4zBz*hxXpT3@vzr^#`|h4fh%{Ou|HUVWXO(*{-M_N@&XA3e#6DL z&a*%6AMLGWfwDgUmMz%cvKhEAdAgXTd8}9`g=4?v69+bDMvH$ zBWM#`4imz$Aoxc+CgH|$+o=%5s~vVp`^MRO!DsbWMj^7|%_g59H=gGkmfZ96Nr@aO z!>Dsg(bIpQ@_z>aYx&OLvkd7nK5o61a+NmxLo64fq8trMnx<+KBG@WKIzf=zbVChV zrZZ${mT#Sn{&yiiZ~<)0w%RnAnp9f|Lw~A@IhAz?cdBHK<%{3|1|HOzD(eyfXBKC? zbzTW1>xtn%iNFB412Q6kT(G)y|21bapJD!8kgz8~#KKpaA-ufuN;BBz<{c6#_t~&T zSHS4V_5(HvKCYn>F~o!iw)5UbYQ_#pt9t)oPxei8-RZN_xd=bHE(dDyTpMo0n7L#3 z6S>Mr7RW+1GBX{JD<11ToKl}ou33~>i?9!#-k0IRf9my@eF`umz-2Jq%$oera7=6Yo0WL zIN`U&kXaDY`Q|cT8~zi*qi0fb^>;&@VhC->usl^CBt8&y*wILOmpK{&fz#5Y6GXbi z));Gl0zF}C-B89&m0Po6qka@F_>#f;U`G{$VAP$6wF$Q+7!7=N)Y1+%?_yPMZ%r>F zdl3Ots~*UrwzTgy=oFWqvY}0O71X@knT0=h&%4nvuy$Y#*WtN@wPbvrYsKa-KTiPmT9e!VFe!KjajLoliE+-5fG<4&@2&DC`|#d4?ER*R=yx z7TqmA!@|4G5v-dsHspQWd$6o2<_C@xzBvMz`Cr91{poxa{C;HL8bo&JVCB;m0?+Qx zmw@MyjuX(2M!neS4W;L1U5@4&$g_Acoq|~VW$j+CmBesjL=>58Cf=}Tw?fU-{&B_`yN;xOnJJVsejJv-slkwMQksI9kb}0ujI)#t-G9EucE0LxLv%JFU${A= zDjvCWu#33KUPggFUqIZc&b{w{Yq#0RdDhPc;~i~-0|?nl@(&mzp`K?x#1=%T?8)uK zih(0K+05Djv=Ip3K?Wv0A&W23{e6+&nf2{7DR?it#k-?!%Zb9%$>H)zh}kAe7a-7# zL!fi(>P@NT_P`SVBrjD3RFCT&%AZ^+-!$tuw9fMSkN&uVEEkw?CfVr#Qtl~Qi6WQE z8w4m8TpGwR6~NKm4jJ*o%y}Q`ne7vi#x1VyVkOj{Hcai#R(0N_|;SRe@}dZge6l_0naBxvt-U zUe~VLpR z`-%rUi#i}(9*X_TV6(Z+T@Kxr0x&S;D?h1mB|!OoxULwGD_~w#vyvrwh}>JDUg-6{ z-2vLM0}IC5gphpZHnVtXhfN`QEpB3LfLxmb4dNl3-N%|iQ(C=U6p&{iD7wQE-HCQ2 zEB|Bmkuo3VbFgIiL+90Mw3ry6^eHw=>l}I+1`(HFrX_3>Ov$;}_sc|o&{E8f>tIb%CsnSUcsJ`ItM93&R-FNeZMtYA z@X@ZF3&bR>&bl{W``Vxsg82LRcZuZ*|VXfO^W144B%DmE9-MjpW z+6#&kWxzIi6bU`&COOAC+CC)X(?&z|s1Qy9dL@;R>~e2di&j$GJCeuLy#f4Aaqb{| zpcwb7I?~NX>aZ%HuVD9XlBiEL(H1r4eQcLvC)_EXqDyr8uu6}rkqys^ISrc6Pqkcl zF*Hje>e?g9cQ78XXgKv?Z+SP$C`GqQ{K;K6E8(YICFQ;yQhW#&S-0EUOJadsnkkM- z6UikF&MAEj>!t9i-xa{ju>~0To>P@XYnIW0B=B-A^gCTUf>@VbMx6LAD{cN-$E{`+ zYL-QX1glibd=aP1isKtJna^Uu9UahRxFWUBDq6ixSGv2caq?_HGB70a%6B@qQf@19 ziY_gHKlMK|PkU>75h3=8`Ig3A;6?E6LLs$t!jt#YIMQ~Y(nbz_j?!^tZD863HbtWD zfgUTW>frb^&sQVU7YbKZ(cd5Oj(w3}iJs(1Wg0nNYe+9N$*6^d969N76GTYD( zJpOg^ZU$WXrx)}A{sP(*XHu>G*ly$kAzsOzNYRfuCs zoZ1FxuyXq{|AW9+$5P*9oIWqxn#xs905+BlpHcqk_eCNXU*{Kq#WJD+ufwtx>A9?V z@gb{}Gx_qYTE>%vo^18Cqou`JUn%6*$HWiIBKO$|)U!S8H=9jeN98uw$PDsrsVaRW zfn%fDlihBz5jlSEEX2wmpZ0VS!o*!{aTkPs`aU#F&o_~xU zl^8auhq3=E%jR?V(Q+rE)l{uF0GUh6vR16AxHmKo{VKtykm_-p@u#dG~l$2PDSs%ds!WN9x;{$vY5Pk$j9Yoe-$i8<=7Ds@r(t6a0yvsz7)zmlA*g<1yh z?HCx*vMu~sS>^o060b}d%@Z9$o6ssI`D_^i#=^F2Ffc%#nfKiVzQ{WbU88@lnP9}^56$?tx! zIqzq2x~^7PBl4>Dpt(cq9re2_i(Fp~{n6rFFG0(L_ajwK8f26wj>VKunoa!$ku>aI z0@T-4fLpA7bH{1+Jp*tsAh+OEVjujZDyiA-CM1qP+-xNzCdHwLt?V^Fxe~4w6)_=! zoCwZRDNNrBP6{%&Rd0LXJI9QKyEqq*0F3qdU7$W z>cZXg$xfRSZK3qJKl!}KP`hh2hwlw&2^ZErjeXSQfF!JF7uC~JrT4sB_g~pNz#ymWfQa4n4q`B#G7ahoOs{or&~X z;!lPOYqX^A^i_c=XPf8Yw_{ovnI+&A;+=rCt5hF|2&eH#d}dO1|4!QH)~TPS{NfBg zPvaW!Yv|IciGtee%Ft|Zk1Zyb@$r%kRhzbkt)r!-Eu{kZhH_m4IpUk(`{sPgy7>mMZ#Vc6cIUl}=+vT86#Rl}0z6e>Wg>Z1otEek~ z-0E(sQ2)u09Gp>9^{{pSRZq4AlCShQ&C|cO5HU3qTaj873&US5?A@)r)a;95Q>p9Z z>)H+~lkA;=3#vq}B?sC{TY9)hUaDt_5+S4P648%ePGK!O3cT6!TX$tFIDcok*D9ky zMu5{?n^f9-pT+B&o_M9^HC!XfgVxwjQHlI68c{IDZiky>vfK^XI^(ad(XYTIGhcp` zv1x&53LAlW;i7S&d1__Muh7;8|Esk(y410ER{7lG`o48TVv98(k-l7PJx;IyMxHP$ zOck!$laf{X3I5z0VciF1Y0h?B?aYtM+y*|AW8t>x8V~N3<#IlQegA!{*^6u^hUF1# zI{-+4TAJb)-Nier6I?5!!UbZciE^65SWxpy$h985*yDi@JuKrOK=)mq;)}nIMk)PNiPx+xX~o#RBu`z-&WV#nJV15BGPqpKb*x=(zT^nzB(SmUOJmR?f;USP9NWJI!l4~C|1U}MN!+l|5hlQs7Xqu1L`l(Q0}@C_&!8$bI`x*Li~Sx4 zn9+OXjrE?>11Rk1o}S2v=lIeB(J*kyPfiv1FGrm-K4U?@8 za0sq)OI)MC_QxKwK2xc|5I!eqfsz=i^uf>%ioG^->S6BR-hQIm;GWDg;v}(axl96I zqyHx)OM-}G?e1ml{s|nDlU^*OE|bhZ$~;MMzI^2_-7fQ-E7|MJUiKl)L*M0wg;LIf zG>%xu2%_HJH#vJn0650`I>Q;)j4H8EE(duen$8ki>m zFf4s_pRD7naf=2Wbf)Y?&rCh zFPF6O=q|{eE?T0~Xd{C6vIQv@IfdhB{|TO22N4dk0dgkP zf5KDm?k)v>#@_vlREVww!|{d2L(8^O&oS3_@S++`+0fbB`3q0C9cq}Z94V@2U2xHL zzM^EQlwWK=6*y<~{q&fSXq)LLC-LpSG2!b%;meY^NY>y`jC06K+~vjbZGUuSOMhb5 z?^JRh#Gc1@{ps5x&WN=^;4e6^vPrT2lXf6K|L&UAyGGq*#-j$?g1-jBVubuvtYO0d zF4|D%ESMQ9>VL5e+Wvsa+^3q8Tul2*>hEBE7!_1}TlbIm{x4YY8JH7vk@RMNK!yM; zyrzr!A9KRPib85I`ST5lnfsHHR)q}bcrJKrjs`f}ITzIB3;ous>Y{Og%|B!}7zl%N zGOLS|Ws=2cW=DLMh$bM(eAi#+w~sbpOCX6`D4l-h*yxmR%-e7=`Q1k*Q_Y2MXBUB@VIO)K?1KqWXX*! zx-pB744vCxp-kmCyTNRDO^e@S{l$QM3%A?5M78<-%RJ3{$7_Z_cNVhLZn9em7#=TP zbcSh}UB*bblU{{6Z}(2Pw8MN#oX5crrsG%$@7OdBY`f(- zbx^U`x7bc1dOXOJvyz;FdVtBdv%inJBRUr{;lCVxko6>HWXrNj;^%DYf}FCkpp{yF zeiv?D(uw*nn@1Zeeo2Ul(Kem?=EIK^{|A(-8bjn?V!ofv`eS3cCP!LIHd!AZ;n*;D z!U+cq3}E((bwWR9ZT%Pz5X~~(i~`mdZV%H^BcC_Cv8WB@nVj5-IbM^z2?@np2Wxkv zy!IqnN8MkqAZOW%>$H(uOe$i`M%nH#p{~iq_FIYvG<_cpW%Ik!d0D01p65g+X?e@L zQ7YNKNl!#}&hW3L)((5-TQ=|-mKX<@L9vdNxKEEoEb6`m*>_$^*luiN$TELYq}GTT z#W*Ci^2(Q;ymqOpB=~=!G|2W+8mX|OJ9qfxitO4_odL$^v=sKpXKx@k&oeiEYmItW zf5jhTzgWFo?yzmNaMF;$HD~eRzJlXu_4UiQqESWy4>e%gjSpThc+PLNifs8|b^FQ` z>3Q|@M3;XZwR?5o#B!LXX1jY!W%p}xbapN8gX;aNYN#^he<-YkSowb$R1Q#mN(e7r-CXP{ z(JxZ&pC4Pb0mWbwzQi6p!9XVit{Jy?J4VLx%fw|J~ztI1fX8}2PC z{B{2@cq6g$WOrI+%co%%foqA>Jv#YtJaKOOr$?7qD`Z@%*law z==_?JeP(I8`gO9zco@ybG$%#+zm1^8*?dg~5X~XYR7HPI?Q<92ef-ao@3zP%- z+DYmhSo5cupmkmx$%*|JodRcI?V1(p)xAKQ?#;ePm%ch&kNw5hM&fa3|VEj)@g-vdu$Eln1S!t_DG<#@RXZK4hZueKd zPpa!|MAT8&>x2T2r6{|SpPsT{pjuiT!=T2IJ0jKibuSOB&9deZ-Rj9>(j}b4O0>}- zQu3yg_&IwQKpja~=b4l{Su!!rohASCzvIcET8@Q$_xjrxJzK}ClUR}28f5KVOw=z#Gbm(+n#RLh0>}d;EDyX+_ zJ_x;6hix8gHJi!k5arAhYzZ#!+HWw;VK!zzaGz|0926!+3v(G5Y47Pb_jTh7qda~e zo^DIylSW7{>GuZwOsDc#KUCsz*da~U2yIUF&U}4WeSH&8X z!+m?q=ME!_bt3e3f4dzTYU!zq@%2?!gGrciCWGxCs@CVkYDto@?~fYd?p3a9x()xS z6Y5U^-4CidDa^HP%~H20=-cO#a<3;PW$3$4Rerw8aix0L-G+T8k zB!Y<(jglfq+^Yp+R*Q#%4RVN(Jmf}&il$ek|IMTQof21xV*Lmih<#TfTdk=SZTg8O zCtqy3AA3Xc^!9>~hB#zULt4P)_o*Gi*tqD|jSi>EM#y=NDZi8A7vsK6ex+D+Y=K!4 zn;(W68B=!^Fn4imPRA^bDRPY5foYuP9OsfJnz!b0-T|4zQ+VdnNWaB9)r8-buB3As z5ci{)1lf|pgjd$5yJRiY>rNJ|vlu=)V?gOfs{Bti<^P59eEnBVdHY{AW&J7N$3#(x z!kG2nJgUwerLBd@*b2Gewx_~fR<(G~k>(P+1f_aLK~r3r>LcIoExW#%)zrCh?iF#* z=paGUSr;ThhyEJ!C~DU}lDtRW7a015(l9B|aFogmm>=oNM)j6wyM*4NpZSAvQXf^} zuPQstFpqh87sS!oIDV_=;zc9(M#mJIu)$B9RS^3rn$+>LYJ6JWBU#KO|H^`Df zK-Ltq@Ok1V={6yeeob1!)y{|3zS~j3NW+&MvxsT(dw3~(Ewk)ZtA0bJTkX9A$d0P=JN*G#XLI$*haixrcD`F0#%yUsni!s^$&#Uv{9{Nx2(R_=0s*u z9IBDkGtZirEZ<9jR*@`&qC`bwANTI&JO#h+sl{SLuh^Em#&`d#5qiyfewPX8nn0P{ zGr)M4?%tL4Y_5`s)~A}1*tn9Z3?;=kbh{vyo_iNZZ^_MPl|R(tJdzKP|)Ho`Z!t^D3! zt;pBW4c<&nKX_@C+x1$`MyqRd7_oosQ_~6m_^6cBLvU6GWWYrQ9DHI-mHNF8^%+<) zosVK%&7}}nxQej2DzAm_Ov??i`iy$9D8OT!mQo%iGE}u*FemL*$Jfa>dPL%JlC~(( z5WKQ5n&jA5pgt9%l_NJnv-6|9A~d232nWE~_&a1=+Y~oOo}Cm)vz7elcGu}YYIvf6 zFUUaYn*FJx(B{2lFR=n$tR*jIoLf44S1Byt(`OrIHoyVE_0DdeG;5a z{jZ3ur5+5un4<{_zM`@?=;Br+ggo{aefEe>q;rZ}(-rV(TPyoyMW*NokUy>r0*73S zHosD?WaMaF)sIX&a#*fD2XgybW0C`y2NDAIN+|)$Q3i;hbd}u`Vu0ZAY=*}|Z1eSb zW!GGS)ZYm=P6Nf>S#8eAA~4#kkjK{$CyI1eckvuV#tA!Cp93Q6^yU8@#Rt-`9DA!d zaift&MV5FoE6~-_VQDU7iwr08uk^mi38Wh6IGIs@6w(DKVvEJ4uC&ws~; z0+|R`3H6(h0_VxS-_R75NTyZgI2YV%{ZZJ?vh<^-{h>Pn=L*`DmX?Qp3)*bHi7T*h zy!qz!>-4-Bm37IJdpA}7PVSps)R|k$xCu?k%8JAc_%yRt^^&DZ>Fu&19QHR1i?`L* zYzM+zWjY(W*l*f26iei_H5_axMz1dqx}0H!Tun6)t6w&&aMZS7Tlg_Gnh-B&K40Ly zX)~S@x;3^t=J12vX6)lSoUjjD?VZ<$eROH~>H+MARtP<0xpQ}HwcNpZ&y#E+q6sz6 z@iANixx!Jo&v;etu}9&-hqEaEL%~#O@9`o3*O9JO8t9y=9*`FP*xB;OuD5u6@Mdn8 z_k?He#H+}%j0m|yEY4$p6h%QBUm`*w>U`4Cp zSX`I}adDfvjNG7s(COr{JmTrj?)57@fDVTa7Ya;!?UHU>Vi`AW>Abhr&9TdI)!RdV z=U~Q94jG=^s-NSxJzr2DCIoTa=+*Je4rA)+ohzlETl<~GHD%u5u)^<2XU48sY*r!c zIqSD_;bv18Ao6@HgSfNIg)1TIsT>%B&P3NxDg8c^4X<72MK#8K&*cO7rb+h2-xlC7 zbFKBB1U>VX`P}%nj?8#)nH?5-a7(;^^*BpWrA0pmB22$n&2rv0vaVh)`|K|3~~7%Sjp1X3PJS!FyVFUV4j zm)~o2+nS!DOSbWAYzrjZ4u7-=oDybf#}b+tzr)1Q^@}Ei|Zo;)>hWrvgN1C~1$wv9f~? z=cB`|5Hmi!=PxVO5okUDPg~7^akiV%!Q6N6JsI3vXlD{0Ea7*su=vuNqDKn6fWxZ+-wP;wylQG}!zIpU0AbsAVY^64tTg0tYCo%G$Zn`0gA7V%& zH&2b!+L`~#=QtA7&Af87HH8d6aRijKTF>uM`?+71q9NE2vLPKG%i<2f1fs(n1jBus zk_i+y1%N6>;ZRF{L$Vza6K>ZPXl4KP8)p;LNBn7x4DfUI=UISYCZZk!1Oc7`A*CH* zXy9bn^Gy)6aBz-*ERIVMh#8|t2UQQSV6Qr&*sUKX)!h%rKJ;HB1_r+Ea-OK#jVzB| zb@@SwoYnkS#OM?F=$Ad7!J>&dGpqVqAN#p?<6BvKP=Wm{zn#5^fg+r}KRQ>ZW>-c2 zDN@C8U`c!`_hHRIoxGHHaf)b4iNIvTRah(g*b&~vesdPy7kLy5j9fs6kY+YE{xfXT zac8;4q6~)B&QhTjm)k1^aadz}QlAh6kZvLFkDkn`MZ@{PB>_hvn80sZ*0tx_aCKto zN`PENssG6sV>xWsaw=ICbl!1_^RB9SAyw|u9-zLXLMcbdsq0P*0ql8sfw&Xhs#mEj zJp}yXJZNe%I}}m*D-yVTrpymx+l@c(s=$DSDo2ME-sCg=(N$bCr=;QFeMFg#(Il;2 zwFL-_SD=%9CPY{|=q{TfCLYoyia6D->%Wn2{>v2E6&k}m3~N5Nr>-}JAZ15qNX~)` zKoBjacH4mTqPR92WRHBX*n+t}&rIF#v1}nXYlp1=y^&Vr60^9oXy?xlP8-gqb-tH* zP0|Aa_t~mQ`h<0*J_hu>zgL4ni8D~WTOLMZM;{D7>nd!bt@|c}vEF}UX#0J(&=lPz z%Q}*njJK%S9lD)3LKI<)4s1$VO&TjyTjU$$-EZ8>P|$uNPWmh=DylD3=lv$5p!nAZ zRYx()oVT6<;ZSt4Dlr6bIFyGjq+N0d`hE82(xHEPudFDbkw-A?bvMlZr*n$+r^ z98px;r_|^rZ)NK>ZW!bnPywk(dN%Mt-mrGA{N+#IKH_#wlo$ip;=2C0C#5y$CGRKq z0_S{V$O}r2^WUBa!1?T=QAWnPk)I(xI-cd7-N?&S2yB9pfl+AvF zPeJcmBKhb_Hh(~=X;n!2a}24aTo*1CO;LW>WN;QVuN0LQZtoSQB1<#+i|p0IWV_>BK#`fQX-Xh_erUOqix*Z&T56(Ja`Ta#~4%#VDen^ zLBONV6%*@@+1#P|TT`{3$$rgcw_~q0JWL>_1xkSKz3LTC{kO&0%g%p^k!vdz7|dTw znP)Mit|T|mcz4nk$5;M}n>Ec-@(hAx>t%hAV5xXXC#lTWa<}Xa=p=N@HBFUsyIL-E zX6!Jm6uPWDf=*biN8_F`aXO#2IyAn)mREE z)9y?U>l9O(gTX^$H&}SS8e+!TWt(D}Yo}a3al593cPyz+2w<;BrBTJzDx#&~FsWbr zxlqr_MD8f~Sao^QSTGz5H zbm3qq8=B4tF^X@nX^7KqCYNeZ@|z@^14=^}-doYE%oegNf8OtT_=biv&u*IVz9Fg7 z#{8)&4KiBQ*Yw&f@5NIlbp4$rW}JdFw?-OH0&y#e%hK}CP>|*)(^I%t9Y9ie-L(=p zu~R01Dl(g)!$m~A{k{qFTg?w<4FMwDPR^CWx|u5i6}O}bDYh3&&n0iVU7OvB*Ya`0 z3GXN^RapzG{j!~E65vu$l7s#B0zt@JGi`~zL(Vp&zAK}cXQaN=A{j2LmeSy|#{w*K z@z*|{OP)F4r=k&7o9B>L856`v?)6o6EH}-{`t4$Xgsx#Vt;XkKv$hR-R32mD*k7m( z?(B}Z`Fh9fqlu+dz=4G(5BQ^5afwvw%zO1eUUi5)E6aXpWmIUQ*sSxJZpju;_1qgU z_tcOUcE1gi9+lji1gLkZi;;8X@8meOi$rbd%o0&AAa-oOlC}4%1EJrewGAHzaf5z) zS3jk#x;qa&1TDuvA$%JIz(Qvdzs%eilvVo>)yfk6@v%)*div;u>bJym6NQ@J_>2la zu73rToO;jO6c}FUe6lz0b|RLpX#N_3eVT3jRRc7=8y zYKBr5uY@=o^XR2%Dc)A%em~O|pKh(BQxxi&H+d@kv3=^dfISc0R~ge~vT6YCYhEF28Au2fjLaQQ23; zKO{wJWSwPC992Gt>1Qn7EAFEB$CF<_{h6eow1uMQB!4g@r%2b&6Ayo49z-99ki#N% zI(KZ)ivH5k_0p=lhX2A}xsIZa9;x||O4V1x{zBNtzpNXJRPX_MAd6l^l#G?cgERb} zTpN?bM5jalPYi#6L_ODnzTEN(^!a;;iomu*-T;|4&iHp0K+sA@;Vk_A3yQ52lKLO* z`4S#^etrMujA)+FTf?aDwURK6C(Xm_wk~eX#|6D>OmERCxDpTc*Cw}JP^ zp7Z~G!{*?fg0W-?Qd~}3GTX8t4ooo^7=bvc)w_7^AG}0w6v-6ex9S}Tx2M-?O6WyB zMuLbKu4d>d-!BS5sQ>j2h+m$kATn5iFS-8s*}KX!KWJYQoZbKPdZw7?QWym%Xy`6H z+O{k78zk7e{4%gJ`bG3m>~mF@$jz7SsBQ*L%ug}M0xLBfCzU7?GyG-$m-!+D%*I+2NdIS zPlq=M3{u|HDNG@?aT|JhmDxP362lVS@~oC_Eh2JKf)T=s_bGV2tsqim8)s67-0tBy zAFR(d<~^Od;jyv?8Ig9UOC1T|wUn(t{uoevJ0wnjM{KL0msggCJ{ml5=Z+aF!tYn& zqYUV9v|(ht_K}%O?)_61Dp&r?r>6P0*Z@2=_j>?Heq}avJOzZcs>2CED`Vr_e%mjj zE9BDjL^@NM7Ova5yuuVFqR06E z_vE9DdcQVOO42ORQB|?`2Mu^wX$=-XoQm4&W4H8uFPMr^#i-+RC{`stRLBl=#cd{p zAD<(W==yCJf+=Dl3y|f1pFK-0mUq7(8rpzscE(pP8XOc08-$U~^(3^^MsX+*n1nQQ zHFAX#x7lRVmXCZ5>q9lftZJ_Ot&GqQxemZB61)4ITd}!bL~op;@nhRjGRAq^r3+M+ z){ZYK`Y!+Dp4vg&D>`U6Y5wT>5cHIaxI!x3{ z{jv1sTwkZYL8hmgl=m9HQpsC!j(pcw2is1A4u}a)gU`CwQl<0c7Lu+(9D#Bfo6o9! z*Bx0NS6CHmH4m_7$@cEP_k^g=raK3hb!Rk&XkL@iX_WDJ`deY*^JPbq_q=ijQQE6= zTDcfHT(jG$)aC+1>_GXiiOu;@-A+ds=F!-5kKcQzd$E$P&seQsd3C)9#LQB~yb#!} z{=b-L8$L23#IJmQ`;RT@Uxa%Kpu4|f(*M`aew!pz{ck%nur?8GmQp@q*Jz8G#yj;k z!&0|1?8a^kNVNIrghq>zJ@jiSg9bRw|J);{wJN{)e*6j(5%MYie7<0Yoz5wQAY~57 zk8XR%LpCFrko?3aOHkm%H<9YK#7Qv(ehjCkY@mC8>fnt_y#6{)qsz1N-h{l1{Bm*~ z)_GzHH(Q>B?wv7Q8$JuoMxb~0NjH@CJeOwJKi);3K-$a0e=@3d)H`TzN(Ne`S^W-x zt};c_c=*)$C(FU@k83vb$X8$VbBFpk(fXtC zE*Yz-6$vjE2lJNdX6#Q=5ao7z45`>hZ5}L_8n)3m$uIQ@URonc700^mq`^5jSOUsDd`P`&oKrp8YrFQL*P zFQHn0p$})gx@$R8S9$4;x%ueW*ccUU(91LLUcOxGaF)a*X!uj!I702VVVY-`4tH8s zkWd{>vmkROSTY`(>BOnaeds zAJhkpbFz;TK2PGc>!jRmLQh)lhlhn2f2QtX^e+|JHAiepbXn|(sGjYoNhve~{>Ca25RdFyjIDhxsJwrPQOc`lVxuE~in$81+6u?M0-C%k!xY zAMyjpF-whqo9t(W+e*R8y5ADX9ghN--;6M;)troI1@Et>R6Q%O+fdyeru^uE?lHgz zNtLfmEZ!2AiCB;~i+p25lWupE5bY8ublQkk6iGi;@*bscBI0s*kKV>z2>DHR{o##% zWTBi7<9f!jVpU>8s~}#obXIctYCG!rU}RUGuMr9@$JlY3c>h?2E7Fj-Vj$mS0|OdCLc{IQ#sZZu2=Z#<>n^kp2eaNW z*tO(-qziU|C^?;YD>Rz$^HJ8PT%^NFjv9;vE%E;c@f9MuF)6eHhmGX zp98|v#6CW*QlGXeJD)m^NE$#iL`-H)<=x9YYmn=0k+iwBG+lejtNlmbevkdJGWDBL z&m@jO!I*>+Mlt{V>v4{HNwtTyb8|ge<@;4yTzOxQ)ZHdWXRjS|quV4L+7%Opy*ivF9FaP5Xx|@9$}nNOX+PN?Hs3v8lAc$VU|u6CQ4w z%$Ru`Mazzn%_T(M)d~GBHH8*=FHNF~mn^c2417#MqKpl+fySp^Sl~fs9rmbh#m;D? zS(W}=Ol`0hygoIp^Q7ipH7yk^t=kJGzv~l3dK&XgJ9T~k?1r1(#laMUk6^!#`Z>E= zew2&%u$mJRh7f!&y&|RL8eh^l3duEiks?INUDptl_B~?ynqtb;KTb%*|0=&wpsCsx z(g^Y0{EsL2eE$p8r)rY*zaO9uJOIzlE<48EFCT0MOjr4B0=Cac9;S+Xs8}j&NOI4m zftZ{`_-X5w$FoWsABVf`ic(-$Q^=`c2xvm;Q3?jX-su~O5!o1p2q~va8%9F@D8T=o zT(+Ti^GUX7Sw|jZzqxGx^*N`<*3ddXlO`PhxbgWM2tCXkcAxE??ZD~bINKM~0v2>f zY9wMoXCv+Pbv)X?7EiJ~QjrNfvdpMR7TH-0q`f@|h$RhBD&bhrOQB<9J&&~u-x|Ur zx^<0E`+Rb8QXj{uu{cuhpeB}YoW1`Ls}jM|(NmhY%yRxgY?{x3Ym%&=9DKB#v%;dR zR=YSE7%+RIIr(mh3ktS3`>}Q98@dClDg~kTJuQ3XE=D!#+t#5wA`Htc{$xkSqB5S{ zZ%aYScT18`o6_0X1Z{ssP!KQ^%?8b~m{D}BQj_&O<_$+j)ZH3Fa^K~!S=-DW{sCLR z381CaZ8Ls36nYoqgXZc>SFo7c1l>!}Gr0tXfPng~1wL$|uVq!5ynOy`*~6|nj}2RL zLM@R6guzQ4VwC$nyOJfzBk!mMjY5 zJ}{EAhbwfj+3~p$%=T$_xV`-?F^Dy(5&NmaNx~#Sheqb<$=IdKm*M^qp%5E58l;H! zl_`lNam#HIzIa7nh+nNey4sN>h8ifeaB=EnK~B_pNB6BxHn#IfaW#!VK|809T8r^b z_#*V~kcl)Z>0aMYvy|nDM-9GZ5Xzdf_e0Gjjul%tO>NdAGI-^7ktL{g zpZ*0;-kp{DMK3AP`E}j2*cF$jy`0onY9l?hIbKJAcO*_tte(7D_nFDB03TfP;RCIR zzV5LwG1Zn*&X8E0x^&zk3`!EN2JYou3yR%d{_JyZSi9l-iI3(zN;vXbCaL=ihT`i zPTdiyHDgP5C$Nb@%U=W^0eN?$4LILrXNm348O*22$%zahW(?8Rd4=5~VijMrQrj;B zt6CkhgX_?!vfHpu_)WS;@2kH|9bOsq&X{a`(@nYq;a~cYdPlBWGx2 zJ~-E5GrlM%jh>#qNWg2kCrCX(P>XsPHm`!y&Q?FIze?%by@a8*Uo>JxSv{+#gWCkS z!MQm&h8Uuxyd8CjfZYD%<~serj(R-5LZplbyOS4nOZoLefs3-cOZ4S2sE|ynRO2;_ zO>?C|slU(E=AF~<%4h43wYBk5KGMm<}7Y4dT` z+u;%Fvu8iZ>BuDzwkfQN3dO=PY|5-jfkuogyKNwHe%kMSwuYBQF(V3<3|Uy zj_~&vg(9I?*Rvzeivt25jlvz3BjpOyRTu@#UZ(lYQe}O08Ty!|B7bb+k`?z>n~B+} z1oRgf4qI?T8H?^IhAdYMe+xdEZ^WZAOK+xT8U&XLzGOc8IXFxTuJ*LVVq; zjUWhI?NzK>6+;9mO>Ua{!7J6KUiFnbC{Aq#g&hPWKK+8jC67Nl%C?2a<65rvz8Gz~ za85;3`kZXFZq@Xas!@Q%=>fq-r44fd?tJY)x$>TTeYf%$uXNt;IAs4l)&w{02O`?q+ z8A>Gqm4G!WAz#!)s`P=y+ZU}O(9W4hWk>Di(G&;u2Vi$uRN`XQx)8_sbjmdAYflD( zIlnz9lt~2t@Zu9RlD;*nN7XNDqMc}8WH9rzv=w@KH$a}U=?5(Qk(>0BFbVqy*uBE_ zjd6O%3D4)D@J~NY7GB+xg46H_5#p&GgZy zf1N0Yh-hm!zv_VU-=Ymi-HszGkWy&khEUC2dfRt#ZNUDMs?9^o@!ni)4&v-rTZa#b zV+Z8aCND#w7?&Dp^sBtRLhvWQgqCa=$S|G~n=3&)Gt`Z)$g8pRJ`5 z5CdnpB!EJs+T)3i{xGcQ(T1^AfUSS0|0A)Z0tG3H(v&su800a`eI*(_$CpsMOT!7r ztszj<_IakGKdF9BQQxvW|*S3fIz zZP#6y04uM%w_ny*P)%!@v1(F@{=#OR-*XZO1{%Z^3P7RXwL|WWrir9I`X1j(}=Dj ze=GXD^)g?4F6HW~QcP~AXwX-Kk9IzWC*Fj_FEOvpuaaz;yUZ(ODyiz#DBs4&SmEf< zpNR(ViZ2T_F1uo@&~?*`z0SGGE(@I`TNjn>`_FJC%v@WxiIlq=!;O7FU08oKP~b0L zGG!ZK2il&lc27y2<+0!$h6O(=<2a}K&b4QH-Z7|O!jwD8R#fd#_nCC>hI+m@Jb6)~ zKO9U8PKRHKxojfDiB8x?M2TUGmNqXw5gFk=_Yi3T%Wrv+d_P$>|8|#3G({SSo1Q`s zq%(G(s(RWJyjOVVv)VY;@Y&PepQi4RXAbh8)uJix8>h4k?kHy~2!S+he)vQwj^oo0 z9`Y_}_l6SXZO1|tw0OaJ&G-)_YMsP>rDg!UX1jm$=rYx@BlwWpNxe8_Oe0vbWwnbl z$H`yhN{p{=-S|3>ASi}oqxoH=EAZq#R86lbN-k)|lUTUat)E?t_4O^6VYj^-K_k`M zmBk79aDk3A@s3S#+g*01E>8FvFoH-iE~YcLUYrG2{;E|3T1kDiT{pe?X`!8}fqm`E zsdZDQuD3LMvG=LKci;2B@V8wX2Td9cDGsj49(=E||Mt?saip-^l_SxMjZm1;@#a{N z43Sfcy-J3#y*mg))i#eTT^hLVu{FmnNu?^9;*+uj1}D!U=9O~$1(aC1DO1;WBc8oW za^2LS>n#UKD6Ih1cz5>>`1N0wxwOlAbm@#!2}MGbVK+=QyX?2r9%MQXpN|>KTt5|0 zX$yHGiIQe^dc?tFo2xu0a|?ZH)EGdYcS6B83C+e>jtSbpA@s5cL&4$q>?Qf7p* zmizr@bs3LYwgxs#NuLN$-d8TR7Zv;gV;Vk>wK9A?8Is&oArOwg5lo$YDnyxE+rPL4 zq#P8@ioCtF&hq+#t7RiTbIDzIC$R*64mG31Wpx8`nXPyheMZD>qE$naRJbyR>hEH0riPuoDAN`F{hy@k1H+;E~r0M?YuT~d(-kd(YUQslM zeuz6PZu5_hekrVGE2z-Z8SlGu>9B+)XoXy~H%-G_IG|Y9|7~A1G^zUbO6UfJ>6Vp;LXNufXM)@Y)eZ5jHO5@ew!+Srmw;cvUu@?(H zl&5o-uujp6?^`ow!YVat)W)-0m*LzWQd520#xy2$a<1$^_A#jsBc_WsTn6Hm=AsCu z3rbma0-WzIjQ6Z398cKAOjV+LOgkU38Sr=Aj{Nva5Il5c3s3QK3)__fc*c3>LkZdOCsp z|bIC7}18!?^?2cqsU?YPg|UgR+##keFy1QcXLs0wYlCieOFN!IJa4Za@YMsjh%fO z0!!AE2-5BgP6Z3QP>?;5x!}IRVefXmI(h^qifbj8HPu@;17V|jmp-7uiSR=I**@6A)FWqyqC z&0e-ZlmkG*l-p!&ImEe`S-ieG9E4M|sCwV4x5cZ97`|f*#XH=rfU<4|x{#$t@_hQG zU|?CMf2tBo#W6*s@He2W83U9x-@E$b;{RI_<+Vck8o!ZKc+#j8*0b9#>bZ7IWCxuq z;dXrN9euwF|F{EP66GXHGBwjebMb-`2gA1nW2BSxm^#`~TFtO{Ida{9wafkPlSXTT z+s<7H)bYEY^&QZ*3yICb%iBN4zb$@^8Z1T_*sjI*k(Z;25r&Xh2G7J0HuC)c;#PX( z0JkzE?+_q3w|H}WIAnr~##>se0I{*b@8&+^?^YL7KQ)}k^Q2bk({UJtbtTx!m-^{L zZFC)!Q5L;A4l4q-Do0(1xsb~GR+p=Ic3f3X=9ypSTiIB^xMR78@ws7Grc3u1U zSG*$_6gNK4de@vo_vzrjm&&%|pQXAc9+Emox(MEg+1*n{2^ej#8=c#xZfjl=FuD1B z=l0LF>#unjCi`?2c9%{%Z9aIPjAvg1CJwJcF6g}IzI!D6X^t^TPp9@}RfN{e3bNfjz4~rpJ}32ON{Jif302zW59fR3 zM#d_l^zOwQwjK#9#GG@3{8w@7&I~|I_k8{R3+FET64~o_W;DW1c@6;H{(xqFiJ+SpL2l!xycu9op70qo}&WiYFlE9lrp_mma& zh8b!68zvQqPca4iG~Sye+6~%R6_F7-F|W`pNI-XGobe?F*c^&Cl*bFh3!5%GwOf0F zBW2}ht2bZq7Ty;5OMZm2lPLSA$t4F`)cJj_rQvJ-61L_}H4_$AkHtW9B*L#KK z6nEiW=hMyBo7AqFE^E{FU9;>)iA*>?_YR-*m_tMkkJ-;owl7BqE7j7DuM;AE`d?ei zly^)$3f?C=cUk{=Y&#`_wREZ_{Q1uw_}|artFJ>sD@QheKL!c&=9@iqM?#bf0#dyf z8Oz^qFRJ*FR*u`^2U0u6uMWNJLh)pVZMB4GFlXNEV#a)xG;Oa0cRTdb^$JaDdjz3C zE#!ig@XK+ztF5fbjGe2|Y`n%*hz=%OTH&=ra@?+`)0NMwaS7EnnceO9c0>{;?G6@g zM|6Mf6)*W;>l)hv@&#InPxZ{`KRfIe; zp!fm*pe)kxQ-lV9`2>;~+Dz*Lv^B+5+ZGAyg~oOlh;!$* z^~{EBUH*Ztji$7_q-UvoRE9U-=R(!Gu!V;+QR5gWMYAB0NvqV)?VjGqSbpXM9l6Y9;WRNP4M zjCGHE%i%ZEz|nn>bvd%pOL)k@%3+gxi1ZTSr6dy~O0|mcJ9cbr?*^@>be&sW+Q zwBrbRxuW91e5j#oW>ebmCBcioF5A!tSU-8Rb#GN_(&w^T$J$%}Nm7{`5sm)bOlJ;; zvlbQXcNvV+k7e!G+tKxZbiDk%z32Eb6{~VeAF*?9E}_OaX5JO*+N(PAhx74WEQNIV z8hfdajcc-#NN}u)qpKg(RjdV-7!Nv(mak)dy07UwIx+b!V=Y$M?*_-5&rE8dBHi7e z^gpOo(hwKPSRo>%oR8={gD;^1g!QGJpD}QD$0Xo=XS^1*Lsp+4^WgF|nA576Z2%symq<#+_x_s46pZ+FZ8QW5H*+m7W?fu zAAN3n^OeZQNf`VhT>?f4RK$_?6E1e?3BXJfO$}V8#d}V}(ueJ$9MG(0Lf>phxRv_M ztTqZmr2h6mG{jRQN{YTu^;jz4dU~o%NY4lYl_!ec-cA>?H}Cn`AcKLWM-y*mX%_HF z$^7Upnq^sh_Z^Dg9!Zc~@0nsJ*$Oo@q|MPR2My?4B|A}-_AqtDD@U%1&8X1OUJ42d zS(#v2QFv+o4O&{-9K$+dYKWb;^4$Sj{aUN8c)@lEt>%I2E7>JAjXbT;Y-kZo*vZ93 z9dyg9NDS%CR{vSH(6MR%tDk2tq2Ko9OLMqE$EcZnG-1XJlmBPu!Of8Xw|wh-%lmNH zP`+Fd>g7^aKjIxSeHcZ?eS_CrV@hv~<Yf;*a!UqY?`35y?`m)u} ze}UeC+hqffe+1k}Egt6(>=-JGLf#)ny?3?z;FD0o0l%4L@P$xcR(_@|r5{m)+_K_m zh6h4y_HRl_$>c^BNo{9jB&x6;=z(_N@-wQ5HkQrXZkEE=F295en%+egX0cMk=SQQ^ zXg~57BkYs&q`%mnvPB$YZ@+@i78E)iOh$9+7jYL{x^g9HYknANWu56}3AJdRaF~@Q z>TqFNN$V=r&whzd^T5zEn3g1Ue_Qi##eJ>gn0WdWV;Z}7^XPW^&K1~rA!N_%yKFng z0yaVuoAXz9K0fuRM<{wdTIK79C)8F$Bvib?;rj;2HpY_{&^l+EJIw7_Wf|3*v2B5= z_7^WkYRWkG#M@X_g_jhXY{wcM*mS(z@*5!vqW2EQLzfTc{34pHtG1Xfe!KTZRAFkK zb?@O!1yL+lIu(P8i&0OCR2G{U1F@~gp4}@Af5AA~AS(U`>vt>8IJUM29l~K_LqkLT z3Kn^B$PpgBG}#w9&r1EB#6UyB&r0{AX_?)wn89dKFx3VWp{r%UDg8QXczC#7B`PQ# z_-U%|{#xO#)9rU%faxBF*_>sAY=0P7dVY2c@JF7WGOlV@lraFb0@UX6E+_Eq#t%Tv z%?Fs*)y}@3E{sB&PfW*Jr|PcC)k*ylq@OAIB53Q1J1D0tBN^*FxS>7jC$;Y!$>;5h z^dUS@UDe?Y%1@rR1$Dc-T;XmDs61b!aeLGY%d$(<*E-kYefgG~M=g|6Vcz!!Y)_o+ zJ6zU>H)r!YO}^BGj`}dW(U}P^nIUTPItRI86JjwLZ%GQDu5al`eG*pR)Ij%iK`@Pe z!0v}vmd_;KN1wV~-Se3ng+wUqn)(^VUlWZjSe0B`9#c-4rO+HoUCuT~*lWdof(Av= ztoE9OP+Cq!C8u`!n`CAHobR5cB(G>}=}34A_={tcL8juM2^HgVq`5N>4DlwVlLoq~ zEEhstJBN732XnRh?p#vIM`eBZh;5X8sHjm_64BS2T3R|1J02y91PM9;k1O9OyIAww zXq018Q-AdcljSpo5EDy2uUYxur@S-HA;;_KzAh?S;iVeq8?qmPUYUS)O5l`Ext|c6 zKwjSj{($}9A-0dKt4p(o@dM{br8HYY>V~TI$_cy{-;kRMpn6owk&1^nJvpf}olbfI zBqwArmzUFLwMX%MR^yL7PDNFjijq;(0%yf^BL@Yh6^tZEDEx>afOfgF4yDu}PcS_!QZ1Hq-b<2{$z?Qy5ePhC$B+e4H_ znu?K>BtLbnYQ2Dgo~7$#tx>f7EnF>%6SsO2p6R`09_ShlCXfK6SW?Z=W3v_bY#(Vq zw?|H#Rdu3}y#>^Dvdj&-K%!`b{LZd?g=WWac6&u|yKnd&ELZGkM^xzT#)F_+!Uj^N zHOVfTmHE0*N+Am#Ov_4`kDRf!n3x#l!yV1p?T(RcwGoEpT*&DnHN#dQwXm1ZGnXFK zfsw#;r^Rb!`)ce%+x+IimHY1d-*hROX7+B@mhpOj=Fk`F z{=&bPbf-bO2k-%kHCfA3(_S3z!6g>Uu_a6o0|l*gi_Z{eE+>HnHkPoD6bV9Q>B3fZ zJ_W0)C2#r$S8jBywh+up7b4%AjqZ$C!4PNL6Xi#-(4obmSq&kZ)v@wq1IGzziT%;Z z!Z)_)%L@*3jhhTXNcGH48{N1QI6#K|NjU-D>m5-BBt33>lRqFK7;jkPy609h%83I0 z?kAQHls*gfSz4k!&@LxZ^3!RdJw{Do(obriwwvrcS$8qp2vseyI%)bh7~j66M+~OD zkxL+&!@g@JpaCl?5KF|Q<1P9#ZgjrVU1?53`vbK$La=pi6CcfV0qMB-M+9;kjGoGeYw_vm*s3f*Ad}fU79GSc5dCIIkuMS^v&+7}$?)1o>p1OpS za)Xx6N6GT@@_^cqe9@v)HC_JN;w;J3!^45_bRo#uR^7VUX3G3ABq1yO?SB19jX+Uy z&Zg5;?GqTh-Ii>&4&nlLqfrLr2LvdFpC0vFyy+#cif8 z=C|MzN|)wp)_0sVzrgtsS;ezsO1$#XgoE<<7(d`k0oB6|QSR{)r`(=t-V4ts-W+xI zr%0(&a*XBy;E`Za>};Qi=oK8rw$wA_x=Y~UV|L@@fCYvi^yHtd7)#l#URvVHbmnk}|bW$qRUMuLHj2GqPLhDjP@j$viGyIGMELD3RE(Gw zf~w8AB$)5$(HXz&1#Ex5++-QYwf;n_s$J<>L^X*8%dL_JYXvW)Gy8i}r(E{zZIuE{ zGQlw`-BEx}qVNlQhwalnnuS+N$A8lYm5{ zj--59IsTFBr7~A(dLpmT5Z9_$e=|zt&b=X|QD{U!hXMYba)s99CcMw@h!u z8=$SqoDGJGqz~_`h6UOzbSRN2Mn$KNBS&a!oDM2py-Ir(Mk`_`Ys^#jDl>X^*(GFi z#Jv$z*V9NDsU_~isooUIl8vP$@=7zeM@zJ6vO<4pr%EvJ0^a-Tm}AeTMM!T=bs`mC z7F3dpG>6yqdU7UxT`H5GDiRf(k!b84*Tne!nMjC;SPKLe*u?R}@585xlL{!(-9NDR zRx4{ltfd-K9@2h^s?|jFwYY294XOEilw99+!6>tAyKF;fVk?(Xd^~-xUzPUeaeSyp zXhCIZqbfdqhg-vGB}zAvM;7YDCvZ${L}3|4)B-g?0-gUbsnosN;qd!=^8sXl zW;II`pECxR9fOzh2vQYKu`IUl*yJc;AQnIY{h5v-QK*j4tIu|HqJtZVQlEImJ4zS8U&%qceAwyXPL9P{sG;NQb2Gs`#;(bbs znW#dmz4h)I!UHel(7eFWo(=6pey&)pUkCBuS@myi-m?_Wh7xTm{nCENyoG-^8w$qR zV#2%SMbnm2*VOWgKwh>^ibojCuEgp^q9Ip~ego7@OjeNTg}6~* z1;D=O^70!NlAg#npuh`%*p=tQEW3FRZ+LZa}lA8>uDr0Lv4L@D1jM28NDrG99-Opr_A-fR4pH!8*h+mC7ITzvXl(u)>;JY$uDiGIAi@X z#Y-@RytAD@U;CSj5sjQgciZJ^ZJ5?6KoDfTZADWqX^GcrlVb8Z%eXy+l%B@T7f^sF z+F~S`ZRp#8eneDMUphAo;qPo$%tr-1d`1aruh9#GPhgT=I+oQ*WWo_TDM65~kGyGM$ zc7QR(JSeyNdFx!*uOh@Rd|S8UGpt?OO1HRv;e$Q{rp&_7tvSz=Uthkg$;JY($QAfc z;_RD0{)(C!V!}4uC9pmhwzwbWyzG`;&!OpL4T03y1WD0(RFXf(4y==m@G7Z0U=Ahs zrdJO?HMOof$~HWBb`Ep>7vDam2l)1Bj>=oyb3mv+9@iyTD9rZ4#k)~78O72`oi0|` zJIGCfy}X{PC>O(91a&q+GEOUw8xKDIb*1YtA=lYoMxGDZOZ)=%`W4U{xOa-_3ik6)sm2%Kd( zcdh=ZiOm^e(6FgWALuzODoK*=^0yWCm0uJbR38LByv&zPvh)~Y8IShxRrz`^rL!#C z!lw*j_aw@ptBh9Q>Ny1PpJmu0!-TD7nVfg-#G$Ep#aDlIeb4{=Ms7sUMEM|OEOYfJ zpLDTZ?8l{@TsQ2j5BZ2!4-|wAEh!2~&RwW~&QpQGd@FKdQ94)JO#W&l!X4Wbu zj*j(e`M@Q-1}2t}(`Cah(tdG{`a0{~zf7MM)SC3M=@riv20}64B^$0O&au98 zwX*?COwcmg^}Fber+M-nx<6z&-n~mX7s5YVwtfKc$`1#OF1Mds!wr%G6n5QRf!o-ym`^pJ#ix^2ez|@o_F?K(U_%ulZv+2hWrX2)pI$iRIwsm#c~t#N=8l&80O>5=xrO{KFFSh+uGrZu zCpOkOl-p}u8Veg8pB3gn+cqW6DgASd-fX0KpK4K1_2T~6yr2X%6#xHS-2V4>N1W^n z-Ly{o?&jlJ5Tcs`fetIf^R*sj?$P&kecex{_I9>CJTgBc7L40{QI9;X9Q!@{q9Y^m zAtVE|cyhAp+Ok={xSqVw8QZ;%`=PgNRroih9khN=d4%hp20s<{H0g+6t`O91uRBB2 z>Xf^a6H)WU0(dj-KGoXA36ClL{tWxelp9A+Mk_5p9A#LXV1P%7(|yj9A;Z%UKI>6wiD6Vs%uQscwm~f>l_-+XJ+!R9I$CV+5*4y1k_4!)JU07Shf64I z9{W*tWao&^={j<%$D#H@dSYry3c|SIBnfbUKUF#`$=01fy!osr7chi&TwrO6?D6Oh zS=o-v<_3ptC((NB$D&t8WF&lpQ(9VBe~&pE9G88xt8a!iwB4knTa01vOZih|YQF{^z3iJ9uzX&vXH_~IkV4hRM<`#n=<1(M2xjI{+!ZjQjrlS88=gjwabl2{Rv zoJ9p0MTC4`dxYi|vf0ndMs0lTON>KQyY?v!^--5kC<0@nm_h5|$oW}Ol8w#H9&pr6=q`vrp)+4>H zS*5Ia_V)2X=E+?lA>Y&I@P_ZoLb(XhXsE;b8=LaJt z&sz#=QRj|PljUciuMd=lAG~}a5zyewsfzo#^V_=CwT~w!CwaUJRvQQ*<))L~2F{yi zhdxtJ+oKDLO;|6?=`$gR0nU*aX2oKM*tUd1K)~M8YKy++`Q}}}Ue_j9rt5Y8JqXrX|rAw(KlPPB>T%p(5 z^~?dYJ@Cxsrc%5>kZ%$428Ui@*}?a1w=Ce}8mAo>DmGs0&e%*c`K$^F`IEKqFrx%pD5f`<>C7_h(7>CVqmsOdd-Oq8saDdfuB3 zb~t4uBH?1zmg=J<1G$>%UU?&{3+^SW8m02a;kB`*4z{TKCA`pJTEUlCo$&G)Um{wG zH=CP-eDJr`{J;yTy?;Aj-z9JLX^xhc`#v(d2{mGS9izBt@I93eIB*iM7{2Kr6o4o@ zz~CGb?t`Zsqbpq#gXp$x#Lqf0514BruS+ce7azp%)?aJaq_RYhS7^cFNOo(Z)2Q!Q zR;>~`+kFo0nBu&J1_z6(yw?n47mECK9 z=k-$~+s^>-$damVHCjDz{mwN&qdz^hWnGTMHQmynwnTsM@Gu)ynAtYN7d=FeS*6hM z+i_DKHwB@cOX}*x%0pyci;^VC$&ap_x$Oy3&z*+s-%d{IoV3{{VvOCf#+8gK9ZEep zb~XDoSF|vi2-u~&_6!$lPZrdL9Cp6;m^3snfQb#y3WYE0?%eYm7*S3U>xkVf-*h_0 zqZU2P1-+xxSsEfrwedwO!GmuR$X|k@5Dp17e7Z^E*s)@c;XNx&K|?*MXS)X%Z)^JV z2pPw`qnQYT85-^`ThTBoJAKkLEv>8jL=bVOJC zplvtg3{2=vMwRKEy(2l4Y_Ls5MEoi4J{GmE`|M_rP5!T_12Fczx&cY zezia*e4Gvf6A>Z2fzGVj|5@a{EXdPukaYFp{14k0Tr0Ab!9~--0>ck%k(?^PzE5*g zyRpf@ajFYaph0Tfh#K2Zx7`^g2*2>j7rCpon5~hGb5!lcgY(wi-Q5nr7VqFIuNmgeTm!CT3z(Z21L=>xL&WL4`I;Ki|hTG zZ>XS|kFg%LF{FH{-eX#NB5{dBJ%eJ*!pIe3(+C-nY-3%0AQ#mqYkI@Qa;${W(9kf# zF6$27hu7cVHV0dp9r*Lw&9$fs`t0@}dAXR14*4;X!vIUX_$%$+5MN;-kdeV-*dO_E z9P8=T+JGmmj&R3?w2&e{@+Kf7`i7)Ak|dDRAS-$ruE5*49?k^R_Qzd@wJ&)=&nLJA zeaBke9>?0$`n1<&AT;7_inXe%lsJ;Gw7K$mf3uW$N!-*vIj{-DtlFJP`R zYC1+bMbcKei;(cjs~(3`c^v1>mp>HijBl|xQ)_H#U*KFBuV}t9*u7D4>*7=3VP)ky zC@UU4-KR9p!VjSpNmE0T(M6p4TzL|Pj#%w4k&|H4E>7!N;c+Ftaw$pIs=_PEk1ux) zVd{A#@JJ5GU*PFtbhzOqCE5gE{qnQ2W%jCPuMawr==Y`qwGL?4qWYQZjx?gLC-rhf zWDj7B#=#V&Ll!*82EU}Jj~;I?g=J4*@s4{W`U1 zr9llaC81yLfoa7NmDB3od2hfN!vv7t@FYDJGQ3ajOV zFmy0=c7=SLaikdV>PB09*|#X@9GrWlu&K-QmdNti z1%A;s+*sby5voeZ`C2R(LOP^u$Rt-YSKdY?;DX2FMLB6tR0~7^mThpsDC|PBiLS~^> z1`BNC9coO+Bt(UIjroW2Z>&0+O=dwS_|y$HK7UP+^Ig4&hefN7B}T#597@`=ERi*NI(4Ps7nODuN8wT9 z{xWD=&NxR8IJm|m2RV9jW!1T-jAi!6Hv|*I5o!=&RVVd3xFK%WvdYf~&oKvm za4!4ub8fbDS!&E+G7IIAsGL1vV)IL|6RWaop7`E>vk@mR@v4wH>m)*T078RE z>)V-P2Nb&*?Ak25vn=OJyC1}wnQDt8yK4+g4 zk)KZm|9AE|2TV$irhI(xXF5zrg-Js%@Z3J{!1V7`*$80)(-%7>!@@e3Dr0#a%VMZc zTDcnpe*r$kr_F&S`H4BXuiN>LVP9>*#zcZ;7ymc^Dseg_#>yP2dFS@IL=+^JUiS!O zu}%=QlNpDP1`5Vw+G1BgT5GB5vh==oYT7y}^E-L&pL6@2Xm2rNKp2ynVEZqMo%2`$C8Iv+FxjnJU>?Dx}V4R))Xy*WKkS--bzB(n>I$D5$-) z_(~ACm*pg?G{qYrlVh0Flup?b(3zOtorSwI@$2l#%nwO6t@;3i8GC#Aerc=;Y615@ z=lf4Amoc#fn0u4|8Ov)y>^-bQ?}-T0U$wV(mWjmPU{%xAh?LWoviHTeU5w<$-l2I~ zsXpsZv^d5j7FrNZQZv5dq~mVKbRSaPAE+_qY)%z4all_Qydx-d%OzZ%g6&b~<5)UR zcR?HWb9N>Fx9xE;vAmb}zWiq_({lf;iw=z!B(!$$JW^^yhEAQ zL-hfhPfH2fv@yuaJ=iJR>c?R-=>)N{{ku8(#n*!-PgyLljJ?9y2;q12j*YbP#;Rno zWB>b-IAOv#VDrZPGmO_vU`d=cTzPhNH~0ovpC!1y^*%svP@B9u2>z(g?8)IlQX@Yi zzq6J{qJ(v_yHM~N%k9R+?lUDaak049QG5yGkNGv1AK~(yyf`&E%)M_QiFG2sqIDkK z{ddsNzbg_j_g4R9GJu_7)e^%$WK&=;(jph6)Nq%Rn=~0Oo zTcrO!MS3$8L@`L77u>158(J6t0#MS0W?~Sw%v=(mbLxlRV-dW)(5*hzUOLEM|AEl? zMp9a1fE|SSTj{q|&6jHQPB(UxwGq7LByJm{{3VvFU6J_+7MLqKMVu#Ah+NS9}t-3A84UsmxiT<9hb5tFn< zain$XmT7}rGem4Kj`+5KWcx7c>;79<94G>1{u;>b{qCc%)Y4-#i1xpJUtVR`sd2yB zZ2K98Z&D;fnJ;(jB0j;Qs&u7j@|xH37+1Hc&D* zp+KsnJGQ;KnW1Uu6ui)(EelHP#URIDt6S#~auW)V;?S3ab|sdUjby!WGjFr6dH#(| z2sE2673JjQbgo&y+=Q}U=GOtMvp36&P5WPZSGwqM%x!Xs8?AxoKUu3kU@O zU;4))VY~V|JT=uQ&vNjKdcbIzrQG(x=p?Wg#azj*v4S28kPyHIbBC)Cs4sS>$4=@< zo1ih}(u0ZdWC^O@t_zq|Sg-AiHbEMQ=MLqm7dbYs^7r{%zG2!LevRVRHQ=dW5@gu!K627F>}5ySty;)1bmes&dnJ*_38z8}vjJbwmypSHl6$DuvM-4{j!2 zoB)Y9Tcf28!2dQbVF1T`jUt>0S9Ql(3{Fz)>S}(m8vSd#Bz#%RHR`C_p%T!R*woTM zXWQyP<-7ZO$*zKxt7TN3+|bN$Bp|5dIjP*&t(E_Q!gw2adScCPw*uPerEI+suKw*{ zT~mPNlo+9`z@8RAya~MY={E)M-fbOSu%8f8EqeLGlyqpjH3Ib60iK_+Yqi@Bv)gFu zDhxVx<11%{S>H3jM$oj(%-0uEYYk@#M80^zV%e#>)SJFIX&`FGF|>_d6*3%rds@K~ zbe(xOJ|@Ogzo{!>)K>6)5VALC=qm~RZ#Dr34n>9(@X*2mxC?~%Pa2EZI(zWi?UVDX`g zB_AtmC=FC!6+>E)S|%bAeeN2p*=+*K#*capD2NOsLWnQ1%DZWC-ym}DGr2E%_=qe) z&aUmNfQx7)AV+DuKYBc~9mW8FnG);7pGPX7<}9i+b5ll(UwP03phciptlH|Rv$Nd` z4-e>~JCxtoW8yO5R^@lk&*TPmYEtLcw!&l& z$cPH6^0}X8>-ho9KXr@#-ZvukrJLY!7KCdh#P2dSl$dEb=0KVJH1!*O0tW$e0Fn}T z1W<-fN9^Zab9MfNk59@X`DEx4vqGi<-trRnLTFQfB5*nY8f@l5q|Ud@XhUaT0LddE zoHNnoZttap^mq4XZoqjBUIB`Ba{1(2AGgGu>#ok5KAEg`XOFH=U(W_=?mN;nD>De` z(4yBbJe>DdYk3TSZvezEiO$Q4G?m9-GW=?vH`RS$IiTv4rNFRyvNxHuNuJRgTPpM| z6AE{aPm^HvA_nSf#zr3NN$(6L_!qKQI{dUA z8iKBGAGs4>#%kxb6XeFWlM9Z-s$^t>;dM)0t%-OaII~Th?ip7Nhju1fzmvKpl;8)9 z)tC9UW*f1KoaFBm>U`0;7EXLLcb9tr#|k({1d|>!UCkjCuS1~jYs5r&07_$h<d& zG-xxDP=aGc>$Y+IvZ_K*QvOc_BSenwk?Y{4Joyyw>dTGLFP>u;Po}0NgReC~gRl-Q z%O_=jXa~f!Lf}s>aYdmd{bU5c>ADUQeyssboJ(8}%mILfbrKb4O-F!*jOUH^@i?l_ zVU%x~#^2PwH_dzkT`~kJTo6l9^R5PSS_wu&&sKK0@%*D-#4t4d+f{1>D;q&&cneVH z*CxC0lPe|)+U76GKxb6pYaGA@rqA~F9K0T4twwoeQ@SqNjDdtptMGM^8Fz-Rr=mq9 zW0L>zI1tQVsF6)+4T!I&x0Je@*|Z}B^W=IW?>cyMygw+yGv{e`$LNY)@68>x>bDPx zA)-dEgWz~mxVtC`|(hTBpsA~8H#(l=oyeL4LvkIrU>sMDq_&yF0 zW@P|x=M`K##N>XrL%3tjxCglwhH%Ujd?z+w-&Og=>j=ii{sKc5?T}|+B)|xFMtvCy zRL;KU8jiT+>19}xIQ)8EUHQ-9h7vP`+0^M1-Do**vQzD|SE4#*&+8-NSFU}-#bNv` zZ1r^o9wyGu)JlSCKD9ws2l=gxH2E_Mx`s+iV;> zzRHVt?{}cPwid@8eccK94I>?y~c{_z=zqUP>p$vv-a1^~Yo6!ShKH2mIK2 zdK}a<_`E^bEA0DlrlLoU#eJD7xpt3tY|vJN^Jl+qsF9V(N4 z*Ez5$=sh#rg+Qv>J$g~%XLpk^nSuUsGt?=p^-{6=czyS!jdpvr48@NX>ws6(Sd4Z0 zOG*{3y9Nz%z7EeVL;`9t4}F9iM<;|O2BzBp&7Kvnt!?<^8pZX@*lT7kTiqP;p%5!N zCe&1VF=xMHWX|gmc19vE9^ouEbYEdsdo~yAB`0GVML_ZmMLIz7b8^RjQ^iP6|Cp)4 zuuUt-jaYq0TN<<|CKDOj(jI*YhGQgPKGOSv=f^|l(3lvwBR})phEc46r_owYHX=+s z7Sg_Wjo^a09bpiXor7hwMKc@93)gPBLNb&Kb@G*S(QSv4_A|ZCoJWX-5Qn{*xkR&> zdq+UNrbQFM_!WK&&P^^J?Lgd&tjLdO$uw>9z}0V?hByU_-*}Vz%8$&|5`iQnD74`q z7;HN-V2IFLktHBFLWmK&yQQ)jvlT#I0%4qNd9`$Dn%?1pbC zJW5@ol9DcUI75zJi$fZ#RR&!1>ml1iS(0_VSPZoyJ`P|n!Lg|Yy zdP5>l^Zk)|ori4Zh^_B=_?Mt;&Wpd3CmChxu zKH7F~^+hYga{&J}40Qk#(AiyC?3qC{MHkI~2CqDf4luI~fFyH@E-gh$0c==S)RTkw z`Ck-ZRd`(F;*NV={PhseH7k<3zfd#)eniJfCc8_%=F~$PdFwwwSG`n{-aOaNYGFv0 z*%RuaHamA?7SCE{*LyYFb11vFXFYc8M*m-c95n_KNT_@)_vg5D3V2pa6A3i#OElps z&L%ugHi@*_59W0RgLx@FO-}m1Wchy} zfG7_Bfi;r}Pw7$N!%ulP0DmmSbflSzdo&@OgVFdaiNjwpE-bHVKn2;UerCkV`6t4t zhmW%=HiPAS@{*#KH$2Kc@(tDr8wED=pA-8XnK<2V6k_(NC(fTpmUtHi`$H{bG5s}! zmfs+XD|OO;_$#5a7yxdQO&t5L0H6&vSPa`oOi7pivlzdTyCur{gTKaVgNyl~#Qmfz z|M_48X2hR_^-F*MAU@`UQoKo5|MNiv=7ZX&0>Xc;B4#zj*8te;yq0w_^1QGfOv@(H z6L7HZ{ifh7$xzvrunYXnNdcT?jsi2dzev~l-q|3MijNrDSQgh4ia$YF%z}H}#u#`8 zCpY{3IVZY3j2ppTKfe55G2&2L%$x+#0^EPjX^s-76?=ycSou;wekQ`q@$t_&cmGCD z!O&j^!3pXH8{a_wjH|^5Z`7?nMt)F%IEk}iN*w9K>rMYxq)!inIvQSX6S(-FAZ##q z+7{vC&A;xBd;+js1q3GXcz@nXHvC==w3R{PXK#(xIN0U?igRJ%X!>EiI)V67qr{X{ z2pC}`2;YFYnS+ub0C0q;MQsA-H&rKAQm(^63j-XO0^5<+pMYxlfb;#jmN)7VcH>T{j zMRY=KL+B9%6i6QcFq30gK-^kNO+)hnGiaV_&`YCPMNt-JW}UGsz?lcAAv29+Li%o9 zgI`tyTDZ>AFY~rUXhDu(HweeB#kws>(z7H9(4XOB(EY@s1j6g=vP4w<+(YQQ>A$6w zKLAA^s)fnEftvHkO`{bB;GHL_(!0QRQ|OviK=*CK8l7uG9*}Q(V*~^QgM6U&D7|$% z0w%lwupwZiXoNm^AXZ-Ln*XenkcqJv3EZ>}^kUtjoP+&}_Oz3D&BnX`QJR_$wAp#V zhd>jvq;0}oy+HU#FNf-Gh2OqwAYdaIyG?xb0t9STH8nMG5`g#s0jD~1quy9=ZC^}m zF?fZ^JReEb8vDyLj(C~1X?MZkvMw| zls>N{*~|~4Qdwl%p-xt`D3rvoTms4=U4C9JF=*>bKuCE33Tw?Ul2dcA8lL)(GI2ql z@_L4?m}Z__qZKJI4?U| zqK14k&R3L>wyu}_*eM@nb^X*}rd+i~&;InzF*EsZIraWf0-}naxVxYXV5Yn#I zF!C3mrIkzOywmJ5s(lEEUElo{SKeMa%0B3DlUov2@X%~P+03^+xg#c~j_l1WNjN^a z999CIcXQUsPFxBc$hB*fZl@6^CUSpFS$PGX?FO`IwXhz$I zx5$Ex^P<uaG{-L^JDePbk~;8xF_xzT6a70UO+QOSB^LXhYT`s491XbeVn ziM-Ggm!^_5kLWR}aASx~@I~IalhLR<|1Tb$O$fL>KsGX(ODvbsjuiFaoa zn6Cve-`mYg@1zvf(T)@Cc*qM-n?rURrQCy#(Y)$Zb)uvx$MuBU6nPR2(t*jMF6fam zTREJqDUYLDMEh>6_qDz@V&s#XVun4VrBtRXhU6NQ;edOW@ct5!>F~zG64;)>GM9Y! z7KX{=nppR|2m@}euh%63wJjB#m+~f{z!6HsTeb|#bq zH{6^1r(;O7_Q6#iH2CU#^li+#$`8}-OyWaO257mb8(nns@@d`amUTYr4iH5?#(>no z$}l-Wuk4|R@`E57-PH2Q-uM5fqWmIAcIR!kC+!kH+6f(xk3p%U4P9lA(7wGGDIwl& zoYgO7eWB9KJ9@~mveiP4RJ;BU#HX|;e9u`sV}bG%M0AYUTblNg;4xWwyY_1A@@rA6 zMQOkwwD_gHc56%y&nK+mVj)t(bRn+-1JT0WkMl`cf$t>o(e99a29=AR3WisSf#9Q6 z$>2@FG?H~EeBK$zcq8y^b3Oh!Ce0HL(@kxdELD&RchPa31gbClTj?)^fbdE09|0sL zY>>)7NLgHb2=tG@xZI{fEGmpt!QE$Tz*-VLQv#@6CRdXtSXwT~PMhE0nwo z)}MS0KcFqMh9(@lr;{1X0G1B;d?IfmH=RI}KX9P?^3%RkO3_2>PtPrZk>`@v(OQPs zg^)C$l+2EN^K0(mZ$ctKBQ#cOde)ibr(b!gt|0J)EHa;ciS28sSx*)74NS6Mwv#?= zD;0e`{UBDrsnT+@W3yn-&Fe>Zz{<8^2Kh@^i4&q}XV{3O_jx|5R3376f_k#87@sz` zeUDSj=v`~-{w-!Ng0XD5*$e#uy4=6T7e4rUO3H{*_>cr6D~x^cCdXG0gLTPD?y6cK z+WL(1a2JCF>d%1ZYs`_;Zmd0<@oiBXx}bSi*(H0(OeJz>hTcnmY799 z4gqvg4X+3$-;mo1$;lVLw6o4&0Gv6P%T3T*6yv{eQ(*dHYS>3)xE@RxI7_=Re}1$7 zv8cr33|rS$BU=qWymoYb+*hmmRR!kURLhJ3*YBr_X(Re%zq@RB88A=ZhpbQBFxQu^ zFge{lIBG&k1w>;CLuW?^!rK;C;k;y@Fm)4hirZa6^NBS_U)r^zBCgh&)R;}v^_WCs z-WSA9_IbbPly>zaq;c9sYn*N9J@ivM*9N;&wf9d?tB^{fFf&BTZoY1vpS^rf>gvUX zaknAM{xmrTaMY+}L1&w1IPKYXX-_JubEwDUZ2iciXF*9f2ub-~!+G+}p0~q53GrSY+c&3W3k6lC&qcO+l1+%6A*nqgdjgx)n88* zJYD7#<9kSIW(#91odU!_3~L(j4aaKxDgyy~Ir-ar_YCYj^{!a%Da?&MvrW?BVt1vP zI)#GlvGvLuYWvmEFkfP7CEaU4j}%44H+O^FF-A~9F66XCtUMrOqf}IxNw65O1DAZ) z^p)Uxz$Mq~YwsR;f=!bm;1EWHOg?NiO;I;r?3t*vPXSc9d;mz>zT7kPAbuD-v2#lZ zC?aXAjmaXUGW)wztL&D<%cqiNVF z_SR8ReSP1sbO@p#Ad(8wp>&6Ii%6$*qjU@5j(%sUaGz>!{Idt~`!;sIx>%Q-G zb6vk@z3X|`V(|yenzPS7d+)RNr@oR338X5|KR9&A3t5{;_F+hOzQN-+3GJw(WGaGI^Ps0j0G^pYl!Y7x^}5M@X$Y|w*y1)%xKO8(5VO6<=+ zbiOMzM^t^?qe&`J`^iFkTEg{9F*@W&x+!XC_qeY#mRy~~-vE+hJEACW9y&WK1oKo~ zbH-oPqo%sc9QVaflfGrM4;eI@R$<#Q=6_g;7Qz|lAC$o!6ywN< z%m#tg=^;m(S7Z@H87w6wbZxl!8nNDW#3`lbM(Ow|wM*3{>o1ui``RB1R8Eu892utw z^As3NU2*>j0rywiiM2|{d)HO=r()V2UlYuK+~K}ONn1;oaKdS5i$ZeaHmWvt&wZJc zYt2N#0_PzUzPbeQRb$S}8>k{>ki$J{F;&TIw7%L=0q z?&Vnga;9!3%U;J&;NNpi1J{-3HcU$HNVj`2by}g)jyCE>adpchz`a1cLzyo?!aX@_ z_Wjxvx>m+iIXE&gyv?0CaLpiT4C8A2{Fs;o$hSHjQ)+KsQM9?}ukzFVrhGYtM_BVLo=qKnKg9w1#aZ(01jUqvuK5cs08-_ti*&)XQ~e`27X)K`9O z^7@ruEETksAIdrOpHi5}H&~*gki={3@_T^5A}4|GC+}oG9T9R=ryUu{r+~+!kWUzk zC;yJ&JV4%x1viE^;A#YQE8grN7>c6th z0nV#PepIT={6`w&*!Q}bB-K~hd3IxoglwPWbNPgLSmE1J_8Pkpvx}W#S@R#byp}k~ zt--2VWFh;{>=CA4z#O7I@u;x{Hi6|n7!jVGX}ls1uXIns<3#|r3%OLdg* zQjbui!-pe38V^~~TFxMzS}!pOzgAokUAfmu|HAOvX^}8Dxh=CSb`H}a z1wM!MA3%X(71|p-23rF94;2seytGueCtB<)vUqB(#jA-}bGfW}PzMfQ*y8g_=5KZ$ ztvIb7zTkI1Hc1WZgtIHX`-6LLyxcWnk(uUh2<$rj#JSg7eRPj?1Vx;;Y1W2f60n~# zzHNZ5%X^>lE`NVQuUf<##~K@menR=w=65a&Tb2SA!lLhZqJtecnxH<~ot5@B_FR3CB~X8^(>3vTDuVia65{^YuYKe${0OC909j!W5^7#>$rDnpTU8u5T2vJty?O+~Y`9tjY zd~(%8`tw;s=W#{9UO5%G$F!k4Xxj)dcmE_VS|6)WgVzKH!yZ8etMDbFvYkQ_9HBq$ zxX9Pw(>+S>3GnaDxI54k`0>T}^(~_@{5Rb4C1%hSiBMnD(12RWpnyp>3+W{6J1l3! zHJ&B-sR(N%2{d^bU(Ws3?QD#nWApoQrsvg*#;YVu-hD$GC;uC1ToTxJtjdv9vn|G~n_!^GL|yv&qUfiXyL}s)KAXlVrs9YZS9) z;N-B7IA&AUaAzis!={lp1VXq59v094;|4^ZQvAQo; zE6bWE3&m3%M4K+2>H%g#0qZ%bVhvUUoClT-N~(%a7d)~GCTJzrYX!;w>`qsy23M`Q z@xg^^Ojj;=wV8b*|6%4o0M@IO=l_P!A5u~Ee>{n$Hb;&t{t)0|Gym4pN zdqJb|Xdzx$Lv!px3z}sdAE+icr1>8wOC3s;CJb(DI(Z<;nKpYiz%erNv49Z3%E6Jmd}>?69Y7th0W()~<)s)r8QF%(e0WKkaWZu*cCPitYrI7i(tP9|-i((<0OOo7_B8Ge)g$h910B)3gnU#Vx5 z*%8G9I=C?q`wzePd`a)(H04~8W^cSNk~PjDI?8zGBIv^ulC1f6o~r$-6>*?EbiITQi7 z_Zs)F@|uw{Uh%7b@rOZM_iMdOZ=e;)-LvJ1qN*$_LNBs&YkcO9UrPzCCF8PF= zsl<400_X9!uyr=h>t=hXx2rg|$h?!Wb>mN2>azws7R_@5Z0pZ>Oq!VwPWQ$FT5s}5 zf%voj`Nfqn4TBp>mUZh2Lr5~wjd<=>u%o6po**Oxc0E?Sp0q?R>i$07dW1+HFvj;onsD3>G|7`J?L zu3qmuZoSTlZ09P1Jx&oxF8gz9O+!rOHj<^~Ezy?^#Nc!esH@j>m1)jLaqq11)_o~| zbUc_|#)rd>OZ(w?me~T=gA9wyN8Tb85T|L=g!i-DQ_0qzle-PV*3|e&HmgBMC#O?- zTM=U7UQ=|0&6h*PHI6;3V*u`G3)eFbB}y0u@7ZM@G@WGwZ=~}wWyWU$9#45{3|o4%dFyfoY#$Ow+-}E7Hy^bs&DEHesbw6QaTRr z-eEf&ToLcyV9}|&D68J;3fl@-s&^29|M0mV3eC^#@TF9GyWKDJu%NZ#qr<}gyLwCW4aaVtqblGWOwQYD@`TSb(cXK%Up zWq(}Qw_XIHKO&Q(mKK0K3r3S7BwDFx0PIPj^U0{qYhUP>AK;?=?TtWSB3=RP&r_}rH|fVY`}4D`%#MBH*>4h z`g?DWUgIt6ZeI;r{WyKkQ@)FE zuk_cS^JehC$!tZn}ZA> z_-5>jfZA+coe(Q%6~wq(=|m@uuhl-|z;f=V>^Qu~V5>KZx^*YE4Ki_l5UM*XpHOpl7}MSj{mluOPV!(|px5P%T;Ly>=MWm)-$ z!swIJ1j;JNyyb_*);{~(d7tx;o1066F$e}?y4}9j6AzW`6t|AT~8LIt29jXIi8xS#Qw_d36P_l38&E?z&y4qhr-g81NDJB zM<SS$L!zr5JiQ(e zN13BO3!_SQ;?Z2N0%-Y|+%vBRWQMx>}~xoHxc*`D`U%wL3co0H-t<8MY`yaK*7Li+7nrD)Q&zL;BX zpHH?Yj@CD2Hgi7#2X-%B@(pDfaPLZl{c*WIW`kABj^Si{gAqM6-mi5V9BOeBwi>O= zGQrI3{f1bicvHZt1XROh{3HUYI`&UW3s565;RwE*20*XOK5)QQVkjgK?-Gw*rlQed zO-``Fg2;A$-Jm^2WP#k?9)hHD<{ogH`TZP3_;BH6>VV?v$A?I0Set64+Fe1m+!FFn z6!sUX0#N;IQL{TYn940IZ`?wCKW#?dQpv?J#>srCfBAamG0WPwwX&i5u};}Bi)n{j zKl&p?$bfbthQiUb)Z<)-+z=<$P&?p*K$xr>9tJs`LbBWdGb;3la6?q=Vla+ApT);_ z2Du??g)V!*vY5hxSv%V8d9dHvaZ^b4>4Xv#bD>>lVOF*U9AI^@$YdAKuJ~)W4z@UY z<00^QM~pCYKk+XQxT6KHDES<|UuuEQL?SBzm$RT7475`HB%g)zec7HYh_bG8>Vc+4 zf%QTO7P)}OmvtEtX`BTow3Avh61c*g_XN^Jk$f_ne#x_m5-srDRPL3q0%Uc3A^Cvt zRc5^zm_HxwBQjkEpO$;JhhV1qSAhn*M`YNjUnY>8=}GD|E3-d@9(1k4+QG4codKRZ zn^pLUNf3zWG4qev>pw7vNW>e{<;5#@{@ymzrTs?aniZGMq}cMhudusV6aFhw?uZPn z?+`!``Er0OdAlZqld@t63mo7)U#oGrIMp?oUc3sILXTq6t4ckiJh}B^egE_I?Y9Yb z;lPC$dKPENyFE4k99Kr)924uZSS^#*V<+?Erdja&qruCkj=1Cx43F~+$ikx`N$toU zTU8%wMD3Cx@O1V~rofl}NG^E6fn_c=D8 z)}3iSO9T>)U%+^LwrhMW=NsGGA7+p{?y&GA^M+#*aHGp{+Pq7kSqF^unBK%@FQm{* zmDarDA5@3Ybks|R#y@-0-7Ok?amto?g*YBzK69V4yY;VpNW^TIu9K$#gtZ+M-j1hZ zwD(^JVM+q?unhuuM>tWeY;i=1k7U#P@;{}HHUu6XI5wq}ZES?bb#JWnL|m3t?4xa{ z09Axv`Ue4@ge0)B_2`#z3YY7z_`HkarAMKD#Y*#8j#PL6YTn=qgv{0vX28~$+&H03 zO6eE%6K*w~b){Wq5P<}_c>0f^x+f`MK;VMch}UU`&>xY)f8jLWh`S4WOM$v7f^N{H zuE?m@_=6fvCQ z)jzv5;i|0l&m$JV!n48PkhY29bn&suFC-N7Y8y}7lPa*xsl*U{M^?L^f>w|?WFov_~(OlzeikGLLI>dFgCnYUZM7FLC zVtK9qX62IX<~RN%WGUsXS!T5_D+=~%awMB24wFXMxaGyPA4zLltZcL+UqWJE_eIau zx5`EGk!H=nueLE`Unz=(L)m6%6J6uS-KR2_`{n+m>BEvdtO$okslY?$OFzg)x=eAX)v0!@8r9(Eybgv-Y4gl-t=8CZsUcL{ z%)>Ii3#rdGRGK<-F3`-q2gE0K}@5t?AVgC@#uzSZA5 zLNAJmeh`jWRbiH#uKw>6$-cm_Q23d_)2D(cYL*M%CMI){{8Ur2tL%a8y{$HIl;@S^ z%W(*o|6~@~*jA~VZ&~rbPV&LvUk57#aZG+yrr_tylQ+Z2;HJs7ou9dg&a?AA0;lug zC;Y!SI&+4Y&C9BoTWxrP!}-UO27{u>u=v0N!;+#RR@+lVfqz8fUcB;)P~o9j zAbfSN)&F(y7XtuL2o*UN&pot@Y7JFl0VeaLD>;s2lTb}`ie$4fwn4j;mA!n8sAT)A zw%UI|aMI5vbbm~6ST(GpsH2&y335>;tnn*y%^|ZPsVS2*`87_FRLY4f-#wn-QtXk& zW8d|dx^>L;b?ni86_)xCWjB~&>DI-h#z+9}junUM1glPK^tVTpie8u3NGwe{mMM<*wS2cgByRGb3 zwf&iwU{CEbj^)EdciUei+Yx!hJlVtMMQ>P6AJ6ZPu@PoW4I?Vp~)le?#Y*@8)V|0#5#N@ShLP;@Dc zF!aZ>1PY*}p`FCiPN<%`f%CsjI7)0+M!C;JDrfLnCP3{)?`%+|%4A`mwaUgmC1h7)rltP^&y=)WDmq{H=J>T?{GGF(1D< z8K6Ar-#N&yz*SOSRgl5d`gL7_M6HAW3i5WEY5SXmTIRnPko&*ck?-bg7V#3wJ#ZJ8 zv-Cj}4B3Cc-yUuT8e?sW`Ek5!TQAit8{71sV=q*W;pC(#?lS+CydIFM3F9tNC*3c@ z9#t92oFo7Ah)2&zBx;Q#(Fz%KDsiNJfNo`(T(fL9OIntT2leDqtSyH`TyKHvMX*mW zcv$4%-{Oif<_wlNhRKoixaGP2KSqn>-DufX;=KV4R7(qe;B}$)eC8>*$6hUy2i7sI2&S`DW%r-^^dVhV%bBsl zAQ->PqW@f^mea1uddiL;{^j9TF+@1K&bAED1R{LDJ-Q9eQ>GwwB|q9fzmmVrxj`|i zRq^aDRB>-Q5H)x!9YKMWs&sbx^CWe-T{w$&``d8Z;eWa-9-y+L-p#9Q(k0aU^XjW_ zLEztc#h>vO?WDprGZ3q{*`5rs&Z131SPtrv%X`t%=z>Moqm_e!zCB_lM<7@u_+-d2 z9@6s#!H-SYFDao+p8zFeF=_;xBon{S0;o0qe3d0F2(X=^mDs|Y zPmM2aWU_5;J&il*3$&NAEn`Nz9b>z;?627ZG-{t4_u^eX04kP?^Xl^RTeiZF*B(xe zO?5Rk2yE(1k<(6UPVZkP{Pj}K9)~wg4D#{JPRpHG+Y|Ye*8+~*acmm#{I1Pma-a14 zZ^-0;bi2%NxZ5N9LY|xPE<7G}#)E%koo>Y;-wbX)A3~xy=;G-0;?>#wefP150{&qs zeJlpdY}Cgl_sFot-4|5bIuaq;t9F=#!=_m9SlTDA4}QYEwZYXx8s*wz$E+vm%$da% zNn|=lYoV4C0x<-oCVGp}4tcG5f)DlH+q9@aEL=>pqG9^-{nihiJ$2)EI42w*4&Ef= z{b^^2vB=zyqqRY0$&JR@d4&1tLK3F!D3RXqY9npE&jM5Ec97Z$Dny>+;I#*KEl_SV z*xY7f%LhQ{=s`hB_Ct;5TpnaCXOjZsRg#WYJ}bCDI==2jnYF9cyEfK12EF92qmL(@ zIe5by%ED60LU8;r3Xb%dc_$RWn;8cPr=)Yf;r>t-xlL|7-Bg^eGRx_4hxSl-w^su~ z*1c@>u$b!3=HDZp=#8l_`38Q<>&g5>zkPCf?Cb08a>CJkzv3Ytav43v+_T!=`Nxcz zfnZTQ#(2M(51Nw>;Of$Ml`jusgZF=K2#>mE3^WZxL0bslc2Ef8hoZsDRHM&cam64E zg9|cV3?3%wLc!Q#xfj*Co|)s%^oI1z(g)0EIflg{uQJFRMuGR2mF5OBY*Ra z0G$*uP$Z7?XSWATls@D2L^IJtgOz-3vD6rML1Y})C2o+eBdW>Jp6c?z4%3EGv;GPU&P^y$u zw&qy++=}Zpo@SD$_NW44exqoCd@17_sn;fJCF&6&gN_Yw94Ed9$c;w+|Fr9P2W$w% zEf?=I=BZq$5|($HC%eh(r;M2cI8KBwS853nEgN9Bx`rP|#!@5C)BRX$UDu~KlM|*l z!P<33N1)tk*0quM;nB>Q8NZP)I-LQqBAVPP#dHp@e15>x`n*EnLzOto%*hI}xsL8~OWUDNcgYQ7jjW2K_C$4S9mu!6I zbTe|bRpB8e{a`{$B2|me1AAzM*P0JtTSEsi`cGqDT-DOIXy)ebO*C7p{n6kvfCcY0 z-FgPkK*zO<w&$Q1V)#>TfqO{-pc=<-a64d5Y>303du%TsGH?MKSQp~NL+z=HuT)~cGw}wnN_}q zQoEW?;c@KB!z0h(SA96n(sEo{H*_`V3mjZ%J2KKWceE|1NnAL$EbK^!uqcM8!Wl13 z#rF8krN55p^X%(91aB!n;o|M^b=uYz9p(6_gv<4}u#?3c<)rCVAT5CVfT68=Y|pH% zycMy)+_6x!miwVX^yc*&!4IMaK)_|8@j^4&U2u%Y_L1~p^uB!yOzh8_U?mEA%?1Zt zJs0Q8Nbn4SQqJiPZ>G!aC8S>ygA$cTrZPSem8p-tp| z4$1k&1|IBtFR6d+si|7<51$T`@>q$Q_Q&SQ#^eKajdVbl@ghFL+kckK06G94C@=Nr zjzmmzos%k7FV89*yLp8rdag*UubI)Gz+?>w`z@=X+w}Ge&Etr0K=J&L=;kkT3$Sv- z$+bIxPn_A0v<(QnXuNxECv#dd+SV*OrVBP~l?ezu%h+JkT&;IscWbcYKD5Ex+V~9s z%VpL2EwHUr#<(yDDYxCRo0y7!#+NHfS4U13=}ScP8Hm4={7LF%UeYRTep|IC7udb@wjj z=z;z@<3BsiysP1J-MUkv(!$sWEe(Mg!Nlzh2{|+>vmM~JNnr|@!y^U-};pYs%-aKkW zX>lri|DoKQJQ>JQXlNU|Sar&(nj^oWIKd`l8bQ@>6gu*Rm@3!L1C@O8wW`;FGI#Q+ z4mWyCe`=TsEwqQUTGLz;Oy^uH3xh=J2#{eIfGVx4%FM2UGMcs!& z{PD~iw7KqvnxS7p!Kuqs){UpT!${vxmS{B$A-c_-A(dtA zWa+Nf-&5b{XS;m{5|b%>;Ooa;&!909YeS_@#h(KZhv9Q2^qQBhDfL zT@9r*z13An^3`v2!fL!I8UWk>88u~kyDg9Ck)~)(@S$8EhK`v(6M3&6<7hHKl#sY_ z+^2f8ez{NXU7QpBDpYl6r;NR=T0Wi?e)nXD{awe%S`VIPD5ncufgmozu|_Ac#~=YK@zYM+BZ$N z%n0Vs(OsblM?zoEL38#5VCY8ls}RzRdYkF(CSmuq?$Y`>i)D?_n;1{b+bw`#9c8}} zdYZ8oN9jE#c?EB9+KQ%CYH+Xi0no-Ql%AB;nL47BJ&-`DZo|yE4K?>{fr=NSM&0*d z1KX;n5&JYp7H8AyHgz0;z?VdEKO%zch%&x~qzH16V8AK~ zDo>2wsZ#CHky%z)hAsUD2+32#f@XR^=SCQjQFVjkMz-hqB~QH1Z=0?hfw_hB*}Cb@ z@{-P+opfcjVs-8}O}M02JvjvU_?xYs2d~>c0?J^Wx=%KfojKP%W^9W9dhck{WoG(d z#~*Lb)adZI#EiNR>Sqf&SnM>rzGQyWue~w(`c0GbG1)2y-_fS$`AoV+x(-O){83xO zHSW5WC`kBIyN_U_i)}bKovw14fh^_ppjt2iaL`R?;Mcr|<#=J^`Gg3C{7w5>6 z6torl74fzFZ43a)e*#3Op)HTZ8&&hvCF&B^)Zx-oKPBpunKI38U1HlT_V)R|bZs4mBb-tfjNmF-Fd z9hVC)kfD_B$%SsKB^^saCi8Sl!PZAfIR*?IaR_o#IGv4uJ`lEAWyy4?D zCLn^H8M8Ry(BBOFd?F0O?Y(F+-2Mlsoe}9Wu zMtXMBm9N`?zZsX1o`vLtv(#?UjCAowU+`*JX#M%qtl!;~_U^+^pLqSLP+|UwAkF~j zCb9CC3&HCS;adS?bsO{Lxr@bX*QAuAH!HN72yt3}-)-1~Cr4W7#0W(IX9Z5$px{e{ z^2LS$@b+=FHz!ym;V?tqn>{elv;~e$q2Es}HY=i-g%NNqyU_*rOTrb{iI3TZq(wNe zXH%;2>QN%Hrqg_)zH{RRhOxmLvyRcDHm#@zn>ehr6D596;vjwGo zL;?<=R==*W7&EH3KNqD0hZzObO0?6sOAX`!9J--Q;i?TgPup2^v827DBTw;plA%_K zC;k)2n%*Y+C5WpHMM&LacjWT?qt|mu(p}SJg7;#;F~Z=p`@@xAzDM>pn;4X=dC1v(qL3W%k#gyhc57w2nlIC^$hh6iNBE<~>9(tHn;G?< z>D7FUn2C>LyH9;U$BVW8c<#23?CrL35HZp1@N;GqD|1*oopVp+2%0zHS&I!Y+Ua#B zZ2%PzNli=}gi!+OV^8O?zZFRRBsciVf&%rl{Wm1HfQ29Bdd{)Gx|f&ht>)?D;WcJt z5fz7ruk=63Uy=g#6rZd&2#%C7(7VA-i7f>27@v!LtRh5e0iSA+B6+R$R@MYy>|7|W z^~ZfYDip%KaO4fma*+t0GVmJ0{IXn)i_~)JtZ{6Oiam^ZH2tMlGtY^EqSvSR$lWhj z?6|!3>N`R>6vB;qVqMmn8bZYIOE{vE9Z3RRxUbV!%s-GM3wq#8-P3lpJa*;DF{V4q z-MLy^W+VB^?UVmWHEu_#M6eM62e=Nmi~TOajrKW0(1F_(pRn2n^&P_-I1M7GZ=Tzx z=UktK;)p(eAqA@AlF>)E2YAnO8*zo!l8IcJHUXfK9g675g8e9Qof{T##lAJh4ts>{ zvmT>T+&2wC%#YwmeltNz?Q5hT@5I8TMO@kdxdoX3(nWp-D3R#8KGrMULQxr;E60>-B~MXjkfRm%qZ&FA*Xnu6xs5Tq1b4V}+ci;74hGJW zTTR-Z5*+U7I%k0}v-ys1720hJXfy3Yq&~K+*<@<`1Bb!1*ha;d{S*4i0DP0 z0?C>C>?0f-HHf0tACuv53+p7RFb&Ej7-hHzAn6DX z7={ZrIF3EK!L5GNPRj*~Z~y*%F7BW@tUc-|2?Nk#ee5v@R@IzLQ=-5yQpT-~_pjt5 zuO zKUCK?edg&T4zcFCf40p@3TWEq9c*Wx3DOE{5M>}{>OQY9pUMuam=OdQSgp*sfQBH# z@*EA&+9M zXQd7Wc%TtanLqft99q#F^9)Nll7u9y0Qzh$%C7!) zgQSk81ZnGxNUbk<&+YuCGn7fRTe#_^P#hhG#X<3&TBL9q2e}6O=*vL2eWBWhnGfK= z8RV;7I!j6sn|`?V{AZhEv~4oJDpkKP2unEY8DjIv@3AP0xEyWaLQ5Hd1Ju(A0WLl1 zOJcFb`JD&-J@6-EO?Tyn8q0$luVC-r-ustnT3l4!OZi?gx&l+RdlF0K_EDWtfhG43C(w=jG88iI}v<2Y2f#zr?1>pilQg-#4@Z z&O0?G_|f}(TqNZ$iO2PZ0vt7OhNpo7(rU+~Onxkgy`vNfs)2q;0siai6-SroHUzsr zG#+xy+~5h!)|xcyKI&Jolb=ejh%|sHHEb?ti){vDEL~0(f;mFy#0g++_Yv zY#is+JMTxl3Faok3d0)vAoP0egD9-5*@*+ramtw=Zl#Smmpcj$EUqwC*EMo*JcDYq z2mE{#^M&?7QeA}L(MyOZS;Df}=3nOh(;>aXmRi=@QJhl<_0X2qvr4?Ge?WmFZvbzY z@~syp-hCnx4ON0KOy-FsfN0W4f5qP9N71v67FtP|Hi1Mo=3w<|r!$jU1PyNE5TsH+ zPi#hqb>y>24$H<1@BhsGRk^(d*h8H02;&v?-FK3>D7!hZ4c4B68Kvd{z{${beYJ$M2rsqRUzr(`?`|-x$VezJLsDk{C;M+MzW>({wG&w@(Dn|IovRQ3*DpS4{52Q0ltOap?aT=@8t(kSE%0*>03e|bn=lR zT8Bz;glR2bW5l{DAS2mIe`6zi@>&!}D!`j%syTm+KW6L`z_YKGE6NnAxt_XdyqokR z?iUD%=W9ZFaLx>R8^d#z268Yf2|SJ9=Eptk!ExHd`8P4kUx7eS1VEOa^L}Kz*R)gpz0xHAt$_`7uI3}unG?#8u#G3biNYGpCs#@IODO_!U5_n zA-j|;${6Yxs1~a8&12%p5fJnWsiY{1<`R^1XWB+Ud^BK$-gx$y`5qeluYT;jQ-y$1 zD~Rv?&x8s&TiXLbitk#R^99+15h5C4lZD*_0N~>lLwp~6!yTju6NS*s&GKf2Sf(tW zmtFpqQ+&}slMj@=luRjepK1h=^Ken-o0c`Awvb?Ey8ljr^waeR9dK%-%jyKyw73F1 zcdpWZxRL<}R_`C66U}#T0$lH{z<`WsRW9PG%C6bGvcrdWq8lWeL-_khvg4}s>&7dL z8zCqYUoD*+woNQtw5rW=2a`C)`eNv|&W~0`=RU~K=n9Eh2oqeU3h8ijyf{CvGPa3=#{ymyKk+kHoWGQT_&IVHjHOTo`E>!Y! z2&R*)tb1q-SrHok-IZW&zF9QN*+Ee} z!q_!lusJw7V5X*Fa+OXgSq?CksCtdmdmYSAZ2@}2;od0Yt$>7usI$3L7*skE#hb^cF_@L#wH>2nwI{u+bi6{toa6LU(p7+*4QS_IOS%WahL;udwX&(Aa zIuDFSY)|c+brfzY1v(m<;qgA7q%;bTbIW9IFZ735pzwAV`+=quD<=`a@mCnOdF|_a zlkj91iW8geO^tj(d3~8k0R5yYt&F6%WwFCa*#?$S7QA`jyzTZ7oKo-04?UGJZ1=^U zE#s1N{AxH~uS{H|mBa6RfS~p|Rh#+)TEh=&`?wv=^r`99)@BU@Tx;H5ED>!+Z)A^LS02JE? zqTuny2d4`~TwI+OkC@BHG0NVZ4e!cnsTEHS#}j=@+%s&Dh!1ECpc^jNfOX1WdYdrD3-o( z>s$A9zXfQLpSMn+XW@{0ZGl`|NR7kV^KB8s*MMeR!CA9Nvrv$$Ve@U#he|;ft*TU7 z#dIlB9$S~>-rLER%t!Y27DI17k8)lBjwt@_fnPj;wmIEx7pX|Uh0zw;>Ms>eA&r8L zlNDF2t_tix3)G6um?JfEOb0Zchx638b&O8~UMDAnX#$Ye4wVgfNSGZEN;IZRv?y|$ z9a_j!ez+*i+or^_UKwaMnEYGNEXnvGF=MZw9Z!bm$y18c#2&Ggrn89Y-rJwQsYn#92Rg6>NIi# znENl6K<&t6eUCL;>emKtAB8TiV&I9=XNzY*zSyY0AcsM`dfWQ;z$TbSpYy4r-#BP-3**0J>Lkj`|3RZ)B75oQ8lloPv67`X8!hisCR0RboqkF$cBMS)^P5YxH|~J1u;}MmPK>7jbB%_K_rj8+E^of zvQwq;!l3O@a>T4v4mg-IhCS3ZGygkj_&s&g-z^KTC$Rq(@w}u+|KEjFQfpCvp&e0u z`=1Bk(TQ_%=w3cTccl-AuucF2Uh-JTJ0Ztzvg8YR8>iVS9mH+BAa@r6UTg6=ILUqCVB_XPK4#?RlTJGX_*%!BB>;NTv? zv1W?Xt;>gi?G5t|r1=nO-X+riwt3$Pwn znUdDpL%RW-1U=MOzs$zcV{c-kfD>|_03VJZ;|jmIu*UH^ydkAIkRlfGxj?v5Co)CQ zrGtVe%gibKT$|+estkD6kH*+g27P@ilGa{)sZR1QbI%ckb$~i(yf+$DDe8xr6Y>P& z-{vv56A3PU%o-! zXs-MyhqBNw`EQh&@;Z9<@uY3XA@L^*0_9ObT?utoifk>1PxU&GfysT{vv8GukUZhf zx*eD@0~G^E=G@OY7^-*dz{7Ms7Wo?a=e(KaPv(7g;VD{ndQekV4v1-9ZBX=`gIHPU z=<(ikbzs*u_xe??d>oZ~+vSpYAcOUpO$W&)I}>O$wyp5-Luif^PUfIi)i8l_iv-|I z&8NHs{U6HSIx6a~?e`Z^z#t5eMx~ULmIkF$LX;Mfo}q?L5$Wz`=#UbIP65dQhOVJu z7`kC-ejo1dea`dnd+xK=IsU<7En(R6nZ2*Q_w{~V@5`!h0^Om%0x&zqZ~pLn(zomz zh+J>j??dJ-ZUV-B9m?@vjTe+t&-sGT(hp4@ZF+BuHJv?DQCMAPzX}7Saha;`8x-!LQZLb@Y*utmC7$~OfG01-nv-8HVhk6mI}W#6r*X)oKT4Jnr9cAw%@dG(w&kA!0+?cOPy7C1SgUTgrI~tB>uZbCE#4 zoyV?KF(vqq^NcfE6|tqCv^Z%yae=Yty?rX6i%jNa3`~jase$m**^+DRV*cX+dn1O|>SaNYSpLG%# zMJ}7FdQE84_RBEtu6}qcLT{cwBLv~0I0BvVihzpVSAPsC-7&)Zcs*iGDbD9rPWh|2 zpFV4d^H=iV&p&tu`}%p-bGD&);VtP0s^nKQjA== z)fLuct$k9$=6uy)hGdZSb7nqvtLhj6>Vs_vFTs&4aA=r(SLd*OgcO3f5f^WNKkniB z_>D}yDPJa{C7REQI!MGM8>sf(IPQ_>pOpE?^eGD)m(W&Jd#}d&MXE|vSP6I)?cJCZ z&tswSs~eWRG#QIx24{6|z;wK?({AXN4anvBNe>IH!7888Jek@^L5iW}_|QD9bWer*`LWIH z$3Vng=Po2*CAssEM_c3*ET-W#*i^fw+)y^n7r!7 zIQ!8=N0v(9ZHU+v+Uqck1Ug^h%?((oU zA@^p!Hsyw0?DD6iM<*J#mRCTE^w6{I1Ys>mRtc;q%u(D>^HY% zR41R+%vXsIO^02(gNUT+Mt~>ztCErzNc}piS|LGI%5AEp$w2x9BglRCUoqA=b)P!D z7D`P-^hPq76)3~QbP)*HF1=DD7ahh!vNxTIlJmMn*RWHGE-TX-JFd_80X6}Oe3cEM zabG-phV5t(r0j6;Y4Gebw$+G^)J)+z%pjHH)Vm@rUd9uO>9s(L9?$_$-e8P=btMDw z+(5>dtCz^B4V{|rQ#{9bH=JF9j6;L^MGelM3UrOlIi7pi&L2;%Zx4hUcn;{MSmj>B z4?vIl_nTG?K}tIf!}9DRvV7A{4GC0Nf4r3`makmTjB0-yxWB)q_S70wu*rAFOkFN| z345$Qi>5x(PjOZ-q-Ni9okX3@G_1#AQ9}nrxs31LJ?Qts%K3W9_zdwUZuZTp$rXmK z@9xKhZDLd-1XC=`f?M#CT01R_e&rdE($+BnXjaZKhhsplk?%9>J?!ah-FuPChop!8 z!G0IGYZbriEIRAgwVPO8lNdInvEcb%$R;^T!-6#Lq;EGw5($~Oo-b%-7)qFctF z8$%uUe=+e!7Baz-niy@taN*2Z^?D~22;z&|@<+Y%oNIjnC<*SV05dCTR+t-2)|o(S zD#6Dro}ZRxWc6CVGV&&+7{WGS2e6`XmX2GPO~Y#~W*!#MyJrDjavlpsC<$p5gSAUF zL!_-1&laCBp6eqePc-?I^^!^_#Do$(tH5oq)o7_x!rUFc!Y&PWlt81FLX!?4jfJ-w zivyHNF-@!#;P%WDk29jHMme2UbD_j0#3l5e|CG}K3GfbylTc0v5v|L7JyOYo0lmw7 zW58ktUJxjJ++$(8emZrfn%cW73dTO%z|h;~m~8>8xVN@o?cIdMZJ+zkxV7LO+Sgfs z+w*SV#SQHT5oR2KL!C_pod-RKlRQV=9$<*4+qFC4*`s6#OTB#xfMPsw%A?hNF5Dj) zrw!~p+f3(6i;fCJps8coYiEhZ1*3Z=duJCOC!2)KKOBK_QNHa@8@<~kI<mv?t{xkM=65eFOzi;Sh5L}ZR*yLfEaDKsh)k(43}98DXN<$D zR~GDXf+C!R6Tb8qsu?}EAHC#pXDWT@BJwYl3mwgFK#iH&Vq>lxT%LjdT( z0OEYBN7(fOH*-lir8h|zRJ;k?9nUP&h02dsN7{~A;LVdVWy2eZJnJhajg?@bdOaIw z`P#ydT(IG_A3j$nmKk@_J+{U|YTag|yw?of0FRLK?V>od)^^1lC|Ibo%hjf&HiP(V z3c<3!UKR29%jT7Za;n*Qly4X*s!9NIu5DFsdYfAGVy`ZSqKPDqH`}75VenYU3NQ%( zM<>~6=2Sm86S6y&xwZ-^X*v};s~xK68}qfE*ob7Xd!J+xjDRe&{T^(8;q95T-MDV> zlaYeBac_@{Zb7^WdfM??r(MKiD2O6zD(+xqBFq?(@XQtcwu<0l{~1@%5#j~yg3!w& zOZLW{teUIS0QcBi)w6bw*m>N6Qe*P#en;!qW88VsG*~4BaoT~l0v07xu(18_@tMP6 zv-en=wVPI3d=GodkkhW`o!i~1i`PFN-YmL%&GBySWHAT79RB%KbT>l1^6mff0{H7~ z7l`dGx?fjkL?mwtq_w<&sMv&P1atKM#;V|! zo4=m(iKe-0YekL%PFANCEDHby_af2kkt`RN*g-Wf|HufmiJvH(Ycs6E5KTtSC(m4< zSM9!y*eN6TrLjT=#cHg=XDvqK^#|)-*0Cbz%BR$H z;Mz)E%@}Z~XzlekQT%eIt|Y=~@hWD*jRB&1WgPk0E$8}+9d93TM)8a$3Na>@C3K6zG6 z`evJxhmDgAxIxd4&mm!IRWzsZSbUAlOZS(^3|IP_00QF22xe+};IIC5D$99v2Psuk zsqe8$w(rWFp#R>0_Hbht-xJy~Xmx?Bb?@s;m$(17Kc8qWFUDd-&db$s1ez`<_j90( zimA)FzvfD(>)eeR5da4v?zA^aK$eHB+Ab-#9a)=mRy`HRk*)7Vm|y!LJz-Q-G~+So zu)ny3B)&CZx|qBoe3D0qjQ^qFheWCY_Noe^a3&UyC><`qSgAH(LNWer+ch7`HtQj$ zmZOxb4-{$}-U^zcfLj$!Az7-3fa+~t&K+f(SE0~16M=}bcf9}%RffPW!kgJV*@l3r z?;y1o2H+aUmYEOh!@c5k`IY!bHG~pB$tUt#uywqq2dcsMT0+3uLwBDgn*eq2VXqHgL;t?+b&s08e{!rW;?nW= zgXjF)>=3M(q7$-}3@LoYWK|8Ty`*n%Dutc|AOMHZD@?QYvMVD{k?&25I+6NH(QtY$ z5p+C(u!=M(nGtT9e`~}JlZ&4{T{_0hx=sthS=iGCjnJ^@0zYg{oJ<<7)XwbltVoi0lw(0 zl8Q2E(@HWrZe&AC-!Uejw=V@P%FDd#-Hd&f@HYFe1So4v&p8$szG<^CV57(s;65s! zsP~P*4)EOKl>gx2kIETt2TW%xrT{v&zJ8XI>y+DJXaULeGB3DAr_Qn=Mn#3VJ1+D5 z?p~>X)nqrQB$*%bp%v$_C(@U5VN@zotl&=mSSfJ4LsqUy>ZXiF^s)Q|c8 zZh|vAUuK}?{15%bhJHArohvO-XW0L8QB$4wcBpMirx?MBmo{7kNlN??fTOq`$ed09 z_{;f=sH8^8`fmmJ0}^h<+Et-sDQti-ZoB!aD!r_+e&=MDIGlc=GCJCC~XmzbUz#e~uOmD|W!1&ZuJ-Z&a z^`3beSto^%aoDyF{;>K%Vk?T=I{~ah-iBMpYm{gvnYNqDxy6$4a_&1$aPCg@1(ViH z=>GM#_{JL(LI;QK~cV@?>$~{^cY7dWGgNe>|yd9J1$KgATdq_I*nK01{1}A;G+gosUkv3TV z`Yw~;NywL%*c7L*6$aVc3!i@0V^6ojB_#K#?H|!eJb!|vgJat{h!#9>TTInb48%ydTudl{kfzxwVhnuEmWWR;w?(C~7xJ;J;i_ zV;}k{tERBC40uFw-fj{S?c_iRUF4b7_KC$2SEr(zU$m&+E>=v^H`K7_5IRm-{iyjc zuF0@vaT+7a6Irj>BOFf%65Vhp@fJxj97x0GR@ZKOu6m==rH zJ)w9tOPo;p>&!>HA=y5sZuyCJf+W4p=x-TjfLFE%PsX~PGvOv<`pZ?EYQJH;8$uVkQs|*Z##ue>K;PO5&4xZJQFL#s$bLhL0;j)~WLy?v z{HMN3StxEXfMLaFGjq`zN#1#c04o#|sk}?omgp=)!5)T2HxI2JS0MpM=FhBzRS=&8 zn9qb>S)lLvX6)xsRw%`OARA|$>$FRNTB^cni_BNT1p4^6tDNF;k|f19p32!!;7kwD zc`$$AyN9f zUQLO14~LUkZV25j_#{>4;fx$8_7fx`p_VhET?xc1*kQcco+&vOcSf;lLLnlr@LnT4 z(xdW8Qg3_dgfD}K5i_+ez~UK?+y>6{040VvS%9UCgES&t@lHgLfoh`#u8o?`lBW}L zph`PgJdOGC6?lwTItx<;;isBC_RdK$Q~rxbBU$!=S)TMuI#*8b14sV)e&9 zf{PrTSn)XbK$J@pCo5**ZGC{WHmTDKME1)M6j0W-*N74|#myr~P1^_&n^JC2Y@&9^(;Hy`LogmsDHk z_(3ep3D-raHhuUh!h;iePDI#g6T6@x%&GQ#d-!?g9^oEyuEg=Ma7*EsfvbrxC2q}DI5b6r$p zQsyEe1<6#_f$S1eF)H+YISK@!c$2-m;5lO-8 z{-GVS+zI@wSqzL4XESV*VKz!0TBDiw%UbMfayegI>njM2K};1Xt(#I(SsUorQl{5coA~&!Ww_q3LLC>! zROB!DCABjvVd*5h9`^0BeYxB8a&a&4Rwk`}|Ib|GbD|^wQyF*rO?+dPaX!aVVHH-D zC3ZyxSyqIAc#nx1r&*x?pTu4wCR)F0;{TfM9N4;{ zANYsi0wy2DCoKqz#Z8he6oHy?mv_QS`znP)w?=As;))AZhomg7AKvt^TQUIe{NO@F zkJis4Oi)h}HeWasr$&`enl^mkh87n_j!Ibo6BQflASuJSqc+a-H}7Q{S5H(~xBgM9 zV1I?@f1H;h0D82z__N_As)qfDXk?I9@@btWCW?~yb(-t&w`>`o_eBcVZn~6=tgn)U z)!!)LW9f(c3S=k&`SzihkABHg1fePE-IWJabm|r=Xbw718pHfRiU;Um0ff4v+38O{d9>$ITfYx$oO6S{0cfT@mf z(N_Ce(zRz!%{X0=-u`MQ6A!p-S(l>e8dxX@;l1^|jjKHdrUkYGP>dlTV6J_>o@bW1{SNZ!`o z+ydpbv>{XCj1imO4ioIpnl=tcW5DoU2>cRDd2uZ0#(^!Y>;Ag`e=1&H++x>DcI6H? z){V)p#O`G+F66ZJUjj+6W166S?T_+h-L|;(;e?f6Pb~lpj;H-`R?3sw+#xrCviUvR zKhxnybuDGJ`SpwCJ^UjxHf(Xh0?##~fMBGkZ7)WVe{x)S6!xRvt@;lQbgtxG)1)c( zt(R4Bj|s>%LG=7TO5*Q<5eh~BCcH@yy}F(8^?wsYFYnrgB`z?jlBqg|0xJ@*_Uw>- z&|NFN^w5^eP$y>g%?qfu8TlU;g_`BX^0lnAXctZ~Rq=7{=NblY^(g*y-?y&*vk}cG z{bqFQ0jyh4cUSo|aBTf;Y#Ttfx&6+p-&$&IhVxiE`*A@(jd4Oj-*dj(zn1Z>PzC1z)HEWMYk0YJH>a<9so@TK z2_LT?KB>JkmM}K}D#yaoDrQxRCbaaw!k+P>7G>VJ*^%I%2>0&VbsGCdjDEY0=g+kE zzsMj}EO)gJfQJYCT|{?^Rt(^jvI{_9dB4P{7-R|tX?Tgg12xAjpq4up|Kb;aXspS` z$-2IyC~@v!P!894p#{EW!D*6(d)EXYNzJ3|>z0S2Kc-H;>SaU#pXJ_vtkypU7`w2r zCr3AQcdv&MV(RbzH+Ply-D7D^ob4W-3)35Eppvj>%|HjE`BJQ4$7dz#nZ~TYv>F85Pr9V- z&X0HV4%;Tveq?QB0xP6RqMi+(W{9&xJxRP+N#>Ijv9x_|`PZ28O6mPwP2$Us=A*#2 zKm2|UJw&G9MxOGB+U(=b*d8DcevOAROhUPX9e+tE8z+1Q#7JbADB?j0M`hu^n&<|kHil~S{2JufCe?i;HnW-Do4 zLhs$N?Hkb|lQ364@Km)|dl;Ey0;friG}O)a^2C*CbIxcQ9?dGxm7~lj-|43gKFsV+ z(=u~wDZAfYT9hB99PZR!JA@x_#|TAQGV<@kb3lhlg{ZnY0m35x$b0#H;rw>Q*dD3I z8ROdbTPTnhaEfA9_ALX zLc1#v+}f8`aM5`Womo8XI!M!WF7XVZ|3!a*JoZt?sAe3B^@M8^#iX7k|BQX`kLXnz zJdPcbZ#g7L`HdTd^iL9>`29fw@VGE9TwS5<+0#hyO?@#Fl*JAF=uhkH$Nboy^v7XgMBh6hd-E7aFa~ia)A7Atc9JS{3Kh|&^>d? z2&*34XvM#?r6I;+RYUXbzD;3ff}(c8ObR$1gDEtGzxt%V}xTUBbLY6q2T0i4=()0WOlNMepZm*QBl4_k@;}g zy7Splie8S*3eV7)g$iMoY zUT%*hKsy0F$#TT8J-7~q+}kP9PLJnuX|CQ2I&9^nw_fG|W!eHAppiX;U!O|Y4!1?4 zD?!s+l54qh|pgM>Pv&n;rzFoRMgtPAAu9&Ud7p+nQ&Ln-hek8p#8GsqmZmIB3 zUaOg9@=$=`k@gJRu(C}R0LMav?(J-jK?j@P`Qxp@D1?gIax@;mB*RZ_gsrBY zp3R1ZfIWe7|D0Vzzy6Y@!_3!eChe8A`tYVxM8_2E&RgI;*)8{IHdO|gWdoI=A0Wl+ z0QENH8?wcZd*@F7Qa~#|`jXph?;ze`6H~@6dKIpedOz#Mduf#q{bWs5L6&V&`0Hv( zqhbB}7ryVsz1t^UHGInl`I(Bs%E{cl+nZmtc45jhT!+m3xTyqzW*?hkg+E_; zto1h%UyOL-0pQp`3HYnF*-_8#br;Bip8;xy+9M^XE6gs^ZnuS=T<9BJ%6*~DFKekOk>+-OiU4 zcD2hEy&{(Y;3o($1&k4t@}37&5`fY89&Kv8t%$ERY}V=G?$)$BQplBB(B@@J_ox`= zK&)xaW@m$3uL;q|vABoo2F&?;&4g(DjuPk~u`D^c)d%FWFhNViT-OBY72=crhyDrp z!N@ac*?@bJp6i*g7R>xE3ISYB2+3qvwW_k`=Ds8fw?aZ(S0YH%Ys`5BT$bJxX*UE` zf|2Hxu)CWxMNu46xe7_Lxk@QV^Cie8=dIalor%F^a$b)X^~rK)W{SQo&&Bf&n#Qbr zHIo65bH^9&Ve%1`cHVr`wN|#R$!zRwxtPhRZRK5Ixcw{LUs*N2A&U@Y+fd}3U^-3H z)sH1j*JYMIew_>sKD^&f8RX|FG!_ViqnSb}Z#j;50 z*4kz5Chto*GemH25Iyj$(^}dE+BkG?_dPIM?y0G_xc3;^+&Xk5X07-tqHK$wh2grWXLnj{qTz z46-U#?Koly`ty`wt57-J*$&aKry@jrK zMd+cO?mk$~-EKSyUU$BA-LCff@APk*K!l(6Dy0hR_*v=-s-D;y1uNCaizoHQv4;U2 zfm6{vx+_{junT?nD|sLqqFhM8u~1wp+#1nO#p=yV^-=a5V=t7qbPnK%*+fIvqvVs~ z`JN-s^Goa^lp}XO8Xh-&xDpyxnZ7nsK#K2C7f78lZMv+A0Aiq8*U6ZEW2WM{8|~x< zz!92`fK6wM>8I7|at4g6Uat%Q;gJUoU5Vy8u8}o7o>kW0cWoia>|I+K=#>R(?*phC zMs&jfxzSwt3APXV5bUEa=!dg$BX=B#Pti|61p;wR8?&OBje1;dBGrB@Gf8;ZUm}VmMO8XXN%BcMK9>Lr=rNn|E}$d%%I)M)%3F`Y%wDFI z0f6X{Vb$ga{@WKnX;CG^KPgKg;YW@FxsmPYV5aER2DU(N;s(*^Yh~t@0ep&4E-fX| zVzmAG^r7j1q_Gl>MY$#`(3bPcHLgj_Db-$XG?Tb|;`DGkGn8sFTp7DTYeX)Z%S6fW z18$1Vv-37!bK4$cbp8}z7GcJFYk&*eE$wsXPKwxwnr=LnRbHCbl zO6vAZw#mm9)K|;NJ>H&cZcX0Ts4WhM0ByEkdp6dWyBxZRL`*4$9)LJ2t;V{S>aDK4 z04Qo0;Gis({0M1y7LwY7jN5=N+IrCvd9gwM>*2g$a*>G1#AM7C#M35b&wt*1WBgv9 z^MfYS+9sVic75N#r)=MBN2{IuOD7XTQY76#L90a^or>tc0cy)q1-rJaUn}tL7ibiH zjkR)eSs#Z~+DCCfJ( zYXo&lGL6~j&!yCtEqfpDil^J~n>ND=`a}LM`frq<$s^^_eQ%glrbvy)$6udkD3Kk+!CF!KYSU)( zEk;vs26g=-o%U3nQ08yIbH0@1Eh;J`mIdnh63oYLA|65+D$Hj=i2hv6lkzX~s)Izc z->&c}gv7>*{j3e!6`AnKfbCvpq~K#ZKxzzeWGAD>V_io=Q!dE|sxmY1`3gk%{yeC$ z8A?XPvFQf>F6QaaT=%m4L{gymuKO)tP(13yjf7*Kdlq}hs8dYXZ4~KfEi|DI?{HTuAd=qGH;kjeq)$s|V zhqHjZuB(u2o#R$o418kaKC8bFJeo}{l-hjehZH`|J6seQTY1{cFM&3U=F;oABkM1JizeRQGf>^M5L8;fDxI*NbY;*w9qXnY4jfvbK2;sb%it2ILyt76(W& z2Z6ouBQZn*zc6^-F?jzRg|j^uvYFD+Bm|@Fxi8A)xRp$Y-x$lTujm&>8B(ry&On1U zQGWET(pDD^Ww54No6-&%%#wW%s7W8pvXYoh2+Vipu*>aEm^`HMXYpQK(r-?pRQ5sF zwmXeS)o7LwSk`Vyydb)-?m@FHh|lsf6rv~VN3gmyg(@XjCTBpGOT7=6toWtY9!yp< zFz-UVt1W=T4W9ZQ%Lb%;ggTlW8nhj@hG)3v;*xpD<%SDYLdw2%eIwU0_n*}~QO*!L z|6r(92J#t5uBQ*XVqeXAi6Q1t`V~6#15F!gFA1_U1W0wESS@V0B1M&_jd- zUr#NuGM|MKS#YF2)c!==SvF6DORbxymS=z3Ynuvmy@y#y4; zWp8P(a$}V$eRqLL>~DN{9Br=?(3(Wmxhr~)J~TOs0!wa@erSoM`~DTXeTF7`?O|aO ztlY9x*fShPd3Tp$7-7<=;;)ls4`TFniA~CU*g5gVe@s*r(RFgAeeamufpP7mxF-QB zFQKlKH?}FLis8MQ>AJ|lUnZR6Luv?z z^E8tZg$9#7j@RTpWNQ^LBL#?MC(X4nj3e)}vVJc1dWELT1Md^<6|)KT$fh_;pC628 zLhm1-5t+j2rAD|B>k~PKaFg5y!;<-J*&!vrP?R2VB21?h6t?=CgS~3MSLE&Qgyoai z%wvB}vt0UAH?luEPg;Ac;H~YI{SoV}eS}m=Xa=?kR*$MdQXz~&_sY}Zs5FdQ z>=)kjJVW>hwg02s%_*e8Aq$Gj4XoZw-7Q#I%IaIlh*|#THqE>H_fE`LWHZT-(iuPR zVNdfk+2aZ76vbS*!;G}k>P#>o-N z%gjHkF=V?XC9G8p+iMye^Z6$`FP5YeQ7H$Pk@baMmab_*MGO|7TRGS4^3r=O ze$&f*{H2kru_;LUUVEX-I`t~6OL^Aw$?})LX1(gq9ON9Pw=gRUdWZGilv9~!_qHn7 z8YY(L_aAynnxV?*%k3X{3WUVrW+_PmX0X|y3CNB~m)gL>-F;_%Plxyi(M@9FYr}^& zLtMHdnjULQ_sI7#GKfndNtruWT4kvjS~Jv!Lw@2k-N@o+MwCo?F3XUo1nggP@Lsg2 zRR_${Mhz$;Pf9~sr~9iNI1NI;8I|e|0v?8Nq?%}|C%wC%lanjjPjP>X1#=v_De}z~ ziGA^BD`g78V!KV=)f?MyiL&rTRJ4GLn^ns=#;h>JL}4Z%c?u zy=CRkemk5N)-Ia^{!&ej&ci3|D$8Z0t(wWkWWl>+1K&eOKF8j6c|c^Wp5(L_rc2FB z%W?5LV1g5Ah8W+gxRsG~)Kg_K;d@)C*C4nV#4%we+8$fLlecL%K2E|fzKgMjNx~$) zQXr{U-t;#t?L4^ok^jr+@$1W5*uo3~pK)&3)yu|;Ia%;?cgeu&qh|EWa(Mc|gmyul zldc0{MUTMl27*-fps{f3^{;7(X6x}%sl5Bn#4z$Wo|f_y8hHG(ByrW4w*j`lmxO)7 z`ob%H3v{<;-#%p)>`Oh|EZ;8W$;jl%V>J=1tfy5*p4|IN8Bv1DVV@*a1jS(kpZYiim%eA6258ogIs#aqGl}^{FMFs z@783p_?miG_?SIqb|Z_)_SPn4V6Pa)x%OC;3nY4sbz?yPOmkVETMrMiRHHg?q6G^Q zV#&NT@3SUSyZpI|ye>=5@eiyeL#GKUgxlZaD>#h0A3Y7T6cJ2|ptvhS&H%Rs4RX2PS6eC}lVPjd3g8d6~Fikp8mwNI@#u4ByU*(9R-C%IX7>r_c+W zQ$%ZC8@w;FQBDR42(~gsBh5rRcwfAB$vC`tcq>V^9LMfe-fz@sUZ^w3M%xgn&Rw&w{!HC$@I&hP-4!5k z*j=3R)X~|oNXuru=zdryG4WURCWIk|8s?ET80Cncy2F1v*ZCpD$l9TT0qwaiGD0G1ku;7qpRg zdW3<~pKyC7<*^o~jFP$%2%dcXA$%)S0!0>UrPIM%INhh@7Xp@>nMU&4P`8SOadCqF z3BmvUt99Iy?_M5pG`D1E{#!tsv!s9CmmKzD5VW0ziR0W<(uVyc6H3Sg25?R2fik zl1YVp`uM3Qe1N1+$s!osv+|^JQ%f~liK>@zDw|S-;1dE==WV@c`s}ARuIPXI{C9z& zrXH^ok=^|Kh3@&EkeK;9#5#KM^f!27Pm@p2l$OfSl8)9l*dO%7V=UYx1ZRO$MP*y; zS9&dd#I4^660%d|gXw8+71{OAs+KZow5LQKg6g!XrFrvcl^NMu1t{hAhZxL(IUiD5 zM7)hfO3#FNBjk)RW)&=YQ%UAC5|=V>NU26>_Zkx2e3hWW+xGB=E_%(bU=hjOzw;%H z(ML;y;KoKs#D4qhx+Hl<3Q8A2ZAgng=TO78*-gO8xCo4`7LzNan-geRR^*=|#42p1IJ z59=JW9U()$(!1qgWnc(T(oCq>uoG&p6lqyor&)qBfCai^_-Hnf6)BcKfBxLkEIW9E zmoiW0Y%Cp548KjY{*~m0O8?JiPiF}X{fXm6`pr)Eb7Ep3@#|O%>|G^~jk#IDyU!D-z%Q{%>e)C7ac zBh&uMhI=da19KIa)`x~(6RU6DLt9-$Umz@OJ{uAxoy_NKo|Cq#JT!)ngo5u!UQ2u<}Ag0=Sk7R}WC^63xh ze_M9{#JO8{A|GOruE^#vMd00d>wjau$3sMLXkfyy`UiW&Oz&RTFrEg``n^w44K<`{ zI1UDLe0vr!S^x`hjV$$r-8Ia;p!TyM`tsY_3F*zh_f`OVve+a}T2SffIqhDB&LUrE zqIL92EMEyZg(rMs)@FQ$|2OHZqiy&M8YrsE%EZO^AwwvAL~x*7Mh&LVj`4nI-k8V# zKG0R)0}gakhkk=MrCn^S(!1^tMT}idoIQ-XlVkRtzLM#f5NjfF8<7qlHwmCehpfYK zP}Hb%=&UbS37Nr#HBR|!{O$gO4}7}z&Y`SgZtc9nvSO9W(eI}oR~>$7o&Zj2eHDjo zLzFY5-97DJ%7#3Gg(-eoZWqMII{sL&dSL#*B$ub1H|Of!gRAfAX0Mbl&+Ybr^UK$t z7g|4~853JaVVzqnQ*S5l&W|=L|EC2@e!XBrTqHk}gjB5AMbB?~pQT<6!9{(a9(Dmcy+Mzd(PX{@+8X^rR$lE=_SoNNaE72X4Do(`? zDRr2DEZQDCmwO7-&tcDw%G((xc>(8?tBKOy@vRXcy7KQYWj}qb+V3SSxG5(Ti#pQZ zAKDKyai}IcGAVzsai8?YGycbhN-Xl_+>jWMbH*X=gvQ0P>B=%HrP?aJg)A!!pOhYz zThFUbRtld7xt_llohf>9?WCbmsPlfTL_f?kt)JCmq)-Zwk>Bvsh}sPD>j62BJngYi z26Pe9;aZQ=trL<15{~o9V}lhIW66d^HI>eMhA(RoVt_r3?i7rPw_ecF|l z*J@m?-If^84TbBFl5!&zQehqgxD#X~@81`2=n+QStq!UpsCCVnTjvEV-TR5u-VQq?2z} z@iTVKzA-6GvZRj3d`+&Y2oyoV=X*twzRPZoGh<;d!1~e^(YGig=-D)ct!)hC%obsIHYc8)tnVC(K)01!UNGGK zApx)e-%^`mA;NHuErDnjmEf(K+?+1W)NO9xF+KO0;r&{B3Z9k$T0%Y!s90<=0@^f3 z33Doasa~}uT<)}MJLx%>1_+POJC0f;;ij9l_%^|4p5Rui;}1i=IaNpZ8%2@H4HhmQ z*@*Cfy=QFio*i4IL;0yXE0Z^|wo20*hPqzqUka%gnHy;}dC-c^rhI{pAE_S+^FC@J z>(2o#ENXT{2JMU+Zkl34;Yi1gGQk5+P*jZiyNhN4%;?uBL9(t3y+yaaBtE_opiePm zMuHqcr@Ev>g&Z(ugQBTgZ{1V;!>~Z5eK~4gIW*r#bkwH0)?l z&deOQyS)zoDj%^LHfnxnt=XPxV!fN~m%*)xJ`44QdtIZ*8_bt(4J zn7s=+7O1np=#5z$pNTZM_A`iJ=uFIlmjs4g)7XGE*9oz5eYxH`tr3*SYn=lVs|$$g zz>-xyTvO84WDie9DL<$tz|4BG0|o;lOO&K5sSrS~$x~dHR_GIS+;U(|ooF133QO8| z2D&}lE&96lVRrZ5_fbwL_T?T6Puh2;3R6^|9eD;e9A2SV1iJck>rN73v5>}jEs0-h zxg@Br82FOgo_ZRj9;6A|v9H7*+Xu9zpAd-*>S@8Dt%fs&z85cE^gTjNy!D(BD*uT< zN8O(+)S;kWH1WS135(=KUhFEBTg{4#KqZsx5j74)Ye07ik3Q$S>tDBA>~a(m@|f2F z?2$DcjI=v29dJQWDcUwV^y-EcYi2S5X6pWjE5unD(k7i^83{p3`32H6C_`8LyMUl` zov#0S&PK;iqt>4}l&fsg*L2qne`qkXt%GR#bNR&F7e{aa6cvtE_A!TuYq74~gZeu$?rt! zX=e4X(E(#*1=!%=;Vu=MzDB<>JO=Q}IXqZd>WUf)Vn5(WoW`LSu2o!LwhQ3V94O&F zFx}OZfImvv>wn=WtVJEEcW=a45-~ukRpAm}T-NcD-@=qXTQe-ywHy$qY%)*P0p=r7 z#J#e+i;bO&z2V?t2#u&un>p3|QOKCy8^YT_TP9VGC<OZx+ zoT-9-lD8JHc)%YdFBRJCd;Gq12NlM0^IMQ|B%CxK0B$>3Hp!4FAMM$ndh?4?cgVUl zTR*$cT?)|UAb~a&AA?EFxOcXYC5g#ng}U2<0dM=<1K8uwj%XH=pOuxadC*41S=4wb zr|`d&_W3sB?Db2&b+K`w^Fzqy*OqT{8_r!_H1=4PcPV{$9li zO~J2U+JIB8m62l#n2EBkn9VTH=66-me&r=s`<@IY9Q8cDddd!5`1Hn*zDJ4y$C4%8 z?1?XNGUPw z^O4nCxOf`%rh3~hcS%1xIVQZnJz7*kb z44R5{vwxl7hoWJMopW>C!uU9#7Kh2upr>pyhilU&wz0mC9U6f=R3X^=wjhn*QxW_P3!_ruoO+ zym@cJ{+2fR#a)HR5hb$WPXOy5z*#FQS6qP`_w(nB0oJNp{^UHrr1Pr>zWcwLN1%V> zV9!@;HFAVJAnEMjSf^-JOqHbzWeuHX18rA28F0B>IUTJs%BBjA6~^Fjl&MVlg7|G! zeH9X3@BFn9pm-`%q$Q&8@(m&Z7XH^d>ZKH!TVHfV23z=k%@ciE2F1i;hJMde&+SAh z_rXrZ%i$mE&lyZ}g2~L<3u~l^96}iJw@`X-Y!gZ|^yC6Z4OMntdIp7L zbMlF5YQC}y-0g-wOiTrX2|m}!ym5#uxt&5>GI0-Mzz8~@QK4@zUe3@7lk&-oRljMu zS6_kLxQu(e3qj>hhc>)Vg>8%K7DtFo!pv{xji3h+e^P_-FFft*MGQ1los*${nB% zZ%k%g<= zK;D8hY5CTz|BtS>4vVts--k^?2|*C01rcef#ibPqK|lmSTIohu5LsFjknUW%M4APp zQR&)c0qIbQrEBSYXYc3vKF3dQ-+wqzmbqr;n(Jcb{G5T7@|c?8CXEsDldMlo`=Xw! zJ@mXCUhYPCh5y;wtlw@VQjcAaaG`fQl}{?8v_L(OES+Bg+a_4>vpYr55Y(0YRClLb z$dzM3`Q3##!%1gvXG!%zg*Jo4!6Umm*A%fm1r)wIrL?WxwPBg2q|`ky-lYMTw-VpF zcp)q6+|e)RKyr2EIvFWmYT3+nrtC}`n~`&@9ROzL%hKpzod$FjY@Tr$JG zk)@SN^qmumlX*5;Zb=PO`MUiy{9=MMwM_3!cU#DKd4*#v;iCoDn{AbHe4f=`sLft( zQ6;^lz5Q`U&--FkzSi4?Hx>i-kgN z*>P^wg(C!Cq5AqCW~Q{5P(?fuGc>ASehvCaTdYtgKAx|+s-^=Cn!lNm zvUO#kQ+rW6<!6k z%1Oz`@fzG63tTZ9iYy*B=|Lo*);E{jrNZ|OSVs;*DhX@7rXL)+bMyIa2HX7FAY(lZ z!QDOIkCdrpO*PKDx^RP!q^W4^wHWluuX_rtb=-i8bSRYa$Yl58-kP>jU02+(E7@hk z^JNvuJE2I^E|$w5)#}|Ah?0)_uJG|qHFlP?i#T>`)Hv(HTRIP3TQd9O|Hp9esY?!xO9O_#(27^X2*-ONky4+F+ag}vziV<(jUfzyRQvgLkaTy z(u6>GX7?vx#brw^rZ&cKvU{$Y(KM&=j@RtIp~-*+X{OVSy@YVLl*g4@b{o29z>I|fUIEAYmSs#aM@P}@l7*wBwUAuYn?4y4 zXk+msK04C#swngQZ*R982QB7p9OptWI~qd0e2No(a;aZyHE#b3&7DE`MmxqDi`l2> z9fcOh!6-b7BVCxG4bL3S)bFw}t(iaX!pxsd5_RT8LBqhEZAfJMerFBsNZQ}24G->u z^0rm_nXwz{MeSbO=R|0tGNRh+H5PR;Q?iXiqt!!glM&~;5A%fXJnc&8v8yh&nB_*_ zaug?U$b^I-X5%SVTB49{{3vQv!VegR6`7E2gL+t_Uj$x7jrunll}ny zZ;7761f=|fzj!^W$s4xlND=(Zzh)vD?`Az-Me&ROp(%mi1tn&3ubgVdgbX1okVPW^ zajW9LXh1TD#xRm3xwLr&XwjXNl=Zt9xwl2_o4bSyn4Za=YsI`>?0b)MIv99l_mQ%i zB9=R(m1~zdM=I{Fj&hhG|{FV_F{tA};-oX@yC@84`GUfhnjCS=T78J28j$ z$<1jOLGLN_J{Y^#lC=6dF(#mZo`fZke-Df7q9c^P-<6bL% zvn(`zTgTzs`)1WfVw2g%we9;MebM|faGXeC?yBTNP|H&;IJ z*~sTEW4#Ddx-hobPkFj6`Sz>aZY$Msi_A8UugL^Hd%rdHA0TkO7~u0K#ad@C{l5i< zGv|okOOyCH3xOM4-$7sOq0lQ&SbE!$*R zGZjpfN6zWv<^VsJ<~#+q0EhGfx}{4I=Zq+S>&|~#iM)+}D3Jyc)W1EH_bAt!wn9x~ zJ_co?9Jp^mw7?AL6VFIpyPYutCPn_Hk^cS44Qn>0ul#>2gGXM&x0){*LHb*(cR;J_ zp}U-(HXq}OAGACM1+s2;%GSP*1TF(mNySf}Bg1~g|6VTp&qVzHJTX+ndoP_1O9OeH zEa$g2_=UVLWdzrSb5F3gEWc^16Y85h))^^|VY%3P>$iUVw}PM#DhN?_6D)$it*Iw0 zKXUU{uvR}WQ->*kvk#q8#4px{5)1G!Wd8SIXq0C**r}YN+pquCNO;EWZ{v{n0>ZuF)MPWKI6`x<_n~j%=+z5Xw(*M&t zAFptHdsiT@f1jTZsnYZ|EB6W#_o{ELl6*2(QS#uDTlQ(VCIK8cvs*_lhv5i5_+M1K z^jk0fGpnKCtsr);dG5Ex_G*_Rz+aFtOnFtZ=+vO~ZttnY&c-CHPJI21E!7%}k<4$C z_22hP@CA3;d8ngFE*{-|@x!tPE<6k^1_&LePWjzrao0c;9&uDXN5!T2j@Pi}u}`k= z{9vIjiXUczmRso0%~kjDQH17Z8Oa94BhZ$!-6VX54G(gC-_Fk|?;?;e)D3}GsNAfS ziPMIcozWb5ZyY+H;UGb0O^=;YIy4FJqWdh1ut_5JqHW!=JY1&T@jmCEST0Uz?hize z>!LOXmPPiG+TBzaN1OZ^gz#k$!Wj`}i%QV0A>(+Ol?I~oFvOCn6^qphFH!rnSo)7q zFtGCm@VUDU;inmdxT_=QvrqSu#VHib(aA7bi%LGdDrWuK*U4ck{QxyPEKO!pGK)N& zb#HK-4TRm0fX3Zz+ZjQMLTNZPydM<##~UAl07QT^yCePP+_i6FO*$gg68hMBN(&QV zMA&YyNk=*AbvZu{wXJ5lM_!N~$D!)By4#cZL^_U{ z@2-LsvykcN^ zeJvb!<^d;f!^F-}4MSt4YhG<+037_uCliNAEUMp~G|DY_DO}7&MUgtZv+mGx z1{j9ZdZIeo+8|i>`|Q|A1hP(EQNnr#!KJ05m@FFO+D5MV5i4Z*R2={s-T3YidwPLd z36Hs_nPSf@RS)mI;p1VIWKrRc+C=&>EZ@AvYrtw2ZVR5*`fOLtG`5L|j}4Q@2*356 zaPHK|d#Xm^$sT8{tCS0*8eQ8@7PVz^yE{e;#tk1f)_jVDouTH)$nUvkZo9)GRck%# zF0zP`OR8su3t1zv!WG(snX0L37!D*dC&Fjy)M$3tsw{lZbaqnA z@9A_4_ha;EE7z8l7_~rZYv;~@BN(_k6o<5F54lhvk5dYhPm;(hmgllYN|J81iB@C+ z?M!dlO;*d@)%v2fS8~$(8;5*(YpLfgCy!e2YN=u=FC`GAm>c00rd~}XHJUE`tVl0{ zfSmC@u!(fw%@iL$J?bAq;FuG|?9=gDsm4=JSvrUf1Zs^ytc+Hs(SFbGV@A2Qp(eoN z=HCDmg`p?&VgYmA`uG>n(XKzAWmoKqkkP$+%fkYPR%o2s!{|o&9W0zty5GFp0eo8WblF4n_0~O7EPU|I zdXzhp-CzI#3B5`Z#NL@VQ)q`)41O0oo|>bax&lV(e$Y{S^omKLNj_MOe}h(h?@Wcw zMtFzobPf;7l&Z<3@7>R>!?p7ko#hGmuD3DO+Xi!YXg9wYN;jyl8K_jgOndjmb=%>p z5l}(%9Gz)g_&jxpnaLA{dI2MCIc9oE_j3N*dY4^Kk9YerS9H}+eVQze50vrhlm6V$ zImBUqR!9G{U-gHme5V*XQ_^X8ECEB2OtNUj$1Y=rk5fQp2avDACW`3 zwjyGfvo!wv#q}X%U3o1Ts5hpqPKc^`pO98+56XI#%Pg?2hDf17`?+@>YVV^$;V-jf|Ut?Dp$iv1N!u=wV%)&;^7NVZw zISAt~6n1JcoYV_>1`|EbZas<4KVzFuvGJJmmcu0&AKb_K?)4{d2_A^Fei% z-!ARQJ>gCK?BP0G-YCVX#i^<)H4#<(vdMD$Q^kqhwQ0Cl*rF<{d`Pat^zF8)l*Y@c z9yv6iXf{YcCLcz>>w>fx%74rDZ_~fgQPomYsMb(39VT@@SNMyYP^X@iM=2noRcG(4lx(NUuj0c`z`M9 z8dY+;cX1(pWXDNljH2{QC&s$zLUc#=joX*7;_-mEeq(IWlZOeC#iyJg&_7dfvsdc- zl0#%GrHo%F@Ft#s#rpNHHSUT|=@RqXFq6p5Pn)?i=% z;EZEJvewGO=W)QlwOJf}^4syx>)-cnVmf>^O`SVz3uOoiW zPPw)p>6sQ&4d*e%^H@rXeyP(fkAtY~{P#g|`_<37^S2LiUZ74*N!tRo78+d`tjOdK zB0#EejQT7whum*G*ffUO@cnSKA_3}9j2@j_sY)KEBpqkGqHJv5;kujRfs*Y`5&?>- zhq;Kx7p3?&1D&-u*cfE1d4*{$#P@GtARrdoHB)`{X>TSBCc}xLX2ZJTX!2%`x=tI? zD~=ij0-Y5@fZx(sMTXTgt^@?0IscA1vD#wYaRPJQ(?|l|-^wEwHfXUtp5qMr^emqi zD<1j!js?<4=VZSvY>RLy8X2=S&(v`G7RdNI(5c4HXfIV-?ce=g{rMWH3D91>g>Z$e z5^fzPI4fezu+|!a9)~@-kqpj_(~?~tFNcX=yJEV zD12+^bU%J)MQR?CNi8Z3u^x=4Ewf{9Xu5V$LX=13IRpQ)YUWpTd|qac=YEU7(_XI{ z$~9O~<{eOfh9B{b7WgspkdC(=N%YP~_nQICbTb&Xyhp5#re)x!6)unu6CaO;G3I2* zE3smMs5B9gtCBkFUZmiyD(TrK#Bz;?eLfiIxqf^-d{kr{i6dxWsv_&Nc+r^^A%{%L z$m;Rj`kK^gOp*MQR;rq8oAIL(cop9QEPMXdZ7zw1opqT!IaOlIJpFt#XP9sJU8rwGIC74sMAN03QzVL_Mb31%`3;$%;4$6WFP z7M`LjtTY8B+Q`a&q(>mP2x15r-1gopO`lbG+ht7X%D4BI98qpmFwb70 zJCSh&A?cKKCPUSPnFB~Ua*+jz+Yi=xJzowT^vOrHk|)9?Uw2Q?Up2#9$s1k|hYwzn zot@;afBo=IzlFvv(`aMeqA?w)*2gEYc9T`6B*dq1x0dDC=2-I*%A<;mWIZ%Cx?!G$ zH0bs6a1URzC_t5Yt9m46ch;m#8bVq1^!-3_Sgj+nrCWG;Sv#3=+Oe3@uPwAXDJ@cW zq%3=H$|QPri};{gBecXxN@?OJKqgx85FWx81gFl6<1iNtPl#D+a(<~zFlMoq)T3!6URn`EqbtrJA>)o?_ji){P4K)?}dSHXV zqz3M*OOPwe+&)rN``Umi(@%~3jNfr3LAhlzI;HZr-&H%pN zdlNUg1aD}MJ8;|>YRftc_I)cDvo+zhS{)#+;g04L{1zmr78x zbdhrG4_4H8&jlk$|C*Tqv9JqHq?c>vGS?*5CM_(;--q$q8L1Pta|ZoKhMZ?!|1Grb zLkc)*KQw%=@PBN)|3D>_oM*2u5LcM7#Br7RoWL-jExzGd)`?{Sw<^Kc(0(_)3&L7+A zxJkxCl;Qk-g(bh^od3IUBp$7KRICy~@n<)iKX`+gQ9TeS-`gepMMcQ<+wA=>dr{hm z9nWeXV(ujW6W$2DkRjMBq6{bYBAkfef&6%YTiroeOAi8zU6 zFZ_Y__=r7X21%Sd)$_ZDYcoX@|4zmHXI)ZKf!QPeB>LoEo_W3O;oRC~*L^k72)~m% zsUkMXlDc92q^lq(#gIC((bscz_@Km;RtlW;3n>Wr|93xr`{w_53W$BBC+n&gGItH| zTHNO6X)}krPZm8J-xb1EIp+iPzEVkbi*Fm%4oeR*R5Z_(Zv6^Y`g`*%%O-Vet*0{PbhEuNO>Ll zEl%pP4u0Dg=#7fwR?h1;6&dr>tCjw2=hsZj&dY_kN1$52Kbx| z9vY?*)(|{y{tssN$JZyU0N^EMJNFw94rWaH8_N%?G$n}3qTnjvTMw4L?fdJbz9HHZ z>J^h3OyP{?Q^H#};F)O1&M0=8 z8-l&z_Z$0}yhySZ1hIIk-INE8V`ak4+b`t0G8SFyMO(hls+{-LcNZo>p8L;1o?pMF zT*zqHaBesS;!%ko%(mA_7&xOnZt@GufniZ%Z8kekSgrc1e4v#gwv&tWe9nslLUU_A z09rV2u~@E+KcL5@dMUJp0#vN6Hvws$Yz(nHVwM011IV_!ZaZGy0yU79J`=(u);uE) zSWS4h8sKM9iYV7I$tQ(&Ki&jZ zIA6Z>qxfeNdC+W;xKU?1JZh5A54_X$&>)=e*@xYe+T#B2`C68DuMw>rr953h~(GmjxW7`H~s@5j{%*sSN6A8ZM=?F7bH>op~;uq$7;>lA@^ zKWzNfVwPsEkQ#bNTcD=g0mMhHc9q0e64rDFbeWHzKlk+Mfz!hLB}3LC!?1n(oI~q0 zfHp+SZHo1N#7sAJVmGH-XbztEkt|rYhpRd5wX4NFF&hh{?!s{KNm-*B0k;Y7WnwH( z&XP9`PN^c;bn9Mh%8INu(j@&W2&+$*kq6c!)sEQJuFHlj1@F6$oIl0&U~cF1rWP*64ClXm||f0D45ia75UHfQG>sm2QgI-D`6% zGuPtOC^-X8yxMY`ahjEXtC!w}?hJMT$lgc28sTZW? z(ix^71CZFRdDQhG4!x>0SFNQ(EVE9@t4G002eq%H8&eae1ZcnNfTP^d@dWqiHsjjh z2gQwzJEF5G5{DW3=&6eUxy}T^Q18N4?hDzU7N1%DKH>6{sqy$l%EB%4>a|+3WYNm(&KS(b?`19AfF= zKj>q%O>MABUb_5eyKSNlvEl*TpzZ3Y&D`Sziyz*G!75PWoKVK&SDVC*kXBVnFn*n z!2J+FgFGR$n6NP0K{J3%6gM2jlQBtJg|6U9gF{OFIiz_vP60RoJYhJgLl^{~KTA6P zg(QF81hsTh#Bk{o=1pG<6K`(Yd9t%OzC9n!q2AlP(WH!uiYXsam2!;}9ubs7_V;Ge z3BBlM)HcLdEq^G~JKLJf zWv-&rVw6Jg)w#(8P9gXj(|-_p~R*du0F&b#eGftWYdyt+A5AHGyX zwz!fCDs`WyFncI8TDV@u3sq_&sLoZ8FsxtXy%P|i?RQEe^K0gjiy)V5Ak_d;UQ4Iy z6~|>Nx2$%Q@Aaci=^9VpnPu=v3ysh)`wnf!6FoxL7p|X%rn#dXP?yfVkCvx}wRgqN zxX%XGraj4c$PWm}^gM>8I&9_~#;KF4y>BlnSz?PO1Qx`FZ2NkJY;@5Cr>5@_K{Q-e ztC&cnpCt#eT&7h0j*>xN z#HeVA73+X|Sdw{!^O8uU-z$xGEsKD)*;h83uRP5%;=cq`1{tzlmWO1!a&Mo;`W;0& zO}8quQX16EOoo>}Nk6AiYv!y|X+$lxRF%~Kv~{w+rvFzqmqxS5hus&)hauY4uO|p% zT9a6MhpKb04dFON9k=3_zq!u0gj@lRZgaVN*OGiv5@iEz#a{OpG#q~f)umF}N$EIr z6*{F6pZMC@_K--}>XlT5C1${!m*eTJTn&2l#~H&U_q|BB*!NHB-li^9at7ExwXpnn zYu6*s8~`ehKfvkLgH1?f3_6FWKk$xMnMv5HS&(pUtmeCg<Vr1;dI^~jiZ(mEic+HrPf$w;CIOU)T_<(3rxMUi#E zD55Uf5Q2Z4OdQU~3R@553-!f>o(<8y1pYS-lPu>8B9KA~UG0he zw33FX6d29hY~wF7+!y#F5acx?55f^GL@}+)EIDnj^`p0q>8v*DHE3~4=fz1k#ORV# z$#HI7Kwk44*4I%D%YA($UN#o6Knoq3G0(9XDFe7?v;9x@Ih0Yy{+F-A;fED<{h78Q z7#@Oer0-jA;z{)gD6)U?`x3`|FmNyd70Jl>H+^RmNLWLDV#o@W&51A^+39&pPYSKr zDEoEGwMiv(M7;yFYvTdDN&ty81m1k{Q#Uh5Eax%!SUgaue6wY7@vD@_!B$Ri-~1z` zFuOK*xln_8zZq9d?%Ru-bUiAWn0hDpLOXT}IEDb5smf*Jc4Te5F)h5@g0}BDKrCXg zbk!(cDpG?-Nij>l^Wb4+mxZ=1 zJX?v4Y&9JRd(GGYa1rtkg%q1(AW`dDuFWQ)LkmMps|3>RO(%KK$tQ*>LM)K6| z(;)0pO%Jw1`F+amr;^arD4etWFH3z`mdN*>B=`^_+Fd!iEN>2hfXxZq9ze`WN;gAc zH-rh)(d{EQvUf^X^2`Nx-Lc;syqne0iBNVySN0=ZLbrtB*XbYbY~~Z@vaTc1=BkB< zc@mXuDXq>sD&kC2dAJ+Uwo|LvD~r%Fp9cTN13~9jy~Vyf(%Bc80Zcy^-VaG-A+YFP z>$8)7sy|7cT=!S^x0W=Ry(A;2ZKQT>j@Lv^X_H(UI#p-di0&efg?rapjdc$_MELcZ!jTfXzocRbxQ`wY!`75O z?s6W(8!p4F_Mkz^dI*=?x4bHsylL0HA6s|TqON?M8d6F>)@wHH#TUcV(tK6I za7q8^A?+MTW+g%DI&5bi6FAB}1baJ_8A4DTkDSN!nkDmI`5>QLM<@J7**5l=ZIIK0 zc3wsP3gq7e5QQrjSm@@vs%hFqq)zTjxo_X3o-00w09>lAc^qy1SS4{xk`zxt_fOI- zQ?9|j&!@+}2fN21QXVO>j4MYMwpT|+h=ROG>AxECb=fr(u*$gvnT1&V8W_nrtJJd? zvyeQLIz1^(xX`59C~PRCfFhWxwHH^V?uz@#7j&Udcg^c-ebJ!*3XRnN4%oka^JD5Z zpfpVton8KaMc@A%h*z)ptP{X@a}9o~R$Zin>B0x8xSqy~N{t3ep8O&__$2SvDfQCi zImb7;`6hYgH2$kOn3;{t9vT5gj{>gm^`CE_T4o*HB7iT=Deu|G#%7{Va96!cjxk- zhv_?%MD}T2xnsMp?APB1X+7!kr&GtUTnyRZNi{$HIL)fzIM`ew;V;q~SM*$F>i9Ds z>5X;5U1{{g6*1>*cBVB8Nq=)M{8PxUY-*$Rr|?H}Dv!dF_hj!#cA z++AV){%ds?&QE-6{hQEOn)yDS1DE#c{QKXmsqvVU#JLF4t)(HE`EM;j(ZWh*{d9Ue z<)&jQiBd1FNVx7l91c}=o>#Ody)MHiKH3vBw%%Wd)3WwgaJ6J)2$9 zdenBWe=?|GA=$b+`geFSJRA?3DpdHC{RuXBvjB() z+ti<4EYzu&v|;r8v*G`sLth$|&k!6m>YpE$`t!Dq-#o_)Y@;HxGov{HymCn)xucuB z`n8YU1j&c8&_=-*X@^Remj1*~{%Q6o3LrSAQRSt4e`2M3gv1tf^G~R zbbnO_0{9!+y+3f?{}3v}pWx@^jfrBFq&GtJJAduJO{P6KLr`-`?!upI#^;^< z50xA9wD@06Sr4U&{wwtBO++9XdhXA;P{_Do;E^Y?y*QE(*%CPgfIf~r|Ke|hinvbQ z=sJ^yl7)7uNOsjn!0a|_?6cjI_KKhgnnPiay>{Zo?JaAk`E|9%!8XOe4w`vg6~L`0 zs!v)ae|rytH{K6J=5Stp9gu&-sF5!hbA2~uas6zkl!M8zsJyOo#?r;0cglwc5jVA< zf`^?6$4*PF&kXY)it+7@Is-xOeBgnbkba?B4&>m-B(Ix7v8Asj`Lq&uRKE<*pl#UC)imjQk6` z_K|ic+2XcI(K5PvES`j=8+q9FpVjlPuaA7dQUG+>_TO_FK-cMss;U&d)l_cw0rdS%y?sQ+yYK4J=Aa(xW)HJv+C`S^Hr z8v<2;{bSu>H=wxXs@Pg$GEB=o+IRupk!BqykH$P^qa6QS3Y*9vmk|q`$Fs$HZ@*+{V*ut^oE}&K~ zw^#P@$?e&9-?r-ZvhGi1(ERi~1+W+`9b>24oD~2AMFJ2pU^e!XWj%Pcx_)h})9ra~ zY@J_`o&&o2!TU0e2E7|PHBM}xrrBK`j7P74y!{6vt}mxY6#Ch=9|5S4_KTp>%!M-L zZrca#hsG+PW?mI5k{7uv?H|LMZBe9e{__xWB5 zJQ$TtlRAfp>-7r(T>((VMcg*frwHp!twOtmYYC*{+V|bpdtb&N@bQXKIN&Ay>=z!l zyhKb4%rpmkFG{xR`%}8x7XkNzR5VvNF4*x>pdQXpXpo6>8N_RApXRCnbTU=K4V7(SHg>d+Q!}2cdhR$`JB43(IjX9XMI!5N zy`nL1$Oju!XC`DFC1G;~ux@u(+!OT%xprZ5^rQ!XfTi)rb%F-fM<`%a_&QX((Lld5 z{HAXQ^v--R;ODwJVw!qrlMpUzm=5hpeu(X8Yuk<}EX;vt=KStZdUt{##&OV5g=+>O zrln#vS(n%0Is`~~ErMga@G?5vk+KN{8kl}Kg}NF}xZTmMERbsy#pA#RD8xwKvmk#< zid1y%>lHi`4Zbt4IHpY#32$&{lE8{OTdASwp^s6Uj4Prh=#-GrN;{RHYkZl>iqIex zmZKT>28U@sYl;QHw0pWb4uxXBJ1ycB<9K}R@O9zdumP}jUwsW42H;=E`1xDrt}3q? zW=8HAPL3ZTnt(tMaG zBgpfx%rsn>$DAEF%wrEW%gvGhYP92vj(pyx%m1nq$nIx8}`LRxKST z%eCc`Y@5*cGJY1A^r*;&&50Ii70_ZTvogKgHBj^lHtHV z!p_O_&ONdoodXG+Iq!%BK_^u6K zBmrq8vNh!Kg^R8u2m#aEmPA{?`k@z_>iNPkmM*N7nOvG(l>l!n+&VfwthAXTZt52( z%@;z@aIncrD*+_jDngE{{k4Q2!r8aof4;5lxRbMZ%uT5XC^i@1t_p`h>}jzT|A?En zxKCk32om8(3F=T<{Npov*UsuW4pQcO?X6x$xxS^~5Q_8!V7f5^ohY1&N|suqFH?*O z94NUjktpO^Q%xvJICBQbyt6yB4XlLx7`X#GFXzm-?;vXTZ;_@Z3R%9uH+FhB?Pakw zPu!X|*slFhje@|uC$WtWu{sx3XA(#u7_sD<8~KW>ZKrA@D&#|M1Wd%P1df~I!H2$H zR&*51FYb>571;uR6-!y6(ZXxiNjMXN1be-ckhR!yUT(qWln zXKFMqGJJ~eQ|NNG=rPg;=@I+x359+9(zMM1(o!$KY1nDxvl@9GVaDX#qu_&gd$VsE zgId>P6s=sa)9na9(rKu9AwdqH?&Vloa6ql&JNm-;Ra3@=JPvI7?i$@16dZp8AAiST z#WDws=>aRLh`u7B(|IozFSj1v6dufUSs9ms&G9_p*UWaX zgnIJD8*R-up>hp@Sbc#(^0YNHsUSyQ$=Jq{3TnQ3E=Gk9Q1k$-1w|!nTdZE07tO+| z*Z>cPnhtGRb4wCOk9%KClLXDr>TBNB%=SsLppy&Jw7AO~LLhmvZX1Pk;vKl&Tu#-m zsM^6JoQ2X$UD@K?|@`MGzr@cDYhh*0Lf`xhRHzGVqy&{&?eU#q9NLLELS zcJ5r?MU6nGEdNP2;^WiF@%q}XS>9?B_V6$Ed*RgyW624-$@R4pyGftLw2v^$V=jp^ z@Y!RR5vCqbMeX4itR3G>poWx5wgJHvZ%T)Fg!7R-{gz9EE1~{(F4xYAT67P8m>chK z@E!N&ek}Pa{1_vM)geAZaq7m9Vx3@47nIAEw$vXs#SiH+GN)U>$gg5$``d;^#Kr=p zYG?@|->26~JEB^cR@`rv?&&og9eM1itCY_r#ax86g{2BX#)!^DTs)tVNz8rw;|k)-Z&G#Kzr`NG@+BZ@vh}?gSwnj?pOJR9?R1tZ;)(3NU(KI- z#wUR~JvejJucz#nX2umVhB6C5bP$mVB!kQ}h0RJ{li-m_0Cm#-Ljl%QlYlf+66v=moj_QNtZW2S=~tLMv^9(o>jWmU~v~`^U_Dz>g48C^L5E2 z2(n2g^u8{n%gV93spCGhl4qD(YLfCISE~63+RqV}sM_VRX#KlJNg=Kf2u_bl+@j5_ zw3O;H!{I)P3z89#E#V-Vf(;1Dj?iF4E-1ck)geJsN|VR&L|pfV~c8w@Q=HfsG!L^R)Uw_+=c@+4Bh3qZgHHI+J|2y z!sz|sYWEK)fgy2)6dl`v!d_EEnTKS7L0aNh;6!gjelcdyOI0wd*@mM7(>PdL6i0u zqv9{-MqF?C2dkl)$Z*byA@8;O3`|UTBVnUW?u56*Ah{xKD4+9=Ho9dA=+CAqGqh83 zfiu&0g$;`zZB>&dwaBxm%@dB8(S@3psul6i9Lh!WIl_Lp{(w7R|bR zkU1h^JvXvHftkVCOo&)dFfp`>-hWwpVj*ca^LaAIUh|>-*oj4cgZ7E%y`>%bF6ZL@ znGN}|t)6SfQ_E-F8nmaw%)Byqq_Vq2x=t*nbDbF16Kw$Dke%WM(p)WHHN|X78ME>xNTca5}nLn~Y>fsn>y=mxN z+$p`Bc5Ft@U7lBIWG?*|y5lXKq;-Gb|GWS?Zvub1w4nFfaBd;LqQpHahB0HTiUx;2 zz1&~V!msKnh_3s1*54p@`x9O`bK&jPS1f5uu-=<#gxDaLfBzi(&r>pxrNv=hboc3> zCuIf7bC3&LAG41+7E@0i2}lb4_4;SdX$66}(R^9j<3E8RX#ut$52ewujh80I4|GMr z*})K6swGfyrRg`*-~VyYe0o7r9eK#r%0GYXvlQ!1dRG5O_nP`{jp>D^zYfHX1mtDf z2fO;k{0YNciT1vFR-d+T$k@k{?i~C{^P9gC!I8Vaw-N zQmvmI<^Sy7e+J+atq6i06_cUC|60;Z}z{;uL?oNUE`pj_sO52%n#bDC2VQ(JPgumN{5EV z*1ZycLMH#YZ?{swm79@GDS!6smHrjbeVV+<Z4HlEg zTwRmZpNnbz3g`sW6t38ntS0wm-5(UN|Nhg5N)AlnrKGzgG=J)aZLWeonD-dIw`*9U zd+TlIX4cF2w{d*=e*4VXW2>w3UVnBlyqoAAO)u9W(Yz|V;L_dS75M+_4^c9hg1Ydt>vCHeH>1db;**; z=E9%QJH9tB;?7)fsujo3{@(E~&sfj=9J^*kG+(K3yX`e|Gkcr%*Z6<0@kl-}xg9Qc zcLjd$+gb9n0ke!$kJi6cd!s(T!V)<>x_mIG`j#T&QSAlhPh+zm2LIhy5&OBb7q6 z&BSda*=M14sFaW#zwwOoI~44mUV0 zME2i&T4|%rp%mM#Y0Ha(bd}n6%{_R3sTX+V^_1EU%>mQJcb-d_7bWTd=>TfR3TxQ? z%KbWtB&1nLS^))xSql1jDU>?qj@r0;N*IF_BZ>0cMIdWcr9m0~SX%*jR0hsicey9A z$5zlwL{u5?-}sa*D9IJ55+Zvp4$lSI;C^P>;AC@UvH}9Q=LC5g(dOL4+~+yvxdpr~!)aO|F)x(zBukezAFzdg1md&CSnRcQvB6mM0*|5ZkCsLORGi_wHd z0EKX_%!})dFATso1b6&uIX>AH6~!Jov-@RG+`Rh>M1C%#$p)7vm_AG|6&ndgDkHy# za~C=Zkr`$Y5Qv=7%bkDUza=xGfMz6ad|#_2Jq10M#JNgR_ldjJkQ_WfJqlf~-oRv5 z+Ms$M{t4E>ICsf8Ayb7^pUs(&nY9&}Xgyni4~k^}w5EZQU$%jLyRc1iqfg!7Bs6>t zp`N=;KgOm=m1%sG%rkT!^7NM1&PrflNuecb(%#hp7XVP;;{t@m1o zA{^ZfBbaXZKWZ>;Id$Aye+l!LErBYgN+N)C*fU=xrQ8_iP^n$SS~!B>Vh3Vh6v97Q z3_I>c&B&<)(6|p#uaBZ-N0@F;`GY3b)jO*l2rBolc%?+;+gDmd ze49g{8tftaqc%qv_mJB|dyqNpB46|irc(u`UQR-XA-4NwNONN*_gJbQVUleST&YLXo*BT7%*4cbt0Q18IWJO1Eg zi>oI^aLU20Pdb&UAFQk7&q)USB9AS7loHBn*&NU5mdHg<-iuxs?XcdzwdyQZkkD2TsPp}#A)2xd|ShL zi`H)Fh7H5$n4!nUS=-@~4_McNn2YQ#6;>+F_W8=^-=qu`j$IjL{xz)FTyVefq#d%G zJ7wM-YTVU3b#TJD`eAroh!q=EB3Q`6M^%>kfA>&v0_PvNgp# z+DDC@3bOx_aZQ5xRYZ-(UD5g4=mY61pORBwlRUQAj9!2wW{OO`>2NnB zXMl_%X2fjE4JLH+)Y~9)!v+mT+5L0fCqP2W4^d4N2foq8^#^s(Q_3V${8nICj)KavepC0idnzE?;q6)M?HvpUlM!F#e|(-BvHJT%oM zz5;dShtCwdPl$v9sA4AT2D(9Y$331#)jT8p=&;Uhmr>Qj$WY_stE=1aMgB!}+fHO- z?Cr{g>xUuN2UA-^lRi;PzM%`MrL;e-H+mT_QH_;jlglpsPM2=+ky(Y?J4k!*+1Q@O73%2k$YHWtc(Sikh_8%UC4gNeqS9hpY$-B9#$Ht;w(Iul zU;=zVq|{(#jvvab@<8PQf4JS21)TP5LAW$UCdj#&J*Z-Itn7`UbwAQ{@%@EIjtjDg zgW||;SN<16i>}i7YAw=d8vk|(Y~-VOgI}}ISfGD4-py6#Z*;~T&Oz>d-}`{T6#24} zqL=uu+bo63b*HNvrf4(GF;#Z*lnNnjZS_xZh~EPF{cs@h62(mlu(; zny{jX`1lAeXj*9g9YlE-PA&X&! zezYqMXL;-9ZOXYW!aEz&25BE+@3E-<7Ln6+qU!CdIHjVdXF2NTlFsJBQ_bz zw{Ar}`DsA>*6BqN`a`hyfdIK=4jj-xDKdDA@*yaKTY0Q<5mYEu+7i=QY-dAw9^hb5 z!ow3HiR?Nb^;KkaiMMjLmC(CMsV{I0-J3Rt>tlvjb?lRGz5Q$Q=k^qJSF{y_qRv(e{p3o8!1Tv5hGFJ-`;yln(yrvLlRrAb(1#z| z;d^*8m^`P#DteMyV>@lV62VpRx=C>bYISz}nR$1iwzQ)qp)*=|YJqyAtcIkV%>y1I zvp~-n>@JsBC&%T{8NHt1X2L7yGc8~7$$zp$)ckkoZK%?2n*7^TsAswz!8>Q2)L-LY zABg9G8k4^Yil{a#v6FU4c#X6rJHKVW&;$>4(B5j$t^8;R-p`=nmlv- zYy05sC@rT4-b@e1=ll0G>)0-S{qdvZx9P=4=AKRobD37LN36wDR9)y$P*!RU_hQdZ-h+Fi=`%wHcK0DezmDp+ie!rs&4s3QO|Lhkh z-@hg`?k>FTAGj4xInz!C2zQF8~Aixd~*G|Kpe)tXSJVFWTNlNQ{$*>vc6^q10eg z`=(M_=tRkJZ~~X{iWMby%nh7wp}M7OF*gWq{Ts}*2=EC}d$D)_g>ba71xn{5L#8@^ zML7g?r4p>J*t4nBY8J}-52SY2vb0Fo{@x6&-I{Ah_!1u!olst}RD8mt?)5IqUGWCr z6WRZ|)XN`tz-??v29p054x*q$C&qMoGlvi%@$jknqDU%CgF2zAU2>fUmvl5QOKmr2 zKhAi;J5XD$o8PzuzHMm`Om@l>e6T6}n(gTh=9BQkXmr;FWOkIsh|w6gV)f@K*5$tM zF<+W6OYJ7H3?s1_blHYi_Md*_E4WT10Rc4R)J^{->v|n1ar)dgmqYil#a7e(bTLuT z?Ky<2EW-mkVn*nj9rA)e^Ol+nt+bcS@FbBz<~Vv0#&6ZA+djj0a;WS2qfK%&o?z9( zqZ!Jq3bpB*+=xtCvI$DCTegm?+V0G)W@S?YnMXM)>tjZ}pRJq)_d zI=6F&ql5XsMhN`)f5EZp6!CpX2c`^jO>T2}q@Tt;lT&4wNfU678+2y0l~?%iV*V9H zz4rTx{cVIKO^%P8E~D#9I|Ft*kAqip>cru*Zx@Vy&DvbH=`S3K?=mA zSO>ioQt_pv2IXkQXM*4)x#Q1-GBaeemmys8s@cqso=qKF@$(@`hTlbUs#bw?i zB)vh?+SN*nQ+!2I=6lIDlaFrpG}OZl%=GLu`XSDCqsEc@-0SXvF5B|q0b8_zelb^F zSI8X(MUzmjEma$XZdYzlW^B1>Ip<`iU1hiNJ+0Si^GGf>+fL`f_a$A8ecq`;u31kN0^Z%W(g|J7XT=sH1}N!jpM5 z0>5{@N}W~8Q=>YZV>8$n$*Vso+O(Xu-Xam1FSeeREx}WHiC1+m>jnLg*1>plRqFlv zLSoX?_j~!NQg>4yezA9OwjaG^uwG5YIQhsR2RpQtx-7&!^D&{hKC!f)gR^mWAerH@ z&O%d_Mfbs-^boFM`u{%{N`e!;v?xynH^_gd(BOl83KDEa8#CFw^_ECP>c2nsX_9P; z9%{<8XcefL6sr>D)E_WSUV=z94jo=jAcODu38yZby7+rn5E7FFdtdJ4yFGRyeo@=N zT%QEJK&b`yf5BTesP#$|ZJ5B}vy86eP9u-dTiJn|;v3B%hUUn8!|IoQ;qp5UF7a5(_yDHc^ z!O}3XTP0n6#MD{M&BL#+tp(KuYpqhyBz@;8r`I=(CpPaU1>X(aLVkhmTf5ywu}Q4U zR~0e*?vHPbknhKFu^)9f6zCViS30e(FHA11pRr(rOfhY&qd7@)5J}MvD@FlXHMUup z;bAV>2B|g!J(+)dH{(XA;0BMg*r&=q*F)F(mY+)#N+?ooG%6G&J35#1)QJ@?+;X*H6bA1Q!)mjziY6f zeCo3@uRg0u78!5Detw(*u2o61Bq(>Q$c$Z(-#Si@ro1Yz_bbkf<)SmM$YDt!{g6(o z0J)8k>(TEOA^Q12Ep16X?}Pq$Z;5Zo`Xh%bgPbYz)kgh~P$f78Tg5qV)7Uwwzko@< zt5sQ*x?@0orH6AFL&~9Sp$5x(e{x_w$#t+0gJge5@^85nUkm`Xq_H*kX89izNRt8} zaq&=x8);&`*JdhtL*0Z(Bs|5ZnLkRf?Vo;>O=aPm%v|-;Q~hoLAL0s%uU=3n@qJ*c z_bJ|n`1oVOeEsgqjVHKXsU)lCm~VC|=2b@hTLz8c-T@6-6+Q70M6&&v=uE=#7ToGN zA;bQKrL;n2w>vF;&&CWsk8D;`J?8?Pl>ZhL^$$6&4*91o8LWf+@3%hi|4Us%CwNJ3 zQJLeI{%^A5@|(c>Y7%kvbiLc0q(g@l0VQ7Pn+9@Oo7RGF>h-6_(tPmIQ$Z|;U$%qe zT(X^Re3{AP#jA>chpD3NAENvCOQ>)g?n)E=F$XTk(=q$9;L0B0+xUZb3uI)VV>6Aq{{viDGs!C3 zzS9;h&z3RKh>FSAYMptVejSEMv84$MH^zo)Fpf7n9^L4fi~W1O<6HU~xqoZ#_H|1^ zCO*xZ!-5TNF{C?-|dpBfc}wrhv)7ZP;Tgd!H+SywyOy z@W%lCXO9WA0Waclbvn-8pLrjw=>hpZI4`5DNr!wHh5+Q~m3gykrY@m)c`jew+OuOM zow@LuWp+qEe8WmmoVWdi#J3W#b^Babei$jc*|tQZ$hL;7U=BaI zpdE@h>}G`-4jtCGq<>7)znM9@s8m3%s;O26mjCgJN`x}fv6%(j&(Qlyk3pj?GA#By zuL%*_od*>_^7i#wLhF0{TbL~cS)HFL9mgKvv20GCm2lxZvcimTW?GA8JEOZpC5#ay zt1NiZ8Qr37ZS5_x)qYiRinz|Csn6~h_s>)q3x1@7XDt5M4B600zX6Xg0C&dN^9+|7 z*XwSZOzc0ewKqmW0;d@>zX@7pK`n=VkW@L1l6B)Dpe!*!6DM_yO_b#G9`xznRKU9g zoPmBjMmC&E2{12ibLQPd#RWi9%x5`7aO|*T-o3dpm=+7Vp=ccEAn8)}yG1TPhV(n* zDK@B1u%nU7Q-itBnK#xH=gaQ1@i`7T&g>wW!KVhI(~dy{Wwa+9JoVY+C7M0yGgoBT z_-)E(cZ{{$uEu^o2()o}V{6pvm%VgQVb-rGet6b3C1Q%Ay8IW$J_llU%chP3w6iQA zh-l8dP#CH{KWkkHXSnQg6)_8pki2Iz7uS#1&>(FgeEJgo5<6R_25}F$mM?FVfw;b8&vYTX+@*srfdE@hJJ_0*YnKhr1YOnz?KL*+7MfVk zj@O?6)q3&L!AP!(lBw+Nw^08afJZxC{*x`?(vd*C>sCiQJeO$#-F6V`SBZMn)`J+4 zaP1O@G1EYKnocI_cTKNL~Rq9pKrj8rHX&{Nml<`WhOXf)UQ!>^&d#qqIYLe(ylq^2u?x z9`PC0y#q=+N|j5A^&ARFw}#K=8(v)CNLj5_kX{b_^3>L(Kf!0%lTJl%Zn2B|HZzG< zt7SYOR_k0~=KfV~iQU2yz2du~Z4lx`7es#Z_0IuYl!07OZ!-l4Z@ul}-z~?B7~+A$ z$<(M)o~r8}lx&U--Ytje5phBs;kjzCP8eG1H$jHT(|P3bT&~rEA)@oCnPT#G8R(l6 z``P90!+kWY#ionj#{GL~;b{?nI{*YlKjbON38T*?X;HTmxs(+x@f!v;3PzanHG*sS zoJKVDD$PaLJDkH@RD^InN>1laO?UTp&;CM?TI|->)R&B0C7pRw^z?9L@-yN}_+23V z((g;+5e1QVuA*j%MSEk^_H{3W*Z*4`;>_~pT+F>#jva{dU zuz%r6=9I5H6Od49(4x$O^|0Jtz}rLS!d=hjuJU`7wjl#cJu2T0Dmxq1!dlyL#W1Pp zc5!;cKF?k9Wr!ik>PYRLey**R*VY;hUjRZOfdkvERe%>3-9M3;^Ha*7ryHI>d~fyX z>@uI|!73gA#8oMtzP%3s@wF;JHjM_x=F<`0(`M1X$j<9KF5kl_Nnfj9W66BZd+!4yeiBYvnbo9!-OU;4IpaqqYg0YxTUB_tOH%)^5{8(J4?LFch|4Lz-e zsv3vax1PaDzk#HS8as7WX!c@f=#aic^Cel4izOrzy!^<}ws#5~IsFuYMi@KJnb)2O zL7nAf(?52KiW*KhM3oC{Brocrij@xTjCL~{ z0M6w$rde{hE55QJNJteiSna({E0*Vft~)5!_!~9TW|i%M*}iNj2(Qn&?eCf{Qa@dK z6rGF~H%GW!(lM9h8@qOqGu9lD>NIB_eBhxz&K7S;M>~6=y94jrPCpvxye_^TSs&5O za@(iyx2Mwf^4Z_civTc$UrD~Hv@{W@DxB{w3|O*>KGooY)B z4sp;PEKkjy@OYM|Z-X>BjdQj~^T|lR%{*`QaX5`@MN|Q>JiL?#;;>+S8V5NY_NcLy z1#O!Y@ut>ki+KjkGxg#^)W(Ss6g+K5 z=Ec}-r0?g0a&ajX_S02l(ZKICP6rEUaHTf-KXCbHO5UlXQd{tAKV7E5+B5Xzx17rs z-!n@`N>H`uB%}Rxgbv}ki|d1ze4F^nTY#7I8~ca2zVVUDI2n#>B{Cp0yN zzNqp(6f?!tyMSM1)2<5voiv~OpH+twQMqo>-G8$$*77c`pM1jE#EhZZYCgulhiItv zq9n_b5R%AkoS=B0TUO~@&st_mA(4x7-rtzq6vIpy7W$X24VffynW+)}x7Ah4?g#zz znq30-JItLzg;W;je52CTUC?K}gaxVGY%#y98Zbl{XhjejyP=DQB$}ybaR@tfTG>bq z+%)~R{PX&Dw%~?DvpOED%P*ki&N%(pgu2onOG?MNh!n;Ru1|M9z!J(4~CK;qOgRy7yMp=~qHhwzk2iHJ-gM8+z`hvfzrgm1Js% z^wgPjS|gr)ly_^Hw-y{TKuP0E3ED4Xs3!e{WY^Sc2;+8lo-aZjxE#2Oh&K{3Pwa-9 zpZOC*d%TM$ovA$a#T;T}s`XbW7`ohNL?U~}Y^Kv18~Fl!u5nEI6=>vTxeC{eB+4+j zvfss-7r~p?xHl8lGgn^#TmU{swI4gz{z{8I8dhgxgoH`SmK4T$Sk~X{;bG`8gyf+- z7kXf5jk(B1ikx}S2A?!W8R;X=# z>kw&5rrvAv0!t;NQafJIiR4{_(=$;LYBNmBjesyDy|zUx5n)i%6boU&`N}#@`E*wn zj&2k)5z0S-i-i(`9!v{Md1luE;m*x}j1IfgVNT+&^V=hdBcS?CB}WwEyBG`j2>rZ) zpi7iM8##x8i5W@I6vtvI{xv{lktz1jH-R*Yhd+0F)X`zCE{;z+(m3t&uubwNC+=uM z?hcf9qbnb>eDM`fwx){r2wa$#PkhAa2R+(av~kvbU}F07!_?e|*6oYgx(|`t&$P*c zHA}UVmw6vUW*z-3K7Y!9a2RmC7SDNx1pGxshNRdifX7eu*(6{2pV@WOa^lc!PM1Co zd&6kiE*tYNYLYJ3Fqf#*>X!3r#ZGWTd?ywY7w7LrSy< zy|WQnQ?^q;b+Xxyw#P$*|z-Iz}OINMdK+Nvq8WO16>mWL zpW|cadM!G;(A!)&as&( z10A{7hlO-m330?33rCPa$&D{6i0Ussxp;%uDNV1HY=gzY#r$q-S^_!m%&A|L3-g*t zmq=ryk}@;lotJq;m;o0!h1P0wt#NKl6WuR8rt;0SLvggQeE74|adX6eCDp4(Gy)4i zt4zl&tPoqE%y(vXSiV{U+3q}FnqhJG9iY&yVrykj=jrkh{+7eHWjgD^!CdztNjM)q zVq_wQ;L&%9x1CMB?{`}e5eN@7=iY0$|Fd<`GXbfg-|Ze+_Rz0yh#%?u1Qk{G*E1YYnm)C&aIU6K*-@WJ?VLfb9o?TZsH&YiE5|nN2e* zy9W$wIg-z{zj;f0vD0Ge4P}C=W*aHe6gPg(#zV&r*eoA>RmAENrk5YU z`EnvOutN56Np>GCoc6_SDrSBj`R?SN9||dO?-`dmd5Qjn3+=#U93DshPTElt$?8)` z@`E!QqlWXkufA-1W<&h0cz8#BHmUT8c8;lA9T37;W z9myXK&L$|ev43C0;t`9>ww{k9X1$V5JaiLgsn^Tdv=ZDh@OE`rk(WTFyInlM%>x%S zleV047xH>O?CZ5VL-#PZT-Dy=7_B<)b8@nX>3lOj7mkrdQI;%KA!-$HhP2)9);0+F`fh}@qs5m6>LK8=vkVYR%& z7IfG$Som<~Tlu4xxBQoR)b`t8Ctaa05{;IA0?hKD?!? z^6;qo%WD2oI)R7AH~%VM&hCL~x#IB(VI4_5x%~h0eN3}S{F`K8BUw!!oa2uB21XG|28K)J{u+SweE;b$7A6lue89gdB$3?2W#WdEa!sa%8W zqJK>GzBhkQb}!?C02*Q3`BG8Rm}AuH!J4q$ZHLX|DIPNtwd{&5JVNkRga zTs$Y>5s2B{=ce5H8-TpyvM|87soLNBg|$?Vl>tI65!Q8+WjRs^^r;!XDl#cKe_sA{caY1 z#$u%a_&2*}#U>l7mAQS*SBhWapLp^W+}@9X(0t2F`hegMrTrI1+^nXn@_yqBS%>kZ z0b9klRVHIMwJvO?>R5YuhnRK^=!el~Jh3&uU5563kp^;KPfikC#$4pm5y;GOW`t2G zK`-b0q4WI)1-2d=Vm(z&z%tH>F4sPq=~(! z@%ka=l`-4ACm(5azuJ_=ZF3}qX3LLOt5^$0Pfwq()i1n?kntbRY!9|t%Mz4*>o8uz zJKm6UjkE_Ad6@_6(E3~a*H}w&h@M&^7xfAp8l0>8Kn3*4A{I!vPV1R~;(XPyU2iTA zbD}YhAlM5KQ2%GUKu79(0GOv%<5K>EaSi;!K-Xt-IU(7)iuMRr8()<2nSX}%4H6@@wRKg%oF|#3N88`TIbUAa`*!aQ zSFL@be)WlLdjEQ*a0K-DN~8BzO@WC}K>m5eG#4fxexYn|YFAMQzUiBx!iA2;C+?6WRe*NNqBn8b(*iuW$68& z+YmqQL5SkN5Rr8WfQU3!I$r(LA@q_XFe$5!r@gdKf3GEGQ&U21m3K(1$F?mR9g|gK z_ggwuiShVyQ*&rNt{ApK9`D>^NL51IP3EfK+J>RG^@c^(4SrQR@ih>j=(CiT>o-_P zep%FWxPhtsr5emlCxVIJ+*CgxD!d{=4@kzhyf$Cn9(AV2bO6YHGJa%e6Yx@E?liB; zBDhUvnARKWroFm6i}2(>FWNh!2NjgaRL4_ZVJRK_B{XtX@>Y~k7|q-)0GuJ*LnHxZrdkDU%4t2g!b zWq*A>##`#V=WT%NHSqAoU?9U#4&k)XMM`3cc!@Rarsu=55eLfq|F(a6Uh-HXD4gs+ zXd2)hl8Do9UDjFQ^fKkvuLpyVEV8RRhU6#HVW zSR=1}6-?@Gi0Q}CTJckPUx5@K#|v0Kz$i6KUaH(=%;U z>D5{*Y|hM?0hSF`_r%?Q`JrCfYX#+nM>~=1xQ54J&wR_-aP3JaU;O?z7nFep;44d6 z{apWC99g3SB3CzseOj#@hEEhBhc{Q%%W=KUlzT@O^h@%biV3Eq^*;kBLrG0NS4bC;>@8dLM(+n+_P{p?PwG(Dk2wA9k0y<oPQQsUIe{Mm5wKfA4g@a33Md++_(R+BYOLRoUo^e^M;+ErhsSugxO0a)N5W#~d~ zQ1!-Rg_N6F?Q1RwCfc51^lWYYU#^wLBL2Yh*kyv{iI7gTbRik%y+7FJ`4<|ydajc6 z6RT+rs5BjZqo1RM&jTGxa9aseYm4@Iob88ija3_ubNmG7i#Zw_FZ&^jjCFUvdi#QT z8^MQYA#S0LDYKPA*J>*Nh1x@lu{w1_J0VLy;AjSc%Z5XTOWsoW7*KV*1jpQ^S-!55 zJVa_{-#8_vTY;J5O|L3`!yao#^uN)apP~sE{$yX@%F=xzK!Xa~%Q)l_4UVrx zTswgUsZewMT$f>t7@?upLHy=vzqu&_rWsf5?x$${Kt`?z`S38ID}pEJe<)XejpGecC>19-c%r|^FYV(ywA{CI>Yz`+fMM$ z`zqlAW;vZiey=~buJwR`PmVY-#?^(ZI=zja;At`pd5V)Ns|*Ss-~2Mu%(tB#g}m6w zzTX**cDYE((Ko1`tIyaNcNi~vUElwhRAOA3I#5{0kIT_d#B{efrSk$Ljs=#?I?=Yk zZ4!kfIP)?L;KPq>z&3iV~_Fd?^;6v(zTXx*YZmJ}qHq2xl} zev zN9S!%T3Q<(8Q*w)QJn6gf=J}yvLBzwaV?PwGVy_Hyz!U?6kjcz^aYXn9M%|oyI zpG7=O-IpfXTtzpkIqH3k@Wk;92i^yyb1*-A|Hnf8H=kYxys`~isg}3?P_@|O<7U0Q z1`4#2EMul^6v>bZzL3r*N3Yv4H``~b6G~;berbF_(=ogV6wLTBV6cA{AXxh_@zG3< zGO+{s-ow=f!>0q|BGf7W5jQDw1}E|ATwJ}9Yx}OT&Xe4zuCI3DtK_c89`K;8mz7wZ z_o9)bh_jCZ6nOMQ)IZw?|4Bu&MuAH5zJS$(r+-wE2Ku<=GETfSt=2ljr7`@V2y&9n z=r=NYq{)!}9^EH(NN6xYwr>dLK51691!r*FdVnWB55BfXv-_I3L~(1EXNEBD1%O>9$KZp93#LmVY+kcyH5SbltT4}<^;}VV7SaYT<}U}n>Sc^ZWCg&*wGui zw{>DXauh=4Ie6g%v!cii@!bafW8v6c0$+F!aN>$5p+ZC9=Ko=#M3l2j`(2vt&&b<0-d;v&Bu6kv=;NRxt zoy=>2;;G?jmZ`NVQfoc;0foM1NLAw$VH671o7-M%1J=k!vcL7A<>bI@K23sSWZd8rk)@A1<3Me8~a6QQs z>uk1`3+;OV_1IomuWRPB8X00B_Q<|UNHz*=VANn_s#*%KQj7Zq0gsVjnxIRU6O?~m zGX=Dp=!T(Nt;jEpqo73Go607B00mAkbd_M`g<)~ZC?I#IP-d>jRW8u}I99XN-(ejn z!$CXM&LC+74e>*$PO*_qCrVf!vV;}2u}!cj5mL!%gqqp@uPgwicsgwJU8u*_+~(z< zj8zkP4$L}PT7?@^a}}MNsFUOE*2GcJm7OY5y`{2Hr%*YJ>`Vm&TKz2y7>kV@ImL!q zu6%u_J!1p-v*#b)_Sb&gR8!KqVf~D(%h<+& ztSi7dO)(+&Sg31ti(Zj@g-3t&4~44>Uk*o4`579fWZBv3Zrjpqg?G<@9XY#uM$H@Y z>j52B)7!Ygh?L79%6k4~+WE1~!?fmi$s>6Pd2dq@ytG-~o16~ZK?Tgy^LhE>F&-O3 zGd>9bC!ekr8ZLGCx%5Qq>)7dRZ3M1wa6v#3H9nA8wlsNOhQyhNHy&=dC3g$2pE#|Q zh-n_}9ZY4O8KP=E-y4`dc%zF`U%w_=w*)uDD;S*gv%-3a+paSoH_4lCe)@YeZLGmG zWzKuDnY4TgAFyj_2dV`rlgiJG=+O0vGdb1L!nj(W?mlc<>QWC{cU|gU!amvRg2+=3 z`>*QRq1cFcq(yT9_H15gSk$97`z}-HOxvB8l=P{QG`z`vzxXf1?SD<@S&u8mJ@WO^ zYfiBfTf6W)uqE4($spKk@?StV*6THo+lMMEuiOUU$eS9!qs$Ze2xw0K`LpN? z9H_j6S#v|}OVmK}k+20TH40;&$Eubt-;k|qiVkHCGxUS>5CR^ zEXM-NH_)KOv$Iq;R3?{uj^=V+>L9kZCqA5;8YclxbczE_T#X?GiyXU7)f~ z*5zM490NZrhe<~~EXOW(hD8W2d`A;U*Spl)ioNS`@}91=V5nzK#4zil?-*`xO6wmr zN=DcTs=23FPvk3lVnv>dA+B>AnXfcTo~jUbnk%(T1K~!-=B!5P1AI~x{TEUxDVBP3_Dv|X_1q)iJs8>SA2t0JI#4_X>E^bh2=pTW>B5T8mPdxIiN zlG;ly0b<$}KDNIQ0qi0P4mGC1kgCYDuEqC}5$;kU9IznInU z>FHfL@#CH|?MpwdnA7)_$eN!}PW>9bkOR7g_qUO@zs3p*x=o2GfJemNwNQ9FLnBWO z*e%omBOU)HE$3xUXxH^tmqfsxAcCh+op1YT=POn=T@RG2p8317o}Yi!%ei~?{!pjn zW7TG)A5;<&u)KWl1P${I0^UaH{)AFjA!G9k9V4$U0TIxx#75?o-N+Z5c$462Kp*L* ziRweV);f*4%mqDbqK$kq$wRW0ZEJo@P2|53Z3q$HiS<88BWDosnCt63x-jyanBpK; zfQ%3#L=~KISRkz$pIx6uO>5)xJ7vx36*A9?zy;V+iC&W3_W9ebD0rhHdKq7y z%Cof+4#DTw{h@tWGh?1^C&079xY>_D=Ba11B*hwnJ2~#bL%lscd{~J`V1HvflfplN zk}qEgC}*-&pqm0bt=L}-rj-PVkQjkewR91cR>aYJJMpc$Ubc9LT;|W~qtZ2=OU%Mq zKWdHrZEeYdlRqZWBxyC(;=w)ns2D}ns4@n^o6AG zJD42Dcj|UWl=tM%E0icS9CF(D1~dVVdo2H(ok*9sg0w8U|ErkNh=U;Ga00ca`^7$(BV1%tMZ=JA!VK;K&xLr>DRH7O~)+lDj^M~2NpAz!MjMk%hhkq{!^0EbYC3t6mob9Eme4%5#IN>-nTw$;-mV0zCs_FO}z zE7B-=Tkp($#(FTB9^V>py73SSzb0a{b!aHOqq^mV6dB?j0&Y2(%~)o&yy~Vl&0Gk; z86Rz5E85#Q_~u?SkPmD4o#Ju)?UY|nsno9uAQ7H#cYms^P;%bz8cD6bb{S;|F zXho~&gS}6JR<}&!i~O4!tOq-hmoC*V2TZD`esP!7-KTO9?|ygfl&#dOcqIC6XV~!l zn(3}ZD_tsjKbdk*-2hB($G~wvx0qdraQU?CP#A~oy*DPIAgO>OeK3KO9Z0rD`P?_& zR+c^DfgEpR7BOncHdV=sku^V{c0H4a^bNhwB;&67SK)8h-vL7xfc;4x4kfpp$ss6*3R+l0`O|xt&dvX_w$C<8bN$Y0c{DJ(g&Kqc+Zv6oeMY0Hr zlL2FU{O3K!u!M?Frq^aUaKu$LIE}_w!j@Uo%#lUl`V6vtccXz9jZoG0OM0ZZ^!0bY z(~hzMP2G866kL7%QhA_vV-*Amcv#&(`O!)IhC%e)(xWoqm5*S>yEdzstM?GxqjY&XN2a%9PU9R$&f-H_d31$@i4onaZ?GafDHVJTKr* zr8f<^+c|ZE39}*c>jg?9RehMsj{7o4!we_PfKQ&Wl^Uv{-g~zbt9tvumaAvb3|m6- zvo`^*JI%_*A_=igpmordXM$N!vSj^_Zq(_M;xj=<1YFhR$l?0At z6~=gwk4AO>DS`{wU)}u{IW|`iu*rc^;D+^JV}XU~Rwr*yl|^#jikz)T?&QX9#XOd! z#BiubI}iFsUFD_>!*@$x5gzF3460(uLay}d98HORA9T1_T-8)8xtE1rmJ~-&sY9h^xrekBmFO!YoWJ}vRfMW@CXE|c=Il=E zw@{O|#`;A)mh`mKdrbtOxI^xf+ie@Da$S_JAI$AXOFc#YHA_KxuB_AYrS4)wQ~AG2 zQb!e`nf}4`w+QYBav|O&ezL{-iuoQF@d$oot*|M&Up7*yr67h($oEXPp;W-HC@UK? z$BFvf)3Zml$%G8Md&9cDoCDJL?Bi)dAFhbtL@IO0&}~!WzocQacXpPi24UQ=dfv>> z9|pV3BidqZ==3kuI|~!h$Rch*s_$x$cyfD0YMM z)$d)UZ~Cu0%+i9S0`0TmH=1$c{vhyW7v69>Y!PnU-c`Q&TaHpZ2lMHG@i!baZ=7jx zNFblum~BPQh=e(4pDIs)49U{|yWh?&fsTd|*!Ue_b6S!R2#w#~3Lsg9 z5GhyvR}PqC^IZbEy(2A8K%Ocx!~yuQrozK<8qTz{R#P6Q#K2)gm}(>VWp2Lw(nFV92Z{=DfZp zc@-59#OpE!JU;*N%RE8}F(%X4=UFE>{xXErPIv(@t3bhzJ1#4NgYS**)s;GzQD%?V zz#vC6L1ecqWXbQ($=nUV$Oey8Ts+){#Op@?w7eM*Us5&1l?0T3T;h+fckhfN{*OyU zvf?%%PT1}QGty!F8hqPQp^)c!_IYL=gUJdb%}RK#I*c%7H$Cd;1Sy18nTPrTX;QT; zLr}tYV{g__t}eid0g(X?{IL&a91~%!<&x5=nOB!@%y~ z_+2&SlHl5#XL{}{PBP(*5#BVhMBFb*Ql)+n;~2hq^v^c4b?N{+Y*ja>aQ}mX4|s-~ z)qrJcWxFH+4tEGvZ-h(eP`{WRc7C~hd9J=yE=QWARX^t3mO9IaXCa32<-FeOMi8WN zP?;J-k86mQ38pA2+8@NXAWKQk_V1`;4gP--t5~-!reea!i+yfl;0doQbpH5b`2JQv z!~_6LjWHmU`44jE$1yYA*G%Kx!d+Ct)a=yYFlTTTC0!P&c~NGl)`zGllJcAOF-d`nhn*qfB&9u2Td@a0sv9&*>$3q z@7@V;CY?~`D8zpBTjT%abuk4)9^O3e6OqvdY-<9B2^kM#{i`bq(D|T1X zO~moe=)hO1j=FY4CQ4DVcG8)^6Lhm&LtsbY32Gwy^qfY!k)HD)Mu5$@gyS}~*hf$` zQ9V~uYZi28kZ8~sHUZu4iPL-`%Dot9WeuP(HhfBw3sT-se=J3S&h zmd4{k)XM(>7?QSVo$G0(d;=MNAcD_}LMazOfh)j8A_a!>8X^uKOcNwC%g z$6C;|s|HO#p+byDxBo#9W_sO!1{Br4o4etqLY%BU65Frt=4t|jE1R?33T;|Y-C!3m@mF8MC^$_Z}I6gnxNSe+_ z*)4BV(d|oh>%g`fZ^-SM zk?D#=%<=u`u`AR?33zC>9ri0=StxFPerG72#rcm(fBD7-Qhoqy7^PY6n?IYYykfYc zf1kgvK<#9Kv#E4%^8v0GJ^c`PaEc(}T4;TQ%S=Oh1$eI1X~@)Blv5SAn+|V~a%;z3 z+dE08r?0TAt+JW4|9>cZ%ebi8wtJWskq{B2K~RyF8oIj$q)S1%q#0rWNhL+P8<8AR zT1kmv=!T&iM!Mm@xvnQ}?(2Tv-|zXtmjPz>>~kN-d9JmNwU~6uQGgKX5WVVMiCaHL zc2VC9`y!3r%815?{%5l%qFk+8ryMNhPE)lvD4*nttTwBX4yPypZ`2E9mJ!S8d454U zB!#bp)%0_~BN6&7ZIUZJgl1bo_0T&II3+7kA$RRvrpZ|bPIeCB zCJT%LNL=_;lQB-7oNI{?$QLJRnl^@&eQ+*>DmEMse#z>YYEgH%C%s0O86ep`ZbQ+%_*WH{cT!7m5e!>DY#lHGd!_ znkRO2`9HE{{C>Jh15==`JZZ16?hsd{Sg2|X|5MuA9}ZM=ij@jvHa}k(x%A$aJ}6KV zfV?;crt~Z?0~y}{xigU-rS_l{+QpNGjMoU7mTdCTBJy7zBkT8L>1ExlXTR4?8V70+ z>d>r@C8h6<2IRy?4RvKeZgmvd+BYYupQa!iE4;ePvnorSxSTuoQ&mASPJz`@S&Fe9 zE-u_W)>FYb#Rmgi-VfXf7XbP|+Q;`gE}rcw0l(aGTT4p$8kRt#ZWsv?bw0;Ex^eyX z3Hyn-J_#!6)7m59!)U^Rt9G-REC-sh9|pM}o{|gwk>$!~Z`p;NyKo`g7@d@e&Fo<&76BqG18c|-C6V3H<) z583Vt!w!B`!u;%jEsVC2aJ`gF;}@S}oh}0*_eX=eAWDwX86IBQ0~uj}lAv16m3NUc7(sm*Jvb{{@j>tVoBQ;czI$ix+!>fI zpK9W#i&cSv%GwqQ_U3_JA^mCMz1bivf=xwWT!{xxcM?UKux1}dDr#S)=8?bL;mXB8npuZtEK56pyH_fj4v4!KJ zkM3W4z}JKc*B@nx1ZGf`%4fr#ZyI1&6Q<>fg0;Lt8fY@V>$aNSUuRZH=Q_x~xM0Ig zTxd?KKjetLh}Mm_NI)P;9@rt81EyTJI}c($+c(!W*2Ohf4K^=bjg0eUjXFcLlcNrrEX0AhBI*nfA`c23UCH}(gH zP=n2c%kPcSNak5d%)=&@%&ym)O3wrwXkM)~=@#|=R$K99OSDfW=Wt^2xtLXb)8K}8 z{xL{H{v7vk&wJ}ATRJT9<+iqwPeO&0i|4H`szIHTXFC7&&;wOy=A3^#o4$C|9#=gO zWh|kb!u{&wR^wUN`e71RF5$YJ;EYA!yA8?guG5<3lOYXNA%$t3f!&F^ z2EuPEWOKD+rSeW&PGGFl^)Kk(-hW!G^5_aAFx zxl14{zs1LS4dXRhe0emjoF;^BnEr&VCSAlQdDnI{2x$dxb(z+BObsZWogkfklP}1P z)BUd(WQ~*c)583iaH^zWs^p@v`_373>IJL6Poqx3i#ccDMSqOL6_!j2vzh7!#m8|Q z%M~K%hUcVQ_A!&CM#2eV>;ZA1SH!@SptdnMQtuDf)(YrM8EI*8`aQzwo`3b$bx!S^ z$Cp~i>EB@(fcG<~wRT2)-qi?&W0ckVUYOW;sLO_Sx)lQ8jaK=Of$b0@yC11#WB{?Z z;F-3OGu6GOmYW&Md%(cOB-d;8HbZ3L!xGvmO1*wBM`!utaOaaok5xt3peHK>oh*hY zUFN%gH|@GLxVW`Y`J66-d)Gpx6cbf#lW-o#BKT;H)Q+m@HilMh!8W1a44FKpzk=@owZ1>om(v>IqckG_pJHCp(s`V zjM$0Zc`f_?Qt<2Z!wuytKykrn+KIbbefb6ZkU61j!0#GC!C^$#&u;KTPPgJcsCD~m zlMyFcQ_H}%m7lTP;rVbAHLs@08WpQx%pnml@i9BO=9($OX)XsBYRRbpM$TDRNOqmP zeB_M}c1G?IkVykh+Tr7!!v*~V7sw_V!0*fC*bvANo4t9f3g?2akgaD)Ww!Lz?jw8P zQoivHV0Oj<$9cfFV)En7?}g1|k`npOItZOhs2)D6_JXmf6#!2d^uj`KL5VZE{|hJq zv`~7X^D5+Q>xBX$@@eM9gZOK+*=`2-Pl+8M1gI4bEj5#}tmAQ?YXtWlG)as3uFb0U zjR{k4sqF9x2;O_Jz^qle8oGP4-dyVLMb~VW^O?A3kV{om1t4kPPd$PjHxY<%nSckg zS;C_n?=L;_r|zhIk+5(kn;B*>S)y-IzjWMrn~!y1dssY4w_bfC?2ul<23JV9+Fh_b zVKbac`E~$=V-M#?=zlY8(2}`@r@}rk4jwv%ITEha+Rg%E;Qm}AZyOhRy37m#@aGB( zxC)R;MR}W97rLhAZF5%EeD;Ne#lRK=0!-q3>Z5KOToCd6F4cU);r#KF2o1;t*XR1| zWYvcPtv+0>{StVl;_VA{(<%TnbATrv{Ki2PKF^9&1dcad<&Najr@2uP1u6$<{wAx( z$DCaaZpb{ZdXdFCo+g>pdyn=2LlT4!l4}(bM4hmd2}?^~dV5tLF;Z>A0B;RSY~3T~ zBB(I#JHM%Pr~aV+c#_7qoCVy?Z}I9&02&?ksvoUMzb>+Wc=*79&l$g<;N$0IO5B%p z{P$nV_4BjwvzEWtIc?GAa`0T-?Q=n`{KG6_>#C`(J$|8e;&O7K=_>fqDf8!cH!qbf z##-O79^13yOnEs}8nk*he3O43s19xqR0nwl|2R<5I(>MJ-QyI~A7h_|<|zcE-%Uvo z#7C#5rl_O~$dd_eO%qY({qU^riDmbNOjXNa+B*Go0%wiAMvNaUeV=nUG3Wbe1~OD zbc4w0Dy+NDF6mdAZZz38_+XWzObA|_dLuws*5KXU$_Z!L^9@V1{MsA{YM60hUG zy2QIYARa((@$a{}a+PFdNl}D^FC2{DkXTBOPorWO{U{VOO+Up^fjI;=1{&d z%|BDA!^!)F=DA;4_Bt~8$v3O!(@EbZ2@Ocbt|G&+1pa}~(eWzZgrxC-w%a+C0Mqxj^+>?{Y7REB)r^D2;B*0u3^l8TYmh^9mb^KdK-9v|l zj)#CqJ45XaC_l~4WMx;M!Uz3#r3Q81OaR`KAg`dHp`_oI{fKInjM9r`k_F+x@-s4L zcFa~KTVaOXUE??xG7D4l_4WJpPcAZ(Et(-jB3T34Rrym$k`T&a45)Ne)G~I z5=c2(5sa`8XcVY?D}Gj5z?$Nmto^yimQDY5-aayNkA(&B^PfyrFVEz!A)s+Xz=3R{ zXW*bB%@kl3-n1)f&mod3dp2PXw#B!YOLl%%eBTQouevoL39B02H==b~LH&Uwn(Qwz%=qQuG1&39l0jkb zGM!9Ij&%OCjF~>M1U=^crfkabeLZK@d9waX)xd!YGetFIy5U*M^m8=wdhZS<&+R?m z81I{DdTwWBnGYh;9Y0j16yv;7iH6`Zrz1xO$}AEupCn8!NT0gvm*y>$!joWP1eR6C z*%D_J`92pupo@||6R0_Cll%Q!0*y^1ywOtX(4^zDx=KxpbtmC@#fxbqUf~WhUvh=X zSwYU`w|Iu{86GU7_IH;q*oLKYD~yEI_I%~{LXQ86hp0JLJ!-crr|J#a#+TFhvU%vF zkfMTf_QpMjuPeOu+EHZ^kpZJNV?#u$!d>vMyYMGaT@j-41S9};F{nvWHh-^5Si8t~ z5s?u-nbu=_rS;3$-*Sq`rgY8wW$TWjLXXw1MCPei5|1TH6@AwpBR8 zJGrl+_5F=M{ys8G=%U-FV*0J^DuT+ZdGh@+ES?Jv8M8H~nHS$`o@`w87BR`nP>m9u zDg~k%2#H{INt>cBB1m5fQrB&~m#S~$x>vdZ|`1QUN4i&x#+9HJ5KB8%~OF1*ry8OD> zQn3;hkKh2~3%vhzkNsKkg?Vp-l|E0>Svh^@Bx=s?H8Z;3{G8huJ08iEM^E?XijHSD-l&h* zpFami{DyfEhuP32hPqN!rqlhfD7j`ye#?OqE$U@~KeYrY#$zLW0($kfL0h06W#ckx z!yg%5Ly1r#!*K>;@9W8nd{L|lKwGt$;?NwaR}Bnp7*)8`ZGk@fjkh3c&9UI?97G1U zzk1#+SY>`BK>o@yk$7RhlcF?>dAceakPr_qcJvz5s!?7C+;igv_3~=j1p=1%Ut4mL ztpPcRi-EhHVEyVQbD>2UYSo9qN8`$mEhY)t&R>t0c?~#R8mdz)1e_T0mVKjo^&yeC zYkkxfi8pSjEbMud6ZBwqnx(9V>e|$I7~T>|Y1;PQ^CazN0Pwz*vyD%|t3+ zAp0V&=_s-zVb_hR00#MRf&%;7MEvS0D9XryQbWDZc06 z+_-b2>y|Z-`C-RRgfF(;(!rG7k67cJtNAN;wG(O|tFdkH?T6`wUfI>o9WJT1=oKOn z>_U`#X8l2=S#W9`pmwxx?9x65fZ0h}adq)UsDFh%zMYamUBff^h>_3Jic#j+G{_2d%Fu?%xaKCc&xSP7fy#i)adTYH#SDB zKrT(6?(Ky*3<8m-Rx>~3jZ#b}1*&Y)XYFloDx^3zhaFzoO+;@_ zt2RDxwI0t+F!qeB_Zbh#SI!`G;6a_nTSaB1mk9^x5Z@}!pND(0G?|=$DreaJVTo}I z$6MpQXYU6hy!XXQS%FzaGF9TI)41C=Y2aqkR?swQ^b3dBZ$C}GOf=fh21j3YBtOOP zt388gollNUgU*l6g5fLOGjP#AZzGqWAjaIw0D>85&7wCT5J23By!R<4)6|*w&Bz#S z-WdyfabO`}8#pK2Fp%3lZ22dera1I+zmtTYxY^HJS37nr_VJ z5QwEs2H9cZk?-fk-KRL2nN3<2$>v4ymgrT#7+Vblc<)`wtX+R1+nZTXHc|Gn>)4~tq2gilM$ffnvpY?D zR}F6B2MPg%+XF<=7}pg`fY5Akj8L?K_v&Im!UZ;kxX4xI+g~P20ks^73cC?0Ja)Fb zPvdp}$oA59QX_z)GGNKYg5aIR-Vc8&@@5Y95*t5gx}p4%oaC`*Qn&wmxgIhQq=Vew@I1mdjDEnt@cFol|qsvvoVY276&V%G0K} zZY*8{-)e9T5n`L}V0&%XvM+<325;fQ6dV@Mir5Z8wul!NrYMW{zP(?&Fl93}RK5L` zL^O}e*Tw#P?*h`P%D-O=wNC+(!*6ECEqll8I(eDV{bz!UrmFK(>>I~U83BtMpu$uC zNs(&7rFm~#vl0baE76|FfKHz}*`DS|TYN!4shcKfgb_`mdRTbjT1^Y|D*BR9d~VDqx8zk>KO+N#Z*?ALTWdt}&;jM{zW z;D`Yrx1F9-L3BD1^BETw(;w_fVrDJV%0Q_|f1n>pw!!6Xw zLOVHNV&+m>30~POno6X&VAbC~bj}Gr@*rt_R|9S$w$RlTDP=ye++P`j3kmkh{5amM zsfwmCzYQSB5uoqyUSF)9frHH3fB%%MlL!+L(N^=6OL0eC^rnh zq%iMYt#@T#DhpJ~8nyl*qG2>lXwn}q!ry^-gJ|!Hh|u-noKT0fEgNY`d(A}@ z(c;U3v0^Ul^74p-^EC&wzbCtyoXPO1iWfYx65Kej4ytCN&x9+AuI|t{fL9Yoaraw= z9%NaMGF91MVZntz8M~j87vmj8xx)44*jZ0LZ}H~q0TYHP&da~$Q$ThLh;sV+0zQ){ zR1;?&NO1gC)0#5Pb#S4078D<4V99X!5$}Rm$H<8C>fG>Gq;Pm0x$#cpL#K&ePh(Vz zrcgWi?Ns>{e{|W7)%n$LN6BSYk&5y6)BKHMEocoq)To?kUU2c5C7?ciCOH-iJG6Td znZ#o=HYigw-5EamU~_~H?4}EIo$?qCIh9-1qeX|>@tL)MARD2JoQ|LX2A}9!gc8Dj zT1rlQ@s~Z?`UGP>tDCu^6^GXG2<9yFI+|L4kJGKM1`nthK>`-7siW;ov;A>NpKvIi zPj>#4gX*Sd@P43iSVnMARPx%47Tb$RKE1M)?0ck^?U;kr>1)=zs+GW7Fru6ianlnF z<2kca*f(d|)0isPox>Q(b(9>2gRlY=M2-8KvtTK;PR0d#dm24`Ea2Xm@0vCxN-w+N zm8mP#61VZI@%~nH3ZWT8ok}ufKk2w;w$E}7fLVK&JQ`#3~7V*&kD|yvXt#lA{8M_$8wgc?PJ@8|A6$}PH`&5!#4FdQ% zXEP7j9{R4bQCMeg#uSKyZM9ArE9n!n1)Qepj^%xeEmG`oH;Gzo`O`;tp_5lBP0JCw zc4bxDDL}H%x(}eZ;|w#NTc-m}CqB0DuH0$-EB8hQZ!6`(WKTnHPxgSe@w2x|Z@VcF zxN&qyo=(aj;sG2XJk4Kaup)|6fiW3v9QMpGRP2987ajukzqaB}U-rriDAr5Q^A7Oy zIvXnU3NAspds~i@BSo~b^o|siEvQmh(J*Z6KapW)LK9L{#YbIYIi7Hd ziAwuIE7_N$4h}{0w)xBC(&;A@w0!pp>Qpo$R&4}&bW)zJg1Mz&-5{uXELI57fTVe5 zWmb=1(S+DMRbao)iHoAtJ6*6jF|!$K!A4+YoQDe(=W-4`Zo}(z3w5oqLv`L_nmFVW zp6~2bn<3_Pu}O(sPtXNceSofB7ORs}TLw{j1thN=`yU3;C}{WP0SI5h?EyLFofkhT zi7nUt?`62uBE-he={l(W2X%`!)=i1SH4{f%EQ_DN(E(Wz|A|QcP04L%VxM3Y`q%Zq z9yO{*^*&S)i&~>thw?~zwN`3oiWRqhXoN0~?eOe;y23Qsz*m-~D$0sKwQ+Sb zpO1I*yu6G_mtF|@+hOVM&rhnr_q=#QsCV}!+y``Cwf}Mr!@}r>Ck4=+j9*qTxPX?w z-in5DNM}CA19(4kUs3=*%Ihr6VN|s0_$4j#Sz5c$JFI%vQ z75$;KD?Y7m%x~*r;>*An7ns=OO2A*H*g*{@Tem0;DQEa6NDO@IsKOlQs3mt)cmB3g zv~M2se$2_hRJm^w-UX^pH04^ISKy&_}cYtm(8(s_Gl02d>S99;z3o2XW z@Y(Rc3G&G++8E&Tz&h9HbsuU(@4Cc7c||&xMfPyMi`Uv;m*1v16}U?otc%57V%>_A zR$_i@9^*Miy7BBV!g)k{C!$(@wT@ampU;^72%$AbqIQ+@h}P(wZJ`$EZ)FvPfD0>D)+Yu5 z9H{q&;mVUxO8e^FWb@8n)cjo^8%kBLC@0*UY!8$j;#;RvZXQB+rB@1Knn)@+^ z`DEbv(Wb_)kC;}l@0wK}gJ@GJ@$t`F&@gd^h_!3-V)7IQ{SRIAnqBU*uG|!;q;pTN z!i|3xv+Grnmgv=eG>2Vs^$N^BeqC9GuPg|~xzK%ZE%4QNX#f?a@kuG4j^9{P58i{- zEH+mKOAT+}IX`*|jURywyVQJ+s{-i_r}MpjJC?LyUHn?e_pekCBPLE)q4%_-C1{XQfS#zm_eB=3>z#PB!m58 z z3U$PkLHMKtmnk6aE< zf>6?8W5bL*&NMd>MGSy$WRUQ?t3`#^AaPqDNl9tu=QyU}`D>p-+d3zgo3^MyS!9!) zg2?ruXrH-Z%V}S(^NZ(du;l7*)<@(nF&4$_`W1=Nmw4~2s}tg?DZhVgPA=JaX}Vfe z_Kn~g>wLJ^eQ*X{mcCy{8&zOa2l{@9M(>&9U(a7inC2gCRQ=)f3FII106Y>PkQ%6v zJ%+tjQ%{(s9yD^G8@{8%`g;Zc%TGRIKnE>q?D-#b&^`ZiXGKucFdc5H_kPE5e!Z-F zVw}$|aus@vob5X5#(f{h)%Kv9kuj*xW~Qv}TIL}6nBVtG`iPu6Q%0FRFCF2Q*E?O6 z2jIbuMv|Uu)!OUNiem97c(Oe4D26hI^VN5Sz%Tjvd0|OB8t$>K#hqclhJIo^7=(#w zSEP3q(NY@GjwQ_xb~3%R0V{i%5C_xaboTJ57dN^#Qadm~b&w(al(G`U=frl&V31(CPxw`t3RK$XKp!!rpz~c)7@O z55w|nCEs=R#6~y=`jyL&^z1)7?umPQ`G1r z{56xafMl?$b06fLudJ;UW$S z2_q%&qtwjqrLN&-AFD$axb4k6YQe4Q+x1G(z(rnu_&LKZ~L0j5nkoMyf1VOk}Hv@HV7s!>R^F|)YW{_4H_ z5*3@s$e&0su}`>Fnt!RraRf>an#*HvV3X(t>~J05`jIc}5GcRUqEs-DmnE{f`8HB) zrCMC~4g(VWww(0ZfbFvm|5nj3J>ZJlb%y(=3)h9lj^)l8{)^d6Ww`C73{-ercY7%{ zs@l%~^wJE!-7Bp6p42QT&Tl4n&!K(ceRG)p>x^9lV3>AKm+jboLz9VFyF}(hbf462 zvg%X4?^T@YFBYSU$5a9ye!wihQg>a<#`$sWYd~>Ys6Srz@=3u>N15%-yWem1nyfKr zKOC^e#>PsuT%HD~DFEb-jMXKMJK-yvk?+za(=|D}2$#{|LaRw8EpZ)kDtoN+#zRxv zQk-^Z=V!y(;VK5Lr+hlE#x7Y*pRJdC(5@;EGX>p(YJuP9a-F)FRSo0Q}wtat-K4G zW1Mw_%ic~FaL9I7j z*@=)AL#j6fvY%K1H->%9mgS}HY;Ar{SQ;u?`hN0pdnC$^qBGPI(_)3{2nB`bDVndL zkGbi}H?L8bw?ND8xySL3^UcpQV9etDY-qTg=#Cb*nD8O-E2TfZZY03j2B>sifYS;j z)3yN3!W@b-I{09MKgLNtD^|k%#rYl)gU+j?zGx6nxnJ7Xn`o!XCPda zX)hhFz0Fta*QVEIHv7nnWj>v$v;C6%LALG;=wCvLK_UZhiF1hqfy-#pX}ZLFQ7WPn z-Op4a;U-tCtYUfHqd`$*MFBnjj82=Y)&I)7smT`KnT?vV3y@~HJqAe`5Y}*W-dV+e z8-&i=0tTUwn%|}^cS>Ef-Tly)PhMb{dBun)H72QwC6KLG2uMvH6zPv&n3*L=1D=pg zz#2*sh(P&{j>5-4DZ7#&O}UTf0z8k?g@pHsi+)YLRF-{WBl?mLXfv=Q;1Gh8aZF5f z&q4DohHB+b1AVkezFtx}V{oF4qlj!>oryN^{rXfWa}~-z*CFadIczk5i|Vg+%?xAv zE_ro9kf&KR{=?o9cxKjzqY(Mj!WYB2-5P%}KYM5QxZUR@$OX0;1YFk3w*&-dJ-J4U zyf^^2vsl23W2e3um?jv$5%EGy2ssWROSit_h+H7Osw@{Ck+*_qggkz`rcv3~`IGZl zi(Q=)WC7q?U1^yPG<#&FH)?Vf!VPJCe9M_+K=$!j4eLJ!<6eC+P5=+Mrtue28o~(VOA8bxL zV6*Dh&zRXsoq=m~GmMNWQo6#^)^Go4w$?Z=qA*2C6gk@fcTLkac70xLmyI|7u$7v`ygZ@0~11vxWzBY}^=2qZWBSz)mltTZjzuy-G9Q z@M)cB1m1hx6=XLQy4zI1b!48h)|Z9^zDTji9it|!fj`EON_9l#jd#ztfPhn8TL60& zcm4eR@24ca1&@3r&kMP!gpP%uH?q@kM!b1uHJ$%v>_J-#nqi*86_)#t!!DX8W%@IN zbRjbV-ry6WSFZ7uQe_%J>dEhhzAQEDHpV&ayTfIplEfMgy~lPuDs?1xrg#b^FDT^t zg(*&+RooCDd;^vt&+093c_xOPh(n13a{)q64D7U)57T%t)L`axq2z51%mQChtbd+X zeY-k4`b{muzM)!pTIe$gyY6lqk|;RNLJL|G*xPh7XTzP2tY*;hEUhom@ajomr-@jA z%r%=9>@r49wAFeZeRuI%i{$?3>p;R9@AYVf+V}D{o98{mEq;xOKk#|rNGI9qpH)|^ zg8a7jTDQhe>ZdTAoX+qZz+&b6Tw0Z2hsJnndr-r%`+pCFusi{8+nXzgC^gY$MEY66 z+VK5mz!kFm=^~Csy>W)a`nYk|KsgO0A23)CE_oW?CreWowOr@@^XI!N$DH9~oy|Y& z`oK_w=fauqYakrP>YRq_O1qZ$Pm?hYrt*6 zry^gnT;&Y{qIMFlk^GyZUMN6hd7H21OmCB?xAf8}zthWU?~i3g_HF1Vv+~4DvApB) zu3NnIC6V8?N|eg>u54AC(^b-r3%1U4Kkm*pziKXjhaf00?TTzDJ(%;dFJRJS8uUYL z>vOJ}#$nd{{u6G^UJ1L}I4Sh~aVz72#8~w?0B>=)&N1#f=QD2SEu^(%>h-cjx3b^W z2k_p?Ihi|)f`WUHP6P`gE<5}8VW-aV(6irPa!Q#SmyRP{#MAUTRqH&D^u~|kST&|j zQ#?OAxb<+~KDNe?JwjNzaFkDF+t~r%u1}?tPxJEPg9uoWR^5qg0+VLU^~FZR$k(e_ zB8JI5*_)`V!GlQ};XpePm)}jd&NV6D!~+z=2_QjuzAk$vhaEQ}+}G>(i3)T!*5HW? zbjMtu2 zQPG>6c<}AEh0aNA!#tAzoCFjw$p^JlpI4Ft|mJoCmI2piU~ZVbPQ zm7J(W&hhGJia|CzU4XMX`3-7K z*^ZiPwhQUvO)KDW6>t=v9tgMU+mk^2XSVl@^>GJQ>_u+NBXA z2Nlb%^u%Y8aari28kE09txvm0r?g7pNWptz;=DF(pLKMZ!-m!hHNE}ik(PEL%g-=D zR=$=M3AeSKS|F-^W+n{gbkOLyWWKu=z;63Rlq-!OBfyZ)m*a`XxqC=(6s>&(u0u^L(z`CuJa2@S02E=-Q zVbNY_ePE-pqsOxW7rn{0BMD3cr0j!j1hY1jwIJ%nc0Pazc;WyUU9?s65wZQq1YJD^ zgVs>n61AW{Ax zaM&^pAu)eIRz**s@4WgMwo8S>W205znP4C_+_73R-MYVvP4=ySXChW@kgzYOx*B-8s);wKu9=&7p#vG^xa7P|$Sq4ht2D zkR}EDRTC~xq_>|u1AHTf(u<}He=W*jeS&QW$vt}&YVhA3C5}le^M&m}c#`kno(8`# z5ZnkrQNr$<6LSXIW2O29x*Q#JYiGQ==Tb%+Y+W#MkuuE>V@@i1##9+#mj2Cp@0&V5 zBx2>9rKO>tc?rB`0F`%c#!1xepc0(hN8=c#X8B{FLvcM$zG`(3Ls*MKou^tQmhLavl6g&l_ z2iybMV6I2T?Z&>aysD<%d&Tigx*tmyY_3NNeTUBRb~~(~x)=^**i8Mr6d>M~V#d7O z+z+B~D5sn@mSL#I8(X7VNC5Yw>knkQpoiEXr)gz`|07PuOt~*~K&n!Jiy3F2P;ID3 zvwa#>3bL+&3lLBb8QiGokWvm6ThTLV6_e6TJ5La9L!#{m>4FM{MZezkLlnsR9O!3! zEEx3q(bhj=hLzyrf7x=DZ_e+(w5b6{x=qU@q0mLXC3{sKe^m(jc!~np{q>Gtb^v@= z#3?uv7a{lw21-Q)oz`X%T@ZJe>o-pddi+eK-X{G@M=%A&Oajt8gt5RaRl+ zyjyekjLUYVx1&k{GJi1zmNJgpnJzS$4!EV!w6D>L2r^$=QsdIY&lUsRdEbsYKVIwp znzGw+qOxnT2dk%fOJCiy+-f%s9w>i~*p_}W_H^$ZU`^KBbRkJFW9R(`^uqfyjMGmV zVzUa(Q1=$F4>&b5dt0LHdq}?>1!|s*+hy?n>;q7Nbo&O!8dHB{!el){=5~y%q4kOCv*U2)CE<*C3Rd|lFgf64#s;|stWIwLv^v3C$XJw())=%tnD8x zi&I=%rjx#oD$@o|6b5xM6Q81oXZ}ojd?1$ng8iD#W!gjRQKovvh?!}7#fZDe6pP81 z)XdfabJ2`>NPqb&(|MP}%8AtF>a$$Tf^cdeaj_gmwre4OI-UCE=@lLS?nh`6Rp1bu zLAAc@&*4O>?bM5Y42rI7yOg|p?`3LcAtHt#3!RI!qY3ZU9$V6PiGuEn(=6yx?}H2R zy5gZp4sI*8ox9MqW#NpS%hib*UHd;PLMoOi2Tr0iwd3$(yZ-K06}E<})Ok?vRVePv zoi%+WT$JQZ7AZ%*+#qL<=GNaF%pg2R{@7V}& z$3NOvqTLa++HLc2NMg(!A{|6>esX@k#tG{6NrO(dG!BjI@TAGJbh)YbKRZbA*pE#! z6?Av9*D&tv_AoE{E}kwEHjo<2%)sFEy6Lj+o8rzR5u1v1|Nd6)0}>UHU&%!YLT7jf zGrjv4=#8`4GnEQ9{5O0_7rL8V@EDFiCuBrbv~Y+?WrRgP zvyX6|_q_gmM(!OVXhF)-NU=ge*?R>&j1UpMS4N<+CGzZ+Vd_|Ye?2SGA7HD3DSGj# z?ww>iaemc~m+Dy)U{mAU)(UzxtYpe{BlBBW>aD0ny>31;5n`3nd_Kr9<;*=LiUJNz zV@&_1HMMKOZ@zpP08?9t(vb^{#ND-^SghNArhiyzP;@%Q(p4Mr!P7R_VayJp1BiZ( zCbtTz982~ugP*&J_6mP8(0~j2y0l)H3+pLnpXZ#7r6H1m?6(UVBqtfz$aSqU@yg;y*oF%n0{VDoWY-L<%PS!HDXsI|&cW zv;|lUc`IMkS;FYLaMi*0S;`>7CS_s;fGDX91y|p%Ps^`#we>q?`EBqDU{jLXA!rbE zl7WUXNApb4ChsRO;SVDr#ut(elEZ|ic4jq;G{48aP%54$(s_sA%;Ro)aW@KFuv5K`ZzOaGj>|!n3=B*ZWs0h)Ml=_4MU5z^7NlwfIrz z$N#8&lR1JTF)Uzea0FKFqD~>d#IUSXt(2@8#l@JXv=bA8l0RL4e=$s{0>?eTHGV%ZngpOwVR0v9i)9VHLRyN|vH5oGdKTY)JwN;UpevF( zYe)ZMv+J(#D>LM+&|_T2APe(9Pzz?)ZL zRQS>IZSkYpQf&Zt$z@6+&r06He^?wa+}LJ$D9t0CoNVVR!ZTv3#DeA;5IZA;-ju7J zIpC1##ZkGq?e$y$se-vauM=bm8waTWlhlDr^fMXvw}s-m9XLNfNgl`FcUS=^)T_)D zAFomKbI#E1KCve@5C2K`h|zl8erDnG`z3ckV}A^KS_S>rp^3yl0GP}{1@HTO-wR0$ zg}J89kaQi3e4FO{obW62Zz<>D0G3;oF^?pAh3;M~7*5H|G^;Y~*jZqK<*60XU4M@b z{q<7>>U^Q{f##o+uXT97Etg6|p0rZiG(51$rq$Gr2`W`lb=v@PHzo~K!(-A%*>K#+ z$7FZlt-tZ@Xlwx9WsqRc#Jlri^bU6DPZ9w80-i)hF){5Rb*-Z7C#uEDgPz=JB_=#G z!|OWc(OXJaetblzh>$e!r*uU5Tk6e6s}$=H0qfPPoK&J@Us^;Tp1LGmm;cq$NdC({ zzWWQkWcegU*sIhIdM8=aHs>zw3&2R1CiOJh@kTfP@BL?VrPvxIx_3Uxf8p?M_W@3+ z;S%b8hk^N>k0JF4m*`_fT5jcI+RTtbx+#_~%c*)-feH@IIH`L5@)p7YGb7s!Vksa; z=>i?-VTL$$^-mT&S8|i@T)y0hZhmIPJ%ASqsOkpj5N2P@tXq&gQM{BpE;LQYCgR#} z7!_rq#h=Y*+bU^{Qbcmxv&RPKpWrA8_A^d^4{RerkO6WAdz#LOvuU(+E8+abYGEj^sKr5PCv>YayoO5KwoxJpir0s zXs4MN%8gBQOh}8rQEP2kK>{+n?q1}%xf0;2Bxa*vd3{Sj0w32AWqr<@gFTUKxvwX7 z$b}Mmtol3L#o}a&o}{uQL#W|^8eXKd~W*)geDX7&f1Swz8_kDdlV9sHx!Uw zVbaQ)0Ur#KfFl}sL$6;vPt~yq{kv#jSlp6*iyM<)a0bK$$EajzyzKsr=G>}+xYbU{ z*1LWj`3o;*jQOKXnF8oP6H7>vcz=WD0-t(u!zdtfAZ50&E6s;nZ~r;smK+9|@UT?; zV%oNmSMGY0z^3gVNh8>kp|8_5D%bGb!$cXp?>*U^Tlr9ZNH^&Do#*jyors*+Ewx{+ zG4Slpi@&Kcq#7R(wZ0eBP0yPF@2kg=S6doSQ7cS0^%^H=3WE-G+?hws^nFywccbrv zbO2?;xQh#iL&xYA^BUfZugA+p4@=Nof8)9!fzdkyxm4Cui+=68&L>-1{ixr(4mx_K z0g*lS_!K)iGcW_!E6P_gYeLUvP(|8(6Jet@_=P^?>U%8xD<$j{#+BPjbHCz06z}=% z+Yp}P3l!FW6TK+>G`>9Ida6|Ob%lkP)##%_Jfp0a+fMG4*YG71cFRl7)XM7LPU%VO zdV0f-UBw^1&)aS;Q|YB-vEgZ>Umi)pwCVfRQ_pZ+>|$F`r#>(M)_kHXNe8R-0hNJs^DqKQ>ufK8|4_k0+rPpgHPls0buj9SB z24KCc);CpB_ou3TiugsVK|u_xxcA?2<-KVqeyT}(*9`0LD}Ce-OnpkNV^*Qw*&$T0 zq7tsR2ELk9iFf}r3VUSMqYizPXL-}y6x%6Lk3BZ|{2OiYj_VLL>YL%c4kp?V!bIUw z3Pa9pF&5IRp2DBoVbz0kP79bW*I1sh3sXgZU3uRv}%fSJ2cPYFG- z2D(cVun+hwD{eK#zB)_V+~}pDFs#ri;Du+Zu) zvYVMUUBmg1vEWzbc82L#;Cqbt@yGv(Ao-0CbTf+(BMFK-G!N~&`=My0F9N&WDpH@T zy?pS5S!KUuB|IEgpxjrKir*zVl3G{-kjR)&sbem`G$A=xnZ|YEQA?&)V;Pb7^hilsWX;yhy*Z2#1fvXjDNBZc|- zagO8Hd`}EZ`(T_d@)Yy46+l4JT=wpMhp1<>(fzq;ntDTXs2=6y+Z(bab-iI|C;wL` zbtfnLo`e<@VmbM&+}vkVP&2Nb*|rR==pOIeD<`VzNpK-)jd#;By?^KS{nx#{jZVPW ze$0LM#R>vcUw_cI(RW(>y8SV>ak0$ZeNyc0twq=#*cS0WgX&a*;$3Z z3@7y4E_~NgUawtf0b=e2m7(5e`-46N2-%j%jPyx!orJd-?@<)bo4Kbdo$sabInsS| z-3wh)$HX|Kjh4G(eD@cB{2m>s1(o6qS)cA!wh0Whjt|EAZ9p;Tk11>Hb6;ib0iWql z)g3J7rSj(xeMtL%biH+4RN?kLEJ!IJB_OS&l2U`@fYPW)Hz-{L($XD*fP@GPB}hp# zFmy@@3`2K!*Ug?wF5ffE=rtJYvwB3TUg}#E zf%NhHqr8c-H2yA)-cZs!#F0s7Yd5y{-?&f4E}m># z6PR_qM=l(#ZlWgg>#)MfQW1a_GXX;BK%Y^2`ZzP}<>7|j%Wte|X$gSbE=>YyxTT&i z`socLX>2*6Y%&^=Y*Uv2XpFsuwJvA$dHZuK`)>@_h5S;_?FJ#UAQO`l>@chCdRWZg zG+UH2m5NvU1A#=8%~6u8<6?+Al&k*aYfjNer?(BHBeVdPcQcgMQ0Qu5rnOe*X-l`LBTQgY|vvvj0 zk^mFFeq{wvMZ>Ak`0p9)Z+{7{^`5OYALlU3a{*O0Yn1hPX=@##%6=(5n6`FewAOfS z{}Yt7cK-**K$R!~_89U$xWcE6p@-3O3_mVjH=UsdP5zU~3g z{|D3}tJUHB04LVXqYhxMkU_>5r%f$-0WSw*YyGlMxf@4vm?QcFJvaogmgT!ZH+mYz z@K5U{dsYSq5$fY7izEc75>bCLPHn@?daJ_vevb)F>o}uHNRE0r55LW1V-k`|MWE^? ze>WIIFu+&h`hx~qWI2-9u;meLicFL_9~jW9D3Sjr4-{N5oH?|yk9QoGf&8c^ym#wX z`~t%usxn*S`XBUbM6Hf=#~Z+G(BGfurcFO-sO77sa6*RTy?-zFVx=MytjPmO2`0U> zs{|s}Jt7P>3OF0SV74Uj*F(=k>Rc|TUk6|DU zduEWr9mlR^V=eY)BLAr@RTv8sd@F_BDPOVamnU%6kaLY3P#v}-7?M-(b#Iaz; zy0-EK?-*p@Y(c`R{Ihs%Wvb@5(#GiXeCz7jMiD@%(ytr7A@Yg3xMY3X#D(%|SoeI? z%bo;3TygF!sRNYZMh=0jcdSkI9;E|8NoO=-pNjdSWw}>CyJdPY>kVZzb>uv3!lKDk z&~$vpqi$_QEpNDGI_(!xw$USi$R%Do2ArakQB zO{AV^2kacE1{(rcZ%h-w=VK@tm*=1f6asDrCOOX-!}dPNarXxF=gC#Rv63Qqn?Sf% zs~4xJzGRnA97D^~M6cki#BqJs!X07Rbhv$ZFFb1`)j!4B6xiY7!9|Pzc>A0gTfR=B~F=BbXqF?l4wvKNr zl46;5Y|D;eVy5Hx#o8t^zP0){ut#{p%E#zDOHMMXQl;`7sEqp?Ga@H#|4RA_?AGO< z7%!a;27#-YH`os~HAv`c0W0d~?(=)th*RosTrBrHM)W2k_~RG=7VsRj01yVQk`5;A zT|$%d8uX?SYztjbx7kd+9+_D^2h6Yq)Rab{CQ(3QR7<=2K_XDAphZLhT-R+lHVo1c z@c9;Y?J}IYlO40w^Br&Qc-^)$U%*oVIy|1Ie98H|Q{j9EM4W>t#^5mcZjMVtZa7`Em^c zUANwSYaQS%dd^p{6kJ%cz^r}hYXuYr{6ezSj8czPk$QIFE+Q3e>raXCsg7V(_d>n8 zI+(6cyJMDXsd&|`ow>yHruTU)TL|L1(CH2Pe(e!*J^fZJ3#~jCn&7ah~Z)@S4NIDqGj0; zcRnWMIRYwR1oh4sBBOTWF`tczoaO_W`6eN?xI-fb07B3H}GD zlrE4RlGc!1>@#X>e4z@-xo;aseIMe3R!xNCxS?`}NsQ^1=!w&PeQ? zy0`Y%oDs2&ww~?UG1(Ve6;97*JZQmQJX_NX`XRY$u%MFDk(svr5{Oyx5|Hx^mUc*e zoout>LDp_>UuZnO*UooVS;~9&vP7y6clBiv4j5PWde$sfK#_;^&G<_O@6FbBcyd#m zZNuUHa4O-?Ua6)l9^0dH%r<7UUTyK+FTc1erE3qdZl*OPNM8sArH*WlUMz2LCD?Xe zi#b)#RLu*%%{Zres_#;{gvHu8^MX#;%kYc;qUa&UuymWj$78)@^Alu))T03MTB+fJ zIFu_Ol4yPWQ0R|s`bVmtiuX(2A57$wsADN=6zWDL$GZ$fg2RhJkKd$lXT45%OEPLh z@pB2}XuMJp@`K@2puKH&O`d&Sr2Fn`?+d$t@+PgF-BXco*tQX=&&L3>1$y&mWa_P^ zR6XnaA|>vu1}-IpbJixourZhNU7Q#v+hV?iCGg%GP>dNo)R*6nm0{;z;TT5v57 zn6vfOG4AzqTMA0yw-AE2)50j3xym*If#ew8Bwq zqvFiht7khbJH^+QF9_^O8iLMu7Z(UhzzpH%V-dFM%(Z3p>6quwWTL`-gL!^zc&FCc zTBr2*Gg$|;(|YZ{D6Sg>{=7!VYxb8$UNZ<) z0+Aq$^&)HUev*<~9gKO8Dr3OpxeP#>5WTeifbpXwfzI6cAdMf&R)X*#;k>kNU8a|k z#S-Xv3ZWw4qvz>l*A#@qQhZtJ;+?)$fIGw=>43UkMT_ulE>;AM#4#!XMif3--)} zlKS?ABAz{KJLTXd1av&QYNLL~A%Myme%Ncko}ppPo{O3HjrEJCKMhP6%U<*sjCI34 zELK>3&-9AZikj~zvv8n+k2zS^cxw^{Ep>jIbMH~{u--9UapB|5o|SJFCc~nt zaBjaVx9+fQ_@%f?$893qqMhZO@Q1^JdPxYbI)L9)+h0$_VNX0j+26)Wm#O}ZXjF3|7 z)+%^7lSM!hhBTsDEhshtdyu1;&8eE1gYs}M&OgVmtEz0B_tIFXc;7E^Fz&zR1d4R- zj%TLkl_2bA5=<{*w~=O5iVycBfxz*S7c^ys?MPi{{FPM^OBrGTOh3Sx?E>zU7kwUp zKZ1((u?);seVcrV_9puH#mAq*k|1N&Qv!K_(^e{^)^%V{26#H`oF%Q=2A&nkKX^N$ zWt5NZs-%zs&I}o+kSI9718tXkW{wnq#>U1JWe;_iB(K(MxgNp#klC#_h72-akUc?w z$3kh~{RxzeaQzr$*kkvvR0}>-F&L~pt^0lec!YJbk9>;s+7EhBF;{kCIz|V%&`3D5 zPq#f?+m#_-m?Y@_rZ{|@Ns-lh%F`9Fb}~rB z8R>SZQfp9z59b=#O_z=6C7fv%XSOFgmHPv)lV=uNIpP0;f@C#sFi5yn@AKOjqzwiS zQ4>S{{q6^^O$0Vz4(?MptKTT6H=v%|$oG4;T)p)L&idG!23&E90%$UKT;lwWDC`1b zyBtm4N&OFa!$*7`C>~Dv!7g%x)RQGVW?z@9f;*q zRw7-vkAALF8X;%%X`5gl@+rZzzV!gbTFIb8R(z81`Mn~qU?hI&xc-l{(ErWkV*)16 z?i`EbcH+drNBkkZ%R{M$l8#atY?)5<%%Ol0^=nk~(t2?Bzg6b{{sF}0pPD?hoOw{*1=Li-=~;cPs2`s zhZvDuBT6=zW{G;cPCwNa*D!A7^opin!B0LObgyrM7SM`}@C zN04LyDrh=?LDz5hXF|#Lp;HqIjoLbdj$)TWZllt&J%}7tVjGX=R49a zI}xKl`qhlC5@XMImgByD~Rx%$%j*3SdXw9x}Ux#heE1svlkl_L zEC3oos!s*B-P%3GotX(UVrh)_HeWAW!}aw0yiAf6)`pB0JP?Fvo;~#p?7CfFP^a}5 zyd{sf`ZU@Vs5x`KZYSh=nNE-wijE0!nR19MMlI~<({kA zS6iS;TnPP&{Xbf}Cj+p#nw{=Z{wEU(pjVUl@}~Kv`sy)iN3M16%ZJnMB6yB%GeWwE zK|N~_Q)>a%zq@pg24fVaqX9!5?v)>~`COd2?BJcx+D9LStoN|Zyb-GDQ3o?59#M?R zRTjSdU*_pa{JS zd(ItSUfDY@-S~>79&#!dqM3jev@z%K9o#TOe$TCgM*t8RB(J~JMsTKX($=F9Lf*~p zL^jH_ImLr)=^SS4$psVkAal0^xc}eawFB-DwB#V`+w=SQ+KXSOLK zv2YhI8d0~`U2QZgo!`Pbzp=e?Kl<6{dl!45u-*Ij{!(ZE+;184oqA8l{YANJH9HP+ zPTi3Gr7i^vi}TUt47;JZ2Bkt>Ico>4JdlE?Ohhrl9WtI235oM^wslZeCcZ(touOW0 zvylwjCW1DA?x!VU8K`Z}5x=@9YtKRe3a}|t7O`H95eVSN9QS2OkRwnAcR*VVFj8glhT59beXOyR#_g9LM{OLQN`{brT zQIt;MIq0Pt9MHZyKb}}yQO=Z4bSH{Lu;>bui2GhxJA8l(BGM7x5U1A2Wnyd8GY7HM zSpXaEK#nY@dt5&oS@|7IaHCjhP|iA0>mFI|bpuZJ<*EP>l#zoND*z!GC~l3huK>QQ zH`Z16Um5X^Fs`KHLs3gzGOeMc+=nzm4{gEN_tyaljP$xmF~FLOOFwDRt=M)(-vMEd ztDM3WO2*N)B2Ou0GI8W($B((QL3L$o6>g!}#Q;LT4o zyr#oXEl$}6KT{bc2Q+{x+@F`{M+4KB=YDzNYyW+g^9Xy_5 zdvtJ;v2|vOOKwQ#;pGlWCA(22j4UqF?J=Wjha&~L+Tnv6pbk+z%zF6fA@DFB=k&Nr z-Nd+ZMEeV3bK)yXPzm57(7bTg_JCOoTNk#7ecCf%`eh3W^n^Wwd&1i2I6xD7W8Xf^ zY}y^^{5iEo2y~76pA_{_-86*z*>)HZNKChd1OG@V#F&AFugoKDLHRyj$9ffgOymZ# zqV5D~K(y3czF{E>t@hX>^1Orq(&8UNZ&(^OGaheYrie^5{~)QYFP+xyE|b$y0GC*y z4zLktQ#VoP`{gwM@#1LXx_@Nm;3oik%9OuR53Jwbe+G!xe1%+1cfLXZLR0|dAw0*d z=&H>6v@_p1b?FQ+BUU=LF?a-RkATLToSNSU9ITDBE6tzbFM7f4eyjp`mF0Bpvm5O` zK%JfF4G9ki0|B3>n-f0>@Nj-IP})~`BWd^A8}HGJ?$Ch-fL1X;T9Bwhu)V^h5GOP7 zC1hza{YCM$3!tA&K*?|F(PCecf3gV}KOU|-4SLVMTM!8~nfPd8TfL`Uz2B*xl}gT` zPgyAY+LeA2k7hEa+H1<+i;AB!3&KEpwU|+1ad{TrW?JP_gFj;6LaOI*_-zp2!32iI zRTf0FNSq8DU*>d;Oj#`28vW(Zom^3sxuVF{!SCbknyU&_H|zF-ta98Vy~+T_L&Xq9 z`vli->_j8?mX5v~?$wP+?l{~NdqQLo$JK@nG6LtH>c4VZpN7#PQX4uTPKw$!;qE<@3BF4DiYV$BWU_)jZBAixVWnl)4Grduxci z`%(i8cRi0tF5Ivln^&$rPMKJ%+4eL)q80`zK01a8!o8`l@@|r0MYRW9sE<=c*YC&S zkl+m=u9~uYq9?Oc2U9LL+J#j&!vny~%dLvK)!Ko@41(|qn<=t)W;JYF z*}=4xF~*I-B(?JdGIPdh=$lVT=IjXwdW0U)rv=YE)fMfz3NaS{BGR(O?~nDiuRoJ5 zcca4792Hk^)tavFbyd{>>xh@6g)Zd%_BIQr9L{E;_3pR`LI6eJ*Q_aO+eAiEN#WA* zYTQRNic^wCZEt@XV0n|iGpPM+mg*Y1zleB+3dYZH=~~`XBPsIHZ_w8}JD;IlLRE1g z;sv`jQlWu!;Dc;m|Ay=zzk_OW+bXueS4S@hMA*eW&+!2jFM;!YBfWQab*j0_s)y;r zWhPyqPNibE!6Eb1b*wA&bAngr{lz=M17AOYTc3$x_OM=03`E8^XF9PJpEDyD(-b|LfzJGGCP!WzGVR;i?Lwk+y z?tC4ZwYM`Tel#}rF=xiX#VmRT5Wp(bEpPMIu;b{$_;&4iR3Hh_!pcJ%e$Xkt?#>QH_`O&{rp(JX=Ay>_ETC;TlPM+Fryq z?83f=?+!z`pnpi`vnMT@=%%Wj6xJu#$nLYHjpXPLiRc#TmXCKz(Swnos?E|YMsVDO zmTWQ6Vmd=l7eEcvbnT4|W~6AXFO&!KozrREpi;`&~Pnr9$p1W zTMEVx3(Ze$In2lQc?OUZa7aZTVlXL^N^mSQ6%Zceg~^waNZGU;>%=EBWei9q0l9h# zx=3fhtGl{8o+MaCo!2nkS2w|IuoH*6L`xd82A@Z{2LIZJecl&ehhxmc0@h|aI9%;S zl|RZ3)98%lM1YD)4PmuX3!R|vP_GAMg!4;dKt{e^Z~}Q*JKecSXm=-efvA-2#8g8% ztQw!5w#>>skfPDBYZ*xGCe9@ORRYBCpyDz%^%pT7uLOh#PuHCx=1&N!M~Sm{iM#9M z7O*ayL;aV7CW8?BN2Ce4Wge~aY@jauFg(exzVdME8zvCJzK}I_JZRd6zPw zT~MK3O;2ZsrK`@XLrXOa!3lQWb88F=O~%XGKhX%Fmc+17sUHbSO5b1hOXUBR;050! zWa(ZUjdPEF1Dy;rQZ@kcVkjLx9@!++m2I8uhm}?&_+U9fhbdE2yu&&!q5A~!XGGPN zSEe$7IsrmeOwu5kGU>W1*>-OReXt;n#Kdyx)Ya;1JbHuJ>NszATHfKP^(xB;c$DbJbh1R1)cNgLY$YKz%B>I$1bI(~`b#+c0(6f@K7#3l99zsB zYH{~X1H(9LeOm72Ke&Ib(X3Dx-Scb4|Jd93n8!gcYC+O9%qMh20R^yd=X~Bs0<7Ut(ukGdEA* zbFoqCWsrf;$lr%$AB=vHLb-BOy6^(@%H6_Ap{){JB={|3wnksmM%j^SRwIx_uj!5v zzx*J%KV%8P<08cp(?=zi-wses-MBI(a6;r$%Z_7b5X1(7?su?PNslrV>_@N~TiSiDO%eZ^v1 zWjS2-^@ew5h$z@0z7alhT{Ol){D`rbzm-1zd-^3oM$T43oJ=#-)j#pK+B&`^q*wt@MW~l zXO}JdlY5t4fB=O;fcR=jdL)AqEd_W{ukTp2wx^%WLQYM?!HJlU{)!)BvHmd@cE4IR zzDhO$%yx^c$2Ru>hbX#QK$sOtA16>fijr9|^e($On6U&g1-|}+1JlvSA8~0wYkRjT z46MCP+Nx-Q6qx9!9a8vBOWuR(*T?6X?=FP~rGv19S&mg0^ zDKquTQCon-s+ef~7}j#VCf4`yM;%Z7cNgE31K_%CbtzGcce#(JGyI~D=g}K1)St9R zZ1q}4GF`SVnrKs4cLx94*LRyJN2%_<)5e*dCYoMBxD4J1gvl9pNt_ZmeY(wZktfX1 zC6uA-tX{R}Zd z3s?BaNf;t6PA)tZGTW47m}~W?UyVHtOsgAMUUq+Jd1zyLq}Ytf<18%cGpo%@@e1b< z&D@Ub7X$YpVVnEWE}mV!+js4(F%tLlrwFD!6Yi6;KAH&1u5*0A3tg`dzWT8AhC92i zY834jPzs=XjE?~1*^UA=}fb_{U?&TEVM+Itdy5B6~o$N3{3Es}tF3s?0597;?a z=%KuFt?wDxrdWpZTnXSK4!?y(1Hhy2fFvMCVu2oJd8^iQR+o_w5TwJS|2&oLA6eTh zs*R=>xUVT`Z{gq^uSClRbfkp1i*_mOWIDRHapXPe)Zxi-0ggUS5e%wKTOumchEkeXVO!WP&i z!%$|Ho{v3;0!LmTlO3e2y-{GK%F0^-EUn~6asYP|^)8&A?w*+gC4 zws2U=x@bL5rzv*!;4hJXW*wd@VY=X~?j7;LZ{grKGu3iKEhst1F1U5XH-H|HhTH+K z$^gY$&!;}==?o07@tO`~UvEg08)$|p3qbf?>*_E08Y65(v3p1FHK_bP``&vIsE}ik zaduE-Mw_uE?0f@nX}#u>(5cu;1MG;DQfg1lfxM;abEd%iq@0%PL_}VwxUE<=4W4go zI&DywX!Xpjf%6&R8fA&M_KOoei4VwDlQ?>iOn|hcSwrgEJkRv%nSAH5RIZumQBJ3` zR3s}LdqrW~*~iBQ`gHx)2>y=k|3@)V;&GDCRX46}afn``zQ?wl(W{D-yB0^2frDFl z1l{Zh$KbV>1|Iv&0KPVCw!n*QcsrD*_QPA`@bTf=_1!lKM=XcyM}nRgVRhJZR!ookie2rj@HvgsF>=M&ylddBJ6aEO*(c&pf$k!M#rU@a^$t*VEH2(2^$%44-+u}D(Y(LT%8iQdIRWBr>|V?aHQy6VdzZ*wTOVc_*){U+AZ334;`86os=5D z8h{~J=Qw4s!G+-LY+}d!ER}b4iQFKqK&L{vsQxln9YJ|;!a&elF>{!nT;r^CTm;T^ zr#+)Wek^*UQE};g)PqqUly|uRCbG;McTqmq*ZEd)q^HVW(#GjrQm$&Dpj7@4e`r`e zfRX-ld8Rx^t|U9DpDbm`&Jg`)4$`i35t*G!13A_N;r8nSw`J55peopb2kFA>uzcZ# zOvsH?$woF+xxPBAJXwh1ULAzIPh5!dQ^X1J zgM5F0BEZ|3ntgWnaxoXyiL>*zsGuW8_&Y1FVYlRDEk$sDX6V2Gh@Sf^-a(vWT`7Bs z-QGt!5v;&x_W&=mFE1Rn(U%d4U(iocvlp#h9SU2M)tLjRcG>hDU+4Z+^-J#9{s#(W@~@u1By1E~cpC#Ca(_0dXKstB6MX1`N|+0~M#9(;B@WFGwFt)XP) zepH*>=m5CbQvAcexZjKv-jC;CTO<|FJwPJE5b@i6%%df$WYE1)*3lUfUYI=I$n}u{ zeFPFTGS9MAs^opu%%obNOtc{F-ivp55b`BdLPmhQw;*lcL;~-C&oA1LixDlar0~>k z-{#QAbO_e?vMGk%s<{=)vh~{ zl70Pg-u&Esnw~vg0DsH?5wyL{@VvKDSoo6 zO#GhYSEUMCH0VPpFFcRfZCYpi zb~;T$J`4J$`X8qS>&UQcRijHZ{m40X=GYV_4C?&WId`e#zeco;YbUZE!1HGzS3gxE zqc77Rx7yLMX}xiWu-Jx#%uA4F#_jZn2*vB#UG@_SW5#dK^w8TG6hnLqa~_>5;yD`C z{yb=aidzN=%OE3TEBJeBj#m9^_((41eh05Rkz~rpi6ggdSbbMv&r;u|BdOwrAk3V) zwINb7-5Pi3MCAi2iooc@`RzeI-wpyEU{@Z}%8s;-n-8TENq}y}k+A~1salFWQ*SdwqFjN(YY!i( zuRJ`Hn@nqpmB|NMX5~o2_DU;LY<5#2bK_0 ze1@n@MwC_T$rCjGUv<9R4A^}cVKlAc6||&AbiftUrXY`t#{AwD`{vZLC6--=Nq^_y z^yDz!(1z^*OEtC(4KTqM8c|~W^^x}T1+kZfx`NzZn~&HUXTQEnEEmJBXN#j(h+|(J zEyu1aZB9<HhC>{#W3&-3#%xNlF;}=T-i%x4HPx!nk{O2;d&eKJ3h= zFu$(2N1qAKW2LWjwbZy%B>r?|1S@*|;zFM6^kv9c#x3!NJ{oV?F>rD8;2}w2>%?<> zb&?+lOhfutf|rjG80=Wpna$IeAL994h(yfZ2B?(KTORHnGe3?hR3YvB_Jif)rqp%4 zv4^lgJL~n~Qw{h-`M9UP)0e@0B>bH@E@^avl)gnP2rToss`i{?ZsT{}Wj0G+Y{@vA zmN~$NsJ8O5HE!*S4cQHyXr*hR2h@14Rw4>dJR-8)C_A!an0ws^b_`FC&PPd!ZN@}f z^u6}Co%M;1N8Be*VOwrjoMHdk<^DDPm$N1;KrnTAVaZ{ z%-MC;BB=xnm6pQqKk9GDmZzuW^Ps*`$rGyEsTD@(&-}Qu^HYMgIraT1=kGD7{5kC_ z=7Z<5vK(G)d}?Vu29KdVtfyY<7fgp_r&wZDTb;w>`BxMq+mn8}w*l6Wx6Ic$^Z&Vle~JE#{lBgnAHq*)z-{OXotkk>>fgivyZ!IFV6aP9 zZCTsHm``UeIrX`)&S0yxl@l%E-gc46o2SvfYI#O1>D6t!a{0)pgXl+j`w5TH0 zMoFn3Y|~kjTQx}R*a_5x%-O`H4U+%kM)L1AW>aOw@eaJ_m7+X}Zy%Dw(ZnZ!wijlb z#_D_tk2Q^Eu@FNTT(4oBHN9%@h3)L}wKHI*5)$6k>%jo_(nhlwu#c6n*#hY`o#kYf zOr^KTnuGP_v!{p#1U$8*z*q~pp}gT@y^(n^*3Y#Enn>7;8{3i_$sP>KC%N4+u+}2B z=!R)v+vfiI+d=VMywOngQG${UH=}j9y^c|HW07Kc&ZE^_dbcOIzy!Xe1&ys;bNhNj zeQh3QL%dCNMoLpg)_5i%=X~4~5Q{Fee~xg`z<35)BYtmj8|}t{PvPgVl*6ceqo*#v3u^X;4CUe|`KG8S(#n z%VEG!=Ia%9vJSpa+G#lSOS~mX1oE@)Vb?M;l^kaJwyAWf9_LJuv6V+!qvz64=s_@@ z)NCStrTcM8-};}Q@{!q( zS$QbY9FuHk>W#|WO7}Snbkc`)_*J_bq}v%MY#dvyicH%5idK57w*0q*z5X`Eo8|2n zA3uPDGwGvKVUu~&n0gQ5UG`fJQU}$+dkktS>`)FJ5WAKLc~{)&Z1$GKvpDW*`|-hh zFC3PsFYb;oh)sKRbLj9%_@6n=zFBcLUM!M!VbMBe*Wxic?Jp{hHJm;gC^Vkf(%M^F zB(R%nW^zh0TA1Ion9mhLB=WzEX*---An^%VP*Q}zR=U`Vj028CrC0%VCFjur$x@*h zF^XU|4X=#&k%eW%sPRO2;p`_JK3jBKI&bQ*5I${Dty4S-o<^k0=TO(K$cY$8g;{IM zJ)=EIAk%J#yM>1$RGQh_f{Vnp9C z=Art6Gl&%0UHDr)fK1m-Q#l&PU~`4bZxt2b2WU(mP@c4LTsk3` zZY`LoC>i&rxjI>>zFFZB*BUaRN_dle>JSbAIL^98lns=RQsTB9uN_KR|Fiyuh8O*q zn0Qu^+aSYXa#Oix7`ocuO3GHJv9LRrVG30_6eu*1;x1}AhI$S5Pp=+}?D|Y|b+w;X zST#`D3>ZFmG32P>qymjH<3{`PXvxFI-L;^(ZKvYVbVDp_t*|8yEf}k{mQ15aA4WEJ zp6YZplm4#BW7x^!^lMZh4Djb|u5-UvnHnD-$C?`l(8aNdiMRV2z-n6Kde#&5R2kBW zugh)ZF{Sb7Fxn5+=47}XQcaw5waaW3&3=F(a?jB6zBkz(3gyof%PF|i zzB*@6$$A|!T1-1|zupky@V z&UO%dMg_vgUxiCeyX2`y=KC|dh7XUtax{NX2f5<;jD)ID`b5JnajppB6No;*ujkz~jDxVfX3>&5~wN8OaGg7&pR;Oh&Ldyaa1-mz%3 z6}y)-735SdUBFXD%u80S(id&vG?_hHqCd`KY~UN`{Y(bG>=qNtXXcvxvArjPGG`Xi z;X5BI@)Zl{E+>L4{4*n$c;KJSt>NTPztR~6l__6ZbfkD9EtwG$WL!yMX=#M{S`|dc zJJ6l$YL|_G`66^|OP$b?I`6%3-Aq=Z6P?mod6)Ak2JW7_Us9Jkm1j!1SZ8$+7CRzF`> z9N&acZb0?48wSvUe2?mi^I*|(`55pcs5KokMZRIYo}puNJNIlL`%TNU>R}8FYw=nv z(eZ?XRW~O~Hc=rS*6{-|Q$!-B!uM_sw*&ZA&FhMG{MVNUMoGdh`W7%dyS=Ol>%LlU zKoT#nY-}NF`OutJ*q!5C`t$*Sbnw-APd3KNEm&MQNLlQblq?^v0x66%tpREI?7<&0 zX5H~b_0wT%lSgflI)|%$sjn~YG+a8rv48=S1f(CDs!UY&X=Edri+VPrQx34b8%$VD z_1q2uBHO+)yT=&n2X9awtezdtNK@?(Y5Ifh2Wx4QIrW6{@pXSVcGAID`!c}P*R&@) z@lKs!zSTabjbZ6?ctKRUZ%?x3)x~MC?X?8Q%jX7z!ZFVp!nz&m2{$xPmJ@6fwT#~k zK5i_D(#?5?M#>ANF(n64)I<|@-@D7g&-i5b%5iHml-CTI*W!CliS>90c&tvBy6=xk zK*Q}IGmI#Rp?uVq#FrY-$nCN{NV|eG_!%Gh6(}Lfs2 z^Cd;Jx;OmO^Mq+P!|lNz9GuEB&a2PcFC_8cm&wv#Scq17%f_ezeWa+LcmK@yX9H}j zQ(gEq&VlAtRVzor@6Z7qv5}|CA35$KI{9YjoA_~cCM6Q}DpAfm6MC)Gp7d?j4!^_* zqEDzr99KhlesqI6n;}}1x(?kMMTz7WG&D5aY2I9$lcf13;=lLGtp?&^D12PpD{52`B^F<8gUItmD6II^oOX1bruYX5U(9YcjY4KA z!xQ0|Vmz9D0^w(Ca_x^JTp9zhg6v{(e!uLb!S{PfL7bP@TMjj<_c#rmd!+KKkr|uC zP0Hd?i`{wBV56KqwxB6+mPNIf9bKIn(o47hA?jnKFY9v0Q(oX+x|;4oc8>t?s-pYZtJha?;P=Tt3=<9&Yc(e8~fQex4_8qDz8UsI#ZeUS#%>+ zFGX=r;u<~oGaRJMMYD6`YdL5pV!ngaKM@JK44CyYRc4}&yWj9 z5!2@=G%J1V5R*^hwVk#5@(QblAz*wUN~CSe?xl9M}&Tek262<-FI&PAsbEXDFTt$hm38C4Xcp1AV$` zx6oO`_1Akb01qY8U`@7v=?@`9PkU}<%01^=q$~K(L4TNIw5$;GoJ_;dJ9J z5o5g4x48^*-%rbJ&mZ``FvSlNX&=dOH=`6zRZX0NMZfCroG(ZmYQie!gv7<<2q{N% zw&yEAx0Z*MEK-7>t!V^<&P~HDXRiXCsvJX*gQu98Lp)TB!ihAy4b&b+MfR^GHY=}U$o{|yc47TZWZDlJ7C$YCS>Br zESlJU`s3g@{}puUvzVv5iv|g)qIo<{2X_tJrAe(nTnqXp?H;=6cX`q-e}48%gk@g_ zZ<@hO3acs2A7*L;;q~*AvreJRh!&QV0G)bkip{(4|FbG%Y&64%!X3*FqEtFwQ(XNf1she!Bw07{PHKndg%0ReJA>cUc(N*EBAMZ2+1 zp%(La1Qc-+*W3xFw0O!zZW0wCr0i9k@SO3&J~9pQp^i-WzOloT&6VGj=Lu&^U)#Kx zp^>ziUSjC&5vKy$W4bsHvHYEJv1gFCzrDnM?i^bmpp*{X-}V4Mm=zQTfP%?dc%(&H z72!ixRoRL1H!iX1QZW;zrQf{<_~5IL@>)zq7aZoA1BWgX+stShR$2}mlrBW+`avv2 z(91~VXtV8__5Mc4<_ay%A=@%bQ?)LDjKi2`e@D*>Zl92Bb`IEn5scbcAE()7*p zrfq~RPZ$^JLveH5BYM0*7nDJ%i&fcle$#cF`)4Q|Hko>2x5|;-oIGQyec46sj~;zIi}u`Ka3&s z9nj$mbUR#U{C0V;Ty2njjjsc@QDCimD`}F1zSSi1UCPtitb6p|}OG2iY1lIVTA045?CeR*>>ZffbYHbd%?5IB_Rt}G2RklbO34YE?x9x z7Fm`s3L>JfcibdDE&lwwDEFiuKYCYD%cGAuWi;B_gCc!Utng9vhaPkDe|^hGpjTdg zF2(UIw?;7g#pixvh5d|4y>pyL!z6{^AJLlz&OVRk8O8uDB+26B__!sZIo~TxmDTVY zrN8y*ujDr?!}Tr>^q=cRh!KsB_8>8a=V&A=+9APjxz>U0*+ViW3}2C#zz|(A%J>*= zz6KDBN;}18@_OTu@Qb5)SL7{}Nd12yce4sfEkIkBjN2aEdjARx zAWVE?Fwq_)DfC_#ui2IGs^3gKo1wD)%^AKIzCJ1+lldx;pErkP-%mw^?EWi+|FNnp zB?A(ZA99Q&AVR8nIiuPB5e$ShzKI1k>VfH)Lu({Lv5aYj z@4tHA{`lXCi>Y*wt7 zFKvZsyg%-r9Rpq#888@SQ#O-LFp##Ymdd*xR|LmQHJvO!yohy?T*;hh2N(S+>Dhlj zz=DNN{SNtY^v7FN_2&Oa*jt9hwJc$yI0PrSTY`i@2<|RP2<{HS-FodcZb2m*GMCv(si=MLeZ)GvUA^ur5jsm}&SfiC(}T&?w-i{P#V2*bv>F z`v>Q{L+9C)Lqt)=zijPqKSl*11%AYyl*-17|M@r(A;2uRZwe~KRH_e!GKE(5W1KoR z2oYW-XM7fX4SR09i!mnomqo>q!g7u{&7{b+J3rdq_C>~M{GW>vWBxpR^B~EVe`e?v zG2BZI`3lT1ne@EHJMo=2%zmz!E2{0K@ zyPk?hINzNm=dxWuaM~J{24meM&X#Bz^%f|Sn+~VJ1LwvWGjRX-VlR0lvpOPd)}bWJ zHSSUg=*3OW07S)mZ>H)aZGkEiylKpkcFk>~dX+hej)&yCYqYWh-z1hR=4b>R&%WSY zH*T{-Vfz(7Uoa`REsTrhw9@qD*3hF33lN!|*WWtsZxhd1x8&X-mK*iRB@2dH3$2j$ z05|qd#{3Dz=}WpMpZmOyU}d05b*T#1yBoI@eTSmQF2ZzVP0hyRxku#7qbob7K%@(Z zPqXqwb+g5|=`vqlZf+YOf&bw%5}x$-;&~*_tMxJI&Dyg~cFR7?Y<){1Gt=y*!`)sv zzF;)`HwfH|Gsur^R$eDx1iAF>Mb&5GDhan%s+--Gg_gN8JuP+316p^8*|*w|$guA7 zfc}Fbtu52-l)i%ikM?xIBsVGi)Wjn4!0$CIFw;2QIt0K2AzM6p53*c#Gf-t|TKlD= zt-Z-KOe_g{c@LE6=F?S~FG)OPaF<*ykK`)o@0*bn6EOc7 zAK7;1B?VeXp*w94h637`>zaDCpCtBk7DJQXUJssj4-1bq`YMgB7|@LwpH`pgPJJGF z&AJcr!axoQ4iNKZ-+IVL`GVOk)k4*n<(03>FL?4NzuN+e-eg`QuD2R>8lgPUi9(?H zYbHjdDsQ9H=M;zT#$xp{B%ApvmFdd{mpxgJ_M92St35B~CEV?fd&KYnrF$}GzD7a3 zw;I@{Bi(johd>jk_#*9QPgS11L9XMg)45uH zc<5|!X?lrnBWR#RrI5zC+Y3Q=zIeLiug&1Qs9KJFr0=xN+~yVi)Ma}<&lEMXSvguJ zdS=^N_$AmG8n8|YJg&C)>jv0%FsW7};=qkSa93TDXFXy{p~(1SQ_XhAaJpv8rs{`x zSb&~bZTb>ud<+_(g`Ga%!FYOXB(urDlIEECy#5JD2C`rbnUEPd#h1zS^fMg z`I@Mg^c|9wH;ZsHr&m%UeF4^3imLrMCsjK-#=>PGm2 zq{)nzyyTNV%pgTP(W3{^dD{?^)cdidiv|h7q}|gYE@?2(vAxK8si>V>yUE>QlbxQj zNyzuStK4CnAT9hMC&O#VC+eucV!T^{T|8g7f#EjC>rG}li?4IE%sV9?qg^xdM<|^v zi5So$V)E?J6yp(ERYhZep_;^)nGU=N5gNUemGMw@okPkAbGjXal>xf*Q3mc%t6FK| z_x{tDt*M~O#dkvbLrKmzrledY$QN>w%Ws9^_Kv0V^YS`erg#6!QUV>{UG+Scm>u=p zUdRWBV92TZ^havlj|Cs|7DV3)?wv~OyNh!7ic(_UFuGjl>}A>2;OH$o+bEX|+tECA z4sq*}9!u|?%%=PLsGs*C-H(xgD)G#+>PT)@xUG-v*bQCLZugyUb4K{D$Y!D)e#};m zAxGET$pQtNUhp7|_TA_LJk9lLPZ&|ItD|k#6ad$5uB;*-1+UUd6(iTrSn?s(-O10aMj#&P3KeT;kGyecO1b@rv&elB3+`-#$|~AR05ha01zcp<*LP7#^@P8Q~|O zsEZLRx(ywDR+RIT@1aU-V;F9J8wCb8kK?x*%UL+7_3EFUV{wUzNbj1Db1PCP#GG^Q z4x4o`c#NH*LROR;E9b=KYknG>Cp$o4j*reElJI3M_azl6aQ%FL1l<YRr8|#WpDavoc^Y=|tR@1HXPaci6#@t@(xZnh8$vxCJXK02Nwi#=5o?TxLuccb zoT@g`YcasSce2q#+zX?~n5$x8FXHk<5|NO-jhwS}K<)MYrn+*`kJnz0wi`srLb}mI z)4Fc2`EH|&yk_Kdpj(WR$4Ok4GcYWYJqK4aBQlSXRSJv&*x;AWSfm#Z8rLRsb~WrL zzcwI{F3I)oCxm%Up5neRv8XA<9G{6Ag~BKByEPYfio8cQK_Ob3YKG*5Z$>K(7RsxW zyIlch>ll!6IBwLm2nnzYvw;VYffe{`-6w!iSRQS6d)tpJ%A$_J`R!#TrWq$MUTJ*; z_uPG84dUCf=F>D1^}PH0aIN089=#$NZ!x(Gpoo`+=7-OZ0edHaKPj<-9bkGV2p^aK zIgnA=$3KSz4vIJUMzsiUP5_6z_L1=AbWsW{$73Yf&GQ?9orCw7=&Q`Nhe9`=?S)$M z_?649!yt4y-XAtf`&KX7XaH-Dpd6?zhRaIcv(*>p>#u{+Zj9O)y1!Rp`ttUI6jYq7*CKF#_1I{DFpR%#fGsM0yPWeB zj9B;eyY(f7v2*73ce}*y2}<8In3TQyd9rh9-_S>MozFjL1w**T?@j;6D_2{?>`YbT zD!}Z9Qqq}gT7L`|?=_PXa-U-b9y&etF2^Gjk_8qGVPTjzpO&M01$VZ|^ygD>Xcf;! zAw7lCNF(0D*mnxg6^e8ApWXE{c+9ZeTFx^i7Py_<#fGcYHVa+^Jp`vikAmXSc9*bC z!X@3o5aUWrU*{f7v-X{Q?$Gzw3l%_X-Q`84Dsx2$Z_sM=)H?WK#M{ka-zu=eBRXUV zyUL)N^Y);(EUSl77XhdjWkD43SS^+=jS5F*2Zjtsf*!+OXug zj%ZTJ;(MN6dxfZ_!DpCFv24AkmS2cLun|4=iz*Vta@6#IicZe=(&tV$TKKj`+b&>n zw$V^vuEgj&Xkc6$rc0h29=cQ4pRATYE@yp&OLfH*#d1GaZB;~>A4N%5ErGS~+&*4p zNzPO~{=Qcq4?Yk`+!=ovBQGl}3brZ?og%~Vr!isw^5J@{tE|&B^@9?tb1@y5@mhuO zE|_A%1)-M z7Z=a^3KM_0hx0^Oa;|aN2Ne(`OVy;on$B!ONasT8p`_DPt4)mmow@2)1#s!{9Wjk_#mI}GhoE$565Tn}SvxJ~T^lic7>6NzVJnvJc*!EzYUZho!+Sef?LG4(_okvAZmwxI zaLBN`-dI7{ZZ4wLOqNUir<&$aYmMDVz=C`FG65wc;?Wzm0>9SnpXF}8Rop5XZyIJ~ zW-$_mw5tTDz6;DIYbts)!=Kb>YdLF-1k%oslfDE>%iFnc%5H7BW{* zBp+{ehlh5B6Lyk2CD@NG5HJsO{|W9kYoG@n{d{WP!xMQoF~YZGDrL7ulQQv+PpvU7 zYU6XhM>-352g8Y@mEBrMLiYMo*Jzk#wWUeZws{rtc1G2ig<0ZL`nWOK!9uPX;aq(o zZL*|Ltfgvkk(G96T!M418G0(Ivhd!Fi@Y5YWZTg*4uT1`pngB(WWJpYygBj{N1|mV zF?`mB6Ac*4(p_0!OUrNpLkr{1qd2j4UqLuCKqxSMrbpA=W)HfJy0fD|w4D9RoBwA2 z{wZUJ*@^S`AP9Wo;Wnc8H0)eaZ=23%nfAxTiEr++CkYp2E}e)see-7IZnDre?m+FB z1-%iHP^N7=;N1dTm9pj1=PWT3tHbZr-wS+ZKi&{1_8wMJwj&X-`*>(L|F$Un-%b{4|2Fy#eo3g6g9L`(Q9nGW+ zw3;<PPbhW;s%j>)|WmA5YxBp_kJ1zq!L7q2AEwyt+WlWozDZ-2{o{5+Pq_ z(R1lIJ2fDzM7mAEgkDJJ2us)6C4yXW9@LjY;mD9jDI;4PQ^R+lX5-wAsskY-0oM^y zBclECg?-3npvvn+{mp@~a`!mUm~rjtY}aN!`ayZnb&i9c^w}4aN~FwmTVD=p5mGM; z3%HQD62Ok77Hwb;}Tz1w! z7z7I3ckHkG_qn?6CjCFe9TrwI64f|jsd$qlP~v~r_Ey==q6m>6dngod4Uvi;NXZ&( z=?E^jVWK7m22qPN&R!k%`7btBtcXmcLvurTlyRqs3EE$Z)cR3}0-%e1EH$>rl6$QG z_oKm~HeKks^iFLwKjWLHQ+_zMtAQgyhTH!^g7p6v;LaZgZj2oAqdz|In`%+fsh5-M zx>ep~yCLY|N72V+oge8?J;wFn&6Md6ecr~^{gKt-J(^8(vn)K5f;0f z(1<6oT;l;KIWJPWPvw1x%$BWhx?~Kd}y{$PeNNvJi{|=#b7L13T&BI z^l%(Jx1UKQt|vr_kg*G9Z>7v9H%(`1Y^m@0uI1War(ln=d5_b)N>RYk{Rcj!q6E;} zbI|C=?CgK`orE}512TK_7ZV1s9;{oVZ9v+-u7BIBehj3yCmM((W1^k>{YF2?7`Hsf zCZ_B&7Cx%&rHr8U?=sCKzsMusjsPbTn$UG14CXk3%Ls-k`@6uH)zNPv6w-o{qHw7@KKBwo^8gKsI z1Ziz35|@@hBUaR+se%{twbp9wPH$vch+5-efr>H$+dY!JkiLYJX)(nDI5=i1kFp*Mp^l|lQr}w)KR_5ui8gs%g zg8wHj4;2Ug$!$5_$NwU+&%Qwi`|DcYZ}OkjyPQFRcUPTLvZNiH8*TKNhR^osQAe8& zHvYsy$0Rnm2U$Sst8J**7f51fs3q$)NfSiORb)$Y%Y~IBvQrCUWcsqrEBvAWgIwx$ z0db~V{wl2Hl7nqNS;rQri@kJ#?d)4NNs-|p^45B;HU2jy#Ui|1^&)gqwrK+@!O#&2 zNbc8H>40JC(x=kM0P7Ts1xs?jh>JKNlf+!sol}+Fl@_>tIJr6KAV$Uf6UEtbo=k|8 zm9;=Ljt*mW!j##dI}Nd2fcJwf0wy_M&{^x^b;?g&`{Onrtp0w1QTPu^68|jz-{hOi zKTd%s14Y6$L}??rrBbKf6GNjlvE>)YuQNdO$0Gc9GQEgQN7~#ULflwZWHH0Aay1b* zb>BgOU+}`- z8qyEK-Wa(F5OVHN+}H~kb6)4w$1(}G#@lWM(#-;BEF`lu!d}M2#s-wFEx(wru?{ej zj+V0E^S!je%h{;<$3wir0Y)aDJ&djTFIPf!?0k~>womv9H56L@><-`m< zyY_-xoJy{=zdmqhoVh63#zAQ6mtI<7GYq+RA@>`fs0Dvf9t9Jpe)`q$b~tkT z9y)=}Kx)cPDVHFFcj9&*!VA<+$YEF?sAE6+Uh)0KZnTm~zu4^QPWhJ;F@4>alV+?y7{(#AH2;xT=g->bF{|9Nu$vdBc4T0z|uCDbn)lES8tr^C0g z2A3#%y)VT-PD`&O5&O?nqQHfT0tOh?u9}Pb54X{QLsb?oQ;}BWV(C3o>xQ+zP{X;) zJ))Vqm}#_KucN~OjX|9-)kBj^Q41wtDwu>J*o~~niI!o{B%CmhBM)IAVLe2$|4M1u zjd@srXClQG1Qv+8u0%ai;)>p5w^PE^@BF+asPr352!kgl6ZocX<+al!>O~*oBpoN- zdr4;OUo~kQ&wN=}Fd0C(w1ox0M1DRj1O4{OX>mc@*zf^OK(a|tPj;ro zIHIn>aK^_lu;GDy*7N=tVYcjm)|?YSK*|bb1NPZ+Tjb1 zer$(;1Bk*iDY7i^=NoB4Gr`v|+=@$P<15J~9l1fo#>PC($73m3LJ`e+x+qp(;r?Sn z&$rI-zpx2lw+NouE#{>^ssBu=xu~DO2bMn9BHPaHm}+mZCpn~O725j+SuG-7l_v^Q z%2nXkn?d>jM^c3MecO2bP6MFC8yJ&&L0+PR2YY+riJ^w44r(ye2-=a-<;x1{i8Y!~5*RGjSm8J%ae}6kisZP6Ut>3FNCwNy`ebI*DeC*!#rvpl1+v z%hY2;uhSDo2UpDL^!ybz%|}>M)b{>J!BXhHdzCKFM&jxWxLdURrR`{eXyg8~o}eN6 zv9A*o9lN+X&*z8i-t-DzV1RWh+H17e{}5(=R1m^v7OhJ;_n!|%GN`cI87=u;P?1L6 zRu0dA`*ytPD(0(Aq%T+6pW|gC_hB}a^cby(7>))+zot5PODg8Qr0g|II45aSTF>U% z3w&trxNgFh6G3U!(M134s6M5^8VX#}6! zVLX5G?K+`(xgD9}AiZw?d!q23Cu)s44OqwxGX!(0$Qagg-Lv@H>-4kEa+7t>o`BVL z;%K9={F0c~9ls&CjN0FvnWD0FE!hQGz>ubMSY^hK**FEp^65dn$(Zp&nF=viiEgkR?);vpSJCT_|C|B{u-?z-4P_$Dzdk%g zfIUU3e-)}s4eNX-M@7(iGe{q^^+P;*=39seXB4JT{NT2r;>3rrZBHQf1ED(o z6v*OoCc>wjaIUM!Nhr*LPjuKKNKxnTqtzc-IMHLI?cGcnp(R=Hk6La*M}x#+BD5Ge z3a)Bsv8Dfus=mR1CHYPBN>hpy$sn7QTDOBk~@nV2o3yVuF-ZRibx(!&`} zhD(q#S71zaRGn+a05Qk6kcirK5FN?m|8WwV<2e&0pu}%SFX#`3Dzo6QwMKmP#RCzm zbN64LdTgyjREx~o4DT`dys>m&?;#S=y<6)m{}W67cR2KX7I6SU5(Ne|@4sl&k0CG~ zMr5%vT*#wN_Q-s94#^dG{3ACfieQ@ygGA7cBAdr_R*H#t{9H;;CL@!_{(Ne7OF9B#2P$D|5n9zCJRaRb zOBV*E5{5%--OkFuWQJM;%EtqNr10=lfnX-~vfyk5&e5FiSJ825@EYbIl|r+^eqc|3 z-Sj}`?r4%hivaM3L4Kw9!rusg*}PiV0DI=|cts=p6mG`#ff3k2={(>n6zsTzZ_u?C zY}rz<5vMSXcam$x%=_<|^Ly5AxQR&u+v)EZgX$v=m3*v>BaftJzLx$f#X@0#SsIz% z+th*gnIipBuWbWvnji~ws~d;5Uu5ez&S3j_JRKS49I!YB-79H6*M9_N`J)IihFY-4 zQ9OC*FJ8QzNoWz$njMyck<`jo-MM)~!=UlFta^e*-2E-?x^|!1Pj6l*L7dp?D~=@g$8>u6cys zqdSGtHJ6YvHpDKx*UzO`y~E%84&A-?`YWTQ|JUlx|7e5$8-_lQi17B}*nu<~ZYB+7}_Kyb;@A0=Em*hKJ6ECCsVlg=Zpilj|!J?E34jOx~f;<%wX zN9SKl=%tn7mdl_R_F`qA&hv*gcaoV9mN_6Z6lV!@2OHxBUPR;j(Mm^Cy`>x8_lrz; zlQIMI4hF>sp4<&010(p=J%b#$AC5p0L!QG6YLrofy;$q~T@G`ES$?;+(voxm($PC#jB9uTzFef0;$}&o;w( z9&G3P?_~eoW-HL=8Tykczs=+~KO-2RqFOA#Pvi6sFJXjYl2;@~@A15T zjR>$EFpaJzolDLB>W9By_m!yBJHFo>gVuI0D;5`6QiQ*EA4N%TM2D852*a?%2*YUSqt;oq*mtUzJiMI2s4!KR9S90mI zkQOp@?lon>{*Zx1cI<1TOg~Yeh*43&F_)%C;g(OK+GI5-I}INSmPhNu7dX=18}(2U8VZRF#53n~n>5%){i(L$r~Gb~ zLV#?erG?{yMeBjgIoMh7f<*grV?Vf=!rHWlvbeh`A(2J-(+%|Ua5ixIa?Ajf_F$=3 zI3@*~=0*_ma%P0LfHu;9pU*ir!VWK#c?h3zv=I=B;02NtdNRyee$`dvNwM34wB{c0?6 z2kOsi3<+PYxa4dZMmYItwfY}&`sp=madP(biKa+x-&Qv3)K|C{;wfCQl2qR3RaJ2? zXy;WmgdP9|fXaAOTRkr<%Skgep|?}tXSN!8Fxx{hn2;ow8ticghr5^MBf<_1O-@=q ztXB$wi;HgHhp3%%9Sihf)5c#$g>s$)bN@oRb8F_kQ$^U8A@592gLR%zmKmXEcRp{y zvc%PMc9um=J?dPObmdSm92e!K72Bxb+h&2e2&b2vHP?wK3V=X^^JpP5JjA@<6x_e^ z5OV(xpw23I7oldlT;r9mZi1n6c`%S=6{PJ~jCC+fq~7A^pNSL6Qc;jb;rU1FG4W+3 z@B8DKe|B#_K%KHJ0+&@_w|nkR>Qw}d%7+9p!vgRVc(_Rj}iv8H*&u1(!wFb&=e zHYzljLMcRBg--2Uo?%zUN0ndaIFizZKx-UsY&F}rqMJRh#B%4 zGi|by-ba>@2y4GBB0I~9Kg2rM`SkFa8SySSYW+jkuS_&TIn`6LvI8QM*pX7uO~o$n z*V2OzVT$W)gW!oaZt^mlij;f94e-Tj2n{0onz=rVh`>;!vNBqy-tXh%RdEHXV-Lz6 z7(Pa6R9?DuiBF%h{DeJg8wkk5yOTe*=y?+4dnIE&sTRgUqr2ojG-PCdW-qnz4mYo_{n8dp>wCWaUDe$vLp7?x-u8mT51Yx7+ z6nuV{g9gEB`Ic%|%p0VS_{&G~T=~x#)?%mgHK%#Wo2*&Nbd+Uln)|WY>n53Be63;k zSd+y$!dm%uYBxF21vkH?v8-j6 zjpw1t^mukVaCO-?Ns?jb0BIE6ysbc;# z7d34n9((M!TK8pMqS0o(q9+$glmBo$h>90C# zv4?~rNlttn`I+7rXap=#Iy0S`B(J2<@8E16@w%hH8(C84Gdevkr!i3?hGfV;qskZ%-XAa zqT+@dODTGrVQ5+dlgi5=<; zi3})g{k;7a%_J;f-v;vPbD!Z;! zP)@FZxvk)moP{ZED2lty=tFbBf!|MTHJ((UPg`g9pBCqqFg_{ty=$dF8fiY3lGu?Y z;SK_FYdU)sg?WZQ_TH${USl}VnMf3z`ns6kvsX->)KP|ZiZszj5pi}G=KhGS?Ubfy zzzU2AUKuEB@y8(L_9({7QYlu;Z`XA@4~`bTdHHy6Wf~N_Bp*r1PbT#A1qGc{sNd@c z%4^E|qs4v)ek-VgdzRy>K_9>-=C86(`W`luBZ2M_?@zWPAOgb3975v@*V;{PdmpZr z-9oj3uDaC44Hg?PS|quwrif6aD{iiU0f zeO~6{GFJffRQICS=55=o54FPHfNHHX)CU$Qq!UQJ?33R$=G0@uEe>exCIr7+q*k}M zl&Bx!xnJ%`Ck&03oY7InhWfB82r7G@9iT9W^V+tbq!18z`(GVQ7EcFRhKyt7Iv`_VBl!1Vvu)`hD4;fRRiO1vN zVvUk-eDwRE$fw6`IU@h7w3N!NptSHLk1as>wA2(V z;uC8wDyo(_|7GdZyWeK_eAQNkKEuE7xWHx#8Tuj0hm1@0!)G5I-wNNa2=;FgvyYj! z_yh)=h+Lpqny9{V7ZP|Xq^AtHfmYJ`&iFqNNkBZ)>L|`A7zEeHz0J;(Ps$07=7AKJ zfCxSp>LnP@&3^oq2RD7`<^Ap3`!V?2bxwBm8&enSmyM_2^hS$R(Khp6=|h$!!movi zk84hc!%9^gv><{}b;-91PydMTCDVe|Ae_6Tfm*;i(gV%~( zbbHi#m+GmPbwkh0atekL7kD?a$7A`7(L6nxYb^*~7e}u?Uf0QB-wWQ}!37BD7_a;S%)-;kq~i`qnz-*qKu;~2TDkn56WD-w zV|kzHoiIF@%a#jNNpe57?aGhL(q8-Ieam>0LwFsGP1s5RcWTuHs6Z$$u1)nptBf-> zHjNdUp>{O`yk_uSlfB5oW~IKq9Lyxi9EQ24?a|$4)Z{xi+Y~(*nT>w;W5$ik6ZOi- zb*JAlhGEZ@x-UD;r@YfNt7#XRdN~u|k3e21e#dS&zQug)SEV)( zXu8N(on>9mR*p`H4%OJy$8!UZwtav{K$LfW6Z<3PmT_*Y{OOTvdt+J?31BJ`0+n?4w zBBZk=MhOl-Q#m0tLS!tu4LUORsR0q&1|s1Uv0SUA?BqLx>{`VzgKwPOq2=|aQWzk3 ziP!sP6|+wWU)460MfCEA%&KgDEa`~IWr&B8xqkYM^7eoxkNhnl1N=ydh{d2G!(}&t zEkflqS7TAAc2wHYR~S_bju|gN;v0ZcD~Ar@eHJ=eaGHQa(B-PN@NV#&p#kUcDmFPk zNApxUQZ72sYMp@&wAAvLxGzF14Wl>=G^7j~RakDRqvBtqxZzcPYQ3;DH3me{pMQ2c z{|-xIPc*LR{_cXzw+W;@5hOC}BhJ^@qD^Ikid*U3J32)>LoOzZ#xJ04=nmKw+l+L` ztM5OQDd;_9sFw|MbenCqiD7)i%qPOP^_?Ov2i<&{sTitCk?ayDHCRbJoHhoc){o?J zpLC7CWNkVS8eNd6I_%Se9SHS4c)Oxk@W*&m24f=|LN+gX)OA9*?0)#rAyZr7{h>^Y z+*TDjTdfkBO9Ojp@D@$7c*wXR;CEZWJyW6a4z+%Lm(&K$Lf?Y2PhvCIm~-e z+_&ndaWDrkU;R#SBN{ub(W|h5EHop(*8E`mM>uORWw2Q=)Em3i`K~arkk)5*4rT zu102^qLOsgMT00)H_=FV55;j5E(k~ncmsCfG3kvcg+^%ovgAbi#>$h|z7>M8w+g=a zGjW}7Bu&B#wa9^8G;^8#Jfg~hSu!-J!mlRins4>%mdKjvo1KxSztLlccTH{Z7>|)m zybEe;$Kom}w5kz&IDc;aDcr|G-i8M}x!+mmLgQXW!8iBot33V1+EADxLCMyB4i6XFp#dHPrlU zY#t2Gy1#qBeMEgGfzZ((H&kU2SZV__*3(~YVZn79He;G0%A|U2LZ%FRF*Q^eLeuUc0yB7Qskrtj7^L6e*#g8AV;{RipU8@WT?cr~82n$cT$^?ph z>27n}vV}bp=zN`JL3%&TR6Du%mQ6j+`gzlF*TJtd(*Xq^Qj{vt`a?~KU-!{Gl}U*; z}D+-HFs(Y;c|0jn2c!#ajndH-OwIRN!WH=O%vX za+Uj+T_V@-om1&rspYrLf)4mXW>9xD_AWBEV{Tc}XqUH5RD&!{5YpQf!j2G(M{bsw zCN*a5f}2<_UyjGbouz|1_;ix^SN}$U9jQh1UWXbFaA~f8g@>N_z^9W-OgR4JbAfOj z#-#IY=3}=KR{!3-SEz~FtUpP!Q?Pv>GY};>)eA(OU-3-doD_V2)AJlOqI)K$$kSMg zhbo5oR~Wr(fRB+Wr&TzKcvssCLdjscJ`G+G2}Qrfo?D|yH3BNpLa`b1-I1$E-GM-_51-8s9iDz!3py%5Re$|-{%YR5~B+vWpIJ^m0 zmE@BP|JEo(k&=*vTMUS{N#C!37$+3CUqIsKxdu+$=|6y_JsCDtbQAzsa#alCl8J2F zK|((o_E2UTEcv_ga-C+DQ0mrokW@1(z5!nfs<4UjVxjS1X`+c^-RV|EVPaFPyokx3Ac-YhPcC(QuTp_ z9Z7d#H}+aEZZxJVYn!tE3*?LED*!1&V{2SFbYKk}mt+(A? ztSfP4w22OU#N(LGFGbSv06etw(?=;DfC|9sk@$^>!?7 z&t8&41(sjn#1n;V6TeM=W&!fvM;i{y0b5pRgwkVcqua?x0=Ry;PlzP^h2L)V+`P&m zNIz}*ZWjK`?HB@xDE!3MJ;jP|^a9$0ZA-}aLFEPl{mSL1^ym}*Qe5v^t7U&VxEJ%~ z8_6Gk=*S0!#i0MqOFX9z?gK>F|CxXq6NHj%JK~`${o(Ig0N9(>vvgJz4ls)@$ev$C zzRYam^WfOSxbFUuJrL5boI^?Tz1it`kC3fM86H^P&ndeivSr(wnqk2akUwro+sQR{%#u} zU#eJq=pFvlWxcohuv};f($aP>d)gZ705 zODb+(yH386?goi7?SA~hN?!PmJZ|qr&!1R}zj9b#J_Gwt)FSX-pN+QiEv)lA=6Rd_ z1{G`6+l+jB)vam;yrM1k0-%0gF2J~xP^#`~7O|B@pWc5bHC+0E*qu)Q>i@+OA;-70 z-d&Q8tk({cZi-rc2NPXyav!hB*sXIc%!CcAOp1Yq5Mh$nr);SCM|b!uRbrls#n&^x z+w`r7Pvh!{{GCqvm$NB-UJ!hee{)KGfiNDB|JlMBrZeGF1bCS)d6^g<`pV^CH^E-K zb$-HXAuRe;H0PG%@1(Bu?f44&+WPP4FD&ZLnyf=3M2^#}s7!h&YJUU7Ei=Ojz$e0R z;<3V~1KZyOr1=_v*Y^!fDpRoXixAeV`L2nixxZ;ib&wW2aPui~#SXl;%UWzm!=P+_ zDtb+S9^?FA3Prho7-`M=wE1atyJ$%v-<8=t=LzVN%2H)9`Lc9>5$(_TKrzg1qg^vn ztK1WLet_X;KaxQu~-8vTWvf6lAZruQJBK3>zd85p`CvxR4xBr@RLx^)h| z119(N9+v%*I=ejU`JdkDz|c;hM)Kj0f}as%;#vHov+X|5OZhPDVJk?GscAQ=7CL(UsFmf9DUtdMaDJxAk7;;J$71nzAAp)?g(W#w3z3cWENXy(dOK`1^m&g$$Ttqdt^Le|Y+#K^p>19H93?<&f=2I2y z`}4J#n)2~(5W6;*TfG+$ua3`V)PRaXLfIHlF^DWtiQvNgFA!;@MH~%0OY#G4*~D*! zRa;Igo{5i~eO61^<{zY#?mxp+^9N|K91JI)PEhaXc!x_4-`_r)r{4fw9?v$Q;=n*0 z+)9!E@~l^Q`PApfopm;Qz{6w#RUT=A6Ay`EJNc1GO}ZkX>9pHYP7ZaBeRJ|3#EF?0 z1oaF+L-}_`ej5&;XW;Lk`8&h;IUdnLiOKIuwAKV1rXmI=1h7_c{MWaLU3R}0x8i-W zR&L@m;|I1vv~2?|+?z;R1Las!^5K`I@*PKnF2Bwd%H8CKCFH3wAnj~V21f=G6x`26 z*M{m|lepay+|mu^sBF&VwV%DA-KUg6PcsOmt_3#kU%{5gS9rFVk%n@^LHhUASzQd0 zkUa5e*0>w{enIPH#f=stK|fmlLE;`FXe77J3wfbSed*s=K&mQ6YE6jt-6Be~z%?bl zfZf#YTle$!yGc3ceDyl4r!6LPH&bo*yJoD_i`u2&1evXN)#)X4ud9R3qnxWwjW$Jh z=omfFAEq>beZ)Km4tmu3z<1tM)tG~sHk<82*xNVTX^qe?r~-qE-$4!~3Ic?j2z4*r zKfys?AgD_K@?((zK*cqYs8jhIK#fGv^2K zP~$Fy9xn2Zo}^L4RQ3!sv8r4g1B;vC8T$X>iKCu9F%CiBe+v{m7jD=PYjgHQx~MQo zDe;gn;q|lifnVHuX(Ub6*p?hEVl`PBxh#H!r0>H9!{>64cdt{$z8I05gQ=2@t&`p96FW0ty6VrbYO51-pXE_K38H^j4Ot*Ijh*4SB?a*|CLCCUe5~S7c1m# zyfat}uYl7qnjxp_%~gE`{h^3&{492Zl{y!P_p925r z)f%QVmmdRt(xW{PFyQa}IKu4n$`dzvj<#W)nMfR;OPusY!(h<;>FDSQweYA#XJ#sKng6HDQBDfG_}_2k9=%BN{Kf@OKECFWC;(vTvl_%Ob)` zq);cPEQR&}1?(pv3j4Vv=Dcm=odtBqdOz?@Wi_%^C4rGymvE*q-H3 zBhyzdo6lmS|0U?70tPz}{I3sy6T(CNzh`Ta36Nn&?0S8Z#R< zhG5>34V%xCkZ{`NxMnZ^xsuX2JyOC)&pf$P0W{WcG~l6S%@>#eJP!xDe*_4gOC6wo z530c3hEciA-DM1)VWc+OF1Iamy|g1vsr>@69sec}`X7nVKc@w>fn&s9KhRpXT2mMZZVJw_m=kUlvtW$dLmbT!8sk4b}!i+h{&8iQ2>d`G{99X5@ zS*Z3lcsbyv8&(F>_zV;SSR2c8gjyvs2}tVTWkZ-XHtt_R098~15yObSdrdEhXY-{% znWDh@Ff$uvKt5W_7zez=^)Z9cUV5u(Qy?{P$UU90aHh1lwN)|0A|pxuzCbqW;g%`f3SWeKK17 zqwq^G#cdNMlv-XdjvlE2a!>p;Z#i}MONVXp_QbVz@i|SEO_F=e+m&EbZ1NGu7Vj&M z0#lQZ^Lg`ut1e?-2&QK?YoioP)Zuatv-yrJ5v zJ@-UhQ-?^m)sS||V;VHpZY1 zpqGJ!`MM*2>D7EjYz6PfxS6U6~?Ut0VRdF0#qa8CJG$znp%IP1>t zk?uFJm`hYPHJO)I$u>^f48Ojarxs6uASO=zb`rl!8`J9Ed(UY_V^FaA+lg0rVl-^~ zeTgAnSVScp1KVMHo8Y=kh{Vg0H}<@M)Hg1s!lQ)bOk%!P7@qK~WhlRcp&9iNoJG5G zCH&M~`$APIoN;fW6w&V77wc&$nCqujM?$lgsHdj~CpYNo=k&_-POJEAg~>`OzpX!z z2K8y{cV=E&&_`v6`4~rC)ASE}&+x5s^OKj?*W>RXSDS*XSn={_lC=}%7G4nYqjuw? z=#>{|5ChTmGa>MliP-R=?p9bER@gq)M?+%J)CB?|_Wxx5&$>3E=POyD1lvCwhN>Q$ z>gbzzg$Wq&>D4@b!^C&`bNRn-kjxMU!jbgG z`aC2i-A0WZsj`h(fa^;ETl~O#O2`M53Z`B_tyQ0?fxp$PJnnCm+y@vyM#T};w@uRc z_qrO^DiPQ6Wc+w?h1%g({C@(GvBbX9NRo|s4CPMO0cFBgUQ3yK$-is9#RT!$qp0zO z*c0)by^HdoC$T~1v%t>#s1ahs*dLFF2)Ht7*bj2gXJ{<7NMJs5y=id93XJDl05$PI zuKgg;91JB)Lw*c6YcqRm_iD{>k?H4yJ1}coIvH~c?M@2`rCTAr$jdqs^s0`3zP_f~ z^q`jioO~tm3-)!!CMzYR$osx$=zZpI4pglOC4~MnDK0P&PQ2bf`*>mLYfGI=iCy;r zkbtsOnMI?mUL<9@NHC$1qIY%0ZFd9&hx@r^JY7GqUR#{1xbM~zLI!x#SH2B+ ziUHaWe37!0@37=^=`W1Bqk5)0rvDW-))iE6ThBEi`aB$95pi1oKa{;yKvnPd1xkZb z0!lYXBOqOy?(Py05$TfLbSOxRfJkn-rBfQDySr=Cu{T|JanApD&be>*zQO~GwZ3o0 zm}8E?X}46XGNquT6nvq-Q#=CtJe4i8|K@w##svj@lyV*XA%_Aw`C{t%|DP}J7z=!H zvJ!TJVrK-=9-wV6q3X-*vUy)&2$Wj$UqRTdzN*6$o?4219p6)2n3nerLNKQ-FXKS{ z(Gm_p;6y@VvFS*Fssz*@-DyB%*U55V6uhS9vMFH^#yAw&5gscp>p0F33vOKe!Fjlt zgi6r}4?sr8=`KlY0iMpA8&W z!d(GJ!4O#MADaicF6h)D{Ltlj#hr^b+zd`*w9L0cdq9|MMea%97ipO@Jy$) zi=P17u%Lh1uY}^D<2XIB{Wudg?6MEmk?$1K4V3|qx?{-M-IyI0eH%H_3#85~7tj+u zn+3Ij680uu2V-@_&M#|`DhH;c2M-`hzlCJqR1;T9OiVigYRPM{=#SY!J41$Ps4*bV z*7>j_`18xP6(7{YVLMF6)=QeEhv~-h&wV2Gd-Ew~$1T22JNEY{XQ1b}qBr#KJce|D zp_sik!u@@nX=r+)yT@c8w^5BKk^ABD5L+2HCpeVuhR-`{-c7j+0M|XG?)Uad`s~EwBg?NAnWS=iPyaHLj%*!NYb81at~TTvyB~l* zUzv#lm$afw9!EmA*8yRN5wS+-N}l5BhOXKC>T6=A@k3{0{d?cmBp2SLErfIB&wFBQ z<)KS9O>GS|S8KwMl8fy2%kx_wNb$(xc2s|mI3v@49pLBFsDT9SA@a9=lK%Q26Cg`s z^zY)3S0F%Dcyws)3&}|Kc*XWi_tyl=jiwVNfOR{B%X~@}{*vHg#EOQunK%a$gDCk* zTr5O*WI)^iE8X(G76+#!Q%{#D04V*D&%>xFIkPo&R8Y^Nzzi_NaOu~4@j#&wDOUW~ z=kAH*I|2YfI!HTPM~>8ztCHJ?1-Y7$u%t`*xn6)tW@e2WA@AnNmBe$t>)q zn{^i_dkE`P1E2}sZXx<6(EF%~eVu>wH{^16tPxk8v&B9|SK@jog|}U=Lr)hG-kPTt z7)d%6ypxJ!g?^Ks+AMIt0X9P+ZG!+U+h4u<<7eQn!ZqSmDxsUA6PW-B?mtTo0o3oG zsz(A$LHz%yMg`_b2Q(?vejIpf0Q+W3j0m@{J>EW@Ysx>k+5muJ6yEvwJsC<72M9bO z{(Pa->wQldS3BHAaNC%vSbRzP{7hoq&+Pvw!-O#B@qkWFDE{A&cx1h`7(*1&Qr+Kg zv2MW~ZLbCGSVj$}5Bza_(1flv5_jB-gJtldQeR@l!D(o(RjOMov zdK5ZdH(|NDKhTP4Pi=s$Cd}M+j!Hl}N$icJvbXIgRpCZA%NR-z#zjGE+;dIP62DLy-h!0k+{R*WfBmM(+ikrHZG>f@d&x&{=!XFBf zwf`1N)ErMsbS4>W_Wvn{g930-dhI_d)v^Kqf4wXaMYWSD^iS0tf`RC-Xz+IDEmg*W z{6`faB><>-FsXnbc|%|C^UsAFk|fv;sZV6&urZI9a~@67D`sB|WjYd_-LmaoLM%-P zen+fG_}@nDWSRwgCDV6vY)=FR2e$Yg!TZ6&M+{W;cFv3xQxSLAuH)?WC2iXAH$7Yp z=}v6yNeKAjiN*iYG{Z&WDHozQs_jdb(~dxf#1&ozxu0 zoJ8nmY3^SR|CVqa&OgpZXP_^^!Oi&8k`Ja^U)3Le87#%FR-oMf8$Z$&L`BaRtYfj9 z<^@rE%P9g`yQXjcBQ~o>BjCcfG4ijg|KSuM_5)MLUqZXWMx}kFE9H1c^*gPc9?L!j z>X{;o-ZO_z6X!DpNXNb>Tv%6d7wCL0w;x@zkv!)j>64-_=i+5s_Q^zip*nC8?g(#Q zotz+FGCzd0)FGoU4e3h70M^sbh)l*AfAzs+x(A|SYLU}WLX3ZykwoVP{pk(--)_a> zsRP9(0fg=TDOUpNB?6jc{<%CT$k6B&@gB|oxRCSSv97Sc`uf8lN`TsK{R`|n=}JsG z#OYYOx+Fo|7EAuE-}xJU0kaUt<&xcv79 zMLA}m%kF&-VEh>qCYQ++;+`>~>Iz$c%7SW0*bCoZKVie4+~qau(spUSif|&b2ETr~ z@}$6*;J3C7JLP2v*Eo?SDH6KG1hP>|jzhlJH7A#0i4cr|_w+S41qn3JsRi-bj$bjqsIyqdlEro6?mL6!NY&}tR>^xUML%f$ z0*J{qndVpg$a_`uvDO91oSN^G=ysQpZ%QyOk%3JHLai|D_n>FiDvQ388{?Q6=C|A# z8l7J1^-gPFtQ+iaaei@If?8T!NFyK2Bu~gPpEER3Uwz(Y_i#^#9clyk$o^Zkpw|IM z&ls4W=5h*w2vGu0b$xlQa!EHL+_N=sm_6>}zE35gA9|WQL@nhliC;TFT??~RM(l@4 zrVDb#@Z^3p>m3sOh*Kc)9nCspjdMBgyK%ev*J#!rt8#o`u~jagYnTyl=k6@ZrJD)A z`KCYp#D4@tTG;C>T<8CY2=!FOolQ#kswaFImvFX$?@y2h4hZ`CTvGP%&+5f3n)D{Iwck)N@Wtp@ z0)VAQD$ifDh~F9S^Bq&={wAG9iDSN`k3tRuCOdIITEgUk?M0NLWeY`wC1G7nP2ej> z^CP|4`2MeQp+E?IbjUHoJWp^mrCUR1z{w`vNo;R2mlbVXyx^j@Q!IqpkyNmH(%OKFV$FU)UjHi+^4+DY6;*H!3PH_rS5dTy5tF!$~=iO{a zbRk|fsM3Fv9->5>Fj6!ke1M}71yIb#UGd>Y(k*#+Xq>Fn7ygR14N%{~Hubs2R=z#= zNYt#bXg$4oY;OW%t^%yEwwC)_^Wp>bKbA2OnSg>mAisY6{yIDX=kEjQo#U^W3-eNH#lLu1=(<#rAJDFX@Y$d1j|_&(QL zFRrc=cU#Ws5COk+EVgS8lE^Cuo`bonnYzJjH^qo2*E>@0 z*K?R@cGfM@Jr5*iYkh^Ws6d9l>tpnAW`r9z44m3gm*R(bD6doiir5#&P#-j8F)9=S zhP{ew|CnXm#>z^w*%OU&bIYHmuQH7rYe(<1U{f{C0BEj7Daga#2MjPl_Dg}PmZrYf zB*fgGMD8AXR^HLqyFMxvMGHf_38aFy63Jbpkc5Dj&W_&6V z5wH3b6^bN-Q#aNSx zeE!YtZI%>|@fwxqH}X-WQ=zbtL!^yz$m0|zBaM#LGLQQr{gvAd6&{cI$0*t+f^Xi@ z;4D4kDWz8c0fy!KP!nt=E$ijWk^9V5t)^Jh)(2*%v9CMr0+SkZ_txC<$%=-5$rIa0)QWg8S(M)$5r?;h z!(s=g0cPX|hbu)Ai>)06p~jr$XzFu;VeNsae4L?!md1t=Iq#u}ATx zI=QSNA;RB_!jAmR(XN(-v}{h}J~6eZH%o9(&Ags}HxwfN~@t{Rw|!W#61G2pr#$$VT5h z2|lhHZDR{C7QQQ__2GsK_YS^0HZlPyZmXH-$6dC!MpqjnBu@_(Bj$yfr+odYh3q_- zRBwYo5&2u}&ATPCBAwqsE+et!?!T?83_3*Cg~-J{<^xThWNaKS11mh)hi@%2EkGTx zAJF)PXIL&LP`b;`)P~2l5h!;?!0@S1#-I|pKa$cKjx2q5^rdj`_X`sn1g>7XfGCb0>*-+=) zJJZn%2KML=v^wJ0#x|^wMCB#C6L6=!A>7IK+6L=%TKJJXTS2F)CnIFPA@Y@)SRJLQ zPU1iiw%`-0bm(3-qjy`LGg zlzV_$t1|pey#!d@ZT&j7@sPKSJ*|L}SdshTu=vV>)7Csmd7;&@mh6TJ@W#r&(*yLlp5~>NkS0K8hvw>{uJDw^y7(Nf z71P#@odRg7Dg{3=Fex%IqxCjZrq*Ke_;-EJyF&Y^RLldT;q*7zb#^rVaI5m$vh~=N z-GdA9nad2C&i8AQj}K5ufoW_Kcvz)DqT@xy<2{l4QE!kLG+6JU*b}Q}GE-on3p$Wl z-+6~hxKfg<%oYk~{$;j;Bkd1=Pb}8{*r8=}I~Mww?bp*8;*Trreh?``(Nnn@bL8Cj zb(L7$5zzkgffk|oKky<`JPMPe{{3N4q^=`X3l#d?-S=w(vS>WdU6&-B2u+q;bz zl5)2NzSi}4f<40nqU4NSjNUj&J>xkgQIwG+VO$(axFgJ(Jz0|72jQB6>Y>iz+>skq zx2(r8dK|?*DMW#4&PV2$5;{5mW#&ReAYcVpok<2r>u>}~zKQp1{JFWWG@@4_{^`J> zxHkPFe)A6C_?BBw5dKo(qY_xvld%eqFxyMh)SD~myjBVJXiX{8!hIAXH2EHwcz%Fm zbV5p>vYYV9Pm2Hx#49eh?MVD$(u~`8<^T}F)8^aP_msE*7&drzD=h@b7$2)1(T8N~ zM~j`a(fYSlHU-X-tU5jeH1FarPNj$Q){~&g%s_WWG@`yw&&zLAut_J8pDW)*BT8(s z!I-}@uz9S2F_pW~OJ9=`rhFHw763M$)HCFJK9qUoPXlVWDv$S$O=|i<R!2tDNbHD1_6r!vCTVp@V~P{h97`AciyS3BPM z2N`dTa!$49nrwJK&C@f>v91yJS}wx zp2IaSVj49yHe4|x|I7ujY=2b1vv_o-jikL?9SXT?u1@1jCFA0c2_TP!HK zjO7XbZ1EbUF5tTJj(S}B9QKoat$o)M3F@&pMwGV^6nOGgTNk0j-eo|r&el>O9MAW8 z(4PLJ9(OjHYun7;qtLzD7KXbm|IfR|BPjZW|E<zM@TN|3s4KDgi zCSIn|ObU?xD3s|^xMxtet_3rw#YC&_xJ}jGhZ0hZzuP$`sd64XYb}ULr)$g@wDx2# zH3lSe7aW_PJ#@6eMaVH-a{xy7Mm|M&#!!`0MWTO7@#=9BA?Q9FF^t!i+pPej?xInG z{SS4D8yl$r8O1M|Q*e~s^cq`s8ypN}%cjO+f0&BBId4|=#SgZduCNN&rhh$3bUL%b zT{EkkLi6TtS4uVBhjSZIzRTua!&krZPC5Kitlx8w8(7u^YQ9z~Tygnnahmsk>4V=R z!#t_2DWA;&i<;aryUZFH%_a)*muoO)Q{MMmQ3q#HouTj;uH-ZdJB2}^} z4M=g>jsH~tah%|es5Id1f)P6;0uZU^GV$o@hidREv=2-I?q3M8Prxz;fcz1dD0~h` z7idI?2KLq3=m(zx=XW*{Fc>_(o^qm3u zaRubGg@XA+dBn|ir6guV^?t>N5=ye?neP}99mEI$zh>Yi-3Y2a+ZyOZsusi~VvqxR z*Q{V}t11d4;qOq7@kS4|j3^aWUb`1`6~3h{?*NrOCRz6Q)$a8ib~Q&tIGc1tyz5oA z^#mq68|_L-bbv^b+j&A_VMY2;1Q3Ap)26YkqLumfPlgCNbg@bsFBCiZ8yEe>)9vyE z(AZn(0I9c=q1rYX{3p*X*c`jj{p}~_a<7eZ-}n*LM6o}>0OLK~8#Z2_st16C%C#E* z$8GczT+8M}j;I~AwOZhZ&&eO3s>I7b(z~)75JG>RoBh2hVHx}B(X|wIz)l4C(3C{0 zDm{W<9#x!O?JxJhT+6oZR_!tlbVR$t_Qq;vZ6LF2g69zkR-jC} zu@83hsfFeh&vvCEs;s#81`S-Qud}V2K4WD7y1nc4xf#!u_kS%;NWWocSHZ5lQ25yN zK2GGvs}V)rd_EXJ#{FOrDi!E%H2yK3b8>1#roz-pF8RoB+JbxO3$uLQ)Th*HbthBE zkW74)Vg68c+}`(LKahew=*xX-v6&^tQm_>b66)*f3zz2?+-M%Icj*c}AG*SeZrvS9 zAD1}$Kcyc0a0umH>V#gV{@CQ3KzW?ys=h8yzvY`#a^FAR>HN9GIcaptc+W@VT06Im z$qOKkzsrxr^P8%xvt}7xSMrwZ59+hM5NCA11&@4Q=Az=HjxD+oDts~A{@hG#XWh-^Wf>Jk*tSZ80FJ;?r@pa6=(RkL&Ky>^TkxS|owt#1d{kWk!BmU@93`qYX`xC-)Zyd9us0 z9ITNK3GWKhd9r&I5d8ke1=??PtQr~$e-(rnRg97n!yll^H*x%RJN?gg4^R&niEW5R zI!)SeqxmUN-lx0KA3d#*9w#m9OS|A3-z&E}jH^M03inK8FEI>>H~6<$Q}E>SZ|zEm z>H9yCqFecq`@g-dass}eU+tGo14!^7{x)7+52EXG;PAEf54${XV$mw{sH+1aAxiWe zb`gz=L9*RC$yS2o1=In9;o@hq7%}1Hp8HO+&uP_Qp+Wj4GcK?C*1dndlIrflv~X)+ zYo7bk{mJj=CTy8`geuBf#wG)@`{1NJlyc=GPP@_5)^N%KxgpsR^jyjF9@g+;I4LDg zu^RS35|kAq8@c+Fc@?8AwsB6p&y6S=BYM&63t-~|>yC?ZLx4ZK)T4(O+Pp8NP%F3SbCy0x(Rxk!suK`RG+RxVk0W5l*& zFFi(&V!Ce=lM~|$CzCjXZ$OD%yRpse#RhEc^^fI*sioKPti1*lbE*i^UU zNW`$qy1t;L^Z_L13SaK{{0lWN*KGFrRS(g3+g7nzw+AMYu0Rd z3b!hWiM9UvKN`BpBiz7h7um2PFy`p2upyQYoY||f=?odHDd+F84rQCFFb@F zA^7?Bvz3f^OQ3_y#^YQgjwfR{0|>tC&7oL>5FRccN2?-)#1ceHyR_`x8BNPXfr7G& z#3JJrN83!NvfF%lGvxm!PU%gbH@FY+*!SpcV`ymkXnEM5zoBn%xU_)}gBWFB0IKHGVb;*;-1h$kjt)&w)1`pD*#wu11ii8Ydc?$pSu|ZF{{6qos4@S+HGKKQB)j zU7u`pI<)TB>&hKN`Th0&=0^AN-MEJyS%X6_UZ-2G0#cr@O?cJSeLTREVIiR*1_wyA zQNQ3pT59|7g7d#Wqkf=9sEVf(thAeM87CdaaNW$-wjv0(C?C5qL5KU#wtZvrrq0t4 zYhe-|)r|j;VWU6ktPc8a^DOEekqLI=4a(L%R{f%s*fRI3&`Bl}5rKi=9;t>_tgIP$ zXIQ1-gbL3To%aN@%hqpj61mv2J)o^OrnAGX)$HWtMq=W&KQzmRJ}M5Gf>>%Z;3W@t zMtom69w&|~8K#Q`@R)x+P)^W3)64xFn%6UwqJWYHM&sXHV1?QU8>)f*i`cD8&nSyH zRDCPY%rqRdlGiBZ!CRu*R3t9wHIpit;US=R*C&O0QAO-5RKhyG&b(TWXvTx-Sk zAz3W(%X$wC6ZGUm_Rk1a8N2U>EKJN)~e9P^7W8gu`?SLPVdsQ^sAF4ni$2%G>p#i4yd0fET zzrdhZ_%TTk_<$>y0hy#Z&77vk{AGhkQh?7hk@yqnUQvUT=P;lKZ$B_h&m?FyT4pyr5XIXVj;>roQiVZ5(vIERi3{!7* zH{#XAwa?`aE6*I%t>aF0?lr+~B62D<6Q>CN#_z0YN8Qx1c>jCUR%L15lSBFO?O|;H zI*k8*VYK21(`0g=KhBl-=JFn3xY|7%x(p`0k=$}g_=Od4^FoAh%R!tJRLzmVW{`@8 z{SfBmw<0#psCenC_;sCEj5rN+NaUR1-+=>nKG3eERr=t;#JXG?$nkPGd4wVXM*ut& z^R*~dBeKWUJ;Txr-HYEnBgVqO(Ptd5JiR1D7nc(&#(p(gw4*0lpl;8Zb+IWrw2i(g zp$u)tiLSe*DUn4VZ@o-*TvRv;**LQ)c2Z!}k$!&L0Ju=%t{AH_8}iX?DK%x)ArNF^ zZsdZ;+u33F?$OOjQ@C!nfK8a=@gSL`58T% zuXGbuNfk&M&ry-0NcVj{WxW>ky4tGGRJ-6kqRn^?Wtxa{_zgBOZOHFR3neXpmBZwc z7g_B!bvBe!)b-}qjSjr#$sxpzIs)D^YlV{|}t_--p zW(c|vjxPJ&HlJ5SdC7DaDklkBEL^=}ogM`YN&cQjNVXBjUR^r*<5CDo5`dK{x40C# z&-0YiRrmQF@}dtH)c1G_t-B)|;(4s`|D6An^kLZGz>=cYF9cFPEv zt4}U9SvzFBhEN1PU{i=W2OYH(SC3YhGmgd5$)W7jOGldT_Z8uOHa%Wzq1?QN?Z31E z9BUn4K~fpil2pm+%kK19vgZFL?f{ ztcUJ}vl3*|Ct`2C{}=5j%^OAUII4}It%&dZKA1_i{}WQMkW5e_k(V&PHvDg*^P@%B zkNGFjxue`TvnpYKwF%S&q4?hZ*8A)H`w);s38?%u)S)_;10QaLj5@|WX3X;zk1{>Y)C_){&H$h+Edl9N5l?H!PX2RPAZZVZZW# z%WFNIJg_ow-V%SyZ8?;kXzCkk2)e*%hJUcUF~70%x{tanJe>GlU>=&^Y`^+}z!7hy zWkbpJY*S$z18&^tuGQiLZr-9c-_I8c*CP6X>}wiYAdAB+69!!Jv0gl6PiF}UN8x&_ ziCWCxhBYB+t9`rs9Aa6|apPf)xIM5?DqjV?>(s8g70)nI*asvPKl3va{+oNi3jKyk zmtHV+BNy!40eBGGbqLRj@w9V-C#kmk}2`?}AY zn*|ZVD6+0my>zjyBUYsq_hSZCH{aD~-OW<@iCq#826Z0LOj4cQlQp`^HpFge`c#FL z^@`eR-yX4eHiOF1`qMk+?_yL@WZcqw>_#m=`p=Mp+Qp(XB*Y%1?{9sS05$al@2jem zG&mTxjn|^uyj?jfl~>E#CNx^1Wv{W%OchiVJ1LzJiU1X|BtyS(L!0Gs1afu#H}Q;? zow}b9kdKMwW!1=g;xpW%yep8RPuUIq-UWM6W_xiz>5^}0&+ApJyZgM)&SRlM{jURZ zpjLVbxR4@q!zs)y%`%o0SeVO_qIG1Z6s3M!%CG`2&{7FZm6=?y-al- zLjyRni%o4gO&~t3<)Q*?eV$YdSNShJp8tQ3Fn8oXN4P>J!?xi)Ts8V7)!X{I#T(nX z=Z051_%`~jIIlJ_9Owl`^GANEz&<^y_G#s0qzQXOWH$m*K;&yiy8YBbH^9<X*mclmQOshi&Wqm>8oZ*2aj*Es7;RHEBNHwO`vCi!Q;3{SenWoLMT zPK8X}CYl~1_NeA{)Wx6cC*ZH>lJk)9AlkM2C?CP6UHnPMW~`(_dcJkYv(_&QRdl9C zG_wriKNmdi^zu5Ttd$9q(8Tv%bXZIk`R4q)9J^7iW;e>%fe^!SB{D?>=_FoskpBCC zE@V2nD8_{Ix|Mtd_$BO)O*(l0`iL~p@9yduY45`3&Zf+Z7+4T;j-_yu_`uJyGp^as zdBiA!KMdJI3Cqyh8U^s1{KZYV{LiulB~2@Y+Yd+x42F5&`;J=H{nbg8ZJEFfW*o3Z zDj+e3?=k{^`(C1u`y+J3HJ<2=$i22w82RyxKL(9r(-gtbZo`}K1&{cj&fXj6L@0GL zp0Oyj_dQXRwDzmGgko640`cR{aPY0WTAvRh17P%(^(lsDvvxwsKwz}OliD?iYy8qB z-^WuMy9VKHSeQ62HVnJo94NaGKEg1bV&rQSpB3SB1_}c{ThransX)K+E(}+Dqrw){ zw5IJwwW%>Xu`v(C>@5)#!gR)nW&AHQK4L{v^-Vy(67-o*Gatvka5AlKEOE)Z4~JjZ zq2sD}BfD8=IWN3UMyveIGoG?{Es}brCT8*X$LJ@K+06(6te-ob!{m=APp$vJq8Fg7 z?e2{p`WBuw6GBb>j7&B(zav;TUIyX_YKT&!4q-7_y!}tR@>C%*A$13q$`jlCQ(Xgf zmxv6)G}S*PV-p^f@fyojba76pA>IOV%3qy9GTfuP0658zJG%M)EE1hm&+1hA$tK$w zdJ(t1CjOY1T^Z_?dSE~Ao|jNOt;lnN1>z?+GN(hL(AWm93xn!73HQ!uw7x4eY}mM-am+togx z3(OS&m2V`2sVP?IJBd_^(Nw$(2Q4(sasd#vGS?Ma}l7q+b}qrQ!-{dD!*_s|xt1fAC55ZT>Co zOv*QDL*%`C;!g@nPP->!@;D;b&cBfQApd*#-sfSs-_*MpAbxT7UlYi>4g->?&)_1PH96EF%X
3aAO+r}dvf?v+5o{HUON-$cfe!`=3y2K<(dhLov|@g1>;)KK8uls%?L`E4#qRb z9Z*(Bdn`U<1m|23>VWXWFFiIwa0gznt$t0o1#_A7zht(4SX{B7p&~rEANBqY<#3{p z=-Am2*CkK*jK{pa$ty6`tFo%imK6f8fflPK8mh z{GUF6EDY!aR^+RPs{ZF1eM3OhHzn_jwWzvCm>$J|PcTNK&w8lOn|EU>F4TByX1P?j zfrDjZlT0-Vx>3THgU9De_-}H`UYX_(0)ti>|{)TfqdPJjGULv}GeBdd; zh3FWk)-6-2HhZ045{WC~H=oX9;?q1}%f)-_&FZE%IIU@V<=L~|?Cqz+*D=#RE}YA% zq<^<^l=g*ye3R>t9%m+kWL`8x*)L8p&HA^g?=5+JU;GO`a6HxBbGi~!FXW>U5?%bl zy;_kVG@Kv@=ck2=ZCJSwljG6&8TG9<*tdmlAe7U~QT2Fbw))7kgC#I5AMt3mjpN76@>A$}K8 z76@CnOW<~it$y5I{SU(T-w70i_mlKFL}mgFD<>#5?y7K-(JX}2-Pn7Z~VsHS~id2%T4qozdV>zItM%8+*f$WFY1<= zFk%YwAl;wp9yVF#o3BYcTW;~~_zUROsB^O(!adUXuw&0FjbGGfUtA7J(W8ph~cn8qRGA(YKJ+-phPcXol!-e(48#B{g z`!ZzF5DZs8_G{k4yYmd))=9*o=EyAtpew(;P;25Wqu*Bz_S{J|5i4IVNLT0B;>#86 zmfQm>EgK>PioLEMnz@P!m^&&7nrVNUBqIo6k|0%p6-v1D@oL0;ipN!9-)yFKV- z+4aLs_#X3`MiNP6wj&Cd^PVfGrQiviL0M5t5PHr2OIYS}z~TDqflbocpPQ&TbXWW+ z(N|&qeX>GFH`$^%f9l5tMmJV4QYile%dEBr05{-~NOrE*>Yt<$h!TZRqVx|W<4HFv zxTlY85*Q*=nja!hWtgjpdXsm{IR28h%yKAM?GWOtN!t?}hT%&5%(7hSrS)ckGu*h$ zk9^D<7a}(lJD<9V_(SH5-uQ!OG+`Zi`d^$CWqb>q?D@H>or1mKS6kQL_TK1I zi4Q@2+9v(6QFhd{du&cJJ*$lA?j(@xNgN8ribAE}QDl-Fc2Pb|-d# z|HzroJ+ZWG;yQu|YU^I!p&!9?){}14bD_g~H?MtKiyb*r@Dd_ta&a_{Zx+KofY*i7 z^N`+rTVQf8x0{gIIGWw$COW+9u(I|#I-a8qZI)%-sk`+151Q#-8rtD${1FBX6Ln2PpPyRQnVaN%g&UN z8_!SY9)0`;idgo9(be{tWcN^Y310j}ZZ_%cflejAo%=jjD8V9gB^Q;`9Y{$NprOl+ z17||ynZiaw3ncqjAF*?|oj&{efb}qACh!d&GcsDy+(K!(x60@8HGRKko%9v^7$gE5Uwxr4tcHm$>_HA02 z`Mn|y69^~2VZw(8DmHPyNUaVIjr=YGjUA#|>q<}cf*%aX^@=Fz&xZ1(cn}&ke8x7@ zOTz(;NVf{PT5TL6NB1&oZL5Mbw~3=jgf*g}W9&OXb4&NfRP|`&N7sNSdS0A%6451v zn0rtRgrr-|Y}M80%jch1kZo2+4JbRVh!fSmEUFc3#YpgnH{Rh=ovG_)o(h=gwd!bH z`2gO$1ds2U(er+!UX2nCb40eRyJKb}Kvz(R_J-PZ>SJHr*k+%|atWsJc@GiTBmb`g zJ^*WJL66xNHc4jZSR9uZvxjwcfQnBj{ZrX19e_~6{7=BLLb+j%7RUYhKL7N0t_V|Bha3)7n~}}V zk0Lz*O;RIfn;>M0g|Ge#&15=7@BmX1(H`{?@t(Qezly?+d%nQ3ZQ^Y94uc zDx|$6&}9tQTIyG#*4u}C zp9eS6WNw!q((tTzGdm{vTdaD028Z*HL?&Jl6vdcb-vvF@Ik<9Y*YC!22jZi*WFLh+ ze08uIY+gzm^|p2!K|1Hc;Lder&xY@u2qJsv{Ps9-oCWT7o*{{bU-Q0j` zQy{8TO_zU&yDD-;hW$p8c_ZR`NDjo{lu%J*(L4;-)%n)rx}XJ(ZuVeY2W-;1oJbOG z&pIc^@ez2x(TGJud3OVZPrtAA0}$`X;~!5pIA-Te_2vWU zR)^}u)(L$OR@Huze&jU;N7$6=sM3yNCS$mgd`mg$0lqsDbUXGb1AoD|vT;afApdt? z)`SBmrX8)ZpwH9i_+$k?OUh(%B zo3J@k;q?a7QDMH)@t*{YWvEtBGrt=XoOsu_ zXVZGGNupYh2Bd$+`mx-Dq)M*x1&Cf&oNo#jeJy0v^hD9!)XJq=%<~hsuBhRnfX)U_0SuEw(JFaJ=S{O(6T|Hc>L~^_ zD;(&86d`ux>Ng+qXDB^>%9YcOR|4#Y0_+G2x@ztjyIQ{P%A{fn!^w|F3nNVS^VRbz z2GXZDDOBUB4VWJ*Hp$9bU--N^#gW#}a5j(Ii@4F=x-WcF%7X_DkIMu;)IS$=Q69h; z?`5;>)1L-a|2Jb3N5McL1>K7P!RBe4f^L8Z4TVlBxMytCmKn=pTDP2ms$a+zH+ACd zYaUpMfcIXW?_Oq0&jV$@tfrmu^+RKs7HYJEM123pXi$Hbs+M$?Es9X`7Xk`Ap}J+m zU)gRW-;i%7M!b7@$)TJ-x(Nf*-`JDxwP`To>J~EX80nR~3$6QjEPuW~m6yJ98HYjo z4Mry%uCxShw2Lv>Z4ccVJfUeC;?Ig)rT?v20}{LBNmQ05;kL-|m?e&6G8Y)ZOS@Bv z0ezmPHM*OX=Qp8Ec`wa2FvdUo_UwfmEls#_R_ts=_-Di`mQtMO??!hd$tnC3_Dc<_ z(wEJp-eP6T*R7ZYnLy_Q9U6bv6O&ieT=&>5-iBf~$iHQ<8~g$zGJ;<2^3PHd^pHSp zsuRNR7%}DcJ70KR8t2YYCq-tv{Os2~{IgU*>fyWJMIKm6(C-x_^vx}+{ly({5$1gl z2Ye52N{DUo0;79#rOD4@4wvzkzHko!;Jk8nkMgSh4c@L>6b1Ac2y%Pk=!-F8pB&dr zVz5gBsDw#Z4?f!k;!4%8c4Lx1+*=a8VvApK&u~HclPc)wmQ-~1PEworCj4+)F6#QL ziC;z4q6~`Aw)7)trJp=Vq6t@c(Pe8;VWjmc#EyOC8(j@2S}+>!Wi>I023=v8cnnsn zl`RRnS2afR4RB1_qNEVQzy5>OnE;5usEq>fcav{(WOu!j8uw#cQkC7~h=2)}OyK zzWTqxyDy~{mWs_Fd*CwX8-m;mzus8_919m!TFd{9+YK4(_7^zJR`u+Z2>;BkA&a|A`hvg` zd00LQp>xLN1KL!9X2Ku%YVa#hu2r%Q^OCRtyyYwi#IV zCee}Ya}VX1G+vA-HT#t~iqD92-7Hy6%aspkB2PPK(s@!qUnoy;sH-F&uT@yR#8j+c zav68jvSQg7KRcBx03b8xj~6&WB9If>(D|lW#0Vi_cfX_~7On8|?*tKOR1Tw(aEcRo zTA~<_;G3F0;bj`Qs#`|u*?eQwLRGK=kX#n(JkYQZ7f$JaUPZbowjoU*cBz0YhKp> zCH1G@@uMM{!XP_g{~jAUk_J-tkwAgf&7UEK0-7Xfa{duZyj;+PP41l7R?A}(A-5YW z<84@*e;SWwWRmagqrC5&^`7l7l_8uidyv_?kJv+VlwdGT_X?W=Le)T09pO7N+Ye#4Uc6X;r#osiO~mXKL< z)<&#%C7axg&tebDS;Ra=i{365{Il+T7hzhf3OO_QP^gJ|!SngXbzE|to=k%kJ|UX; zrvrxfBYO5@vh&QIMcTyP(fTk%X=jyW9um0_8=bBr_t#pR>iqSg^Udhv;WWJYj zn{(Wjg?m!iyR>p%iA)BOA-IQ(L!E&S$E8a=u1=0?3)P2%jRz&UANzhNN%KX6BwVlM zsQhJIEFW2hmjEE}g?mU~XPsNIoCOyHd0EoiPw`7(z|k309SvzBhQHL=qVsnm$EKYs z)-B_}eCAKOE`lto9gj;Ea=m_0Dw38FM>26U(l^=TlPM#`Q=sDIu zoUsy*IYAmxrC10@ZZ^P-7T?*76c3H#nyTfDKv(-GlJiH z;!d!2fVPwB1`BRCGraKEAI7Y+X@As=<64@(K^Jv(l8k$w>lsG7?A%_vGrCbxterN` zE3JlC@r}4<<{OHs8ViZh{UISA19Tn;Ib=Jp7Qs0l z6)B^~6JR@E+KwKTtk)5FMtIP&Ryl6VO5`-*ZL&ZdgPoxVm5+x@mM^f0d6eh(PgVF$ zw9Wd$(-JdW(1nqg_>m|u87dkHH23)^te5^o#fuv0Cf_Ys8K_^6vpQ-`vl)C`mQ-aM zZbL#PzlLw^Ouy@h*41Ys_tVU&2{~&o+=~%I20jLO~HtHXRs}u}ZihvZUDCGO? zRfqH^1Y9}@!9@F?t^f$2G<2g4su;fc_6xd`g;GY#6&Czv5%$o8*&}a0IU#czT5_n9 z^u^USH~8I?h0q9(*I`!T%*fB;Q)_2aOSzV2` ze2yQX2HoWDT>hk+s$+j=i{j<6EaXEh9GA)OzD41jZg);BeEgW(?Gumvon=U4AQ((U zmm{kJ$*?*2c(__*0e3T)?l4KV~H zgrB{;ONuFy?1+}Sy`+lw>6$83v+*m@Hrx~w0`<#@tDtzHWF)ctPa#D*`Xt~zC z|AXT02P2el{PUru2?C?1kq%w%kj)-7v^Y+SjwGy}`b?0w+J#%GNap2NJRxe(|F; z6BMEIxe8lfqh5c-Ux#6G2G^sAKcqObOu*GLfTFh>30f7 zlFbDF2)p_y)D}2SW|UUD9W4sp#MViNQo`nojSEQ}jbo}7)>dWXFQjUcM^6bDJ@;Wn zaBF_sK&Fo=qLzV?9>t7?^5~K>88HNcX9LbEWv!<)TLX|#+0XF;t`wrkzKd49$q+z! zO3JakWOpLh(x4UIACBRApvdVlU4~zXV=3Y;AbA!g4&n3tDFCk^HmByuBIa}=hRi*b z3v6gy#BQtgPs8`b%i1ex0GObd|1$-SYpM#BLw{=6j$Yw{b>Fx}8_!3 zknV=x&H0}5Jm>l2UF%(I{$dTxuz=l|tn71-d9 z=~F#hh_>BeTYfhrVL=s!j=3*M7*0o8%(pT!nL&!;g z49}x6dY+3cK>lj8Ge5$0IBjk-LOHDo0GOGh1~tV?)XS$^vhL)=rI(hW=mw_~2aNuEHuLCLpO-j?-FSBB*^%#()4 zf=J7cMqh%Mn}?&r?*Y0YSct-x&!FldR8<>6L6>Wq#Z{^oYzO8U7k2q-I+7th703xgIYUb%ZoeWyqD8J7JN&gTP3v;kybs81<8)qk^!04&+QeE?8`?y%HwZJCnFgG zw2DK&jwrcDIphCVqQvHJoBNZN5^tEEl{G&oWemuagIW`Ka-z zwYM0Lp}+mgjML6KFPCE$eJVBu2%i}ZrHF-LY%oo<9a^utZIeUBqB9CT7t(sSI|Lus zQkAfp&@m{dqkH)Zj)p;jVL~&K4k``?K|o`s=}{>9)P1k=cbNUnRW8J9>UAGjKZ!}nok1zx=on|!lP?p7u z0!LP_N|fa@2AXzd)|bZz#46w18CN|?yhuuOF%1%A^41WPB2Y5@?(vF&P*ElLo zzBaccrR;(fN@6ligniH}O#?bgSCV?T_JD6(Uq4@?7NN!Z9?dXARV1jmRVCU?4Nlm@ z&Bbhnmi+vBih)uQ`a zSEw1Q1lD$cvIuqe8-#lr7uFN>DtHM|!5-+hbuv4~_B6gj`J25{3Wp6@We{_ZAVPY2 zMz_tmO_NTO4O^6uV4qe?!K#k${AfTkpPU?6yqn&6js@!XXQo89#ZAVQ`&WRN4c--vkusUnNK}v zQF(!D?o3tPD3`-IT8TOC%$D(J-I5> zTX)^4z%Jg8hC#^qS}N^phEJJ*YKf*)1&FI65N;rbqM)~B$2mFcJUQ&)nN(m`&^ z9`rVxO>wsoHg_x!ue11>DojirYPgbQQh zM!1I)WNMa2+VW`$wp1*qL!7 z;ZN;X)$Xx;04tDkKi$mU1gd^vBBP!;X<3cqUhc;hW38ISZ#U<=Gx>`>*0jOf+lIHz9422oyphuY7@ zF-!(1=Sd24&k#9!R?D!?{5*{*Mef~x!ojjB$X&ppYhs%cTUDD3n`Gq`3_glYU)1FY=$?C&8XNM(${ROcg9NNkOxq4t|QirYsTIrTY^*)VkHe=aEDPzWtL?i(^wZT?hsFPWMnV`t#c2s#9g4D zUe4@8lD;IXR}S9py`9ZkOMT+m-%b<%)fXJ_Tk>^|EZhK)GiJ({2YIE=4o%siF_`0)?rHik zXPShTrihoCT#qf?wl)kIVM!;IiZWX{2JgCW=jca%e%Nc=Yz7$7Ter&HS{QNKST`3p z=c%YYw&WneVj=FM87tbB+cKq-kfBtKt?CEUYTkjMP9U_UL-LuJwepZBq}co40f^X7 zt=gWUoLgC2+rO8Qzye=p6Cx8NAAq7%K~RJdJJaUWs477G#hqb z;ckKtS-6y*xwyPuzkHZ#<5If(NZc%2rv-^l%nZ48)jZJVDe}7ZG^aCK+KCzrvFbeZ zw(o#QLKOeU2py#v9q^ zwO7xa6W%-p%(<@$sG^QObpWx=lPBAkoxXjD3LFfHiR2Wfl}chJAK6!3n+&TH5Z(QC z9MaHY*xNS7NqY`h2Ym85J(T1|r%57y&&sfRAHh~sc{;)p^>ylrk9d|b<43W9adx&q z81~5Bb z#30-F@F9P`$3M@>v$H`(_oJ{(ST;*wEk;;&x*cYL-;jVIF=)-A^xApx9goK;<^M zw)XIEgAgpwz9Y0<;R;pG{CY>guq3~G#noYx0Yz&glAs6%1{of!otTUKf?uTcAW}Lp z`n}_ab{RT!KpDj@xI0FmFH`Svj`rBw-oqrk^+YUvn*qIAin_o!t1%xB`{br>9SGk0 z=>L-loj5B=yBX#$kV2#?q~|n3k*;aARWY>2wk~eb-elZ9jFg&AdSQ0gI+FRmyf0#r zD@V3S=|`Ln;g@io=w13xw{sy9Nh_CBdps2#htZw5xY9Of-BFOwLhY5j}Mp<^e-(BV*Gvl&*VDa5L}LLzf* z(P5!#h;Wsq=Pn2noQYod85a5Tn(M(pH)Jbu5c?RH-R=FlBpH_#kH#R17M9Uc6PJf4 zJE+Y%!$aHZNP{0H{;N`Cd$#;FYmNye=i| z#(v$$*z@Nd=}+CCXnY1?p@TLr1&(sh0pK@ws##%jLGsBfhTHsAHR*6cu~*PbHB9*p zdzVgo|L#`+c!xZ27tWov5Oe)RUbwJ+992o2!`xO*=*Lz_9(N5Jgu340ITHaG@Q_wW zi)`-ICr+=*rdVB%Y&@R>mx)PpwywFC)~O%BoegZj{h&@pdzF1RS8y0Q7*5C(S?8(r zc)iWr7tP!kk>pmh5ec5el{HeNeSTz}ndGyEr19v0Tn1(ZkS+mx*Ekz`30dQ<}CJQtQ-g9uT z{p4N*ipkttkGbH`PI#v%N@i2NJ!ooJB0)PArAZ!kx(xg|;WLz%NsE#n$kG96K^rAu z^(yDqNOeyB2*B`6^urM!{LFaTwih2V^a19YY41dW+986S+Ov)!k^Ac~+%d-Ql3dNw zz^s7y5xF@AH=GkdXJ3qK{Os<`)_4t1^R{P>sQ~FLALy!vu=F1UQ2*rV@0Ht~DWU5b z0qkl{x93<#wQ48r4%ikO+>`UQg^pduf88M>4bzeVPD{L@rn;65lGLMZtBz*0pRUqr zE(i;KDZ(;(1Zqtrm@wvM!Iu^2=%K4Z9d$+P;OoeA7CBlU0DK}@_P!-_w49JHZ{$XzFoELJoTrY;DGkC4v`sAAqn9|R4>f3cuA1;43;fnZv#osxK zm^^djGGvnnxU~4@k9=ofl~{ALzsR1W!R-DW(kIVF`2E+>{h2b8Ipc^mgphN^$~>(o z!oJnvjR%Sc-n3NJ(cDbi1LqJyfbPScdyx)wnwl*Q)h5p-vsvVH`~6yF29qhR#PnNT zDm8yYTE6!-?UxrI%7Ba~(2-Xcr4dMEr*9+QNkmzlq1vWL9g=dRa|o85K|%$ql^MC- z-+h*lnB~Tnpa&OFJO4RBW77P(Jf1=Hst^_^>^5|noxqT={j;ySUv2#b+?q0zS@32N zDBAI8&MjNi@nISe0e@ec?p?rb(8yNN#ErXf>GKez2d&F7{2VtMIK{*_sdoAw)(%qdjU$VLeA+=C+^j9ieAOWth%$e{OAO+_*=YaNQ^y} zacK|n*{nI03JJ)7(fRShv(*}Y{9;FKEl=#MVN;{_gus=lpjnKAOCgid=TRU&8qvGr z{0^iWfn$xtOGwpxA+1ir<`TzU^YJ%9(j8=$B2o+$kUSBr`-j;b-Z1RQfLh=nRcHBJ84zP%p*nzLq9)L#7v6Rjw%3z)k5 zlGS3Ph!cwr*$d3X*;je^D2xUuFSx+B`naPKl+EUICUavjagXcnVJn7ZI#xr8cr!dJ zb&Z%vLzz0;N80(VcO38IeX-y@*;^eKB)P>6x61?=tNAWqVFU*So`(%G;F5;Q7|QUH z)~h}5*lYQuy|EEz>-RS=_Tp`3b1Ur{0UYhO?a5DQ4tAOt4s=G(`%|uXBoS#U>H5MC zN51PJu0U5rhfa#fz(j)21p(v+1Jf`fdSQvZ4wuMS#%is)IRKofVs?(m|CD|ZCO~pZ zQkSNuVIl~%`ih8@iXP?VH+V|K7*;{>XfYf4937}ok`Ma*?PqgW{|XiTe^i=z$$OBv z(k6a?qg^MEc;>#j4<4x4d#@!*mUHx_B2bCEK921`*1(tp-$7=pouJF!8kh1yYqroA zRhSF(-3>&@@x}AQjL{Y)#)q6Qn_8br%sLauhiOZz<_8!`7vqS7k|zp=kFbYn%3k6h z(L9&#@k6xvohF)t!~&5I(zqsP25VxKoXSQ#Ifxm62h`T5JMg~6#h66DQ$|hhqW`Qim@F0>7v6}Y3Ue*V* zcCKuEd#V@!VJ4tXIR=CJN`A|yM}d#gGGZAduo}C>Sw+wEZL~~iI54`?^JP-iKqGG0 zD{Skwc1j}OfJ^QHj)4W~weh!xK-8=w!4Z02 zkm9u{+4Gbz)=FImU{p#XG5VKhEbjxOVj*7^hJ|_!Mq8TEe;@n8l=!(0o(EY(V3WPD z*>iZ-g6C*T`JT$L6X#jzN__7hFr%6S%}!#i`}?#U$}rPQPU0d&^0xOCod`Fu#6dSf z%2}U~sI_JTq8B+EcyHuhukVSUz0A3BIKHmGb!X0S=~zLMv*>mq;(N@lr_l+zS0#=wdspMg*3Z6Rx6{hHsy+}a3K?F8Q84r zP+0dTv+=FE@|=;m8p%mrpKQe*eg-yc`!iV2%k4hny2@V%Z-37{_>)^P>sGk_tOn4i zu<{wd1AfGG^t_e8E*=d?XA^x&2sz_f_nLM@u&qDMs>o8S0O#EKevxwmO1JGDz-?m* z>fL#9D2Mvo`%tTy8Ey*l^Tb6;CnlusiKlpTJI6Nb{#C02pQOonGtiXL#<395A*BR$}#rtp$0p%?=U7TE8(e3bYviHWf?$4=H$9zoi zOJ`LdPNQ}-S}#TTEbpuP{k@(|-CQ~@bQI{yt9h-!k4Ld~S1M0&QJQgARZefMJXU$Uf3f#^Ihiu7ZRej~1No|ZaZ6EcU zIlhrkG0!xslN0^kZ5cefw{X2Aj!&^wXIb~TPC_KwZh$j<9r25x%M68%0$6#q2^win z2(|iHA#u9U)6^dM%;xizCm$0qLi6iTl+U_EYFUysp`Z5gJjGCx%3k2m@EcOES9L?b zhBp!gcwfz#EY?Vv4#sqFK&=cOB7?-l4(4-DL!&2Dp^Ran=o9`{~8>1D%C`;`{M& zT|bdUR%B#xw%P#Q;g9*v569835p*d0hEejelHq@R3X1gQL0s(lrTx@tKPHE7*7&Ez z>5F*J{eD~!cMZd-B<&B2=D=g6|30rJ0~rm3R6REvE}Ik$R!r>Q7c!ZAqJc zLDsv;@B>Um6{v^S*S?sQ44%R;{GVke0#x^xVx8JgP}AQ~8V)wD@8_Y|zxZ?Jn5IaL zTg&f?P3XKcPF9sU-WAWuTniPyVIP0vg82SYn)g?IXovdsha$9Rvtx4ppLk!WZBGfo z;Dfr+x;`IArh9xb4$qod>5FzGK}~NaAvF9E=Gxq|OsUt+@v=52VvW#`83!rdQ<#l_ zMI?X&$^!^CptHTVjpu&&N z*L=9c_`0AQ!7nJbXZ%+)n=jDjtpZ|pn>&udy9^5yRnW(NHz=|edcmWMs0YG~8EDd`k)ZQ6-0&=em>VK~eN$gS{wm?i*Xjf`kRVfutv z`@!kH{s+i>LaeDr6-Aa?IaOQ4wG@|_S1)zd?iIC^K<}ndPQGonp1dk_5#}p?Iu|J} zpU(+~Anv5wvqa3rpw4)E_M5yM^x{4X;E^hlV_Qo+(J9vslY71o-*G4Bdu89~TdACj z$A@2FI3-Nyz#^soXO(_Q0ISs5+aLcgM~szx!2KfpzpM1502Rbn6^RCv_zuNEf*d9E zwva+(VY=)fRSq6H%6NzOKZp~x4j3rks=k6JotN^J!gQN=-lPHGXAxG@a*?!fc4r$rz--CWcggPzS!Hke3A9 zA?81LsTVmcWt(#fl2=#ab)|AHruse}J5Tdhk$f?T=JNhb@*u(KYvpzHpJ-lY&Z*c+ z2bE|QS+VP*LKHPMvY#iw!8#Y6Vo1PFwJ2bz9ot8W#b2&h#PiFbLY{H@uT8}x1ob7# z&Ex(kNymTA(9xs6P}Zt4#wXC)9KdC7*%v^(-*{d}d2(O>5~miIeb+L};Y5>}LWx7! zd?p@JL}D;?SM2pY{@97cY-+VMNQk!l+CHCN6yeu$3OB6cEvdU0|J&jqP2f@a1X$T2 zTDLoG+g!={bn5{Q5^=?-GtYhS{L0{#$t(bYB8uy&3ed-u(X zD8T%$-AM7P*Uj16K*u4m%jc_xTgH|ii*s$G1_Y>1iW=hi<^?A!b_3kltTh)j5{Dr z_+^?!#A=!aoNJJXzy?%c%+Rctg(P&}AtBcoA8m8MBptWvng@?*Gsk!Rd)s&Drm2++B9q6qFuPjij^z`Hu3OPg=;5(Cyp74 z76F`!K)#TUv27N#wi&xZw;u7@0pM`EiK!~=$Di020L&SdW}YVl3#@~!qNiQ=8g-=` z_V@qL6B!~jV;U%P{c!0`@^hm`T7)JFxh=AZi?_W@e|m! z)T*tN>Ic~+=I;M`t*mtavt@$BvoeqP5Ov6hG^O2}jYfk4{-CR#0{$wsybzlIb47>) zsa+$95-%+E=@1G4aUXW{(RC~@ z2kqQ{ryGv&$$1^hgV0g|fQp~~$Au;6FBcYko6`^fLXFm<;7Zm1c?w;Geq~#4n999A zq(Fq=E#?|Ay&sNIoc8~yMyE?)`dVZtEfoZIg~B3Tmw*B2mjtx#4Jh=IDz`_JD2iL9 zpNqV=x6P))Hrd_BkyuJi`s0(sDklWA>VUz(qpxl(e^WPlT~~pqyox`<6FSktzlHjm zIxlT8c_li}%Kep^>cf?e;gbO1Iw{3tdaTlrP7WegENoH)cW0_!jRrOi^& z`<@=pDVlB`eAT=2)|2C~oAr`3=WMOc5a1JUWWlAU!!l{q=L9 zk0Nl>)c@S^fNa?TnRmO`oA%|Dv>EaxdC`V7M7cJlw$SvloXpj~8Yq_wYIepa_FWTm zSjl3Sc2moD`V$yb+U)Uoz5lQerGO$eL32Un@`~avDUaM){y(sG4xyxNnSj~kr z5JO0;oDJx06^9}j*Ca5j^-1kizovfm5ygf1^8CHLfQr@x>#x;)?kD6ezvK?0eiA;l z{KSO`j|g2liaP47Hsm)?19Eb{LJ(@STvj;^uKFep%fbXLA`zSh_dVoXin3x;=?ND1 zR^;ebXePbM`x7fIvXhi@n9vn?Z96PgO>YjZAD?3VGxksTWpD4*=WFZ(j3qWdT2zcXaAt<|FF0bW{(Qi(oybTF!laOF z;;tgRq_mz&ad|k!PB$n3Kllfl?IyMrX-1};0YhJu^BF%E>hv;Cu9uAL+%=rqv(0|= zYNDa{_ZSvAy+}v0`+=aTquQ)$x7nGi9iBSDB=iil`eY!XKWZ*Cd8bj7Q?i~1+#zA* zq$cxij~{nMabt!JY;LKP zm6u5S{I0(Tn!N1;U5~$4IuYy(It=dT%X$Jgcl)CNf?N52m;vHQjdcWJ$jyiKR316k zl@mwg(|%ay;JDT==b`=RQ&DC-o1J;upW@K+y!>-`5!4D)Z6|S+K&-__D4*eqqK$8s zM`@jvu23n0<^6&C;_iR$3zMh(mJ8AO^wYoZ3qYRa_|JWT6fLyv07e#NCLb(YHwV0@ z^3L3LX}jS;o;^<1yjCgiec(OTiVBM4%MW3&*v`qk>s$v;j|cx}=6$|kl{^i5di7R> z^TS1%`BGH~MFPDe(f(XZxY}K9Jg1e0?dQ2);2a%=?r}0GPGOKX$Edg@v4peG0Na({ z<34JRP#3|DU13zToa81`F_4;8VSr59GQG76+);}!24ax4)7i%>SZAVx@cgix?v;ds zmN4B#&|kmQeCLXlve-nbBNwmHnqQ{g>!%?NviUf1T{69O+utP1@-yyqz1YR=TiwCZ zw##Y00Px?hJd(hB1^lz@xRXjcmxa|1>*eT$zviwJKr(lpRv=HN?vou*+#x@t z*ssNIzxMh}GQ1adwxc{vylDZ$Q=jJOPo$wW9>_?KBIEspa&d@h@kK1;xAOk{Q?6oc zc|pQqIpOVvbyFWJIMIW-Tlnz@eLStLE(ymSZLOnXm&1@($kBNR zG;aaz+4r6&$>k##lD`+vU!HYJ`2J#9>(CKM-U-WCPcv23m+W6_ul4HOVkg_0_5@T~ z)W~_gjycqQDNM@tNHycFm$E!gY}(7sNwY`(x(|m9J_adNVddW}3OMwh(y0$XD2i1iCz1)MIhVBEU9kG#&1xUl z6<_{Ef4&K*UqSBL;kOEB5?Yvj7r#+8`|X#3L7D$LJaNhd2KO zhdb&~4mR!no*IBp=P;^BePg`va-eFxP^-a$qgcC^_nyaxV(2?(ws?K1q<|!k z<0hc=;2VQw22U)fa=xG*x4&?Y2AFB++at4 z^r_|x03r6uN93=HIz9Tk%kx`l+gq`$YKjeSdERpZ+H%YfO#MwL1PqFCtj5EiNN36m z+7cP#nnoX^j%-BgeMY=5CwxK6o|Xhmy8gpyASOPHSEo@*Kg=InTbx4D_a)8HtxX0J z)3GRdMSYlb7!~&%*rI%n2+tCVTGvdvu8GKf3^^elmcvJ@TJ|6tJKP1E_A3RzHA@P<+C4P;#X0gjdHfxX$*b>{i`r zZC0_8O;vx}ebXWl(feld_d=_&+urE;GwDo_fKIcUVnn9pOlf`toBORR3#0BjEN@=r zrmV{8`s57{wl@MFtX#K2+WllRmyk(oEOfp1YVQo+>8Kq{!e{YXFvB>&t%JAtdR_{s z>Yn=-`>L4|$OEUmuaF2#Dh^H~!AVy+p7sLtt=(L@>xV$XPeP1|&z^n6n`tiMU9K-T8l9PnlN^E55Gf@=w-?wkY93GoGuX8^6CPr@D}v!B*g_WX6Kd zcO_{?&jmD6qSqPqRO}^?8R}Pe^;Zju;CE5tNK(_~?YOCT+ z3oz|jl(9=osWdm^A%%ZJ^>?Ggs+Gj%e_Mi^vmOAA+kt}hX3E|g&Nix(6Vw0S=eHaC z&+`N0!s>rsQ0S3{ZCSoaGAmXoqWgSetXHxHJX1|J?`_x&toFm;YRvfVC+IadD`!$A(orC@S261`RL7US>h_Tl7EGQ9 z7Q4E|Zl2;(wl|KoW^V8AGF%K?vT`NVd}?)%Fj_1V)R|u?uM5fd<(l5u0Owkp66t8^ zAHPx>4W!1)mfk=G8vIN?c?yqueM6GT0xq2fmjq3F{Ocaf=0HBaiE!>tw=;L^^ma zrmCJPg5VXW@M;G*ZJ(EV8L6yXFyj!oHOz&aOKo47{a?&-UNb9*cMV`|U0@{K&{PkD zY~3DdE7)U;ck&>TkDSQB6MJxNU^0Cv{8xH`3@`X(qlb;v`FD_dlD^_R;`koFk#sQ9 zDy8{WaxIpHV!mZ!+7Z7FwS?lBZ9=`SqI@E$k_cz1b|a`DOsaQw`$Df(Is)LH_q_e# zZ?O#s;|YA8(c{~0TGHvnbUc;Q!azBl=k-!$KgvQuWl2W$t6N~(T4A~}$i;$Z6g`L= zk^DU16B~mT<(7!Z(h=Ra56qqRS((tP32Wc+a2 zq{vd~mg&E@^s?#-onwRjDoc;&SGVa`s6~Dw7aM1Xe8T|>zns0}aoQh$Bi-p7N7Oon zK0fDc2|EjGdnYyT6rU~4A)Ue#ewm@W^10VkE_WapTe%o&Q{Z&DR1WbUm)XKN%c}qW zloC^MB6)Gp%76aLq;|fT+;6z|_NtXxVKQk34OBC@XuOj>@7%)k+l?!`M7?ZuyxV6y z!d$uKI-7blnga9@eQPMzVZ;gc$!9o|0T-spBGwAQ~l&hf^+i>sJiwQRzPzM=`xgvxt?vf z!C@V9>hHYx9KQm+LG6XQ@~!U{@n%e$D1R{w8@WbD_bZk=}7@FnOa1-4RNj ztXwr{&d;iM6pSNIk!e&J^{Mp`(CA^=r}S*}h@0=P+?%VTmW{tm15EywYq2 zwwLS(@II|=IyH==Wj+MFs;iP8a=aZ(W^)QX|Ip3nQf!C9TUzbebG_hBVG8Can;Va; zS9q4ZTOc#S>L|iGT-jerAfns{ELqR>N0mc*f)I`6VQEth`IJfphf}eo+rPvyk8y$H&;etBjM>u% zX0Dg6L&g_$!|LjM;+QGnIGwV5mNZixIx(I`Z4 zG(iMq=dezGG+~0@9wZ|KHt z_lIG7noXm0q)by7j?L@A&ewO)XKeusaXfefUcXe6tE&f17ZZ}LabFSvu&#JAt1*5P z@)DCxUK)>v$#g9X(W2zeL_q;ysX0oQNZiJeBm4WxleCe%wR5;~3Ha0w)^*p{01{s= zp19|+qd_axzwF3;u{Hii%cSY6_m7bGpw!J&%mIJBmuVYVh)N;++DZxps3#T(~P!KKixN&}{!tLVVbA^vsox=L5p} zAhuVdH%bbhar$?4=O!a(JMgU`w5^*#gSIG?br>V~m3 z^bC9`OdINzwvW|W78}3pX{xc@I*8ukzw@p2YWIF6e#0>?wv8y8a{Yd9%GNClNSTVS zc-@E8!wykqDTj~h`?3IgU#1_A1mN8@wvYx`kL&jcHXqydMF(B5U~WJPf~ng|my(}V z3J+d18vwDt3 z(+@w?N~~XQOqIaAo#=#TO|H3+t&WbTD7*y#82LV8nc-kAcaC!aJx{w=s>Lqr#ruR_Hs8FrW$)XX(ke{@A+d=Tx|~zv34G|A9`_(J6B2+ z$jSgDd%!SL&UHaI4jJnql+$+Uh4Z8S(N*@$%$dlcAG>$YULjNj5%JbXLMEv!OTSZM zAiCU))O&9%-$^}LiQR7W!+d?HPnKwMmFbA|{q02-CHSS}ME*!v6-S9j8)$hvjG$zv z05@06JhX0XYlvjb`*KMZ5g98N7zic-B{SYk8eS1zp12gm$SRsJ5nrSrft`xDYF3*F zM4!G>k3f@6oh(4z61>Szux}C&I(FrDDdLM2^ zYCT!OtXib)cY{qPy*oXJJR}JXoG{uLK`ffNyf;6I--+d1S1<2{WLD~b;lsQ)&-1ds zS&Z;4(re>XE!X_q<36OxK*Hc}(yV%iu{RcVm6RZVP57TK2kszJFy(i+O8~ zPtMm9i>;X7=uDXdceODApv$5!{BhVIaX(Q-49r&ln`l~aFnSADtD z{t&=!G%E?BJvPH<`mAa>=@vu6<@FtKjQ+@=exeqw`1TI&+VGECVgsdY8aq$_bm^Z( zkbAJPfVfD|!>3@LUvKAC&O4+hYs|e#+8sRsp8CTO&mIBcZr{d}O@6bMcqEB@^|t1= z{0$e0YnjMx!5ofr3i#HWggkQPv~-K!+hMLrMtLY@3YMe?-oEU3TkfYY@Qe$o7?Fp2 zf8v&ugu{buC`CXoE#L*q>-;a^;+LiM85&iLx|n|1W=*CU__VQ4clj3y6b>HuA#CA+ z_|||Y=5SBsqpT5@BBR{g{ViBVF#0CYWGIDHMnFt0obFIQ)4j2l(XP|8Jq0@LO^(fh z`P9svve(Zj>44MvGYl;hzUO5J|FS&+Pv3nnTqEoEg>DCzn-9)Pu{&;QKOQTLtw}=p zr<8cZFWa+21V_SX#sFutthc4%k(7@NKMB%BX~V>5LWMF}-5D{BfN|AVp`h{AZ5=gc$9f>wUu{mjqkzCe&N3A=IDAUB zcXhyo*?|i->%!q>)o1MR14pLAH?`e;KWRCC^@C0|h*NMngmtP+q9M~WkaO~7?toHZjQx!>V6ej1a^gVr z2N1(E-8o84?q~by(LQ_vT;K|DX?h#iygXVhJ?&4hAMBCtn;g|Ay}P!rf%&yrAa9fj zz-BxL5aoOVgnE&ihadX(f`rRu3QXTIklv#_a;H7$xA4uY{CvH{81 z1NTEDi~gDs$?>U%dW07FXIj5|W1N9zKwhADS!p(=wF9X2aFVx&wO%@VnteIz>o~jJ z5Y*UW0b?E3b3|PRB^-f=Q{Q7_60@e&*+Pu#0Ai9REfvC|2WBYf8<(EVpTT9QVXtf+ z;PBCX1G1j#*~V}J3t@5kQdg0A5Ih(%)5<$UCNk;vHF+OlQCNxeb@92aKmhwJt)e3) z$oQo|;&>MMaQIqru3t*FpaKt8~IrOy^@?J`e!Fq?aOV*6XQWI7C!0v#ns zHk#d9@-nF@7#V{^nv_%2Hacj+2o~Ma4c<%y_?IS&H<6`2Q6`=>we{>I4Wxi?R|5S> zB?bSm8Q1*Rz}P2qEtme^ch`8Cua&AKSXM z9O4#@Wz6@3^a93PYpJlviyfNoaQ+9jKX&!=cDE~bGJHS&+d`Fu&%>+z&E*S`;K&>r ziCoCgv{zyAtiw$w=fHC0*jvV>w42+P1Dv~Y0yot*hhrTg!NzHe6+>#}YRM^_=2V@ktUht54^?>_LiV)+=8?l<_ zdUga13nO}oq-KDE^hujDMsd`Mtzuc$Fuo3b%hWLl&*!%mbIAhtd=IMUwSSJ;#wX+VoW6Le!n z%B1l@aJ6^!t?tXUpuwwby`L6uT*k|2Ax6wTk1cBof#YjEWkk==Hohfj#av&7!U(9E z`z%ySK9*buztXL@d3~qB-P1pJvCmifuK6A5&`8f${szb2*vwqdeC@CeGUgXxKhtRykTHdK)IAq!7n-~7dYtn; zoQS!q-eMFzfOW~jjzHsO0b?@Tz&7zA2&OL=kn!zut)w0q;Lau#v|W4qtgK|WABVtA zu61jeL~V6)>(#%!U+BRixIW!Xu)pw8?`B<1!y64Q@AH^Qw1#S2c5!RrDQGFjwoIh6 z;rn#YNQDzHQS)K!tx?WiVL2|%*pGU72LaZ`BhRR-(Fhr-4qLzQO1;K~eW)qZX$y5Z zym0o{tm4b0#L{b=BwyQ`t#WQ+;Nxr$d0s#o5dcUp>k9h-Z(*B>$5}q~t9Mb5Xu;P= zlBloio0^p2(P|Wg|9o83S#E^39WckQG0DYx6e(z$g*P=J1+NU>@;kZ@JGeG9rN2(v z;PUF*8yp7ljksR8m}Z)qxZsj6w3zhx|FDTdzj#YMe?Q^+wrzgm{b%AEDml)4nwR~jCeN?M*M0I6T$lQWPh0lvY^9ofZUc)!4`e@F_-L(A8u>FfdA@WW zGZ5HJm$Hyex$+aZ>Z}_zuM?8@mLA14PG{_#S_5Qb2tT6SHTAJ*8@^8~9 z#RoGrq1QN$FEdjo*2th;7yfkxPFV>9NfZ490}b-_@loVF(!cqIbWBEMuIoSAtHhPF zNs-n~q%Fl$mpn1^)X%lwAut0G|PzlSbNI7y6-Z|xPHCw<*R;pjH#(VH05-pp=M|`mP02+ zDS^zt>zvM`Mn(aboaYd9WIfRsSUD%fH(((K5=300l$&pADONB4_~uHP<%BH8QzQF| zyST?YP4;ds+1nGL<+KT$E#1+jG#3%m)rMynZL*!t6jxx&Sfe80nVjLOxvJOpQ_azCV^8xh$Mf z#OYr-bkJWhRLs=tw?Z;ilKi>Wpt@Q=H97OqO$6uYawP@ zDKX#~15|`sY+BmD7jr7pOt7xLi*T8vLlj7*ZVFku(x-EYeq~Z9P3Gy46Y9cCK zmGrMr#r*hj<7?E^Cj1OJ`ijjzXv`;+2eq{$YjO-ChbOch%s7o|sD{uz^eZwHn3Goa zJJFF1VVt$k6iqTcBH5zcAX7*&W0@T)9Q0cj{Tgz;;U1N#;fqugbmICA8ACr_k+|-0 zH)ugtIUT|_n`kPZ7(VP#9;2Uv9pz)HR2dQ#KQNAny4~Q;k1X@pa*Vz4S^YUP_MC!U zseD;M(+TVa>m_eV(b&sKtG6`{+G;)#xcCY*>Rhs#c;_d-h4w%EPP7;VT8tj%;I8GT>wnt8i;6-73M=!WY zsasm#f)V4z>w}o%qlp>$xu>5e|N8bsHOj>xVGTaZ)Umfe^>eaPN)+|dR`x)5*QqEhu)8} zxkLAY1%&^PvbTrtd zM(O6gcw zzpdljBwS{l4-ET_vETX6QFBtJ=Wu?vWZtjXYC?XS8NJPlVTKHAq#k>`vLF~I6vq%N^TWZ}r~G4FI;>yBo8cfDZht&w=Ke^}Mu&ZK#s?86)nz z=a@%Urx@xf!Sw_1gw>E2WOpaiqMS90uldOPjWYY@8-0O(^w_LTK0=0jsyUTu-#)RE z>-3A4aj#mHCRD6lb|zlXl8{w8Lp_Joc)HjU)6EUh}&;V5u^Dlq)-ODTB9@z#3(G==G`;LMisE(@V@*B%TJvAF572dr^ zvf)wL%bM8J(oUop-^$U}vLb8d;ae8b8duOp+m3C&ZziNk)u4t`$$Xf?d%!WFRZ{V@ z=$y=qH&hJ=ZVDc&08JD~jeXtf5{V2J8JrAxYRG}cAKjbC`t-vPxdxS7$2=~y=xn5b zIrk->k7lzs^Z7jP(GnAKXr#|YUa#$S%vcSb1tnX`cfKR^9O$mB6fP0w2x&^1~r-!Aa(Ce z$##_09cmW9s*R6*0LgiUc%9mHg#_tVF>J>7bj-00t+IV|KKlA%{A5D z%;U{Pv<`8cIlcP81k|alDxi0`f?-1Ih&HzP$#83Ig7@AFpMTF=3iZ7QE{L+uMDRWI ze||DwFIv3p$A7}!=NJ%yrp+I{mtaUzPfV%L2TQ_bLwu8b{UC2`-CGUENkds~#t?rD z3bnydS=;5+87)?;HL)}1-BDv;l3^YX>q+-lE=tL@XlJf8yyuT<4cg)9S~)O@vC1wB z9$#kP4OgZa+~lDBt`?S^2ua6c7zfNJ1Cl|Q%V%#0)tMf51^k|FVMCu|Rde-u1NjCW zY%p`K`9@mv!O^Pv#^pDIJZ7cUTZ&b*r8n0Rx+bHSRtH=?ryt!EY`$9~=alLu{jPSk zGs)ZhhfelsM&t&t)yh+hp4BmneN)Qjh5(p1XtVMw8M+ z`|D+fmf2x>`e^M%-Y2ANV8bJ{%J&J%{o)Ue)<`98<&)b+1+>VE;v5Sr`xUPm35SBk zNg0Uvbw+i{eo|5uo#X0pc?dLC*nRu>hB>v_TZf^WYe@S6csp(4O1ezPJTqJi{O<9Q`geYxjuon9~h2@MeJOH@HiS&!*E^B|jrd&JLkh68o~ z{yiyibjE6IJ~Wp)BFy*RbO^qRKhXB9u{za&ytTGA<*4SE>DW%26-e zq&(Qf7LW;eYEhZZT8L_)pV+N zsH02hyFXZYC|adWZ$r&y+a?K@3}W*3NU1HpdadBmJHmjqZ3Pj+K=z5OV{RGOdo?VM z2-ch$UeieX=iBN^;kj91%UHy@^y6kD{*?urElrI&dRp(9Cl{orbx7(G^RxH5qiq~` zhOR%0Y=ckZ2_&B59(^#*UG5^uJ@ZKG1 z6TpMV*Yxk%386r=)-8{E@ay1@%eDp$os(`9KVX0)bv=$hw0*m6$JVD-@FK5=HsZxD z<#*Gha9*S0uP3x^d%C7Hhfh82T1^Jnn8}o>M!1&`>~+5-odlml{b|#BK1@QNmZf&i z34Hfc(HtexgL$V`d64hUQ5OU`Kd60vEtHErM+pre0%QNV1>8eT1Omy_hYwl*%2F@d zOU>{ft*@lPt)=HM9geFA0}^p=?xDWydvX}EPNtV&JAki12dwLa_3iUSHz6}N-|VID zUe)9hhsr?-!IeKW=wt$j%|PFl?i7pkQ;@R&to28pJlba!o7X;i7^?iTr}g^EbgJhJ z;zgl=qoZfzyU(CnB6xX{J*sk3#ZCp0B0ZdXisjQ=T{=y%P|9!>FJK2@Wfm4}0 zT3Py{kIZ`>TRw2X^+1~|$=%yaSc75YD>%r^jTSzPqX(d zOeKdeZ!(1}RQq{e486>O=kY-f^eB*df%)>w6}xsl>KNNb_&vizv5DnLT}ekRJ37LA zO%_dt6te{|C2Nun2{*l_y^l2z$6kE^Q|NfSFpk^Dtw!S?#fHYMyD<8NA6L{~T#ZRi z1uNUc^X+ocW6A`OyYl?Pc}-Mujd#nf<|%pz4Wm~~s`R+&L@vMio4HE&2wClS>^GLPMBmOGrO*!emkr?U|NJ#alBo6;1r6&9!9v*?vkx(_Evw)bpd+ z+sD73obo++gB*Ko`&;KM+^*h*qv-mu*AK1_nmcPg-OTeenvTZ{0_@Cr3OEuJqgzG0 zIsJXK^x^JwW%NjgH-%v&GDke~M7&}7&E?mv*7XPZxS!aSR|tr9Zn8avO|a0zE1SPD zJz=!SdY^y&A*?cOBW=R1`n4^J#IgR%H-8R;n_nd*2_@wqL(q7(%=W}cQ{yImpguA8 z|2VDxJ;Q>4`J;G>ZpyXft=Zwb?yD^i1K_QG=%&V#n8a*uIGeIK1*-*?H9S+h#1&K)(8{))jZOX_FH=Gdt+nz}-N+eqMJDtW`; zb<1kSQ;zARlFhgJ%tl{&hoPy#Y&8aB)9Rih&1v~H9J)ZDeO`Y_+xu=-jmOMcrb}t3 zk8AnM@^jOs=qYMZWe45)l+UH5POIDOh%p@(1+1$OBB^Y^dkoFQGHm?HK%-C8uYHN; z>y@a$3>-kA$ov1=QD#)@^kvEa-o^nFFO1&)gn$aY`^3PxN;1}n)+MLc+>1wKlkae& z<;LBTEml$)Eh9mF<%1k=Z=#5KnZkn#o(EXWQ*41Tg#k5DxU_pv7uW9}U&j)$Xb9y< z=6oEOsWcfoqul8JnPu9%%6ELg@igMDxwkOvT`&B*CS`q;SX16+*{C9^nL`9g|&)}<4>1cDDum~1u(GWtm+X!-y7?LY*I2C$J1Jxcu?Zr;W z=XhAFi1h+j=pOD7)u%`tR4V)|^oBG`9cApu-5y&D8bTI9WwVLQqGZltZ?sF+OHJjJ zr-NA$MD~~k^mk6uc9E2^U-W_R7FamUK0U)z1F)`@Bku-unXxC2SH2hLW`$I0AAM1z z3l<{pP1={W^S1@&T;6Et3}&3~L5nSG*o%~kc=4R(<&4sE>IKT>Bx>1!_(A}PFJ55$ z!2#}hTi=-hX1W-(KYt2DL7n%fZ{9+mTJ1S7-;_VlJ~Bvkk4cCT@goR6ll2%E3e*n< z+x~vFWos(6b*k>^*r-uqEHqBap1BEJ9}?RSI&B=1DA#CCPxp~s;fFA-aj@6`mA||^ z<42n;>mhO@mKy^ZKq`eK8R(c^;KIkdAqY~A4rJH5YTLa8hWKxwM_tKVXWfPFN?P@MuweErsnQm*yY z^WRiTlW5eN)^37aB(P@5pN^yYr-M-A#QuiU_b}f8{wWsyUsum31P+zWJlRbO=G<{IK6jix9f7M)Ib;_^nh0P$!^blyl3M*}*y&=%Iq65C7HL8P+dLv6vM&Hq^J4K6F(8@HCzvBBhyRfaYh<~`z)S_?lzFDsLi1DC zx2N%4A)bZo?7E}Q^eP=%vEluKyZfalas_Qlha9`Xv3b1G9OMp#Df&sxEDJ5fQS|sG zg(FA6Rj0$vr7ixyXq&iO%!y`T;qP5i83 z%;K(L7w=Vk$5(8K%~Fy5PY}@bqgS_hw7q?4%}(>d$!-qdPHUcviyB=CWXB1-Q1su! zsT0Pc`kzyjekYV*4A8#;alF?v15{r|xX)8eD*CI!ok)_KX)L`S)1u;x8HUga0VcE`46y- zfV0DwoAr((f;aQFI+$Dd6Ll^l2hvJi))i&EF~?3akTcf%FVZ@9sSqCusCoL-7>c?~ zcze&rPKv&%j7wHH{d^)R!6cz+Yi0%$HdZ0h9B&Qzuh?rTLMnN%Oq~|HyH^aW%V}I{BYx&XUR4q8o(*OIoI1ylO zfSWgd=!^km)^h<{#v-lSnR+)Rkf&q&CKh5}a3F~z_a}AdU7FcTf%#i6RzA>H*rd)USitDY``eQRa^;Ik)wC?X zE1|azJ0dfli(n^9phn&9U^m7&H@6}`N0VDUs+i?`W;|p1H#MF^}HO-=eE1T8swKXn<&nE zr%yYS%I`|vw=-XaWxwT=H0=C4$}oJD=HTG7`+L`T3>*rvXb;$qS}LZ~Nus#PvcC*_ zlD(|l!m`?0qV`~Vu{LO=^s~P4@KR3rDk0DZt?6n>IO~ul`o@}34xR6KMYsY;fO*bu zgt|Kq+fy@&N03X*Tx|c$8fYBO2RnbJWM3r6Ivi}i+EYs5xe4mJiKyD2`V5_~G^^H6 zgz>_})?uRq8x5aY^|}romF28$XXp;K2M zH(zzoZgcx{mT^-ALhtv(Ctr3(c3RQi=Kw{6(CN>Z=sW`k8yIF;P1{K>`@NIt6#1n-8lPAz;1a} zU13pg2h+5r+!+aR8T~NfJb&evxJ4<`8GkL1E>GP#9ML>=&cEQ$cbxltxzkfzv)Upe zS5ly>!g${ILd%9OY|$~4XEx##J;G)bzB--!Zn~SH4bEfmYn@%Mo$F}V@x3!Ex%r%P zmrP8{Aeu@1KyqU|c9^WO-Nd>7c1dPQ({oWa$Uu;6;DWp`hP#mm1HtQ?#cJBRm0dRr zVg{AaI5*H&oqVM+rj@GuChTnd2d9OvWuA`|ap{T+tFM!=LF!j|rXv^NS^>c7_C-Q3 zr!=i?51jIdo|Z$7s2~Hw=!vjMbFZfn|4lLxL@u@h(Qto zrJsc^bHpSmM2VFnfCkXE^x<2RCBmok?xfX+A1+Rt6T?@7yjTQ0(>%^>qD;04Xl1$M zoUiRw6{gD#jaRy@jIaH}!dJ(OG?OW6_HnZg-`^tcl+EpCv;GL{RJ@4p^RlZ5z%PliIm0bf|VydW3{83xp-9`jfa;$DK)OA#T6!_Mp=h8Sd$LI9#w%B zHp7`T2FuzCsjk}fPq`<+_Q$txCJB^CEcV z`qiys+QKED>2L!b(x_|57VBhVBu8pk)2boibMiIv7Al{kYv>;`jdJGZ$rXl5@*1+8190tQKg&>S)Quo8wNZpQmLixcHgeN+P7h@WuO*6n|e|^ye-69+~i?uNJAW# z0nchHpEkVTF_qPP@nhp^S+3bgnXm9Vg3pceGPa$9-`fI?uyDwOO{6j}LZILWE|(4# z-xP8lvD?phRt$V8W6Uv-;EdCjZoU@q1EC?vD}{EF3Rv<8cV^Q#>~P6>T-X@{Gagl% zA%o*xze_9T-I#TN-{6%Olr)6e-$a{p=jH6Zv^q|zG!N8+MR1cyY`=T~i`^P87+J_X zoEYkh_0!9{ToH7b<8a?l9iA3~fLC;;*Pm$AJNUfgrVwyblOwdB{jO~HT^a5_tWM@E z>%*;LWsvo4gE8EL^k;A%i(VEHDdY5_Qy`2$R=AHyCSMjdb9B0JJ7k2#+t+rvMq0uF zBP7Lj9^OBYV#$^<&rW?O3>8>WGBlkpd`Nn{HKLmdzrXDG^`+>C6wK%vAET^M+DAGr zJ?9mEl=GSJFhQ_PGJB?&xe`8Xbc7RR^aim4vTE+Zex}yYUMs9a4}Av;UP0k>?_^|P zNz&ss{o!)k!SW-Saf5L|q3IH#8H(VxmO$J@zeW-UP9U9Y`d=rw9nZviv}$&0!h;O{CHcJn$W7n{wUu04#Yyw75|xE$nI5YeHjXvjQ37Z?i@~HQ=55J zT!ZvLIe{T|?1gL9t5w1;dy5Nf3vD-N%T^35Hnw6WklztkUAV1Kk8|7Srb;DZkQ5tU zyl=3r?I|0ov($R49AG%w@T@Zy*@xd`GrNV%PrC+N+OXC}K$Qx!piKL8)dCH_2gmlT zK7o@5_asPvT2fR{Hxn*+6Aw!XZ1*E#fTWW4ho<@;bz|vcKXA*YUUA_)(k6EAXb6w- zNEnlYR{aF;&*Tu`#k}cFYfKQVcK}h2spcz)@&=^c&je-~%wV!6=wBR7v%4br7a?j5kjogY7Nk8YyN zH&n2FNUvGU7t5^ZPs>DBVM`u7`@?U!)?H4+ME@%>F)@TJ%ywX^=6yHF8)(A_79>5e zRan>K`*m4}Ta>>5xn8Y!4J2d*ZgHB)-Xe}D3iDjG%cLNd#Cz*0rEfn-WD^;o)pVEt z)+BPC9z~Sp9sXYK+ITA$%3EqhIsRQa_Pa{qhtZp4L1gtlJ~m0~I0^K6oG(q=BQc6m zgq2$a!|DdX2_i+9e03R6jvtnEVC`-BKw33jb%p!NJ+FBh!>WywAzG*1=mu$}I7PAR zwiLPHrko}g9pdf@6kl~t`)D&7abx!AraMVKd78blDi_s1^& zdP)T(-;(fY=c&kCNI~yxw=|;|*SvLQ;*`Mvh5NMVnzk0S3djp;9kt({c(-fb#;I9h zEMK>3R3?6FGQjCNokLtY=fqGApE80WvywS1q;W{c?52&Frm{K`8LyXhr;Q-`UlOiv zm}_geE-xKQ2N0?J8W^=QWrh)JkWJXt9xcI?!D4tARkUA@u6Z8?(}kp_+-Fb@c8cBe z6=#C`bC^SHZX{bGWT`wfS)WR90Krv|rnb2Soy35N*TMGp85p(I_T*)ln$B zzRZ(qpl&u2|lIm@&x+b2+F^SIlng|x;6JB3bu#$fqzl)b45d!qoy&z$I#pJ5T2 z31!sEagVo(FRN%^&Pugo11g}9Xx-2STfFG)F0s-gc>hMU*-#Tg9Ehj=5k0erfd0d2 zSx^Xq)lM9vdfG{)c82Fg7VP>GKM^NQK>EuLb83nK#+%xUSAvhuF(1L+bh6lH7AV8c zFNmBuvHKJ*4JcL|y7kgTLSuoh?jEn56#iv^?!(j zgoL>!3@3WBjL?=HyT(DrzfuPjZO`((3l}g`=m~qL0WcIBDeb(J9D|CE_sPwTL6rRT z?xO31-VGUck6){FzpWU4PXtM}u5h(LTCTq=w0U8N2hU_9tqwk*LNDMwbl2P~8&R!z zY_M1Yb$AD6HcXSkR%bW|>2I6zVzAv( zzCJck&73MZ<1v(4_YHLrFf9ZATL4%10<)U0`3WMaC3V9FFyHT~P%l@chD za<5Jg`vF2lW@)4P1ml#o=_gkjM{riDzlRCa#dZkXoef*QUGshZH=v#2`-7axpD>78 zDPSEkRPC0NmE_AO##Md6bGRoP_&6k;#XF?~k_&*Bqn;NSUn*yUv3sw7I?+xEeQD}Nq(c6Ej@|Rz{2`ZAd!49Jy4a-T zqc8jIK>}P3^dU*UREuR$mnzh!c(sbX5OpfAj|TUty{yt7Q2V$6)XWquekLXHr+C^v zIALx0V=_{~;qGm3YJ!*XLDEd;J90=k$Q_9Rh0T;0JdNdn!*EM3ncoNqo3vdxC&Hy zh88_RXYJfnK3;`!pUnJ3!S<3ZD(?qqXwxga5;gWCL6!J_AvF|yObGxSjC3j`__Izb zEBx*s#w5S@7;(EvKOQJEB^z=Nism<8*q;s$xTd+`X~VV!{35$WQJci17nHnAd06nl zZ_3MG0yLb8Zz&~^&QSHxcccFP)N1l-a(My7p|?s?@dKK7_d|nelMt=!OU(G4bO>gh zFuL#TrT`k}Ula#S^%Ma7goj%N{-VizLR8qx24kW>xZg&6>qAxGEELiaDkEVAKC_L+&lW(?%A}}7 z(wJJ9@8H9ot-vI2u;6N z!f>@RRM>5hXQE0WL!!d_$|QlX$KU4HTqyI->JoiWc?XXf=T~Xuj57P)VNOd>ReWVV z(kXHAQ-1r;njQjcqR#Luzgv?8mB=k)H}it|f|B`GpoTZt66AW2#6RVAXID-+u`e=MXTu*dS&whTh_@U*MNmZmBjNhqgPnZi~)qX2!VVwYgZse0F0P=c66)C#U*lxL?-IRaT zilN~JoQhw&>*u)*Tcpr{xSnz^B9Z^0!$04tJ)`s+^gj!Uti}{EJ_;>$n%DcvWtyx| zOT)2!9A00}x;@J{-DDCDVoAaQMti!XwTW`ZIi3neu>}QMBtNE%WOG?*g9F?;9w{Nc zH`tpdQnZVCX-=K&<0l?r8+>b_mgu>YwMeVC zj`hs*q%YuBKmq~G-+@Ofy=@Va!ngj=Qti2({D_LT57ky$%?}k*oam1(-or$v1&*3E z=M>ss0kB|lvpewe@##`k6#61f5D2^dPey)&&o*GOOvrKM{3o-$)q*&}Nn=-PmczZIj6E0&vUqxvx$Aa@1ntU^nw%5mnLwLZEZY5_ z;6apb?)!i+n4biuFAxh3TR$ylWKL1c#KtrLp=>wS1fq*6aU<0i(f^*v^Jrip=Wy)| z!auqJo0IwXxZ)WQ{Tznn;5Ysee)hPApEYo zH-ourOJTpBYasl5`XmEd3CHV{1(Cs3AN`e)o}ZVGN2-sozNqUM8oDhEJT~{^8(zPC zto)xHyyx9;=Y<-kEx&R6eR47Wo2j_bnBJl<%md5=Kh;y+9f~VtEn!OJTXeA+UvHz1 zHE1Up?TO>(8baN9K@*3%UZ2@ms_Do!oNJFiI^JanKUCWoRmXqq+-QT2x$**4rOsc) zcjblgH~PO1DZE4eRWwkK|57g9UFZk7e#&HPPgY#lyvf>@p2jU+2fdai?v zM}kx6i}7Et)*wxlj@IaW_*$t<$T^9_mBM17sjbX|CMr5#wZv|tc@u8D+B<%S7v3TA zq+I{GmjboUS>cA9W)lhYs^34rjRtZoh@i|md_7~on{SPVgJTQGCgspm7B00UrI0^( zS+apDQ|`Fw3JL1(8kMXAPhr&-j*KjH@4hlOl+wzqJLok12uC!!qzY?RTe9uCot4_b zNVzP*;{~eA4ykRnux+hEHTk-grNe>BMz?dxE^JZoAO$4Z%zAJ3KJ@I@=Pp)KjzT%0 zE}v`igve-<{V4#GGuIypUFa=krTeeN7hSBt02txpvs>w=DlRLBV!NL| z{5HQcuw;0iQ&d4WZM%*MP#|gUuXpF*@oT^C-XaLRWuiz1r?v=!ND($kuI7#duCC&f zTJE(b>ojrDJ%9?>UGVg>PnCOs%RN~u3)6VjY*K5Lmt)~jXqO@6w^3(y`3we(g1dv@ zxwi`uvrPfUdnxA?w=F1LtY+ii1rHjc9OexKA^AhcliEF}yCfA(WiYdPvv}Dx4<`2~ zCRTYFMq#F7h3W)J9lTZ**@rljx)QfHpy+Z_i2ao$S@&n=x}eeOvEfbV)tOBan>`u8 zg=npBjeq~x7dKD~gN$~{AWAIq*MavL7VnCN#7mCYaA%EfLYLYf9v05jTY0u0%)-Zn zOO-P^%K$`)KWcjGz%})qKIimEtzy@xbnIF`Ox*QUtQfwPWiw76`%NZUC0DN8qfKnU z;b{nB#8bD_x{R8~#kvbOPeIF#Y~ZKs6orQo4^AwDYPRA`bqkGP9gYFtlzVrF8r8sBg7xtt?igS2@LdPTMjLGgmWsT)EMl~NTsj99$ z$v|3jJS%))~+pu7^E-6pf-%EzGh)lR$4zk)xxaUfT z*#U=?H=aS$A87S!T)fL+HmN>*4xn>OO(&1C;C%Wi{tgzJ@Ci$R$db5QYun`j z`9Mk^4&o_udui9-&_=hd3Ovh6sbK+Qe_h(i`Ad}~c2@BJ6^+{%~ zl{7GxhEn39RWq@@xvrYQ4fYE2;h2&x*1|)#DsmvpIXb>L(XHu zYSDoR7QsVAS2caG+MBp=h>OKb3%96xfYdSiRoQ47OOD(*Z|vZ#AJ#&ON2r$$a2mR@ zpHdqQCiVe^M!P8~S}884ypvOOs%qupci8nIGxNR&UU8~Ob-swxVCjrjA|Ni6#-g)Q7><28?e;pcr<_N>QUWhbU;XSkEI&z^v^ zfe>k)h4E_tT(9pjsCqR0#JH>#Sw7Hv{kba~)+uWOATcL(Go75EfGTsFV!OwC^-`2V zS5kezLk2h^(`SL-H_y3~TJ)Z}+o>>Pi$_00@!>$27%U;p6+z z^bj^FkDX;&H>-|&ALDe{DLs!+p1(UFJzOFz9?9tCKRlwIeXJ|;{)W9LxJ@QS9p~&d zSGaxE=X-CI>0?tpwmxUUuNEzQ^HQCv*M|FH6U@cn4f#??mK z=`g*A_vOzn2^EhW?K)^SXOYGbSK;AzWizr#d5nODgAsV)p;{8>?)y8&v<+{rt+^hj z@Q`IE(8e;Im{$DKp(qRhgLj_-OxV2B_5J4pZj3UplM+H0rQ0I&OpA|~2*f!MgroQC z{qC597=)BXV&t9sdQrUbv+b+XG0zm0K{C%q8)WeEL^lIGnFw|)NVL~r9vI1bu1o%z*XuRpS^5FSR1VUzB8^)p#) zoz7FnE_jH{^AyH4%FNjxg5V4t*TdWc2Bvq0U&qBU!E94%L?uMZ!sKLznF*dW8! z9Ivh;-XHeG+vSOvaR#ThT|c?xjra`9t3yun*Ej5jy=IANbK4C?$qi)&d2BOHCVRhK zxc$XO`VL0Ap;;c>AoPN)YDMi{blbwdedn=_-$z|5&0m6o-sobyNHGXNI2OWj zsEtNn<7~d%5Pv*kjHU^-H z2m%_giL_3UuUr~Uqg){sxKNUACjjd-K1z7Fb5*(g%VU2|b+dloBUktJ^9#%RezAvTX@F!vjdS)0+WO#x_HnrK<`p~I^{U?(*7L~4Kg z7nraw*tG|JqHZ|u3@O|<$b#cQ%Act#PLuH3Ea9nyYF0XZJYeF|h3vua_=9<;-TN2U z5p+j~EX~Q@CbEWiR7Pjd8(!oyC$&Zwb@^C&!tQ7u zZ}fJ~lXjbL9$UzfUM}2oZBsl9ehIW;$!a5d-I(HN>G*xXCx}-1>g$A+wE%YkFvGL+ zUOChQ;4O{>vzd>iEh<)Z^)_%Pyw z78z$T%8Etn=pYQ%4<1)%`#qhDg=1q+UE)eQ9-G52CxPAuJzcjE3&Scy5$1UnKi#4P zB?p`qTev5tJJVrNeAYXD>#1*fZ}od9uBWn2M}b@n;0V9QNSUrAs03i&4MtAgJVD^> zA2sKhpf~T!!VMxke0PWS7~&n~s~2i5st!rWzcJz25|Ca=0+eNTwusAa?0P@wMoH>q zBj9G=vZ>iJo6;je#PyU1=#F#`T80eMPlkmFf>Wnc zjwOdE%dF2dZ%C1aeuAI_or^sr3n|QE+GxN2)Pc=<)zxyH-!%m*e&bUUO_A`~b~s7yE%qi#8r3 zsFD7dKx*roU2|bbBFD#ak%yM+Oez>U-7?x@k1eNadV+;NLW$5)Lb1zp>`Xi)qEN_L z36P!Aki% zinxClSq(4iKi^l!pzpzcTw7&*Nm5HHh8$f#E>q zZV|i~m1-Tk@`=|@{Nk<}*MZ87Fln5u%`Wt;`gw`cD|bXu9F)1${jH5{H`e9NJ+VX< z0rwEu_eD7O9b)TazPeUAMtbxF5ztOQrkG~zBmlCxO1~~|ZwD|}#snqL%Ll)-%5S~7wnz+m4No)om2CK-^|?vBTbhLw!3b318}$bPMzwIB zx3AJdKizlM>d~P&O%~jbaMnOY_t;b!P%ZtTk`2cFr~?lcK(p3^3Hubk`o4TsP&A!D z^9FulFkV+t!OnmN6J5XHQy0GY!fH3;k65~jjs2;$>!v3A^G%?4lTp|f$z+wCew|V- z1yaJ{w>=YYT4`yfZ-0B7_bzb3t6cm>(^dj>go=61S??YPE!OP9e#OaaciUrv4>Hs% z6EWlMqdK3dG_?i65kx7kGj+8mcwi6Y`r!1upd72_VzOqu?nZ~ciCvZdgn|yDBwG4? z6c$KMhJxe|3lc^JGo=H)q2n&I%&ec=lK7KI_C6zbaLaVnYpTKkz`k3v-3am-U#My- zYdYe^HL2!_al8fRbOoFc2wttWxIVd7Of51M=Duo%sP~M1=TEXwEuRU{LIP#~(|Qrb z%A@^=N~2_eW~-=x+tGfjfR~9Me*cQ>0D6p0Mp;x%;{a4%HbA`@D&PMNnkrR{(?gom z=_iMM20nf}Ti1M)g2`v3-H4VAY2lkIGi^k>YyR1uJwL+2wW4g`U`rgJ$T;tT{}w{r z!<>3)y*I7BSr)KwWYZ)T*MFehXC>KmUsK!YxBPla_T>CkpvPLGJ?YjV#*D^ksxKLe+4?v>77uW!OQ7a;pSaMws)jD#0(7{jTzZlWSw?LM&&RZn8 z#(xvV!Kp8(r0iu<+2eX+WP`pV%9cJX)SBVh^B(^A7Qd$?UVAIO^6BH41brDgcfZfq z-n8?1#IZH`wvbQ>X-1p zWxFj5U_k#tn7Yx>Q+S6&JJhT*#h1XRj&ZJothi$aTSvZdK78^OXM$f2g;E}c+>;&N z0+wT5ky16T+n@DGfo~cNz=M5gbbo-Gzjzh!i}x3)I~W~#GDP-Y=Hvh!1$d8@s9j$o z1yBvT|0dqW-!{&bY{jl$`ZHZb;;IlzD~5ME;`pt~@=+6K4)Z1{wdVOfTqr5ihBOA* z2IB1JUx?`c@uVwZ;7M1baYKK&t^*nrB<|6t$nw!L3gR>{$RT%hJR&p$fSY9qg?M^% zym((t%2IynZIicryC}!}_$}g?zGxN_VX#(5_Ggb4M-h#FHL&~&Bu(hAA1(lQ%!cn{Jh-Z@zGq?N>Na2 zKeRMP3HkfX!HRcmY68BeUSUKZ29Lo$|l^hKs5SRJgDlGod)b!CY?>=Mn&Cm6P z%@CBfjVLTcmhSBtl6x@CV@IHR!x_LAfS^=r{d7YXzRr|=eR-XJkchxK)Ic=ugkb#DVMj_}4jkJ0qf4zQ}>`bE>ZG678-y_0I zg06@tH?&6mNvwY7IodW1%%|3nv}Z#~&`-fYBa!igtxlP~V^=CN9hI z-@E}*$U29T_#w>vhqEEjON;7{`>E@RB-^wRcGIEOa|4c}dq>d2dVw4Zsk)2Y%YOI8 z**$IJ`*^9FHnOU0+!(k}nzdlSBK=>=j2Nvcm(KH6+~=|p0#&PUBC(h+4@2yXc&EDU z*+;3bRL{9_@E@%79SvPq%fmZyHvEy4nPw#o3Cz5=T6X z@vd%0#~MkI^w*`G-=zk)dUf3o(KISuINYaC71^Y;KJy)% zDK-qT4U?8MN*Ab>WVli|1yZ>G61XtiVo$mlgt{DM)YFAQ((y<*~0O%US|&3z3p z2<-H71;=G~SI!vd#Rpx`<{CzIIH|Oln{_K70kkCa89FQAY0|gnFDac<0Sq4;!S9sU zbKyDzb=hCOiTQ!1$lRPwd>g*}O*>iTXuy{;zpHl6ltF22p_0bqcPu^g zp%e|bxKYav`_qHr6+B8n`)`-S-P6saf|6`3ms9q>;sv)$ZD?Di>S#{9mcdbc&LoZ7 z!89?uRqM-(vpH2`M2hp#6~q3>rBxHSO*|~`oR7ib7x8|Q72`rT+YAd($8%h}_=kS-}{ zknZjnx_jvS4tG8G^ZlQ9E!G+sW=>qOuj}kz>~3m}218EeBKfdXk5iorIyU01r;PVL zfd0&vcDqZx)8uLla zs#>EEw=wM7zdn9*a4IUa*LDi_U*JyJ z_Tfo}M|LB%;!z^`Yq|gTmP4TBJ+L|yuYCFNPfO1s0k$6RHREl_xRwl}+0LMxr|0fu z3g%MJv?n(0SXino;#s!pAXYi!<4QZTSLlSSYqV~w6Y+TpZFQ&XKCe#~u=q#f)0zz8 zzUnl21gPWO@9^ROwD-bnge~KbFIQIIHlAWsIv?^89$n3yPP(T>)7>!0p76S#Cs8f~ zUPK)^jWe};zG7i_lY2y*#%~y3d;7^+Kxa9irQ7z{+8(X0mrv_Jr25ZM1jr*0c?HUS z)Tv~XKQy~8xSp&@rYTdhK1`PzY>Yii(rL6}(RRGqjro)l+cSDto~z`PQ>B$RJ=dB# zWm4MaOIncgZ!eDjv`acUfYdjVSAiU!{JEd;<&h8|-cs~R6EKp>{p2B(IkL3W`qp7< zSns{!k4j1J$r>{5mfUO8i}g*iM1r+6Z_WrzQVxj)7s(4C^mP+E_br5fx&hI27Nur3g9w86j4oltEFm1HYW z&gdeo>+2WkLU%~p5JW)oK;f`Wzd;EmkDOp1?#L(^bi}q8Rp6Rhzf?qZ4-Wx$j+9 zN2G0fysV=q@=SgfPmMglchFq3U2G5_FXbGr(0_ly;aDQA9v$|(HZ{8MH-HW+7wYr> zuiF+7d<`uBf@Wr0e-i;}MFdjT*=83E&7b!xluSe?CgM!W9$K$7YHp0INgBAKUc8JR zECeJ%LxuD~ZktGtcWF}f#ocL}q}X2jFtncKAhOz|7&xakJ!b8H!u>xj3y0tuDj?c%_gl*rctI6tBHd?@Z?Hd>Wx>yWw)RvAx_T4p?>GIRL}Ai#c>q zRBhAqMtyTD&g-m1#KVJlS1akfd=Qy_n+`pkl(TnVUwkt=o8rC3c`zjbVTJxq*z%zwfJ0lhcCwgoWWdj!yL)}e%3b($) zLlh`gav*qwlPxxkjNj@&4^*<>-ab9W8QYvgU)NjF5jY4?_upR0d~bL}aV?GQJpxYF z`>Q;c(gD2D{4Oj;sfmCx&g}Pz+$U<2+8P$>X}UiBTTKRqOr%hvGs;*7Bf$4&=;GkT zymQ%z_w%o|wgKpU(YMLZ&awg9Vx5pQ#Zkr?i;5Il!%7x!v_&xqNgA`uo(d_iDOrCW zNHpxrpy4%zlLLcLPfyQA6TPPfQR{8$RUN0zk`IP$%%`8(k$}nK-Xy06a$Pjko*db& zCyG)c)z>bq3g9z(HB&3Rxpg{sD%oAilD|-}G(DtTe(P9NIEb+lHkKPonZdAQQ*wq=^b@b4QFl3OX2&Ut{pG?Bl)ztxM%H>UwPIA5#jw{)>GwK_L z_sWg37!jjx9g4mDUBvQm58l%HWuq5$U({eg%w@QF0dylD1sCkhGK(<|0MG!keksdb zz}f|VAy@p06JC8_m6I{|w_DkwZLQ?FME+32;*=R#^#X^2+Mt;##o4hLBZ$n0i?ZqJ zf!Rs*y%B()JM%f*cy3Rxx>l`196;|RoTiN{T1hRz6?ovI8NJ+oK6eN7IpgBFEX5g8 z%P9agBd)au6ga>3VvLx<6;8L9=av-&=cY|T9YvOt(@WtH_XG@XBOQPcNw#@uTRmV` zKRrAHuqzGxRs zQ=^C1m4@4Pxtp0%vG0dX(i()xyhp>m?$@0G^hn^ri)^&{VIUI_l_lUa<_eY8oqRGM ztD`)-vHL);^Rpk&*o}B!bs(RPbmaf}NJAG7HIc9Z~+o585I zDf&H*3)duh)BmZ*Z?20hGVi1{09udOH3kG?VkWlJTo~~>cJPjSHt%TLdkxg6d2^)-Rz~Nr0i=*#a zJKP`Kq&-3gWK9oT^tw$^PhnM!?#^`viJKRE;dRcty?@RnxO1M^=|^o7q1EdO#vuhIepR)>D16)*hcDjtaBk+l#=FAhPou;1Oc$ zQJ}Xw@G#ODYQ}1M^kKHvJ{~wt%3Nr6@r@T#1c1aIs-=1qUI!hDL7YhWm)#$6x!qD; zIn8QtR%;RLoP?XCkEh;UweB)=u?R*EzkrbNCtB4m@`j%ORf#c zmNu7jwna%(2;VMugmJ&kKqr=T`SXBx5BKynuuWFuz4qrhzoBQpVR8ww2kC(BaD!g6 zjsJs0k1UB?7EwuaFyA4!&pm{XRwBRq=fmwC_w`cNkQo3mMUuSAKCKwyG1s|8#=K1$ z1xgEjdVjk&CTAg6vuH=+d42_VFgA$((DIzpJB<~vM=#Nn=Qkr$JbI3<-xf_!@x%|vM(|f&@DW!aU^|ad}9HZ&4G>nNO=sTLf zOgXjP){FUaT>$7b&&ES<^2)-?*b>-O%4Ay3sW?CZywNF@Yrol4B*9>Q(BXcsqe@j* z@PT1oGzd*~;FAtV*qJzKQ#>*$Ue-$0ael z?d=CbKK-JRnZUMv-;!ihY%tpR;JM{l5 z7dGop_;xx`rT}1RfJm@#x6z|TUZd^KDmOzR>-uz_B^CAd~7-Ewa<EilM65 z_Vg?y#g1SOGdU2Tw2a;yHk$9nJj!eV5v>&9G!yFwqcXD7KV||3|s&j@HI9KaeH}L7?bYWy~D~J1KaqjaiIXCl{%3qXjcWCU=b79GTLF*{{zKQ9BqT|iU%@$RSGr2;d64b42|x0uw*UFpvbfO-*FgPouu-{(n5@(@ZJPD75% zze4qPtM`3_MPjC^EU4MZ8=Eq>=kA$!FrE`XgN53O_LZoVzuh~G7k?NHbUxEo(TO>*Q}Iy_zb!7xZ- zt7f%AR31u>=~Y`^yyjHRBvh(0K&Vu%?|Sf6nw#5M7Jr!lv@p0MWPdrkSV=Pw*gL*_ zs%VPLt<0ny zX|Zt*P~fRxa>jEbjb35r)`Z&3=I{468V69O6e<@A>0d^~eaht)2Q&U@0}wx82gYrT?$L6oOO;R;aFOoBsaa_n?!8H4Rt zUk$ZmuCJj0TfEv&qFEKMUVza-kzBo>qh~5MN3Jo5D7QH(EysHM>2R}(L)Ezf06qPB z{voE-lG}-k4E*L9VanC4&Jbe|NC+}X-|zSqu?dXcl-KgK@#)!a60{;To?}EoXF5mj z4P!e<$inGMuOq+>d=Ekr=X){JjLBS3r_gr&15lKA-{Ls8RmG11voRCN>mZQGP^64U z{Ovvw>=&HdZi?J?P?|?*KD{RBt-6fR>`dpDb)v#twyiLZaAGLF>hdV8p(3=QYFA5E zzLCGL$p)Eq+f&`RaEjV}yrBh8PE(w0oc68$$FGyV41sgTGDOEMGVWX#95wZ>i>2zj z^QiB8dp;j?=rDn2n5|6EXhGcqVH*RGB&QkUWg~3>toz_fRs-Out}9<6`SyRV(-ytD z?NFna#Ux0#kTzlBkb+2X2f3vN^DP5l5n*IQ>9^IMPIT+?$jS{9U#&R+PeaRnrXCmR z90D{TtYx-J#UbuSM&V$S&LG7Sy$&C-kq4HOV!1t^f?;z~LOWF*sxA;p$d*n4`XJ5S z_t6!=?n7#4HpVCP`O0}Wes0{-7}}biX+On~ z@=;e4Xj%8b(@@2BP`!wQ(?Lu0H+T|R{)wClwmGo)Y&R%sLO-{%?7cECK)U+OSg!a8 z;x%R{QdNsrq>RP^wK@n$rZ;Lk&_4xNKF~~k^Umu#tYdC*XR3jbqD7k;8fQ=UTh1B} z0bymHXl>C_coV4nkfcm%ga837w~LJ5y$AyDfB}|PF2EcEwy(t+YO*2t6-NhwefMbBOk4F#EV!8qtj9KFt_;+5a&mD$-xA`X zVc?l7dSIbYyN)@x?63EkUa~|te3|Bt{!6Pn+Qxa`j*mumXQ6(x7VK~0UP;Mq1Q4eN zMAC2{TS!CAs9O9xBU3se?<<$Jf9#Leyb-v@mm?yU87Z>n~9fnjAlhf z)OUTYI2fXN7$v#jiL1wLoe?DOMD0#b#EDaZpyU*wYfM)!41Nxk-|QLijO3c^V;~Zf zBRL5<37-TA+FYyi|MPK}{Ifs6`36x7{S8&F=^^YGqO#d5q^vVrFpVDW&I-JxHPaf~ z)GyGiiNmnvWH5Nk=)G{!mtidVK+Ml6i}IMY&_Oy)WmIRIU0hpKbbi38b6=-DIPG#w zWrYe%&FoyGVmiZv|49t~*DnB9A4mf5S*qpzov8Ii`MN41#8J^_-=|KRx7huswNW=8 z!U@bx-N=eS__(O2H4cOdiR51UH)GSo}% zVMI^F1(=7KBn^BjqtNiWd|EVZ+70zIJ<9$+{QrL^WxznrYXMKP8~(3@Yf+?E^doYz zE`1p42yD%N{0@2ebCfRMgVJEGamr?~IifpfNF~>}V0inPc&C)&x| zI1VlG56A!e=sp5gkED9`yw&8tTKqXY{M1K#j)AuVfti24g^kkU<`ak-qKRe~Gt|5MOjttA_{PtJ>0h*o+W61YaVp&)bK_=jq z^@wt>A=%CP9%#3H9&|{IZNdw4uvYrt$Lod%6hWJChRFzjC5V5%`AocN=^1avUNJD^ z3YY7ua{8!_0|r0~jqkiey8D!NoU>g1eDFh!gGnh%n8VZ#?ZLA;9$Z z4>azJFkBeShB^cN&&zar@f8=jZYLiqSvrX6h+&gWi~29=*1u-K%%jp`FB@&<@6P!m zK;Fd}SAc==_ggcvHjm{X1Yq_*_KoCpeP6=6U&ea^Hm45f>?I z8S)#HKo9AqmReeU=y82v^m&$eA%MdewWXG{LdHM%kyG*CKfh3s9MW}N1m(=9%Sc0s z*}_zcHC7pj_rzWkBMN%$>fc)p-VgVtBA3MW zi?s`RmXJ7BKS!NvkDQ?W15Eg^PdnCpFQml>|E|0v+;wBJ|B zKN?a|@0_D3XDR+xEQF2jE$?Ephn=16^-|-M`X=_{)9J0Vc&yi>OVj@B8Q^FGV0epp zhAw0dlj;U| zbH@_3xyuJvO^;V{uvuXI={CzQJ5v{k_E?b9UBib3g&Z=kEgg;WT)h3S)&^nDb^YKV z%-mtlg(239vVdD28^UL}??n)iJ$~==_<1Fs1_9aX%OKKU-$yU@GRkk=7i;YSPpI7E zyF$Pb%j~kdyz=N_rXJIUCOBL7Fw%aIfFer-kz7*b<1403IZ*{U-ZP%F(~p}e+x^(~ zvR|KNetkp!)NYV-GnqwKdFbBR$;9M*|KQY}XVBcMVYc9)?)%yJYHO&ZF6Ju{A`lq87MCY$EP z6@|)pma@4u-P(~CFJEzk%_mMw>rO`(PkX3zN3(kr+X`6Dpxzg3sV|U+mkt9pMy58e z3nfaB5u1ob3a3giiYr9F8#}BU;g3?Kd3J#0P>bbWJ~EhDmkOdzla(lbSa zN9SMrZy)b;WOA`A_%0Tiy{Nk`klr2}YRdi=y*E972)F<(r5|q3wF+n6wS^nFP}O-5 ziu!Hhh{#cD+n*UJWs0EM^Ob5;amBuu>|?bWk0_}>kTQUML3Q%GENwbL_qm*n@51cqUp0-&Aq+ckjsABuG!psFct$zhtk_xPL%6-g{5+Gntzwecn8yhC` z-Y6#(0$lXasnhB%d;1tOpW)IDPO=Yf{r{$@LtKki|(zApf*o zzhd6mi{*%$%VtTyE!I;?*J=BSQjV070Sur_ct~Ya6{yx=LO16{$UJYK0GKCVV=sKp zqtOr{=L-9ElI3Evx3@t4cn7|&1L;JOLW<>mU>u!xoLh%a6i`PgyZ*3lV*0jg%N9Oo z^acOJp7+)iRBUVt>LGq}zAs(+Q*IlLkt2R*JYQ1R;}`?T6p!bMxMdQCKs_&RfiK>W z8H^PY%U>tvlC2sTCzrb(E;hpFe8Rurd!RFyZ%#d)q1PZ@+;~hHAznNT@Z8*;-z1E4 zM`f=m%JB)k=#<+qa4)Nsll{b+Q)EBH8BNH})q6a^pppD3#V`Qy92CLgcPTAb4-+Ta z{?-nIt)K>m2&1f_yr2%Pr94e zQ_TNFxy{*B{gu(=5=Huc%lkeBk6uIHgLAGr;6nU@qn5VH+8bz|uqROP@k+<`I(`}} zpNnRbdMzfMX7xOe=FE|J+ITUG^skd|4adh>DjnrQ;fBc^?ya2ks*CB)iE{ zE!JoQVZG73-?CbPtef~H;(1g&GGy4HF1~!9A7bgDd%Vt2D-RCc@&O2t!{ud2aS4&* ztcAE@{#=I8%csLN-kkTnJ2+aEbGnW3uHR*>{O0+Sd60-b;9|gO&xR+(X||`+gx#~V zX!unyzlKKzjz#1qKhgFex7Eh4Q73~63}{jNnA(LT7*iTx`&?D?rlt@-5o*aPbI>yo+?MlgHuP0L~IBWKGD4le^E*4q~zVoBFQy zb`;I*wQhH(+;d19zmEiYt<%27DG`)o&L%jHS|Pbazbj;#$HeLqE_K>to3fF!tZsg~ z!DNUOJ3z|LU!_!s1a?$sEaQ@!hol17$J(@ckxGIPs`QgN;Nlo{fjCilGE6zRz|S`M zw->-pR#<^GSf|lVhWa+!ZX{E5yUwEY1Ah4op4C&4P0>9#wVuJg<5jAWl7(fmx-iZk zocyMsY+cvZ!S`q&dusWZiZ}&rauEd-c?wZzvh}{;32y zZ~fFZu>jOKc^ibs*Hb&6gc#Z!f&*c(^{NkZMc>-5S&-nr=kNp1Zv_ulxTTmQYymYs z%T8<-^7IhRfv`Gv%dLy`o4YwD-1+v*xZ0IN#=s;z)M0~wH#GVqxcb<}Q_26@;r7@v zbO1zrufLiChRKs|nvLz#3304eDw+Z!mQVuj#o&Rn-KiYRJ^9P^c&Hf-f90HncVlsB zEmlR@TA0J~=so_7j<6Q;T)Ql0MLk!MOo`mRs2+Dg7Lb>V$mV zuTyHbJ1r#rxCFn;a4js+c}P3r*KyQ5!=)3+y7E~3N!jos$L(y#fthhBMzmd;4ymGj zgNT!Z*R3BZ={Fh}j5J*DwB3$I8Jqxfg~c7o#+WM?j?&~N8-jHjT)sJQIz-*<+pfNY zJOMd@`6BV9Ub1%$zC#PkDo`#=5cOo)!m_>?6~68898;RzI4#bS#~y zcL-@88UcF%X*7`CA5k|+mEwo}HY0-2W(!Zj^rL~GRxuELMfk!iLnQab9Bd_m`ILJO z+@`Mb4bkdnz6e<7vz~pxG#y&(4;e_00X-;4m-@AA|FiWzh=KcE;1L>u5#m>#LJTs# zVD^|VP_5UX?`a|jRqL14FE|@$PEq`H_HZ^)n?@1fOI|ADLWY{ zRvWB!FFRgS5ktca7t7Znu`-PKuoW3~a_a$h_l^#+DHk0cV*}f6+dW9JLSs_-OsAjn z%&FZBB|b;fjO((09P^-RS*dgD?~c8x1aJqtI~^lMNymk3v&M;d-;a1&#^&iu8YAJT zP>NT*`XtBVhcho0$rh?*qzg22tIKBstyK^Tr?ebVZ|tVYEMLphCP)$sr%HF2n6v?< zr6!oh(*?E9XF`8WsN9Z-ZqHgUnCK2N@*IA_8V?>|<}X?jjnju&p*79K(QC@T)v@6W zUd03fpSoJ{Oo^OTZBIJcD_l^3bzuIA!26+&8cP6wi>FJU1tHK zk%J!8=l8ffJc1pFSlpYcUmD)V;NiXN(?5sY=yX9m#OwqX!>m;fY}cC9z}4k6Rq7R; z4j;d7oPMW-7V&(Cx=E8&l=sA?ooF@1FZ2_gx0{$4EGq&tH;>J_9T&P6jGK6AybgFG zt~3)~zqm1`M_Lnw&6uQEc{m!lX26`#h-{6p4dy__w}@LfbWs9+TI67!_8vj`1{bCm z=#{psbUxkwW%Cie7)hZ)7B?W1@_~;1+5T&!=EgY7#lBUKmfz33QQNzJb%(KPuF}0C z-GA=9RV=ve3sV4pxgB5pnTi6)P|NUt|x+(Z@0$<)08 zmrL>&#Mfg5;TQRw-`a}lz=jFF`mJGG>1#fm#J6i|m1o2+t?eTy^ZWMa zk_SNeeNP}4iRH2jKbN99H^8JOe7fL}ieug5yQ2AMH6jzvw3+a?T{WQ;Q=cl1dVUN{ zdOA^tG#?BZ*?V0m7FbVqUd2Q2Mrwg|{q;Q{&4eg>J4~<69nX+_xX=To9$Vh**$t6d zj(Eos-J3S?$34NQzEu_fVm*1eM>pGl;fQk+P3qiU z7;~UD_T73!-n}n{XSnd90@(6M;J45DZ1n2I3m7>O%ArOiXKwLp0c`APdS-td)bRUmOJ^u?>Q^p~ zz)(lYEvNG4&N_=~>;q?)bg>H3%RkTd@0V_Dc#QN-lt?`J$Nw(bytYAdgJR(;U6pA! z$WG39-plgsWjN7rBJ7mOg@tdtM-b;iz$+=oQ(B{X8NLQrw={iMpqg)!0Vd?GB(qdA z-@F;WGV=(VZc?jF)=;o(7s?}1{h#M`M`HC!y-2Y59_;fb9c9g@ z<#)olMRQ5uPTE@nc*l9No~&F?RiqW{?ju5oUe#qajNO?6d3fl+@mcKh6w$$wbqX-} zKv;K)%?gb+lJ`$#d~*^T7hEg~hR!a7*yeLU5E8ivlii`vT7qICZI|W9UR5 zD`u>GM0-j7JbTMqWwB^U@Y*DQLE(mUMkU7hU?}Y5>P$SyAUwltwR2=|29~M4XFuk6 zx|)YtsftjM6o!HXn3e*LgH{>z5Re@mZ!S!^v_rf--JI&Lf&A~G#o>oN40nyq<1*_A zT==D~067)1o=+%&3fk}w{qHEr)5{#@t_}vkj*cakK|5INmJ7&8CR&$2Q8vQ$4e>l8 zP_3NxRQ>_@Y=g`4IrTvZ5Y=sFs$#ioX_4gr1)Qub=$=Po- zWcwN-r0e^45sB23Tt|t+DwJp=Xc)a>+|Ijx1FWt$d}_F{o2hIg??$gjekh~8*KGKJ zabuR9uiE6nxii{`=E>`#mPrcdPkdA>I3nhSh<>O4tl~978qrHjik8S&nE;^TmVSry z(MoJorS}7$Mzb-S<=jsK{C9u!&%c}Ck4e?Shd?M$6P|Cb^w_)PPmuH%7C3Q- zGT$gArzA7U{&pzX%+`Swzm_jlaoicN#r3@2#s|ESfus}Q-2KBDdi6eFL20rsBlwdj z`9@6Cd*p4n{T%K5$qhTwCuZY-vcu<1genR}^JhFpg{Xpw0t3HwBKD#zz$!myf+_Kd zz9t-S^(4{fMFV_luUfd~Vi=@KLB%1)d51++gS7LC+5!>s45JF&88Zd&VWY6Y9myxJ zVt{We7S~moUFI>d0&sHWD+wl9YPdzxftA%&`F&y+O)>74!A$Az$UNWz zB5Qs4m4b2tI3j>l){}|^qt&{qO1*`omrBo2lK!7&d5n86;pFxP#1L0pfe~9Kvl6AN-oOAm3ahEAZzukx&;2W2hS}U*w~L&V=824f!ZFl- zvBKSO9u@cAq9k_M+~{ObkS#1Y#OD9Gb5J zOi$rFo>f}yCn*6gFB)~=C;QVqQ(6v4FFlV>A~uD01d;Qy1+mMJiI*}skx74Li5>+q ze-^_)BMw6mp}eGXIWB|B6xCz@)drmz#JT|G@&7 zC|~10xSA(TBqo`piQJfB6=yuDyc#y2mEr#NY0xp+0&_5c2$!;D^NHi^7sRQ3-wFfT zSkm;caAZ7nSCL)z5AIFzCvZP2qCcoIn(cR4tj5A|WE{6#7DeHFIW;%^p@HbSu z)%W_}5zw{}tz+M$qvkBJq+3eieuMh*L7daA7Yxdn;6GK`zM?6S+CsRTPnRY7c0q+A zO7tw*6S1G?r28Ug-fx?u*B2{YHfVmMb<-f&RuKP%E57a~4}6-a5v>wAYBr;8gAab~ z-j`sw_epFt{}sspX|nU@!*F%Hbjs0x1bI$!@{-mir4T8_IfTZ%_iWj&L1JT>{;S*a zDj!L@Dz>;}TZv-#I9Rz?6PWCdY?M2X7D*hYGjU@p!V=6!NV9kuMd3?{dB%NB2sg#U zdcvc#w8XCVe2m;h(xO{#mLfj$6Nbs>eFkCTAEn-o4Q`3d1e}l7Iqv3p-DR0BH%zcP zosPHLqUD7jh~$fVE2XM^-u` zq*W~xq>X3Tcp}SWjS@M=QSIjNCb&DxB+StP8^&;{m5sX5O7}z0IUTaiCPvop zL+rsLGx@dzwZRttX&TOYT!`BiJXyw;#0h5RxL$_fRys?!T*}E=_(PA0uU+Pb$misN zyBNm%`R|682vbTlAwsSjg%d?G$e4S%xZkfFZ<#kF6=km^X)S*IXNmo%XMhp@JP25B zSC{0){+QWWud*$TGk_8FcxY97Lg14y~)Q5*HVl*7TT1~`!brv(f+luclv1AO} zx|pUE#f?)JA`Wn!q786d+GD0XzhMQ&P0O;lfN_IQyZXO(0q-cK2$YdJjjyD5P1Qa>HO2A?5eM#z3CoIKc8C?2Q{kK6$y_7!pp zCQOfSYb&j>nlS|5Q^;`QNoZ)Y{Ex-+F*cH?!#j{?DE-l7TEYT6kRC-xbD=JLAXAKW zo?A=-b^9Vc@>FKAdMy(-1|4WK$&nvZ4@5&jW=wfcy2Fv4-HfO{&TJc+N&%d9;Z(>= z&#z?O?8M7!LED1zXc1I^i*#|m-X!YjTx+=5L`LnmCo2@A?2i%2=JL6O-9V@z?)y&( zJ9coM1HtQ+23=06pE=%|eP)RYZ=voL4}IBDYky!+Ev&d?@4r!rNaFc5cMCOp>Fvv3 zE3$|lQZYah@fFz-cJMoUEU(H4rYD5*aD0wCPxnYzTB$y1(X1j|huD|wlLsl(hIj*U z*|e>WtfRe~+bf6ck`Vom%~40rAxm6=(?V8O!iT-trR) zAMF=@d#Tws7L7kxF+Z+dZ!=x@GNs=r{C!fnfF8Y)ud`WZ79r+NvRV7XkT1T839d^} zrVOKo4(U23))O345|8&U9Z%v#w_@eh;0avfjTH*_#_~{N)5*@8yKyO~)O%_8)wLhD zX}96q9r@w1e;%6U2isi3QYne1)Q&&reRM5jTrhHfcjMaH$IQCBv}c#9J4*Y1tRn&(z&b*T z$zNRj_d0U(L2?B=q#ksMM>6yS_gORZsoxN9 zZ${{wh52Nm!i|@>~z%0XKOW+0~L_t~{E%ze~5}=;iSypZH+- zX%~O(ol!m^3B(ig&UT4K!`4&>O0?$&m!l;J!Dk$j1>~qsdM{#EQlB&%|KFsudnVcpP0W?u_ISR5?XsdCQ~; zcEB4l?+(eO*3q6h3=834MTv`%7PAR(QVj zCre?ez88A_CW>6U-ohd5@AQfT8{L@Dn~Xo&7py6~^}3hvAv1xGKRVkw4>HcgT*J5nD7PDZ%F*5X}a_h zOYSeS-`W_LBIZ&btXjFdYgK!Wo|Fi_6njnPL8=HP+1m1-#!rYcemXe{nECy&Y5o7d zJnlS5k^dM%YtIl?EYX3%J@J!Bv1?D-w7s>)jBy`5FM?>LU%TGQ8nZ%tVqYSx`7`~q zcPHP%w$oGOzny&bPC`WYf^r~BrEdw#^7O7_$cP}7+$7VuZB?P1w8{Aw2@lP)KWWi; zSj|vTK{HQqMDVP7U{Z3-V}1UgOOrr`PxMBJl2c*z;qzwr-4rO~zcAC!g7~^DB(Bv& zq_#_p_d;#wt&ypKl?UR20dLVD?oLAZ718H(WcZiAvc#7>X?9u7-?$)dk_R)8ligyA z_!8D7@0^X&2OUdfe?x=&g=;l@hxENIzVLY&qBa*L3HnD#7DNlP!2iAIlcQWd9S*k| z33&vYqTk77VGw}=#R?32a2&49cujA=^ipQ5FDh_Zks+Cc*24*g?@>xw5Yl|zoCSM#Z5NVi<0l7n!V#1ASof<1o zQ_n{<$bOK|zsr?*K)38eC7*(g`Ov{e!wbm8UZghv@~72KjC`vU4OX*JB@K(#jW?pt z##}egnY+##uAIOJuDz0#wqwTj;KN0d#SJkGBKgK^M>FQRhQRQ&vi#@X&c}X>CSx3Y zX`2I@5q@09F;BB4R4$_HhvNn`&F(4@#}8UuS#Os#D=(fS58n}2Pjh`IvC-+3>9N%= z6mv{sapoiAhY$y4PU)d7O4l~oMs#+g;n4|DrOn4043Ls#CI=WuUX1GAgXa;}@@ygy1oL zBiMZ9t}rNVVaT4UG;KcAwG-H;WoYj1pYzKK9j~=Vo^7_~mr3FB2kJ^;YvlCn(TAG}zHnXCskKu!^~_xA zrDezjaEL^MuE5{2RMolu$XKH>`D11`=SQHaBJ~uF8m>UgR3TcG>GG?SK8EaXFLDzA zTbTh8?$pn+h~=(x)%zLBlX*I!v(=VH*S|ah8XPG->Q18;H=3*_i)Oy{t~H!fVE52w z4A9PdI;Hp7Cd8%~m0Ublt1>sxIo^^AC{`;+VB2&*0Eov%4sw2laI-i~R8ZR9yRSS3 zmf9%k5}*6-!x@GMg$UN85|*vl?KC^n&PoLRWtBk5cv4q380cg0{wINxduv(rukN%N?(1DYNKqQ9- zp0ys+aMA_Skxa@#mZx)j2`R6>cK+s5^gF&-h#YJISuZ98zoe0}ztS1$pKD>_VwmqU zBnggl1`rs{IE^ZELBM8}Z78pd37xmrU9}0s$YwzAB=H;bbo~~?VOvM&1Xn7bjs+UC zws}J1Q9Rq=DlIhO@WhVKd1pNJoovF&JnfIdmRyfR*bvBo8K~7_A9_p^6(_MV{fSa~ zG=^r(H2Mi-efj{fjZFGr%S<16^A2tM7Uni`d-dIP_B2riY&CZM3ed))%iuw-5}Cp8 zRZHkn003A90C9=KL8G;s$!3ElA4-ZY-KYzmjy=HZKFm5lzq6q^c9nT=T67-xur@sm`s@^D@1=Kc6 zFmMO7lo`ssjya#!gFEHS+SoMKkJ@_Mv*1<+?yu5c#mcS#Dy;6xs!SQw>@WGyOWlk@fCf31L7pgX@n=<<(%i7 z)MHI})bc(IO94_6G_12gm-Cks`BC`#U~Hkh9!mF;}=Jl2)ma{%3(S!P>l z%Nas%NbSK?*T!Skf-7bNoW^@Z0`X{B#J5lnSV}CiuZWquCh%?K2vi_b{v53 za3Gd+8xb9^!yj%0ufJH{--R=&fbZwDYY2n-0lrT+e&rG7H3M<~BS+$NLKXUR%vNEG z506$OCe`gk&j4oKoU0GGK}wr)h8ViX{3C!8Vh) zP4)rM{~in9oU|jL;v9oI9lBSUVnq?1@6U`0Puf1LhDaSK$jBeiMv!*}h7xU0>59Y3 zN$x%H+&lC@9+!+B#Zn!{P%DCry_jBx2pMNsxhZ&q$b%!>{)YgSE%G&`#~9UMAJmIT zW{zX&>9_%n8)epOI|@cdJN2c>YHeDUK~Qo6jzII~p?40MOETrx%^%aGnN;}-wYAV{ zOXR!jIWiRls}*^U<;!Z;y-nSa7lZo-8_oB&hbcUR{N*guB+t68?Yvot7ov+^D-0NqNWdk5fOd?6}{blq$t4BLE(azTOZb`OhgOBZV06zZa z+RrpF@TP&WnOK6wMKCDz^ty0p)FjPYlpoe$`c~V0VBI0YU(LUXMBm8Q49zL{+&EX< zyIS2(6g1k$$k)*|Kwj-?n8}q`%_Au?0#|2(T(_3e5w=9<<#T5=eptzg^R;ykaBalV zY}V1En-wTWGG*SL_LYnLO{V%^<9t+KG+dO4;tt(??K8w;(HekZ0uMl_5p;V`KV zArqDxrwh~g^AYpY&1}{=FGA37x}@*LMs@gaQC40f-_50XXDBq^3mSBWgd;^^ z%VYkKWkKP{QV`Fg8>RpB3+3))_pow=W9uz_$1*`H6XECclcF&tZbRZszpRi7TdR!K z@|W7(dOY$U58AvQ4Cq2h=|R#tHj;ktD5de(yQc9Zp*_b&M=y3JG zb1rv%cTvu2xFttygGQznD|*XjuEYXj)NOjRa6l1LI*6uX-CPW?ru?ixqs^pO$e|?M zBhz35#IODAKf6zNef?!3xWHtT67EweH3>KG)TyoC@}3MgZd?havVf812YFUs77U?% zYrZ%BK#y_NKzpuSpcHvfd*f8?3U^~nHTxlv^m-FIAcP5!0gO@9=NGy!jgd9gD;0h5 zwQpCOxVKqWAyup4UQ74QBj`=xjvf`u0Ffa=7@=49m;)lZ|62S9Am1_8YA54OI)s2J zT)pDpx&T^XCC=AKxYSEfyD<@(nxSBS78caxcEmfRdNsT${*L13l`yz^xtVJ_d(il} z@>m@9X1YugM0xu30fR|+c zm9f3pI)hi|bFzZVEK9mqhy7^uavb++ge`n7L}*07hCyhE=Z?_;R=THJ8M)@=P6@^L zzF3f-t+UiLjioxtya^)-&xPOfm$MvGKY7OVbZ=G07&N`PqHFSh7+W}Oe11mvgk@t4eWL|>{BL2-}=0>lNkGrZU)xVNA z#dF%XAvXZWoTVLlmA25W>s6u0yx=kB_(ws~{;uj!(JE%|vo%pr6Z|Q#jZv^&l%XpW z0~a;XX#M&?GLcS7zxjDY1r0f$eDC&?HPTIB)(O6iz2Tb(x9qa$|Dfa&9R>~%l0BrsKoB=PLJ zpM30`>6Efm?|&_LIMMK_GjL!93VJubI2SPuBO^K->? z#*Sd~h?{ebBXS|-=i8J}Ikn zrsASa$;G$pCjqDJ*9%W7Foa5(qd$K9D*i`Qj#NT=7|*Xa{EA5_u>)AUj{U)soXFAJ zf>YW2p9Uqrv6&wgn9n3On2p7Cb7Td^3n8y9t>!29>0h`xdCVo$bj-~83^lWP?9bu3 zy$&+)4mF=J#cHM$_4{yx^|gWyJO1;OcjjG=bZq9Rk<@YWD{GcYgx5#;!%s1G#5-Lz z^Awy|bRLq}lM0&7m58E@yK?f2u+bP_L+aobRCr1M3ACt1e%_;*y;l5$d9$4|QHlpOa~iHL7e#q}lN=SvY9$3F z5sCA|oS^l$m5zb1T1O(#tDv_hZSu7$J^vqLZ{Zg8+pT@mAp#;O%^)Bl-93cT9g2v6 zNJ~j~OGphJ4kbuScXxMp#|+XibiH4^?|bikJp1=N$NLA&F*8@LYn|u%EYG#6o?AW> zrHZ8Fj7?c4udi`1>RP0U+3Fa*FYjM`Slks~?y|vn^qAVd-xk@lsLw6r8FQcg`gS+vlG3Tz}^>yj1%$bB9;#D`wrH2YsclMfqb`CMZwm zN$}vfoH&cz<_gb~NubqbgD509GnDW)96Rh@Em=B=R=*pk0A3H4IV_X!>s3yLKPvoS z6y(b9N?-fY+p)$4XwNvNebOgGsl|_h+0C9Bty`JcS@!cTXAd5;Fxe-vLO1JZ>c^%I z8&hzhp~q|Hvz!bzL2~DMoMyQ{4%uD`RvA;3a}Je#5`T`|Yu|5;V^hf7hv()Beg?G? z7)Qr+B~0BVT5g#Zt4+Xy=6`L`Swb?itU8<$!j|t5v9L4|gND;Jy{MgzhiQ<5>tA^k zM)QkqSuC^6v)Eh!b=}%YWxEQK%?M-Xu=i8vlGU8pYUw@^=3%cX zkMf(a(sm%Kt`J?e>;R?I+3>-Vbe*2|>vUUQW^N9OmIWL0V&-}wZP&Xf!}%^9+l`udk{A~GDuePc-v(@(%d)tx^QTUVzq}TODhJ7b zZ_f6G3{vRizb|d@Mp~2(dK>gNW#g@khLVg5{!B1A@I`(P=VQmRI83YN1P?I~vlFZ| zI7zNURVWWZ5s74sR$+95?0?FMNeB`V{gaNw(?Db;PSx?+%Z2et-?xlzJ)9R}H$?A# zGDwJQY=<{HW2U%^2nde|k05yk)Wel!nS2`V>$3=8HZr!bSkXWq#_2}RFjJi1W)F6e%f(#1{ z+DMESDrJODC{NGwV;TD6JvAqv{sPfi$hM^sD(KmpV_mR1hlwZ>>L((OuWR^~!7}?h z{+Ox>~zDuJuWDCCUHwl`5rdENq@XN1G>Ye!{{UIwrL}nH~2c!UHwMIlWu& ztE~YCk}ZB~&KvDGPLk6WOx5R5H0l-@Et%KN0y;`W(h4C{*9)`mx?Y%V%|k=#t)_eD z8MWwY#|5ux)SEotuf^C-&gl%;>7kXrFw0L}2Hmp4|K*(dpHfVuH(>E_yQ3uEO9Fl_ zg)LU_&N#zm@QV=K^aGqq)WvzWk9B;xTIW&ShL~KEguvQ3BC0ImCiO5O0=f~|ef=6* z->|4JfS3O4Arou|c~|)E`;bVEup@jTkc>aiMo%a0i^VbE>^_MfnGwq#IbV)QpXDZ| z)9k$Xj0R{95=XcWxakby`6yB=94~wd@^V~3rObAy@EPCKf)h@ZemIQ_Ig!v&h~put zdNU=Q&=u1y@ms-T2cK#Q$`mY~de#o~L;^S_`o133u|S`YlFTI*AO|B8=^j$4M#7Ne zbYXz|TZyV7_5%-S=$8q~3lGqZcnB2`srhEoFFJ{5h0=j)X_DbV-SBc*Ro0 zA_Sy;(tJr7)Bh@W|J!mUD=v--$BxYXmwu&4@OAOTmdvB$NhiigOkgu^ z0P9%Oj^E>PE*`30qO*342PF?HdnDxP%86gb;Kk87#!WkdZp=?$;__PAdC8tq*HPDo zhx`W~)djcc`Nn*O+h*Mis(_k}9cp;%J|04*wrM7;c`N6no1h!(kedS)J@ysgESSUU z&?7Z`<=(-zbJNDWKx}&W$a%oYOzDDc(S2l13diiI4al?(o4t0U6!v#UHXNBIImSg)1G1a2=d0I7 zBDn)bl0RR60CS}T&mccmXfc%@e@$Su#g5<-#A|r&;`-jr#jFvd68_!v;Ai$Aq-@D| z5Lz|X=nR%?4yQQS^;thb0vvNs-i(s=Iyd{K()qAGLs2PjdCZgfFx_L4;zNpQ8wB;g zPYOPw-(k3IzWPQ92a|nN!$TgfJ0(BNNCQ)d>&qTA9tox0d!)*s4lL+#Q5DKoYM!)ahV7~4$MJcF-7V_9!6WBt&!dYMCH1#B;BaGvT-P^Ok%r1cbe78 zg_;Fh=C#E7uA!;ktGF-$OhDkFlAv1ESn_4tM34No!K}!bh4Oz&7XQ*QsGkG>r90zi z^WPJ$hXncP?2_(ZJ7|c1#Q9SA`OoKgQtX=^>*C>2?aP*CieAad{idnIcs5S5{RIN0 z#DaY?l#@{yVHT=mc{>IYccY$Bqt}ntI|4j3*IScwkY@=_satn&f-C zd7iDNV16FWS`m7GJt;%}IcejXVd4$j1y*N(A8#U3-8nhjR(PcLWi0=LavkXpQRPW$ z@Mm;<&p0B!4jf(ICv=fk60yOiAviVH^7boOeT`hWZmvDLbJsU|h>qriAX27rR7||Y z`EsB-AyMeS>yYzp{(IjcfkM+hA$6u2$9cXl?l(*TqP0~sq&v1Z;}~0bJtNzLO z<$ACo+68H%FUrFe`42La?_VJoGl=W2%w!beQ=0ULu`;3rgGl^mqTX@*O8%W5DNU;hNsS26-_gjV7 z$_W4UO( z;DmeMVvX^zQ^1#>h_b`QM+cx< z&x`I~=L!WN2}Ar}lm49_F2HzD0PdxA(uNVEz)diP_Kr_M@^ZGI+Cxmx!bWNSC7{;G zd;|k-nDS{rc5pvF-J~=_;s8{4{!a&CI2UgoQLmv~wagOVyLi|)M78@OKgtB;1Wt#! zj#bSDmm?Cnn_+Yu;?eH3(;QxlDb;Sgz3ph6VTLGO`&*@oFQ;Teji&f+a9{WBXy$v! zpUFZ6T*w#k6JnNMsiJk`vyIC9eN#3w;s;IlrBT&iTzPcePoHT9H!IyC%GuR{#$D#~ zXL}P>j?6-U3U(AOQxinx`2&2DE7p$WUlPe01<}1oyufu(W2F7x7vcZ#2LIQ@+XCi9 zQ9L~3ziN075wh)_tGaqr%y*=#V1IxEDsb^p=5t5+VA*@YISIjxdN)ziZz}k8iz8m1 z(yD1*x=M!)4oAm44t+Tl);sc=ir?ew^HZ6$YLv0(IPF;Xbj$qD*p{Nyb!}cCsO+XW zA6XDmPJUH|DXxzqze+b7?|+>tW<~>?ai))Oj*A0&f0K*cQ0Bj_7YYl`bcx$#Vn};) z@45d{=me2Ek>Ofi!!m)g^PGjyse5!o#2~q`Jc)HiXxxxz6#+-t*CztQpF!8ArBF|BYS;^ug*} ze-E*_ItK!`N;zYqtR@O^l`~uiA|j*3q`}5i8QV&tKkV&nq_+K^1)tJWOZ+C;e{ z%5OaS>R~uSm`hnL#BdWeCtu4S!t2hOa_OPZ*?;Q6^xgo*u8qabaCrTT*Gh-if0)@b zcdnP@fdb{se)if(>&8ory2G*p^Q*AgiJdSFprY9k(VbX|xa>YqtLH0(SCqSH02NFC z;qH6W%^z~1B3<%7=Y2SI^;4$9Nism@B|Ra{`$wGvnDZw-jmuv6`S!M{-qD}}J!09H zjn@HLhiidH<)4Io>W2-FACxxr*E;V*@HOSw_UlD3WfasG$!*Lg-QM;h?)GNzc+P3o zmhEoy-pd{O(`JDqTJLTZECid`F<WUQ}1jQTL7C@)Hw-#)5Wqw@H}V|H^J z5foPcecv^g@cdJdD0945$4TA_ittYo5w&%b)4coY;<6U8)wg93(HkUQ%cURsOxFUs zGuI#Q+stJ_wfC-9YwHIz-Ar0xbDhn#(C-a~rfd8oS!#Lzdq0TFWA*g@i4pl66i!Uz zliJHaviW$wZCUi7uh=r7rKX|DQ{X_PxO-Wm`A;3Wtl#$74-ax(lt0l+n_Be>f>8>d z?5y*(P6!>?<|Y4eR?)6VLb@C0CB2&iaoq;BL}mh{$DfTlvU>ywkIG#z-4^&qYH@iO zh?uI{BR4fd_A}i9V-eOTnBt%;cN1TeEme^4VZ?|eQ1!RF-Q4;M?dnFMc+8%?Nq(Fm zBKzsfka2&=4HSHb!1=-JC?a|RZzj}R)oXZf3vhmefZBJeFQZpzb$4?GuG3f84y`Y> zfQW8cqS9hABP|HBL476uL(q{$*Jb-Hpq;akfe|{dR9i0N-6R>_9}ZoojWyrh{?>mN z_Lz1xnC{JfGpg=!7R!I)(#W(XNflD#f4;SmiGcvC{ym)iUsuK(agoHoSH?MTW$fcQ zz*UnNdf+yEMi5aMFYLb`!0e ztZV*MWcZfIhTrLTyFLC=T1G78BOT+3tN!D8LWD*1j~to zLFYWo1Nr;a?zCLH%|3_z@agf3?=f(oR6AF9e#&8AuyH`ShPqQ9K-b~l+>&yql1x)S zqKsyT73_;+(#SvaZ4dWJj!PwC3>g+jTD-&%+L{bzLyP5n?W5wW=co%J6WnF^(C`&m zw5Qm~`vRQ09Oh=dJOSeyNgk2Schbh0Loj<^CpSB@F*(6%K{k)Hul)Ce>x`DgWoi$J zd`(G^z~bCrF~9!HgMxAx-%edc>VRx_DD&i*eVL$HiY4Y1D!!=nvX|I5`iQ##{B{-q zuc)#OV8QuX4;WIt*k1ThUE?C~bd`SsUF1f9%x@-`e^K5v#kCH*4K<_wpdqEG?hND0 zLdgr!`vJly*JIOHjoroWFe!VzhQ@{*r!q>ox+X`k0yawT>9el48h@>-|2q^5mI7XH zrDkxOf6b_*xQO?^W;CJUY29nzP{sjZr_963PCU-~db@y*?PHtuu<7%ir|WD3trAx9 zR2k8_`6RECscRAAwgO!k!<)`aXrrDSgYW5_YHGUOM+{7#NM-h5f)xEfyvE&tp-VxJ z8i&)3EvLfKf4vOS5~RVSu;a8WuA{}Fny)=rHPlgV5j=V#mO`#(Tr3F zET29~j7K~pJQov8``%wkQPO19P|_lhmujp0DWs(iF`?KgTz96gw9CHPO#T$RRn*d- zPPlXGw(?jMiP&#CNYjijhAt!Szv;PL^14+n>mXX^Z*w;}S#zgdiH|In$WO@i?+hJY z_;~dzdBSTV7UB8*?3@ya1buh^{6mzgZjgIVLz`$-gbl;{ixpaxQ5leTuT1Inp8Y-0 zq}>_MSs2-2JC1_)|5CQ#I{{d}+`b5yGyPo!(bUwbDt}*n*Eq-y&X{g)L!#KR{oa>L zwREm=*D0o8Q!j7Pebz5p&E$jc!GZdV`vONw($Qk2zQ=8j3XvZ76mouBG|a8Y2&Ds7 zzZq45vKQZ8PXGY`l@b<1M*#6iU^-c7YJyB{`)glSdLq#w{pUZmBn_S!eHTG_hNU9l zvE2MonSap>J^$s15i)`u?3yN2C6=nMaQ#U1av5gc12~BRgU9x&x2Uxqv#_J&;rS=i z!9*3}6h{R~cR}o%W?#0-j?e#jVo{5E2BdQJJ){SQuzHf=a#T_Mb2ZI)BvJo0Q?aFq;ha?;a-}Xa~|NyspkEnR=D|I)_7u!V2>V_29MKZM`rw4|<*) zn{?s9niD~nfn6*Cs7k6pi(pN!A07Ka{gW2fz%{wNapY>(!^_0aA4fXhEOkyx;Ck3y zbrsxa#nfGTyMLT>yPV{|Am6YVk1y02v_4KqyJvs91$?zmli!`)^)g^4ljGT{HgMR( ztp-eIAWt|5v0k!@XlaLx9&a)ao)3fEH_|35GE%(0WSY)tnB=UI9iMuWo-@dOI*zZ7=#N9H+63=t<$Zv$$hf%L*S)F;$3-Y|4#WK z1XRg1@l>$-|XC@w&8^LV-b;J$t!#Ow$bC(U)bW9~PLCv4#)RR3WC zNHq|OzJv6xh>hS=RRFy(8!U6?Dw6y+r7RLSKc9R#?}T1DztI?IND&Y#-}-f_>TkC~ z`2*yfZa*ANzY;3>aQ!P9er%~HD>0ml{p4Y9B_o9QSe0aR^9AH5EY(v;^tQvF^XYvW z(AG2yu&-R3W-|`H=?2QU^K`FMJ&1%a)?6^s!sdY3k%Yl8U+$l#Se!r>7r38u0%PKV zNd_|0*eGkW|C!~7lwR0Q&8b<0p>cQZuDG|`$e!&n5Zb?S&gwSaXhGZKkg1%Z{1cfa zxk?hCyP}bq`|Lzs#d;!C{R!mmg&6RDb7t%!oNLXIRO_}t-ndNroII*=XqqzZPvst> zpG6gE$Ls^XfTnGy3a2X0$5{g=021c)qK99~B_qD|`dB#>FkNZ| zh6HQn_A?xi%=+b$x%$`K=>g6rA$H)5az1h#01n?Ek85m7M@`Y(J}ty7u^vqK@W5yy zXN3};p}fy1!X)8y^Z!V`y?ICi6FvG{nG*7ld{Y?z?`0ALfZ#?S%P`FmFr>r#))u#h zNmnWEFs^`ILssiS?75%cE6wJe+~9nnH8j+oPVE{gGAEmQ=W}aD^dEOH+C>BK93r*) zsz;X1FcQ~*9$O}-LouERzWfs)9#{KJ-1}y98lLIgyyk>nEh~wH>G7Bs&LzTI4pqzi zjmPupK9jTdczl#FbFf$CN&1IR7>^j=0_*t~R*%Of5a2$nXV$;#xnG+p>vLXg99nJr z%X^(_FI0`!P5##Exky~qVON1=(&G|6tundhk@vsG4JHhJkPDCaGW@_^d!drHcE=cn zf^~prS8flYO~Gqw;SFw+-S8i#oy&ALb=%v_InJ$|fF^MQl_;J|Z#~{0OUibU3t!+W zimZ(G+o18Y7b7DkN~fa#G4b#}E1Pc237$aG{`+bc79MeSjs4ZNRu8+sJ#!*t>~*p#CL(H#>Dawv@Ml;Kj{rgJ*8BJ+!L6wkT;}O7Iq8zG^AOk3Pb=g6ql(qoa_D z`7uhk`RnXwM7|#-sqa3JS#{7ly@y2N(E2nPwTk5A6GeUB=lIoe`RwG&zDD=Qfq7-5 z+c^5TOvVKscUGlX-NUD&2wZ`35jLc~O*LG*LxR2zx?D_ew9(}5;m&DJLXQw0`W4Fv ztV0ZEj6R2!&hvQaJ;eH6A|!%5wm{#|-TeQBQW6asSxap0L?)s_4(t?|wjh9PQz41sl>e zdFK8;tIBFTvYLJ}Cckl`7JWp!c zy>)AOqRhPh&xzzLw|0aFZoTA!fRC$(jO9T(E%!+9L6J9YUs z+rb<_7a_$h>L$#Y>dMMB`wqPgOXIwi>INnV(t>2tM#{*P?GLO5xIiLx3WnXOf#hMRl8X2ovhCuJ( z{v2g37{)Q=Y26fiXV?Aqw}{Qyxxe&CGsunivppRLkw4*ATc&enE^8n`P3WASuw}pb zWO!P5BbCkxFUGBjiK9K1tKCsh{I97);KBvscsq1A*uaW#zJ$yfQ<5eEF(f8 zRG=T{*(+HofRi!^OI_%eVIU$p`_kosCU`;ep-Ry_)$n|Ts+QqBdMNKM2qj?FbN>=pQ>!_iMqE97+BkE*2<@{Z_EMCP>Ofy;>agA4c^`t z7H{`w8-j+3UAypYDlJafnC}XJKXNFLz&HI$=fQ-U)a-ODTD=ex1duP{4W+jzYQghm zvlqJ;BBjRmwoNK59kBG|VSuZIE?bmy9A-1Q<*}O+Ya8$CbbTfkHMk3nGxdNr&D!lQ zFVzpHijg&G>wqs7jx|E=TT^ZL)*w|IIRi>iXIl{dhs5f=Yak)J|7n)L-xd>KKha&8 zaMzdhRjQ<3{=Ui#Eff1u%xaM4-$AnuDQdYhORXln9lh+_q5*#U^wu$bM52w=R^n?l zud{OF@~~e=v$hm!sVY`~P&s&a~nl5j(b!n}Ui#M+whXN$_ z=ZG%o!T7FVR0`<}>7oyn`4b0~rrTKwOX(o58^Le%)qjL-niUPZ71PhfE+!Vl>jjf; z>vqR*07T$BM3>IN_-}1{E{&%e1$LYAh%+err~>&GbSRTB!p5}AwQFG3U{WpM(dM@Qxn3r*{vVlT{7WXF%HdE5#6O-rlz#4#Tx~T^ z^#O+7IGp`peD|_xkQp}}cvPr@a+BtP$QG=^^cLfJ_(Wg7$_b}H!jkvB#$1`Bg;@&F z!XXr>dwp|z)u=F@J^Z*(g+7xV>^>cKQtj`J^#GDXw8ik)ETM1@LHTX43r}{-m2))Q z9M)~U60#4E2>HJIuNAP1r4#Gz%6#X;6cB&7kiA4tJK8IZdbzHS{Cgb+iu8y~FzaR* zx=)j=iZb>J%Hb#a`nJk9s&PI=5mp3L8XFFwvTxN@6jeq@niwSl~F zHoqo}$cj<^V`-Vn>s|znS}nHJV?6rPKb&Yw1^%oV_@U!f_Y1a1cXsw5ZfD#pCoHQu zIv95sZ{peg+1oD*6<%F?IUm$eoM35F34p(K86Kn3{A;y+yj;)>^XkE29I=frb`>^N zqrSnQtc6vV-?nBTMsa=ieT z&upZegEJ-zrmy1VWM=X-+<_);{czqY*WDTHZB1Mj;V&EKh&G(!QDS{B!uRwkVVL)q z&L>Z9hU!7%KzYV^-XEQ+*Fo#K!;V_x7RAj)v|y zFW*Dz6-N$$9w8Y(InO2VWsE?#`qRdEJD?>z$L%v&IpU3vEpRX}Spf+COY3HzE zuA0v=+a**lVAMXTKz2UEqNsKPEe!*gLegTY(4LIoS+9ytPV(^b)*#tEQrDu-(8cTQm7P3+SvrB~LIs(t9V`Aw3hy9V&n0ha(W8k_53v%~!mYI1 z14G2P3hnka$SXw1tqpAiz;n5A4R0QJya`QUQ7T~f3V`xiDFVbkNRp{c*udr8j7pH! z*<=D}jDPuYHon+hvcKOd>JtA6g-)Y1uiMTZzWHif=hD{L7CEm_$lY!6_4cOL(yvuo zFQxh>{eZi)W*Prxa(TODBSW?8aQ=_49luO}hKd1Q37=o`CdL1j@OMLOtmBhIy{YaL zJ~;Y6@;>XG=ZW<+0C`-v_MjUn<4o&2zXY6-;Kts&0su&d!3!;GuwI-)?2bc56-0&r zSX`1_qY=8;-68H((m|!z%^^24RQ~SZM}MANOrXc0ciL9j0kvhmZasbD;R2z^kj)50 zEF7y~XyX2XZ3#4(&%wW#sVeo+C}NhWHZ!@2wP(37vCu4*21t?mXBthd-RobO`I^s% zm=WEeb+{*$sqI*hQo_k;t>cd2fWW-VQsIY6uEyB|^!4T|gIr2)#Jo+DS?cK0{R%f8 z-M8~WJvTWUp(4E6#MRNINIL*96k(6P2!>G{$@Jf5gd-k&rOl-slpWsB_)Y;ng6UZmK~Faq z?+P@p%{dyre%?uGx1==At%20;rdbuU{`kW?xc!izzcM&4^r%&(@{od#ToVIv@1y5& zIO+O7jMI4toZhNXlQG1$F3?UI5~ z1{sugJ4^B&AIs|BZafG742SXsymRf^&H*}VAb7wRaxpnTAW4qf1iC@LUeN~@vllTy z5_P%ClQEy{n?wM;fJQLRMgx^{&UTeysrK1Nvr5TCH@gX6gv&mic7rfRPDFl;N5tLU(j`#-urI7V-cEVDy4YaK62Bmmyw%8m{9TSBb^ zG01Jq?FiI0SuEmwuq#eJMKDjtG}@xV2B}f4J=5ix25B_(896bwt+InjH=M{R{dd_) z`V;Q@QhQf)&;<@LTiJ2?Y-9>m*{k;`2|BE3tj1E((`*c6IZ?O_1v3kAlrIWq|1%P?B^kHOu!V7zmSP&ieboX`3*l=#DJ00W*60OTwk1deS|uVet+_yrPU_H_B?_xn=yI*OQ? z%Q%B9n{XRlWkvKQmv=$ZD9O zyBTc!y%L9X3Y6^$!YQ$-FJ_YhcM~(A(70s4- zzTuyCwAf+F@~pKkX+5ws0S2Qb?6LTWg_!|nz*`&tTU}*J7=C(Cs8KGz*{50P_O_{& z7`o}(r9#W&E6qdWLon_$!(u;VNebhoKzrsW(VNNk^F3X&)Jb-ifl*hIRRq1wElE=+ zXE=NhlEzg)nz?De+AZqs>4$+&p{#p3#}(S`iVG>H^Z%^q-NG%hZ`*Pyu`F&gU4>hW zv1FWFRaVc)cKP`abJ)9rAww4BQBrP`lLeByEj^iWCm92&?J2-6lfJ z?C`g(K==3;wfC`|R(w=u9aZ);eC(A*_VrbT7a~i;I89qRwjq5K^bsrOyW*wBCqPXW z_c*zX+H42VoXzg{F@M&o|sut=GS}HF;@Ux5_2$iW58Z9iof}%1rXx zXUy#K0_0}w8E2s*G%u=8P^}^ZPh})hTEH3OZEAwwQU78-S(iFAo=tl{d6;C%fzju) zf8SvQ7&}AZP?>7Z>Q}@bgkNHr<9t3Rbu8TN_}h<7C?N5I*%`jXHZI2V~X{6&t22f=n~&A}bVco8!P}p4S&9=6 z6FKPfb0^2i+u+z90=|({7KpxQ77~d|dUzeH0A!%$WB5drXZh}Q$tG0vyQdG(ifPSo zKEo#dAf-dayc?t&b{j%T31z)x>YQdi|HK8EJsj*4jj@z{3Tg;X&uWNL^Vlj29-V3g zny?sKmuy1f1LWhF-|~3L11A3=S;2d{9`^Gme9`L=Js=u0bm)I;#r_sme-m5Wnv#J@y-Kz zU?jOZInO6^)ly7`k_}%{vy*!&Z~8Q#WdlzSL6Ec8OQWMc6Fq9qG~$)n67A=lQ@`K( zTcuiU3g#kRXR3%529>(H`r$7PXCxcjL1?FAFDaZ$j-Kazuq>WoC0Dl+x$fPfKeFf% zVAoH*gVEnUi5$c^W}(JmwQ4ZWF&X@$2^lso)gB@xm%Gu50mi6zYaL?IPhJSTkcQuF zH+2pUQ)#;}PBSkf)#;opX$e%6=bTbV9qMFIJBhfu_QYs7 zeY2AiDCv}k1Ev17^rZOO9 z7D<7er2pWC0-{K*P>d146$*M=_2&CY0e7*a*s;C8kkuzJo*5_o%0jk-!J`{Med<)3 zG8Y;mAs4*Bl8WI%Q{Tr@1oEMj4GOv*0G=dAmXx3HQ&$S%1&IiOk93Qm#;aF@;u~!) ziJ$nz%j8`RFuk)S^rD(QN|)Jm3^mtQH-kk*kDj^pF&6x@l~@39lITLv{;EZiFR1lo z|1)k+#DV-j{t=$T*`A=VEe+n=^}Sd(PLZlVK5a3NmYDFQtYH~;nRrp7v~*&R!Yqh) zl}gfZ9ep3$=!eyj+&dX1Kdtw-p3R7GCGqC#4l{fz@|3L0rTxj_CFEV+yQuw!b2p*A zfuaRLVe^|rbHr|p&FF;O%?}>?k0p9XKglUHG6jcG%LSsi%58 z$l26_D0V3B(J~ul`|J~d=6gU6FC9ni3&mL0m?aX&jl}@{Ocduh0%#@DWD!F<{{`ZZ z1qfr3#e8`btO>n%2(!%|{B&eNK<5Mc%ctp3c??Y`BErh~#rO1DYPF;}QO&wgJwk}2 zwc9{{j^k0ZD;_QVe6LE39J1O+dR~L>@EhZTJn`{COZW~4-T^gu0MFt5rfR9&|Bc!~ z28cE)xD2dWe{m;BC~J5KIjYijw{IVDkr5@!;^*5EKWMtuLgc?jx>05W+?O>b`PrZF znDhkaB9#>b|299`ptdgvlJ!sm^fWT5!U^-8szLaGeVs=4kX4@aU_M;B&tW}qkY@|h zi?~fE;91KVWtzregHvY+kN6XeQAknS5Xd&9{#embWLS8*&u#=HL+S7V6Wbv54}sGHkBlAe6RVjO_3@R{iff`ac*Qz71~xW!m+7Rn^0PzxT%qo*o69UwAtNc9j=M zbskWHqM7%PZq@ZSWIp?e!i6t18ULWo03 z#;8h7{~4)X_gi=smV0EguE$}EZ_RtM;}~m>jVs$ra0B2CIQzFZ;4pQAp!WAQ4sQlB z;?iFG+a+VP1wBe0VU+)Huq5tIDD)G}v24s}3&TbG8=!ngWv-6F798#%hVe3e^EH9^ zkYIY)Q!Ea0s+!l_7Xl)vhl6?kEu@<*AT^%53n?T}H~Yna*+$7`aH9sO;kE683B?HJ zkl+e19!OdLXtbtge~=~gG77C8z)$zoC~K!D|6U5LV4tRb8MQANF6VhQN*d1Z+tnx) zr9caF--$$tc7A=lQ|x6CANFTtkpw?v!71I)ma+^YFvGmVP*cc{Ggdc86K*7Y(cDnM zqWORZkfsiRGij;0BmR|^mkbA7WB!!zMBa2}Km>a1IQJE1bGZMQrONe#xDTn_VH44$ zHa@JH>Fu{>U%g)Q!22a0cS?X8*=>a@d>LY{h9_LhI!<+=Xm)W&Mq;>& z0Wek0>hYXRKQ?+C+9F@4xQ+HWg^AXRrIK_jGIdd&0y07T@gWNz+``q;?I7u{hJ%rsGc+j z;^zvpv$L8dpCtCDUd)v{>sMfn_#%05NPQw(M*^FE`NCms-9jedXUDGC4rG)m%sPU1 zS*j{kCARQJmq>JIXD{P>E{sfx(*cab2!KnXy$u6lv($tkb$;CptFK?-vYWx!RXtzo zTDwEDJM`rlGhFmdwZZ4yOZ?3SpA(DDYEkfRn_|Nq#$UN+e%BnT&~--fg+@#!FOZi-n(xwEB*DBBAST@Cn7Kx zefK++TpysTkuIXV5;ckYH7^(Rdm;cZ>5MNPWuE!~8zcS_eW7~gBga23e!eRO#Z6hL z6dccPmF-Y7u++NM&=9rKfPt$NHH8y_cHw>w4&ASZ;mwuU(w+)zirnKM*HOP-al#s- z^^>n27F(`7jZKAd582(KhOjkRe8+#*yv7 ze7KP}sN{G?SCcYL`Tr0EruaeNy@KeKfj@eNaYOjl+;K34?6}PY>t5mriSRT@sL+>J zoitvJt4|$@=8m|R3!l+D%TuhrJC!tbNAlI-iy<@wllI?lDK)EW&8&D7z zdou4EVUwOo0Y`_WDV;C&BO`Fhw>MH@D5BXlRP^c}bDx;;iLA%Z3-{PJfVYs;4}gdn zlG^Z|T7~e5Oy&yZqXyd>KjN!2Z?P6+M2vvX9rvR9a^_=y{&Yw?$nFpi-Fmd2Gh$tn zsV?kzW&Bqis;E*+Ma@4fvk;$W58Ov|>9-lcA?~e**1`O5_k}7g*oO+>XVfUD4Le4* zQKrCil^tT+B>rT5t-sQ!U}{Y8q6We)`wZ<}7Iw{=NTw?G<5v^!^(eNt8K`5ir7V&@ ze7kDpnc8Y}Kf@pn>m^Y9wRGosy*<+E-vtTZeZ$OomqdY2;!Huj5(9Cz{yEtQ`|fb} zGG{y?%|oW$>*5(bOxzbM#CQ0cqzp}A)MPA~v12;&HArb7ESYlb^GEq7vPH!lML!<= z>5-3-({{f74IIqEFh(?Hd?U2c99yUJbYtqA4?$yl$0N@WP| z0cd$KQQB?`v_7%DGM)hOSl|4r=|lqhf3DuF=Yhus?P24LY~^;7f6Tlsrt{?(P7*Z> z-UUcQNL7G_iJ3*ZLSyt)G5pI{J|8MATh~X!opxs=Kwb#J6xXdhp7?8F?^T-omYD~1 z$_E}BdBDAx9!2Y&$-6UkBw%uchtL~cJk!#fOn89lyv@`6-W35AZx4ioXd|`gwjxKP z0$rb3dfS3BAtshu#aUuqVVR`krckZU2k5%jsT&_Ih%>Atea(xgThD1ivKK z=U<=7pB$aY(!kVs=aj_%Wx|JSqF4q<|NjC7-YG6o!nsTwcP6rdjMG?2nm1<+cIuvr zK>So$;eq#IQWQY@?GvS87mbv|VS+VapwKZ{M@0O+@J z*wJtN9=)P|s`=!le=tpmA2sQB6#3^*I^PV*4hb?B=bKoVI60kLA27RNoPim^< zT-8^bX6F}NkFyWh_7i~b-+Zlo(ct~+pd2On&?(f zOoGF=1Mb>FYM+?{9AApNkdA=T%S1@<`CfGu1;w@{AGOK!dRjgqYZ^C}MSUqxY#{i2 zSQq)etS2?lBj0o(#dpK?<+4n33hMzU|cPlh@zntWpOqT7So>Dwhc#uyV;kGs0* ztEX#RejRnWl+4jK@Uv>wqLF=!Zfz0``ZS#u$N>LkejwQ@dASOqtWTd0z4edQS&6wq zIeH4x&ZmNld^=5}d4l^S9eap?LhZSQt{mRD50xCg1J{Z&REVHwgmLxah%i#no?gIf zjcy`up5kc+!9_XKI<@}NLNpp3uGB>CBwF~G=}(QIDoZ+k!qv`w(d%@JYOU2rWSd&W zACdCZvon=fXzus9(-JyIdxW<# z&#A2&YV631KGr_c)%U_Z+g-vGv@;4C79&5XR#8?>o3O$DjA#bBLMIEjK04kQLSJ-m zzzf0$`K#oF`6`lfCmN4)Hw;^#ep)_dyVR5et{i6yQsg7!<(-aXz=># zL{eD(>-oN}9%Ce%6&ai1;Ybw6!m2OWwoR5%?>FNJO(?!}gv^AkuQcpzq|{JHFLET~ zEyKr#NF;EI2*xo6Nj2{dkDE?#YE+nUA9$Q$VPTCt-Fp`D2*AbfLO{sbGpy;#ZJ2%(7Mi*m^7 zBMQLp!r#b!G|VeGTzYLc|7$k2=P~zlMWd!?F3*FQ8hKx3?96W zREnhcyu?!P-{!r2{*F}_-`$Baa*bBjf7{u-GzOzcUUUT2Fw>dHH&@EXt2$yOxa|Fo zyj>y&f?8zcB}DayE;vbqOtcFplVP9t!f8nIjG{P*YKGIW6NPJPs%RQT$oUJV?udX~ z;OIg8Z(!REOK2e0l;T;Kh@~(bxWc?eZwa?#!c(!j@4(2#XZsl1BQaO=HpDH5_+2fp zAjr4JoBbT8C&e#BQ=OOcjR}WN)M2S5Al&6WeKT~c5B%#j&KUSXp76aU(*rL)k$%a- zpJtYYZ_8gPUVp~$xSDl%-|WQH5kUz(c?{yj;Io;<2nCa1ol7_O+alPN5AF`9_q+{` zW7%g8Bo`7O@y{;odYKubf4e+ zipDKpi`V$&Faj4^CEmXkK4W&j6S4erT|x9{ueDqpiM4+UMd&v+W>^e}|H_jw;Pmzq z#zqa&%;##<@w5!&=CS++ejJj z{;8C7LbCDt1_TXc{-uJJTbGsR$HY%zI$Y_5%B`5LR0?>j{+Ztj+|qHJ{(oNvGL^`p z=4EuMrSeSF?d!g`GN1ap^wZT&K7^f4^e(rYTJTdR^2GjB9Ys;ys6~lHseXQYa=uuy z+HKV^$z<+hF_DK*g<=dK4{>=N6@!}g0O zCak_zI$lBH7S){W_dPz~aYJ!_5nyXIJDmzuSS=?SB#XI|#^xQ07!4%iRr4)kF0<=m zavFB~ec|nsY|623&m%PbgiMxTUZC^m`^rQq5_;Mo=Cdc|I&itFn|hEo-|?6Ex^1nQ zrc-CxI3aY+Rv|9q-j>u+=7b_+X_OXtj;^ZyeM5w5-}pFBgq<#P$gN(Plt@ zLQz}9o_8y|&b?8+>LbLM&3F^RG^gp-Q^vRZf|m9s0^h7It?$mi5)<1V*0lJ|oKR@Y zxef*2E8+`rx5Y%d4)|hobq*Bk)Zl=DG>A+H(%|F&K8hUbG3%z$e3{HMBE_X#T!cOt zj$ds?QcIvDB91hdic+FSNjEB9^1G*JB74%1F)PHoP*qu7qYF9jw7;yh8mDkO-|d?= z63?-kxJLZJAhuOFbH!;|zo4m+61Fn-T3LC)_c#=@Zn5BIUVEHcYl!0Tbil@7BKN!5 zVdaoGd!T9ha%GC|&B?c0;S+LV%nFNnKhcg}W}%nX)5@ml8oARY&u%<-(dd-JbOybfBXR3# zr6U3WP@y}$9@+?wJ_G&;CNW-x^9<~bQ4;s{0o>uhDIHgBk<(!~s;cCxMMU@str!{p z^1u&Fz_{nnlEkgfb*7mUfN9e>in+hsj31XjWlHPW`N(*+9Y638V0gSbb*Xn5dNL&D z?pZMXJAhW^E)bxB{+K!25J_e=9G7;fl%HO;f$SQjKCrQ~%^rKBUHsGe;X z1T|U05jqR0O&0u!yKdx(?6=(T$#NWpE)0hSJ3TM_)!>~zt5b1jbZOQ%|Kv-g_QT3W z7WMNnxBHvaYapO5tfLjkV413mrijP9;;F&m*9OVuDXP#n(7ZOK1{<`YaTxq@k7QVm zQKuuTRw-RgpdovA;3L1d!+|}%fk%t$`lfnkf=I~G#yI{p@wF4^o1X5H3GL1bL9s;E z7vq(-%g}~BKG!)l>VRV$B{xY)&199!FLd3#%}M%fK607OPvXs8sNk`(A`|}~V{aW6 zWxQ>V14Fk;$Iuee9YYAx9U>v!4FWQBhje!gjRhzz-3>!1N;d-%Ll6BMJ?GqezxSN` zJiq_o8F+bkKe6{-YwfjpOq<6r_1RYkU*`SHHmXO|J;oSJDu^J@y|9h+tEECFLLYja z?pjd_6g8rv`rbXs*;0RS=GOc8GO?*NYv|?ZT<@w#f{P0tD2-e}g9etd1I#>ryF%pA z-L0>b%qyjqe_c(!Yz>=CW~U`r#60ti@mbJNM_ZM|t~O~wjr*Y{(CDs6bSCeqA0`uFt!0J<6qEQ6xY%(^{R7HwR`@HJ(4ym zU^fJt)MY+y>OtGEmN*!OrG?p^GgyLJB>N@xUq*TO*{|SK}bT z{KlPw#T8m{6mFAm_@i2hZ&~oJD4;|G@q>lX*UdXCoPzN)o@wzT_1@qs74Iubwd_v@ z;#kvV%HxIKi`1@q3c1F9HzQu}!Y!$*Og}Rsd-*I+xR0P;ut+!&f;?LEny%t23zQ0w z%KN#QO7|jsnWPh&XZpwfNO4g|eBgAJ!i`%;!D*t=YmZqG2A)a)^1Ajqn0H))FU%>8+J zPLsg7qH#pb*DeH6<+|ACN{C|7MY5QaLbiw#dpwmCMidwymTo&he!BLUs%aQj$n0Qw zVN`fC?qe_Ng2pxfqmmhZxL`HKbj9XpwjsO-3`d!A{)N0Ai@!ulAj4Z*D0qG%aJc9D zKA(EwJW2KI4%|(Ld@(1?D$`c9!2}9^RI*89n|0fd6K9-cL{=pPxv8j2aJ1X2%9WqL z+urtaBx7uqyAw-Z4d1k=MOp>`Mf!z4E@4R4YWDg_t3_K6r@hIwi3*zhj~vrJjeo8hmD2FQAbscpmEzfw2fxvomB1M4Od8rLx zdLT?;*&mNF`Xjg>^V}wGu8TuZR>Vy=DPbv^2pBF*U>2ImgM{t zjURd`+G64;RkC<&sh|tOd&75Tn=O?V`;~Hrw)~K^tgAeG&dNxPydQV?(w`P1fIigb z(3^I3z%@OxY`W1lJ4@yXE!=r!J`vAi&pV0#(jycQ*-_Jm$C_9WMu|rNxPE?iQ521s zs>Wb}S|KZD39Z)rDGmRJXb~y2r8xm8<=eYT>=yuZ*z;ZNM7FqwYUXGMnag9D!TQjr zD2aoQ;a6?uhh9U4lGMVDNTL;3I1`8=UdnKL zdxZx-tVL}ef2-|^Nhx)@GuexIPA%dp!zHLT5JL%|60RaRo&w}fFh2x+Puix09d#sO zdI?tOP~1(mI=^O-gRL?!LA!3#cT-+}wq*(+7o9xypJJRmx!Uxq5RY{0yvir|q?iZ{ z{{%Mm{}th0L<`m{KGZ>%mEP5GTmZV_S0^IqYLGR^RyOjy@AI;@KPRakz^ zp)=0^5ocgvfMA{_ug;A9{S=8Z{6cXB4Ez&i@T0JU#G%bYWsHDQrBB<#mu(|GBz~`~ zWlR7JwU=q@(Wd3<0*op)c;n#zk>7va3W7py4qdogflY?HPHc^Y;{7g{To z^KiFoZsSRB+%Eix_C!91g2~sVc7OR2R|;KkVstzuI*Wa&ngixhXk!N%Z#wizbYdM9 zno~htfUtbx2Mj7Al9f*`CdR*-t5A;N+^yuLGaKg@&Sw?;ZmT~IT&OOst%474HGA}u zPzymL!`KB3Rz|``X;Y;dk>snN86+Xrze?I%{2!#4s<6T9!<_Nm<#DedZ^Jn%P$iSG z#J}$z;h*_H1x!HDXvo|{YQv0g%E0EAnel^FJMLU;!>=|GZgv4!aXZz5lF*wD5Pnb( z6Mkx?5@wvf=GL)n;8%}>x6Z}i9~M`YZgq*yltL;DNkXH;Q0@w8yC2zn`H`{7O8^U+ zkjbiyE;}Eb(ccL4^Vwmp2giaUScWKJlMwB^M&(M+5A0gvCvTqexZqmLbSCb2Yvu_* zdLxw+F4^oLlE!QwU)gf81q@tc1!XP%$(zKtef%K0??$1g5+?J4&h@u@zYXmd9{^2v zzzLi0PSD2$H**6>iA`xJ(M+EJ!TXf2bJ@{F!1 zynt-90={q*EbD)U>$X@`?aM9Nzu+P2x9?`ZR8g1YBUFw@B6V%f<_|M#c28+y<}t2d zwo=4Ab&-C+m0R{rBRI28FXuWMhNxNW?U;QDExJK(>a8?)!#kd3_5R)Tg2=0M9C;}i zeSRcG8h^>oKR~qqAkGqLY@b|taK%1$YfBg)=$N|OGI0e36kM~q9lqxEj5-dILFPRw zc}bL1Z}{DBp?=GlH?($>1>yA$_|Czs&K|07IieFY4BOvG`48##e0){&^-`j} zNed!R67dM5vMm%@44PF7`ZwwVIihRRkJ*7R3_K7Z(N8^S(YB$)5VCKd{{mPPv{Z+y0iB=r5D;D6bq{LfSW zLI}-KCdsj{K?+&UAd(nXKW5vtL)2CNpy*BELJ(Cy*1LdhwxMv~px2L6_p_b~t!hh; zMbjIuKWAVKe2`axUg0~VHd7`~YyYbn)&B#n7(!%lRGJ@h@uIgC45=|}L0vvnT%I9< z7=HAFkNl=xW%kn&dB2oH0Am^?+WfRtiYUHXb0P<)zZ+HXOYgC?@w#v7Ce`Ui0>%a9K zqxtyYpwoSO8j;2u;R$v{8_|K92OA)@<;vcvlBB^}1FlqM{w`p^jUHU+j&F_w$ z&kIkwqV4yqXCPm(j$ZOw#;iG*W!`KOOeorjxN1M6U-qFmdTxq&773ncWv9qm-zid* zIl(8I&C(IxdU|w++6$GS={4&-imX0ZUgc0(ZP{UB#kU4@Pe@;^nLM9>91&1qxpgj3 zpGak-wYMu0iQr>rqppsEO_9FOlQ!+(*R*l z=B(52v%c`*<{Giqr4S9{k(!^P;Rzby*Fosg0i7wv0)8j-i?;WM%i9GL&P8j#dfL6i zvyRoSpS)fcQN!hw#i)K$^1)TU{e#7EsiS1%i3<=8m zZy@9haC5cers0_y@|9vHrw;+GE-;HwXm0R{-5j2(x+ULVOu%#iJu{k+sl?CV$c%I~ z5tm7%`L%Y`nN+v)c4ZDK6B~*TtDMa5XVOmXMLC=2HYy{gntal$eMU$pd0}<7eE)7{ zm=d-gsxeMwb1Fj@287Knp4g}&G2PstYI_EHp6&j)tWrKt8rROCC%KLGlLOb5{joiR zPZE}FH`TmP8Vf^={dEnbEYS4k8zO6VSK`yfAYQF+_Y2fbPc~_#Zo@sAVSD2K{4`>&|1sz zTm)E{g1%f;y0n4U>NE%7!VjMpyhD?weoO0C#6x!)?%}5wwE3*F!F!=Vj<8&_{|J5X z|A9XEs~muTp$|25CF=cuWu!JTSpv|{pEcw|Ra%rR4o3lPi3`bTeA$gctW1~vr&;Fo zR3gL8N}(X=!g}LgY7_{{V@|~%G9tE~{*;GmZ5X0q+O6*BEo+j=ZF&UY>NP z;>c?>3?`o>7J&!hSPsNscvHQ!t>HRBJRIMQKN1C)3`9R4)#Du4Nz?tM?&N)OWm zD=`+>@~#O|^u-;dtJ9}TmvCdw;g(P&XlQwHUD0d`d;TEdw>~!@kNSSPEW%jC4NxGt z{LxI&KH?4V;fBO3c#R7*BZS&E-zh*PYS~q<*|EihIz+5dynyG{Q&@ntn&4w9++@lM z5N|jtO~UFvWhv~(p#QiU&XC6IjSseaK{o@qD+nqz-xFcF0$sT@_uopm#Vbsn;@d?i zSp3TifY)Qbm6&>xq2d7cUbDlLxA3>)5(JAz9v~h`O6@>>{lc6AI_L4a>R}cjsfRAq z$+${})-kEePdFdmZ}jTTiXcRB9p7ui*Lkc-P;(%ToX&Nrqkqz&G=NF|`fQg$s8Ay4 z_l1k7?+J;Y@J7nxh7Uc--EM2^U;VWz0@R!Rceu$Sb{&8NJ1PMI+si%jdfR9~M{)9K z@_Y?2kq}4OU_}b@i*n!8w)pYS^|Y4-&TGua<~pXm)VYV=3!a2Q?Qf%Z5J)~gFUrwQ zYMp>{{R?aRUC`3}cMtLu<_or(-|fxqw-;u?A{$Dtb6eeBcdk8BU5-sTI0Skr8^?7Z)4KI#?;LQ6-zQAt&d`0?B@ZJjYj#$s~$ z4j!kjucA0P39lv41NP4_O3_H|YXkMgcO*ybINt~keDpR$P{7{NB@KS2*VH3J=}Vv{ zA}3w8kka2b*cL6l@mRO~<#R+0xAX{;Ceqz*p>AF(T;q`j@NnFf0%7_`Fyye#e;2FD z%>K#Nt9p%`{;I{8(aH$D&=u2>tB~5u4~h1`{Kj0dpJ|$Qo}($9M*cM5f}7*nhC9cO ze%@wip|Fc%k>G`kH9LVXAoyepTM91^G;vh=Mw*W!RC)JT<$^>l!6eR~QqzCjXaNKr zHi*NVy?Q1UUcakT$A6x;i|6~y>qMpUYwq`hFuVK#LhyFWshZb`lJpyFNvO8#44_@w z55i+I8awBq>kNc3*;9M#%tqB@X9z%lg_wS!wHwNjkO(;bJPx=c@q<`}R-m?D-qhX? zeuny(i$K@cvH>X%_^Bc;|E!f!IOb zrA6DYK)YW7?qamp4@t^Nc4$ec=HR4x_(jC~-VhMxQSqywF#|qdyp8r&!u5;$8mns( z`1raqv2+k@4ZXKo^PdDCi%7lbb)(R$jl7jOl}?@onD}YE zXM|Thg3x{V7q(o1x^yNpCQB5!cgk1UqE?(J;$zS(3q>+)U9 zZF9-9O_ns)N1RR;KriI24)I>gy8D#1q#7Ys-`haA(*=P6`68c5FibA= zKmvt+96VzMtrSQyl*Z8`nZpK`Lf8vs1Vk?EEZAs{-vBL!efjEiX}X-5)W$7KAmpSM zYKJRD;JM8ehP3;PG@@Z1Ja`=4DH3C2+G*iAw55%DwA%cu^9Y5r&xOswzgJ7H;R9*0 zQi`1gq3c!i3P#Ru{z#GnVJKy@H;aqYt^(M<>o=lR0vC^*OO9#bJUQ-@P7BDj!2rFZ zpcVUqA+s2Xtv~LDX&)7jd72-5ODHBAyuxZbr-J(C#A$7RoU4v}ZHuHpiuHzf47b~1 z2TvT8lIf-5sU^^C$Kb324y6Am!bn{refKsUel~9Mg8W;#Ki$p2Wr#FFIo{(NHR|Ai zx4vH_11H~0g)6KWnsM<^P}b_3K%T`@*6Y;%1o%G&=mKUCuq;0qz=&_h|Kng{S(Ym64o6#GGThBfVisf`yf zPh`xH^q-#opbV#RaY_@3^+;+c-%9OUYBdDR^9Fhr#ToWZ|-5tfXhgLK&L`wVxfm8x9+iacimVK zJ4$~XY3R$3bH{ZfTJ`=(SX4IbI?R!CEj;a`_YQY&*+Pxq2ZtEHPg7*1Y&8~g!4<=` zjo1fAiBWd7)*nP)0%b0x4j?p%d0v#$U00su7WbdF;O9H12HKio(Dj7s_IVj3Kf8t& zPYs?x2xkig@}+Ydrb=SfEViN1wu(f5`X$i47Cs1#X(#-4<~&LfgXLG!hf{3TM5a|C z%U50L_x=a`dcnjQ^GsU{A`@i)y_&&#A|-cri;E__waL!6 zImgAX1J1Xs5q<+rwI?@pvg`w63maQm(3vwrPJ`w@tgHEe;zd~Dczxu_K2j1vXaCxZ zDM0A%N%rx1Q|v+~%}e#vYoiWn z=0EnYY6JQmMPfI&zPv}ao@P?mfuapK+I^7Y|6&%Di~|*=3?^qLE8+*;GUaP~80s>5 zzNKEP0lv`@oneX+TCJ5p=;mC}#gp=aKI_SvJ5oXtP6(u2^P71+Zn4>n>TC5z&!`Ih zm*1a0Lue=*EgO8bKqZjj!+FKIiyvhEFxMPNEwcXhcPRD1`LXn)uGr&mA6_+K_Q@r! zdm0OO-%!R!YNpG8pzV*bx;Md$)U(}U&RZjN9;5K>WYnhDL?reaxnfSHkcA5HS+&v% zIkmf!|7eIaC4|Wq92DZ|OmJ44NF=?ke_(ffsdgn65m~tki?$stjGU~ftm}7ZB4bY+ zJu721?8K4NXT^5wn4(6GR>S-~=VqHzaLF_>_rZ`L2}cQC`nI;v=AuX%jPliXxNm19 zIO)wn-@%Ck$p11NU-KRLvceZ}VYsJjh{*Lt1ebA>r`b#^_8my}SjO#F2~JQ}WuMr- z#D)RtgDe*94Kr{M+V0Pq9eLM$uwxcJL9X9(j7WGM_!T^iD>W(gslI!$R7{xGbw`@W zTP&DP`jJ{D?~L_iA*o;cbmZXS50}n}9O0jNRtg&>dNgy$tpr?KdUE@H3O}i4D4P#3 zme)y{^Y(;+17}d%F%wh8ED=Ac0xWU z30S`LO%^YQ-zFre=5jOW-eGjIm~?ws@+6$0*5x-(Ew)XEKe^9q96rH5Ch<3$Lj!&z zOp1w-22?Up_`8sYyqd4~v595wOz}u3yF~|da@o@Ry$jmKappd)PE+-vf@iG1_X$fA z*=Cc4U8tn)!_U(5gj-AV0w0!|t`KBY3$8F*2a&AXtCQNXXhK??m)1ym8R_Bu=~Ft9 zI*&AOHFY)&BIbwNUv3jxU#*jaS)xMPEy~QjbCKEyRLH~V- zA{VJb4tY<<9hMwdVp7mGb`h6=p~&wU6piC z7G%)MFuN}LKz-+jO1trb->l`0&?f(A23O>ud&c@k!`qER%VNwoI@g;UQ$HtHdZ+>v z-!io&Dn=Fzx^-Z%oof;d;;jX(7ZFA@zkhTh0vJb;v)tzYgk!(Uz1Yr;-=8Ym zz*EI}2`>n|(1BtK2q6ghU>+rZS+qLh&2!vHEr8^Z(c0keJ-Qz)mR+S6NOc0*#gR1n zLhx0_WEnw2!enVEpFrW90*{{EyNj}yQ6%z;xg8`gD9&N^;nv%}Tj$${45W~p zR*k4-NeQ{VSN~Zk?n~J8v>iiePcH&h&i@NqrOJDsOxc5af6y;1qS>2=eFT8+*CHBx&&ZMKgQdp? zZ)`&71W4rfn=>z*_T$Rs3cg3go9rkqvIx!wcRCkTvP9k1lLogTJQ5DktUvy2 zEW8yYOhVFeX{aU8Ua6#($!;q(L|e){)+s&vDU;iA?n5Hx88zZ&p*s$%&2^qZ$PJMo zc(grH@MI{a!*%KDtMnB;Uaon=mOVw&0DFCs9t@XJHDleGqyB^>CqE6+!KB{}%@hd} zvB*Po;YdW^n`n&bYvinaWX4=aO*dy9!~XKC=t$cXX{5Bph2zoTEpR*Yc_Eqc^u*IL zH6OWcR%k8KBuLpbggm#WD5yyR_AZK62XY&_le`fACm7x!m3n4E%`B zC#s6e16(k;y((!-Q;HAwUn_N1Ry#(<*|F$Zt&axm6VcO#i)SwKcIK!4*_#5Yi3ns9 zIKkDYD@p%?Ba25oac`iP-0U3px1O&=<3!}mOM%_SA;eO&R4hnAe(Xv`XE$)a0d&H-mL_@1(+K_1Q)EVfAqKSs6ll zq<271Bms5&<%Sks-)(_-{nq89Tqalxl_Q(U2hlQ>j8vPqRt!uT-B(k0HI%c-#JgsM*QjW8*3q!Ox1;1X79!n>+n^f;>C zy&oCi2paN=*|-ZQkOc7Pew3^5BPxBffG?z<6chnX%cRDQ_VFv_tRY@-SI!vr*Vx`K zt2!+M>+!z*La$0X{8>ye693JR{Cw&Rs2Yd91XWA}C><_pejT%}B@pV1BPS9sb$v@Z zyq_wSOSXoTh-cb#Z1sd?hhY?k1zf#3Em=w9hLsLytKnTKZ<}yaaJ?tWT@}HU)Kv=J zj|#U9tvs#M-Sc}5Q&Y@}q_Ql%M{FJPbaTv_wOCY@U;LplSK=J_VB>kkJZOF#!KQ2m zaCTE-zaESnWjb{~g4C;;f>yc6*t{^?X;9U24JO%0HtbRzIy|0>OxbN3fH1Z5$;m>S#Ig+ifB8+UmmMLE?%@ANbZHOchIHRukLfdZL zI6!lN>4i0zplkbwze2>K-ck5LX7HUUgcDki-HYk(5|2uvR{@81IjEr7TJRgB?RCVZ zDr(XA7kX}Bo$Ns=Eq)siJ}c;dtnj4SAd!6exx;eB$>O!-j1RF;nM_+YFM}pWo$ySE z>j1)3>2!=gxv<+g@@dvt_Yf>s`7*EO6!!7ZF<%UsR{ACOAbMSiLB2JrMH}5w1;%`{prS4)Y}-3 zk41D*W7f@Oc6GxPD8LSW~?ZeACG1mo^Lj9OZJR82w{>i?yczcf5E&rfC6Y>CkLbH>W^-Lnyk z7$B<3Kby+@;vhSHMlS!=*)8d`#{_;c4fgDeV5u2m*W2_uc}ypj5tMm>O-DLQ5p7@9 z?V)~-Lswm~vf~3sk%XS!oXZUA@M*1`6!9bkE7Ws^ao;J3Pyh8M2z>;3H})^p7$1br z1|UA5lWKomS7!T*1-nnRjfYgOA%?w;hbvvMdX5sd1jj!!t1-muYDTYlw-BBuW`;}0 z+>e|A%XdHB*QH0CMbbm9M>atTP{W_<))|ZJ9|}W_dguG@vVuRcTq1%H!LZ z%D)KLDYDo4W3&73(`~}!WC&p0Yh~~Meu^CuV)FSwdHruo2W}lcY=9EB43sW$B}mXQ zsb>_iD`bUO=aeKf+3EjmKSe1r47|;jSg4KxkqMCurZ}~v%Yr4VM>JAal3A=V#L1ZK zU!*G#31TD-L17H24UEZH-=<`(K2bfCk1GW-!U|WEjGwmaA$y~1AN^5hGCPb~m)BNr zHX^Cf&VkyUmwbY%u4|w3zX~<2r1q_jb^3o&23Ht7C1*PB!BQ?)W(0JLh#iPT|m}lz@-YQOKWCt8Se&Bsy-`(0!BAV5jVjudR zFN>dMwpDge-Fm6xh?7SaG~zb0F-#$N^k`h4u9Om;uA&|5wAI5bM5I4u+i&D@s!MfH z36u4_B?%+8%akA!(Ip6_geg3EuO;2d%QUZKd4Eamu>2hlxy;x^sJXQ?|5Sj~B5(Xd z3++Nz=SVy{P(xub^E(ROJ8fUBcNRImg*);2rfqY#(U&Pi_A*U2O`N5|#hV>md&e#5 zg}g?b&3-_Vr^_0q(~;_wKSF?;6N72+=enq3MjJ;7HLDbls75GZXenpaA~pH{$E%`1 zcK2ZoC>;G&EfAxll{NeiULy_b8rES)WT8xMR=0!b-me14XyWF)UwmMXf&Mb(BX zhfhwHadPKxuUw$vmETt8FQ}KCJRg~apTS>3M9N+!-(x%r3qF@2k3ddUOQJ%}W%!r! zg&6QHKa@Mljt5W&qU=LC$g+F~(l)0x#dLT_6s;0o^-gbsi4c_l*7$a~jZM49STKF3 z9K>1x+*l5PV%l~K5tA}(cEG1{ai^!1*6wuq;G=mlaLf#YZWAzdzooAUmHr?v&QY{V z=a9JY8bId}1I_JsfJunXe;jH^obiWp<(T^M4SCX5rDISW?*40i@L{mf`_)W!t9R^d zV!Lu;F%^#~6Uhr7gCEBZOe-g#Aq~?{D_2F1HX=WZvS(KPB-jk1H(g&UW2`yFcM5M> zGas>Ys$Gdf&(z(FyBFktgU8L-nU)1M{Q|7s~l9$&iow9T%Ly1VD$+%?C>!$jcwv(539ei)mif+bAn$Kd2 zu$ON?CVkyof+VCri)&Gr{yX>8^+13;PY<2Dj--DaC4huzW##{4lNcjmZI#)$7c5F` zaBbBCuGCC))$(#=J~SAOOmvJm1KjnlXDWCH!_X7pRkQ575lKcIcRUTabd(%6ASt~v zJ)*P0%E|ij2XAzHg?~L)4j-vnIJ7ap@I$jQ<>OUtg9owy=z0>h`9umO15^76`^w+V zVNtKTt*Fe{CLb<>P?ZL2_vMg}G04y$bpuHamhzTh!t3t6zJ=j9-XsjtvhpuISc+Sc z0|M@kG>xH!Duq+TD_MHZ_pzPxmxK7~G`qnBLl>7^Om)B#;olxEy0d zhVDdy!}$6eSIf#gyki__LB{&)1u9V6be{eylDz+0w^f{I=pyOz51l#`O+2;bw87{> z@Cf&FDj<+&H)4MJBi6T&V!cP;S!^iL?0?5Atc_6ay5vn#sS`c?^smV(V6zeSK6B0^ ztP1%j#`LF!%3m@TdDDM6$z~yH9jx(TVX9}v(*g={ef0aNm}|%VzedfE+T41J6jD*8 zxnnlKYXlsVLc7RrunIKBSIlZ&rOOE1>XhC8lR7`WLzO7H};I0^F@(h>#W%)nue_i6wjK6fQQ|qm#l+alKW$ zo!EjR$%Y84>^8_lsaHBCqELf4MtkZ2Foq z{e$zblBz#s0tkzeCt-EYr?f61!>by$(pY&X7h88?$d@#p5hG7uR!pEuOyZH*3>!FGq1!d|@zr+J8rmOTKjcRlxO1sMfel-qCU7REN9J|OfYRrs5@fY2&G2va5% zNvM4&iu!-`V8x(HQV@9!Z9W!<s2BiJ2@yB2PjOXY|PX8s&&O)V?geteQh3YK$z6%x=Jc(8RD5}ykY=PJK(CdCe_w6%wq)Id&elOwH+kSJY2Jjr@ zURv!t-f%}ipI7PE2|F*>0)v(vDW}S5rxK8F<46YZjkde%v^?g>3D#7YWNFZ-o!4D7 zgX1OuH^zez^Et>^SD|R2Y2Obr#z3YGknNWzpeL&lB5VVed|y|?EG$gddn+tqKHMno zC#f3D@U?O3d!AXqk~BBff3(zsyFWsnzTW;A%EMmDBy2t9gW@-CdOSxHB5cD!7&&c~G<2rxrHLP~ z#4SHRe-uN84Slixuk4EgIt(zeWoRKs9^(>cXdEft;c_cpyJ;|B+24+SvC}81vMHWY z;CX8zPdC;Pp+(2&sANl`vX!FTBH|bV^XxaDiSni(2c$k##ces+3_n`o?79W!aCs_@X@U zXUvn>(N-?IVkfEH0CkVtK%Jis|780_q^oGFJWX_587bVKiI)jD+O1JnbvYE^8^Dh{ zdP5&kO|E7Cv#XFE;~$;K3*+B^bt3d=Wy_xb(uugrB#lwRaGtP4FqX&3Mt#az9ccJn zdzmEZ?tY1?FXg9~WI+1;cWFrk1s`f`1`le^SG%aw%E|8Sm(**^?Kg|{6ns{z?dy&H zd2pcLoO~Mnw327QRV~S82?fy$46YU)q@9Bcae8EM{VaKk%m)1}Lh;$V&Ut_`tHV}@ z{1OFG2oUDW;1i$9`*5%IenjEDuvPp|#=92?^{ou~Kf@c5dFjdyyY!y(uTLaDG8V}I z*I&Z1ZZ-EOaF&MsaaA%o#m)lS>Dr_zB%zX%Lw2~NQH=J?4*?^CNI8+bQVs_x=+v2k zLf9UKSvi&D+ZhapDT-kG@mK3{Oz%CB2;~h00kdY9r_5FGeEp;Kb+Y4n^;@2WMwL1y zc^hE*SdX#Zwql5^@_Kh!n*%eu7gAvKNdU@H5MIJy2FkwjWo*j|r9(Yi|APAwHIzzxY{Fk_ZbboFFXV2~aUich47q;0F-h>L;fl?oo5|bze$h@c#{9{GJz5?nU4F>B92zZ<9dOQ}S6rrXu9x=Mbo+U?;F=h$_wYjHy&1`d zhUq$!0-zWsZMsKthO7$dg!G7G@e^{%qIBYvSZoj{gh|S`Cg{sOukcx9%ZtwOy}Dl^ z^AOnI8kuSG2k$@g5LaJ_Rjv{RM4iYBxiHVzP6GJYwBVG@!4FbM&jBi=-3EU9O$SbI zG=3O5&f}=v&_X_4TVyXWpI_l%XnU(q?{>oPwRxmFkMk>g$W;pNrJW4X9AHVT$dB^& zi`FNofs8yk<`Gr7f^PB~0`AO(k-6+>hq1nF})63_0cKZKK+y438|3(~890=~vjqAM~kd`Q|sCV3psW22()A zT^)y#Oj4s3)0Z){tYn|^TNjT4m9sc~b-=shuDsWuQ}!K9OLW+ka)e1ai`vn-&6@iz z71ktAw3=2V-on|+lnT&ty_Px~H_@8?14;D=T61+NXM!kQ z*SPHJkX^pTv-}CI9vj;|}DMft_KzjE~<# zlwqN+MXNKFlPKjQ8PP~JQZI6%CI{(_4re%k)V4@ka_SMwJwA-+=FcvfcRW@|!lo13 z3neWQQ3(2zVw0YFFpFZUmQNw8C3|@rYt_;SpKz+)M6xo+2pD7Y8j8(LR;K!qPzxqNt4p$t^MIgo>6xk&L##2-Z~MijK4-dUl5GolD_Z5L>}^s+YV}65-#j3+$}|(OSiR^ z;D>WA&|^Nv0&CLclFV*K@T5gv4-H=9i6Lx$3MW3ZeotHHx$HcB0I%N33o{&fT!;5? zvW)vIZLR=o`^_8a(!x+~lQLu#(}#_a(rkI2XdmwV!crCqyGcaYL86J(h`bgkVP7`- zLSNmqW1MyrU|?MwL>hVjmP6V_n;(by((iWWi(@1L#n2j#O)qFPdm^Uj-aOMOQ^U#P zHr@Q4-||N8N+rW$Z&UeJFw55Hw8T@&>*P@p9UjZO-`<#hmy$bcbH9BSPwB}Ss#KMd z2N|)~|K?tnh%DA5c*S$NxN|@0Zp&~X(NU*DPt=gKEHB1TBDZBQ33*uQELWR++Qv~> z(5t~Dn$#D3*DJos$bT#nPJc*8CUTEJ z0ZY^5u}?v4xtuoq_RB3S0iEV7G|QfBi(C@A#ZA69uOxbGK^wQo&Sd!Mx6jby^9lGy zs*wBfEelNM67GC_!{As{gaN#Mb7!yxt7uSX&q6!kl2qX|2AFi+_ z3MBB-29x^F_~6WIoc5x}t1%V43)UsQdXy?;ObLxgDnr3sPF zIjGWZOK487YQhG_jb#SL3|9xvorqaAUK=!6iqU{-Cr<;4eFft&ubir|S*{|8WauKX z{6Z>zmdNwId%O&>OwKHPEG5cO^Q;lW$KVsF5aS3AR^E{-AAY{hGhCX9Td4C9?y9v* ziic99!xwhkzbBRf7e+ckO^f`WF$t-ntSlVCbC@X$mA*&daldXOdE@_>RRjI|2mJ`M z;5z^zofy>{Up<*|s+WIyhTwA(lI}zVpyH_8t5cb4uiaIj)T^+yt4At1I8UDT3F|B5 z1V_t7FQv5jHH^4#?;HMX@f2B<4}Vp~fOM5g=Ckrk_hrqO4o2!y;I9V}S|4qN5Z+iL z0WhUMQc6MBiGtRUJ2n0lf|fCm@0;hDW0Y%6D-UnvZFPcveAX?vxq_(jzk08O%Dqm( zoO85@W3s*2QWWvY@GJbMl+R(n5XC?OHTmA$N1V>ppHg7B`=6&~Jj;q6&EoDg3qFWi z@HvcljkH}BTYa~0xaPCJQ^!3-gVk<|{1psjUneRSp?JHQ($db~T@cHOY^8eL?nl6V z=!Y_SH-kMC0{ntZpz2^#vq+yb;`z2kLXAUd%siy+Ai3{yssib z1&BLw2^j9*le1B2v0YYU*!ydvz%(FmYn`;i=l48;!Sy*K;s&9d?7yQ?AQY({w$LSQ zf^YzL$a-W8dV#3d$Mb@?3!fQbyDqn3U=h-UNiZMpiaC_Z@??L-2gRl%-k!K4 zYO$Mr!UflJj^$PtY>fVB7UCe~?n}yh8;F5LT{28t(6N*gIj;>Km`x znyq6M7`CD$@bU*|PUv!f&<)ZY&igu^l5RwPlq_KlIr+hgMvh(aB$L~OhWzJZwDiL~ zhKTE8I!tM14B^bJwaPAM8_?;=lst9@Y(&~CiNuEMu*4l#Q!}5Q{}z$u7I-pYF-a0Y zwW53@!&U`kMqg8h6M zYH*#qwDDBH3^Ia(74cy|C}9-4IR?lnVS|%uxxKI)8#0oM$sbB~H=0rBFPbs@UxOw= z!5xc;g@vTa&>QG=(t-nWlG4UtFg(D9B|LYP_H8((T9vrlcoWl{|LH@=rf3@IFq1C!V)Su=+p1C$}n zv){i0?2OhXSCV6tEoaH@e6#h&9S+qNrHL3z<>FL1V9LMTkQgR-4I_3`%3%}(-FXEa z?g-dTSa@{faNIHM9t3Qu*k~_Xv0YklCcGs{))S8X9$^=fiGhOjQw&8Q?wi~mp3B*4 zhQ3*;L_?M5AX}D8+XzcWvhx>1&h2#Ksq!(O4T?EwB(Fu1T&P|#JFo#^XBw%z+Omiz zJGIYV+51#LWk2Y#7Yjhw$A+~}xJ^1}-aK(|o@t^A zyrGr6@cDCwkq(0)NFCr>>pBpDMQ##F1-oPLjkS&%L(tMXDj}V`^gd}}dW*3M++!|b zdpo|`@o|}UYG&{ZMZ01kJyd#r7jUct^BXBsPLtVE*)p)*qZW}pF!~N(xI#~I@vV;5 zIh@hl3Kw+HspB3*3=owIhV+SwK%3|iT;{y$gg&nO@3a5-|0sLws5qWA3J?ntf(O?G z2oAx6y9Xz@6D(Np;1=8=ILzRLU?GFs07>vLSa5fD7MFhH9I9xqY(>8pHwiCr=zd-aUYgms4nueCU*O0%6O7f?BPdO2?w z?OeZIY70sNjo}3Fb*M~M8iacuceSyhhq+vQNL${YIb%NK)H@&bIok{c{N_%ly<}*q zB?A4wnzT~v(}^*Po6TdstI+v{XhIUvR44z2h?z}g@>ZKPO9q|hVhewgp*0EXZ7^z? zVNKJdjlYov{=0>fNw1peglrXp8e99dm7 zIj{F}26C>7Zpdr`mnFQm(hs;V7o^5t zT^Iz(Eq!qQ3>2{Hs>*3J*vx6Hq_?6s^75*Ktjo1h?69b}((%@Zo$I|fn~l6N6_lR& z!MK9ETVU_F)JEO{;>9`Z_3;qkQUdy)!t$HqlL*Z)_KA_}#9YjSE(;T^5RPX@3f_ z`4d((7kKiG?Z=Qv*y1wjmi3@s^Ql!CX?~lXhUxc@SV7aW4JIjx;(Qo5``AnQLq-kG z8QV$QbEu}hc-8HaJWFmC%c-u}rhQvQL(XlAV+vCvT`sjl5>D4b3Y_(5cHu`cGe1LPFQXSrVR-2BJ^i1@ol34Z2EL`aheu6nu5QH%Fw) zZiWg)Tt+I;JIRcdTj2Kjp>!Pf`Y$UqAgqZl+d+9L4dP_Y(zcTk6vh(3OoUWq#$9 z#m|U8ALHZVUfMG3XkX>BCP26NB`SaVprjckv-g5rBp;?UHve7~M)2Hnz+ORB7bR)7 z^F-P^xz+|x3_N#IWDHdEc8Koev&&;vb4^f5SN`O|srMJ#r{eH7EO*P+miNfrmwW3e z{}IX8yZhy595bruHvO1Rd5-YZ92K@*w44qfuomo|Q1AU2m*xJ+)T5{^L%4n|TzaWs z$eFp(4%a~gN;3F1mVN;f9LR%i^T)IkN-8+82S}+X1Zg@~kJ4Lz9DhfmAfptl#&F-{ zT*YhdYfKJolF#H@p}(tQ?+G-uy#sBaYtg5r={iRP#wn5+Rm1V;@7MG3rh)Yk5^DM^ zTh_WeiK8)VD|%WW8S^MB-#uTt;KSdi-+liW8b9}q#@Qz79m(3Van-8iN<&vUp+UrV zv>!LN5kM>EuY2gigQ#?djcEUFk&ee(0&MkuPr^-Abof z;lURYae@Ru=Efps>Q|59*aKI)spg66zoghOq&Qq>TrA|*xK{01uiq2!At{bF5BYkY zGEcggpY?G?8`aqS=6;v<+Am()!ZXxcmSttJOx^p?O2)?OuPtC=#ZaJrEIWph^SnRx zjB2wdlus*tls5?2$Lbh-S}sUPKpRih11l$Hn`*Lr&#+;%t5a@1x@wI-E6pgA>1Y6v zY;>kJ7Z5kFF} zIn8PZh;1=_tCKWC-+NhcD;V4ogZXTEUI>t5&f{FW%n)Ytk^D%`~8ny`yKmyCGxhj+?@gSlX2R(1+v6=AuIwcu! zUJ90WqS*?nhYhv`2krbw+n*->P_S;SS*n8tm=@HsElJh&`P@wd{z{(5#d;p}=Z7=I z@wZ!}hu;ZF|C-`7_oZyxNyc(0*|=R|@dLZ{l!(M-jRvU#6>#fC2Og9Yhui3E`p43$ z*}rk)V3g+tcMLP|wkBT--RnKil^c}mw)#jlLXN;sInJM(<{QmNQWT;Ff z2fZ&WTbyS>KU$imqS`wSD5AG+!uJxX88+FXQqK#Kr2$8 z3>f652nnVuSm!BZJ-DvSTlo5vVjPx_mzksp2@K@Mw!f<}b|u4qdCi;u_d^|PjP6!* z?FS(o&P~NoF5NOUO^L{kn+V;|(SiQ8An>n~?UbRWPvLQDd<_dpStQ>XWU|y-7+fvDE6xEKZBmnIgk_otjp3$*vX6hddAB-cG#Q{IL?Tr%;NW8b4G z)O8sq3xWEEd&-)&v}L&Mo&xX2>4oEZscP5lM-uOvkS6SOvBd^U_Ax#Z>!}9{qu{K7 zj%T9+{rSB*nQN#;v`Uq*;!Pd9GsheD0kjyCDS-^cSUy$~VPj{`WUY|<$+i8En9~3) z<{dk--pHPactz(OA-lZh^W_56Z=ABEsj61zQ2K5=kg|z;02|)1FeHxUYKOy_0=@ zx)l3lY}WAEYo95^Q-YEK!8nbG*ZyZV8WA1SOGg7&hYw%KgSebOnU1^^^?eWx0ZmsA z#GN%z(ugqdUYtpM-7b6g*SYU4bPF&MsGUlneEReQJ^9AY^?v$L-6{a7*t#a8ZErdh z^H}-CE8@I!%do{!p?6fDMoWfbBRu%P{r+};63-W9H%9$3J4a)hkfE3S4%ith-B@}Gbx{$AtVfxx`;R5fR2{LNI>#b& zyCRRsDM&=V$fTu1dq8W-Px@FK_=2&5fa_zwJGKuy2xkxz`RB_vTIEG5Qch#(M8kbs zBgn7U2W3K91yWCJ@DO!BjoBu5Ar4LjqLBZbf{kTVA*WTa0`f@lG*4v z7xtIF=6I^o<`G?tZZEF^|K0I0Kd#Ul>hH7g_fHBhV?a@JNsekfkh0B^7rLky_GLyC z*4&rTeZPTZ)^-C(y`n4v_zvI8w7zP6!OOA#YD;1E8sOjWQiVX~0etaJf?g^wn6zf& zWA|}Gpje?n*iq56;&l+xo#lmTgQyRsatC!H%zOKyBG4yGFPFYBxF~xl%+UUyUI0`} zb*cH3JgMASUb7ZiCa>>)3|(>hQFccqre9CEUv`jnss-HO~sNNaIT zUSio}fDpN~ThhEgsDlqd zKH7lUa!$FUSX2oJ(9q;N;XVWQ)T6+}(-h{;r=-~i+2gLUq>;()=3OQ_Yl>I74tyVmB}w1M+h8yDv@u6)_G#f&f)M7J--U^s zLl$Jjs2vwc#p#HPX$cWp{&o|TW?g@&Kcu~M`T~E>#C1}ju@i~umrsNiDAP$9sO?bv zzZ7wu4}vIxVj~aG7n(c&kW*2ZD8+IYL{$IWGADPw!X)Z~hPCp&&gEJ;Ro(OL2zEeJ zBv-)p)EnULM^VKDi5FjvuE_+H!1;B;k9-ivw?Kci#2FTfLnXnO(DJyK=TM+wStR76 zEf!mk$IVP~JHLVtVW?s#I8iCZ9cT18JHL$+bv9L^NyQaoNYo6FB+<{KeYA7jqA^;? z@_*-(^QEO=214Bn7@~&x?r8AWqXDfXWt1RSZ3-@sGEU?$|ItZ~;UV8Dn8)j0DLhk` zW$%yebnP^d{>fF=m;H8-E{!TRJib&O54csp-Ri^ z^1j4ySK*bg0#KmL_0KcO#-Ck(E~D`+P!^NW9syc9W3=twf19E(12%mUP2iwqOUX^F z?mD}h9NO2s_x|Ng=FG3w=yt*rOZnd5c@WIwch5<(N339K{4zSW%_Q?0a`BVQ1~OBE z8Up&4cJT^baO8dPwbl{hM4MvGDVB8c3G2`KeXi&zIvilrus*pHoOLYfGKMEJSn#g+ zFEhZG|Ceevb6)A4mx%dDN zEN4L}^g(-lcD?lG;F~&=iwXq}CkI1lV_lT2+RgTF0{dddZj92^YvwU*T31%jr>uZn zFesBx6wjmkC)<^5-$Jk;L&zg3gh`5=P5dvxBJl=bDHe(Q=IKXpnva$;$zmY^nABPU z7i8gtZdV(6iy@}Isykpn$@h#w4l+Z-Rh%D8B%byNo|P9EsCO?$hQ@z zJVBw!#Byc_S#)D_lSL{#(Nkh-Qv!s`PlJ+mD45zPbq3B!^Z|;EwB&z^o;0H$jr0kf zs#JiULKI;p75>|oQxlgP&|%z!8BOYf2sAolR{0%cp8CE8F!Ir3ine#jsd$yhJv>8h z#cjRuhHeF{&HpC~;Kx z4B;6O=*xh=f3W%k8_Fmu^3BP=B|QUxq-X8fc_+Ff3@v2*v&Cy`@ynL8SrbiguS$Q&|C=~&s?E&|WU;?xCzsi40>&h|b9A!V?_c`Y6?UyK zUtJd+e#+Mt`TyD#okrZS=zj`wJgydzp6D$)%0e6$X4_AdXyY5bpJ-Ld{Ugq1pgH;Y zAv0`K>}7&ghEm?uI!`$^-3H-M1}ODNP^64c0Rqhicl^IDyB+STvZkS z75iOuM2QoO?_aF08~y)wmuj#~=G~}F}+COM#T6GmJdTNzxuRf~Y?Y`2jq6hF1PCF2@FmM8alpdLPUIYzH zN8l#LbT8$PMQl$$00O!8A{~M#T+9*nN7|2LvV{j{n-hqYv%)$cRtE0K`mYH&@>ivX z88Kn$nw5XQDY9aMl#8$^*dw296P*>O>u&=Ddk**Li8NAD12BgM&QyYl7A1b`=s%;3 zs{It9bVc{IZ95a)25*>pzD*im}y4ra{AL)oy zZt`Gt#J*fCobuX~t#kG8@DP8VADIx-Rm4px%KfBu-7>?NlT-2;78dvX3&oFEhld?a zbE|rPA?*+zh=_;@cI$EbU3ZQxzsW=3ktZBD0dh&KAMeHWn{e+~=%zrEMEcJo}RF5Z880e()6B!qsVK(!50>Ou;f+JOhlvhiyw|IUrce67qC(G)+9d|gP?=TB- zHhMU?_L_T`U4_iwInCX(G$$B6G|;5}{CPv(^e}le0ck*##yvD3JZ$m@e?~k(lkChx zfPVCFN)VfH?rqf?M(!of8Mcq+|578jTWj)#Od9q~z*T%`Ic(MLsApP}9 zL!JfsknqWESD@|PO+3^Irqb_e$vYCR6v}v+cxnB{eZp_sw1!IjX#G6mw{uI^A;mId_;??*wQ1(QrXPWZqzWFpClmb%L8^ z)Kf0$7dM?xaEM_LP)sD#ibHei!<@!|6k9&yu1E)&kJYCJ+DSv3xQpIeE!bo1^sUv(nz%V|h4b0$hT8)#3d6bJ?;JlCF6*na_{9Q@Uet)Cfo48N?E@u8Jv+L( z+Z-jAS`U1nvtGR&9S2^W4n`?Df0ha@JPExRbH)JTf!ZO0CP?${CIRX+m?Auhh?=m)505Uy(`c56`&aVSo&c|m}qZ1=RiHp42#Z3rS~3bG0(Z2 z0!eN<7Z)}ItZU$+2bb-y9Bygqj`&-t*R$#K1(qEu(Udzm{JNi_8Ad(9&hD=+(x8*I;Jop1Gx5&}_&+(dqJmU?y_HqK3XI$_&kTQu@d zPL>8ma+vv*F5r6Park)+bV;mqIE@Rpc${d}k+Za71f1i-1|TBC)eN{-Z%F@Fe|E|s zA$!bY(IKJG6#6@#+u)=+GY(1r+dc8$P2Dd4^>Mn9h-&O zj9vE>2QEXd5{NmJ*%ih3s?w`{1WwP)Ra~Ndf(oUS?#iA2i-Xtaa8Y(jA*dx7n9`)1;kvs7sh%MaEW#J!LMg6 z5`BjojUFQ)_&*LwIy8Jqyw8Ng$d4h)fEC5QV-?*v*Sd2*!FZ=S>j}7HwYN-1hU`HG zpRVg>Uo<+wdrjslEU`BCBR;Vc0%*u;fJlSTFGrzEA0 zLT=g1^WkS(0c6U)83A~+ya!$*4!G;LxdqNk)SxL%b1ofM{@OFAf+AR}IPZwli@T~S z)XSy?Tt}z9uF&sup!x;M3TU^DHv2LaRJ>Bf3%bxz>M%GOSiUf+ZueG+X>`M6Gp zzNB0IXPrJOg=;T-=lsG|%VrQY1$gh%OolsbQtJox;cX#!dnxZA|;HV z;}gQ^4}te}#Lt7A?i+85_PEm@bc3giw+dhC@SY1~dtZCUw6!E-zlhaox}WktN3030 zF9?AB597RX1%TBuMM07#kXEH_VNvy{Y-9wVRlkzKx)0yNrcfbIrCFk6wdk{Q>w5d@ zML&9!n9D0lGF->JW0!o|(v32hyE~99I$SpHKg`kIQZ&vX*Ki=Sbgb=h8%?t7G;0YB zqzDlm%t8_coM%5&iBwk=O~wH|J!g1|qG+DxP1bRDvB};_-8nlcb9WOx)4=HiT|8`# zaM-K8SMCV&(L40NX3^ecuXBN`2{S_R2_Svf;?AV7!%WC(pyIjtymsxA{~84(lu;m# zd#nB#99A^hfb-g`>4Sxh>n~}iTDQkrcH@7q>&|wc(#oeSGLGd>RTICrs_~Jhe6S6_ zSg)$0_G@3btaRF0ztygh&EhM_R1AKXYc*NE^s~4fa^ccmHy2Xr6tGlzanuZP2={so zP>Cu5`#1#dy-ZC$;@bU(nK+tL&S(JIr~k5 z+6{i#lyXwfjci^3)lmDjb?SYIXPeRJcp%#7UtT{-F50{|srM z6V0*cf}NEfmtQ0lH8fe{bI31|bztH8MmQP68o^h>sMmYEB)ydLd2DdWig=K(bu$#B zsFLD|=rzJ95|1EEY87 zxHh`252^ZKldMi@V&;kXrE}&c77k``F}?PoKl%#djuYn~Bv@Om@GG?25Yj_GU5x!R z>x=M)TcaJOMk}NkOUeOZ8d+0+07}V-CquJf^<_-nPE|8|`vQ~PXpS7S z)LtS&4nCV>CF(GVDNsM!Fr!`IdS_tE#}^3jJNyc6ei!H%R&MW$W1M0{z0IW&7qR%_JKS~QFVOFM zsck0iHR#AvZtD|i(%jAT(L_)gjsB8dvU&@Tg`A*8&_f~Wi8~~Fffs6JA)Es!*tV6AQE_R8}0-IRHML1XP5X!L7 zJs+D)kEEaJBS6MYXiaa}FrM&f=@a)O_V$avm~g%|)2lEA0XsKjnbdPjIdZ&n8jc3i z*brQp$BM!ySr2eM7-DGJx1sl-p>iW{t$Yi+EiIRPr(;jVst2<*e zZp(3zE47;yWW*7~_$oHE2z>l^6rp&TRF zx{bc;^ee=cY}r~kV7Ja*eQjQQ6sH2#I=ZC>`wKX{t-JbU`nWJH=*8W}cQkZgld|!+ z-!dIQ&J+1b$W=_25v_6F{3JUrk>TOruGyjJN#VCbyW$X2x`Xnblyv{}24$-_u7FqN zlonrIUHtZh2I7qPc7M;$8N=qtC*Q4nUQs0(&F|PL%7$nug5Zn_ar-&cN3vXBJQ0Q*tRqdHdDm^iQ0r7`_}*W zmA|PRl#oL>bd6&ya^zDamcHkogxe!Rm#9&efJxJ3k$BkMVLE(YF8qz9@-_QxW!*ap z;gY{L0q7&NK3C%3G}~D(4kVJoqf|pu&)sbbLZ5tf0fQ!P_r!ssK^_vej4tp_T4+f) zcE(tz8Tx1z?_p5QaRJ6rfpEQp#!?`vJ@vyKcrS&^%=E^-Q*ffy+bsKy&$kL8_xoT( zyFEaX^Br$VdED?}KT8MuetYMzQwGKu2Lx8RPK)l|wjaitkS{F2Q;1do0YdSE@;i+MQ}BON z2kO=pspd5NVh#96%V<7xR*`5ovYgE4aA`~{FIyFNkhyRP4ZIGw-RUfl;d7ps74DL# z-ECeh^3AUJm1{dr=@^Bu@tcntA1&i|)K4_zbCCTxvlUM%NI5Ee>AG|K%GT&}kqZH{I=C(4A;Mw(?R+HS9*D{qg8Eb^>H;C36yW^x4 zzJQxtwB9xFQBV#bzl!n|y=9m>QVP_4HOB1&ssaa2(XiWSP?%WQ5FZ4!x73%LwCeKt zUy&I?Ev&{iuY~>Dp%-}fzN1HB(_7>aB3*@Z_-4Me4o}^yjl%A_^sueHJ2cpnBfkfq zpLLq3%~dJB@b+KT>`S)O?Ht*H1qw7B86CHQNF0Xyd7Ac#7!Z)4A7Q}XGZ6{_%eSvH zn3^{4;Rq+;Vbs8zc)+kPY59|zH=om35(2Ru*0U8JoBj&!2Ht|@@J9Py-hm=;X%EhT zUv%6KL$?W;OrBkx2*%7eN;GFMy@3)ye}eWG<2&x1kvR05-gKRqdkukvLOQM+Byh-u zSu+QTjL#->YO(`R(eIS~=;4fSK=agNNB);Pz0IJYeCQ^Z^5KR!3@q;X?BYNq2_27m z<*dnvv`6G+Vzcc@;1563fNLKj|C^1e0;Q|mV?zgen?Gd}mG9y~y9nkbL1=cZ$`E%X zUXeUP8j27es(XTKW660ix~7y`2l=p$d1KZU$9>ycGb^K_mK!^RdoJUNz`EE1`Gf|N z9rG^D3B%5gEl}+-;Y$APKL;|{{y;`5#*Yg?aHIO&v~u(0Za%jmS*Tp#;SAGTjpN3p zh|xo_yFrNrl{8l3op2g;cd>^d_$8r&186Yvl+5Z)Bm8I6{W!ieIB@q!KJz;5=)j~E z{wAZ=vgDKjHi##2_x3EsAB5_F8i_-$Sfr3xCM6^Pi%_g>%2(nyYX>aPP4u&`=v3b6 z2F&*G_rY9hMlQu3rKPy;H+>M|>~L>Wj4ePe-g@Zk()1SA`Jju;JG3TSA)jK!73m4^ zp7mY>^{^ZuhJSW{6xO)2=W;U|mo=}aqc4y95ZNFl_ez#9L6Q|Dek&UZXcsg#CP3Am7ZTNLwvil%uu=_i8$^@`7`d zJ3H=vIM;Q`hPLU!`J1;2$J*Jf=i#j5Vw=zQ7A=(b=I<{hVPB7jj&{4>1ApZc#eYXT zR}Z$s=aqO!2*m*Dowt*KUUe=53~YBy^lH}*63}rw_C>YM;F?IPdeFq=D1WokELxz| z_m8liyA4l99Y>*pO3t@hfB2s*hNEnt7W?Q45zt=7Fm<_Sg^hd|sne2YGTpWv4aUWD zF<*bnS;yrihJ((@syTy18rj~$`kBrewNIM|dCnz9bB&w6h-5Iix0qoHT5lrtka&;_ zU&H66%sVkL=v|W6!K{n~?pPhwXbOE*J?lh?5=!~Ajtd@0gblpWc-|O_( zO3&7)6+*ECnO4vLWG0A(#|?UN-IrV>KCHONS#H$~{rxDteRX6>Yda{rS*|Tl3064K zSG2lbUwar?kx|%InB2#u7GHJz;gmz-0b4NHZWTwh+ZkI%+&S(`E4uYnlCQooVHyr#SIeFC)LH)iK%D%}pTzI> zlA13}I(|R>hqbaMNYtkZB>44;9dczF(`J-~S?!z$7uU!UN-IG_}5=4V+v6`5D@Yd6qce{%HHhkp8tQ1o{7H}sGJM_nn5I9NaF)f(^Ear4{X z-CxSrStuSN8&lz{<6NT?QhVFKPJZpZ4)j@djurO*JK;J^C8R0_7GpvQw+&i#a8PpY zZeoR?Wdf4EUZ=spK&F;P_^GO#klhR3D?D1U?gPjzSIhgYVrp`H+T{kxA0qrD1f3;k zR6!bK%A%0CAKJr7-CyEQ$FPJYE4b9xz)p!^{jC){`^hS!*r0l*7=aT%Bb$L+)BTx_ z#GXZ?%q27Hrg#!=sUml-hSvGfE`}9*G}T{=9IL#r%()>f5emrWQ@0-_=Mr*nHO{gy z_)PlTa3#W2 zjdP9Ly8b714ihm3{{k=UadT*$EHx{;5Y?;Dn&HDUz?B`F+Bp67y!Li?8D-nYZKDwQ z)W`;^JJr+zcD6AB$I~8L&Py=&01{AzUio{%ued)nUOR2hZ@(8?S~9J7V?2ws$Pt<@ zeQ~ulkTs4bG+Di%7@0tId0)~$&vt(hPuX~Uw!gKgR@%;L%(P;$Qd%X<25G0x$eqWi zFs+Iy-;#;Ezoz;$$MoBNs)5mI)z+mq!DLhkb#bp9Y)L%7rRFF4nW6TD%+0eq+_}n> zU``69yjgysF2DGlpjj6;gF!N@@zSa~r(4`KS*-FcOi&&>wD03hoyyYqT3Q@;@gKAP zZa9#YOpN238^k1FJm`dAew#?0YU2=|6EIRzKW*7_|KVW>+1^x*)j1mnSk7a#lJv@+}4Nr%3&_#d>})TR*k^g+e@I>!e9EtWf&{+VOkK;B8gHiP13|IpakL6OUnz zC+%(8+DUL-IZv&ULgH{=i=|bOnf*0;|8UA%HYq`3r}T?$(Mp9_7^^SeLyAExiK@

3?7`jYchD;CAU-26Y|N=Az)Ia6x!*f`S(t1nDSlx1Z<&e} zlAqA3J_dPzmj2C9mc7-Ngh_&Yk960m$)buoapfEHmY>#A9jX&@B1y?mm}%Pj^n53$2C(S78QeD!P~Q6q1f-)OXO@ zTVG*RN2E=x!2j)TAk82u{_O?l;G>Y+`4#%@O4K^k+Jo*N>M_zxs^_D;CUf}~%N$zP z=6Ua1O3FjRuY1HDd%-b(tzC0a65|#Ek%d>VN~-vES1hj@S9;O zt%^-~cKKmZwsiw}OkJQHAg+l5^6ikFa0J!$=^B!>gRt0K&Zs{G%4aXiC`GD0ks6x~ zq(?M_LFkYlA}n(6s=ko6z%WDUJ0wj_eKi9XWIl>|h8EJ|*c+tYgf?nOA|t$KEPSNzLiJ}0YCgp68s*7|e_7DdVUQP@1i?!iGt!0{!aD7K zy2qTn3VIDc%7WfxlQvy0NV8DiauUz4J01RHt-#|W=P>F)srTbzyYW>#7IkSWkj94W z6w(iSrqKD28LA&xrQ!Wv9McdM3hKcy%413T5~$8uR{h(`%MorbQTQi~=FI?_3wYPt zk7?oCG$s=bna{fkq=T|Bdjw%y`9gd)%%5Vn9;x4;`Ylx7bvbz=X0dq%SSC6F^|6?{ zPB0SQy7*?g;qlj%;Fiz>s2(#F{*%wE(oexX!UVlvlcz~$k#pV9w^L^zuE&bYy})VZ zG|U@r$ty31Bk58oG}Ni$EhE$UgfUmD__uuGO-5rv(j1U|q8anukjf}-GB2_<8s)u2 z)n`@tuegnT3sS#J^%7L=QqO|+@Ta~g#h5VGmD|r5D5(F8cH{3O`3#MUh4L&>Wo==< z?(eETg`vkPOsHkVEz5eS+~fYE!npSsJJs`J)TdBuA_zmPJfP18U)5JeW2a_3ZSetx ziCFnCP)mJ5bo+Xakv^^R1HKl$ka{*?S8F&YKIU0d?^StN@7{q>d1kvHexw@N&d5_} zR1cB*9_ZB8hZ9za;E5?fo*l@es2Hhn1NCu`Uc(@dZeX!@0qvubry<@Hfo>m}t0)4K zkGmkk>@N_vBGns@E~pOA;(ihLH$(DpBDnP~^s`+De@y?OL$_d5IaR*JhqUHAD4lN& z&RrdpS4R6)y;Q#w?f10PQV)ZXrX}pWjSjw6(%6aiz|X27y`J>u)N9A|ikedOW{BGr zPI7ub>4#Xz+Z`_~k9DTL|3RQ0lhcf#b;5X6BOX#Y98=rq$p@B=;CR+z_?R;SWP?;_ zv?vLFyS;^>r(#7TzjN5HS%TQ_Swk*awjDgY=7WE04>|18dz`$$3;Jwa1`|GQ;+C&$ z;c~!oI2u$N_4gzQ$C<-fdQ5*O0a@3Xx;>amqc;8!NdGf5- zOv~CDKDb&oZ5K9-t%!dB`U^M8QDf=cfD{+VFf)*bU+?0Mp{H3zK?JKb+oEF6@sF?6 ztx;DRPaOg@M!uY$kLxt?BKQouE^4=|Bj38-5NB?;glpypw6spNm-c>a6ztFMx*5oX z+E>u1$!s(@wg6qHQ9mGY4$GoFtlBz;@E@Dw#VVI-Y96>-w?1zFTLx+L+$`mR=Ip^m`d_`KXArtSg(hxQFKkc*>aObhcENrrbJtkaV1R2R(1r<`!4IVWQneX-Lm6 z=`S>vV_7HpeC#!LE5ub&JVa{Ehp1llh%j#@1lAGPuLjR|4abNopa;-xj`P!4t9i42=DSo(f zW&n5TSSdc~IWo^|d)_(C1g)%^vQhqee8JoeyqDD)>6x9(E@(Z(riXfRu~~Jx#t+=@ zzUNHogzj8BI|F9~>GSr7-(rGYJ)AV6xeS+xTlb_Ptuus}TeJE@ZlRh>>$=CVo?#ho zJ7>TtR~XeJ4|uc!c8s@@b1&7E#TRw>xdHdF;`R{~X_h=|+b@;>Ja5)YuKT@>lr2}Y zr=5-X1IwMbug-q9_~mS#{U(dK?W@5nDlTZ>x()T@QNs}eztrTIi*zME; zXiVop-Tb*z;fHXWH5WeF1PGS_mtpSa2J(V7k6k&rnZ-^&#HyYD2)`TO!ZAewDrO=v zc?>(?G70bc3ZWCZ9bbgLM8YNB*k>lz{yPcopQ#~x1eCzY$(8uj_$iFOep=A@z+?D2 zpyMdz-XeXO+QgPGx{%2ngJUz*-@In@eN@+zNHu{r6YOQ=*U9{j=|=2dVWEUOo7CR*(OgYc8+s?klHvKPcf}f>z^S2p5Z{ap!N_q21&?*rB2W?#fPu zoa$kaN6)DCNQx2qzKKBBlN1y2H`GviCQg*Yjc>xMI!SD^CRWDqtr{=0F=C8PSN{IA zD}HNY2M2xrV|Uw~g^?>R3pM`+QJwkmtZ$I=bUTjoy9zrx2gx_p-$R?%oYvo`sG0;oMGieIs7TcVNMj>N1PYhp>DVhhB7!UvS|*S@6LFJI!_g?QCZ}oM6Fd zPPlRKIsBNL0W**7V=;ym5IbZ7Z#lz}g?d=R%j*|}s^9(0nO*jrf~&o|5dQyxJKwW` zYN(1QcFuhVIl9!J>}L-OS3QQ)UYofU8iV`$@pNJ>OetJLJTsjaKe(;%4aY5NPv<}H zW62Yo}2+S2+Z=?djg-YPm3IYd?r-PUjS!?gTvxcjDAmw7*%vAN*9` z3R8Yx1t@w9bB|nxb#n&GmO<69dWW@=Y7pynw3baRE<=xi+2U*ucbe-kg<%+!)C@<^ ze%IP%dw|Bom&{HBRR^{1y7P@8S>QUDz5_OlMdBdYecoQ;%??OB2-3Epob~g)xV$og z5hsg1%YMM%Vt-U~ooJOa>hino?H9jHE8Scfh6^~Q4?eB~T*{~2h1 zcskbTG80tph8Q`^Yuz5IoCl3hgqZiYHl5)ZPXErpFLCY+zMK_~5!Ly3uY*8bLY&!x zudE(~q&Y4MY%8Q|pA77JG-QuXRAl3Ccd>o)^jaj6$ zfS}zR60D8_@qDI3=0P^=v6D;Z9p~`EyB-~zPYyh)?5&)nUVQylr(nUiqv{Up59 zw*fAGS;9Kp-XW%UqJ1gG*szEz{jqi3_H0a@T}U~O=`qz*{j*`EyI@-VO@hWN2ER1H zp{E*SsK;)695V@fd(>C?qhiV}YilVlnyMOO(+7rON5gH{{I>x0d0?_Z^t{tW5_YNl zQ=bQvSCDDBK;;hmJgyi|&)tethurMl6dWY-Ve-cPSYz->APz>-XN=YsNF!42Bq{CY zqUyp~*B9{J4#@a8icen6P|d;NNH*LZ_!+~x{1R@Fd*EeJZH3XeWSN=5ReajAB@ZsC ziNl_sMA9)z_whYP!_eaDL!}WUjb9v#eny-cs&uO${3Q%I13DpTnb((R%714=X+5vN znZrxu(6E~FL}nLR@T;rraN8X8gGS58=S+FEpE}|@jK!0Gw?PY!*@D&*9hW-E z*SpF!TRQT*p0APg6Rnj6Gip)-N$(jN366;6Gh8YcSvG?I08Eds(q?3;cF2@^zIz-}ZR!w3sB#4$*5$Xqw)ChjhH z1@(C#@o6=*K5r(wochj43y4JpI^1;bSh%Z6z^vovkaU+U)U{+eF6)qZ0b6*Kz>_~F zoU{<1-YFZJ6)u1WwN5b7B1~x|w|#+BFP^f_ifj#WCn{aPfAj|+EXE@(USi+;Y~kNJ zkK(%zoS)o)uKFB_lN9&hmvw)!#N?;abuOfpDId;dQasUkeis>VFA}bP`X;XRMqGb> z70{e;vDsQUy!|9*KN$lhJ&kc_%y5~!Aup47PSpLN;e9s#hGWs=FfL<<_<7{M=xVDk z%eD*A@KLm!aykQUWX0eHgXcnxgY+i*64a14{a2yg{)hI$zIF(kTDWnSw$b?Q=P#go zhpQzU@nOG?yq@7)%Z>l>sq+ux@m1d^I4{?;czcsPb?WAu)JqVn4P-z-X>-T>J=@KMO$P&-D z;Dll9{{3$JwR;QfX!(jMt+f4*F8i^Tj#_q*=(G~HaN!qBy7z^Czf|=9o}Rh5;x~Ja4iB_$k$nPQiOS9#Jwq6u59oz7haVIoQ zTOH9R|5u&-;$5uX&=ar z4o7+dafgaK_PjVmD$ddQ?*;@sDrbi*?!#NVT^i%5xr}&F#2BuGnQ>a^Xh{3x&fANM zmnPQk1%6Mr0BL;1%lx(Nck*kUnZmAtzDHGD1g$}po?q3}QMNUG0923cx1%L_fQE9a zK>%#lG{!?cg9XJJZ@xc)V}9gnmh8C7#;3Gl8w@5%;#8>Ktg%p@u+q?hw>s8{a|hsW zM|5I-xemB6F%$@wtMKr2;wC5xu;tzz51b}lEfqgHTna<-v{iaeac{ek8NxQfm}h@$ zA;~)^Pe^rLgZm|wOR(W% zehiVzI*${?>0mo=snWK5`S@>8yHSerMnLO{qyf}_`99ja^T%4L@=^>g{timNkZ#l# zUkqZ|SHs2BRwpRmsQG_*+qliBqAmJ#eTJZ0+3&`JA z=|*}#l3o^sL-_Me-C`p!BJ@WAj)j4r`TF%w)^IE>x$a zH+sVmdJd+=KpmiKLdUM{P`xj=!*(dR84JWM;*!GvD2kYe^Bqkd$ zW18TM`LH71OEx?^0H(aS1!)~VGSW^Ib6uF5*;dlJW3}`1ka`cG`T?XFWbcJ*F&8VBcD# z#jX+e?^isHS1yi2@;p_%wWH-;I8$JRl)w1Yago4x=E}QL9=MZ^C>9@|T37kz$1%^T zz6Y>+9rwWP)Cp`=+ZgKod;`-yeUSVZSbm$w6qb^|gbo$Q@lfAVR+bt9=N1GjuOrUU zS*aeYnsV}2&cqGEcY8aS96J}w&rE^G=eFSQ#zUFzTUY80=-|R#aYFT18U=r6q;tj7 zmIoRA&gI_L5gOIIptWc8S%kS3(x_hyBQ1r5i#XQZ41WD>tgwPjH~+y%M`cpKL}m`r zm7RC=rhbGjd1B(ZWp6bL+{W;h(YfH-!2pI>hGB8WUO~DWXpIP`qe%}Ypl4zso;mzO z+x3sP7`ZJ1HjYVRfqgf@x1aw3eI~DSg5%4g&204kCj7VKZWuUlYE|z+X@2tQGA<<@ zNMmZEH!sE}_g*7;dpOd)EhF#5s82yXhrUSjm4p-IK^vWDxy?bH7#Uh`~xJn^lYw%|h^9QF8v z)T>~pM!F*TdQ5q0CgGkjlypfH>$4h2(sk(FJd54D^$sh0wZ`yBy1~h|A(AdZ>Q_M{ zzuwSew0;hOQcd#l8CyYVJB4%W)|q3o+~!r>`**m%{C5D6`W%w<9Q#-@i#T^K^#vM9!e|`0 zFp}v+nh{4H5Tp~4a9ynSX~Z+TG6+hFSA873T9~LIt*vR=r7oE1v{QXFXwkii>f0$i z1JaL39+t*^4z`_YuKHUrtt1~?*Sv}fQ|P^_{sgT>RXr1K3#Cb9gN{6QNC7 zCpJLEab???hVoGLTSfY2D?AZ*F1x%zK-gj4sn)1zDj9Trs_MS zMGexppMmm#Q5*#Ii7+~BBU9_$C9j9lL?ZZ(hI(+xDz7;CH>}-qtClp3(jrwox2vBP zQ;+2r+g&hKt%WAxTO|R!_T_BqvcfUpVOWh z<@FiqMCB#8Zv8^Y8*>b(N1=3*%C*eFFT*h__!*sb8w?Boo`GjeJt0@y1Fq9E^3wzC z(R<}I-hDZTrKe4xU4317!}k~(y{IY27wgCd6Qkgj?8Z~7Z;~gb)}6C6%-A$AJdeGOZR*taBFq_L#_k#4X(A zr~^#$Zi@?@(!s>h3~4N~UPM>!=(idlzTbv-taZ?!ek-2zu#YtETQ0)&nKZSbGZ8P> zr1J!vK#l9W*#`DK{Wlhd=<{%W!Or`)fjsx?(BSAm{_@9hyk>X-p1$jkm6tN`u8SqS zPn-z3d8;(9AKyTmlxlKtsttx}4&jk2^~A!9oZC4q#*Fxl+D{Lw5jMAi!fnDaxJ~v=u!34P|{53b((^;r(hq5DAaA;DL5=Se%psDH?iS(APuF7tVJZg_~%?=OUqV`c-1<2cp~(oTAN3Ed4Blhz z+Kc#CFC6nfHRJskSg_;i>v8U2kUC?BL44DXLQI|whkWy3)$NB6{pmGc81@nC-(Huw z`cW#aMWI-}D;Ybr{z-EVVm0*W`<#IF@}16h`PsJ?%n5abuiK~KpP9Luy9=XX@1)wC z@WWXDQ6fVzoB5M%|^;Aup$q%!oJU`^xA{^wZ;T|EZ>u zw>$!xtGy)64+?F6!S1)=+{SGIzc6f`d>iHtYv*jk@XQ*Re6X23`{R}pSvr zbTSq8 ztfZ-n z)9SLp&RT-v0%`h|pjlxknr*huoZ2fMd;6SVK||9)t&?3$kgVL)9o&d7M_m7aCFg#y zzQ1<}pS*T-2K5qHSN#}PEJ)Hkx!D<7-#kFNbvNbqNEfTM)Q=f^8{URp0m1~#-oJxuJ zKY}~fLHKDe?W-w&3Zv;~FgLFf9nS2=u!RrQ`)ED2&?G+&=Uc_l_vsqSownfE=n@jX z;k!onM4!wHFyb`b$EWQ?`=6W=db-~<@mtQAK)ttYcv<4FPZBrN_X)J=ULsQ$L z3vaPN10h{J__o!bKyOA6t%W7$6B6;%ni%|iq95-+IA65V+m~rut)6`NzM&-ShQK-# zh2NfV@x|aFmMrehB7Pb2QQuE$96m)?tzGrcbRJevPp)c}un@O9MX)#BoFrka*lnxN zpS+w8v?eMC#rCu3sip#D+q<3kg86OPvDeAa%(FStO%3u)>U`paXx6077;bZarFd0o zAYb;K&LVCu##?3%a_aKN7&$nG{yrgNM(tBzQW+x^)~vj?no)gH{sj`=?PVz%eLk&C zBic956kb0b#NQkX=GJ?L;KK!1aCWDIlE#OMe`?bC@m-SkNE^4$m|asxZpbr{#`Egn z-)K*?{mVNXqVbEAp4dpT_ zUYP%KKRut)ko#B{3*tqfn$QprFyc!bPg`1yK3`v&T(D&gPqttkWou5H0!7mvvD_}T zpefx8Y91H_hyR>_!=?+lsSR;+r9J&#E0Z~`4c2-*5w|$BksFtF;A7jpf`vlOsaqM}_lIq&i~b%X3&S(|ER=?zN$LV1MU*g7O-#Zv7#M|EjQki$e<5YxD*` zwbbB-S(V!6jp^*5p%KI%kF;&a_z;%;AkLU8Lp64&^3kFDO;lX}?ph2Ke}6^$+)f3; zQ1F~IR6~3tR9-pi7!cPqh5uGX!rYi|KpYE0`{~LSze=Ef;zC%YTbK9Gx(zkf)RI9% zJL8;Rhw+c+R+!&$G}ajTOVoTf4i0x|FQ{e(@dRvmeGPWou9JjO*pPVxVH|q5@4!AB z%}{I3hng7kz;lU=@ zu*r+gW22(q&}+|8Tou$5ORwfZL-r6lbZ9JJ1q){GT_(naS;(cQ!!a~x4C+U%Vb*(c zMK^jTf^dRQ&wYWT=GKseAE-EZPF5=1t7gNG4_U^3PyNm)$Fa=&srE$kG*#E&emF{y zz9S6LL;H*WTnLAdbOfd>s*X`pMss?vOg{oV*pjbiljC6cnzdjc9>nccF;5i#hUi`eJR2Xh{^(BqL3RaqPDy*N@W+%<1qvl_- zC+XgDcw4s|9UHjmi?WZ%J$RE#JD-9 z#rWbNQ25Y)@OVs~edvE2U1eBROBWUsK}A3eLa`MSQR2)R6Gg=C7Q4GpFDi~C!>|gCTuU^zm=#T%fLHYA&XlV zebWRjKq9Y;F4{eafiWCEg++-PK9pgH@gO(Q-ZB=s!x& zu|KP)pobW|f$!*d6UMLqEnEWq6(gWKIXYV}5tT95lIL3i4V=#6>xBQxYSV|k_AKV( z7Z<78C%>p-uUlK)58G+fu-f^!qJ|Dck4F*oOd9HglbZivqY0pqxbvD0`cjR<{H)e~ zb1zisY)55qsn#5e=(;HxT;`&G+^MgoMOD_&pM;v^&>=6yvG^kd4CA2GFq*e=4YW@J zI}8h?d-*IfcGq3L=Dr`j)q%{qnz0-0XmIjm_E)?%zPBFsFNb^-a@g>v0#=D%`Xrmo- z;2C>h`1I9-EcRcISh&qBHDyf-hlM-IU)xG%c{JlEyl)<$4>o^AACG>zRnL`1p9%1{ zUz6k84A*F^fyD;74s;|mPDh^MlVLII{(Tnq91OWazaF`Pw-1qb^h`CkZY{d@h-4?6 z_3Bf#CHHlRfY$w(^?Oab#4>)yf5Niqq4@`@*X3pkV5n^RupS-sEhC*T=Fz}Y0`{rZ z&+2B?_@;uj>Gd0XY4B~p)UE7vdMA0OXX8CDQy4W%=p9qz)?+$2{)E1hisG3BdgwYUR3g0BhQ zFI{L+Mz0<=n$fR3bH_^Y*y#j~s}Mo(O*k|64NqVFlAv|0Ve>DLXFxuQ>(k0JTdDcn zKIGn5Tj1{qnk~!Y07K=*&*fyuxg2czWaj8~3Opyu?n1_RZ65_rJ*}cVj#=Rm3B00B z4;Wwc{OA$N_~|4XZ*!*H`|&;^?iu<}RSR#c9Ov9dfBGrRJ(vA=Osp$0Tn}G&j^X7v z$lcZoZ?2%b`F@2~3fGX(zk2JZTLO4sJdr?MaO(=|ti5bnkkS6&BsQJ{{!2`DtVBj@ zqmN~k3P;7bBp<3{Eyx|e)D-SNb8^lV8+h=CRz~O0Bi|gvb8RGiy3sZ?wqymJx58sf z@`M6h>P|b}Q8+}Jc_{j10IakiHgxI!skmK&l73duLQ-n0;JrL z3IFX(1)n+ciKlJ2CKfP6}vBSvJgBBHu6`S{u!}tg{VrmutYf+|X=`c}03lywfzFJI`6h&=Le4 zN5E`@pL8gCA5|P5#1;HL)3fRC%-jp(QO8Gr!}m{bgz+n7w$&t)hg7T>D~bl^<=}s1 zDBG@`mf(@g7}&(nnZhrAfrf^(u4@uW#*@X>dBO9qoTF9upK*`OOw{Te%YR=M2xy!v zti3dKf;QY$@OKM_=3(#>a-IYpdEl`^-bM2dI&fs%3_>nMlM<#;WTuA3eJIq7Emm+7 z?JxC&&-=t6wLOX4t(pEYZw_P5xvhO2y)V^;T_7F|zLEG{B>k-z$A169OI+udQQ+Zj%S_{U7ox>()=5X;1G+g>mTA z+qtC4J&^sNw#zE`QBDd_7UTbbui|0gbGr5Lz2bM@jJ{-eLqZoolY8_Q23uXRnG)^`>tUmf z_H_NUT_BTDCTBa+sgOdRD3QY`gDd!l>vg+D$S_RKBH#}Ne48*H3m9YiiID51;hyPR zPF`xeLSJ z$s?Pa$~UK4WZ`36Y6`DBQ;|&u^S5oF;YN$O2WSO`PL${~HOQp^8Q;2~&{~0T<~{>!u=@Bkti@f8~y9Din<*MqP!LQ7+mBc9fs*O{vLr?y$TnCsxIoh$U+24{N*V1W!LIxwSs;dm3%^^(MF%D zx?g?|?tO6&EiLOtt(-UVi+BIj^L<~%v@wsk&z5@9E3?1s)GL=|bN6f%wR;DLW?U8_ z6%#mTD?~WneBR{qpB@?4!{@ z#GlAQraZ7HrdbWJdGO~W@zJi8Fe zJnIjSo?MB?-+HL>J{wI-mNcUQ#qqt)?vXsg$6i<5kjecYd&&H=D9!7&g-^?6{Iq0Y z)YwcNv8tUOFS~0~59$9}a2?S_Z~SkVjt%gYCG*uL+nAL+_R&l_RAf7+xK^W+PA3>O zqxTG1Ak5nLa{bJhUoPKd7O(AGSG;KbO9kV3vv*ZaQB3r5Q#-sA&&!abZMv*)&^KeeX{m8Jf|CCk-kn zGef7-i@b^=vwO=S-%9F%J0hsXn)x(dX7bp|C#d-5hn(4N3jcB|VCr9__H7~ki!>AX zJn?+rIQlK_n3}huW~u5h7mf~XECzcPl9L9-35)o}^BV zPm5hGzf;ZIkNBqZbmcqHTNW=hhGiwRQTx3}yDR zRLSr7ai@a{^@_Th##36?s?~YrJkE2iD>?RPM3+`nSDWlO;Lfk|=<%IYBJde+o4Qlh zO&ZG2hxk&Vh|jA0f*`qaTcBP)!%2RA{Yy2OyMjD?T3X`!*kYcSa>F`qB1e&Jrq?M? zKu0-HrPA!M(z@o_TO3+?sS4hQbN8N%)WZhXl%HJQip~Mf8hE5IAE?vqqH==aJj#))bpMK)$%a6*qL}b7gCj^^?0? z{PQHw^}EKcU+z{{VLEm zJld_gT7Ef&7mUTJUFAJE>8qzSbGO`mJYRe~Li|p3QiBJ!Rpr0dm#I_c2!nTbx~~_J zGbSpWF`|Lt5_n6ud@d@~mkzKk%rzd*lICZt9TG&H)$P+dO#P&APx$O;dfQ&`pr>}y zdD~O}JsH>|od2=srJ8a{&DLyi!C*z971boLf%b)OLoMB=DRU3dBRqe1d+04Pc$mA` z93HxG6XCOTXx_V)*nkvry4+D>e@hY4yP`I8fqvn3m##B-LSf9bWS6Zr{jhsd6TTAM zRKE*~q{JtADWheac3jp%SKG3hhVA`q{d42Kc$PW9a`7>=SlzlJcG@^yC}NBralwJ6 zybBP?dp^;#%;&r_*A2QgYmvFObwaLM`uv3Ktiu|~w!ebq!lW(AuWlWKaWZk*JZidX zwFivP%} z<%vY#2DmyJ1dnGZg{T8V2`^D=UrL6oTr3UX@*Ok^hO(bBh{C;l(`xJK+ zCrbY18gWOt*L^(CAKKVb%CDgQ>2)l?EAyG<^Xo};`!LQ*u5*Aq-Nv$ea&B3I? z@Ux5!Mj16g74zH@I|sBeHA~P1ddKHOs{Pi!eD+mMHKenLs{AuMAE{8^A2_G5Ce-24 zTMDmNoKD`*Y?Iqr*8H-XJ%--oX&;uQHJngJ4n3bFe2T}@Qv3VpC-}e}Hd(r%WWEFR z>^t;owJoA#Xm&m3qh`CvFg{w$o(xaeSB~IzQAOnJ+pFoW+c=JJ&8MNOM0Ana{9@BV zj@TcEd!J0;AH#_nn#ph~;WI?c+eK`2!u#S*(sHjfA3b&)?=QEMmii?y&b7j(9Vm zy`g?z;lFUYihTaXa2CEFPG#@|$Cj8RHf+34KBt>%^c~-I`^xtkH73j<8_Y_=o{5G- zr(Mq!bN`&RKu7Az@BWfgO9#4qJiA2wQm5_5jFzG1j(Cos>F6x2hkjvjzQW&Sad~aG z{IbNaa%4EI^(;REpUS6QN}wJc4MwWM4o8fZvj(m!Y;csG9&qKSdAiWd{`vUX_x1V$ zeE+zFPKI;2@NpZ)-|)fpOZiD;JTIMH*orzNXi_n4UU3bcz_U+PRNG&tR7BfF>=Ths z*E+j{-#n~fc$m}L7uByW#tVFow7Ws{=SHYCZ zIPRjtc}A7#h9ju%sV!EcuTaC5ujgZQTJ|dp9=1Bn_Gf%1UFkgwTBb>hcev9V`A&e_YwIC_{V>fhKXhtKV9deD{3~Bp&xboqn;9)Otr1m zMxxdv=93oOa7ArbGdy5whidm}CR(g=hv6xkqE$kS>Psocs%WL!x zp{KRSg)5>=ufUq#HoVS;6aOZ?WsI4V?mQ8#lD87{1e?z>+5q=Ss8!|Szt_6l zw}wu0cgWIY8NOGIM_}+FiW>#l6t||5^8*WDT&%=uSu^d%{^8G%~5=N zb8p~YAYp!0$c}*;Yb{YzSsK~s=}P;)Fz`nKZwR#|lFwD;+jc$pLiyFya$ExN{Ut+# zsE!`rY2uyw!gh2|4jd4}MOMY~*5OMSxGxPBgEOe=#W*tfi+LCI5>j|TL^b`@wus)@ z$(`c+%qE+G1!VZ7%?z(24K|Fp5R1>uVV&^m2dAYuTa1P}`l%=l8`X7Vim4;@#io0)<1rQ1P<&!z>HX;dUsiv`_zug+FKUXx zP1-iqc)?X1R&}$wv$V2=worX~*vOEIo|e>;r!Bs@eYNSc5`q2UX`8Uoj3*+02q)hw z^yFMCZnUQg1-k$+on+UOicU!3l8xwE%D(&Gl<1Xt`3~+ekw4cxMDR59_taJ~ zarR{JfP*r+4mAr;wHTgnDjVGVR^OAiZcQ-yTEK4$@GoDA_+qpHLx&me65t4H^Gewy zxI;q=kWbEQ!ryx~L32}uL07r|wzY&YGk96S2dK?~4cXi$xL#hm8bH=@{fus7%&E~! zroI%eul6=5$No7D57ql^|8h5l~i+{Ns4rHo{TZs0+|GnMD1 zy%cW~6ELV+2!EM)NC5v)9EQ5^oj?)E@U}I)_@`bd$q3YA$`HkTU&}^X-1|vD#MasulUUciTUv zu0{1JmD%Jz@Tdy7MDTtDo+2|>eSXi#$_}2@do1o>S!7KfYFMl``o}?8g zK8dnmY&I#p`FV{h433b4f^*9#mo0?+M7?#d!Qu5* zle=?v!rxMY=YF0z;e>*BwZ2?p3(fM2Om=f}>=CNgurz@eC2|VRHb0wuIL(_5xP%Ze z)Oa5%$7#U-ODrEuw?Myu8-|D(HxHP8XUwArE*gcJ?Wk`xLpEJ}zJ?bRy@#bzp~vs2 z_T<^hXj3y5WK8^NU=8S*?)0nAd1NoU$atBF6M{@_GBlg&^e>9LCYB}C48i{yJ*kit zkn!+Fi>?XY%2k%kqP9!VlgUp?{h7)08o#mbdh=VU{?+u+do~O{<~H3MYVauolPw=Q zx@pHA#}zm`wWe}G?s1zY3`jP3uAzT<%DXf|HfCLsHi>ax7G!>s-dnWmjJhPUSk^s{Vdy{Ofsk)p&eFKXL&5Lq|28Zm z)UR4PdoeWEAVv?f(OU4_%6M4hj+DOg3LTnQPU3+&h5A#b|5N*BSWM;=I@1}Rs|tf_ zX>jhF+=7-U_;#~y#&fF389%M4G4uCyb{F=E1u}Zj_+7*EBkhXG28B+s$=ZPHn2YAJc=uAWxLv04M{Imjw`V)E zG;+RvcF-1M3D>6c6Q*T()m}A|$m9a32TmTeN5zcnK@%5D6U%;AFnN^fa%~Ck*q+Vk z8Jg}m0A4+TFmGh?Oq01}4!3a9g0eJn=tA>*e0Xn?pZVQ!Ij&Ogw&84YFXT~lAa@~I zX8Sru?jhmP0)Ro#+~b&Y)@0+^UR`vQ$W|;LZ$Az&+I7(UM*OF8S?W@-r19BoI2?Ws z`Oq4bIo!&RzaFK#lRt=z%bn;ETNyd4bSsr6%+JhqpFe%je`d) z$hhcv2ENRH8D?@klNG{)k0E4jS(*r52OMO>$e0wgf`qo!(9sN@CSZ}pt43vlZXn|| z(Axraqk1;zM;%+&p(7=uo znbRR{&JQEUEs^`ng+7Z_hsW(&jjqhT*B;CgZEWgJ6W3FDEU zc3BfJx#$j2yU#hX+QCyn*9veH4W0N3e4S~y8<@Gm=mO(I()#*3pr)P*csnwB$#694 zl!h)WXU(~{s%m~Rr_oy^QoI#39GkoznV87^uQ_9Fh0%Ur7Dh6%JoM30OLr$53EVT@ zmP3wjAm|0)cMKUUH9Dhldb+Sbm5V)II>^D(WBL4r9cf4QAF;Ndb(wk|?jXN+nnfi8 z%jgMFrReRIS#o)^uiWAg-uq3wNlqQg>%>x?dS{0(+-PG(IySZdRXCHx>1Y11Rm|Z( z`Ld}>3x|t{T(_y_<6@$Jn-p3U8%4Vk`mo2$jykm~_UfKJRwT6bR_*4Olk#6IH?9Ah zPgR&D`T#jpU-(yt-S11vq~d zdjZ}p$sx^JaJ3Ul$i85p4(-q!KCYYia{CblUR=xv0!N7>c)$J3{5b0Tv$Vc{@v*f> zueD;Mx4quz9gXu+YRD_S>^U#Ki*%KE@g{eBc5jtK=c=u_T+ec{k?TzInOIP-vOhv! z3Kr$e8Z+_TaSZ0FjXe6Gw0<0$tgnx2glDs3Xkp(i);agLQ2Lp0NezIl|c z>l~tUyqrjRR((p_b~Bmg9-Nm}vzm*V-U-iNuky6M{(|S+uB@VB3;g7`x<1lUFP(q) z8m>Q7c!hT%kI;d@k5(~wqY7wRM854&NuRzOEpkSsVju1sI@b9kx3FKqFMIl>^&Qhv zZ(31^f|g$sPG@uI>#s`5Qp09wtRYug_>PKQzDE|Dl^l9@8@Gra%x6#ZrSMsW_2A>V zb=}LQ^ot>Oa@#UnN~*V0VE%dZv}fu^fqZNm6v;13#&W(S8y#HrFn#u}Aulx_sl0l) z>ipF%@s-qxdPeGH4(m}*=W)8gi$etU0{{(b$7NNLke?j4vzVI3@}5B}`EoAGx{o?g1;moa)s^@7@F>Id$y ztESFA$V(Sk)&=`+ucta&5A&YS<#<4N5goL=y>2slDc}1zSz|55t(s0+jp(7VzItcV z2yxxFxC-iA73YTB+0?`BiU)C)cmx%;^OW5R+~cj5Ppb98-PE(*EP8fh z4bMKjoU(gm*SN0SA6}cQwia?~mx?UDo)mu$=j4sQ+w1YQ?yJuuJImt90h}?zTkNrW zYwgtKscPNhFE@^K(5Loz$s^wevGd{TGPV2&?fR##L?4k$J2&~y6OVM%bdp^+w37!r z&ecmF4Us;%_HuERPdD|m6{a3%&+W&kLpl1}QFZ2$zlL21 z`79ezM{ZoA>|%xbx=Y-+-+~$k zEaV&u?Pc=U-PX{DA9%?RC-EoWRC-mI>Ery*mK~{!+1)*Z9|!JJ>-IdOsIT}gduy? zvUOeu*|+8vg*9PIy+AoY*P#W!ile{#C|-Vudcw0_{%#{NCTZ#|V2KkyZego^&yQB9 zRf?FnUsTF_7Ux%laF;$+bf+%;_)C`sSkFlAGJGm04qXwjYH?rrBrTrG)GtbPYWCGZ z4nFds7^Ztydcbo+C$QV7RRXxF(BE8qgd^vS@&`8L$M}Bm=S}mpYh%()I+ilML*t^W zQs~_5Vno$QaeR=X9&gfV@77BE@4l0sdSM2C%F~L9kBHX;M+I{JKDp`QhJ$HS+-%g9 zDjmprJW-d)|5(M1nk9i7GHe^kd;M=y{g263)SG%1Q-(Zp+j4jRy7VOSuFCayp_=>8 zm-{=<)9r({b3y-N)U7}di(hJGVACJ;qn#d9K3F;qeJk>KVBcc&&BqfJl>P%D$YPnypY zxwjlQ^JTC^%y^SWPMq9=fVBdj!PULjV$ajv{5WTRnJvVe@3R8y_t)alD49#&n8*38 zU#RJk?c{!E6R#V65VcrX1N(sq`L)N)8|2%(H2r#fKCe5&fSfMd2kCtxXO{<$dHZF$3A z3N9sJsoJlfa-Y-1Ws=QId49!rPWJ7>vuzs4`u$@VJV@weIjqHMvgW>~N~V8SJ5mdi zxtD7Z#q|8H54hRdc%I@G!L8QdIg1D5$T?!RN|B4%&;JiUOm4#V7u@uOD;`<3`WJgU zEC~?o>>GnGH+oRs5Bs!3jwgiArQI%vMPz|eod4ZcUeg2TVl2k_CvA6u53BQ=m(AsY z3FX+olrKFky+W-G?#`;ZA1w}?hw(Y#*$wOmvfqu9=2q0+Ps_^F&;QYym*sVzc1Kk7 z)pnFGDS`(yenf_I&IIAT=;g~eF8r4YZ1qgVg-;Sr*S}Gbe@*2%@8NuYMHnp*Cn)l0 zIc+$&k6jSw`c#jCGUjOST_bg=(sr`Kp{{(fn7bIBbG^WI2tB8VEiY#3TQ7FFrNB4z z{qZ(csFFpS&vzU;O#Q8Qm4I{9(rc^5a7_93ZRFV#cz?mREDd%aM{~z4AYeJ!ZL*4= zN8`laG2>}YLN+Oz*wc-{qq)sr#iu=LlEDRN5izRSeD1e3QZ1jhjSIf<01nM3qdnq2 zUg!B;dzv1?IdjhX)M)Hm7+saOCS@@Chfe<8NVRS@qW!ubk1Cp}0*@c1Se;!1LpA!8 zOYN*8*X`+#`fe<8eLBqO8Co)-w_I7+MSh}Ovc&rbs`#(HG?@o$)C#r3dqUu_)WkQh z#IA(L7I3(j`!5B!R)lZweaH)|k4ZD1vp@U*z8mR4a?TzSgUiyD)SnD4q0KFe1q`gQ zl7P+Jz}<<5J}E@$hoUUGeO_W6f@Hp9Cl#&>&f8=?QRodH?Ky@3N6eT*;5J^gu_bwVJ+%ZLZ$Lkf7SvlB9w)3Vy*g7D zXB;|f%pc7O{UPGr0|dB%W>!gN^aVJ;S+6~S_n-bbDAW|*8HpzJKh3`=b@K;lTx(fz zQ?~^E){k7hkoB z!n5-$HtU}6SYv&$&l36cZ9iHw=)2nVG}cmdwVg&?YS|=OfDf&v9yZ6-qk*H+X<5;= ze8VM@mw(GmHQatFU_0-2sV4#*4~h-*n_<1-y${x6)VN-CrWk&9n6iI4hxHsry2UZR z?7vL~6u~>QIZHFR88}plu@3Cfs5(7R9@=p0oz*Lh=GFTy&s9Sv*^AwCR`KewXLxXY z>ojN;Q)?<^cDzX5?#Ec)wCy>7HFq=V-Km=T`MIY2w(Si%)~vdAxqpCstp1Gp;z3^PvDGAuk5l$+;u9^_W(_- zU0zoH8%74311}b))7`pq=-g@K?%sks#kpp20kp2}+;9!Gclu^AI^%Dp0&>UnUIH9J z;=YaL&Df8;@mKT2t2ZV)#U?yR5PWXRNM% zOSuQECTN4K-iZ6yfd<$rsxrDAC2!xa5=$j;-hNK9M$2kkFZ_xKu3Ar@em;d--AlxG zrf$6RdKuIADy+mZ`g&|7Z=bi1&(aZUy8j0C&+NqIc9bOGiWS&u0r$(dg_5B=PO#CJ z&>Q;gy&+AxxrouXc@?IAHQ0iGErtjE0;AS*9V zz++25kw*f!A#HG>YLNsyw{n{R0Ox0wtbXtbZMg?+xJl>1`pmkf8E>sK1Os)m9mW22q2e`yAsm?-Y7 zcGJ-5a(#t#qjeZ{D^ErRvsnjddgH}}!}Nv9RWG%a{IP{z*$=dAZyv+B_AE14NrhfH z;yL5<3{2+Z+a9IuuW>Yss|}B%7pxyv1u5u5b>n`9GCIe5*N3cYho1FPyS6$^Ro2*|YT8`Zy7R)>%{_D0+^v}Gh;CFb2 z|8(Oe_{;6ys-YS|vpU4nBG*J79vDJ7gX6IG?I7sZH&pFJ5j;Dk7=3QAkdD>8#of-Q zFgTS96tmaycj7qW*lsrX@oggZE_50%41bO+?JasXog_|8{z~9zv8kLtYQM3bOm4dB z-R_LKH2y=xWV*|aw-+%q83R|?X#dGL>AC0O>twtzbcfMS23HjP6?wgsylKTk=z*h# z!?2&KRRI;J)R1+s>&~@L{;E`!`-DXghj{?@KbDPf4^ZHvUpQS9r z2U-vA{6#rW-On0>Q`0M?M%<&8Up{lY`J0r{I6qUh6_{^4EVNTe39TS{ya^0I-w0?3 z_HA&%@HEwR#$K+=ZkrybLreEk?Y)HrxS61Z3Eq^(ydJ@G$2jY2XA5y!`~-zMuo}OE z>r$6$6X;T>K;gE#A#dcW5}uB`BvmllfsKD2w6d(!=_U2UvgJ*$rk?YDCvp!cE;r=y z($#{VlhG9SpeyibcZgKu$ZBB=-x^PdHSxsv~T%Z8UDEVxW9ONB@f*Vx6uu{S_wLc zfVDinR;V%>8Qz~-2m8@SrwwfIrgwrS)RfA3We@l=guf?^Bg0XCVUeQMpo_d>hrKp^ z4g68-M%VFQQMs!g6=UTAwn776vwRBHP9Z1J(9l+|9?PxPNxRj*!71GN*l-1ZD+ZN6 zrHseu*FCR1@3w{DJtLcHf{pOn*qZK=%+335y#b;aK5$Mg!^Hi*jgG3+F z5Vc1@i%Vb<_6+MoNoOw#;IPqjmKM>kfTJ%s_K#QAn9%Q{MaWULrbnc?2V^8U3EZS^ zwyUh==`NyZW)0mv>^bs-63{yhBzhCxz>&s(*>CXDWE2Y7mAKcx3!9t>xX*Oa0xp+Nr~40Y0GuW$-Rd;!LC6q^SjH2{sv7#0)!%usCR= zSI4#g3H{Mj0~ZC>78<$=&aquCfQ5R?g%>8*A>>r}{a8j#GH{k`rjFzzmDgZ>m*T%a zFmM(g#ZBkmP*ofM(s#@S0Ss2-zxZ+Uk`EbJP4H%%tG1+>4{NgXxLO8xCFa%WNQQ1l z&a~W$9D0En?F*K1_Gbl`%yePEDChlW*LiBo-3heLej|@WnX<9)-JE~s~S1aHy zKiydyTE&*(d3n<4|17s^&L`ui9^I-Wn~qQBc=t4Z+Pi@z{aQsIh_4K0J2gUGw_hn_F{2{H~RDhcinK+%paGo z`qz51-42smnOb1z+0=nG+&NQk703lSB>!fDJ`8xY@f)M=H8K?oIE^p|gsew|^*Ag{ zHWNN;aq5z{b*Sp96ha@W#5aRgnRCMEQeiUM2JZ6(`a6r?%G6)VSJ&;y@?s`?2$);P zFuP3>b*eDL|9|P#mJQZW8^X>_h95yDyZyFlY^%orskT=?)|MN??Q63CjQmLCusI-+$$-7*0F@8`XWwae~SBCdi zwzP}Jow=qi`+HfzzZk~@Re$$vYevH;TfhFKOC`+ zQoC+o`)OCxT3+kGSAoa(rZ$$(_CBP>MIW){t0R|cG*9mB@2ana_*;S-CXubZgQMYe4EC$(sOUs~*xCK8)2;aY=2 zIBxl7k-oaH?0fnW)w=(l|Ab`l<=&mNKKq&-;4k(aD=stJ{HJYdZLmHZxQgvsdB{-> z8|i<2`q7JX6F6{$gN&L|M)zr3MK+5Yqgyr&=Zekl;Mt9AGJpPZmV8U5vizJcEbM4iaNrBFT03 zvVi^e*=3oK_t+nF3AxOyLVcFGa&ui*Kg~ab{mWgUjhADsvp%=sYb72k`)TcT|K)M~ zfLd|U)Z2Pd!W8zf1}R)yoz6a5Pdi*qZGE$n>kM1Z!eJ||+0a4H@IS+i{)JHD5vCn) z=ac`+AH1!2J3X~`CE;GAy?&i?o*m2Y;j=MEsbuUk)xxcz8rkicz}yhcIBzL&xjV*l z-CFIijh^=Lnecr5h5DXu&1q$p>6y3jnpyBH?Kb8y_gJ!%Q3H(m;&Wq5$q<`VVea#I z=QaAp&xYKW^6`(?SE%q~Z#jB>6S>-^pDM*Zddbs=>iNogG8xZ43>aq7d+#1pJKx!A zaa!>_hwJ>YYN!rwSWRDe4gEbX0%y>#=1zXKWmLMOJih+BifHjw%<#l>AASYd_V`(T zxynzYhQ(x@&#`9o8okl|1%FOUR>9sA1FGz}Mou&Qbo)uQjc;Fmw!+B2QvmaB-*S(_Q&P$d5 z$_eJ@Jng^wGUtwi+^20K$K#!g&ts0#;szrLb)|mg8p4G~h4J~GLv_$IFRD{9g-~PW zXT0uGF)9~-S$#;gsLfY$G3t{_%*jQeQ(|<=aa}TkRIx z@s}d|BMx(KJWSO~%;c3-*zzJWfw~-@ zsN3CWAhX%tASdU^RJBE8u3G9hzmI-PP7d?s%kuNH#^Y0F09|&?rk^@~P`MTsrvl|P zPi$O4$Dg^!LpnIfEx8T|;1_lBJHXay9z&Fv@4!#nEQwb`ob%~Epf8+ST);1|ADe!*=w zM|0kRwPno{o%Ho1ZB)0(MJQoz3SWJ@l}z2(`p?v{RGNX&a=^?kbTYLx{dc_&p$^r! z4eAm^|<54M^pTC_{3}6r~Csku16bA{dSfx&oVgM5ps|D zMW*KG&G=9JQ0{tHQn;MH=w?8Th_AeKRg@Li;w=zm2FJc%*5cf&6Uy7Z6&+mZB@UF! zWu107iFUr5O{f8#z8ZU^w)xKr?htVm-qPkPGdN~QU+NwCj)wl6%Dvj`r=8nYs4n5y z*QxGz#<&T%K*#1hu%--%?Yw7BY1M0zUCYw-?Zo?A8wSWX7- z9NOg7UHofuTaR6;XlfTrvaP4cQ~I{0^p?N;yu%gJKT>(+sI82iqK>&|%He?*cw_KB zeqdirR=v57OS`7hjlRbS%TL)$4%b*a^$qXFwEu96AsckRM=#jD$TeR0V=W)Aa+&{aW4|nZoDhOz*fHCjeSm0vjb#ydRQiz zP8HswiapuPvnKE4@Yk{2;bt1G9pfUa{a1){q=(9@Z!5ClNA#VkH+Fc?!sv>`h!DoxC2Cm(rALFX&-FXY^9wqZ~sqx3z zaF|EnI+_r(h~m%m;OZrE%lX6LbEclO0$-`j)h23SMLgp^eSx?%ejZ02*+`ui)fa=k zj*{UOtOfd`tDbvombJ#t`qZnyX)@d!-R8W(WKp%nNoDTI@CzO;C6??!d`NYNqO|;tMfVN*)}>o_Z)ch z1f>ipDD$68RiFN@vH&Z^A)F^BC~uKj7YMUXkI)XQ~~fDj$wodrh5AgP&Zb1?{V6-8XuWPrkTq zO(lyJ^UOvApl0cG_D`IV^Pb3aD?*@u=v?)E0vyGGY0riA#2mdW#Z%TFzD!Q6K8s(D zc%|?+T;yy)4ecgIKi^LypVc#Xg>!Do>MFk8a&xb4d%&t>=eo)Svp&wUp@vdY#f-*?<2@j;n51+V)zl{E~N4jfipV`nR^& z=su9a58U&?A7wMqk&o4zO&71Y)1^bt7+7xEFyR~3*;9~m?x>^DA6fb_$D~<=9#KzL zoH0GGb~n!@7Vl}nW=3r$xK6^vD z*{lp7eKcLA4gJUlZvsvo6TQDy;I(5G5q^){)+T51+2|@8O|KaYv951?kn`8>PUu-N zyQC+9zs0?eHozi3y}XkXs|K}uk6~J8J zWl_F-^Do&<{$qu97SP{x;6V{tsGzN^yx^c1a&E8S_9p^9zkE)yyHE11$#>Ms=jTO4 zh#lkefoCN(dV*T_tRRENj^t${_oyl#8o-DAQ-uo7W^gB?=b^VwQoCFQ`JY>iz-Jp? z(!-;)>0y4ex{A)(zcr)JX!o69@+tG2TTM76=2-mX;szf$D*s?rE6E?%oI=u9G zNSS+C_HP39d=PGGU97fwihsvJ(==mn0(%`?q)w);7Y}#NA@4=oxl^Ax+-qqWiJFwC zD^+^SWc6_OID!ULyGr^Q&g0X6bE{J08$pL88=RoWCra|b_QfoySsK^lvJj^QzplGg zZQW8$*BW$&FkXrA1LHfJ8r55C%%Ngk&RNh?ly6BkxhK5>!(XVyP63SH(Q)?{npzMY zdKUt(%~iqb3MF|S7SMhA+^Zm?^Ho^bAmdp?=Mo+g8eBs^;rwl!d-wgPN)dsw*U}Bp zPG?2-8!poHjb(_R1=vII3^cLkCuKPEY?0!!?e_76wXhmJGqz{)sVQ-{|1N=bRW4^-)uVRLx#o$R7I;P;wlnzC*C==R%!&q8E`5Z9m1(@|@uMhp;|u>J@rlV4MnLA#)AS=TR8z39&xMC0@C| zu%5B=Jzwo>H^UKJpqwUyLZ!Tn>(p~7`PCE@v%&mKFp~ZD_>D{g0vC*vXUDD{z!fVnQJfv5Ssz@Wd z_2;gAvzb0r+j<|N=N;NokFsAdXLUHNTD*d`Ro83A0Q+MZSS?U%wB^PJ)pq+g1zjrY zb{g8hQLzx}}-gDT-6wdpp zs6=gW(c#(o@|NRj%Ao*R9eWC34mJFoEUUNjhOzc~SyWqExgY0sMbyUrD@10l;ZbzP zya3&6Y7Y1rdui_Lh;t(apTjoWgC%^M#Z$>FzX)H-WkN=z?b;qsz*quK)7a6r%G99o zZoMP%e$DNd6!vmGH{DQ?Mm0R1g`?2eJoN7n^Z)6J?$~SRqLqPZ@T7UP>+E9e^}d;s zL#q(#+LC7x_8i8&9r?8ZUj7L~>k>S#sP_loyIi}j2JiKd;A(KsIl_Dryn=9#iseS< zo!RJ&lP|i+SXyscS0sZCEke^!w;xyuNWAVeoK8p^fy_x=9&7j((6|*2Z$%Ud3_dVZ&kmCPJL zms+2!2;nU^MjF3E@FE27%w7EVa^YcfO^uMzn(x-E&+=p@!)fc`O|MQ*r~1Q-Y4EZ% zS|9f&Yxu9?@7e713{Nkvb~wU{@iXv-tNCrVLdzKp#5sRaY&c*do@brZ^Pu`3{D>Ud zL@{(X6(0573a?__ez=jw_1Sm{qYI$t)xHAX)$uBA*yzCbv$As$mq8Lb&1e|-q$Wmp zikA<|(6ElVkRe=B;Af-DxN$}lba8t&dJo*9PvCteWHvf?;yyZmd6U3(x!kq7+UTNA ziI2$4yYbD?G2FJ+B`Y!w5m;#*8=Yiwoi;gL>1$wwnqFeBFlzyhQ2})ZQ@`E2*m-vj zjd`-5uPufLEA`4F>+Q{L=EvZ?yj}7<#?n*Rf1XFkR}7~aZ(xD0(9j52^Ij6VmY^RL zw1d&O(7)3N{i3>mCdRdZ!}GZFtt68LNR#1=%TcNmY-f0 zb%p2Ox6_aE#nR)XY})j#cgxxg|6n{7K_~Gqp3e_j&Q!jwx6zqub%2*4R%Au!&Eax? z=01jBf>vM0vp!!|>3EIJ=yvdpHXaWd6raMLL9*RkzP(_U$$%^-6RVhWI-wsVI8npz z1Wb;NP~c^Ov0H(sykl!wdHu*`o_p$(z(2I)Db5RNwF>)}Z)T(0n+Lg3;;n9~{DBhm zDd?Nhwackr>;#RHU1q2mqFm?-ew;LwTD;7s3?H`LY%dqi8lZyKX{+b@m25p>r&~65 zq{egH+29=fCByforR*9g(dXp6dpaXCQD6TpAfru7_b993p&4}pE&2jF-itmKK1fN} zliXl3)`Gwb@E$X1*5sP0vVjpXNMTfNW&Uvj9>h&k*a_@WOuK(M*b?GAy99=6lzHz_$iae(dy{Bj5XFp zTwaRV{bC4Si3;MmX6SzS3LD)$V4E_yJ{sq=f$NmfQO460ul`2lyctc-~&3UxnJubgS)R#fBdOIG{O7xmzZ1FC-4xJ)!6qHeg*y%9$2%vMv2Rrba3_; zn%a9i-?8)+#h*Fzm_|e)S6->nRmw`^$&B`{)bA6r*&oO#uan8Yet6fnV*V98G?mF- z1a*E;q0g1c@xoq~mPU8NGfHSP39oNzL&Q&;Nq&P5o2;39ZqFl=$ADuQwMWML94I$~ zz;(!#PJkn}iv5|jB(gbcfx+A0VUrl1MsnF$qYq3TBA_Xx=j*Wy3^blmEg2!@7?+%e zqg2S1!DgW!&UGMvz1C(o5@vP~S7fAKTDzr|R_^Yl|I+VP> z)~N@4Cg~}=jOHpaoU>re5F{>{uZ0La4ST})2zoB3ts*t8pc#V&OCLW<04uEwZO^{uhwvpw=a*w4d3g_ zjIHDF`M)=CEiwbHS8oH8Iy&=aAG_n6RIe@|vmOx8XQm zJ#WbG|JH?x7pKdSn`etSiyipur@4HQa*yezjF5c|ZeeNsZxKC2h#X&6exnWT%NAn%Q?C7%4KEf>yiCq*Ya z?X;ZZ4Cc0wrlo$ks9lM;MA~@$u>CUc{7)>7ZopEf+=6$5iZFk|6mS`@C!_9q$|*Z@@z&YR zvdaryY561``b-=zE2SroPJWCFD=Tr`#Bp>lE8Qp8DIG7=+KkP^JJFoy!i|0QGS2@j z&Kma{>}S=~eB&?MbJ~@A_hFpBAGqzIeN;xr!m?{aolsE0hK=>l)QxU=_V?IG&#M@%_e4-z()(X(Mq)OU*~O?pXPOKVyDd%i|v zX(j7aM&P_ZU%>YF4{amOd2~)#3a6)Dh34UBgjGo+PUDVoEjEg_`uAD0Hq$W1tDkIi z*GqJFa))v2z3@)whPcYFKALnlkq6>7p_P3;+!CE7T1!he(O}4 za5TDr#}em(S49&(J*W)T|0O>+fEcfw>P+`hcqOz}tCn;2*6w(tZ&3ElH77Ptd*PPw=qT=QC3W@E*I{@aO*~^0EjmKD2q-G$8MT9&wJVh z85>!r#0s#OIuPu(yc43Kt_O?E_h8oS<6t+ZjWkL8j&?1g1dSP5 z8ujMe-G8&Y?n_a9_nY#XVyj+pNT*!mLs*!%LQsr}@GrJ*o8&H{2K zK0}&U2s{4^3jnZ_Sr(4EPJ3={Xh&n>2IWs`@I-akFZGzvb&%hO?UTAOOAn6=wudhTVi)Z2M!8rn-#89m- zhMUuh<*~$aXzEr^j;Zbe2Q6!J^N1YqyA}XnF9nOGW-Zx<@#U=F`K9uCTn*VX?3~cc z4CU$ibZ1)0hn@<@ch;uM+$p$zX|3~Z#%n&i7G8inY-RLLTpSd`u&3{NZ&WYn-W>pY$)6M6X{mSQlnz-~ z$Cr4O#+rCnx;ne!mq3kK<@Ge!dhOj%c}~jXZ)~|PZL5s71s7_+MX|U zP1cOK*#qBJ{={+4`$br8rfi%%M!K5hFya%Wxs?aKd*a;ky*SF_IS}u%#<5nC_z=IB zY@r&>WasDYVN_#m~ zA}*5fAOGuasCb3A-e2_!+qZBF9BKE6QEkhD?Y%Ioc?|Aw&|=M&OR&f8qv&>aE8!a5 zD@WX@Fp=IX9vEKE{^@g+Z8;Z(YW`EsCJDrp=rH>vTe6YXh`&AnNUt!n={>L{?HsJGMtM)_HAwUrOFcD0{QG7rA3RQ=o_RvY zxd~_-cxVY-*Pf*T;%Ih8?;-Wi4@jQY3zSw;SibDq9mP@nNt27{ZLPDlrZT| z_d#C@MxP9Gpju-=;T>UaxpS3<>XpA2nvbmxH@n2KhP7j4R`Xiu@cAT=w&KyX>Y@Ih za>&ex#EGHB;J1hLdP3@`D1Gx888HxrO_oZ z#wCs3Gg^ii_2g=-iMyoZr(EI#N9pWhC*L28=l<~U0OpyOHbZtNQ4U+zX53XB)xDX;mN4axkGPsY9 zN6V(Cl@>+9C3$g9CjNS8FWWshz}`kRk%TSic6kEMyF)o~;#Z`4MRUJoAUz=RmQ<(z zdxbVu;c)HxH6%V3#I>~ECarg;>6&fUY!9k>ta# z#Cb4wUsG^d?=6*UAdrGQhPMjcK{h5Uho1EdGcd-aOYywIrUjFqL zdrp2J;%RTmW3D#D;UBPl>J3I5i?^JekzOP1(ujKKB-G#D7Z2A=!PqTtF#XYX;!RrH z?MoYY9=B0@s&y&rW1mOZJP@d!I32Ut;h<4k3`fr>0K#@b`adnDdfeh3QVO(0>G zH~Lim7LEVTlxl1g=bgUt7)d+gOG``D7x?8_7^9ldDy}5_kZUIgGNn-$-{_;b1~R|2 zIvcsNSjnb z8ng)J%-;p>7fVt7KJ^I@_Eqsfx2Xd#@_q}&wXE?x7f_m8@oD3~pV*-9TVdvzML0R53*@*Vr@Z+hV*Ea*X$Bc_vlU2;ub0>X3AZCt3RKkOiq_J-oyK3ezmQ$X5HWJcL)&n&9L zlqdJSppO^4u7JVdPB?!qtqn&wllFTJG;8w#lwRNNzD)frs2-)_dKxo1u1kW3^ncZy zHI7YSZ`$9W-=q7qa*yEXZjpSi?o+J2Y#k^McJxMXHZim(w7&2igP(4Kyb;@B_~#S& zC!U_k@M{1Q9FGF|QzQ=7jE>CIknZ32x_*CjbIAX50yK~*xw68P`IkIRW!eS=(vlq+c+as-+AqNf1x;xbcP@u3S}QI zAo)oVlJ5j}&!mZiDfh*cogHB9lUPB0q4WjS^D9s|)S-E8Aib!L54$wm4TS$V_}ea# zHQPh0vIJGbiWA&gebW$j2pV_gfyI)UK5)k|i;k~8+ItRzGY=i$*i5PQ#2g%~DZ)KX zF0psbYqF=QSCP&O``cbncmoFmJk|Fg)e^HR*+yPFTo`6pNHO`IATA{?64=V{8Ys=7 zdP;E)WicA6Av(^hpyqFt-xg5W4CNU-@_q)8Z@|1AXBcTo_NAZ!jA=6!;wr7=P&vL&}2aIr@D}{|?vaS|&p98GG_3qsZ5%RMlkj z&fDqQ(IG*(3?qGj0rQ#x%uUf6%aG|&Wi!7e*;lXF{; zCk}^#Vf#dOHD3r>wib-d_ra@7Jxr`|0K=BhK4JTJP>yh#-rt$KFJFasjs1{%0~D^P z+=(N$z$Cgi z)~NDN`z0G)Rc$N(Cr=gD){ z!+I?y+_j8LS>Ua8Jkq^YG?m@r}8Gc zUiTE}-y*Z}7m`+@UTnZVB+f;>^8Lg|Qz4+`I@oz(KKXlVZ^|iAc~j~cB%cIh$FIi7 zcvGOPN;Iiu!(M35Vp)U+w^TnyV>$p8Uk&%XPxY=x#}Xq{#*lfdGmxi-LwB@n#LWwe z3t?NhFA%OHaX9%%x;L17>N!oJcy-4cF8X)olqV49Uq$cv>7v=b-MBIWfU*{#`k{S? zCgaxeI)XGNWhj$n{Y9I#ggY#2(0TE1>{3<~Ybh1)x{t1-bd5If_d;#bj5AuDe^YVr z(6-X2av3U}Pc^GF3cGx>2y=7YRes4-u5x-@2&md4&xNJa3q}3ySFnburYb9?3`==7 z_UKV#@(!yghbk0mE>tc`x=t*deS)4To((GJqGLqznLs?x#!c(3FjCTY0lklWZzW|2 z*OGBQH>k*hZCm^ zm2%z!W`1QX^*Ui3ye!|X@-U&eR$d>@4{jYL21E?Rdgf;_Y-n59(oa{KhaVFG zHU5Be=ZE-`)}M(x(FeTK^TBa5?SY-QOT4uD2<b2(t)I{131gSS%`_+M+Tx%%^@7%!sGp6& zo9~MS-+P1RS~MOQmZ?4BVTm)lpW~$|Wq7OlQONt-2Yp_|-S;IFAQTC#qR>+;)XyV=jhhp^MX`QmBZKHdoLf^Qu&dG-8$as5I= z@gQmh(lPSyHQagj&&OE1qOQp7_K)4(u|?+pi^g469W|bmtBn88P5yqp4YN{Rfwt_C zG{#`ws2QacJ8@n`AY`7e&09X!mBpXDW$ul>==UL9zDO_!ntxe>jiv3o`n>(3FW4(N ziXZxH0%?2FoW{_PdnJvPCf)itYWn@gL!TqidVB=- zxIYv->%7BWS>s@TND_8@Z;yNH*~n!vI?Qz40~S1OIi6R~a%o(6Y1lSQVUOWRk5^0` zbEj3Nd{*O;(6sqZ_A@Cvh1k#y*X2J}qL$i{d%%ox4APl`>CO0<8RE}wRp>59jC4&_vHkkES{KAJEW zyXsv<`V3BW0H+Vz4UQjjG1klVD`^Za^mzd zVcO*&^bMN^g@;>0^H0>L>;u^fk(1PCWr{c}@ z>fmpehE!8>NpubVq3jG?8x;d~lNa%hE~5p_ANLBf1gc&5*f18F#61(ad*|@a7mi|w zLEYuC>J53F@9Uw#>pWg;lZY89XSENOM2OVb2&@^g2g=TO@J^|8=kzx-(obaj4(rNZ z4=K}|xEV@fItl7ScJGBd|1T?!FF0LO9vp7Z%^s~2xi_weV@-auR~>=Vc}N;dKDXC) zsp?hDDgB>#B4Z$ANFN+{!j4LR=&nJ43Es~2vo29!4eaB z=v)iZW$nN$v=4v#aWZ72_|X`Qk?RZ7(b9C0u&H?fjFL?8mGu}l_&e=&I8m2ZEIcQ? zygg+!t&b4ddY2H6?wUSFZnNoaRtbKhA}9@O#C?_@!6g$~Ga3W= zcVG&J99k!fGSbl@QL9lsN@E0s9oQ|&6YE~7!|B+Oun(=;YbAXSeCzcct;QvytIY)H znfVFC)o-vQeY%)aJ{KBcfD9PlPMUWQVVZu+Fz&=<$nLk9QO}FO_I7?&l#kVs zan(aH$>As)&e|Z2Ay9ovf6uQ#=YdPFA++ChMfC>~KC=t;9kd?`Uwa#`EeFB@I5ICE z3s%mQms?-OMZ12nA>+=l7Dp*-ZEz3!92v}_j+W!*jm2O|-YGDs8Gq+z#T^nge4#@M z%$*3^OIm+~TkfGd=EJc|K)gu%YYc>8+&sK3+good4)yL0zkmJF()YpA z%6f8?C9+eCHo}?%O*rAS{Lr9@Y`s1c$0y7M>NlLUB^PME)Oo?Om{mBhx24l<)d7DahZ2ocRLnNJxrTp5iwTxt5E z{Nk`e4EV91aLR;VD_Mp`0kpPqQ-KqcE}_Ekpdszix$|%wUUP_6v)hBO`{02!j;7#C z-wT2;94>!uf-Zhn1aS!?{18q_X@YQ3Q<^f1mju6tg4=u5`?Uj_XcV`A&8t2*&ME3U)0^@1tV%*(5$g?ljCw9RdIyk_qKMX zT)m&>oOOJ^lhu7O5hqTc1y|d2!X7PSpp9)^ndWnf&d-X*ahiPl@tyWheISp$7bLn?r)$hu;?5#1Yr)^C>g$XzQ8#5q9OUzH}eY zvv~5p@485(H4YYP@ZQD>5plM)v@&~)^j(T$_$%FwR3lsAkyCpx>sE`+dQ59oG@3$t zjyGp#S3Spm;iEM)R_H(WD*LZXxG){|S8UYSAZ6dMqtgvWXiiGK$FcA2F zabnPz5zwL82T?L04%i-Nuxxh=$Gj_~xhg|{J6pxIRlUFP@(ZBxz)i-QY*iacY0ZVs};i~Uft>}+7HC^HA$W##+v`<5Gg;aCe zzXKa`su6UU`dGxR{|vctmMpTh1GvqO#{BqoaIt}jnlpA_U3KX?tC76rxe2vedoaov zDgEyJ6RUje&~%nDeoh^Nr2Y6?T4RZ7MZT?f4wr+_%xYYsZ4|Rq5=U|Bci}Xh)_(3q z`w-uCkXPH#+WwzvQ2o%poTCkR@a8}<;`KBzofSqscNzaU^^tB}k3)k!wOGMzXVqU2 zrneNRe%ZEK-_-HR8^so!Gyyu#I0ZFF^~7&GGDVkwI)Zd8?mF0v-)z@_KYDE}cjwQL zs=l43`^idLli``qc+w(ksy_FfUjl2JHXnMn3h}&sTb z-ZRgzfYz%Zo`5kqld*T`PaKo;4xTjIMm28EzZCC=?_Q>Yv;rGT>&6zX>CTSyn81kN z1Ze@*K)hsA40Dh)9U4t{r{{HjdG^cdQrC44)zOU8L)yRgj@cdt#1l|i(-=r2uumh8 z!{nCE^7Gs!FwLkn67Rw+6I0sTsxI~ILGXyQ!V7^n*wk4)V8)!5@YgwywQSrP^>ie? zK2X-Z63_S5-w0uAE$}ZM2bv$^vbl2Y;9=1C?H-Jt>jk7gf%p(R`ZIa^9uG89-^-MKDp>gm$X8(Wv5Cl*USuKGjluKsDM9+56E-uf^pm6;A^lE> z#;MaJr1RF6U#!O`tO=FWC$e=<9sW@3MLuag)EM!SeLQ!c5y#Q90{g(d`&RrEUK2@w zLuf!AcF`Rn)cDeSsJ2Im$Yg<~Uz%|`COYQ}z?SxcMsM1J(zw)jjC={VdmqU{R`16z z-*h?gGp58?lb&eENfT)mR??V&`N#mJEm?;_=kY&!_O##JNbvj-2iHw{lLlIk@=hz# zr2c$QCw-|f+VId6P+XD_c~|j`R5%mv-V=Hn?qi+Z3gP!sOLg2}7w?ZWcWP{rdIS3& zJugaP&fq_}myy25N$$t-z`70MQfM9MHR~9wW?%xU-&5Eq*pp$-6>cjneB;S;Ak7a7 zOX#z?j=zPp^>^f?Q}8LD1;qI*B_>f^Y!IO^hH9}p9I#mhq<^%prq$D^^K6pCiBAW> z#DZqLY}8&xb1&z93ubTbSwq_mJAr=2kFm%Mz`bi7WxE}dQ2863n@i|EgMo5D&qPK# zM(bv@4oLemy!KYq2+O0om?Q}I@v-v(X!qR~4G+E7P9CPuNsEd6$@^LB6V+wz-Y<~9 z|DMR2dqpS>tHzG%7gWC(&)9){uMPE44jXJ=o0I1yy=aF0M=ii}btPnWTM6BgPAbnY z%#TIkgQ!Ou>ly=at?OUqX$A2EyR$PA&K~pu<$=kI;oawCODO-i-&aeR^o9)&Hs_R^ zXq5I<+C^ba zjiLMq(C6ZtiWkc$GjP2j$e+ktPj53if2cnE zv&Q|)Gokc0VY(ncOnJs3*kI_-PxU#XG7j!ie-2yXxCW??F)05G(%6W`vbf4qsB?YZ zVF){WFb6W{JOmmiRc!J zl_|ZYd?9H+w(Ykakp5sveK%=n4&|vuH8|lY)nC6Xb0r)t+|nsW7F@KzV3N&`XOz6lU% zR0N~n4%8UwTPwbY#>*QZ^*azRX%$W`Te?tE#)Xa@=ZkjV(}28%Al<-Jh7cOyD>~HP z0+dDImVT=l9SLs#WZ`vMwkuOoG)Q2CwEaXH2)PXin6XEDWZ z2{x|1R&k~5-(wdROtny638%~)!iqAQLh^(nq3Yk{K@+CNlWGw6htwmDR;+EftQq@A z>tWtrz>%~*`!j%^_o@FyTcBg0q1r(5AlUuzZ&Y50v@1-T4Kl6vaD4x%AEoo?4abP1@3- zs*Z{81mPyG;O?YHg0NeGr6f+$oF6(Gt9NZo^FCgXMr1clJK&%(kCacBf0nJ|YgQIe z4){oAp={aG?`UvmkoI|Gh4c za(M~W&O7E66e}pdQamG9>^-MZe52=9U;fvnn$m<2w7NEVgjg^*_?#Q@GsWEHW_y|&7%@q$*E=OA9 z3@Tovo&ddsBM^VmSCS@CzL~P&H9+TCm3^q1Q+%On!ey(Gq&meKj!aaSJs+N-rXDw9<-{8y^Fe=N<3W3qB?{D6dDk`CelgT`&q43|lXQ=C6aspMGik12jTj22k8-EgoRxP=x)~wvr{DEqOp&g!SdG@!^Mb=s$QHl$w!Oyw#Y~G1F^JvH4al z`f7KASx^_5z9$lo&D@H!Mi1ewKgZ+C_N(lN-^o4Y{%A3tf>{4U-}=*g3t)}d=qPeiHNMx5Wo+563i)sUaJ zUM?+d%(ItXLLFO2Ie)}6+6U4Hlf^de2)*Oza?cWa|Lr7OHQ9$=J8b3sBl(y%Di99O ziGp7nCyI5OCdlAi3%>Us=Q&}2tBx=5HR9`5UHAa!^<21L#*8$3nSSCJE>4e!hp~fs zA8#k1x#4uovgSY+c>iS@MA$EpsoH;7&D$9kY(9&|6SbP(8=@io%XIi~W(`cp59XEU zd&;H{jb)2zd(krG7}W4=!&k)&gL+dBYI{%ghIwD7K(;!sWFwU^LL~~e)mG(B$%(*(O9^XyUIEkK( zT8f0VnIdWX9vbhJtayN{wEFhad&W#!Uq1g06c0GSPHjC6?sos!e`QuO&2Br64cx?? z9An{7?j}4{ZYjBA4<2j8v)nE?38g&o0M}EMqomYvgr(?Q{mjZ&w?qF`0zMDf4Eo{ArQ3$uGP?O-X5ooajmbb--@zE>G&^Hot8@A?;>vqQBMc0^NqgFWACl2Wx`Gk-|n73}PXjM40s{gh+ zO@*BFIM)4UQ>0@@`doP6@`QPL9Y8u>_Tp|F1`VAhn}lSDZc`3pL`xyY%97~yM|14sK`!#aSjcRdvnzjgGDN<`QtuQ_|(HZ9M)Q& z*Oa*q$DpGf*ud?30d=$(P_bJ6TR%sJ+}{ZIyI*8hqlWWUzxr#NgtU+!Cbfm70ako* zjp|6}Bo{GSyKJ^4w{ZH1E@HU|o{)!`@0-gj%ewQ7_RVSD)=0~WHP~|fH~bJ7$q0{C z-HM*aPqP^-&SZ4NH)oqe=g zGrD(v=EY8Gp4G~=OTZXnGSGx}Q~H8I1{WZrrx-Id5^+^Z!!tjxvKyYY;02%a^q4P~wm zS@Ms&3fsl-_12QGQ64s_4}P!m@WjTqz>RmxtplBrFi*V~Yj_^jv@31G{X6UOZn|E) z$LB9}?ziRsU7LAu+#vSo@@bfEjnH}eE*kGAnvd6AgTVv(NVij|FeoMlVTR81vN7$#a!@)m>^6G_CVc^=)lQn6Fj(A*bhA*#7M~8x`?GSe-r~ zh@<$fTidbgmuQx~w;CT)R=27rR4>qY@jZbX+4x=c;M}i%up+A~?my}yRE-;bY6zqk zw5`HZajIjC7?2YOcdun=J!(XPoZJdXC&xf{%jE;PIwkv{{38iAg4OUs0eLbKh&q6(*;D`N&kwKUw9iyD&c6jOLp^{V{|e&g_9f4Njs( z)ObuD+ZTqfPZZU5ZiAi$qjCHG)p%r+5n2}wz~?DhNVP0`CG-}=3s5=QAD@a8S|hC) zoSnM?sK-$09)&X=fghDl;}zp*T~i|msI=b-+7VBc9>4?nufQt#x3=r#jZcZH9Dh;DiUe)W(NtM~qSZAp5P&(w5KJgv9Ac{Ho3e6jsu4 za$konkdd~NQBA|IH=dv~ax^?^tKkUz`IVe0ZO*yY?Ugmx#0zublY15=zy>jII^ zkyNiTd;M3K{LlzPIvGgS$Ljd~?3dzUlP#dMitm^~?A-XSVnEPYSmvU~X7+E%JOixh z{*rc#Fi+m|PRCo}zi2TId#UtidT%ZVM(aq@!Kdl_B%KRC_`5p`NI8lN3sy0=(RW3Y z{E4R>2RX_eU#nx7L363}T>sd+Xik}r(pM$h9Xa8@BwR+~Zy@f;CXQk1Sgl)i$3Ob+ zV4QCvn#>PnwP)DMquVm@iJZ$^?!}6BXI|inA>TD-mzku$k@`Zsow^T0BR4_qyNSYK zPAw$-hpgz4NSq6sog=(mnwA4;4N2nwv+PsY;;@x8wollt&t>RlzYVweuOPn4hQz@Q zkbPz4vw>= zf88!XJc5RevQ+=UtnA&;)cGB3d0--`$26+HkFV$`3)i#()mN7yYw(sebz!q-7R>CQ zsd^APMn>VrDJ{@uc(9D?-kTE-%Qgk8MS{yypx3BgZMb(sQ*0O*rL;K^uPV==8NVwDmIO|NvqLKIbM8hU-{GNZr^Ct)yu>q;S)F#J zv9^W8pw7ficaYvEY|f5=rSbicaJEWwzkID>_E$~;X?%I-ZyuxHgBH`A;a2z(Ug%HH z8n!=-Pk-DI7M<6?wTOB^nig+NIfQhclCT{r94$yQ==1H7)09Wy^VZKp&3~?7{j~;n zzukxtZ));`1yfi<`VxcY8_Ef;CE&Nu2xr+hlz)9{DeZy}2PX?P1~f-F+%8RH5-=YN zn~lf3(nfN#GhG*;8dDhlAFW+<_gV`<^}(j14H6eujmhF;3eJRb;PkJcK^8>fy>} zK``R08Bf`0DjV$bf;|iuLIUucMhVqzEI#WCy9om#r?UPSFqa`e3MeW*ZlxGXQ zaGrW22-EDEuniGgFxK0#O3RXN!aJX~0DV7_mIcy6Yz19MqdG&k^L?<`afbM>#Wm6+ zfk<9K>yUX->(Vg-FC}}RQ9>Uf!c%~zNcAM?1X_o^K!+3diSmabqQ1pC_@jRqj=wnt zGvAyB@_&qKTqcjbq9M-*XRfx^WIkL+oHt7oWJdewkiHyJ+y&_SgzEVSJi%A>q|(wzUQ^raQ!=dk{t)WF=+3Cu;nKIYqI~rU ztQdE}n=ldy7a8FktoqM|v;kc|qzp%%%V>QjoBVbRNHD0m*DZX!7hu4!j5N?MG zI!4_3t`+%OA5^@0#4%d=Avk_7T8#nkWRO8U(^&aQNRRlAZ${`=jS1;vAiiM4Q6h-e z?k?`6&vlbJq2d_QV}krHCT({DK(SDxbqgAz1b8LU{)2gSpXI><6VTtk!k~}ulw=e{S9}jb4fVhi?eKX+3 zt~$K#mK>z{M4Bfc?e3irdk4bWbYQK*8?%`FFr@E+zdkWl9CB;71>6}L4N>OCO2dM} z&Cj_tIP>2@YjGBVwvV^UMws&5fxc#pdWIP#T*ju(laY<@4)^DUX_ao7U0zKd$ftFC zb3dZ51Ks)TmMh=?vl$%E5w(|4kq)`2;E*_it|)5(fyvUr8JRq(>w?P0o<=>zCPQcA-`h6iOL~ysc7hGI4xlnBbzZ5<3*qNI3x&7ZRWT0~etDM*;yii&HLx9R@T9rBfBBj?nW^Sql)BC?AB`K9-zvF%i}#7pM=#`jI76+YOL(2`4=$ zLmX!+zaVX0M-kpSYe{QpDHl{8TvE2dDzhCp;S@V&bx~;~JbtexZ{>cKvZ0ko9uM0T z=z`9<+pNIU3MdW7k!} z-)R~u%|ZE_Xn*w>Q=0ErMTtgvaq3~5_p-By{1T;b2vi<=?c5Dibw++w(;%@2e*Rsg zd80>rNuDvFyzUoHY-Yp=BY=E?T$6f$4eLiHU~^}zzlW~>^*<~IdL9w)8@mAeHGs8u z(U*dgroP{2Rua?wQme{)dB$H;yv{elRwPUn7iL+jj7?k{`3RMNJQz1nWlW&I zI~LO;7IM>)Rm!&mVJ_AEUWKH4b7#a$%%kBkI4&x+ zT~4)7UKE>;T*dAr(6e-(7FJ=O8aLAZjCxoQwu5l*gB}grR-NPi9_R7kM>|f(m94O| zFy=n{y`iS?ex$E*O<~;ENRHoJ26w|;a7CNia^968{Q8AIth7-W>WuHmKYp(ZM{*Cq znuF^h7Im?7&OG)hX}nB%R|4&_CJS$uu`+MNN3p0|UpyMvo4?J8;>rC`>^w0NR@nRy zO@=12a-VrnvA?dI|FHo-H?x|w+Ej^SJnQg*flo2{VT9)7s2CnLAQ|Gnmx>$nYVu!+ z`8f942^-|7;~cDw`|My|(He_cNAPhWXA zVL7I)wvqLX=koCl>&PEhL&fTK4bY+Y6n=`oIKkaxz5S(dB0z&zE1e?SoUF!(~LR z<|1N+2hM)hTo#$Q!mvLZ(5A&W^jQB3A55opO}x!nzyW=vT1UNx&3O0bA#&v<4^DF` zdag9%h6zo0OroWH{?nWJ*(QjUz7w&>x!o9>9!$ql2)YqjeCvs=_$2HD^Y`)udmn4Q z+^ng5pks%Bszo8yKkV3R0s|NAhS7}|i@p;g<(9<3l6nbCUfc3zbXo7Lt3RjO74?2L zh8ml^c&9NTxZyeg^#I>KZZJ=)V=13z42K}{s2h$hWI^%?)wiik*twcby3iNOi}UfJ z@kQvov60L_@`&x}VT}!3+FV~(sair=G2oo ztr_hjne|Qt4qOb+ZWcq2%UAJ0poyG$yf$>NGg)T)#b~~^vXyI(ddR$8)zI2|6Ri6i z3%`flfY(WBn!7f3TGe|8Z%u&9J?J`u>n%K9zXW=YDS{QXuRwNnYh3w_fWVTxSSgiA5|08k%@$TD5-HuZ~eUYIJLY&V9T8eK^^ z!Io?t!nT+0#H*WX3iq5}sJ_<1YtL&w^MyV);;S%3&4B~mqd|DU3gtzJ zFx8W%x2%8>XU~bdn{H~>Whj#V$@`C82Xg2ABSpIR8!ZrarhkB>$g7mORWd|-Zk+p^4zx~6IYFsuMorkr| zhe40PZQyhDGpKr1^Vzbb1H9~#gwsy`K;OM|FGh|95Z+->j~2Y2(NOqhIuB@Ekj{&% z*Zpj*cx9^>*zaEp-n+v#O#0mx^>3R?szLbIt{LAuHv&41F`zN1DSr zt?X6bog1~J=T5?|gXf)lxcuW|>^tqG>Q6isqC>UtRpA7F@HxW#l0V}JnF7xLUDNJ% zzm4g?YT>L0dOVe~DQjOZ>3?Vv&*%`Lec9p%&l2VzUmyiHi#S-xM9_CHNm9adG-#8DI} z#YVsa3$Tm0JFDocfQbdz3D_+b22v^}VS-91wuAxPofWVXuoJM0FtI@WX7BTSfAEN0 z?mfG+Grt(l+10*mO`sNC4HE7q;Ldq(VDgLy3LmwXyh{bGgY0DdNVHt=h?(>rEftRZ znr_Re{zL=Iethe^#LS>Z^{~~x1ylzcA#d|myz1~D(rYjy=(JjE7_-+~4%jr2+xHm7 ze!v!3I`}6Lf3wU~3zctd(Czx{p-XKbyvHexqv`phKZ3AQEVeu?>UO)zj1M_U;$}Ry z=?3d$JPKT&wZc-LI4qp>2O{=>(kdC3=B~qWFRAC=)ko-o13ylm)mg~BPYB8iVG*c#Sg2RiJ}X_~&_MY((dc3Qy5tauSjrvUxMYCFjg=6Tmc9Bk8%1P_6gA0vt_nv=Z z%2wK|>3Ftgc~m=ofSy}>v+M~@IN~Y^M>Jz)W8UROh{A0-e^E=mDPj;Oz7dyRYC-YC zlHHV9 zmqqCH(-o|W_QIzEbHzjad&4-5(qdHa;QjBDR^j&BwmZ=xDx22cTu?0VY35os?)p63 zUj=dLQ1;6)!tXrsrOJHqj`Zcx2Ri?CTn$ zY7S}b(aDOQcV5;=E9Tqt@=u*8Cs)aNkAqMS(Zj_RbD{RT8Khm!(9NI$sQLfQGmtcA zzHUi4-euRR{%0X^kyh0d#Y1FfoX3CN4`H^09;ckaV_^xPIA(j3LSZIuDcvhm3@!+^ z;yv+fF?6K!pD1tG@~C6tMzRKLS$D!Q_O0+oN-s&Z0XDa4Vs1qv0Wl|l_zMLE62Kpyi7 zsn4rTGf?%xR$QJ-*E=9A+rMB-tm#aL=VJ(8JHyAFFEzeT_lwWP#jx~oU%8>jMV8St z0Rplw;&x~Oq=L-+LMgK{;;_l_0f_NYDV_%5@VGe?HEROroU6N)3 z#RWGS|0h_Lnw)eDTN4_i+0p-p^Sc`_#Q=*FxN|ySP0KJK{X^qRWNlux<{=MT5RPqx zm^o{ZG!MK?i-Wd4SAb&7Nk8lS*u<+F*ooVLir@H?-Mx`CA~a1JC0AcG&TLEPTX=nL z%#&v&;HHS1pqGDD)vG4xs4=Hl!K?NwRlX9Zdun^F`8);J4USkS=88p+v7VyT&@UqyG|A zT8eZ9w2PRe(~=G9H6w4b7h3*Jg01sfh?&WQxs(0^_@&>Up0jMtN%P=yzjLhP^i70c zv!&ul`yP*+Nmpq44mtrD8|)=v6?1)d0R6vRMB+#szc`pQY%M`~07pv#Vffzqo-jZMF8cP#gL$+iqF zMAE5(d@EBm_rNIqn8YyDkgpbra z^s}}V^x!z@m)5Fpfpl>@FDi8e{-b=pkVZH zUi;k#<qcG#@^G5DwwIJQ0rKry;%YR%(^0NlVk{}{P;%>#AS}nPj{PyO>VKeb zU&+3Q50iwG=J`YTvE7N6(-~{tU9SA5K@e6dfM^nQ#h7F30UW6BioKt|*UP}og6k}&8A z^-p#H@dROYHJ+Kei*>B5M!qeLc*F(LE{zZwO>I zq)pj{#(ILZCs!W0;f?CNd1x|_4^|peG+wZs5k4X52j^BTw=7zJxJwe(N+6C8y zC#|=#ewSL2@8}2eLKp0Fa-*m=q*yyYAXmh#*-aYoJW`y&AoeOc+dUWie@6iE9n55n zVVhYvJlH&(^20__?*u~^Ty&1?)J=?EtdI0rke#@WuU~Zv%pPx7Yfg0;h{U<9_gx22 z<9(s$gD+d(5q=+j;s~cR>U*fIXP~qi^~tw``JH6YJlUgpu`!kSvNnI4v`A?RPTGL4 zHx3e{H7T}+I?SKvwjpzRzy?(Mh59h6_Eg_T)gM#!L0UmW`k5G~t zi5z(Bfb!Nl4Mo}u3MZ|kK2i`gUY3od9h8Q{txE<$MW-o@`fTjq;m6|MB(3OKK2e-@ zO3?MMNXKd4UChLYRZBEO3-#rK-L$s~`TU!`yXg8}+8Kd}_7O<_8*{jQCgmFadxVw! z%^`feCX{b(wet!nFMOj#sYYq;<-6}Q>f2zm;!V1|SDKx)JS>|{_u1WjEmS{3U1vmR zQ=mMMbPe(sMmnAG?D!L;8iVDn_lr+wZv$x}ofdii_!|qNG9o2Nx6x1U8P|JX&7chX9TUU zbBvKL1=}yn;f(8d=smRqs=RyjIF;oKJ0MNWl)kCGD1)gyK3Q^J*L!%I^i30x86yft z-+}%m(MaRqhnAld#4Q@qdyM)ZKweHu^{M`qs%Mgvb3;IfB#r0o2S{tdNPlRE*O(eN z^(_SHRhj(Hf?r;omg!!f&Yc|RjO6Lza92yxY5H>PkZ4AFkR|kLOFl7-saPnTUN^fT zD4(tH@@aM|dsE$xtF={nG^(`&vwT-6&x(Ox=$^0iP0-nHuTIA+9w00iwxvUKJ(M!h2CZ8X%YRJ{iHm~KmWVxas7#p}6d(4!vma@9ceyzR}1M}T@j zs^>}f0Rv8rJq^?!(RMRys>Vs!Y)d}1F?8{AtK}9OAH+RoO(| z1NyYdQZ=LX-p~WLB~(H3_Qbi5Dev7CUjlIeC*0!1r$D$0)I%fuog)-)D?XyWsxHoy z_l$bRaG--HY2c6I=vRBL`iKGT2eZxT2qXKnBE6A>>n6-q8lL*F>v8nbcTCj*=>q=k zeHhHyydN&KpnHSXPXN++8p0MNZAch293r+yGSXWas^1m;V~&oOhP1n^i%Z~yW7_E- zB^h+^EmOIBef@P&v%&=rRkn~v>on#E5>5l{{UiHMtIFH?-C@^Whr|AUZ{Ya) zQqbGh08Y?;P*X`yjE}Jg&YR=-#f`Ew8W> zGmYi`%}+(u?cHdv_kGy&mlePDrwG4}`-MfHTJpZvpP3*!C50RAUM1Bb60;P{)y ze7N5ye3rEq^E<5M>~xij;z4g>@@sRd> ziK9FkW2m{DUswJZu^C_QUxV6!dYt9=mz}qsM}1#&d9U6QY@F^3yNxDfroCE@y+RIR zMXQdynzakBNRAQPraxgLK8?j0HpaZKr#tU5f&NB2r@!p4Ifr$_8}Vn$BY3UGLveEc zRGwR5jVteT2d%>%Ski2i+|si+bJvAwIRE@HJkY2g24!>v{bA$qUB{ZZHDnU6+hv-> z6JJ!mPJxbJR`LNxt=P646KTA)HLBRoA7mk)**q79U!H-Da~Rgm+YJ4_wUhc+ z=3&l?Z)n-^P-@E!nIVbjD}%9(>mI zpD418IDIyb+&Yn;t9gPZPw zx~22c#@T@Hb2XBk?(W7N5u0%K7ZXwBmm!?3)KK`^yuq|BVEkrSzZv?6OPk8%iv zJFO!5WY+-L>obe)*Y3)v{eFO|_D-os|q{s?a(E<4>{ zt;e0iyQ6;1sbE%o5-ydMLr&u}+WCepw1?7+$2IuqZZw{;VfV~)tr=M)mKH*cb`y?NJf6q(pfw#0bezi}q!R{`0 zHh6&J{U5{Wp)KU1CRXa_m}RvCRBZ%oiW9^2&E=a2H-#zq?X550cH$Q5U-`sTjVzN* z@$=mJIJHMIZrKqdVjkv*frG8E)x{J>^+@9>2Zak~hV{ms_k%UJYFkP*$C)A>IMc+sV5)*oG100$IIc%>|x>H3|KKXnC4)IUr*b~p-W9<`|f8TbX_W#8ZN~? zl|Nzr%u2R9sk)ppdW7a+MKwt^f+czZP}0e${u3#R>xJYKhngKtwY4qgs=#PwPN#Xf0k|(SslF zvjoSjt1n%PPD#oaw3yc!FCAVjXdaT*TiW|}<8=;I;Hrd15L;r&e|rtYA0ujkXTNsx zAG;(6n@r)||AqsuNP`=vHv;7f9Gb8aBWw<$;t#??*3kVQJ4An*r}!a#PgpaIt!h^t zW*6PUfe+Ve2#e*jf)B)XeX!pu3oYd)-f2b8d}z0`dagNQWX>N(ydjAX;g)A@{>USU zcRRpv%IsSFMRpQy`|1YAy*ta+(e==(0{N_IbmmTBni#qDnb`aIAK}`5r;R6FaQg5z zU~c|G`)Lfrk<w((Ot$}i#Psp`Xn5~IjycV0*?#|~q zyT~JddgF-YSA~i#ec`gI4e*)GBzWXeRa(zTgcju|u*GUCzQ3)$eD7~Vct29SUO5L^ zd+b4qAyoflL$Aj`yQ5cyQO@H zEZye0r}kUO#>_f;$MCWonHfHMK7@MLhx(&;!_;}jc&j%(%Qt?ln0u}!?{&2aU%J#5 z^26F|8#X)-0q#d|^nPcdb!Z~rSe??CzdnFf9-DwoiWAVhpjpe!GUI3j7Bp|cGLNMQ z(lbKw7;!FSO{Dv~HjIE3<~5P937zT%OPVkJ?5?K2p#dC!-b8zfUxTFiJCW80{*==h zz9~((;_2`oJD}>&voQSV8&wn7f5uX*z5EEYnG16a* z>KWFj2656Pn3{H0{QK^q%L#S8scrq?ja4`B*yf?M4c1yzMJNs>F2?*`j_Nu}F9_mV z@#{uWdN|$B(egrsu=u?PuGv?SVY#lH&#BF$cQ1u%JMOUY+I+&LpGdWdNoQ(NEk)qa zuG@sd*ER2N(1X%HfiP36Vz8z%3}_6R$*vU{^&7O*X*$wXNcG2W(KE7jyS#(%W{srb zmbIDw-e?v`317T_APnq zN9(awpceb??jaQpD=te748ks!DR^?uTb<@Se=rB*8`5(P#j};RWMQ%4pf_Y6WZiF$ zkyD%Cjo($d72SKC21n(|pTotgt%rctQ7ZoXaB#eQ9*}}1;g%R~+g-XjRpXXd^)hXn zHIrlhtYefvTEpWrfMPBmn;aFY&aaP^aMUXjH_W+)q>GU>7Z0U(-v{PZ5#wFf5Xf-m9y4q-%-VvG#3n5?Zi|)5RYe4O~}_1OyuCEO@zmg zN7!k_2W^dj6U>s%JR$vtq^GfC^$ggx`V08XX^(Q{Ei}Jf%q?*VIqI z`uXiK@>mp(y5|K-GimDEphNKxaLP61%j90AF=5tmI!in|M@tWgK;0#U*rur-SDHRO zWEBp+G)t`?TyeXouodY`(m3$)(icMi2<@j@)dRM#?1l;th}$qFwVB*K_#W&SxC5uo zn@QKMhm@;=#w34D4|DRqn~u$<>o|=O4+>S+uM7}L@3FJzE~C=hgFLhlKKe5}NCh=M zK|BERUKJpIVx@Uo36uKaDc+yBa|48GO9g2R&6|NkDQ~y3w+EU-wWn7V{^HN;*+|-q zo6n6z(oN9GaS?Q%^a^PmxqnlwnA)}+4=r%Sb!{3-!U1SDJp?js`iPrvuYd+Wv*l;G zW{HmnDo(mTvXqg2f_ha3V6jMt=~p%I+OmNloDo}Uc9bsj1`C7bW_-!vi$ZSQz*G*B z7AD@@0MCyNBHcsh_}yE>NTeNfY(+Agda`O0bJ`2DILbbevm*UQ)% zUa}OnQHOh#&W4gSH5=lSrlA~Xq!-xMkmusKcPhT`)J}%_FM&>XE#bzzhNO`Tfbf{H z200K-&+?DR%EH58TLtM!Buphglp;vakuHe>($yNnY2V;{NjmN}DgwGDxU_u;6kksG z##fC?5L6?AI0#qty9#Z|<6e!6)-2FWP@fVq9F z^YegC+;B?`;C8#k_07MaX^}Y+)yQBS}->O|=_4phZo7q{8?vMlv zE?h^-6`^!a*A7O~C3qcN?7LQBDh#9hF{lRFw{~Ok-pqr_ckx3L{=kh!etd7U@1Xd5 z-{HxOd?Jgh+66ZSB?4g?pS^nwyWGG?1}@l*hZo*Q(sDpN?)+euE7%`7BFu`r@~zf% z-g$%8s64jv6{Pv)lE;(P_kyqh4ZloeZJ)kjPq#a8S~FPiAQbMW>=A^!7<+KPHn?|V zMjQ@~+O8N|vYTzN8zD;T*Wh~Fw&C^ouH;u+pceAZajEF(W+- z%Qseml<&$N?HhvT_l)IYr4w8mRJ?K86x*1%0{Ia9Y`+%?qxm+2s#J&9VPI5*S|=c%#Hh~UeNs>Ip`D07 zO?2FJVex+aplt+qXRO1-v2mn_4k|p>kT=qjPnSw>tLsx8z}M4<;l#)xjPwIEf93$> zS+(Tz;q-P3{^!MFq@DzlrU&x8u(eh$8-6wp>2r{J2M|a1^uE5=R6PHDUZkzDmJcsB z(5+34&%NaGvy#P3K0#-R>{Hy!-cGcKw%yF7#L-wH!Q#%>S#IcN z>^#1P9Dg%h?2q5BvCV z;kKtm>qka7d-oF{{*%NlW0wxEoR%v6 z!p?Ov(~w4h@-yzrtFX`ZO|j;IregAj&tQBnR***k;!jYTfN~SRhqh%O{?$T?tG4xz zS(<4f7Z`bH6>IXIec;2vLL|MU^X}v=Vcm;nOljihboRW`Ca>@9Md>t#dQ^Ub=Adef z`ivvF#qTE6OVR>)XQ6yXmwV2j`XZ#Kz2y6E@@7cJ4&-J+KYhjM9RGhH1YZ{|o0%^!P@^9)W{^as-4QU71)p(cE5fHIct6~A<8>n|u zq&$qI+SP8gUMCh!t^?f{?Zt_EOl2ChMZ$gf+Exlx*UAg7n`NrJKB##0F#o6ejO_UH zJrI8@I`il57BcYNah&*~DwtYz!RHQI*fxC(QXFaXd`F<#74*86y{x8PVNU&+qM?HN zYns`PK^pRtuxK)!mrj_`#^e$_%c{YLlr^PZf|($X2UGT4!+z(@_=ruxsCrO~yF{U9 zH3vbuf$FO-^+qc2a@P`}UiUlI8&v&}z7*tbv^x!56gR{D@_Mv}ai}=qSMP<|4}Gf3 zqT#E=&)VrgpDC`m^_3KBjl+Lybo~kP;4E!TJgR=E>W5LSYYeMfsXP|chgG<*bQHEW zK8Wd0PcYKZYV7Rska)bW3D+F;s;ldny3sS7q@PjwyJc=Bc&~LT;mlo}8ch4&-;34K znn>bxByTDdma5)h*@jG~`+KIK%Kcrd7XbAtGpXMQN{{cqa~l>KokPNVIFU>HP!*X2 z)u*lwsRn@72nka(0n7bV4}hI$O@Gt4W-Xrtjl;gz$A~*It(dxo>bqIFbR{pnS*?x2 z1TAqSR3ux8yvl5#_2;Cyb-gIFPDeBazPq$Drs`21S2F7JkazM0^Vh8e=|D-j&pTPv zQGCYRtQp09>>O2}f_faaNypjXPtW7T7kBVPy~or?T8nmvO(5XYFUZ*GP0y|W)F_Qi z*u<0{Q#n`j!DBI_APVNKa^U3EIcamzLUTYUop-PzMDwA^0#q8S`(I;Dyh=GrM>hCn zK%sSSU4QhCrVUruA>L%9ePHt5ql&lWE={GDc#inp9;x4~O@09i`+1<{3|-H*PYYA5 zvp7rj(Lrg{nuF%3948F@$;hWroPJZUBAU^A8DTY}UKy${;$}uYb)b7=qurph-GnLFyb? zxul1TAJ-fFZEab^8$W5{eIAW_4uhbB(|Ez8nLO{vPVgGm9#Zb!M$0(|Fnd4;OsW5j zWm}nZSCjX`UhYEY#vweu`UQ0U))<}ccH-(az=E z)*tQ1#bM+b5AL_Dx+v`23~o;TEy~u_l0iRaV(By+ZWTs*JUFJZg+G6zamS|+ce)&0 zO{PP_>CZ0GuU{f733LY^i^X7^{9SAM&rDYId=9jgzGm?>J!!Q#4*W7BwP^#_ zVUYK0QMtpO?%(-=6bp>Z&Bl0NF703I0mTfn&fnB}SrXp<>cvfKnDFHNpPc=jn)0Wg zD}?8VQds1EmDBHH(ApI;wWYP7xv;`fj4yJ!!wULcgw*|X)KuG=%ymo&c=b+~d7~Rh z=dpEU=sv=wp-s`L{#kZ2Z6(lkb?*y)&|kMU{_CviETfB%eLV{D&YeZC-Q6)OIGdRy z=ZJVI=o~n$2=<_JGXA48LVm2_6%z(Y8b6d|-&6BoQHwXBU!6Lb)~BX(h~pL>l6(TA z4E93hgLD?@<@Ntukd_S(qFZ7X_{Y_S$fQRir>mi$&y!jDPeg8-yGE_YpS&7Sw$%uN zm)v3%AIHfGj|VKWbryIoUnMAa!1AoEq`0H+GL4K{TvO(>-6V6O)<9eX+8_7E44EC1 z3fU&s=v;q2*f(e-O+2oEcU5~CfA6_4?YW21HDp?>35L+K7k1@GnSFzun0xy@nC!cb zcAf{o-@U$EXgHrgt?~s+KRI#>+MhDN?k2v_{V&)Knt%y?W7*jjm6+o55<_|#%e4Jr z;IC&Yyl3p?`7=yp{lgrKJ}y?zEgyE@*<+UWSml){tFcFYFH3a;IbccuCxJnfz)cSib3l zX(Q;2>ymf~-qKrUpWg`;bk8;Agmg~~hxpSj(qC_*`ahaZSPzu%BDLENIz#9*$-`>g zG%JXw+#QCgr#j1$dBbR*itVhlc1un%1gcY{vCupl^Z35E@XzWHT$nMNt9dPK^cMa! z+W^^LJA$eC8sYuIkQb(AYD_;`F#DQKG*)5TAwIMa3)eRhR^LNG#n|=w7%ZssPN@2f z^M9vx9vcA`wI(sE@6~xu@CHz8Z&`R%RMK7memOPOHMFi)6KHPMys*wm2=PwA5bsX# zXLU8{zjFuu&L(iWph5qMp-45RHCb)XO_n611VM~Qk`)37OiCdx%E8n zg*}>jL_t}nt(b3VDCzew^_T2|U#pR~KF4htn=oU{u?ZDNx^kvg_3gnl%j`;Hx?kMm^6 zOK8Hg|Le*e#|B8Jjke(U^cgshXebxDuM$PQwxO}vC&Ch6=C%AP6b;=VDSjB$BL!$| zEG5CmI zS=djj*0W^D8wi=2go{!*w=2-1XZSafI?IkQW;pTNVU{&zgtTuG1u2V0LAihp@_??c~{DfAsp&75tpr zgXs@?uBkok!Aj4K7as0G^;{FJa`&RssSDt1be(X1BlA6civ{=W=|s5Cy?a*`gaOXk z`bo@hzA^e54UnbTam>>7ETlptdM~dJe$%%|`g{y=G=nJDww$mUsV2j;eMx@$?2>(Q}e0d=v+2J)-v7@rbb5m}@f|OEX&W${j~EY8=1k{zT_W zb4l3D?7BOn+sV_ASJfQsH&x|?@iO=$lfL~%V$>N+mKAW3a;J!uPCSTl2mXoN$fqLV zqJvBgu~e~uiq&(t!gN}1pgPl<%smCZzwV;X^ksyBAGpcoE|}`mf!Wr)joy=6Fm-KO z2OakMjv6coD=?&ug>)-+1NS@DTwzGzC;^_or)VfoKw-D#_>Pb*nt*fVjtn4G2UofYE`OKtQ zDi#?9V}fU0@a#VojEhDxlN#ATuOaa@cYnEqDg4W}+QDf3FqqB^q5MQ=A5-)W4~OJF zKZF|_K-hL0^TOM)gq0^Ce{yYZYGxo6c2s6vQh3IS4zJhoK~7W^u3oqFI)!n*Uzq>a zSd3i7(VhCVi5J&kNXKY2twHBYeXal}pBz+tOZ9<1kKRF4tzF>ty8~A>?BA({{L{vO z=3HG8mt%BTFnacX%Y3Kz5ee0|K(ONlDCl)f=_e7gHWHny)stmAtMQ<>2fBuim+AW2s1wb= zvd4M!vUlaCJFK|#h3epQ^$Vs(Zw31T%7r!Fw4a|yC&{3)#rrLCUkbqO$;fjjd_nds#we1$gz-eVXV@r zj-&;7;Z=KH9Jo?x7^ZL_DrvtUUehYAQR$IDy6wA&s%62gnkI0W8+aahCMrG_pqB=P*M|%&{h5YoCAHA)$t{db8ihCOE&-}XwMN7tC$);3i>erM zzh1Twf1tlitEQ)`3Cr=x%xSDEdM~gLZuBg;k6j7+88zf87b3qu75?tgK=}X^U+!%g z!-#K~?aIZ1a6~-qwHo8gZ^F}|HIVaW(N5W86(mYSi6XQZLEG4sI&;>&nY0@W~rtr=oQpQkS^mzyXr}A%~~kvO!v?>dxLrX_wn4S zH!$@=8!h<`AY7O3M?=;7StRY7Ot}aN1#Qsw<}Xq9Y$MQVIl}kwP;@`3#}jUL)|e#s z!t9sM$`b+czt&{!0W7dCWTd|#p;afeyp%}T2jFj5Lpa3_S8Gl@^BnT}H`99V{=lNf z$Fc+-4}^QD=4U+NDDh%pmn)3aY1n+@x=i79?&OO=`bOHXG(}VIS~@NwjjiI(J!7!r-PodskEk}P zpyS2@rZk0*-7Hr5qlGLBjADh$+K_kY!Qu|AmK1-ac!5=qS`hWUjqqNuOH14*2%D%T z-$CW-Oywzo^t(uuVQ6{gI8%HU)uR*8oH^B_@clI+!|h~uUihfFq}YO6+j~fSDx>}; z2pS*rdclFTJttowr>-Htw{swpFJ;6dYAw;{eIYA*W(xK2kV)E=I}eP3%8!STbV-KtFBEeiTw#@OkBg8ko4E2Yeuo?Dv?+}b z$*Uo)KP&3h0Bl!&7bzim%-hlmNJDVqJ0y*Z3F9^q)-H9i?T)N zd&>zZmtf(QDd2uR2}(yEgM>}{(P``@D08m@70t?7a`bQJRCz-ilH7r`!%HXfb3i^D zD2_;V2#K?UrT6k^=02vXHnnjQ&2c7oj%~`RW*K=GtXN&9AzTIjA!gihYz0PrKZxY} zgyV|0%(Zw0*h>o$>Y=B#=<$;g<`K57L)C+D8{-K6fvt%*o}rbi5fqIIgFhYDfbuJz zqYGG>i-GcNVA=}4YL+BR8X zZiYsMs~~8{0+!P42Kk{PbdC9i)OP{9R_8JDQeQ|sO+Cv)y*c$*gwpeW(i@V0ugCo2 z+T0bC$0M(S6fdWom({r4wl18o zPGmKHYL|@kd(>wtBi-gIsPCY)Jdwss(g9tID~0ly z(GGgj{&=F&XvBZ!tTf|*F5ijAx#C~)JNSq09rNn#C~4e~-Bk+55-$jHn~P4}j$owU z4J19xLedWli>BTh>X!)8px}f(xm&%CTE!j2XXJ%VIQ0Zrsk;OBdv7O;4jV~|9~k{x zrecNUC$y@MLEfLGeKV3}?`lZGBtgDMy6abCQExhP zw$@V5gN6L-Onz#bj9z8P%bYgoFwL=X8B(0kPUe#D>A^{Jq1E6J9cGZGqI)S%Bk6cX zeFv>wlPiqsh_s0n_`ll?q!~ozzrElTavXBIyhOiUNm~1wd(geOE+fsPdOpf)GUsox z%)V7Ep7)`Zq&y`2x=Y-#7uDx(}Cs! z-gVlXT=>#L@^L{+(pe)QDPS{ql0G|%^r`WAS>Nlu<3)OcWPM(NaYz_d+BuAh= z7IW7RLBd;S-%K}5e)R|35_FapQ*lZqQ%oQE~R!q3bXiP|&1F}!rFyHkljN&HqBHN&G z-V0QHR+ayX+sL;d^}AFrLDeqxEOJ5hMJdLFv98>1_5eZsXS5x656Nd}R4kNdi7KNr zU#tJp^(l-zR$^jJV{Td2Q{^7_9~sWQN875NAEQ2nS`$GSs`G$8>wQ&U8BH8*crrgzeSnu9J@F#t{AbzA8+0MvQ zXjBi8`X1=m&W96Lh>!_Kfjm2qM^Q0S;|A*EAiajc9^JuW?|fMFE>A}AKG^hL6uV#TYN}s&V|x> zO~9#}z4FRvx3ZV&ccR^%_FC#;Fw(Syowv~5*Gif=(jNF(Ri)SC-kf^vm{wU$s@@E7 zyTVCM?x!)qS+xABleFMYsC^S$7|4a;Esr%l}Q zr8Qg*Tn@*BYhjH&iNayVFqRe*DBsWbfiT-*{8&Sc&`QvB3;cwV_^zUVX37_h75nhaK|E^Wn!rzm+Kv_z4$@_#s z?En|K>D^uM?$lBiWIOVMhHLTIfC!BI`aH8mJp*3(VmXu6+t{`{_cMpJGM9Vbe}c{J z9r-;mlch9`7pAn=rC0Th;MMRkPG6jWPa3)KhhBeiW!+n%epDKpw4phhXLk@5rjL+E zfA!;4I#$7VwifbOy=ZZLZBu!y$1$Y&h@PhcVO>_V{Bfrm-g>fFx?i)D>4o89bG=e{ z9(RllYqbN%4qOf2njYdwXUOODS^$5SwZv=Pz4_NSN4Sc!_4XvpsN~YTLti?3eGET5 zGd9z*mWB_~%ckwZhO&T!`%q)S203yJof~k}88R9gN~5oLaL14{tgVq@6)MhoP5d;PtTvoZiQ-M+eJwX&rIFM&LIN z+`==f3mN@f23|fcA}#C47A^1Mx&EFqro|;V)U_FJa(JM0m~k6DZ`R@#SG{3^?FMP2 zmr1cOp>st$%V%S*vHVb1N%0UV)gGWh^9wAz>r%YC^^R!Xr-;4Y6o$oywIs!zw|TQo zoE=q{pZ704Q+r)2J~GOg{~cD2TYOKknKy2u*xy~QKK@d4eY&4nWEf%f@}sD25XMwrh;1ERs21kk8KR+V`EwEZN_r5+ZyQi4Q6g^LlZv#?bnZtJR&Q`Vs=2hOa{}uc zmEpE?hMaPRQ9bg&hXJr-oRyq&sg8VeQJIX0G7X%iBuX zY889Z#hBl>+==^Y$FW5(X7TK|S&x08`yC-K{68?ay1oqS_t zOYGb?Kr^dOcRBNG6-ihmI;_qY59hgKH@a_<)<*W2oXM`-&S44@ryAI5BAYG;T1TYM z72E&L0m^HjSYhDZy72F;0slS6TwdATM^5W;K*GO7t;W9yF20`1oxW{{_LKHt$I6bl zD?0^ps)R${%u4KeFhcW$==xnGb+i|SlRoKxbSiY_D0ouOev_8+_ zL0dnFII)_$6gA>Uay!%Yjd>B@frXE&$SgV^!t!N2dKKr;{wn)qVX+5xo@Ec(Sx$H) z(Sp&OaKoT;_%SyFW}Mi^VvH&wcJO%KYPtgp^RR>$*DnZFzj4foUG|)etGv4s{{M#C z<=H?rq~nR5bKXNvb?Q&{wF8B#9>Zv_&D=_j;k`IKlT;Ir7Of_pna1DTzoqaE$1d!E z1>5$q!{Z%g9Gz9V*(`*3VIVFtiWTD`CdrHcQgP|~+C0>J4?k_!0Xe6IK(pX3C}&QM~JE+Be1V5Y)1xdzbZcVc^!j5Yxg{HaXK7T#9z$l-6AZ%@;q_ zv%;glt^&M%1appEhPAT?$yUL7Sf%3{Ni~RdJ2jEbEG|ROz**vKPj{N@P=#R_e9#7m z4@pMtvOPd!;)`dcf~td>cij1g&@6BrM1KP|ibdie*<l{gSfmR zf)OW+J%n4F$q_+?enA$FKf4vR?fq)hv!}5hEN=> ze@Z)GL0vwjVmG^5)SB)|FhakazViLV?btfY4=4r{dk2}eZ#uKG4MD|)REPNAf&bv? zj2d7s16mKnYI#7MG@Qz#X>y2kP=*l_b{&eVY zWExiOIs;Vhh8jD`Yuz8HoCl3hgqZuc7JvMEBmF-Ezr?#U_;OY_L{#DBUI&1Cgbss(Gk)*w}J8v$7R$Olb;WPf`n?E>J~?i0jaLn@ne5gYs}(k zO9Mgp+}(wD2Fyph&6_j|`^??VIK`2loiPyy-8+bv z`qaloFG^U)TRX(G&MxRWx;cxu(hu9z>A*(U-i4Iom=>d_>YoiQ-33$YZ4xwIG3W)I ze|WkvhI#D9M==wzw?{pdKPsl&vX++eqM52OHf=ySb~4(AEq)15p9iMvh0E=(lCVqV zpZYwYyn;;2`6_qV=do|_%$%)Ab;ujOpNs=V0ZiHuhgAoi0^(pKea2{QfixoJ&OxQ! zoK;;oRrdvazXLKpj^vY8F;sJ~Kb8%527Jcwu0MraI0<_B#mF3h<-|(8m4rsAp9i^NeBJV)XeM4Gv&XtVYHrC zApOW57XIueJKi#f8o{IFqjP3l?}xr9hcS5S?>1=ZF-y=|qQeqLnf#^& z<$5={dP^ss=kf|kKhauQFyp2rp!6eQ5mMcPV{$i`AwpQSYxTHe@~KSCmc!8Cda{Q2 zOpFX0L3_}HR5(>WC0z9VVJvI!S_p)N*eC4}5+;cFz}-kXh7lHMh+~lAkhymDP264h z66*0F;!{0ra=xMLdb*sE77z;y^|{%cF>ptdh*`as6P2!y8&wR1#dxgc3+z*nE&SW$ zQGEA-)8iY^tp-QpB*i`WW$j-qG5w)*oik}=%7=5A6i+mn*Hs4IjfAV8zKLtS5!anx z2{b2M)Nl?HS`1NB6_Emg{lHS`UxyD)+=to>l5dydIFnMkMQ3PZy5brk2fEBLF@E3l?6H4%9r1tLhz=Cq#5@x zgOR;B@tNXtEbr%n+e_m>=_=9-*x>9nbWh3030AG)wci8ymuk%iZoMdCI<@7k;!kRt zwLYdjytE4W#XDG~fg{#mHjQa~>hPW08WEn4l}^=PGr}5Gg9 zKjs3&9V+hVa&fR!oTI<=1_VB=V23R3!8_Ys8j~rxjCfGQ7_ERA@mlC)MEm2;-HVEs zCe-Q$eowXlX?(@Y{FQY%`L!-gVb=iP<0>wK)hzrA-u5Fhgmt0`&;Hg@ zl6O#^km|ZB_d8gDtD&3rKuCaO52?}{l(E5Sm^E=W3<)6bA<4@mdpbb7wq$TAqBhHb{O#O8eT z_rY>mm$8C49jxarQQDR-ANLJvH9D%i5zu-fX#n*v-`AwN{Juu2ycC1pl!MYQq#Lzw zE(Wvgs~g3X)+Z_7sQ<8~;#84JWM;*x!TcpWhp=Q)^y@*;!}jOrMDW;T$d z$21`s^B^F>OEx&$A0|J~ht!TA8EGeqd3BhR*-p~Bqu%+sNWBNB{_YZ^+z?cInI)}r z80j!9NGipVVRyBMT&hXJFCgro+^|-j(veopi(-@$pF$oo{{=^Ni}d zKdak`&U8*0&sMgJq2A9oF#GR6BtHh0-{vxfrQ|Q6V`VZP>Ql!}Ol4Majz$O@HJDKhGJo}(hl$PDGRN3ydw4Us0v%xs~|h`Q%FLb8<+ zGP7lGk@Y^OKi-cwe%*V|InVQ5zkAQ6JfbprV&b`lrMj73!?kRD9N!D%IUJpH@Yp?s+DUB4EhhV5pdo-y#XYyr?Qx#567G!c<&E=kwnQ0rWF!{!Cn72NxG;9U7H1(9+N zNqUZbcsqqScRJ+)^(A36_MR2b4CCt(N2UnUiAcCEmW{8^b2=~x+nuCx9K2AHrX#Jb zvvRHtbqyUaVB*^c*s{_ERG31)tNKsST2%Bi z;kHnkM4CJCt$iQDqq*jaYr&)8KW+VuAzWQ=*RE|r<(s4Izp`V?M~W)CnmjDV4~Vx| z?+FgXr~crX8-sSAl33~{6Wt!qX-wq|YJ93blpaxdMD;AXty;s#w^ir{m%Du^D>P?^ zDh(Bcdq`dh#*aLN1s0bvZ|*cW>zRP0o!O*GQd9XranO0fky3oHm|8Q@c;y<8zf}&YF;eeqoQQcu>*DRo1 zke*%WgxgQ}spANh!QLOy9E;K$W6|<(_NmJeq#706^fR(^Y0xJ?$^5?|si;=Ka zr@R#TQi#~}ykf1CfBKbXj*8>nZ(usALzTCR%-Du_EV>(4JxUs>BD1AzP}H%`W-8ZF z`DtGl3FNDkR%NQc!)**m<9-6F14i>8C?~>%$S9`P+quw9X(F-Wx{flqy{fJ_`8TX- zyGA1oqqImx=62!Z9LiXJuq_YAskP80fBgf5w`}mWGC@9;uBQg&v-wE*oAUdb+m>V4 z)N_>5Bo+A()h<%5c$~(_pA@%-O3BY)!jh-rQQ$#f+9{^A_3wxD{xsUZjkrrk*{(*| zh_>?*b(En(?%G(z$WDJ(g<2&%;&CBq9;|+Luk|7$Hx8McE-DIPK z^e1kaX2jcH6D+un9oQ_n1YQSv^EYJ&vE)-8 zlxFwDkL$D1U`GRBdq$~i(?N|Z_~neAY`du`j;Xc>Z>5@xHD@{Za-NIMEuyp|#rimG zZ435y?lxAb$wC>F7tW{FGM7h+LuBa8D|l`+?UDTQ1+s`_>{fmUmN&nEwms7DMnGFU zM)#~;$bH5hWRBn;Q-^@_kO^|bs}HbyS}VFHN64OM%-4GwW1^QHUnc*fymcp5DZL>! zxy+F?=lCI!zViy{1%B--@wT+L)tui=L1(rJ_EkSDJrXw-F}7k1^Ki?OB6mcw&osDmMrda1n#3(NyUxn5BUkhj)yrqopZ@yCMh^Sz~myMqL~T z9MrlSv>6qy(Y#`}3Gr}g;V#f(_2u^nQ!)KjG*Asd+~umG@v|B^eqEDrnM)>{YL!XXTSm1RCie_=gT^xq zCY|+0B!fU$&K>R%Nr3Y~5w=`YD2F}n`FBM9rZc*3f>XCRHXkMZ6?DG|U zGh*&6MwkFI=y z>(a-WH*nX{#{T`d!k!1AVem1$0q;C24R-`5VzsT^IUQFV-kg~8N3Ra`TwVEuo)frU zdz3hEGfH$azD>2}&bpkd#yeyj)fsRQS>Os~yYzyBC6PSYxNx4G0fs;IdW;^zH|LezNLP zvGaLV>E0ud5eDMbdTnIcpN8n)C=qC0vlrFM0_t_yAS)l!&n{<~>#jnnbtonk?qKSi zI|9~F0FQx9h#n$#24IIT!)jhJHm}$bKzF6MLKEIPP2XNBA@F1z=509bzO#E z!^dwH$gxj%LikR4x3l(K5HCW<)K|Ktd1;KW8v`1F?D@b<`gLv2tO{(H{fU-*X%*D+no%_7{%y@?jNF7(rv0hl4L7h54I3QcW2&`6tAZ4m zf3*Nb&R=TrK<8ssTKGxi-%1vU%_W>c*S*3IQJHk@5};t zXUbQcyauEjx zGJPmq+cy*cm^v}D^Vzt!vmtkGQJoVuvBzRN*i9;9`k?{d+rgh{#jcz2vXX^p1xRB;k0{Lft1AM|_muDGUyxy?lb zcTEsZ0xx^2_+M_Xlb2B8}CrDdhdb0-nQ@O2F7#{ok z2~O$Q5C)}K%A!l8Ij=2+>O&0pw{9gsbEfmzm5f8)4dkR>boMNXGKyR5-<^>w`buqK<(+(-`pcwVh99`fwNQr9`+D66$dT%)*@{dk#4 zJRJcex^IGRbDeZe77m6>dw;=LvtQUCt2?-!@J3oUP1WP1DJR*&$m-B;YG2Y(W;o(e zIwX`dk}aEB$pgLWseThZELQ;OU6|8o1pb({Tem$e36&1N?{E}o&4hlNIGoRnh?|;e zq@!Wzz?0x$Z6Q5Jd0|J>=2+=3vOQ0nVcddJEwJuPFmkBOsUP@4s~d`6mBs^wr5A?H zN5z?E-yB2L*QAG#IE=k4$OPhD`E>n8!pCSv+68&pI7Zy1Gz4MaQux(kGiGeCm8a>u zfaaka6b=a+{cqw?KtJu>DPu`%CiXvXt?HF}JQ7@8cjV0LZHr2y6VCx@B;2ucsQl)9 z3hDeX*m?o1sx?I9b`@^?m8u_@VJu6gkj`aPtHWTqX$yYkp;d=SaNF!v^JLIZJw^Jt|YvNy) z+ay5pv0H(#iLJBf>QZ@JIhe~$PYN+leUInp$d%to zX=1}4T{DuM%CiZBc8I49JSu8Dse~&|r2F|Q5uZ(ly1~z9<3m%~jiFl6w_O3GAHdk?` z;y?INcJ+AY8E5dou#Rx9zl}UweKx+o_Y3-n}F4NY1t1McVZ^WDV{l%6i^ga9yy00PX9wWVggyYQhK)TXL8gU~i z4Eydd50&R2zG2gUwgK|Kuz9I1w|aQG0yCb^FvXl-&P@4w(iyn>^8l#RtrpO^V0nx_ zC!bC}{tb{0k#9de(~a}4Nxoq)s|rsvp>DeH+YP*@`fslaMxt8EGw`e-p%QNO;1$ z&2m_`*8=v;UyP@(4;8JAJK^pT#!!1<2UK-oaxa7RKG%jgeH;459%6kH{h->`e8Rax z5pqAAIIJ(MToenc&Pc;E)vH&2(7P!gIzo+`k66tHb!44E`>{jf2Ke;aNLOd}B&G%B zg31|2otekf-zyxi71tQ4zvTXJUSM6e3(hnfE`91ARN5Ej{g(&C$AZSdC@Vmv=ZFW8 zG{06Yx+UIpZwcgG$UoEfR`&J9cZcstx)?@k)`D;qs2;(+ZY-#E{op=Hc?{4-?-E7# zT>16dCFIl1fwVe#kU>a(Q>SocV9{*ynDl)BJv&L+h9Et~YhT- zLeIO6sqV9Mr1gkz_Y0*f&#o{YBI(Jt@~&SizObFA=^W z)h-6zNXBbR>H+CWBrnNGQ*z={zT{3EoO7=tYvq=S;s5DV?s5yP=-w>K5)_Uj`3Q8H zwVC>TF_6~*!fZy@KpH;_?w#*}V)9p&SEw;F<#XsbP%z>hqg(?ilLD28betQtz1Q3!36^@;Lm1`zxe=h1&;TqQa-_d67EGPZ;rZ1=iBo zRO?OkG!`hQe|KmZtEsVtKx&q2Xr^W9wd*a>we}aOt+lqN}7>0WjT2#1F3Q=nnzAp zsQOUSe7h>&F#f?{`SL^pP;RG@55QL8(a?4MTXp@gXvIk^b#BV%zBn$H$Al;?pi?;+ z@hgn;j^(7CWxY>X8s$LpeEvaP-4|59g2=Cl{Rdk}%6%$)cFzTAg7PnjyPqc18dzRB z0K@?-pb=se?MbygtvPw(BVztZC#iVr%QFBPD^BXYLL+S^0yfrRDg$`p?#Sm4Dr*#^vxRFEp=#0^jWjl#I=@2YCNkuHWv+4$ z<((U*wiLwgxO~6VDgQ_Mfw}i6B`x|%wEt8_eBQi59}p)A;vCd_K7x}b5jTE))7|&^ z2u-^=;kkR8ut)18)z2FFA0Q8hxlLX$(z}fOHf zB^OmjLm3K^Pbd8u%OcwbyAa_RET2`MEV*t0eqWStRQj7{LgGnDRKW^Ieg6g8c5c z9F+s~=sB`tzA0mXhU-_0^I5f321`D%IroR5Kw7XO{|mR+g_Qg1enO}?-p*g;a9IBM zrP6RZ(mUGe>$jO7?VtIjXS(iIuW|@p)k%a~G*dW5`k)i(Fxu;;_e+({ss081!mm2= z$EdvWt`GmIIpg2=^x=k^`@t(dn|}WoI(KSKy6+5pGWNvv@^ITnsV#d)L!!LDcxqdZd(-wDbx={>_1(0a#G z)=_T-qx@2%aR|aKxX@-BKE8yExRck4+om)rdA>F9+v#nE?(8*?_PdAm8f7eylp9i! zLyw6afGWQtO^&M){-IB&H=^=vZx@wYC$3t6rP&vh{}QADMY1RDk4czH$A-u}FY?Xf z6)vLkC2=0KM|h|qHu?LPsqF8{kL8r})xoh(n`-2{2uH4B!aqMz{Ix#Mtl1JtPjHpF zsO8r?C&*+~ZE4Z&f=yyQr z!(NY>quo)>BDB49(d`gNHn-#~AHB84Wp407sIKHJj-gj@w z^DdalviP-_cVQhIxwcT3I57h^*U|^83(w$y|68o)Q=Q)$U?rC>h{VC(rZU)o-Z?bU zPTsO=hvkjhNb@^Ad2jpCSiegSqz?>cGfS%Qd%qgMB%cquC(S2gpX6aujVJWPUZAn# z2fao3+t>;trvHWHcE*^b)skl^A8T}P25wt)LGv>X!d_oDi6@QL(!WQ_Gc|Ye^1hpJ z{h`+wd9n}_4GVEdOd}jUd9y~JcYS6$N)8)-3YYe1Bu`jv!hzpUu#k-E+-=NsJP;lt zvIbIe-$~a%~amZCxr(=8B2;?gnt`zT)ONjgfDX9bC*cGy}^z< zZ}=@nEJ?}fb-E3|HFSa8JAbc8ebigfHOt+4j_g-{Jj-o=EXTTS5!2q*)#TkM*>7zj zI#r3%-Dn3i$8EXgOY#w4f9I6@e!{18KXUTFh9YmZ zd_eD4iPio&{U=8UeHMG~RTlK{CbuCz?>y?f?$ll<{MW`nUe`QurM0D`YsYzU_t?!x zOEL6&5ai!)2M=G>k{7cZYVJFm;gOC*z&3N82sAL}dM25;a_}m7Z{rEfH($uTlf3vZ zM{j(4m&^6$Uhw8;3{KCrmZzKA$w##=!=$Gr6P55n#OcUJIbt2ChZ1sC15k?i*}`0DNsn;+ES zbbLv_3x3ltL+5XGB+wZ!u5D{#+YH9G$6E8Ezmv>>`gr}A6prp021Xn9gLdrd8-X%`qw4kl+;$PoxcVemv*SYn-NQF zFzw4EPV=dmwW-Q|t342pX72&x@PXKK>2{{D!FQS~-xBSMlMeUhQ+J#c)W2d@?0M{U zIvwq2IG|`}P4!bV+jP3V>~PajcI?xJdmM4YTKV;P2Uio)&jWQ=s|SO3mp6oY7opP3 zT{_k8=DdZfc}|!@W1pn@Sz21I0XjBlHX}KIw#BufeOtLbW+If%EdaTvF%(|zBjdKZ z0v$&bnV8|J`CTD)sDm7^+*4k?Q=1Q1en$H>=9qNoRY?wubp-FEb3mAa7qe*}@C+^j zDM#GW>IYu$ZvsZ6>hf7OW~w$_OGZZmT?_TmIr#U(O!YN%ZmAFN!&>nDHI4YXZG9wR zFE0Mx2?7lwb!j4r=6jRwxg^#mwf}z%%#p*AZ{&+3C&H|LftQEN5@d?O;KjwzH6@19`1KCHQ%T zD-u^oJEuiB{DwfU5e%03YtntXvm9_ZQ(W$P6sd=R>J7%YLtA9xE4Y-HG-$s`0Xb-}rQ0bxwT9Zk(%vySiC`>NzjFE3kCgIR0jR zJ07z9JK=LMpPvQ7PHb6bC09&)1Ed3xxSAjK8Ut~9b=iStwz#V# z8w&qwl+(3B8Y56Y!TZ(!fUsRsj{@O#1^3hM5H|0l`x6(V;v&KZanJb)JKpJ)szEWL zS0{`)aE4J$=$`&L3H5_^6CeG@TzxpCH;oq`&bJpGOMUQIjs=V_i~!Zw#DDT&hdhjo zs4593fz|}uS?#S%dp+Ly(CT*V|rh}*GEGzO*EHT`>sNdFO?z7`zzJbPAKUx7pBK# z(Q%&w=^0jOZz=3gPuCi`T}Cl(t!RBDjoyQEN^=P9jVA`I#C7_6Ma-l2(#qi!ZvWJc zJCE#$gm?5@bp#L>@TkNVlC%|JRxjCX&{Ck6w_u+s>ofx~n-S-Uk`CYC$L+@K;qhJ4 z%gzTa2MiVJGnu|Eq&lwBgs`I~Q(8l55gTU7WStwBHK!VeET1QAJU;7aUZm;+|DMU% zy}l!F+^4owx^ZuJfj#0{bJ9>a&+ea?V7?wsuP}mb^epj@YS9(?GW7dL_}%q|Yv*5Y zv^kaY(DLOjJX~uFn51U0C)pijk>_2=+SgwW9F~a1-QS7OVOKC`cO*1iW6by8ug?fK z#IaSoNuL+O%3Ora7Q+-*NW%>m*>&xdXm;Gcg4e62|H6(dKB1hDi%o@PeEJ5KB2I)hCo}x(1{>u)mI;S*Q0}t-4GP?%y7)v@pyXqR&h3q@$_n zG8s9f0oGcVF8Y466U2Y;V$~h-Fx(lc9NVvRnHobq=gD54HIelH$DKDUp=UnE>8K`f zOYnhd?|l>Gz0-uLJ2ZG2ZnARBy|mb6yHlI*hysFT3c6ZvU+p z?!|#>>_~IViE}ljQ{KM}Ra_#QrU+16Of|{cEe!{g*;YQ(P&60!@&75a}3qM&odO*z7h0j9SE$j;A?gCM*~}54?o^ zQ>H=3LsdBSF!Oeahr)!)GWy0l<~HWNs!v9`gkLL}%#AjV5RWoGFydh7pYjfrPji{| zqN1jh29O8lLE{F=V3ikz!)J_w2~S#JtXN&q*W+)F5!7#Dp51bdYLqw;| zUWDkmO#f6{`7)t2^zHMOc*$`CM*TBD<*^8taZ#iNo0C{Wo;_YwE}4=K*Dk_;Fk;etdFZLaZ-qgx?;{r_ksWO9(cE@C)hca zYnT4)hRk9AG{yfk-*RJzHlnO;N64GdSt!lQ>sI3yFJke$$1WhwR=p=wpQpc1*24ES z0n!_|*6gU}UoQ&>KRW{CS7FiDUfdw70ViEa`v43_)9E3gI5kiI9)vu-&bke6#TQ{1 zt=$kwJ+9UmN#F5vzZ1|d;TBLoQGbPi@{Ux$Lg{tFQkb6q8_64?7NLnnEnJbbpl->O z?$o>9oIbDfo8AZiU0#NIon6AAfoVf~5EpqW4a;kX>_E~Z;&kLA zg-Kv)`WIWau8HjTUtu-M5zYS2Lh=WoyneI8deFW9Jl3vvJ-%d{GroU!56F{<-u92g z(jHrZ{FLrU$6FA1wIMcJ9uMh$cOdh`J+ajMfWmQlPh2*p(!D2hCRvfjGUlZ3p@^zl#qsLwgHOT8!B~(`UIG>JV?x_oCNT1Ua6kRN=(wyt+qHN&HXQ#A315iEBb4q# z@|-YeRFSk!>SxxAauMnWNN^2;MUm?j#(+89gG-pmMi&(T%|Fgu z+(tCI_W?-D>!^Rx9rAJAgv#>svE4xPTA@=(vuWhJGQ@?<~77joC)`{>hPu<+!5#M>7I>2gjwS^Pa>#9eN66~Cc|B<>_kD8YipQ9!s; z;d5!-#fTUFtkKO4u<}zIdj4QLVdyR8?Z{iEVD-_hL9JCQ<1S#^w=7EUi6DkdJcB z+n6eLPae-stoDK0Ayz;b2Mz5HC_f}NTij(C6Pn<_Rt{oSpslukfi)I+4y(|L-v>Q} zTLX5W$!%-)yK4aY8GmG?g@EQ4r|k=8q;n~QX^OF{6Dz!?${UEoUHGAVCv=#3gM41Hs!Mje;4@SGOMDM%Zt2<}ar5R1ynL3{8VKv* zqOTht7+nn;j@*jm_XK4+g7gwDw49@|BJ%a|Kpd{EFg%9qoHruzq&)n9_OOo$164!E ztS_;HR|CLj*2pS&TWJ&0c2n@l${sn2KNnQFrD{|Q z+IfRT+Ux`BbA@H-6y&aWMXEf9bOnwMnkA?fIpt*XV{$d5bLptpD8m_nm($h&WhZ!J z{ice!q0g{2E)QVirj?X~yg=8dt3c_)Ext7vWg?2BM7?oOsK39V4cjctcNsvcXDgZZ z<&}=~29Q@0pG((^OA|+MuX91F&f&_UUr3o0P&SBcm;vv&VkYUT*Ep-qTb0RaiJL8% z$_KuTTBGJbVX=1nvNN~-9xV(5Ac)4XhjCpJ(DJLS`&{-Qksfr-2RiW+3 zV-TPAhf=-@2Ux_x+&Wc}&cWz-u)aq}ZN3DAQ5gNx5M^i24^Q z48rzd*~Q|V{ZD2p4ZyOxHYAKb$S4ET^{;vnN$U!#4W&u(FMYbK-R}?8LGbSlYAFe1*QTUeNTCKoE7UWmVJ8wqz_rfl=)Eq?tR^^ zxnU4g=@2MdlJ=z5ho3+^r)sdku(w;zpVj~$I(>wq;%ed9$}39 z5!IKeq+Cp>bJN;a^oRTPINTmX-*yM1ga;A0wXw3WvMI7%E+i*WVrl+h-xL?Ar)F^T z45bT^bPXq8S5b%Li9l&+%F59C;(YQ~dv%mS!OMj0;P7I%E7dcc9kh=!;Tc@zPUKy6 zDr=>jutMWmTAFf?dRqm|UMQ&31oadr{L_&(V`)RrR^-s^yDjCJX8d@kNoWdYk~{#r zH1PqW{>S_2!^Qslo4{pi4W2`LAdpS~(hoxE2hul4`2q1}?+Pv^kEHTbAe@xMH(G%I zE>L(!J&xDl0)(=%yL>sqE2`<-F*p&HV){t|E~)n`48 zJo&7e8Na zh`YXRr1!gSyqa?sPXA~MZuV8AS57zRvhnc?t{b*3Ro1<54V*zlzmsOhNg>k{Gp8<-)7JVw5zn=P(&MOld^#K$fW&U3_|6SSN&w~ z*Ae)&`c$N!olJc_Vv#f+YGkCq9=6EMdE!ZDGOi@?hB>Gnc4D8#%Ri$%;nW*yRqiSnXq$v zBf0;~4qiR0qtuQiiPSiMsO~!x?E>_%_tmQCv1lcYr!wE*-9Y|ZJe}(udM}c`6tmOa zM#BQjLUf5<0akkRbbL)X{F##}0%u-g?VB3PXX*B=S>Sh3p3+VPMc3xzkJjY2Q|Duv zO%pkBMJ?&|VioV0WGLMZd&nEXduV?MUknMgkjr=Umq(m-$;WpyaM$&%=%p7S8=U{j z?tXa=Uz#|<-`GrinD+ww+nC6#&JS2|)N^?GFHjzPu?w^NHdkYWxo6^O{*z(r(L@Z~ zUW?EAx)$HRH~`TZgJj+b8yVL_p z?)-&t)VU+S7~ev^acV2$M_TfQ2h8~E#7W%pTQ-=jK90VhBeC7gF8q(#botvo6SsdD z%Bfb_@&*>H!TC9SUA(27;=4u*uYMi2ytyU*>wR8OpYsvsHP9|NgkA4L&&Ygu25Z|` z%8iG%;L-8T@WRtEvZ%C`>mld+&~a{V1wOQ?AA)Xms>p5PwkTTl45V9$NH{|G3Dutg z8N;pkmnMhco7-D4w(><-@UJh``dhf4)CTWkx-KPYo7QV|(vKNNWVX`8{Al3pH>i4EWZa&Rthx6T;-B9Wede zFnrl#zud86D-4=X_vf_@hStYHNB9FScJ9)y_y;3wmi0$}#cS>Nfa(kJIth|n-l6MX zD|KIK&&U?l+3@Awyyk#o;N|xZKIl2}-pOb16Pv<&UyTsq12^Q@oBV{9V}oJj%kyko zqejy2eiRS}s9MAU_qxOWasSETo4rw8$EC9^c(4)ed1>aw`|s|Kf6U&qt`kqAntPX7 zEAU9+eZq80*&2R8>!2U(S^7ROo6%Xu#%_X*hZ@MZJ@vu82W5I^H$&o}+pJ-?9He=} zANQu_C{Cc`0O1Ix<@rnLHVOxq^pL~qc~gB~ft3I55Pl|#LdurgTtjsBHPn7(Oygs= zU0_l3ZxH?+V!WpZgkH4}BlRzUmtIwF8a)eH&B}7k{rZx69)4xK6FGs+*_isVQsJP& zs^pgbvfu1{*5Oe(v^u2cvL|gjsNShJJ6k3!AIxbj_^R30;lJMA+GWy*#1}AhPZUni z>%)2JZ}2ybX7iTzrMHbg9ALT6Im%AVwXT%4xb6IZA>)-ozYYzJIm-hRG;@kzJ*J&RbI0E$^;Z+JIPv6Ez zHtdOyqV2RZEo`OVK`-e|dulfM=?ZDa+o5a7X)*kfiKH51L&uzh3!Z5e_()g@1Hjnz z{f^i8uuBHJy?rm#^LJs?1G-7q^Dh;szhgg|9~M+oSR6G$9!}T^>&{wY;fCFKX!K54 z+L^Mu#(8M+s-waw(mM1!K-GGjdKw6$;bWkqh#yJMl^KM>a=rC}jv+p6JqS-~?nB~x z(PgOzoIm&&8m-O2;_SWHEwL-AV=hcsN55Z{I;5Ree&R5D5EpipR}d+5^tPtMsr zwm^?&iI069(YWbM7|}KcgKzXl*0r+yHq}W|U2v*@?Sf|u=$I6SSO2!8T1jC_2ZqiV zz$3fezzq$DK>7T4*m8p&pRlW!>`8k}DC{CV0G*60f#TEK))V9i^BAt?^y;&ge83Q6 z9y6>y(i*Z6UOOm!yb=!VHIVe5!Zpvg-+@SD2Z~kaqRB=7N z&+#4(cCXHJ(I z!;KrZuvJBMtYR#F2Ox^!!WnVnmOwXVR79A zLA4IE?$(y2&Xcg$3{O1kH3n<>&4Nd1c0hU&eAo8CenqQsZh9STpLzmloGdb>o;SLfyLcTC2Du`MOlhEUqit}y@%jq4o*@Ak7 zyDoGG!cLr;WXj?fZ04$eOy6~a9-CHa3KNv4AuJN)bt>@xkn?cqyssQ8J9dB|yDLIz zueY6FBl#8>zlGib`=tw7>3wJIs~8H>ER48S=~&UiTMtPWquV79N!pr`KTy|(&(G{r zHAXm;1P|%G#>LlLBh?Zo?8E^>B4OcybwK>dd_NnDSzl|Dmd|2S|GNw^?wbVlPz4{7 zcGu3%v4z>cu0Y|T-CFIFpOJc&yv9z<%4`P2A>zdB64H7B#P@SZE97f+f6Uf8+f?F& z_h?d>CCWv zp^i8Y$#aQCiG>PVk#r+zo$sLE-5H6~Fy%jg=-=EJH`h;OJqF|{eJ#tX9%j|v28p4I z&k@E>B`h_N-RiD_zW&`|a#K@gU1>feT_zT1^%EE4)8P1x5rVWf7%X&VHw{`#y+hOC zZqsz)h^+@`{C6Hf2pMljB#hWcn2~6p)%B3IByFmn&pJ>NU z)f&y-6i!n0%?~Fug@g7x(Pp_|KZ6d7mPKXY;e48r7DTFfrgRzMufj}h(>?^$bsxX+ zfc1;%E`8UYt-xg(SH=1&%zfq-0N$e&=X9mkNg0OXrJW`x~3d;0F=t>J5> z8P%Fs)DR4>4AGxAvX&cyHPYo8)sjZ*1jasDN?&57qch+`2S-V`ie5YNMEl$m_}@Mg z{_lQ!ZsRgvJO4cu?zIdMPtHvQ)vM&cM8BAPq&@)hwmR$H3~Ds)1n0lovUTyqp>uvh z|M%4-@d51_-4+yX*R+|1Lr>E?PwXagz1q$Y_If>bZTc6kwB5~)>m>nsUe0#OH!O2MLcXnOd(7)5Ab%4yEy1 z?3@ouo00cG(sGROpYuk4MBL&c(rAs4bg~Fbz6N%ELKtC)Hn>HUpmDI=_sPsVdO3`q zbOF2lUdhP+Yjj?!zfe5zV+YVWY5(nu#!JhehzX@NdGzOcIN#okleW~kkFLUrhmkS? zl{@G*jBOxkys$I7k@z{}Fp!=lKD`A#w)t>&Pbzvicjf(?M~d04>c~59_fij~GMX2m zJOS05An$~PX^{D3Cq8p51*Jh}w~7b5%2Kq@r+s!wzpkz|Nc9<~IuOUEjDYb4H5ERv zm;XwI;%WLl?6Jv+AN+TZDc!o__5i4w9Yh&Ggy>dyn*8?!xU_sMC=aOcckjt2^uC|A zY=LDzmeF;GAaBg59)-d|Y?2J0U(?<}q^Y|esKP0;;nYvW-D?^75kb5TpF$nst}a~! z+4+dkVLg$25j-08lDL%Ki+nxQwd%j4N{8f-meK`vzaq9CZKya{w}0F{Hmc)=3P0Pu zaW^zt5yU%07($;)tAVl};adN9j&A=+(%}8z{pGgGZy;${K|TgcM?2v5{*B=ES4D?AJJ zh>rX!J|9==(kcw`?Xm+ZGXa&~5H2y&$ZSiWM5JrOv$3hj8!g4yIh&wyNsh)#m*Ccd zL{MH{aSizpLDyMP&%`Y{mF>iO`7+X=EX`&+aZq#Vv?3IZeL7;RFo3IJPgULFIIeN-n;H@#s7avo&wY2E-O7tymty(7R?72ze<|!ZY9}rtR5%cB;HCDUCx9fWlKQX zO1zr*3Oac&mYom%64PzgDj%V#>zOk=9>~`q=>ug-o_t(+ zM2$44h&HK@)Q^I^v5tHj-lF@&A4O;A$Q!C0gO79VMY?mhYfQ-?F_} zCGk=Pp8pqbA_>EAa&B*=YvGg;Fy*zMZ!E=e!Lbz{fig{z@#`)$aj2*p7#I`@;j-ryw$NBP0eVDlICMzonkVmP_=M?i8fck$Bp5`-j^? z)y(*5z36!vw~A|odGx*RnJ@7Cp*<(vs^*X>{igB{%9IJ;PGdy#jS$>-y_jyi7AWt+ zpS9k>Z<9?=RL^Gf4)=x9 z1P7qGWNqgx1f|a?YfwI33@VOfeRo#lE!ygFTXWh2(9D-G{2%r5C_$Q$@a+jK>6O4# z*6hSj=tvj9#wA<8E?A$FSAa3cd*W`t?VxHPDYsbX{Vq@CmZEK^P?eKc_$AUjKv)2B zYBD`9K<_hb6vra&U1wAes)tFlb*#|$bJOWX1MA$8vT7~S>NcBv=TeT!Zb+-4(re=f zT@>U;g-OIrZ1z-BnJ*6?SeLx*Wu~xNd1}IGpj?YB{%wYo7t(rPA}oMkm*jRATsEdFL#p``|V-zt{v>z<6+Wc{bx4*CFe8z!=SmK zJ#a>S!v-03VQA|^nA^7oCIxNd$uDAcvoC$X&l6h0)=!IJ_w-dT`m3d^S@k6>zV`@^ z+OL-uNiV>xc0HNd@hwILOvgTb+;UdjK8o*-8_TES7|!rFlqT;YzB#kax?amwzIg`11_hspaHixX*#_ zN*{^!bDdd0*bQ*(5G$MP4aSl#mEe$XKY1deFOIRT1Xk5vh*OSj9@*<(3|)Bpp!@3^372Y%nP$<^ayIR36uN^`uFEe!=ppW^{Hn$o*>OW54w z*V&uJcSZjTpK#i=rt;v!5tyR?4oq71k~xKMpiW{ce(JzwJL3sB&&61NZrYUh85*vO zzFz{_{^g?PGCz#AcaX_Zzrf2p0Xh}Lz`oiK+4G^7>0NXS1^tPs=j_;xH_dpp{JEIQ zw;|1sj^+Vdk!G(EKUWShjF#H^bS6Xr7Hpd=Y3*3@i<;bVp#}f7<~zp#*w{EoOj}%6 z{U`+H8e&oHvoP04*(VV~sr_SmxK8oO$Yp`w3OZKxVtm_>Uw{j?Wn9n9(d zjtLO%lP0JZfv#KT_KgwG9^b|M<^?#QS8GZ2u50CkzH0s zWbf@~RZ1FYpb!Xt%F159=k$k9=-&6f=bZ2J9nXE=^P>KNL`E3H zSsQxre>W$?Jm0@Cz@iKHxao%5`WeHqwLigOR~gp(XN>gug5CoMn*TzJvSE1ejk~1z z;p2w>!|OfHf??)e$f-U<5dX<})1#`!ba%^fVRLtdLr z<}^lldf!44m*RglYcq4>5Y1IyQyN!Zz_>DFKH|EeX8p|wDEZX@3&Y*yHqQ&tdd@_} zqcUc_j{F|kk!L)5sIV!1I>(89Jsu&gGo*}35Z%mL!_tZGp=?+`F#k~rlR}5U^r9&F zIUx?#yb>a=wK=C+hDEzN@`Nkzz(!KOX;hCly_SZJOFc2ai94q?74@5EyBq2_$Rl)Z zvO4Z5-~JGq8B4sr3W=+5LBAHfg~c$qn@0P+_B5mUe~C}_XQ=a(H=mT?=Y;yQ)KFLM z*+X0K{wE6UMWkuy&cVm>hs1MD)GhL8a&f-J*Z>cUu}!;=GD+x zS8vW+dV3100UYT(7pbmkFP291{htL6u&)W-*CtUNbbuQRk0`7uPU6ER%vE^PES+wiJ!#jcXgeJe&D(=J zj-_c0t~2)T&}p0joj7qnZ2RrWi9g*-MlAs$=w73dTY+>Jy{GRKCl+M#_<;=X*!h5I zHyycZ;W*LqG2QnRmmrQMcI9qq&q<$WpjT-tcCN-7xH~i#ChqKn(0P}_Gjq4Rt)=?r zW#i*n#Fkl_x>t82y`M*nZ7WrMjSN06f@Y<_; zFv!7cjd$Ys*bI>x6O2zyMxX_UUt8`pzp7Az(1I0dil zFT>zvCOCP$1(fZz5Q=ALZIo`}x5iOyPEZS4({NDOe(XxmO?$FY9^0v>Ar4_m=ar{f zaU&9kBzy!KFCUlD0BB8ErPpa3YBWLBHqbc=lQj_z}K z>iD*@X756H>uCWAcaO3m33GvP4#cgvs2oLtjbP8dd_#IaDxRis(K@WuCbvDLFv_XU za8jdvnD@gJu1#nqBP!{chzG{-^Zz{%UZwO*t{z@;)A(*8HB3WV&;=hn$iNS+7DL|t zb70!$i}-VSkM^b8D>gswF!cNV5&UP^O6Bp$pF!wXbIruW4~%M&(|NHEVYQi=T{3x# zYiN7K6N3kUEc)jmlWyDarH+R&u1OI%kId34k5{}VH!sq14<0=@x=Pdfd~t^0jfM0K z>gtNqCGotdH+CUBu+4=(mmMI!S_SO8*Is&C6+8oiX$a|Su0c?%%}l4>X1OcjB&J^Q#JR5rOV6?btg+)Il0TG%R_k#2xk40Y zP5F|r!GxC@a#EdVK(A%@yIRtExq{#KGk89%nN(U&o=a@m`VPrwK*htuw1!s7uZgqk ze!#O$4WQYgV6a(Y#@9F%;*OsOfv_lEG&m!?TDqXmDQCI0-eVvS#Z>(i-_gZc5i9Wu z()B-sZfHhm{|Vw~m^Nw)9Q}2G&1$(tWWTI~#A|H($gpNM9@zgj|2ro>;xnCNw zz&2;ti0jb`i!%GcIohAr8Uk&D=;t;~QusjSnU#m-RKuccb}`hO#$b)zAGL-!ac2%x z??hiWIeMY0sbg!Gx|3#V30HJYn~5ZTg5u7v+3)&J^mCd}eA&G10^(yU;!=A_-dxn& zTu;m!cN)JJeih#%k7>U?yNAt0Ej1?G>1d|>KAgYM3g=&%P4{44!x)e3Djr$t=p+|y zo(;sCV#@3sFx1&261$IO%ExX9*^TtsQt4&#!!h!d?^VTdX!NWbP_1* zy*AkQyMwM@&k@HB<01blF#XOpaXWn!8|`x()BLOP^gGtLUj7zVw)*VKDx)gjuGU)d z&axd%l!w<+25=7xp!Fi3?a`PMg?07aTEaJ+?(~K<{swLuwj7#A)Phl8 zGm!i~%sUcCSegt)1B`ez?-5A;#+|YX9{%(bj4S&PSFB@kQNKasr^i|^a|BJ_6F?r_ zP2qs*Q&VSn2UME%L)(OVB{$PPOl*nhuoN4n9}?uP`K`;xg~Gw$PDZkRFE5}^+8fUM+56u2MenGZTyZwlh$g7^W54((zD3 zNxaA@9|`hx}W}Bpwq}vL%q8f)8O)WpA(-k`kgp# zW5=(oqI<~_=a5J6gV@>0Ncw^g2k&6?8rW?A8@A_as_;)Wm#_4V{DP$4Q${cqq|1_g zJD<7tDJ1XO&PH#Kh6a9(X{~$lTC-jPaVZXe-cp_|Hv-BEFx@OZKil>iqYRVRmX~SO zIwn7Sg!^l67u6o83-Z(Wr`u-b3yDL!!}Ji2>KM;9#ZwM>#{7tt7%&0Vfh9YG+O~s9J(5cr5$eW?V;e)8M zHuWBuDqevbj|C7tw*z+ZcwVKcg}3QD_uMFa*7J;3@B0_HRo@F&9(5o;WXo+@ z_Mv{rIP4x$3^f1Bzkp}QbvX6@Q?`5Y7#5y4h_WbMdwQ3i*Q1R9>t1aHWgfhzJ55l} z2q;Se`3X(yIb|Ytc4J8%1?p@UxMpQ6s{BdV&=BY4kp`go7UQbkisHEjNzJQ#%VXE$ z7<|tgrwts!zqW4y)WZUmtyi>vuOPn)oQn_EP1|3qMV`dvj~$$<eW_;jFJ;mTiKc<+Oz-N>sQ-oWU*sITDz124=)#c!z{-YK8WXx@N$kP%m6n{%Hb z#-jsy)rFX1VJsV0)T-i&>-Q{$>aoS$zfd(F1G|(8i;lFn*2A+v7|SQV1G#N_)$uzO z_7ciFP@QUD*PN-Wc$xl^$4LZIlc>Hb>oNRYMjD2-e zkXCV;OVK=H4bE8*z{qb=-j9L;13hWh=&tf)TFM`)-zF4isoYHb$ti22@^~KS=OOiv zs{F0Vic2nn?caEzzS8*h5D^3 zgMi8}=6y~HolEsN=_6h$aR%}@e6ht|?f0tyq&N~gIYCR1Og zKUeG6KH8cGT%5y67ldoTe5KFmaWfD9js6O%r?q|ZOrhp>hRYnNzO0aRYbT@pA&CF- zp9Jn>UuKk3zwa~@TGfG_O|O%WSOWuZbq3W3A^m04|Kqm;O!8lxS`4)AO3$HjvQ|TV zNoJTiT;W6zx6$6k1u$`RJ3-z_VHoIrP*k}@WeQ=ozK`NrpbU+at*Shv>hFyV_NeN; z5x?=yPM^g|`p&CmOm9X!q77@YDxdN?teNkPo}1<|>h~gXtWcU^_Q))sIG1{9chO|| zDXgnMnY@oJ^=|AX@g^fqt(yCd-^?_*6<4VTz7<+}pTvt33-Fss57Mq2?c<70T=AI7 zMK0IvBx#&by4a)oI&^<4@#l|12>ZPacZ9COytO0Q^kF7=tHUaBtz8`SY?}v(S0Av} zi6_Lk!gf+*)sf#?HIg6d*~1#wW5T6RZ#iUaQ(4DlI~YH5sL!&_;Q~OZ>n|wo7fx1&+!7oy;d@ATpRAXV<+x@ z*iOE_Jri;scNbsHr(m$9r+gJ#2wNR;#Z$2d+cu!<<^Hsk$CeC(4GY@A+RFZL`pQtb ztW!MPe%grJ9!`aJH5&6YttESYtrho-Im&&SPm`W8t}>GD4XJVBD%8H$jc@+=k&``%+i-QsMcAu8_}`KgIJV>tw$7^#{jaCPUGL$zp-v&qqbKiLGYkJswB_389i+SS zCOq_Y7kU{ja=%7>Q+mC$Z&goY{dYPK+-=Bc4$y7S27GbCTz**|!sZ%K{>{x-$K}KF zeQ3NHqJo~aaO%M%89vb;+FY)QmtOSN41s8PJ1+&YR-57Sh67~|eV3GdU?ulnR81Rc zzY8~9m@eDLd~(md?jqBShsjWSmYq-fW(=Zh^YW8x%fE@|MZ;LZre28ysOBkU_7k9W zle6?z;GO0Fgu`tI`8CB1QjgI!Sc{E-)(;yQ9)$7P+p(jgJ%)^XgVR1ZKwjr$obu0& z(^`WbJI-4=h43Y=3y|gj*0nOl!EwDX*ECX2*f0vR0&ihv;{*KJBPL&#w&C4JuLU|k zPREg{KZ0@o&j>+y1eZSNalYX!kl#DQlxi>I!wMgI-Eb<9LGF99_HsWoMFu8 z8Z1JmE-xG52!EU(VDa%9+&5r8w^^(&>xyJe$Nu?Z){W}uz)iUIyKowJ6xO9Z%YU7- zmWwGb67KLz z0hfrK(8t&m<3c@geQY(DsoNU0siS4W>Hi}@q7FPeI`F+5(I!# z#i*oKFsPOnFFA9L{r#KG+Sqo3UXAHHyAz3k#mz81c(;gdSRba+cM2s2DR4EwP5hg< z1E(FjhH>XM^V1(*VU6XB!DU?&Z1+ST4)z_#Dw=fSrg~bu+~zY-o$&=>QDX7tb-1O# z8|WP2Cr?sy1B6TNyzIRoOyQ|Dt)&{@_&9x;6SanC(Y2YKH%?$t(Z+I3#79wMdILEG z3}x+2(}`2Y!k@e-Tsp;Fjh*mtj%wvD62`&v`D|F-t(P1h7lBWQYS^1JFYKUijD%Gz z{Pzn_&|dJOudO^*EnKvTIPAVc?>Hn?8p~Q|y2u7al~8B10XHvi2;a)O%JJDYJk>e@ z7W^(jQTU0)9BnB3)}Ji0+qdPz7k*{K{@BYJ)8mowp*a|nORu9RT~|h9O3ks z24jy+fs!-5_}$DNijTmpPBpo|^Jr$g)gKji5-+=d@7I=}Oh7nwDFjG6fbcFVN*>{M zizVRDe+kA|R_BfVyfn5&$8g)q`dI()V>ai-YHl8~AKaZA%7;JKve+M8Fy{6FTpHaU zsg5DhFB&a29inTZljP+#^KtspeTsXl>LOt0I~?-e6`q6#fZ5UO+IGW7!pHXk(6Y-5 z;kD6T_V2t^(wv~G+qk;On>{DvpuXGT(5!`g(_d%eqhR!D z{#yFJ+M#$i-(J5z#2lq-yuVm+r~jQ-^Tt-~i=9o|@aq~F1dg-7AL{%veKc?Ms&w~2baTv6H**R|S67>dyl zr!kL?dT^jm9XT@c2M{mH>QC$O>LbR>o5<-6=p#A#0P9@q!tY}G*Y<0 zKE@lxl1H7{v1)qYf80adA2SNACe-9*hhGYvVKG!^M>O=4Q;p&`y&7`foLYoId)g;< zkJkK5FiiWf2bcExmTPe@R6FcXnp{+OE$RhrV%3-0leefY?N052lb*q1Z00`^)yoai z;i~AC)f*QN&4F{hPvh)czJ%NAoUj0=0vqy0b&aq)cyrPeBs~FDN8P@r;0o`CP!jTi z#oTVq$~q-5@(AcVurBdXFe6-}`rWH<8F1*Uoov0}AQJwVD4d6DTsw*-i!Zsis#Bmi z1E}s*Em5{vuHC<^gM85<42F$&{6R%)j%A!=`jq+11WCCv_*wTxP+UNT8ZI-aX@(IJgS>^K(d}3=U<2^ z@f#U&ks$q7^+2`vKx{0r!teTiXc(p^W8-h)d9&?e)Z<{tn7aWbW+z3b&NRJ3xHFi(H5FAKyPk(mL73_7o6r zu)CS3f!T)QHjS38%+z7M|CBPNb6t~rN*ae|llC5SInn_){XK#Y=Z#T%2o)3Ra9!_# zK-!|Edc;2j4nD8Hx#mOniWX349XXFjISqE?O4Cl z1JV&$GZW;V$u6=lT{Ai2fWRg?wdIOYKf!fn9d0#YHIiO4H~I$NRXb8WSrFBn6aHhsr(O(<;OhIY3W^2gpu}k_}`&=k~A649zP|lN^Yz385qJXppzsKrI$8KSG_URqWUqR1jA^oM_;qBfvp-9#%2a7E(1CSrnR4e5i&q_@>{X?62 z+mQU0$_Q|4m^r_8-kwvQ5d{MqaFr9Op2$NlVGfT0!-IMVkk5!V8RAvJvt}x?<(DB3wA(Jli$ousS!*-yw6k?)nx;ngBI)9k^QW zg5CXT-}rIviX+yh#EYiAvSH?l!HjaO!iju&Y8ueGYn2D-`RWNhKjjHhMuRltJ3yL? zv|dtWCdx6|)V9m$8K7TDS9TExc2@X@FO`F_;VtS(C*%=BeN^)WOYO(=iRYWrKA_uC zb7>jYUEHh6hZ}ay$fbNNit=d>CYnd`?C4?I16t>;(q8NO6_gKIyt$58H6UC%^#ATo z8p6tNS+X(HeR;Jh3wTJm1^JUNilgO*yl~uECk%-@HLoqtpy{dcj53_&WCHMZPa<&A z;GOK&utY}n2r0EEvCG-EIP%JMG$~EPfY#fpe2nr)RI@-H2t?L8Hf8f=`X2QwBh6$s ziz5X2Wc5=^+zI4gMBn-^6>f#f4P)0&gDE0FQXbbF8CWEgr=$F<>R0(XxZ<%((8Jw@ zN5^oWEX$ANN~KLK`7}fFsz7U_{Is?_b2StgF+4GUFSLBUA9JVn#E`UZg7ib>A5_^n z*>PVLmWdCr%Q9y=XMgf6blr{Jee`a>pOFv3<;%%F|6Atv(t8VwF3K0nPi)XA&NTbB zSJb0@EhsmMv&}OEaf_yDv?1lK&w{dm@>l$3v%%2OF%1YikU3rlh&Pbx4h<)qhSME9 z8R;l&nz$Xvm$B5gznJO;z(HN4&%j~#jis^91=hc_B~T_I4VX|hUoE>#0?H#W`GH+$ z^170|q|#+kH1t2J|8pv*Q9t3VAioE8EBhh&WBJ*>1@wyw#e4qx%)V7Dke(|%3+kh& zYzc&Krr+ie{Ox8$d)N-Z2Fu;h^JWzBewcR5=TM~V3*C!HYsm}1-5cRZ+6SdQ_hHLW z&h=uKR`DQZN1^;1k2(e9?c~f8HwYt3SVhSg$}Y=Ty=ZUBw?=~U6cBcpT37NC?Cm+q zCw+I&dCy??UqlkNZIJ%()MNWQ&+Hu{BK$788(lo2u#c{qPHgU(?&OIdvGG2cR68$N zpQitbZ>J7w31e`3#5A_^!XZ$-4DyhwAE4?VZHqd>*{O6L`y6jZ{U5RP+U`8c0D`zP zZ&JHD)EnH5x1Vl?ot}TOmVvpX{s^0;_lNcEupEhV>G_vEc$b1&JgG7U=)LN^k-RKd z-adT4D~D<)1!*_7s?!C@!^5%>dugr5VD(Nupxc58%58jr*2>NL&G|E3;wRx)6iCsIN#%^>smATh5MXz76 zNIe8~-kP`PMya}GuW!Gjd+e;a#ilk=<%P$4cL8|{q#gogTz6DsraC6y=nu=616aH+ zCy&IS7Tup;k#|950erWKo<)>jBvkHDTK+P{7n`N5XQUVW@m@PQAUPcMn1>|37kamf zSYyAlu;bTFdj7&=TrB>^?!lh)rlaB((#xfy`O!Eny`L~}8Vg?0J*<=&sHVufcCE@v zCq1j-mKx0{&#VTI>$FGv2@};Pw>21Th$|qS)By+N2m|CLlD=Ie|Ab4D@-k7*?PJ?_GjUk((zoVSQ8KQF0_ zhe|_U0`H`0aeEGfrMiDjt8j_?njLQ$u}KHD0kl z`2tX0S3O1yTTr1PPt107ISAqJqEM|hX)BJb-;%s_1QNzk`9|f}%gg==)f*vx<1o1` zu52{~Nc%bUjcJ}+DehMuRj6JiWgC?pw5n%E{ZdqZmaxQ$cYykk++ctwCr<$PceV%f zkXEYZw5F$?(N6m#Q1v}Yqwz%xUH;2@1oeb(YBT1VVCfWzT?*C+m31gXfy!%y1(^Ei ztMXbv{U>eH8V|*WJU!wLW1)JVq#KO-MogRfkcIC*iqvD{1D_wk{RM^)Uax7DA3s}t zD_Z}3qv>A!5ck-P#JwN0v`>XY)n}-hqxGrk^Q!ZsTqtPG#jdv}p>MTS)C2jBbzE$a z^0=y9@l7ujKDxTAekWz{Tg{{h!>w?kxuGwk=DDD*D#g~}s?v1iUx9MLNT zMuwQmrcLeS)hEIH%j+^+8oCv?{qW+}j@{h6&UD9y1L>NpoM+fz=RS;kKaa2WZ3R&o zS1~i72AY_L!0m$HA|>0De;iKF{@J!g+uEiSn~v92`86qopxOOwi{KZ`h*!((~v;d&_hwZ-`Aox1flu z#Imr)d~Vbtx%{%ROb{Q?*K(BXX>nJC#{lg36N!`7<>2#|rcf^|kX!kE2IKTzyp@+1 z>tSA-tMUFYH^PB-ZKUtuRAFMO!)5IZpyRUo8@#wf^!NNl`LDsRm4v?sd&rHwyfJ0@ zH7u&OL6~&OVs;OYL+35V=sUP6A9~D;(|M_R7Ta8mnAeau@N39SzFWH+KOYXmb+sJO zYu!pbzBdm?zM%VJ3nkdLtBDSqjrd}E255MoAujT()V40YhW~nB#nWYV#g$q6pna|8 zy#M_i%=~MFNv{L(@7zp;>WT3Au&%tiPoVCRHLzfX0Uss5z{_dd(Q()!dFWarSZMu} z^|TmHbF=_lzIWq0&h&sC>w;u(_5;DrhjOd@S8QN-Gf~`hxB5Ib>jB~yAAgzL4P~d( zCq!+lhoYKrkoBEbVb4J}>~e=qV40gFizY;Bipw9P<UDq1u_cDYy&e*#MI|q1+ zzXwm_x9O>18FU*DFAHPQb_=yvX0_x4GHIWJ^Sa#l@{`w|UBx{Y)}YShc=);e zI@=R>3ifA&W1Gq#=^x)ihR2Yedk^7Vj$8--kwLud)p+>RxiK4%IfR!~%)vpM!thMk zB>w7X1AfBlF|Id@0Ky9xmhOG8*7OaVX!QsTdUS=AU30+m#~Jjr|AsVPezdq1-^ZPJ?BJZLaGP(W9cdk8qf$HTO|rxx@LFB9|Lr7=)exry^l5;d)_(V zA`V?K2or`c1?_;RkTbkJbb3D*U9Q9thRm@vT?g|#GGXI^uHuPJ8PeS2o)Ig!`kdWO zrop;t+eH2mI;R6C)W3P^khiFppa%hCy77two=E4Gcj>1N|Lavv%radCjb`nk@BVc7 z)4;Wun6d+@&M}VmVoKUrUFygMeC5L~IQd*3-r*5F3;M$%!v0$Py>+7?9I(Ho2AI55 zM~>Fr4smOjBk=%4zTCj3tuja5*IUKbBZK5=@0;Lk>i~P5E~B<#fB1IQjHz?;%?*?j zYCFmOrhS>>LRwE4*=K-YVz!)d$p|)uJfxal09Os$%kXm{&}r>2cIamx%xvYMX}bC) zs&yr-!NQ~^#HQt?KQ$33Y3T%S{&LBA2d*?)a%G2%pCS#Azjd-fCtPcMKb#9ym0 zwDm}lh&?t|8f{Bt4I^66wG{W@Lq6?WofE(ne+;xcgoJzUxiJKP<&>i5MjakjIuP_6 zmJ4XxTaGw%2A8d#0$+R{pkdw%s)(B*&Ia~>D9py5mSbOA$xVKT zv{oOT_(ZEQl4=-*{So24YAl`8I#68obNN8Nv~RKYeXmPMTqNJTpD)Zim`I(Ge;L)D zJl430oYOoDGFA+h_p?VcL*Eae)^@0f>&8JAQ7y-l~ zGWl^&PPL9dE$$Ql4#z$RPhqh<%mSZYWO@$2p~&wGR@8ChA40~!2l270=Kt1ZvPs`t zoVFFg?2v{K=^cfCZneOnPkL#JSHEHh`agmUOMl{?e|T=gc?=#ji8UWomVbP&n+(^^ zg8mR(&@ zTRI=l0OAac3bWx4HrJJN22Q|`Up1g(RJ!6DY}#xtam7WPa{HiYG|Pbn4cIKJ>uwX@ zt|qYE+UByje5|~4xgJj$)EG~v_u&r;3q+6X4Wt`kKv=~N1GnOgOKZjJf!5qHMnH$7 zv5;1(FZcJW$89gy1=2{mHzo{9(?gWLu)g?-4cXH{GbdsnENk`&TRqhgq=osh@9f2b zo;^U}EZ-zps`;x?lBapy=nk&E(f}=1MBuNMz#n_{=80`Duq?M!2r|E^xJMFJz{%SH zcdr`9h^s+)go|(JIYWAO+$?G#?se+NFAaPSQLmnY@*J0<7Ndn_JUX5W#xpM;pwg64 zav_WieTJlIbgnwQ`_gK>mTNtY(ooaPC`Mez+~ypr;&_@DpmX92EH|L4dwL!9NDd(3 z5UD<()k;HndBp@g7D;eE-j37x;91)_;#0~_w7HY6S?#+Yi9a-3%gul=E@pLcMZZ>D zlQJk-J7|-xtZ9E(s2bf<>4tUOcK~TRkXCAoS5HNr(v)knhRJ?Tmw>zxU*uQ_#EqzT zG8g6bBy_+uozh~ves?s1%k?1iHEC9L+T$w*FPPTA zz#b>j#jX;Fiytk#59y+LQlckh}~M-@t-rKDc2{I>rTD z(vttdXUQo@S||E8d4+}_4gz^8^wy@ikM=kMd*Zsu22(b%&EUvCo?nf_&U#5*pZef- zvl%_lcQqr=0bYOfB~M8uOz6mf$oX*1awE&RSX&Y|gVCoS@ODcacl?wI(~mEKuCFJ+ z)b#Tj(tGR~?W}ZC{@!Y;YJ}0cYiS$xH zCSHgI@}pdNZl#A*b3i^CD^2>4rai!IE^ATwX|-;7JN6>^M$Aew?0o zkR}6pe2wC%{1Oe%{J2CEhbD>gG-rO0?h8LO zx+@I6mdM`zUPJ!5Kc*e*j&B1lL(=Q#I4d(6h?`;J5IaHsfm7`&z6IqM`;@Ms{g9sl zWed_RADWv)B;F;z9sq<}sk`Kq;!|mKt~V%ewaBqMr#vEvt9j+30ug_67fiREtx>t< zZ6_;y^2t@>z3MR-=1pQ|@21Non%$tK`zZHLio=nj1qN5V7uwg>7!v&-(oad}3YVI@ zBI$#6XrpyFs%hL**C(yn4%?@v2=Y!WdX!cSeR2+-B{zZUJ9Y!Duh5Z|7}ak$ z%$wB$t`BVi9gmIS9`{_U^#2)aRh558vl!J9uJ?48Dz8z7Rlb{^%f13iBHKvv$>8^K zF;I=6!mG+^aV^|{YJl18a>Ra4ePv{NHNMrSgxTC_%U1{Auj2K~iP2nnuG=H?s{9q5 zyXe|%KfDN^0Vk}+a;iTKc^6d2r5aNCk{xOv1AYOk$j8O8QEh|y6K#dcuBwLk+@IAr z@vq7^KyxUkkLZ9Neb+MbXiC%hT|Im78(N`NS=7*XjwCOoyfKn@5xa+FAbB8mJ%2h< zE(ZO*ozUoC9a_^R5V-Od`&8XR(zw~g4{b!k@SmvilymrNcs+0%P{stp8_@i!^TMXG zji9kn=4p>QU)(#B*J7R4E`!snHdlFV!XONMGl|AD6h6jwm6uLozW>EY;ro8S;#%UF z|9G{g(_y3a7D1dK3M}gLrR8Jz-bwZ4^^eXtw&4|_J};@@C=5N8pfWfptx+D7>Ie&R zwralm9~WzMHetOHN73s;CG@_v8-{d0t}+7>=W?oFsDFGhBY!NY-odV+E^$;6QoRbQ zL3cV|)StNv<8RtwvUxNRZ{=T#`X;(fPsT~D$)8)60C6a57iNuW-W9I?JA8?)J@J=5 zw;`azU$HtbMKrE|5?;S+4(1&;u*$x5k@QFGXelVW-2`LrL8Nvlc<1ytDES#K**WzuBSDCos&)j5z1FhO~?+KMuqHgP(&^81)jM*VYCKYqF)$ zQKs~_Eo;F^U#fD$!k{~-GUdRCIY9ja42b-!`Xl61Es=UNNd89ghRVP|IRSs4 zIm4n}1>l?apS4uuf_{SH6Ur`}GC%o&Jh6DY6*o&Xkb7`GP)=5!TZ~)NT={Dtd?+r5 zDBBi5xI@Aa9D6Z>{Gc9?KB{cQt=8v>uNy<@HG5Tl5#$SSj>C@ZO64Fo>*M(P8=i{qJsxP~o0-w%uX-e&94GEn(eRhQ(~@Nr(MsF_uaYQ2i3 z6MS9sUik-iI#-Lfe|0X{>3LDt+x;LP(G2Lzrjc=v9;AE z)k{$8P}NhoXf_DRTcYxdi$1TY!baikHYyjfzjXy%eNv56mQomoJ5w#mZ|TDOpn6c; zl&+y8UZB6rk$tX#S4#~N9&u_}9aPyw<%RHJ-^A25vMH2 ziu?Mpq%{?|`Cc~lqg|Ndg+tAkLCpbUq4aKhX>Wc5sNRKXW^Ji*K6#)#;u_YulLZ)9 z#Ay2T?gs}qtp@AQ%YiUU8q$ToUHgZ1Y~;%*o6E`WJK6N(-+?qxE}EbZW>H_*dNX}# z*t&rB(`<;;YZFR;nuJG8|rixITzKwb(e5~ zr3qm%5f!&wJbE9!(f#=}wKedsvrt&J{ z-5e*W`iL}UE#*X^ju+q>hqZ?^W5gFgcmT5&|Afv(ebrlID;ut6qngg3>r;b-^7*x` zM$>C|vOdA}DO=Rz#$`t0$73(m`%--`g(psWA{4&|WvwTk@)6XBfV%ZO_>rNrRj*78 z8eO0vUkbUpjV1Xt4P_gx;=0r!GgJ;zJvHf2Q2~^ts`_iQ%?64RhfY9V2S^h!^yw-_ zoF}MnfUcf-?mnq@uyE)el@-K@=e6mcQhI*;$dycS?7C^z7%=7(^+Q}Fio$*2#gyrz6HO5msJTR>$>2rOcz9krKGv~zV7`LS!S2Z*;bUbn3ZX~b82J|!Ln*1GA`HIHF zwj2&+RctJJDuI^*xoo8xPUQjhfdIJ0SVWnM2yg=iLrGEB(P`;e{6d3z1jgj|;U(xOP z*Nrtuha!P8Dcro1!4BG`>{#dopxzqv$TmU3EvWtwX*A7iH~!-LX{7bi8l5{2 zW{HugdQr+teT+Q;$_J5lLX!Owpk6O0%yCQWC!jOA0a9*OK8h(%tGMjuopy9wBjwNB zRlO>ot@!(_?FZ^Pt`a>f_F&1N$@CcpNIqYZK1u3ru>gk^g1j}52UFe@yInO^y*5=h zVD|0}ew-MCs&An(BF!nE6VVciKK}*kD+1vMsfPhh3$-HfV<6V~uNU|CtdB7t zH^RYLjnu@ zVzX5iGPH9w-uh`nx@YnL+;r&%-4=dk3FB#RltznqdPy$3VKE(BA9##+mRs>Q{da1k zuZKhWj&$re-&E7Q$0!`p+@JqD>gB%pu_1K%Z!g-+z3ASed^h9Ur+^;-T>BP*(c|hKz89sl%qixq4yjm!mQ3 zyLBQy>Y&3roxcG8-d)EIwyu19h%WxQ(Oj;k=N35M9w99=^kFF0kOM~Cgt-CESi1cl zTE1!^8}%G016wBH1HWuE&kw{!=Q?0{jWO_MW&-!|{0*ngjo96y>)_?Gj`nUz9lULmPsL6=^n?k?Q@~$zAtck{Q}ni%4EF%YA?FAug<>D{vzfi zX)%BGy8J2NAr5w4$Va{@LKByCwRW^$l8<~J;x6{Di^AN&(eQEnNd8`QM;fy`%{AEU zt0qggt_C_UwDxPFG5xs+4h`rB`uE&mSCXX+5bGs<2Akni2Mls+;$6CTH)6Z4Iycx; zG)D%#UkYpLZ{e015x6TL78V{dzZjABys5

h!lve_|TV>|fefEL5{P&4{~H93g+wo|}fQ2ivddSa;Y)SU=nY)mmDN`-MJB+Vj8m zwPgID9DerrPOyCYf)NhkZNy2~Fe3nIe$;Vs`~KE6FNN?W{U}@v4#azjUUK2WA@cH( z29f~96CAGz*K32g#kggnS#M)H*8;fcbst;2`KbLIAHlDcZiJQNm}E{@S(CxjY0Ubt z{m@D{dVQJ9iJ6L{E+j+L%H6bXE%0ULZ=o9<4WHxn<&qB6_nv!EG_O4fdY3E~?YmDx z-#boJPn(ebEwh$+ahrX?K=|Zt`=_zvbYF2jw~q46Lmj?mbz9l{uARg@V?Go^k)_my zXMLCOAK#6n^Uc-ZH1!I2o#-deem|t0+roiw{C*i+H}%3Fzn))MAE}lpAnIFgMaXL1h95oMP8fT-zyWwUB88-MJZ1(!8eYcCAN3zNS);0&; zF+CZpr+e^#_KP`jlfs?!?s)`a-N$0*C#jkp zVwS6p#i(u}_IEIBzE)r6S{1w3pnFu4s&9m0E6-p*he6o&lD_N+kD&YCC@AhXQ!0#= zFGv;!A7kLo@+4fyE`Uo*rcu02yv68T!Qr|WSI3-m&4smzIHYPyZfF-;#YvYo3*a~E zLH%t)IRZnhG{b8_nXCUVumjX?T@mRS$nPju}iJvQsf>gmVTyvSdUF&g6+L-<9n zk;JX-_>;-RXGW*`%T+;h_4Oppan7HAW2(C3){#bLF5szsLvPS{& zEyg@=CI1*5!45B5V&Rzs(C712@_*l<&7eh9pXD8-A->heL6IS_!cY&hJb|}~Xd|01 zi_>(iX(AV9JS3ePh@?OG&GH0H+Pn_YPw9=MjqsJu(oY#DE_V-E;}wNXM}S>`lq}G?)Y{G`i?7M zhrd1(VZK3xr5mD2Dm^+nIejyz&} z7=Lx)rmA1CIXD#CXFJHjgZr|&8|SfQlMCI0-a3#Mm;^z~CXtWWigyi5&@9=Vt2sRQ z(~M7}teCoUCmOx31+%8)X&qu?PC6q9T@)OB$7_!mH+%n`;6eqq6p%pSTz>jEButU zZd_ycy2DH{;P`m-+nxsfV-4VfV>{A>uo{*o}b4ifz zB5$>duB~?F743#YrbC<<^KXINANz*Gg?o2 zJtO|Ajl+8|GCI$4`WX zosE2PWero~3NOeNL#8!D1E&gFUwxb~em4vA{RgKl$H{FrU4Z!kK@&4jWe5&)>Gs4tvcskyOH=s`>3$1Y&Jj9s?hE0*JTH0`4c3r6k}_eKMd|CD#>-fQxWNcZVJJ6VZ{}pfU_M`fHCj4VI0{H;t z>!D)Ode&gbIMJ;BJ*|F(N3wUoQWr>=X&FQ-!IZ0RU7diMbG9-!BLr}7pmbI9wntHDp7Q{W$C z%E>2l${zF`VmtDqb5Yl_F9f~+2?6cD;r*<+=+v$d*XfN$@_Y22Zv6O`BZ730%bOZG zBWgMnEG>eo!&)KLAOzKgZh@P-Z{ws*d&&2QF{+EI97MiXqqK{- zABN>!L*ggQbf_;4$81Bw0HaI;xmLdavvk#ARV`l|1QSGTEW|>^Mie-+Mp4A>?(W3C zwg{LY79s|S9S9aGXVzdM*e!N))7SBpt-o$o zV_*k#UbyjVX1y%H41L5e&HTS8xqJqd4Sh?Ey&ekq1Oko{_|5oE=yN4@wrDS{yf+%II@}+Ew9` zODU`YFadk5EZhlRbJoi}GYEQ1S_`!z_#rZVR5`*+qW%U>UT1{n>aM8 zE-)#PqRXdI%!y9CdAgt*4m~Uh!zYVi@7ZF@iUMrB9(aNx3;rNzKk;qxRZbXEPC|=| z#mkzSK2aCC&mgiUGw_ltO}?uVhJP0$e(yAzgpvJ*~>U)@TC3Gea5JL0DtVT@phZTgo4Lufko=5zCyFBD{T~d^P|)SnpnE#) zsXRzcE&PZI-@v|G_1hZ_omHp6RYKoe(JKr-%KH6j|G5m!NHM<`p{~c%yQ(1!ouZ(} zaGa-@?+_k2FUqoh#7H$K;RVBk(%sPx41DF#O<^=;Zdt=yR%lc%xcR8j(RwoOA-8WA zBzhk!f?VJgL%XS*^Sa1Yxl0n}m%&L^n;hF1bxwBaFU6>W!Iar=5Sxq(`Ymf5=w;*8 zsd^26K5()$`ZKrjsK$qo(aWeIdH3mfy1$^VoMd}At0$YrMAMkr`E<7N`+3%zErw%_ z7p38)`Y6Mpt0z@rOTmricgZWOw2|;q&_*?k-_Fo8ynA&8hRzpbKa3&ZFB`p;l#ni$r9nEjhClZ4W=vTMEv(#<#eVJA**3zLlL~wD~{tAT+ix<^jLqs`zV&9Bj<7I$Oa;)>b zicN2tT+h;~nrP8$o^t(L2bmV~fY5X5RwCX_?%c@gZ;Sh`%GA!{dFV3Va5Qj$M6CsS5w8J_8?E&t4yz- ztvHPb`8URTy%*3r^fIEE1ST+Y4P}=;Ouj9&i`NfzF_^5fjrZb$BfRypqVZg3m#Y;T zgG|4~)LF^Mz~t_AQ;=Icq#kF-NpPXSykb7=^{Io!*?5BD|FuWvcGBQKyygvp2Bp@^ z?^4cr85U$V8u^mZCCL~oHGEdv3M{jhEImvb?bqJrsL8pqG>`kmlhFD-$Yj;%A4*IP zF}MrO@`Q|*F_~OV=YbNMNFh&P_#kV-xBdc|tMO3A%hJGuxTkMV8Zxf!^zhp;ifmpT z{w;-o1u89M75RA1p@W_JFnlYa-x>Kr7Uv@SfJfR*+w4D6>fIpdpKEO9<8|P0T3hNc zBZt+ro3FWZoX9=;$!)2jpQZy;jcSh1Y^9-o300xqO>kwl5HUatm5K z1bgbbmDgd@YUs}ezbl6YdHQgDCZ3gm{yCr?sbjVf8+}x2$~^*SKTWa)X+sX@JF1R;p3o8W1^?)5mv{@4qbI zn7;RU+Nsw3wX272_BoknpG)E%H(pZMI|sZE`z8nE>Y=N?t&MR7QSfor!^JOBR}Yr>0;^<|%jvAEvVi)zhG;p2FZu60T- zx${9HEvlnw;Waz`%WgWCbb85!QatqDi(b0vay(#ce7sZW{uW5H?FY;cqi!nYkm)ie&bG5JLJA6ky_D;fHf!m7d{V$1jRP0BW`s4ns zH%csPA5S%XTIx&F2Q!XErP2=4_k-od#faQ;*H3TX)bk_tUjO`J%!}+Y@#!_5*DHvU zkNE1$tIIj+@d56=-B!OFx><)@?!h(Uhw#wY5&Ba9SPs8ERN@>YYE@ghKNQ0~aE)+o zn0gu=`Q#5?@n^*3uK$3%RfxEA$v&U)R|!FP}F?zEFir{1T?ClfW!ljl6czNLemI64XM z;Vrd}Gfp0+3V*BXpn{e3f@Y=3>h@Wcd$NTxkA4v0IakoAmnXPVjl(>6?g05<;t%ru zwUs`NI>QeNAExUa9`m+@(sHKL96i#%m>CQ9?wxNroAObt&wiHB7ucWl89$ofOOqXL zSx?2<^RBb4G!HC=UH2r^Ea*}n%>$&*5wz}qzHHTpi*Q?>~X z<=)5Bxxkdkz@Hv`|H4_CkYhjRJTpU%ZDON)EcMa$pXTVXSIf!3#kKTN=O1J{b+%6I zx|dyN&*g`48&&4fK-Bf=WQ>`|6<=xn61sxF#NJj|C(iIKNoDpH@vVIA6?;1^#J@b+ z^nQ~S^{K>Ash{?7;rWwt%K=}Cad2`6-RRmP-Qo8aM+^E{Bv#=VvSd(z92SG@zgLh-9lZ~|Qvuhyzwv||UJ`38^iCIvV-ePw z9-4RFw8ArcBRBEbeog3e!O3dcsq>bLC9CP}U-FC6(M}S0&Cd?E!#N&K-da|P=%jG& z9NvyjR60rq9~)hB;NBlR$*sOcxAFMEfp~_wL9Yc|?@&qZG~pCoEV)YzZ*+t9`Y&Z* zsvJJHA({HnKE&N$hf!wn9JI58qkc7`G5HmZ66Yo#A$%VD{1oJ8Ehp*+J!IdI-bA1LRg{b83ZZk8{bY;#8NSXo2Qa_363r}DjVZx6AIiRW zqP&)t$%7tv>obeWa?RBD;z8m18uP$YqEhg@!Z*I1{{{ow_0s6OYVFlu0&B;U9lA?{ zn?Z9!^v@!trF@#hvZCH;wV+un-S7E9M0H+HjiyiHmjmm`+uxT^f6)T_bVTz8`@A|h z)=^%JSU?+h9AeLQ9YuwUC+KxuT#LK4heu(PcdViBto_yW$$3fySFI_qUc#(T>3yB_ z`R3QDZ{lFtq|9Ri=5pogHVm91;Er@%eU6I`D5}AI(kV`GH_v)%*rD-^@6)-1!@2I9 z&0_kTQW9J)=5#C0|9NNU#a{wt^}~6%ddmdf*&$4%KdYl!m0!g6(QkQR-m)U9@^)dq zv+z2wa-M=C3Lwb?z|*^Hb1#iTVBp}Myx0w%+ZC@_{`>E za%8UuRNkYQyfDdD8hnS|u$uGqPWobS2-$rZ(f&eDdik-R0B=~JZwNfdQ+GGUGnaKV z&XbPpu1ABT=Zbe+187 zG`hw?UNvTo8Z~LCYUvP6m0sl4)oO2{6?^bb!0X4TYwa1lescln9v^xdHivKixIj;f zZQ<76-Kbb@e?7m{1k~_Q`5~<|S^QQ*FXdDDp6}uc3oo;8#R8&RpKa9bvb_#JT83i6 zaNqyho?^n_w`}+MC;{`SRe1+$Hh87clgO)@>2#YJGQw>wrOd1@fdRtIy}e^|F>_4? z4gQq%8wF9$JqJ+7AE-j$B7I}z2rl_0j?OIFz_spuCvYXP{0cc7yNuIkOXYx5YjM%@na zGJGuE&i3LaIZNt@!Px(~q~@Qm#vAQx`j1>!x3az$ir(Nlr>U8e35lF?ZwPhjo=tCG zGaVYnR2`ggk|?wD;qsv=j)|Z%NR16VD0mDY%l_!r5WiFyuaBu0nH-7 zMamN%sAYxb3~sc12`$9n8zsgMlQnl%Qov|hR-vJKIWUF3#P-P2)-y*dfxeSB+Hc|l zZ>}h4P32VmK27U9j-a=#fwqe|x$kTRPO&t!OJW?0`X^LGoh;*jZ%eSlm?{PXs7C2V zJhx~BwDC;_?sMX}pKP@Je`kt9_P1h)-iaJP_x?ThI%g74Zmn;G#a-xS&f-k zS^90VNX!EPhmygi)NONdeXi&{?w|h7x2ff|_+|Hp(SO{z+A4aR*qNaF(Zlb=9gn`+ z)KZZQPw7{*D0I_T0Ud13>~~Pq%XE^P2ISN&+Iw-(dgu*cm+vThxoq!KvRToD?4w7~ zzp3>(b^m6*->|ZJY~yP1j@ozxf^%GXS!ymyuDz1;H5sCSdA#LLQ*JmZzifBls@kw~ zGFx-R)37QRRkbNDYQxf+)Z%^@&Yx^APY2YbCykd7w2O6f+kp&CMwPD(XRl$Ur0;&b zi|6lOm3$_X24 z>N^>|DhE8A#BZkOZhb1r3`fr_e7Sev1ZDWH6|DdP ze>_#C{ie{>oM8-YD|b|U$ocz@flj|h=tFY*nFg&gmRuhkR?vO~Pb3Ol4uk&bZt9=G zN2XtW=N8S!=bLvHHAtsojpKWj;~3|Sp4)1j@#wk$23r?y@5(C! zyTku>G<+ssANLXPfaK96mj?Hln#$_4y?Lcee5SvK zKU5LB>Wh?n&-v=DGBSt5BI7j#FrB~DFH2s*1K~SbNHZ>@XA?q8stxwxyy=67@rFjr zs)*0Es8Rn!Dwlt$O5I<9pvMWGkm1L;T;?cxp8W*<&L+vMbDQAo(&Q1~BaUh8!V{Od zNO&^cMlQF^{M}o%NOjix7J%n%V)$^>0Zy#al)GIHq1Y*SSJlKH@Q|+=`2+QAo|7jW zp2I~ySgFP`H-k+i$9WjfKsXlXEu5jiX)?Ou9<>bnU(Oga6A_LD>yibTyw_|?8FS*Jxp;c+ppz1neb|<-OWFd7tU$Q`d zlHqp$l9E4l2m!y8(!q@3w7%^kbpb)Ez^U zS&(5^!24pyYOTz;fAw4{R<1ii(AA8*!uU@DztLK2Z9@H6jF*~c|3<;@3-lki`f*>p z`mjO4dnto+Mpu_T6GI(e45Zn=VwLf&;7=Yjs)zBK%J=+q1)fo}=lO$grZDP>UTt61}~eslP4-QknP88REAft zf37UyO@KZ1sm*Czt4i-}a7~+x%lOPAZ9^G4NF?Wc4gAh9Si=qN>hSy}7gb@`fePN7 zhF|(k=uNrjU{$WRybq)I`T3x~*1W&->0)9KHXqv;)v^MI=u7<#;6E3G!2}okZ{}(xYD+gN;_`1%gHrl_u_C)5GFffYAk6 zI;_~R>1s{&3yhpdnsqxy8>#2VkIH0M(0-ON+0I$hx?fW8+^R^%UP88oJh7-o-jd~$ zp$n}>6JVZ&SGk?O@b(O!!ZF3Fqju(-+Ti9L#)I>RnmV>rSlC{faRn9JX6nVK!|$AI za=N_p_ZhvxUB>6(E?17z(9lh)&APMd&C2tH^UuOo_XP#<99&P6jfq?Jj`5zNgL(9Z zCF91^k_M-+0RcEx|_Ld_6N4>)Op1S)u}_Xyf&c!3PmT$g?0Z3%&Feyk5{(0DV zbwb`rf0`x|`h!}u-%o$Hy&xQ$rVWc^Xl^xO@F;_ks#Etl0{tO^8-G`}Q*De+&&m|d zIQMgVPG9dYI>yyj&~|iaYc0`b{6gcu40j^`k4gSNpY@k(f1d@MUa63EGtNaoqY-1T ztJif4Ta3FC82(&wMmXe{J_j9iv1#gf>u*)aL^Zxb2r zUWk`(2vvKn{&ebB43EG5AN|2IL(?3bWT^vnWv-ByoPBH#ABX6f?D-;?D!h#5@{g9W zSL2H6$A9H?j{ow?T0YV8PHF@N^s6SP7br~lJ#{Rc!I@tQ=~kklh_tC}N%(r4D`&4O z=O=zwt42HOPrv?&W3FALZTI@}!<&!79RJdcD7MRAOYZ6E#$ly;sA2D}@u-B^GKYI( znwi#7wQIjX#I^cH50|dwjX8?Tmm{j^#Ub8!cB>_qUvh@r-;GxHZ*3yArcd&w{g-K7 zZAW>jN?Wee;*WYUu)N+e?*aS&nM0rc?BPRq2h*qo8{KoYt*kn{FPG|5oGiy&WcKv3 zTq<97eu#HQ?pRPj4*M{Vk|Tm?=Bc@|^od4Xyj>XA{p%t(_NuD~O)JH2XU?cQCF7|{ zkx}|XR8C#DV;z0sS~LCA{SMErxPwdXnvpfWTJx*RB4J^2$b%+y=>95^_h$!wj(0$1 zKN%=j=B!D=jdXWQylDOM7v5zxMfPrURAqj8WVxL(pD;GgXuFvf9NMirpKYpFjqI+^ zB=+V?txr(fpB7cXbvxfkNat>Ma!E6WV-qLnb3?Pq+V$7dg;K|O(a98XyC5>gwO`fN zaV_+(YL6AZOZ|P_Llm{7vh%@~*qi(>Z=cX$r26wzQ1x28Z+LZm|He4=v1Jnq zJo;FTX!(ZigM?YVtU(@%Ut@z3BF;ogfVZt{pMDk(0ZBr z6Yit4G3^TevM+=3H4)jlUJ`&69A zRXQTNdc0v^f@M*Wj$Ge6*s=uAc5l`WydYmI+$Ufom-tSWm`lRvP0!KtX5PBN+Fh1H z(_3=aV2iwet32>$1>3~W)dkYR~2Nn8a;bWRt$m)8&VE8%+-|O<9Qk8&5TB6_A%^6xF~C4m@61 zPVeu*{_Wn1y~s)2N)^;!``w|zHG8S9*MhXc;URcN*`d)2^ng9L3|}DQek@X7_Iy-# z%k86Har1?|_M3JEbdx!&wc_qRzbvQ|y=(16`ZhdL6_1Of>+3CI`Y<&e*6=n+u!y)0v<8)30g+_`vo1g!)z!!-vaT2W=(hj`tQUAe)}d zQ2y;kQTNCBG}cIBoP1R{i(7-N0zFE(?t04}4Qf;4{Hr+rc@pJ$oysFS?dA5nQr=rLXWelsllH4_+X6$%K`VVgc|0{o?VpR zn`2zdrx^I6h>p0MU60I+)Y&KX;CZ%(fx91gX~UY_Y|=5S>EAQ9?y~G0&DPfw^n@B| zcy93)d-Mc{EZjypXBEOZ&z5%iW{E+SuZzd;eF^w!jktY_dj?G?@TdC#TBIF{ikJZ+sGV2MPM~0Ksy8JS1 z!gsFXDAl&g(poPR+QGe=etE13d-M*GyP7P@!f*7jICm(@awhSfZ;t=M$<$xmmPS;j z>69$7uf>V++#n#o zp3MR=9fcS=@e&u5poyQQke=eY;6{L?!Y zu+RR5T=r@Rc_g{%*B7rb)Sb@yx$nX=bip5B&0XZASr`=_sf zY5r9JM9tfIND(S6!xOjRpE(52wAlN0=)$ zwv4Z8cCNSn(*2s{S&!HBv~v+@c+_!&_n}P(>}ArWg$5tEdbtpu<+c&e2u+sWJ(x3o zV!yRMzu~b$frUxb?aq7#b|};U-#dZ$;s$*?{Ett? zwPj#$GJ4Ch+wQWc|5VYJp7TU%ozkLuIeYSaVJBNH>rHPm?ttSzaqlg?ptaHny1>*Z zP3gE+VLS@Y4RiO0ciC_=^oYk9`m!I@tnq+4jovJtJ9Jk6Hnakd z{8Tfsza8dBuE@l@?+$zO)t60mW+M-|ZvG?<-d3ZNa%R;Uu!;+euBU-{`v0+4u5TJg zwcjr^V-O8i|5nY)PgAemUel2y`wiCOyo0&3>k{&-mq#!0{f_-%iI1i!iho{*mPG@X zui5I3gWiZW(~HyR_>%IIia%pguj4rx&X#@e^I3Ukb%jl$e zqUiF1664iVvvs0_b5DzN`MfkV3pB+8RbX@q?;TK)HugHe1Iu1O9bBcu{g*QIBKZzG zE$}niW-q}VQOp-0C?N`8HHNSgah9?lvoYc}w__lsI6~~%NjqDbx z#UTfAZ%!}pMgac`nZ@DmEBM%Ccl{yq8jT%q!_YBYD(@BOn<&2K_mo{;)v!WCspq{) z>49b2@U;(ZxN5-@hVymqx^wZYdwpuSalg@2rf29;gCGf=Ku_94_@ciQ>YAauQ11>j zv2JU|u?Y2{^QUy+`$3&)z=V}FwBBoIayvS;F`G7C;&9(Sve?H1WbkgsgX&`IxHIZ# zv0#~V)(TVK1l$+EM7l8Ho@iEj3+~HZiTVg9OJF-H7W`yH()Div=;K%PC;97c<@&>mo+`gSOcysdU0v;BX4DGs{;D5x1f6K@& z-#-cpJ;$e-?-vQJ3&?icH`BPO^JsG8#SDB*t~EaaJ@Z+ZUW3n({vBeuZQBfX9UN!8 z7S(?$lo)46&_PzEP)bN-iDs#=rWSjULqJEuA1YZT*9bv5>Hi|dwY9Y}J z>Zf~8GfxUWQ9)C3wmdsj;i?CrPYQdjoplB9w;JXB zd4+m!i(sQMP*=kAW8m1<61A+>2FICW84TmP9g~DdZ+kB1S&6~z5;|6(ekfnxnzZDJ zqehQX*{k+=zqE_y_Ll@Oo+A_b5$acpv{8E8rHxtt|K{>Mb+LRiMxVj+g<9Ht+9zt= zZ!VfmYN+8?eBsNKnaBEd+DYhMx$=kV7C7iS-}1|kUw4pyhtdrG){VMvvBFQN z(nZ1qIE&W?m*J!fUAfZMWwi8lL8G-5>X`k1UF0S%rMbB~@)d9#dR&1kjILngvC$LO zTL*(^(B))s@IF4`++5)t8Javx*BDJu_)Z!0f{Vu58~;w#inp-Ts{7oM_~*2+?S2?p zf~!pb9F>I?riOtf3=Kuanuli9_Mc~St*BQ9k6Y#Jr3$(Qe$JVX7Q-{xvG3{A#sT#1 zK_mgkb(=L&W?n2QJ@U}|B{mv9jt*S!!|ijPVsHbzdKHe`T~Nd8`8p3d$o&f)wN{QF zlBJ2@U-@UH#ccG5Cy zc8ner7bh)|B(LLgA%__bfQ>>KPXa|E6C}ZnU70_oInOx*TsfKjLD>tET%si2)OK>=galtgo zZUli>tYxcY;Zhc#^c&b&SOPBz@T!6i;YTZ9vv&_7Xe`1utP1$6fZICWFNDnbL-VS_ zx5k*9iu1_48aykS6nQK9?0O>Zq~_Dp92V31re&lVhu#@0;6X`uw39vGzZQ$HVvj+* zGt$%!G%i&?odjJ!gONQbnD5jNdYT6S!2)Qh#x2>z<%`>ie?v_Wk_URzI@v%95);kMz3;L}vdX}JF;5klk!_Upc!5Y;WURM}hHZh_kne{TdcHP4hWHjl$ zscpo|Liu!|3N>}Wo`wXRht~Y&3v5V+SEr+CLiTtukbzxU+&I)@fXl@v}vF=85ce~hMk>D z5U`iJ23rNVLPR+o1P3KSC+3&G@3!LJ+w$omVU7x!B-Shi{%skXti)y23m)-(vA}oe ze7jSX$pe93*2pfYVqS+Fz_zhs#r7~ZJd#I-Q$owW1pUS#tNZiCbMf%>eGFb(^7Y+q zddzTCmL`FAQN7}F>GdD&8NF^rZXuB2XydbumwZ#XyulX>@+bAAO+IO`&?mW%c>lbl zg!kh8r-S*+zH%mS<1zWC=ze*vsFz0It2lD9xy@s&J$b_YS`wNCfEHm`zEUeYYWO;kGIqIKp*~BjvRtWJ*xNPWIw zIDIU=Ma}c`mNuzHrT^l4qHNe1+|w|~H+k$|>h^Up<*xtNGBN)&)oDvPj@ogG!O|1@mqm2o;Z_Bkm{VOtvcf@R=GwXeM@~8~%b6$vvl|HGN zjnNBlmvV(7D`n!T%6ej@alGq#aaljVli(*8xyQdkoQZpP^R0TvPv7Qb?>w)}`Re7j z%jzrF8uFz4_PRty4$hbq!kf#SW6Xo~>$|fw=T=8KFXJVZ2(6hl@9mCcmtQK!%h+$t z)an)vy4Bd#;`eR`nJ1*btb1n~W1KRkR%NzHSkHgM%E_EbN@owe&fVMf*N006Sn+$V zkLTLLiw#tX&7RU!>~VAV*<*HZ6r$TqbCauEEF!np1>~0__jp{lGwd?3wm3PluAZ3x zH`_OSk^H-66P2(=(Nw2m()|2Ws}=NUs+azit(!XA`Z1NgSB76~_U3obYf0JiiHbbG zhpVm|Y37jKiv33^qu1~dyHpMuve5U(mg!WqZYnk0+ZgZN`OQ10rtr1mmDIH1Z-nom zwOES+az~wRWRn`DD?cT5b67!%@7GC-`pCAGo9ptIYVrUtSN$-lpuQnC>p>~^+2z0j z5q5Q+zPDnfwRpx1OY3u^q;uj^-qf=y7bu%7M$V4mvJ*RUu3zi*qhIeu_Q1~C!G0^H ze*MT@vf0q#&;p{VgAWbs6USqB#>wp6`p{S0S5hecm%8MYM>cP}jy~q9K{H%pMeP0d zTP*m5YBTo?RJp|@n;<=D@*-fG@Dqq6)~+tV_9Qx5WM z>Lqcm^6R^ndQXM9mVhhA`A9zxPIA~pX00&a)ONd*JU6|at{ZSltgTmE4-b!^xzm8L zZ#Rovzufp;WDo8nJ5!xud#K*y(lW5nGj3ADO?Q5sOYXm$L{m=HmU@|;u5|D`)^mV* z(z&G!!+pJp%?8n!XWy}>Y7KFz|9dgAXgeBVb4xikbJP=mInt<9JN8m|<~4q>oZ=Zo zs~Uvr!)tTvz`vu-@#O9YZiI8^CU;MnI-;COg4V5mYY8v*gg1SLrYvSl<3G%0FFeOm zXh@8*Uy;I-4p-yW=iXWauK3d{7YCMg-ilEPj{3RNA7wWqjl{7H);ov39>U|HykT}B zx$Wk3-e(&_6Ot-2`i%ND$fxl)^zzkOtNq_aR6FlA^mY)xoB2e=wRe|SQqJ(qUiRWn z8GrsMC<5vgx_NXEqXZsh?LEPkXP_pjxZ*$UkY>x09KLBTM|n8uWsQdDn?)rjkDWolL|HeX2)VQ(>EVIB-@DxxFNb#}QxD#$ zd2zmHsM@1T$#1@F5M6d}*O(Ve@K|U0zOb8aAG(F3j{1p(r)|0B$dZKmqv7HCWcrH* zS>pi)sV!|3S8EZg`F9hQW8OIhET!V_zL0$(JNf8r9+_szpgvDM<-s8~;`!uLqW(5N zgO$=?WAo=1$nUT>b#u(Y%g1$+_0DdybRKa+l_)tt{pp>=cmH7wym zrpaLXy?c?4YA{_qeEeVXqsKuKSj*kMIvwI^j{IwcyF@LhI~U#%=0c|4v(sbR1Yqtz zbHh`bfP4CTzxAkINnZ9sy~dLJ58s{OM3C}^2gY|P%y8bw3sk2qI&<8^m6zO;di~d z-kAD<>mZ98T(3MWx|+#dc2~9zO6fyQ7k#!G?f`e{Pft!_^O%3MtXiP#w(*eRW^MWb z^?*6-qWgH~X7BL_rDcX>&u_gL^PpqwRmv_0wulrl#rF!-D4&RS6?X@3qpNR{$-d!GS!M25&NFI+>UpA$YEol8b$pnO z=9j!6md-kf@qFMGGZ&Fvp{X3b|AH`cm$QzI2LGweaoBsO{cdG?0c)*8TUD~O4|S)_ z3s>vilRL26oB;8HUUBwgRkf+V^jS=|0;QZ0kAsLu(qoHdz%3pBV>310eQx%!&D zk;L!Vd=F|s8Lk74srpM#SsFH*!o$@Bx$({rf{vk1@;U(z89FCfwQ@E6%H1rldFW~z ziq5gc-~_E3IiLD>ol9pA?pF9dZZ|qnfm^LOS8ASlQlyM7AqSM&Dlm8YK=L`u8Mldg z>x}K}xyP1^^xwObncHL5qs-5YsyIDqPaGdNj=p8|q(DD;i_{rg7YS-b7 ze55vUpQn52XoL&r$hnMe;XR*NKSH0PmmlNvw}g8mdpW1-PC_s7&DnXh=l#LZ5(Bw; zW@BL2Id!j4dp&H@Dy(xKUHa-8OVeIX@*3vaKw3X`J`a3O7TgiSyP{48qMS9i~VTDr)XZbYd_AVDsRC3)~yaq;))9%==;Y)Om9f= zCSTkip!@E)!grqyV5bV@dHChmJo|ZoT(^1@;cxl+{dDv|8}8~jLqQ+wFB#WF!xDuo z(1voz$Ytc{Sy;DUyN(+V?hfU5Kq?bI9xAd-d9M2a^8y8MR~TJp zYOeFu{Mz)*-dFCr!^0^Q9$krsZTibrJP&u{au*BEMYfr?o>IDO5{C+VSO!h8<65;e zpB>cQig^RBI!RzILo?GEx36@6XI=>oP)QSBh(6PwD)4p|KlqgC$91D(sY-)l4DC%j z&lWQHX>BrLI)C0igg)av7|;sz^evuQ?&n39rVJK$7CIB~TY%RH$K%*OQ3^dIUpR$g zZ|-AM`1n}#$4E*nVWTfS+Q;)pZ#EokJ+;e&YAmsrW1dY`NA}s!u`Lm*)}H@xjGAON{bfvg57X-soWyPYj#{>V7*FT^L}IPVX=qTf z&m|K4Fy6PcE>G$IW{b>!cn=ahL_$a-xonpgy-Jk8D(p3X^fcplgmV^SSL{%gyqD9? zsr3}JBNZx~u1u}9elm~^)_lZ$AX7YTMc+T$33Vq2PMK=fTcWmU>8!?Ce(Hv(peH14 zRnXMR_^_^ye;L?AM&}!D-(SN~EN;?XuA1A=_!{-!npEq_au3)oTONZ;8Xij)&-g2v zUF@niCd^@@k)N*FL%|ry+I$rjKcQ&;Skn>gzskm{}i% z`XXo}fq)+TF>;tY+s5jGRYRJd}H11{u0T+_NtLoFV zpGGSx_$U> zv7u-aQLyb+n)nM^?NuXT=CarAs?ra8dx4`Fy4TF9@k`qIWKW|jw81RYGeu1F6Rtn( zR!eRo6jl3k1KNLXK*Y9{(Yz5CpmOrEYExQfM+>W*YF@J z;FU!f#QP25A?ro>Z z@D}FG=xKsx)!iKDTEHP1{3yakSW!DC`M)&*SsEm^shfQMJ)Dm`9F(PJc0P8H(0jC| z{~W$P>9^@ghIXfe&y_gx@F_3eT^d}ooWSAg&Y_9cYAs3|U(E2|jJi~93S5B?2{v49 zFMOSB1(bxj?WRySS92nf6 z${+C^D#JFmHMx~!8YNB+h2_zIOcvPIRksTU*6hL9@x-1 zE2}Y>W_)&PTA0{5AWbapeaiZ~Nj-vg6`3!-@xH_~0+%7@SggR0M)Rn?%X<>$-}3ry z6&@2;0@;sNZ+!oWpdoq0&0)(KI!eN)h%bH{$z&s@&VjYk9s7(f{I3dQz6pLY%X<%- ztgv4v6y$BUBReTeSc>-_pD|op_M8A*BtcyG`{@PgHPwq zAgK0mH>kBmRW+>`fsZ;32QdTqk<&&iXiAldL!d_1lsl5+V(wBHLeR;Cu9_C<_$VXrVkBBMpmcbm-+PVReWH>bphP6{;PI}4Zi<;{ZP~{f7WVp zu~xDFXjB09Aj{+|FAFRY=RNFo04)^@_T@=7xQ~oSF4(t_fpeVu@KNz()bXrr&nIp( z&krprooEd*y^U-z`djFHHK<}W!nvxtcYgCU%Bi0(K4P>FH|!f|g$_VHEMlAk2R@sv z98WbN=oo9Kj_0!Kpl-rS)92QP&3+1*v`4k;;zQFqe1%V`q@V>g@LV9fQyJ=k$r?nh z^!MBj*=^<4dFX1Vobv5Ot64KL*tMhOMP)p)$z~EgNBbHNZR+x3spVFac|bc#a}I~s zE)dWj?D``Z`zd2@h&tUkEZe9oeGILtwytoK_B)e=$@xlTeYDf7i7#&;`VC?gkA z@YC=IKe_6D>>G5wJV9SFw4X)~n{3tRjl*jA>vkHNUcgHr56L0Wlj6erO{TxO{EMx~ zcm}Gs7cE>kGEC*p?}ByfY&@ynT&5L4L$bl(4Aqwxo(^JY5CKdO21B>ct3mL+vR3hv z!uU{w(OU-ZW#ErB_Wot$8D|ANHXlfC#Zd)|%9&pd@ca17$k$@5W8&5blW8Dt6y+~r z@3~`6Y_ftT6aFY@FtWRVd!CA>SuwZpSyjMiV+eVifS%$=?1`52rK;GM*n~d+*1XxL z5%Q|?eEw)%%XZgCc&}YsZFDH~t)vf2=zH_+R^+{O$H%IS?}I0k$QOY>j%?Ns*_ePH z(T4kD_6(#W`)1R_n6=1ba;#BgGEbYHr@8I+#R43Wm4!9*#QiA9fw#{~WbIdtjCVGC2R)#&GLHe;^5psw zE{1GHe%LELD(^eKItXcweFRxS( z_)dx3M1z;4mzsyXK7n?$Y|A)j1-wv&W=7Hg$I9Aui&ooRM$51{dFb-W>3G)>i6uuI zbjiz1vwP2#A0FDs+@1$$hFd5{It}5DJ2vuCk1%<&z$^-B*pJ6IxUJSaiBKgklUyMz z(ml*i%=J$aE-MuE&eoB9!W;8}E2-RPU`46hr1PkyRk`H}H<{VclOuW`;yTHxv=Gl1 zq}}y(wyJLSEi3Btz&u*S9ME7ixTUYTaP%; zmn)Q$@xcxC>L_xe2QMn>JJ0?J75qi4{O`HIIq<{w+h}g!d^z0xhS<{REf+qNS1+>+mYI`1xyrwz zVt9w0l(zMUD1WHFh{~rpZpaASlk-20t~(&dE{aQ95;D?2q7)%pz4u(JjO_8*d+$9m zlTw;AjUuEa6iMT~=Z3PSP_nnojL2T!x&8Hh>g{>%J@=g7IPde^+XkV_MJ(8|j)}5< zGBm*oqP+4uQK6qr@hk!wuIJlh+Z(u_`sIG;u=hQsxe0-NY^I?xxjW4V2 zXWECV2j1EhFSb=?v4!Vu3;&Z(!Q$&SywsPT7Z2&n2hTVlwk+u^)_r<_@q2qntGS3X zZaK(X3x>;GiM2t`#vioFma^WuU>k8L z8n}AEeLQw)sjU3ejyvr$gM(}#Pb!X}IV8gZd04~-aWJ1k=XB<|OUKT=!D)FuK72V+ z>w2h(>{YV~;Y~-ZUB{Vw{qBf{7rXFMBTsqk(nqjw5$W>rh!>>KtpM-EkFog5Z=m}@ z^7000#z^9^43Gzl%=e2SfO^s!BjXm{|fFJG?&I|gFEhOk=B^KS>(t}pE=XNH8Q%{ zJbtt+N=({S%=r9^;z!Xj!0bw#p{e;=X#7N(Z|C)3`WVb@{1(R=Y!_!3Z^ zTXR#{U`PlOws99*OQ~Jok9(LMlXX*_>>o;$K6z}(x%S~WV!Bw{J;(X>}8?DBUp)qBGY7X+o?G?QqxC2z{5A-vP zyZ#m<{?=lrhV7AkeFwt+*12%RCy&wf39H`&?Y~T+YB*EW!J@D#qLgf{o@#rS{ zGJm#ajgO=n;un(EK+Bx^!tAsbdnUU}i@zfK^BWy-|F{pL&4F#b>(A42b^zrb4AvQf zU(Y+qXB7^b#5#-c?#dC;{!CBsJ~)-~d=^B+zrgPMF2bmn!{ol^rkYq~jqAau(mX>Vf&noB)So_H$+8lAf`27&&`Vt*G=Yg?)H(vQE6)5+phT@Pg20A7; zk_~CUoJBLoNy<-oySg3321)E=HBe+5MZ=NNOEgoum4I=lK)LZV^@<(c_%vq+(Pmz6 zp7-x7tkkQHgJ;Wc8WzyPrIo02u-~+x7!c7;_(f3Yg6h13u=?q)2d)Gre*feF1w$jUf#Q#?qIyZHBW&C`6#9J_h)2ulyo={`z&WokU%dMc zT70cdJ*^#B@%pIxf0dW~rp|R}TdX*J3IuiaBVeU4R`4T8MpD zSESM2Rf=QlEHmZ>5BFozy`DhzNt`p0D;_v;*$qxlT@7yin~HUVYVaF7e#7mWHz4{1r~MIqjKwD`M9))t+$Jp-`VF_nnGH&{6Wfi1-I;Tdu94H+;uL9}K2;dN z3t~{^!pBW*Szu-+-r9eiQSGC;2Yp7yFD*wG+B0HF`=i(@BofXR(>~3lq0U5o#r6Mn zK;l}Y`^ye1b~0!iBMiS7VpzuGHp_D~*1Qw|4qs0r$z0{^(_D2rI~0toLkx$pZU)gzJ)L*hnErT2Z6zU%w4DbLO6FDchq zka-|Xk4qN+rL=%`Z`$*<{*(BJ0Y|ajd^5#yII&%^*0!u3?HQCMMmN4L2%BKV>Ir;C z$|ym4Nb1?pK8zi^GQt^*$pSQ=!m#rOb2>B2M3eHoR~5f|{Z6O%HwMbDpS^*&TSnD> ziJ$%3$w6oLV{CS)*!XKQ#GaN4pJj_7Jy2=QPL_S-=Qo?N=%o!eIBm!YW58G6kX!8q z*k)sl#Bb~=Iw)MxUVXm~oAt@n(mjy$6b3Zv3R4!AppVHDp*ZKYt|P8_JrOC_faZ=c z-AWGYQG|q-O6RILVuryd&15Hgbq^NV$%HHISF-aOo(^%>7({e|;iYkkYsJZBpCH9F z3QBek!#fZD!UeUTeRofc)Yo9fv|VtbjX8{VkB1h=!eN^6D23gO@Fr)PF@1mb`8XQ3 z4v-;}o-5rf_|6#WYwkeeXsn)jLVJGscbxY+MT`~K+1}Aoqw8(J=4@&$m*|d1(mR-N zx18yf8(=+e7fCvRjY_|UIcY24cDDpcx)JY?|Mord1nVRQi9_ppf$|R9gMyIq4$m2` zLE=eh-6jefjj1h($8)Mb^Mgjg+i-u!1%!w9VS9%%XOAaNU}6vi7grRB;tpzJ1}cr8>bc~K~S?S z+`6WVoOSpR5GRthbd*Z(DQ&W>sI5@rR_l59&<%*}6jHTLNvVsF@EOL9zX7ff_k!WY zlM3H?4)yWU^BS=yO_Bw15~R`l|D-{spV?D5V)FqOJ9t50zzq0&QU?y|m&4|mPQ2Oj zFqpn-lpu|Wr8DBOYm2@Vb2FY6?21~8N+fPa!c1r~uP#tMi0EdoDW__3!ei~EpDmH@ z!#vwRg2R0q3exegVd7-G_MwpN8uSW!_q(k01duj@fVDuB(a(R^gyztswF@X?7RhVNFw!v^-Eu*G^o>xFkq6WF1_i(!YsHl}dX<4GE$ zT!56U_dq!=2=}qL?KQ=1NS+70|LF6J-3ydI;M+UQ!qJU~D^H1E!<<;ao9Qfhc_dIx zkak*t5ld!s;%s@~j42PWo`F>VGCzN>AT5kttt@zZQ#W2|?TzFy(LKLFlgt}PrR6j4 zEyrtlzu5VGR~Y5J;(OUmyIV_GDv6UJboMN+#?juW5fA&5O*~Tzos#k~YrX~T2_FaA zAuBoMJEwfW3Nszj1KWXouH3a&DBMxr2`=|&#uE&BDxZb>|IC*MS-EIhTo)GqIzm`` zR_Sq|F~Gx@#gs4EE-lBLhV_{>lupBzV|p^z4Fj0UN2*)qtM5*CpJSBQ&K1#tzC78RY!+^Y0LZtxde`_!lE zdK7;_oQ0%!K>7dKuS!AH^~&4S-WwMtui|MAoMm`p>3LD@-t<%t{1@MP>c z7VJ&u@y)Fe`JKNhey24ZD`+g5WpQ`OBMtyxX2_jNc0j?%{>ODuTM+pY?WMd0R4#o0{NKGhzmI5h#zLuZ%4#r^xh=6)Cpz341_Km@^(-)Hv91* zh@2aOL2ok!VS)6TtS@ZRx)Y`>2I_$*T_KciC!H*OG@VgD$dEMQWWrioP8x*vas_cI zxd6iFxbQ^iDR$POS19}B!iBEZa{KR@7&B^~X5%PdRKBIW))D0yA%DkZ>OC+)n1e@y zmS{-#Dc`5HaNMpu4yU=hgnlu`tmWs)FBA&WSXey%pjZ(@`+|}0RXw$v`)CcX)xfUr z&r#26CaOC2x&0W)M`~!zSz)>j?JZ)>RBo=+cr)$#3)pb9mXW^U}!#cq8#H(0cHW4;rc7 z3yl3Z91Dj2#EBDHa@B{?9c&=HI&=aR_wTdX1M#NV5YT}WH!;FMsrqx*Y>U|4e*c(X z?JiY%hWbEA7>dLhRI3$Y=ITSJynlQ`IZGXCCLe4p)+n4LZz3GN_H?HDhGoCK80C*P zU~K@>JjB%(gQ35lH4mSYBT~P7K<$cX@oOP`1q=b^OsjoTs(^%#WW>*cd`FeLmrP+w7dzke}K z?>S0KeH4-Ytr7lN;*0XQ6;dyRbiy;O#b1WhlTy7`Abq7#{(<5N)Dz{AcV18*v6?K> z7K$@bt4X&RNu>kmp46Y6AXSe@=|aLY>Iwc}!45O4^b2856)){hWsG~{Gr6+f%3M8IE_iY709vzkLpq{I^ zT(cQr#4JX!5yTBhoF%r64$>%1L)eX^Er99<6t^nBM7;$hKc;#lpEew13;bC+Uo^6g|V!RquuRwT8-ruy?DE;Nc;8F4QB@uAukGbO&!`Az6V? zkJ*GhmpHnV9x;9pNi`7{c#3%e10naD2qB(a))z|f?tt(X@k7B2x`*7kxEZlg2 zde1q;Ir-Y2H}?SfcOacEX4E=FUUMF(ejNP`PWqWhe4r1}eYUG!7o$EW<_51K&u_;F z<5>$j7wy#7RY2ZOEL}8S_0=?#3nC@$43aMpO3Msx6@Sorv9Kw@NOh|9 zoRqF|LChU-i`B^5TBV7IzmWRWsy{4OEvc@0t=SY8F>d?_VKk>Jd0dS+`mQ(ih6GSf zaKZu2s(5dt{@S4Xv3(Y^WD zrS_V7wV&hO<0f)owQlnCm%6;^%_iJ%hK;P;Jpo@Neq%Sib$MxWUnpn$S^Uv-tk|{< z|2tzu`%ZS3CSn!svABw79v>jz<*&u5seb&`Re?Kso-oh};=z?s%*n$V6NdYM{rJ7O zBykva?A3`UW^WS5oGqYVd(@tdPX$)p4}*sFfqTC?%ZsHwB#lu%9rqE`|9ctiW;uVa zW9wOOAn@vSwj}Wygp59dzjx^H>0KWS6Y+)ie(EV!>~xTA>oiBVJ_EV+*f1JrZPtCL z37lV?m$O4#h^zB&i12<}_;-iBc*d#(4^=xr-#a_-%2xO~{TyD{ZY?Xd*$~>Ri9E2`P_CiPq;d!ONOUcv*+X z#{7X86VB~pM)QU{yW_9=+u+RIXkIh(<3WvGZjUUN7h)-rwiTXucGiEs(@k$mXuqdBnDO+}Cs& zFMgQLm$Y(|H=pl@&c+wCMMEp#)dK2o`OJa32fQHnLn~f(AWW*y_IVux>D{aG!H3>q zoih{hlc_go^E@Hua3VT9s0-K94SBV#W2K?}5UF?fJkIje;gesFh1cnQxVryzvl@I* z%XOUMAo|vM1kR0rLH~y5;YsOlEG*avS1&Ze7qfEdY)=|XKTG*EDxdA%@)W?~5}TT; z&zp>BjZ{O7avTr7_C@_)bdK4c;rMmN68><@c40)%Xi_d?fRUYiQyd3>XfLv#o#yk6 zlcpi98CX0&C)SVo38T(0k!#=VVAh@`@Gdt3&g4kgG~*(?{!Guk{1f8v+*oMt>J7Kv zRFk)tj|a`BdfL3{E7{v0v$Sbrzc6~c4hbu?G&al)UyI8>kHOGKe$ex~E2SL5_EMzslp%6e)-JdEV_X=}wk3u6^m&Gavc z(Cy}UR5^bqqX9Nzb>!z2=F)Gx0r$`g!C@Wy^OCqJsQzw+<~#c9=Aza*4a5977CALU ziZ$%g`9Qx9fbA3O%V8DUpwGaL(rS>a7_iY5_O8-m{^QNC({sJHBp;;8mz#DYp+$K% z*1E1I`vfnLCI9L`lZ>7`tYs3+8N42gH#C+XE=^@~QVzqMfI@h6aJEf#Rq^yB(7733mSW$FI%1a>`S9e~|osiwAIJN;J|ucx>m*;=r?} za))*ZA7FJAR1(>vR`NRPL1`pagX(txi^3hW7OFD;(;0qe1*l6V9zZ(2?3_aA2oqX?H;_SPl!5kCS?;DV^eQpJ$&r}zgn&7U$yBf%b*2cyr*-|*4jUiMm_C9WoF)%@Y` z+@^5LQ6uR-kajYGz56$aFCAdR*VJ5rvEPp(@p(?_wHU3!MEhHZP;m>b34Uo@q%E1& z5(xic$*e|lr{^zhwyy*9EwqzkZ*XMQ^`+tih3SvC-)F^^ewwD;GO)&&k&-Y0yvv)T z`@|Pw(f)Z=@w`UQmbY-I#sAQL$QF7n=v|P9U^%oGkJ;Efj4%`~F6At9{QW$}xV@NMN5U;K>&9bG>1?e5M9WoS>x|(pszXPjf zY2S1qk8>>sJ8bmBF+oj`)&&V0HBsH}!H9A6EV5A@dF79;+}!rQm@w57{(4mimuN$7 z(EF^od_R%tIJQF4BjQuUO5(~iruf6=^fvgfx~mI6jKrJ66$Vj$3`H*k3;ylJQ_>8l zLB+qmQ!h|lKx<3;W-JsQefy$Mr+hfkA`st>HUrW`82+XhNsEBN5a=3%=ov*1hPq4Gah+WUP-k-|ukK*#KizirFuvz#zb zIy?x4Yol%>VH>kDu8I8{_JtF^&EVy$8MU+x?FJ_;jAQl0^21=@JYRE?11f9N%$#OZRm)p9XpEjgx{#NB56n*zIHA+ z_NgsNH;9sHiNvFvEo|~t;RLVzYK;1xJHfJLHKB6#$lEYMd{GtC!l1J__|PJxv4PTw z@0zt%V_|8f-c>wGa{vO)WUG?Fmb$yaA*sdDjsywe_!(x24|^hl9$MQw`sN zso@_%^F+7V$vBUE%FJIT@HCU&HT|;& z@aPU!+j^S~FWrSrGETF1%~HhWb)n+^sV7K$0`zzCyF(NCd&g!NT1yWvo_s0@uO#IP zh(4e2$mPy*_(^)q(%+u%7@G~P4)AU&w}*}>v2~Mpv-%SfCWFG+&|Z5L zPEn0d6Wt;e|DR?`$929;@9JKQ;iR3gU-%6mEdXEcHN-jn>HCkkDI)Zdz8GZFK-S10 zk4JMZaEpi7n2kW18rG(|vNdxKYKZ5NVuaVy{b7Pkg>D^IwhsvZ^2t{)Kz<5$KNXe+GRkPi1f9iVpi z*O21>6BSRXu@a}kZ_6~a>?!aOG#5c!$_GH8;wU`W`V@2>F$deccSGWIShjR#mA1_|{0#F? z#ba6{WQ0+|xluzG(iEak@LQF`;P`5iX5RKL__Eboq4@IFhZlls1x(Bnk+dnCds`^@ zd|j#X^g_i+aG89Pkq_m>FI8S)!jH}TXj(3khp6J%*d2}Jq3Kzua9?TqQ_Gu(j(U%k ze<59^FNtrF=75xMn7PyxNXsgoMCF&3(|7w_wrR0lcogh@QjZ^tc7hGNJ9AI&tGqJo zymy$i?h;Viinxcb>N$lI{)u5u0-T+r&+id*Eg+s}#!KL`qk$)h8be_`u0QfU8}TGDf03@APS%IO9@+k6E4^R}Q@$}3QHOd6vq zW{Oj$omj|2pO?eb52wh_1qk9SB+iD_(}3UD^Ib$coDc(`CX(M{ykIx$x#K7jCo$3+ zjPx`tb8?i*w|yM+ANkQnv{u!WM}(j{!BE%j8jvR?J?N=4HIUB-(tO(7kox@Npc2xX z8Km{I2$P2(<+#wfRFC%1Wk?*YFpjkBF#NehV&sZ;N)Paz_Z;aur^V*LTAb#NEgXWG z=1DSp@7)trEyeXYiRt^zcvk0J&CE%*Q0L5jXjp8DPnLKA;e_(w7#M7hd9nSZ!mA(U zmmm-5d66b9m}N^Zp#BGqc^LHU+z83ji@pC%!n-*?HQ?Y5q$#yE+n$8GW`{NRdynG8 z=j8kTVD@d|YS#@oxvvr5bNzLW!W&b+z$z`P{Fz!qr59){gdq;3S&Fs93yk8!CgnH8 z827sHrntF^xynr}^Xkk`kFiIEqlAs@SLOpC4-0fJ!CZSNKj~7Ek8J6H13+_N)DsaD zBk0n`R;s%79^%5DnRg>!`9Y8dLDDf)cSDE^Z!^C?rCRC-QEV%LV#G*OYe*NsXR}wL zOVJAIcbE$05pB*@vhyiXhF z*PJIdY%di@`^NOdO*$Rn?A!^;v#34@Q~rp?PJBw|e3;D@<3=49vmQ4B)z2C7)}C*q zcXqAb9KqDc{^TP&lNMPG{9(+GiC-{*@WrzqS$F@oZBfcAw-5UQScojzHWV?Wz% zLE?6>*4pzHfpMbN%29B#peHB&iOLU=c3}yH+nLp%d~v&|CiL0r3%VtbAZgb@LHZXj zecPiY%$HhgCynB@4c(e@s$(S20ybNiT>URWjCt1y9s6w4JfCu_%DFGpj?gNvPUYRDl*b%>=4KP&9fP#;38t}}Z^5G*~C1wRhg z<>SJ(Nb;j-c&?iy-9@^5s8CvTPk}CZgG_whl%A0<35LVQ??qcW>y~4Q3M-C>_ZL`J@mh6=>`} z?Gw-LKhr4vLRv@M+a0TLfK?dBu+(c-eCey2kZjdRae#*SiV+8r7rX_vI_+UK7cYh} zLkA#fd_i7KUNs!fiMNH)qc>mw(~y3ziV^j#U~XG$ta&9CpU1T3qkJ>b(s8y>xI}tY z)W6zVk~RX$34Zy0%PReBlDQqMsn??V56Z8&cWog^!->uF(tv))>c2NYrNhd+GH~6Z zKw2+4`{7%ERJ}Cnw~~kWmy>L@QxFdEVTZ|gFU9$#qu}GAEf9Xl5;~R9-bm|%K;hf=X+PM1 z)$@V$KJ#7ojCxajadk6Y^155$dDB>|DC#Fjvr;T$fOrZ>56joX(vbXARlK*cOx1g1 z4R3p>J{%|?xuQWAE%|vdDJ~mH8-UX8#A_PrAt-HzUnd99p2}_MS@}&G!ZoKt@x4&tP{IPrysG!Aj}Ev0$1qy?aNkD1g1cnT}!0*cv1M&1Ml>&FV} zB?9?6rKgdwlwvg-$TQ%uk#>ywt*qPEZoK;^`d)KM8?EYR6K4tX9WDniCPT|r^*Q-- zyfL8%Q}s#S9En#H_pmWRd-2k^6X3eJDJcI&y#Z;|-%;_9>H&ct?iH5)X4pEpood zl9f-O{uvq%2`3-yM;KUN+uAuD*M_*mAv;U#$+~Ltiy6km&%q5Z_oG_}KmIJODc`;D zD^A~k4RZRAlXJZq%GhJQ@#^HJnx(!q`Q`m|w$PiCqVD@fP~qN^x2&8c(nj0y)Ab_I zHYWw2o;$?)Y^pBL%Jy=@KnJMt;DU($U?JDjzS9S8e!|14!>jH?KZCPNGPJLfv%xLO z1g93vmiuow@c2HXyk3X5mF2^NS*@qQFU=hy|2;bjr`OKJlP;ZQl5S_{H@Yi-Q8EOE_h}~2 zB_$wT6U(z6u<$pwnA__!*d4USF%C{yg9iDb*`j@-pJf`iX94_lY7m|ei^gzg8(B1P67Jg64bFeD;oEu-=MQV> z$Qs+PK$~1^X!x>&eERS&8oZ8!z2^tZ_$ODfczY{&vmp*vq-f-Q_k}#?$yT|3S|{mq z^#=>;x|%gJ9mC%TU0{uxzr%rljp#Gw%(P_yhU@;sUUrVy`|LZs?Vr!P{v9g!*avBv z<=2yw+b1B|zxKmEwKTH;RZ9#)!l^8wJ zMpk?Ijh>gRBk$2WgX_*|vFyzq4Xd=~)e7U_Sz4@%>Ag}>)7VJ%?|B%H3>d&ZHuxv5 zKsH;QNxb~fT^qh{Ic*dXpyDPAf9{aOCa;2ojeT=MdKkc}PRk)>k}*8+tk07JY-k)k z;qJt?^2V5K?Uld`93OK}un84FbB1Wc5AgRe$R_12*&v<&(3IW-q#TtF7l&d{lLP4Z zZWG%Xl?T6?`|_fJ4}jt=pQq1-!nEh&$H{WsMCbahe0o?sA2LnF2Wb5f+Jx~kAAg|u zLTSbw&DwkCaLieIT)kih*U4@KYu@YgUctY?w%KpgpV38*aEg-dVHIFxJDE?ReVFLm zd**3fPY%ob4IB5ohkHjqLtt4NZ0yp6KlM0@`M#%E8@mdTo3IZ18ERSN(t5IPU?@KI zXqyvOtvx-BqRS&!+!u#`_(1x31AO2a!H#b;lS5B+l&V%k%@|y*+ngQQbsj0Mn6=Cf zRxfx4lso)<$`a81Jc8%NTvoXZ{!@A*ce;nSVz%(gQ98n7#5U1(yg9%3Jptnq>p^YH zcKq?de3-LiKQ#5XhTltdq>&89Ujd`punC7Tuu~hU@;{=)0EOf7hUMoU7uTC***Ka;0*IHR@WhkxkGeS;Whz4u^h(+qg_$C6hU z(?ns~OsLk_8Y!1Vw^weWMbSy{8h;hOo3(~sE9~)LWo_9i-3D6 zP_Lm2PHq1ln#87H+cWbZ6Fjvpn2H_rx=N>Z-g3!c3*PRrFHoI>gINK3E~+i5rd*cS z&VwbdP348BQ{|M%FDhO@m<8AAJ!6%(n~b(%dG%0Qw;Awu(?WUN^*PS&Z_Cg88_t(% zZiB)+x))nNb0b`?(GL1ts^uU`CUWUf(LsW%kkcRobX4)#J|Kw%@?7?c0HNjEnndYuB)FV&b!89VgLG2 zFK;vdVBZt>cWuvamk)pkWix4yqjqqg&aq|_mN6sShgdU7NA{q7z&@IGVaHo<7r&Qo z#@B8gaCepi()!Tv`f#h+IfR|dq3rq=&?FwzK52DaY}g&IFrOc~`w}UK<)1SbsKz5S ziE{*A_4tfwOMbF3s}E}__u(-;a!c5SD!!EldSX#K^2)|Xu|GZQe_*iSONW@F%SM<9 z!Xwebu$ay-8(t>o>}bXbUtq+J*>EZIGOR9ogM@_`NX|1JHg4f-Q?kZUQ%%h z)t#1d0u6m*SnbS~3WM0;dUt>@09AYxf0T6z7IbZJ3|OOGGI%%I*RL*4Os^)_)$hj5 zdl~Vjw!8Sz7rn*($XKeKfpRwKvB}?mv$vbdp$5H=Kpcw|$G(Y_GvVr58igC}!j16V zhe9Ac(zFbntUdOL-f1Zfgu2jxCk;P_Q!h1^_n$^Ht$09s?Q= zqgJWCTMl}G%j@P1oN)n6Zej;V2h78qjUe-2)AcOl723KSggdivv+FK z8*CA6lOx&BJ&ok&1;If1A^)2{Oq%&sy7-mW#3v!yIhD=siW>=?!L#I%n0CSxQ&vxP z>2znG$mz44>aIQane8CWdj$Yt9qi6Z2F+ONff*T~U4<2&{3RVa+wY;eod7P_PDazS zarAk-R_MiYySEbg-CTv@+KdzD@j=0MY=6}aY_Sq@?;en6ejA{@hfx(r6X$WtE95i1 zVg49*IQFTDh#pC2EAAhrQP*7CDh~!83xT7ZPeD|&4`dAO!NZ*oVCco}yl%x4Hch8C zKjqLA2b|vpca~=Z<)5TA0KzKDi6^w?R)VlW$41W{fO>yH$i$KDyB#myc4vxLBjMvHe zh<0P?-DIzZs4#%W1fetO9E}$t=>Fxqw*RMgQoGTM?mr)i8~Edj9>T}`h?cY}ueO%^Wg4!LU(Dksn z%fdv7XciC*4E!sd$UErsr=}J+}KXUSSTD znorZJyjOWf>x`^&pTcF)%=8xVR2O#Pk(sMCh(53vd^O?X z$-NAyGh!d3wZKhFZzIhIhzGShJ1}^2xhc^2<*d9+rAdj4i)$*Cu!N$EmoS1c4#Y9-PyC|lQD8ZJkOyqz1?sA#!V0^Qo zIsPi~hkeoANV8v8_t(&TtFSroWH%t4$0!z}<-NL?r2m&K)r`de_ijkshI?9%5`@8^ z^62xHCrH`{yITwcrEy2l9=`R?T@fjDwo|_O!KU029C0NCD0e|=35u~KPQm7zbFldH zF-G}ZrHQY4tl&OZ*J8&Ht*J(Bmirq3BU2gGpG%w8)Ao}Av0(+hWdpyqlMxPrT=38<_e`lpZ$p@-FTRly%~qrwL@T5U3z}f za&J`*bUFVJc3B;Rn;Tw%`~4PZbF>H_&h>|X12*x8y) z_q-$6cKj~*Q|BzZ=s8a6tvyW`W}_`m_C&&WL0S*bjcdhMXEu~KFIe(RmLrg~3sRnd zSF$@an`opZEhIYy9fT6MZ-O{U;UEuQJO@bYbHaxzZSA>e8%z#!#CC2W;>vp_RedY` zRodEWwhIp4JBZhw+gdjL@R8!|!Is88BK`=%0d40){rVSyYEaEfl3ozRJ#1Tuk0hN- zxU?0C>qXYe52)s%_?&nc*8g#o<*|nR!umSW-XlPqtL=e==cHr%G8H$7m<~oZkyKx2 z!Mj}+jGOTW%&sTmfCdeD-==+F`rGPU>4B%~zTv}6I)l820k7t>RCfKPhh3(asBt0b zMOafY4AdNHyj6H(Ymmga(*zuHlcVnE8)`hvu(%Odxg@+UYbow3cHE;zI1mo8Q6Hu9 zSBg978dKm_Y$2<&KL$E)TLeQ|XcZ69XX}yI+l>{!S}A_OIyDZeSOW1nTT^k3_CxL< z+Woi-7urT>-NTj%$_W)OB;SK|Mp(n(xBVq)2IY+zX-V7?Ye9axKGJ-UG!yE~w~~)5 zPUU>sOwU@R?m^FzDcU*D8%v5640bsO-yUWQGryG(u(K0TUXVW9$)=qM!|xl{o2N(c;UiVB__MDvzr4lgcI1O1Sy41{WlpgPKX3 z)p}#;V1HKB(GZ-sNL)N49+Ymp6+1-e%#T6xMO8Jg8@yTZrAzRraMs9Z7g8Sq_q(j4 zIX|NQlr7~#HzcnnXbo`3k4rdF_JX!&UW4DA675^7GE_Q{YD=g*Qt=%*$B(f12@ISb zCOU6>2W@IK=ES9ZW0$=^8k=h6gO)U(ywG>9s696YRu$}r2bRI$-g-YI>1L2d&xg^$ znaUF&VI7!wc9FzIVzXNwBkYzlZ=b;edLEE;J&T=wUh`n{1LFOz&?vV|q(7SIOdO?t zF0?ytLfhm9iW~TKaY~rpu7%xC_Gk8OZiy*vZQ;~eGiDkx9w?VI*2^1km9M1Hm5MqDuw{|5YXDg0@sp}0U%M}P94enRDl;(F4iyut31OnFJXfP${6 z@EXaZ;kaF4+MmN`;cCk!j4%?+6Y6rx3r1^Sx z6Hkd}W*^1z`{6L(=&B&UE9y^ktIC_m zm&sFYgC7=^>N>FSL05k2!CUf7k@I=_@T0LD1}5Y(q&#B(`Q zFYN*9IcQW5fba>dF82i0M^XB7%djS-|4s?wFgV`24)5*o1*tD3Rqug%We~KWiOcXl z``yacxQd1IU9*`nUKObsUF%JBYE<`{LruUx4~+N-MF@nOQ(R2_TOonrIFJ z`8J{SE%`u<;bx$;RY*lmd1`eu)6s9h+`p`mgaOz%>z^PDP_^&U@k5G+{1$ng5n9r3 zV0t?k>4;;t$KIHHqaG60%5QDDVD`6qoUlw1FK9>`C~ZrcGlu#e<=E-dN}>7pVt!KMb>}-yNQ5gSFmoR+*PjDIlZ_TanLfiT( zcabzSagGsm;>3&Go=*-tj3+J)$J$N4V=kR(UgTASPbe;cl{?qskgaE6pXIq@=*c5w9q(flji4}`daGAXg)UBgyj8lNPAK5 z^^&0cVJ(_;M8ZPV*CAh1O_EnarH9NGS*t!0kOv^W?uXwm9@SDWM02%%IeDCwq}Muf z@>;|{@9@a)!K$~_?Lk2uY+7o?<0Gy!s%upI{qDdB@%W%C)o?pzkg=XLmI*4|sQbt*k;eCC2i)u`(4s2cjd@t5i^ z3&I5?U(TlQ)u;YX65E=838+tu_UC+odYM)FgnT)WMu3{Ni&>UQEOw=H+tLhMaq34C z#x#PP7ouqsa}(rdwD0P@pQ@Ke8d+$_vuHXVNL4z$s_&-T-51WMd;s$MtmZgh)o)Th zgfFx45o!z>C;CEm-B(J#v;Xv|U!eO_={XnWrPUmjHY#%-jY{YH@%yST%m@P@uBIjV z>%)pS&@|*O82QhFxMgpZSA;#SZ(<`Ob58vUHBU9KoQURTs%)IfU=H?^P$fmvTsXVZzCVe0jr?fJ^SCzO9tYc6U%1x=6Py&!vnCCqnQqWSDu0FJ$F__7m;=(OQD=$&~C zFAN>Aa=te+Ic|=zW_GZky&XT8;*VqMb9v=uPxjgN2P<#p!8_Sb(OjC{6KbtBl7rUY zg2k^hFtTnO8`rHmx4X6(R?u;V@Ydq9u&Q4m z1X$_dBo|${=+PW{=w;%Yk$X{d#ZDS|_{%&rMTe&MRV<~Yy%v2}i)@PzDJUIMab)wR zm_6{0=0cD9^4+MWB5-6V-yX7Gqw&tr)H?SRHw+AB7s`6@gFT9wbP1LH*OuatZ5muZ zq>{$8Squzz;djlJYd*c+3`_rQVqSwoNt}~|F zg;#rW5Y4NZ$b%_{fFGO6jI1wsC!F2`Pag#Cvug42Et|=U`RlRfcSBi&zGGhc?;lG{ zTZUpuEKBbcf&*T!WPbm7fXXWy%keT~lqs#nTI_Am4Z2@VgeHEgdEB&}Sh}gDTu@4H zY?`;^Nij*dp?ZvX+Nc(&n9}^PCp~{dpO>3gYM^L+JgcnhBrnxk1m7ndfZpa7ysYsm zusdbVUwrE?pUx=5cE8%=k9WG#KcFcun0toKO1X)&j?!RH2DS_Rg~#Su@#68s*OpVU zxZYC8sP+;v7VH6sHEF6&c+8>qY);T4h|%rM1C9;kg{}>;`YC#M`^9dU$@OF}zf6s# z*=-!BnU7RIFsi(c?ANOw-;mr-y7_m5l9L&DIkpHbufE8p{aUeMyfeS!z6`#$XIiJh@$_2U=K5UN;nkD(@fd||Yti1$ecQ`37K5ctp%>T`R%3Rv3XtZ+ zn>==xetTbG4SMdX;?ET}xzQ=`$a3JfCQXO6AF_~YTY4Wrq3$4yB2FH7{fp6t@G%NfD6u3&j}b``!{PCE|e^X9VET^+#ULu*L+ z`HYS8nV^|dmX2QF2!HJ(SyVTe(!{7*M>mk z#9z27z7H>%Fs*9di~BAWb!@U=x63u*y4ylF8(dAE&#jj^b=N8EoqJ9MSZ$VvjQjHA zf6H)+R+ke7py9fukd?CZGcauGf#0H z1=ZUA(VjP`A+Me2gh3q!vD&ZgwN&d!p4sfIH}%b3;=$w`3SZc=o{d3x*=qHkPlu%qS0} zQ%D4Ky!TGKZ;BaA-TFX$ZTVCE7phOU1%<{k#w$Bqgs;2j!t*Vf$EV@{S*#A?IPfJVg&!VU>?o;7I-}!Bj+~hjsNt9(LL+S z(AqZW60rja8~9w0D6DSLNql+h&5G-N#DcsAEWt5ACKq=VMeA)bB{dL#M;(Vlv#6d= z8ehU=7vVITVhg-y`*so{i9~w{#!d#IULi#R(=U{lBxSz;m30bm6`X1Q)vx0 zWoRLea-IitADD8;hVL8FS>>zd+PKH6hMgxS?H7a#f^tXpx$Ynr&h0D{jrzc~x*zb{ zk_D`{K^AnfeS~8troxD>bEU$YkF`^IN&E=uJ}VfEYK?^LrC}PgE3;^A`pSt(&3JZZ zZT7_X5O%uILVD}(WyCKqDjcM3Tgeh&9PsIp(Q=gYYYa&*K$^QCe#0fVM`%y!zp47njGl`a<+$+s?;9REz8p<| z2Lj<4Ol38A_Z7u#PWV=pf6f!OVph&q_HnL`q+DjCCA745oN`|fcSx20ZPpzV z#xD~UXW@dECqU)#t*n#GuJHtz@$Hv*lyXxLwhH1DR)3?hyf@_;s5%I>_Q4@0w|2>MSJZkvv5F;MJm(eCN>`yLCa`C}7%_gDZ2&xSj-DNXz z|2XF-h0*wM+DxjS`nzo1CGKBE=XzTt=FYmoQ?zsJ+Q2ir8fak7T^%m`JU{k(FD_VLl+sjRZrdVEG~AXGcTI8=1*tLLc(PDP}-F1R{9~~ zH5M;)1(jEn3#?*)U0InqL2lHv=7#G&YX2wcy5nm8-gqfV11%a7g^bdW=$_}0os~V4 zk?l)n_NtUpT8c7KAtAEWJxG- zq?)JwpQX|>K>dkJHM_7@Wi)7J4Ui6Ev)J2p2;W)1nN(}#)HhU~?ywg3zSscS<5Kac z#W_^3OB`y>Z_g}&(3U+I%?oL_XhFXR_rh~n%eX~a(gje^wl>eQ&StdUGPS2CTrsVI zT^<|Jx_(uf3~4=O|F_>TDmq=J?sy9cC2lbL^c9|o>C9u(yO&w?^GQ|t~&;Y=!RE?slBi7Lc(UkQctC+SwV77KL3mn)Lm{u+Tpb3 zzcpig8n>ss2HLS-miw`${b6`}U^qPY)w1u?!m-t0dbYK56tlU11{FVyXqthZ-JGxG|9K>9n<#=Q~;+o%(Aj5?6o>3&p}Y4Uu{dkI4n(2ZrL-Z;ya98|jU; zP`2%ALHNc=V6C%8=6RU#8I4Y|4Pk|-G}zLIHw0-bT(W)wlBNaXO?mjuCRq7&7>?}I z2T7Mo;#v_gk74a)vq?ub6e3aT0Q$NcRs@Sz0A-{|sj8+>80Kb=Q??WG9s8;7+UwZ#UrUBq8!OPK!W zFVI{np9H-dY*N~n=3^{U9k77>FqZ3PEad})DJLSabm(aG7#e{eCWmsGA5>b1Je4pT z-VNK%=nty?NbiZnHzVbZgecD0$tuiW_3Rd`yKN!A{JWr8LeJv1>$x4ryKIGDdY4F> zRxppGC8*Y3|H-zhHLf||kW&qc+o8@}&25R%G^GWAw5Vo5?>VS2GR(jppY`7aK|vmP zc$|eR<`qH^ghEut3MQiZAFofUrk0%X<7=o(D#` z^^(*JKq8L5&v6y0!g9C|^uH2#sAi)bLnNn&Pkc+$a%EN`lxSv$INvD}ALQeq}$K{bJ1v8{#_0s7-s&}volTS+syjUt^n6Ba!xEi+#zMpLmeaq=P8tYqdG<(S73vjK5*#Qy$z!s2&a&l_y#Z3@5HHNAiL~C-s0PyJiVo zs1*$NG)Iwim#DB=qk10S4o+vyCzXQoYNY+d?AzB^@$NjfCi4~Q?Y823CfQ2G$!ea@ zyL<9zBL_?#xtKhm4a`n_jN~hTGz-3eu7fuc3>EIPJ2_=Y844r4Q9$|x6@N9Fy9fJB zGUZD5(fbv&opD8cQO{7GjDq(|da2iQ(!5Ch1-Wi9gnyPmS&Hf{JkmI~D(@gX*M4_0 zXM}6SJ8w{b!Fw7*HBcCy9<&VCC1xp}RNetz?iBol(>^}Mq{ByJk>P(pIg$4CCV#@U zCggX$)cQ%%dZ_yD!RKyDJEG1bCr0B2=~IWRwN!b7bN_1aF(U!JjubLqULy8x*$KqC zO3!23S|^y- za>5wA+bazgg>F#&O`gq&r}%jQ{Y-i!{btIO(;OEZ*Vf@3Lk1CkISR^JfUr`MXJ*9f zp!~v?FJqu(j5+=Nkk%{@(n}9A%1o-XeV-w_kuU^QA1~M6q@lbG)(l(72p5!|L#3hU zJz(Oo>VmYS^ln>(V_P_~ke;)V^m3KHQP~IOX3Bd&Z1G-nes+P;oYMXF$1&!`ZK1cJ zH4OXsgZ3e@z=Gs+^i0MvvHq-wydAoOQMSbQ#&wt2X9TC*MAK}68*F0z;dP7)`PU07 z2P0kM#Z|sl+<&B?{6Oh!t~|cV)kgOvk3~5HT{G#LOtHLQ2qXQdRck^xrfnIMgnQ`x zGScyEJUy#&Vd2&)A3|QY>RO4fsE1R?(_VsKfsVXx*cX)-q4iNaMtrFCdGH>#JzlT0 zxx!yoE5n;xPHPUbSt}&XjDxqPtF^+PYxf~#e_Ux2@_yc_0hYwsCsdvUAL1maFc zI!`FSMHvqe=gQs<>ch%an^EC*=d?6I{)5H63c!iu4}i08LrL`xw6@qR+Kioy)Zs zVat86>mtI`;xUsu6 zcpd3jmDf@Js`{A@P7JEbq$q<{W2Ic$gffOqLH>#pe`=MtBQF7~o(+d~x`$ZH^4CD` zqdXO!cR!$2x{qo_s=P$bR^vFj1_nbeFS~pDg?Bj#>+5P3vCw|bS;5z)heur6{ zGzRy}Cz)%XS1>q4hYuZWi|glfWRV*SSaq9Mc%w^wjuXzJyV(x-+BkwcPt$_i!=p&o z0ZGL=a(co#wy0@SuwL1L4_Gk>)%%r=ug34pYb1@{d|I6Ksb=f#z4H#ox{B_Oj879<0UEVx&1msw-{Qi$Pp_^_N)E-BC9GS3`EFnZmh^p`Gk&wNH(&OxxAysz4`Ae$ zg#PXC3L3k54IY~}08>uZm2do;^StBTWZdQnxLxB5?{b%kV%z4j^*{%{Z81G3`R+9w z+BXGa$C&W#XWl_&xAn5pDhDcl?Z7Gad76{APGRDMK=xu;H8@B2;GF$+Q=EEr36A|( zrya1O43@j$0cJhbm6I4sh|lmR8k>6z^6F-*Q#7Dn^&592mh!0z4;7}YDM z*VFeBo#5b{I($~Yp6JnO9Td%<%nfVRzsNJY)@LmTnexw{1u2G5&J(8__|d&+krbC0AW&CaIUW z2thdP$oS)yp-}yTJ@3qH<71MciEld={$7TIvD6=rA>8wL*F~3sddJoPtE{ci&|HV|JNCM)b9yqI(@aV zvKLT|X-7G?ytGo%d%MH z#SMk7x36e27nuOfDZYFAkVP%NhCdFT6(Ri};`2ZQp1mm@&jM5HiRF4)@#Iwl*{tO- zJaT0McK=lh8#+&his&2K(CcRTvxpxHq+g(m*seJN)Zfr>Y8)O2*Mrb@U8R?~ z)UMUJttr&RiwhrC!tmn9aCUDj+}V-9=o%%}0(_h);Lf~EZLWKm=yTbEcQR_pQyzXs zV+RLF4$6m)FIzF1Ps0BPBBahT)@>=h>D6DbBx)t@z2qaqhtqk5(=*?xb1rgtlu+~@|F{>vA;dykgz!x$ENr)dUV?Z6!({}Xqc zTtMnEIdcrhNxu)^)*Z8{4%f-Av-(Rb-HL*&sW!6qg8}S5?S+2+k|ADL)*1|OmvnIn zmapwK{N518YCpKjMosczNns7RUAt(}X0DxlVpoel)0V&iGfz43&Ji5GBLX}NTzsA{i4O>qo5=9;-Aa>#mBs>+MBL(oest zABLY#^I1VkmZ07T!%g(eSy*4HKg4fcb$MXUMo{=c?*j`5%mb=*BwW#si$1Sv5o;~| z!irxOz=Qcmfp{4`;)C#r`59ILM)KX;F+6O8lZJYQ&fn-Or}{UL*42JtzYGlqm(CEH z3GLxpwZ@Wa0lK{E0#imMO0O#;fN+exozX=0sDBRaYHT2GGlc*C`0<~!pR>#DC*c~0 z7GmjwL?Dcn&U-d#JEab1efRBwmU}(19yuHZ)PqQC3EihV@JsX@z~89a zd`kQZB;LSo#)U}NqM;g8oWc~IMEv-n(LKM4U2$9ugSsTDx|QM94{-kgOU(cDh%nn; z6MyWHSifx_49dKWf!^l|X6!d-3R^?;lLf6iOaJas#U;cY@NYqVPC81hI~JaiSg#}# zS~k?jtu~ZN4RME=S8Jh{Imf!!lkt5*rf}`3gJ;qz8R-%B=xmloz3&=_UC?*Nc}=?` zYmw@XP1zl&){%RDJOPU)8KS~C;#b&)D^)FtFQ2VAVY}!zdKf%-X(+uv-N0^X?}2cW z=3xtrwWIyNrX_1kMlQgoX+21P(6hqS+tCdy!FW$8+wjc}oX(7s_bLCdrSl(5YUje| zqNXgX`B9*HluDo7jy(R%S)w7m6r*&024r`6Arz!mDBB$89@%@dqvrl8rpsJZs zlL~O9*di2meNH?|_^rk2e@uAb$scMwSo5!=Pqp%ojPOVf&3dLZq)@yy<8dgcetdJ` zDb{w zS3otz1`eVuzOXukE_9Z&>733Le)+I#^C8XK8QbyGRU4pwz#*Rn&|EUYX(TNS{S&%l zAKh>u+@ksMfXcPwfa+Qho^jH5tT-Ts^zcL+S^W-7Yt;@XaVt*OBX(bO!XXn(vB88H zT9LgQ)OCDx8;*$$eYM2Nj5M0MUJYq)&Dd8K@VNVZ*8JZpoSC_W`Y2OtI?G7bEN!hk z1ZLmUk^Axn;>zElLiM@oA<}55Jc9$BXuB@l3uYDB;eYL0%F?_nsP(>tDek8F0llfm z;c-h{N&UkWcVBTdlGM{kT8GmgBtL?*p5So051fz7hB_TR&@y}{#Lr5Bn7^)kbHqc; zc^3`DSM0`^g^cew0wY&BV56AT%=Fj|)Lp$D55~WR4rdOqHgh*({`l5(Z<&GEI-+R- z`2_aDWItHVIpS0HX&-U!6Z}wGshPx6v?qI|g8%aN_{?QJW;JgM(;Jk)-SwMsva2H| z)hR`#Q^)(ggtZ6PAk718v2LX;*V6#aFVlXfdm~P+!%vi7U)E&v#aG3oWHdfPZ;YcUhjT^^oMZ% zBrcj1rF5_IPC!4`C@rIJsw1lPC??D^h2_I`gR4#hHBSD%?{TDBhfU+3LRvCc^RDI$ zeCOXorS-mMolshUr%%z73fsu5pz(bKdQWIlS`C&Yzf*XsRr>asOFinp=j?Y=4z9m%pWt($S<7 zkZKCvEVssLi#)aT-on&(BN8`bmub;J>&KKnBkU5|E~Qmi^QXceRR7a!IO#IJqtsN@ zEvFj6joVKF^(Fa`+_2E{k#>99RlPaG0_pKbm&znoLs)`DE?6RN`8l}>vWaT`ja24!Rrm|#|h_nx86<)e}Fg}ZXXSY%CZth8lGFPJT6*0 z?n8QwmTHi`ukOVA=<3nF9X()WkBg)Wefg_Vn{dD<+OJ2(YX}dCgA;_Rv1y~7fwZdl zl(~!5Ggt|?o($yizS*Gk8R;vbYh6ZXU=J3pKD0pb`Mdb&c1z4k%oTC9k1C!~cq#&( z-X|YuBUcB!g}U3C$c5fENLVC(9P9yf-AMJo=6z|X{H2DtL)9J-221K)M!b|CwqYx) z-M=y4o|`C=I-=?oJYWC1wpU&~;)s95c^!GBRWtHuHp&Z9{hnt^Prl?0Ir%502Lx$( zoY=>Z6DLE~i!wp=MjZTEV>~7l$S1+OR-N(Ho1UoZ?ZO9rs;gka%D=?rUzzXxKk2PSJSayvj{VF{s{GE<1!kPqRZtHpoCmMYF4~KuN(+=PCQr(VcQvGU3Y3nS|8x)g z*H#Pt8+;Q_oqve@?N+?mtFNqK^IUC@JCnqrarSa#el3_Y>l6f~wU?t(o5^lVlYlY? zAYO#Evv;HEu_0P>k7F$It{v6y5KtaIewG#S`3l%uXA^jIsw-SBwZadz0vYLM%7IL% zwvyF4kw!UyYQ6|FtNLKc^?sbZEmOYS`db1>pKxwcqS9)g3-QJu$6WsiK3T{~Jq3pp(s?VX`(J0S2Y4un}|7Ixzx?-adc6_j& zo6>Sf_@doEAi0VQ)NAdBS7!>#2q)PJZA~7q-dqgLnuwi@b_3~=f_4#&!RyLx^1Ty4 zc->}h!OyhH8z1_3USSxV?{gBOHM1aK>k_2#Gc32Q>U)*L(C^^3o(I)@Q!epWL)u51 zu=JnS-g+xenM&tH5Esik90Cb)CY4*1+} zFmL={W2MVAEsiyjqn)cq_U{$eQEq`Wc2{8@=K&K`HCCQNMA(2;GzpEqXO~{_-1*knjat>l?&Dm1Jbkw zNDk~!`KtAw+=;MD0^K_aE z`|K+~bIL;ckHPTO?%eG78&KI4)rGL9dx6Lg2*MjqT!RlQ{z2bInV-bft08H8r2(}~&;HajvAu*>uN=~;tBR$%DYsLC_RSc&*5E0GG%f)Qsq=_ zQ`U=mC5u1I+vQ2*h$XnJ<^?ggcL zLHS*(b1|U9C-LUOcu@WAy?wQY@Jsf3Xn?ApDaX`M4uSh07Euj0kSh0}jDwyph{VMe zojLhj=GJHf5SG$fdy)p+=yP@SIH5b-8}8HDmt!g;(RTcBL7EHBY3Hdd7lUhE1=l`P zA#BPaD1PaqxSJ8y6exX8nGx&Kaz~ZUCSHIs1|P`-Syqk5|3pJE(Ii`_c_4o&xMuEXw8yvHsi1s+_$A{~8#n4nM zQ;9%&2F~C4gp^;Q$>!TYc@Fh=JNeXk0oyX65>zI5+U~ekX%VM0Pk=He$bG+s#=M{{ z<-(kLlFe(Qr*ak_>TQwN(U_fDuncQAJgoeimiSAU&Z1}cJMKry!I8KEXx)J34e9ss zW@rd(Zf8mJ{aZ^Ji!A;94$1psi}%Tz64r!|?8c?n%36|YhEpad6^>oc44^#`Tse7Y z@)b#fG8aMFGp1}X#C?QA%dCz=LP2Nj6#NW+-Ma?z%RSV$9}dB5?!iTSe<)lV0lpnh z;iacTq>tNJ+2*?~U&re4yt9$=qI)m&7&ZqF$B%+lcBN3?NGn42b%6Je>igJ#N8yrh z1;@hPW9aeEELVR5PVm2p2h!Sb(>kwk48%gqdaJn0{dlk(ehNSS=>Ux{wvg$k5-{!R zMc~3%ZY>#&yNbGFKOG0&#@d}E;o>N-)Q*;8{2LH zPm|^JKD&fnd=sW!XV1gwOvFs|fu(czVc~RRIVL0=?2nabr0p;qzoEBuIiJL?++GfI z9b3V-kGgWzpe}rzXB&9CusTAG6&UMhz{O214%?rEWBS$vH|U4I9X{gNd$gyr$v@aJ zzJ@%te5f2}!YDUbD1(OO(R1u0Q73mJ-i?lhEo*Gxnfp#{L8ttt5?6(%MPy(Fee2;Q>*+X*VWJ?+Zu1hsd#dR?^}23;5?=1LlNof!F{& zaGGc#7kDjVuBPtjwZuU7jaaQM_!vC>PMfsl7M9KR&bM;bGu`sQUft>sdGZyq>*D*R}e1YVBJGj?kBcuADv!u_0`MeF{)betwsk>08k&d+07<1|m z=D%B?T}lnYkjQ|uZZdCC*U)-~q_K1=ZMl&3i0Nl&PDxDa;CU&$kSHNeFV>!1#u zak98;9bRxLnfTQ~QhjpOH-_i)s5#c zTT3&(a;25(8K5=79!ERD{7-k$u|%IA$UVRf1LFnNxfoUbH`{V>t#F-|36**mQJ8u1 zDYvU>*1WWo@FfiC=TPy^Sa$f+jO$G)h2=;3U|XGzGIc2Rc8oRqmT!dABeG7daB%++ zfaB+0g?_giOO5;l`@7i5N+^M0m$(s*DyJ_bw(TLV>%^CRECLfwr|MBmwKgj>)# z?Aw&|-~&5t;O6m=+J(yp!v%hCy#=aA%oMr$6~d}@Z&YI!W(O2Di#ykKSmLjTNcbVT zmDQzQx-Wu`W=K=-r(jkT2lQTG7#@x0{TlL=6V>?75Y+A}It~3h0r>A;S%tIILyYhe zeKd>zhZ{n46c)F2j9DG0_q9r zcwiMS-`Y?0E3Urz7r))t=R=)z`OrU+w2#)^g2L%B@XoiE+?2JH{8l_J-Oz%sYIP3x zO#6-RQ<_nY1h8vn37q&xES>9)FD1>fZ`Ws1UdsD1hbPVT>KTm5tfNCs-HS6$#(R)g#g$W8j`HYS)#fwy5D9jEMORhCi zJ;Fv7kHV=now$>0eSX8%TYhitAQ%6)T74Fbpfjuk=f7fm@@EPfzi>&~sGZn?0j6Jtm ztf-jIE)JoaRv%Rd>K#GY%@Vs0h6b~*K-b~bc+&NWBJxcxTr;b~|6Kbm`u1Lnw_@f9 z;&_M+){%Wj*Wv%{S5uEpgpm1uivOieA43d2ST2Z<#OuMcwH~y86#WhRw#FijNm4J1 znU@x|vwc@rCU(D$*M0XvtG&lyZfy@vYYb_*WmtbmtAcSR0y@0fja27Cuk$sa-XRR| z;)>sig9wk?z_TTrNgs5B^=EUGE~(NiTT32jSFSgQZVin|e|%;E!Cf%@jgh2z6#XY3 zfxb=C2#>ze9BzVp`kA7#`585C4F47mURgSP%OJ=6@)?)KjW0~A=G$zOB_}MxNAMi0 zy~`K#vnO+f1q!PP`+TV0IdLtoxI@oiWUPa!ootct4`1x&I6|9`{Z8+|8n{B-yPu8i zZl6*(!(;s-!LX-!6<1FpZC;vHAPTS^WOUpPP3!ovj8-|Kv2d02H!Gb|#9CDzp!+z> zP+iZr?WV#(XB<4WjD)1TX}p_hElGMr5)MjQYvK2yK&#uQ1rI$whuzx$7ORi`hRsWV zYQw+jp=(3BhwZ{+cCBX(PB?;21J^O?UATXKCrmkH2#W8h=Ok$@nujb1@axHHo&2gj z$u;19?h(F?t%ln0hCrAm3Dcpk&pP~Xy_c$S)}%)VcEy-)``{K-kgw4Fc zY&u7~cd+JXPzSL1*h?W7pJ&g-i1f%nW$M~ackA}1kX_lk-{K`?F&qEp) z-pz}||8{vmen&moed89Eb0M8k&4`Fz>1tiMt#ce{gC9cSOo@Imqj4dPiPpC_WKO?%QbJ;4` zo!1k;FSnq1*Fn;%K)jC2mkzlff#Z2J2f5fiqzQi9-wQsujKO)<}lXe>tT& zv#}DX?$GR4b!J9qq!8BNAoJHgiqo&?EJ9;nfyCdq{%kX)%e4ocyK4^}zKJuw&V`3j z0jP9@(xfY8EWFBmMw(`*922Djgtg+GZ=C9JB+buhjT8saxe?RlC_3{j!h8qbn(&h~ znc;(^^CY!@7TVnR=oGBkBY-dS2q*U>mE^``0$D|iD)w? z3>rV$p@}l=CwRlPp!HKV1^&lEg~Ha%oyf@3@Xrrl zt98T=Blp9s4q-rCsd$sEsCdh4buFdW5EHnye>(QuyX@zA7;th9lD^Qo>MwqjZN%JsIAtOKWN=iD(pJkc`Ezhr1fd@)Smxn=%_pg zm|W~lePGAU`Bv>yO9M&#!M)QOj%bv2sSnJV4jGWmi_aJcg; z;<*I)S-21VXzxM7SJa_BqT^OO$#Fj=oh@lD)axqs{Gr#i3PE#Dn6MV+>_`JuU!*04 z3+=CrTRL-9-=sO1;mlZY9p0D`50D4$!>j2Y5xSZ^IB?Vhrs`f{71ak4r_ec?LF~di zS57_wsi)AZb8YTkcZjB=!6l?#S6Z9AQ%CrC@(fd2K9?&Sp)p{ zozCa^sLO}DI9K`D@6+l++EstP&GkE@>jC?x#~En^ygtwsrn=cxUz65YQE0EfY3sSp84EI;UnbyNZ#?G|4rt&_y6M|@zzPR3KV5DBVRNjHfR4-Wuo zWd6G5Ur;*vv%xdgpy@r4eyR?qUgsZr975tBSnO#bpH0bNOINu{heN?gIK;?PD=#Yu zGcn`qKBW3Y(u^24zbVw~J42*ZoY#I@PzILWdT_dCp!`JK?Pn!P=OMiZe->{m0)kI6 z(#m-A&m$!Lh52^T5I24&vpej7x%Xc&s#)6CsteHP3GzNbH3f$b4`Y_wHBgQAe^m8D zb%{4>6tHJQw&Ht{rBTUYG`?eF?{P!J^(y+|k~YtBfP07=iK^@;m;t=c%=v(Q%44 z=fOsxzNqqGs}{9aS_#Qx3#FZ@7LYs+YG*n@(XUOgu383?CXj?@O!ayHOgbB)YXT~b ztMVkuW`yD`%I$zW161}cV#@nafAY3Xg~-Wn3=yO36t2pUeaE#*dyw}h{~70_*4ey8 zHti3VM;bJV`Y~HjAEFw=n%`zp`DfB5)Mw{~%kp%k2{>V@@)0a(*-lj6h2}}l(XPXb z?&lf#7buI}29G~{V#M3HC{dq0!uF~xGa#Tl>Eym>mS+Z~OOBIIkU-u+P(9Ju^)SDc z0V;g1TX+w6PA#nO?uFz#eduTp^o`Nt{q1>3^Qm+!`MhwT{899Kn~CIuLG|&?KVw1Z z!IoT?I~;l^luxH@57c!p-%6S?ydkQrsEp2O$-O^Ske-CMv0;Mv8I-nNIV)9|gx!RF zN1}1#nfc;lc~_*&LgCoa_B}Qeh7Tew76X)D$ai1T6z>Ch96`NEIxB_75~s2l(!32M zVJs@G&>+IUN*gaqTuGjLqA1SG0m6j>@%Io@UUOQLZmK7tjk`G~okG6d4&vIoA?3_f zHP)bMps;`1p(+z4z2igNDwN(P4bIm2CzD_8Pdu8;(wl^_SBs~SpYlQFk(B0D-t*M( zz$%QVYz{`&zE-eh5$!!leNJ4h^ceIVoj~=~5C&)b)@JqWMH!kupW0~y_Ut+cGQ*3M z9#ZSTDAy&udPi%xGhFo^k`An@OQS8hKzg!D=TPQ_q}jOgTi0D%OO-p2<{&KGU6r>e zJgjN95h70=0F|p#UVvF`{o%*+U5K%#P}j0Gs2p`yBOjnTrMzP}iq}WcRuhB78;ray zWi}b&Vc8|>!6?#L*O4+VtQ+zYh_|r!*zZ7Dr)(_tpwc+SMNNdK=TF!+@FNmNqsm&# zURxn)6z;UJ4yQhcR`!U_*#JUM=UfNf%eA!s;+-&TC@yyxFQ#PAsCw@c zb+3X!!w!HuHMqwpLJO!^8OoVKMrr6tE zUz)U>$RFx#N58`$SAFu7o~;tV^3E3weI6qNv>m~~c|AsNzlx6AMqt7k59#3k8hmQk zqj@bvr*)2!=8UU(b)TDx`vW$?#JY9hvqcc}$jIeAGF)T_?`+wB!g)M4&Qn;HS#sS= z@v!7TZ~pxKIE=L^0^7H91bv2dPgsF>H@wB-L>p<-q>Icp$cCQtVxeB3rJNOb119(k zlYe5Y<&Ga;wO?KqipW}a{6T$F*6Penv@n^@GQKp$h;~fcxYUTX~2%zbbHh+XgJc;|WF;YT%%bzFa*q3$9r%hoEDIyk1~3^lt7VU;Dig*WADfJ~vs7+ukBCa~nkGw%~6+ zw~#aAE{eF4P%$&kAL4GC@fz<>;mSc3`}@%TYk%6wn?vclnm*C|S;M;`>|`$y z(?1DbEf*s6-D|bJns;YcL$=)!Nbh@Gd+5_OSUJ-e>Hl#2tY%zO@fV*g{sO-+LDRn1 z5!S)&qNsMzS*XvU8san;xZE`v43B=J{?lc>E$gHA&e0mLM{ZpAUUx<{2~C)hw6NXG zT%0}%3yTt#X3>y4zuSom+TVlsdmPdERvI64H3(PzG~(~di(uZ4?O;D+Esk5003UmK zwkzqC2sfgFIbjdh&oGuV=zjc#rWHPYejH>MZ;qCW461XgKRngwC%Px-NIBJrXVJ5e zh2@!5bD4GculUsEHl9tHtO+^yR*dTZ6PNY7i$j-Xsd>PrbpKx0)>iW3%?F_Rq-)1M zpyuRv(hnTqx)>H2M2G_|wlRfC2fD?IulZ+@dQeVX(pTO&U%}+8so>c;gJsl-*J>)3 z;dGN7*wwEIr#~=dOKmx8$TChf$y-D`!l07L_@&GPoAl^}>b19;E(5|Asrrh>%v8_3 zDCsO7=e|K}KYd(bmJ2VF&!bxN$(L)$?#~Ws)qE=q%G#SQ2kPAs`<9J|9($jP6SXT* z`)3jCi$J*LIt0Eiv=QIOM??L?cJlVS?S$>4wKvwKW0(Kl;{7RKp<@|+fAg{s4a{rE zvnem|egN0}@N2}WSJC<5LOAcI!)czi;jzEq&pg^sVgD>3JjRV>^QD7hYoxL9kq4*n zyBnq>)h^Zl223ehK>G;ZhY_C=HSK(kii^)`BlQFJb{xdL9_48R!)EbG74;zJ^dG`j z6yrAf$+TdNK^T2AFQ6}DBE>^52N}ogo1{G zC9K)+V>X7)ps8I6{|&F9A&j6i)abb%{U4YoO(eY!{KvP*29pJ@_*#rl(ht%6(!Ol9 z*TSn=dR)yh%^~fJwMdq|8pTi9rZF|w^zVZ0?%km0(rK9drjERnQG>g$9|JFr#-O=( zgnTtC9E>9Wil~EOYL3Ly{$-eL=Us)7!FO88wo~0C;XJO@rM=h3o6`H*0AWi()Qz63 z%>@VfbUN^XW-*|bUXNG4HI%f5>{7-mQRBrsc+lZG4q597kl3DIaIwZ3kK*V%-4z&o zs)6hmWlWrW3W+hcGWEEReP4_dWYW!PXcgKKu>5Ey-y1wW32J69#F(f8!3U99=h>2{K^4o1B8fw#I( zU|(!9o89^d{upvg)ZQ|f=B_%6bZV(R8)qmhf1MD%-~WiV%a4m=-OsU<%Q;|R^Bk>? z>r28MmfbuUmxd0)x0Ti4$B^NW=&#FryRU?>BWq#LZxp}I`Qfk*m2m0sda-#`ZCT^? z5u|bQYo=SU?}>4o`il`i!|TCz{M*(78b?!3bt;m}SKyb&QxxZj{Q>o5Oy|c`D=w0> z13UMhAJ02-NF4LeQuTt42UgSAqgg6Hp`~?^Zt3~#Ze~}7ZE|DTcb0SAwMsi|sQy7+ zi&o)cZ_7iX&}(&YhI1JhP1xiy6<=ySE9>_YoXTJuNdJmjvZ*x zn1A_@hSoz6{hv5Nwqq@s{P}}mVw6<6g0!2Uer3TgpHaX4#56ZUN&UuhE)HgXR~mEs zl}>WWfeXxr_6;M9z@j2ENt%arO|H^8H0S!?ME#}T&D0Eyf-?7T+oSXB^B+i=t0kw zCz(TgJGE|T)8LJ$yX&{u`*0AXdcR=K&#!1nYpLFsZ#?h7psSBjTpJ+=JxmhndM4fP zK>bU52Nr7h)R6g_h!%Qs^TG#O(tTKSqXGV%JPe0h-Nr9>qLChYrj6#pLoyFsz*|CmiONmv+Y575y1i->kYd_A7Yt~Reh+`Jk|BUaUPt4sTETBD=*KKmTfHQ<&8 zr zrye3q;ww$YKBc9nx(SRQ(UAKr3EC%wNDMz@Jp z^C-@4I-z)&1>6f&dPn&F9V`ZT4OAWl3M+2mcK1^d)8CwM=(8AnvIQR+q(S0Sq&30J zw->SVrV0o?m99ze+kvp*9lYN%Tk|rx7ZQFjQ^Rz?tjEGCpPqAguY>DD>Iic;SMo%Q zIQbb|ZmkbDLTf>Cxeo8(=Fh_9oGQE>KlL&auCfs|vhb&i50X|Ew63f=osd3seor7R zj--QN(*5IA96-E7*j51B#Zs)SSEiybPZj znS;l4U!}ca=*&+V(rc1*Gu#hKQR^*IL@Ml>5Fu49#G1cC<-fu=(4KDjtAO+g?(WnT z2nVRAEG20hxV+K<^V^(gukV+@b5<6Si&*u=|Iva@v0V5Kwa zEgmXJ%j2xTA*{}lH8^s+hq@j{euT7^qr97A5AM;;M90#5*sa=T@`M=}baWrx|6q;e zHwAG4RPP?>CzcpCVJv=j4&7(S>0IL@tAT3`ERO4t-eT_CR?=IYA25Rg` zW5LY}M@!NvRj(a2F%&$V^C5jmJ>oH2P#C{7v!`rOJts{>L{-Una{o2 zX>Rh2f_R1c?<(o5cr5US^>X8GAozKY|J#=!`?t9X`l4W1k@ zi=F=K4l9H1RACGGL(#g1A@*i5tXKM>im^3%`m#Fr($^+C@*gK#8}m zJd*GP=YFX!E-yU|^cwM{jJ|tyaO9+MxsFw;mNWqACmXOVOU8wEDgVbI^JX|RB^yYG zF^A?cK2(2{H6#ntN=*6rz6IfOmCoxb-*?liRTV#ueUYQQFI2QLfe(oq!ig2^^1S!z z+F;v=i6SgBob+`YPM(x09sT^95VgviROutaIdP$$q0-H;X}=q{eft>`{XPs$Rv7sCVQqo#=A{G!Hrq)-=y)1f{>W zV0Ccjzjv%)q-D6m1?7c_C(zd<5<4`iD@enUKPeUi&5Xs<>4=n*h$CUUnNBTVs@=}e ztOP~- z5naZw5#)DK6z`kV3&Uir%xRBe$DmnB^a@abGj>_2TQ z?7g)F$jec$4V5KsJAwSWxU}~;{-vzjX^~b?eS`91w9fQd{-Aktqbeu*y30(G{-9jz zB9P94TSm1cMuLT zxgxzMr|TdMX(Gqy)82xs={uq=@shkfkiS&(2reG<%xOTOhPYm7b>$Zd)OaXs0LtBv zyfjeu#!3Hj8k>A~`IY$B_8}@CPkBW_R~L?$yi1UNki;Rl_jxY)kGFy{BJr<%J1yy0 zPIJd9tJxssDe}-$J0LAXTt@ePlU{Dpqli563r1LuNuD-*@*xK{M&Asn=D_)DsD?5! zpn6u=bYzxulxC&!^kHH;#)#r?+=25b_aYXw#LbE-L(yv2^@ik|gZ zGK7(SMdckya{%$9(%;zo$rwR=hrK#(N6J?;t5yVbnU~TT+E_&~h{{3Ag9t~Oi-ISo zmHxnE_baIf=-#M1b=CXeMLi>}(rA~nlE5$MBa7T-Br}5?<-V(-csX}1kX~nmA&lyP zbY~wynuw`Bu^r!{sn1R@HG0v=o+)L&8zd}y*dfykthSxsMjlvLG?-)_f|Np4#X3- zbUsGf7FU?m_KJV9;3g$Z?1zOAw!V5j!Fv7T|SXq#vHQLsW$Nt@-^cN_7o}5~nQ-;G| zrguc8<0w;w#QyEDj*voO`l-t&RUS${d0~~F?7aIhaoq5dGwvuo0uQQZ0%;MIZ7||OLD?u+zxjwahYzNEk?+9!1HPE< zYRF?Nqv2}3dEgQ01>ctZfRV;0!Pm=`r+9};y>E_ka>gEP-)0{(SsI8=Z>;2S+h)h>d$ z;W0e*)IzRpP*-LqY=MODw9l1sM}G9vO7IwT34hoggZ#IBvDcu^;=-p8w?mX~_Rn94 zqZfPO7Ru&ar`>0rEp2%Er&qZC_8r_3-A?-a+>F!Xj^O^y?PY(rQ@mXJ3V!?wWApq% z;o|5fcpGuB-6wv$!c0QT2S)t<(Ln8UvjXUM(~9RE{Q)c9o65xU zYItqOe>lZ+6yFkX11_G>!E5JRa5auyu4^!Pb8T5aY@7UOdlS`JT%A&iMwvUnZj?FC40qCU|37dg#996s`<<;w`OMzgPJk7~ z#}OB%f$lfN%R83wFSgy}-${|UZYI!kO(r-=e;S%K`htG>O?X~o0q=eZ;pOip8^vC! zTED=&bTQrY3V!{y3lePpGQG=lF!g&PG;b6S7el&N%`06W{P;Q#%KxnwF_-`qt*rP# zyHII!T;HuYw>NKlq%|5^yUSP8>qx3KAitIYzo-ks_8gO*rXA!ke|>P> zULSLe^m*L{p0cmK4mfuGAq*md(A+u#-~Z|rP?e-6}@&s(+Sw`#1%^uFHm)2|fvXUTA$IeMX_@oAp!Sc_E0B6s&W z_OM>8Jovr0W@mXXOfAqs---3OvEfaq>)f5y=<|)y9Enz=0;#q?X^V>vF`8$YcXTUqL zM4q-W;KtTz+EJwuuz%T2NGgb`TFb!bPk6594Y+aC1z${0fKJ0&$kj1>z~gWV4)))P zHnE3=&cR@~eEt;MpST%fD=kE@M-on4oJ?mcH0DpIpA{8m9M<`ipwrS#}!xE)4^rgG>5#=@g($hDP|r1-oj;hlX!r`EcjQpUezItbLq_6+$~^qz4Z4? zaWkY2jLL7v*I4|8@e0t+EsKu%Rr!)fW$%9G0O$UDX z=?HD(>ZPRVitdB2i!W~xd>Vs& z*Aw>Kfa^W2c$3+`L|DK_Og-HUd@??2|IuDe)vwn_RrmdF?qTUhS4Csz1ggP+ym?3M&w=6YH=Nj^|!G+kqhYn%!N*PPvsr7F@oDcKr zBM;cjkuIzIqRaMH)Ccu>x31<@HF#oSDKrXxCBpJdREc9b0)xm}~pI4b`bmaOT30luv%r|!gGXeQEoZzT*|js_9B z@?P7TkaK=I()vmJ*}btbJr6TvVwO1BVJzJ0xlt09iSWRh{4ed*N&P3oXQtw)jDfJI zy(=ev66xdH@+H1rKs|(iAJ3Hv^8y@RiPur@I|wi@ZNPO(%WZM5+fA6!qy;Q$P^_I);vp}dSc?nNa)EF}Gvoaf_;)Rd zp4)t^))uFI*@F2sk~Muzn<3RIv^?TPV|a(egILq~t487Z0dG26blxs)9m8(;z5i`g z*g<@W+ou$O>4JMecrOV{=xoMU+FSqT0M#?px}1-9>d`F<-t(~M!~liwZ1KeiuxvC5 z2+MJKOb!rEf~ptaCjRWxF9*IdZ4+^N7##3+gx#A5vmq|ORIl^ZDHAnYrhEWL%iC<| zen(zWv6H^L9S(uTb$D<8ejv{KLut|}W)V9J7J95g;vJ|Lyb}AHn(`rDBT+xoUQ*q_ zA){5;xcWru%@<L44Ql96&VA%wfBgXfdz2< zdv!_gj~C8+qTAp0NVp8IG=u5freLbkG_ZXTD_%yqsP)B?;kEfNmzRP*lYalAmhc{^ zpP<L&+uzfGhChL0a34X+!RLH51U>!Z=NqUG5R-Yg-<~E0)+D#yHQ)QWbk4^_zk81 zV!&gyo}BvrTUKhqS=KLG;u;S(HojyCR*%20UFh*gP_3hh^K+qao3H~$HaF+Jrdi^z z6>H_-?A=J%q`k3bDe;mXf@wLeeOF$3>j8GN@2KjV(U>sxsV84l?1NvgjaGdIx9Kdg z+$+^oO(S8G@VrT9)KnV*ho9a9;zleyAH?SR(szxqSy*%2Nm1wcLk(Mfm|hcziOagF zwUQ?iY~Wql7}X2>V#rzOe77;--LjQ+Jp9g0 z)zd`3epoM-&b7fhIOwM}o^R{`Zz@9JM0lPyC}ty_2j`Bf136yo*_HPEh{cToPuaDj zy|DQ0It|s4;=C$Mj))Zia#yF|dHXzuOwnl|IK)Qg5=q{psvK!h#eNJ+52Z3TTbR z7VrYc9%dL=yam<;Hb&B2s>TTuJrw@1dvi>m4K>xJu25r{Q({8T zVjoo4iSvt}SFLqPtAXf#bD+j4=P~>_uuZf*k_?4MZKa3*Y~tL(Xhh%D7ES*OIiFfe zizkNC?@M>h#hy7BTKN-5qvC|nc`!Z07)W~xx;7+S(ey1XWexh6V!_)kShglo@w=R_ ze}H%@#H~U*Qd78Q3WmHouT|GZzpLgPUM5)pja{~>9>Fw`bCC2LBTdVcXV@4p9+gJC zR6~PtCVjQq4xltISQ<8wb>nNucEL4xu*EAN{m(1Q8pyqsy0YuE)7WCRFZ@|jgmnya zfjE@W=fbD5eCXrm3Hgn~)S5^&M|&%SfOHq>i!$->pBC;_yaCVBMPf7hU{C+iOs((h zUv#GQ>b{`(J2nkIM(l>I2+=_NKZ+->g}b{y`dgEvuiAMQTjl0 zzD9l(mZ{ID&l1ngyh+#nM!j#TqS1mDqLK6_UVB;;=U!6p3AcxzK{hN% z^Yz+8pqfOR;LDJ_tR0ZP(5U$!yyI_^g3#)mA5IUw4p(+Z;&$)%qTp>7+PxWwq)!-e zkT$aX1sL?1g7+$ByV132Nt1%+aDSn6vcj~q(~+wFV2JTBuE|NIta2+5uV9sSb>712 zhS@3aASW5G)e`=5@(fV%%a6Be(*u2iwiI!%Z_AaWxmJerWC~ z|8>%g)Bk}E=XN3KH!bxBG^qX=OaC>HPp5YW;v+Q&jI=D&IzC%emc@u`JEn5o%d1Iy z*s@P%HKpC%`I5Lx{e_jnHgYiGLW=dfU=VVOYxU&Td=r^)lc{3bzy4bw%-MrBy* zNecDjGq%TdF1#3=2dW2bX3xkb&Bn5x>ca5m2Oz+sA6z&(P0gRku!(`DqcXs_vk4}i z?txKwmc@aAAic(a+s3hzeYx`3O!>2hhMVMTvo@N-^STPJ#Pxb(ak^(;q;bnT6|3-_ z?K9Fq6)Y{mnJ-A|#_fkyK<2{13>UUnb0+d0W@^ay(({2YP-om!Ft%3}>)fmq9%o5uw>73jJLDb{^T8HhU zMcuGO+yTkL>8`|C^GXDHGk7)FiSI4w31@mf#m<%qgh^3&WLX?4+}c?%m31>6%5@@I zP+!gzs@{nQaZ5^V?)EngS_He|mYGJ}WN9@ydSePGOzjz|f%ZMtX78+QQKj{=t{gzp zso*%fv%+p9ZWKmz-xSpv5MH3FyKCnU0{I}q)o{AL+LAI0D2*z@TF2kG5s#t%b=ohI zdgkA$D~vR&nonV{^niAc>pMYu3!j;d6U6QOhiy9`9$_>F{zFq+qj;J$BYa;GrMv_7 znHIvpbP7<;fs50Q0dW=($1IfX0uOA0^&DT1`H_^;(eLgVRPyq}LX4${&!{0~WnMfrRmLMP6YQ{t}iE?py(- z>DzUE!6s)|asT$)VN_{tPWZ>qjPAyozZs7-W}S-2 zOw~ttWrd)c#*TN7FnX<2zVg4^U?5y(41j*NLQa znfuBQlkQ;d?jCGJelAm*mwHIVZ+4d#ckfgFk(n+SLHyfDiW4WvBayFlcu>XNgl8~0 z^s$CA0#H7@@U1=f{8`AzlS#rIEdOT!qxXj^PsKXtAkIx2#MF4L9-zv~M0R+*CbIG( zDjhT{ZnyUA?hazAeo<9cM%RvZZSIf{NEH{3zMyk)pWvOgyMa6r7OtW1$wN~aVZZ1- z@wKotj6?Fan!JadHTUW{BIycBUF6nUGk}TvFdu`4g4h z6x3*d3L6O5$dBA24M6wu>A!#xrF#Y82ip~3#a__aq*fnyGs1Kze0!Mk!b9-+mL7R7 z9Zt2vC^J(W#wZIQKhXeVp#gZ>N8u2+X#76ugqHj}ql^t-x7vuM71_A=bU7pbqztk? zWfb<@=G--iEzbbLMy+Gl8O)74Zx)2XS*wd}sK$a}mhb9pwZ??~T=6LRBgZt0kY0lS=XgWsibt0QMG~XUgLc?o?r| z(p~eL=tA9bYmxjO&m6yz_VZ0s`H)y$oS=1Goy91R1k%0CD9o3UPLvh5vQTkw|I)z_ zmbV$FUA!gOc72)P>gPr{gOs6ZJtFhPwu0{(S_4V^rSeG`YH=Uy#)lwfjH_ed#+G!tgQU!rGR>(_GWZKp_QIM3 z-v_EGY!@5?A0`czq%*pN1uTFr_QM!uLQ02`jvG$-S*95OQ@G)L3xzhj9 zb`QG{|4Nllk%kA-)>>63!`*b{`&;#pcsZN=iSpAzd6Mb#=EBC5rVtiAh@QV#4X;}n z%ZIlDsxlK*^W9B*AmyJ-+jln{8Q)duKSr9Lz6Y$Wp_+%wM~eW%H&ZU|J z+gtejMjLK^HWLnhZj0+<4nb_Iwcu4a9^EGtWZRwF2n{P*a(cEL+F$jOX{S%1C}@pN z&r+EytV0hwkO6m^$>J}U*yKYULA{TAk6zr=_BSlCKFTaa1U!G+NqWUC1OElf`MQS_ zgwDEwa>Vj65EbYr4kj>(mvq>+VL@0P9Shfc+d})1H*v%$A21&*pqBOqJ22o1G_iUK zGkW&n?^=&T_crBh-J#_eDt%>BOLrO6U^kq4o{z8ZS@2r#vxLsKesa@cx(BB0vSw-L z+VYh}OU;T-Q_;$ICDJYI+VA7^WpOZR`-2O)!P$5=YoVc>7nX;8K1O2nB4b&5yo*d+ z8P0W2ZhfGdplOU$S6?C4l9!8!|gks}Xoc;|3V}rq~$0JA? z=EPrp-GeUE$74p<7wFe^FgMfC7i@diGlQqC_@%{m+Muy?Tt17PPOx%1JHd z6N@_Hz436o-Z~b-#?-~KFOS8$*avKW{60yaE$H7c1iNymQ(ZVe(B<$ zAs1u@;``%TyjL;}l6w8bcT+yHG|$uUE~^Ybxb3J~OB%l{-S zdb=NB%Wn_dAoDkjrTN>;t7)^a;mGE4ZqxTTtE@UYd^jkIHhe|L9khq@^v0lf^CRZv zI%&P@SAz`QyV@Z?Zo)73BHUy8lj`myhS>z+TI=&PH&dj;1RHRovu~(|L9_WLz7H5I z)V!!V8QU^Rrd)KCrMXKmX+^1KRR3yR@7)Tn)_`hT9B-z_ulGK{Dm#Q=>-*U;_+0n_XBzgG zdMBpiuDcO@g!TYtYUh8KRvv#o>quj99%)!-TY>zS*VY;y@}?V)Fn_M}0pze{C$t7DqVxjUfY zW@Gl%`65&Ec*b=Y=r1i|OCww1l2_;8+HATu!!Y=8xsfDn5SP2q8E*QQSoZ~bGJZo4 zlwS_Upji=^>D?6d^+vN9xBKJB`Z1!@1Xt-a=y`~+17yJujTj^k_H{Flav>2%$A=EUVEhT^X)k11BOZJPrl59Do ztro61^^j4MhVlV$5`)6)QQn*>k{4xbQ=VsNhx!Lgs~2mqc6Uek_O2!nj^K!w_cY#z z*5TN-=g=$eJ5+OYK@DMXMEk8;M>`YwDx?`c`Kl+^h0TM5o(Y(1vqY||{{_$X-h?ei zy7ByrAE3Mj?VnK=ES!HB^L*FYeDTg(jQT(pHCzGuz602-NHfV?PNBhHXG!b9GR-%! zK0WCy-VtGRADw|5VfP9M!+1v2eXKn`1A2S4K~*z)w$E`AOb{JAeuq@=^{_S5Q_xuW z@$_(+pY#HJCLTZslVem%@7%uAd4)c=_oC`OS`Vq`F~^O@r)rF=Yo;|tbE_e;$=T~H zV1+(m{!&>|?kx#>F>iK`nhUOKr>u1!o@Dd_2;)WE>x&rgYAWw}Pm*Vbjm1f&y|DJl zmHH2H4j{nm_2&ratfDjUrY z0>VK~HKQdg;q-I0hO#VplDt^cM?MeGLb6S7=~=IbFpsz{71r84+eZDijj~!-@b0t$ zE?ue*ga=4Py8Q^OTT+~=Zv1j;p@Pl<8FA@`U$A@pc*Dj zexxN#6a$*3VDG(6;pN7w;*9HE$W8OdXkAlT>a@V^VmB0~wkuViAYm*k?yW^ydrU1c zCak_E^xpjfI+dO9sJir7djvanTqd06r)!4ySOogpo}lxOjr`ueVHj=G5DBMwn2j!U zFV*1zD?;(xD%z9&)Hz1ikD0;iHBScaqjj3bqIDy|>dH5WcoZ+zziz$VA}-x~whi|p}^b0CiVX`=WBGses(-s%VMrYrzf?+EjyXLnW4S=_HD3FA>`-3{&S zM$Pz4ZVfBKikaEcw=5%SBzEgJS5^+HT{T~EuRW!i?{hKkSanV~i?2fVL0Rux(0;QU z7RMf8#Q)G;rzR&}6e$gx0^uawGan2z=P(zRt1)KZzhNoWn$5|tgsm}>S@&2gHVr+` z;xBGvBOk7V%zdVk=AZS?7$FYKDnysim&_Ng(pk>0#IDKaGJR!&pc-P2AKUT3g_+R1 zP9qAAY-EdnO=Q^hIIm!z%O>)q8PX(`_N;6qv`I&a$Pfo8_-YYLP2l51K> zooyqvac2-hXOmUp&Ggr=ba=l#SkBOmI?03&0$IiWVD`|Z@s zZ+8fPM_(2V^qpi}uZdD!@9jAjcrW)c?D;egh(pnDZ+|gqiw_hwyUJ>BZzAizAE-18 z6a>^FtUX6-N9VI>2l75WgRAt@@vS=2F=MMV8EK4@w%pD(pnax0??|Xx_oz?4Y|?^$ zq?;~Nmi8FjO&%fX&ngTgO-6Ir6a9VWq2gS-#2~h_b0JvA?Iz86LECxxP7#wso}*;H zT7R5q@I%n`i~NRVEVX2#n3!KKz0YrE8EF$?5${vi#;6gN_T z;W%z63NN)&nh!|FapG@5SOZE&>>YDfgNx}o?oM4~UhrNP^@+aUTyPc5G!H@XlfmGF zNSc%Mtd5Krup0=Eu~Xa?m|Q;%e)j$+dLw=J-It5N$ZlA>MRTP3$MPjn*kHVuJo)+y zQ?*d=)<9gY)WOO;L#1i){@7R~4XUs~oNCxfzIV@MfToI@RUE^OE&5>IVNN5U*d+7k)uNZ)_b z&^Tc1PC5_w@^SdKJ4RMMp>rB`lxq#F7*J1Qe9}fF&j7@?@Ow!vjy*-^3=v`cEzo|yud7VI3s@L4Nh`7&Oa83%e6E%Rhy{qdmUpM z^aauixbERIpx(`med#T1@)OwubVM(Fus}^ z1M6SMh$uNvkbk2yGA^-kqs^#pQZ(;+S3sv8$Dr~_6eHcjO-61c4VeL*Gm^#p{_SP8 z+KynAI!8FIkCXwgtD)1m=G^I-Bd2lmy^HAihS&vMX>9NMf!w5`4sB?0MAZphe?#&F z8al6-bkp)NlImIGtCOYHUhDJmFr&{EGmkZrK?6dtsNpZBa9CgOA+mFBlDGv|9C{{b z-5B*UikhfAX%()=^`bq6x2{$EBm##X6~x78(6O=XwB`)+$Z&^J^XZ^)n)IA@reSAB z{LM*6Xnyuy1E+pF(%Q|C1L(X++*)7xQgD6{#K~hI=_I8m;Er=O;uIr6{Q|^|YE6;6 z4AnI4%kndk{NH0{cI63>-s3ACUd26uUw||oqj`qFYC=Qn1;o8>Cs+IgHSgAK65N_Z zJQp#)x(Vu07Y~9bDbd5Jqk4n`?X{S42pWEwYp7{Yx z`+Ob<8?|*!bU5)CG-M5VZr9bA+;tFad#fpf`HTKS<=lD_ z+HO)_0S5S3D4(k^Ln^E$?L+$5mj&F}2c+?!_OfkAx`NS~!u2Uf(002Q^|U?Gwc>O4 zR$}6g0DNMxg^|XC+(j>_W{a4~RejQ9 z{TZ}vag)W_--nYc{84}Da3H;+S!#`$-Lty* zGwYYCZ?~{9+mvU7C58i#uqNA~a5w4wM7r*IU=i9v`D+;WYCC!ubYx14k|sxoxDH6y z4CYUBHSPap2=ZvcWBD<~VMsVB2uJz#FX_-~-X}Kq=px?YNUBg8-7qu)T2yd_L8R}! zv|k?IuJWDmqdBMS0{r$~6DdjitFYkt+fy`0wh`TAfaJ%Qw#fP5o9p5wrE>JF8) zPaYzVJRA)z&9e;;?tphiv$@hqGg@}TC(aExaTKgQcA09`2Fd3Oz(^#1VWdTH276A5xgvgbq^hMO5DXL)6mp4Su4tDFGYHt#>0-DeOnkWXbkLP&D->B_CrBfpz;l@ z{L=_8`dv^t4{=Z!BP_=>GjH&@oR!XB%zLbolgUQd% zhu#m;N#h-cd$Zp#RUeeYXz6`q(61=rOOKOox{!OCbdKou{Ws~*I=t22ZyMz>qPsRn z;@xb*DDq4rfieQF;OWLZ@US&H^r#Ps zyUaPwFOWy=LOcZ%cT^(bAZf%1^lDRyDno#_wHRq3-tp{jAWTe`&cwM{?xTP>a78m89!MwK6<>KZ7*a8uH>m z8c?b^ApeVMUS|3`ldmw4C5vw}g=@rlfaT#p*vcpeV#L>+ay-)ihMd+wWpi+;x`U+b zt16!(ZUW_9=sj@W%r1OsStD5AD?{NAkX|P5Y{QhcP+EFa|3GYga5QuWE2Nx(r(8@W zonMX9T4BR~p(<+=ia$vwz`ft|8Tlmi^>IL@eMy%w%8>;9o>HK^Y1s zEd^$JpFn5YMM3$FhP1uPc(~6;d)f4UTVZZhNb~t0s{G>M-4>ks4>FxBx$?KlKW#dk zi5{cus{ASC73}54e!OhTDEOJRf}UsYfRxh_W{nW!PZ;?|Jk#)+%I_%CQGTf$Rb~=c zja99MdB+ZtG9`EtvVrcJ+9QZp1+AqB$c&+!^aks{yqB66LEc^%oV6CIxw?$B5PsZb z2wRQZ1kEq6++>Zv#y%0fXV6}HAzT^@rLRXFpQ9SiVEbED4jwF!#8p zB6Jc*rDustHSqql00UlCakNmB;P1%Z;wOsjwgV|q;eeXit{s3 zVKU`QTGyjGIJfi>5*JBb{U9VSO4(Z^SR6b`UU`HdO$0wYjAY z6{PV4;R(D8UQv}%!3!>b?bigJV9oOAkQb1M^j#@{ndQ9 z|8_4>Hi?eQ*CMR}%sX_IG}jwJxdcn@dXteJU_;u{9$>c`kng^WlsQ&q3_~7V(o$_8 zWe9-7_0Yfcvs=^YC)kiW&h$IOIr&(jJP6f-@^hfP@)T2jM!7ktZ1>%lucS51IQ0w? zuCWmrbAf7!GL*-x)x3qUH_8!5j{EZcbTeRZlAoOTJsuwF2?EwpGWc%7?>&N7seQvuMoh>ndO7Gv1tEd}GdI zSG47wR<___Lx$qfZ!a)MoMerg&xR#MyLbo==MBdUmg|3-h|XT?wQHUBi6D9=%-b&) z-S0Zev8PVqt5&D9?srpR{*~46`84N73)01r{rb>zR%=;!%$WO6>j-a0CyKfw|00iB z#4WqfS$rSHOMW;W={@nyyMa7AYdoGySt?RBmb`tVB&Zb+pjT7?WuwYaMA-ABTHzx9 zRWD&;FNEy88Ue$J(Hjg_9BGCK2sJW!($-(?rbm9F*Vo?b_Q z)(>f{GV0kFY?ZSGyr&t+fBu=Mwd>6FioAK#v&TjJaw~ozWFKA{Xd~|4JBiEoxM0A> z?_!u^6r0w!Iq3Ij!EQq#-MJl->H*FUcIF}ZbiYlzUfEIqouSWaiK~8G%W~8Grs_bh9z9C)kk^$Z zhS9js*mSMX>qgP0_k#AoHI%5dSS6A|LIR@F-DFCe<>+C+DZo z-p5{5WA9av$y%qZk#`GkV4uqF_-o}Q<~O1ikIC2y-J|}pam_c1=eO3vH%R3yTe78cU+O=DO``$~OXzF30hDrB6`*1R1bj>qP#1F>}@q-wTOVk_OR z^09O&?0%sO&JQ1JD*Q6h*uz3Do;;ozLwgx_G>(-`PK9DUchvYtLqP)*Ih*&z%O{Rv z>t;=0SN#N-Y*rwoPj}6r4xw0J;UJryb%vTZ?rV=tZzc^=Epc$@H2gjySm+)Y%U90~ zMXGhegJi;%jPgUz_rryieljLu&g ztxIG5CPJf{5C$0WIf)u<@cuWueo6;I*AL@U|Enu~$5sQnE}Ta9DyBMJcN^7W5BwK# zn-MO-kUt@4|MIfDvGg=lV{_!kkNbJ??`868kg2r1cw1wxlg)b0kHDT2CdeDJ=i%$n z$1pVe9^8FYA86j-&%yWD?~(`1Nz;X}5nb@klobje#qeldiC<#~cW57Wk4y-Ne2Akg z=gal4O7T(D5+wZL&vqSwSFJih{lE5f_OTz&FZu{K7Tjhbd0MPFZ8~O+I!({oZ!-K%J5veB;`u+D&C|f$9YlZw%sJtlLZT=rU|D zp_68A?KXVi1t;{LXp8NR4w1ein_$hy>ma4@02UtXA-`^_A-&pnTYQSvEkN2=80AkaNu=rQ)Oo zorh}m-NwM&>d&D^?X?=;fx|?-))TT7FGLJGtkq4T@1<6+#K6t{uydJZ_K+V5V0)p4 zRQ#AcuO9#B-&s=6V#h;%qI-e^);ee-iC5&2=ut>DPvcvNRM%ANJ8)Z5OWti^Q?7X6 zUd$`p9=Kl&YuOgk+W`pwA2KbR+wxIJJHdfnEI)*sN;7L(Hb1H|PqrE&kX%t^z z^zmGXUzAlb5%hoz_!dO#O z9Bj078~#YYj8p&HM%R54<`+9~;$ON?=Lak|+91QV7t~x}zgtF}aEOm7FJu9+2S96= zNj2PB9*#e$`cF`u%f{Oh@pWWNzHDENpm8$7RTyR6MR7J|gq>)fb@-NgW8wZnOKkRa zAN*{fg&$f|pgyS54T*O$(D~_E7Cf>qFB>&UP#vYF`$!}VVGld@r?cy>;Jm3HVGo3W%}nTH*(;VsWt zng4K@X5*Gxazpde5PiHG8qL0jxvj!s!doNRxokVC{-!=a!bA3HXgW3=a|`)?CO>`) zMQaCJm^XC+k}iZ5Hx8$yCxYl%@^7z>YKbGT##}#~rdbNXV~@hPCic?g&lG5Q zm-m$#(- zXjikCw05K$EV;qH0ray&K->pvjBSQSuowSo$;P|FS;K-2V)gj0RM*>q`Vs%z*%(&b z3ByObxzO(u#_0Ot`-tkKNuxA6#p4)h2f014Ial1-Jt2k0kf$LW!Ep_G(0sVz$2VQ6 zj}GCdp-wovds|k&`X(fn{yV&@Od5ti3aH ztTvGDd)>Hpyr^z`6INUpBO82qf@cD~xkI=KerXYn>slYdjDl)lkh-0Ex4JCSc;LI7 z4#H=2J9yvBolhvsBOPW6b*}f31tzqo(1uo6HtrIb?XQELtQIPNMA`xA=h8asCElNN z90;G44uR)qhr(tVgC>iE@q3?pg6cwJ7g=3qU)ltP2hWMDVYhM3bqoG>yb;a4Q5I=^ zMzsg0KV87X@f~rjb{E4B8G=5aPrWyt{VvNw@?D@ydygq?w5{%Sh}-)S>i;|^sAr+l z^P+h3xg*kefIUh8x;7wQ#oW3-8EJNGQ+k^jnjB!iEB3%N%{Lm~9q5?qfvz6ixI=x~ zzpL^rTu;ry&`oY~=iqfv)X7gg+to(KFW*D=&sgzq@6AYKH0Q(z7!Wy{oBS!qoHO-# z;*Fc2nU{~#k4K9kFTZH#Bow)woY01b&`erwC{b%dJsid&!|Joc2kQ&c;Yi~bY5B(# zuZgE^NJE^Dpf#-_%WeKo6>B?QR3iG{l+@Gq9rR2}f5o<{t(*)9$FL?E3h-n^W}$n(PB?G~!{v3ufXt*Rlq z9{KHk6r)}OszL3))s666PAK|~*d$|@^c9sJrSPze6V*sGw9a+}Ylj0M{WiFrc)yUv z?OlMLom(m2N_u-F^PaX|^$xS_m<;4SQQ--B8@%TBPkUj-Nc8AlO)7n#Nqe)`ibtkX z`~(`?->lNLgkOTV7S0a0r2QY0NmsALI^!4Nq^vNF!X45As@}<`9S6EbAkT}Wzq40d zc!Uby$P24=5p7C+xbg&aE!1b@@Q~|S++~hn^}PdEII46AX-~Dr+DTc4KzOK8b4gl6 zj3`}$mwNeT5BO)v>B%Bg+O)~I7OKapd~zRsV+~=p@*nW=O%8ELf2hCd9s19%&&k75 zF1wkfcT7;cBRiZ5$AIm*jCg{c!yb&gOEio$o**7Zl{Jv>pf$7y;u}WVP5C1+G)I8t zMIUf_RTE>^M6m~V_lr9BOgUYXP+EmFC=2?=SeIjUi9i1nN}v6=bGP_$>WEOBPk9HZ z%%nvtbF_axnJdmE9aA+IGzVDSGy_RnVcNy3Vpn||^2vsrxL!lK3ES6WH@+R7hvZ*4 zd1NFk1X_DW^TA!#hOyrZ==_{T&tb~q(d5@Nh)bV~96HNZc_N?jTjAH*R6&>_l=d3F z=99*}W;K5Kw3#H2LH9*hB8`Q#-Wv_&9^|`&@y6`;LTP@gaYkMUNMmRSXYhRe{;0ep zafOK8iub81xH}cno>~^hF;uz(JFEf9O32(#MpXA#tcxbtXt^?=s81ltz>YhuIt7cz1+WX@dc+qw;^hT< z`EPYB5>}JZr9SkJb}^q&^4Akbve0CwdM<8@pVw{{2M}8QhWA-!xafub`X> zK7(lesUqtLYq;zm|BgZ4!2bwuf1 zq}&Cx4vl!D^WDMNHD7%ErjL}(fb#7N547gWldJirxvk=|PAd;$Bl=!P@fziMjC`e* z<^T*W3`kSXW`xfe5Wim4x2jD+d;u1dJe9{lZ{LU7QLWeFKDXPd1{moxZ1RboSJSIY zX9+$+2divUm`GYdoC)nfnVh%ssEqo6lfGd)`|pF3WBM@i`|MEL6{N-M;{HQPu)9Qy zwr3Km^riC5Hy^hWQ|Ua})K1RaYMnp4vb}|hSEcu3&BJ!n(P#8A@o8jTMxF-~@3_G4 z3M)J|@+)zhr=)R1+=07DSgiaVOK(kQDuu6O9n%xp@!e5fD3{Pu&w%1&wT?<3_W3>* zm7gJ;lGg@oAzwchjIS;sujs^y8_0i^vJoF0A#_JIPHUia1uFifxyx1_`N7@hJa&y4 ztkT^Mbqd>PY-~pJ!sNqjaAbQ%xIlhThpTK(VLfG$NO+I8JUTO#lTh~3Wve%xaobBr z_Pb^$scvBO@>Pl_={klfeSnmENz#_2ll!9bZj?8%HRrsjcH+hN0YWH$G2}BS{8qkB ztjU=M--pEG`d1Sb7YXWRApTQYN)%Y65a#J4c^F2yAdufg^4k#M`5Vpknn?1^Y}w91 zREXvh0#E{0lFB?f(H-uVdk8vgn=7S;p~Z@T2)t7UO{=en3Su8K@T;dQE}qI zgHXuarK9pOm9HpYE0r(17w1ix)fXT`kS}H17k<<9I{!}Tb5Nd}{5mVG%m!7r ze~z7oorCqHe%CXMG%}Dsz|rfInTyeD4e3o(oJE-c-`V{>UFS=n%mxVaP-%jDaRsRG zgyx1*j)tV~S$or`NWaI{e{n$acUtl*n09k35;qFUl?3I2NO}s3G2kS}IpT&o<-P@f~d?VKskm^+^4jYX7U4Ip@Es?3{WT4ww8A4Rt(tJQ9uN!U%y z#mauhvS?dVsaJ9fr)_0$=394OgYLz5{R(uB!!B(~X)OB2Mu0`^eeihXE!6vbX&1#m z-CmCynj6Tp9B-Uo5Cgxbc=Py&cG&&!e_*t{8P{*{1CAJ?=y&cZlwbP9jyLq>_v>sF zK>;K2YdGx})Ye$;URjBQR!4(DL`ymRRTel_7~zfw@93W1LlAPW6oUfp|tfeYo6@Q)z5m45TezbCcN@%E*@x>g3b-Yr9qd`822GXPMr1&2erP8 zjosG^b?qa%)zDrXlLD>@3*__42iRK%&=hUk3*BeC%l7R|aMc+@d0(50 z@495+)ebgVo638rKF_Ek4R7u31r{wLVD;}~_~_|){Bp1a_rqH@A#**dG3S1$A-`W; z4cm6rlv(lnG5>57o~&`3>c>Yswy|do<_}~&U$lXWohh>7#2ohM=~J=sZBrbk7l5t* zY~~y3d$(3|;$%(3XyC6c`Qo}wWZ=@>BKwyk4h|VinScR~o;3-5o{W-P8va#tC&#kJ zQq2+72)r2}p~X&njJi@t=ijc!UFH3SC_jYsS<>kz?L7p0(a<*sGe$PW0^9reTe}Na zZ?plkw6XHnI8*J>9;LW)ZFl%kQHxIPWAFRtpMTRiCJ%$qF;MN_w} zaslZjtB?LH`u!q4$LEMBoZ-j)hCjwyl~PB$9ExE%-ijI1#-c}Oz6 z_4$qTbJ+a&o~ZZ0S|)T(V_Q1f(m5JA!hNbPUnwde;Sk5TwN~=xZTi0YMjrc;qbn0@ z*-25}5;L~-Qd+^1MutdV!5v7Y0nHF<*Z;7OA({y7jKOKPn69y~MG-v*7#HzhG;SDJs8q!_Ct- z$`{wn`RvE5WUmuFxnF8)sS~wD(i*`Mm$llIg|)E5bIN8L{1E-ZG`wx=xp4C~oujGg z%08L2lQ&`vD5Hsu{)*mz*zcs9dd+ee`**7bg+w?rnn(i#5Ud^h$&oRD?onud; z`u)G1^f}cET)5T+XBKzk^?&K|U%g(7#w**hM(Nwxn2T?~C9R9x^mz=pC7Q`&S&=Xx zItC+(8_Gj_>fwuJy@h6eGOW3H5&l>n&|D7k;@P=+{Biz6y2jVo__a_yudY?>+5H65 zz1{J^@!e9*x59(1Egq>}QS}6Nr#B$2J73gSPoAxJNVC%MJ1h=y;%c*?6 zr0d0UtFAosQ4p>_+(@pf_eu36?^LrpSDa8_JCd8ane!7v|FR(iUy6w0UE+D3i=@vI zCpU+R;*3C6VUP{#b*HuEaC4G1Z2uC?bf72RtN9w-rv~Az`>!#x=PooDvX<_n3Wj?t z{CV8eWnAHRJxfpNG{XgMeU8L~0Uf1l!bk9bwG|vLA7b?T(AsCYJoikBc71QLHYfK1 zVGH4F2e~LaT9fIpm@}Q-sOECHUJ(-K{EwvTj_30G;-sV@l2VEi8EIjy|t&`xqg3mHSl@vbI&>Nah~Vii!8@mK5>k| z!F_V*%(?ptSMmPTWO?V1t{mVpKy0t5q_z5Y1tid!%ug-E;JeML@?hfR#q!IxG5leE zDy;fclXEvZQ+|UTVPH)*yhpf(I1Fx8Z_B;YZm}hyrvzanr0vGPZaHhKw_OI= zm&(9+?QdLqzaPX0)uF#HVKuUH@N#Y}1oDq8vHDNgbeQ(w+1d{Zjz@_5Q%%|2Ei;{| z*2Rd>mu$h>D5i39T7&0!zhNKr&#WdNy4L2Gx{pP#wjCj0db)6?wet>7Uoom{Hf7ln zY#pZq1N3%cfX`QqEUk7-*GgA2(7HA@inu8*zjUJgGCbwMd5;BY1mfv7JS(v~zg|dZ z9#KBdEodN(&OF72ecQpa*#2nRx}O|4wgY}}`34i((0Xl0M`$rP8%Pg`{!iUm==0rR zkXwuQU85DtI&1*q5{Mqt76{`(>6lR`+Htx+IkKdacsqdeh}B0?)qvv35N~&OBJ(?# zPWpsPj1R%)kFBe?n{-kY<~mhAW!9rtR^G zeR>A?ReW%nukx2_tN~VA8jY^?yQ#R#)LEhYe8othI(Ifn-U!v+HkKvjeel-@I_EFb zUbdRpL*{u^Q?U~MtH%hX6*6^uE9^kxGy3~E_Vz(LpfBvPvKO|`puHaR)+s&^X?tvW zk6XrEaY1fsF1CGt4&Ki#)9gEDEa&g(4`t7EkoXM#J<_1!pRoUIIE@=9hH!ji0gjF- zl5p_=G&EXMr43CM&&R^?NzOD^Olh0#y)Pr^*LHSZ#+X4QQwjY;`-8Wa|E zY*LGlj`4))Wpwt#S9+C_bW@&J80z$Z(w66m2RGSVIf=fX*IrpOkC%pOL;&~K$9~__x#b0U%CP*cj2NB zt?P=t1m|zBCl5V@cxAP!acoyFhmoGdEsZYW`r|q9#IXi1KY57$-CvDYGycC~rLBSH zTe$Wg1Ia5URn^mtC|CTE#}wyj)OB6)4dypU7ubUQOt~kLu7WOy=Zi^yVxWD-Cit^; z9txe;ta}H4AWj7vyEJgT5`cd9KZ$4qI%{XAIc$l%#|+M2B0e8a&BBdHx#67i@{UkA zGW|yuxE#q96IX6wy%tzX3&${^G2)-CUo>UULTM}lPeyi@$3KSCdw3p#Vy~&6SEh6= zVT(6@FSkb9uq@b_&*xHKAbyx5DTwmiv2UQVP?D!RtE3I>xK5R%i?Q5SV?iiF4Ft zM*M$_5hrWNQ$euHPR-WA6-XGxb8-(LaUE#7=pku&XkPk7gtyrTicc4{DsU!^sqmZC z_1}b~^Wa-$AYYd}6)CqlX$s{bv{ZX=+s0T{Hl4uqPsJ+^a8_Dt)S#_MSSghMAT7(4 zE^}{|qN%u1lW*|e%v5d@j)Kw`2QTcUpKpObuUg8VS2Mt??S93HQft0et8|R=6oe~0 zWLrI@Wq6GL8|D4@^SybHI!i~I>sPX)^!}UrzN*omhkxR39|vaA;~ExB)w0%eN09~@ z&KjRh$I(r@V}~h(f1#UTP{4a!-gRiVY^&B{>+dz7eA4Z4N5qND1<+vJLB&5Th|Vx1 zEsLfr8c8)~;u>k%`Xb6V<=Qr5hhb(>3vlwygVu9t&!oSJY>NFfPFSo_Ue#QG1ycS? z$^ki$_H9<0*7wW?K2HCjAiWDW2AcC_JD%bPy=$bk&OyZYUJ&%|45k;?l548j$mrA? zqW|13Qt{W`F{`<4MF(8;+600R&4mNb0+f~_-h$VggEUkVAZtVu7ukW9&Jo%*V>;E6 zACNZZhk7g`j8DU&w@t|x{8aNx{rjct+nEz8hXr{m(XQYj(mazVUEh6{o-WtlHAO?7 zOJOD`?Ms-7ca7>Od{Ono97iLuS$Q)=j9{wK@@NzkkPwZxlab@S)F0 zpC#%4FnqGFmUw_`qb;=aXW8OHzf7dqVbr=5;yPcP8Bb>t-`Jx#f;`x2nzwKCZpmsb z>2)M*u8KeTCmGkk6@0cG5<}OvNAiiRxbbfdXDqmRz4rZ zo=~G{&I9U62E}r{;iMwpAUr3Gm;%HXnB;N1fa0xj zbeW3^3zZI0ct-jS&xc*aevR9JuWv0l;?YEIKRS|y9bLg4lU%U4@e=1ZCT65x4LN;= zw$G3AjCu`>=8RuixJ`5_UQ(cRjM9?C&L{I!1Lh0cwp*2k29 zEDgm^5Pu?ZJ8{1we$Hr)gcnHhlB5@_`ZmrnZ^3zxr)I~d^+I>U0DL3+0daQ~mMg8< zJTeBUhoXK*G@6=-{7nk0@_a73Dj)D)mq_XrNT8mB;(mW1b6xx4yaQS|R^!#) zrjqXe%cy?`q@$VYQ;oN6gx}YW;KTfS%b=bK;#WGIgXVXRk>4aQ>!8K~iPaau{5_Wx zJ^}Se2qU*cegkt>K79)5oKxie4hrhwux{E$p!!4Wr#>QowwW;OrXbD{cJjO+3>NF# zYe`SfW~aN>A`QO}ntApT!&^NR>SqaGu=|Ods?MaHiD}i?$3FK)#eWw}BvP*eOPm`D zia%F)M*W%YyJoxy|N5di*m04K!np<*4}gqt`Dz)$`efbQ%4@T84aDQhcoM%T{&S4 z^uBJVb&s{CK1?1cuBGuPu2p^xRNZZJGnB-o%1>kWzH4yPcxU1+MxPl0q~Qhe6HuR2 z%=kB6^#kzS#Ft{rG&6b8XBDQNoq{&k2e4X60Z^9E1?Swnr5&`q0VnKL{ue(T+@^3y zPHRhh{j9DJ#5FMM)qPZ-MV>+HGbLUqO-+2m#{`r1rT!UtTh)uu%uegT`#(L6R3{qM z-yr?2`4=+_F6(_0)HCizb1zkWQT{RN@$$nqT~+TxtFVVO32BJVoO;7R9#~8F1xnAG z{R-1e|It$HZ*oUVJw1B<=~u;LH6}-hHe+v*FNkCF}BJ#kgsEO zZ63vo;rZ}##pOcz!aAwdIrS4YF5qUz!F*#%W|dbWuG4zIx01x=Y)tSf z_?YAd8G_#L?U_Ir`~s=}jMPV#%{GZK$s*?9(f7!V(|8*_K{U#T`ake z5$&C}(3MZtRzS&UUG|XH+DCVsDeu*(#pWMw#|^T+v9r~_CNz_Qc_UB9f5rw(wo zeYZXK$#2R{m+7)`Zwj#Y$fj2UFj2hYSdXw%nO zmi%>+sei2a@6+3H&9$cdchY8NGVUh&PjzOwcV^@Jqt`@@u(Qym#(LZr7zs)5`)Mxg z%$Js>*TFfwA$JLwj4SnCi1PF&NY}zl$7!rqhZt<*bqqed7y*%wJM&YI_4tNOi4Z+u zJ~rZ6aA&}D{<6hZ8nAH+|+8hpJWnapiCG5A3jg_CVtlOe9jiZn? zp?yU{hFbDxYii2sEe}I29X;uNA_U+5_=F8UOqZW_{sgBZ8MJQKh_i+tF|+k<&D9U3 zV)m^JSkW_F4jh+_!{7hG^@f5i47mz*PEO)Kwsqx)cLbrDC#%*5-1~z-%-$|O-Rc4F z>=%oFCY!le3q3jaYy`J2UaW02T_T838);50wHd#m33i2hIJ^xRhJWOw0ok78rf`f|^j?)>uDFg`f16n~$2!RUUa zgTn<@yW?ee;+!Bm*ha$Ly;^OZ6V^C;=N%}$xEp_Vih~M$FG)Eq7L|-+sxE_enc=ao zhJsDyKdq? zK$<7$Y0{SeroB$o2!kp`uj>P7&vbX%KcCCBJv;JD9a&O^0L!>-}{XmywDd$u9JYU7y}FP@kGn*NUP=` zY3*MijKj&V7el;DHU48>yDI$9nKw&(`MM7_zF7k_ZpydIfO9v?J}ZYh6Bdf(0p@&i z6E|$LQ=e~{YJh|}RdYSJs5kCi_*nc13FiG=4}pV2ANg#go3u;IXFZ<=!kZ1vfeUW42*aeXcXNEvI?oJ(6ds*vpzdpHuub zazU@&(hr=((_Jx?T?=3`x1p>tZ^iFNVD{K4n%->I7#L`5< zl+&7odQSHb@9d62toKHkSbH~H(K?jg)AvQfW$5Sj4ZeHW!soNL za`dHQXnf_5hUN*U+-WX#o$AWcOK%F|LZ9KV__r!gfH)Kv-L#P7n)YN@3)kUy(+vD- zy9oR4e}Jv_^wA__nDE#ZC2+6w6n1ZQ4!5Ky;D`Yi;c$&^+}YPqPMflrN9k zFsEGX@Vy7-t&azL`_-6wDGet1J%OCwW;l0!4@nD0`7?(Ck!*RBrH}anaiNpgvQ^t5 z>0K>OxW+KM6;Q0J>by_>MjZ3!I@7(m0tkzN?u8`}9F4`s187}-4yJA%0)z|Dps*$H zWcGFw{dEw;~${ z4UC2*lNLf*Ofi0(Pv=?u+KTEvF1nVmLEimYj+qfm{~nKj57Y&&m%=D-VbhXK?U+A4 zXm4MWt2y_W!*Q0`3DN_-_>2*cnfhYSUa;k=)Ii57~<9jE; ze;jMBM4--G|WXQ0#S z5kS1ppBD_Sicy`9Ihg-_JXrZ9K)&rx%!>KV{OYzw;c**1Cv2n7(v?Tg-bR1+8P9!e zii*R&Mw|mx8^sU9xLcU1+~MpdNekiqjaINuuB7>E2~&38tn;+v zq2izU&yKKlQWr{pca$nOTU`2$h1C~I(tJRg3%wKGDc;4Q|Ar`Sr}PW0{k6gM51eGp z*?Z|py+2<4Fq&;@b^|>V_kzN~S~{zs_o?|x7oxXin7lIX3>)LvM6PJy!k2uGg|RWa z1jbo)73p}Thv0*7@6_b z5N(yOq2&ugm9exmvGX`FNf8vF;W zIXnfWMM$d&;(4WeRGna-(4;DTLt}!-`dq$?j0Dr>U!h6cepq^m_PvPgCW|^s#V>fc zWFI~oc@iK-2PU=l(@slu1&ULZj#V1KVsBsk9C!x_FJZ%si&|65P8@F3#lYyPm{fQg zTgKc&;yk5G#qxt~v2Ke)%<1SO@!x}$bY4e+;!A}`aBGY^-_)-O80Pnf`t53CIGrWr zHK>k?A-1|l`&k9lkcFLGV3I>Bwv0~2J;zP}VG_G7hbS!o3YU{COfcZrf7%CrI+Acn z5LXipU&2jGtknI$RC<58b-{!xydrIdXR-p-XG=AQge77`e>$J8%W=$`>;>s@OBszD ziV|WV%zpu-wS8MKX!$yE_WDfb_uwfOPOJ`BkNe7$P6epIgY9+5Vr!&`1*)aM{pt=^<4Qun<+wEohb4RhV=XP=P%3C0v>P&fuf2_L_ zF4cp*J9pyJFIR-ppUtQ3#$MHivEcdjkn|Pm*Ka1ttH8D9lSCGspZ4mGt2mc#O>*n8c3LHb4M4Nf^E zC*<5y+6x~o?#MSUeTt-UNLSetpDaZ?^2kL_m14`(HAsE{`d_WhUHn@qu4P3XIVVpc zNDl+Wx2lGFe?s^gUIX?SEC%K2j9i~-*UY}mC}-I5q$`Z##xni(Lzi3L%Co@8y-j%S zgr@v;x3%5W{1|WPB&Rx8facIS;!WR81vp>xbm~cvQe5v9f;w`Rp9cdZGQ=s^i@KPvk zb@|#1!X-;i*uf+A@5MnMmmO1@m2eHtooGOw@im*|HxZZQ=wSbA{lRzEGUXRpXjU?i zzoMW2E8JEM6P^LrVf+?$3Dv>8y)LL+Fq>%4Z?kLKdv z(_v6JF&xN0Vw1n8h+7Y#@@Ws>__5HrSA##yF-SX(VmeNjuPVnJ&LEBOR3H zBrF%{v07|YHU!8kC@cZ$86f4O(w$;w+F%;%R?=fe5OX{XYmZN&_b6wB!w4fz8cNpE z?FGaicxbz=(i(ViR#T~P;?mU|pnKHb!bH}nUMZvdL*hoIxy6^FAU^-dN6nFd2>kNx zBTO~0CtP|C3%8{}<3BnIlW^3=P|^DnC|*M1SI|E&0QTFxa#rIZpFtcMgR@@#WTpcJ z_B6>=@f7BmiJR~5$4&jxs^T?ii&Lq-o};{xtMCH$tU1Oy z+Ib;qA&5=@=K5Je$KluLtlSOs-b#!nG<%pv#qQOvWBf(qtr(qvf-sL$oU3Y0^&ga; z)2rAkY7D6aijQowGZG?v%7{<1wWKpy!WDD07EjpUsiESDv$5hn_IF4jC|^Q;8Okr% zlU6krJ|meaJkW#%c-odA7L^l5b#F#(D7S z)(enyr22hO*km4eA3pBr2C5%J{R+I;qk(9+?;`rn`iq8n(QIY4R=i*R_i&_3k7LUJ z5MF4AcjWMQTBh<~WbS<+&A}+|wMyTO=pThGHhpGqg6<;alqPt=HF0E8Z)vl8HA|1x z;P}PHNcMg*ZFS9`THu6>4z$kmUb)yd_Ze-raJBhjLkBi@@TPZKA zp?P8(u_qo+8mRT4GjkvBDS$Vrj|tntfyPU9Li;(MnM|DgSoj&fg)c=f@x-@M(w;DEY`Iw1eg>obhgZ9Q0ck_Dzi^DW*cwO!C~Ref z8O}aN?*z>SZ0pk)VorEe`76csq1m@^YOSg0(#ZmZJFQVHP8EgzMKCOWx*+bSc*Zlj z5BT%Bt!UF{F?t_Ah{Z;y zPfBiU_B^VOur}W>s9y)2cl2fvK4bCNm(7CmSK%6FH!)UzRFID3$4{9G%6kzJ7RU&T zGy}&^B5yn$hHPGm#0{i1W>)#-5lia>=?}uzjl%eC+Z+cfC3`{k#{145Rb1 zg2%vVlPjt(1k_u@)3ckg9!YKG$KJ2#dQExX$CGeC`=+!))(J>!!?Gb3K)p58OBySK z`t(7gSvjOr3{d$_;$R?et@>w54mq#>uXMrAi}@Csi+y#>{41oFdz_yLb58?w&xk88>6V554Yfw)oCfOhT7 zXzJk)f*rMgS83zHD{it;GyX8@wUNi!#>j{AC$-kV$*{w0+PO%0`#ldwE*nBzod%@I zQB%HM_0YwJ=}~m$)nb{nXq$la6JqVKDjY6r^5{NWFX?xApad zxI9OBH03I3H`+JeWURC{@5sB8!Qj3683TB@M>!rCM~Z7=R5V}n%_1t^z_p>&_j5$2<@X`_PptUU zu@|f@AI@hiroA-t({aMXp1fdde|ieMk!8;qj>F*tjPA9uV8X*w&?_ikTY2LKjr}s~ z99tV#ef)zgth#(Nv^h&FxuU7zn2n95Rl}H`eI$LJcwVb1R4$$b3u?a8j?SpXbH@(k zKf|ZNZKEIP+Ke>a7Jq)S-)f$G`<>9U_=58;Yyi`oZ}^(tU7lg(CM&kpgwjJ(@#C-h z@@M)=G~4eYO$v931NG|4+Li^dI4~Pcf+P5~_R0Ks%y0Pf)C$+V-qM}M1sRLoxW|zj zV$J&pIL%=fdeuIT&9>cyt;@EHyasFG;vElemDNR*jUA8AjwMOQ_Mb&T&IakkR^jcS zKuu5yVwdYhxX8bUT%mK3#UxCZ>dQPc7#59~2h3qThL|_!MfaX!T(7oJ_d!Y+wAvs^Wo9s2WG7rVma0DEj`V#>AI`S^N_zBD)$C=cmxhiBuqi|6@muxMon zM)%DJH`89yxK$28b`#Q{7%@B@ zf~?1*XTeb!GS`qhF+D!sH5H6!WMWCnJ)mz51^UGUB;6COP4kyIjZNU=2TxS#zG zm@;p&P@i}Ba|Mpv4*ExRXI}dUaW8Djqk238^C6e<(R3r;)@eBR9QqG~ia()#Nh*%~ z)E<6z_2N5>0_C-f_u#-`3%16`OymcwXWj==q_0~A4%imRO;6V46mw3w0@mBpWHW<$ zg63FD`OI7B)!~{2&b+;=KE@14R`&%eckMrI#$M)e{N{`)94o8ItRBqFRao}Vk`CXv@{>!esGq|IA0FAW3Mpx3lT`UBn@p=LT!g| zu=<^i8>5ZnAKS&8>a)P0X$hpnwvz+$_OmZv>&cgXE%|88Ao(?!o|DxWj|MB^HJ0v` zSm*pcN%<{&d~M{4uPY#^cy?8+X`IaSOf~6z>WA2RD+zu2+|}F;dd@E1IRhrabiUa7 z5rS$4t^chC{n_Ji+oX?J$I3#g+?jT*0FRfy(wu*A4iD(L!HLX7d{>#w=(7;Nbi-A? zN1Ss{t;RsdbND^hhmXCxSn69Nto-p5A5QQ98|Ov%?n@te(KZOpcFg6+KXg`eB@X$X zN6K%o@$lfI-|oddGdGDNwFq-h_LR}t8Qt|u8uO5#p15n;G3XLb``~|BDZkKp2H10y zOu09hmmcpT=Up&Ehq7seP4|VB%OB?ZuZawF?9B`hFJ%iX^yI3KOJS*T6YZ4CL&0dO z1gaC*e!)RF+a3^0YYdd*;)S5=v6&~{;HJ(s_?PaJrBSnV=@hjCtu{c3_TcCg6vR z9=Q0BC*;xom5WE+fv#P3IK=`gD)wmnUY&>1z;obTC%j;6%y5O5yfUJLG`QRm7VV!% z_v(&>WjL>8Bri;Q0&(A?z{98~_zpMV>k>Oknn%Ks27H}+JU6R+43z7_c;YbWA9W1W z*c{)cYes!(APIl4MtFU$>UdsZe~QOX*sy3RFQ~51&oXCRP;&;-*d@hAtm}RnR*t4U z`rf9)J^T+SXMk`S>HZ+s*ikM$+M3^P6^{e{`eK&{iP~Aq8gSPTbG`?A@FmCV^8%eD1nmlm%z3kog>g|Bb=-c&eUmD?AV&cR+MGhjn_Z(b67SyW%-DHGBf z)1u_*fM1R>!bqUGg)i<0==pAYq`IRsT_R!TK`|GTeI_$_XmpN8f? zdLY$4L~NJfHlvfuReAg5O*P-_&7TfD&3O&o(_sFm<5?EzeUP}QE~nq;Up*T^nzMm0 zn0FfXoQuY|@3g;6zvt{Iy&q54tg3fa2Nf06N1^A(Yd^G<#zkw{^1siCClYZ|Y%~16 zVKdGPCQoGb9on@H=9;skMQ8i}s(9pLM_QL$`It4|G8hwHFurJJG{P(&B>uGZUJK z{55!BbbDwUa~8FKEE&xqeCRcp6AvPB7C-A54w)f^v=8!1q?jnKXXhXEmOG4|V`xA) z9{BMUU;n&>)56v(?3C?G`!lLjP-B~F>!EqnzzWO8KEjkvC#cqsI>*PnVjCRqf*Lc` zH4K2E=~g6Wqh3lp;q!uDr%)@)cug;VU_xn{iJ{Va_BbW(9ImKxrr zdM*LtL72ASA`s?C!WXpf{EY=Jnu@v|Y>Bf@i^yG83S%*HS0i)lX;mpllqvmTq}ZGqwM`zbs^AKz_Axh7%~-YVQyTn3tJJH?r(Qf8j{ zQSmE#se1?}bl;Cb?HuJj|EKu9Q-Fl?Cs6T;&O=MI{Ta?KEUgf~R$nFl{sKccdq~PN zx#LnC9M#RjvQudVxiuca-GX;`cz7EqJ35fYPHW%}8VkRk-AEUa{xLV?go{8J#JiR3 zVWcD2mrYwmVXxzG)GtBnV$*>3-fBSc-dGjCm7^_SmqkPApR5N%y)$VYsG&S`Y^88~ z>@BI*Mj>lJr@NtCI%GB+{dNnBTr60muCaXF zs0Od4t%O-k?m_33bY0z+aLuNV3=BWa*5uTXo@c_;JZmfBKj9Xeqm1fM246|nR<`NM zY0j|eQGM;{hk;a!zVh0fx0p~7iT7K?h&O-o!E$5@q^~%GZbeqizeBOOeW|(FH|z!T zD_e#Yt^>7A3==i&yD&~VM{F~@EGednj}_KYzGt8lJ)b8XhI5*?l$BNsP+{7Nf$zmo zZ!4vF3aD-*{XUX5BYgWTEWh4oPbb#}HK)YcQt3LXRUWi-p!8op19J?|!G)6X5cl}A zX4iZ@@bYgfDd(}n!wwL$@hDSz(ei+v^yxEF)hLFH`Hm}gjz*U`M^NSPA23E$AL?Eh z_4gOm9*pmk<~a}di2%~CoH;attBY?4-~PONRF6qe@Kb}{2O9G9JG5uPd_7rqstLB( z(+qz)E`tTrc7WmJk@$T3V~ERWib2+o*w9@@DsI|>>XD#(yBjBMsv#^CMX@IYakB7V z{tgwVxm?@^*F!hrim#QzG{_PbHEhNROMo;ftb5&DkWK)_-!v{f-}MMwd}@RgTP)~Z zhNmNc2$cuf3$sAs9>@^PGdLqC7m@M^DgR{B@)hFt-2IT?x(7e5zoW(_X4!0K6iabv zTmyPXivbR>C7k|>?vpOzr~zS=uf|IM$d{WJW9tMj?dk4AVZ%!ue5^AIPrGlWcPl@k z<-lT~yvK6idu*)JD@J+7h@T*>9fvxLW3;5(8DRvpruQN5o}s;NygLcvKQ`lrg;x2P zY4^kNT9f74;&xZ@H$6``F*B2-FTpA|R1#0YyGm0qJ>rJsXPE!(Q3}VzFXz4J^YOLF z2|SE6Mse4|jw$WgV$UW)_aTU5aoX9LFfzL?BmIEO={y6YX68V8RXmoBNdK=rs)8wN~LL{&Z`At$S_<f)4NVTw{wNa; zh=TVYfP4;`jkqS1=B0VmywvS1$a8YiWmJnNt8flxUDwPpy)BIMf8yA3>I2i5>dZQU zbbrb_iR~>@F=N(z?$qHJx|=yj!Wt;d$W@+K_zqt~^Jb~?2=gVq1KwhU^lBI_vWw0# z(j8Uzs^ajpo~0~1dWFsjY^=N^BzG9i5BYi_)t(sMo6ce$ZJ{OIqVfS0Clw}5-|0<(( z5A`Me42GP{XEm;O77b03MbWc%aNtJ>oK4=TyoSOO%_awp*2>oel%IGvpu5uNiYK(e zQzsztC@g8LLmniaJVb4&d|djxpVUWa$ek`N2lu*VEQZ$h=a1}%$x+4H=M$r#!^3=3 z8o}|cxrqFB2Cqg~qWb--i-#0so}Pf@kD%Y!JZvfVAkHn(=PXuW-$Zk&0th|RSZI0LN+ER|U63X)s zX5g2^67V=O1kbzaD$d8qZysp-zJL*T$~i@yps!)J>PJB6ft}j)j+t8J%ZV=p^-c;V zyf(z4-<%ZJt9jx+`YW|}=chp1@`FGg7O&mfPWmy64KQdSyLJtwo=XnR&ts-A@nK2T=NIq{LR%ZZU1 zrP0oZ-&Hk49``9IUu<-2JoSAFwMt9SS{gY0dOvB&O6{s~E49RTT;ZFkQ9dZGth7(i z(kp_rB2r(9fmI%RpY%nLE@oxM^qitN6iSPAXflG#qCu%5U8E=F7N$8tTE=bnds z!zMHG%b+l-a`7`T^WP{c(h#ltJ94UBNqsVk*Hy$FrsU}(30JGDegM*EGx9i+d;oK7 zKbHqCdJRoRbw=V&M)QmmGnE@K{AVMiSc}#hlTh`}l(r_WQh5Lk_xZ`fLD5Kf&xjWw zIk_RY&^|0m?Bu39}4ji!YmN5WO|81ZmA za-pPL#nQ$$-2B&A)b{O3aZDi3)P?-kN3F3qLwgFWM$1hzfN~DHwM&DKLv)D$MuJJ* z{^GpjG%ax)WSkEG@@ah68CxU|&eUgr|5%4Bjr`It3#oSlo$s|3Us*q#+R+G=rd_U$ z7eCh?L5eFVd>}szy-r$!edqbud<&h$exU}s7iv|$(S7^C&DM!aGBfpI7wWy45+V$XF^Hv07f~Brq!Q`WM+p{ zpArY`VEu+XM2bDtSGQxTALG5VSew$xRXFyFg7&4C3Kn6m;$$T4z=$&i=?(3ntTz}O zvX^N;-C$)?a+L2x@>oFetkQtwe^B)alvXDH4oXiGhcm0J!^)e0@8^N!Azi9`?)?^Z z*!h?1Pi6u6$e5v8k9lC9SHz=v}8eP|t{u z&WM7LBhT2N?biBHde!&JDHF5NByNTz{=F9W(8kqUo3+ z=AV4=nm64s>cLJDzOK*UjFBHM{kfMu#1t6}@*;&-@wGOR!-P8Y|{okuN?}X!usA zAiUM-58U48z$Tw^;Ufb(qwDHYO?bm}k^MbetjJuCztWBP=mZCO+_VDtjU8C$;!7ca zSJOVx8G-NoV@0i^#h?kOk6s&1;o8LnZLiDVcK6PgUF?Ve`bTPJ*ZY`}b zCtw*|-4P|aq)+Cjk`3YJt9P(?U4NkYfr2HIS(^cN;D2l)H%mQ(O<%Z5)@cq?_da{i zS+>3g>C%jou;l0sTx?KR(!9y1OKM1uTX(v}@*8;Bu@rKryb<@0)!@r*r|@4MuORW; zH$h(-7S))0-uL8tf>Th0mf(-WTC{U{06TScIL#;i3^Cww78<_0Up&&>V57NGz90St z`##H(*BfrZx;frB40>SpfmIk2AH!azJ%REXo-*uW9hsS%0z6)y)BNGel;vgyBGdwG@F`ZBh*P$KtB2-+PzBnQ2vw?N0 zfZT_zF?dodnN~0#*3$c4-8GwJ+5Y=5ZLco><4}kuHYc&>brXz=+N`<9@8Ha{A2ELQ z1lc3(J-l<8fw!-mq8y5V4bSX&gX;Eh>{4@S6Kx0XcJ)b{n_z6%XB^OZi;TIE1A9Hn z(Y_!WL;R=Um7*+kX?zZ+gvGM(<8$deg1T_^jRWqjbcg&``h5898oVT&-W!_Um394S z$OG+ufJ0e*S>NLx&dEEd4H^@HQ6rY~XV3Oxm~k7vIy_I@o)V>UUuG?xFKalJY1$a9 z<{P%wgvTvr^Hh4y*eY+IG)^&*tA<{}rKNuS!iFB4>Vp?7c_DIAHet2p_7Lhg950lP zlhf)mexOqdx3~0=_biS>PD(Smv9k>h&HTYSY|55&En#=4nYvGYzh@ELZs3nAGU*vN z%~^198B%?*OG{1J*nl5Z_epul?{uGvf66Un^ptRJm%Iud#NGt@U8H$qH_Q#CN7J7$ z{XYZgk$4P~*5^1kd2$`gPqe~*uM=>olP>t#hGF9oCOpLd73*Ld!2Z^1#SP33;`8Kr zEO^p(RWmGbb2nVG`v{xK@`c;pr&V=#to{keO|LDfUhu{WPb_^9iw|pt9bQ?-h1;0E z#1WALVA=3Bc)B1N9uH2yTTxk<(7Qb+oDn@^i?N!)E1)_?x5@6Bw=T2~<#j9WwJ{k^ zCC6cIdNiKAc^oox8}ih%^>~qeYYZ*g3{fYK0&p>*eO;UZU_{r^m5_$7RHnLVTD@i#Z*X|8smP;NKG-_udoByYWsoqzi z?#B=KuiakOwrxFOHe?&#xslJxrVE(b(O2b{!0{lDw3$YKBC1>&UD5~e^&hMpa!^k0 zUrBY^fpV}mpE0Tr|B+%Ps9x~2|4fcen{gez-f268n8GV24C1@Kz5w(7i^Y+! zMC>qpm25m>9k$zWn4S68nT2)e1twD-!>aVR(9$(cR5a@#G#83d<*-hnADtiXuJE3H z33Nfz@_Ib!cR3qmFq2nbwH_NUbd)~3S`t1LW0BWsq;aab)2KR6ZZRFQu5UuK2j!Z@ zx8K4n;^}mw&Dg)-6Mpq<#dSwo0nIzy9M_iD{!>%kx17ApQoeh4iO%t~z?n_WWbuL> zsB*x+T}yc)uRES>@(6EvP=ATm^Y;AjwW>dgSEkBsxoklur{AF*H04xdTzpFgziXqk zggxNjE(;V6e9^4|l&@%0HaxN*l;y!^VTb_cXM*+FJ%2tzd;8 zroy5F&xCW2Xt=SWh_wj42un9;fUt%u&D4Brfaqep6Ut9)7o(=pI;OKp%&b!kP(2CC zFO}m!G37tCyD)sqJ)t=5oIxV=I&vJ24SR&|O?=^tmE~ch!#SvHx)Pn~tPK_0PANYH z;i!iA7wc=Az<*u3^Mqw6({!+P3OG$9(=sVqy_o@$-sY5tvBJyK5Wzo|p{9bYc{`9$xbU&zSllVgt zmk|fN#HcHr&!GLyNp~P=D6Pr|-=&{KI_()Z-lhoe-N_)mHXoE$e(h`~!?*as8TX-_ zxJGswbXUy1sfB9J1~8GHpE)`DzjBz_d6Jp^$+O#fshjho+~yg#q>AUHn>PS+Bp7F3)4biYn}@ZLog&Fw zOcx3ZsU9^utkT#bTF;R*I0lUa=Az-XVeGs47p=c8x@$xR4Hf)sqmVb%n z>LqwTeGXo&N`NpxIz8_wCpa8J;u@IIXg7v0z5&FQV%px7u+%3VoH7#`=_#e(Fs=O& zO^sa^5Ku1)zkPjMrTMot%Te<|dOZR=>R;4MT9}XiuXJgxS^yIMF~$GwOH07+&?c~M z5vq8ZbaZQ=*kOLHt!kd}d$76u?A8rPUm?{sP;TLwH6f_xYSD~d5V67y2A#f)=jR+_ zW1Ei<*_nT}{pbA$#Q#W`g*zgBn76YbFZ=cz_r+JJ@3Xfq5klcjyQ5dw#Z9@QUWo@I zKIWvErB2~J4b>uV5dYFyVOPQ4NIZ6F3r-v;;PP|96jl+h;sWb#g7Q(t0tTerq37K6X^idH#w_Sn5gvmC+LHu`J}gV9KQUp2CJR+B0m|(%D2X1 z^R4BCU+)T%?_XpmzTAP1C->mwQ-#bs!xQ&WFTLqGJASn8brIa6hN?SQ-^x!si1o!i zhr95oD-s94yeS^dc8~*2BSplS3E0AN_~ZM7xNA>jL(Ptn_h<>G zGrggQMHf!7(yAI%wcvTb2&RP9mFA}kG^8CF-JjCuA}Y!P(|x(5dJj}F4-Kg!b$r?i} z`zXzX3P&hM@aC%-(6r42$`yT1K9`Y35^Q%zBu@ux=nTzU??0;3alGrhLK5O*vTSJ(T9j&XNIQQXri&q+KEs%{RSI<4J5 zw>QfxZw%?aQbTisRO76q-dEV-Uy~~zWA(ZM=u71|Q+Y$q%H%NOZWAgtvy6(6fyEop@ zCW%NwMUoYg^qlLEO=Lzk**kknQX!FqjHql0QBu#jPGl7+6)Cgqk(tdK?{)g){rrBP zJmWs+T-SF!_kGS1OEmPIqO@Ptm=)G+U$V_(^3dZrzVtA&t)7k7&(m{*Z!Ki6ok!A1 z?=#Z)a`?41Sbn)!U6Y154;1IC+9#ce6K2qUhU=#SaS(J)t^t=D4FLKKP}(W1(+f0O zHCtqF8?3M$=ES|xD&ADsr?5$XTm)%;cc>h26;hlkHL708FNv+;17S`upEj?h^Ftc(iX-#1l~Gf`q2>zWq`ll)=7?1f?_;CSww!89dv*F4u)A8$ zD2GAffGS)doh`|40eL17`er)yOg?KLUkm33=2!juoVXdlj-L^SySC>)2kCMeKY7_= zLEeC^8e#*~1JKv&q2dux8OGQy^U>8TT|@c+M>W&q_6@d))K=x%S{qjY`F$RI;R_?J z&rUQRi_Z2!{V?q}GNj5Af=SE-G#`Svv}{7j$BTmF#VRI}kUs zRqfYu#rH}dU;1?oXuNQC*8uqY#*Akq+)(;f`t|jOMZYq!m9dd*y`T*`&NS!V9U44ysQC=|dUvO$P|W8F3Dh4@TmBp)>?#bJXuYs%q~*g&VH#LeI~Bjz`j& z^7_M4&Hi!qA{{ z@vHGuaqXE2k9k&JrD>brYa*$Blpj(xzzV;GD2~RM{W(>&uegsgCZKs&KtJEJCUV0tyW$R%!{tz*6>>z(us`tiN>~uaP$sS z{S|nvp@z7F^zjPHGuEazN$`}FCWp~D{tLL}gi=ifajJXmnY348MpmMT(5uPm8pxwC z@)P?o!_E{{zM@-4ms78^(cdP**AwHw%4DY+JDiwYsn&#znYR`Cly%_53FM>mk@^8i zb7(22QrV2kNd)ng_R8Z1DrewBExmBy{%i1a(AqT88%TYYzRT1F2#0ZvVLsAMWUwg3 z9_l|Gg0yyG_R6P>vOrPSJXrM`X_ZQ$aN^FPT*~1F3VI#49Xy?M;yO*)pBaj4h2hCf zxO~o4ApfGV+!L6Vd8doiS+0?EJ&+UAT5+P5xC3t;F;QLtsP2$5DfXy$ig@g_7xIS{ zAoV@m{qTU<7>z>9Qah>E%WNv0*FM(=NW0?KsToSgtE^Wj&9AcIu8;IouB$YP>M^l5 zV=}ZjSwT5fXF(oOQeFbxdM9Aa$d+i|z>?ofj>f^+49sn&C~VYFhQU;hNj=NrHf&*} z#TeycsPx3DXe}cj&znV9aevQ7DkEk44|=FR0?X1ky0+6oWydOeGCkr)y-=UL$QH3C z(}nc4iBy^^S#JqaCJf7;Rl`lzH}S{TWVky0p&A#uERBH+VQ0uA*Fx{dKLzO#>Z1b? z@T3oWX_~?g6xx!8HkX9MjCf4tqM~-+Y)!`%c`&`zRJ=d2lzpn#jg9VSB1-!ofX{H9 zkq%`kPsc#{!3-dbP+2Z244)V^8c8Fd>ose>r28OT)+d9!%q1;lHInjK_N=TkbU#xd z6t_@Dp&@UTPIZXnb!4Kpi6jnS%KK1%ibuUW(4CWcq*E$8HLE`ykK6dC=f*dXN~0P2 zzEIg55)N?kIvSI;_kpq{p!(1d#wc9JL0v=P;huchxS<9c-}bZSTKfc8u)Q|FvY`d- z0r(i`$O&Lg^T6Ckm%rZbEl-Bn$e24D@XR(BFfZ=zQ3LMdVLC%R-{7st8Cy>0kkKZr z!$)$hb3LBvXoX=%(%DZZSLu1mj$P`|lE<%F51LtN*r09;o=@3Ba*t6E~~L zz6)-^cqz`{DMsV%myBL(C&=Qd zf1zxn2ag_k6CIXzmOf<*rK4{xeEGm#JM~{8Iz`RkPK_n_1<%tyN&Q2ge?{c=_rYE| zjp4L+FaDw5Q`~fAiu65`q^OWof}Ik)^WSIr?t6#Nz|wnj?tdDpRP-!S?5;4@s5Uyi@a`mjHL7eUUO&NxxG z4flS&SRUROgDDI2pfJmV({;mFGh^8NyOSJl)Xrn(t{u4g%Mmet|1LCJcSXFt5r_AO zP2>068Ld|w#(i>yUDGFub?F7T=VT6Ex3A6b4|@e&A3PNnwyouVj_qaJs~XwuiwS#l z{3UKl9)KSCd73@@-C%!Aiq<7!8npkjS%&=8Xlbl)wA)!Y(liv;WlsZRyVGF$W1e&< ze+yn8jrh?Cl--tm#I3WR&Ab7Oga#3;68o23k?QGmt%HRZLgp7PJivtYEuMd}<~h*=Mv(PpJ5OCI$=s9M+d z?k?5)65qC!(|go~Q=Qv_$Cv|rUw|=RTXTr;7~{aJIkcs7C&r?0R5e*TYqO?&QJjeT zl+354T@nZLL-~mZHsbaR1hu~7B7$VxzrHwqVmD6fp}p)khZ`+9Eq=`1Cxcr5#c8el zWpwT`9;1mKwup_3=<; zA8gv?EmQAL`f(MGq&~)_7kuEv&0ro->o%i$g}Ya4g5~oOn(0Fmk>-Tf?iIGZs>Sgq z+i=|rdxX1j6S*#ZFkcFmd{muYyl~_j;oISh#y&R#opdAVo{0`}RbMA;Gx4zYdF|@B z?ZA1QTi;hxQIx{m3U6WCKC9u5&|xalwCFM_URAP8x@Q+m)NPtN-=inXsPCU_Jpn=Zr6o9C;Q3RVP4WD;+6QfAqzfE z+6Xmk(ESwQ#prV>3%(Ek4yyOGo#?r`*e9^Jj~g^ULHEg~u0++-wXU^7+YWbG{Yx!y z?#?>0X#Wt%yIu?E8o9y|x<;70b36JyTMHku8?$4N_QLCfllc5!&SHJhDzUfEIdJri zl#lb9h(TBS;I=cl(D0%&6nfo;N5`k4vz{7z)KyOX*NV@+^Bz}fB4N;#Y&>VHhgX}_;RSKG zVYmGOW9%5=i9DtsIJbr)Mg`>lUm>jv4s zhe7-vS4lX8E#v;8!dAtB(`gSHS}R8D%|~shfQsig(7L0qAnZf^YSp;Hm(izBtO4gZyFsC9~Z1F0>o zajN-c%(>s3J6SmKisxReck3s>XPn2da|8MEWpB}KToZOD0)g;_?(z0Tzo|X&=HhKI z$G;doJ%5Ru`^WGe<^QUd2~*^X*rO~fHwt^cUW)Di)Ifd8%H5%{zw?|IJ;ytT+!x zj!weMk6REY++h)ItIInPz;oVYV26~0sQB5Td?xksZG}A={_YneJqE@H=Ob|&Dx9Hy z#0AC|amx+|-tch|jG<>o$5~z#9^T=EA^&0Elp(OBX}KUSMVd1tjFiONQt7vWM`AQI z4o(>9ArnfF>P4e^E!aoEv3Knx;R?(;+R|fIYF9!1gkSWQ!pm{Lp?<+d#UXr>_Ofuy z;gWC|IvlP+80ZItDM*?Go3_0p`gdP1mHy0aybY4G^@v9U1nEV&efe^sdUxKy1VMU) z4O{sLXzux#+|iJ2GMfb?LX}o|_10T++3&reTEO$ybP1DpDjc9^KN1Q5vyjF`bMaYF z|BLe@?jwD!tUI@*RGQNyQIC`M(RN5_LN&gP?fQC7kp5$(9|p?F*GJ+a@8d}I&GeVG zWzDrmaGd=E_M~YO?lYi)3|#aYNDEP2R5HawuJtwC1+IqWBr!x9*6f4Z-(JTZOZH-{$>_BQdwl zF~w0zmui#y_2f3QTPPjP2(ys%qpY5=7F4fL{Sb$2*Nl1iS(L44!rwKDgVsZ9l5RBR zr!p&4zhV7L2~}8qz-^y6W3>RO#&JmaSNAt|_L6i868?#3uT)|4akk=GJYQ6gE$mzy z{)F{`9g%TJI#4K%%}AdsViqqIdDkOx`p{M~D7+>oT`c@6qH))YSV7}v-a{zgB;8JH zgzL`_s(RhKb*XG=QfGW_x)P}N1Zifpd|pD=QUUZgs-GX}2izLU&wc%b=7*V7HGb?* zZ(gGN8rla8;>2fi%BP(uo+hHzIV-N_rlsmCLm!=73WcXYKbef zfs3Y7{cdKKUp^q=mHhF!6u)0-$Wy94z*fg2i1+9o!Wt&XN7;)zfk&z|3|%vhc{pD~ zYaqTkcR#4j zOx*2Ls3BbeHPY+JEB_70e1n-H_sm<(JkL6$4}T)@r+oS-4USy>%9Jj5E-8c`tCO|Q z*Qc@OZElNOB~I*c^-`qQ^Fg;CE6>M?%Yn3*rqjW2px%T=Go2w~KqtO^bS|UX!Yj+S z3+h+mr!Pw9X$rlDLfqss_PgM$*csQc3Xf>51@#Q<4&2Cyld3S9I07z|ouKE@w8SmV z@n%#8D7+*;oc4Egh~T5*$(vHP0qdF_CfgOKUXZb-+lGaA)9zmSO2>T&? z^&du>u*$b8O}ADD@Mdsx)^}0|_QW+EmEN$=?JUWwVs*NQYmF!bT1RO4kIvEj@f64p z%E+JR8Tl#7ChTx|wa$uXd0ziJxZk~zpdLWQ(TYdsubPc&9di%dA>K*=HGlM3j5HCu zF<_XWc?RMh(ievX=|mjkwHv9g#1mH!r5|8lzy{?h(CT)ksMhWpJ&)ZNj^-Ii^4>iA zcM;k;dg3EDEoxTe5k@-;s`V;O#`~G!+1B+n>52E!^&VZrP3>zakI#oDpJ7HJJyj3z z=R5Nht|)!Qj9soO9>g@g`)uG5TYhYr0V*HCLkzy&%p#5LwWNZ)djatB*2=>_GJ(b_2!yDhn?G!D1<_zs&*J<7W8 z+gQb~1J;Zu?G~DT^~hb=ZTXZ1mzi_&^SJ7{1%IDzqO?Mq!bF8ht?w3Kr?U>q_otE9 z6y(1|bnZ`8vy8^7G6bZ#<0E)5(mb()L$5Q+M#w)feB*MSsdZ9%=HSFqI5=^G{_c-H~9JICt2K+eM&Pb zti-?X$8z$SjPQw5R-6`BP}!Rn1pvG8&SdH#88 z7_g>Pgzj#Ntmyz+S3_*HybB)sa8|Q@w72p(NS zJB_^MGeQ0gNrQs&e}rwI@Sf@k{=R=i|NlI6UY0A4EeiqSVJ+z#PIw|Imtl!?Z!pa% zTkAnjTPe*4CqwpV zsR#MwUeV}(dARadf_h$IBqY5)&5Bwy!wDWiN>_r?9!alV$X|S8lnV&T*2Gt{4XAj) zs`m#Z&cFl*13tI;93ai0>X!PkE08Y|0sES9^3lrUL&#q(Dh{mTIj{O$;VShAP_NPF z8FR{!NK^$v*2CjK81F%vm(v=Fq6S+T#e4Hgm2CuzZ!z2rRl`Y^|;c0m*0D;Jb_Ubf;|#ral`v2s-~#d2dgCuClx=)LN{5Z^iTU(HBRsI+4np_|~F$k1q zr)yAtm-g81SCv<)>-@L8UApRd-fxrme%m$e@UFAMgZA-NdTO!}&V*kPNPyfj}!xe@uy808Oua#}6rM0nQ&-AR{YuSNZkauGqfASyqkxZUb@S9p0d9@b^Q zLggLIi!ZBd5Tqd(VXd~=VhKur&^5BZkF#La2#J*cX;z-%oaSEncR@V@0f~sDMaYLw zWt0gkEa0Tg7=5mW^tr-hg&$N0H7JX`sOChHo>m%&^n9cAtnv%=Ofq2*Wvb_5ui)q= zW1!=vDISzNB586q)x5jPX~^3(k(8?P+T@9p-eMrVVm9->?n{myJabDwbGWf2FkvAVQ7zmxXX zO2@P-S8?vz{d{=904(2ahUd1egplMs{1p_6;}^U~(eDT@`uz!BFS3*-LrSq*&9*R* z?p=J~)&qPNXt;H1Gd97wTyi$gAa!lpue7&ZzQ z+d1+77X8F}|C)<|tH<#}!oN2 zsSC%-!gpKn(Ys8@KeKo9O7X=289hil1QEdU8k(W1YKRYvy-u~ zphY;odkwie(vdrnzYD$TDkCdmpnUfxcx~AdKb~>L8Ba=RO!qPPVk}M?@Eqpch?8Sm z&^?q(NAT#n_h8?g7}?^>Zx~!Yl3%#$#;=ZO!C!Riqs_Ch=0)BfJl1p>7EJyu%4sj3 zrPF6KAB&+dcWonn@Jkooc%qJ^HG%(n?t)K-bidZHQRuUPeiu$0Aq^tDWt+HKvg;Im zzT?`DWBST(J8ZW)@tBEBbY~8tdu{F7oiccjC-L4W1YtjlJkT`*pKS z_$ygQZqS_Ot7g`hw+~jwSsCGoaUko=TFOcmw8!d8OGNj$AJE$RE1e&hB2Hehl%D#w za-GQmoEq2$Q(|wT(cUNc_-7G&{BszqxSJ?Cw(pCyp4jAJM7q} zUsl|0B_}22z^BD}{A^Sj6oxs;sK0HQL+U;D@%>~q2AT3;wLGWxNhq7n&a6MASvZK! zlS?t*%&OlE8;LHae+|Hg_{AE;U=+p2J_AK3o zlke(+&P6A<`u$Q>O?J5AjTM&T=xo9lNbg~<4=&)I@g1X!UMAw4^JB1o#Wr}s+t z_2XHATP$2$7d8~Z@Bp*YpIf)^_ zw9ni5of0;N!>gprB5QJIC|h_y`z&iEzqG?zqjyt4BJszRmb>5wQP^>>b^OG(Y^vRP zK)nbnt@I&2^BeuNZNn8d^wYluRL6Wk^FF*;Y&^C#uOX+oCu+xukMQGgA+kw#p?1o0 znwKK{)}y)XS(vHnfqCl3u-9}3;KH*lp<}$R%-yhp)>2QtwmiY~FU*v~--l|}iGKF`JO^2!Z$0jS9^LDxj z+4P*kwsn*FecRG1t|H9E#yY#@&b%nPo)uzVqu=nXyMe51`JEA0$@1)S#dTO#iwU>- zllhohm*MG?hBEI|lH3=w3Um8kXWi!4l=(p6&k=rH!id-CwHn%}3t@%@MTeoFHhebhBLxhhEO*-Zdv-Y`yEk z=I<<=l++KdcZf&gO;)SdU(G%Zk-vO&-DJM`O9_ZL2Ta@#U|cKsXP_ZlxlPo7oL> z0m38F33cVC#X-{J+bA)qaWJahC!EH}t>@Tiq~9Ya^pk|wc(Ea1bgJW#x0pU@MN961tBVDh(Eu$gchZg+JAi=hK0)g#XPSRX@< zRBB98u7Sn5R6LZxiHk?_OU=rNXPyh;IFF5pPDaDeDKHva(ORBC2ghxA;_GSXa3vQn z_peLqw+(dmd{+1m3dfV$ZvpBN^zRfy-+xuC->N6KpKBwwvc_VfVXTOw^Aw+D{ej2| zTPVdt*kMBqJdAEBdhOgQh_A6m#UYrz*px4w{tl?tG41?USbSz~6<+Rw2+>trAMbTs zi^Do@6ogqC(odZF1s@GKF8n*$s{iC_`c2p6n-f=DM7RRaf9uJZ)cva8X%F+}pw`%F zTRwCh(H_=s4Z?@Nw&B!9y@7NM`&heyIK9$BT3c^nL7z|*d8b3uTb;4%_ZzCW8F2>T z>c8~k%O~L?qi(Qb^9%gasg8_!5dc)vqTQr}glR@n>7Wa`(~xji`yyaDKY3*v=`O(> zy*3Hram|X4H${xKq3TgoJw$WLWYjE}=H801Y!+8_z3NXHjPD3Qn5OZ*eMZcxeNfU^ z;cMVXG15{;zI0oP)E83mZDI{8PW7cNTj)UhIkn;rj+Uy%+2Oh2kdc^=-t%w3;*1Lz z;$S3{)*!8ky*fC9dq2u$T%u(6$=3vF3>f*2&fhwo%L?C_3&WwdY+uZOY1d1`G&8z; z@?IOe!2F^gc&Ec1anY%vIH*$#5@L5FX%beMTw2Njq%kWSB==iz` z)4Lar5e{1Lm|UA1SN5T^3XTEkZ~WZYg*)rq#B%pTushgC)~eSCNZX0yp>z%%%}vq9 zw|FJKm0aQx4okN<$hpr#Xx*ly_Z{f~kvCro;xA4Zg|lim;eG?{xxRfN5FcQRFDt}= zYZHj?C8{y)m`(Rj^Eq(uPJQ`%v6~E-@sKp=eqs3hnK-G_Lup}6{oZro*$Z;BuM zoDnbjzMY|PtZI$_CP%T{k$XUCRq7u$q*@nDwb>2y9b!Xo9Y~|IY?W45SmoDkkm4|* zbZWzp{=A;wR(Sv}3DSa$dzwP;@@Aq%{dUB|%h2MIktBSP%}==TF-5hY-oIn8GI=jp zOo$USSK!%xmEwL8Fk=kX?bA}t0S*Xrk)(y`9M`pc*|RilN@7FO4!_6`d}Y_>N5GSC zBRXGmI4+uToV`_OA9&CurVK*v& zPHj|qI@O}WNq)pxr@A+smWOKM#*^3xvgko-BlTjjrBQC9HqZ@4} zeS4G9yaCO*RNBPw+&WM?<)@K`lfM(BHyN!PuSvfj_#HYAJL>JkRW}3Ke{1`K{~mqX z7s*iZsObAcgM{xMYa-}(p81tHak`=O@Vx|m4jh7(2|qQ*z4X!GZwQj#fwU_<)qFsI z?M;Ymlc#hJB=)@~lrBAYHHax6QD<2nP`tH-a)an53F%aqa>ijNEol@m?=2v#S9c)4 zO}ySh%?)_ywPOE0OA@Wx*M;b17ny;^UDEYQ@({AY@-*^T9hmZ=+dr3ThX>?|m=}Kq zeZC;fmH(c8VSE4CNy1>H-orkgT+(Z`Lj}OtY|knlr}by6?W3{Qas#|wD)DoxKvX^W zwBf+C(3v;cdm|$#S+kCG$`ByW3MNCw1I-a0iwa@nLrBxFL*hKbt0FD&fKb}6=(HVI zp4WYSDNNXPki0{>mb47y+a{vo-PY?{@!)Z3jJQTvX5Jv*TLN=h(SDOJ`gK}8>oDqRj;R{jOj+AwNbT{!X8g?wKfVDp@`%DeCVNBdGesMIJwR-2xWRQ{;FO?*{O zqI{Opu8Dnnv0)ue$OFeP$~Yj?Q%_Rui0g$O@afP*AYP(9-i$e6G*h)f-kz=cGXYyU z_7t|9&hS0BK-Am%3F_QGjpXCx$^8!nH&N|0D%2fTf{Sz{G> z60#;|Ao(rS8Rv;Je{*3wn=;x(mW*&saR4JfgOz*QK)b7^yngQwkz^L}v zyrN6mH&r(U_ ztcGlVQyUoYZYT0p3<*Oed0UOjYHnriV5AW#kFex~11h5-jp7B*-=ATmRY)U^1k$r| z;xBKwGs;xf>J@;bK_qDo(z!*<$aIfJ^`M%c-I0gk)ATc}W4<3#V6h_5Y)@*j|Mo4#ya)+ z9#)7|uWLM0aXcThu!Y>ihCuq<3I5b$O%v6M28zlr%GkZVH1?%`5p1h<|kgdzT+Um7&cxEM&(n(H{559ufDhM?K^B z(4#>Q7`1bWC~V*ldHTH>X(@%dI5c}Ryq=bcv_^!B7d1<74COQ*K-VUI7u(|d!?83s z!K8tHgR_4y%E_BxwD(s*>nE&Jk1*12lxv=Z)%Nw^(0OyE-+<-cS9f#LH>qQET@m2RY3l`D^Usoar$|8R$?d`jh!%kRE|0DAsKzs*4;-vUZ2DvuG^ z%YwY@EF_HcP#TeHB7MF64SZYc1FjesLSD0`pq!QXN>A2JuvBZvz6RbSJiUVR2J1p` zwt*yH%*nT?%#b+pgGTjcvhR9~eGm;DnwQqAVd8sScTy!#A9B!w%2;yTG?25rT9B7_^>B`Z5s+Yk>5QXm>RXNOuztdjsWK zc=Jbl@*anoZQE2(_0_|7F{mC@xsl37-gocBsoscVIbR&K1&uT73Bn8@Y-hW2hXd&i z>48Y`!{z;SCjGs;k_vJvGM2770!IaZ#7JHu*~K{Luk_|X-1I5GMqX`)+# zd>@Q5IIUfGRLiWs)IrixbWhb{p*Tcc^Sr@pQRR6<4*0V1-8T!uOvnYC;#05%!qw?Y@<{?P=gCbdPd{XWPQX!Qiv_#OouNR`*0JXE*sQwFu*_Z{pFhU%}>M zF}&Ow4UrC3eAL@*kdpL}ZJ84zQl@zctxcXVI;+or2H(?^x0}zqtvH82S#N23y*~dD z{S)c>xzna0%&eO$EUC;8jrzo5-td#+RIhSecz-4jj~a#NrwqfItsBb#+v6})*A{xw z@6+v~8%j;;dYm*QLbIggn>IYHiTqXfFS~kpKgQikXGc6*%A_sEoc_PY^;nkHf9xn; z+j+;QqgK=lO8|+jH?ZGptmyB-IQYLp_ zixH7iaA&L|^d1rf?FMX;-tYnTghgW3gGdSWQ^TZIp0`MACgS!}h0Q<^%oUV(h zzuyjP2~i_9$Pla{W+dE|FYg^^c{Liy7N?rwlQH#0;;1le-f*dQS+NdOf4LTEj-*r4 zUe@*QO9-Pq$`;;#i;dc@kep=U&}< zJs2YI1!*?a^^!@&HkjnvUpm--7l0^T}}h{=`(;_iBM_H$=dRJ`?1f zA>E~I;CfAuJM*zD%?o?)-7dcxU1!^iH_PZ1V=#9{f4bH;Vr@Y@jYp49P0Z0o^tmlU zoo9fv=?K^`-;itG`-4kMcg>4~9kH!n3trytGtJ`RK3mAj62lbPDHVq<@aN1KJqeM>6XeS<9RnX?6&DU;HnV@?C1dH<1~ zk?esAJL-A|$h{YXMPoM|Ve8lr5*`He7E`wK!Rdd+yYSXXH~>LLR{TpVOIfYF4CB7) zaqGve)jGo!-F*B!Bp-U&MquH~F|2`C9Zd*s#;m4If$Eztjo(bY<%P6{sQI@WmiM4% zxbFC(=L|D!vwk}B>_O)rvDR|1M=(ZOlnaGRE{1clG^kk2zkU(v_00DAZbq1g7wbL; zHhZzGpFWxD=LYn-Fjh=7T>yi3oI{se2KZZh6DR!Yjdj3>wYA=bPd`X;BuK)qo$GK) z!FEnhQea%@QQkmT52;W1ew$1=FxX65KR$_3@iy>givto~&{-2d1l1&udfSQ<-lcjE zu;zMA`by@uS!{M(!?XU~2Y1~F7=Gw2T+AH|hnnkR(;jZ3LDUhu&2u@up-6ossHb=*cRjR< z-pX%OzJQCtU3fiBDQjPCIqF=m&hHlVfs^-Y!g>~kAHzDynG5E!y+7U&Rv18;e;U5& zxs_$$QLTC5Q&9D9pJKpOJzt0(2U9i#Va?~&vG%})a`tCB=kE>On_BFo@CFI<;kVUB z*cS0gsOx_1I#oWs{uTNs^p!RreHBLF@KoA+Zc?kNYj{*@On5&=T042lUQ_I45alHO zT3*3OhaE`Q2K0TBdJw*a4v^td_wmc(L(pcoB~Q9K4_761!>_yFc#zov)jRfSoeu6^_OLi5k+4}0X$@qn z3xgD1@~wcZl_@HhArh#Yd$ZV58Gl zq`E@-EKoe;VE;t@9q%j+$NtubS;vo);lO|kX#B^Duin>0`b2qx-GR|`%^&a=omIAD zMtj(|!InGyYArMS`@*bXYt@I+f%ccN7(5+&4qeJ=zHqzwcW9m1PMRf|OY@ESte>2Y z3UiFku9Kx6SE-*I}Y- zIY95{M?P6|fB$OY$Aqn5ICCmJQ)7rH??oW3C;lqkLHzBnK8x04Ak9-gcF+%2ngOY% znEu~?XnWpYDz4pd?1SPO`FZa|JpI-~9?HGKh+A+`Ksw~acL16@2=mx4=ocW(mm zlA!q)>8;-*@fF(lN?@D(Ysp6c!a+a4Ks=bB&$l!U6Zia#A3G=o`zKu2{`qtP{`2oi z{n3N3z3>B<(EXyO!-jx`O(1g%+6Y&g)t3XuUt+#Z=E27PT%&N8bQdFiC-y9R1^?y@ zf>7r-V)4jNu*l7czmCtOJps=P!>b1%dc{K=;7rf&{&*)kzcr9I98Rk`L$hCTN@sAV zX&d3ZE$wYYa}{b}2NC}~ahdNQ%#K`xdQGll-^2{4`JDEB+pdd*=jgXA7q%DIpsY3& z!g4M_a(Hdl^v?l2mev*&&eK84@`KBN;IgPMIIP+SelMHLO#Plhn^*y)EzxAf-cBPd z2Fqzr={e#?SAnoWkVavb9~nx*4Fd^O@U40Er^$5nr&Fkd$P;{Z7$9AI3# zwwi+#n~C?TWXC)Fd?$N?9RoQKqRVoAwj z;)?dEN@&RwJSV@!1Cn0-Ov>FdnpGi6y#y5)8WId=wcy$tI zF+;Se6OUKNglq1%u#l-;tHa0xi$xRjZVHEF=aL7q-=g72+Cj*KN<8k~5eKISpwl!H z{<9(l9_bHJIH~<)k^xhVHb|p4<=RbkiWq4-2;AF7Hh*cZRhl8hZ!qD8jwI|ss&mqn z=A1AP1~?B?^~3^)#ITa56;*RO;a3*zpS@g;2xz7B8Sn9OFLQjo4#(IT;`^i}nhKlg z+L$57;mnK+tkc7uqOQ27`X9<3)t1jkHRTuSnX|>SY`DL!v(O!V2Af^h;OY3Ic;xY1 zJUiS@Sd|^X2D(8&bLvt0=`6%N&m}GLTKo(iiPN5DFx^9!sn;z;Z_g3pj!8SE3q)L~ z6Kps(7g|qhT!n9>PtvItLHAI1c#-vwvHOGI<*YoR#?Wm4d`=i734efa2Z)OV^)w8h zqX(;9`tYLXuK-*Aq&KA1j`$nww+(jiklk*KEkxTW@ep<<85d0DdfIxz$%-ZtSI^-mBU7;$>NP#jj) z>2myys(C84r}?mK9bHSouNnm?LLs6B(3%wei+=r zYnCgpPJ)Z1bw=t(p)jC<*LI}^Shp2!K%R{Jq_1{SRzIwEAP<%HT_5ZyN!!9i(+W76 zGmsHy3+1&47ZhKLNBV^x6E?kL>spTE%&ms@k@f<#rhV8~7HtChK2H7+h?DW91G4JN z=!_@QLW3V?v8z#w6rL)b2ZSqJKPwvdPRT|!C&aH(X;l5cYsAQu#$5S5fU3B;o8uHg%AV>Tl8l$)LDSX(zg7 zoZPw>dRV68MB5mVFnSH?)jOatUtxN67uwO0c$TE)`)a>L%e_YA`E}qPchM@YSJzJemVRr+cf!p(g#9a- zPt-;H6Z9MuR}UQD4V^QuVsXZCcu?WQNy{>&yNTNc`BLJzBUPTKlj8_b{^jnZ_efrX z{7)f%-uo1zi>&#gtQ=NjiV<;_WXIBylx9WUHNO>paq<|<_uYC>T$n?@seQV5f>A9Y z@8!<&t{8J#6RGk8npZ|R1m}EO!pySk=-95VB-|4cR9d~|#<2+60R8r(QwsjGLf|>`dcY2vkEr z{EX@~);k)|c@ay1u!60x(BWGwCn!%@)!P-{8{q-8p}&=}lCLqOf4czXB0M|tACfL$ z*DQaN9^4`b%UIxZPp0yZ@za`+E^!y{lfH}B@m(3^HDcKeM+{FpNItrmRQ#oInQ$Gx zTQ4L}UjmfhP!`dc>Y}Bj+Q+^&dyxDUTN+%AK;re0N*&dYe>6z?7%nd$Ag-o+3?8 z?+F9j^B5GotwH);d^YDYs+uD%Q{EXVC!jp278+i4 zATPgBX=IgyXm^`c=agZ1bj;Z+25Ej(c{9?ToH8Er1LQBwmk6g#g+ROn3l9bX%?~@~ zRRqHV?WM|nKAhH-cTCP;#rOT}g_0Ad z^1^XrNj#u*Ui!4%t>kUne5fr42<46b9;UrriX4IbIQ$LtC){*Y-U%)9->Ym-=)NDO zw7)z%JWfM6jwRa#A}ohYE|xZB~3EL@hgkWB;g#VK4wMVBcWTn zCOlLBs;2AaJFxC)4$vGZyl1UD4+rvP+7CZ#bHY_z<5@u**o#$;IU;Dz1!Z1B`5yI| zzsK!mj7@}tOR;t5yP!UcGGnA#6_jTJ@j1*5YC_qAh5A0WGxo3`PbvtD)0IEa z{HtG;VNz}YgoWgR21sG#BBzJe;I5zVq0+h3Q$X3CAk7RNKRy>T45E}rq5RZLlHP!u zTXqnK%n>Rdiy7h!QLbH-Hxl7qePHu-Z=SKxO6gsV%7>^&kn$I%G*@)EAt(I~^uK7+ z=`E|Qy&YC1T|$-75Y7SNu+p_E6F{mtJY7>?My)DfDvKd)hTk~i5IXm6@0326-X)rm z4wsaNP~ZErh8@2tTq2+UoIaDDZR$^FzJvyPNb8ZjZ(?VlwGouTKbNZDxxW$=q(=#lYM1G|WCbY$S(Qz#aP;r$j{LCqO)qVU%}&N>Cw z;b(e$7sR{T@k!?h<88_F)>8VP_EVzgE1H_7sWFnSQ<<00i#$X=^(wf;-vk%hL-2g? zKK{L2PhOtg29o|QVGAyPK)apaVdUe(@U-3#eyQXF_%2PvkRDTT-%De$YE6Bp8Q~(W zEqddH{Y-XTH9&MrZ^uubN(PhDR_VSDqge7%bBqmd#J9c91johe#pCvIa4#bo)4Ntw zy>|Ojx?eAMhCFH0Qzjj6AY0)c7?Iis3I-l!{Ui2*-ksWfRKsq}FnkbXyKTgkiRLiI zWd_U2tBG$O)ZzCklX1yX2RffkK-%SWbe|HzXEg}~!^M&KujB;uHlX`X#3{U9mWur~ z*I~iT+Op80JwCV8$ZwY&d4AziO~A1=u>Yrv{O(&vW-NIu#`<1kZZnYgvs35E9ez*2Z*mdL zJJlQVjz^-6!(6%L>MA*Ty^Y*f5QGiPTp%kif_?Tf;5JJ=Wt$q~py8P-Nb?0L-+J-a z$?iB<{}AhEIU6cA+Vcqgt&n@ykZ=Dz9rTYj;-3yLlbt^rLKU@_5)87l5w%r#zmNoCA-g9pO->DgW%(14nKg%GfMHWY zu)F(P=4$gEyR2y~1Kladel&|4K5@Y8w0pSq?jB6oG*>3Qrr&1AKR~Jz5qof~?B;DJ zwHfhPmNbIDZ0e_p%h5vlNN>?1^Q>soq>gl^{T1vy_i6{YwG&;+Gc?n61R9y>u&$9l za)em}oVvR!@9Y@~5#16czw8Ib`+u;$I#-eEhAkiC13J1Jd3@i6xT?Gb`J%OQ!Ns}K zG{}y7H9x?*w0MBf^Eiqj7JPLE-J4c9SZ3Jii{_^{3IE=Cg6d1w%*zpe-aDmZ&P0eC ze;m)d*A$IUW<@5>*8 z`~D~Cy5nm8-Z;@#WTX-jm6Z_EJsY< zn|MWRUui$y3mqP;l`&4kp|$x2G5&BheoKEcSgah$H>Qlkj_sRc#1T9Bs>M@QC$p9u z>(httd-h8#4avvTkAhLve$S_-a&^)_@wUS!SW`4c4DqW_rU1hP312;ed>#*G9}=cq~C+cVjXrY&<)E97DLyrR@g6Pfuw6?9x+#-<%h#E zW8oS)=l2gTq*^0q$mqV6CEjfIOu54=I#-Bx1m-eR) z!m0k2vQI~ER{Y;h%<@TMOFJ!r|9(b5!MM3F(m0lf#jv_&rF9!&z&V*Cysh#$NUFQ)xK+B$@`^z!C!X4xU=!m zs`~af>CZRS9f`LuIPnKAUD?&ZQ>qqm@8s@C^$lq?^e|%BP{Hr*1nLz$AJd#4|Mwl& zh9&XG^O4bKP%ST!lW50q_lRzUTHi?6PtTB|2r3+bGMlmb!gIv=Pf`SByMkmJ&Wn?3mgM!KJ| zNMwO+C7o-!tXMp}V$Z_cZ-8wxL-6;NI4zBhG}4`qXT~>`{_$P;&m#wKd~z{S`B}n#qo|SMK3N;FB8}a#iD^QX-9!EdFnpjBQPO{cfzl zo$1yh?{`gZXVaN5dK|w$nKE&zQ>tJ7Ib#PX%%Of2T_S60|D1kBbA1v_UsQ)?^%CKX zktet~ti-(EL1=7VTlV!IA*)^ZhT-jXf#%yBF|@%I zG{F`!ZSGF4=6i_8NPZ`6na20>TzIKzC{yEV$+clEk#L7_RF@O(Gxx(r81=jr^+_;P zU0)c0{rF<289~n;OWLKr1hh0nwzMuM@+iN zuVdQEHAN4q>Ot|L(T8-nk+C1C4q1M{AsEJ!@VCoR_kipG}mC+wFOuB zdpl@5oR2vO<10r)Ug8P78ImGmf)0bzW)mDCo?zgWTrKGUIq~3p`MjHx+|@f2(pM&m z`bS?f8Y`@3(O14QvBgJUYbb1__1vNOaq1Yx(!s^^<1mx^?Kd)s7J-FooBzLw#|5 za2<91xZlV}>zjTY2x}ncN_*j}x1RrkJTr{-^J*OX+wWmPhwEZ|C zqCZd{;$qt&;tXl+0l!lj)dah;u@K7)HshQ)8^tw@a2W|J;QZZggptF6=9Up=vc`H1 zk>(7Q&QQIk-ovUe(v@@>p408DZ83RfIBd-`DWbUn5gZ#<(p_Z|7UjV;mtHk zSL|F+O*RP{3{CFE!=8u?JkE_|kwbxNPnVfhIG0trR7>0?Nmt>{1#59@kG4vu@$So` z!EEzRHOA~`#4c@Xb6py&u(B>` zI|p8anF#AguChyHkhmOg#XJ9PfGs>*bA!Itn9;jBCw@WdQMPXHbugg4ZG%39fYK+v z>Fx0tJ==6;<3PMTu&+?PM*2}pKU4k7QchmM&DRd%RLA!4u}*cU-Fz<=UMwRXPvqO$ zF2+}Wt0eV-(t~Wg_fc?YvJKfG=!6gj3th}aC zJ@i+%8TAio*!l~hPG%VU;BXfI=GBz>O^31-Iy12q?P;~A=!WJ=%;?U{dkmiRG4;Q95l;l`365!})g74~`` zEr!@zGpo4Lpl_UpFbcaz+@!J37Dq=-6ihEXoAe}_tlET6KL$X*HyhykN^6CQaPas$ zSobwvxH$ZQyqDL&tn)<-v}vfgOT>3PB?{u!!D0EE9q=_HJvADvL#BaQlZ~LT;ls*E zOl(tvW!nccrLS>?&Rcom+*;AsQ3IuRW-_>&r=amB&KA_O$HlVV z4XSjFs@=;!7Rl?|?+dzLuCO_AnvHbmI7TeV$id9mF}N(1G(x)Az{;4z|-NFBj{!?{Nilbeo)CQ&9#+IK_TAU?! z-Yy6mFu7S;NKkdY%uoKWT4Cwyvkv8)puC|B&YI`xNOSnbrqJ5rA@)~C1zuLtx z^Y;Pm+l}X4mj|U2hPI=_Ec9e#S$7t&QVZ(-O>pnY4v9c24Ytd=11^K>SDg z$GP2r>PQf-C{4xQT6KVc;##OQc93p9R(C%HOGEl$@4f~+`H4G{r_iYWuyzj9=>B}F zGz(Cl$Qh`ouoV8Ad{7XsLDwydM316Rq$}nF^*5C4e~XsC^!b=n3rY8ZWlj6Zi9dU4 zX^d6v0{J9dG`P3eLC^3IcQd7Llwd0NrT=L@pzpaPQ9U&Sv&+rp{h}P~e57Z#;_iL4 zcY8(bWufNUaqA)S=r3K#EAxTZTreX&5+^J-M$$~$Tly~)j|nv|^*sF5{DZCcdE(HG zjOItI{h6&aAQCo{&l(M@U)81i4#Jn$bTNA3B*kM{#Lsxodn5d~RR}iamsw6&eVJI& z0d}X>lm+8HviY|Yn9q{7SZ9)!u6sBT?*QQ@bWKPhObF6c1cfM%1w!6FF~ySSxSVIyTQKY8JLnhXEa|>w$u|_Os|^P7gh*ONTwfWdZEm_n zQ@#9#ApeZIo~K|`NC`Cc@nui_TfnSqY3THQFMD$Qu2A(ZBHBvQ3F6bCpEN(Uc-^tK zX%@{# z(T?>wRvjDu-6xFwXOizY24My*MYL@np|Xojzb&f#p~iU~P)`HZxzevp@iBQcB>#(< zLA3XMh8v?E2c?r=E~B-3d@W47`w++*fz28h_O0{kG4`P0*NW?~V0oM*~Uhjd2$@uS7~%KNDNfxXch z>|t6HNfW}ZPhF9;LY1EFHQqyz&!nvC4N#szW4;AdPf$H$p4SB8^L?UuHl5MZW0x4$ zx>)$wmSXDtB%q!XPkLR#X$_Yv9WTfSRpm%_VR;(*d{!?m2cCC3rXj5==ihDtJ03;g zmWzKV=X!{QQAl18!sB)TjTQTMw++NqUJ|Mnqcb`o)hTA{jF9B>@bCPNIK7(>eXpOU zrAz{cwxNAw9nKPVmcapET6@0b8&0}%O$^T8CrGpLiIaybOc5$eQLkT9`I1F9jC6;a3SSxFtEvgDLfS9}<5kPe7QFPT1*4{>Tf757DKL zRNN1grU}x{;PIic)N}dE=2F~4eT3Ai1+qUp@3F4*ruvl(SakWc3}@gO?eZY=Fd1~1T3R0#5P3SSj| zaOHuPE-)f(TN9q8xTErx>b{$NnjlE0A$b^02Lsx_tmqW$8HQRKW2iqjpY_Re$2VVw zg6fM)%cSx~3Ul$TRl3s6;=E%djNLIw`PeG${k)_xsL#Gv|0^inGIVBRRQmTA?Jc2v z3-K!Xu)8?-J|GXeh%ctqlJqmF@}x(5(`lVyGliFudKIaU#qi=C;;;1+AWuv+U?w*> z+B4GBcWrD?p9IQos`S0mFLzrXfuEJFs{HiL5ZbpPGDzWdHf0t-9+azm zm1>4Ey{*NE0#)@*vMtoikE7HZoC`I z0+=T~Wn^pFxGDIzL94`1v&w5YMRlb8V*AG?oWf#^5DDx5Od~Mb9l$%o? z)L(fsJp1qtSUoR>ZMUsZW#3fqe0!aJNO>!UB<4Z8Yv;7&<5eyviK_*TfyfWAqelp? zl3()G4*P2?Dn9LoiYrGUenq@!_!Dc@m;DR>Gaq)pJF!}HTSmAt|-J0lwIoEz*;e;RHzPKf98T%R8zjAO*Y%B9l zPUFMyBsRQsh-uEpiR18-^_^57FRrT3r~f`L9(wk~r%AQt(VwYU`YIHb<|MKjo6Pz0 zCeOrKlbbNM=s)_IJuk3sD_itRM_hkg^joqC;sflsj}F!JREfneQ(%6LrtlzhGI#FP5qMlkN1mh24Yp zBaHzT(DNCSW0teaF9y)A={ln4Z9iVu=OuQcwaRDm4Y22n`*8YXCrzK`)Dy-}h5e2S zwtsv&I?ndz7Z>lscfUL0d&3kKnb#TrT&yc!nD>;A=N*&N4Vm2h^dc^f)d(6x_U+pa zSZE(AwWBVxZw7y9@7v#E`FA~Tns*UT9YU~q^8(UGPS@zBtmE^IVqu5XMld;(uMK=X zQQB=%BX!0uP zzq+QZd_NmzYW9iLOP7U5p9Fk*%vd(ry9ZwfeaHSA)?!{v5bwY5xk#*aj%AkD$NsB> zu%y)zadq=!@%Dl@+}(dt`)HDjoalN6V=qPXCfg3dxZK)s>2?W>Y`7NNt(c78%)f&n z*uoL-7_p&WYaIT#Y4+s9U+6n;PkC#4iN=V|l~wcG^41WU)a3|l+_+3!I`>!;ts4oe zm$l=wcb7s8?X~{euc3r@LDDH?j2w_ZfcN@RB9=Xk#NG|urTG(S~S!Dg9%==g)b#3@_&~04UsRp;XOy~Uij;4M5+Tr15oz?rnR

ox(`utBN^#41=XW4Kii7an6WYO&E<@SCt=c{liI&^N@1}1ek_~0 zU7E2)n5D1Fb*3aUx(_h&+JUd!+en*pA!uOJQSS2Zr-64hIn_U3n{Fzb9*P0#Ax787 z-4`dpGr1m$?>O@x?&GrlJYK}RNA9Bc{02jO%h}DLj|Gio*U8~qK|89iq?+dIo1V}1 zo#dfvS;l>whr9PU@>7QZBCmFU(l6hk%d{Z~jUHh9)HDpacnj1#r>qR5&-_o+oAMCt z*PF;I9rlYM0~gAB#XGT}c@hjSNWrmkyCzG2gEYyy3W>VC(XewO#?NjgqwPNfn7p96 zZUZ@@j*Lj}ExHlXOEHhZ$e(&+w%SZTYeh(~v+=IV+uaRRj8nRD=*W-D& zEVvWNc>C18{P8>|VLaXx;%6UWst@cYG?NW>oTJ3~7Au_dk-kHY2AW$*bpXffHe2ebxlpH86aKD(_ClU9l^;yqfZ6=6IDM`yWWLy?dLLFVI05YQSD^lvfoJq2^*>&n z6a-~UL)qk|jrjMDQzg}+%(&b#`>ftbJgq$r11E(`T*`jU%o7Xkjb-_(9Ig5tReKLQlrYt=Yqr=zxI+{Ep!FSCH8uw)cDw)$ zuf;?1*6E;2`yACw*eLsSI1XD!uERI`r^2!_Q%U!X2M>j)>w^c)@6+0!YW(MzL0rwn zX>EJC-l7iBGizwGjXgrudqGv{__Ta|X=F;YR zAkz3yjqK!4^XKrb%btQsdNIs7HVD_zdNx(-3S$(8_FMlIGj*53$I)}-@+One$It-p zzYk{A%bdm#?-evsy?}AI!qIEGA2{|lJYt+oG z2+{~hJdZZ_-(l(F+H#FthKiR1X5I$sck0&&_|Rk@bhu3CICa^G^PY4>!WB)$6I(cD z(oEv!R8V-a*TDjT*Wy zRQ;~-&8po)QB;(vX+dj)lP8z~@raD>^oA+!ozTVu*F-k~su83f!q;OOp?=LC;BexF zxY@B9#-3=VxEh9?c42h?(5YnwTU!F?-vcKoKPSA@nrAy!adoi&4%}Rtk4rDw z!m7zy@srl`e|x%d8*~SIOBsG;!w$plpyCb-n<0`xplPPmfMPc7bQ5bo~6*g_v zktGHe(r@NLRJwwC5SD-6$Vqq6_e~Sz)k!tklmj(zz0oz1)vOF3FG+Frn*JQtW^Klr zneRp1H8ZJioZ~vUg$3`RJ&PN997W=5>@wiF9eKP|-4YI8At zWEtAl@&dvQAnifgVM7&G8gF?CqgQ7tUe#W@d|wcLs{7T34_=1f|3tDY9c(yh0Z!a2 zGk&aL)k#(?96kw1J2J%$2a=aU)SGxgnjWbKRV@j^JJ-oE|B=RTCWu!VVT?wsF}n{d zjg0EwsYd7-Js|y&CY~m3g{CJXs1|lr-Sb1w3%GFjGx1b>N1r-D&~N=hS2ZrnD|O_d zPern>lMyp(775FAzM$f|j358BSetqdpV&hBDGI++|T;d37`8%vyKe>TNG1 zp2X@tCqT_3@sd3IbG&T4&Jm+NHCA<-r7(arog{rOI$ZAre=bJi%@&(L)ytVX)d&}h zfb+Gm1D@2Nh;@PhBDE{5>SVWU}k;oHWQ8 z*h}Sq4zUsM8*}zgE#Z^-AIMoav_wFrHAAsih z@2YXW7_?2J`0mh~Velu@ir0SA0qy2k%GRIH62>4H_1_J{iya{K#4s8sb8<O;)zl zSi3meA9r&p6r{mryhVG$pu>F23 z-ednZp|p$QzPfy!mVBD*q5aAnHiu}*gTdU~^O|AkEP7QNNO&6{^YybB)g2pp_zdB= z3%e8A4aOVXXJd9daPme#HIJ30dc6MfVx%!f`$f@sAjDFT7vM9R9Am2YJl|!Cp+^N! z-KzQk>IH>wFy-`2QC?iY#%xcazH6o-O@JRqf2h(&>&^NDVX1~VRA~`Rtks=UZ)0>V zQ=U6Fj`U}HKB3Je7}@PU{ytzY5}u6&!Wv0^h1sUn@pvmUM!E@RIyXnUXAE9gSKHi) ztNy}a7o3p%GNbDRs#!)DNZNTW{%hMGl}=YZqWo>cOLodTDUQIg$6^KbD^}~_hV;4G zv|H{tqlptI>__KcaZtCVDd~t#?De4Lbd6gusY@$x9AAMnZ}k61kT_BCCfe@^KiQPl3CDjsEzJCt`i!KrOZ-i&G?wa~R zJpbf0*YHWz_SsRn|?DOrWyb%TtN2@%de&xj~KRh(%688FaoqER&ZLU}6&gBV6oDQwt(w^7E zd&e8aRPi`@FQA&K()`4Ec<(|}xD(?d$jd1_V3fg7pUqJ?foZqSqFJ-LK=TLWQ`vl@ zMw~Pn)#RqEY^MqI9zT)vgM1pl9SC!9aARvpxcI|geRm0nV5r=!!%XACuU9h4=HER5Qic8bMpM- zk#Zbu*N7}oI^$V}J}>m#285G9dPYp%xSc7^Y3ua|hjgr?^c`&5Li>@H>VbY`@2Xys z5iOB0E}L>PHn7M;IQEX9tZsx1Gj6B)j5Ou}R#|AO>AUDRBc6h9AFMceFPbkubXl1z z_O`pK{owgsH1CoPZ5p*g>Iw4PBPC%nx~8>M_>C41Vgz|0sQoDo)%!SXHblxkfOHb7 ze1~ufT3#E5cOnI*p17&9Lm)3k7?guIrP;JmTbvK~v zW?;G{55=Pyw6?kDUYONA7Kl5M{1RHv&KC6pJu&lP95|P^;Oc$G8;r)Bv|XZ+SHAer zZM8PQuN26Wk+;28m9fz{@_A2AU=p{ZypYbSG>O{aF@fd=#B_4#S%r*FulW<&;NkK+;G1$dara?ag^3vk(Qyrq z`5W><*@OvITW{^tzo{Xa3%HrbSwU}3jiNV*Ufttp`Ml&_=Cl-8PW-^}@RLxGfIQ*M*PD32gK zoT26x4SP1>lz;Q0^WE^i=`^4*g4nZ`t|tQS;w|6%NLmgl_o({(dj9tq<+KW;$bU?i z|HrR5^a9YWwk@;&G!IgoX+Q2wj_~624UFD>9_P9zs{e-r9uu%t39aqwxEHr% z{ez)vw&Re}vAD-*C?56bjwPeMWpAn##Y4O8)Gpun4$CuY%7>|zj9vpFD;D60;tg=} zd@4ltErOK#IZ#q>8&umdOh(_ekwe4>mh$ZyJ7lcOyI!XK(%RRNP5;C}>zt;t@s~)h zKCf>GYW#GjVPtL?78ESPdYauVtH}{~W?(MsX{?0Su$K(2o(;V%?nD28JEF(h8ZvI| z4m><%wf6R#-Z-r&nqFgtrf1q|zxEE|r^^qaX=N?GC^1!dtm)`_jroX9(ZS$&j`D5gE=W-Z#@gAERnW;^@)1JT1e}#el-06&H13v5caDF9d8!uZQ$V0M+fP2?i zaQE3PO&{%nEuB!l_<9^44jw4`biax3*0rZxI|FB#G>4XpGvVXNS2)hr0gqKws(E2< z?IJ|^QhTi*28v&UIJgWPgloEWk-twziP>GO#lHEirGef{ZQ<8B+{VrvY5bV)!f6`1 zE}SrPt0wmROjdHM12amo!o-*xV%)`1T63o}{Dz?(zUy@ZzYR8pd^*RYWM>I`ZQqGC zD%*sPBbJC8h8qDcWecY@pI}e9pQL}ogrhFJ6Rv`<_6=mP_cAb_-57lrz7{v$?qIpJ z4nClCEx#Q92h~_a_8lV6v+r0^uMUr2aY|J0P)#1$@&g@jIP)$!)nv$uFSzVy6C9RQ zOWqyQSbMiaI~o0BD@-uyA;;CoVjHJD5`8aOsn3Lq;nM}pA@+M^$X9nxg?E1EV0M={ zKCVVrsM$LTcgM`9_g=!ouD-yBa~OUouwb=D`e-lIug0_fxx&mEmU8C>ArimuV>4g0 z;!*E6;FP>jS=hl7v{NG>awDB*5?%)FFOAHmdc}1O{Ar9c&|+j)2y9&pZ7(0;FFI_M z?LHrX=6hE|a^Jepp=Sf=Si3unF6zWfD)v)7`fwU|xou?vl&!ChKVQGZx1(ILukA{J zuVu8y`kP=?vusnF^>TegOL;!CBfPBCf}hT7&^hDDaH|ntrgg_DhplDZoUv#g7AEN5 zaQM&P%-u4zYW(!(m`M5^QCN4Jq}sysx8vd6y5o3y?hF`szny${)|68{piM7bscUu? zuh6rP!)CUVhtCfXSxs8WM*ah3a^@$X{@|e(d|_(3JsbUz)?F^_D^D9m;}|iRH~&3A zZn(D%rvKW)9;U{CxHXsW?c0IV&%`0)4Pfrn4Yv+$quxU<*J&k3JL>SDu{ZJaqtq-3 zmk29Lkg!G4e6ga*MXFYKr~mGw)tRmQUAUF6^jv6 zV_Lt}2Zh41Q`L;Q?%*b>f25a73O;QAP$W+a#8EFTd3fuNvW`7@KfONk*&BPv*!8Rm ztIpqUM%d;k0#-S554jC|_RzV%9^JF^m+NtF+df*)8GoPwtwH|%`xpG)aR>;<_#KA? zrs{;P@si(j{3VU0e6ZdIEKci68$Ac!eD7a8aVLUjj|t|~m&~u(ZPzP(I?Cy#O(oS1 zf9|jWs9v#rXS6t%mW1O!+`!O;awPnL{Knag=7LlI@b*bFxDVR!m2GV0wcG2k@u^No zxPd31#^T_Q=V8Kt{ZL1Lr`SF}A3H_s#swibD`F})i=K>&f;m4vPYZ2d zTgYv(+rebgM1`O5+(B15d+w7)VQpY{r#++z`ZCqq>le7HIb$o2y@V6r*FxTvJh1=t z2|}Os<7q3RMgC~o-?%=VS8B9C^fdM2o3{Cg2(1oR*hctET=7X*26mvm+2ZitnctA< zY>J}ZHB@8O*lZm)>A8UWG+P9ZcXr|}mKez9zpvmkgB{q(rinX+yi92oRxrpDt zs4wC${%&>otiVMECYs1G7L6hI-){Vn(uyliYw|En%_&o3n%iVEj@W*f+ia%Xq@)M` z-bh1SV5i<+qv~_^@K8xuje)^U_=pqc>UH>gqz7?dO|IsBb7vDip)EaoGJTMA2(@Nk z^%Di%H}%+2=zD3mcJBCSAl?mu-uLkq5%FiHBYZZ2JK)Yr1# zCP)~EPOip0yk9&^ThV~O)2o1;8y@1jvs=XJv`gw*z`t!>*t)<qnx>B8 zb=rHzP+ay3XT76#;>?R0BweH-j6mW|Z2Nkbn7WYmka$<3S$6LjoH@9cDQqXaR~(Do zJgdn@$;q&}b2!rVYx%2|@Z#n zK=sS7K@xAqk0l*6n%iCaLHO_rmvs6MCIwjH-MVMcEnqR$+R<5QGg^k+OJqGM&@R_G z0*XtiKebK|IC|H(!}i_&i(J=24(hQH2AiHB{iX|FlSiV~!xjwmrhsS7Bo;n>n>PM# zpjcV!q|yO+Iew*d zkZ?tl*RsB}+aHJ>UR}b+qefI=c4eSDkOsgGlWJn%yDvaoj>{UF%7k|faN5TpPIHWk zGut@QncEAp8P%mm>C<_&ZeZQD|LAo=qHCw9$SDhaPO<3D(_`0PkkUr1xi zo)u&QVXycwdw|CEq@5&vC5Vfl+Sg{1uu!x3g^~OjUrj2Ea#}M_<9WCg=Nwv&O0V`D z=)~y$H38#tk@x^AABD5?%Wi6kn}uc15~!Wu6>2S^{CdG|G5qHWoH8Q^NvGiFGaTJM zozV2$Y{J`pwuO@iB|b~JtR=pMr)Sfk;qRI7^f9fUJ~9f%2Ct~n9W+Py$~sva-me*~ zvAG0C3WqbgM?AO1RlPTKS!)Z#x6JO+5|Mc4II21!{O5C?kJP-@zeu%iz^M;t{$2vj zGwV5PFM5rz;?KfHVNO;JoJ%=YXt#!nck!BanW$-R#|`~X0F9-Vyn`fM5XUN3;q=md z>b+Uq(|e-l=5VxZFdcs>BTNg5Zzo#IR;=w`j_O*^@46r?c9=^VV@Q4Sh7n%~@&uUY zQA6%&bsmY^@$TF(Fr?>5Eqe|H@;iL0`3gL~^#~FV@!w16T;#ldu&t>N8uq%NjlR2| zc4n&7(EGzR`aWdSu{@YCvo>RWOECCBE}gM))AKy5AB7Rcc&oS z_yVJTC*9!6qsL5^q{UqqENsP@Jch0J9f#$I?vsAe;cMe+Xo)-7ohj?U*y`q0Ou@3d{J;gn@#$2*#g& z2VB2vjxgdE`rhgkk`@Ey7h?LYQ`n8`uWu9!e)ucAgrc74Xnf03j)iM^=eFRa*9KgkN9_S9nk<;oCkvU zgamvO9*SxV$eRiB8n9{G9o5G;Chm!vTdm?0^0y+ewHLJX>5HWCfpjg{1x6zIB0)WX zpT3$&CY`GM@n8SZ+I_cID?KgDKR?0JUAnmYyBl~}AS|?b1%~V5$@8ZW=Etx_{l-Ia zY8H~W!9ul!D7m%Ly0v0zMm=m zsC3GJ)f3hD;MuyB!t~5hcIJFIv`>1Y{To`KS`-^ApW_z9!{ruz~ftW zvD(cOOzUw#)LN3qJ_n9x-J55)-tBNyL)ioBSjWPro9g_wQhYXO1##pD^dC?isU8LO zCGq*gY|`H_xTpk(KUhHA0YTb|(RDNOtGs{h^>CNIt6Vu_t-=ef!gzW=E%`*sWa=Y& zwFSz1XfN(dO5*_IhwuElL;RSF`1G$ahCa)O=Su>}b50YK2XGs`!%W?;-=4Z! z@*s@75s)U(>Qtt%iZ5rGn!APFhT}K0_ElQ>;ImL9Oy>$C4s38!{!u1Rs}3c%(%Izx zgMr3Y`5pLux;|Hbn(pt$RDMMB&T5q93G)2fBdwZ~H>#<9JvkCDruqQo8nCjKv(hx+ zKOj)d2p)s_OLyXL*O4m!A--Krd5yDtvTm?k8tWxf|J|+14D4&ux#>Swu%dH~Y1}&^ zX#kwtEVD|#T&m~>bUzx^7hmn8!Ml5BP@Yct2l6~h=VR{?*)%S_QH`mc?rl+1KLbbJ zuZ8x?A$QNXZuc(5+$U`q-5)yrZYv0j zk#ZlQ`0wdiN0nCr)hB6dOQn@rt5XfgJEuSczj{*Xg698BjKXWKKjEj-C$Zv-J1TuhoT*+130p6_lFp|wNF_~cA~SYPV%3T z+)fMw@}ppOYAAeb-xMj=BK_;58FiQ`ji#ZDh?n|05U=b(2pP?ecN~*VURfv(rF;o^ z3w`N$vKP`A@+W>1uvh6yLD~s=pVZ4HABfZs?1<4R()mp|WlvIJ)bc~4aZmYkE!W+p z&HUyBgq>XJp<8DMg0sIV(!DeCoGKeYosrSt()b3^m{K?Wj!lx_5I<9D$$rU(@2&%|Q7R z?ww`>0pofpf5B6m^i_VJGAcS7n6e+rE!cMpYxDp1rkX@Q~6BDT|W*jmN>xaxo=cHqNxl#i!*88X7cxx`^!AO+ML5NztTWuRrYE$8)R|&Ka|h+z?Ww6DT4eHSUGNnkq@ldZoe74@@E{{dRp`Mp|zpc_>DNkt`DD;mBgCXY0X#I zWw3)~&Ma^GN?h4`6#D6XgNU9!IIbnFpC`2uHpLgqhn|J->OJ6U*+JB}-4y>~4rBHn zOFqKjl~^^YrARy%j<#!Bg6EoDxc$U%Ub@l;=kBnRdh2G&W$$0%(W765`^L7+sgD6` z)%h`aw+Vr#?Lu+GvF1=cVYYm+$58&QJi~^@yw?2pWfb3b_#?h~aSr$1ZOtDHF_i&y z7T9r%>#)@8A9HV3!TLX(jsCVlxZ-OoJZfsldwxsCl1T>O*07yya<>##ZZ+ehEo(z& zfsV8vsRIsCHzDLqDtfmul#_Ptz>7O{G494R{@ue+9?WpyX-A4+e0Y2A=~+v3dz%Zo zmQENoNV4!<8}Zyn7yf~siwK>Nscjse0?waDg573Q*~B;X7u#vqqp3a9AZWm|Y)`vOYcEgN@O}Ine1U}sVFC;~^ zfDhvmVXu*kJar}x9tKNE_pIG^c&=3U=g=kwSB~9)=Aq-WpF|#HAKUK60n2Wo-yT;+ ze~0}k=d@}}=o+*4lmv@uX+1FHOj~K~Qlt%=QYsF8`Gw)m*Px)!GQLS0g5_^(;?>i1 zhIh-Kcp02Q<2FoFnO6s2o3)WTcPvGI_uCqWv4{AhunG9avy!!J4>%@!8Eb2}1HuaG zLe_OVF>&W&G|FAW%*<9`AKEXlyBsYi794=mr(3yp-f@;xIbI&#O8cV)(Vl@V7Srzx z1o|vKSwEV4&|2`=tGW2M(_9(1f3$42d;>h)*@wN*yNLUjoyNU~R&$o*B$uD9#GHP# z9}spPvJr!rkHQ+bF#8GZm(q-G_pQ z653lfG)y4y9MHN7sA2e=~(Y&I`9>F5aJuaHS;FO zoE7H0a^hXhi(=D)4I0xLX+0UV@&{7g2wSU9 zSljIdt64t`w=H^(>FZj{ofR`#+k~<5^VI~T@#od{6$sx~li_jxmrytDnf6HgF#hX(21ajqr=9c73Jg9^hg!~m*}j0f z*#F^sn$LXzy>^Q1Sthc&VIp*%aECpg`4Fbl+JF5f-bnKd>8CP8Qd9yI*_lhKGq5f5 zlb&lVvfoz}s6IpbIrU*ZaA;%6Ph^jR^dU{;f#-Gbi{TF#GbBeum<`}zfA+F5uXXq& z=W$5m!F!DA$^1jlP+xL6dz7=x_&rAwR=}g?cDSan20z&JIi3ug$B(ZH=05jV0#Ebi z8(w+K!slyoX#2Lj;;s>Io^c0<*XRi**I(j*)Ih0lrRRmc%y4urIIVKwRFi~PTcJVY z>U?*HvC_8eIZ&TU>SyL%xdv8_)#W#~SFkgg(=oGPovK}#Z*-Wc>op&C90&V&^O+}g z)tK=R-wK#Y>tnaoN&^33+n`{-Tn}l- zC$^&m;%QU9wCn=)WCJ#Da*RCK)Q_v{p>fu9*F)#N7sDk;V=PB5dBG~3 z3TdyJ1ZnWO3>0T{cpVAtlJ{3(B-MnNSYQhuEj;0dM;J`SxEZNz6KU{eqErxO z3j1?hHvH63E?U-GPFqahNu!a>x_*f2KUQHO&yAkKkF>AFY(3*Nq3znpdr#(KNcet! z%6GEnQO-LU)TkQI+&Kc~O|}%lO>AXD2M?MfJvtlRj2pMR4H7*<`(PZuxMKs-`}0ZFZwOb{4xsvEr0Ee|b!I=k|UEk;|wSoiLUH;@~e!O0&ZC7uM=F*HfFlv8Hd^T`eVhzChNpfuQgZ@unsQ#s$|nwWZ;Egt8m)mYk0?ma$T*Z z;suO;+X9Jas{TAV{2aEuW5|srOcS#2Op!J7J*!h|B+LD8kDxzs_vL2TzlS~VZd;#I zEg@+=B+hef5i$oJHqn8*?z-~75es1M8Tqb;_x-b9GZtTjlCs`i4r_kzZ0O6Z~ET(M3R3}nt1pcblCL~=Wk6alF z+xzWit~ZyWM~MSowQ4U0K5q~G5_6O;g~#6o=EWxA(u_h;en*y!|~zA z>3pzVst=jf)e5T9rQ)j9aaiA*&ir~{4e!spF{&HUvOQ}1r#yuPa~?Av_s$~O{}Efd zW)pgPhM|XUCk^oi_5N=*e+I&qJ*|LhPFlxp$|ijdq%UE^v4d2bx|*q_Z5Z)4ly4}^ zBD}`b+pe^Jy9=5RYtKmw3M>u96HVyM#A*@jcCj8Kp2E$!*V*3guN2n;y@o5!Ucclf zyc_AqYb3X4RKMKsdT-`2XRZ)#EqK|Ajr6|RXf-*TnM_UA_V1C1#3SUN-8k_skZuqc z9Q36fT8VCfA=rBNP7F$3>q>p3bcPJCejckQyjOZyL*v9ou)2yTG3P}w;btft?VSb5 zH4fsb@H(0Vn;Lw;qi{(*j-UP=hek8T0L?8bo=DMnOS%T7?WLL7Z(%jrRjafYX)>6) zXrX+Yc^8xh`%feT)hDYqJBMxWXC+C%P?AOg2pC+F*UO4-e)>SJtMAgZ6NiKuFSI1hu&)7b?5~W@T;e}U><&vJ ziFZJ7P3qnulp@E^BKQ@B>#K<`>x-1&x^k3Fs6IQXG_QB`Fxmk1{)LV#j3vV_;2BQO zc3pVWXXDu@v2+>Ecvu9z-cq*4J-GLuPmeida-Vzs;m7#Iy;Qp85z@CQG zWJGRodtaP;)SJPLballFzGgQDdTExJiF4{QU(Xi64+dw`t*wW$xbpU+_dN1nPu}&S z9=BiPL<^3!(#C_LAL#ODXXEYg4*N)|F>k$q2jxeVGU-{ihIFuHCFL|L2RL_`Qn~Vz z^QY|6tS5L!?Tt*K(T=6`l&<}((MKL|&B&+w>sRa|Kb#81e$0wyPo2P(Cwjn7p^OTyTIs;W-wNx?&`P|j_%$+~)8ODozn=W7^jw^2dx_Un zyU(?~lC2mo0q5v$tKmkU)7vr~2|AqM4HPsO1rME`rK|IQwXo4uRa5(0`X<~XzvH=e z(`~(|L4+F@YuuU6Zrjb*25eKNZhm(7g!*qu=|dLNgOAY`!`Eis@Ht}k<0||P?;jgZ z^k09PuiGytgB|dY1f66t-1^|aPlQPsnz8PAr*H^6n8uzQ8MHZ{$&CQH{BzThFTPBh_Z94*ULPGa5Jnm`&o%0G(M z7vK#Gw5x)iBGjDf`E8-We@2}UdV-b)+>8-*>@5Wr;RmO_~s|40S zG{QNp(Ph1~(W&r===n$sFhi}Y6=!}|nYuK+S-SKMexnPq@rm$4!sHgv5DIkwPN-@H z?<#PKsVxo|qD9iB@v88J6_%Ri%Ms?5u&xR|LBoHTe$T@4F4H!v?&T(O#GSPY^{$}l zm0#tv*24?_DP#|F*G_karsYN#9Hj9(&ZSmL_vA3~X;3Z!55^bTJXD2jic8!N8J~eH zgok&(A^a}}i^B_M2xxp79hYgX*1Dy7QX`qblLiZorlU7{EhX%p&B#e8Xl!)?p5h(8 z7~}25v=em*nTeSTLeHRg>sUJ6xXq)7_fTcKm6kJNCL=eIvPOrzS^m0Cn!k?RQC-1b z67;635Nox-pVIt~{?g0|_#}Ns&4XO-$0F)J&&3MeO^q(hW@I=F4+8&$GrJspX}sqS z>g0-dggV!^cpX@n2JLO~8DuNja2C}Wqa%?Y;oRDujrG_2J9zQMy=?D36q=GPSUUyZ zr3^-1UUFIN|Lw}~U&facG9*!>Lj>X9EH9gMp+YwC2HQpLJzp8#n^Mc&5_XA;1K_u< zdAA%Cvs_oJlCNgb^s0rC3H>DGD-3S6B4g6+dZ&q;l|#h80fpd)ZFT3zf#Q3?IJ}E& zD~dI~t;Xe9!0%UH2hQHd+^jUc&F_D@Six0)Rqq-P z*zlv@@uh~(X>PNHYHeR<)LSVDtW)rKZ1^McVQqGt)>uHN6MUO>^85>8Li2l6V#XF+ zFPWg-6|}M{?s+~PoX0;u-KOJxI$Pm2`DH$YU88KY!{Z(bwPZD#Ic#(_h0IJ0|SCH{f7vG+s+YPr^)?LYfhW#zz zzuD+Qli{G2M2=(sSlkgN`?5eoD&#%JH;ARzZgJagD|t}M7V@LRHE>*SMh_F}-RM?} z!QpEI@a$cmSLng(Tp*$Z`1PFnIX_0gKZv(qnyQ0^;|Vyfwg-+Q=qi(wSk<_8@?n>2 z42>oluBxdbA09}D7RJBZXm~w?sWh^>3-o^u4ZP8F@;@Z^nLiAMh>dS=C{tH2Hn&pH z>vDMa4%Bh-=e+LXTTEe`jU5>RZNVgLeck06dz9xm`tL@82izg*x`r zTYDYIDND|7rl7@@i*F2thj)`@mhNWgEnza=pUMXrMh*#GpfcOlktPF1zw)N8*9m>W zGX^w6ruUwvt?-pIFPD*X!>nTK*{jG37G!x{WRu3@S_Z#P6|Fx9@SJI0IwbW*n#tOL zcZ@8XI!|Z!7pX_}ak){1i>|6y)Ro4k5?tO>VIb&(fE?cy_fLx)i=Qz zQi5}!m7+`zNSsw>!_y(j`God zgZYBXZP32C`;`!6E;rynjYEi2Ol=iKPoVm83>~5(}BM+6Q?!Pb7f{jvt zJGFrSwdl^y+i|wa4^N$YNq=#3#BN?Oz*e^EQI~$F+sH1-SNY@Q-<)~FhWwMp=>3&! zb$G2PdTNC}in2R*zP3r-9EXF7`E_IFpxij1Qo= z`%NjpDHjc_AFQ4FucbnTI#by#&9GO2mkj^mr>6OR=FUN^N-T@vextmsH>+L7pHWoc zO(a)2lwB;TT3D|=zMdzaZjQZi@k|DIa>6%PI(KxH`jv0Kh&VP$fAStH92@shL*9kZ z^~_#!f2B~GS+oo7YhGTa-wG$|tiR&G!u9;@`DD&!J6rsghq%BSHyzk$q^>t?zo;o& z=yzLcQ_RSXgbrmGi8JPwZ-)1Ik9JG1hdnHfF7|y^e||N z-dgw|*GzYjg}xN$mi`O$!4Ij*W7QmrzPFD*gfLQbFI6b<4&+37UXyUAHvQ3k6>R|9W)qcWoJ_|fPbUBww zi7!Dhoqy4)&lgql=yPO~GLG^cZ>CHCX+TM#9k@`T+pNlr*XdmwYuhV3%(`M7pVF)S zr>OYbP5j^1wmj{_ax$MiC21pVi*uEqZ+ghO*^BAqUE8T@bDVQs;k9^PZ#kQ5;=btU zhNir5_)v+@(mtM}vwEdv@m@UguQQc8*q6L55$aN2PwC$xlk08s(&#O{VO|@3eL#ZM zY4H_0S~MqDcMa$5_2+WNJUe}<<5*p-QWZJ1#X9zSHcQ2K&Z&QXYpt^nPh#ug?gI59 z%zumAYo(=feJVCDw=8_%5;B%q-2cyW3;NlNf$@7}t~G16E%h0cU9agmS8wf7lq$zx zO22Rx`*6k7q`}Sln)>AY=euyxK{sh$_X|2BS3QA$)4`oi(1}I6CDuiLZJ-Eq#@U_S zC9p*HK9NUt?0KDCo<5|j(}j*M7)7ri{Nsg6EGsPON*0{_-r;U2T*{P7nL{*})eXMq0@S31rXyF*g=N=FCP zFZzNAa!%#1VTI(@F0uT`UCOTeo>=?;=_d~@-!4LJs>|~4tUTyNbA4(>9WL{rFeT=S zr;ius0J9u`GZkd1E-hq_Sm0!{`qVeqOpdSjM7{2Eh~}g;rs=l_tER7W%8?CT@!_!! zS$$n0?G<+jC_;gaK2Xi{#k6bkUG!}%=4FV&Sh$?aQxW2W_p!IhEvCPsW{YQ$>)e~PD^2mso#}*LDq1Mf{CF+PS{~RJGl?j$zztrO=h0d9A>hxPqG~BZ!Z^k(u7$eP?`oy}< zXD`Lq>#YM$SJ1#m9Z|=MIb6rpBOB=oqZg2zT326AU4*%rjq{&(i6Rb%NGCOv(pp^m zm+{s1`PXxQaRU2m&l7V}PEynD+tLgU0Mod;WsfCX-7x*3D|_eFgSHIi&gZA-#9YPY zit_P{-W7RrY(^5Yoko?eYr!08gPombEH*r3oiL@P1n1CiSw~tfTNP?kKPl9lv%S8P z4vaDVA)0?$L*NmyZ_FdQ*27)g3!1L#=h;c7R))O0Ypw9#X=T=7MEodq{`GlPvc@Xa z2HAIgJeNARL!i|&#)C>vn!|ftPs=Jp{?VB`J1Nj6f{$f?A{usxVds^-bjGO868*=m z>lBcgH)dMemY>OgU!FjJ+taX{NBB>X{Ius#Q|;7hv>5E@C}WZxbfaSDdB6iHRhj=d zp`=yq`I|(+1z+%epAKBB;XMWh)2dDx++<^6a%%NS)N0sOzT6pL)?bWX(1{i=E=8xi zj}SRh3kci`W4-mPZTI7&uP7d5~K_Jy2gw8+%n; zy6S8GTP}1C=3W1-&BC^-t`lf&xMshEX9D=Cuy&L$Jd=T^V#`;YQ`of(_S_aS!j^XgRfH>tk@ zUQ_nrKB7SK9{w+*I+q{)GvL8MKlyV^1iLo*Mq=z9gB6xOZyQt6pH=MGJ(n()>mtn= zH$d!&_`)dRCYg}Sw@7o=`H)qN|zpSd~4N9YUg%Sw7P3c#d5V$ zhi|Xt-Y1qRqdzdO>d2AN>U*UUdfl7$sPp4=dR(-!EWe{Fu0G7ykNGg>rzW%v<-Hb6 zIScG&=mhJ8^vdkA9`8YbXKuNbm%u|xDcze^jmV=;cYm)w{YM5%WrLJWM%QFv3+jf7 zk9wiNzZ^gFh#EKMpfVhjbZ{R(dgG`gpH^e=8k_#R7vm&5pZ`S#+xS?P`d8p6x9n=) z`#2T1Z#j?q_7_~=lD1d8PwoX9a-%12L}utWRke?eT&pzqdG>KnAuR*pFKLzJ8}j=wEP65s@NWU9OFKWS4IRvY{?IDUI?F%2vc2?r9W z=;m5F&?c{*{VLk%Aqw&F(0Q7KGjK^7{;o8+fYFSE&oW$Su#yeF%&IbxJK3MNPD;Re zOW*EuY*dJV<{;n%16!rhwZ~>Bh$gFb$3euto!R zv+3B8Fuwb1sW3H}DWc6hSca{BLIs!R(8DKNpx1`6@M|mm`+De7g@;l3(Ua9)*C5tq z<}&zH8GSr|+FXl!r_!oisYNt2q8;>Y6xD4sQ@)AY$(G7T1;&##?zi>J;!Lc0Y&2E( ziEc)_>xr}O16#Bh@vsdQ9n_JaMW{mBATBWS3U_|Ck+(aRQP2W7S2Z@vD;TWZG%8Jx z#r_+pXYySWtjAPpZMj%gl261QGFlM&j8XFe!&dh;xT8bP=R|#Ykikdu*-`p$0B`|o z6;8_v*y|sW0s3K(E8Tzdm4*ynN9A|j;erz%lV6e2*cYuem8tJbC6*PTqR*qaqNklS zYq5Q>q%O(rl!ID9sC9vB5O_iE&B#lBkB*VSMOV9Msy<{X=j<3EvCg{J=34|jWY33f3%qU8ST{491Y!GF+{%Fp1TUh?g|apr#toNDxhM12wdM9ti#ht;1e z;0i(SsOp!>(B6y)9(iahjYwM~UbU@84%2+O)WK&|C2Wh)Fs2X87<9S$8w7q{;`5Ck zlCkdV2=gP0)Qh7dHdhJSTY(R$>VjI98Q)7tgQ3&C&RWybThYX>n{aM{w-`{ki2Pe- zvzc3lZo~SPkTu4{kXbX+BhW&k@b9V;+C>b#m?Dg~a4&eAZsb@)=6B&02|h{=jkpSL z6)%i0cs%hMG5M_i_uo&zD?(PRXS#AMAQlIy)=;8X2x2VTwP9$Z4(K83tH}pGCt?m ztu4HVa_ZyTZWFu~<4+!uR!~Eu%2!{v5Nb+8XNus0{f$-?)poyA=n)HgOO~5o(qOsq zmBMIbzl6RT*l&1-<_B(~_|6OQp57>ECp-GrH=Mpa+(76b*?Z>z)b}f4>ge5zBC1Zm za&l^J7U&_mu(%*kr|(0;jCWELyZ}7to24lW z-JeL%jlyV7@EXT-oE&#C(9}`rk;~SI>1*VbyGIqgsx~?l{!R^VHw)b5 zLf;nnaD1U{hOgoEhYG{Zo-r#J9LbZNb}(uQYc*EDS7dpbqeT}|<3LA=brd1)LkJ#< z?eDPay)%QpX1l8}cMLx&dpEsAZmm~kX~u-#b_TP+5hb*D$$_*F`^CJUzlz*HyXb9k zOQ?3y0D(R+Swt3}z+(hJBc`v}^qhA6SIA^Tboh25gUtm0%B%Gf?iOVQM!ZCCjZmh} zkWGO*moxM38!^eSjFHg=enn%;EySsvA=I2q3muZG-fB|~>B{W6iCJEREsJ~1lZR?oY$ zgui;vMt^*#4;5Xh=K)*OM*@9C@K99Z#vj#o#6yEi(4rG0FpfIatt8;5)Q?LAvV8UW zoG$pg9qD6+>_Uz)g?kN}C`|9JKVOlzd<{2T%efPmDqwBa8u)mAqJ|xc&@{g>1WwD! zOyG5RcsqOGTnx^pTg5Jk|3sD#`yOh+{WCa%jHdbXsFvwDbyrv6^W`d2k(oA>I^;4V z4-z-dr8D%ANPd@#4DZ3);97WBu5cjMe?Et_IcxNdPKi$t@Q?J~LJO?M`J8TYw+l71xWZ^A><|koI+Q*5be)*OSM}PrHa2TBca*( zQ5$dd;`su`T=M?*{@UZ$I{37Fvex0k8vYX+%8t<2=6)1bxZsb##Q$+C<`q}JH0dEfhD3jL#AY)(=(DPKja_3>mGxo5z#vbb|7m#9codVzSaVH3WvPFqrO9}W zE-?9_sXGmfR?aIQGyE0-8%4}u8UJf)9C;^CNKa;XEA*nfva8ilAg`5K z`U$?Qdixhtt0ybI;&-}mv3!7!-Ii*D+KiHJdTt z@?E_X_&l3ituKyjseaZRq&&2fZ8M9?Z_UqGkX;h)6&M*JOa?XhWGOb-39rn0=yL_X z%p=22qn^A88rfpJ*xebe7#N@}NwfHO!c75>rA;j&^QMcbQyF||xEy@bna6BhCaPux zg3o&q?niELa6|pC=fg#(u-k<51no+GUFY!QFDpzoPgfV#p{~=OQQyC{ zdNa|TtdR?7(VLC(_NN+jqt`+?p!7USx;}>=_4m@pUasYKk!koDoO98sE%r`mq-SlI z#Y-<-p$E?t z+g?`v+ft-f@Y3G9ipwFDCTU?GB4-yo$1N%}q<<$4S$BQwz~&nNO)IK{2mD8^ZS&}p zXM!x_PF2*u=X%Qzt@gGp ztLNeU&SD=L=ppw$vr9r0Epq%$mq+zX-{{v9I2Wp;b4I97cz1U7plmdAY7P1P+%%#* z&8YtM6r7UxnkGD1L`8-!BJYBe)z33~X#Wt4cJ{tv4gRxUA3og7+_Rn-5l+*4V=wz@ z=Q&S}L!4Ou5)HMyqM7H@xKyb*d@a#m$lReI+3J!7S#XnZidZ$>bvV0Ji*j|Ne&x_M<-{0jXGf94=_uLfw zO)m@Z*Scg4J}8&#Q*%>MLlvwTFK;{-&T3mXUGUBg_N|^9@0eQEyDoKUNyi*Gd#J2_ zeE%WgXVlP3d1Rr6Oc_)0jHqo{8PRA9x6G)*A?~?Ek%=SdQ&5krny55v6BqvCrwY!$ z9FSmNNuJo4P18Z_r{GbBNAwEd#i^e-x$beO4!D(5Z#?Hj6J|$A)VLg9CV?=1;D^7|$2IjoiSMME*V3bW zJmkpX<@NC%&#X?bi|EyZ=JK%xn^fzzo>F=%9)EouS8a8T7j>8>OUqj#+x^O_eaj8( z`*tXyhv?dWiCDv(qCrFq^~}A7kK2ymi+jgW_TbqJjGzT2n`(n!iS=8{l&OXFRgO&e7P=Ns0}arq@?sN_>(;b*yx9y%hH=0*=6fpIkfBGeiWa$UxYPW z!W+teAisT$fj4*Q@%Z9%;)JT zouu3T@4%1z(#(m$gvhTY1lEDZFPlu}=VFe|r)oQJMsu4kdg@|Nvb+vw%%OC5&MEyz zSC`!uTmuGJ>9J=dnSN+{qlHes?5;|$Z9$iQM$y5|nKWtt8GdnlnEtqUET3()R5a~9 zSdPc@REHcNlRwT$=yeu*3N0_C>lO&Ip1K)8U$9Ty`D)T&FK3)zpl{x~s*1-3QsmbJ z5!25@6gX^eO*lK5fNhoz>DhJQ%4PCSauImI95P>Kn)n_2fl3Zsqg%|k(w+R@3=WE6 zjc;dRUDCT-mTEhK>g!=&>K=8B3iMfGbspZ4 ze2}wDDN$AXH0q#xP8zLm9<~4NVQNPF#oh>)i-pj4+fKsFH8@2Fr~TqjOG0_d%+icC()(SG^SuJI_2_x?^}g5s z0D1Q)tlCVx1)lTfHm)}5QLQ!!T$cl1tdu^`%D4Q2=y~r%V9ZoerF>q>GrOU7vL7wM z$L!XBjQn~ymA9Vqw4$bQ&B`=*-!88C*jd<=-p9wI@-gO0hV8`npLXupS7oyNaJ)?V zh&-vvsR=SK@SJCr3v5a3$EGi<{=LBVPJIP&cF#?ex@g(u5`557m*a$ottf5#Dh8s z_Nipgue+xR+?Vos*^_$g-pD=LRk75*U0NF+-E*j#__O^HV;llJX;~zFH2Mh7T0iG1 z_4+cfPPujuQ>Y2ZykGFz9oF7XE(J@5ij?s!)}-P$)Y>-dGj>_X86Z=ET!Ol zqv+n?`OInORa1v?bbr}7tHH13cFQeyPktn))vGD_{(m%Q;$g}xSVaB#Xh+RDI6;rh zQS;Jp#+TcC%Z_6kc=>J@37w*h_Bq_=25Nl*^&hleKnp0-7o+DnOYjU&DY1)-x3nts zE)AWLlb)v}iAUKJvTAF5nG(8vY&p3)e<S+_~v)?ohIq^6PYk7rVWrfXd!-&C(!h*!c~Y-Md3% zyy(WY>;L6kFDlAh%Z#POw{UgGKMri4C}k?ryWFTA)|Ycn)tnL{y+O5j1g+?a` z=ni0`s|M#A%}1DDMxRr|_sgx&6EfF}?F6ou?LX|u>WAY!`f8h56=Y!iXJyu|>G!aBmrj^sBJmy`M2px~b>Fn)}#lVAfbWpOvx3U#+z0r|N@E9H&v$(p8& zFxmk)X5kqlj?%IPlHEPu89l?&Tgbz1ZDhHM|5@v*b~^O)uXLkRs+5mFfA%t*X8JUJ z+^Oxnwah|ZZOth=9M4qo?W^m4T`SSfD)V{o=f2#-&PSI!vR*~Jtj;|reNrQbZ{?Uu zPx#OFp5ogVoR9jqmX7hrEjwNbVf2i3XBB7uaD0=Pl)bg}!lPQMM2^yyjt|cUOnCR+ zI@h~Z~s#lkKCLZly2QBBrTOYp!K0Bb-YHR2NUEynMj+s6{K!cb%6^19z zdfn%t3*Ry7hOn-x?Tz#MZDwsF7n{X=s@y>ZO+={;2D8y3(84_7-3WT!y$0rQCIJ)u zS6~l_jFp|#<$YlUeZ?;uwBw8GN@#FC9k{Z?;#+&P(YLaH{`>}SmF=>%-0V;fGj7W( z8=NmTwv{%022R5MyPFvGZGpbvo8>!aVMNfTe=0p<8t)mgT|u{UMAQy$Jf^w91=Z?H zQC{_Nm)=rfEtl<>Mi`&o6yH#x7DR&mjsU~e##c;R<0uV3HK9e1TUa~{q*T!HuWK2MZV>C8h8~! zMr$RPSWQ*qIt%z0T3NFZ1=cF0LN2@#InJFE*BV#A`*rPkq<1MfvqCZh)8*fR%?!U- zV(}b&iwei_41PcA(y=&OeLcBN4@W{Dajm2Kt?)E5=e+$!AE=A64uND+>vrhRWcgsOP@diPLmelBEO^_qs$>2X#Vz4 z!&7uEx~haGWX!YaLkswY@+sAPPt!D*iP>A~??iO&7Mjpg7qFc(SEra94hkV<4 zPjAAzs8wlV9Xko#%tqf>{u?ae5rprx)ja!jqb#q`C$661YXz>S0u_+A)peApL3L|I zZCd!ymzx&}mMQ0p>ywj?QH`bFfUzH~@Dggkcoz+v<=!>B$>PvDzMuk)Rv(*N@1eRj0I(?$*+FkUTA9Zle7#y#`USA8hk{p57a zu{2ukcik!6y-^_e+*V&tjLXtkSQCRiSy*NCa=?8r+4Iadqi6KD1-WIuH7Npm0X>`~ zpc#$75Jr>0Cs6-tLkQSKzDna1VW2)0YD7M~OzB9ckI!dJ6X|flq3E zr&5N?-w*NZ)lCTxiu{0&dE^ z2|eVw6E~?`-*WO;KyhyORI6V%6AYHp(Tcm_H}+9R1sm(=Pmh)Hz;9zZ;C-uWv@!g- z@kj!AWPMU1Oy#V#hsrO=WQyWf}Prbqrv^QDfm(2 z-?YgJCe(;zJ=R~MUyUc_m2=OEzXQ+WK9aKht$n}S;Ol$;$1A~QGC&SZ$EAz;tWltHnW;J6h z9(&N>AACU+yvrtnkG4F|8%L&>;br-5x79S#d!x}?Mxzltr+ByRgmT}u()_pbNOP2V z>+C{J95LRKqyEM*JW&>RW4&qKpSd(K!bvV#Ta-|T3Oy;@eizi}6A?RlHva!;gX8=x zxF&-y40lj;*L>8>F-)P~^_bQj8NDLmVT9kllia93_S1(RXt~OVH$Lw|;|A|%Xk7NJ z(vsonq49H*;WW%IcmA=>ct{Gf8A`v(Z4m~`&79tjPlne>kl+FZuGLE`o-+PjY)d*S z@~x?%W_`)d$Qo4gqK^a*Y-#3b2h1r+$X~F3T}uXDQ&48S@LXJdHBzzRL8#}%k9a6LZwa6Jtz zo=?yEGEWrs?P7%vri8OU1JWk?AS;^1;dKjB(KDgc==UtcZI*u41aaKN>ePYiMhRZ{x&IXB0dL z0k8ZGC&Vu7$VOvanckY91=N|3tBwB@nWyv1RvQXhfCr||DK5IP_AkDJb2TWT=lX6Z z&Q~Ar>$+f`E{M&^d+GSsoCbFo+L6i^TTZ=(E#)FZb79PT7`j579r(oj$ua#_u+!=S z%IkYM=<}cITH|f}+9jEhmuP5f;aE8pUaE$wH=(w{b!GCS3LBe=S=}EqydS<7Si#U~ zGGpo--e+@N)E(S13)lbDn`iW_#eK|EfqBltag*&BE`<;0mkFnU;i(EfMqIaFg#K!x@MW1B%E_t-kv}@O; zQ|SB+;Jt(7da8>YlYCAX%tu~B>F+-Y!|B$#bD*>5fe$Kcx0poQ`}rB)yw%REIW>8( z+hjY?mIW=)ul!%df_$v)Sn~S5#k!vZ#WY&Ru?ufNF6GMt6$8*hzV24}>U_LI<};EW2G{~TF*qeMs&fftO=RLD&* zzxAlgmQ7jt4eHohaktPZ=i>!@3OA{;pWqV|I0)HHI2V}un4gGXUe_#@puq&RDH%Rl z^|31W_&+lE4!(mH$U}>_EVW<`XkJ=hyaN^^YMk84k?o?v8~eC@%aZh_PA?j6f6Qc# z3ONncT;_n>?;2lEEJi~r;mj7)Y~qUR!t};~(%A*>A351t29~FHeAr9?H}&Vu?Kz@R zISKDb$RT*gv62d&h>kak5XgT;df7ywoZHFSr%#Yu_P-{pR^Ud}zgi}Iy@N7*4ZKo2 zAIwTeHbTgHIjqe`a%mIErB2|SsxzS`f8g?WKM-^#*ByL;kmnIHX@ak0%u!ahVeSEb z)6%BL9Q=;0$x^KQc3wn&yq!?5S?imAqo|mT@45^Jx1G}j8Ql+y$v-Qooiyy+7{DW zpZ^{%FU3GOA+l(D9=|?2Y#+mag#R`c=Y3^S1NJt=Lb#7?BRpo1q>*S}Q`@m6l^F zETyt;HTC0#&fIk8F5XiuFSRe{s%_>x6$?8pRGuXk@t4upxNX&vdf$;s+FYaLjU@bT zqBVQSKvinQJ+XUfs4o2O73sIfcuI#dWb>@BbZs8aJL@^fu$s=&ZtiyK)a4@$j{7PG zs@%Hbn$O~w<1Bq3x;pm`U!%J@J(chqv}+%GqcqA+TM_o`df&8O_IZlzH*O5vQOIimCC;qqcc0Dr!8Ojq}tNe_LW z<8yzgj(skQ^9@LbKCC3=^2vOp*hdaNvtBnow1FFp?uK^)9;)h}29ctz1D!SUjNcH(-e=b>~2RCc-M3s1$oo;r2#Jd7E z(Uo&QEv5TU7f0r0mk(rpnv}YV6Fu*$IU}3OI*XDmAD`GrQ$KZQUE(vrJ893Ao8+^7 zryf)MrILsXaF+|DGPwxob6E^mC1LpHV|+8@5rc z!QOAv*5K@|9C<>A}68=aAPg4cSoTdNW9!K1Mxf;CnDh-W(#CFxz@W|>RdQ64fbg$tp#&~%_%V#vG-FK@? zy`FS4Wwc)Na~Sn2zEZ5-AI0yQWYd^i>hdQ@H|PZUVBm~lGqu1~0S z3cb;a6Q=a!kr5uMtS%utx@z4y_gq@?$x+vvy;_VA0dhs*_HuLYTeQ-Byt?qB8NYBp zMH}Wg@#N0iG*z{hLqGy@-8Jv;!suN!%mN zc+eO3=Vh%?>X>M{6P~$$Yd-1|O@+QS7KfI3i-`jY=qanG(t}3vDt>lt3fWRix6H(= za_3{LriOyjT;=OlSGeT)=9b}>k@`fx+x+bi_Oh(emIwXqCr%!7)3^rBh}=bCp?ZJq zM-wbTor5{*;P!yHT8HWBt+_n(?P!Tw=eNIbc1nwqw90!uJ*vG3Isbw0BRAmjXCZl*I86qUYaqD>Q`}+9bCRpuWErbpr5a9 zblO(WF11QssgO$l70D?Fs-<)~rKtQ{=NRR1id9FhvFi4w1NR$tn!n|0#lS9lR5N*h z?&oPLX;+L+!r47FM;*3YzTQb55(~-k>JKnuoR`IWR zdMg`Ga^9++TzF_X>XV4InYj%@(;$in&Am-ERYFRqDk z6BiM%O8;H-UG=_tLHuyZ&-XJ1a8ar%Hx;#$$#1sMrHmC4HO9yDca`t`n(6?%?!xBT zI%R(^TsG=8SbppEScDJ6dn9q=bh&}c`2NPc8hr^~D6B8M--0>T3aB^Ypc@ zI%VttJ-Ay@U87V3y}5TM9s9T+xi)uV{%0fp-Tq2Dq4Ruf>oL|qZ#?*qPmY)=tE|J` zg*NMDc56JL-pGCYb;kX(|G{vUpK4fd=>l@&m}V54xk!MkaIKkSaM$Cyt7^5lDg6%o z#X4Y;>3=R&a*OKdRfyiSUdzh@XR(*lXsUkUB^S%^R(bB-!FN^)I#U&U4Kqu(#=Di5^HI^zaf>p;iFM=dZsVVs0lBq6Hk}X=iWWyu$uE}Kmi*K zzEYtJ*0Nk5uP@Iph4sECa_q^a<7XdWa0lbMyg6Sf>JiYM&UAO7`8|uFZ@l^43X9C1 zJ2m}>OEeFw-;%=JH;7ku!Ftc!4P-xdB#Xp-7W5_miAhyDK2N~f#zP3)Z;isfP1SBT z(j8pJ(&BYV%466B%*jsr-aCmdV6To3EA8db;04OevAa)kdH=(4%;~Od8zaToM(}}x&k-z`4$B!iE&qV)C{$@w{*uJf~t$6*_wB#fjOa z;l0#xyQtOTGu8q9QY~Hzd%A~hV}toOMi%D5-C`MB%%~M@@Fsu%3KDqFa|)&LsB44K z9}!|=%6T<>b$JQSDpeYG=6nc)EIp)&fI5PPE>|!~Z*06>(+pto2M(Lc@ zXcW6Ek7(DOH*D&ybM9=?x6ljzJUfpb*!2cA>7ByI&Xwiuf7Y_;tM{*0a*Ln}y2{G{ zeX4&6;Ph7VIJZl@%$Z_^7F4ExQ0v+yezmo5iw&kHIKnTHuNOq(UbZ2BDHBJ1u0;W( zFM(G)<<%R-+4MQE2Ab-hs`K~|12@UPXG<=$&0Tu0?x?X=s?u7f3V6p89HhV*8kj49 zFY4gj@eJ)pSI-|cbwfqg6w{VmbEvg{n&C2>)iKp*OrwLi@u8BYFC^BAhc}4QY1U{_ zxlxj}SY%GC{XHjbbXwtry=w7CW^g`XPHEqf9bCRgb)DJhi|8}%0ec?3f?l=P;30k= zxW($T*M;6MyeEw2@pAgal_MWefs`%mJGrPdYmUCl(zU2Z@O(~ z%#Xd=C)IBs@pldc-!Z( zl>9ZD#P3Sy*{M1@tiZCx}cvHE-)oz)SU}bs>I^XSWR>7#_iVNnj%b zM|tXolG;Bf>rwOk_|m+)$|=8_Ho6A2DBuOeiYbdl(RqD2Gu`&tId9j2Z*ZW5-_Tz@ zo)I*vb$|Lr_9%W-lsz$%`=7I8yDL4#r3?=p($!wp995TgFSTW0CMWyfH|xeTeGhTp zv+1<+=^0wx8|Q9&W|uuJ>kY>$_$A;}lrrOLdv{qn{0hFecrWaxwd679uBTla(T-53 zGRWa5bx3>6&@ zBj`kiwx%7i735cszTn1dz-?!`v)4g-f6qhXj@7|^=h4vPYH<6y5?CtK-&!0Kvy|GO zZVx^7k=CzxNtg@M;}RTX`dD0>SWQCjfH#kbhGRCVjb)>(f9o_b^B}edUE=aNis{%o zMWx-e3&0y^Y3}1vWEk6*Bigd_ItAgJcWC)6ZN7Kt6`nKTjc8NmH?MW@wA}sP#bCNR zJiV9kDZ+RGa1ej8@`Dx+7?6XL~aTj`PiIrp@9%ZJNjL~hGj z3;YUa8y}`ZZ(KCEZu*=&hacibr$goXCcb*S>dlx7>X-hVCXvZ7&JEVg2}ICZFB5fDFbTJ^O$&e+DY>mT;IJWi$}&Emnk%&hR0pz9*Dh^<3La7Ae(rDdGlLVdJS6&qKhA2*Zsj}ixbsEm z-j9-mx=`0jLOb3sNYDsWVM80rS0I^i9nmB?!(!?R`qTn1%X8hj7~QI#+>~TEW9ZvY z>E@b&fz`6GYe{XZnCo_j2j(eZg)h;j=JpROB$iJ;#=EOF;aY<#nwlds2V1)wRgtq2 zEJa?lXRFgHHgjsc!spq`2;LX>eVnikY-$VZZ}fVAAKv*KMRsKLW1jF+3ZJWwm#~-5 z-DXhNrj04tznk$##-nA`DR_bEtXxFZ9&O~`ZD&$=kB*$pHIwX%KA`25^XO`iO6ljL z{PB#QpENl@#H3#;BJ4I#>Vc$$G(eu&{ykY8A zDr8eeV|=2^pVJoTBwC#uprI*vXVfZ=t5uiV1lWqXCrV57wv36%O;a~)P)Em~v)t{t z!rYTGoP0~KraRlqFvh^tx)g7AX~Zi})@5L=6}u)vpNGnes zX5f>_0@AOXn`|(V;gzgcd?Tq@lirpVU!vLMBtwI9h>L{?p{*2tkXgyl(`_F0R*%-xA>A@Y};IikRhpdV}1x=i-P|W zR||X(K$c`N__{u@gDM#RCQCDz8h{V9VqPrh1A-?J#y=JJU8`R7n9631zkZj}#>YaB zFg%4A8h-%$SO!v{U%LQcCLLO~o=1eg<@bJP=+?k+h6iW#G?jDiNYr{Gfgd?nv)Tlo zqT$;V#sYm>l1vr={m1`S#gg${RbPc=^)$vPzxVbre$V6-7XO|X_}-e=3OK5&Ep(IV zUAL(7-<{a_@NRJ>gz<7?iVx)7&z39XM#eYdd*z&2c^2kUfY%AVVI4cXANu+vo%>c> zq!bO7=I6`m1Xf!U1$dV*_X64-zIp~%=(rx4+G6OLJ)+0kqPn%eKZB=Lqrs;P&nwg{ z!P`*h5Gz%ycpL9HlvB_!{PomMHlKrhglw1Q;GA2Nv*xBwrEzJvjsT8}^-WR<{HDMq z{9loUj9iT1=>@QihctU?JT^7|+)CSDEpNg2)!Fzy1gy{ltS8L6i9Xl77`j{SJ^YNJ zPZTsh`lTf9Ka(zBzs23_;T_v0FDRe&OO}>{rWeR`%sOcJG{ecl^lJBd)hTM<1^!#> zg(&BIQ{Z~)*YAy@DD8sBSPTk#VOhAt+x!mrI*e_xpZs0BFll_zt6od6Ryi2@klG&_ zFKnw`Mt#RwhClDdexq~fPJQzm&4qIcI*3OL>c}C@`k*$PB-Tn*+_;9La`xiAhx^ij z`f(il_&KfrjQ1PIdMMfIt9CRkK)cIg4`yVk;4mwA(qfIpxh_q72Q^)Y2R@<_lc|f*b7UQwtMXkUk%cvQhWS8A5mB|gS_x~zlGjl;(k;+V7 z2rPZfO@H;}%MbRE!GJj<&xjH4XQw-*cQU>%-QbDID52{_QG9RjcI`MP*Hz%yG6esl zY27V`*GDGM3ts0nrxd*|?tf^=vyb1SxhE1;D|?jb9ped1KQ{YaiCRA`t-SKIHn}3d#n*u*->v=V8>fuT5a2F0 zIXOHRCAH~C=o5=ii}7T%%k&S$WJ8a)!f0gpID*$C@EkYJ)yV>1$;ei%YdUX6Jr$#b zJFC^p?LP#(0go+F&G<`g=EP(x&^ZzrH+?;LS*7K!pf*e`FX0gxy&}GzszP12+L(L* znm3X$?+koas7vHSm3h~aG_m`cyE0kH42MuUvGgE)Xggh2_Gl^GR@@f_PTA70s@G`9 zVpo}SVGm&b0a~)OJJs@a=CV6G@>JU$+_zjb=d6;!zV-n;YISctCRa4)6WHtZe~zv@ zpvEqYYpH}}MP?!qDWdK@&nZz#WMq%*tq<97WS5bWvdXA?o@1pDQb|^1?~#!$ z-*fuw^Kr*{&v~BTy6<~V?`YJgdt_cbGZtGP&{>u<^?8lC&0!wx1>I=PY|vb6jeqAw zVs=Ak{wlvIE?&16yVdI~`_^bGzn}gJ?;Gj!0KfWt-I^10KToMh!-2y6)D-Adavg^z z>gaqWFUWHIB7VK{9h|lYjC)pqk*jt>dC`FpG2FSy4cZjAC!NiGg*(M zX$R}}v$oqB;AeK0&b_Z9pU3yXA9+slMDR!LpM~QgphcOeQ}PasF71%>emL-_;SA^6 z+CxzJJ(y{F5g#3{0pm#*xz&CQNiNpVr`A_k|L?3wNiRkh=QXmd??H@v5hEWRj)Vz= zH_NWS_v(h~cf#Fs9^&b{0_kE&7Q>2C(va9!6V9;kn)J-y!g-@IFR|git--cTl%`JR$i^0&X<>IZO zm%Lv!O?*5upP%fxTlD*RLUvdk#^yiEhKdb)1dU6W49SGlff0OA!_TgLPb|hi|5iv% zpIvgy3gGlwF>vS#+5XxYJp5w=3^?6a(rabEur^R^c?muCEeBgoZ`wQmJ{pFOl_4n) znKsrQ)7OQH-Yp-9CVP$wd3A|4y+buVzGgTKKX;wcdU7M1I%se|j6J{ZD+e{C8tu~< zyrVX&T4aW{Qz1&Xl1CR=3Ma2PN$+EedQ8#PpL!BXUFdnn&d#!PlWRcp!C&bG;mVLx z5OnDwcCA=|thx){^6D<_S02@tUpD5)Dm0wxR%Xv@$2H|Yf$EGKrys|-y0v(Z@iTZ; zZp)l+0S|t;U7nwPM^`N9e5)x((RIl-3}t`W!R3x}epffXXx}3Ye)ADu9r!`DJQI8O z4O8~VII2y<=k3vkOH@9}gb!vmVx1Fdt~X`0z~yv~HnD>xg>_48A;Qtnu>V!E(c>_0$V=_RbRxvSV)WEzlfuq=09? zbFh1WC$`x+Lr49Fv6I(A-IhzZi|JD&JmB_^k`xwlZ{zv!;9_s6<2q4V?+=qNPp!jj zZ%6W&%dqa<`Yh1EN-pm_fK&bQ2{+1NeS6@i#vjAkvFmaBPDh|Rg57(E17RXO_mVr@@H>Tc~!8@Ck1x4)Lp^q~^SizN1GRR&O8mgJ|7=|y;~1Fk6f63* zb`s6?{jsA+maUK8)_trt69*QIV}||B<+;3ksLB(?=2S0!ph-{K}W7PtZ2hnGQJ;{T0}TZ8J~5k{+Jm}?U3cOl7Qr#o+j_$MNG0I?MakI8&gM-BH=kd$&T=Xg45As_J+1jk1Trj*p&AG88JfY95fYD~h z2{+B*>b-q%+vy&9Tt5I*>-e?xV6DP~BlQ+(2=l>i#Rwj{xf!l5)d%V~=s`MB?_v+) zlTk2n&R@LRrIGw+)Q*klWFW6^kr)fn&d4f(dpFM73utq~f`b?|!?+ zPOc^&4Sb7EbF3uw7nlu?6d&*If_mBQ)%w7cDm@@$X%-M?pu%=1uckED8h1S!p;`!_ZLb0=7GY#C4Qj{>`XYj}^d zFWCFC`?x2;f&Un4%Slr}O89Uzr?YyfwprEfag6YR_nBHB=TvV1zMIcrw0>RmnyUxx z>|A88YQC^#+ekHDe&FI;LEjDKGbh55o5!?mc7D*h9)2V8rqLM;`st*V+=S=NzM|dW zPPk^KH4-<8g%v|_u>2|FN6!O?Zn;Qv#$TSIz1l2WfbC=-oa%FneP-?=->!r><238h zz!a&LCDjT}vka4KJzA-n;RX-yYD$(!9JcTb3+X!vLpR6c?zokeS|#*CBP{rCD(;~r3erLNryv}N@4?S{HHKgIBJB2OqFQxKz5bfcz08Jqhy7SLWG)c5@d@@H=p3mX z;^^>>jQR=3(Y~LebRz!rl%hg5COrL|8T@9Z99nb=Mn7-{+pMm-k;zdwBgU1v-icAoG}&q_Rt+VD{8X1)$D{@bIAdr=1#cSr`}WVYm{BVo>GaPs-8Ti9-pB+drc!_#1| zl@5DEl5ifJ&Rbs5hLM2K6R<=uGp`8wq#WpM_6x?9Ll-_v2S2%oP{+ z+sWOtOjx5;yRhhQC-LaVXGWZlcM>)r)wFQ(nagb_U&ion=jmBGI~ZJl7bDCDHD=O} z=49Sk1VTS38<&7+I^h8;!?NtMqHcpSNO5i~iW?$>vPLRv^tK6z_I> znL&JU21wT+@je`A5``0f?WB8c21>#c^!Mqvk2urZS6 zDIK4fJ3^A_0kjG<)z-ugpm2-uQRg&g2OQ{Cjpw{QjgG@sgYA}!_@KELzQ`O4E55u} z;}UJY)`Hcgq43CNJAVCnmsxwSW0BrB6b@4D+(*677ifQg*6{s_6;67Q#uon8M~mKh z!mEdoq}NGn-?2(-Gn&UreM5c$TFClJ`+)nYS9mV3A7vSZ@X@%TBz{8o$2D1iADu1r z&s~y^QaT3T_a2CWOMc^_>E39Z?xf}t0zNh2Ef+gU#Y1^v%{3$RJs@9aI=vyNW zXnlBD${x~e`>@Z{Sd7ou3OoLDrt28SNE^ZLQTeI{$(Njjox`pw?Bj&%a$&_C?f6F# z;!e_YoN!}ar3WHh5c|hn0o9iZo7=}-Ag#QGyux`bv}hvVG&q8BmXq*Rg%xSOMX)G- zEhH?e$_ckbrgbmj)$@KO?in0=fL(uE!{2!|#2@{k;1Sdw=o*3eCG%oYFkCG91yu9s znD7oZO*ln!6^nevFit%Sae~fIZ|bjfC;2%&T}bgtl$ULlj^(YIX_U6d6UCFkJ-oi8 z9$;IKHdH+aE6OaXpT>h%qf+qd5w43)Y$TH}okhRwC?HM3i6@xjFk`qmEEUxIy$$z> zge2Opi+aI)j0^XjbO9fiH>UF#hVY6F%@qfT>7#2ikN9Mv-a~c7tZG@%Z$X7f>(3A8 z#$fr(=Rg{_Qp?4UE@#;dPl$Wbwldz}ul8Qd9~|d?J9FgXP3$h6&qEp%0v8=%&l-n(UhJbOZBPOzVm*}VhZAA8!76Ti{(pO?jl z@fM_6&I0*cPP{JabT*fyla;n*q;nM)fvP`8do$T$!w%9)w%n}4WF+23(zi|$c;iXRARF4hhm#{oRb63exG;ZSfR!Hlpz3^(h@_dz89y{%k zu=afnDd~e*aMS|QD$|6j1=6^rKjTD5$|zL*NH_wcrv1|rcM9rz7H9baHu~CN=%oGd zWcop9`s$2OzP%ycLwP4@q9pv*{L#Oo^o7DAY&>)i(3%4IJQ#Vq8g~A8gArz+!n5k_ zEEw?<6Q$2#jpuJ#OB+sghq;^R`QicAeA77xO}9*d<8!@$xRPqE8wM_FrRtZy+YZPB zKro$I8T5H2;dU{ss&$LBUm%bU2C6~1`ozdeU8cr5JGMIM#2Um;xx^`YI9t0}jZruz z1j38jE2##ZRm~&$Is9n69mwl*;$t}D))9J#48oLSA;Kf*Fit7F!boQ#)sLVoftd`^ zRAQvULdBKJC#c-xu8$>GT2gsM!ceBLwP}4OciG=y@0$+Q{rGbjNqb|+!3e@YPaW}? zAkU-pRi>(Mr6-bH_OdIV##U-x!Yfq=w6F9TZ0r6FNCz_Ijg{sjOvJ-ML117SQt34= z7PQ1KHL~DLtu8#;?aw#WEY}u`JeFyF6~ZrP>qaiF z4a)0Z{MQ_UyVu59{clpH?}0~8wMC`Fw(mR!4`cg*@-!-=Dc@nnZk|kFi7^fN>wWu( zW4~hZ_AF2tPrSiW)^^%1pgH8T=>8naBQmqrzeSsS|8P$DPk1{QcVU`W; z_q|}gP(GP_r>K2l7*30RhRVk&oQU1BiYeY5EEmZ<$i^!l%phV=O+n*=*qT#F18uEb zYgH#-N_!ChzChw)DEPjH_@E#A+Oi6#{7I0WXVg11rlX`ET_ojWY|Wl8OzAJX0i!9) z@De-RGZ^VA9c33V4(NBtrEyw6Yh(PS^TF9uY9Xx)+G?&r-K9+x7KkpBJ;Lwx*C_iLg`Rr7f&3d# zrp1z~|EBpH!$=SEfdx;n;@oFJT1oDyl7z}Hdo36Ol#^lkj&-OwPx)cW6hP^x!LMj< z+aI5Wi_2*0|K$*Nq$BZ3qWI=kN1C+Z^jWi^1)ZCDqWC3y8L>gcbkT~eHCvGMotDNA z9{c{`mahYm>Kusck?;$P>>X7m0EAbPawA#&<_tkTLj>2W&&Zn~aTXAtf%0=|Jx=J6 z_TQAI(`~N8uk~|~DG z{4FlOJ6v03i3^a&#jk@38F?2z?!TKFuemE!mZ2rCmQ+iidSizLBKb?5a5W(eS_FHl zwATEf%zVcA6VS71x;XczHD`xfP_Q741e6)`4Mx)_i*8TcOV3iI7iXwkxA4U`?a13}DX)I0T@`p;qz6>x zO=fq(h}N4Z>uZ5Wy2dkF2Szy!s%(HX5hs7ErThm{uTKTa;wrgW5O)%D=?s$71wPJ(NTuWo((z6Mw>aQEQ&XJQdiP?A>ovuTvyv? z>n5aZS17Ji+5YbEcB0R>ZQvBqlXrIB3S*id1aRm_@dca#0U90s@OAMmNT7c~DJoH0bN z6Kh&@4SlLXnPx?gE3=sF;9T`c4lY=n`ALvT@(ba*m!Ed-3(1!>{?@Obf5tkptMSH zi%RTy;Z;F%V+%!lTfiGjd&p^Z5m)%Xz}6WV@?CuS=}yJP;p&OD$S%}ib$ z2P?aOc>CirNGqxeG45;d%FDjkK(D{N`v z=n@9Uc)^sY6|zRPO^_Qfjt7icuDf&hoQ&}ZgpYGKYd=((D2C)Di%nK;ykFip9v)#W z8~Np;QjmDM8wafh>&a`KdP)!4FK=Y4`MA{Tvd%`> z$cxz@#X_f=^21txm}cJ%PbW{rz*cpjoq7GT zG2owY1)emUfQ&L&}4v zS2u>2Dp!b#$Isy_>DiS|nDNAn$3L&a6CKRCf!=*hHSk4xF+I>2*W#xnogfz0A^hbE-9&*87j{Oj$IZwRBf? z!uxs7<6mAfsOGau=uOyxXHFV(svYbwwvpTxH;CIkFp$&>qQo~Go7x=5g3|hO%%Ls1 zLH50AJ)@|fJXQbjqZRQmw@ZmgKWC3K;-+%K0(_cBdE&|2u<)B9J__BBlM~KXj)~?6 z-ntFr)xykW!((e;l<#@0QN3KZcsV`uzWf57PiqYEsl#Q2?_n{=*M+4Ot-^xnG@!m< z{lB#5)C+LD__a3W@&uf|w@45!FB2=~x`>N)&behy;S>B~bUbcodx zZ?V#En|RpbhMHSh3{~J{Y;BpD=g6-m->2sSCvn0u$x*WOAA(Oj?L{TlLkrJx2TJ}owf-fMj`S@~)AJ45>fqZH%w0cs2-+tJO<70-X z{^y?K=kjLZQ`9_z{=IgRxI>2Q&DEAzjHY`{IeOZfunAAnsm}amL3Bs1?gP8Ayc>>u zWy+1SCc#!{D%CjlC#@F~^X>qRO;GiGo zSJthwd2gI~+niHBscT2VF|62I6MSrTqrwrw7dTqcOty_2)V8E3pKrUNF6j8>e{}R=bV>)jE8f zzd-c}*o0WYrSNT)@4eIarck_iEqMg@rZcgqcR|%^N$*!^Z$AbV9}ynOlLd6o@+2F& zu6QJFkgb~B$C@E6!PhwweP`C^wFa9hev=PVQDKnI$EJxan$%FT{{sUUv#_bLAiuR?S8mu^+|< zTC=Tb$AqfOO@q8qec!big-Qo;;!NqVU>LMa^3nC{wh7E;I`d{>M$-0Lj93tQ3(8k6 z0qRFQQ2q)IO=>TU-ZqB&JFmgIf(}R+gZ7hNLvq;;ptZ(;A_u%!QIFQ;l5TLls7elU zfPAPM)dnX|?ZOowExS^MKRCA)sjgV(pON%jTC&msjA~fS+_egc1F`GLEL8PESPH{; z)q-K~Zwul}_Ta)V{ByOnw9T6j#AlV-AdB{ipxzZfOm|_nrWxstzHpSz8gi-rSdfOQ z)FX-0e`CXWRrr%8dJx}yEp~Wo!Y|QtT59cFy^Q7cCT5B&k+>39M1I7ALCHet9O5Z> zy5$=lta=w3&I=KW8^(DTi@=;_`1yCNP`YAA#8wvn@+qS^RveE#G$U}^_};L<`x>L3 zf!Yp(pw-X`;OhNGtsfF6!LP>?L2=TI_t#_OBQu%Pd zc9>x5DG7^pL;6_~2h0OM?=f29WvMhxjA00_tZj}aubg=P^q2U0|8I;+st;7BXfnGQ zE?#OZsXlezn+*q%V2LaJ8o)rqI8@s3*ii=gkAGrVTq&F!VJc6&yo-4;O=R`v{g~pL zmAAujid7*bc+&Z)`O7iJunOtMZuFi-P-p5W+-y(JIiw$T`<2?c2P|tp@?n_WI+$>uqXxz-!c(?YWW*vKV zda%;MsI-ZzT*6l5)MJXn=RL0r)aSTy>OMUD{;0w>Otw#e-qnY|5(gWkJ_FyM23%p7 z@&t~z596=Ik%Dck!UqN)6NI^HZdrImIz0Yy67}P4;K}ZKc<#-) zns)IU;Pn*p(t%eXd+tHzQ{)PE55jo!-@jOoiUPDXZbDp?0-2^-__^F6ll%aqegV=- zIAzd%;*wN+o)E@sdNq<$to)ID1YWQ^s(cSRRg91k;awQ5Hz<8UI+3YGRHQk|y!WM}9@c@-?>G-M%+|3FV)9bAyE!;+T z%l`~9J}k}iOiw~f^I z=oC*rE6=6RT}g8u&1#Cna5SQZ+`6DO{BArG-3EQ6zS#hsY?9bVjT!s<`IaDGrF;*h z>QRPeco3B~u`eqotn7`yhx~xCu{yS&_BLxWdkYe$;hHY~8mbL&HowN&45*JrZ+GE_ z7>0?{7c-lX9k4#Mm$u{4@1VSp@_P%vt;DIpwvxD0=cIX#gxyN#v-5f@RLuy|M+!mWp zFGJ#|N^b8mA{%Z+UIJCCgu|?~z!>k3>5eoWTo&9|X-6D1>6W;6w6!E}hg5TdIF={d zw^X>sI?kQJ$ZIg_QQhEsD@ACU4x68;$(62Lto!0ZeNNoDS`=6{2f`9eoVty2+H*{4 z-BwTfW0&V?x{$rT_Mze-R1{C(dm=r(}vBmMCS z$O}P!VwoU4T{$6E2L;5Tvy*KJ^|Bw)~{{9%()C{DK&^B623u+~VN9qeM^h3z+HIo%DQ$Adkdt zH?Bp`_m3G}1M~7br6G+$HFIA0mll$)8^hGR5q>b@8c}QTI(*-^C;V+cT1@N~A!t5X z&dy?ySlU+kN%BFlnxScVY|&7}-M%(-?{G4vM2^6^QOQW!Qeik7=-!&R<)8Red=+<_ z?|{=kdUB<8h{G$npS&TQEZE6keIBg~81oN{Ap|mqN-Wj;PcDR;w zsE)WEvft@}O{gyaV~w8n7+y7I8>R#kWA( zMfEz6SEf8?A&}QnbAg24G`~k!je_Yi(#epUZoiI{0f3WcqNEz9u_Wn87h^wTQ?T0Q z0FARsfHV~J3@?Ha6_-S@+fzn8CPF5i*S620^EG~#U~-w2ToZI3NCz^yZY_CYrZBK? z%v~|rp}oqN6pzcW_uV0?NgLVZRcpCGKBV;=4h2?wNS|(C#HpA+eKKMGV~lE&EXV^2 zm17WIC|nWCu7oJf0#`%ZBVD^7{UntZIe7I3&O2l#=vr`c>1BK$*Iui%JbgcsHbvqp z$e`a#dly_s!cURAZ94WY@DqPe_h!VKkW*Z$G7?n2f9#0kDmTO8MelG=>@Hkh=Afm| z1HwJU1Mp%{JW!scu9LoFGm>tBn5s|3;F6bwO*N3P8_7?A(j`h)td|4P*>{kr;ax#< zF|3kfr4`u7&x zbd;wc?@V>CL(*J=w6!+3<}!spU_Ii7u8vhO`~Iw-IAd=mAJ4d~B}^heW}*BA>vAX= zf9g&%(rc`lZz;SwU5xa7Qe{iz-34hRkX8H2)?Yl7W@A$g>oUq86^1F@L-;fYl+N#G z`cAD4kY1+zPhS%Ev*l$+1Z80=`vIql`>Jm>4d0h%5GFv=?cH&z;bxVciKSLP_%6*v zl3qdbS4cS=sN8_O8I9#cr8Xl?#9`?HLH%BtZGG<2T&m1ttKJq+e3aEU0BQZ8`iQ&Q z2N!}NA;w8ey;2Ph+i5CwJ8_wY>KIOip4L(A$hC)7h!~G|l)crGihJRU3;wpME@@2# z@e&`|>Kmh8!=qXC`B0$S{oPwDp1{6C^D{}C0PD6(t+7;jgwoflmuOuo`SXTnCc6`N z8f-UKL(&RRUUM3oHtDLE>DPj|YYWxhN?n~&3O}t5A?XLDt;ipZlxv%BmK#j-aC*rD z*nGM|_S z;y_4=yf2mxDqzH$NZN;L>L`}4{DT*>XTrg&eSz{;Fm|brYTZ<}MmmVucv)1A`|syL zNSuU}2QlScJmSL{)s>vrylG{g+AJ&_;(`jaS5kX`${;-Bmcr=YBULuVsjq2{8VkyS zfOHhJn^2XlZuNwG*AY-zEnSaRVHIV1oG@Mc|6h(P7XUjy<``SqZ={qVM3D;iffg~6GY@!OGbivROJ zr0`6)f9OHbuh#{!v^gj3k7GwT$&%Hd$R~Ui%I}g#MapYI=?V)s7w-Dkhd0%w@oU=; z;ILU6;qveK*s8N3dw;hrpI7^p=Fc&E-m%I(*!-m(8ve`_^Vgfpx7nsxb#*8np1KN* zYK)M3-&dnOj2!ss>N_&;HY{PocphfoJc0Q#74+hqGj}opn}aq$W4jK54Xh)de0dK6 zr7rk>gpXwFEM(5qIojD%Ot^0PaA+w?VAw1RzV}phzOZQ>ek5ZItKT{t7bLu8tBxE6 z`hRV&!%o~R>p6~g`+-;UZQyAaOKFxhk(-@u3;44I+v|ErG+?%9kakP#_lZWMbrS9r zEmInzG)b<~-|c48+BrV`XWz>3Y^+p8{9*j?S7)`at&r#W$ieC30|N{ZnxG ztqnf6oX6HXEr4lv>+vpeCVca`*D#@V4w^KqD%CtZ(AaU)6jS;8?F2S{lqb|Sn~U39 z(^*ubJS8sfDzoVvF4=Xpu6~E#og4PO1gzK*YQ7AVcBN0@&&xJ650`;n&wmZFmAkJP z@QLrL^IEB$xHw|Ki_Z)e&5R0l{^e(3@YZCx)8q$xcr``VeECl7ltX2!s(&&!yYhQrs4M`Zt~U~7p-m219J9YG|_K7 z((mpjU{^XG+i&@xc~^G=AHK39uzS<-hetU7?{8yVM)!=*v^aw&|4_!6{6mcXuL`SL zXsv6UxDl@A?+5QShoH!A8~sl2N59Q4z)KTd`RE{lu8!;F_g~E++RsG$FES6eTAl#U zV;6Lf?r(&**Z-mT`v9G5brl(Ri(!B7UcCR25mI;Rr>;etVIpBGozp!kOoWfG#i?Gn zc0+G$@7tH3n-C>wej#kgbF@6a45?4F{y%?#d&Os6@9t_(kbKeZUkB+^)KeCpDTX6m zvS4c6zTh~cSY0QlTGkYpT*I(i-tu_KWY+I-Q<*;FE!6s6h(7+Q7=5Fa9C5Wi&KlZ6 z%_YpO=)oFfn&^&iybP|+m#Ak3iC25|A+&l9UTaoCbySbjT*+UzvoP-QXnc7mL>`K6 z$^!zd=zOVAINRh9biTDpbZhD^TeP_={+Yjn!A~yB*4vh|Y7294_2NLNakCe%ZLHxv z&K+b~UV%W@46A?ULC(ut*fKB_$v*Ofog2zsq4D68AB7j!=d-;!Z>(N*12gIx$CeJC z4>!KH#j*bkc#m#5Fe4|9&SauJFrST3f5x3IRp706YmgfrGlz|xApISkdAh6?{`Yqm z)K6}~U8ZLv^{g&w7VQJMX0WFGLpk(6av$mSLSASo_q1Fi&P}-j1J*5rZcWdE?zx`a zxGhOD(&`hqrX?f9cf`q69B@i6Kat&kEA+D8ENK4t$toX3`IOys)?Fxcw=#qtZtJ9~ z(fPNJ;G=*JFnGcXe9iu_ePLHLnavq@JGzZ8>gWvm&R^i})-wp!`S`iUHh36P9jM0H z@3+sO>S}#7cjzyv4)9u|?zm^pQ%G#{LG!aF-P`mji0i+m--5mtf*EpfK zZzkT>3xP?r2iNP9`(d=(LRsTRG**vDrG1vPyx-&nr(+IA^0oaM!kqNh+~w3Wv8)-F zck%$1z5A)%RV5Tw9qGV2W$%%BhkLO3H4kGh>T^{CVFM0CG(o%EqZ+dA=fpzC-G!--9XMwSoRl+v`U@D4fi9_l1AvfjA_^m;b1}R9^nh_@~92 zF|%<9gu0tb&Cf{wBdZ0E{3b<}TWR2yeiKhtsk|2X_q)0LkT@Bwc1!|Q52>~RlJJi3 zthS`N&a82x7T2HLg}*2fn0BX~Al?G%TTW|=J$Luw>YA*d%mv~%v^SWJi+kL{U7^in zqq;SP(U}mb{_euBYs6E|7-XHGv-KMT!}aTNoBdUA{J4L@s=*aZO*qT1KU#xJ{X=*I zlc(Um(2AQ(IDkI+HRX!zvC#2PoGzu5_AuUF0ONdZsGkkwD#z<|=E5f2@pUkGPP9>d zEiGp!=voHq6W))9^GDmmr`!(;b8%4NI2P>}fj0X?fL;SsOS-LFV)@W-ui2BJe3<>_ z0FGU;7tR!CqvN!l(r`gzK6t_;`g0@xZ%rN4UNGlXhX!L;k6pN@r75SrWnYRL%1x!~ zWS)H+jdsH|`h74D)z~|<{*0e|`qMoQ`&Iqn!yA*3@K=2{B<|mW(?5NJWIsAfVY4SE z?gttJ#62!$wNlf?@2TEs;dUC#JbdJBBPKiC*^19rbdk}6DsbxP4@heZ-jUCRnaAbK zA09@seuu@fxa(JZbl(D~*U&!WDjr?hS|01s2B~(~6OB#t+Paa8l}?ufQP>BcZ_1f)kd}{ZAJBywg=iyu*e^ z{3pB?UBn9ukBak04~w|EHX_$}A2eL$h0VfFU{Sl9?BxOJs_=Xgd&#bRsUi&)--e&V z9_xr_)#pR3|9i!cOko!F0!(sA#k&)Ji!H`|kZK9py|>zKgE;q}vk%UvzrwiKu56s; zBe8MUlgi!*2-qQ->N<*IT_XtTV1xA@?Gb++YD%hGr5E6@Llic3ttT#4OGjEer1@53 zWKY9CVacpSMz7_s%1W@^(%SN5a4aayF?n%;O`LH^y+%Z~(UViIzoFOK%1=H8u&`+i zj0rDc6U#2ansxrzO8>kVxp^C&u5JYf^@}kxZnrLf?Q4A9sVXD>6lsGuz_w>)tnm6K zFgioe+m6gB9P^9SI?;D;f(bV3#Jf#7aH@L)x#(D|Xc$WOTLkRD)eD@& zvc1FQ@QC-=t-n6DI&F;;lE>?4?J?fY6A0H;AL(jt+k`a+O~ts;7hp%(P8@YmgR}0@ zZwIYs^TYq0$M_MeF7-A?tFLOk(xc9vT;lo zeYyOZ0n%rnb9EzK(dhb|cm#IuSPWgG=c}~QAomY%d3_s}Kzh?SL>sTSngK_(mVNH|Ct~$~mbe zEYb}V$MAI2X{b8zEX&1EARQtIUj%V43kmLqUE(6pzRMB9{zVwP^ro)sh+g)?fzwU>a_?uKlp>~X>Wz~PZ|r- z%wY0lwWe$IH_~PI*!lD+Kp24!PFLfdx*I4?$MK^sVo=f_89qK52lew(d@g&RFVqlM zi3!P1sW%tESc9|pp{1{?Z^*biSf0v0DH^WIhNkrUS^LXQa@p>9AiW@8Y;265H*}Vp z#KI&UGbHg5RdEFX5{dGd6M-*mvI53dr`5cgD zLed>b>n=KZpMh~(y3y;7gW`t`E2{$OU!_s`laC#x@*v|b8$qMd?NND=ZY_c#yT2u; zng!BZ@UHGN*PZeXqdI05*pTNmEmC99zN?iCYYV$bz2=2Fs%Bu@&V;FP6bcHiL*K?j;qHXR`2Ox8SlrDD=GC5ptuL)sIHrwUI0A@Asekgt zMk_DA%F&Pbg`T-gv{m@cNar#&mxSMJ&1yO`E@BH?w)Y-k>IS%zyF!rH&|WPX1^?E> zK%W7{Ncva#6!b55N8)tW*zTyX8@3Mm+lAp?-6d9Zq)d4#h4&cQW`*#YXDapfeMJ4& z1C%C!MIBF6YD0RD@??TfIRUwA&f@%M!FsMo>~ty4%)Bg zA*_x60;Jb~a2`$G{8fC$ss0t_qHWJsbbe%xaHD%R)b(5qH{`R5R|@|*EjVF`pk5G4 z6VWeQjPMt2<}MUptu->Co*#Ow7otu+og+i*K|Bn1ddD+A_mkN4%zHd>@`i@yp51%c zg0E_~iIKKrON*LF-*L5pu0?k@#()!FV?+A=Wj5{4*QP}ch6}>UuU92+OXtk{?k6k` zVFOoZ5$|z!f&U&if<=Fhia)uAEma`<7Iz15z9|@0{ zW!Wmc`M4AIx%EXTJ@iyt2s1tPpsr;zAZ?9H2Mh6KbwYSas+x=IZQC+&=|rKB&hh=*TO}KNHuZ#%BuD zT2#g;FQ7RJz(bpdp-ro}N*<%N5ScB%u`G|{Kv<_dJ&zKGjQoFPhM>56sectt*s63t zsB2VQF@0u{Xk&YsJjP(9)0lVUMD(Z@r#=I`&3uq_CBL7PAgJycd0Sp(2%TY?-5Dt> z;A%aq45$a!EF)1Zi-jKE%tZjb)%~vC@1%JPw_*n@{@trGqy}BaQeGO^mIf{dkycjC&b-sNqdH>roE#)_r*HZGq1aGgt zBkJy60HkS@7vhx3aJ9zu#+_lx_b5+#>8X_nvTn)mJ=`Z~jA&HjK2EK*S>;TUG`*aW zdlzTq4$~<-r!kWLZJzo3vK54`wbK@?a$w~52_sLz@@HY#HA}I1J&+EDMLtNKJ*FI2uqt+(gMCIXGu6CNrbZd#3GJlJD~vKN${u{+s=tSZ z{GXQcp~@VJ@T&Lt3h$?<}azFr*@&|v2L<)yJ<=< z0QIdPfH9~s< z{LG~15;h`bDLTq9c-$r?&Rue z^`CFkb_~`7!bKfrHoVN)k#8~1(@nd(g>XQ>k~24MYfU}-21xs>oCIoG(LTv;&E>pH zmz6KTqrXj6uW8%D$E<<7d4ftcrsbG`qhMQBS zh;ww`ZP#F95xuL1yl`&_v@o3_o1_+M)AIcIcl}fBy^oo=`*SP0U3`J!G3^%_Z2-3e z&am@|&!FF`jbJ~G%P$r4xc9c9XjUggu1ll)AJ=D~%RLkRH|7j@u6e?|%%UM}Lrb2! zc{l8I61e1FDXwdE91=Bqpq+&uM(2m{Zoz@*7H=p^bOXS%evX*YZxugPbp==q?jvb@ z&>(6O25)pK7oz{yJ)=3YRTXBLwReDMzAhzvnYPE7ai+GVzs7SU^34~ zi@|x!sb`4Je&tGh_%vEz&l_0pdRrc3aRD+T>hOkP$6)6yI@_gjoM_u6S2T49!SkJb zV0K0U_Vtg0(2Mt>VA*24ax(|V{aPWjN4~+*S;OSE#V5swy^qA9kF9vyeil4F-B0c< zSqrga*J!4PXrNVtQT+3e7pP~}h?!q80uSSS^&K$6wp{zU0moyl&%m>nou!LkEAVcG z@=x=9G{=)<>aoWd>HYvO>~xSdb#rmz$749|zc{gAytOReH5tP<+wq2;tHFG6FTQwv zcYyhq#HEY*?C_8vY+tya70&TOx^6MX^D&%rTgK~ky@bEt9l_^OUa-T`Tz;ykm6Mm>pty2WHv)@ej5rEbe|&=Jc|W=H>bLkSH`rJ=2yb;Vz)DN-)d_-pEyCTIMe}L*5@J36+H{-Dl-m8 z)937wUcQ;IDc}mO3LYy>^6TTC>x7J<+44IFRo@L1&ntY+U94AkFUki1d3Rv|gO4GW_cB5A_ z=c~2hQf)K3SLO_K{%6D@>N7~6_W<84bd&dsc3_iKe;6`rCb#TtA*p8A;R7kK{NzM_ zE0jTb>rVVsMh-l*Gz5cfPa(Bpr(B%47S^m8BB@T~qt8`!2Y=m%U6ymv>vm&qFlPmx z+4=$JK8}*73Tr^u;MFk1sRyg|^thm2fvV*(y8WjkF;hECR!zFgKI|PM^GNRxY>|&N zeysk}Rj&QSb=0TwcZL9FV9%d7Y9n6lW{`Oa@$@^5IC^#n9HujrJN2B12kY8!_e@9m z{lstK_vWkrq;64)_B|(X2m;}nRDs39il^X@Me-Ll%I$dBctxa-o8mpVL|O#!OV5V zHq0^jg>Xm<%}v8#MU&pR+cq5icKsF0yY+;WA=Pt7g)Un_l}W7!6s-ziUARiD3qx{o)+#PhThA?`R9X_78AO0LRlJg&b0q3imcmD6O&8e8l-SPb zxxWo~zt{u{OJe;OLcoPQyuPNlBDU7e(?W=m&9 zn8~w4vqj#rSlHO681!3tD=g4`80R8OXWeCq|JBLdcuK=F3TMNU>udPzjQ-f%ryZx> zXOHikMKQiU_}3W@vg{#@j2ZzAO7=+C6%BB}7Gp{Ct37hJDJJc$%69e&WgG6(zIlb8 zabM>#e8-i6e8u!tjCz@EH>uFhuDV_>C^IC!xCsu6I>Cc2#bD&jsYj~uHhJT5tj3s{I&=OiY%YGeTSrpAF{(*^%p+Wz-zpJWc02$R ze5^6sC4wc3j?&EVqxgKFx%`>+sj@fDv@Arc({zvEq}lkH&cex)XW`Z<2VR!D8{MbR zN5e=uSJE+(#{EY9nI}8+aFQ9^H+?h%bsjs;@zf^gdRu>ttFPhn`J(ee-|wfimR_4q35!U`$@t?ZPzj9P+?ZDKBE*qFv3+v>kc)y)MjP5O@OeC z(f5;12*W4U)3M9Y5#sBlk<4aYJ$cRg1*lp~Yiy#U`qdhp-@}Nj*(t9>Q1{LloVJ7R zNd`-~$~aql&aFb6jN73je&=_xTasqlh9>#33|!)1*!5b(P7^Z_f#}byQz03N! z4q>)u-@{tz>pFkNZK&ew2}=_{>H_+v;qb1&R6W$a^9XnLqTk~dUj@Q5etY9_Y`^j~ z%)8_bv|cbarzTijnTA!9hM{J1fAv1d-*zF>cgg{rZ9fA|14hHx|5-ZoxSX~xj2lui zmLVjm3?ZbDdiUCfGAkJ~lX*xokD);&NkmB!B1w|UP-m~5%#zGw$~A{f88iRZ?(_TO ze(vSg`<}D+TF>)+)_dM_mX8oOE45{A%XUIzcZql3Sn>@utLciWZ^|C<@;jT|csP>B ziZ@PC8r3P#f2+Mzdxmd^pT?x&u8ew~tGXfv64hNY*`T<_g7!%H0Q4E?Bw8h&W^3Hm z3ZwZ^%yidy^0k_>r0BUAw>S-&7&XAwu0zSQ8{kq~y8k0(yxR3&z_gz)aqYFk5Mo)z zl^c(O98Sj! zv-LvpTbaEce_CLu`q$}k&Q?q@yvbo!V~iUF_BE{Fi|#>OT8K-67zu98 ziogqxwgc7tN%OsCyxD~&a(neAXw|}&kLwz&BTv;G{5eh@nNdZ033KTGox`g{5ETM7&GDupSB zFXE%>i%{*Is%g?n)G!!GOM%!0Rj(=5^`P%-_p;p1#MEf4mGW0{6`j-i3cXx*mt*b8 zyJ_zqkaj*4AxB2=f=gdnA$^{s_CEj8X7ZgYbp7F3K|IY@e0&XSb~-74$AWwM5Pdrl z6pJctzvuE@`4+tEcn;`TVXL`?+_$SQYt{G>yZLf595?QkJ>*9RPHafmne0IIPZ`}K z=7qZ%qkSV*v7zhIBSrmBNkB{pu3x>enbQeio9n>4pV#5PKZs3w8;HRvbXxiYEVV_m&fZ1n*j(pC#jnxm0N;l!e!VvXBB#U=P-$Q>ur z8{&;@L2D%rhVMq|>Akk@7?Ip$bp=*{3;BZ7aReRPAQa=T5WjW0AB3 z`@O6O+qFlm>dR)N-_7L9#cLFUL#HP@Ag|th{I#MUoV1xLRWEzL+6{xvZ8+6F?%gy* zORHTEyv??vVlMI@jgA57_mEzPhxnx6%QKFgSX0{lIuFFJEbQJl)el%{S{NIVmMM}u zUZcKiC_Vn%gpnR+;qm?}@bAe4jkpI@EqxnzK-3I2<5e;aX^J(C^WLzMqG>p9$t<`( zta`=qT1Drp-h^o(m!YPfPX&+Cc@tMgilH4Wxr>hjpKn%<#BpNsg-uE`MWun~pndU} zids}Urq1>Hm?msc-Qz$^3gk6Fj4O$I7_lqZe5VZCunNl6&TQu8&O9=Xu60i}fPCOZt zt$3Ys9oxIo!h7#TvryF%;>d{j=d%GY4Lzd91X!I#fHPF1p0F;x!o4(U_Y>qWNNGaCMJ$4w6nZ0AgWiWyB=l4 zQ?lo~eR#IcAb8`{kiS`;BN75*;oDVbm~UeUl%+ZO4HB12%782~B@k)$Lo<*D+M?8^ z@FTvi%!t@u!84=}u%r4Kai&c@pnk+j?T*9#iOpbsd@E48NEwEnojwlqXLaJF<=XI7 zJ#j{h{lw2l;Mw9Gkp8LxsNY%kaUGCXpw*}Z9Ff);U9Z>Vim$wC@1iWR5P~l?t>~3m zVKt~1^_jKlCXT;oCZuS%dgEC;- z`84b!M*{V)+CQz$t9CH%pp|@G&59}ScX-jAQ&yH|uUX0uh8K}o0|Lvhsy$#G^|!;t zb3+s(q8-NJjH3ovrEIv4GAq=YJCg0Xf0>c?!tm-XmER(m=!X|k!Zq&a{fe`A!Dz%Zo=9t=5^O+6td1h(P^gI}Zh-nfd{j5>GHOl(%VN*%Xi zPpZ8rL7WU!qe!(|F}4`idKfp*MrMcQWgBYdr1S z9Vd2tKH&Q0LNNx-yRf6p51q<9YIa968s5BtlThq&$;*>bcF+;uiR_C&vzHp6v_N^q zsU~r7$E6c*r5u~9`YTh*L_EFY;k^!Dv(G2FeUSLbs$3t=2E-X*( z^Z(qniP0JhgSdUTF{Xe~et>Z$`GR}{Rez}r;*mE=`8Lul5h$y&8PU0Fwn_8VO5}Hj zPQ6IK zKVI>bpt%Fk97JsSIt9pgE9RE{O@nDR_e0H|fND(6PCS96MSR{_E2+(M#lB^in94$w zm%w1~Y*gz*451@-L$$Z+oN2Fd)&dvOg|TcY8l%d+XLcSD`)@dL@;s!ujG7$_wO$YA zc7b|>Vl?tv?zXHckDA*<^bb#9N^huN1syA=IWI)aGNsIa4Q_D;BtON@ zQ`@UK1yDvpnh#SQ(EYncUl+t^DnFo_!4RV={gt$qNOLx%e2KK~tWnd;jLwOvnU9xS zBOdl^FT2*(pSO-3A3BRjc&Ag3 zt9{TfEeFj_7BlsCZ|%R*wE(t!Qiob#cYTmHD`&2_NYBvjQ`dmcd)-Tn%m?glR0F0< zV_jyPo?OykJ6~S316v+gArBtEfmhbek=>0pkT>iC=Kmjz`(_OL*No!k=`Jkqz%Ps) zW-agRi>2#i_2I+m-FRzYT|VV&9TE1+QdYN}&GWqXiNJbW_|;8zPM7}LNW;rntZ1VR zX4M*n`?j4GHy>_?{kHY-Q2Bi5@!U?jF6zR^Jx)a5DWgRSJsU{xP8E1i>)-k+=D0QH zHR;|>x4eG=y#}z`8jrHnKSR&ECUV3^kEI)h`$yWk13R zPIhwh-uklA=`W(rsd-R#W+gY>e4JUX=*#QL&A3Q}!No)i_(s={o}W^g)+Z3(Ja{dv z+~%S4nLW_=N30wl{{+|cyvR#_{Y1?qUeexz@JXi6t+Vm@q#0G2!_ny%C z&l))(#g2y@Vbb2KsWe+>#Eky41bL|ou9@2$^{bo^?Zgxq*JKUvWqkvNuWyD27TU`Z zo!)`xkn-#<18T?!{oWY<+ZWCiH{);Zm+(&YHp}mQD)F3dhuPWoPoejN5@@sBMdFu@ zFnN9o-Fv@W3mH(8UtNA4Yd6{rRc6$bfn8Ti`Wd>HbT>X`S1`@>3b9>44L(RuLYE^t z{^!&Vr@tMIdDGXqv@cT_x(o$%e!q1NVDhDg>~Azh7RC%gsuAGl*5Kt{gLsz#8}RXi zE$W=*+^;hr_?$K9+T3OrzC6bQ&5%diH9@yJdU8yaZZao$KKz~eTGjnYmwtM(<*4Hr zb-1g1YTpdjJ~WZtmU8;s`!>*;z{nXJQFH%>qdP{xzTBPIAx+}1{O;MW`sq^-pT6fguNSlrRjmaXh!$3NsQkh z2KKGYOU_vF3pIwql#Vk+bkkJb+OjH~UQdR^T*K6HE_|JTb-DfQZ)P*sT!PnU$iHv| zy{A25W8Pkd?R$)1bDWt;!7OUz*Hx_GV~UB8qcu9784cYyjvrXNhkuXn3+lMYjL z15WRvXF;9B*jZcAvC9ge+L7z7yW;OV)x`1MBVg$(dYY_bN3OpcqP&Ka2E^-c*gqLFM^KQ>tNM>GwIQ?KR>WAgSXj~1}An+;#TbfazHU-$3`bk^1=SBr12>6xC-1|s z_)xiV4tX5ir);-8;(pmn;0%Ns6~Vy8boUxFCe^^bOU z@?Cmf$3j_@(1`Yn?u+PcB_DjO#2W|C<7(fJo%)Ew-WBMkC$-`&dkxSH(&KQm-FWz1 z-h>-0NM&a775-S5DGaAqmpz9kN32y!ED?0Pc(r&5(s>JqhwsFtspH}Oy)g80 z+s;>)Fs$Jo2fNmem4}iK!WaU0ZTH{}5C)(c%@yqJ-^0dMX~AYa^R|Ho#-2 z=2!fq^v`x*43Oty(Zdfoy_N@5?mHUax4te^PlP`(k~wZMQ2F~-;s2=^jCRrUt{!?r zgEuZP)RykiX7eA!QVw6d;vy)2TxRnKw;8{L_79`*J%F6}{jEqHcUHEvDaPVo3CjOb z`Jd;I<=E|siDkXH5kk<+TzREjrhWy_t~&}lR+J) zsz<~4$9Uvhq=+y(inM?DTXzC0E$qaVpQYzCge^ujnOC zbQDV$({lx$tVLhfw;J^pC|#nu#+6!i?s)AIyiD*R?e=5y27hA3zpUVN!=`X~R5w8k z!N-Qg0_hVR|E~oWG}mz5^(*)xsvm4H-UL;LG_T-}9?#Pmc_}eMEb&n(6g~7|hz^{ko)pn@%9viAhp@7Fws!Np4a{~K zf{$LDfDUP#ee4^Eq?PQU!6l*iA$f=;@lhk`VizN3?er7=)d#ZqpVw3GH4wW)_u`4X zZWuBkjCEh4A!$4>U7U@iJEC!1p(wvMl{LIQ3XYBKDfbTDPu$apH!z9QNO#zlN!J*8 zG+r0e;CqucT9jWgp2E6(?$?)4S~nCQHtQmlc82){iA6Q8GvY$LKoS>=_hlJrfU_ad*aKrRVCFPP%lX0x{7~a>3Tq0aPg>6$M`EhM{yDB z==}rq%IA@P8Ou?30)hB|O|v>f`at)-vi~eDe3=HUzk&QzCs;SG$zNdG^YP1`*UT?) zh9rHXHFFRbS6tEtnkJJM<_q#9`n^Ru;nS8WPDz`-f!O4|7ErW6BVJSWgNE^*{NjrK zNPRDdHfarKov%U@%N)gLSZRHnSd;i2#m>soY~FrGeXVp(QXj~EVSnj$lW{4&WK;{9 z@-tdjC*l^^#=Aq-H~Ma2Zmgn@|Hp#* z2NLf+1@Kw~rvLpwVwo)R7cp(HDGn>(V? zXr-GXf`&Scnaz53fYni{o&0|1NNEl~4E9@qtf^9E8$9)t6M4!Xe9F zTN+r0P5eGrte?A#kqhvnp@-Nl(uHQj;(H6r^;dLc`n9 zwQs7Yi8s_SvhmXmWy7HQK+HfM@f=9ci7$?$)0`Ng7$uLc7a@J3dv-6ukA15$+INWR zS1pJ19EqQFCmqv}bO39tUySLuD|1>;L2Fi_i+k+K*v+y)d86kg@n+8!{Mc!`(kWR_ z8fghE3v+EkpzV}6@7WB6M}Qc(xg;)O#P>=Y=zEgc(A;M&@v1pL@6eWY z%+^!fBgmt%bK84Zw!02a&2x|&yPt;dYeoar5>O_g<1>(>R}ZK9nW1(1-GX~f4P@j2 zIONa~T+y|a&iPk43(GDB(hT$)xQTpjHTVU-gr={1QU+Pit_-RNE4n7K{3266@n9RC z^gRl9-rFFkKR{P+mM-i?SKi|0P-fMDo{O-r47Ln-2Ir^v>z?#C1c!u%+{VKm2A{Zv z)|!DR40ulYrIqZNHAYbW(CGc(lG}VF&w$9E@lYOLlYB536?6LSxQQQUe_*}1gd?_I zIKTa5P+sOxHU1k|jL!!=r+a(W5=ni>GNa5qx`ug|%7zvB!F1&S{_n0QZWuTN z)8e1PkAd@yHbnIZ67_Vj3x`&i2W&hTLrDEI; zJBG3u(#vw_xBw{hPBp)%R#y<01oy=THZ@NHWO+{zo_yET4dUv5*R8WKjk ziec`71gy6DCR(nzfW1H564y%*d+(_W78Zh@D^m`YpI8$IY=)GfzNkkRB9)!_3B>L? z+GC(vM2{hRuu;YZ{B?Yo(i8ArIS`V*HxviPUx1G0U4`0z@(Nl*>k3Bj&!&4&IUZqp z$%h#667g_t9{>J^;!Nd7Vn9l|I(9*QqRq+Z$4IL|`Lj#4T>P}?1YG-b5UEe3V%;x8 z69vr}G-58McgloU9kLrA(D!5x?+O88F4Ao?PJD;@cPBu2n~_2>(b>N1CucV`<=YYd|l@v^%mNXw8bq0rn0v6w+hNt#L~Gq^G$CguM?XKOOf&evi>1( zXW1FtyWzL^{4^C*e~!PLjl|Gc+~pTgrbW^-LHmrfcXD!ERmz?jG-I%4sy^GFZ_6kL z3E~%Rnw2wB55lj|l{#WfJZJI^O77QDa|tBf#ac6-G0O0G`ri>^o~3x+HHuh#3pk&# z;KV5C&oi(z6sSDCxAMa2j>n#v2ldrT;)HE<~^|I>0ux~WaMQ) z><^t8L_C!puFJoWU=3NE2FFjG<#uoFT0`D)IwH&)fEx< zb1AKbFC2N?8h5QNWRKer$9}IbiRD!utC)SXj=c>u&r#Y67ep-{)T4KHMu*xkAgT}P z`+cE2!YghZk9?k?GOVPZN26&$PAcE2y;d^}%4m!{TKjeUCe~l&sxmUov(nT)vbGNY zfb@v8bT|+z;q;`$Z1eQ9xYOtvQjUaieLMttl6)0C5Wj6ZM}2#PDJ~~o<)gOtpcxij zub(fpHv5}OHKQ>4Z!cT-(vfmOCaPRYYlR7Q9}CLiNb7+KcMb5B-$a!y*@E=m>U<%& z&IA~@_>A_UFzVEHr1^?Qz0YXgM059-G-uihPT57`jC`Y0GpTPs7Hd=| ztlzA9k~B{-lw<6$B$jw@3Jf3L2YOsRuRL5VbU26}q04p0<{wrwK}I^NW`x383(--& z0hLixhW-|`zmjG=j5r9Ch7((Aow*bu?`Jv?YP%~$%*a-q@P6L%|`%&)k0jeLx=*%{}z5H@wB%^tb*8Y4l zkcYtJxIk9($v<}NP7zb=dg?$6*0;b}(!7CXRz8j`>DeOf%PMo4m%*-1(Ll2)Q1$oW zbaS{i_{prxrdj-&}PorazK_fef8LJA|kRP8|<-Yfs+7o(jJag-= zj%F{M@+;NidhPG(JK))9kFc;{x-z{}g4PcMipm(x7;$BDCu}=q6lTnr4u|hfVzoNd zgqEYaLZ#D-L9fm@b|<`}cC*1p_I$uJxE`<(KbqRmahl-V16Js|=osA24#x8i$vA53 zDb2CV4=k}BE+emAK;lJCYl)OYD*Q%f^w&{cSLm$TCz>hIJ&iI1F&$dexWCPBT{>KlfL8261r|s5xhPPLva6LHrf^ zldbb#ubrwnBFkr9BB;4V+54Lq7u8qI?cmwR(Uj#n$%_$P@NeT*oP0%+j|<9l>}TJH zjIs&vFnYG-`Z!oy766?Fz0!znkTjZVp|-ZhqqCY>0Qm)J=hTX0Zth@Rk)>z5TxV0p z_CTt6ZRO|3+EFKah)c=f-IwTi{r2ne;O^<5=8!853$o)&6Zfu$fLN& zZUHu)S_NoUExe~qWRwL&yAmDEpZBW#PWAIqH?GfJq(1}ks7~c|+Fy*n|AGx)m|o#a zb22VsonRBbWa>lGhA>8Z29(J}r^zFrPH;3Tf2MwAYR;{ARB1ZpVVLh+AZ+(FMl~-{ z*}`-heQ%jqNX@)RlYugkqMBJ zWUlFhjT$$gdnmpW9je{OrEa_M+kz4DV5K;blUa!it4*R)^g!mCcU;#jsV(?~oq*bb zh0y2xJUq0g8FW1MN*HdtfR7S7^6~HVw4=72WzcU8hYvvPmvkLGOOv3@vxTrZ^*FeX zrSI@9UnvSRP3e9~^VyB9kq}#{E-aiFiYEJ}@c5pQxIes~eB^l*epi}MxFycQ?f`rWtrWVXx@n@IdT0rfar-&%;E@b1jxPmn#SLjiTMbq14t zPjO}V9{B#UnjD*V2Fi^+xW%!DNNWV!eav`~G4Mds2XGDkvCD3y;QNcn^wbRs82+8^q{dA5xNNLPo2d4LtoPU;`G7!;vm$oH&vAPwuWU|0VJ2z zg0~4m#FUPM-H*4@n!ORvZY98y_>GwC?J6tvqTV+g&Zm0UlpY@Qu;G^*!gA_cY_#o` zcx3(_GkqK|tgf*v4-P@^4pBOTet_0pp5m(o{j#GTCBuiZ0?e8?kn0M!v2=@;+;-{@ zpc(-u8$Glwankkon9Ktlm$7TPr*QcBU)t+O%YbT$FY#i0OZ)-3F7GufuW}KF!UlN0 zXg}KT^Ofg28}MzD8sx$T^?J}LFd-vGEhaT;#^&`{FDcjy_x`>^eR zD9{x)z;VCp%NeP4q_bgrh$(W#aGPN$;tQbY+C#by*Z@=9JIduLIWQ{p6y3A!3Whw2 z!|u`d;C{qt=^8x^9g;1iF6Nkgd7}>>maL(HLn5nD`8$lCP2cOcag?d?xj;3^hZTj% zu!2tX9rp+9o9!3YJT#0~r@Z=W=omqJ0S)@}!r+vA_>n$J+I{hoo<)uM*S>l()^{)j z#_z!Jv`D;X)dt791&Sw2{Grw}`o7iT=}>vPEpO6gq^f01s+)q%``Lh1-X^+Vs}pX_ zu@ng@FS0k8xoEeawv<&oETsLu--2`t{-mvxk!~loP9Y4b=BdsuQ7^rs?@=9qwcd%C zYE@aP`t>tApw$D8+zjyFH3lN(;SKzomV ze458EypmZPYGzgEeHZ@&jL2czbHm_#*#Yp*qUQ@=8;m#CRb}tC9)^pxU%{5g`W%ie z;dE-8YX4kMV-e87!XR=yJuC$Ih47sCs^_}&pxq> z0Xw)M-IsTjPXP5&Q=xi`>I+=+y27mJztHb>CoKNFORhA0495%ZVf~WkmG=|dhw)DBCGzm!?rf%M51>DTexGA-zy1#J zjlIb{ORcrA0rPqH=uy&&?h~VQ>dVNjf;3p}^zOm^CYo@&FQs5Qx)EOmc9JxdGmfOQ$`kX8?HxXY2CC!7r#-Z1C;#xBLGXNQQreo>5ggsunkk}g>YsyC@71?h|kP8o!Swqv2j#qAIo z(FnTN426y>L$U63gz?tXkZK9n4|0Qu{Naq&N2h$dT1h0tr=G#|M`tT^+pIhWO7ym~ zbG}P35$?l{x;MeJ)di?+&=mijOI03%`De_KGypc^Idlyf38Z_ld;Bd%zN-3+uZT;7 z=xe6vU)vgATimbcRim44yV)e20O9qU&F)@Vl7C>+$Zxv1iJf3=S~VDro583-0S+1; z1iNbYf=3BQAhW|=bnUYQ>nEPlNKeJg5^vmRu^qQ~-65^rBj~m083kvcn}IWbx@Rcy z#3#H4YuOsz2;JciwiUh@<5^p-t2q!z|KR)nMCKKjP@&iN-IhZ(F(!E? zyv~SQXIb%_l(mdJfe}l=j=J%>HRYeN;L!xwSgDHA47hWvp(HH?@+eeHN1nvm$5_MH zGiRaYKA(!$T$eW+>U}!`b;`qVmYWgJv)T#jH58Z7u@OfsgXPw1kobYo9oOUu-&r_& zxhGH!iTS=s_`oj@bB(QFxIqyvxBjauPM4@WX?56H_VH$cX7Q;x%07 zb_n~M1>huT1l6shA$m}uXll@=!iV*~S;85*&kAWdR~+o0e?c9Wbf;&glIN>lK=N(n zvARZ$w?Uhko#j`*XG#a5-HG$rch2b}?XB*U-B88aU^#Uz)oBgc!(j||2yG0rY919c zy*dKb6Odl<2WOHoG2KvV9{}iaeavlnU-lxNp1)YOv4S06SalU9vjd@v;cjS9Z$1!v za`HMxwI}kdnz9An^!={$zeJmvnN0CiLx&8-RbsB;Eg+9z4yAq-9;Es)<60L^y{9|v zb5^XT`Lbf&4W)y1v$dY^>BD^Xspn%Hh=FWL$QiHm{>G@3E$--Xi^}@JG@qp%`UJ+i`gLMha;|70AqO1zne36DBX5wfF0G0dcq_ zJ{I;b##i)d`Xd*4drxnsyptFSJ!$0Yq7KVQ~b5?^JL zf0BldPGbinU4J3P&Q+v9?J&q;({YtICfhgosHUs&mo> zJXpz03^(z@VKc|VfznrEo_R3rvuH%DoQN~W??$Rq(R*frVjc|5`lI-bp2cRv$uH@i z`I2;^7)hhVyrNc;xPVn@kehXHT6La2Wt|we{1#ANQp|$-(Jg?o4KZRJN!bgZ_q?V3 zZP^}(fB2XZPnDSjWjWT?vIOp-Yp#oo5ft~>X|-fGgEP3SdmU6gf1~a&WYwm#wBJvNv3mpY zJhsN=H*&w;Y3j(#3nHMT^AqXJj%uB9TRkM-rTZdnW2Q3?5dWLe^EK$1(q;}~SW!(D z@V60?ma1%`DISU`eTbz0Lgg5$E4X6oR>2wlJ)Ttfr_!?0t~K#*%fpnb>DrBVTh!07 zZYDQ~1IEBzy8dLF?G9p|mRw6MLbdkN&z4gz6x&ipGnIW+59dABp~}!Z)0*?r^nqf& zuAkC3#E9-#bLl~C{5!gbklRpI!^G~oiu%;=(?v%N&#C`7y@vd3 zdt4OC&!|_F{y9>m+{nequ!8JM_Ji;10!6z89GOd-D;m_v> ziG>Q;yrM+W_WTH#5Eq4Ylj4YF_mb~e@=~8SqDC)9+!WUfzoN5srO@$}Qa4r4M;D zUy7s*-I70i^afpinz3pnbWLUNaHKU6d%GXk$j8+@fl;5cDlwJtV2)rkTM@*r;*R?r zoUgk`y&Hf_OkMe*iCjlnTW2%f1CnjtGR19EbQ(Rs=7ad{e3@oXyFhvRThDQlvJ~$2 zdaCs3KlD!pLXg({W>%X?hBCul2%4 z!xkYOi^}E9*nc&U7RVC4Mo4*2OR8%F?F$Wnatrhx-JQ3zK8h|`kgakCX$O+Y!_Vh! zIB6e}-=oriAgvHQXU=49zx5??IFhbtp&jVHUv`1avZOtzoUQbAYxjdnW+{<(QJkg8WO{Q6L#{ zEE0oM=;i&0`=GJ~%?udL3MaoRG@W7wN@*`El_Ft51(Z!yGhlPsy4d* ztSi*~%HQ2Us+=;&Ynz%|NaAjK=F=P~@$U z`HGQlN_AeOo%~*WiPBD}O3wwSyewMIHsX{^iMRKvx}y0)OZ7V4o;9qhvW7Mk&cUxw24j^8F8iHYQAa0o=$_J)nJ3&` zj4GQfc6_5dS++wcZzc9a-_p}q7Lbdb@^(U{6XsAkDMHdr1}R(cdyo6cY=IgKM? z?QuiwBvkVzI!?tsNO=LtkA<3}s2OBR?W;`HB6(;$ zHs<~f#eA^9CQHyPPozGkj1`lF#%GHwJc+U?5J%EH_LiDq2+|KpSyOudbV2Vm>+sT| z8WntUz|&M~YH$!~zqpz`5I;Mqj7JOsisRLO^_+c3d5X9?nL!>ecEP1xrgDd z&%~2C5jg5`n9%!l1$%n66U*N^$d9M#+61pY@N;Y+ZhvtfZFkUjs!eNRgKhKicI#Lz za^Y7x4+bMmenTyzqv$<9Q7d)*D`Jb2p%2{)c>3OIJZFvU zlo!l`p3%KezZ`)E9qY?Omlw*5Pxs5FW`=V6i_thhZx1`rJ&;|!`W_~wH=fLT5HC#{kGfK|KmN?iE29&@5A$w_HtY|5t++-SBG z-D_XPdg-_E%gGE~y!m3Dn;eQgPh1m?HlD^U5o5qPUr){mu;rmS7saLJK6vWCA1wd= z0E~98!!@y0p8nDiug%)RbH_V^a5x}e)$9UWp&AT~eS^7mXNoT;?K$lM&zsNqtIMlQCb^9{mX>)WEETR-`fG^^U3 zS@OivGH4NGj9U|)(s8WN)ob!dyS-^2zB8%AU%z)}lb4Nz(yBpxg=x76%=E^PZ_lu? z&jdblYJoUXd$uq%rSErltp!6qMM2>4Cc2kxVr0UvT98_|p;$0uAg6u8m`jW0?5Bph zgK_B)(cmJ+wsD94q2XBbu``Q2bwNCScO3_}v&W{_=w6e29S^>+RHktcXzWuAA5Vu1 z@%lF2c(zZ~FTWZ%fc}00k1km!3-)wC?^cy?!@jyQ#N{~lt>q5SVyJM1_Z43Ik4V?c_h9kF zo!D#e1)%lk0YUBg_2`?}_Q5ouT7*uv&mnKZNi?t5Q9f{O&uPDKnSl>n?s-d$dNc`z zLue#zL$1AHBIKc$} zPCVCmE3+?(fVXyim39F204z4XjE`@|=&0|2>K;xSHQ*0dzQd?F7l5<^hj76U|Ca>T z^*it`qY|Xc5eqr9@lX29cs2Xto{Yg4zCm;6rZDSIFHV2vuSN&K$peqH{RZ-2`v?t&CBdO-Q7}Gk7O3;c=)8)zes&7-!^$zJQ+rJ> zaTQt=8q@yn!mQ7>;8WTSPOUJ*?u^S{yg7ty)|aOAU8R5dI@GV`!An}tg$I=l_>zr9 zVs=y((%NzQS$tl9DvX<#ghuiCGC#Z}^}U~V)Z&vav$2c3x9$@7IEFHN=XdZa;ScGI zBX4MI%lEt-#ecMmhV$2&!u?}irAOb_S*p(ut#t$UGi`XC7EW+0#Xw}=%@_N&26Cz& zxtH$6;TYQ(CtUSM%U0dww3hz-;K|uMxc(*hIeHZLp7(>+Ybk!7TbtguzpOid1O7}r zsO#k072Y2nE#nfynOdKLanT~WM2{N>*5{k@_M%IJHG(vejlFx6^z;xTEy&&qL9ljV z3#=Zn4PzhvVWfl7qNk6feZ$keD#>a-Ve&!c5Ge0+6)k#xfa(DaK+f0-6JBS-s(~-? z(B)6KqfaJp>k=!Ty|CmPPsG4v zS5)RaUI)1o2VjOQiVK|nT`{GNN>(nc!;l{HTvQl~g-1RV!#1Gl3_L>-sfcHoA zpp)-$k!xHZca%SYz|40@-oiZkH>Bt1_7>zZeCdM@&~jEP=Fn$WCdr9ve`q#(L+Jvz zPPvN8zhfHq#j0l}0o6S0qx&U~c%^C28%8kYo7baiV@0rFHbvy$7E%m4l+>vTPBb~s}OYe#+)w|)cy~c9t#trDvKMHex7qf?x zj)O5JE8XA0 zzuBQH@sN7vETcLQeTpVSpSGQ0MnD>x^oj%b+~07d_G|hcYayB>{a{;)4vFm0mE@nY01;khl_>8O>t>H{(OzgqJ1Z4)TlpNxynEl1VI zO3R6VfG*4hS_^pCU^7gwwg48S`(VQ4Ig<1iTD&u5t0&CEsOFY@V2KdTZ0qrtn=R#l zxyg+D7!DWzLOLE@+-Q(PJ~foFk?V9Vn-(E?E_xk2jbXtlKuo3Hi^XTrJ)kWXqWjq0 zaMSn&^+*-@dwp*huOH8QR<)#Fv6ib`jq#|r3y_ZJ=y+x8=y_^CMV;;o$fx?Uk@@tb z&fiIx)ze=_)@ls*oc5vm-D7`h;FCFZ)w#j8o9<9oeD8(}*>yv4YZ@;`(3u{E;|;w8M!sS}I;Gb{-1T zhn>-)j(dUo66zTnSESw))W7(`vkM!%EKsPLEj`hhudX*hZvD_1-G254v+YfBFWsAH zg;`5RwWjKXDZd(hHc5235{99o>h5^u}rrJ@fOu zR|R{03ZeV!w27f}+KJ>XY#n_j)X8?5Bn`%(OIF-p7XnltO2gsg!5WfuSnKH8Um6Wv zi6P~6AwJ857e`pZ`^hvPSZ0p?jt|+}mli62p!v#fIOa2X>I8q7rVZkT(~m;ntV{US z{=AM}2S`8A_IY>3<)YAc2hjRR^`1Ppog`LNtRrtubbvouc9=cS8Rx951PMPLSLg`s zaX0EkWScpVmqOX7(QN&q1fkewh1q^$^`^qv?rDW?>3jbNDQP->eNYFD^f`Og_7m`I zb&RSZF}Ax-yz&?WCcVs<^(6zXUZx-X^NRxORVUzk!zZvVrmG~DqkI3J1Lce4yFBT3 z0et9vO=Sr<9_0^97wm`jZ{IP`|0Y7~PR5+Fg`}R%8c|S%5A$v#ckj#q(oG;{CLTMi zw23Jm_}F$kkY?cQ4z{x6v7t%Da>C@-YE z$(0`0ACO;>@7yApVs=f{`9VD6v0TB(9hHPxls&Z#I^P`_cW)yRH!A z*RU;Ujc)P8rjYTbGOrx@2}zs5Cw>O#QoahFdq_~$vHNTavqua7{lOA|}TuCM7;SiIp08R8gI9Y$;Z5NMWwd-HeGdq1P+lfTbe z?zA4l=`%0&8hc(5Yl3&BUAH;K|qMGWi;e zoBNBk8k9#f1#{l7%tX$;-%*lJG3sd`-z3g@3$;HF0<(bwnOVjy(es46PUU;z75LHS z4O9P3UX5xEW@W^o=aUT(wCaZPF8EM%o;l~ywF4dVFk@FITDNtA^@$OKx`ZS37^5B& z#LbL2!jXEF(SFFU_I(7^j>TQS#N;zSwcfo;XGjg|_`kR~%4jKCM4`O)vwhC;U6u7ZhL6F#}~*Bwl0x z*1X0GGb1U3&Bq?MIy1%iU%5cbR<#_t7BG7P8`z^ ziPOa5Z^oQ77)SV}1ND%2xp^;iFb|Q$Kd^i7WEj!elX!fvpfzIW11`hB62_h!??V3S zPuaX!qdsKc)|!w9g`&#C#CzCflohAffO<1*;X+2Y>?z}$F}v^mv}>6Ta+FOv&ZB!M zu3fl~S}2vwV;8PIN# z0sN^}l@p7IL2EOz`#h>GMSPm#biC)(0&MN*nFHyLiaVT?&X6|Jy3P@I_a0WfqgY-< z$7bR*p984QL$S8fT=hGfY}bl_t^ariW${DhLlIpT%jM(oPlP< z%XC9GY=n1tfJ%qxJusuIHKz=TMThm}rR5)VJ%XR=djB}jNb|&pEf&PFH-Yjf)rB#x z{MSQL-^#d}rGi+R(QAq~G=}qCIgG1}{?50QR;B~goi|a$OlgFQ5C7Zq4=PPdccQrq zw)u7m%WD0PrR$FC>3ic^N;D)^(~&BBGSCN+c1|_?+h$ zMG=wA5ZQZX^fiCa>Gk`=D^#C*?>W!&UiaQ}T$Hz9rg!V0^4a9&w4_Thf8q-*=@hJ$ zH-R;(JLHs9{7bFVR zgSzKyBK<#U@h$3lsLIWvhD)*c zRS4dJazU9S5@%tZzXyQkTI^lfU#m2W?<-BtpMi&RsCHQADgS^nIG`LB zDU*MeW17?jxZ!j!Xrp6~bDtV9%1A-+)Bm}Flg3frORF@=t_6+dts1UXdT?x??MONd z$tS>|TgQR)x~59M4)&sRb^|(hL3)s~6(czPdoQiB%XhVpJno<2K5j zvN3-2z*zkFpa<;8ic#~xY@J`Q*vfH&vYckge zN!tKXn3lQij32 zIdMGb;d5FmraRv|AQv-RO8oK6QJ!7151%&)1<%_Xap0o!82PlX?Db_5lzV<4ENKXj zUgbjG&rZ;=Mq5$;%TWG!^FX%8*$?vi>|_Dv8Q@2YpEOQ#;`%0+@olFV+;oN3p}giT zk9pI2!wqQfPA7dhVew4!;r#{m&W791-)22t#{LK#8%NuCTYLdZVY`RCP;7X zXujyeUsz;G>kruf#^T+la9`6)IDW%LVK^uO?qAvq+Adboeuh22w!9y6Evg|h{u!hF zzULzBKp1T75()JhfUDE zd!(k@oDJ}zU>kI6+FXY3@4^}d=gGt{%zL_xxGP2F$txclLE9Lt^I$FpR9 z@9%h32RX-u8_m-Zto!FVWTrjF!qKfH&ACux^Xd!Z#{Zn*ateLHWu{%vQ3OB*flod?arOmoZp)Vw*#N!;Cg)n|-o*+8b;w3FjC zw13G|{i=CB{^=*ix7#b0oH&4k>egVQ!6ad@^`jV)_ei^I+bABByNmvR1GL}XOIG7G zxnJM4jII}Br=+lE7RSNm^na|(i*k_eO+;vRx!7>cowwfKNHVrr)j3@26;8_nZst^5 zuwD0qIAJkC>Ytv1RBxJs9VeOUr!z<6MEmUp?5gt{jm?P=)nl5xpQoT@%Wzmbt`j|{ zpo$>xon_UFi^ko zJ}+;>`rjR38nct9h|_Fa?_>#sH=v;|!^=l%@pk5k=-&S&?pe;*@7n{gEsqDi%snt> zZV(Ud90D|!tV@c8OiOwM!{95$nRYn($~b5g+zlE&=p+>;G^}08uv2wD zZt_^9KH+n2+!7vc1)778jv--?{5tQxAiU)bYcJ+@rL<1hjq`HU=dE}#t0CsC97(l2 z5qs_uTEY<+ImCy*SlmwbJLbb3gaLn3rwt@W`e6NqKUrmXM>)_-m)7dI2|IT*h4Xv6 z@hQvZc{5lr{#PW2VKFB!}O%PYj~M_qWnhmqWO z;{`gOHNJ#TsKMTIoy5QSrRreBZnkOSST&%Su)}WY~-VZaozr7 z?Z+DM!cqf1>T_+`eMyNl58KIKf9{UQyk`|t{rssBdzh^FL&}BdCWRBf zd&y3zd-3b^ZF1!PwvzB1=^Ehpr=?Z%S$Z%F=Xc+yup4t67qV|X#}g-bwgLAv*WcF@gk@0pxIZ{8XeJvU`9^DdM*;mFG(NHb?_Ylny(3#o>LqqI zGY}LO&~ZR}UW;_DP_V;=dNEk2aR`ZCBow~(A36$*|ILFsYp>%o!{<<6M#1%Ox;Sd@^_pEHH@t$vN zS*vUHc$=8b?uxUoEx!Vt9Zre1ydmzpF$oEqrNib;@MpfIpgF=C^VYEju7`wsVJ`Lg z9weSneaZ-mrrxowfuM|i#2{rz1KZNQ7r3+wsw<$=x zh~qlgVsPFkJk!BbPVe>#yw;9~wdwU>!?hetu9c$ccThiv>Llmu$4t+qaw(ORC8SX9hz}VSeL|H`%jGO!j2EW?ENLO_xu0hgjJZaJ{ zq-&IY0y|*C2d9|sc2^!3Zh^N)#iK#08R^ZroW>c`hW^38<|a5ROpE3t_h9Qg0g8i& zm)7Ex9ku0`J}G#)-Er;878UHY`E9%~uMk}{w=iN_XP(i~N6`4c9|B2Cw!%0fqCQ6C0pMMpGa>5$7A)QGE`VUsg^B(gQ$T?e==X zRHW;aq=|tr7gzY$GklQ<1=e?wmbitaU1h3Pc~|Go&@f_%;#NVJ2})xT2cgq~1H|Kd z80mU8OV0}bIRAnM+J3TT0_7Slo-p?vtD(ef4TN?CLD-2$y&MG9ik9$P)_YTn5#DJ6 zwvRwFH&?cH+^HPmU`#z0r1S&nt`r>6egX!JjTT97JXw#68B$>%>1!4e{R8(WzgKmO zr1{l2Gt$gRI8Rz^6})QXAxX>Ol-zLf_wpHNHm;65eLqS=*AGLdzNfl4q9MJ6go#z> zs5J8^M_-v$u!hmkvkx5xl82e7DTz8DvWEm=vCURoREyRoqB^`=V;-YE){rh|$?Nsy z@?#H$sg(~?+U|MFt60%6oiJpOCeAcV{Jwn+XD@HXhZ%OjWe4jb@gH;9kcBU=jS?Tm z_F<%Z@yC?QTGw@-2s3sI!*!1own)Mt*t1+{vlj0GI&ZxGK1LlIX}-YV?s_2a1I0GA z74OTV73aVdn1L?$Oit-a_}oNN%H}}Rykl$N+Kh#A;%jy@S07^P~uoGrKa_M(KI%TjMNKI)e1M{BJZv z!bd#bc{@}5OnysHEg+3`&Z>%bbj&X>*tP`vOsDrY^|FCH8V<})rhNqV!x5hhMtCI& zZz!+w;^gH}@@x2c;eJs5)z=~y$lq#mo44h4|4c-Au7Z5`tG`hec?Ukgh66V3wRd6jpYQg}lQ-nUS$xL%-D_&xld8~n3ritC0Q5C`1%(@cv4)LuEWUJN%CIQgU_lspUxQNO;fpgbi@xBZI!4z5MggZSlVPsP>RV_Vlj z)K3R6o!C}+B{6Yjw&u%)wYYS|Nl+S`ysjX9gGu#r*v(U$iQ7(NcHDaLBJB(5pKmPs zqa_@<97=up&N`v`!}^JhoLj4l4ii-)KRZ&UTf>2pUWBjy6D^ znGxp0467g{9U)GY)P@<4odkJZj67OH+MLkSs`{EX`!U=+Wy;ehm~isd+Fhe#(WF~G zmIWOpeVd2Gi`;ElIq8WRQt@Zb{_8^F*d^18(Ee!()dM|4G-m@lH7paqT+pYvUgxf3 zz70BLrc*sxa+TSL(wXYXeSN0?2dW zsztY;X>3DYYqlr751Fb}_!P_9(fm6Q=DCsv|A$JOy>0MA{K#^K850ModLqqf#sf`e zQa#N@<##roPeGbv5#NsXJnvJV(;TBkHD~wL-Vb0z+8LE`NjjcJVKmJTj=a`_6K^Vy zs->Q1K1*xL!L&{bjS~w>u?Na`fUuqX%t7M&kK*)wS_d~{E6lmP3CXJpRa?(W2$yz! zfUO%!f%GhH?A>3ksz}G*%}!7cn@G~`xU|M~@)>$?CF}t-a(N7-K^XN1)z0Re1-ts; zfX3GH!pCOXO?5Uw?7JeZ@~ksP)BEkruIjTDmr`cHGD7H?c&go`U86;uX=mZNWv=Qq z<%6+#wuc~Hr;g7T2X^C>0kDH@ER@cL*eNYp*W>ryC_h5-`B>6D4?jG8k0lo;Q@%ip zk&=g|Op36;6)6MJ&^ajlf{D6iB9^6B>9_paW}36bdX!BzlPZhb6qBy>DWh{im4T>h zCOxesy#!A?K7=jzgIT@@X@EML1aYjk%&QZUuVjQ@g17>=_X$IT@k~T<||%A6zs;MNzXRg8GawH)^T?7`5T--@15lTn8iu!XitxI)B0|L1F2Gw;Nz; z!ab$&7-6!)3V8B-S(WcH@K1xQDHq_|kAJXzKXrF;X| zZF~ue#iCmErOLR;<0&2oHHPXORPL#CfXYsl$E0(@`}Or;UrSqx=5IIdr=Qzx+ar4l^!6-AW7IXMim|BSG)Z(%zF33e4Em0Uk0kdXdMj{6 z>&E;m=z+=PzR+*}WIk47+_Ub0=mxNw z-p4ZQJfGKCG62tg4?q{+NO?|U!zVR74(IQ^g+|k+!-q2on9SR%>)}DVHqifyu8d9G zMtkTQLs-IM8B)=S`4x4C-OYor*+LEfm&@fmT9ZNJ7^D3dIuuVBCi49QT;x>ET4-Hi zhaakyA&nD8I1G@M8KwBfVw@bj@B%h^J{R2^CgY*o8z_IYao=$K9z;Cdg1)zg!|bhk zeBP%WIH6z*Zg3sK^V3W<^&i-(a~ERCFWBVksOAFi(mk3MxBAJrc|w~qN>`TO83w_- zx3jiRSD;Vbp|W?6Bk(3-ia4i%|54X} zCxcVf(E3PEq~oI*ZSZ(#0_QV6cyGEY)}zyG5dv zPe)F5g8TGCup-MC$1i!#yjEDrG_y?ie)C^Kc>{Bp9&06UUa^-pgptw3L9*-VTWBA@ zmk0TH<7%^1nH0DK>aN%b&%EiG_ES!h&VhYh;l(>_e25wT-32ZDB_|0E1H z-Vm+-nDf#;HRyS@V>rtHD2({ifYZ)&+${q5zeOfIDX?zU`1~v^)~*|Tkr};t4wfI9 zuvI<(1NvN^(R8_d_gs(nXzmA8$EvO|-FX8JN;g5Lyx!cx(uhsf=<~E&?=fT+#|zK= zrH|eqS<`C`)HoRfGa8xkPG862yiaxcRee)XeU#{zBqE-g@oB{qpv|3DSh6UXr;G}g zYi>Oe9VafveXGJH&8^s45C_dK-^Y$)bU6O4z{Dh9ncpH>(m5iX7Yu4$7xtdLgoYDm zV)O8ZjOr14EzpxaUc^HE2MJ)f=ah)_+OL`56@&4$qLJnY`$YQ7<^~Q-OY8SfIkgdJ zY$VOU%s4n-?9iCWRx8EBDu6xP+QA#ZNGFa8`I1y;IisW!igq@x$W zci>pD;aeRU7QF`E-rol8#VI-d>pRk3^`Ty!cg%BaH-01c1o)cKJ`j{~`W4YSiBWUl z*|j`;HP?);(Ng>GcPO_06UqDB+#!lv-lg{}T#?2GRL@SWhFJ1t14u7t-ecT8Ja3ty zy}4}^Qk~||x$^89I)p>-1vhRg4_?^`yH?QN?6ob?t6P8ElR1K0w?8Ak@2ZPEGrhEZ zKYqZr2XDZq{$ZG~e_>S}Ra9QWw+&0+OX4Bq#eSS>3F1qN#JN*VWRvx$H7zH&vSY6Y zh>aJJt=nYI4`jPa+ndJxwb5+ID_+Wtg2o`#3cUIeCL5e>A(xsIVcPbi@ZeAb?)a+% z3>{g_y1I|%G$;JPOj?_3g%xc7ehalzqM+~5V;XNiQ+mJu0%Xy9`c&KUuE9XCFB^+f zy8go054kWS(UA6iy~+|FHstfqo`!lmC&;XSW}GleyVS%-#@q`)^RCZnJiEz9we67R zMqWQUP;8#EUUaMT15|Git^nZ^vz%wcpRKI}r-;{8&wq~X3|VW9_(Y4dg6c%hv7M*t zP-E&?3!cnAj!^A5OdoqyWG;>2x8kqB$icFzX3|frsltk)RXgz1>|_~T{1G?VJjC;{ zPUvyZ8_e@Zi~JT=3bUn7!W~AnDXm|aa=+gFWVEpxcRjfu^NN2$b?@!`WbN&&@3imi zyOpi`tkVJb*r^h!Cb9g^ZS^yvRmVqgRev)MxjYH>{@f~utj@%h={u<}|1gW#)!1ac z2Mj*AO5;dox@X_3)7w8a0z!BT8@$ z?eBE6-c6)B=WYf1()Di%aZh!4@wAydv8jPJTn11Hd1Q`j%7QL&s7h9Qbk+&OPF2Oq+Jn$pYlpQg4&$lg6LHhEG+bymh!egbOWneV zYoT`!CwbpK5nuiNE1+6?47QI@HG#wvka6&96)suT_yz9*YVr$-39S3hj}YkQ2s93O zYg7a~@ZSPl9-m2EV<~SZOOyo%pZ|eY9mhiL;;XE>whdABs&b0yJ1OoYw>u}S$5>|a!vJXo1jrkCq6a5f=y1ffl*B-;b_`dulw+J zXnJB9+U+E%Lxb@$Hwyf@onn350Q)p$u38%!B1xeV?r3H#YZ@6GzT93noN4P~7}asPmmxcK7|ZO9=5B+e4Az1*t) zf5x+(j4&2WQ}t2tMxbnj#EFpc7?_&R@THru_RwC8I7E^@+?DK?iLG!hQg-F_k)8FyWD|<;8peb4TALu!C#f(X(P~q8y zskhX8!;Gn(XdKc-F5wO`rRc%78u~W3mOjChATU5 zg}d54FsJzhsLcF;KDo6uZ->Rfm#G10VYwcR|Mrrdjk?3Tecocx!`etaO>=ULtsC5% zIhp>4{2}#r9{R!h`kNp8vns?5FlUk}DB%YG%imr+4uXe)T1=fT&^`&+5{=`d}OkqG} z`-Mo@iER?b^Z$M}XQW@n5QDaG{7?HTJ-j;n1ncR06W7<;MJ;n-{MQ-yCoolMsBGK1 z-q`p*UnHyt-uJQSTA2WYe@PmHUsyT)G@ILgC~?6=Tt)BK&{*(2?E@4~sIg|j6Rv2C z{}!O`V{>hbkr&jQ@uRzGe^l)T4QVb->zq5JHJ9ZyvSa-8b9-7ngT z4aY}fa&B`OS9wxyacj>`-sua9*NLy`GirkB%SroEFwxIJ8~ecvI|aQaKQK+UdrRwH z_zz%Zi@U@89$B;}MF45o&)T3c3+{3)21z$d&)Awuo73K2jIY}iiD6AYW0a$_()D0J z{w|PSz#%zDX?*UM~)4j>1QuD4q zV{-~{)`}*u?23sTy1NEXUjGQ`xY8yw3kmz!!ny&lsdP6EF*t@FyGMia1#@3Gu*V1g zY1>Y+gjB7ipkp(2yi9Rhs6KkxW*Cs4(M~gV2J>pkKyx7Icaby_d%CnKCojc%PTvM2 zf-R9SMvQy97D&56{c%?8<&;L?*fUU2oygY(>7e+Sj>nbWziUv;>UPV+!>wCmpj$F~ z?Ul_)i@8(p5!O|(Wmh85Yeh7R*^-W=@qzFi-#O;NvkN7VxwM`d3mU5tigzKuLl_(} zxIpuHNRUpVy@W@y&yCt~;zeHar3Q@Y;?6H0`J((AC(Q$-bH&uYGZZEW@(eJ$#0!=G z+vJ;!9WEpz{Tq_?Ub1$x4S2}w^USjCI3&Hzl@2GLBh}Xh0~9uXLef=0eXX!Vagb2s zl(7H1hUOJA8W_m2J4VvJbjC=%j!Sdnl{b}zyTXAqKE#J5V4u3Tm~PWzAnr%)(>OtM zrSUauEa^>f(XeJIlBZO9nUn8?flZQFx6La#`48c@$pF3mHY?AFm6@}6V90Gox{KcH zD}pa+u|T|}^cz$+?}y5d?HpTL<(=r-dGLgO!q7%y$m{NkU%~8q1~y(n*0w% z*&ahnlr|>5oT8!gR=5KO)$cRE-dBLM9QDf7D(;MwE%8jlUb1A7E`K!mA%3zvgUWM| z9v6gPRbKy}-XCnRt{iUNw&c_&;`FdB*o!fsabkaGHdNkJ=}>I3-9#RnFdU+3oy6~3 zlYlh2hIowb@jn-u$+v}l*(3b>Wjmg0mqObwI^wve^gK`A5Kx~{*S$m>3Fk|_lGSn#%wcfrEAyGncF zjrG&C1L9LPXC2+x8taCV{1xf=I^1e=1yg=}YG?-RY!nCnlV}~Qk`auwfqQBAMeJ;J zzDf^EqlutA)v-g7;AU}3d5kK(OXtQ&ca#2j#U32m#i%E%@RrVza*J}HTtYru8>KW1 zJ8f|r>HMqaO7X6$E6N#|tJ^MoF(!>YaJB^NBl)aoRb3#D4?!2ND{sQ zX+JvBLe#5B9+2rdodfdNRddpqn*w<)P>6Da}CpA}-e`T%x+i20iv6Y5yv%OUJ76 zM8qX5KFpRsncW_^@qQp*iOQcUpH-0+4vLG3!=%a?ewl_N)i|SkKoCYSb)NKl!f|zL zj7!s_EG7hx+3ys2mqNtsS~Y>j0!$rgf4r2zjPMspOOp>6M13O!)$q2Sr&OX zeXjU`e3EeSHHXu~%s6>ob}{S_pHbooCVpw?WLg&$CvM|h)be1m7g+&lj#yx%c2$SeVPx^p`MJ!iS{t<3F2fo zMmYs(j@Fzq20=9<4yW5=e2EXJ-|KbO1ypv`_4H6i8cP<;?=?) zP-kjO*(^O6N!QW+%t!QWM=D-->7z0-m{oZQNax|}kVRP)^R5Hsc9hBNqTVye>25oP z&FbdI0{@ml*@;?GWre!_^)Y#6RliHB1#l_)iOW?24Jd*Husf-pvD0I`H8K}vE5 z>{z-Uh!>D@5k^=|c~fg3y{hJkZ#n74SN*!Eau6V$O&P@uq>PL*KMRc5bGu4ot9m63 z%#-yd=Qz{)0Dr$&QU-7hXnuj_fDtDNmA$B}k+LL4m>@{gXn#NKfCZHvtaqVJ$arno@XHjW_XBa^lO|HiK;W|cKA{A$l_u~p{ zW|!oG8jI-etH}d)QT&W)0~bJH^PfO9E!5bqf4g5$t|rJI!}HRP%$nAfDmv>6N^_CM z*QjGFz9alo{+M<7;KWJiup1^%HH4eQQ|2m5xk_IWACZP?p=yD=;b!W=rCiPXiW!e`6i%p2pNOJ=cA$FC9KN;GTB%+0oMza+<#g!I^QiZiLY~ z`g#l+%+u4}%+aDF>mn0BKBjY!ulq5strv+WD-OVafsSJ9 zURyroOix)}@>tBdKSAORcRsQ1b7 zdmD-&4@b$BJ-(wyi!d3ssH=Q_`W~9<9$_P6$7zPlaN*|vf&5l@mVIBKkL}Z6i|7uV zAHHJ*M)e=C@a*qud<9)Mr(^S!RU@F%I+%6(&Sbah^RsSeZN*Ll>!RtQ!E$Bt4i?qD zfwsrcQ{w6TOyL{Yoz~21i5)8^$n*40YC@ex+;r#}jraDhvf&H~PR$~PeMNULY|>X6 z=h=zH|V1r7T=_`Jv$V@!zWglaMqab-@OPtudIcR2L{R6pf7DR$HAjE&u~>j zGP`=a9S^wn7xR-Q%9bskz=gl3v328lXjt%BOrBuD1MB^Su-U(`+)03QP=<_(oFbPE z_vN`YhO_c@+cC2DdmR1v0jr%(`P+Lr$&XsdsmpO8E}Wu{K^x+c?TmKUiR#aczV@LPDvjNPm0~pv;JUa zJi8V*xZ{X4KA^5CJ-}W1cRwJKJ?v$-ljSUYMheci50JgR6KD;>&*1pVNf<73;NsR3 z-0M38Cr){WUY4sR)smp&z^UY#{CM>MIhwihC-(hd)hBmV3mCdO9iBF^=Tk;s5uuL5 zdE1#A<(;1&nO};Pj7WQkt3KrbbMk|vUKa9_=X~%o(a3knYp{4}8=0869Sc%)`477H z*x&hJ&Y%XD@b?H)j7S?pHE1Ab46F&mM;yTjLpS{UyQXZAZzW%xnl6XrZ$j@6HnMlO z;n?fH6EO3u5m;Y83IiAQl|L*eqJ2-=lh5ou2GW|FLE%fuBkpB`&IR$I{ww9>jt|uD zs=lbI*Mv#!0iB~T;C6T1IBm5sso8)Rqy)>i{ja0_-sAB1!zws?ZWNE7GLlCaX1m?- z?ZfCR3-WNLewr&%TsETS{s-tV%302O@5O)r&A`5%1{m1rtoW5y2!>a)#OH^-7}cGK zifKd7;N_xE+hUy6Wf*MV?3tad@#WQv%UNkbJ%3K^MW&(@dBrx8#*)4q=K*O|JXo zwB~Y0f1K-57w9^L&9zYWx&0{qQB#P^b?ftg74h1`ch>W>mO;2?duLwvX)PI6=qnv} zSW6lc@K_edx9>0jZvO_THu;eJT6{+QU2!2b6;%CN1(TLOehzK~MZ+82wHlosVR&KC zVDy?Cg@ncIh0!XE;0QAwoQ@|hZ|=mKr0j*r{XKcSn5kL$WxO2uvl^#s=5w8{ z!Kwdti<3k5N%toi#0mdlZeTm0ITOsOR78gs3x!#*wg%sO^a!l!>cG!UEQQ+L9wRI5 zfrLl$p4VG6-}6*ZouTnmde=m6D#TrG$~Ve7m@w%$)z)RHa6vq1A*m0*#c!=_c%~Sr ze>h<_e6#gM)1jWy(W|CE(H-x-Wfn82lim=(su-rvQ9y7ZJI$M45YP)beaXb;e z<^5!*9g=X+xsN#9AQpfB?Fb6P2@5sN-rUl_oravokeweF2Ct9DqSIn~x&C5nsqmu5 zF;i)ByemIV?@7^dU~Gq8Fx1ot;_DV@x?0iuE7X4h%cpbqElp(Xyt%x>xrV}EjeqxM zG&fV_mj}Cnm2cD#?+K?y-n{>64W1ZYh`kfiVeYyX(({Tw25;;Qg!9<{20gFT`#tME z{9#tw+Tl@uM~6FIcIEX}cNfGd5K(O| zQ+-4Ak7*YUYo{ORg@i%UWv~ZOe@pM}=kU&`E!eO9683Da3Enn7z|L9Zi{>x%!`3)7yNIzlA(hR<>Y@R;mDp>vF*&7jLwrQOr)C97B4-D9s4du;uE+% z+Kv0ixS=un@pcp9!RkRzyw-FjSh+Uly}R9k_IiH=X%aj)(GR1Z{?b03`3Xq>AoYD` z(k*QL#Z*R^&t}Arz~QN$sz0?I4_s#>eJnLEO&Y=&_Xzi*(0o)i6LXuMAO8;^d`3&O z+Hf+*$2D5y`M<$)*0p5jr?&8SfSqvh)93L)dj(w=e%Y`QRPE65p!=94<~g@DzU%P> zry7|fag!i?U1SxbO7olPFji4XAsAX&?wDE zdW7n0hiE(^6AW5upyuQROtaIMUpCa_Z-&;xg8c}Nn}eT zl!?*dBUzSC3H-(u2POX#7yU(@)6nKWo$O=(9ymzGL;+9XlZbt36K`KZkVs`=(v zdf+5>IC@slzopUx3+*=Zmf^&$H)*}2tY(6?F~HeR4htVwD{MT%6(4lT1*%8Qa06@N zj%#>qY75%?;SH$%&h)M!F2CBWjcK=C1a0jiCubcHWo1jyVfT7WDQ(Z`??5;u62CW> zL(b$Y{Z)mt8v|*d$>8^_Wr%tUVh%X6v!p$b zA@#1-cxq2jdd}@_qSC31t`$4=86c=m@!SkMb>7-mYqo)#M`wOQKTBxt&&pYL+*uHI zQ-)p*d~3YL?#DZ8Olt08Gra!6>w!PT*4W`dniXyvH{+x`SaRnwO)2dyMe`&UoB8pK z{`Zk;0=;(`a?+QAK0|x=Uo9NH`4U`yWvaa~r5Fh#@%mx|nH1Lyc4zIvZ;R64c=dA8 z-ng3_w}$S|>0g$UCJA=tViV|aYMs0=s00ZspmcC!-gnzhh*|y^gE~)UO0NwYIaDe= z=K6IT)zKrEJYgG#TNJ~Q`DZcU+Df53fZ|HUvE*Y&OL_odlO(SJ#KFR5ffGzSV!}VU z?GscFxJ;j(Wirm;HrEU|;SQ4CM4C?|>;}X65262Rtr}C!uWDn^^MZ$l`VK=K@6Z_A zOVVnO9w$Fw<>P=TL-xJ7G z&yqd@st0y)ZxoB2_eI<4<^i}_$CAI@_l$kZTM98D2k}^*4va6lCCDpK|Ik`7D{@%x zA2kRk1d`toB@IKM-n|5^;!D181=w6X11o+Fg1!$YqRH@ASUPwwVMRVzuWSGZw?u%_ zAweE#NE#W`XA<8-^43hY;HteO?q&B9!$~)70bHHTmi_rjKFmqHK71pGu7^JNI;-v zE*gv0g|Apj>2a~OA|2)z_XBl&;$7?${icdnh`;3dG+HC#{yjmSjVtV2&?JlLzFy1r z8dc{-cm0&Vgs8|@uz2bTAiXCu_R(JEE;>AH(K$x`PfI?AO}M1T&ot-+mKz^1#ZAP; z@TJ8&P`=9A_P5f6jPw#Hj#oTOei2$Py#fI(M@WTx>U(9+JmF%6qlUDLmd+<t$u*QF!T2fn5FLv^)g<{}r14eSK@eYP zM^i(M-y@syqw7w__zLoY&bI2jt(7&Ow|!)KT>@pV8+6qo0uj zuP1?;-`D!iugUaa{Q1$-&wy%!FoT})9o$hyKT1LI z;1D+b_8v|z>Wnu;pr9Iqq__iMu!8pGq0B(>wd%LNZl0ii|D{R1D4DhdK6W)H9^V5r z$Eb3Q><`sBVYZ-JP(38c@59BO8pTJbv~lgE)u4Q}-|rL+X)`#~=7aJRK)#&QIM8v9 zK|siHDD$vJ@~2gNL*9&Z1Y(_CwH4mGQ%(V-31pqFZjAmdYSX^A{%X#+d zx?A;q;y-utJFK2_dqzH;vXU$@@VWzCs};XlXSTNcg24)SwS^#Q~&q)({@jK5aCg$sURLaEl(N+yqUs4Pj%Zk06dl@)|;o6^$=p zfDib`WJC7Lv+TCsBgi|p0{?#NQNg>FBK4MrG&$y)CkUs-iJ-7o=dH82Gs;Y+IQ77|#b(lq_M+OI)qrrd znp8R~|3@WVTS1jpB`zedauX`EpOA-HOk=o(QAR=fMHf~M(dCUyEtEbGL7iWs{;GxI z!24|NvcoU&Z|_o7GkEI1M*QoN+uF01AITft1j@Ro*QWw$B}Ta%Tk|1O5Pt)WF?JZ0 zhUX2B!uN|YV3*+re|!5e@(EN&FM!5Yc-AjrpPrjYooAE5s!o~W z`;&(OS#cc82{Ru8}MG zDSFp(R~U9MiUX=Oq}pNc6SgTIt;SoD#)e6?#xv46uw?QEt}yP@o(o#iv2dhG8d6W- z9^Lioo>^JR#IM zfpW5O*@~0aH#mbjJL!fn;TVmJcs=~AmgY*HnblkAca=ji%0>lwb5L1hQ20*`d2!9e zTn^dW(@5LC0WWPbPOlt^3znG!VFIUXMB*RTIJFPZwXh>st&#MU!X$9mZ6trzC=w(0 zHrJ5G1LCx*42W_!q*~VW+T$lx4=TS#*U3nCNW&(V@j=v34dp77_svs$3n~jD%}tqF zQ%3xP`YtVT#`oi>a!K-K&(>GZFT-MIgozT{Ops2v?R;J@|j`k$hu-Q*qVO9R+T?Z;_E~H#z8MIns!AUDsc^c|Br6ZW)f70rN zt6KOr`y~=LYslkLMz{?E7smqWO7sdS0#&z4zkFHt56Kt7y^q6jMq(rQ_%f3%J$W0H z&b%R>0%;!{VsTH9e?XO+`D|#(RSry8j-<)8Pv^Um&$xk9gDT(ASQk!#&hPtS(9U(r zYcTb9x+V?jS5{tnP+VemQw9!`AO~1_~9-2j&r!j6$ z?ZgSIVEngq5m~txZ;deIAwP6s##Sr-=5ICb@Y4rj?>2Zay(c`5?ZA7D8v(PwUlm1f zyyU`BgCTdSF<)`XioZ|nDSk~Dj!m=+ST*l&(BtqZK4-!TjIpi9XPlWKTxjj-tY1IH z=z{a&?B3gAd5t94Y1stEckS4er6XuwY0F(StKz zY`C4?e~>geL?%tvlTA0-Ngfc&jvqP-F=ZR%kIdq1pPdGL&^l|`vsG9A=t(EYEpLPM z-HrIV12uSy&->v?uUX>e&h6}B>3q5Em@O2}yMbo|F2VkX{!sgJ9-BF+2JTN^hsWE8 zYeSx2hM-lg@XxkNapP_iZRyuxyu9^VTr+VJj0&3p?QU;`@FC`eyat?Yll$qjD=wwQ0^xFE57mb+W~Evk$O;_cix<&&G2D9nNc1)`qz4)kTkK z{iTJa1v<6}l5{=jSx;9i9efbSpSG5rUl(Hj9X(m{I0jvR(f)(nN_vFSI=IIj_`;;d zka^oiUcY-AjNI(d?MqE;UsPB6j{lB(CchVFPDG$4fYt~Pn$9CCZ(-7oQmTnCvFLu5 z*pbj4E?$kmIseRnY64%ZEXFm960p{bW_-dstu}x8O7^XHQ=YWLj~TXo2{j)ELkP@X+gUvRL52@IQxt=y<#kZx5&~ z4*z_BG;iG8cD*cWRSaW_Lz#KsG3>{`M$kI?74+U2gmg@N_w)wb?|2NRCI)lUNj3Sr z%uVQN&{sO!nQ@P+E&`XOV&?5Qpn4Vio2P=qoG-8~;e{G^>^g4=u5B0y4R?0|od;Xx zHpg6{dA$Q#JbDXs4RB{&3%ofXAI~=q!&BW?%6XZ!!1D5DC|z+v^_Yw>jsU%=&Ft_` zf9UY#8@#oD3^NC9;4P-~LWgb*dFJ^*>2iJ#&{zqYYZ=_E6#BH<VX1J#XYa^PW>JSIZUb3V>a3q6*q9U!AeJIMJBenNIc zJ*a&D5>%hwPaQ(fWLogfa~(PLEDP!#1$q5P)BT22d^Ei_QjhQ&FGk{ko?Y>(OKsep z(;gNLegM?FeAfEL)YEa|&YymoaSJX(;qRlkuiZZMx2^%dp8>}GR{_h<4wd0UBH0H2 z>T*n27My+UfL$&~hD_zHc6{vVt2q8!I}B-K!@GeqAC_Ujsm?XjU(nXdn)YB?jR6I; z=AY$0_!Owm%dLx`z;u^PzI7aF{L$m;NxHAljGJtK%%Wa>GyCWZm4I_$jeAlw12? z%Q;5;>qS7DgRY$V7YIva^Fue8_kbCiQGQlz?T1Y;X#RNAuBA1rn?%ELyCM9_$<{L5 zI2U63y2<*J$H?==?qi15`Jg*mXn<7@Iik# z_b=AcTHt-LXx?sx#d1%&V7lQ&**i8a1xHOXv^($1A2!W{tY1sHeJumN!*ZBBX;ntJ z+MQ(&t0uiF#z3lGBk1K4EON__((&3!tJ%Id&$&HR^CtgnXba;O^yM_)FfVB+pJwz4 zZst#u2I2L2YWuG+|9v7Pj`2e0vGMYbnLbyT^3o#@0-XLZ!UIvX?<-q+Uze*|JF+wv z`kIa9Mmk2E#zYQHJ^w$Ft~;*B?~7NmD;W_DqevzH+;iUJJkN8lxVqO)Ib_11qcgAvOy<+9hJtNdjg-H3 zA?2uap5+KRfZ&S1AuYiWTMn&;_ZMIn}AW5VesXT4l}O#UfD6fz?J^HNQAveL4g++{lRY zIOQr%tbB>ntP0?0(JP#{;jwnn)IV%{dMYaHI6nLaY#z(lnr+=t;lZ(!yO>FkADF-1 zDcO@Kp{_sLYq%s_=CZ7qIFR@_Z7$}l&B1P+no5Opqek`TCoO%ktkRVejzjaJR>axi z;MQ)rX!FxlKKQej6aLAk^t}VkKc~8fl6MpFRZV+boaT@9y_<_iOKYk*$0tW(U`>yO zjPML-P6cr{+)MgPI5(XyTAl<{-%wHiGE=eYa%!U9byvhtZzwt_8%&)#MoC zY_Y1KrzGBDNzE;B_JgKkp~EW(?XVGJsV5LJeXvm6`exY!Y|#G{hMlkx^~RaNz`@;s zIElw*?1C<9A|dPdbGY5|IQu$e9}?%|r)C!!{f<=iUAxU2M){B4?}k*(6=90zuk{|b zWtcl&8$|CAT^oZFFWlGsE$KsJ)Pq6!=A8JM|2#Go_jm==?PT|;^1u&xN7f6UoM#@W((WDPyT0^MaW+wGzgOKtB4(a(aIuEktC07$wY-f7K{zo*fig!-mr~xA*?&BFZJNVr99(%A9F<{zxpqSwC zL(S00+go$HWosnPz_GV9+;!&(@yD!CgcknMKC#p1q{B4jRH z(EVX-hB3yy`NT?E{fB$n7-HJ2v?_hK>s1-jJmAfCUh3YEYE_Myo`;0UH9bBdVFhuR z5i{~G1gd!!V-g5u>&;|P;7R)awTt|=!9}jz_67Trrxe|IX*TQp)nqZv{r^J_Lx+3s;lAwE&e(eWy(^V#{>B1;j zuhX918E>!Mmi~om-whA??^jwB2xpYWls_FVLytjCk#H3icl>b(ViXHGuKO%acAg>M z8aP%^{m2d7GI3d4W0?@;tX0^r@F2V}8!zs-p)?4+qq~WZJX24Mi8zXhZ#J_|pO@hs z(++~Ti*mYJl@938I6{zygrw$&P=A;oYhHAOzMr~_9!`2yG3<#RNI5R!&mYqeH^~b( zM^XL7g2&rLSTVIBt@-;w%>(?{5TGzvxK_`FCMm7pjvdjBA#gEM^V`VJIc&^Y1pVqf_7?Ge3WWPlVWab;E@ z()GZl$$N#v;?;UDM!bi&-tNF5eHl=ApnO#r&3})f zeJaY+!AeuYJeJgx25p@9wmYYhG%M-l8eGj+Mg1Ol^X(O__fiMiV|XOa81h9l8~+AB z)?Y^WZbo|PBkuaI1rTNl;&|$l*=c{p9hYMwthoB~saaR4Vr*HuPxHd~2n;>5iuUUL zD7FmCWXU(Jd2l*?i~cTB>0EB3txNbi4TCGMzZV%uqt!q%-EEL4?ibF^z87fT1OW`|^{0Yl+e~tI0EBT^8 zq+CJbL@beYNITleL|QwDa7Fni@Q?iCvTWWq=#k0s!ee8pGz9U38Z+DSu!SHEOZuk> z37b+Dx>_h*#UFm9{iTW1{m*um^S4Gr#?IMy*NJDKLW8 zu5Wq6UbLh=(faSx@T^U)@_y$RjE?1`HL5g2-svtp)5ntP_5xZNY!ZZ*pc|>re|6Pm z%Kblu-z zgL(~5fy$#?r)$u$w+5SypGos$&JTUcuF^8&yjsA{kB7JOvq%m9Gv+;k&pwhME*PvP* zCrKVpntn<_6TK7Aty2WeO91vWEg}!;%AOXxg6Dk?L9y2kJ@bPRH&Ncr0P-trz0O(O z)95b#dr?#N{vCzUuM3oymyhR!p#Rwmb1RJ`}N*k}yLDHFYuWfL= zaj4Q3oU}30I3b|i076cT6$5IN!EE}bjChkN9iy~!PG%FC+`FIB#_YwGYl1WlBP|TX zSEynT9A5yWhuIE#PI;mCUuZisOX(Cmt8FF~huh8Et-K`IMwKGvI;O070&h~=3i4)b z&%yIRd4pdz%;#$w?1wrXY`NFhwXnLxAD0(=!$_9_SUqqb^nY4QX%{VdGS+H)lu$8W zzm1-aPv0-nPc?zP-e=jqUf-2J6BF&%Y0W+bh@$*6%0r{Vqgf%j#E&}U8%*V?40lQQ zi_dQ5Xb5lcv_T_&FKIoCxbhI`I4$WrFlh7}>3mw!M_l>ooXovI9$Gt-zWv|kN&Wje zy^uT`%WL@n5B4%toiFm*qT2<@KuSB<`d)cAxRq?7O>`|(QdKsYpEh+p8E3RP#)%@-O`C~>n%1A$n zC%YDD=H5OEg!vdz*pT9$y50?-Z82djio5&x+FqVbr^b!ux&9R3Ar-I5LRht1pQ| zAb(^9T)g_6Lh?a8wwr#}M=A>f| z;L@qtic1BJi9H_m0AfsNPurOutf9+i;W_v{BmYP`(4LWRhg*|6$z2OFHJ8E-WhFhw zCqIRsNAIliDRhrWSWFzTj2Vo(PIG0*o1A=yH@+mRxfAPp2%L~Q2$~>0!l8MvAZ?{hoN$J8<#$1IE&Ntk5XKl(am065dtTJ%7JD}8IDK*84BOI|khfTe zE4M8{!VKu#DUZI>TMS(joZ$W3c=-2Xi-z)*kv12kA((!0xftgajkgY`kq&VZgwasA zc^vWM3&wnUgX)zP9Iiusmmf-xR{1Q7o5E+}q!6KM$1Z%PB=03`I-Wz)8{*CT^+IVB zU*CpOCp5!_bSrxQsRa}lp!g%ryXNO|BN4K)hSGpybn!Vgj~HLk5r@iiSpMLVAilwb z5pPlTY5G2uN;e9tjEm&;Ev530rDrTb^*M|l#pBl@rd(kL{gD(`(4lplhISq<$TL>y z%9np@!AvL|Ti%hH`wVG&pzmb53+u478OjJ@Eyo+0O7FGVE@kp{hFxwXw?`CdKdFKlHO%Di@4$d zAYYH>uZ^U&_A*)wSWBANSQ^us^ooO24}kIkU2Kn53* zg18JD|LA}%24pd%6ErR2s(ciA4^TL5cXI=~0X5~$j}u%-H>;koXw=I|s(vQT4|lv; z2!yASbPPOix=(p5O~}V~KsrZxOg3e1Q40Bft?K8^9+HZLJ2Lgu4Mv&{2>bDpVJDok ze>)f~(3PZVwaSOM&&~tlTGgi%7JIG(X+(aHKVYP>29Sws+N2yX}u^^T(oWMX5tU})~u)_^gHax&&RX}rPZkyOFT86dNjU3 zeNrT>1Ja><^H@=Zfus*n`H;GgULeJT{%sq6`!-JXPIyZ}|gR_js-6x`*<_lF$n*LK6oCfx0YzcXxhzv>$&wfj;QP*r%Z6*^kwBX>m=%1PWS5agO;0c=iNq@pm4lv3B(yYI;`J$jAe< z*LaUE9Qq|Qnk$~{bO{H}8Y=IdDndH9yelumn*k*-wsU)~@_6o`3V2|x&u?1HU@xnm zXVbck<^>DWAZG0~9H*(yckRC;zm!I^&|j{2y23)v`O%4mr>$o{OQ;vocNQHRfK7}3 z;pXE^&R+Nl=Xv+#y{ESkL-DjI|5_p5wR@<=Cc#K!m2Quo!_!0aG!|a%RsF%7#|tsx zYd_iU1nJ&oqa@Y2Tp0fd^lUNIVK}YXdT0dARrSURklRE=K3w$L9^*IsLv%-F6ZxX7!VgpH!ERhPbjNFZ$LAP1+Ga2~4 z@gja^;vwu;YdYb}0NLO^y&w3@jDIlChD#0;!Si@1ztXEdH-6O}jvvZ|Zr>hb>$i?j z5*naY<3?X+Ibazb$h7wz&|}%@3Qmr*(5rBw+W{dU9QJYyREt2)8#2 z)KL7uYhXJ*C@W5hl%7{jqA2~J85$BJNVo~4DqPeG!!WH#%{xmsWtUY%WhS{Bw zS!+&$>#k?Gto*um$1q!@yn$(jTX=`DkH9bgJ0on6Th0)UMuGfW9E^6hmhw!Whe$cW z`mdOdS(Bz==L4STz26Rl-S+WW{j=e{z}syjZjIR_MHN%_jR4;;$=+kb=+wu6bwUO^lt zO)j3{HreNK%CiH!&BI9gHn|x+Q~HdjM`zL8onw^aK=YGwrENazu)Mj7J=J42+?73K z`SC#EdfFNr4fIzzh4mNcs{8`|mn&KM@j84cwu5>hzA(SXIJsiQ4lJOxcnRmQ-ilP7 z-t{@$>qF1KvnI0*6IZ~y-j?*PmXRbLTu?Yi zId_0jO)C83r{*UJ!Zgf%7z);Z4~X6arV`HdftY_Cv8?EzXku4Qrr+5GS;4!tiq~4! zD26{?18FQ#!f;ao%;<1XaUE{V(hB{58rm!Go=`YW*o%#vF3Zr}jpgUhK{)bttms$k zm7qFOIExKW|511hy($K29A^fJ`tSE>g0ppGo4f9s{wrQ-PQAOPZJX4GrB()OdrvqU+Kl$K=`GZ; zG;c`v3XckX#J8PGh2qxqlspaPy;#sEiFCmhAT7j+%Y~Zb&pp4u-x1c(+4ON0W)$5% zgFzl^SUW2ZuJV954jZ?&hl@S?YgN59EY($9EH|Z`K)2PjC&&ztf4puoiXEzaJbSXQ zq&yU*J{g$$F&V#~HN;M`pESO)R^f@TYEToek7|nniGx8M-`5e~-k2-HcnwAug^+m`a+ch_uj7Xdf6T31>yEPOEYDml9R~ zTB;F2+{Tnf{nvMw!e~kQ0kw6Xf;Ps8)}Ux2ON=Afx0tT{<-2mNsc*X6H^@}Fix+q{ zy^f&z!od4Kv7J>GQVl6gaG7CyjgQH6z-ns;D<0I+dF1E&Ihy;XBOv-sGu-g~CglHm z4fi{lD!i4%34*i~SNdhBYdGn}X+U|&er`9CglR1OjbJRmJIPatHZx zYb2yJ9{}}-)A}4+9)qWm#6`~08sk@~K)OK?PEm|z(H?TNe@>^vLUCgB@9I4J?{1*{ zBEAR^y_Ra!y@=Y*Lparm`0vkB{G=1D=1=o&=T|XP_Y-SYxsMV4K-yDl(a^&V?P6&^ zoz;_Y$dQ@w++ho@+d2^1_CKgezHSK4Z~Eh~v*EZPBO6FhaMHa><1pe-L3u34oSDPl zJnx95i#Fk9&pi0`^ah3(Uly%`%|w7_8?n_n{18W{#1Boub}PTkxGe2(BD|jssrNnvFJ&=&ZLBQY-7p z;=}v!zb2P7mTB#)^pu+O$#>^rk@sGMU?!liIxN{qh+t>5E^19VP_{8$~?|K||lP%{cIc^rXt zTW>PT1AIWw3zQZzFh2#E8>ebWYhlE}4Wv18&}PC`?c44iyxJ~}AUwe3K1=a{eN#Sg z_-h>aUQ!(QfzlkmJ-=1)jEbf8j7*qtsyb30C=Q^l$y@T+3uD>GpM9%v!|`Pl){To{ z3lDr&7|+LVdyk}Vkp5P+CrCH4YX&F5dBYJna+#h{d>a9Ey2n7_pvyq`jY@-%mKM8D z=<-%aH=|)IU!eOU-S~;A8l&-X$}OmD{*_U^OS(RcyVR6BohoIomY!E$NgN(>6}lh0 zsrVj=Z>XM+;Fh1ONXr;9((7>O(>){(LjRK1Ff(B*C@mY~e-Nf7%}0IzHn=dSIV7LU zWrXE0(X@tKV82+i$i64n$|MUb{< zN8Qc|$~n?Yo{F!ipLYZuK0HLjs763}MD14Ot+;5C;aywX4v-ObS*)3iseCl*{{k=7}8A_-=16uFa6!cp4pd? zcm*3Z{fkbg#sg_*ac6lr{;YjeSicyB#N&ee1dyL4@8~av#-suHE)DSl>L%}ksnckE z%`NSf=LW~zanL`ct(ZEkrr6ZVOU&5)5lFKG45s(*$v@Jy$&V-EGFlIu<{s|eqqWLC z)@asTHDZcG$>WIC$10I@1S?%s9ZB=4+{Zy%jnty;A8+kGyk43$V=j4CV7-2Uf{Vo+ps@%(u^+nP#sQgL&;jUst0b-=KX;`{S8M303;}yk04gN8(ra<}=dRU}t*>9=nYcFPcUod1FR> zIjWZ(KpwoxU)=0=N=rR17<$Ezu-aJY8RjGPGRSZICzNidnnTsk@N#-1$X{u`x$Z#K zuT%In>P|YnZ?qZvb!{jK=W(K6JR=`1HpZ=FisOh2k>(QzPue3UG>8{8-{McoH?a1r zsf~W-MtvG9q&ae?o*+=ah&UmFdKsI*w8=g+Xk5U2eRpByEd#OWa6hCvb{P%t80kVr zUKxo?1@$3eGc@3xJGBJr4WiPr8BK23$V8btr%eUmD5#)>Tb3mL%d3u=mr9W5K4)8qT z5?b#JC@iCx$?z}!s&+J_%bDtxsNUkteb+$6f2V7A)f1udRp&=zM?w99J1Ln5>5ELy zJm&MXh{X;_gdYDUE6<9mW~i4b7ah()@?c6&Xu@=x$V!*4lKP&!;Z;{uT7kSjkoJ-^ zhm3k+?0upoZ!_vVVf#O^FV`Lx*ndVU z7{$>dzFiXYD;gxK&u6&K{~KDZIfp713Oki{R$7XBym-yPMv^zj;TH6)bMi(pZAc`` zMD4>rA$nRoBVMBT z9Rlhjz}~(+<$K3OLHgK5mOZV~O~g-}dPXcR{}7~(d?u8(B^-q-prM|p3lcU0;f>1E zD$N;jyBfU{I|>O``OYh+=$S|WsB4gKL#4r2r*`Ai2PPb?#l82RVGbY0BJnXdo-&L% zzoBOM1JH0BRvDk!`iQZ zsQ5=qc?RnuofVE#PSUz}chZ$67sa%PLvA*G2XB+&qWaAf*4#(pSlScs4((@ctD#;C zQ&>wJDaaS8ev*oxB&-qz#cv?k?ha|V-l+2Qz$pjb>0S`78r%)H?zMr0plGFIIMs+u ziF`u%ZlL-Vf_%Hu=$g@Kca<*DjJw?yKjTCgn;F1oxcG?Cox`ZV_z2?LZ(w6Tx8w(Z zU1V=|Ea91FHN1Ae9x`C%R=lvKF?%&@3+)j+hT8^Bg^@?Cc=^1^(s$!kc+qJH{JHxd z9;#Ua!MRUyxwjAZY}!b!s+fb{bKXOQ&3o~Fd8k}{v3=4QG@qW%tlZ1rIVzf#>_e2>gWuOI&CnBmX&m7m5n{$t@*JP?{HZ$9*i7uAMaq9^~ez&L;o@_H^U+qV7w=J}H+Up!_@ogV8p`PhhhnMgx z)sZLZUB|R5Hn?fmEeshuRBmaR4x0jE<-9`;c;xc8%q+X1R54n%%uAN{$i%D$JLQzO z_o3D0>o_kh2!5H);Wy@;!-REzv^M4W=xbYtFPwQ5!fxc?kCu_x{p1aZo)8FL+yBCa zBq!m0b^^0AJ`cku=Ah1HQ#4IoE$N)NwsRjzF_SYxHuH5U4RCIsDbl;-H|9S&3Ey|w ziDygQWaIrt+$|=FO>DH0xy-YIE)`o~+v7Z-6|7iZw`bzWh*7jQ$SFK~U?X(2oy7N< z^@UxFY~=77_3+F{TYPdl1PeA##O(fqd35*sU{5@s@nS13c`<=Mrad`p6!(=O9lFT) z!mD_DHBb-Y8>~y&jis&K#oaZ>pn)vcGuXP?aY*;QfODJnVI7ZblMA+SFx`|Q1{yyXNAmqJG|3NqZ3FSp11`T$ zp>L4h-lur{M9s)SlI~MJ^Jv2x%&iTtX4S-pV;ah|UvDs@dj@Vx=*a2brQOGFg3b$L z8k%7$AH$E99ET>W_P|g2K2(k`gy{~6tj#k!*}bp~3!4lPZuAZ6up%bwR@z-qn9?6|Y_PYr_miMrU&OD%8lYW!?BjpXat=Wqy&yoez z8tC`E&$h394h~BL(8wZ4&50`|r{*|P(0HtLS>?fK+^qkG&(O#EmKg37i!ZxZqTc#8oN7d@ z?R<@5v<(;TnS>*c9>q!j;=s68SB&zl#=8&P&DQmpjSdf<;pWJtnB}y^#k*uJn>1>s ztlMxgFKD4BGiz0fNNy_^&g@NlHxGetrRi|ZTSt2L)#W>M9XaJCr+P>s{Ls8&TeyfT%%ytak2}tiK9)XUI?`M=IPS>_KSbp}4fnX~ z!fnzM;AZn?(%t1awp|h_wzTZY16I}o!XNB7V!d>Gx=h5c4rIL@bfrFYmQ;86?@4`% z`#HYCsg$tQiV?dXd}`x&0E@vHrPY}A!W2j8~nx_Ow~pF9>P zpKrp6qfm`~a%pE=Nb7e}41r<}{|q~bzwKl2%%(nsTPJ~H$CJm+MXw1@S+!ZW6^}`a z%`N4yWrxrvY!4>p)WY!_cgsCzyl5SYrNnpR*y`C~P95sa7q1j;Ppd~R~8%;Cwshx;G^3(@YxmhH3tW z!U$I{&AX$HQ1^kO{JtQ8Rh0c11;70J^ zdm5BHn~f@N2HTrSns>~7GeyRgMPQB(?e9`6gGKzzW1-C^;@C`mK4R_%APmM+Ex+Ja zooomX^nsRJYNNvZ)Y&U>*{oQgeA3p}HkXGjOjW%~;#K<2P7B{;HY2@37*v}lT4k_h z5X^QO7BUaVrAT-s&hIXO)aNb8bKTbV9aBST2naZzqXoAgc;iF`CN!aU?n2+=-taY& zcnt1s$;Qz;2Z8U#-txq8A4xeVXOBt79(ALMqed!j!9MSdu>4gZgeK`q#R>1O+CtAJ zy;;b%kAxK^jIc;X#YLlW$v$yle5-E6J<#0ij-a}QhCZET=ar9PzWyTEp;;u4&kcvz zsRL!d^cPI&g!l1<*m>myQ1e#)st%C0Aw9T2^Yu$(xw)4XeSA$}K+mW<(6 zi#vl^X#Qo3u`3v9 z5~TB}SZfz~AV%e4n z^sV<4r4?1Ik@OSs*5#_X9W-;dsFPpcEO;#G)<4pZ2K*3>3#KcNHo)uQ*hFIfQ9 z4nNRw6XvwLi?Q9#$;bblg#1S~8mcQvdR7n~T_k;ugb9T0btL5=60U=X<5yI?MAs&6 zse$#cxyh65ti^6UQ&xH07c8CY3gQ-Q=9Z<|zIF=jG5Aq4O1#78Z5;T<2tO^wA2$~C z5|j^I=^f$&4aJBN=W}h?5wW<d2;+;9`x@j*yhL}@jp z71$0^1!a=+LxZ>?BuFKG|nk%M4w_%G0DvjXk<5yqx*s>?jI*qzC$r|ae~o5?oy`k~_ZsO=|V{{}O8pks9)F9XZX8gt@i zNOKOlip^qjEu- zRYSEYv|+7yjp&-VeQ+Nj9|lIAZRJ9CQk!An38de|`h88Q20WN6$gFFeFWY>2X74t=TS|e1)8uYK3%N@NY3n zd1$Z=dcjC{G1uMSL5=ab?+o(VdZ;abD3m6p+)E)%2%m0u5x3`?x-=U+2Xp=;W1`m+ zMtVxzbJ)q`cyFb@l8-f-fuxTa;X4u@;mIk}Aiqu~D88qB1a<9Ce}_ua5`07hC$PCW z0!XW(LCPg4ZJh~r&qJXoN6U!+m0uU6kC8Nx2(PBYW4iX0G+q(CFBeHOC=bAjA}xqB zTHsKN1*&GDAaR|Db1T$P9K?hJqe*k_fk9UW;N;RI{Pm7%SAPd=IL6`lwOdSilPZjN zZ-V4^xs&!bewpuyd4HoJVcj|?e4EI~xACVRmy^%%)lgqT%@_arB^>kYx)WEwRk;fa z=T)prPMu;j$3R*~svM!70sC~!R4C4*`cN865GFCIL0s2xAnCwO^u5*vR34u6CgDUJ zPz^HjH2iFKG?<7j_@$9CJksnJ{*5jGX+0nx4-2oXXM5`%gI?b}IprZc9+3zwWAxD!APdWsui#$$8EUF?W0KzYL$$em=sht~)uzp(8%Cdddjg-E+Vor5o2@Rte`u)fehs6h=@_04cuO2iIG1eMkbO=c&g5gYrI!&s}PB z;zuMMD#$CdQ*SzQ$|FX;ffe?i4~@U}g`QS3wI&PJGR0x`dEFr@cEI_5#7%3-lXc}QYV^eW z1}4&V^Bv;Sk?f%PIgRRHk*6n4-3Wv!e76zE@|pqQHDMBT@-ybEuI8y;6Odnkvy~iH z_dB3;6lsF*SkS`1>SrpZ+VZjNOQT(=Jmf#k06C&TBD)n_pA(*{9vWe@)C>#XD=tpA zsPdYNJo?LV$ASd)td!oz|9b6#!z&6fcv>~pmyr`DM=0G1q$#+6i{g}uvURZXWfqWL zz{SHBNUBTX>cMa*uc_*F$hq&^k{^s?g{Af2`olDQ|FA+s{Q|-t%ZqgHReh7}{`o@r z)604NT`n%}Noz0$0nMFg*YPyWNXe`*j16fjM5L)m-uAsn=)gSfo3zVgw8DQodnxOJ5u?(}*->6A2) zUC{{X7(RPz5#`<;P<^GEL7hndbwxY#;Ywr6+&9k^Uz6rMrL+O{W?#eQ!)H|-lxC-I zAY5fe{S%CGkn~f1w5R9F{|`(pGH6Pjg|BmKwjXK=Ha`LCwF~O0tNsu5a3aam7${sr@{>~G^Rij?{KfsoT;&4gp~&%BB4})q zZds@~XutgidE^?Lj+07*{4L$5mJEzPPfAaW$jp85TY8}hgTA=_u5Kp(e2Z; zp6&&cOlglEyGan?_JBQ!35L5v7ePwbg=m?40(Mc%R@K+%u{sTyW6NVAHSa1^tlurW z_t^sPviHG2-Wuxun$H*i`zI!C+9GKDFi-Crn(z0cf7>9tn_dx{b}q&`9RuXiDN}H1 z@&egmr>#0(Iwo0)OnO)F-#TB|K6?`Xs6R#K)xU{r-2n#n`D*3#vo z9>3VaOgJVTg6uK0hvbPf_@U88uxZ{yy8rft9#dm+TKa8>vWbvgPu&+^uf#&uw}a5X ziKFy%T!S<=eDHaX?9#Xt!>3lV!p;M@eSB{jHGLFRyla9jfA;`7zdR6p6?ErMl-BcG zV8-|ynCo1Zn|J@ta{RId%`+4iKT}s>om*Z?vKj9~OHushK?#t9Y%Gcw! zr8KzrNc8-E0#?`VEax<6%CIA7!(p34NbhYh@T8oxcNH zwdyLH4cd!WD>5)Q>VrVH{aCTS0mP;Fap%d|*k{x$(aAMOP8%DErZb*FQPwqfD(NJ~ z{G$Cz`xwJ-+FRE$aFT3yYXTk(iNQNY_XOpD^!L0kmhUaZYe%MFQC1(yldf{P{%8nI zEYh5rw->gx@xZei-$U&6-ymlN^V?VEg86=3(Ro}so}qOu%$6F+Tan-4W2d$}ctu^= zGv_H})Bf1q*R7E7GZIEMK7_Bw*M=E}i00i_;IBs|>Yn)3jP;t*k5MrGT}Qr$F?M{sx8~qj+$?J{aKR&$oGR zmoJYu1)p(eDUbA}vsEpK`!8L)d&flV()%bZGiWC-ejcvkENj;d#@co3@fSlTuxI-| zYD&&*Lc^p|;S^d;#)fXk;*NJgoAZ`=zsu7kKGczoQXBBg8(yHQ>wdZJGW`1jJiKHh zn{Z<3#?E@G+xPvf(*89412w&wI@z2L<%7QnOxeK z!s)&=3I{rTDHRd=&E%pkdze$V9~9#(X`JHBJS(&~cv~=hYB!SiHvA2AZ=l9OdBMXS z=)02fwPjebr|f&jggesSOnyV#@`G`+*u3$+tl{jDa!I*9K7VhER}-p%#qRT1ZD1yZ zwipBBeB+p*N3k%f)l|}X_{$ZA?5>$EV?K zZ#L*#7~j}&2lV=Q7ib(j_hu*FJ5K=BKCR!@7}n^%0Gk929R2ISZTU+!$XTjhnC=Pdv-`diVAC)X-hy-8765Uf{g(I#T7B!WzO+ zR53jVV}bGud5hgVvx!TJ(|^e@$>$2KZw&m*ye1gm__bJa*_EGIXvkO5{tFZ*G1xp% zM*SFuge9WxuX1dbcmo$6X-0XeCojx6!>Jxwg&fK^KeOk?-qpC{=3<<;;FtDH!43Ih z$YXKIG#85xl`x&!cKBb`W{t{q!VPh6)DNOdM}OmtvAnYRM;W(iVwiO$??o5az4^rVOc#} z9AnY)=0RDhwhAr!vTNMb9@h#pPcZDtr!_5;D!(Que6Yz z<{9DGN<9d){El-rm?7Oay}Q$f8MJ(X+4Ri&U(4m%qakgz6bJF@U_a5RAVWj*CPMa{ z6VL5tgW@mZOi8>;9D7>+8_d`NFMS!b_&l3geJ=Jm`kqzOdSWM?+HzCNdBS7E7%;KQ zM#U$To4BIgU}!hx0qp+yQVcPejs2$Y0-e5dw2xosgVF&lCoMliWFIDvf)n}Ny2b=0-l`^$at$bDyS?-?u_rS3+;8}X(r5p4!m2by2LJTn4M z^y&*SI}^mlL%WgiP3Z|u`(I~a+|W1Ln28@WV`}`Z!u_F9REHI-VgK)b@b_(X(gve3 zu1(Di9~B7teP+1K67yVlv)!V#s27?cehxhko*9*F zNqKZ1cCUY;?#?ot z@pw55yww~}&Xh2sn+xfUaxuLARao)P2nsvjsEWml22nto3Y#qGM9*gZn67RKCL8}^ zDpwW^nh3Qn6(Vspr`i>CUfR#hk9gxWD;Pe5&8>3{=4V_GNA4%%my#81OtiPB7Xu;i?`-Y5e`ZnT2F$Gzl+zaL;;Zgm*7 zW*C0TY$aF7el*?`MtCl-la`<{Z=)w*hNi@+RFhU;gRX7^0i!V9?`>%x+sgDDJ%# zxq@AOUxvHe+QI6`_SB2nkNexzNrHGz60b4mtUu_{tPTHt z$V{HMsxKcMcqvE|$fz(&p5DWnuj>01LkC4`DriqL!bz5^aaMUD-+;F~WOJ`G_e;m*M)(8&T;M6{{a<7qM>-Gufo)2hv-Y;mY&~wyJ&r&Bq3X zVfdk zSg@kXAjOL^(RVqVh`$LntslVgO9vU@4Qb&|q;V{{!lTS4DdI~vJ<4Srs-Xr-<08$w z=GL^Dcyn_rbu13;^cei>?h|NI1n-B@`zB`piQ*eA2%ENw`U9L~$-2v$F1@W7VHWA# zfpSH=e5Ips?QlyNvHXYPDzRw4Cp+<|0Uz#K8|x29)DUK9^41r?x-|`D?De%+d?Ol& z*NHR3(P>(DAijgr_7fptYaf~F^wWj30-T)5HEOJ+ouqY-5*PX%RGPK*o#8;oXrJw) zJ$haqflBRIzBoG=d^*tn;A>lpl-n9X_$AtFK0>XRP5GIE0pggomAsMpOw~VHHG2hZ zet6RN=OHkl-V>zX!%FQ#3{2?>ch)=g(VWAjC{|IFv@Ko|ow zFYKW)!(JBVP-*jweT;IDV1@Nl zK6SOjMtY0=?Se@Y(SB!Fnz<~_u1C34mVCkG6>B~H7R(>vD5~H8m@;b39kJ(M8juFy zZYBc=Z(pl?6cjUIR-6u`vtiky$3WKxrR8T`v*Wj>ZGo&QN12N4u*_oZ;4#-zXzsNI z!(W5KhSia06>p-KUz!;I@)On3ehTs^;x?&loR zoVt*v!a-B^vtIr~SlXzM(EQv8xvzga!f2|XKMf>dD_<~(=H%uqL3K{r@`=KHBo2b% z^?NG}5kA+Q6c6Flt~NN>ypAMq$omI$;Z(0m*Mj27Ye$}A;OiQ4O$S>^bIwTb@C^~p z;M)BXi|-u^h56w?+{$U3TGCu%*cVG~NWB*FaboDiW8gP5M)3xcC(!gd*O)(le@X<{ zN_-lf#zqb=rTrj&K*Ghk-0Rg>b~Rz6xY^W#6K`_oO4|SD?>tc4ptStp%m46tksh4X z{0CEW4g=vIymSszxkdcqCX`P-H@hWio^{$cBU~`SEFGvW#ic&GaEIMt?C@ngVaE$} zHD1igE6Yo!{RQQU*!^=_6>pJl0`ie)+=jl@AuLw-#$Q2tK+{UnT3wZT zRo;X6gDGuGIKfVa`(yLc147L`=@3B}g$KT_Vga^XD6FJ&@i1DWgtWS>r_hb)oV5dqM{5s4^EB@)?}LL}=u(;q(GOZVR2%$)Px?(a7P#`DsU z^Z={fbBc~?RgzvK9cm&83!rXm8=!UO1UZdVb}zK#QyKM%^5*0%y7L*Mo68?AL5%7ET3F8mnmgRy^d6GmWncRKVWfAFu0e4! zT5ei}?s@*WufbF`2cmn47mh#Pgr^+K6RM_~ZhX(aIZg)B2#Mupp+Ym}hdBT7m#`{G zfQM@@X-7J}1;s06PUl(t@%La#_j4(KNgRo!NwB2D2~4x!j7ndupLZLq?@!ZG7NaHI zi*xtP1LX&%E$_y*_WZ!=?!3*qTnOUziYz#35hR@8q*0XiWo_&JB%kl7qxyrExo3cS zUXUkX#20LR)xJQP1_nNl1EswQ-GIP7)|D*vvsB|!dwm?Pj;xNxPi_kGlw#hZ z5qwnY3wDHmV$|P~a1xbXv)L1^DdyUU6Lz{KdDZIP*^B9oUApH5Fqdq53)<9n8rbEC=IPhLr;1{$Lmy`{V+^tR9iYh`^qh7driBexE3qA6M*%5_kl~>8fp9H>=Ez1cLUX; z)_Zbwt}8#IG8W>XT})+5UQ3pVg2nTIcnl|PFp}Q~JreUfSxTkdb0A!hU#!#)%7i1Yji}crC+UH5#MBwb-~vl*=7ZxWZ_S@v+TTRC!f-qy_hV-5V>~pdg*P8RefC zWlzE=ISRYpbB1lL8?&>24zM`~uK?Am@)@+pb2UMlRZwOjR31s%T;Z+Kh2)>N0M(FK z?_5*W5xa6K1}VRgGzPu_X%Ae=dC6N!UrSRSn&_l*7HHrz8+*<$#J2MSD!nIZK=yi1 z1tZ_9CHx~DU#_+4$JxD7XUb?gG19NFsa+^8j)-E)2N7<`C!Oh8`@XXnWl(}R5Slc( zgygp=pRLWeU9QiRhgX?xtEk$}Uc84QkE_%keA}AmKicFJ%sM z4-3UztIW)RxJ*)?@R=VPkvA$=Yk@<&`-8$6%92z*B}lKp;-|$bQ?2ySkV&~C{rC9) zp5V5f582^CX8e5pZt`hLe&t#cp0k**ua*8|Dhr)+uq&9i+fF&^a-cDz^1bB42v61! zKODirbz#aE0L>qg{-?g|T$vvbcM6MhZ;e{@l$rBLseV9YD9v{1t@>7)C8Fw z-Y^;iR~f?2ZcBhZm$19G%4G%d5vuHldJzuLK4Y7LF5`!{?Rm1ZH*2t^Dt>ZylE+Pt zLY4D}xb^E?SnRHc>7)n4Oi$B(LEU7v=e9CtnIA;9q5D+~M$4@ZAWddkO1E+iKZZ?V zO3z7f$#x3(&^^#oNso{0IU6QLJ;%)jTuyux0zIP7i0hdF5PS2JxRWvvm;D|n2TqF? z^q#_hb}z`#c+m6p|6uUQV`!tl3CFpIVr&~1X<2Wp3^Q#fOP8IJrGI~; zSJ~{!G1U*N##29Rk~L^go}_?aIN=!yC!!9s{dLcXI@v)O5z>;M3~4C7wfh050_=Ex zN;kZ|vKm~|dw>U{TB|>j=94!tA1+S?ghRo(WY8Pd0Fpgx$kdQcz}IxrO^?_nuZr$^pZ{C7E+IMw$yTCD}ql zW7Yi#ZpI^xm!j$UI`pi`7MxsGBr=NbR$ilp)?C`?S7%=72l3kd4UqG>kEHb$vmb5d zo=%4`JLV?-XE2z~-?8L=A4bUZ$Q*2#zm5;>(^Q5Oca%42&xQK-Bk=NOZ{a#bi_Nn# z=$xt%yl(jz^l(3-eOPo&*6v^?S#@4xdks{{JVDkc$FC)-pm9I{?o?-(V=j10pp zi%db+2)n{NN$-`@bt{8z>b~j4i1qP3bS0<%pqjtl+GaEjztSC?E~xd2$t+v z%YCLfjLEPPGO(w^U-;d)3D?CG!l4f> z zIa!-T^ybgdXX8oS|8^Dcv$3_jT;z?GsU}cu+FR(908pjyqxPn$zFgg=udJW$iL~}` z`{r%gXvk1(&|y7X9B3(K;uyYuY!7~N#vHg@^cWkS@!<5{x;eieX)pB*!S)-wQN1*h z)F1qG%uLEYV<2PmUGXR{jTvQI%MHI%SoI;V(5kix-(lb;J=~kfepxN$)H6H8T>I); zn=#d4?!ZUTtK<~g{tOffFMcm;!V*Gu1Jw&O-EmvcweqnJZy5a?=caFj3bQio8^4@W zpG#{KV{BAD1lD0sPI#!jcE1{ghM+U%M=q^fMe^*B7dW)t6N3-0Y_Xd`W4IwRY{`^9zQH zZbr^BvcM5qn)em;`Wwo?vMX*2hg9X`>%I~V60$Mhw!3_`AEaLFK^@gQA7?jBYufvArG5SsR|vZu==sQ=`%Mc3)w}OJE@*9sZPZ>3+9+gjfoQO0y}}fE zaM^N6J*c@d&;-(qAL6Kl19D~YO)ZTPsNNy`cWe3a@;`|8+{@m1bcE9$@8C+yF|4oC zNf37ClCTBm6?-zmLl(VxzMSE1D!;|Fz|4Dt<$=w=!KZkF%&xe>%AWow?6dXg{Ij}z zl4FQ;a_uf-ZRc`Ucu8CM8CfmALN+D52x}#eN7KURnBH$V@37RJ(VWVB+f>TPT2fsZ z$*}lSusBptUXC}#_}_cjEYo}(zhf`vw(JSp>grhD{O-&=td@@E6$edd&S%Gs!e9PP z@yPw=(oSnY+;yCu5ietZKl!0W!Z}4hn#SWwGM> zoh4{o{t{>&@o~}>7;`2JXS&vq%fkooJGQsA>gQ7n#>0+yH)+=2LiY2lV0{W(6F2&3 z`=_^%pEf%o^{u=*V47tpl1HF}5TWo)>B1T7V&s&YK7MDJe+Uh-cG?|I6&L z&d9OU<9Bh?v~b!JuBvz&(-D{W48h%W&sVRs-}tNLQFv`}360Ls{_bhE+9}q%kgx&* z!>!YK9_AO zzD(!Am5{b6!gjAy>3-2*K{c3EJ=zyrSl?vE3+T)<(l7PX@53C=Tt+y-hA#SoO?M0e z%aT0immG-1J)HVN=G{z%{!#RN?TpqsS`VDKCIl8b48vh{=d%dgWw?K~6a71#BV?cl zG>6oKBhht;ju9X5kf4stE;$E@t3cr;%_ZE}Kdh2ZyH4wX<)52#!bCW7|2P`v+i=of z!fsapuD;&~X?;b_M@5XVkB=^XSJ}72H#TBQS4_6A!-#W|sL$x<+mQHA)HWK&RNwnX zZ%5KQu+(%2)qD7e^}=VPMZINchQV+#sxQ z`HMUaZuilM@RlniaT=R7b~F-~sd`d+6f?%^@d@$8iYI|E5WVah@~JL`Ox5Y+^j%n_ zq3>G#+i~h85$|awY2LNO$J*gOwCBT53z?jkk6)8wq1U2SIG|@aEb61rl}5Y}w^yj2 z5oZzaZe?{^ZNmv0_b}2@NlE7>;`1UM629PoWJfTZ@CK|x^Mu_y1HR5D4WDn;V!b1Q zm^i73G}G=%oTvANab)V36aCwzKl0XtcI zP*8uX@h~&TA{ag4KvMUBH8hv@K!4*x2f7cEzLzJR1m}xteFMhx@veK}eZTWqHS7SJ@p>z9 zeQQi9+YENv0C`|+ThbI)arnl=Nz?8>VYEKr6*UJRI$uM=NNilMirhAvYLjZnujfOc z8j>_mq@{LYrwM>jX%5mO^ayqy$^Sv~q`9E`4Gba7ryY_u zV|6pOYH#Ib0dsZbG_OFt%5v}Z5Et@ZR(dm~-)4lG^1G7;(7d*g+Sji9&ivbg)>KPg z1XAbc;*pqd;?m6*bT)y85teAkL*l{+eR(|o2rkO>;Do0T?%76-h5r|K8%9mDld7I- z=Wk=r-ZI58q~k9#&qopHuyLT!|7phzJJ~4 ze_A|;Hmv1@bn?nM&}C^n8XCR9$pa@RQU5_{<5Nue3AM&lE0rA6&?p-e*MRPjeUew?JaQKp&F+(Qy8keryLlv1QiC*n6Y==mGId%PIyP}2l5?Q!=x@K z{gqe{oMh0wz9iknljCYg#ZTntaQ6onOiU_*#}+B7*056oonvHFjZ^KzWz%hpG&GP7 z&=C)7OZ-nWrO^~eD7{KJj^x|5$|D?3TB`IHyA}}0zOOE4TiS5K!PaGB~d$f%;k_i@*r6$x(o}BR!hSmKH z&9nL|?Z9aLVSwKW(7L-(7I98-2h4B}f}@$oxWecK0Xa<7o*IwJ7pO--@jlfeBOV6A zJGQ9LIt_V59dWN4;Sn16y{pV72&WZi0c9ULsttCqtd+tnopHUrP=of{s`YM%F1KYG zqpU+bcza!8j;_PfWa#nljE+1G`4@9O!E`E8Mue9&W96uX`aA*Z$QfzgmHwA%M@u>w z|E3&#x1wz=YDaRWHIs2Nh;BT2K!?g9VMZ0STsI+xZrP^k5lZwqkE%08$x zl3K^15l0o)Q6|wJi0gpz5=K~s)HjmGhGWjuld1=(UdTKApq$KKoL+f=)@!}eZo>DU z8D&__AT7yD_DWl+e7x!r`9d1Lsmdrg3N9%$@1BCDbDIF&W2GL)X>?SMRsAe`l--EdF> z`B2K^6fRZUV$1&=DrMxyu-gP@NGRTwME({m>(vmJsaqKRE}N1Z4Ep`;L_%?G$_O^$ z>q(tJ`TB_l#ndAaNL;LZBVKTRg+GFiDeaV?eVw52T44$v^EpbKeMXH@`oKi|W7}9z z-T(^UT&77J>DK_Ke0auA6@+VjrUheQSy$fM{S;~Yy^ulsJ0u!U!k3K`mG*>Vbsbb@ z2Hfj9I=~49(c~(4A@mAHG zTjP>Dl^lIK<{cw0gBL9|@MG>bpu9kNHNu#4tlkkr@n?P{)^vKVdRb5&Ac*G}X%R^} zsj`QajwD?V(FR9w!AJub81fy-LsL#Mi1#mB3ZIB?aYlv6njnBjAT&;8P>+}NnGUKEmf6A6cWzrr+TXf1xthKw& zZu+bfUnjMqS~y7cc^u{}-2^{3M=8%NnlJ5-gjp)bU>*IsfX<_kyv2WvGBvcGY)m@w zq#!@1>V%n;6tl-OQW*JfxolvjP-{E3#9C!-iW7zU({MsIJDgO`ln)LmzJQZ{KdFP|2s<>ubAUGiZbRM@L|6*Us$d$$PWn0W*OxVs%AL7FXcCZ zUW#E*N|!R+6COqUu4*{g}!1dNdXE2VL`2qZlS&U!w)o;_xfHFje0ec zXUi6gaQ|bX*V%i}*wRqWnNRzB9~=ympD*Ao!>hs1tpV6#umx^2-@&$RYr^wfFRFFH z!HwJL4&3T0UYxLygjx3{gDadLyPrEK`w47DVTdddbFGqBh`1IXR9ANt!{y7Xr_cPYA>z}O* ztAzm7E6{v0i&+gN4&Q`HQ-{Hj&4*xIksGe=RGqKZ+o`SPafH|GHxE58-$iGxhaCg{ zz_7!=MCXV7Sy*UasBN?n2YK9vxF`GgQ1`0R=w~phZ|AnzHjDu4-OHbE~dETjQd=MOzu4SzZ2k zpDNVakDZW7bzuW-oR>(dM=V|KE`yq#r+bEnaT+7&U)upDboS9Avvla(g@M_vv5?n5 zBlVV&4*Ajm%r7RvcRgQ0bI!_VJVd>vzYKd6+r?cG9h`zJltk1u!+b7?L`r~7^UCpUz=s1+~Z+Y;O z&eo-R!DhV+fqKpLq<0+V9>~OzyG(fB#Pu?x$9ZP--w2Ui+!g~Lu7}a`e9~xJP&2wkI@1h_!`6^f) z9?ESlAH#8$&ah(AVA$R28XHoh2G?)9R;oEo$sU1JPYM%Y&&t}I<^wNQ(UYsDMnkxN zGuiH-JMZ zeXe5lc3KQt*jz?=?T0phR;@0m{MV{hL}XuHS+Xnv_Czl#eu&H#-SGgnmQbC20co6F7k&}JJo z*?(i;(XIFJtVt%DIpd3NYIys~vD5W1ns=s2TPwQ+r((lAZ>(wgAJ|__#PEv^K|wCFXfLNaByVZOmtT9S zTYwcTtJWfXZW_Zvp0(s(riU}tXKnYtLQk)W(5TljoVorJQ<(JW$1*&6*ah6%wwBF* z_fR+tgm1Wg+HVZyJrdxm{!U*cv{P?k7SZsFNOIh9bIH_*Ea~!FIC;L zcb0qb+|z7m;j#y7H(oDv^<5d&AX9inbI$8dUxUQW(B+y|TG1KbR)L=UpUE&meTo)2 z+hN!C0&UO9aJrqNp#ZX`Nkn8QPL+L(NSw}yHUl0S|js+SsGkVrbHvBqLLtG0* zmtC-PX>ZaA!Ju&1By7D%eR!IAdw+$ZFKwV<&HG3g$lg5Z0gsKWRJ|k-#$$d?1AcXZ zwN~}D(Jl{3Jju)tNKJ5VIjnr4BThRBz0P_d=>}D2tVNZBFm*8E98UX{>eb@sV#i?I z6B|q`TaS_T{9xPTy-4+nS2};Btgp3DuivwGJJ@bFBu9Pt|h@S1HLZefbJbrau>Z|T@gZ2PeB<*5^ z32@Re9=!^VVarSYn#>inC#XwTBy5rwPIs*2y0c?_QPpL@&@R|#XdT$>LHiCpjm8_F zi}8oO5Zlbn2@}m_T+CruJtRQYG~Aoqjp_6RtaNrl(uZ3AzopF2(O4TbZ;zz0^TDhY zo?52I1JOv*xcGrvSH!)QT-K_3x{`~Dn`BMPAFSm0Lu@dTz7I>W5 zzXfSf+Nb^xF8pMzukFql^f?J@xi)7Dy7)5E zms+pX7%Wf7AilJa79UNd@&`*!SJ##-_ho7wCqMrph|ff?;+uG~+gW_u%Y{6Hu^jyI zBA$r+goOE$YMr?A0X*NnM{M1ElhK+3%{fs0YiB=TlGX$We+BU`X_;g}J+33{z~JVy zA*jYBB<%|?YONE5SCaaJb$zje&3+)j_niYDJ7J(axg!eN)*mj2JCue2;!up?kJ!Pg zwK?f}t~g5ho#nwF=|0mLf_ejQT|1=vT(1S{nYV^Lee21bu-d$Ld@R0pEEM!}g*igM za4Q~*swrw^b;7e{--JW#5;XTu(RFr6CQtO3RYXR!TNQhO>PqPJ#F8R;f?Yk_dGTYGNZgv+Uefh){Ep;-+y25jMCSy^|q_CVZ6dOVcPtyz=SyNRUt zL#xAgA$H_&Ade>gJsb@Vv9wp{nx-=6j)~HHNF2`MGdDr+fm)@*pyat1s5v10h!z{_ zL3oy}Jk`rsoE`fL$P0e zA!%jpJGw`Za1q+>_vKxk4fxyLEqJY}?RbM@Q{-Ty^J=Z}pobx+I%TxRjQkGx8bm^~ z-V=02yY4~zIgO}}MrvPo^X9|p`(V0$M!FB2jW+}7U)p2lobtk)@Lx`#bAJd+pjQU@ z2a90#$FPFVtu^PW56Mr$vL4Gozru(s?rM=41*QGd6&4}!7>@jJd*yplZ>Tvy(!A(Y z?JO?d*%8QFVtCemrZl2$^S!M7#&BGJ=U%0+CQXcVok$uOhI}?tcqhnnA?YYg-@K@D zUU`o@xVUyBc>UFv&*5#*HRudD?u!$hMd{pLT4UxUBOqSVE$dMsTCOwZQS&VMwr{({k`PmHZdwf-o_>Oo<*#6U z=Tt`A3O4i3Vg81eNSqG6eLE{26+QbFB~g8&dB;JTFPC;A;SO5Une6XZRTBs43|n2N z3zZy7ID%@fVvnT@(#k;f2(D(~K(B|Kuw6*njt#I~saw(3ONRBk5BalB=~k@B5UyqR zoUm1=H0x~Pu5>CtN$37i-GDfF6jBTaCn*j51Q!q@Gi_1ii!BF~zyc#}j8GX#?CacQ?>` z62|-zHy7T6K6dpb^%Nd>iHy8F{IF+0zDE$(gIQ=Xs(LC6d;uy?A*_1obmySw$K^wIU4Snn+XAl?SS}3Te?r^C^rDqm_p-U#(YB~drTj(Qn%9i z3~#plkZ7nXvQdy0pe=Kzv|k7d?llLVuYo*c|$4Tt-kd7PWSkJ zXbx`{=YW3ODvWr6_BTn@(l{%1sKRB(^CzH!?iW*B)Hjj#-%hQmOD$~$?@l==9RcKx zarW>b;>Gt3Qu!Uycu4C9q}jx@z?u+umF_1g{mk@&t4_@C5mUp@3e*Gv)wJ@)5W95%j%-y= zM{BO~8KfKnUmg1bg}TRJ?%xfnPwoIpKhy75WwA0>j}H2 zJ{OYI!V&%;)z^7p zwzVfGUSeGbtQDO+$AiMooV8Zuo9J2dzt>0$Edc5v`nRWII26;~wxvk;B$TcUa&Jf1 z+P@NCX4#K~b_bKOs%9V74 zrwPq(+|sC8RGFrJA?>Zz?QxP>s1H(JiBwZ^!-wNK#e*tyB@ZHEN+zqihnWZ8>69*| zn&#y*yx{0;Abzi+@>pR>_tPtXt8^dbL9EC1GL@Yu?84MdI?9#GwC8(Yr5y1xjO@7= z@5U7{<(a9T*tsQ>vAqbvD&N`>hPD-|zXAd}qnObevO65X*^yU)@*tu7j`I0_-g=mA z(_CKo?Z~eLOoCw3QmmNcq_P9pH=!IzV`@~cdc-Xa9{UBN($vcLQYIisON%)pPax@9 zZKDZ|F(bIS!cLXZAYn5a)_SSD6pam$bexVfv)JtMgAo?f`m_@_9=%ciMWej5%86%H zdkhPYdnHj`$;g93#e)dZ;#DIgou)E2HYg&6G>oC796}NoL6?g2f_^TJ-fzR~ue#xV z{i@PpLl|-A_M`#+A7JX;6FA8A9z63(Q(1&~{q%dl2JOPzw5*IU7ujIW% zt+(g2VNRns<;&!a>T=2&k#K{&(i=!WN_w+bJdpl}-bHjS=7&zC9~SU(dN%3jky1^| z<_~b|cRjGT+|AGZ7%l$QkJWztIFw(%`W6zrs>y}T(=>e5XlQZ%rp90emrjn84aE>cAsAHIOZqRfmt;je-6hw&cCTZJVy(bW45yZ0|rx zbD+i!>T_@IT)-cXaAP}$Md{{`od{jqIN{j=HTlY`%h9c>0nTao8T0CW!p3uJ$yI(Q zWx|f$a#-IPven1k7ivT=-{P72dF0%uj9!P0JeDy-$0o9U0BLwSyKCRQXdHGZMUJri0;j@V(S+^=ZP$;^ z7oq0}r~1ZWvy2Y%rXFz=4ufg!Yw@?UT1Y3Ef}a{Rl~>yQ(WP!!j%inV^7zjEH;xYr?I-ZhR9j42zhg-@E9*-czz+NV1(3yp=YI3Wt323uF zUe5ZnkL~<75A}EF>lR*oLi?AtMonW^zI{#yIpJIoPRvS{7ylT^=M&q5(Sf!|WKsXBsC9;#N2~CNL565%ZOYeZ%+aRBU_3YY zJZ!Pp$*298iADQAvlT;1Fubg~?7UzSjL!@cDHbKTd*^QGXzK%g=1oU?%Q_QK!(t62G*XnZWb*x|OrKT;NV>w{=O`-a4jov%qM9(N}f4UDI6s$%yhe5Ol+c@qo z{|43KSLwe0t_Llt*3QfJb;2NY#Slj0fetPUc<%AZ_-K=pFf92Ef3JD4=*-RHN4uAx z)+^}f9jF~wf~`K*VI{$*MT7jl;`Q{ZmDl*V*aNF|Z!Zr`uPd$NQ=wt9j%Pn~5&oy@ z0j)ic`nQ%Jru}I~b)r65Yp;8?J{a;{H;9#2f5HZ13Ex`O=Vpbk@L1d+RkLFL?cOr$ zQv=@UrvU`jy@-47tm7MuId^~f5I=_u$G8Xo;iHsNMq`3w`pv;fc9YZc6Oigij!Nqz z3vTv6ll49M?oo4L)rr|~a{n%%e&m-+_o97NoG|Z{3KZ#_~(!u_OQc9e!imv@Ab5$_Wt30`03>;{2qQ>KC0Lw zu0GDjD{WR`S#~P4Xxg3+?j8XZbsmX`SsG3lEZ;`Gg`GuGHzIHp_@aR%u7O4!dvk1l z0baMKbF;1=qPYsgiU5g(hxlw<4O!CYh9FMC5rMflc@ouu<660{adYjvq4kik7Y3eN ziS2ZD7_xW++py0I2rtlWR|DSP{xY1c^9J1$o9ezLxUkY{TCQro(dLGdxQU+g)8c>M z=VQ-J9@4Pa2&Qmlg;hInyEUFAKd3Hy4yh$uXK&y`FSU{%o{!>RLjQxB9qO^6_n*My z5s65z;iKl3h?+kaLA%^6xNe&&raz%<@^44vrV9xNle9G=mjdr;BVW=zmO0`$z+F?E zQ{^(@MI9*X5hH>&)W>wcFuLA@&}+_B^u++&JhhOAwQ2^}x110TFS}CzC9pcLHS)r< zQo_1He0bS@Y}tGSw5Df|#=pC*xEljk-C!MUiy3hX-*I&dX1?{4uLgX8gX@p5rMen~ z_p2Z}(^!^d_~G$Jd9d%Eqa=R7;=X{qF@vTte<x3l!!1rm#=(~#LTB-?o zY+;a~YtH#=XZwGwIItZ?mBOP_?&aL+DK+-3o;g4Se zCzPT8;>B>eWiFiW7*)BxW70pUpUL`PoYA^jOZeP$4rj)9v=$A#1l1Qm({2g6o;nFP zJcC$hvvNUv4bPgrN-{n)46^q166a+$5qPLPWOhr|HT5m zmOa>Dg7kh!_%3u^?Bs)jziidBV0=(F3Z_yvYj{Z@aTf1c;R=my+=zPx#*FU^|LV7r zdv+&6#wlZ7yEp`|+jd9lYr;DloOh-UsL%9_^aaCS5AfQtOX7;p5iRuzY`4ybn67(} zG?gUISO3Ot-XQ$^;w)!+Ooz^TD=@Vk>GXafx~T18Nb^GLv_QA>UnzRWjR)P>EI3B@ z?HaY}gSnl4i$Qxo;e!oZD!Dmuf~Aa@l?0tUPvh+QQSih*TswA@IrMlOK=@q-=EJ`L z)h?=34-tw)mTC@D?oF0s9g9_pRK;O9$JS@RJuS|o@)T7T6<7Gx@$E{&1JKyWo+-> zaoBN7hSDBzI-&sQhiuT1{srPsP4I|*j5r^v?W-=mi`S!P?c2=8q`v&GU8v%E@vU?* zk4xw!^~cz9y~U2~RCofqzWRXP3meE0gF;!tj_W|$T1&sfh+kOW5o3Tpm)D3a$0N^Y zNTu6|({XXBBR}=UT-CX_ZD>SirmPa`a|j=Fbu&+5zfu3RC$oo3aXuQ|#{8!x9Vfcb z9{884oI=t*Qhlacv%j4aC24l*8AHWKq;=N__lI8W2_M3V1K{f8Vdyn016CaCBdB&H z_4}oe#npJGdtdZO$|4P)2i83b7~v)i`f#1IiG`RqfWiMYkz3@Y89a_h1bp zjrgEDyMc5xqk3di@0GYYzi9-7A36)e)^3LeXerV*R+XK$`GLm$9Vkt|{>~R9%;v-a z`1WTCOFE-3C(<4uR3muRneKOfaZGeOtOe4qI`RbCJ#TBF{@e3F{ZHOvGB|eWLcI7# z>{_^zkxrsL`{%*BTZ6#7le>IlIaFZ~vgPqw{oP}Lv;mTK0;MmC_Rr_^dPeoGdS7;q zHDts!V(HgR>XEHLYosHt2Bne6AD~=W1H3vI^LpW<)p`lr5>v1r>4-nUCIT$TuG}5QOop zn?Y!~=9LNUeJ3P}P*os=*LwmzYto9he_Vz%25I-r3u&z2^Q94#H0mp+ zMtp>&>wICo;XWV_$4Nt1)=b7oXMU&qM=@%yA6^c-4CE2S?wg*Z2_CT8`h9t`s*ALQ zv*O9?C2UIXQl*mx^$xdQ;Sc1CKyeD;2-h@D0Md?FdZtKyrmVfN9eJ`eHa7UKj{LCe zm9Ijx>GN<^wN_55(?*(M*7s?D0A^cK^y?8hG%ZxR_WVLcpnAdZ`bgc{&7Lt zo$9_6sBbypnNIm(^4U0IkTpJ^corK1E#J^@jU59a7eES zLrvF1+pkYprx)pBV^1T#X2DQsT4e}QjY7(_bZFapJ&>PKb0K0^uE3Km+Nf~@=|dpA z7SAtjqQBRYC$uCkGnKYAyYe=@E%?{?9k6Jn9jFY$_)xYWeMWuV40oMu3@*=(=!i3@ z)~~_u+4mtO_ms*;fcz8AzkOZ1ulshO_r*7scPsOcWU~x-Hf}2@y-WE7P=9C@Hho<= z6NtA3@io4lzMgQtD%XzHBduY|@~<~kUVvFQi$T)iNLo#h&qTr`?X6qLvdzE4`KT>O zeFs+K-@&YF3vrGYeYbddEUQ(|S*qWy7oLZNy~>{gc>*|K9xBLpiP`lt#qrnHs5~+0 zKyeF>Domhp2g9Co@5Q5(dCHF|JmJ;*{)FCJm*B}sAC(4D8dZ=_hm2SK;luM>$~hd6 z<{P{lToZ4jX5gZ*A<%Es8us##J>1^Y2tTH0py_2ZooH$X#BD(HCQi19!wFrFlb_I| z@3$Ij$xjNW2s6AC>j{(zkf%GORUS5>;Y7Tg_*wN0;h_s7Z4WTvI8vseOWgIIp5?3~ z{4yGI@@r6cNeHmuMP2C%34soNPdL!6MC-xf)$YFgg51O^4-ud z95n6}wkdTZoDIV&df`Z#TT+b*&*{h5JDbIfbci5b2WMwofuehRL3B%@T1rXWR&kti zDEC*ctKsaw~M(=SLrq=SL9ErXI{qqVa7l64h&(NHg6CP`+ z-uwmOC)CKZQ$7Y;UHnAvYeDb-NpY|)v+*A4=cZioD!mV^w)>%a58DS+!3oP=SMmt? zGUZWdt>%+9_Qj%mR!XBY$B?}^#%(wf-@$*mUC{2)QP?*z1rKljpjC52xrfYsx{=kJ zVuQ?hFtl2E3e#N|V}Qdgh3_nK?syzs?=T}ig{QltVYTN$g%c3lL&rQ0go_qU&$2ZO z?qcoYx~R0O!ULbyli92F7O3(o%5T7F+6?y5%SWipgz`6Dw%=c~!5r{bZ$d4U%uJ^s!a%L}4wrV)nAFhHY4W6h{`05R^j-#pg5^ z(B(7jH8kK5ypBv0?_4BIB9Hz2_t=LMuj8h~+VH-UDeT?*1nK`N!;zr8(MLCDm3u0DK=J~3KW{W)b-GsNw0@%* zuHN<(Twl!=G*^mWw1=jLROS*7J3Mj=E`F|QwkV&GHw5P&HPGSZ6^txQgYK8-$ffIE zVcDbt3_sK!U-UX6Xx(x0{-$7fiK**k$!7i8vO1fI?_QBMJP7pEGyKfARP!U1RvJEa zEot`$V%iK3X6b!R%!sPbcP;$Q-W6WJn68#8lj6j6f;dc4c8uh&f%pnGt^J5j5!-bN z8^}k1-)MnH-BQ`V*sFptL${^>J<7p+8F8|_KYbxm=7?GQt})6$bx$*=!^!L%g+18* z$OUMBq$803p}H~j8*M(MqYOZdThhJK zH|EdljSgPX_)jwqRSrYBh0<_9ITz$_uV9K-x_tKGlnIOR-tn}~J(V{l-FFHu*z85} z|CE(o0>!CoUR+Z7HD%4WvD1q_l(VEU<@alpXMt5$SIir>o272}$udTsV=6~d{Y_&; z<)0{XP`nGpt?FXIsBui?qKAi`6vbcWqRL-Q`ma~oOAuFb$_ZfSyhk{E{RO1D!}acY z!XM&c@}$2?BTLE?V2GmyC+tG$NH_DOp}rU~sUrj|Sj$c3yMqIH++{1Lp}{kq&SByLUK*Mvns0Z6yKQ@MiyEKc zyirH_yKE=O`P*Ri&MMsA zqFC|E8anF5f!m4ca)ebT9-NzkA|ywAJU0=%gZBy34uG zHMtnGvp?sv0`W%o`bal-#Lh909j6z zzf>TC7ET7MJ2fyrG=e33zrwfDeoKGrEChAUyj~4%)4~~H`#`?D;Zr!2j2fc?FX@v| zO$NUVfVV#VWP7g~8d_^kbtX-67s$?od%$Vk0b~yk!c6)uCZ|qIo;|+sQgku?x2jUm#du)`c2T_% zvSr`0@7XINNh!(_N{J#ONq5d5+1F%=D6+5FW%oPN?~nJBx9)xJ%$#$+%el`pW1Hu) z?u!5!X$yq?Zl9PTv_>1kygT=epnqUklK``w2RPF5!df zwM3Qo{k>nZk3Mt7sS`Fhe50AXWBUhvnm6X(Z@z-y+%|&p5b~18$fz@=xW%lRtaH5& zJWPD1u^K;^Pxjv;O{wQPZlXS<7%$h>yi^^l4IK?DDzl;f_891~GlY9z-6CJ>c0-l> z!}t2i`AxP!zU@$LZQ_F_j{;e=-et7oZKcW6?(#%y5Kj%)vb0U@aP|^9_r(1R+Vs<4 zojfDCzow@f_N5S)kK0Pm}2{GaXw7*?wU zm(@Hi4$bL@CULa4jCl%r9heP+m;F zPg{p_!{o!kHkujZ7 z@n2{D&AzXUu=C=r9`2T1OLVy2WMet3)-d+u6`k>$o{rzf)#X8xrt!pMv4n4Ru#e6S zSa&1>Kect_xsUX4$gGcW*r%pez4yf8u9ET}_SPAC=aVx^sV#%-0YT`#cg zpEfwB?=+#V@nBs8T)8y^D5kvjgFEo)en(Ci!zsU@p2Z^^Qs)HZK3XePycW+4fWLpV z=svFlPRh38s@6ZeJc(C(kKqC1o}h)HG5_V)6g%8pfE@-$%H=qV&wAgB6P82p#6yHz zvCSys~%CI=Ow^MNZ=$G_+quMkO`gi9I}(zwcjYCtwl z8VuJXuZc@XzQgctJAi6WsvH^>)tOuG>yF>2(fMI@*Ma-}Cs22MN3JlZcI5Cp9j_vB z=R{xHOL-%ln=l6FzIlwi*Dq$%Ih0jT9N_API9G@~^H z^P4x=C_5c_EV>>)zNIU?@u(rG{`r>8Z`iDR8wmeeQqI+37lz)%i(Y3ja^(j&T|E|G z9%za^j=G9qp=C-&~R3pq8eW53}Gq{0*Do?9`Y6oui5!O(c!PK1lm zTH+RP@yJp5%69io5X3vIp2cW<=ayVLlSKL7RC7Vo8RD z+;?}or21gh!;K~3C0x5{Dfe{mBvOq3V$I;%%qr(UXxeeOR2;R_$VJUlcKOp=687T6 zMWgaMnT{6*tt-UQF?Ye*!${0qXoa)7dGgsyYSG`zaLbBLm>1caC)XV$yZ_eb#2ebR zs|JZZ8PDO%HETh&2R&=$h=AXR*z7CSczJX&CY)adgLmr_{$}91_VdC0tOF#EQyJGR4;e4j=0Z7W4biGgQrC`mj+#Vg0LJ>#@H}7U(!{V`gsD`yFi#y06B= z-u@%uYys_&W_F3z;4vFeV#gJy6YkL7VVQZ1_BEi-R~mwyPqM*?sO2ya)8J9@K1SD< zgkyN)I{A!E5$HLrF`u*T1*n>>)8Y(xKk}F`X&BL)rir&gF9G{?772 zQeR0JEmgfy9pSDc?cj6VXkoI<2s_eVt3lOVfjC&)`)ZG*M`Y>8GqCctt!UM2uy&JE zBDT4ILe(Z1{%9hJW3lTlH~zJF0P*WG*V*4^$*1!LQ2tX6*O!*V+oFzhQ@k8~1U8ko zkV_i)^5{>o_^shaHokeD$WCvKgkkcT=@I;Mz@2YfTgtW;9+WB;o182p)e5}XcZ@h< z1FD}7lIUCk(TNAWUP*X#j@7&rN?LBaxaeGuv|Jqi`_NWyD`=te8fbpvyF6Tl(yoV+uIC+=wwq|bg&vG)~55T zqsJ>w1*LHa8{t*X8Hl&tpc(l_qIWFqle8=b8vESW92?XR^ODO&7Z(FO;XV(QHc~Zj z|FS>*eK>^n@Dfg0S%fhkAi}7TR2X#27vSGmtz1 zdK}-OV$NpMS=%w2>d0m~YqW!RuXFt|f3#*}|Kl2^QAV$<;yWaL$fw|CNSa#8Nar)c zG*xqaZ~sz_*PpjY7{*CgBH@bm!20J9oqHYJ*G&e(c2w9m3i23X66H!4nXo<|+CH6( zd%Nt>EDM@X`(+*yRP(rSWGzkEvKrFpZ6rLAb>)YI9*~nrXFu&}A%C8`uO&Z0Yg7nB zKCcGyT1Xg=A6~i(${lR(SDkOQ%0RcGJn$U0iQV_P2E>-cv&h9 zsPyoS7G05a0P*N*%`cr@IR433;?HtHwTn*;_4x5uPJ+A#quLRd_e_OrnwyONpNFis zkaw$h1?$*Ig|Vo7nc^E&r^JsCzNC#X^;)QOp13tWRW|K-AHL>qhQ6?g5swQu*Xg3o z*>I+3+Ei&&xc}D=_q8_V^>po}L)<$gjYavk7M|Li0*Z;)y(Lz8A#o{Oj~7ho$t!Es z5Txn(n3m1>vj;P9aR8k?`oj%6=M59nK4*x!IR}`jWo_Pj+YYEv9E1IOo)Bd}XHZPt zS$~%f&|z8uPz|^$PEon&J2P6_{^%xD+G5)7_EOKWi1fE7Yp@}dO}ZFB_klZN;+1}C zUP!NlQ(Dk#o;0OBCYECH`MGLts5f~tQ{I@?4fRa7Rrwdf4EeSAD$2 zYTT@yWov6A*8TUos`(L?!|yLfOmW7tiaOl(Z3pFJMA@=1ao21nDqJFr=ae7pvAKzS zJw%{WU=HdJug$;CYKpY3c+JLKYdR~Gkron!{doP{MxdO~5QanFynk?ehzXzE)L)R! zk%UK3xV#%9?xQ(({zZIctw146@vO48tWk!lpfZS530holvnk5li&kNSMv0j zODON2B7Ij~aSvL|zaSJ>D9ol>hmF4gzE3+wPU5ggD3R{iZPqv6OKS06&Ak8c<6^Ak91HQ|S3NtzBIZ^(|PkUr2on7#1Jk{|# zQCTZYs2VkMsjUP(eu7z!LY=P=bl2(W96B6kA=P+oLIbP|2 zDm*9r097Z{XVVY{F%={7mZJ3jdL$gzoM==NOEQN@<#R{}Ak`sT{XPhkPo!LslRuY~*VJD*#!gzalZy_nMEjz~8uFa*MDrc*X`f++Q@-%)Fd4}I;nVr73ezH2 z&Z5JY7XjfOk|z_oHumJjmqNsP|?pVfDug}!0QVh+1t1yv~mSw7MN7n_? zsBmEYL`bTUL30m;DV2su7@$dP)=Q0DliP8XiktFOeC?`R$}g#$0P;`5;Isyh|9g$3 zQ#30NJ!TX;5R|FiSkO>4trwF`oapi;*7S)Ptb@LnMn!D;496>%$Ps z1>!`^d*vmlzE%)l8pu(0AxJzaRX>DkRq0}i-DXC0q_AF4KZuQZ97jEbXQ+BuE*{Qo zO*1Q%>oPTRl~{Y*N27TA65WqNpQq(Px;(2*uqa5?lPb4Xw`tFBE*?StWD51AkhHuG zuhsDwsNV3*S}T~UY3gIa`(76*$5Vj#2wv@~E0(7n&m+$QVbAjwFQMX=u)+wYVntq# z`8Cag-6OB7{xXpE)sl||!Y)SoNHhDf3vX&}4HUB~pE~S)GOLppQ+0jn?P+dwd9P8u z2EumbX<=v&9VEPlZ}u0Jrsso3HB-GD(W=8vM)|Ix{xuSZ@h&crU^)B%ES`A}sb`1! zYZi*1St+Qzi)Wp!(CTZ6Mrj76#Zq(UG3v|Vn3g-(#0CyPK7lV!b794!yK#CSG%%RK zPBdBt@wZ}8`8S1+q)GEqbHAv*yLj6#6UhHzZTgN)^|(m?!Lo`7)o0ZxZ$LFAD39dL zM(Ls?!BR_pN_lCZSS!zu85VU^Z-=j}R)(8j_tVnYnd+xd?o*H5LVCTR=P>7GG4dU3 z^}SAF=Gr(xnio|KbbhR6uy3m5JrZ9{PM1s zVB9x>u3^J7&8qX{o{n-vaTmGu)+(8reF7k?H9TM6Uh1~r0}Jgl;pTxcve@P}d~dTD z-qcJ%mp&gM&qo(@r;SDny?xMhTfF#pv%54J(VN+Y9YME=rZkSz@Hc)f?O&+R=U3jr zhEdG~eID5OJJhn5x&aXD#E9{T1&5Lo;g@rpJAh+;5L}i~BKZh5>w(vyO zsd6^XFC5F4P5!5i44sYnr`ll$+m3j0^7y=QPG)kUPM)kBYc4Aey~o?1m!nabp#4p% z^V?15O0V@Z@TFW0LGBHJPMgGcFJeThmJ9MR9xa9&@p%Wu9hW43F?wN^c@ok}dOiL_3N9TU!#$!pl+`Nf?`I=9GtJFHN z4*9erWw&%;pT@+R!gLN z1j~c%E{c5)N8s{1U#YvS9>maIM3XZNv0qt~*lJ8haz08qXV=<+BifAHF=JI#SH> zL%VkJjh_MUTltcuP8clHtO^;eFJ3maXD8r7L7|`UvoE_l}iktw*nt zJ20kX0POr5&2_aG)Ob1G93>ZK7aqSI?Zb3ab#kG7s~RziWt?SJTgT`OLd47&o@KexLd-y;s87^Nnjp*S3{G0 zefe+HBs7ib%tl0ba~dPBvCWd^F&J%pp5hRa!L#YotMhJF)dhrht7PNi?K8#lhCktIQ&pknP3+~)lY7iRmS*P9xQFoGA& zwBtjfPEGi|$?*BjT68#nl--C4XH zYT6cHa6lNAHwM)B4TLn=f_KQeMRlvo=jpBDQFN~Hwe(A>?(lZg&rDbEFs3Z10W=?Q z$dsV-Wttkh3|ZP;9$9fgL;1~hYHd^3fmFA#V3%RXW6wvrsxjD&aN~*QbT)4_<9Oz!CYb%CQhZn%4<+roLX1%AE6xvL7V46d!`Y7F~F ztYsat2BB}w$Cw+pUgbGx*LyHpYw7HRIACQ2EDtl3MlYsHN6!LW9<~9BX-_@h^L9vU zizE8SX-#wO72g2mFVcJ9wqIlZAg~^+?XLsLamKRwndGY1>NSkRUGE-i9BZvbJTO%d zUoeaEZ02Wp2B_}Eg|to>6Q6^BNu6>0JGtYelJS9PYwMwo(Qb~J=#(VrpM zslJBRneeR;RqfIHn7g|*JyZ3A(fIgtxAS<$zcVcKNru}s+fvQdkrrBaPBjl7^8Q2m zJV}_NAxv_Hu^!&;oMnQ&Z> z_sY5q|E${b+n@j9uocsUQLk#U^`-0Vd3K7X@|_z5neD)KX>*`PKIvzjx{UG^!EgYr zw-tX-fGp^vJy*3bd4x4OmA6s43QqX#fH!`0&tQBZtIP_+%UPSjsXPOnbq0y#=C0!E zv}owCtseYrz8RC7=P}}9?7A`*Paf-!4-zkEy=OjPl;g1K^&k;;D+(#sG0MjbHcSXa z_mmzu3XpMfB4M*8bh1xF4(x=sUT%FhTVP_fq2E)q%U{>vsvKcr9=} zmv{p$cQ_(lpU#qMBGaBWmcB_x*sIk~X-!r`YKaq8*j#bNo1{1Y{9;dfHk2c5Eu}|z zHFvUZ#-jz*CNr{J4=c9%!J*|gIw$Rxpm<8cJII}8hP`a}F%`=y zuG^6c4>$iogQqufv)5Btaq1$HcA|Y|N6XX?KeREjwRFtgMf2~2sp~IkjAt1@)3IL# ztr6WTdR%<2HWRHY+OiV={VX!Bxg_n1RR5ebD=R;DQwEteW0d1)70?R5|5t&!gSX<> zFLfkoHw<#$NI9^X#|B%2V=X;UnulVEKhuxnknpZ-%AD8W?w&$tvg|_AFJS!ZAUc;k zgy;4vA#z$9#i>}iXsmRx`lEf|vQ_Dos&(kHpBRz--Nr_A@M1&CXX%kqX#KGR6UA>k0vnD;VNrR=iiKfdI$@3jA8TE z{?Kq@I^?Dgr1*X(oYlqYw@kF%b4{4a$APDYW7w|{yzS6}Uobb4v|fZ=!I}r&qcJ)C z0a%ZVz+GeAdDE;?eDFGpgfKlOZ~Nk3+SXh zrcwMu+#*i|Zv^5zej;hL>-ato#q(@yxybDcBmFCtzW!Y8y=LyaFPKKps1%>F6+e1L z$i+|7fYy|Uq~v0|FD07Hk5TwCzJXBs$uKwoi~l?VRWn(Emf~umn_POnvEqHH#<$Sk zvC60PYWxdJ@=7Rv5h672FC%@WcvGmc;>`9y+J#Yl!)4kdo^k_?daZ==`q@6CN_NFXI?t9uJJm60f>01;uGjgV$)@&FL%?F1193 zpT94bBgKYdu^E&W^XOZPkp>q-T(0AhDIqMOou5<~PIV~*a_a%r5q~mGmvq^AK{{So z#gC=?UpsKcRTt8-!J@qdANJ(}OV-wrbrzh~5-tfEBcL>eZHeO;c`G@;Jes_W3FI(O zq@3eacR*_cN(ZUmM^4LtRMQzyY~#;jlM+Dj$^I7(U?m4bh*@L2yl5WF{8*3Y()rQW zQ-`9_(WeiNxqrNV@@wzNlu!FGzxAk5#Y z5v1+l!qvUn5B|FuaS_zfnTTmOC&B#6!MH1;6A$sf&KBypYG~c)cXwdK-wf<9cPr>N z|L$6J(_fI*){_6xl72$+VJxuc1);P|icd5qq#Tqw1FLi8Qq?3KW-Bfz|Z+OxSO@- zx!RSeuzmitY)17>I-2&ZORrS(;0=$`{pGEmFw(Rt&ndSivpIjQkYd2~8a6^zFaK8G zX9@#SeCkVw@QYA+uAfx-MEQ#3;ehfRCTB!q;rio@{3lX8#JSgbqF4V!s@*#1<i3_1q^NFx|L<6d0+9UpkHU8*Xhj*p>qEkNv z3+0`HE*({RS4)1H<`SX&754huiBTVcEAHr{byONkT4sM?^9J&gP|r9+Q0zG6jPNK}qH(4}f!lCY;_6W?!CY zsP{p=2wk+gRs;P%b`<6BdNcA}OzCFAP9EOz6O73Y;J-@@NneI2UeeIoN#BZXLg5nm zYk9D?Kaj>{s&+iWPm%Y#iv}|n3exIIljD&UeFXUgZk*W*2`f?X)ZnCN1z}VbXP#@&4wY6UU!>VQXDI3Gt!TA!I|j8IkE7a$;HdH~#7$F~ zY0ObA=~S)iGx!?rrFnt+IARW+z2&vTT-6ZtZ9bCe?KlPp+x2CwKX%846I(<1 z*i^V$ctYem8z>BOB`*cu8l!6xYp>I@8}|EdG|GI6r>6nuR$0_Ey&!q3?S`{`X8S=pabrO1m&8VYI<*m}o)OEf6fmmX zj7Nrkgs1gpKoBnD=yTZG)%zA z!T_MY29oE7NNubjUn>@toMF_*qgaOGu$ZmN7m-IfCiWX9A!#Wj-GZcNG=vv;am7g_ zO$Y@`_fbD5Rzn<)XKG*3t}r~pe!ICz#f7oKDG*t%t1v-QtqO~{%j8YpL(}5_)Ht=8;ND2uPD9*Jy{9Okz8KQ=$%|PjZoxBoPc-C{q3PV+V4Ay+ zQQwPE4NxEE0i&29)jvJ|HXf9YP`*!b24M!Lo@HM5DF;QN8{MC<^)UR_sX=`YHMbhB2cIj7#AB+ttTuP|q97x4dh z7%InZC(qedKA*tk@AY=fxAYzS8}$NA&F{hhCo6n7)Kl?;2wK{o`n6$PVJ&HIrO7lZ zr^v4hr9X%ZHL7R!vpIb)rL$KM4oKCnI2XQ7GsWwhE8(y5fP#8#j5HwJ_L#&9KCU9o zHj_QvybPK~Op((AwV2rPl}Iiu1nQBZeNs|rRd zug@^vT6rZ&W5-1kUNW!j$7tHN7LtBrK@(TQkihP&yfJE2Pek<`>s2^2?87idrbEsC zT^V^~pjw6JZoQS}C2d%fFYa?ks2(Tj87Oq~MT#F#9g|^j5EM{iD@^h}l2U?HJqmVq!N|H7P z>WMO{2cfh+XW;;9Hy_p#P{du8C(<~@Ah5OO&Z4EJX))LOAY{gaw|C3oyZsUk!)-vMUU`el&G%gu8x32Wesx3#& zGvK4&%mg*Qi+Wx%7`}=R^-t4%O|CHTH}w^_+K5EzQ^&ks4N3hu?E|UHFUv8oy_OFe zFKUBdr@6?jy(jT~ERxo9smM!!2UIs{=UcP+R!o_S@?)_Y@#2^&ksBb zju+d*%sZa)Lw!$Kz9=2`%q!By-yr+#lIQ2tlXF2IO>I7C={nkHx9NVcH#5b6YILuleF}X4TVL95xd^?Q=D_D%>d!PB z3(fW4u@GZ{0fuK-$hR?Yi1yUF^eYY;9sVeeBulUkuEkdj87MnUc0-FeJ=uMZKWuJj zPG>p&#yNo(wOi&^LSUDBqIv6cV4iLy29=h&=v%f|W0GesWkR=C-lD9qFK<$?TvDx2 zZid488C^u#``!30P=^(*k% zlVE6A>j2VvXb#L8BHR9r5nqC5N~&R%+t9W#9Txj8gss|{kC#4q;sz5F{PD6jwn$%s zt*5!d`yR9x+v(#_Ez3cMcm9W2SFf_K10KTpIn*l^4q49+4%i~c z-q3-{x6?)cww{#p??F@S!8cUAVebD1ga3JdUT5kSwtvrY9FbXs>NWHJ4w3lr1d%kRZb7>Zpm}Bq;mhG=6 z3ye)c{&bP_KAMjQSczu*e(&zG?D-8=YTgMc|DBU{g3hm_Os zEs)M|YIhO}j1xr6YdUXx^A)YaxsTNifNB?fa{55GN6Q%D0w&XUHB=`!!@!1*3^|P{ zS3be5sR^ncaAcA>)s`POPddpxq?=saat-9P>A^p5=|k~I2bJfx^$SGjr%|xvC+)qr zr+X*NjYd+cRv^xgx_7bw@z$k$k5 zAa3pbLal?wwEs3qxu6LgRRihT;`X3uoTPh=5#Hic{SZDYyM~yVJB9bJH-)RQrhG|6 zOFCnh)=7?Eb{@KKY$t=p^$^|k_MwVZK&{U(7uLa^xczXycY!1fe>3~5-Ur9goEuOdkZMO(%i7PVcBJN8Q<;0Q2!1W6Gwpvh zg81j_U}Mx`pxmMN)Q701OPRukF=Z}l{Q#RZ#C3EISj#<|`Jw8EKyT;*Xj1SO6Aq{2 z$VDG50q=#?n~_C)|m~Eu$Xq6I)N$|$1m%Ry8^!86Vu~JIsy;cR-k@c zx(C_bnrEF^it^`iv8OtWu%wb1K&*|DJUMpHqTP^2DwGn zipRk|cQ;bLv3vWDLV19%x)%TR zcsVK!sFno9O>=n5TX_0rFZkK_MqRpRvFv?I_Vz|lj7w~x@p28u*Nvp;_UMl&u45<1hnvZi+XW&r#Z;P%ZCQmK6nol#sV&l>Mo1y(0 zQ>=(yPxF|Cq%nBKtWMxIuz_6R;Uyg((%GUuu550bm!eshWmULx<9Y?kr&&O;6r`u9 zPHSKXzb=yWn$oW_yjQYlVBQ&mLU)Lv0gjUN2oNqnM#)z3@9BEkymc=$o_S9^F?^xo zjp-k(r25(EKF9_QpSWhdwd2dD4~LuatzcZ-bwM1;-ikyt?a~kxzRZ8=1D0hYVDb|~ z^gd9jcu;hh9Ss}mokgV={@rI#65E!}G;@}(`s~CGlTYB%LvwNV{2OrF%MBVd9EUZn zGe{@ifO88zllHDDdra5~O>ehQwMO;69BA(1QRX_)^mY;uFCgKBh#m79{d%|1)H`p; z>z10(vvnWXx@~mNVznQdTJ?4d8XsYQ2CBRX_D-O+e@Ps)9j4Cu1r&FldQu=^Gbc?V zut_Yg?7T@sIVAOle$%een+dx@nsd_ISTQS`&0fA6o@IW8W%--Mw(dHTcnjALZO3*L zbm6vUn{dY!+S|4~hF$Z?(;WIS67L0n(kVrb+8z2S^LA0afScb~cmrJB8(W`X*&iJANPQyXrCSKUjx6z;aNvrD{UO zd}5Dkbf(;7oIQ9E)EN{5l+$p!##cHE`Zt>}Ob>nqP6HYn*tWBfD_eeo9=&>MNn=p% zy#&%ktjprdT8ayKfR(&~`BCQLu?i{Xd9R#I#C6xhg78bbro&dcHDn^BUZ^X{i&WJi zc@t=B&_~Y7I0D;zk2AN<#}yU{;&<4xdMjSOPUnmMX@ET@G>0E_-wW{zJW9>OEwR2V zZe&eCTrb}x7Qnl3!IW;x_by{aDP@8@5Z*B0V zP>s&f8h1u0-I+G(l2E#I%0WjY4T3Ze>`3=fJ9KN!N&oQ=w*83DF5^porA@aZ`5f4$ z=|DL1EAL@JEL`9Ru;}tyM)(e-bBRmWqo3shM*ayN7Br;%ILHV;@LShnpqk>3GWGeH z;-`ZA6!UYt&YC+VVC(t%oIDh9kPc5ce;7AZ1VMP`72?08I!NA2;KxZgHTo9l=1szH z-S%lO%W@2TbWclbis`x;jA96c5rQ-x{MqqE;g`0!UW>0ad3I>Js@4i$V)MomJWQ%cyr+zqV3))Oe7408|6ko8UcdayU9VNBPr%JMW>gamCjLlEQr6^l&iXYlCC%G zcpNP#2XP>MUotq=1S*1QpItM$|9K9b6-Ju)MNlbIT=Fh%4zx)Bg)_dI^FVsOmR(v$ zdZRNC#!2$zNO>flICkUv!ykgri6-!F&!(!}P}q(}9@IC8Cw)84!NapIrf#6dmjy8ke5`4sK7q3eJ&I+%3t%XS6K(~!1P zJq=!Vei7Ea{6#$4aF~%+hE`|pf#QmX z5=YC`8}$`-s(uFaIlT!>YdLVuwOL5~uY4CQ+l^KJd%=#0f^bJr{Q>n<1ob{Z;U48V zBVASH?P&}^Iv!ud)fS!8?qg3^Lpsmc8QK*QPIe4r@|}(#{mS%KCIM~DET~_^ygV-f z;SN%Lu-2jZxXEolnmXo-lQG|vJ`kk+=vj~oI_n`pkY+>T7np8#ALug_Pmm66FUU7B z;&hyFE{`bhOLDF$7a-T*bW2Ahmz)%V)pWuvasU3EA>*4`ZDt44_C!$vfC$1 zpuP<6U10-vbAnKL3gT8qnopy&HT4;gVglhKTFcCFNup-c03dxx8gnd8{t=Hm6YP|J z)+ipMH502ht^>jt`kez)o{BzS^%p_KTKQY$e^f7lG(OC0){$`E5poQI;EvlGB<_Q& ztG2<+hm9cj<$gxq7YJ98unqEJB9X8}^>J9MGXXI0^91sS&#C7WPTV#gM>lGUM?ssE>?`Tslo>D*R2J4ThU|0Rn_AmjVF|UV9PhJrhZ*jY>?KF zEjSxP{jSqk>)ApV;#^wqYp8lkst;BXyo#xO|JHQ~QlEnpzCx0}Aw*X*1l9M6zx7h6 zJSLnFS64N~Lsn~<>Vv$C`=mTNP>)b?pVH`gq~(7I< zN6o-^^wStgbBZR9hamOUU{wDx>isM!^=Q#3j} zTM+liFUea4`76pnH}KhY3KuQji~7!8Ie9Utc6}A~#SX%c41ZYZ(48yJaO-yp8>ec8 z>LGoJY=HC{F>P=?!pBND?%xUOzI}p|+WPRGA&r8rl=H@`C` z56cF7bL&U7r1y;j%Q5dX~$T|akS^FOvcHKB4mf3WUxiWzR+lKPa5MS zoE4IcCu`M~wySHZ*YZpG>+#0mr()jao%ph!E(8wf<1llpBaa()7=nLzViUbq z>^%C)fEU)B)>@V?zlSYP#L52QyP&K=6q@%pkowhAS$vO=EM@djocJP2t}fQ%)qY37 z<6t-N>T?U7_DL;Q`f4}pl-HE_^ zughrMv7)`_3t`yb3UF5ksoCKtMcG}fX80CpJ!Go^bg#;ie%gTv^>DxcMeKCx5R7~L zAFNoD$_zjKMqyzion!xif5=rd39rP|P&>YM=YDZx(;<0zPd21Z9H8Cr`T-}uOGCfL zIy}C|Y;NMwMTW)QVd^!db6UxST7CJ_Q`ziWbVq4So8J6e_YkJtsU!FM_myAT&l5ie z@1}FGdxFilSUJ<`sWw?S%kpL?nT_pv(WY^Mc2BAq1|3a+UR@#}{Ko+d-_npbPk08M z9QyH+eir=2wV5oOp7nj-EFE?n3d3f8Hhgf4PVj#0RDAj0Qua)@CC^)EEBl^o2Y*j* z#33msVb0lFxI5oluA^rK-~BoUOL!|ZqWip4L(}ldX%3aXb!1uYOY}LI0X8pwfmv@K zesjb!dHmQ_C{Fu%yheK+Oe)@hYHfH8`o67PU-k(Z$0JVdkb7!%qP+}G;0yc4Qv36; zw&>n05M{Jy`ipwJdcFEm#kw)uAx9l`#MnV*{KnUdEcWMT42v6#qkhcL`a@lL@AW8| z*27~202&wVn`6NX8o0_+`yb=kMQ>qygC)?-;V9ltDhJKb2HFN)=$x?!i#4vd zYx1(M+48e)T~Kv6pw0o9of5}ci-%wtuP1U%Cu5uWBl#8kQ(Bcr0c|HBz21z0&t{a`PGmtMbPQ;EL`mDjc16rypnOLzOs0Q%APlkMM&=`5~ z^eZuRd;+9e)rFN;CUC+dpzA={{F`y^*Nm-qXH!sX+A!1K#=Z*P9=~ z+j~9aqT)hmpBRZ5BRAFW2c6YjM4r6r#AZ>C}i5mv{+C}uU>>rq42 zs`TbX_dbY2EpKAQ!;!GNXOHE=zo#`MVHJ8i?3CA{=W0)_Z6v!7bVr?ap#UjwU_;h7Al$-a zkptj)LmnJfi>sR5YA}|MITa{R+7E_s<8WSi*cx)K)D!JSox*-+d$Yk4=E;~Z!!WvS zlqSb6_hqWpdN_bfwSjkbL5szQ zK)ovs!1>|F z@a@-T^2+%=oUX%(v(#GSy6zdGw&gHMc`nmC4dFZd1Cid(8^+T;ei^lhN0$3ksP$Te_rS5s&(uPxl2AYU~3oVGJM0RE<#{a(lxz3PbR&t_#$$oy&F@?F7nUx<`FE zqp|Rrr~2ZIoyB>IlQ%@QLeGu^7~u<;o;j`f3l8l%hb{pXNO;V{=>9Ehj~4vM`5U5} zLuYu|&XJov?~L_kegM)jV6q|vNE5Iw18<9vD2}T>H)52tkhwKeVKo?LGR5D#WPcZI zS^gNG)?R^x`|`1SMR;4Hd$1aTCgc*U+d5ohPdvw0!!Jo!UQEp5LmY zYEdW+wD5CloTxt%+hp&%O1}3>x-XQ@vg^v|X~mm9z*H&XJ_2V!?0zcD#1=geGwQYC31H zkCr%vhqSfEoeOW^=F$DctQNKT`ixR+TvA;=Iu@zofR}Z>HD5nm)+&9MWZ{eTXT%Y< z|3sIBJofQ)EBK-53v*r$!injpH41O$dpqE^zK-&?dv9r4-4KiDE7-<=Zm>fw2g2+L z7vR|>M4kS*us5S4pX{BFy}HzqgvWw3B={PaA=MSO8sGacA#t_~R-e&Y zHp@Q26sM?lB@CBKt*)V}&mLW$!_+|~NNa?xJ6I_E$E^J0Rrx@92WS0vi11s@H^L$hL(mFN_##UPfL>4dmZl|eV*Q4e)oOOxvuZ}-S;_nsQz~W)s^;` ze-bXgc#UxVR1Rrxpm~$IFU;s%@=(Z_(^6py?Ws~zZoIW7dqPn&p`X1Ewz!u7;h^|1 z_7<&U4Nf)3`|-cz4UUSx^q%(fJq7G~*>>y^=M8h)Hs^7AHQ}L;izIAiN~7PPcf59H z(05zgocSERBRDrd3LiO-RW%N}_Z|wuAoT1dXw7Cr%Un}Au;3{WZz)`WlQVpd&9q_|6So(LW2(nWz6Uui5AKF(iz2Om&|#{1OJ6usi_V~>aV413%YN( zxo(^IHOa49pBmmgjI=gLo&(%+4#KO;MZ`5T!JFO@Adf~G$puJ%E3Lpuzsr;Ly3#$o z5TsS;nYk`V_$n0cs~UaR=mSLC{OWdjnJunp8!OU}UDZ5WJ_^^*xuK=G(GW+-;mtDz z)q|Yis$~Ja))W5z5FuJec>6I@kcSb}7h>rIE4Wj#83{Kz=~1^%y@-d8Efi{Ojat{> zA3`(WZdEO%lgM`@3x_2YIQEDcC+`Y2H!YPXkZL|0zSqa&FYIJZ-$GV2(M<&C^?`wH zq8Q}_q}zlD?G}vWrTN_^_t^bQh7dApn}#$BBi`avx2*8a2)AWn^|arQr>NSYb7v)+ z;hG8!>^ixv@Ej#cH(=?f@677z7*0G5QQd}#nM>+$#rgH8jlkKdpBVK&a~Rn!XKGv= zIOflT6>SZG_y}IVz06cLKw1g^i%$@w&pG)KY-gt{mtPzxmpxB|0V|_1s@pyMUJ*cl z*JnR|EkoslsZZhkw?w2m)hdrbng_-Y-3-?P*MNzO9wu32A$eN4&`?)|Ii%287I}g^ ziyQe?AiNSMGa3V7HER@QElWQwV!3Ia;@w%x8*}^0X-HTJ zu4acZW6oYd>xxb99jo?il(9&{PifmQ11V=<&oEW|>@f+H2PaNO%DPZ#k{}0qhbuK4 zn|*JIR2!Oww_KH%VMVR}(^A%jrw3c(Any-&d09Hp`T==c3@#2-Jfu~*OzSsGSf^$? z;Aea^7^JVjn7{cN@_Hho3u*&W+6$*?=A86~%3TEUI?`MV!emt1hjI%+bt{hPS3#@2 z8rWIyIntPb^bU}QhIt1L0bgmuhU|DNsE=TK>XB|QCZ=Gm(gj+Dkym!d1JyVqzbKzr zw&i8lH=)8g@(VJr<43LXEaZ~}WqvHagP)-M3{-~pc_O`QD(|`d?VAKs@^>kXrnnxQ z2LX`ZpxnS)UUcgNVFfWrUJlGt#?%co?w{<^&M>8$Flf3pG`;62 zsW%023{(D2>41;8jyS?dJl?Vst`F#7Nngg~ru8W}yyDCoyAF-ia z^0yR8Hwx12Y|XG!K4IAVgf-(M2tMeGrg<8T-rRxZ*76M&Y zc|w&HfbyIp11$N^czukbvjvEo1ZAa+YDkb4XDZ7u^3~$QkJfIg&Z)QLuHa_)!O{qH zKf6eUjmi^|wr2T1Lz$ZMM|aL*X1n#+Z^>0yy!2)@_OA~21&ioK+9V5Cg_(@9F!J}2 zIQQSXT9rdkUIHp3B5sD1v~Q4c`bBk4Lp}`%1I5=vZ^SL9t$6dJW3^@_4JLU&Zp-kp1Do>nU-iY)|U0K?( zH}&^8p!|Te`*^VEd0KgKq|A;tx7x}A*Onpqcceam?iXtC(34>~-Ma3^))yRrcoCHk zQ8{i}txQI~LffeIO0oUYZ;jH~ot|AI|F;(jqu@bEFuuKG4`FL1ke*Yz0y4{b1Lcic z%HsHhr9bKC%qR=F0Cr}*S&Y9wQU)dHdiY9zFJ(`ESmyP&K%A>l`3m_k$|INJO`ifh zGCB$=>)?c~ytw5?_|;@HQ=F>u%9%+cIAN>UeaeQ@AB8U>^ZIlkPbn$CslMiT^ZrOR zEh!^M%9#cE;cAU5mgNHFccA>(@5p{Y9+7ggpR9A6c)lvxRAThVIpPVtS8830$2>s^Y zf+^BTEFAw?@ySJ;c)nxim++<*%H+GHYcT9L}OWi@QBQ-QaStNd|d#eNW!pa;bdK zQIG%ZaGlM3{7#&CI)D$Svzqje8?x9w-^H6@8`U`A%+sqm+SZ1Yxx8nO$9sqe9V4as zd<-AQlM626@+NO#OBfdw_FKiybuA#S1?}bSehgmkix(&6^^^m`P9gm)EQs$XJ&)#L zj@SZ|dl%DOZN6v4#HHaUHcX@A2ROy zULObdZvrRRbdd+n&4cW+-MAeOX=vT#lx|mH&e|~eH`Gwh4Ic)3ozLT~(b-^@u~Dq( zu?kIp#;ZD_{gYm>&rOeOYDeg4(=(F6@>LB!!?G1GAJv?136n@;=XtuGfN7;-Ybq1M zYBEOkIwJ;K&*Sz_HPYzC0SFs;AKO{o5$2y7U|gSVGAUOVmwJ6;s(=38>dGC5FUB48 zoJ(n|k#zPTm&+#y@&)l1;p9|jem>G!z7CxY>+^LWHILy>gPZuSz=tn7;mXzTta9zb zKP2YM;9G6MXJ4=^YSCOy%6beNw8O;bo<}uvrt8AnkZ{bm{fh@oZDh-`Pg7B?W~tH|Fjm9cGZ$sM@|IGbK&AKil^&g& zFpQrb*N!hgNqZ>wz9~AKILdy_^W;Cw$6`a{Q&^He9aSA1tg{}=+C_u+y2TLwatJ@v ze4_j?;tI_}3$`ZqtJr$UjVnA@eAQNDby_Ch6%?Z8msYg6m88w6^NFqbV}&ipSyIn! zfmyBp#e$DfjQW~UFGKr={Y1a*H)*f;!L0k%N?2WYKTPu0m3y6gbN9itUuRH=wDapF zU%%f*zdM!FIK-g-f3SGPOy0Uv93S%j4M?k+qORT$>ATN>aI!8CE^wMZMt{e0`-Sq! z+cQ|%TOW@bT*9)<2@u!Ef_t>;&;7fa@JHVU@(+tWWSuS#q3O1xIH%$vu8rL!vc&@K zeYXN>U8`%SWbAtE`DY_^nfVTO)NIf1Zf(a)EKKoKxdU(GYalDUO{GbR4W}B1Rjwz1 z?h6lZSta9IG~(CniZEf{Cg|+cSekWxr=4UU1uJ^g=7edqcfkub@mwq?JcC6u1UlI^ zl(T#{NxClk*gjXQ>XXI>FRfd$u(Nj{Yr#L%Ph)V}-{bO(yax0Ky`Qw?mfmCKoV7Z% z-&GV3E>8u^-I1D#-){+X4&ml+ePq_fWK_7!KOfdOCM<+YCl}$%eLqFS=F3!b6R~8h zA-{dT3s28zftNS#5aXJ*0PWi8q;+pX2g*@4O}hrCjyP&xhkjy}y`ymm-vBqKfzMrY z_AcEQgcY47UHFlmYpjL%@4K{n|Le!!PNF^JPEkJ9WI8l^XfKFkP+>QH9#{3{*KP## z`ud8g^};)~U|7+0Xjsq<2(#st=ov_#qp%c-_jnD~nP;tRL;qhYW-a(Z_+Lx@nF@0H znhf4zTqs&KAHuC~dZ-!~Ti<5EE`JNiw_U+(E@)wojW5>R`Iq=Rl%I3&hI2fkv==^f zM9hsyh{1vAF;h=U|aR?I5sv6?HT>@a%j)vH~P+!fxP@MM9 z#E!h*{2|&I7Z;sSo9Iaou#k2Qzx1>$3E7u#9HY1VMyQxkrDwr|enZx`^#p_NE{gZ59gny{o{%+Bu| z8z0aF;;+nM=O@zMRZVnwhWC4M;i@D5I1n&)ZZdNmUIkv4I*ZOuyP()@uG`j2Ik=*@ zzD!BvXgH!X{^&MZP(Ptd^jL5kbOEeZ7ib92VQ$(`s=LXO=G4u{dJ_;=%YO&F0^$W0 zxV#^0JSGpXxrA$I&DqZHzeKN~I0#vp$V`7)%g;SKk>>c1wbjvK%Zfih%(9NMW8)0` zd+RiO{nUmNmulIRQ%Lj5KL0#{q_x-^!>#ZpcZPJHJYL;5Ht*XOTP-Nl5{|GY9bDjR z*Oqd1_HsyBd`3KGS2RNnqu3VKQznJ3BCRo6CU&WZw5D3B4Ne?`x(oNfp6))xvmDEJ z+sHn9AF@AF5ecK&`XOU*N6iD8%9~FR={?DRpRLBayN2`G(_doI*KZnW6)rE+Sv#tx zNMmV;Z`jNov%r1uQbzwjis@^hSD%Kms83VSkC*}lOC~Urrd4P#;JS9n89IYC(}R95 zQ<7HDP`%Okk1)DtWWBPG<`}7N<;)!)#itTyuG=P*)$g7szM8hi=C@Yh#5${3yNbI= znghRN901}84Pm*+?R*#49ySuh7clWvj0~hR;ZpO2;sPLzsMeBab^5KP`jh_luc5SG z5nQR-#0dMPMOQjYq;50jyN~w0o79hZGN)Rb#hkYz4r+=s>(g`2lV5iu9j$$I`v(5A zKSLX~q!@18zeX9<6f{3KnKhW?4qspW0Mb}Wi;JD#mkOF^p&LA(^zt5Yc=Z@~qc@E` zxzv;YyQ_gDoz7qJJ&gKCyZ@)1XtmOu_}CEaI`_ffEqf4F9%JUF^|8dwm?;iO*cHeE zZr*h5A9s%Ranq4)cIflrKME8tV9=X9C@_hHt{;m~@uix>Tdo>m3%5{>#b&v?^r$gF zagTP~&z6j89WT$cRvd}z_P&&tPJY+^7*hgoe~gi&okZDyNH8w>4MX)t5ymeO#JO6J zM+NNcJx=xaSk(a3xv~;hnVrNDQ5WE0+-;#b?GDsi+D=Bx1?eB8URK-!s(uQttzz{y zcoI$+alPZ8K_?p}s%bS9zG$si~SvcXHWnqy0i6(=rK{e-;^ zmjU&Q?DCxQ+yrw;I842jEs{cOqtX)87aIM$4U}es7yf_M=Lphc;?t?FV(PXm=)K}G z`~CMUCP!VvQHxzUjSI-@5a(P2)1H=~yRbR@v)_)tSl1p2bCEQ;s$t&H*afG~%Ykoo zwllq&JE3LDd<|)P@ZI$o>GMS8@6`~z@(*@?J{Oec3_fB@>*6KWkF5>oDurgD?pgL` zn6bFM=OpPlQ@L|pvY_?j3Tw^Jy~ufer4gh0ATHaA#IZ1assq0}_#pMoD|UAMSxgT| zgM?wzL6AqM`9<=Gq^Wd}G%6e7-;k{--36&H+&FnD!Cmcu?va; zj5l3?)`vWCwoYB78pY9hnMk+`E=|JVZ@DvO`^F>96y`p^B{Y*zLv)v{_ zRyN^3R)?eF9)W8!{5Vt|6$W%?` zmq$Q-4!6ue8iaU3^OosinXQFvKDBkV9_>B$DTJgn<`uuE!lA7hthjx^Y7DMhI-s`&S-cXmM>_YfOI{Qe~{##VPn4=Y=&iu zQ23y*ept~U7!nbuNy?3Y153X%v9>O+Z_|su|2_{!`nKec97%s#aq=%pkD*zHF&`Y< zSdwOvRm+DX@ei5Jdt#*GnyrnkRKHlWE(&Uk7 z)hR^jBmDiax9t4Sc35%XIhqA*70OeR=AnCCiOQEPeNJb<1?iUtIX%F6m zIq+wVhUF~SA#xWLvg1DIF>O{C#b0RLejiQ_yeG(0qw@6Xo(?^DDY`u0h(?JWH2$y3 z#F$xj==(mJubdnOL!Uj+(tT>b(|#$dKD($q0Pb$xuX-E_518uVXpFy7DksuBtJ@FLO ze5e{N$$u+)w3vfCgHzbFoKtMYfl184^Aud)>?6r%E1tl-wjP|;kC6ru#POVZLUVgr zSIQ5%Dg7nLPvCsN0Z_iQDW}g>UYNZ%re}Xmx4=&RkJ3ScdIfeo$w0zSNjxp@4|$;> zKZzzy7h~_(SRg&8=~-&3@Q%^kYt`D5w`G)s&inaw<>iwW(s^S{ zwQf`TRB7h1C8^NotDeGpM(e~jH#)C1ZED5o+Jbxu5;h_AxjZ_ikbLWUAbrFBi9Ccs z24}>8@mY-K3x;qb^b5Ho4nNRkgulFOfF6*RQaYWHwqm4jl|Mq#EQ0z%tBW>#>6u-E zGz6b5>I>55K>UsO4*wtw3DeZs7Y+vPNZ)n83~LM50j)8&mq;GDJ9xL*okN-mY%XYk zyf=Ps+8O3A>d(o?Nb>jMVnQ3(dAu1Xu9386In=A@<39_Wd-x+|M2zwjK{<-bVUTbi z$7fjOxbI0t(f}~hSkiulg^+%|8UEUKNeo(Oi?#jr1>qhNw!_&Oz4^mwccH@GoVU9( zT-7P_URR~*c&`J?uHTgUIud%c_>8Z3b9v;@eTufx-z=98ZQF`1w|cat~WAXPtG>-Dx1sva31Kp7#`OHIerYHvHoY78EJ5P_~s zZlcoL!)<7P*DvG1Z@@4xUm66nAZL@ zc^X%mV}Ec>y3UBx1?6No^1oaS#5ke$&TA}K@&uF?_1|3|l*grBgr3>v{N?vnNInUu zR|vC5Q5I4I)i@pLyIIm5jOJ29eWLhNdBke|GdGO`$_|zG!?c0UsPaqO@Im;hOF0{E zGoU)xB=1Uivc$O1dDiQL?h+1IpsKSmvohS2W;X~a0LogbH3nT5U%$5i z@>;Z?*+EvLQKr(uKzXcCwLPa|EY#|6uDpe)=64!PnI7%wKpLLrOb|yh$~4#vO$%Ap zPG8MCWi)h_NAwJck60kCy&T1?R(n-@G#Vd$&v2P~Y&VkUWmOk^@!69uZ0+FPL|khHdyV@-+N=|J>edm;K+UDfTobNr$Vr~$E!{Tfsl1~)(_f7 zUyc356V*NKcy$jWzk&s4$6$vOognZ6?K`peB~l(Jh7Jn@lYk1W9T6#1MxEBat2W&3 zhT>kuuUgU!oG=o*oh=YlGvsrcK>k1N@Zr%yTwB;sQa%C7chWj4F9s9mOyZQKVsp!1 zDziiCb?LrrAMdw)Fn_T938Va^+BD79>CUR z>w$Vl=|H4962zHmZUpIj^0@5<`A}$n>o0UQ83vS7=P2#$zqBDIP6D6rrpjkNAqm0;{B|y0OF9$Z8xPVO@O>uHt{8FI@%veTZ_d;ZTJPF4x}me!oHG2F!OOR^;;}u;^9b| zuR1?zur?J2lmq?||M0eRH$&jxGIp#-1`}R$G+P~3@^e@8*}mI$eCQ;EZ9Q&bEC06g zMb&2*lUxIqx5|LIKD+3Qi#KTT)Q$EScIOxGj)vn$y5hYw+EZ}j0d$-60{#}!`vvpY zIri$$iJ@Y%Qw zVG;cYmU$SnSFvH}?^1yDcdpm0ja;4gl1=n+5jN}pgQySvdAVDAInaF){(ABhYoCw8 z>A{=$;R-#O7;sQKBYqHkx@#p1Y;17&`~$Mrq~UVjiXvR(o~(&JWdt@b5Kfgg;G>M^ zU}(r#R`%~A(h<7++z}go%~fBPyB!c}&h7QblP;XXkA0{Oe6B8+1sYq@Gbd&wU+UDS92DwdBL^V2lK)5 zoX#2H4vpmXz_aioZVF~p{wru)c!V;PH5J(!sv*3PwwvYW)#ay3*K#NQA^iBPS74q$ z7%zrp!;t`EFtFXq=>BQXni#C>e;qvU`|^hA>tucUPQO`lsitn`EQlU641@0{$x@vi zNWB83Rm(M(#tq^9^TRNqA_;wdAHsW;?c~Ld7Thf372FSP!NaUO$Z_x5fYXi(u+!BK z_kS>x8aq0xQvVi&T5Ir^n+?@`3hlxEo6$N;wN}d)UW2C5bMTsLG=5qMlf4aL92fpi$jV#uI)4M=)&g>}ccHE*`pN`2d zTg%hq-U)*xuCjLINqqOD3TTa4yXXMyI%2(~-^b;x>|})JzgV?ZV34&R3~y0eM)>IP z6R|h3a9kg`yl@qCaykVix=X3f^m)ajx-iQ4Gp@Qak+qMk!H?;Tl6TB2m~-Y4pwG-P zX^_MN-35fS>I1YMY(P0Z>Kf9RdZP#*Y_2B1=odvR_YB_#l0g*P@r*Mor1FQJj9C1s%GC*8Z7jT6UU!OzMkU zD(taO-#XaCX&3(^e&F)L+Vq{$zv6k*`&hPh82mieMlSQ%2y_iO>yQ?2HJt~9Q*3U< z1pdsRj-WMX4bvOq;)>h2DQ^TA@2F%aUd93jmTFhEY9cqdJ|hirSJ0et>R&nUzA<51 zPkEzbUD+-AD#EBhcExQv&kDWAGG1CjM1U`EAGwmNHH&et&GjwPanHg=oaR+)3gj1Qpi4#`y zwe+0h>(BE=>#_tHykau!pz~3aW6PlNk|v=3OZ^FI-RW~7+4Yhn4&bW(rWAIB12%=2 z>GcPwA9+&fFR(Qn19b1Q=~@G{ZeWb69|$L5ukUgEujn^(q6 z{vKH{(ET~CZ?yr6>i=Nl58Y*Vj)cnr$<`uig*ncT&%=^q_xXo&hd}*r#V=ll?bwUg z&86O4Iz#u$MB;;0IHt~gez{;A=rv1&9ShS$1E*>99r_7SyjM~5i9K>0T8-yp$xF;B zT>ymrK)1@tT(jj@X5wDT!D)@~rOq0bberD&&jIap59Tg@=$ZGv*ZFh+hYS;d;1nla=1ZQYQcYA zSO{sJCpFzXerQRzVBbrx*y#imMv?8{QS3TF{0i@@=F`feQ%Dv zruTi03D(zKct#(C9?={M&OOZuk8x92z=-t>IMJ1t?6)@er{F**JLn^JfLhwCST!b~)EllyLGfLE(fIQTaVmc?g6pHcUT zZ)-~8C&G-%;_1!KtY%w#x9`^@a# z{93E9xT=?xns=;O=7aC|_hQ|HheD~J2E9;EQcvM<>zQ1w_r1#fK-?sserzIVhm>K` z?JRLE)|Tqv4o#ItTFbZ~$Bz&$2x)bJ5l=vn^65r#c@m1-O$N2zzxStC?YMy{0RDr@-UNc%=1`Mv?oW zW?3P8|2$f#d32%ozixC~4fT)HJK4JT1l6J_n3W69L$^S`^&Rla<~G=B$p)5~b)Avs zP<5zy77oms3#4)I{y0~>*=sb=b@}vMXCPh?g!w4L`ihSEwz$!^0GHm6RP_wAFSX`M zGZp0&u_x_@i~9KwwZ!`(U{Xue%bx~Y5@rkHXeK+hLh3udz0OW}+A9w}*1wHbyCdPW z!4(+1zB!|N=Q%pp&`d9x9SxlXq+KBJ;5npmlLU=hkj4S(8M$Gj z4H7nhKF**z>MH+nX$^y)pVPXp*D~8lzp-X-I&1KwE*RXnuRZ4X6HK?6$z>mg0bv}3 zOfu&*Ch@5qB4HqI3Frp(GUoEP$IcK(C8AfWN^!yKC3AZBTBx-l9{}fcx+CE}o;#AD zsoh!CR0p0>IaTU=KZV^#3h}*Pb95Rt4{f~tis!`H8|~eElk%5gkMkmQT52!Nk@V711KE(wD>V2z9X$W5NEzy1BAt5 zV(@O5Rxtxf(<<)bx826^_ICN`up|O@#x6v{KH~6!s6M}b{#d0|U}IW{(jaiv&kwxk zccs0Pa|QVg+*7B%lzA4aw(#uc6VN830LJZzVC|gFk!Ez_YHm-4=wM37E;RP1GbT3n zR^u0)r>!RZ^#jsn3R9E@gwA+SkWS$=C&Fu4Ce8P5ApZf~Gg`}z8SZS~iaNZH`$so3 zy%t*1=0Ni%LN8hJvu=w-W?ByW(XJDe^~wbDTs*e)hSKC9T$^*ME7<|M$l|+sNWO>8 zXxt~h=ney#f424Z1#}I5s%iqupL{^_7}Z=xx}3LOKT$(-!^poO-5Wa0pN@|M_rujA zhH_hceY_uRoXc&AYauS5t>JbA1{s&%|FvOoc`k4Y6(hDVFBxOg=5yjq(nb z6%YxuCgS#MdN8b#LYD$_|HWp{4_CScy8rHn>OO`}0`f=)fwU+2tn2K1Qa9$;>K!z0F-d7C z%2cKc${2vOA@xOL+!S(3*nO!FPNTj8)d&#Qu!pl+^U3A({A1n#rT1CGRf9+v_O1&a%yT4-nJ$P&#FE^2By2{dk(IYr`NoLnE@~X~Ozj$>uqq-YMUa0Z z|Ik_5Y^%*Me@>uGM8#BBOigIqb=Y`m@%iENFFr3Lv!yeCd^MjrMGOo9k>RY^1R!+s^8p{ zuAucpT2IQrE&=5`Vq@9|4fz)Nxc&?n+v>4UYujPHAN6>M;!&k-AU4K9^&V5&{GPLa9J@a*t2*l=pAQEXu0i+b58*`YV9~yKuiFjNX871k!r=r@sd|9)3TfKM z^p0!3OI%qYh~mAjEXmABf&X{aW&-quU-aiz7YSL(|kcG@$Zay1i9H4bMC zP^0p_+d_5+^egTiTn+c#gVGwYMEU zzkSXf8$*%W@hVJNR-g2p89uq2DM;(1cZ)58v@1K3m8rBAs;p|phmS}e8EG8k-Cnbx zQEB2ImsF(Olxb~F3;GPLhtF7m%od!qj9h)lv|5w;KTj8w0n1EZOF?}}xm6$HxLnNu zj|nih;t90e@mQ?^t{Zb1$+vUcpR4iQtO6Y4(O2nxikkHOIn zV>s!DYW#{wX~k4;5pGtm&s^gHI4OFy(kB{&C0XLQO>YhLv2@LE3=`(tRAZEn&rNKi zw*~o)HF(+JsQCObf^v#w9 zv%;UL#bQWwE7vuirOK<4hNZ0C?XAk}fwF0w8siVj3msTk4-b{bGsT^?BNCxOK(yxe zvw6H{{udbZWGvim{D)26_YmV;rttfnj%!l8euQP%l7FTgxV!x-xqW{S)Gl@5ds7GU zy}trPhi<=d+mtP6U(p`Erj@{v+|z7H_zpPaumxvy4dI7p24S}jw2$BnKbiBi3}U}- zl)s{~!8CCdKK@GU{dFxm=jciEPIjC{rZcCrgRtM~BIaGb15$J9@MdwJwEo|n<+?9* zAhiEFEOOLp9u*n#on}a9#>~U%sRQ_enN#E@iyM&lxLDk&IEodigJI_E@2uVa7f{>D z8~z5_;ayV~d2!Zf@TX_b)b$-qkAm5g2Y5bvrI@NUlzjtYH2>t!7j!LnW4~672~5d@ zewZ-=je`5(SL+I)>s$<}c&)N0;}!*QF69E;sp!P(RBG|C=T4;g0PXMp+#gRCol~@;MKPtd1?@KfXW>U>@xz-G`O;2ijy$LLfq%_BM!u&4Ps{9Q%o+pU={uOy$q zFk!{r1~dgHt3bIETk)`%o6vm7IO#goK{WZI$Ne4+;It;PUHDd9cP3YMrt^Kjtc(S{ z?ZvYR;E}H1MH};;khF4@*q_&*PZ)e1CWSTRw^xtBM8|A66%fY?zh|?P zyPKg7y^q|oVVK7I*F;Y1fR5S*aBp%X`d@fkU0-xfx=(W%QdEzh{O*dO=^vqR@DT9p zV2$RTGGO+-nsV^8uJlUtGZ2Yud5`oExx8}^p!(Aom&D`#wSO@9>|nmcZY5tHwjRHi zH|5W&?n;r^fUPmIl;@4CWUuGn;h>?R{9{?F`7dlUR6f3sJt{$yRI(fGR^Nl#_O~f zeLn*NOA zoZ+uWK5} zNjdJT&o%i``!(viysudf>>cPV3u&L4G9-SJ*8xcR4h# zucT|D#lTK{!+{@QnG*p`d;^*I)`s6rvXo=&>&S*-^?1JT47A_(0H)A6T(|Gr$wLli z{Eqi!>{4zC@4x5Z#DLZGJ$xct>QM{+$uAeJ?DctWc>?{h;d0ew{6pWlUhltA{ugE{ z$NCJXvtQRS*StPr#KW^}Q^P38_k9nG>{sLX2S+7+ChcqU2Q7Y0fMsS(KA!y(Y8tNQ z|5noehK&N)_XOG(+_;&f-@!po9unp_)Q z+EF+uOwtNr*^E1a{tXPm&3NF-C@?cg(_Z`%ivw2b$%)TqsWlZ9sW-86r-Raza!$of z`VDKy%}#eT#!n~1$-#v}UGqyxOL>#t?fxth`Br&IhE3Mt>w?T|+dIoEQyqWfGowDJ?z3a*pD|?UTy8R`8=my2&FJ@7{C!)jY3W;y z0YT?VFg?mevt>^^puUD4fmhW*)tU#By%1`HAZK? z1t#I-Kl7z=i8VI7-%75ipbRXmfz;1+RG21CVQaBtcM{NNNzxPe(qS8$KJY}&mOcB0 zgJ~!6>T@^{H-W0Rw4Y-H@egh;)v#5@d*IKRbGZG%as0hkLb*u?B(BE8KRkKHneNzi z_BYYqEE^&c*I=Y}1FYTarhPK?1m1r3l}#8t2(>F?uto<<89XhE6|JnxzZLI<2PsJ2 zsQKOZ)kG3ssD6TBRW~84;t0cYWB81&AF%zi5S(7V6D{Zb7MC98191~09M+DXVgf5F z955)+0Bz}A?9sWISbBCSpZBnrpuR<)oLZc8k7oE80G+C9)mTE;LcgdJ*!)Ky5mjss z`nkvO#*}UFBl;{$D{3HE`!rB>3KRBEft98wNRQpcQOO;FJ?z1aOpoB%j{8wtvIlzq z{YCd}4Tn9gv@>#|<)1UXaM+h+X76XqmHO`g|b+ihD zA9npL_BVQlAI!QV=|QcpS2~b}WKpXwvNuk{`KDYKF?04JnIm>#CJAt zzojCr7q5}=o5eQrN7ANpa>YTWYWhq8NSTm@FB>kXx zRx}OT%P&VBW-o^Xh}U$kz-dp$QO)Rc&!3-1 zG}DAV4MNg0p!lR)#$Pd{C>E?6J;d{QSHLHlq2ia?_E9iD(hdh3H^ooqUxSzRY_RFD z14z?C+b$OgOaB9K*g#k^4>$a{3_qtj!K$-`NScPX@HE5t)%O+mO1dU#xCWAVS@jSq zJZ(~SozeKjFPAdaAA;6J^*0bFih(BW7;yp`JaCXMLq7ra5ObY+2+3c;2eUIw^%M1u z`b>;2k4MFo?zvs~EF%lKGWe=EoP1p20Q{~R21nPtSH291->3%alRmCNd!`RiIvt)q zy6JWysw?>hU6^#A_VoUK0*Hq|>9W;D_33_XQSZljNT+>aXbpk-90*h4rdDHpb^KUTBcPh{tuy=NmxZO^No3A=ojiHJk+;dMb(p^m9f9;4S ztlr?OjPwe7P@V|?b?(E{hRz4#8}@o;8S_r4!KXN$2AV^B_uwSVb1r4HE}XCfJ|*qI zyqvnMG@%x6?qDcOw3}7E;VrXVMqY*bP#66v3$VQ~6$n?s@1Bk%? z!s(q#OA5kTB<@A!PxcNx3BguVWk||dY#TdX6hyQ}n+}cS%gBz(Hw*G)K-!ge^=hYd zJGfPO)B4qiuix51LgQZ=;yXr|jhigI(Id%3D&O~L=t8AU@kAFdw=hRnK|hD&p9T3r zL7X9eX4m8ijh8~?Ia<5Obme`KG?&u(e2`z2Al)ooU7oW=5y>>a3xG5$Ty`+$eZFTR z-79NN@2-@YUsjqKh_9&Dmc!0~dhBZ8ZuM`PrQcm-9iKaJpz1bka4=r*7w?#)rYmA4P54bQ)<0n+$D-a|GvGLowJVs+c-G}Z>Kd=#x^|H&Me1yH<-2w(qKf?|_ynrDcQ*p`Q8)}^?fB1^D zPV#6(3hAg9sB~A(~FQ6G&&HD$YK0_O|LjM)?N$;b`8zW6J8=Fi0K@!1|pn{v98Z`Tlpprr3D$rrd0zNxGVLw$;|%IgShi=9-Rb-kUk9S5Y{W?tV~ z$TfK$;`x_)oH&DeJBqwnAyb@q<;MV2dBci*CABrMMHvK4K=nr>Y*2N`v?4AE!BQv=esGb0)Ob zTEY%VdK#1mnE%5AsgKa1aiu1%LjtQmV=%OeNCNUTtXbrKruy*xU|($cVg*F|EfsH^ zinPNm?;+_zL0q35X>tI`cgU09v&6sY^T<1#(ohCLn!OiX&2`1`u}=lziW_MMadZDP z2oO628V^*(&+S3{7umBe8R zv#7UYfG~$qU10P;CmBI!RMMKUkwc!Set`{<2SwLLj|9~d)af6~j#cJ?iAJ6(5q)12;t4i}Xj=NkeYSrW^-oKA6&Y`fCD#bPery`C6!4^=iXaFe4gBONXJ# zYib51(>yg%+Kin^&_l(?locqy#;A5cWp+C*ya$?d($Vilqv#x{E88JuRXL<@(RN0m z@`#Llrlbr&5Z9v`M~^+JsPr<`mK)`yNI4iPJR3G(H)+vVNSvpow>TN6VI2R(mpQ?H)7Br!6=&&)B{?jQJsvN@QQ)1oVZeuo<{RxWBAo! z52jeX6;HBtxZaS5)pJf>m})fwE+1Hev{pzOU96k+kx@NkhnRtEpviuCm7Iw)zZ!wk zvXl=|2B*hEiq@cW*I*!j38d}d!o6CG<5BSf`5N~0(O@V}D+bba%EwX8@m`R2#C2b2 z|EeL6sXo?oa4_N?6&b=gS_4kTiEN~Uu}1X(%^eIsgOoSZciA!58R4ZMtXABsq5A^j zbLBOvwL{^Bnc(=OROvd6(nmqx_R@I)XO$lYr3=xkx zT<#}doe4dndTR!J+K6v(FI=;p0ZSh>!iZ;A;H8fcfg9Z*b!06_N`HjSnx}%&)g(w)Qlwdt}3_xMKVdHZ0DR518WyV=g%Gh~Mtq zr1ov>F3KS}CKrp&EN0E#z1H!(%sW~7g*L}chRd+9>pRayE!KP9=&O~ycW->UHWT|6czf)N zuZ6RD5+pwk<2dv;MCa$=?}$i|%g(?Q+wXAwax3}H^*m14nuwdGwFF_j1GGkZ+B%MMXnP+F4dEjlzC#x90mVeF-o5P779JokGs3<&PS-7;U| znu#4`onQVy*T!#O|4%yKIgUHN?34HMo8Y6ten8g&j_W>St#6xf)zLy&)Ay=4QgIr` zxY0T8bRPq)ueNAJpsaD<9G>l|E9JT~@FpW!Y%wckSz9A8Ec%$pjtqc@w5Pp=p9RK+ z*5omzR{X_~l{{^1SIT)$;X|)y+AfCkV0ZjfsQ0H#97sG(^%jJ7H^%c>_U`z>zX$J| z{uNHws>9zn$DxhQ5IOX*8`=gA;b#}Uqm;QOpEuN9JbbePceSh~JuK)M+EJaj>Y=ms zy7LiB?xShlPdKJmHe1{OH9nj346h!k!;@n=^4T>i@oVBi`ffN%PO>=#U)vgD=W5H!?g8I@n!Q=iZ2bPRvxg)pjrc{g_jYPJmJhx+89Th~tXUZ#?I zNKSj3jrIN8px!`p+395`?mROAbyg2%wSTyxPJX1|y)Qxhs+#QD#X9nEzW@lU)k3;| zuP;0GEeGlgUN_EGuCjV9RvqoajqcNa+Oey7x_LEhIH4y>(|n;`^R8Px`Ejg^r2fXz zZDW|@ZZBDDq=ZR14du>}Q*p>cOPRH`8t-#+4N{-Lb=s?=ppy;1ugQdUr3>Nc)M;oM zGfp#hTtoOr-4>(=Wm~6ClCTr#^I>Y>DWIOhij4*^r;|Ik z(mf6?4z!1S$XrSD3RLH?_En)s3#0vdk~WJx`^9p&c|*F!A0q9-8=QOM6g>A@$?9hX z@t48L@XU;1ADJqtm%(n(XVOYTaL#c@IsH}zRE)e?)hplhW=Ti;i+FR{3N6hiPbz51 zH#RAPZ(Do7jh8X-={b|MHiR`jv4M$LK?=0D^e{)g%XgSXRXh!dS2ggq; z5&dpAud4AHr;9~`NiL|d*z}2F)z{O$9(zCH(ZGmNC)^-J)*S0si&$M&gR31yTS@szilf} zz0aaD{|YcW&B9Qc2Jy4pOb&DnV+g{41aPA&{j`YE@139UoL|E>EPLC z#^RrmKPN4~L;kIW;fqgdZC6AoZ2*Lc$S)RXZ&h|zYmoo5<1HK(wij)3+QATUWG%P-J zUG*p&(XEXxHTEF!1-r7P6@T0MAZ|FOBRyRjaNiEiRh?AvQ~@#a(eKPJ|!NpmEo{;$TlGSW~3=_+Km#C7kB`keVHLT*4qpb^>mohMm%IQgJOG5 zJ&p~sKaqCmNEkjAsh`-?*NM1vL=9C3u(c)m$`Z5mBR^y6< z7LsaLd#(3QamTu@EO5w&=qoxHIv|a3L11M}Q^>!OhkFjv*`&>LwTE78WUbdFVESSw zW>Pzd&Z)k}!rS~{rYFsj>Q)j5!@9{=!QewP!mHP4elP|44z`rjBF&h3Z{l4^I!2@~ z-jCF0vR-QoykEKksfQsuCxV_=?*xnXHR5_+#q55{4InO&%gZX!(8P%FV5H(0(WE3E z4(BamjP|J9`(qicXR25;F$YPHAYrQL4)Yl49SB%&sj++86RKS|l*AW#iZ@9gh=Q~; zo~wFVN!uxQ@*my*zWl#J;mGQ1iAOa#wof4Z9AW*D0+!zE7mlyE4`I>nc&=zU(zx+k zRy|hjdL-e>UZi=IFMSv)eDI$UjHJzko6B6(O#$g(djwV85Z80Rm9!W3^HMCopNWx4 zBM7sPF~yU%fp>8Bh6T{1>1R+pbG~Z~(l~MHqa=t!zXY>o)M$z*T8OSXI0p0v%1Es%LiOpyI2Zv%cb@ zg;T(OXB11x$b}e9kx<<1k-1vbscQk#raD&f=!f${o8;Y5OEm(S7i!^U3w=pFswqjR z1Ef9h*`8RuW9SEI7p5Zh9H+U&4fGjT8)~XF+>MuYu+{7%c>QvQ>H*xgKa2&PK1zD| z5fUE2Zr_chsm3CWQS03KCBClfB2)cN%XxRwJl&sMWOSWc!Ue>ki-}+D6`zYymVMcR z&uOF+JdrqFQcWv;3O6S1go5L>rHg|x?2F&1xwBHM$!HV+rarXifc zK9c4tfa$j;vyc}KobU%@O1FBR+jJdUud9zMT*4L4aC)uho`a>H-Ni)MS+kbR!vK1o zG>$3#;B?Clv!+}FF|(VvTYW4WQNO(;F5`rylGcqa>6$LubT-C-^;=-Lc_J$-s4JH( ztRY4n-V4X_tTD-P0GLzm;pws&=NB!&!Euv7@y)qb8QNFF1^lzw#45gJDo&v_*F&ch zLnLtrcX}ViiHp(b6F~T=7BFR31%~`H<*u~~V8ZM9_~&&G+LP)qC=E{>E2zGCvkD8b z_4WplZR4z2tFu8+AH($yH*jFOC0Aa^y76&QckykdMZ}Ms^Vsl(L|r?7M!F0M&orhj zWtA_W>lfoEQC?}|!!5@hCTmDdCN4Py6 z^t}AB_5xd_(}8pqJ5tdVnsoG{HK~WTE2>v{ppJ_QkaQ-fIemAwsf?4M+Ah}K@b>gd z_?7n@cGW#CY;q1mxBN@spKKx{Z}=Yqx};} zFP#I$VWjbqu2(1z+3$l1{}vH~H?DNYejnE0&ys`i`srUrILZw^?80w)A6dG2gqG@E zBs9~d`RJ@Auc7_n4nX53Pho&xESt%MS&vx27jK|Bgm;@i5dH?hH@97=X_Ey?XOhOi zuG={F>B^eH?p`{oNi(=TEOjsy7}w#Z21Wc!6D3P6Xw- z^a}<<@cBtlZ^YNx#EP+ay%}V9~xL{CMFeX{TLCI0MR;?-)J+Xnf?e z{?Hl>2hw*iL0vn{}<{Tk0(0W+W)xZ>rqKp;Fp zg}ub}`1tlfAgt!?=Nt1g`}E`{ub<-BP7mUNRaF|8cop~mE5*mvSHj@9S3o*edwKj* zM)L^AcIIKk?_)r9j{5HmQDKk6uH6a~z+=-MP0v~Lz~~byY*4zJ$BO<+Uox6|h4D;R zV=fwnw-Ka4JCX;(8N1fONY_N_>5c4%I|6w_&rxmL6UUlM!fGw`H+V10()^i%V$5?> z{`tiqF!5d`UWeTzUfUu_KVib^eXMZq4oHbB6{IC;?Pk(BB4JEvbc?nZ1obTRN=aoj zW_h%6ZB*L-zR6#t{6cO$=?l}RpJE+DzF<(sWexEqX^}M1$g(<6<)tvst_8d9Xc$ zneua=7T1$*XD%vl!KjC^Z~8nmy8jV6U6~8&y-wbVrc7odP@QOe`erG=k)tRSL`CZs0kgPy~MFO z^FeVt)jcEpV#-^Q-qjKpL-)$fTGE{Ab8%2h2T558ji(4iR5YB8^#p?%n}Ehk`N((T zn|aEcgX&|dd9d({C;m9W=za6`nvWA#kHi3F6pH)!7t38poUd{mk+|2FQyptb7YNE# zs&rs@n}r(EjbfWmDf_dc8Tq1GP;EJ}>^q!vSpe%(r5@|l^{B{de-b7_1wD!>D z80~{iwO{5a#EJ$vxI6xh>N%<}cTRc=Ukzt6|E;yq28|})dsh&?YgCp*e1Mj9TRXVzEQfb!>p>WrziSoqZkqN|5V z%EkEiUWPQ^UxmF9-CLt)t|5$4nSj!^z>>ZQRnMets&YE2NgV1oOz8!%Ge}UpRMi`+ z%hux@hbyW-kTN+9cy6doF_HJ8hNCL#F~`0%u;yxLGpHUCxh1{~0RD`*{Xa$Yu_C$?QsO^L_V4X~A| zu~hs*JcESWNLn6A&kE%Ok{1l%D(}kibOzG%NOcA&g}RJ1FjA%iZ{ma}alJU8J5A_( z_^0v_wKklv0bSD)82Ljj<&YZXV<`{Co*&8?^)HgX(~^I#^6a-mZx9yQvtRuXC@X=H z;q%3mU6YC1YJlo7)eEE#AR^?UP`-fpn2}yX>K7#3*XCZ^&JG-|g*1jLZJuL0T6r`^ znSjz%g60&rlK;4HWIFlJa$yu)hUEPPaT{eh-Nmbi=LBg_(p3McxmIhA&7-`ulo64y z&;Vt%qL$w$B)))c!MRE|!JcmW@jE?$OL+o|3*CSL59d|&9rZiu=Fu40E{C-%I)cRY zG~Sv(Sr$^B3o4iS_x3pu*K0Lx+EFIuB1xYMs$FPy?E-1)IM!gS1t;u>cL$le21!|( zBwZ{H_3f4Uma>{|}>9&G!Z#^|@wgp-dElxv9rW~Qo_s&JKj zS5;2qH9m{c&lJ}v&4Q$vwUj3?rTf=a+gIh;$UjmI09P5Gx_-(zx#C*FGvx)f3eS{R zBK~BQS>`K`s&uX4tV2M$S$u69iK8q_G?bOId3PKs3%sK|A|LBsQRV+tUMKB$Ft@G+ z;IpkAl)QW=l&7BGrBYn_y+ganP?zsH+fmvLnu97ABn>XKM#~kZs|;OvHr@j1du7v? z`GNB;Ygv!YXn1$9Bus>MUF?D8iGAwzoV|X!1PA%h`A(F*5}yF+c}eRd)i_j*Q!jA! zxs->o@e%E{r@90xp4X866iu4N;qN}4F!v}ui+pJ>ULCa#F8X-j;JZOE|Hp8+-1rRJ zQ$0;{bEzZu9qA_P)@&~AR~MtZ)6=|vx*J67hI5$Tp_z2wvJV$pb>@TqEeEeAzhSu5 zEtuZ>F-uzhRD^UGEA_LI=pMM&aP*5Ub{|$Lf)7RWBb|-p$E7h?aj+Z)+kJrt{WnwY zR)f>OW9`>;Ci?eqZfG-F(|=2xc!UY8Uq3Itv!WE{O?(Y)xBtUkW*ubU!doAL^BMkM>XUo zpDfYw@-6W)Yd)rIPZRVrI!AVj%s4iMhrgYV*^!&@{fIAYrCBfV%=N_sJ(|ms%#Qr% zmlRewyS7|dYL7HO>_F^E95FTnGrK(iYn^XMW0f`D8Os}~4W;+$Z1Bk#!LqC@_|yE$ zcynnQY(Mu7SoB)3+SG=7PHVw^FL&ne_xkg5cXi~%FxtnZw~c4;;agBXaTBx+*e0Ia z8S%;}(gokr#Ef`%aeLPqsn@KfbZhM-d$?yI_f5lW%ii#J#x`bja4m6=Igj~s6GJ*Y z!@OZ}qV!{t$lUk?7MAXTTa|C{e0EK)=C1ZoV;MX_;C*8s-o0*hd3bFFT>P>v1oe zW7zChGLCcKCi6S&(ij~ym2ndXNxEkFj-H>u3Xm%cUFD%8_rSdQdc zxX`4g40Mn<=E7L~o1cet-I~JL(OA9ox5jo|EEy90z3&rX$h)RR!7j5L?!B7s*Dh(q zFB~}o8Fy(AI2H1qn0wF7t45AsGZ#WB@b*TOu&&Pvw%JuGP@0z2RF2X z&SQOe-Q?le&nN|!Ug*uvOxK0be(7L$2-!7@=`bg|pC)`*0X&PlkD>J3kE+K}mk;3H zId4Gq^24Xq`Gj`Yv6cHd*1lgL9{YG#v*~0{b6%Z{~uMIOVgEC+2O!^bsn;K#9hQ{C=tc9zi0~qxdh@cg)ptBoyT*>&h_lDVFz*O_yDyg0ny_x3Cz(z3F3%>->T#N1vCQ3&j~W$%^qx?!JPV@w55eyX zG+6eR&Zv(YhUG^NYj3+1iZ6?M@W4&_Fvjx$4$vPeXZ7gM9S&q-k0pJ0_UKo5Y~OLb za%HrP>VHRI1>$bkQL^|}9?p(&;8R>qT!+blVBOUUKfnKi`eLq)^bJB zS1~eK4a8<1hte^22*I*E|7Y1QFaIAqLr~c6R zd018FM2;T0T|;A(^|!9lQhnk2>6t)d$FDVL zzMnqUK5_=}-t%z8mk^jXyA&__ald~kJoER4jWN+w)3&hN^)W14lEPI#Q+-M5S^RD? zo6ekUBsa#?<_(Uw<-{xb;ln!7Gbg8Hn!cr^et~Z<$-BE7%1uAc;^)f2;JQ->N-}dr z7g!C8BE9j)fOBlzl05w7yipEa-&#(aJwP@+dI5SSACXks@|91DHt9o4nVK|G92=E_ zmk<60o0y4Imknip$8vcxsR!Sj(U^^y@{YL6etb*&lOyzis$yjx%v#it11N z>llviPr3@i7&yM2?oY|K=i8SC$Q?_}G^?FoBJ~LkhoL$P((cEzzvicOkxd$QsPGfH#)SG=Exjmx~)@fpMc!O{6J3%Tl-; zy-WM4x`cC~o;aYT0azLu@%cVsxHV!c{PTBWd!5$E21c~ELF_Fyxhx%3zf9LJ0###A z^ZT%BcdkMEzBQ^ijrc&Qepu~10ah#;3elluaDBKhr~bveW+vQeSu1?pI|>@<6|fHa zGuV_9|FP%yj^yk6UK4~HS~U+whf1N8PID`8(PQmgX2ROu{?P5hLYbOK&ySB!R=NOp z3>huX#?tehvGH(k!4)99(Bv$?C<1!9qR)ZNxWqFasD=ftwcOLcuu7K@-pHV%P59!H zVbCJI4E|QT0aFu;z-`_cFz8|?(-W?U<1gnh^EOd*9$Oq9i0#aiKI{_{+t%RGw$+rr z;E{p5G_`j(hbJu_;OkX~U||T|vqbpX)?uInvmd{Q1oaJy!x?{lOBR45pAV^(0$_NpW^P0Q<#Q6ursKU(iix!RVXespNur0 zNPH&N)0ySOLF~wy-T1TBO&~61#JiZkRB9CGY`mL+Pg)G;^x2%*`T}8Q)%Y4ayuz?A zeZ*S*gH`>xJlOzMkGs!aBKOYT3*O#aptk!%Xm8{Jgw4X}nbVJ16D`HPlcDln zn?dUns&NK%U9e@JtYnpm+2M>g2wM zK4r(j_QpZner`HY9}89cj~{tUL+d0}UvfglUm)Ft8*(GC|C2*_+^i`ljfjL3IMczA zrzSnZ>H5A%I*k`^*haW$NgB4E(o#U!%2p06r?YrXxT-_qMKnwQ#R%VIhTTzY-iY>b zO}CS@KCtydF3>t6X(MReURUZJ9Zfy<6V~8ukN7qD_(%?iKM9okT3#aFyq15bs6E~q z;*OmTIcSvrb348jNPBC857idbTdY@GJI?Mn0_jE^=3R|1uP9f&1T+WyOx#DMIq*{o z)Al%9K|0eBekN@vodkFop92lR!y!OvHO=|#4RZUNB<-#m|KX0a=}LFQ6ql}i z;PfHHPosq6r)pw(#YBv#WzPG!+@t$uE{fnEpHTJTuEIp5wWu2N{=B0=b<3j)niB3< zNrUz8M9cRYp7?PF)(Y+i+dBSbe!=!o(YOs)9PLzZoR~i$mCj&oEgu;i1;P?g>v{HG zcldAQDlj?Fh5DCt)5?u_d+|td?cxA_f4;G788KUs_J!NKOnCLH$530_6>m~D=~C;a zX!ZI4qxEHJ1|yMh7%WWc^G`!|3er3{bbY2~3}q5jH*}v?1`J=MfVTdstr zzhB~mXC&g zKej%HKWw+6_x^dKK^yk zVEAF&ij!_v*M@tYOprVVySBUoUwk>91y5j-^c;)n+6qZS!O>N829ViPLE0Dh=egn7 zVK)SM0`bmb2*z~Hz-t!oknj{QhtM})kv)*iZAkUj=kR4kNNZKSo8y|=$& zJ=;yzsx?>o&+hweg=N~pKI6cuSFl**X(H_SGl<28C@+bSW^O7UnlX_XYy75 zfxd45P~Ry`kfibW?9g98YcDLfbiz5dmYSd`#n|KySKgQWh8d8?V5CuquSUz`X32sy zGl(HQmHt8HZAO0>C?E81AbM|~q&SB)9~;Dnbw01O9vk8tgWlejlKMvQn~$gte=A-_ z^3rf=O%3c8@qqTqZK!$NyE@W*!=M;s&gYj2bJGWS{*M#s{jRJ-%dJS-2B@|)O|$fY z_>p}-v5^t(=aUx_q^np;;9Al=%M?Be#WAF*;YWX6(sZ*$udq()=g`jO1L^&5!t9a* zkQVn;zC7ejH?G$1`UO+rU+Y8_&d?lir9H?ii1H(Cv{@l-VM6j&(plwVt*8UN==>Pc z589TVgR9oR^^y>#d@{P-5eZfYz`5=Yv1w*^BE{l*iXV@aX z2V;J3M6(f=q;s7_+rY6veIWk(P=H<2sss5`mHBAXd$)(&3(XkeB@!kB;RKkUI*~`~ zN;p#im0x8Q-|7GUldtBtt!a0DXV9Dc*Isd`cYB@6SGe*@$e*Lpl>kuVaUn zR+ObMjK4OJ`nsu>FoV+?@VwMRjPwOv$4`aR`1|n(I5EEoTq#M?nyy$QNFPbkzx@7y z5mkPS=2FHs(O_0)Z&9RQjgtq^9Ju#@6}T1y@hC4{@keDPK$?<~UL?(^Be$iVR{j;% z#qY%9+hfE(|NTPcC7l~N0L@dC#~}X?^m+I*$r{LqF!E!Z>Qojty@9gEODO&5%gEp3 zz-h5s;t}O1aYyabDkIU{tV8=;A2j9WfeXd&%{P?C$|J7;Nt!ck)MaNN{h}r8Q@Mhg zQMdlyxDsRyB zy}i<2NIFkT`6MfN(Hh8$kQX~I&f13{jhFm?ButyNmQfbTsMZ+ehOE$_2No=N0Yh&@ zume3(h3B+%B<(AXj9b7|-44|0DG1MDNY@}w(&ANhykp1{M&1F5_qgfG{xq*m;X_A| z$}_GkJcK&a0=eQZf;^OX-LRZd?_jU; zNYXEx80Ds5(|slQUOtJG2MP!N0+dc%w)B-Ctx%<}>3VscGql$e^+1pRRzcIrhmi76 zt;($2j$5J1jEMVCWm<$~q_1jlm1}P1HDSx>r7-%#4bk*y&nliFd`996Eon$TZI&rh zT5Yd$f2F5U;jPNJ(hMGH$v0~wYTwh)|0~Y-Brn9syLwXIK{_B;EUjm!w2{WL-((nH zYd@--v_<{a{6lyUn-%+pGPN}3KKm~t%mFo5^zV|eR#HDyX#w(ysC0Ix@g@qdQF&{n zHxfU-_ng>vz2?b`Eb{Jtg7A>#4ZkZWw_z>QY*5$lDTD-XV81Y2Lq1l_KYS9&i}RX(MnM~C@)@i>PUwhT#pDS+((nBmWdk^93Oi-Ry`Kscs;o|q>nS%7V zP?;z3rIyCVUadXOC>yNOYvHNENcfpgdIVG^?b&>t^1dqT0+UZ$v}!%>4|u3@DI^Z4 z@_z^J4ad0~9*S*mM}w*xy=MD7%};p?l?~GGa+S5ZZ)vGG0#xR;&M^pv_|}nsTAT%? z6;-bMj*hD868TzC&L#)niJ_kAC`ogWj#@)moS9tq<}h<>y&pc-@fW=VFA378k}^9q zSeFhS4{{h`5nR(Gk^lTm98^;hk0Rk4-0VR6X_p(yfw?YRW#{BiIsEPq1+M;}xQ=pW z@-41aI9Gf80gV087MfIZ;dYhB#gmf$@ax}Mv~Rf@X43Z=)qfPx^JyQT=a-{knH?uX zd|SvzJ^gXMQJ!$waafo&9f1AE^b&6(>Y!VX({TKfC)EFV5@y*rY6ngZX6;RTz@O!P zrDnks?U&g{=vh2-S-X2{c=s}izY2@MC2@`Av-ZJaK6%OKUmwCH=UHg-BNiIY+pG$pJQT`i_m@Lz$~xDh;hxGq_y{bw3t>Mjqi4o9s6HrXCI`B zwNrZ|*BsTn3f7kuk2v?Si)9-JjSy>Hufj{1%ybS6!ySEF$|Ki-2in`<{?ud<&}AFe zEKJS&G0X>PjIzYJCO>?A8ZOT2CDW?)mhDZ?W7Ie^Y@?^apElii?0-|RuwEQ?9$k}f z?D!b!RfO~S5vyUzoER!l} zjehnM*UYbqUSD)&qlVVAysJYGIpZO&WYZ>4ov z+KwOn^$RD}Wb*uI+OKzXYd*Nf6IAb28r^_*+-M`;`dY~L6YIzk*8gC5*g|}6F`TQ+q+VfsF-DP@eDa9RT z+}*T+ED!DgUt(QBt;fL0_gLMEENDD(C=Lw>#d7OVps~qv>(?;s)@)2%Ss!h@op|EP z1UU35l%;t$kWu5-G1pPqTGgj^nV<21-(H}az)5R#vHiptd=_Je)%uL#wijo~2JvU% z^OXZIcllwoXyXMpa`)iw!G_u)F%wznm38dh%SPDw%rA&9oQH!ax0J6Qy+gVl(U-0F zoa-9hmZd!c|%qi10+a~-*Pr5{uPozMC?P0Z`sUPE=yhll-u$WxK% zHn9+BPPID4cQJKISDyXrye3Y#^WnEn!uu(Ka1ru=|xvqyVGl7X!~h! z=D}d3`N29Z4rwl#eTD*4+9&g@og7Ths;1ssU)76i4xGoVgcDGh zNJjGlcaz;!zp(HI_WbwBo;ZL1bo?+SMFtp@GOAO4Wd3IiwBLa*Zn=QvwK`bG_y}&8 zd!MDycYyTyoG=}(Uap6l&5yJVvQMLdYdE`fW(B$~4F|Q>3ydP6S>!Qr`1B7J?3YmD zya8>Rwc^xs5_`16n7iYE#xJG?n!{$@jcSf~OFHK$VsUGEw9!v7&af4R26mH8ik`u$ zZr;4u?g&13aRR0VZj#jBxGypg7M8E#!Cz-eg~xuE9pL!uj{MRjpfTuF^;KiE6|1Y) zm2c1Wk;dB=huaHOh*g z)|8@A$CKi6nxRzOkg(^w>Ssyy#s8HD5zen?saT6$Jd=Y|i<@`zhRJs@MA$n4`m|rSLb!iW6>BVJYzerj4|K-Zz(vXNQ{s zjgig%m(IAxjd|8{kcOozfqDkRuKtFYyJh&Ten+{m7VRmOeOgrR>de->iUX>TeEW;u zKz+}M6BGwy;MswcVMgG`UiR|xGvRCV+(dB!W!~yomQXGMVnV4^ziZ<0A zD=vpCZ|d;Mzs8bq8%#xgg$F>~OFxehqr&2G_Q?@KVO;2unk?G#DXO08)A%IDiDS6= zX>D#S&cWo?M)GM%6V(@NLFjC0TR8*m`-OQNscS?SdI3$M~x3F)*O*X$gAE-8=OIbW_yRb{B>kAmu2$y>u$tP{Y z32PxZMxQCHm=hAhceFXcOt)Ez8H=uII{ns1!f4Pb9*XOxTJf1SLtu0FziLhd^)P9u zu8hVc)1LK_mz}*CVKEL1s1DQWTXNz-c-{OLdXGLqc;q1#z4sH}le|2i#tc+-gFa>5 zal)FTINEwEK0CY%iEn|p2C2WK-iHai)vx`6Fk9X?DA1g&*Ogzr9EVMc#hkKX@J?DnWGjV$K^=`}Q%`!3&TlQGoqy9WQ2-)4i~K1IS&nS8AcHlKcmJ+9G* zJ0p99dW8?5;Vix=Ba03ra*du)GX#b;Sbldgj&EO|ihH&fN@q_; z?Jr(e)q_^mWaKGB(o3(@>y`Eqo5XVP-f03&p1I0jk9$CF=YfLsApi5Yfi$*^!7R@O z_;_-CP8=gHPI`;x(==6_d~ZxXQjK^jogA1xSRRwTvD&&;oaPye^~Z_hP7%zbUJxVQ zDwM_}eFJ+Mnuv@3?=f_?uU6?B_t*ZA)S@wvp2e2ew}V&s3{bVbEn_$MrX}K;-vha2 zc0C^49&qR^52m@%5;m_qS5RP?|<@?Up^}vZr}FaKCg>m|v&{ug8*3w5uzp962FgM|6Q2 zbn~I=^Xa#(V1Hx^hWMsy)tnH%f!nHDn7LuB;(65@7;G^^5>E-zBBTi(B8`>Ce_ED1F}ark6udT&N;`MjiCP0X>w|b7 zr`9hPaU{rBi!O8k|vY(b!jt_@iynY)qjOw-lGz&)Kss8jFd9OXF=7uy8kk%8AdKR-G zCGAkvV&oHjPIZ9OZujOX9va?zsAL(LPrzG$0Srn{g<_vW*mcr&9GMq`21oM+>3!nG zICfRF{d@ILJkq=(y;gg5<^|E=^%k7e+EZyyIX^@P=ao+Z+m?c@t9?!AR8NJR z`wj7OS|pP1V@fw1J~pFDi~6MfNB(9XY25vSdI(jYFYS;*_g|idnS3zsF*5^+XF%z+ z-5ycW_e=xyJCqK6hnhlmBR@WMc3nAPjT`^PV_AAND{0(uqaeM3KW<0C`EY0O_s>9@_}R+ndPvr=O8NU#$CZ9mzjI$(t_NhZVrRWIe8GNO>R6oZ-+svo`8Z zeJn^H<8(kT#fQYMML_jdWFBV&m;Av>VKSTRG|qA z-KeqJK8QY}5T4GuMSFe?0JgccaOigqjs_Je|BOEuZGe7%qETtWF^_DhuFbg8=slvH zaMAm7itBjwyWhnTdfdyf)IfP_Ts7{v!XbrC8p0ej9Q1)TO+8A!`64cvW5d?h-i3_6YN_W%ZKSjjK-j)EJBLC8L0S#`j+^) znf!KJ160F!DN2jIZXTn1v`@pKe#b;ZkKRE3o*NwXWW*sbs3+Z5@81O2kZ1W@#uqD1 z1$GlwVsY6Er2d4`0ljfeaXfi>Es*{|Uw=AJZPQ)6P?inkpJ7H)6Zt4T7S{ zW-Az;_zjy)$YpANPg@1@#v|{d@0rcGaK9<1UcgSrHKgel1J$9@QHtj^#KS7f;1BFg zIrSnVPm8x$23B(oQF^edwnysd>gNum%n23NChS=ts1Mkkhi$98 z;-#``Fmo8;#Qt|k8iIwr4}&=;GjP`9Vc68Kk`1#s4w;n)*p9yY*qU`c$rHb3-QU>b zoUSh*?(Q<7{H@7Ar z^Cq6u*oMk~D&L}W;65xJm`uLRTWK^cX;{TOTE+X6$Kc%pUFq0%i&k+VWm%kF!^*?j zW5b3eETNMrC%jhk0~5^C=FDSrj0y7=a$Ma?IM6lcNmGxycNEbE}ad&mG`Na)Lc@L8A7nGggaEo8)jv5KdNjPZ+P~LKJt#}dF z>7DWb%G*K3BZ<`x>_@^~B)^KJJ8;$RDCnZUNkera7P`#Wv>6ovFTQ5LzGsak^%7Q} zYpQex@pLUr&^@WJw2E&?)55)FE7_A)Ni6W}4Z>?zR5>8^HG39tr;5v)rn;fxu3JN$ zxWZM+m6-A&it{^%uVXQmhcRO70mXkp;cN4{OQ6N4t)j!mep2a^8#iqw;i-nSuqWXL zP`)8lrfR*!LD03bc6$p%)~6`Q`cz#KuL9+s+QBtOD?VZe+rDDv8Twkv-2`b6^zn&? zC`TXV^Wb3HNvOQmd7sg!K9l-|QQt`Qekt)0u(nTM@=n1DBPeq$COm6|3d<=I^9=s_ zK|?-Psyu`^7L@)pm^&TRnvpiGs>SF` zeoZ<}b2s^z$_ug5_lLqGtgw#4n+wxbmZfwokmmwBzg6TVe<}}3JRB(9oh&H-ny&O3 zcs>4!DzF2Iy`CVblMOUkcn6IO|Qp zic?~J$jhB&d+;Q?Vpf~0dQ-XaES@Uk3%`m*F7;6PTg5}ezIGzb<|+?=(hz1?N~&R{ zWu@27x3qu4M97v)@npL$TzSYjA&ev2*mzDFf%ymcyVgJ4hh;;evQm&2ima@gtl4)W2WzVceAFKgrO0eem^ldI}R z;NtbKna{Ny`LE6ke%>aC(a+TDF+0YY&Dr}9_b<`M;VWA4HK9K8_?}WcyLJ>`(JcU< z%}o`7J6rLkA3^x&cEjzLZa`ZDJ$!3jD!B1qSQ6Y)3@xmII?Yqr&EzseWJc0RDdXPR-><~5Et5uw^*`} zPP?#t%mY^9+g*+dkH#R#wN|> zr1;a=>^EJDQ5^_6yHB8cS6zkax@>L$LmGr;g-kSVe!bWP*C~=X+&phO$akY7lED{?}+!ix(EmfUp zj-DI^v4~uI%uJ59sLuztNx*QMn_#xRvF!e%hP1ABk>)*?_E+j8)`Xsd(lldf)oG{f zxXK%Mjp!uDSL?%*&*$U1t97|o-Ef@Il=gw~i{RY6rmS2?=h3DP<#Bda{BJ>^tT>wq zUOvarb=*`m^!f_)K3M&sDa}o5aG7N)MOb%EH6_{=bJZ6zHlQE03@gO<5h)Nbey*l_ z^e*gjwKlg(H$^p$ADahrszWF~RtY02O=x^RVpP|jNWBi>?_K1+1`cf4I|n|ra6NW$ zeZy}2W4!0mmmV$3BtGcdfzvt&RiEZPX9LxlT5Hr@_DOTz<~F`;pMzt%NO0=(7bnGc zm3{;7qwn|xNN%$QH)rc}o#xHO==%>fG%l9nV#3SYwZNrKpNkhe4>KAs+B;;@8A@eMXa9?{^9Ac~B%k-f* z+dmqp?)eaxD9l-3OS+Bh!kZ81hDV)x^Er>=p+R^}Uar3kRXx*dq^dm{1O8hd!&N_? zKIf(#7F16<`|04)Q$|wtUfi;8+R%b4c*}Vprgy(5_C)!^yaz^nRM!iPdKBotkg(_y zP97TpE}qxL!L1+gL9<%C2Wsd$sXPoxuOZuROCVf`fFGre`LT`Gy#KX7@c3ya-nZMX z@v5uC2eq^3j!pICarT0(y5kSO3TNQ_E3d@yJp-h|7IsJitqbAbK1uV*I;_47E0?Z; z>EYAS3Uf?4~2VyVG+PW2!!ElWa$XS8nO@xHIx!<*?$*9>PYUYP@1H6!H17v~kO z8D_J!6^Up5z6;#P>6{LYNQ-!;W+ZL?R&Q@-{ zIunAM48yNCJ>;6CS9E^sEA?6Q?CljWc$y}|Z3f8)<+s3lS|3hpEHl$<;qEoJH8n?_ zXG4cJ1Je%2;bZtF_|u{@sy>~U;v=h@xxwF!O=RN3jp9MGjj9%8%U+H7e{?UA>CXK~ zeF&fL2EyRPYJ5u%6In6hq|6!@%fA}FWyBH8`05+x+OZqheP4-{>n1{pd0kF_kZ@Vh zlP+@j8%wM)W-yfAXv2#}-w;&Gm>o0+X#VkgVi|eGP8jy{Gi=IB#@|kTS(MQ%pxTH0 zmwm;T+h?%S?l*i*ufr7&P|qt47VlQH(IhA8^U@oKS)Zr$oMhq?9#AqG>GeREE}Naa z50{oT!Vy^k$huvHRX(|RYDWt-Zbm%J&z&+>y^h4$obUuA{NFL^2k`c4$c?W~h2LW% z(J*ZcH(1E1e}}}T5SpwBd?LBtjDI6eFwyx7@B*nKzP&u7#9b{n^% z-ityY{zi|%Yw_=(iClNxaz?nuA`k2)j*XL1-YsEw-aoC~#)i15e4sQ;dkQAI^3bq{ zAt*kKMbLzPU&WfZlxayPKu%zgX!-0AY}iyFe6HE>inFaH;SBu!ZVFBQ9wMCULfH3O zP)#tx3Sn%Ls#*57DfBGArg#eow~%oCf0E8Rtg2=U<93S$2o|;|CI)b3jfvQ(sMsA~ z7sh7?*kS<&Dqt&OVZfO+f}moeVxm~s-Cg$`?jQI0o-45To|(1Y_qW#BXGVRyR)dDh z60)Vs73<#>5h{D7wRC7G*6HVGWhv#(4lU2pRVpI?{9Mm1hvc)Iu2WniPmw3nNwtp@ z4>D`%4v&w)6Y|I@6AJ5nGL(#8On6jJ)_52tDmn{!=}s?M&F&K;@07Sdfg=ZWQ0h^A z2J=wKeMh+LwQU^M0dM;>@|1N_(}kt=eMYRaSM?TzzbEGzojLE6T5@ZJ*VO6Wa&fu2 z2S3SImd5y;;nu+=wBPWWY!f_=gNH2P?se@YJXF42(3qcXs;GPRsY905{T2R=i=J99 z+>iKCWt}K8@2$c;cd)%n)l?*J!ja;=Gdm#R0le0aS z4Pm2mZ7VtI8pXPcW&Tx-Ka#`VYC5E21KD%gV{6LElCsaychq@O1;bXl(68@wp%jT) z0ggrs3EG0OhQ%+4uA~Lt^&GvtFi#E$=he&dl3}Z!r@~d!#_Q>tB|C>6T}^Mx`d|+5 zxln@!+VOU(sO`~J&zrx);(a(rT*aRyXk)(8>e8LtD*LjWh}pG#aep%%IM>1CspZmy zF?=XA#pJy49t~f%+I-94$ZrGW{SLe6qC;&BPox=Ndx$NcP7>xxml^G4O(uy! z?sQ30&^VsrH`DmCoc1$-yX9Fbz%R(lD1I_NKbIeBBjHWl`$GeY9B@&T-nG=q53BIf z?5%Z=FPCX}@@wnFwnfb|sp#>;G;#sH|Hf)L*p3&Cxh$~uB3JY*v8GZudAu&Fx5mw; zi(cL6xEN-A^T8cstx9=qPB3_fn&!?Fy`3`H=%8L$!R z?#>>G*gBw)D}3;8eX<$QhUdS`rLX(%BE!4TZ!0)4u0fq(qh&ng#$oZ{KpT!OQ9!SJ z`%ipsal#rAz8bl=LjMYF!?SAT5GMzA)9^;r8gHn{O)F*Zd|Xt)o=Lqw99DJmuA+l~ zV@+Q0ieF{*sWY*(BC&_@Oy2pbkdzZf5IjhXJGM@MP1Vo8NBG(KHgZ<2d7SNKf8lgF zpVhlvn8}4K&N!+)!7PpV`W0Kox6v2O99WHh<#^timS4iTYKJ;A znBL5#DtoQ6gl6i{rz!Z{uZT?8;we7O!h0FWLCk+#a{bzhFexL(srF6~00&Fpj&UG&$!Jvj~bfp#RMbhS7 z>fiEW8ro&zllI>76mr5#v2ffyg*l+x-*W0eudAX-(`tG;zL)b6N36sS&ZUtrwCg|$ zN2Nq7_^txOD(~HRw|w1Q5jyh@_6XRiE;{U^Hz(!`aBxVRWfK@_2inm`oXBe@ z-P=#czUL(+_?^K7;?blk^xuVx{N_YXX|w}eq=qMtB5*nXee;X{`aY*pdmPoPQ%(e~ zP#*Zc$iThTWk`j>x?1`96g~c_+S}|P!HWp|EL`iA5vyMg7M_20i((a?Ta5?Y4lXIu zS8nIoi!bw&OGAy%5Lg7y+E=5drPvb>D5ZBaq1V9wZLZ3QMFt-zujN_jBBjt zF~lX0Ke9xD1BHptISZ2%&iLbj6F<_PS%H?}HPZ=xm(>R-Ba4Z4v;7q6Xqx*rlAq_w zp4Ar_d@IRP>Q*A-o&p-A+{$$|b*Kcxa!^b?MlHhceDi~w0>sqF2BvlphV{YIJo)ZI z!W^pwwk!Gc=L>wLX|ysmKgI!ODkz~vs!>u9T{zeg^=dKQsONfBEA0z>q@Wz{y;q>u zBC9A@1>Ys-Cz9rtIfL3aioP_9Rp3kRe03%xrWCwK7+tzvXFMeYY%*~{ORt*q`3e*2 zhOAx=^vnXzl$Brk2?sk{U1-P?!u9$4$m52$$m+dMJ^%4u<*1OK>u%o0w|dT??F*PT zY;DW_mzRi!y?64W-F6IK7Vu{2jIE%jCK(Zibw&)c@*hMIO?{{U;dgl+_FQwj-yh$5ChX=8q+J(Ub8wAGJ`txX^kq zr_L@!&+-%@!@-6tqx0dNAi2h)=c&>fb&nOXi+IC(W?0isG{PEPVhMDz9rf(9pXx*& zV)Q34Cm+bf7Pw#a=%?9d;u?mQWj(vi`fNte6y}}c?WR!VSr7TS1oqR8JfxPN{A~P= z9=2I#$q`pZ4$SDsLH6Y|Y7g=ozL(L5(FO9YwP2ma#D`l6tVS7~J+)!s&6^_4`Z97} z8V*IyTFHJRp$3gdZmUL%z4?RVQMP-Pk5AT%gm-S|b&aQ4Po&0DR{uh@RxzMDgVpqAo6`bGNfa+;}68 z*Q0NT^J!alusqJlslhV@78Y?!=hD7#yi+}AEVvF#iJHZ``2F}sJssta!-J{hy_Y6# zbTcZ*s81OFLhvb?Huxc(Nm`uMC&QTHOx^ytK(sQw2C#!-jcS#=wpQ=o^*7rm~FXL(jwXZAft>vle6^a)Mv0T#>4{BXO`P}3){ zf@>|fo`;DG^wAi+%;=qpiBpd$uV3M0opGHpem&#sEplktM!gtt$x?T7WoYer^jY=^ zc)LdK3!~NPKZ+4pSo}@6rV!t#iv|%mgiLSi!;5UnrAQlsj`NGUr8II*7mJ%t**MOM zdYQqB#uKvoaMg~58HOO#?E*CiICLpH-u7n45({|mo=t>ai1N#pXgo+o1_v;-RYGsF zyaIh0M(4l;)XN80J&G0AMl@20eGBSA#Oz!XFBbG@8Js1-5TbE#v!%d<`gB@-)DXMZV!Q52&}5kwVy3Khv9n~zskx6pIa+b&%XJHSjC^fG)>y!^_Z8U1aIXfZsEkTw`9$7O+Os#B(Vy1n$0FuB z5*S;&ds9aHWov5stLW?BV)Sz)uIq6iTXTtV5PF!XY3pg3%P#*NvS!d{MiD3hIV?FZ0{2fi!5k{lXWQP100UO#c`MK^M`mdi#D)0>Lfs;Ktg%ylj2TWwb5zI{_Awk+k^k8XPG zxe=~j^gCqf*|U|7X}Loe9ad7UzU)dF7ZR=M6VhncMXATHK20|^mDaQGd=_8aedY7? zpO$ZT@~Z#pE#&^)eCcJIFlxBB6Hg4UEFC%&;ER8I>Cq8(+;7ALJ;8o7PdzbGci*yv zp5ScFIKSU~!8L~-aC^DFgFRJ`wk@H5`7fgamk;yK%yjBh!Bfs_`;Tm+e$w~;wV4LQ zP{~jxPul`w;P-`kMxNt*^R0^%8F}QNZF9t~`Y)B|gG~Ayu||Ds)_}c7EYvmhN-lFd z*qS>bzg`ykTl5YU6g2)OSKPYLBe|ycA)#B9lhy1nA$9TsO0H%%lF%p zDO*ejRq9Gzc9;{%)!o5DuZ}9uX^rH{kj8rVt7BZxCzIz4sHq|bZBvJT%;QVfwvbtq z9OqM14P6xDDMWo@8|r;wN9oL)Iy&c37pfW1fs=5?*RzU^t=Vh~igMqk=qty+P=OJP z)Y)yLXk*XQbnm5=2HDT!d!77r#jSa@MNcQ^!>+2=p6+`5sXqGmf(3kGS_4gMlCz$F z|L;Nk@%KROH?}_)tv{F7yN)4TQ{)S2s*gVEMyW@gb&sm`sN?XjV%oVjIx(`SUXk%W zZd0)kzVomTJ~w|PP7hzqo-yqys{Isx++iVIT(^&3&cNqU^GnEFqn}g$*_B1_Uw-=0 znT|4j`*W^UGn`#~uX0Ru8u^yJN^;gtxp=6fZs4_zQvAOQ+)rL8@r^V0bko&GW#i-L z&d`+LZaP=wtgJZ6X1i1)4%L+&ZM5Ul_Hua3YGQVgjco6^mntv6Ew+DOAV1ZaDw>bK z#WOs*%2L?VRkkap_5H`{>wD~d=~a=2b$&v=ov=sE_YZ95ZpxR{oOX8(ow2T0H@n_a zwx3FCGvEJCykOp&N^9+M%MVi(pI>pA%rj;D^VPw3=Snk2f5)Yh`_)}syKN{XEqTQK z+%nY0#QFMHctxJ*ZzG3XJZ^HIC$tTbcy8*qX!Z!-)lPXju|n8800v=8Jv`nOP9zOM`k=h zw``@Eg9{G^(8`o+ymNmG`L&XT2D)rkeXtkAre%G&#fmiUnJ`ga`#Ic-amgyt*C={R zfV8W6NcAjcr=ewfPAz*m$*G-Qp0H5LA7j~h?nc#R`XP3jQAgo9_`$N?y34-S-17Ws zfjPrG^)Wxg^EENC&xv`o_;2qe%bTDA`fTH-;@*};+`nQXpXrhziX=b5*k7}6r{Owv z{%T#CZMA2s@+!M)Zt0&mmZm>X_Gq`eH#uX^8kai-pC7iattMUT=ke!tn;L*oAAVpD-I z*RuGrYC6^?g1bKK$Mu}ro4;kR`61S|`%)P9Ba_eAYgjJ$YNIjN5mEpUJ`hS_qJ=&t(X+4G#^d@Eiz%uX-8ww3Vw@_q3Y9Mt#(*^U{a z@eCR{NxI$wTDYRK-d7iUO$^>b2}kx4G@oky$VD^qH0JOvi&gk_CvCK%>%)7Z(w52k z-b;(zyS#^9btsNM&0ViD>nzv9i?70*PNsMG&PT;^mj(2mEB897{{&5vmpYD=@H*pf zVvMhkxU=sxkE}CIy@`vIL(6r9x2@$(o_jQ6L{F>xm0Auh1ue%rrhz}Kqtjf3f8sk; zrFe1OYsv}sEb$+~FRY?SS?YB6)5$(0zg7N~7Ji1N3$(Pk}_voj~m3*&0u3y%Ff(3qfk zgnT0t`%9w=%VXh{pz)0a{GN8lRgivT?^D%#BRJ8ci?ki{m;>J&P_yp@Q1!H$)U|## zd`??Pk9wA3-EeF>m#;rcVlLGC#L-lHri%{Gca+zycM*5&0#z~Bag3ajPWw+;-AeZ+ z%qQKyzFAu;d+_xUK_+*&bCcs->_Q7^bYt7ZOJZZyy3}1<6om^_k3ZUW4Gq{aOnj`n zSWNT}rp`?ovsIpiPgr>FENm)+*h5|s@|dQUSk4Xa597Wg=hMCUf4JnJs^a92(H!sd zK&(3C&aI+?)!@wAv_^%IPv1B&z!b)qJ@#V10RPYS>X?6Fe(X4a+>T%$fz2EEDX)SD zPot`J3+dx)4~gS{?uqIV_IgK~lH$-}Kk_}cgI75`vAWz@LWLH7Fk`2|b{_~@Cv*Jm z%aKppa`}r1|nd^viUFrIVF52akA{ptCRA&i_CXScoP4P|}s!P+W|(af0QMdePvRlF>(g`0m} zL6|$E&+6@s4Gg{FTj@n4o=N_5u12N1-Naajk}vkbxYy|!kEz7u_8jkc9yVH#p7|*tbU~l~CM`}rJcRUMZo^+vbxSUgC_! zPvPYAZ7H3rUO8*N4=sL1jfXF#8<*=z#H9Kd){tL(4-#M=eIUYCeyp2c-mH+s3zPi# zYKZ2LLOT^0Q9{GDhsSkVJN=+C_k@0tbzCno?)Fsey7goHq-{C);&tre)G@je8DECOrB}dK;S2M0w zGhRcNqL!%wIomOGUhm28YH~-62y@d|7i(z_VZ+@`7Y1=l*IfGEmVC0@$${XRv#P@( z5Bkt@J41iex?!OLz9wRAob;65O$gUU%zXeeFQvt{M)1T`3oUvu(2SSh!>CE>6V{!-RL@zb-$eY?ZEEH;7;BRFWK)zHjSDpOXmzbA3xVRewn3{dWHRU zP+L%rzjqlPr@+XRFXe`c4QMA(ugUt(dldX%6)$^D7$58X>lvq(>1Jv_zJI+OU+Hny zlKyQmp{`ISKX>j=X5ab$2)r%9d!pBrW^8<+f43VZ$JL73tJTW(%jj6D2i?zigeUnt z|Y|T17G9Qou{Oz=(VT`Ns{5&x|i$|Fs1^EDt<8%|>G}W^uOQ9&R+G zoNUpDi{PRg>AKxpBUMH6*x2LUX|3oiuzte(R!SrkT(lZqj;Le9`9cz@=)SX5 ziQ4jzi;c~*>XK*DC@CXQf3LZYQ*vD3FS!$y;nk)KarRbVF~Zu4j;f^be)@AWj9nJL z@IXz0+BU$)rE%7{UM8i+8TWK(0S@l3JNb60E z+Lh2Nu$sI|-gTG!tJR`>wL{ptbsU8(e5b$?#>>^uN7wk1_k6_gb&pM+HBe8kR~Q4~ z9`5)ZjXB}4l^0DuWR3UFwrui=+wJ&8G3RLj&MNadx<^@-Jh7tRp>aP2F5#$=^9a2U zwc%J5t{3mb-Z|Sc^iM9X_mL4}H1)(QkFK^~2--+tD?7;8fb%A|_`xR!+Li5^f=4J* zw^XjS4>|ST=!t4_$&LHJs}5btqelK7%&2z=-pC`uj#IC;9e3P~@jQ2X%ExGljFA8 zYr6&YrO|fOkeauEPJcfq}SQ5s`_8nphm!1 zve2oMwd~}Yw>8?zPQh#wshihWehbLxx6dRFB)+)~KVN zHe?h1Sbfs;qZD#cVBGwC$72EBphoA;agndn)yT6Q2s|m@-Y6wcwJRd84&2F8oU5Cd zQ@!gH(eO1!-^^nCvC*Fa#+L;6gxe(K(RxTwL4D>p8J_0Rxtr?+)wf%#=Wc;M!EPFVD#O}4RPn=NCA7Qu2WnI= zA3sNcfN!mdd8B}+Lvugi=W7_gNS5v$30g>pbTAv%*tKD2Ry|?3A0B5#FG^r7xO(nn zf*+|XRbr`D|ATSpUov`J=>HWoelI3w$NsaMH&By?LkKL8#hmCtsU0WpXXQh+v)+^| zs)*k4ZIUpoTYF|S?uB<(pzDSa-Adn4dzas0XsI$Rc(~?zg*ZVEbTHLz@LVKmoUI$V znu{OXNnrD=dS~XS5%i{92hkP%;|j0hxpc)7s_Nj19`JFIvM?vs-oezo=w&3M_j%0v zRTozG#|*Th?ofC(VOY>{b19A928B-OVtt>8y<%kxGCdOS5qTwi)Wp7k9>*isxa{{u z$}qlRP4pfl=2SGQw}{bm73j;r+x`fnIjD8Wtf_C=E{vE35A4~nJ#@y&r5Z9a!G|w zxXS3Y%I4eN@{4}164#QY7rjvfnp(xQA!XV45_-;5s6tCFwm!!AB;OeRnT|Dhs=z|@ ztlKpK21idcFYb%8RL9-U;#c&Wz`QH1|9VS`oKr4~=}?zJcXyf|ln3I|^i0hCRYB#z zsDr3N+*asNHEYu@_e>8^P475Y)N{H(6K3w=u&@8o5WnJJfdoQtnZN4g3@tOYnTWqx zQlSq{WeQi}Plxv--UcyhA|2dfpE6#u&sO9g?;BiR8@(|4jk%<@cjs9V z2hzS+m>G-d(a@YVN!BS*rB#9QskFKC8PRq}5Rd4wi2GRP@u7AXDZNQ|`OABuTK6ZX zEO>CR{BR|Z_i$m6gTwh{#u005`9%6RC(@Fv%TfQNBeXZxMV6|$R*uQ>jNb>`w60Nj zFZAU!E_m>f%Co1twaD!X^3Y#fVH4n}pExcQHD5*Y_obbAV*ZIVBg|8}_lw;HTtMAWkjCeg{K>hH%$Pi`Xgpxe8ghoYuwf;9l#$QK9BP==F-PRPG|Q zGd+Wz>?tE6(&y2>VWU*Q^(A~WS08Rtw7z*>-nln|K9tim?cield6S)d(XPC#V!>w= zt6y=9V*`FNtO@*jIrYjnNc0|Bgirb9k`JfO=Za%C^N!L5Bz{Nk$$y&rUi(OXoxkw2 z3$3)rsWx?3tSJRSohY8q^dHEu0QZ$yQ z{cT1G!@I~8(Aw>O&cWD7s=872k1) z&#o^kXJe1)`(BUCyogRq`-?jp1F76%DZb<^XRZ1;gz``JSBJkGqQhTYWuJu>N~rRc zf9nyJueXZowUIZ(&52%odhvcWt8Qc9SB|!OmDG>8r@q`}1V#Req53b*iBC=aEu*jxz!97I%Km71{!-gTW%_pJ z+>XszO)sg_f|hXNhr)99iE(;Z;(d-t&!N@ywS3Gcov|)@YSG!e!D+wvxZg)zdb~hK zZ7-p5Z8Nw0C7~&w|F@mX`r%!Cysvk2BED0#F_23&{OE31dl-iYxry`b?%pmdiy%6_W|8yy+%v)&;LT?j{zsM;zm|{&sYo2IW~jtydcV$=A+HLWp1dg z1Kplcpqn4}!x>{c)?DJsyVq0TnHl<3^TyJy_EWCg1b7<9UHQD8yl_Y7*6& zFrRAovkiQq!WGK%wt+HpbNC$Ix%;Wa-2L?>7m87dF?FeGq#I?QdYmqPO;(%Sr|Qr? zb0n@o$XmT=Kov&Zs$FweQ?6_udGwLmTDe*IK-c}0^3F|ToL2mt{70Np`SZ>a2i6sm zCfCSjH>EAWGamu@6R2TJCSqjnvMwQn18ybH3BHPRi zprM-&i!R=wj5(#!-p$1?@4OQ0D-cgKqDMa2e$!S9;#z$QJf((au9CkGuVBOo-M=}N zYkr=tmp;Fx@>@@F;n)4FWBZ55)fI9hf2G8~sRCC+pHDdIAmdA9`AdILav=s)0#)Pz50?{ce=#lMc3)7&q8ZfPHy<~ zTQ&OOE}f$N>7pfyKDjN|`-b8B)0-~yET6)b4hgBeble*Ged(1fjYPcC>buqWic=2x z!n%Z?yIv-vugL{&QsWEPxkINRy5F1TI=5pI&G>R(jIO+%F%Q;}jc(A;I_E5Ta4z%n zCu`LZ`=1IiM$kZ(Z}Ung`l~+oj1wh3q0e|{8*Ta6POtfwMir<0r<*>hB5M`Cfc4%d z>t4=Ir5aAqw{Rxt-fxSvi6^(>i|Jk1Vru&d+SuGlcde9+B5% z@|_;lxVrO5cFxmO9N9Hdwks4Y=6+g9AF)r#h!>l9eXyfEyX8CKXE^U@Dvj#6gEkB< zB(bqAL9cP`Fn;W5tMx-C?G-UWV~unkFjiX1tqr{NzcD)W z>wC36cs*r*=FE}Hl6hh_2mPjQc5X9h7t1|0^zoysp_z-wX-i>gzqAC+jtk|S6}Qo= zgS+B03i?aQSB-Ab zq;CzW+v1f9o{*&nE1xbl8Y5?X=}%5Hi)`@jA#zc9hv1#*+J)$#>mO^tu=4SU4>qx% z-DLs4?ro#tk$RK+cf?6g#A|=PSJ|C!wvHE#WSIWsR#-qEsl$XY%jn9E`fkUT@_S=P z9la`>thDkhH}bEkDQ{Hxg4{ky3CSc#z7RasV>~&%r4V$== zZiS9#%&G1-s|R~bEvBt@MWo)8q>O(XCfU7JGoF=9Tm*kV%`cLNa?WAc7dUh~4@$$k zH~X5AOe+=9j;Jd1xVcTJvO^%}pZuKn^(`jDk`_tiGaY#1 zsrAdB;%i8(f|e2HjaSW?MmIKhp{j4*sh8zTXlRpO5?)@9JYJOZTU!dZ(nC1y&n1Oe z6laS)BHvOw7_ma0S0D0UocRSVpoEYC(&zMTONp07w8ghSzu9%#I$`Qvif_7+gF3m% z2U}OtraGHc&l(|S%yN<^&IOw`8|UHF((nqyC?ja}AwU zqt@gUCPtt&yz<=;S!dZcHoS_FDQFp{g z3;0vH@BPmjv#6ZVBMej za^}oZ+Nob%Xx29o6cR#SHg|>H!&-XU=)JV$bS*Y4l^VI5obx0y=AUQmswNS0;3roG zN3-$2d|f4DK1H7vcd6#*UnE@82!3ZcS9F>eXktV3&$EWU4d0=`S>Uhdyt~0wo-nnq z@f*v}d1Dpwi0rB?wVwLEo{`V0RQO!W_zguUVQFr#NE$=y$fZ|7eewKJ27k%uiuSt1 z^zFu5wbPd6;;qdk3u4{y3fF1f3jTIi^!k^Fz?E|D{#XibIh%I%f8fz*{9k&|A6zndjAb0r&nb@OeMYmH%!RNfX#slN^YR2n8-Zi(l{u)wWLnFA+_*v5U z%%ALMC}PAqHmta1+5)cCxjIix_F{O2irV*G_YAPfs>6cT09yY zR$p(eH0&V25z2GOlq{Atyz%~bF>&c}9u18Gm;B}TaShbNx%X(>i{VtP=^kG0cM!3i zjVm7tpeK%bC456axn7+XOc-QETnRbsB6Yc6gMW{6HEV^~4l&#yU-e5WBgb-bm{$++{5anJXe3`vYwmIAYF+_< z7SMjafAc$~Mpl>GYPAv=zX#?*M2v7y!L*IR3}BTa@(128MUBG7`+}owRKGV#G-**g zK6RK?tFX0*mFa4X*U2pALOmid2KD>?M(FQMimTS19b%3Xn2e4FEXBMY<)d|L%Xw#V z>N5AcQ|KEzIj!9^J$K{{OWDKcsP%6rs&d{Fb<{A`smvsLIr=@#I^dz1(>~nn$;m^|M%D;;I-5n-xtrk5!_El}{_f z{^J_tl_sx|U()%`A#z*dt&yMf@7)r@d{g*YTdMH%5gTn9ZtKGS`?fGR1MIq8fL#Sw8gjg9QjNETFHd&F#?yjiT;!h1!%m9}f*v z4R0QbgQN1H&xATpoo<DjH)@Ck<^NP>kov2X%^P-r(9!1(L>Ly)_Qr~M6EIIObCG-WT z@@_{q+5qn3oU3f~qp=m}$F?Z&jI*vd<%fA@`n)Mrxz2Acq3uSa=zZ?d##^QP-XElP zY_mM#bTO)9RSp;yX+b^5&?~*;W=%fQTXAOb&+10z0ioP1biPGS3*uipcCRdl&$m-x zYWlM{2X}tHjB8ij81J$Z?@gcvH+={&twJ42%Wss<$`j+2;~FfqV&g(HHih^V_^-Y` z!&#!oAjH@@%Jb?#RkF-(+LU+*by`oKY6`)cuElRnYWFvY&{xKkR9e7RAZJz(QfQPw(@CyZgZu$ zzMSU}CDqI?n&s_mJe|SX8uhEGEqIbBAq_W!n`mJCRmAaiGQAV$wcW((_2)#H)lPcb z#(Ip{;p$E+O|GF9Y0IUa$7lKNmzz3jV6|=ILB>4Ni*}K! zRLzpoHN^?F^KW5%+0+N8zB|fN+cHdzNBA3o`i}~ot1qGVrYEFKFJ!^+ye8+A>1Ui% ztrsbYfJ-1g%dc2BC0Apm!00dRcd%Ajht4WUM~pG zVwG<(^VPmRXKD0!ypIGfmlem!Z4YcDV%5|@0`q<5nC4SCjV)J(b*ZD1H32pTIZt z{`V?{x`a_%3UD-ydk<#1b4WmEwB^}n-g)FFCI8oq;r-mNUU}?Wa#5fk#HGEHOw9|P zR>q^O;5lVj5A{P9KlbfYi!QHC!F?yIF5Sn`iaB$IsmZ#}*~fJUmXtsLv&A#iu^7#) zU1tiTwnBf#F{>w$-sGkSdRwA>Z4Ze!L2u=Z@=LFy(5oQ-U$;%~33Go)xo_QUef1@X z!42}k`!igB`)FbK%J6Ia@~(zAQCo%>4OOT!#qt~BV#A*qj9#i}zvC~Tn75UN9SfvJ zd$SoGpr+O5GxCSKpTs^#L+@a$9TXVU@>CoWk-usgF5!y*VolE#>*7mue~(lqr;Jyk zr%J}VOrK)$2WT1QamMKr?5G3#q=3W{Rn@&tI!u^%qQOCMN7758W?{_7Jm+J5=ZEt3bU@Ax78u^rUZJP zG?A?guI5tP`cSiWwPjFn3>-QL%dE^03^ z{K!Jp`{W{VajqwgLf?5tHSFIPoKEI>+%|Vr4-c!$IDB!d3YiH6lW@z z?b=4a6{~2X#Yy9Oj5%RvyTN+#%2vAhvlon$M(LPeIsLWk2{LnWd*3P5)89cJEP0+c zm0cr#hLn}K7qz{=L&r{3V(?veY3sL*l782u{q^s&L+`vYtYHW3edYr>1sv1Kt*dgI zHMR65$77V4GKsvZdFhBFLA?F6ag$y}OY;R$5c&K_wLnDO*0E{(5 zWSq@=*SC&rpIn>^WEPT$ZT^z)48^xAAd5{r!DT$t#CpegnuL9JT;`sm{bL8J_Wx`- zPORa+%b%#4tD14&+#{&ujGwAuo=jzv_cq1Uo;S~cYTX28MUNz0~Bhi@_ahkRMk^DNd z+(OzIR9;QAL~t4Q)y|vxan!6#PEM@BpZ4D5-G?e@eBO(nI|-}{-D%W}uJ$ZJb7LK~ z(`z9bp6f4>zsfhDAqB1tvJ`I7M`w2Yph}dER#*PudnHrWNXJ_v)Yy5)p*v-{pWQ%q za&9bE7oDSS?0H3RTuR9~_bZE~2O7%x?Ut#lOa4&hsrWwgpbO;Qs8yCW<(*uG^Cgv& zSR-Bg%|UWlHeR;fTA71;MDVaHrM2tul{#?7WQwv5(Q#8krB}7n^t@Uk4H$m~8a9DD z#Qn73_qBe0hd1@K-UtwBmDvtLmmTGycJX5E?ScMSsm;Zc+acRVcKFObKjq{oa;c%5md!Zdz_R z7sdMnT^&wQu2iKuKFX%677e6%qh{-W&7N9M`G;t0@452iGiTrzRTX1tTaiWp77zC^oeoEPabbGj0AMsDgE%7{lCk&2J2>U$D)sRB^!>{Ea`oDL`QUECxMcQ0@CbOz z*|URjuN90qH8COA&h_GoWunCI_shxTU!6bB5?aAwKiX<+JS7qxn(4Ryg8A*^e|$0| z0y^j?Kdp;breIDAOQWgv|63?`ma0P`cK59XKj!2&@76+3 zn$Y}(!NOtfV%c7-<@Ylt$euOB7#c-YbH8T%zKLy)4l7K^W%eI#qs@H0FOr9GqBSQ4 z9ON_YvmqvT(yUf^Pintl;d{ioBekKP8a-6lZr#ClE){8Vxuw*2!3G5%w!)LO|Mr#Y z7~ZMsj`y;>dk4|P$x-y+?>hDkIO2il=0k%$waLR5`xVtIyH+6<#DUqj#Fu=nbxO9p z@YkO7@Yr99nl*@BYuV`cGd?QBq`;=>)XkYp{P4;Ozb#Gio^UeGA}KofoOl#If?RLR z$nw<+m#fLAdDA#(*=V|Sw*eo$Ur`o))l%0Q+QzUDf`h5|OvOczK|JhWI-t0q&oA@@}U1+_d z2lRB0fEQ__ftls<8n0KOReVLk+6!dC+Ie{D*uiW#rSR;2a#FA5bk6QB&4_4i5nJ~0 zvlSiqV#_3|w5^OBndbq252zy}_lzbVzQ*u=uKuFCno-S@I*+?U$Pt>9wu*Eg)9WPKe_rk;l0}$N!iI4AAf{opc~pIpq4vk$Z4<#m^+} za-p5^Ingz;Dd$_Tk-!qVWDy%i{>0x+jaAmQIr)nxKGU7NNfmnbigwl5L9f>A6ypk5 zbh`ajg1!jb`@1AOR3I1Gc*e*)19(RPcWcCd^XW5tbA{G5v-G7&&cC$7%S!NK1)pN? zwRu(f!~=#oMaM^T7|g_TW7pH~!v$rlUsh%KEn}WlbgGsIBtraj$eo#SFHkaSm=6*9N==1N#Mz99=QrM7Oh(~P)55>_x&xR(~2*V%T?4dzs;h0 zmD_Bt34P)gv-@y1pT6ej4DI5O^b9sS-7I_@MNgk)Sd4r;FH@7SRl>ZZ*L(C7`sUoo z}VZ?vB4e4iG1dRyP;8e{b<8!A%s zCvr@Q4~EIOwug%x^YT1_jTv*D#mk2xy!6ZYbp?0~eBYIO9bRRP_>fmt>pT?ik6+Nn zlke5oCU6g3YQR$q@`TTI?k}+pv~HEm;)({YqZ#=te1F&Fx1yNt|Khr(^WM8`{1Y)i z^QM%fD<0bwSSBmKI#nK|9db>U7xigw;QB`R%<bNQ!$0Ebu34)aq@edka@iQkzvUp%AO&@P% za*0dzc>PdUX<`l62cI-0aJ*hW^bjXz92ck$Xrgh?L@XRnDy?ej%>p2yjJ;OaJ)_QNaQs2V@KMX_H@cq2kW54l2MZT-^inB{Q7 zVe7pbm9_tdsq#SHYa++Xjs!kY9j-3+==Ef}VS0u>T9EIwy4gz;E802zfd$-wy-I(Z z`ISo|8%xACC**5O0ZFA0JB{gm({VDjn}>dwRyb>&8=T6fkZ-DYwxt&QJq@nzDh(9$eQYUo(N?yA9amPB-C!yBczgs z))8`;{`+>15PM=(y-xz{$_bwBE#}(LbP+h?E2Bm=eF!jF8U5#E4tgGOl1~h=pnhD- z2hMjkeGQFVmX_6ShBcM-YLM`CkELcGvr~yR`vp8goF8}tEa*nVv(L4dI;)JQk8Tv) zmrxVa%J-}2VcXRTJ_Y{2LGWS==2zD(94Ug!;0!Bar&FdDlqTm;7w9VO7noXH&zrJV z<=%1t{O?Do@x-oSyYRe|+0{FxhZz~9wcau z2D{R_ZbC<=t%!^0_nB|FK2t^qEBS8_0ZAJ?mSiL{n3*tE@i1*US-cus8n<9TQwucPETyeMOyq^YTlzZrg++;I}>vCe90r4dXuYJlr2SmWa+ z3osMC{bwU^FDgCmF7JzeB#O5>YQb0)>I0l1)rfnPaf%PjH6g3!Fx+E2>T>mcwC3#4 zth(pZ-tO2t%SDfzHN^Vr(r(fJMK|mHLr#Xf-FlRXU@&A>orIp0KyJyS_qWie${EVk z2!?%4kKp*6RMob0_5aay-ElR4|37I;Mp9BpMnq+l>Ymq$Qp(;Wdv8AW9xWmw6-uRK z79phWc^%0tD>AY|_RJtG@KdyCUm8Q)Rb4(fMV~*-Jum*o{ngTX)bQSnEcpdP~I$22fHlZ51(V9pj-nD%3eOkehovZF&l7!2fTMV567 z{%Kn^SkrtA`Ccp_ynDwA@p^0a;wJmQz?{=$zbUPzWp)@U! zPJ<7V6Wj(4x`>C0XR+7gy^(Y%>B?4BT9xul(nGZ*VKKW1>qzt3GR0+&9;KmL(~%n| z!WGw^oN$&A2cV4+o&CFV42G}1j-;c>d#-;6z+4oLwxHas=nY>X(W)w5yTaY_)tSR zp`bhh2|I!O6dWopLv_8XHt$coc|hCFw4>*(eG~YslC`{lTZ}ee%X%RNq4J zkLOyYJ%eXdVo_L+AOiaVj{Mp!jXu43>sd(KvjP}r!7ZK*tK8bFp*`8CLNb7|m-+nXdFNt?2 zK&96tB+smPUPD}o6-~>LybhRG2kBKi84tc2!oRdl#v2Eop!Yg!C>gm=TQbs~&-^=$ z7e0x=$ptOB&yacO6j7nQvM>R=AHN4_$*YCeq*XYrCY`BrIMn0kWctiK#F3AGAz0$K zTtUAF?QLtz)q~$L`XfGR)4-=P8djKD%k@^CoSi@nJ9$g{B6J+sKRwM>zAA^jQ$^@E z(?AX?+l}k3Xz$UFp>RyleMIGxS%+Q%7x_1cRzCGUJ z<_i^Y)o&4coX=-dy4uO#1Bat$iX*oF{$9Ja`E!_QzY!+h%Y;R!kzE59vhSbizKDYZ zX`kaWx_>GIw`jkC^})rs>hTv?{&6M${&|it57K}z-iU+Wrm!d78}P4tcc?kz=G77D z^|0FENi{1?5Iz7;?5m zW9u5kg6D_IfY!62%>RTe^LK@2%LmD8ucI{fV>*k4rzIxSXSl8<_qAcJ17Y8~5BTJ= zm29_)P7G^n%X8@L`q34=A#&M1ly{!P=w-d+~S9nYdu zW&*oqIS_YuYlK~CZ{PUIT*llp1x;rk_*(5BraW{6o8Hkmzk+qJOKfxgxaD%pJ2e0* zJ9dQ|clG&RPfIi?sYF^2Iq-`;=-1q%N&h+mMr?YZt&>;?i#uhB-?vA|l)Gt|xG)b) zAL^i*N4wR<_)Ki1I;kfEFBssy4v}($aV9=DcuvnbKPR2CcH%5+J?0$p6U?HzNep<2 zf4dgad>V3XbPVXU<=EKQln+0pBOmuqVuh6B-LBmm4fJ;6RA$5JH8oUUJSFfT1YR=c z#&hfP7p+&rvgAF$+qla7NGCppcvWv}Te*2&Q*3lO3NR#G7_N-xG><5YN}*BmM?vp_ zv)@0$Sxwf0#m^3!3k^E)q3yqm)m|F z9&j+C@h{@r8a%+EBP@aT*z%ki3;2Z5CvjuQAvV(VepM|lJa-!Yxn02P4~<~p4|?Xy z?pAX9)_Z_!BjCU1ciGQ6CenIv9JJq6r>Z834gIuDxFK|yJ%!h$^L72EhDfSWp8Vtp z7~cCU3fh(yPgoQO+>u$zUw?jTI?(9r)y{ttHhabZB4%s(C-gUC`l@lXP-vtmXEZ7&CN(>|=Hs>ymdcdAgS$Uvdtw)|(_L=CQ?m~u_tFC(%)x%4wBM&Zk(|(Rd4a>ubO`WLLYEh3? zle-g_OXw&RHqhBz&*6D{18!E&U+CH%$Jl|!oca}lFF#{Z9};lotHqMmSM?E7*mun9 zq^vP35>GADl?H2U(e|V+UzXg0ceXyj;%W|nB~I;FM{@(sGP`|X>sn2=9+ah}^}>4V zx`=s0`iK|lL0BB-E47=V#K}}!@ZOlDrM?HF8$W<>7uFo^BdPvi+=o28yCN53`>ug4 zFSfE}jhpkkD>}o<4Hja?p(widE<$#gHysO~*OJtmeAL>(+Vb^u4nnVN=$&v8bGB_` zT?2n0^$9<5>>${W$yKk-rt8g-ey5*mAGiFg-D*1)8$Pxbv%Wn?95EFRwA6>eQ<62* z!_dK|CZ8%3;kZY8h4qZ;5?#hWhR^N?z~-hUe_Ut-_ZH;94xR4&(8MC_=}-iw&Lz0~ zBRvOT$#Jx;@ff;&Tq~$gaQ^k~7*JH38-Low32W(VV?=oBc{sVRnV{=KhknPPS8@-2 zqJ0}_JU5CR?cGJToIvOK|82kt`?%_PtBEaSh}T1$uJ;)Tw_$5R5Zp`&#LImvaKLXz z88E1>%t~*`>9xdpTNJP650SXmMDEqZz<$H`>iuPpBSUaicr-J09?7r1H|7)Q4)liS+Y?<6ZQvKqPM-70+$%k7#1LDhU z-z(E`^XHxD_o^E4*G9rNQ}7w^30suWJ?8O;wF+;kHjp^3s=www9KZw7-GcB*Bii$Hk;M_FqhJiY|xBmKy?qTMj)!WN}uxpb_DkT z-vu>gxBMx1#-%5gj7@_Yd2iuz-xWX@$A?|pfJRs6!GPt?GG|9aLD!J8yIUftzP%N9 z3-u#ga@&)^;!I`%+ooZCC#u}fEz)1r$*vvg9vj4v$ICD<~dw-`hSA8FFXNxwnq7|m?4ndDRUnJh-Ki*8o zM-ON(=(|6l`t|y#^;PSBE4IG}^%VH_nF;%yBDC`c^hD}=+~&6$vs&*F#6x^{N+pJ; z`eL!+MUR__#X$WktS0V3g@4oB;&9)_|1|k^jsW2nryAn3l7tQ8_q-G6r7c)4?XwygY=L0a8OxuxU;b*mR;z` z_8-|m^}P~^-&h-;G>w`o!a+W7z-5+1(H`NXxYBbe`)0ch2>%J6eyU!8C^|D`R7erq z8}$|4{d$0NhLwyittWl^d|+F9hM>|a^ZWSI*@wM=deh@c_wD$+{TlqXs{w+uI}Y!n z3-$lF;>{-akT8({8afI-`gEvTYZ^BjUjK=PYL3BC52kDDg3oJAR&$F=o8(7kP~Tr; zNuL0y-spZte`tHAm3UOV3x|wph@Uf?%bMfDu-&R}_`)$nz8~ZXXEQFq*NWc|zAs-q zeB~rWL@k&zu9)uet>@v~&RU*pbq2;w?;?gZeZyihrh62uZ^OYkn_Z;O$(%Fd*yH|< z;ck$(^n0fVF}G`Qw;TO=(t8h3{qfkLQqa7h$8(mwGm$dKK;mrroc38N>{xkGlZ0#Tw;{| zAi7V$obPl9+oHU64u2Hl?Dcd&3od0 z0bLI?F?+$rEb6W>NHlnN0DBy~2@l@2r_T*cCGj`XdJwnKxy}9QUiYnrQjM!yw^yn+ zVD8;Q^qN#jc+g!cj3iAgJ8lVuos(SQ#t=OgJ=#oCoe>|ll%KQ?ng=sGl0NH;iZiL_ z;quZZNO&iYdmI%39}{54G(W|5l^Nj3#C3ldsV#jKRjZm+LiLYsm;k%7FDDBkt)E-=(U5i!^0nVj$rH>xV;f%{=xY~8T)~0to z-fr1vR3QAbA3_()y%<;LHnW_Q0r5sO*KSs_()l#th&MoMNnT_= zS{KKIx~9b)IzrSND`|b?5B1;~@HUKxiMJ0^@0s!|bFHMppx7I0N%z)O-$Um#3wS-F zpP;#i4H2~0^-eC7mpsMo=6AvBSu+`RW+}~C3`hMMY{2hB_}RTRZnZ6i0LL#L*PSOS zeT-XPSjc#zLZ*`@Fg9(9aKCy@>u~KTYoBB)$m;-cAiHr(S5oaDX<%l&Fh+E(ZVB6L z9^uzkiNFtZgfAH>q9U(1)wnyV_t#CzMDmfS*5uB{%Q%+q^(EW~#lzHVNSFo6Z+y<$ z2gIRtEe&OLdZq`xmY}-D-93^yaX6#?7t~W&IBp|TnswcdT%1sHLXc(v`Wbv;w?yuo zmJOTdrQju<=A5{cQ_XSu9?%#FGdHroe#F_nkKxEoA4&ZPr?CMB?q09ZpRyUX`jr|M=a{hZ+KmH z111hVio_K_btMgTY*5ue(DL<~8wbWzc|jUG+xa?+xVsRukDP z`BKhj7kV}f@l7wgD}wYs7?sBY&68$bqdYY}_;1pE^3r$3gyF}bbE1PZk*`>J=?lCN z?Qu63S6Ut6p2 zn)~Bu+?%}xjFxU=qyZUeU;d*(6ua^*o^+`JS}Zs!NDnA|uRN#1E)Ugbt7^n*3Ads5 z_H&FpfgpVfgjwW?_4$j?r-H5@8cf|UNQ2w7&1 zAHBum>gUk%V_PH~7Ua9wao3ET*c*mwe2jE7;c<7-K4~MqSUnogrJZHwL0X{qlH)#j zX@*^^38ejy{3*Qpa*OzQJ}Q4t90n0}-NCG0K2Q${(s4NDe5#;zK*g<8yGXqTtN)pB zdTpM0k+C5je~8O!^6Ade+I5Y*$Oo?j@>7Di3WILGQU6EsH%iAqnSjIOwf-h`*Y0m0?UfVJWCiu}@5% zs&lFI!}|a)Bp!lm)9L|n7PCHBE+`jLc?5H~l%d^=u^ypfD12)k57b{`eSLv%pXM{A z<;b&XD1*Y%v(eb;UNIxQqw&NuH4iGgpm_lD0_ajo_d;65DUHO5R}`L)VM=cbnlGT-1*u7 zk69P`T)VX-93)-Ri?}FO%`=;FzM0BsAhk~_>)fPmRgN<02Awt5eY3WS&_}f{@Y2Q$ zwFmYaDE}^%Ug+sCfV|ozAgmRg6Vq@_`+h(kNJ}^hIrX=~&+)G?I_ooj>~feH811Uc zH*)>&qlk!v*jBHAd@GRt$H6Zf%XmN?PbIUE0D z=URC~T}#r~?{g@(ST2V3iXflV5Giv|nncphCXZkD9wTWcykpV>C?f;%SgNkD>{1qG ziQ5@vf?Q$Gvc|iyjgJG8#?}V<=qt|x3f~DI1Z6ow`7O#)fbtGdI8NLKHxq|)oBiME z-tjWZI@(p?Zs|q3_q@FU$ZCc#zBHT7N({pz{kvk{TAX^jE6g}l7pRt)qt;Fmo(jt1 zfPRLNXM_SH2TqtHNT(BD(>Y6vZ!pRm;YDj5+2UI+nIz6n<&f3@MYWntSrdijVQs zFCADNS_j))NQYm;$CB6B4ne<-Froa6mikL+2)Y)Dr13EK(r|1!;f&JkVA6Ri+^%=M zN^21|Y3!eNfhm4_kn&`(DKzJl%?QFqMwrh@PiU$B%m}kU;gp&q&y+*r-J|oAqXfahnE@&v2Bi^{UNi0Z1P=Te z$4FahR8B;i6S~CN!=mhy@Z8lHsXnO>>Tt`M>7q%@B=GZ|z*Mfh>()N@!@L&!($ken z9}$-j9+eYc1t~qH^cnN$dJ(7>uzsaGQoU8_kN>8XV7pfJNlRLBwGQ-eNqMxU#`~HO z{38tNuW;ikXNygvXHkuy$3DkDB>z&aDtGx|9*MW3uEE)M{*p94VZcV9jE=JGpH*BJ z=x?MADB6VXJ$s|^H#1E|-c`!zcaUzBDmRr14{U2aWrqh}sJbrlChUC?$k}naRbGQQ zhI@}&4wOx47J34fuc@x3-_brbn+bP6?8J$4D`BT|dv2NpxTA&xL?l1QtGlm>{@F>~ z<=Gv`<)cBj|6MTMx)mp__=C$$Or+O<_w3?-*~0xzuq>m`v)csMlaJh|v*D(#`4aP4 zylIzotaP%+r8WI|%HZ$f+F7tL?f7k%Pa zv+nBxgon{6X7Mlz7Ct)!C_`v;DmjA84R+RAIkM#>xK zy?H7Rk%R64zn)<#=>RynIWGWKx;}-{j@Hoe)s6Z+QWJyz^wvyM;ZjWw&d@rGigCmG5k7wK~tZ(CkOdrlX8{gty> z`trbkabUE<9~SFBW_{%>d8b;XrZ6>1#+kgrBm4Cwt&^Ct+*4gYmz{pV^rYEx$R|I- zfWg4qFXn0uX>A0}m)v2n4P1*GVP???+?Y!D2ers$q21`dot%k$!F3bqoVphO7~9JG zCY~reErfqf8py|OJHUYrWBHpUTT!iDv#!;+`k6?277o<|F4L(kubuoPcGQwkJ7o*3 zbOikN;ScT^?}w{hTHv&Zt6~v-&iT~afoqKm@Mx}9o^h`YJKNBn?I~umVA*lpJ)O?y z*lEvbK4s4hPV&R1NcmKwCmq)86>CT6$}Y@}WnRj{!qmEqYCyi8)lgRR?w&()uNl$m zv-|NIO(fMEZkFd&y};T)!*t7k0>o_kde3!Tpt`km;6XtheYgu+D8t2G>)I|vljJcFXTG*)c_ivWgE78a{3tz zUvrPe-fzhp=+g5hHh#x|uZ`fh)eL#MR~Z}Y)tL`{?;_nEt$}x`^z6}n@fiDv&IGus zi$~91g&r%K$N@JesWo9#BZ6v<>vqr7KCETV%W_`eflwv`f3M-ct=_}Vjz0Y6bx%xa zIvTDW?twXFI=t@*+M_$gtg5fh4RI9B`f9MGMR)M-b_~>5I=l4YgbPCTW|Nl5Y-!DW zcs6Xb)K+UI(-sHg_j?K0Cn*}Xzc+zlA8Sy}zs#X}&@9#;jT8Dt5w5JlQO%m-?m^ki zc=I$q+VhPVa&Jq{)c^ER%{$=;?&@8LR}=Q5m)mZk)9oR?-F%twhW0)!H&JVZGxh|i zIg>T_K4tw5Yyy)Vli&?Oan{dgVCSwc%Rid)E9(q+Mv4stHd;V<@*VnTHx`lGf)rLk z=*Z6eTDlwVU(i<0xo`!kKKS->A35OSA^!V!9eHVeS8?wzeU2fPAk8!C_1jT39_npP ztpnNAZ}~LWy=mS(VCx5eb~ArFgbW$N{#onGaf8OfFtcBx<=7?|XLw87Y+!GZwz`#! zE_9Hurx{B9(P?=5W{l>}+X8qMyafL{yp633zQBxvhV#ERXS8KWjbKAeI;>qb5|-C) zjOO=*>KnZG%}5d!X%5UZfDdUy zRnTjzHNw-5^>AD-fxZFT<=+N}HE)(!U}L9l+<$3z^zLy1n)y+l;qX#8iC?1C+Gj|$ z2`159cw*gXm{(wjSGUK?;_v50LjApvd90c&%5#>R(weFF!TQhLc;=mtc;bl@@44>> zyc%u7sm8edd@s5;Ym>YboX9q176`YBv8>&xzWn)w2W;?*%WU!-Z^-`9lB>D_i*`Wx z$4s|`;90-M@FuK~samJkgf(O8^NstiilojJNd1_zP3Il5>XFp9V^p8P!dG38YE5CO zB!0r8yAxPk+fH5>djQu3chejd`>L?Vc=KD>p-UW-eG%#PvGt;Da?qs^pgzU6 zM=G=?n;hla?Q`Vbg>T?!u0}d0AI14khT_0nL$DkEwd!l-Kedv0hgUd1XG2LZ8BU6Z zJ8!#lg^h%zV)9fQPJIcV_88!}e%G)pem!ffsbm8#cL6nLZ(|+_`kwfDhn`1$AytfQ zum|7oxX7H2-Nc5*^qI-0BzRC63fE8F!?eY^a$3kfSZ)%*R=SSklSbL1;Vcid{CSVr zdU(UT&@HGq(DLIk{=jaY;w-o@5plJd38cPG|^^;b-;v%zp78PMk^4GtY)(?}12| zB23@MLIq6U(g{QQv_sqWHn5{c8ngOh z3^Y$L_x5n?^fnhhAFnU%hR=iW*}GwNmr+RXgW0-cU{srnNI#?aL(bZt0mOMZ(?aa! zSdTbp+V3Z9%IGAYkdGp~R<$6%m!4syL$nv^yz4tJ=}z7^Cm8V>U(qmLWdGQUiibWQ z33l(1uoUU*XcSfk=jnVO>S;K%Wg}Cye!FZ0s=E4e(M|C+u9+e{S`10W?+g_I&{zLlQ7}gh;qjCLu`j`D6|;TjDG(l+AqEZ#Jw;jd=Yy#?6JrE zTa9I4qwi>4p)ai~!nAa)a4jYnsdwPIwj1C2y^J`ZA)M_RLHIWwHnrl$~B4#8pkJ%cv{2kaUwEe3iG0_lX?_mc;QhWSh8r=CIxan);mb=zlx{i*k(7 z@?A2no#!S-cRCC8Ud)$PKWrekaVU;7-=g#ewwe+SXZoHK$)2adcaT0;dgJHobZJ&p z1lO}V!g%uLqyx1rr~2aiXnG#T_6V6-^pMTX(osJvW8TgMrK4h=UJ>(NSAb2;STqhl zM_6V~_2t4_tsM{abwSvOlV>{PzMvN@A}m21K7NW(zbYNWTTbjCe8-%U%_n^nq!rlu zsM~_(L%!d6A3L{-#l5r63epvt0cYc3;G?IYxZ-ejfB2>AmOUdf6|(Lc$xdYh&~TiN zWF{`0xEx5&O5!>oZNs;JYAoMw?g{l@_rTe?<8ef*IWRlfh${_49DoV+TWSWE>_Orx z8ULrDRJxkDMyT(f8#NG?>eFj^K7`FM1&wX`}v?kkGPH&H{iO3rjmFZDxLNTH9sexQC$tFBMCR)=D9HZc&M!)&IQX?8!>chIe2?+q&Zqa z7%&_voF~Bd+s15+QzIBqIFJ#hXUAL&#g**~wMw6o-i0Zv`@>y5%G`%rqBA|CX{5QC z!YsU*-vCH^qD{^EY<=vMD$Un=6$jEBxMBM#k$J5ElWaDj(UKE5aZ)!yTAEGYT7a$( z+5wFT&J3Lf#lP+%)i#@*91iKxJtQ_81ibYC=n-=ZqK?+Yraqpa>OAY=4siQ*4eN?V zGNn^In%!RvG%u*BcOS_k0P!4<{=vdC@3l(rke(J6f8N9Kd*^X@{X!u91;*h%+Bc zviP5dK=a56Co%B1f%t2C8{H~OVbA11S~tBa94XkkT#&ZY46h6P+a*((^CcQG_BgZG zv*OX^)f=GtB270D_l;Yx{18vJk5I2I2s63r3pKB?cj^J@68urVNX$4e9jHc`M|oXI z_ydGTNWM;M`O1_jEKs#U9!=E;;c9bE+8hW^fcytEcDg|Qrdy@SNiX5{@^f@%KqA=K zj|AdGVAX5EiiQnIzeUjZeE8}1Ey4UoEaBxWg-68C?Um<(p|6`N{eE&}gBEg(Q=(A* zhx8*0_?oDpdSp>2=pGRF>9DHx0m6eeFvg*VeD1qiOZdkA7$+-TMzvlO==#N?AAMOO zWvQ-TUNGeg$9D;4v-UgTY;hePHtY?=cJlynUmRaf6jy;YqMR-9E@9I0R6_wUX4BESwRJN1@R!hZ^b zxxMhn!R<&GFG$mD9D%uZ+mvRYzb(;ki&E#G6;rHRQF#?p|iHiJd9VyL!TduuC)# zq~}w1ZYmUpe<>^h;!H{YUZea_LZd5S|6c|Sxo5_y$CY*^O?es!Gmtb3I3~A8r4wka zk+@Hgc7uvK#u(bI2D*Q!uRMoP8tK@XI)tf%v_lOm6dwZBCjVv71FY)4z+VQJ6sA!v z#=^;m?d5t4kNr+FDQGU9L*Vt zN2|P4+aoK0_?{_VIJDIk$8>)IQMZ1J!KK;s4DWA(@LiHtMf$(CHQhI1@uWGNpGnU- zbkBxr-UXy<<$!0o~yQRhfge*8A=K~2o0=d*kP<=fR5$MUwcZ_}Op za!0Oy=Xd>k7&>-t(!J%HPHrm`5kSH6Q4Z2c?xA2-n`O;0t0ihIb55ym&H z@_3|~mG8kDS6?IPQy^>wgSl6#IFsg$x!$b7A6jS>1|roGf9knSX-n9GM$&Xkb$LBw zBXk`$7$3O=C_Teejp+3<;#AW(DR3iJ_u7EeV`5L=@=H9qUTH)rsRo*v2*niUOT#9=I`XCV3cgFty77JtqpK8i$oALi|y ztfd?aW~|?=#wExjRCxu;7BqUj!m&TLXDSOMe1Y~EA(UH0pwhzYRvr^ACuRzjRedcw z2HJ|+(liNC)oY(!Ejj5*@u%(uXg==Y$(gY=d54EHl}^=?#sR7U=#lUnJe#(ail0k+ z^dk*=SmhJKZ*&l&906!t@a-BsQ+(lR4ds!Hd_T5Xxf&1dx(Brm^~AK*`$_*4fx2du zrI2S6euI3vOcwEMB8{MfZzplN2ef;)oJM=7zuiaD_dsg^l%)Y-rl@zuKwLZd zADj#90#DqW_}-zBf_w|3-xKMZ-0*k(VQ}klZ`A{ucN^T9%Ap2bs*bmwJs}TdqOvO- z@4sJAeJS5f8B1+xJg2s#oLqEnRX=BUzdhv7w^8o*k}!HXd(!n4d9beJC3d0G9;aG5 zaLU4D9@ufJUs|Utpwq@2J|QFg?*ZY{qr?oIbbJ(H%^^y|-}jurcxFrS@JUjmq%S|&zC~eZ*C7dX zb&TPq?kqU}uMOUDvc(~tEqJp3OZIW}8`h#*s?al!Qin8h&Hhi)zI;AFw#S?tc zBDp>9mFFZ&joct@`%>A{?I4EE$k3#wr$M~!2yj}^la*An!j!24Fvs&fTk(B@%nH!( z@i%nE)hUtq?&EPdQu+x!7rkfqk_O3wj)`Cx)tY~7z85$DYsj>t16W>x7S{KQgqiCP zf%T6_*mlI8jjwqD$3OZ4f9S0Fm!o1Z;?8C?zxa`5IZT!hYw1YuIq`H|jisw;HL0`E zKwi}_d=T|PQ`7De+?hI_KkHE&ZZSiy#^rbG6{N6An73#J7WO%fZ==It^RD?YH*^C3 z5osjVwdfX|0AZPd4hcHA^KoD4_%0utb`Fy?Zt=VAW)`ELBWTWK)*4ea-{5@XH@?{G zDO+CO19Qfg!-L!|++)QFc(qDLdYJFTdd;@t0k~YbJIcSs$rJfaV^%9a{!^O;@st18wAma3@LYM0<{>$Q2u|Vb;e~e6rF7Cf9o| ze8YBvU)*l+p3_iv?KDP)#0SxN$KK3g)*%+4H-fe9?Zq4W7Gs7@x;!=M2Tq<{gR3>$ zHDWsU8{J8&dAw9UPC5o>qVdfjS!7So82mjJs?mlq|L-@kVAdE`CoB?P)V_^a<076q zVjWjnsZAWZIDm;*Fk zc{GmBid~wo)(tI;^Wp5v!S7hiIztbJ}%n z>^RLA2IX~zOl=3O)u~X>m~nz*2aM^sL6i8a5!*xO{`lCeQ*{j2&T$#_xuf(gJHZ#t zNr#h95%vzTk~6b);OZ3Ge{tpz^fVqOy^Zu_)6V)lv%d}66&T90_-62`*?nRBQNz)}b1wKE#uC>d4CSd2+OVE9P>_OH#kXoFU;@pU%ulnOcN(YexzHX7-S~ z_9<4%BckZ?gPhFo-@(N`fUlldN2BIcz4w!xMX42+gkRTl5o4Wa8*pDMrgSDW;-A(k`#6*o}L-OLmGX=jAQ1I`F(** z3bBQnImhtil9pKH@&)s|=&{9D@8g`@B=D?tNBr8gkv^Bcr5W*gmvCBeSuDS`SZ+@} z&GtR9#%sg(W!nrm1sz>FGgTX5KiWxSXdnYi8p*x6hhgDLGk)#MW{gSLm1A)&l+l{= z36+na*Wb^=iO$FQHD3$RYB#IgG!jbZT*XxH7^L6Bu3i_y)B8 znE$+l{?5g~ZZ4d9oE6M^g6zLOICP9IocLQqex6qb)K9YDk^!jx7=I%eECUS1zV4Zv z#sqUSPhvswLZo?AI0b5s2ye9UFSg??wpP~cmk6DX-p4^P_R^;LJG?Z=80!pW*sZo5 z_ec-Nie&+O{=7>9=tD*Ws^|)NoEL$AC7{>^Mp>OrI5KGT&az3}-{ZMN&ereMc z82@N4=5@FMJys3DyfM33Y|A}Bm>>#Yw1ySU+(Be)ZR1mg7rlG7hrAB`<*dH*w9B{wztl1dzqUDtLzWH$3paWuQF4uv>x38BldId z@;*jPw7o27jnOdbfQD*VwwzpBM4KgRG_Db_=wKN>zVL_rBE(^p+Vc29o zf8<^?i;D*J+He0_^RsVTO7n}eMUytGVO7nqp7^gu5d6X*%S1&JY&KN9j6JM2EYz zHM40CXv(%daBqBFiStS{zGbdedSpXEYu^7&Hq7!4ku-NOW>Xs6_Wy~Q8%D#yIjH!b z69@2UGg~08XMx?0qrRa%&V5n?lKm&a;y;J5Q^U{NU0-iOi;|ks6I$beZ?>XdGYk0F zF@sfKZ$dRTi@zLYj)b$AH25mpM*G$}Ufz#S4jS@LzGk>xlg2(}?1!ag7hr!iBkY;% zEvdH9M%;o2Lmc?ckfRvCT$k7TPS2uz^aW`xwUcQ-WjQ`a7ur`%^;OmT{o0RL>%r*j zFm2&}xaaB#U(YzQMfdY^+;kes*h~M!>?``bqf4&vSmCPmu`!l(9dhP~`=W7^0)(NYn`h?%T7YXYoy}*+_Z))3=JJZiQVHFqN z&l4Xr-rSOQ-0NyU7XKA+aqB(F8C~5vFcm(J*8JkXMpOzt2JLUzv=xo zAKGsu{)*KoeKR1X7FTsluPsUQ$>Qu?jP$AE7^yh)+~N0_Z*~+(6X2`Y7Q98bW}tNL z+ld{3cp7Lu)HO@u0udG7NG{jy#c#g9$le?pfsrllX-IF8Hc)*qNtAi6z$3SGV5E) zmiCzGq%+g|O@O$KTL!F!mvKXRowD7acw)-9R7Sl@95hE1-|O%e%HjYZT6o`?xZyUrp|_aBazUUG}F*YxSCNI4ln`Ux#6;xfd+V87!Xu=_}~kIQ<;< zc>SJ=r_Lm3Kf9;1`m4?hg#ikGss5l%%j&57kLq>On}V=|`h5y`z1jwk zFN^}C&@<5E&;bqgHtRUfj4y5-gv531LDWo`9UA9W(RJrSwf>|rf>3vA z6Zo|HwoqQdDAXIi-^mo+7H88QS6zjNNSaUKxR!X22kbZxq^k($Ht-fD8-RL6^a|Jx zZd2QV!V>Z<+O_V+d}Qoj&Fi^Uy(CC~2!&VY4$pvy&ii0@Qdh-IoOH5O`h>6?B9=}A z<%y!`ehI<>d8$oE&FYD?|D13&zK=6kT&ed`E2y?eyPC*W^z5x>JC37sK)W$rYxLnRQon+#6C>+`LUC22d1-Lv-7;0<bCL%42E{@4^D71KUI%?ok7MNL*t27Mkn|5>$ZX>FIfN@gsZ>UhVl!dV%c37vh<#C8JsTmMm=P8OSzg??X<_97~J!WApJ@jY&Puu z5DW?rsV|{CsHvRU)RJBN>_hmF$6Ccs2jWj~=yG42Gl^E5L1)(*$(0W_!mqRL28QfENO(qjs3Jb2!V>alV!7@W4UG{J>MelA4^IQ(rqX_#dI74BaF;X=;*#U| zzUOnMdZLF#7p%K@ABN?Z0Ijo@=7%zo{}}01QD{387HogRW`6COL;V87uV;bcxn6l0 zaPJD;$K2Qo6Y6~@jk$@H-hT#kZG!x$TvySD=01QE7qKb7HZTigf26F0Hy`u6%HL6b zqT*V7L@zt4JD*@2b7`i+5I8W>4Ngo={W_X6QW>wyC!^f@<8qF^Xr7dMwKm`|Cf%djo-rCXnh%P zOK0pgT#w{&py_^VuEs*+#wLZsAUf+8K5g9$`u%9a)gR(`jB`8-3;T6O;x+WwZWciS ztp(*K8l~%%4kKMD)cljr! zKd3xz)!gsjQ;G?ck&?cz!ZY$#ptLICY?Y29kAVvRlvdoE>&(gj!lsf8p!brl{^vmH ztQp(PIgJfd*s7du!Gk1u1cl*Mbr_wsgtDEdcxuu@Bt6W@&&s10t(5*nmBA?9rmu@- zrH?V(_m?JYv|p7U9zDxbE)6@vl;7PFc!_*v4PM#O4`1;8V7vA(t54^Ndgb^ac?so- zK-)aF>NNs`7GdPTgJRpew<5vIAF}eU!_M6`x$+3Y(g|o2HDRtH&byh)91i@4RO3il zi&PqhI9&O7o)nUd@0Yl#yoV7+Gs@@0Dxck=yg`I$wu9>R$$DrJGznkomWjLqO;z1d z4GQu$;%aJtApEV;WTYd3JPMp#vX>Dj<0s!Oj66MSIHM-aY`RTUKYvZDI70P0=@^X~ zhw?2d2Xwx%l#ynpK3f39-=KK=>W$rC_`5SGELFZ};KLgj_4ybQzo7b>&FU!S-6dt3 zNVyN~`PEk#$dqOxeIqG%lGVSaF!vGN6<)BeTh<85K1lm|sD5OQRyL?K5an4K@(?02 z@}u(fBDe7s)-mxmylj^T-uK%9Wp5taj_gx;9#YOh^I}5hl)b^{Wv{i?`3z{@QR(|E zhgT|>A}tXvX#AY=FhLoi$|fn(7^!rp_TCkHH5cT^_K`1$Ve028XNA)H*}k(xRx*Z6Ej$2nW%XgS4>&QQ=nW1 zNKdKp(tNBIswc_oRb|yf3$_F0Onj7CZD!#8mh^B6<%0P_afHfRR9@x!*pX3o&DO8& z4}=+2*)e4mFf8&B8!(Sy(dFSl{vX_Q;_=S5PN=f;?Kh$!y}l`IdpC{mc0DB4WEiNP z)>yYMLh@{^|7ts+Tuq!S$pxx+#nob(sTn6tDJIWeuRS%)fRkRAb_Ht{zbNmaGCE-~ z_Yo)#4qj4Ut(VFT@r}h-u_dsFcyf@=LO)$o8fDh!%3D$fBX*jm=1?XHG>2FMBN){x zBhSH(&XjCMqcAA@^bF>H{=za6H-L4oMo`pYGo(gF^QNOtuxYg&c*{4Vd7nw9nxcA4 zu3CE!t)iM~qqBU$mOi)M{L&mA4e5f~jYq)zm?f`lcL{Qbgn?0qDG=m6P^9*o3co5x zVA|K3yu^;q@a?ok`XmqK?w%FG_trCt z%IqmKc*7>-PUh=_tnoOXrSZvaEA@jmK$j_z?7iz-NQZtvUk9tGRJpBZE$IEYiL6-A z566D&go75hz=IhE^6~UVa@F||v~Cb319rH{-06s(hs<%*p}sWE6!3p~iZ9=OlO>(^ z7ZGj6o};^HW1Y8A09l*h2DeiyiW)6+P&$1xuu`b@$@E9{BRRAGTa6s6}=$NAY7*Q zdm~;Im`n3x^qeaD8YsT|LuJWpI-5C>-&yA*lb+X;uXBG0$KZ*0|4A<>{L~7q=JpXu zQJJu0ftTEBsVjTx-+<5c7E0G)_2u`A^<@04WLQ`G1=btWiH|x|DCqid`>J?(j8P#15NQ+ z=`?9Se5?HY&r?*`&6bzrYGBJZkHKz>7u0;IFCYI6mS+OJ)q3Ku%7?JH@E>*$uZ9Z; zILk`wdQ?A6Wy8Jq>E2ae9MMjP)u?$AKZIWv-{*IbeT7!4%){ttw9c&uC58FRWk>p7@eb-H~4&$&d=7=GZW$luJ> zdb)I-MbD6HYbwoS&Y*R#MA!iJuvJZul>SwaGkAz50jp%Qqeu_0Djb zTgcJ$WdrM&YQy5@z=7m+=vcH*`?-<1?DC_SWlYtP>bkO~CE@(HTLrBljPP{>7Xuej zZc$M~&`1Gc;OotYV zxpnCCHHX&VJi-NUeK6*a*I&UcGlL>z%RytUSHpS?OCSepp*3 z?=+Rgb!_CQE4#5`j1{ym{~t-$9gx%a##>Z0h!RRAN=AfI?|BYcWk$BFkWE&0#!D)48$Z9v0m2-w0l(^H{!2rvfI~H(^El4J3TpoLeu&7RT>zgU`1H zvA>Pq3mRWWoWWgM_;TuhTo_327`et^>`^0Jd*J~y3)?SL&4#_QaRIG?JkOG@b0zup35w@r>yg=b zBC0lz-S?U;N~C?$f7n*x4%Ht2y04?C)^`PuZ+)9tHq3;EKLvj8HXhV>ss2b#^rz>1 z@^I%gUCl7@N>S!dJLpRLOF-H)rP`#EKNwR)(GT2IGkHZ2o}5IL@&v*WL7ljdU8v z2CT93GTC>R5f_TyFMf+GUlVSd?au6TKe0sK4xQIGX6JXlK)<&C!JfHG(Ei(244C$c z@S;@Q89oNb42+kv3k_ss#Q=pPxN}D|rdsNP`Rmj8eY__uEgplXy04WR(k|lE4>xdg zu)PR+0CHI7J5bm_mA1OHZ_Vwd1JN>l5wvf*M$qwr<`eci*Qe(< z>+#@;MOghu#YXzJI9r_~$}Mf+E>=!n}|>NV0p)#Q!0Yt+1oj_d(GZQlnT zJI$_|v*>~_%{x~!xTkGM7|;g^UqJDNp{*MhI@V);by~vQZZ{BJ1}k1>ZfA-GX(sgd z=*aI5(C5=12B6PkCrLF0+DCC3!edZ4lfsU|zalpg*z=1pz%R_lxx4i7*(}s}zWQrl zrDM8V#Nhj3q41`{i9f2XVQy&)KxUo>-}U#Av=Y`n6Q{XzF$q!vR$y#3&Oll~OdIxA5SHQ2J{d4{QVTVPjQCN$v2Q2hcf}|S)q1xN6Tc%bpyTrt zHII@o22L4{#hI4_F(TJTwm8~eeqB=zsk?gt=_YtN^Z`zK@>4u7i($!$S5=>~g{Cg@ z=j${qccXh0T3-m8{}A_#J_DK6nj+N~1TU!s)88mUj@OXO23~@5SB@#};M{4iu$y@p zJDt7=W6jpd5PDYgow*lR*lSwr3)C>_ASYJPz8B?&5Vc_d4$>?E>IGJ##X4Bl(Te!O z3ce;7$`*ywRIRWE&sJ)w7TBhjZH3}Y!WkglD0?k?Drm0p+L!03YHs$v!}#aj6Qu#M zrgs6J|6E9OzKZRj{XF7m{W|I&cysA0c(>P+FASCqc=|N>cOpR99R05*J{)cNZYX8F?GPCFcZx$zGb9I>H6K} z*MwD?a-V0oe9s}Ic@cDOZ2b9Zpz4=+8ScN^%T^6~gCm1XaD|yKw+gP!Zr1zZ^}cI& z=s0sKE^Hi#H!ima!gSeWdM%)uX2yAgQPm!eou)9SnYzNs+| z%M4bUMq=o3g@gFqY>Vh_VFvWHJFQRf70>qU0i+jlBW`U${d4hn{)7-zTcB_waMkuI z?7XzPmeO!Q+`v9fyaf;QZ;KYytbl3~Ns~kN5^Fi6(n16bag*(2g1sk`bw+f_tvx~_oS~t zdLLU<+rTR3)`gIZo#DCg5*HV{V~U}ZxW44N2!F5u{OV+g;f`Zr469z;J^l`_qcmTqw-p$WAURc?frA&rABFhlvn+Mv=JwLCX{|9i~#a5 z>YS1AgdfwZE@;eQ`i0%h$X{Z{tb2^co`nV#!f$OU_FHmBkiHP4U63$YkS39rmwkeU z*0uRqGZV;;EdlaOP=8cCrN7CW-9q>1UDRJMkg!kG9Ak}M&pL^F+ATo-OZZKUhNpIx z{MYO{lJqusqzvQKn{d;?3O2UPtKwIsy-KIvL8=3x`~-cE(!;o{;|9XKBhYs0dNKWi zqte-+@P9|yJSg`Gl0k0UF~0RM%t(FZRoRfOP0(33JecMrSd%f@? zAbqbT9_Q0GY|LZR4zlMW4~cgfU$h#E3l_4jLlZ%H$Br{!ds4ktaj^1{uQzqa;LSJS z;-Su@b))faq`6r9=MdrCTIIhvaVJvkqQVH`ZYa4rkUKtaEYB{z0pz>TyJsFf|Lm=P z2K)NwLooXIqPbA9L{P!$X3!yW&^w@hA+M70|m>dY$>bEk3MG ztT*3qW+y%09fArQNmC%pj}f0zb(uQGs|UT=p%$fV)?h!sP(oYhKAK&PXRy+>G0rU{nTbi$9Sbae-62^mL{nqS6hc@7Q z$qBtUg}4=}jPPR9+lO(?e$AVpwKJd*QY14a?@Fuv^zYl9&0U zC9D(F$IP%imDY0gQQm>A5YOp)9F&d&^3qi}7@E_J5&VV~nS=EAe&iW^i7g!pH;Texq)D+OQeC7e*stBEJ*; zM&%#)cit88IA{y*Sz4bHZ{#`qF9g#4#P`dq`cTilKe~D7OY+{zm#O~-;vmwFrkpfB z3|g2*IZqhjp%re3oUgEkH1ZnafKw`y!nVD&q;+be!iVI}AH0-bB5yC!n&%?*ULNrh z<~3Xlx)lX%l5UA0ZGe=I2*N?`WONJ%UyKCuB2sxfr5&jcKwQ&8dH|!G0lqj~ z1-fn|pUF-ys>cbF82MCoX^Md3^*E^S7#2DXhCGsJ`>L}{KcT~VZOetr+kAPK=cB;< zxJMr898Owal1E~SpOgk9FN>r9&$-l`-_Nn9nmMg9$ ztk=U8t!yOqASbNQ)Q)_KO0ze663&9$8o`uppYs%td^m7lv*PD^=)3DHw0LpSOZmh{ zZOmbxXM%?M4^<6za5*RHcRY=jFCBSFlVjL?%yl?+W&|#+5yeTt|rg!_%N^VIp$99DVTQ-<`+ z)GjR^7|~P~I&Q>3(Ofzix%0xIj!KU(;sccFl9;_)q~_ct2~t7ms0U`;4U2au>&I>!xWoQxTIgKhH3z?$9|rDWnZWrIf*_y9(~u3Lh^M&^(FmWL)j)% z{U2L+f_mo=BQ68N38b+G%EF-BVi=H@5XDdWGR5hiFV93#`dMW$KsBwUT%0|5v${&N z`7GYkVP54W!Xgd)%vho!jfV!mwkypK!6H-|Ec|aV+kYdc$`4T1P^EvUeu45LHq)dgGrrv%$m3yUza3Rr zNBOfl26@YTpjzYPvy@jJTyeWR}?S(Ce`QqspSWGg0tT1C;G(=83iFTLixsU6AVfO@g-o5<-?6xgjb9RIW z+F=Dmnva3Df0HoEevydTY{!?a>4KqsX5!@=y|LN$82LEwy(TS57jG`2y*IW*YIp1Q zrRNn(h2POk{4{Mfc9;;ySAXpfrDII^+gf#{scUoEqcs`g|9-=^d$OT}SsO1;pD}Xp zr8eAs`3}5#&_hn!@hjKSBLVfC45ag>sZysr)2r{~A=1ctx*R@v6SDaSpg4UuE-o?V zGf#Ko3H1D;^|eWOe$g#_o7Yy1?Yy|%8eSk)7SMU)Don`G;<7n;c0I_3}Ki{xmo2Fj0n;d%M zS%+@#y?JZ5wshY-6AiDsV|4%Sh(5+lbeSx6}vR7cY@>KGo!DFY}>d+*Nj%_P(lX-5PaRF!r5XisL+M$u>(&#Jznth2ujv z=@B>sGkwmnS#+OZ@x;G4BVsybX0y=a%uVdNKT19d@sicsE*5lM+L(_zGUdE2M%_J) z3BLYfN`6b;-K(LrcIip?Tkm5*8#6xU$P=yQ&p%p+y!6;{RVQ&n$5IsBLlEalNMOi-T{Y(IYW=3z&i|lfns1SsWm>X zea`@%WK5>#WK8AOvr!V;kA@|~=VN|Ryl@My#-HvOD;Khx?4qfOoVc?lfA#AojJE8} z?|-88K;P7rwxx@?r>=`ExRb%2_D$uf+n%w`&2%CB*>N1&;+)r^z#Cv(^EpJ;4HxqR zR>7`Tk^DwuFWGyQ5&e7u^bo1Au)H3)m#$#JB|dPz@q8>yHs&-h@K3K3ztirzwujCL zT7!u4+5*N(GYUl^p>UcP6eG0r*p-e2hp0J^iD@Vzalg` zNAF%eJpf*`?w-NucCz7cOJ4ic6ES*v27CD5NqK2~5~lWcmZ>SL@#KI?EC}l@!#zU4 ze_36&f6o@yc}5PTb==9CX8p+fRM#1X-dQdBrKQ7{-5o?XpA;x+ze{7aw-fcOKCLlk zsQOo{>Zr|>$Dod-#^%^a10L|5yxhE<_)i+~!q?|;@iYg8Pt4DN-X%PcCr;{g;q*E9 zF|Mw*``X($__G!NvYVc7zv9Vgjxc*lO|d_57RGGe0d{r#Xs`1P+-3Pl-2T^#zmAE5 zb=j@B`-~9j)3R=!4todn9=;RlHCD1Uom$9wwGL`W7nt(A8QbBH%MDi2$VqOiaY`u6 z8hE?_qJKUWQI`e8=X%4#*7`i~H-gpRFx6W!NV^HmXcfohp^P_Ka$c+OC;L!m3@$kb z3ESI>{`YTdt{NmG{T=w4J^`m10hsXNJtI7j-j1U{;pmZ<|G~9S)3LdIGLCG02TmSZ z2~_h?JfI7Vjp{2~R*%9(4!%4!#fX2JX2i3HJ|evA4}*R$!hI*x<=xXyVPCzPvT4>r zPT#A29aDq1`l-k7S01N2wctsmW2LwIYFu&75(pdN#c3zn6Rr<5KYv`zh^i$Q#(O}m zuTx~m%#W~TNt>#FJD;(Z{ojZ4i@yW;?`PX!QABG_V~A!G?;u?xY)Ux`&L1u^!f_cf zw-6fL-OfH+O zs=|MsvCdQ$4>&D6&J0GY!Fv38J1gmDOnZccv`>c7$%-LAaqraf@7hir^Dd0ou70p`g`cErWFB*@X-`5I zwk=m*jEvO6%voV@+H*EtOFnmrGZcCyjj7L@Kr!7rSo*d(UVlCTw1ISg-FF^wm>DBn zmDL{s4nJ6ek8Fp5n%jm4t_h2~BY^Og>Z1m#x%&`dB`*{l0X3f4LjxHdPt^ds(b`*- zEp0`8-h$Sb;pp$2h3|Hx3gR^=y5C7okDrT6yP06{QezS9)Ql^AK$t0rC*+3}bD+t! z<}&u$WegkaBouEwi!Vc;2Ia80uDAC75Nm00tghy_#|WY3`@!>lSnp$f(Q4aYvECyG zA{=VadNT`2M+9r@pACd_X|?dieLdp7IA+-NGb}kBhR@;yWJpL7Z*=}MhO=bi*yB~> zp?VrWSDo$U}S4Unv-kjE6*hO`CR2#LX8dqbMG6&?-Z(aDf8+#$> z*gWv`d5C%rfpqQrA%FG}yf@ws)OGZFQLJrpvJz9joP|0~J#cBKSK61Ezk#raPwOku zY0pQb^AlT|ltBDMeNK3V|AyP6WwsCKwp)u%V+9^hY$pkqWoz1l;x=XX5?+Q%MYK+fITH%*ILy))y`~8Um(pG}LLlT#>zczh= zdJ!K_YKx=;wCie`L)3yY4EOkgFQ#=zbw0H>&^~Ur2C}_PEX2dx4!p2)Ga%iBzj{;$ zh54$6i9_T8_m-0Q9N%WNV2aP22ep@UF7$q^6JPJqK@vC8JnY1gx4#2v3#l;ch*u{W zch(n_&Y}8dgiTf4rF+jsPN*zl-=}^D)pvO_8gbHbNF2%tPvxImR=BlZ4^TD#C?t=m znj&v0EuIylO)3U8p`qchZE#BPTT9RhMbKfta{^^D=v%Uj2GhayO z7>Z*5 zk706M_CbX=@TPeRthKhn>E0e_WE)HOPL81evP5(nSq61N>q*srALmZfs`0YgdjeAr zn+vKjrO~ls<}lITy*o~*?1zCH2gx5b+u`B;u1I(!sSg-wc{H`DRN4}*)V2fCUt0U@ zH8^jQ8Is2FBL0v)dmTf?j-=slg8Wec{!w!RS0FTOAWJ z7C#ZY_tQf=@AndpW{n!?vUN(V8cbFTQz+g6fhMbgMpX_w14F2bglzp=a57I3R!iy^kzNY_VtL9q7g zoRnvP>$gWE@dJ*Y-V((hSH=&Oc@fTH^=@gPYHX!z93wvAs-OG49z(ox8C0D(q&z_4 zJ>_Y5nNu-&fFrC^Vjm58C)sGFWMw64cG7HY!V z%KblXGxAV;N@GjeV4@ASS+b6q>&@0A#Uy0w zy@s<^I#O}OlFXr;c!ghjd=0t|bPuXa@;={0 zGtC}G9)O=bWI)`u6G)FT>dQQuJFf7B#!!&1#rN^s@<@Xq;UUYKAJ55~q4EfdGsyD^ z!=`hE;zp%G)m$XC>dIe^?g&HHA41YZd_?D!Vy*QCAbv*gt2>nMLDHGn>Svh9w%A|Q zuj<+f-(<$(aY(*g>*M^PsxL{asv1SYc1GS3mUfzqN}nEZ&p=;0(<)3L&c~CK`Kfw4 z^;yG7wz~alOyY}p z()Fx-OB{?Yi00%kVSJ&zq#l4Dqh?{d4>2$=ueYiVLAe2v-{ZvV*n$#Psxu7O*#nRB zYs_V(4xcdO2<;_vosDl429!tOqIZD(tZ zKCQ20oad~Kvi}LA^UFoGebK1$9);^)+ZN&Vgl1mv<89fTEzKqImltVt@-lS)sXN*9 zGuBE2L4?gjr9&9`Nvi9Ir03s?czSP@FuRJg&4&*_@}of9%TD#O!J{`zfw+~C{t{X9 zEx5`L4(rAW<*x$#YRVA_rvzzUVf`##lpZ+`J$G%uF2?;xt9?*f8I*oh*~^2NLe|>t zsM3I342%^Mx4cA$pC|IV7?%JY8;xrj16>3B+FYPfe)xoGPnh#*t@1jka*?o)oZIv< zg9i{r*c|}my-|AU$vxqvKwKo1ZXj+X?y`{N?I|-^g-XjRUA;{+!_PJYe-|Gt{-YXk&|a8+g)MR@1k!r+u3|mq`9N#zA!ljXeSoqZ2phZy zPi?&eT0hzw@XJ{dQ*jR|7vO{wUKcE9s!U33uGk^SOM%kys`o=|+eLU9*AJjRPQjRpk?<`vCAYKH5L`;@v?&dw=d`j4dq@kQ$DCwFxP??>sPZ)O znkr*c`JrgNx=vLMk`EKwql4JBormf9939--Bpz>C9;Li!a31+$X?EchT5Zl{)5{0p z-9kf_XT_mL#!eV>JdzL74gux=E;OivA69%Ko;r^=8t2g(amTf)Pn`bJ^KgF$P%c(k zmFcC|Fu*Q>nRv-Fo9ccgl7H6B?nd`DR;H>`*nuwXD3651zf}5!*1LFloR2>Gk{HlyHFm&kZ=>`>pN)) zMVVBAMrDV_wbo#pdQBu{L!fX~C%&Jgj1Z&$8RCC=W{k88ZY~R{$`nYaGU7?_oqYig zbjbq3AZ&f|n}wUi|$C701vx@{+#Sgx!)U#P`LAf-;!37;m&%`ty9?FljPeIg+=`SlFqM0bNgYA` zI9#aQWx)=P9t}I7rYwQ{(S5Hz#rae>!SZ-w77%`@Y!oKN1S;LgNEa%91uvY;aU)*G z^1Q|@tiLg>&%6LH(;k1BgX-}npFiQ6zNTVLUs^kMy9r-hx(VLY`U4XTEAilmZDOn5 zD|EZq77qrNv*!9IMU#uVICNVJ{(a|Tk+_92u9z6KSyv3EX|%Sa_YB--zFIb&^#Iql zZ^&q%GW%sPf?3o4tp2tM!sK%;R_+nwWnOrbv8}&Z z|5yLv(du>4>Fs$ansgeL$4%E%+ft53z8qg{(?_=+jrh_}xp@Db9%dc*KzlhNd;jqf z&O5mq)?aEWH;>PTmG1%RyqQS3$mF0R{Pa8q@0{%li-x<(5AEWGuHG2huYC?qEBOe& z|MS6s*|YfX^E;t2JzuRmDhw~bEW@jpUt!mTQcdTeUEai||JTwGJua{D3fo@l^rZaDERmxjRP z-x-kBc^fNyJ%rzzU61SkahEIKBcA(E30o|WK>HvmUiMqT?wB^hRFidb+>*NTY48>q zwCx_Axtt>#Of{hS3B~%eTFIirTk-s$Me5vmpA%g$<0-u(5OfawZ+c?Z=U!Z=|2Pbl z&T{g(hCIe7Q0^H&K^+Tw)}gigzL&7>(Fft#n@H?_(~zHAk?tNYQC6-_aG8v{TS@1^A*+D>lHSUR6qRMisv}iCmhR;{zChvx5O0}6FK^cv1}H! zPWJfdB>kgD%Zm>TRt7@E05iH^pNo)KYyzHVcVDLtq$+u^&fvSb_t(;ueQ|f zl?Ud3Ua-$6{=m(UI`XToD`-7kc)xf1dEs9>*+}1cszTo1p55-*Sr8(!B6X?xXqBkF9yc z^k&$n%TSoqvkk{v_s}taEXMYV%bSyGAgJ!)-;dR@IQNrAzgsPC|Jp|0?s1#7D2iY< zHvhrnPFXoo&cDI#!&%(DekVjIpU{btRnf?*rc*zhT0}Hmd%-Zg!$~Dm`Mvy)Or_AjpyrX=f~V&eg@mrS(+};(|H$ z-0J6RvHcY7n-@4m_PJIQ4j9Lx^U&$6V~-NtWj$J+-#U;bpV5Iff1HocZO5K4c^R!+{WA|!Lx)O}W{t&ts}P_*=3C>p;u-&9Xe8RpHq~d~ z*SzZ(0Au*~to<0}yaX5Sb;Ip{Izo)mBTP6H1=?eEQ3TB9P5d^%^}koJ&I}zs_P8I6 z&rI{m`g{Sb2E|}fQnGM&30Gq;3De}SdZ~0DVTfGR#FmGDeJ<2>U)5hM-RhV!cmEJM z)bu-^D>9SMW1qsr;xiblmjl-y`UJwU?H9qB&7eayt%?Bd7Vw_N{3>z&v4VPTaTWd_wyTaTy`hX8{t$<>TLV{I*5TNH%_P;1w*I3Zcp^L#+f;uG zs(ue!Oyt*!I;ry&6*uWQuoin!VaKfvBOud6!mm~D$rCi>L*5(`16R32Kye*z6i{8V zkQzKs-&!u(wHosM>|jv(QoK5v_Is&3NigPT=i*VlL^ppM-p1C*#Qp{9rjMSMy=(9p-A)2;<9PID*uKu_j05dndP8(1KeHo`Rdia z*l$mN8CorjO`i6HnUyt?Z#zAseYB!f4HNDj0yQVuX};7WW>wetw_R5mxyuU!iW7vw z=nwyNXpguk8mFG3xQDZRyvh_h2fY)4xB#{!cbC+=xU+>b4;k_wb}~&8{ikh!pm8ah zEfIgj+Zxx|#Nuu;q4YPlZ=F$vAG7A!$mOZ-IBFO@_ts$}^!T_3HoYifgw1l#fliu- z!I}l<`jsg{hAcF%yF)-I8oZ|o2G5~cLNJKSIk~d+kq(-$8hqwE%+h0 z44EavCGhBJ?K_yfnfGHT{c{1f~aUK%>dHPzZ7;TL}YZNLYXn5(*h zo+Ek+;*P3*-kbc5y~BQJZccj<1WZPw^&9ztQ^(M9eGlI8wmt+k=m8IJ#?tYp0%;|X zYlahMn$fyqms#S#Ny4p8G=A-A1nT(O50}}6vGsYePG5fHYFHH~zt|S8y|v-4IMUe< zs4vu51L+M${3r?+uch-y!}(vcfiP0WYVVPjnMM1VcgKZeZh_0DZQS))6PWfyE4*9Z z#~%m(Q+z9K#~xK&s4@8*3zhpLaLlR~pn8aUUvV~nF*sd2dw(mY`Mw=^oqQZq;w`Qv z91`E#SV78#GNd|#r%s)T+jg+Gt2(M)sKVQS+n!_clu^+3>>4B-gwP*j<%E92<;k># zOySJ1s3#CIM3>!sHX5%-S7Sb_Rw8jTQy52e3umk>BxxEbcX=l^ZGWc?UUC+W=6gZM z9{o7!7-($roH#oT(}&h!=Uc93B|$!1X%W&0Y}Y;u8PxtEHcrZrEZd8@fO=FWpIITc6}Dn(EQwQ*xQ~v7thcET zs~eD`w611Y)Hvw#D-`Yay~*4E=e-)|Dvv?v}*tm8Y@cdgyIM@&f!jJ?NR-vy6I|6W?>36vrMdehUthj5y5`DBf98&|jJy@W`8<*Z~Pw*x2*+ zq{7ffa|2lA{%SzD%DXzQW!48?5>}Thzr@{i&R}tmy<(}R4ibm)bq?1UVX$6ul$qn)7vj*@yv*w7)d())`1!Vt{U%SC`o_{EOi&K^tem2F+J+SpQRe zTUMW!*i2VGSJtb(7K`awy!4?7z*@H9fdbo@ zxKPx8v@n-&2ew=)g?Vc&fbc^o+_ne|l!K3N!$BX1g5L5AOQAiGT0w?!W zB-~})hHb?c*Ag|`|KuyZ2t%JoC@e&pQ%2)U+LG1*w;Kbs77f7Xw>ei_Li!o1wP-~1 z8YPq$*zN11ye3O?9fV5fHer|YZg$ec@UIi{_QhXJMr+T7TKO)mb&;{HUljX95CZX8n{OXxhZs?s-`NQI-Ofp-9Eir2mlg77{M!OurY$ z8_g{f!wvm#^0HHmv?F?*HYLr`o>e<^RK)hGt!hg1;_MfUn|?=)H({c?pz#(_mBvV# zkzKqf;M3qTxb?st*|VEWcPBlz3MC=GY0X=_d%8RtCA$IL++ zDPIEe84GBBGa3C{IJr0B`F=ZqxEm?Y(5lRddWlg+#8sX^I+$uLP^7uuptp)%A!%kW zIv=cTa8>0TnnkoWE#(yCTbn|`jX)%fmE?)BUiF%A%c&Nx8_s%5;?Y%LuPfJPp2)l6g8KVqL^`xBe z&{W3ibxIeT$PP@Hu1BfKFC1IT7YUERsKU+ zBO{y=q~{>s&5j?fo`}a9w}ap3?UA@%)qGX2PM)@qDIH3g6uuc&%4l5G81slGemG)T zEZ#jq`l!H;llDd$3-aXoxIFb3l8>ePM8)i1rYjHzYu$9ZG4j`-`Q8X4hmX;Ad3;BZ zeiF5e4LRXABmUBiDLDz{dxnF`A4%8c5e{;~YgXO*rl4^}${2u_(WK+MD?A4B;aB^UJMKse;3 zyfWodQt5Wm9&+}$flBkT0RvL;32sM~Stxy~yeVZ#NS+u@upxpn2%!3csT2BP{oGYv z_vk%Bt!KJ8?E9VGDK@8je5bW@>h{Nftu~X_|HTw;uT7oEHf;~43s-|sWiO^_rYchd_f3~I#j(1ScbIVXH;(y_(SC*- zd5_xsj8(k0#HK>*|1%Rlw6l-~ZiRyCf>{JU(kO3#W|I@Dyqr8WQuZezcjdv%%u=9Q zQFy_OP4xM`G%KD|EeM+C<}%V^!Y{8rDzCV|+XZO#`jSR{|I-H(#mz~dvD_sX$&)IK zL(diW1dStX29;{OT9>L@7Rl?wCeT!D|UmaDeZ zA`fM(G_@qX#VIcrghe8pdsF|dW=ivtkB1vy7l`Va_u=KZ5u($TbTK(D0h-f0CbP!7 z;r{%CaMEZk=q_2t&ezNY^ITKeEy`Bb(d{Jt?QcV?#J(^(#amum<4t)mJu6qSK5zD* zzidqMMp!#Z58Ag7xn`$5SeuHra)kK-oITu#NBcEJ$G}};T}oSiVk6}lvDeTz+?8MR zs|CZGt%jnqebDlOoBa8zp*-p{i9g-mQM=_zH1hIDc;qvjHxJ#7aW-$nru5dbc)X*wi+QtT{qgF6p~S{w~QEt-puVwx4=|9uezla9*0ZFe z{R^nEC<2-#>A>pk0T`t-1(Fvwm%W@@e@uiovwAe{}0b) z^#W7>5YDqga*v*>==gRM9$=Pdka zs{z3g-Kgn(dL&%wm7(7{P=VR(hM{_w#5$IO>6KiAYU-v<|mp5 zyU68MyO5cf%H*uuK=aK_?$%Mq!Ir!KlX1J21C1BIpWXoCHkQI`?MZORABrP&zT+X^ z2I9H55zjg`K)PMOiC2et$q1jX?A5Tgvg?iiWVe#}a@(gz*!#t1100va=FPuh`4)E>^{ta^ka82t>|Mo*ViX0Be!_8cb8*6El%#V+ zsxe7(#`7wSaM7MPn#({z$AdkyW^>?m*-md;O*`#ESW*8IXX#D(`w&2ks49yqN&+8ac-SK73 z=h8c7{TYo7OK~d)oobDEttQ>E&7|#`O207t@u?fkY`Y(ijo*q1UOU-+*TZ7(n~@mm z#ZRK?gZGP zCfq!@vADZ#0{%*jVKaKK($)-FtR3lLEM0nUl1@rXt5$nNsU;4LyN{W#>kTngc12EeBD!$>tC zt4~`AgfEZq!s^OWdwX>`|wN9Jyp1w^?2Sn!D8`eO@MQIfUs&rEK5G9r(vv zSEhG8&2AMH;jY=KjCu&u47A`~Gk_l%bB2Ae-wV^yxg>rdEFYloQZ8%$SvzT}J1?_} z0K=_oLDgldnZ08${Y%;Or#p? zX~CwF<^ieJAn^Ql7Nt`zTHP?>sxJ=~yULP^|KQS~$FOKmZTTj1BaU6Jh2aOcdO2?K zr}{nv310Smc497iJ$IFnZFkE@x;pZ8r<=I!`6gPo@v3~X?Oz< z&G6d2C3myca5WCSeq~6)QvT!5Tj3b!0C#Ku0$=BmkU3=rQq9Rz`Tsv(`hIcO-(5a# zy9ygU9wnYS4#eJX7UGPW#aMFuJzO)pNu1{(7c9@hYX%uG<4QxEW8MV9ru>Gt(R9!8 z+Y~{(q^=n%Hfo7$x&vW#Zk}@<%=HKZ;z1bFI|hG_HG@=An-AL74-uKJOsVxJu$4Uj2i$m~-}o7?*hpG*%v9WKp5;0pI^D z0@a69TktQ%98Uc#h5cTev{ZN6U-fsxy}!%)qBH=D2sOkHQ`gG6GrnQFd3J(05XbnnlQ$=7f%+b(H=x34A@s1#(u^=~ zfy7-3qcK9)1`BCDCE^vrwjk!1_6wJ0#{=;$FC~wik8G$}J1LB9 zJ7}n|35&-kiGmRm(BQm=j&BnZgHBmiuk||Y zezq?>&S?nMGe=?RwU1yPoX)7%X})&TKBJCUD@X_QW)H@aZ~L*=ykzXyBnLi+9fQ@w z|KlT$Ic3U4k>kd-0%u0lT-bj`ZtF`%CnW@uc6= zv)TKxi3#nK{$K^Z4{5JS_!-2W^>X8?CO>Ym;7w1{vwjYnsK(4N$~(16f6)AhdQk^x zo_-+xp80R-D!Yp&Qeo`4J;|CwzMG+F^9)cpG-lXHsIcjW8CG@p-P(&~?rdAUkduPM zh3uT?7+g~HjCE;d&bLIm@D8*V{pv(F@bRc64>X^k`VO1JWcpn?n8vl1#805K&B9-e zc(VpcG~Wi|X`&_jo8c*^Wxc?N$TmPdsSU{(hY9)FAkORY8J6>u4$-W7V2N2d>6%*Q zbun?OGb4WVq`C9Rd-)fJK^boFzaaZOF2UjBbD>swA`X7`Md>wE{lCDV2NDm97a1qu z51UUl+EA{ESeoZ{e;M1nBND#FddMSVy2<=qAJMkOX)rr|Tsy#|I^j9J>$xKxe%*PA zv3sKE8TV(<)b|Y1^@xkFT==fze~~l~OL}gBS2E6tTIs{{(xdm_4@hUM*dLzNm{0h0 zNUO$~=7;p>5>YsYW8FEre6%g?YcT!2wVZ}8o+ z`#}1a)(79E^a`$bFVZ&JYCu|ZEv~NbBYV_|LegGH=MMFqr$SL|H#VN`L3a+_O3xhR z5&y(u#jdBc*STQ3zU4CF9SrOig$+^$Lu$fzq@I!APjN8+%#rvRi61a^jh_fh_t0p^ zl_Q-8&ggB(H~9DB)DPI*J5Lb4VRiR8Eaz-|^_nH3XdQf|AxIN?HawpN27dj( zcAgVY_1mtkwEjTarjb@*-lK~%1Yrvta~unb$BE}T)dA|#`X)-V>OZNc@QYLZDGi0x zkDTyFkZzMlBO9asD1BAGV$sA>xHzMuoO#ev&^2RlWCyOe#b|)D?2+7v56kJUB|W09 zk8uA!`o}c@)zhShkj582mY&v@mA?Xfm7^Bb0Q!woGROxF@x)2F# zNJrHttSrO}ki$yc?h;RHAWPQ_HKmJ@G#ugSV)Usv0I`p?Xdk|p_KMs9(;KXSv7IU* ztVv_8DgBjqcHw5xH@b_`bnHS-W9--{T%`3S)k4MCH+RRv1ej+z@;EY01;2X$habO8edyI|j`Zchb55c?nVfNiF3Aw8>e+ zFlKqnDo&=k1rLkSyyW;yXkdG)N?%hwNgIP_5FdRIe=Ha-iSv>813t?M(0G4@FfAS< zv)d*yx9fxPxlUiapH+Yjk~8qC^)gz|^BfRYW4~;B*|C8c{~t-$9ar=B#;MFABFRdG zL==^}=Q;My-h1y6U$VDUl!%P%LWm?;b&F-)9(*2)UEqD=RD7Q-OuNo zKmW}pcufWVlCy%dsXV#sNnjuCtmZ86dreQ#-Nz@@(JH&Mv^KcMI?nDm8}6ETttf*t zgY5ZKmTI&~fN!IX)H0b}ww6u}C{(EPi^=k9(`mOCK#@~D|R-3vu z+M81U>>}Vh-_Jf>f3+{e@L1Zd+X)f>hYaqhQX9M(^(xCXs!#PD(}lr-sY!Rm*cK^l zG)%JxE!q6u)R|4;jhxUYZ^WCJ1!B&TKr-A6Z73QITuRlg->vYN;z{@e1)acH16p&g zivqUNRVV0)P<&39wv@mv=KU3Fi%ni|E%z)5-zJTY0Ou&^0ZwSzkftW}kjL8>W~>i1 z@N7=5S63LVfEriu3uJr%dPHLFxyQg*zStrbydJ6GnGOG0>OOC4aK>u9#v0ds&{OS2 z&q85TAo&J|`i>)To9=KkSb#5hx$`s3`7zb^+8Ty`%upCF(gv+@jRa&ihTYcvekz#b%~S0lF0;~fVsQZDz>R`4V4 zcl;D|bHy_1;@XbjH4LAL3NP{*t#0xPHkbij0e~wFEX!SwoNTn>v;__tIfy_;Vfvkru?Xm1;Nb-_{CxIE zS+)9ZehI!NWIDp=k~}fjEH{hfkVOMlbM>Fqt+nlMik_WrQ?2tAu~%JcmImr_cbzD) zFUnxBrGZ)x?r_rvFH9~`claOy+$CIF;1HRU4@`ZfuD{zLOzi{rbr-_iUP@QUl&|&|H9ogrZ z^^<4LHOs0=^bLLAQyA+zlX_3HNMuZUK7e(Br`p8CuYYJ$niHlwQ;Bo!Yf&jDr6O+WVj~J*bw84on@YO+6T2 z0o`D(PTW>4D14*NwF~{m>ySCyu&I}E%w3C-~9gRE@&o<$Cw+nGE*mLo89{41I zfSs6^W7eK`MhfU8c#9Vb+K@l*VBS^sfr4%ZhR@+crTxX&OSL8Rxyj9Rhx28OR|+y1 zw5nPOld(~+`gwKg%#&)QXLi2Z6Z@HuPeiWt8CvN$A@?%589`4A=ypQRlxe-wPC(;O zHa}g3_sDH@OL8b&Rr^Vt%o+8HKxK@`dxd5>c96O1Ctf<8UcJW z`a;!ezn7sS#E7kn#r=rF0&A^-&B_M%LSby^<(gC_M=XJx;l(S#n@to$0w;-VIaV95 zEszZeWIlXmQ3-?jh6|xZ?+JK6?1}PTK(mO|ealkDf2;V&_+r}RY$+=$FtAd&wModz zCAGiR_Y_A1sVE_kH;QXC8o8c z{Ebc#G$(v}N33_G!9HGd?xq!-ONF=kvfZUe##>p2$KF>*cbv4eafnb}_?#$i>IAjE zMJbbIQSkdPQ9F1s*Z6A>J=wv`JA45?oAY`CydKZU^cY;sPO+1NjL)sS`fHXKyfFm# zdgLZO`1B!m-+N6#mkIog$&(n~9R6&L!KnojA1}w;QsL7lmS-fSk0i)y68)AOHS{y#(#>Vd=4wr%)ygX zkCG{bHBhc8QyF?#mdl6`z=o`B1m22^29)B2sO*F_qsp!i3E3O9sMkYn9B`bTJ$ok> zIMtQ>bB-)>+0pntx-`wj%(-gStB{=9D_l&9@gsO2UgNkV@WcIA1l_0}JI>%M-3qd~ zM&z0Fc4~Jzw4yX4TY)Z|PDXR#`ULvPWQhb#7-+J_KfcY$4*Mm5H%-kb;IqWKYVbiO zFjXQ)P?!sovl4O+gQF&|gFY<`tT;<-w?pjhR#cgs*S@N|d{xPo;GGyeM#xoaYz}f| z!$C#^8NZv;|Dz~-%Rw(1vXK*xY!~n^@J0{BfY2V8<9-(CW(6MA0c~8h^Yh+xt)3tI zb{b-|j=^#^-m>_d@$ArR6K&b{!~AYW=1#^x47YD4moJ}0aZ`U#+T-iuaHBW{E~U61 zk1c01uBp2hv?c$(O6pk9HqMT-qtdf)HvgspcYUPq!T{baq z?WrGo&!%3}6RCE{f4WidUHrH#UH{V_t*1od5%Neflbq7cI7Lh&n=a&WUH4^jAcToJ)SJp#aWK->$$)w^33jqnH@HarSBVS>G`q4^}G-V{jySixeoWTXY`&*@o)a|wb51N zXjgBpLfXwK3%?lS1d)&(+Ib zS#)TpRqQ&^MknVSz@sNkqWrbbiuZZliP@BX4~HWl=NZU<>evpU>1*A%&_VOzNm zpF^M8<49FM;n|L)n&{9QvoFUXWT} z7uowG!_Mvstv^1B`_`FEIUXrXRP0*4qwYNp|5#4%z85DC(|fS@Sp1P_ys35 zu$NcbF60Au`l+UyUs9tc%jC#nUOL;+-qicRRk5$+Mb&1&UMf1dk)E}1rKq@b2Gtu{ z731=jUDy9s}8ik@MhMev$fx#(gWI8+#pLtg)bm$O_@b)cpA^3}XE zVIA3prIX#-2%hfWUalXoUbVriLDDt03+z zIB&U|{(|1VTu1fK{84p-<3!uTGiB^>N7PJ|%H6vJPy6hKaZDzMHS_uCts&O;(S7); z^AJXV)5R;>>1NyRyl47FE?T0dL`~E4w|(f#Cbpm-WMi)ey5Qa_vfJ7{_$<3F`eHwQ z7&MDFKRM0gU3SoP|0~=&@|wt%J&u|+=_g12y28un{$SLn&eouo={cRR_dvPj@D6M2 z+wJU8B!}MrJCF&_2RJ0Q)7>9z}T?AD7uJGRUc5wErn)7LU zR6X13O)F1$=o$s@SvNmvrV92brz1ZN*HQ1c=qs0ZTKz{op^A&WrL|-vw@J-SaJSCOeO3Qc}+wWY@iJ;2VVFNETdjMbIYlI`RLr(a>}g9kP_?l)0Qtp_dX}teD65+ zZMg8YsT|e07(H|Rt+LPBMz3C#)_;>4@Zz~uZwc>92^^80LE9c(U=Fz6qBdF*?-v@>4osn>DMM22ggoz8Aukb0Ol4je}2XWyM_ zi5hvPQJ0c@{!;%xvw+!WY0Bj=avx`}x6QEAss~{&pkT|z-Q7`p%`o4CdF+p_l+pXI zIB3mj)pA#=r+!_{SNhqBsmOQG(YDDZ7I{@Fi9 znI5duypxKk-j#YLTIgQYnHrp}rhQ&R$E>ZD!Aa-8PpMv17kGJc$OkL9i1HjLC$G0ztmiBoi29!`E-gOHPOBDEv~O?j7>4Jh%O+FD zuy_FtBF+Z{@|^8L8=L{&TXWW#&bVfNfisG|F0bNh1q+iIS7ZARSi4A?Hm0uOWr_OW zCeIi0y#lGcq}c(}M-2WEm=6my6`iSp&misE%ZLkeRZxp4&T)7kKTaD#uD{j`@EjMy z9$$&i4^g?ROL-X8R;xDI>uDWAScH`4*%#aK*rI7_=wxp;y;lFs6uI215~KFGS>#-< za1Gq@Ul?CoXd_VzDzr-<{uTL_+dX|`9jKS`=A+*&7!$SHG?I941NEpyA^`_Qrv@Wc zOnzGpETcQ`3+nFmz8YOeD~8?=I%28J8B=2{#eZ((;N08P!goKYN_8)t$9)^m{8UDx zS9I*~`SfUnRe*cRp;iQYjBjgk%=SP*zwkQb0Y*E$v6lk9%+4j10ypxrDyQH z-oQCDbHw-ZtFR97rhlzn9uJmt7CsZjEY8}6{VexJP1ZTr;hz6D%hk{o*|kmIB7Co^ zjsEm9gqI9mOtn&6xqqorn9pZaL8gq>VQyuT57j z7dU|CVqeW#%?dyZr}2%@uY96&u*Un#`b~>iW8S5*J?^hKY568@IC<(m(>%lx*Fy62 zjgbc5X#ejF0@E=#IH>aHOksa;I70`RI?&)!@@bu2xOPqDpHH`m?CWfGmzl-&)|uH= zrGQ}!9ZTp-GVj@H(|dusP$84jM28Ju1#nLpZE+H za+zy|GeW-?(gpq&*XuScVBhUg3@wEozaA9&J&xnB*L{l!M|S`2jk$9d2X48M!SK3U zRxu@J&bj0PtWyH#+$JoA?o4_A= z|6O4{SZazpdtcypk-H>+5ZtC2EjO0PC~yK9nKeb-w{Ti zd`!+XHKQ;N=};@D24B-dM_>98vBC1H`8zK3r<5GN+)keUewYWYt|!g)_3hP)lf&y# znT-A1Ev&DIKDUtj-rGuQN&#(q$c-1AiKn5z@>+nubYaN><+AENwSSCU1KJZhQocB4 zCnG<-Q5jS7G1e37y48wW<83o$vC#y;Nz~IK0qmE+L^b2fKv^l^wK^!0G0xptI0)^+ zU2OMqxsH~s9vC0r9-QZ)M#twx&pm^NykdFKvJ`sY1LiiC!BaZk;W8QjGX)h z@U^;JoqMr*bgrHSxUaXwV1(M;)1hVX8h)G!tYYhxzr%~6yf(+%y>(?HiF0LDQwGy7%WzuJvvHo zgf!~|{baZa{?kSpPifQt7Q^$Smd*(D7Qs7$Q*ZHr?)mv+6CdT%dIn)kTyuCbXX{r8 zn&`PgeKI^1RgK#r;Bl#csnz_Y@eY9;fH42&T6kW|42$szPeQc%b9kAuu1;pdg~rPw zli*GD^ApzH_-*zWi*u*udGcS6vJ$$2_D?Dzu6)WZgU+Q`DlhVZ@9F>@VVC7S@NX6J zGlu>>{V6K>Z4`vP5Wttvarn2-RI1QHXW=ETRiFD}LyPC8QgcsmL$g=`e?)65{t%`% zk*`=^SH8-?f6n-~P4p2>gCf_LWHF-PD9Is(sni8H&MWb*o@qJ94Z3K+*mM-;vtPS+OB_In$dhJ2=_k>pg%aMMP7!UlH8TV?k9pVPk&zH%CixtW%iL_-bN@ zb=GYC;>ctL=;I)G77G8*R>JGj@rjQGbd8xCQ9tFeLXMCLj;7oXYwB+UqUc$b^Cqte zGJYrU{So1QqjMG(!50LY+$dL>%KRgT9T(i2D*FDN!TF{u>$XR2Byf~=W%D-Pfz`dc zZ0hvGkQ2hY9rkHB842u9FqojA)kVG6tp&6yw|w56eEdFIZ2EVVzJ)q7vIv4Fl<){N ztI$4rKDV&M-|SQ>P#3OwSs2axB)^}&UcC-YtkIIL6)Hzg(t!rwsSaH=KIrS5L3;R< zFV^>SR*|2tkDNJbIvXu`X!2b~w!$$VJ_)Rk^}ipnL6`%hN%g*xZRi^I4l-KC)XUhV z`M5?b)3qy;X~+Fo_WH7$JpI>`=h9JxaqHf-cLKYI5cq==2S4TeBax>=2iRagV(2iA z_I;p`x#?3bt`u@B6B;stSHGyueWtBf@1q}DfFs5$%4SU-8(qwWdcMJZP%#Wl=e~cl zlkrH#U+?H$ok|83wHj=0oLolMj%mQpPS2o8MRs$C@H^DAbWsHlug<$X#yL4oLiJ#J z740H#j`G#VC-!6HeHz{k_dDOPMx+Ol@fpEG{&1C@&BR08qm5q4;tc#By*iIjO9XyY zMvJ@}cF5xB{xj3)4)|!+C8AV?Ydx9Qd=SG_NflhdR>F6S;~8^}_9x_QMrRZBgwm^0 zEYNhukMOSz2UW*@^^wJ$4JujeF!G$-a$1MCSswi75j+RmFt_%L`wQ&c!95eL1ii%Y zVQh3S^oHEMe;;jqm=`&RALF&OY)CwxyzU8aS|KYx1^=2p%UZJu3E#o=ix8XlrSWoR_1;+5JR|#2b^61bL+(mPmir@1TVduK>(ZZ0%^2SwAML9v?ZDabFUxYf92tzPl)Lw7Dv&3s(aX;T zV$Gy@W2=W3-$(|7ksDYqUi&IAR&}%OoveHj<1oHS99uDkOnzHBa0QnPil>QJvKhZ9 zO)i@~BGlS)pb+RqWzF_opqI(KPwU8zWa`LZ^`8c8{E2AQu#7NsTH=P`nKUZ9bfZAV zE09A1a}$lGfgkf=_z3bCn_nV>Vc-BE=N8G~hX~$)r|kJGj2A-o$|gH`ai)pMoVcES zc{Z4J)%hhuUvr1`zO<@h6_XdzyQW)J>|0-veO6UjaqJC@132_zgEjE&bVA<9duzTj zxf3C$Qb|X*TGW)-tW472Ff#3+`BmIR=#72C)Z4#VDJCD{Z;b*pa%)DGOfw2I^6g6^ zPub7hwm~_)wSH%n5H$^TU@y^s8tbnO=ACj`#v3AP%lWZyvT)2~JHRi4f2PO9+$-rS z`gF3m^=Fv)xNHGGYkfrJZ4ky^ts{B*)MYY!t>(K8)TORR6Xpwy-`R~ULDzapWZffTw&n&Gk)6R~P&T(;+eh246 z<$6o4{btE+N49X`sJy!D*ROnhY+38aH7efuPNq(>9|8``USpMl)mwLcI47Q@FzmeiH~ zEvnqV1DIP+ozttlxN^x(mwB0@t2Sz;@0W7YnR$1hW?XgOUB{@vsJFbTX=T-A^dXf! z=YECxpzW8x6Xq+cK2A(a;qWyUjrF$9y;6-f#X~CZzR4}i?Vtr;7gJo0zB+$jJm-HR zT3XNfRH5KyvDdeRHJ5iqInpVa@8k^3y!Nn=zWHG--~8sO-^b==Z!wcl ztJGD z7jGuLquDzb5PFO;4*1~v>TuWD)_-4pX!H8xJR(mIUO#Sy+VQj*b)8mFE-vWHZIM-_ z-<-;qj|K3I$ouU6cmtmQ@RH$c#*3;=3(`yM6a5CyxfIvGL@4%1&fn`cJ*oSd{{5&f z|9ws4eSU5-q1$g2w!~M5*e;h?S597SrwYdeQ{?v3%BT57;T&5)x~y>{-~sjDd!F|B ztzq;!C-$F%wSCFOcJAbHMTw{94prdpW?F=uP}V6z&TUsB7{+Zkg-$KKm5 zMU#@SPCX1ZSi6kPt;aV`WULqIN;pFs{r3AY?&0e4|0>G}?VE&syfE zD_0$*w_np)J>8;An(0-U!@Vo;Z=V(GVVVDO4&BaHEvq(a?cPVgBT0#yu_0hE+Z8P= zuRnL?z&Q&o4?H?BuuO+~*XO3WE@Axwn}ZBy6Aiz7hAsUe2Mk5m8wJ}v0Glb^Tj#xY30Ih*Ox2ctR8sAooj?OHD5>n z+hp||=3X2fvXqyWwh>{mTSTWJRyMT=ER+lX-BuG?Z$>{|r^&s(p^yGzEvxGEn=^UK z)(adkt|nK=kF&u2mx{sns$FC-1AVHn6y!r0bq_l@0E{;Ow^AntE^(#o6GV z@uAJBPf9kOIB2fC=bT3`t6C7a1|D5ohCN+M=y}*jAYfcJdHD?Ulwpym(^>lUm`cFd z2uq1WIb^d!?)vkl1m4PJWWMgj)kU?LCbaefcT6Gec>#Q3s~f!@qrp#l$iv-~iu);l z-CC^22F$0*)oSRlxGLzM#|Gns+!!M77kA`$k(!@2{%O{dfX8$`DW(-)f>t1!|Xkac>ToHR5a<4h)CLxwK*etEO;PJMF-NCoyaAsF5v8W4l(Ay zvhF`;!u$X$8e3isUc4)1t+8n|3rxLq$M4S%ekaxV*uM&v1N>Q6L!XSGZl z-?Pur1s-e(HVG#G>X&j&!$#YNJk|7&G?j5G7ICtRl-|L6?3fZ zW#^Go)m=whirCVS4E}&`I5^)5wO@LQ8dDAfFIz^m(M>WIAZ=)gs4L&Ro`eJX+X)K}|!+Vv6Vitf26FsHP- ztv|41MV4+z=y||mEwYa=ucDRjAL_I1AN4=rtVg?l6~F-=?IAQUo?i|qLH{|YU@NzL z66-8i1TLq}L%uVvgHUhK26dHXaVa)^)K57|;E~+C@e~idw2|gNo~8oE?PXwrfX-6$ zpB+~BiaIdrQ&g%rOpeG|Qd6QAIJF)Fhxl0wJDJ!&kBogYm%(rJ_q~_tJ8*BBnABxQ z77jkDzER9tHi|~%T*WGIhBZASpaB^51r8=v>f8bXuR&)- zK`YG6Kw>ZK~u6ds$@VN(KhXzn@-N&-jJ95au#-b*quIB^ciYEyD zAl8KzWz$E8+WaDDGXAvoy13&oon}U1pQ-&B=#lT-J6Qp5Yb(4jixqw7jzdjtv|pRr zu|cL*q4ntWf|?TAQIyzkBdgRrq%Q38kj-76DDWAV{p1ggRtj3Vtymj!o_^ge%HV4~ z{7)QLs=AX+jTD+OBU8B^6z$vGho-uuUJdRpqYfqT{AW333pa1+(`qR}8}UWuW$lx) zS3U2T4;Z)#n*6+)I>OxZ@2P7{$*J!-&mrgx;f}qDfRU)9-V8ht1JAGLlpV`BdBqH) znZ=A*ZaT4lBQA0DD4mM_s^AZd_GjpG%2R6smA;Gp&KpMvV6!%L2;HJkpN#PdXalx! zIBG4I1AB{9ctF5$URL#-xOBL>4vd(u>|4y`h)svGG~)CbC9-r3G(6rbw=DCfD)_8Z z=7O(naeb?R*HL;#WH0EWzXG@;!5IV`p{kAMsg?B~8;;bEDMW&!3EGEj9HT{Mfgs?- z5O@+R*UGdryv6xCIvd??nR{`##`?o2giBM4r)K-Bp|56e!>1$Yp?9KaUShLVmB=N` zS~h&K6SXs)(@t#{fpgBY(M}jYLnA?>l%ba8J}8gh(|PiM>=IfQ*kdCTdII}Xt7y}A zQwB_tpVlttsH43V`c<@W8>*L8&MzyLEF&+Zy9-mVC&SZC{aNmM-~PQJd?-?J#A&%+AY*qJnNnmR)NrSl(+6)6)-W3ZJo~v@PIN}(J#bbdJ7w1 zT9^gifxu;iv8meo8Z+<}o&e8Q|J^_*!{5`CVGWG_)C+O1E%1{+Z@Nq1Fw66f!F;4g zw7?pIM}T?oMZidK#X{@+XAuf>pp6eHIHo#dU96}*-uZeF16L%p8l|LNhL+EuKhN$# zV+Uq^@5Hp1*3s^Pd~)VC_gjDQR2_r&xWH|g)23K|}aYNlmE8||isuWQIM z@6)rq>qD>GC~#yH>^W-Xzp|DG!@TAJlw5}Z_BXT zYJ4tj-WUV^>kkiFlA}*QWB3g@;N)t852DyNC#nD8s|-CUQPY-O-p`;h3Q@-%RR}z; zB2(>UW&iQ?^WsUhHl&sID!)ZR!_db0AE*+t0o0U)#sF5>%Q^eI3g|u5`2y;>G}-z- z_9%FBvRYQT89d@bgYgO&$d7}(!EqfKy32a~$r9rSpg*buTRUsRHKi`LF}!0mEj;ug z>eO$i*!AuT71t7dI9BvtHyXVY&Oc5^815usC~v=Pg`YVB?RCbCCkv}O4cuc5&R1O3 zIdWF~e>^tO>+cnCJjm#rSu6LbN|k-3eT$~T_}8^V9pO8h@RDLyIx%esdg(erGZT71 z^i4mk%(cSP;cJ`e_~--?``*!FGJ__+j?$Y+BNVhzmN$fk(T@T)lhIT)!cM9VUAJJL zui_e*32ih}l(Wgjz1Mvb*B-Wp-U$~6o9$6K+)GM$3F^JhANaAK4|hFk^Z+l)?MYiR z-6USGz}Gx&%vt!=O%`}Oqt)R7KSEPnp{rLTtwW!)sUd@j;QCzzt-_W4t(HxTzwv&# zNgQgUh4K8*s+hBIf_BQxhWq%cWqPtG@ z33$lJ6$rYE##BEkxLmA&KQ_1+H2hB?nSRqoudH8Q7MhVAcvXhibXE#=$p+7N|LP5` z(~ow(D9A?h8%+j$rr|$h@!pxdVa_YD6nm?b&mGJWNmIo*x+Ly8PO}{Dl!HS4v{uK4 z7el`X%HI!)OQX4giRSz8_9qxTz~?qxGJcrRPpW#KXY5sWK$gdYu9HXl6{Nnq@|b*u z%e<^?){cyiMqZ(8ofk0PL!JyDY4kBU4)BoPfzk?{3ZD}Xe5t91|2#{;Hy%~|J$feC zYBU5e0XffHDzfYvyxx7Wrn7+le6G%?6gSxk_1+L@H95!iCq*yk*XUVMV|pWT zx@hF;ddTp>Ydi{koLKKzAC=PQM?l0B6)OSTBrMv~fJaS1!#8gx|4L z>sHoqtkK%K+ss9l-QP=VqqB2(l!mS=iMnjT(bL!PM*qtBeGI)SzHG0kbNN)( z$QY3`R#g6>Kaf4#G4o+^3xQr!^)9>d@zJHpVCvo~9VPw^jdR0la)Y7ICz=|;xjAd0 zwZb@Peg^`t5&B#u4mt_G9Q4Z>@)L(>KGfRne8X&eLhxNGQa=#&J8mtyn|S>oNL(V~ApZKVMx zOM&C;@l54hVCttVZXa+G=U}?8g2z6f23>t_1qP%3z8T%i%d0J-r@{NwefLWSV+Gbp zb?Wy~L9;4kO~?}-T1_q!V%teHD}d*Das~>_CG?k*MlPxjwV4DQ?93)_G9Czdjo}AN zV z#;3B;Zne1qL3^3}1G#C08hE}BLtDvt5B6r|aL7rl=naC8GkTK3lWgSs*x72I~_pZ8nJ< z6k7DKFnPfGDLvu6-$Eyh6ShvLvpjsR>f6Y!xC_A_h^sg9GB84#95B?YjzB(2M+aUM zZR=f7s~RQK#Ew4DHUGr5+cBy(=Q2HKutr2|$&Fh4lj$F-&>uBS{z-#x`%uu_#g;gm zGRR;*BERj1p6g+OSCGIg1${!jR&4{m?gyq{RM7BbbhXJSCifc0$RrJqbI{yN#=8n= zaFa(PdrcsW1zb9b`en>Dxiz@hPXH$l%-yeqdkSr;R5f3%$l^7iw6bHP!C#32ZX< z)vc;Qlj8pDj*G!f0gQ3Ud|ir~`bQROrC+z4)kwz;*hgiP$XQ{Xf;I~}5Wb(gOo|Y) zVISryS5+a~Flsk+J`Y{e&+0bAU6vkvRBUWGlTK7$OrxLX)+?MU>DRfN>-?XhDYm&Q zZ{U17=ebIHp?^&^FK;~ETboC|OP)$e{%?ep*_!Iqg-bZ*)BcVIh9HmeWH4D=h9xf{|^t1_my}CNlyM4!?%-LQt|LO858zR zj4QZ}HpDj3xxBmS!)Aa@Ld{>=OQ$;Kq{i>-OFUT2r;1y}uIqDn{-EQc^5X*9a=EcCo6<;Lh=?)Q zEJRFuKKAJu4IOw|lq(*ipQLA#ZwCBPnYKO}V^&}4y%eSMzf}1@*=tL-eB`|E5$B7q zs&7|6%moHb&{vntqQ|=~tL3wI(}jq9dPa^LbfS;9>}hot>4WRbfwg{8{JDxUu4G?X za%C%hW58K%@3E5Gf4@ezO5~D-w)|9%DMjV=+nu@8w-l;T-G$C%F4afy8OX~I^ZDGF zY|>_c$$yYO+WL1WH(i#&Lyv2D7IwlZ_K8voau#)s&O1#Gxi z(T4ikXFpxqehEK4A&LJ(;`42KnT>4PHE_(*^E5x{Tgmp@96`Yz6ll;UmWnixzM9hRg23t>i&RoD=?$FjS$;~AHoc&>Dsthap^qn>1y3Qm;rccdJGI6ts#eA%-FmgF&aBp0A3fYsK6wyNFF*H{SBKQ5?Xy17@JYL9RCHxJTW_uy z8W&3gYfY3p7U$M?i#k)gTEXl%y`4B%vpEl*IZ30w)S0Sf6wkcX@EzN8;S5_4KfC zSu4h;dt{#@cLi8Qtos$JzIBl>xUr;r0sZpxBbt4-8{e9^hxT{aB!U`c)7>|ua1WnE z%Hy&+!(!FS6zq0Cx!Yhn$3aQG8a>mTr}#$BXyvDym1gUmeqTsocK zzFEa}zT-TkgPvkwzIl|GQbgZ!>uUKoX$L?0-H5)9NX7n4d^T;}@ zaNb$Mej0MqR(IPQ$r%%bT;>%)Yx}RGzok3sip55VPZ!QpuFj?T+$aw#){!FXEaFF% z?vq*M=5l3PakaLQ>r zeXZpJai)im4)@jqPk+$O1Dmz!?|viF#J{VK^8CNo>~(%4&srBCYyK`q`$EFRGU-Iw`jqiCnvY9W8CJOVz;VEU0Vw zv3_p7Ht#W=WIIE~wChQ2Jo8J`1r7E%MY+6h^O4(L`orsKvSiH!9OWgXC9Z4rA>|R88W2r+0t{;wAWtP0e?1d5|Y9-Hx*} zesDy$dR%@JK1VyTPhsw8ZGgSLH}(h>=%1u?rJ1_UuiupVs7_F88BsRh)DK3i#2H|zrr{0R5nBDSq;A%Vs0w>Xn*iY=hUmx}ZA)OiZr zU}{?|nY+mn_3()K8fEKkC!T%3B4+;yrB;QLEXrk>-gT=xqjo9d@>27@jBDW7=F1F6 zQwhh}V(!-}7Vv;-^~6Ds-F=a;W@a9_XvQiAr&FIlMYwRa;_{eVWowP=7T|bM2~8u- z8s%M?$W5;g=89uZS?T%q%!zl`^V)!DVd@vWN$X1a$imfD(AMw;L2uub%;L}`ssQs%7SJvfbEO4q9rXuO5G|a{aBxzgop`lZlyTykdGuKf-ItndejM5i?}yn#I=f zE7q}U=&J`Uohc%p)?m|XiMOf%ci*YM2REqa_a~}rpLe3yT?Fb%>?w6mY#RSo{X8*@ z3fbCb@s0Z@tJ=LkmC#E(xRs+kVvqeHpGBCuwnRkV5{bB<(r67_6YY7|kXNj9Q%1LJ zy3|yEe{hUmO|qk+I3Ky>%Jbru+pWx7_)KV-S5C&*ngR*M|xG-Q$`iMOKak{TFo_1FS(Pzx5CT;u+cKJ*DgMsHb5B7WcbuOB!>m~Zkf1k zE<>kbecD*QM+`MsCO-XH#K3YvE+=W7*+31sRLizkQCWkIn9_KdvXiKO!1x%$P z4ej*pD~0%P>1#N{@hHzem0LC++ep;KIXOpeZx%!Hl+d5M9^p^ed=O5FAmb>L4ujZO^DL37%Fn#>aB8U!{a{akV~(s=^_DbuD2plPwo^B~jq(vShwL zs^=g9ZERgxei7MqUr1OxF*edFZq(SVj0Q{b@`sMgiO+1SNN8;{KhRSJ#fV-xCHhNm zTAxmG)CTHdcZa9N)v`iU%1!I5(W3RX+Hi0Ar0f#CtE)cxNYI3OUx=+}bR!p~ee=@8 zKGa4HZR2SNuL*cKK2hCSG&oXv&h+XuMs{jMtd%%6+T zerBD}&PPIL3$u2fF&nJA@{VQnk@f8JZ9#7q9$=$c!8z*cvQ7f^D$vhr@8fy6zc!sS zd*A2KwS(z!+EbBodKS2{9qtG3#KH5*aB`K6g!!Ss1;^-IBC9&5`{}1AG~KqVY{o?B z3t8j3m5TjZZJB8kZsrHqGtj#CSszp5dfQ}3L5^G>FnsWZq`~1vfx%g7J4KHf4 zQS~0ZmcN~h;%}ZpT>8EttB&U5UVLat3P>57bv@hfwWOf2@#4?Grm}3V^6mH$ln;}&-Zuw6uQhmgziSary2kik4Kev+zM$=CH57VEz>|u6gKG zr!Vz}2xLT7_&a5^dPoj00*d<(aPIxBfF{?h1Ur0V*PCoAPozlu50+$YE zWHi!bPv9j9USRLK2NW=Z-P+_)BIb(GfWVs@lu#dKbM*{%d``c_lQ;XEFNeY_oO*4~BYHjO>h& zy9MIsELcaXZr^}YympD_+xt+X0q3dUp)1@p`2y-SkS3IiSL07AqeF%7w_8*>OzOU&g=t&l%Z z;n-7Z^}Ph@yL%nw3~fo!0c?EGf$es>RpD}S{igMq$USLQ+H7`sIF?7gIcu^FS*iO- zO0vz)#l7|*Z*#=C>E$&r#{z$-Mh5@O(ll5*5%#hoW1d9$rb&E!;%pw#`u4Ofve(WA!i zi_PjC!=J(XjaJYd653wD>+tjs@8Fq+2ylWj^=-)(WA%UUtjwA_v!kWH*bLpX%dEXb z)+C!=e_=5`>iX6Le4|DH8~tniugMLHj45G77AnS^-OAZ=dP?Mv!r!fmJbk&L{2V!l zz~cnHDt>u15$FNJdRc)PrbmpYA%o+{mDuF7<%;KHW%66&on5LV6LL#svPjerk93VQ{y{)rtKX?wd2-FHx~kZ4hsos#UYMH| zbupMtM{49EXoO6o3-)*L)SkQ--nf$T>@knPD+c4y=g9&&1j8SYHO52#c6e*DPHFPn zi*-XJ_zhlrJ?%`vXFhwTF>-Ti`}nQZ=N-;P+A)S|zdRY#eO(53?lVP1xQyfpt6b=} z@|K&=Z=~!K57W5fL#alkk6eGyb9VYFbZY+7mTDn^+~%pDp5yAM8*DC4eS5d00locX z$x(ylw?9Sb1)jGI44KKx+Fs%h^nwTXVsiC6!ZVk((WO3imEV5d=JF$ZSn`dlBcqqi zP=6=6S{7d2N`CKh;qxk6T{pS2j3|1B4o2tT{q}Jb(+Ovwj%vZ)+v>}1eIDXx?{eD; z_WFwLTWVUdh|ZtkLd)8Wu%twhH4Fzb5>|-|t03&&%AU{~|6twurdWkiQJ*=WKr9zJ}oZv zg|8f`BA0Z?!8gOO56Sp2F*sotzu6mXJ=S<1UqASp>b-K%9R@h)FH=19m~P$ljIM?F z%29jE@VlMqc0FJ1>-~_D+C<5|iH)Vpu5I*P7T|x%Nd}}Ykz=~u=FJ!EbiJ3`_+jnG zH0Or1^sGAEGWS|-xwa6_uQ?V*%bXV>Pi>_e4aubw7To4WgU)c``R|zrPaIZ@ovT_` zTslIn$~TcNyB+j}@bkR!{6Tdk$0qXH)>EdP{zEA-liAn1kGgK>!x65V3BRAKA`2g! z;LkHxILHyJp0jqa*O@0raj8#%T+a*GQ)v`W>Uu)_5<>p@dBTd<<2q+??N=LFHCsci zJZ<%=xb56%uoq2OwN!b%_osY8HX7@p2e0WQKL4F3+&`66DXofGdabQ3J>I_($7cuX zCKYbuv#Kyzc$_`e9J-5JZ!D^Ra};5&b%p=Yblu@tes3I6N{F;H5YeC{74NxE(oQAq zz4tEdp%Nt`qmYnNN{J@%p8J%R(jwZbZ$nFIYH0rM_+2dr4W5>a-wW_^eK)qNxHSwqumc~T z%Y#uZ;*ri1H{BaT{ilm`a~ZDp8pwy+{>D+ZPsMQiy40g%;pT<8Xq0^%sAn1VChOd` zCf`waG-6~i zUU+(bv#9;;6NKKGB2*7i?&WR$tAw{E{P@5ca^ueJ=sW%xIA`>SP9JJjLuJ{>%O)rao&Xv$qV#=}C*xJGG9S^ehL5?ze`f zCp_e`$D4)ek2+Y~D-Pa!#A9jh9avMsG^?7{lGy_q;L`E+(QEn-wE5f>61Q&^i~rM+ z1qS2g*VVJ-l|FX3zVS|IweURT)wIReo<^#U2zPEmozCeRsx?{5y(#Utz5z~LdxqD0 zAJ$m8n#vX9n#nfKM_}uYnqu+8SNJq-j^M>fnNnIyUD$55>#d)eT@v>?O>tc^_zF5v1DSL8I5hpp?gypMCHV zzfo_u!bcluAME`Mt_F35v9`UW@3>or~|Lthhei+YN3o4Eh~3rTPgYsX4?9?I#Wo4h=Zfjo5W7hlcZ0?{-^*bxzJd?@Z66fqYSG!j&K~Dj$B|0hc;Q( zKuwzwvU&SQ&|`LUG3*86*VSd(A=3{4aSvwpzYQDWimmKpq`G#0*)4n#2e|`*|-qXIT^FkP5s;FJL88VC)aMuq(BB?Iz z%Q*KrQq7CCw04X-x4==Sh0ENvn$B&5k>ZB!+nzy<<4M)K>~h|BxVv#DCJ#=51rrW3 z>fIdDGN>@};~8VAaO6~NdZ%RQbQm@Hk2rB{GXD8pP4+pYBZo}80@3x3RPlU`6}Rw+ zcTZVV-5M?iePg5%XrJS;wD*-B5XOm>kLoB5#xDnrkqs-ZGSZJCs-BnPI5f+gCJ29& zuj3SdPC5)p@2Q#v;vYen#CyHzEGo5gFl6{AQBa-9x#MTR>EP*_q`|qwQQw8e(N(H* z$ggP#q_ISbc_b2FgVKwA9mXQn8&i6Na9^XXZO?9Od<%rtITUZSJ5~UMql~ypo~nHo zO@Bma>34+jyGfIRB+QVFD~_V?*zdS;r-jn4giU@(wWhE_)8XAhp<;Em?_dmXU?thS zTHI^;3asN6f;CpQN2d>*bcCCXj5QN=JDc;?pQ79?mz>5U;f?w6{tFrPDf{6UtgTn^ zQnNCC98%57{gL)S`jXM#OI0U*Uwg>+e_m02ZWb4>RF{PDO6M`JmN__L?|Z1|vKdJi zfXRwoF!iY?<+7C|j0S_8i#c`NYH^yHk><+8j`OAtyBV#{H4|X+4CRFmBIZrg-i7h6a+fBkm7> zCmcS9!0Rp*u&ZbQwrp1pH-b`-&2^H?n{5EX4jf#%ijkg2r2|g)b5Pn8RQx-o{X^ac z;e4%l*mbK0w>e=an^iWc=){yN-s&JjoUg~@M=GPEBt{J)1RPA z&(shW1L-zCF*(=W^27tYcOeT1dtvr}H!;5GH56T{CTBlhg35D|SCT(?1U?FA%_t|z z^GLgobFfAGZs7EOA|oDz4jJub=9W+3p56>Td8`LnYY_KqV$Kgo)WyR-JCW)JES4+* z%2`!C5Vqs?XnOa5bpepx1iQ~Etn5s89@QWauk_XBgHpzW);IvWwy{Bb--bwffL}AH z6vxAQ$srC~phQ1f{9XAUJ_@XX#Pdj8E~@`*jH(U@^Vr-87cjWNZKUh1(nw3k<>cJ& zSsgYtYAr}}VpM$xLH-iodSpRKH8&uy1Wgy`G0Ua#)PuD+e|wJE`ksJ_za~6(+YA)6LGYcmN^855-<9*N zj;g)~g^Ruuwqms(YvlbNt*Gvopu@3~a5!Qc6c{`P(uthS{sK>R`-n8}i5ki?Oquu@ zNk@=hwBe*HwfdT)2x)!Mf6O+qHKqW<@97bCtr2U-ZXsQC8`{k;R+ZsLTmR99=k$~ByDQ(vm%9toR-VZ(>BH4k^7v0DnG z--}iYyYrq0yQsR*OS}Sry`I%Iquz##=9-HwbU+EP1yK(}k8rB-+ z!<6I{B(Kj%*AovWixK-bYO4Q?$4w86;pw?FEU2*qNGr3Jw63lB_Z^Thc#Ql7J2Zq7 znzl}UV)`B@Zg^@TCmv<=8F9}bjC|<|h>UUNk9D6TajC|4>~fYFdPC_B7_?t#hm}qM z;xa9L4x(vx<#Dh=X#_#%RMmGT*|DT)j}zXTaq4aA`%^4#SeT~TuWhiTSt4Q1Pj=Sp z2r8U;n)?awcUvGxha>5Ip}3H=8l1U;a4;eRlg%BKc7yM~-T4c>S}K0f+(iqW9SeZ8 zAk?dvuDl0pedIqJ{K|~=e_2+=H3@@DRIf3Ll{@hiDE;W3M*A&P)I-CZ)-r6w38r|W z>pOjV2JO30KI_J2Jy4oU#f5N&_AP#h(RG(;>t*iLgKdeg8Uv3M>FbsIeco7 z(hQ_!(g zRe8Iwe*2)?p)C09cLpimK>CU>b{Smio1nZ4kXJ*xetycRRCy0I7Z3{XO;$8R(kjY( zx_?-E>QukfHlV-ipddZZ0!j>MP7nZ{-VH-z_YF|{+ahqA*Mj(n_ESI8lEp^vQJ#n9 z4xhD)69?eIh_)Ki!HjaHw4zY;uI4~$?nCt;7d}3%C49~%tV7aJ?ET*!oaPBtwM99{ z4(|`4((5-bywe=n+J#5m4IwVd@aph*u!qO7-@V>3l}TdU}=_k=IJTv zdx$^Y!+%}>!RoR0YW_o?t0k4EbvSMY;dfgQ{_aADBR?7SiBue~yfpb4#o==Bt4OB2 zu8Q%jqG#nz=dhFqZYH^>das5NDJ>=WLKQQunvD@x zGdd^y#Pf2Z25ky+c^sQ9NepXs31+Jv>JCf<|=I$M^t%Z zr7P*WG~Z+*)lr(_Mx}GL)qd6Gu1=SPVa_3%)!Cx*>dr@LuO*o%!tbVF1#iO^Cj~JB zZF4BqC7scD4r}S6&uDhTiVV(+P4}Ilu-8c--en56$j7m_G>%`KR0Any+E-_HpuxXg&o6tf zJmRfQv=3QUZ8cj2#m{v+Woq8|>x&^%%b`;H7L1oo#T~aKj%~3XYPoeq!j-DoW9Yr6 z@ao?PVby*Ge()PjUTG^*ylG}PpvvEr=NhP-gPJjX!`$po9y(xsY&Nr9xDq$-@IcZq z*uB*?M!n5adFL?zFGvp>a>5%e<(rd!!c^1$*lXP=!RGh~>&0nEa|pg}rx$za zZ&l^h2w$leX28zv2Suj0DclX1NVT_$v{XseJb-!?$R`5LgZcSdjis6;)$P==YED!s zE+frmg{*ax)&!>6GwH1cCyNBtie_`{Wa#@Q0e@;9Bh^0^_E@F)#CvI4zh6z9g8{Du)9GlgH|6`=H79X?=YEcIhwHse_U>S-k?ZAZAx#y=ZMu~?5| zN)M9$Y^>r;zF;`cLcB;doxr0lid63k;$t{6usN;eP)+mYbhPOD)&c5ww{EPMA{& zq^rpD=AcJf1~gr=SIsa{&BsV@Lg?LG!dPP<-z^p1D$b<&h|&tmo5Hz}mdX>;Om33g z80&|Gmylj?gdL}~Xh|a|Ud7puE3_VmjZoupW-!urystT~vs&Jj!RERn%(!MOSKZRY!txWj!m*ErKN7^DHlo7KmcVRdtQj`WeY zNJv68=l**_E#CMay%wAzZ&)>lbqde;rsat~iCm{D^VD>(9l)%Fj64 zc?I;C6o@~auCwh)=6sq(4S7B_Pz)=kHPSw;V026{7(PXwpB2d+^3K5R?G5E$qc~{Q z*+c&4KVK|_MC>`T(7pBHMRH!8nY?D5DvmA5$F5zYP|}`*yJ`(YilKahZTX6|{=Bo7 zA?PkCg`tTl>iYQD-Sv4cbAz)#BGGliZQSqk61>v9d8%~5oGd5a{drRipl24G{qDet z7k%XAUmMWlTpIgS(L@G5yaPF|+v!29x^QS-3*P-zAuIIUDW7O)je==QfnvkI)-&Y= z)gQ2X?5=1tZHKU4pUS4i(7G8>SF#VfOoO3~Yis)8OkQjoCtS*s@p5iAEC}q1u49ss z@|NS!!yZEuZ1||of3QdMNGx(*kG+ZlVDj8mU}Reln~bR@i>IGqZ(Ikno6ASR;%Q&u zc6}FFJ7E<}>cp`50u$OFasV8u=Ec+YPDP7&H=OR(3roiY@uhbvfd07z~EgRiW4{vYT}N_W{-4QXRnI$NQww!AYFXU2ZB^Ao?y1$If1x z=`(kj?q@G{Zr%k=+pP=HGOZ5$srygpbV*^smG#i)XCvO+xv}ik?+x54StRPL>c!P{ z87WEYLZN{1ap3+mxFkzGWen*w^B3Xx;O~=q5bldPr6MXGROi3!z0& z+C0%*j~>m7C)eg9b-PH1fH(+E*drF6?t~r&Mey^;OA&Obre;)UKdxfg>6#5s3e=Np zY{GJ!>KS2Li@%!ZPlsS_%csyge;_;#Nk;Q||G~o55x8f0V|jMba_;swov9qmk9fo= z9^$m|e=I%b4?A_~66*Nv!0ohVZp82Ea#ZGXLEq19zK&LLW>e@no}8?7v@Pu_w}xvp zHM1@Q9ZN%XAT|Rc#ZC5@q%S8JPL{D3PKm(_t|9dj(9dz7r2`kG)%k>r4EA4%)7V+* z;)e2tY?k|Bap>q(@HTxV?)3lUK09N){B1oQpS{=O$5n4c$|y_JqxAv_2k?5`v*P&_ znlpIo$}4mA;Cfw8pxT9X8(W}paSgt3-FaLxE?j+v-??%ZFaOe$gXc8jRJ(AXrZIdS zswdpeXkq1-4s2xCV~pY?A12ZMJw=|9zL#f3ZpM`^2WX9mM)FzOc6{k=4L@*zJkb3g zXbN}vnKNS(_(-pj{JJ` z1EhYHC9|C6lYh5>>YTeQ`~ceY%ABEzEo90LV^z!W;?`WW8})_O9iIwc7G= zzjn~!>3mtE?;1IwdvorUc7w&gI4E;K0vb_MB|^ zb3YWf95mFN(%URA-W-ei*8AA2^MfhoqpAO{()xJuOx0dW(LWs8I27(@hH%66HKj{h zJE%6pPky`pO0>9pUl3kkA5DRby{|AiNigT?VGt(YH;AgQPVs7q&fc>#}q{biF zb6W%_yvN1QO9UNL@a=;I#SSBv&Q;t1JKG0po?r%BGKsKBb5wioaAzK&qZJF+bw}br z^jSY0sz=(&Huf##haewVO>2v<%wCKGVs!cKVk5cCSPwM|oOs)ndXo(Xp3P!x7z87wd z8i@t8XThoS+u@znMtqwxh?VasL&aD5?a!gY4#GhojzPkF^jtQGfBI3NeP6RoL)?hR zPJn!0b2j@mGfY@I9isS+k?}8D!R$k6sz11&g*^~9=8&Ec1A{FA6DA<_8XDZnuZls9 zp84?lZ~r zFf@=h$6ZC-DcUo2>>+HIe-sr?9A6ki%QE{xk+Y8M`#lw@Ur^}^&x%tJFvAXrzoF(= z26XOl>B3i3ISXl%jGF;CaVE^!^PROk7s~7ROGD}ls-xMsJt+!R&WW3}?*~PSm)^VC zuF#nfzJYLIk2`+fWCAB3wmtOv+JjTJxjE17Tf_GkDpr37BPWk}t1qf@$~j+^Ii+a17LW5q7Fv@dJI=s=4i5IBD6{E!aIev zMTZ{yfwYQDWpO|n3JIH_OMy9(uEeyxrJ!oU{)(@3jK7N%3pTBxmg?3+>0Mgu%SU|& zqgY9WC-oX7fx^b`hN)OI+ZV>w%h&GyGecf+GQ|enmSe3?o8kD8n)sl`E<2-T7~w5r$vMW1T(o30(;!o)|arp3!^)scD;zs9Cw^zEB0e8Uhe^_krL z=M~{rUKMus-+5AT6}F;h$-nixL8~N9RCxiyd~BZ`1C=WscvsydD7GbSQui#Q>&J2A z!Kfzaopv3mwDZly`_WAVu^N}?onFg4+LJ&ZNQd*Wjn>e;&0mTS*pSID*qJ_7aHNKx zupbnr+2L=dbQ5L_+=noJ8ytGO1?Gny5{90%9>d4I_~e(3(vUgC(QLp76P(si11mhX z;(_k#ut9z#P)y{9fJAXGhYSBx$HeDZ19ARI@fv^vsvtkm%2S|+Qk}kC*4VnjI8~sM#cAb#)u&6(9JhmvdXQYRKG=_Nkxty)giN|VFykJD0ACMN8 zW1ge9QD}*ztI&4(Xg0a*38swN2|j=O^LjVGYwl;(;#3z{ct(dedDH}^jc}4tM;)tr z=xVYju3xhVDjs-?cUA_Zr~Qy>07(Z3!V})fC>lwt0bK{PH@8H}3j}SB0pePvqau{F zyG^y=Oif2d=jJZ&TFH=iZF%wQd%|}50~Ts&D1SN)!0{og8EJjK(!~KvQukuXs~AxF z)OUU>-cnbGD{ZTMuKrVJXja$}^E)nuJy|A7zkt#;giD+_nbE&=FQq+hY|uaw?$erW$#rbvWI7 zrH1@BTzMa_^2Gk>y#V5TX1wDPY-v7U^*gHmpj_kBF;8&C*#j`R+yw^K&?O&3dyPbP zg9sfBDC{Dgh7TXKgIV7;ialN80?5hxd0 z_1($ut}|8tQ)c$oynA8>Z?fJp^2CgE9xJFmT2Rax)j9aS3nwn~p#F)&MBOeYyu?Tk$=>+|f_yyA*|QwTJBitI+&Jk1t@6<{yWpw6Qv`7%Q@xA|$6Vn*dEjV; zZ$KJU9{V;K)?H`_<;AVVU(;($aZ9tvb4Z$;IK{Y1J5oP_&zcVS^z%GMIF9AThd}wM zzTaCyzn3O_el0UlHI;RB6_6e!J}s=m4brq?*^RB_Z9?gD_2@a+wJ`nKe+-@+CVU$Q ziNg=#vCl^s@gbY-*i$Uu;3`N{G15qMK8LVbVJ2Zhyl{OalH^5iz}n!E?BRnWNS>LwA1_AzT_O0Z z)lP*A%#GVXacK+Q@xCEfbAoC!{xH(CaBN9mP`;XS&K1`Y58wxyHCjykp?n$A+$QJz zvv2AcptOSW@*(YpDIY1H{Mw_X`4;b^83^8C16ae{aiIE9)hZoR5Dv+U_UVa7M*^=N+Z0F6>I zM9HiA+##TmB!9ukbC6D>XJd!8z}I{e(5ympH|Ut^4TxB~ThKg@a&4u25qkPIBhQ`( zEisb+>sqmky|f3LLwn)4y`jW9K^pgjTe z-Ba$aC!O_O#g{zx7^;nnY}KNjGUGaV-`2Yn&N9j$T!?RqE(;gHC*NwI=2^t`EU`fr zQZ1>OqBwW>l!o*hqZpIdUJNIu91+jv+Njbvf zT|nN7p7H*xbTa9rBJ4h`GwE#|F|H_?=3(nFet#A`XwA7tKsL>R8gk3_D1b1fSbAoYRx*3TR2+HZou@e7I5>r-E+Yto~|RalAf3@D-^C|&63REKc%rk$8A$VMtDA;^n$^B6r2F(c zW<%4u4~YXd(feNOX(nNhiMQxJ!^SC+yd2W(F~|3vDUx1+NhfWT?ndHq&}-iXs7Cp- zvLIB~X#U{`tMmImg(rMZ)>f@=dJ{%eO>1m6(JVn+{=Lk3-dt{n~JaqmaCJShho*?CHxF9vXa zR8uKZFN2p^B;70BuiabJ84`ZZ1?P<)@L73Rd8YUuCI$z=m~q`DG^mX?Gy3zoB@tLL zBn?w2Hf8Vpv1W%Hc6!Ay*iEzffR6`Yyi*|*d(j?_y=i}rhF97Bxf7t!(Ge>4G~jji z((@$uECIP~$Pch#<4#K38_xbfk!=+$f>I+z*C>Fxtz>ZuIOPrj~MIe9ab?W!lA zGqJ?*3Ru#D+?JF`XYwuxV5moU)xGL%0L#1mP%^55`uI6C5id&A8$aV3kw zZjOd{@5{~oAY~{10TcYtRQ~A0iPn+|iQvIH_EwGfsSWVeDw~-8- z7J>u+#mF(HchNe-3cboKaTU!!nvclh;o-EdQhs;d?80a~ol}i}G4SBO^w*20bFAgh zgc7!7^c5H$(}A0mhk(YqCbux{NoxddNBe`%*;#o(ej1yKv%G4_XHWFx={!&Ejfi56 zbLu(!@|%#SvOW~Z9C_rEj^qk4{Dole?vim&>aw4V0~)^@ss zGi@z+i-Z>PMQL{|bUX|bHviOyd&aFY%My>{blD98Fm+i0EKO03q5v+hBpfrrQAmfOy9(lq~54&MiA|7@e@sxyYk zYjK4tPoV3dl>Xefc)_G%5zYLrx;^qKEcnW;h}FI9mia z@qE}#?A)mbkF~er!+wtD3wEyn&xm&XeU~FzmAA2JN1%IFw=Lp@Hpk?MG+eL9MaJ+vOeviBGg)Pp~GdX{22MB3?GXL?KR zW$b`@Jlb!w_REj;+-E>Fv>=~EHH7f79=6mrL!E2k+@!b&E)Sg~T`X$x33sL=)em3z zzLfGFjD96;;7@uC-}PyMJP}T70_aH`U6RaX>o@fIT~NQMuBuB&Q0hncOTG^R8yq$zGBO{E>IdAC!17U2g)-y z@4t-VwgKMTE$4aT#>4CsSJ_|d0Il8+hL@x4rFs8v!hAYCPqW;Z6}p!(zq4O3adM=b zwMa)eZ(P8${USBjPHcu?_j?Kh#KyQTe40UHxc!BJ<3D5A9d=oB@cL@8`{ZfiZ{1jK zew+^PqIaNg>@VnXdzUC_*H%)kiUIYC1l1g`s~4p0u;w#FhvcC@-Jd_~xEH(5$^pV6 zMt#T5PcH!%3tHQA2d(jM(S#F@K)uLmjB?8d&G-j|Wm;9sjqVJU^A0ZMw$~bBr^QS8 zU-+sp1XbQ2)u#LHZ|ATNC(LBt@)&4hvPndm?bROn-A+2z?hof0E`Y6;*797bR`nqC zzW<-5#hCA)citSv7PY`Xw`xP)MLixhAzY3M>x8>XkIA1|+j7)-J-oV!FM>h*qMj=| z)c7?T=WS>EDxEmhJ9s{hSGdX}&1|K?(=B+YqqXd{wj=hoG~wMRB1~)$ExWzF&*(bT z`H1Guxfn9ho8Q`D#@pvLl_$$O!acK0>bX_CYsZan^7~?>9HHOY#dzUlZ`OZXdpYgG zJ7~Vv5yrF%=dTV#F)v!@FQTx5>AL<9D;MP9j?6!x-{zO-@oX%&ukpJ-Q%0e_~++T%a6z%CZz zX;giF*wL1EiF4#9zkdK%z^lX^{h#8 zaEq%tkNG{lVVe1UHjD+!`0}86p{Aq7A+DeJ`S!KL4lfjEa*&s!u2x6m`O z9Q6_s!12XxFxZyF%Af1QthHTWXH+0kf2g>K^8Eht)|nu@xvHVOJin22d@+o#S+6Sz z6QREKdK`G{5PSAy1MEmWgWH}Skny*dPgtBvH2_JJa$x2071SGd@A z>s*!%&#`@R3H8<~coX;t24&7s{URwwVn}OmxnXrrd3oqlF~?yTtSBz0-j7D%KUlVF zT7&N?ISKR5r=yCO-4_!`V2$~T@)k1Z#Y&8+{YulQK>)4i^B;bE5Tvmd;}i~{iivLs z*DM)O8}-|))`-+T3QKboAAilbjb$#BfIRM4b2RKvC?}19DLP*t6HOL zA#O95rhPKOslz!`J@9x*QzRaPO|M!bemjZv0uR8?^EdH1bAr@q+i;F$EKHwx5$U>x zfA!}`wSdhoJfNPi;AsK1vDey6Rbw(vzROAWiN(a2o-%*+2vJ8ikTY&O zaKn2hP|_{~GTo9zHv@CwVPYy=Y%aUiJv$QU7@+!vG>IHEV;y|+%D~xQc2gep!9ZIN zx7VAh$++zfikA-CvKKa(*zcr z#`K9JIMouQuL_5VgswcY*(AK)u@&kZze{~FN#$Cd8(V8S5sBZ>DDyrP4r$JRZ`#c& zEd>(CfR|ek5JxjSrz1P8@r6DP*MP7{JoK}~pW%tPx-=CXz84_r0!Ufy4ewSz!4pf1 zl|JH=e9sU*?AHEz6bv!7XTYWEOBrFc=;`H;>UgBfaAW9tyig*6>P&uWybAn#N2+rG z50h}ZUrs*QXRDxNV2y>dkuU^v7mVbGA7?@MQZH@9`A#^@<|eGZvJf9=ui|Q#dWUs>SSWTVPO?HW+by|{{stKs{Jf>OM$43<0)Ri{$Of~fze--usuXfpw3k?(5v1ao^G)D8dtZj_|-EziBV=vPe)d z`lYWUsE33)FNMjW&SPXBPkkgV(Gp%`--NdE)s4+nxUqB2R*k}if%mNWmf!YxA|W4> zJRXAUmB%QSYzNt6k9_M?p>!AOosVLKnWQ^P1U~#NsOR|SQ$-k`myO519H;lXGchP2 z0Bnyx(KL?R%Q}@bs;X(*`@`{PR$aK<+g4KFX^$?7Qn8ivol0+G$&_cL1x&&0Q$I;P zi%n-81=4=FqIe!yi$@yLZv5Aynz&%@1@_~bx4bj68BFTeS98+LQRNeAOgW6-7ulij zy;joSI$lsNQN@b1htgm0a9;#;bO~UTTTM85)|(mEC_a^J@rze1)VT!N>JBn$?sfHD zTK7tGKJWV^*uB0M(%&-DxLoynWV5GGdNf>=qX|17HV8+0c#*E_4|ZR+3O;fz{@d3G z2ty=(bCt16wqU4jQzXqTiQgI3mBwn~37~$Jsi8ITxMz|&UwNjyGas6CRCy4tv?O7m z!bPQxac7SmRodIEVK?b_vLhpmku3+Wf+IEOYlFrb^M@h#*s+venjX&{YU8TS*3=7h zP){6t;Sx$aLyt-j1cdoc56n2*5RyObZQ_>kPMEa0MU{qj+v(}!5 z?ZeGEajWRHWigu+mkFfV1@Rn^9)kmRFW9Xeks3O#`djziVSTXoH5=lVquGkL0~$2L zlHqn$98Z3MG{yu@It+h&A)kXAq3?M+qQ&NW#JA0X@R}{zx>jj9c-#9hcG+JB-Q0Ugs&86zv?C|ZhKEot-AA04W=eTqp#;3NNnZ|rJ5&N>Q;erJp4ecm$9#Ce1{W?b>- z@)hTSya+VRj}e6X_}k4D7frcD+Vd8WzhvZ3*qgv{%1iJ@f4q?V5QKWw<>WOO)eWs5 zHvzNK^~Lz!n<22vcA#rvW=8a$RG(3tj>#w|*se8s-7XeLUKbkKokfZ_k`E=lJ(9tZ zSFHWdn>fGOJyE|hIEOs0P#)vd`(?N}@t+`!qem}yvl{{RSj)jb#pu^hgj=AqC=G4| z*DHPs()8l_&%YXnp2!-kZq3I#`Cu-yqctPTMZseaw)kI?cyZT2sPCp)m(+VIw?N)p z%yb^i2@`lv=X`Nav*`_%IN1zFg=G;AnM#GV^gT+$ z;(~2j{FyL@Q4dSfSK9QT0Sd2}andf#*s~GoT(DP48g`!ug0KKK99ao7ZE63<2Gw9n zuCBEFWqRFun<{-k*UE`Bisi=E2_ zeTOh99t=Hgx4^pPfjIMbA|sp=H18nYJ5Q3==h?#>bLBIrzCig#$`Kt$AM2Ec!RJ$h z!Fdpa6&4Aov=8;SyLLdcD*xFdBNe~dW<%%KcNzIrPTqqN=R?lEAV%?090v_FYgqm2 zmy!4Z5-d_w-!u9d{xUmVr4jZg(_RtdZmT+i$**j1afhLj^tm9c;FJgA=Rj0iVbVH3 zAUwp}$(B_!mnHtWg7OIAGed#=u25X~;Y4H=mJlwxkq3i~p*n(kRq0rqqcZ_ahkb!X zQ+h*cTyN+gccbE1!adFXx&JZJT0;4|^P>`R#_a>JtF%2@nfHddGXu1w-=N=ICTNDi z>!l@X>?{rVh*B$FWZRCd8D+#zgd5>V7n)nG+5zOz*wubhCC#OfYDbb@fZjTH#Fl68 zQTe@>9`i{{&s_!5{z6OGaL}0`G`taNBr0|~71-E`_1#13Ay+=OlsN!bD z?Ly72y1D0(4|=F-lP?(%DpcMGHzC;3lJwF(;(@+E*o3oFPO=Za&v9x*HY%S_wIhko zh4RE|{;+%cU@)=p2lB+277!|Eo(1Pz4rq_1#+QiOM-iY!# zQO!ZLYJNa9nsd*s4fRw#y!}NG7a4H+Jz8|HgrIJ;A3JHf5v55Ab6E27Ft~0)dvqIQ z;Xdnqq_d4B^#!)v))MJhV!ET5(wImb%G_rEsG4u7nZ)Glvp_pw7Z3*n$B8N&>RUeGP!}+Y*#gh`xLMXn~&%uX>O=6hWum` zZezQFHO`G=q&IMcJV-N$NF=W<2={UM90Ppg(~A7VTSj^o74MQ(#M-Sv3_LwkOWH@1 zxS&96-2WRl9H|644UnC?p+^SNoM}YAb7Kn@xBSjXL+QYaZ^aBmC#rkQ6JCIrom5#R@eOGyE4w zb?F8)-x8#q#riZiKDb~nJRaZxO7}P1{2B;1w4_&oJd-=kid0+~%{SE?6VArmz(V)g zRorJ~eu=GaI1cu__T^{FPoeW50i?m?)zBz5`&w)G`hlJYx$^^jI$YHrXm9|?kMI$D zKMCuMBvCQ(4Po#pO;0afn)m$>q;-(Al_35D)h84?>3OO})f`3nm$R}5_40hUb*2VC zS(XWk`)J+_#w%I^;i*>XkfUpTkTjp9oM;r!6OJIo842IjtPLA|3?-dnrFlAMH&Fjl zpZ$SXCksW)>7QV^ZjPE;Yw4Ph=0SwL0u|Si2bG5!&lKykmjUrO5MPpqH&r!-v!-A7Qnn`M$E&=g7=IpD%X~u^k z85&038{M;~l1`69nzbQn>|{W25>#J41bLbnme@6ivT6^pmV2-`khFxyKJ;Mqw0i{o z3|?=lhjYeE<@vL(G#ZIaS&Jq*W{CLZQ)O+?850T>CDYwl61^D zjw9}@#PX;0@%^gFyzR{M>}BXM>2+=xw5g#d!zw;QnfDgpQI5FRvlyDBc|*P8>bU%~ zC;C{Pg0wz#@9$ItdHwQycvsp9I_|TAOTnMnmY$R4p{ZA}q26COaPu@g-uRr2AGZ-F z3w!C56~#kV^@rMtv+-y5T2jaDE_ToE4)6XnhSXQpu*kimJmhx-7Q8KB<_}-840|2^ zcTXC->G2i1j?j~-uL6X*Lw(-2P78K9sWUWe{2Xup4dLPMU3liTKdeqKQFUx}4p+P{ zYp>VcMQdYk*Y>TGqSZh02-E7XmcJYf#b1=W-`aGoVQm3|NI#NNSza~WtCSAx4QIbx3+o7t#0 zv<}O|4ak1$upiDZuyAiM4w&Cd#@qZxAKfB`ntd|Xq?SDOBS_^$?D!lg6J{B~*fp&s z#SXTNza(sL^ptKsbzs)E`S`%(k!U;cv9_0ABk+0s4qJZzjm>XP!@8f_@bo$YV!!U= zPrclMYLq4Yj)CrFVK~7w3znL_!py&yuxkr9L3Iqr6F0%xj1X=8T}`B_g{)r=^7dbP zmubKzG&KGv<}NyjwDTmI#2(Vl9eIJBZ*x;Tl=I}2p}si2jlg-}!TnQd4K2IcSo-J! z>I78RoS2!w)p4|2>hj!Uhd9+VwEhILS!gS^;O#(g`O#J!x#!F)^i4#|F=p9w_HW}V zS9_w)!OlJP#EGWm_{6mW$4q~L{^9>&%Eu`XbfZM&7fyC?;g^!{qFy5xSy6PJPfD4@ zla9SY^M}*fjQTA_{ap#z$jcZ$wpxsDW=F_1(_8YJ^Q|cMdU!i)H6LNq3hNJ<2WQuG zx1Rju$NXRcEGUSQ}L2l<-4#3}Pei2A4e!O)|xw4XFc5`Mt2Q60GJ{a||L zWCv_nei+^zpUeJ8XMXX)Q_X`>v~Tjp2!M?CJfor`zj$^HT-8~rxz=tksJd?&-j}I7 zH-6$KzxfZvW}_$YJrh@O>Rai$bSZCTd=6t;o5H#$-q=wF@`G_Fq5NqKj$2uar?72M zJ8K}Wt(OCS{}$tet`0zb&o*_eBllaG(7h^saQ-`AcIh#WC#MF1$=aUW=xq)TAHEIh zpVF1if8T2&-bR7p$hJ}%Xshstuu%307zMKq57E}?#kj?m??^o$tcNC{iVO97Ro?fF zXb6NC;^~@de1fNrJW#EsoHM2rmzi8biZv`98^hlpKAbZ(pb48lI8b)tZv}lHtXUU^ zorfHS^llqraGnFVyz&K0zqOKtM>6NY4kVm}UVd?CQ@v2^U7X0g4t`WM&6yuUOO7WZ8Qv4_4wY688V*KiZtkYY#AMxQ~#RUmv3gx74{rCCr+ z>n%MhcHsxg_JVz%xp;n=4tLG9LmR7(^2xfr82nU6b7$=*ZTjPd+WdqDn0u_X`#jsx zl5h!H)_I1yGyLI0#DN@Ds}V)%SQ8Ec^*nP7dM!-))BT@-QEbTh8uIm~Xifbg<=}Ph zHu^QZFLv&U=6379VeP~+DBN39)`V74VMw!-t?JrvON5#Hnspip*Ld|enw+bdo9V=f=_3`uZSdb^h%@+4e{2{2%;oge&5;Bf4s!>MV#ugTMp_lPS zLGhCQ4@|1AuK}&iLN%R3xXwRB(EQqXolLrN4PFoJ#|iVrtZnbnty!K{al};@+MiW> zO1R{e;=hgdP}8=DBs{?HYfs{$oloFE_9%=!v=j(y#IgAgq2K;O;tvbZG^`F(H?S=> zfY*9|pV^g$sJ~|vNA6;_9k+fOjMV%5MALalIK-%aVCdLl_O$&_NjMKVW8VmhkGSAj zU#k?^>N6mYvAoZR_Zl6j=%4Bgq*D>gyr*lj4%L;U>NUKy)NGndIl#ad&*y5HJEwL zPpY^)t5+bh&o{-xj$Nrv9XRosbS#P$SNo+>pBk|Ki(Y~?(?G9&1375|oHipCyOpQY zbv_cy?>|KIul-Ez42$mv&UXHoBJu&a3scip%J_fZ`(9^LVsBti$8>b!3GhcA#_u^@N6S0#?VH zbHYgC^4E;GK=m9OZr4{T4v1+Tjwc3o1~L4g9Q5KEQvN8HlhEDOh*KZojCNrvhpO-K z;fKzhYP}z&I8k9aT8%ivh)X1K6DOR2z@Z&Etr!kLnctx0m`MEZq%n3Kv4Nj?X~7?u zjAOPs5rh|qMDKQ9Nbw`wEmm4kyj=VdKYyAj6`tt0jo>xL+(5CwM6SQkSK~eSEUf=l zmlIbr>On@wgSILrNk-B1vRKh$&f9AFi%?Q2aN^pbC?g4t^rOPjtlE7ms4SnOBgw6mIsO z3TN!hAZ?+98Eeb5W^J4~%^UjkJ*YII!fuV4r^TmzIIQJ#bg#J{jwjCN1(WVDrJdpI zMrd<$1l7qpxOSnthWc8R7QYii4o#Hlc4bIC17-I+%c0}u3&M7oqnCrSq8wx0PpLhY zr0a2KtJyH;$_&ih?1a>>k~EXJo-|xbT2EeFL}w(p{Dqydx=K4}jFZ}EltwHxxquJp zJN*UYj-mB$E=iBFxuL%pc@a_LOd7;}o(F30)V{=x)MunM@RxB@{>wNOsn%2tXh?h0 z8in)Ry}7v5B}DOscze*3yh{t>e^*+Y7s7A+F7|H8V@4XdN(+#GVVZwUCFu_QI_*8U zmlrFH!TA%fqks5TaX6)>B&-%|^ZMerR>|}Z$0*IBli!%>;t239Z6Ms@W1)NHThJ>$ zMdzs3K)a0$##hqeZY>@#$H_aX)-1#{i)b8jz#aFLZ3EJ!?DZfcM)k^8vcayex)x!> zAC39Nd}BV~<`xVeF-&o@m{@Mat{wY@(N9yM{y+ydb8c(F&O39T1c_eGpryS8&z&9Q zzlaQ^8h6#1s>4T5nJfq|l~&-!lOI;?4{1zE*u>g2kK#{F;^|rks2^5PZ46Qz$DD4} zLcVT4?bmDJ)wvOt-6!lkUnxH4jnz!8w+E&)=&MOOeH#;2jz!Yk`)kokKL;RjkbC7jlgj1_O$fJM_Y|+O9<`3JXO*>p_71{!rK8eJ#PdqeyV6>bw|uOf_A?DGw5u*U z-_67itt-h7^kLMWFysN9J5TAT` zLh_mlZ+J@&PmJucRy6#PB4EP>=(Y6(JZ|p@1#ty67{n4!3va?F}Dy@HNU4*1L)3YWWk=8`3G#st5HmaJv8UviU zJ%RL~9vqq0ND?;U(|t<>&5J3GWNzFM$zRClC$pyD zT|v0U`&yiVK>_wK{eB9#Xa>OAH|Bi9ucx^ENw!dWV)ML!svahcffKsc5aGRnu%Wsf z7_|?ScM39_D#$0n(4bs4WxzUc)h!V9B96nGPBn#_-znvhWP9geg_H8r$CtR(rGU;I z)aSzJCsN&r-koM3^*>?SE7w8OYF26Tljq~n!2T)QoSsvqB}wy%sg8Bz8qif7tronxoNGqEG>2RRF!F5LrL|>11ZO*gDKpcUzrm7xU z{?YS5eISU-kh}ubTPa={vc;A5Th1ER0+l!Wa&c$X&xym(%*-49T(`vR*<)Z`s2dv0 zwE+4&jBKj|0(~CgpqhvKOnD1vrT!5ciq~4*HS)! z%F}wHC2N;jN`CMo8=;|dv5aHL4~^jUrS8+aPqPwOerXO-dt*zkBc zDxSS-OJ`Nly(3D4(^&Zn zcyKgMkjGNF1E@MApA5}j_h;9h*(p6s`BoUc$399B=I8kybprZ4y6^cfga>Nqyu1c5 z(qmDA=AHeOGIAgNrksv;%GmKU$-vXprG>fkTC+tJwR;GA5*{7&0 zWgm=k3QlWQl@U_DQB`ZCADRD*<02r#Lel5qgd6*mH{g_+5SBEQIaUYQzP=t1V(F^- z3zz@sOde$&p19AZf%K}%WO(PhtyK;nR0fq7 zR0b*^BMyV!{!TI>r6;TV-bgA9qcj|00Vq!}Fx8YRKTlk&r7RJN=Yc#gk_V%_?-kJe zfIJ>r7@5hkGi9!ho^CK}b}djjl=}N`lf8uDwFO98R73ue?vuHIOCsofs4Fh2Ycb^; zJDJtxBvpMA_2MMVYJMB350LUNuIhk%og~dg z=eaF_oxWY!PqXf-hA3MsL*gdNE^Tmli*s!MSVKaO{=QWismx|l%*C&I# z`W`8ZV4(|tR@E_arzrZ60Sapkx}6iB-|P{uUoV8fmyV*feM=xeM0jqA^__}cv-6B3 z=`MUX;g9mmFf+!O6aH|*cjB6o03&GpH@6ewNHDUYRI?FYt(;@P7~Hl#HrtoC?JS|XeoT`7kE_kS+RZxO!$%?C^! zt_!MhHAkWJkMhCS&o#wvJ!IEOk;q7cNa81)cP>WtBcpZTO;#!G{}QPF;Q1~MxHil{ z(wj(q$u*_dS;n2uTYD+Fbec=l~gsW68)j1}i@RoO5)1$_~O2iUB72g-^*s~jC2{XSrA z&orp%7Xx>k+8{M9?61)hoi#PUHDQ5eR=!aw7j6` zk?>`eMuvorfE6=Md4Bj_2)+PZ+pUQ8T96JKdUk`NL?@XOvkwm@TcD}W8NPP)LuS~f zgIv1gC`-`Yj_3Wi!LS|`FnHJ|P407R-tPQa@c(oUXCF0@{$edVYVj5>pY4DiO-71$ zuckoPXSv!Id0fs~vQ8f8)A0mMFPj~~K6Qu}b1ntr zvWcx(WqK`oHr1K8fAxhut!yjX>DS_`x7QXfb5GFo_U4+YR@LOM!(Q;tz7?B1rI6K* z+|GARv*OhYN8!3d_2i^b3wo|M0}3;j;_)ZfVPJ>Hc<${uDGn`T4}#9)*mxH?xM-04 zYZfksj$v}ebWfT7aU{|>G4;R#OxxrjGv{bUUB7lR$a_2V`xL>?Z%gLSrrLwg8$G$~ zU3(lgD2ekMb)iSPMn1ByDSzAd7hAsHf+FAJxOQku-Z9J`CmLSE2%D)8Hs}x1+qc>6 z@ORiHHV(RXI0^0a&fu?{_2_#36=vL}_jdnYBkuX!#Ik@Md{_G|ptGwjx3#zAf9U>= znHQ{Od;8-!sZIk~@7M@!}$xD!t(ww1Ie@^u4xw_|+^X#XmRQ;kTs0VZ--uR}u3ze4Ae zxW9I@9D1pXSWtH?#5vBD&a;=ewrMn&_Td-!M%R^xYM+PhQ-dXq6OP_Jg6}HY$ht|b zMfV>y8T~$M?r|FXr?~Q&{4%D0ya{(7_=!`4FF{hP58_5;E&jrD1lPIy6w1IB0JLA7FS4L|o#WH>Tb@(Da ztS}%>c!Y&IwfU~eSFzsmZPI0~8`OIH4lnsQbLtxq$5x_+?@P9^cR2WrTEy4v43LMe zTmZE`l{0EnZ;Y2uJe88$mb_!?b4*^&HYn4ZiIkLGU1Wsp}=f^ySGM)e6A z{e2i!^DZ3fP46PgSs@*IMDk-@cVhh=YZ*IyA1-*V2jBdG__Id?{@MID+S+fB-STU3 z>IuwCG{=p{^RNZ#VBZiU`FFqypn1yA^;gOdmq!EDN#2xuVH%H@HmV0f)uP{!uZ(Jr zn=!g)%!_M{os!gEf`i*BF>$kv7-o0`c5S}|)OVWVWv$_qPa?M7GfYG44z_-Sc<(Wr zVgB^JK=Z{;Uz_5Tx`>Crh4Ez*Gcmeniq<1*2d8Uctq+H_FaMbH9(U%8Z;vN4RVP$S znDp@&Zi(87clWfA7auUbtwdi^kMgUp(o|pKrI`0Zm&5{Oynt$_U%TQ}jN4~S)gF_Y>3c?Sb9d3Y6KTMVV za-*@qoj6eaMt#EFhaASg%Y#Jl#cJ~QNk^*5Jh8gZGIUuG2GjEAh?-5$F*7|KDS~^- z-2;oUeTA#01%`@|sZFGFwP+{{D1c61Q`s%^u1IT*Ll*R={&f)vuvM&>egQ)t8uIa3 zlacBIXSXs`xFR=Ov?dHU#+%zXk6iZ%8mTCic|CTesna_1x0KmeV}o#)mU7VVo`Ou4f`m z8uUf#O=eZO35G0+LEGlu)YD&t8Po8ni)pSjM(qpF8#wmdQMhKbK~#@x%1_0V!kFEs zwDVJ+u9j>xxtRxIpk`H(=^LZ)CY^_vtaSaQ%D#m4T3+j;UNdaYKtgA zYayL~E=R&7bSB?;kiI+LVf)_o>4z6=TS5$&p3TQI`}1W=V^f&it`FAOUj|Pr_h~HM z??FQG4t8?d9n}MzFp^V^0D#c=nrEu(eAZj~W8J#ZhseikK$@Hjzu z0OPnLQqO8~3R?;Dd6jGluf)mIqcLLdOuX~69Vg7JpDl(`Il2((r>GG#O#pEE|(^fTlYddI5j zJNVppw(#FZcIBWQ-&0tVA6R$~h#$o*lh2U<`5`v&ie`orUSZg3H(2<4h&-$$iG;^_YTb)y0}KIbm=(ie6rW+3J^-6XCbD~59)n)8-kZ3Ok6pudOI z1E0Z6FO=5lCOTezou~LWanowT?L39kK>eT{vY?2m{%u-eCHqzUM>Sk5XdSf&C)5*H z4(_7-#Ffw3+X`tO;QXg2Y*{%@X%*(=Cm>rOU0(GGoXoE zQKw_VJxd6?sywF1`L4wM%n*MeTq2GZIddBpufeI{`~be|z%_M5&uddYCG zIloIh&1(;Pt>1}#Yo?*9E!6|-7H(GcA;Qlvr3ZLxr#s+P|Av~MYo`v|v`QlpKhpVP zwczgTmiT3ejwI~GW$&^ystyQ)p~mw8xVq189G=@oUWz+{ZJuq!MF)&!p-wD>c^81f zE~-P>+G(`Xj2iWnpP(tDvTm*YKoy5aoRfplfZo5eEd;r zg9gjGiftuPs5Alfwn+GQ8Iu;a<(cdY}rq$~@ zl=OfBS3JLIayEP6)sT}X!cy6f#fq;Qx}LhGY||tW`;>Zd(nIoOz;2_MAElipl$%(zASF~rmse~&+ltKJPJ9=VTwd-T`#Zq|r(PTxd* zzC-mb|6{aNWFP-e7z8gs(wVeROSoI#65^2sa9GxmTj$j0V~YY^2}hYY^cxfg(-`o_ z`P;%{-#x__NFE0x*G(YbbR3r1O%v-|Bw*s$HP}RNryyMe9XuvO%w!W9OCO+hV&or? zJR+1q__|N&MKt^S9S>bI zl|8GqsjBNygRFRk&OFh8&d{VM8_9nFafn90 z>?HJ`QjNQwpU$rtS#qjB$Q|L0Uw?kk)J*ut6wXs^!mp+EIQa-D*wskZEt`l-_#(g)%;_AS14vjcB5n79Uqgu@{cSj5rj~B& z!Xa6wk+d8m>=1;DU>tBsVIbjzrNVrne82#+qi`qS3A4W$0B36i(0A||=w81IkVlf# z4>;qg9X+e#$i-U@b;}+LT01)GU+^;IpER{Tbb^PkdMPdwp7;X)+x!?h=8Z%0OCqW5 zQEiCnj;en<=21S&?zb99r)Y_Lxb?S1+N<;q$k3hVuwGs=_PJz18YmsA1!SrI6XroK zNF0siuSh?gz~BprQ>|8rn>UpvZKiZT`Jw?x8c*p}3>z9DcsbR*Rhou8s-StQd;>3p zdm`yDc5v(!F>BEvxavC!sNOMX^;V(0!mHWmV1G#_ny#&ho_%M*`qvgfIsm&2p)**= z9s=?w|0Hpgh0w$1OfA1Sh&jfpoVFtiz;h=>cZH<1n<@x0zW-_7FkQH-+`LZ>(=GV?mh- zS6ub`y(uR=WWEuD`Q9f-Vf#Wsn7>i!4^&!Zd$%xc+LgV6IEwLqY2&FkGIH^ zYCqN9h(Vq0In^LOO8o9jo>D&0O9aY1c*nk(_mhOzkoCB{V&} zg=$HUm0T;=-15vJ99WB_7pT^zvf(*XpvEf`BwS?A{2v32OW|MF;L&t;E%Ew)*PBD5 zJ)NOv(SAnzEDqJ~Pg;3A@#afLeF)@3pqI-NW;@=Tb^Ce>M|Un}Vb;;A=48yo60kn$ z#{Fz`@}}HA!N}txd2O`wpIeoSRi>{Jl(X=I=|(u&v6+^*1Ski9&*satgt>D3n+(vuyprlrD^zYo z+{hGuPv7m&XCI9RuSY3h_IEL+SbK8P3aIm74^pq7pG_!4)k@V8zjERf%Gg$+@+R!? zSav-x7S#Av9-uM{eR{{Gs$0^@NSPm!w$PGi0lj(ND%(NIkC1d7kQPwagOdSG=<_b% z^P6-o$}v6agRi8Gbr`KH&b+V%FV_3Q$TzDj&{d7qy?&1NEuAme4Lz_Z@u=2#b|O+v zq4Fenzoi*pecgj?{0}G#8mu&`AYSKN3Q`HHD}|~>%6DMmMqfO7w=?m0BU$DY%_vut zlwWFzqhLnEaX4mgB;{!vz|Q|8JXto2va>CyvQm}N{57jCDTCuGpWL$Y0g`r=$GSF` zhd-YXgs&RXl8ola2>ZpK8xa^gP*3;`S%{q+eAQmEyu@|bui}qJ=`+GiSpMZQBdsHq z=OO)|G!eLa%?7G*u5~x$lppfi{qD1IngHCi^|i_YK=u5muV#|?3O8-fVned^Ir$}2 z*f8YTZzRlRq@|Ug0`jA<>fI&c*0BnE1Zg%-f5r*T#&hyaOlcbHRT1QKbG4+Yfv^hx z4yXpib1Fl^j@VqDHE^Wd%LbHY63e1g4$4S7A!VLO+Ju=js!cjG2?;YW&hewd5b>$9 zR8zh1mh#?$GF3sDrt)s$=2ShT>^aXNXelahN!}4D7giYsZ{Am5D4lQUe*mok8t|qU{guk%dM*1{Eaka`VtFkiJ^@osf zQ>>hS1gG3sQsq09?rPv=0P7Y~j^@Q-M+-VbE%AF^nKV;QH{#;9&e#D*T-z8+}wsd#%*}J_>j_tc($yg z@)XJkRAsw${pgG}yWT*02fIbo=3WhxfpW^c${lAkyX(aR`4*=9o{q~sthZJlbiNt$ zY4^C{A+`F z>i)^N^R)@KcDEIy2YrMoQf#Wdk3|VroFuR(Xuyqx~(0rmUxlNyE799#eVcrH{6_F+x_6-O?4B=2>s<}dx{HGxp5_5xzGd*u4ai^R2f;BMj8|S>g@{YL z`JsOFoaN8i(n(WY2EKR&3Ga`ARi9!^YWo!iM;_4Jd|HS{ZL-;)xohPbohvxld72#8 zyo>xDlg;i`yGPGZcA<9<#6#A)cKl)47WtLVxcJ!Y2^`Jp#c7VrJRk(yReOm=4Gdw( zpY7VsKRV-*R;$r9KNMbk?8=7xX~HX4ILYaS1K9OhSH#{r?Lm6o!9W*BDRe@`c=e}2QsQ6nX{_{W{k zocIQsdV9zXiFdVBKS*PM>4hb5l6RLU<7!avJ)`|59U7*iasGVH1`$*O%38klw580DfO|_~nl4!O_iJdcCLff_mJB zO#6$dsd*g_weHOWcQ=$-%}%ljhIM&Mrw!2kaT9qe(TFd;oX?_WpGDIaZ8&`{Ca)bR zx4qkjcCwjvc-S`j{4Lt~9t`PsB&|K`F=`#{XN;KHtT89tkYmUE=W62jNqagufq$I0 z2X^fkjB!5)Fy|h%xc32HC^TP*oo1c^s%fDao(wIn_vEox>3JH@^*EwIF18CSXO$i1 zxYD>mFJ~C)Rg88zznMQ7^GU44_+|d>xQ8uOyEgxT8e(*&f?2d3;E(>uJDhpEk|SykhJEcKbElT zaX;8Y$18iT8#ZyP=UAu$ISgX;e(6s!iw(Y7fkb0Ni z#kV?KW}EJkPiAVh?FKu*@Gu)rV-rnx81mQu3Xy6Zt7lyV!U|bCe+c?TSaPrT>*4*r z584UZL2|BJ7g>GH9?+}NP&{4#0!6JB<>@t?r zBiO1W3*!1GfXCnCqz%?#QSfrv;#ebCY+FO_Zqkp3ly<;Fn>+EeC$&_~YYDr-$#*Xj zjsamPSJ>G5*+#Z&LYPb{sU~_woq=n;?c`3w0j zBX+o8YouI&2k>`HM?SgRK(6*JZbnzpHqS!1((&bYSG{KIr)07hp%i(A{)i`mHcoFIe^X(F`QKQE3E{{oINNkBL>{jx z^>=E;QJapOxQEUYe}-PmceC}q!*H5ICn!4Ci_c(g3M=65kLySr#gj(Yl{4Nf#=%o+ zLh)38LAWkTx}<53mIZ)i154rE_$-KoL^P!DH+v5Yms>|fAgz(AMO<{=Rq+sw*+Dui zx~*vhN7(tl|4I6{7&;(RalUr#;$G7I(n_G(7B`1x!_C6RP?6%F7j(KaJgq(y6HM-l z-Z|febnz-9< zs~A_^SkgRTfaaU19A8J=8hubchM3`hu2DmYS1( zFY;*ZaP7#(^2(kFae3||(AnJ&n{{gg^w~UeY7c%opb}GK-6Yi)9(5b(x^3GnAPff8 zm!Dp3$y;n2!IA^*kv;=oMqk0-nhG5ITVE|q(bFRn>H?EJ(EuVTJ@tm9!xJVLb z193mI`5CNukX>w32iABex%%{4fY0d+*=xtlWa6xYu=!T3c+%(#EL!pz32&jn2@UJ? z`zHA6H>UISGqr@Pve#4_ehe>(D_uA9ZDUsph1&){Bj8=Rfh1ih6^Gn5UySSN-j@MC zTS?1vUht-73Clb-QE`T-?C@Tc9kJ%s%N8pBfv{5zF|I$IX%rU$CVT*FkLb!`Cw0R2 z0nIpJD5I}J{qkmcrQC16=CQMgCdK z7VfN##O+M+wBIgdHMSn5wMho*6+W)|8q&b4VM2e(Iy~Y4`(6_?SH59E7wJBJ6B_L8 zu5_JHb8WTs6>c3-L*6;8i=^%0-0@7|($|E2%3G$^3u)iPH0gmQL4h#GG)xql>&d&* zPvhe3O*pA*A9?aXJ2<{9jP>4d8TraVsJ9_YbRSZi2f7@TXAkXUX+8gSRlH%Gkxg}z zz^wZ8B@WmD$JaT1*077d!BC?u{> z`VYyYC|s58U1EWHK-l!p2O1ymaBUy1DX5NZZ)vO6&edxJs9L*oI1EOd`vjY=L_&l6 zkMdsZn6KUIa82ZP?y0aiX{ABFy!1a=N4XmfjrhHwT({4m9KCpW{tmtk;vfTxU$83#tK@@U;$ z!EN7CnC~I*q46u`ec(3Xld<9gc=Edx^Pew}Yd5{Yr%&2(+IRdibv;+TPhN#loikN$ z6L&Zuc_Li4a5B*Uhu05M@L|X`eCo5Adh!I4w&OPoQ`8tZ=`yI-{gm<~Fmml)4PiK# z2mD2v4?EOv7AI{Ege6#ha5Z}B9K_ttr@?CK93ZbP2%lm3%DKGy>`>z2j5j`dA=;&&apHoRi2}>Mf!GfaYC)F()o(RNPD*7&X>2UngR0aq(S#WM*W_!ZLARzZi@Li zyHV-(RxBCFLvrFlNIvu#3p?$Dkxkv8Wx9^4V&)`;X2{mV-@9kR&Z0VB9HvO z*!rSAVaaMbj~0+PO+1RYBOXmYilM)rz&rm&g6f1d*k&PV4>g-torS$AJtX_DuTWaW z!)Bb)G|+p4H~J*zVa%BQD0*B(=LsD|QppO1<)nvuDG$!b18QH#(|JeKUv9rnqFzuG zkVj{Edvk#HQDG?*Wv0NK!IyCH$LFAQ(zUM-km^qPYStlVC0vgSR=SqGEe#{BeV>uG zLekL6H^7s-_Qa82;n8F#PWqUU#;ej6gkfUMqrE^LmkoSbM`UiKdYi`xtT`Z%LvPQw&(M6T9OvFrdmr>SqvW>>7e?R{);D{SE5<= z+*|oh@#tDA#w@$aoIjox8Pg18xT`Ucj#qmMp4rp2q`8oC03d&>bP3M?@&$Xmvcpzy zu2G#O!M?)TNWNMSma+l2Lvdf>J4QGRl^>d_te|R-6qeICC`bA!$h&EB!=q7gbrn|{ z{33k#RkeSlB~bZo%Dfalf;5Q(PuuIDbQ$>ywa=6}_2dmJ-m0EvLyg+;nd?@G%;zhV zr-bwEw^CMg7_ar-#VEr9>J^xE>zTdf`-AcGe~fe% ztg7({1FfQ2=>-O5;lBm(rc8UX3YB*7STg|g>#fI^^`{UwoYGL;DV+)HuW!JiZLgw1 z%x3EQiKz6EcU)&4Ht#ysrz?%Q5vObgN#mlU+dB9(DhiYbx0`hjEHn2Anunx$!Q_h% zLEmCGD9@?*m+~+T>0TCF>4D@0;bNOOy088XJbc$f9=p2;v*tx0^V{J{I$29~4jIM& zkzOD#Y3K{4y)HBIlvTc)`b|Ug0@AQ-x>K%(@Pl|G5vrY>A-?Q>BoxL}`(OaX59GTK zBdrx9U#X=WLS;{&v}XIW+d=Ux<%skQ%0TTRO$L&u71T?D`?r)emer}sHm=n8f-)YYb=A^mQ?_#yJ~o>qT5Md<^3pJP zN@SHUB#lEHQwD?|lpWDonE7-z^geyXmAHw{QB1m%zwXf8Rng=dAoOCY~jT(s|e!Y-zi@rzKmrBcc&TWXV4=x8*8b{tQ z9K(h<0_9i4c^x5beoB=dA#cZ29zgj6E*pL#Pvx5G&y-Knxu9IO8+<^J1}9yAFK_PH zyZF`Z7m!b3(xi@Dq_LNjgNf|au0URjsqs%5`40Eez5djHl>e9^<(^gk|5Gh@skEia znC=Dk1zvX|_??+ll~a>XQ(i=)6ZA*S{Bs`K-)P2bho-?G|K1{QBc0>!UZ`ab zi@5E310Hj$56@s-c-Y2DJeqPJr^N5U|FkjCD*7!3gfkqqdkwpF=r(?tV98tf(OI5X zUc#pDi=n0U5i#P*7>EpLfZqOE9DhF!e#hqHtI8PJV*g4$+i4`xW&^lO= zl?e-<1>%Z;j$+=Q!*H>@7Jom%K_rE|f?Km5W4fLV{}DR?utFc!j7x=+Mz6%&?(}Sd z?_XH*$5@6stpVFqrl~>qHwQS)h7#XMU?pa-bkR2aHtZ<87)EDaxCUbMxUTZKvk6{C zA2bOnzy{XuMdh6iGVh@q&al*x)rPOtN?&)_UTQAopjocw_AlAITG4PYCly}WjN;~< zIJemE32GhepOCM5wh~Vbx+cbY4n@!VkKlsyFYK9LUv{>xPWM6VlpRV6@Y$?Rc(&XD z1}D>*G||KOUcE>@-7_83xbFUI2pjqg;d#wnxbc2}p!JsVF^}25J1ZD{K94pHpmEXJ zWg)frx#9~L_HhGTF5V*_+sws%EJZf9?}`Nvtu=pF+-C!KpFo5DNvy;u4-yMz$?An} z^8K)zu*b0+3YuHU^zX$mFY6iRJv;$lhIN;;ZoJv_O>9Tb7`9E%86r#n!bf^vH?1vh zV1LANCu@kkCGaf0V<5881sETH61;57M1pA_*&|wqZ`E^`h0XP0cD$Zczf1GL2biP@ zA9qRfuOAcXdfVg~-)$Jx$`%~_EKvVa6u#Q@63lWBu*Hk^%BJ@F;aM9E-;r|{kA7bP zhvv1CH=RpFtwMWwto%#Y$VQ%WwC8yIy(tm*g=~=P(^iNY`fcT_7dh}PT}Qq?WG*vU z4whpd{MpY3f6#eUXWSRiwW`U~ishKpNDoi?9>cAsRlYXQ{D-5`7C*TxBdeySef7lWs=q5fHr zJ5Nt@CHPL&TpR2^3e$@6F?nMYZ>{?uk9yWh2DkF#t)H04`-j|NZ}a)!gB1+5oWN)fa#7YmaIj9sUp5g~J@*w}{nrf^I=S)Z&hbcljrkAl zX?}(BR@ZTj&j73yG)fxvH{va)uRt+nrx+jpM#OBqD3&%_Ci=fVg*|QmvPbC? z=nTT6s)k@`K)))Cs2h}tpZf*Kfv=sZ*NjEP$6849Wd5}Z*-q9&mKSeDzbn1i#G(=6 zNvX=BS)QfAVy?SUKe-<3YY`lMB;~O*N@ywQd-L2-5aE8-b$kyKHS>vD@ z@}YA}=HlThEq*mX!Vz)O*BM-+?Nt9^?Lup?+24p+2PI&w!VJ~3`07O$%%=0j_Jvr= z2WSPokGNp8&m>hV(yTCyO$+ZM^)Kn;HlOPBvIkGz@O>)YLPt8hbp>=T>B4D@{L_RT z_$i&z#CM&0IfOk)AEC{!`~?w2`{~55nsSXMk{;wZD-j%8fSg zF_s*^Ui8xXUKxzN{a#dZe5Ui@{@1qd@_gwK2JLEjL(7qJ4~-RTQ=oV#%s z?=|d#)f$DZ>{EIJoL1D06Grks!x~{|EBc;)UUzv553;giWlB zY3+FJPS^0f`(2>x;?!DQ(at&_SC88dDQuLM@Pk{t??u<#B`PnD#`~38a4yLdv@de5 z({*?~YYTi%ufyg$oyWni+sJ;l&xKF2H+0u+%oXOSIw4LG=ZZb~eRl(N_o&5boJbhK z4u0Q+cmE}0X+jFronDUDoJU~c!!=U%%jn&@SUa?tv@7|DpKAWoY>IIciVHt(>Lmxd zu9Vpp6G`J>AA{EYi&Xnf-}gxp#LL0Pw8a?t?j=Gq-P(|1a@BWe>qtkTZ$ ztj!A^Zo!3%ob%F3BQZ8WM-qSFs#Y7sg6`XqaF#fs2c8*p4wqyVfd5D{9Ob_ezQ*1F zhmwKd6`0K3zY4-pD^0~Pf7-ulJb%yx^!VzC2Qy3f(7at5hu#MVZc`_ zx|j9=q?B$$OShW2v?h|S0XzCMJ_#_Y!z((*r#n9AUHGAhibk z$iwSM7>1AA6rrjQqyDS$yG??2Co6!rv)165lzQAEb&>ec#t@#r=ph@XUd4q);Yj#^ zsUc^?dY{gGwa*QOH%LE+*XG}Gs4b-_k>Y_!{jK%hcZ?ah#U)0qo7m)f7H7 zmRmynV0TU-Qcq|J7v;Sdlf{gpBg`+J&TDXOCDTf4Vk5shUBfoc#}3veocayFoc^gW zlc(nlMyffOqb4^av$Idc3lV+c*gU#F^~h=By1H`Y>zcBSUlYyE7yn`Kv!~+4 zEPKAUd8P0UjIYvB*9Yl9JCEr|d;r9o=xe(TXVls++;`8$sg`k=6|zdNBZ&hW3{G-roh!?o_Z*{`G*cnGL%y+4w8DkeBcgNZUxl0P0t3wB2vY3Coc5 z5fBzaHH-S3c$F*cyjXlx)_=br2t#4~{W?6ck*T~~+zL&eEJa!uNj-w3^>{|b3#4^G z`W)FfC<0qQiO;J@?}#pmkw93F0r6|$h+_*uSjPo=gf5E2G5#^w zBGr*k={{PFu-&VTdbS5%RT{Dvhql4*JGEusBm02x1lp90lDi8U2;v1zkLXbJwwxut zn>I(csqs%A-3y6$p(2p1y|N(b%2J`R0w`7=+T`e)R8_db8R zZ^ev0hH|Tq38q&36VxMGm#=RKk5^$#%n~G=g*)zvG9oKMX`iaGKdEfSOa4vL+F5ra z+$~h=qWXw6PrR+6`QU}(lh|x}eM}3Jg<`IvHm% z;wCL&4^YjJ7W|AL zT2G*UJ5G9EjF0b5?`&(x1DxLB`i=Wg;S1?;W|r%~LKoHKvudS+Hl;UYu>?3#ewm#u zKftz|u7)PoBf#RjCi0X0lXuqkPnz$|5|OSzY>TzmkS&hJ@vy+usq%W5PJgY&3P-28MCr za)?8cri*biAiZ9tu}@mLfztZq1617t=?t77o~0h>@@-7v(UdIa#Vc|;H< zb=9U6X_DDwRQ2$ie?szULgBG~sD?Y_e~0*6j{I?(-dy2Vsrx1N80kHm?(I1FTuE!i z2?Mw&zXubq>{mVlPBh=nRKF9>YSJ@*5pTbTZm;KY!Zbl%7|45aFNfwz14>#C(umb$ z>@CL0+W~1AHq`PFkf))!H-=96;|1}b?Cn1T%B*yOGzZf46fZ&@w>DVQVg=IYA!&No z|Newk;c1iUEu@p{2kda8p45v=hJ%>{Krgg6dA#Y+JwILHrRcG2Kae)jG`7zr+?xb_ zkF-V7vS?PQi|xEmu$Y*i*yYGSp}73ipkefxbC5J7;pSraFDnN|dp<AmBQ4MdKHdQ& zUXrv%st;IF=`m2aJv)9SDjiI|y-E+4S%nFu!w7>F&M6&4&q=vys4pb_8Hm#p(i75Z3C=6V>%yG@gOGA(Lfd%jwm) z>i3R`+n}j^JD{w9g)B0cDKQyJC#Y)+RnwE=kHXFv(n!8G^6e~Rem)@+6lW2x;na~e zu+Or(xW@D8Y&4k4}1z+dxp+*=Tb%I6Y4ib>%QkURk4JfPysuK9F+ ziT_4Hx((K*#bTBt5L!;dt-D9DSz=YDTswa*CUd+=Ux+p1iLIFWBHI0pv5&WY)s zEtH=H!gEGgOMU%}aBQd8sCOHbmLy&gx^XSV2+Lt`Blt8BwrXMnLP^s_Lq?w%-t@*; z(vZeT{ZNJfglBSSV0G}bUB_x#RIBQ9!g%fRlq(w2M^wK}vDj-_DkIG;vmHAiaTY9X^jkwe2h!M5)k}ZZwRk?F9Eb;n zXCR%qo6=hlUU8)Xl+U0s$ey+#DnsJMDTdT*pIIuM51XGb3P|5+GQK0sU*wL8cL}Rt ztYXjvrn?98T5=aj*V7%v-w3PNEr?$j*AwAw_3_(Agv&qe9JKTOk;O(HYK)K(qsx^3 zR`re#53>Qg`!n19YvE*Ocoi>w8g@baqxV*kz7tJ5>7ci*J5&30>0b#@9-~p(rvGbq z)!TTWAQW#GuLR1{fHW}*tKk^mXb)jgzT#1(i6Fbeg^~UcUz7h$exXKxvmyJ;vjViO-XnGJSr_t68YAp$Gh00MV1ES~W^x?;yP3-ERI#^=l z3Mx;bOaxWm5BG0|!yJro6us9#Wd#lP(|J)P1CjiSmUtXU4~T_D)$!J>iP-$fZlnxB z(?0qdC@p`axtFATLy$*PI)UwU{2xcx9oJ(Q#ZgK|NV1hAlE^6ayXVSYDJx_qp==ph zAtNJ|5*iYsL}?)Ex#yatVN0_2-XrU^-gEoo{k*8(^W1ySIp1-9&vWlvK{`-*Nk*FR zc*`fPx!>UM<#v z@{Ot&x~O{;d%CX_l#VAI05k_dJ!&AVrQXduHvWDijBMIeDm@i?@i}RP$6`QWwF&?=9fm@8$yD!*Z~%UI8G~RB3Rihdxh*KSB=ttM0f8HoCV~TR z@1VlD!ZBN*!_ZFhJkH>R5lr>#lx}h|-mdxv=sx+JGcFq~vuA#Vp6-Dn)a$j*x^*9_ z&C|H^aDVNTXE1-FjbLLlyYaaWrLc~^3w{)Q68DWNz>e3>p-a;|%~-Ko!8Z+efO`^V$d z_kr{*;9A_V=o@7BN?=xRKjYno+4#k!6%TGPpV8;>YNfd>%0G?K^{{I0Z&+Uc9bR^D zfspV&+U{g-odk4@PF&t~qnNo{WPr<%1t(eDRJ?Ic!P8%B3=jF-m zcDm}_Nn^;dEYC^?X?EVq0^q@*k)B^ z*2N{9)`lJG8iU<~W`Td~U)IU-Bbu2+OIl-0yApxX3$CKm(`vlN_xWI4sZ^v)Y$N}9 z)Z}9Z)yF>XzGL*F4nRMrHP(mgMpt3ptRm6ii5}4U;j6#Z(4l7ssQ%woDDKo4?rY(6 zhEo$YFFCyHLXlEjOJ2NMUw-o+55@*{;laRn>^_|-61m%j_QO37Hs`v_tP%&-sK+Oy zdE&^vCh)^-8=SE=MT76{xUrhI0DkJxKS*1tzN?AfPSgk8LZh z-p&Jquzs>2qPy5|>OIcrWG`F)strL_)uloCB{BQYM6_%^S;Z7bKkmSOx!TJ8KI&?zsoPPu7oS!3xm7IsLosIdqlfz}` z1v^f8g_o|y(gXOIbh-mIQ8bV?&DrV=pXupMa`HCP77YZ)K9m? z(3!pE?xA`7(8a@I@0XqIWaCn7)G!WOZ<;Lwo3GC;s~@3iMqb{v5B9XGjU5cPW8?Y& z@3y|yhQ$12$*o56kGl^D69Z5DogJY~2`NGEPxa)VHRfm?*;cxhtQ8@D6GXtn+A0n* z%DyB9?GQ3wl{RO3BQ>_l1zPCwUeo^gOR3GAgaENsLH8#M2=SH$B}b+P+L zW0b>IgX#6D7;#|%EN-ZeyG8|J%pFgpoFtr>O?&*h@_>6CdFG9J3M*LGdQCays(3Xe z0-~$W6LJ?@4;BfS3l@n zk-~|v%qw)5uq$&TwMD(1*K+?nJ2=|>|}U7=`hlo@qEt<;M-x7?w7j}JloV$KHRVqug>bsMlUXAs`jS7 z|4sP01BT2C0)^4RExtfO%Rg+&_wCwg@-M8O4@{kqfF=evabid)j(^#S*L2uNXF_ko z^!6)c-xE)vdHOP0bbbq`&%o~t4|%+c{X)ycfNeJy|kaT8`+B25Bws(`akJiAUbLjc^>P&v} z-;R`@I#o}hj`i4(YEAIg5@+5&bUv<>tAVfugRQQ@e^2+KiYM^^WZ!b)l%p!=ab|~W zxHH=tnl;~vwC=d=z;2O!NRO*Ldpx>2Up5dV)quohw}5J1tLs!Gx-M-g%MXO89DrS~ z0j^ygCyZhZWc5E8;$3thw(-i9R8x}p2PZZgz?Y;&}1l9LO$2J%i*nUEr7dMj%`g8_pfX50idEqWN%K|M`J1 zYIm0EJq!q^;4<~lOs_x3FW0=WB&8~7hX%_1KP_ba)Xn0TYpT`}(%9bddfe^z9TB8i zaYvV-&^FqR&p6)(T3GF7=ePIb3jhAA^$XS)Rbx#b&g69y+q2Xy4>9k15ZA{sgmalt zhqYFHuaOB7DgS`~@7pa^X8yp67hhZD^9JqNdz z8wU-0lgI)AxArtmuQL@4f^bfyy8IUZmMP1`H3lPVW|P9`$KL!3R`N}fI$%69gD zg7=3T@*6$!a|!Qsucp-DMzQPQsTE_CThKOc6P%B20hgS1!kUq1aOf>VZhy0i?#;Qe z3NN7~X+OUDaRDob1>w*eHu(5dB*i0|`N!X7yGB(4iW`nwYXrFu1Q1W4)#zi`ea(4x z@bnkVD^JwH3%d%8)_d>D-%lyS-rwp1)v$1Ots`rmKLMn57{x>qk0Pu01lmmj*y52Wja${X|5=fr@C z-K1}a8tPi&?`vzBzT-4@h+ZI{eQnK~2k#Tb`6Hp#DJh62Fy`h`(t5L5_n>!>Ic zSkR{;CzcIFtn%pq=I7U@c+**{_jbVDh7P>nwlL^1YP%#of`r9D91N!#ECS+4c+@Zu zB49h-cGKtoJ#9(hYO2Kr)saej-pM*7>`SeGNG=&q|N zNh=VquRy95h2cWw9qA!PI!T%(eie^CSCOG!)5SdM5!IQz0#%Jg)>^~sO#X}AvJuVS zrefugM4Y%j9f^B^KCcnafR4V5Bfevb=Lq}p+=%xIBSeI=lcYS+HrUsZA-}pyU7zbn zb&P5r-~8_Z=`@|**xJx6>lW^K>a6|9s0GC1Ks6*=SXqF@$hvGu%o4H0q!0`zbdgO0 z&x%cx3XwQiH#uUn;Pmc>(wC>+Q*STpmclrAer_woruBgnN8I4=a-RxqNqUn#^9n|38RF@vc3E;XTLK^FJ zCce~{q|d3YBGJdDns(Tj&Y}%ljHH=J-!6t9E^pbYqkhtR)x3&0JGxxOi5B}QP8Wgr z4nE&(mK*QBi;f5yhd27I!n4Qb;iu}G#e<{6WP^uEu>P|b`kv{hG#3yrD9%RZ1yqh6 zn)pH04^W+HwC+ObWrf?lHdp6|_v8tMsWs2jK5C?&6E`~XnuQP9gl8VobHzHs#xeN% zLIYV+yB4;}WKgQ^K zNV=LSUTijS7?!0TWNA}vb(YPaR>Xj?N$R~n51o=D@k#4g-L~YOnE8*+CwsaVVjFD~ zq_xO1?9n{;{R6ABO{CRmM;>f76q*h(2IbHCZ@vm`#@(T3QBP^glWeKpO0@g&14;XG zT6-klh0gsuRd4~t-kEfiMqUc+XE&2=qYq)>pEbmhd&Lv26c+VsAXl|$2mgkq!_Zsn z>HW2Ruz-0gKO%o;Tqi%53I_jG=kF?~w-*X0*Fe%t@N;=n zY1yJDzhaOE+u$BtsAL0*?-jmN+{0`e77@69ai zd-{yXTmJwF3v?loci>vL-AEdXz3FVOG&fd_NMg6|O{(yMRAWeff%Ml1Gk)w^kQ)yszR&Rz0d%kWOXwQ)^4Y5ca!i45;<`Z)rE>XT;dP zx6q+a7`ocdVUJ(=f!>>ULV1oQ^{o|;h@W+C=}0T$Mc+)Sk;8)45a##TiZf%o@ST%^1s+%V&NgpE#( zRn1Y=$oVrr80mb4-H>d#7!_at&aQ(>BM@J4${U>PFhblOXf4Tmis{~GabI2ts?Sk; zC{O)?G#M(Mol@=u%Y(y(L+K18|H2h!lHcdStvjIN5;?32-g~u&mG8Yo&+4yYiRL9p ze1ZoGG87)Dc*3ZILiJtH?TR&A-!((%ouTtYBl98U%Tt){-3xOEwBSu2zJ=W$iMoLu zZ54iMH)bsb{gXLLn*rThM_LGeq-6kk09ZdRAA5f}B38VuuQURZw*^`!DDpleOnV%J zqVpy&{!Iju?j(i)gxkHO)`1m!W#Y*#Mo=`F?FTPyK;%?{*=D#88d zXNA(6M;2~|U-x}sNL2|lV~-2c_A<2SGLVlH3e!$`R;3<|G4(aR;@dcDu6hLB*SHFm zw{MD^wPCy?5AoB-J|9Q0cE3h)@@}AdM#R%V`ic9tJ0wQ--BOVsggaQ4asfzpgThkM zJ@~QnR9riJvWPicS(5%zb;9e-<(&A3aO4?OuSD-7t(qsvLo>xGw+AK*@?pfy=E}E{ zcmGIuUV!P%)a5lA@Ch-ZQL5vk6w!1|7O@8^JuQ^^Ma^|>_Fk5PK%PWol+|JxLZ zBa!kQNe@Gn^?5q-MVg6`BYE&Jq?*Ec(`P8G#1qGCfqaEVx);-YF2acceoWUh0H19- zgQOXOydS9kirduF;FxAA)!0Z^KzVW+_@0~w<5s1zfcb-Xt@u67o@tz?Cz(?*OvonCpdfTGp_r}C25)pj6MC*N4Mzqa8%xw;=w=eU9Y1U zRQM(80XmUCXUUd7ko15gKab=Exat>@CSYOqqrjlty&`W^UaxBIj<;$C2*OLC8fT=} znEmXXSf$*M8H7w@P78bmX&DUdoK5{UW6s^P1?f)Z#gKTG52?0;#nro5u?9_tgrnbf z6InN@CO7mpk(>Wcf!-T0p}pTDXMLOIq;Ucn@n5j5DN20%r_t=28@4*)c1iS3C}gs`;WvHGDcKt*@Rzq zIEm?9bU>bz+c_I5zXl(G_6Q-JLVYXhpO&H0NUGoJxxzrw^{C%hnJb?~-n~L=C!KDG zvnrhwge&-<1-;+l7stqZO0}jNPR-VQ^XVLeoW_j25|qyMMC;c1YAitgWb$TrfIfp% zn{=+YEC0ItqnZOo&2|^0%?0)C1nC9xM0<29F9U&e1%_8K1g{9<@+vo@ zT}0NEGBDbnf~1L5zf|FYAb$&_tw{5wJ=(p?s7jEb?aNo=fhV)Ge6qLG0Yxo zb*?F0vNuA#(et4FpO3Ke)lBG}3Xt!3Om_|2$kJVHL`a^A?C%d?du*U?>828N49jE# ztv2F=gO#~;c?d6AOV4(+=?w1*Ye}o5r?~B3b39$`3NHM!j<3?UfGJx|W$}-NP&nWf zCdIEs&wy%t^SW!8@MSGl>hVPcjGx6Cn19CoEA_c~gBZy0_>Mm>TSL~a@%-$XX&CUc zFVyL|QvObV4(}%J2a9Fh`1XP~;*a+cxPSIE_ORcJqZ600B)_`Sc6Ax{+*-ss`c#*D zhbFLpX|cFrNgA{2af036HWr@E?1BFthGJRcGa~KQJqUmAB;I7jL3~Pc=#a_9m0C@) zN-KXYYB-WKSY7l^+ahhg{nb7jEz zcbJ$9n61rre_4SeB5?E71lI~64yMd@O2Y@vSlr2v4a!xMBoi) zxwQ2mF=WMKUDV4|*0T_?O91Eh3MxyZi=S0)h+9SOIC14{dHks(x?~%P^@nzWPko`` z{R5~!zY(X{;h&;duw6Y$9=)o^FQlGof;>vQtpDZmay=TfqWl9 zd(TYLr`U{=uR5J(hqXFD>mi?lv5c?TM+E#l$1-rd?4Oh&Hfw$4;*=7~&rN7!*iD+` zHu<{RMKh9EH86eFjP*$Qh0ufv9kHRSK5^RYvw zWa}Sof$1l7{CiGkJilx+dS5b?F@5vl#hb?ReY>L-^C~xL&7;jsFgtb#FW!2Y#it}l zii=FT&{c-Iq(LX!QS43q9uQLAN7A^3#e>Pb<(R+d=i;p$3}3{{+#Pr$$7k3hW+O|s zzpwHLN-PHR4-YCqrF$3H(y_Z?|H`$pXS*j5exj=U-t;!B5;|H^UaFX&$K7R!0lk>2 z53RtFr`3%VQ7>&}NT?U@UE7C`1Nxq4-g0LC#7euu2TN5$RA1^k_~hLSX0@=R4DPfJ z=e(TAJ-d|(BZphUxL^wR+g~6aZ#<31kLrnrI`SurccW)A!`fyw<&oEC(M_)i?iH+( zr}iC?m#oKW_q>|$?41z;1J1IGTdGKdkeaxrK?zV@!-%L2P-D;_){?#}b=&b6R^~bb z)g^2Er!4Ox1jf; z0i0?d>FqcOh@8l4(b+WxuU6x>&L;5hVNI~<`U!q?t;Jj8C{*>aXZn4lYsu*JRKlR$ z`0jTl+VAf^CX7lI4QI!o(ZzX$DgVUEp(WzxKsw(vwm!R1dm5(Ix~dDEzYB7s*T4pP zKVYzXhR$K$A2{=2Gh4CIKxR+5hr=^Bl#Q^FXcRr z)1SF_+#c=9tu_+x+d!SZF1%AgW<~6db$ks8U-bFl=l$uNp?qvpRvp#cRNfgjpDJy3 zm9fkk`h38i55jW73AR6@D=&L(44;YfBAm@d#H=k$zs3=uyoZ8SO=bMep@cnaW!dvf zFlVh!?stvknPqiwdPqP1=2;k&HSQ^A?a?T2T^W8`Ocwt z=-Y6&%Om7~(#MclM*Emv%TpMlQEiBkttW948y9YRcN4s&cNNVWG{m77tI1hc-@%>0 z9q99HJ%qZnlZ5|_IE81{Sc{?a&H0PLr_tQC2K0W=L3Xk&6Imrg1B$RU#U3 z!hdMK^)IUY-?FwUCyb}DRY$59pnOvI5?7M4*_c-KCB;^Lw%$+gBP0Xy6^-dJx~wuI zY#GBTX2NGcd!W2_E-$5;%~*}cGkf94vkd!&gFHLjk&UZXABi(4_kH=B0Y`C|-3Mmc z{ji|ihrx}3+Xoios`y;lU+q&>)Svh6b%UNstO{pZt|4BK#GQMw(!%FJ@y0v@Jz2ln zKYZ~fj`dztq>Bp5XPc~k>In0}H+4OJc~9SaylW@TPHw`$8@Vc%MC7z#;_IGM@ZxxD zzNF@VLggdz3pf6C6DUvl_m|Hxu=*jKwEdB8#fW-b<^1Bs7GlcQt^9L%7}gB@$KE`v zRqcnWaM)0{FW5~PB{skQ z!G^4uBu($ukTC&^G0r9*j%S`#cn>rN@LOamX-vA^vzr6)AL{R~DTilP6FzkQZx8#H zyvY6lGINe<>}y`^BCe)$?XUHQ2Er8#kNvB_jN*xR(ebMpA9`_vcB=ataeUBDAg;t2 zmXC^c{?d-D9?z~>(Q_(VbxF7ZOEeEP_DTjPMRMCW-s zo$6|iICV1~@)P3G>RtnWFDhKsKQCAnFFQ2c2Z~3{K6K*?y=gDc^`4A$6T5ukD^?ko ztaCiwl5g{^3UOg?Fx#Q6szu7Jx1i>)ct1B<4+u|z@)|y*)`FOTA0n!u7ZzQp!i}n0 z^Ad~8H0Swfl{6W$9S-7t*Sg4#k7S<7g58R30>P7Vk=8*cOqxm3toZ0gIn}2#FL^r_ z2+J_EhBK$NlB5;EXy-}d&p)b8rR{4j-afhr9fCrz8tr$MeySPwG`5tMTRP(JrC+oo z$A&P)zgodDh#Im*JZ{y5>U_WOdSj}z7^8UNo&Iga0HYhs?xMtNADcKSy|}X3LnLjD z&9|<=v&Y9_1M_*16F5Y>ZnzmqBPmV7sb=6rt1Il))T20SkB;)|JyYD@dc#vtdJ_*G zV};Ad;q&2@Ne4Md(gWf|m?IQqZx;ENF2mZ4w?O>O|8%79lJ8W7_yx8V^?OoIgH>13 z@b#nxm}Qs;U!8A5Rl_e@T8}(o@V*mVXH|H;(Vw+u7qTia{_L7p*rn${R9f+B=}WLF zDHhXDXrvkUQazh1?m?Jy5RFdTZ!4m?+3{CPB%hJ1ID_U3q`5T0O3LLF zB+br7cx}T@-+iEOTQfdqttP7Oy+E9OMv(T$PF}?{kEgnK6I)B2Faz>yeCQNQ^=HPb zNR4!@x`s1x8;}n`8lQ~so+ElKGoqZVD`_2wlME$cDwO-zg1ZmtOgmi?kQRX8;$3Xy z_wMRiELeVR3Ho2Q0L3joW(0xmgAorc%hxQ= zOx9V|IuE2VdCIapM)i#A>#t*r7ffW|Ji?H)6H?q!;WOnLVaFjHygHG-Q?De+V<6=X zj)M(ahQ}iEo^#RTp}?Za@!-{@KYY(2Z)e+vU+8rgsh;F)3sYI*aYn@DCy>@iW4XT$ z(DM`vV347aG#k|_cTh=vKJNM+agd%pBHV#X;XS4H{S~GiIH;*O6EAbIR8M-`{RLDr z*e~R_AkBi_otWr3aU{~*fjk|O|CM2ECKA5jjpkK&=?n+X2G6Cly(8eyHftQ(u>;O& z>H#ipEy*{nk}i$x!GgXoAe}-U`ZLCb&0yp!kn{%p{5Kj3ms4-kZLj#{9>j+SZo?zT zu88qhyhY)FX~2eOlJ0G(HEGsLUJ5^hjSLP8(xi;E4A*SObMi&#HlqX9itZ#8#vf=O zDHLWky4FqBPESJ8O5k0k8YAC?Nf(amIz@4Gnrkl-TIPV#j`;~q<=$qg8u6{@`^}Ox zvOmj?y@GTd9qA1qPJ*N^3A!3VCmH23M(hcO$D_lYpPPrn!SL@?hvDMq`Kdr_rjZ8G zR{8an{L%pBTXC58B-|G5#6v9HB&@|$zz9SKv}(`rp5aTn`< zD-lkNK8xe^M*#UTd==Lk>-0S-)S6PD?&OIK%}y z@+4qU?Qk^gyTXQ5$kiTZ6-xRz-UjY1kK1gS?rzPb`856mgQ zvD|1sKHJ@eO}B0#$tzLK1=1PdEy>5VW~G^mFB_ zuC#;uWADJ@Py3-qj1kN~yc;MssPtBh$u+vpW)^Ol2BL2ro@Lk@i34>(GuL7K&8vd) zUttoD8Z@k8u1cTeJp6{_RhaVdOoYRh)v) zuDsBw7b2FPJjck($O5+m6*2ay&#+|eSETiq(T@|9&ZK&-D~aQw%k3xZ3cn0q$M-O* zW%56QJkJcKu!eG0P;Y_fUE2(&y6a)CPaU?#wJ(ytAs^G2(|B~VoIRwf57i?h?vh(G ze+bnlp&kWzaY`Y8J~*VAVOdAUwnbi1$Eg9MT{_JvGe0dq_D2q;)I& z1^HSe-vir!P2hxgg76EQ+j|1(A))#nb$`~71zCfs&$1PoxZTv^zFyUlZ%68xh)ds6 zz)(A>^fqr+DM|SCYQZL$SL3EBF|cd>1TAaEYoYieduJVd(qb%pywe?BJ&@K`L+`)+ zbd>u#>PwKPpyy?W)MS(cSn%u&Dh(Xz@Bobqp0SB5-z#2Y&t{&+)IR#^+SuvrM}=Lu z$!du+X+EI*)Y(mIz^TW@=y#~cnV@tq{4)&#@>Y28+dZA-w=$tL4`Beb-Y{9t^f%flwqSDXW*I90oe3uNISC9It)w$}`L{?1` z<^Ew}w#8N;4X2~~vh5wNsT|Upb%{aJL2#>MHq!Vw^{jA1pIkwl0MlyBM!!W{!P#lK z;#l%cjcGrzej?7Mk4D$S6)kpCPo)S#+BlO&qBEIplq;S<<>~3VK)obZJaHW4cGxMD zM|&1u4&;d#c}t`}GpZ@g$wl z*{FwX`Ro!PyslU)!e8Z4sV*j=>Zx?}2~oXPOqzUE#f+YfAENwl2fd((h_T?u#0*- z%Ja&pVzS*R|EMM*5bct z!qlT^QO}e55mRyUqkhU43c?MkyfgJ=6)yuO2O#z91=TJp-A4Z0nLIr9r#(GS>8`V! zy`7=w-FR$x=PzvOUlVmXbpDDv(+$ENc=aDb(B4I&!^=9k`+ubB1~lG`e`9XI1h z=#H%%UD_S99vebn6KnWrZNo!-3c<2x3y2#A7>3`Nok_eLx1)w^>TAlU`u`Lq`G#^s z@4>tUonO+j%_+Q>)DGU?+lQ0ee1%9)M;@Lw2-m-{kgdDc=5ud0;nNNdm##ih%;?)` z_;PIvT=+N(tTrWK_2&)H>9HAHJHM2l>b(ieMn!0@HxjYLsR3U3YWerHPyr^}TEn{m$O26Bk|Kt4u15Uc8Jg&mix;LM^y_&BaHkIpob^FtQP>&=g0 zLDjCJTGTk$*>JtOCtCcB2NjLZ28*5efjYJj%Y^{c3@J*ez_{jjW1YRReYbk0~Wly z$0%obaf?cPCb+O=1Y0Oua(m2rk4y)yMSw5jswMrb)oN(eTOEo z8B=}bq?wO!N<&wvV}0OCqvrh8OGo~#vLhz=u0YCRKC?s*RSi)d@lEqT!N!^WwVz{UEvxU%{iZg<;@CmnX9!m_kh~9b|VaeMnaLM_zS{E(g>@e&( zxSe#pVJZ2TZZg8}A-2mnh8O;`g6pO$@m9-A&^R>+irfXJ1}(?k{SH!Hl%wf@Z-VZL z53gT_9Q%EY<|KAwRdA)v(+=Pr`}7gcaztc zGPW+CQf-f}cZ@y1y{@y&u3VXyvl+ZUGTS60X?_VRb5-o_%cem znYRX2zx&^p!{HIpti`0S;_0myEbUz*PGiMA2W!ie`vd7*f-u-zyA#Fw2kM3Ap-H8A zgeQ$~rSyauPweF9Id)w|pXVz6^sZr3sN!J8R*d=LOnHVce>@~UtHt$K#BsV8402hf z8}DC9wCglix>r31UlXnfmFFM2SLdHUKL^LA7j#k7$FR@)0pCB>kXITdU_E+gnQ|T% z-a7*S8Mw=ku;JLtrHaCC9OIM>Dqj`;^jLBnY=WYIa+L7QkG~&sl&xPJsW2b!Z>u6# zPdme?J~03AGHr-wWxm+Vn6KNwxEQ>Xot@uaP~FHz-JZgED|;!!r}JeFhK%YFDSpCg zQ!5z!wFz(BJVI<+)Iqj6n1FQ~m!RwDyW)}E3anXQfXZ8^DMOI%A@{zzOSqf?`MtK3%oBgkJi_3_c+J1Z0(P#$8StH9z*PSBD;O?Ed;SMh?|5uX1{9wzmJi96pQe<>V`n?B~K`sOWWwquJoHe;VQh6}2BsC}oZXwxhm z6C={$w8ebzcsLARnDoSEx2fkld^iuKcOD7bMD@}8adUj4*xAq!J&p9_mH0Atan@0_ zj=GpelPC|1byq)_z^G8vCH!rLwcYFUNY7(H+{KA&G2o(+!Y90RB^HQx@kyl}Y4e7!G=Ss61p#aE;)4G}gGT-By_XIzSI7Tg7k56@Vhk3M`2x6ZXc>Mo7nbl}8` zNL<5+JNfsE^Py#SCh3vyNSMQFwweO1o({*hw@$R)!gKn`^_W->=~NbLIrh zV@@zyQ|6XP?_P{{--Cz88Bo>B6U&2o-%b3&W?z%H!oAKKD|8T^W6mj5^ z1=MgkCtm)TC|ZoG!n<#Lq{hYkwB4?9+wS@$7F2l~* zNp&yR=NH7t&fz-32fTdDi5JqI!_}hhqW+4;NEm{h-=&Dg&1tV#mo_rs>^1cKum?#~ zRs77;XbqaJz5)~jN%@KKJGw)O7{U}T80gw_`yMXP-5>?1_BGN!I^soPd-5SBn}mrE z-9v=%D!{r{)iKwgktCkO(p5W{TUs=o^}9jLYS#%o#>WEHzpna)V_5KHEgyqH-y z`>BhNi(>o1b~h_Hy7ehsEA6Q5^4_F0jhwnHQ@mW*nv*UjjB1ZR3Mz4ffu3;jVK0T1 z>}`ukm7`Agb}WR~H$&0I!U+5IZY+N-880U8ZYej-p9k%rHUG5H7%4tT`Gnp-hthvn z!MQLS+_xOViS>>!;s&G`$z+pSpm2B3Zcl|#BJzAb>TGwQ$7>^eIMfU$jM{}|#pd8> zt;g>K7-9B;2>G~~hi*syd7Z-GyO+CS5;Rn}!gV<{pvSZ@Xz=v`lExP*=5vA{;>hG? zvc8QakcP*(54*0%!StX)(dzF}+~yn1-~L&w@Q@vBeVI`{!K1t+ z8nYwj4```%xIdaaM|Vkc<8yw@VN^?kI0z#C8OozuC#(F?ZPm9x$IW+e>4-a!yzQKp z8q!5;FfSRqloT++4K^mA?xN9XVMAM>GLV*LeD39{&QvT?^$t`qw(1-R8H!YCYZ-oq3xidSS9B& z5VuRhaY32_8$8R#W~X{q_=F~lPQd+!s15CA0b}|P;l$UpZ^v=EuO)`wD5JfRR>6a( zOBnGdc)Z?-rICFgZqihx=aeo|T8Z?qsq`J%iYx7TcG`DA{OkNTF^o;wbQ?k%=V9KB zda@ntXQX8$#Z}I3S`7#%G2AAK^1B*-s#!?fL1$ad8|#$d`&UGF?uEv)U*YrIC^p~l zB)hUY0Br*93c@DnHqaEiM>XWyk`z$V^c|+oBG3{uGB<^QM^Q!^*Jlx*v9p#oGCvFh? zgPW2rzJQB=w`1w;d^u@U`F`RAoLuVslbTv% z`geSNv!>z|ARVit*t2l+ahP0fBrIQ1gRA+Fzf~RwC_m)k6LYZh+{f7Kj9^JgbJ2N3 zGFx-^Dk{$~-gkg_lU`X8-eBKHM;ZA;7!a1AsTw0+1YJt{seB+@DJ6dW?cB9yA`bSm zC13Fdly4wy09FmRD;=kNV}-Vu`u?QW`CS{?v|L}_nR$)4FC0l5kw?uVA7~1h3ugdf zDJBN3Wpw|FTqI6IR#&HWpl7*ujq=c`{2@Ip67{+x}RPTVifb^ZQ4pa=O zH=}d~thqZ>aT~Vl^pUAv$f3T48sRoS)VDuBSN{ftc*Oq$Ykz380b)^<19{vWnT9!~StGXe+4LtqmyjuBUjPPLX(zR%Ifg9Cj|s(u8G zse;4#y+v%}@d>Iw#hw*Oob%&@2sgj0DUKiwji)>uKn!li=ZE|N(n+w)_6pA4b(*lS zGXM0u8Rchd)pL=fYJXEWLVmp~x9qt`Q@&04BGS+rtv~;-PH@HN6%H$`SvjysEWOr( zFL}FKGti}rQH>4(^((O1m6>?);aPSkU?bS~ILSzNifPS!kp9jn52VR*C#|9$}kS)72^J)S8%gVTq6&}j;t@$fqh2|tH?>aH%5x5B=10)HbEoJpd&AjN%V}c(%=~f4x!S-idzWlu(d|}evj#< z+rR7?W`8fz6yNMQxY>DpxifQ`W5|E%>PXcWO1WPQRI6fSJoOOUEQD1C_n_sD%NVss zCz>aFBYA54wZ@ct*=+@?F$h1V2c+Y8jl(0L)~{R4XHXcrZ&(SWDHwSv%E#)G#wSSg zNzy>L;!~W`0BOH`c=qQTaaSUsQ&`(XB4vw%kSf@-KG`Tsq};M z2x~~{^ax{UAK^Nu_fZdG9(oL)r1}d;>nkXilm=AZh28#n29LBT6%;S=wMjHkFH%s> zi)72=VqS~gNM1{=Cy&{l!c>e2vy~4<9sxVQ_cRIw7NtGM9Oo%vF&^DI4%xJFSEmi$xw7B70Aa( z!e+&HNIFH(yjk7oMs(jDXi(7H+0~~Wbl>PL0vsnX(qrU*sDDniZC3jWm;P;(sp0sGevp z^$wJ7*JhhsW#rchf9A5+mqsA*7p|+in!ML=yb^i{-Wu8~%_oR2SjOf-aMZ9|#!5`b*Lx&uA)ylxl=nyzKHs`b3o~X_Q9!`cj2z$ zQPBTn1Ee{?F_zw?=--4+sdH4?i_`TbBy{DNFIygwL+LT`c$D&3x53*OCGzz+ZJ`X@?f;4 zK-{JJOVkshJ&z8xuc%GMN2bmf@Ys;WaKttgZ~yE`*tePb3SS`Iu{lyN0&E9eWr|b( z)o&)L2P{dmQ7-Ofv~J9L)DzX8mc1NYHS)8pYn2(m>S{$$WPWm+PFjVCkLeU|rXU%Vm9`$70&E$aM{#sM-w& z)cK}svd(@IuQa94Qww&OFhK8Tw?3|aj*-SD%+NEte88tmC!mzQe3_`<^% zi*GH)T02W&RW)n;{?Qq)rMU8E-c4k-dk}V+JVp%6%+Q{fUJ=`N50j?e{W$AvCz~WE zO6!l$1kDS-y+{=?h4%8SZ4m^PIm<%}E}%tOI2PV8GBjWbMdEl{*%jnLY z(84uOyqbTUpPR6OM52 z-)}GCBdU7HCztBWwXIr;0oFON{7s6s>hlvYcZ-I-oug&%*RS!yJ72jpXBxK{*$NVB zh4Mu66x1CxgU!$0Lspa1Vo~x%X4m#1P)+e89qeWBzFb({bepO_(PP$r=pX4NZDJ2G zRa2*8dZA}}6HeC_zY;ftF2O^)ICDjAk0%*ObQ+(QwN!b+fBx);SxtJd^*Jfxzgd4kZyfF0=#!5-ZVy6(B|8P>AYO=B z4G(vWM8cZf8-u&c!{ zr2k{{#NTXSW(PU(_gZJCZRZ%_6yB>gR2mnolW}7mD=_kJ+X`G_k+BWrNz#vALtn#$ zm)o>SYi;C%tcJ87_FJ$vXd|Kyt%IQx9^lWP1-NEMDx{mPV`EQsmCN6>5ek39{hjF< zne95lAatJ;tU1*f4DCJY>Ilyuj`mJ8m<>=;&;-^5PZLqmd2-g27?%6susFZA9v>(o znD(+eZ?tnOJJ#a@8g0sj1{;%*at=q(e*WoydqLs2Y}WEsTN!iseQv4t0%#pnZ9-w+ zFFJ)U0j-b2?n<6`*5W1H-P(lT`FBP)a_w|Dx#SDptJevuKFveIZ|T?kHKW|%RL2yn zqYzMcA#=M~OILzA*`~H5-{iFuX)fIGu|H7$$iXwBSo@PXqRgiQzxmD@BEyg2J{gXk zpYyuBV zoaR1yr^L<2eH1>*;Tt0Gr12o>+;$hcSM92#&&s(ohqM1ly6(81zb>wnN)#%EhLwtt zvijb0D_PlOkL;bjXCfptlqe%o2vJe0?>(0aKNLcC$d-|jZ1SAj^T+eT^Q-UY-gD1+ zkMq5sdlhaI*I-Y&7t?EG76cvI$~y!d#Z-fvvh?3tUPgNjXs$@LMEBb51?}Kcyfdna z+%s|#@A9Dz5EqGW4|~gc_Ya60?d$VY17AG8*+c{ew2@Q;@IIowR9Jj>P6U>u#lXZD zb20t=Cb;7sz=*df|HgrRa9y-tl?ufc8zA{>2OeG{62oiM&NY?4p!R?kbsYk_5`xRG*` zFvS5*e0c^mJ|64q$Q5^+T&%%~%Ww-tK|8^5YULvwtBH zqWtI_iU%;@juvk^JfppnqZ!pF^Xj@pxJ)NJx9Wyn{f97BzwPzRp%dw*VhepacmbXJ zPr2LW-ZfZjX21g$??;uZ#Bsvyw?65I0EL}MYe8}SDApPt(#$Gu%&x`P;Z#3VJ4x6m z;)!^R@{)SGdL3L|Hh*BOA|@!4b1Jh=(;3>@M0(u_k$1J znP}6^h?6$bHXY~26>jbQz<_d-I|aQ%>l7D!S2s>%7SOv;nKQA3&LU2Uvc`>8AG4pP zT!3Tm9r&ObIV@n@Skb1J3l^c4_ARetL5FJL$h^&Pu8T3>5%&*gT|bL$@?$Z%+D`lx zwjYOVi^J{4D;VJ@Bi;i0hf9r3#S*Ee;rpj-taxn0OGoEkA6XiPvl3s?5b6WYPkdAGMSWo}aj zs(!ESXsh(4Jlb~-e4QRmcw^6Nwb$Wu=d`G*A>!B0gb(Ro2aKq}o(TC`fK zb*+=~S#hr*-Gm1wm#%@eg$ zgAn?<3%ghT7Vr>F*NvE~8Q zIgMXwWf+~435_SzkYgXUhY>ON*|;$UBI9a(B%P~t4Ui^bq+gU?ft8c=B;hY!`Qd{{ zt`0}HPl>8VpwfH+7OanAHttDSo11dd8nEj>KS7*`gc(42kEBb$DafDYw%$W|9FLD| zqj2!*6gr=+nFtv7Sdivm^xp}`7xK%aon*&1ztT#1HuvFwqZaz8AHpPDtyGk zR)#F*JjVPc?ibUu0`u? zffsH?WGj56-zR+EfXgq~NYbsaA1?)dYf~BB;TUN{W2`Z0v;<)A`HBvwXYsOF zZzS&kosA1c5gm9Owe>kRxy_KcL6Sxw&2tNhlTjC6v6jv3Vg9##m1|;*=QaFyq&Ft} zd+^am7ppuWE*%HD7cUB`4nqy6=@NLNF7uv%QjQQfdXSIZ1xWv^8ZcovrGo-vkaKE)c`H0=u zbQPVJEa5G!&g0Vy2GF-_1}c1sFx@XmOHm(u4;*M|L^W;+#KSOj-fyjTY%i{K^s*iX zO8YU=|4QHCxCNo0lk}aPs63~<6<^uUo0AUD?pCd}G-XJ6AFHJt!CJPST>VKgVar@D zu(QogWXom;I}@kTUg}~P7EXKL$|s2uhj>=)jJx~w?GJ(Qgt+3C`|0F`P+XJF{xS36 z((t+;<)jZBy+FtS-Q3{9YoZ|IRVx7ajtx0Fb>aYY@%4~Hw%NkZ53e*MEopz>E@#YW{Ro|pUB(feqmZ;D68{S5 zxd%x@i1B~)WZ(YvCB=(8L-%=i4 zReVR@;xENEgcmRFqs@ySMCZzQ@%s}VBIOz6C7+~c_L91fLeT$aJ$|ae2l#g+AE-@) z&8K&Pj^D=-eN37a*#A5>VLGS`!ox)jYD3+;-tpzgk9oSSYvTj=Pu~c zJu6gGXxhUDtWCc&m1~Aed&7*&>*beTbUw;P2TZluE@pBK?{03*ub6BBr7cEuHX~o> zS{2XE$2KYNhAKxJ>@^q4`@UYh8D|ymSN?!-VhWU2G$yRrCVtGlj+B$)TlrQosxO_R z(aM&)+nr&JPy1KR=lg^2>|B8^b4oO+@;nr`?AV?ch4QqkC!}c>C+!E1?w7$KVjuaM z*P!(Ammw|iZok<~;rPXiV>rm>JSQO$NPh|MJHOR`gT%vqd0M+5R6Ig@n^7+Sh%>A7 zrSjCOUawj<;KctJZTCh{9nzY)s{Vn{@kwJ>*X0PyfNr38kLnc6%CZJSgMlg^lk#evSm$@G2MRCHyZ(DbJ%k4sAMGN@0}F+n+kQTRH5I2>{F)C+ zODms^V{Z-y(pT*Bm9^9-h=R5r19)UJQ{4TsHJ@?H8z`SOge#md2GtsbzRuN>R$&Q8 z>_FuKc{(<*_a~9o;sEGPIf4E?F2T!T8mv2@70>8ur@SGbb~K$0D7cTAcbZpuKEu$h zg0vP4>^)D_B0F-gGbijs-fk3-*F>eMR?mtA@&xF2<^?l&_E|$d6o})`mq^-MOZ_A*#Y}lo zARWz1EB1rpj~^@2aK28m`Wa6AOnrw^Xm{flF4~@k#8o1usBWhDQX@uQU&KeJ!mxv( zP=4E#6ZYWC3FVJGv7)yN| zeJex6;))53`WLwUw7HCZ9mwLo_rh+*VM^1A-&R(vrbRng-mSgz(R|H6It$J$7`=6? z@hRgw(s?f0`U3-+q$S zn1h1qKug+&tG7QM!17mkfm*t-vEf$Wkf^>`Sb09));qx3WI%H4=61HXLjv? z3HoLl=hH3e4!zqN7vD3CK1*C)xk2^yvQur$8Tlt>WoAQtEF%t=HmLl7fF~D$)`IxQ zT8`@I#h+>qA$fGJaGN|S-A%U_mk+SuR6n3_lJo$(Vq(k>wmV;iGt^5`{uvY(su(KG zI5{SU_@qR0!zlyxe{EE~5S|d{0W0XPIqJ^{>QxA}o(pSg8O4-QZD^?913TUBu~$#p zA^8IFId?yjXC{(AhsR46s#;=%*&3z!dLQgh8u1cP&j+ZFqETMFUtR&u9q~e(_smkb z%m}Zm^gZ#J@|A)#nH-UzC2rrRd=U~4g8qP6jJ!8G=NcMzW z{R++f#tUOQUx0cdpgbk@7a?s=Oa8Uz0?Pl>tWWbPgk@bA`911O{$V36=Ly0#)#GFX zziuXddr#?g(x_v2EyLA3+HMA$R|e!&yJD=Lksuy`b`LYe_umbnnr8w&(5YXQ7fQR| z>fwr~hlk97?`?jv5lUQjrNUeL@+=K`IMQ`1VM@~oW|DSZLp9E6 zoxn2SJSZ<;KX4~#XB@4H75P@x(_*Ay$=Ca;T#-{O+p_rm+ZcH%E%oEr{`$L=USp(H zMfR;hl6vftdT&5^ibLykpmltY#8Hy^2^cnY1X9fc<+!{(WgSxgQS}pO4Hk37-Ku^` zyNc)ztyJI4ov>Df)?1AUk0LPB?X~E-g=6DNe~=crTBA%y?SbpSmwM3s898}i{+sr@ zZmK79YqXJX9jv4@oXxiW%7=DmX3AlgCy2Gv=Y!thMVdB_X0U!&0kmm!R?KX20$b
y!93x0}h_sQE=k=Ls_nuN$SF zWX)la(b>#Ou%6(=ehPfb$kYrb6*va4p_@5YYx8x*1$0S-U2h{y@0#O%e)6`NBgiW~ ztilr5Mnj{>?LeIIDU(|i*i3;L#Ko1%XyGdSe#)o31ZNpe)8=@)HC>ztzBC@shKrE- zMfkJN^nJX$g!YQEC8E^y_)~;A599CB$SbQ=e42%ZMkEXS^p%z-MRp3?^-~DCnB@}_ zDn`<0pJ?$Ysv7P6wL^I~|09sgsd%A3V6a^(ZT1$0b7#y*=;QT7VZ0DJYR%%6<>mF? zGK+ZOzY+qqstgBWuf>Kh@8NCr=y`>8gq&&k1NSx0qwYTndCjHp>S<>g6y95a@7ZXg@n|rrXutF_a>Zd~_zi0zk)>%=O@)0L1jf!| zcuAng#mzVQ&0NXM5KQgT<90Qr(fny2rxVsn;pZ?9anNUvyNdC7hl#ygT5yrib-0#; zgT~sD;rfwdZdlW*`*36XW(xi+F$a*(E0khriSY{ND%UJUV~goSTSBPPzgVmBZOmZ! z5kd-7uDbJ__VV=yxfJ+yo{aKxrkW)6p*F zkmn97%+-(+$C!-6|E9%J!BbCF@P&5z$EJ!4dS|l(#!Az+hgAwpM>lSpZU~Wdv;;QG4 zaM+CDW?sdZIjHS>9q4yvc@>y*QC3YZPHrMit~GO}wUuy9(X8DH`CsBZOy8@!v4^ra zu1ujLoID{PW=*Tfa3uVk&Yhj7e5Szn9ge9z^&To{l66W{5_qa6gEN(L^@?WC`F=$k z;hx%=F)z#Ng^Lljw5dgCtkuka`nArl+LnKn#U$`OdL3Owb>h@nRN$o!s=DnYqLbLGf_1?B}&Am{SSgxiOp)cf?g2b`QIHNIUweU(n7o zqrg$L`{Es{9oL$SC!7Cnm)2Gd*zd@g4|>h7ThJ0PaD|$W-$CvyQdFbH+3Zu5#4&{V znDt=2C_>((20eFh%oNOee18Ztn=#x!@OT6Iw4{ti-5UL{ATN7?VWgQeKzp*h*|3%A zyNu4bum8Y$RsGp9Q9pAI!^?S3y}p(tYQWxK8cL^redKtD$(Hk7HgLNivt;V@L6np; zj%&6a#53onlG~aXa+%?ypI=$Y)jmh^)XR%lUD?5LTU_+U&37pEZMty(Ya!PcouvwU z%;&q5mY2WRlQWAcHTyw5{V}*RU3$7iM!nl$2`wZ=-;-NpQ?IweX{Rf1n7Wi=gEFjl zN4}xxDL>Tq`3dy=+HZaqFkjkyt|-sHSZrBc^o5AdJ4fDHah|JPs3UFCW@$SIdp+(z z75y0ZlGr>N%&k-UsF`zHSwB4V*X|lW_G@WV8yRQx){H>)S3lxlZA#1Z7fTzp(Du0BCbPphiMkU$ zy)O#KT#MK_bw0BfWZ3(wA_3InNz_ zb+MQZI@3lcsX2u6(f&(H@~t8z`1_Fk>~Q^!TD{|z`1r3Z4=MSLLfjs)V?b|d_jU(O zN^U0)x)&7o|MgN~|3M3!4{*Tq+|udda5lAN>g1sN67_w`4IVsigT(QRgp83CRo_>e zdcryDz=wG_DfEembjn6f7ALFO$ABi?*rwfh-FMS5ZgFQFdz`E%2X`oj-&^#Nt3$qW z%ArubI(xD?Z{C@=sb?09)2j>%1@{M%1~Ut(4Aj^_NlWHKd(JXJHr2FsalE;96R zC+*}^SH`(^pqry?aSz5ZHh;H!>n`_S5FJxlECkZHkH$kwr887dDx@bOIp=? z4@J~MJ$#AvIzOZdqkgDxqYRpfYZ2Q0up!%4h3N9cA?*6%8pVv8O!iaOlJmLuoE#ZV zaes5`@A+DA#`A$PX;(8`Z`PdyuD8@fVqb`M8~!wiq?YWzrI8Mc2-n-5ZYI+MuP&}4 z{I_x&e~Rq)Y@34ema%LCSqlUIUdxFj0gd|>} z^Df2oxJAwN(y%XTQut|3SonyrevCRW>p)4X%5lZtZi#lX0wU+iIyw@9`NwJC7Vn?dNyd%j$$C9&FBUdyTdnc=0TIMAXnN zDzudTJtEclNwX+^@k6#-A}Qmlw_F-~L&Xo9rqRz7T1X4;D$QkbNC3}VIbW`-v`r*` z{wy4RwpBP5v3=qsIv-w9rhFKqtM#oQosYDFK5Q`SuD5rLQ;%x}^N33IrNiL++F;T8p5QAfY1=7*wk#E~78dFi-_ zyyyj;e9C@(myrtV&tdN6Bz33^y^Z9^wz(;2>wje5wSjbRIhf)PCR@?J%5U`yg>}%^ z#)RkrIG88WeT;g{LAMj_u;K1>XM@>Y_~f53Yz(- zuKGBlmiFWR>?igKKYmQ6R$N+;Hg!BWsiI8(Q%-N3{+(}qs?X49j&-|4I4^Dbd|IBV z@?%eXIRY;e&RlI#afgpnz=HLJ&-aR{dXb%vq>ASyn_&H(DvxGEbjsEi_?}wQxxp%m zFHnOLj%TNS2M22Oqk#5X0~WYY^slDMcjHQm&Wj$t^DC>@(h96&_<^wbyp7IJc|p)D zPW^L0L{+H7_BZB8r=455yJwu}IAE!aT;*tznAe~;kr>UpDzwB%b!?QNOM*ZMc*_{Dw6{5#fx+!{7wd=Ar< zo@DgE0l$~Ry2ws7M)9P5yEtS13WlHX<@uEqJU~1TyDYF*3Yx3Znr*T8Y%7xGpB>k% zPCz~D%SU5H{IEa5{&ySN{=J!qYk5%gEwhNG=3dF;@<+1SGiVuIjyg;J!#xFh!?I*; z9ewxyBw_pDrHY8_!hL`55=md|rAyC8G_vSK&U_Ti_`6io^EjCr0z=3a_2!XF{prxv zXyNwsI$OUj5FNi}=Mnew%IL=S0_UYN+c}WB;wGWvYP`cQ-j+M3j_y)RzN#`Cdp?3A zbK<#xJ?)uxFH>)OFHqY%)+Xm=HhlhtgMKyOgP3;o5@i;%;{^MfGNrzkESoZlpb=_P z?cM~>;-IasRcz}60vscrcS;p!UKijPkzd%3y2I{=$0=o3OS&w^v+X{*SdV{PC}#W5 z79+Oh&|o{u!CMtQ!Fk$$%TddvJz-+wpA*W~|Bzae6iHwJzI?QXH6?f;8Q(`wXya2J zw{D~F$BJr&sWSjWuhZ&`)$ed5=O}QEFgg z{fa8dsh0~#kNpKzHU7frZT7oR7VKdwP@lB4@j$^4ZSORk~=os ztK4#XYSWu`J?d-!;YY}&XFtpFN=0-;#!R-cZlK71yE$#xTjksBGPgM7rNF`(EGb*m zAI{&qj^WiI<8Y5aTZaEB@TQKL>nY(y;$7d5oVvLs8|IU-6S?oJP{V0F^TJ||^H?l8 z9qq?pV&VG3k!D_SkiJc;oAZ@@|4U@Ay@kl{bBKsP*o6GX)E7zp)>7ck>_)SUmQbWq zFrRt7$m*u?OoYr441JX6r$ArfF&~#M6A7g|SSH~UFt7v@mleQNij37b9y6$j7CR3zv5td1<55Y%`|2 z2A^q%=T)d(?U4js;xkjTvFW+Ys?{xV%lhaP$D0;tEP3xNti4qNz1mfet63xD9?t@V zS`toeEgC(@U=D&Wsj!od8g)epL#H!-mQ&v4hlWjJ$9cynBPvlj&q`)6llCdRg<@9U z=eUu-DPqAwvqpIT@Hg^!waV~{H7I%xJ6?H9ouV@7T2)DIm*a$~%RLXa(7Vt5=%ZsZ zT}^mfqhD;InJeuiYEeYHR5m`w0j0kxRq`64=lEUUQ51K$F{_fDIrQ&phOek=kCqZL z4NZM|K+NRJWkIVUUBEl22MHs67MB;+PGJp_+a*ZKPp7Xl^3bRGm36 z59{h>e8}qkC5gQz<~95YPJC%H9tFLt1>V?3;7(4eQdll6IZ)gBd+6dG}K3?g6HYn8i0^B3@Iol-xMQYsX+ z_*MNRzTaph0$2G^!1~4nelVV{pd0GJf1#rLC_D~&fs39xeycf8^1s_$c@1tY;YZ?H zRZZWA)DfsBZgJ)T4QX^+p=LQ|Oc7m;$7Fe6#=ry8)JD|5WGm{NQXRS(FU|6!%yG9U zdGt|^`qzX;6kevG$NYTI2zL9roHi~xWY$fBNs#mA3UIK3_kw{=DZ`nr>vlrldQ;8T z&jq+o7;l@De38Kilsf&n73`k~RuZn)_R+^P&RORR{U;~-O}B8}`Q7_JaU36wA9D1D zyQY3Qs%H&pyvKO=-S-KEJ=5udah%>QpIkC8zueX^p9Y^%dXFIMw%l`tVS;KQxpdjr zR}+!x34S3=uFvZG;LceY)zsDH=igX1dd+6vP3?oNmG1!y$1O`!5rbnCj)T0boMJGA z@eZrq@SY}P3mN-^uN8Bq%N@&Tc$oli3jDWzd^3Xr=aiAv9{G_+*Gg>o27V@OE7p{e zq3Ik|FgxC7RaQ?eVKrPpPB|Qu(e2QdCkge+K1Yt283LHUILE(frG2N?)yVSlyGt(a zZc`V>RnmmfLS$!c z&aqw1u&jLlZgfp@?%kK*9|}6$6!u+&X8X(FcnaN8Q+hQSsyij;iCmOH#)7Ef&*fllE0LzAnsu8BOijv6C`76PdFyVg6vT#k;id znH;;Kw{TKt7+I6OyhFst3bqU`Ab63<1U%wgYX~DD;AcE;YBWOwY2)+> zj2RDl{VvY4kleGq6l3Ouc~pos`EwA34tJB#3c5Rdu7LL$y|a|PS(EqJ=t|XM1ksYwb3{&)l&eo4`qq3j84AW?JMyJWm0$LAj** z7WDCAbNtFMieW5~VO=qkEFHi>6%xp9I__c3^IJWuKFeaVJLW=E z?a>X@w@d*--4hr?VUA!)dvkJZZR_3GX*V zHnE^)d3F(E$0xglO~@*SPBVOyFZYinv<%qSZt5waz@RV%4Yj-Axy7)3N` zE<1L-YQg&Gkvm?o$A4!z_3c;=oU>7ED^Vp8y(GYb0-tZ``=6Icn9K@mY=O7xY9XOu z79TSkAm~fhj0ucqg-!`*KAq{dUZs`u){$S<3Gkq$UE>;zOv;&aopeCwm2B7ma~*QR zdj^8)_afWnB<*hJutLjp_vLFTBJR57-RQl986~9+8)ov2a*5uKV?HIKg2$7sO>1TR zWMt7Fd};h_JbzzXurHWD6k-||&hB?_@vVmw*)Y%GE2(UH@viGp%p~e?l7 zjK9}RC?M}<^Vc(NS0xUK9n2g5_~=Ppyanc{0`pO2*wD=6;Nu+qqAZQCdmrntinq^i zpluU23ghpAyIvBGKWj#UER{7Y-nhua0*{dJAmy4j!HS$^^xX2Sx{c~rZmj5jb~Bqf zG3wq6$0f&aenUo+F&kDsz1vCDgX;ca4psYHM1n6BW*`bZV7!?o<;x)Bd0+w>>|9Vg zTn{9}c;Em=ZCN9~l+yTa_NF%iJu4#B3c_sJ=!eXxu^S9n+hRE0)I`mMMGCVcayc`S zV^;fP240)f7ZqkRqrq%2OC$F@s35ItH-U3+n;8$swXAFAaym9>3TD?0td5s*$%Nyx z`Bm`~T(0h4GN0F?MK$E*_FOjqVM5-vV1AewbSGRHo`&`)^sh|mSxQ59;B_&E?+Co2 zFe|lUy|7k8ICYV$#@^GVJ$VROPhciUb}fIHz10!x4WQ5H>jjM2E^5n(-94IU z!vM(p*2mY28jc~8Gwg9)tD9bEt-;y#gl}zR;)|V>K6*d*FFJ)idlZ+s^eGBIQA9XC z51~n6dHMPMg0kd#XKh>TFAta&&YOxKqsm`qtG{)6YyZ|$%%531uIoYhhc}l+f7Fuq zdak2lAFQHr1uq_Wvb(+-Ur48w%q9Qbs>oeG2djT={?hif%eipJA^h#g3<_9PU0z?a zlnS_(pnJo2{Lj&K$JO|SaVsj3rV+A|RVl0Pd5&zMVQtnR(%ea3e@_kGV{udWBlCdLc-{w~pCKu#;T|i|?)$JY>dCbRTITN9=rwp{9@ESKonh z;fofsoX+OF;z{S$jWht4dycF|ViV5k9Fgy#!+GZJcKk!pa10(i2KIH>1SvP_@FJJ( zJaAcwmOfklX*W==s8b1}YZ}1S(Oty!pqAYHeq*H9i}9Tc;F?2}Tzj$v_pQ3YBFE(6 z$q5HFj}s>g`5;|b(Y(Wc-xglz?;vmG*OrY^_p>cGkHfsN`4H;(0dD6;q$R(eWlU^V=#-j62K4yb|3${KAQEI)QK3meTLP7~wxJ5&hR) zKw2L*x}Lypu?I*s$3}E+C_CG)=Fh7IYlmje=kZq$Vn*>uPV0aMHfK;L_pWAoP%Lx{ zZY`T<4V7AxQB<$hq;J<)er>@6 zzdJZ3sS|r*VYEkbG};+WZ)>(1@Vq4~E?aMsyazIb~`_)RmzHb4;N7}&jH}k00J95f}pjhy*&%G2L zU{duEFr0Q5M^3t=>GiTY^yx(REYu5t-(`lN>Dh?i`{p9|_dgEAHF(XT8SPax6Y$j_ z*+0_&O6K_BoCcx%@Vil1>wIUtKpe=LY|su{-dFyZg;+oFFH1ku4(Jz)ti4MTN8p+Z zkD&YFM*Qe@>fx1jgt_rBc(!CR-|KFPm;2R$gMXXzb<27S%DLR>wMk|ML}}_|7l~^v zb|7(=j2tss5+?Di1;#RMP&w}L>L5zyl(8varty?d&E&Y6CnbGu)%Cu%qQT@1wWh==ZG6ewKG?D5}*9SwI}8tgM`|n;dsa~JbBm#CzE99&MnwnC=1Xo}FA;?FdU8u8ZyZ_Gg6C zxN=7e{`UG|Sl_z^%&TKAW5!fKWvvEa7rOx*uA~9wP^5f{5;fo6g)3*?()!0aWvtvt zwWkX^XuhK`Cc487_8~MJ60dqm(hT^_*a%kCSpf$PJ5YS1Fanab-?A3L)LU-wzRg;o ze5v}83cG9mnaHwN=7{6(${{iNA&i{V7tPyDWQ3JsfF% zh}Taobr!=S3)Jh;W9KEL&(w}}a_4JC=!4<|>L)>b1wH?*hn4HhIIXQpj~G@9gf^p2 zV~yN1@<_q;RQjDMZE<|*!74479%d{_=V~Iq>ynoJ1%!WmL7_gI)wLSve4B;q?RLY@ zc5UU73uZWa<~ZrPelLvaoC;lPEx^3+rtmZL8*?4^L%xpIZL;JqBMpT)o;e~Tt*6wNuQAB3JDg4~7sQ8beZpK! zu^lCdXPMG&o_p)^-}TxGinoShjrCnRL)wEV&Bc}N2U8?7aR# zP@nNu?+F&hxgc>cw4mos&5iHUwVuW0n>H&wBM2XHb7e9UO?|<+(92EQYNCDj>_b|&q;O=)6- zjP^i!Y{N-=YfL`w*Q%e5*iu2Wybo)fiy~IP31icJ>;4U3wTvp91kHTyKAr_J>>z zBMqtxiUV`nm&>M@{Q}j;Z8fLy%-rjOG&{J@+{;RxC-YM0CqQ*2^uie`uR<7s5$?TU z|CnT`pSVWpARt`P{0v>Bun(QL{bt3}9N;7aB8~n{=}FCw z$_lYFIun~mX%*iH!eY_$Uv>WQ?R{ZW?+TDt1f^e3tUd2UdS04Tvyzr85(IfLe&x(; zu_~gwB+MqCWWvT6v_h53{d?wA^_*JA6N_L}{)x01j(@sGggVZZgyVudAEXUhiaOy3 zl-~ft0_~|DAxOD|2m6h3Ht%n5|$g|57J>oi6>SyDYy zPFLWg-1|T}T9EezQ;PvW|IQ7EW#e3)0*6l=gpkjcpgYY_()Eg{{zD{rW~8{XX1fwF zzQcRS{btQ)jx|Tpex&P-kzMS`-aM@)0T~K%N&mJ!>RH(|5vgU_I_Xu?3R;0GCV~r1}=5Up2&wTIEBO zCRl#0w;*jx-1!abjTdn7U{9sZpx?c{tl5VBIC%MbuJn@9nKl0uF{Pa;7qIMP3%=)4 z1?K7e&?^1wI;lELxD^Ve(-tGeUyyEKCkL6};lEx&`5(poMH^kP+wWDlt;Po+4+yog z87R#Wv1J(=I3Lg|&q+E_sv0>z;Ta?DWzY66M$hK+@qpJ2HfXZ}!`*eH?zAc&z(&l! z&BAK9;*x~T>bf{#EmJyL`Eljh4|>f3JM+ilk^go^UY+)Kt`0Mwt^mcKU2l^H?za(2 z!q?)1#7Hr^W|;Cb%piOs<*2J9i~#k1r>A!%uNtbfAN9sBE%h#(JTMC1XW2`ok*I%> zJP7I98iM?dFsxP!DTeB2l(s?gq=I@Lk00o&r8U4?y~#k>0@rBIEz(~=+5;7Cn3cAH z3C-P+JS9_od1m`FrhEh8B6Nvut)*P2k{4y9eSqpz&1Qhsz?F9=-3lisG_S%^#eeim zRJhWueWDNQg$vjVyP#l6Q*b<92)p+TfuTbh%CW!pfM#fG_Qe0bmOK?cdj16VTv`SX z^IdpC?K~Lf6b7#M3Ixq$IB8w9Zr6r9U<58|a0?FIp}ob<)A!V7dxgU}Thd&&(p)F4 z$4^=J=vuph4c)a%x{E0vMRlOK4|@&FWu%YA_o!l;P937}K_-LJjgdZ6@PZZXS4DbH z#gaak-G44cz~BqYqrvO6lnw3P-JS0WUkFfEs03PO+x+;ALyT|OpWf_U2RV?_1iM@fiSCOFm$#C4>gR}& z2J7KL{kP!l(Gp&i>LBrjCeYLjKHV?Y5{JOwuZP67fTtq3Zj4Yf!y3{`U5mmq#g$tB zc^%=yiXcI=N6o@IDa0 zs^JQjbtVUXjItMMMq^d4rgXhG4mQklmGQ|YikooN$=;l>MDaFp^%Kqfu6@{@77vs^ z&@OB6QRqEig9`gq-To`zR<|ej}T9c=F#uT=G4z(u^ zd;%Ye?!hqY>dM!!K0OPSN5sYN>5LW6qd?arsco~V2QVF;3TW0Y$o|Ha5-s5z|&kG8~{EzW_$ zlGrZKk!ER(G${UQcNJ*{q;1_UfZ`BHzV?KcX2ODI4xD_R7&<8qwrewCqiJ_FBL35pZ97QwM>9+b=q%Tz!W^mQ7G#iCmZ;kkw?Ow{?Ak6^r z*j!yn>#Fk0^s^fGnGID9VMFINjOI6lf7=xftK8uue0b}wknpd#e@0utj3MZCF9A-maM;zTA^+fFCl@rmg8fEq!(RW|r-sIz zLF=e;oDgIt>lc5*uJ!djBPTG!az&$p{o1P8LJwW3-iRM7yQM7;q9e!WDjttJrtYYGv*md=~(mG zWVY(UH2Aryjhs>DAxG98Cj);6qUohk>=ZqZ`+k@X_cpecfmd3<;jcZU1*d0#96RuZ zgYKjMkN}N}$CBD~Z`i5`2oBZZ)xPTUC#?peZPRm@Jv9>+Eq#vFgD*nsHB01Uy9X*( z(oo+Y8#QLqv*ufDHopZox!DoV51PTne?i!C9Ookz#)9>djkv5vN0w=3k1zVTaJqYf ze{paH$5sbq$*FMo)q9h${MUjfS=pp~vo*uRUsC`#&BB{S#n|}8XzB54j9eW53Dd{qIUTbgBN_tOA$b7{ZnYqcTRQeyjQ6Yy~BHS%Pf6ZC046%K8?g1c!B z{?WMrJ{S){o;O@}a4<)TJ*S^z9py6I?3|8=C$D3TlZ@o?6;si3zJu&H$`FV5H<8)D z_sBmHR&t~35RrCsEJXZhE2)Oia?)Z^8u=0|IGse+v;k#7^j;_%Po^1LoR<-IR$|moS zL$dS0al}(m-((~ob^@ih@e#Belm!D1j{>S)ab;fu&ZKjfPR@A}LrQ1pt;nKqZRq$ccG#ly0FRk3|n_7nx)l{ zsu#6&->zo$&O8UU`U1W%c$ro^YA$biMMryRTYpf!8)I+}$_Fh&Zjh2Xb>Aj@vHXOf z^+NsRb@(UZ5WY)21vmZm`F|6Ar9oN`?tCu->M$E`OuV)BVca4PpIQlCOFW2*Xa zQ2qu+`Gt+HH^Ag(BDOJil@l9OAYHF)Yi!2%`V@<0j)wePdVpx;VTqUSIm`5*&b+?K z12n18fRElYPJYx|l6G+QFu7&gQu^-DMjHS9j7^?KV_$Rx%bof7!I+*c4%#i6#+1Sl zypIbT)#o!T=?u4-BS2nQA`2altG?9qJ2zFWNla+$COeIGmlyh?|){rm%HJ=mhtX^QnAs>z42HgCF|IA8*NbRH+) zxxQj;{oWwu3|6##kyfXBKi=ocR-_zZ!&XKjVP6FHbn1^?|E}QSKX>EVu+c#ME)|xI zh@U9~w}IjYd9B?cTx^(%ryVr77zMzyHytJM0sdT-z}{xn;tP^3!Z+>_QVS1VX^+SKtMeA+gXJ-cZSq1vDLzOKlbXi;#PyGQ3VUSt%8Qu# zkWP=hvKLoe+Xh~pe?md@2=M#V9gLu(y#8?$r}t!s+co6}cAN*QJ;hrv>-2eCv7;kj z`Y;mKJr2Nnulmczt8L_*X&-=igsEB^dgdDz|479&$5^a9>VOmm=9i~PP zmb(qjSxLKk8-oQ#j=4a{mPzXxtoM`C3BDD?t!_%u`(IsViriKVgqnS@8Xplb~yt zC#dl_iktfWP;o`aX1@h-JENY3HLG2?XU*C0^nHoOS+^Q`1kAwkV>hyCt>0LL!|j`Ebxjc(d&h{B}GiRBz_eo@@h+4_5Kv_zUBK zYJv9FKZTy89|!9gap#%$SMGJifb5M7{XMDvu_y>*kd#d~_LF9Ant^=6Of~I@@{9S*UzsE~)2m z-o>MsHcv+?U6NXNXH{=s+1CcI;Y9SJ=YAUWUk{sB6o`Vlk<2e(9#eR^WMK#X({Lg` zJ9;6QkI2J)j*a-P5(jD8;h(nl{kw23y(X@H@r7M{OwR&F4wv38bvgd9)6#pur+hp7 zJ+Gl4o)Dyg*c^v;eEj>K(!T#@ApVrQY2~=?kR@O9CR`}YEyGgwZ&M}fT+$Fp zBZ}k_2FWMJ@H3t+qUgV^n6~G;hH6LneY(KhMn8i8Cca~oV|HPcu~fLQyk6y6cJC{S?;&VF`n$?fBKndLXXFG8ISSccl81Mhmx5S1)r?@ul_4 zd$wWdv!pTMcn6%`PG8Da>9FkP0Qg#cqtGcjA}IHA{QGoJdh1htQ|+ke(?~ea>|x>eUcNpS2JzfIbpZn*|r5&`y5%PHHal*+V*m}(m{Nmjil%^ma!l|8lLl~>W>&$7#6?arL za;U=8F_lB0?ctkfW2Gn8F5k{bTgX;lcQV37^wAqeI&`nLd{71S_(6NacCrJvi_t8^ zA{dTZW`p~ulW@GRKU~QvW7N;k@WM5u-VrsA|I+j>AHhEBq{D`P42W~ku!Q!zTRTD0 zH7fndNiR@dmqPexA6zYJ$ntmb08fu9jRvakW^QPUq{Za5ubEJ^BO99@swGL&z^%l6 zjAD#b7gF=W6ZF5`V~UTiTQw=)Hu}hpxM~Yuaou<;`baN4i!gg)`?uqM=u}tsE9TL-G$w zlZn3V*5jp~A=)3qZ$Yk3H#zCpEhw98jMIZB;egU@NSrH3FQb<(ooSw|Cq3sUv4KTS zD#!4v?jfO#*ojmx;GUL))xK^fZ7_q?ICn>DW3>tLN-i_vZb(X-jxXou!D_Jqsn?w7 z8h9(T;~}Ljk$3{M5hl=HM;D%KHRP0g^!EEDlt!JrJcK2Ex~A4g-hL*iUVRX96qFt` z{_97Y_5halGy?J?U^=6QbZpgxTTc3}(U~*T+V-F~5l$a%fL@t4*wH4Z?DVElhc!9Zhc|c{z>Jmz$T6-!`j9y7(a|&5<@Q z2nZ9@Izag&$^(!;K*CwY?MS*;umDBy&Ot@07NMVIA|o&6L>?G*jKknaX#FZb*X52c<-T1N*54bw zij&@BRM$v;Sdv!&(*1(2Q*$)6sXTr<8|2?Uh9l@q z6Y_b)3s=~%M)X`?qrK2-=xZ40_eCg8K>7iG`}}16U7b{J>3UAV!{iH~{GFYM;KJih->c1FeM*IGqUfi|0dBkTqs^nn>@_ozrWiyOlBj zx;q^SgOyH$X7gh-BY`9NF$}+W7V{VOhl_{K5*N}QJLa$P@t(ih<#uUsJ@pW5POL2! z$IoXrpAxiGpH(_T<*#twR?@sL#jr*}swPH9}0C_JQq|$glX+~cKI@G;ms+y^PMhN z`YR^y1J=D*kCV@*8OIYLuCi*E0Zw*bfEV-1*$ac|qEy#L%_VSF-}g$hGi&G8O6T&jcDC4mLloTk^8`q9 z5jMCeUcgDm+|abcT3XLs{NTS6s2}C~VL@VkSU;L!q`(H>T9Rf9eEWnDq&RWKMaRtj z)Et5|{#zveQ$AJUIPP4V32KheMjinM=XR2$)f7f5TqW+d7SxmUvxZ3cBS@Ple}{V(o^zxg$2azYps=aV z9MVjC7UPUj`;>;{7Ywh1(m1NespdfWPE)()Aji^q%B~OacK%`N@t+u~SBRufP}Mrs zDNsENii33S{7FOe1VQgzP+F9FThcYatQB|E>jdQ%|J>D+7BM`+2%i0d1& zkp3U*Twf({(`&I};#DVuPZs=SeiWUbqA$H7dnsJw#1|@7Z2i5HpyEaI9u-d{J%e3q zH0B<|W7zKghrl;C8A<0zr7Z{Np1^yy2Iv4?1@S4KoHL4ia}R!_vuM&*`fe>8WKRai2W^s)4CVYIZBL}+B z98U+8o7f3Wyk4N{P11a%%d3;7`VDo% zjtR;WBi>0PP8GSicW^_u7BoNku36{wNL>s2w{;&7)~UELH48j_um)Q*U^K5}`;s5Y@b|cGQQe8Q;D=?}Wl$r4a~kG=y_%?!ibe z!04Nt;<%nLzc(X35M%5*Ysd#H96{1vjMk*3*$0IW4i)>Bf-fzC}aam`3yEDt$wG z4d}Ck(!*+QLwpP?ow*lcj+^}dQw|MVn5kC$K!Xen|V2?c}CbQDmnv&*el4bxZ#~O;Gnl(zo zK~%Yd6ak?J#(?HHbfq&;M}8KK0#EpXr4fB5m;eBN$$0mQDc#IlEe>|)ddOlUkF zQoSPJWMU93OEbh2Pivl0{Q&;---dok)3CwcyKJ#-I1ZRe_e^KBkp^C^r9*H7erL3X ztM@p5^cci$c_oG)qWg5a?Psm~T5^XyW4Hx98SDAbQjWAMhOP~o%6=oZ;K(CR;Pt8y z!XoxUw*%HNGw6$;*Wjne?fA;kJ@HBmJs00F7Q-F0;bq1Ger|A6KGY%`?&j*t(Brew zcD*ZoMq}Bhz(sT{zlk1t4Oy?AHPB?aJ|D8$P7bLx0@b?ALQkStoP_!7hRV3Dh3vJa z185uvqy3*++-Fm7d02lVyvi!T__cL#`&o0&U7tI7=0yv>WK! z;e9^!&9htVokueBZtw*msXlL+GhUv1@eA<72B|yYDXPz2>pMu+JW(C88&{V>0iF51 zXIHSc3-Et-4}o5n);aK~HsEJxxLve_pBcM~UpqgL)1Rx}=WX6g@Yt^5vz9hz%{rZc zzcwFW)ffli7*!M3zgjQXet(9?QdY}5xA#EOy$ts7k!=bpFH~&9H=GeAcQj>VP%l5guh)L!P5>TyeF z54JnIB{n)Z6e&j9dSx1!u)RCBE?NWa#v9A#Js%4nqf)B1Xy)vmBU7hrWAquE-Y@Na zKJBUdbu}9O*I&@*%PX<8HtH>$gUnmA7}xv1 z$9^N%^J!Cd!nK5TGT6WcTHm~Yr*8em37NaFR`)H~$lM0sdwteY4rRZh1lsfL96O}9 zlZQn#grzxo;I?fz8#t>CcG(yRIz86oH}3}QNTQ@Ut#>$qdS?qUcwxJJ)d+Qa2D%Q(dbdtGfTp9F_c9h~L|K0F3`FILy{ zbXv7zhwyLHBPIJJ|1@1_M6}q+E03W#*zXyeC-N=}PK-x;70T{H-Coe5S(Dd)W>~uZ6)R0q)Np z#)s;MvPH?xwDw`pw4)=;J?JV&wa}N5Pjuz%&B^H0HUgT&o`Bbb7s&5lEx?G*c+Bnq zc)REaQmvwMwiW)4o2zxSqH~DbbmV!Zn^^Y<3vNA*_KJD2l9ydhW;aszqSveOKsiN< z7f_wajX{ktW_2Qcx7-WEGkxUF%TDlN%1dn0$c(MJ-9!ZV)#JwhoWM^1B@|S);PG|n z`)H$Zsd5&xx}z-IK>OMLY6KTv59ZX5c%v7SW+r#5Fl9?*C@wfRhEMyoKwh^s-~sDK zN~#AKnfe_!8yU!DeU6~#gG4cL+#)P8=!aB)BFcLgJl^$MJPFq2ZVk(E7uQGopr(?r zLOyNrm=QkFGyWqmA-s>wGUf7*MLJeXdyi{#Rr-UndcN=^27W zhsz>v>v&jS?!s$l?gO(L=3Fdxz$b<=(tmt?_*mE#PW2zl{Mx&6dN2O{%MnPMT2ltq z9my%i_xL8wD zqR2@WD#lWA+R|Rma`QoRr-1c0jR4f7jg33>ib z6`6uKTexkb{ebNI&vgfEV}n>n{_DE&D*P0{4zdw z7GxY)j-}Qc;aaCQpt#?3a~d98;Q@q?BF^7Lvu>_E=Fi^^*>Q~ss}sO7B@vr6-UMHs z1TyMdm`C^aD_-#|oe7fmQB!?5rte~OX#SfKW}(V!?Ykq`(L=Z3nNbM0SvMAPvX_eZ zMz3&tRkG8f*5U5Dt$FD2w{X#FIg~`! z!G-56L3=UA=o_2Bjm4L)z(L-sFjBy(@If)zI%z$v3rtf2jLPxM*MnrHtf7Q%eYFq%R? zmja}7kTL0ZehhTON$0 zp$KmawNET($>ugK_}H#3dF9#w@uJ&Ixng7<5bn#h6E=umkDkG&5jv7yLwc;S{CC7& zCffJrQ{VKV9JhsIb_I+u7=w5D(!DR4g5nH@?^?+tO}n$yG6$*uYBJ@wA>l_;zOsJ+ z((8Es&rVvZEzmCNDXAYIc-K21OjH~tGED~I!7qu7a0)l9^`cs@%@sZ-&-*1P=1A+o zpWCj3-UB;Hu@X81uc5)Cx;*H&0o@1a1h}NFm@AlPt z6jxC#GwM0)d6~;Wv)9RcV_WfjdJbG^;i39W(En_g-NPx*`vMi$Q4Ub)<$fdT;TUlT27PS}xBKp8_r}Hn;iFJ^y=n~2 zvOhC8CC{^ZQ}!~Yd3{Z8Q4S6X(jKaBQFnr;F#I!#4~_S6{BL&+xv$VsF1tQh(w~W6 z=y{y*bg_4+4HEZYj}v)#X=`;p@Y7YLC*j%tj#V1&SxXN|_*WG#qu??mj0e&t8k6Ol zHR;PY$P>qnVDGS&K$wGsLojcv4rw4APQ6yuXBW5Bg3+i{VkYv~@jFOvSR{AMx?;5aQ~cu*7)?)#zX_4~x|*eK*z15-;Q# z!q3`WxHPPUk(PUq>XFXk2vodHXUqPB3v}+o*~|vGaraYZJ@&ZDGcHZF0Qw9;_$cT- zMDJ@`#N~DmDA&`Fyg+J=esh@8f}~&hPk%!;`Z%2tLHdAt6cS?pVyAh>nBr5?07&tN z_g;PwV);%>{X~3l1(sdE&ZzIfA}$8zpZx?>cR)2JsBTFkuNPEL*x45{CllWyn4>1-06)j|Cs*3Yv7!ZSwKAQexJc5sp% zevHMQ50_)Zs3Le``I-?9puw8~p!#Cp)FSL|T&pTa#MNoK_XXHiwgcj{Dm}hKW2^lz zWxAmJan*x!&AP!Ki=L#xHe=v|^-!rB!Fsxl0^)Kpzg<3g*FuQ;1oyg4ZCVGW>pVZ-1CvRacwd^Us|1$mW1X# zClJP^AzXY2q%W|0;#BmATcU6Q*89)lrPg(z-js%XoOFb+#K}N?oJPJyaXRZa-Gvih zaMA$cT0%9Rx!ImKJvRYdw>}ZXS@Lo;LfDDnRrT%HSVy|n7AlXV`AzwerWYXc($5wYs8I~J_JjHA-LkvtxBYjp$t+TRrZC1*u)=~SGUrOW-2UI6jF6ZJW9 zVLkFlmua8z&RlskrJG29VY}JmSmyw`m!hku5@ozAmo|LEN8UU1#{DpWtdUvW2K@CR__Tk+JueUG znZVE{_gH$h!>mpJjjY~+C*Zc0_H*_6jD$IY^b}XuXl~jcjvdZY*vO1NY{nL;mR#R$ z9q7M$qRGiF*D5dexj0;Td@(-vHCX$UYRIpn(xQYJY{q+IR9sEEOAxke)!&u2@B3~U zqnN;^jq5bFl}EMOz!gwI=V^j89m|S*CelL&$sUpPJo1FcY2@pm zuBRQS`30?y`f!g>n$X5?J87?mQKW>w3C9#77m5q;vmhnN!ayt)}qi zutgudZ0!Z)Mb&f9JmpbMSe?5Ssekz6fKH6&Kf<&06*P>R=x7phS3|y+`g0aijDdI@ z98V5L^3;qp8AC+Mw;G9Ye~5;@&GUXMlL0>M&M}N_)dpkCO)E#G9-~n{%MNJbfN6=#Yu8 zFS;pBtWk3a(gdOmc3@cI6V`r0ZKSm!;k44k${TR?9^}D@6WzG#=^x*=RbgY)=M)^| z6NZ^Jwu+%UY^BmXy*wn%lI{R$74l=B1>vbAAHb;J={g(Wn@d?r!|{jTC#rcG?qG3( zG%(FB2E)^RDT zR&RwLvQHuK>JE~f8T(s2{;ews=Tt3d6qnBUQvj8;mt0opAvQ8~vF2vZJs=#$h#8J7 zK{vW8w_DoK-n+zCR9oZ)(^&By4>lnqMsd34cay=SscR{ZBt~`~#_RtsL$jKjtMHmM z1_b|EM>;Q#)k^P7`*bcLZaarGBS7;1RsMQ-vr|Yo2rY*Fr)ox$P6HL^fu9C~n%A}a zO8XFy*RM-kiB)S^192%m8%^^(@`I2SY6Zl*N~ft=2PluDG(}izflA9z{z! zes#IhnA9(fG^CT7?NH9OgmHXsTR)|vaqhvJ8t(=RL~-*)aQ?v&;>7m6n$}B8eaZAE z4dm2wuqZi0LzqndAq+}SA6IiNH9KRE=Cx$RX-;zEDDvTHnw*{^2zPWv_tqKW{@3qX z#gC+~1=R%1-rNnzlOpLx+)#l~_{$V&MyK=-QXL^-IVer8xQe)ucTX%4r>_WP-v#UgqWo6*|rkxS%>GFNBA799P(eapwLs zhyJ8IH2MA%P`;h?9A0Vfg7d6jGKw9NC*q~2JE&PLB#nQOwiC=5&69-E(lmcV(o^&d zSV#Q+#hR{A(JCz1hy#=H*)Eoz1 zoV-YUJRfTGqq7d1A5b$vAUzGF_r&KJ=Ry3d|tQEX6g57i?lt`w%mla#kZ(lc0Vz)00YXl(3@nH3tY6A}g%a;m12!#@08sSEK5 zxTbMipd#A(9G@$ehp&W%{>}C zDlW5o#)WW>z9XiiX|$(;ddp4@jl^5AhEFLkz#6`drJ3DiUTkJ2*N3dfj~T5XGPo1p z-5DW%<{xNh>_hKqEw|dv=BF~ZqqUo%!Vnv*jw%B$WxI36JVO#{hgDY@a9AgbaLc!a0Fj|>{42pAxA*< z`nWoOCCFfUZlHW3dmRL_z_k}X&fLP=4gQNQ+*{+TqCH|o`X%NX$KkyB23Y%JEKQy2sRBK7+lPJAtl~XI5yj z=FhHtY;Y2c8oYuP1xe`Q(+4v1zeB9i5?)x;knb=VBs&|e#nhrEJT0>UPjT(bPT2Uv zUANuMxE1GCn&PAQ1H8%4Bs_0k zLt2Myf_Fn`UyaNd(PuX8zmeHM(w)mJpmGOR*j*PhOx!dJ@C(c^*AshV-FV-M{_>o= zIaGU`g==D7i7(PaKAYtwM_2BFQln3xuC1_WFE;t%a#A@5G>x+R`Gy6uM>U$dv)#PzHQJx=z?WyR~!+?!rw9 zN5jb>t zktCXiY}R(IYz%%Cy|J&u9*wYlrIqPi%y(}pgJvg)PriX^2g)A+mfyt!8`{h3G4Po& z-f%PKDLvQQMA`9-qR%=+&^nN5J{-|8QP# zXI|Ib2#Va=({)~hkMWJc3Qv>_QE>{ zd$`9!=HoNiAc*D%?&gwenZGJ1#BRmgAUS4A75;bzxv<#IjC;AAhlW@Q5pmsNoO^Rr zF`pK&gx1$Y&WulhwVl%$#gg0k?xyp#j^bw5f%3!SPP}18GL}3(2~<~d<&dW=a&{}} zZXODDuDPi2Vq2w=bc&e+>xwL8zHeGu^-+qOKdnOR%e-pon{R2s`&frV-lT9a1aDjgi-AE%%##gfzGo`zpnZkH~DVCn=zgW_vG7jCurk3 zl-KiZ1V)3`^84Z+PA&A10cOqMYT*DJLwnw5+qa~8b>I_yw*yvgp}P75;~Y$-o4X~i z%!n0)ogi#y!unVbeCex$Nzu!Z@Ck6oK z2}Z|Ts=TmuMbYw?_dweJqz!vEWVAfdnf5vLrDuT(N5XHTx!@Ulk6j2HBRvzga?gbN zK(!4^g9gBsnf{6cgmL*ZZJ*g4c<9fg?A+`Y_|JH>Bn}|&YR8FpSmdmkxTL6sToX7^ z?23=UWo`~=8JC4ieE`zk&7|Ax_K+5P9jVX6D)$J@MAspdb008sXu|6hj)Bjn9fZC^ zg!ZAGn{49|#x|M^rWr=O&@JA{t$r+_obM6-zV$i9g(U{&<3ao&LJLZSuUi*%w=19; z(8px-)d>4mLe(^3mdKgioL@J&tfBhmfyKHo-C?I7?nA;==#}n*Ln`S!qsmY*+N>o{ z@JYpciZS6U`p&}^QmJ^O~s$Z7;*q(Gnrif1nA-pePc@?RcH0$O699?-_P1_ew zGL%vhn#{9O(m;2wU75#9JTs5Xif5=~CZ&XkP$5GD$sD4)*G@b{C^A*bJWoH*kfivn zegE(w-Fwd2d#&{y);VWyHs$Abpcoa^w4GJT zLY>m78~vSC%=}yP&)${TGW?gqb-WqX2+vs@$3GQA@O^-dBwZ$;@;L7Et)=h)+wS|s zVrFZzRUO}2o0EMrYKt1XEL+R@SjRD(@YnrZ*-mU3*@ORG)<#+ly{+kxn*>j0K0)G1 z)_v_z#f5TZlqDaZ(w?m!ng@~H%}^87LJ;2z!d7~BbpU71dNL$;2dww4EqCSQYIDzb zkdzN#eBOxvXiF2yV0lgSws9=|#&~4|TkuxXWbL1cjBH@C*9; z8p3h!u1w7>v(U;lgF|3w&HzxklQrum?hD+2oy*PC|M#H!m-Q;1LL+bLkEcfAwx}IC zU3qcVP*8kF^N0VHHId(Q&OxM)HJdi%sV;X2`*_+aQ`P+3!0$?umBh6bnB;x;qdwURC*>e9E*N7z;l(UNL(weoWsE>cemmRh4q4t zqgi+UFpH%3O4BMv@KEz9&@0&kWRAr8Io?3^jJd_Z_^aC~Fd14z?*W-99L4`~H&D*U z!p8jt>Ku$XAChyof@Q=@X5Qo=G;}uLlrQjnUpW$f@g&FL=&`OI>0T2Y{(XVC*tJ^4 zNe~WVky9%?T4p2L&Kd{Th7`kK?=0+Gde>b$qbi$tS7z@Y3>A^7qVvz45bD%P^Ralo zcxhoL8$8T~%)OP^y1SKBK7#Nc|2XZ&$I0CYgU8YD7UP=POcH;HZqy@8b!x{A%DUjK zR0BL>(GA8#c4tc0 z8p$R;)YIDcm}zHcf$+AMq9r^P76HfLYWW%@eFIH=+5ycWT9>6mP40T67^z$U;&6V?Cl0cr8uNa%e>LGX zl19*JnnZ8MY*hye4+>?G=KJ1-imN$kMQ9h;R5M}LNc=q`n1@%4;ycUhs_`hE1>y#N zb7(J2YCKRnvF4_+ZXN^ELx1E1l^Ssh4@NZ>`(%YCY_^VJkt?-_7KZKh* zy`bWj>V1rn7LcLdnZR>x2(PDMkM7O*cHcFiyef?g64w4J$ZsI|3_ic{P$VzK<8vGH zZxuszK3DnMS0&?cam8q88`VlSdw8Bv97xBs;4T#t(PQv39hXp!YKRYnL&_oL)#Q%v zf#gXKk>8rfZqIh%4I7 z&7V+XQ+%Rb>m|EZ)mD7Z2#0xApdk#eJPPE;$xl6ls)*aFM%mLJJ=8g%I4}}Am&XF} zGN?IKF(F<@gR%^fd;S3R3!dTDrgb#ky7fcSLP$P|6W(%z{iSe8j({%7x%l9r1K6(h z;SIn4z&!Ku;>(&a8NTsp7a;{?Fx^VVl=y?ck^b^Tj zlm31IMNa$ZxU}~h>1@68J@~zV=Ct-qsPgWrMi})sNK;5SPS2-MO=7CUEb+ITz^yDK z`2t^Y#oLbeYnb)H{73N$^!#y!?fJoV{-ANWIrTe#Gj+X_ zyrNwKm=x@&L&r;Qa1enW~Y40sC?HkTOPiEe?kN4=Vz8DsI+dX&zG=qc|!S z)MpkvJj;~drF>x>_CFw98;uo_?UXmyP@ZWNCy-Z=^tU>W=5gc!pgNU=Dg3wh0gNw8 z0xMq=KGLt*zCFxguo{|WoDoSbWTSQfqiS(^nEyzod2HS4^(00|jMvV#aN~4tMmiibQd*Ffw&djTa9zrNL0Sxz?_B9? zK;Db`ofdAY9+9-8mOKVf{wa?MtpbiR`m9iVKcwQJ@&`ckO}>5rkbf0aPm(ZQ^Yq&< zmESu5=IrEv^c_u!#Sw99tp{njG$cPRKKL|(JLO?2x8;iS_DI(amVxntIG9u4O<0$8 zl0Ak@huPofLFACXL3!7`=Q~rc>Kdea>+0nkVLDE^P>ZWM`8xPLBMpzF)1?~wv-2sc zHpTjGT(tlGU66-E!YDP*tV#Esg62)}s_KI=>KQ0pk<<@hqxiRn_BOrA11MLW}Z7 z-0P<$KVoqL2{V!E6$mGkR$;0Z2*U;CHy+!6opjZ3pgIwa=7hP=F5Azjzla7U|KR@0 z4uqpSw3dO*w9TEmQNJlrsB0n2ryh$V5`Ssj0uD2odFKK~mMzjq_=2Zn+Vsa}oE|mxTLBei~a1c`bapZzm1V3Ftf8ye7?r z!g$9(>Xni%@hTk-R9oC6Dp}Nuc+IN6Pl1URXT@MAdhdyJ@}DN=*yf>!rg@VCsE5CR zI2(xjq4LKWReLJmnd%$2Iq!m(ayJX=t7R+w=~doP@uuA3cn1y-?W^=R;rvq_W=)8) z0qT)y<}_)El(TrLED@=Pp?E@2FBD0SBjt$VPWaZOF{d2lgdO6^ko)3A;9BCH4C+Vs z<%GGCv?JWf5vZN*M)r~c(lcv^EuLnn$Zxqza)ShozOH2M) zlnwr^yRlS`g3>VL5d5u?!fd=|9xHI~-`_K2n7LF}Jo4XU2Q zuzl|^-^T*Ue*@_^;)pFGGucHbk2$XCdZsv{UtnXTn3H~~hy8t{&#BLxvvAb)L*5B` zUO6B|kQP(lBQHDbVKh?xY7s0OcQ6fA5QTT`vG1C)p%l3pps* zmUk*{Ah&qE#sw~|$RBQC3%sVmhDHNvPX$2xfVD8feX?AvoSr!8d_H?)^AgssUy5`txK-E! zi$_O*->oj#=4n0dZ@EP?(P1{w^`c(oH<)d96dQ*A%|GyT+)`bSN0}ywFLMTnbE9j? znL$zVNwYdoHoa0i{aG4r*x!gdxme?A*BtopatYk>90|NlGS6{X2!08sqR4A5Y%WZI zCExbpH+p~DHpiY%%*lj{>xRk1-QU@QY6DsEI~Uz!)A3z#a|qnuP@ag_lTkKhIChK~ z8q@PrT|FG2pdeDScK9Sr=@+S?G4a5cAHivkD{Kxtz_!fE!JwMm{Bu1?@4Pw5(69Be z#VI}B_Chdk@naIcGjr!RN-~*uathkLe9b0Sc*~}D=AeJ>6lq~JU(k2q-9B2ICANoj zx6cv>*4F@Cx2(T<946vA-M9|diGqBH5x>Xz$`uZC1LT@Bw7-Qj}7L4zD z8&3OgM<=hpQE#1xJQBhIWm|rv#8HloJq%GJLS^Wmp=@4YTbUaDl3iSGOtA`a&lpJ0 z7^j?r6u*3QzxuC8ym|v(HH_w0vPQGw%`N3-&qDaLXgtfhvRC+@Zh<|fX2Yd^hVoNE zG*`Km)8-hfv^xR&hjoXbxg&6dzp;!CZX;*flrXvmRAW)GXz|`go}N1vCyvV%LA|O4 zT^oEj`ZxcbL%zso4NQ!T=PH*@8@^_5R*mL8@(x4zg%xn&p(PrJi%%vHpyrJG=eSYI*1$vIZhx?aD zLucFf^lZ^CaP~Kp#>f0+(B)d9Lr@so{=0`f;W-@gU;NFpik=H+|7f^6q6Vh+)00Pd z8@zkjh%aj$i%AaiaEnV@zW4n>#>a6UJ#dJ;zwXy4`yr$cMh7SJW6Ac~lzw$Z!@`Zi=~hP`YL|*tV_J&z=Y4S5 z+BI1DbsP9PuciF3#1%Pb$lJ}Lcexsi(lNGjdjAB?+Kd?K>NXfDk6Alw#^b+kU=>rG zd7R-3xaHhP4jAztj{TMda|0U4s8v3E?6kXh%tM2TW15N^jS4WNXrjD(+lMR6I}tjU zs~jZ!Lbkx!DxRSnQxYbblp5^#1DEVWw-PnW-q_hq{E1sM4n}|N}F*c7ptaC zg>##2@teO5_*I7iVWP+{-bwv{wHSBVK>B$-Mf$vW5l!nZo4JEtyBnyr?=A1vv^-YpgS9l~u!t7$Ye={Zq z3XTQw&!ItLz;mtW@u$D^TU-s}r+=Ut3s;yctH++jXsE}n$F`A_8}vC1q&$s7HHNU< zm!crqk?yspz{Gckto}ex^er^!2J8QUEpt4ey7g{mSCYbt|J)Nj>{{>_Pp!~5u$5Fb zll0CM^Itq-#3NYU^@-?w#6_M6JcPe}8W0zp1A`1&4>8sO)@J#sc%WbE2()k;AYF~V zg75KF^h{q0cQ$+|HXOajh+FaM0Czda_7i>%{fc#MhH>lZGjPDFtFXZ91^eCRv>-kO zcY8PNJn$kA9`fI9=zN8{u!r3q2>Sr;@tYHou8&{LeUINRn@H~Pip>TCSi3R{%oF#E z`evyrHnLz*u^;Lh;3CXA4=1o0yu{i|cOT>D$*un56|?HEjWx ztTN$6bfD;D))D>ax$o^=Hsi5HBe9QF2WZjuEL*Vn4>QQ9#S04R${{=vR-K9l|I>SP z-y>W*S5&`<_--js-m#)vS>RoF2Y5!=Gs0uq>a8=pIeZLmg?`q?BzLBJ{-5y4DRXqb z?TJ&Z4x!@E=^5*pnupS9lYzJvp1ePbgdu|R3`h$}{O%!N^fN~7w2AJ|p41|(6aY`6 z7*bBF7~}8A17fUw_L-m>Tgb5e0qy$J9+I15aMJtzpzwvT0BMf+8`IXjpS1@LIqECI zpT-Kp2Mu8-#MHIrbZpW-TOczbfz2GyjZ@ttX&&CX@&^XFp3^G4QX0l}%6{OqhE`a+ zVs_-^SIGVGk4~q=Z??j3h94CMDIF8;i_x1KuY1=lOO-lbiT_ovNeqj9yxRTLW8u2*s zIc2cYlUk}>>09p@63=Io<|5pR$CzUuQQ;$T8N~UU%LNHrF{$DXVd55H-MStSR&mlp z(Dl_Spm<@_^IW#JzYQZim+s$o3cHsxp+_F=Sa{2wsrp_r?Ex0K90I-5O|VzHBBgPq zUeldo$>?6zwPJ{byB@lf?5?|`eULV{b=yDj2ywVhJ748AjFp_v36YV_}|L|6p z4K&0Hg6b6>y`%jQyyo-ju9HPbQ8{VuacC{KDXb>!Y|cr~$o56vW0u<};a^#US(^-G#}S{z73;zB-HWG4 z_#|A7f>cf-)g7b!5TSV|MXz?lm9~Qrqo1S^y2!K)GZDY59H{M zXM{%)#1b|=bTb+!K;;?FsQhpPCLD`U4fN$ z-8khDE28(~du%r2c7saE^N+$+=e?0Ikrlag#@R6fgL8IYy8-V5HZC zc1$8#{6388_e#T%M?&Aw7cjtP6k(E~Q2C&5)r#%)wI;6nil2^EupEakNLm$$C%Ez! z18nZ=a_C@?2|D+x=&AT;USJB;9r0da4R5nzJKY;S0}Tsl&$u&LRPU|j{tup-HdbvV zVGF(U*jCbaL2)Y8J529y2?vHhW1WJnKI9- zBhO8}ELODXq~r8s0b6kNoiE~k(JNf_z8$uBzl+`%^ybko2v_!7NW}+ny{MnswLpiB zbbYE$$%AwhN?($9Q+)Q@p%OD*JS5F+3+V&gh~okw&TyI_jAr*Y z)su4<`Eun~E)H{*r2CPsi|Y-eHCXz$7Tb#^OGnQu@ND^J@Xvh)pX2MI@>l-L!zs74 z5Wj08^L*bzhIH8oG#^TX(;Uz}qhpIW=^r3Ghq0*7Jlkra>gqk6FW&Apg8C`Vl{Y5c zbwyN6IVHZh*vn$OuF6+~@-7!GcLM2qM&nWX8GTa(q(7%MNypgYuGWn~;jhv(irojR@8Dx6B z!>bS0z$@#UihGdqSCS4T?A#0CetGcZ{3z(ccj$1nesELb153PdbUb1DB_K>?=Khgj z`K=joTvItZ?-;09#eZ$UJfn_~KD~%lv0C!Jon*XSM@~AHcQQN3C~j;(kO5Z9&V0XR zJSPo6ezP04u}UF44F(lMqYwvSZ2p&qbON0Fx{2QXM>g}hR5ioqIj=*SA5gKSTFIu^ zkiIpQq-Vh3nJriGnjPGdkv?H&bGj@4gt^Yq=q{6Dxu!@!-& zW9aZPDW@kUerpS@+7+^1mYW!PY^2Y~uI8iR*@w&SPP5D-$(K)Oku~QA1|r?H?eocw8x0y-;8h*za;FzETccl ztApqJJ#7E!R9F&w3`pYv;kr;cLwZcsJ!~Z!9{-894pUiR$sz3Xu>ht&YmfC`+)(&K z^_YSmLjHwz&-QBL|Iq#Dg4NU`N!31EcZVr$Lb?EmFEQl|K=J0ixL}UZX#&!}jI;vj zk@>WT;2f^7xm^amZ&Q;&|YQl>VnSG<#E+J5u{CE?CyEs zw|F_QvF#bvDq-mr>@Xk$8hZA@0j67F$jbn{dibyyaG2g}b!FH!c^^J<2yq|3jQaZvG>$QLH>s7rxx0k zw1tr0E?TAc%>C;^Si3Pwm#{ZMjYWo;Gt%6EZN&f`hWMr4(Y%9L(vuEAzED%SIT;9# zg^DA2FL}i}pV4(uo}OUgR!0aE=fdba3xPb7!W5x;PK23QCOZImCNT4QsHHvx7ERV8 z|MFc_PyYp^8L`)}L$H14Wl_BAIg*#+^m|Y`ROKUiByHT>`+|BYg8CASa)WB=5Kg_Y zRPiJdE&<^$b_!0^#k`<2UwLzgT2+A5KLWyCB;AWW5)wHSIs(+X`kM(So;!6Y1IJNZO52KZ_}CG3dxyu$a<9&{)xW3+F+GPgPDZ@=d6E zMWh>Hm2Wd$p3%=(eCN0FuaviGSUL4N#w;q*RPLhnm`B%xb9Na(HHOM}P;Mf5X)qXm zhxB)I^3r9@{hKx1iVsKSNt8w-%>?s2hXQ$IjcrAqpgxPJxY8a7PpEe^f%#R}lGHn7 zgvF@pIRD7M*=DKUs{VzmTfQcd_TVDDCWvi8~neB|v|LxlWU{bX*41N7HfD zfK>xI_4*`#7m2SjcED%P;_OE`IfVJ1{KJ0h>}TsnAoWLBV55FIEk`&@J))M}>{cqO zIUzkmJqk;?1nY3ZNYY%jC3z9*%h^(&$4zN+S>*K*uQZHgMuSF6)t@8|BhIXiDrQud zK)!`=DFt|rBUH7D5Mjd{LHS{eo!J`Fi%9(h+){t2Ak8Z1{M091uXq?#&noomI*NA? zBkn@V8=N`f8#@>j17XV}#b@Vjr0c&R@i0HO=$?uXqqqygUpQ8p2lGN-D=egXT0@%Q z6_Cfo&a|J??}`aXJvJbJMcO46-vq_6icRZ<^IUH}q=eoxemEAA-fd=DEh3hR*Yowe|7$h+{S3ImYv7*anz5c+410_D4)-VTt5(2%}V-j3Eb ztF3&X;s8PXrT7(d#tou=*i3%qnS%&1TEbO-y2bb+rhKKRZzD+HPMpXP4b7 zw@l5knmB;Uo zK2+H9$frJ1&ksn4a5Yx#)k39%l!w*z2$Zi4GW2)<=6Xr_e+_XRC*6Q5c5j{!5|w|( z(0A!QC9eRgyJVc&Z#DI7Vu82;sc+~`*TF2hgex2b>SdtnfyUpdR5&f;61#HJ%0kti z{)Gi>rs;Nd9AMMmim_*DO%ShQC|ddg3PudY?x)kh_Fz|Lzj`g(ezpOwxs<}Lwmg!3 zW5;n=7Cn<$^Gv**mxzb3I1JC}!tiz{o!p{&w zC%M%LsFkIMtyhkNvX;B~OY6hzOLUaD{~zshJE1neT2vvzw0^Xva0CAB>l)3XGxM>i zoe3On*Hs38Pr^{sV`A_+W8U~k3pry^ZP_t%DLtq1S>(-Y#y%H~f)kY~Vw=fxy!gHW zAM(bPKDP&kUvq(r?>A)cUt$RUX&3p74hc~G!c)$rwGC`OFNYV}FrHKgMbemw{OG21 z@hj>EULOAyI%V&W`J zAQ`Z(SEY#DbQK?M^wwI(j^J@8=E(;`(!hN04y?Lc3kJ3xpkjj6PYq%5`HPy}zuv;9 ziGnl1Oze$#_DLDJ`F9mi-%$79b=y@Z(Mh4Q`6b!Pm}5y;4M=YMa=#JqX&c)sHZ89lNG zerlrO;S1JC;}#dd_#bw(UbNg3>XDJt!;Ea-#jo^>`pW1>nG7>8)iQt?_y8) zpk0%BTh|6Ky5=rEhG+2iLt0I(5$(sM-g#v8U}@(Z@!@a|FBrB1vn$@fNWO-RxKr2t z;pIm-_~B8!rQL#QpN&~gaDBGTs4J%&gsmf%%jK`e$zbXmw#=dXE*s0mn29U-5hqu; z6OqMEU0%tnXx)pJjXLvnbWgK>YCYbA_E_j=G*{$CRin|#&6GRy;aAj986Vt9voeL& z?wfQ7#|IbVl+1kmWFDcVnv*8WT<}-vM;u-Am!`vTKR9*qs;2+U$y}?)B*k4WyKf75 z=k-BaUkBG^7O43{_c+|#@Z`v5JU_ale3|cwwi#3T>>aDb@EZZ}#zg?l4|aY2LeQLX zn*kAc+~*hm<=l)U5;?_Z4IX8ok+V>P2Gz{l|J?|@D=0)S@HobAJ zLrdA}pWPza{x9~&?>|k{r~gFNilty|vw>Yc(@osz_%GFBEnHnM3aEC(FtnG;V{60A z@C`uY5mb*Doz{lWjvJ(MS`IVyk@nrJS*zAF`IbH&+&`W6Y}y_{ak`81S|5aoD>Ino z{SUbAdyI;=@SPBj*1S7bE$vGf++Lnpew>X;IxHIXZYwW-y1+VRH>Gz5r1n+EQyBcc zIgGy$!wB!hy=~vH_s+F2ciMWM_O&+++x|j|8xMN39Tc{= z`iItMojDl^kNE4mjaaTzdqEfs(P_!L-w(1&luw66LW|R9bofAZB7T2cfhva9@63R{ z!!xJsW!K*hX6E z`gT6FyhQK!noQ+6p&q!@{+s4?LPHQ$JahUx-O#%3+yvEDt-o3y1bQB-xZzNEk}^G8}q;p1}Jai~*}B zIY{%Na$F9>BZ{N2X4_pjP_s?WdhQ});|>AkBu+a&9Cufqf`RQ;0L7LcdGG`O_$`o2 zu37LsQykIxA7kFx$_t$T*$5xc#A;Ha^Hgk+aE%q^`|^gx2gUFXouqa|89l4L6Z+Mo z_jQu(k!pm|_~cmaf0~TYg6uDwzk)&dc*3V6aI%g)9-3(e!*9Gp`nSwGzebl&RJ%M$ zKN4Qdy#dxA`%5I=$$!i~uQ~(a zt(!^Wa=E%*J#M-Vz~P;qq#WlzOd6v}!9A@1BORt3-zHv1O@Nf>0pj682`c};4H`$! z^dxI~Y$$?NH7+P;>anay+QYJ22A*#71<}X^RE|w(5vFnk9e6U*anOC>Jd9)=b=(W* zzpx1f+fiZT>T&6~ZJedZxI}BKcyPJ!=@RgxURCt)TD)1?WDz~`EI@W$Im_)nHu_&J zm|w6K_RTvBv9I!BZ<8u9s?R@C;i;ZsGOWso1L8q=w@VMcMAy?2c9TBYfJ-~9gRq$= zu-5GBP%r5{D(o#Q>;}1Eu~1USSoXI#kBaw(&E9~uvfWr`df)GRN;6J)2827p@F2&n z!>-`nr=2BXJ3EoKMyS}1xDyE<&s~9c`{#(hN0&f+#~`4w@Pvq0pmLh<4ptj+^jI>M z5ho(?E_`0)s)@^}Ei$LyW2NTX;Jwcf;g~dp@1gZEGpxsB``&TrY~d@OPi)9j-+tDR ze&G-2)#KBx%*3gS1Cg)^zaQ`xPe&jcgg?Zs`;_!FX&3(QkTb$7+zx*G zi0qi8=NH1c#P4{c54|Jg9|6ORE#Trc zkC&T)w3zs>EFHIwsMhHq!aPXXSOYQFB4GT2WPG@gqpI&oYyg}b84ttGfsE z@GaQ>d%4O5iq|3ZY^%>Hp8Rx6M8Xz!a9n*E9^4aZJ~a{^*Dk?I?|n+!;-rgw>0|ac2{GrAaA!^!O7JmkElI zB#c#hRRryNLiyA{eBEIxiPzahkFGGUHNCHTe3;ZEq@Pr^M>)z=%t!~)el!)hGO2<5710#b&&%DeXesBAbSC`yWhF@? z;&yE>mH(V@Ne~Z82i^vZ7fivpns%UgUVV;qn0C*U<9N%&nO}ICjTc_bqug8r$9Hvu zeJzJ_;}*Sokd|VE+X_>3{7}}?mRlH_%OfsTjWE+8Gs<{TXhqdwmN@&K%^*J4>ai0=Ozc@s9Z0U86i+I1E`rUvlV z*Ei#eICpGb@4>lFrk`*c?q*-ycY|Hqtr)&QUx!B;{ZYh&ZLrpiCbIeH-lTa4GQy^F zq+6i6hd*|1x*jO@j7|B2&6fORZCy72`3MmamJ7cgoE84b8=;T)Ef!okf^cXkCoTnr zU2%wI0~c*VXJp6J{z@A=~1X! zQ6&ID!;YU;F_CnwvQfVh&@&#xMh`a9m=X$`+Q8EtuNTxZ$!$uSHWnTJbmdK^%q*?5uN#{4`vz+&VKkZXhS7VIXU%RtTrvLC>ac_mwxHsBe1NZ85_~LmF zFs=VIX?($3+%h?(GzyX)#}@wEP-&QSqa<TQPNM4{4ikh-IgIN%i{FOG!8 zcq08AqjSNn+!`H@2lq2%Y5iX?x=yL|;jg;V7HoY+GC$?BjFViKT7f{{PYcL(xy3@!Rt1%jkX8Wb;!l+LRD3&i6`Fv@*;S2n)rLiQg#-ZCSWV2hVk>*E`f0fJK`g4Ufje4g* zPH2wEDoJ*K+xI8wkJVgVgVG6vTY~r#{ ziOROR8a~~UUMH!!ghinaGTHtvX{J0F)#sDumK-3pdIF!HnvTkYQ-0!z&W{wPLdltZ z3R}eY125^EnhW61DE(mZ*ns6RL#AHU7j zkY2^~i5@U~QDb(p{uHM0FkO2R_bvPgsc}{?DC{#@(R(fAZP|!$8^WDo+EDXpFm+OA zT*%JC1E2AD{_aU=U2s|XFqOB^E7qBlci@Wi$X|2P0g5M}&tQ6Q%Y2%KbPG;g*@X{S zkcs7MegSzP@TRr#Hg!Hmnq?FI-g7`{Zuf)Z)+wA)I0#OoyYp5pPAcssmfhbCq^HG5 zuXA7@E(Fc9u!>0{yia!@*sex%f6o=>v7`iuUs0Wt&$hv%TmA-<>n*h&i&Eg><-f6T z@?WHhV^zMR_m_G|*v(W7m9`}u6cgF)@dK9Y8Z|Yok@wMQ-QkPka3^af-P4}2`c;oa=9%2={?ob$;t1^h@c@v9<#WdMgyM8d z%#EP+rL6@}-BZu2JAWNT&nlkyAKrD^PC1-`6AGFj^*`d4!Bzp_qr?TfJRs3G^w949K}L8@gZPFjk!Wuf$0 z6Vm*T+5J8Jq1(acq@5-4U02|?GvKUSQ#M844T-ZE^+L3in-tsIO4qaa)=|*@%RwYu zfg$Vx&b#C+>3cZLI6+kC(O#)1j%#=1|ACHw97VhBr%BVql5RPO0ZBHn#=?|SAB?%K z%oobjSr~2=7aoP^9U9^@H~>ILh7dw=dDK4e1d!e9@w`Z2*2>cCQAtTx=>*kI_})A z#>Nzu5q4>b&!MT+Fq|`b4WoEM)zV9#dIpYzGuSPe2ZSNy0s3>rQG=I`3yL+Yr$ ziWr(OnD^Y?4nJK?z?f?W?z3a#XpNbX>ilfZXnn@di@OFf~@H6Y9B!qv}KNN-*N& zg|R|!1H|r_qP7O`5eJaGkJ9IGo7UQ{Q=UM4kcu?dg#Fuq^fJ;pHRNfKFjnC) z{P|r74J`i07WNFI=gKdTXQSSc=`lumseRJ_boT#RZq#W@np1b>lM5%`Wy%LMz2d-? zzt}gg2lU>l{LfhGXa7*VhvX@gFTt55&+yN0UvN*GLVjbMP`ut5hpOvU7{JKu0Ud)= zJmCH=GbAqn)UOh%M~>RoIV|I0SdEetZF7Ry#%RK*cPbhuO3>Qtn$owN( z&DlDl_OooQ*N{%E@U#U?-4%y9+p|PP-$HyGGEvfZ_{3K&c*LR-HpVGKv(w})_L|ZY z3c})WWLhxhKl0}PyrA`#9HgN6K)a0l%*3hPAxvjzXG zPQa+tpSV2q4$yd{d1e~+JvdA5>NiA=Y4C%UnLdQ5%h$o<;d5N{W(z;M+M4FvAFP&K zgZHJ=@!^BTU|e&8uiZKr*il-?B;*N>UsMXojc&pEp{?Ng!^dEr*&f^64i~SN^o0e3 zkIKD=Z{j2VE4%SES6OS}cl4-sh3;Fa_w0BEw^*j}bLo?$kBx==`ezSxwYGtrCH;7} zY4$*Wq~W>Mn(pykaZBUA@Y_3Hv^irA9gPvZNRK8@w#s$BGJd_`oqMa!qxXrGGZXECwaVP-n@6dP6Bhz)3V6!T7Wg*6Qp z(S6Xfns(1Kp@nZO3z%{i^UiF7i>0r`g~tPAhaQor*|iP+e#-HbX&GiexxkVVTcF<{ z7a4VVh|H`UBCUdF!6&a-qCx6%>Aorj)Nvc6R>Qg$dUCkuLf!W++HSxv2LeDZp_6=? zvH~dP=zZW5m{?q(xo8im(^IhBBR~8x*-P%PIu9{t!*NoL13$WZgZ5O%YRMYV5z=|Ce?F3etENr8<#!8HbMWukZqoLq|XB|I#iBV5|gmO z)AQ`&uXFILd^pTGd5G4>*5~V5td^-^7P6wRublX55C1di3QU{7ic_uu)q(7sRG-&h z#$m<6FA%iCfsd-aUe%f$w|X}`uRbZ3bWg#{UbA7AOEcMfWoI_JNj7BH+9vwGHRisJ zhNI0*TlAWlh(T*iG54mWq~qXwTO(1&Y#dN-@@lsz^*edP$DFUsxC@Q1oq;g3(PCD^ zX&5(?*4y{f=krS6VB5pPaEbAJ-pTVT#`Suk`R{~1Z(;HQ7v-JrN%hVLIPZZ@Va>q6 z+Lm_=s3&QjHEs`K~vhpV=Fv;w+0@c4TFGQFExZC zVhaDwPS)7-!cadPytshvUeQ%^$Bp*iG5!J`&DSB-6<^wSxNuqqNI46&;s%J0T`!@+ znP;`)_!(ZU%aN@^4&Z89C+ps9L)oFx2%dZOEG%x?U56c&3CTG0w!Vx!RGLj#B{wdm zb>vqk!hGlE{G|RSOg8?Xwy1oB^eOot%*is6O&xpkl0#!uedFeL2J&8?`*`0Y1cSHb zVCy~llJXQN*4Qy13YxeiqS5Ef}XFFzL`jtzphtW#dOM6S&TxCF5 z#mDb9LBdn^SH?&rEXKmn+d#2`t;3?^>y_=b6dQ1Kvf~v2`Vey12ZFo;g~Hzvp?|@; z2UX;wI`Zp1^!dLgvtYeZHJm+~E1zWUgIcGp6dnT23v4Qg<1zDB!r%I4V7Ooz3{R@3 z34K(EbUdyJc`DLEi`eBgCi2zH?byHctq#*lQbTd|&nr~>>u}}u4)o4M545}9i65D5 z#McbWM$7O9{LRS@l8z-er9WW}X7y<~#@j~H^}@Y@JwP=67n@k7;b&jOH^|vgM;SCN4+OV zHCEw5eadStVn(eUQq@-Ep>fcCK^5i&y+d2QJ#gsfWguJy;nGC@@cbkRANZ#=8?!T- zoQ0NwwRqdS3ygk7!ZOW)<1T#1$xc{WR-#?~v%f6K?}C>G+X2-S?FqD!)mj)L+T!NGk0aF(6d!ZPWnoQN_LE*{xPbPyeH#N!EIWxe z!O!t>zrol&{-`GX*%?(M@~`7rczEDBNUyz){pmB3H#!^1{(dSMuN9AOu3N$Q!3v~$ z#leaD7;z7rGFpZc4~<7>dJaMH(~zLK;PdDfBP`&G_o^~(D@e z`@&w0AHP;{xjE&JpmU08JGvubqcn^^#>!`zN_CyFwYF2;fC#x=3!h#)p*gnon#zAT zSyl=b%?+WwO%pixG8ySy{N<2#ocKzI=@eTvf9~aWLviHm1UyLlB2YX8VLv_7(OO=; zv4@`Vm_Sbn7Y z`e_NN?8}>m!Y`mMr(`Zpt9!X=Gv zQx0~*L2GFJ+z35!{$xjf%1@8~@cf5Qn@?*zdH#pxvo_J`C-z3=mam^Pzlb3$#9Ir_(&Du3IPW$9IqWOP5zIWn{iR*WO-7fAv?Sfuo)7cc4#bZ*&(;J;L*msuDdwABC;fRG z3Cn4VvA=<|sIa-Y8~^mV2QT~o3sk%Ed9@RK-X_4Se;&L&a#@G#cDJJGy^>;}d4bWz z-$m=izA8uLtMIzqwQQnLoYisA8L{K0l_VUN#Ifx0u2>|Fg_1+X;Iz6q5?3P49cb_W zfrl0dleGb`n6N2F#LEv?eq9%GfsAGge8{;fyxuA z8RE*n*yFRc6sPLABWA}*B<%*Nef^MXPy4k>Dh(!xE7;Fl29mUqRQy-t)Iod>qV-D? zTbw((dM6V0vt~2tf!zXH52^b;R`fd!HXXgd=rg!+sh8%$i%zoYdRuhKcMw5d?~ybV z)%G}~`en0+bd=R@_4sqk6g)@o68$`$$^53LLCpN$K=pvln>h%j^Fj@3XW#2HO7^$i zgbn>7epe4$IJcCFUzDyP?2x4QVa2T|(ciWq?k}0k ztFBx`(p%WvcC*rt*!W!(SW>Sm!_81CP9(hs>uS0J@id$H@EI;TFjwge-YF~|N5u8Z zCcmQe9=v;TP{j4WO#HM9e*I2`j}GoS-FIfyKKx>^n(>!u@Ld+^tU@##t`$Q1rXKRJfsZIq5^D zIML><2_5%{@-foHDp~O;v@iJ#q`4;@%rANssu`9?9c1 zLk|}U6{DQs7d;45dBWp?Xgc7c&YR3^UJtF748nndddz)QTRyYeMYMerCl{E|b7lqG zHA=I^AAE&JJQfj#Jx1bFkzeuwi63E{$6}$hb{#hpX_aLxOZ8(2``chjfPtWMfQRKl z&APJjFvhGHiVGV6X->8><1cZy!8hzQ`7&fyURQk0S!O$~IGt)lbR6W$3Fl=%ujyjk zqdw3B_d)EuJWIqjuZyadh_|$)S(Tq8{}u|Q z88Ei`6htf<%YV>bJC|qJ@+f^%SUSC(@SkN4Z*MkJIu!TNUdl(Nb>>+ij~HoA(w7`x z@7%$nyB&uMde(TOs~6ZDoK4vEN2mXY`^EMwLw?HWm%#p)Xb!J2()QA#c{I(rrB(h{@tiJSJ*v;Zqp8QBD#H&HH`u3r2dZoFW4RL}-5^r?-WJCXjddsACfxZd zFma6|fA~c!drbw>^VtUFV<2Ek52?I3c@}ojwgKz+crd?fc8~OF0X!{eB)ZbNdc?6n zapgUXeq+5pGBI6kUH_Wuz(LYe^v0$vRYZR^xPlG<7T;W26jTd{Z}vq(2$?WFQO_N;@kL z`J$me)|mapo?EuX)4eBxid#Ru=1`cBf;!!N)Qh-jqu3C!hBWe2Q1xHO$zG>l6oyc( zYssgRS3Re>(*?O*jQpw6V=6vO)fjONqho`?&uiIwp`(!cATZ-eAF7iWwslxN zHgNDUpx6qEDS0C+ezWf_F{*qNG){j`oHYhXr@-psV5JEe{Z45vE%764ewm_TO#Uen zT~e266wXkMfzqOS%X_f42mU80-je(ztX)uvbi8b($Eluxd>UKRph64`T#Xy1p1_-v zFW|G6t--f@s+gYckF{>CSN#~mxEOeH^gK{5;)DfD)HP_m4I2T`9yo6gBMMC***P7$d)bSh;e?#uUmq6oCeIH3ai*0df ziqG%=#>&uPm;D9YI(~}oAEe{%HJfzWn=~j>J}blQzJ~lDd-wPX5FSCDCIabq zaMP34dRm&Jp?M;VPXN-~OnH#*3zh=)qXhYFN!kR-`vBnvF6~>S$*#^6q;oN1w=s}k z=fzvE;-nT~N}mFGN6n+&Wk}Z!^Ym=_llj$HZg+$Hv;VB=uvUq`#$HqUeD$IK1T-U zi)Cj@u+)_H58L~ee1Q*w}e&aNy;Nicw0-% z*f)u|dJOvajl)up^+IWKm6dfp(gy1qEr%=rbpVAGTdTZP{>m0SZ;$%*9%x7xD&4_c zJlBF@q6_7&IjFPX4p64aw02QKH}fvGpHfFEt-5$>6in@LMv%_L=?Pxg?VJWE+fX@% z_!n-)7YF1b%_UR040$^AsXGxfU0We>h#*{3*)V$b-pK-1#0&HNIY#AIo)?yuUQ?;Z!p!I^m zNd8`}Hyq7rM*Dz%5%UkX6Er49nnU4*P~IlwNik4ChZl) zZZ<&rMCE5d94aZRLzj9%8p>diJcounHj;lNj6YkQmr+gJ$SB@OLg8%8UKu)@)&kc4=k!@c-V*>{ni1(4UywuQ^-4BnzC4=;M z#L58me%?#Y`N@?fhWti=ojCQ*8C%$8Xs`Car@bg>AF;jJMRhwp5PxHi!tAPebZ9e< z&bds68#}JSGWY&quyUBZA5Qzxb#v!OQW~Py_(;~(ybU_sJP55*4P>(A|KP@h&shI` zbD2HjF6wn8-RWc|i_>%P_m97#?Vxqy;=Kku$J3bKP525qXUFmfXCDgRSX1s$u$><} zpC()yUcmb)_4#d+2e8W3oYyzZN3%wC`2@GGEPc@h^j!D@GEKv!@QuQ<27{z6|H+bO z^@H#r9Zc&RAttzm^1ZT|jYU*`6S_O+%X=Mw1t z?c1OHv2fybxUtel2ARgfoOegjt@S_6zxU?^!`xwq#w6xScXct=F4FZ^IvPg-C9h#3KNzFt+lHX>jj6QdgN zJAu|{^sKd{Il!{8DAg3U%|;!4YoYHxR`m}1=of_h z%97wm|C<`SxjIN=;}*XjA~cvRUED`-iv}O)taEEfw{Iv%ej17vr5AAVH&=+w9E5dW z?ZvOKls8^pNYCfBgL+!Q){VPRs`cZZ%%ZcZgdP-VXk)Y?tyS5i>x9&uI(zcD< z-Q+u4d~XM=XLaQJ^n=2i&PDVbQI0_?1Ho2r2vh4lq+xfl^IkHXm~|0@S1`y z^dKliLa&~6QLWRs%=_?UVsCEC7g7e-KBiUYlKL%^kMA6s_9gj?s0;seBH z=Hq{rp?R`YpC8%!8;e|0SJeQBya3*0)?*lH@&>2o-2}ogNf;yu3+Q@&LRr7>7_fP@ z^gjEEHQT&i^Wf|V^v-;QW@kg;gC-p#lhR;s|50eQ&w=lqIg?+ztO3GX7#5_@M&9Tl z|HgC_m)0i>Vf+UpOp)>|Lucx6Tj)Hj) zABp^aRX{wdJ>&PNnm>E>Tu)rqj=7o4#Bx0egxzv?mMeZ;-+=fa0={=k#yxbVEzN$S5z==t_K9dV7}UhReY-h8w94m@FH1tIg@ zq3$b37=P735>CkMR~uRVoCrL2U_BOWWPJVnY@AW96}R&p;rEYa@>uy{yw!RdDh#7> z$*)b`gXe{KL6{8Gch%V1+pb0R&%eo-OwV{6RF|2t)-Z$5 z6dSKU*5*3qK)%~lF*JXq-SjX9)arfWet5LX&TKa z*CeR=XDbzdR@okc8Rs)#R(BJ=epo76Ha3RAEmKw9u<$T5{wJv(TDUC}*Dh<3@Dv;h zY~_^LSfD;p7{RBv_hkDw+3{yjUAU{Msnl8Yh@ENEN78lSm`?pLJ7EzTe;UV%f4&65 z0XVU?6q0|wWlz#RiY0?qV#dmDaH+{0I9GfE6T3%qsy*&uLHARxE!Pkp%k4u^OPnJ< z6|6+U23+O3N#oh98%+3h9cbO~OhF8z>%$YXdSS@0=1^;w5irNjpvK~`bqzXH920Yn zB#5$pVOVF{X&CpSN*Iqe(fV8M1FCGSTi4}@3D4J9E936;peaYBuu6=k-O`P zdFhBHIyDtX*P`wx?(dCpwI!6Uc(ZmW_SWR%k?>kbSOQdk zNMq1;k6Q~t0SB@_ghUG zG>Y8Z-c{*H=2O>MUeX<_@mVp46~*evpa3TrxIUAx*$~!m^yS2vcwYJ zETrY&cYu{796;&|I38?{G%tkRN#aQOD$+VJEJD6x)NfGg=uA51C#Kdp$@Vp5lHO1C zH*7l+PxE<3>1>EzI0Fesk?L0UFAgcQmxfgV>n*65!W zCbjhj8Z$pxRSO25y97FmMr$hV*Wqt^wlS${4N?!Gx2H8H4#Rb&4Ek)X1*Ct(`$@)1 zW3&7QK{);X3a#p`Pd7K?lvpD{*95di;>Gt!M!FsomuFD_7^t2`(pvodqV6#A#u>J5 z=T`D9yRoZzrWkM7mW56Dg^jy!LBdhhtIFFbjf_-}g#Bkx)wtb(os4t{Bg|v=&bsto zCAc9r3O9bN@P3}x4fCVuEE&-V_J^5(_iTU}!~d~mGrut6DO@l;5!6_WKRqQ~Qvv0} zXA8PsIVvy%NG~(uLHX7|!xgvXN39e!Thi};KmG&KfHK`akxfnP!f&jkj4RFrMg_K3 zJ*CZF_*!ut4>)2ihi=-3O*}YL^`fi0NUJdD)RpaEo!A?RAHe?SBAoNCBX6lZU@2qW zn+9{WF2}pCApcN9{tn3?kMIs7ThGT6bmn;Ug;lJTeuC;H{^EO0PIIFvrvct$UTp{RRWfIwEuZ&umUdZLiX_Zt&p(!9+VqVW5zrDAF1pH|OuVEe zo)NcNzeE2jI?ufzrigkI$;U~0UsPDC@P#x8yAg7Tk(XpH?hip+9}cGseDGJ8E`L1C zf;7h&r2a&DPY5`&4TkK};iSKSum_a(R6TX9=|HfH+khYFoM-jk4JS_IVRg=c#pG2= zhe^V2$TMsP&E9qPg5!BeUxOf1BUsw95qP+!u$tY9B>cL=9P0GbR4zJE?Q6Z)U4}JR zSMs#I?$9qc3~xBCWoxeL@sUZpvAk&@d7|S0QS~|bY8kZhis<7v11o(}Nq?P(jFqQ> zum`B`d4?<(twx>`q|xBPY}#|t&RXsnSAdZ>P9otIINWq5pQnSwg`E5bd6r5I&A%l5 zDZNtml7|=sp*nRT=S8K$XNURh~&x-cWEAVbJI;fzuJK8tk3&28X7ewHRePJ^#=qM((% zBj{|I1?N_$;*`@)T=||!X-mbxIa?v)!2%o>6onm1%q7hk9CEFz>Ptghh*|#}HN-{u z#>$*@)!#yTUu=D^4aYtl2q#&O?i=vZIj$j{gOAxt47<4tf_99-mebRLbODeKWKF89 zf%*jo>V6XB;nbLLo55+-$He_C3oYD`vMZTPb$ z5Ip-RP+t^{)pii19VBTkXnXFu>I3qCCxA3C54+hM3)8+(&-8}I!7Y`}_Hc{LLq@x7+HqDy>9twLk#gXYiYb(}8#o(VFKUjom6xhuT*XEpk z!jvBia!Q5-K1Jz6F>|>K2;-$*t}Ab1%@0vR=|=K0f_O~(CgU|z+KfDeHhj}ge7v!M z{aEu&yZ^8Yjv4WkIQXuXY7wYTwPz|a@k3}ak`KgpmY-qH6-PPs zdr15al)n`1o)d?Er|lJ_`9;*UeB~2l*wzTr0?veUo0WzXR~vdl-Gh;!c)rZ-rBHd4 zvB^B3ECwdsibC?$!v9gE(%0y6VheN{wjW|&gjH*jncIDUFk9UG*Nkdu9uS789<9#o zel$FS(XSH4hJ6jV(vioS-V|4V?*{U{lyd-6JY-SY6I*{1xM21EYW+XsKoSta*SgYJB3~p4D@N)?3l6S9AXXIt-+fPViLel$1+cj-D zc?gV2v6T@ag-E!>4`Yq+(8m%Q$6&hUiHW9n!AB~i6HJ{ z4t{glN?Q)QFiYjNT!Ago8KQ|L6mNKjUVl!>r`FGs2I zRA*UK12}b5Q;Z2;1LQMssgZ%ab~r<$@VUG^OZhJ1pblu${)A9IcyPCNz}`3JNA4D3 z&Wkz9|8vSwaM{dxLS?5(9xl~>aFYK8K^YNHT|m;>r`5h^&3g+v^QRkExx&cSzkzU^ zk%s}{t54a>9Hp6r^7${??;|}N2{YnqD_uf7Py%{cC4w?A?S1Vo!lv!oCAx1l*ABPF z;r|X(Uf2g!E~~gDgw^LBfrs!(-K{|Up~;zj5DY%+a^hOFF>i-ALR_l-3Hg4AzZH(; zy@cX<@)NM6$r7Z@2UBmRV%s^XKv@U6%>2WY_a)6u-sK{Y232ccv>{d>JI(i|`($lI z#LDxO3BG6hFViy~ls!>ir?e)cc?IJocFZ840BEf_>1w7ts`Ag3*S;{NwaL?HC=12( zk=|l#N;A6V63}mWRb_Nq@&-tF4}=$NNtUO$`79nc`Wk63-rGWX;Bc<$vV3@jxM}MK zl))h35>R%+63(|^lq-`D2*%xWO!?a*tAxk2;}BglL-7TY-$BY`m|D-+*&QThJB+vu z2(xI{*>yl(6DRC!%!vzu=9p6^h3ATWIr)7-`n;MS-&942#>>5sd<_QA?nt?Kq4w9| z-7w_D23520eDO|HnXh|N1{(d-NW~9IYpcv~U6cV(eJaf@DHqp9cj_g)Ge>jc3tT#m zVRt@{@|rTF90Mo|Al-Nhj7pNK-&5rhDqr?;wZVVm>+?3VC-CJ>r=#)s=8#(Zn`VCF z2atBk6bwUl)Au*R)K4|}y4MG>(SpHJGh&pq2r%T~1{X9N8~jGo%@tznlU#V1UI^3M zjOO@c5;N&Vzmw~+lkK}*sA)B*5HBXyhqUiy;5U4xSXTcI7SNeoi(ah8IzbKP6rJt( z`M>7e!`TenI_(!p7wezSu|Hgtr$~?Zz_@Z`(S19g!RssggdU$<~EfOpKa``gz zHXQi&5el#PVcx?*BI9ikZarCxuK$sUG8~9!p8bN-g8O1}r|pU*SZ`$znD#Z6Tkp4`XR^KJs2awyA3YQPYjImyxZhOJ zIMAZsW;UkhZ}E3zZNB(d8(w?L9!Px({Ap|&bhU55m%QG@-|imBhjyPNZG7q3$Ke|O zWM^kSWXNBn|Iv3eCCGui?G}Ya02yJ*(nj}^h{%=`6p15=s-fu$u z&vK}2_TOuqWYkJqxw!<|2lbNc>Xx&CLCa_Z2mj$5=L52 z1&7Jk@PAo9*_ulccr#&}7+A-Y#~HsA56{kqS6hz(eV(d4xxgk!{+wup?sLQVyk-xX z@Ax@z zuYhOAK0xC+17%V6HlRPay(Av5CD};oHSOn&5bY@1|DaRIXK%fs`s~2BnS6J}Lp-~? ziCp?98lGfaf_kIUfz}xRH^xA^cy7j}%|F2Rgh0tV?1moBZF$G$`l5A26-;z{D*V_< zP5847=r#Hg6d!Ry^`}?U#ysv(GtHlnP1wiIPEwzvZ^H+yn2WxWN0+IP5&& z0rRso6tw2p#UdR2n>RsP3&{NWNl^bY%QYWR^+3HLTd@5AI=|R23;wL_#c!S(1*2gC z&eL^~FV<&3)$ArR#)9^)%rAhUjvDHxS2(yelaDuN;psDdc*(w|(ye0(--UJLn$OAHY3DCYrv!xA4`xRnWbAA8trKeu+*2 z_UL^N?7GEM4K9SdLuM>hM8aaLVSL*6I38ki91WJ8VU|PF2#fZEchg7~Fmwh)mBy*Q zr!{y1FT==-^**L*6se!659xQId48H!^InPxwNhn`aQfb`^^86TD<-WF8jn5jbVDZQ zznQ8qPotAGR;>Z2x`nx46CtctA-J?#E#EGv$JREXGiV;}K!r1EjcScB<+T1VqP{(0 z_i)*?a~ycGeMmJ556@PJE@Nji#TNlXKcG>lhR<$M3UQCp@kVq_=+mMV-E*;3ejBI5 zBX8H_QKhD`XQNGU?(ssYu&aZA9@}5qm7m#SsOCxz(hQ*eU_L_6`UX7r{WN|nawUv! ze1b5f3g?(;q*0)*!YoN+!c9MV%TCt1vcvoz%vIA^Cf{i(*S&V;zPyMP-+ckoYwktD zWm-!=PS=bhj1$3m#&hp_EAJ~DV(oh6D=uWGkDtOJ?R#*q(W6;|=_gP(;jxx5hxX=~ z0w*85#1_$i1oetctv#QxvkEPj4ke6RA@r7V{JF6Yn)NmU#fLOb9A%pB-L6|3&|i!& zzG1$my^{gZ8jG>s&!ImbD2F+hh*-VrykkFqJil!dD@#0p$J5p_;$|rPx?JHBwC&%K z-(OZks_QaH`BJ1XX-VDQl0E|-?(fRyl-Iz+BDFBsGZ?7H6)tG94;paYS^F@g^AGml zr&PsJK>V$tH4)oiNSLITLgQ~(q&|oGk<(hrhf8$ilG>f|)~@z=YF7h=mAux7IMpAl ze4`sEoef6;7PN&V(t3lIoIu zU9=OT3~$0plR@%+lnGYOTnGEhs9+BqEbA?ruKUNFyc2MlaWAC$ zV7|b&ZCAoVXqTEDHNON zC-a>tIh#1W!L+my!OHYHrqj@{)J@)oFi{APkRkk%2&|9i}6-9RtWpD7+X{c0R@1h*3 z=jK<>z1Xwf`)JW#+;A%a9bF^v!-_phqrvjuZa`}){M-v@9yg%n&?fTmsSN1UtPU#N z_8niT-7g!!%$8-e2i*W!D|j1_CW1a)2TQ^PMm<(kc6t_%sMsfbS|x$c;1R^T7d7g= z`q-u5?qVz1ZPX_q{V8hf{jU54j5W`KQ<=@+ddX;5_jISyG^jj)Tl@Ob-EkG`(7JXt z*OB()#CP~-+f#JxcZjsq3=C~IM`0h^dm@q1&jNiuqk4lOMs@L0s-4{RP|Jw-`LMcH zXg#bIKXt0BSbNh1Qhq%`kAdfDuY*A7?QtBeqozS8UoH4HUW+4rUJ9i}=(7d&11PSg z@ruQt)8Kr1W95fH@r{>jGF?|~?l=6SXzymuNl#(I*NcL*veK_i=_^$?qyxF?lSM{d zfV==aJv>yl9Ni7?#wEdkUT>5Z5cR|FL9_oREAD_#sn-kby4AzHTLEH}!4`4i;7%ak zL^bF4a0JZ!y%i6>xC_q@-a+zI{Pz?K{xWkghy#uInS4*awD}C?Fu7l~p0S(ik1rDP z&|=m}O;4{ASg|S_nojbgdcF(FuXuSma>7@%2)ooVxhx0B18En0+abd06yt+!&YbWa zn+-WE_hpAO@<`ZZ`4mAuPh2l~qXyp-ngH3=V1RP&bd7#-WxBVM4hV5?>Wae=~Zg}dVP%ipk2?q=S|%TkX*cAh!AaOEkSI)6pj$1Y=pA>#Zd zx=(3^gY(7 zCqMqt5N9y*4nSTGXkIZ(YpLoP>GP4a7EG<1gLBFwxO1Q<=+8Ss+QLjWnAZ!b-ob>< zxW2q@722+NE3CTKmZU3KWEFzx=GpAWQYSXZVw)h|*H9gaSqAMfmhy4JY%TB5LMpz$ zMH$cTiV>JwQNJD^{S0`^v2yo5!HlOT?dP^m|nXRu?8ed@`X%pt& zyA#ebILt@`(zp{~{?ByM6KhcQ5n+ZbUA+a_lzB*)C6oU*hOW^Hlx8OV!~9rhMi?Tw zcRP9GW+>i|T1%R6DGaO=2fwFmhCkcJF;$;4TaJ>7gGlRNQn$6n{Au zt{wH~-)C(pqHLfVO9|(Zath-ARG{^Roa6iPzwfP~*^u^Xt$=z@^H=Q=o67Q%@4pi;XOm%UHH;8vFawb{aCP7%!Vn1|G((0RacPW?_=?3Sug zmQ>jjsD1@;CG;9Joid(5knpva(j{88CPQf7AMyi8`V0v-k=7U#j*aIHGm64*YRsF)DvV+6xBRIN*?u{c+R1G~8UC4+=l-##srMA z<4mQE#PzhjVt*;=$ceF#anXr)Yn`s0_GcR-T;#MSa4+Q$>uO`nLk+#C%KdJls! z`j(8ixLV^-c0#;0fWCeY$rEWvSHQIm@oIjm=Zm}$=I&dC$3Ha* z=D*nF&-#4#gvA6AM~Pg=qP!nkM+^%7Irj~&-%^jcLX8r#6_l(F4c{RA(!`{01abjGzH z9r@`!X@>!jQz)h5#GpzW|z!hP;{tT^xr=yeL0 z6hBw%#pltfLV5X*@l29`6AD|8-0fN2dz3HAds}t^;X9}Aq0Zd>Vm{r=NZFcHI#cCt zb%GKx_cGm+w!#M}E7KAOs9s79hXr&_%8GmX;_ni78t+)z(R)ZAkhwq^7G)*DjIusBcm0_tFPz56^Qjus zC=IK)cUFTlBA`tsX5Kie`Vc-(sDXL;wxlz*0_AnpU%Q^>$gdqe$0&agl)F&2@K%tX zN8$;1xpbq_Xev9vQ7yY7@g}H@=A--zk5A6zkCq#W+s8}6Xht?+@DQZDlyubsNciAe zr235XtvBfxPTYZ!UtM9_$Md-At%K6aK)O!#0%o0Tid65^@6BjEJOuSR<;?3rc}T_k zOLO+F3DrW16*b2N>?i!-*(lO@|u$c~RAU(9bbZP!vj9)unbaAxg2_F6U3Y~%QBWEWZ z)^@@`KAq0d8X@a?c9ZJ8MqikX(WmaB8=d12_uEhodUXUtT^xAR{`qj?!%QejSqiPk zJrdUz&y({;#PQ`bqOj|XHCUOkPYet8NA`^c`Z_x@ zKS49-l`DGNT-L76UW8y@C9aI91sg}K1@)QwzuKd(iJ=Tx-$nf_r|aX-GvgrTQb*15 z8T!&G-V!Fq$6+tSTjKS_0KW8b5NSp%Mt+0k`c)a?EPn#2-D$sh{shN5ZsgXt zf_S&Q8W388_L!qRPzK~Sf;;_p@dm#wVeGI`(qK64Kl6z}=&ft;YwmD3x~v}m;Ib8+ zJx>Z}@;q&HGvrvu`f|+UWbM@&9c0H5325Qt19~M^kUB$GZjRf{%Ll~B83Qb2WIH3c zn7fWSmiOh4{7%4TIukTMO zEL!uZy__6xDmT1ptJa&ZrZbA}v?#*7JBcEr-XeKI?1IdppRoC`=j_ji$)u?m6t+Md zWA=(29y?V=pBf6c%T7Vcu%XcD@GAtnZBTpeT%6-{4|9V%;;8=FFn#JxQSW;dI-~_@ z*A6hl@*}i=M39eM!$ag_-9)IrJeukK7%#UaH4%N6N5HM(0`Pm?NWPxF8E<%8z)puJ zpss_XItq7MkL2z_=TPGj1T3RSvn3pZaJ5 z_Fsq3#coLRjP6d4&~?BztP@}_qdg08QovS5>nDs}H{laP3$*lJIMK2Of9rG!+U~yv zMwYRfK(95J@#rA#xcL$aOIBg3`5_$mZ#MsXyR|$>XG*pT1^z5vPhOa_5M!tIeBICCkz`gF{|TPU{4hdpxB5 zo?f#i9b2=s==E^^VsF0oCt|jRx%QazQ8w_h1E=?b#|t1H!+d4 z212cIgMVR~>_(w5cZV0uemM?J+<&l^H|NMluZ?(ly)TGqJ$a*b<3-NLJJ>C{p)A#E(4+R zKUqli0j0wmt2IK8d4o8nT@XY4-J#PmTRF}*6>0qV(0o7MGaDhi=BJ~{?UQKWd>$V7 zZq$5=v*F*88gTunYdGD@fQQAklo8QpvMPN8`vmps`D5LN@_&0j!;%(VU>I6Rnj5T*3pMQ0G=G>8Q=rIn@J2b;HiSDv#)@s!kuyk4> z+iY{1&Nv>WcmroW(UHwcET!tJ#A3kbOZQ;*ymPRA&N8f5xs7nEwjdnB%k#Q(H7^q_ zBlv;bc=qPyVB%*7xiBLadW05vS2?=^y`Cp+8Gt*|w}Ij(EA3^iLD6fWc#_@O1ztOi z*~_VC@$Qd6j7`+A>0U9QeO$kqx2;~$bMBY4+NCBddHU;kF(siwOW2DOg%hMWn&46s zJxMqxBBD0{tvB&ZIu7db3a*S;g)6Q^!}h3?-iyn6aauRIB0UA`4@hBu-N#^s$63Oo z*Fe3A7kr#)O*bR0nLIstsi3v=o|XDebeQfaJudGT4R@TZ?q%Xju}e2asJYX=m6rey9@9Pi0q(4vNt|t>_z*AcXvh05ufYw5=R$*j-!=OaJL2uKE-ZdtFKx`yLs&bn zwwGSgA@sCJ6ZHq!$qV(nLXLl1cs9Nf^+SxbvFS(Wv`v8D!);*EgjiVKb~a`=`U*5h zFf6W(rt;xjc)Y#mHy!Db?(%{TE83V?@kS7o;_2g+H!i zzTJAfddp8L+*BIurk$~vYe8os|Dt_S_YFsbXggHctaMT6*b4mCq5x`4`GGx8m9a89 zE7HvefUuR>Wq{%tP`qZ`ZVL_`Q6HA{cVc%hOyKBi$+j*tmxo=_NFz*A7{mg?YVp?X z%yH1X>-eEYHXi(4%S-%9g-iclpyd-oIqc*&_PN52Ydr32iAUhlZ#~&N`yZW2I0_Pi zwF-NY`bOm4X$ba@)3N-~K6&u>DnaWb+CTD^wt5@T$nQ4({ADFc>k8@}Nj1djwXot! zZLIfw2_xM>9R7@{xm9C~aOQH8zYDwOzh7+a-x`;8x&rfU&I0KYNw~x4Ga+Q$FesSv zlK9q$|FhkJq(x!Q<(<%I-E;PMTyIo8U7vp@Og^doBT|rfp<0)A_(|uQc=QDQsD4{LY^-TgkrINJS_afc?g-HAXonI^mg?FlE3A52Q_zWWtq5P7Tu#Gh? z_7@B7Hj_8{#%xYj!C7NVZZ#rFxH`0Fbx(~4(&CJ`kWU%EM_jtRRbe*#yyU`jX^+gL zjwx(^;?ZiH7%`y^?7H1Skhfw_13zN1?Ovw5mPW6x{PeM(>MeXEcA$CWYoWYIsdHYn z2MB3bhhOkHh~_`fX(NhCVX|$upxQpF)`t8Vkp2YvH&3`@P5QJuZ|Qu7Rc!uGlQbb( z`(f-0i2pYgBZ@{M@sw;nP!~xHqtbPxSI~HUFE#(99V_6-z9+EfQzzV5!%N{lkmlnb zYP7=>uh+@c87rXA={7KXl~Z;998XU~nrnP#Q3t6Xkj4Xw-)(HJ;llsUqrKS&oN_k` zI{YjVF3rb_h88=4ya^+`5wmj}@ES&4=yx#9Xy5TA8XBK`o%$H28unNG#c55s(o}g5 z7ovJC;iR;<_=r(2%K@v}N$LSnP+Ua#P5X`=p1~9Qj+8a&S#Qzyt?~|7va}V|Y9!iE zI|= zXDBRESU9M2t!mG&^nH5hVe(&nkopr>_d5+uvkX!BJmO73xWxQ*9U8YYam$wtg?S`_@Hb8v~Cv~kc`EdescN(o~vRe1ec;qIJhaLpNG%#=uko8Uy|XTy z|5-rz+zmIb8w$!FO}bqu4r`CmwNGZJC;t)Zz13R3_Q_We8JSrxDasi#1B=+%{Vp=eqM z=vID-y_?rX=_SIk9MWRRIOq3THq@pkkhTC*!v|W@-kMZ%7x89QQ#t;MQMHaBzb&#Z zUIm-D_N3{u1g)!}tVh)_hMOP3FH^<Z+&{>7!t+(y>ry$p<(Wd510V7zp|Uqaf2^ zH;^8n*PMeV`@7KD2X}GKWqWuMuuc$%BF!uQF6sw&r)8_mKoAyDKGPY4GCU<^GVJZ; z2AnbouJnqUAB7f=pX0~IFC&fOvo_#7s$6ssY zDT9EU6-~(_*X6NZQRKh=GRpo)8?C_@!$9uSzk^iVNm!t~B&MzXgi2=+pFmo9SCvU& zq-YK22CTu6Wmkk_v^BFen2UQeEjiT+(%PurVJIo(+$X{s;m@B*Svn6qC3L>HsgybTEh=VJ<5qj^R~A6TGFN(m48ys zNqK=WpC}HH4%PtiDN??{{6seF(zPRhHUyM6+u^KbN)zrHcLQp4bW*;oS_4q71L&Ke z)=nFHx&=SJcOlxmsVU67Hh_$#{a_Q`7Ug>M5#%p{GEqTUv+ScUArzAA%5m77nwt1l0{e27O6KSbg@PP!KRZLK5jmgq_1S4o)^ z;Ycjy0e#>)#^B)*hLZBa>bj*|oh$#Ku3^&tKpbjw0V%Hp;x6=lFY))UVu<|Tc1&Nn z0t?p;XVZq7<1P0U;%d7M(A~KZ60el9Hi;+1*y46l_{)J~ zn#ww^DPa1@jc=Q}9!5^l<)c#mgPU~U?aR-{M4N{VWiR*tF$t!8p}i>{I@1^Zqc(DS zUmRE7jct4O7*a-E6;`{Cv1I*ktWL&9ly|>jbj%!X^Kb~5ZYyOo*JpuEm#%!)t=odG zkKgNg4-MzuLF>Yk7_;ax#BUrZCq|y9-`Q-%fTq9T)$;?`@1zaf_8bgVW83qJdb^ot zz5^C!f5MbeedVM37c{*FAA{IKyTzehT1}lnIoPm72d@xx$}_P^8B|4 zY*vcPduIj|JnkyKH=IoSe07#@;)`LM<5BTc z>_zAL?d8cotz`bfp|B~m9jyHt2&XR(l1n@{!kwp9yxE~0&`z&0->tP~FRr%co#S$O zABU;3bDW!8{iGM_owx!<_q*_<%1SoFzXl(jeE?h9UT1k*w?gxVW2Hme=Q!i}E9_tU z3^pHe6kVr}0{s~ig>{`}u&>7~Ugtvzz6{Dj`V8%<=uB{!rQv@cra}I~yV#~s7Xq&x zgnK=Q;U>Cg$GWgH_tnqAe-oQ=?F)D5-97;ieBXsVOv1dczVnv!dfB3NJ&o<(X?(&S zV@7j;UVArT`H6<|+kX*kj-ie8J6fdPmk%xKL*vaB9}5HURB4D@HE|?3UDC&k<^Gz1 zum;}GO@o}M26S&@KUt9c9`iDn@t$Ei+SM&~;imJ`WQVxV-bb&w%H5_z<;wm+vd_UJ zTz;#8EJ`(!e-qCM%Xq=2T;2en(^<;ACqU~aXVzMZcmMk*9Pc>F?`aKSM}BLJUSI;W zerRQ!1;KeK=+U+XMvQ%jQ!8i>w~nbe`JWf3wT4>k7;ohg!56xPBFzD;Yi)r8Hu&LD zi`8=crV)@c;T9fgoXMX(V)E5yikHyre%U=eyg+J{}@%Ax2+HYhYw_9K! z>xxv3M_`edc|#W+xjDD}u!_bVi*-d~UUk-1E~xhysP6FF*Bgwk52#K>P_(0%`}&^v z)%G2z`TTaB^NVeoVvV7uxV@nR@BU^iG@46$`Gics+n%?f;p+s9c)b;xu31Cbo4dB4 zM|+-nS}>i}1?YadBd2;2ZNvVuC##-<)!Bj2;pTZ%fefZBV;o zl-zP@wdx!B!n}s;-GhAjur%1XcZi_Rk*Y3d|mYijO|K>5LW?jIq zu_G@(kqCI(9uJ1^5o;{XV2b?}d}x>kSH^jXe-qPj>Vd1c;cOB=UGWC>{#yX9>zZJ@ zCkBw!Yb^WN#FJap(&8njFF<|9LnC9wf~0l0^>|O9YlPoCS10#o~Wec;ekcDS9=N`8unFQ_LanrL~Sh z@>f$?yTnJ_PWa_}%k{I0QLEQ3x99?%@&0QE5NfDI>Em35T3e`Q&$ZtX54aOIy}1A zo8QmusrU%o>Q<)x;*8+jG1E}6b>7%E(-*Oo^4qS$t zt*Y}z-kzE!g(q;|8WS`*@{G-Y8_F95rhhW^*s}{JiWhB+mW({!~^XQ}p2m98Q zzUzMg@uIB$!kAYdHBp`p%Er>)v7GH5Ud1z`roI-WUs~c;x$NmFY(bm4(tKl=m;3PT zs;-jw44`L2p)}1k@)f+?^O{YlRSTLp{s$Azr$gA3$85%WEgEcZsOkdy8byj_PhHuG zYI-o{w1;>&ZZuj?s>$CRc_(yyHWSV=H1v}bM)8|o4f)^vT2zBgk+e?R;9>yGEKkN2 zy}zBZd=RYl`I9J@)Z2i1{!y&@il*c(s!NA+32>G&<@n5hBD}X7q(Q#uezXrR9exrn z_ql-cZjYe4tr!{M(EvelwPNcG3W`TuaOODC~x={5J3b#oMF0O4L?iIG>) zrqXvS-Zl>b!+QFH@Qwa-|NrV&DRfPaa=6D++5(~-zK$5mHQF6ObUToB(*yplig*`)s1exDN%hSmIu3fkL%v`)5Zat??$*!|24z?uZ( zK8=>G$<$$e{=8yJ=ekDsl5`!KDD4C0ltK4Q|INTB|BX|62&I$ia@~G|fwVc_+9U7h4MRr zfoU4zU0#@7UrzRjQCzR;jq1ch7+&o_+;E6^)ceasiI1ZXtK(kAdL-tntUZ zb9j8jIfVs2ae$M;m{>h}1t?GYYeEM07;DJC_fJ!Gjl|{IgM8aV#=^Hq{#E7--UhSx zb9g0}qmiu0BAu4~N(*s)QwLmr*jSP#!==;bg!QAlYP>u@X)}6^+^gn@6*^63kGD6L z$9G%835UJeb~@oeT7%!W=*rgJL-5jzdziiY3Ml=h-{IXpHKFh18$kY^mxaZ{rno5@ zss)%awOJMZuFt35J*y=NFKW!F`~m3=?K>kBuB7Gr=}7()S(6&P=A1WFZ`OFFvKId& zwRmrqmCU*FDx1cF#3y3E<#!EfoaWm(I(H?cHYZPj2WxhgXC7{cqbY~L;HNGx2pNF$ zGP^1t%G4U%-;@sI)p&=QwYm2~2fUJ=A=F&YY2JlutcE0wMbb%mc&st+RXYK13{A$) zHubrxt680gaMCwc;gXGECAHCbdMBHo4&aWLf* zNjyc_;0`p^&qvBL{7;85taj7~B#(YJ6T6vVmZH}|6wd(L6R$*vy zcMAP^Bew9JILj1O4a2m>t~Blrnveh4@yJJek@ANoVBVjqdLW;{ZsnP?q$X>j7wz@X ztwnFRyuq5c?yifGJA3j|GcJ-RO-8~Iz_Y*5;Kxt&$<9;S27e#f38hzU%dEJ{ShSYX zyZv>b`6Z5PFXw-I52W4dy4kjNH6hl94p9=ru6Wc{Z+-C$+S$c+vD9+8Xae@>?n+z-=G8C;v)Q zPI*S;46e^rP9!{$hhE7Z8PI+U&(?@um%IdV3EcdL;>{-yInb=8(mGt?)I^e}Le0)V z)DJAunr~rvVMGl{*QWFUo0!*?dDiu$T5qS>PZ;?#pe*Emq;_+v$qj<=fQ7pTbE+2@ zy(|*4eO>VL%tBl|=?Y62c0`R$^LN++uDhu@k|sb6T}Q6gJ11$NpseJsIATN0Hqp5E zahP*vD5D&!>O{UfHxp>xwaSC^Dt?X*4W1)qG)OeM2c)@3>m^lYq8y`*ZMzZ($D}I> z#DT7={^85#p=f@48~E(VB8I9^^93uKPUKUrG=>BD`%!akCDvQoyUK^x>6(6u^06q) z-UezO$+M$}Sx;z{6{Icb`W2K9S-PvP2pSY79S44QCk5F(^ktO*P zI`1yNo!p)khVgYnkhoLx-s&Qnotwxg!)ea$0N(!jI-EK*p56A@$q0`SQ+q1Ae!L0# z=G{QkSBdD?YG0L)Q67mfOM4wv195aCo4)J1c%J%|k!CX6rRxOwWc5=^+zI4ggqO)X zRkuRrhVh$b!gS#$DUWM11{VtD=_vmy{3>4uc^(M@uvmC>3IocrJmZv9+QgzSFeI-E zv^L65Yl|{NA;*wJ<(==0D>8tLS+fv#3SnS?Z8Qq_F5=rR>3kHECY4zA>NC3#7u%c5}ja>D;*mD8x7 za7mEg1BW&Jk^Hf&Y}y?9htt_tWAs_mmRo@IT-CFnK8nhgK=sY^+t7DXyBnhGuR&OU zl^b@u6;8Y#q8(QmjFf$$$DOfS@&a)GW*Cz8!K+>=*dmy7y)7%Mc#yKAQ2vdFp9AuC za?Y8XR3pn+DeYfF^T&)M`cb|$6qKicYKN(HB`?7~(%y2yD~`rJn>~EHo@(0`=?{-R zk?K09PoP-mebwFYYKE$PbkTHX3oiB`PyCck9FR%adCU4XUM{|!JEWx=gS(?;UAk6c7jVY!jSZcQU6D*C`ihp3?PU*v*@!vY}!8w@4nat@!jYw34;cb z`Xg+v-XG@GaTOBhLY+xHd6%48eDCMYK+jdC;9WmUIA+w4U)#C?37fKOUjrz5;}8D* zi}2XBu0nZH(n$DIw>OU6{~H_CpXY92#jAP{R7X(b%s@_K6{=69yqn5_JuEwEI|e*w zRKr5`oN5{Dq&~qtsF{hvZ6e}@Wew%g$-{-x>Mw4sxWcT9=A=9ICfLxU>*3keEHz)M z-ays9@(B8gt2N3$=i9ze>%_VY(htl$QF(c4EW)ma5;Li>?$U)IzsK-1c@x9QyUC0`FUxK(_w~+dMxWUa)__ea3 z++Lf!i=G_V=bunnW5;y^PCZ4OE?(frIxei!wk;Y}3u!Qr@`D4m{@X_L`+X*BTw^Ru z{m@Bq3+d$w(d^h(Ej^!V-~#3pHzQ81t}sR3wQE&YI@_%p?yg}^c_tJ*ZhVAQXG~RZ z0w}w%o!)gYs);A3-T{_8sH6NT-l9G5sP}^R=gi?<&-y~Y(jlsDao!kRFl$+Woe~x! zX{l&G@}csvO?ca7ci%bIU*&aiH;%YgbBpg+Hc=9726 z(sAXbMS5GrZnL8?U|d7l|5+Cv{cf;G=&)Pl{k*0!9x4qbP9naX%r4~J#?na}(M~$U ztCs=g;_G`?-F z%YWI7qMqy8SlxA`7C@<}0f#*H&szo8R`TURV*wRJ5P9MZa z{^yKGk~@hm1AMvphy6mFb;swnQ<$HN3B1(Vfjg(vlTq36Ftt@zoY`-voYkZZ2d3AQ zvqIhgjQjxwr}o48`WM;nKjGMqzQg!AV<`4IN%tJ~4g_C16SZ+;2bupofPZ=a23G{r zUM6(To=xlSZk`u=p!wjJ5PR|^){jrYt!4l5&=D;mJUt&X{c518Ss>ia`7L6OoAHW~ zEAiaE-P%^RuduOiXC4^w1J925q)|WJ=C|QScConph!@N+naz^kc=NqEE)Zk+0H3X? zD~neSkUOq~z=jS6{BXf8ZN{lLV(N$rylv70GAq90twq^rHqeacthpmJHoL^f&U-}{ z)5~z^%tgQO47_>-GWu<42`I3P9wx)hnCUGQ!c>p$b0kVbq#_!7D5I^EYJ%5j9% zXxYp1z6je45cg+2PThDCU%xX01U!Q!sr@WrXo0|g8wVLsP4^Lv|Uqjscelq@DkcmLw z38_4yEAvwX>SnBkMXL??X!!-+&D@WzeU`|>1r1@b%?sAcawN^sB5?ZNoyT4532__! zWx(;rf?Wyb*4f2uaG1Tg(>O`JADjCa@ymcQGP*m;&KJ&z+SX4*HQ^{t+67^+A-3## z$0)EmwO1BSTCcfN^bDHbB3Zfy8k}{R+|5ub=pP=mLJ56$@7Wck#%|5EkLE zSerMu1s{}o3wK=6s0*r3c|{1NTHJ0vOuFNRFz#mDONGuF>=lf^oqdcnQ+ ztyrCjx|}c}3;geASq)F3HJpWYHlBdnCg*WOhxK4TqoFK*dR{d7mV?zAe_&IrpMpWp zuCS)-N$B?DB6e%~4e5IMu{$k!3U9}^44oko>S?igrycBT(;r}+Jwp=Su=OV=-aj{t z{djgsI6U#-GmLZaQo9hXT1Toe7Vm0{<=GBEb-`^li8%GsPwZ!FElxHr!DaW_iSodw zNO+*Vu7c2iP(yfTy;C&FSwZz~fR2tG*`ak6%vRHsyZT+l;j4$>j*%-sJLm=Nv+fL? z%NC$h-d3ui26Qi%4rX~|Lgc}&;<@b`q`AlBQLDLnpCr4PuyN)-k)1(fI(SC?o5v3O zhh6cF8&)Cl0IYwvoy`nt zfV%JZh&>rYphFW!IMD7oYRw12w|onx#x~;AWI3sJJDFzfdaP==rQC z^`A{R;SBnZ`puU2t;L8F`RAerkl(A9FnEy*n!qtyRYQ9+3dNQae4ot z0Lrryh=cvO;*Y@&hmq=@cZ&?fUngH-w@4ix@@g>XIj$1WwvQZj_#&+1lqTqw9kII-2oDCfzu3Iz|4veBQAKsGfKGNPl_Mtuai$+ZuW#IZ?e8 z1Mw)L;~MDvyC!6LIEnLpYVwYQt(e~W|M<`2QJT`1Vc0f$KSqSnJ>>mvLg24pc;WM1 z)sZktyDSP`4wP@V2O!NIs5%=w@QJV(e?g9aZ!M#|4{NO}+VLsY<0N4iMbiw?FK9fC zX(K4E`nhT_U*UB}Th{v;5*Nu2WeY`vj;2z__b(&t$rFw2Fep_I*#ox2p3ufX_`&w+10-pY z$k=ckidW{5=G_FPXMQ_sftOD^xxQa*N%JXUEJp!xh>U*LixbxIr{zQ9-;vn&&^f## zkFd!vt};Ex-%#lN1xxF?@$$fNP%bK}F#oq6lXhOW`Q9-cj|ZB=`hMZ~=XP@({=B#5 zPG~VZIPfW?Ta6*^`G=RcU%`MOQ(3biZ?aDxaFb!$Iee563c|bQ;YWL3bI>#B?Ya*R zIkZ)^%QZF)W&4nQaPRsSRM&X+(@H!$F$|5~Y+(7TYs(I&(}6ex!|DFe$Ghsu`GY56 z;IA6cDLhT_4K}u4KwNPZr{6s!8qRfO{)2YO>bm>HxBML}N!v``DH<>DT{q@2LmJ_Q zw7&fD`5e*n_;%8b5TIJcj)V8$>}wmu`@uH6^=1Jbk8Od(SNbxwzcFuey&jN8!lPdy zw0A|I(ii51pV+YEj+*)FQedV1Cv5pbN01g~Zz*Xi7WL{0s?M@a1EiY28jrFx?;GC3 z4SDs^a`ig=)dKi4&pv!-+gx_kEf)M6+)~^lsaBv}KiW$wXaXay2IUd1exNnXeD@fYri_-0!8iCNlBUsEb$E{z)p#uz zV~x^Kv&?WtT*utzAFkqfnirrk@kLhKQQ@8*M?I2*NOg#W4`{ij4!p}V1&<{XI-G9L zX*_tT$$arCCLV3?rD;M(q$2T$W>1j?P>qYZU0l$+CD+6ZiPjE@(v>xv9uW$o$)DY@ zu6rDirUPlE_D<*wP9uRN)0UwIXzIT4wcF0-t-=7bd{sYZZ#AHuE8vw(b!Jl`i2 zIt?viq|bax);nmrq7Q8AY$dHXxpC5OrMLW>(?*dPunn)oAJowNs-LXu`&VGmER)Wb zxQJ7-rb(Z@QF!9*0~kMGn<(%fsn!W}&HM8SAB%DB@=J`ak;YAXMV{xV{GP=*Ew=A} zPcwOy9w&^jk#|yL*PNqD!-0ISY*1)Jnnm|Y`ftWuvl=+L=UH@e_zc8FO!>jW$~7AA z;d7PW63sp~mgL!x)|VQ8_FBmAH6Kk$bM8XeS8Cs;t&R7p- z7K8XScYCB7fESyk8}Xg??7z)OUIvM8VA0C~xP5*aZuPsSCI5pjqhpY?PI%cBW1aFt zKwb*_X%pSYdSpQI*6yQBV6ZBbk>>!b~5hdlfFz8*WnL zgNV}^EEy}|dy)$an;ao7hP=hg(;LXUIjgF4ihR39Wezpwc>|;HbJ0##E%=2X9b;kI z5?G%WfF~Nou-e1o1!)evT8yYXUw8c!LHGmmC*pN_0V{ib8-EQe(JCENo|x8&_RKiQ zdalY>^$Hvgs>cZObkhA`4pNSybF8hzyn*ICdPN}l*Cc#!bTbrAGT>*1B_e4XIR2^* zW3Cz?;T=emfjquO@l-bLLzY>wOxy|HD~b|3@I&hs;Nh`dVQ9fl_VG8}SKenJCLZd6 zAN{Vw-uJI@Ze|1!H^Y=+4ubpxC+sS|1?3m}zFG_F`YBtGZVjNh*@?uvQ?G5 zJE!80Uu2>3$F;w-cvf9?>Za^4d4hgNX ze>*R^KCK$xGvE=kz1Nn920X0d_3Jw$xbj?geY2|k6^&hVwNHh&VYA_k^>|MB(~x&T z^}2*1l`q+0=gr{l7eqd8D;wQ5fIruks_d#TME4R@Yv(X~%+>W`L|LlhTH=}IyjtT~5NWep5GRNnOB24L zXdFK<)kNN)J$c5P=Lz+GdvlJ#@Dn>!1_z}z%7YS)Fz4hR&DSxf#af*xG#+&fJRR*!%Db%5x3ryCB8-m(_;>({36H~VC3st)PLBleLI@5V8i1~;WlT@2%xN_dNnZlML(oI z0K7e&DTr&>(kuP(j@MnbF)Z;l}f47awMeJ`q0r}6XamrGvhT+}}EAm^qwAY?7+-XeTp(9?Pzf0e~1>o62 zgH(?=<4s*u*+k`qFrRN?hH)23YYB>Lj`Zk)x8Isj_B7;_<=7oBZ?<=BDeii3ochsD zO!30uW-FoQpz-kPzO!uF;3g2>g;{28sd7Gfpe*7V=Gxg33@kTm`u6D$hoVBkrg9Zf z&60+6;U72rVVxR|V3f_}H1~Km>-2Xz_t#V|nWPUE;a}J$3w>FqRSxXSH%IEV38g=F z!Isn;kJhT3rLgIDHVflgF!(yggHta?qx55YT~~4PYPiA#F0wMETHJ|>Tdp2^2;{k_ zH=xT^hM{`lk0X9E>d(QWA@7I>N6|IT1*%KJs-K)`7pHne(s-1MhP*W+Er)+^?%~XL zv>)2bJj&!v`0>{fRXO`0-Ak+bki=&|JV_pSKK}kw8!2lluOdq3x09-mNY|{ToG8@m z`MJ>jo5SoG@dZ#lfJO6vLg%W!>aDRg=AmqK(+qy5g_(*AkvYpQzUq$fi0yZ_Nm#8U$V^&z01u?No>K2P<^#E`K$8uF!ZO1F_D zzowyVqg7lNJ8ZVfL8_-F9ZO4rvQ$-nZJxzo@yM|=kkswR6`Pu07_Y9bjahig?|LCq7@Bv4N(TX{Unlzj9Tjpl~UOF>+MK5jAe znUlnBV?gzJpl`iwh%5g>--q0atg<5~-v^{0ochJQ;FPVn^wk3{q~%h-c%mQ-viuZA z-g%Bjl!M5=C1Ine7556OHInD z=NKes|9&$AP=UzDR$2{Q@u8Y8?Y$(fE80Vqv~6z zj7W3J=dWu4g_VDS`iemHgVe)-c8lp8lZwe$cX@9h8aAyj+s4+AJ>1Os@lo6P zk;+Y)-xVS5&5vxynzIj!3lXvCJ$0_U86O1itxuqx`9|4uSXbVld=J_OS;}D7YP{78 zbKY{qLAd4A9l9^BWIHB?;9ab}W%&WLU2xUCdC`7+Ss$?dpdZC zL#~Ut?}tJ(bxKof2P@+S$g)6pk-9M)PYsQLiiy6wO!PpyW_OxvuuY*oJNJYFjSFqO z?KEaTm%!mc{XzeM8zk(tl73>7r1xO62h;_FQ#J9vel0rhMOTdtk_+cc|FRXZ)?_!g zN?(Twep_JiVFOIjv4fMFi)k;k=IAzigG}~ar+rEI-gX`N|z}Q(Inf{bSn%Tcl zD}w3gg9bHHICf*DJh^!W zj?RsS@HI)aZq4ys=5L`J5doFk^yRXS)c0O+RWz$T1o}K$Dx7;v#S!<~5uT!u{w{FL@BOXq+Ux$#28^;j)ww zRcqMzQ4l0`&qN!$Vn*`}F@wADcbTj3SG!5*YITSO+O_19kL!WgP800-{-eL#^RKEEk?M7Ex!X` zSAmHe2S#9NHU3XV5Cxq8j11x~EZy2A=na(nyWDo(n#O8~!V40Mh!KUV8#Ro&_i zzXNASf5O)%I-qFN6qamkg*PpBiX&|;Wj}u{;inF9*eFKVfW!+JdHNjgnDPL&&!`La zB0oXm$zI&^=Uq5nG_(pIKTZe9c3-l@B8N}dw^dC%ZvTXBOV|m-ORR297Z`ZeRMjAh z4?YiW^CrNex2F6}w|&fVrxQ|rLAx(*w5N#^pW#ys<0B^ELjB&j%X0-*Uf)LiQWHaG z25D!XZGzhp8lcuISStSaDzxU&f5-D*F$t(cXVRP<8Ld&*wQTM+uzgvZx9@Gm|L(FD z#B*}SCOa8)C=y7Y(CX-8_cL94OOIW8vU=KSH81j4>&+UYx5M~VPhaBJ_Wb!Y;_^es z<=!jn`K#MTtVxF~AWU;r7xcaub$2(*`LGoV-cASN8$sAnoGq*AwctumCf&`3uMq)g za4M8O*Lj097}~*>~p#lAh0Nz#4^N;)Y)$pw>|NW9@ZDJf_V#J{pK`ar5hz@{eH# zc6`?Y&tE(UeJk_H|9yuxLzYy%S3iFZ@vS}%Ssw_i>*(RpZou2DYa^Sj+^XqX(^M`_ zr#)s%1|#VYezQ6QQ+KVzqH488{xyzH6MOO9sAXdownyi)xvSt z%_;CJeko+1nHlXmM=87PXW?c88P0Iza4W~8*h{j#K%B&sn!<0z(UZCsDneQ%@lvfnaEjx z{h`&t@mS4Z6-3<7BMnUlYh4d|{v|AJqYGn3KGh6Zb`+<*(-*y-wdU*ghw$RuTMEBm zduTX1A9s{PhkCJkk^ixk)6Tp5e{>`-Fcth)P9-0)2k+N;gci~6T+QL3pB8*3WyRR| zcr<)p3+7JG(zc$XBk8))k+%O)9Ps0;H%xoC4(}Kl@lDr>qI$_Twy)PGjEkF{FKlkhf$V@zNG`ExdQwIc>eIjoj<` zn4#b4iRisQ5e9BCfZW#YNfTCKfCt@&)_p6Z+7uI>oW+qh95u?%nT1WH{ZY)JU!7#^ za?FY9_>!2`#1z^Lw3p+4$AaRA@ADQDr(#y#^2g%Oo#oi9=Oa}3p4I6jcK^=~o7=aL zW6g z-Rcg!wEalPblfV&{aYkcw>%{eP!8d}=W&`Rbnfw1P;Cmr1Cq8wmJ6L(y)YM-dVdz` zy`Q|a(|o-c4km*)uscSZXB8+Kt8$XY}$;hkXdu!USvtCX4JkgK% zr40|SZGvki2uX7RHr_EnK2`=_>LM3LI?FkxHRR&M>v8izJ4qPhK4qbd){`F3h`(y% z$es)hOCI3>j}-TpUf*%a#)DAbB1nwp$7o%18P!OYcOLe2Bi4N0)qSm(0|XtJ&OV;= z;-hP%ip;WyKyy~*{gjt&SjSV6M_@ZoX~4eEW_-BV6hHjFgAdbf<;`c$g{6b7e4DqH zsp|^MIVFb8v`2$>rL?~KIB8-M3mNebE?7;F`)s=a`7N!|ccaGR;MzDRS#N8!R$WJJ zPwH##Zz^ccl@I1+z1u51@lPus|9(t2tm5na>OM}*XAotWBQPw*hX-?7h z%~LF}wUbLNE!g#@2gEA;j%IyzVUcf&crX@#{HMIHzZS?hBJqz@97CGOqB~h|5t{`V z_&`g14&*_Z@^@@X2(G)g30b>=*t>rgo^2GVQT5?!>(6wDk6?;#Ou~2If6HsB8s)Qo zwU&fyc5PfH(Eov*d0oC|`R{D1aV)=O#Rz{`^3a}>zJXQoHXI-E9712b7kiekLc+Ir z;6VEgJ@FI&buWYzofvkH_KSLEZjXH?8`1B-;;kfa!rx0VW^*L%XVFyodMKT`iPaxA zLD-u-(CWvutK!<@cI&}z&`xn|yDiMv6a!Sp&|=a0?Ax*}`G5sbsb`BmWi`3-+G;Hb zmq?FSz6&QLxRS@K!*z$~Fr&A(Sj6rd+9=ayswO4X7iFbZR1pph${zF? zVtewV3sBd}3;fG|f}is@e0X#LwrhVLH|mW=@_Y20?)>!b3_&``ez9X<2`An32Y4jpaP;Zn%|n7pF!YAm1Ot2p3g3h@SjB)}Hj{@nN%AG3bA^BrzV-tlYmp{YF8h-_OVXnOI=-fs~yvS0ARYP^X5q(&FszV_|4}IlQgVD-7P}pH0M)4fjqEV(V|B1 zZ#nRdH4K1Wo6d9Zi;2y{#FW*8fxHAG-BbP+NT>K%`Wz^5J6C!Iq?1726;oJC47C4) zThc0#<~W=D8v8Y6HxyntLHQ|EE9>^c+=`~sG^rbqzhZ@dbAUWH;-}pZC-amZ!?2n0 zRT;DV)*d)e+?%j94Vz$sNRHe_H8BZGVjF^f%n?SOSgkQr`6AW(HhArMgS8DAd8sGl z;n(BfKT*nOLRnxo8{u_caViqd8P%@XxYSnA8W1N0Lt*$K(lukb-T5An-{M-c0YG^O z6pmVM?#QYB_=g`lvUqbYPTln@o3b0rn$wv$DIIn!#ayWSRpfFU~e?>!WZY3Vc@sXtR;{5?d^NB96@b#%BFG zq&x{^=j_Cq$tR)FdCCqwl9gx2GkP+cd?Gyd?ufrDbOmu34|>xG9QOBQl=-u}cG(Yi z?&!K39xGJ3$@@%urP+B(3zX#)_L<_7z){Cwuxn$=fqMn3NGdtu^e}XC(8w+ywFl*mKw!Xs{}*if7;16aCKm zNL*!y_Mvl>wm@vWJ!PVuptAq-nfC&uW+CnI86N2PZ1o<&~oi{Y!Avk;OPEG8r*&^sj5mE+) zCv)o{@hgTVZh+-s_DZ+38$e6aw&a>Tz5zy8+Xov`U3Bfyql=C)`#yhumPH1 zBu&y*(@R0BbEt8pn9VT_hQEGOQ1!UTzg3Ncu&lB=on6)gPhO&ZqOKUIjH+@7D8EcK z#Gkxb2+!7c=1X*Psxa9ndN<4qFp||59!LL=$x35Y7KNF%9y?qTyIxnBk^a0Wh z^3!kN@3^gERI|s7JVN%9lqjWXRlPQqB`ND^h=0i=Epnx-EOzgDgj?oUWfZZiU@H>W zi=~!31l2Qo_}xIAkp}Jqbfc^Mi^`F^#}y*=@nC>Q3+l&NY02-o*SVPi#9xduB>oQQ z$u*`9lGa16c;o}0-*K3ba0*D524ys z*4$3{p{txb-Hm-&n95W(c;xX!aWRUiN5lZ#M*Ox+qfECGThG74$YTNd9SwPks^|Uj z)Q~m|65S8e*{7ZtF=J0NB%jX~l+Od=UsQhU#E~nEyb;ixGL<=L9t7ztTOJlK(zA7e zvLf8`pQjTOe<0re^{^yv;z-udtuL;kGcS6se+iRd$J5~^MzWoeb= zVL-Q4f_x(Vy+qWz)&i(kgQV>m)jOC;->)6Ki1uBZcScZONcH64yYo(5P+rEUH{p(1 zoAFS`I6S=mZWSJ`#Pq-?FJo|b6zw-RyBiC%Uo9q0`N61OvrF6G7v$?er^zr-x|@1A zQcHQ6JVL0Tyj6vpS%3ct${wmF;M&bzRXC+=2w&Xos&WAJ2+{A$E>zf5eLc;9w#;kn zTG8lFC+cZYKOi(uJWSsY6ODUnd+5;ju9~=1={)%}PTHV(|DqM?nidrEGq#UF@@^vDko~z1GD$By;Q}ms_%*)i{N`)6+&p}kbc9h>v z195@49I^}CoP*%hi1A3d6$tN0{e>#crQQc+qzp*Wt%QqD0?Gf}K{X#A=FfzEwmC?B zSV{dFF{Q^PMp;|+kb(FViSu#Ryski*jsCwjZ~Np0?&{$v6a0)-j~S_F$LTrn^~yM{ z%8Z2@{e|-P#7US(djprP?F=bHc5-(EM@`BD8$LSjG_J7vA%fHFdBfth&>+N!rySb_ zJ~PV2`ze0trq^ChSXzg7?w<>(33cT`CmkMoaxEV084Tn4*z=CGM`Y@j2v(Y=frKu3 zFm5!R(=@BSywLcYh>iMBy3cAZLp|#9IuR1_rjPi zGJE8_r;ymbJJ4ga<9}O9qs_0ud}{*T_nm>C>eQvP-U`Ks5#FeYwC9P>yGy@m)gZ=p z89V>_he)z>#S(cDMxDK#-7)1cz(0Li(m6XzqIa5cd=l z;s&DG%2qPzRW_F0J%OuU7eoG+I`Xo`LtK5tSC*Z3k)Qj~=UoB%@CoZ;qF!$_&ue@# zWb#%XWbeqYd)RXCwhO`c9`#WT(|GQU9MPn|{tS(J(DzDhmtkr*+Qa$hMAY1f!Amd>#*|C^oaWU$DPw z9n6Ndmo%U1_uzcxlPE7s0_XTrM)L=wBQ9dDT@Jczo5a0OdkYx_=nrwwdYvZLrGkAta2-B8d4wayTTF`7z~!O$v;}kY(4Tgqy>^#& z?C4nsW9!Zl6%Cv4gEQA+WiXw4KxbPkOc&G%W$cy(pU}cky01w^qiZi%YC{tkb?A}8 zBOg1mErxuwK;jv8)hiRIwnXP6{V;FE9?jTZhfv|YVf|p1nAi)QyOw~jdjT6=atYjO zxbX%do3UZDP`pN7zTv=XJV>?`!FTr~;hVl^>CAzJq5ttKxH4-gk6%0neHUMcHcaSW?fB=neImG6fA>{n)2kGPEyUu=iLEvY+eRd(!H|5zvD#l-uW`I`-SZM?0Q)7 zX`eQAW2C~k7{As6gX>#^v+jI%AM%>u7moQa4Wz<;ZeUl!6_@HUK=&qu{A-BD--gNj zFA=ig-B3oa4}@p#b8P`03J74MLhpgX6RkZ2lHTQCHiB}Q_ z5LYfm=cxF+F}W8XG@f`!D!htAK(x0dP=149AU5RGb9f!RC|yNE(1;8BS38+g)DRRe^*l z={I>jR&23WYXN29X>8QH`;gjjD-x&j+^;=A)ko3+pj^L^-d_(I7#Yg?|Mh}T$&t); z&>1+mdOeUPVcq77WuUW@ByN%AtsS`Q6nAcJ5UtU?rMrTc4~M|KETpw!xd$#lN?8<=wh?a?VbQzq?$_?NmARvl zktVX_1!?T6QAch(YJ|L8eg{U1sr>!rm#RjvXx?8zSOS`3?hpAmzPN#Oo1Vd4TLQY4 zcnIfKix3tW%KYRFgsEg^KGIfk6w6y-g$?i4MB+MLxx)aH?T=t(Y6LTF*-_9>toP`n z(oA;XH{Cn$aJOm>iQna_({EV46GiCPx(1KG_e3P`%)|l=g@86aDl?7DdumCI~;W@^Ez^3`%F(Yn!fv>bn|0(5#t^8~;iapZyG^e|Ydq zFa})hCX4E>X2fAUwWblg&zvEgH2-kT`+IoJuN9}-mE~p4@Li!lnl5wU2`jI%ur0w@ z|Mn%M$MO#~o`b?s!44gMc@yms+GCaW(!`0l^7Cgfz3fan*O-y+1I-s3XLmrOJP2_X zmdy3Us|z~G1HW5Bl7R>Kd4#gg{riHisTUBwalCyaU}qxHd-+P#7}{2OnGdJ#F^jo7 zFed2_q{LeAy6G!u&06C5*UK4=PbmGM--GfTZkvu%z0nz_+pEi@Svi8{87e;6@Y0YR z2)dQ5%}a;^)96#EIk;E&&{>>*D}rDGt4G(G1lK6X)*pW!R&-sBEUi8d%yWaVXF+IE zSOD~UB(8^&AKm53Z3}T#&~zTWr!kEGy&F=8*Ar&tY3hBE6&o@0QwPW^>q_%AO`!67 zZF%X!B(}0P^{t67+-+I_&}TQyeg{n0Z-gbU`pOZHR`Kf__f_d)z>^+KbI@AS`>>z| z&1CtdR#YSOUGQ5ck#q^DZsjV^S*UpxN8A@r`2Gf@l|a|YD^u%8zsJEE@|t*U?Ge`L z=S}fRe}|UzM@w_1#?NyNC27PM26D-9CL3%$QQk!pV| zUVm&YnL4+Io<}%?x)&=^X(g?fHmR3@x|2RtaYx{dbc_p~27YuO0bNg3jh7AVpiNEK z2tKw&}#^1t%IyFh39+RHzg)7gmLTExwNtj`)jUOYumgYjxcXw5% z}hS@=x79vZcMMQ1M6gplf6P}O{JaSY3i>@Jj!kS{^< zts3H7G`793)|+1sZzf1TAlJE@2pB=kCnHpe7Ae|!$ABj=zWD*jSfIQ zv+DJ!4xmA&Ef6|nvsOKC^ur6_^I|5BUtr6}BzEGwdNrinQ1y7ig-EHs3yR(?g9B*? zHF@poYt}>$f(Z{cfZ5el7;BROg`HJu+d zZw;diC{tUTx{|lns9Zy9CFzT%%-8gqSo7WoP2ODrpQIA^geTXO4&kcC)0pW*Esn9w zM&d;|F0ch3+uj){N5~Sxi=Y|3kJhchTt`~_IPpKTd~4wIZSA=0aXps znj?)9<0I|>X$Sb%p!cCVEPZ=MROW=hb@O;eK40mxs(+-MA^p4bawi@_(gQ{{M$dZ% zSJw2!2IC@t=1-0{vf-cVY{Y`^L6irk2xD71?}t3J#aC-yQ8Na~Z)p|(PB5=6eKT{> z^&x#H?ClnJAKU4a6JwZ5hnE;^l&JVdT-db$MwfW<$~XF)-jmby0*#x#*V-6eCwAv5 zPm+IxKpQunWOoY6_NQY}=u&N1yXuf)z6ah%Oo0;TT!@YOB-n40s`@A0V5D5mZ{7lXsw}m{6gZcIJ`bN4$R_?XfJQN&59hBXpQ|xVZ7;R&)U! zAHhLueQ8wjMyNHT^@8BFjZ~(>+#6jawtIy3cgpz{F$5K>Y3x(4=W#P61VnX z#4!q&jC7lkXJX`=Rjq-G_gi<$&YbcSjsJ}%(Xub2Y$oXa8F8gt6I~!mJ9T9j-@n7_ z=JwSn>OuUcWz{*YiBp_uhNX`F!5%d+)hu*&_*6Z;oqyhH#iuFPYL! z=~a2k2@_5{fu9;!5e`^!C)x*=u1`mrRD51J6x0l=ecUt1LZi z;Bw>YDzCtzz8XDaWKBI852T^N_j?Ag!x6aF=BAkBWzI=&3HTSJd<9z@G?EbyfYN6p z!bg$PjAYY@fN)R6t_a!s4PR00k@M|6ESTxR@$q3i@ zy5;o!aGf6fqj?4^ywq1({aX*LzZsU5>}9lu;=>y!PMCl+CU_nAACQ+<{*{xC(vjzr zrM)iD{*gU3#gCMIXymmBLsRh5h&sG(v&~2vj;ViBwj=VB{|KrFUbf5*oITphF1a3@ zjw@>G<&a*Es~8W>G2vDKSk`DPRbPJT90bm*NYmIRp}$uwfO!ZbPKQhFBXp|w>;4Ib z@vAgYm}61aPDdGrj`S#V3X$T|MY`@Q;jDDr88Ax!RpBuRKPtxI`@IuTzv@=^euo97 z?*$!?1zOcW(pGF@UwY2!R+O0V*nxO)ph%8S0FzJCfO=7omP6u89CK)#B>f~v#|fi% z{|TA{rHPPo05InBfv|{?PEdYKl0E}nbPOooA{~Si4>p$tH$EU~97b7^7~RQ>u;Zq% zOlrx^2XyCU=Mx2S1C&{|;p9!gX;(L>UFygaT|A}5Oj~J{?+?mn()n>>bW>K4_7+H| z!TULb@ng#_q)Cef=|SOp)gH)CkPmR?#F03<*b}Mu@k*b{g0zxUJV`taNtaylT%EB< zJuH%^#-QqL;!Gsp0bPC#1LeQSKPi6>q#re-;O|hjau1RQ1v&=h7PYj%Fhi;I1L;>F z&jHND0t+7#&y{-#!Xi-hqkQX?&g%tzJ}3EXeNDOQE%de?{;ck@%;CE@avK+%obc0loJ4DU{H9e5I%pa4QKC-myT&g zjC5y3ZZzr0RxR0fXNA^DShqsu3CbgK(gE3K3kJc5i9%^^@cLL)QpTY41ggw%d>PlV zE<30ncIoOw#u6`i(Eh7A`=N3yip&-pNAw*eN9Z&lY8sZKbLU zl~Yo_Cy1vc^)3*X3c@Eyn%S55FY#L-tdfL8R_ zmSHPNhd|Mvj@W7STTwdBP1N~mjxJm6IOQLLaxXM9qWccKB7kx)ZB)S=W_G6-Euse?Ggtz()aHt!hzNvKbRm>Sz$t><&+N%Qn+7% z@suSf55h-&^5%w1${Fcl!j+Ze4>#f%qX$6vt&Pvh$|3z+;dNJy^r`SD=FQKdrTrc# zjXo_tkBWk#iJjnLNt`h0IUe~`U+%Q)Bz%a?uJ~K$YU$u*RSBFQWWZ{_*Fc#PT>4BI z(1ZJ8)V_J-0hS0aD_=%>g>@YALs0IGgk7+^6YUGyjpMnt!&NP_q5vl~7eLoY>$?di zMkgvBr5rpdr}SqY`K)sENLYsf@zH{?7wMW9`4#vyJ6dOH-%Ll@l^_mOc?^0kwW!F% zh%b@y7I`3lufjR#OQ2Kvj$9zNi`PV;|*LXCqG`1Trd;f{4;}@@q z#wunV`O)##7}{u@nDyZ%&P!X*tJ(C1wdY2Np~He%;=u3lWb1a^Tdh8SGsjMD4sfDp zw+3K~Ef>J%!+i1RUJ{5n&zz*=*C4)=AuqNYf?fLm5wEAVkOwzD#-4xt;PaoOnEhZn z%uAz;Ft!qJJ9`o~@~jWKJS$$cuo*TosftgY9KwUa=6qJ!M#u^GhqdQ7@Fr*6u>R0c z!f-3TcY>pwwyF_YU$`Jj8m2;@I`d>%bam-CqKiCp(_Q|wDZ%xv4r0T@YbvgvO#ZtX%ov=@QH&@i@%?r>h9+T z0bLv390x=Qzy(Dr{JBmLL5_Kz|}S6c`uchW>)5% zy6uK5jq)($Y%aTB`wJf4`Ar;H++Y4`_e{{aMESdMtgfyAO|u5j-sI(&vVIJBJL{oO zw~fNLeMiig-CjPrvsio@&<7TtekSG&_=)5DQjW7X4jW4e!;R^?<)h=kxmGKF_ydX| zLq@@vPA%c})FioUt3BqoGSfwTjukb+_TZMMW5wc?)uEtEf}Cb&z|ZgXqcL`83$zmW z;-RNKD0-fAzF%j#EdHSQ(Zoa6a4E8gUtgNCEkzLOy9CpMLpy?P@ZPYYXfg#YgR7PnJAwm)x@ z<`;T{Ein5jRt%esp+2oy$=vRo=92aD{0-KN%URmDo^qG2Hq!ba)i+#g=Z7D+tmm}O zII3k``G$G~j$KKA| zy@?H&7u>^JpDW1&=V!r>~E}i~V-OxN;a9KJf*9+GQ;N{d^;W(wD){b!}Ko z!(Oae>yM&dY<(@=qaIAN3*y7Zn8KE)_rxQA13~Ku%i?}eUxuRZZ42HyEk?9(S_31! z+w$hYXKnv6ez|sCX{^(u{!k6Vr z0BQSYHTZB3N$V#Lc5oA;+swsrZ?|LpA5!f3XU;awjAMG0KIxKI3`57mRp3rqUmeYd zHs;U zDSjc`-gXRO+YWrZDI8)N7GT44dk$Ck1Fb8&<-SYgU(Uk-`mQ*)Mo;a*l0w`!$4$H8 z@(`zYc)*Asca5Sq(5}2 zwmb|}{Uz>qgQm@1qfH-9E@dK>I>g>%^%Jpw0MUV$|x2 zYaxAb6wkeV2sBq>QBxP;ercloI`*dS<+yk7Vq_hzc+_x@Q+Cn`eV$poN#O&wZytc_ ztoPD0Ld&?*IKkk-)QtZkgzZl#*_phVTYg^z=tcZ0QG7G22VQC2nh%H{3)l{ zt=$^rMBah#C;#F3#34x7n?t>&J5uEu>-Ivx*YO)!L6;`1eRF+qcwL1LjF<@TZ@U2c!#Bjlzl%I|W^jpy!5h?2`xd&h0Nq`bLIcD!?bVz;8Fa7qXJKCq+K${rG32L)a)du))(BhQctHpI z$KWx4AA416r1-aIAo0jgv6l9?qxs>Hf4{?~lTLW!O<(!ftqqS4@{z>bVt!`zidrMA zLi71uC1IZYKlbw7eM!@P+@!97b&vfm+*_;?@2ua!sZ+-l*3!8Bab)8y;M%pO+~V>d z?GFoh`ka8&S6v_?hH&|2B{^c+dlt94K72`S$ge-`j?}L>!)dDcHpN)g3K(o?z)9!P zUh{Ruar*8=aoSZMQypoV3SIeHxB_Vwa6B{=<5tJQ_24D==lKYs^uZ;Q<1l-dtGYI< zlwqXiT(tJMAQK1Mg2}|S&^bPz_LTgFW|j-lFLZ=#Mf-iKbM!W|;v;f(5Z7inOS$b0 zji)!2Hqpa$hj+D-q&J!3f6{zFT8xSKtuD*lN^r&RfjqF^KuK68=Xy8*{S2(SUj(I# z9v065!Z++ybE|HC=0;X`TWwC)D_2(bfUFHg?2-w+Q~vG&(iD${`TSfwHl-Epop1uD zHoHP~a05s0vhIK(8@C*qTlAC=`$VTz3%lPdC8(xdy1T z;}4%jx;3*pLV4s#P&_T-9x>u)_H;))7CiJr@%1n|Ri3XfiT7`JTo49}ps_|so`8|& zgaMtI%b+(a=~yelt;!~`IUoxj2TdT}=!L%m)?t`UB<3x%k!4XgVbKyjBpri|jINWu ziN>3*FVO0(y+#^JyqeHRPHi?AZq z*8^!#7}?kqmiTT4J3|RCMm%SP-E7_E`x@~$&^)r@8=WfjC}As{GA+Wne@J_lB(lM~ z-$TO3VWNH0Yf4ul@jQ^ehZZjBN)yA~sxNe}#VIW7<%%C3Tk_d`S}5Go;jp1x;mz3> z!8j{AhqUW9ApgVm{G<0roBRXcz)N_(-*J{U$ea&N*$0bPZiHpMHzVO=PRF1$>X|Q0 z^&0sc-g9^Y4jFPA8&6MEUJJUtV1g~K0;Gd9!Y@IX2{e9AoFHcJIwk(boz2lpt;V`_ zDF*UYgu8RJZlUvV&*A=@dO`fO8L#F^kWUn(DY0_RgTk_H7U|Oy;_dlCfO}q&<|~Kk zeo5ketHnCEOz=Qs$(e(L&p)Btj$Dy zZ0=x-#O<7PETj4XlSv(5%X33XJqpck=))=6QxS(1Gs1X`O7jA$U(U|WlZ%V@R{a0_ zYa7MY7Vb!WMxHNFH~4b~>-eZ2y@R}mB){T9zD%fj?AB!{C*8}pXVvF~*9y;A|NC>1 zYLGis{($-?-vIvZ%m=$)1=Ujt-D6Spn>^HnlO|NXEZ#P#!;_|RzTuaX@_j7c`H6VE z$%J>flaH~@FJkTHf62=r5RNhHrRh2wdS|85fTR<3#1}}KU(z}Yy-JIKIGC`YmhAG~ zjDPxe5PA-8g2dm_ZR~!H{Df#yCzKIq;ZFain0aM1+8&62t?z0ptu3gIvFER=*vF~~ z_Om$*isPuq1#yMa6{tKm^@MgW%NM3y%OMWVz*DCE1dS0%lUL{(r3oxcno=)VNm_g5 z-=SUo{<@BiuXO2uiddzwCn-y4%nQFnS73#zVZst54F&c31y0;c==>wwx;fE zmOlKIMjA>qy*5k?j;l+4aT{!(c#-an%LdZ_%%*I&7JJbiYCO+n^chK601{p>^6APu zOTrwkZLi7A3&z6n!l6j|Q6u~k53e`ExqmI?xUGG$ve#MVnZW7T9msafSYr*VWba;^f=Ffq%f@><&o2QQRK%1>UcB zWrVvTv$$rq@}Gp4IZC6F=aQ%E4n^ugcGYJKU*c;F%S(x9uLzmA9Oiag+EY2yY+gPTDaTTE2RvBmD}6pI&I?rAEpVi#;RlA;{pY znpffsA^Ouk3bzL>!47wN!pp-BoOCpNKYf&ycCd39k*mR9w}VKW0JjgE7QH-Ifyza$ zH0lA@JJ$fJ8#Pa$>XJAd*)~8M>eGl`TmC$>;SJm1KyZbif z>N@111V$bdp?hc%#{{WbwHYw)&IejuE0(oYeJ7N=f%SODDZxnt+YJp&#HXioHNRA zDZfd$)LhqXa}!~`cnwoI5ostfFT(?E|F+lOp1+I<-Da{S>q>N81E-5%&!-qi_er>R z-Nf9lnDT%qBfNHBV}^UOn9}W(1ra74$1$5u>sIP-1mYxh9P#X94p0V{L+h`yCguB~ z%FIimm)Buhzb~Yh`m;UdlgK|bK%Du@wXFzGvhbsrOHx?$+qGbo!>u3)koF}dM-G#6rhrBwRKF9dD z*He+OtAbmv1?*yqcg`nXMaoiS+taJDL9#7UE(8}n-QkzLrOdFJ!4#G#?cF@s4XDO+ zq|XJdH`?I=_Qhj8ViINefq_Z~0Ii3Z>9SpME9s>+7(F8pc7F;07t=qWJM|LDW8JP?xv9UPj-bnc8zK*o3x1cpe z@(qGAQ2cePH>=Yr7w4u95=wiLPf)oOG@kmAxkl|*9!F5#hjb39ybIxl${3Nd1NeG$ z5iDNr;iCKn={ZQ6DpkfspIv~arVW6w7e@PQjJSeD8)p#*Ws*)b zpiGk1ESLOS1gfmWYR7Aw^WSDh=a42x3WUl73BR{i^7NhxuXQGqj;J0}995x7NV|v$i3W1_Cw-(|*C@AOl;KF_vz3>8 z-@1ju7aiqK;`v|`uCUNK$CG`0)sU0-!sA8$_%)-s%G+>(*$U}pq*1-N5nm;OvEF@L zppD0+k2-SFEOfoQApHJtPS~w7Kxc>WE;6PdS?F6ibIM(iG=%Ci)En;1XKY*nzc+ML zIWu8>PDS29`Iz#hS20SK@+#8f;Q@uc5 zNO^Wn-beVqXjbvF)uzvcNev50HOL;Wt|aN$${))1%jYw~9-QRq3-b+=fOG}?cH9Xn zZ&UTRV}2;899s1#^`U67rN1EDs^G&Wbqta61mfca`fe*5$XB4^SK@IDc)JWg4LFBN z&*>c;fiC808f8=!-X~;1GdWZ*Pms=0d9LDAnRUlZ65m6uxDJx|kGFXISk&tA6v+!? z+tYv8bcgT?4lM3p30@vchOEC1Jk0MXt~8E;pCjzisU$!g>~a@QW-JrIJ8N@;XDN`f z&>8>LdBG+QS|(3s{0EO0wuh%vxV&6D3_dm4FE+Y5alL}NyyxbJtXb#<@LWC3C1+7N zjQO@1YIG~td7G^g9{ZbP^uhC(n%tIuOdW=c^Q@$YtS`Hlw&bq1fq27XBZlmcLEoqa zn0%-%58k;AH;zh!MqZSw!EC-D+Z@NO8-l0$Wx&ji8MtG?ThZcmD_(7vn_P9>h1cv> zP3p=k33kX1N@vXD@$d85QDYOFXdMBqr&WUAF&1cgwgj_gtcQUozQf+M$}*+#WcXKS z8XnxWA1>^6!DaJqqvvJDylp>=wY{kqO1EOmI@@{r`PS0gb}=5lRiAe=GiLbwDvtYS zfq#dd5hgo6;EU4g=v4Eq8n4`TznQ##vj_T^>B|PUtKn^*0NmH~64E?q-#%Ofzk4J2 z#@lb8L12f9c^`VJD*xI#l_z|67qR_o$$^Vv+3zE@c+G&x+_7X8(slA(9b2Pb+HNcl zrFXMu3R!jGEgU;^vOL+amqx$GZuDGRSiPwt-Teiuq&;pPynKc>ZUJ)eDr+9wKLo7b zne(rw9^&FL7tm^ICwA7`QF@#GM*SYIbAET|BhumotTe92)z3cS*-v2FZZHHTe{Vr_g zJpv7W?Uqk}ePmS^j+C|Z4?w}UPdKWw9-Is`XMJlq!z|B4Ty!XrR~_RC->RMgi_~AD zxN}Y3Z%90RGO&j=RtYTO(J-)C{05z!?+CjEe__WRCkSvn%JKI(@o-HOxW6O@IuXj4j>A41QVThp2Jr!AC-uSx2GkvnG6@#Y^mCZ!Jf@GvLP` zWWtK`ow$ssB5kuS(|S%3&qog6p(A?AboXhn;N^d`r>Z?Gp8S!`tveLv=-m^B?uOF) zmm$n6sDhmZJ@cA6ov(24g&o}k<;k5jUjLfV{4$XW1k1-RUJX~Oa}3>rMj?s z&oSu>Y07$)U`g2q?96&{iy6Cweq=8CoNR}KE|qG%uJ?quR<)3KDP{B04CO21KcdQ- zbHMWUXeHU-vgz?q+_T<-C*NI-8I^Xy(#%#!eFk3LO(gvdym`A*(=QK!PDVGWw|()$ z+UFv1m<_+4djZ!@&}Su0d*T=KE)}15nciCxo)MO`=ev$RN9r#g{xD3|`SeJaKf*;; zz1v)>ak$|CxSrczr+V|RwiS*{8jrq~$F)xe>Cm*EFN|Av70(*J!7-tyacX}{UhCKp zFt3$|Cz{2nYZGdX9xqJ84Xd_GyA9r`dT!xMJx+5eU$tBU$F9?TA+a_2pL3P@$Ncly zc0qL<*u)Lzu5T=#t~|~<`RqaSr6%&?w0pwGEC4O^$q1?5ePmD{Sod>I%4?bb>L4)o|Nl58m}+vTo#@GonGmDdNw>Ongv2 zR*tZF1Ycd7^S&WVuxX#^@cT%JjPAagJ$be~=jk(FPFRa$z8PoZDntA=$A(iah>|Ps zf#!l2Jgg!Y4DLm9{{?%Tmq55D|MS{SwK7$z@zY$f-yP23q8=ll)Wv|S_0Hox=-m(} zF{V+xR5cLXk0alu1F9Lgo%9A2zWOiPf|jXOxKFoP@XB~KT-!1MXSc{heG7lw?XwwZ zURe9d+u_aZJIwV~d%3ru47*SkH@#~M2)XeWM;&RcP0RI!z9C;U#T~?*^7Hevw0X=w zh-kNvkJ+1z#LZIm1Jwh~;V9|pR0ExsWO3aZjt<`^AkBm5WaOdN3!Xl!#7mrY{57Vj z{>EAj8cQfM<&M$xjB|~(=xSS)PwgMYcGW+|s7CQjOk?(7dNSO2p9A_mW^%g?->~Mq zIb!^oE~1b9ZW!^TG6Xfe!nUqEOV?9|{nv&-6^oS^bNmWZb635Ko+SPg`x0sIonc3W z>IGVB892~ZH!RQw_HBxlN0tpoYrkIXHN3^Dd2OYtzruA0_j=j!jjmacQuYPnt1uw$ z7XgkP<<>eY@P6W7AbkPDf0R*;50F3~Gdn7brTSdTn`=2h zxB$~V8sOH~+r+o3Zk&FH>O52fLUA48nCQ0kysn4)3Y;yL^1UT9fOHIa@LNE5h@^9J z#K6X?UvZ4?9nOr^gP3aj6;43>+#r}ddIMa{IwI(NICOrxAa2!YtkCc8S(ZD$0iV+7 z0He8+Cvq<7E?9fZ18Y*y;bE4D=3<%l?#5YR6iJp&-O?lL%Z{AK+jZEW3zRt&9~P1iH-NSgLW6j$}ze*k~ofgPWu3) zOJLg>JA8h!o;ZARFP`cI==tIpoDR3bYSkm)9=+$2)(@ypsh6M8=kGBaH+}Bc_7G4n z;obE$rNg7?q$Q?ezrVc+yDo``7Q^H$pB=Q$p0de}I9=Zf#=LB7pp1Pgh1R5lByA-o z4*Cx^pWBXZEn}f1u{z9oGnqfzYA3ImHdFP4Mmwb>OSKgVFi;o~%xwzy|jBR=qi z54H+^EFa|ssNUejn|SqjFFEPpb$sx03L3X;fwQl@!}YIv@%Y$zK!1z39u-m#493xh zYXs?I`Stcq)}w)ijaJQaxb{?%G%!3WBT?r%afxF9oM%RvQ+UyaESD^8C z822v{6@Ql(|APsi_liP#KB~WEu1;Z3-?CXa+%iZFl$>)r_uNrVKB3Al_m1z2D}yaSrBGny@*8kHs4IB@g?^tLc&{7 z^OiGV-akUJ`g zWm~qT}m(L&*;)@SKzx;JbFe4+PXkXC>fr5llS8WgXX&Pq0!0^v6!UIRKF zCU`~*>LLExIFR<{&I8N*Mbsa2A)THjEPk4SLG$-14%W^eZUgpP^!b99%fzV+J;>W1 zE;=4vPRE!Dox)<&Ix?GfVL-SEkH%!d51a1Ts>VKu+})G!DjcVJof9YF5ZBYXl(Hpo z=}#1`RdY!ilx0{&5kD+(X(C%d;lK6l?>~AE5_yQU05=|SxHG&>=Y&#` zdRO?VJFKN#nt4EwrWVSFjW#St!X8jMUvc|!dqcLl&oI7YgO~C(VtHJFcDDIrw60uJ z;gTee#p~ZLXYN-=%e`rSs5J76xTE0zsXrs_&aU=fjti#)P@RfP+X`Vum0qfkVYTlR zN&3~rX7MbwK7#5CNE-q9GxQiSP^UbF>#a1}o9s7`7Z9X_B>4+Qo(RY@k~Xc%%_dku z^g(wKWo^LC9t@{`*XMmFY(~}hnn4bdE@Dat_~k!&DbvPPpHeyVBlX#L-B)SIw#NOVaf0c%>_( z!Yrz12={hnjek_(AO0rmh|dJ+CnSxaG`YOjzO(Ye>T^QzV`L38B#woJXPS!sInX244i-%S}kv$;Rw;W?_CE z@oKf5RL>@?Msg7--l92EdKyTx$}xr;bi^T&_>qOq(WrLL;_|p&6&fU=uQh-5Ul^Wx zGOa?-9C}uZliq{4$$ogt=eO!fB<&7auLL{wM^7bE#^ zq`DM?&99RW@mE}}Q(Ce-bg#-52!EplX=b>1xfSu+G?+Ts5J=z4Q$-)p@ccQ_t2<%G zh9|HwC4edZ4VvGQxT?7%u3=6!XM@kkNl1Q&k67MCXWunV8$0u^P<={k?m`|#hTK>S zbra0t`MjP;V?g5m3SVLLUnX>B`7~Fpq#94J;X&v$-W9w~_mh-ifUevZY_kSIYI3UX zX2%3}+&)yJtVA{{iXyN4Lr@I^T?^1>NT+zPVLKb?l<%#P+z*sDq<>VMX{2>vgZX)t z7YJGpG2AN`TOKY1(yD^I8Xmizj+1I90{I%&YWFCrTN5M>7mb#8Q`p9=V}L0?kY{rt z=Qi!>e(k^reqztaimXOqn)2BN=R(<`Dd*VciRZQ7eYyf^SN8Yycg#pX2gGHRb8HsG zk4p21N!!K)&A;y513O%h*nqMhDc-yMW4<$Lvb(dkA?YYiK81aC-3uxkQFTsO%WY_% z(M`+kkmei6FIIT(Sw2=kcV6PftPbFB(36p-(P&+CKf*WTp^K47wJIMS3d8a3s&LZE zkowDy`hFE&``HGIYW&60A>W-SA7a$IsPwxTy$jRqYZ1I2{sgM#r3%6jB+aBW3-NFu z+Kt`Ass3c9 zR-HK2m)v}=jwC+i#APCGa%JUpC@-tWm9JCt;JvjQKC^uY${*6)gVPcxPFWQ1oMeiN z7v^D~-qGm0))&{L7c?g{JE;-aQcv->{cDaqd>;+p{a7N7>h5MB8@ZgM1 zNH~WzPM&7Z=jB&qd(MgbFetDgH-LX5tl0Jz%gyR>T|7!`%lI*kBXtAoMF}QZ{$;K z1!+M^c+M!h6Su`^eB$s!$$TWi#mx^IepO zR&{x`(KbzG9;6-Vxv714a%V6j?SYMd_|tyM3z2du46Qt`LLZY>6$c`%x&EO{rgFXp zRo^r6r11B(57M~>VJK2AB*;&bKlq7lkJG+Ew^{({OC;?lsfSgz>ioV|Ecxr9k~BRd zFF|=oHAX$ju6#^V{f#YNAE1n9swlmxL(8a8QNzrN)@`Knq;g-=fj}A(6$alHlWb{LvB~U86kXA|sEDCvpa2bnUu)?bj3d zG4&edYrC`siE&J28k9G(7R9vpT%Iwitf0@*KY}zERJlU;Jk>p?(cF^HvLilQ0F=iu z(o+~kd!l82ZOby!`@omq68AZ~QC`&oFQ0bQr8Is@@3k93d!L+F`j$i47Wl7Uibi=a zlsIcb`9AVwobpA&A462@N7)!7Jt7tN?>agaPGzizC%bl1KDgPo z$TLB_zYoTlrs|ZI8GkreP!5aa&5`C>BWwh!JK6JbXF)h5cDKkDt#;oA>VuquK83n8 z{b2E!-d{28z-yq4I%iDtrGjupm+fohF%ta+d^PrdO)I3#5=BaW!Z!g@H+{V0?J zr)*3|>DmsIcUF8ydO%cU9#f?5`K>LK3;bbL0}+UOA?k8Rpqzx6^-hGdp@;EmquNN> zCz3{DAKTU?eEls{y%8rc}eqjPe~O(gLW?<_V_UY`aB2Mj?Pvmm?>7Hgwm zieYQnc)t`WR`YqNUro5SeKo!72-ucWwPeF<2W8?g3dATF!ZfnC=tXFkj9<&vwFWHcDVFm%m~(27EFs88$2o z;EeOXqjy!BXraG-(?PH_wO{DJ^T=yJMe?G$nM6XO#~*+m_zsEv_2L9KLd_H&qUyeZqFu<)xL>&!W9r`BqHNsUvGY55+zoFJO;u z@p6@~KKk@4!6WAv%CDc>bNhrku$Rr{2d;0UF(ktrc}VOE;b7%S@9E5Qmv)_dfc@fp zeE4d()-$D%>|V1G;f(_tn>h3Je;m-DunWIcW34=Lz6@G5k8~+ZT@S}*exiK{9^>^3 ze}K*h$%`#y*GfBKzey=pw%Q2!UPU}2(oh};%2>|n@gX&@h}f0k@NUl`TsZHZ?oRSw zrq;6O(IQxUdn`Xj&jhX4e~i0Maa=_EV=Zs|0F18H&Ye{{}!Xl>Pil7Hb{mUosvE?)lu!>}o&Y^yWqZ_s49DCCD{-F*!4v9g( z`IF#VaAn?vn@Nj7VMy4eYVW(JVu!AEx&Msn*l5=lplizcD${UxD1ATBJen;%bRQ4Z%R@UaOHRi_ zKZokF+Q=Z8)$0pk(_X&HHvkU$eV|%y1OqN!VhM$_nM;dUb?vw%?uDS9gS;{EqWc4P z0KL9IKf^u6?{M2+Lw0h=PT4!KKkROk3#tBjjE+xO{Q;|d{qRG zdB%9rkrj|}?GrOuku4Lxaa`8ZUre|(oYPaeyote97*?w}FK)Yc@ zuK1KkH!+X}vE|GCCG`+LcVIc#@Ms571>|jRs#331cP@gyJ9RUTC8_U8a17sxA#O)<(W&W&XT1f6QDDmEp zGaB92)_tu?9Pn1W*wPwr-B`n)eWG{3wwfqntq}W--3?JaU!h&+Jg8O8iGTf^22^|0 zLwk@g1{{*>N=twG{(tHiN%bl3RBj2oLL~NT&|hTNh=tUV3v`}NH=tIhO>)f_%8KpW zxRG zzd-Tk!CbFyCwao)EQ^Xa!iCOb!P|1YJg}lPr}DcD-tWf%TzBpm2Hx$A!hfl}HKhf# zEbS&9tQig`_J3xTpH-7_>EXES_;_BHs>c^yIgB9#Yw{YN!Dtgu8~^)lFMmxBP=scBSDStmK=)I@E~9rFYsZ&eFCzxZsv;*+BD;t)se2>LYB}c?p!4tf#A1KHNr)t zacPS>`U>iSoYQ&dHPR!r&(wTLcm?ra-vIFkgUN9?a%~Iw`1JvJx-l7-%x{lt&OTIg z4mz)MuzjBfyGl((dYdv-9BZrhtXMR| zj=kOBaQ*Fq@L!Vdfq>%byqQ@9L@m_m%+o`~!U#R2d7|}e$FGcDg*}!w;;I)&=W*f( zq%rb8cRF%a+ru6X=AVuq7T%U6cz5nTUbgNq+qU2(5Z>u>KlDZFMpFtx+#Bd`^h7Amc~iv> zm%ka0RBJ$EN0{6|4(WCU2``n-Rr83)tAEx_w5Po?su{4zPDWg5zZ=n4@l=?*uKKnv zF!a_Q#kC@H(Publ76mt=hv3}@f8m_k&%WzgjI5@^<6b);qpcN;bWeijMo7CbtRa?R~X9F3JvWN2p>|G-&q4vNvDn8lBjx+;Fv z*^G5<$;(EIwq+w%u>%2&OQIG<}Svh+qMIiYFE}MDNv!I6jM3;KhA6ba28z}uC2DVQG z`W@71W{i_AAAk_+EZn%fi=1{S1&9+#TiQvb_mnnSbfuk8*R9s`UdlCy>=ah9P6yKF zA>lKO8G8+SK8y#0!c2wlJcn|;V|jJilSavcI0@3*ECkY^azmY`klOSk%(q<+n}Vmn zm&{7Ax7vH~jqAj%7jK2h-X4N9BHo&kgpRa7QCdJ9o*vp0wR&HXxE%>Iq0uZ;pneds z)^Dh$YI4G3ZNe`br1P+~9Uj4<-j;%NJggo+5wCu{%ytZT4L$l4DLnzCjUaeM4W6Ih zh|fRU79!{&`(?)ds=twZ9FVTTpr@H)QsNDI@x zl=ywCJqvz2nI$id1nLRWPIGYEg6W(%oA%c<J30jel+8 zhvYHQJ^!LEnOjJuojxa7TG3DC%a-_f>aQJ_~o3 z&z5`Hd(rs1Da`+!N?3bF>2aWIfQPTHQ@v!n*o-;_t4`=EorX4}Tv^Z6{g|pp>RT3A zEr|F#76}KmjQCGEH6m*OjT5)+%zS+N~niPMRZS=-+YZc7>&!B$x+_KjayXD5QioyFD>T&8V!H$90kSqMKyP8$`^;d z--L-7`jje+uuZw5&E)W-|3E~PM zEh;*_aI4S{xwKEkil_b+$D{cd#92ss2bBMx{`wZEzFvBV^1TVPXS+d9B#$LN7_Db} zCYj@)9Zk{iY6cdIJ%lk|a8yPMY)J1~UpLJYkDShD>iUVZ@pO?7o}>565%1vjAA?<% z>}}5pC-8muw%D}93<VQyuOL!vGI{!RZg*$(YAx$t82xG;ctNM_6VF#Si>n}epqrFcZHsZ)a`;m>DwhdROk*`A(=_?>x8{Pb!KKlx^?+YRgftlNk9@IDaz%3ESD>(mXsm=?wa$rVG-4 ztoHjHB+a8WC<+JFW78j}K;+CY40(4#5EjVw6RU})>0Jp^<^yFQN>>P_+es%2e_dy+ z7Ggk}a3W!C3r-pY8q5T7KKUX<%y8lR!Bs??&?A(+a^aj~6B+(zD#m%t(yj3bMCDuF z8>TAH2>IKKD0^UnFb5BZEYOkeQ@&5DXBVzK4yUoZZ249TvlgEvzi?TQ#=`4k_lhNP zmB=GKuE^AGCeRvQ)knt<1(aD$Mb*dtcOE18NFA*?yL_xE7Vd7sRBbNR`7v$PIkX(9 zG151jJccCx#KpJo0nH!GSa*=PBm*zytVHFTs0JAIruf`A4SlC)Fv4Q-YHAzUQkV^d z@#ww44~chy)`L4dsH?IU7+p3LFAn~NmBUmSNPEV0>;!7we@yED#G7Jua7Rwu z#0UeW%IB`OxWexB`NuXGcd5`blmj7QC=zE-uYM9!eNs?)|D=8IS=!(_^1+(xI)#(u zO@!@tS7+*PSoCK-qx#W;R|F%CLtJ_}5c+Ot!Xswnh_r7XQCku#e$S_SPcKA4*d`~^ z5mhNOENAhpXKUo|xY8&yhmAt3)0?qc>r6p4MOwZ-DoiB3C)E6^9#(#Ow`Z<+J}!lQ z3-(j|!hS^fE1is#eYm``KL~ddTS^|?fHW;VAG|LM53HD{g->(_>Ki2PfI8naTGQT0 z{+zdL<|@fc>+&`?f=y|`R5M-WREv=?J>wRTJ{0#|ZKTh-64rTe0ql31r=yHPD861i zy%L5+JO|1ZwUWNqak8t2MmdT&_Fr8rUl54$Xal4ygml6)t=?aTlu4=V6-Zy{lz*W4 z1j+x&fy6WD zzWE}MM@OYQD03A>HLVFFrZJiuLEM1ESz`0Z5S`LAgxyHm0;q35ajWu6lr13nF_rlU z%1{~Mp0o00q?1|HLJfjUYVe2dt>E92V~nsA*G3#u*$d^h2Fio5QlC^jG$_x7G7@bG zJx5k$;6@gfwFy2QZwGPCSU0*CbN*oI{+Guesjb3FO~_bh?;gm_lB27N|UqK7$i}?I%8{2C==uRo2BQ z=fvDlZ}R*t8DTtYPVYrK`Q01H+lhtq#;RORN3|dhrk_Ug1wv_A!Zf_&)Dpu^I#uuo ztry$s8H&`W+S&=nR4s^^!)`PEtc?|#i1-UBr&jr}^j=U|Wv$sXFJjEtVWP$iNAkEj zak#VxWkUj}COF}M&O6BuDSwvA1C+hxa$<4<3pzJj*LUPU+!sMx3jfT-O9sW-p96&-%NSqn~k`^l&11)%s70x|37xquL{4F+#BAr z-7M+wG5oZ7Gmbc21A?k_l}5rF*R}HICyw@$rTHsxQrZUo`jWtCo+qkT3gMw&qnQ1& zCb(~?KeQSfj|=t>L5J?0`2Os5;)ruS=+gnUGf8Q{DsRA$A-&-K@6NLDRyRr4D4&ig z1NGnQtH-dMzs1;Q+FRIkshBO;e-*+;rs5ykt9P>FV__t|LAbT6SQ2e3+nF>&H`^3e5eesrA+9yarkkxSBe z>q!APzxf1dcOVH1m$t??#|rRVcoX?m%Z4r88_7gp1G&=l9{T)iDSf;U1_f`0@l6VJ zp^KVHcxA$O#g#*x5$CPqMsmyOuK2t5W;lH>mRAlt$&TEoeWLA$$zq>OsJ(R*?=U_E zJFo#fto~g=^TnKQ-eZ5eIm?qbGH_#JD=^zoQ_``J#-?jMNgt0*&BrG9_4(ShUFE$x zcIX`+&K%S0Lds+(wVrZC*mUWB_cIPm+K5|cOp^8OUSj!=Xs{U4Sk8^}=euqXk@bVN zGvo4ftk$qn#klNVKMQpH35%TiS4+q?^6|}u{B!DE3@gjTi_7i{hYNRcXw5cg{NXWPf8dEmFOsqK zc~fp4{~qscUB+m<6q^{>&1T3Z0oC}rHJP~8ZW_P-Adg2h?&=kYPKS5jQ;D^Y4aSdkqCtB+5;pJMh0X4p!IeBi zuCsiAG<5DEYuq`Fqa$_skk=O0n8I`8H*pHmz}r`ivpZG&GBWO*7M-~5f2 zFD`;hXRPqW=wjO2ldh$+t$ecn1WS#10?_On8jyqm2L;YX0k6Bs> zejORXAH*aGBYH-Yav4_`Im+_u8Sv-NANc7tfiDUljx=Xr^X!yZ(Elg&JslzEm2Y77 zkvE{ScpV%olCXH>S$O@4o_qNx#NKh~&?F!lZk6lEvgu)_@M_-!G_`hu2QM5U`an;SZ&(+aKbeg!rnQy%6I`_WyJu;s z9_4~{mvEQm0XVllmK_{27+jhj#4)EDBh?&6=+XK>v%4}L@fs-qRh|OHQG<>fB;gaJ z{Eow~P1m_3QQ#-jq+k*$!->gZlr#uV3d&oto0moH+2_ZBKASbKc0>OlteTP)f(SitW^ zW>w8^#_q1@UHS>N$N!>NTpw8fwJ}#XP+=PY^^I4`J?-z{+fI}5XW}Nl@J=O;?=u1^ z&sm4RF5*EDU{dGm+&bEwhuKj7H4Y{M}ljU{0u|9wj<1CIBX>((@d+?&

rxN@7GKY-_ldY+{(^(60R6*NZUKTr0TF8@0w#x_t6XYQ2jlPp>M^ZzAZpLRav%HV(mCv-u zVQ!XRL=M2wgO(}OyZG4nGW25)u2^)IR0ju|fPhF8Qo zNYy#}kTmL%bXd9W+yb32mmWIIJF6@o(--e{PueyPq~Xe2!!8wCX6@9ShpU>u1=eZ_|eUv_ofFF zu*DY`NYM8dlNkW3WQAXY_P6l=UJO64kDTF~#WSJ=G;*dG zyt5>EHK|N@GUpKLlz#tl(#UUUiAO&k-8h!!7PZ$-J(7&qwRYR}kHNiSc8+bBz#$66a>< zC9!z=M|jRwDxt$EWpt3qyefDl(6=+othfI3X6QBv&tdWn=-|VF7y!JLz%)YM!1?^K7R2sOq`!O>%U`TE z^GqrJTWHT?I0ar2Wx_QnO1tu*^; zSsLBAAR!;nciQh_gYVPV92A%@Wo_P>MEm3HI$^Bw(_D9ghrCqN$|fg+UiPhgbRG3t zmLWDc77=4l4dTyjW^iUHrv5IkO+Lg`9!N$#)4c-qgwcoKO5U)(3;EgpXMxvM39lCr zG`B)-!CyL8SI`vd@u_*rxGCvD1r-haycp5g> z2#m4dT+j(!<(l$GEWlIrzv6(X+*lveTLb@1Z_(#{cxL6Miv&l(GuT^9p5xzgqktyT z-&Z#0_k$>+*F?;1(YsR{=@JYKBuHEEIB<#wGN6}Z67v8mo&Z3Wv z+wnx#A9am>tP)p@x{q2QY_E(HEv#!9{zDA@eTYV0+)PGe82$F)TTMb&j-$_CvqeLo*DZAU7>)k4wqYozV9Tk7IR+dnb(M4m$5{FiLuiDNEsfET-;^?k0o9 zMvs|sHoh2|%Tjk^1*5eUunl~CLIKxk?KdkUJHVJ-<#w$L8*gE?7QLi^!2)wl8J$8= zhYT#VG1^WA{YfPE`2o_@n3-?PvHBMCP}HuNpO7V`jV#tuqSnY7;v`Wsoa}RspnEJv zn}airjl;Kw~FQp}VAzZ+NE6jy7+@FxU zqKJ652+!fC6L1fCNOyv^7a5NCQM<7M`4Jh+|Nh)l4oK~;;I#?b9{R0{5Bwm!t3_6M zp!q+?wPf;N5Z+B7FiFCc4INc#vpg&bpra(=sAz^j_v-HP0bBTF^a$eFFiXElkeB5$AX ziu`E;n+#@h{UZ#WN2Z31Mng@iz0(u8?#4~VgTgn)8(*wUu3~)0BfmY=e)SYRObqhv z)%-kl+`f?OI$otC{Z_DJ^)yv+VieX3tti(G8z_@buIGQVM)8^U$nwg4R!vh2$^sQy zN%t}bS?%sBE1kE~E2giIb1(c9yHjuT>T5PSEoeMnSo53PCHd$i=hs|r&}3O^dW4Rd zwTX*_6q9*x?B)AyHgWkv!{x|&Si^31C0aGMgRJY=fs#tBw%lm;hHhoE*H=;^$l=-* zQPa7T4E*ay%ci8MYNIOa;D?fl^NJZ7%(Two&GXv2w>ysZZ|4 z-Wv^C$&V`|$!3wO?!Mv)Utcj<)fzWTW{(ZiDf>nd?onP}@l##@vxAZc2WpHP&zyUZ zs=C$Ed-M5o?Sp%j^OQ`H(JfT%%ioCVAIPI~MAl=w>G*!G(KUfFldl_`;rc0EwEvG_ zo!`@$?CzJ*$GV>3c33}Q*+tTynuW`+crWl?>l8J__cu?i=OQ*2^Awi8eq!>C>D={a zDs}2IU*GPOP1bb2Zms5CL3ixDMBG_>fWv16@tHi1bgD-gdE=iu-bwLJ%NJ2X?`JR7 zVa>qQ2~n-Y_u5ZX>u!m3EN64IIIN(#F82TNRXk5v%{TrlsYhbpu`u64dgI6z+J62L zu6S;dKAvKszt^Vmhxdi`$ENwnjAcjLSQJCIdY9u%fzOEf6?$>c{~Tn| z>N%Ngr|^gt`E}9EJaj2X34LMBR4y`Zh!xi(bI0awv(r)?){XNc2{`9`21_s#6@@F<+ezI%KFK2&l zN4t-@YSb-FeO^p9>d{EAcfH9UE$vm+qx;F>QBH~9ku~~1_APZ~4u?f8l>^!o&}WnU zW#E>E^2d^B^4{}~PyU)fm`5%jT$(GobfFX7u+~mDTRla3kmrGeY(0pKDU>>=>%pCOR z7PcoS&-E6%(~g|VG4miLmhqs&+lFz~yfY-mho6dRR3d%6L=V%f`BgOfgr2S5EbcY< zXX=nqe{{WFZN21HcD-vvg2KJ&iszhlfxEU6bIezQ^2%m?^DBcF8>WPt-k@&pKJxrI zg&BRvUD`C2&b4P-oo-g)hTYE!i&H~=(v(eesadES)a zbfejDMm_Q0M>W_#sH*ZkigkocdWHr zhieUNn^t*vbN;;O2sh6j$QeJ+aZ-sA`q`(g^lkBaRjQJ$9=IXIV4$2DyPT)r>8NMk zYovR9YN?8s+bV0F$|+wZ+Hm*HMRfPgOAUUCz1EZ5ykCS^FgK5>b+Q(6W8g5|UelZ> zyq~E9{~i_9kC&kz4VLPSZpS&aZV9fI_JN|Sr1IH+7wE^USPt0KL3iD-S4Z8)yXh`n zr1zmJ(u_%;dVj^Ogv*+2e~EP%S#Ql$(w<;-}5h z80V2a+85HP15-p!uUb0WGiRB-9=_lFQUrX^fqHCNz^C>U;{|i)5yoG1So4e$#y6BD zM?U8>`|Pu7ty7NNx^U%cS=e&&*EbbzagZVRQ)zf}H!gCmq#n2|K%KO&tZ^Um@%t8h z(mu&|(KRpj*>_8sI`*>vBZf4oi(U?wxEK7aSz6@H23()Ax2w)x95!O4ZrGy0H$ zSv+cSIjtL zWWHN#>A}zC!tHDTzSqMZDzRarN_s>3*mPmq(H-ac*iJvWcQo|%pEiUVr?9FIDZ_52 zru@|mwgFq*$>rn(%y&Eix9Ci-eAMx4FKO=c&qH5IOBq2J7x|=VV{I_`cu+%q)HzJW zX0G8G4Q5$xe(EVB!zBm%ISExbTHE`?@~`{nECGA$aQ*KH_$)9+9Pj>KtQb9*F70rW zBV$|Zzg54{WJ<#QcM>tzD_doj-&2}nR80}jA!8Sn+yz5+dLd(->+54WoCK( zs(uI6c=HT$xmror54tZtKWzd`_`)_d9`XD5VmhRbC)Lw&3``^ZOpSfL8Q%?U;=z7L zIBsGm%6F@j#$0o3@GDh#%wKA_`xsBY@s~pC>=I7l6{Y$6qfN7D$ooCOq-F}dO2H2^ zNe4K=g| zHMQx_pQ6vQQ_T&$YeY%;7W*p#=lHb45({t$8Nmb^kuXP{?zx<*j`SkKDd1@WHjp_s zxSLk zA2=Jx$p>4a-r~9a)@t%jt7BRB{@0?icqBh;!>O{9%+^9Pt>Wsl$6;`zm zwaQr={OzD>4ZKLuW4z~>lkS1_{ZDN83y#}C(2iMi^Q%!PR%G#F=qH0cW-ir*?7O%P z*1hSpBRp$drpPs7z>g>ft}(QZ81T9qPpdy4wO&?2H_06?7U0SrntJP{GFrrNRgVun z1jdew5C2X0UAp=!T@8ABlFNBi!gFn2^23T4ORII$H2R4K{>;I9t<6Qu>50mtVqSi` zY$sp--kVXQDhJkeDw;WuCqJ!88%A6aDacnkv>c(T7hAT=G-8goGrng40%9K&JO8FN}l97nX?FxdKFSwGBYI2E`QXu16T0WYe$Jq>^72LY=@ z@yXR0eISjliJ!JXjBU8hXjq!iB!@0OdAq{c$mBD*v_6czppN3bd4I(QJBkmVYB)?C zu`cG{KAO)u&f*cs6+7A%q}b|iG;CFI?zU;I==|NE_YV-dlmAV+wrL-9_9ZdXn}vPG zKmojvz@xNNfurk`8^4Xx^G8{y-WsC&`EOKw$dTsNUP%Mi&%+qBr?0pE8LZ&oy6q^v z|004eveYVYj!KRGkAYVLT!XnRO3+MMeVA)*5~sLsq;+MNi0N@T<&v^3Rgb^3@)NsmA`VcZJlUe{v#sE zc^J8R+@aV^56fA{>~i>`2fS6^HJVr&T?ws3#?v%OdSy-Te2>79YW&+ma+c3#XiZ}1 zJ$=xz4lf@$O=PbXz?FuzBjB6yGX^7lQJ?CZ&2ek{)xS`GL2Nh*oS02DHBuj)-L2LO~1cvwMdLe$SvV5siFNTDjM8TLa!Lk;E9$4 z!gUt*r?r_3{k4zl2lbMt|2$GP$Id6W8`(5)9({FE;Jm`+>K2CPA?PR?-`JJGTSk-9 zmJ`+Ghp78%S9&79UgDRFZq z^oeRxT{Y`tE zuF;PX|5tFulxeRSWn`i=s4 zC5{fTqt0DtsX>kX`TqJf3R;{DzXvU!WZ9?c5w5vzi_WLj$?nMv4XnI-d==YH=Vy39 zf%#RyECIbv$SaJFF`AL2a~TG{8NMRt$hj2uDBc~{V;6ujzN_ZlXT*^kC=;ou52WaRr)`w@Y$K`?&T3=w?4X!bc&A2D( z@i&^0H;8f`TNpjUZU3CNqHn~lZDDHm4ZOcsHiUq4z{u^?BxxuQUtLf}%?nXQgRfC) zf$Efhb|G1RPz;T0R2ur&PO9#`E$~vp_VyVv8VGgFr+TatRf;M`O~?Y5F8QLL_1EWE z%lzL89vBeGe|@)*@nlXnw}R86sLz_(()hyvIwlF+J2peFrZP61PR(ELUfq1iJaf+3 zqmZX)w;Lf;?N~Cw=abRnlcM*FZ3Q!h!IHMv*E^e>N99W;D0mrqvXZj6-1y18{Q@Yc zt*5}Bs(X`RxULWtJyU4vUlW?`F#n!^h{wNO%!|+VvtYbs_~ZO)QAmm~8OEC6{mS@w z!xv}q-Sof58w?*9{SAFQ3L5LS1s_=0S%*|H8}KPSuSulR7p;-Hnn-8T6Q_4&;#^lsYL-3c||2x>m~n@1Wqwp z%L+X$hCO-9@U_TeM({Z7LjkO&W*zSe^aMkDB5T{s-xilO_-gvf`nJv#LbgO3F4{A6 zH^Zx&Iky}@=3Fgpv?x4dr*-Vo4dkCvPQQ7r1;z)@(;u{C{A_J$*0BeNFI=Ea@3-vI zfXw-j35eYz3(FaI{mF24v9f!uMniV)nw{Y})T?iyn4@YuQJ8=I_!~~NIW!- z`qY~)pivD!^PATa*!r8C@8o6p1k9&^9?vb4gF^{8Zh>at_YdE&@x8m!OB>H>fhXX} zH6|+Pc0yiB$RY)Fy0vL)Ecy01PxHMBXk-Yuw*$sYDb%wCpXdA+`fKEwl=JR>E-2>cgVhq!HraUOiP7hF*sNu>#`b6gI=Q- zYZeIMfxsCB?7X@2(cLmqA&@Sg9#9_?#% z!t5G;bY@FO>!VHCk#QyxbgWn~t&8#AggnQxZG2urE+J-k9aqK9i!42iIu{KF+gsp^ z4d?3B*#E<;a1E;L+QfJm1x!=$%Uq|3JEQmZLiZa_=nKrCs4JHR>O|zJmToz*tENB> zM6)t})8KeT!&k=&!*R&rL}RQu9kl$1LM|hK18To>D!4G&9FKfcoALMhYoVdLNaPh( z)FuH_HF}kyGYOgue&dNS<9X3`Gu>bBLH)j!7ES*9C`^8sp5MvjFZBF#DYFbjB1CZfY-pr4Mz@m5!IACBm}+GO^que-UPzWHp^0<<2knxds*I=y zKUAASXTB52_$YHrBGz9Ts*n#UlVO>h4*bE$RN!fDk$R>!HB{aB2^YtdEer zBD<}}$jz}{0KOwDD#&EPKbPj>ox^Gqa1|Nu62pcT45`=WWUJuD5cWwMDk~4xwo!pwG(><)rqTc--daJbheeycJ62{`D_)AA-oqBtogH`SbGMyog|YtBnO7<7Ic26yeKS(l zpHhW9d@rkKtNi4D$rWkwY)46F-Szuq54@w0SF>+R>0kIWKMY<#{d(rt(OrMh^|!h8 zFqbF}Ivq(hBg3fTrCh$t=eH*Jd~bO!bfGdCPat|Fw zksB+^kmh;xo#;5Vr`I)#8Tm#v`PD$)@Wa|v<1*<alXo+x&9(CO z`-iNvU%KePU#|r2h1Z-}r)#|O)%~QKoWI>w?)RFbuihxaWsVQke}*~g+CGKoP4Syt zZ&fK%2iAfsZ(Bc2FK@<0J$GHsPW$?C$iNIf-gJ_jt5#CZrZeS`+Y`BJ4-bidlc#H} z=K0%~(f!axD!O|`-TW;pj0FYdX|DH{Tq;qg%6sSq<%Tt|7WZEwdo1g%YYvFw~8@`R-{EgPOZ%S#?D_(_5$iUl)+;zk>dHwAiYUa6x-3z$Ojjhu7 zJ>D;$+`5$X_#T75`|9@}=9B%E<*IlKThTc2A-&9lXG@pOGqokrAAECFQ3|ZzS=8f~ z)+&cPuuIi4+QV%QSFVssriN{L+3UD}wdIkto(f~jg}zuRXZFQn0QLndTCyaq9@7M0 z^Pf{4%Uz@r_dkiC7ngYN9}hF9+@@MVj7u?jGH57I?Kyzgocv1H0_$_vYB*c9c`{{` z9pgGJd)7BnfgTIRdG_HkOUm$*W>_<(-F`j#ceof|Mv~Y2$>eV-LI|83p^aB^{jJ3EMK2pOjm!bxrE-O9^h(iPjgrUcKpT?DgO-cT$O<~5^;z}ic070g+n8q5E~-o9M8;l! zmMn1KGPlW7oC@4;uJdop%Ma)~IS(o&Fc*4fosXjCrq&#M$WET5(i;6Q-&)tJ%`?W) zxzcXByN88({`Z9%6h1}o{yV~dhRr8-3(*Pr@|&^Za}E1aW>Rq)d1RFIz4npr_g=7xt!`GhU(QG;oAMyS8QY9X*L`M_0f1qUQ5v(~#TWDZ#O?jDNCK z{Aje9${c^CN*t~%%Q$4JV^|~WL}Cv0b5ee}_f}=;;E-LqdpUE^iCYw1lnkfnsSQ>s z+BA_&9inH{<;szI=%jRIc&bh10kUnQa6Ucij^$-QHd@fUj@)+Uymfq&-CWDhRc>wj zlMQBXNLj&82G8N2w~x`CU3pab_|nIZ9H<_ zCXHS+yhstP{l(5YV+hzxThelqhg)v_XI=;V?4v@jh-f@tjQZ7PY;JX_Cf(AQfKmEK z?+>EH{jnO?Er5II)4KBc*V1hJa-77qav|ReEdA1X(xL^xj!1g{_8uiI#`^V*D(dn5 zepAX77HLh}NMI;GK6`*0j@TsEf9%RD(!HhO70ioeR*nSj?qTKW9Ttkg9#80}H9+G& z+2Dei`vapqb@Q`L>HMEy!yDq?^gyYv6{6mQpQ&+kN@~=Q1YVnK8`YG+8G36iOJ_GV5h|f7T`VwITuBOJ z$JGOiseNf!7vMxV{r3UiBUefX7eo{4Lk-Thk~gLIGFVD|r{ADmtsYx2?$rLj(;R-a ztf>c{T&0RIdSph;*HrJp0s``-eGRB;G>nRC>>;FrOgk~xvD(=Wn z#Vh1B{f_z1r=rRqQaCpQ$2GK>C~*HI_5YD9MyD2~remL5!3*G~{Cv7b2`Ze@MjzdQ z90k30>e@ju?%X`W{n2xuKIj4LYrDXWz?FQt!Yy9(?HII5MQz3v3lHYEwlaHAi`W8t4D!DxNk@< zdL7ENyj7Ge7_^FC*$?6&_sYpvM}Kk2ggxM+Sj#}CHC$!dVCB@lhJG0kLKr7=E!C{tw9@V+- zuc865t$5Js-F&<7;j{tm9OVxWPYvzE=l^^a?)gS=a@k}Cz7ce(xM>qWs88#_*IRg9 zXDukHobI*LMs8SG&a%1JRt8V;iG=IG-0Qskz-a}2$sZrvYiK~|?2A00<`ROwqyg|F zYFZsiyfa9Q{+&}>50@rUTbtwR#U^~D5E=vC;wH2 zzF0JLE3Mg*Y^~j8F0cAoTu$v1n+DA%ZR*7sUB+2j5%Xs}Msm|Kvh$DZ>RQ}Z?4$mb zDlLfTJWuW#PG@kSW&Xz;^6^_QdQ-nGm9E-G_6><3XfM&ux{W7v-2g2xk{0d0LEGl$ zW5a9EyGGv%V1Nq7`?KH);o^|2pd+|#?>2P%(`i0+tv;PSrKmy19LD&`bEV_ar!~c- z-J!y0>}EB7Q@i~$*sjh6sy*=vq5pYa2aAM{qSnzTt*?$wQ3j`-#!Tcsj|GVufim7M zF%5cxcOOsnjd0AXc1)Ycp;M0&^a->@6KKIqstHeeosZI>x|F7+(io*PGJs(~DjEfK@-~)d}pkGJ8J(dlmRneS93rtG>M# z1*gUGlxNuA-ESE;-jL6F^WZ{CtB_Ly7sZ2Lsr01WDr!-#953x0Lr2(A_gDel1iS)f zLPI-qsgy}N_|RR%g;tLa7dFL;InB8 zpwpQQXByDq0OxyK*EB-as zQctqL2c|i--(fzZ%9T9L(4PctPgfFa$f7@gQH744^x#)rHZ@o+E|P#7y8guJV!|Y> z!F#(jeLHQFr7JP-+~vBXL~l{a1TO^-MMvMCP|&|z1baA+$n7A{*_4&g;d;`@rQEq& zUET6QDvx~^!Uq=qvKUP?@O32#pFjhhJPe+}+d8qq&R!Qz(V17Z;Dy%^w4qvGD3B@* z`o@?~t}=6p0v=I$Ud*|N`!KW{W8U~t-s$jhNp$g30-nLTr{M9pPp)0m!J6_H zx@5N)FMR6zUV^t1ALbNseKCSzSFx%aJ>vV z5k$WWRyX}(v<*!S%PozsgC`g3I!j%~p)(zP6+zOkG5g*~phe%BP?ec3)-JJY)XMHx z)PQlb1UQNRL}fSrP8to5z9FOKQ2Sy)>=^;fCzCCFjUGxDCZ6G;-krEXMsAK76K{M4 zj~`e`bUYHzzz=aKd_Cc3%J_QJoPusAcs?#1lT(gq=VO5{Am|#^W7z|Nx&fy~GdNBR zI$fK(VE;65v6&Ns_u}{`RcKGcywYgN56uq1v$|^VR2GlVY0zB5+tEAEvlLT(yW;N- za%|0pw53>mJvdu2eV`3n6C58YWD#^?W|T1FYdj2e1fSh>jGvZ%FW~LqQy*KQ<&2M} zKT)%MQTxU>qtE;D>RJ(WdaMH(e`kF1QfEMMqaiVar7*Ah6_c;^3Ey7IUjo9-{76r~WAC?Z4@QTLodM1+WvB}+nfB5Srt z$Qlts%F=>_Qr&YVLUt)4YmY6mZ`mS#XZpQ=yr1_$UH4ow=X|&Ox@MrDFj^f zC(TP5?1;$gcUY+WM)%qgXg*uZ*3cesSGKRvlBNa6=FKJLM@r)`^29>vK;l5Aa+$T^ zogt>n9(p$(J z|1V`VJ5l+;{F3%uVLo9R&|Hb=iDpQAB45_9Bi%fOQLcbW&yjAE4s*AXCLMuP3)=Oa z+;L)tn^0r;Sic6R+Cl0`yjW9%mB+4W5)U>~*%EXvJfR)*x|LM^g}4_~Hk9_wNsfOZ zkaZ|jdQ4UuJqjszB8_0l2x~CLcOp~s(7-hjRIido!Emz)U|talgg>a8oFPb8kWZXL zJXRZRwx$8$tO&Loq;dlw+=Sd_JxS9#$xWrzID0z*NDtE-hPe_SvvBh$gp@-{m*I0K z8>N{r;c*%DX#Xnjes`|3RJqyqf2W|Jp&8$MFA=ix50js6$cUf#q12h=NqdmSFv2Y( zLxH%5*?gObi#^McvKrDpW!WXu<}lJAO!30VL7kO8rFY=^fXXMRR;n1e2DRi==KDlo9=6l&PS7 zLLFM~o#ZEKio9u*8QDgP?}Ltt!i`c4tw--qd6sL(eLllTJ0bgEBA#?_$w{vxTIjGh z1AM5z+Cdln-GcO+d~dx5Xuh>7|Dm}R=H1G0(xCqo@2Ol(DEwAAK}p+AoHP*d61?zoh1a zt+;U-RqpUD!-y|8Ie-Vh>A|8+3(vvc*y5DS)yar2&*qmw#JZQ~rCC?jsBv zuk=CcEkQi)nq1LWt*tnkJe;&+GbkE!UQ>2u*5}q`~nXl?|!}vLrotYucAE<1WastKoRsI_ie=0rT`ais8o8zMe^*5bOKb>^% z=_<}5f5P1Z4v3kpZ)Yz$RfG08c%}RV^xsQoO%O&CKajqcRa%*P0Vo?0TW8Hs{){Ou zq-<0WUJ2q^*7>U)S2~0+Ow%XRhrKR33JYC20R6vsZ>>jsA53{|HQIag4fCJSik>~+ znVnI&AAUwOP&yT-mp76LIdNiK{nJR9H0soJ)|;k$|NB*#}jhh zva0^PXY^3vpJa*-b4^`|6E&253evSeJ)m?Ys&yhito*rFX%otTs&pLTvm|ZJMtt{G zbtJacae>D(n^EpHO=g;I(@+-aN;y1t+p?FQ&vB5-$51YWq=A5VLl6#&*!VN-mH%oa zE=QVIO%8O$9$&g~vvFhiJmW>0>puI?>qi*BvUDQk)J#K%{u6Qe>MLNkD;cbo^^&E9 zJJ9}fG=@?}CJGZV?SPexi@y(7-hO3He>GtHawr$;p5k?%ro7;LXTH3V5Gx;dmLFFP zg`3Z}@Yr|zG!Z78jlFPyySu(+W=XcVFEk2{1y^Vv%zOh0i+_m2Ycrwr_HTRCL9FoJq&@?V&T5DeBWhrA_Hoqxa2WkZ zG|*Wu31nsIH;Dk!$=29t6Q;m&JXHgz0ArPpTmuKIrPf4dqy9oJ%PRA=sgIuK`g znrHVgOTgvH?ZEg}P3an$ilzGdJI;0={%wO%KK*ngUm?>{46V6Fcb6okX6d`Z`J6!t> zl=N=Fa)d5=ba@Br{v&P1bKrEok83}OH4W0t2;0vxCs%@1Wj&E@<1Ov& z7RT9qrDw11K}n@Qi|Xzn4e5QG%E2L=>W=jqozF~bZ9?-aYcct29~neiD72fEEV;i3 zyPWzajDOyc-s4R$aFP~gx{idLFY9GN?fY1k-$({u9w2i>IMDq_|1FkSOwUt)4?2Oy zMQ_B;#A5dGpEU+qeuGtZPtY`QG}nKd2n)}7@l#IqnBkOBV%?P1+|>Le2A1k^v;66( zYD?A0DW@&s`{Fw|Y0W-P<7b%_Gars-GR+--l&R+R1L$ z8mXaE67XF^cfRDzAk=AC8=ZT`(Q zbGdH+aYk#2J;vPt!XQ>QZXK9^94vGn2jP(RA4TkCR&`(cmo8}SbzPyX;EzU)+w@sm z)!#0rA$yq4fO`Dn{&c6k!`I_|YsPKA8d3%T-pW(R+;_?tZI_t?^SuRLpMcsZ;;_Had^t)@oxXB58 zSKV4>_H;&de}~%-#HsmL(RpKg>CvJu_a6~0t}hIO^v(aF{uT}uw|elXV|PUQv?Y@I z86r$1>(QnUKes9nm+x51V_F?z=RSXC#y}n%(j#!4bPbZ9G-okk#(4c#CsA_h1SkwkwR z(2y6eOwyzq9TOh!&QeVmXKyKV;fB5Hix+|VSTXb~Q#uaeTq)an^He* zp)Agp>gX3Zd}zpLWFDirudJueY3K$OS z%ANX7g7j18wA8=Ccjq!e^B`}}S%u%v#cIAT^+Bo+Os+Les&!3Y!oX}?3E2MFDya5_ zfAhNFG`%0Ey2RKPhqdOO43rOv8WtUN%fI-Zo%l7IbUBjc8S z$Kp?2kT4pL+1&)XFRA)`N$pVnaik%i@i7%DcWOl1>0Ahm+zWJl*W|YOXr6V2m0I?s z`Fk#$dk4w9gC6ubln-=`p=aOPLnb|kI_}g2N&UzO`?Z0QR*>+^U3kXrfCK~T;rqX; zdacB04-y|RozI8h`r?;Bn1xsVoMf>rTA|^caBh}#5Qn>@G4JtFJUYfn^)|P9SBMc? zXQ6M~*{HLsk@)^AK^t-U7WnQkm((M2OHKpAo^-XgGPhMU$?bUX?N6OEv zpzy%BN3@z3KBWCDTzEVME7opD;u9=b+e#1zu+nt}jMhpJ z=8BTu2Qq_~ z4m_`mE4R*@1%we|VaL%xoQ1&$UyEb?PcqUmtaQi~cKNjtCbPOsch-G$IvpZ}eg|%~ znfA8(Xr;~lxPh(oU&&@1sv$|+X-*ZlbRnH5{STbc+&sFD#r-)ibp12LmYirH4S)q_ z+H3RZxdFxd)Du$ilymVt>ifAYB(5gEog4{i(|3qd`}d)*ZjvaNa31siIg%cva|fe4 zaqs8nu~*j!pwHBjZmiOZgbREXJvU_<_!tShh&RUzXQy6ZdNYE5FMEy-+m0x{1_vE& z)jCl>!S!=T1=ST?nd&CXpcM}-FyhW9>Ir-Ng4EmSd&8Qvhc2V~gec1w#G4l(IXRX% zEei5R?m;un4tlxxBTy|O&5K-K>yIWgtru9|O=TYMdMMq72@AJM>Lc;>?RYF6lLzNq zv$B=;m~=!3-qAaO-rZ7#ot}$Sy{SKwaNkQ*zOqYj<37EL*R$+Wk^ z)2{$MGUz+uzZywel6!QSj^BsP!JVr@U|!xpB>oemH)Zl_7fxKn-i`f;SK|Ak(tP~b z0vuZwp>!H9zp)w9oHw%}XT~9EVrKip6qesG=AQ2D)cx|YQ?k)D)dYM$)4MLGdV=qq zE-YuU165O5{NciftO!z(%MPd6QObj5OmyU0mMsU-33!gbp@G|mm>Wx=Dn*+7`6~i*PMll z*lb}Qb`D81$zJsjK=RHBhwD2tqiLHd&Z8_K# zq|tYlw*2Vh1Mu$iIQVGVTiUZ8+Ju+;VfhYyPPNDq=pAU{F$`>4E`;T6pq|5bV`o9y zX*U#>Lowp-P9$Ctx1U4-c^U|Pev&Q60`Tv9M{6}Ljs<({)r7*jtC2-n!xJPS!4M?+Wugg4^*`UXI{2_vd4tJ2_v zJIcF(Qy(3!G#_DZHqEKHd~yxVtE=K6ZI3TG=(@l_=}*#$uaw7w;CCHJZyW;BotV?6 zK991TAa(tn<>A^>QR$1@Uo#ZW%2nHEveF-8T)+N_7sip@>6wJd!hcIFe(chklTSj@ zYTD?OyV&cRKO?@Dd0nz0uSG+RZ`)7o`!kji_=w$ZE8%;PBcpmkkIRoCsQW!ZHH*n}8c2JbQ^kQ56Hanf z8?pb+X{pZPim{*K(kecx3@L_lv(I4czn(Co{#HqNjHMH4kLvPm5J%rvkayvvNyz7| zW~74!aW840i$dv4+dWByZ==DXe<>r~#}v0MJeQB858%quQY~r!Y|={1_V#i3c-g$_ zehL4<4CtA);Me$ozB`N9`cd&AChU8l>K9{2w&#j_6n8&(p9!>P^6S(z{OVVSZ&_29 zAL>>|l0Jj4BZ9Su^j0&a3G7$ZlGd+pWs|1|;tQsHeO2GD7+U2~Rb3U7enZ~VghaTz1M$>`7W z>D>yL8s3cCKk)_UVXaW<8{#vr+tPtYM|{QDde!MXD{C2Qp;fv7%xV}5h1(H*F96jm z_R7jq83Jkk#+d%5pY#mvB+1M3?^d;NxK5iYEv|6%a_9=Qx>%bl{B>Sj8<%^X6Lfvy zJ&$V6+lVcReMk2+Ly#7cW1XC^vQ|Gvx>8I{2_jzcBD}o}9`z0i@@$;=gYdg4c}{)O zR~;c|{6Fo{S=*8Lm3VL&#AeWboox=PIS|B^?Bhc&v^h24YkXJqe4R^w*IjA@cR|iL z3((DMCY5#!Y_gfo+AR_LtnDaQDPal=$vaDRk6sSzs{H-C*$!YndMprs2*N3sg}3WU z#SejL6S2Z#4em_z1JX?@uYk(I<%~2SX{MTzyu0#YP&W4%sJc|Wzh!U)kennqHuFwcq=K-wJ%Kk2>_fjlixjdG=L2#etRIZM1di@tX+ z|0HPLSwywJNID)O4mfbvo|DDL7U8&jPD`Z;1!*?9uWk)anw)gvVCuOuB6qYURD3_6 zQGK@Xcw5EmOs$u~RB!JgLg7qwO*f$YL8D7&Y0)+D*!I!POk76tn&{eN75msi7fBZY zagp)_IOAG>B;CiT#@W*}C!9KPCoXS0772IYyU#{u>oXp%AJ^xmM-CwARc1@iM^y%N zg2(#_Hx5=Peqe^Z7pj~9wiFWQFFTKvGXQB{rZNrNg}sGcL`^Wx?ZXq+cR`00deXt) zR#J_qI%1^z73U}q0@^@xBpigi6)!L(Gyw?%A#Uk>q>P7p{Q=#l2|s$c2~y@ndDJTH zxzE86G2M<6R=|Vls~K^h!aEI}_l-w&^rUYx2l7aoiZe|l@fi}%?fUMDo8)DwB;IpYJXEn*&yH2?f~^My$AHfL@14-G$F0MM)yN2 z=}O8?9yTMI_G|bSiICF;$eU=WW^pBLbzu4|h3Tw{ zLBeSy-bBh4S*W)gBfi#N=|mZ&TMm%71Uq|6seJ50pR=&BXCLnEeM>|0BWEU-q4N~_ zKL71poEjd-4q3Fxre0#C1sPpmGj@tA%Upzl=2%1AD(zQAkY?$nvH>7`smhp0<7o+} z7-2qK_S-8;YHp#^=$d(_c{6 z^YE}IzdQ)7>z)Sk+ZxIbvxD!i7fMeP_6X`}* zWs|L>wTGKWYXf;bMwuY!_x+*PR&LyzMp}JiSHfrN>qw+*mukxlot%40r2)wAYo8W1 zR62$pMqPl0bvB@#+@SsOsk+>AZvbyTJsF10I)Rq;7vo*O#=NVz2#a5vuxFQ70UKgK z=dPG==}CLqH28}xdauWa1J1%9?Mm^x!)VzmaS_h)-KzC2ptCWi_Tfvub>LGs?*TX3 z8gzRtR}A_&T&~<{Li+%~M3*Tm1YHLP?b?RFyn^u1>1A;JT00)TGzyZMTfjg} zWXG?pmamNG@aB`_L}65O82h^ktRHlU_r11GbkMGV%!M`Oxq8MjwdWgbJ7pq2xSrk# zAN2xy*KR5gO>kjXo+U7=D;_X%6`f0v_zO06jHOr8((%@-FEpNe=w{u3*WBKg=iffd z7U-Ia@Pi%XvTJ#F`ik^XsTUv9vIH{8Pk z>D$EGNtWFE#}BMIvySv^`2u(UxC?fpJ>d-WADu6s%rz;#72Rz zVVNJKema1gH~rJLY!xO?q^{PQ8qqT(=Y}&?Ljm{JvON*C@KY?E5lC}AbZl*&YB>O_ zuU?7$zMsGsMe&TTi5W4*a)Itv*`l{0#JGL~hx?K8bX*I@S6>t-s$GXWL+Q-ffR55W z_!riCTY;Ge+SBu9Hc~fgzxX))2o|KDMymy3nB3|hD~>F}lofy3SBFNj)L{wN8?;i} zJZBcy%-e@IANufZe-FZZ`!uZn@Ro)?AF{(+NOo|rq<_P)bUI)lD|6~#-)psbd3X?=^=Tq&&fK8-73Wu53s-8p@!!KjVdeoZ z8U3&xrykYN{lbldo2q`sAmTFK59*<%HAdUGQlwgj9!(-~lyjn7`YRi(7U=Mu7c8)s zjsdq`e+r)$Gy>3E1B`&%T=FCJp!8|-a^!@dEj_1n*DxE zd@9$nBO}Hk^&a*KTnMvPXkfvxdw6c$SC`CuUuhdRk()jb!|37&ew;EEr??e#HnF3m z`DW1%x2T^(3z>{%JL|m$lED`be5D8RI%n25%TnSt9M-H~l?fnw@+ws|Y7HdX8HwcH=#u_Zlv} zz;6LVHU1Mu@L@Y{vLZTHxNXl%wANbjy?!IOb=C);^`0a9xC{p8yB8t1(*d|y>~ z^Wp8!-{NPVq3|g53tHxOf*En{yve>e*k$n+lNU~<-q|c4SKkED#kE-bAs>P2TKnon zKe}gQJ|vk5s%O9%?b)m&;{{vl#PQ5moU&#uqdJiFUm421n|6cE>4#eSY&3D~hJ@R4 zMW!X3>)IY%Ue1Is-uC>(u~ovvz?xtC&;y-5U&NW-YoT>OA)XzXt1*490|9MsLdBhx zkW!-_JhW-dEA0A7g#%7klW?S?6>s1+32036XL%jCFm(#HY`dNxZ#E1=`gFv$mdWsD z!7AIv#sxSKqNH~82L z!=m+|-JhnMYE=+F$f0A`(Vh-_HQy&b7KADIVE!Y1>rN+RC#@mC$SPOnL@v53_G}ioPTI3K{Rm!rqK5Y+U0O1l-ey?7Oc;ac5mQ zIklC-Z1y?i3~cms<@7&k;+TVUPij8#%#ilD!)Ty}cnXPUK)WFm@0;r)VKUxaw+Yo} z5jJzZ`TK!-kgIC10VY3|ry6b`-j-j#TT_$5hu^<_Ij+^Mxn zNCx^WuwIjla_1d99d|%HwYUlocQ27w>Rw|L^Y$>R?dVs==14%W=a>ra_dm^{M{=9Kj{6aVe^b{^MzSoB4cQ2iP zkf%MjZZ1Tm4J5o?2H(4#QS~AZo&F0nX7=g@JxlZYJ7m1R12legn=_ITR*LNgy*Oc> zX4A9do__v_M`jSg6^O$w3@;)+j>U#Y2T6Cze0QkqH7 zn#tyqn@O5u-j3c)_@>E#Ej`X>$IaS;M+3@*ol`U1Hux=gopr4G8Coa${7AU#w-ak! z@P}{48#NmaeFMT@xbJ0i5vO}lRtl)q&cJidLq&mXs=kDQ@^`_EdYnX<5St=dW_=F)R z{$i0A{;}7ef>2#^zFjLx*F?YFBjkq(F_7Yr1vK9{r&|*s?1X;b9c2f7&S=iz=IU_x zUOxs51Ji+YBc6Or`%*WIr9N6ny*2^wxfsbsUA=fgtxERG>lJ=kctz-Db(0^pudwuy zGf?e`u>+s8q4#EBaiovXeg2XuEuhBbJIzked~1mJ6i$NY?Y){Fy;}1>`e!u6!8kC> zN|Lsa24hncXW+VCdhny?6;PUSiS=-WYe2Qi=$@E;c_gGZI){5|%!3O1+o*H98+%b? z!(vSRsxW6Qz9o(J80$>yi@k;x0F4_u)+?lYJ%Jr_BJt5+PeHhh5Br{GyDX+b+>TUX zxw|1Y{MrUP*7KH%<1Y=Ji}Qzg$q{X8tA3_FuFk1PBxzirxnr`=X5!>h&6Br{VCgMO zZavOWOTDI@)a*xA=C`k6>uSTEmBtmS&dqE5Ky4)@Otm+nk}&7sOHIuXZ^1|JC77T#LkLw>+q~d@<0m zCh+%tj_a=YT5`v8TdC%!d-tY1`|cD`klsboza!~n_A1*310I>mg=3~y`LE|coFVz& zYE=G-`iyj9j0o&9L(+9b+eUG)#A7~g|8F;3=+$41Qybi6EgsR^4dF{oIQ1_)oI3^; zKLkpBUbOfqDDE$*)tDD%oyGfG%UHx1Jzo9c5S)1+A4vb8;)3katHu7jCsq1`)<~_l z)E~Nm9dfM)I!CS1cWZA(c!@9dN{BChX^3xGlRPO2f0={s|IZ`&0hp$B=X8Cjc5nmn zkrmZ{7xJfjAi$TrIAM+Q)4L0@nB532iM!DH@ggiMyAAC;M+rN_Fi>+%-iG(?d=Uw= zfV4W0j#4})iz7V6mAd__v|}q@I>+1pAg=oI7Wx<4aniR)+7FkXGLa?zWm?s@%XIP? zVYxIb*v`nmz@U%MwOvXc;MVV3u&&hxMsqA^yoBL%a7}tK+V}BB!eVH>_9Ku_SAJVi zpNlVT>vG*FYb5;v$IIKRdJ^O(HRNHmrHv?uh=M^E7|r_yoYZUqt@{q`-Q*IOaJ! zV?wa?+F5w1^)*J?h$&7d9jxY`P0Pt=ho=v~*2e3_nsh&|FjeVj<;Rt0U)Fj$m^FVS zHk(ATgyk+(W3&%m1d2b0?HGilF<>=iL@P35g&O;*)4Le)8Y(SBnFG|O=b;pzHoU$b zw%WJ?`BCjvSV|hg7ZoQe%vWoW;ybJg7xMa#6_5Vbz*>8|f%cRytWL^C?73RJU!Q@* z$=bNxF7Skpfdw5jaOaQ@+BfS2G?z#@2kg3R3&eX$Q!&+-& zvN`LE_Yp?i4dILQR9*x9A3AdCCBD5@HfZKGt-?~pf6B*rG(_U^Ug)z)KsOc*GeVjG zBt3xi(*DZ3$*&GEl)bd1XJGGZyVP8SYVJp^UjWsM`|`~EJ3#&sybJD&^lddc-5>dF zTTXe4@~2SkW;#&4Dx4IFy&cHET+LRVyyn?kqAb_B${&ztW;o**kbYF$r#!97VW|Ed z;|g&I2qURyTjO@e04yKTo$hH3`Lwx8Be1TSD2RWs1MO%@dK#`Z91HY2%<=U}K^X`m z?;`yA*4L80AYD+M(OM}VuI+pawZwx!nhJ<}l=cFZanTw9>0w6e#oG^|vzpEJLY{S| z%8&5X)>Ww1ROMaumRE?~&6gu-J>_A*Hbg?uVN==nOQLvSMSk#VYq;=Pmxo@}Q+y=S z{^T;si;y%B_472Q)`YwSbGUCnSwR!|&c6mHy{U8p+YGLvkI6~Uu5AzHqptvAr^*@_ zd3*9VC5$i{)!*$o;eaED3)D$6ljEOMtr^?%=q~=4@mcV+50oYAOUf}+z5v7>SZ-KD ze4jX4aS)6ecN4OY_vc0NSFuKVXT19ID8R62I@|aNM)tR)T&xFB9ihnlk8+!PTBnOm zrRw?dPxb)$0(hI!T@s%0H#crmt&CJYG<)eUSA2GTBkY?N0bNcxNz(F?`V1Smy#&(W zs5tlTD0+9J_;M8vo~rW(o_?gWAQ#p|;sp(92OvM9CCr1lJr9esVgCu0|EbJy)&7>U zXnu31c#ODG`>l&TT)G=Bu3OF3ls?K}c7{5NS6HX~lR{}%;yffjrskVb9znYPA^B5% zAn&FA%q??I0cisEaoZK>)y_#g3Z27gev}^s!U;*)jG70`I^xDv26xmrm8G?KqDdOq zlM$9H4W=Dv^g*O~E<-gZ%0E(mq5j7y^KvCWCRDCTnwp(Rtx4L6&aoS}o-OV^9w-mP z*rJ{|UQgiNm1k9M#fiJMlxwj|cXdJeAJT`yZ})vvxN>$_rcm1X#GMv!WW*DuQ}Kuq z$E#X^%6hkyw3y@&>T&w;d^8VWhJl z|7de0p8#&VehSjoDpS-@*2UfWgFN6*?^|1a)eu+UuIs7zz`Y$>mCu4eRfEK$<`20s~V#B2$wLbE7B}cK=}lyJd1L0 z-Z963J^N~}bcP_T1cf92869OE^d;p(fsFWy>WTIP-A2#p=X_*FHamdwUCOL#$3dCi)?M#`P4sn zI?SI}Og+J_q%?-!Ck}|+A)Wa@+Gl|N%qK-#!2AXJGU3NQHqdYsME)I%=Tn!6(~a85 z!k8}LGHNh>es9VfjLO00jmNS!`hoCdv8~)|Gnjn%aCsndHP)Hz!#dnqjQije{~Uyc&XaQ-ilDuBBl*K-DJFJivg_s77~I*Ke}2;r1MDaA*@>&McT>)j zF2%!Kzco0_rX3q&(h*yS_T|&qR6cR$K={XZ%Xi1NK;qRkqV8X7zIW45|cUk$IAyu|JMCc@A3K)yD_9=yGJ$()&s zA)xhgEsNiWh3)BmvUE#v*ysZU_J4^qUbTkW8*eN)t+|-FCLFTjALB6Ft=J~Mp}cJ~ z4gd4w())$6-1;GqdpRbtSe=_9p&}d$bPH&0H^>!XxuBmLE;AyHpwkHpes5zfFl7=g zT93l0QA=gF99PJ-84toI506j|{yh3VwC~>^x&2T{wSzSGe0b7g_AYDzdUVUd{H0-R z%>omt=KOLaCmDXp7;_Bj%U_Ro$mhFTO2^*=L~7JGX1f{t{A};@Y7Wy3VgHuyCihcoy z@Th?SOU}Im#X+uke~kw0d_#FrdLb_S{fYL|=pow}9anV-%SVNXvHE|t?KA$0sLxkG z)w0n~AEdeA&fPNjr4N6RdWE-eGnEZKM?y^Q0N&x=L>W7?jNJ>E!Kb=6!R@jCuyJe; z2AyldL#_|!GE>?9;=QbQQ;Ge{zG|sn+1aS4P#AL>LoL5#&xxJE>qhBm-zU1m(3uxd zZ)YJiy)z$&l$_7}@BKP#(e0?%U>QoavjQ7@IEW(yQsK!`10J70P3|x1O0`0JRu&yX zsso_@kOg~J;`ICi*Qhbi)mUXWpITsSiJCi)!zGOYa&0eHVMy=v+IXSp3M|lDhl5(r zXWK&mg4I2L-Xla$?A`PlY8qMb`N06Oy3V4;+YGU)t1<86W-D(+59W2IX=$(QBiJKk z4Ib0E2ODfIA=NJocdswI2JB&vI}ZbXfu2E?tC(ZdF6cPzI;;&jf>#<0kP46J-z4=5 zZ#tRI^|mnP@l(S^ZLemSmQI=_rz7v)zX;!2)#J~jN6>dubF*iR@s{DS^MQI)sx_Mutr9_(&CiDGRXKaPd59TjFW9LG^)P$w5=|# zbsi{h_;^WL7d8KQ*3Lnr#zL4#8SoUiynrC6r`XTFgc@9%}CZCRlYOFJQBn7hwpA;~%(dX0)G_T*`<-odl-s3p_qVEOf zm)wDhq6u=vhml~maD@EZ;1zSGz1O--SV@}izw8KacRuLCCbWBd34NkX#GaT~96Hk- zBY!XAqZ{l%lcZtbS~gX_4mt-{<0r{_M?i6dd}?qAHAeJ4daee4x&6hXAyVVPhI1M# zFnv0wIsG!0>-CjXe=I&|AMMYx7kvJ@^K~=F@bX)<`E`m>L87mpKTkhZA|A; znk~FfHStJnTlRuAGCV6DSlDp-47T6RThbiKl94UshNg7Z?&nMhYtWiEPqi1iCadVV zEPMK1ke=DeJjCui*5!L*ED0kv<9WA0v~^z0Z)>f%ey%POhNE+l4!>CLDNkE$mOC6? zQq69Vgx!LAR&M#xP`>$b4*fsV*`^m_u&K@#co+5qiYE-E{^_P_VFY16FLP_FxoP1IWfMFZ&>FTG>fzd>Er zBI!Szb14FwH(Z44bdRE4STt9?8r*q*)%^XQP|AJ>T|=4!Hly51e#-rYTP7Swi=0Gl zv%hT^^*#&i3~-ZPjm-%YaJM;-t!jWU zgsn_8;$Ihd;L@NCklooGSDN&MtD0S?z2L66?Q~w@b2h6%Gs5;}K%5GneZ~nlU48lC zU`bU!MR<(?`fnUa-xHDMV7~*Fqn+<>L0rzlFC@dan7)cvf#y%MYjHLFQ$7*vMy+8J z9=_CMX(&7SyC3H`YO|C567W4Y5l`!`hY7TA>YI(m-00f}dVjVUu68{oY}Sr}=)^`q zb&oegZZhH$_;{%|Y;;e>8$S79ZDYcTV;Sic>c=3u2JNln;w3g0(s!6gkaZJ7fiwvc zM+o9->^i24%wMV_b)3s+4;b3RqbN+|Zu-rr_85BoVpA+%D?Ub@Yw6&@=z099C*T)- z?>T+KUVL4VgrrX-jf>WLTUBqH=C{DX!XH=ddjz`;!qo4G>mzrwI(KF;g_qh(ZTU(g z|0@1-r)Ma(cWulApE}77cm8SjyA{IBHT0fe(r0%5K|=|#LuB6xhWuYjCvAbj9w0qP z*k~rE6?PHCk<81nbyaPUrUTM7(r5C*Dt$Pc_Dox+NAEZg=8lVf!A4#F%{l}(M&I9K zL=xpX#HF0@gRput4qN+6L-j3mLr=5%mx~FH-Z1)oCKsE^TKe8HFMp{Z3{`c?BLDt} zmRX||*NaZ+D}ZVdjhiMwMtLq9l-rsw(5nq0KJjb>3xH6|Ir#V3E@>6K8Hnp$RsGsK zbd+W$8_+SzT)v5HgMHi$q~d8A(gzHKBg9M3M3H^9o^1Fw2aY@)CKqQKXn$yr<84`k z$puGU>oxp@lYR%uK(7IORp&NR>A?KOv6@az(}mKj*Dsxd+TJx~^VB0O#p@IJe@kaWb*$1lTO|47V zvQcudcN^I06~sy3;Ol_-K>dSzwbf-Gw}Y_qU=sSJn}X`Q{$Z_fDVr-B$DV~2>>{#* zx{|awR9fv~guD15exXF~vBaUbnBpVSy+GsRgqfT$Tj4KZ+X!{f*x_EO=<(EAo(f7t zrA?39c9$t$v$IKe3x}tT_*A#kKs_kFEUhm$WNy=x?`^<;dR#@CTUR5SwE&KBP&Q)| zr}`wlPUn(4e-?yISVw1#cBl6ZaPHu4e?t0tO=;JBXKXf zubu6BKhm0CDDJ>Rd~A?(27ImC01j*PAT!07)0mNZT#O8GKsVjBZ1l1#81iZjIIRsL zowS!_tVsgZQ>wlRyOG8U-=fFUwX4g2wErP-uqH&zgt-^?Qw>MruEi^GbI^9S!|^&$ z-(!*IKILm5#%~vPe-S~rpNIw}aWK&EA@tgW&?&tdr}_udwMf{6i<=H-?zXg_utBPr zAJUxmoD)czn7w+uQT(u}1gd@Qw4x?j(k_DPfbZ3= zG2L$PppihujWizo&&N;s8m7iV*v@D?IO*>VOl@nZp}CRuQkz$41f{1FCS(zwttQ=E zE*@!`la83miL2oH=oLtw4}AxI@|$(5OXIb782JoFYmND@I%D?LE#kF$N7kS|o^*Ra*})X0 zdEux*q~<_mP5CY6GOT&gmMbkqnhEMPoQ?Cm?jYeg^|u3Be_O!sxV?kU?m9r)74KWE z!5$Bb6y}2RZJyzEfOv}5%9_q8eJBX);C)I8BQ57j9vBIa;B}=D8s582+G;u@T*1z% z=~bS>GUFQ~FN2j<=JI#IKOpZb?iStB&|1hdx|bm3Rdb=VG4(GHCMfJe>Iv107#yFi zG@|@_%pBJS=T+$@(vYO{UIO8}Q2snVsHxIFjQB;X^zP=OR3IFNf%|e{fKe_!(j+kQ zk22(y9(iF)jrLLs?l#a7d!jS6RF6nn8fYx!JC6YA3^)~JhnXg$!R132PPHz@8xx)y zor8mY6wWV#TKo2DOc}?lgXhtqu$+d$QFt@u@9NJ+0Mk3qaGRh_21S&Q^~1oep^{jE{n)GKxrC_PB|2>ckh zL(Fm1@G(YLHA-XByc1Tg6Qn;ylEW-P*M%m1{i%MsaLRJv>w-(DaFetk%*`o6TbCz5 zUQgu)DpwI!4|~$Rya&2A+qJ+#lNZ!c5RL))RPCggYCPoyVBOzZM)QrN7uoan8^{OV z6{?n7ULFUL(e32ps|i3@3lEOz3F;fM$?Y<#>>%X@idPTp;H!ZH5=VC>4S+Lfulv;K zZcsKg3C*{M;FXk}Fl_x;Zf}1Tl*S=^ugWn9Ye3JTDX7mq9AAWu_Z>!`$v<&f{(bCi zS*$cUedQhZA)17M zA+S!*l)S|$Y*?-A0dZ^Y>^)U4kzdJRKsh;hUpMl)LRg(6i^*M^q+tT`W4u3U|zqvfp7=~`MZpvFaRM*2t9mDuRf04c*E?ekqwmVrHWnp0llz$k~ne&vQD z)WeXxYFAKR@{`|1R665-JjKc1Na|mq@=?;pXutD0&R?*WJZg&K3Qn3Bnv^67!W-u8 zbv}D?W|+7YU4RGHG*h3aAueHW${(@cl`$}@(Q3ZewhJRaf%LnKGC?-*U|lw2)LSI| zMf>+X(UQ*Mm>P=6l0vq%wYy;Wv)?W#GniQlfFdCqQQ)B$(C+J#=7 zH^ILi)x???DM(%#2~$X?Wx(eAXFzzz%ZhU;BPbL~vk!Urm^^<4kXJ=L$8e@H6;tP6 z?q~Ug(HtU;Lo*=s0gfna&+bR=hkA{NF_mZ0da&c0=5XcFEOP5|(%y`)f~!o`pzm!# z`3&SQw&djdLF@1noBi%Z=awyjI@2^jo*W(~GV+t5NcG6BkFJLMi>>&wfU}HpQ?1e> z_189=j@|Q#sf5=g-B~IRLwnk zB|&+BY26Bd@^(ofxCWFHRO8hTj#Aq*>Cb{%T0d(_~vmGrXSrN~5$sWp3oZ z^_Bj@krNtls&7GhlBwQ1oM*6z$^=k^xiLGzK2 zH7*v89a#+vvW+owtQ|k2vm1Xd+l;eM24cO+LKfmc&)7~%!s+sibRW7AeG*+^+{*{B zAvOk_ciX|F)t?3Z4t{*tnlB#K4fFQXd-09paf{1EcyT6}pYm+N2U}i*!kY$ieacjH z2pd43VJ>^#?0y2E6~04zj=QP*me;xc(T5kdt_C#bBAR=>eme z0*#9&I{wk+Q`Wi3!v<^MW$t}Euv`zLPMC2%;Hm4N+A;7bd_U$atj@z;>M)DO+kx&K z-rc(nt)ARqZwFq#Eg>I?i|hdV&Om)$WuNxSN2N63FPkEaa}<(JbvIQ@6kb>7^40fVA6JZOGXX4UQ} zRJMN)p?*%nWp51(dl@E|mp{Se%q6lQe+Qhrb%xz}u^o-Z8_MvN^nL!>*KqIWQH=Og zO-}qX0;*+C*LWX%j8B^Pf-4z}-s5WmHF`fGeSIw3A-E>?diw+JOwVQ);((_Q(3SP( zIpF*IXVE6RzLX0#;oD0l@<_53FJF8HjDAnS)EeC*EM20O~TrL}pp z?t|F&i#EiQgOTP)Tlcj_9*pXOEgmg})+0@2({2yN6q8b_wK&$@>x#?@*v#lNI9)IM z-936f`P*C2=gYi!+8b)hT6s{v8Qy+&3~6rVo$^O$x?l$`_Pq*r!`sTcOGC(; zzC(APFdi7N4K5#CAtUPag%;OOW9p6HwDo&5>UN?1+RfVIyK$eiw1(3C(Lo3~eTp5b zy^U{(Z3OeL+=YRghcJ&ITkJ$0vu4*Y{5rWFOKew(QRfQbO4!i>oy+u&r6XKOTfk{fkuv{r(})wV2`9 z6_Ag5Ukl?F_hL!gYw^M$$`JZTWBC+a z-pa2FIBpLFk52+=t-0v}Gx2LnH)uv*I8yJ^y=l1T*GB9@+7Fk>3$`-)UGNMQczdch zA8fFm&CTdeWA}z8_AYc@%K+(XZ6NoQ=*y{*8R*(77A)eA!Yj{US^mWmOy(!R#kPR? zkG>(*Dt5ov5`QPm(7Lp=z*Ro=8+{=+bYl<{q zKy@bnkE5#$sH$ngSRjZ>i--jl1}Q3cX9c?my9ZnJ4C)-Hiyd#KaXxgj0PF+>#X_W3ITt7vDE%Wogcq=U9o5x%Iip zKR0kNcmw$rEqIC^o%?0HMXH!3uIL~O)?K9Mbu8ih`+=P55pR1iX>R(U8dEmKZNPbF zM)9eY^W-giLmskbxTJi5;TKDAJ!y~No(FN#qa$Mcm<3p5*c&PTM7;lYDBS*DJPWVO zy&9F`c5Z-9Vb+qcLOyR%$OxZkjsGxA-O^KLn{ipyBm;HQKjZ44n-s_8yxFx?l5$9_ ze%%_jU+u&f-FOAxXB^TvRfS^vl`Ckz83YfVE!M4mq0#p=loSK;@FhK0=xZupAL|KE zR{1miM<>|1$4xjtvyKPtor|j8R*s?b%f1*$%5P4X2G?1zngi{sS5C6cy=~yr_!+hy zc?nwFn4vI8#UB1P+k=m1x&eI-eja=nTKlc!-=EdSNuH4)sz%5$@*=Qywd9m}1sL43 zt^DB^CsOo|V5NRTXwv?QNZvda)|U3?x>>uyyru;g3!U*4>R8O`R} zd2;$JUh@4Qq)(|O!|Dy^G{^XU;CG;WfVWxGMSevh)0{j2{{PNtE>=`vN^}l5tjl6g zR((KCw_B*{fN)CoxKW=U>phKAE?387Y4I%CW$#|JPT8(Gs-+EOgB!@Xjp+;k`;9{9 zaBJSkJRNFwZVhH8#hO|Y#m#d<&9PLRw%Eg6uBV>gss3ki%SjuBKS;bMH|(*MieH}0 z>I9Ly`a$R2?zAp*m?SRaQ|7({g^!8LXc;YfIb2|p4b#$A8_;fq+||3w^gIZV$ZKZQ8IM6|LSjFxRbz|!I`_;huVnga|qm<)-v@8PCn*RX$$ zGWPJtLfGj;=N5l#!&~Pxlbwy?fU^TYxGZgk&5(Al?SSSLSNPA8`foqdUejA~ZRrj0 zTf7gW+irk*Pj+b)zl^=-2bp`8;M=wlaHC@@P~2~}J{|Wh^#Q_1k-W-Ovu2hP=FQmw z*ODy>t5c!*g(GOyI10W$3uRQdFuPGa#WI0SdOHIo?W3mZa8$2_=-li#Bg{e-SKSB0 z*rEM*;e~M|@3dw#+`P6}q*%Vgt=Ywbv9|EwyW$@Y$@;EwSlE~oWU)#LM-ha z4M%&fV9l=m74u;ZW*Ym_>n_k)-i_qGn(yIy3a#aQnT@BiwuohS+u$5JKk>}NGf23j zacS6&otxl}7oLVN#RVUEIFg1Uye-f^YwjnTb!x#!cW%imR)>h!t~2D);ZJ~YU#=cU z=RQ7t0sk9TN78#pk2RM64m!yrPCfXP4?QWyZQ!s&J|hgq@a;k9R6k45JVV1zHu9i# z7k2SKXKCPs zh3i(+8m<<)T;b!n*_DFk9O-(nJo*;sKXQ;XSE5e88)*2n9uNC%$OtO|7qt<-bLKOp zSH}-H3Ntsx!&%QzH3t5w^A99FrLYVP=Y9m6o=5P)+*df_&@rWjB;g$%?;MXb->@_9 zzLs*G5$2FqO~BFE8wn4jf3OKCuA*FKRCCz<3YP-{*T~$_Haw5kfh#RM*q|Pa>3d&W z!Qy3$^KfLExukr# zNcbS!PMlV`&z`Ox52TmiM8+X-s5hGQ+-JpMs@Cx2z1OI?j$(jHFZUi+A4iG%FyMzR z+@WCko z-Tv;VDR&oilfk#BcetY_@k<$0Y{?M226sZ@9_)Jb30~e@gZKO2b)_faMQn#^4fmp@ zk0ku7o-gC@|Bx^qNSkO(m#o)hELkUyrWwPoAuWM02MLE@_U1aIf$DIowdy*%w5dLy z`rmn>c#!f&*6CUUkM^u$%bvZ&fFSA=q|$ zJ)Rvu2&esi3&hvpP#MRE9k{AA0MvUwR8(#~jW;`4QoNh0`M}cJx1mol?K9lt0&I2D zFjYg8BSJ6R5SqK+L*i39A2<*G(jMX~$5_e1+m544!$WX+^ID`i6~RNlV&U&d;_4V! z(>->ZB@19WY8ezna z=dA5$I=9>VCN93%1n4^i;iI6>5It^e5?9(iqF7Hw@&XrY_MXX<79{=3%U3mIBhz+J zU7TQ4qmY{P2RqI_%oLxJ20)sB_&j+oL^l7VrFtShxC+6yZZWERXp)==b58vaDDQxB zOiEXa%*wA?zJ8E*8N9}C~ z9htbiD=h5(RNTHXk@vh=gOiqoX5GgT#?e7QmuMZ?9$V~kWD5EuFH*PwYgf(WZ`vlbblm|2d*}^ zajxARvFg<+aqjID9G_j6&prJbi1*#7&WQ`_lSjG&%$bQdtpw06!I4-jVOnwqW+&T!ycA+yn**`eh&jW6`UxylLv5LD1 zgTH_~-;8&Llmhh?s1_=;P0#fL((`h^z9|g0%4Hck2Ux2<5v=~aXW+G(_H(5*j)Xaa z^b}WPv@q)fhYw^cY-Gk?)?-m`=V9>P>P0)xV(YqdMFvlqT#ncPnYGVCBtJy`$2} zb?!eGitD<2J;tYH-ALzFX_;I1YOSX5=0KC4c!l=dBQHvMxt3peS_@X(+l*8{yfCCA zqyCSW^!6PziXY$2H1dIld@t3fAJQBH@i=rlHW0~EGtz8Gx?V(v8}b_B5%m^2x{d9k z&q;@gJ3|(;7shotX&O~mLh%HCsmT;RKP~OeXD4KV>J?F5il6>lAb!a>Y~gVk9*s3X z)l2yK%|-PwNDtQI-LIYk!WLEA?B~+kg7AXo-<34I8#|Id4Ym#XPbluans^e3_bCsP z#EA3{Ow~APKu)~Ly0$t4%FEOD;k@=)`2LcY(!?6ohagQL{)26}@yIh~H%=Gn+K_Ns zX=3FKxcUt8V8n@DT-EfipV8IW82{}84hYzYSv5C{!Q1Sm(mWnMl6pz^fwT(wv2TL# zRFV&1RPQuSBm8hVTWL7{xMaNQr{Ol1EJy=W?_wZ4-%aa&D&xefR!2yKhteFJfrZy+ zaF*=k=-;C7cU?(1r*c7~ zxO93|K2*?Na@ia9v*8yPYVO?31;Sz6JG~oAt(#CC+fA)#?_J_6$}RGO>8yCC4;z=6 zs5o8o+iDe|X9iQ$ezc!S^np?R(K)p$)B1H!A;kj_hHwdq;9p`8{Hx1B-i5g_^h zYJWYn>2V|+gcd{os+^Ie)98#|;^+VM1J$o<_oGTEuV0V064Pl}6LIk#RQ){igOI(U zH4yJAou+ynpgfM!6dP>?DlK!RQ7jZUI12ZA*W*fKQoS(JkZ!8CLowG9#_?Hg<|-Xc z=U?B^_#4d=#myGL*+&P76YaQ;cCwc0k{L|s$EoIE!MRKgVKVuLjqsNC8>K#!>YcHt zvs*IaG&dPBf_!+o=4SU{ggbRb7u!tn@JESO@gwPLK{)||>s^sNDUxo)b!7+zm1ami zI;DS*@(2mbL1}WuRm6?F%aK=tJQpfWc6R)2t->$GqeFKXz;mPRKyd`}mD-sbn@WmH zbst6f6v9V0>T$y92ZOlk$B^a{Whpl0vSg8D70CkUjcf%KmEHvKG! zKQ}Q%|FejS&{4RB*V-dY$i0mD`uD3DOt*{KPYE?=@-I-+#C2 zks-wvvg0OK_pqHCoTatWHDy7}Pb6)n$uxPaQJRY81{L>EK62tpVP-Nxc{?OMgSGn( zS2cttCPA20rr~uWH$oFRMbpuFHD=gQf0M2;t6lcqr_IOwi+0ofjd=x9W zZ)rZ(3~DUR9VYT(b91>iaxH$%w1K$rj(mqBLduLPXlD{YpV?M!wh!dTv$o<P%P8AWaYf6sx~+6-h^ayQw-Z??Kl7;9mGEAp0s47Zc+rQ_NAFp&A49$1*QiMJd0 z2U~dC;=7`qVrj-@=9x^-Oj@ji)xTzezmG0V4(Kk2M~1-#uN;Sm4S;rD z>G&>dBX;nkJunFv{24rURs-@oKzSWX02IJ@W<6UhHV6Rq((oO*0@M3)?xJ5Qd3!)iBk38dT0Xl`*xH9pb_%40q3%|*7WW`Q+Yy3Y@V=E}y zg;wRxm>scA4hZmIT?f6zZ)R!mB<}}XleGh0mMoVPUz-15>0+s)Zhed;5!FVd8JGQiFh?&kCWW4icEI0L#CKc9hltW@~ z@_Bixpg*?qye}wM!8+gydYj%uoy-dunb`r1XV&08nT~R@{cLp|_$cI)7!au|o1~h7 zYjz#EEaWH3kS|E%gsp+L(ktABn-+|OW4{7;>vGy}HL|Y!Ur=rMo}Z5tZ(N+ZQMNRt zb1RB0(KA`{{za*h-lrLrOt%|x7Gy`5V9T=pVt8a7wD!`%IsC+KTdV=EK~s6#fL?OB z>1|vbxlXnXxXj-NI76S%m*8SQzuQ@7Lur_Dl+GG22fv^#aMN!J&-cvY_9=GqgI7C# zs<^fsS#aO&zY2Z+C#ko*Zc?Z{5zv*pWu6w+k?XacD;mSxvL4vWd8bC$zthSLF6MY! z%do&y@xP$Z^gX4I081ZWey3&Ftq}N(M1Q!G_#FCYSxNiQ*CN*I1g=a1qe7>@(*W+JMwxK#!%#CN8`K!UsEjQae6+{FJg~K^!Jg= zosH#TixBwfMCaEHnks*U3C$2X@BNndc07j)rrI zzSa0MDXc$Ba%6n6*I8(U6|gtC3ykq@hHB2IhAg7%vywAXQt1rO3`TRw9fEeCp7|kM z@7Yg&DeTA_WuC)Vg~x#MN-i7poW%v&NN9*6_@l;V2C1M9G_@gMEJ zLKmJ?et>dyx56`sw+Pid3U4bLxP;28U#~DBrMZd=TT_%EEB*V)i{-7@i$NphQAgV6 zG^iH0D;N&Hjc386q+E7Bbd;P#`(8~-Z2*+pusEzQY?`r3aey!>eWC3c*q(1FKg7-i zw!nWTBPDSFc~=Kcyu;%BX5gZt7IJ0ic(FYt5re&)v3YVfE)D=X587OM1=>M6t-Yl> z6U)8#YQ}pGqL>GOxw945D;Nde%-RbB=e^p;4qmdA&qfwuI*@u8DWYz13~&8w5ygC` zSQXTO)4Z@Fp?SCuzlaU_uSAfS3wk@`Qw|v5ISkSWCmW%1nlMY;3~a`4nO@OQe)G`c zx-iW-MiBQQ;VO7!^v6LJjbv`c1~JmSB~J~wh`FUcn4CWuCK&&&#@G3w6SR-~e5)~* zry5IPzFQD>5C+W<4drY2AFb!icRUZdrT2t$(nxv9v!}ZDYCKRpWl}ns;(ZSv7T18s z&iYvJ>n#fk>xYiU|HP0W6D{$M_VTayFuklN45u@Ph#T;d=OwT!oyb(K+%9uN;%@Dh zknfBzPivL`na;8G7Oh;?i4k6|c+$yMP71rhwuHL&g(5Jk5gsdVE&j+H?HPx`A}rjV6OM7pUv{q0 ziF8GlNJ*tV*8EPZ+?$c5&B|;e zDIdTvvkOyswSfezZaRyUl|waId>~IZP3f=+#&4a`zspCI~agw;oXP_7Q+i~6D?sU)Clz> z9^2_QJ6Q`F&m5SC^Qv|0*U+Kx?blQw{>0phdU&xs6^UzQW0w$c%8FAwp|D=i=V&4` zX)gzdN3g$i7!Nj?0zHzugUpavH^U34o-r$LA^vte1NA*~vCP{@;V6F3+CVwK1vVz- zsPACJ`H-Bo4JfOr+k6eiN#3x#p4}@V)sZr(!KR@=&yO=Ql}~vCqX!f zxlS$cSWy#c>pvE54$6apUgy!S;E|hlMrAtju1rrF2&Lhv!meTu1Ut3Ue94C@m~o)Fv@?3nD75kyh1S zR^oG}4LA?NtD_`;qx=LvoVOU}I*eq*X-HZX-lwip8j(>A3i2glcR)I8o*kj7={P{~ znxNWZD^qNcumS!^Z?0-fI3)|hf66OsnBb!aggJapU>nx3$PSN0Y!{WD5*~#g#EO(% zY?^mGPz~ap(8qAdVFyculasi0L`D5=GI3Hok`vk=x;Wd&*sMa*G zUJmK14ip~TlewB-3G)?KbJB{?D#$`J-hTxCow1OIl#b+kignd^6wd;21HbFp1LIBm zOR5WTCNL3|wpN%x?-RC(vvE-ICPp|fO3Y(`_!@h9w?L{7BwvH%)8I$yTIdq8l(;Af z=KBvrHMe#ZRGx*tMJUczX<&p6j25;Q?(+}9}$ql@(* zv^Wy=L;m2$vy(NNEN^x&;TP9KaI+7sStjL6{Mk04i>&UdI_} z0U7Ld8hBO{!s}_+)wvPh;j<2uSEX^m?y$>({05TG;B!qqk-QY&m1W9HN@+h+^0~_2 zzRMqrOG-z9ZA1&%s4$aJ97xAB{AXauO@f?3L;N>nEcip z_8_1$Cy&JyM<4O`#=Qy4s(r}^`zc5sgDD&&j$ov@nbKW^_Zpf9*4KEYnm?h&ruak~ z<{>*()>3@W2#5LkAbl8Gehlb3$xpq4%FqX@M%l}Z z+C^z1EAs&L3tr(ii#i%d$393}2+1dL!dqTDsQ|9XVbDG~3!fI+gLRlUt(hys>n7vG zk9DgtCCM{=h+|J!ls^p)B)cj+z^#KOsCvW@6JOkHG8xKCDv)qad#d0D>?ycK{&*qY zD^F&f3OO`N=^!?y8gSwYNp%9`Cxz0HN@ICs+`yp75cbA16BZg*AbD%j-)|t-DUm*x z&VD1Et&`c6KMrgPoiJE=cU2>d`WvJvB%GkZ2})!tuBl;e40GfBR{M_luwwVtLA zw5A15a7~suae7bupZ^Us9{5l33Un*G#`cx*YJXr_Y(o9czqCzDGMntx8DIMkz{1p8 z_;}D$B#(e|owqSnBL@SMFm6y0BfJ&|L;i=QfqN7;Yq8)uQyL>Lg7yHQ*UT%tz?9#m zd|_>po|3Mm^ToqkDQ~W!JkuynAg>_lU-dbf=ivu{>QoY@@V{OMa92@0H1?^_2ini& zSMrNtbYMHFaFaY6lGeu4;b}0^WIFktt%C3y3`|F8u2*b?+2JSM8aVdHo#k!eM)GNT zws?U0wBq%`4s%$qRRf?>Q-p*t8jH-YtLXA}m*v*I3&*Al?BZz_F|&vRUjXMitB1bq-bZPv|<^4Q(pS16A`xCx;dm*9NpQ&3zSn$Z)^TI-n0kw!UI*cVqZ?mtItb#tcZ0V@Jx<=7cR`2dJ3aL$7uu1x^Vh#MmikRQks#Lw&djTFft`ckQPJbJ6HMCChtZ4PO~nm z9+9-8mOKVf{wa?MEdq}*daY1=Kd7`&`2(Q&CSTtV$iE7zCrOyDd0FyX<#)Azb8)gq zdXFaG?5MaO)}6H6ek4CGK6@L(!{W6nx8=&rj!4%HmO;A&aWJR8o3JWsC%X=s4gtUB zK=`1EpuB5BrXBUFZbGV8b-i39OvfqNHMp9Sp94QJ((p(+U8=Fa%1lwUDb_o3(fZdf zK^_hXqtrYzbLTyR=1uXc>Vq-r87N$l)DK{!d4PI_LiPC?JNDD6n33)V>ZerWt&3MT z(t2TVBLf6sD1P?d3e+!QGGrE#W}~y-Te;9|M;E8#!lzIx2ty#q z>kJUDpz6_xLPPPyBu61gr;4=9LvCv$YXJE&pk5eX7Jh;ygyxI(*}9x^GM%tY zxMmEHs-Es-?q{=#4LE%labI0YxR2zgvBjYG!rOTVX@GV>@6ld2ZzL4PI|Na$lyr$l z!BC*u;`JkvMUBw+tm@Ykm|%8640JkLt&{(mo1jf$cTH3C1E_<)fjAq8`=Pw-oT@#Q z?@aZLZ8AIKm8@t%eYJF@KRt>YDBhG?9Uj6F&)!OZ6VAV^#;oxXO@Ml2nwjR!k#ZKV z6vZO-FceP+>V+cdaiknk+zBP-rkrw+6LyFfgPw>tL1Dx@Y1EJI%?WcQX-9aNAy6C8 zh45)F3^Y&1n8ayrs^?3+B1S%@S}&@4B+aAZ$sPRj6$WTk4}v@*C%#g8mUVUB1=JVd zbe-MQ{B4>+`H@hr-Cz1p>r~W+lcv^EuLnmJZxqza)Mi@L(2{=^MFYQRRljq7Xff*$ zejb$W^>u2`lk;CN>VL89(2;1Ec^+E%7|NT0`^2)41#EV*235~taN(3-c)t0%X5yjT0y zkU!kO=6Otm4aWUxeHUQIz%Ur*yAgF8Zsd!m(DUD}&eBkK3k-l7IBsh%Zq#`f?8toq z*M?4KK|Bs)x12)XR*!LM+z-0{IG^^rUBO=XBw|pVwfvF(M?7$TIL1yZz%~nNuqf+= z?5wXjk62)UtdlnecWA{w*QpOfW_H6#nYY-7CT}5Z{W7HQg8TQHVcw`v+IOx!+Pu`` zzLr}x6YK+kt`~L6OCZ4L7&Zu=$UpN#xV1`;M>O0me$4DAE{>`p{T4*X7meyb(e!fd z^jG_FLy|Ff>THE)T{GbG+of>deFSiuWS(I^ABOC1C~`e!LG-=du(Tu*OJdKXbw)=% zA>%Y$iX1Fs<9@MuRkda5-z;?7atJ@>HHDz02J+-C9U0N22*->zLPL6fs#AA+$jJ%U zgbkg9DSg5+w6tBS=l7~ zXynH42-}TtV-JkBRHBU`to&*73YJu>KFwb<4VI z#$qf+R*&mQod~!Eo5bI-K60h~Ou6;`Nxaf0N^V%BMQb{gxy4ya+2@rFpH~$szv*#t zYvCc#?ROBW{#yejv0dbwm2E`_<4m~mw5EJA#gPXY(}Abc7m3s_qwvbhofx*hl6~#& zp`r0|nn#R@JqF{(9Fn*4tl-IC3G?dSgto7{LW{GznY+VZoIbP&V_X@OUzgD9r2%`= zeg_lp(j!?Z#yh zd*cqgYY@e+ogc;WqMOTT_j~Yl@i=z=T7vLB+YGx-O@}Le^ySx_D6VoV!{#_E?{E^5 z2Gjnlvxea?UqczS&_?<-$!By8sK%mV(d<(bd3M$qoG>;^Ea*`s=-S}(v5EX|2KgfI zbub}(7gxD-R{uTwuzD2ldi@B5WUs{IIknI*xv7i~Z;Ayi*3z?9REKp3;D|k$%wp+(BqOVNGYV@z5R1$Ne-=-HyZ;Nq(< z4UhZE1y^f`HVf9W9e=yZlkP*|)|-j^eC})E;u{4whW&x5eRSkeZiA1m8t~;Uw_v>e z9NgO3mM45V$oN>!qxuh$Pxjnq1vhf(|5KR5J-Wx7;VcVE>`;5d2V8TX(z9|)*p4Y_ z+T>P++Rux6VbsE8emuFOHl>fQXmD?%aJt`?2X{!t%F)fmq1U}|Ii3Ab{&PF{xP(!D zSmMfzbL8#((d3V*C>U)mr}y2h2}|1~UAqiK%462bit$}PH?YzvE<8s64cvDzmi>l( z$1x@GFe|X0j9Bf>$4q;K$GdAVcC>}KW1NGFawo_~54^d;ypzGRxXMAoFJxOiuo7 zK$s|Q{c6t$}J54p{YLv2@P$qHB%u zk$#|xDYJ|Vkd`;H7{!D;+=~Vtz8xZt?8i}iM-l#v2Z{@3+4kj+Tzh~~T@UzdJcJ(x z15o%)*n{ozHn>f&*$+RejJUmh~luXjO+Et-~>Ieu^g{mX3WLw^6 z)k&CiCY+~6O$Wja_|RlJ5^u`y>0h{Q4GSv}MPP60PwW!5DwHO59#ZqVyAkn%DH)fm=hy%jmh4s@?Q1txsd zXLb9#qt87PUVHs)*gCU2RJEk@0P<5<-oMA9Ylmk1&CAB<6VyVgnu-6|5O2MC#)wC- zs?!T$ceJxS8FU!`de8aeNgak}UHVB^gP-7YVl_R}m%?52 z3&nVT7 z*M`3D-d^n5VIQph3~sxkW09_pU&{J~C0FZ9ZvT!2Ky3(HbskJ&lSEykR23VUvp7!> z25Z$d5GKK>sUVD&EXKsLPk6B5Ufh&?7#)Irx%I~o;s+^QdMrjSld(wHjR_I1;=@xq zlPBvF%qbj$hTS_54|S1I)+J(7jt*3Iyn&zJR)Fr2SkTGJhvj2Bun~LQ!E(!fRC%E| zqEms;G)Qp;s#m^f&n4`n8zZ@m4X-(bu^yk#LoC+i+PEb!Y20X-<5Cw$w}?w;54vef z+DNa(p3tfLaip4NEh4jp;)`!CX0iY`Gc~W5BJbnb2e-vqOEbLDuo+xwd>Y zZE*-a_r0TiG#+0(0(&)X1I=tNuz5@VvD#@hcutP49K>U358Ej4J-e^^eS~WlbL-|3 z-z@{mJC=K&_DIm(3GNXc8R0P<^=1bjjvR;k!QZr-lI`f8|5to*#spm+xZ~8uhf#6p z^tAO%%|pSo$v|8SFFqYZ!Vp1u2Bd`~{^~B@^f5&3v@Q!UD1uPME{GzAf+Sa-moyuonwWlcznYm4PK0n6QiO{@!HV0u;9Vfbf3Z}%wq2l z$gr_T9&RSpHGf>#0bHlF`nCRz;#Ihj#TUvyR+8fGA z5hqBKMPr6{9iUv{8+dcDx;a|DDEm&=LFZ@UX5qI$Pk0^o69!3PjI}gSdkGZ_6(`2zIx0p2dSUTrsyTWS1&ZeC7oD7b(Al>iBT=d5w z5~=PO<%bBqeoFLcHB@OkSY%K^8lk=1pJpU>?JWkXKVBsdgIW7pRF4gX(vUPSj9!Pt z@oKKYL9YW6E}_cd^M5sJ?qCN!*F`ntMmkx}(rwI*cI)FK-`ybuSv=|a6qT-kH?YFs6KC@?~ z`Qo^40ywz$g@^^~l^+4cK?iB`!;XP$ZJroojzKx#PSY~obrg} z?rwrzqm6in0R`mwN8;*CFCr!aOExfHF;8< zLkAbsN0%O@-4y@K2}%LoVV@M%aGRAo=-%i#XmF3tj5~Lp>b<2*`s}W;X>2PATWHOr zt)%yY;#8`4nBLbC4h((8+AVA>qwiUYTh_~XZ92;%q)!Oo^k14e_ssd7ItEhd$ct02 zij_9)syY35;8q;<@P~Mk`wmxsYK6@{?WOgCUOegz;YyO3RD2NAgZinRa;mYBu20n| zd62e3=}Yo%{LrvUaWeP|1V`^=iqHPi+WfRPg`~Ny;ZVOW#Bo6oqd!d$MzbfI_2jI@ zK3w^gOM_h{>3*c^;yQzE<+Z*o#g5#`(!u>2yjl?rzFDu}+b&&H{>pbn2<4U*cI}(EL0YEa(glI0#Cea~n-N@ z$63fx*N=mW)vllQn0v%g(x;cOa*LL{Z#%iGLt9QdmbWuH$S7{C--6m$CGB{U`;|3U@d{YjoRL0ZMl+q2|H7@icOYp>@)2f|bOmT}19SJ;%6vb&L4%+@%41aH zV|+$8j4iQ+7On2F9+sOKd2FQD$WA7s;MM1=Zsak<^z~($Cz~U9Q|kLr&%=#!T57j+ zmiP5E+_Iw!R~}J$EY**w8TACot1yKflgi#w{cmO;^WtHc{zOK&i9dGl!}A9JlvfA$ zPy1NX*;H7%EnJAsRVgNXLW9-wEN3gz8u9m5JE(-!+wgQ0?iibJaT$q)lK<+#DFPWCgG> ztr^uSVd*t&(~tHub?=4!8g7F@Zv*khkt3qt5n9*k%CJ*%B0jTUO=IHf zpiz7j^5v1zoq{-!x%{ypJe-CG`7P)Zq35+}nwbX~e+{rrRUw(A!(ABlZ}47Y*NL{Kk9P+x*kZcr^9#;MuM6i*`I z5)l4kyM?jUG0!QurMx*rtj1b+YiRx%^~3OV zl>w4JgMg!2@apbFdbTN&c4O4fVoF;KIC=rhrZf{YR5-YgBD+HA%Q5uaj6U=e<1oFrl>(c9j`YfXKT5BLYq2AGWHl(VCq~0MTEJjtw zw~k&;H%j$V^)Fnz&~BllcLM1}F<@bwhWeLC+`*_X0X0{eRBN*44$Fc1Xw_WRZ*_l8 zy*|l5hU2@mo$$>)Fa23Y24TKC|D0r%{wi`9Qh$U68TYBya)h(gBWli#?x&)f6Vfx( zqp*}qu?{DUB+XSzk{6-AoHg}%x+qO9b3MM`wFcqLV8AG;`jfE`vqWtJBoluzr73&%kG$7q%lxV;+V6uznNrT!-p z-!Zc(`vmcX!a5{;XU%*X@b=t7VE_^yL+Y2OqUO0{K>04Hw*%xMG^8(;w?oo^$_FYA z5X4`KUom6s0P2VN@oTT_#Ug{HT=l1$jmu@qSGxNcOTsA9j1{EwTdSU3dW)HlLE9rnW1>`D#gCJ<*49|yyz ze*;+VIU=MU3B^X7EJ~HgA$8ua&fx# zH7xM|ghS>8^E(47#plsY$;<5KO$KVv=X3$So_rg-u=fx& z_?_6<$d&&tn1z;4W-{waZ?^Sk?gfu7r8srUB)KXvo0%Tn2kWlbN~@2x8H=XpQJC*r~gKj!?__T4 zS`SA3d4$j56-+FQ7xj$O@ugbx$n;>DU55C4B!lM+-ihg@A7BJu$A&%Bbt}C33KZ8zw|DFEf^_O?X-vwWA)Suy+HbaNNnM*e`ef=hLtqzkEce(tDHC)fE3DW8| zxGpzC%|E)w(WMKX8qtW~ifSw0-f~Clw5dE`=V~$ZP9S{fEP&<*JH37*XwG<(exZ26 z`!^1EapZI6nSfXM0uaX<)7&j}3w_W*n)q~}`t2>>wRMrxn^$2U>RnJSiJza}byG35 z!#gb6MF(e_CGeoi%`9N&KQ{elDqiaG8lv4kGpaig_$3@_TBKpOKC}*TMKp7NXhGu~ zt)cqGPrvOzXH0#W8WSsWmraxFPq)W5K21c9P6w`Pi*kr+=^O-`yur^$y33iz@>!qS zYvpeHhMc}bdn%45W&xHas9^fXZn(xS_Y~=xD`SwDjX!dv)i&hsqUZQk)*)oR$Y+!m2c8`Q$T3{@SGC zEqum@pcQw<%4NL?gImjUD^9SH@kfMlPg{BEYc^|_Zb54Xr1o9gGq~_qQy7=Mi4oq3 z$J{6NAZ z{{E3M%W`Th2!kPNe{%KT2Xu&)F9(N1v$N-_@qy|@{4H6DDuz}cjey?6Pftl;xBnZ= zFUBr{mdjHie1m!}z)Mc-wv{P7kJxP`{qm=X zr}B*8?zpUDiROK18~m?FFE)B?e}2|lmo1r@qz!3xR~zee6{*%FVGOp+=>v;TYh+pd z7)`8GADMgW9>$L@hUG3>_}UA5(X-EC$)>DE!cfAOq0sZ&c;5TqCTRR318F`~j?2M# zRB;sk+5QL){Mj!3Uw4*UVh#i4Bu>j5igD#g_b@t`l7EMajbroAATw}Sne0YPrgJNi#c2Ya6h@REn1AX*py-spRq#9u~ zJ~>AFU6U4^ll~+6C)5rZNBDFUPSxp%hy9FT=$(&9Kg;Wx>#FmKYL~~?42L(f?ts;o zzOsuw-81fT0H+#83a8pzIn^8f*S$Vmc$X&5M_oXb3&i7^j9bsyx7Ft$q-7&XTrSt> z>G6h<0QMhsB;`0St8a?+a~@;evO_TC#CGvMVmzco^%I5jC8+!_88DWf=}Fdf-H;2b z|8z#_r^C*N*AR^z)9|d#4@86dpmJ<{v$ZNm(4HqFeGa6%ZXatY(yq|nD&4u9|8gk|pFmvZL)|OaRDG*SNeB(g2w-=0dsT0AaTwG z*z)ccB$!u?VD(0hhIjgtdMru|~jc(2M_s z3VVz0IYQRjEs$TwQ1&&;M8$i912$ld^e)Vf*8BZRX~YT7fN)3XALO`g@HKq&(oPb# zvy=PR2^HI655wWh#cS{}X{P9XY$@z&y8vh`e0S(OP&rL_2Wt#Cc3(P%5ho(?E__?< zs)UJdMf#={KA2x` z3GCgsSj{PrrV$gIexXbDF(54>Xl}Gr6X0ST3(J<;p~BZGb!M{kQh!ETOUziB4*SmC z$E7#7(|ex-57-O4AM6#7IUg>@{=z%GXpN9>C=52Vl*^;uYhz!Ikt%nHGx*$`Iz0Ae zK9Ei#&WjYAnirwvnjHw~;q+{73~GjE!-@V4k+g;8#ZMD)yGu9(#N1?E-$nyzG4Z|V z5N;b*Rjq>v^B`s8AJ}v=6voX<#=`j=Reeul{ovGyT|j(*G=BVisIFWzbSt+0RjhJ> z;&m9^ZEJFhCqLU9k+6jw99vh0EbInc<#GnK|;^XyNs zJ)b&A!zT6Q%@xf_1I<*N0)1-NkWOR$39FZY!ya>a&Afm;d;S%P%LK(p62>aMDi-X0 zLHSfq{M^}460fsM-8;dYmb6~=$wsQh`dsO@4{z?VQ=`4Oai1|z=S2>L1kV;z4eJVn zvwd)Q_z@w|E+=`iiSDJSrLeHWhnG99zu%%R*|2PgJALSl1IZrH3x zSJG09a9d$YH9r(Jx8`R0Ch}_mh66I$tK|Z?J)ty>g)T{Vkllg?PR*K{Q&r6oUR-Vdb(dX00`n16FC2R)Op{ z{}ZH*1nDjy9m-}N8v`T1-2w6dOxI(HthIpd{~LG_HrW7;0bD!SiL$AIyyfj^Tp8nr zP4%8$Y}fEBPQy6%!z~Uv*lxp+c{Qu?NK1HF$eSnP&%Nd8Al@!@{qJurAuV;=VG zj2mf2dfr{1H1R2=BT2{6y;#FMh@w6GW~SNlEirbo(~IM{XvB3~dgqqXa7g(^`b<~G zk08y`A&PWCxtV)Bj1j?zkGiKi&}S6e$|^ED?$B`J7~zz4uIHZxLU# z$jT@wQb^lus(U_%%w(0V$liO8tl#JK`@;*j?sK2#oX_XI?(>{ef2@44;|83mojzX6mEjqFzw#gP)F z=OAf&330$PMqV5Vi}6TWKBLcq(!xJgIKHTd5lim*gwc6Qr5Bg4$)sM)0>Tw(`92ut zE;?6ms!s=4pOqm<%Yt)|flxJ}c(CoCFWN)zt{~wR5?6J<_JfG`-huWE<^M z)OpF6=AR+^%wJ*uJyu+qkpidFT44CQ`81bL82Nl@vEdWEuWu@etFdo%ZD~^)gfu>a z{Hy%Wy(d>#W7%~d99wcs*^`5UDA z!*{kHSpR<(vRY76aGX>diBq8UV?Ur8#oZR>vQiUghV&|?jq`-T0kv4Bp)XT-n5NCdJqzAKQdC{=3I2d}>A4p2wrohK z9pTPE?Gh_Lm^`5+E?`IDR{t@0{CXxdynIIaFjcqEWlMWb-hnI5BY(|F2PmF^ZoTQb zEh|3_=@uNfvK8+&KMN~DegSzP=tpbgZEAUlG|MLZn~`^l;w!?8^@u_ zC{&+c+M`Q2PkXcx>xbYd%W%%O*Nc>V=r4HN8)ToS%{Wuljin{ z()BE+VK_AXl!}Ba(1+c^xu@GpdLIrnjTPT?Oyr{@hqc?!|AXfL+K496IizW}kZwuE z8S!=yQrm)44#qmHJTH`|t8ElXTTX=HIM>BcH^PC&rTjT06n?l zDXMw8=C9hneGF8tBKpSm=AELO;JcHt7*Sl!WBQgTT4QFI`g}HPgr4F(m8tN*eO62A z4WulPdMutYscVWy1?gO%I%ISlnX>N|+U;yAa^^Od&ByM;tSn>p{8katzZvyBDBW+^ zt2R*91giG!zuQ8Sr@N6bwBYlbskr}&2{u1Kd&3;FsnU0pFT$X?-$BhgsFmnH_Ce2_g7@_|R%uHbQxD~n zOD0}t$_Lar>%x`4*fX~ybls`^&nU{XzboED@)XLKpnusT{P%YNcqIFh-xw_vueZd$ z>U(CsNzYS+9qagn&9-Hk$(q>TW_d2Te zhVo6?TJl=fxaC=IR9QG3Q#NpPm9PKaXEu!+Ny?tN^3>(ly-Dki5QO8X&YiA{G?EGN zI<498J{TX@90e+iAK|_MBeEKCi>?pZ&0XthuYmoSrfmuJ+d2{nY)np3Y>ph`v1D;-S9jZ?^hm-{$S@^Sf z+`Ye_9I{{`?-FV(?;0<|M-RNPe(yH4?$TZOSsDiM3twZw?{@O0=U1FRz#Nu${^$`~ z{+t;FjKwwQ4`?+<4TS!q0C;yuWyyr1$V~&m4JJKp7k9 znxWZg_7c1Jc7n^nQ8+Al5uU%-kI$j~C%j#xpz%PbjGN4?ZX=kycp?rhHRLYV8j&=m zv6P{d?b>a{e?hSrp7ay{TXGHPdZblWGImd$CgXbakt0pMv%3~|ApFb)@VrC&e!SSq z4+hnvF`oo=1B>Bx`4qf!YbluiIl@;*_6By4)-hT90LKKBL!xCVtnb?Z9^bhSR#{E4 z(Unl~Jg_^=?|o42-hT=2@m~eCiaSW%1z*wgcL!(}N!fFoT-<7#%=6PG$o_V<<>!A1 z(7K)-91HBh+xj^J{gFobL7H|kt#NCu?(nx?jA)c=1d3`v4hS7*WskT z4TaXWiL4u~#g{J)DeG*{_w}zOmsS>H=UQh)hWk4Cu(&Pg{C5<0*z9K(le5`yePhJJ0rWlSQB9M_S>QNe3!CA46pL~7)4vRTJ9sEOMK;~#R&wz!K0 z4^FW79gaB2$4!Qx=_9j#^pSNJO@nt{(}YRVf6`;sK2XOsN%{?I9d+d3&I_vEchY_X zeoC1EIFt41h(6UXS}AvG&dXBb!AI7!nyzo zbhnA_FHQLX%fV=O$sWD@ci_Sh3oN{3E9p4++TK_g)Eo^|oBX$XxcZsA*x!n;%(xD< zigO{j<_Iy(+z+GtX}$eHdVFsA3v9f9AO@Pw<1IRW#;7h&G#`&Rb4Rl`7*KS)6ZJdq z)jk1Q1lvHhdiK2ej2e>0S>t}|KXmwo`1sBQc_sOyW}?MCL1QbeF4v(wJR;%Ys}Q(< zG#F-dd8#2C5x)E{%lzZaOO_16-b*jDT`O8^uDKiX=BA&((_t-AU-4z#2MgEbfK;=f z8`VoRZ+#jS&OFkM;<@~HRgFaUNx>jmC+o&^BiYPy2roQ$6qeR$SA`uvViU3N6+O8o zy}W?1N^V?6>&OS~fO+j5c&6SaOf-G3y;?Cu_Ah%6v+|5(oi?3#S^7xT-?;fzHF=}k zO}y#37#D3jh7A++B-JUaot z_lug+?i>TcDn4eH84{kdp&7%Fuoz30TmhOZh#VL$pRa7HrMZDNu1@^h3_V!9zdtPW znkf|i4p}l3*50ZlAJv>+=%~j}nN5TB#=qg{!9w{UYY*t=)Kz#0G%m2|auko4w-Uzd z)dZvY%VBVQ4b75!B}m8Pn#B)A@{+6UOo*9$<{ync%U@PuT3ON(4ElMNdVejhywHrE zndpd47h3QG(~WtEcLCaln(!By%_JR5Zc2N=yv<(Iy4ruJw+<2hzht4^ZnHg)4-*dr zs&kRQDz1X;@)#Q~Edi=;VYpo4Nax~$q9X?}*nK^BPF{=`XM|$eGdqZRl3t*9p{3lm zvN`T9@`NvS+7PC`0RP`Pa%4X{N#_eUygP!ZbqcL*)9}NNT};)vLnj>`T=N#Z*4d%P zOfFhDOWx}`R;p_iZj@6NyNRi~+okHQHR+?F-TX={TKEd>brK-`=NTYe1>t5bzjuBo z2_N{okc|Zy)<>cKOkLi%=mewxBVm~)<**yyp4kG+@0MwUe)g1Q=Ud_F-cCS0#U9^W z$#fSi79L$6Xt&lnjfy+RmYB;}oj6>WK;MU*D}byHo3VApEwRpG9L~P96E96^#Je9T z2EF6}Zqp|n3|tQ|op>X8;z=6RUGzd5*uOU9Og_xd6>+25YaCxvbD|pZajpqLn zb}u8q+O~ywvFI_L>Cqe8#T?Y6J<3%*B8MK%!~Nd*kfy(u{p&W2TOM7*#y^yd>&Bpa zhe$BJwF0SMvGX-+xe9{R3Ig#1_f@FK4UcY7TKvs0f_F8&dc;|(PJ zn-MQzpyd^+!4~KfLhI*->4@W*&3X199sa%Z9PT%d)_Cgt5i6!`#xEUp`SHB9uycT! zjPkePvldN-Ee}rO;Nb0=w%y86)v`~IOG5woPV79;2&#Jn8U@@1>I+`Ovl1f~Pry$_ z07;!cV9A1c?1E*9Iu@p0NJdp(pY67=Ljfa^a9O=gtj2_kg8HjqMb36D zU5n;Pt&@y;SGwQ&4IS=Qfbr{KbUxclR3^`ZioT9mFVm??^QgXVxMMHAy5Cc}J*zL* zoOkBho2x|ff7x)p(i7?5xH5Smb`3m-xvLuRf1Y2^q5Kt~e;qm6&xtj&yoN7R{RnS< zF-NO@oU{>;CW2P6sYtqoYW*8)-nzRQR}JBk{AxNITz74fGeV7k_>3zIrkclb=MUoM zee@l5d^jA>Gn9Q6zE%HM{m&bOY}F{> zHJ6=|V**Cw=>euy8m&pS)p+N}dA#z-P3F*gy)*o6Nw+oS1N%n<@gvT)Hv`g;c&*ea6Zt*Ziq=DJhNHYSY|EX8s<&}UaW*OpKYZQ^ieGHTi5)Dl z?2HenI-#B+t{lqlAJtWyTE!g^+cS~08zglfgw%W5&y`YXFhN|wewJ2~q>ZHFzdx?c z#D|5ne(4U!{DVO|k+7fH_|px$muWquc6->>zsa!a;0Z>*gBzE5X-+(8AuBI5Mz`}W zVxiY-Bn?HqJsPQh+4MfmI{ITsm%%^9Gejd(ZgQg@y#Js;i{eTWOE<)+NCDrr` zZgd+id)jY8^Fd*9b+*02W_2uf^q?0fZ^5>O8uD7NYM^8L`cmT|K3`m)+bOvt` z9D_rmdK8dfQF;$vJxLW&J%ZLI>=t5`hhSQZZ7Y{mA4@s#`fI1 z45S;8dKx#M)CM=s_fJz%Zb_nyRm$@IOxy8WMzfGmW z4W-LTA2P*>b}!B7xCfMvk!E!h6^}yGvJXJomy^ck&(Vk0%(%J^@X0mRcsVvuvAnb31`({)V z^cmo3o2prRcMOcIc^ht*m;h-`wlZU=xNhJtCU>bl`;Z zaz>XaV)TUb*gHX2lE)N3mXxaF(Q^iVN^{EirG2YDch%oiZ24cF*y2zfRWA{5X-Tsx zKS};=36N&MEx)rNEMOG>PJ8W~nQG6&^(UHovmwo7*6Qa!;7uGz%F$ng9zuCOzVbo}(6Pm{E|0_?=N^)DnIPQN?lhzI)9tq4T%BE_t+!x}7QQ3T5QF5A zweV^U;RRIT)ZAZM<$o2=Rn@EK?_9Vy`4CiQ3=rW27nJbJjA3%o(o0oiNcxkJ zPX@w3p|rE|kWb7f;h&ncN3m^V%;`D~)V%f3ae$K8eORTN4|)+dZ4?{ALP#S|2G#!t zuFh5ZMPUf_x|V!8dDVQ)HS0X`a!!Ec{_h@?Aw64lkXb#0= zhiW{rqdr&s?Dkp*?shPjMcbgtzvgtUVB}Ag9#iwfRF4tYFgi9=`&nF|vt&3@4gymj zbfZ3rV37lBFz?=nfaX@voRT-H%P(~=6~ilrL#?#O#91SebP5FBUZgZ3qn|0wr6qoZ z%}@8KIVS(K2HldDYZT5s1~@7#9H#4ju=pMI1XnP@RLeU$ro}zqkTs zA2_G2Zt@e=xv%=$9Qs#wfuxBindNpjK{}jrn5R&haaAba-Qmw+sM97Nh`*sQ^(oMG zsN6@A&thBM>fqy>fAPnXf#jL}s(6F`rgJcs!;-&fl^4Ed?amYikw)kD<7!Idk}d36 z=o-{L+)0u*Al&K!HCH7u(k_f}70*l(PnW&K>6J*z*VTeq)8xMe>vXX_Vh3t)O%Q zQ{xj`?l0SzEajAC0A+@ZekMq-q2f{U$^2(dElKkuGxf|RohMUuuUU3dvphH&RWCYa z499;L!Xf_*1Z4LP%E$L3KdM5M5kU|G?bqK zc|PiV)&zuIAp(AH+q|chI8ybz$4ZUwK3cud0jLss6;(6VQ9WX1v>B zlTeymWo6wmTcNq}D!BY#J5X4$#$nrW!%EH)6JyPs>^1qh8aE23euT4E4mYQKc@l8HdKxw{sr0cWj@(RW64x5 zL!J&hn@_Vrk}$D9S2-l{V?pub zL#VKG!!%1y$4B4EuHgJA9|?=AYJzkYYL2dkBS}AyGF7I0j^aJyRn{(Gj2a7-LGwim z-w4a?IYMbFg)8csRFVzh7N>-p-Q$oxYavhxSL0tEW6851@OYDMts&gH=zf`~paaiNe8Jw&l7G|8R=&a^1gJGv0~& zf836z{A==H3j-dnvyS!_Y{tuO_$eej<$My707CA%(jXG8SnB23ej(G5n@JRXG2 zA+n=&)|Mo&^Wjaob-;0v_NvQJz3}q$6g*D?>olUumO>g;qKUk`=#KDOZ^4`9?c&FZcMBJr^H>&d#c!CF z!y4CG+{!2yE$f)X=4>vW%BSFhT!3|Q9DjCTopkg~WjAL!$>DQ;6pX7~gTJ{mfVF>5>sr&6bJ2AF_Ko-6m_OwTTn+6k zM_H_g`LB;+`x@-WQze7_2q-94$Q^tILQwCIcAdBES#dqs+sHvU8cRs_S_eA3{00(!DXKT@6q59TDtF6^$nAXUeVwC=oyH~%!SrR z6fvF9BcdM|4EiYI_8w&y_dE04yY7^&$vXkSG4G0$*RXn(NRAI{W#54Y(d5^u*r;oJ*2O3p*u(|OGQV;vr7V1nzPe1jH& zqi7GIjqq{%6$of`3%}^yL8=$n;(eyT`K=Dsu@NV%;*l{NyLxS-o?nL}j&$Pw-o2&U z_;5Zy(^|S$%RbuUKs@@Jw~)58L+Kb}SfZ7u!Y$V3h-3o|w$QpQ*0O73eLipQFok;;q~X1kYo-$_+FTJDz!dn|B}8J4S&qZebvU041lY!^&* z)D<7r)z!T0kc4r^yd`0VnAag2=-iQNQe!BW$-~u}5GI6c-}%kp?f&Ir{zqE7z-T|! zRt6@tx>|*I>3fC&@rpEA9#5RoLH7Q!2!oEam&8wu`ar%pc8vX{tWPX!E_Doc;`3c? zf$9?SM%GpM$@;arOYB`YZm(O56`l!2XF%YmJaa!n{bvH!VJlv!7|jjofBX`F*84_Us5=Z#fechS7D&FZJJm`}r+`Fd3-t zs<5@MLzAlazf7M-&v+bBwE(fdO!5zV_Jl9+2Ya^A;WdNm!E8Q9L|(bC&2E+jx$S3& z5dpNXPUsNg?v{LRkByA9fo5#WWf;Dcp0Vw94^*G8wDFRpX*A};!klFkb!cn-wF(Sd04eiA$N<0%jhK;g!_5clI1d$9YR zSUxlq(?h$%#rpH%+^Is0=@HJU_IQWdbU)?BXBxs|xoZS!iF3sJyig=;z%{N>8uy0X z!SCl4pn1cyd21P+9~RE-gHuN~hU)u_fi>#_>ROua*nmwx924_1qeaEQ05qF<8YVsY zB}~SfYQ1e!f$CrAZ^~h$uP#o>CzUV0^-uc#iJ)PS=95t1z zap$TZ+eA6TJljUxzQJ=^(Kt$T;RP583)5dy!rTsx6^(#mkK&u&*W;GlQxlVF^(EAzg#E$L5VN%I5(7 z@;ELcj<>49U0sm?)MsF0-VSLV3)+k_mc`lruy^|%@nM#seD=%@@t3ytSlrL!S? z>1-q%MXFoXzc{?2k&M};WuyV&(P;2VY;xLT3%b?$m>OlHOyzw_t8k^)7KjdIwznx5>LK`G1B!IvnrkX$58b&lGftIfjwaC)w3)jaR>R9BjUcJRawC z$K2JlmyDFRmSAI2=GOo?0Fs^?~)l=HTOJ69i<35@7 z*tdIE{^_Fnu=v{IwBUl|(|s@eCIBez3=c z=evM>mCPDq&lmoft6fuu!NjE0kjo$+UY9=|`bHffHtNd1X)Oz_Fv z3B&j4aME8u*aJ#?s-8O5UcR|nwkyNd<-nC^0gijmtaFlDBqpp1_QGL@T&7h zwxL9ykBv>jXASz3Cpr$W+KQ8}mZL&1i+;Uk&Zuj&#*SgM3Pv`41t}#XIv6l`fZ3E>I1scD* zUNpX?kTU0;ARSsY2g7a7VEe0mq{3dTWd{dx_iKXXO!zW?HMD5d40N{7g>&l?aQbOy zu6&RG?iFIl{2h>9z63{(T8-__)RHu2aLCnM)t8325cmIUrXeoEm$tP?SN+YW<6_I( ztvL4MKsd>I_SlLS&v6au9K6p$G2q%h7`0~tHlMW@NEZOSN;lhe-JcP-Xz+_o(MrG0Gnx?%261FLof;sS1;oR~!APKKRZ%3e*<`6SeIG zX$MJ~3!KhfQGGxjunY2Xa=-XWBT>gE|6K(C)Hb`}h^EOUJ<;{Eg zHd6eqC4Rv4JvW4SHxuZb@+oH!$<1qX>Q7MDn(*N;_9#9iVr}DL`Tjie@#i3+*JMHc z#_~ga@uJ=X=o5AXiKDo}KlMAemOf^#KJ+YNf~76fUtiXJfK$3{8FK{TJP z^r4uu$_0eUicYSf*Ra~#)k5h;@-l*WO#3qZ1ykCLJcKqVDiQBT=CSV^zG~BsxZs2_ z4~c_sX{i>0>QsC7Lk7NGbPCA_;%n83`9kvkjVQ|q zAs;YaVVs~$0=mZZk^y;7eh1253X?qGf}3>42-w0|m89%W*>5GacQ|Ld!f ze6{et7pC+zx)g2)&yi`c{z*WUCYiIVGZ1ErYyTQjEiDAX5Y?kqnca7rObmY>Betg2 z;YvpyYj91JR3-uWUdlOuDIThQw>P%@DsaiVv?~2S`#>xZ=3!j#YP`@kmUO2HQa@;F z*O`rknV{xD;aH38U+7!68sX<~)-~sLzRuW7)VJ@Et_ex+7wpnFaq1)f7<7(>XRp!CaB*%DxGV2`_H4iMUBp4{(5`KvP(FBA_cp-ZIP%O}1(@|@zViQ^aui%S zXQ5EpX>13VDnID&eO^#T1XLFgyYXR_@7eICHtqS-ovU17Y|Ba@9B1TVKzJEY_A*~- zCZT-(leVd(hr?j@<{CK>VS}ns*QkKk0GeTC}Ux2Cq(asq!b}`(exVASCZ46wi~NfaUd< zBV|6Aa4i9y<|hDU9q2OW4^!TkG&gyd3qTrF&3(bv^# z84t>yD6dmmlhL?>$#Mr~=#vLD*PL`UQyx|M=Z~eInbO+iX*86D;@+_yVq$zlI_EQB zU{j(pIxTqvB)kX03$}c}ySVmf3r2bwYcJg1PI=&HuIlpH=nvwWeS4q`1__sdvJ)0v z+=NlCOg_LDljd9SSD9->hndGAyjr^A3nagTl*uqPpX=wfla%c+;x-`6qE%-jfV?L9 zB|38ALZC6`lu6;aQ(m0>z94;G#g8w41q;VjosfJD`p)Y}xp=vD1p5H1*>~A7|o7+s1IjpVlgPR|aGETfidSBz)8^ z629K5#*fn)wFb+F$-WcD$!u36Zrk9T#>uV{y)S+c?YC#c=TG^NQ#p?B=sS(g?x)Me z!M*I``=gqTBl6Mpg%$m*0@B9M5qlopK+6-Wx$g6Icw>x>?4`d8Khr%UMLw4BEGeWK^M}O&c&|N%4j7~F$F`GPObkC{0*Qh%%;bIS{8&v^|i|fkjZ%yR&7r{I%X^~vk zH52@DtmLmjznIXU1Osj?4x)XLg?Q+Mydz(fU2uHtuhH08CgH&7$%uMVo^) zc>beSyw%`juz3nxZ}e`c-n0&H|2m4(x$yJW{_;n;0d5_q;d4{EaJ?~qF|uoQ{&)Uh za9Ey!R&H?+~-Rc)w%-jBu29T5iT)gOG{DrSRVhq&?I}$Q^xO z;7YR=+8=Gt!1aJWv|sKsW-xx0^q`q9?rX(5uGHaahHa#(fur*lNdrSKe#j{tsODi^ zdJHyi*Pj;}hC!<|Kk?^54c02vK)zhK8_RYyhT0wNA*4?Zp8j~4%czT3?e%nU?o^5& zdwgdNJQd4g=RyCY$6%134utGoC{tgLl+#0uX@C0wRk!R>`}sUP zWjjoM`wM-d!f7sBA@6q!{%mN@r@VLI&mU(a+nomiTPCmcMX`XZIL`#+qvQ-^BH^B!}5dyqw(a2zudw26ZPev z2jM{H4Gnwlh1dEU(C$RR}?f8q<)p{ssPBg!5 zBT?1-KE2lXKF1mR8aeX*ru8*h)1%Nv;~?usjl(tVgR#P|9;o@9RAZ3bcCD1rxWLUO zv+!G=K=ci)!Ka#C!&v++Ug)gnKkoR6t>d(s)TO78{tv$f+VeLrJD~Yq7f^Hku<#Dc zbu$*x1eh>$GGL|ytEnY|-3|{N2%|kKyLaQvI`b{X$wzmIm+sfXv`1b{)sg;ad)p_&n;`f#HSzsuf<(3D9yzM_P615&n?)} zHV|t4uE{Kpg@M<=k(|z%$Im*B`&XV}gefwco}D~tvYOp)J{#2aMDWSHRoWAvno#Eg z_C||vRl+B_-^_?|1JJbI|5Q*POI5>V%{H;}qqix?2^HNtB|~Vf46Occ2HgD^rfINi zB3v7}o%?t;2V*L8APj3t}$fu7ZH-iy}`4Ffeto0nK{dJX3OYRC_K9Ibj9 zmK&#Hx3hCV@y2tHh3wxGe>%sv*nPi-Qw`I)1bdPA2L5eofhU&lkTZJgaK%5Z?psLL z7m={($Wp1Wi!h2UI?#<17vY?>rqb1OFy3kP4j$7!PgH|^$ePyD(o;()#oz>-zw4QBqdHi*2e%mOYMRt1%k#Cc+G-9`mN$ADX#%)Hz zez4a0=wT6FMr#6=u(1BLyKb#v>~pJASlPZOe^)Y*85kDg%i??5rQ2#kiQROd@ua%> zBZ6JVOB?+Kgq@X`{9*)s#}F#oUFLXvdp~-P#|Q}9aMR%(ILKm;$F83(6|TV!muwC7 z4?VNBL{M!4;kTS{@r-a`SNO9b-nc40iru_?07oQ8u)80t0rd+g{KN(wxh7awnp&PB z+|Z-in+MFQuVlS%f*PxLmvzxGHYoq|4PS71x)o9{?bFzvH{@+b(sRT2zOh&l&(4Na zLjN5`8p0f*qj!vWBOc!ynH3ZoRs-TCu6TiJU9MT@g*_Iw#cLz$VAH;htH!C#p{a!P zZlH8RBj>*Ov#>3CU)YNMcdU~gTdjfvAIn9L;`c!Mf+-Am&}J+4yZQ-hm~^3@|0k$^ z!M{~Y*j+c6HU4q}iO>0tva1UF7-1xip8pualD@F?(k56q{uCaaYt1jMcSK|Ju2@pf zSkgSPr1OcO-ZQP{P&q8o6q}Ysz^v6KlI8|nquQ~3Bi}$^jTF^WqGPLnOutbyCZy87 zA`@%!ZfDm+CvU|z2g70cgm7qee>3UF zzFf^&kUk0J^Jp<#OD&#(J7d) zh(7L2X8S$aT-RXJ-nU^33mJk;WHq zWYTvHl{GZPH&7!tn6I5<$_`Gpl)i!CP;N?EZ!&5-<{5Lp3GL(t%_E_7-b!CXP~)mT zk2HrEImZ_+RJP;%yN#rAhEKlzuyKN(cyekijQDktDZZC44v2tJ2Hfe4Mh0e`k!-g+ z(wsx=REgiMYAPKoRUJ`1<4bKM)rO z9ZkCUd~)h8-T!~T!0OLj{%1of-AfunKO2K%bxSk~ zABd~**iu`m^!ktf%f!Zm-3>2gjm z?Up{A`Uq_ox8Oshmyp~H2tzP;eKC@D0zHG{v=78g$mphp>kT#{^@LdIG!7;PtP$ju z;K!XOK-VTZw%h|bNshcR-6x{>hBTwn%?fM8U%Oh|r@8~ar+cso)4`!xZ>IE>svE_f zs!yQ23yj~Yf!jGFB=s1Yx$J;l173pur5us^`Zl=bOv52odr8Ob&nNCd{~O+dG?Gv_ zc+$iG)tJ|g7z1M-@2Jua>X_uI_~pU1IgP*aQGDoXcTVHNoI4Ff8VhtY^1%VCa3WKbpEPqzIa_4v0&{*N`T%)5mDiY8HIw@fAuZ0&$n1UV&8!3wRUkD3!7aaCM&EtVZ;`4QV_OwG3wp@Q0s1~8extfq@ zA0kQ13aViwUj)}1&g8z!o`L18ro4TZYfNbnrByEssKx0TQTfizjoR?osa(_ZSr2Lc zeKvj89jdpFV7~VdqaM)=k%d*f)uGvL>~_Y2ypbhW^+?=KpJA)CxgZYw{6^1y=n zU*RK<@it^LYu&&N8ef_BXFNvts6oDB3|zE*fjseOl@=U6^C3{ri};f(S=#t_g18;; z?QUA-v2`O`%Hh_H2-j#&Mbq5{(_6PvH7T|`wS#l3mqV?3J(X4m(h8!ZX>URL41B7` zqrv!p8p0b^r>F)LS%yP0?}TCdtVstXBGnb)`7FY=OjJI6Pe*sK&d3Ds3zqWj*gi=0 z4hub8c;MJIsOFK*S(1)tPi!TSHeqxQss|PCYkCJ}iPIn3;DWQUu(OYmwo8X5Qt|zu z+g})IN9Bv)_V`%Bnru+oHu$!q-1#jHZU$Zyio<7oTbZxkLtKmp&0UdnN~bk$57_*T zvj`*ZV?^XB&ESw+!f69Bsm2vJnb!nJH}Y4^8YrXiFevSS#0wBK#Yj7}d^?g?1snZO ztp8C@+;4J(H8hXGnKBv@i}pf;Ya1}}=}jb#lCul9qtene7jli;MAF2x#=@**j#IVGTsIXe;;Az@zK=ndgzL|P-oOF!Usq!KEONVOv`;J#y2}1VN zW=k&aWYmYOZG(7J8A?LxdCF+!BY9Tz4sgZIOE!zM!Es{tOm}g*XH6tO2PfaT2&y%R zj_Zl4j;g&{582_~oUp%O;4b4TELGRDdZr1ix)Beg3E<*My7!#)=Y#UL#5pfv%)Dyc zz57u7GsHpTSvU(R?-Ar}z;VX}X7V`#D(_8U#Ano(E0A;}Hgdj5?>!>ArWHWW?1ebU zF9^uTs@f$jb{%O&0_I`S2S_W6ALXlI-^f|0atWndPVXwwsyXqhj77?isAiKf^v-VV z(5yZjJCX3P=is!N$V`7l>7OHVr>mCBH54Ga<+sA82tD)Da*{C>u+``q2d@T=v!!(E6 z=sajG+l%+HUFLI@H)xfHC2vRPV2y;sGJW}JK?|(l&BkWfJp2&t0h&lUlJWk&_u%vO zQ0BhF4U~82ec6gNvf5Aau-nA#wgC2KTB)D z#KDG=>YeVZYzEb1_rtj`ds$;*Ltgt41L`w8)l$opFC^>(U0$s9@l3@n**}#pkd#Fz ze~9EiL20g-gd){v8s~^3(Dnifn=N-S*KNI4eU{&UYc}*L21z{Z4#OTAq>bgUr40ikH|b@AlC0WC0^jPWovk?&`l@8&or0 zc|7L)s3w2kC5A1Heos2ifN#kTBpo`Oveqsd(hgLY50EsS$~}2(PO3~0{Z>4uKDf^m{>9Jg%uf8b6v;b_X4Xp~Y9f=RI=P~AyM2OuMOE)DX*62W zSOe)8PCvtcw3avNJ@QNz^260hVC6=8v{Dw;;mR@Q_^vY$-e{FyB+t!yZmCXL#{YYp zxSZ-}0yNn50LX6wWk#@MiH7B__W`Owq&mUZ0S(AEMFQ0=PJ9_Byk;N5rQ0YkA`Dfv zqVP}E6ZN6WSv2H>aAkgxQ2)OW?!W_UG*cP~2H%QSy@uqk3f#M2*Ak~f?~?f#*=-6_ z+V1lW(&co1Dtjq#aMu(p{EBo9N+$}+VUWBuy0Ae|@J|P$^X;L0(hi{U#@}-W!NbI6 z(5x#ChUZ2Q&*Ci3>Ns)so&*KAr8^xSb7H zNw;%E%>Mhw(9(_bpKEwOoH*T$@ojB7k{J8*S3i{g>y!H#ogiN z+$wXh=6O>`K6bx8kN?>acV#(o`XklPJ>EUS<%15eIZa!^-bHPtOXJ64^2WuYYcp#e z-C-aP(HR2Yvl8Km))V{lS+tMV7-{a_U8-Y^KR*w{Pu{}zw2#B)N+UV+c_uD$Y04W6 z%7wzWbD$u81+<)WPn0fOC>M^|%va4`jooH%z>n#vVx+GZfAhQ%q`2oo*~me1&*f?| z$jeSkLZPClrNKW0AK^o{@!>8=MTGC_93+d^UhT=ye7n=}ZXSPwSB8 zOVHelg^eo3ucUgU`!?WnG4m(! zNG|4-MT@k$%jFrd7t)8nN81sP*`K#F;7$;OlPwU(*L%*gCd`x}=Z3@0`{!Wyh~dyF z>p6mTEYw`E0R0_qWA-F>96KlzX3xAX>VEr$c8Mdk;e+epvm>;B#7Hk0#;3^&-Ds%4 zI)v$cpDbfznu-Cd*TRjwBwp)Sh@G;gqjjGU=85L=OQ0d77!)xz9{ap^ z;EIej%ay@T1us@8ipgl#RUa^c_8*qKf zU;G~AAy4<}mRlJTgvpJ^aMzI+QPXiGu(VuF;L&#cRmn~?6dlpT>ZFL7Qj?Dw@Q6`; z!!+H!KtEIOh23)}Nsr_({@9=ZEPBr2G}rK^(L1cax+eE(agouu!Q$Q>_??!)y!KWL z`PncX9HN#8N4GAx;q4+ASacf?1lWsYrw}lC-(TaMdKEt9xgd=*x;i|-4ufOS#M?%O zxSho5-n$shpD=vUluuc7TubkTQ_XbvE7E25sW-sTY?H>vBMj3X9Kt=*vkCL@yz&U7c^dC zEJyDgA>IqmoKf;Dw-!xBR_!=<8rJp2b@W&>00 zDaR9RNQoV%_ky^>nW*2=k~iJ?fc;H9t)=${sspXpprd%mVX$1P7s|&N=Hk+_-QYCy zj;b*L>&Eg_HQN8pE&(h;b@`AI@_2!p(8kzU(i{ji#|{52(PTDS1Pk_f!2G8Z(b)An zYjxdUK6qispVj@0nAnFmikK{pzAMA-A?EV<&H=c8aw2m2x+YssSB7j2hdARkGzM2# z`7^G@iA^$fm1%WIr71$8qa4gpaG_NvaRHH>!!6BkZ_vD96M?F}$}c zbPBYR6FVm$-9MI_rsC~-qlL%fBs9Kx7Hc_Pg1eo!Yu<0MYNa{ZtyINPHZU$Ui@ zTpLnP{z}@$zQ*iC$9r$Yii~Q!dEpYUDr+gC&eFcG-{yhBYbTG|yh7Jp{!aM>D_eAh z5!gV|*q}~^F}|HJg|{l*ir1~faAnp+a8B@7wJgo*4@1?{)C=O5BY8{R6q+A8`?^yB zP;bNEn`vk}gfi4lL)oB*dOT!uGW?|HZi-I61Bb0qxa8d!N$)Ki>Et8TF^*kdDJJ|p zAXgZ=$$>>9faVkr3_OeUdR7(~0fqJRfa+9EZTyJvGm`fxH36!9=DMxp{t>;9%WY-+eN4v^q85>n!K)|e%EYdQc&Nv>ccmwA>tR|Zmm`T-F(RqN6 z1NLL)!i%ukKM?DFj3wNvDF}zKWML1k#$~G6T7EElD|`8L81b{6T$Yv%Js0JA{&MU9 z^m-n%b1?2niUq|_7TOYRt=tzv@gytT3m(Zv>?wW!bnCkhZi?2h*&d;wt*Bqc+ZNAh zoqMrX8(_SiC%xDz{GvW+343v>aDd${jWNJjPZG|FwISPp=9_pX35WK44yB`mab0N$ z#0Q=ATyeiIr+Jg>l6GVL!MoXC*YWtF;|0Q_7eKvC<@+-4;q)T!`A{dEyI6dwY(y+ zGH@rrhfGcDUe~}^=M;>rHJek7VYGEU>GQx%@jUO85ez>34eAfk=M78jVav$VA|v6c z_S>w3;^%_9*y{R6F}6hk_S)P79PiD>#Rs0jF`Icj<=#X3`!t2=5ZH1Hj?O!$8CUoL zNb?B7Ibr0~P(B~-r==RioHR$Jq0n_?Z-^?3WdrOS1@X7|XVpxozE3lXfe0@bIU(o* z5D$Zq?m_C4dz!s9|H8eHP!Sr?4rz|r$?sS3nDu77e503CxT!SQb!#KBpaGqU{FC-Y zJunJug;=A)W~GZ3P56LcTO0=+zwg-R+E+ku<_|g+a`FNe$k{$rOh!yozsiGVxGl4G-}%0gC@UL9>Sj za>UuM?9&HpuIYG3OFRO_m3p#Y=07@Kl<$W)3zLN%-u+0ePr0SkU~4 z_76OzmEJZqr0)`b{Irmybp`c~q#EM%T3Aze5NWvXPp9#|@j)3ESPl<00`9G^YNLm#9OOl~c#A8-5u^+0Q zuFpRaCZE;*q1=Rcp-PuJ|DbbCI`#qmpn+JBz8&;v{YE5i_9Q&j^v_R+e(6nkXu5^! z5Ao02kaU-=s%fNq)!Zt4LTBY2$Jf`sE4ad$HPbx#)ImMm-Ra{s#B!?(a_jP^>fvKXPv4`x;I#!=Y=7YZP?CQ z)8u&L8vNaXyVx$kT@r3-Nh`zQYky$E&sOaFlR4t_B1d|bFJ4gn%ENn{iBo3Fj$I7k zz{fD8S|y$w&S@NBN>Vf4xVKi2?v>;th)Zg5njcB8fp@*_Fw7`UOI*TL4HadZ2(mcL zpjw4EG!o#%{T_T|tv5)02Uwf~#377uf;903G<=Z=R5w+c@A?2=k@T=rWX3H<$A}Jg z?OE+}lYz82BQE5AllP0_l3fb3;YV>ho;~RV#<=fhsnI8@aANcn6WDvxS&+A4|M|Sb zJgXF@yp~3nt>0yiW}6IXyQhwXgDU&@Xc}&a%A^ z$MZaSo%DUs)2ur8uQXP@UR4`2=sUi#1$E?&DLOpgpf|5kd|m`pSjZuJnBp)E^(9^N zFLBPhE>ImyqcPR0^mK36kJ?q=Z;>yYAjl`;%WOkX-b3ji;v`1gA?W`2;XOxz_?xZo z<*nT~u)A#Fby{3}(3W+zUs8pSwnyCs%@dt_cL8eVgkZn)1cmv;-8JBHJ1g9}un6b= zIE{qoj5rX2mQ>^2IyZ-9<$>hAhT!RxAJDw;n})n+F3qWWFMMjWo$ZgBDW>)+2C5Cf z1rvnwVx)(J!tj7jK1%CCLFPi18xzm2p7rC(XDBRESU9vxjVjNt^nKEzEb?Fdk@^!i z4$OyU83w3)9`U9iTw=X-TS8K=o{T&$BQ7Al_#W3hJ1_2EAJ3@$*u%fn#s_O`0>6QmuhdTG|cUM!ZL zd#7b$yD&edNLN zn5wJXPNa_}X_bxzla+5N*DhmAI}QQ;AwiICw+~1U&}%Nj!_=;H_Q5UmFR_7#-VuU0 z6lq+sGIt={nw6rG^YQ%Vbw_kIPR-6N5-C;10` z#mC~vVl(p4^jt--EwB0P8t%G(QDGe@uk>Nk5m0(J$aNMF7b#!H+Xb9Z{sPY3bP{=6 zgR!+)2vFX~^WB_f-H(kWX;(b|%@0V6YLqv+vUodg*nFDw=^^oP6^D0=x4`27JEc=J z%47LWwgbxOp#5rE=XBt%mgW#LW;e#BhNrRrx5eUwVHWt0JjX0C76@N7q{%h5EoqPP zpO!!x)$^}zq&RWEfL)l`M^OKY*+xx(JT%SMG^7jyu77Ao9=SH(o241kF-%3 zh8p;AuR+dIaVKGc@{*Vs{vMUiAU=V_XWdjLg&Ra0xHvcr$J{R!EkhbJt6B>%CEbiu ztsu>f>OD>wsi6MH=RaJ4e5%Sy;a>PlBs~Z*5J>v&3ax|JN8&{6>vjr>`&G^-NLvZQ z1@u_lT&w&LWrSjp4#P8>b15TmW0(!$8}13pTrgu-b-A}@ zU{&oD`u_mR#sp<7SdjS{eQ#XXQr<&3f+>0xY)7hJVV2#L#(E==$0lF0OXWfEZTEMW zzGOX8U$A)(*Pzln$&K&fxzrx$yEp+@ z%l_D)XdGK^vzWDAVxY1&KEr1Pl13uGuFJ~?+2VUkhD!TWW+9ZWBCSI_JC;dPdqKPn zGPg|hzRCFV4{hLkOFhbo#&LVAV_MRt8kK)i&PjQJ5uYj!k`C4Y@hMWi!Fq{IdUn;C z{FyH(Z??x#%akVEJMkLmxH~9cR;2+b*8%JtrRGk%DZd3jow5uqUsf0OJhp)hq5WW^ zUgc(Y>@Ub)0%f9tvS_g=S)czJzlywtXH{mfV4)9^rxPk?qVd!EJB%PL?uOO2t$|_M z3qW3q|6X)}5xyejMw~n>#I`uCxX*)loe^h>9gAbA7wS;HnMm(d0|={-ts0;*I8?d0 z-z|N4@M$@o$SOzTJWje62ktTvw+i$m@vEdvig08T`da+UQ z%|y3)BeYi+TC*0*>cPu^wb18KePLg=RAbw)5ij;EN2hbKVsYn6@EBvvx9RIkmlGl2 zL+iOlbo+!U$2I)cv@T-s+CrG4oyf-ZnMdEj&|2@0w0FL9g68M!I&wqS8yx#$3D5a& zxV#Z`2uxcU$(BYnxTeho_R!}VtUF&D-W+u0`-gUAN$o~r)|4RZ<=#b3to;zi9&XP^ zy*e#CFZf`pcc!pyxC@Rv-otbkbd&$}sKsZPjm0bV_KU|~6S3l`Dfo_zWDf>f%E0yO zggSR2wPTs+|8oZ9_S-22%+tuw&&x1uLNz|Vg|0jd*YVQZ>T=!Dj-tuw?A%#b^pHLa z$M}`w*U6XI+9?frKYu;GVg>NV<2SJJE!M&0`M3GsdXf0hXRz$uw=v}Wt%O&NI`iCq z4e`u;H(HB#Qq1^~3un@Q!=R^Ovd6+JcsVZ!eK*y`kNhD!7*5+cmsRJ>s?TC)(pPcM zbbF~gteI5faQ0Xj^o{-iZ_m)#GS4&u`;^c*VY}h-!`5=-tQMl~?g?-)EL=NbOLv}Z zw?(d)_E_x6;<;l)`a$fTQ9q{`-N(( z<$5x-Ais~xXd-xVby<+v7|DkM@p!N0C`W##L1?)FTJ zTR&z{-u)InCElj*IG$kWEK`)1W{AIi??Rsko8dX_=TCp9nqDD0Ih)AMF@MlE-&F z!FVZN+<@uJ6Tl>~jl5@c9^9w;${Wd7)ce86&|cW{W^Y(Hei|NKJsvV$Hp2Yw^sG?N zi`aJf80qx;A>3XO3oAEA!Ivk8*oqY>=o&Hjb^;u+evV}p$(T5?38&fv-o#bwzulhi zeqD?UQmSKt^8=WqzmxT`xXDh}J_Xe4Vm9k3LxyYcomZN6Og|54u)H&>y1Gj_%9LvZ z!FS0CFsHLNo_jd(Lwg!%FTLFWQ)r#&n}a#9s_jrnES`YbOQOW1tUf$wa(7x17dD_S>Ao zU-+eB@xKPV=go`adX4%rpt+&AG~xn2$|^+vyaQ~0;9%CJeRtY_wHs!Mckp1&DYoCe zsW39`$ZT?&$rhJai0w&vpzvhhCvRyQxCl#5eiP2YVe;_Rt+M!Eq^P#d3C>=lH8~*` za+^(SN#g_&buAz;yq~yB-whtS9icI~^Pa8n?u_ewzG#~bZY>E{MYY3&A%2Y^R4e+5 z@?a0?)7%mgCnmrMO>4N@_9)g|X`t?%rT(hJsh&hz>z$C^@RevWt>D)BDV)ZQHOTWu1N$WT`dA=4 z^!*QnEE2Axdd%p5KORGEJ)eF z=`-NI6bG(Ma_NQHho6k5GN{_TxrQ zjXD6+^AK}@gqwuHrls4g!E{y2502{Ar#T`68;Lq7xq#B2F&q74IfC8kS zbA#n)v`@D^Kv8rAgAUlpKEFD^__}R)Wob=Lyd~q;Yy!rcfmdk}*t&o$t9}WGMD69> zQp~WjmJdK;45qEfz=}6_#Co4aVqH{y-fASRWl8_7AsoX+XCDAz3ct3mk^F=!Vfn9S zaCqbr@ZRMKv)-l%>R(X&cNm6nlev!a;vIW@ee5r57dwL$TX}%OWB)&Ip**bwozGl_ zlK!q}uyZo+TgONy7C*#T`u=;}`OVbJg;l(CcghdNMVd{2Yx3W;7<) zdQ%d%A9qJ_3_O2j%uj^w2jX}3Uk4``ZHZ!=n~przZ6NEk@f#3_0dW9((lQ>8SWo9B zcjiHX^EjICy$S*E)*XOj03-= zH({_#2AtS`QCK;g#_!$lz74Fh}eZj|Hn{q$n9dxEn4I0l0sy2{#1%pQF$*Gm= z(8ZTcGz53t`oQK;+b5)}IeLR!v8?XC4sgkya@uDE63dQ-mORyCe-{l4latkEX|sGjlQ zJPTIe)R~bklceXg?dxhJaf2k@Ry{@QD`+43#M@$HNf@KKW}fL8)Z3B#_KL=`a(5W~ zOu5PGMefAkGkZ#hziH4Q?-HXv1awS?^6+%_I;I@`7uVvqqdE9)+RE=vSteFqEJwm? z_P3q8oFDf!SB>8w-Apx>LTOWRcr)zT-k5*yK2}qtJq=}rx{3q%(&SILGkqh-E_#ID zz2HepD|}hhOzf^TADWzYQST2lH?T%GK`QJC3$P{Kld7;y#%&ymhjk;+wGpjhb}}RE z`wO9EH=*_I&Wgk6zh_VzOW*O|d5@Df9Dz9lno8165Heh=RX9VMhIRYm%Bd!$i+(F; zPHW5Ce{P7Cr6HPBi)@%-cAp(H*pxfi#hz8X8Hvkwr6TdHRCPc#>G`&D2oCO5U1@AV zy@~W*w5Epk{*3#|u6;U5`&R4GIU3<=JkcS`P()9Dt@$1DQpCPBlZ17Q##cMfbSq4X zSgCqdP%j9Fl0!K3aaX=@{4>?#TBW&Ce{BKcCsofv>B(7lNHI~o)c&FLd z37c}+z&&ozyKgLcmf@7aG(h|K4sgF|B3m<1D!!sU!o%TdD6QRXQUFt3y>V5qr6OVG z3lGxWxXr6ChJOCRSeTcf-bT8=T+$+-w58Io&i5CAakJV$wVPh0Ta0^tw!3>&l(HTnP8o9msLAw4|%S%PV2wrpmggyuQrl&3nz_?cIVq;QHx!9$B1ZS zT_7!?{hWQovs>Ot+S6oP6(+e??$C@ax(?4R4R}vF)98y?TYhj}3@W{wCFY(O(eg6& z-+J*aH36yqpfs>O=+;~-gKsy1d7c5F?px`cMtw6dWmmM=(X4_M-I^o`@0qGG(r%=4 z3UI-#HF)TIfVTKGgJ-#QgAH-bm;5Q zj?;a!b(crLIX!#6x6T~=*L5pg{?nKIMuH?wLwn_(0arI$@I8x+N1 zbtg9l<=4U+Jz{+x)CR*J@~`Z~{@(a=<58UV$ds^C@j!1zn4`%JzQ>jow&1rf zdJ^YmXiMX+1Nm7P`rT4?7*+>y`Uli;oC~B0M6Z`gP}Y1l|J>w@;(N`W(w1;qho`f5w$pF5=b>e z-m<&r_|N)O<7H?X_(j{rG>I@ZhY`n$rF{(;y$|GPpTk4a9W7oS#S^o1h4Knr+UJu; z-6sYNHs>oGtdV*XGY2~Gd#ztW1jljf3iZxpKfCk{o*FO;vv#qrNB0st(5Ly9b@8PbC*a# z8VM<9sM6KBbk0&Q_p=awxG7N9f*X3ItML=tPd$T&v%Vr_Hwu#=i_XV*Up`$+^(9C* zLPSLz5>7LV;yQ5Fc(ImrC?mZFNnf^Ww`?KqDsKohUfBBUV<3I2z4)z^QT++Ur>pL{ zBJqUsn!>33B&hiqS8-5ze%x}#NlQH?==z27u4*2YzaSog!@4GX>a#TH7-I=7?`~8c z4JZd8oNt8+qfAGHAaSNq)b4m5-1HWQs3{V{i+dH%7jI?bKNaYAjii^5I9NO? znFCEfIB`+2NhqBn4%g;X2jnG9RJ~DNT&n6p>JJ$X@8{d$xN{qkxPy_Ofp^R2v3;K| ziG4R)N{g4yNMj>sH`yzuy#EZXcFf`-8gs}q-#}c^Uu8zLJ|G@`2PPu%5F0Q!I+rpn zNtz6*pRmxX_c30&6xY33iOOdXCbD{z*OER4`dv*}Y)>E$i=?eYwMF6ZCo_&weju}|RC~MGAe+%M3Y}QdC!|kRf2%x0<#jSHWhq#BPXeWZiT5ab(&vO@K%NswyQ_Q!s8)pH zHmg<>x=JT?LOB)wZ_z`WEu|@HFC^1yz09Gj1IWYC4$w-Tx1!W7UvYEkC4v>~w4>Y!7ZmAZ?KZ44=Y@WtbE~z64-??$K&B{x$i>9B@ zGl!xz=a&MBE$` z0F>=&DFcMS^}3QUSgZ0zs#mtj!v_c_=?ofUNtq%{@M?iXzwKNLWkk*t3 z3QZ|z(ZdIuCPF%$3rDq#U)!t~r1?PQ3(9-`V5Xqvi@dGMd@(R>o=~1wasRBu5TMKn zDRbfE_Y~$y-4^?_Hdpf)Wf8)xRw7W3Dh;QlJe?8dfbzDamzl~A6WZHg&fgU>sqb&} zD($PZ36eJfl?jjrpfTtLf9o|s@;qGWqWmjURd%iNI3&E{ii_60`2~bG=GSbTgB4mz(fO&)m5FroF69y9WICyKrpfg4nat36}k&btd61ywli?tkamaY~K)j z4wF6Q{Lb&e)_)ylm6oB|qasMyxeGgu+eVw`4dR`>=zIE((eST$8>9Dv9Z#RL@b1l} zeprUMHK4oL@pL-Y@@voI5=P=_ucPSrp@ez*(X#_zzcRgyhT81&JGIV*MUK-}&@a@f!p@OEVq*w;A%_w6=9^u0v-?sy({ZQ70d4!({12U&9MTFTj))WE*9 z{`APRwa~`VSn~0^U>ogG+V@5z9^W(p8|J+bKkKYP@2@r4tpWL{msSI=FT9MS?K|=% z=HyZIR^b8j#eDnorRY1`RD7|X&#q+ZbC-;OvPw!>exqpZn&JpZTu~}c zgwpq+k7$qRfu{WJ#AmqK?i9X?k3{ox4_W;2>e8V9RAf`miY_+g^luZizi#>Q%h7k? z(K9Wqx;YHDzxaoA&-_G^9uM+gL}#ZQg=?GZv-`%r{CL=l+;__ipj8J4xb$VcsJycr zEOTSz7t4oub=i09OwY+=ULMB}83chzn^CC7YpldchxewVEIIY@0VR!l0K zCF`^}iUTv(Vi#6hUfH3MYcsX%T+<6|@S^Li{R`&2$zGy<@7U_+T;=Wyo(C!SyJ3Dr>+o(Q=pR+sK($HG5| ziMs2g#oaKb#-D0QT{B_&OzudWp`-`gD*#DY(FT-@m7R4n8?;c#~D5Sd&$EF?o0q z?ist1?$w|6{@n`sjm*)o?=aZ~8VaY^w7%wp72h<)7_4_W^1#b>Fn!-ZNLt*IcY&RV zy%wO}*6TcJ)MKFf7vXK6iK7==%CcEo;MMMUh&giuyII-ups-9vb1eJbXux!L%)(f0 z3$E%4d#_+taU2$~3-I6bb8OfgOJ2)oj`rZkJ9u^JJbAz6HW)V7md{;b21AEY4*9L0 z%x~OO(!5snFI^M5?K9^=^JlZG+mDJdCU>EKd?3DFJ4mQ|r_Yv8{;Mx7clYNm7h>^s zT)3RMVX9pDsWsc&S%-CRWXdbvep0w0*Qag5&mJy(i)}j`YZoL4Pe6@5^)DMb--X}q z@Dk>Y(W;-pt|Wh$zW+AX*E=l|SNLdZ4fraqE`5(c&lpU37>v2=nxP!O84-KA_ zW54Uhl+%`bs@lj&%M-~X%f;;ueyFg)e`GRmIUt0M>Rk(|CqUuX#%b|_Y9Iez>A;U( zOcC?O^^tUsk}!<7f3A<#s~7ODO_;nnw-sOcaSx8JSE-p5Uzh#9ITBhq=b(d8J?THv z23EZb1^PX#vC)!;IeZiKn$?%H;^=wxs9(4`XA(|1UnJfyU5oAMxs8I~-MCm?OHO{= zL>8Eb$ZyViO#NPbY_3?JW+)F|4`iM*{?k(Jz}rdhU{-83>6))A8x|}(N%Ja8j!W&E zfDUr{DI=M%(wEH}R|rRnPGOouDBfxR9}FAvkZI^VxMwL-VB+&m^3>P}jC^z%)*VV0 zenE4%*+T(hGU*!ct(ayyOy=YVi&2g}#H{$Onr8=QE8O8m-aSNJ8ykiDeC~w1_$DwH zIuv(>r)jG|GyRS*d&nf!5zzg^UjG`>-)p*17_DmhOUF2j{B5Y}oU0!42+9#lD_>B( z{6&}Z+vOea2GZipKzvZkE!U_w$9+RGpzv0Es*eTueE%4}&?D1>YDOD){1dJ&iQ*Gq z^rUA7U091Dtzf%eH&`%CmuFsnfP{@|z9n57>TaJb_bfXFwS3;f#24qWhxvL?Jx+au zZA=I8)kg<&g*CCc@o4tDhIC75#fhWv(%s&W+CCN3XMfUk!;O7MV#%?NqHe9{I5p2j z2F-hdI)~fI-rwr*5&hfC>8E!Z97*$yvBH7R4(uoktyj=afEGoV_~x|7GreA!6)4j#d$<_p+Aa0ePp-^1cgM#9ytHdOQa@~mbP zBODg0J_xf2^OCT@q7Ki~dBYm!f55rRBVd!aH}|qz$TLjrI7BXX-69%mo6-D@)C3-z$!d4emzB3aiX!cOMt#PQ zZyX_rC)m@}MbNvPo`G3#A(!S54;t^zC5&UAd}}K%mV~dGUu6^Vi`!l}zVWsgU&BVu zS+yUoZD}kgUo3~4g9-bHpRx;TOX6#sP<^{>-F_dAbh;_7hwE~MnU%Nw@kf0Ne)z4Y z-0L4qeBB&_L(L`qo?Kop9N$b{0>{H<@Q)*!Fv4P>o@ZlP@5UJ+0eF00oH&*$pxv88 zu)OXD=rq(LK}yFfo;T+nP?##I zwpr$tY1(E>oq7Lb9VA^l_I&t7m~@L0R3{?y1Fqsp7%+? zF|{Z0ekZ+&hn>-P;6AoAgaR{(kIvpuPh>AqmblyNpV2(D-3= zc{Ch(rXzpfxC+ltZNWyE#U>U1(h#@8v$L%rt(}E@ns)*_WG#i318VY_342t3u+yEj zK-etr^syptd#L?xmW8A>WNEe?I(!-nFZZ}&i=j@u-P>cZ>VpMNev*QpSP`3kDvY$1 zv8;CFJX`4L&gM=Sg>0%f9)5EKFCU50`W)SbcY7Q2pLJ_-rGxt1cI8y_g6@mOod_3Y zv+D4HnL6^rXgB_k_J&rtO4zFy?Em)P9JfRC>GNkijUHE#2|t_JVB-9>rZ4oG9h z2+x4(SQ<=k#^nUsN8>_c;s!s(wMf{LOWFgyIv+=+MG5m2j%yDZnaTZbBcQ{Pf3Ufg z4-!{Gcgwv{bJiL7@#7v&EyyDtuz??@dqCAkMC&p^SSYBcSbbf0eyVGsX4<(7NtnZw z)>wPk8LvckRdWZO#^~UMpBdEufALJGbl9-qw|M;6Mh=~y4GQzNnj@i?77X*2>5~?2#|h(+^b=h8Var_8 zdx1`-n^ZW4lR8pg7hv7yu`K597F77q^S@O{bBcHR#N(V*cC6otv$$#U294r`_9cbj zo=oSmmKUQ(FtA+`6lRGRNhs3ma?$T3t#dXw1zs&zVzr~CFy+6^+T;bBp^fQP(A&~i zTk+-1mvD;kpKUeG+%B&D+UO z+G!M)jIO)D^J}9kSWIWIQY~Tkf5XAw>v=Be7EOAcV_-Y{0+8NjquS9Px|RX$s$Rn~O*xvDYuOH5@EZN||(G=T%o z9%!{@h0;Qdya+t0(B*52O)%8Y6-iqm=>&d^@^s=TL3M(r|0v5I_5r4!iIaAN6UE9; zsTgo#1@3pdtv*+A7W3IP4czuOl~hB@+mVMT5>yXJSfFrMT3LC&J=pR1&2oC;H^#MoB&6t*M!!AkjFB=X9*rbuF~{h z6XmVMdRmi{G-@&@-$eHx!GG4NM%v&QPJXl#$)j+?DYZCxD9@}CAD(J96-Z-N;Rk7S z$yQfqZ&#&(En7v;%2bDX)q5$f$1cr>(0DcBb7vhbz-SsFgD z=VR);QhJ(Q&N4x&ZTxq%jPyk_^1N}Z@(U2}B@^~FQ=Sb}4Fu1{ zsjz%`J!1?GsiUrw&dS-(hAij?en}@)-(kh;N33Cm3v_QBr~OYi7@RJTLaH&P#o=M~ zhWyg0KSI^*g`f5wuKDXodtDVCA^Aw!h_uUmQ20c;gr~KOp&pH)9yZnP8@!je#+a+| zpnFi-Q7YX`nw7M29_!Tgd{sS?-@!u$`rLEIL?B#Lm@Ve4v<6k{$}3nMuLcwKHUVi( zI=|^0P(1;0BO^_Snl<6jzVsUI-gShz92&1)BiPhSsQS7JLjxRfsP8iJwGl`-MPAlN z_`+!QicjJ9MDr%}m=(-ts~GSJH^DE-`~kw>`uH7=U)} z_f$rJm7jCq`2M|uK2Lac>4cfD(==O~Q_atJkxPI7fMurfU~&2)=>!wXHC{6E^@4o0 zypkD&4GVr~?c9cnv8|#M25_orq}~(c-%yRC(h}oqB(vMj{^D2Je=yEKizx%baAe00;vIp`Bn)tZ0_>s>1sn={fQ<%AN`aaTw=IFNRo*+!5xtfE7d5|>vwbIx~niA-9 zRh~lLWhE+JGW)L^UXJPm@y9oS(odwfkh}~dZ;voJ7wXompgOpV;XOORXOCL5fVB-ul$|Ex3K+ggTT7#<{%PmiC#o`kM8uF6#-Gzaw z3*{w9v)pCm@vHKJVMAw-jvU66mLR_(taly4s9(C0aJdRwsdtsGL)Wy^ShwbQ85HIO z&X?Mv%Af+SaXBE`SA4OW0bZTmAl<&2RG8wI3)K^-{ zw{XXk@d_h=dV@I92A6-kNb_YR45!v*H1?qSn!KN&dh=BNB-Ag5bWJ40$9l?l<4U1p zsyjyh*$x(Wjkt5>Jo2{Pu=rXi)qssScECmcp)fNzWDBhL++Isw2&v|oHSO6EZvT!s zzLP+_BK-L+9FS#@d(Kmr+Yavqq+?-BotgM*-5od!v8?^eDD3s~08-8(GEDYU=4>jP z1WeMB76j6Xg8U5b9me>*HfhYb=OL;A9hBYg3d;UOST9e0EF}Xsr0Obv56b_1H)~Ix zq$ytfwu8N$G#3o}l;Egd)ui*`>9CH@#kuZ(nm*G^s+^5_7uo6!$`2{tLE=Q5wWa`T z%)PC-wj~yx{I?x`zi%K(i!fC$)`71yaS3;@<-jdiQS&g6_EEZnDKAPILj3&Noh`N; zgU2F%K+~1ils|)v+KycH%CS_YcpudKdTuyKV>JiKL*wTr%YbwZdf!3OG%AAhb&wio zPWn~N8JFEhDLn&2B7*T@^_@yLOP7PUU}|1AQYHc;`e&iSI_gPAI-I>;`M*WKEb)~kORR)sZ;9WJr3WqeLTNIa^O!3i^O2<%7QucF2`ERB&(s28^Y_D%O zIkHa-wyA$fkWY}x4-B8Fr}TzUm}+%=4I?hmD$WUWSW@MSl_%@@+l%zablCBvg6&FP zuDA}SobQVrinMHZOd5u_ora$4TWA8WII`@F>$GMtlXB@RDvx2L?|?EiW^qSP{(c_` z+G)d-zh>m?RDMe5$2yQV8xAVhBD@09x2g^Wd88`sFyWOiY=}4w1F|+&Vb-6w)y0l# z_QE3=1!Xm$SsRJU)6r+)<*W!gqreU*gA@lkWI>HkQ&3qX^$4$Hr$x$21--YzA9`l5 zhP>Bm7ZL{nWpWzI$Hb(f&mybTmnRSJ3jx;)#M_C}c=JWE_@IRjDj!`ov!bK*i&c|2VM)YtP; zc@ZZ+3Y0Me;XaaA6kBVQ1Nl?5ZZllIqH`#G`_$x1TNk72zsKTA*EN_K_eA@1Ni~)* zXC?Z0MRD_UZ@}xA3*6t{iw9nLuYKTl88*1z#r*L0SZ7HW{(NLlX=fD0^SeyvfA*42 zj!uTf-NLYblofkvc3r!>*IHTIdOZx^+XFrfF~Da2(_r|MK&%|y1(vq%$8TSL0$tnJ zltBY3pv|l_7}D(s{1rE$;n(*3WEAbU^zFOwaO)2CaV~tv8eOnFX^QJ-HsK?i?9yBv zrO&rCe4zC`wSgU79*pllXmC!mNKAi3=R%Kri|KPC+4#dJAlK80_5wHo^mADMAP%3L zr}bWWTk*!@{(NfT2*jLqFxKUXrjLA$Ek4)bXHPbfJ$hV2>qX7*dHed@!;j8wy19~t zW-b(TjiM~^E_OQq2zFoB6J@XV%65;tg8K7dZ43^d6pl|9>C0=`d922T3s|vhH&%aC zgKg_MgFUk_fxCT9(h(I#aM67;d^H}UeKagxZtv*H7u=b~50nd>RI**%-khPi8?}~C zDyh$()eeC18Vx=ehInbWsp{Bg+H?f$4 zTj23^6P8vS5l0%H6p=AC<(7#D;N$nLsIiLz^Sl1cmew4svG@ny9v{%wT7Ls<-1Wq= z&R21DwMt0q_YYdU{0C0gOnKpS3!d6(6aD)N(w~>{KfWLO&eG#{XZqrwNzq96Dx2u)*b=Q__Bf26e@sjF z#yah8$9CPn$F3Lk=HKSEm1_PqcZ=xx)SWo4ub)_KA1t=&JIM!MqcC90aaP$l6du|& zjN`Ei=Pv>twY+Hrp5@n;x%IUB}Ley4GBdWW5B ze_|Ue5ytI2ioIO-!q(*nHG3>g<&MqY@y5J?a^>r9EFASd;tdctU=c=h&miVEOO*D4wv#OL*?uj`SMX zn7bbO9t2sm+L~9l?JQM)n;vQoWA55XkEH?Fq}pCSV0dF_uwty7a&(BS=AH(<`?aQ? z(dDVGD}>JsZ|T*h16wk>F8@@gA&y^3=dw7Pz^0cGZ0Ym*YE1a3*EfLb8o#yp4~CpL zEGF)*#r2FB_dh!k?mT}f%El#$lYPcjz3;6P73h9npV#Sc!smK4l17`_LYs;d+}%ru zuxK_TJkTDfy@ti}k8r+bb^f-W3lFN@QquMEtMnYL!;E>j^I=1&u`}cJSx93py6oE^ z({20m+T(!HnBmClm+^|nd`=i8CuRh|#W&r#UZD<8a*2W9PVMEBdhXE8>>@1t7cF$H zW>VHP0Qc@2EqhOKl+@eUrlO^s_Gm2jzMaloQtYICD>rW4<`$0jv6XXf_L1vnCcyhl z9e#Dj3UR4C3fZN2Tox9~YTGY^RM$#Pzwi3ea!w&W8B-0L-*CmLg|{^{=JLP*Pu4Oz zS{yrTM$dsa6gJBcZQgilhB>bjbYJYfTmsMcZIRj5wb?#=h)ueTVdpn4;dONL;nB{= z5K8O)Qm1#4E5trQbOLFm%ewq%8nGh zgM|hkfqD{(ZTD+?rZ&Tg$LP7-e~+<|;VST6byQQgu{W#tu>sd9&ysY1&?s~xuRC=+ zTj6BG2f=7OV6_;%=@}lvB&4~+VL4ym!h~~J!rH>;R&liL|6cJX)gFTej2Cp@a^XdH z`RmjcINtaM&YBVe{xOc6=2%93?7^dFHou@Y5sAoCK8BOz_R;JZV)$; zddz``>6j|K2UVYyn}e|Koh`h!wI1Iyrk||+>Xb-uI0xOA(OxGxi}32Rnn<-nICcv) z(@F{d!?4)aj_a*H34=`!prEsE8}ZIncuzb@&rR!q^^4t@@VXYqwY2y9p@Zl?NRNLS zH~@93Kg}iHmWnrSZ?+c}b^?9-gka><%&*0X}sau?A&?*=CIiF7kgd3 z1=8<{_iO{Uu2n9!^?V@=|F*)*&1T7?Z}rsgFDSC@de zL!OPaGW#jq&^F>C{galiM;={XgNL=H zGo!!Q@pjtUQ1iG2tPTtT!hBd`Q%BmJDPnf3>!U;VNibYsC&zwDhmy<#npUhSe|oHw z%*gC7sIRm}$(b<4zp>(Ls;SBRbOUp~Z)GdCD}JvWd3}JqkYOaL_F?q7JuGX}F&y^X zLpwRP6teXVFuP}0#SvIIJq8S(*+40+{oYg&plS*q&2f^X52$AC;Ni59vfG4huxe=s z!d_$9=ywe?G5ySl8^GMeO`iGh8urs)33tsG@jvE1Ks+pIKC!|iP7p>)rFp^|SxCBm zgq^qD@BW($#Bo5F1c%x-=QR>`K&YdR!hC4f+Zn$7a=@+a9s_ALX!M(OPo@sKZwHaw zcenUzd{0Y!!4Fjb0K?zauHsmwLr6dHjTe>y@htc)c9TX$ld0w}LuGs=BVDKY;?;~B zjM{@QCs@%wLpHeybAD-Zo3|w%bdxvQnagJu5xh_23820s4^>wz{&7~U>DvvqzHKA( z7TI&ETSk2dO)XN<_G%>~{E~}Ib>xJ4HL(A{?&L@2iE~$M!Kvpqnc7mHD|~o+CJTsf zv2;i#J_|SGL9wl6Q~>R@M|z{^Oqpik$JdY??kh=~vH1BNiL3OW`HdWr)6o1V0jU@NrO%W_sl$_#aDW9#`YHg>eZDN~R<-B~gh; z)Y)s3S!S6d88Q!>_@NRtyoAXZ&03#jjNtc1n5tSz`ZLop4 zmVH!w2jiQY1N|O-moyuCTCad@vIVi!7oghEl7DC{>fL5~wvB-H4AMsr7ab04X4Q@z zb|Kag)czLwC`hyp#9yO|an->VP#k70={Ru&JjOFlkw9Jy zq>ZqCMs;G1rto!mo~EK!3tVEb2oB+792GD}X(6a}%U^7W-7>=@M$1IqTp zpps?S$9@RA99avFZ8X<>|9&51e!S4=_u4JV6Pe=u&zEcAiMRGZeuSUaa!!5-IjCX( z8E(;(dOJ&(p;swG#A}mpX{ElbWHQM&Obc-@dNAH|C`cc7&_}JTeE5M`9DqcIe7~xZYVr@ z1Bmg#(1U=Neqwx!6XLjS91=gn@>=pxP#)dvT{CW2kpRIukwAQh#A+%7LCpmQTzPTikPEQ&=1rjf zSD6K;`dx)ThYdM#vi5J~0PIm%q#-uNKig_T$`&Vn^BG+?b$APEPMXPHi(EnNd&#mQ zbj@`}l@}2ul_@qET3`06|m#`n)2${sLfP9N<>n(I=ZGv^J) zU>flf&P=-ee@#kd>HIpZE$)D-x72x1W`P;@31UnAOW5aQB({FMMfrv>+jU;pUzd!$ zSgMRM=$WtLX!7#NDh5mLbr?OmHiv=wPT*KHN%4+0hpyF7dUMFWm< z7Mk#0Xuv*S-iVFAmWj#rHUcp-rc`tmS5{sYb3A?m=_8ac$q|EEYOtT@Sfo79x(uXy zEQ)KyUpkWdSx~l7xe8v6rq2@E_C)e^P3!#CTHjLGh9KCIhb4c23K4q^97n9C3MPfVV&>)xQHjy~4;}t<( zpfp}69io1Lg|k|t{a9C~cP^SK=5dREi(@Kf6=xk?&=5w>n+`tVV^OjF{G>*bw1TPG zLQ2Ic{5!C|r1wI~wm^KvK0gb_)?qzSWz`zqzgg(LL-5sP2M~wwxbzt9!^0`!-JddDBrEtBz8CCDode*Pc|$|)yFu;#xOJO2t4k%3XfbflEKUBBJ~cCzoUG;6^WI>)_J-dbD;(! z|H7$dIVxLd)vSkdInbO)@ZX2=#9{|Yd`msF5wj1SrG8l8vgu_it#JmkO3XpUZ%V&t zb|EOcV22KLy;{AE=#|_}qv|vGoEy02p2q_Z_R-oklFA=@({)ZNi;+$M&Cb+Zh0#7j z?&bEl!(b3DoOKWsOBI-XXUb!{Jq<+SckI3=nMH0}p)}jsecurc&63#C=cpn52IJdl zRr9iK6%T>t5^Vpwn}X&^^7a-JB)x&w!#1%)YZ6c$dzbqsG-sC&rddk|(vvixSpd>& z3aShK%(y8hy%1FQjF?a=kB&P0mn{#-LdE`FS9fEi$>6!(o;@y2hq{kUxXM3j_M~)- zw2{$#0JYgCDXTgQ%1}VOCojjX0s4Kkk6tB+y%_bU$e;Cx)!bMI4n?+?)T=-<5X^Yt z4vUPAsdHC3ORM&qxQ?_oNvM8PbE3SK#ZaxAjmj3dKfzSed zFlilbgP;4ZDovu}f+nN;%9e}X5-ZN1Ev<%19r0q){NM^FQ&H?oEB`Dx(>! z7;ZHNdN!_(l=C=c9O4HCNh9xp>4+mj&8YsBn$s-mPL&RM|BGcRL$`1^j*8tXBjU6} zzh759L;IA(NF%TpeIJT4g`ggn4j&q0llI|!{*)n5=5v(p-*|xa&uGJmf2-J;vJ1_F zvQ%zkr2T^GmZ^Cx&4kf1?lU8;hscX`fAo$Ah+VRAGC`~I2sOuZ*hhIXuAI>-5 zpjn?NFUo{{=ldYd*sE%u=6kSe+!!EsW5fcYvbhz%)2KdETCmq^qwuv1lngz+io(L28$kGdFx&CBEY=TUc07E}(}7Dd zhjE#`9gnzUBfp*&nBi|Db?e@Pl(05%sYP`e-8c?4-w>lCvyom`7S~?}rv}||3EBM% zmi;M$to8$h^^sFx)T9&Y6jYaOJ8a_<_S*39p+jXI53N{D_l>?$FpB4U?*-4ZhhhDg ziTK@GpREe$1nW;7lR@6PXeJjDCl5WixlVJIf4<3^i|6Ht0SK+v9Y*`%R#zg(AZ{u@T; z5|2qmwWtnW&n(o zj>N2j6OhoQlbkW_w8*pCjzf!+F|GC|Sa5bBESc3$v?Jb6OB(}4X5IO}fBP~0Km|;= z>?GGEE>-I-oo0W-!Q(?@-H#KPc?_|f^)(sY(_R#1x54fMd-EN(q)Q#&iTrctMBSK7 zNXs;4RIk$a>lr$?SLnCRSk}2WNp6~7fh%@oxHLav%oB7LfaaP5_nfv3ho?mHH;w_I z8D<0_Kl?M7VSrX<;q2gHM12%nwxm#2sYF0Lu&ZiyugCYHR z|LQ@w`)N(tVAB~~cw;_%t$EkwOz=ed9+~XkL~IYrD0cL{4zv!^ zZ(E4w;J9HJL)U#fnN1M1=aBOy31528g0cq{aC&_s=;r&G)$-Vmi;g#z`TE0n=$rH4 zJO2?p_f5u5bUnfQ4np-eOmU*?kCO^S~Q7z`g!=g+#S_1=FA%* zEz6#WCl5?Ggl>mxF@t1Cd`Gl&Phzxx@Z>=#sQwTM{pELye3sU|Gf+RKt)9?^7Y)?m zS2}g&R;2-W?)w**u{E7N*jI?5lb?dd!VO3_peQ?@Jz0|hs{&4kH|-4%zLa6q zwy{#PlfLsEG=lc2fvm6lNbKuk#FP%iZu=`P4X7h~nk3>K8-M<*RXYDY%8v~>KUePg zUY&P*62odG9S4huy3%xc8$PFz0o2jAlyjUWNx#%x+{}M27L|75#n&20(k7t3mX(f6 zAU)|U)@ZU7^as>}{E(xVwf;FcPPB*k1EJbce%+yKs|cjy7tYS1n9|*t(XnH2z#63e zX5NW`=&9F(3(JpG=N%+zr7U>ASG=#;k?RcltPSq`3aMWO^*A&&b^`i0)ba1f3hLX* z6jL4QB|CaX>JF;m5W0taH96nD7GE^tD#WxY!W#v5S>J|@v?-ob@nFYHyp-9QuWC1y zsrUXpsya|ziN2v4{=Ng_)T5Af@}R~os-65*x`uS@pr}_C0ry@zK;7<3dCiJ!uuI-A z=y*V9bX%!EOBj(#@h|pPo zT9nJ3oL++uIbSSD=Q!0q(tF9&1CLlqa<16C;w+RIUJ{kX;c73H*0H#y+rivjPqcjY zOq)KOyrt7fJ}>yD_GyOHfXk%HrclF-h5ge1Qs?JU-$)3XhNKj^TKkvl=OKPYC6u;OJrP@cNg zLCOn#cwvOZ@T*^0Uv5cj+ZyH#p1`T@81*=L)I&{Da5vOgSCi^|XuWtq%Z~KCha@p4 z{5Xp^+Dz4&@*d6C(+n#c?$y#-Nb|C8ocaXcA9)5t7aAgY3A3L)6iSkZP@UE1d*grO zj*+1BL1`z1uPDG7>3&Fh#=}>5;Ppd(sM$Fm!{$UuyTB-X^WY4;ve2XFvzW?dV^Y!c zm=89kXFlY!2tH?4D=~=fOMdSABG$2HTc`}}1@XnJz9_ zNcG60KSd!p=HR@&blvosOza(Z75b0bCr1673Rm=P4c4LO!;2kmA<_H;ch_M#gm>6V~J^7 zS*N}!!f4-CRDByaXA&QMmag6FTU$nLd!S9}9>Qkmy=M6yGeOl{amGXp>q+-G^n8g_ zhosw^(R2J*@Jlt9iVgBTij*hu_z#~&C-$4A)%F3$2P2>$qdw#({iltMcI31MJhdWG zLt2Te+P%b~3%6iw^l7!9=(~chpJ`ky;y$G*pF&kL3vZMtZRNV#Q=G}aSX$UMpkBs7 zaiwVHP0w`iRms#b-6`u)^|ueLG$QrDctO6YQ66JuTp$XweerpRM%34Cd_OK%Jum2Q zH0Fo)iOLyW<*9@Ns5GV7@_~}-5N9=i0rd+83+vn6@kxtCU}|NoCH==GyEg;*IsUUg zj(rWEfyL|qa7^wi*LMrTv-9-i!`rP;XT2jTEj81>48&PrFWyrBq-#hwz`d6f?D+W+ zBQ}KNtkn+OZEmd6M=^ZeE!3T7#jkkul9k*R*Jf!z^>)uFUD%;xMRnCp+Aj&@LEVyo zYKE0Z1*7}rFm?Q}ely*-GP?*=FPWW72CwFbMqds=$1Q%cz`M7I7#E0ioK&Y>aK-#H z5QITec?9V(abg_U9(s>nt7~J{=uto(DT!N<)`XKTBh??gAKPEfX|x$$)HHgaEHzz^7E~D#p9A@-$pA*%z&bOYfk(HrsdTxD1IFt z6y)+?`E6{;I?5=Y>a1u9&G6iIXk$MNl2c#p2l}`0-5*=!r|+LwN}qR*xYn(a%xU5R z?sG3HEhonDXLNi_vB%(JZK&23O44~^pin{Yhq^^qfOr_xe%oYp#GcM)s_KMlmpRQ| zLY`k;rdJFR#IWQW>7e#_c-}1Zoj)D};|9dNh#MfqAZ- zc(B}EjCX3N@rYV1==gwYQ79hjJ**SnpS=}`TUp23uSDFOLSl)9NVSXotLu^1_zChk zAcmvkYbVAmJV9%Fkm|!qDn7&&ecH*;Zu#uZ#JVzf6M|yUhqvRI*q-JHbX!*Vor`H1FNX6(FVq2SA&9>oG-?CPcJ#(9DA+5(YVU}yoGdFpLkyeVY zCc&T>u0|7LOybu%`Uhajq`p#l^^AH=;nZ&}P|d13;ff;`Xa>Wyy*DAlV-FU&4g}(K z9Q(sil3vj3I>V~ftD)qG`&)=&^1iyk7H$b+R~wai3hE)bR)-mV_PgqquLM zT*WG!JdP`WCrwa{!rYxJSnriw|2X`*r6YImbyg6w3Hv2A<&5;|iWT5ZwVv#hG06Z6)Ovd{WJV{W(3Ed{rL}e*YE3`WUi%C6n9FBXKN{HVNWttWWo0 zvzT2&&TE~ADihXRH3vS&UJzE|BC5PV3@JPv2BDdMu^@kA{?m2%>GeCHscwa!Y%d!a zhauIHFl_Hi`}L@b-PF1??RpU?r-Dx(T`g%NU4vJfJpBQB-8cw+iw(Hk?o4?h92FBQ z7NB)ur166GL@cvEhVIVm1^I;NeV4u?-@#Fm&QX7#R4l7{SCD@wh84uIbEHJa3;1QVnWVPm(TbXikDha&A}ULdx;@r{*g3 zO<#n>SVG)llCL!e6-Kr z1GMKn=FZLY6VkQhb@Jdwi*j1Hebb@9;RyMLtN$yRQG9% zDjgK}2-hdxj54fTpt%9#gWA(9R8wYpVy$X2DLVbFYwj5AF)@ z%U9q)^BQpdP<5p2FYL3M(K(E!eQM5WmQ%%#tDpOVsv+`lIjdbXeQrM-#`ZtRyl%7x zT1!DaFM>8_X;nrx`eFr?5vq8EG7;#76{(qxP`Q+NkeF>MrY40UaUPNvxKIwl-YX^0 zIhi%YUhxlbQ|Al?@DXnvb{>lc1<^#NjaFdThDP(9PLly~9i!Vp3E z9%5Gf2S&Z6IC^U{oLc$}D3gl(q)H&Kkd*nTS4IKx4;HXgRJnj&5B^##ukxru*EW<9 z4q!^{Ye4fYP4R*(c<(#%{JnzT81FL}n>8tCPg;CY`z2`2q{`FOYgjnD6U;a}h?h)s zLSlb8tFktdUr`39&)8eu73=M9X)0%&Lt<8>%nQi}x?ooOE0kwyas2@sQDbqHJk$~| zeY~LNLK@j5O0gxt&xoM6V)tlUj8|zn33iLxZ&0k-&pKe>eXiSHgbGrxa5< zJUf7KQx2weC#Kq}$C&i%$xb$%!ATSnGQ3HY_`T z1jzgGV6AT8S$GOHafhnvIpxG^p=OGoO2eV@jHdBK4sphR1)yT%fO^7h1VCtLo()whm0U zcQp2#;tI`g?;@|>DDvxXgeUH2@MD~wniD{Bzv<9DdnRhGrGT2fss5umDtV3;U%t;& z`H1pf0;76Y^^KIPIn`5DrYp#ZP;(XadWtE9(o65b=YaAIsQRSiWUD)y%cfqrG_U)N zQ?*eTvi6{}Uf4{iyQmwC`u7q@b3qelCFcFJ1mvIV%HyNQAF_J!u28bR|8q5PaOh@{+& z6F&FkU!U|>oC+#ilfR?i{RL?Hg|2rMR|U%j)l70z)3<6JiGN!`@~T8A-Z>nI&wyq^YNia9 zxx-!V&o<)f{m9!f_jPSanxUcJW0V0YbAQ4eCu#%vAet_Z*El~(2bvpkT0hqGbuuR0GA0)rOqnjq54Pd+Hrwz^{u$zMsuVlK zak@6Nr964&Dl9s@8yf!n25u37FlO>B?vuHUJ$rBf-gj>)-=E81?Xwn(C6|7~hUp_^ z%bq)-_B2nVJAER(C!YKg2OHXzqo1>eO+89JdG{oq%iPL}ciuA^_we}F zW-#?fec7%@3sEuFlIfOoVg2G9c-F_WESu(5qpm%|-p&HPZZAZaI*sIeoh#bwqfVef z?oE6?pRNV}Vk!Sbz9nt#A-DXV4WmNq$#Em-^Y4juWZ|$(apI1n^e7r69I6e&TH#;u zBsDBYK~7cG@IN9@TM^Jn}tVJ5&o+WEzX$aD8Sy$pBjo%fa~X+j+w-gXI#il?{I` zfrh{RVV6~FzJ7Tf*`?B5Sn2kb=iB>%Md!Pi_qaX=hi`(*qwUbK{XG8T;u`5>QXhU) zjD+lT3(>2@5zuZUY@YU5{QftVKmGofmD>eEb~lcuq4mY(uy0UL{{*)DUJ2F59u|&Y zTJwv}Y49QI6XcC_lnM9j`{7x>0C6ETSHfZbHy0EW{@bBRE~fLvAv|@nd@k(cN!bK&w_dN z40uzY_FVmqfnkD}pJ2vkR_`Y#Xiuqirh5b!@{H-X@khlSWVTDO^Wj)%pY^q>CnK*t zr2W%H_5M@mp9ZJfE^@BUHF(|eDIBhzjUP5AXsK@C+Kpi-ZI;97?ayF%n6Yf!J5;?M z_0JNBa(Ctri>@M_8x~yg60;}#g}aky$OUuKam~L+P|d{|+@5a5q!y|0OXGmS21ii+ z4)rlp>-u4HTTXkREy-LWgZ#1?)iS5LK+T~;K=mg=uaA^6y^W;N?_x;KT$)C8 zESo=!gUD;+m`Pi6G3Q$h?D!SP#`fNZ(Oz?r))#ltXSVICUuMK1JnV5Kbo-pZhCB%a z{cf&&P}km2!mX;bpu@y)KJTsrp7z*@!5iG76qh=IRWD6uL*zw`Sd;@0cUZZZ5Fy zGj1Vi7UXxiiqunBXJ1;{jv5 zzh(8|NvC@_O79d~HtVwTBRR#ZhIHTwGCA`*)*00q&sB_)M{}w{LC8+{7a(xXT*lLz z43bJ?m2UBIaqv4Z6(3q1fQMcmpwf0TH(Qv9sFeO8gzi<^9K z<)=N&Bx#Enws@ba39f$rGI$?8^f|)oP$sQ>rX%W)JcqOgq>-hf?Y#3ihrSEm{dqOW z{q+$ibiU)PdjGbb16O)sy=w$t8TDFh9Yvo5ISs%ep}(P4p#e^b8O#P7*z)3?W8}qU z^)#e0V#3cYnl#(RIHRK%UYX`0ns}9{*VYgpsMjVf+borr&2D){E;XqqU+uanX3Xox zjW-NJsta_tjs=e-e|5gBqRBuct(Gt7`9wC^3uyi3h(6^nVSAevZ19bI)*+#>He{-o z+EZ=#lvl#_Zz1#H)|~Vb^#9Ov8$KOjfo=a|R_6sI(=)#tOnIaIyvkQgd#631OZPd9 zOV(^(XpQ8zka56P?i!Clz#y@ml#3@tx+acFs zVcigM)At6}3)_a|_p+DmRxo=<&rKok-O{y}>J8AHKAQ4XU$w66QRfk~HyO}3QU?Zy z8}M3%n{lZ}V|oU_XXSg?>hcUacMJ09t*G?+nU|mNq~`&_r3m3Z#T$n$ZXsKyrNYr1 zUpOYqqJwygoa!(R+mew1?foIGtaN2orjQgnEvw}mRwqe+3B&Oa9D47EnE-P z-ps0BQ+t-KxsCa;3fJ4L5OMQQ;VH0@V-~jMn?@(0VwF0fe>7A(E}zc~kOmPOLFv=L zX-`16-8z`!xfC82DR!C7GjCpomqRbZ-8OsiQ*({1>vSbe`EK>Reypo~H9oo^8aL(M zV5#{``3`swb>mbkjCvOKU4G5zJYeStM5=%Ktb9MlIe5UURvpCZ6^78Eg$rv|lR-b5 zx_rN-9TGs`{v$ zo=cAn_oOCW=i@Yok|<3*z3d8a)0;9eL`~FLph2h4L!V@$6-I^i!WdITi?} z*F2c=${9yOsdhGC>mLoIVOW{4_!&&cKT@1sS(CXM?Nu5j)ca6fQqQ~wF>ar@U}T|q z4U=|`l{cmq(kwkmD6ToT@r*{-))orB5hWx+3o+u(`a;giYVz$ZHJe7a z=hSXY%0ELiIwk+^eP6eAyC#HM2R z@C?Pg__Svf5T_zBvf>;@TEQ-rKL^q}B(LEQtt^Q-i|Btp)6Uw`Mo!-~oQ>EEY`NVh z@GqZ&Urj=Vo=rF_ioUHOorJ}!{KW2>E|gOuNV9KfQ+7OtL0vPEyagIN-ct2WURfY0 zH?f}2Y7jeI6I;4o5=(EN7Y~y*039!@R+<8FKkD(ONdf4am&M{cPU3?u9AHm6jpiq3 z_X6Sp)l;JM&M6$R?H?3edB%trD6=eK zV7g~?Pg7Q-{1r~Dso^!2MTvIR`^&YF?=-%7eTbWDNTrQ>-j9&9OglUJE}P!UK?Zt1 z(1g+VIW{hB#EIRM4hv#*7W=RTkoRkxOd{B;iOVtYV^?}!sw?X`whgK^qFgE_(Q`_c zdXV=ws9vRysup@3XwNDCuns#P!PY0CNV);Ukdi#4O23*|wGp(w;-QZXsAKBY?k_HG zKbMgf(0>=>mrEorFdO zyNCr{u+|t`UVpDtdjWRecJrG2FeL8 z>YPcV*~dBbQQDO$u!41eYDD9lxecVtS^V{>9qjk~MjR}tcEd%d9ZwY)Yz6711`~RoI zc&~ZbYHM?BuQLd0x7#j0HP^zAji*(5nlkBd#YgC+SQAJe8K(Ch}N?geQ# z=osx|q}7Zx4Cie=LNyo!S>E??Q`l07KpQz$Br68QH>to5SMOWJ>y$^pWbuBu79J#g z$66^?7Vo}AGnz9XWfzTF7d0EW6H>`2mvBAr@t7JljrOL2rq!^1aN~n?Zrs~M^)st8 z>JRz3XF$t)3{(6slHq0(vOUtkOR^9;m(l zERa9IKclAfZzrUg5~n7Uh1vDl5>;!i6+d)4a8fP`%Zxrckps?ab5*sQhtz zn|&%{f@kY?YCV8vM{0(o<}f%jwL6f8LUF=&Y&N`?q&;FA|GWW}#pv~5=+x&(nVvrX zFknA2gE4pU9+mTvvkpiBV7^*A6m z7ZjIo4gICK0%%^t(j(@=((TVx9%f!W={f6@^x^qY1GKcX2kIM<_0b7R>jq1Q%rYd; z!7KYu0j)DV-?t4@eO6Ydf&6;AE8T;or8a5j1GOfoc#(Pp>hI-{nlCUhdZZY#?>rFa zA=O=4#(|?qIwy#UP|X@+I^PCkgJW2KTLmMg0yVd3RAA1jhsE>xog~f81hKHNd=QW1 z@5-OB<5zml?UdI*^94?f%@@q+fo<;F5TDnVr1kKb?gPB6nJZQvID#~fV5=;$Ffud@ zj@|8uG|xegBwbiHBM(vzH<2H|T4-qwBuLiK7?b0?iA7-=b%UZTqKu6_j4xeU~jVY`2i^?-9iPT=^W;KB~P`z2#g%&y1k^ zldK6$qgl#)Ts*rsZhd0HNyjO7hKjLe%c*xdvxRkyHELF>Sdw@FX!Zm%IzH1-&XfH< zb`!)|{PgWq!9LkR6RUW6X5mJl{t#-O;(D=>;$wO?%6vR) zS1M>8mqs&yDi)$yC23Uvm~YzwG$$h8P7zm^m@C#}yQ~i4hLX-$SOYXF>lyc>=X>N- z!;;KztbI;(473SVJi{rYE2f6c`%lB2J>9`^c2*TH5yt@axFA+z3@--CWTA+2SPWa3A3ta7bQEM{w1%PoA z?W*?v#Av5s{?2jS!{-=~M!V3w3`|2K;BxMD(uezC5K)U0S5(zC@rQ=yqVS(lGfeXM z54`D`2Achf?lXo%^GRQ-@)G51Bral#oOe|K5qGkdkeOWMh2 zr%gmewJ83thdvucv-!+kqhU%TiHBF$lx(RHZ+o!??=tx@Gx&RqS<-iJOBPI$qU;~^ z54-?1$7Dct=7*2(w&Shn9*i}Xw1Y>}^yJME)nuPJz9P`hg}=ULA>qw&W;e68jE=XJ z7kk_CmlMZAM(J+NbbovJ^z1)o|1gQg29>ZD%dX)3yc>@kYGo2X9xA~ z8`1mQ32op+sP6FvJVrN=qpkk9JX;-w10Veqc}G65f_^Pzvl{e_uUHSfRk0Eu&3MJy z|A>;+BR^uFPJvipd8&r0?<%#9!wEc;_h&@^J0Bn0quGoU8&EYOQ(K z{J-c~QGl;bzQI-&2K>i>9wM)CAg)ZT3v{j`&Uiadpm~*7;tIfTt?-BTtk`((9%P;w z#F`gf5U~OE`1)Us#jBIkolod^frZBw4XvTJvb&@F-PW3w25pDexxZlGpi)uU{WPTg zbeEpp#^BQKe_{6SP&k|y32tRxqU7&PG#av5c5df{=92^jrd`5QK8J+$yl{C>HkA`9 zJIWIp51H3(scdw2DkO(VEd5zCW|0S&I;B`=Qg?e5@`79l9_yI}44 zL#Q?%V0N4NxPGH0zOCLH(;SBKzP}9Q$eu6R_Xm5ht6nTDS~(bJ)tw3sHimL-V^3b5 ze+ld!>3~z)$#@I`u>Fuq{Pg4?TE`o~;@kq<7TKEX1=F)+ChQhBn@(ZsZ)gpenO9xd zSSQ~_H>ou&+G=AJT55SsoCt643BJ$=XU)4dk z?QP_%mcHN`Y|iQT(c$wpZ4Z~3Xwk}qSEd}%SUOwq%kMHU@8xtkzFJLAJq0CBuOOw_ zS9rDZ1L}02gcF?Y_^iEi<%?5V?CNR;v%js<9J-T*V@|Z?X3@dI?VCRaou_AiPVFp? z=^DU?kuABg<3C3Ghtyjx^nTE__cx?^fb);jQ9P(49s68ihR-tLXWDZdapt(t4xR$P zcUj1;tLWYqtDnNnoZB#MP!LwjMZ8|^AY3%;h*^`~sUC&3raJQA=k{t%!ES#oFtb@C zru!eq2mXVA^hcuLoO%}T1+?U^ifUlcWPkBr!}hY?0A29^ zR$uBT?15%A+ETx-5K1SM?%W(TQTUvS)%>)3fWP{GK&l-GU-lL=CP$#*2o03X^^>!0 zcHyn1Ti6`bdUqvzN zyE-1A+Jb*QHepPo!K#Ltm(O3NM{?I6XLOiK_f#odz;MGcsz(pa;ayAd(49W$+3h}# zckyNJJNt4v2RlS#PP2 z*za*1y!!AQcW1mqS_^nsk3M@YH|FF+0Ac%pG*4+7JwGZ=d6H<4yTIdBEH++WseP5a zS#Ioq15FAfsJ^5gXA?(tmsth{LVEXxM)|cQ=@tJyIgUSk_z0DrtULUcy=C3u&)JUh z-NH@G)HX*BvOB8w5`(6BNyl^DnOn!l*aC*YuX{`Qk`~7Jd1Hoj?$L`6TdT+SJlcn~ zyWD23|3WpSmHb4~En!_4#B;hIg2o2NVfXJ>NOgyadK#|kwOUk9q4&>MBwsgmaBfqFz=YKb+S9dMuubrswlS z>!w{eogb(Fma6xaesv!+j8RWQq(uUdp7NAzJ)pJYv8F2|`5pJLXsnKr_i{R?p_&zk zrVNE&ed>_6+NY5YaO!jZ;f0OVqwAKa_M};KZ8`fc-7l``6uz;}2u;+Yvlw^27S3|> zlvLO3?QwdRM*0uDF}hRL`PiC15~U-`)E?1&r<%ahrA=Y-#bj()=QML%m&s;p*8{PD z+H28mqOBxt;)+QgeW}ai`h`I~6MeX`t-VNY-e0Qzxf{0$nvZwF+(lU~O3S;{&SiUl zoPxZCpMYvyW|s7mORpY*0=G3R$o4fp9C(Y4XBl^IlLv=l+*p4H2kBMVNDd5mkNLyx zfV4@}`5BMyJG0TO#GJ32UyMHnq-*-kK8B>veC3PUpe^`-Sv#lklnOs_vdIon=YlN~ zf1v7(kM4sY(eU%fh#_>z=RqH2W&!ZOb>0mp)XTKGEf6xqe zb}^82KHO~TM^Fqn%k8N!j{ioiR4J&IS@Q@7ZaZ`oQVrm{BmPpiRT=*H=D>kI1GRV; z2{UG7B7M&|rXkTRjtKJ?Wk(t$2aoa!BxO z2AjP)$eCg!S8KD$uqCJU!o*X@klqt$UnT7m=$l7@k=Y#OVRY`RWP|=4SoX&+;AKWL z&_)u-^MRNY!dgDWB`IH(J|XQpvyR`2c}M0{vDI{JN6-FtgW1{&Mw}o?3-RH@@zir$ zk(gD|d1I{UPbB|^Ju_|O*^CX`AkzV|PcO&aou7g7iuFV2Zc@6paX{P%@O4Pn(0YUD zNVB;YbWfaR17P~V&YXIX{BHm!O<>h>0sOUQKxw3D*EK$NXR<()=_rJ8f5b>$6Hot=v3_h8efDx|vo#A}HiN$? zc4Ef!enFSpBY@Tk>ilejlO5yXfK4K58_dA&SF%~QWjK4bx*`9UTTgCHsZF{y6^SX3 zdX?YGUMCt(yNHc@B_jO{NQ0_8qFtk8Z2j^)R<20akY~7%Zeirn6wqznL^_pvBe50t z-w=yE-S^V>_5CIJ6;KZ<4w1x&%qzYr?^1gR&zm2KtojZ{ena`AAv)M>!P3TS1o1nX z*?ghrnI+-kaDVWc*OE88ehiuyb%!I{dn4%^Lh~aKJ?{i>wtSD`RywD4jMhRDmy;g& zQ11_iR_vFSSVB&4J_d?!DF28z;jJiBBr@^_&5dnII8duEM@L36(q>NDCWt*59fu%I zRdt{xPgOm^DI>wdeUr4rBg{5v7@Rw34s%j16+dg1y+M$BE@1upTZt*3YlyYr`r=(x zEV*;@PSW4DoOsY>#2I^+yu)>9<}y=5IweNs6~nB48x*sP^88*vo`8SOo?zEqnv1(} z`s&}9eYy-f-#HA0ZEDG1vA4OePo1Hz?R z7wUC%{bwS{Um5WpmS%orq%+L+$y4&0EtIn&@ovZZlxgnKwNHDHIMRjq8rE+-0Ms8M zHT(~-e9UPUtf!buipdJzR8~E2bE7K#$)||cHuFNZn#j6Q~V4{uk1R6vGWJ| z$W>iqu_0fXZT|>D`5R(JtMNU}};@geHL3>(C;u&qg zo{oqEQmC(YR@IH;y5798fjy9CX%^n>=&~mBCJ^6gMxGuAgZ8iGM#HCztXq?i^i#wR zY9##!w1SZzM=JfO@;Bl>ICjlgUa!`=igAm(I)meh9X8rsgnxRc!MP_Q(CZC@^f?0~SYlK_{aFY~jQ-raXo8S18TxvTF?S z{t_UKVc+VS3(8&ad)hjcTNNJ)-4m7gy2mF~>x`J1K2LkX_J7f-%8|qfy!zE(*8TNE zRIiaV&I}Y!`nXyXpFa?adx;(S;oCkK=zCea7rQa1;!Q$jn+9K7fMU1VyG^i~CEc^- z{%+{1XTd1PYB%-!i4iWLtmWtc%3U^SS-GEb$TmE_)K<*2zlM~Lk=|1joZrrxUAL1o z&%oC7y*>I}Mz15v4KE1gA;eacClkm+7K_ov73|kK18Fk&j8^gV+AVkJ9y4@b*^z!{ z6N(Sxy7Hw!T+is7u)wX{Wqw`&zFq2qbi7sdtLpY&En`Tz@(LQS|4(tdAn(z{9J(iV zU6iO;j&ckVSHi#z#+)(9BiPjl>L#;y=ho=(2XRK_HhhJ~#!RP9e&#~oB({i|j* z5tkbiPhL@3M382K%9{=EbyIb#vcE`}{2SW-i2;@G2krmMNLN|Id#Rwfb;_{wqK3v^ z5_9sj3lGvr518g{buIOt_!YZSBGg=hDW<31kPCJgV7&qUng<;u{`~qF z700Ljyban0Z`JGuS7jYkdUL+r)6P=)A2Bi~ui>OI+BjoV<<&^NiH-V=$4*9;GII0* zAdRfbe2>H;P+5=uKS7LuYyP_7rvJVou?^%8KSs0AK0uiVXkVEA^TzV!%^{R2yThpHJ3DjhsGr#U9BO2IWweNp+T-Tsa2Sx<8e!@{_yl&q?@oT5l zNH{;_9IW&)m8wpaevmGUf%U&pb~@uivrN=U8O6OHg^6EveDLJ4a-lT*#FEk2$k0z^ z5TN|ZW*KkPsu>qCg~~dLYXor)(oBF6D{7TD+6Lv}TGI_c+Qf+a(P{01DxEOD)k&6S zglUMM809pyc-Mf%pRT6nZTPU>Jz}ij|8aEPaXoHfT&W~d5+NgzM9V1kJI^6xNA^ky z*(*Cc8cHIfC?jN~j6~yio`YmGr0l(CS=kwv`<(u`pL;L$d(U{D@A|#(c_F$>eMuT! zOZbWzLl8*=seF=fPmk+fo}#X&rTS&xP0kBFoerR~%DWTSLbQHkzQs)=sE*bCD?H>w z-(6ShK%9__lw+dHsWQb&Ec)^;=)W_G@S~88_jZHluS0=616|9W*9*22vkZfo?XSk_ zTI{QR8j^-#lx30jXi$~iowc3lj2fO%x(rl3lP{H2m$W|KsmvzmZ-KK;yVzT^E`|G2x=EylUYre{SEQ zw2e^fQux9|&i1V%*X#;q@1ia;@?AXav&0ph0!fz{a`GXJ#s||T1;Wb8ZICmqKy=SL zgOtewVU2cJE&7hh#g7jN7zo`;KcH*xBjDJX@wg#3u=1!SNXHT()^_JJH;u=DZ;o(n z3B6CJ)^1^Vemp<^Wdcm?(MRs@R17$FJ?{8q4FAlR@*n&D6Lnu3@+FrCXK6ZI zz-_avrgDb9K1D=D@(VKUD`yJ-`39!5;^xZlV_O^?}XNW1K5C%55!w|2Zjcwz-PGPWR<85ijuW>Im&Y&t>x7 z$(tA$xdAdaH-WbY{@|EuJ-PD=iEkVna>`uJvub|TrCse@xY=efuc%Rk7rJx<&lVeT z{%q1$OMhwDyo0!T(ghfQus&ubWQf)g*U>0tG}PNrpX-(g?VE`@Y+s(a%yW!I&2&>p zyT21pu9_lqx}C*`Zh3Hv?w7%e=Sb_L<^ki`Eby?KApQo`5;O;lnrg`DztO~~joU|; z2uN>#1;53XplxCVU)ys(ycso7KK-mGV`6n>;WWml==+M3tLSXI-)UNd@b7T=FVY-B zWVNA_WSsaW0!HNE_-p<7!j&h*^4HsQX0$nrv`;*k_mg%{*WuE~y8LU~qmVkqkByFR zDYHt~qUT#JHnX@5iM^M~1NZOZ%G@u|D?brKb35}{$&_jQmk4fC8Wxmw@2A;W;?0^uDh*vEev@F-1JHui4(IeCU2WS2^Rqn%wnKE$-=c0aMNn#$UtR-~jiA(s9Bk z%nBGMO^go7Zc_%pIqgvK!Fs6Fxs=SP9-#H71^j!5o^Y|(DBM%G99?RjV_VJ3)Ev=Q zYasV}-xO{3(b;@g4r9WjJ8a$8KC+G8VUPhW@lKDOAY=n+p8E_2RQD8j7fq+^v>wns zHdz0L1z2`?jF&=tNUPHm<*GP|G&d}1w}r>vT0v{MMqKN07v`13@P5OR#B;Z`_`{?% zp5I*rdW-tN!jlmwt2Y7~vz)!?E-qZzN#<9sVN>SnbNX|>e|ifH+VBj=F1Mt6dg{wb zTasbgGs)9SdqDBPj`DPTpa>aIso8x?pHuA$*F&8p^%`1s7^nGs^ft?j?*uDs^rafJ zc2z1%)(Mui2ffO^!WdJ*a0nza3W(PYL6$gk9u zpO22D{yc@Aj&5Cz3ovoWHd*bAG1h$@!Dvn8y3=2^^{0NskN#bt`+jWnF2-VLXX%xwW&bv|{&C~e3A1Z77a*-XvZiwmS!%)@3vtNnGuBRj6iQ;cQ zzk8dgAsHq_gteNJUUXm9u{Rpil&0FD#Tn4B zaRPROc(mNLmK6`oz@~TKYOk{HF#onK5SEFksS$Xfx2}BQu?q;(dCzsuaQtsIL0X8Z zy{oawTn25JAo`s;jC7BTjNV7@r93qmX1RS}rt=FR_SS!d>o1v_6U~)0M^mKQM^-Z& zXpU@RRu6pOGeN|BU9a7jw*?oTG$h|KLP5GGJTIMyjv#LLSb#4Pm|IGs1 zgyKBsMDg>Tfuwqe6LpS(!YZeQt>OIc{-k-_h=(Ub4IdAtbEy@4h%@8FRm7|Mve(!z zK@&qN)kPxNF$RIvv0I>*AI5yV!&!ckq$1 zF&nXbBy_KH533D*iK*AaLFqZ2O@pMazA5S3<)nvx0`)%7^(E;OB(9c5VMrF6amjWeplN!9mA`z1$@FVVP6zbxuGM8Q)ZdUvw%!5ms! zJh9&xD=nnkzSkJS1+;Ue9+}-sc z9JZS;_;E9!9wQ&oleAzlY}S|n^#iZflTuru%h=(zSfl3p~oCfrK+Ho^n>Cl9blwNV@))DGcwpt)@J4c)zCk@N+=C4YbCvr*{{i zz7_iQPve^nooPS)NrS`*wTH78jmF~88ddc@=#2oI<|n|%a4Zj=-%Q5Y?t-+^f8zAs z4T3%wFPr8w;wQeuAeDE#V0J{wEOi`kQgpQ=eb7 zlT=?wbuQMm-APr6*Q|`J#N9HUxN5Vb!*FpzLw-ZN)lTSH-5g{?C{B-8nKB#=v1`B$Z z3He^?71fta^(WPgx*KUvn|D{bpt!XD^Ek(=~*ZP}9;zFsvoIStkc(ma}+ZXQy3N=+vewimm|WSwe~Fb^h}W)c1yD_?-! zcJ2Y?TSzCN@?G{Zj!bC;s!v!O=)(uCN>CaZ7u)2z4{B!0YOQR^ss6Q;W8j>}zwk!+ zOB}d3guO882PYCf;^5W?ac$szp}ezOt2!{~#1t^LafYnY8rWscOfOB@p1iRAT2=WNS(_eJ;hSK@5fOa*DKD^&^jR1FOJl9NBd9NQ0Fq@jS+R&V~;Un zR*4sDm1!#DW9UB4Z(Tuo`0B>C=wtW*g6HqTc4aLzls8Gz?X1_7O;wn%==DuTKTml7 zgZ8+c{2d#s)?V=uobDEilA5% zaHe{1vt}dKTDcDn9I6dxY8|HIFSEN|w(h>BMV@0lO~Vi-Z(XAEAgU!mzg4%YbeZxV)!lqK<#X;-KTK&1UFVC8G71gt z3%k0*3tYO!iS4!HKxGV+jS8v(T%LY`IPd`y-w+m+Fv2kPsFH6jRwi>Xt8|)A{aK!TrkS@o_ckMCdyA4;^nvfc$ zJS6GFS#aDgOEel4hPBLo1NA%P#h-*tiQD1mn!{+CvJ*n-xnJt>oGJGXfH+afsJ`Lk zDaHwlL1_v1?}J2#F}s-3N|XhI>Vr>#Gm$hV>i+tKl$*o5#)z~pQ2y^DcKGlO&$}Kb z{I5m1jUjLF;3%q}SGu_8m%)UChE+A(_4^hP8mB|K>SL|aPo}$8VXfO;H1`G#2haN- zSdsHLeAeTmrbozJ)U~+5M!u?m)izz(gi8)w+?>U3IU3>{_jUMr*LO5dHsO_THbeXq zCcUv89Km(CZ|!Ap{prd*XKjOm;9soC@5Zu=#}J5F-cJt5c9MHSN}==qQ({Q?GI?O* zdGH_j2rJ*1p>??%{}>rt;-f$(BCar=iaf0|OlS%l@yi z``3J7``sR*%WPzUSu&&;n)9bCEVVm4(qUt1e?h+wExTv34c+$PxK1_shRY>5s#PPT zF{8_#90=dL4rZA3lw0?k^N5G(7~GO6zDhB6)gAd#9Cj?F-n)-^Vr{t?;Ky97oBUyNBNw`7Ctd(A6?8%t3|ro zP5p;QzMR5obqZkb{Q=y$nc9hv?9kiv2pbx>7xK7|)QM;(k4~-0uME$l&kTm! z;bE{~X$SZuF5|06dJpCFiE`S6evpXxiE&o8K&psAEd?S2$ zunQlK8bIg6&VwI+XTXq|t>vGJb?6;dO$ObwW$`I7(#vkGG#$21uH4)h>&M&S&TYgw znSP?zp))zX{&?^UTkqgki;G&L9TqbEw*xv^eb%UX^@|-3=Z3oG$_ySAaCY$ zlC-vRX$Kp=a@c1G9~jBMAE0v+(j#zsNE7ZpFB}{A{-^qb=SKVUpB|=?`VcfaPIC3w za<+JPHO>~Tk#h#W(e|^c1;d)O5UbKY!oH40IK`+KmU{gXrHkszf0IXJ-{NTOZ|K4& zHcwT10e$KFtm4Ky_CuXP z8?a@qTJohiK%ME&Vb1fqn3L6nhpo=W!abLf`V7X;IZo@l67#hOFn+xj2PTf@w)JRi z=bB+F8-GSUpfy}}0spo;0kfmb@z{@aDA75G{sa9u?H%O^j-Y=yh-Lb;F0HES4ArL?s|^;hbwbNZ1@6e63k?uG$#MH>p*)r9h|J@fytls zFfynyCLd~sah>R^!QlN6^)dx)`#DzC|GC9A_{0&n=sGDFd01DDDviet-CFW1&9=e* zs6KMrk=ppS`aL>ptr}ck8O<^`U%{mv_T$U;X8gp^R3IFHn-z~?ifv2&$z&kERHu!& zxh7KH>$r-kUV5JR4$i$K9r0inw=RCh9_e}EPEQAj-P2Gu{9-Nr^V;*Agp1Ez)=!Lyn<2hicIT7JmTKF-KL@JMy}yi5JOM_ly3tufy`;IJ zj{Lb;V7tG`aOzkq=(8{eZYPc5FCWIhUbCOt9@BPU^9$wB>>v5M?ylTrdL8Vjtw;4e z6A>K|!0@GztQ zh9>m`!PnUs7xoT;A2Id8mpHmFJqtrP!1s9VN5uzwoxJ6D%X3<+*MKlNUg4OeHNu}q zUuj;Lo<+i3asGrOBp=$vo*4GwieHYjeX7RB?Vk<=88%h;)$0s{rnVIk2f|^|m(@6I zsF(UVsBC^(Ou61dHhXp!2lc1(f#c%f-i1Tjw`HM7>!Q|&+5KuGbuESwN7)Mcd&(D@ z!I-$W8ifJgUv9A%rXJWTp(Ec}au>qBM=4&?ScYE`eOUlLxw-?cd%UD~$~Hh6D^Gsh zSK$#f9lA|nkGy_ugV1m8#ok&ubH6h#jA~KgzgSst0o+sC^AF4I^|ICT(H%ZD>OT+>R|h&r`npfp3o!Y z6IkBsBekFNuyRBh5Q;uVKqO34 z`^*~kPl69^4pq&+oQEUmw}bgtI$}ZaQnsk=5yDV^AU+gHw+CbIsA*~+A<8`!{(0%~ z`qm?aXQd}wv0)b~U7&Q!(cLGh4u?VBm~@3((5Wv%igg)eWh6;-n1R$!!sM~5X6u>NyyCHs+)!@aw?2K)4MZw&!9&UwXIrpcZJxoMqopbbe=#k8F9PA{OS*8C3tu zft*)@IbocRq&2`P*Y_Z43zP#T&{$w|a4&Wsjm~Fg$MJn>TeLb>7k+Px7XK#Sg@rZm zvsy{?UfDStvH8d?Fzt3z>gTsrT86Ni6E^Yn3$C#ZK?8*vFX=W6tv!joe!8CSp%{oq z&M-zMl~7E=UilhIDx(Ewgx?N_Ncx?tw{np<}Xn7i)N$31!+CpxZMitHujUr zj|;?xV?SA&8P)jv_tkiMhtaBMuzB@*y!c`Qkgnu39`W91r2PIOj_vShFYUr^LfO!9 zeEO$(3Xds!S|K|Hzh-X&UEot_3$%5(qxc>af1Yakn>6VncoTSv_9&0E(S5PDeMcbd z!hsIeQCe(7RW z!I>dBxS+>cG@ExHmDVJD{X-*E*gJF z5Wk6a(e{$|RZIF$cvN_?$ir(`g!x0HHw9`*L*vF8i;;86tcTtOoA9|qoFuIR2R3@*iFLPt_#Zn?ruWrU zJZ6L|uxy<*>61N-FhNd))!_Eu38Z=F3<>`Y32!o#rU7lQ3sw4?xEH+M`tfJk=TKpv zmt76)-82@5|0{$*W>R%+!hG^GN02lrY+x73TIcqMEHbL4)I9Fsy__7+${>@?~g$-okw_UUzyfr?-n}NKL(n$E$>NuUX>80k$3ClRu1;0P{Al|LK&5ZiQVfbAasgH_oLY>%qt14@)Hq}&R`!s00WL* zN5V0t_*VJLrz;YbM?vdxwej!mN~cnymhB%$M`0#rJ?M$3Gl z`iFU={k1K+Xe4Qo5dj^$UjZ8GQqZH;3|tiJ?ILMR_$KZvb<3=p0k>6p=1Y z{6W*!tn2tzl01sG8J#6Z_zhGW;?VIp(5?s+-6kc0@^~L_v_|48_90^!9-FBzBWK=Y zn!@UG{*??QToQXL_ld8an(&7UTf&KTBd|lA`eMrf3qflqNCUAe!|8ke5z~-(NKkDu z-znKZdyAF~(R!B?hJCNaz1}_o;(KksCAGQk`hzH@SdfPrCOvbvvtKQ>LSfW3U&a@W z9|el%%NMj4groS|vo?=7V89QJt*`c94!Efg+uU*(^{nDI(JXZ3-A13o9!^$M4KU5=z-@Qh_0$nIXiF229%P8y<$@5vKTt+yn<;sqYY z`*6bFQB@ho(rKlP{I7<*w3xZ)y&w&%FjKoM`G+7pmdhLUmsYQU{FgDOI+xYwr~}o^ znR~M^_;Km1csZ4m)@dWW{7>N58JF-=uO+a0$yuhZvp9!pk=~<8_$p|A*znXZrZl+9 zOybvT=)4PeUfat|=`5!7BF#y85gzwnU7-F$s(DmgWZ&=??C;#0@|r+4r!g1iUDJfp zM2&8FK|7a0_^h@kJoWDhx2HLA<;B%H^P;~n{(uh>?`TM~io34CbY6!S`#PgHC*6RT zXE&w(n9UULt!_39AzKh_ox?v13UVAHs-pwsEFR{37SIT)VW7ggRt zo=0@6yPNf8bPwQpdOtH^`t_B@+{owyVbMR84dDg9Be;9jFi=>0ae^IgeBS~yyAeP!d~w5NZ*i~VKQTA9c2(bww{+qPV~3vK zN4;Pn6$aA2!R*S)+vc zbT;xuMtY3RkB{JiDK8a&NSEmwU{jj{q@R(0ll0-&yE;|g`Oc#?NM47fCA#4adS-uf z)+f>n^iB(UyXVTLmk2vAD=jE35^W^m2vSW*#XE=nHxUokC5(OsrHi(bhWd(BQ-U-d z#t%2a$72g%!N)M=^MP;{)Hs8SbJ1*Ld*aOV@VCxb%~QL^RhlFs@;Fd^iL#-O#mLKv z+K3_gQh7#|b)*OCNXlb0`P$a(Q0!Sjy`wTO-ip3&B8#rX%sO3GoZ* zeD5a8hjFSsS~GLXGAhArlmSqE(c1Kqgm--Zx%o`blAhJvW{H!_e&yV{7hZ)WlReLoIEsP z$sHgMDXa9{922HLe}JkzL0N++YjP5u{Sw&Uui;v%3#z^KVs__PQRKW3S9h-ulyzx0 zzTcxAH{+zr_mHp?4>X=j`~ILRhnUpzAfuW`%GI=lmpSAWg~~gWb|ud%RE>_XYAO{D zI0wyv+94dBcpsISAmu`;mVvM$M_~u~9Iev6|0>tuEyoB@I)rdoVFvy-cp*qn0rj(( zvoQwejeeqdBZvGMBmE%h=Y_&q$~_o)4@uqzA2dCn{GrmbKwh7Pd^Vw+V+3*bTlf;u zluxVES3Eu0i?ASpojvTTu%2puBcq&-5l4aAe}w@GTSYKa2~gBRnF zeSp`Gg7oWWS#G0Sis zKdX^5>T%M{qNeS6Xt9}*e~6=Xq;rA;qVZdvC)}Epfuwa&5PnKS7mMF#eh5(()<~DDb41-mUyuL8$7%! z4YnEB^7NT8$VRMjKh?|!JagZ{){LFpvO#xjwtN)d|E@J}Ydu!}Ew3iWYwzR5A**5Z zmO*gYvyWVy*j{j5^L}+`@d*kJs*W{BWH+7mL?F;ag~&GHj35p9a`h2E_~J90K7l(Bc17U0|uJE zhVqG5-P=JIt>NGZeDZ0NZ0B7B<%2wVpKdQjnd>v`JA0T6PdW;rs}|FJM_D*z%VXB) zR!{9RonVk|N3kbHLKC|ZsP|_!#QK}_DbWvchVd&?$7B^_M9p0|!$e&Mq0=|Zuum4hpjB#4*El+jh=dG!6O|LT+CJw$PVYt-Yy1`0hf6keZ%ux9 zbB?=7KYF)bVxGpsa*Rys+7)^mp2qzBM)-T56OFmLh<~ykQi5)4GyC_GeY#D;mC?r; zTX;#(exm*lJAR4IJy>UDD^<-+yl=)=42y&v+wED-uA%OAU&*);mkcr<%mFz=O{GWqdNwXNxQGSFuOpHPLih ze~eDr4b*!u@5XNIT+#{p!WyiZvViKgCeNB1fDX$w{7_eCY*Cg6H)F`J()-eDbZ$*+ zL-&CDKZXmQw^=LidzfiY?{#o%E<#2>#H>m7;^Gh;e(qDTnEPZTU%YxPT+`#wn4Vi6 zmcLJobEzi#PWdN(PR!8E?>Jsg_W!`Hnl6MJo;k2DJcb2oxms)NT2_N+bv5JEH*#)q zAtw6ki&4?P*^(PEV(bS;wA#55a?Zbp2Y&0ZLQjVW6*GCkW;3I4NYx+NLGEzcFQ5JQ z-3`O&T!PLYmZ0v+aP~a%7jCjIW%ZjLfC|^^So=o;4)zX)*j1xtK~XY%|K6M5idqMg zyQOHle3-8ANh*9AG6R&~cCxU-y%T zH@nL(`%a4!*`A<1W5n;Z%?IiOucX(ZtQL3TShg&BaIX2yd5+CA~aqs^e@FTLt=1&tsN}=)fNah zw4eAP)hnQSW!{bEveyS6F{@i+$eNT!n#P(_J;6V^&t4~_Ow-P;y=)!Q4BaftsGmkb zrxA7KhHG;r;WiBaz`1KmWBKR72vFFtV*Wr#8EpptJ#WVgKXrkD-&*i>RtL3&G1#`E zrqs9l14~}-)Ob7gMXFIa-;DFtd!IAHLfmcR3m?mOVKklf=I_&(KPjhtcJyYU`pv3j zJoN3Og~a)0P(E@yarc*D-=|Vb$Fi*mPP^ z4)qai?zmsj{fqsF%JQ{T??B2=*4pMF4pgh(*}5S=u~TA&;s8*%_|epzmF#Lwykn@XXqza)(k2VSDLirh z4|Fvw#%=bwSifm&`Pa!`QqPK|_Wx;VzSL(DZKeUFZm)yplqXJ^-H_f#x(2zV7K6ppCjhOzP<6G&em3=Z zZB9JFRozg(f&c1G%rRsm(6#VU`d0XI>^n@zb!Nv$SLYdNwV_@`Cg|HOS9=5vmcM2i zyL!0XKNsgjU(qN$EhwMHKTqtA&qub=-l42@$qiExwQ4Hvok?eeCvTFlya`gzqN#8Nq=Hqd;1Pk#^$^*KLA}lj?xd7=c?_^1FJD0MktP5bYB7y8H-6{#q}dTG-3Q zTg>?^%WCe+N6%zRYY;!7VZc@`t&h@0Y^H5Cd-6z6tTq@1Z5t$G;Y$OB{dnYEJZaAL z*s5s+8@)OY&Tg(P!_r;>aUi39lje4QID5)T__Cr^)t+jz={^hE|6%u!V)T0eMfiO&?DZSmk)$I#El<_#tb7lj2AvWf-4;mVbM5vaKK%a3 zEM``?1O5&FCAMCBCW8O{MdA_O=G!}LxL8lBeahK&9V3k!WB#sFsP=U3QGMZ=I}<17 zhrx^wF|yvn6jAS?op$mtS9x*L1|aT*E+5MATj>$dS=WWVlBa=mFsswUo9?U7m7W_L zgUiNjI$z>4J+E_9?CH3cEs8!P6z>i0RB3ll^C-*XzZQQ8XAdeeS?g`xUZH$q;<1w;QOof%qES%IN+u!k{e#{np;$f30fP>NQt-4F9>F5`+cf#`K}QZeDM>@rfl|wh3Wpf(~o*Ue@PZlVXJ0i)w={ zr1CtCvVzrI#E09PNRQ=fJ1uVqHGIzs;vwuG;hOU*JzVt*5TA<*iw!vNnYqFZo}AVe zo5yLXw}*hLchV)W+h7}B3bK@Eqv)NGgkg$%{rp!zn_ITrzDKM$>UJ3hdsidB^N@7? zeGIxgU!&&y(djRa-_nhfcOq=k2>;o(Rk)z`E<64&j4-sq3fH0XfLSipp%wpnI9BOU zObGf0&pi_b@wC)?c!|BPIbZheqk&2~w`cVY9UyIn-^(`=2492?8EF{&ZzU|(9Arv| z<(!`dJa?IVUy9iNPY2bnc(tfLSGxUHlo8zYO9ZM37Ub3F~x@*LI&h5ohYqv;T1#IYwJ9RxWLcq{Wm5 zVdRl$4`-rNNgg9^&^AxGgxOKOxVMR(++@{|*BCK_kFa#WXU!-Nv2o^4Om8D`0Mgj; znnfX!|G?1sUHP;2bT&EhOT?Efr3taOVHdQ`Ux7F09zxS~Nu8*lwD;TZWTa&v#%397 zlb4|UrREOZ=TxiW0`6pieZ?UK{ z7VGXj0ufKzR@KhAM`zi94_*qVp~pfu(tEX#YJ@2+Qhu`Tag8KhhH7oEKB9Lbldq=z z;JwVhgYE*@5c&>RCt7F~kJImTrNB znH)V#_M*~Vx6@m3!aHakxCw5{u_XSUiuZ^2qQX7$U5s!Dt`=>j^AsCM|JetjWY>Wz zEl>JVkY^`s8b$NJLVQ=i+M1M*j#;dFTM*VO|3Gy$7Y9wL(2%y)(!ZI)f8q?hRvC&b z=3k-tWQYWJTmy;w1DL~dEi~(A0Fm~O;ZBhO+Y>Zd=|6m<7o&0nLHk8o zZkC`v!^h93U`bRXn40keso$7s%1bu=$t%L{|5*14spLJ@f%2sWb=O1lz4Q2>KB?@} z*jd=2L49b_G#+519h+zHna!CU2}&!Q_@)zX9Tb#VFv4SPYS9xoGpQ*jOk#!s^#x%X zk`~iYonh2~27I~37IL%cc|ocdbq&%B%qM?~;-Z{iE4FGVw_sO7&m(aeuD-Ea`Ale- zatoUI*5lK2_rSZ%F=+QE7rRerhqSJOvH+RB`7oX<%0S{Cpsa#X&uDa48dTxn#2XyS z+8!3s+Y6CuO63|t`SR$b=0F^+I0#q!t&*=IrCNRaLgr7+Djw3XmneL`~oc2od zuWxr$*$veNBb|n%S%exxd&?js9_MK~~mYZaKc+LO^*3!YWoivJvm~%~>jq zMp+OLhaqVUB#+NJB?iLk*Bg{Cf!pcH^qoVR%D_bYzrj_Wk+Lg%BnNQHLFn&Wk^W!C zzR?*hPyXx6tpAjP;;{0GgMqYwv|soVbIU8$JeWx!J;y*knY@flJ=2D#%xq3sNE0BA zWCNdVQM!teH_&_@xsFkOL%f=ZeN5_cRTn-tya*?(VSHj7cCg#4C4ACS)?P z8uY7Qwf|I0N>^(s@1yKxrck=6LT@!|@ogF5ZbPoJv44Hff#P!dJbtzO9?iD`V&}I| zz7A-Q@sHeujqskQojY>M_$ygwrzK=Ntov$e4^y~r;ViWX%(fHWN=e1K&guL;fU zd1A_x9?DmXJ-xO7eI}^vZ<1|q>bW>ZYr(9o-$KG@XH?l0^)nDgRApT!_IJj{h0$7D zyrA+tM*IV6?UYARnDQ|60O_o+s+NVYSWLKjiCKgol7)9l-7F0C??4)}JE|P6AlpI` z#%QCX=^6H&4kBlN8?Ne;_EaHn%RtQplM|D&{98!+t z7m=`y`1c=^HZ6ETc06xs=Ry831Skih?}0xs@)JVk6JxZ~Rc3&%O8c{%&6a%cDh3*F z2hw7~p283s4<0!fKV(L4JC7^wk08mHSX-NARrKYVyrKS&n;e2-3p% zO-@Ula;2`)pw&P7loL0FDlI9m3y+JN{-4w2VItIo@heXd)l)p{+#|RIR z_*!8H%vqWXjk2oAyPvE$VXRQKrE13^a1C>qb_73Yzi zK5mz0^SNzM-PfECo#Z9={5J!c{EXuUjbOF)ljz+_PVPo^`^yA+|JVwr>d^k8gc}Zn zc%zCG_?9|{&ehwCK6ApLdfy7XeZrnwzxIPUo7W)yIZtlBOkDZCToxa^4l7osu`6Zi zFgMnkKdXHV$~LFMw-qk@$aeB%*0GrFazt!QisX44kBPwtww~AZUGPSAuf8e*& zkMxeq?HHUSup+_({4-ken~!_RvWx`r;YT){d9V$SHm%7!818^;JMwX>zJ)wvbUo*_ zPAlH!i;c!}iM6bAe!N_{+Y%kD=1KqE|FlbgmWbrit9blq42Jfq0ksx1WMivE3mU)X zMCoq6ZMeUjJJ?fRn_Wkq^Duy%&|7eJmX&PObttUMevCPxBQR-ZfQ-5vB6YLhVC()~ z+M-L=T>st=vGvnDO^KTizhcyatFhA9LD;R~7EL{Q2fJGGW_oSh9)FtkPD~eFLmeQp z&n3;|hV9YgVlAi>cMz`RcZdE1^X2mQtuXcVaFM@17HCW|WY2xHo8=?wWW|Zx4I4Gu z*Zbs<1}$-F;X&4FhNTEj&AK?7<-)Ju#c-oB3a3tuVys`WzTNDw` zEoRQb8b=;N*n)0s#8Oj6bCVtyqwxLY_R_@QEFKDZgLDnKE_)nKt!)6e1Ft~iy;jn0 z)_qjh89M1Q?*3DRI}I1hnW>d*;SUp8-%nRs=F$B|0sElA{YmiseKCxE_Xbn%tyAOE zZd}z^yC}kf#}2iEj}6=Kf)kJ6)YY!Md)NR$zsITe(KY`+NSgVA{dD%?m(15=>BDfo zwDB2sy8kBDar-{pb!0Ez@_tP3RCnMd?P9Qt`Bu2)Jr3RsD#mAprTD&QclS->ip18| z;W+q!5vQM%mBlOB!d^KT**IEU+tCMQTWk2UB?mSQ36?dE`0`gPUSo3UU>UN-RF-XS zClBn)65pNDU~kucY};^phg@+>-YV!7r1fhlk7yZ8)0{;-@DhHS_)3k336C+|%hG0; z;i8mqZKDc1UYuYkb2pgqq;ZWT{W;J0dlzf#Tk+E~gWWG!WwG$QahA(||t zv+Pz>z~R0jX#V@8X3e-2nuJ$VG)tKyI4|CUBi*lw;z{3x>V*>Lp>i-Z}gFRD?=dq9(`9=w>xwCGEzI%vA4WBU^fPuOvcfL zcZKQjG6*zzh4go_TaP(F;}Et93uTW>?d0IOKU`^Tx&MvLter81go@KW=C;00+?fGVAyA!&!BD$@Z`9WktkU z$V#sOp57ZeYRxnb^$xPRN$;sw;(;)aM_D?+@X;;!3H<#(O;;Y5^YZ;kC6$y&QizCT ztF%0G#+Jwyk~RBIcGtd^M0+TbD6&MPM9VX02qhw0wyb5}_nr8j>G%5mabNddp6B_@ zXU;kA<$UHd15_WHjC*}>-sat~_(ywG__q=+X(_Kc%dqFIDo^9C%GYdYYB4CRT(F>2 z%bZ4uJ%0>uRg%fWJu8GXLHn7qEVtjEm>A4cQymBzAFo0SlA$&&{=P1Ig* zx(;ie3dEJG9HqzK4lw!VH8}pMout~3io5#Au0S})&OZDhtms_9(|_%G%%7F`m}c}SO7f@55ngQCxFI`4p&;qPBTM!`)9^z z6}5t|J>7@DYMuqno7BxkWNR@0EKO|{buT91qIxUkO|do4SSWzD?!;)YFlhv2bj z_?Y(Zt`$f*cuUhI_%IUB;JIKsd7`fwfAcLKsRq#V`Zu9J#zp>qbedT&)aM=dUIO)* ztdAz}oc5iZ-M$k)vGhXTs{AmG>5%E1@D}|~q(j`tPl7lZb3b2#M-7)z-Y#bK1N!54 zSWI~Ro@UnRG%~J6#ESYu=9R5xa zFO$BBjC;kXa+c}^lh+t%owqfYq4vWe!0ID5G;1imL^WBFXd(VPo{7XW+|BTw=&^J+ z{B6+cr2f}Pg{Ep3k z>+q)aehT6rl@G9Ox-Ql1FhSY_#~iJJAEsVqmB&+I|B!ZImg~x`om*i_)MqufY~1y_ zA}p;Q>4y|`J*tsPMmj?2DkPkSS>;ApyJa0n?tD<#F763+XB?*Y|Eo#jax~~@rTt>{ zLVL)d5W}WtE9_#U2e^XbSif`IU{$^$(wqtNJB8qQ)f*=@FjG1}(wHP^J{Hu`6G&S@ z1Dk?8;z%&mttJT@!7IUx`(E!MKkv|Gy5tvKUi9O{-Br2q!ZQdGf4AY+M$X9D!42ip z+5_a5+B<;oi1w$wRfU~|r|7)z9dV4W{E?ppO`n}$G-lX&xEWG@C=IA3%+@64^%oRR z-nQu{rnu@&0|Qn(EDQ-(NtfP4(zECoJp)LiVRUjWNjc0dFKb2YnOxcLL^+uqD?VZV zWcGd4kgEFF;@1^Tw-zzNWca#s4`h9eVs4eDl5`Q_S~Kj~Zv*~Heu9e|CgCuncsA@+ zDkyGCy0j5DcD{!J?c#v&0f_^VzAs$vRu{STPN*^P8olS@lO3*7>E}5WZNzn-lk7`v z8%df))3cuey>l;di}N6@;z6oy_9Ll)5xz@xed2$9G^!gCX5wBy6Fyg`5htD_Y}n)R zzQt!uD6@vWLEf~t%@d*OGIm@5iw^h(-i{uU_=GLK)m0QWb;FP|Gk`Q0Bi;tmMM{4m zkGTz%$6JBru4=0GH3#Dl2{o4Hzk{IK&wUh!+4z0c4SGMTt{_aV;;vzi>(S)49#6Vt zhDxh%y8IYPmk{5)VeRYn5`-U0KjWQ!fq4{bsrZUC9uP<4z-B&JqY>rUR%5O>H`^r= zN|tUT{g|(fKJ2IIY+J4+PKB#R#J6sXK>NJ8xEFGi5zot}HQTXctxtQ2AYUg=+79K5 zGqfJJJ)YWrhbdn`_rkk$(p9m?K8aTaaVsN@F3Q%`+ zp4ic62q&%*V?Dz`#mi{7FH@XAniZ#xjE0vLe)uEbnY$)aDr6T~0T|C%l) zy(O` zTS%CWNq0w)HhiwpN!uYJ-)+KS-#@d~NAg&~Ra%4WCUC=+fy!sFvUQt~u$<8~WW0V3 z<=H1exT{ffHMe;f`gGp`J{x-~ZK6?H?8^35Jm8J1_Defs*!Q%)jOhKmiohnVMpgpt)9PK2BCpJNf9WLaMDxG&v|Q;Xg-@yp%of z&?y_X=0qw_t696tAEH+$F!v)hdC_NUsrN+k+epSU2$5d~N z>!(@3^oD~J2GKrQZRN%5vXz1zw1PC+5Cyg6+e0~W3 z_UwSyM!q6l+81^P6cEOlkiJRPm~QQb_LmK$$rgLQC1{yAxR}-*)OyDZI!{KGUnfOt zscJk#uQ$?vL%Sh*oH#%?LGZNQ3AVaJ3iiTeAipOnZ#Lkh6>zgtM`7R_ic4C~z~y1XWV_6z!q7>Jq)iz4 zaC)}V5`6}|Aph${Iw%rIH)ErZy_JrmXF2A0HnTI|K4TA%F9Y(iBK__`Pe>!!ZBR1yut$C#k<2KJpaM8+=SZc@t@bNJf~?ao=R+O9|r|GV(W= zUTd;8b6ia*l{r*zAwaza&C&~TkP@{(^$P^WLsA|y`-HDm_e&flNz(xFnb5nlSae@k zqD`bdc1hziiW5FNSxEh~Ld8AUX=W`(o=BSgs)v;4aQ>_gR~o))ZZNFovr+k_P7ZCb zegtspwP=)vpt@iYy{9S+@*v&8KK{7PuFpw_vukxlT|n*?Xcga3qKqY zhz@^gf1*7t=)9@js^(aux*nW78|jKQ+utSG$@chByBkqSG?UrxO54qw}6^x zcMNV3k3HKoj)aOI#4TwTFN=*QU6k@AQmB^*$oNTAtcI5~zMOY91$kvz6~hhT!nTBv{|lU6RKK%1J@KM^NvH z`Z=$wbVrL?v(cvD2x+Wx*9!e^6S7JW0;BT*VWc zFy_6JB;Mn-)_78_uT@yenj824;goon8qJPHSFo0y1DVp!%3G4pQ#k}l@2k;RGX-Iq zP(2jNb!{~|gNyDal?sz+t^=XpiE&7CD=7wA$`|6~I-(xm48_9?sh?33m9HfZ7KPpRfy(c) zW`*qWiXshpTp(RfcrX@kmrp^}^SHOUl97+3zF;_Hq?ik3ZsCdvVaTMlbLisKAS?c*9jZeiID4%vP_GT#qb&Kjq}4$G zkUy?am<`m|gY;TYiI0!rrEh;g?_DEKzJXWX+{*VojbqfWAnclkr~mFJEbpLnxrV%e zhBPpKE7z&gdvqkm^^>4ba^1PD`(Y>hC($pQ`#UeD^py=i*WV zOg)fGy`pDuYvNksm0+$sg@$~4&bt%1JKQIl#07}_$rAhZYLiiULVs^|7|Ea$;L-$k%E_Ml z$-Z*hmjbLE6vj=uo)Zxr?f8uQ$Ds6EA6d}JQzrZQ;*~--{?5XP-&#{swk)Lwt5GL3 zZ+aEMt9`WIGV=}Vef0%AzLX~}om;^demBwFweN)AWHhJWV1DE8<4c|CJpAPve6#Qb zV>1VU+g1naHywt_v=>Fzyd>~RIR_bCjp0ilEe_~45?3_%taY=c^+yj&z{IiA{Tl5b z6MwirAL|^;I;@!qd3!J6jm$HmWp7(dsVL=6186-_SvFjmeqLibtUXWb z+KE4K7$g^z+e4$IHSopSQ!YAwkwu?qAx9fLf{zXDvBd5**7h^!cZWFAnl4=^Su;Ym zTx`j$_m0QD4x3<_U=xmkq zXGBVq*!-5gOj|AG3$^yQ}m zvmpFsHE#6#B_wR&n3uOrt6~#-up{?r+YWuMo|Hiy7lHfQv2ZNBFLbf1A>*3%lOv+r z@PbZfFpl=@K2e?eTPtH(MC2wo;Q5Q*z3L%rYGO1LXPkS!E581hf_ASgWcMLkgv*o$ z{FH768_{b6bWO04$%$W)VhZ(J*U^MTGC5(@U|GZOsiu+VCeYh<1|2tUhe`b#;P)mE zG5z;pQQCR7m^#pzZ@ghFO-s&;*a;?hdB6_pJ)9EJ)O+(AKbpUZeuX*XK7c`xGaqlQ zE3XV$C28z3+NG;_oZLMxwZehU#IoWvHo2hH1sv3&7#-h+s+L%EjY!U4cqhq`_Avgk_y7{T?aS0zVA}GWGBLlKkGr0zEPZ< zI1vrGqgHSAR?)=ian=2n*agTqS|3mMgq!Zp!a4WD^UU)8YQ7)n0drfl*E*87*?1!Y zI+6xFd2$e6e!MyOTCavm4?1U)&N8o&QHtq#0o*g^DR`T=!JdCSVb+XnrshmssRrMd zmx$MG8*$rkWB$waEPfwv%b4kSxNq75vpiFwIL(FcXx@c;KXr!3AzKKSnh6>!j?a(7 zu8(LgF5bhhn?}RJE>7rF*oU8}9t%UYA+WyhPb_MjMb990WzA{pVf;H|?CHRGpjS7R z8el5_O^Ox0+YaNmeNAOrVsoy0cO6W)(~X{uX(8ju4n}z=qZS2ePU&`l^6~&&+1gdQ+s|VO(|W4BQ`ZNRuHG_q zPcvcLs8rnkwo4S3rQyr@yGf&75;^1MVzkdp7IyoWHffp-&Nw2W``X%k*t;=kmwKIk zV=X+pdIL6XE*6C=r!W=g#9HRusjwIhBu(RiyN=+9h8wY3z*j6TYeQIZ5SL#bAna28 zWWi!R(pi$3*G<(Ztgvt708VL*rA7D{AS5$AITZTIHlv@bm*n|T?>9!Q5@lQ<7^na(Z3ABdz+ zI&$}*NP2#^LObw^8MgSspzv`W}Td zJeu79~wN%?07m8lKTLLiqf1m9Z{c(fer@x_!2!*tP~{O6wSojsU-ft;FEA>jYu1 zFvvIvoqrC579YL~l?Tx^_afto zjV!eNfpi^d@w}gyR=yN&&jftgqm3po@(Bi?_>P7}8`-m~L-@*Hftc3yI4&649STmS zGN+o6Q1CVhyPCSoKey^~%2BAup}ur*9${W{oIUh4QoPW1bbU$nA^-G0tZ-P`jq6Bx zd=3m|Pg8h-S8Cry>*@n}V@o%xM_PLmHHZ^$h>Y25p>fZ4l0V#og!fdl7J~3fbi4Ew z0=-Vb%N}&%)uQXd`@(ui+b|p!%$-f=!q<|Y`u~HAjl*!v_;9@Vp)PkDP*YAk`wbnI z9m7W9$L0F{&vE;Ma%gB-&U!EWh2HegfttqzFwUTNMRBh$8KPHK*ZYStw1I}gnuIU`?;iPjOFd@rJ}^*^LBpx7OXMIV@4I+xCV z)%k;jx7yaa37E3(sU~$D!29doK>UiU>a|1PC9^PMY$Kc%e;Cc`zeTq~TMVv7XQ_SN zz(@7jgRACJ|MEeU!W_s-vf#wUv`%O(Q*n4^9#3(2A(mgh3}*B0!pB}aVa}1*yzZI0 zkiFYU4%;*rV-IG6#?Fy<^6e*f1WXWbQjVk1u-jl?_M92C^un6P4JGBXc)oBdENr1K zLuvoQ9RZIp^Y3{^IK}Dqck&*Il-7&Uv`z$5{2Kj+-aQ~+>qz}p71Qkh zn3|uem@Vz@!#_6Mj~R92aL3*4n3c4JO`>NgJ}KTB!YZ+*P5r9-{%LK+bqX3o=N*rr zTEIdWoYI8t{h>wTXrOwdwE@Q zrC|AED{jBBfRUcjl0L*Gt#*T_`E?l9htBt*-`sTft%^UooYs@0Ck-Tic)(gaZPeTy za)>pX_>JC8v&4q@5tq%X2B%{eu^ZEqaY?W#|3940&nvE(qEY#x>eladcbp#|4AqJz zYH!+);n^djg{m>7GqdVvf?kn^6*YFwqZ~uR6Q;M#6C5Y*5mz#2a?&LfPe(}_6pfOW zqr$nlUGCz-^9>~ZF3`V`#)kEF&z6lHgYbF7pYZHyGJYCgjEa{rsxu#S-Cx}c)YOcCMy1Y1=3%buwo-*{nq6^p@}fa zVkf4TxZ=N6`B1cM7n1hDh^0k9I-j*oD`F2WJ;GlG15oAW#m0s}BCpA4<-=ya|J*J9_r?-nIJ+1@x zY$Wyg5cxj8Qq$N`2MNQmc1Cq=T)}9m)3i1!-A;2(__3o(L#1uF&DOMu6UCozVO+s| z%0sI~ZOJU95#+D@HhA;G zdl=y?S=X8S*sAU=_}-wVXu576@s@{p*0u){&IrZ~ui(mGqqC$v|v%XGNA zHU-;H$%l{8-T0s}-BgTJ{9skJ$Aa*cy&IFRyaK1(f}hq2NZw77UV>IDe`*y5kS~LN z7Db}uUXkK<@&#M8C)ykFV|k}>LbIo+cunCC)gKT~G0IE$^^W#oT{ReHeQ8vMWvceR z=rj2+zr-`EntV&nagWaf`L4dpd2DC=~GpUs*d7TXa&^|ZtQE%-Pb-u@-RR+$yKg; z9cwD0^?UQ!2~j|p&wD1{z$X!l=^c?BLg_ib*QbGgn<-ow7x!2=O~1!Ny`vd62*UY*?`vpKzxz6{lPe{zwU?PIsQF5UNo9l3&oW+@Ls1Y zP!1|Ss=^S`_Ka!}dxv^4uSX3y`3k}ZeI8oP47}%TgA(^_@==YX@)@?(_rtG&^O3Z; zq#EGyhf9DwDA1hZt9{ED;S}8MHXaFEt8mic^e|Oxpz>tSeIq2CLE;A7YE!d{14ti= zPyJVm)3M(e;Tn<`1@epNHekKb+}j}L7}coK5L6pd@p{4*2}@ehxqP>Lgng%>7 z*ei}FtpTL7xZ^}SR9L%c!~?v%vL4~|0JgKv7saW<>sYep#zT#q_jo4lb=e>KjhO*O zAB57gbOz{4h1HyLil2M)XO*rQQe_G5}MPp?0cFE`tSbGP@V9IzlgnTs?gHCB}$ zOvku8t<5mts^*{h3y=3p?qJ`LYD%YRSB)A3CQr)p2)9u68yZ@U1LxYa;hf<#p}a@m z;kBsFLMR3U82Mq92aLQwC!PZGI?4kmPlI{kyNQp!G4g^)8d1d?R_!!rhbyY{7X79m z-mDAl*>s;&&-0Y_VU(LN>0>HH`@B*3qx2jueKuD0ShVD6nDV9z zXB|*JjFEo@i{D!@(an&PF9PyJKzYX}EjLk~kJb3`O$1v_0rFeySo`~|J)Nma{t8r2 zV}?fxOlyV^T5Th=?7d4vJqwzzLPnnzVKKMJdmG7q7QL{^%`a#^+k!9eQj7L$%EtwB z-y!)xLAilc-*QlgK7u?HBW%I7&ua5d^!UrV_X1v|yB>a=nTw>6K=D(8<}w)4vy{te z;V>i43{@NhUtYubuOB>K=d7Xqx6Yx$swX}Cgv;r5VB2Ugt;^$Jo>-gHcp&e23@E>d zJ;~F=2P1tBI6YUUegqJ6S83Wh)!Ci z$;g`^<%v*!GjsRd+-?q&KQ}19uI``u3+$J{ zWw6YSLDFp$2Q!|~v^ftrPhN_2lj_&J>k$Nb;mue%?U6xP&Pt{nMy0KkcOo6G_3>}P zQv+hEJcQDP(G@DnCnp4tEc#4VCvli&6O}RR4{UzZCOLk{D?>TKCXeC{3braqvIdJet* z`WiQtUqte+B56ykhV+cWbY9x|9UvaZa1lyAur~E;u8K};&H(8N)jwf-Ga7Khe>LBn zdY(Xf0(6>2itRz4peQvSD&wvQ@~&Ei>J3y4T5m3&8>RIY& z@>Fk$e&@34XKIuuAq^(*)gce14+cfJ3*rb5(zwi_aZmW_I)~B!YbmF2Ky)RNz69O7 zden;yWJ+hy{1gA=tGuDVyNmQ^u;ge>P<@LUev^^CBca-0Z^oJ76G3xaBw4gMFYbSY4u zl~E5E$>Yd!`v}5(JJm-Ml)3F;4)p3Bn$%i!m3|dyHW zY|H*t`Y5jHMd4|m1k|^Jz9DOnv?29t`?2yNH8gvT^*QNE?qutsdQjS5@0w9xYN5h? zrg}5Q1I`NS3uqNTtA6U!IU!0ra_W6B>JzBGum^c@q<$mz?QtSc^#bfqGD-N)#_HKX z+W}=r_ykJlQ;!TOAK=e&JD5E9hnl;*jZ1fu5A%cU{{;E29pL!37LxDQo=!c5N-yP& zZsV%7&Io6&`m{iGLVcwoM#{M7pM-;7 zGhR`44%(y}akgfw$*GQynq2`__pUAyt~YyJ4H<;TVLXY?Lu z@>Sg6Yb3W9naZT1N!#H*DMu|tBR>jdzPkHg@3 z*5c6PRS@=L5w4sV4VIM|*e=NrgV(gg_CFEV)cP&Fn@z;x+jbW=>^Fg8EqR#1c?C^#{sK4l-d?;LGSaD66cKr)3xnIUSUgWZag)^Yl z>-Nm^oC9|+wv%gHc9#8ZXF!LYHTa1}M({W>1)kMQ#wJ$hpiyuRd-BbQVvvZ-h71Mk z%k|}gx8;!XxuYy}-;J+VPQgx|jp5m<#~3=diEPoZuO`~A2R~fe93IZSsqJ5P9KIP7 zDD&cmApN(bIly0!YqPX{>tOVj?Hc#`H8g&OS+e=$Un1dF0Ma;t;)*>YZZiKz#keNP zO`43L_hb8v=KA|i3U`-j*qip<-ulcImj*SJHdDT%>!BD~F-MR7?u8iJc!AVry=1S? z^~i1CV>>KaU?JCgR;Tl4n@IhAT~r>xsonIx@2o;NPh9KNX)=5Y|BNRc>hpwKuSKJL zBmTw5M_zs1n~(bFhvTPJgYc*;ZV?6}3$k(0vg(pM2f*$d zHgfF0NIp&DAxGAUgR~~4ILiY;#OlhPUF~u19}9jmWdf(gnPRV5Ps-O3`1{jVxp+?y`E`)GB&~{?KNkBlPb05s$}J=f)!^;oebuAn<3BN8IcdaJV!@G%@O& zx7X|}i`G_?7Y+USRBy&`cn6ic;(6kCX7Bb8Dej!=3)y*&+`)_97+P2aMJ z9b6`j-o^AvYl&+~R`Q-pI>aCB4-^xKelbVovaoU(z&z|CMeSq#cx>|`@htMKrsau^ z813u>1E2JOd6)Z%hV&ldIL_MXSufs`_m@ zxh3BpdRp|jkg)3+$3`4Kgp?1=XQnrQcKZU*{o}$n zb#T(2RQd8*FduF@MNW_1#VhRgvk^@`;IX}36DfBq3K{&M~I7U3FeE%3VpZP&F=p1emecx(y-i zc(jJqizsHV;BVpCOd+bD` zaM7FgDyBKlYckZDC+CF1aMNe3di54;NE2)N-cIn#ZjF6AB*UrAHKA5xfEm_5vDX25 zuQPm|9Adf&;#MAE*9{`!(W+=|(NI_J>GX~@?p0ToBsYK)R~*5sqYYH-_>WV)KFiZe^xhZU`>WaaKw)OWn8tGB?Rt!`mg=_|WYB(AYK)0Dtx%ga9ceD)?i)K$ z-AA?l*O9mpe#IZboUHm%aoyUx57Fi2aTrm}i2rwg9i2s$f>f{2ozA$UdW1g-3YL1A^sPOYZV!o!fg z#|Q}D2)YN9woy+-4`r*T~mU$hza z7D!u2&z1ey*C&Q(vCf4pa6c)C1E8U?fFAUIVblLQ$YGzZvV~Tw!P)Y;u|d7FlSm$Rer7Z8o=*HKL_G<_IE}-97E4h31_8F;XrNrzA~U#i>Ci&Fv52j zzSWX@_^uGg`=lV@worUSyeI~J_Q8kWH$Y12U`F}`-){efZN1)zjfq_uaS>{q8_2Q= zy=kAEVEB;BIn4)OS>j6=rz58i?~j%KhD@AZKy&ew5r$xDyISrqR&K)YXZy%*1KUZ$ z9a;KQ2h=r`zL}Tb9lA#7pihVY)VMIRyDq=CbDmWB>p3o8k>Y97y>x4Em8iJ-XhTFT1w^NS|OrZit5PO|EkF z%A=YhY`=(p)i>a%wyov%um6~h`vxGMgUZmQIDc{|>FY9hl6DxUt!sd}S6ky%ucJu1 zfyG2-Yl$z#;QdX|(e)ABog3s~^Y(x|b!riIt9OU;@n%(>DBLaku}*0uF=c8E`Iw%Q zlm3Jie(NYtjOiJGK8W;yR+5Gz zJ}4JgT~qPGt`|W0rZN7|lr&YOhH9GmEWZY;y)Ls_jZr9U&YbQ7e9lP?VKg_`u$qxZ zLBdQlJfkBY9a_cfPMe2Uf0d%j=`G*>Fr{~+oiBlny8)~?;f;@0)lvGG?(Yqll{b`0 zb!lz-3t!S5(KxkDd*pMv$o`{GpsInmmF#DSVx1@D09R@)L2gUDz$#TPm+d$e2pFFS5h`(6Gi$+M_hcy5Fl&8f?Cxb<7XN9p) z?Qsk&Si6IMqdGkB?T#nbUBb(ju|RrOD81A3tG6VN!2k5CCjXKLqT1JzF61pHwdSwB zMroARp}NG?7pp6MD$ieXQhHZIbImAsfpnN4oI+3YP<)-IMRm`lDe$S*4V7l5yo3M# z%)sQCZ73GEaA>=I*fyq*xOQSSTif6m?tAjvW5Iv4Cg_fr2rzEUDxzknvB5FcP)piH zqwimV6=mK`yXPT{cRYlx>~*=z%TD0#IbZwgV1`C%|BZ?HID505(pw_;>H*02ZjIZ& zuI9z9wxfy7Nwn)(AC+#O`)5m)?)i`_VX@&Jk#@WeH-syiAlVnr{wYD@56NI>*bUjB zw5rc<7$1jJlj8S&4?#69zmKXZTK~$!o3HvoY(1&&3Fa7oB5n!Kt2Z@S5T+|F0>pQy zJXZZv8+hiixtH>3$KeCvtj z#V0@-2Wrg!9!3w}KZ0V9XYyNcmz~3~_&&(7Z7a3YhnI)}H#A5!h;L^Z;k?Ti$uE?< zPp>@(y>sbY-v>JKUug(7@+iZdxp(m4+@bV-dUp@$K}hQQK^XTdb2BTu}UAV^Cgc_&sjZ!ivQ+K_zlb4Gqj>*7)m z`gWkT7+bC(#f4oDyaNq}zXBJRSlpvSXQdvrkl8k6?mGQy;P#>z!tEfw*wBtsy)nwe zs@$CRek78&%4;_CgHT?HI1eaSm2ZK92Oi>4ccI2rg z?+G=g)~->o(vtdmo}^#Ctq>{?bo#ADHTMOZ{efx&c6GQ_g`dVFpJ)b;I#uQEc6O;v z{%nVs_3l0+{3c!t#@FWxlqZ8-VO<3Ib9g%|ob~QF4?iWW#B~OL8TAh!X|ye0Q){_M z^S9@3nzaSO5EvZLml3Zs>jzD+0X_RwHAA``KOD%Sy?v)D9+ltC4gh%u@qWMs{2lOA z&^4sWKczutPSKa7EikFBCv=V+49cSs&LepgRDR)P>}^bGA0qnsxB$fm?qH_kJ0KsU z^0aEqH9}XY&rtuu4pfW?ljzLrC7^l}VFCT&iWrBlSB#A6pCQ zINue&-?$EayU_ZoBPS?6f0?C+o-|ya4vJIg_m!5c^099s=AzBp)|_i?c=v-_l%~O1 z3r#usD;(Kw8T_|tzo@-riQ)}Dz4m+UuH~Ir!$qsKXE%ib`CZW4Ks{xuL*g-hCVwyU zG_QpeM@Z=$gVg&VUfhmXk9<|}LE>XZy(4XA>38AZxCzzhXH1R1h%xkDDDel?QnJ$5 zZ1P`IPIyW_2(`C*bVB=$MOgcv0}yr!)d%{#+!AY6YlYM|LFygApKMPdjjMC&LkRMU zxag!8SN@iI3{d0x9Z>UDzfC{brkf4Qs|CNBjswnSXxBw&h&<~JTJi`uVulY>8j5fh z)*tSnydNXYNxs-NuS;`NKHkxhlb;k63o*{bQmXq={k2{Rb&xn%OWp;>wp}Snm%-W0 zIT$rAUPF43>a;!1dN`O(dauu^mw@+H8v}QKO`KJod~h>)dvKSk`uRwE_)Yu!1Yz89 zw&h$TC{3+COI}dWeN^!c^*gG(YRM1}q)CZb`SLy@xD%}f9_b+4bu!B%{U(Mk4aOh?2evwhDCco4|HseWKpFN3(8Qx6o+EG*Kfp6L5A zL-4ikHtLgk^W4)-IC)DA<(XJJp{{1fQ#({VdSTZzM%>Hxj-tK8`i9`RYiCGbe_^e? z;u+P1cJ!7;%2#@*-T`S`={%UuT{pR`Svj!|?St1t5H8}5FYS6x)RP_Kn@rlq9*3gLHk zp}Z2!kMgC|H%!tjX_bq{(`SSCEH9?w(5~frX1@40Q{(&pp2?`T2Rv%_)kne!I1@Yw zc6N!RbqyDg@>A?>I*==01mimKHWLp)`o0m$%PO6T@drzYn`;C00mM_iEllY)_tj^q zpH-pyyEM-Q)bH4Y#9JD|boe%N0_!-Ty`WgJPCrh9M_D^geJ-TjVdM!^PBQZDtZ7Ct zQ+z;uaHUm+!AMh1If|sIHLqrmqTbnNd@^nTR@m7SKL0?{d8!XdJ&dP^Cg5raOBTCOP<=aSuX-G54>$07O`i5XIGKTPHAoY0? zL5-h!NJ=x%xeAw%`VB1l#e2>E!Yx2GA^l79@le4w;-?&Jwy6jISE?_SulQJ0#Qyv9 z3W*~*^@oXJ&q55dIXzvHZslFl!PIMy4_7^1e`W<=Qc5wRj7R=Eacw z&w-a2Ed{qgTO2#06^1`+0J|2p!hz2#JONt2x9o-0zJ~Hw3F&=5E8cvX37q6BtA4kJHSinrXJhZh_N+kexJO7K;Hf9GxrSUPiyQKw0Y_`XCv^=Q7Ob z%F~ZnNd4*tVnhGh^2N#&5mubX==-=W@H&(~o(0I`*$09WSE*m-Ma$cVO zS;%I6arM?;__F6l+&9fv^i6PKWldA@G3{qlw!l)-_2u*C^lnAZYVv;GQDI;gCmge@ z^IW4BNcRPQwz%?8|JN|RxE1ehmm~HEXTZ@vWBHnM>v8{s%UF3d7Ei{^;@iIu0*gRX zyglY6c-t6=YDsm#e^y&~bWSV7?aDB^MGZOm{9l~r*G8TiaaYr-I0Sx(n91DS#ZcbY1gwf2VCJqb5I^}FZuB;Vu~|i8*vZdY;|3@3T+cOd{Nf6{>)DAv zzC2xiEqpKN8u;PD7ND`AfBp{iH;)3{E5Beuyg4ih^u)m}Ut&YOnUZn{|LmBI^!w7{ zdbPa4EvJLbzapM=&J_l23go-gP<;708RMJkb5_J?J)jYvetapuMiXhbXeP7$WXkU> zD?*wNH2l(0CYpC(=G|M!N3$DA>zH`y?cW=_-aLb9Ts7&PoVl8R!f_(K&s}FRj!c^k z5eYYuz9T6gS-1P2aM7@u&~=5K9J!|psJQx@oA57_Y-L~m*3jf;g1@n;nx$i z#;)xms1`PkzqoZ8G1&2Hu zO1~HTVcs8!T2DK9GUhFuoi>s8o46Z>8g=6TT5ZDA{yXKNh+1+Lzl%Yy>(LpOdKhfd zjMIE-E05O0r-${qVXxhI<-$7l8@BR}^SAL12inN{=DwnMdTpFvn1)L~Y=rd>I@6r4 z5#=$fR9rD{nJbs(r$p$PFz95ohlO-JqTOLN7P11`;h|@#@b2wW+&ex6x-`>cwqYM} z!O~Z7ru7mwCnO)P(fie9B{^`9_Hm%P<132pYy5|=Vp`8Vu)kw|UO?BDu=>ye2ys}$ z%Fb@UuK`yy7VVN%yYPs0;FOMFy|tITNAGLT3F!(g!W&8B0w4I9 z`AVDQ+lX>J0iq@r-~i1#dam7%Q4Vu8pIdZO*;3CSQR3l%l+@g2bBB3;<^j*fM1CwC)1nPdK-j2-2rwrswI0>ci^^Ofqcf| zT%@t7&^c`_XVr?8R8PF%b+&BucmuZ?5h5H-cVWQW z7nMye~!_dNuQGFwuPPLXwAtdoR$jB*rm4mL!E3F8)q;F*nEp}6`xIOxAZ_*xoC z&wb_aev~(e?-OJ_qs`bosUrlN?55w@r*%oZ!In>{BfSmC&ldLQ6Z`q9_^@8-c|frR z6_Xls4v}WI;QJiR^19R>s<|5SnR4YX_{HVOt}CcVK5RXnSuqVLH$=#-I^3@L3ZU3R z`QEmep6!MIMQ!tl>@|gldl>L>3-xj2AlfUQ7U54XP6gjWD;XE%fC^`y#t(zptIhcC zX1Um|%!Ct`D-5g37srWalJJmK3o92?+X~N-upbubZiRrZ`{2KdT71}b(tf#%WtxqE zfX!QQt(OawJoA8WcXkRKtHZvhM8NZD|M445{^GFf&E%}?6iB)GP0hPJvP2gyoN;8C z&hxA4eU9@$N%O@%#p?3cNbfg(87xmXJCB{SvYGZ}u;5#VQ*H99G2U2j!yMjjRyZS5 z;*YcOD>`BQ+}&*1j$QJLSx1F8FrmX2Ou4y;%`!a$s^*&4Z_JzCA1{}u{gcmX?&E}! zBDTpWZgq4yy!Lw!7atnRKAYEx=+(OwuK?8;b82}TADs#pI@KNs!XKeiKT=_dIJ!GO zcZvNTJS$2u@`xVNHLB`z$+CJ>+b3}6TxfR~At2Vg5gw|7ZGK-N&$HHF)lVnldKjDo{?N&CYME$%@V3-Y1&4@13}$ z6Av9~TmV`!CX1&>s<=V@y4|_KRXeHbB)Y{~k5T{K!&Zwe1Rd)@7}6E>$=@XHHG*OG zn`Nb*9&<=YQTPm#Og72^&RxLvQw;kaH5v}<)C9_R_GJx5^5{DC>L-4U6$QgD3(?`*s3cg_?c>E^#GOFHQ(9XuO|HLw8z+f zfU)ee>Lx4j8VbLmfh^y9SS&Yh!v6(zM8bP`=sFgwJz33SFTQ4VU+D3|k$U`B$vI5o zGa+zc1L;5OFj{_(#3q;5!s6T^Y+;-?GxWI%#4&`Y)$r%phDaER?>66IbZtRn_E6(r zPy30CcHK_eXeEwoW+4eP^Xv_1@AC^53O~d(Zzovw>J<>i@Fmx?u*;b12<{{OR>@nvOX1=Q~yZ^ibr8(93#sMca_l;9vM4x1Q&?-=@ zu=2sensE?WtA!-J#ppXcvez?(ZD{jx3VSoR1pSBq6*fB;iEduB@2ZtV(ruEoo+PYb zR2$&T)VRI9J z_?Rcox(ub0VuYj25H#nY_;G9Q6kfV}5FDJF=59s(L8T+|O788WIzQgf~G>@I3v~H?zl%U*}>iTVd&%>pjI+C1#{ezGu9ct642tO?WP1 zoGYNhmjhi-;R>Dz?F>DvAF#aZH=vbWOK5VrhNN)|;%%H(opIt>_Qb80ByG*-TPA|h z?{U27iVi%;uMo`!KV+|xGk|akKQyjMnDQ5Y92B^sXfzT(;TOXoB#xHL_FPdls5nte zdJyIfNXHdcmyk|OVWk5qP`CMOHE&G$g~PoTBk5ojKUCb&IID%iQaJaZTqtd!_~=u& zSQb|7A=}6PfdtwcKQ!H6>Ko)h-IhgFIB}#!J<2I7Hs+rJy4Ed){Jn`tF%(qia=J}J zuJl|==PeYs93UPiZ%_&0b|cWa^tsY#hPY|}aH!K>Hjij;I&U-uX@~9kaUVkT0?&MV*x`*x?2I8?M zZ$WY9TPI_<(PR}&H+aQJLom1N8}VTJOxbF}RLs%MW8FGDCLE~2l}?`ebQE^}_+1m~ z9{@9Z-9Y>FRGPc)Fu$-3C=J?h_$WSRD?QI$(o$|Wt`5Ezdvo#{I4U$8Of>z#D0ME9 z|A5B}`vBFfhVGB<(Uy}A$h%_&`=ve*vydD%Bpeh(OW+(6O+V z%=ETJ(urVYwVkcJ7XeotuOo2@AGzlwI&{_LRAW4GN>lI@(So=r?@a3-g1i9S8*45H z?v6xHQ3s!FY{ma$2Y~zz4_dnr=zh4@`B^xc_W_s0AwYT$d2e z_;qHe5G@j9(D~_bxVdR5x?6_OtI`b3T zIG_u^_O32zu0ot{vkjCdApMPzUlu~!f`>r(K=~Pni;T|Vm>xgL7w9pc)oal8uIg z%*c}n3URXadn7GPc-Th%_aav)zSInVi=UK%9c3iY_wSq^4+h{vHzl%SrT}0`Z_`=wv#Fdc6%N z&4EfgDC{KMrrO*L6kFyT*OE_p(Dr{MU3Xkh-y3hzA`J~fipbvTbDm>mX3Hi;_TJl9 z5s4_J(y%F`B9-c%=OCNNYS=>d%HI5*)9(*2)aP?Q_nz}S@AbL&993tWdWUd*i}+Wx ztKzyzvy zG0Gv8db$CS|G_T~rSPDkz8tZwn!;g5S^-E$;q>^k&~mOLpHa z5TpG}>UMBQEqT`BQ^Exk_uv?%qxrl!+`E zU`~0_B-v|GG9xbxx*1M<*wWR?n^2ZisC+b1Ukl1Z1ZiPZ`Y6F7NaDnF&`&xpbSJ}b zU)2n(AL)%b&aoAmm-01Kp3P^I2{*mvqqMV3sy>dc{UjJJybFD&UdKs4H?SJ#4+`o% zByWxvKZO$)e4;GC6=vQDLdw>J@^TM~BE)U;AL7aKB|6I1cy{^)+&W<)J{xIQp{LBhZ}KpcvqytT@+n923M%A4c3*Q*&}0_%}%MmbCuAa6#R zbd^w=i~KPDYiP~mx3!Wv=96_%)3&j86TFag9E*N&ku|ASSG^AdemoL=0$fDDCe`Sf zfDK~)3lA|SWow0&q5J`QX1!1~fUWAd^IO9-m~qUN|DZKsjmv4+V-lN&D$k z4N~rc(9TSf9_Ig==P9fckMqoRq)`>F;Js~yikm&tTYqHBXLR7CrI0ch^6`(gLyJb@ z=j_e8Wq)S@@so&V8Y51|Sxt5^l|$~W9IUjEPWhBS-d_Z11)zM8@Gwohwrd9DvBhVr zbR^6aDm$h8gDE{g`3QF2x(e3)OcdHnBf_gsy0G}6oOo5#&e+GuZ%N`GR@T9TJkDCU zZ{eyLdDH;XcT(vi%IQEioc5iK%2CJ(g>9Z7Ha+Baa8<>qJtc&G0B?GNvyi(%urYQqc6R0jG=!2zy8)U1F5E64q=4(p)MN#ttK5bu_1F zbvd~LkN0eCt1<~eUL0QaJ;})5amq~u=`YIl;(+uCQ?GFwYzmIXZwND{GRo34#ov2g z?^9fo?zqX>hPg{JAC)-641CPUk*Muzhm#W z^|;5nYm|r4y2C%6_#cZlEXXApDf`n%%PIav@}9((e}&?ZA+(>A@?{erq_UEOjbvq{ zby+MTLHSG4kE*ZafZr*oJhkGunomp=|60YtbGRkjTsvR zjjLAn`n(v}{4E9!b%)PJWso`74mQM?z=>loaDeYRsDJ7b`>~-9JR7?K{v0sie*Wua z%RjXD;;>+})(1X4z6JM9>?)gW--v_O2eOjrRAv}oi}&7Z#r>P!W{z2j(75cAZfTo9 zmh*QDd;5FOYCy96B*-Q1qjK4W8Wd23%jBRXEL{%-?ZXl5xVdB4%x zxKBadl2n*{Zx2LQvEcPT-|}4eu)ZjpmW@B(zraU#cH?$;TAOH@sjR;t2`ArkmA5FH zu3hyNqw7W!pDobx>UWlT_ky5n!+vd(VESYiv1GP}n|^l4`dp#kPM4n78#EHZIJMi{Zxd!O@dYerpp92|h3B zb7knxHu5>G_4=(r3wb}gnEe>MA7~{a@ELX$rnR$`?%BU^u=62!Z<&!>gZ5Wh5w(_m zEP0Kuh9di&<{+20(UbmPGts>7DClF-3~%hS=2^w9WbB?}9<{+(zPD@*qbkMYq=VJv z_fzHUPWXJFKNvbC0)5^&q3*GtRG;Vgyph~HzmZ(=+e2LP`w5d4hp^j?hl>+stL5*f zsM|g)7T@2m2T^Hjz%0;$dsg3!r>l*S%dU3P*?oNji!Ewn+fZxmo|B#YHfFFWoY7mp zd#sZ*K2lxVrc=IXvfe{3owgSWZ>(42O3!7URAY{#Zv7SEgQL+0tHO)OuSjFd8oWP+ zvweNosy9CPb8S0_+202PH}-|^r)J~zd+m6HJddqM$HV^ieLZ8WR`QyeOL(hB7V==z zUGT`ONc371hmRIK!8dOv3aTB9e0qR8*1n2k8+x%n2Ug+1U)H?!*~@s~qLFOebsI#u zE#-k<|LMN89L;Ud>Whho4@sNE-*`W;l?({!1xLGW7oV?mq&ZxJieZc(&n*#j z?rfRICz!CJ7XP@jha6LIgU+#BWRdQ^b)N1Kk>T=#SuLn(y9er&oW{pO8KYcqB`~B7DH#_;@9k)~H-Svg`ZpAX_UAzOX+l+#N z&1XQG%MGe;TmE!nUAen@Juxi(1Y5X2NcS-HAkaL+f@PspKmFzLL6=3n^Zq)j8?0(! zPQAU&vvJp2SZkg;?mg5GK98>=hb^ticLayyq0L;nUjE89Onyt(Gmqt^0F-ymgI6bJ z!XdxIKtH2(>-sRgyKgni+avJysrtBOSs`IYQ~9%2FpTIimC@M2$C;k|<)|Avx;E|Y zf3&9I+h*MSXbnEi9P3OKAO|`Dclg#uNR`hWJ?|uIF;&&?mG{owTFja1G=7E zmzKw{LqQ018s7>gWOf#_8h4ZRPkBrF9A4dhyBtz8nmZfO9xj19*p-lbII7SbyU;q` zp=^QNF(^nkA~^~kS+vv+BupVp2}i;OUFk!5erah<2zX;6oVusuJ$%E0To>cnt0qYE zgV((OF*$6UT##xnr7*jYxr9|DA}eCN>5x-O?ZfbdI4IF0xF z?IlcEFW&Ttz(J#*!qclh*t45C{>n(xS)>L_Z|jjrHH}5rpR0M*-X3Yf37>U+uA5@Y zNJo(jPjvH~7h*C5@b)hp=v)m&(yi{Y_P+L_Lwg<1Z&-{=Dz6i>-40`$VMe6)Tj0Ax zb}VN9Wr&;h9;i2|mIlGOT{Dns8y@u9Bvk);9p4QKFSF>`$Qnl~!NFgV6@Bl_#>uaz zZD3zg4q@m=;s3P_C=4eoMfF;b)gyT6ep{ileTS+~+STExUp!1+%`DVW-EqQ4?BNgv zk1Hog+i&6Mwze8<+r;?3`=cfC6dq`mjhllu!kzcq*x*6AaA)`c-XLeYrut;{fY~x@ zM^&6Q!$UUASf_)|6D8G->~lR7Hoxj9Uk`MH&M_x3Y1i(GTK$#LOw#e>6VC&ja2E1X z5D4G+gx;3C{)nxZ`elTqo`=jT!{7rnM!mUA5-wxWKLeg|K!cSZEtujK>D&SB&X6}7 z>Wo&rt*Wx$-yqEYkPdA&H6)C##K)efCf^;3(X|bY zB%b;WRBJfNrYC;=na%Ip2V!8o1n4mH0mjVPk92;xd}|`kbut9PblhU?B5U8fi+-)% z16^kh&4KJVr6;ucc1QGhwFudKdrthsvoEg&sug@$uNi-}?h>|&IxaRwPk@w>ag2Bq z|8-1d9(}Sv@mk)}G*Fz|BHx&=IYjsLShbP96P#(RYa!JctKxMS#`;ZSeis2QyLLk3 zAA3|^Gllh=BKmt$jmzL8aqxBX8pXe$-b>sGO|0Gs{L~Fk&o%%$CM4~0p}Ml?hciPV zV(lKqm5_b866VAe!Jtn^>ACFXsCrIu@u*w0PD@r7!nM12{a{0W?eKBDx8{O~Dz(8L zufBnvsi{WTsZ*S(=7+El2@5f8#6rS|jt~M45LCyQ>oI$z`Ipp#Y`kL-?*6YBM|LSh zT^CbM=SlPY69|K-XX0qh+CVVa(V8#lJCk#>Q&?yeryHK)E)Qp#s(<6aa63+OrSyk( zeV!4gvEco--(nAzCE~fM7qu2`JYZh-Y3+1!lJ35}9^ZMamhR%O`?~%uA_!xqDja3> zI@}R#DM{CW_0~|d`#K$R;)Za;%ZG6L#}XtiN7euGU2`P9%5@FsDz2_*f$NQ~W9)Qm zxvhRE62{=z54Cyx#M@x;(7IyGr(Yjjp?gSIXg53Egu`oR%8R2y6d%CvBM z6r*a8t|t_!=9$t{Wo@Ep%)a8#V?$)nx@>>9*jfhXGrJ3iMN^J z_`2=-KvdwX3Z9)%>odFiDg?*p4%0r}7$RYoJ}CU8c>>Z@^nB}5W_$57Y>K#wb{jP@ z*J(BOk2|I24K7^l59!{Sf^b~vL`WU78sq;Y$%JzT^7u z^ZS9ch&VjB6iIs{u8+bW@{a09vAa)CT(x=^q?MnBzg@lf_yP45E(5#Y3-0;osJGKl z^G#57t#q`YUKP|Uv>vG?-ZX746`ns_SV^`T+X55DdSRRaN5#9I|9Oc{SH|O~DX#K& z+E(aq@CEuBI_Q*kqj_L{zr1Cy(tNhS&YKaR;jD2lE3oHQgguxCuat3~W1;SqIKr?o zQopza)U^$N(4M0Tz7O#HzAUuM9>#1~XfX zg2`1|fn%1FSaW|fs(Q(+QUX5)H;HWlQ%IWb8Rg9l zc(ckUh2s4QwdRA<;Q&VZQHHiR&hoeFde5pvL9=RzAGGGoV1e3iD6(~ z^9to0t+1HZ=r`Drp|lXX2M&Yl2lvCK5r=ZMb6419tqR!HGl8DVoAW0I#yqBD4I#qa zrPb9`@(Z5F~WQ?h_Usq3#_Sny^t~esP{JclLV;ZgJ z+gBHvQB$sdE|IQ_a4sAgS%>33UJ2LO(!HxV9rr9b2~P}YPoBnA#PQJCss}(0d#w5S zn*reu9cwhK{*Urv;`3WSF0$_dt*wIzvLOiV0G< z^(61dNkb_fk@JtH!n+l}fV^Erohv+*?+z2jbeDre?~6?hw$U-G;qccw zi`|?%7KW}&NAlr7-US~V?S$lMne&aq)T8ZrYKRs8Fz+fT|McoDJr8BtPoml5*?nsTC62hac3Q~n<(J@Q1-4;s}C*!rJ> z>-Ub~ncWe(vcf4)GxG-{Plbkt1HftV2{q43qf>odQ65%i?yZN;3FeaCmouks91wm` zAC{7LIe?en)a2(|*Jpcr92Ma%m3jAPMlA11Dcjq{35n|<^~-HsYW|D7+fk~Ko>(se zbvD2E0eu!yPNF0I0ZxxT2s#(EOztg7mjdA|#8~~q#6~-y%gD7#4+-NRu6)PaJS6Vd zksd+BT|)2f49`FIz2SW8eL$bbid$N7C(9}dJEiJB-`3Vb;T~ZlsvZyD;E5`GA&uW| z2z}?UcJU3Ebt)1a-gMHDmQ=We?X^EZI#&5(raQ#Y{=-o4o@*!x?{rnH>Q-<8`7q&Y zyoi%t!m)nOD|9mX4cbeKp8vC^y}~Oeh|yEqiA!I5K)YjRQhdK#f$2(%t6EnY;9r#; ztf2R9pk9G^x;IzV1o1fB8n-;Rqh1W@@AjPfuEO(>#}br(u$LSCdBpo7;eDV1;^^;! zIE7pE>;}r;5LSp^K|86omx-|-Xf7`|uE>WLKHR6X>=1{mvJO-Il#BU+bySThJ%Vrg zR0Hq3Z6)D_Y|=3VjmHhu5}IeJJ`&_<0qYmSp@!zrpzd|1vaQpzS5@GeU*uLG4}c5z z>r0wR)S zcPm!9g?1U=s8J3gh<_x_C*@xfl?PEe^341%I;s!iZ7-2-Qw_<-0QCgFy&(vco~QQ; z(*K}1&-3{LNppb1p7aIcNJ;)znAhy9v>Z0~EP!9HZqOc_myvXhAYNh@{I`i~i)%oz zV;}zYbQE40;DM!9=PPP`y5SQcv(3@(*ClrQa|Yg=`-t*|QTRSdPZAgLZw-zJ(k`r< z%LyIzGUWWV#YW5WFx@6X;lJt&TJ!HNtc(mG|I{8yTg&?|%XGH>U37bvbb<$E1xQ&K zD$W1*ZZ)p*nSk1{%6H*@y#sU)%T{o3eaeO}FlPQ64T#I8YovF9>R6I4)CLdoRD4YS zxQQ0&)K6?bd67}Q3&K=9x$+DnK4D!O6bZr&R5hx&hO$VZ@+ZQZ3SU5am=S-&pI)~; z3mHen*V>kjyg2?e(%7(e$JS%x)c!d4(+X^Qrw1qeBb_xDHs`d&+B0o{@Lh2YJe|0U zsWC0J>cOc8QE`^_R#T;48R-W>++2b4i!HVbn@jeRyaxGTW9VD-TO%&Uj7}!3_ukD& zeiW!bVD!{uf-(p}e9VX+aQ0_sd8EHLP__Yg>o$S2vmPp4CW>hv!DG7qKs<}2S9p5H zL{@EgmTp&iWqEyGFulJ>7)6IDy+JucG@~2|DH}!d_^KYUYxxl*4ihVyoyQh!a$sA1 zQ>2_0+5L-*xB`@(FzZi*9+z%As{9*ejtA8-byGT> z6jS$(hI9J%lKLKejlThTY7gRPrLB=LMP)sN9a;D-VjL7rZp*qfp|w4pROd6=&cTOv z_nG2o;&=EoWe=!qp;4(Grz}ThfVgk4jaJB1y4`Hw}v9rr%DX80JE2PIQ&w z26R8L-+BCV_@nrg)tGi1~a#(_->T_mR% zPmmKb&jbC9*-q>5X?Sn$7}7?D7r&*oA>7&Q>05Q)Yb;RJgyXjM;`X7>?1lRy;r!((?4ed zu@Cm?I@9;A9d@;qbvw?b&wk3D1ZLyH>RsSUj+4A-V#FVHNW&U~JIW_@QC^HPmiv|j zv#Gux)R@EjejCK24$Xk;wZy#vwBO2o2S|CJgG&yV2u++}=&TM;hknGF}C*1)9LA+(12P;A-LUlu3&W4GPY_{3d0_E_@+(&pu(%e2Kj_g}QE^=`1(ux5a~3eE9o+Ea9#(^Bp;sstxmcZN!{j*2M} z&*0awu^8|iAQexG6UR?MR;$IDJew|l4SU9xeef4_E~s#9+KZ1cx4)ZgI=!>(yJe?* zvE2`6*@l5@&@vd%>=?R~e$93Lx`e-rHH3~&j>E%IJ2CA>cZhy!D?2o4FU?N&S9JqU zF&CI??0>j6DoslnpN+oTHqrB84*W?9$1W@AJCg@f7~zF5ZtBiM3ogK~VH5G$zJ9QE zWh35yV@oJIbrSE4TQ1G&onBR3E}5ml6<<_rQXi zIqcTw=`yrwS9$&5ah%mC94BPl9^oSVqe&nmI8 zM$MU@d2Rmjg|m#>vq7ZwILnVs2*j^vI$~G<$KqkPZRj#&0T4IJw-&3|$IyM4+OmuM z;9EyNo|;MgT%r+2vF|-^VECesR8!56)!HLw4~_ObdgcWicvS~$KDrYw8cM2FwjsS3 zh{wQw(;oD>&>J6Jq zW}I_SjQ?|j`C4y;p?${45dT~qy$(8`>x23J)g*B|pOiWrzR_L;`3s{!)gEyV-YNMh znq+z6JC`+BH?y4d!bezo_72dT6TYND<)0Y}`{4KLH`p6&A^A@X>XvWMW&8I6@e4DE zic&oZy}S(Nuf6(c7*Oe1^(^p@$O}x zVMyL+oP7I%mi*j?-+8+NtK|9v^*i=XkK}4@(tJsSf7Vs(Mz7##9wDM&kkQ%Y}MGUYX_FUuE=ZQDvZR~)73 zfq#mN+*iXCJdeVGw1*ZOu00(|6}^^eAIa z@8g8~QjIh90RPme5oxenbdB*)qXXi$hJ%^<8EHc~PPAhV*P6)RRc8`5Y*JbPCXe@L zCtBx;JL7)iT+cnC?-plI^WFpbs_bwiE&%$uM!gJ5TKaT{r;_ z^=rckk5Tb(_A*0=^{FRk=SK!@AcR28bD`MURcUjnVFrDWQK|L*Bth>hu$FbZvN)SfK9tpp3dxsKn zd3PMGA9Ei2k8LcUhUb$ONrM|FJIiO?eL&T(;ei;8Tx1MKhK_*hd0X*z-Xe8QjC2Y5 z`1A$RYf|yIcUfgY8cI?vDUFBTWg$pf0@l`Up)?U@R+=kFi(v850NN*c5^l+@%Bf#i z`_4v^#x9q18YsO@bqZz&FJq5t%dmlqjnXLC*fxt%-!Q@}d}?$FsORD7lAciF^MdKc zuVOhjEeTg2u^nH=z?ZF_pjD5qmP~or@S=7w^BVCKY#<1$2X+>Hip= z8xF{A&y?PXx!{7Mf166GaYj0oz4I_(CtqD)367oM_M!R2)eSt$Zk-kzs@=e#4W_X7 zoq;^O<2ozxnJiBYACAUN4=~aO*w=Ii5YI5fqC}YM>ddcAZYwt(e9lN;@ERGq3N3xp zp#YV>pW5Rc`hAH<hG1#mKCk(mFHr1IZXfAn^9qpj=y{mYCaU{D@dZ*-%vaGCZUsosbi%QV$H*Y0O% z`Th~m2=`;go!1q-Kspq<*}uZXYdtufuhwE&5!15_BAly@bS|hcC-|HzKGbT$sBWu7 z!}qoL_;q=nSJP94(g~{m>$w{ckLKZ=Dz$i7z**+wbA=hbu0z*p3knOLnCFsK2tlR& z8ZO;IKJ6IcVJUXJB1peAAzl3${+wL^Insi=2K@rNakY5hDdd;K(98V*~h=G^6&CLo>$$PK>DhO>bgk>J%io}me`_F2~&EU zMeZWc)ru?MU|}#qDvl2HTMdV9XNi{b0Vr%&=T74Rb+^AouLC2IJd@U|X&{guV$Vkp z#p4r?;h2Z9@OW@bB>g2W(>`9}@=i^S=KzP+ip!W+?b|RUx>)%-)}tEjOMS_k>m4!1 z{qEb4v|9oIm-AqT#oIXiV_C|EaGjk)u;V9hV5D{wTl=@~}am2`HD;$Hkx#RI8N zVDL&GPCg%KJmJUuTeR-Eo;>7{4yKb&W00LW;`byV4~zu~jW}s*xEmb|&C zf|R8iD4jvRUfkT+M-sMRZKL6WJQe<{yprjEtSY;FyaBxmg3$Xn+uh3 zbJsuY+k`VfJ{+13J&uIOKsdw6pPE(mLW! z^LM$~+GQlu=^$x}@RNy1hn z?T4f#k#rDFZTVXD4;FSUr0aXFl`q{a`fp5Uv`jEW76(G?=l$yZfp`fO){-^@(rrTV zmk+#?l?~85!!5INAdi4rV<#QyXrSJQK~}WyWo{{uE(hX$R`7G5;vCAAvN5}o zu}tW84#=x3f2wo=b}ySR$O}Max_3froQUfdg7VqYFVKkv*RP{+5J=Ney_rL3K~Lp> zkhF=?mx6FoqZ~vd{3lJ?j&cQK%rDv`R2~2PcN?#75%_-mrix5V z{&5{?Ny@#tu14|{NWCCPSLhU;5k8>81G~kv23r4D z8fiX-S3vU)6xS8@fX@ZAmJvfk)g3$(R26*-yLkwuTGSSCR0rv1Jb&Jyrib`IMM`E z|Erm?y_-~8CeQ9Fy}vCBxbCTVSj(6&1J1-hV#I6GZrnDZ@{q!21A%-X<(DyRe`amU zDaQl(JH^TD$Nb?+&nsVpEy`)_Fxwxb#S4M1o0Gqg``v4iHaRcy7I=G-&#%Z6r|h%$Pf5NKA2`OcO}*df$P**Y2dKU&n6;NO$ZDK0U*x~Gf=XpAD0BP7&cu(+ z{jxO|C^v=ow=IBpk@a8H3>}^|=foXQZ@V7%I^oOd+H@^j`Kmdv0Y&cxq zSteE;i#AiYLZ5mZBR$;Uvq=WzIaTB7alq;|Z_MYHgu}*nC!uleHhh426*=Z*fA%q; z6I^(8N-NiEOy2>&q5Vxt*-e9k80=X^eI8Er>c&eN=OQ=y2jO#l!I^wuLF`Ovk!Z&Q z2XX1Y=@hOiru%wc*W&L+O~K!u%V1)SW7vHt6 zM%&2L&28wu_|s@K?Fd*^&V-ive&UOFwcPR9&RBoWI=u7YD61lC$>VKL;*FCv<=Dep z={dI$zU=u9gg`ys$MBf8->s*-G_x7Jn*V~)aq;TU8@ku?E6EKF?eJogo!qWPBp+wp zOrFVGg?Ub$@f456(6fPJedTbNmG&50hB~vMKNmsu7N7A~v7xkmH-W{|`qk^7RhD}@ zo`;3Y4e-hke~IaPa7ScK+UNe0b~1M~qu27C?aMJ}UX&c4y_4>(Y9t*!^4o7N>MWNA zo1t?hCtlssTypw8jQKU-FbjRp;?oYwR7Cc5^d3Yq%NJ82TSx3L@f) zSouB~^@E#h2lGSWw`Ux>te>Jj6Zdtr6F;*ausps5-W;=)2j6@|dJWw-az|ae%>Vey zbFR-`aqLN5KJsKZH1Y3}Q`NZ)XbjO}rQ#KQyM-MbX;sI6h_> z*!0>=dOTelR?lAuM8)39#b6<9E;33p#Y>ob=3=DoPNRP(*_%Oi0OT41I3HCbjL z=VQ#0ws0pxZ_&L1!$b~0f-i5Q$#^Km&J20{8BN|sn zKGo}NZc6_W&m)T`@-F=hR3Aaf%SQ6nrhjl?@=&<`-z?l0G=y+LM>rFKshgit4fKUJ zvj?H>_ZA$Qz#KCmYLiVeY!0Gg+&dAzlP&&JMn>i zYx1Biyjp9mQaY*%SOR$N}DdcVQbE#-ML< zHSoQ#1ozgS!_MRx@I!B^5w?W^f6^TaV#Dz1d89F4iw<8AT2+}Ta}Hc)g|6w^7wtuVGiA$JSmryL>j(InzOwn6<*MVYS&DpCU9g zsY&bJ?LWSg z<+c${a<3)9>5Aw!At;d}bC|!{R*%X@ezt~(hD{IPkWbB4pGiYDOlFK5lQbVj& zCxM>7Z4dqSkI)p(pZqlylgm7L>53Bc`FLDs=5g4WxWT|SjeQHy+~LuRy(XEoEj;X z?wcp?#S~LNRF~W5bmR1YHGcT4QC;pYK{w=jWc4?Y|qu0JB?93K)j(C258>;m(RjNxg+Byou>;d-S3G<k(QF`UU4;6D%b)??|Sp0w>K$Hz$1&FA@Mg7=JBP$ z>tT&_#|!4agv*pbS&(xE`xrh)I5ivQ(0xeid})kz33MNGu;)gmxcpmzFFM_qXgQA7 zec|RP`gH{3uxxR+^;~#R;}i693WmND^D%zPOmW?}68Byi$EenY(|wK&?NXS*K4ZS| ziiyxH(($h};vLs{LjPCB+C=saiT}7)#9XW~qOQ)ev#C_r^x6G2)o)!`60jQoCN_g@ zRa+3BWx(RP!&tlO-3U)pkT4bXx(w3!&3J>vk=$t7Sh)Ig5E2%nL(wxjzey+~)Tf z97$_dH{E?raUlGdcMVC8sq=%^(HoTx5+?NBQQwI*)@K^jdwY z4?7^7z`cJ$u*KwG3Xk}@y`%V$*p`y83+MO@VZEH%YKo`AQd)`G<(uHtnA)nwMd-6B z6}-F2@&{=lBS~|LEqZwIr{_0dliI71bRcYc7ml;j^!S`s+h{HSc3NDe{o1Qm?r56T zU50JAr^W~tZqw-b`4Mu>vII2yR)~*XyfNYOAcZA_IW-mE;o;9)@z|4H;?|~>w6|VU zx$(*?y4C@tE9VMFdm9+nIt!^**oYk_oYQ?=ZIAlXJ*D-aNzhVoU(%WX%o>BMza18& zHTcJ1I~cL!6`bvMP0Wi9g8b`|ETnTZjQB4|F0SjwJH7sgKjzf|!YD{Mw45Ex*U6RD z4&&tQHz0mWnArN%S(1*xaUa~|u$uj-rnf`OgUP7yqNmd{q4dShFJoE9b-AL|n_yA7 z&7{e_H_i+D+;ejBa4}hs&nT`ec`__YvR4iVBJul zUfp=sagu&6af2ki#t1`k#DDq~7@_*U_`+Z`b2tR*cP^K|>2A&F!A|BjCVo50iZ2x6 z(Yuc5WAhD;+pbf!uI3uj<2DfPG?BzXKz++OauVt2CX)0FxLjVu8VTBC86H7I<}}6c zNIDE}Z`uarC3JLLN%O{9R|yjfmuKMdp^HIbbY`H0A+%Nj`30cOo=N*1)<~PPMcYg< z)6ZB^O>i|=$z>)ypmi{;2>Xmlr=Kw5WI<4fkrAZboTs!H%loxXl1Af$ zjrhC1_=IZKgtHENgwj|FPmUSg!9MjQ`73Wpnp`yglLB>~eieTc)1Yg*y>8NqdFlUJiX!$@bqt{fcLH-WrOI*@)L zu86~B z=Tf*4_8RjUNym_euFPEzZGw=_-vnU;Hn6)7O**be@_YDvdUHuw1~i9Yk$4_zc*HC3 z0nN(B@Ff9;veP9y-hJqLR+Ri7v~FgDj$bzrSNF>$@5Cpqh*y2XlEtzlTdNLOc1wjiCP^tM#5`&FDt9!SGa&+pM5m8+F^L-J|R&9-TU9#B4W zRn{){@1M2wU2Dpn&{&dIq_NQ>zu_dP7Zq*@g>9>f8}g{7ZScj_1)v{XillSFy7Fo? z*;3BBwS5YU^>(QFRP}@?vRUE>UR*p)TUZFkO%&tqE z`GhdX0_*Il#4cPkL-Lt~(T`}3dLVg3zS^^c^7I%!_dc_|(v(lQl??B6EwyVk51?D* zNi-`?#A>T!p;EzFF?aqUwnmO5&OgFP6X|G7m0khr9Ub{RAdM`@?+fMC+Euq@!+q}& z*SE(8h1=k(xu`ERzkZjwfPf|tUxuy@{YMjTI?<((i6 zz{r2m{=*is()$Fgm)W&erX{t0Qa@sk57Ms#`qs*c1p$)HzDZ4dAWgUFvyz z^01aj+@`!3Cm$&e9why(-(E-k#7OUBjin<l*w-4c2b<&hqnCn|7(rNh>zSV9ZY2#Qu`Sd)98cQ3J$N62+%Q2n&ko1@&&7^at z`@(2k2~+c#(!Xa6Poc`18vnU3y0?YlYH(ejI$Po{i^0VZ%fHbF$aInG?lBVXBsui+b!@I*2msv`y;?^^9 zN;?5*Lml}*cvLDda8MX==)4Ngv-V#$Cm#ML6#Hg;mXy|?u%a1!rx>g`PK{|q- z8S1P9@>4WU$W;B6A5T<1k1&k(n>AL%J)G&)E3Zn{XbZ-WUh>nr>q{nJM7cZ;p4<@>kB z17%A1q01}EZ+mK`D~9PPdxFjO0|>W6U`WnOL3t{Cofb>_GC)!Vd-FO=$_t3A>UoswjmB1QjPQH4>5@DG>BT*S_lt8Wmz1Q@boNon#NS=%wL_q} z+Xhd159zw274=GW@7XBco?ACGBR(#u$Z8YTpV0`5=^3blq#eIgeq}9B42=QuS*TeA z3i92gc?UD%FGl%-%El=398bOO1oNWxwK-NXOyxvt5?xqagHnz10w8P@l)Z4unmJ`Q zgxk*i;G4NH{rn@(ra=yr!vx8JGj}W94IS5+Ao*}quj|D(hy}X(aNPeFByoqp&>_qRGK+hiSRX#x@El7LM zA7s>{Qq>9N721hK+l0#6Dci=w?S|*x8{CunBmybF66A9!mkvPEn^db7+#)dsssEAc zf>92J@p2@8+< zi#D*%E(?%;$LrppHKc|-29J%4VVOlNZrRfpuce)U@I?oh^Bybi@Gf60DjO!;Iu4Y) z%u^QCq-U8AwUL*SE;2sx4X*nz0Y7^z;^Vb35SwTQFK0(V$JN82Ucf&#FUd(emKzLK z3w3;j%;)yk{aL$_n=$83A!PYIMZe{HgyYCnbd0C0tTL0WS`C7lKX$NPpKrjrR{wBa ztCe`*Ry?xv1)5jKQfxtMU4DtQ!`urTLbqpWmL*R7w5$S3ImKut1EwA8^{U<2w3)LiL^Zu zA?Eb?!?su&X$y?%%4y9uKxdaP@Q|Lf(KopSWBhVptA(4~VVelMT2|&6GicAc;V#f? z#9PtLv?i>Z)tcran15c-SU#K35pK%GQhGIoF*~nA@MkZ$ZL5=0taf6+H`-fbl9WsI zy>Lg#bJ(u^#(}?S53&6ITp;f#yJtEb9q#E?vUz-&RXz@3=8q$9H_}SRyw}11I=b?JoU%5ql*U$e`66Q}QDcNu?|F`OLegX; zYh%e)gvmM#LPUuPi7XLAvSw`4d!A#>mMw$HGBH^ulVvbj29xi1`Y&&}@44qZ&u_c; zo;x${H62Q{)$dAU^%m@L5tFc=_}U7d<+@kR^HZXE(=pO(&|Mxg=%ZNGWIcV^cRk5Z@fRNZQ*t$ds*@X!UVFO8=<>?|Qw6>(`paa!j;*bg+u- znVLb784LMA|4O>S`t|a%Q&YKlp|5OrDUl9UJ48O7PP%zfZ`tF+V=Lx^eLR2UehE$G z!UL@|zC%X1bkU`6zvaqhJJj6UhiRw7Mymdy7iAzvK<%l)56_9V=}Yu7hvm9zJrtCjz zSYloBDb?mzbt5g0dwhtL8_w;e3SQsSqt{!d*Xb0I75&ikEjx^SL7A7j(~O4&${e%* z&dIV~$X*eB?-=Iny;?d8_g#11qHU*E(0%vhbE}z|6xqy|j+Pej^d?E-`wqT8o+Wzs-Dx%B zIzKv_vP$-g7qchwj0iVgyZLWkX@_gbTRY-<^aw7TvyytR>PF9pq|3F#egiJwCSVDn z$92Cv!95GCIVEI7hUw<|M0ciHmR$g z=O^$U-*08y!X5NOa2rBBNUPs1@^U^+wu!Ux{J*OXbiS^9p5LZhsnHzW_60q;T%RxQ z$9;p=p)xPDflLgtmjiD6$kk`A6%pZs=}61Z$$8^?SgNACZvx{$qT~k;$ z?f6$a>f)Nr8~u7=&;EmqIbkk%!|!W8;i)%VbIU|2D9|6!W0sA6(#sW z4==l6=0p@sYbbxHR$rH0`bUi}2-JN&UFqSX&vfI!SE9|y4m$JF86LjNjStW6!*$zs zW2}Xa!8;>xE?m3+Oa6GrP4)@Fo)`f)DSlOvx|W0eUT=4h9+j(6(5Mr_f9C`ASBgr8 z2beR?MWPS1TX01^r)-oAntqe<|J1na;|-oEoGY7J#aKkFOCJK)(C~59EdSUi%fkGV z3TsN|dIs>hycvwo*kGr@f_fpXDIoj;&Cx-W;S;Sd+ToqAcTyF2N24EsQ^WPS*D0o# z=DCVLF!3QvYoOiyR!zRN*|q}SIA!rV3RN}a;c$ZdsSDJJ9X84 zl1KBk_q%mWf;U&adylY2I^#eNJEm6D&d-C$%)_8|bwtXF!(8_u9?1SIlAF8Q$ej+2 z%4@*PV?&RoV#%|4*4#^BvP;n&^SSKnna=awn`>|@`15P^V9h2*&kEob`$zTSwbi!s zx^X-847ZnR!X*zj9Q7vkgD`c0Ib`q#?t4y9J;RnLe~&-)c-LD3Z`JnbLNFtBB z&(-lcai$kg6IPnhXE__pdK%lC9`)=e$KxK6(zk^Q7%fbX<8xDk4BD3^pC>dabl9I{P}nfN_Rg%Q=%dXyIjhJlUJzjevLH7 z!NtWdh0#Pe>aPW_eMa>h>oWS8IV{%-%w^1r0OsKxD31v^$M+`vVg5}ieHvK}E*O2( zdQ@8l9VKrJ+)pJdJA;G8mZ*5Y;o z&v1+3K<9o2s{}YeN8WEF+s{2lC4cPYtA)7NA$uI{dTlS!M|$nB{<0|P0{i{4nr6@b zSwNf6{m4c<0?&x1M6}oadQX&}BE8g}?eJ2AZK+QZo|)S-k>k4kNTq&n`GMzOV%y77 z0USc@T(q27lZ?8})4&dC=ApPahN@e=DWLLtQ9ro{r->cxHhYm=fBQ5IEh@2giYwrz zaXod$KkX-k76>d}p~UV8qcM9Nw-9~$NY&nm2=pSzAD;1CZl3*+T^x43rQn`X@Z zzh`t7A@|yt8dvZZ&@pxl9=B|**^$pWc*$vY8RfdixA!nYzwr?rML4F_^=u~^JK0vB zOBhOr-#60s_R$8{sF%+%#=MZ}*97d3w!vi>;~XrXEnjAIGI?ygLDgv#gQsail9TXE zt)=m^0$-YXFV}7DlYc}0v7Fk=?bQD2-{8`0f)17FnO}@%+upF29P4u#ShkI)#fPG9 zgY;4FdU{W2BE{GIlItg*#TbLguC^;bJ^qNHV+j3+_v-$nPJHr_|KPgayqVYORKpCj zzKX-X(xzXa3HkHPH=J+t9fvkpOe3Clw0vq&yBsf?=8oWqjsK|JB&L*ye{xyUU_Re` z4NX)v74)O2d8+W_F8LR)A=BeHCU;%#DGO_c)8#(GaIP7LWyI5?jIl^?sWe{eWoA{n z-f|xWHQOPKrpPOf5Pvpz*Q@@pA%mUpL^6K@o|z86%~)G)6TgP4kDV`KE9cNb*WMI^ z{j_Fw{aqP<^Yihu1Z<=yPWvglNh=*+a~~b(It$Mjd02)9XYhA}_Yw3xKl5q`&yhml zYYLe%i&r&qkxg@p82Zl&tk>E$B?sS+!oG;c8yRg!Q`7N&v;V~T(QOF&OaVJ` z;FSp&3Gee9HFcYcQ%=z6)HVb?t#Dj2M^!4{T#cOpuNd>)=goD_{#C$7{B*@6BfY+?=cp0Oa>;G zuT#Re8`PxeGvv5q9UHF+UC%>)*vMxBF4#{f$P$8!qSmQMVNIFSR~$-*--@x#aKREi$|pW4}Qg2|upjz4Zm`r7+-z z69NC|H~abg)2Hn@SZhXZAmJzZqTOXG%`}{8fY!S&DG zo!!fPbKtdP+j~tk^dP)!EQ9}bqqUNJt9IAGEE=+6t%3(r=t+7Z!Z_&2OM~0Az2Qn; zUUiBvnS=4v(5(K!*TcENcXMUMF<+yeO4#mKyK+u}-eYJ^LQPX_coclVQ+T%N!su%7 zx`O@@m?r@rp}|$$wCg@zS2l$&zB=J8*FT922ApqRpWeP$$M9Zq8J-_CUJU*~8+~B3GrS!EpN+1S6$Y$E zwvx!k*Z!}nFGskXF!>129k>wK{GMtyOXSx-eW8|QUqr^TOd;G;>BZ`KVIO|=+cJaq z;FzV1j7dW`Q0F$8!2I{PcJ;d|i5S7~l*)TjJRvJ!>$GKT{Df`dM{8&U2MyiL>5p)~ z;q~#HRpL#+CqD8xp1+ZYxU~IB<2l*r+`(@5O>SXoT%-vPHZ_hMzyh35o!hivtPg#i zy_Z{jVHNP58a{y0XM`RVosKURKMXs|$lxSAAV2*t4LWHRfA#cB>K`@8a0YL=F9?2} zr=D6Ydbn5OHuD?FqopIb<sPcXceKyUKiQ*I0|U`3uO$2gA^s3VD-grKz<+F9>BQ_-6C z-q8X*C4fKB7gs2CZ$^36fgU%$g)pB+3&1n}$428AjA&~AK|lwhhN0_Mz2LahC)hqN z$LItLFhyPHVk=!NIbv5r2s?XLP{Twb71wf-z!<~Rz}J>Ey%SEGDRSpG(5qmU0Raeo8cF%2EV)tJdxRXNN7O? hJ~G*e!GB>g#R2uQjJBtIn-?ZqR}G^Cy_Vnht)}~C=lf9$tZKS@dU|(yx_7E)ht-0{H}l@C z%;~JmA|KsT>(Rzw2@HycU3OZmd3l#Guozpm_zNN|OAZmX{*kf6#sQWe;TVU*5SBw5 zmK~0eBW!=Z=lA>Fd$Y2-x>kUg-e`iI?ykCz-~Ij0`+UziZ+`XP{2#yaPkn{{@4@?~ z$?RY_PA0Qghx5b3^=z^_uBP+hay4IMRepMy9nKfW4s9~Dhi^T+@WT)P z>ca}^ui9H)ebwIcm6P=0_-h_~?P0b`(qxs`55H=? zm_B^)jpNnoXt{g&@@#%DTOU@F2`gMpR#Tq4yLThHHCSegd&yuj6i*6|zZBh^ESH>| zy{vMJy`D}E#;g08{u}LI{qQmxCP%CFB6Ev!dudy_JW6I!GF;4gKsY6PQ%dHHqDg+TGEV9jHdvvz5POl|}a^%ZVdGJCXT% z6&+;@zV0xY4YLS>Sd6nItycC|$)Ze^qE_vhJIH43Vzigu(>-Z)Gg~bt!=;Wq(PdG> z@f~thI+x>UIfSSENw^J+CXgtUBvOgiOVPacU*ECF#@TE+fi5%mLS2&6OEtQ~ZFkwL z;hMj|NXv|c^CoS~WST6DyOzl)6PFDp$>i8&oIR3izgKv8E)uZFo*7IuiI=q71c=5rX zFa2~h$)@S@)n&HgSD$*gGyaD0H;>=-=?CvFU30XU++(A4?|Oi28}w`SPCt#4q?yL8 zZle(={Va`pwfeBzAM(>t?cum$LUCvO^~l5VyT;!>{s#T!o5ugj_}$|_H+~ONWBlg} zQ8E7J@BeE^hku^$HOht)kVb4}wt#?>>G3Bx;4&RaIvQzjk^!tYzeyB48&|NNbTD%@?a>1Q8t0(`*`{GiC=Fr;6}8JX!@Q6y$;g(`pXa!X}b# zCf?y4$v>T&=zY9~WE=_JgPS?%YUR!X!Ne)Y{>XLTH4!O?vCN{EV2QG1oGhckdNN(v z2XLvsE?z?MCQ-|>@RDPh<@!k9Vz)R>MXW_fq~1G88i=h zN24r}6hR7IK}5kt(LQfo8dMqn+YPP&=t`wj@IpPNv!`Nw1eA_O?ID`6{MZvlD$|?4rZ)LbfWWL%uMnX!mYBO zWljh?wID00w(sCrs|BUD=%f~8tKYG+?Wj~DHH<%9Y?+PGY`k# zg5((g#TUolT8M-3w;AWx8_B3XXtd*Dtp(?II^DR}>W|`DZ8WN7y-v3_*yjBAJ(lzT z0i6G&Z(AiOwL)L8x!z2XG;!vJliLu`#BV!dgQGE~zXXFAnWx!ceSk1KaH_%>J4P{> zTAi3$lt_W!Zp^qL!SoqL%cE?T>P47LQX}`WEO5^6`!j}ZFzvgsU+v-A{eCZ!kHwQB8|%37XB7VW>VSG#$! z;yD458A4wf9@?WN>af+Rck2Ci+-vsh>`BU=w7V_-+3EKC$*7$*xA&yw_vFF5w|u-E%ubP-NXKOU2GIy5xeq^3>__vHsvr?R_~pUBh;x zosJr5+{tQvq{pBiC#^v*PP_fiu#G1+YH!;OAAX$lc(R?YPCIR;DaO+<9n|7hquY!7 zNoNqJz2>md9n@Nlbld5ADL7q!;WVe~@?bhYkmvO+MYif$ezcJ~!vK5FSNo`U+WMz| zSDw|2#J)CfAvQ-uM{YMU4uUw*EW`PDUqtp}G9+R_Xo9p$anFt-NGPp?$#Czs#q%yy zD=%aTp-GG1C~e^a+c>)}rp>cRHQgV}(anzauSlLM0+C#af{pmD#0M=KqA)~=`xGxd zOzyIh#dBT3x zkpD89lIXKA{r;HL$YFGy2tZQlU9LCXieFK-PODUi!D2P@x_k4zWpr?~$```>>z8V^ zT4diyN@pA`6D7}5poyQLCr0^$=UCmo(yjoxn?@IchX}L*mj7YE4<$ck;4@|U&*GBOdF3&Qwm^h4{(@UC(cW75w+ zOf2_c$(bbNVK-&$k);f0E^RT8{ovJ?^0Qz3+|Rxly`nGX<@pJJu$&>`#uM`VOfn05 zU!+D!L=7h>P)lHH-+i9+gy646FVCivyYS2+k>;P1wcm=xU-FBL)+8(zN>LG#7j=Tq z`k}S$?o==&)Fr0CmqPB!Rb%#uq2@E}kYy!icRu|Sfl8p7Xs0qm>#YxCBZ$g{m~0o& zkpxr}nF}KOIs54|^>%%<2*f}+j@UP7=PUu$YGzrhH|$`Bwn?rgwPxJwW4NcIX0J2q z4}0y-wpDsJ=%xd7(-_^fB~X4Bf$}Husg>C93x-lrT5+`~gov{-PfNCOmoNliRm8}^ z$DGvk$ux3(vmpqJJo&FPd+Xmip&>Z8zvK??a(-}mlChjCbXM_;ysY{fr*`di?4sq; z{w}@q{CqZNJ%##-M-yn!Q@YbxsH$lsxD95TAU<6G@(i%hVrc>Z6L1yKO*=0l58R8= z0}vd}*aQ=_02IXy7mNbm`Wu1db4)}mw1-0kmC$4&IfrzStz{qTQwkYg3dt9?qxJWFUqRPDGb;X0HhY z2c88UkbTbqed@18ACVM`_9fFQ^=fowJf8z%`It05j8BSZe!53YjLsVFKz#v4(( z#?G~Lwe-yraK*N<3HNhtp|px!+GVL<{M^HzUt%xh5Ac6~5DTCX@&Zus=iMb{RKaAHb` zYqttRSI|DlP}KahCk%#t7za2A=~xg3md&?q%+gAHrZlP5NBt35g0z=l={83l{QgEK z&f3k|uvcsM>W%i6xwCkzH2JTQCi`e%1is7zZ9(sWvQ)GJ7>7gtA-UZ0Jqt#-z^TFi5Gf_^MTc_5VSjI& zP-c5+8eqY89=24hu*GB`;bohj0@s|(;$RzX`o(&7S)SZ-KxQXaUq)C3(lijGNK1gN z$>=y5%vS)qXW0-ub4gsEMEsSJkkv|1EYWiww|BNA8Fjlg1V%S*4!~}-8pxDEqg#vX z&1Suqf%s|+hg*{5__31YrzA=Kgg?7-H3w(sBI3p5o?ugKdhP@XacyWL+vCC%jqVaW z^T1s@L#O=|th+Y5Hvux`P?NX3l`rFdgVWuiIc14$KN=C@fh7vWC5}n)9Fq&uyH_^! zE^!+S*`S+kiG^YYKs`GVa@nRmg$-ZI?B!qruR0jJen@BxgUARt;NAE58zt0pP-4aL4GCvp_M&R#6W#+L1O7puaO07_YqfeGY! z?%WkZMp^**Ohlt3ZGh^Rq@Pe=Cf_Qj%RiHN4wsIcHy|;1)aXS>x!4%7jE4r-lRoF)BX7i>b> z9B$_!-+Zhb_?Nz#qL6p)j#O?0S*tL1$@jPvWv_$Mv&sPRN^t$mXcn$oB}ACjj(d`! zDn3*t;zGz}yIOAf!_UeffD4z12+pwLgB7xlWpBOpD_9Y{cfBv+Yk<5TYtQ@XNQ>TxP3;ClMJwSq%gUy?5*G5}?VH6qHa)E` zX(6$kZhhOSYuS9aTMu^UFGe3m=st_i8%_NE55GXAiwn8zXRGzmu5&bUt!enc*U;TW zHbxkM!e3gvKmi(C0pe=!M*JdT_Wlxb;P*|6S z(BF-o#SoZgu?yBm1YR5>-_|7No>G$|nQu{lNW`A>@H5fA!1XNV+ANK!0sz~KKpHNN zF%6LDb~_e}Mb#{HWu^h))%5{m_XBT|8Deg3T5~= zGHZp)n592>mhfqhx}!!LtE1IuBYcv6El%nn&&k_0>gj0I?sf-TM##VUSmE;nU+LVL zCvB^2OK~cnT^|^`Q<1V8*#E|IiL2{2{imZ3&*t~3TOr`^EXsJ7=q*9UD^N+cutXy% zz_}pcA>jByu2A}MZYOxs)43j;n*p^UxZK`LF}vo3e<7FLbY#>HZv1x`Nd8)~AfCDB zObgzVharn9kY}Cs<-#q5=fWCibFAJ8aW~6pkpm+XG&>nD(Zzx<6G$eiQjPYA8`xj% zGrdW}HBRCB0?25&R$+>QDx*b)dL=%>DvSH{Q_FKkufh3dWrZwSxsFDLgIo47PpFwj^)Sp!HO|_LP8STRN-`-X~|BjMz9T8Tah6V zT4=-tvgHj?QDxa<;?3xPq@7~8lNp-L+jEfD437$%IAmfEc@lv&!gRQ(D?H0kJQ`g? z8qN=oaR8wgXd4n@k@j|IfgB?;mdEP{q|4{iwZvrNN@DgbT$?{ajue?tVtSj4m+8qn zK!)o(a$AZLQALFHbX00pDe7$H_C)vFt_(8NKPFiIdo;s2^4(B*9G*@F-C|iO8CYz>b_{qVPN$&z37k z?mQmVO>!JuD<|9RQ8r(v7e8ka)hq7&mA>JuQ5~pQp9f;69V*{)1y|OKC0l*t>NR^D zz?~IoeSNEHEU$0`DjZqKQBrqoioh3L-wbyEGrZTBcH zPNpjeTsu?tyATREKb#SIHz#XXw}I|bFv;zM7Xvc$VC->3-EFq0R)i|zofu?%^U~3U zl;Z(mUtMeTp}n#6kG;T7ai$n}5u`BkD)1XzpO+=L4LVcS43b)R)a^Io;joSl?GM`V zpj+?6{bpy-9CnkeKH9e3KO5BNTjUlVo+cCi0GaUQZ(Q*D5c;md*LzpKV$?$Sh(^ukmaSQ0bVO z)cbbnQ1VQanHVb&`yeO~1WXXlkf>JYB>>^oW2xyOk4?#+j0LFfB#ORv1!hCKzWzl? zRBB-6S5(!;W>R0McTZ*9P~eljZYUckl=Q($7>h^QaaYkRp=oakQqjpTX9y$v;OzU< z4ll=(BR$v_3c$(zMBJ*I_4$|JR9EO3BDL!k!AqXtN<+1As*ocQNu)qYMeG-Sc9MsL zB%SY+l%efnA=}Lg+K7i<0;z+OLv1g~_yLmH(H~mPGC+58qtN>U+A;|uLL;V>Sb{0_ zTYQ7M2te*1Jj<3zs8ec>TJ4zf(01HP8@0GkX;a*2k1{OtX3`ucTju${{aE?(HQ;Lg z#J`)(76(n|bBnQv`i>IY5>X9mot%6MQ_@jlOv$9Dx6=#=beZ{%j%!Y50|&`U3Tkvz zeztUP_M7#_2vk#+4%vNR;=R_00+Ea#_G($03`SeK|I3fDB+=Axm(VTmx#IqgN<9Hz z`S2Hlk@)-X`Ynva2VW58v4UDAC2xOocUD|vqH@KVr^o|>3op!ZA@YjxwIr1x5zUl= z00SJNieErL&+K{QZBF1&rj08^mG1(puI6iWuKC4lc=JX2c~s2dtlT7o$+lCNjXZ(v z^pCS>xz{HTUvFeFq>HQB>9enm#xU;II?b$K?*N}|ZtdoOaK>(8H;yl}8*utezgNDPz`gB*Id9IgLRMRNO4P@esW_d@Aif8IB74%xtvk2< zHsn8g4$#S*^t!*~gOz=ESLK!(UJyQ*VXSE*E3U3I|AQZ`2RcgqVW;QzTTdbq&Z*Ac zPcwrA5o+B-FyxTE$Jy$G6|<4cKR)XKM%kb~8gw~;G=ZE)15%fr8U+u{Y|u>yO@WxV z4&XP!0la71F#>k;M+MW$7T>4+d?3s)*>k5CVUZIZ^IJ!m9@{%E)j)A~<>sq=Sm zrT*hGOe+YKr!Xzy>k2@EV&lH0!x;bs+2l`QT2EnGA>WC2?%H%%`1=WfX*D`v(9<^7 z<*+fJJgZMBQa^1YO@{rf(W|wRQ(#*E;IY!=DNGAU&B^4&Q<#>-rG+J*!nC~Q_2mQ8 zsx_O*pf?=FNm|QDCyfvygHDg0GJ~YwY1aEmZ@bX*w;wA>p2D=`?1*um!n73T&C48u z$~}o`b1{LZFs(l+m{!*74TuAd2w#DZXpOp(1N9^x)CaXbC63*0x?Sh?-#k_hJcVgJ zg=w7z8G8!TdJ5B`U$Acw+G>wF^-Qm&|e(uH;SyT{a&f9UH%7U_Tgul}Ku71rwMM9q>t z3CNF&vX#rdLja$(%w3jA(bxjN?>pyvfHK+8O%(uYm}GK}4s&%*J}z_aRDqJLr-BMC z!AG^;DFxQAj5E^McHai#L3IV?<(~*Q^Cs8)+ETNYu{S_IE&a)DZ~$2U8{O9Pp80vl zl+e3fUV?Tnue3f{_UM2HAJc=uKU#Y1eGz4SrdYvq5Vf*o-1(n#2`sgfcMUmw|^D zVdX&ZhQ;!gzbrD3HCKuv;Ef;qO1JXQq8-Nq7kCF%yLpjYURCAG#rBFyFzSNTYj9cF z7y3jP8@|C|xDB;!bpR^Q2SItojb;EeSf;+S@}})9ZTGTH2as2d4i_T;r3s*~xL41* zz%u%2GusAX{Ntc}f1k|6ACQ^Yk{*AC0*x=TU4-`S%*tgjzMrq!FmDy{f()|?A}XCd zvO3hiib4eo*#${>7oeZ_`ow*zPeOLMLY&RO^eF|&y9Ftz6Gad*hKQw%czKylO8V25 z{;A*iC%^eAwT!o+FS}G!X%fK<%mc|Z0JHoBM&a+mFI!Sy#l=<=@HcX{c@6*t%QhAR zWM(K&p_%eEYd*|GwJ5qVuV8i!zk!=mh75qg$-A&Y!>N!lYX0Eq`K=#2S=oI{Q$2z( zQp+L@0P#Rkr!}IYDsMHO~4)f%mCd(>>j z{aO>`MjM1_k~If}B!}6sRcAEIDUkMm`dA5c6PT&K_=(;(JF`!H9^z&Y33Xno;T=+; z<$EER%C*hb#3@uQs7H<8;I_9v^|!gfSN0hDG6e|!!*1|ZnRZ`Z{6lb`u}A9YAO#1X zY9s_L8{l-Wa$HGyETA`C;%MPOm$tyj=PW7bfv0O#6!TSvP^eXn@j0lZP)?p#9nr^u zIxQ`#T=^Fk7(cESocjt9;Y?zlL@hePPF98&BLQz>(q=Kjlt8-=r!;TO%*qKy?#E-?1_{(0r2=$uyZ;w}Y7yg? zijFdPH04+)yC|J-!l-$8w7(MvC^1mx6{_we#O6^Bk2=a|NK9NFiH!tFI!t`A$T8@3VHk@qaOQ{1`x~*X1=~i3 zmu7XHu=!kJuUUQhFsQutL^Hk+TWIY!k>GAR+ADy=&DF$!vjS-AA5yM2=iJZ)fGA}z zA-=7Iq+AAWkNm>tO({s$JDcu#kg=u`4&gB!lr^IPVLqUTJSu?4=8Z7)3P`yqG;zxe z_HA%p&_tAwjwh;XISTEUe1(FkE_M}?wFxw~C_K53xJrd$yPnOc5=iqh{kXKxQcTXm z4FwLiEbLNhGO!rWvN}40L7&!*?YP|oKGAAoe+&}yd%B(4sFl`7jjfEwYhMMlJQyDT z-}v3HeTDx0o|{`z<)3)lSKQ7EvSWfTl-pSVN$hm8@Y?8YkgTjw zUvUmR#O=;vo8|6m?o?Po!KqPf!752l?JV{vxSB2OkG7py^p~^6JOX*mzKvp#t-fQnPNZQ_Q1aiz=$~*7q)sKY`|(Z2-bY1j-h0fs!_L^WEjN3U zyX1E914j3S_E_Z}2?#6{DRoZBHH+Px+Jem1rXm+}eq6IMdkN#{D3O*s^%JiBq5oEy zoY=|I^$?Y7WCm3H(erxLOMBGp4#;RAaH!eEIzoTM&0cRrAb8Lnj9MqL>91AJDQkYm zWX+SleVnC>M_5OU5CD-v&i&?S$FVRIsES_euG)Y|!fIF*W0e9bc81cd<$f<*0(WcV zVwn6WC2~^Z<_+I`-CXF`v(YU|I**zZYfxCI#&XvrTS2akxcpjBrX|&h46@Kcq2stP zL@5c3pn$0#+5o{ z5gu~DbFROmxs(>MvggtwS_R)s?auB8L5`HF?F!%f021B z+K_Vb3v);(lv1mVmJFktifT7Y1zBTSYm!3E|7_@@{YakBM({?_6|>NnrJ-`_3$T$u z^Ofejy&Qku(02CnB1Rb+zJ<7|*Om^EW~;Uv@}Q$x;*hsfro1Sa;s_y%>lJ|wPJz#D zp)gHH!qqP5cugKhdDbkcseIn6746$AQOMivX?UUeIBryB*iadr#I)3{%habCjo2`qsd~X>8BxO9c^_XFVf2YU^=9v%)y>!8E>!;NhLE z=4pDjGZ*qans3;o(bw^4Ju}%1i6}IyrG{~-r52~|&1Ta0w!nE~Pi`omu@Y>q`;Y_| zYo(EeNgqy5Jqj$`S9EL%<+|al>=hg*Oy=`CVo6jk3oc;0y>=5MeOFwN9o>IgiBml$$vI|3BxERiJko$A=w_tby=7Y}--bLwk#|RcP42P#?1ghC*aLSq zm`;}brH@PSJ?y#OvN~x7x-A%ECaej9JMP)|2HJ5D^>N~lj#p&^>&l!;T7T0y+Vtt; zKk9uVQt@V!$#=<{m2Q||MrcJ}KU-67h|wu{evEPX&e0@!Xy@rkEQl$GUkqFN@5mu5@O5-6kLU;so0uvffw`#A;5|6=xwJMee-9?|E;7{!M-PiTCMM)(^yVXGYh7{X?3i&WE2ZBi`=)DM?iuSrK?B7xB$;Kso9>s3ndgCBnisp>TKj{a zkv?D9is@Gu^4%7uDL2W?-z$#DV}3HE)G~-A6egbeqk5vXh46WPt$`0j3gvZZS1hpM zK_FTzDQhVMCg0F^1Q7|+h=wNNgL%iZYvOqXLo8@vG+_oEoT&8@Zotdrva)DK%u2@O z@e%tlTOqL!?_h309`Ym_5A&!8`Mg8+jA0XQ2fgh9^dR?~amETXge`AGBR|RdlEvvm z0Gv7a9*t@`352j4vEF3=oEdWI)S1@RpwVcn&Y-SVALFH8t1$?*O*7JF)=nATvK4Z9 z-#I0WOqO@>zHl%eMFgTlIS1Rdfo4}>IYuA8eZwU8uaf;Iz$F5CU$8=?$~K)RPft0@ zs~p7Rm@{~9nu@UY=yC?d;}&_V)Tn7aTPx64)?U}t%+j%G$T>U0Td2bp*U~!1^&*rL zw=to8pSD7DrMZoMTwGnR{G=@#JO0Jb{oLn;PAK1VJ47w8E?vJPp1zmP%|EMHNBn5X zqLBcY40LAIA2zd!7f9spvmbyQV2~r)Q@V7wXDAv89cJkSw0$DY$9h>}5=zujM6?va4x3WF5#XrC7bOWKpt`^P8t%ct&+@ARJS^gD5l*}U^A z9kJbsUHcF)ZzeIZexsLgps_JSBn30bVVp`pIQ`C*4f+ulpbCK)>FY%<0UM{wo&x}M z*S7!BJDpBto|+E%69Ywhr zIg4a?mgP2z#ufzfc6gV|bEy%)rd-cBB`kT%>scBpM>8DJE)tE-L_amGqtlusyANoR z{?%?4|E`vx+eI4O2i!Jy4rPQ4%n@=fkxUOc&@~KTBq$?}1?rqH#3udewJ9w;EU&5U+7Qw<=xu0Im9gl2pG4mW}>HtWDJ~*>%Cf98)mIx zqq7}l`arSM55DbG7!m0P*H=uCp+97Q+fVY zM~~4I=_-1uJd|B&bBBz?T{yT}Y2c-i(t$<@QM>5SL)4MbXIx&HRtV(H+PQ}Kiz|Te zriBgXlNoMi981xCNG-?BaL$YkT_RJvX-`#plh$!)yE%uYep;D(A#z-!8ZS>cIdBhA`WFqEhM<&tOx-8Yq`t zEiqYMiv(kDfMy^~1Ccp4^+^|ZGN<8h+>vkl2xl{-Va_21klcAXNHYoyhR)AI)ltAO z8_$&!K(`FKL*DzcaGx9#`JYFQa zLj+h=aUmy5vXn=7QMw@*{868pN%OIOnVZW>Qy;&fqJ9;fW3g{@$JtgVO1LV(%VlC-)u^JJ66FqV<=)Kdly$b+{ZXgaZS}T8)*n8< zd=L;>XZpwgmA0AWYY6`maD@Np%DFUMU~qS)kGKSq2VxcJOsYwQ&7}iBvmuN_hX4Pks6d-Hz<=sVP*b1M=+$W$kL+}$TqwA4^W9Ev~++I zW?i|&rf;rq@Auih|%Ny-|-0N=62Pxj>A$um(PHOZaa3 zV4JFRoSHt#?}Z9=KxQv*Ty!HyBQN;)jJJRG7dKq$TP2P_kmcu>D^1OqB3^Htv30-Og`u;gpW$KC^oSHh3Ye2vf#?9*_2UkWR~QiJ^aC^ ziHRwMo}01umN4A3bP|4?X^x}NF`Q~N*>m6eg&cV0;)pLhF?n*o4>WkhrRK|`RuA7C zby&dzKOx?Jys*8WQt)X17yubIR>hSdbCrc4`kX|R;O%ji@QoeS7*KOVYvb0l%W?4R;O9GMQ7$cl6EugH=4b;JL=JRU{r6? z2ANr|%sm?PsORmpPobud71Z<~KGlgHfV8`;y#?OF9P4D_=4rB&cRx7L<~c6o6ZVw4 z=KE}3_?97M6L*o5nQAU z_DD*~%}&%l*a%{G2}e5stwg{&Bbp}E{jLg3EF4Z~i>M{!>;j0y)reHiQjlqowW{op z+J`j%S51d9Y8hBIIhkfcp?x*=7!|JcpIVh4SD~&H-8DsDTLkd!H-8?1Wfk*pzxfNH zJY*Ry@3Lg>MZr<-NB{o<>c9^LIpK$dA(&I3s~R=wMyfH0>}hmiv}95Ca+-)MPs_jo zAa;DP+S+TljFF6D5~}K}8U^rMegPWEmk_EExE{`Hlo8Zl*$90yiKTcsWa=~Vh52Y@ zfc+(-WinSJ#qbF|$rU2XBmh>4Tgw)^QI2{NtKdmK_3fM-${kXYIdb+hjb+jfMPodC zYzamm%$dG&L+|H0QT?UpKJWV((lnMBQef9e+kKhBpb))1SxQ#faJK;Zsq0EY2JnDi zDbCh%6^Ta#XFnkhzoaHI)nkQu!BjrU2|IJagu(hEA6mFlx%v$%Hs%Rt?hvNUGAhnC=0s zX1CGbY7_9KThG}|HsVVCu_eT_Q?z^>Byf+vh*NL0(}hEjVV?& z`^Z=@r;=*)A*jG5>_mF^=j&-YgF-kM5}$AiKXn9?4;S)>4TD>*!X7BI7qM%vQboL; zMbzk&hmofV#jxTkfSu2`dSgtYd@Bx|J!vnt#v}`hDG-aub(u^@PsJhS2Zr)Iq@D2lA8kfn75evVBZ7W(cP?@N&_o`- zv8;Y4ue?@5^qMt5j!zCax{zx%5jSMLFPZ4KF+!SQLynF4HPy2W5QAhQSKQ13gKr$i_a<|~ zE22Snnml9QH==RAWMZfx>Jx^yB={x=&Fl(O=$@BVa4F;lIgTR z&CU<$50yLq)Q@AsyU6^m-<$(=1(4)bKN4d6nz0qV;>9&W@Gw|_aDW?~wLti5tGJ1A zZfnL7QZ*>OS7_xIGgYE!%%CyKTCv{z>anXh6E$HN58!JS^)uL?v?d+G7zSor_dKJ7+DYp zPS}l}gS+L=xB+sh6|J$o5+~Lb0ga6ivuF8uAr_o$5V={108n&#n$|9+VUDWWl`JbC zlU1pSVs@+JGNjfzmWpAtq`7#lLm*LT`*#%=^^sOjlPpo<9Z+LHxlRgMhFR^ z0UfcBeU{MRYz>SgIfCKgnh83){&f`fuJj)KLp<97sy&FI07$@$d2Ud0&!E+wv|=+i zFu`Td@F0mQE|<-$9EnQ+y#3~{=xe4EmU1wci$GYQhWP7k`__drDu%Fab zCyR+%mKKqmk)Lg@h$zG(+=IR`;3ZkkDg390>|K7=bd-M+e3(0Jm?J{I@FN!DRA5nKod9-5pGDt5R2=xB}iE?PS%1DiSa}>g^x@ z6|dK?Ngy z5L5cLm-MkxTrn2XJ!UXdENRzsnK7kxT2vHUYc`Kwx80h>p!p7spdgR*gHoErXvTA* z#%D^CE|VN+I?|?QS6vl54J8^5lVO3_?a`uee%pRV-i`!_mmLHjO}m*q zS=tzC7pf_EOazD!OX7?s~VtpETWq5dT<^~8(r<0dn3Rr=zaK16HzGaHx7r=PSTzK>t1SjgLj zg{2Upu8uId?}|Ivi+I_lEJ9TGuC`j+%9h$OCnK$VYRF!#W6!Xk=6B(^SS^r!$;yS6 z%<6J+Y?V7vTHZWE*(>;QXS(RkwUO z>>`Jtt`qQ+X-vqTI(wII<0PIVEU%cOkAXp20Er%r$ySSdADbmRh04|G_;PqnhQD%j8GPI-f}oo2)n zFcpjGZH2TU0ID1-*Oh*q1Q*X1l;=QwGRtMVeX9crtH^PS{)*?3174MmC7Ts;^}K>2 z+6if5pEtIoy;ct#!H#WTWR>%%{S z>Xs3mTARbH5f3^w(z?AibDz4PM;h%;qdj6UEy7^SN_gx1tY70&3!J}GHly+(f;+|) zL=D(|t141xOXu#Q2i{T?QNTSE7@2kGSzd>!%v@-88W3J0VGWR&&kVn>n^mrD{5A!M zFK^fS1R^^iCA@1ZeTE3K!^s*~KgYqIDXP<2r=Mo6UYvFMbdsa?Fz$^gPD(qitVV~q zRyx?4EAraU7U%WgcAzRpLe*q)nb9H!?>RW4ClX-2d&^jVZHK?K=IU5Ai4PB_>sa)^ zwqOTlm5@$C03BYxeJ8%PzaQOOM*3?z{Kemj=k_qVdGm#r)gfy*CSZ8i!l52-xU=Wo z3cO~r8gSbsgdeGO?cz=1c*_eXaX>F$_Bfz}um7=8m{#(~zKY8Jm`>`4(e?GwDyj4? z*P9LsXQgw5>3G^ zGPsxPbsZ`?c&^c`?cpHbLl5e#1It+tUW1zPa~vF|U8@xVbV~+a5QXBN3FT>l;9w#~ zJNL(PVTY8;7tV-SmCwgH&7pV+QS2^(8*>rtOlybk`I8PQPAl_MXueJc}3VH?3H*yH0zd&(`9L-N%FCm>C7({|x@Q5R&HgMx`eEWGRYi*GFo zYuNIWy@EE@Y=Mb;Bz(lDf>GEE2aBT>bx8MMa+4SC05@qP&+95WNDNNhM$W9lt`D+o z2-rTZcN@gq$?C=Z;ed&S!)~{g4*PV%OSkmZzf;J+*FLAezx%b%>yP)`+!7-{f>-;5 zZ(rpNf9h{&ToC5k@EGa&k&sQ}?-Soc=>y;{>nX-^B``|0tAhES%dy_2?(I9i4dS@uh(b; z4PpR6f5;H@W_`3J1b*%OLf~IR2q?<)1a8|&@otz^>=I0bWC*n!s$HtL6%LumqkwklD(=K4#?#vwHV90pCG z8oYkH+O0nmfjY6FC4p*1*(#nPfOLBd71*V4N&gX-lF%Y`hpSv%U8}ZX+pPzizP{oC z2`3gAUU`M}q30Lmo;uBHr+La_iuG-LPOn~VoqqZAk^%b;!@PM5Z#eC<{N@3ol%+%9 z4E(%j2{HcjgrPAlROsUNDPMH7)9h56e7jmJxI4wU*QkoH8%{rpXvb~u=1snD4p0hr z3Q$9GtOUO};sd2(Ue1*6H&$~*B;M$u5?Fg*Rp26}GJE?cpRMFu-A1dI;*=(X5vrZx z!(fg3!#HgvP1>!{)oHX1`1|$q^LNeo`<=QomAwHQE#rA_dXNnkNntiRh9fJAKNj-Y zil{aS3JFDmp-RYw$d4vfjm?}k@qU__C{7-ekq|TV8_>Gf90G}9R4-j7MLq%IJ?Mw* z^`T6CegY?Zc82{5GY-fqF(ymz2pUHq`iL+G9bw&ay}?@y8>=#Dc5sTcax>|re~beS;1=YYftZ|6BTX!W+-y;Y(j&M-@v0_{@sv_q73Ho2YKcGr zK{G_=<|k@Hj+Cn6i6v>6o5nglXUY~|x54{pU#V^%QRugd%~xL)(t!7o*z^u2LP4?p zu`{~B0edIgQ-NW$3g0B`ilspl>r06OCq?H^Qr~IN?~Rk&sO2NChIlb76CP>)J$m1n zL5X%_#Ec($qV%#h6>J$T6#8v?tr3e55T?-_HW_yNAM?r3xTkRA%M@EK(h%9-0e{60iOSC25Y|Aw(F^Zm{ zppoTbd{V`cuYcnPSq0B;Rbmh&X~OE@48c6eH#osBpFoj55>|&yL4pW7zXJ(f zyiocKaZ!f zxX#s77A+W`#YZ_Dh)AVG3z#oq1veEbxLqO>LH1mX$~#qf0wten@7_VN4K{dS$ptJo zgt7P+8xwl1KH95#xrlghOYY({do4Vr=+^ZMStdKk0Y*O}krd z_49jus4m`oh8a9OBsfHMQ2NzYPmifL>vha39@OaetL>Ja^a2?LIyCuBVi{-I{b@p} zy`)Q*=1Xy_%b>s>{T$==pj*#cS-01yZ4DWF?eCpm&isAk%$LhPRy2>G>@wlPjDm`3#hr^~@8P--;Ig#W~(^n~Ug=Hpd@`wEj_kaY{BDI20j zv+@JV1pd8dm+1ZZ(`B-tY`whjf(#tQ0<4D%eaZBdDeYgXsyBNAyi8>vb_Q=-W*<<@ zosc1dQF4~bI)p+>T2&B$FnvPUj!GcjGPmE+H-ZTpnTs9vYd3;*dcvJ*$l_P zm<$gs{t7&6!GtirwAdq)Mkt$xzz}^h9KsJqJD=>=p1JS=W&a=z%M}`sc;H5Q)w?x+ z<;h)pR!P-}${^4UbVXl9^)?S~wK}CIH>#bs9-b4RAmJ#fvm{8Z-L7}*LyVMkM0Ir9 z0TeSLkk}t(O<;DIFQ-Hf{1|5SLBH3wqdQ1!b(p-EPFtrtFtulFM2RE~E-K~FWpcc2nEI^GR4@uQG^ zT;TglU0!vgm!c4ON(;}Q< zk>!SklDZ7aI8E-CwWJ;AOSf(?(A(=vo@GDu8%ld4XJ<(`pCxKKwN9^kUx`-2`BD&6*uwmN@7{cp?v`}_dS?Eq z8a%0Fzw$iCL_Xao{&p-8=z3|Hsgk^&ODSSzE<0#J1fETRikznCu;9{-i{uSjG)2}) zp0~t+!uk+dI2L1h{=u8cxl`{vG~(grJZ|Ulpt-fQ8-0|78V~~+GA>amyuk-t8d3-n zhk1@ylwkSpB+E*N8s3nmB;T_W!iXPL_6?8s0&y0f#pKsPDn>g{IZwRuQ9jD>S@~6~ z3*O~X3yzza9I_zo-_Z5^=fnf$LY&MR@eJGTa{5OZBSIed2-S8^f4bOgc$#xXg6rs+ zt>o0yYt?#3w^`oEzd$EdA{;$DwW)XC$jQweT*~IXr}dA z0urQ?Fo~ztpxUw5syE`%aMXvJsDbQ^wt_CdbAI{qpPDoD1Z~y^g_mWzqd$<3`KK1+rHX8d zf1b!!`h?J`MfLP|!{wLPc#1qhy(#L#Q#dDkfDd}gT*_9iuCTQHBI#!jJMA?<`n?S! zQx8b5RrR>YKOrFCZbY7~reW?_yV5Pif_xC9Y?tM_-T4;xDCf*2$(3TJ1}H}&S%K%u zn=vCQX>SDooCkRF=uR%0z00BKPOePY7~{&D=19dvLpoF}S~yVN<7rQp)+kGo{MKfV zQuA;wkGj)~tNf-$_Z&A#z_>f0S}Ia4UPKX@s$^vXmA+d~W8tB!aGSnVY7W58JZsb7zm1+Am@|G5!d>N$t;J$iMw9G98m~3C=@Fb#bZ@V z?8xI3Vbpk=;_^e+*UXxAQ1PeweQWd~?=~#f)^Q@-$+B{3ae7`t)a&{x1U75bp5;-- zBH__4!s3LbLaJwT%SO0cmm7IL3JTDT}78^eYmn6ZXYQ8tMouGtyfc~0ueyi z$_`x!`NeS2DXCDt=`YhbiqqJQ;zqUIY`4VryqMA;0c_Rap+GAElh)f@XscxCgfuXZ z^9!<8yv{Z7$H)Yu5b`!$#>r%sm&c{o@p4C0B|=b=jj~o>P|y1$3w2yl2tEUrTcRRA z1N&{Hc1}-W@hGSL{L570tiW4Hn7Q_120MfRNN6k#sja}^S-wq!-U|I1FyUUmOOu5* z0IXhZfNxXpc1YcJ5=eeaY`lGbvGFS=Hr@f7!2aD46Sxg0>3IyNn&%;KU*6>WF4gJO zc@YmpTosL<=6X!G89B0*6ubr@8?mN|+ThA=!ncd z0Ht4yst9wpeEKJ_g)>0P?7&+%a&pJ?O%y#N#HbEWVYPb`07_PioyTMXk8X({dkW>a zcU0W7x!9aFU9@I%R~%z(N-Q56(i%4N83)Y_ z6ca+3h%rJH(F>|6al+Qv{JFJySXf0XBhzt|NK2Kj(a97xg>|4D2LG27i`A226nz2% zaVHUE2T2NLV62={Nx;MQAvz>O2}sU5EmZEL2M;fB;1A?6rSSi{8_o*sC zvH$=9ta>C!3Yohv`B=QG>$6dyScqnjqYX9?7-ca~@x+!3-m8RStu|Gmh-jn#hY4!v-ReJ^rnVejUa82I%tsp}!U{m643h5wH zlawNF3u6TFsJDkiRW+)1;#_t*4$2*UNomsxAvalE@3x>)Fd&38r5;v(u>hfJ)^7k} z3Q$!LADF{C3RsKk2tZQ;T6HIMU>5|==w8mRQ|yYcPVjl;+mu$}ixr#~YcNt%fMu@s zB@4Ove#vJDu6$Io3VQ$nMJGbm$^ZMngl(20o!CQWhHo0jnCa5GvXJmYb8T$7Bo?j+(OeQ>0+r{zbD$JtV zB?k8KBm-spxML~c;Gj?nIKqFAh&XOX`_8uWY3Wlcn-JAnmugF=!{mgLEbQq6c%{X- zrXjHF!a>Unk(LXNLSVS%5)FBj+YY|f>MAcO?`TOwFAtEFK@I?%P88TGB`CZRUE*!4 zx)f_(AmE`68<^uc8(I+Ja=r$f3In`xd%-=t5RnOMuRZ{~`7EtZ$67o{*2<&Hwuk5O99Jz+YuJEyjg3E=+ z0?cDOZeHgop@IZNjamrUV8ZY=2Zc4HoPqVdm4om{d}F%MROR8{{LG(%&-lObw~W7T zJ^qU?j=%L&K)1hP{LSNceR}+DFRlj<-aBeDW+J6wNQ1FR0IKT4crcV*jyA7%vZU4N zZbLHv^jGrCZx9Upn;ut>`DreFvbT|evR3|02X@Qp(O*<;LQbl6%l4SvE-89?Iqs#X zRnUv67^lxqEA9%)eA}5@&FLHOT|S3r)T5@VFDTT{m*^3Y)AL126q1MU#DyqI;m7=l1()*IlL<)mkv5k zq9)en)|8`+xOx5UH-Ad0S{@#zgxJ6vAq45@yJ1$NI`Zpxqb{JfXZ#`Zkk;Hk$pDN% zOm;5TJJtHp5xM*5dw?s~YIU;_2@%mu6Ooa-q1pu3=*D&R(ZZ}$%tU}J*8H{Xs-`=9 zvflo!&pH*41YXD7y z3*d$Y!EK(v2cllBFJDJ`67tl$ZH>OoPd;t+vQUPyph&DeE~7`bn#Zc2v!MqWEHoEys6 zAvVo<{nwPcrlza5U-!9gh6!)KH#*YR) zdX)kssE@!9w3uB+te;-UDI=Y0qy&+Q+ilnXTp{;f`$ql!-LHL<{y0U<{2j#1m%$dY zR$mWkO0(_$ihezBo4Q_#JU~Rp2wJ1;E-<)(gXO+!HVoh;Mf~4DOh|GxJis?8=Z@E{aoAzY?s_uJ1X9X66sU}=V$L9*{gA1A(E zqj0i8&DAqM$d%>Kxh+7*ZuEUoAFyCpwY6m9C}}949$&E9H}CYwo^~k;HXVmJY|G~Q zErNe!qQb!wFu#-nqRZPx3?&5k5Q=gotha*1u9Mz0K=zrW`;Y3Xu+dP0n!cN;Pgj)B znO?O9^g3;y?mnONDV^}@yG1Q0L7@8gS+b)&ND?M355O%Q!LsV~P_Ldp5hW z6o8y|-PISHVBSuoST(0Qo+J3E+OWGz~Y=<4y z=4=X~BF2Z{A_C*hK9&8FF*Z{&@NfXa@`M+d(;)Ou+2XKvE#*%s@4}Az=0fU7Y@S4G zm!3v~;?#a06cwsg+es-zXOE&VeNnC%R$-Q{&)s%I(yDBLTQd65BGiUCoF!&z!!+sC zKq7QgTF1~&qDPf_H;YI0;h^14YlJbjtJJ^i{9@+6K+L?uw}(yla>LJzveC;bWZcb7 zOKme@0qJ?#k0c9es_g=LwMKAQH~*xi(-}*lu!BmKxA76$vw-R=G{3Vnl_m71aY+dy z3vVhG0Xqcqxb(QwtX0x(CoR}*1;k@F&FEUSN0Fi|o{c}bE_jPuzYEt^tnGj&<+5@w zd<*=5umb`6u>sZN_r9Opwx6J;=nTx7GmIxkr3d~~q*a@LFIo%O%e^bGUB&0r&wV;~wiLu}6!C+TQVuaUFJMpTiuYILGX zXsgR;x<=Zq*ShUNt+QpdeE0dqiKaEEmd-q2t?LQfJ7{df5x)Y)RujwoYzNAwm4Vjj zK>$a7X7Y((#Fnfe>Vxw05#biFS~=j9ub(&>Y3bU8v+&d~fve%HsWhW#{92@zo!^ z@cv8DcM;wb1XVTynv*7^JH?HvEU#VQh94OCJAPnb_jqg%!!LE~#vQe4Uw0(-89%W6 z?#Yt>l;8KA?pFdOe*br#ytk)&!@H{G^%)xyjnZumsqmt` z@0K*FpI@5%CennH{&Ltx&f=pk_onI=kCpM+I7t&bS&~@l3#Z-zv9Y0)nq%9W^O@iR zj&U`uP7y>G(69*rR_H1Ag^`(SW>O;$_C@>0#4rz4$bKN}NQ@2Tk&d$o0+z+0ag)p- z1VSm?5ey3#Rjs#QN0}3yHb8bs7e_nqwfJq9QQJiA?H{#z-FjYy;HzxsXQA7>G-Iao zg#{r|)Vj8Afl@FT-YD|icxZv<@)FQmzD!=EeT&xDW2kh@#BJiIB_OwA*(}qCdn~-m z=oj}YGss=F7>P_C!lO#wfHp;~t!apil6RF`To*E%zJa`HcrzM4S-;fm<4xL2_ylog z=JwIP)+Su%JL+O!3dOr_)L4;(t4ju?yFfq1(6c{~FC3f2uggE$41w~SB67NIN-CDI zWlm~4`m{25qjX;3?+wnxI_JnO+%PcTOH#XEZ-l#OXsZUQcH)gDoi1=9E$)A2?4-_Y zTyRtg4cmvBjSIYbFd%@^=r)_cC>lglwsDcY^UIBY#-*nQbpAze0`I#@W5W>gjxMr& zjef7iHeIwmt9NSc{2z^a%l@Ifi1yvp9=g6J*AVqI&>Jo*WxjJ)1oD!F>iAq6piU+gtVAe*IFf-s3Uu zeMYb3#8kmVOh=JgePpxM)~ozW$`^GSvZU>zcCUG%5_tXOXSsh5Bt49u*a)<|@+)-1 zLk;3ESmSL93)S96^33{%-)Fif00RDPkOd$kL{W2TJDbm9fyoc~wQ9M5l^OsgQT0Cl z%4))3$`yUiUS#6IuG`JX-=K%&kkh&=VvOGY@gF1CUAP^tji6zYxxV70-!)0UaAK9b zLZtM|J3*)|!BIHNs`T62>fLju;5~$S+ZWKLhmS zl^LNrHD@ZwD+f5H^cxY2DNRX$%lwtwje_i5eOi}E^1J(Ybg_P3ryo8e`{ZK^C2e^~ zzv3lHgNyY`4869x(0*)4N{# za8Th(v2p2_`W-k;RUSE4v_kLOmua@Px*-VOFe(c3@z8GC5> z#&%?%TWpoydPSFG`PVituU{hFSM9g+E3~60F6msV(cOS+K02SS>233@A;n7^l{2G; z_NWly7j}{8K^wd0A-y*xvT!PPTM?Jp*6PJ0xqq*)^n+S|L_&hMtHGdC zxKhzKAWd+V35rSkhOAr{;}Bvu4)dBbK$T!DI_ zK*BwQgHUWJmL-G4&?!NA;>3#l)ZtLtj@i|Ul*f1<>&=p(Xh#hHeWECfY-@hgOx2;$ z9bklF1m`v&x2{J4F4u-t(+H=jUtPyF8YX}1R-_zG>~nN9hqFW~K`tUp)AS1FQJS->OK z|A-B6x!Ol=G~-pmPUq!jggC+849A-Ex3sqxXf#-sgdG2E!+WaXkx}I7 zRkvrEEA2kxsPLu;Hn27+Ea_? znBouF$+ENURQ+R|^o@qP``BdJBOwRqhexC%$w2#Lu33n9-qJN@LiU^d742&p9Wps3 ziS6^VCvR<-D^F8eyY*2iRh)XCP~N^@MhEDtt^14Ya>y!g-P@D}(*ouqLnJ^0%9-$U z+qe(uSYPtq~o*Yu!Pw*TXSQx5vcYIluJyQ5hw!5y|$j1U~H@ zyJ_O_f<=rVv9I%iX1F6p{w1ou5Mj)tlWJy6C?9WEAmD8doaYR)>>oiyganLybI*yK z7DZteNpmx_^i&s*3$##5>vOw-LQcQk=Xw=alOb{vnnEzofGCf+c}`74WSRI}vH)^( zV<1lR6T(-2i1dkzJB06#c1r3v4^DyuGv5Uv;``QYlgqAHx#c3ECd0Y()NSChcC+ve zPyJf1>9$JOgav=Zk5jV#2#;Cw_eJlO6>DqW7AY(j?2HTWhUK0vx44qM2NfH<*%N!P zAK+zZi}yC^dRMwK{~gjF`&In~rEs1XuGC_=kCLxD1fI^=W_rat1t^L2z^IF^7dR7MlRH41@t-9I&PM zA=Hi5NTD6qcgAKB2@XI4IsE#mm0H>`;08T%ygJer$?h$=nO84tS z_i2I#GSwc+^i;!)Hu(7l@>6)w29cFt@)s!JVZf-;jbu2us*MGd137GYY}V4ioq~=I zi(e8(%ohCe1aj1}8j51&XSG2cHhqNlD_Vec85?LqY;oA|Brzr`3J~UoNcGiYT>tWs zWWb5*3A;||`|Ikw<_^7D++xAb5Oc-Zagp(o=1|Jk>O!Z?3!fe{$4@D!s)oxDCB{@eX}t=9)% z-mYyIhY!v#T%3dQr0$%^xQx|Ra*RACkp(w#XUFA=a!O0mdx>?ZcT8$CN<2~vTR~Gt zqUC&Q!+(53L@A5FnNX}%f{UL#%q#Sku*ju_LA*0Ufmu!vK3kZt0+JG*;!p_zzTgVT z8a2C1RH|g1*!=1hc%4|4Hp$hacyNr!VitP-kp(>oIwc51Nsx+xWI`29e3dOI*0eLB z1@;0jhRt#eNcN(D9!>#l3yWxxU>x-V51r34A_3}3D9RsJ>bc}hk+@W6vt&L16-jd& zg~aU?hY}{{=?Dt3S;+i`2eJY;Lv|V;trz4}5gO%h@D1}fmRIb~?c(Hc0n0v|i_j8& zkr*lgq;LR(dMMq@@V<%W*Ot?@WeMSljJ&jMhwYnvSP_E-pyNA~?SP)|Du;ctxDKCIiDMcS!j;$k*F>JWf#seO=^TOMj6fMMqPp$Et*=!Nu8!*oqoSJ9F6$h zDg9w4=a(Lenr>#d-`RVmJtic1C7BAuDO*qNq?F~eS}o|&8RhC9I#eK3Qa*4AKERq9 zwMABWX|kjw&)Tpkh*(iS1N+%?3A=PGoB(s@*XW4wCif7t!69OCaBMXD>0u8 z#sR{DEM%!YkutB$)9}TKyw71G9Zk77%tVCb3EVw}pWBrss1@7eFk2<27=@xQcbB6@ z8-S!B3&wzn9v3Lqhr4YTk{&n zyhvjiaJGUP3k|9&c8(XDKfHUz;AFiwKV%0REmgvS8Qlzf-yjB_+KjfnypIp$Lgid; zxXGH^_>v6)Wv5k`MC^_X1qbffyZo$`{z(9vO?Z;{ELV*EtcTUSM%w=<75-9xMboPz$pJ z;1>9*@gng@Al`h>4W=moGjhetM>6=5D3$PIBcq^-teiYAb#=4^<0ujdp3UX$PG#Z^GU zEwY%-Alz>0CKCv=?7rWxAf<8~1}QQn#^g_Nut+0(KXPx@6QkH%@w)4Fl#)Xiv49}U z9Zc&!5**7yd8{Z|pd#3xX zUFe6GXoX{`K50ikd8o|eMx~*Dddc{#>`O=~Bss+1IhZfZ?Jv>gA(N?m+kX*DN>>;s zCQQ1y8yC7!6y@Rz8&-@ z6pW=u@o711r+t(G3`S0UQq71ikvd`~7%l3m;O6JK20o>>0}_Y~nP980 zN>VqSND!j@8&2Pg(0Z}xxOkTNQl}gDXxMJX?OKcGT0^Sc`sr}Mz{FY~Nz-ok27@hB z`6tgWSNN}mVn(kid}$~=dV((ARq`~a5eY z6GXsR<@Nl}#vLmdfK;6?_+&gSsLn){6nklp&}MG2MN`a>cwSbn18j4|i;9%Vz(?kd zVxrr)oUHHg32)}p%{s%R2c~n75shpmbgAp}gE;MF%}#g3ZwI3-_W6;5eIC5}=w?1@ z0u@tXu#8vpSfbP6P*klT>9*u&ayR1{iM{nA1 z0;M4{8i^GNQmiKx$3tI)y&KW3!BRnMr*QKVBiEWAqx${)%Ei#{=Tk3!Lj8Wam=b@a zZ`^ME)4betoqui!dE~#N0@_y~3UHN0U<^$-)i`pP7lZ|EjRs(~Er^|B^-rZZX}&K9 zfVNb$vz$j`sS%o+(fnC2NXw;>~uTKZ?T|8 z3_zM+J6l=L$8KJ&vBzPY>3n{KSA)x3-1Nc=H_ff#vNfnt4Jl>F&|MrMmhpZfg41lo zOjZ($R*-aGt@ZBf4WX4z>cXp4q*=1!5VO}&t~^ET9D4MXj@bgdYS{Ub90K3_e>AnO zL^#Q_|DV0Lfwe5l>ia;B16)TKk@g4?WL*k-x*F=fy5Ap#n&x#?b$2mc-9=UPOb8~W z`*Ci)SM@&1@4naFHQghjWMq+X4Ks$ei9CiG0@je&76v51ik!%!I5tv9BF9E{631ks zBpyUeD-ywu5|ZD4t-bfz=id8ny&02C7YqXW)w$>Fv(Nrmd#(TaueE0MkI#P)_G+nW zSCw&v>FVg$X@6K*GZ9|^B~}FiDc~Hjl1tTDq6>_+^&x!e7;JA#zk{7 z=rjtd?vOkT7szlnr=rGuIO=z&-DY>P8cq7crKq|0G5!72y+5n}tl75jvTbj(PX47o zqGpW7@SXWq+)rUu4D%u1OrRq?W_yz#9RkYHkWuh!6S=oK|vFFy}kmGMjoAQHX(Y*WVANlK07cg{$@5;@A7VqIK z9;>2&G5HNn4uBGHu6peX71bd)-dlL*2YJwC$%mr+h#|%1af;_az+)x~{v>7QT@UKD zUi|c5AV;0VGN%*3lEbB;5d{-4nz9@PYq2H>*tL1233De|6F>T~(VX;HLm`g3946YthD)4_L24T=gb-GaAlE#b7XO6eFYz z5vIz%LuGX$j0UZ7dpd6oSLvXC{_OJRZyDn7?x&xe(GnX0A#K$~(U@?72*l7^>gVbc`jeZ7P;lZ%Hg z?us|HzgQ$wj18fj9GQxw@pVw2gQIzYWps7jVmgCKPL4AAggt@Y;tk{`ybf>HAy36~ zO*)awm93iF091UEih0NGC1N_TQ*{`M|CnOOPaf3rJGTVKov)1_?B2OwJbdH0pihh$$o`~7d7gBf3(yHr1?x176BKL;KnhB%ih@BPRR z)SjVz)9k;$`2$;_3EpP6yPU?RBH+Mg5?v+m#=|$is+mZO2;C)Hg8ATtmEg3JcR&5k z$3cSZ<1gNfX+k80mFyo0BK68dNszdV<-tORo3D^f0%STL^m;u4)>}mZpM2OEH74ZO zX>=x|KE7BSu;bP6x4(3D0rGbcAWz_sA>Tj$iJ$weU;acT5}G6r@Wq0$|e#FX89}M{q!e(Nu9VUm4&yf z(gTxz-JlGkWE5f|T(DJ2GpxclkG&_pU)l9^8FMO}oR2=Xbe2oYQKP6?(z)Y#9>q*N zNbS7)?ooRZNZx+q#Y;1~&ueKGYCgJ$|OORSr7p>ymEYCt1r0n@;bVbhsPQ zdix->csFS6`uUq0cbUzN?UTC_=j^K&JwGuU{F!KEu(2Tiqx2+ab+kC76kxuTn=N$N z)VBxF@sPwA9qJli&Ol1JK*+zII&x{9xH^9hk5nU$(Ya?#bh$!%C(6qtv#4pUU;}yZ zE5eIxA?kn*{_j1JZCsi*^rvQO<{C+dP}V-MvK;*E?yYgjP8iOa!Vpo&MP9`54WbJ2^d_EyTlt=`m2ci@q zb+I?Ge)PhRTa5JGg!hPdwgC;nW#;Uq^ni3zo`qXW5V|0(dq>rj%(fe9C{nR z%tml4d>Hmw92%bb2>+Lc1i6p4@9rnDUxj@ZvuA3kCUNsKDhEU{d}ocH&&Q$K(v6Gw z=rfFRSl3Hq$ShgojqvmD%^1d3A3KKe#yAoWd4KwZW32DxQH&4x_~FTAL0=AXG>m^3 z!jEhcF5~{FMF30_G8kn}a0gMdc0L?%2z1ENHR?{Aqg9s1&z)Vk{7ng$Y!Pl;aq*kJ ztfN&fIgY)X(ZDPvdD#B+)iYF9w_>=j(5|=Nf6vyDFypffjJN2J`t+=X7m6S9)x`p}c0YLH;ybVV2& zcDxQMA*sSW4N-sh(;%iSFN{A>5X<5edso6)l57hwJ8RRh|6{MaB;G1X*pD81_x|F~ zRJm|i<%W_=a2Qe+8E^GNk-vf)L6A3txJM&qm*+DDmg0`4jlyF%B+YQAL!ori7l5*2 z==54W{@I^*`mI*8KUrmx{_@#{#}6PpH2P1*NYxD8oW+_L!ZA|ry%O78amwLwG-r_3 zEGx_44ny>vnE|d)DB8-np%rX|Wqq)H?E&#P{b7*8czz-sz ziSJQmguV_GU|p40DaLYzth|D4Jil1rShB9JMQ)`#A;ewG)~hyr`a(9sJ}o?dqBwp^ z^%h5RZ-5Ab&W6ZX^CbIMseG8+nPqybn5_e@e#lt)eDM$|m~Z%8hubOk9^gS{#~6T} zyAw3HY&Mg18Bx@bC~dMxXEn3a(VHcd3CbddZs;yFgJKMW;^8Z#EhD`U*o)nfjlB(<*>Cw+;QIxRSCPh> zN&yL;1O+H$RpQ+Nv&Ji9Sy$@F-%Xo)G*ewC3Y+%YEzFK~d)gRxXT!#H-W|-ly<$Eb zug0#4JQb^;J;rT0$2dGA_Sq>qJH(@pe_ud$%9{zBc3PLNkgx@{QE^wD{ zOwO#?4dV<9%5>{Y5w2&>MqjB~1HmcBiZP0KG-%nBFX3wD)nqQupgQhzT43Y>T%rh% zu|kZXY!FH9ihE50T|_^)_?hjpyu}-dU!v7!XB|!g?U1YSz)Hs%>(bM6w42R+i?YFw z3eKl6`VSfaBHs@wYU(hVVT&e$Ndh$LL*h#D)#y~z;m3p9%UFU5AWsb1)`FJv8zaZi zK2_rE!my6c6!t?bepI`b$4%stK$%N%xtd5ND9qqJ{`*fiIbeexM#yqng4p>Wsj|j+} z;hx5S#u!f&FBi17TIFQE{BXKg)bc1e?u_ffW?J8US&;&{3L^Oqzy_rueF^bsPVFS3 zAbd3mQTm?m37#v94cNCb&HT8$7yiPEC-WUGr!QiWLi?zHG5ReD{5PsS5U zqasTPydQV^9U|{%%~c=me|~nUqL{WPYqXfFTdZ`p5EHM}-r4FIv&8FQuA3}BFqw*O znyL_%1elI1Rx_jN_R5@o)@X_0Hvk7v)K`po2$L)sQQJ0Bk=0IvGAZAIZJ<~1)3Od( zk~*$fJt^I^REerm?k-y^hZYZSc-j`MB#{?tl_@ZTZaTLdf3238^+4emrZibd-|m6D zc5?n9rKq+5sp~+Cjsf$)kBIccFT)xjlIDcZCV_SEzQCN2B9Mu}nw5r_vd@-?f<~?( zQ<|TVaxPh)%_SA#5zEq#PkC#+t5WYGd`R(U(ugprdrV3h(GvILDXtRR*aycAo#F1f z4i1xW!5~J-mh%Ee4JLBG)}39V>E$xH1UgG*Kt4Kzv`1M{agQwdGIZi>!2(zu-?6GH zfSSVHbs1NAy6pbt-5IFnrq;qoB*;P#X+Waf_B=SgBUx$R@R+vBf!KOL!jyzAHM}WRpk*3RHSL|`Vz9p1{Rpy&`vJf3r|;&(QxN& z{wq3i>|yeYT$=?*ukw`OHLzP*c^@tuzC6fYl%sEthk)Z}Zh@4~SVSZ_pwD5VbnUV( z(T%*>B~UN5fxy0dg637W5U(ubP@(si1z;ld!uvF0;fp^aG?X*)yahA~#swKRCV)pc z1?gwUo>5>e%e_x{D2{Q9abju^c$d9ehAy0c<~uj3E3FRj#(dbNk~~ z14Ae^5&nIKC#mNt%SHQ{r+2~g)4#83_lIS%+7hG;iaefw`s&S_uijkqVwplAEh2EH zxg2hzH^1NkFW4>kjw;H6(VDQ!5J3`4udqa=ImyM8Gf~ND3qcI&j0Uned_X*?a)Y4a z!%nIY&dog`Ey7hUsqPuW@G7dF#jOIIcsKM?n#S_1a>o~kME8lHX$?SA1p^4#RN!Wn?UbLejArHR5R$j6$0aCx#7_X21` zgziDHF&|KWhM6-S%o~Fy*#^7AIne2>Xf}#Y2ao%#7!}i1Oy$2jyEu8yFqPjgBgvY2 zfkaU(y5vz%g&F`rDFbNw0qw`m!BQ}g9R8_=XPV8Utsr7x10YRl^>e!r-WW;<#(`%Y z&BE{v+;PAlS~|3IL!tK;shuy(%XTKAE+Cg!I(a{W03giNJQW5cthKwKgJHi*MPO60 z#LSqOoXl^j|r4ongOlY#oH}8QL0H*xr4N_YyvKD z{^Ac3`L=in>YplD8KDgtu6ib+V`vOAkCFp>v`p|7+|mOmqqg-SWnL+81YHYmgPi1s zXcL?>R&0$?jj@}I@g{G)#k*J&NRMp;I!$ZamOg{bL$ETh~^1T9sSkqHt^+2`x}WT+&YKV8!7e&x3YQ5PW%S z)0y8e1m)!<6m9}wM%F1*?#RO55FH^-J8mz7iE>z!9q;Hd{mJN@hGR$%k@&H~g~d!I z7{@aj=L-I(*knb=QBu7SiRhB=IdNb!k#T%zha!Np;pe1IVc#?Q`yu4RT8$BROUJG0 zMJm&xF=O})7c~%xr@|aBO)jT1kwHSGn34kw6jf16!ztBE^9Dq1;Wk8{a9q3KRXo7h z;OVV1MgeaU6N7nwHsPdiwE6-gIup{8kBSy)ygJPmMDodewi;slg)@tZN53K$+Et6P z|9u>&-0tH&9sv>$+CqxuT%Jt36Gm|~ZMP@gS4C})-_&^u(fMh^%L6C|_{y=%c3Z=}?RaW8ca7JoPADgFZX0byqBR$O6WHR|~4n1V}xw?_`{k9+|D-P|- z4JCV=#w;9h2MJb~>q4*)O3XUR_FVn8-;JM@VJN4?52D?JExDfP8Qc>9>kIX3PZR2C zPC^N%=w7pE=knoQ214FEXnRt8!mzwS=;k#>8odo?i^anMP9eB7JyK32FNJLf^9C6n zqu`gs-&Xzl!@9_OilU^>)&lj6M0a<`NAwJ7XE_~Y!DZ?q>;Vhy+d-#9)piM~qfnv| zO9(@EQlN3A*K36@ILoyvJdxe@wTstZP7&83uN22T0+mB>Ji|lkPSlyW0vbC-tEORj zZj&T!&b!Sv8Oo5P9g-DH@IjB7%~7K@9*m1_d(s+oRu!&)QObaO+Eqjl{~!9#`>wBS zy8jV{tJ<5=jy!#!`3QH9#H~=S{A^E%?W3*2NUH2^8Toag>QYmtFjFc}FFde=SuiC? zJ`&4^>eC`D!9(bV)LMnxS6nno~Op!Oz0f!Yp2xS}8iq{y_Kyr;oc zV0lqW`wUQpG-xP&(L;x8FL;3o3aJrEE@u!hPsS*v zfNL2@>_3~bENoUL1EIg!ZVwZIOu!IFv|C`X<`%-s&MAvOw;#6Z=&=SY^1>srADn7p z@xcHMiqpGP3!6x|z;(HmxaJwE0)cHW=zs9-|6R%3lbvui?r_lYdqB^Ed( zVcs0?dmE^{44AGRJlIcN523ynD}8^9pMiKdSU+5NRq^^qSm5A_Pgm}L8kF6eheP#XynCL{hQYTKoS25(Z`yj*2$<(LI;OiMeeX|OUEI->zwXX^ zgT}Dvt0Xp}rI@yQq%2zzFkf?a0rNR-4O`^@_wSa&*%_73;`mo?!QbO>bDsspna2o{ z_{L3f=lLP3^TSM%^;2)B`g!~R5G$TNcLxKtP!hev^sQC=$U#6$JG(q5E{LwPww z$4C)KS>F!ABpjcpuc_Geo_4q;Pp#y5hGjwUfq$s3K$>hwGiS_v!Gi zwEWDv2;T6lwP3>$-$oh$59f^+!kz1f;bsievE3_Tj{~))_$0ni#FgXBzI9GYu+k1t z-Cm@RP|b@g-_MmF_((M>dgG2xnzY3rhkq4!qF;KUh|VMAV+=heyRuDSmC;!3e(Yxm z`S7Z=j2LOP2!NVn z6OC4Fpszo>1o;oSH5@VFU}XJIm24uIBTvd8R-$v)OuDh0Z04WIV!*IV1Qf<#nZ&iA zCIoGq934cAdQJL`Ba$9)8XHZamYw)Zk~pnZKe^dNE&@D+&qn+F)9PKrdbgH-Vt1Kp zRz~twt}Qw2)!f$Z43D?|GUCXl+wH~CzyGwaY(@P+vItqE#rFu=U{59u*`Btm7eO6K zkiJ5r^Vq}`5LMnmr$QLyIucAcT+7>I%96J{XUV1d(;D#AfaRwz5a&+<5KmL`z1rfg zvM7$whu%L70mTF?chomF1vj*-H+Ru*z4Ck%0a;%q#FPvrUEaC5J5ujmPXwZf z-xSaa1Unz1q%gn>ISLX8zIpDN9PkJLF0w?5lM+G$fF!izn(>lN`c;#@1ZGs6#L$VD z1tudCY#Y2M?432Ik(HC%pdZt{c;9}^tp_OMJ%~xyG6FU|3WVhp!%aim;3P7a)UJ8z z?R=7%1wxH$6_ZOR!F}qWWO;OAoKWqpO>oI*JR0&(i-PvF8L#s`hYkH)`3yYi8!^O91 z=j}|1MqTX& zqs7`LG8U18i2{VK#K_`xn3geX#57Cs$41VH4rsy4Ixq}dR#_HL0`I|^VP=(hmtJ!+ zm<5{LLJ{zYR>87xvMxLGB?Vgosl`-U_9+!0|Ip(32*?TazspSCN|0j=GmV+GY{4O- zGyzdkdBj96>45XgH1XeivOB%66n`-URFcUM+(^U%q-NO$z-W^o8IXOt*Cp;}JemP>#K6`K>xzG&OrH*c6GZB^)DKJyMv%<1)|6 z^IJx>a=u#JpV8vV$<-O68o@LYdQgZXr?6Mw1aDZNJrC$o{05wf+74iMGNl;a4JX`IK8=fO@^h0}~lvnJJ$X zzQJeFbCZ!IU_4tJaElMTaYIEgLAIBJZ)?}~RldjE&my>F{#87(W%}qCvm0YcLl+!M zspR_AbiSrfGQ%ee_XSu2Kee>0S!tmd{BSc``OKOTjYB*qZoK;khi~8?B5+d4J87PA zk>$la%VZ7;(9M`qJfSPbz+oPwk6I|0RJ0dUhB_j^71Ei^>X;CjWniC$Dy+9`k{U@sKM*uI6NR96DME*k)EqR59uzuhw7(dn*KD1XqcCEt zHP=yODl+aEaISk)lMPDj_!Z?>i{6y9*3y817seH!G_th@lH5hyuEA##m>%F&KJDr#b$1PYdD5ZLV>DgOQAkv60VI#WNng^KbW#Qr~F!-kykyg3?h@yn$mEak~jD0$fH6iD5Q%Yw;O!6%b4TA{YMt3477y}Q-lsUGBuSSlTjyXBF`ncX4{1dT4d{ge zl`)AWI(D5_WlH+4I1y!3@C#7^&Rt01Sgg?c&g4BweBI^;NxI>78QaPp1Lq8PZ zA-W`@Pl?>p~qN=H*RWO;2he6L2Fa3pyeeNzs@L=BO&9TBC+{8{Ix%G(~Fz^NE1>LbwLfP)O1tc=Q+k0<}9}>9|ctiesc|W%P^uEuro}v&5 zY9q69A8LsG=E{LPDuUr;dmM<;@*cSUdiY@Mwa+ZsTRh`4^(5U8&1Hvh6NnSt9zmhG z&T3Xff<>3UxOX@i?_G8^HvA&w)`8mU6I)VwZz&E{aM7|~1TVmCap@A#fxzy-hSKfm zU$M)faWOYv!Ne+NuSN%r-Yf@%XpW(BeNwEHJPd2&6;yeQrYpv`i-9r~DxNGhX#g4h z1+9=QPA=wQS!~}RrcfnPXmXVqciq`M5@3YiYNo{Z@fHXXu$VyKcmdv`l`onO=3W_- zCNfxz6qYDrGHpDO0$BEJ!><^tZp>CDeWR9X{qF^sc8a1{1`j{LsFJK~I z<6IK`#ioMP`K}O4risN87P5z|Ir|q*tx7u;?e1zq1BL&ck(l;haO-<; zNElmI?m_()bRvKNQOb<6Sw?f%Ij#sXhvwPODzM7G6}Vw+vRl5o$^aJ7a3&`QMXIb* zR%OKG@&&HlaZ88q$pgiQ`1u0OmCL~=Iu_cc)7y?Bt!!_Ha3>BM?u%Tyz&|tO!)`$v zpCj8q24X}!cW={AnHXAdS)U^7#J1L(BtJs=f(GYhpb!CSJZFdk(vG7|}D6>L1OxO%@@xW;I-u#<3Rum1;_=wpW z`j%$J`lA>DGom$NX#)8?%Vnz>Hxww4xc&73=d~B-YcoM5A#vT-CRx%SO#1yknn%%V zstywdemYzA+PrXfS@M3^_3!DyDMf)|*+L|U08PN`vj&7f7f;j4 zTq&?k8^jbxcGA9uc4#tf62pQ3l4m&+V&UKl+sH#9{2to^F&vHKD&z&=Alkl0yhcn@ zeb$h)C%iD_xZRx$2PE#9jrtT{nG8FPQBh1AtpTMe$KxKADOXRt>*Xqa^vv3Luf5{# zc<&;}rw(j@Rpk~^K-V|NGyH~KTCNfKjb9z&vv&T-U;PcmXMN*Wf7Mr2A3-2+!1KS_ z&xWnC4=MO=71vnHJqS<+$3NIRSk0E@6=E31u)Ktl=-7^qQ?zl=g$!gw{7qC}Mga z!|SO>_1pT(_8SB0ZtunEvw8WK-j1`m?!hyTsJ(sZou|&M-TZqxq5ogKe$#9Mwd2c{ z20ELLh&dc_*9AOKWFS;O0dRYH4o7V47WE%y|8q37rHi1NtQ6$^uH|PZ_7{f_eW4Jd091_;s{6w4rUBVdLoJ-+rx@g2G^`Z zujS8HG_(w+jbsCRib`3=ypVPu@{*VCA{#&t76S!Pa&mj7%p`QlfZ#VmE!9k!Qin6w z_`9Ee^l8vuz9tHYT8p)A8+DUhqu4>eIpkO>MjTb$)@X@iv;YS=(T<`4>A)bC-YA0Rr`Y$6m` z%MUU3MGAf`5iw5bOl{9ZX$6*rAjQ{c(ut^LHX3NXZ`9I;vx?aIP0`6=JEH_?j={EY zNs~jDOohgsv1RmU-l5i-)qK5nyghIelUeXQ3Aj0hQQeU+T`ZM(U$p3X*?Aw8g zug(@6#h;z47vB&XaN;HbsxXkllUDX)%W0Q>X$ncox5-aC@h9(w6a&IPY8UJeWlj!% z?j25T_xXbVs18eGE94K;X-He4oKx*Wbg-rirVA46r*x3k4*9BLy63!|Kqf#ISy_eJ z1s;rem8Bo9T^Q@c$1vZLOaNN%BDQP`Dsz;Mnk1K41>6u8L)5=KZ%=ep;PqwKN`J9k z8}hs_uLkK1f^4-K9G%zP6YPcePyFb8d?E=R*=gt?S^60YI%uUh!v`-&*GMsk3A&orCkKQRXbLE5n}kkt z&}$DSlXhc1CaS$Vm_ps|%({(1e?IRIDMvZ(u7;n!es-brPr3EIJ2I8}aB)VAiZBEG z)3h7oESjn;3d{ zFR_^7(P`0Sp{Rh4QT32Lqn#i`{*n{|W9^r_j6^gf1XMD45GYE7vN$NiB0B18$PS!bTbCMNIf&{=p~#WCPEkbo+yK^Sq&(YC6*aF?NGBF z6$xX!DyKJtqlfq2)xANe0(6B;ChLg{P-IH^`_nCNeSQR*hfCbAcq`o}4oHmISIhrU^?Gy^f0l z!%;d+BEj?McHqjJDQp`6bYU)GiCVV@YI?f6a?l%($!^wdwA+-3AyH2uIWuZ>=KK%& zx>}tzSl>s^E@wW3i{XhJG+Yl9>Y5aAzA4VMobLt`R!(%+Wu}LFdm3vtJ)Q|=mNLJl zIkYd}R#Wasw$sKEcr9>9T9KBuc> z_5nY+2kK<<4wH3%_hi44I!A-%-F`?5n>}aq;e6N}Od8Ya7)~I2LD?I6jcIc>AS!#( z>-AS5?2neq`q6iv9rR381IDfd6v72DgcmG3FD2XggR&3zZNN$WA`i&ngA=lYl^J^S;u#6; z3VCDl&$Zc&Prp%2vKB7X_q{S7zyOJ{EH&k-XJ2AVJBY=zc6sOMr=<|*^J3niQqr0? zoIR&w{6Rd%9p)R)UY87}2wBkkq#BYw*Meg*(#oq)mc_=ay`5cryZ&-Wi4QWJ3H-W( z#|2*RDyD)+z=!p>Yi~CijmrPgRG0>C&BD#!u3z7s&Gw2nM9ch+x_RowgnYaH`M>r_ zXXDUXHgg*uUwY=HLm1ziUUsF-$g=dPQa7_Wcsk@KS?cEUlj`erFJ{3|XxMB~WshHV zu7|h@-vOkUQo-1ZS02A|kD*rQ&-{8dk&e{2nbsfu_w!*+TZhGr4zU!oovtC%B@P1s zVi~^YQA(x_LX_J+7KrjWmzz>{i)C2C@ln5tMaX`W9eV@k!6IDPs%1ak z618K~;oTdPqO{kUH>N|Y$u#RUCNAoz`Zk)5N1Z``1p@teDdq0{d;0sSdmqz(-gkXP z?)(PVpS%&1)}H0Ra_2v`+#v_{%VUJg@iWrLwZv@}qCsegP!tm7ucnL#=xb5O%cY3TWkfBzoUezpT2ib+(qFYyq)J%!06!4-lcd} z_{opQt(@(|C}r|j$7{S@MJ3UyFzapG#3Y(Kbf%G+EP5SN6*J8xLTKcp8)T9Vn0ckgu-86YO2cSi)UF=9zI$|dAj6`BLNEs5Ob1+d zlcX6

a;kN=ot6_ir}1Dy~w-e``-!+v*8l*O8Sp6^Jke8WdzNPgqvB+d77tB>fD zJARSoo-Hpo)SuTRjo~5Xt7X}w4HRF~eag+I9c9|x3_TB9q+-5$nzXD@OhMyN`jVBp z44;-v{5<7$XILZ2)5Zg@CI7gYmASt01NuNX)bnsU*bj@z(8sNvXcbf*J0 zbf-s=*e33d{(RUT&08y~!>7s({iuJsffP0Ei-ym65`0Veh{O{kHT)E32;QsToMHhE11SWsj|0RzMgpt$_T|K zOx>=!jm%`p*K_^A@EkRU^1R3jN3IO2I!(zF<>P~yhaLfcF3Oj=!}0VJ zE^{Dd>)O_ah(5KLcti5agZSjZH%TJk7=r`PjSWsDL7Izct^5k(yCk~+etBMwB?y5@ zvCPLSTYY=;#;Zqnalw69QqV`w=Y)Bk5o{>pieE05vaR;A+?U&>DnYtnJyX;D0T{M1 z{MDwk#{DdZmy5r}M(^yfIU5aoaL@WJkY&svQhVaBYZUx2Eqd))vsewx_$z0ZK0nK? zn#}vla{mb#yUJ0oWvhWU+q(?KU{$THX{7o==rSlIxf>)ZA&%vG)ZetC;)QvmB#G6M za~Gf9tJ_wzxfCACX7SZ*TSksw^n+aU@_$PaN}3s$5{m2*OFHV-#D>_)!xKl%7Q|I$ znJuY>QlSuyLfe9mlN2F2FQMvD!4()1LLOoXfknkNu=J^BU|(fIVY1rrIA2Zxj^!!2rrwYudyc)g358W=Ouz z@wnd{&*t-Sb5&XS`BGdw>Yl;aIvU??RFr3#TKx@c=rI=8uYPUtLhVlf_OHp}`b|1? zdUNUENDERW%HwMEZ7Yji3*7R&_-e659=5Jah_%}lZ!G7tT!fA7le=2Oh=>&MzKz)yTJu+WJsh}JR`i&|fs}q`5(OEBun^nn7$Yovf^b%J zam#6f#g&~kZX`v~6j|KjL;dL!v}@yPiz-gJfMut0MdW3D{L{))cU-0{<|pH6C@0v` zrcYjfs87ayGS2-&8|A75iAI@bG>-p@F;<#ij3smnghR?tnH`}>zo&6>&t`ld`iYVo z#qkZ)4y-gR3$z#mDi3zO=FHeG3V14N>Q?GBw81Y zWn5fNfW^5yQ#;ujc>x_x<4Aronc9U1#)p|Te{}3unD*4B8cGDQOe{tkzT9_eFi-Xu zl{SrUTm}ANxF&OPsz#L1Q%eT3Obj+YWr3j1$0(S6&v8t+SytJJ6|>B^^!oX7YH_Vm zW@;|1V)Sf$UZ!E@o!zJa^=6ZKk4#>T=2Y~wE-8x@7DlS`ahD7kz20;U9{eAd67AkE z>F=lRDGLVX*NUw9MWc-v$MT67tK5>+ASRBHEBF=jtK1yf1bq;j8b$O9G-u^N$PkHI z42-JZq%4@0sP_C#j(Pi6N{;a%HWvG%VjaPAl^Uss#CI(>Lwu^SZNBI12NZdeYS1|$ zU;BZt7rua6rpnQ7b5&Mfwj)yZO8gD8#_?f;|CC!N+DtaD{uKfBP%q8=IL(~W3o8Ol zPB4vMsoA9+8a`QGlgKP91+;njExmK(3SKs68L8nhh!x61^yo*)>ub zp7;SY=50lh*=RY)F_N@S3&?llA%x@~dX~*_Qo7xcPXER@=@NHKUE)hKBvPYEN5F1Q z$?{eXYLmOz89EYWJdg_ma`~zh2G+FnfM91y2Z<+URH_!_)Q@bxOeF|%}(fXzU zaZt0|KvnREBgrNI*amFbMy*N*9VF4^lo%8dV$_~=A(4jdSz;tIIK;n0P3!!N1C{x3*3Z;)(Z9Vs0c3>feM-ZIPYlO?nbdcAggzh!$5;_%R@v1l&ji}V!3j+Yfpw=|g8B%>TN<+J+Q>#O zjrgANUL5@hW(FaK2f)8O?yiVzl9HQI5)8|gNvDEV5doG`g}}r}o5|o)Y!pJ|$vm>B zFr?$1koW*YLyTFxCK;xZNl&kO2?`ivSJqh=8HWUs|EbT&X%nWN ztF7y2UFzP)Vsl|49ZyXm0`L6!@B44R%Km2q&%8Zvwp(PqZw#g*5Yus|Pf_SL@hrp9 zVAAVPhUh;l{^q||3bK3urT%{E-hZY4yzlyog!#K%e}c!5>b2F)l2aaP*@{pyGy>nW zbFY=$LLK#=jk6TiwC;=!K(b>f0-|wiJhw1wLt!#Y!JMJ&E{A9G2hGEhK>Dyo5$@=dao(NAzDt&RwoQ;dxsx=hNnbTy^ zCX6{JVIit5RX{0gVcH+=j|m)2d#x%-4F9-$h@6`g56M@ecwDi1W)358AwMtTIkX0i z&dxf7e%eL3+L-3ePlY7bqpU8~M#r2vurfO+17>uLC}CY+t8sBsKcWfs7&-DevIVUo@n5Ea;8ZZ{GsY&m#vQp4<@u9rk7YP8m@;1hf z!(q{!kJ}@tP6IKYU@?*dr`ecwp*~S|Yg+VINz=bxuAoQXc?OmGL1_U|a*e3gfMH6h z?w-1u9-~%&;cEgSfA0_dpRYwk{BQ>F)ZkpT zuY51cl{*-dI-LuubwiQrtSewJAXA}~wqA}JZAB_%CCJI{yDJ}~Oo?Vd@&S7_TLfTq ziaf@FT}{Y?Uj%1%7@h~^3K9z{A1YI^6<6LsqeHk0PJ_6l(X-wP7vCq+20(#M z^c02@{gI?W4FUGZnGeFyks1#}p0M(%suASok6{#{dmJ@Na&31rOpX>Ra9dhz)FG-A zt$ZZJ1>zL7liC91$w~5IP=s=J!RRIV;tWVn;DmXdq#gk!Emws-mP+=eWk-{D4|Z|8 zDlS}WhF}YIZVUiq-iND(moR7+pUS7|rhvXP&7zfRr+~=h2W47yt`+BB70D+4UAKdA zB?YE-Eo?fi|NN2X%hEu#%JQg!$C=)=>0oSihO^O}`hnmBByF3_C^Xt14`FCi`K#L< z51MPT;48;XJQ)9H{r%Lv|3&|S5Vb68{xR2|{1Ig02>D9-Y-QMQC!-l(Ca@bZ78ceC z6p-JCD>8_N*Nmf7`OTo?e3J8D>s6YN5bUdHkrQaolwHsEyS6ndGaCvBJFSNLi+JDU zc+A@roE^>3{JI`v6<17$!(0Z*GG3NH$wx`*1yjLMDzt133^fYp4H(!><2E46wniH-WJ;W9C#(Rpezh<~S>-r@RLzV)YmD&o}X zj?@fZzFY_)HKQDo1gXLVWb8RcwC${~;rTq^#K$^a-5|)c(ez( z3XL&k-AZJsO$S_a-X2VQMYGWv_R*e4WLg?_NclEV^{9TQHG$Hy0#AR{zfx+?|CglK zQ}_OF{b!9}`7N$z$s|+eu`l`fW>vPZ?RzobzT}}asbR2nYK~T>Q(E4 z>UpFrTa6^C5M1?;(l%V)0oT1ye|c}bKfXrX(3`$qUuCUrUsDST1!_>`1#@p2FmN_s z0?%=1rwSzGNKz)DV7)%B;0WqFJHL{Qz`pyt{lhI^vVV9P2&sJ<3znPI4iWg(3?2&G zrt4)giH3WWtz#ss+N&QcGsObH->m7^4ltl0s_fZe!f6$9%^;6{tnj4kq%$e#w=7WEL_U#U9bd z*3)Zlld6F+iD7kI^}dCmg>*XUC6t;`3<3D?$32sA_*Tvl+;}w|_bjRqNw!JcOuDl< z>9-ZLLV1eqN-#8v2n$@EFru^bc# zeD&vj7hiyQrZb9CY!{TTi5$1289JVBY5itLtgQ&0uK-$mWYU;@Z49@2Wmw{m9sn`bBM5dCUN^!l1x$cox`g9{dMYs5rx45QL%Niq`ZIJ;A_&!QK&b=z|C<_(%$2)IRbI5kVgj>Pw#`7y0 zLm0R`(lL6Yc;D@$P1t>KyxozfEE+a>+(X_M3%+F67GcvOOBaMO@aZU->@@C*o_)eB zr-Yj#|Fu!m+^b)U==>}98V@E1s=4=*?wdScZuN3w@-kKgt;Vv_TMz~fuH5tLU{a@O z0pFKmKudMXJ{m8+VoSYdI5Gn5g%q|ZEC!?)Q5GZ)U>qNYmhP! zjFK}>)qHHSxU#!%@>&EBDDxc7n`8=fUaT%yLY=AdVGny$ZJ1D~dpH_RhOK7NZjL%D zD$LiPU8el+CR3i|(PN;mF>A3#*sPe3F=C|gKy8-|BlWRyIB%fTG^lOF{WnDlQ86o@ zld2(xNm_=}aP!NgXx#%FvndvOl(LuEgz6wVlzV%+cX+~yh0?(?fn9;{36U3(BNST}@RojE8?eX664)^B(V&%-UmzTtm=md(vhvCfJSER9FUjC^G;xm<;74AnW zFN!)ja)_V^7I3l-0<#Le%}z@8jn2v+fn390p^^(05??hFx_^r(Qiw3vrIv@z`pB^? z$}s-{e4idYwDICzDUo|t@iPsmRJ)_%t^_!5gZT{LCAWfFW)jRdVd=U)|&;+vG9MCK44=>=(Qbkb~bH<2oB2}~L4VQDhLGSiB)uiS)b<57J zMyv(M0yswJP*+I-YS<+S(yD`R9SbV3nZLqCc_5!-DaZyAoAjZtV3H<)m^mF>2=_+) z1F00a^ce}VXvx_tc$Bn%K^I~z{*cC+qf(}%h<=u_aXH5!8oirq6Nmt>h)t}9qG&MOp_wUFF$Hy4{~;ubThlNkv?BkK;8}L7g1dcgP@s$yMQD38Lag05 zl1s{<4?(5)d36{(Gm}fepZ@VmjCvkMeoDHMT)EIn_+T)OBH3C-41Qt{A5Rp{^f4v4 zW8aX}F;9`Y$ULN zTwGl(RqIV!OncOY!84sJvKpB(DB8pnHQSB;WKQQM{n4zq%J2D>v&$5DIpdta$A>98 zh0E$JnS97i3ayanWUen%8%G+ZOLd#a^6=O8y(zL*!<_D6gG)Bprqe>$I~>o{VD#RZ zrW~sx4wq~@Jf}BH$%LGAo3W9LxijGhc)Qw4lY?KN<6c;Jqt8Yx0sMIOZUTCOP z^vp2T!R%pPrM=4sB8ZXzzAOVa_Dh|ecsAV&^|=qH;qd8$dB^i7NNb)k0K?XVsdX9` zCLi<~bCcVGaS1>~2%7R5F4iSHlPf4Fpa` z#_#q_X)M}sQxGe3HR+uJ3t-OCWUw2fQJ6-?zDp<pu^-j|3RS^*#Kz{vEETH8(vwaH2)fMY ztUF^aP~GO4A+-iVO5kinbZ=PA(wZvBxjd064`A)`wTqXp*xC&@)eL%Da^S|e?c9t< z!UzZ27!l3MNY&H%3@7)HNUeEm(il&NfVjp%DaAGjB?;ojjxLtHYf#BF#ibcHR0I_}=R z>-I~~IY8>hN?4K>jV*9_OgQ>T#iXztD^B;KSGk0UXH_oCqQq?Xg9XcSKlv}Sg;`x! z&(&YTrXu>x^ehPg?HkWM?t-QiXF=g}WJLZV#-vA08zF>R?iw0N8LT(2`xw)E${Dwm zz_au#S>8+)s8GsyPb$i{hivQ6&efm2&`Uy5r$*aNdms&(bv+D_#Q+^<8)_@tZmZM9 zW*Ls#Z2cDI%A`<`=CA|xWilL%XYCbrqIKr2|LEPbb@2~IE}eOA-k;B!O%iUk+j#0H zy~b#+vF|tA?P)RZk2|ZN)36-ub%{&Ya*GzT`-H?M;UJ1pOPldpc`f9keGJR!dt&O+ zdw=@C?^6ITni(8+lE5i+o@3FaQM4-T zgp@|+RlriI2NSoJJ_7#}>e=^6mazB9(tM1jtIZ~z1?LR{@fLWn62{>naGyOwfy*K4 z)LcaO1m`8SVZ7Lx9FC7SFmG^IAd!UAHAn_B>5^OXSOCwXwCs5zNUk*DVn#siG%YUSv ze3tC{*;C9M=_)& z7bK5Y9hSGoYNuM0P@F_v;s7e(3!Pr&jr{KI`LKsx+U&Hb;eaSYIQ1KYCIuYEc=6`F zRc^pbXBS1+%{YC+2hb+(6Q8jhT!;asqOd@?{8+Y84}M@KP5xmXQ|0NK>}-r77K!#T z)5JzZlL(NDJqKYn+!ApuJM!4Dh1I3jmS1bvg}TS4$`E0rG%0QrRSlYI)hqf{8h_b2 zxyA$(=q=$;f+bEwk+Hcs2qPl!Xa^`yIS60M3NP3|OMuLHAEkE$d6En`iJc?-m<8uO zRWTLpSFy>?Xr%qYuz-e>R;z^q(HJ%Rn46Q1vWU%to!-?>zrKvWPy9{5VU)wbEm( zjiL_fWIc69GBP@=MMh|Lq zqf0(S5{k}9&7d;aqiL7)>Z`EMzgY^kd$;xXQ}!u*>L9-f=)?=S9LcAr1>^c9~sj56M zhH@_EA7uZ?nwByJV@uOTL|L&_Qy;*H){O6ueyXmrp~O-Dv{T#tW;6QXj0i#$|Hp&6o@g2A1%F9Y>5@@Z;$tT|&7XTp}%(gr+sj z4=Qxdtm9BTx$;tdA7T&nC+#_|pH90wUXAp+b9O=V+dTrz;)$7&C@CH)x-_1u(vM0HkclJqbQ#=ZDu9Uxyfc@bkIHeA zddL0-l_!m!7lkUhmudbl)%N!Gw~X9WyP!WKFyQ|>F7`RGjCW)UE@DRM4Pc*o%JZgd z8O{@)v`i{n4*B>cF=Bk1{ImxK+PaRePH-yEogB}IOZ3u>DJ6MsytnCdH^dQlY9|a{wiv zHzVAzK+h@K{mHaDp7qDARg-OgcA@i65jyXF`n@_bl^~-qU*u&);e;;=`W8yVq|=wp zZg@ga&81DtiEAK12w=9uGUuVnTY(J`tXazz_Leu_&{2rMw9++}#~z%48jtII`OE~==Pa7Q@PmfgeieundRVe+xKW0nP4=?_B9J>(o6(u zmGOEl#vsVsbb)XR^1D*KZ6Q70vLykHIUQZ~I!Q1KiNSS~tmLW6Wp<6xkL)h78OjLa$aKg| zKZItWnx2l7B?zqy15wJ|kk-(vswP)c{6a_*iIP+$d3kDEDc`(21H?+SOhh?H$g^{> zG~_qCoW{+0e>54j8q+pa!MoETSyqRWX=B(PH%T7dYc42~1Q@PcrOiw4JWohQ3bS}63Qonr_3v%Ozk z(c~Ca=tRJ7W3YzJvt(X|pq-VL@&^O)B+vPvJz+%;CnF)AO%gMVC_&xoP6oZscr>1O zSJMw2m&^Lmvu8v+6Il?OBMB0y#AHHGf*_YMi^pY{`H9H=xc|!U{+5u=f6ht0zNwmc zSq&@poW0x=d^-VlZ6bi+3BM3Xk~&?S6}ZaO%#uAMs4IByvp4#?;?r*wlhP}|fxJI9 z00C40f}CWqr92lju*xHeB6oRh=g?=!TwV@*(jqMsmJNsMmw-e%V8>^qw?5!#gMnar zVd8g`!}}^ogJJ+`Z)XDjZOaL9f`-B5<>!TXGn!ArpLPiiCkpWG+S`prqw;?=70Iq| z=S-Wo>t)&R`ghbnbR^Xnqv`GX=l>f1K#dix<^0#vA76S#sUB!2qxCGkjL2$L1U_7S zU8G689eE&+c~X6q&zja43Jsgpw9g(_{4cr@Y7k$!{Nk0zuiPVSS)C6wl*hkbO{DYm zZ7Fq?>=|ip*Rw_9)F99?7b5y$0DN4~X{Gr(Us;>cH5F47t;;%MMhl+m9eF)+uJa96 z)goNjs%1ao3OsqR36`LE?YP_P3>qB-8igJP(1-iec4OWnqLM7fqvmimB;g~ah`aYt ze?N8a!}`zruCK_Q-$3rf(a9El;@rG9M@%|rMp+7TbN=Qbr}fKY4$k9eq>PKzag%=( z$=*W^O{=K<)g1akGmCb}Ekp!f$!&|FOshPC9k22B;>{^;1KCm<3(Hup zH|)@vMrN;=6H$dAY?jJ_tVG#o17^3CZjvM1C)}X%{sPNIVOPcSM zhVz7vU*rmA%gaet&qER=>HouAabR1J>qEjdFgbF0BbgH%bMVY8s2$5_l za@xX4dJca@0U1pU+aRj)Z**n|n+OXZ(i$Fe?lZ2Hc2?2^%P|a*NMEszcCu`n{i0PA zREcU#=5n1jTkuQ?S8tHhu0Ni1htqX@kUv{)=trGXtfO&Ivo9Jl*Dy@Z0$vf{vWYHU zw;Yh$#{G%w@iEkoFC>HLuYS`n${_kDh>JDtoDvwH7)EwLNkAWYzgR+9Nc`G#Eys=& zS87wc4_kS|gIAR-AYj(wE3oBEag=Qnf{b$!e*rx5^~1f_b(AIK(uOEt#R;7>JTZ2T ze`tRSJOD;8JzF*uQPm*vvD)|D!|~oC512{jn(YatN#pnm-y`(gOF}46lE#EOE<ub8&pr^~)ICNs6xl*K0HudhQG^)7kY{;-2uvKap}!%m9L^lO&A*4WXW%kRi7w9JQCT#$A2~P5OoZ;MO&J&D zc*V-`i5Odk3R||qsXXcW^y zb3U822JOypC3fnaQh43_r2c;D-k;NdNL#-wUVazx0?g|G^b?OB{OHTa!Z?1+bfBY# zuO7B^wt2!~P`eH*j zz_=&(jXejuK#5|_)G+yBOZ8ICJKp#Ze$UF*B(ID;Ko^Lls3z#QM%JB8DEI!=*Sxet z;5{bSjTj!{?8Ry;)(1-^pJDpuMp{2?8b0T;@8;B!sOEA#|x6yZCT{*TpGf zIZaD?nsqG7aC0uLm0w|emt>bG_6T7@Nvq7qE6UIsl@#WUSC8)EeEYD3sE=M;OxrVj zU84N@<#H+8Y7g{(Jto<2ISCfDNQ&hLoX`im0AKo1Dx)35K_OUpHp7j6*?x04>~;)m z>7hDxNjXEN`xezUJF`K1G@5tDtHAQ7&Mtj^7U`qOq=KJk28(k42^qVJuB0PC18o*b zBC?j)hcqU|jD*ax3a^!xa+taF$^tvC26;W+s9{xBZen?I zGr1Jtdaig6X~fx-}rlav8M>BcbY{#tG;v z7!5R#6YO2+E>c<4!LW{{pnM5c9e6Akn3V)CDc_43m&_G!i<6upUA_`~yz@stx%0$$rkG!E&*p;14y)xC<2s2A-N@C>w3e%q2R6{pnCB;l8M)o~$ zyfv6bAZlrw#-{O&tH3`Dr(`ZmBNbmu2AfR8H9loQdf0r7g3b3_vdPqL>QlugGYY+a zzMNWIYmu3b3+BhtXM3!4cBA$)CviomIVC{3JEsz2f8K8lTk}z)IcZT=d^G8_=c`oM zzfy{`d!Nu7q5{^_$}Z^Po$kxXBrB|H5eVb^jqY77BaCG$OUAbV#jhE%tIV#HSkDR=Obj z0WIDsT35xnc0EEbd;y(I*>K$8s;s_jM>qpX78i~aW#r{Q-o#99w6_FY( zk7?GXFhZ@3m4n*kE=GlpM1f>5Bb13`MG6C0?qT1;&xj|Ls^>U6Gb&Y!<3>6a`7vrG z!X9#L(rbB zO%kIy?azpB>o%IRCN-<)^A;!WWZ0NbTFp_b)0%bHm<@lUlo+R`EJ_ZVsab}Q)+%gM z+zz5 zX2pD&0dfo**>eh|aMfeZ#6YqRQEq9ES&$v2mMjD~bi5peRE*TdkGKf0rK!^{e21FW zKjT1Umcsg(dM>)Pmq!4GHy}qG0(od8AnunnHNu&%geHc!t<;cI&QJhLB1x4t0Uz#k zN9n2eVlJl8q_9_a{Unf&@VliUtBkgNeHghk;#T5=CbjZ{=0sTG0l+V=avyLzZ8Cpd zN#8D4CN0G82?JK@a@ZGXGYR6NdI523O5eVx(4*s>{U{nNHKg&mgF#TqZ9@JT8_ML& zUT0op{Y2h-p3=}G#oGA@aRAkvCXoHK$aU8Bvo2kawADmcNT&$$+R# z4Sgbpj?{-(oLJ1!kx*fVWM88yAH=w6!(Kzzk10q^(+iUmjR5XUwa0FgLU%lC+)`G` z_adnx7btkSmiZL8_D|bXpiVk z^xnm0cTk~EJZx>OY%Waga8Y#)xS=*-Zk z0YDpkZt7(ExJ*`5to^>9pM)t)__h zugewmX!tnvJzM&Xupdk!N|ElKx|ANHN`K*NLb98^U+n#*2*ID$n1v$7?P*D)5O~Rt zoa+wKnbV%FI-z1d4hvGt-;{roDOT0zGJLy?!jqhktrK)Gz+{9+%I_f1QJ@s6-X0ou zsI8JK-Rh^roG4V8>YMzBsPmQYMc4A$z)`+eq|~~hDDeT=&Cb5DniDRBQJrZLcofS( zUP4C(@-fO3X9gr6uvc9u+9CjyQ?Lm%glec3IV@;B49|m7c&cVwK2)YwD}KCz7KdcF zc*#pPdR`#|Abj78+^C`zjs)&yOhd#?^gdFWB?QtVWIhNV$NZBob3j|NGCEg|zJU^< zdz>>fctv)1gcm%K2JJ-6qDsihM?%aWj!-+MEg+uUBT&h**#(rB*o!kDC4uARb+S?f zSTyUG9!te}(z2t;y9bt}5U#k^42*ttZj77Q+znTaZSm|4P$@!B@q#!}o zS5YA+Fr0U)D}!zAxg^1?(`u-{h`UXm#=K48*U|LMuj?^ZaV_u5VJ?FNbT7*vW#EDD zhdml(8|!hZaD<~n5h&Rs_))ctlSGXIH|l?6Pz$SdstgQ{G}x*Rpa$Lvvd@e8SL&#y zAuE3bK5Mm}OLXna64Q^b(|J~E!0SbMdVL*ho4iB_%6tHjhmswMFPjMro|K&TtP#l1 zJr{{m&le<$)ng2j9}lH4mF!>Ep1&xaS28d6;(f`g-j@dKZW1otes|hwk><1Ljlmsf zI4Q=n=CIM4GzXLEXfW$c1}nnl*Uv6oes-k_-am-Csp|{?~H3JbLC7{W_-kyf}-=(WOvtl{pVyMTl6fuE(g-55Fm@ z(*MtY_H|OFKg;p9+6;ATUf;dT_BX{f1P#Vcm>?+C+ZEENS=&n@wzlRSx6z4{$RqAQ zqv9P>XJ&>3 zZmCU%BLU|h^cA}EI%T40mm?qz8!ad8!P8Xpx=S%*HL0pOI+VtrblC^5?-ER)Q$(jC zNun4Z4lWWrSOsQ(+0?P15(MG}ArJNoCnOCrN3yY0tip>19mjqd(d6?M$SNwT(lFMG z*QDelth&976}!>0O6U#N9pFP0c?D|NG#(Hd{!Ei#=B(gem z$;8$jjyn`kAp_*R+3HWmlTp7pTP^zdZYg>0{ignY>fZmR|Ge+|ikSGEX~9qQ@G<>H zxQG@@xi8p*KCLUWp&cWg2u(|_XBJ>NKePw5e0XwqM|JqhJ$Na3ghOJ-Izdj*nEd%5 zT(sM#Ztr%i&$t)79x)r^V_%lwIopJ-j$f~)qhjA zdYn97gH4&`eQO{D7e%D%X_tod{b{92>D1nlA0-Afz#dCSgv7M4iG2e)30fq*EKAn3 z-(c?|yQRPj`Q@z?uh-xaO2t+xr=yG#QZ7|-$+Y;f=`}y6XLF5~_YFGCA!x5v=5SJD zV&bavUQnK+%w{Vyo5djre99pEOa6n9db~cR{JoqkWlz+(K-9v^vN^EC+d9t#qlYs^ z=L?6C1aUYTj^8-6f|~klkt^W{UM9>})j`Tu7?|;ds@z9jdlPghNwF}P3*~ncrJ4}9 zlYPq<=~b}G1b|wpK@{fa$|hB}Gw%$hs<}ixAS6f^2{IlK2-%qzz2S5|?oN8m6;=0l z&MrZeKEc-E6F6?HR^K)@?0$yhG(ITy{HXAaR37FO3}TTtZg59sX0KpmPA&DTHUWvz zOv#d=EsY}XBX5mx#v`l0Zky#6slksMdZjN5-M`q?j^mQ7N`Gi_gdbkZOA8?MGwW7% zhL>x_0I9G+=aJp!N4%k0B;wb0y;>gY;nZi7@htW+MX}!~EEgRL&*6h~^2?|V=ce&# zb*R(Q4c8P>D=$FoQ?`-vaL=$OOJA6k>j+1X_fMxc6*vwLP!Ga!jVeIhG<-j*zri3y z(Oya
R;78)TXC{wPmOmQ@zGjRm66PQOYTkctZJusSYJV_o{!M?k*{VHA@IG5>n z-f>zm_02?^Unvczr@mRt>ADinS-l0m2gj&$y8VYFJY4667JVX`167vvS{eQy54xtGHA$Lp zk~Bpz92DaFO**3{1*(V8|60_Jh5iRzM@_PEv1(NR?%5^HKQvwCJwHS&jvV5-Z7Tb< zEEM*tuqU7Spo1v1HzaXqSG(hp44b4))FNPAIJ*Ig!dT}XI;WIOrFljJ57~-Kojhsm z5*C@(Zfn%CTSruz2wt`zsi%3E#U+8IzxCWYJ^p6YaoJ~dLf^Ru4rM?+>&^6E&RZ)U z9)gm+v2gl=Sg~rAJCd&4C=I$ym(D8^Wu_b3ijk;KFk9}!0=Bc!7lN-(0pD3|ANvB| zH1GuvO3H9>9o2X z9paoB(JTxSPAb64NL%lrPT5@ZJd>@DcY`K$AdkH+wK8Ur`8_ZXwk9$2ifP2*@yArv zNGni4Cfv#=tx@`-YXFnCJg=13zuFI)&6QQV;WFMG6<7-vSV|y}MlMZ5?C<6T_`~_2 zIUZ4>XEtpT2;H7o7Ka&msrx-j$j%1C=Bg*)5C35do^SZbJLh-4@o_eg%a!-?=rGF3 z|3m@;j#0^lQWqvK8+vtc4Mnd{upxNXcM1#_vU0`rszJOb(?jXR6*l6kv=b|gvch3N zRr;jI2D(&OhBZ$$We7fNiFXAVA+Um!>wwUhO@Ti_CTk9E)Z_cdhm;LueeNF;5$T!_ zAH>trtngDKXs>*9dA;PbA=!IUi`uN*%`EFS}+=b6kn`K&a=%j)i7dKQ9#^7t0Q^^Bn*t z&(Jvz<+5m+^c;G3*-#U?4d2*FRK1Kz6Ux3?e+m^)L~T_ME@N(5P${quIqs2+!mYyL zqNBpTkgy>qA&tZA%LMu%!X=I^L;%ToByWN#j0_CJ7b`V}3}dV`lC*i>rT2uwkXc$M zkqk?QcNM+EK*s3fU<*aP(FZL%2Z2 zXh04eQpD!2F=)d(eH&w}u*$>xU7Dz$nR!FYw?a~Kq|oC{ny|m=VE};;Zg*wDlhVBI zi?al`XP73gOy>qiR=&?wbQ-fF(^n)QQMXJqhapWmpzDUEeJ67vSJ5M2w5WiTx2c0j z^TEQq>fO{3%&q#MO$kf~C89CO;OLd@pZVD8k4bj;$VtC+Ly92_PGbleC?b}~H$uMl za2WXfvrUqC)Tv&D5#>jNjUJse<@}(V?PK9Am?WJgBAFupt)rGlCwMH%%Gt?8l4#v%_H>4WQD-v4 z&Ot}PIYiO1ZkuAJlj*!an@(HP;c5lJfBo!YHIobD%6)h_1Q+o~|h!Mo>Kp%CiqW95tpvi`N3FNcY={AixcIv(xmDVC9C zn6^~z=;Y+Z+%`XeA*bmy_pO0oVS(%`b&=>>dF*UBw0NGbS;mb>?Ui(~%o`+mOZ7zg zDwaRrN=E;9swU&ifBHfZnMB9cak45iqkHDQCvFq*Vj1dmBDCZhEq zltIPe3EeQYAw3-Cfh~q8!>YoHi!fCb7alLXVx8!-V;8EnfChkO+yP`FrVM-F*j9Ex zR6f&^7LddD?w9tKKGD77E^gG6#7>z%EPSQJ!(=RQDj5-a)>?C&C{u$BxW+_{LzDw+ zd(!Jd}*~O12~+J6Gnq604|y9hvbH8%v)2Hdmr{j^VMR>-+XrY^50u5@{>M- zHk(OH%)K@xE*Esi7!4ke*`O+>Fm{^C!HL|dXmcBuuxp;4G%Ub+<-w(uVTBm-#Fo!V zZ8uda%0t!JF7a18qLWlK8*E9FHt2I25=S1d5sgw6ETN9{Z3mLIOoBBnSJ$>S1JTVX_9FdR)`=?j*{wmI#~bxfx4#Hn}(Pwp@LtAX%w z@J(-hg9snLfSPbMw2xSS!CdF4akY?OeEZtP>o2p~ya%k${X;mUiFapbuLYEZO#lj}fLi1iB}Twmxu}=%p8Y_Q{WUvdB4N3H&As5Oq8Ut zHqAK|5+J&7TWF@AvJ|(7|x~InO)<1HecA0dCQY0-Xdk$>Fd! zQ_BFO0UUtU?Xf<7e>S~Q#gW)%94{`#G7$_A=fY1(&l-{OQ)XF!EWVcqh$+0zQb+I8raKz%hMhaX zla5v$MXGjI*q(M{QV$UvG0i$fv~M~iSguU1yWV%y6{Bu~$OUBHo^4Pib6PZblL-)( zcLEOVJuYU{9?fyQ7dYl&+Tg$KyU@pKG>ZiUyr9-TAQ>$lGOZZlaCq_+tLWeU_#4t z8uffmgS2gFz;X+on80BLbViap=h^wfm&^AMQOc&R`J#ve^g^qaQAOFfk$N4-$V6jWIsY z`Fm5y9Pm7(P0f~Js+>(vKbZSE5W+eytVRELXvE7s%_Q`ZO zbwoJ#I$3tGgW(GHXo&dqrEo-Yo*F!PAZGA0oR^p%Jk4>LtfCPT9V_akhFjGaOI#AV z#PA*0ZlZz?Y7j3-n$}`Zc~~DH?AS<&g~I*lb+o@*yhv;^*OpARCCjl)k~YjVatCvX zaw3+gl+w9$wl$q@YcxJ2>l889egz5)!|St-`>Qzu;e!bi&F}sgesKoqc zF(rn2%8+{a+}K?OO@LMb!e-@!3?k0Y8+C31!_ze~zqPSPyXuLC4Pmjw(z^N25uG)i(5l`xX zrM(QlWDoD*2h$(N(V$eUwairUGhwyBS;U~%>4Vcg;u}>4(64wb-!cGgX9V}a7wfhd zpQh}dYjLCJbQf)7;W)XzMfnrMSn^;7*y!y>10S+HOh}ueM;b-xRPExhgoXM75=R2Q zqw!!PvpEyAZ~q-|7%NV<_=Abt`&<{Q&CFYn;3sR)pm1?eIs#X_N!VPU|C{)D+8u># zYCUez=Aog%mY$(Od?h^axYUh$6gJgT^#CS_KN}kAC^($M46Kjy?NtQYc7gKkvS2tF zYGsXre@1KLvj~J4fpWh%JBIj8*AmEwVUN!GNs7W#420b9;dBzcsas#F{-W6^=(Zfl zIZK^jH7IO21fUz7erKXKF}4(ArI2A!E~9EfwiOkv(`7hZa`{3kmVgt6q4bDxZS>}D z3o+KpM#j1dRJ1rEb#&+U<^`;N!51-JMUGTV9Ygj9j!M^Lsvf>O(2ayz-c=NiKqZC3Ga$4kiA3;*H#xSz2#RY^~dUdwq*2 zR!*Lm)ob{u=~+7y`!nI6(9{{u@L*`AmgteYr4L!pEGyed;w8a)!3mXt$P^n08Y<#Y zG*yOqlR0r1m?|7LO1D#}sKL{20UDiMto}L*s?0bG7&bo@Bke&Rk7&%E|Dt3Uw>kn7 zsJ@StYk1cI)NzXR_HuYcf;JB+_k-HD$Z%Od%co z_bR!Su%@?Rhd+oN{(7ESS0`4pJ1T(ov@PkpBo*sO;PRwPL}8mXS=xifek z4wQA2Tji1qoPlIkgwyb>V^1JRo~Cy@$|F3wV8aluOk%ZSJ6{@pgr&~SsYe~Ge{z^x zv)&A*T_pizn6~bXBla2OLQkAOZT#L2MqQx34-Pbtils2S!OTHnYdcrrQvbS`ty|5# z9^$j8UG{#D8f62!+Yk$e^@lURhL}&Hk_uk4ex$MzC8!nRG=e~NViR=R9%m>I=bkPt zT6M5}_myCrs^s^xB9R(5=Goc;cVVPXL27;KL!c~4!EZo|_fU2Zdn&F9!`lHNuhh5f zLW2g3E*J|M!thL7C%%G0_No&U>b=9uO(M0AN`B^ldN?PRB-}@6N(laf36K2a(+-SQ zD16%qXPfR$o5`3gr*|-!1WZGNX3rMev<#Q=?2Tvh_9T}SJ=TpdJvT0OENjBDDz4OW` zfFTpu*Og{f@-Fz%XBgqQtv=y>AWWb1uF3I_13=!#> zST>hLfJUOy-d1VMqPy#gX=Luy>v&!WTxjQ5sq;6hZZG=zsa;NC*+cSIAO_Ux2!ory zbX8IGDLfHRY>doM3qWe<&dx;d)`%W7ppRhAu;$B4nvhy^IkbsRrDCaa1@SAYaAn)m z(7LjP6vFw@g{aZLyt_U)bDJ6uWMCVw1tM!?oXIV^>+lS#=5FlP&nySHKXfecZJA^4 zDE60?v!tS0~nkcZp;Daminr1fqtc{XK za1Hb<{hoW0cw`5E0U{Mw^E#9jguaZ6DTS-GrPf#`lPc%S7`H+i=D2jRl8F`anNlo~ zEM+T6R0m8X8jbb&&@irBWEgpCcF0{dx6Y5$fbIqr9*(2w1tZtnbd5iB<7-~~J5RAq zC`8ckt;jF}8nBB!=xAJM-~kTJENK|;Ds=G8Ko9i?7y%({x1;_t%;^~Nf*1#mD~se1 z7p7;QC?4szO$$*K!d)SRIA8eqs^<19XhGgAe`i<@QM3UiIO z-n_xvWR`m|^F}XiuJvv)Z}PUA6`nC~_6E&cyger370g?`QFEO)Y2M}?G)>-N^LFoH z^A7L0x!!x+tn{8X@ARHEtGwsTYVUco#`~3dm-kz9gZF#$ZtqX#J>H+qd%eGzwccxH zo%av(KJQ;U!WCg*KCPGzK6GYY$Mr8wGpKKwFNs>^L9AR8K2ousx_@wj_CS@04 zO7;?_#UsqfIAOo+C(O$Igah&r;h;Q9xKB?w7NKL&D?8DVmpIV;RXG2a)4j+-9{LqE-P!U!$%Lt!w9`H|3TYt9Kh mGv>!abEo-<(3W9-Dm>4ap9%F5^K+r(HO~tr$IQ`_@&5wUJQMf; diff --git a/noveum_customer_support_bt/processed_datasets/agent.llm-rag_dataset_processed_dataset.csv b/noveum_customer_support_bt/processed_datasets/agent.llm-rag_dataset_processed_dataset.csv deleted file mode 100644 index 92b1c61..0000000 --- a/noveum_customer_support_bt/processed_datasets/agent.llm-rag_dataset_processed_dataset.csv +++ /dev/null @@ -1,13 +0,0 @@ -user_id,task_id,turn_id,ground_truth,expected_tool_call,agent_name,agent_role,agent_task,system_prompt,agent_response,trace,tools_available,tool_calls,parameters_passed,tool_call_results,retrieval_query,retrieved_context,exit_status,agent_exit,metadata -,f1f37bd7-0851-4659-b493-b80d3800d920,ea4d61c5-b8dd-4677-b21b-2b17170e2163,,,agent,agent,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, -,52aacb67-c361-4445-9b72-c157f79f47d6,7505f97a-cbc0-42fe-8a72-95387d723daa,,,agent,agent,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, -,2218f641-604c-491a-9710-b51a9941b982,8a6e18a1-babc-4166-926f-188f344d514e,,,agent,agent,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, -,255fd49c-84b4-4b18-887e-6308a412d535,5bec4165-6596-43ac-930b-acfcde1cc02a,,,agent,agent,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, -,dc511122-c0b6-415c-9a49-c7b45132dd87,f9f79a07-4eda-41b8-9634-d4460114caad,,,agent,agent,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, -,04bebf38-a343-4563-80db-0154bef8d927,edc52a5d-8450-4d0f-ad14-958ec4d147f4,,,agent,agent,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, -,5e043630-6493-42b5-beb8-79faa19bfa37,6c362c73-e063-4813-8180-d26b6a52b798,,,agent,agent,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, -,7da9814d-a2e8-4c4e-b750-68b26bd5fd22,1393d065-a9a4-4475-808f-2d00b3c68939,,,agent,agent,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, -,16143f74-2831-4753-b33d-ce4b645093c5,f419377a-fc24-4abf-8a1c-03857824e374,,,agent,agent,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, -,fc64e6cc-6739-4256-ac4a-7b80c3028233,0e58e31f-a8ab-4aee-8fc2-abcae1e21d9d,,,agent,agent,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, -,b7945c49-f584-4c70-972d-536a805d8a31,48155aad-6204-4a52-8624-8fa5d6fe407f,,,agent,agent,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, -,f5c40ecf-36c0-45ba-9cc9-dc0329b0324b,57ded2eb-0f3f-474e-8704-5ad9974ab4bb,,,agent,agent,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, diff --git a/noveum_customer_support_bt/processed_datasets/agent.llm-rag_dataset_processed_dataset.json b/noveum_customer_support_bt/processed_datasets/agent.llm-rag_dataset_processed_dataset.json deleted file mode 100644 index 31d416f..0000000 --- a/noveum_customer_support_bt/processed_datasets/agent.llm-rag_dataset_processed_dataset.json +++ /dev/null @@ -1,266 +0,0 @@ -[ - { - "user_id": null, - "task_id": "f1f37bd7-0851-4659-b493-b80d3800d920", - "turn_id": "ea4d61c5-b8dd-4677-b21b-2b17170e2163", - "ground_truth": null, - "expected_tool_call": null, - "agent_name": "agent", - "agent_role": "agent", - "agent_task": null, - "system_prompt": "You are a helpful customer support agent", - "agent_response": null, - "trace": null, - "tools_available": [], - "tool_calls": [], - "parameters_passed": {}, - "tool_call_results": [], - "retrieval_query": null, - "retrieved_context": null, - "exit_status": "ok", - "agent_exit": false, - "metadata": null - }, - { - "user_id": null, - "task_id": "52aacb67-c361-4445-9b72-c157f79f47d6", - "turn_id": "7505f97a-cbc0-42fe-8a72-95387d723daa", - "ground_truth": null, - "expected_tool_call": null, - "agent_name": "agent", - "agent_role": "agent", - "agent_task": null, - "system_prompt": "You are a helpful customer support agent", - "agent_response": null, - "trace": null, - "tools_available": [], - "tool_calls": [], - "parameters_passed": {}, - "tool_call_results": [], - "retrieval_query": null, - "retrieved_context": null, - "exit_status": "ok", - "agent_exit": false, - "metadata": null - }, - { - "user_id": null, - "task_id": "2218f641-604c-491a-9710-b51a9941b982", - "turn_id": "8a6e18a1-babc-4166-926f-188f344d514e", - "ground_truth": null, - "expected_tool_call": null, - "agent_name": "agent", - "agent_role": "agent", - "agent_task": null, - "system_prompt": "You are a helpful customer support agent", - "agent_response": null, - "trace": null, - "tools_available": [], - "tool_calls": [], - "parameters_passed": {}, - "tool_call_results": [], - "retrieval_query": null, - "retrieved_context": null, - "exit_status": "ok", - "agent_exit": false, - "metadata": null - }, - { - "user_id": null, - "task_id": "255fd49c-84b4-4b18-887e-6308a412d535", - "turn_id": "5bec4165-6596-43ac-930b-acfcde1cc02a", - "ground_truth": null, - "expected_tool_call": null, - "agent_name": "agent", - "agent_role": "agent", - "agent_task": null, - "system_prompt": "You are a helpful customer support agent", - "agent_response": null, - "trace": null, - "tools_available": [], - "tool_calls": [], - "parameters_passed": {}, - "tool_call_results": [], - "retrieval_query": null, - "retrieved_context": null, - "exit_status": "ok", - "agent_exit": false, - "metadata": null - }, - { - "user_id": null, - "task_id": "dc511122-c0b6-415c-9a49-c7b45132dd87", - "turn_id": "f9f79a07-4eda-41b8-9634-d4460114caad", - "ground_truth": null, - "expected_tool_call": null, - "agent_name": "agent", - "agent_role": "agent", - "agent_task": null, - "system_prompt": "You are a helpful customer support agent", - "agent_response": null, - "trace": null, - "tools_available": [], - "tool_calls": [], - "parameters_passed": {}, - "tool_call_results": [], - "retrieval_query": null, - "retrieved_context": null, - "exit_status": "ok", - "agent_exit": false, - "metadata": null - }, - { - "user_id": null, - "task_id": "04bebf38-a343-4563-80db-0154bef8d927", - "turn_id": "edc52a5d-8450-4d0f-ad14-958ec4d147f4", - "ground_truth": null, - "expected_tool_call": null, - "agent_name": "agent", - "agent_role": "agent", - "agent_task": null, - "system_prompt": "You are a helpful customer support agent", - "agent_response": null, - "trace": null, - "tools_available": [], - "tool_calls": [], - "parameters_passed": {}, - "tool_call_results": [], - "retrieval_query": null, - "retrieved_context": null, - "exit_status": "ok", - "agent_exit": false, - "metadata": null - }, - { - "user_id": null, - "task_id": "5e043630-6493-42b5-beb8-79faa19bfa37", - "turn_id": "6c362c73-e063-4813-8180-d26b6a52b798", - "ground_truth": null, - "expected_tool_call": null, - "agent_name": "agent", - "agent_role": "agent", - "agent_task": null, - "system_prompt": "You are a helpful customer support agent", - "agent_response": null, - "trace": null, - "tools_available": [], - "tool_calls": [], - "parameters_passed": {}, - "tool_call_results": [], - "retrieval_query": null, - "retrieved_context": null, - "exit_status": "ok", - "agent_exit": false, - "metadata": null - }, - { - "user_id": null, - "task_id": "7da9814d-a2e8-4c4e-b750-68b26bd5fd22", - "turn_id": "1393d065-a9a4-4475-808f-2d00b3c68939", - "ground_truth": null, - "expected_tool_call": null, - "agent_name": "agent", - "agent_role": "agent", - "agent_task": null, - "system_prompt": "You are a helpful customer support agent", - "agent_response": null, - "trace": null, - "tools_available": [], - "tool_calls": [], - "parameters_passed": {}, - "tool_call_results": [], - "retrieval_query": null, - "retrieved_context": null, - "exit_status": "ok", - "agent_exit": false, - "metadata": null - }, - { - "user_id": null, - "task_id": "16143f74-2831-4753-b33d-ce4b645093c5", - "turn_id": "f419377a-fc24-4abf-8a1c-03857824e374", - "ground_truth": null, - "expected_tool_call": null, - "agent_name": "agent", - "agent_role": "agent", - "agent_task": null, - "system_prompt": "You are a helpful customer support agent", - "agent_response": null, - "trace": null, - "tools_available": [], - "tool_calls": [], - "parameters_passed": {}, - "tool_call_results": [], - "retrieval_query": null, - "retrieved_context": null, - "exit_status": "ok", - "agent_exit": false, - "metadata": null - }, - { - "user_id": null, - "task_id": "fc64e6cc-6739-4256-ac4a-7b80c3028233", - "turn_id": "0e58e31f-a8ab-4aee-8fc2-abcae1e21d9d", - "ground_truth": null, - "expected_tool_call": null, - "agent_name": "agent", - "agent_role": "agent", - "agent_task": null, - "system_prompt": "You are a helpful customer support agent", - "agent_response": null, - "trace": null, - "tools_available": [], - "tool_calls": [], - "parameters_passed": {}, - "tool_call_results": [], - "retrieval_query": null, - "retrieved_context": null, - "exit_status": "ok", - "agent_exit": false, - "metadata": null - }, - { - "user_id": null, - "task_id": "b7945c49-f584-4c70-972d-536a805d8a31", - "turn_id": "48155aad-6204-4a52-8624-8fa5d6fe407f", - "ground_truth": null, - "expected_tool_call": null, - "agent_name": "agent", - "agent_role": "agent", - "agent_task": null, - "system_prompt": "You are a helpful customer support agent", - "agent_response": null, - "trace": null, - "tools_available": [], - "tool_calls": [], - "parameters_passed": {}, - "tool_call_results": [], - "retrieval_query": null, - "retrieved_context": null, - "exit_status": "ok", - "agent_exit": false, - "metadata": null - }, - { - "user_id": null, - "task_id": "f5c40ecf-36c0-45ba-9cc9-dc0329b0324b", - "turn_id": "57ded2eb-0f3f-474e-8704-5ad9974ab4bb", - "ground_truth": null, - "expected_tool_call": null, - "agent_name": "agent", - "agent_role": "agent", - "agent_task": null, - "system_prompt": "You are a helpful customer support agent", - "agent_response": null, - "trace": null, - "tools_available": [], - "tool_calls": [], - "parameters_passed": {}, - "tool_call_results": [], - "retrieval_query": null, - "retrieved_context": null, - "exit_status": "ok", - "agent_exit": false, - "metadata": null - } -] \ No newline at end of file diff --git a/noveum_customer_support_bt/processed_datasets/agent.llm_model_execution_dataset_processed_dataset.csv b/noveum_customer_support_bt/processed_datasets/agent.llm_model_execution_dataset_processed_dataset.csv deleted file mode 100644 index eae155f..0000000 --- a/noveum_customer_support_bt/processed_datasets/agent.llm_model_execution_dataset_processed_dataset.csv +++ /dev/null @@ -1,21 +0,0 @@ -user_id,task_id,turn_id,ground_truth,expected_tool_call,agent_name,agent_role,agent_task,system_prompt,agent_response,trace,tools_available,tool_calls,parameters_passed,tool_call_results,retrieval_query,retrieved_context,exit_status,agent_exit,metadata -,eda4fe22-9a2b-4b73-856b-f4f3309bf719,ce8556f4-bbeb-46fc-8bfd-dfaedb61cb70,,,agent,agent,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, -,0ffffba1-8a37-443c-8866-d53ffbfa7718,cc3073ec-c970-4821-8335-8d0fc8f37930,,,agent,agent,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, -,f1f37bd7-0851-4659-b493-b80d3800d920,c24fae6d-6314-4342-8214-7e9905ae83db,,,agent,agent,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, -,43cdf081-4f01-49cd-b566-dbd1619e6cd2,93879d8f-77c7-4fd4-881b-be500e2dcbde,,,agent,agent,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, -,9a1983f4-09da-4b53-80e6-38de6878e0e7,f92643c0-44a3-4fae-9d29-45321342d12f,,,agent,agent,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, -,5d2517e1-220a-429d-9d59-f701bda25eed,44f379aa-10fe-4a51-847c-5288444d7843,,,agent,agent,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, -,52aacb67-c361-4445-9b72-c157f79f47d6,e8e79f2a-2ed0-4981-943c-adcf6f7d2b15,,,agent,agent,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, -,a81ca3a8-80aa-4c39-876e-8d40ea7a0aef,e6d0eb41-5f64-44c5-8a42-a80c336de03e,,,agent,agent,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, -,230aad27-f3dd-4968-a45a-3c2f07ac28ed,c545cb62-5a51-4f2e-b020-b3f1d1dfe217,,,agent,agent,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, -,83c7dcce-3d89-4da1-8b3f-d419885d4cbc,c2f21e4d-a5e6-4134-aeaf-96e2f566cf96,,,agent,agent,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, -,2218f641-604c-491a-9710-b51a9941b982,39d99d2b-518f-439c-8758-7a01b970fd9e,,,agent,agent,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, -,255fd49c-84b4-4b18-887e-6308a412d535,9e5ade74-3815-4dad-8abe-d4281d7e62da,,,agent,agent,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, -,dc511122-c0b6-415c-9a49-c7b45132dd87,4f38a85b-0721-47a5-8a51-d0808e460f96,,,agent,agent,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, -,04bebf38-a343-4563-80db-0154bef8d927,f4c03a8d-b90d-4c1a-a259-826d09e3e57a,,,agent,agent,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, -,5e043630-6493-42b5-beb8-79faa19bfa37,48fb40ba-a39f-4e13-9d94-18794c7e5e2b,,,agent,agent,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, -,7da9814d-a2e8-4c4e-b750-68b26bd5fd22,d4d544a0-f052-4889-82eb-4346f2e447a7,,,agent,agent,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, -,16143f74-2831-4753-b33d-ce4b645093c5,304f9e3d-9063-4661-9777-618efc49f5c1,,,agent,agent,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, -,fc64e6cc-6739-4256-ac4a-7b80c3028233,e0b4ccfb-18e7-4d71-b2dd-5b807eb27513,,,agent,agent,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, -,b7945c49-f584-4c70-972d-536a805d8a31,f1425131-f8a2-43f7-aa17-d009f745ed06,,,agent,agent,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, -,f5c40ecf-36c0-45ba-9cc9-dc0329b0324b,f41e836c-1a82-46e0-9d51-011713874e94,,,agent,agent,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, diff --git a/noveum_customer_support_bt/processed_datasets/agent.llm_model_execution_dataset_processed_dataset.json b/noveum_customer_support_bt/processed_datasets/agent.llm_model_execution_dataset_processed_dataset.json deleted file mode 100644 index 6fa4b1c..0000000 --- a/noveum_customer_support_bt/processed_datasets/agent.llm_model_execution_dataset_processed_dataset.json +++ /dev/null @@ -1,442 +0,0 @@ -[ - { - "user_id": null, - "task_id": "eda4fe22-9a2b-4b73-856b-f4f3309bf719", - "turn_id": "ce8556f4-bbeb-46fc-8bfd-dfaedb61cb70", - "ground_truth": null, - "expected_tool_call": null, - "agent_name": "agent", - "agent_role": "agent", - "agent_task": null, - "system_prompt": "You are a helpful customer support agent", - "agent_response": null, - "trace": null, - "tools_available": [], - "tool_calls": [], - "parameters_passed": {}, - "tool_call_results": [], - "retrieval_query": null, - "retrieved_context": null, - "exit_status": "ok", - "agent_exit": false, - "metadata": null - }, - { - "user_id": null, - "task_id": "0ffffba1-8a37-443c-8866-d53ffbfa7718", - "turn_id": "cc3073ec-c970-4821-8335-8d0fc8f37930", - "ground_truth": null, - "expected_tool_call": null, - "agent_name": "agent", - "agent_role": "agent", - "agent_task": null, - "system_prompt": "You are a helpful customer support agent", - "agent_response": null, - "trace": null, - "tools_available": [], - "tool_calls": [], - "parameters_passed": {}, - "tool_call_results": [], - "retrieval_query": null, - "retrieved_context": null, - "exit_status": "ok", - "agent_exit": false, - "metadata": null - }, - { - "user_id": null, - "task_id": "f1f37bd7-0851-4659-b493-b80d3800d920", - "turn_id": "c24fae6d-6314-4342-8214-7e9905ae83db", - "ground_truth": null, - "expected_tool_call": null, - "agent_name": "agent", - "agent_role": "agent", - "agent_task": null, - "system_prompt": "You are a helpful customer support agent", - "agent_response": null, - "trace": null, - "tools_available": [], - "tool_calls": [], - "parameters_passed": {}, - "tool_call_results": [], - "retrieval_query": null, - "retrieved_context": null, - "exit_status": "ok", - "agent_exit": false, - "metadata": null - }, - { - "user_id": null, - "task_id": "43cdf081-4f01-49cd-b566-dbd1619e6cd2", - "turn_id": "93879d8f-77c7-4fd4-881b-be500e2dcbde", - "ground_truth": null, - "expected_tool_call": null, - "agent_name": "agent", - "agent_role": "agent", - "agent_task": null, - "system_prompt": "You are a helpful customer support agent", - "agent_response": null, - "trace": null, - "tools_available": [], - "tool_calls": [], - "parameters_passed": {}, - "tool_call_results": [], - "retrieval_query": null, - "retrieved_context": null, - "exit_status": "ok", - "agent_exit": false, - "metadata": null - }, - { - "user_id": null, - "task_id": "9a1983f4-09da-4b53-80e6-38de6878e0e7", - "turn_id": "f92643c0-44a3-4fae-9d29-45321342d12f", - "ground_truth": null, - "expected_tool_call": null, - "agent_name": "agent", - "agent_role": "agent", - "agent_task": null, - "system_prompt": "You are a helpful customer support agent", - "agent_response": null, - "trace": null, - "tools_available": [], - "tool_calls": [], - "parameters_passed": {}, - "tool_call_results": [], - "retrieval_query": null, - "retrieved_context": null, - "exit_status": "ok", - "agent_exit": false, - "metadata": null - }, - { - "user_id": null, - "task_id": "5d2517e1-220a-429d-9d59-f701bda25eed", - "turn_id": "44f379aa-10fe-4a51-847c-5288444d7843", - "ground_truth": null, - "expected_tool_call": null, - "agent_name": "agent", - "agent_role": "agent", - "agent_task": null, - "system_prompt": "You are a helpful customer support agent", - "agent_response": null, - "trace": null, - "tools_available": [], - "tool_calls": [], - "parameters_passed": {}, - "tool_call_results": [], - "retrieval_query": null, - "retrieved_context": null, - "exit_status": "ok", - "agent_exit": false, - "metadata": null - }, - { - "user_id": null, - "task_id": "52aacb67-c361-4445-9b72-c157f79f47d6", - "turn_id": "e8e79f2a-2ed0-4981-943c-adcf6f7d2b15", - "ground_truth": null, - "expected_tool_call": null, - "agent_name": "agent", - "agent_role": "agent", - "agent_task": null, - "system_prompt": "You are a helpful customer support agent", - "agent_response": null, - "trace": null, - "tools_available": [], - "tool_calls": [], - "parameters_passed": {}, - "tool_call_results": [], - "retrieval_query": null, - "retrieved_context": null, - "exit_status": "ok", - "agent_exit": false, - "metadata": null - }, - { - "user_id": null, - "task_id": "a81ca3a8-80aa-4c39-876e-8d40ea7a0aef", - "turn_id": "e6d0eb41-5f64-44c5-8a42-a80c336de03e", - "ground_truth": null, - "expected_tool_call": null, - "agent_name": "agent", - "agent_role": "agent", - "agent_task": null, - "system_prompt": "You are a helpful customer support agent", - "agent_response": null, - "trace": null, - "tools_available": [], - "tool_calls": [], - "parameters_passed": {}, - "tool_call_results": [], - "retrieval_query": null, - "retrieved_context": null, - "exit_status": "ok", - "agent_exit": false, - "metadata": null - }, - { - "user_id": null, - "task_id": "230aad27-f3dd-4968-a45a-3c2f07ac28ed", - "turn_id": "c545cb62-5a51-4f2e-b020-b3f1d1dfe217", - "ground_truth": null, - "expected_tool_call": null, - "agent_name": "agent", - "agent_role": "agent", - "agent_task": null, - "system_prompt": "You are a helpful customer support agent", - "agent_response": null, - "trace": null, - "tools_available": [], - "tool_calls": [], - "parameters_passed": {}, - "tool_call_results": [], - "retrieval_query": null, - "retrieved_context": null, - "exit_status": "ok", - "agent_exit": false, - "metadata": null - }, - { - "user_id": null, - "task_id": "83c7dcce-3d89-4da1-8b3f-d419885d4cbc", - "turn_id": "c2f21e4d-a5e6-4134-aeaf-96e2f566cf96", - "ground_truth": null, - "expected_tool_call": null, - "agent_name": "agent", - "agent_role": "agent", - "agent_task": null, - "system_prompt": "You are a helpful customer support agent", - "agent_response": null, - "trace": null, - "tools_available": [], - "tool_calls": [], - "parameters_passed": {}, - "tool_call_results": [], - "retrieval_query": null, - "retrieved_context": null, - "exit_status": "ok", - "agent_exit": false, - "metadata": null - }, - { - "user_id": null, - "task_id": "2218f641-604c-491a-9710-b51a9941b982", - "turn_id": "39d99d2b-518f-439c-8758-7a01b970fd9e", - "ground_truth": null, - "expected_tool_call": null, - "agent_name": "agent", - "agent_role": "agent", - "agent_task": null, - "system_prompt": "You are a helpful customer support agent", - "agent_response": null, - "trace": null, - "tools_available": [], - "tool_calls": [], - "parameters_passed": {}, - "tool_call_results": [], - "retrieval_query": null, - "retrieved_context": null, - "exit_status": "ok", - "agent_exit": false, - "metadata": null - }, - { - "user_id": null, - "task_id": "255fd49c-84b4-4b18-887e-6308a412d535", - "turn_id": "9e5ade74-3815-4dad-8abe-d4281d7e62da", - "ground_truth": null, - "expected_tool_call": null, - "agent_name": "agent", - "agent_role": "agent", - "agent_task": null, - "system_prompt": "You are a helpful customer support agent", - "agent_response": null, - "trace": null, - "tools_available": [], - "tool_calls": [], - "parameters_passed": {}, - "tool_call_results": [], - "retrieval_query": null, - "retrieved_context": null, - "exit_status": "ok", - "agent_exit": false, - "metadata": null - }, - { - "user_id": null, - "task_id": "dc511122-c0b6-415c-9a49-c7b45132dd87", - "turn_id": "4f38a85b-0721-47a5-8a51-d0808e460f96", - "ground_truth": null, - "expected_tool_call": null, - "agent_name": "agent", - "agent_role": "agent", - "agent_task": null, - "system_prompt": "You are a helpful customer support agent", - "agent_response": null, - "trace": null, - "tools_available": [], - "tool_calls": [], - "parameters_passed": {}, - "tool_call_results": [], - "retrieval_query": null, - "retrieved_context": null, - "exit_status": "ok", - "agent_exit": false, - "metadata": null - }, - { - "user_id": null, - "task_id": "04bebf38-a343-4563-80db-0154bef8d927", - "turn_id": "f4c03a8d-b90d-4c1a-a259-826d09e3e57a", - "ground_truth": null, - "expected_tool_call": null, - "agent_name": "agent", - "agent_role": "agent", - "agent_task": null, - "system_prompt": "You are a helpful customer support agent", - "agent_response": null, - "trace": null, - "tools_available": [], - "tool_calls": [], - "parameters_passed": {}, - "tool_call_results": [], - "retrieval_query": null, - "retrieved_context": null, - "exit_status": "ok", - "agent_exit": false, - "metadata": null - }, - { - "user_id": null, - "task_id": "5e043630-6493-42b5-beb8-79faa19bfa37", - "turn_id": "48fb40ba-a39f-4e13-9d94-18794c7e5e2b", - "ground_truth": null, - "expected_tool_call": null, - "agent_name": "agent", - "agent_role": "agent", - "agent_task": null, - "system_prompt": "You are a helpful customer support agent", - "agent_response": null, - "trace": null, - "tools_available": [], - "tool_calls": [], - "parameters_passed": {}, - "tool_call_results": [], - "retrieval_query": null, - "retrieved_context": null, - "exit_status": "ok", - "agent_exit": false, - "metadata": null - }, - { - "user_id": null, - "task_id": "7da9814d-a2e8-4c4e-b750-68b26bd5fd22", - "turn_id": "d4d544a0-f052-4889-82eb-4346f2e447a7", - "ground_truth": null, - "expected_tool_call": null, - "agent_name": "agent", - "agent_role": "agent", - "agent_task": null, - "system_prompt": "You are a helpful customer support agent", - "agent_response": null, - "trace": null, - "tools_available": [], - "tool_calls": [], - "parameters_passed": {}, - "tool_call_results": [], - "retrieval_query": null, - "retrieved_context": null, - "exit_status": "ok", - "agent_exit": false, - "metadata": null - }, - { - "user_id": null, - "task_id": "16143f74-2831-4753-b33d-ce4b645093c5", - "turn_id": "304f9e3d-9063-4661-9777-618efc49f5c1", - "ground_truth": null, - "expected_tool_call": null, - "agent_name": "agent", - "agent_role": "agent", - "agent_task": null, - "system_prompt": "You are a helpful customer support agent", - "agent_response": null, - "trace": null, - "tools_available": [], - "tool_calls": [], - "parameters_passed": {}, - "tool_call_results": [], - "retrieval_query": null, - "retrieved_context": null, - "exit_status": "ok", - "agent_exit": false, - "metadata": null - }, - { - "user_id": null, - "task_id": "fc64e6cc-6739-4256-ac4a-7b80c3028233", - "turn_id": "e0b4ccfb-18e7-4d71-b2dd-5b807eb27513", - "ground_truth": null, - "expected_tool_call": null, - "agent_name": "agent", - "agent_role": "agent", - "agent_task": null, - "system_prompt": "You are a helpful customer support agent", - "agent_response": null, - "trace": null, - "tools_available": [], - "tool_calls": [], - "parameters_passed": {}, - "tool_call_results": [], - "retrieval_query": null, - "retrieved_context": null, - "exit_status": "ok", - "agent_exit": false, - "metadata": null - }, - { - "user_id": null, - "task_id": "b7945c49-f584-4c70-972d-536a805d8a31", - "turn_id": "f1425131-f8a2-43f7-aa17-d009f745ed06", - "ground_truth": null, - "expected_tool_call": null, - "agent_name": "agent", - "agent_role": "agent", - "agent_task": null, - "system_prompt": "You are a helpful customer support agent", - "agent_response": null, - "trace": null, - "tools_available": [], - "tool_calls": [], - "parameters_passed": {}, - "tool_call_results": [], - "retrieval_query": null, - "retrieved_context": null, - "exit_status": "ok", - "agent_exit": false, - "metadata": null - }, - { - "user_id": null, - "task_id": "f5c40ecf-36c0-45ba-9cc9-dc0329b0324b", - "turn_id": "f41e836c-1a82-46e0-9d51-011713874e94", - "ground_truth": null, - "expected_tool_call": null, - "agent_name": "agent", - "agent_role": "agent", - "agent_task": null, - "system_prompt": "You are a helpful customer support agent", - "agent_response": null, - "trace": null, - "tools_available": [], - "tool_calls": [], - "parameters_passed": {}, - "tool_call_results": [], - "retrieval_query": null, - "retrieved_context": null, - "exit_status": "ok", - "agent_exit": false, - "metadata": null - } -] \ No newline at end of file diff --git a/noveum_customer_support_bt/processed_datasets/agent.query_routing_dataset_processed_dataset.csv b/noveum_customer_support_bt/processed_datasets/agent.query_routing_dataset_processed_dataset.csv deleted file mode 100644 index 8578b36..0000000 --- a/noveum_customer_support_bt/processed_datasets/agent.query_routing_dataset_processed_dataset.csv +++ /dev/null @@ -1,21 +0,0 @@ -user_id,task_id,turn_id,ground_truth,expected_tool_call,agent_name,agent_role,agent_task,system_prompt,agent_response,trace,tools_available,tool_calls,parameters_passed,tool_call_results,retrieval_query,retrieved_context,exit_status,agent_exit,metadata -,eda4fe22-9a2b-4b73-856b-f4f3309bf719,0b19befc-5dc0-4b00-9a8b-5f2f2c914f17,,,agent,agent,What are the latest breakthroughs in artificial intelligence?,You are a helpful customer support agent,,,"[{""name"": ""query_routing"", ""description"": ""Routes queries to appropriate handlers"", ""args_schema"": {}, ""return_schema"": {""type"": ""routing_decision""}}]",[],{},[],,,ok,False, -,0ffffba1-8a37-443c-8866-d53ffbfa7718,30644d2a-5ab1-4a73-b4ff-cf9fa478e36b,,,agent,agent,What's happening in the software development world today?,You are a helpful customer support agent,,,"[{""name"": ""query_routing"", ""description"": ""Routes queries to appropriate handlers"", ""args_schema"": {}, ""return_schema"": {""type"": ""routing_decision""}}]",[],{},[],,,ok,False, -,f1f37bd7-0851-4659-b493-b80d3800d920,89eac2de-b7da-4db3-bf7c-8e87c429481e,,,agent,agent,What are the newest features in cloud computing?,You are a helpful customer support agent,,,"[{""name"": ""query_routing"", ""description"": ""Routes queries to appropriate handlers"", ""args_schema"": {}, ""return_schema"": {""type"": ""routing_decision""}}]",[],{},[],,,ok,False, -,43cdf081-4f01-49cd-b566-dbd1619e6cd2,962afebb-26d1-45ff-acf1-218163cd6bb8,,,agent,agent,What's the current status of cryptocurrency markets?,You are a helpful customer support agent,,,"[{""name"": ""query_routing"", ""description"": ""Routes queries to appropriate handlers"", ""args_schema"": {}, ""return_schema"": {""type"": ""routing_decision""}}]",[],{},[],,,ok,False, -,9a1983f4-09da-4b53-80e6-38de6878e0e7,246cf822-4562-4d3a-8a5a-e38e0a0b02a0,,,agent,agent,What are the latest updates in Python programming?,You are a helpful customer support agent,,,"[{""name"": ""query_routing"", ""description"": ""Routes queries to appropriate handlers"", ""args_schema"": {}, ""return_schema"": {""type"": ""routing_decision""}}]",[],{},[],,,ok,False, -,5d2517e1-220a-429d-9d59-f701bda25eed,20544f91-8828-4a0e-b253-ed1f8b95b205,,,agent,agent,What happened in tech news this week?,You are a helpful customer support agent,,,"[{""name"": ""query_routing"", ""description"": ""Routes queries to appropriate handlers"", ""args_schema"": {}, ""return_schema"": {""type"": ""routing_decision""}}]",[],{},[],,,ok,False, -,52aacb67-c361-4445-9b72-c157f79f47d6,5359c861-c51b-446d-9ad2-b3d8c21ea70f,,,agent,agent,What are the current trends in observability tools?,You are a helpful customer support agent,,,"[{""name"": ""query_routing"", ""description"": ""Routes queries to appropriate handlers"", ""args_schema"": {}, ""return_schema"": {""type"": ""routing_decision""}}]",[],{},[],,,ok,False, -,a81ca3a8-80aa-4c39-876e-8d40ea7a0aef,acf6a99a-39e0-4f87-a07e-3a61213ab032,,,agent,agent,Tell me about recent developments in machine learning,You are a helpful customer support agent,,,"[{""name"": ""query_routing"", ""description"": ""Routes queries to appropriate handlers"", ""args_schema"": {}, ""return_schema"": {""type"": ""routing_decision""}}]",[],{},[],,,ok,False, -,230aad27-f3dd-4968-a45a-3c2f07ac28ed,0b47ea07-1eda-4b6a-a183-ea9ef047e431,,,agent,agent,What's the weather like today?,You are a helpful customer support agent,,,"[{""name"": ""query_routing"", ""description"": ""Routes queries to appropriate handlers"", ""args_schema"": {}, ""return_schema"": {""type"": ""routing_decision""}}]",[],{},[],,,ok,False, -,83c7dcce-3d89-4da1-8b3f-d419885d4cbc,3866ddb3-b17e-47a1-8b3f-bca1e32a2ae4,,,agent,agent,What are the latest AI news today?,You are a helpful customer support agent,,,"[{""name"": ""query_routing"", ""description"": ""Routes queries to appropriate handlers"", ""args_schema"": {}, ""return_schema"": {""type"": ""routing_decision""}}]",[],{},[],,,ok,False, -,2218f641-604c-491a-9710-b51a9941b982,3ce2e35e-4518-43b4-8a76-4413da6832ea,,,agent,agent,What are the benefits of using Noveum Trace?,You are a helpful customer support agent,,,"[{""name"": ""query_routing"", ""description"": ""Routes queries to appropriate handlers"", ""args_schema"": {}, ""return_schema"": {""type"": ""routing_decision""}}]",[],{},[],,,ok,False, -,255fd49c-84b4-4b18-887e-6308a412d535,c0c39e8b-9a42-4c18-8737-9224cd8a45c0,,,agent,agent,How do I configure Noveum for my system?,You are a helpful customer support agent,,,"[{""name"": ""query_routing"", ""description"": ""Routes queries to appropriate handlers"", ""args_schema"": {}, ""return_schema"": {""type"": ""routing_decision""}}]",[],{},[],,,ok,False, -,dc511122-c0b6-415c-9a49-c7b45132dd87,80987c51-ea82-4a41-97c7-ed591082b9bf,,,agent,agent,What monitoring capabilities does Noveum provide?,You are a helpful customer support agent,,,"[{""name"": ""query_routing"", ""description"": ""Routes queries to appropriate handlers"", ""args_schema"": {}, ""return_schema"": {""type"": ""routing_decision""}}]",[],{},[],,,ok,False, -,04bebf38-a343-4563-80db-0154bef8d927,2f89c1e0-9b62-4651-914c-b71cf561c33c,,,agent,agent,How does Noveum handle agent tracing?,You are a helpful customer support agent,,,"[{""name"": ""query_routing"", ""description"": ""Routes queries to appropriate handlers"", ""args_schema"": {}, ""return_schema"": {""type"": ""routing_decision""}}]",[],{},[],,,ok,False, -,5e043630-6493-42b5-beb8-79faa19bfa37,e30e3e61-6edd-4b62-ab77-627faa2bf741,,,agent,agent,What APIs are available in Noveum platform?,You are a helpful customer support agent,,,"[{""name"": ""query_routing"", ""description"": ""Routes queries to appropriate handlers"", ""args_schema"": {}, ""return_schema"": {""type"": ""routing_decision""}}]",[],{},[],,,ok,False, -,7da9814d-a2e8-4c4e-b750-68b26bd5fd22,2f513795-764f-4eb6-8dd2-108d4cdf8432,,,agent,agent,How do I set up observability with Noveum?,You are a helpful customer support agent,,,"[{""name"": ""query_routing"", ""description"": ""Routes queries to appropriate handlers"", ""args_schema"": {}, ""return_schema"": {""type"": ""routing_decision""}}]",[],{},[],,,ok,False, -,16143f74-2831-4753-b33d-ce4b645093c5,29c7a515-2383-4b7d-8fc7-beac3f403641,,,agent,agent,What features does Noveum Trace offer?,You are a helpful customer support agent,,,"[{""name"": ""query_routing"", ""description"": ""Routes queries to appropriate handlers"", ""args_schema"": {}, ""return_schema"": {""type"": ""routing_decision""}}]",[],{},[],,,ok,False, -,fc64e6cc-6739-4256-ac4a-7b80c3028233,12fee687-2a2e-401f-83c2-9c8dc7ecd853,,,agent,agent,What are Noveum's pricing plans?,You are a helpful customer support agent,,,"[{""name"": ""query_routing"", ""description"": ""Routes queries to appropriate handlers"", ""args_schema"": {}, ""return_schema"": {""type"": ""routing_decision""}}]",[],{},[],,,ok,False, -,b7945c49-f584-4c70-972d-536a805d8a31,9c5aec32-b6eb-4c42-93d5-3970fb118c00,,,agent,agent,How do I integrate Noveum Trace in my application?,You are a helpful customer support agent,,,"[{""name"": ""query_routing"", ""description"": ""Routes queries to appropriate handlers"", ""args_schema"": {}, ""return_schema"": {""type"": ""routing_decision""}}]",[],{},[],,,ok,False, -,f5c40ecf-36c0-45ba-9cc9-dc0329b0324b,f132d053-f82c-4e66-adf4-bd7ce68b660d,,,agent,agent,What is Noveum and what does it do?,You are a helpful customer support agent,,,"[{""name"": ""query_routing"", ""description"": ""Routes queries to appropriate handlers"", ""args_schema"": {}, ""return_schema"": {""type"": ""routing_decision""}}]",[],{},[],,,ok,False, diff --git a/noveum_customer_support_bt/processed_datasets/agent.query_routing_dataset_processed_dataset.json b/noveum_customer_support_bt/processed_datasets/agent.query_routing_dataset_processed_dataset.json deleted file mode 100644 index 3bca440..0000000 --- a/noveum_customer_support_bt/processed_datasets/agent.query_routing_dataset_processed_dataset.json +++ /dev/null @@ -1,622 +0,0 @@ -[ - { - "user_id": null, - "task_id": "eda4fe22-9a2b-4b73-856b-f4f3309bf719", - "turn_id": "0b19befc-5dc0-4b00-9a8b-5f2f2c914f17", - "ground_truth": null, - "expected_tool_call": null, - "agent_name": "agent", - "agent_role": "agent", - "agent_task": "What are the latest breakthroughs in artificial intelligence?", - "system_prompt": "You are a helpful customer support agent", - "agent_response": "", - "trace": null, - "tools_available": [ - { - "name": "query_routing", - "description": "Routes queries to appropriate handlers", - "args_schema": {}, - "return_schema": { - "type": "routing_decision" - } - } - ], - "tool_calls": [], - "parameters_passed": {}, - "tool_call_results": [], - "retrieval_query": null, - "retrieved_context": null, - "exit_status": "ok", - "agent_exit": false, - "metadata": null - }, - { - "user_id": null, - "task_id": "0ffffba1-8a37-443c-8866-d53ffbfa7718", - "turn_id": "30644d2a-5ab1-4a73-b4ff-cf9fa478e36b", - "ground_truth": null, - "expected_tool_call": null, - "agent_name": "agent", - "agent_role": "agent", - "agent_task": "What's happening in the software development world today?", - "system_prompt": "You are a helpful customer support agent", - "agent_response": "", - "trace": null, - "tools_available": [ - { - "name": "query_routing", - "description": "Routes queries to appropriate handlers", - "args_schema": {}, - "return_schema": { - "type": "routing_decision" - } - } - ], - "tool_calls": [], - "parameters_passed": {}, - "tool_call_results": [], - "retrieval_query": null, - "retrieved_context": null, - "exit_status": "ok", - "agent_exit": false, - "metadata": null - }, - { - "user_id": null, - "task_id": "f1f37bd7-0851-4659-b493-b80d3800d920", - "turn_id": "89eac2de-b7da-4db3-bf7c-8e87c429481e", - "ground_truth": null, - "expected_tool_call": null, - "agent_name": "agent", - "agent_role": "agent", - "agent_task": "What are the newest features in cloud computing?", - "system_prompt": "You are a helpful customer support agent", - "agent_response": "", - "trace": null, - "tools_available": [ - { - "name": "query_routing", - "description": "Routes queries to appropriate handlers", - "args_schema": {}, - "return_schema": { - "type": "routing_decision" - } - } - ], - "tool_calls": [], - "parameters_passed": {}, - "tool_call_results": [], - "retrieval_query": null, - "retrieved_context": null, - "exit_status": "ok", - "agent_exit": false, - "metadata": null - }, - { - "user_id": null, - "task_id": "43cdf081-4f01-49cd-b566-dbd1619e6cd2", - "turn_id": "962afebb-26d1-45ff-acf1-218163cd6bb8", - "ground_truth": null, - "expected_tool_call": null, - "agent_name": "agent", - "agent_role": "agent", - "agent_task": "What's the current status of cryptocurrency markets?", - "system_prompt": "You are a helpful customer support agent", - "agent_response": "", - "trace": null, - "tools_available": [ - { - "name": "query_routing", - "description": "Routes queries to appropriate handlers", - "args_schema": {}, - "return_schema": { - "type": "routing_decision" - } - } - ], - "tool_calls": [], - "parameters_passed": {}, - "tool_call_results": [], - "retrieval_query": null, - "retrieved_context": null, - "exit_status": "ok", - "agent_exit": false, - "metadata": null - }, - { - "user_id": null, - "task_id": "9a1983f4-09da-4b53-80e6-38de6878e0e7", - "turn_id": "246cf822-4562-4d3a-8a5a-e38e0a0b02a0", - "ground_truth": null, - "expected_tool_call": null, - "agent_name": "agent", - "agent_role": "agent", - "agent_task": "What are the latest updates in Python programming?", - "system_prompt": "You are a helpful customer support agent", - "agent_response": "", - "trace": null, - "tools_available": [ - { - "name": "query_routing", - "description": "Routes queries to appropriate handlers", - "args_schema": {}, - "return_schema": { - "type": "routing_decision" - } - } - ], - "tool_calls": [], - "parameters_passed": {}, - "tool_call_results": [], - "retrieval_query": null, - "retrieved_context": null, - "exit_status": "ok", - "agent_exit": false, - "metadata": null - }, - { - "user_id": null, - "task_id": "5d2517e1-220a-429d-9d59-f701bda25eed", - "turn_id": "20544f91-8828-4a0e-b253-ed1f8b95b205", - "ground_truth": null, - "expected_tool_call": null, - "agent_name": "agent", - "agent_role": "agent", - "agent_task": "What happened in tech news this week?", - "system_prompt": "You are a helpful customer support agent", - "agent_response": "", - "trace": null, - "tools_available": [ - { - "name": "query_routing", - "description": "Routes queries to appropriate handlers", - "args_schema": {}, - "return_schema": { - "type": "routing_decision" - } - } - ], - "tool_calls": [], - "parameters_passed": {}, - "tool_call_results": [], - "retrieval_query": null, - "retrieved_context": null, - "exit_status": "ok", - "agent_exit": false, - "metadata": null - }, - { - "user_id": null, - "task_id": "52aacb67-c361-4445-9b72-c157f79f47d6", - "turn_id": "5359c861-c51b-446d-9ad2-b3d8c21ea70f", - "ground_truth": null, - "expected_tool_call": null, - "agent_name": "agent", - "agent_role": "agent", - "agent_task": "What are the current trends in observability tools?", - "system_prompt": "You are a helpful customer support agent", - "agent_response": "", - "trace": null, - "tools_available": [ - { - "name": "query_routing", - "description": "Routes queries to appropriate handlers", - "args_schema": {}, - "return_schema": { - "type": "routing_decision" - } - } - ], - "tool_calls": [], - "parameters_passed": {}, - "tool_call_results": [], - "retrieval_query": null, - "retrieved_context": null, - "exit_status": "ok", - "agent_exit": false, - "metadata": null - }, - { - "user_id": null, - "task_id": "a81ca3a8-80aa-4c39-876e-8d40ea7a0aef", - "turn_id": "acf6a99a-39e0-4f87-a07e-3a61213ab032", - "ground_truth": null, - "expected_tool_call": null, - "agent_name": "agent", - "agent_role": "agent", - "agent_task": "Tell me about recent developments in machine learning", - "system_prompt": "You are a helpful customer support agent", - "agent_response": "", - "trace": null, - "tools_available": [ - { - "name": "query_routing", - "description": "Routes queries to appropriate handlers", - "args_schema": {}, - "return_schema": { - "type": "routing_decision" - } - } - ], - "tool_calls": [], - "parameters_passed": {}, - "tool_call_results": [], - "retrieval_query": null, - "retrieved_context": null, - "exit_status": "ok", - "agent_exit": false, - "metadata": null - }, - { - "user_id": null, - "task_id": "230aad27-f3dd-4968-a45a-3c2f07ac28ed", - "turn_id": "0b47ea07-1eda-4b6a-a183-ea9ef047e431", - "ground_truth": null, - "expected_tool_call": null, - "agent_name": "agent", - "agent_role": "agent", - "agent_task": "What's the weather like today?", - "system_prompt": "You are a helpful customer support agent", - "agent_response": "", - "trace": null, - "tools_available": [ - { - "name": "query_routing", - "description": "Routes queries to appropriate handlers", - "args_schema": {}, - "return_schema": { - "type": "routing_decision" - } - } - ], - "tool_calls": [], - "parameters_passed": {}, - "tool_call_results": [], - "retrieval_query": null, - "retrieved_context": null, - "exit_status": "ok", - "agent_exit": false, - "metadata": null - }, - { - "user_id": null, - "task_id": "83c7dcce-3d89-4da1-8b3f-d419885d4cbc", - "turn_id": "3866ddb3-b17e-47a1-8b3f-bca1e32a2ae4", - "ground_truth": null, - "expected_tool_call": null, - "agent_name": "agent", - "agent_role": "agent", - "agent_task": "What are the latest AI news today?", - "system_prompt": "You are a helpful customer support agent", - "agent_response": "", - "trace": null, - "tools_available": [ - { - "name": "query_routing", - "description": "Routes queries to appropriate handlers", - "args_schema": {}, - "return_schema": { - "type": "routing_decision" - } - } - ], - "tool_calls": [], - "parameters_passed": {}, - "tool_call_results": [], - "retrieval_query": null, - "retrieved_context": null, - "exit_status": "ok", - "agent_exit": false, - "metadata": null - }, - { - "user_id": null, - "task_id": "2218f641-604c-491a-9710-b51a9941b982", - "turn_id": "3ce2e35e-4518-43b4-8a76-4413da6832ea", - "ground_truth": null, - "expected_tool_call": null, - "agent_name": "agent", - "agent_role": "agent", - "agent_task": "What are the benefits of using Noveum Trace?", - "system_prompt": "You are a helpful customer support agent", - "agent_response": "", - "trace": null, - "tools_available": [ - { - "name": "query_routing", - "description": "Routes queries to appropriate handlers", - "args_schema": {}, - "return_schema": { - "type": "routing_decision" - } - } - ], - "tool_calls": [], - "parameters_passed": {}, - "tool_call_results": [], - "retrieval_query": null, - "retrieved_context": null, - "exit_status": "ok", - "agent_exit": false, - "metadata": null - }, - { - "user_id": null, - "task_id": "255fd49c-84b4-4b18-887e-6308a412d535", - "turn_id": "c0c39e8b-9a42-4c18-8737-9224cd8a45c0", - "ground_truth": null, - "expected_tool_call": null, - "agent_name": "agent", - "agent_role": "agent", - "agent_task": "How do I configure Noveum for my system?", - "system_prompt": "You are a helpful customer support agent", - "agent_response": "", - "trace": null, - "tools_available": [ - { - "name": "query_routing", - "description": "Routes queries to appropriate handlers", - "args_schema": {}, - "return_schema": { - "type": "routing_decision" - } - } - ], - "tool_calls": [], - "parameters_passed": {}, - "tool_call_results": [], - "retrieval_query": null, - "retrieved_context": null, - "exit_status": "ok", - "agent_exit": false, - "metadata": null - }, - { - "user_id": null, - "task_id": "dc511122-c0b6-415c-9a49-c7b45132dd87", - "turn_id": "80987c51-ea82-4a41-97c7-ed591082b9bf", - "ground_truth": null, - "expected_tool_call": null, - "agent_name": "agent", - "agent_role": "agent", - "agent_task": "What monitoring capabilities does Noveum provide?", - "system_prompt": "You are a helpful customer support agent", - "agent_response": "", - "trace": null, - "tools_available": [ - { - "name": "query_routing", - "description": "Routes queries to appropriate handlers", - "args_schema": {}, - "return_schema": { - "type": "routing_decision" - } - } - ], - "tool_calls": [], - "parameters_passed": {}, - "tool_call_results": [], - "retrieval_query": null, - "retrieved_context": null, - "exit_status": "ok", - "agent_exit": false, - "metadata": null - }, - { - "user_id": null, - "task_id": "04bebf38-a343-4563-80db-0154bef8d927", - "turn_id": "2f89c1e0-9b62-4651-914c-b71cf561c33c", - "ground_truth": null, - "expected_tool_call": null, - "agent_name": "agent", - "agent_role": "agent", - "agent_task": "How does Noveum handle agent tracing?", - "system_prompt": "You are a helpful customer support agent", - "agent_response": "", - "trace": null, - "tools_available": [ - { - "name": "query_routing", - "description": "Routes queries to appropriate handlers", - "args_schema": {}, - "return_schema": { - "type": "routing_decision" - } - } - ], - "tool_calls": [], - "parameters_passed": {}, - "tool_call_results": [], - "retrieval_query": null, - "retrieved_context": null, - "exit_status": "ok", - "agent_exit": false, - "metadata": null - }, - { - "user_id": null, - "task_id": "5e043630-6493-42b5-beb8-79faa19bfa37", - "turn_id": "e30e3e61-6edd-4b62-ab77-627faa2bf741", - "ground_truth": null, - "expected_tool_call": null, - "agent_name": "agent", - "agent_role": "agent", - "agent_task": "What APIs are available in Noveum platform?", - "system_prompt": "You are a helpful customer support agent", - "agent_response": "", - "trace": null, - "tools_available": [ - { - "name": "query_routing", - "description": "Routes queries to appropriate handlers", - "args_schema": {}, - "return_schema": { - "type": "routing_decision" - } - } - ], - "tool_calls": [], - "parameters_passed": {}, - "tool_call_results": [], - "retrieval_query": null, - "retrieved_context": null, - "exit_status": "ok", - "agent_exit": false, - "metadata": null - }, - { - "user_id": null, - "task_id": "7da9814d-a2e8-4c4e-b750-68b26bd5fd22", - "turn_id": "2f513795-764f-4eb6-8dd2-108d4cdf8432", - "ground_truth": null, - "expected_tool_call": null, - "agent_name": "agent", - "agent_role": "agent", - "agent_task": "How do I set up observability with Noveum?", - "system_prompt": "You are a helpful customer support agent", - "agent_response": "", - "trace": null, - "tools_available": [ - { - "name": "query_routing", - "description": "Routes queries to appropriate handlers", - "args_schema": {}, - "return_schema": { - "type": "routing_decision" - } - } - ], - "tool_calls": [], - "parameters_passed": {}, - "tool_call_results": [], - "retrieval_query": null, - "retrieved_context": null, - "exit_status": "ok", - "agent_exit": false, - "metadata": null - }, - { - "user_id": null, - "task_id": "16143f74-2831-4753-b33d-ce4b645093c5", - "turn_id": "29c7a515-2383-4b7d-8fc7-beac3f403641", - "ground_truth": null, - "expected_tool_call": null, - "agent_name": "agent", - "agent_role": "agent", - "agent_task": "What features does Noveum Trace offer?", - "system_prompt": "You are a helpful customer support agent", - "agent_response": "", - "trace": null, - "tools_available": [ - { - "name": "query_routing", - "description": "Routes queries to appropriate handlers", - "args_schema": {}, - "return_schema": { - "type": "routing_decision" - } - } - ], - "tool_calls": [], - "parameters_passed": {}, - "tool_call_results": [], - "retrieval_query": null, - "retrieved_context": null, - "exit_status": "ok", - "agent_exit": false, - "metadata": null - }, - { - "user_id": null, - "task_id": "fc64e6cc-6739-4256-ac4a-7b80c3028233", - "turn_id": "12fee687-2a2e-401f-83c2-9c8dc7ecd853", - "ground_truth": null, - "expected_tool_call": null, - "agent_name": "agent", - "agent_role": "agent", - "agent_task": "What are Noveum's pricing plans?", - "system_prompt": "You are a helpful customer support agent", - "agent_response": "", - "trace": null, - "tools_available": [ - { - "name": "query_routing", - "description": "Routes queries to appropriate handlers", - "args_schema": {}, - "return_schema": { - "type": "routing_decision" - } - } - ], - "tool_calls": [], - "parameters_passed": {}, - "tool_call_results": [], - "retrieval_query": null, - "retrieved_context": null, - "exit_status": "ok", - "agent_exit": false, - "metadata": null - }, - { - "user_id": null, - "task_id": "b7945c49-f584-4c70-972d-536a805d8a31", - "turn_id": "9c5aec32-b6eb-4c42-93d5-3970fb118c00", - "ground_truth": null, - "expected_tool_call": null, - "agent_name": "agent", - "agent_role": "agent", - "agent_task": "How do I integrate Noveum Trace in my application?", - "system_prompt": "You are a helpful customer support agent", - "agent_response": "", - "trace": null, - "tools_available": [ - { - "name": "query_routing", - "description": "Routes queries to appropriate handlers", - "args_schema": {}, - "return_schema": { - "type": "routing_decision" - } - } - ], - "tool_calls": [], - "parameters_passed": {}, - "tool_call_results": [], - "retrieval_query": null, - "retrieved_context": null, - "exit_status": "ok", - "agent_exit": false, - "metadata": null - }, - { - "user_id": null, - "task_id": "f5c40ecf-36c0-45ba-9cc9-dc0329b0324b", - "turn_id": "f132d053-f82c-4e66-adf4-bd7ce68b660d", - "ground_truth": null, - "expected_tool_call": null, - "agent_name": "agent", - "agent_role": "agent", - "agent_task": "What is Noveum and what does it do?", - "system_prompt": "You are a helpful customer support agent", - "agent_response": "", - "trace": null, - "tools_available": [ - { - "name": "query_routing", - "description": "Routes queries to appropriate handlers", - "args_schema": {}, - "return_schema": { - "type": "routing_decision" - } - } - ], - "tool_calls": [], - "parameters_passed": {}, - "tool_call_results": [], - "retrieval_query": null, - "retrieved_context": null, - "exit_status": "ok", - "agent_exit": false, - "metadata": null - } -] \ No newline at end of file diff --git a/noveum_customer_support_bt/processed_datasets/agent.rag_evaluation_metrics_dataset_processed_dataset.csv b/noveum_customer_support_bt/processed_datasets/agent.rag_evaluation_metrics_dataset_processed_dataset.csv deleted file mode 100644 index cf8d2b1..0000000 --- a/noveum_customer_support_bt/processed_datasets/agent.rag_evaluation_metrics_dataset_processed_dataset.csv +++ /dev/null @@ -1,22 +0,0 @@ -user_id,task_id,turn_id,ground_truth,expected_tool_call,agent_name,agent_role,agent_task,system_prompt,agent_response,trace,tools_available,tool_calls,parameters_passed,tool_call_results,retrieval_query,retrieved_context,exit_status,agent_exit,metadata -,f1f37bd7-0851-4659-b493-b80d3800d920,ac166548-ffa5-47b9-8cf7-4787f270cb5a,,,agent,agent,Evaluation for query: What are the newest features in cloud computing?,You are a helpful customer support agent,"RAG Answer: The provided context does not specifically mention any new features in cloud computing as a general topic. However, it does highlight several recent enhancements and features related to the Noveum.ai ...",,"[{""name"": ""rag_evaluation"", ""description"": ""Evaluates RAG system performance"", ""args_schema"": {}, ""return_schema"": {""type"": ""evaluation_result""}}]",[],{},[],"[""Evaluation for query: What are the newest features in cloud computing?""]","[[""Context: Source 1 (https://noveum.ai/en/changelog):\n- \ud83d\udcc8 Enhanced Dashboard Analytics: Real-time request tracking, latency monitoring, and cost analysis\n- \ud83d\udd0d Improved Logs Interface: Better search functionality, detailed trace views, and enhanced debugging capabilities\n- \ud83d\udc65 Team Management: Advanced member mana...""]]",ok,False, -,52aacb67-c361-4445-9b72-c157f79f47d6,e193cd4f-cca3-4b94-95d3-a1f2446b9fc6,,,agent,agent,Evaluation for query: What are the current trends in observability tools?,You are a helpful customer support agent,"RAG Answer: The provided context does not explicitly outline current trends in observability tools. However, it highlights some key aspects of observability that are particularly relevant to AI applications. - -1....",,"[{""name"": ""rag_evaluation"", ""description"": ""Evaluates RAG system performance"", ""args_schema"": {}, ""return_schema"": {""type"": ""evaluation_result""}}]",[],{},[],"[""Evaluation for query: What are the current trends in observability tools?""]","[[""Context: Source 1 (https://noveum.ai/en/blog/from-logs-to-intelligent-choices-inside-noveum-ais-evaluation-process):\nFrom Development to Production - Inside Noveum.ai's AI Observability Platform\nShashank Agarwal\n3/3/2025\nIntroduction\nEvery AI application tells a story\u2014but most developers never get to hear it...""]]",ok,False, -,2218f641-604c-491a-9710-b51a9941b982,432e74f3-3291-411b-8d7e-85493868f8cf,,,agent,agent,Evaluation for query: What are the benefits of using Noveum Trace?,You are a helpful customer support agent,"RAG Answer: Using Noveum Trace offers several benefits, including: - -1. **Real-Time Monitoring**: The Noveum platform provides a real-time dashboard for analyzing traces and performance, with live updates as new t...",,"[{""name"": ""rag_evaluation"", ""description"": ""Evaluates RAG system performance"", ""args_schema"": {}, ""return_schema"": {""type"": ""evaluation_result""}}]",[],{},[],"[""Evaluation for query: What are the benefits of using Noveum Trace?""]","[[""Context: Source 1 (https://noveum.ai/docs/platform/dashboard):\n- Bulk Export: Download multiple traces simultaneously\n- Comparative Analysis: Compare performance across multiple traces\nReal-Time Updates\n- Live Refresh: Automatic updates as new traces arrive\n- Connection Monitoring: Real-time status of your t...""]]",ok,False, -,255fd49c-84b4-4b18-887e-6308a412d535,a0475dd5-ef9f-4eb2-bca4-9be72bf6bfd6,,,agent,agent,Evaluation for query: How do I configure Noveum for my system?,You are a helpful customer support agent,"RAG Answer: To configure Noveum for your system, you can refer to the SDK Integration Guide available in the Noveum documentation. This guide will help you trace your AI applications with minimal code changes. Th...",,"[{""name"": ""rag_evaluation"", ""description"": ""Evaluates RAG system performance"", ""args_schema"": {}, ""return_schema"": {""type"": ""evaluation_result""}}]",[],{},[],"[""Evaluation for query: How do I configure Noveum for my system?""]","[[""Context: Source 1 (https://noveum.ai/en/docs):\nWelcome to Noveum.ai Docs\nComprehensive AI tracing and observability for LLM applications, RAG systems, and AI agents\nWelcome to the Noveum.ai documentation! Here you'll find everything you need to integrate, configure, and optimize your AI applications using ou...""]]",ok,False, -,dc511122-c0b6-415c-9a49-c7b45132dd87,ed8ad539-f008-4f1a-beb1-12ad5793950e,,,agent,agent,Evaluation for query: What monitoring capabilities does Noveum provide?,You are a helpful customer support agent,"RAG Answer: Noveum provides comprehensive monitoring capabilities for AI applications, including the ability to monitor, trace, and optimize AI agents across various frameworks such as LangChain, CrewAI, AutoGen,...",,"[{""name"": ""rag_evaluation"", ""description"": ""Evaluates RAG system performance"", ""args_schema"": {}, ""return_schema"": {""type"": ""evaluation_result""}}]",[],{},[],"[""Evaluation for query: What monitoring capabilities does Noveum provide?""]","[[""Context: Source 1 (https://noveum.ai/en):\nMonitor all your AI Agents\nimprove AI Agents today\nNoveum.ai helps you monitor, trace, and optimize your AI applications.\nNoveum.ai works with any AI framework \u2013 LangChain, CrewAI, AutoGen, custom implementations, or direct LLM calls. One dashboard shows everything.\n...""]]",ok,False, -,04bebf38-a343-4563-80db-0154bef8d927,c7358391-9ac1-4312-a4b3-79edfa8bc0d3,,,agent,agent,Evaluation for query: How does Noveum handle agent tracing?,You are a helpful customer support agent,"RAG Answer: Noveum.ai handles agent tracing through its specialized Multi-Agent Tracing capabilities, which are designed to observe complex workflows and inter-agent communications within multi-agent systems. The...",,"[{""name"": ""rag_evaluation"", ""description"": ""Evaluates RAG system performance"", ""args_schema"": {}, ""return_schema"": {""type"": ""evaluation_result""}}]",[],{},[],"[""Evaluation for query: How does Noveum handle agent tracing?""]","[[""Context: Source 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, vector searches, data retrieval, agent reasoning, and more. Without proper observability, debugging ...""]]",ok,False, -,5e043630-6493-42b5-beb8-79faa19bfa37,6b21de5d-456f-4501-953f-0ce5bc4fa5a5,,,agent,agent,Evaluation for query: What APIs are available in Noveum platform?,You are a helpful customer support agent,"RAG Answer: The provided context does not specify the APIs available in the Noveum platform. For detailed information about the APIs, I recommend checking the official Noveum documentation or the SDK Integration ...",,"[{""name"": ""rag_evaluation"", ""description"": ""Evaluates RAG system performance"", ""args_schema"": {}, ""return_schema"": {""type"": ""evaluation_result""}}]",[],{},[],"[""Evaluation for query: What APIs are available in Noveum platform?""]","[[""Context: Source 1 (https://noveum.ai/en/docs):\n- \ud83c\udf0d Universal: Works across frameworks, clouds, and deployment models\n\ud83e\udd1d Community & Support\n- \ud83d\udcac Discord Community: Join our Discord\n- \ud83d\udce7 Email Support: [email protected]\n- \ud83d\udc1b Bug Reports: GitHub Issues\n- \ud83d\udcd6 Knowledge Base: Help Center\nReady to get started? Head to ...""]]",ok,False, -,7da9814d-a2e8-4c4e-b750-68b26bd5fd22,2ad4e155-7b76-4040-923d-d6b65a97a45b,,,agent,agent,Evaluation for query: How do I set up observability with Noveum?,You are a helpful customer support agent,"RAG Answer: To set up observability with Noveum, follow these steps: - -1. **Sign Up**: Go to [noveum.ai](https://noveum.ai) and create an account. -2. **Create a Project**: Once signed in, create a new project to o...",,"[{""name"": ""rag_evaluation"", ""description"": ""Evaluates RAG system performance"", ""args_schema"": {}, ""return_schema"": {""type"": ""evaluation_result""}}]",[],{},[],"[""Evaluation for query: How do I set up observability with Noveum?""]","[[""Context: Source 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\n- Production Ready: Built for scale with intelligent sampling and batching\n- Privacy Focused: You control what data is captured and how it's stored\n- Open Source SDKs: Transparent, auditable, and extensible\nGetting ...""]]",ok,False, -,16143f74-2831-4753-b33d-ce4b645093c5,6b822089-be40-457d-9fa6-0979bafdfb85,,,agent,agent,Evaluation for query: What features does Noveum Trace offer?,You are a helpful customer support agent,"RAG Answer: Noveum Trace offers several key features through its Python SDK (`noveum-trace`) and TypeScript SDK (`@noveum/trace`): - -1. **Decorator-based Tracing**: It provides a simple way to trace LLM calls, age...",,"[{""name"": ""rag_evaluation"", ""description"": ""Evaluates RAG system performance"", ""args_schema"": {}, ""return_schema"": {""type"": ""evaluation_result""}}]",[],{},[],"[""Evaluation for query: What features does Noveum Trace offer?""]","[[""Context: Source 1 (https://noveum.ai/en/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\n\ud83d\udcd8 TypeScript SDK (\n@noveum/trace\n)- Framework ...""]]",ok,False, -,fc64e6cc-6739-4256-ac4a-7b80c3028233,08db68f6-3623-44b5-840b-41400fac7333,,,agent,agent,Evaluation for query: What are Noveum's pricing plans?,You are a helpful customer support agent,"RAG Answer: The provided context does not include specific information about Noveum's pricing plans. However, it mentions that all users will have access to the Observability suite for free, and early users will ...",,"[{""name"": ""rag_evaluation"", ""description"": ""Evaluates RAG system performance"", ""args_schema"": {}, ""return_schema"": {""type"": ""evaluation_result""}}]",[],{},[],"[""Evaluation for query: What are Noveum's pricing plans?""]","[[""Context: Source 1 (https://noveum.ai/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....""]]",ok,False, -,b7945c49-f584-4c70-972d-536a805d8a31,6a57c92e-e473-425a-978b-19dbe976a46e,,,agent,agent,Evaluation for query: How do I integrate Noveum Trace in my application?,You are a helpful customer support agent,"RAG Answer: To integrate Noveum Trace into your application, you can use either the Python SDK (`noveum-trace`) or the TypeScript SDK (`@noveum/trace`). Hereā€™s a quick start guide based on the provided documentat...",,"[{""name"": ""rag_evaluation"", ""description"": ""Evaluates RAG system performance"", ""args_schema"": {}, ""return_schema"": {""type"": ""evaluation_result""}}]",[],{},[],"[""Evaluation for query: How do I integrate Noveum Trace in my application?""]","[[""Context: Source 1 (https://noveum.ai/en/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\n\ud83d\udcd8 TypeScript SDK (\n@noveum/trace\n)- Framework ...""]]",ok,False, -,f5c40ecf-36c0-45ba-9cc9-dc0329b0324b,efb19490-bc9e-48dc-a365-2f839100e41a,,,agent,agent,Evaluation for query: What is Noveum and what does it do?,You are a helpful customer support agent,"RAG Answer: Noveum.ai is a comprehensive tracing and observability platform specifically designed for AI applications, including those powered by large language models (LLMs), retrieval-augmented generation (RAG)...",,"[{""name"": ""rag_evaluation"", ""description"": ""Evaluates RAG system performance"", ""args_schema"": {}, ""return_schema"": {""type"": ""evaluation_result""}}]",[],{},[],"[""Evaluation for query: What is Noveum and what does it do?""]","[[""Context: Source 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, vector searches, data retrieval, agent reasoning, and more. Without proper observability, debugging ...""]]",ok,False, diff --git a/noveum_customer_support_bt/processed_datasets/agent.rag_evaluation_metrics_dataset_processed_dataset.json b/noveum_customer_support_bt/processed_datasets/agent.rag_evaluation_metrics_dataset_processed_dataset.json deleted file mode 100644 index 78aad41..0000000 --- a/noveum_customer_support_bt/processed_datasets/agent.rag_evaluation_metrics_dataset_processed_dataset.json +++ /dev/null @@ -1,446 +0,0 @@ -[ - { - "user_id": null, - "task_id": "f1f37bd7-0851-4659-b493-b80d3800d920", - "turn_id": "ac166548-ffa5-47b9-8cf7-4787f270cb5a", - "ground_truth": null, - "expected_tool_call": null, - "agent_name": "agent", - "agent_role": "agent", - "agent_task": "Evaluation for query: What are the newest features in cloud computing?", - "system_prompt": "You are a helpful customer support agent", - "agent_response": "RAG Answer: The provided context does not specifically mention any new features in cloud computing as a general topic. However, it does highlight several recent enhancements and features related to the Noveum.ai ...", - "trace": null, - "tools_available": [ - { - "name": "rag_evaluation", - "description": "Evaluates RAG system performance", - "args_schema": {}, - "return_schema": { - "type": "evaluation_result" - } - } - ], - "tool_calls": [], - "parameters_passed": {}, - "tool_call_results": [], - "retrieval_query": [ - "Evaluation for query: What are the newest features in cloud computing?" - ], - "retrieved_context": [ - [ - "Context: Source 1 (https://noveum.ai/en/changelog):\n- šŸ“ˆ Enhanced Dashboard Analytics: Real-time request tracking, latency monitoring, and cost analysis\n- šŸ” Improved Logs Interface: Better search functionality, detailed trace views, and enhanced debugging capabilities\n- šŸ‘„ Team Management: Advanced member mana..." - ] - ], - "exit_status": "ok", - "agent_exit": false, - "metadata": null - }, - { - "user_id": null, - "task_id": "52aacb67-c361-4445-9b72-c157f79f47d6", - "turn_id": "e193cd4f-cca3-4b94-95d3-a1f2446b9fc6", - "ground_truth": null, - "expected_tool_call": null, - "agent_name": "agent", - "agent_role": "agent", - "agent_task": "Evaluation for query: What are the current trends in observability tools?", - "system_prompt": "You are a helpful customer support agent", - "agent_response": "RAG Answer: The provided context does not explicitly outline current trends in observability tools. However, it highlights some key aspects of observability that are particularly relevant to AI applications. \n\n1....", - "trace": null, - "tools_available": [ - { - "name": "rag_evaluation", - "description": "Evaluates RAG system performance", - "args_schema": {}, - "return_schema": { - "type": "evaluation_result" - } - } - ], - "tool_calls": [], - "parameters_passed": {}, - "tool_call_results": [], - "retrieval_query": [ - "Evaluation for query: What are the current trends in observability tools?" - ], - "retrieved_context": [ - [ - "Context: Source 1 (https://noveum.ai/en/blog/from-logs-to-intelligent-choices-inside-noveum-ais-evaluation-process):\nFrom Development to Production - Inside Noveum.ai's AI Observability Platform\nShashank Agarwal\n3/3/2025\nIntroduction\nEvery AI application tells a storyā€”but most developers never get to hear it..." - ] - ], - "exit_status": "ok", - "agent_exit": false, - "metadata": null - }, - { - "user_id": null, - "task_id": "2218f641-604c-491a-9710-b51a9941b982", - "turn_id": "432e74f3-3291-411b-8d7e-85493868f8cf", - "ground_truth": null, - "expected_tool_call": null, - "agent_name": "agent", - "agent_role": "agent", - "agent_task": "Evaluation for query: What are the benefits of using Noveum Trace?", - "system_prompt": "You are a helpful customer support agent", - "agent_response": "RAG Answer: Using Noveum Trace offers several benefits, including:\n\n1. **Real-Time Monitoring**: The Noveum platform provides a real-time dashboard for analyzing traces and performance, with live updates as new t...", - "trace": null, - "tools_available": [ - { - "name": "rag_evaluation", - "description": "Evaluates RAG system performance", - "args_schema": {}, - "return_schema": { - "type": "evaluation_result" - } - } - ], - "tool_calls": [], - "parameters_passed": {}, - "tool_call_results": [], - "retrieval_query": [ - "Evaluation for query: What are the benefits of using Noveum Trace?" - ], - "retrieved_context": [ - [ - "Context: Source 1 (https://noveum.ai/docs/platform/dashboard):\n- Bulk Export: Download multiple traces simultaneously\n- Comparative Analysis: Compare performance across multiple traces\nReal-Time Updates\n- Live Refresh: Automatic updates as new traces arrive\n- Connection Monitoring: Real-time status of your t..." - ] - ], - "exit_status": "ok", - "agent_exit": false, - "metadata": null - }, - { - "user_id": null, - "task_id": "255fd49c-84b4-4b18-887e-6308a412d535", - "turn_id": "a0475dd5-ef9f-4eb2-bca4-9be72bf6bfd6", - "ground_truth": null, - "expected_tool_call": null, - "agent_name": "agent", - "agent_role": "agent", - "agent_task": "Evaluation for query: How do I configure Noveum for my system?", - "system_prompt": "You are a helpful customer support agent", - "agent_response": "RAG Answer: To configure Noveum for your system, you can refer to the SDK Integration Guide available in the Noveum documentation. This guide will help you trace your AI applications with minimal code changes. Th...", - "trace": null, - "tools_available": [ - { - "name": "rag_evaluation", - "description": "Evaluates RAG system performance", - "args_schema": {}, - "return_schema": { - "type": "evaluation_result" - } - } - ], - "tool_calls": [], - "parameters_passed": {}, - "tool_call_results": [], - "retrieval_query": [ - "Evaluation for query: How do I configure Noveum for my system?" - ], - "retrieved_context": [ - [ - "Context: Source 1 (https://noveum.ai/en/docs):\nWelcome to Noveum.ai Docs\nComprehensive AI tracing and observability for LLM applications, RAG systems, and AI agents\nWelcome to the Noveum.ai documentation! Here you'll find everything you need to integrate, configure, and optimize your AI applications using ou..." - ] - ], - "exit_status": "ok", - "agent_exit": false, - "metadata": null - }, - { - "user_id": null, - "task_id": "dc511122-c0b6-415c-9a49-c7b45132dd87", - "turn_id": "ed8ad539-f008-4f1a-beb1-12ad5793950e", - "ground_truth": null, - "expected_tool_call": null, - "agent_name": "agent", - "agent_role": "agent", - "agent_task": "Evaluation for query: What monitoring capabilities does Noveum provide?", - "system_prompt": "You are a helpful customer support agent", - "agent_response": "RAG Answer: Noveum provides comprehensive monitoring capabilities for AI applications, including the ability to monitor, trace, and optimize AI agents across various frameworks such as LangChain, CrewAI, AutoGen,...", - "trace": null, - "tools_available": [ - { - "name": "rag_evaluation", - "description": "Evaluates RAG system performance", - "args_schema": {}, - "return_schema": { - "type": "evaluation_result" - } - } - ], - "tool_calls": [], - "parameters_passed": {}, - "tool_call_results": [], - "retrieval_query": [ - "Evaluation for query: What monitoring capabilities does Noveum provide?" - ], - "retrieved_context": [ - [ - "Context: Source 1 (https://noveum.ai/en):\nMonitor all your AI Agents\nimprove AI Agents today\nNoveum.ai helps you monitor, trace, and optimize your AI applications.\nNoveum.ai works with any AI framework ā€“ LangChain, CrewAI, AutoGen, custom implementations, or direct LLM calls. One dashboard shows everything.\n..." - ] - ], - "exit_status": "ok", - "agent_exit": false, - "metadata": null - }, - { - "user_id": null, - "task_id": "04bebf38-a343-4563-80db-0154bef8d927", - "turn_id": "c7358391-9ac1-4312-a4b3-79edfa8bc0d3", - "ground_truth": null, - "expected_tool_call": null, - "agent_name": "agent", - "agent_role": "agent", - "agent_task": "Evaluation for query: How does Noveum handle agent tracing?", - "system_prompt": "You are a helpful customer support agent", - "agent_response": "RAG Answer: Noveum.ai handles agent tracing through its specialized Multi-Agent Tracing capabilities, which are designed to observe complex workflows and inter-agent communications within multi-agent systems. The...", - "trace": null, - "tools_available": [ - { - "name": "rag_evaluation", - "description": "Evaluates RAG system performance", - "args_schema": {}, - "return_schema": { - "type": "evaluation_result" - } - } - ], - "tool_calls": [], - "parameters_passed": {}, - "tool_call_results": [], - "retrieval_query": [ - "Evaluation for query: How does Noveum handle agent tracing?" - ], - "retrieved_context": [ - [ - "Context: Source 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, vector searches, data retrieval, agent reasoning, and more. Without proper observability, debugging ..." - ] - ], - "exit_status": "ok", - "agent_exit": false, - "metadata": null - }, - { - "user_id": null, - "task_id": "5e043630-6493-42b5-beb8-79faa19bfa37", - "turn_id": "6b21de5d-456f-4501-953f-0ce5bc4fa5a5", - "ground_truth": null, - "expected_tool_call": null, - "agent_name": "agent", - "agent_role": "agent", - "agent_task": "Evaluation for query: What APIs are available in Noveum platform?", - "system_prompt": "You are a helpful customer support agent", - "agent_response": "RAG Answer: The provided context does not specify the APIs available in the Noveum platform. For detailed information about the APIs, I recommend checking the official Noveum documentation or the SDK Integration ...", - "trace": null, - "tools_available": [ - { - "name": "rag_evaluation", - "description": "Evaluates RAG system performance", - "args_schema": {}, - "return_schema": { - "type": "evaluation_result" - } - } - ], - "tool_calls": [], - "parameters_passed": {}, - "tool_call_results": [], - "retrieval_query": [ - "Evaluation for query: What APIs are available in Noveum platform?" - ], - "retrieved_context": [ - [ - "Context: Source 1 (https://noveum.ai/en/docs):\n- šŸŒ Universal: Works across frameworks, clouds, and deployment models\nšŸ¤ Community & Support\n- šŸ’¬ Discord Community: Join our Discord\n- šŸ“§ Email Support: [email protected]\n- šŸ› Bug Reports: GitHub Issues\n- šŸ“– Knowledge Base: Help Center\nReady to get started? Head to ..." - ] - ], - "exit_status": "ok", - "agent_exit": false, - "metadata": null - }, - { - "user_id": null, - "task_id": "7da9814d-a2e8-4c4e-b750-68b26bd5fd22", - "turn_id": "2ad4e155-7b76-4040-923d-d6b65a97a45b", - "ground_truth": null, - "expected_tool_call": null, - "agent_name": "agent", - "agent_role": "agent", - "agent_task": "Evaluation for query: How do I set up observability with Noveum?", - "system_prompt": "You are a helpful customer support agent", - "agent_response": "RAG Answer: To set up observability with Noveum, follow these steps:\n\n1. **Sign Up**: Go to [noveum.ai](https://noveum.ai) and create an account.\n2. **Create a Project**: Once signed in, create a new project to o...", - "trace": null, - "tools_available": [ - { - "name": "rag_evaluation", - "description": "Evaluates RAG system performance", - "args_schema": {}, - "return_schema": { - "type": "evaluation_result" - } - } - ], - "tool_calls": [], - "parameters_passed": {}, - "tool_call_results": [], - "retrieval_query": [ - "Evaluation for query: How do I set up observability with Noveum?" - ], - "retrieved_context": [ - [ - "Context: Source 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\n- Production Ready: Built for scale with intelligent sampling and batching\n- Privacy Focused: You control what data is captured and how it's stored\n- Open Source SDKs: Transparent, auditable, and extensible\nGetting ..." - ] - ], - "exit_status": "ok", - "agent_exit": false, - "metadata": null - }, - { - "user_id": null, - "task_id": "16143f74-2831-4753-b33d-ce4b645093c5", - "turn_id": "6b822089-be40-457d-9fa6-0979bafdfb85", - "ground_truth": null, - "expected_tool_call": null, - "agent_name": "agent", - "agent_role": "agent", - "agent_task": "Evaluation for query: What features does Noveum Trace offer?", - "system_prompt": "You are a helpful customer support agent", - "agent_response": "RAG Answer: Noveum Trace offers several key features through its Python SDK (`noveum-trace`) and TypeScript SDK (`@noveum/trace`):\n\n1. **Decorator-based Tracing**: It provides a simple way to trace LLM calls, age...", - "trace": null, - "tools_available": [ - { - "name": "rag_evaluation", - "description": "Evaluates RAG system performance", - "args_schema": {}, - "return_schema": { - "type": "evaluation_result" - } - } - ], - "tool_calls": [], - "parameters_passed": {}, - "tool_call_results": [], - "retrieval_query": [ - "Evaluation for query: What features does Noveum Trace offer?" - ], - "retrieved_context": [ - [ - "Context: Source 1 (https://noveum.ai/en/docs):\nCore Components\n-\nšŸ Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\nšŸ“˜ TypeScript SDK (\n@noveum/trace\n)- Framework ..." - ] - ], - "exit_status": "ok", - "agent_exit": false, - "metadata": null - }, - { - "user_id": null, - "task_id": "fc64e6cc-6739-4256-ac4a-7b80c3028233", - "turn_id": "08db68f6-3623-44b5-840b-41400fac7333", - "ground_truth": null, - "expected_tool_call": null, - "agent_name": "agent", - "agent_role": "agent", - "agent_task": "Evaluation for query: What are Noveum's pricing plans?", - "system_prompt": "You are a helpful customer support agent", - "agent_response": "RAG Answer: The provided context does not include specific information about Noveum's pricing plans. However, it mentions that all users will have access to the Observability suite for free, and early users will ...", - "trace": null, - "tools_available": [ - { - "name": "rag_evaluation", - "description": "Evaluates RAG system performance", - "args_schema": {}, - "return_schema": { - "type": "evaluation_result" - } - } - ], - "tool_calls": [], - "parameters_passed": {}, - "tool_call_results": [], - "retrieval_query": [ - "Evaluation for query: What are Noveum's pricing plans?" - ], - "retrieved_context": [ - [ - "Context: Source 1 (https://noveum.ai/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year...." - ] - ], - "exit_status": "ok", - "agent_exit": false, - "metadata": null - }, - { - "user_id": null, - "task_id": "b7945c49-f584-4c70-972d-536a805d8a31", - "turn_id": "6a57c92e-e473-425a-978b-19dbe976a46e", - "ground_truth": null, - "expected_tool_call": null, - "agent_name": "agent", - "agent_role": "agent", - "agent_task": "Evaluation for query: How do I integrate Noveum Trace in my application?", - "system_prompt": "You are a helpful customer support agent", - "agent_response": "RAG Answer: To integrate Noveum Trace into your application, you can use either the Python SDK (`noveum-trace`) or the TypeScript SDK (`@noveum/trace`). Hereā€™s a quick start guide based on the provided documentat...", - "trace": null, - "tools_available": [ - { - "name": "rag_evaluation", - "description": "Evaluates RAG system performance", - "args_schema": {}, - "return_schema": { - "type": "evaluation_result" - } - } - ], - "tool_calls": [], - "parameters_passed": {}, - "tool_call_results": [], - "retrieval_query": [ - "Evaluation for query: How do I integrate Noveum Trace in my application?" - ], - "retrieved_context": [ - [ - "Context: Source 1 (https://noveum.ai/en/docs):\nCore Components\n-\nšŸ Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\nšŸ“˜ TypeScript SDK (\n@noveum/trace\n)- Framework ..." - ] - ], - "exit_status": "ok", - "agent_exit": false, - "metadata": null - }, - { - "user_id": null, - "task_id": "f5c40ecf-36c0-45ba-9cc9-dc0329b0324b", - "turn_id": "efb19490-bc9e-48dc-a365-2f839100e41a", - "ground_truth": null, - "expected_tool_call": null, - "agent_name": "agent", - "agent_role": "agent", - "agent_task": "Evaluation for query: What is Noveum and what does it do?", - "system_prompt": "You are a helpful customer support agent", - "agent_response": "RAG Answer: Noveum.ai is a comprehensive tracing and observability platform specifically designed for AI applications, including those powered by large language models (LLMs), retrieval-augmented generation (RAG)...", - "trace": null, - "tools_available": [ - { - "name": "rag_evaluation", - "description": "Evaluates RAG system performance", - "args_schema": {}, - "return_schema": { - "type": "evaluation_result" - } - } - ], - "tool_calls": [], - "parameters_passed": {}, - "tool_call_results": [], - "retrieval_query": [ - "Evaluation for query: What is Noveum and what does it do?" - ], - "retrieved_context": [ - [ - "Context: Source 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, vector searches, data retrieval, agent reasoning, and more. Without proper observability, debugging ..." - ] - ], - "exit_status": "ok", - "agent_exit": false, - "metadata": null - } -] \ No newline at end of file diff --git a/noveum_customer_support_bt/processed_datasets/agent.routing_evaluation_metrics_dataset_processed_dataset.csv b/noveum_customer_support_bt/processed_datasets/agent.routing_evaluation_metrics_dataset_processed_dataset.csv deleted file mode 100644 index 42daeec..0000000 --- a/noveum_customer_support_bt/processed_datasets/agent.routing_evaluation_metrics_dataset_processed_dataset.csv +++ /dev/null @@ -1,21 +0,0 @@ -user_id,task_id,turn_id,ground_truth,expected_tool_call,agent_name,agent_role,agent_task,system_prompt,agent_response,trace,tools_available,tool_calls,parameters_passed,tool_call_results,retrieval_query,retrieved_context,exit_status,agent_exit,metadata -,eda4fe22-9a2b-4b73-856b-f4f3309bf719,95dac0c3-83fd-459c-af4e-c6a1734fd874,,,agent,agent,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, -,0ffffba1-8a37-443c-8866-d53ffbfa7718,516060a3-0f58-46ba-8a6a-4b502a376492,,,agent,agent,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, -,f1f37bd7-0851-4659-b493-b80d3800d920,8f8d2dc4-150f-4e43-aad3-96da4d06f586,,,agent,agent,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, -,43cdf081-4f01-49cd-b566-dbd1619e6cd2,cf0a915c-668b-412e-9c14-69119d059dc2,,,agent,agent,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, -,9a1983f4-09da-4b53-80e6-38de6878e0e7,f75b51b8-4882-4170-bc08-2adcab250f50,,,agent,agent,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, -,5d2517e1-220a-429d-9d59-f701bda25eed,4180abcf-924c-45ac-8dfa-17c0bc603eff,,,agent,agent,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, -,52aacb67-c361-4445-9b72-c157f79f47d6,8b0022cc-91dd-4065-91ed-b77217f92bd7,,,agent,agent,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, -,a81ca3a8-80aa-4c39-876e-8d40ea7a0aef,780df186-fe5a-4742-b937-2a142ad528c6,,,agent,agent,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, -,230aad27-f3dd-4968-a45a-3c2f07ac28ed,f80ae3e8-b4dd-42c6-b17b-de7805cedbb9,,,agent,agent,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, -,83c7dcce-3d89-4da1-8b3f-d419885d4cbc,209259cf-411d-4b97-8f7b-40b2530eec81,,,agent,agent,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, -,2218f641-604c-491a-9710-b51a9941b982,c3f3bd32-9954-443f-93e5-52aa6cf412a0,,,agent,agent,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, -,255fd49c-84b4-4b18-887e-6308a412d535,940ce621-0015-45ed-adc6-c4caa2a95e81,,,agent,agent,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, -,dc511122-c0b6-415c-9a49-c7b45132dd87,ea28f049-6bc9-48b4-8386-279abe087ca8,,,agent,agent,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, -,04bebf38-a343-4563-80db-0154bef8d927,01a03da5-64df-41e3-b9c9-556e12a8edd7,,,agent,agent,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, -,5e043630-6493-42b5-beb8-79faa19bfa37,8d9c632b-b2e8-496d-b1b0-a89e780ef810,,,agent,agent,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, -,7da9814d-a2e8-4c4e-b750-68b26bd5fd22,72dfbba4-0672-4adc-a1bf-308157d831e6,,,agent,agent,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, -,16143f74-2831-4753-b33d-ce4b645093c5,92c81ee2-728d-4ea6-bc52-b81295d2aea2,,,agent,agent,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, -,fc64e6cc-6739-4256-ac4a-7b80c3028233,807627b0-41d9-478f-8a8b-26540d70af68,,,agent,agent,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, -,b7945c49-f584-4c70-972d-536a805d8a31,bf860672-af22-44c1-b13b-0eabe8f72503,,,agent,agent,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, -,f5c40ecf-36c0-45ba-9cc9-dc0329b0324b,5fa714f1-eeb9-4617-bc7e-67062e6d1eab,,,agent,agent,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, diff --git a/noveum_customer_support_bt/processed_datasets/agent.routing_evaluation_metrics_dataset_processed_dataset.json b/noveum_customer_support_bt/processed_datasets/agent.routing_evaluation_metrics_dataset_processed_dataset.json deleted file mode 100644 index 295df2d..0000000 --- a/noveum_customer_support_bt/processed_datasets/agent.routing_evaluation_metrics_dataset_processed_dataset.json +++ /dev/null @@ -1,442 +0,0 @@ -[ - { - "user_id": null, - "task_id": "eda4fe22-9a2b-4b73-856b-f4f3309bf719", - "turn_id": "95dac0c3-83fd-459c-af4e-c6a1734fd874", - "ground_truth": null, - "expected_tool_call": null, - "agent_name": "agent", - "agent_role": "agent", - "agent_task": null, - "system_prompt": "You are a helpful customer support agent", - "agent_response": null, - "trace": null, - "tools_available": [], - "tool_calls": [], - "parameters_passed": {}, - "tool_call_results": [], - "retrieval_query": null, - "retrieved_context": null, - "exit_status": "ok", - "agent_exit": false, - "metadata": null - }, - { - "user_id": null, - "task_id": "0ffffba1-8a37-443c-8866-d53ffbfa7718", - "turn_id": "516060a3-0f58-46ba-8a6a-4b502a376492", - "ground_truth": null, - "expected_tool_call": null, - "agent_name": "agent", - "agent_role": "agent", - "agent_task": null, - "system_prompt": "You are a helpful customer support agent", - "agent_response": null, - "trace": null, - "tools_available": [], - "tool_calls": [], - "parameters_passed": {}, - "tool_call_results": [], - "retrieval_query": null, - "retrieved_context": null, - "exit_status": "ok", - "agent_exit": false, - "metadata": null - }, - { - "user_id": null, - "task_id": "f1f37bd7-0851-4659-b493-b80d3800d920", - "turn_id": "8f8d2dc4-150f-4e43-aad3-96da4d06f586", - "ground_truth": null, - "expected_tool_call": null, - "agent_name": "agent", - "agent_role": "agent", - "agent_task": null, - "system_prompt": "You are a helpful customer support agent", - "agent_response": null, - "trace": null, - "tools_available": [], - "tool_calls": [], - "parameters_passed": {}, - "tool_call_results": [], - "retrieval_query": null, - "retrieved_context": null, - "exit_status": "ok", - "agent_exit": false, - "metadata": null - }, - { - "user_id": null, - "task_id": "43cdf081-4f01-49cd-b566-dbd1619e6cd2", - "turn_id": "cf0a915c-668b-412e-9c14-69119d059dc2", - "ground_truth": null, - "expected_tool_call": null, - "agent_name": "agent", - "agent_role": "agent", - "agent_task": null, - "system_prompt": "You are a helpful customer support agent", - "agent_response": null, - "trace": null, - "tools_available": [], - "tool_calls": [], - "parameters_passed": {}, - "tool_call_results": [], - "retrieval_query": null, - "retrieved_context": null, - "exit_status": "ok", - "agent_exit": false, - "metadata": null - }, - { - "user_id": null, - "task_id": "9a1983f4-09da-4b53-80e6-38de6878e0e7", - "turn_id": "f75b51b8-4882-4170-bc08-2adcab250f50", - "ground_truth": null, - "expected_tool_call": null, - "agent_name": "agent", - "agent_role": "agent", - "agent_task": null, - "system_prompt": "You are a helpful customer support agent", - "agent_response": null, - "trace": null, - "tools_available": [], - "tool_calls": [], - "parameters_passed": {}, - "tool_call_results": [], - "retrieval_query": null, - "retrieved_context": null, - "exit_status": "ok", - "agent_exit": false, - "metadata": null - }, - { - "user_id": null, - "task_id": "5d2517e1-220a-429d-9d59-f701bda25eed", - "turn_id": "4180abcf-924c-45ac-8dfa-17c0bc603eff", - "ground_truth": null, - "expected_tool_call": null, - "agent_name": "agent", - "agent_role": "agent", - "agent_task": null, - "system_prompt": "You are a helpful customer support agent", - "agent_response": null, - "trace": null, - "tools_available": [], - "tool_calls": [], - "parameters_passed": {}, - "tool_call_results": [], - "retrieval_query": null, - "retrieved_context": null, - "exit_status": "ok", - "agent_exit": false, - "metadata": null - }, - { - "user_id": null, - "task_id": "52aacb67-c361-4445-9b72-c157f79f47d6", - "turn_id": "8b0022cc-91dd-4065-91ed-b77217f92bd7", - "ground_truth": null, - "expected_tool_call": null, - "agent_name": "agent", - "agent_role": "agent", - "agent_task": null, - "system_prompt": "You are a helpful customer support agent", - "agent_response": null, - "trace": null, - "tools_available": [], - "tool_calls": [], - "parameters_passed": {}, - "tool_call_results": [], - "retrieval_query": null, - "retrieved_context": null, - "exit_status": "ok", - "agent_exit": false, - "metadata": null - }, - { - "user_id": null, - "task_id": "a81ca3a8-80aa-4c39-876e-8d40ea7a0aef", - "turn_id": "780df186-fe5a-4742-b937-2a142ad528c6", - "ground_truth": null, - "expected_tool_call": null, - "agent_name": "agent", - "agent_role": "agent", - "agent_task": null, - "system_prompt": "You are a helpful customer support agent", - "agent_response": null, - "trace": null, - "tools_available": [], - "tool_calls": [], - "parameters_passed": {}, - "tool_call_results": [], - "retrieval_query": null, - "retrieved_context": null, - "exit_status": "ok", - "agent_exit": false, - "metadata": null - }, - { - "user_id": null, - "task_id": "230aad27-f3dd-4968-a45a-3c2f07ac28ed", - "turn_id": "f80ae3e8-b4dd-42c6-b17b-de7805cedbb9", - "ground_truth": null, - "expected_tool_call": null, - "agent_name": "agent", - "agent_role": "agent", - "agent_task": null, - "system_prompt": "You are a helpful customer support agent", - "agent_response": null, - "trace": null, - "tools_available": [], - "tool_calls": [], - "parameters_passed": {}, - "tool_call_results": [], - "retrieval_query": null, - "retrieved_context": null, - "exit_status": "ok", - "agent_exit": false, - "metadata": null - }, - { - "user_id": null, - "task_id": "83c7dcce-3d89-4da1-8b3f-d419885d4cbc", - "turn_id": "209259cf-411d-4b97-8f7b-40b2530eec81", - "ground_truth": null, - "expected_tool_call": null, - "agent_name": "agent", - "agent_role": "agent", - "agent_task": null, - "system_prompt": "You are a helpful customer support agent", - "agent_response": null, - "trace": null, - "tools_available": [], - "tool_calls": [], - "parameters_passed": {}, - "tool_call_results": [], - "retrieval_query": null, - "retrieved_context": null, - "exit_status": "ok", - "agent_exit": false, - "metadata": null - }, - { - "user_id": null, - "task_id": "2218f641-604c-491a-9710-b51a9941b982", - "turn_id": "c3f3bd32-9954-443f-93e5-52aa6cf412a0", - "ground_truth": null, - "expected_tool_call": null, - "agent_name": "agent", - "agent_role": "agent", - "agent_task": null, - "system_prompt": "You are a helpful customer support agent", - "agent_response": null, - "trace": null, - "tools_available": [], - "tool_calls": [], - "parameters_passed": {}, - "tool_call_results": [], - "retrieval_query": null, - "retrieved_context": null, - "exit_status": "ok", - "agent_exit": false, - "metadata": null - }, - { - "user_id": null, - "task_id": "255fd49c-84b4-4b18-887e-6308a412d535", - "turn_id": "940ce621-0015-45ed-adc6-c4caa2a95e81", - "ground_truth": null, - "expected_tool_call": null, - "agent_name": "agent", - "agent_role": "agent", - "agent_task": null, - "system_prompt": "You are a helpful customer support agent", - "agent_response": null, - "trace": null, - "tools_available": [], - "tool_calls": [], - "parameters_passed": {}, - "tool_call_results": [], - "retrieval_query": null, - "retrieved_context": null, - "exit_status": "ok", - "agent_exit": false, - "metadata": null - }, - { - "user_id": null, - "task_id": "dc511122-c0b6-415c-9a49-c7b45132dd87", - "turn_id": "ea28f049-6bc9-48b4-8386-279abe087ca8", - "ground_truth": null, - "expected_tool_call": null, - "agent_name": "agent", - "agent_role": "agent", - "agent_task": null, - "system_prompt": "You are a helpful customer support agent", - "agent_response": null, - "trace": null, - "tools_available": [], - "tool_calls": [], - "parameters_passed": {}, - "tool_call_results": [], - "retrieval_query": null, - "retrieved_context": null, - "exit_status": "ok", - "agent_exit": false, - "metadata": null - }, - { - "user_id": null, - "task_id": "04bebf38-a343-4563-80db-0154bef8d927", - "turn_id": "01a03da5-64df-41e3-b9c9-556e12a8edd7", - "ground_truth": null, - "expected_tool_call": null, - "agent_name": "agent", - "agent_role": "agent", - "agent_task": null, - "system_prompt": "You are a helpful customer support agent", - "agent_response": null, - "trace": null, - "tools_available": [], - "tool_calls": [], - "parameters_passed": {}, - "tool_call_results": [], - "retrieval_query": null, - "retrieved_context": null, - "exit_status": "ok", - "agent_exit": false, - "metadata": null - }, - { - "user_id": null, - "task_id": "5e043630-6493-42b5-beb8-79faa19bfa37", - "turn_id": "8d9c632b-b2e8-496d-b1b0-a89e780ef810", - "ground_truth": null, - "expected_tool_call": null, - "agent_name": "agent", - "agent_role": "agent", - "agent_task": null, - "system_prompt": "You are a helpful customer support agent", - "agent_response": null, - "trace": null, - "tools_available": [], - "tool_calls": [], - "parameters_passed": {}, - "tool_call_results": [], - "retrieval_query": null, - "retrieved_context": null, - "exit_status": "ok", - "agent_exit": false, - "metadata": null - }, - { - "user_id": null, - "task_id": "7da9814d-a2e8-4c4e-b750-68b26bd5fd22", - "turn_id": "72dfbba4-0672-4adc-a1bf-308157d831e6", - "ground_truth": null, - "expected_tool_call": null, - "agent_name": "agent", - "agent_role": "agent", - "agent_task": null, - "system_prompt": "You are a helpful customer support agent", - "agent_response": null, - "trace": null, - "tools_available": [], - "tool_calls": [], - "parameters_passed": {}, - "tool_call_results": [], - "retrieval_query": null, - "retrieved_context": null, - "exit_status": "ok", - "agent_exit": false, - "metadata": null - }, - { - "user_id": null, - "task_id": "16143f74-2831-4753-b33d-ce4b645093c5", - "turn_id": "92c81ee2-728d-4ea6-bc52-b81295d2aea2", - "ground_truth": null, - "expected_tool_call": null, - "agent_name": "agent", - "agent_role": "agent", - "agent_task": null, - "system_prompt": "You are a helpful customer support agent", - "agent_response": null, - "trace": null, - "tools_available": [], - "tool_calls": [], - "parameters_passed": {}, - "tool_call_results": [], - "retrieval_query": null, - "retrieved_context": null, - "exit_status": "ok", - "agent_exit": false, - "metadata": null - }, - { - "user_id": null, - "task_id": "fc64e6cc-6739-4256-ac4a-7b80c3028233", - "turn_id": "807627b0-41d9-478f-8a8b-26540d70af68", - "ground_truth": null, - "expected_tool_call": null, - "agent_name": "agent", - "agent_role": "agent", - "agent_task": null, - "system_prompt": "You are a helpful customer support agent", - "agent_response": null, - "trace": null, - "tools_available": [], - "tool_calls": [], - "parameters_passed": {}, - "tool_call_results": [], - "retrieval_query": null, - "retrieved_context": null, - "exit_status": "ok", - "agent_exit": false, - "metadata": null - }, - { - "user_id": null, - "task_id": "b7945c49-f584-4c70-972d-536a805d8a31", - "turn_id": "bf860672-af22-44c1-b13b-0eabe8f72503", - "ground_truth": null, - "expected_tool_call": null, - "agent_name": "agent", - "agent_role": "agent", - "agent_task": null, - "system_prompt": "You are a helpful customer support agent", - "agent_response": null, - "trace": null, - "tools_available": [], - "tool_calls": [], - "parameters_passed": {}, - "tool_call_results": [], - "retrieval_query": null, - "retrieved_context": null, - "exit_status": "ok", - "agent_exit": false, - "metadata": null - }, - { - "user_id": null, - "task_id": "f5c40ecf-36c0-45ba-9cc9-dc0329b0324b", - "turn_id": "5fa714f1-eeb9-4617-bc7e-67062e6d1eab", - "ground_truth": null, - "expected_tool_call": null, - "agent_name": "agent", - "agent_role": "agent", - "agent_task": null, - "system_prompt": "You are a helpful customer support agent", - "agent_response": null, - "trace": null, - "tools_available": [], - "tool_calls": [], - "parameters_passed": {}, - "tool_call_results": [], - "retrieval_query": null, - "retrieved_context": null, - "exit_status": "ok", - "agent_exit": false, - "metadata": null - } -] \ No newline at end of file diff --git a/noveum_customer_support_bt/processed_datasets/agent.web_search_evaluation_metrics_dataset_processed_dataset.csv b/noveum_customer_support_bt/processed_datasets/agent.web_search_evaluation_metrics_dataset_processed_dataset.csv deleted file mode 100644 index d415212..0000000 --- a/noveum_customer_support_bt/processed_datasets/agent.web_search_evaluation_metrics_dataset_processed_dataset.csv +++ /dev/null @@ -1,14 +0,0 @@ -user_id,task_id,turn_id,ground_truth,expected_tool_call,agent_name,agent_role,agent_task,system_prompt,agent_response,trace,tools_available,tool_calls,parameters_passed,tool_call_results,retrieval_query,retrieved_context,exit_status,agent_exit,metadata -,eda4fe22-9a2b-4b73-856b-f4f3309bf719,3b7b6838-22d4-4edf-9b00-7cb011f18a52,,,agent,agent,Evaluation for query: What are the latest breakthroughs in artificial intelligence?,You are a helpful customer support agent,"Web Search Answer: The provided web search results do not contain specific information about the latest breakthroughs in artificial intelligence. To find the most recent advancements in AI, I recommend checking reputabl...",,"[{""name"": ""web_search_evaluation"", ""description"": ""Evaluates web search performance"", ""args_schema"": {}, ""return_schema"": {""type"": ""evaluation_result""}}]",[],{},[],"[""Evaluation for query: What are the latest breakthroughs in artificial intelligence?""]","[[""""]]",ok,False, -,0ffffba1-8a37-443c-8866-d53ffbfa7718,a5d480f0-ea5b-4190-a3c3-f94e26b48573,,,agent,agent,Evaluation for query: What's happening in the software development world today?,You are a helpful customer support agent,"Web Search Answer: The web search did not yield any specific results regarding current events in the software development world. Therefore, I cannot provide detailed information on what's happening today in that field. -...",,"[{""name"": ""web_search_evaluation"", ""description"": ""Evaluates web search performance"", ""args_schema"": {}, ""return_schema"": {""type"": ""evaluation_result""}}]",[],{},[],"[""Evaluation for query: What's happening in the software development world today?""]","[[""""]]",ok,False, -,43cdf081-4f01-49cd-b566-dbd1619e6cd2,2fd285de-68a1-4b7a-a4ed-05f539afa737,,,agent,agent,Evaluation for query: What's the current status of cryptocurrency markets?,You are a helpful customer support agent,"Web Search Answer: The web search results did not provide any specific information regarding the current status of cryptocurrency markets. Therefore, I cannot provide an accurate update on market conditions, trends, or ...",,"[{""name"": ""web_search_evaluation"", ""description"": ""Evaluates web search performance"", ""args_schema"": {}, ""return_schema"": {""type"": ""evaluation_result""}}]",[],{},[],"[""Evaluation for query: What's the current status of cryptocurrency markets?""]","[[""""]]",ok,False, -,9a1983f4-09da-4b53-80e6-38de6878e0e7,48deb050-478d-486c-be3a-b8c0069d7abe,,,agent,agent,Evaluation for query: What are the latest updates in Python programming?,You are a helpful customer support agent,"Web Search Answer: The web search results do not provide specific information about the latest updates in Python programming. They focus primarily on news related to China, including geopolitics, economy, and lifestyle,...",,"[{""name"": ""web_search_evaluation"", ""description"": ""Evaluates web search performance"", ""args_schema"": {}, ""return_schema"": {""type"": ""evaluation_result""}}]",[],{},[],"[""Evaluation for query: What are the latest updates in Python programming?""]","[[""""]]",ok,False, -,5d2517e1-220a-429d-9d59-f701bda25eed,4ecede78-e7fc-48ed-9fba-1241d6890a1d,,,agent,agent,Evaluation for query: What happened in tech news this week?,You are a helpful customer support agent,"Web Search Answer: The web search results do not provide specific information about recent events in tech news for this week. They primarily discuss the grammatical usage of the phrase ""what happened"" and its variations...",,"[{""name"": ""web_search_evaluation"", ""description"": ""Evaluates web search performance"", ""args_schema"": {}, ""return_schema"": {""type"": ""evaluation_result""}}]",[],{},[],"[""Evaluation for query: What happened in tech news this week?""]","[[""""]]",ok,False, -,a81ca3a8-80aa-4c39-876e-8d40ea7a0aef,91fdf43c-7170-498e-9035-1e2b11b32b48,,,agent,agent,Evaluation for query: Tell me about recent developments in machine learning,You are a helpful customer support agent,"Web Search Answer: The provided web search results do not contain specific information about recent developments in machine learning. Therefore, I cannot provide a detailed answer based on those results. - -However, I can...",,"[{""name"": ""web_search_evaluation"", ""description"": ""Evaluates web search performance"", ""args_schema"": {}, ""return_schema"": {""type"": ""evaluation_result""}}]",[],{},[],"[""Evaluation for query: Tell me about recent developments in machine learning""]","[[""""]]",ok,False, -,230aad27-f3dd-4968-a45a-3c2f07ac28ed,096e6592-23fa-432a-931d-8f39656c7fcc,,,agent,agent,Evaluation for query: What's the weather like today?,You are a helpful customer support agent,"Web Search Answer: The provided web search results do not contain specific information about today's weather in any particular location. They mention weather forecasts for various places, including France and Singapore,...",,"[{""name"": ""web_search_evaluation"", ""description"": ""Evaluates web search performance"", ""args_schema"": {}, ""return_schema"": {""type"": ""evaluation_result""}}]",[],{},[],"[""Evaluation for query: What's the weather like today?""]","[[""""]]",ok,False, -,83c7dcce-3d89-4da1-8b3f-d419885d4cbc,b56248e6-e526-4000-8c10-f5d13cf2e12d,,,agent,agent,Evaluation for query: What are the latest AI news today?,You are a helpful customer support agent,"Web Search Answer: The latest news in artificial intelligence includes several key updates: - -1. **AI Spending Concerns**: Jeff Bezos has expressed concerns that the current level of investment in AI resembles an ""indust...",,"[{""name"": ""web_search_evaluation"", ""description"": ""Evaluates web search performance"", ""args_schema"": {}, ""return_schema"": {""type"": ""evaluation_result""}}]",[],{},[],"[""Evaluation for query: What are the latest AI news today?""]","[[""""]]",ok,False, diff --git a/noveum_customer_support_bt/processed_datasets/agent.web_search_evaluation_metrics_dataset_processed_dataset.json b/noveum_customer_support_bt/processed_datasets/agent.web_search_evaluation_metrics_dataset_processed_dataset.json deleted file mode 100644 index ada83b2..0000000 --- a/noveum_customer_support_bt/processed_datasets/agent.web_search_evaluation_metrics_dataset_processed_dataset.json +++ /dev/null @@ -1,298 +0,0 @@ -[ - { - "user_id": null, - "task_id": "eda4fe22-9a2b-4b73-856b-f4f3309bf719", - "turn_id": "3b7b6838-22d4-4edf-9b00-7cb011f18a52", - "ground_truth": null, - "expected_tool_call": null, - "agent_name": "agent", - "agent_role": "agent", - "agent_task": "Evaluation for query: What are the latest breakthroughs in artificial intelligence?", - "system_prompt": "You are a helpful customer support agent", - "agent_response": "Web Search Answer: The provided web search results do not contain specific information about the latest breakthroughs in artificial intelligence. To find the most recent advancements in AI, I recommend checking reputabl...", - "trace": null, - "tools_available": [ - { - "name": "web_search_evaluation", - "description": "Evaluates web search performance", - "args_schema": {}, - "return_schema": { - "type": "evaluation_result" - } - } - ], - "tool_calls": [], - "parameters_passed": {}, - "tool_call_results": [], - "retrieval_query": [ - "Evaluation for query: What are the latest breakthroughs in artificial intelligence?" - ], - "retrieved_context": [ - [ - "" - ] - ], - "exit_status": "ok", - "agent_exit": false, - "metadata": null - }, - { - "user_id": null, - "task_id": "0ffffba1-8a37-443c-8866-d53ffbfa7718", - "turn_id": "a5d480f0-ea5b-4190-a3c3-f94e26b48573", - "ground_truth": null, - "expected_tool_call": null, - "agent_name": "agent", - "agent_role": "agent", - "agent_task": "Evaluation for query: What's happening in the software development world today?", - "system_prompt": "You are a helpful customer support agent", - "agent_response": "Web Search Answer: The web search did not yield any specific results regarding current events in the software development world. Therefore, I cannot provide detailed information on what's happening today in that field.\n...", - "trace": null, - "tools_available": [ - { - "name": "web_search_evaluation", - "description": "Evaluates web search performance", - "args_schema": {}, - "return_schema": { - "type": "evaluation_result" - } - } - ], - "tool_calls": [], - "parameters_passed": {}, - "tool_call_results": [], - "retrieval_query": [ - "Evaluation for query: What's happening in the software development world today?" - ], - "retrieved_context": [ - [ - "" - ] - ], - "exit_status": "ok", - "agent_exit": false, - "metadata": null - }, - { - "user_id": null, - "task_id": "43cdf081-4f01-49cd-b566-dbd1619e6cd2", - "turn_id": "2fd285de-68a1-4b7a-a4ed-05f539afa737", - "ground_truth": null, - "expected_tool_call": null, - "agent_name": "agent", - "agent_role": "agent", - "agent_task": "Evaluation for query: What's the current status of cryptocurrency markets?", - "system_prompt": "You are a helpful customer support agent", - "agent_response": "Web Search Answer: The web search results did not provide any specific information regarding the current status of cryptocurrency markets. Therefore, I cannot provide an accurate update on market conditions, trends, or ...", - "trace": null, - "tools_available": [ - { - "name": "web_search_evaluation", - "description": "Evaluates web search performance", - "args_schema": {}, - "return_schema": { - "type": "evaluation_result" - } - } - ], - "tool_calls": [], - "parameters_passed": {}, - "tool_call_results": [], - "retrieval_query": [ - "Evaluation for query: What's the current status of cryptocurrency markets?" - ], - "retrieved_context": [ - [ - "" - ] - ], - "exit_status": "ok", - "agent_exit": false, - "metadata": null - }, - { - "user_id": null, - "task_id": "9a1983f4-09da-4b53-80e6-38de6878e0e7", - "turn_id": "48deb050-478d-486c-be3a-b8c0069d7abe", - "ground_truth": null, - "expected_tool_call": null, - "agent_name": "agent", - "agent_role": "agent", - "agent_task": "Evaluation for query: What are the latest updates in Python programming?", - "system_prompt": "You are a helpful customer support agent", - "agent_response": "Web Search Answer: The web search results do not provide specific information about the latest updates in Python programming. They focus primarily on news related to China, including geopolitics, economy, and lifestyle,...", - "trace": null, - "tools_available": [ - { - "name": "web_search_evaluation", - "description": "Evaluates web search performance", - "args_schema": {}, - "return_schema": { - "type": "evaluation_result" - } - } - ], - "tool_calls": [], - "parameters_passed": {}, - "tool_call_results": [], - "retrieval_query": [ - "Evaluation for query: What are the latest updates in Python programming?" - ], - "retrieved_context": [ - [ - "" - ] - ], - "exit_status": "ok", - "agent_exit": false, - "metadata": null - }, - { - "user_id": null, - "task_id": "5d2517e1-220a-429d-9d59-f701bda25eed", - "turn_id": "4ecede78-e7fc-48ed-9fba-1241d6890a1d", - "ground_truth": null, - "expected_tool_call": null, - "agent_name": "agent", - "agent_role": "agent", - "agent_task": "Evaluation for query: What happened in tech news this week?", - "system_prompt": "You are a helpful customer support agent", - "agent_response": "Web Search Answer: The web search results do not provide specific information about recent events in tech news for this week. They primarily discuss the grammatical usage of the phrase \"what happened\" and its variations...", - "trace": null, - "tools_available": [ - { - "name": "web_search_evaluation", - "description": "Evaluates web search performance", - "args_schema": {}, - "return_schema": { - "type": "evaluation_result" - } - } - ], - "tool_calls": [], - "parameters_passed": {}, - "tool_call_results": [], - "retrieval_query": [ - "Evaluation for query: What happened in tech news this week?" - ], - "retrieved_context": [ - [ - "" - ] - ], - "exit_status": "ok", - "agent_exit": false, - "metadata": null - }, - { - "user_id": null, - "task_id": "a81ca3a8-80aa-4c39-876e-8d40ea7a0aef", - "turn_id": "91fdf43c-7170-498e-9035-1e2b11b32b48", - "ground_truth": null, - "expected_tool_call": null, - "agent_name": "agent", - "agent_role": "agent", - "agent_task": "Evaluation for query: Tell me about recent developments in machine learning", - "system_prompt": "You are a helpful customer support agent", - "agent_response": "Web Search Answer: The provided web search results do not contain specific information about recent developments in machine learning. Therefore, I cannot provide a detailed answer based on those results.\n\nHowever, I can...", - "trace": null, - "tools_available": [ - { - "name": "web_search_evaluation", - "description": "Evaluates web search performance", - "args_schema": {}, - "return_schema": { - "type": "evaluation_result" - } - } - ], - "tool_calls": [], - "parameters_passed": {}, - "tool_call_results": [], - "retrieval_query": [ - "Evaluation for query: Tell me about recent developments in machine learning" - ], - "retrieved_context": [ - [ - "" - ] - ], - "exit_status": "ok", - "agent_exit": false, - "metadata": null - }, - { - "user_id": null, - "task_id": "230aad27-f3dd-4968-a45a-3c2f07ac28ed", - "turn_id": "096e6592-23fa-432a-931d-8f39656c7fcc", - "ground_truth": null, - "expected_tool_call": null, - "agent_name": "agent", - "agent_role": "agent", - "agent_task": "Evaluation for query: What's the weather like today?", - "system_prompt": "You are a helpful customer support agent", - "agent_response": "Web Search Answer: The provided web search results do not contain specific information about today's weather in any particular location. They mention weather forecasts for various places, including France and Singapore,...", - "trace": null, - "tools_available": [ - { - "name": "web_search_evaluation", - "description": "Evaluates web search performance", - "args_schema": {}, - "return_schema": { - "type": "evaluation_result" - } - } - ], - "tool_calls": [], - "parameters_passed": {}, - "tool_call_results": [], - "retrieval_query": [ - "Evaluation for query: What's the weather like today?" - ], - "retrieved_context": [ - [ - "" - ] - ], - "exit_status": "ok", - "agent_exit": false, - "metadata": null - }, - { - "user_id": null, - "task_id": "83c7dcce-3d89-4da1-8b3f-d419885d4cbc", - "turn_id": "b56248e6-e526-4000-8c10-f5d13cf2e12d", - "ground_truth": null, - "expected_tool_call": null, - "agent_name": "agent", - "agent_role": "agent", - "agent_task": "Evaluation for query: What are the latest AI news today?", - "system_prompt": "You are a helpful customer support agent", - "agent_response": "Web Search Answer: The latest news in artificial intelligence includes several key updates:\n\n1. **AI Spending Concerns**: Jeff Bezos has expressed concerns that the current level of investment in AI resembles an \"indust...", - "trace": null, - "tools_available": [ - { - "name": "web_search_evaluation", - "description": "Evaluates web search performance", - "args_schema": {}, - "return_schema": { - "type": "evaluation_result" - } - } - ], - "tool_calls": [], - "parameters_passed": {}, - "tool_call_results": [], - "retrieval_query": [ - "Evaluation for query: What are the latest AI news today?" - ], - "retrieved_context": [ - [ - "" - ] - ], - "exit_status": "ok", - "agent_exit": false, - "metadata": null - } -] \ No newline at end of file diff --git a/noveum_customer_support_bt/processed_datasets/agent.web_search_generation_dataset_processed_dataset.csv b/noveum_customer_support_bt/processed_datasets/agent.web_search_generation_dataset_processed_dataset.csv deleted file mode 100644 index f3c8e35..0000000 --- a/noveum_customer_support_bt/processed_datasets/agent.web_search_generation_dataset_processed_dataset.csv +++ /dev/null @@ -1,9 +0,0 @@ -user_id,task_id,turn_id,ground_truth,expected_tool_call,agent_name,agent_role,agent_task,system_prompt,agent_response,trace,tools_available,tool_calls,parameters_passed,tool_call_results,retrieval_query,retrieved_context,exit_status,agent_exit,metadata -,eda4fe22-9a2b-4b73-856b-f4f3309bf719,2aca614b-b016-4d56-abe2-43a144dc681d,,,agent,agent,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, -,0ffffba1-8a37-443c-8866-d53ffbfa7718,03e3ae4d-0107-401d-8ca0-05fc0f28fb24,,,agent,agent,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, -,43cdf081-4f01-49cd-b566-dbd1619e6cd2,dfdb9aaf-9925-4a5c-a0c6-2cb00f70d4d3,,,agent,agent,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, -,9a1983f4-09da-4b53-80e6-38de6878e0e7,eefcfdfc-6433-459e-a134-e2647b8418e4,,,agent,agent,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, -,5d2517e1-220a-429d-9d59-f701bda25eed,c57b4b54-b9b0-4a4c-b5cd-57a4cfe5301b,,,agent,agent,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, -,a81ca3a8-80aa-4c39-876e-8d40ea7a0aef,a062bef6-d973-489b-91df-5d40a2aef077,,,agent,agent,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, -,230aad27-f3dd-4968-a45a-3c2f07ac28ed,41f82a95-620d-440c-84e6-36d6d1b9e079,,,agent,agent,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, -,83c7dcce-3d89-4da1-8b3f-d419885d4cbc,63f2a0d5-9016-4483-8ec0-e6aa6b27689f,,,agent,agent,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, diff --git a/noveum_customer_support_bt/processed_datasets/agent.web_search_generation_dataset_processed_dataset.json b/noveum_customer_support_bt/processed_datasets/agent.web_search_generation_dataset_processed_dataset.json deleted file mode 100644 index 105a46c..0000000 --- a/noveum_customer_support_bt/processed_datasets/agent.web_search_generation_dataset_processed_dataset.json +++ /dev/null @@ -1,178 +0,0 @@ -[ - { - "user_id": null, - "task_id": "eda4fe22-9a2b-4b73-856b-f4f3309bf719", - "turn_id": "2aca614b-b016-4d56-abe2-43a144dc681d", - "ground_truth": null, - "expected_tool_call": null, - "agent_name": "agent", - "agent_role": "agent", - "agent_task": null, - "system_prompt": "You are a helpful customer support agent", - "agent_response": null, - "trace": null, - "tools_available": [], - "tool_calls": [], - "parameters_passed": {}, - "tool_call_results": [], - "retrieval_query": null, - "retrieved_context": null, - "exit_status": "ok", - "agent_exit": false, - "metadata": null - }, - { - "user_id": null, - "task_id": "0ffffba1-8a37-443c-8866-d53ffbfa7718", - "turn_id": "03e3ae4d-0107-401d-8ca0-05fc0f28fb24", - "ground_truth": null, - "expected_tool_call": null, - "agent_name": "agent", - "agent_role": "agent", - "agent_task": null, - "system_prompt": "You are a helpful customer support agent", - "agent_response": null, - "trace": null, - "tools_available": [], - "tool_calls": [], - "parameters_passed": {}, - "tool_call_results": [], - "retrieval_query": null, - "retrieved_context": null, - "exit_status": "ok", - "agent_exit": false, - "metadata": null - }, - { - "user_id": null, - "task_id": "43cdf081-4f01-49cd-b566-dbd1619e6cd2", - "turn_id": "dfdb9aaf-9925-4a5c-a0c6-2cb00f70d4d3", - "ground_truth": null, - "expected_tool_call": null, - "agent_name": "agent", - "agent_role": "agent", - "agent_task": null, - "system_prompt": "You are a helpful customer support agent", - "agent_response": null, - "trace": null, - "tools_available": [], - "tool_calls": [], - "parameters_passed": {}, - "tool_call_results": [], - "retrieval_query": null, - "retrieved_context": null, - "exit_status": "ok", - "agent_exit": false, - "metadata": null - }, - { - "user_id": null, - "task_id": "9a1983f4-09da-4b53-80e6-38de6878e0e7", - "turn_id": "eefcfdfc-6433-459e-a134-e2647b8418e4", - "ground_truth": null, - "expected_tool_call": null, - "agent_name": "agent", - "agent_role": "agent", - "agent_task": null, - "system_prompt": "You are a helpful customer support agent", - "agent_response": null, - "trace": null, - "tools_available": [], - "tool_calls": [], - "parameters_passed": {}, - "tool_call_results": [], - "retrieval_query": null, - "retrieved_context": null, - "exit_status": "ok", - "agent_exit": false, - "metadata": null - }, - { - "user_id": null, - "task_id": "5d2517e1-220a-429d-9d59-f701bda25eed", - "turn_id": "c57b4b54-b9b0-4a4c-b5cd-57a4cfe5301b", - "ground_truth": null, - "expected_tool_call": null, - "agent_name": "agent", - "agent_role": "agent", - "agent_task": null, - "system_prompt": "You are a helpful customer support agent", - "agent_response": null, - "trace": null, - "tools_available": [], - "tool_calls": [], - "parameters_passed": {}, - "tool_call_results": [], - "retrieval_query": null, - "retrieved_context": null, - "exit_status": "ok", - "agent_exit": false, - "metadata": null - }, - { - "user_id": null, - "task_id": "a81ca3a8-80aa-4c39-876e-8d40ea7a0aef", - "turn_id": "a062bef6-d973-489b-91df-5d40a2aef077", - "ground_truth": null, - "expected_tool_call": null, - "agent_name": "agent", - "agent_role": "agent", - "agent_task": null, - "system_prompt": "You are a helpful customer support agent", - "agent_response": null, - "trace": null, - "tools_available": [], - "tool_calls": [], - "parameters_passed": {}, - "tool_call_results": [], - "retrieval_query": null, - "retrieved_context": null, - "exit_status": "ok", - "agent_exit": false, - "metadata": null - }, - { - "user_id": null, - "task_id": "230aad27-f3dd-4968-a45a-3c2f07ac28ed", - "turn_id": "41f82a95-620d-440c-84e6-36d6d1b9e079", - "ground_truth": null, - "expected_tool_call": null, - "agent_name": "agent", - "agent_role": "agent", - "agent_task": null, - "system_prompt": "You are a helpful customer support agent", - "agent_response": null, - "trace": null, - "tools_available": [], - "tool_calls": [], - "parameters_passed": {}, - "tool_call_results": [], - "retrieval_query": null, - "retrieved_context": null, - "exit_status": "ok", - "agent_exit": false, - "metadata": null - }, - { - "user_id": null, - "task_id": "83c7dcce-3d89-4da1-8b3f-d419885d4cbc", - "turn_id": "63f2a0d5-9016-4483-8ec0-e6aa6b27689f", - "ground_truth": null, - "expected_tool_call": null, - "agent_name": "agent", - "agent_role": "agent", - "agent_task": null, - "system_prompt": "You are a helpful customer support agent", - "agent_response": null, - "trace": null, - "tools_available": [], - "tool_calls": [], - "parameters_passed": {}, - "tool_call_results": [], - "retrieval_query": null, - "retrieved_context": null, - "exit_status": "ok", - "agent_exit": false, - "metadata": null - } -] \ No newline at end of file diff --git a/noveum_customer_support_bt/processed_datasets/tool-orchestator_dataset_processed_dataset.csv b/noveum_customer_support_bt/processed_datasets/tool-orchestator_dataset_processed_dataset.csv deleted file mode 100644 index a057016..0000000 --- a/noveum_customer_support_bt/processed_datasets/tool-orchestator_dataset_processed_dataset.csv +++ /dev/null @@ -1,21 +0,0 @@ -user_id,task_id,turn_id,ground_truth,expected_tool_call,agent_name,agent_role,agent_task,system_prompt,agent_response,trace,tools_available,tool_calls,parameters_passed,tool_call_results,retrieval_query,retrieved_context,exit_status,agent_exit,metadata -,eda4fe22-9a2b-4b73-856b-f4f3309bf719,b6b7c1c0-3b7b-4e19-805d-a16beb7ea871,,,unknown,unknown,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, -,0ffffba1-8a37-443c-8866-d53ffbfa7718,3045d971-b7ff-46cc-be11-6db7d65dfefd,,,unknown,unknown,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, -,f1f37bd7-0851-4659-b493-b80d3800d920,42d4cb1f-17ec-433a-8087-90292c2fa9d9,,,unknown,unknown,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, -,43cdf081-4f01-49cd-b566-dbd1619e6cd2,e22b809d-5320-40b7-b78a-e3e125538a6b,,,unknown,unknown,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, -,9a1983f4-09da-4b53-80e6-38de6878e0e7,813a4450-1cf8-4d43-8493-eabfa200919b,,,unknown,unknown,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, -,5d2517e1-220a-429d-9d59-f701bda25eed,6102352a-833e-493d-adbb-887e2af17918,,,unknown,unknown,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, -,52aacb67-c361-4445-9b72-c157f79f47d6,2926418a-fe09-4df0-bf19-a527bc24a478,,,unknown,unknown,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, -,a81ca3a8-80aa-4c39-876e-8d40ea7a0aef,afe85a1d-13c8-4476-8154-52d58ba83d52,,,unknown,unknown,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, -,230aad27-f3dd-4968-a45a-3c2f07ac28ed,db8da393-6d72-421e-87dc-96f56b6576e5,,,unknown,unknown,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, -,83c7dcce-3d89-4da1-8b3f-d419885d4cbc,a290202d-1b3f-431f-8728-fcb937380f8c,,,unknown,unknown,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, -,2218f641-604c-491a-9710-b51a9941b982,5fbaef83-95fc-4bed-9695-9d69c37c35e4,,,unknown,unknown,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, -,255fd49c-84b4-4b18-887e-6308a412d535,b3ac3924-928b-4d48-a2a7-aa749802fef9,,,unknown,unknown,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, -,dc511122-c0b6-415c-9a49-c7b45132dd87,b8055ffc-ee7d-46ad-b6a0-84e80f9399e2,,,unknown,unknown,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, -,04bebf38-a343-4563-80db-0154bef8d927,532b9e1f-5c4f-4b41-98bb-e9a1510f0f1c,,,unknown,unknown,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, -,5e043630-6493-42b5-beb8-79faa19bfa37,14064e3f-f9c9-4bb7-b8cb-1ad52938b075,,,unknown,unknown,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, -,7da9814d-a2e8-4c4e-b750-68b26bd5fd22,a74fe8a9-2654-4b9d-ba87-dc2f86b6466d,,,unknown,unknown,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, -,16143f74-2831-4753-b33d-ce4b645093c5,c700d0f4-f1f8-4339-aa42-a3060ae2ae11,,,unknown,unknown,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, -,fc64e6cc-6739-4256-ac4a-7b80c3028233,f7f5981f-d840-4265-8970-d319b876366d,,,unknown,unknown,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, -,b7945c49-f584-4c70-972d-536a805d8a31,0c7bbe2b-18af-4fdb-a5bf-4a91daa3bf50,,,unknown,unknown,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, -,f5c40ecf-36c0-45ba-9cc9-dc0329b0324b,dc869742-6797-4016-8061-37530fc7d433,,,unknown,unknown,,You are a helpful customer support agent,,,[],[],{},[],,,ok,False, diff --git a/noveum_customer_support_bt/processed_datasets/tool-orchestator_dataset_processed_dataset.json b/noveum_customer_support_bt/processed_datasets/tool-orchestator_dataset_processed_dataset.json deleted file mode 100644 index 5e81efd..0000000 --- a/noveum_customer_support_bt/processed_datasets/tool-orchestator_dataset_processed_dataset.json +++ /dev/null @@ -1,442 +0,0 @@ -[ - { - "user_id": null, - "task_id": "eda4fe22-9a2b-4b73-856b-f4f3309bf719", - "turn_id": "b6b7c1c0-3b7b-4e19-805d-a16beb7ea871", - "ground_truth": null, - "expected_tool_call": null, - "agent_name": "unknown", - "agent_role": "unknown", - "agent_task": null, - "system_prompt": "You are a helpful customer support agent", - "agent_response": null, - "trace": null, - "tools_available": [], - "tool_calls": [], - "parameters_passed": {}, - "tool_call_results": [], - "retrieval_query": null, - "retrieved_context": null, - "exit_status": "ok", - "agent_exit": false, - "metadata": null - }, - { - "user_id": null, - "task_id": "0ffffba1-8a37-443c-8866-d53ffbfa7718", - "turn_id": "3045d971-b7ff-46cc-be11-6db7d65dfefd", - "ground_truth": null, - "expected_tool_call": null, - "agent_name": "unknown", - "agent_role": "unknown", - "agent_task": null, - "system_prompt": "You are a helpful customer support agent", - "agent_response": null, - "trace": null, - "tools_available": [], - "tool_calls": [], - "parameters_passed": {}, - "tool_call_results": [], - "retrieval_query": null, - "retrieved_context": null, - "exit_status": "ok", - "agent_exit": false, - "metadata": null - }, - { - "user_id": null, - "task_id": "f1f37bd7-0851-4659-b493-b80d3800d920", - "turn_id": "42d4cb1f-17ec-433a-8087-90292c2fa9d9", - "ground_truth": null, - "expected_tool_call": null, - "agent_name": "unknown", - "agent_role": "unknown", - "agent_task": null, - "system_prompt": "You are a helpful customer support agent", - "agent_response": null, - "trace": null, - "tools_available": [], - "tool_calls": [], - "parameters_passed": {}, - "tool_call_results": [], - "retrieval_query": null, - "retrieved_context": null, - "exit_status": "ok", - "agent_exit": false, - "metadata": null - }, - { - "user_id": null, - "task_id": "43cdf081-4f01-49cd-b566-dbd1619e6cd2", - "turn_id": "e22b809d-5320-40b7-b78a-e3e125538a6b", - "ground_truth": null, - "expected_tool_call": null, - "agent_name": "unknown", - "agent_role": "unknown", - "agent_task": null, - "system_prompt": "You are a helpful customer support agent", - "agent_response": null, - "trace": null, - "tools_available": [], - "tool_calls": [], - "parameters_passed": {}, - "tool_call_results": [], - "retrieval_query": null, - "retrieved_context": null, - "exit_status": "ok", - "agent_exit": false, - "metadata": null - }, - { - "user_id": null, - "task_id": "9a1983f4-09da-4b53-80e6-38de6878e0e7", - "turn_id": "813a4450-1cf8-4d43-8493-eabfa200919b", - "ground_truth": null, - "expected_tool_call": null, - "agent_name": "unknown", - "agent_role": "unknown", - "agent_task": null, - "system_prompt": "You are a helpful customer support agent", - "agent_response": null, - "trace": null, - "tools_available": [], - "tool_calls": [], - "parameters_passed": {}, - "tool_call_results": [], - "retrieval_query": null, - "retrieved_context": null, - "exit_status": "ok", - "agent_exit": false, - "metadata": null - }, - { - "user_id": null, - "task_id": "5d2517e1-220a-429d-9d59-f701bda25eed", - "turn_id": "6102352a-833e-493d-adbb-887e2af17918", - "ground_truth": null, - "expected_tool_call": null, - "agent_name": "unknown", - "agent_role": "unknown", - "agent_task": null, - "system_prompt": "You are a helpful customer support agent", - "agent_response": null, - "trace": null, - "tools_available": [], - "tool_calls": [], - "parameters_passed": {}, - "tool_call_results": [], - "retrieval_query": null, - "retrieved_context": null, - "exit_status": "ok", - "agent_exit": false, - "metadata": null - }, - { - "user_id": null, - "task_id": "52aacb67-c361-4445-9b72-c157f79f47d6", - "turn_id": "2926418a-fe09-4df0-bf19-a527bc24a478", - "ground_truth": null, - "expected_tool_call": null, - "agent_name": "unknown", - "agent_role": "unknown", - "agent_task": null, - "system_prompt": "You are a helpful customer support agent", - "agent_response": null, - "trace": null, - "tools_available": [], - "tool_calls": [], - "parameters_passed": {}, - "tool_call_results": [], - "retrieval_query": null, - "retrieved_context": null, - "exit_status": "ok", - "agent_exit": false, - "metadata": null - }, - { - "user_id": null, - "task_id": "a81ca3a8-80aa-4c39-876e-8d40ea7a0aef", - "turn_id": "afe85a1d-13c8-4476-8154-52d58ba83d52", - "ground_truth": null, - "expected_tool_call": null, - "agent_name": "unknown", - "agent_role": "unknown", - "agent_task": null, - "system_prompt": "You are a helpful customer support agent", - "agent_response": null, - "trace": null, - "tools_available": [], - "tool_calls": [], - "parameters_passed": {}, - "tool_call_results": [], - "retrieval_query": null, - "retrieved_context": null, - "exit_status": "ok", - "agent_exit": false, - "metadata": null - }, - { - "user_id": null, - "task_id": "230aad27-f3dd-4968-a45a-3c2f07ac28ed", - "turn_id": "db8da393-6d72-421e-87dc-96f56b6576e5", - "ground_truth": null, - "expected_tool_call": null, - "agent_name": "unknown", - "agent_role": "unknown", - "agent_task": null, - "system_prompt": "You are a helpful customer support agent", - "agent_response": null, - "trace": null, - "tools_available": [], - "tool_calls": [], - "parameters_passed": {}, - "tool_call_results": [], - "retrieval_query": null, - "retrieved_context": null, - "exit_status": "ok", - "agent_exit": false, - "metadata": null - }, - { - "user_id": null, - "task_id": "83c7dcce-3d89-4da1-8b3f-d419885d4cbc", - "turn_id": "a290202d-1b3f-431f-8728-fcb937380f8c", - "ground_truth": null, - "expected_tool_call": null, - "agent_name": "unknown", - "agent_role": "unknown", - "agent_task": null, - "system_prompt": "You are a helpful customer support agent", - "agent_response": null, - "trace": null, - "tools_available": [], - "tool_calls": [], - "parameters_passed": {}, - "tool_call_results": [], - "retrieval_query": null, - "retrieved_context": null, - "exit_status": "ok", - "agent_exit": false, - "metadata": null - }, - { - "user_id": null, - "task_id": "2218f641-604c-491a-9710-b51a9941b982", - "turn_id": "5fbaef83-95fc-4bed-9695-9d69c37c35e4", - "ground_truth": null, - "expected_tool_call": null, - "agent_name": "unknown", - "agent_role": "unknown", - "agent_task": null, - "system_prompt": "You are a helpful customer support agent", - "agent_response": null, - "trace": null, - "tools_available": [], - "tool_calls": [], - "parameters_passed": {}, - "tool_call_results": [], - "retrieval_query": null, - "retrieved_context": null, - "exit_status": "ok", - "agent_exit": false, - "metadata": null - }, - { - "user_id": null, - "task_id": "255fd49c-84b4-4b18-887e-6308a412d535", - "turn_id": "b3ac3924-928b-4d48-a2a7-aa749802fef9", - "ground_truth": null, - "expected_tool_call": null, - "agent_name": "unknown", - "agent_role": "unknown", - "agent_task": null, - "system_prompt": "You are a helpful customer support agent", - "agent_response": null, - "trace": null, - "tools_available": [], - "tool_calls": [], - "parameters_passed": {}, - "tool_call_results": [], - "retrieval_query": null, - "retrieved_context": null, - "exit_status": "ok", - "agent_exit": false, - "metadata": null - }, - { - "user_id": null, - "task_id": "dc511122-c0b6-415c-9a49-c7b45132dd87", - "turn_id": "b8055ffc-ee7d-46ad-b6a0-84e80f9399e2", - "ground_truth": null, - "expected_tool_call": null, - "agent_name": "unknown", - "agent_role": "unknown", - "agent_task": null, - "system_prompt": "You are a helpful customer support agent", - "agent_response": null, - "trace": null, - "tools_available": [], - "tool_calls": [], - "parameters_passed": {}, - "tool_call_results": [], - "retrieval_query": null, - "retrieved_context": null, - "exit_status": "ok", - "agent_exit": false, - "metadata": null - }, - { - "user_id": null, - "task_id": "04bebf38-a343-4563-80db-0154bef8d927", - "turn_id": "532b9e1f-5c4f-4b41-98bb-e9a1510f0f1c", - "ground_truth": null, - "expected_tool_call": null, - "agent_name": "unknown", - "agent_role": "unknown", - "agent_task": null, - "system_prompt": "You are a helpful customer support agent", - "agent_response": null, - "trace": null, - "tools_available": [], - "tool_calls": [], - "parameters_passed": {}, - "tool_call_results": [], - "retrieval_query": null, - "retrieved_context": null, - "exit_status": "ok", - "agent_exit": false, - "metadata": null - }, - { - "user_id": null, - "task_id": "5e043630-6493-42b5-beb8-79faa19bfa37", - "turn_id": "14064e3f-f9c9-4bb7-b8cb-1ad52938b075", - "ground_truth": null, - "expected_tool_call": null, - "agent_name": "unknown", - "agent_role": "unknown", - "agent_task": null, - "system_prompt": "You are a helpful customer support agent", - "agent_response": null, - "trace": null, - "tools_available": [], - "tool_calls": [], - "parameters_passed": {}, - "tool_call_results": [], - "retrieval_query": null, - "retrieved_context": null, - "exit_status": "ok", - "agent_exit": false, - "metadata": null - }, - { - "user_id": null, - "task_id": "7da9814d-a2e8-4c4e-b750-68b26bd5fd22", - "turn_id": "a74fe8a9-2654-4b9d-ba87-dc2f86b6466d", - "ground_truth": null, - "expected_tool_call": null, - "agent_name": "unknown", - "agent_role": "unknown", - "agent_task": null, - "system_prompt": "You are a helpful customer support agent", - "agent_response": null, - "trace": null, - "tools_available": [], - "tool_calls": [], - "parameters_passed": {}, - "tool_call_results": [], - "retrieval_query": null, - "retrieved_context": null, - "exit_status": "ok", - "agent_exit": false, - "metadata": null - }, - { - "user_id": null, - "task_id": "16143f74-2831-4753-b33d-ce4b645093c5", - "turn_id": "c700d0f4-f1f8-4339-aa42-a3060ae2ae11", - "ground_truth": null, - "expected_tool_call": null, - "agent_name": "unknown", - "agent_role": "unknown", - "agent_task": null, - "system_prompt": "You are a helpful customer support agent", - "agent_response": null, - "trace": null, - "tools_available": [], - "tool_calls": [], - "parameters_passed": {}, - "tool_call_results": [], - "retrieval_query": null, - "retrieved_context": null, - "exit_status": "ok", - "agent_exit": false, - "metadata": null - }, - { - "user_id": null, - "task_id": "fc64e6cc-6739-4256-ac4a-7b80c3028233", - "turn_id": "f7f5981f-d840-4265-8970-d319b876366d", - "ground_truth": null, - "expected_tool_call": null, - "agent_name": "unknown", - "agent_role": "unknown", - "agent_task": null, - "system_prompt": "You are a helpful customer support agent", - "agent_response": null, - "trace": null, - "tools_available": [], - "tool_calls": [], - "parameters_passed": {}, - "tool_call_results": [], - "retrieval_query": null, - "retrieved_context": null, - "exit_status": "ok", - "agent_exit": false, - "metadata": null - }, - { - "user_id": null, - "task_id": "b7945c49-f584-4c70-972d-536a805d8a31", - "turn_id": "0c7bbe2b-18af-4fdb-a5bf-4a91daa3bf50", - "ground_truth": null, - "expected_tool_call": null, - "agent_name": "unknown", - "agent_role": "unknown", - "agent_task": null, - "system_prompt": "You are a helpful customer support agent", - "agent_response": null, - "trace": null, - "tools_available": [], - "tool_calls": [], - "parameters_passed": {}, - "tool_call_results": [], - "retrieval_query": null, - "retrieved_context": null, - "exit_status": "ok", - "agent_exit": false, - "metadata": null - }, - { - "user_id": null, - "task_id": "f5c40ecf-36c0-45ba-9cc9-dc0329b0324b", - "turn_id": "dc869742-6797-4016-8061-37530fc7d433", - "ground_truth": null, - "expected_tool_call": null, - "agent_name": "unknown", - "agent_role": "unknown", - "agent_task": null, - "system_prompt": "You are a helpful customer support agent", - "agent_response": null, - "trace": null, - "tools_available": [], - "tool_calls": [], - "parameters_passed": {}, - "tool_call_results": [], - "retrieval_query": null, - "retrieved_context": null, - "exit_status": "ok", - "agent_exit": false, - "metadata": null - } -] \ No newline at end of file diff --git a/noveum_customer_support_bt/split_datasets/agent.llm-rag_dataset.json b/noveum_customer_support_bt/split_datasets/agent.llm-rag_dataset.json deleted file mode 100644 index 5f77f7d..0000000 --- a/noveum_customer_support_bt/split_datasets/agent.llm-rag_dataset.json +++ /dev/null @@ -1,734 +0,0 @@ -[ - { - "span_id": "ea4d61c5-b8dd-4677-b21b-2b17170e2163", - "trace_id": "f1f37bd7-0851-4659-b493-b80d3800d920", - "parent_span_id": "89eac2de-b7da-4db3-bf7c-8e87c429481e", - "name": "agent.llm-rag", - "start_time": "2025-10-09 08:26:35.915000000", - "end_time": "2025-10-09 08:26:40.841000000", - "duration_ms": 4925.434, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "rag_agent", - "agent.operation": "llm-rag", - "agent.capabilities": [ - "document_retrieval", - "context_generation", - "response_generation" - ], - "agent.id": "noveum_rag_agent", - "input_query": "What are the newest features in cloud computing?", - "query_length": 48, - "query_type": "rag_query", - "output_response": "RAG Answer: The provided context does not specifically mention any new features in cloud computing as a general topic. However, it does highlight several recent enhancements and features related to the Noveum.ai ...", - "rag.context_length": 2579, - "rag.sources_count": 5, - "rag.answer_length": 883, - "rag.mode": "retrieval_augmented_generation" - }, - "events": [], - "links": [], - "trace_trace_id": "f1f37bd7-0851-4659-b493-b80d3800d920", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-09 08:26:35.915000000", - "trace_end_time": "2025-10-09 08:26:40.841000000", - "trace_duration_ms": 4926.342, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-09 08:26:44.053000000", - "updated_at": "2025-10-09 08:26:44.053000000" - }, - { - "span_id": "7505f97a-cbc0-42fe-8a72-95387d723daa", - "trace_id": "52aacb67-c361-4445-9b72-c157f79f47d6", - "parent_span_id": "5359c861-c51b-446d-9ad2-b3d8c21ea70f", - "name": "agent.llm-rag", - "start_time": "2025-10-09 08:26:16.545000000", - "end_time": "2025-10-09 08:26:22.201000000", - "duration_ms": 5656.22, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "rag_agent", - "agent.operation": "llm-rag", - "agent.capabilities": [ - "document_retrieval", - "context_generation", - "response_generation" - ], - "agent.id": "noveum_rag_agent", - "input_query": "What are the current trends in observability tools?", - "query_length": 51, - "query_type": "rag_query", - "output_response": "RAG Answer: The provided context does not explicitly outline current trends in observability tools. However, it highlights some key aspects of observability that are particularly relevant to AI applications. \n\n1....", - "rag.context_length": 2943, - "rag.sources_count": 5, - "rag.answer_length": 1074, - "rag.mode": "retrieval_augmented_generation" - }, - "events": [], - "links": [], - "trace_trace_id": "52aacb67-c361-4445-9b72-c157f79f47d6", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-09 08:26:16.544000000", - "trace_end_time": "2025-10-09 08:26:22.201000000", - "trace_duration_ms": 5657.012, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-09 08:26:22.904000000", - "updated_at": "2025-10-09 08:26:22.904000000" - }, - { - "span_id": "8a6e18a1-babc-4166-926f-188f344d514e", - "trace_id": "2218f641-604c-491a-9710-b51a9941b982", - "parent_span_id": "3ce2e35e-4518-43b4-8a76-4413da6832ea", - "name": "agent.llm-rag", - "start_time": "2025-10-09 08:25:48.107000000", - "end_time": "2025-10-09 08:25:55.878000000", - "duration_ms": 7770.424, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "rag_agent", - "agent.operation": "llm-rag", - "agent.capabilities": [ - "document_retrieval", - "context_generation", - "response_generation" - ], - "agent.id": "noveum_rag_agent", - "input_query": "What are the benefits of using Noveum Trace?", - "query_length": 44, - "query_type": "rag_query", - "output_response": "RAG Answer: Using Noveum Trace offers several benefits, including:\n\n1. **Real-Time Monitoring**: The Noveum platform provides a real-time dashboard for analyzing traces and performance, with live updates as new t...", - "rag.context_length": 2773, - "rag.sources_count": 5, - "rag.answer_length": 1460, - "rag.mode": "retrieval_augmented_generation" - }, - "events": [], - "links": [], - "trace_trace_id": "2218f641-604c-491a-9710-b51a9941b982", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-09 08:25:48.107000000", - "trace_end_time": "2025-10-09 08:25:55.878000000", - "trace_duration_ms": 7770.953, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-09 08:25:56.565000000", - "updated_at": "2025-10-09 08:25:56.565000000" - }, - { - "span_id": "5bec4165-6596-43ac-930b-acfcde1cc02a", - "trace_id": "255fd49c-84b4-4b18-887e-6308a412d535", - "parent_span_id": "c0c39e8b-9a42-4c18-8737-9224cd8a45c0", - "name": "agent.llm-rag", - "start_time": "2025-10-09 08:25:43.830000000", - "end_time": "2025-10-09 08:25:47.099000000", - "duration_ms": 3268.575, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "rag_agent", - "agent.operation": "llm-rag", - "agent.capabilities": [ - "document_retrieval", - "context_generation", - "response_generation" - ], - "agent.id": "noveum_rag_agent", - "input_query": "How do I configure Noveum for my system?", - "query_length": 40, - "query_type": "rag_query", - "output_response": "RAG Answer: To configure Noveum for your system, you can refer to the SDK Integration Guide available in the Noveum documentation. This guide will help you trace your AI applications with minimal code changes. Th...", - "rag.context_length": 2237, - "rag.sources_count": 5, - "rag.answer_length": 613, - "rag.mode": "retrieval_augmented_generation" - }, - "events": [], - "links": [], - "trace_trace_id": "255fd49c-84b4-4b18-887e-6308a412d535", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-09 08:25:43.830000000", - "trace_end_time": "2025-10-09 08:25:47.099000000", - "trace_duration_ms": 3268.862, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-09 08:25:50.304000000", - "updated_at": "2025-10-09 08:25:50.304000000" - }, - { - "span_id": "f9f79a07-4eda-41b8-9634-d4460114caad", - "trace_id": "dc511122-c0b6-415c-9a49-c7b45132dd87", - "parent_span_id": "80987c51-ea82-4a41-97c7-ed591082b9bf", - "name": "agent.llm-rag", - "start_time": "2025-10-09 08:25:38.322000000", - "end_time": "2025-10-09 08:25:42.822000000", - "duration_ms": 4500.0109999999995, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "rag_agent", - "agent.operation": "llm-rag", - "agent.capabilities": [ - "document_retrieval", - "context_generation", - "response_generation" - ], - "agent.id": "noveum_rag_agent", - "input_query": "What monitoring capabilities does Noveum provide?", - "query_length": 49, - "query_type": "rag_query", - "output_response": "RAG Answer: Noveum provides comprehensive monitoring capabilities for AI applications, including the ability to monitor, trace, and optimize AI agents across various frameworks such as LangChain, CrewAI, AutoGen,...", - "rag.context_length": 2740, - "rag.sources_count": 5, - "rag.answer_length": 839, - "rag.mode": "retrieval_augmented_generation" - }, - "events": [], - "links": [], - "trace_trace_id": "dc511122-c0b6-415c-9a49-c7b45132dd87", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-09 08:25:38.322000000", - "trace_end_time": "2025-10-09 08:25:42.822000000", - "trace_duration_ms": 4500.425, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-09 08:25:45.028000000", - "updated_at": "2025-10-09 08:25:45.028000000" - }, - { - "span_id": "edc52a5d-8450-4d0f-ad14-958ec4d147f4", - "trace_id": "04bebf38-a343-4563-80db-0154bef8d927", - "parent_span_id": "2f89c1e0-9b62-4651-914c-b71cf561c33c", - "name": "agent.llm-rag", - "start_time": "2025-10-09 08:25:33.693000000", - "end_time": "2025-10-09 08:25:37.313000000", - "duration_ms": 3619.1870000000004, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "rag_agent", - "agent.operation": "llm-rag", - "agent.capabilities": [ - "document_retrieval", - "context_generation", - "response_generation" - ], - "agent.id": "noveum_rag_agent", - "input_query": "How does Noveum handle agent tracing?", - "query_length": 37, - "query_type": "rag_query", - "output_response": "RAG Answer: Noveum.ai handles agent tracing through its specialized Multi-Agent Tracing capabilities, which are designed to observe complex workflows and inter-agent communications within multi-agent systems. The...", - "rag.context_length": 2736, - "rag.sources_count": 5, - "rag.answer_length": 691, - "rag.mode": "retrieval_augmented_generation" - }, - "events": [], - "links": [], - "trace_trace_id": "04bebf38-a343-4563-80db-0154bef8d927", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-09 08:25:33.693000000", - "trace_end_time": "2025-10-09 08:25:37.313000000", - "trace_duration_ms": 3619.672, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-09 08:25:40.339000000", - "updated_at": "2025-10-09 08:25:40.339000000" - }, - { - "span_id": "6c362c73-e063-4813-8180-d26b6a52b798", - "trace_id": "5e043630-6493-42b5-beb8-79faa19bfa37", - "parent_span_id": "e30e3e61-6edd-4b62-ab77-627faa2bf741", - "name": "agent.llm-rag", - "start_time": "2025-10-09 08:25:29.902000000", - "end_time": "2025-10-09 08:25:32.684000000", - "duration_ms": 2782.976, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "rag_agent", - "agent.operation": "llm-rag", - "agent.capabilities": [ - "document_retrieval", - "context_generation", - "response_generation" - ], - "agent.id": "noveum_rag_agent", - "input_query": "What APIs are available in Noveum platform?", - "query_length": 43, - "query_type": "rag_query", - "output_response": "RAG Answer: The provided context does not specify the APIs available in the Noveum platform. For detailed information about the APIs, I recommend checking the official Noveum documentation or the SDK Integration ...", - "rag.context_length": 2284, - "rag.sources_count": 5, - "rag.answer_length": 270, - "rag.mode": "retrieval_augmented_generation" - }, - "events": [], - "links": [], - "trace_trace_id": "5e043630-6493-42b5-beb8-79faa19bfa37", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-09 08:25:29.901000000", - "trace_end_time": "2025-10-09 08:25:32.685000000", - "trace_duration_ms": 2783.326, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-09 08:25:34.918000000", - "updated_at": "2025-10-09 08:25:34.918000000" - }, - { - "span_id": "1393d065-a9a4-4475-808f-2d00b3c68939", - "trace_id": "7da9814d-a2e8-4c4e-b750-68b26bd5fd22", - "parent_span_id": "2f513795-764f-4eb6-8dd2-108d4cdf8432", - "name": "agent.llm-rag", - "start_time": "2025-10-09 08:25:20.587000000", - "end_time": "2025-10-09 08:25:28.897000000", - "duration_ms": 8309.805, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "rag_agent", - "agent.operation": "llm-rag", - "agent.capabilities": [ - "document_retrieval", - "context_generation", - "response_generation" - ], - "agent.id": "noveum_rag_agent", - "input_query": "How do I set up observability with Noveum?", - "query_length": 42, - "query_type": "rag_query", - "output_response": "RAG Answer: To set up observability with Noveum, follow these steps:\n\n1. **Sign Up**: Go to [noveum.ai](https://noveum.ai) and create an account.\n2. **Create a Project**: Once signed in, create a new project to o...", - "rag.context_length": 2870, - "rag.sources_count": 6, - "rag.answer_length": 951, - "rag.mode": "retrieval_augmented_generation" - }, - "events": [], - "links": [], - "trace_trace_id": "7da9814d-a2e8-4c4e-b750-68b26bd5fd22", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-09 08:25:20.586000000", - "trace_end_time": "2025-10-09 08:25:28.897000000", - "trace_duration_ms": 8310.205, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-09 08:25:29.588000000", - "updated_at": "2025-10-09 08:25:29.588000000" - }, - { - "span_id": "f419377a-fc24-4abf-8a1c-03857824e374", - "trace_id": "16143f74-2831-4753-b33d-ce4b645093c5", - "parent_span_id": "29c7a515-2383-4b7d-8fc7-beac3f403641", - "name": "agent.llm-rag", - "start_time": "2025-10-09 08:25:12.442000000", - "end_time": "2025-10-09 08:25:19.579000000", - "duration_ms": 7137.042, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "rag_agent", - "agent.operation": "llm-rag", - "agent.capabilities": [ - "document_retrieval", - "context_generation", - "response_generation" - ], - "agent.id": "noveum_rag_agent", - "input_query": "What features does Noveum Trace offer?", - "query_length": 38, - "query_type": "rag_query", - "output_response": "RAG Answer: Noveum Trace offers several key features through its Python SDK (`noveum-trace`) and TypeScript SDK (`@noveum/trace`):\n\n1. **Decorator-based Tracing**: It provides a simple way to trace LLM calls, age...", - "rag.context_length": 2787, - "rag.sources_count": 5, - "rag.answer_length": 1014, - "rag.mode": "retrieval_augmented_generation" - }, - "events": [], - "links": [], - "trace_trace_id": "16143f74-2831-4753-b33d-ce4b645093c5", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-09 08:25:12.442000000", - "trace_end_time": "2025-10-09 08:25:19.579000000", - "trace_duration_ms": 7137.402, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-09 08:25:20.290000000", - "updated_at": "2025-10-09 08:25:20.290000000" - }, - { - "span_id": "0e58e31f-a8ab-4aee-8fc2-abcae1e21d9d", - "trace_id": "fc64e6cc-6739-4256-ac4a-7b80c3028233", - "parent_span_id": "12fee687-2a2e-401f-83c2-9c8dc7ecd853", - "name": "agent.llm-rag", - "start_time": "2025-10-09 08:25:07.932000000", - "end_time": "2025-10-09 08:25:11.438000000", - "duration_ms": 3505.745, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "rag_agent", - "agent.operation": "llm-rag", - "agent.capabilities": [ - "document_retrieval", - "context_generation", - "response_generation" - ], - "agent.id": "noveum_rag_agent", - "input_query": "What are Noveum's pricing plans?", - "query_length": 32, - "query_type": "rag_query", - "output_response": "RAG Answer: The provided context does not include specific information about Noveum's pricing plans. However, it mentions that all users will have access to the Observability suite for free, and early users will ...", - "rag.context_length": 2105, - "rag.sources_count": 5, - "rag.answer_length": 399, - "rag.mode": "retrieval_augmented_generation" - }, - "events": [], - "links": [], - "trace_trace_id": "fc64e6cc-6739-4256-ac4a-7b80c3028233", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-09 08:25:07.932000000", - "trace_end_time": "2025-10-09 08:25:11.438000000", - "trace_duration_ms": 3506.0989999999997, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-09 08:25:12.651000000", - "updated_at": "2025-10-09 08:25:12.651000000" - }, - { - "span_id": "48155aad-6204-4a52-8624-8fa5d6fe407f", - "trace_id": "b7945c49-f584-4c70-972d-536a805d8a31", - "parent_span_id": "9c5aec32-b6eb-4c42-93d5-3970fb118c00", - "name": "agent.llm-rag", - "start_time": "2025-10-09 08:24:58.518000000", - "end_time": "2025-10-09 08:25:06.920000000", - "duration_ms": 8401.725, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "rag_agent", - "agent.operation": "llm-rag", - "agent.capabilities": [ - "document_retrieval", - "context_generation", - "response_generation" - ], - "agent.id": "noveum_rag_agent", - "input_query": "How do I integrate Noveum Trace in my application?", - "query_length": 50, - "query_type": "rag_query", - "output_response": "RAG Answer: To integrate Noveum Trace into your application, you can use either the Python SDK (`noveum-trace`) or the TypeScript SDK (`@noveum/trace`). Hereā€™s a quick start guide based on the provided documentat...", - "rag.context_length": 2803, - "rag.sources_count": 5, - "rag.answer_length": 1151, - "rag.mode": "retrieval_augmented_generation" - }, - "events": [], - "links": [], - "trace_trace_id": "b7945c49-f584-4c70-972d-536a805d8a31", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-09 08:24:58.518000000", - "trace_end_time": "2025-10-09 08:25:06.920000000", - "trace_duration_ms": 8402.516, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-09 08:25:07.620000000", - "updated_at": "2025-10-09 08:25:07.620000000" - }, - { - "span_id": "57ded2eb-0f3f-474e-8704-5ad9974ab4bb", - "trace_id": "f5c40ecf-36c0-45ba-9cc9-dc0329b0324b", - "parent_span_id": "f132d053-f82c-4e66-adf4-bd7ce68b660d", - "name": "agent.llm-rag", - "start_time": "2025-10-09 08:24:54.009000000", - "end_time": "2025-10-09 08:24:57.510000000", - "duration_ms": 3501.569, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "rag_agent", - "agent.operation": "llm-rag", - "agent.capabilities": [ - "document_retrieval", - "context_generation", - "response_generation" - ], - "agent.id": "noveum_rag_agent", - "input_query": "What is Noveum and what does it do?", - "query_length": 35, - "query_type": "rag_query", - "output_response": "RAG Answer: Noveum.ai is a comprehensive tracing and observability platform specifically designed for AI applications, including those powered by large language models (LLMs), retrieval-augmented generation (RAG)...", - "rag.context_length": 2334, - "rag.sources_count": 5, - "rag.answer_length": 611, - "rag.mode": "retrieval_augmented_generation" - }, - "events": [], - "links": [], - "trace_trace_id": "f5c40ecf-36c0-45ba-9cc9-dc0329b0324b", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-09 08:24:54.009000000", - "trace_end_time": "2025-10-09 08:24:57.511000000", - "trace_duration_ms": 3502.062, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-09 08:24:59.716000000", - "updated_at": "2025-10-09 08:24:59.716000000" - } -] \ No newline at end of file diff --git a/noveum_customer_support_bt/split_datasets/agent.llm_model_execution_dataset.json b/noveum_customer_support_bt/split_datasets/agent.llm_model_execution_dataset.json deleted file mode 100644 index 1ca42e5..0000000 --- a/noveum_customer_support_bt/split_datasets/agent.llm_model_execution_dataset.json +++ /dev/null @@ -1,1782 +0,0 @@ -[ - { - "span_id": "ce8556f4-bbeb-46fc-8bfd-dfaedb61cb70", - "trace_id": "eda4fe22-9a2b-4b73-856b-f4f3309bf719", - "parent_span_id": "2aca614b-b016-4d56-abe2-43a144dc681d", - "name": "agent.llm_model_execution", - "start_time": "2025-10-09 08:26:46.681000000", - "end_time": "2025-10-09 08:26:48.685000000", - "duration_ms": 2004.578, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What are the latest breakthroughs in artificial intelligence?", - "query_length": 61, - "query_type": "web_search_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 2.004547119140625, - "model.latency_ms": 2004.547119140625, - "model.start_time": 1759998406.681066, - "model.end_time": 1759998408.6856132, - "model.performance_tier": "medium", - "model.prompt_tokens": 303, - "model.completion_tokens": 105, - "model.total_tokens": 408, - "model.tokens_per_second": 203.53724594656313, - "model.estimated_cost": 0.012240000000000001, - "model.efficiency_score": 1.036764705882353, - "model.response_length": 423, - "model.response_quality": "high", - "model.output_response": "Model Response: The provided web search results do not contain specific information about the latest breakthroughs in artificial intelligence. To find the most recent advancements in AI, I recommend checking reputabl...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "eda4fe22-9a2b-4b73-856b-f4f3309bf719", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-09 08:26:46.384000000", - "trace_end_time": "2025-10-09 08:26:48.685000000", - "trace_duration_ms": 2301.488, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-09 08:26:49.422000000", - "updated_at": "2025-10-09 08:26:49.422000000" - }, - { - "span_id": "cc3073ec-c970-4821-8335-8d0fc8f37930", - "trace_id": "0ffffba1-8a37-443c-8866-d53ffbfa7718", - "parent_span_id": "03e3ae4d-0107-401d-8ca0-05fc0f28fb24", - "name": "agent.llm_model_execution", - "start_time": "2025-10-09 08:26:42.090000000", - "end_time": "2025-10-09 08:26:45.377000000", - "duration_ms": 3287.441, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What's happening in the software development world today?", - "query_length": 57, - "query_type": "web_search_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 3.2873661518096924, - "model.latency_ms": 3287.3661518096924, - "model.start_time": 1759998402.09009, - "model.end_time": 1759998405.3774562, - "model.performance_tier": "medium", - "model.prompt_tokens": 165, - "model.completion_tokens": 198, - "model.total_tokens": 363, - "model.tokens_per_second": 110.4227467330248, - "model.estimated_cost": 0.01089, - "model.efficiency_score": 2.190082644628099, - "model.response_length": 795, - "model.response_quality": "high", - "model.output_response": "Model Response: The web search did not yield any specific results regarding current events in the software development world. Therefore, I cannot provide detailed information on what's happening today in that field.\n...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "0ffffba1-8a37-443c-8866-d53ffbfa7718", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-09 08:26:41.850000000", - "trace_end_time": "2025-10-09 08:26:45.377000000", - "trace_duration_ms": 3527.831, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-09 08:26:49.422000000", - "updated_at": "2025-10-09 08:26:49.422000000" - }, - { - "span_id": "c24fae6d-6314-4342-8214-7e9905ae83db", - "trace_id": "f1f37bd7-0851-4659-b493-b80d3800d920", - "parent_span_id": "ea4d61c5-b8dd-4677-b21b-2b17170e2163", - "name": "agent.llm_model_execution", - "start_time": "2025-10-09 08:26:36.433000000", - "end_time": "2025-10-09 08:26:40.446000000", - "duration_ms": 4013.1009999999997, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What are the newest features in cloud computing?", - "query_length": 48, - "query_type": "rag_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 4.012948989868164, - "model.latency_ms": 4012.948989868164, - "model.start_time": 1759998396.433433, - "model.end_time": 1759998400.446382, - "model.performance_tier": "medium", - "model.prompt_tokens": 780, - "model.completion_tokens": 220, - "model.total_tokens": 1000, - "model.tokens_per_second": 249.1932996219951, - "model.estimated_cost": 0.030000000000000002, - "model.efficiency_score": 0.883, - "model.response_length": 883, - "model.response_quality": "high", - "model.output_response": "Model Response: The provided context does not specifically mention any new features in cloud computing as a general topic. However, it does highlight several recent enhancements and features related to the Noveum.ai ...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "f1f37bd7-0851-4659-b493-b80d3800d920", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-09 08:26:35.915000000", - "trace_end_time": "2025-10-09 08:26:40.841000000", - "trace_duration_ms": 4926.342, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-09 08:26:44.053000000", - "updated_at": "2025-10-09 08:26:44.053000000" - }, - { - "span_id": "93879d8f-77c7-4fd4-881b-be500e2dcbde", - "trace_id": "43cdf081-4f01-49cd-b566-dbd1619e6cd2", - "parent_span_id": "dfdb9aaf-9925-4a5c-a0c6-2cb00f70d4d3", - "name": "agent.llm_model_execution", - "start_time": "2025-10-09 08:26:32.526000000", - "end_time": "2025-10-09 08:26:34.906000000", - "duration_ms": 2379.957, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What's the current status of cryptocurrency markets?", - "query_length": 52, - "query_type": "web_search_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 2.3798890113830566, - "model.latency_ms": 2379.8890113830566, - "model.start_time": 1759998392.526488, - "model.end_time": 1759998394.906377, - "model.performance_tier": "medium", - "model.prompt_tokens": 162, - "model.completion_tokens": 92, - "model.total_tokens": 254, - "model.tokens_per_second": 106.72766620002568, - "model.estimated_cost": 0.00762, - "model.efficiency_score": 1.4606299212598426, - "model.response_length": 371, - "model.response_quality": "high", - "model.output_response": "Model Response: The web search results did not provide any specific information regarding the current status of cryptocurrency markets. Therefore, I cannot provide an accurate update on market conditions, trends, or ...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "43cdf081-4f01-49cd-b566-dbd1619e6cd2", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-09 08:26:32.237000000", - "trace_end_time": "2025-10-09 08:26:34.906000000", - "trace_duration_ms": 2668.841, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-09 08:26:38.618000000", - "updated_at": "2025-10-09 08:26:38.618000000" - }, - { - "span_id": "f92643c0-44a3-4fae-9d29-45321342d12f", - "trace_id": "9a1983f4-09da-4b53-80e6-38de6878e0e7", - "parent_span_id": "eefcfdfc-6433-459e-a134-e2647b8418e4", - "name": "agent.llm_model_execution", - "start_time": "2025-10-09 08:26:28.250000000", - "end_time": "2025-10-09 08:26:31.231000000", - "duration_ms": 2981.5170000000003, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What are the latest updates in Python programming?", - "query_length": 50, - "query_type": "web_search_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 2.981428861618042, - "model.latency_ms": 2981.428861618042, - "model.start_time": 1759998388.250087, - "model.end_time": 1759998391.231516, - "model.performance_tier": "medium", - "model.prompt_tokens": 285, - "model.completion_tokens": 145, - "model.total_tokens": 430, - "model.tokens_per_second": 144.22614791708833, - "model.estimated_cost": 0.0129, - "model.efficiency_score": 1.3558139534883722, - "model.response_length": 583, - "model.response_quality": "high", - "model.output_response": "Model Response: The web search results do not provide specific information about the latest updates in Python programming. They focus primarily on news related to China, including geopolitics, economy, and lifestyle,...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "9a1983f4-09da-4b53-80e6-38de6878e0e7", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-09 08:26:27.992000000", - "trace_end_time": "2025-10-09 08:26:31.231000000", - "trace_duration_ms": 3239.706, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-09 08:26:33.463000000", - "updated_at": "2025-10-09 08:26:33.463000000" - }, - { - "span_id": "44f379aa-10fe-4a51-847c-5288444d7843", - "trace_id": "5d2517e1-220a-429d-9d59-f701bda25eed", - "parent_span_id": "c57b4b54-b9b0-4a4c-b5cd-57a4cfe5301b", - "name": "agent.llm_model_execution", - "start_time": "2025-10-09 08:26:23.486000000", - "end_time": "2025-10-09 08:26:26.985000000", - "duration_ms": 3498.348, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What happened in tech news this week?", - "query_length": 37, - "query_type": "web_search_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 3.4982941150665283, - "model.latency_ms": 3498.2941150665283, - "model.start_time": 1759998383.486767, - "model.end_time": 1759998386.9850612, - "model.performance_tier": "medium", - "model.prompt_tokens": 280, - "model.completion_tokens": 125, - "model.total_tokens": 405, - "model.tokens_per_second": 115.77071186088594, - "model.estimated_cost": 0.012150000000000001, - "model.efficiency_score": 1.237037037037037, - "model.response_length": 501, - "model.response_quality": "high", - "model.output_response": "Model Response: The web search results do not provide specific information about recent events in tech news for this week. They primarily discuss the grammatical usage of the phrase \"what happened\" and its variations...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "5d2517e1-220a-429d-9d59-f701bda25eed", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-09 08:26:23.210000000", - "trace_end_time": "2025-10-09 08:26:26.985000000", - "trace_duration_ms": 3775.124, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-09 08:26:28.174000000", - "updated_at": "2025-10-09 08:26:28.174000000" - }, - { - "span_id": "e8e79f2a-2ed0-4981-943c-adcf6f7d2b15", - "trace_id": "52aacb67-c361-4445-9b72-c157f79f47d6", - "parent_span_id": "7505f97a-cbc0-42fe-8a72-95387d723daa", - "name": "agent.llm_model_execution", - "start_time": "2025-10-09 08:26:17.058000000", - "end_time": "2025-10-09 08:26:21.807000000", - "duration_ms": 4748.643, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What are the current trends in observability tools?", - "query_length": 51, - "query_type": "rag_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 4.748546838760376, - "model.latency_ms": 4748.546838760376, - "model.start_time": 1759998377.058629, - "model.end_time": 1759998381.8071759, - "model.performance_tier": "medium", - "model.prompt_tokens": 872, - "model.completion_tokens": 268, - "model.total_tokens": 1140, - "model.tokens_per_second": 240.07344535272622, - "model.estimated_cost": 0.0342, - "model.efficiency_score": 0.9421052631578948, - "model.response_length": 1074, - "model.response_quality": "high", - "model.output_response": "Model Response: The provided context does not explicitly outline current trends in observability tools. However, it highlights some key aspects of observability that are particularly relevant to AI applications. \n\n1....", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "52aacb67-c361-4445-9b72-c157f79f47d6", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-09 08:26:16.544000000", - "trace_end_time": "2025-10-09 08:26:22.201000000", - "trace_duration_ms": 5657.012, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-09 08:26:22.904000000", - "updated_at": "2025-10-09 08:26:22.904000000" - }, - { - "span_id": "e6d0eb41-5f64-44c5-8a42-a80c336de03e", - "trace_id": "a81ca3a8-80aa-4c39-876e-8d40ea7a0aef", - "parent_span_id": "a062bef6-d973-489b-91df-5d40a2aef077", - "name": "agent.llm_model_execution", - "start_time": "2025-10-09 08:26:09.227000000", - "end_time": "2025-10-09 08:26:15.537000000", - "duration_ms": 6309.442, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: Tell me about recent developments in machine learning", - "query_length": 53, - "query_type": "web_search_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 6.309402227401733, - "model.latency_ms": 6309.402227401733, - "model.start_time": 1759998369.2277088, - "model.end_time": 1759998375.537111, - "model.performance_tier": "slow", - "model.prompt_tokens": 309, - "model.completion_tokens": 409, - "model.total_tokens": 718, - "model.tokens_per_second": 113.79841926731602, - "model.estimated_cost": 0.02154, - "model.efficiency_score": 2.2827298050139277, - "model.response_length": 1639, - "model.response_quality": "high", - "model.output_response": "Model Response: The provided web search results do not contain specific information about recent developments in machine learning. Therefore, I cannot provide a detailed answer based on those results.\n\nHowever, I can...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "a81ca3a8-80aa-4c39-876e-8d40ea7a0aef", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-09 08:26:08.903000000", - "trace_end_time": "2025-10-09 08:26:15.537000000", - "trace_duration_ms": 6633.662, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-09 08:26:16.227000000", - "updated_at": "2025-10-09 08:26:16.227000000" - }, - { - "span_id": "c545cb62-5a51-4f2e-b020-b3f1d1dfe217", - "trace_id": "230aad27-f3dd-4968-a45a-3c2f07ac28ed", - "parent_span_id": "41f82a95-620d-440c-84e6-36d6d1b9e079", - "name": "agent.llm_model_execution", - "start_time": "2025-10-09 08:26:04.977000000", - "end_time": "2025-10-09 08:26:07.893000000", - "duration_ms": 2916.6910000000003, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What's the weather like today?", - "query_length": 30, - "query_type": "web_search_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 2.9165940284729004, - "model.latency_ms": 2916.5940284729004, - "model.start_time": 1759998364.977324, - "model.end_time": 1759998367.893918, - "model.performance_tier": "medium", - "model.prompt_tokens": 343, - "model.completion_tokens": 94, - "model.total_tokens": 437, - "model.tokens_per_second": 149.83230293069235, - "model.estimated_cost": 0.01311, - "model.efficiency_score": 0.8604118993135011, - "model.response_length": 376, - "model.response_quality": "high", - "model.output_response": "Model Response: The provided web search results do not contain specific information about today's weather in any particular location. They mention weather forecasts for various places, including France and Singapore,...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "230aad27-f3dd-4968-a45a-3c2f07ac28ed", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-09 08:26:02.424000000", - "trace_end_time": "2025-10-09 08:26:07.894000000", - "trace_duration_ms": 5470.232, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-09 08:26:08.586000000", - "updated_at": "2025-10-09 08:26:08.586000000" - }, - { - "span_id": "c2f21e4d-a5e6-4134-aeaf-96e2f566cf96", - "trace_id": "83c7dcce-3d89-4da1-8b3f-d419885d4cbc", - "parent_span_id": "63f2a0d5-9016-4483-8ec0-e6aa6b27689f", - "name": "agent.llm_model_execution", - "start_time": "2025-10-09 08:25:57.553000000", - "end_time": "2025-10-09 08:26:01.415000000", - "duration_ms": 3862.011, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What are the latest AI news today?", - "query_length": 34, - "query_type": "web_search_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 3.8618290424346924, - "model.latency_ms": 3861.8290424346924, - "model.start_time": 1759998357.553477, - "model.end_time": 1759998361.415306, - "model.performance_tier": "medium", - "model.prompt_tokens": 365, - "model.completion_tokens": 250, - "model.total_tokens": 615, - "model.tokens_per_second": 159.25096456685014, - "model.estimated_cost": 0.01845, - "model.efficiency_score": 1.6308943089430894, - "model.response_length": 1003, - "model.response_quality": "high", - "model.output_response": "Model Response: The latest news in artificial intelligence includes several key updates:\n\n1. **AI Spending Concerns**: Jeff Bezos has expressed concerns that the current level of investment in AI resembles an \"indust...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "83c7dcce-3d89-4da1-8b3f-d419885d4cbc", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-09 08:25:56.886000000", - "trace_end_time": "2025-10-09 08:26:01.416000000", - "trace_duration_ms": 4530.08, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-09 08:26:02.122000000", - "updated_at": "2025-10-09 08:26:02.122000000" - }, - { - "span_id": "39d99d2b-518f-439c-8758-7a01b970fd9e", - "trace_id": "2218f641-604c-491a-9710-b51a9941b982", - "parent_span_id": "8a6e18a1-babc-4166-926f-188f344d514e", - "name": "agent.llm_model_execution", - "start_time": "2025-10-09 08:25:48.496000000", - "end_time": "2025-10-09 08:25:55.406000000", - "duration_ms": 6910.508, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What are the benefits of using Noveum Trace?", - "query_length": 44, - "query_type": "rag_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 6.910433053970337, - "model.latency_ms": 6910.433053970337, - "model.start_time": 1759998348.4961321, - "model.end_time": 1759998355.4065652, - "model.performance_tier": "slow", - "model.prompt_tokens": 828, - "model.completion_tokens": 365, - "model.total_tokens": 1193, - "model.tokens_per_second": 172.63751644545212, - "model.estimated_cost": 0.03579, - "model.efficiency_score": 1.2238055322715842, - "model.response_length": 1460, - "model.response_quality": "high", - "model.output_response": "Model Response: Using Noveum Trace offers several benefits, including:\n\n1. **Real-Time Monitoring**: The Noveum platform provides a real-time dashboard for analyzing traces and performance, with live updates as new t...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "2218f641-604c-491a-9710-b51a9941b982", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-09 08:25:48.107000000", - "trace_end_time": "2025-10-09 08:25:55.878000000", - "trace_duration_ms": 7770.953, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-09 08:25:56.565000000", - "updated_at": "2025-10-09 08:25:56.565000000" - }, - { - "span_id": "9e5ade74-3815-4dad-8abe-d4281d7e62da", - "trace_id": "255fd49c-84b4-4b18-887e-6308a412d535", - "parent_span_id": "5bec4165-6596-43ac-930b-acfcde1cc02a", - "name": "agent.llm_model_execution", - "start_time": "2025-10-09 08:25:44.242000000", - "end_time": "2025-10-09 08:25:46.618000000", - "duration_ms": 2375.3199999999997, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: How do I configure Noveum for my system?", - "query_length": 40, - "query_type": "rag_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 2.3752830028533936, - "model.latency_ms": 2375.2830028533936, - "model.start_time": 1759998344.2429838, - "model.end_time": 1759998346.6182668, - "model.performance_tier": "medium", - "model.prompt_tokens": 693, - "model.completion_tokens": 153, - "model.total_tokens": 846, - "model.tokens_per_second": 356.1680856486205, - "model.estimated_cost": 0.02538, - "model.efficiency_score": 0.7245862884160756, - "model.response_length": 613, - "model.response_quality": "high", - "model.output_response": "Model Response: To configure Noveum for your system, you can refer to the SDK Integration Guide available in the Noveum documentation. This guide will help you trace your AI applications with minimal code changes. Th...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "255fd49c-84b4-4b18-887e-6308a412d535", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-09 08:25:43.830000000", - "trace_end_time": "2025-10-09 08:25:47.099000000", - "trace_duration_ms": 3268.862, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-09 08:25:50.304000000", - "updated_at": "2025-10-09 08:25:50.304000000" - }, - { - "span_id": "4f38a85b-0721-47a5-8a51-d0808e460f96", - "trace_id": "dc511122-c0b6-415c-9a49-c7b45132dd87", - "parent_span_id": "f9f79a07-4eda-41b8-9634-d4460114caad", - "name": "agent.llm_model_execution", - "start_time": "2025-10-09 08:25:38.732000000", - "end_time": "2025-10-09 08:25:42.276000000", - "duration_ms": 3544.302, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What monitoring capabilities does Noveum provide?", - "query_length": 49, - "query_type": "rag_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 3.5442299842834473, - "model.latency_ms": 3544.2299842834473, - "model.start_time": 1759998338.732058, - "model.end_time": 1759998342.276288, - "model.performance_tier": "medium", - "model.prompt_tokens": 821, - "model.completion_tokens": 209, - "model.total_tokens": 1030, - "model.tokens_per_second": 290.613195127697, - "model.estimated_cost": 0.0309, - "model.efficiency_score": 0.8145631067961165, - "model.response_length": 839, - "model.response_quality": "high", - "model.output_response": "Model Response: Noveum provides comprehensive monitoring capabilities for AI applications, including the ability to monitor, trace, and optimize AI agents across various frameworks such as LangChain, CrewAI, AutoGen,...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "dc511122-c0b6-415c-9a49-c7b45132dd87", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-09 08:25:38.322000000", - "trace_end_time": "2025-10-09 08:25:42.822000000", - "trace_duration_ms": 4500.425, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-09 08:25:45.028000000", - "updated_at": "2025-10-09 08:25:45.028000000" - }, - { - "span_id": "f4c03a8d-b90d-4c1a-a259-826d09e3e57a", - "trace_id": "04bebf38-a343-4563-80db-0154bef8d927", - "parent_span_id": "edc52a5d-8450-4d0f-ad14-958ec4d147f4", - "name": "agent.llm_model_execution", - "start_time": "2025-10-09 08:25:34.043000000", - "end_time": "2025-10-09 08:25:36.950000000", - "duration_ms": 2907.183, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: How does Noveum handle agent tracing?", - "query_length": 37, - "query_type": "rag_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 2.906853199005127, - "model.latency_ms": 2906.853199005127, - "model.start_time": 1759998334.043113, - "model.end_time": 1759998336.9499662, - "model.performance_tier": "medium", - "model.prompt_tokens": 817, - "model.completion_tokens": 172, - "model.total_tokens": 989, - "model.tokens_per_second": 340.2304596387895, - "model.estimated_cost": 0.029670000000000002, - "model.efficiency_score": 0.698685540950455, - "model.response_length": 691, - "model.response_quality": "high", - "model.output_response": "Model Response: Noveum.ai handles agent tracing through its specialized Multi-Agent Tracing capabilities, which are designed to observe complex workflows and inter-agent communications within multi-agent systems. The...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "04bebf38-a343-4563-80db-0154bef8d927", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-09 08:25:33.693000000", - "trace_end_time": "2025-10-09 08:25:37.313000000", - "trace_duration_ms": 3619.672, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-09 08:25:40.339000000", - "updated_at": "2025-10-09 08:25:40.339000000" - }, - { - "span_id": "48fb40ba-a39f-4e13-9d94-18794c7e5e2b", - "trace_id": "5e043630-6493-42b5-beb8-79faa19bfa37", - "parent_span_id": "6c362c73-e063-4813-8180-d26b6a52b798", - "name": "agent.llm_model_execution", - "start_time": "2025-10-09 08:25:30.278000000", - "end_time": "2025-10-09 08:25:32.086000000", - "duration_ms": 1807.8110000000001, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What APIs are available in Noveum platform?", - "query_length": 43, - "query_type": "rag_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 1.8077418804168701, - "model.latency_ms": 1807.7418804168701, - "model.start_time": 1759998330.278618, - "model.end_time": 1759998332.08636, - "model.performance_tier": "fast", - "model.prompt_tokens": 705, - "model.completion_tokens": 67, - "model.total_tokens": 772, - "model.tokens_per_second": 427.05211864758854, - "model.estimated_cost": 0.02316, - "model.efficiency_score": 0.34974093264248707, - "model.response_length": 270, - "model.response_quality": "high", - "model.output_response": "Model Response: The provided context does not specify the APIs available in the Noveum platform. For detailed information about the APIs, I recommend checking the official Noveum documentation or the SDK Integration ...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "5e043630-6493-42b5-beb8-79faa19bfa37", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-09 08:25:29.901000000", - "trace_end_time": "2025-10-09 08:25:32.685000000", - "trace_duration_ms": 2783.326, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-09 08:25:34.918000000", - "updated_at": "2025-10-09 08:25:34.918000000" - }, - { - "span_id": "d4d544a0-f052-4889-82eb-4346f2e447a7", - "trace_id": "7da9814d-a2e8-4c4e-b750-68b26bd5fd22", - "parent_span_id": "1393d065-a9a4-4475-808f-2d00b3c68939", - "name": "agent.llm_model_execution", - "start_time": "2025-10-09 08:25:21.471000000", - "end_time": "2025-10-09 08:25:28.496000000", - "duration_ms": 7024.773999999999, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: How do I set up observability with Noveum?", - "query_length": 42, - "query_type": "rag_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 7.024708986282349, - "model.latency_ms": 7024.708986282349, - "model.start_time": 1759998321.471663, - "model.end_time": 1759998328.496372, - "model.performance_tier": "slow", - "model.prompt_tokens": 852, - "model.completion_tokens": 237, - "model.total_tokens": 1089, - "model.tokens_per_second": 155.02421554068192, - "model.estimated_cost": 0.03267, - "model.efficiency_score": 0.8732782369146006, - "model.response_length": 951, - "model.response_quality": "high", - "model.output_response": "Model Response: To set up observability with Noveum, follow these steps:\n\n1. **Sign Up**: Go to [noveum.ai](https://noveum.ai) and create an account.\n2. **Create a Project**: Once signed in, create a new project to o...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "7da9814d-a2e8-4c4e-b750-68b26bd5fd22", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-09 08:25:20.586000000", - "trace_end_time": "2025-10-09 08:25:28.897000000", - "trace_duration_ms": 8310.205, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-09 08:25:29.588000000", - "updated_at": "2025-10-09 08:25:29.588000000" - }, - { - "span_id": "304f9e3d-9063-4661-9777-618efc49f5c1", - "trace_id": "16143f74-2831-4753-b33d-ce4b645093c5", - "parent_span_id": "f419377a-fc24-4abf-8a1c-03857824e374", - "name": "agent.llm_model_execution", - "start_time": "2025-10-09 08:25:12.892000000", - "end_time": "2025-10-09 08:25:19.220000000", - "duration_ms": 6327.166, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What features does Noveum Trace offer?", - "query_length": 38, - "query_type": "rag_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 6.327087163925171, - "model.latency_ms": 6327.087163925171, - "model.start_time": 1759998312.892979, - "model.end_time": 1759998319.220066, - "model.performance_tier": "slow", - "model.prompt_tokens": 830, - "model.completion_tokens": 253, - "model.total_tokens": 1083, - "model.tokens_per_second": 171.16881306375637, - "model.estimated_cost": 0.03249, - "model.efficiency_score": 0.9362880886426593, - "model.response_length": 1014, - "model.response_quality": "high", - "model.output_response": "Model Response: Noveum Trace offers several key features through its Python SDK (`noveum-trace`) and TypeScript SDK (`@noveum/trace`):\n\n1. **Decorator-based Tracing**: It provides a simple way to trace LLM calls, age...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "16143f74-2831-4753-b33d-ce4b645093c5", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-09 08:25:12.442000000", - "trace_end_time": "2025-10-09 08:25:19.579000000", - "trace_duration_ms": 7137.402, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-09 08:25:20.290000000", - "updated_at": "2025-10-09 08:25:20.290000000" - }, - { - "span_id": "e0b4ccfb-18e7-4d71-b2dd-5b807eb27513", - "trace_id": "fc64e6cc-6739-4256-ac4a-7b80c3028233", - "parent_span_id": "0e58e31f-a8ab-4aee-8fc2-abcae1e21d9d", - "name": "agent.llm_model_execution", - "start_time": "2025-10-09 08:25:08.383000000", - "end_time": "2025-10-09 08:25:10.938000000", - "duration_ms": 2555.3149999999996, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What are Noveum's pricing plans?", - "query_length": 32, - "query_type": "rag_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 2.5551459789276123, - "model.latency_ms": 2555.1459789276123, - "model.start_time": 1759998308.383137, - "model.end_time": 1759998310.938283, - "model.performance_tier": "medium", - "model.prompt_tokens": 658, - "model.completion_tokens": 99, - "model.total_tokens": 757, - "model.tokens_per_second": 296.264873413499, - "model.estimated_cost": 0.02271, - "model.efficiency_score": 0.5270805812417437, - "model.response_length": 399, - "model.response_quality": "high", - "model.output_response": "Model Response: The provided context does not include specific information about Noveum's pricing plans. However, it mentions that all users will have access to the Observability suite for free, and early users will ...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "fc64e6cc-6739-4256-ac4a-7b80c3028233", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-09 08:25:07.932000000", - "trace_end_time": "2025-10-09 08:25:11.438000000", - "trace_duration_ms": 3506.0989999999997, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-09 08:25:12.651000000", - "updated_at": "2025-10-09 08:25:12.651000000" - }, - { - "span_id": "f1425131-f8a2-43f7-aa17-d009f745ed06", - "trace_id": "b7945c49-f584-4c70-972d-536a805d8a31", - "parent_span_id": "48155aad-6204-4a52-8624-8fa5d6fe407f", - "name": "agent.llm_model_execution", - "start_time": "2025-10-09 08:24:58.886000000", - "end_time": "2025-10-09 08:25:05.872000000", - "duration_ms": 6986.124, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: How do I integrate Noveum Trace in my application?", - "query_length": 50, - "query_type": "rag_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 6.985715866088867, - "model.latency_ms": 6985.715866088867, - "model.start_time": 1759998298.886451, - "model.end_time": 1759998305.8721669, - "model.performance_tier": "slow", - "model.prompt_tokens": 837, - "model.completion_tokens": 287, - "model.total_tokens": 1124, - "model.tokens_per_second": 160.89975910075202, - "model.estimated_cost": 0.03372, - "model.efficiency_score": 1.0240213523131672, - "model.response_length": 1151, - "model.response_quality": "high", - "model.output_response": "Model Response: To integrate Noveum Trace into your application, you can use either the Python SDK (`noveum-trace`) or the TypeScript SDK (`@noveum/trace`). Hereā€™s a quick start guide based on the provided documentat...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "b7945c49-f584-4c70-972d-536a805d8a31", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-09 08:24:58.518000000", - "trace_end_time": "2025-10-09 08:25:06.920000000", - "trace_duration_ms": 8402.516, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-09 08:25:07.620000000", - "updated_at": "2025-10-09 08:25:07.620000000" - }, - { - "span_id": "f41e836c-1a82-46e0-9d51-011713874e94", - "trace_id": "f5c40ecf-36c0-45ba-9cc9-dc0329b0324b", - "parent_span_id": "57ded2eb-0f3f-474e-8704-5ad9974ab4bb", - "name": "agent.llm_model_execution", - "start_time": "2025-10-09 08:24:54.382000000", - "end_time": "2025-10-09 08:24:56.965000000", - "duration_ms": 2583.161, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What is Noveum and what does it do?", - "query_length": 35, - "query_type": "rag_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 2.5830719470977783, - "model.latency_ms": 2583.0719470977783, - "model.start_time": 1759998294.382122, - "model.end_time": 1759998296.965194, - "model.performance_tier": "medium", - "model.prompt_tokens": 716, - "model.completion_tokens": 152, - "model.total_tokens": 868, - "model.tokens_per_second": 336.03400051448244, - "model.estimated_cost": 0.02604, - "model.efficiency_score": 0.7039170506912442, - "model.response_length": 611, - "model.response_quality": "high", - "model.output_response": "Model Response: Noveum.ai is a comprehensive tracing and observability platform specifically designed for AI applications, including those powered by large language models (LLMs), retrieval-augmented generation (RAG)...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "f5c40ecf-36c0-45ba-9cc9-dc0329b0324b", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-09 08:24:54.009000000", - "trace_end_time": "2025-10-09 08:24:57.511000000", - "trace_duration_ms": 3502.062, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-09 08:24:59.716000000", - "updated_at": "2025-10-09 08:24:59.716000000" - } -] \ No newline at end of file diff --git a/noveum_customer_support_bt/split_datasets/agent.query_routing_dataset.json b/noveum_customer_support_bt/split_datasets/agent.query_routing_dataset.json deleted file mode 100644 index a2b0f8a..0000000 --- a/noveum_customer_support_bt/split_datasets/agent.query_routing_dataset.json +++ /dev/null @@ -1,1222 +0,0 @@ -[ - { - "span_id": "0b19befc-5dc0-4b00-9a8b-5f2f2c914f17", - "trace_id": "eda4fe22-9a2b-4b73-856b-f4f3309bf719", - "parent_span_id": "b6b7c1c0-3b7b-4e19-805d-a16beb7ea871", - "name": "agent.query_routing", - "start_time": "2025-10-09 08:26:46.384000000", - "end_time": "2025-10-09 08:26:48.685000000", - "duration_ms": 2301.289, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What are the latest breakthroughs in artificial intelligence?", - "query_length": 61, - "query_type": "routing_query", - "output_response": "Routed to Web Search for query processing", - "router.classification": "Web Search", - "router.confidence_score": 1, - "router.rag_keyword_score": 0, - "router.web_keyword_score": 1, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "eda4fe22-9a2b-4b73-856b-f4f3309bf719", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-09 08:26:46.384000000", - "trace_end_time": "2025-10-09 08:26:48.685000000", - "trace_duration_ms": 2301.488, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-09 08:26:49.422000000", - "updated_at": "2025-10-09 08:26:49.422000000" - }, - { - "span_id": "30644d2a-5ab1-4a73-b4ff-cf9fa478e36b", - "trace_id": "0ffffba1-8a37-443c-8866-d53ffbfa7718", - "parent_span_id": "3045d971-b7ff-46cc-be11-6db7d65dfefd", - "name": "agent.query_routing", - "start_time": "2025-10-09 08:26:41.850000000", - "end_time": "2025-10-09 08:26:45.377000000", - "duration_ms": 3527.556, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What's happening in the software development world today?", - "query_length": 57, - "query_type": "routing_query", - "output_response": "Routed to Web Search for query processing", - "router.classification": "Web Search", - "router.confidence_score": 1, - "router.rag_keyword_score": 0, - "router.web_keyword_score": 1, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "0ffffba1-8a37-443c-8866-d53ffbfa7718", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-09 08:26:41.850000000", - "trace_end_time": "2025-10-09 08:26:45.377000000", - "trace_duration_ms": 3527.831, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-09 08:26:49.422000000", - "updated_at": "2025-10-09 08:26:49.422000000" - }, - { - "span_id": "89eac2de-b7da-4db3-bf7c-8e87c429481e", - "trace_id": "f1f37bd7-0851-4659-b493-b80d3800d920", - "parent_span_id": "42d4cb1f-17ec-433a-8087-90292c2fa9d9", - "name": "agent.query_routing", - "start_time": "2025-10-09 08:26:35.915000000", - "end_time": "2025-10-09 08:26:40.841000000", - "duration_ms": 4926.115000000001, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What are the newest features in cloud computing?", - "query_length": 48, - "query_type": "routing_query", - "output_response": "Routed to RAG for query processing", - "router.classification": "RAG", - "router.confidence_score": 1, - "router.rag_keyword_score": 1, - "router.web_keyword_score": 0, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "f1f37bd7-0851-4659-b493-b80d3800d920", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-09 08:26:35.915000000", - "trace_end_time": "2025-10-09 08:26:40.841000000", - "trace_duration_ms": 4926.342, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-09 08:26:44.053000000", - "updated_at": "2025-10-09 08:26:44.053000000" - }, - { - "span_id": "962afebb-26d1-45ff-acf1-218163cd6bb8", - "trace_id": "43cdf081-4f01-49cd-b566-dbd1619e6cd2", - "parent_span_id": "e22b809d-5320-40b7-b78a-e3e125538a6b", - "name": "agent.query_routing", - "start_time": "2025-10-09 08:26:32.240000000", - "end_time": "2025-10-09 08:26:34.906000000", - "duration_ms": 2666.057, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What's the current status of cryptocurrency markets?", - "query_length": 52, - "query_type": "routing_query", - "output_response": "Routed to Web Search for query processing", - "router.classification": "Web Search", - "router.confidence_score": 1, - "router.rag_keyword_score": 0, - "router.web_keyword_score": 3, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "43cdf081-4f01-49cd-b566-dbd1619e6cd2", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-09 08:26:32.237000000", - "trace_end_time": "2025-10-09 08:26:34.906000000", - "trace_duration_ms": 2668.841, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-09 08:26:38.618000000", - "updated_at": "2025-10-09 08:26:38.618000000" - }, - { - "span_id": "246cf822-4562-4d3a-8a5a-e38e0a0b02a0", - "trace_id": "9a1983f4-09da-4b53-80e6-38de6878e0e7", - "parent_span_id": "813a4450-1cf8-4d43-8493-eabfa200919b", - "name": "agent.query_routing", - "start_time": "2025-10-09 08:26:27.992000000", - "end_time": "2025-10-09 08:26:31.231000000", - "duration_ms": 3239.343, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What are the latest updates in Python programming?", - "query_length": 50, - "query_type": "routing_query", - "output_response": "Routed to Web Search for query processing", - "router.classification": "Web Search", - "router.confidence_score": 1, - "router.rag_keyword_score": 0, - "router.web_keyword_score": 2, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "9a1983f4-09da-4b53-80e6-38de6878e0e7", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-09 08:26:27.992000000", - "trace_end_time": "2025-10-09 08:26:31.231000000", - "trace_duration_ms": 3239.706, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-09 08:26:33.463000000", - "updated_at": "2025-10-09 08:26:33.463000000" - }, - { - "span_id": "20544f91-8828-4a0e-b253-ed1f8b95b205", - "trace_id": "5d2517e1-220a-429d-9d59-f701bda25eed", - "parent_span_id": "6102352a-833e-493d-adbb-887e2af17918", - "name": "agent.query_routing", - "start_time": "2025-10-09 08:26:23.210000000", - "end_time": "2025-10-09 08:26:26.985000000", - "duration_ms": 3774.906, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What happened in tech news this week?", - "query_length": 37, - "query_type": "routing_query", - "output_response": "Routed to Web Search for query processing", - "router.classification": "Web Search", - "router.confidence_score": 1, - "router.rag_keyword_score": 0, - "router.web_keyword_score": 2, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "5d2517e1-220a-429d-9d59-f701bda25eed", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-09 08:26:23.210000000", - "trace_end_time": "2025-10-09 08:26:26.985000000", - "trace_duration_ms": 3775.124, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-09 08:26:28.174000000", - "updated_at": "2025-10-09 08:26:28.174000000" - }, - { - "span_id": "5359c861-c51b-446d-9ad2-b3d8c21ea70f", - "trace_id": "52aacb67-c361-4445-9b72-c157f79f47d6", - "parent_span_id": "2926418a-fe09-4df0-bf19-a527bc24a478", - "name": "agent.query_routing", - "start_time": "2025-10-09 08:26:16.544000000", - "end_time": "2025-10-09 08:26:22.201000000", - "duration_ms": 5656.666, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What are the current trends in observability tools?", - "query_length": 51, - "query_type": "routing_query", - "output_response": "Routed to RAG for query processing", - "router.classification": "RAG", - "router.confidence_score": 0.3333333333333333, - "router.rag_keyword_score": 2, - "router.web_keyword_score": 1, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "52aacb67-c361-4445-9b72-c157f79f47d6", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-09 08:26:16.544000000", - "trace_end_time": "2025-10-09 08:26:22.201000000", - "trace_duration_ms": 5657.012, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-09 08:26:22.904000000", - "updated_at": "2025-10-09 08:26:22.904000000" - }, - { - "span_id": "acf6a99a-39e0-4f87-a07e-3a61213ab032", - "trace_id": "a81ca3a8-80aa-4c39-876e-8d40ea7a0aef", - "parent_span_id": "afe85a1d-13c8-4476-8154-52d58ba83d52", - "name": "agent.query_routing", - "start_time": "2025-10-09 08:26:08.903000000", - "end_time": "2025-10-09 08:26:15.537000000", - "duration_ms": 6633.4220000000005, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "Tell me about recent developments in machine learning", - "query_length": 53, - "query_type": "routing_query", - "output_response": "Routed to Web Search for query processing", - "router.classification": "Web Search", - "router.confidence_score": 1, - "router.rag_keyword_score": 0, - "router.web_keyword_score": 1, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "a81ca3a8-80aa-4c39-876e-8d40ea7a0aef", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-09 08:26:08.903000000", - "trace_end_time": "2025-10-09 08:26:15.537000000", - "trace_duration_ms": 6633.662, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-09 08:26:16.227000000", - "updated_at": "2025-10-09 08:26:16.227000000" - }, - { - "span_id": "0b47ea07-1eda-4b6a-a183-ea9ef047e431", - "trace_id": "230aad27-f3dd-4968-a45a-3c2f07ac28ed", - "parent_span_id": "db8da393-6d72-421e-87dc-96f56b6576e5", - "name": "agent.query_routing", - "start_time": "2025-10-09 08:26:02.424000000", - "end_time": "2025-10-09 08:26:07.894000000", - "duration_ms": 5469.894, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What's the weather like today?", - "query_length": 30, - "query_type": "routing_query", - "output_response": "Routed to Web Search for query processing", - "router.classification": "Web Search", - "router.confidence_score": 1, - "router.rag_keyword_score": 0, - "router.web_keyword_score": 2, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "230aad27-f3dd-4968-a45a-3c2f07ac28ed", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-09 08:26:02.424000000", - "trace_end_time": "2025-10-09 08:26:07.894000000", - "trace_duration_ms": 5470.232, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-09 08:26:08.586000000", - "updated_at": "2025-10-09 08:26:08.586000000" - }, - { - "span_id": "3866ddb3-b17e-47a1-8b3f-bca1e32a2ae4", - "trace_id": "83c7dcce-3d89-4da1-8b3f-d419885d4cbc", - "parent_span_id": "a290202d-1b3f-431f-8728-fcb937380f8c", - "name": "agent.query_routing", - "start_time": "2025-10-09 08:25:56.886000000", - "end_time": "2025-10-09 08:26:01.415000000", - "duration_ms": 4529.776, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What are the latest AI news today?", - "query_length": 34, - "query_type": "routing_query", - "output_response": "Routed to Web Search for query processing", - "router.classification": "Web Search", - "router.confidence_score": 1, - "router.rag_keyword_score": 0, - "router.web_keyword_score": 3, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "83c7dcce-3d89-4da1-8b3f-d419885d4cbc", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-09 08:25:56.886000000", - "trace_end_time": "2025-10-09 08:26:01.416000000", - "trace_duration_ms": 4530.08, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-09 08:26:02.122000000", - "updated_at": "2025-10-09 08:26:02.122000000" - }, - { - "span_id": "3ce2e35e-4518-43b4-8a76-4413da6832ea", - "trace_id": "2218f641-604c-491a-9710-b51a9941b982", - "parent_span_id": "5fbaef83-95fc-4bed-9695-9d69c37c35e4", - "name": "agent.query_routing", - "start_time": "2025-10-09 08:25:48.107000000", - "end_time": "2025-10-09 08:25:55.878000000", - "duration_ms": 7770.734, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What are the benefits of using Noveum Trace?", - "query_length": 44, - "query_type": "routing_query", - "output_response": "Routed to RAG for query processing", - "router.classification": "RAG", - "router.confidence_score": 1, - "router.rag_keyword_score": 2, - "router.web_keyword_score": 0, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "2218f641-604c-491a-9710-b51a9941b982", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-09 08:25:48.107000000", - "trace_end_time": "2025-10-09 08:25:55.878000000", - "trace_duration_ms": 7770.953, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-09 08:25:56.565000000", - "updated_at": "2025-10-09 08:25:56.565000000" - }, - { - "span_id": "c0c39e8b-9a42-4c18-8737-9224cd8a45c0", - "trace_id": "255fd49c-84b4-4b18-887e-6308a412d535", - "parent_span_id": "b3ac3924-928b-4d48-a2a7-aa749802fef9", - "name": "agent.query_routing", - "start_time": "2025-10-09 08:25:43.830000000", - "end_time": "2025-10-09 08:25:47.099000000", - "duration_ms": 3268.6980000000003, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "How do I configure Noveum for my system?", - "query_length": 40, - "query_type": "routing_query", - "output_response": "Routed to RAG for query processing", - "router.classification": "RAG", - "router.confidence_score": 1, - "router.rag_keyword_score": 2, - "router.web_keyword_score": 0, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "255fd49c-84b4-4b18-887e-6308a412d535", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-09 08:25:43.830000000", - "trace_end_time": "2025-10-09 08:25:47.099000000", - "trace_duration_ms": 3268.862, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-09 08:25:50.304000000", - "updated_at": "2025-10-09 08:25:50.304000000" - }, - { - "span_id": "80987c51-ea82-4a41-97c7-ed591082b9bf", - "trace_id": "dc511122-c0b6-415c-9a49-c7b45132dd87", - "parent_span_id": "b8055ffc-ee7d-46ad-b6a0-84e80f9399e2", - "name": "agent.query_routing", - "start_time": "2025-10-09 08:25:38.322000000", - "end_time": "2025-10-09 08:25:42.822000000", - "duration_ms": 4500.249, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What monitoring capabilities does Noveum provide?", - "query_length": 49, - "query_type": "routing_query", - "output_response": "Routed to RAG for query processing", - "router.classification": "RAG", - "router.confidence_score": 1, - "router.rag_keyword_score": 2, - "router.web_keyword_score": 0, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "dc511122-c0b6-415c-9a49-c7b45132dd87", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-09 08:25:38.322000000", - "trace_end_time": "2025-10-09 08:25:42.822000000", - "trace_duration_ms": 4500.425, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-09 08:25:45.028000000", - "updated_at": "2025-10-09 08:25:45.028000000" - }, - { - "span_id": "2f89c1e0-9b62-4651-914c-b71cf561c33c", - "trace_id": "04bebf38-a343-4563-80db-0154bef8d927", - "parent_span_id": "532b9e1f-5c4f-4b41-98bb-e9a1510f0f1c", - "name": "agent.query_routing", - "start_time": "2025-10-09 08:25:33.693000000", - "end_time": "2025-10-09 08:25:37.313000000", - "duration_ms": 3619.46, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "How does Noveum handle agent tracing?", - "query_length": 37, - "query_type": "routing_query", - "output_response": "Routed to RAG for query processing", - "router.classification": "RAG", - "router.confidence_score": 1, - "router.rag_keyword_score": 2, - "router.web_keyword_score": 0, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "04bebf38-a343-4563-80db-0154bef8d927", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-09 08:25:33.693000000", - "trace_end_time": "2025-10-09 08:25:37.313000000", - "trace_duration_ms": 3619.672, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-09 08:25:40.339000000", - "updated_at": "2025-10-09 08:25:40.339000000" - }, - { - "span_id": "e30e3e61-6edd-4b62-ab77-627faa2bf741", - "trace_id": "5e043630-6493-42b5-beb8-79faa19bfa37", - "parent_span_id": "14064e3f-f9c9-4bb7-b8cb-1ad52938b075", - "name": "agent.query_routing", - "start_time": "2025-10-09 08:25:29.901000000", - "end_time": "2025-10-09 08:25:32.685000000", - "duration_ms": 2783.166, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What APIs are available in Noveum platform?", - "query_length": 43, - "query_type": "routing_query", - "output_response": "Routed to RAG for query processing", - "router.classification": "RAG", - "router.confidence_score": 1, - "router.rag_keyword_score": 3, - "router.web_keyword_score": 0, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "5e043630-6493-42b5-beb8-79faa19bfa37", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-09 08:25:29.901000000", - "trace_end_time": "2025-10-09 08:25:32.685000000", - "trace_duration_ms": 2783.326, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-09 08:25:34.918000000", - "updated_at": "2025-10-09 08:25:34.918000000" - }, - { - "span_id": "2f513795-764f-4eb6-8dd2-108d4cdf8432", - "trace_id": "7da9814d-a2e8-4c4e-b750-68b26bd5fd22", - "parent_span_id": "a74fe8a9-2654-4b9d-ba87-dc2f86b6466d", - "name": "agent.query_routing", - "start_time": "2025-10-09 08:25:20.587000000", - "end_time": "2025-10-09 08:25:28.897000000", - "duration_ms": 8310.025, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "How do I set up observability with Noveum?", - "query_length": 42, - "query_type": "routing_query", - "output_response": "Routed to RAG for query processing", - "router.classification": "RAG", - "router.confidence_score": 1, - "router.rag_keyword_score": 2, - "router.web_keyword_score": 0, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "7da9814d-a2e8-4c4e-b750-68b26bd5fd22", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-09 08:25:20.586000000", - "trace_end_time": "2025-10-09 08:25:28.897000000", - "trace_duration_ms": 8310.205, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-09 08:25:29.588000000", - "updated_at": "2025-10-09 08:25:29.588000000" - }, - { - "span_id": "29c7a515-2383-4b7d-8fc7-beac3f403641", - "trace_id": "16143f74-2831-4753-b33d-ce4b645093c5", - "parent_span_id": "c700d0f4-f1f8-4339-aa42-a3060ae2ae11", - "name": "agent.query_routing", - "start_time": "2025-10-09 08:25:12.442000000", - "end_time": "2025-10-09 08:25:19.579000000", - "duration_ms": 7137.247, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What features does Noveum Trace offer?", - "query_length": 38, - "query_type": "routing_query", - "output_response": "Routed to RAG for query processing", - "router.classification": "RAG", - "router.confidence_score": 1, - "router.rag_keyword_score": 3, - "router.web_keyword_score": 0, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "16143f74-2831-4753-b33d-ce4b645093c5", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-09 08:25:12.442000000", - "trace_end_time": "2025-10-09 08:25:19.579000000", - "trace_duration_ms": 7137.402, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-09 08:25:20.290000000", - "updated_at": "2025-10-09 08:25:20.290000000" - }, - { - "span_id": "12fee687-2a2e-401f-83c2-9c8dc7ecd853", - "trace_id": "fc64e6cc-6739-4256-ac4a-7b80c3028233", - "parent_span_id": "f7f5981f-d840-4265-8970-d319b876366d", - "name": "agent.query_routing", - "start_time": "2025-10-09 08:25:07.932000000", - "end_time": "2025-10-09 08:25:11.438000000", - "duration_ms": 3505.9610000000002, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What are Noveum's pricing plans?", - "query_length": 32, - "query_type": "routing_query", - "output_response": "Routed to RAG for query processing", - "router.classification": "RAG", - "router.confidence_score": 1, - "router.rag_keyword_score": 3, - "router.web_keyword_score": 0, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "fc64e6cc-6739-4256-ac4a-7b80c3028233", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-09 08:25:07.932000000", - "trace_end_time": "2025-10-09 08:25:11.438000000", - "trace_duration_ms": 3506.0989999999997, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-09 08:25:12.651000000", - "updated_at": "2025-10-09 08:25:12.651000000" - }, - { - "span_id": "9c5aec32-b6eb-4c42-93d5-3970fb118c00", - "trace_id": "b7945c49-f584-4c70-972d-536a805d8a31", - "parent_span_id": "0c7bbe2b-18af-4fdb-a5bf-4a91daa3bf50", - "name": "agent.query_routing", - "start_time": "2025-10-09 08:24:58.518000000", - "end_time": "2025-10-09 08:25:06.920000000", - "duration_ms": 8402.101999999999, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "How do I integrate Noveum Trace in my application?", - "query_length": 50, - "query_type": "routing_query", - "output_response": "Routed to RAG for query processing", - "router.classification": "RAG", - "router.confidence_score": 1, - "router.rag_keyword_score": 2, - "router.web_keyword_score": 0, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "b7945c49-f584-4c70-972d-536a805d8a31", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-09 08:24:58.518000000", - "trace_end_time": "2025-10-09 08:25:06.920000000", - "trace_duration_ms": 8402.516, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-09 08:25:07.620000000", - "updated_at": "2025-10-09 08:25:07.620000000" - }, - { - "span_id": "f132d053-f82c-4e66-adf4-bd7ce68b660d", - "trace_id": "f5c40ecf-36c0-45ba-9cc9-dc0329b0324b", - "parent_span_id": "dc869742-6797-4016-8061-37530fc7d433", - "name": "agent.query_routing", - "start_time": "2025-10-09 08:24:54.009000000", - "end_time": "2025-10-09 08:24:57.511000000", - "duration_ms": 3501.8790000000004, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What is Noveum and what does it do?", - "query_length": 35, - "query_type": "routing_query", - "output_response": "Routed to RAG for query processing", - "router.classification": "RAG", - "router.confidence_score": 1, - "router.rag_keyword_score": 1, - "router.web_keyword_score": 0, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "f5c40ecf-36c0-45ba-9cc9-dc0329b0324b", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-09 08:24:54.009000000", - "trace_end_time": "2025-10-09 08:24:57.511000000", - "trace_duration_ms": 3502.062, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-09 08:24:59.716000000", - "updated_at": "2025-10-09 08:24:59.716000000" - } -] \ No newline at end of file diff --git a/noveum_customer_support_bt/split_datasets/agent.rag_evaluation_metrics_dataset.json b/noveum_customer_support_bt/split_datasets/agent.rag_evaluation_metrics_dataset.json deleted file mode 100644 index b8091d7..0000000 --- a/noveum_customer_support_bt/split_datasets/agent.rag_evaluation_metrics_dataset.json +++ /dev/null @@ -1,998 +0,0 @@ -[ - { - "span_id": "ac166548-ffa5-47b9-8cf7-4787f270cb5a", - "trace_id": "f1f37bd7-0851-4659-b493-b80d3800d920", - "parent_span_id": "ea4d61c5-b8dd-4677-b21b-2b17170e2163", - "name": "agent.rag_evaluation_metrics", - "start_time": "2025-10-09 08:26:40.446000000", - "end_time": "2025-10-09 08:26:40.446000000", - "duration_ms": 0.08399999999999999, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "rag_evaluation_metrics", - "agent.capabilities": [ - "retrieval_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What are the newest features in cloud computing?", - "query_length": 48, - "query_type": "rag_evaluation_query", - "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/changelog):\n- šŸ“ˆ Enhanced Dashboard Analytics: Real-time request tracking, latency monitoring, and cost analysis\n- šŸ” Improved Logs Interface: Better search functionality, detailed trace views, and enhanced debugging capabilities\n- šŸ‘„ Team Management: Advanced member mana...", - "retrieval.context_length": 2579, - "retrieval.sources_count": 5, - "retrieval.context_quality": "high", - "retrieval.effectiveness": 1, - "retrieval.context_utilization": 2.579, - "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/changelog):\n- šŸ“ˆ Enhanced Dashboard Analytics: Real-time request tracking, latency monitoring, and cost analysis\n- šŸ” Improved Logs Interface: Better search functionality, detailed trace views, and enhanced debugging capabilities\n- šŸ‘„ Team Management: Advanced member management with role-based access control and invitation system\n- šŸ›”ļø Security Improvements: Enhanced API key management and secure credential storage\n4 months ago\n- šŸ“Š Advanced Metrics Collection: Comprehensive telemetry system with custom metrics ...\n\nSource 2 (https://noveum.ai/en/changelog):\n- šŸ“ Documentation Updates: Comprehensive guides for integration and best practices\n5 months ago\n- šŸŽ‰ Platform Foundation: Initial release of Noveum.ai AI observability platform\n- šŸš€ Core Features: AI Gateway integration, basic evaluation metrics, and monitoring capabilities\n- šŸ“ˆ Evaluation Jobs: Automated model evaluation with configurable metrics and reporting\n- šŸ”Œ Provider Integrations: Support for major AI providers and custom model deployments...\n\nSource 3 (https://noveum.ai/docs/platform/dashboard):\n- Bulk Export: Download multiple traces simultaneously\n- Comparative Analysis: Compare performance across multiple traces\nReal-Time Updates\n- Live Refresh: Automatic updates as new traces arrive\n- Connection Monitoring: Real-time status of your trace ingestion pipeline\n- Performance Indicators: Live metrics for system health monitoring\nReady to dive deeper? Explore Projects & Environments to organize your AI applications, or check out Team Collaboration to share insights with your team.\nGet Earl...\n\nSource 4 (https://noveum.ai/en/changelog):\nChangelog\nStay up to date with the latest changes in our product.\n2 months ago\n- šŸŽÆ Major Platform Evolution: NovaEval Framework - Launched comprehensive AI model evaluation framework with 20+ built-in scorers including accuracy, RAG metrics, conversational scoring, and specialized agent evaluation tools. Features production-grade deployment on Docker and Kubernetes. (GitHub | PyPI Package)\n- šŸš€ Next-Generation Traces UI - Complete redesign featuring three-pane layout with directory tree navigatio...\n\nSource 5 (https://noveum.ai/en/blog):\n8/12/2025\nDiscover how Noveum.ai provides comprehensive tracing and observability for AI applications, from development debugging to production optimization.\nShashank Agarwal\n3/3/2025\nDiscover how Noveum.ai provides comprehensive tracing and observability for LLM applications, RAG systems, and multi-agent workflows with our powerful Python and TypeScript SDKs.\nShashank Agarwal\n3/2/2025...\n\nUser Question: What are the newest features in cloud computing?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", - "prompt.prompt_length": 3123, - "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/changelog):\n- šŸ“ˆ Enhanced Dashboard Analytics: Real-time request tracking, latency monitoring, and cost analysis\n- šŸ” Improved Logs Interface: Better search functionality,...", - "prompt.instruction_following": "rag_optimized", - "response.answer_length": 883, - "response.answer_completeness": "complete", - "response.response_quality": "high", - "response.source_citation": 5, - "response.context_utilization": 0.3423807677394339, - "output_response": "RAG Answer: The provided context does not specifically mention any new features in cloud computing as a general topic. However, it does highlight several recent enhancements and features related to the Noveum.ai ...", - "evaluation.retrieval_effectiveness": 1, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 5, - "evaluation.context_relevance": "high", - "evaluation.overall_quality": "high", - "evaluation.ready_for_production": true, - "rag.retrieval_strategy": "semantic_similarity", - "rag.vector_search_results": 5, - "rag.context_synthesis": "multi_source", - "rag.document_coverage": 1, - "rag.information_density": 0.3423807677394339 - }, - "events": [], - "links": [], - "trace_trace_id": "f1f37bd7-0851-4659-b493-b80d3800d920", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-09 08:26:35.915000000", - "trace_end_time": "2025-10-09 08:26:40.841000000", - "trace_duration_ms": 4926.342, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-09 08:26:44.053000000", - "updated_at": "2025-10-09 08:26:44.053000000" - }, - { - "span_id": "e193cd4f-cca3-4b94-95d3-a1f2446b9fc6", - "trace_id": "52aacb67-c361-4445-9b72-c157f79f47d6", - "parent_span_id": "7505f97a-cbc0-42fe-8a72-95387d723daa", - "name": "agent.rag_evaluation_metrics", - "start_time": "2025-10-09 08:26:21.807000000", - "end_time": "2025-10-09 08:26:21.807000000", - "duration_ms": 0.063, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "rag_evaluation_metrics", - "agent.capabilities": [ - "retrieval_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What are the current trends in observability tools?", - "query_length": 51, - "query_type": "rag_evaluation_query", - "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/blog/from-logs-to-intelligent-choices-inside-noveum-ais-evaluation-process):\nFrom Development to Production - Inside Noveum.ai's AI Observability Platform\nShashank Agarwal\n3/3/2025\nIntroduction\nEvery AI application tells a storyā€”but most developers never get to hear it...", - "retrieval.context_length": 2943, - "retrieval.sources_count": 5, - "retrieval.context_quality": "high", - "retrieval.effectiveness": 1, - "retrieval.context_utilization": 2.943, - "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/blog/from-logs-to-intelligent-choices-inside-noveum-ais-evaluation-process):\nFrom Development to Production - Inside Noveum.ai's AI Observability Platform\nShashank Agarwal\n3/3/2025\nIntroduction\nEvery AI application tells a storyā€”but most developers never get to hear it. When your RAG pipeline returns irrelevant results, when your multi-agent system gets stuck in loops, or when your LLM costs suddenly spike, you're left guessing what went wrong and where.\nThat's exactly the challenge Noveum.ai solves. Rather than flying blind, our platform provides comprehensive tracing a...\n\nSource 2 (https://noveum.ai/docs/getting-started/tracing-concepts):\n- Explore Framework Integrations - Framework-specific guidance\n- Learn Advanced Patterns - Custom instrumentation techniques\n- Master the Dashboard - Analyze your traces effectively\nRemember: Good observability is not about collecting all possible data, but about collecting the right data that helps you understand, debug, and optimize your AI applications.\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to O...\n\nSource 3 (https://noveum.ai/en/docs/getting-started/tracing-concepts):\n- Explore Framework Integrations - Framework-specific guidance\n- Learn Advanced Patterns - Custom instrumentation techniques\n- Master the Dashboard - Analyze your traces effectively\nRemember: Good observability is not about collecting all possible data, but about collecting the right data that helps you understand, debug, and optimize your AI applications.\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to O...\n\nSource 4 (https://noveum.ai/en/blog/from-logs-to-intelligent-choices-inside-noveum-ais-evaluation-process):\nThe AI Observability Challenge\nWhy Traditional Monitoring Falls Short\nTraditional application monitoring tools weren't built for AI applications. They can tell you if your API is responding, but they can't answer the questions that matter most for AI systems:\n- Why did my RAG pipeline retrieve irrelevant documents?\n- Which LLM calls are driving my costs?\n- How are my agents communicating with each other?\n- What's causing hallucinations in my responses?\n- Why is my embedding generation so slow?\nW...\n\nSource 5 (https://noveum.ai/en/docs/advanced/rag-observability):\nšŸ”§ Best Practices for RAG Observability\n1. Comprehensive Pipeline Tracing\n2. Quality Monitoring\n3. A/B Testing for RAG Components\nšŸŽÆ Advanced RAG Patterns\nMulti-Modal RAG\nConversational RAG\nRAG observability with Noveum.ai provides the deep insights needed to build, optimize, and scale retrieval-augmented generation systems. By implementing comprehensive tracing across retrieval, context preparation, and generation stages, you can ensure your RAG pipeline delivers accurate, relevant, and cost-effe...\n\nUser Question: What are the current trends in observability tools?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", - "prompt.prompt_length": 3490, - "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/blog/from-logs-to-intelligent-choices-inside-noveum-ais-evaluation-process):\nFrom Development to Production - Inside Noveum.ai's AI Observability Platform\nShashank Agarw...", - "prompt.instruction_following": "rag_optimized", - "response.answer_length": 1074, - "response.answer_completeness": "complete", - "response.response_quality": "high", - "response.source_citation": 5, - "response.context_utilization": 0.36493374108053006, - "output_response": "RAG Answer: The provided context does not explicitly outline current trends in observability tools. However, it highlights some key aspects of observability that are particularly relevant to AI applications. \n\n1....", - "evaluation.retrieval_effectiveness": 1, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 5, - "evaluation.context_relevance": "high", - "evaluation.overall_quality": "high", - "evaluation.ready_for_production": true, - "rag.retrieval_strategy": "semantic_similarity", - "rag.vector_search_results": 5, - "rag.context_synthesis": "multi_source", - "rag.document_coverage": 1, - "rag.information_density": 0.36493374108053006 - }, - "events": [], - "links": [], - "trace_trace_id": "52aacb67-c361-4445-9b72-c157f79f47d6", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-09 08:26:16.544000000", - "trace_end_time": "2025-10-09 08:26:22.201000000", - "trace_duration_ms": 5657.012, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-09 08:26:22.904000000", - "updated_at": "2025-10-09 08:26:22.904000000" - }, - { - "span_id": "432e74f3-3291-411b-8d7e-85493868f8cf", - "trace_id": "2218f641-604c-491a-9710-b51a9941b982", - "parent_span_id": "8a6e18a1-babc-4166-926f-188f344d514e", - "name": "agent.rag_evaluation_metrics", - "start_time": "2025-10-09 08:25:55.406000000", - "end_time": "2025-10-09 08:25:55.406000000", - "duration_ms": 0.056, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "rag_evaluation_metrics", - "agent.capabilities": [ - "retrieval_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What are the benefits of using Noveum Trace?", - "query_length": 44, - "query_type": "rag_evaluation_query", - "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/docs/platform/dashboard):\n- Bulk Export: Download multiple traces simultaneously\n- Comparative Analysis: Compare performance across multiple traces\nReal-Time Updates\n- Live Refresh: Automatic updates as new traces arrive\n- Connection Monitoring: Real-time status of your t...", - "retrieval.context_length": 2773, - "retrieval.sources_count": 5, - "retrieval.context_quality": "high", - "retrieval.effectiveness": 1, - "retrieval.context_utilization": 2.773, - "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/docs/platform/dashboard):\n- Bulk Export: Download multiple traces simultaneously\n- Comparative Analysis: Compare performance across multiple traces\nReal-Time Updates\n- Live Refresh: Automatic updates as new traces arrive\n- Connection Monitoring: Real-time status of your trace ingestion pipeline\n- Performance Indicators: Live metrics for system health monitoring\nReady to dive deeper? Explore Projects & Environments to organize your AI applications, or check out Team Collaboration to share insights with your team.\nGet Earl...\n\nSource 2 (https://noveum.ai/en/docs):\nCore Components\n-\nšŸ Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\nšŸ“˜ TypeScript SDK (\n@noveum/trace\n)- Framework integrations for Next.js, Express.js, and Hono\n- TypeScript-first design with full type safety\n- Universal compatibility (Node.js, Edge Runtime, browsers)\n-\nšŸ“Š Noveum Platform\n- Real-time dashboard for analyzing traces and performance\n- Ad...\n\nSource 3 (https://noveum.ai/docs):\nCore Components\n-\nšŸ Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\nšŸ“˜ TypeScript SDK (\n@noveum/trace\n)- Framework integrations for Next.js, Express.js, and Hono\n- TypeScript-first design with full type safety\n- Universal compatibility (Node.js, Edge Runtime, browsers)\n-\nšŸ“Š Noveum Platform\n- Real-time dashboard for analyzing traces and performance\n- Ad...\n\nSource 4 (https://noveum.ai/en/docs/getting-started/overview):\n- Advanced search & filtering across traces and spans\n- Cost analysis and optimization recommendations\n- Team collaboration with shared insights and alerts\nšŸ” What Noveum Traces\nLLM Operations\n- Model calls across all providers (OpenAI, Anthropic, Google, etc.)\n- Token usage and cost calculation\n- Prompt engineering effectiveness\n- Response quality metrics\nRAG Pipelines\n- Document retrieval performance and relevance\n- Embedding generation costs and latency\n- Context assembly and prompt constructi...\n\nSource 5 (https://noveum.ai/docs/getting-started/overview):\n- Advanced search & filtering across traces and spans\n- Cost analysis and optimization recommendations\n- Team collaboration with shared insights and alerts\nšŸ” What Noveum Traces\nLLM Operations\n- Model calls across all providers (OpenAI, Anthropic, Google, etc.)\n- Token usage and cost calculation\n- Prompt engineering effectiveness\n- Response quality metrics\nRAG Pipelines\n- Document retrieval performance and relevance\n- Embedding generation costs and latency\n- Context assembly and prompt constructi...\n\nUser Question: What are the benefits of using Noveum Trace?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", - "prompt.prompt_length": 3313, - "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/docs/platform/dashboard):\n- Bulk Export: Download multiple traces simultaneously\n- Comparative Analysis: Compare performance across multiple traces\nReal-Time Updates\n- Live...", - "prompt.instruction_following": "rag_optimized", - "response.answer_length": 1460, - "response.answer_completeness": "complete", - "response.response_quality": "high", - "response.source_citation": 5, - "response.context_utilization": 0.5265055896141363, - "output_response": "RAG Answer: Using Noveum Trace offers several benefits, including:\n\n1. **Real-Time Monitoring**: The Noveum platform provides a real-time dashboard for analyzing traces and performance, with live updates as new t...", - "evaluation.retrieval_effectiveness": 1, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 5, - "evaluation.context_relevance": "high", - "evaluation.overall_quality": "high", - "evaluation.ready_for_production": true, - "rag.retrieval_strategy": "semantic_similarity", - "rag.vector_search_results": 5, - "rag.context_synthesis": "multi_source", - "rag.document_coverage": 1, - "rag.information_density": 0.5265055896141363 - }, - "events": [], - "links": [], - "trace_trace_id": "2218f641-604c-491a-9710-b51a9941b982", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-09 08:25:48.107000000", - "trace_end_time": "2025-10-09 08:25:55.878000000", - "trace_duration_ms": 7770.953, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-09 08:25:56.565000000", - "updated_at": "2025-10-09 08:25:56.565000000" - }, - { - "span_id": "a0475dd5-ef9f-4eb2-bca4-9be72bf6bfd6", - "trace_id": "255fd49c-84b4-4b18-887e-6308a412d535", - "parent_span_id": "5bec4165-6596-43ac-930b-acfcde1cc02a", - "name": "agent.rag_evaluation_metrics", - "start_time": "2025-10-09 08:25:46.618000000", - "end_time": "2025-10-09 08:25:46.618000000", - "duration_ms": 0.013, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "rag_evaluation_metrics", - "agent.capabilities": [ - "retrieval_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: How do I configure Noveum for my system?", - "query_length": 40, - "query_type": "rag_evaluation_query", - "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/docs):\nWelcome to Noveum.ai Docs\nComprehensive AI tracing and observability for LLM applications, RAG systems, and AI agents\nWelcome to the Noveum.ai documentation! Here you'll find everything you need to integrate, configure, and optimize your AI applications using ou...", - "retrieval.context_length": 2237, - "retrieval.sources_count": 5, - "retrieval.context_quality": "high", - "retrieval.effectiveness": 1, - "retrieval.context_utilization": 2.237, - "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/docs):\nWelcome to Noveum.ai Docs\nComprehensive AI tracing and observability for LLM applications, RAG systems, and AI agents\nWelcome to the Noveum.ai documentation! Here you'll find everything you need to integrate, configure, and optimize your AI applications using our comprehensive tracing and observability platform.\nWe provide powerful SDKs for Python and TypeScript that enable you to trace LLM calls, RAG pipelines, and multi-agent workflows with minimal code changes. Our documentation is organized ...\n\nSource 2 (https://noveum.ai/docs):\nWelcome to Noveum.ai Docs\nComprehensive AI tracing and observability for LLM applications, RAG systems, and AI agents\nWelcome to the Noveum.ai documentation! Here you'll find everything you need to integrate, configure, and optimize your AI applications using our comprehensive tracing and observability platform.\nWe provide powerful SDKs for Python and TypeScript that enable you to trace LLM calls, RAG pipelines, and multi-agent workflows with minimal code changes. Our documentation is organized ...\n\nSource 3 (https://noveum.ai/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nSource 4 (https://noveum.ai/en/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nSource 5 (https://noveum.ai/en/docs):\n- šŸŒ Universal: Works across frameworks, clouds, and deployment models\nšŸ¤ Community & Support\n- šŸ’¬ Discord Community: Join our Discord\n- šŸ“§ Email Support: [email protected]\n- šŸ› Bug Reports: GitHub Issues\n- šŸ“– Knowledge Base: Help Center\nReady to get started? Head to our SDK Integration Guide to begin tracing your AI applications in under 5 minutes!\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability ...\n\nUser Question: How do I configure Noveum for my system?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", - "prompt.prompt_length": 2773, - "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/docs):\nWelcome to Noveum.ai Docs\nComprehensive AI tracing and observability for LLM applications, RAG systems, and AI agents\nWelcome to the Noveum.ai documentation! Here...", - "prompt.instruction_following": "rag_optimized", - "response.answer_length": 613, - "response.answer_completeness": "complete", - "response.response_quality": "high", - "response.source_citation": 5, - "response.context_utilization": 0.27402771569065715, - "output_response": "RAG Answer: To configure Noveum for your system, you can refer to the SDK Integration Guide available in the Noveum documentation. This guide will help you trace your AI applications with minimal code changes. Th...", - "evaluation.retrieval_effectiveness": 1, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 5, - "evaluation.context_relevance": "high", - "evaluation.overall_quality": "high", - "evaluation.ready_for_production": true, - "rag.retrieval_strategy": "semantic_similarity", - "rag.vector_search_results": 5, - "rag.context_synthesis": "multi_source", - "rag.document_coverage": 1, - "rag.information_density": 0.27402771569065715 - }, - "events": [], - "links": [], - "trace_trace_id": "255fd49c-84b4-4b18-887e-6308a412d535", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-09 08:25:43.830000000", - "trace_end_time": "2025-10-09 08:25:47.099000000", - "trace_duration_ms": 3268.862, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-09 08:25:50.304000000", - "updated_at": "2025-10-09 08:25:50.304000000" - }, - { - "span_id": "ed8ad539-f008-4f1a-beb1-12ad5793950e", - "trace_id": "dc511122-c0b6-415c-9a49-c7b45132dd87", - "parent_span_id": "f9f79a07-4eda-41b8-9634-d4460114caad", - "name": "agent.rag_evaluation_metrics", - "start_time": "2025-10-09 08:25:42.276000000", - "end_time": "2025-10-09 08:25:42.276000000", - "duration_ms": 0.041, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "rag_evaluation_metrics", - "agent.capabilities": [ - "retrieval_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What monitoring capabilities does Noveum provide?", - "query_length": 49, - "query_type": "rag_evaluation_query", - "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en):\nMonitor all your AI Agents\nimprove AI Agents today\nNoveum.ai helps you monitor, trace, and optimize your AI applications.\nNoveum.ai works with any AI framework ā€“ LangChain, CrewAI, AutoGen, custom implementations, or direct LLM calls. One dashboard shows everything.\n...", - "retrieval.context_length": 2740, - "retrieval.sources_count": 5, - "retrieval.context_quality": "high", - "retrieval.effectiveness": 1, - "retrieval.context_utilization": 2.74, - "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en):\nMonitor all your AI Agents\nimprove AI Agents today\nNoveum.ai helps you monitor, trace, and optimize your AI applications.\nNoveum.ai works with any AI framework ā€“ LangChain, CrewAI, AutoGen, custom implementations, or direct LLM calls. One dashboard shows everything.\nMonitor, Evaluate, Improve Your AI Agents\nThe control plane for AI agents.\nMonitor Everything, Miss Nothing\nOur lightweight SDKs capture every trace and span across your AI agent ecosystemā€”from simple LLM calls to complex multi-agent...\n\nSource 2 (https://noveum.ai/):\nMonitor all your AI Agents\nimprove AI Agents today\nNoveum.ai helps you monitor, trace, and optimize your AI applications.\nNoveum.ai works with any AI framework ā€“ LangChain, CrewAI, AutoGen, custom implementations, or direct LLM calls. One dashboard shows everything.\nMonitor, Evaluate, Improve Your AI Agents\nThe control plane for AI agents.\nMonitor Everything, Miss Nothing\nOur lightweight SDKs capture every trace and span across your AI agent ecosystemā€”from simple LLM calls to complex multi-agent...\n\nSource 3 (https://noveum.ai):\nMonitor all your AI Agents\nimprove AI Agents today\nNoveum.ai helps you monitor, trace, and optimize your AI applications.\nNoveum.ai works with any AI framework ā€“ LangChain, CrewAI, AutoGen, custom implementations, or direct LLM calls. One dashboard shows everything.\nMonitor, Evaluate, Improve Your AI Agents\nThe control plane for AI agents.\nMonitor Everything, Miss Nothing\nOur lightweight SDKs capture every trace and span across your AI agent ecosystemā€”from simple LLM calls to complex multi-agent...\n\nSource 4 (https://noveum.ai/en/docs/getting-started/overview):\nNoveum.ai Overview\nComprehensive AI tracing and observability platform for LLM applications, RAG systems, and AI agents\nWelcome to Noveum.aiā€”the comprehensive tracing and observability platform built specifically for AI applications. Whether you're building LLM-powered chatbots, RAG systems, multi-agent workflows, or any AI-driven application, Noveum provides the insights you need to understand, debug, and optimize your systems.\nšŸŽÆ Why AI Applications Need Specialized Observability\nTraditional mo...\n\nSource 5 (https://noveum.ai/docs/getting-started/overview):\nNoveum.ai Overview\nComprehensive AI tracing and observability platform for LLM applications, RAG systems, and AI agents\nWelcome to Noveum.aiā€”the comprehensive tracing and observability platform built specifically for AI applications. Whether you're building LLM-powered chatbots, RAG systems, multi-agent workflows, or any AI-driven application, Noveum provides the insights you need to understand, debug, and optimize your systems.\nšŸŽÆ Why AI Applications Need Specialized Observability\nTraditional mo...\n\nUser Question: What monitoring capabilities does Noveum provide?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", - "prompt.prompt_length": 3285, - "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en):\nMonitor all your AI Agents\nimprove AI Agents today\nNoveum.ai helps you monitor, trace, and optimize your AI applications.\nNoveum.ai works with any AI framework ā€“ LangC...", - "prompt.instruction_following": "rag_optimized", - "response.answer_length": 839, - "response.answer_completeness": "complete", - "response.response_quality": "high", - "response.source_citation": 5, - "response.context_utilization": 0.30620437956204377, - "output_response": "RAG Answer: Noveum provides comprehensive monitoring capabilities for AI applications, including the ability to monitor, trace, and optimize AI agents across various frameworks such as LangChain, CrewAI, AutoGen,...", - "evaluation.retrieval_effectiveness": 1, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 5, - "evaluation.context_relevance": "high", - "evaluation.overall_quality": "high", - "evaluation.ready_for_production": true, - "rag.retrieval_strategy": "semantic_similarity", - "rag.vector_search_results": 5, - "rag.context_synthesis": "multi_source", - "rag.document_coverage": 1, - "rag.information_density": 0.30620437956204377 - }, - "events": [], - "links": [], - "trace_trace_id": "dc511122-c0b6-415c-9a49-c7b45132dd87", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-09 08:25:38.322000000", - "trace_end_time": "2025-10-09 08:25:42.822000000", - "trace_duration_ms": 4500.425, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-09 08:25:45.028000000", - "updated_at": "2025-10-09 08:25:45.028000000" - }, - { - "span_id": "c7358391-9ac1-4312-a4b3-79edfa8bc0d3", - "trace_id": "04bebf38-a343-4563-80db-0154bef8d927", - "parent_span_id": "edc52a5d-8450-4d0f-ad14-958ec4d147f4", - "name": "agent.rag_evaluation_metrics", - "start_time": "2025-10-09 08:25:36.952000000", - "end_time": "2025-10-09 08:25:36.952000000", - "duration_ms": 0.18100000000000002, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "rag_evaluation_metrics", - "agent.capabilities": [ - "retrieval_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: How does Noveum handle agent tracing?", - "query_length": 37, - "query_type": "rag_evaluation_query", - "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, vector searches, data retrieval, agent reasoning, and more. Without proper observability, debugging ...", - "retrieval.context_length": 2736, - "retrieval.sources_count": 5, - "retrieval.context_quality": "high", - "retrieval.effectiveness": 1, - "retrieval.context_utilization": 2.736, - "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, vector searches, data retrieval, agent reasoning, and more. Without proper observability, debugging becomes a nightmare, optimization is guesswork, and understanding user interactions is nearly impossible.\nNoveum.ai solves this by providing comprehensive tracing and observability specifically designed for AI applications.\nKey benefits of using Noveum.ai:\n- Complete Visibility: Trace...\n\nSource 2 (https://noveum.ai/docs/advanced/multi-agent-tracing):\nMulti-Agent Tracing\nObserve complex agent workflows and inter-agent communications with comprehensive tracing\nMulti-agent systems represent some of the most complex AI applications, involving multiple agents that coordinate, communicate, and collaborate to achieve shared goals. Noveum.ai provides specialized tracing capabilities to help you understand and optimize these intricate workflows.\nšŸŽÆ Why Multi-Agent Tracing Matters\nMulti-agent systems introduce unique observability challenges:\n- Complex...\n\nSource 3 (https://noveum.ai/en/docs/advanced/multi-agent-tracing):\nMulti-Agent Tracing\nObserve complex agent workflows and inter-agent communications with comprehensive tracing\nMulti-agent systems represent some of the most complex AI applications, involving multiple agents that coordinate, communicate, and collaborate to achieve shared goals. Noveum.ai provides specialized tracing capabilities to help you understand and optimize these intricate workflows.\nšŸŽÆ Why Multi-Agent Tracing Matters\nMulti-agent systems introduce unique observability challenges:\n- Complex...\n\nSource 4 (https://noveum.ai/en/docs/advanced/performance-optimization):\nšŸ”— Next Steps\n- Multi-Agent Tracing - Observe agent workflows\n- RAG Pipeline Observability - Monitor retrieval and generation systems\n- Custom Instrumentation - Add domain-specific tracing\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nSource 5 (https://noveum.ai/docs/advanced/performance-optimization):\nšŸ”— Next Steps\n- Multi-Agent Tracing - Observe agent workflows\n- RAG Pipeline Observability - Monitor retrieval and generation systems\n- Custom Instrumentation - Add domain-specific tracing\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nUser Question: How does Noveum handle agent tracing?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", - "prompt.prompt_length": 3269, - "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, ...", - "prompt.instruction_following": "rag_optimized", - "response.answer_length": 691, - "response.answer_completeness": "complete", - "response.response_quality": "high", - "response.source_citation": 5, - "response.context_utilization": 0.25255847953216376, - "output_response": "RAG Answer: Noveum.ai handles agent tracing through its specialized Multi-Agent Tracing capabilities, which are designed to observe complex workflows and inter-agent communications within multi-agent systems. The...", - "evaluation.retrieval_effectiveness": 1, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 5, - "evaluation.context_relevance": "high", - "evaluation.overall_quality": "high", - "evaluation.ready_for_production": true, - "rag.retrieval_strategy": "semantic_similarity", - "rag.vector_search_results": 5, - "rag.context_synthesis": "multi_source", - "rag.document_coverage": 1, - "rag.information_density": 0.25255847953216376 - }, - "events": [], - "links": [], - "trace_trace_id": "04bebf38-a343-4563-80db-0154bef8d927", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-09 08:25:33.693000000", - "trace_end_time": "2025-10-09 08:25:37.313000000", - "trace_duration_ms": 3619.672, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-09 08:25:40.339000000", - "updated_at": "2025-10-09 08:25:40.339000000" - }, - { - "span_id": "6b21de5d-456f-4501-953f-0ce5bc4fa5a5", - "trace_id": "5e043630-6493-42b5-beb8-79faa19bfa37", - "parent_span_id": "6c362c73-e063-4813-8180-d26b6a52b798", - "name": "agent.rag_evaluation_metrics", - "start_time": "2025-10-09 08:25:32.086000000", - "end_time": "2025-10-09 08:25:32.086000000", - "duration_ms": 0.04, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "rag_evaluation_metrics", - "agent.capabilities": [ - "retrieval_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What APIs are available in Noveum platform?", - "query_length": 43, - "query_type": "rag_evaluation_query", - "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/docs):\n- šŸŒ Universal: Works across frameworks, clouds, and deployment models\nšŸ¤ Community & Support\n- šŸ’¬ Discord Community: Join our Discord\n- šŸ“§ Email Support: [email protected]\n- šŸ› Bug Reports: GitHub Issues\n- šŸ“– Knowledge Base: Help Center\nReady to get started? Head to ...", - "retrieval.context_length": 2284, - "retrieval.sources_count": 5, - "retrieval.context_quality": "high", - "retrieval.effectiveness": 1, - "retrieval.context_utilization": 2.284, - "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/docs):\n- šŸŒ Universal: Works across frameworks, clouds, and deployment models\nšŸ¤ Community & Support\n- šŸ’¬ Discord Community: Join our Discord\n- šŸ“§ Email Support: [email protected]\n- šŸ› Bug Reports: GitHub Issues\n- šŸ“– Knowledge Base: Help Center\nReady to get started? Head to our SDK Integration Guide to begin tracing your AI applications in under 5 minutes!\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability ...\n\nSource 2 (https://noveum.ai/docs):\n- šŸŒ Universal: Works across frameworks, clouds, and deployment models\nšŸ¤ Community & Support\n- šŸ’¬ Discord Community: Join our Discord\n- šŸ“§ Email Support: [email protected]\n- šŸ› Bug Reports: GitHub Issues\n- šŸ“– Knowledge Base: Help Center\nReady to get started? Head to our SDK Integration Guide to begin tracing your AI applications in under 5 minutes!\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability ...\n\nSource 3 (https://noveum.ai/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nSource 4 (https://noveum.ai/en/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nSource 5 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, vector searches, data retrieval, agent reasoning, and more. Without proper observability, debugging becomes a nightmare, optimization is guesswork, and understanding user interactions is nearly impossible.\nNoveum.ai solves this by providing comprehensive tracing and observability specifically designed for AI applications.\nKey benefits of using Noveum.ai:\n- Complete Visibility: Trace...\n\nUser Question: What APIs are available in Noveum platform?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", - "prompt.prompt_length": 2823, - "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/docs):\n- šŸŒ Universal: Works across frameworks, clouds, and deployment models\nšŸ¤ Community & Support\n- šŸ’¬ Discord Community: Join our Discord\n- šŸ“§ Email Support: [email prot...", - "prompt.instruction_following": "rag_optimized", - "response.answer_length": 270, - "response.answer_completeness": "complete", - "response.response_quality": "high", - "response.source_citation": 5, - "response.context_utilization": 0.11821366024518389, - "output_response": "RAG Answer: The provided context does not specify the APIs available in the Noveum platform. For detailed information about the APIs, I recommend checking the official Noveum documentation or the SDK Integration ...", - "evaluation.retrieval_effectiveness": 1, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 5, - "evaluation.context_relevance": "high", - "evaluation.overall_quality": "high", - "evaluation.ready_for_production": true, - "rag.retrieval_strategy": "semantic_similarity", - "rag.vector_search_results": 5, - "rag.context_synthesis": "multi_source", - "rag.document_coverage": 1, - "rag.information_density": 0.11821366024518389 - }, - "events": [], - "links": [], - "trace_trace_id": "5e043630-6493-42b5-beb8-79faa19bfa37", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-09 08:25:29.901000000", - "trace_end_time": "2025-10-09 08:25:32.685000000", - "trace_duration_ms": 2783.326, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-09 08:25:34.918000000", - "updated_at": "2025-10-09 08:25:34.918000000" - }, - { - "span_id": "2ad4e155-7b76-4040-923d-d6b65a97a45b", - "trace_id": "7da9814d-a2e8-4c4e-b750-68b26bd5fd22", - "parent_span_id": "1393d065-a9a4-4475-808f-2d00b3c68939", - "name": "agent.rag_evaluation_metrics", - "start_time": "2025-10-09 08:25:28.496000000", - "end_time": "2025-10-09 08:25:28.496000000", - "duration_ms": 0.039, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "rag_evaluation_metrics", - "agent.capabilities": [ - "retrieval_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: How do I set up observability with Noveum?", - "query_length": 42, - "query_type": "rag_evaluation_query", - "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\n- Production Ready: Built for scale with intelligent sampling and batching\n- Privacy Focused: You control what data is captured and how it's stored\n- Open Source SDKs: Transparent, auditable, and extensible\nGetting ...", - "retrieval.context_length": 2870, - "retrieval.sources_count": 6, - "retrieval.context_quality": "high", - "retrieval.effectiveness": 1.2, - "retrieval.context_utilization": 2.87, - "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\n- Production Ready: Built for scale with intelligent sampling and batching\n- Privacy Focused: You control what data is captured and how it's stored\n- Open Source SDKs: Transparent, auditable, and extensible\nGetting Started\nReady to add comprehensive observability to your AI applications?\n- Sign up at noveum.ai\n- Create a project and get your API key\n- Install the SDK for your preferred language\n- Add tracing to your AI workflows\n- Explore insights in the Noveum.ai dashboard\nCheck out our integra...\n\nSource 2 (https://noveum.ai/en/docs/getting-started/tracing-concepts):\n- Explore Framework Integrations - Framework-specific guidance\n- Learn Advanced Patterns - Custom instrumentation techniques\n- Master the Dashboard - Analyze your traces effectively\nRemember: Good observability is not about collecting all possible data, but about collecting the right data that helps you understand, debug, and optimize your AI applications.\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to O...\n\nSource 3 (https://noveum.ai/docs/getting-started/tracing-concepts):\n- Explore Framework Integrations - Framework-specific guidance\n- Learn Advanced Patterns - Custom instrumentation techniques\n- Master the Dashboard - Analyze your traces effectively\nRemember: Good observability is not about collecting all possible data, but about collecting the right data that helps you understand, debug, and optimize your AI applications.\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to O...\n\nSource 4 (https://noveum.ai/en/docs/getting-started/overview):\nNoveum.ai Overview\nComprehensive AI tracing and observability platform for LLM applications, RAG systems, and AI agents\nWelcome to Noveum.aiā€”the comprehensive tracing and observability platform built specifically for AI applications. Whether you're building LLM-powered chatbots, RAG systems, multi-agent workflows, or any AI-driven application, Noveum provides the insights you need to understand, debug, and optimize your systems.\nšŸŽÆ Why AI Applications Need Specialized Observability\nTraditional mo...\n\nSource 5 (https://noveum.ai/docs/getting-started/overview):\nNoveum.ai Overview\nComprehensive AI tracing and observability platform for LLM applications, RAG systems, and AI agents\nWelcome to Noveum.aiā€”the comprehensive tracing and observability platform built specifically for AI applications. Whether you're building LLM-powered chatbots, RAG systems, multi-agent workflows, or any AI-driven application, Noveum provides the insights you need to understand, debug, and optimize your systems.\nšŸŽÆ Why AI Applications Need Specialized Observability\nTraditional mo...\n\nUser Question: How do I set up observability with Noveum?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", - "prompt.prompt_length": 3408, - "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\n- Production Ready: Built for scale with intelligent sampling and batching\n- Privacy Focused: You control what data...", - "prompt.instruction_following": "rag_optimized", - "response.answer_length": 951, - "response.answer_completeness": "complete", - "response.response_quality": "high", - "response.source_citation": 6, - "response.context_utilization": 0.3313588850174216, - "output_response": "RAG Answer: To set up observability with Noveum, follow these steps:\n\n1. **Sign Up**: Go to [noveum.ai](https://noveum.ai) and create an account.\n2. **Create a Project**: Once signed in, create a new project to o...", - "evaluation.retrieval_effectiveness": 1.2, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 6, - "evaluation.context_relevance": "high", - "evaluation.overall_quality": "high", - "evaluation.ready_for_production": true, - "rag.retrieval_strategy": "semantic_similarity", - "rag.vector_search_results": 6, - "rag.context_synthesis": "multi_source", - "rag.document_coverage": 1.2, - "rag.information_density": 0.3313588850174216 - }, - "events": [], - "links": [], - "trace_trace_id": "7da9814d-a2e8-4c4e-b750-68b26bd5fd22", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-09 08:25:20.586000000", - "trace_end_time": "2025-10-09 08:25:28.897000000", - "trace_duration_ms": 8310.205, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-09 08:25:29.588000000", - "updated_at": "2025-10-09 08:25:29.588000000" - }, - { - "span_id": "6b822089-be40-457d-9fa6-0979bafdfb85", - "trace_id": "16143f74-2831-4753-b33d-ce4b645093c5", - "parent_span_id": "f419377a-fc24-4abf-8a1c-03857824e374", - "name": "agent.rag_evaluation_metrics", - "start_time": "2025-10-09 08:25:19.220000000", - "end_time": "2025-10-09 08:25:19.220000000", - "duration_ms": 0.053, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "rag_evaluation_metrics", - "agent.capabilities": [ - "retrieval_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What features does Noveum Trace offer?", - "query_length": 38, - "query_type": "rag_evaluation_query", - "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/docs):\nCore Components\n-\nšŸ Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\nšŸ“˜ TypeScript SDK (\n@noveum/trace\n)- Framework ...", - "retrieval.context_length": 2787, - "retrieval.sources_count": 5, - "retrieval.context_quality": "high", - "retrieval.effectiveness": 1, - "retrieval.context_utilization": 2.787, - "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/docs):\nCore Components\n-\nšŸ Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\nšŸ“˜ TypeScript SDK (\n@noveum/trace\n)- Framework integrations for Next.js, Express.js, and Hono\n- TypeScript-first design with full type safety\n- Universal compatibility (Node.js, Edge Runtime, browsers)\n-\nšŸ“Š Noveum Platform\n- Real-time dashboard for analyzing traces and performance\n- Ad...\n\nSource 2 (https://noveum.ai/docs):\nCore Components\n-\nšŸ Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\nšŸ“˜ TypeScript SDK (\n@noveum/trace\n)- Framework integrations for Next.js, Express.js, and Hono\n- TypeScript-first design with full type safety\n- Universal compatibility (Node.js, Edge Runtime, browsers)\n-\nšŸ“Š Noveum Platform\n- Real-time dashboard for analyzing traces and performance\n- Ad...\n\nSource 3 (https://noveum.ai/en/docs/getting-started/sdk-integration):\n- šŸ‘„ Collaborate with your team on insights\nšŸ”’ Security & Privacy\n- šŸ” Encryption: All data encrypted in transit and at rest\n- šŸŽ›ļø Configurable Capture: Control what data is collected\n- šŸ  Data Residency: Choose your data storage region\n- ā° Retention Control: Set custom data retention policies\nNext Steps\n- Tracing Concepts - Learn about traces, spans, and observability best practices\n- Framework Integrations - Deep dive into specific framework setups\n- Multi-Agent Tracing - Observe complex agent work...\n\nSource 4 (https://noveum.ai/docs/getting-started/sdk-integration):\n- šŸ‘„ Collaborate with your team on insights\nšŸ”’ Security & Privacy\n- šŸ” Encryption: All data encrypted in transit and at rest\n- šŸŽ›ļø Configurable Capture: Control what data is collected\n- šŸ  Data Residency: Choose your data storage region\n- ā° Retention Control: Set custom data retention policies\nNext Steps\n- Tracing Concepts - Learn about traces, spans, and observability best practices\n- Framework Integrations - Deep dive into specific framework setups\n- Multi-Agent Tracing - Observe complex agent work...\n\nSource 5 (https://noveum.ai/docs/platform/dashboard):\n- Bulk Export: Download multiple traces simultaneously\n- Comparative Analysis: Compare performance across multiple traces\nReal-Time Updates\n- Live Refresh: Automatic updates as new traces arrive\n- Connection Monitoring: Real-time status of your trace ingestion pipeline\n- Performance Indicators: Live metrics for system health monitoring\nReady to dive deeper? Explore Projects & Environments to organize your AI applications, or check out Team Collaboration to share insights with your team.\nGet Earl...\n\nUser Question: What features does Noveum Trace offer?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", - "prompt.prompt_length": 3321, - "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/docs):\nCore Components\n-\nšŸ Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frame...", - "prompt.instruction_following": "rag_optimized", - "response.answer_length": 1014, - "response.answer_completeness": "complete", - "response.response_quality": "high", - "response.source_citation": 5, - "response.context_utilization": 0.3638320775026911, - "output_response": "RAG Answer: Noveum Trace offers several key features through its Python SDK (`noveum-trace`) and TypeScript SDK (`@noveum/trace`):\n\n1. **Decorator-based Tracing**: It provides a simple way to trace LLM calls, age...", - "evaluation.retrieval_effectiveness": 1, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 5, - "evaluation.context_relevance": "high", - "evaluation.overall_quality": "high", - "evaluation.ready_for_production": true, - "rag.retrieval_strategy": "semantic_similarity", - "rag.vector_search_results": 5, - "rag.context_synthesis": "multi_source", - "rag.document_coverage": 1, - "rag.information_density": 0.3638320775026911 - }, - "events": [], - "links": [], - "trace_trace_id": "16143f74-2831-4753-b33d-ce4b645093c5", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-09 08:25:12.442000000", - "trace_end_time": "2025-10-09 08:25:19.579000000", - "trace_duration_ms": 7137.402, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-09 08:25:20.290000000", - "updated_at": "2025-10-09 08:25:20.290000000" - }, - { - "span_id": "08db68f6-3623-44b5-840b-41400fac7333", - "trace_id": "fc64e6cc-6739-4256-ac4a-7b80c3028233", - "parent_span_id": "0e58e31f-a8ab-4aee-8fc2-abcae1e21d9d", - "name": "agent.rag_evaluation_metrics", - "start_time": "2025-10-09 08:25:10.938000000", - "end_time": "2025-10-09 08:25:10.938000000", - "duration_ms": 0.043000000000000003, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "rag_evaluation_metrics", - "agent.capabilities": [ - "retrieval_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What are Noveum's pricing plans?", - "query_length": 32, - "query_type": "rag_evaluation_query", - "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....", - "retrieval.context_length": 2105, - "retrieval.sources_count": 5, - "retrieval.context_quality": "high", - "retrieval.effectiveness": 1, - "retrieval.context_utilization": 2.105, - "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nSource 2 (https://noveum.ai/en/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nSource 3 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nThe AI landscape is complex and moving fast. With Noveum.ai, you'll have the insights you need to build, debug, and optimize AI applications with confidence.\nReady to see what's really happening inside your AI applications? Try Noveum.ai today and transform how you build and monitor AI systems.\nLet's build more reliable, observable AIā€”together.\nGet Early Access to Noveum.ai Platform\nJoin the select group of AI teams optimizing their models with our data-driven platform. We're onboarding users in...\n\nSource 4 (https://noveum.ai/en/blog/from-logs-to-intelligent-choices-inside-noveum-ais-evaluation-process):\nGet Early Access to Noveum.ai Platform\nJoin the select group of AI teams optimizing their models with our data-driven platform. We're onboarding users in limited batches to ensure a premium experience....\n\nSource 5 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, vector searches, data retrieval, agent reasoning, and more. Without proper observability, debugging becomes a nightmare, optimization is guesswork, and understanding user interactions is nearly impossible.\nNoveum.ai solves this by providing comprehensive tracing and observability specifically designed for AI applications.\nKey benefits of using Noveum.ai:\n- Complete Visibility: Trace...\n\nUser Question: What are Noveum's pricing plans?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", - "prompt.prompt_length": 2633, - "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to ...", - "prompt.instruction_following": "rag_optimized", - "response.answer_length": 399, - "response.answer_completeness": "complete", - "response.response_quality": "high", - "response.source_citation": 5, - "response.context_utilization": 0.18954869358669835, - "output_response": "RAG Answer: The provided context does not include specific information about Noveum's pricing plans. However, it mentions that all users will have access to the Observability suite for free, and early users will ...", - "evaluation.retrieval_effectiveness": 1, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 5, - "evaluation.context_relevance": "high", - "evaluation.overall_quality": "high", - "evaluation.ready_for_production": true, - "rag.retrieval_strategy": "semantic_similarity", - "rag.vector_search_results": 5, - "rag.context_synthesis": "multi_source", - "rag.document_coverage": 1, - "rag.information_density": 0.18954869358669835 - }, - "events": [], - "links": [], - "trace_trace_id": "fc64e6cc-6739-4256-ac4a-7b80c3028233", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-09 08:25:07.932000000", - "trace_end_time": "2025-10-09 08:25:11.438000000", - "trace_duration_ms": 3506.0989999999997, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-09 08:25:12.651000000", - "updated_at": "2025-10-09 08:25:12.651000000" - }, - { - "span_id": "6a57c92e-e473-425a-978b-19dbe976a46e", - "trace_id": "b7945c49-f584-4c70-972d-536a805d8a31", - "parent_span_id": "48155aad-6204-4a52-8624-8fa5d6fe407f", - "name": "agent.rag_evaluation_metrics", - "start_time": "2025-10-09 08:25:05.873000000", - "end_time": "2025-10-09 08:25:05.873000000", - "duration_ms": 0.229, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "rag_evaluation_metrics", - "agent.capabilities": [ - "retrieval_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: How do I integrate Noveum Trace in my application?", - "query_length": 50, - "query_type": "rag_evaluation_query", - "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/docs):\nCore Components\n-\nšŸ Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\nšŸ“˜ TypeScript SDK (\n@noveum/trace\n)- Framework ...", - "retrieval.context_length": 2803, - "retrieval.sources_count": 5, - "retrieval.context_quality": "high", - "retrieval.effectiveness": 1, - "retrieval.context_utilization": 2.803, - "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/docs):\nCore Components\n-\nšŸ Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\nšŸ“˜ TypeScript SDK (\n@noveum/trace\n)- Framework integrations for Next.js, Express.js, and Hono\n- TypeScript-first design with full type safety\n- Universal compatibility (Node.js, Edge Runtime, browsers)\n-\nšŸ“Š Noveum Platform\n- Real-time dashboard for analyzing traces and performance\n- Ad...\n\nSource 2 (https://noveum.ai/docs):\nCore Components\n-\nšŸ Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\nšŸ“˜ TypeScript SDK (\n@noveum/trace\n)- Framework integrations for Next.js, Express.js, and Hono\n- TypeScript-first design with full type safety\n- Universal compatibility (Node.js, Edge Runtime, browsers)\n-\nšŸ“Š Noveum Platform\n- Real-time dashboard for analyzing traces and performance\n- Ad...\n\nSource 3 (https://noveum.ai/docs/getting-started/sdk-integration):\nSDK Integration Guide\nIntegrate Noveum.ai tracing into your AI applications with Python or TypeScript SDKs\nThe Noveum.ai SDKs provide comprehensive tracing and observability for your AI applications with minimal code changes. Whether you're building LLM applications, RAG systems, or multi-agent workflows, our SDKs automatically capture essential metrics and traces.\nšŸš€ Quick Start\n1. Create Your Account & Get API Key\n- Sign up at noveum.ai\n- Create a project in your dashboard\n- Generate an API key...\n\nSource 4 (https://noveum.ai/en/docs/getting-started/sdk-integration):\nSDK Integration Guide\nIntegrate Noveum.ai tracing into your AI applications with Python or TypeScript SDKs\nThe Noveum.ai SDKs provide comprehensive tracing and observability for your AI applications with minimal code changes. Whether you're building LLM applications, RAG systems, or multi-agent workflows, our SDKs automatically capture essential metrics and traces.\nšŸš€ Quick Start\n1. Create Your Account & Get API Key\n- Sign up at noveum.ai\n- Create a project in your dashboard\n- Generate an API key...\n\nSource 5 (https://noveum.ai/en/docs/getting-started/sdk-integration):\nšŸ”§ Framework Integrations\nNext.js Integration\nExpress.js Integration\nFastAPI Integration (Python)\nšŸ“Š Advanced Features\nCustom Attributes & Events\nSampling Configuration\nšŸ”— What's Captured Automatically\n- šŸ“Š Performance Metrics: Latency, throughput, error rates\n- šŸ’° Cost Tracking: Token usage, API costs across providers\n- šŸ” Request/Response: Configurable capture of inputs/outputs\n- šŸ·ļø Metadata: Model names, parameters, user context\n- šŸŒŠ Context Flow: Trace relationships across services\n- šŸ› Error Detail...\n\nUser Question: How do I integrate Noveum Trace in my application?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", - "prompt.prompt_length": 3349, - "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/docs):\nCore Components\n-\nšŸ Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frame...", - "prompt.instruction_following": "rag_optimized", - "response.answer_length": 1151, - "response.answer_completeness": "complete", - "response.response_quality": "high", - "response.source_citation": 5, - "response.context_utilization": 0.410631466286122, - "output_response": "RAG Answer: To integrate Noveum Trace into your application, you can use either the Python SDK (`noveum-trace`) or the TypeScript SDK (`@noveum/trace`). Hereā€™s a quick start guide based on the provided documentat...", - "evaluation.retrieval_effectiveness": 1, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 5, - "evaluation.context_relevance": "high", - "evaluation.overall_quality": "high", - "evaluation.ready_for_production": true, - "rag.retrieval_strategy": "semantic_similarity", - "rag.vector_search_results": 5, - "rag.context_synthesis": "multi_source", - "rag.document_coverage": 1, - "rag.information_density": 0.410631466286122 - }, - "events": [], - "links": [], - "trace_trace_id": "b7945c49-f584-4c70-972d-536a805d8a31", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-09 08:24:58.518000000", - "trace_end_time": "2025-10-09 08:25:06.920000000", - "trace_duration_ms": 8402.516, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-09 08:25:07.620000000", - "updated_at": "2025-10-09 08:25:07.620000000" - }, - { - "span_id": "efb19490-bc9e-48dc-a365-2f839100e41a", - "trace_id": "f5c40ecf-36c0-45ba-9cc9-dc0329b0324b", - "parent_span_id": "57ded2eb-0f3f-474e-8704-5ad9974ab4bb", - "name": "agent.rag_evaluation_metrics", - "start_time": "2025-10-09 08:24:56.965000000", - "end_time": "2025-10-09 08:24:56.965000000", - "duration_ms": 0.048, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "rag_evaluation_metrics", - "agent.capabilities": [ - "retrieval_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What is Noveum and what does it do?", - "query_length": 35, - "query_type": "rag_evaluation_query", - "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, vector searches, data retrieval, agent reasoning, and more. Without proper observability, debugging ...", - "retrieval.context_length": 2334, - "retrieval.sources_count": 5, - "retrieval.context_quality": "high", - "retrieval.effectiveness": 1, - "retrieval.context_utilization": 2.334, - "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, vector searches, data retrieval, agent reasoning, and more. Without proper observability, debugging becomes a nightmare, optimization is guesswork, and understanding user interactions is nearly impossible.\nNoveum.ai solves this by providing comprehensive tracing and observability specifically designed for AI applications.\nKey benefits of using Noveum.ai:\n- Complete Visibility: Trace...\n\nSource 2 (https://noveum.ai/en/docs/getting-started/overview):\nNoveum.ai Overview\nComprehensive AI tracing and observability platform for LLM applications, RAG systems, and AI agents\nWelcome to Noveum.aiā€”the comprehensive tracing and observability platform built specifically for AI applications. Whether you're building LLM-powered chatbots, RAG systems, multi-agent workflows, or any AI-driven application, Noveum provides the insights you need to understand, debug, and optimize your systems.\nšŸŽÆ Why AI Applications Need Specialized Observability\nTraditional mo...\n\nSource 3 (https://noveum.ai/docs/getting-started/overview):\nNoveum.ai Overview\nComprehensive AI tracing and observability platform for LLM applications, RAG systems, and AI agents\nWelcome to Noveum.aiā€”the comprehensive tracing and observability platform built specifically for AI applications. Whether you're building LLM-powered chatbots, RAG systems, multi-agent workflows, or any AI-driven application, Noveum provides the insights you need to understand, debug, and optimize your systems.\nšŸŽÆ Why AI Applications Need Specialized Observability\nTraditional mo...\n\nSource 4 (https://noveum.ai/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nSource 5 (https://noveum.ai/en/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nUser Question: What is Noveum and what does it do?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", - "prompt.prompt_length": 2865, - "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, ...", - "prompt.instruction_following": "rag_optimized", - "response.answer_length": 611, - "response.answer_completeness": "complete", - "response.response_quality": "high", - "response.source_citation": 5, - "response.context_utilization": 0.26178234790059984, - "output_response": "RAG Answer: Noveum.ai is a comprehensive tracing and observability platform specifically designed for AI applications, including those powered by large language models (LLMs), retrieval-augmented generation (RAG)...", - "evaluation.retrieval_effectiveness": 1, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 5, - "evaluation.context_relevance": "high", - "evaluation.overall_quality": "high", - "evaluation.ready_for_production": true, - "rag.retrieval_strategy": "semantic_similarity", - "rag.vector_search_results": 5, - "rag.context_synthesis": "multi_source", - "rag.document_coverage": 1, - "rag.information_density": 0.26178234790059984 - }, - "events": [], - "links": [], - "trace_trace_id": "f5c40ecf-36c0-45ba-9cc9-dc0329b0324b", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-09 08:24:54.009000000", - "trace_end_time": "2025-10-09 08:24:57.511000000", - "trace_duration_ms": 3502.062, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-09 08:24:59.716000000", - "updated_at": "2025-10-09 08:24:59.716000000" - } -] \ No newline at end of file diff --git a/noveum_customer_support_bt/split_datasets/agent.routing_evaluation_metrics_dataset.json b/noveum_customer_support_bt/split_datasets/agent.routing_evaluation_metrics_dataset.json deleted file mode 100644 index d52d6a8..0000000 --- a/noveum_customer_support_bt/split_datasets/agent.routing_evaluation_metrics_dataset.json +++ /dev/null @@ -1,1602 +0,0 @@ -[ - { - "span_id": "95dac0c3-83fd-459c-af4e-c6a1734fd874", - "trace_id": "eda4fe22-9a2b-4b73-856b-f4f3309bf719", - "parent_span_id": "0b19befc-5dc0-4b00-9a8b-5f2f2c914f17", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-09 08:26:46.384000000", - "end_time": "2025-10-09 08:26:46.384000000", - "duration_ms": 0.041, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What are the latest breakthroughs in artificial intelligence?", - "query_length": 61, - "query_type": "routing_evaluation_query", - "classification.mode": "Web Search", - "classification.rag_keyword_score": 0, - "classification.web_keyword_score": 1, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "complex", - "query.intent": "general_knowledge", - "query.keyword_density": 0.125, - "query.domain_affinity": "general", - "routing.decision": "Routed to Web Search based on analysis", - "routing.rationale": "RAG score: 0, Web score: 1, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "RAG", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.019230769230769232, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: Web Search (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "eda4fe22-9a2b-4b73-856b-f4f3309bf719", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-09 08:26:46.384000000", - "trace_end_time": "2025-10-09 08:26:48.685000000", - "trace_duration_ms": 2301.488, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-09 08:26:49.422000000", - "updated_at": "2025-10-09 08:26:49.422000000" - }, - { - "span_id": "516060a3-0f58-46ba-8a6a-4b502a376492", - "trace_id": "0ffffba1-8a37-443c-8866-d53ffbfa7718", - "parent_span_id": "30644d2a-5ab1-4a73-b4ff-cf9fa478e36b", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-09 08:26:41.850000000", - "end_time": "2025-10-09 08:26:41.850000000", - "duration_ms": 0.055, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What's happening in the software development world today?", - "query_length": 57, - "query_type": "routing_evaluation_query", - "classification.mode": "Web Search", - "classification.rag_keyword_score": 0, - "classification.web_keyword_score": 1, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "complex", - "query.intent": "general_knowledge", - "query.keyword_density": 0.125, - "query.domain_affinity": "general", - "routing.decision": "Routed to Web Search based on analysis", - "routing.rationale": "RAG score: 0, Web score: 1, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "RAG", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.019230769230769232, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: Web Search (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "0ffffba1-8a37-443c-8866-d53ffbfa7718", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-09 08:26:41.850000000", - "trace_end_time": "2025-10-09 08:26:45.377000000", - "trace_duration_ms": 3527.831, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-09 08:26:49.422000000", - "updated_at": "2025-10-09 08:26:49.422000000" - }, - { - "span_id": "8f8d2dc4-150f-4e43-aad3-96da4d06f586", - "trace_id": "f1f37bd7-0851-4659-b493-b80d3800d920", - "parent_span_id": "89eac2de-b7da-4db3-bf7c-8e87c429481e", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-09 08:26:35.915000000", - "end_time": "2025-10-09 08:26:35.915000000", - "duration_ms": 0.062, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What are the newest features in cloud computing?", - "query_length": 48, - "query_type": "routing_evaluation_query", - "classification.mode": "RAG", - "classification.rag_keyword_score": 1, - "classification.web_keyword_score": 0, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "documentation", - "query.keyword_density": 0.125, - "query.domain_affinity": "noveum", - "routing.decision": "Routed to RAG based on analysis", - "routing.rationale": "RAG score: 1, Web score: 0, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "Web Search", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.019230769230769232, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: RAG (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "f1f37bd7-0851-4659-b493-b80d3800d920", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-09 08:26:35.915000000", - "trace_end_time": "2025-10-09 08:26:40.841000000", - "trace_duration_ms": 4926.342, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-09 08:26:44.053000000", - "updated_at": "2025-10-09 08:26:44.053000000" - }, - { - "span_id": "cf0a915c-668b-412e-9c14-69119d059dc2", - "trace_id": "43cdf081-4f01-49cd-b566-dbd1619e6cd2", - "parent_span_id": "962afebb-26d1-45ff-acf1-218163cd6bb8", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-09 08:26:32.240000000", - "end_time": "2025-10-09 08:26:32.240000000", - "duration_ms": 0.05, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What's the current status of cryptocurrency markets?", - "query_length": 52, - "query_type": "routing_evaluation_query", - "classification.mode": "Web Search", - "classification.rag_keyword_score": 0, - "classification.web_keyword_score": 3, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "complex", - "query.intent": "general_knowledge", - "query.keyword_density": 0.42857142857142855, - "query.domain_affinity": "general", - "routing.decision": "Routed to Web Search based on analysis", - "routing.rationale": "RAG score: 0, Web score: 3, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "RAG", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.057692307692307696, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: Web Search (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "43cdf081-4f01-49cd-b566-dbd1619e6cd2", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-09 08:26:32.237000000", - "trace_end_time": "2025-10-09 08:26:34.906000000", - "trace_duration_ms": 2668.841, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-09 08:26:38.618000000", - "updated_at": "2025-10-09 08:26:38.618000000" - }, - { - "span_id": "f75b51b8-4882-4170-bc08-2adcab250f50", - "trace_id": "9a1983f4-09da-4b53-80e6-38de6878e0e7", - "parent_span_id": "246cf822-4562-4d3a-8a5a-e38e0a0b02a0", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-09 08:26:27.992000000", - "end_time": "2025-10-09 08:26:27.992000000", - "duration_ms": 0.060000000000000005, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What are the latest updates in Python programming?", - "query_length": 50, - "query_type": "routing_evaluation_query", - "classification.mode": "Web Search", - "classification.rag_keyword_score": 0, - "classification.web_keyword_score": 2, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "general_knowledge", - "query.keyword_density": 0.25, - "query.domain_affinity": "general", - "routing.decision": "Routed to Web Search based on analysis", - "routing.rationale": "RAG score: 0, Web score: 2, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "RAG", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.038461538461538464, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: Web Search (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "9a1983f4-09da-4b53-80e6-38de6878e0e7", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-09 08:26:27.992000000", - "trace_end_time": "2025-10-09 08:26:31.231000000", - "trace_duration_ms": 3239.706, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-09 08:26:33.463000000", - "updated_at": "2025-10-09 08:26:33.463000000" - }, - { - "span_id": "4180abcf-924c-45ac-8dfa-17c0bc603eff", - "trace_id": "5d2517e1-220a-429d-9d59-f701bda25eed", - "parent_span_id": "20544f91-8828-4a0e-b253-ed1f8b95b205", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-09 08:26:23.210000000", - "end_time": "2025-10-09 08:26:23.210000000", - "duration_ms": 0.044, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What happened in tech news this week?", - "query_length": 37, - "query_type": "routing_evaluation_query", - "classification.mode": "Web Search", - "classification.rag_keyword_score": 0, - "classification.web_keyword_score": 2, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "general_knowledge", - "query.keyword_density": 0.2857142857142857, - "query.domain_affinity": "general", - "routing.decision": "Routed to Web Search based on analysis", - "routing.rationale": "RAG score: 0, Web score: 2, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "RAG", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.038461538461538464, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: Web Search (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "5d2517e1-220a-429d-9d59-f701bda25eed", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-09 08:26:23.210000000", - "trace_end_time": "2025-10-09 08:26:26.985000000", - "trace_duration_ms": 3775.124, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-09 08:26:28.174000000", - "updated_at": "2025-10-09 08:26:28.174000000" - }, - { - "span_id": "8b0022cc-91dd-4065-91ed-b77217f92bd7", - "trace_id": "52aacb67-c361-4445-9b72-c157f79f47d6", - "parent_span_id": "5359c861-c51b-446d-9ad2-b3d8c21ea70f", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-09 08:26:16.544000000", - "end_time": "2025-10-09 08:26:16.544000000", - "duration_ms": 0.087, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What are the current trends in observability tools?", - "query_length": 51, - "query_type": "routing_evaluation_query", - "classification.mode": "RAG", - "classification.rag_keyword_score": 2, - "classification.web_keyword_score": 1, - "classification.confidence_score": 0.3333333333333333, - "classification.confidence_level": "medium", - "classification.method": "keyword_based", - "query.complexity": "complex", - "query.intent": "documentation", - "query.keyword_density": 0.375, - "query.domain_affinity": "noveum", - "routing.decision": "Routed to RAG based on analysis", - "routing.rationale": "RAG score: 2, Web score: 1, Confidence: 0.33", - "routing.expected_performance": "medium", - "routing.alternative_mode": "Web Search", - "routing.decision_confidence": 0.3333333333333333, - "evaluation.routing_accuracy": "medium", - "evaluation.keyword_coverage": 0.057692307692307696, - "evaluation.query_understanding": "ambiguous", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: RAG (Confidence: 0.33)" - }, - "events": [], - "links": [], - "trace_trace_id": "52aacb67-c361-4445-9b72-c157f79f47d6", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-09 08:26:16.544000000", - "trace_end_time": "2025-10-09 08:26:22.201000000", - "trace_duration_ms": 5657.012, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-09 08:26:22.904000000", - "updated_at": "2025-10-09 08:26:22.904000000" - }, - { - "span_id": "780df186-fe5a-4742-b937-2a142ad528c6", - "trace_id": "a81ca3a8-80aa-4c39-876e-8d40ea7a0aef", - "parent_span_id": "acf6a99a-39e0-4f87-a07e-3a61213ab032", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-09 08:26:08.903000000", - "end_time": "2025-10-09 08:26:08.903000000", - "duration_ms": 0.06499999999999999, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: Tell me about recent developments in machine learning", - "query_length": 53, - "query_type": "routing_evaluation_query", - "classification.mode": "Web Search", - "classification.rag_keyword_score": 0, - "classification.web_keyword_score": 1, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "complex", - "query.intent": "general_knowledge", - "query.keyword_density": 0.125, - "query.domain_affinity": "general", - "routing.decision": "Routed to Web Search based on analysis", - "routing.rationale": "RAG score: 0, Web score: 1, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "RAG", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.019230769230769232, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: Web Search (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "a81ca3a8-80aa-4c39-876e-8d40ea7a0aef", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-09 08:26:08.903000000", - "trace_end_time": "2025-10-09 08:26:15.537000000", - "trace_duration_ms": 6633.662, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-09 08:26:16.227000000", - "updated_at": "2025-10-09 08:26:16.227000000" - }, - { - "span_id": "f80ae3e8-b4dd-42c6-b17b-de7805cedbb9", - "trace_id": "230aad27-f3dd-4968-a45a-3c2f07ac28ed", - "parent_span_id": "0b47ea07-1eda-4b6a-a183-ea9ef047e431", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-09 08:26:02.424000000", - "end_time": "2025-10-09 08:26:02.424000000", - "duration_ms": 0.025, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What's the weather like today?", - "query_length": 30, - "query_type": "routing_evaluation_query", - "classification.mode": "Web Search", - "classification.rag_keyword_score": 0, - "classification.web_keyword_score": 2, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "general_knowledge", - "query.keyword_density": 0.4, - "query.domain_affinity": "general", - "routing.decision": "Routed to Web Search based on analysis", - "routing.rationale": "RAG score: 0, Web score: 2, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "RAG", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.038461538461538464, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: Web Search (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "230aad27-f3dd-4968-a45a-3c2f07ac28ed", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-09 08:26:02.424000000", - "trace_end_time": "2025-10-09 08:26:07.894000000", - "trace_duration_ms": 5470.232, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-09 08:26:08.586000000", - "updated_at": "2025-10-09 08:26:08.586000000" - }, - { - "span_id": "209259cf-411d-4b97-8f7b-40b2530eec81", - "trace_id": "83c7dcce-3d89-4da1-8b3f-d419885d4cbc", - "parent_span_id": "3866ddb3-b17e-47a1-8b3f-bca1e32a2ae4", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-09 08:25:56.886000000", - "end_time": "2025-10-09 08:25:56.886000000", - "duration_ms": 0.017, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What are the latest AI news today?", - "query_length": 34, - "query_type": "routing_evaluation_query", - "classification.mode": "Web Search", - "classification.rag_keyword_score": 0, - "classification.web_keyword_score": 3, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "general_knowledge", - "query.keyword_density": 0.42857142857142855, - "query.domain_affinity": "general", - "routing.decision": "Routed to Web Search based on analysis", - "routing.rationale": "RAG score: 0, Web score: 3, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "RAG", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.057692307692307696, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: Web Search (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "83c7dcce-3d89-4da1-8b3f-d419885d4cbc", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-09 08:25:56.886000000", - "trace_end_time": "2025-10-09 08:26:01.416000000", - "trace_duration_ms": 4530.08, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-09 08:26:02.122000000", - "updated_at": "2025-10-09 08:26:02.122000000" - }, - { - "span_id": "c3f3bd32-9954-443f-93e5-52aa6cf412a0", - "trace_id": "2218f641-604c-491a-9710-b51a9941b982", - "parent_span_id": "3ce2e35e-4518-43b4-8a76-4413da6832ea", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-09 08:25:48.107000000", - "end_time": "2025-10-09 08:25:48.107000000", - "duration_ms": 0.066, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What are the benefits of using Noveum Trace?", - "query_length": 44, - "query_type": "routing_evaluation_query", - "classification.mode": "RAG", - "classification.rag_keyword_score": 2, - "classification.web_keyword_score": 0, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "noveum_specific", - "query.keyword_density": 0.25, - "query.domain_affinity": "noveum", - "routing.decision": "Routed to RAG based on analysis", - "routing.rationale": "RAG score: 2, Web score: 0, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "Web Search", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.038461538461538464, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: RAG (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "2218f641-604c-491a-9710-b51a9941b982", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-09 08:25:48.107000000", - "trace_end_time": "2025-10-09 08:25:55.878000000", - "trace_duration_ms": 7770.953, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-09 08:25:56.565000000", - "updated_at": "2025-10-09 08:25:56.565000000" - }, - { - "span_id": "940ce621-0015-45ed-adc6-c4caa2a95e81", - "trace_id": "255fd49c-84b4-4b18-887e-6308a412d535", - "parent_span_id": "c0c39e8b-9a42-4c18-8737-9224cd8a45c0", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-09 08:25:43.830000000", - "end_time": "2025-10-09 08:25:43.830000000", - "duration_ms": 0.022, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: How do I configure Noveum for my system?", - "query_length": 40, - "query_type": "routing_evaluation_query", - "classification.mode": "RAG", - "classification.rag_keyword_score": 2, - "classification.web_keyword_score": 0, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "noveum_specific", - "query.keyword_density": 0.25, - "query.domain_affinity": "noveum", - "routing.decision": "Routed to RAG based on analysis", - "routing.rationale": "RAG score: 2, Web score: 0, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "Web Search", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.038461538461538464, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: RAG (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "255fd49c-84b4-4b18-887e-6308a412d535", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-09 08:25:43.830000000", - "trace_end_time": "2025-10-09 08:25:47.099000000", - "trace_duration_ms": 3268.862, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-09 08:25:50.304000000", - "updated_at": "2025-10-09 08:25:50.304000000" - }, - { - "span_id": "ea28f049-6bc9-48b4-8386-279abe087ca8", - "trace_id": "dc511122-c0b6-415c-9a49-c7b45132dd87", - "parent_span_id": "80987c51-ea82-4a41-97c7-ed591082b9bf", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-09 08:25:38.322000000", - "end_time": "2025-10-09 08:25:38.322000000", - "duration_ms": 0.041, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What monitoring capabilities does Noveum provide?", - "query_length": 49, - "query_type": "routing_evaluation_query", - "classification.mode": "RAG", - "classification.rag_keyword_score": 2, - "classification.web_keyword_score": 0, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "noveum_specific", - "query.keyword_density": 0.3333333333333333, - "query.domain_affinity": "noveum", - "routing.decision": "Routed to RAG based on analysis", - "routing.rationale": "RAG score: 2, Web score: 0, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "Web Search", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.038461538461538464, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: RAG (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "dc511122-c0b6-415c-9a49-c7b45132dd87", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-09 08:25:38.322000000", - "trace_end_time": "2025-10-09 08:25:42.822000000", - "trace_duration_ms": 4500.425, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-09 08:25:45.028000000", - "updated_at": "2025-10-09 08:25:45.028000000" - }, - { - "span_id": "01a03da5-64df-41e3-b9c9-556e12a8edd7", - "trace_id": "04bebf38-a343-4563-80db-0154bef8d927", - "parent_span_id": "2f89c1e0-9b62-4651-914c-b71cf561c33c", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-09 08:25:33.693000000", - "end_time": "2025-10-09 08:25:33.693000000", - "duration_ms": 0.044, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: How does Noveum handle agent tracing?", - "query_length": 37, - "query_type": "routing_evaluation_query", - "classification.mode": "RAG", - "classification.rag_keyword_score": 2, - "classification.web_keyword_score": 0, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "noveum_specific", - "query.keyword_density": 0.3333333333333333, - "query.domain_affinity": "noveum", - "routing.decision": "Routed to RAG based on analysis", - "routing.rationale": "RAG score: 2, Web score: 0, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "Web Search", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.038461538461538464, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: RAG (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "04bebf38-a343-4563-80db-0154bef8d927", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-09 08:25:33.693000000", - "trace_end_time": "2025-10-09 08:25:37.313000000", - "trace_duration_ms": 3619.672, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-09 08:25:40.339000000", - "updated_at": "2025-10-09 08:25:40.339000000" - }, - { - "span_id": "8d9c632b-b2e8-496d-b1b0-a89e780ef810", - "trace_id": "5e043630-6493-42b5-beb8-79faa19bfa37", - "parent_span_id": "e30e3e61-6edd-4b62-ab77-627faa2bf741", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-09 08:25:29.901000000", - "end_time": "2025-10-09 08:25:29.901000000", - "duration_ms": 0.028, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What APIs are available in Noveum platform?", - "query_length": 43, - "query_type": "routing_evaluation_query", - "classification.mode": "RAG", - "classification.rag_keyword_score": 3, - "classification.web_keyword_score": 0, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "noveum_specific", - "query.keyword_density": 0.42857142857142855, - "query.domain_affinity": "noveum", - "routing.decision": "Routed to RAG based on analysis", - "routing.rationale": "RAG score: 3, Web score: 0, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "Web Search", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.057692307692307696, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: RAG (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "5e043630-6493-42b5-beb8-79faa19bfa37", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-09 08:25:29.901000000", - "trace_end_time": "2025-10-09 08:25:32.685000000", - "trace_duration_ms": 2783.326, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-09 08:25:34.918000000", - "updated_at": "2025-10-09 08:25:34.918000000" - }, - { - "span_id": "72dfbba4-0672-4adc-a1bf-308157d831e6", - "trace_id": "7da9814d-a2e8-4c4e-b750-68b26bd5fd22", - "parent_span_id": "2f513795-764f-4eb6-8dd2-108d4cdf8432", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-09 08:25:20.587000000", - "end_time": "2025-10-09 08:25:20.587000000", - "duration_ms": 0.041999999999999996, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: How do I set up observability with Noveum?", - "query_length": 42, - "query_type": "routing_evaluation_query", - "classification.mode": "RAG", - "classification.rag_keyword_score": 2, - "classification.web_keyword_score": 0, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "noveum_specific", - "query.keyword_density": 0.25, - "query.domain_affinity": "noveum", - "routing.decision": "Routed to RAG based on analysis", - "routing.rationale": "RAG score: 2, Web score: 0, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "Web Search", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.038461538461538464, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: RAG (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "7da9814d-a2e8-4c4e-b750-68b26bd5fd22", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-09 08:25:20.586000000", - "trace_end_time": "2025-10-09 08:25:28.897000000", - "trace_duration_ms": 8310.205, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-09 08:25:29.588000000", - "updated_at": "2025-10-09 08:25:29.588000000" - }, - { - "span_id": "92c81ee2-728d-4ea6-bc52-b81295d2aea2", - "trace_id": "16143f74-2831-4753-b33d-ce4b645093c5", - "parent_span_id": "29c7a515-2383-4b7d-8fc7-beac3f403641", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-09 08:25:12.442000000", - "end_time": "2025-10-09 08:25:12.442000000", - "duration_ms": 0.043000000000000003, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What features does Noveum Trace offer?", - "query_length": 38, - "query_type": "routing_evaluation_query", - "classification.mode": "RAG", - "classification.rag_keyword_score": 3, - "classification.web_keyword_score": 0, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "noveum_specific", - "query.keyword_density": 0.5, - "query.domain_affinity": "noveum", - "routing.decision": "Routed to RAG based on analysis", - "routing.rationale": "RAG score: 3, Web score: 0, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "Web Search", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.057692307692307696, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: RAG (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "16143f74-2831-4753-b33d-ce4b645093c5", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-09 08:25:12.442000000", - "trace_end_time": "2025-10-09 08:25:19.579000000", - "trace_duration_ms": 7137.402, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-09 08:25:20.290000000", - "updated_at": "2025-10-09 08:25:20.290000000" - }, - { - "span_id": "807627b0-41d9-478f-8a8b-26540d70af68", - "trace_id": "fc64e6cc-6739-4256-ac4a-7b80c3028233", - "parent_span_id": "12fee687-2a2e-401f-83c2-9c8dc7ecd853", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-09 08:25:07.932000000", - "end_time": "2025-10-09 08:25:07.932000000", - "duration_ms": 0.043000000000000003, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What are Noveum's pricing plans?", - "query_length": 32, - "query_type": "routing_evaluation_query", - "classification.mode": "RAG", - "classification.rag_keyword_score": 3, - "classification.web_keyword_score": 0, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "noveum_specific", - "query.keyword_density": 0.6, - "query.domain_affinity": "noveum", - "routing.decision": "Routed to RAG based on analysis", - "routing.rationale": "RAG score: 3, Web score: 0, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "Web Search", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.057692307692307696, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: RAG (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "fc64e6cc-6739-4256-ac4a-7b80c3028233", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-09 08:25:07.932000000", - "trace_end_time": "2025-10-09 08:25:11.438000000", - "trace_duration_ms": 3506.0989999999997, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-09 08:25:12.651000000", - "updated_at": "2025-10-09 08:25:12.651000000" - }, - { - "span_id": "bf860672-af22-44c1-b13b-0eabe8f72503", - "trace_id": "b7945c49-f584-4c70-972d-536a805d8a31", - "parent_span_id": "9c5aec32-b6eb-4c42-93d5-3970fb118c00", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-09 08:24:58.518000000", - "end_time": "2025-10-09 08:24:58.518000000", - "duration_ms": 0.054, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: How do I integrate Noveum Trace in my application?", - "query_length": 50, - "query_type": "routing_evaluation_query", - "classification.mode": "RAG", - "classification.rag_keyword_score": 2, - "classification.web_keyword_score": 0, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "noveum_specific", - "query.keyword_density": 0.2222222222222222, - "query.domain_affinity": "noveum", - "routing.decision": "Routed to RAG based on analysis", - "routing.rationale": "RAG score: 2, Web score: 0, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "Web Search", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.038461538461538464, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: RAG (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "b7945c49-f584-4c70-972d-536a805d8a31", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-09 08:24:58.518000000", - "trace_end_time": "2025-10-09 08:25:06.920000000", - "trace_duration_ms": 8402.516, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-09 08:25:07.620000000", - "updated_at": "2025-10-09 08:25:07.620000000" - }, - { - "span_id": "5fa714f1-eeb9-4617-bc7e-67062e6d1eab", - "trace_id": "f5c40ecf-36c0-45ba-9cc9-dc0329b0324b", - "parent_span_id": "f132d053-f82c-4e66-adf4-bd7ce68b660d", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-09 08:24:54.009000000", - "end_time": "2025-10-09 08:24:54.009000000", - "duration_ms": 0.022, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What is Noveum and what does it do?", - "query_length": 35, - "query_type": "routing_evaluation_query", - "classification.mode": "RAG", - "classification.rag_keyword_score": 1, - "classification.web_keyword_score": 0, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "noveum_specific", - "query.keyword_density": 0.125, - "query.domain_affinity": "noveum", - "routing.decision": "Routed to RAG based on analysis", - "routing.rationale": "RAG score: 1, Web score: 0, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "Web Search", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.019230769230769232, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: RAG (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "f5c40ecf-36c0-45ba-9cc9-dc0329b0324b", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-09 08:24:54.009000000", - "trace_end_time": "2025-10-09 08:24:57.511000000", - "trace_duration_ms": 3502.062, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-09 08:24:59.716000000", - "updated_at": "2025-10-09 08:24:59.716000000" - } -] \ No newline at end of file diff --git a/noveum_customer_support_bt/split_datasets/agent.web_search_evaluation_metrics_dataset.json b/noveum_customer_support_bt/split_datasets/agent.web_search_evaluation_metrics_dataset.json deleted file mode 100644 index d4f2542..0000000 --- a/noveum_customer_support_bt/split_datasets/agent.web_search_evaluation_metrics_dataset.json +++ /dev/null @@ -1,682 +0,0 @@ -[ - { - "span_id": "3b7b6838-22d4-4edf-9b00-7cb011f18a52", - "trace_id": "eda4fe22-9a2b-4b73-856b-f4f3309bf719", - "parent_span_id": "2aca614b-b016-4d56-abe2-43a144dc681d", - "name": "agent.web_search_evaluation_metrics", - "start_time": "2025-10-09 08:26:48.685000000", - "end_time": "2025-10-09 08:26:48.685000000", - "duration_ms": 0.01, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "web_search_evaluation_metrics", - "agent.capabilities": [ - "web_search_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What are the latest breakthroughs in artificial intelligence?", - "query_length": 61, - "query_type": "web_search_evaluation_query", - "web_search.results_count": 1, - "web_search.context_length": 708, - "web_search.context_synthesized": "Context from 1 web sources", - "web_search.search_effectiveness": 0.2, - "web_search.context_quality": "medium", - "web_search.source_diversity": 1, - "web_search.information_synthesis": "low", - "web_search.external_knowledge_utilization": 0.708, - "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nJul 23, 2014 Ā· ꌉꈑäøŖäŗŗēš„ē†č§£ļ¼Œlatest čæ™äøŖčƍå¼ŗ调ēš„ę˜Æꗶ闓åŗåˆ—äøŠęœ€é čæ‘ēŽ°åœØļ¼Œč€Œ newest å¼ŗ调ēš„ę˜Æꖰꗧēš„é—®é¢˜ć€‚ęƔ如čÆ“ļ¼Œč‹¹ęžœēš„ newest äŗ§å“ļ¼ŒęÆ«ę— ē–‘é—®ę˜Æ Mac Proļ¼Œčæ™ę˜ÆåŽ»å¹“ę‰å‡ŗēŽ°ēš„äøœ ā€¦ Breaking news from Hong Kong, China and Asia including geopolitics, diplomacy, economy, technology, lifestyle, entertainment and sport. Latest China news, opinions and analysis, covering Xi Jinping, Beijing's relations with Taiwan and China's tensions with the US. Latest Singapore news, including the city stateā€™s relationships with Malaysia and Mahathir, China and Xi Jinping, and the rest of Southeast Asia. Horse racing news, fixtures, race reports, features and analysis.\nURL: https://duckduckgo.com/?q=What+are+the+latest+breakthroughs+in+artificial+intelligence?\n\nUser Question: What are the latest breakthroughs in artificial intelligence?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", - "prompt.prompt_length": 1214, - "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nJul 23, 2014 Ā· ꌉꈑäøŖäŗŗēš„ē†č§£ļ¼Œlatest čæ™äøŖčƍå¼ŗ调ēš„ę˜Æꗶ闓åŗåˆ—äøŠęœ€é čæ‘ēŽ°åœØļ¼Œč€Œ newest å¼ŗ调ēš„ę˜Æꖰꗧēš„é—®é¢˜ć€‚ęƔ如čÆ“ļ¼Œč‹¹ęžœēš„ newest äŗ§å“ļ¼ŒęÆ«ę— ē–‘é—®ę˜Æ Mac Proļ¼Œčæ™ę˜ÆåŽ»å¹“ę‰å‡ŗēŽ°ēš„äøœ ā€¦ Breaking news from Hong Kong, China and Asia including geopol...", - "prompt.instruction_following": "web_search_optimized", - "response.answer_length": 423, - "response.answer_completeness": "complete", - "response.response_quality": "medium", - "response.source_citation": 1, - "response.context_utilization": 0.597457627118644, - "output_response": "Web Search Answer: The provided web search results do not contain specific information about the latest breakthroughs in artificial intelligence. To find the most recent advancements in AI, I recommend checking reputabl...", - "evaluation.search_effectiveness": 0.2, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 1, - "evaluation.context_relevance": "medium", - "evaluation.overall_quality": "low", - "evaluation.ready_for_production": true, - "web_search.search_strategy": "duckduckgo_api", - "web_search.real_time_data": true, - "web_search.external_sources": 1, - "web_search.information_freshness": "current", - "web_search.knowledge_synthesis": "single_source" - }, - "events": [], - "links": [], - "trace_trace_id": "eda4fe22-9a2b-4b73-856b-f4f3309bf719", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-09 08:26:46.384000000", - "trace_end_time": "2025-10-09 08:26:48.685000000", - "trace_duration_ms": 2301.488, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-09 08:26:49.422000000", - "updated_at": "2025-10-09 08:26:49.422000000" - }, - { - "span_id": "a5d480f0-ea5b-4190-a3c3-f94e26b48573", - "trace_id": "0ffffba1-8a37-443c-8866-d53ffbfa7718", - "parent_span_id": "03e3ae4d-0107-401d-8ca0-05fc0f28fb24", - "name": "agent.web_search_evaluation_metrics", - "start_time": "2025-10-09 08:26:45.377000000", - "end_time": "2025-10-09 08:26:45.377000000", - "duration_ms": 0.025, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "web_search_evaluation_metrics", - "agent.capabilities": [ - "web_search_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What's happening in the software development world today?", - "query_length": 57, - "query_type": "web_search_evaluation_query", - "web_search.results_count": 1, - "web_search.context_length": 159, - "web_search.context_synthesized": "Context from 1 web sources", - "web_search.search_effectiveness": 0.2, - "web_search.context_quality": "low", - "web_search.source_diversity": 1, - "web_search.information_synthesis": "low", - "web_search.external_knowledge_utilization": 0.159, - "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What's+happening+in+the+software+development+world+today?\n\nUser Question: What's happening in the software development world today?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", - "prompt.prompt_length": 661, - "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What's+happening+in+the+software+development+world+today?", - "prompt.instruction_following": "web_search_optimized", - "response.answer_length": 795, - "response.answer_completeness": "complete", - "response.response_quality": "medium", - "response.source_citation": 1, - "response.context_utilization": 5, - "output_response": "Web Search Answer: The web search did not yield any specific results regarding current events in the software development world. Therefore, I cannot provide detailed information on what's happening today in that field.\n...", - "evaluation.search_effectiveness": 0.2, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 1, - "evaluation.context_relevance": "low", - "evaluation.overall_quality": "low", - "evaluation.ready_for_production": true, - "web_search.search_strategy": "duckduckgo_api", - "web_search.real_time_data": true, - "web_search.external_sources": 1, - "web_search.information_freshness": "current", - "web_search.knowledge_synthesis": "single_source" - }, - "events": [], - "links": [], - "trace_trace_id": "0ffffba1-8a37-443c-8866-d53ffbfa7718", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-09 08:26:41.850000000", - "trace_end_time": "2025-10-09 08:26:45.377000000", - "trace_duration_ms": 3527.831, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-09 08:26:49.422000000", - "updated_at": "2025-10-09 08:26:49.422000000" - }, - { - "span_id": "2fd285de-68a1-4b7a-a4ed-05f539afa737", - "trace_id": "43cdf081-4f01-49cd-b566-dbd1619e6cd2", - "parent_span_id": "dfdb9aaf-9925-4a5c-a0c6-2cb00f70d4d3", - "name": "agent.web_search_evaluation_metrics", - "start_time": "2025-10-09 08:26:34.906000000", - "end_time": "2025-10-09 08:26:34.906000000", - "duration_ms": 0.019, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "web_search_evaluation_metrics", - "agent.capabilities": [ - "web_search_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What's the current status of cryptocurrency markets?", - "query_length": 52, - "query_type": "web_search_evaluation_query", - "web_search.results_count": 1, - "web_search.context_length": 154, - "web_search.context_synthesized": "Context from 1 web sources", - "web_search.search_effectiveness": 0.2, - "web_search.context_quality": "low", - "web_search.source_diversity": 1, - "web_search.information_synthesis": "low", - "web_search.external_knowledge_utilization": 0.154, - "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What's+the+current+status+of+cryptocurrency+markets?\n\nUser Question: What's the current status of cryptocurrency markets?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", - "prompt.prompt_length": 651, - "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What's+the+current+status+of+cryptocurrency+markets?", - "prompt.instruction_following": "web_search_optimized", - "response.answer_length": 371, - "response.answer_completeness": "complete", - "response.response_quality": "medium", - "response.source_citation": 1, - "response.context_utilization": 2.409090909090909, - "output_response": "Web Search Answer: The web search results did not provide any specific information regarding the current status of cryptocurrency markets. Therefore, I cannot provide an accurate update on market conditions, trends, or ...", - "evaluation.search_effectiveness": 0.2, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 1, - "evaluation.context_relevance": "low", - "evaluation.overall_quality": "low", - "evaluation.ready_for_production": true, - "web_search.search_strategy": "duckduckgo_api", - "web_search.real_time_data": true, - "web_search.external_sources": 1, - "web_search.information_freshness": "current", - "web_search.knowledge_synthesis": "single_source" - }, - "events": [], - "links": [], - "trace_trace_id": "43cdf081-4f01-49cd-b566-dbd1619e6cd2", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-09 08:26:32.237000000", - "trace_end_time": "2025-10-09 08:26:34.906000000", - "trace_duration_ms": 2668.841, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-09 08:26:38.618000000", - "updated_at": "2025-10-09 08:26:38.618000000" - }, - { - "span_id": "48deb050-478d-486c-be3a-b8c0069d7abe", - "trace_id": "9a1983f4-09da-4b53-80e6-38de6878e0e7", - "parent_span_id": "eefcfdfc-6433-459e-a134-e2647b8418e4", - "name": "agent.web_search_evaluation_metrics", - "start_time": "2025-10-09 08:26:31.231000000", - "end_time": "2025-10-09 08:26:31.231000000", - "duration_ms": 0.022, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "web_search_evaluation_metrics", - "agent.capabilities": [ - "web_search_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What are the latest updates in Python programming?", - "query_length": 50, - "query_type": "web_search_evaluation_query", - "web_search.results_count": 1, - "web_search.context_length": 646, - "web_search.context_synthesized": "Context from 1 web sources", - "web_search.search_effectiveness": 0.2, - "web_search.context_quality": "medium", - "web_search.source_diversity": 1, - "web_search.information_synthesis": "low", - "web_search.external_knowledge_utilization": 0.646, - "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nNews, opinions, infographics and videos on China issues: geopolitics, diplomacy, economy, technology and lifestyle. Latest China news, opinions and analysis, covering Xi Jinping, Beijing's relations with Taiwan and China's tensions with the US. Breaking news from Hong Kong, China and Asia including geopolitics, diplomacy, economy, technology, lifestyle, entertainment and sport. Horse racing news, fixtures, race reports, features and analysis. All the latest news, opinions and analysis on Hong Kong, China, Asia and around the world\nURL: https://duckduckgo.com/?q=What+are+the+latest+updates+in+Python+programming?\n\nUser Question: What are the latest updates in Python programming?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", - "prompt.prompt_length": 1141, - "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nNews, opinions, infographics and videos on China issues: geopolitics, diplomacy, economy, technology and lifestyle. Latest China news, opinions and analysis, covering Xi Ji...", - "prompt.instruction_following": "web_search_optimized", - "response.answer_length": 583, - "response.answer_completeness": "complete", - "response.response_quality": "medium", - "response.source_citation": 1, - "response.context_utilization": 0.9024767801857585, - "output_response": "Web Search Answer: The web search results do not provide specific information about the latest updates in Python programming. They focus primarily on news related to China, including geopolitics, economy, and lifestyle,...", - "evaluation.search_effectiveness": 0.2, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 1, - "evaluation.context_relevance": "medium", - "evaluation.overall_quality": "low", - "evaluation.ready_for_production": true, - "web_search.search_strategy": "duckduckgo_api", - "web_search.real_time_data": true, - "web_search.external_sources": 1, - "web_search.information_freshness": "current", - "web_search.knowledge_synthesis": "single_source" - }, - "events": [], - "links": [], - "trace_trace_id": "9a1983f4-09da-4b53-80e6-38de6878e0e7", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-09 08:26:27.992000000", - "trace_end_time": "2025-10-09 08:26:31.231000000", - "trace_duration_ms": 3239.706, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-09 08:26:33.463000000", - "updated_at": "2025-10-09 08:26:33.463000000" - }, - { - "span_id": "4ecede78-e7fc-48ed-9fba-1241d6890a1d", - "trace_id": "5d2517e1-220a-429d-9d59-f701bda25eed", - "parent_span_id": "c57b4b54-b9b0-4a4c-b5cd-57a4cfe5301b", - "name": "agent.web_search_evaluation_metrics", - "start_time": "2025-10-09 08:26:26.985000000", - "end_time": "2025-10-09 08:26:26.985000000", - "duration_ms": 0.016, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "web_search_evaluation_metrics", - "agent.capabilities": [ - "web_search_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What happened in tech news this week?", - "query_length": 37, - "query_type": "web_search_evaluation_query", - "web_search.results_count": 1, - "web_search.context_length": 640, - "web_search.context_synthesized": "Context from 1 web sources", - "web_search.search_effectiveness": 0.2, - "web_search.context_quality": "medium", - "web_search.source_diversity": 1, - "web_search.information_synthesis": "low", - "web_search.external_knowledge_utilization": 0.64, - "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nhappenedēš„ē”Øę³•å¦‚äø‹ļ¼š 1.happenedę˜Æ äøåŠē‰©åŠØčƍ,作ā€œå‘ē”Ÿā€č§£ę—¶,åÆęŒ‡å¶ē„¶ēš„äŗ‹ę•…,也åÆęŒ‡ęœ‰č®”åˆ’ęˆ–ę— č®”åˆ’å‘ē”Ÿēš„äŗ‹ć€‚äø€čˆ¬ä»„äŗ‹ä»¶ć€ēŽÆå¢ƒć€å½¢åŠæē­‰ä½œäø»čÆ­,č”Øē¤ŗā€œęŸäŗŗ发ē”ŸęŸäŗ‹ā€ę—¶,锻仄介čƍto引å‡ŗā€œ ā€¦ what happenedčæ˜ę˜Æwhat's happened\"What happened\"和\"What's happened\"都ē”ØäŗŽčƢ问čæ‡åŽ»å‘ē”Ÿäŗ†ä»€ä¹ˆäŗ‹ęƒ…ļ¼Œäø»č¦ēš„åŒŗ别åœØäŗŽčÆ­ę³•å’Œä½æē”Øę–¹å¼ć€‚\"What happened\"ę˜Æäø€äøŖčæ‡åŽ»ę—¶ę€ēš„ē®€å• ꎄäø‹ę„č®©ęˆ‘ä»¬ēœ‹äø‹what happen和what happenedēš„ē”Øę³•åŒŗ别ļ¼š 1.čÆę€§ļ¼šwhat happenę˜Æē”Ø原形åŠØčƍhappenꞄꈐēš„ēŽ°åœØę—¶ēŸ­čÆ­ļ¼Œč€Œwhat happenedę˜Æē”ØåŠØčƍhappenēš„čæ‡åŽ»å¼happenedꞄꈐēš„čæ‡åŽ» ā€¦ what happened?和what's happening?ēš„åŒŗ别ę˜Æ什么ļ¼Ÿ \"What happen?\"č”Øē¤ŗ发ē”Ÿäŗ†ä»€ä¹ˆļ¼Œē”ØäŗŽčƢ问发ē”Ÿäŗ†ä»€ä¹ˆäŗ‹ęƒ…ļ¼Œé€šåøøē”ØäŗŽčæ‡åŽ»ēš„äŗ‹ä»¶ęˆ–ēŖå‘ęƒ…å†µć€‚ happened\"ę˜Æäø€äøŖå®Œę•“ēš„叄子ļ¼Œē®€å•čæ‡åŽ»ę—¶ę„č”Øē¤ŗčæ‡åŽ»å‘ēš„äŗ‹ęƒ…怂 它ę˜Æäø€äøŖē‹¬ēš„叄子ļ¼Œäøéœ€č¦å…¶ä»–ę–‡ęœ¬ęˆ–äøŠäø‹ę„ę”Æ\"What's happened\"\"what has happened\"ēš„ē¼©å½¢å¼ļ¼Œä½æē”ØēŽ°åœØå®Œęˆč”Øē¤ŗ去发ē”Ÿ ā€¦\nURL: https://duckduckgo.com/?q=What+happened+in+tech+news+this+week?\n\nUser Question: What happened in tech news this week?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", - "prompt.prompt_length": 1122, - "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nhappenedēš„ē”Øę³•å¦‚äø‹ļ¼š 1.happenedę˜Æ äøåŠē‰©åŠØčƍ,作ā€œå‘ē”Ÿā€č§£ę—¶,åÆęŒ‡å¶ē„¶ēš„äŗ‹ę•…,也åÆęŒ‡ęœ‰č®”åˆ’ęˆ–ę— č®”åˆ’å‘ē”Ÿēš„äŗ‹ć€‚äø€čˆ¬ä»„äŗ‹ä»¶ć€ēŽÆå¢ƒć€å½¢åŠæē­‰ä½œäø»čÆ­,č”Øē¤ŗā€œęŸäŗŗ发ē”ŸęŸäŗ‹ā€ę—¶,锻仄介čƍto引å‡ŗā€œ ā€¦ what happenedčæ˜ę˜Æwhat's happened\"What happened\"和\"What's happened\"都ē”ØäŗŽčƢ问...", - "prompt.instruction_following": "web_search_optimized", - "response.answer_length": 501, - "response.answer_completeness": "complete", - "response.response_quality": "medium", - "response.source_citation": 1, - "response.context_utilization": 0.7828125, - "output_response": "Web Search Answer: The web search results do not provide specific information about recent events in tech news for this week. They primarily discuss the grammatical usage of the phrase \"what happened\" and its variations...", - "evaluation.search_effectiveness": 0.2, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 1, - "evaluation.context_relevance": "medium", - "evaluation.overall_quality": "low", - "evaluation.ready_for_production": true, - "web_search.search_strategy": "duckduckgo_api", - "web_search.real_time_data": true, - "web_search.external_sources": 1, - "web_search.information_freshness": "current", - "web_search.knowledge_synthesis": "single_source" - }, - "events": [], - "links": [], - "trace_trace_id": "5d2517e1-220a-429d-9d59-f701bda25eed", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-09 08:26:23.210000000", - "trace_end_time": "2025-10-09 08:26:26.985000000", - "trace_duration_ms": 3775.124, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-09 08:26:28.174000000", - "updated_at": "2025-10-09 08:26:28.174000000" - }, - { - "span_id": "91fdf43c-7170-498e-9035-1e2b11b32b48", - "trace_id": "a81ca3a8-80aa-4c39-876e-8d40ea7a0aef", - "parent_span_id": "a062bef6-d973-489b-91df-5d40a2aef077", - "name": "agent.web_search_evaluation_metrics", - "start_time": "2025-10-09 08:26:15.537000000", - "end_time": "2025-10-09 08:26:15.537000000", - "duration_ms": 0.009000000000000001, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "web_search_evaluation_metrics", - "agent.capabilities": [ - "web_search_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: Tell me about recent developments in machine learning", - "query_length": 53, - "query_type": "web_search_evaluation_query", - "web_search.results_count": 1, - "web_search.context_length": 738, - "web_search.context_synthesized": "Context from 1 web sources", - "web_search.search_effectiveness": 0.2, - "web_search.context_quality": "medium", - "web_search.source_diversity": 1, - "web_search.information_synthesis": "low", - "web_search.external_knowledge_utilization": 0.738, - "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\näø€ć€Say和Tellēš„ē”Øę³• ęˆ‘ä»¬å…ˆå°†say态tell态speak态talkčæ™å››äøŖčÆåˆ†ęˆäø¤ē»„ļ¼Œåˆ†åˆ«äøŗļ¼š ē¬¬ā‘ ē»„ļ¼šsay & tellļ¼Œčæ™äø¤äøŖčƍ之后åÆ仄č”Øē¤ŗ谈čƝēš„ ā€œē»†čŠ‚ā€ć€‚ ē¬¬ā‘”ē»„ļ¼šspeak & talkļ¼Œäøčƒ½č”Øē¤ŗ谈čƝēš„ā€œē»†čŠ‚ā€ć€‚ 而ā€œ ā€¦ tell sb to do sth 和 tell sb do sth 都č”Øē¤ŗ告čÆ‰ęŸäŗŗåšęŸäŗ‹ļ¼Œtell sb to do sth å¼ŗč°ƒå‘ŠčÆ‰ęŸäŗŗåŗ”čƄ做什么具体åŠØä½œęˆ–č”Œäøŗļ¼Œč€Œ tell sb do sth å¼ŗč°ƒå‘ŠčÆ‰ęŸäŗŗę­£åœØčæ›č”Œēš„å…·ä½“č”ŒåŠØꈖēŠ¶ę€ć€‚ 它们åœØé‡Šä¹‰ć€ē”Øę³•ć€ä½æ ā€¦ Feb 3, 2024 Ā· \"tell sb sth\" ē”Øę„č”Øē¤ŗ告čÆ‰ęŸäŗŗäø€äøŖē›“ꎄēš„äæ”ęÆꈖäŗ‹å®žļ¼Œč€Œ \"tell sb about sth\" ꌇēš„ę˜Æå‘ęŸäŗŗ叙čæ°ęˆ–ꏏčæ°å…³äŗŽęŸäŗ‹ēš„čƦē»†ęƒ…å†µć€‚ ē®€å•ę„čÆ“ļ¼Œå‰č€…ꛓē›“ꎄļ¼ŒåŽč€…ꛓ倾向äŗŽęä¾›å…³äŗŽęŸäŗ‹ēš„čƒŒę™Æꈖ ā€¦ tell apart 和 tell fromēš„åŒŗ别åŒŗ别ļ¼šäø€ć€ę„ę€äøåŒć€‚ tell from 分č¾Øļ¼Œč¾Øåˆ«ć€å›ŗ定ē”Øę³•ęœ‰tell A from Bć€‘ä¾‹å„ļ¼šI can tell from your face you are lying ( to me). Aug 23, 2023 Ā· tellēš„čæ‡åŽ»å¼å’Œčæ‡åŽ»åˆ†čƍ都ę˜Ætold怂 čƻ音ļ¼šč‹±[tel]ļ¼Œē¾Ž[tel]怂 释义ļ¼š vt.告čƉļ¼ŒčÆ“ļ¼›č¾Ø别ļ¼›å©å’ļ¼›ę–­å®šć€‚ vi.č®²čæ°ļ¼›å‘Šå‘ļ¼Œę³„åƆļ¼›čÆ†åˆ«ć€‚ 变形ļ¼ščæ‡åŽ»å¼toldļ¼Œčæ‡åŽ»åˆ†čƍtoldļ¼ŒēŽ°åœØ分čƍtellingļ¼Œ ā€¦\nURL: https://duckduckgo.com/?q=Tell+me+about+recent+developments+in+machine+learning\n\nUser Question: Tell me about recent developments in machine learning\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", - "prompt.prompt_length": 1236, - "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\näø€ć€Say和Tellēš„ē”Øę³• ęˆ‘ä»¬å…ˆå°†say态tell态speak态talkčæ™å››äøŖčÆåˆ†ęˆäø¤ē»„ļ¼Œåˆ†åˆ«äøŗļ¼š ē¬¬ā‘ ē»„ļ¼šsay & tellļ¼Œčæ™äø¤äøŖčƍ之后åÆ仄č”Øē¤ŗ谈čƝēš„ ā€œē»†čŠ‚ā€ć€‚ ē¬¬ā‘”ē»„ļ¼šspeak & talkļ¼Œäøčƒ½č”Øē¤ŗ谈čƝēš„ā€œē»†čŠ‚ā€ć€‚ 而ā€œ ā€¦ tell sb to do sth 和 tell sb do sth 都č”Øē¤ŗ告čÆ‰ęŸäŗŗåšęŸäŗ‹ļ¼Œtell ...", - "prompt.instruction_following": "web_search_optimized", - "response.answer_length": 1639, - "response.answer_completeness": "complete", - "response.response_quality": "medium", - "response.source_citation": 1, - "response.context_utilization": 2.220867208672087, - "output_response": "Web Search Answer: The provided web search results do not contain specific information about recent developments in machine learning. Therefore, I cannot provide a detailed answer based on those results.\n\nHowever, I can...", - "evaluation.search_effectiveness": 0.2, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 1, - "evaluation.context_relevance": "medium", - "evaluation.overall_quality": "low", - "evaluation.ready_for_production": true, - "web_search.search_strategy": "duckduckgo_api", - "web_search.real_time_data": true, - "web_search.external_sources": 1, - "web_search.information_freshness": "current", - "web_search.knowledge_synthesis": "single_source" - }, - "events": [], - "links": [], - "trace_trace_id": "a81ca3a8-80aa-4c39-876e-8d40ea7a0aef", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-09 08:26:08.903000000", - "trace_end_time": "2025-10-09 08:26:15.537000000", - "trace_duration_ms": 6633.662, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-09 08:26:16.227000000", - "updated_at": "2025-10-09 08:26:16.227000000" - }, - { - "span_id": "096e6592-23fa-432a-931d-8f39656c7fcc", - "trace_id": "230aad27-f3dd-4968-a45a-3c2f07ac28ed", - "parent_span_id": "41f82a95-620d-440c-84e6-36d6d1b9e079", - "name": "agent.web_search_evaluation_metrics", - "start_time": "2025-10-09 08:26:07.894000000", - "end_time": "2025-10-09 08:26:07.894000000", - "duration_ms": 0.041, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "web_search_evaluation_metrics", - "agent.capabilities": [ - "web_search_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What's the weather like today?", - "query_length": 30, - "query_type": "web_search_evaluation_query", - "web_search.results_count": 1, - "web_search.context_length": 899, - "web_search.context_synthesized": "Context from 1 web sources", - "web_search.search_effectiveness": 0.2, - "web_search.context_quality": "high", - "web_search.source_diversity": 1, - "web_search.information_synthesis": "low", - "web_search.external_knowledge_utilization": 0.899, - "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nLog in to WhatsApp Web for simple, reliable and private messaging on your desktop. Send and receive messages and files with ease, all for free. 4 days ago Ā· Latest weather forecast for France for today's, hourly weather forecast, including today's temperatures in France, wind, rain and more. 2 days ago Ā· Get the latest hourly weather updates for United States today. Detailed forecast including temperature, wind, rain, snow, and UV index. Stay informed about today's weather ā€¦ 2 days ago Ā· Latest weather forecast for Singapore for today's, hourly weather forecast, including today's temperatures in Singapore, wind, rain and more. Aug 25, 2025 Ā· Sing and learn about the weather with the Witty Bunch! šŸŒŸThis fun, repetitive song helps children explore different kinds of weather while practising the que...\nURL: https://duckduckgo.com/?q=What's+the+weather+like+today?\n\nUser Question: What's the weather like today?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", - "prompt.prompt_length": 1374, - "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nLog in to WhatsApp Web for simple, reliable and private messaging on your desktop. Send and receive messages and files with ease, all for free. 4 days ago Ā· Latest weather ...", - "prompt.instruction_following": "web_search_optimized", - "response.answer_length": 376, - "response.answer_completeness": "complete", - "response.response_quality": "medium", - "response.source_citation": 1, - "response.context_utilization": 0.41824249165739713, - "output_response": "Web Search Answer: The provided web search results do not contain specific information about today's weather in any particular location. They mention weather forecasts for various places, including France and Singapore,...", - "evaluation.search_effectiveness": 0.2, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 1, - "evaluation.context_relevance": "high", - "evaluation.overall_quality": "low", - "evaluation.ready_for_production": true, - "web_search.search_strategy": "duckduckgo_api", - "web_search.real_time_data": true, - "web_search.external_sources": 1, - "web_search.information_freshness": "current", - "web_search.knowledge_synthesis": "single_source" - }, - "events": [], - "links": [], - "trace_trace_id": "230aad27-f3dd-4968-a45a-3c2f07ac28ed", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-09 08:26:02.424000000", - "trace_end_time": "2025-10-09 08:26:07.894000000", - "trace_duration_ms": 5470.232, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-09 08:26:08.586000000", - "updated_at": "2025-10-09 08:26:08.586000000" - }, - { - "span_id": "b56248e6-e526-4000-8c10-f5d13cf2e12d", - "trace_id": "83c7dcce-3d89-4da1-8b3f-d419885d4cbc", - "parent_span_id": "63f2a0d5-9016-4483-8ec0-e6aa6b27689f", - "name": "agent.web_search_evaluation_metrics", - "start_time": "2025-10-09 08:26:01.415000000", - "end_time": "2025-10-09 08:26:01.415000000", - "duration_ms": 0.095, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "web_search_evaluation_metrics", - "agent.capabilities": [ - "web_search_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What are the latest AI news today?", - "query_length": 34, - "query_type": "web_search_evaluation_query", - "web_search.results_count": 1, - "web_search.context_length": 981, - "web_search.context_synthesized": "Context from 1 web sources", - "web_search.search_effectiveness": 0.2, - "web_search.context_quality": "high", - "web_search.source_diversity": 1, - "web_search.information_synthesis": "low", - "web_search.external_knowledge_utilization": 0.981, - "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\n1 day ago Ā· Artificial Intelligence: Read latest updates on AI like Google AI, ChatGPT, Google Lamda, Bard chatbot and more along with latest news as AI technology advances and makes ā€¦ 4 days ago Ā· Get the latest AI news, live updates, and expert insights on artificial intelligence, machine learning, deep learning, and more at xix.ai. Handpicked daily for developers, ā€¦ 5 days ago Ā· Jeff Bezos warns that current AI spending resembles an \"industrial bubble,\" where both good and bad ideas receive funding, potentially leading to lost investments. However, he ā€¦ 6 days ago Ā· AI Magazine highlights this weekā€™s top stories, from Microsoft leadership changes to new language and video models from OpenAI and Anthropic 2 days ago Ā· Explore the latest artificial intelligence news with Reuters - from AI breakthroughs and technology trends to regulation, ethics, business and global impact.\nURL: https://duckduckgo.com/?q=What+are+the+latest+AI+news+today?\n\nUser Question: What are the latest AI news today?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", - "prompt.prompt_length": 1460, - "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\n1 day ago Ā· Artificial Intelligence: Read latest updates on AI like Google AI, ChatGPT, Google Lamda, Bard chatbot and more along with latest news as AI technology advances...", - "prompt.instruction_following": "web_search_optimized", - "response.answer_length": 1003, - "response.answer_completeness": "complete", - "response.response_quality": "medium", - "response.source_citation": 1, - "response.context_utilization": 1.0224260958205913, - "output_response": "Web Search Answer: The latest news in artificial intelligence includes several key updates:\n\n1. **AI Spending Concerns**: Jeff Bezos has expressed concerns that the current level of investment in AI resembles an \"indust...", - "evaluation.search_effectiveness": 0.2, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 1, - "evaluation.context_relevance": "high", - "evaluation.overall_quality": "low", - "evaluation.ready_for_production": true, - "web_search.search_strategy": "duckduckgo_api", - "web_search.real_time_data": true, - "web_search.external_sources": 1, - "web_search.information_freshness": "current", - "web_search.knowledge_synthesis": "single_source" - }, - "events": [], - "links": [], - "trace_trace_id": "83c7dcce-3d89-4da1-8b3f-d419885d4cbc", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-09 08:25:56.886000000", - "trace_end_time": "2025-10-09 08:26:01.416000000", - "trace_duration_ms": 4530.08, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-09 08:26:02.122000000", - "updated_at": "2025-10-09 08:26:02.122000000" - } -] \ No newline at end of file diff --git a/noveum_customer_support_bt/split_datasets/agent.web_search_generation_dataset.json b/noveum_customer_support_bt/split_datasets/agent.web_search_generation_dataset.json deleted file mode 100644 index faa3fd1..0000000 --- a/noveum_customer_support_bt/split_datasets/agent.web_search_generation_dataset.json +++ /dev/null @@ -1,490 +0,0 @@ -[ - { - "span_id": "2aca614b-b016-4d56-abe2-43a144dc681d", - "trace_id": "eda4fe22-9a2b-4b73-856b-f4f3309bf719", - "parent_span_id": "0b19befc-5dc0-4b00-9a8b-5f2f2c914f17", - "name": "agent.web_search_generation", - "start_time": "2025-10-09 08:26:46.384000000", - "end_time": "2025-10-09 08:26:48.685000000", - "duration_ms": 2301.089, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "web_search_agent", - "agent.operation": "web_search_generation", - "agent.capabilities": [ - "web_search", - "content_synthesis", - "response_generation" - ], - "agent.id": "noveum_web_search_agent", - "input_query": "What are the latest breakthroughs in artificial intelligence?", - "query_length": 61, - "query_type": "web_search_query", - "output_response": "Web Search Answer: The provided web search results do not contain specific information about the latest breakthroughs in artificial intelligence. To find the most recent advancements in AI, I recommend checking reputabl...", - "web_search.results_count": 1, - "web_search.context_length": 708, - "web_search.response_length": 423, - "web_search.mode": "external_web_search" - }, - "events": [], - "links": [], - "trace_trace_id": "eda4fe22-9a2b-4b73-856b-f4f3309bf719", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-09 08:26:46.384000000", - "trace_end_time": "2025-10-09 08:26:48.685000000", - "trace_duration_ms": 2301.488, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-09 08:26:49.422000000", - "updated_at": "2025-10-09 08:26:49.422000000" - }, - { - "span_id": "03e3ae4d-0107-401d-8ca0-05fc0f28fb24", - "trace_id": "0ffffba1-8a37-443c-8866-d53ffbfa7718", - "parent_span_id": "30644d2a-5ab1-4a73-b4ff-cf9fa478e36b", - "name": "agent.web_search_generation", - "start_time": "2025-10-09 08:26:41.850000000", - "end_time": "2025-10-09 08:26:45.377000000", - "duration_ms": 3527.3050000000003, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "web_search_agent", - "agent.operation": "web_search_generation", - "agent.capabilities": [ - "web_search", - "content_synthesis", - "response_generation" - ], - "agent.id": "noveum_web_search_agent", - "input_query": "What's happening in the software development world today?", - "query_length": 57, - "query_type": "web_search_query", - "output_response": "Web Search Answer: The web search did not yield any specific results regarding current events in the software development world. Therefore, I cannot provide detailed information on what's happening today in that field.\n...", - "web_search.results_count": 1, - "web_search.context_length": 159, - "web_search.response_length": 795, - "web_search.mode": "external_web_search" - }, - "events": [], - "links": [], - "trace_trace_id": "0ffffba1-8a37-443c-8866-d53ffbfa7718", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-09 08:26:41.850000000", - "trace_end_time": "2025-10-09 08:26:45.377000000", - "trace_duration_ms": 3527.831, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-09 08:26:49.422000000", - "updated_at": "2025-10-09 08:26:49.422000000" - }, - { - "span_id": "dfdb9aaf-9925-4a5c-a0c6-2cb00f70d4d3", - "trace_id": "43cdf081-4f01-49cd-b566-dbd1619e6cd2", - "parent_span_id": "962afebb-26d1-45ff-acf1-218163cd6bb8", - "name": "agent.web_search_generation", - "start_time": "2025-10-09 08:26:32.240000000", - "end_time": "2025-10-09 08:26:34.906000000", - "duration_ms": 2665.69, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "web_search_agent", - "agent.operation": "web_search_generation", - "agent.capabilities": [ - "web_search", - "content_synthesis", - "response_generation" - ], - "agent.id": "noveum_web_search_agent", - "input_query": "What's the current status of cryptocurrency markets?", - "query_length": 52, - "query_type": "web_search_query", - "output_response": "Web Search Answer: The web search results did not provide any specific information regarding the current status of cryptocurrency markets. Therefore, I cannot provide an accurate update on market conditions, trends, or ...", - "web_search.results_count": 1, - "web_search.context_length": 154, - "web_search.response_length": 371, - "web_search.mode": "external_web_search" - }, - "events": [], - "links": [], - "trace_trace_id": "43cdf081-4f01-49cd-b566-dbd1619e6cd2", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-09 08:26:32.237000000", - "trace_end_time": "2025-10-09 08:26:34.906000000", - "trace_duration_ms": 2668.841, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-09 08:26:38.618000000", - "updated_at": "2025-10-09 08:26:38.618000000" - }, - { - "span_id": "eefcfdfc-6433-459e-a134-e2647b8418e4", - "trace_id": "9a1983f4-09da-4b53-80e6-38de6878e0e7", - "parent_span_id": "246cf822-4562-4d3a-8a5a-e38e0a0b02a0", - "name": "agent.web_search_generation", - "start_time": "2025-10-09 08:26:27.992000000", - "end_time": "2025-10-09 08:26:31.231000000", - "duration_ms": 3239.058, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "web_search_agent", - "agent.operation": "web_search_generation", - "agent.capabilities": [ - "web_search", - "content_synthesis", - "response_generation" - ], - "agent.id": "noveum_web_search_agent", - "input_query": "What are the latest updates in Python programming?", - "query_length": 50, - "query_type": "web_search_query", - "output_response": "Web Search Answer: The web search results do not provide specific information about the latest updates in Python programming. They focus primarily on news related to China, including geopolitics, economy, and lifestyle,...", - "web_search.results_count": 1, - "web_search.context_length": 646, - "web_search.response_length": 583, - "web_search.mode": "external_web_search" - }, - "events": [], - "links": [], - "trace_trace_id": "9a1983f4-09da-4b53-80e6-38de6878e0e7", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-09 08:26:27.992000000", - "trace_end_time": "2025-10-09 08:26:31.231000000", - "trace_duration_ms": 3239.706, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-09 08:26:33.463000000", - "updated_at": "2025-10-09 08:26:33.463000000" - }, - { - "span_id": "c57b4b54-b9b0-4a4c-b5cd-57a4cfe5301b", - "trace_id": "5d2517e1-220a-429d-9d59-f701bda25eed", - "parent_span_id": "20544f91-8828-4a0e-b253-ed1f8b95b205", - "name": "agent.web_search_generation", - "start_time": "2025-10-09 08:26:23.210000000", - "end_time": "2025-10-09 08:26:26.985000000", - "duration_ms": 3774.711, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "web_search_agent", - "agent.operation": "web_search_generation", - "agent.capabilities": [ - "web_search", - "content_synthesis", - "response_generation" - ], - "agent.id": "noveum_web_search_agent", - "input_query": "What happened in tech news this week?", - "query_length": 37, - "query_type": "web_search_query", - "output_response": "Web Search Answer: The web search results do not provide specific information about recent events in tech news for this week. They primarily discuss the grammatical usage of the phrase \"what happened\" and its variations...", - "web_search.results_count": 1, - "web_search.context_length": 640, - "web_search.response_length": 501, - "web_search.mode": "external_web_search" - }, - "events": [], - "links": [], - "trace_trace_id": "5d2517e1-220a-429d-9d59-f701bda25eed", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-09 08:26:23.210000000", - "trace_end_time": "2025-10-09 08:26:26.985000000", - "trace_duration_ms": 3775.124, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-09 08:26:28.174000000", - "updated_at": "2025-10-09 08:26:28.174000000" - }, - { - "span_id": "a062bef6-d973-489b-91df-5d40a2aef077", - "trace_id": "a81ca3a8-80aa-4c39-876e-8d40ea7a0aef", - "parent_span_id": "acf6a99a-39e0-4f87-a07e-3a61213ab032", - "name": "agent.web_search_generation", - "start_time": "2025-10-09 08:26:08.904000000", - "end_time": "2025-10-09 08:26:15.537000000", - "duration_ms": 6633.127, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "web_search_agent", - "agent.operation": "web_search_generation", - "agent.capabilities": [ - "web_search", - "content_synthesis", - "response_generation" - ], - "agent.id": "noveum_web_search_agent", - "input_query": "Tell me about recent developments in machine learning", - "query_length": 53, - "query_type": "web_search_query", - "output_response": "Web Search Answer: The provided web search results do not contain specific information about recent developments in machine learning. Therefore, I cannot provide a detailed answer based on those results.\n\nHowever, I can...", - "web_search.results_count": 1, - "web_search.context_length": 738, - "web_search.response_length": 1639, - "web_search.mode": "external_web_search" - }, - "events": [], - "links": [], - "trace_trace_id": "a81ca3a8-80aa-4c39-876e-8d40ea7a0aef", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-09 08:26:08.903000000", - "trace_end_time": "2025-10-09 08:26:15.537000000", - "trace_duration_ms": 6633.662, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-09 08:26:16.227000000", - "updated_at": "2025-10-09 08:26:16.227000000" - }, - { - "span_id": "41f82a95-620d-440c-84e6-36d6d1b9e079", - "trace_id": "230aad27-f3dd-4968-a45a-3c2f07ac28ed", - "parent_span_id": "0b47ea07-1eda-4b6a-a183-ea9ef047e431", - "name": "agent.web_search_generation", - "start_time": "2025-10-09 08:26:02.424000000", - "end_time": "2025-10-09 08:26:07.894000000", - "duration_ms": 5469.763000000001, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "web_search_agent", - "agent.operation": "web_search_generation", - "agent.capabilities": [ - "web_search", - "content_synthesis", - "response_generation" - ], - "agent.id": "noveum_web_search_agent", - "input_query": "What's the weather like today?", - "query_length": 30, - "query_type": "web_search_query", - "output_response": "Web Search Answer: The provided web search results do not contain specific information about today's weather in any particular location. They mention weather forecasts for various places, including France and Singapore,...", - "web_search.results_count": 1, - "web_search.context_length": 899, - "web_search.response_length": 376, - "web_search.mode": "external_web_search" - }, - "events": [], - "links": [], - "trace_trace_id": "230aad27-f3dd-4968-a45a-3c2f07ac28ed", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-09 08:26:02.424000000", - "trace_end_time": "2025-10-09 08:26:07.894000000", - "trace_duration_ms": 5470.232, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-09 08:26:08.586000000", - "updated_at": "2025-10-09 08:26:08.586000000" - }, - { - "span_id": "63f2a0d5-9016-4483-8ec0-e6aa6b27689f", - "trace_id": "83c7dcce-3d89-4da1-8b3f-d419885d4cbc", - "parent_span_id": "3866ddb3-b17e-47a1-8b3f-bca1e32a2ae4", - "name": "agent.web_search_generation", - "start_time": "2025-10-09 08:25:56.886000000", - "end_time": "2025-10-09 08:26:01.415000000", - "duration_ms": 4529.669, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "web_search_agent", - "agent.operation": "web_search_generation", - "agent.capabilities": [ - "web_search", - "content_synthesis", - "response_generation" - ], - "agent.id": "noveum_web_search_agent", - "input_query": "What are the latest AI news today?", - "query_length": 34, - "query_type": "web_search_query", - "output_response": "Web Search Answer: The latest news in artificial intelligence includes several key updates:\n\n1. **AI Spending Concerns**: Jeff Bezos has expressed concerns that the current level of investment in AI resembles an \"indust...", - "web_search.results_count": 1, - "web_search.context_length": 981, - "web_search.response_length": 1003, - "web_search.mode": "external_web_search" - }, - "events": [], - "links": [], - "trace_trace_id": "83c7dcce-3d89-4da1-8b3f-d419885d4cbc", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-09 08:25:56.886000000", - "trace_end_time": "2025-10-09 08:26:01.416000000", - "trace_duration_ms": 4530.08, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-09 08:26:02.122000000", - "updated_at": "2025-10-09 08:26:02.122000000" - } -] \ No newline at end of file diff --git a/noveum_customer_support_bt/split_datasets/tool-orchestator_dataset.json b/noveum_customer_support_bt/split_datasets/tool-orchestator_dataset.json deleted file mode 100644 index 6855e0f..0000000 --- a/noveum_customer_support_bt/split_datasets/tool-orchestator_dataset.json +++ /dev/null @@ -1,1162 +0,0 @@ -[ - { - "span_id": "b6b7c1c0-3b7b-4e19-805d-a16beb7ea871", - "trace_id": "eda4fe22-9a2b-4b73-856b-f4f3309bf719", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-09 08:26:46.384000000", - "end_time": "2025-10-09 08:26:48.685000000", - "duration_ms": 2301.412, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What are the latest breakthroughs in artificial intelligence?", - "query.length": 61, - "processing.mode": "Web Search", - "processing.time_seconds": 2.301326274871826, - "processing.response_length": 423, - "processing.sources_count": 1, - "output_response": "Final Answer: The provided web search results do not contain specific information about the latest breakthroughs in artificial intelligence. To find the most recent advancements in AI, I recommend checking reputabl...", - "final_answer_mode": "Web Search", - "query_processed.input_query": "What are the latest breakthroughs in artificial intelligence?", - "query_processed.output_response": "Successfully processed query using Web Search, generated 423 character response", - "query_processed.mode": "Web Search", - "query_processed.processing_time": 2.301326274871826, - "query_processed.response_length": 423 - }, - "events": [], - "links": [], - "trace_trace_id": "eda4fe22-9a2b-4b73-856b-f4f3309bf719", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-09 08:26:46.384000000", - "trace_end_time": "2025-10-09 08:26:48.685000000", - "trace_duration_ms": 2301.488, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-09 08:26:49.422000000", - "updated_at": "2025-10-09 08:26:49.422000000" - }, - { - "span_id": "3045d971-b7ff-46cc-be11-6db7d65dfefd", - "trace_id": "0ffffba1-8a37-443c-8866-d53ffbfa7718", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-09 08:26:41.850000000", - "end_time": "2025-10-09 08:26:45.377000000", - "duration_ms": 3527.75, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What's happening in the software development world today?", - "query.length": 57, - "processing.mode": "Web Search", - "processing.time_seconds": 3.527592897415161, - "processing.response_length": 795, - "processing.sources_count": 1, - "output_response": "Final Answer: The web search did not yield any specific results regarding current events in the software development world. Therefore, I cannot provide detailed information on what's happening today in that field.\n...", - "final_answer_mode": "Web Search", - "query_processed.input_query": "What's happening in the software development world today?", - "query_processed.output_response": "Successfully processed query using Web Search, generated 795 character response", - "query_processed.mode": "Web Search", - "query_processed.processing_time": 3.527592897415161, - "query_processed.response_length": 795 - }, - "events": [], - "links": [], - "trace_trace_id": "0ffffba1-8a37-443c-8866-d53ffbfa7718", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-09 08:26:41.850000000", - "trace_end_time": "2025-10-09 08:26:45.377000000", - "trace_duration_ms": 3527.831, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-09 08:26:49.422000000", - "updated_at": "2025-10-09 08:26:49.422000000" - }, - { - "span_id": "42d4cb1f-17ec-433a-8087-90292c2fa9d9", - "trace_id": "f1f37bd7-0851-4659-b493-b80d3800d920", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-09 08:26:35.915000000", - "end_time": "2025-10-09 08:26:40.841000000", - "duration_ms": 4926.2519999999995, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What are the newest features in cloud computing?", - "query.length": 48, - "processing.mode": "RAG", - "processing.time_seconds": 4.926196813583374, - "processing.response_length": 883, - "processing.sources_count": 5, - "output_response": "Final Answer: The provided context does not specifically mention any new features in cloud computing as a general topic. However, it does highlight several recent enhancements and features related to the Noveum.ai ...", - "final_answer_mode": "RAG", - "query_processed.input_query": "What are the newest features in cloud computing?", - "query_processed.output_response": "Successfully processed query using RAG, generated 883 character response", - "query_processed.mode": "RAG", - "query_processed.processing_time": 4.926196813583374, - "query_processed.response_length": 883 - }, - "events": [], - "links": [], - "trace_trace_id": "f1f37bd7-0851-4659-b493-b80d3800d920", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-09 08:26:35.915000000", - "trace_end_time": "2025-10-09 08:26:40.841000000", - "trace_duration_ms": 4926.342, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-09 08:26:44.053000000", - "updated_at": "2025-10-09 08:26:44.053000000" - }, - { - "span_id": "e22b809d-5320-40b7-b78a-e3e125538a6b", - "trace_id": "43cdf081-4f01-49cd-b566-dbd1619e6cd2", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-09 08:26:32.239000000", - "end_time": "2025-10-09 08:26:34.906000000", - "duration_ms": 2667.1330000000003, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What's the current status of cryptocurrency markets?", - "query.length": 52, - "processing.mode": "Web Search", - "processing.time_seconds": 2.6669082641601562, - "processing.response_length": 371, - "processing.sources_count": 1, - "output_response": "Final Answer: The web search results did not provide any specific information regarding the current status of cryptocurrency markets. Therefore, I cannot provide an accurate update on market conditions, trends, or ...", - "final_answer_mode": "Web Search", - "query_processed.input_query": "What's the current status of cryptocurrency markets?", - "query_processed.output_response": "Successfully processed query using Web Search, generated 371 character response", - "query_processed.mode": "Web Search", - "query_processed.processing_time": 2.6669082641601562, - "query_processed.response_length": 371 - }, - "events": [], - "links": [], - "trace_trace_id": "43cdf081-4f01-49cd-b566-dbd1619e6cd2", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-09 08:26:32.237000000", - "trace_end_time": "2025-10-09 08:26:34.906000000", - "trace_duration_ms": 2668.841, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-09 08:26:38.618000000", - "updated_at": "2025-10-09 08:26:38.618000000" - }, - { - "span_id": "813a4450-1cf8-4d43-8493-eabfa200919b", - "trace_id": "9a1983f4-09da-4b53-80e6-38de6878e0e7", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-09 08:26:27.992000000", - "end_time": "2025-10-09 08:26:31.231000000", - "duration_ms": 3239.601, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What are the latest updates in Python programming?", - "query.length": 50, - "processing.mode": "Web Search", - "processing.time_seconds": 3.2394051551818848, - "processing.response_length": 583, - "processing.sources_count": 1, - "output_response": "Final Answer: The web search results do not provide specific information about the latest updates in Python programming. They focus primarily on news related to China, including geopolitics, economy, and lifestyle,...", - "final_answer_mode": "Web Search", - "query_processed.input_query": "What are the latest updates in Python programming?", - "query_processed.output_response": "Successfully processed query using Web Search, generated 583 character response", - "query_processed.mode": "Web Search", - "query_processed.processing_time": 3.2394051551818848, - "query_processed.response_length": 583 - }, - "events": [], - "links": [], - "trace_trace_id": "9a1983f4-09da-4b53-80e6-38de6878e0e7", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-09 08:26:27.992000000", - "trace_end_time": "2025-10-09 08:26:31.231000000", - "trace_duration_ms": 3239.706, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-09 08:26:33.463000000", - "updated_at": "2025-10-09 08:26:33.463000000" - }, - { - "span_id": "6102352a-833e-493d-adbb-887e2af17918", - "trace_id": "5d2517e1-220a-429d-9d59-f701bda25eed", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-09 08:26:23.210000000", - "end_time": "2025-10-09 08:26:26.985000000", - "duration_ms": 3775.0409999999997, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What happened in tech news this week?", - "query.length": 37, - "processing.mode": "Web Search", - "processing.time_seconds": 3.774940013885498, - "processing.response_length": 501, - "processing.sources_count": 1, - "output_response": "Final Answer: The web search results do not provide specific information about recent events in tech news for this week. They primarily discuss the grammatical usage of the phrase \"what happened\" and its variations...", - "final_answer_mode": "Web Search", - "query_processed.input_query": "What happened in tech news this week?", - "query_processed.output_response": "Successfully processed query using Web Search, generated 501 character response", - "query_processed.mode": "Web Search", - "query_processed.processing_time": 3.774940013885498, - "query_processed.response_length": 501 - }, - "events": [], - "links": [], - "trace_trace_id": "5d2517e1-220a-429d-9d59-f701bda25eed", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-09 08:26:23.210000000", - "trace_end_time": "2025-10-09 08:26:26.985000000", - "trace_duration_ms": 3775.124, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-09 08:26:28.174000000", - "updated_at": "2025-10-09 08:26:28.174000000" - }, - { - "span_id": "2926418a-fe09-4df0-bf19-a527bc24a478", - "trace_id": "52aacb67-c361-4445-9b72-c157f79f47d6", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-09 08:26:16.544000000", - "end_time": "2025-10-09 08:26:22.201000000", - "duration_ms": 5656.852, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What are the current trends in observability tools?", - "query.length": 51, - "processing.mode": "RAG", - "processing.time_seconds": 5.656759023666382, - "processing.response_length": 1074, - "processing.sources_count": 5, - "output_response": "Final Answer: The provided context does not explicitly outline current trends in observability tools. However, it highlights some key aspects of observability that are particularly relevant to AI applications. \n\n1....", - "final_answer_mode": "RAG", - "query_processed.input_query": "What are the current trends in observability tools?", - "query_processed.output_response": "Successfully processed query using RAG, generated 1074 character response", - "query_processed.mode": "RAG", - "query_processed.processing_time": 5.656759023666382, - "query_processed.response_length": 1074 - }, - "events": [], - "links": [], - "trace_trace_id": "52aacb67-c361-4445-9b72-c157f79f47d6", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-09 08:26:16.544000000", - "trace_end_time": "2025-10-09 08:26:22.201000000", - "trace_duration_ms": 5657.012, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-09 08:26:22.904000000", - "updated_at": "2025-10-09 08:26:22.904000000" - }, - { - "span_id": "afe85a1d-13c8-4476-8154-52d58ba83d52", - "trace_id": "a81ca3a8-80aa-4c39-876e-8d40ea7a0aef", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-09 08:26:08.903000000", - "end_time": "2025-10-09 08:26:15.537000000", - "duration_ms": 6633.559, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "Tell me about recent developments in machine learning", - "query.length": 53, - "processing.mode": "Web Search", - "processing.time_seconds": 6.63348126411438, - "processing.response_length": 1639, - "processing.sources_count": 1, - "output_response": "Final Answer: The provided web search results do not contain specific information about recent developments in machine learning. Therefore, I cannot provide a detailed answer based on those results.\n\nHowever, I can...", - "final_answer_mode": "Web Search", - "query_processed.input_query": "Tell me about recent developments in machine learning", - "query_processed.output_response": "Successfully processed query using Web Search, generated 1639 character response", - "query_processed.mode": "Web Search", - "query_processed.processing_time": 6.63348126411438, - "query_processed.response_length": 1639 - }, - "events": [], - "links": [], - "trace_trace_id": "a81ca3a8-80aa-4c39-876e-8d40ea7a0aef", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-09 08:26:08.903000000", - "trace_end_time": "2025-10-09 08:26:15.537000000", - "trace_duration_ms": 6633.662, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-09 08:26:16.227000000", - "updated_at": "2025-10-09 08:26:16.227000000" - }, - { - "span_id": "db8da393-6d72-421e-87dc-96f56b6576e5", - "trace_id": "230aad27-f3dd-4968-a45a-3c2f07ac28ed", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-09 08:26:02.424000000", - "end_time": "2025-10-09 08:26:07.894000000", - "duration_ms": 5470.179, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What's the weather like today?", - "query.length": 30, - "processing.mode": "Web Search", - "processing.time_seconds": 5.469927072525024, - "processing.response_length": 376, - "processing.sources_count": 1, - "output_response": "Final Answer: The provided web search results do not contain specific information about today's weather in any particular location. They mention weather forecasts for various places, including France and Singapore,...", - "final_answer_mode": "Web Search", - "query_processed.input_query": "What's the weather like today?", - "query_processed.output_response": "Successfully processed query using Web Search, generated 376 character response", - "query_processed.mode": "Web Search", - "query_processed.processing_time": 5.469927072525024, - "query_processed.response_length": 376 - }, - "events": [], - "links": [], - "trace_trace_id": "230aad27-f3dd-4968-a45a-3c2f07ac28ed", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-09 08:26:02.424000000", - "trace_end_time": "2025-10-09 08:26:07.894000000", - "trace_duration_ms": 5470.232, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-09 08:26:08.586000000", - "updated_at": "2025-10-09 08:26:08.586000000" - }, - { - "span_id": "a290202d-1b3f-431f-8728-fcb937380f8c", - "trace_id": "83c7dcce-3d89-4da1-8b3f-d419885d4cbc", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-09 08:25:56.886000000", - "end_time": "2025-10-09 08:26:01.416000000", - "duration_ms": 4530.039000000001, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What are the latest AI news today?", - "query.length": 34, - "processing.mode": "Web Search", - "processing.time_seconds": 4.529800176620483, - "processing.response_length": 1003, - "processing.sources_count": 1, - "output_response": "Final Answer: The latest news in artificial intelligence includes several key updates:\n\n1. **AI Spending Concerns**: Jeff Bezos has expressed concerns that the current level of investment in AI resembles an \"indust...", - "final_answer_mode": "Web Search", - "query_processed.input_query": "What are the latest AI news today?", - "query_processed.output_response": "Successfully processed query using Web Search, generated 1003 character response", - "query_processed.mode": "Web Search", - "query_processed.processing_time": 4.529800176620483, - "query_processed.response_length": 1003 - }, - "events": [], - "links": [], - "trace_trace_id": "83c7dcce-3d89-4da1-8b3f-d419885d4cbc", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-09 08:25:56.886000000", - "trace_end_time": "2025-10-09 08:26:01.416000000", - "trace_duration_ms": 4530.08, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-09 08:26:02.122000000", - "updated_at": "2025-10-09 08:26:02.122000000" - }, - { - "span_id": "5fbaef83-95fc-4bed-9695-9d69c37c35e4", - "trace_id": "2218f641-604c-491a-9710-b51a9941b982", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-09 08:25:48.107000000", - "end_time": "2025-10-09 08:25:55.878000000", - "duration_ms": 7770.846, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What are the benefits of using Noveum Trace?", - "query.length": 44, - "processing.mode": "RAG", - "processing.time_seconds": 7.770798206329346, - "processing.response_length": 1460, - "processing.sources_count": 5, - "output_response": "Final Answer: Using Noveum Trace offers several benefits, including:\n\n1. **Real-Time Monitoring**: The Noveum platform provides a real-time dashboard for analyzing traces and performance, with live updates as new t...", - "final_answer_mode": "RAG", - "query_processed.input_query": "What are the benefits of using Noveum Trace?", - "query_processed.output_response": "Successfully processed query using RAG, generated 1460 character response", - "query_processed.mode": "RAG", - "query_processed.processing_time": 7.770798206329346, - "query_processed.response_length": 1460 - }, - "events": [], - "links": [], - "trace_trace_id": "2218f641-604c-491a-9710-b51a9941b982", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-09 08:25:48.107000000", - "trace_end_time": "2025-10-09 08:25:55.878000000", - "trace_duration_ms": 7770.953, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-09 08:25:56.565000000", - "updated_at": "2025-10-09 08:25:56.565000000" - }, - { - "span_id": "b3ac3924-928b-4d48-a2a7-aa749802fef9", - "trace_id": "255fd49c-84b4-4b18-887e-6308a412d535", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-09 08:25:43.830000000", - "end_time": "2025-10-09 08:25:47.099000000", - "duration_ms": 3268.8160000000003, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "How do I configure Noveum for my system?", - "query.length": 40, - "processing.mode": "RAG", - "processing.time_seconds": 3.268723249435425, - "processing.response_length": 613, - "processing.sources_count": 5, - "output_response": "Final Answer: To configure Noveum for your system, you can refer to the SDK Integration Guide available in the Noveum documentation. This guide will help you trace your AI applications with minimal code changes. Th...", - "final_answer_mode": "RAG", - "query_processed.input_query": "How do I configure Noveum for my system?", - "query_processed.output_response": "Successfully processed query using RAG, generated 613 character response", - "query_processed.mode": "RAG", - "query_processed.processing_time": 3.268723249435425, - "query_processed.response_length": 613 - }, - "events": [], - "links": [], - "trace_trace_id": "255fd49c-84b4-4b18-887e-6308a412d535", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-09 08:25:43.830000000", - "trace_end_time": "2025-10-09 08:25:47.099000000", - "trace_duration_ms": 3268.862, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-09 08:25:50.304000000", - "updated_at": "2025-10-09 08:25:50.304000000" - }, - { - "span_id": "b8055ffc-ee7d-46ad-b6a0-84e80f9399e2", - "trace_id": "dc511122-c0b6-415c-9a49-c7b45132dd87", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-09 08:25:38.322000000", - "end_time": "2025-10-09 08:25:42.822000000", - "duration_ms": 4500.335, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What monitoring capabilities does Noveum provide?", - "query.length": 49, - "processing.mode": "RAG", - "processing.time_seconds": 4.5002899169921875, - "processing.response_length": 839, - "processing.sources_count": 5, - "output_response": "Final Answer: Noveum provides comprehensive monitoring capabilities for AI applications, including the ability to monitor, trace, and optimize AI agents across various frameworks such as LangChain, CrewAI, AutoGen,...", - "final_answer_mode": "RAG", - "query_processed.input_query": "What monitoring capabilities does Noveum provide?", - "query_processed.output_response": "Successfully processed query using RAG, generated 839 character response", - "query_processed.mode": "RAG", - "query_processed.processing_time": 4.5002899169921875, - "query_processed.response_length": 839 - }, - "events": [], - "links": [], - "trace_trace_id": "dc511122-c0b6-415c-9a49-c7b45132dd87", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-09 08:25:38.322000000", - "trace_end_time": "2025-10-09 08:25:42.822000000", - "trace_duration_ms": 4500.425, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-09 08:25:45.028000000", - "updated_at": "2025-10-09 08:25:45.028000000" - }, - { - "span_id": "532b9e1f-5c4f-4b41-98bb-e9a1510f0f1c", - "trace_id": "04bebf38-a343-4563-80db-0154bef8d927", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-09 08:25:33.693000000", - "end_time": "2025-10-09 08:25:37.313000000", - "duration_ms": 3619.5789999999997, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "How does Noveum handle agent tracing?", - "query.length": 37, - "processing.mode": "RAG", - "processing.time_seconds": 3.6195271015167236, - "processing.response_length": 691, - "processing.sources_count": 5, - "output_response": "Final Answer: Noveum.ai handles agent tracing through its specialized Multi-Agent Tracing capabilities, which are designed to observe complex workflows and inter-agent communications within multi-agent systems. The...", - "final_answer_mode": "RAG", - "query_processed.input_query": "How does Noveum handle agent tracing?", - "query_processed.output_response": "Successfully processed query using RAG, generated 691 character response", - "query_processed.mode": "RAG", - "query_processed.processing_time": 3.6195271015167236, - "query_processed.response_length": 691 - }, - "events": [], - "links": [], - "trace_trace_id": "04bebf38-a343-4563-80db-0154bef8d927", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-09 08:25:33.693000000", - "trace_end_time": "2025-10-09 08:25:37.313000000", - "trace_duration_ms": 3619.672, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-09 08:25:40.339000000", - "updated_at": "2025-10-09 08:25:40.339000000" - }, - { - "span_id": "14064e3f-f9c9-4bb7-b8cb-1ad52938b075", - "trace_id": "5e043630-6493-42b5-beb8-79faa19bfa37", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-09 08:25:29.901000000", - "end_time": "2025-10-09 08:25:32.685000000", - "duration_ms": 2783.257, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What APIs are available in Noveum platform?", - "query.length": 43, - "processing.mode": "RAG", - "processing.time_seconds": 2.783203125, - "processing.response_length": 270, - "processing.sources_count": 5, - "output_response": "Final Answer: The provided context does not specify the APIs available in the Noveum platform. For detailed information about the APIs, I recommend checking the official Noveum documentation or the SDK Integration ...", - "final_answer_mode": "RAG", - "query_processed.input_query": "What APIs are available in Noveum platform?", - "query_processed.output_response": "Successfully processed query using RAG, generated 270 character response", - "query_processed.mode": "RAG", - "query_processed.processing_time": 2.783203125, - "query_processed.response_length": 270 - }, - "events": [], - "links": [], - "trace_trace_id": "5e043630-6493-42b5-beb8-79faa19bfa37", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-09 08:25:29.901000000", - "trace_end_time": "2025-10-09 08:25:32.685000000", - "trace_duration_ms": 2783.326, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-09 08:25:34.918000000", - "updated_at": "2025-10-09 08:25:34.918000000" - }, - { - "span_id": "a74fe8a9-2654-4b9d-ba87-dc2f86b6466d", - "trace_id": "7da9814d-a2e8-4c4e-b750-68b26bd5fd22", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-09 08:25:20.586000000", - "end_time": "2025-10-09 08:25:28.897000000", - "duration_ms": 8310.113, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "How do I set up observability with Noveum?", - "query.length": 42, - "processing.mode": "RAG", - "processing.time_seconds": 8.31006908416748, - "processing.response_length": 951, - "processing.sources_count": 5, - "output_response": "Final Answer: To set up observability with Noveum, follow these steps:\n\n1. **Sign Up**: Go to [noveum.ai](https://noveum.ai) and create an account.\n2. **Create a Project**: Once signed in, create a new project to o...", - "final_answer_mode": "RAG", - "query_processed.input_query": "How do I set up observability with Noveum?", - "query_processed.output_response": "Successfully processed query using RAG, generated 951 character response", - "query_processed.mode": "RAG", - "query_processed.processing_time": 8.31006908416748, - "query_processed.response_length": 951 - }, - "events": [], - "links": [], - "trace_trace_id": "7da9814d-a2e8-4c4e-b750-68b26bd5fd22", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-09 08:25:20.586000000", - "trace_end_time": "2025-10-09 08:25:28.897000000", - "trace_duration_ms": 8310.205, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-09 08:25:29.588000000", - "updated_at": "2025-10-09 08:25:29.588000000" - }, - { - "span_id": "c700d0f4-f1f8-4339-aa42-a3060ae2ae11", - "trace_id": "16143f74-2831-4753-b33d-ce4b645093c5", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-09 08:25:12.442000000", - "end_time": "2025-10-09 08:25:19.579000000", - "duration_ms": 7137.314, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What features does Noveum Trace offer?", - "query.length": 38, - "processing.mode": "RAG", - "processing.time_seconds": 7.1372880935668945, - "processing.response_length": 1014, - "processing.sources_count": 5, - "output_response": "Final Answer: Noveum Trace offers several key features through its Python SDK (`noveum-trace`) and TypeScript SDK (`@noveum/trace`):\n\n1. **Decorator-based Tracing**: It provides a simple way to trace LLM calls, age...", - "final_answer_mode": "RAG", - "query_processed.input_query": "What features does Noveum Trace offer?", - "query_processed.output_response": "Successfully processed query using RAG, generated 1014 character response", - "query_processed.mode": "RAG", - "query_processed.processing_time": 7.1372880935668945, - "query_processed.response_length": 1014 - }, - "events": [], - "links": [], - "trace_trace_id": "16143f74-2831-4753-b33d-ce4b645093c5", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-09 08:25:12.442000000", - "trace_end_time": "2025-10-09 08:25:19.579000000", - "trace_duration_ms": 7137.402, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-09 08:25:20.290000000", - "updated_at": "2025-10-09 08:25:20.290000000" - }, - { - "span_id": "f7f5981f-d840-4265-8970-d319b876366d", - "trace_id": "fc64e6cc-6739-4256-ac4a-7b80c3028233", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-09 08:25:07.932000000", - "end_time": "2025-10-09 08:25:11.438000000", - "duration_ms": 3506.0159999999996, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What are Noveum's pricing plans?", - "query.length": 32, - "processing.mode": "RAG", - "processing.time_seconds": 3.5059969425201416, - "processing.response_length": 399, - "processing.sources_count": 5, - "output_response": "Final Answer: The provided context does not include specific information about Noveum's pricing plans. However, it mentions that all users will have access to the Observability suite for free, and early users will ...", - "final_answer_mode": "RAG", - "query_processed.input_query": "What are Noveum's pricing plans?", - "query_processed.output_response": "Successfully processed query using RAG, generated 399 character response", - "query_processed.mode": "RAG", - "query_processed.processing_time": 3.5059969425201416, - "query_processed.response_length": 399 - }, - "events": [], - "links": [], - "trace_trace_id": "fc64e6cc-6739-4256-ac4a-7b80c3028233", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-09 08:25:07.932000000", - "trace_end_time": "2025-10-09 08:25:11.438000000", - "trace_duration_ms": 3506.0989999999997, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-09 08:25:12.651000000", - "updated_at": "2025-10-09 08:25:12.651000000" - }, - { - "span_id": "0c7bbe2b-18af-4fdb-a5bf-4a91daa3bf50", - "trace_id": "b7945c49-f584-4c70-972d-536a805d8a31", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-09 08:24:58.518000000", - "end_time": "2025-10-09 08:25:06.920000000", - "duration_ms": 8402.378, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "How do I integrate Noveum Trace in my application?", - "query.length": 50, - "processing.mode": "RAG", - "processing.time_seconds": 8.402259826660156, - "processing.response_length": 1151, - "processing.sources_count": 5, - "output_response": "Final Answer: To integrate Noveum Trace into your application, you can use either the Python SDK (`noveum-trace`) or the TypeScript SDK (`@noveum/trace`). Hereā€™s a quick start guide based on the provided documentat...", - "final_answer_mode": "RAG", - "query_processed.input_query": "How do I integrate Noveum Trace in my application?", - "query_processed.output_response": "Successfully processed query using RAG, generated 1151 character response", - "query_processed.mode": "RAG", - "query_processed.processing_time": 8.402259826660156, - "query_processed.response_length": 1151 - }, - "events": [], - "links": [], - "trace_trace_id": "b7945c49-f584-4c70-972d-536a805d8a31", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-09 08:24:58.518000000", - "trace_end_time": "2025-10-09 08:25:06.920000000", - "trace_duration_ms": 8402.516, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-09 08:25:07.620000000", - "updated_at": "2025-10-09 08:25:07.620000000" - }, - { - "span_id": "dc869742-6797-4016-8061-37530fc7d433", - "trace_id": "f5c40ecf-36c0-45ba-9cc9-dc0329b0324b", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-09 08:24:54.009000000", - "end_time": "2025-10-09 08:24:57.511000000", - "duration_ms": 3502.0229999999997, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What is Noveum and what does it do?", - "query.length": 35, - "processing.mode": "RAG", - "processing.time_seconds": 3.5019419193267822, - "processing.response_length": 611, - "processing.sources_count": 5, - "output_response": "Final Answer: Noveum.ai is a comprehensive tracing and observability platform specifically designed for AI applications, including those powered by large language models (LLMs), retrieval-augmented generation (RAG)...", - "final_answer_mode": "RAG", - "query_processed.input_query": "What is Noveum and what does it do?", - "query_processed.output_response": "Successfully processed query using RAG, generated 611 character response", - "query_processed.mode": "RAG", - "query_processed.processing_time": 3.5019419193267822, - "query_processed.response_length": 611 - }, - "events": [], - "links": [], - "trace_trace_id": "f5c40ecf-36c0-45ba-9cc9-dc0329b0324b", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-09 08:24:54.009000000", - "trace_end_time": "2025-10-09 08:24:57.511000000", - "trace_duration_ms": 3502.062, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-09 08:24:59.716000000", - "updated_at": "2025-10-09 08:24:59.716000000" - } -] \ No newline at end of file diff --git a/noveum_customer_support_bt/traces/traces/dataset.json b/noveum_customer_support_bt/traces/traces/dataset.json deleted file mode 100644 index 5578d29..0000000 --- a/noveum_customer_support_bt/traces/traces/dataset.json +++ /dev/null @@ -1,20904 +0,0 @@ -[ - { - "span_id": "4149abdc-acdc-448c-b9b0-b09e166e461a", - "trace_id": "6fe93abf-1413-49e8-8464-58558e25ab7b", - "parent_span_id": "817c9703-0062-4bd6-b3ee-b50a1121c2d2", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-22 16:58:42.084000000", - "end_time": "2025-10-22 16:58:42.084000000", - "duration_ms": 0.015000000000000001, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What are the latest breakthroughs in artificial intelligence?", - "query_length": 61, - "query_type": "routing_evaluation_query", - "classification.mode": "Web Search", - "classification.rag_keyword_score": 0, - "classification.web_keyword_score": 1, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "complex", - "query.intent": "general_knowledge", - "query.keyword_density": 0.125, - "query.domain_affinity": "general", - "routing.decision": "Routed to Web Search based on analysis", - "routing.rationale": "RAG score: 0, Web score: 1, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "RAG", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.019230769230769232, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: Web Search (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "6fe93abf-1413-49e8-8464-58558e25ab7b", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:58:42.084000000", - "trace_end_time": "2025-10-22 16:58:44.766000000", - "trace_duration_ms": 2681.9289999999996, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:58:42.084000000", - "updated_at": "2025-10-22 16:58:42.084000000" - }, - { - "span_id": "817c9703-0062-4bd6-b3ee-b50a1121c2d2", - "trace_id": "6fe93abf-1413-49e8-8464-58558e25ab7b", - "parent_span_id": "ff171ffc-dc6b-4cb4-9833-64c5622d9941", - "name": "agent.query_routing", - "start_time": "2025-10-22 16:58:42.084000000", - "end_time": "2025-10-22 16:58:44.766000000", - "duration_ms": 2681.799, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What are the latest breakthroughs in artificial intelligence?", - "query_length": 61, - "query_type": "routing_query", - "output_response": "Routed to Web Search for query processing", - "router.classification": "Web Search", - "router.confidence_score": 1, - "router.rag_keyword_score": 0, - "router.web_keyword_score": 1, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "6fe93abf-1413-49e8-8464-58558e25ab7b", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:58:42.084000000", - "trace_end_time": "2025-10-22 16:58:44.766000000", - "trace_duration_ms": 2681.9289999999996, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:58:42.084000000", - "updated_at": "2025-10-22 16:58:42.084000000" - }, - { - "span_id": "ad476d62-d6b5-4142-8771-29855db3f39e", - "trace_id": "6fe93abf-1413-49e8-8464-58558e25ab7b", - "parent_span_id": "817c9703-0062-4bd6-b3ee-b50a1121c2d2", - "name": "agent.web_search_generation", - "start_time": "2025-10-22 16:58:42.084000000", - "end_time": "2025-10-22 16:58:44.766000000", - "duration_ms": 2681.732, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "web_search_agent", - "agent.operation": "web_search_generation", - "agent.capabilities": [ - "web_search", - "content_synthesis", - "response_generation" - ], - "agent.id": "noveum_web_search_agent", - "input_query": "What are the latest breakthroughs in artificial intelligence?", - "query_length": 61, - "query_type": "web_search_query", - "output_response": "Web Search Answer: The web search did not yield specific results regarding the latest breakthroughs in artificial intelligence. However, I can provide a general overview based on knowledge up to October 2023.\n\nRecent br...", - "web_search.results_count": 1, - "web_search.context_length": 163, - "web_search.response_length": 1040, - "web_search.mode": "external_web_search" - }, - "events": [], - "links": [], - "trace_trace_id": "6fe93abf-1413-49e8-8464-58558e25ab7b", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:58:42.084000000", - "trace_end_time": "2025-10-22 16:58:44.766000000", - "trace_duration_ms": 2681.9289999999996, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:58:42.084000000", - "updated_at": "2025-10-22 16:58:42.084000000" - }, - { - "span_id": "ff171ffc-dc6b-4cb4-9833-64c5622d9941", - "trace_id": "6fe93abf-1413-49e8-8464-58558e25ab7b", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-22 16:58:42.084000000", - "end_time": "2025-10-22 16:58:44.766000000", - "duration_ms": 2681.902, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What are the latest breakthroughs in artificial intelligence?", - "query.length": 61, - "processing.mode": "Web Search", - "processing.time_seconds": 2.6818161010742188, - "processing.response_length": 1040, - "processing.sources_count": 1, - "output_response": "Final Answer: The web search did not yield specific results regarding the latest breakthroughs in artificial intelligence. However, I can provide a general overview based on knowledge up to October 2023.\n\nRecent br...", - "final_answer_mode": "Web Search", - "query_processed.input_query": "What are the latest breakthroughs in artificial intelligence?", - "query_processed.output_response": "Successfully processed query using Web Search, generated 1040 character response", - "query_processed.mode": "Web Search", - "query_processed.processing_time": 2.6818161010742188, - "query_processed.response_length": 1040 - }, - "events": [], - "links": [], - "trace_trace_id": "6fe93abf-1413-49e8-8464-58558e25ab7b", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:58:42.084000000", - "trace_end_time": "2025-10-22 16:58:44.766000000", - "trace_duration_ms": 2681.9289999999996, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:58:42.084000000", - "updated_at": "2025-10-22 16:58:42.084000000" - }, - { - "span_id": "467cb806-a6ef-43da-9739-570730dd244b", - "trace_id": "6fe93abf-1413-49e8-8464-58558e25ab7b", - "parent_span_id": "ad476d62-d6b5-4142-8771-29855db3f39e", - "name": "agent.llm_model_execution", - "start_time": "2025-10-22 16:58:42.438000000", - "end_time": "2025-10-22 16:58:44.766000000", - "duration_ms": 2327.706, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What are the latest breakthroughs in artificial intelligence?", - "query_length": 61, - "query_type": "web_search_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 2.3276820182800293, - "model.latency_ms": 2327.6820182800293, - "model.start_time": 1761152322.438459, - "model.end_time": 1761152324.766141, - "model.performance_tier": "medium", - "model.prompt_tokens": 167, - "model.completion_tokens": 260, - "model.total_tokens": 427, - "model.tokens_per_second": 183.44430065903882, - "model.estimated_cost": 0.01281, - "model.efficiency_score": 2.4355971896955504, - "model.response_length": 1040, - "model.response_quality": "high", - "model.output_response": "Model Response: The web search did not yield specific results regarding the latest breakthroughs in artificial intelligence. However, I can provide a general overview based on knowledge up to October 2023.\n\nRecent br...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "6fe93abf-1413-49e8-8464-58558e25ab7b", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:58:42.084000000", - "trace_end_time": "2025-10-22 16:58:44.766000000", - "trace_duration_ms": 2681.9289999999996, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:58:42.084000000", - "updated_at": "2025-10-22 16:58:42.084000000" - }, - { - "span_id": "a80e4596-01ea-44e0-a590-13e347857f2a", - "trace_id": "6fe93abf-1413-49e8-8464-58558e25ab7b", - "parent_span_id": "ad476d62-d6b5-4142-8771-29855db3f39e", - "name": "agent.web_search_evaluation_metrics", - "start_time": "2025-10-22 16:58:44.766000000", - "end_time": "2025-10-22 16:58:44.766000000", - "duration_ms": 0.008, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "web_search_evaluation_metrics", - "agent.capabilities": [ - "web_search_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What are the latest breakthroughs in artificial intelligence?", - "query_length": 61, - "query_type": "web_search_evaluation_query", - "web_search.results_count": 1, - "web_search.context_length": 163, - "web_search.context_synthesized": "Context from 1 web sources", - "web_search.search_effectiveness": 0.2, - "web_search.context_quality": "low", - "web_search.source_diversity": 1, - "web_search.information_synthesis": "low", - "web_search.external_knowledge_utilization": 0.163, - "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What+are+the+latest+breakthroughs+in+artificial+intelligence?\n\nUser Question: What are the latest breakthroughs in artificial intelligence?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", - "prompt.prompt_length": 669, - "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What+are+the+latest+breakthroughs+in+artificial+intelligence?", - "prompt.instruction_following": "web_search_optimized", - "response.answer_length": 1040, - "response.answer_completeness": "complete", - "response.response_quality": "medium", - "response.source_citation": 1, - "response.context_utilization": 6.380368098159509, - "output_response": "Web Search Answer: The web search did not yield specific results regarding the latest breakthroughs in artificial intelligence. However, I can provide a general overview based on knowledge up to October 2023.\n\nRecent br...", - "evaluation.search_effectiveness": 0.2, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 1, - "evaluation.context_relevance": "low", - "evaluation.overall_quality": "low", - "evaluation.ready_for_production": true, - "web_search.search_strategy": "duckduckgo_api", - "web_search.real_time_data": true, - "web_search.external_sources": 1, - "web_search.information_freshness": "current", - "web_search.knowledge_synthesis": "single_source" - }, - "events": [], - "links": [], - "trace_trace_id": "6fe93abf-1413-49e8-8464-58558e25ab7b", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:58:42.084000000", - "trace_end_time": "2025-10-22 16:58:44.766000000", - "trace_duration_ms": 2681.9289999999996, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:58:42.084000000", - "updated_at": "2025-10-22 16:58:42.084000000" - }, - { - "span_id": "409f29f6-21c7-400a-a9e0-11c87c4313cb", - "trace_id": "ec7c6cae-9bbd-47f4-a8ae-2d6d9b5011ba", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-22 16:58:32.801000000", - "end_time": "2025-10-22 16:58:41.077000000", - "duration_ms": 8276.701, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What's happening in the software development world today?", - "query.length": 57, - "processing.mode": "Web Search", - "processing.time_seconds": 8.276506900787354, - "processing.response_length": 1424, - "processing.sources_count": 1, - "output_response": "Final Answer: The web search did not yield any specific results regarding current events in the software development world. Therefore, I cannot provide detailed information on the latest trends, technologies, or ne...", - "final_answer_mode": "Web Search", - "query_processed.input_query": "What's happening in the software development world today?", - "query_processed.output_response": "Successfully processed query using Web Search, generated 1424 character response", - "query_processed.mode": "Web Search", - "query_processed.processing_time": 8.276506900787354, - "query_processed.response_length": 1424 - }, - "events": [], - "links": [], - "trace_trace_id": "ec7c6cae-9bbd-47f4-a8ae-2d6d9b5011ba", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:58:32.800000000", - "trace_end_time": "2025-10-22 16:58:41.077000000", - "trace_duration_ms": 8276.788, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:58:32.800000000", - "updated_at": "2025-10-22 16:58:32.800000000" - }, - { - "span_id": "43389a18-a566-41e7-8db5-f688f651a106", - "trace_id": "ec7c6cae-9bbd-47f4-a8ae-2d6d9b5011ba", - "parent_span_id": "af400e17-8b20-4218-834b-9330ccaa2635", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-22 16:58:32.801000000", - "end_time": "2025-10-22 16:58:32.801000000", - "duration_ms": 0.047, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What's happening in the software development world today?", - "query_length": 57, - "query_type": "routing_evaluation_query", - "classification.mode": "Web Search", - "classification.rag_keyword_score": 0, - "classification.web_keyword_score": 1, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "complex", - "query.intent": "general_knowledge", - "query.keyword_density": 0.125, - "query.domain_affinity": "general", - "routing.decision": "Routed to Web Search based on analysis", - "routing.rationale": "RAG score: 0, Web score: 1, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "RAG", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.019230769230769232, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: Web Search (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "ec7c6cae-9bbd-47f4-a8ae-2d6d9b5011ba", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:58:32.800000000", - "trace_end_time": "2025-10-22 16:58:41.077000000", - "trace_duration_ms": 8276.788, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:58:32.800000000", - "updated_at": "2025-10-22 16:58:32.800000000" - }, - { - "span_id": "6b431bca-af7e-4f7e-aedb-b103d7f47cae", - "trace_id": "ec7c6cae-9bbd-47f4-a8ae-2d6d9b5011ba", - "parent_span_id": "af400e17-8b20-4218-834b-9330ccaa2635", - "name": "agent.web_search_generation", - "start_time": "2025-10-22 16:58:32.801000000", - "end_time": "2025-10-22 16:58:41.077000000", - "duration_ms": 8276.225999999999, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "web_search_agent", - "agent.operation": "web_search_generation", - "agent.capabilities": [ - "web_search", - "content_synthesis", - "response_generation" - ], - "agent.id": "noveum_web_search_agent", - "input_query": "What's happening in the software development world today?", - "query_length": 57, - "query_type": "web_search_query", - "output_response": "Web Search Answer: The web search did not yield any specific results regarding current events in the software development world. Therefore, I cannot provide detailed information on the latest trends, technologies, or ne...", - "web_search.results_count": 1, - "web_search.context_length": 159, - "web_search.response_length": 1424, - "web_search.mode": "external_web_search" - }, - "events": [], - "links": [], - "trace_trace_id": "ec7c6cae-9bbd-47f4-a8ae-2d6d9b5011ba", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:58:32.800000000", - "trace_end_time": "2025-10-22 16:58:41.077000000", - "trace_duration_ms": 8276.788, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:58:32.800000000", - "updated_at": "2025-10-22 16:58:32.800000000" - }, - { - "span_id": "af400e17-8b20-4218-834b-9330ccaa2635", - "trace_id": "ec7c6cae-9bbd-47f4-a8ae-2d6d9b5011ba", - "parent_span_id": "409f29f6-21c7-400a-a9e0-11c87c4313cb", - "name": "agent.query_routing", - "start_time": "2025-10-22 16:58:32.801000000", - "end_time": "2025-10-22 16:58:41.077000000", - "duration_ms": 8276.465999999999, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What's happening in the software development world today?", - "query_length": 57, - "query_type": "routing_query", - "output_response": "Routed to Web Search for query processing", - "router.classification": "Web Search", - "router.confidence_score": 1, - "router.rag_keyword_score": 0, - "router.web_keyword_score": 1, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "ec7c6cae-9bbd-47f4-a8ae-2d6d9b5011ba", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:58:32.800000000", - "trace_end_time": "2025-10-22 16:58:41.077000000", - "trace_duration_ms": 8276.788, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:58:32.800000000", - "updated_at": "2025-10-22 16:58:32.800000000" - }, - { - "span_id": "37b153b2-d02b-4b7e-9229-1ba3eaae44ee", - "trace_id": "ec7c6cae-9bbd-47f4-a8ae-2d6d9b5011ba", - "parent_span_id": "6b431bca-af7e-4f7e-aedb-b103d7f47cae", - "name": "agent.llm_model_execution", - "start_time": "2025-10-22 16:58:33.172000000", - "end_time": "2025-10-22 16:58:41.077000000", - "duration_ms": 7904.722, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What's happening in the software development world today?", - "query_length": 57, - "query_type": "web_search_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 7.904651880264282, - "model.latency_ms": 7904.651880264282, - "model.start_time": 1761152313.172655, - "model.end_time": 1761152321.077307, - "model.performance_tier": "slow", - "model.prompt_tokens": 165, - "model.completion_tokens": 356, - "model.total_tokens": 521, - "model.tokens_per_second": 65.91055594754174, - "model.estimated_cost": 0.01563, - "model.efficiency_score": 2.7332053742802302, - "model.response_length": 1424, - "model.response_quality": "high", - "model.output_response": "Model Response: The web search did not yield any specific results regarding current events in the software development world. Therefore, I cannot provide detailed information on the latest trends, technologies, or ne...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "ec7c6cae-9bbd-47f4-a8ae-2d6d9b5011ba", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:58:32.800000000", - "trace_end_time": "2025-10-22 16:58:41.077000000", - "trace_duration_ms": 8276.788, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:58:32.800000000", - "updated_at": "2025-10-22 16:58:32.800000000" - }, - { - "span_id": "cc941b8e-6625-43e9-a0a9-7328af24f12f", - "trace_id": "ec7c6cae-9bbd-47f4-a8ae-2d6d9b5011ba", - "parent_span_id": "6b431bca-af7e-4f7e-aedb-b103d7f47cae", - "name": "agent.web_search_evaluation_metrics", - "start_time": "2025-10-22 16:58:41.077000000", - "end_time": "2025-10-22 16:58:41.077000000", - "duration_ms": 0.026, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "web_search_evaluation_metrics", - "agent.capabilities": [ - "web_search_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What's happening in the software development world today?", - "query_length": 57, - "query_type": "web_search_evaluation_query", - "web_search.results_count": 1, - "web_search.context_length": 159, - "web_search.context_synthesized": "Context from 1 web sources", - "web_search.search_effectiveness": 0.2, - "web_search.context_quality": "low", - "web_search.source_diversity": 1, - "web_search.information_synthesis": "low", - "web_search.external_knowledge_utilization": 0.159, - "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What's+happening+in+the+software+development+world+today?\n\nUser Question: What's happening in the software development world today?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", - "prompt.prompt_length": 661, - "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What's+happening+in+the+software+development+world+today?", - "prompt.instruction_following": "web_search_optimized", - "response.answer_length": 1424, - "response.answer_completeness": "complete", - "response.response_quality": "medium", - "response.source_citation": 1, - "response.context_utilization": 8.955974842767295, - "output_response": "Web Search Answer: The web search did not yield any specific results regarding current events in the software development world. Therefore, I cannot provide detailed information on the latest trends, technologies, or ne...", - "evaluation.search_effectiveness": 0.2, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 1, - "evaluation.context_relevance": "low", - "evaluation.overall_quality": "low", - "evaluation.ready_for_production": true, - "web_search.search_strategy": "duckduckgo_api", - "web_search.real_time_data": true, - "web_search.external_sources": 1, - "web_search.information_freshness": "current", - "web_search.knowledge_synthesis": "single_source" - }, - "events": [], - "links": [], - "trace_trace_id": "ec7c6cae-9bbd-47f4-a8ae-2d6d9b5011ba", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:58:32.800000000", - "trace_end_time": "2025-10-22 16:58:41.077000000", - "trace_duration_ms": 8276.788, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:58:32.800000000", - "updated_at": "2025-10-22 16:58:32.800000000" - }, - { - "span_id": "3b06d098-6eb7-4bb6-885e-443ddb28cc41", - "trace_id": "343764b8-7ac9-4549-93cf-8abc11ceba6f", - "parent_span_id": "5c2aa1cf-56a8-4fd5-af90-90a94e723bb5", - "name": "agent.query_routing", - "start_time": "2025-10-22 16:58:26.542000000", - "end_time": "2025-10-22 16:58:31.796000000", - "duration_ms": 5253.835, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What are the newest features in cloud computing?", - "query_length": 48, - "query_type": "routing_query", - "output_response": "Routed to RAG for query processing", - "router.classification": "RAG", - "router.confidence_score": 1, - "router.rag_keyword_score": 1, - "router.web_keyword_score": 0, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "343764b8-7ac9-4549-93cf-8abc11ceba6f", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:58:26.542000000", - "trace_end_time": "2025-10-22 16:58:31.796000000", - "trace_duration_ms": 5253.973, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:58:26.542000000", - "updated_at": "2025-10-22 16:58:26.542000000" - }, - { - "span_id": "5c2aa1cf-56a8-4fd5-af90-90a94e723bb5", - "trace_id": "343764b8-7ac9-4549-93cf-8abc11ceba6f", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-22 16:58:26.542000000", - "end_time": "2025-10-22 16:58:31.796000000", - "duration_ms": 5253.913, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What are the newest features in cloud computing?", - "query.length": 48, - "processing.mode": "RAG", - "processing.time_seconds": 5.253868818283081, - "processing.response_length": 837, - "processing.sources_count": 5, - "output_response": "Final Answer: The provided context does not specifically mention any new features in cloud computing as a general topic. However, it does highlight several recent enhancements related to the Noveum.ai platform, whi...", - "final_answer_mode": "RAG", - "query_processed.input_query": "What are the newest features in cloud computing?", - "query_processed.output_response": "Successfully processed query using RAG, generated 837 character response", - "query_processed.mode": "RAG", - "query_processed.processing_time": 5.253868818283081, - "query_processed.response_length": 837 - }, - "events": [], - "links": [], - "trace_trace_id": "343764b8-7ac9-4549-93cf-8abc11ceba6f", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:58:26.542000000", - "trace_end_time": "2025-10-22 16:58:31.796000000", - "trace_duration_ms": 5253.973, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:58:26.542000000", - "updated_at": "2025-10-22 16:58:26.542000000" - }, - { - "span_id": "6b838ed6-f5bb-4bcf-a76b-bac283bf7571", - "trace_id": "343764b8-7ac9-4549-93cf-8abc11ceba6f", - "parent_span_id": "3b06d098-6eb7-4bb6-885e-443ddb28cc41", - "name": "agent.llm-rag", - "start_time": "2025-10-22 16:58:26.542000000", - "end_time": "2025-10-22 16:58:31.796000000", - "duration_ms": 5253.519, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "rag_agent", - "agent.operation": "llm-rag", - "agent.capabilities": [ - "document_retrieval", - "context_generation", - "response_generation" - ], - "agent.id": "noveum_rag_agent", - "input_query": "What are the newest features in cloud computing?", - "query_length": 48, - "query_type": "rag_query", - "output_response": "RAG Answer: The provided context does not specifically mention any new features in cloud computing as a general topic. However, it does highlight several recent enhancements related to the Noveum.ai platform, whi...", - "rag.context_length": 2579, - "rag.sources_count": 5, - "rag.answer_length": 837, - "rag.mode": "retrieval_augmented_generation" - }, - "events": [], - "links": [], - "trace_trace_id": "343764b8-7ac9-4549-93cf-8abc11ceba6f", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:58:26.542000000", - "trace_end_time": "2025-10-22 16:58:31.796000000", - "trace_duration_ms": 5253.973, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:58:26.542000000", - "updated_at": "2025-10-22 16:58:26.542000000" - }, - { - "span_id": "f51d8f1e-eae3-488d-9a96-49ab97377851", - "trace_id": "343764b8-7ac9-4549-93cf-8abc11ceba6f", - "parent_span_id": "3b06d098-6eb7-4bb6-885e-443ddb28cc41", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-22 16:58:26.542000000", - "end_time": "2025-10-22 16:58:26.542000000", - "duration_ms": 0.082, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What are the newest features in cloud computing?", - "query_length": 48, - "query_type": "routing_evaluation_query", - "classification.mode": "RAG", - "classification.rag_keyword_score": 1, - "classification.web_keyword_score": 0, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "documentation", - "query.keyword_density": 0.125, - "query.domain_affinity": "noveum", - "routing.decision": "Routed to RAG based on analysis", - "routing.rationale": "RAG score: 1, Web score: 0, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "Web Search", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.019230769230769232, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: RAG (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "343764b8-7ac9-4549-93cf-8abc11ceba6f", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:58:26.542000000", - "trace_end_time": "2025-10-22 16:58:31.796000000", - "trace_duration_ms": 5253.973, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:58:26.542000000", - "updated_at": "2025-10-22 16:58:26.542000000" - }, - { - "span_id": "93a0fd69-47b6-4e7c-b8ab-8eaa5aeac240", - "trace_id": "343764b8-7ac9-4549-93cf-8abc11ceba6f", - "parent_span_id": "6b838ed6-f5bb-4bcf-a76b-bac283bf7571", - "name": "agent.llm_model_execution", - "start_time": "2025-10-22 16:58:27.411000000", - "end_time": "2025-10-22 16:58:31.373000000", - "duration_ms": 3961.481, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What are the newest features in cloud computing?", - "query_length": 48, - "query_type": "rag_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 3.9613850116729736, - "model.latency_ms": 3961.3850116729736, - "model.start_time": 1761152307.411936, - "model.end_time": 1761152311.373321, - "model.performance_tier": "medium", - "model.prompt_tokens": 780, - "model.completion_tokens": 209, - "model.total_tokens": 989, - "model.tokens_per_second": 249.66015600243944, - "model.estimated_cost": 0.029670000000000002, - "model.efficiency_score": 0.846309403437816, - "model.response_length": 837, - "model.response_quality": "high", - "model.output_response": "Model Response: The provided context does not specifically mention any new features in cloud computing as a general topic. However, it does highlight several recent enhancements related to the Noveum.ai platform, whi...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "343764b8-7ac9-4549-93cf-8abc11ceba6f", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:58:26.542000000", - "trace_end_time": "2025-10-22 16:58:31.796000000", - "trace_duration_ms": 5253.973, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:58:26.542000000", - "updated_at": "2025-10-22 16:58:26.542000000" - }, - { - "span_id": "48cbc1df-f03a-4c49-90e5-19a2047c1c7b", - "trace_id": "343764b8-7ac9-4549-93cf-8abc11ceba6f", - "parent_span_id": "6b838ed6-f5bb-4bcf-a76b-bac283bf7571", - "name": "agent.rag_evaluation_metrics", - "start_time": "2025-10-22 16:58:31.373000000", - "end_time": "2025-10-22 16:58:31.373000000", - "duration_ms": 0.061, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "rag_evaluation_metrics", - "agent.capabilities": [ - "retrieval_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What are the newest features in cloud computing?", - "query_length": 48, - "query_type": "rag_evaluation_query", - "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/changelog):\n- \ud83d\udcc8 Enhanced Dashboard Analytics: Real-time request tracking, latency monitoring, and cost analysis\n- \ud83d\udd0d Improved Logs Interface: Better search functionality, detailed trace views, and enhanced debugging capabilities\n- \ud83d\udc65 Team Management: Advanced member mana...", - "retrieval.context_length": 2579, - "retrieval.sources_count": 5, - "retrieval.context_quality": "high", - "retrieval.effectiveness": 1, - "retrieval.context_utilization": 2.579, - "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/changelog):\n- \ud83d\udcc8 Enhanced Dashboard Analytics: Real-time request tracking, latency monitoring, and cost analysis\n- \ud83d\udd0d Improved Logs Interface: Better search functionality, detailed trace views, and enhanced debugging capabilities\n- \ud83d\udc65 Team Management: Advanced member management with role-based access control and invitation system\n- \ud83d\udee1\ufe0f Security Improvements: Enhanced API key management and secure credential storage\n4 months ago\n- \ud83d\udcca Advanced Metrics Collection: Comprehensive telemetry system with custom metrics ...\n\nSource 2 (https://noveum.ai/en/changelog):\n- \ud83d\udcdd Documentation Updates: Comprehensive guides for integration and best practices\n5 months ago\n- \ud83c\udf89 Platform Foundation: Initial release of Noveum.ai AI observability platform\n- \ud83d\ude80 Core Features: AI Gateway integration, basic evaluation metrics, and monitoring capabilities\n- \ud83d\udcc8 Evaluation Jobs: Automated model evaluation with configurable metrics and reporting\n- \ud83d\udd0c Provider Integrations: Support for major AI providers and custom model deployments...\n\nSource 3 (https://noveum.ai/docs/platform/dashboard):\n- Bulk Export: Download multiple traces simultaneously\n- Comparative Analysis: Compare performance across multiple traces\nReal-Time Updates\n- Live Refresh: Automatic updates as new traces arrive\n- Connection Monitoring: Real-time status of your trace ingestion pipeline\n- Performance Indicators: Live metrics for system health monitoring\nReady to dive deeper? Explore Projects & Environments to organize your AI applications, or check out Team Collaboration to share insights with your team.\nGet Earl...\n\nSource 4 (https://noveum.ai/en/changelog):\nChangelog\nStay up to date with the latest changes in our product.\n2 months ago\n- \ud83c\udfaf Major Platform Evolution: NovaEval Framework - Launched comprehensive AI model evaluation framework with 20+ built-in scorers including accuracy, RAG metrics, conversational scoring, and specialized agent evaluation tools. Features production-grade deployment on Docker and Kubernetes. (GitHub | PyPI Package)\n- \ud83d\ude80 Next-Generation Traces UI - Complete redesign featuring three-pane layout with directory tree navigatio...\n\nSource 5 (https://noveum.ai/en/blog):\n8/12/2025\nDiscover how Noveum.ai provides comprehensive tracing and observability for AI applications, from development debugging to production optimization.\nShashank Agarwal\n3/3/2025\nDiscover how Noveum.ai provides comprehensive tracing and observability for LLM applications, RAG systems, and multi-agent workflows with our powerful Python and TypeScript SDKs.\nShashank Agarwal\n3/2/2025...\n\nUser Question: What are the newest features in cloud computing?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", - "prompt.prompt_length": 3123, - "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/changelog):\n- \ud83d\udcc8 Enhanced Dashboard Analytics: Real-time request tracking, latency monitoring, and cost analysis\n- \ud83d\udd0d Improved Logs Interface: Better search functionality,...", - "prompt.instruction_following": "rag_optimized", - "response.answer_length": 837, - "response.answer_completeness": "complete", - "response.response_quality": "high", - "response.source_citation": 5, - "response.context_utilization": 0.3245443970531214, - "output_response": "RAG Answer: The provided context does not specifically mention any new features in cloud computing as a general topic. However, it does highlight several recent enhancements related to the Noveum.ai platform, whi...", - "evaluation.retrieval_effectiveness": 1, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 5, - "evaluation.context_relevance": "high", - "evaluation.overall_quality": "high", - "evaluation.ready_for_production": true, - "rag.retrieval_strategy": "semantic_similarity", - "rag.vector_search_results": 5, - "rag.context_synthesis": "multi_source", - "rag.document_coverage": 1, - "rag.information_density": 0.3245443970531214 - }, - "events": [], - "links": [], - "trace_trace_id": "343764b8-7ac9-4549-93cf-8abc11ceba6f", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:58:26.542000000", - "trace_end_time": "2025-10-22 16:58:31.796000000", - "trace_duration_ms": 5253.973, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:58:26.542000000", - "updated_at": "2025-10-22 16:58:26.542000000" - }, - { - "span_id": "7384d697-d7bf-4709-8377-ec26b094b550", - "trace_id": "3f7b6a62-8888-4a5d-b12c-37a3829aea49", - "parent_span_id": "e631b22a-a621-4ecc-bb01-f731ca66f3d0", - "name": "agent.query_routing", - "start_time": "2025-10-22 16:58:19.885000000", - "end_time": "2025-10-22 16:58:25.536000000", - "duration_ms": 5651.335999999999, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What's the current status of cryptocurrency markets?", - "query_length": 52, - "query_type": "routing_query", - "output_response": "Routed to Web Search for query processing", - "router.classification": "Web Search", - "router.confidence_score": 1, - "router.rag_keyword_score": 0, - "router.web_keyword_score": 3, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "3f7b6a62-8888-4a5d-b12c-37a3829aea49", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:58:19.885000000", - "trace_end_time": "2025-10-22 16:58:25.536000000", - "trace_duration_ms": 5651.67, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:58:19.885000000", - "updated_at": "2025-10-22 16:58:19.885000000" - }, - { - "span_id": "7abf372d-d9f4-41d1-9e3a-d4f01173fa5e", - "trace_id": "3f7b6a62-8888-4a5d-b12c-37a3829aea49", - "parent_span_id": "7384d697-d7bf-4709-8377-ec26b094b550", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-22 16:58:19.885000000", - "end_time": "2025-10-22 16:58:19.885000000", - "duration_ms": 0.037, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What's the current status of cryptocurrency markets?", - "query_length": 52, - "query_type": "routing_evaluation_query", - "classification.mode": "Web Search", - "classification.rag_keyword_score": 0, - "classification.web_keyword_score": 3, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "complex", - "query.intent": "general_knowledge", - "query.keyword_density": 0.42857142857142855, - "query.domain_affinity": "general", - "routing.decision": "Routed to Web Search based on analysis", - "routing.rationale": "RAG score: 0, Web score: 3, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "RAG", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.057692307692307696, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: Web Search (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "3f7b6a62-8888-4a5d-b12c-37a3829aea49", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:58:19.885000000", - "trace_end_time": "2025-10-22 16:58:25.536000000", - "trace_duration_ms": 5651.67, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:58:19.885000000", - "updated_at": "2025-10-22 16:58:19.885000000" - }, - { - "span_id": "e5d9c3c3-6c18-45d2-a6b6-58bde4d4e461", - "trace_id": "3f7b6a62-8888-4a5d-b12c-37a3829aea49", - "parent_span_id": "7384d697-d7bf-4709-8377-ec26b094b550", - "name": "agent.web_search_generation", - "start_time": "2025-10-22 16:58:19.885000000", - "end_time": "2025-10-22 16:58:25.536000000", - "duration_ms": 5651.161, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "web_search_agent", - "agent.operation": "web_search_generation", - "agent.capabilities": [ - "web_search", - "content_synthesis", - "response_generation" - ], - "agent.id": "noveum_web_search_agent", - "input_query": "What's the current status of cryptocurrency markets?", - "query_length": 52, - "query_type": "web_search_query", - "output_response": "Web Search Answer: The provided web search results do not contain specific information regarding the current status of cryptocurrency markets. To get the latest updates on cryptocurrency prices, trends, and market analy...", - "web_search.results_count": 1, - "web_search.context_length": 251, - "web_search.response_length": 416, - "web_search.mode": "external_web_search" - }, - "events": [], - "links": [], - "trace_trace_id": "3f7b6a62-8888-4a5d-b12c-37a3829aea49", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:58:19.885000000", - "trace_end_time": "2025-10-22 16:58:25.536000000", - "trace_duration_ms": 5651.67, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:58:19.885000000", - "updated_at": "2025-10-22 16:58:19.885000000" - }, - { - "span_id": "e631b22a-a621-4ecc-bb01-f731ca66f3d0", - "trace_id": "3f7b6a62-8888-4a5d-b12c-37a3829aea49", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-22 16:58:19.885000000", - "end_time": "2025-10-22 16:58:25.536000000", - "duration_ms": 5651.588000000001, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What's the current status of cryptocurrency markets?", - "query.length": 52, - "processing.mode": "Web Search", - "processing.time_seconds": 5.651373863220215, - "processing.response_length": 416, - "processing.sources_count": 1, - "output_response": "Final Answer: The provided web search results do not contain specific information regarding the current status of cryptocurrency markets. To get the latest updates on cryptocurrency prices, trends, and market analy...", - "final_answer_mode": "Web Search", - "query_processed.input_query": "What's the current status of cryptocurrency markets?", - "query_processed.output_response": "Successfully processed query using Web Search, generated 416 character response", - "query_processed.mode": "Web Search", - "query_processed.processing_time": 5.651373863220215, - "query_processed.response_length": 416 - }, - "events": [], - "links": [], - "trace_trace_id": "3f7b6a62-8888-4a5d-b12c-37a3829aea49", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:58:19.885000000", - "trace_end_time": "2025-10-22 16:58:25.536000000", - "trace_duration_ms": 5651.67, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:58:19.885000000", - "updated_at": "2025-10-22 16:58:19.885000000" - }, - { - "span_id": "34e2f906-724e-4116-b61f-82eaa6b75959", - "trace_id": "3f7b6a62-8888-4a5d-b12c-37a3829aea49", - "parent_span_id": "e5d9c3c3-6c18-45d2-a6b6-58bde4d4e461", - "name": "agent.llm_model_execution", - "start_time": "2025-10-22 16:58:24.327000000", - "end_time": "2025-10-22 16:58:25.536000000", - "duration_ms": 1208.66, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What's the current status of cryptocurrency markets?", - "query_length": 52, - "query_type": "web_search_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 1.208608865737915, - "model.latency_ms": 1208.608865737915, - "model.start_time": 1761152304.327749, - "model.end_time": 1761152305.5363579, - "model.performance_tier": "fast", - "model.prompt_tokens": 187, - "model.completion_tokens": 104, - "model.total_tokens": 291, - "model.tokens_per_second": 240.77268357809888, - "model.estimated_cost": 0.00873, - "model.efficiency_score": 1.429553264604811, - "model.response_length": 416, - "model.response_quality": "high", - "model.output_response": "Model Response: The provided web search results do not contain specific information regarding the current status of cryptocurrency markets. To get the latest updates on cryptocurrency prices, trends, and market analy...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "3f7b6a62-8888-4a5d-b12c-37a3829aea49", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:58:19.885000000", - "trace_end_time": "2025-10-22 16:58:25.536000000", - "trace_duration_ms": 5651.67, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:58:19.885000000", - "updated_at": "2025-10-22 16:58:19.885000000" - }, - { - "span_id": "5548e02e-2ea8-472c-9c42-f811d6e4d30b", - "trace_id": "3f7b6a62-8888-4a5d-b12c-37a3829aea49", - "parent_span_id": "e5d9c3c3-6c18-45d2-a6b6-58bde4d4e461", - "name": "agent.web_search_evaluation_metrics", - "start_time": "2025-10-22 16:58:25.536000000", - "end_time": "2025-10-22 16:58:25.536000000", - "duration_ms": 0.044, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "web_search_evaluation_metrics", - "agent.capabilities": [ - "web_search_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What's the current status of cryptocurrency markets?", - "query_length": 52, - "query_type": "web_search_evaluation_query", - "web_search.results_count": 1, - "web_search.context_length": 251, - "web_search.context_synthesized": "Context from 1 web sources", - "web_search.search_effectiveness": 0.2, - "web_search.context_quality": "low", - "web_search.source_diversity": 1, - "web_search.information_synthesis": "low", - "web_search.external_knowledge_utilization": 0.251, - "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nLog in to WhatsApp Web for simple, reliable and private messaging on your desktop. Send and receive messages and files with ease, all for \u2026\nURL: https://duckduckgo.com/?q=What's+the+current+status+of+cryptocurrency+markets?\n\nUser Question: What's the current status of cryptocurrency markets?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", - "prompt.prompt_length": 748, - "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nLog in to WhatsApp Web for simple, reliable and private messaging on your desktop. Send and receive messages and files with ease, all for \u2026\nURL: https://duckduckgo.com/?q=W...", - "prompt.instruction_following": "web_search_optimized", - "response.answer_length": 416, - "response.answer_completeness": "complete", - "response.response_quality": "medium", - "response.source_citation": 1, - "response.context_utilization": 1.657370517928287, - "output_response": "Web Search Answer: The provided web search results do not contain specific information regarding the current status of cryptocurrency markets. To get the latest updates on cryptocurrency prices, trends, and market analy...", - "evaluation.search_effectiveness": 0.2, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 1, - "evaluation.context_relevance": "low", - "evaluation.overall_quality": "low", - "evaluation.ready_for_production": true, - "web_search.search_strategy": "duckduckgo_api", - "web_search.real_time_data": true, - "web_search.external_sources": 1, - "web_search.information_freshness": "current", - "web_search.knowledge_synthesis": "single_source" - }, - "events": [], - "links": [], - "trace_trace_id": "3f7b6a62-8888-4a5d-b12c-37a3829aea49", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:58:19.885000000", - "trace_end_time": "2025-10-22 16:58:25.536000000", - "trace_duration_ms": 5651.67, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:58:19.885000000", - "updated_at": "2025-10-22 16:58:19.885000000" - }, - { - "span_id": "a9e34cea-d4bd-478b-abf2-7a3a38dcd40e", - "trace_id": "51f4534f-23e4-4be7-8d6e-e93391d6e0e2", - "parent_span_id": "8e9b2d06-3e11-4d17-9a32-b1529a2c10a4", - "name": "agent.web_search_generation", - "start_time": "2025-10-22 16:58:15.254000000", - "end_time": "2025-10-22 16:58:18.876000000", - "duration_ms": 3621.674, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "web_search_agent", - "agent.operation": "web_search_generation", - "agent.capabilities": [ - "web_search", - "content_synthesis", - "response_generation" - ], - "agent.id": "noveum_web_search_agent", - "input_query": "What are the latest updates in Python programming?", - "query_length": 50, - "query_type": "web_search_query", - "output_response": "Web Search Answer: The provided web search results do not contain specific information about the latest updates in Python programming. They primarily focus on general news coverage and updates from India and around the ...", - "web_search.results_count": 1, - "web_search.context_length": 753, - "web_search.response_length": 580, - "web_search.mode": "external_web_search" - }, - "events": [], - "links": [], - "trace_trace_id": "51f4534f-23e4-4be7-8d6e-e93391d6e0e2", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:58:15.254000000", - "trace_end_time": "2025-10-22 16:58:18.877000000", - "trace_duration_ms": 3622.355, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:58:15.254000000", - "updated_at": "2025-10-22 16:58:15.254000000" - }, - { - "span_id": "8e9b2d06-3e11-4d17-9a32-b1529a2c10a4", - "trace_id": "51f4534f-23e4-4be7-8d6e-e93391d6e0e2", - "parent_span_id": "bda936a0-24f3-4841-af4f-227197e10b9d", - "name": "agent.query_routing", - "start_time": "2025-10-22 16:58:15.254000000", - "end_time": "2025-10-22 16:58:18.876000000", - "duration_ms": 3621.863, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What are the latest updates in Python programming?", - "query_length": 50, - "query_type": "routing_query", - "output_response": "Routed to Web Search for query processing", - "router.classification": "Web Search", - "router.confidence_score": 1, - "router.rag_keyword_score": 0, - "router.web_keyword_score": 2, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "51f4534f-23e4-4be7-8d6e-e93391d6e0e2", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:58:15.254000000", - "trace_end_time": "2025-10-22 16:58:18.877000000", - "trace_duration_ms": 3622.355, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:58:15.254000000", - "updated_at": "2025-10-22 16:58:15.254000000" - }, - { - "span_id": "bda936a0-24f3-4841-af4f-227197e10b9d", - "trace_id": "51f4534f-23e4-4be7-8d6e-e93391d6e0e2", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-22 16:58:15.254000000", - "end_time": "2025-10-22 16:58:18.877000000", - "duration_ms": 3622.271, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What are the latest updates in Python programming?", - "query.length": 50, - "processing.mode": "Web Search", - "processing.time_seconds": 3.6219100952148438, - "processing.response_length": 580, - "processing.sources_count": 1, - "output_response": "Final Answer: The provided web search results do not contain specific information about the latest updates in Python programming. They primarily focus on general news coverage and updates from India and around the ...", - "final_answer_mode": "Web Search", - "query_processed.input_query": "What are the latest updates in Python programming?", - "query_processed.output_response": "Successfully processed query using Web Search, generated 580 character response", - "query_processed.mode": "Web Search", - "query_processed.processing_time": 3.6219100952148438, - "query_processed.response_length": 580 - }, - "events": [], - "links": [], - "trace_trace_id": "51f4534f-23e4-4be7-8d6e-e93391d6e0e2", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:58:15.254000000", - "trace_end_time": "2025-10-22 16:58:18.877000000", - "trace_duration_ms": 3622.355, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:58:15.254000000", - "updated_at": "2025-10-22 16:58:15.254000000" - }, - { - "span_id": "e27e3812-d49d-4f4f-98b0-144cf0ceffc3", - "trace_id": "51f4534f-23e4-4be7-8d6e-e93391d6e0e2", - "parent_span_id": "8e9b2d06-3e11-4d17-9a32-b1529a2c10a4", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-22 16:58:15.254000000", - "end_time": "2025-10-22 16:58:15.254000000", - "duration_ms": 0.047, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What are the latest updates in Python programming?", - "query_length": 50, - "query_type": "routing_evaluation_query", - "classification.mode": "Web Search", - "classification.rag_keyword_score": 0, - "classification.web_keyword_score": 2, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "general_knowledge", - "query.keyword_density": 0.25, - "query.domain_affinity": "general", - "routing.decision": "Routed to Web Search based on analysis", - "routing.rationale": "RAG score: 0, Web score: 2, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "RAG", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.038461538461538464, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: Web Search (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "51f4534f-23e4-4be7-8d6e-e93391d6e0e2", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:58:15.254000000", - "trace_end_time": "2025-10-22 16:58:18.877000000", - "trace_duration_ms": 3622.355, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:58:15.254000000", - "updated_at": "2025-10-22 16:58:15.254000000" - }, - { - "span_id": "b0eea8ec-5cff-4667-bea7-aeb6d0f0daf3", - "trace_id": "51f4534f-23e4-4be7-8d6e-e93391d6e0e2", - "parent_span_id": "a9e34cea-d4bd-478b-abf2-7a3a38dcd40e", - "name": "agent.llm_model_execution", - "start_time": "2025-10-22 16:58:15.568000000", - "end_time": "2025-10-22 16:58:18.876000000", - "duration_ms": 3307.5739999999996, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What are the latest updates in Python programming?", - "query_length": 50, - "query_type": "web_search_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 3.307521104812622, - "model.latency_ms": 3307.521104812622, - "model.start_time": 1761152295.568949, - "model.end_time": 1761152298.87647, - "model.performance_tier": "medium", - "model.prompt_tokens": 312, - "model.completion_tokens": 145, - "model.total_tokens": 457, - "model.tokens_per_second": 138.16994223711538, - "model.estimated_cost": 0.01371, - "model.efficiency_score": 1.2691466083150984, - "model.response_length": 580, - "model.response_quality": "high", - "model.output_response": "Model Response: The provided web search results do not contain specific information about the latest updates in Python programming. They primarily focus on general news coverage and updates from India and around the ...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "51f4534f-23e4-4be7-8d6e-e93391d6e0e2", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:58:15.254000000", - "trace_end_time": "2025-10-22 16:58:18.877000000", - "trace_duration_ms": 3622.355, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:58:15.254000000", - "updated_at": "2025-10-22 16:58:15.254000000" - }, - { - "span_id": "d0bfcbe5-cb6c-4837-bdf3-f2c65be145cd", - "trace_id": "51f4534f-23e4-4be7-8d6e-e93391d6e0e2", - "parent_span_id": "a9e34cea-d4bd-478b-abf2-7a3a38dcd40e", - "name": "agent.web_search_evaluation_metrics", - "start_time": "2025-10-22 16:58:18.876000000", - "end_time": "2025-10-22 16:58:18.876000000", - "duration_ms": 0.04, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "web_search_evaluation_metrics", - "agent.capabilities": [ - "web_search_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What are the latest updates in Python programming?", - "query_length": 50, - "query_type": "web_search_evaluation_query", - "web_search.results_count": 1, - "web_search.context_length": 753, - "web_search.context_synthesized": "Context from 1 web sources", - "web_search.search_effectiveness": 0.2, - "web_search.context_quality": "medium", - "web_search.source_diversity": 1, - "web_search.information_synthesis": "low", - "web_search.external_knowledge_utilization": 0.753, - "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\n3 days ago \u00b7 Latest news headlines from India & around the world. Check out today\u2019s news coverage live with videos & photos on \u2026 6 days ago \u00b7 India News | Latest India News | Read latest and breaking news from India. Today's top India news headlines, news on \u2026 2 days ago \u00b7 Top News Stories of the day, Latest News Headlines, News Specials, Breaking News and Latest India News, World current affairs & \u2026 NDTV.com provides latest news from India and around the world. Get breaking news alerts from India and follow today\u2019s live news updates in \u2026 6 days ago \u00b7 Get Latest entertainment news from Bollywood, Hollywood and regional film industries.\nURL: https://duckduckgo.com/?q=What+are+the+latest+updates+in+Python+programming?\n\nUser Question: What are the latest updates in Python programming?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", - "prompt.prompt_length": 1248, - "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\n3 days ago \u00b7 Latest news headlines from India & around the world. Check out today\u2019s news coverage live with videos & photos on \u2026 6 days ago \u00b7 India News | Latest India News...", - "prompt.instruction_following": "web_search_optimized", - "response.answer_length": 580, - "response.answer_completeness": "complete", - "response.response_quality": "medium", - "response.source_citation": 1, - "response.context_utilization": 0.7702523240371846, - "output_response": "Web Search Answer: The provided web search results do not contain specific information about the latest updates in Python programming. They primarily focus on general news coverage and updates from India and around the ...", - "evaluation.search_effectiveness": 0.2, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 1, - "evaluation.context_relevance": "medium", - "evaluation.overall_quality": "low", - "evaluation.ready_for_production": true, - "web_search.search_strategy": "duckduckgo_api", - "web_search.real_time_data": true, - "web_search.external_sources": 1, - "web_search.information_freshness": "current", - "web_search.knowledge_synthesis": "single_source" - }, - "events": [], - "links": [], - "trace_trace_id": "51f4534f-23e4-4be7-8d6e-e93391d6e0e2", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:58:15.254000000", - "trace_end_time": "2025-10-22 16:58:18.877000000", - "trace_duration_ms": 3622.355, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:58:15.254000000", - "updated_at": "2025-10-22 16:58:15.254000000" - }, - { - "span_id": "5f5916a7-b9ea-4348-861d-db69b25d6b62", - "trace_id": "e30e9ba6-ffd1-4eb9-b7ae-6d19e04343d6", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-22 16:58:11.033000000", - "end_time": "2025-10-22 16:58:14.250000000", - "duration_ms": 3216.77, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What happened in tech news this week?", - "query.length": 37, - "processing.mode": "Web Search", - "processing.time_seconds": 3.2166178226470947, - "processing.response_length": 510, - "processing.sources_count": 1, - "output_response": "Final Answer: The web search results do not provide specific information about recent events in tech news for this week. They primarily discuss the grammatical usage of the word \"happened\" and its variations, such ...", - "final_answer_mode": "Web Search", - "query_processed.input_query": "What happened in tech news this week?", - "query_processed.output_response": "Successfully processed query using Web Search, generated 510 character response", - "query_processed.mode": "Web Search", - "query_processed.processing_time": 3.2166178226470947, - "query_processed.response_length": 510 - }, - "events": [], - "links": [], - "trace_trace_id": "e30e9ba6-ffd1-4eb9-b7ae-6d19e04343d6", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:58:11.033000000", - "trace_end_time": "2025-10-22 16:58:14.250000000", - "trace_duration_ms": 3216.801, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:58:11.033000000", - "updated_at": "2025-10-22 16:58:11.033000000" - }, - { - "span_id": "804613d0-a701-4c22-add1-d315d7a29813", - "trace_id": "e30e9ba6-ffd1-4eb9-b7ae-6d19e04343d6", - "parent_span_id": "5f5916a7-b9ea-4348-861d-db69b25d6b62", - "name": "agent.query_routing", - "start_time": "2025-10-22 16:58:11.033000000", - "end_time": "2025-10-22 16:58:14.250000000", - "duration_ms": 3216.594, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What happened in tech news this week?", - "query_length": 37, - "query_type": "routing_query", - "output_response": "Routed to Web Search for query processing", - "router.classification": "Web Search", - "router.confidence_score": 1, - "router.rag_keyword_score": 0, - "router.web_keyword_score": 2, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "e30e9ba6-ffd1-4eb9-b7ae-6d19e04343d6", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:58:11.033000000", - "trace_end_time": "2025-10-22 16:58:14.250000000", - "trace_duration_ms": 3216.801, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:58:11.033000000", - "updated_at": "2025-10-22 16:58:11.033000000" - }, - { - "span_id": "f6c8d34f-185f-49ba-a5a4-a9d25ccb4fc6", - "trace_id": "e30e9ba6-ffd1-4eb9-b7ae-6d19e04343d6", - "parent_span_id": "804613d0-a701-4c22-add1-d315d7a29813", - "name": "agent.web_search_generation", - "start_time": "2025-10-22 16:58:11.034000000", - "end_time": "2025-10-22 16:58:14.250000000", - "duration_ms": 3216.4959999999996, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "web_search_agent", - "agent.operation": "web_search_generation", - "agent.capabilities": [ - "web_search", - "content_synthesis", - "response_generation" - ], - "agent.id": "noveum_web_search_agent", - "input_query": "What happened in tech news this week?", - "query_length": 37, - "query_type": "web_search_query", - "output_response": "Web Search Answer: The web search results do not provide specific information about recent events in tech news for this week. They primarily discuss the grammatical usage of the word \"happened\" and its variations, such ...", - "web_search.results_count": 1, - "web_search.context_length": 724, - "web_search.response_length": 510, - "web_search.mode": "external_web_search" - }, - "events": [], - "links": [], - "trace_trace_id": "e30e9ba6-ffd1-4eb9-b7ae-6d19e04343d6", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:58:11.033000000", - "trace_end_time": "2025-10-22 16:58:14.250000000", - "trace_duration_ms": 3216.801, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:58:11.033000000", - "updated_at": "2025-10-22 16:58:11.033000000" - }, - { - "span_id": "11d68082-e7b8-4bbf-833a-79e296c31eda", - "trace_id": "e30e9ba6-ffd1-4eb9-b7ae-6d19e04343d6", - "parent_span_id": "804613d0-a701-4c22-add1-d315d7a29813", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-22 16:58:11.034000000", - "end_time": "2025-10-22 16:58:11.034000000", - "duration_ms": 0.017, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What happened in tech news this week?", - "query_length": 37, - "query_type": "routing_evaluation_query", - "classification.mode": "Web Search", - "classification.rag_keyword_score": 0, - "classification.web_keyword_score": 2, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "general_knowledge", - "query.keyword_density": 0.2857142857142857, - "query.domain_affinity": "general", - "routing.decision": "Routed to Web Search based on analysis", - "routing.rationale": "RAG score: 0, Web score: 2, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "RAG", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.038461538461538464, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: Web Search (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "e30e9ba6-ffd1-4eb9-b7ae-6d19e04343d6", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:58:11.033000000", - "trace_end_time": "2025-10-22 16:58:14.250000000", - "trace_duration_ms": 3216.801, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:58:11.033000000", - "updated_at": "2025-10-22 16:58:11.033000000" - }, - { - "span_id": "420b4ca1-c9f1-488f-ac2c-c53766717634", - "trace_id": "e30e9ba6-ffd1-4eb9-b7ae-6d19e04343d6", - "parent_span_id": "f6c8d34f-185f-49ba-a5a4-a9d25ccb4fc6", - "name": "agent.llm_model_execution", - "start_time": "2025-10-22 16:58:11.468000000", - "end_time": "2025-10-22 16:58:14.250000000", - "duration_ms": 2781.777, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What happened in tech news this week?", - "query_length": 37, - "query_type": "web_search_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 2.781723976135254, - "model.latency_ms": 2781.723976135254, - "model.start_time": 1761152291.468626, - "model.end_time": 1761152294.25035, - "model.performance_tier": "medium", - "model.prompt_tokens": 301, - "model.completion_tokens": 127, - "model.total_tokens": 428, - "model.tokens_per_second": 153.8614196346811, - "model.estimated_cost": 0.01284, - "model.efficiency_score": 1.191588785046729, - "model.response_length": 510, - "model.response_quality": "high", - "model.output_response": "Model Response: The web search results do not provide specific information about recent events in tech news for this week. They primarily discuss the grammatical usage of the word \"happened\" and its variations, such ...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "e30e9ba6-ffd1-4eb9-b7ae-6d19e04343d6", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:58:11.033000000", - "trace_end_time": "2025-10-22 16:58:14.250000000", - "trace_duration_ms": 3216.801, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:58:11.033000000", - "updated_at": "2025-10-22 16:58:11.033000000" - }, - { - "span_id": "56b8bf09-8ebf-44e1-b01f-daff23d3c42c", - "trace_id": "e30e9ba6-ffd1-4eb9-b7ae-6d19e04343d6", - "parent_span_id": "f6c8d34f-185f-49ba-a5a4-a9d25ccb4fc6", - "name": "agent.web_search_evaluation_metrics", - "start_time": "2025-10-22 16:58:14.250000000", - "end_time": "2025-10-22 16:58:14.250000000", - "duration_ms": 0.060000000000000005, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "web_search_evaluation_metrics", - "agent.capabilities": [ - "web_search_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What happened in tech news this week?", - "query_length": 37, - "query_type": "web_search_evaluation_query", - "web_search.results_count": 1, - "web_search.context_length": 724, - "web_search.context_synthesized": "Context from 1 web sources", - "web_search.search_effectiveness": 0.2, - "web_search.context_quality": "medium", - "web_search.source_diversity": 1, - "web_search.information_synthesis": "low", - "web_search.external_knowledge_utilization": 0.724, - "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nhappened\u7684\u7528\u6cd5\u5982\u4e0b\uff1a 1.happened\u662f \u4e0d\u53ca\u7269\u52a8\u8bcd,\u4f5c\u201c\u53d1\u751f\u201d\u89e3\u65f6,\u53ef\u6307\u5076\u7136\u7684\u4e8b\u6545,\u4e5f\u53ef\u6307\u6709\u8ba1\u5212\u6216\u65e0\u8ba1\u5212\u53d1\u751f\u7684\u4e8b\u3002\u4e00\u822c\u4ee5\u4e8b\u4ef6\u3001\u73af\u5883\u3001\u5f62\u52bf\u7b49\u4f5c\u4e3b\u8bed,\u8868\u793a\u201c\u67d0\u4eba\u53d1\u751f\u67d0\u4e8b\u201d\u65f6,\u987b\u4ee5\u4ecb\u8bcdto\u5f15\u51fa\u201c\u67d0\u4eba\u201d\u3002 2. \u2026 what happened\u8fd8\u662fwhat's happened\"What happened\"\u548c\"What's happened\"\u90fd\u7528\u4e8e\u8be2\u95ee\u8fc7\u53bb\u53d1\u751f\u4e86\u4ec0\u4e48\u4e8b\u60c5\uff0c\u4e3b\u8981\u7684\u533a\u522b\u5728\u4e8e\u8bed\u6cd5\u548c\u4f7f\u7528\u65b9\u5f0f\u3002\"What happened\"\u662f\u4e00\u4e2a\u8fc7\u53bb\u65f6\u6001\u7684\u7b80\u5355 \u63a5\u4e0b\u6765\u8ba9\u6211\u4eec\u770b\u4e0bwhat happen\u548cwhat happened\u7684\u7528\u6cd5\u533a\u522b\uff1a 1.\u8bcd\u6027\uff1awhat happen\u662f\u7528\u539f\u5f62\u52a8\u8bcdhappen\u6784\u6210\u7684\u73b0\u5728\u65f6\u77ed\u8bed\uff0c\u800cwhat happened\u662f\u7528\u52a8\u8bcdhappen\u7684\u8fc7\u53bb\u5f0fhappened\u6784\u6210\u7684\u8fc7\u53bb\u65f6\u77ed \u2026 What happened \u548c What's happened\u4e24\u8005\u7684\u533a\u522b\uff1a \u662f\u95ee What happened\uff1f\u53ea\u662f\u60f3\u77e5\u9053\u8fc7\u53bb\u6240\u53d1\u751f\u7684\u4e8b\u60c5\uff0c \u800c\u5982\u679c\u7528What's happened\uff1f\u5219\u5f3a\u8c03\u5bf9\u73b0\u5728\u7684\u5f71\u54cd\u3002\uff0c\u5373\u60f3\u77e5\u9053\u8fc7\u53bb\u6240\u53d1\u751f\u7684\u4ec0\u4e48\u4e8b\u5bfc\u81f4\u4e86\u8bf4 \u2026 it so happened that\u7684\u53e5\u5f0fhappen\u8868\u793a\u201c\u78b0\u5de7\u6216\u6070\u5de7\u53d1\u751f\u67d0\u4e8b\u201d\u65f6,\u53ef\u7528\u201cIt happens / happened that...\u201d\u8fd9\u4e00\u7ed3\u6784\u6765\u8868\u8fbe.\u4f8b\u5982\uff1a It happened that Brian and Peter were at home that day.\u78b0\u5de7\u90a3\u5929\u5e03\u83b1\u6069\u548c \u2026\nURL: https://duckduckgo.com/?q=What+happened+in+tech+news+this+week?\n\nUser Question: What happened in tech news this week?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", - "prompt.prompt_length": 1206, - "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nhappened\u7684\u7528\u6cd5\u5982\u4e0b\uff1a 1.happened\u662f \u4e0d\u53ca\u7269\u52a8\u8bcd,\u4f5c\u201c\u53d1\u751f\u201d\u89e3\u65f6,\u53ef\u6307\u5076\u7136\u7684\u4e8b\u6545,\u4e5f\u53ef\u6307\u6709\u8ba1\u5212\u6216\u65e0\u8ba1\u5212\u53d1\u751f\u7684\u4e8b\u3002\u4e00\u822c\u4ee5\u4e8b\u4ef6\u3001\u73af\u5883\u3001\u5f62\u52bf\u7b49\u4f5c\u4e3b\u8bed,\u8868\u793a\u201c\u67d0\u4eba\u53d1\u751f\u67d0\u4e8b\u201d\u65f6,\u987b\u4ee5\u4ecb\u8bcdto\u5f15\u51fa\u201c\u67d0\u4eba\u201d\u3002 2. \u2026 what happened\u8fd8\u662fwhat's happened\"What happened\"\u548c\"What's happene...", - "prompt.instruction_following": "web_search_optimized", - "response.answer_length": 510, - "response.answer_completeness": "complete", - "response.response_quality": "medium", - "response.source_citation": 1, - "response.context_utilization": 0.7044198895027625, - "output_response": "Web Search Answer: The web search results do not provide specific information about recent events in tech news for this week. They primarily discuss the grammatical usage of the word \"happened\" and its variations, such ...", - "evaluation.search_effectiveness": 0.2, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 1, - "evaluation.context_relevance": "medium", - "evaluation.overall_quality": "low", - "evaluation.ready_for_production": true, - "web_search.search_strategy": "duckduckgo_api", - "web_search.real_time_data": true, - "web_search.external_sources": 1, - "web_search.information_freshness": "current", - "web_search.knowledge_synthesis": "single_source" - }, - "events": [], - "links": [], - "trace_trace_id": "e30e9ba6-ffd1-4eb9-b7ae-6d19e04343d6", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:58:11.033000000", - "trace_end_time": "2025-10-22 16:58:14.250000000", - "trace_duration_ms": 3216.801, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:58:11.033000000", - "updated_at": "2025-10-22 16:58:11.033000000" - }, - { - "span_id": "82eb19e8-f6cf-43fd-ba56-640a6fd3ed0e", - "trace_id": "45efae74-ee25-4004-a509-6e0ff628a0cd", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-22 16:58:04.883000000", - "end_time": "2025-10-22 16:58:10.029000000", - "duration_ms": 5146.146, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What are the current trends in observability tools?", - "query.length": 51, - "processing.mode": "RAG", - "processing.time_seconds": 5.146108150482178, - "processing.response_length": 815, - "processing.sources_count": 5, - "output_response": "Final Answer: The provided context does not explicitly outline current trends in observability tools. However, it highlights the unique challenges faced by AI applications that traditional monitoring tools cannot a...", - "final_answer_mode": "RAG", - "query_processed.input_query": "What are the current trends in observability tools?", - "query_processed.output_response": "Successfully processed query using RAG, generated 815 character response", - "query_processed.mode": "RAG", - "query_processed.processing_time": 5.146108150482178, - "query_processed.response_length": 815 - }, - "events": [], - "links": [], - "trace_trace_id": "45efae74-ee25-4004-a509-6e0ff628a0cd", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:58:04.883000000", - "trace_end_time": "2025-10-22 16:58:10.029000000", - "trace_duration_ms": 5146.244000000001, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:58:04.883000000", - "updated_at": "2025-10-22 16:58:04.883000000" - }, - { - "span_id": "e81617c6-c2a1-4b08-a5fc-dde3439fe2f4", - "trace_id": "45efae74-ee25-4004-a509-6e0ff628a0cd", - "parent_span_id": "8001ba0e-95c7-42da-9087-603a38e1a500", - "name": "agent.llm-rag", - "start_time": "2025-10-22 16:58:04.883000000", - "end_time": "2025-10-22 16:58:10.029000000", - "duration_ms": 5145.655, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "rag_agent", - "agent.operation": "llm-rag", - "agent.capabilities": [ - "document_retrieval", - "context_generation", - "response_generation" - ], - "agent.id": "noveum_rag_agent", - "input_query": "What are the current trends in observability tools?", - "query_length": 51, - "query_type": "rag_query", - "output_response": "RAG Answer: The provided context does not explicitly outline current trends in observability tools. However, it highlights the unique challenges faced by AI applications that traditional monitoring tools cannot a...", - "rag.context_length": 2943, - "rag.sources_count": 5, - "rag.answer_length": 815, - "rag.mode": "retrieval_augmented_generation" - }, - "events": [], - "links": [], - "trace_trace_id": "45efae74-ee25-4004-a509-6e0ff628a0cd", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:58:04.883000000", - "trace_end_time": "2025-10-22 16:58:10.029000000", - "trace_duration_ms": 5146.244000000001, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:58:04.883000000", - "updated_at": "2025-10-22 16:58:04.883000000" - }, - { - "span_id": "8001ba0e-95c7-42da-9087-603a38e1a500", - "trace_id": "45efae74-ee25-4004-a509-6e0ff628a0cd", - "parent_span_id": "82eb19e8-f6cf-43fd-ba56-640a6fd3ed0e", - "name": "agent.query_routing", - "start_time": "2025-10-22 16:58:04.883000000", - "end_time": "2025-10-22 16:58:10.029000000", - "duration_ms": 5146.036999999999, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What are the current trends in observability tools?", - "query_length": 51, - "query_type": "routing_query", - "output_response": "Routed to RAG for query processing", - "router.classification": "RAG", - "router.confidence_score": 0.3333333333333333, - "router.rag_keyword_score": 2, - "router.web_keyword_score": 1, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "45efae74-ee25-4004-a509-6e0ff628a0cd", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:58:04.883000000", - "trace_end_time": "2025-10-22 16:58:10.029000000", - "trace_duration_ms": 5146.244000000001, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:58:04.883000000", - "updated_at": "2025-10-22 16:58:04.883000000" - }, - { - "span_id": "79b7d138-05e1-4f18-8b63-dcfe9732180b", - "trace_id": "45efae74-ee25-4004-a509-6e0ff628a0cd", - "parent_span_id": "8001ba0e-95c7-42da-9087-603a38e1a500", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-22 16:58:04.883000000", - "end_time": "2025-10-22 16:58:04.883000000", - "duration_ms": 0.051, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What are the current trends in observability tools?", - "query_length": 51, - "query_type": "routing_evaluation_query", - "classification.mode": "RAG", - "classification.rag_keyword_score": 2, - "classification.web_keyword_score": 1, - "classification.confidence_score": 0.3333333333333333, - "classification.confidence_level": "medium", - "classification.method": "keyword_based", - "query.complexity": "complex", - "query.intent": "documentation", - "query.keyword_density": 0.375, - "query.domain_affinity": "noveum", - "routing.decision": "Routed to RAG based on analysis", - "routing.rationale": "RAG score: 2, Web score: 1, Confidence: 0.33", - "routing.expected_performance": "medium", - "routing.alternative_mode": "Web Search", - "routing.decision_confidence": 0.3333333333333333, - "evaluation.routing_accuracy": "medium", - "evaluation.keyword_coverage": 0.057692307692307696, - "evaluation.query_understanding": "ambiguous", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: RAG (Confidence: 0.33)" - }, - "events": [], - "links": [], - "trace_trace_id": "45efae74-ee25-4004-a509-6e0ff628a0cd", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:58:04.883000000", - "trace_end_time": "2025-10-22 16:58:10.029000000", - "trace_duration_ms": 5146.244000000001, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:58:04.883000000", - "updated_at": "2025-10-22 16:58:04.883000000" - }, - { - "span_id": "28c233a0-d42a-4200-86b0-dfbd2d5be06e", - "trace_id": "45efae74-ee25-4004-a509-6e0ff628a0cd", - "parent_span_id": "e81617c6-c2a1-4b08-a5fc-dde3439fe2f4", - "name": "agent.llm_model_execution", - "start_time": "2025-10-22 16:58:05.635000000", - "end_time": "2025-10-22 16:58:09.222000000", - "duration_ms": 3586.924, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What are the current trends in observability tools?", - "query_length": 51, - "query_type": "rag_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 3.5868873596191406, - "model.latency_ms": 3586.8873596191406, - "model.start_time": 1761152285.6352398, - "model.end_time": 1761152289.2221272, - "model.performance_tier": "medium", - "model.prompt_tokens": 872, - "model.completion_tokens": 203, - "model.total_tokens": 1075, - "model.tokens_per_second": 299.7027484337129, - "model.estimated_cost": 0.03225, - "model.efficiency_score": 0.7581395348837209, - "model.response_length": 815, - "model.response_quality": "high", - "model.output_response": "Model Response: The provided context does not explicitly outline current trends in observability tools. However, it highlights the unique challenges faced by AI applications that traditional monitoring tools cannot a...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "45efae74-ee25-4004-a509-6e0ff628a0cd", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:58:04.883000000", - "trace_end_time": "2025-10-22 16:58:10.029000000", - "trace_duration_ms": 5146.244000000001, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:58:04.883000000", - "updated_at": "2025-10-22 16:58:04.883000000" - }, - { - "span_id": "024c05b6-edf9-40ef-931b-9660d48b4948", - "trace_id": "45efae74-ee25-4004-a509-6e0ff628a0cd", - "parent_span_id": "e81617c6-c2a1-4b08-a5fc-dde3439fe2f4", - "name": "agent.rag_evaluation_metrics", - "start_time": "2025-10-22 16:58:09.222000000", - "end_time": "2025-10-22 16:58:09.222000000", - "duration_ms": 0.020999999999999998, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "rag_evaluation_metrics", - "agent.capabilities": [ - "retrieval_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What are the current trends in observability tools?", - "query_length": 51, - "query_type": "rag_evaluation_query", - "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/blog/from-logs-to-intelligent-choices-inside-noveum-ais-evaluation-process):\nFrom Development to Production - Inside Noveum.ai's AI Observability Platform\nShashank Agarwal\n3/3/2025\nIntroduction\nEvery AI application tells a story\u2014but most developers never get to hear it...", - "retrieval.context_length": 2943, - "retrieval.sources_count": 5, - "retrieval.context_quality": "high", - "retrieval.effectiveness": 1, - "retrieval.context_utilization": 2.943, - "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/blog/from-logs-to-intelligent-choices-inside-noveum-ais-evaluation-process):\nFrom Development to Production - Inside Noveum.ai's AI Observability Platform\nShashank Agarwal\n3/3/2025\nIntroduction\nEvery AI application tells a story\u2014but most developers never get to hear it. When your RAG pipeline returns irrelevant results, when your multi-agent system gets stuck in loops, or when your LLM costs suddenly spike, you're left guessing what went wrong and where.\nThat's exactly the challenge Noveum.ai solves. Rather than flying blind, our platform provides comprehensive tracing a...\n\nSource 2 (https://noveum.ai/docs/getting-started/tracing-concepts):\n- Explore Framework Integrations - Framework-specific guidance\n- Learn Advanced Patterns - Custom instrumentation techniques\n- Master the Dashboard - Analyze your traces effectively\nRemember: Good observability is not about collecting all possible data, but about collecting the right data that helps you understand, debug, and optimize your AI applications.\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to O...\n\nSource 3 (https://noveum.ai/en/docs/getting-started/tracing-concepts):\n- Explore Framework Integrations - Framework-specific guidance\n- Learn Advanced Patterns - Custom instrumentation techniques\n- Master the Dashboard - Analyze your traces effectively\nRemember: Good observability is not about collecting all possible data, but about collecting the right data that helps you understand, debug, and optimize your AI applications.\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to O...\n\nSource 4 (https://noveum.ai/en/blog/from-logs-to-intelligent-choices-inside-noveum-ais-evaluation-process):\nThe AI Observability Challenge\nWhy Traditional Monitoring Falls Short\nTraditional application monitoring tools weren't built for AI applications. They can tell you if your API is responding, but they can't answer the questions that matter most for AI systems:\n- Why did my RAG pipeline retrieve irrelevant documents?\n- Which LLM calls are driving my costs?\n- How are my agents communicating with each other?\n- What's causing hallucinations in my responses?\n- Why is my embedding generation so slow?\nW...\n\nSource 5 (https://noveum.ai/en/docs/advanced/rag-observability):\n\ud83d\udd27 Best Practices for RAG Observability\n1. Comprehensive Pipeline Tracing\n2. Quality Monitoring\n3. A/B Testing for RAG Components\n\ud83c\udfaf Advanced RAG Patterns\nMulti-Modal RAG\nConversational RAG\nRAG observability with Noveum.ai provides the deep insights needed to build, optimize, and scale retrieval-augmented generation systems. By implementing comprehensive tracing across retrieval, context preparation, and generation stages, you can ensure your RAG pipeline delivers accurate, relevant, and cost-effe...\n\nUser Question: What are the current trends in observability tools?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", - "prompt.prompt_length": 3490, - "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/blog/from-logs-to-intelligent-choices-inside-noveum-ais-evaluation-process):\nFrom Development to Production - Inside Noveum.ai's AI Observability Platform\nShashank Agarw...", - "prompt.instruction_following": "rag_optimized", - "response.answer_length": 815, - "response.answer_completeness": "complete", - "response.response_quality": "high", - "response.source_citation": 5, - "response.context_utilization": 0.2769283044512402, - "output_response": "RAG Answer: The provided context does not explicitly outline current trends in observability tools. However, it highlights the unique challenges faced by AI applications that traditional monitoring tools cannot a...", - "evaluation.retrieval_effectiveness": 1, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 5, - "evaluation.context_relevance": "high", - "evaluation.overall_quality": "high", - "evaluation.ready_for_production": true, - "rag.retrieval_strategy": "semantic_similarity", - "rag.vector_search_results": 5, - "rag.context_synthesis": "multi_source", - "rag.document_coverage": 1, - "rag.information_density": 0.2769283044512402 - }, - "events": [], - "links": [], - "trace_trace_id": "45efae74-ee25-4004-a509-6e0ff628a0cd", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:58:04.883000000", - "trace_end_time": "2025-10-22 16:58:10.029000000", - "trace_duration_ms": 5146.244000000001, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:58:04.883000000", - "updated_at": "2025-10-22 16:58:04.883000000" - }, - { - "span_id": "99a5e8f3-a9f5-4953-94ea-ad0fa0cf6ec2", - "trace_id": "c5a64d08-d00f-46cf-ab3c-d8d473c10eca", - "parent_span_id": "2a351fd9-a7ee-42da-9e2e-86a643ceed0e", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-22 16:57:57.908000000", - "end_time": "2025-10-22 16:57:57.908000000", - "duration_ms": 0.026, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: Tell me about recent developments in machine learning", - "query_length": 53, - "query_type": "routing_evaluation_query", - "classification.mode": "Web Search", - "classification.rag_keyword_score": 0, - "classification.web_keyword_score": 1, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "complex", - "query.intent": "general_knowledge", - "query.keyword_density": 0.125, - "query.domain_affinity": "general", - "routing.decision": "Routed to Web Search based on analysis", - "routing.rationale": "RAG score: 0, Web score: 1, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "RAG", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.019230769230769232, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: Web Search (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "c5a64d08-d00f-46cf-ab3c-d8d473c10eca", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:57.908000000", - "trace_end_time": "2025-10-22 16:58:03.875000000", - "trace_duration_ms": 5967.0779999999995, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:57.908000000", - "updated_at": "2025-10-22 16:57:57.908000000" - }, - { - "span_id": "6e891e51-ca6d-4e6c-80d0-0791ee22d5b9", - "trace_id": "c5a64d08-d00f-46cf-ab3c-d8d473c10eca", - "parent_span_id": "2a351fd9-a7ee-42da-9e2e-86a643ceed0e", - "name": "agent.web_search_generation", - "start_time": "2025-10-22 16:57:57.908000000", - "end_time": "2025-10-22 16:58:03.874000000", - "duration_ms": 5966.728, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "web_search_agent", - "agent.operation": "web_search_generation", - "agent.capabilities": [ - "web_search", - "content_synthesis", - "response_generation" - ], - "agent.id": "noveum_web_search_agent", - "input_query": "Tell me about recent developments in machine learning", - "query_length": 53, - "query_type": "web_search_query", - "output_response": "Web Search Answer: The web search results did not provide specific information about recent developments in machine learning. Therefore, I cannot summarize any recent advancements based on the provided data.\n\nHowever, i...", - "web_search.results_count": 1, - "web_search.context_length": 768, - "web_search.response_length": 687, - "web_search.mode": "external_web_search" - }, - "events": [], - "links": [], - "trace_trace_id": "c5a64d08-d00f-46cf-ab3c-d8d473c10eca", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:57.908000000", - "trace_end_time": "2025-10-22 16:58:03.875000000", - "trace_duration_ms": 5967.0779999999995, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:57.908000000", - "updated_at": "2025-10-22 16:57:57.908000000" - }, - { - "span_id": "6d622376-d48b-4a64-b0a2-1402117febaf", - "trace_id": "c5a64d08-d00f-46cf-ab3c-d8d473c10eca", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-22 16:57:57.908000000", - "end_time": "2025-10-22 16:58:03.875000000", - "duration_ms": 5967.035, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "Tell me about recent developments in machine learning", - "query.length": 53, - "processing.mode": "Web Search", - "processing.time_seconds": 5.966889142990112, - "processing.response_length": 687, - "processing.sources_count": 1, - "output_response": "Final Answer: The web search results did not provide specific information about recent developments in machine learning. Therefore, I cannot summarize any recent advancements based on the provided data.\n\nHowever, i...", - "final_answer_mode": "Web Search", - "query_processed.input_query": "Tell me about recent developments in machine learning", - "query_processed.output_response": "Successfully processed query using Web Search, generated 687 character response", - "query_processed.mode": "Web Search", - "query_processed.processing_time": 5.966889142990112, - "query_processed.response_length": 687 - }, - "events": [], - "links": [], - "trace_trace_id": "c5a64d08-d00f-46cf-ab3c-d8d473c10eca", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:57.908000000", - "trace_end_time": "2025-10-22 16:58:03.875000000", - "trace_duration_ms": 5967.0779999999995, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:57.908000000", - "updated_at": "2025-10-22 16:57:57.908000000" - }, - { - "span_id": "2a351fd9-a7ee-42da-9e2e-86a643ceed0e", - "trace_id": "c5a64d08-d00f-46cf-ab3c-d8d473c10eca", - "parent_span_id": "6d622376-d48b-4a64-b0a2-1402117febaf", - "name": "agent.query_routing", - "start_time": "2025-10-22 16:57:57.908000000", - "end_time": "2025-10-22 16:58:03.875000000", - "duration_ms": 5966.8640000000005, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "Tell me about recent developments in machine learning", - "query_length": 53, - "query_type": "routing_query", - "output_response": "Routed to Web Search for query processing", - "router.classification": "Web Search", - "router.confidence_score": 1, - "router.rag_keyword_score": 0, - "router.web_keyword_score": 1, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "c5a64d08-d00f-46cf-ab3c-d8d473c10eca", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:57.908000000", - "trace_end_time": "2025-10-22 16:58:03.875000000", - "trace_duration_ms": 5967.0779999999995, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:57.908000000", - "updated_at": "2025-10-22 16:57:57.908000000" - }, - { - "span_id": "ba457b3b-9be8-4e24-af63-4118833711e6", - "trace_id": "c5a64d08-d00f-46cf-ab3c-d8d473c10eca", - "parent_span_id": "6e891e51-ca6d-4e6c-80d0-0791ee22d5b9", - "name": "agent.llm_model_execution", - "start_time": "2025-10-22 16:57:58.324000000", - "end_time": "2025-10-22 16:58:03.874000000", - "duration_ms": 5550.059, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: Tell me about recent developments in machine learning", - "query_length": 53, - "query_type": "web_search_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 5.550010919570923, - "model.latency_ms": 5550.010919570923, - "model.start_time": 1761152278.3248482, - "model.end_time": 1761152283.874859, - "model.performance_tier": "slow", - "model.prompt_tokens": 316, - "model.completion_tokens": 171, - "model.total_tokens": 487, - "model.tokens_per_second": 87.74757510524871, - "model.estimated_cost": 0.01461, - "model.efficiency_score": 1.4106776180698153, - "model.response_length": 687, - "model.response_quality": "high", - "model.output_response": "Model Response: The web search results did not provide specific information about recent developments in machine learning. Therefore, I cannot summarize any recent advancements based on the provided data.\n\nHowever, i...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "c5a64d08-d00f-46cf-ab3c-d8d473c10eca", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:57.908000000", - "trace_end_time": "2025-10-22 16:58:03.875000000", - "trace_duration_ms": 5967.0779999999995, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:57.908000000", - "updated_at": "2025-10-22 16:57:57.908000000" - }, - { - "span_id": "7b70a916-1c2b-4898-a052-cfd98a979167", - "trace_id": "c5a64d08-d00f-46cf-ab3c-d8d473c10eca", - "parent_span_id": "6e891e51-ca6d-4e6c-80d0-0791ee22d5b9", - "name": "agent.web_search_evaluation_metrics", - "start_time": "2025-10-22 16:58:03.874000000", - "end_time": "2025-10-22 16:58:03.874000000", - "duration_ms": 0.02, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "web_search_evaluation_metrics", - "agent.capabilities": [ - "web_search_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: Tell me about recent developments in machine learning", - "query_length": 53, - "query_type": "web_search_evaluation_query", - "web_search.results_count": 1, - "web_search.context_length": 768, - "web_search.context_synthesized": "Context from 1 web sources", - "web_search.search_effectiveness": 0.2, - "web_search.context_quality": "medium", - "web_search.source_diversity": 1, - "web_search.information_synthesis": "low", - "web_search.external_knowledge_utilization": 0.768, - "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nFeb 3, 2024 \u00b7 tell sb sth\u4e2d\u7684tell\u66f4\u591a\u662f\u544a\u77e5\u7684\u610f\u601d\uff0c\u5c31\u662f\u544a\u8bc9\u522b\u4eba\u6709\u8fd9\u4e48\u4e00\u4e2a\u4e8b\u7684\u5b58\u5728 tell sb about sth \u7684\u610f\u601d\u5305\u542b\u66f4\u591a\u4e00\u4e9b\uff0c\u9664\u4e86\u544a\u77e5\u6709\u8fd9\u4e48\u4e00\u4e2a\u4e8b\u60c5\u7684\u5b58\u5728\uff0c\u8fd8\u4f5c\u4e86\u4e00\u4e9b\u8bf4\u660e \u6bd4\u5982\uff1a tell me he \u2026 \u4e00\u3001Say\u548cTell\u7684\u7528\u6cd5 \u6211\u4eec\u5148\u5c06say\u3001tell\u3001speak\u3001talk\u8fd9\u56db\u4e2a\u8bcd\u5206\u6210\u4e24\u7ec4\uff0c\u5206\u522b\u4e3a\uff1a \u7b2c\u2460\u7ec4\uff1asay & tell\uff0c\u8fd9\u4e24\u4e2a\u8bcd\u4e4b\u540e\u53ef\u4ee5\u8868\u793a\u8c08\u8bdd\u7684 \u201c\u7ec6\u8282\u201d\u3002 \u7b2c\u2461\u7ec4\uff1aspeak & talk\uff0c\u4e0d\u80fd\u8868\u793a\u8c08\u8bdd\u7684\u201c\u7ec6\u8282\u201d\u3002 \u2026 tell of 1\u3001Tell of a turning point in your career and what you learned. \u8bb2\u8ff0\u4f60 \u804c\u4e1a\u751f\u6daf \u4e00\u4e2a\u8f6c\u6298\u70b9\uff0c\u4ee5\u53ca\u4f60\u5b66\u5230\u4e86\u4ec0\u4e48\u3002 2\u3001I have made the Sovereign Lord my refuge; I will tell of all your \u2026 tell sb to do sth \u548c tell sb do sth \u90fd\u8868\u793a\u544a\u8bc9\u67d0\u4eba\u505a\u67d0\u4e8b\uff0ctell sb to do sth \u5f3a\u8c03\u544a\u8bc9\u67d0\u4eba\u5e94\u8be5\u505a\u4ec0\u4e48\u5177\u4f53\u52a8\u4f5c\u6216\u884c\u4e3a\uff0c\u800c tell sb do sth \u5f3a\u8c03\u544a\u8bc9\u67d0\u4eba\u6b63\u5728\u8fdb\u884c\u7684\u5177\u4f53\u884c\u52a8\u6216\u72b6\u6001\u3002 \u5b83\u4eec\u5728\u91ca\u4e49\u3001 \u2026 Aug 23, 2023 \u00b7 tell\u7684\u8fc7\u53bb\u5f0f\u548c\u8fc7\u53bb\u5206\u8bcd\u90fd\u662ftold\u3002 \u8bfb\u97f3\uff1a\u82f1[tel]\uff0c\u7f8e[tel]\u3002 \u91ca\u4e49\uff1a vt.\u544a\u8bc9\uff0c\u8bf4\uff1b\u8fa8\u522b\uff1b\u5429\u5490\uff1b\u65ad\u5b9a\u3002 vi.\u8bb2\u8ff0\uff1b\u544a\u53d1\uff0c\u6cc4\u5bc6\uff1b\u8bc6\u522b\u3002 \u53d8\u5f62\uff1a\u8fc7\u53bb\u5f0ftold\uff0c\u8fc7\u53bb\u5206\u8bcdtold\uff0c\u73b0\u5728\u5206 \u2026\nURL: https://duckduckgo.com/?q=Tell+me+about+recent+developments+in+machine+learning\n\nUser Question: Tell me about recent developments in machine learning\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", - "prompt.prompt_length": 1266, - "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nFeb 3, 2024 \u00b7 tell sb sth\u4e2d\u7684tell\u66f4\u591a\u662f\u544a\u77e5\u7684\u610f\u601d\uff0c\u5c31\u662f\u544a\u8bc9\u522b\u4eba\u6709\u8fd9\u4e48\u4e00\u4e2a\u4e8b\u7684\u5b58\u5728 tell sb about sth \u7684\u610f\u601d\u5305\u542b\u66f4\u591a\u4e00\u4e9b\uff0c\u9664\u4e86\u544a\u77e5\u6709\u8fd9\u4e48\u4e00\u4e2a\u4e8b\u60c5\u7684\u5b58\u5728\uff0c\u8fd8\u4f5c\u4e86\u4e00\u4e9b\u8bf4\u660e \u6bd4\u5982\uff1a tell me he \u2026 \u4e00\u3001Say\u548cTell\u7684\u7528\u6cd5 \u6211\u4eec\u5148\u5c06say\u3001tell\u3001speak\u3001talk\u8fd9\u56db\u4e2a\u8bcd\u5206\u6210\u4e24\u7ec4\uff0c\u5206\u522b...", - "prompt.instruction_following": "web_search_optimized", - "response.answer_length": 687, - "response.answer_completeness": "complete", - "response.response_quality": "medium", - "response.source_citation": 1, - "response.context_utilization": 0.89453125, - "output_response": "Web Search Answer: The web search results did not provide specific information about recent developments in machine learning. Therefore, I cannot summarize any recent advancements based on the provided data.\n\nHowever, i...", - "evaluation.search_effectiveness": 0.2, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 1, - "evaluation.context_relevance": "medium", - "evaluation.overall_quality": "low", - "evaluation.ready_for_production": true, - "web_search.search_strategy": "duckduckgo_api", - "web_search.real_time_data": true, - "web_search.external_sources": 1, - "web_search.information_freshness": "current", - "web_search.knowledge_synthesis": "single_source" - }, - "events": [], - "links": [], - "trace_trace_id": "c5a64d08-d00f-46cf-ab3c-d8d473c10eca", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:57.908000000", - "trace_end_time": "2025-10-22 16:58:03.875000000", - "trace_duration_ms": 5967.0779999999995, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:57.908000000", - "updated_at": "2025-10-22 16:57:57.908000000" - }, - { - "span_id": "1e1a341f-66c1-48b0-95eb-fb092928c473", - "trace_id": "b1b13e60-5b32-44a0-9769-197a944989d0", - "parent_span_id": "1c5c4291-bbb5-4f85-b6d9-b6e8df66f742", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-22 16:57:51.967000000", - "end_time": "2025-10-22 16:57:51.967000000", - "duration_ms": 0.033, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What's the weather like today?", - "query_length": 30, - "query_type": "routing_evaluation_query", - "classification.mode": "Web Search", - "classification.rag_keyword_score": 0, - "classification.web_keyword_score": 2, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "general_knowledge", - "query.keyword_density": 0.4, - "query.domain_affinity": "general", - "routing.decision": "Routed to Web Search based on analysis", - "routing.rationale": "RAG score: 0, Web score: 2, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "RAG", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.038461538461538464, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: Web Search (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "b1b13e60-5b32-44a0-9769-197a944989d0", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:51.967000000", - "trace_end_time": "2025-10-22 16:57:56.899000000", - "trace_duration_ms": 4932.393, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:51.967000000", - "updated_at": "2025-10-22 16:57:51.967000000" - }, - { - "span_id": "1c5c4291-bbb5-4f85-b6d9-b6e8df66f742", - "trace_id": "b1b13e60-5b32-44a0-9769-197a944989d0", - "parent_span_id": "49e7cace-ba21-4807-a238-9a3880487583", - "name": "agent.query_routing", - "start_time": "2025-10-22 16:57:51.967000000", - "end_time": "2025-10-22 16:57:56.899000000", - "duration_ms": 4931.972, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What's the weather like today?", - "query_length": 30, - "query_type": "routing_query", - "output_response": "Routed to Web Search for query processing", - "router.classification": "Web Search", - "router.confidence_score": 1, - "router.rag_keyword_score": 0, - "router.web_keyword_score": 2, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "b1b13e60-5b32-44a0-9769-197a944989d0", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:51.967000000", - "trace_end_time": "2025-10-22 16:57:56.899000000", - "trace_duration_ms": 4932.393, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:51.967000000", - "updated_at": "2025-10-22 16:57:51.967000000" - }, - { - "span_id": "49e7cace-ba21-4807-a238-9a3880487583", - "trace_id": "b1b13e60-5b32-44a0-9769-197a944989d0", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-22 16:57:51.967000000", - "end_time": "2025-10-22 16:57:56.899000000", - "duration_ms": 4932.316, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What's the weather like today?", - "query.length": 30, - "processing.mode": "Web Search", - "processing.time_seconds": 4.932007789611816, - "processing.response_length": 260, - "processing.sources_count": 1, - "output_response": "Final Answer: The web search results do not provide specific information about today's weather. To find out the current weather conditions, I recommend checking a reliable weather website or app for detailed update...", - "final_answer_mode": "Web Search", - "query_processed.input_query": "What's the weather like today?", - "query_processed.output_response": "Successfully processed query using Web Search, generated 260 character response", - "query_processed.mode": "Web Search", - "query_processed.processing_time": 4.932007789611816, - "query_processed.response_length": 260 - }, - "events": [], - "links": [], - "trace_trace_id": "b1b13e60-5b32-44a0-9769-197a944989d0", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:51.967000000", - "trace_end_time": "2025-10-22 16:57:56.899000000", - "trace_duration_ms": 4932.393, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:51.967000000", - "updated_at": "2025-10-22 16:57:51.967000000" - }, - { - "span_id": "58d217aa-4350-43e8-8813-468149dc4107", - "trace_id": "b1b13e60-5b32-44a0-9769-197a944989d0", - "parent_span_id": "1c5c4291-bbb5-4f85-b6d9-b6e8df66f742", - "name": "agent.web_search_generation", - "start_time": "2025-10-22 16:57:51.967000000", - "end_time": "2025-10-22 16:57:56.899000000", - "duration_ms": 4931.796, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "web_search_agent", - "agent.operation": "web_search_generation", - "agent.capabilities": [ - "web_search", - "content_synthesis", - "response_generation" - ], - "agent.id": "noveum_web_search_agent", - "input_query": "What's the weather like today?", - "query_length": 30, - "query_type": "web_search_query", - "output_response": "Web Search Answer: The web search results do not provide specific information about today's weather. To find out the current weather conditions, I recommend checking a reliable weather website or app for detailed update...", - "web_search.results_count": 1, - "web_search.context_length": 995, - "web_search.response_length": 260, - "web_search.mode": "external_web_search" - }, - "events": [], - "links": [], - "trace_trace_id": "b1b13e60-5b32-44a0-9769-197a944989d0", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:51.967000000", - "trace_end_time": "2025-10-22 16:57:56.899000000", - "trace_duration_ms": 4932.393, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:51.967000000", - "updated_at": "2025-10-22 16:57:51.967000000" - }, - { - "span_id": "d89dd84e-0b5e-4c92-942b-18c4735713d4", - "trace_id": "b1b13e60-5b32-44a0-9769-197a944989d0", - "parent_span_id": "58d217aa-4350-43e8-8813-468149dc4107", - "name": "agent.llm_model_execution", - "start_time": "2025-10-22 16:57:55.269000000", - "end_time": "2025-10-22 16:57:56.899000000", - "duration_ms": 1629.7140000000002, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What's the weather like today?", - "query_length": 30, - "query_type": "web_search_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 1.629619836807251, - "model.latency_ms": 1629.619836807251, - "model.start_time": 1761152275.269459, - "model.end_time": 1761152276.8990788, - "model.performance_tier": "fast", - "model.prompt_tokens": 367, - "model.completion_tokens": 65, - "model.total_tokens": 432, - "model.tokens_per_second": 265.0925020932329, - "model.estimated_cost": 0.012960000000000001, - "model.efficiency_score": 0.6018518518518519, - "model.response_length": 260, - "model.response_quality": "high", - "model.output_response": "Model Response: The web search results do not provide specific information about today's weather. To find out the current weather conditions, I recommend checking a reliable weather website or app for detailed update...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "b1b13e60-5b32-44a0-9769-197a944989d0", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:51.967000000", - "trace_end_time": "2025-10-22 16:57:56.899000000", - "trace_duration_ms": 4932.393, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:51.967000000", - "updated_at": "2025-10-22 16:57:51.967000000" - }, - { - "span_id": "d076719d-da79-43fb-92ea-6015f856146b", - "trace_id": "b1b13e60-5b32-44a0-9769-197a944989d0", - "parent_span_id": "58d217aa-4350-43e8-8813-468149dc4107", - "name": "agent.web_search_evaluation_metrics", - "start_time": "2025-10-22 16:57:56.899000000", - "end_time": "2025-10-22 16:57:56.899000000", - "duration_ms": 0.049, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "web_search_evaluation_metrics", - "agent.capabilities": [ - "web_search_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What's the weather like today?", - "query_length": 30, - "query_type": "web_search_evaluation_query", - "web_search.results_count": 1, - "web_search.context_length": 995, - "web_search.context_synthesized": "Context from 1 web sources", - "web_search.search_effectiveness": 0.2, - "web_search.context_quality": "high", - "web_search.source_diversity": 1, - "web_search.information_synthesis": "low", - "web_search.external_knowledge_utilization": 0.995, - "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nLog in to WhatsApp Web for simple, reliable and private messaging on your desktop. Send and receive messages and files with ease, all for free. 2 days ago \u00b7 More videos you may like 00:13 Wanna be yours #kimpolin #colinbridgerton #benedictbri... 2 hours ago \u00b7 101 views 00:14 Viscount & Vicountess's Marital bliss #bridgerton \u2026 Nov 15, 2024 \u00b7 Learn the correct usage of \"what's the weather like today\" and \"how's the weather today\" in English. Discover differences, examples, alternatives and tips for choosing the right \u2026 Nov 15, 2024 \u00b7 Learn the correct usage of \"how's the weather today\" and \"what's the weather today\" in English. Discover differences, examples, alternatives and tips for choosing the right \u2026 2 days ago \u00b7 Get the latest hourly weather updates for Thane today. Detailed forecast including temperature, wind, rain, snow, and UV index. Stay informed about today's weather conditions in \u2026\nURL: https://duckduckgo.com/?q=What's+the+weather+like+today?\n\nUser Question: What's the weather like today?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", - "prompt.prompt_length": 1470, - "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nLog in to WhatsApp Web for simple, reliable and private messaging on your desktop. Send and receive messages and files with ease, all for free. 2 days ago \u00b7 More videos you...", - "prompt.instruction_following": "web_search_optimized", - "response.answer_length": 260, - "response.answer_completeness": "complete", - "response.response_quality": "medium", - "response.source_citation": 1, - "response.context_utilization": 0.2613065326633166, - "output_response": "Web Search Answer: The web search results do not provide specific information about today's weather. To find out the current weather conditions, I recommend checking a reliable weather website or app for detailed update...", - "evaluation.search_effectiveness": 0.2, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 1, - "evaluation.context_relevance": "high", - "evaluation.overall_quality": "low", - "evaluation.ready_for_production": true, - "web_search.search_strategy": "duckduckgo_api", - "web_search.real_time_data": true, - "web_search.external_sources": 1, - "web_search.information_freshness": "current", - "web_search.knowledge_synthesis": "single_source" - }, - "events": [], - "links": [], - "trace_trace_id": "b1b13e60-5b32-44a0-9769-197a944989d0", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:51.967000000", - "trace_end_time": "2025-10-22 16:57:56.899000000", - "trace_duration_ms": 4932.393, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:51.967000000", - "updated_at": "2025-10-22 16:57:51.967000000" - }, - { - "span_id": "41167ca4-e1ad-48a5-b5eb-aa272ad88a85", - "trace_id": "e4f3cc20-c137-4905-80fe-2d29b03c2231", - "parent_span_id": "56959f47-5219-4674-8c0d-e5fa928dcd45", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-22 16:57:47.477000000", - "end_time": "2025-10-22 16:57:47.477000000", - "duration_ms": 0.061, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What are the latest AI news today?", - "query_length": 34, - "query_type": "routing_evaluation_query", - "classification.mode": "Web Search", - "classification.rag_keyword_score": 0, - "classification.web_keyword_score": 3, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "general_knowledge", - "query.keyword_density": 0.42857142857142855, - "query.domain_affinity": "general", - "routing.decision": "Routed to Web Search based on analysis", - "routing.rationale": "RAG score: 0, Web score: 3, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "RAG", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.057692307692307696, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: Web Search (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "e4f3cc20-c137-4905-80fe-2d29b03c2231", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:47.477000000", - "trace_end_time": "2025-10-22 16:57:50.959000000", - "trace_duration_ms": 3481.856, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:47.477000000", - "updated_at": "2025-10-22 16:57:47.477000000" - }, - { - "span_id": "73005751-9eb1-4540-aa49-2ac580cd7418", - "trace_id": "e4f3cc20-c137-4905-80fe-2d29b03c2231", - "parent_span_id": "56959f47-5219-4674-8c0d-e5fa928dcd45", - "name": "agent.web_search_generation", - "start_time": "2025-10-22 16:57:47.477000000", - "end_time": "2025-10-22 16:57:50.958000000", - "duration_ms": 3480.8469999999998, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "web_search_agent", - "agent.operation": "web_search_generation", - "agent.capabilities": [ - "web_search", - "content_synthesis", - "response_generation" - ], - "agent.id": "noveum_web_search_agent", - "input_query": "What are the latest AI news today?", - "query_length": 34, - "query_type": "web_search_query", - "output_response": "Web Search Answer: The provided web search results do not contain specific information about the latest AI news today. They primarily focus on general news coverage from India and around the world, including politics, b...", - "web_search.results_count": 1, - "web_search.context_length": 887, - "web_search.response_length": 503, - "web_search.mode": "external_web_search" - }, - "events": [], - "links": [], - "trace_trace_id": "e4f3cc20-c137-4905-80fe-2d29b03c2231", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:47.477000000", - "trace_end_time": "2025-10-22 16:57:50.959000000", - "trace_duration_ms": 3481.856, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:47.477000000", - "updated_at": "2025-10-22 16:57:47.477000000" - }, - { - "span_id": "9e1b87be-bc29-4084-b37c-6e1220dde8eb", - "trace_id": "e4f3cc20-c137-4905-80fe-2d29b03c2231", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-22 16:57:47.477000000", - "end_time": "2025-10-22 16:57:50.959000000", - "duration_ms": 3481.7580000000003, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What are the latest AI news today?", - "query.length": 34, - "processing.mode": "Web Search", - "processing.time_seconds": 3.4813451766967773, - "processing.response_length": 503, - "processing.sources_count": 1, - "output_response": "Final Answer: The provided web search results do not contain specific information about the latest AI news today. They primarily focus on general news coverage from India and around the world, including politics, b...", - "final_answer_mode": "Web Search", - "query_processed.input_query": "What are the latest AI news today?", - "query_processed.output_response": "Successfully processed query using Web Search, generated 503 character response", - "query_processed.mode": "Web Search", - "query_processed.processing_time": 3.4813451766967773, - "query_processed.response_length": 503 - }, - "events": [], - "links": [], - "trace_trace_id": "e4f3cc20-c137-4905-80fe-2d29b03c2231", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:47.477000000", - "trace_end_time": "2025-10-22 16:57:50.959000000", - "trace_duration_ms": 3481.856, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:47.477000000", - "updated_at": "2025-10-22 16:57:47.477000000" - }, - { - "span_id": "56959f47-5219-4674-8c0d-e5fa928dcd45", - "trace_id": "e4f3cc20-c137-4905-80fe-2d29b03c2231", - "parent_span_id": "9e1b87be-bc29-4084-b37c-6e1220dde8eb", - "name": "agent.query_routing", - "start_time": "2025-10-22 16:57:47.477000000", - "end_time": "2025-10-22 16:57:50.958000000", - "duration_ms": 3481.254, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What are the latest AI news today?", - "query_length": 34, - "query_type": "routing_query", - "output_response": "Routed to Web Search for query processing", - "router.classification": "Web Search", - "router.confidence_score": 1, - "router.rag_keyword_score": 0, - "router.web_keyword_score": 3, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "e4f3cc20-c137-4905-80fe-2d29b03c2231", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:47.477000000", - "trace_end_time": "2025-10-22 16:57:50.959000000", - "trace_duration_ms": 3481.856, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:47.477000000", - "updated_at": "2025-10-22 16:57:47.477000000" - }, - { - "span_id": "3dbf95ba-3ef1-4294-93d4-217f4ada5ffe", - "trace_id": "e4f3cc20-c137-4905-80fe-2d29b03c2231", - "parent_span_id": "73005751-9eb1-4540-aa49-2ac580cd7418", - "name": "agent.llm_model_execution", - "start_time": "2025-10-22 16:57:47.864000000", - "end_time": "2025-10-22 16:57:50.958000000", - "duration_ms": 3093.442, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What are the latest AI news today?", - "query_length": 34, - "query_type": "web_search_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 3.0933361053466797, - "model.latency_ms": 3093.3361053466797, - "model.start_time": 1761152267.864976, - "model.end_time": 1761152270.958312, - "model.performance_tier": "medium", - "model.prompt_tokens": 341, - "model.completion_tokens": 125, - "model.total_tokens": 466, - "model.tokens_per_second": 150.6464167261171, - "model.estimated_cost": 0.013980000000000001, - "model.efficiency_score": 1.0793991416309012, - "model.response_length": 503, - "model.response_quality": "high", - "model.output_response": "Model Response: The provided web search results do not contain specific information about the latest AI news today. They primarily focus on general news coverage from India and around the world, including politics, b...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "e4f3cc20-c137-4905-80fe-2d29b03c2231", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:47.477000000", - "trace_end_time": "2025-10-22 16:57:50.959000000", - "trace_duration_ms": 3481.856, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:47.477000000", - "updated_at": "2025-10-22 16:57:47.477000000" - }, - { - "span_id": "f7fe795b-ce1b-4fcc-83f5-06a5f074f38d", - "trace_id": "e4f3cc20-c137-4905-80fe-2d29b03c2231", - "parent_span_id": "73005751-9eb1-4540-aa49-2ac580cd7418", - "name": "agent.web_search_evaluation_metrics", - "start_time": "2025-10-22 16:57:50.958000000", - "end_time": "2025-10-22 16:57:50.958000000", - "duration_ms": 0.026, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "web_search_evaluation_metrics", - "agent.capabilities": [ - "web_search_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What are the latest AI news today?", - "query_length": 34, - "query_type": "web_search_evaluation_query", - "web_search.results_count": 1, - "web_search.context_length": 887, - "web_search.context_synthesized": "Context from 1 web sources", - "web_search.search_effectiveness": 0.2, - "web_search.context_quality": "high", - "web_search.source_diversity": 1, - "web_search.information_synthesis": "low", - "web_search.external_knowledge_utilization": 0.887, - "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\n3 days ago \u00b7 Latest news headlines from India & around the world. Check out today\u2019s news coverage live with videos & photos on NDTV.com. 6 days ago \u00b7 India News | Latest India News | Read latest and breaking news from India. Today's top India news headlines, news on Indian politics, elections, government, business, \u2026 2 days ago \u00b7 Top News Stories of the day, Latest News Headlines, News Specials, Breaking News and Latest India News, World current affairs & Political News all around the clock at \u2026 NDTV.com provides latest news from India and around the world. Get breaking news alerts from India and follow today\u2019s live news updates in field of politics, business, technology, Bollywood ... 6 days ago \u00b7 Get Latest entertainment news from Bollywood, Hollywood and regional film industries.\nURL: https://duckduckgo.com/?q=What+are+the+latest+AI+news+today?\n\nUser Question: What are the latest AI news today?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", - "prompt.prompt_length": 1366, - "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\n3 days ago \u00b7 Latest news headlines from India & around the world. Check out today\u2019s news coverage live with videos & photos on NDTV.com. 6 days ago \u00b7 India News | Latest In...", - "prompt.instruction_following": "web_search_optimized", - "response.answer_length": 503, - "response.answer_completeness": "complete", - "response.response_quality": "medium", - "response.source_citation": 1, - "response.context_utilization": 0.5670800450958287, - "output_response": "Web Search Answer: The provided web search results do not contain specific information about the latest AI news today. They primarily focus on general news coverage from India and around the world, including politics, b...", - "evaluation.search_effectiveness": 0.2, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 1, - "evaluation.context_relevance": "high", - "evaluation.overall_quality": "low", - "evaluation.ready_for_production": true, - "web_search.search_strategy": "duckduckgo_api", - "web_search.real_time_data": true, - "web_search.external_sources": 1, - "web_search.information_freshness": "current", - "web_search.knowledge_synthesis": "single_source" - }, - "events": [], - "links": [], - "trace_trace_id": "e4f3cc20-c137-4905-80fe-2d29b03c2231", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:47.477000000", - "trace_end_time": "2025-10-22 16:57:50.959000000", - "trace_duration_ms": 3481.856, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:47.477000000", - "updated_at": "2025-10-22 16:57:47.477000000" - }, - { - "span_id": "b9c9d21e-5f76-4e5a-b99e-2b4ab821caee", - "trace_id": "823b7927-10bb-49ef-a94e-1b914c298bfb", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-22 16:57:38.501000000", - "end_time": "2025-10-22 16:57:46.469000000", - "duration_ms": 7968.089, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What are the benefits of using Noveum Trace?", - "query.length": 44, - "processing.mode": "RAG", - "processing.time_seconds": 7.968040943145752, - "processing.response_length": 1396, - "processing.sources_count": 5, - "output_response": "Final Answer: Using Noveum Trace offers several benefits, including:\n\n1. **Real-Time Monitoring**: The Noveum platform provides a real-time dashboard for analyzing traces and performance, with live updates as new t...", - "final_answer_mode": "RAG", - "query_processed.input_query": "What are the benefits of using Noveum Trace?", - "query_processed.output_response": "Successfully processed query using RAG, generated 1396 character response", - "query_processed.mode": "RAG", - "query_processed.processing_time": 7.968040943145752, - "query_processed.response_length": 1396 - }, - "events": [], - "links": [], - "trace_trace_id": "823b7927-10bb-49ef-a94e-1b914c298bfb", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:38.501000000", - "trace_end_time": "2025-10-22 16:57:46.469000000", - "trace_duration_ms": 7968.149, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:38.501000000", - "updated_at": "2025-10-22 16:57:38.501000000" - }, - { - "span_id": "bc85c745-4c27-4aa4-bc00-4c9f10a2f049", - "trace_id": "823b7927-10bb-49ef-a94e-1b914c298bfb", - "parent_span_id": "b9c9d21e-5f76-4e5a-b99e-2b4ab821caee", - "name": "agent.query_routing", - "start_time": "2025-10-22 16:57:38.501000000", - "end_time": "2025-10-22 16:57:46.469000000", - "duration_ms": 7968.001, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What are the benefits of using Noveum Trace?", - "query_length": 44, - "query_type": "routing_query", - "output_response": "Routed to RAG for query processing", - "router.classification": "RAG", - "router.confidence_score": 1, - "router.rag_keyword_score": 2, - "router.web_keyword_score": 0, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "823b7927-10bb-49ef-a94e-1b914c298bfb", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:38.501000000", - "trace_end_time": "2025-10-22 16:57:46.469000000", - "trace_duration_ms": 7968.149, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:38.501000000", - "updated_at": "2025-10-22 16:57:38.501000000" - }, - { - "span_id": "3ab8cc4c-f5ed-410a-b4ee-48d0954b756d", - "trace_id": "823b7927-10bb-49ef-a94e-1b914c298bfb", - "parent_span_id": "bc85c745-4c27-4aa4-bc00-4c9f10a2f049", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-22 16:57:38.501000000", - "end_time": "2025-10-22 16:57:38.501000000", - "duration_ms": 0.09899999999999999, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What are the benefits of using Noveum Trace?", - "query_length": 44, - "query_type": "routing_evaluation_query", - "classification.mode": "RAG", - "classification.rag_keyword_score": 2, - "classification.web_keyword_score": 0, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "noveum_specific", - "query.keyword_density": 0.25, - "query.domain_affinity": "noveum", - "routing.decision": "Routed to RAG based on analysis", - "routing.rationale": "RAG score: 2, Web score: 0, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "Web Search", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.038461538461538464, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: RAG (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "823b7927-10bb-49ef-a94e-1b914c298bfb", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:38.501000000", - "trace_end_time": "2025-10-22 16:57:46.469000000", - "trace_duration_ms": 7968.149, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:38.501000000", - "updated_at": "2025-10-22 16:57:38.501000000" - }, - { - "span_id": "67292076-ec8a-4856-9c5b-726868552157", - "trace_id": "823b7927-10bb-49ef-a94e-1b914c298bfb", - "parent_span_id": "bc85c745-4c27-4aa4-bc00-4c9f10a2f049", - "name": "agent.llm-rag", - "start_time": "2025-10-22 16:57:38.501000000", - "end_time": "2025-10-22 16:57:46.469000000", - "duration_ms": 7967.679, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "rag_agent", - "agent.operation": "llm-rag", - "agent.capabilities": [ - "document_retrieval", - "context_generation", - "response_generation" - ], - "agent.id": "noveum_rag_agent", - "input_query": "What are the benefits of using Noveum Trace?", - "query_length": 44, - "query_type": "rag_query", - "output_response": "RAG Answer: Using Noveum Trace offers several benefits, including:\n\n1. **Real-Time Monitoring**: The Noveum platform provides a real-time dashboard for analyzing traces and performance, with live updates as new t...", - "rag.context_length": 2773, - "rag.sources_count": 5, - "rag.answer_length": 1396, - "rag.mode": "retrieval_augmented_generation" - }, - "events": [], - "links": [], - "trace_trace_id": "823b7927-10bb-49ef-a94e-1b914c298bfb", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:38.501000000", - "trace_end_time": "2025-10-22 16:57:46.469000000", - "trace_duration_ms": 7968.149, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:38.501000000", - "updated_at": "2025-10-22 16:57:38.501000000" - }, - { - "span_id": "d044ae4a-c463-4c1b-abce-a9a18f045e75", - "trace_id": "823b7927-10bb-49ef-a94e-1b914c298bfb", - "parent_span_id": "67292076-ec8a-4856-9c5b-726868552157", - "name": "agent.llm_model_execution", - "start_time": "2025-10-22 16:57:38.901000000", - "end_time": "2025-10-22 16:57:45.885000000", - "duration_ms": 6984.242, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What are the benefits of using Noveum Trace?", - "query_length": 44, - "query_type": "rag_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 6.984165906906128, - "model.latency_ms": 6984.165906906128, - "model.start_time": 1761152258.9012191, - "model.end_time": 1761152265.885385, - "model.performance_tier": "slow", - "model.prompt_tokens": 828, - "model.completion_tokens": 349, - "model.total_tokens": 1177, - "model.tokens_per_second": 168.524060809631, - "model.estimated_cost": 0.03531, - "model.efficiency_score": 1.1860662701784197, - "model.response_length": 1396, - "model.response_quality": "high", - "model.output_response": "Model Response: Using Noveum Trace offers several benefits, including:\n\n1. **Real-Time Monitoring**: The Noveum platform provides a real-time dashboard for analyzing traces and performance, with live updates as new t...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "823b7927-10bb-49ef-a94e-1b914c298bfb", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:38.501000000", - "trace_end_time": "2025-10-22 16:57:46.469000000", - "trace_duration_ms": 7968.149, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:38.501000000", - "updated_at": "2025-10-22 16:57:38.501000000" - }, - { - "span_id": "5b1548cf-0de7-49d4-b9b8-9ad1f9f2abc9", - "trace_id": "823b7927-10bb-49ef-a94e-1b914c298bfb", - "parent_span_id": "67292076-ec8a-4856-9c5b-726868552157", - "name": "agent.rag_evaluation_metrics", - "start_time": "2025-10-22 16:57:45.885000000", - "end_time": "2025-10-22 16:57:45.885000000", - "duration_ms": 0.061, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "rag_evaluation_metrics", - "agent.capabilities": [ - "retrieval_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What are the benefits of using Noveum Trace?", - "query_length": 44, - "query_type": "rag_evaluation_query", - "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/docs/platform/dashboard):\n- Bulk Export: Download multiple traces simultaneously\n- Comparative Analysis: Compare performance across multiple traces\nReal-Time Updates\n- Live Refresh: Automatic updates as new traces arrive\n- Connection Monitoring: Real-time status of your t...", - "retrieval.context_length": 2773, - "retrieval.sources_count": 5, - "retrieval.context_quality": "high", - "retrieval.effectiveness": 1, - "retrieval.context_utilization": 2.773, - "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/docs/platform/dashboard):\n- Bulk Export: Download multiple traces simultaneously\n- Comparative Analysis: Compare performance across multiple traces\nReal-Time Updates\n- Live Refresh: Automatic updates as new traces arrive\n- Connection Monitoring: Real-time status of your trace ingestion pipeline\n- Performance Indicators: Live metrics for system health monitoring\nReady to dive deeper? Explore Projects & Environments to organize your AI applications, or check out Team Collaboration to share insights with your team.\nGet Earl...\n\nSource 2 (https://noveum.ai/en/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\n\ud83d\udcd8 TypeScript SDK (\n@noveum/trace\n)- Framework integrations for Next.js, Express.js, and Hono\n- TypeScript-first design with full type safety\n- Universal compatibility (Node.js, Edge Runtime, browsers)\n-\n\ud83d\udcca Noveum Platform\n- Real-time dashboard for analyzing traces and performance\n- Ad...\n\nSource 3 (https://noveum.ai/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\n\ud83d\udcd8 TypeScript SDK (\n@noveum/trace\n)- Framework integrations for Next.js, Express.js, and Hono\n- TypeScript-first design with full type safety\n- Universal compatibility (Node.js, Edge Runtime, browsers)\n-\n\ud83d\udcca Noveum Platform\n- Real-time dashboard for analyzing traces and performance\n- Ad...\n\nSource 4 (https://noveum.ai/en/docs/getting-started/overview):\n- Advanced search & filtering across traces and spans\n- Cost analysis and optimization recommendations\n- Team collaboration with shared insights and alerts\n\ud83d\udd0d What Noveum Traces\nLLM Operations\n- Model calls across all providers (OpenAI, Anthropic, Google, etc.)\n- Token usage and cost calculation\n- Prompt engineering effectiveness\n- Response quality metrics\nRAG Pipelines\n- Document retrieval performance and relevance\n- Embedding generation costs and latency\n- Context assembly and prompt constructi...\n\nSource 5 (https://noveum.ai/docs/getting-started/overview):\n- Advanced search & filtering across traces and spans\n- Cost analysis and optimization recommendations\n- Team collaboration with shared insights and alerts\n\ud83d\udd0d What Noveum Traces\nLLM Operations\n- Model calls across all providers (OpenAI, Anthropic, Google, etc.)\n- Token usage and cost calculation\n- Prompt engineering effectiveness\n- Response quality metrics\nRAG Pipelines\n- Document retrieval performance and relevance\n- Embedding generation costs and latency\n- Context assembly and prompt constructi...\n\nUser Question: What are the benefits of using Noveum Trace?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", - "prompt.prompt_length": 3313, - "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/docs/platform/dashboard):\n- Bulk Export: Download multiple traces simultaneously\n- Comparative Analysis: Compare performance across multiple traces\nReal-Time Updates\n- Live...", - "prompt.instruction_following": "rag_optimized", - "response.answer_length": 1396, - "response.answer_completeness": "complete", - "response.response_quality": "high", - "response.source_citation": 5, - "response.context_utilization": 0.5034258925351605, - "output_response": "RAG Answer: Using Noveum Trace offers several benefits, including:\n\n1. **Real-Time Monitoring**: The Noveum platform provides a real-time dashboard for analyzing traces and performance, with live updates as new t...", - "evaluation.retrieval_effectiveness": 1, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 5, - "evaluation.context_relevance": "high", - "evaluation.overall_quality": "high", - "evaluation.ready_for_production": true, - "rag.retrieval_strategy": "semantic_similarity", - "rag.vector_search_results": 5, - "rag.context_synthesis": "multi_source", - "rag.document_coverage": 1, - "rag.information_density": 0.5034258925351605 - }, - "events": [], - "links": [], - "trace_trace_id": "823b7927-10bb-49ef-a94e-1b914c298bfb", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:38.501000000", - "trace_end_time": "2025-10-22 16:57:46.469000000", - "trace_duration_ms": 7968.149, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:38.501000000", - "updated_at": "2025-10-22 16:57:38.501000000" - }, - { - "span_id": "aaba719b-36ec-46ab-af2e-3ea8c18e9872", - "trace_id": "1ae0d6c4-d1fb-4239-b536-3333187d44aa", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-22 16:57:32.800000000", - "end_time": "2025-10-22 16:57:37.494000000", - "duration_ms": 4694.408, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "How do I configure Noveum for my system?", - "query.length": 40, - "processing.mode": "RAG", - "processing.time_seconds": 4.694374084472656, - "processing.response_length": 553, - "processing.sources_count": 5, - "output_response": "Final Answer: To configure Noveum for your system, you can refer to the SDK Integration Guide available in the Noveum documentation. This guide will help you trace your AI applications with minimal code changes. Th...", - "final_answer_mode": "RAG", - "query_processed.input_query": "How do I configure Noveum for my system?", - "query_processed.output_response": "Successfully processed query using RAG, generated 553 character response", - "query_processed.mode": "RAG", - "query_processed.processing_time": 4.694374084472656, - "query_processed.response_length": 553 - }, - "events": [], - "links": [], - "trace_trace_id": "1ae0d6c4-d1fb-4239-b536-3333187d44aa", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:32.800000000", - "trace_end_time": "2025-10-22 16:57:37.494000000", - "trace_duration_ms": 4694.468, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:32.800000000", - "updated_at": "2025-10-22 16:57:32.800000000" - }, - { - "span_id": "4254e6f9-10b8-42ba-939d-d7c124f80f9a", - "trace_id": "1ae0d6c4-d1fb-4239-b536-3333187d44aa", - "parent_span_id": "6d5d074e-8fd3-4d33-88fb-7f54b0959030", - "name": "agent.llm-rag", - "start_time": "2025-10-22 16:57:32.800000000", - "end_time": "2025-10-22 16:57:37.494000000", - "duration_ms": 4694.155000000001, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "rag_agent", - "agent.operation": "llm-rag", - "agent.capabilities": [ - "document_retrieval", - "context_generation", - "response_generation" - ], - "agent.id": "noveum_rag_agent", - "input_query": "How do I configure Noveum for my system?", - "query_length": 40, - "query_type": "rag_query", - "output_response": "RAG Answer: To configure Noveum for your system, you can refer to the SDK Integration Guide available in the Noveum documentation. This guide will help you trace your AI applications with minimal code changes. Th...", - "rag.context_length": 2237, - "rag.sources_count": 5, - "rag.answer_length": 553, - "rag.mode": "retrieval_augmented_generation" - }, - "events": [], - "links": [], - "trace_trace_id": "1ae0d6c4-d1fb-4239-b536-3333187d44aa", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:32.800000000", - "trace_end_time": "2025-10-22 16:57:37.494000000", - "trace_duration_ms": 4694.468, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:32.800000000", - "updated_at": "2025-10-22 16:57:32.800000000" - }, - { - "span_id": "6d5d074e-8fd3-4d33-88fb-7f54b0959030", - "trace_id": "1ae0d6c4-d1fb-4239-b536-3333187d44aa", - "parent_span_id": "aaba719b-36ec-46ab-af2e-3ea8c18e9872", - "name": "agent.query_routing", - "start_time": "2025-10-22 16:57:32.800000000", - "end_time": "2025-10-22 16:57:37.494000000", - "duration_ms": 4694.318, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "How do I configure Noveum for my system?", - "query_length": 40, - "query_type": "routing_query", - "output_response": "Routed to RAG for query processing", - "router.classification": "RAG", - "router.confidence_score": 1, - "router.rag_keyword_score": 2, - "router.web_keyword_score": 0, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "1ae0d6c4-d1fb-4239-b536-3333187d44aa", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:32.800000000", - "trace_end_time": "2025-10-22 16:57:37.494000000", - "trace_duration_ms": 4694.468, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:32.800000000", - "updated_at": "2025-10-22 16:57:32.800000000" - }, - { - "span_id": "2cc2ab0b-2572-4a75-88a3-18b3a1eaca78", - "trace_id": "1ae0d6c4-d1fb-4239-b536-3333187d44aa", - "parent_span_id": "6d5d074e-8fd3-4d33-88fb-7f54b0959030", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-22 16:57:32.800000000", - "end_time": "2025-10-22 16:57:32.800000000", - "duration_ms": 0.04, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: How do I configure Noveum for my system?", - "query_length": 40, - "query_type": "routing_evaluation_query", - "classification.mode": "RAG", - "classification.rag_keyword_score": 2, - "classification.web_keyword_score": 0, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "noveum_specific", - "query.keyword_density": 0.25, - "query.domain_affinity": "noveum", - "routing.decision": "Routed to RAG based on analysis", - "routing.rationale": "RAG score: 2, Web score: 0, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "Web Search", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.038461538461538464, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: RAG (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "1ae0d6c4-d1fb-4239-b536-3333187d44aa", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:32.800000000", - "trace_end_time": "2025-10-22 16:57:37.494000000", - "trace_duration_ms": 4694.468, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:32.800000000", - "updated_at": "2025-10-22 16:57:32.800000000" - }, - { - "span_id": "f1d9f779-3312-43c1-b549-00fc874429f0", - "trace_id": "1ae0d6c4-d1fb-4239-b536-3333187d44aa", - "parent_span_id": "4254e6f9-10b8-42ba-939d-d7c124f80f9a", - "name": "agent.llm_model_execution", - "start_time": "2025-10-22 16:57:33.281000000", - "end_time": "2025-10-22 16:57:37.060000000", - "duration_ms": 3778.743, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: How do I configure Noveum for my system?", - "query_length": 40, - "query_type": "rag_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 3.7786097526550293, - "model.latency_ms": 3778.6097526550293, - "model.start_time": 1761152253.28163, - "model.end_time": 1761152257.0602398, - "model.performance_tier": "medium", - "model.prompt_tokens": 693, - "model.completion_tokens": 138, - "model.total_tokens": 831, - "model.tokens_per_second": 219.92215507730066, - "model.estimated_cost": 0.02493, - "model.efficiency_score": 0.6654632972322503, - "model.response_length": 553, - "model.response_quality": "high", - "model.output_response": "Model Response: To configure Noveum for your system, you can refer to the SDK Integration Guide available in the Noveum documentation. This guide will help you trace your AI applications with minimal code changes. Th...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "1ae0d6c4-d1fb-4239-b536-3333187d44aa", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:32.800000000", - "trace_end_time": "2025-10-22 16:57:37.494000000", - "trace_duration_ms": 4694.468, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:32.800000000", - "updated_at": "2025-10-22 16:57:32.800000000" - }, - { - "span_id": "5500be90-0cfa-44dd-a6e3-e72ac9f0a79e", - "trace_id": "1ae0d6c4-d1fb-4239-b536-3333187d44aa", - "parent_span_id": "4254e6f9-10b8-42ba-939d-d7c124f80f9a", - "name": "agent.rag_evaluation_metrics", - "start_time": "2025-10-22 16:57:37.060000000", - "end_time": "2025-10-22 16:57:37.060000000", - "duration_ms": 0.076, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "rag_evaluation_metrics", - "agent.capabilities": [ - "retrieval_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: How do I configure Noveum for my system?", - "query_length": 40, - "query_type": "rag_evaluation_query", - "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/docs):\nWelcome to Noveum.ai Docs\nComprehensive AI tracing and observability for LLM applications, RAG systems, and AI agents\nWelcome to the Noveum.ai documentation! Here you'll find everything you need to integrate, configure, and optimize your AI applications using ou...", - "retrieval.context_length": 2237, - "retrieval.sources_count": 5, - "retrieval.context_quality": "high", - "retrieval.effectiveness": 1, - "retrieval.context_utilization": 2.237, - "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/docs):\nWelcome to Noveum.ai Docs\nComprehensive AI tracing and observability for LLM applications, RAG systems, and AI agents\nWelcome to the Noveum.ai documentation! Here you'll find everything you need to integrate, configure, and optimize your AI applications using our comprehensive tracing and observability platform.\nWe provide powerful SDKs for Python and TypeScript that enable you to trace LLM calls, RAG pipelines, and multi-agent workflows with minimal code changes. Our documentation is organized ...\n\nSource 2 (https://noveum.ai/docs):\nWelcome to Noveum.ai Docs\nComprehensive AI tracing and observability for LLM applications, RAG systems, and AI agents\nWelcome to the Noveum.ai documentation! Here you'll find everything you need to integrate, configure, and optimize your AI applications using our comprehensive tracing and observability platform.\nWe provide powerful SDKs for Python and TypeScript that enable you to trace LLM calls, RAG pipelines, and multi-agent workflows with minimal code changes. Our documentation is organized ...\n\nSource 3 (https://noveum.ai/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nSource 4 (https://noveum.ai/en/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nSource 5 (https://noveum.ai/en/docs):\n- \ud83c\udf0d Universal: Works across frameworks, clouds, and deployment models\n\ud83e\udd1d Community & Support\n- \ud83d\udcac Discord Community: Join our Discord\n- \ud83d\udce7 Email Support: [email protected]\n- \ud83d\udc1b Bug Reports: GitHub Issues\n- \ud83d\udcd6 Knowledge Base: Help Center\nReady to get started? Head to our SDK Integration Guide to begin tracing your AI applications in under 5 minutes!\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability ...\n\nUser Question: How do I configure Noveum for my system?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", - "prompt.prompt_length": 2773, - "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/docs):\nWelcome to Noveum.ai Docs\nComprehensive AI tracing and observability for LLM applications, RAG systems, and AI agents\nWelcome to the Noveum.ai documentation! Here...", - "prompt.instruction_following": "rag_optimized", - "response.answer_length": 553, - "response.answer_completeness": "complete", - "response.response_quality": "high", - "response.source_citation": 5, - "response.context_utilization": 0.24720607957085383, - "output_response": "RAG Answer: To configure Noveum for your system, you can refer to the SDK Integration Guide available in the Noveum documentation. This guide will help you trace your AI applications with minimal code changes. Th...", - "evaluation.retrieval_effectiveness": 1, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 5, - "evaluation.context_relevance": "high", - "evaluation.overall_quality": "high", - "evaluation.ready_for_production": true, - "rag.retrieval_strategy": "semantic_similarity", - "rag.vector_search_results": 5, - "rag.context_synthesis": "multi_source", - "rag.document_coverage": 1, - "rag.information_density": 0.24720607957085383 - }, - "events": [], - "links": [], - "trace_trace_id": "1ae0d6c4-d1fb-4239-b536-3333187d44aa", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:32.800000000", - "trace_end_time": "2025-10-22 16:57:37.494000000", - "trace_duration_ms": 4694.468, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:32.800000000", - "updated_at": "2025-10-22 16:57:32.800000000" - }, - { - "span_id": "3068013a-6844-4f3c-8ec2-854a7353591e", - "trace_id": "fb0d31fc-2465-4ecd-973c-4cd443d9befa", - "parent_span_id": "a80d8fab-d5dc-4417-a5e5-9a2a76ed7419", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-22 16:57:24.828000000", - "end_time": "2025-10-22 16:57:24.828000000", - "duration_ms": 0.038, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What monitoring capabilities does Noveum provide?", - "query_length": 49, - "query_type": "routing_evaluation_query", - "classification.mode": "RAG", - "classification.rag_keyword_score": 2, - "classification.web_keyword_score": 0, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "noveum_specific", - "query.keyword_density": 0.3333333333333333, - "query.domain_affinity": "noveum", - "routing.decision": "Routed to RAG based on analysis", - "routing.rationale": "RAG score: 2, Web score: 0, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "Web Search", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.038461538461538464, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: RAG (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "fb0d31fc-2465-4ecd-973c-4cd443d9befa", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:24.828000000", - "trace_end_time": "2025-10-22 16:57:31.793000000", - "trace_duration_ms": 6965.302000000001, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:24.828000000", - "updated_at": "2025-10-22 16:57:24.828000000" - }, - { - "span_id": "189b89e3-fbbd-41de-8521-efb6a68f62f2", - "trace_id": "fb0d31fc-2465-4ecd-973c-4cd443d9befa", - "parent_span_id": "a80d8fab-d5dc-4417-a5e5-9a2a76ed7419", - "name": "agent.llm-rag", - "start_time": "2025-10-22 16:57:24.828000000", - "end_time": "2025-10-22 16:57:31.793000000", - "duration_ms": 6964.955, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "rag_agent", - "agent.operation": "llm-rag", - "agent.capabilities": [ - "document_retrieval", - "context_generation", - "response_generation" - ], - "agent.id": "noveum_rag_agent", - "input_query": "What monitoring capabilities does Noveum provide?", - "query_length": 49, - "query_type": "rag_query", - "output_response": "RAG Answer: Noveum provides comprehensive monitoring capabilities for AI applications, including the ability to monitor, trace, and optimize AI agents across various frameworks such as LangChain, CrewAI, AutoGen,...", - "rag.context_length": 2740, - "rag.sources_count": 5, - "rag.answer_length": 582, - "rag.mode": "retrieval_augmented_generation" - }, - "events": [], - "links": [], - "trace_trace_id": "fb0d31fc-2465-4ecd-973c-4cd443d9befa", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:24.828000000", - "trace_end_time": "2025-10-22 16:57:31.793000000", - "trace_duration_ms": 6965.302000000001, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:24.828000000", - "updated_at": "2025-10-22 16:57:24.828000000" - }, - { - "span_id": "f5cffb7d-341b-4cde-b162-02208042463d", - "trace_id": "fb0d31fc-2465-4ecd-973c-4cd443d9befa", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-22 16:57:24.828000000", - "end_time": "2025-10-22 16:57:31.793000000", - "duration_ms": 6965.236, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What monitoring capabilities does Noveum provide?", - "query.length": 49, - "processing.mode": "RAG", - "processing.time_seconds": 6.965206146240234, - "processing.response_length": 582, - "processing.sources_count": 5, - "output_response": "Final Answer: Noveum provides comprehensive monitoring capabilities for AI applications, including the ability to monitor, trace, and optimize AI agents across various frameworks such as LangChain, CrewAI, AutoGen,...", - "final_answer_mode": "RAG", - "query_processed.input_query": "What monitoring capabilities does Noveum provide?", - "query_processed.output_response": "Successfully processed query using RAG, generated 582 character response", - "query_processed.mode": "RAG", - "query_processed.processing_time": 6.965206146240234, - "query_processed.response_length": 582 - }, - "events": [], - "links": [], - "trace_trace_id": "fb0d31fc-2465-4ecd-973c-4cd443d9befa", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:24.828000000", - "trace_end_time": "2025-10-22 16:57:31.793000000", - "trace_duration_ms": 6965.302000000001, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:24.828000000", - "updated_at": "2025-10-22 16:57:24.828000000" - }, - { - "span_id": "a80d8fab-d5dc-4417-a5e5-9a2a76ed7419", - "trace_id": "fb0d31fc-2465-4ecd-973c-4cd443d9befa", - "parent_span_id": "f5cffb7d-341b-4cde-b162-02208042463d", - "name": "agent.query_routing", - "start_time": "2025-10-22 16:57:24.828000000", - "end_time": "2025-10-22 16:57:31.793000000", - "duration_ms": 6965.17, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What monitoring capabilities does Noveum provide?", - "query_length": 49, - "query_type": "routing_query", - "output_response": "Routed to RAG for query processing", - "router.classification": "RAG", - "router.confidence_score": 1, - "router.rag_keyword_score": 2, - "router.web_keyword_score": 0, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "fb0d31fc-2465-4ecd-973c-4cd443d9befa", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:24.828000000", - "trace_end_time": "2025-10-22 16:57:31.793000000", - "trace_duration_ms": 6965.302000000001, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:24.828000000", - "updated_at": "2025-10-22 16:57:24.828000000" - }, - { - "span_id": "a2b632b8-fbcb-480f-9292-9f513c64df38", - "trace_id": "fb0d31fc-2465-4ecd-973c-4cd443d9befa", - "parent_span_id": "189b89e3-fbbd-41de-8521-efb6a68f62f2", - "name": "agent.llm_model_execution", - "start_time": "2025-10-22 16:57:27.824000000", - "end_time": "2025-10-22 16:57:31.155000000", - "duration_ms": 3330.989, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What monitoring capabilities does Noveum provide?", - "query_length": 49, - "query_type": "rag_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 3.3309478759765625, - "model.latency_ms": 3330.9478759765625, - "model.start_time": 1761152247.824474, - "model.end_time": 1761152251.155422, - "model.performance_tier": "medium", - "model.prompt_tokens": 821, - "model.completion_tokens": 145, - "model.total_tokens": 966, - "model.tokens_per_second": 290.00754018607677, - "model.estimated_cost": 0.028980000000000002, - "model.efficiency_score": 0.6024844720496895, - "model.response_length": 582, - "model.response_quality": "high", - "model.output_response": "Model Response: Noveum provides comprehensive monitoring capabilities for AI applications, including the ability to monitor, trace, and optimize AI agents across various frameworks such as LangChain, CrewAI, AutoGen,...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "fb0d31fc-2465-4ecd-973c-4cd443d9befa", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:24.828000000", - "trace_end_time": "2025-10-22 16:57:31.793000000", - "trace_duration_ms": 6965.302000000001, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:24.828000000", - "updated_at": "2025-10-22 16:57:24.828000000" - }, - { - "span_id": "40ff54c4-07ca-4b2d-a9b4-7e3ae7adcd81", - "trace_id": "fb0d31fc-2465-4ecd-973c-4cd443d9befa", - "parent_span_id": "189b89e3-fbbd-41de-8521-efb6a68f62f2", - "name": "agent.rag_evaluation_metrics", - "start_time": "2025-10-22 16:57:31.155000000", - "end_time": "2025-10-22 16:57:31.155000000", - "duration_ms": 0.018000000000000002, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "rag_evaluation_metrics", - "agent.capabilities": [ - "retrieval_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What monitoring capabilities does Noveum provide?", - "query_length": 49, - "query_type": "rag_evaluation_query", - "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en):\nMonitor all your AI Agents\nimprove AI Agents today\nNoveum.ai helps you monitor, trace, and optimize your AI applications.\nNoveum.ai works with any AI framework \u2013 LangChain, CrewAI, AutoGen, custom implementations, or direct LLM calls. One dashboard shows everything.\n...", - "retrieval.context_length": 2740, - "retrieval.sources_count": 5, - "retrieval.context_quality": "high", - "retrieval.effectiveness": 1, - "retrieval.context_utilization": 2.74, - "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en):\nMonitor all your AI Agents\nimprove AI Agents today\nNoveum.ai helps you monitor, trace, and optimize your AI applications.\nNoveum.ai works with any AI framework \u2013 LangChain, CrewAI, AutoGen, custom implementations, or direct LLM calls. One dashboard shows everything.\nMonitor, Evaluate, Improve Your AI Agents\nThe control plane for AI agents.\nMonitor Everything, Miss Nothing\nOur lightweight SDKs capture every trace and span across your AI agent ecosystem\u2014from simple LLM calls to complex multi-agent...\n\nSource 2 (https://noveum.ai/):\nMonitor all your AI Agents\nimprove AI Agents today\nNoveum.ai helps you monitor, trace, and optimize your AI applications.\nNoveum.ai works with any AI framework \u2013 LangChain, CrewAI, AutoGen, custom implementations, or direct LLM calls. One dashboard shows everything.\nMonitor, Evaluate, Improve Your AI Agents\nThe control plane for AI agents.\nMonitor Everything, Miss Nothing\nOur lightweight SDKs capture every trace and span across your AI agent ecosystem\u2014from simple LLM calls to complex multi-agent...\n\nSource 3 (https://noveum.ai):\nMonitor all your AI Agents\nimprove AI Agents today\nNoveum.ai helps you monitor, trace, and optimize your AI applications.\nNoveum.ai works with any AI framework \u2013 LangChain, CrewAI, AutoGen, custom implementations, or direct LLM calls. One dashboard shows everything.\nMonitor, Evaluate, Improve Your AI Agents\nThe control plane for AI agents.\nMonitor Everything, Miss Nothing\nOur lightweight SDKs capture every trace and span across your AI agent ecosystem\u2014from simple LLM calls to complex multi-agent...\n\nSource 4 (https://noveum.ai/en/docs/getting-started/overview):\nNoveum.ai Overview\nComprehensive AI tracing and observability platform for LLM applications, RAG systems, and AI agents\nWelcome to Noveum.ai\u2014the comprehensive tracing and observability platform built specifically for AI applications. Whether you're building LLM-powered chatbots, RAG systems, multi-agent workflows, or any AI-driven application, Noveum provides the insights you need to understand, debug, and optimize your systems.\n\ud83c\udfaf Why AI Applications Need Specialized Observability\nTraditional mo...\n\nSource 5 (https://noveum.ai/docs/getting-started/overview):\nNoveum.ai Overview\nComprehensive AI tracing and observability platform for LLM applications, RAG systems, and AI agents\nWelcome to Noveum.ai\u2014the comprehensive tracing and observability platform built specifically for AI applications. Whether you're building LLM-powered chatbots, RAG systems, multi-agent workflows, or any AI-driven application, Noveum provides the insights you need to understand, debug, and optimize your systems.\n\ud83c\udfaf Why AI Applications Need Specialized Observability\nTraditional mo...\n\nUser Question: What monitoring capabilities does Noveum provide?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", - "prompt.prompt_length": 3285, - "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en):\nMonitor all your AI Agents\nimprove AI Agents today\nNoveum.ai helps you monitor, trace, and optimize your AI applications.\nNoveum.ai works with any AI framework \u2013 LangC...", - "prompt.instruction_following": "rag_optimized", - "response.answer_length": 582, - "response.answer_completeness": "complete", - "response.response_quality": "high", - "response.source_citation": 5, - "response.context_utilization": 0.2124087591240876, - "output_response": "RAG Answer: Noveum provides comprehensive monitoring capabilities for AI applications, including the ability to monitor, trace, and optimize AI agents across various frameworks such as LangChain, CrewAI, AutoGen,...", - "evaluation.retrieval_effectiveness": 1, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 5, - "evaluation.context_relevance": "high", - "evaluation.overall_quality": "high", - "evaluation.ready_for_production": true, - "rag.retrieval_strategy": "semantic_similarity", - "rag.vector_search_results": 5, - "rag.context_synthesis": "multi_source", - "rag.document_coverage": 1, - "rag.information_density": 0.2124087591240876 - }, - "events": [], - "links": [], - "trace_trace_id": "fb0d31fc-2465-4ecd-973c-4cd443d9befa", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:24.828000000", - "trace_end_time": "2025-10-22 16:57:31.793000000", - "trace_duration_ms": 6965.302000000001, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:24.828000000", - "updated_at": "2025-10-22 16:57:24.828000000" - }, - { - "span_id": "b0f0a2e0-c1f7-4ba4-8b75-7090e931a738", - "trace_id": "3bbd0b36-c30f-4126-b62a-dd0fae1c701d", - "parent_span_id": "4db80813-e62f-48ec-9091-62d569527caf", - "name": "agent.query_routing", - "start_time": "2025-10-22 16:57:19.339000000", - "end_time": "2025-10-22 16:57:23.822000000", - "duration_ms": 4482.909000000001, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "How does Noveum handle agent tracing?", - "query_length": 37, - "query_type": "routing_query", - "output_response": "Routed to RAG for query processing", - "router.classification": "RAG", - "router.confidence_score": 1, - "router.rag_keyword_score": 2, - "router.web_keyword_score": 0, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "3bbd0b36-c30f-4126-b62a-dd0fae1c701d", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:19.339000000", - "trace_end_time": "2025-10-22 16:57:23.822000000", - "trace_duration_ms": 4483.158, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:19.339000000", - "updated_at": "2025-10-22 16:57:19.339000000" - }, - { - "span_id": "24da2221-b0ab-4161-98ad-e7b5775fcb27", - "trace_id": "3bbd0b36-c30f-4126-b62a-dd0fae1c701d", - "parent_span_id": "b0f0a2e0-c1f7-4ba4-8b75-7090e931a738", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-22 16:57:19.339000000", - "end_time": "2025-10-22 16:57:19.339000000", - "duration_ms": 0.026, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: How does Noveum handle agent tracing?", - "query_length": 37, - "query_type": "routing_evaluation_query", - "classification.mode": "RAG", - "classification.rag_keyword_score": 2, - "classification.web_keyword_score": 0, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "noveum_specific", - "query.keyword_density": 0.3333333333333333, - "query.domain_affinity": "noveum", - "routing.decision": "Routed to RAG based on analysis", - "routing.rationale": "RAG score: 2, Web score: 0, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "Web Search", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.038461538461538464, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: RAG (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "3bbd0b36-c30f-4126-b62a-dd0fae1c701d", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:19.339000000", - "trace_end_time": "2025-10-22 16:57:23.822000000", - "trace_duration_ms": 4483.158, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:19.339000000", - "updated_at": "2025-10-22 16:57:19.339000000" - }, - { - "span_id": "d2b495eb-7e07-48a3-aa6d-1835aab18c11", - "trace_id": "3bbd0b36-c30f-4126-b62a-dd0fae1c701d", - "parent_span_id": "b0f0a2e0-c1f7-4ba4-8b75-7090e931a738", - "name": "agent.llm-rag", - "start_time": "2025-10-22 16:57:19.339000000", - "end_time": "2025-10-22 16:57:23.822000000", - "duration_ms": 4482.564, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "rag_agent", - "agent.operation": "llm-rag", - "agent.capabilities": [ - "document_retrieval", - "context_generation", - "response_generation" - ], - "agent.id": "noveum_rag_agent", - "input_query": "How does Noveum handle agent tracing?", - "query_length": 37, - "query_type": "rag_query", - "output_response": "RAG Answer: Noveum.ai handles agent tracing through its specialized Multi-Agent Tracing capabilities, which are designed to observe complex workflows and inter-agent communications within multi-agent systems. The...", - "rag.context_length": 2736, - "rag.sources_count": 5, - "rag.answer_length": 663, - "rag.mode": "retrieval_augmented_generation" - }, - "events": [], - "links": [], - "trace_trace_id": "3bbd0b36-c30f-4126-b62a-dd0fae1c701d", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:19.339000000", - "trace_end_time": "2025-10-22 16:57:23.822000000", - "trace_duration_ms": 4483.158, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:19.339000000", - "updated_at": "2025-10-22 16:57:19.339000000" - }, - { - "span_id": "4db80813-e62f-48ec-9091-62d569527caf", - "trace_id": "3bbd0b36-c30f-4126-b62a-dd0fae1c701d", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-22 16:57:19.339000000", - "end_time": "2025-10-22 16:57:23.822000000", - "duration_ms": 4483.031000000001, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "How does Noveum handle agent tracing?", - "query.length": 37, - "processing.mode": "RAG", - "processing.time_seconds": 4.482964038848877, - "processing.response_length": 663, - "processing.sources_count": 5, - "output_response": "Final Answer: Noveum.ai handles agent tracing through its specialized Multi-Agent Tracing capabilities, which are designed to observe complex workflows and inter-agent communications within multi-agent systems. The...", - "final_answer_mode": "RAG", - "query_processed.input_query": "How does Noveum handle agent tracing?", - "query_processed.output_response": "Successfully processed query using RAG, generated 663 character response", - "query_processed.mode": "RAG", - "query_processed.processing_time": 4.482964038848877, - "query_processed.response_length": 663 - }, - "events": [], - "links": [], - "trace_trace_id": "3bbd0b36-c30f-4126-b62a-dd0fae1c701d", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:19.339000000", - "trace_end_time": "2025-10-22 16:57:23.822000000", - "trace_duration_ms": 4483.158, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:19.339000000", - "updated_at": "2025-10-22 16:57:19.339000000" - }, - { - "span_id": "b80cb5cd-24f2-484d-bbff-9febe2aa3ffe", - "trace_id": "3bbd0b36-c30f-4126-b62a-dd0fae1c701d", - "parent_span_id": "d2b495eb-7e07-48a3-aa6d-1835aab18c11", - "name": "agent.llm_model_execution", - "start_time": "2025-10-22 16:57:19.958000000", - "end_time": "2025-10-22 16:57:22.709000000", - "duration_ms": 2750.9480000000003, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: How does Noveum handle agent tracing?", - "query_length": 37, - "query_type": "rag_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 2.750890016555786, - "model.latency_ms": 2750.890016555786, - "model.start_time": 1761152239.958973, - "model.end_time": 1761152242.709863, - "model.performance_tier": "medium", - "model.prompt_tokens": 817, - "model.completion_tokens": 165, - "model.total_tokens": 982, - "model.tokens_per_second": 356.975376728983, - "model.estimated_cost": 0.02946, - "model.efficiency_score": 0.675152749490835, - "model.response_length": 663, - "model.response_quality": "high", - "model.output_response": "Model Response: Noveum.ai handles agent tracing through its specialized Multi-Agent Tracing capabilities, which are designed to observe complex workflows and inter-agent communications within multi-agent systems. The...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "3bbd0b36-c30f-4126-b62a-dd0fae1c701d", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:19.339000000", - "trace_end_time": "2025-10-22 16:57:23.822000000", - "trace_duration_ms": 4483.158, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:19.339000000", - "updated_at": "2025-10-22 16:57:19.339000000" - }, - { - "span_id": "d85fcf7d-7eaa-4299-92fc-c969fab63a6e", - "trace_id": "3bbd0b36-c30f-4126-b62a-dd0fae1c701d", - "parent_span_id": "d2b495eb-7e07-48a3-aa6d-1835aab18c11", - "name": "agent.rag_evaluation_metrics", - "start_time": "2025-10-22 16:57:22.710000000", - "end_time": "2025-10-22 16:57:22.710000000", - "duration_ms": 0.036000000000000004, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "rag_evaluation_metrics", - "agent.capabilities": [ - "retrieval_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: How does Noveum handle agent tracing?", - "query_length": 37, - "query_type": "rag_evaluation_query", - "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, vector searches, data retrieval, agent reasoning, and more. Without proper observability, debugging ...", - "retrieval.context_length": 2736, - "retrieval.sources_count": 5, - "retrieval.context_quality": "high", - "retrieval.effectiveness": 1, - "retrieval.context_utilization": 2.736, - "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, vector searches, data retrieval, agent reasoning, and more. Without proper observability, debugging becomes a nightmare, optimization is guesswork, and understanding user interactions is nearly impossible.\nNoveum.ai solves this by providing comprehensive tracing and observability specifically designed for AI applications.\nKey benefits of using Noveum.ai:\n- Complete Visibility: Trace...\n\nSource 2 (https://noveum.ai/docs/advanced/multi-agent-tracing):\nMulti-Agent Tracing\nObserve complex agent workflows and inter-agent communications with comprehensive tracing\nMulti-agent systems represent some of the most complex AI applications, involving multiple agents that coordinate, communicate, and collaborate to achieve shared goals. Noveum.ai provides specialized tracing capabilities to help you understand and optimize these intricate workflows.\n\ud83c\udfaf Why Multi-Agent Tracing Matters\nMulti-agent systems introduce unique observability challenges:\n- Complex...\n\nSource 3 (https://noveum.ai/en/docs/advanced/multi-agent-tracing):\nMulti-Agent Tracing\nObserve complex agent workflows and inter-agent communications with comprehensive tracing\nMulti-agent systems represent some of the most complex AI applications, involving multiple agents that coordinate, communicate, and collaborate to achieve shared goals. Noveum.ai provides specialized tracing capabilities to help you understand and optimize these intricate workflows.\n\ud83c\udfaf Why Multi-Agent Tracing Matters\nMulti-agent systems introduce unique observability challenges:\n- Complex...\n\nSource 4 (https://noveum.ai/en/docs/advanced/performance-optimization):\n\ud83d\udd17 Next Steps\n- Multi-Agent Tracing - Observe agent workflows\n- RAG Pipeline Observability - Monitor retrieval and generation systems\n- Custom Instrumentation - Add domain-specific tracing\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nSource 5 (https://noveum.ai/docs/advanced/performance-optimization):\n\ud83d\udd17 Next Steps\n- Multi-Agent Tracing - Observe agent workflows\n- RAG Pipeline Observability - Monitor retrieval and generation systems\n- Custom Instrumentation - Add domain-specific tracing\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nUser Question: How does Noveum handle agent tracing?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", - "prompt.prompt_length": 3269, - "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, ...", - "prompt.instruction_following": "rag_optimized", - "response.answer_length": 663, - "response.answer_completeness": "complete", - "response.response_quality": "high", - "response.source_citation": 5, - "response.context_utilization": 0.24232456140350878, - "output_response": "RAG Answer: Noveum.ai handles agent tracing through its specialized Multi-Agent Tracing capabilities, which are designed to observe complex workflows and inter-agent communications within multi-agent systems. The...", - "evaluation.retrieval_effectiveness": 1, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 5, - "evaluation.context_relevance": "high", - "evaluation.overall_quality": "high", - "evaluation.ready_for_production": true, - "rag.retrieval_strategy": "semantic_similarity", - "rag.vector_search_results": 5, - "rag.context_synthesis": "multi_source", - "rag.document_coverage": 1, - "rag.information_density": 0.24232456140350878 - }, - "events": [], - "links": [], - "trace_trace_id": "3bbd0b36-c30f-4126-b62a-dd0fae1c701d", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:19.339000000", - "trace_end_time": "2025-10-22 16:57:23.822000000", - "trace_duration_ms": 4483.158, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:19.339000000", - "updated_at": "2025-10-22 16:57:19.339000000" - }, - { - "span_id": "93d82f68-6c62-40de-a675-c4d7c40c1d01", - "trace_id": "fee3e6d3-8da1-4ef3-8214-a7d7323d3a49", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-22 16:57:13.334000000", - "end_time": "2025-10-22 16:57:18.330000000", - "duration_ms": 4996.436, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What APIs are available in Noveum platform?", - "query.length": 43, - "processing.mode": "RAG", - "processing.time_seconds": 4.996389865875244, - "processing.response_length": 270, - "processing.sources_count": 5, - "output_response": "Final Answer: The provided context does not specify the APIs available in the Noveum platform. For detailed information about the APIs, I recommend checking the official Noveum documentation or the SDK Integration ...", - "final_answer_mode": "RAG", - "query_processed.input_query": "What APIs are available in Noveum platform?", - "query_processed.output_response": "Successfully processed query using RAG, generated 270 character response", - "query_processed.mode": "RAG", - "query_processed.processing_time": 4.996389865875244, - "query_processed.response_length": 270 - }, - "events": [], - "links": [], - "trace_trace_id": "fee3e6d3-8da1-4ef3-8214-a7d7323d3a49", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:13.334000000", - "trace_end_time": "2025-10-22 16:57:18.331000000", - "trace_duration_ms": 4996.502, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:13.334000000", - "updated_at": "2025-10-22 16:57:13.334000000" - }, - { - "span_id": "1be727c6-14e8-4b2e-b8df-a1cab902454f", - "trace_id": "fee3e6d3-8da1-4ef3-8214-a7d7323d3a49", - "parent_span_id": "93d82f68-6c62-40de-a675-c4d7c40c1d01", - "name": "agent.query_routing", - "start_time": "2025-10-22 16:57:13.334000000", - "end_time": "2025-10-22 16:57:18.330000000", - "duration_ms": 4996.354, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What APIs are available in Noveum platform?", - "query_length": 43, - "query_type": "routing_query", - "output_response": "Routed to RAG for query processing", - "router.classification": "RAG", - "router.confidence_score": 1, - "router.rag_keyword_score": 3, - "router.web_keyword_score": 0, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "fee3e6d3-8da1-4ef3-8214-a7d7323d3a49", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:13.334000000", - "trace_end_time": "2025-10-22 16:57:18.331000000", - "trace_duration_ms": 4996.502, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:13.334000000", - "updated_at": "2025-10-22 16:57:13.334000000" - }, - { - "span_id": "58ab1d39-d349-4164-8ff9-26ece4b1e1aa", - "trace_id": "fee3e6d3-8da1-4ef3-8214-a7d7323d3a49", - "parent_span_id": "1be727c6-14e8-4b2e-b8df-a1cab902454f", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-22 16:57:13.334000000", - "end_time": "2025-10-22 16:57:13.334000000", - "duration_ms": 0.041999999999999996, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What APIs are available in Noveum platform?", - "query_length": 43, - "query_type": "routing_evaluation_query", - "classification.mode": "RAG", - "classification.rag_keyword_score": 3, - "classification.web_keyword_score": 0, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "noveum_specific", - "query.keyword_density": 0.42857142857142855, - "query.domain_affinity": "noveum", - "routing.decision": "Routed to RAG based on analysis", - "routing.rationale": "RAG score: 3, Web score: 0, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "Web Search", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.057692307692307696, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: RAG (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "fee3e6d3-8da1-4ef3-8214-a7d7323d3a49", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:13.334000000", - "trace_end_time": "2025-10-22 16:57:18.331000000", - "trace_duration_ms": 4996.502, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:13.334000000", - "updated_at": "2025-10-22 16:57:13.334000000" - }, - { - "span_id": "f11ebf68-a416-484e-8e55-f77a11b9ea1c", - "trace_id": "fee3e6d3-8da1-4ef3-8214-a7d7323d3a49", - "parent_span_id": "1be727c6-14e8-4b2e-b8df-a1cab902454f", - "name": "agent.llm-rag", - "start_time": "2025-10-22 16:57:13.334000000", - "end_time": "2025-10-22 16:57:18.330000000", - "duration_ms": 4996.146000000001, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "rag_agent", - "agent.operation": "llm-rag", - "agent.capabilities": [ - "document_retrieval", - "context_generation", - "response_generation" - ], - "agent.id": "noveum_rag_agent", - "input_query": "What APIs are available in Noveum platform?", - "query_length": 43, - "query_type": "rag_query", - "output_response": "RAG Answer: The provided context does not specify the APIs available in the Noveum platform. For detailed information about the APIs, I recommend checking the official Noveum documentation or the SDK Integration ...", - "rag.context_length": 2284, - "rag.sources_count": 5, - "rag.answer_length": 270, - "rag.mode": "retrieval_augmented_generation" - }, - "events": [], - "links": [], - "trace_trace_id": "fee3e6d3-8da1-4ef3-8214-a7d7323d3a49", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:13.334000000", - "trace_end_time": "2025-10-22 16:57:18.331000000", - "trace_duration_ms": 4996.502, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:13.334000000", - "updated_at": "2025-10-22 16:57:13.334000000" - }, - { - "span_id": "11394488-8802-403d-a926-7fda3a510b61", - "trace_id": "fee3e6d3-8da1-4ef3-8214-a7d7323d3a49", - "parent_span_id": "f11ebf68-a416-484e-8e55-f77a11b9ea1c", - "name": "agent.llm_model_execution", - "start_time": "2025-10-22 16:57:13.862000000", - "end_time": "2025-10-22 16:57:16.049000000", - "duration_ms": 2187.29, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What APIs are available in Noveum platform?", - "query_length": 43, - "query_type": "rag_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 2.187239170074463, - "model.latency_ms": 2187.239170074463, - "model.start_time": 1761152233.862141, - "model.end_time": 1761152236.04938, - "model.performance_tier": "medium", - "model.prompt_tokens": 705, - "model.completion_tokens": 67, - "model.total_tokens": 772, - "model.tokens_per_second": 352.9563710098141, - "model.estimated_cost": 0.02316, - "model.efficiency_score": 0.34974093264248707, - "model.response_length": 270, - "model.response_quality": "high", - "model.output_response": "Model Response: The provided context does not specify the APIs available in the Noveum platform. For detailed information about the APIs, I recommend checking the official Noveum documentation or the SDK Integration ...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "fee3e6d3-8da1-4ef3-8214-a7d7323d3a49", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:13.334000000", - "trace_end_time": "2025-10-22 16:57:18.331000000", - "trace_duration_ms": 4996.502, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:13.334000000", - "updated_at": "2025-10-22 16:57:13.334000000" - }, - { - "span_id": "341cf8ab-cc33-4d57-952f-ba1065f0432e", - "trace_id": "fee3e6d3-8da1-4ef3-8214-a7d7323d3a49", - "parent_span_id": "f11ebf68-a416-484e-8e55-f77a11b9ea1c", - "name": "agent.rag_evaluation_metrics", - "start_time": "2025-10-22 16:57:16.049000000", - "end_time": "2025-10-22 16:57:16.049000000", - "duration_ms": 0.041, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "rag_evaluation_metrics", - "agent.capabilities": [ - "retrieval_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What APIs are available in Noveum platform?", - "query_length": 43, - "query_type": "rag_evaluation_query", - "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/docs):\n- \ud83c\udf0d Universal: Works across frameworks, clouds, and deployment models\n\ud83e\udd1d Community & Support\n- \ud83d\udcac Discord Community: Join our Discord\n- \ud83d\udce7 Email Support: [email protected]\n- \ud83d\udc1b Bug Reports: GitHub Issues\n- \ud83d\udcd6 Knowledge Base: Help Center\nReady to get started? Head to ...", - "retrieval.context_length": 2284, - "retrieval.sources_count": 5, - "retrieval.context_quality": "high", - "retrieval.effectiveness": 1, - "retrieval.context_utilization": 2.284, - "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/docs):\n- \ud83c\udf0d Universal: Works across frameworks, clouds, and deployment models\n\ud83e\udd1d Community & Support\n- \ud83d\udcac Discord Community: Join our Discord\n- \ud83d\udce7 Email Support: [email protected]\n- \ud83d\udc1b Bug Reports: GitHub Issues\n- \ud83d\udcd6 Knowledge Base: Help Center\nReady to get started? Head to our SDK Integration Guide to begin tracing your AI applications in under 5 minutes!\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability ...\n\nSource 2 (https://noveum.ai/docs):\n- \ud83c\udf0d Universal: Works across frameworks, clouds, and deployment models\n\ud83e\udd1d Community & Support\n- \ud83d\udcac Discord Community: Join our Discord\n- \ud83d\udce7 Email Support: [email protected]\n- \ud83d\udc1b Bug Reports: GitHub Issues\n- \ud83d\udcd6 Knowledge Base: Help Center\nReady to get started? Head to our SDK Integration Guide to begin tracing your AI applications in under 5 minutes!\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability ...\n\nSource 3 (https://noveum.ai/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nSource 4 (https://noveum.ai/en/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nSource 5 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, vector searches, data retrieval, agent reasoning, and more. Without proper observability, debugging becomes a nightmare, optimization is guesswork, and understanding user interactions is nearly impossible.\nNoveum.ai solves this by providing comprehensive tracing and observability specifically designed for AI applications.\nKey benefits of using Noveum.ai:\n- Complete Visibility: Trace...\n\nUser Question: What APIs are available in Noveum platform?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", - "prompt.prompt_length": 2823, - "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/docs):\n- \ud83c\udf0d Universal: Works across frameworks, clouds, and deployment models\n\ud83e\udd1d Community & Support\n- \ud83d\udcac Discord Community: Join our Discord\n- \ud83d\udce7 Email Support: [email prot...", - "prompt.instruction_following": "rag_optimized", - "response.answer_length": 270, - "response.answer_completeness": "complete", - "response.response_quality": "high", - "response.source_citation": 5, - "response.context_utilization": 0.11821366024518389, - "output_response": "RAG Answer: The provided context does not specify the APIs available in the Noveum platform. For detailed information about the APIs, I recommend checking the official Noveum documentation or the SDK Integration ...", - "evaluation.retrieval_effectiveness": 1, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 5, - "evaluation.context_relevance": "high", - "evaluation.overall_quality": "high", - "evaluation.ready_for_production": true, - "rag.retrieval_strategy": "semantic_similarity", - "rag.vector_search_results": 5, - "rag.context_synthesis": "multi_source", - "rag.document_coverage": 1, - "rag.information_density": 0.11821366024518389 - }, - "events": [], - "links": [], - "trace_trace_id": "fee3e6d3-8da1-4ef3-8214-a7d7323d3a49", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:13.334000000", - "trace_end_time": "2025-10-22 16:57:18.331000000", - "trace_duration_ms": 4996.502, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:13.334000000", - "updated_at": "2025-10-22 16:57:13.334000000" - }, - { - "span_id": "24930c28-7f93-426c-8914-bd0e0085589f", - "trace_id": "93bd37c4-4289-43c9-b34e-19eaaef610c2", - "parent_span_id": "bfc91518-f818-4f35-9113-70b04eb55edc", - "name": "agent.query_routing", - "start_time": "2025-10-22 16:57:06.605000000", - "end_time": "2025-10-22 16:57:12.327000000", - "duration_ms": 5721.561000000001, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "How do I set up observability with Noveum?", - "query_length": 42, - "query_type": "routing_query", - "output_response": "Routed to RAG for query processing", - "router.classification": "RAG", - "router.confidence_score": 1, - "router.rag_keyword_score": 2, - "router.web_keyword_score": 0, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "93bd37c4-4289-43c9-b34e-19eaaef610c2", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:06.605000000", - "trace_end_time": "2025-10-22 16:57:12.327000000", - "trace_duration_ms": 5721.707, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:06.605000000", - "updated_at": "2025-10-22 16:57:06.605000000" - }, - { - "span_id": "11fc18a8-0380-48ca-894e-cc26f648d290", - "trace_id": "93bd37c4-4289-43c9-b34e-19eaaef610c2", - "parent_span_id": "24930c28-7f93-426c-8914-bd0e0085589f", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-22 16:57:06.605000000", - "end_time": "2025-10-22 16:57:06.606000000", - "duration_ms": 0.047, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: How do I set up observability with Noveum?", - "query_length": 42, - "query_type": "routing_evaluation_query", - "classification.mode": "RAG", - "classification.rag_keyword_score": 2, - "classification.web_keyword_score": 0, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "noveum_specific", - "query.keyword_density": 0.25, - "query.domain_affinity": "noveum", - "routing.decision": "Routed to RAG based on analysis", - "routing.rationale": "RAG score: 2, Web score: 0, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "Web Search", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.038461538461538464, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: RAG (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "93bd37c4-4289-43c9-b34e-19eaaef610c2", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:06.605000000", - "trace_end_time": "2025-10-22 16:57:12.327000000", - "trace_duration_ms": 5721.707, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:06.605000000", - "updated_at": "2025-10-22 16:57:06.605000000" - }, - { - "span_id": "bfc91518-f818-4f35-9113-70b04eb55edc", - "trace_id": "93bd37c4-4289-43c9-b34e-19eaaef610c2", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-22 16:57:06.605000000", - "end_time": "2025-10-22 16:57:12.327000000", - "duration_ms": 5721.635, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "How do I set up observability with Noveum?", - "query.length": 42, - "processing.mode": "RAG", - "processing.time_seconds": 5.721601963043213, - "processing.response_length": 815, - "processing.sources_count": 5, - "output_response": "Final Answer: To set up observability with Noveum, follow these steps:\n\n1. **Sign Up**: Go to [noveum.ai](https://noveum.ai) and create an account.\n2. **Create a Project**: After signing up, create a project to obt...", - "final_answer_mode": "RAG", - "query_processed.input_query": "How do I set up observability with Noveum?", - "query_processed.output_response": "Successfully processed query using RAG, generated 815 character response", - "query_processed.mode": "RAG", - "query_processed.processing_time": 5.721601963043213, - "query_processed.response_length": 815 - }, - "events": [], - "links": [], - "trace_trace_id": "93bd37c4-4289-43c9-b34e-19eaaef610c2", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:06.605000000", - "trace_end_time": "2025-10-22 16:57:12.327000000", - "trace_duration_ms": 5721.707, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:06.605000000", - "updated_at": "2025-10-22 16:57:06.605000000" - }, - { - "span_id": "2a82e4b1-c93d-498f-9dd5-e971f228d642", - "trace_id": "93bd37c4-4289-43c9-b34e-19eaaef610c2", - "parent_span_id": "24930c28-7f93-426c-8914-bd0e0085589f", - "name": "agent.llm-rag", - "start_time": "2025-10-22 16:57:06.606000000", - "end_time": "2025-10-22 16:57:12.327000000", - "duration_ms": 5721.352000000001, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "rag_agent", - "agent.operation": "llm-rag", - "agent.capabilities": [ - "document_retrieval", - "context_generation", - "response_generation" - ], - "agent.id": "noveum_rag_agent", - "input_query": "How do I set up observability with Noveum?", - "query_length": 42, - "query_type": "rag_query", - "output_response": "RAG Answer: To set up observability with Noveum, follow these steps:\n\n1. **Sign Up**: Go to [noveum.ai](https://noveum.ai) and create an account.\n2. **Create a Project**: After signing up, create a project to obt...", - "rag.context_length": 2870, - "rag.sources_count": 6, - "rag.answer_length": 815, - "rag.mode": "retrieval_augmented_generation" - }, - "events": [], - "links": [], - "trace_trace_id": "93bd37c4-4289-43c9-b34e-19eaaef610c2", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:06.605000000", - "trace_end_time": "2025-10-22 16:57:12.327000000", - "trace_duration_ms": 5721.707, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:06.605000000", - "updated_at": "2025-10-22 16:57:06.605000000" - }, - { - "span_id": "520b4e59-da14-4db2-af02-0973cda9bd86", - "trace_id": "93bd37c4-4289-43c9-b34e-19eaaef610c2", - "parent_span_id": "2a82e4b1-c93d-498f-9dd5-e971f228d642", - "name": "agent.llm_model_execution", - "start_time": "2025-10-22 16:57:07.109000000", - "end_time": "2025-10-22 16:57:11.837000000", - "duration_ms": 4728.018999999999, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: How do I set up observability with Noveum?", - "query_length": 42, - "query_type": "rag_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 4.727963924407959, - "model.latency_ms": 4727.963924407959, - "model.start_time": 1761152227.109188, - "model.end_time": 1761152231.837152, - "model.performance_tier": "medium", - "model.prompt_tokens": 852, - "model.completion_tokens": 203, - "model.total_tokens": 1055, - "model.tokens_per_second": 223.1404504915101, - "model.estimated_cost": 0.03165, - "model.efficiency_score": 0.7725118483412322, - "model.response_length": 815, - "model.response_quality": "high", - "model.output_response": "Model Response: To set up observability with Noveum, follow these steps:\n\n1. **Sign Up**: Go to [noveum.ai](https://noveum.ai) and create an account.\n2. **Create a Project**: After signing up, create a project to obt...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "93bd37c4-4289-43c9-b34e-19eaaef610c2", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:06.605000000", - "trace_end_time": "2025-10-22 16:57:12.327000000", - "trace_duration_ms": 5721.707, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:06.605000000", - "updated_at": "2025-10-22 16:57:06.605000000" - }, - { - "span_id": "dff080ab-238c-4cca-a910-cf530f910bda", - "trace_id": "93bd37c4-4289-43c9-b34e-19eaaef610c2", - "parent_span_id": "2a82e4b1-c93d-498f-9dd5-e971f228d642", - "name": "agent.rag_evaluation_metrics", - "start_time": "2025-10-22 16:57:11.837000000", - "end_time": "2025-10-22 16:57:11.838000000", - "duration_ms": 0.259, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "rag_evaluation_metrics", - "agent.capabilities": [ - "retrieval_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: How do I set up observability with Noveum?", - "query_length": 42, - "query_type": "rag_evaluation_query", - "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\n- Production Ready: Built for scale with intelligent sampling and batching\n- Privacy Focused: You control what data is captured and how it's stored\n- Open Source SDKs: Transparent, auditable, and extensible\nGetting ...", - "retrieval.context_length": 2870, - "retrieval.sources_count": 6, - "retrieval.context_quality": "high", - "retrieval.effectiveness": 1.2, - "retrieval.context_utilization": 2.87, - "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\n- Production Ready: Built for scale with intelligent sampling and batching\n- Privacy Focused: You control what data is captured and how it's stored\n- Open Source SDKs: Transparent, auditable, and extensible\nGetting Started\nReady to add comprehensive observability to your AI applications?\n- Sign up at noveum.ai\n- Create a project and get your API key\n- Install the SDK for your preferred language\n- Add tracing to your AI workflows\n- Explore insights in the Noveum.ai dashboard\nCheck out our integra...\n\nSource 2 (https://noveum.ai/en/docs/getting-started/tracing-concepts):\n- Explore Framework Integrations - Framework-specific guidance\n- Learn Advanced Patterns - Custom instrumentation techniques\n- Master the Dashboard - Analyze your traces effectively\nRemember: Good observability is not about collecting all possible data, but about collecting the right data that helps you understand, debug, and optimize your AI applications.\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to O...\n\nSource 3 (https://noveum.ai/docs/getting-started/tracing-concepts):\n- Explore Framework Integrations - Framework-specific guidance\n- Learn Advanced Patterns - Custom instrumentation techniques\n- Master the Dashboard - Analyze your traces effectively\nRemember: Good observability is not about collecting all possible data, but about collecting the right data that helps you understand, debug, and optimize your AI applications.\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to O...\n\nSource 4 (https://noveum.ai/en/docs/getting-started/overview):\nNoveum.ai Overview\nComprehensive AI tracing and observability platform for LLM applications, RAG systems, and AI agents\nWelcome to Noveum.ai\u2014the comprehensive tracing and observability platform built specifically for AI applications. Whether you're building LLM-powered chatbots, RAG systems, multi-agent workflows, or any AI-driven application, Noveum provides the insights you need to understand, debug, and optimize your systems.\n\ud83c\udfaf Why AI Applications Need Specialized Observability\nTraditional mo...\n\nSource 5 (https://noveum.ai/docs/getting-started/overview):\nNoveum.ai Overview\nComprehensive AI tracing and observability platform for LLM applications, RAG systems, and AI agents\nWelcome to Noveum.ai\u2014the comprehensive tracing and observability platform built specifically for AI applications. Whether you're building LLM-powered chatbots, RAG systems, multi-agent workflows, or any AI-driven application, Noveum provides the insights you need to understand, debug, and optimize your systems.\n\ud83c\udfaf Why AI Applications Need Specialized Observability\nTraditional mo...\n\nUser Question: How do I set up observability with Noveum?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", - "prompt.prompt_length": 3408, - "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\n- Production Ready: Built for scale with intelligent sampling and batching\n- Privacy Focused: You control what data...", - "prompt.instruction_following": "rag_optimized", - "response.answer_length": 815, - "response.answer_completeness": "complete", - "response.response_quality": "high", - "response.source_citation": 6, - "response.context_utilization": 0.28397212543554007, - "output_response": "RAG Answer: To set up observability with Noveum, follow these steps:\n\n1. **Sign Up**: Go to [noveum.ai](https://noveum.ai) and create an account.\n2. **Create a Project**: After signing up, create a project to obt...", - "evaluation.retrieval_effectiveness": 1.2, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 6, - "evaluation.context_relevance": "high", - "evaluation.overall_quality": "high", - "evaluation.ready_for_production": true, - "rag.retrieval_strategy": "semantic_similarity", - "rag.vector_search_results": 6, - "rag.context_synthesis": "multi_source", - "rag.document_coverage": 1.2, - "rag.information_density": 0.28397212543554007 - }, - "events": [], - "links": [], - "trace_trace_id": "93bd37c4-4289-43c9-b34e-19eaaef610c2", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:06.605000000", - "trace_end_time": "2025-10-22 16:57:12.327000000", - "trace_duration_ms": 5721.707, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:06.605000000", - "updated_at": "2025-10-22 16:57:06.605000000" - }, - { - "span_id": "dad7df65-be49-4245-8721-c91da4962b9a", - "trace_id": "df0c6344-1a2e-4c53-bb7d-4e43b3bce1b8", - "parent_span_id": "9024b0d9-2b75-428f-af8f-75a921a7c69c", - "name": "agent.query_routing", - "start_time": "2025-10-22 16:56:59.213000000", - "end_time": "2025-10-22 16:57:05.598000000", - "duration_ms": 6384.249, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What features does Noveum Trace offer?", - "query_length": 38, - "query_type": "routing_query", - "output_response": "Routed to RAG for query processing", - "router.classification": "RAG", - "router.confidence_score": 1, - "router.rag_keyword_score": 3, - "router.web_keyword_score": 0, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "df0c6344-1a2e-4c53-bb7d-4e43b3bce1b8", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:56:59.213000000", - "trace_end_time": "2025-10-22 16:57:05.598000000", - "trace_duration_ms": 6384.386, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:56:59.213000000", - "updated_at": "2025-10-22 16:56:59.213000000" - }, - { - "span_id": "9024b0d9-2b75-428f-af8f-75a921a7c69c", - "trace_id": "df0c6344-1a2e-4c53-bb7d-4e43b3bce1b8", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-22 16:56:59.213000000", - "end_time": "2025-10-22 16:57:05.598000000", - "duration_ms": 6384.32, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What features does Noveum Trace offer?", - "query.length": 38, - "processing.mode": "RAG", - "processing.time_seconds": 6.384284973144531, - "processing.response_length": 1017, - "processing.sources_count": 5, - "output_response": "Final Answer: Noveum Trace offers several key features through its Python SDK (`noveum-trace`) and TypeScript SDK (`@noveum/trace`):\n\n1. **Decorator-based Tracing**: It provides a simple way to trace LLM calls, age...", - "final_answer_mode": "RAG", - "query_processed.input_query": "What features does Noveum Trace offer?", - "query_processed.output_response": "Successfully processed query using RAG, generated 1017 character response", - "query_processed.mode": "RAG", - "query_processed.processing_time": 6.384284973144531, - "query_processed.response_length": 1017 - }, - "events": [], - "links": [], - "trace_trace_id": "df0c6344-1a2e-4c53-bb7d-4e43b3bce1b8", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:56:59.213000000", - "trace_end_time": "2025-10-22 16:57:05.598000000", - "trace_duration_ms": 6384.386, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:56:59.213000000", - "updated_at": "2025-10-22 16:56:59.213000000" - }, - { - "span_id": "d07fdbdc-2b8f-4de9-ac32-988389701771", - "trace_id": "df0c6344-1a2e-4c53-bb7d-4e43b3bce1b8", - "parent_span_id": "dad7df65-be49-4245-8721-c91da4962b9a", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-22 16:56:59.214000000", - "end_time": "2025-10-22 16:56:59.214000000", - "duration_ms": 0.060000000000000005, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What features does Noveum Trace offer?", - "query_length": 38, - "query_type": "routing_evaluation_query", - "classification.mode": "RAG", - "classification.rag_keyword_score": 3, - "classification.web_keyword_score": 0, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "noveum_specific", - "query.keyword_density": 0.5, - "query.domain_affinity": "noveum", - "routing.decision": "Routed to RAG based on analysis", - "routing.rationale": "RAG score: 3, Web score: 0, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "Web Search", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.057692307692307696, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: RAG (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "df0c6344-1a2e-4c53-bb7d-4e43b3bce1b8", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:56:59.213000000", - "trace_end_time": "2025-10-22 16:57:05.598000000", - "trace_duration_ms": 6384.386, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:56:59.213000000", - "updated_at": "2025-10-22 16:56:59.213000000" - }, - { - "span_id": "83f86157-5af4-4c99-aad1-647866cd804e", - "trace_id": "df0c6344-1a2e-4c53-bb7d-4e43b3bce1b8", - "parent_span_id": "dad7df65-be49-4245-8721-c91da4962b9a", - "name": "agent.llm-rag", - "start_time": "2025-10-22 16:56:59.214000000", - "end_time": "2025-10-22 16:57:05.598000000", - "duration_ms": 6384.048, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "rag_agent", - "agent.operation": "llm-rag", - "agent.capabilities": [ - "document_retrieval", - "context_generation", - "response_generation" - ], - "agent.id": "noveum_rag_agent", - "input_query": "What features does Noveum Trace offer?", - "query_length": 38, - "query_type": "rag_query", - "output_response": "RAG Answer: Noveum Trace offers several key features through its Python SDK (`noveum-trace`) and TypeScript SDK (`@noveum/trace`):\n\n1. **Decorator-based Tracing**: It provides a simple way to trace LLM calls, age...", - "rag.context_length": 2787, - "rag.sources_count": 5, - "rag.answer_length": 1017, - "rag.mode": "retrieval_augmented_generation" - }, - "events": [], - "links": [], - "trace_trace_id": "df0c6344-1a2e-4c53-bb7d-4e43b3bce1b8", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:56:59.213000000", - "trace_end_time": "2025-10-22 16:57:05.598000000", - "trace_duration_ms": 6384.386, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:56:59.213000000", - "updated_at": "2025-10-22 16:56:59.213000000" - }, - { - "span_id": "3b4fa9a7-88bf-4ee1-aeb7-bfa9573f28f2", - "trace_id": "df0c6344-1a2e-4c53-bb7d-4e43b3bce1b8", - "parent_span_id": "83f86157-5af4-4c99-aad1-647866cd804e", - "name": "agent.llm_model_execution", - "start_time": "2025-10-22 16:56:59.591000000", - "end_time": "2025-10-22 16:57:04.869000000", - "duration_ms": 5278.925, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What features does Noveum Trace offer?", - "query_length": 38, - "query_type": "rag_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 5.278868913650513, - "model.latency_ms": 5278.868913650513, - "model.start_time": 1761152219.5910342, - "model.end_time": 1761152224.869903, - "model.performance_tier": "slow", - "model.prompt_tokens": 830, - "model.completion_tokens": 254, - "model.total_tokens": 1084, - "model.tokens_per_second": 205.3470199263535, - "model.estimated_cost": 0.03252, - "model.efficiency_score": 0.9381918819188192, - "model.response_length": 1017, - "model.response_quality": "high", - "model.output_response": "Model Response: Noveum Trace offers several key features through its Python SDK (`noveum-trace`) and TypeScript SDK (`@noveum/trace`):\n\n1. **Decorator-based Tracing**: It provides a simple way to trace LLM calls, age...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "df0c6344-1a2e-4c53-bb7d-4e43b3bce1b8", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:56:59.213000000", - "trace_end_time": "2025-10-22 16:57:05.598000000", - "trace_duration_ms": 6384.386, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:56:59.213000000", - "updated_at": "2025-10-22 16:56:59.213000000" - }, - { - "span_id": "f9214dbe-adb9-44ff-8361-70c872cef66d", - "trace_id": "df0c6344-1a2e-4c53-bb7d-4e43b3bce1b8", - "parent_span_id": "83f86157-5af4-4c99-aad1-647866cd804e", - "name": "agent.rag_evaluation_metrics", - "start_time": "2025-10-22 16:57:04.870000000", - "end_time": "2025-10-22 16:57:04.870000000", - "duration_ms": 0.043000000000000003, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "rag_evaluation_metrics", - "agent.capabilities": [ - "retrieval_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What features does Noveum Trace offer?", - "query_length": 38, - "query_type": "rag_evaluation_query", - "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\n\ud83d\udcd8 TypeScript SDK (\n@noveum/trace\n)- Framework ...", - "retrieval.context_length": 2787, - "retrieval.sources_count": 5, - "retrieval.context_quality": "high", - "retrieval.effectiveness": 1, - "retrieval.context_utilization": 2.787, - "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\n\ud83d\udcd8 TypeScript SDK (\n@noveum/trace\n)- Framework integrations for Next.js, Express.js, and Hono\n- TypeScript-first design with full type safety\n- Universal compatibility (Node.js, Edge Runtime, browsers)\n-\n\ud83d\udcca Noveum Platform\n- Real-time dashboard for analyzing traces and performance\n- Ad...\n\nSource 2 (https://noveum.ai/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\n\ud83d\udcd8 TypeScript SDK (\n@noveum/trace\n)- Framework integrations for Next.js, Express.js, and Hono\n- TypeScript-first design with full type safety\n- Universal compatibility (Node.js, Edge Runtime, browsers)\n-\n\ud83d\udcca Noveum Platform\n- Real-time dashboard for analyzing traces and performance\n- Ad...\n\nSource 3 (https://noveum.ai/en/docs/getting-started/sdk-integration):\n- \ud83d\udc65 Collaborate with your team on insights\n\ud83d\udd12 Security & Privacy\n- \ud83d\udd10 Encryption: All data encrypted in transit and at rest\n- \ud83c\udf9b\ufe0f Configurable Capture: Control what data is collected\n- \ud83c\udfe0 Data Residency: Choose your data storage region\n- \u23f0 Retention Control: Set custom data retention policies\nNext Steps\n- Tracing Concepts - Learn about traces, spans, and observability best practices\n- Framework Integrations - Deep dive into specific framework setups\n- Multi-Agent Tracing - Observe complex agent work...\n\nSource 4 (https://noveum.ai/docs/getting-started/sdk-integration):\n- \ud83d\udc65 Collaborate with your team on insights\n\ud83d\udd12 Security & Privacy\n- \ud83d\udd10 Encryption: All data encrypted in transit and at rest\n- \ud83c\udf9b\ufe0f Configurable Capture: Control what data is collected\n- \ud83c\udfe0 Data Residency: Choose your data storage region\n- \u23f0 Retention Control: Set custom data retention policies\nNext Steps\n- Tracing Concepts - Learn about traces, spans, and observability best practices\n- Framework Integrations - Deep dive into specific framework setups\n- Multi-Agent Tracing - Observe complex agent work...\n\nSource 5 (https://noveum.ai/docs/platform/dashboard):\n- Bulk Export: Download multiple traces simultaneously\n- Comparative Analysis: Compare performance across multiple traces\nReal-Time Updates\n- Live Refresh: Automatic updates as new traces arrive\n- Connection Monitoring: Real-time status of your trace ingestion pipeline\n- Performance Indicators: Live metrics for system health monitoring\nReady to dive deeper? Explore Projects & Environments to organize your AI applications, or check out Team Collaboration to share insights with your team.\nGet Earl...\n\nUser Question: What features does Noveum Trace offer?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", - "prompt.prompt_length": 3321, - "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frame...", - "prompt.instruction_following": "rag_optimized", - "response.answer_length": 1017, - "response.answer_completeness": "complete", - "response.response_quality": "high", - "response.source_citation": 5, - "response.context_utilization": 0.36490850376749195, - "output_response": "RAG Answer: Noveum Trace offers several key features through its Python SDK (`noveum-trace`) and TypeScript SDK (`@noveum/trace`):\n\n1. **Decorator-based Tracing**: It provides a simple way to trace LLM calls, age...", - "evaluation.retrieval_effectiveness": 1, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 5, - "evaluation.context_relevance": "high", - "evaluation.overall_quality": "high", - "evaluation.ready_for_production": true, - "rag.retrieval_strategy": "semantic_similarity", - "rag.vector_search_results": 5, - "rag.context_synthesis": "multi_source", - "rag.document_coverage": 1, - "rag.information_density": 0.36490850376749195 - }, - "events": [], - "links": [], - "trace_trace_id": "df0c6344-1a2e-4c53-bb7d-4e43b3bce1b8", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:56:59.213000000", - "trace_end_time": "2025-10-22 16:57:05.598000000", - "trace_duration_ms": 6384.386, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:56:59.213000000", - "updated_at": "2025-10-22 16:56:59.213000000" - }, - { - "span_id": "138a5089-5268-447a-afb5-4f14fb504281", - "trace_id": "29a862bf-d7aa-4cf9-963c-0a4244343bbe", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-22 16:56:54.719000000", - "end_time": "2025-10-22 16:56:58.207000000", - "duration_ms": 3487.959, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What are Noveum's pricing plans?", - "query.length": 32, - "processing.mode": "RAG", - "processing.time_seconds": 3.487928867340088, - "processing.response_length": 441, - "processing.sources_count": 5, - "output_response": "Final Answer: The provided context does not include specific information about Noveum's pricing plans. It mentions that all users will have access to the Observability suite for free, and early users will receive f...", - "final_answer_mode": "RAG", - "query_processed.input_query": "What are Noveum's pricing plans?", - "query_processed.output_response": "Successfully processed query using RAG, generated 441 character response", - "query_processed.mode": "RAG", - "query_processed.processing_time": 3.487928867340088, - "query_processed.response_length": 441 - }, - "events": [], - "links": [], - "trace_trace_id": "29a862bf-d7aa-4cf9-963c-0a4244343bbe", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:56:54.718000000", - "trace_end_time": "2025-10-22 16:56:58.207000000", - "trace_duration_ms": 3488.068, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:56:54.718000000", - "updated_at": "2025-10-22 16:56:54.718000000" - }, - { - "span_id": "7cda4dab-0fc5-4cae-8d8c-9445f661c67b", - "trace_id": "29a862bf-d7aa-4cf9-963c-0a4244343bbe", - "parent_span_id": "138a5089-5268-447a-afb5-4f14fb504281", - "name": "agent.query_routing", - "start_time": "2025-10-22 16:56:54.719000000", - "end_time": "2025-10-22 16:56:58.207000000", - "duration_ms": 3487.848, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What are Noveum's pricing plans?", - "query_length": 32, - "query_type": "routing_query", - "output_response": "Routed to RAG for query processing", - "router.classification": "RAG", - "router.confidence_score": 1, - "router.rag_keyword_score": 3, - "router.web_keyword_score": 0, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "29a862bf-d7aa-4cf9-963c-0a4244343bbe", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:56:54.718000000", - "trace_end_time": "2025-10-22 16:56:58.207000000", - "trace_duration_ms": 3488.068, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:56:54.718000000", - "updated_at": "2025-10-22 16:56:54.718000000" - }, - { - "span_id": "922c84a5-b14a-4b2f-8759-427d6e089812", - "trace_id": "29a862bf-d7aa-4cf9-963c-0a4244343bbe", - "parent_span_id": "7cda4dab-0fc5-4cae-8d8c-9445f661c67b", - "name": "agent.llm-rag", - "start_time": "2025-10-22 16:56:54.719000000", - "end_time": "2025-10-22 16:56:58.207000000", - "duration_ms": 3487.546, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "rag_agent", - "agent.operation": "llm-rag", - "agent.capabilities": [ - "document_retrieval", - "context_generation", - "response_generation" - ], - "agent.id": "noveum_rag_agent", - "input_query": "What are Noveum's pricing plans?", - "query_length": 32, - "query_type": "rag_query", - "output_response": "RAG Answer: The provided context does not include specific information about Noveum's pricing plans. It mentions that all users will have access to the Observability suite for free, and early users will receive f...", - "rag.context_length": 2105, - "rag.sources_count": 5, - "rag.answer_length": 441, - "rag.mode": "retrieval_augmented_generation" - }, - "events": [], - "links": [], - "trace_trace_id": "29a862bf-d7aa-4cf9-963c-0a4244343bbe", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:56:54.718000000", - "trace_end_time": "2025-10-22 16:56:58.207000000", - "trace_duration_ms": 3488.068, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:56:54.718000000", - "updated_at": "2025-10-22 16:56:54.718000000" - }, - { - "span_id": "c8d7bb28-e763-47c9-b3e9-77f6a91a22be", - "trace_id": "29a862bf-d7aa-4cf9-963c-0a4244343bbe", - "parent_span_id": "7cda4dab-0fc5-4cae-8d8c-9445f661c67b", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-22 16:56:54.719000000", - "end_time": "2025-10-22 16:56:54.719000000", - "duration_ms": 0.068, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What are Noveum's pricing plans?", - "query_length": 32, - "query_type": "routing_evaluation_query", - "classification.mode": "RAG", - "classification.rag_keyword_score": 3, - "classification.web_keyword_score": 0, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "noveum_specific", - "query.keyword_density": 0.6, - "query.domain_affinity": "noveum", - "routing.decision": "Routed to RAG based on analysis", - "routing.rationale": "RAG score: 3, Web score: 0, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "Web Search", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.057692307692307696, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: RAG (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "29a862bf-d7aa-4cf9-963c-0a4244343bbe", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:56:54.718000000", - "trace_end_time": "2025-10-22 16:56:58.207000000", - "trace_duration_ms": 3488.068, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:56:54.718000000", - "updated_at": "2025-10-22 16:56:54.718000000" - }, - { - "span_id": "a378123a-8a79-4bc5-8e20-8ec54c2f9783", - "trace_id": "29a862bf-d7aa-4cf9-963c-0a4244343bbe", - "parent_span_id": "922c84a5-b14a-4b2f-8759-427d6e089812", - "name": "agent.llm_model_execution", - "start_time": "2025-10-22 16:56:55.154000000", - "end_time": "2025-10-22 16:56:57.166000000", - "duration_ms": 2012.174, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What are Noveum's pricing plans?", - "query_length": 32, - "query_type": "rag_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 2.012089252471924, - "model.latency_ms": 2012.0892524719238, - "model.start_time": 1761152215.154625, - "model.end_time": 1761152217.1667142, - "model.performance_tier": "medium", - "model.prompt_tokens": 658, - "model.completion_tokens": 110, - "model.total_tokens": 768, - "model.tokens_per_second": 381.6928096288395, - "model.estimated_cost": 0.02304, - "model.efficiency_score": 0.57421875, - "model.response_length": 441, - "model.response_quality": "high", - "model.output_response": "Model Response: The provided context does not include specific information about Noveum's pricing plans. It mentions that all users will have access to the Observability suite for free, and early users will receive f...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "29a862bf-d7aa-4cf9-963c-0a4244343bbe", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:56:54.718000000", - "trace_end_time": "2025-10-22 16:56:58.207000000", - "trace_duration_ms": 3488.068, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:56:54.718000000", - "updated_at": "2025-10-22 16:56:54.718000000" - }, - { - "span_id": "5813673b-a1dc-46c7-9bea-92ce6aca2b5e", - "trace_id": "29a862bf-d7aa-4cf9-963c-0a4244343bbe", - "parent_span_id": "922c84a5-b14a-4b2f-8759-427d6e089812", - "name": "agent.rag_evaluation_metrics", - "start_time": "2025-10-22 16:56:57.166000000", - "end_time": "2025-10-22 16:56:57.167000000", - "duration_ms": 0.059, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "rag_evaluation_metrics", - "agent.capabilities": [ - "retrieval_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What are Noveum's pricing plans?", - "query_length": 32, - "query_type": "rag_evaluation_query", - "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....", - "retrieval.context_length": 2105, - "retrieval.sources_count": 5, - "retrieval.context_quality": "high", - "retrieval.effectiveness": 1, - "retrieval.context_utilization": 2.105, - "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nSource 2 (https://noveum.ai/en/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nSource 3 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nThe AI landscape is complex and moving fast. With Noveum.ai, you'll have the insights you need to build, debug, and optimize AI applications with confidence.\nReady to see what's really happening inside your AI applications? Try Noveum.ai today and transform how you build and monitor AI systems.\nLet's build more reliable, observable AI\u2014together.\nGet Early Access to Noveum.ai Platform\nJoin the select group of AI teams optimizing their models with our data-driven platform. We're onboarding users in...\n\nSource 4 (https://noveum.ai/en/blog/from-logs-to-intelligent-choices-inside-noveum-ais-evaluation-process):\nGet Early Access to Noveum.ai Platform\nJoin the select group of AI teams optimizing their models with our data-driven platform. We're onboarding users in limited batches to ensure a premium experience....\n\nSource 5 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, vector searches, data retrieval, agent reasoning, and more. Without proper observability, debugging becomes a nightmare, optimization is guesswork, and understanding user interactions is nearly impossible.\nNoveum.ai solves this by providing comprehensive tracing and observability specifically designed for AI applications.\nKey benefits of using Noveum.ai:\n- Complete Visibility: Trace...\n\nUser Question: What are Noveum's pricing plans?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", - "prompt.prompt_length": 2633, - "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to ...", - "prompt.instruction_following": "rag_optimized", - "response.answer_length": 441, - "response.answer_completeness": "complete", - "response.response_quality": "high", - "response.source_citation": 5, - "response.context_utilization": 0.20950118764845604, - "output_response": "RAG Answer: The provided context does not include specific information about Noveum's pricing plans. It mentions that all users will have access to the Observability suite for free, and early users will receive f...", - "evaluation.retrieval_effectiveness": 1, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 5, - "evaluation.context_relevance": "high", - "evaluation.overall_quality": "high", - "evaluation.ready_for_production": true, - "rag.retrieval_strategy": "semantic_similarity", - "rag.vector_search_results": 5, - "rag.context_synthesis": "multi_source", - "rag.document_coverage": 1, - "rag.information_density": 0.20950118764845604 - }, - "events": [], - "links": [], - "trace_trace_id": "29a862bf-d7aa-4cf9-963c-0a4244343bbe", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:56:54.718000000", - "trace_end_time": "2025-10-22 16:56:58.207000000", - "trace_duration_ms": 3488.068, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:56:54.718000000", - "updated_at": "2025-10-22 16:56:54.718000000" - }, - { - "span_id": "c5baf5c2-d18a-458a-b922-a19b96db53d9", - "trace_id": "bc60c122-f30b-4114-9f78-ee40a024b525", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-22 16:56:44.873000000", - "end_time": "2025-10-22 16:56:53.701000000", - "duration_ms": 8827.979, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "How do I integrate Noveum Trace in my application?", - "query.length": 50, - "processing.mode": "RAG", - "processing.time_seconds": 8.827935218811035, - "processing.response_length": 1145, - "processing.sources_count": 5, - "output_response": "Final Answer: To integrate Noveum Trace into your application, you can follow these steps based on the SDK Integration Guide:\n\n1. **Create Your Account & Get API Key**:\n - Sign up at [noveum.ai](https://noveum.ai...", - "final_answer_mode": "RAG", - "query_processed.input_query": "How do I integrate Noveum Trace in my application?", - "query_processed.output_response": "Successfully processed query using RAG, generated 1145 character response", - "query_processed.mode": "RAG", - "query_processed.processing_time": 8.827935218811035, - "query_processed.response_length": 1145 - }, - "events": [], - "links": [], - "trace_trace_id": "bc60c122-f30b-4114-9f78-ee40a024b525", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:56:44.873000000", - "trace_end_time": "2025-10-22 16:56:53.701000000", - "trace_duration_ms": 8828.055, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:56:44.873000000", - "updated_at": "2025-10-22 16:56:44.873000000" - }, - { - "span_id": "8819dde5-9cbe-4cdb-b919-9c5140582359", - "trace_id": "bc60c122-f30b-4114-9f78-ee40a024b525", - "parent_span_id": "c5baf5c2-d18a-458a-b922-a19b96db53d9", - "name": "agent.query_routing", - "start_time": "2025-10-22 16:56:44.873000000", - "end_time": "2025-10-22 16:56:53.701000000", - "duration_ms": 8827.883, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "How do I integrate Noveum Trace in my application?", - "query_length": 50, - "query_type": "routing_query", - "output_response": "Routed to RAG for query processing", - "router.classification": "RAG", - "router.confidence_score": 1, - "router.rag_keyword_score": 2, - "router.web_keyword_score": 0, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "bc60c122-f30b-4114-9f78-ee40a024b525", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:56:44.873000000", - "trace_end_time": "2025-10-22 16:56:53.701000000", - "trace_duration_ms": 8828.055, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:56:44.873000000", - "updated_at": "2025-10-22 16:56:44.873000000" - }, - { - "span_id": "932c4b0e-0f33-40c0-bb19-78b9f81e48a6", - "trace_id": "bc60c122-f30b-4114-9f78-ee40a024b525", - "parent_span_id": "8819dde5-9cbe-4cdb-b919-9c5140582359", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-22 16:56:44.873000000", - "end_time": "2025-10-22 16:56:44.873000000", - "duration_ms": 0.043000000000000003, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: How do I integrate Noveum Trace in my application?", - "query_length": 50, - "query_type": "routing_evaluation_query", - "classification.mode": "RAG", - "classification.rag_keyword_score": 2, - "classification.web_keyword_score": 0, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "noveum_specific", - "query.keyword_density": 0.2222222222222222, - "query.domain_affinity": "noveum", - "routing.decision": "Routed to RAG based on analysis", - "routing.rationale": "RAG score: 2, Web score: 0, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "Web Search", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.038461538461538464, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: RAG (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "bc60c122-f30b-4114-9f78-ee40a024b525", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:56:44.873000000", - "trace_end_time": "2025-10-22 16:56:53.701000000", - "trace_duration_ms": 8828.055, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:56:44.873000000", - "updated_at": "2025-10-22 16:56:44.873000000" - }, - { - "span_id": "2ece2bd6-e4c8-47bc-b3c8-26f82e9910f1", - "trace_id": "bc60c122-f30b-4114-9f78-ee40a024b525", - "parent_span_id": "8819dde5-9cbe-4cdb-b919-9c5140582359", - "name": "agent.llm-rag", - "start_time": "2025-10-22 16:56:44.873000000", - "end_time": "2025-10-22 16:56:53.701000000", - "duration_ms": 8827.666000000001, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "rag_agent", - "agent.operation": "llm-rag", - "agent.capabilities": [ - "document_retrieval", - "context_generation", - "response_generation" - ], - "agent.id": "noveum_rag_agent", - "input_query": "How do I integrate Noveum Trace in my application?", - "query_length": 50, - "query_type": "rag_query", - "output_response": "RAG Answer: To integrate Noveum Trace into your application, you can follow these steps based on the SDK Integration Guide:\n\n1. **Create Your Account & Get API Key**:\n - Sign up at [noveum.ai](https://noveum.ai...", - "rag.context_length": 2803, - "rag.sources_count": 5, - "rag.answer_length": 1145, - "rag.mode": "retrieval_augmented_generation" - }, - "events": [], - "links": [], - "trace_trace_id": "bc60c122-f30b-4114-9f78-ee40a024b525", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:56:44.873000000", - "trace_end_time": "2025-10-22 16:56:53.701000000", - "trace_duration_ms": 8828.055, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:56:44.873000000", - "updated_at": "2025-10-22 16:56:44.873000000" - }, - { - "span_id": "e6315110-36b1-4503-beaf-12d972e25a7d", - "trace_id": "bc60c122-f30b-4114-9f78-ee40a024b525", - "parent_span_id": "2ece2bd6-e4c8-47bc-b3c8-26f82e9910f1", - "name": "agent.llm_model_execution", - "start_time": "2025-10-22 16:56:45.576000000", - "end_time": "2025-10-22 16:56:53.143000000", - "duration_ms": 7566.311, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: How do I integrate Noveum Trace in my application?", - "query_length": 50, - "query_type": "rag_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 7.566190004348755, - "model.latency_ms": 7566.190004348755, - "model.start_time": 1761152205.576988, - "model.end_time": 1761152213.143178, - "model.performance_tier": "slow", - "model.prompt_tokens": 837, - "model.completion_tokens": 286, - "model.total_tokens": 1123, - "model.tokens_per_second": 148.4234468543009, - "model.estimated_cost": 0.03369, - "model.efficiency_score": 1.0195903829029385, - "model.response_length": 1145, - "model.response_quality": "high", - "model.output_response": "Model Response: To integrate Noveum Trace into your application, you can follow these steps based on the SDK Integration Guide:\n\n1. **Create Your Account & Get API Key**:\n - Sign up at [noveum.ai](https://noveum.ai...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "bc60c122-f30b-4114-9f78-ee40a024b525", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:56:44.873000000", - "trace_end_time": "2025-10-22 16:56:53.701000000", - "trace_duration_ms": 8828.055, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:56:44.873000000", - "updated_at": "2025-10-22 16:56:44.873000000" - }, - { - "span_id": "ae91d520-8442-48e5-aabf-fd58aed23694", - "trace_id": "bc60c122-f30b-4114-9f78-ee40a024b525", - "parent_span_id": "2ece2bd6-e4c8-47bc-b3c8-26f82e9910f1", - "name": "agent.rag_evaluation_metrics", - "start_time": "2025-10-22 16:56:53.143000000", - "end_time": "2025-10-22 16:56:53.143000000", - "duration_ms": 0.091, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "rag_evaluation_metrics", - "agent.capabilities": [ - "retrieval_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: How do I integrate Noveum Trace in my application?", - "query_length": 50, - "query_type": "rag_evaluation_query", - "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\n\ud83d\udcd8 TypeScript SDK (\n@noveum/trace\n)- Framework ...", - "retrieval.context_length": 2803, - "retrieval.sources_count": 5, - "retrieval.context_quality": "high", - "retrieval.effectiveness": 1, - "retrieval.context_utilization": 2.803, - "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\n\ud83d\udcd8 TypeScript SDK (\n@noveum/trace\n)- Framework integrations for Next.js, Express.js, and Hono\n- TypeScript-first design with full type safety\n- Universal compatibility (Node.js, Edge Runtime, browsers)\n-\n\ud83d\udcca Noveum Platform\n- Real-time dashboard for analyzing traces and performance\n- Ad...\n\nSource 2 (https://noveum.ai/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\n\ud83d\udcd8 TypeScript SDK (\n@noveum/trace\n)- Framework integrations for Next.js, Express.js, and Hono\n- TypeScript-first design with full type safety\n- Universal compatibility (Node.js, Edge Runtime, browsers)\n-\n\ud83d\udcca Noveum Platform\n- Real-time dashboard for analyzing traces and performance\n- Ad...\n\nSource 3 (https://noveum.ai/docs/getting-started/sdk-integration):\nSDK Integration Guide\nIntegrate Noveum.ai tracing into your AI applications with Python or TypeScript SDKs\nThe Noveum.ai SDKs provide comprehensive tracing and observability for your AI applications with minimal code changes. Whether you're building LLM applications, RAG systems, or multi-agent workflows, our SDKs automatically capture essential metrics and traces.\n\ud83d\ude80 Quick Start\n1. Create Your Account & Get API Key\n- Sign up at noveum.ai\n- Create a project in your dashboard\n- Generate an API key...\n\nSource 4 (https://noveum.ai/en/docs/getting-started/sdk-integration):\nSDK Integration Guide\nIntegrate Noveum.ai tracing into your AI applications with Python or TypeScript SDKs\nThe Noveum.ai SDKs provide comprehensive tracing and observability for your AI applications with minimal code changes. Whether you're building LLM applications, RAG systems, or multi-agent workflows, our SDKs automatically capture essential metrics and traces.\n\ud83d\ude80 Quick Start\n1. Create Your Account & Get API Key\n- Sign up at noveum.ai\n- Create a project in your dashboard\n- Generate an API key...\n\nSource 5 (https://noveum.ai/en/docs/getting-started/sdk-integration):\n\ud83d\udd27 Framework Integrations\nNext.js Integration\nExpress.js Integration\nFastAPI Integration (Python)\n\ud83d\udcca Advanced Features\nCustom Attributes & Events\nSampling Configuration\n\ud83d\udd17 What's Captured Automatically\n- \ud83d\udcca Performance Metrics: Latency, throughput, error rates\n- \ud83d\udcb0 Cost Tracking: Token usage, API costs across providers\n- \ud83d\udd0d Request/Response: Configurable capture of inputs/outputs\n- \ud83c\udff7\ufe0f Metadata: Model names, parameters, user context\n- \ud83c\udf0a Context Flow: Trace relationships across services\n- \ud83d\udc1b Error Detail...\n\nUser Question: How do I integrate Noveum Trace in my application?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", - "prompt.prompt_length": 3349, - "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frame...", - "prompt.instruction_following": "rag_optimized", - "response.answer_length": 1145, - "response.answer_completeness": "complete", - "response.response_quality": "high", - "response.source_citation": 5, - "response.context_utilization": 0.4084909026043525, - "output_response": "RAG Answer: To integrate Noveum Trace into your application, you can follow these steps based on the SDK Integration Guide:\n\n1. **Create Your Account & Get API Key**:\n - Sign up at [noveum.ai](https://noveum.ai...", - "evaluation.retrieval_effectiveness": 1, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 5, - "evaluation.context_relevance": "high", - "evaluation.overall_quality": "high", - "evaluation.ready_for_production": true, - "rag.retrieval_strategy": "semantic_similarity", - "rag.vector_search_results": 5, - "rag.context_synthesis": "multi_source", - "rag.document_coverage": 1, - "rag.information_density": 0.4084909026043525 - }, - "events": [], - "links": [], - "trace_trace_id": "bc60c122-f30b-4114-9f78-ee40a024b525", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:56:44.873000000", - "trace_end_time": "2025-10-22 16:56:53.701000000", - "trace_duration_ms": 8828.055, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:56:44.873000000", - "updated_at": "2025-10-22 16:56:44.873000000" - }, - { - "span_id": "9a9c611d-f552-4b06-a764-4463edb70e2c", - "trace_id": "3ad192e5-bbd4-4236-8144-019d53512769", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-22 16:56:35.188000000", - "end_time": "2025-10-22 16:56:43.863000000", - "duration_ms": 8675.25, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What is Noveum and what does it do?", - "query.length": 35, - "processing.mode": "RAG", - "processing.time_seconds": 8.67501425743103, - "processing.response_length": 527, - "processing.sources_count": 5, - "output_response": "Final Answer: Noveum.ai is a comprehensive tracing and observability platform specifically designed for AI applications, including LLM-powered chatbots, RAG systems, and multi-agent workflows. It provides the neces...", - "final_answer_mode": "RAG", - "query_processed.input_query": "What is Noveum and what does it do?", - "query_processed.output_response": "Successfully processed query using RAG, generated 527 character response", - "query_processed.mode": "RAG", - "query_processed.processing_time": 8.67501425743103, - "query_processed.response_length": 527 - }, - "events": [], - "links": [], - "trace_trace_id": "3ad192e5-bbd4-4236-8144-019d53512769", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:56:35.188000000", - "trace_end_time": "2025-10-22 16:56:43.863000000", - "trace_duration_ms": 8675.356000000002, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:56:35.188000000", - "updated_at": "2025-10-22 16:56:35.188000000" - }, - { - "span_id": "99eadeae-0733-4bf3-982e-e9f3014f5913", - "trace_id": "3ad192e5-bbd4-4236-8144-019d53512769", - "parent_span_id": "7e2c959c-157a-43d5-9870-32994cddd388", - "name": "agent.llm-rag", - "start_time": "2025-10-22 16:56:35.188000000", - "end_time": "2025-10-22 16:56:43.863000000", - "duration_ms": 8674.699999999999, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "rag_agent", - "agent.operation": "llm-rag", - "agent.capabilities": [ - "document_retrieval", - "context_generation", - "response_generation" - ], - "agent.id": "noveum_rag_agent", - "input_query": "What is Noveum and what does it do?", - "query_length": 35, - "query_type": "rag_query", - "output_response": "RAG Answer: Noveum.ai is a comprehensive tracing and observability platform specifically designed for AI applications, including LLM-powered chatbots, RAG systems, and multi-agent workflows. It provides the neces...", - "rag.context_length": 2334, - "rag.sources_count": 5, - "rag.answer_length": 527, - "rag.mode": "retrieval_augmented_generation" - }, - "events": [], - "links": [], - "trace_trace_id": "3ad192e5-bbd4-4236-8144-019d53512769", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:56:35.188000000", - "trace_end_time": "2025-10-22 16:56:43.863000000", - "trace_duration_ms": 8675.356000000002, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:56:35.188000000", - "updated_at": "2025-10-22 16:56:35.188000000" - }, - { - "span_id": "7e2c959c-157a-43d5-9870-32994cddd388", - "trace_id": "3ad192e5-bbd4-4236-8144-019d53512769", - "parent_span_id": "9a9c611d-f552-4b06-a764-4463edb70e2c", - "name": "agent.query_routing", - "start_time": "2025-10-22 16:56:35.188000000", - "end_time": "2025-10-22 16:56:43.863000000", - "duration_ms": 8674.961, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What is Noveum and what does it do?", - "query_length": 35, - "query_type": "routing_query", - "output_response": "Routed to RAG for query processing", - "router.classification": "RAG", - "router.confidence_score": 1, - "router.rag_keyword_score": 1, - "router.web_keyword_score": 0, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "3ad192e5-bbd4-4236-8144-019d53512769", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:56:35.188000000", - "trace_end_time": "2025-10-22 16:56:43.863000000", - "trace_duration_ms": 8675.356000000002, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:56:35.188000000", - "updated_at": "2025-10-22 16:56:35.188000000" - }, - { - "span_id": "6f81284c-a9aa-4d62-a402-1b474f2eba51", - "trace_id": "3ad192e5-bbd4-4236-8144-019d53512769", - "parent_span_id": "7e2c959c-157a-43d5-9870-32994cddd388", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-22 16:56:35.188000000", - "end_time": "2025-10-22 16:56:35.188000000", - "duration_ms": 0.054, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What is Noveum and what does it do?", - "query_length": 35, - "query_type": "routing_evaluation_query", - "classification.mode": "RAG", - "classification.rag_keyword_score": 1, - "classification.web_keyword_score": 0, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "noveum_specific", - "query.keyword_density": 0.125, - "query.domain_affinity": "noveum", - "routing.decision": "Routed to RAG based on analysis", - "routing.rationale": "RAG score: 1, Web score: 0, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "Web Search", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.019230769230769232, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: RAG (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "3ad192e5-bbd4-4236-8144-019d53512769", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:56:35.188000000", - "trace_end_time": "2025-10-22 16:56:43.863000000", - "trace_duration_ms": 8675.356000000002, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:56:35.188000000", - "updated_at": "2025-10-22 16:56:35.188000000" - }, - { - "span_id": "9f42d18a-0235-4ae2-838c-19a88ddc95e1", - "trace_id": "3ad192e5-bbd4-4236-8144-019d53512769", - "parent_span_id": "99eadeae-0733-4bf3-982e-e9f3014f5913", - "name": "agent.llm_model_execution", - "start_time": "2025-10-22 16:56:36.031000000", - "end_time": "2025-10-22 16:56:42.012000000", - "duration_ms": 5980.947999999999, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What is Noveum and what does it do?", - "query_length": 35, - "query_type": "rag_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 5.980915784835815, - "model.latency_ms": 5980.915784835815, - "model.start_time": 1761152196.031816, - "model.end_time": 1761152202.0127318, - "model.performance_tier": "slow", - "model.prompt_tokens": 716, - "model.completion_tokens": 131, - "model.total_tokens": 847, - "model.tokens_per_second": 141.61710856178715, - "model.estimated_cost": 0.025410000000000002, - "model.efficiency_score": 0.6221959858323495, - "model.response_length": 527, - "model.response_quality": "high", - "model.output_response": "Model Response: Noveum.ai is a comprehensive tracing and observability platform specifically designed for AI applications, including LLM-powered chatbots, RAG systems, and multi-agent workflows. It provides the neces...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "3ad192e5-bbd4-4236-8144-019d53512769", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:56:35.188000000", - "trace_end_time": "2025-10-22 16:56:43.863000000", - "trace_duration_ms": 8675.356000000002, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:56:35.188000000", - "updated_at": "2025-10-22 16:56:35.188000000" - }, - { - "span_id": "7bcbe4b7-585a-4c9d-add8-16e8aa62254f", - "trace_id": "3ad192e5-bbd4-4236-8144-019d53512769", - "parent_span_id": "99eadeae-0733-4bf3-982e-e9f3014f5913", - "name": "agent.rag_evaluation_metrics", - "start_time": "2025-10-22 16:56:42.012000000", - "end_time": "2025-10-22 16:56:42.012000000", - "duration_ms": 0.018000000000000002, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "rag_evaluation_metrics", - "agent.capabilities": [ - "retrieval_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What is Noveum and what does it do?", - "query_length": 35, - "query_type": "rag_evaluation_query", - "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, vector searches, data retrieval, agent reasoning, and more. Without proper observability, debugging ...", - "retrieval.context_length": 2334, - "retrieval.sources_count": 5, - "retrieval.context_quality": "high", - "retrieval.effectiveness": 1, - "retrieval.context_utilization": 2.334, - "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, vector searches, data retrieval, agent reasoning, and more. Without proper observability, debugging becomes a nightmare, optimization is guesswork, and understanding user interactions is nearly impossible.\nNoveum.ai solves this by providing comprehensive tracing and observability specifically designed for AI applications.\nKey benefits of using Noveum.ai:\n- Complete Visibility: Trace...\n\nSource 2 (https://noveum.ai/en/docs/getting-started/overview):\nNoveum.ai Overview\nComprehensive AI tracing and observability platform for LLM applications, RAG systems, and AI agents\nWelcome to Noveum.ai\u2014the comprehensive tracing and observability platform built specifically for AI applications. Whether you're building LLM-powered chatbots, RAG systems, multi-agent workflows, or any AI-driven application, Noveum provides the insights you need to understand, debug, and optimize your systems.\n\ud83c\udfaf Why AI Applications Need Specialized Observability\nTraditional mo...\n\nSource 3 (https://noveum.ai/docs/getting-started/overview):\nNoveum.ai Overview\nComprehensive AI tracing and observability platform for LLM applications, RAG systems, and AI agents\nWelcome to Noveum.ai\u2014the comprehensive tracing and observability platform built specifically for AI applications. Whether you're building LLM-powered chatbots, RAG systems, multi-agent workflows, or any AI-driven application, Noveum provides the insights you need to understand, debug, and optimize your systems.\n\ud83c\udfaf Why AI Applications Need Specialized Observability\nTraditional mo...\n\nSource 4 (https://noveum.ai/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nSource 5 (https://noveum.ai/en/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nUser Question: What is Noveum and what does it do?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", - "prompt.prompt_length": 2865, - "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, ...", - "prompt.instruction_following": "rag_optimized", - "response.answer_length": 527, - "response.answer_completeness": "complete", - "response.response_quality": "high", - "response.source_citation": 5, - "response.context_utilization": 0.22579263067694944, - "output_response": "RAG Answer: Noveum.ai is a comprehensive tracing and observability platform specifically designed for AI applications, including LLM-powered chatbots, RAG systems, and multi-agent workflows. It provides the neces...", - "evaluation.retrieval_effectiveness": 1, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 5, - "evaluation.context_relevance": "high", - "evaluation.overall_quality": "high", - "evaluation.ready_for_production": true, - "rag.retrieval_strategy": "semantic_similarity", - "rag.vector_search_results": 5, - "rag.context_synthesis": "multi_source", - "rag.document_coverage": 1, - "rag.information_density": 0.22579263067694944 - }, - "events": [], - "links": [], - "trace_trace_id": "3ad192e5-bbd4-4236-8144-019d53512769", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:56:35.188000000", - "trace_end_time": "2025-10-22 16:56:43.863000000", - "trace_duration_ms": 8675.356000000002, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:56:35.188000000", - "updated_at": "2025-10-22 16:56:35.188000000" - }, - { - "span_id": "d39297f4-b4fe-4f9c-b45b-854411ba770a", - "trace_id": "9af2b4b0-0c2f-4d2e-ae99-30df0d2f0fa6", - "parent_span_id": "", - "name": "system_initialization", - "start_time": "2025-10-22 16:56:34.565000000", - "end_time": "2025-10-22 16:56:35.186000000", - "duration_ms": 621.02, - "status": "ok", - "status_message": "", - "attributes": { - "system.force_scrape": false, - "system.config": { - "noveum_base_url": "https://noveum.ai", - "max_pages_to_scrape": 50, - "chunk_size": 1000, - "chunk_overlap": 200, - "max_search_results": 5, - "rag_threshold": 0.7, - "noveum_docs_file": "noveum_docs.json", - "vector_store_path": "noveum_vectorstore" - }, - "input_query": "Initialize system with force_scrape=False", - "output_response": "System initialization: RAG system loaded, vector store ready, agent operational", - "system.initialized": true, - "system.documents_loaded": 38, - "system.vectorstore_ready": true - }, - "events": [], - "links": [], - "trace_trace_id": "9af2b4b0-0c2f-4d2e-ae99-30df0d2f0fa6", - "trace_name": "auto_trace_system_initialization", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:56:34.565000000", - "trace_end_time": "2025-10-22 16:56:35.186000000", - "trace_duration_ms": 621.177, - "span_count": 1, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:56:34.565000000", - "updated_at": "2025-10-22 16:56:34.565000000" - }, - { - "span_id": "8c144ad8-a439-4986-b741-3bc4b7fc6110", - "trace_id": "bae13042-cf2e-4813-a673-7dc2a611725e", - "parent_span_id": "d37fe1a1-a126-4e4e-a914-93dd402fed3f", - "name": "agent.query_routing", - "start_time": "2025-10-16 08:50:56.153000000", - "end_time": "2025-10-16 08:51:01.908000000", - "duration_ms": 5754.43, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What are the latest breakthroughs in artificial intelligence?", - "query_length": 61, - "query_type": "routing_query", - "output_response": "Routed to Web Search for query processing", - "router.classification": "Web Search", - "router.confidence_score": 1, - "router.rag_keyword_score": 0, - "router.web_keyword_score": 1, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "bae13042-cf2e-4813-a673-7dc2a611725e", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:56.153000000", - "trace_end_time": "2025-10-16 08:51:01.908000000", - "trace_duration_ms": 5754.578, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:56.153000000", - "updated_at": "2025-10-16 08:50:56.153000000" - }, - { - "span_id": "b294f9b7-7762-47a9-b49d-00801a984af2", - "trace_id": "bae13042-cf2e-4813-a673-7dc2a611725e", - "parent_span_id": "8c144ad8-a439-4986-b741-3bc4b7fc6110", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-16 08:50:56.153000000", - "end_time": "2025-10-16 08:50:56.153000000", - "duration_ms": 0.024, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What are the latest breakthroughs in artificial intelligence?", - "query_length": 61, - "query_type": "routing_evaluation_query", - "classification.mode": "Web Search", - "classification.rag_keyword_score": 0, - "classification.web_keyword_score": 1, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "complex", - "query.intent": "general_knowledge", - "query.keyword_density": 0.125, - "query.domain_affinity": "general", - "routing.decision": "Routed to Web Search based on analysis", - "routing.rationale": "RAG score: 0, Web score: 1, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "RAG", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.019230769230769232, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: Web Search (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "bae13042-cf2e-4813-a673-7dc2a611725e", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:56.153000000", - "trace_end_time": "2025-10-16 08:51:01.908000000", - "trace_duration_ms": 5754.578, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:56.153000000", - "updated_at": "2025-10-16 08:50:56.153000000" - }, - { - "span_id": "ab6a281a-4989-40c8-9011-d0ec7dde7ea9", - "trace_id": "bae13042-cf2e-4813-a673-7dc2a611725e", - "parent_span_id": "8c144ad8-a439-4986-b741-3bc4b7fc6110", - "name": "agent.web_search_generation", - "start_time": "2025-10-16 08:50:56.153000000", - "end_time": "2025-10-16 08:51:01.908000000", - "duration_ms": 5754.317999999999, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "web_search_agent", - "agent.operation": "web_search_generation", - "agent.capabilities": [ - "web_search", - "content_synthesis", - "response_generation" - ], - "agent.id": "noveum_web_search_agent", - "input_query": "What are the latest breakthroughs in artificial intelligence?", - "query_length": 61, - "query_type": "web_search_query", - "output_response": "Web Search Answer: The web search did not yield specific results regarding the latest breakthroughs in artificial intelligence. However, I can provide a general overview based on knowledge up to October 2023.\n\nRecent br...", - "web_search.results_count": 1, - "web_search.context_length": 163, - "web_search.response_length": 1489, - "web_search.mode": "external_web_search" - }, - "events": [], - "links": [], - "trace_trace_id": "bae13042-cf2e-4813-a673-7dc2a611725e", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:56.153000000", - "trace_end_time": "2025-10-16 08:51:01.908000000", - "trace_duration_ms": 5754.578, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:56.153000000", - "updated_at": "2025-10-16 08:50:56.153000000" - }, - { - "span_id": "d37fe1a1-a126-4e4e-a914-93dd402fed3f", - "trace_id": "bae13042-cf2e-4813-a673-7dc2a611725e", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-16 08:50:56.153000000", - "end_time": "2025-10-16 08:51:01.908000000", - "duration_ms": 5754.527, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What are the latest breakthroughs in artificial intelligence?", - "query.length": 61, - "processing.mode": "Web Search", - "processing.time_seconds": 5.754457473754883, - "processing.response_length": 1489, - "processing.sources_count": 1, - "output_response": "Final Answer: The web search did not yield specific results regarding the latest breakthroughs in artificial intelligence. However, I can provide a general overview based on knowledge up to October 2023.\n\nRecent br...", - "final_answer_mode": "Web Search", - "query_processed.input_query": "What are the latest breakthroughs in artificial intelligence?", - "query_processed.output_response": "Successfully processed query using Web Search, generated 1489 character response", - "query_processed.mode": "Web Search", - "query_processed.processing_time": 5.754457473754883, - "query_processed.response_length": 1489 - }, - "events": [], - "links": [], - "trace_trace_id": "bae13042-cf2e-4813-a673-7dc2a611725e", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:56.153000000", - "trace_end_time": "2025-10-16 08:51:01.908000000", - "trace_duration_ms": 5754.578, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:56.153000000", - "updated_at": "2025-10-16 08:50:56.153000000" - }, - { - "span_id": "f61ef76d-4887-42da-9fd6-e2cdb25a9e09", - "trace_id": "bae13042-cf2e-4813-a673-7dc2a611725e", - "parent_span_id": "ab6a281a-4989-40c8-9011-d0ec7dde7ea9", - "name": "agent.llm_model_execution", - "start_time": "2025-10-16 08:50:56.502000000", - "end_time": "2025-10-16 08:51:01.908000000", - "duration_ms": 5405.44, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What are the latest breakthroughs in artificial intelligence?", - "query_length": 61, - "query_type": "web_search_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 5.405392408370972, - "model.latency_ms": 5405.392408370972, - "model.start_time": 1760604656.5026655, - "model.end_time": 1760604661.908058, - "model.performance_tier": "slow", - "model.prompt_tokens": 167, - "model.completion_tokens": 372, - "model.total_tokens": 539, - "model.tokens_per_second": 99.71523976044487, - "model.estimated_cost": 0.01617, - "model.efficiency_score": 2.7625231910946195, - "model.response_length": 1489, - "model.response_quality": "high", - "model.output_response": "Model Response: The web search did not yield specific results regarding the latest breakthroughs in artificial intelligence. However, I can provide a general overview based on knowledge up to October 2023.\n\nRecent br...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "bae13042-cf2e-4813-a673-7dc2a611725e", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:56.153000000", - "trace_end_time": "2025-10-16 08:51:01.908000000", - "trace_duration_ms": 5754.578, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:56.153000000", - "updated_at": "2025-10-16 08:50:56.153000000" - }, - { - "span_id": "550bf5ae-c816-4ef0-a6c1-1553b1c258b8", - "trace_id": "bae13042-cf2e-4813-a673-7dc2a611725e", - "parent_span_id": "ab6a281a-4989-40c8-9011-d0ec7dde7ea9", - "name": "agent.web_search_evaluation_metrics", - "start_time": "2025-10-16 08:51:01.908000000", - "end_time": "2025-10-16 08:51:01.908000000", - "duration_ms": 0.018000000000000002, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "web_search_evaluation_metrics", - "agent.capabilities": [ - "web_search_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What are the latest breakthroughs in artificial intelligence?", - "query_length": 61, - "query_type": "web_search_evaluation_query", - "web_search.results_count": 1, - "web_search.context_length": 163, - "web_search.context_synthesized": "Context from 1 web sources", - "web_search.search_effectiveness": 0.2, - "web_search.context_quality": "low", - "web_search.source_diversity": 1, - "web_search.information_synthesis": "low", - "web_search.external_knowledge_utilization": 0.163, - "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What+are+the+latest+breakthroughs+in+artificial+intelligence?\n\nUser Question: What are the latest breakthroughs in artificial intelligence?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", - "prompt.prompt_length": 669, - "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What+are+the+latest+breakthroughs+in+artificial+intelligence?", - "prompt.instruction_following": "web_search_optimized", - "response.answer_length": 1489, - "response.answer_completeness": "complete", - "response.response_quality": "medium", - "response.source_citation": 1, - "response.context_utilization": 9.134969325153374, - "output_response": "Web Search Answer: The web search did not yield specific results regarding the latest breakthroughs in artificial intelligence. However, I can provide a general overview based on knowledge up to October 2023.\n\nRecent br...", - "evaluation.search_effectiveness": 0.2, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 1, - "evaluation.context_relevance": "low", - "evaluation.overall_quality": "low", - "evaluation.ready_for_production": true, - "web_search.search_strategy": "duckduckgo_api", - "web_search.real_time_data": true, - "web_search.external_sources": 1, - "web_search.information_freshness": "current", - "web_search.knowledge_synthesis": "single_source" - }, - "events": [], - "links": [], - "trace_trace_id": "bae13042-cf2e-4813-a673-7dc2a611725e", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:56.153000000", - "trace_end_time": "2025-10-16 08:51:01.908000000", - "trace_duration_ms": 5754.578, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:56.153000000", - "updated_at": "2025-10-16 08:50:56.153000000" - }, - { - "span_id": "04cb7d29-9ea8-4ae0-9834-030cae944eca", - "trace_id": "79f9d235-d46e-4740-be35-ef0006071f6c", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-16 08:50:51.539000000", - "end_time": "2025-10-16 08:50:55.150000000", - "duration_ms": 3611.008, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What's happening in the software development world today?", - "query.length": 57, - "processing.mode": "Web Search", - "processing.time_seconds": 3.6109254360198975, - "processing.response_length": 526, - "processing.sources_count": 1, - "output_response": "Final Answer: The provided web search results do not contain specific information about current events or trends in the software development world. They primarily focus on unrelated topics, such as the meaning of \"...", - "final_answer_mode": "Web Search", - "query_processed.input_query": "What's happening in the software development world today?", - "query_processed.output_response": "Successfully processed query using Web Search, generated 526 character response", - "query_processed.mode": "Web Search", - "query_processed.processing_time": 3.6109254360198975, - "query_processed.response_length": 526 - }, - "events": [], - "links": [], - "trace_trace_id": "79f9d235-d46e-4740-be35-ef0006071f6c", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:51.539000000", - "trace_end_time": "2025-10-16 08:50:55.150000000", - "trace_duration_ms": 3611.084, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:51.539000000", - "updated_at": "2025-10-16 08:50:51.539000000" - }, - { - "span_id": "e5e3c927-783e-48c5-aec7-b32bae98c0a2", - "trace_id": "79f9d235-d46e-4740-be35-ef0006071f6c", - "parent_span_id": "f630c3e2-091e-4bd9-9556-e0ebc1c50291", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-16 08:50:51.539000000", - "end_time": "2025-10-16 08:50:51.540000000", - "duration_ms": 0.041, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What's happening in the software development world today?", - "query_length": 57, - "query_type": "routing_evaluation_query", - "classification.mode": "Web Search", - "classification.rag_keyword_score": 0, - "classification.web_keyword_score": 1, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "complex", - "query.intent": "general_knowledge", - "query.keyword_density": 0.125, - "query.domain_affinity": "general", - "routing.decision": "Routed to Web Search based on analysis", - "routing.rationale": "RAG score: 0, Web score: 1, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "RAG", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.019230769230769232, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: Web Search (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "79f9d235-d46e-4740-be35-ef0006071f6c", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:51.539000000", - "trace_end_time": "2025-10-16 08:50:55.150000000", - "trace_duration_ms": 3611.084, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:51.539000000", - "updated_at": "2025-10-16 08:50:51.539000000" - }, - { - "span_id": "f630c3e2-091e-4bd9-9556-e0ebc1c50291", - "trace_id": "79f9d235-d46e-4740-be35-ef0006071f6c", - "parent_span_id": "04cb7d29-9ea8-4ae0-9834-030cae944eca", - "name": "agent.query_routing", - "start_time": "2025-10-16 08:50:51.539000000", - "end_time": "2025-10-16 08:50:55.150000000", - "duration_ms": 3610.876, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What's happening in the software development world today?", - "query_length": 57, - "query_type": "routing_query", - "output_response": "Routed to Web Search for query processing", - "router.classification": "Web Search", - "router.confidence_score": 1, - "router.rag_keyword_score": 0, - "router.web_keyword_score": 1, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "79f9d235-d46e-4740-be35-ef0006071f6c", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:51.539000000", - "trace_end_time": "2025-10-16 08:50:55.150000000", - "trace_duration_ms": 3611.084, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:51.539000000", - "updated_at": "2025-10-16 08:50:51.539000000" - }, - { - "span_id": "d7b0f399-3b43-43bf-bac1-b2101268841c", - "trace_id": "79f9d235-d46e-4740-be35-ef0006071f6c", - "parent_span_id": "f630c3e2-091e-4bd9-9556-e0ebc1c50291", - "name": "agent.web_search_generation", - "start_time": "2025-10-16 08:50:51.540000000", - "end_time": "2025-10-16 08:50:55.150000000", - "duration_ms": 3610.679, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "web_search_agent", - "agent.operation": "web_search_generation", - "agent.capabilities": [ - "web_search", - "content_synthesis", - "response_generation" - ], - "agent.id": "noveum_web_search_agent", - "input_query": "What's happening in the software development world today?", - "query_length": 57, - "query_type": "web_search_query", - "output_response": "Web Search Answer: The provided web search results do not contain specific information about current events or trends in the software development world. They primarily focus on unrelated topics, such as the meaning of \"...", - "web_search.results_count": 1, - "web_search.context_length": 589, - "web_search.response_length": 526, - "web_search.mode": "external_web_search" - }, - "events": [], - "links": [], - "trace_trace_id": "79f9d235-d46e-4740-be35-ef0006071f6c", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:51.539000000", - "trace_end_time": "2025-10-16 08:50:55.150000000", - "trace_duration_ms": 3611.084, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:51.539000000", - "updated_at": "2025-10-16 08:50:51.539000000" - }, - { - "span_id": "4ed5dae4-f850-4644-aa55-2f1c00d11c22", - "trace_id": "79f9d235-d46e-4740-be35-ef0006071f6c", - "parent_span_id": "d7b0f399-3b43-43bf-bac1-b2101268841c", - "name": "agent.llm_model_execution", - "start_time": "2025-10-16 08:50:51.976000000", - "end_time": "2025-10-16 08:50:55.150000000", - "duration_ms": 3174.51, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What's happening in the software development world today?", - "query_length": 57, - "query_type": "web_search_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 3.174463987350464, - "model.latency_ms": 3174.463987350464, - "model.start_time": 1760604651.9761698, - "model.end_time": 1760604655.1506338, - "model.performance_tier": "medium", - "model.prompt_tokens": 272, - "model.completion_tokens": 131, - "model.total_tokens": 403, - "model.tokens_per_second": 126.95056601866197, - "model.estimated_cost": 0.01209, - "model.efficiency_score": 1.305210918114144, - "model.response_length": 526, - "model.response_quality": "high", - "model.output_response": "Model Response: The provided web search results do not contain specific information about current events or trends in the software development world. They primarily focus on unrelated topics, such as the meaning of \"...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "79f9d235-d46e-4740-be35-ef0006071f6c", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:51.539000000", - "trace_end_time": "2025-10-16 08:50:55.150000000", - "trace_duration_ms": 3611.084, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:51.539000000", - "updated_at": "2025-10-16 08:50:51.539000000" - }, - { - "span_id": "eb151834-9322-4f63-add0-dec9a2966367", - "trace_id": "79f9d235-d46e-4740-be35-ef0006071f6c", - "parent_span_id": "d7b0f399-3b43-43bf-bac1-b2101268841c", - "name": "agent.web_search_evaluation_metrics", - "start_time": "2025-10-16 08:50:55.150000000", - "end_time": "2025-10-16 08:50:55.150000000", - "duration_ms": 0.02, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "web_search_evaluation_metrics", - "agent.capabilities": [ - "web_search_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What's happening in the software development world today?", - "query_length": 57, - "query_type": "web_search_evaluation_query", - "web_search.results_count": 1, - "web_search.context_length": 589, - "web_search.context_synthesized": "Context from 1 web sources", - "web_search.search_effectiveness": 0.2, - "web_search.context_quality": "medium", - "web_search.source_diversity": 1, - "web_search.information_synthesis": "low", - "web_search.external_knowledge_utilization": 0.589, - "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\n\u66fc\u5df4\u7684\u610f\u601d\u662f\u4e00\u79cd\u6bd2\u86c7\uff0c\u8c61\u5f81\u7740\u79d1\u6bd4\u5728\u7403\u573a\u4e0a\u7684\u653b\u51fb\u6027\u548c\u575a\u97e7\u4e0d\u62d4 \u201cWhat can i say\uff0c Mamba out\uff01\u201d \u4f46\u662f\u7ecf\u8fc7\u4e92\u8054\u7f51\u7684\u89e3\u6784\uff0c\u8fd9\u53e5\u8bdd\u6210\u4e3a\u4e86\u4e00\u53e5\u4e07\u7528\u573a\u666f\u7684\u8bdd \u2026 Aprende ingl\u00e9s gratis con nuestros cursos. \u00a1Mas de 100 lecciones con ejemplos y audio! Distintas y divertidas formas de estudiar ingl\u00e9s: \u2026 \u77e5\u4e4e\uff0c\u4e2d\u6587\u4e92\u8054\u7f51\u9ad8\u8d28\u91cf\u7684\u95ee\u7b54\u793e\u533a\u548c\u521b\u4f5c\u8005\u805a\u96c6\u7684\u539f\u521b\u5185\u5bb9\u5e73\u53f0\uff0c\u4e8e 2011 \u5e74 1 \u6708\u6b63\u5f0f\u4e0a\u7ebf\uff0c\u4ee5\u300c\u8ba9\u4eba\u4eec\u66f4\u597d\u7684\u5206\u4eab\u77e5\u8bc6\u3001\u7ecf\u9a8c\u548c\u89c1\u89e3\uff0c\u627e\u5230\u81ea\u5df1\u7684\u89e3\u7b54\u300d\u4e3a\u54c1 \u2026 \u8f6f\u4ef6\u7279\u70b9 1\u3001Windows Desktop Runtime\u662f\u5fae\u8f6fWindows\u684c\u9762\u7a0b\u5e8f\u8fd0\u884c\u5e93\uff08\u542b\u5e38\u89c4\u8fd0\u884c\u5e93\uff09 2\u3001\u80fd\u5b8c\u7f8e\u517c\u5bb9\u5fae\u8f6f\u4e0d\u540c\u7248\u672c\u7684Windows\u7cfb\u7edf\uff0c\u89e3\u51b3\u5176\u7a0b\u5e8f\u7f3a \u2026 \u77e5\u4e4e\uff0c\u4e2d\u6587\u4e92\u8054\u7f51\u9ad8\u8d28\u91cf\u7684\u95ee\u7b54\u793e\u533a\u548c\u521b\u4f5c\u8005\u805a\u96c6\u7684\u539f\u521b\u5185\u5bb9\u5e73\u53f0\uff0c\u4e8e 2011 \u5e74 1 \u6708\u6b63\u5f0f\u4e0a\u7ebf\uff0c\u4ee5\u300c\u8ba9\u4eba\u4eec\u66f4\u597d\u7684\u5206\u4eab\u77e5\u8bc6\u3001\u7ecf\u9a8c\u548c\u89c1\u89e3\uff0c\u627e\u5230\u81ea\u5df1\u7684\u89e3\u7b54\u300d\u4e3a\u54c1 \u2026\nURL: https://duckduckgo.com/?q=What's+happening+in+the+software+development+world+today?\n\nUser Question: What's happening in the software development world today?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", - "prompt.prompt_length": 1091, - "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\n\u66fc\u5df4\u7684\u610f\u601d\u662f\u4e00\u79cd\u6bd2\u86c7\uff0c\u8c61\u5f81\u7740\u79d1\u6bd4\u5728\u7403\u573a\u4e0a\u7684\u653b\u51fb\u6027\u548c\u575a\u97e7\u4e0d\u62d4 \u201cWhat can i say\uff0c Mamba out\uff01\u201d \u4f46\u662f\u7ecf\u8fc7\u4e92\u8054\u7f51\u7684\u89e3\u6784\uff0c\u8fd9\u53e5\u8bdd\u6210\u4e3a\u4e86\u4e00\u53e5\u4e07\u7528\u573a\u666f\u7684\u8bdd \u2026 Aprende ingl\u00e9s gratis con nuestros cursos. \u00a1Mas de 100 lecciones con ejemplos y audio...", - "prompt.instruction_following": "web_search_optimized", - "response.answer_length": 526, - "response.answer_completeness": "complete", - "response.response_quality": "medium", - "response.source_citation": 1, - "response.context_utilization": 0.8930390492359932, - "output_response": "Web Search Answer: The provided web search results do not contain specific information about current events or trends in the software development world. They primarily focus on unrelated topics, such as the meaning of \"...", - "evaluation.search_effectiveness": 0.2, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 1, - "evaluation.context_relevance": "medium", - "evaluation.overall_quality": "low", - "evaluation.ready_for_production": true, - "web_search.search_strategy": "duckduckgo_api", - "web_search.real_time_data": true, - "web_search.external_sources": 1, - "web_search.information_freshness": "current", - "web_search.knowledge_synthesis": "single_source" - }, - "events": [], - "links": [], - "trace_trace_id": "79f9d235-d46e-4740-be35-ef0006071f6c", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:51.539000000", - "trace_end_time": "2025-10-16 08:50:55.150000000", - "trace_duration_ms": 3611.084, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:51.539000000", - "updated_at": "2025-10-16 08:50:51.539000000" - }, - { - "span_id": "888b4441-3678-42a5-a752-8daae2433482", - "trace_id": "77b81bd5-5a04-4fc0-9fc6-571f7907d6b1", - "parent_span_id": "ba6ffeeb-78b5-44dd-b460-7933ca6e7739", - "name": "agent.query_routing", - "start_time": "2025-10-16 08:50:44.888000000", - "end_time": "2025-10-16 08:50:50.529000000", - "duration_ms": 5641.062, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What are the newest features in cloud computing?", - "query_length": 48, - "query_type": "routing_query", - "output_response": "Routed to RAG for query processing", - "router.classification": "RAG", - "router.confidence_score": 1, - "router.rag_keyword_score": 1, - "router.web_keyword_score": 0, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "77b81bd5-5a04-4fc0-9fc6-571f7907d6b1", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:44.888000000", - "trace_end_time": "2025-10-16 08:50:50.529000000", - "trace_duration_ms": 5641.226, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:44.888000000", - "updated_at": "2025-10-16 08:50:44.888000000" - }, - { - "span_id": "b6bae768-93a6-47d9-8369-61897f9d7828", - "trace_id": "77b81bd5-5a04-4fc0-9fc6-571f7907d6b1", - "parent_span_id": "888b4441-3678-42a5-a752-8daae2433482", - "name": "agent.llm-rag", - "start_time": "2025-10-16 08:50:44.888000000", - "end_time": "2025-10-16 08:50:50.529000000", - "duration_ms": 5640.875999999999, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "rag_agent", - "agent.operation": "llm-rag", - "agent.capabilities": [ - "document_retrieval", - "context_generation", - "response_generation" - ], - "agent.id": "noveum_rag_agent", - "input_query": "What are the newest features in cloud computing?", - "query_length": 48, - "query_type": "rag_query", - "output_response": "RAG Answer: The provided context does not specifically mention any new features in cloud computing as a general topic. However, it does highlight several recent enhancements and features related to the Noveum.ai ...", - "rag.context_length": 2579, - "rag.sources_count": 5, - "rag.answer_length": 983, - "rag.mode": "retrieval_augmented_generation" - }, - "events": [], - "links": [], - "trace_trace_id": "77b81bd5-5a04-4fc0-9fc6-571f7907d6b1", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:44.888000000", - "trace_end_time": "2025-10-16 08:50:50.529000000", - "trace_duration_ms": 5641.226, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:44.888000000", - "updated_at": "2025-10-16 08:50:44.888000000" - }, - { - "span_id": "ba6ffeeb-78b5-44dd-b460-7933ca6e7739", - "trace_id": "77b81bd5-5a04-4fc0-9fc6-571f7907d6b1", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-16 08:50:44.888000000", - "end_time": "2025-10-16 08:50:50.529000000", - "duration_ms": 5641.156, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What are the newest features in cloud computing?", - "query.length": 48, - "processing.mode": "RAG", - "processing.time_seconds": 5.64110541343689, - "processing.response_length": 983, - "processing.sources_count": 5, - "output_response": "Final Answer: The provided context does not specifically mention any new features in cloud computing as a general topic. However, it does highlight several recent enhancements and features related to the Noveum.ai ...", - "final_answer_mode": "RAG", - "query_processed.input_query": "What are the newest features in cloud computing?", - "query_processed.output_response": "Successfully processed query using RAG, generated 983 character response", - "query_processed.mode": "RAG", - "query_processed.processing_time": 5.64110541343689, - "query_processed.response_length": 983 - }, - "events": [], - "links": [], - "trace_trace_id": "77b81bd5-5a04-4fc0-9fc6-571f7907d6b1", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:44.888000000", - "trace_end_time": "2025-10-16 08:50:50.529000000", - "trace_duration_ms": 5641.226, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:44.888000000", - "updated_at": "2025-10-16 08:50:44.888000000" - }, - { - "span_id": "f5538bff-8908-4a56-a622-4ba2eeedfd29", - "trace_id": "77b81bd5-5a04-4fc0-9fc6-571f7907d6b1", - "parent_span_id": "888b4441-3678-42a5-a752-8daae2433482", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-16 08:50:44.888000000", - "end_time": "2025-10-16 08:50:44.888000000", - "duration_ms": 0.032, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What are the newest features in cloud computing?", - "query_length": 48, - "query_type": "routing_evaluation_query", - "classification.mode": "RAG", - "classification.rag_keyword_score": 1, - "classification.web_keyword_score": 0, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "documentation", - "query.keyword_density": 0.125, - "query.domain_affinity": "noveum", - "routing.decision": "Routed to RAG based on analysis", - "routing.rationale": "RAG score: 1, Web score: 0, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "Web Search", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.019230769230769232, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: RAG (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "77b81bd5-5a04-4fc0-9fc6-571f7907d6b1", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:44.888000000", - "trace_end_time": "2025-10-16 08:50:50.529000000", - "trace_duration_ms": 5641.226, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:44.888000000", - "updated_at": "2025-10-16 08:50:44.888000000" - }, - { - "span_id": "856e51cf-fa4d-459b-a046-1ea7316b903e", - "trace_id": "77b81bd5-5a04-4fc0-9fc6-571f7907d6b1", - "parent_span_id": "b6bae768-93a6-47d9-8369-61897f9d7828", - "name": "agent.llm_model_execution", - "start_time": "2025-10-16 08:50:45.423000000", - "end_time": "2025-10-16 08:50:49.997000000", - "duration_ms": 4573.782999999999, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What are the newest features in cloud computing?", - "query_length": 48, - "query_type": "rag_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 4.573714733123779, - "model.latency_ms": 4573.714733123779, - "model.start_time": 1760604645.423261, - "model.end_time": 1760604649.9969757, - "model.performance_tier": "medium", - "model.prompt_tokens": 780, - "model.completion_tokens": 245, - "model.total_tokens": 1025, - "model.tokens_per_second": 224.1066747291299, - "model.estimated_cost": 0.03075, - "model.efficiency_score": 0.9590243902439024, - "model.response_length": 983, - "model.response_quality": "high", - "model.output_response": "Model Response: The provided context does not specifically mention any new features in cloud computing as a general topic. However, it does highlight several recent enhancements and features related to the Noveum.ai ...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "77b81bd5-5a04-4fc0-9fc6-571f7907d6b1", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:44.888000000", - "trace_end_time": "2025-10-16 08:50:50.529000000", - "trace_duration_ms": 5641.226, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:44.888000000", - "updated_at": "2025-10-16 08:50:44.888000000" - }, - { - "span_id": "a2fa7400-74c4-4718-a562-899b91503c95", - "trace_id": "77b81bd5-5a04-4fc0-9fc6-571f7907d6b1", - "parent_span_id": "b6bae768-93a6-47d9-8369-61897f9d7828", - "name": "agent.rag_evaluation_metrics", - "start_time": "2025-10-16 08:50:49.997000000", - "end_time": "2025-10-16 08:50:49.997000000", - "duration_ms": 0.048, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "rag_evaluation_metrics", - "agent.capabilities": [ - "retrieval_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What are the newest features in cloud computing?", - "query_length": 48, - "query_type": "rag_evaluation_query", - "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/changelog):\n- \ud83d\udcc8 Enhanced Dashboard Analytics: Real-time request tracking, latency monitoring, and cost analysis\n- \ud83d\udd0d Improved Logs Interface: Better search functionality, detailed trace views, and enhanced debugging capabilities\n- \ud83d\udc65 Team Management: Advanced member mana...", - "retrieval.context_length": 2579, - "retrieval.sources_count": 5, - "retrieval.context_quality": "high", - "retrieval.effectiveness": 1, - "retrieval.context_utilization": 2.579, - "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/changelog):\n- \ud83d\udcc8 Enhanced Dashboard Analytics: Real-time request tracking, latency monitoring, and cost analysis\n- \ud83d\udd0d Improved Logs Interface: Better search functionality, detailed trace views, and enhanced debugging capabilities\n- \ud83d\udc65 Team Management: Advanced member management with role-based access control and invitation system\n- \ud83d\udee1\ufe0f Security Improvements: Enhanced API key management and secure credential storage\n4 months ago\n- \ud83d\udcca Advanced Metrics Collection: Comprehensive telemetry system with custom metrics ...\n\nSource 2 (https://noveum.ai/en/changelog):\n- \ud83d\udcdd Documentation Updates: Comprehensive guides for integration and best practices\n5 months ago\n- \ud83c\udf89 Platform Foundation: Initial release of Noveum.ai AI observability platform\n- \ud83d\ude80 Core Features: AI Gateway integration, basic evaluation metrics, and monitoring capabilities\n- \ud83d\udcc8 Evaluation Jobs: Automated model evaluation with configurable metrics and reporting\n- \ud83d\udd0c Provider Integrations: Support for major AI providers and custom model deployments...\n\nSource 3 (https://noveum.ai/docs/platform/dashboard):\n- Bulk Export: Download multiple traces simultaneously\n- Comparative Analysis: Compare performance across multiple traces\nReal-Time Updates\n- Live Refresh: Automatic updates as new traces arrive\n- Connection Monitoring: Real-time status of your trace ingestion pipeline\n- Performance Indicators: Live metrics for system health monitoring\nReady to dive deeper? Explore Projects & Environments to organize your AI applications, or check out Team Collaboration to share insights with your team.\nGet Earl...\n\nSource 4 (https://noveum.ai/en/changelog):\nChangelog\nStay up to date with the latest changes in our product.\n2 months ago\n- \ud83c\udfaf Major Platform Evolution: NovaEval Framework - Launched comprehensive AI model evaluation framework with 20+ built-in scorers including accuracy, RAG metrics, conversational scoring, and specialized agent evaluation tools. Features production-grade deployment on Docker and Kubernetes. (GitHub | PyPI Package)\n- \ud83d\ude80 Next-Generation Traces UI - Complete redesign featuring three-pane layout with directory tree navigatio...\n\nSource 5 (https://noveum.ai/en/blog):\n8/12/2025\nDiscover how Noveum.ai provides comprehensive tracing and observability for AI applications, from development debugging to production optimization.\nShashank Agarwal\n3/3/2025\nDiscover how Noveum.ai provides comprehensive tracing and observability for LLM applications, RAG systems, and multi-agent workflows with our powerful Python and TypeScript SDKs.\nShashank Agarwal\n3/2/2025...\n\nUser Question: What are the newest features in cloud computing?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", - "prompt.prompt_length": 3123, - "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/changelog):\n- \ud83d\udcc8 Enhanced Dashboard Analytics: Real-time request tracking, latency monitoring, and cost analysis\n- \ud83d\udd0d Improved Logs Interface: Better search functionality,...", - "prompt.instruction_following": "rag_optimized", - "response.answer_length": 983, - "response.answer_completeness": "complete", - "response.response_quality": "high", - "response.source_citation": 5, - "response.context_utilization": 0.381155486622722, - "output_response": "RAG Answer: The provided context does not specifically mention any new features in cloud computing as a general topic. However, it does highlight several recent enhancements and features related to the Noveum.ai ...", - "evaluation.retrieval_effectiveness": 1, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 5, - "evaluation.context_relevance": "high", - "evaluation.overall_quality": "high", - "evaluation.ready_for_production": true, - "rag.retrieval_strategy": "semantic_similarity", - "rag.vector_search_results": 5, - "rag.context_synthesis": "multi_source", - "rag.document_coverage": 1, - "rag.information_density": 0.381155486622722 - }, - "events": [], - "links": [], - "trace_trace_id": "77b81bd5-5a04-4fc0-9fc6-571f7907d6b1", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:44.888000000", - "trace_end_time": "2025-10-16 08:50:50.529000000", - "trace_duration_ms": 5641.226, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:44.888000000", - "updated_at": "2025-10-16 08:50:44.888000000" - }, - { - "span_id": "4c391bb1-1a62-4e2a-913b-96f83feff494", - "trace_id": "32170d1a-d211-46d8-9cdc-5a4265f0c443", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-16 08:50:42.124000000", - "end_time": "2025-10-16 08:50:43.885000000", - "duration_ms": 1760.974, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What's the current status of cryptocurrency markets?", - "query.length": 52, - "processing.mode": "Web Search", - "processing.time_seconds": 1.7608659267425537, - "processing.response_length": 363, - "processing.sources_count": 1, - "output_response": "Final Answer: The web search did not yield any specific information regarding the current status of cryptocurrency markets. Therefore, I cannot provide an update on market trends, prices, or significant events affe...", - "final_answer_mode": "Web Search", - "query_processed.input_query": "What's the current status of cryptocurrency markets?", - "query_processed.output_response": "Successfully processed query using Web Search, generated 363 character response", - "query_processed.mode": "Web Search", - "query_processed.processing_time": 1.7608659267425537, - "query_processed.response_length": 363 - }, - "events": [], - "links": [], - "trace_trace_id": "32170d1a-d211-46d8-9cdc-5a4265f0c443", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:42.124000000", - "trace_end_time": "2025-10-16 08:50:43.885000000", - "trace_duration_ms": 1761.047, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:42.124000000", - "updated_at": "2025-10-16 08:50:42.124000000" - }, - { - "span_id": "7ab2ec7b-a617-4ebf-9bb1-82dc6b18e160", - "trace_id": "32170d1a-d211-46d8-9cdc-5a4265f0c443", - "parent_span_id": "4c391bb1-1a62-4e2a-913b-96f83feff494", - "name": "agent.query_routing", - "start_time": "2025-10-16 08:50:42.124000000", - "end_time": "2025-10-16 08:50:43.885000000", - "duration_ms": 1760.833, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What's the current status of cryptocurrency markets?", - "query_length": 52, - "query_type": "routing_query", - "output_response": "Routed to Web Search for query processing", - "router.classification": "Web Search", - "router.confidence_score": 1, - "router.rag_keyword_score": 0, - "router.web_keyword_score": 3, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "32170d1a-d211-46d8-9cdc-5a4265f0c443", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:42.124000000", - "trace_end_time": "2025-10-16 08:50:43.885000000", - "trace_duration_ms": 1761.047, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:42.124000000", - "updated_at": "2025-10-16 08:50:42.124000000" - }, - { - "span_id": "db01d228-7c63-4814-b2ea-bdf3898f9c0d", - "trace_id": "32170d1a-d211-46d8-9cdc-5a4265f0c443", - "parent_span_id": "7ab2ec7b-a617-4ebf-9bb1-82dc6b18e160", - "name": "agent.web_search_generation", - "start_time": "2025-10-16 08:50:42.124000000", - "end_time": "2025-10-16 08:50:43.885000000", - "duration_ms": 1760.701, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "web_search_agent", - "agent.operation": "web_search_generation", - "agent.capabilities": [ - "web_search", - "content_synthesis", - "response_generation" - ], - "agent.id": "noveum_web_search_agent", - "input_query": "What's the current status of cryptocurrency markets?", - "query_length": 52, - "query_type": "web_search_query", - "output_response": "Web Search Answer: The web search did not yield any specific information regarding the current status of cryptocurrency markets. Therefore, I cannot provide an update on market trends, prices, or significant events affe...", - "web_search.results_count": 1, - "web_search.context_length": 154, - "web_search.response_length": 363, - "web_search.mode": "external_web_search" - }, - "events": [], - "links": [], - "trace_trace_id": "32170d1a-d211-46d8-9cdc-5a4265f0c443", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:42.124000000", - "trace_end_time": "2025-10-16 08:50:43.885000000", - "trace_duration_ms": 1761.047, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:42.124000000", - "updated_at": "2025-10-16 08:50:42.124000000" - }, - { - "span_id": "da7b0550-b13c-4518-ad3b-050c666d198c", - "trace_id": "32170d1a-d211-46d8-9cdc-5a4265f0c443", - "parent_span_id": "7ab2ec7b-a617-4ebf-9bb1-82dc6b18e160", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-16 08:50:42.124000000", - "end_time": "2025-10-16 08:50:42.124000000", - "duration_ms": 0.029, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What's the current status of cryptocurrency markets?", - "query_length": 52, - "query_type": "routing_evaluation_query", - "classification.mode": "Web Search", - "classification.rag_keyword_score": 0, - "classification.web_keyword_score": 3, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "complex", - "query.intent": "general_knowledge", - "query.keyword_density": 0.42857142857142855, - "query.domain_affinity": "general", - "routing.decision": "Routed to Web Search based on analysis", - "routing.rationale": "RAG score: 0, Web score: 3, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "RAG", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.057692307692307696, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: Web Search (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "32170d1a-d211-46d8-9cdc-5a4265f0c443", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:42.124000000", - "trace_end_time": "2025-10-16 08:50:43.885000000", - "trace_duration_ms": 1761.047, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:42.124000000", - "updated_at": "2025-10-16 08:50:42.124000000" - }, - { - "span_id": "950e18b6-9f62-4ac4-8788-3cac08fb7899", - "trace_id": "32170d1a-d211-46d8-9cdc-5a4265f0c443", - "parent_span_id": "db01d228-7c63-4814-b2ea-bdf3898f9c0d", - "name": "agent.llm_model_execution", - "start_time": "2025-10-16 08:50:42.465000000", - "end_time": "2025-10-16 08:50:43.885000000", - "duration_ms": 1419.5690000000002, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What's the current status of cryptocurrency markets?", - "query_length": 52, - "query_type": "web_search_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 1.4195220470428467, - "model.latency_ms": 1419.5220470428467, - "model.start_time": 1760604642.4658613, - "model.end_time": 1760604643.8853834, - "model.performance_tier": "fast", - "model.prompt_tokens": 162, - "model.completion_tokens": 90, - "model.total_tokens": 252, - "model.tokens_per_second": 177.52454111224782, - "model.estimated_cost": 0.00756, - "model.efficiency_score": 1.4404761904761905, - "model.response_length": 363, - "model.response_quality": "high", - "model.output_response": "Model Response: The web search did not yield any specific information regarding the current status of cryptocurrency markets. Therefore, I cannot provide an update on market trends, prices, or significant events affe...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "32170d1a-d211-46d8-9cdc-5a4265f0c443", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:42.124000000", - "trace_end_time": "2025-10-16 08:50:43.885000000", - "trace_duration_ms": 1761.047, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:42.124000000", - "updated_at": "2025-10-16 08:50:42.124000000" - }, - { - "span_id": "e87846c9-ce82-42c2-a517-e0f67e9d14c1", - "trace_id": "32170d1a-d211-46d8-9cdc-5a4265f0c443", - "parent_span_id": "db01d228-7c63-4814-b2ea-bdf3898f9c0d", - "name": "agent.web_search_evaluation_metrics", - "start_time": "2025-10-16 08:50:43.885000000", - "end_time": "2025-10-16 08:50:43.885000000", - "duration_ms": 0.02, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "web_search_evaluation_metrics", - "agent.capabilities": [ - "web_search_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What's the current status of cryptocurrency markets?", - "query_length": 52, - "query_type": "web_search_evaluation_query", - "web_search.results_count": 1, - "web_search.context_length": 154, - "web_search.context_synthesized": "Context from 1 web sources", - "web_search.search_effectiveness": 0.2, - "web_search.context_quality": "low", - "web_search.source_diversity": 1, - "web_search.information_synthesis": "low", - "web_search.external_knowledge_utilization": 0.154, - "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What's+the+current+status+of+cryptocurrency+markets?\n\nUser Question: What's the current status of cryptocurrency markets?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", - "prompt.prompt_length": 651, - "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What's+the+current+status+of+cryptocurrency+markets?", - "prompt.instruction_following": "web_search_optimized", - "response.answer_length": 363, - "response.answer_completeness": "complete", - "response.response_quality": "medium", - "response.source_citation": 1, - "response.context_utilization": 2.357142857142857, - "output_response": "Web Search Answer: The web search did not yield any specific information regarding the current status of cryptocurrency markets. Therefore, I cannot provide an update on market trends, prices, or significant events affe...", - "evaluation.search_effectiveness": 0.2, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 1, - "evaluation.context_relevance": "low", - "evaluation.overall_quality": "low", - "evaluation.ready_for_production": true, - "web_search.search_strategy": "duckduckgo_api", - "web_search.real_time_data": true, - "web_search.external_sources": 1, - "web_search.information_freshness": "current", - "web_search.knowledge_synthesis": "single_source" - }, - "events": [], - "links": [], - "trace_trace_id": "32170d1a-d211-46d8-9cdc-5a4265f0c443", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:42.124000000", - "trace_end_time": "2025-10-16 08:50:43.885000000", - "trace_duration_ms": 1761.047, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:42.124000000", - "updated_at": "2025-10-16 08:50:42.124000000" - }, - { - "span_id": "305daacf-990b-4e23-8720-48900f75d9be", - "trace_id": "9424d420-deee-478b-b727-9dc791a7457c", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-16 08:50:31.476000000", - "end_time": "2025-10-16 08:50:41.121000000", - "duration_ms": 9644.752999999999, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What are the latest updates in Python programming?", - "query.length": 50, - "processing.mode": "Web Search", - "processing.time_seconds": 9.644662618637085, - "processing.response_length": 1408, - "processing.sources_count": 1, - "output_response": "Final Answer: The web search did not yield specific results regarding the latest updates in Python programming. However, I can provide some general information based on my knowledge up to October 2023.\n\nAs of late ...", - "final_answer_mode": "Web Search", - "query_processed.input_query": "What are the latest updates in Python programming?", - "query_processed.output_response": "Successfully processed query using Web Search, generated 1408 character response", - "query_processed.mode": "Web Search", - "query_processed.processing_time": 9.644662618637085, - "query_processed.response_length": 1408 - }, - "events": [], - "links": [], - "trace_trace_id": "9424d420-deee-478b-b727-9dc791a7457c", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:31.476000000", - "trace_end_time": "2025-10-16 08:50:41.121000000", - "trace_duration_ms": 9644.828, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:31.476000000", - "updated_at": "2025-10-16 08:50:31.476000000" - }, - { - "span_id": "56b4cc1b-0646-4dee-acb3-db98f3d44477", - "trace_id": "9424d420-deee-478b-b727-9dc791a7457c", - "parent_span_id": "fb234e30-69be-4d06-9248-c05bcc22f063", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-16 08:50:31.476000000", - "end_time": "2025-10-16 08:50:31.476000000", - "duration_ms": 0.036000000000000004, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What are the latest updates in Python programming?", - "query_length": 50, - "query_type": "routing_evaluation_query", - "classification.mode": "Web Search", - "classification.rag_keyword_score": 0, - "classification.web_keyword_score": 2, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "general_knowledge", - "query.keyword_density": 0.25, - "query.domain_affinity": "general", - "routing.decision": "Routed to Web Search based on analysis", - "routing.rationale": "RAG score: 0, Web score: 2, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "RAG", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.038461538461538464, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: Web Search (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "9424d420-deee-478b-b727-9dc791a7457c", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:31.476000000", - "trace_end_time": "2025-10-16 08:50:41.121000000", - "trace_duration_ms": 9644.828, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:31.476000000", - "updated_at": "2025-10-16 08:50:31.476000000" - }, - { - "span_id": "fb234e30-69be-4d06-9248-c05bcc22f063", - "trace_id": "9424d420-deee-478b-b727-9dc791a7457c", - "parent_span_id": "305daacf-990b-4e23-8720-48900f75d9be", - "name": "agent.query_routing", - "start_time": "2025-10-16 08:50:31.476000000", - "end_time": "2025-10-16 08:50:41.121000000", - "duration_ms": 9644.627, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What are the latest updates in Python programming?", - "query_length": 50, - "query_type": "routing_query", - "output_response": "Routed to Web Search for query processing", - "router.classification": "Web Search", - "router.confidence_score": 1, - "router.rag_keyword_score": 0, - "router.web_keyword_score": 2, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "9424d420-deee-478b-b727-9dc791a7457c", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:31.476000000", - "trace_end_time": "2025-10-16 08:50:41.121000000", - "trace_duration_ms": 9644.828, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:31.476000000", - "updated_at": "2025-10-16 08:50:31.476000000" - }, - { - "span_id": "f8a793a9-63ee-49b7-9931-d7dcc2bb0ca7", - "trace_id": "9424d420-deee-478b-b727-9dc791a7457c", - "parent_span_id": "fb234e30-69be-4d06-9248-c05bcc22f063", - "name": "agent.web_search_generation", - "start_time": "2025-10-16 08:50:31.477000000", - "end_time": "2025-10-16 08:50:41.121000000", - "duration_ms": 9644.478, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "web_search_agent", - "agent.operation": "web_search_generation", - "agent.capabilities": [ - "web_search", - "content_synthesis", - "response_generation" - ], - "agent.id": "noveum_web_search_agent", - "input_query": "What are the latest updates in Python programming?", - "query_length": 50, - "query_type": "web_search_query", - "output_response": "Web Search Answer: The web search did not yield specific results regarding the latest updates in Python programming. However, I can provide some general information based on my knowledge up to October 2023.\n\nAs of late ...", - "web_search.results_count": 1, - "web_search.context_length": 152, - "web_search.response_length": 1408, - "web_search.mode": "external_web_search" - }, - "events": [], - "links": [], - "trace_trace_id": "9424d420-deee-478b-b727-9dc791a7457c", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:31.476000000", - "trace_end_time": "2025-10-16 08:50:41.121000000", - "trace_duration_ms": 9644.828, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:31.476000000", - "updated_at": "2025-10-16 08:50:31.476000000" - }, - { - "span_id": "b968e3d3-d4aa-4357-85d8-34a612748fbd", - "trace_id": "9424d420-deee-478b-b727-9dc791a7457c", - "parent_span_id": "f8a793a9-63ee-49b7-9931-d7dcc2bb0ca7", - "name": "agent.llm_model_execution", - "start_time": "2025-10-16 08:50:32.016000000", - "end_time": "2025-10-16 08:50:41.121000000", - "duration_ms": 9105.344000000001, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What are the latest updates in Python programming?", - "query_length": 50, - "query_type": "web_search_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 9.105276584625244, - "model.latency_ms": 9105.276584625244, - "model.start_time": 1760604632.0161095, - "model.end_time": 1760604641.121386, - "model.performance_tier": "slow", - "model.prompt_tokens": 161, - "model.completion_tokens": 352, - "model.total_tokens": 513, - "model.tokens_per_second": 56.340957381374714, - "model.estimated_cost": 0.015390000000000001, - "model.efficiency_score": 2.7446393762183234, - "model.response_length": 1408, - "model.response_quality": "high", - "model.output_response": "Model Response: The web search did not yield specific results regarding the latest updates in Python programming. However, I can provide some general information based on my knowledge up to October 2023.\n\nAs of late ...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "9424d420-deee-478b-b727-9dc791a7457c", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:31.476000000", - "trace_end_time": "2025-10-16 08:50:41.121000000", - "trace_duration_ms": 9644.828, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:31.476000000", - "updated_at": "2025-10-16 08:50:31.476000000" - }, - { - "span_id": "80817711-e64c-4a9f-802e-2a9e36ff093d", - "trace_id": "9424d420-deee-478b-b727-9dc791a7457c", - "parent_span_id": "f8a793a9-63ee-49b7-9931-d7dcc2bb0ca7", - "name": "agent.web_search_evaluation_metrics", - "start_time": "2025-10-16 08:50:41.121000000", - "end_time": "2025-10-16 08:50:41.121000000", - "duration_ms": 0.02, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "web_search_evaluation_metrics", - "agent.capabilities": [ - "web_search_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What are the latest updates in Python programming?", - "query_length": 50, - "query_type": "web_search_evaluation_query", - "web_search.results_count": 1, - "web_search.context_length": 152, - "web_search.context_synthesized": "Context from 1 web sources", - "web_search.search_effectiveness": 0.2, - "web_search.context_quality": "low", - "web_search.source_diversity": 1, - "web_search.information_synthesis": "low", - "web_search.external_knowledge_utilization": 0.152, - "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What+are+the+latest+updates+in+Python+programming?\n\nUser Question: What are the latest updates in Python programming?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", - "prompt.prompt_length": 647, - "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What+are+the+latest+updates+in+Python+programming?", - "prompt.instruction_following": "web_search_optimized", - "response.answer_length": 1408, - "response.answer_completeness": "complete", - "response.response_quality": "medium", - "response.source_citation": 1, - "response.context_utilization": 9.263157894736842, - "output_response": "Web Search Answer: The web search did not yield specific results regarding the latest updates in Python programming. However, I can provide some general information based on my knowledge up to October 2023.\n\nAs of late ...", - "evaluation.search_effectiveness": 0.2, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 1, - "evaluation.context_relevance": "low", - "evaluation.overall_quality": "low", - "evaluation.ready_for_production": true, - "web_search.search_strategy": "duckduckgo_api", - "web_search.real_time_data": true, - "web_search.external_sources": 1, - "web_search.information_freshness": "current", - "web_search.knowledge_synthesis": "single_source" - }, - "events": [], - "links": [], - "trace_trace_id": "9424d420-deee-478b-b727-9dc791a7457c", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:31.476000000", - "trace_end_time": "2025-10-16 08:50:41.121000000", - "trace_duration_ms": 9644.828, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:31.476000000", - "updated_at": "2025-10-16 08:50:31.476000000" - }, - { - "span_id": "13af0f3c-bd5c-4024-8fa3-c9e347a1ef20", - "trace_id": "a3735d64-89bb-4fdf-a851-7a1f3515c52d", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-16 08:50:27.687000000", - "end_time": "2025-10-16 08:50:30.473000000", - "duration_ms": 2786.078, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What happened in tech news this week?", - "query.length": 37, - "processing.mode": "Web Search", - "processing.time_seconds": 2.7859997749328613, - "processing.response_length": 396, - "processing.sources_count": 1, - "output_response": "Final Answer: It appears that the web search did not yield any specific results regarding recent tech news. Unfortunately, I cannot provide a summary of this week's tech news based on the available information. For...", - "final_answer_mode": "Web Search", - "query_processed.input_query": "What happened in tech news this week?", - "query_processed.output_response": "Successfully processed query using Web Search, generated 396 character response", - "query_processed.mode": "Web Search", - "query_processed.processing_time": 2.7859997749328613, - "query_processed.response_length": 396 - }, - "events": [], - "links": [], - "trace_trace_id": "a3735d64-89bb-4fdf-a851-7a1f3515c52d", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:27.687000000", - "trace_end_time": "2025-10-16 08:50:30.473000000", - "trace_duration_ms": 2786.145, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:27.687000000", - "updated_at": "2025-10-16 08:50:27.687000000" - }, - { - "span_id": "47e8f375-35e6-4097-86ff-df73e8181c23", - "trace_id": "a3735d64-89bb-4fdf-a851-7a1f3515c52d", - "parent_span_id": "a4f1a7b3-dc35-4f13-96d3-b6dc8fb21a37", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-16 08:50:27.687000000", - "end_time": "2025-10-16 08:50:27.687000000", - "duration_ms": 0.038, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What happened in tech news this week?", - "query_length": 37, - "query_type": "routing_evaluation_query", - "classification.mode": "Web Search", - "classification.rag_keyword_score": 0, - "classification.web_keyword_score": 2, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "general_knowledge", - "query.keyword_density": 0.2857142857142857, - "query.domain_affinity": "general", - "routing.decision": "Routed to Web Search based on analysis", - "routing.rationale": "RAG score: 0, Web score: 2, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "RAG", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.038461538461538464, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: Web Search (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "a3735d64-89bb-4fdf-a851-7a1f3515c52d", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:27.687000000", - "trace_end_time": "2025-10-16 08:50:30.473000000", - "trace_duration_ms": 2786.145, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:27.687000000", - "updated_at": "2025-10-16 08:50:27.687000000" - }, - { - "span_id": "a4f1a7b3-dc35-4f13-96d3-b6dc8fb21a37", - "trace_id": "a3735d64-89bb-4fdf-a851-7a1f3515c52d", - "parent_span_id": "13af0f3c-bd5c-4024-8fa3-c9e347a1ef20", - "name": "agent.query_routing", - "start_time": "2025-10-16 08:50:27.687000000", - "end_time": "2025-10-16 08:50:30.473000000", - "duration_ms": 2785.964, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What happened in tech news this week?", - "query_length": 37, - "query_type": "routing_query", - "output_response": "Routed to Web Search for query processing", - "router.classification": "Web Search", - "router.confidence_score": 1, - "router.rag_keyword_score": 0, - "router.web_keyword_score": 2, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "a3735d64-89bb-4fdf-a851-7a1f3515c52d", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:27.687000000", - "trace_end_time": "2025-10-16 08:50:30.473000000", - "trace_duration_ms": 2786.145, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:27.687000000", - "updated_at": "2025-10-16 08:50:27.687000000" - }, - { - "span_id": "ebfff280-5eb6-4838-8de7-35c85e2e3eaa", - "trace_id": "a3735d64-89bb-4fdf-a851-7a1f3515c52d", - "parent_span_id": "a4f1a7b3-dc35-4f13-96d3-b6dc8fb21a37", - "name": "agent.web_search_generation", - "start_time": "2025-10-16 08:50:27.687000000", - "end_time": "2025-10-16 08:50:30.473000000", - "duration_ms": 2785.8230000000003, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "web_search_agent", - "agent.operation": "web_search_generation", - "agent.capabilities": [ - "web_search", - "content_synthesis", - "response_generation" - ], - "agent.id": "noveum_web_search_agent", - "input_query": "What happened in tech news this week?", - "query_length": 37, - "query_type": "web_search_query", - "output_response": "Web Search Answer: It appears that the web search did not yield any specific results regarding recent tech news. Unfortunately, I cannot provide a summary of this week's tech news based on the available information. For...", - "web_search.results_count": 1, - "web_search.context_length": 139, - "web_search.response_length": 396, - "web_search.mode": "external_web_search" - }, - "events": [], - "links": [], - "trace_trace_id": "a3735d64-89bb-4fdf-a851-7a1f3515c52d", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:27.687000000", - "trace_end_time": "2025-10-16 08:50:30.473000000", - "trace_duration_ms": 2786.145, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:27.687000000", - "updated_at": "2025-10-16 08:50:27.687000000" - }, - { - "span_id": "835b9351-d381-4785-8538-fbe1064fc8cd", - "trace_id": "a3735d64-89bb-4fdf-a851-7a1f3515c52d", - "parent_span_id": "ebfff280-5eb6-4838-8de7-35c85e2e3eaa", - "name": "agent.llm_model_execution", - "start_time": "2025-10-16 08:50:28.026000000", - "end_time": "2025-10-16 08:50:30.473000000", - "duration_ms": 2447.269, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What happened in tech news this week?", - "query_length": 37, - "query_type": "web_search_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 2.447227716445923, - "model.latency_ms": 2447.227716445923, - "model.start_time": 1760604628.0260897, - "model.end_time": 1760604630.4733174, - "model.performance_tier": "medium", - "model.prompt_tokens": 155, - "model.completion_tokens": 99, - "model.total_tokens": 254, - "model.tokens_per_second": 103.79091340501853, - "model.estimated_cost": 0.00762, - "model.efficiency_score": 1.5590551181102361, - "model.response_length": 396, - "model.response_quality": "high", - "model.output_response": "Model Response: It appears that the web search did not yield any specific results regarding recent tech news. Unfortunately, I cannot provide a summary of this week's tech news based on the available information. For...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "a3735d64-89bb-4fdf-a851-7a1f3515c52d", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:27.687000000", - "trace_end_time": "2025-10-16 08:50:30.473000000", - "trace_duration_ms": 2786.145, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:27.687000000", - "updated_at": "2025-10-16 08:50:27.687000000" - }, - { - "span_id": "93c6f427-6fcd-4318-aec8-095553fed469", - "trace_id": "a3735d64-89bb-4fdf-a851-7a1f3515c52d", - "parent_span_id": "ebfff280-5eb6-4838-8de7-35c85e2e3eaa", - "name": "agent.web_search_evaluation_metrics", - "start_time": "2025-10-16 08:50:30.473000000", - "end_time": "2025-10-16 08:50:30.473000000", - "duration_ms": 0.017, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "web_search_evaluation_metrics", - "agent.capabilities": [ - "web_search_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What happened in tech news this week?", - "query_length": 37, - "query_type": "web_search_evaluation_query", - "web_search.results_count": 1, - "web_search.context_length": 139, - "web_search.context_synthesized": "Context from 1 web sources", - "web_search.search_effectiveness": 0.2, - "web_search.context_quality": "low", - "web_search.source_diversity": 1, - "web_search.information_synthesis": "low", - "web_search.external_knowledge_utilization": 0.139, - "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What+happened+in+tech+news+this+week?\n\nUser Question: What happened in tech news this week?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", - "prompt.prompt_length": 621, - "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What+happened+in+tech+news+this+week?", - "prompt.instruction_following": "web_search_optimized", - "response.answer_length": 396, - "response.answer_completeness": "complete", - "response.response_quality": "medium", - "response.source_citation": 1, - "response.context_utilization": 2.8489208633093526, - "output_response": "Web Search Answer: It appears that the web search did not yield any specific results regarding recent tech news. Unfortunately, I cannot provide a summary of this week's tech news based on the available information. For...", - "evaluation.search_effectiveness": 0.2, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 1, - "evaluation.context_relevance": "low", - "evaluation.overall_quality": "low", - "evaluation.ready_for_production": true, - "web_search.search_strategy": "duckduckgo_api", - "web_search.real_time_data": true, - "web_search.external_sources": 1, - "web_search.information_freshness": "current", - "web_search.knowledge_synthesis": "single_source" - }, - "events": [], - "links": [], - "trace_trace_id": "a3735d64-89bb-4fdf-a851-7a1f3515c52d", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:27.687000000", - "trace_end_time": "2025-10-16 08:50:30.473000000", - "trace_duration_ms": 2786.145, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:27.687000000", - "updated_at": "2025-10-16 08:50:27.687000000" - }, - { - "span_id": "622e7353-4eeb-47d5-8558-4ecec28cdcfd", - "trace_id": "fa5bb641-f4b7-4741-b06c-392a5f2a9aea", - "parent_span_id": "c0af6629-48b1-45fc-9dd2-60f098ea140e", - "name": "agent.query_routing", - "start_time": "2025-10-16 08:50:21.645000000", - "end_time": "2025-10-16 08:50:26.683000000", - "duration_ms": 5037.965, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What are the current trends in observability tools?", - "query_length": 51, - "query_type": "routing_query", - "output_response": "Routed to RAG for query processing", - "router.classification": "RAG", - "router.confidence_score": 0.3333333333333333, - "router.rag_keyword_score": 2, - "router.web_keyword_score": 1, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "fa5bb641-f4b7-4741-b06c-392a5f2a9aea", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:21.645000000", - "trace_end_time": "2025-10-16 08:50:26.683000000", - "trace_duration_ms": 5038.112, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:21.645000000", - "updated_at": "2025-10-16 08:50:21.645000000" - }, - { - "span_id": "c0af6629-48b1-45fc-9dd2-60f098ea140e", - "trace_id": "fa5bb641-f4b7-4741-b06c-392a5f2a9aea", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-16 08:50:21.645000000", - "end_time": "2025-10-16 08:50:26.683000000", - "duration_ms": 5038.0419999999995, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What are the current trends in observability tools?", - "query.length": 51, - "processing.mode": "RAG", - "processing.time_seconds": 5.0380027294158936, - "processing.response_length": 897, - "processing.sources_count": 5, - "output_response": "Final Answer: The provided context does not specifically outline current trends in observability tools beyond the focus on AI applications. However, it highlights the limitations of traditional monitoring tools for...", - "final_answer_mode": "RAG", - "query_processed.input_query": "What are the current trends in observability tools?", - "query_processed.output_response": "Successfully processed query using RAG, generated 897 character response", - "query_processed.mode": "RAG", - "query_processed.processing_time": 5.0380027294158936, - "query_processed.response_length": 897 - }, - "events": [], - "links": [], - "trace_trace_id": "fa5bb641-f4b7-4741-b06c-392a5f2a9aea", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:21.645000000", - "trace_end_time": "2025-10-16 08:50:26.683000000", - "trace_duration_ms": 5038.112, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:21.645000000", - "updated_at": "2025-10-16 08:50:21.645000000" - }, - { - "span_id": "ee55906d-ffe3-42fd-9899-a476d4c33ff5", - "trace_id": "fa5bb641-f4b7-4741-b06c-392a5f2a9aea", - "parent_span_id": "622e7353-4eeb-47d5-8558-4ecec28cdcfd", - "name": "agent.llm-rag", - "start_time": "2025-10-16 08:50:21.645000000", - "end_time": "2025-10-16 08:50:26.683000000", - "duration_ms": 5037.803, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "rag_agent", - "agent.operation": "llm-rag", - "agent.capabilities": [ - "document_retrieval", - "context_generation", - "response_generation" - ], - "agent.id": "noveum_rag_agent", - "input_query": "What are the current trends in observability tools?", - "query_length": 51, - "query_type": "rag_query", - "output_response": "RAG Answer: The provided context does not specifically outline current trends in observability tools beyond the focus on AI applications. However, it highlights the limitations of traditional monitoring tools for...", - "rag.context_length": 2943, - "rag.sources_count": 5, - "rag.answer_length": 897, - "rag.mode": "retrieval_augmented_generation" - }, - "events": [], - "links": [], - "trace_trace_id": "fa5bb641-f4b7-4741-b06c-392a5f2a9aea", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:21.645000000", - "trace_end_time": "2025-10-16 08:50:26.683000000", - "trace_duration_ms": 5038.112, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:21.645000000", - "updated_at": "2025-10-16 08:50:21.645000000" - }, - { - "span_id": "fe610ecb-cc8e-4466-b719-34c6e0d45d87", - "trace_id": "fa5bb641-f4b7-4741-b06c-392a5f2a9aea", - "parent_span_id": "622e7353-4eeb-47d5-8558-4ecec28cdcfd", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-16 08:50:21.645000000", - "end_time": "2025-10-16 08:50:21.645000000", - "duration_ms": 0.032, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What are the current trends in observability tools?", - "query_length": 51, - "query_type": "routing_evaluation_query", - "classification.mode": "RAG", - "classification.rag_keyword_score": 2, - "classification.web_keyword_score": 1, - "classification.confidence_score": 0.3333333333333333, - "classification.confidence_level": "medium", - "classification.method": "keyword_based", - "query.complexity": "complex", - "query.intent": "documentation", - "query.keyword_density": 0.375, - "query.domain_affinity": "noveum", - "routing.decision": "Routed to RAG based on analysis", - "routing.rationale": "RAG score: 2, Web score: 1, Confidence: 0.33", - "routing.expected_performance": "medium", - "routing.alternative_mode": "Web Search", - "routing.decision_confidence": 0.3333333333333333, - "evaluation.routing_accuracy": "medium", - "evaluation.keyword_coverage": 0.057692307692307696, - "evaluation.query_understanding": "ambiguous", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: RAG (Confidence: 0.33)" - }, - "events": [], - "links": [], - "trace_trace_id": "fa5bb641-f4b7-4741-b06c-392a5f2a9aea", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:21.645000000", - "trace_end_time": "2025-10-16 08:50:26.683000000", - "trace_duration_ms": 5038.112, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:21.645000000", - "updated_at": "2025-10-16 08:50:21.645000000" - }, - { - "span_id": "69a4773d-c738-489e-87ed-999583e02187", - "trace_id": "fa5bb641-f4b7-4741-b06c-392a5f2a9aea", - "parent_span_id": "ee55906d-ffe3-42fd-9899-a476d4c33ff5", - "name": "agent.llm_model_execution", - "start_time": "2025-10-16 08:50:22.600000000", - "end_time": "2025-10-16 08:50:25.786000000", - "duration_ms": 3185.257, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What are the current trends in observability tools?", - "query_length": 51, - "query_type": "rag_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 3.185206174850464, - "model.latency_ms": 3185.206174850464, - "model.start_time": 1760604622.6008554, - "model.end_time": 1760604625.7860615, - "model.performance_tier": "medium", - "model.prompt_tokens": 872, - "model.completion_tokens": 224, - "model.total_tokens": 1096, - "model.tokens_per_second": 344.09075577390337, - "model.estimated_cost": 0.03288, - "model.efficiency_score": 0.8184306569343066, - "model.response_length": 897, - "model.response_quality": "high", - "model.output_response": "Model Response: The provided context does not specifically outline current trends in observability tools beyond the focus on AI applications. However, it highlights the limitations of traditional monitoring tools for...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "fa5bb641-f4b7-4741-b06c-392a5f2a9aea", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:21.645000000", - "trace_end_time": "2025-10-16 08:50:26.683000000", - "trace_duration_ms": 5038.112, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:21.645000000", - "updated_at": "2025-10-16 08:50:21.645000000" - }, - { - "span_id": "5e5a9d30-4cd0-45cd-9494-192c280727ac", - "trace_id": "fa5bb641-f4b7-4741-b06c-392a5f2a9aea", - "parent_span_id": "ee55906d-ffe3-42fd-9899-a476d4c33ff5", - "name": "agent.rag_evaluation_metrics", - "start_time": "2025-10-16 08:50:25.786000000", - "end_time": "2025-10-16 08:50:25.786000000", - "duration_ms": 0.066, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "rag_evaluation_metrics", - "agent.capabilities": [ - "retrieval_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What are the current trends in observability tools?", - "query_length": 51, - "query_type": "rag_evaluation_query", - "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/blog/from-logs-to-intelligent-choices-inside-noveum-ais-evaluation-process):\nFrom Development to Production - Inside Noveum.ai's AI Observability Platform\nShashank Agarwal\n3/3/2025\nIntroduction\nEvery AI application tells a story\u2014but most developers never get to hear it...", - "retrieval.context_length": 2943, - "retrieval.sources_count": 5, - "retrieval.context_quality": "high", - "retrieval.effectiveness": 1, - "retrieval.context_utilization": 2.943, - "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/blog/from-logs-to-intelligent-choices-inside-noveum-ais-evaluation-process):\nFrom Development to Production - Inside Noveum.ai's AI Observability Platform\nShashank Agarwal\n3/3/2025\nIntroduction\nEvery AI application tells a story\u2014but most developers never get to hear it. When your RAG pipeline returns irrelevant results, when your multi-agent system gets stuck in loops, or when your LLM costs suddenly spike, you're left guessing what went wrong and where.\nThat's exactly the challenge Noveum.ai solves. Rather than flying blind, our platform provides comprehensive tracing a...\n\nSource 2 (https://noveum.ai/docs/getting-started/tracing-concepts):\n- Explore Framework Integrations - Framework-specific guidance\n- Learn Advanced Patterns - Custom instrumentation techniques\n- Master the Dashboard - Analyze your traces effectively\nRemember: Good observability is not about collecting all possible data, but about collecting the right data that helps you understand, debug, and optimize your AI applications.\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to O...\n\nSource 3 (https://noveum.ai/en/docs/getting-started/tracing-concepts):\n- Explore Framework Integrations - Framework-specific guidance\n- Learn Advanced Patterns - Custom instrumentation techniques\n- Master the Dashboard - Analyze your traces effectively\nRemember: Good observability is not about collecting all possible data, but about collecting the right data that helps you understand, debug, and optimize your AI applications.\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to O...\n\nSource 4 (https://noveum.ai/en/blog/from-logs-to-intelligent-choices-inside-noveum-ais-evaluation-process):\nThe AI Observability Challenge\nWhy Traditional Monitoring Falls Short\nTraditional application monitoring tools weren't built for AI applications. They can tell you if your API is responding, but they can't answer the questions that matter most for AI systems:\n- Why did my RAG pipeline retrieve irrelevant documents?\n- Which LLM calls are driving my costs?\n- How are my agents communicating with each other?\n- What's causing hallucinations in my responses?\n- Why is my embedding generation so slow?\nW...\n\nSource 5 (https://noveum.ai/en/docs/advanced/rag-observability):\n\ud83d\udd27 Best Practices for RAG Observability\n1. Comprehensive Pipeline Tracing\n2. Quality Monitoring\n3. A/B Testing for RAG Components\n\ud83c\udfaf Advanced RAG Patterns\nMulti-Modal RAG\nConversational RAG\nRAG observability with Noveum.ai provides the deep insights needed to build, optimize, and scale retrieval-augmented generation systems. By implementing comprehensive tracing across retrieval, context preparation, and generation stages, you can ensure your RAG pipeline delivers accurate, relevant, and cost-effe...\n\nUser Question: What are the current trends in observability tools?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", - "prompt.prompt_length": 3490, - "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/blog/from-logs-to-intelligent-choices-inside-noveum-ais-evaluation-process):\nFrom Development to Production - Inside Noveum.ai's AI Observability Platform\nShashank Agarw...", - "prompt.instruction_following": "rag_optimized", - "response.answer_length": 897, - "response.answer_completeness": "complete", - "response.response_quality": "high", - "response.source_citation": 5, - "response.context_utilization": 0.30479102956167176, - "output_response": "RAG Answer: The provided context does not specifically outline current trends in observability tools beyond the focus on AI applications. However, it highlights the limitations of traditional monitoring tools for...", - "evaluation.retrieval_effectiveness": 1, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 5, - "evaluation.context_relevance": "high", - "evaluation.overall_quality": "high", - "evaluation.ready_for_production": true, - "rag.retrieval_strategy": "semantic_similarity", - "rag.vector_search_results": 5, - "rag.context_synthesis": "multi_source", - "rag.document_coverage": 1, - "rag.information_density": 0.30479102956167176 - }, - "events": [], - "links": [], - "trace_trace_id": "fa5bb641-f4b7-4741-b06c-392a5f2a9aea", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:21.645000000", - "trace_end_time": "2025-10-16 08:50:26.683000000", - "trace_duration_ms": 5038.112, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:21.645000000", - "updated_at": "2025-10-16 08:50:21.645000000" - }, - { - "span_id": "2e06953f-5cec-4873-92b5-91f7281f6d7c", - "trace_id": "c0a95e8e-513c-47eb-b746-4e6016aeaf7d", - "parent_span_id": "9e41eab5-cd8e-4e46-a15b-e5f52c9a4073", - "name": "agent.query_routing", - "start_time": "2025-10-16 08:50:08.944000000", - "end_time": "2025-10-16 08:50:20.641000000", - "duration_ms": 11696.524, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "Tell me about recent developments in machine learning", - "query_length": 53, - "query_type": "routing_query", - "output_response": "Routed to Web Search for query processing", - "router.classification": "Web Search", - "router.confidence_score": 1, - "router.rag_keyword_score": 0, - "router.web_keyword_score": 1, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "c0a95e8e-513c-47eb-b746-4e6016aeaf7d", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:08.944000000", - "trace_end_time": "2025-10-16 08:50:20.641000000", - "trace_duration_ms": 11696.768, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:08.944000000", - "updated_at": "2025-10-16 08:50:08.944000000" - }, - { - "span_id": "9e41eab5-cd8e-4e46-a15b-e5f52c9a4073", - "trace_id": "c0a95e8e-513c-47eb-b746-4e6016aeaf7d", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-16 08:50:08.944000000", - "end_time": "2025-10-16 08:50:20.641000000", - "duration_ms": 11696.671999999999, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "Tell me about recent developments in machine learning", - "query.length": 53, - "processing.mode": "Web Search", - "processing.time_seconds": 11.696574449539185, - "processing.response_length": 2001, - "processing.sources_count": 1, - "output_response": "Final Answer: The web search did not yield specific results regarding recent developments in machine learning. However, I can provide a general overview based on my knowledge up to October 2023.\n\nRecent development...", - "final_answer_mode": "Web Search", - "query_processed.input_query": "Tell me about recent developments in machine learning", - "query_processed.output_response": "Successfully processed query using Web Search, generated 2001 character response", - "query_processed.mode": "Web Search", - "query_processed.processing_time": 11.696574449539185, - "query_processed.response_length": 2001 - }, - "events": [], - "links": [], - "trace_trace_id": "c0a95e8e-513c-47eb-b746-4e6016aeaf7d", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:08.944000000", - "trace_end_time": "2025-10-16 08:50:20.641000000", - "trace_duration_ms": 11696.768, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:08.944000000", - "updated_at": "2025-10-16 08:50:08.944000000" - }, - { - "span_id": "7ddf938b-183a-427f-849b-b31b18a98d87", - "trace_id": "c0a95e8e-513c-47eb-b746-4e6016aeaf7d", - "parent_span_id": "2e06953f-5cec-4873-92b5-91f7281f6d7c", - "name": "agent.web_search_generation", - "start_time": "2025-10-16 08:50:08.945000000", - "end_time": "2025-10-16 08:50:20.641000000", - "duration_ms": 11696.328, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "web_search_agent", - "agent.operation": "web_search_generation", - "agent.capabilities": [ - "web_search", - "content_synthesis", - "response_generation" - ], - "agent.id": "noveum_web_search_agent", - "input_query": "Tell me about recent developments in machine learning", - "query_length": 53, - "query_type": "web_search_query", - "output_response": "Web Search Answer: The web search did not yield specific results regarding recent developments in machine learning. However, I can provide a general overview based on my knowledge up to October 2023.\n\nRecent development...", - "web_search.results_count": 1, - "web_search.context_length": 155, - "web_search.response_length": 2001, - "web_search.mode": "external_web_search" - }, - "events": [], - "links": [], - "trace_trace_id": "c0a95e8e-513c-47eb-b746-4e6016aeaf7d", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:08.944000000", - "trace_end_time": "2025-10-16 08:50:20.641000000", - "trace_duration_ms": 11696.768, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:08.944000000", - "updated_at": "2025-10-16 08:50:08.944000000" - }, - { - "span_id": "f3e4c594-f822-4f04-ab43-98fac161b9a3", - "trace_id": "c0a95e8e-513c-47eb-b746-4e6016aeaf7d", - "parent_span_id": "2e06953f-5cec-4873-92b5-91f7281f6d7c", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-16 08:50:08.945000000", - "end_time": "2025-10-16 08:50:08.945000000", - "duration_ms": 0.046, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: Tell me about recent developments in machine learning", - "query_length": 53, - "query_type": "routing_evaluation_query", - "classification.mode": "Web Search", - "classification.rag_keyword_score": 0, - "classification.web_keyword_score": 1, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "complex", - "query.intent": "general_knowledge", - "query.keyword_density": 0.125, - "query.domain_affinity": "general", - "routing.decision": "Routed to Web Search based on analysis", - "routing.rationale": "RAG score: 0, Web score: 1, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "RAG", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.019230769230769232, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: Web Search (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "c0a95e8e-513c-47eb-b746-4e6016aeaf7d", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:08.944000000", - "trace_end_time": "2025-10-16 08:50:20.641000000", - "trace_duration_ms": 11696.768, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:08.944000000", - "updated_at": "2025-10-16 08:50:08.944000000" - }, - { - "span_id": "b47d09be-a43b-4b0a-b7d3-d98bdf1773a3", - "trace_id": "c0a95e8e-513c-47eb-b746-4e6016aeaf7d", - "parent_span_id": "7ddf938b-183a-427f-849b-b31b18a98d87", - "name": "agent.llm_model_execution", - "start_time": "2025-10-16 08:50:09.490000000", - "end_time": "2025-10-16 08:50:20.641000000", - "duration_ms": 11151.141000000001, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: Tell me about recent developments in machine learning", - "query_length": 53, - "query_type": "web_search_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 11.151098012924194, - "model.latency_ms": 11151.098012924194, - "model.start_time": 1760604609.4902723, - "model.end_time": 1760604620.6413703, - "model.performance_tier": "slow", - "model.prompt_tokens": 163, - "model.completion_tokens": 500, - "model.total_tokens": 663, - "model.tokens_per_second": 59.456028386763236, - "model.estimated_cost": 0.01989, - "model.efficiency_score": 3.018099547511312, - "model.response_length": 2001, - "model.response_quality": "high", - "model.output_response": "Model Response: The web search did not yield specific results regarding recent developments in machine learning. However, I can provide a general overview based on my knowledge up to October 2023.\n\nRecent development...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "c0a95e8e-513c-47eb-b746-4e6016aeaf7d", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:08.944000000", - "trace_end_time": "2025-10-16 08:50:20.641000000", - "trace_duration_ms": 11696.768, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:08.944000000", - "updated_at": "2025-10-16 08:50:08.944000000" - }, - { - "span_id": "bfea8500-0e5c-4556-a014-5b2ec49b7ad5", - "trace_id": "c0a95e8e-513c-47eb-b746-4e6016aeaf7d", - "parent_span_id": "7ddf938b-183a-427f-849b-b31b18a98d87", - "name": "agent.web_search_evaluation_metrics", - "start_time": "2025-10-16 08:50:20.641000000", - "end_time": "2025-10-16 08:50:20.641000000", - "duration_ms": 0.018000000000000002, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "web_search_evaluation_metrics", - "agent.capabilities": [ - "web_search_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: Tell me about recent developments in machine learning", - "query_length": 53, - "query_type": "web_search_evaluation_query", - "web_search.results_count": 1, - "web_search.context_length": 155, - "web_search.context_synthesized": "Context from 1 web sources", - "web_search.search_effectiveness": 0.2, - "web_search.context_quality": "low", - "web_search.source_diversity": 1, - "web_search.information_synthesis": "low", - "web_search.external_knowledge_utilization": 0.155, - "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=Tell+me+about+recent+developments+in+machine+learning\n\nUser Question: Tell me about recent developments in machine learning\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", - "prompt.prompt_length": 653, - "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=Tell+me+about+recent+developments+in+machine+learning", - "prompt.instruction_following": "web_search_optimized", - "response.answer_length": 2001, - "response.answer_completeness": "complete", - "response.response_quality": "medium", - "response.source_citation": 1, - "response.context_utilization": 12.90967741935484, - "output_response": "Web Search Answer: The web search did not yield specific results regarding recent developments in machine learning. However, I can provide a general overview based on my knowledge up to October 2023.\n\nRecent development...", - "evaluation.search_effectiveness": 0.2, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 1, - "evaluation.context_relevance": "low", - "evaluation.overall_quality": "low", - "evaluation.ready_for_production": true, - "web_search.search_strategy": "duckduckgo_api", - "web_search.real_time_data": true, - "web_search.external_sources": 1, - "web_search.information_freshness": "current", - "web_search.knowledge_synthesis": "single_source" - }, - "events": [], - "links": [], - "trace_trace_id": "c0a95e8e-513c-47eb-b746-4e6016aeaf7d", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:08.944000000", - "trace_end_time": "2025-10-16 08:50:20.641000000", - "trace_duration_ms": 11696.768, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:08.944000000", - "updated_at": "2025-10-16 08:50:08.944000000" - }, - { - "span_id": "33297286-d440-446a-ba73-7cabd9d4b2e6", - "trace_id": "3af8c6e8-62d9-4893-92ba-cc265934ef03", - "parent_span_id": "4a3c230e-0dd5-4ba2-8e87-2a301f654fba", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-16 08:50:04.544000000", - "end_time": "2025-10-16 08:50:04.544000000", - "duration_ms": 0.038, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What's the weather like today?", - "query_length": 30, - "query_type": "routing_evaluation_query", - "classification.mode": "Web Search", - "classification.rag_keyword_score": 0, - "classification.web_keyword_score": 2, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "general_knowledge", - "query.keyword_density": 0.4, - "query.domain_affinity": "general", - "routing.decision": "Routed to Web Search based on analysis", - "routing.rationale": "RAG score: 0, Web score: 2, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "RAG", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.038461538461538464, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: Web Search (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "3af8c6e8-62d9-4893-92ba-cc265934ef03", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:04.544000000", - "trace_end_time": "2025-10-16 08:50:07.941000000", - "trace_duration_ms": 3397.228, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:04.544000000", - "updated_at": "2025-10-16 08:50:04.544000000" - }, - { - "span_id": "4a3c230e-0dd5-4ba2-8e87-2a301f654fba", - "trace_id": "3af8c6e8-62d9-4893-92ba-cc265934ef03", - "parent_span_id": "eefcbdc5-99fc-4f06-a9f4-f51ba50564db", - "name": "agent.query_routing", - "start_time": "2025-10-16 08:50:04.544000000", - "end_time": "2025-10-16 08:50:07.941000000", - "duration_ms": 3397.056, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What's the weather like today?", - "query_length": 30, - "query_type": "routing_query", - "output_response": "Routed to Web Search for query processing", - "router.classification": "Web Search", - "router.confidence_score": 1, - "router.rag_keyword_score": 0, - "router.web_keyword_score": 2, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "3af8c6e8-62d9-4893-92ba-cc265934ef03", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:04.544000000", - "trace_end_time": "2025-10-16 08:50:07.941000000", - "trace_duration_ms": 3397.228, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:04.544000000", - "updated_at": "2025-10-16 08:50:04.544000000" - }, - { - "span_id": "eefcbdc5-99fc-4f06-a9f4-f51ba50564db", - "trace_id": "3af8c6e8-62d9-4893-92ba-cc265934ef03", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-16 08:50:04.544000000", - "end_time": "2025-10-16 08:50:07.941000000", - "duration_ms": 3397.169, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What's the weather like today?", - "query.length": 30, - "processing.mode": "Web Search", - "processing.time_seconds": 3.3970956802368164, - "processing.response_length": 518, - "processing.sources_count": 1, - "output_response": "Final Answer: The web search results do not provide specific details about today's weather conditions in your location. However, they mention that you can get real-time updates, including temperature, humidity, and...", - "final_answer_mode": "Web Search", - "query_processed.input_query": "What's the weather like today?", - "query_processed.output_response": "Successfully processed query using Web Search, generated 518 character response", - "query_processed.mode": "Web Search", - "query_processed.processing_time": 3.3970956802368164, - "query_processed.response_length": 518 - }, - "events": [], - "links": [], - "trace_trace_id": "3af8c6e8-62d9-4893-92ba-cc265934ef03", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:04.544000000", - "trace_end_time": "2025-10-16 08:50:07.941000000", - "trace_duration_ms": 3397.228, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:04.544000000", - "updated_at": "2025-10-16 08:50:04.544000000" - }, - { - "span_id": "9683465c-e851-4004-a858-c8fdd1a2b3da", - "trace_id": "3af8c6e8-62d9-4893-92ba-cc265934ef03", - "parent_span_id": "4a3c230e-0dd5-4ba2-8e87-2a301f654fba", - "name": "agent.web_search_generation", - "start_time": "2025-10-16 08:50:04.545000000", - "end_time": "2025-10-16 08:50:07.941000000", - "duration_ms": 3396.871, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "web_search_agent", - "agent.operation": "web_search_generation", - "agent.capabilities": [ - "web_search", - "content_synthesis", - "response_generation" - ], - "agent.id": "noveum_web_search_agent", - "input_query": "What's the weather like today?", - "query_length": 30, - "query_type": "web_search_query", - "output_response": "Web Search Answer: The web search results do not provide specific details about today's weather conditions in your location. However, they mention that you can get real-time updates, including temperature, humidity, and...", - "web_search.results_count": 1, - "web_search.context_length": 1137, - "web_search.response_length": 518, - "web_search.mode": "external_web_search" - }, - "events": [], - "links": [], - "trace_trace_id": "3af8c6e8-62d9-4893-92ba-cc265934ef03", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:04.544000000", - "trace_end_time": "2025-10-16 08:50:07.941000000", - "trace_duration_ms": 3397.228, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:04.544000000", - "updated_at": "2025-10-16 08:50:04.544000000" - }, - { - "span_id": "6413eb7e-e5cd-4c46-9651-de14c72c260e", - "trace_id": "3af8c6e8-62d9-4893-92ba-cc265934ef03", - "parent_span_id": "9683465c-e851-4004-a858-c8fdd1a2b3da", - "name": "agent.llm_model_execution", - "start_time": "2025-10-16 08:50:04.986000000", - "end_time": "2025-10-16 08:50:07.941000000", - "duration_ms": 2955.55, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What's the weather like today?", - "query_length": 30, - "query_type": "web_search_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 2.955509662628174, - "model.latency_ms": 2955.509662628174, - "model.start_time": 1760604604.9862766, - "model.end_time": 1760604607.9417863, - "model.performance_tier": "medium", - "model.prompt_tokens": 403, - "model.completion_tokens": 129, - "model.total_tokens": 532, - "model.tokens_per_second": 180.0027950261957, - "model.estimated_cost": 0.015960000000000002, - "model.efficiency_score": 0.9736842105263158, - "model.response_length": 518, - "model.response_quality": "high", - "model.output_response": "Model Response: The web search results do not provide specific details about today's weather conditions in your location. However, they mention that you can get real-time updates, including temperature, humidity, and...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "3af8c6e8-62d9-4893-92ba-cc265934ef03", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:04.544000000", - "trace_end_time": "2025-10-16 08:50:07.941000000", - "trace_duration_ms": 3397.228, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:04.544000000", - "updated_at": "2025-10-16 08:50:04.544000000" - }, - { - "span_id": "64df69fb-0bcc-4845-8bfe-5a721c1ed912", - "trace_id": "3af8c6e8-62d9-4893-92ba-cc265934ef03", - "parent_span_id": "9683465c-e851-4004-a858-c8fdd1a2b3da", - "name": "agent.web_search_evaluation_metrics", - "start_time": "2025-10-16 08:50:07.941000000", - "end_time": "2025-10-16 08:50:07.941000000", - "duration_ms": 0.017, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "web_search_evaluation_metrics", - "agent.capabilities": [ - "web_search_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What's the weather like today?", - "query_length": 30, - "query_type": "web_search_evaluation_query", - "web_search.results_count": 1, - "web_search.context_length": 1137, - "web_search.context_synthesized": "Context from 1 web sources", - "web_search.search_effectiveness": 0.2, - "web_search.context_quality": "high", - "web_search.source_diversity": 1, - "web_search.information_synthesis": "low", - "web_search.external_knowledge_utilization": 1.137, - "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nStay updated with the latest weather conditions in Komna. Get real-time updates, temperature, humidity, AQI, and detailed forecast for today. Dec 31, 2024 \u00b7 Komna Weather Forecast: Check the latest temperature in Komna today along with details such as humidity, visibility, air quality, wind speed, pressure, and weekly weather report at Shuru. 2 days ago \u00b7 Latest weather forecast for India for today's, hourly weather forecast, including today's temperatures in India, wind, rain and more. 2 days ago \u00b7 Get the latest hourly weather updates for India today. Detailed forecast including temperature, wind, rain, snow, and UV index. Stay informed about today's weather conditions in India. 2 days ago \u00b7 Today's Weather Report ... Forecasts Urban Meteorological Services Short to Medium Range Model Guidance Extended Range Model Guidance Seasonal Forecast Quantitative Precipitation Forecast Latest CAP Alerts Specialised products & forecasts Cyclones | Monsoon | Satellite Radar | Rainfall | Agromet Advisories MAUSAM : | | Video Meghdoot Agro ...\nURL: https://duckduckgo.com/?q=What's+the+weather+like+today?\n\nUser Question: What's the weather like today?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", - "prompt.prompt_length": 1612, - "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nStay updated with the latest weather conditions in Komna. Get real-time updates, temperature, humidity, AQI, and detailed forecast for today. Dec 31, 2024 \u00b7 Komna Weather F...", - "prompt.instruction_following": "web_search_optimized", - "response.answer_length": 518, - "response.answer_completeness": "complete", - "response.response_quality": "medium", - "response.source_citation": 1, - "response.context_utilization": 0.455584872471416, - "output_response": "Web Search Answer: The web search results do not provide specific details about today's weather conditions in your location. However, they mention that you can get real-time updates, including temperature, humidity, and...", - "evaluation.search_effectiveness": 0.2, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 1, - "evaluation.context_relevance": "high", - "evaluation.overall_quality": "low", - "evaluation.ready_for_production": true, - "web_search.search_strategy": "duckduckgo_api", - "web_search.real_time_data": true, - "web_search.external_sources": 1, - "web_search.information_freshness": "current", - "web_search.knowledge_synthesis": "single_source" - }, - "events": [], - "links": [], - "trace_trace_id": "3af8c6e8-62d9-4893-92ba-cc265934ef03", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:04.544000000", - "trace_end_time": "2025-10-16 08:50:07.941000000", - "trace_duration_ms": 3397.228, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:04.544000000", - "updated_at": "2025-10-16 08:50:04.544000000" - }, - { - "span_id": "719a1a8f-fc50-439c-899b-436146accd23", - "trace_id": "d7b8b811-ad01-48f1-917f-ace7e8d289e6", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-16 08:50:01.375000000", - "end_time": "2025-10-16 08:50:03.540000000", - "duration_ms": 2164.4809999999998, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What are the latest AI news today?", - "query.length": 34, - "processing.mode": "Web Search", - "processing.time_seconds": 2.1643810272216797, - "processing.response_length": 298, - "processing.sources_count": 1, - "output_response": "Final Answer: The web search did not yield any specific results regarding the latest AI news today. Unfortunately, I cannot provide current updates or news articles on AI developments at this moment. For the latest...", - "final_answer_mode": "Web Search", - "query_processed.input_query": "What are the latest AI news today?", - "query_processed.output_response": "Successfully processed query using Web Search, generated 298 character response", - "query_processed.mode": "Web Search", - "query_processed.processing_time": 2.1643810272216797, - "query_processed.response_length": 298 - }, - "events": [], - "links": [], - "trace_trace_id": "d7b8b811-ad01-48f1-917f-ace7e8d289e6", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:01.375000000", - "trace_end_time": "2025-10-16 08:50:03.540000000", - "trace_duration_ms": 2164.576, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:01.375000000", - "updated_at": "2025-10-16 08:50:01.375000000" - }, - { - "span_id": "644a712f-1c9e-4e9a-9073-74621b754aa2", - "trace_id": "d7b8b811-ad01-48f1-917f-ace7e8d289e6", - "parent_span_id": "719a1a8f-fc50-439c-899b-436146accd23", - "name": "agent.query_routing", - "start_time": "2025-10-16 08:50:01.376000000", - "end_time": "2025-10-16 08:50:03.540000000", - "duration_ms": 2164.3219999999997, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What are the latest AI news today?", - "query_length": 34, - "query_type": "routing_query", - "output_response": "Routed to Web Search for query processing", - "router.classification": "Web Search", - "router.confidence_score": 1, - "router.rag_keyword_score": 0, - "router.web_keyword_score": 3, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "d7b8b811-ad01-48f1-917f-ace7e8d289e6", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:01.375000000", - "trace_end_time": "2025-10-16 08:50:03.540000000", - "trace_duration_ms": 2164.576, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:01.375000000", - "updated_at": "2025-10-16 08:50:01.375000000" - }, - { - "span_id": "8a4abdbe-6ad7-4d43-b234-81667c4abbc2", - "trace_id": "d7b8b811-ad01-48f1-917f-ace7e8d289e6", - "parent_span_id": "644a712f-1c9e-4e9a-9073-74621b754aa2", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-16 08:50:01.376000000", - "end_time": "2025-10-16 08:50:01.376000000", - "duration_ms": 0.051, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What are the latest AI news today?", - "query_length": 34, - "query_type": "routing_evaluation_query", - "classification.mode": "Web Search", - "classification.rag_keyword_score": 0, - "classification.web_keyword_score": 3, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "general_knowledge", - "query.keyword_density": 0.42857142857142855, - "query.domain_affinity": "general", - "routing.decision": "Routed to Web Search based on analysis", - "routing.rationale": "RAG score: 0, Web score: 3, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "RAG", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.057692307692307696, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: Web Search (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "d7b8b811-ad01-48f1-917f-ace7e8d289e6", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:01.375000000", - "trace_end_time": "2025-10-16 08:50:03.540000000", - "trace_duration_ms": 2164.576, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:01.375000000", - "updated_at": "2025-10-16 08:50:01.375000000" - }, - { - "span_id": "904621e8-c5fa-4b2d-a628-8f10eb9da51d", - "trace_id": "d7b8b811-ad01-48f1-917f-ace7e8d289e6", - "parent_span_id": "644a712f-1c9e-4e9a-9073-74621b754aa2", - "name": "agent.web_search_generation", - "start_time": "2025-10-16 08:50:01.376000000", - "end_time": "2025-10-16 08:50:03.540000000", - "duration_ms": 2164.076, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "web_search_agent", - "agent.operation": "web_search_generation", - "agent.capabilities": [ - "web_search", - "content_synthesis", - "response_generation" - ], - "agent.id": "noveum_web_search_agent", - "input_query": "What are the latest AI news today?", - "query_length": 34, - "query_type": "web_search_query", - "output_response": "Web Search Answer: The web search did not yield any specific results regarding the latest AI news today. Unfortunately, I cannot provide current updates or news articles on AI developments at this moment. For the latest...", - "web_search.results_count": 1, - "web_search.context_length": 136, - "web_search.response_length": 298, - "web_search.mode": "external_web_search" - }, - "events": [], - "links": [], - "trace_trace_id": "d7b8b811-ad01-48f1-917f-ace7e8d289e6", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:01.375000000", - "trace_end_time": "2025-10-16 08:50:03.540000000", - "trace_duration_ms": 2164.576, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:01.375000000", - "updated_at": "2025-10-16 08:50:01.375000000" - }, - { - "span_id": "421dd1af-2dbc-477f-bd59-b3194b14ae3a", - "trace_id": "d7b8b811-ad01-48f1-917f-ace7e8d289e6", - "parent_span_id": "904621e8-c5fa-4b2d-a628-8f10eb9da51d", - "name": "agent.llm_model_execution", - "start_time": "2025-10-16 08:50:01.708000000", - "end_time": "2025-10-16 08:50:03.540000000", - "duration_ms": 1831.237, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What are the latest AI news today?", - "query_length": 34, - "query_type": "web_search_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 1.8311803340911865, - "model.latency_ms": 1831.1803340911865, - "model.start_time": 1760604601.7090185, - "model.end_time": 1760604603.5401988, - "model.performance_tier": "fast", - "model.prompt_tokens": 153, - "model.completion_tokens": 74, - "model.total_tokens": 227, - "model.tokens_per_second": 123.96376029926071, - "model.estimated_cost": 0.00681, - "model.efficiency_score": 1.3127753303964758, - "model.response_length": 298, - "model.response_quality": "high", - "model.output_response": "Model Response: The web search did not yield any specific results regarding the latest AI news today. Unfortunately, I cannot provide current updates or news articles on AI developments at this moment. For the latest...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "d7b8b811-ad01-48f1-917f-ace7e8d289e6", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:01.375000000", - "trace_end_time": "2025-10-16 08:50:03.540000000", - "trace_duration_ms": 2164.576, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:01.375000000", - "updated_at": "2025-10-16 08:50:01.375000000" - }, - { - "span_id": "159aad6f-7174-406d-b7d9-eed9360d993c", - "trace_id": "d7b8b811-ad01-48f1-917f-ace7e8d289e6", - "parent_span_id": "904621e8-c5fa-4b2d-a628-8f10eb9da51d", - "name": "agent.web_search_evaluation_metrics", - "start_time": "2025-10-16 08:50:03.540000000", - "end_time": "2025-10-16 08:50:03.540000000", - "duration_ms": 0.028, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "web_search_evaluation_metrics", - "agent.capabilities": [ - "web_search_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What are the latest AI news today?", - "query_length": 34, - "query_type": "web_search_evaluation_query", - "web_search.results_count": 1, - "web_search.context_length": 136, - "web_search.context_synthesized": "Context from 1 web sources", - "web_search.search_effectiveness": 0.2, - "web_search.context_quality": "low", - "web_search.source_diversity": 1, - "web_search.information_synthesis": "low", - "web_search.external_knowledge_utilization": 0.136, - "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What+are+the+latest+AI+news+today?\n\nUser Question: What are the latest AI news today?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", - "prompt.prompt_length": 615, - "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What+are+the+latest+AI+news+today?", - "prompt.instruction_following": "web_search_optimized", - "response.answer_length": 298, - "response.answer_completeness": "complete", - "response.response_quality": "medium", - "response.source_citation": 1, - "response.context_utilization": 2.1911764705882355, - "output_response": "Web Search Answer: The web search did not yield any specific results regarding the latest AI news today. Unfortunately, I cannot provide current updates or news articles on AI developments at this moment. For the latest...", - "evaluation.search_effectiveness": 0.2, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 1, - "evaluation.context_relevance": "low", - "evaluation.overall_quality": "low", - "evaluation.ready_for_production": true, - "web_search.search_strategy": "duckduckgo_api", - "web_search.real_time_data": true, - "web_search.external_sources": 1, - "web_search.information_freshness": "current", - "web_search.knowledge_synthesis": "single_source" - }, - "events": [], - "links": [], - "trace_trace_id": "d7b8b811-ad01-48f1-917f-ace7e8d289e6", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:01.375000000", - "trace_end_time": "2025-10-16 08:50:03.540000000", - "trace_duration_ms": 2164.576, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:01.375000000", - "updated_at": "2025-10-16 08:50:01.375000000" - }, - { - "span_id": "dcf2b38e-ab90-45ee-9a65-e6fea47a3fd7", - "trace_id": "df8340b4-b75a-48e9-a2f3-99725b46a00d", - "parent_span_id": "d0e2b320-c18d-4ec6-98a1-74da5252107e", - "name": "agent.query_routing", - "start_time": "2025-10-16 08:49:17.236000000", - "end_time": "2025-10-16 08:50:00.368000000", - "duration_ms": 43131.384, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What are the benefits of using Noveum Trace?", - "query_length": 44, - "query_type": "routing_query", - "output_response": "Routed to RAG for query processing", - "router.classification": "RAG", - "router.confidence_score": 1, - "router.rag_keyword_score": 2, - "router.web_keyword_score": 0, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "df8340b4-b75a-48e9-a2f3-99725b46a00d", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:49:17.236000000", - "trace_end_time": "2025-10-16 08:50:00.368000000", - "trace_duration_ms": 43131.554000000004, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:49:17.236000000", - "updated_at": "2025-10-16 08:49:17.236000000" - }, - { - "span_id": "d0e2b320-c18d-4ec6-98a1-74da5252107e", - "trace_id": "df8340b4-b75a-48e9-a2f3-99725b46a00d", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-16 08:49:17.236000000", - "end_time": "2025-10-16 08:50:00.368000000", - "duration_ms": 43131.481, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What are the benefits of using Noveum Trace?", - "query.length": 44, - "processing.mode": "RAG", - "processing.time_seconds": 43.131428718566895, - "processing.response_length": 1415, - "processing.sources_count": 5, - "output_response": "Final Answer: The benefits of using Noveum Trace include:\n\n1. **Comprehensive Tracing**: Noveum Trace offers decorator-based tracing for LLM calls, agents, and RAG pipelines, allowing for detailed monitoring of AI ...", - "final_answer_mode": "RAG", - "query_processed.input_query": "What are the benefits of using Noveum Trace?", - "query_processed.output_response": "Successfully processed query using RAG, generated 1415 character response", - "query_processed.mode": "RAG", - "query_processed.processing_time": 43.131428718566895, - "query_processed.response_length": 1415 - }, - "events": [], - "links": [], - "trace_trace_id": "df8340b4-b75a-48e9-a2f3-99725b46a00d", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:49:17.236000000", - "trace_end_time": "2025-10-16 08:50:00.368000000", - "trace_duration_ms": 43131.554000000004, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:49:17.236000000", - "updated_at": "2025-10-16 08:49:17.236000000" - }, - { - "span_id": "fa93821f-c809-4071-87e1-a6c55014d37c", - "trace_id": "df8340b4-b75a-48e9-a2f3-99725b46a00d", - "parent_span_id": "dcf2b38e-ab90-45ee-9a65-e6fea47a3fd7", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-16 08:49:17.236000000", - "end_time": "2025-10-16 08:49:17.236000000", - "duration_ms": 0.034, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What are the benefits of using Noveum Trace?", - "query_length": 44, - "query_type": "routing_evaluation_query", - "classification.mode": "RAG", - "classification.rag_keyword_score": 2, - "classification.web_keyword_score": 0, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "noveum_specific", - "query.keyword_density": 0.25, - "query.domain_affinity": "noveum", - "routing.decision": "Routed to RAG based on analysis", - "routing.rationale": "RAG score: 2, Web score: 0, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "Web Search", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.038461538461538464, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: RAG (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "df8340b4-b75a-48e9-a2f3-99725b46a00d", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:49:17.236000000", - "trace_end_time": "2025-10-16 08:50:00.368000000", - "trace_duration_ms": 43131.554000000004, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:49:17.236000000", - "updated_at": "2025-10-16 08:49:17.236000000" - }, - { - "span_id": "024ad5aa-f394-4c26-af51-564f3f51c6f2", - "trace_id": "df8340b4-b75a-48e9-a2f3-99725b46a00d", - "parent_span_id": "dcf2b38e-ab90-45ee-9a65-e6fea47a3fd7", - "name": "agent.llm-rag", - "start_time": "2025-10-16 08:49:17.237000000", - "end_time": "2025-10-16 08:50:00.368000000", - "duration_ms": 43131.19300000001, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "rag_agent", - "agent.operation": "llm-rag", - "agent.capabilities": [ - "document_retrieval", - "context_generation", - "response_generation" - ], - "agent.id": "noveum_rag_agent", - "input_query": "What are the benefits of using Noveum Trace?", - "query_length": 44, - "query_type": "rag_query", - "output_response": "RAG Answer: The benefits of using Noveum Trace include:\n\n1. **Comprehensive Tracing**: Noveum Trace offers decorator-based tracing for LLM calls, agents, and RAG pipelines, allowing for detailed monitoring of AI ...", - "rag.context_length": 2773, - "rag.sources_count": 5, - "rag.answer_length": 1415, - "rag.mode": "retrieval_augmented_generation" - }, - "events": [], - "links": [], - "trace_trace_id": "df8340b4-b75a-48e9-a2f3-99725b46a00d", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:49:17.236000000", - "trace_end_time": "2025-10-16 08:50:00.368000000", - "trace_duration_ms": 43131.554000000004, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:49:17.236000000", - "updated_at": "2025-10-16 08:49:17.236000000" - }, - { - "span_id": "3d9db951-d8b0-4990-820d-61e4b1011709", - "trace_id": "df8340b4-b75a-48e9-a2f3-99725b46a00d", - "parent_span_id": "024ad5aa-f394-4c26-af51-564f3f51c6f2", - "name": "agent.llm_model_execution", - "start_time": "2025-10-16 08:49:20.224000000", - "end_time": "2025-10-16 08:49:58.420000000", - "duration_ms": 38196.111000000004, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What are the benefits of using Noveum Trace?", - "query_length": 44, - "query_type": "rag_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 38.19604015350342, - "model.latency_ms": 38196.04015350342, - "model.start_time": 1760604560.2242246, - "model.end_time": 1760604598.4202647, - "model.performance_tier": "slow", - "model.prompt_tokens": 828, - "model.completion_tokens": 353, - "model.total_tokens": 1181, - "model.tokens_per_second": 30.919435503098253, - "model.estimated_cost": 0.03543, - "model.efficiency_score": 1.1981371718882303, - "model.response_length": 1415, - "model.response_quality": "high", - "model.output_response": "Model Response: The benefits of using Noveum Trace include:\n\n1. **Comprehensive Tracing**: Noveum Trace offers decorator-based tracing for LLM calls, agents, and RAG pipelines, allowing for detailed monitoring of AI ...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "df8340b4-b75a-48e9-a2f3-99725b46a00d", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:49:17.236000000", - "trace_end_time": "2025-10-16 08:50:00.368000000", - "trace_duration_ms": 43131.554000000004, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:49:17.236000000", - "updated_at": "2025-10-16 08:49:17.236000000" - }, - { - "span_id": "8c33ffac-4d76-42ba-9737-89596e136b3d", - "trace_id": "df8340b4-b75a-48e9-a2f3-99725b46a00d", - "parent_span_id": "024ad5aa-f394-4c26-af51-564f3f51c6f2", - "name": "agent.rag_evaluation_metrics", - "start_time": "2025-10-16 08:49:58.420000000", - "end_time": "2025-10-16 08:49:58.420000000", - "duration_ms": 0.037, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "rag_evaluation_metrics", - "agent.capabilities": [ - "retrieval_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What are the benefits of using Noveum Trace?", - "query_length": 44, - "query_type": "rag_evaluation_query", - "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/docs/platform/dashboard):\n- Bulk Export: Download multiple traces simultaneously\n- Comparative Analysis: Compare performance across multiple traces\nReal-Time Updates\n- Live Refresh: Automatic updates as new traces arrive\n- Connection Monitoring: Real-time status of your t...", - "retrieval.context_length": 2773, - "retrieval.sources_count": 5, - "retrieval.context_quality": "high", - "retrieval.effectiveness": 1, - "retrieval.context_utilization": 2.773, - "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/docs/platform/dashboard):\n- Bulk Export: Download multiple traces simultaneously\n- Comparative Analysis: Compare performance across multiple traces\nReal-Time Updates\n- Live Refresh: Automatic updates as new traces arrive\n- Connection Monitoring: Real-time status of your trace ingestion pipeline\n- Performance Indicators: Live metrics for system health monitoring\nReady to dive deeper? Explore Projects & Environments to organize your AI applications, or check out Team Collaboration to share insights with your team.\nGet Earl...\n\nSource 2 (https://noveum.ai/en/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\n\ud83d\udcd8 TypeScript SDK (\n@noveum/trace\n)- Framework integrations for Next.js, Express.js, and Hono\n- TypeScript-first design with full type safety\n- Universal compatibility (Node.js, Edge Runtime, browsers)\n-\n\ud83d\udcca Noveum Platform\n- Real-time dashboard for analyzing traces and performance\n- Ad...\n\nSource 3 (https://noveum.ai/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\n\ud83d\udcd8 TypeScript SDK (\n@noveum/trace\n)- Framework integrations for Next.js, Express.js, and Hono\n- TypeScript-first design with full type safety\n- Universal compatibility (Node.js, Edge Runtime, browsers)\n-\n\ud83d\udcca Noveum Platform\n- Real-time dashboard for analyzing traces and performance\n- Ad...\n\nSource 4 (https://noveum.ai/en/docs/getting-started/overview):\n- Advanced search & filtering across traces and spans\n- Cost analysis and optimization recommendations\n- Team collaboration with shared insights and alerts\n\ud83d\udd0d What Noveum Traces\nLLM Operations\n- Model calls across all providers (OpenAI, Anthropic, Google, etc.)\n- Token usage and cost calculation\n- Prompt engineering effectiveness\n- Response quality metrics\nRAG Pipelines\n- Document retrieval performance and relevance\n- Embedding generation costs and latency\n- Context assembly and prompt constructi...\n\nSource 5 (https://noveum.ai/docs/getting-started/overview):\n- Advanced search & filtering across traces and spans\n- Cost analysis and optimization recommendations\n- Team collaboration with shared insights and alerts\n\ud83d\udd0d What Noveum Traces\nLLM Operations\n- Model calls across all providers (OpenAI, Anthropic, Google, etc.)\n- Token usage and cost calculation\n- Prompt engineering effectiveness\n- Response quality metrics\nRAG Pipelines\n- Document retrieval performance and relevance\n- Embedding generation costs and latency\n- Context assembly and prompt constructi...\n\nUser Question: What are the benefits of using Noveum Trace?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", - "prompt.prompt_length": 3313, - "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/docs/platform/dashboard):\n- Bulk Export: Download multiple traces simultaneously\n- Comparative Analysis: Compare performance across multiple traces\nReal-Time Updates\n- Live...", - "prompt.instruction_following": "rag_optimized", - "response.answer_length": 1415, - "response.answer_completeness": "complete", - "response.response_quality": "high", - "response.source_citation": 5, - "response.context_utilization": 0.5102776776054815, - "output_response": "RAG Answer: The benefits of using Noveum Trace include:\n\n1. **Comprehensive Tracing**: Noveum Trace offers decorator-based tracing for LLM calls, agents, and RAG pipelines, allowing for detailed monitoring of AI ...", - "evaluation.retrieval_effectiveness": 1, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 5, - "evaluation.context_relevance": "high", - "evaluation.overall_quality": "high", - "evaluation.ready_for_production": true, - "rag.retrieval_strategy": "semantic_similarity", - "rag.vector_search_results": 5, - "rag.context_synthesis": "multi_source", - "rag.document_coverage": 1, - "rag.information_density": 0.5102776776054815 - }, - "events": [], - "links": [], - "trace_trace_id": "df8340b4-b75a-48e9-a2f3-99725b46a00d", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:49:17.236000000", - "trace_end_time": "2025-10-16 08:50:00.368000000", - "trace_duration_ms": 43131.554000000004, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:49:17.236000000", - "updated_at": "2025-10-16 08:49:17.236000000" - }, - { - "span_id": "5b1a7121-d0bd-4525-821d-7721c6fbbd70", - "trace_id": "325ec189-2876-4ebe-b33c-649491a5053e", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-16 08:49:11.192000000", - "end_time": "2025-10-16 08:49:16.232000000", - "duration_ms": 5039.753, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "How do I configure Noveum for my system?", - "query.length": 40, - "processing.mode": "RAG", - "processing.time_seconds": 5.039698600769043, - "processing.response_length": 712, - "processing.sources_count": 5, - "output_response": "Final Answer: To configure Noveum for your system, you can refer to the SDK Integration Guide available in the Noveum documentation. This guide will help you trace your AI applications with minimal code changes. Th...", - "final_answer_mode": "RAG", - "query_processed.input_query": "How do I configure Noveum for my system?", - "query_processed.output_response": "Successfully processed query using RAG, generated 712 character response", - "query_processed.mode": "RAG", - "query_processed.processing_time": 5.039698600769043, - "query_processed.response_length": 712 - }, - "events": [], - "links": [], - "trace_trace_id": "325ec189-2876-4ebe-b33c-649491a5053e", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:49:11.192000000", - "trace_end_time": "2025-10-16 08:49:16.232000000", - "trace_duration_ms": 5039.841, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:49:11.192000000", - "updated_at": "2025-10-16 08:49:11.192000000" - }, - { - "span_id": "761c92c6-d15a-4a8e-a16a-a365922f8d7c", - "trace_id": "325ec189-2876-4ebe-b33c-649491a5053e", - "parent_span_id": "5b1a7121-d0bd-4525-821d-7721c6fbbd70", - "name": "agent.query_routing", - "start_time": "2025-10-16 08:49:11.192000000", - "end_time": "2025-10-16 08:49:16.232000000", - "duration_ms": 5039.642, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "How do I configure Noveum for my system?", - "query_length": 40, - "query_type": "routing_query", - "output_response": "Routed to RAG for query processing", - "router.classification": "RAG", - "router.confidence_score": 1, - "router.rag_keyword_score": 2, - "router.web_keyword_score": 0, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "325ec189-2876-4ebe-b33c-649491a5053e", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:49:11.192000000", - "trace_end_time": "2025-10-16 08:49:16.232000000", - "trace_duration_ms": 5039.841, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:49:11.192000000", - "updated_at": "2025-10-16 08:49:11.192000000" - }, - { - "span_id": "8fcf969e-a577-45aa-9fe6-1870d9665189", - "trace_id": "325ec189-2876-4ebe-b33c-649491a5053e", - "parent_span_id": "761c92c6-d15a-4a8e-a16a-a365922f8d7c", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-16 08:49:11.192000000", - "end_time": "2025-10-16 08:49:11.192000000", - "duration_ms": 0.045000000000000005, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: How do I configure Noveum for my system?", - "query_length": 40, - "query_type": "routing_evaluation_query", - "classification.mode": "RAG", - "classification.rag_keyword_score": 2, - "classification.web_keyword_score": 0, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "noveum_specific", - "query.keyword_density": 0.25, - "query.domain_affinity": "noveum", - "routing.decision": "Routed to RAG based on analysis", - "routing.rationale": "RAG score: 2, Web score: 0, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "Web Search", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.038461538461538464, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: RAG (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "325ec189-2876-4ebe-b33c-649491a5053e", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:49:11.192000000", - "trace_end_time": "2025-10-16 08:49:16.232000000", - "trace_duration_ms": 5039.841, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:49:11.192000000", - "updated_at": "2025-10-16 08:49:11.192000000" - }, - { - "span_id": "9e5bbea4-cc68-4fe6-b3d0-c87852635ec9", - "trace_id": "325ec189-2876-4ebe-b33c-649491a5053e", - "parent_span_id": "761c92c6-d15a-4a8e-a16a-a365922f8d7c", - "name": "agent.llm-rag", - "start_time": "2025-10-16 08:49:11.192000000", - "end_time": "2025-10-16 08:49:16.232000000", - "duration_ms": 5039.39, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "rag_agent", - "agent.operation": "llm-rag", - "agent.capabilities": [ - "document_retrieval", - "context_generation", - "response_generation" - ], - "agent.id": "noveum_rag_agent", - "input_query": "How do I configure Noveum for my system?", - "query_length": 40, - "query_type": "rag_query", - "output_response": "RAG Answer: To configure Noveum for your system, you can refer to the SDK Integration Guide available in the Noveum documentation. This guide will help you trace your AI applications with minimal code changes. Th...", - "rag.context_length": 2237, - "rag.sources_count": 5, - "rag.answer_length": 712, - "rag.mode": "retrieval_augmented_generation" - }, - "events": [], - "links": [], - "trace_trace_id": "325ec189-2876-4ebe-b33c-649491a5053e", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:49:11.192000000", - "trace_end_time": "2025-10-16 08:49:16.232000000", - "trace_duration_ms": 5039.841, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:49:11.192000000", - "updated_at": "2025-10-16 08:49:11.192000000" - }, - { - "span_id": "5ee706ad-c4ae-4c0b-94d1-6b9c43d173b4", - "trace_id": "325ec189-2876-4ebe-b33c-649491a5053e", - "parent_span_id": "9e5bbea4-cc68-4fe6-b3d0-c87852635ec9", - "name": "agent.llm_model_execution", - "start_time": "2025-10-16 08:49:11.607000000", - "end_time": "2025-10-16 08:49:15.718000000", - "duration_ms": 4110.915, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: How do I configure Noveum for my system?", - "query_length": 40, - "query_type": "rag_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 4.110846042633057, - "model.latency_ms": 4110.846042633057, - "model.start_time": 1760604551.6074958, - "model.end_time": 1760604555.7183418, - "model.performance_tier": "medium", - "model.prompt_tokens": 693, - "model.completion_tokens": 178, - "model.total_tokens": 871, - "model.tokens_per_second": 211.87852596934326, - "model.estimated_cost": 0.02613, - "model.efficiency_score": 0.817451205510907, - "model.response_length": 712, - "model.response_quality": "high", - "model.output_response": "Model Response: To configure Noveum for your system, you can refer to the SDK Integration Guide available in the Noveum documentation. This guide will help you trace your AI applications with minimal code changes. Th...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "325ec189-2876-4ebe-b33c-649491a5053e", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:49:11.192000000", - "trace_end_time": "2025-10-16 08:49:16.232000000", - "trace_duration_ms": 5039.841, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:49:11.192000000", - "updated_at": "2025-10-16 08:49:11.192000000" - }, - { - "span_id": "d13bdeda-81af-400a-82ca-a5ef7e1e8a6d", - "trace_id": "325ec189-2876-4ebe-b33c-649491a5053e", - "parent_span_id": "9e5bbea4-cc68-4fe6-b3d0-c87852635ec9", - "name": "agent.rag_evaluation_metrics", - "start_time": "2025-10-16 08:49:15.718000000", - "end_time": "2025-10-16 08:49:15.718000000", - "duration_ms": 0.118, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "rag_evaluation_metrics", - "agent.capabilities": [ - "retrieval_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: How do I configure Noveum for my system?", - "query_length": 40, - "query_type": "rag_evaluation_query", - "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/docs):\nWelcome to Noveum.ai Docs\nComprehensive AI tracing and observability for LLM applications, RAG systems, and AI agents\nWelcome to the Noveum.ai documentation! Here you'll find everything you need to integrate, configure, and optimize your AI applications using ou...", - "retrieval.context_length": 2237, - "retrieval.sources_count": 5, - "retrieval.context_quality": "high", - "retrieval.effectiveness": 1, - "retrieval.context_utilization": 2.237, - "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/docs):\nWelcome to Noveum.ai Docs\nComprehensive AI tracing and observability for LLM applications, RAG systems, and AI agents\nWelcome to the Noveum.ai documentation! Here you'll find everything you need to integrate, configure, and optimize your AI applications using our comprehensive tracing and observability platform.\nWe provide powerful SDKs for Python and TypeScript that enable you to trace LLM calls, RAG pipelines, and multi-agent workflows with minimal code changes. Our documentation is organized ...\n\nSource 2 (https://noveum.ai/docs):\nWelcome to Noveum.ai Docs\nComprehensive AI tracing and observability for LLM applications, RAG systems, and AI agents\nWelcome to the Noveum.ai documentation! Here you'll find everything you need to integrate, configure, and optimize your AI applications using our comprehensive tracing and observability platform.\nWe provide powerful SDKs for Python and TypeScript that enable you to trace LLM calls, RAG pipelines, and multi-agent workflows with minimal code changes. Our documentation is organized ...\n\nSource 3 (https://noveum.ai/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nSource 4 (https://noveum.ai/en/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nSource 5 (https://noveum.ai/en/docs):\n- \ud83c\udf0d Universal: Works across frameworks, clouds, and deployment models\n\ud83e\udd1d Community & Support\n- \ud83d\udcac Discord Community: Join our Discord\n- \ud83d\udce7 Email Support: [email protected]\n- \ud83d\udc1b Bug Reports: GitHub Issues\n- \ud83d\udcd6 Knowledge Base: Help Center\nReady to get started? Head to our SDK Integration Guide to begin tracing your AI applications in under 5 minutes!\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability ...\n\nUser Question: How do I configure Noveum for my system?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", - "prompt.prompt_length": 2773, - "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/docs):\nWelcome to Noveum.ai Docs\nComprehensive AI tracing and observability for LLM applications, RAG systems, and AI agents\nWelcome to the Noveum.ai documentation! Here...", - "prompt.instruction_following": "rag_optimized", - "response.answer_length": 712, - "response.answer_completeness": "complete", - "response.response_quality": "high", - "response.source_citation": 5, - "response.context_utilization": 0.3182834152883326, - "output_response": "RAG Answer: To configure Noveum for your system, you can refer to the SDK Integration Guide available in the Noveum documentation. This guide will help you trace your AI applications with minimal code changes. Th...", - "evaluation.retrieval_effectiveness": 1, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 5, - "evaluation.context_relevance": "high", - "evaluation.overall_quality": "high", - "evaluation.ready_for_production": true, - "rag.retrieval_strategy": "semantic_similarity", - "rag.vector_search_results": 5, - "rag.context_synthesis": "multi_source", - "rag.document_coverage": 1, - "rag.information_density": 0.3182834152883326 - }, - "events": [], - "links": [], - "trace_trace_id": "325ec189-2876-4ebe-b33c-649491a5053e", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:49:11.192000000", - "trace_end_time": "2025-10-16 08:49:16.232000000", - "trace_duration_ms": 5039.841, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:49:11.192000000", - "updated_at": "2025-10-16 08:49:11.192000000" - }, - { - "span_id": "7d6564fc-41b2-4d74-acb1-e77b575c0006", - "trace_id": "d5528586-c741-4703-8791-5fccf5c3e123", - "parent_span_id": "e79a037e-82b1-41e9-bdc7-42353e98ead7", - "name": "agent.query_routing", - "start_time": "2025-10-16 08:49:05.152000000", - "end_time": "2025-10-16 08:49:10.187000000", - "duration_ms": 5034.446, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What monitoring capabilities does Noveum provide?", - "query_length": 49, - "query_type": "routing_query", - "output_response": "Routed to RAG for query processing", - "router.classification": "RAG", - "router.confidence_score": 1, - "router.rag_keyword_score": 2, - "router.web_keyword_score": 0, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "d5528586-c741-4703-8791-5fccf5c3e123", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:49:05.152000000", - "trace_end_time": "2025-10-16 08:49:10.187000000", - "trace_duration_ms": 5034.604, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:49:05.152000000", - "updated_at": "2025-10-16 08:49:05.152000000" - }, - { - "span_id": "e79a037e-82b1-41e9-bdc7-42353e98ead7", - "trace_id": "d5528586-c741-4703-8791-5fccf5c3e123", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-16 08:49:05.152000000", - "end_time": "2025-10-16 08:49:10.187000000", - "duration_ms": 5034.531, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What monitoring capabilities does Noveum provide?", - "query.length": 49, - "processing.mode": "RAG", - "processing.time_seconds": 5.0344908237457275, - "processing.response_length": 552, - "processing.sources_count": 5, - "output_response": "Final Answer: Noveum provides comprehensive monitoring capabilities for AI applications, including the ability to monitor, trace, and optimize AI agents across various frameworks such as LangChain, CrewAI, AutoGen,...", - "final_answer_mode": "RAG", - "query_processed.input_query": "What monitoring capabilities does Noveum provide?", - "query_processed.output_response": "Successfully processed query using RAG, generated 552 character response", - "query_processed.mode": "RAG", - "query_processed.processing_time": 5.0344908237457275, - "query_processed.response_length": 552 - }, - "events": [], - "links": [], - "trace_trace_id": "d5528586-c741-4703-8791-5fccf5c3e123", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:49:05.152000000", - "trace_end_time": "2025-10-16 08:49:10.187000000", - "trace_duration_ms": 5034.604, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:49:05.152000000", - "updated_at": "2025-10-16 08:49:05.152000000" - }, - { - "span_id": "78297793-d73b-4311-b5ba-73509ae9f96e", - "trace_id": "d5528586-c741-4703-8791-5fccf5c3e123", - "parent_span_id": "7d6564fc-41b2-4d74-acb1-e77b575c0006", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-16 08:49:05.153000000", - "end_time": "2025-10-16 08:49:05.153000000", - "duration_ms": 0.029, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What monitoring capabilities does Noveum provide?", - "query_length": 49, - "query_type": "routing_evaluation_query", - "classification.mode": "RAG", - "classification.rag_keyword_score": 2, - "classification.web_keyword_score": 0, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "noveum_specific", - "query.keyword_density": 0.3333333333333333, - "query.domain_affinity": "noveum", - "routing.decision": "Routed to RAG based on analysis", - "routing.rationale": "RAG score: 2, Web score: 0, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "Web Search", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.038461538461538464, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: RAG (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "d5528586-c741-4703-8791-5fccf5c3e123", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:49:05.152000000", - "trace_end_time": "2025-10-16 08:49:10.187000000", - "trace_duration_ms": 5034.604, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:49:05.152000000", - "updated_at": "2025-10-16 08:49:05.152000000" - }, - { - "span_id": "6a101d9d-de83-415a-8fb0-01a9fae5ffb9", - "trace_id": "d5528586-c741-4703-8791-5fccf5c3e123", - "parent_span_id": "7d6564fc-41b2-4d74-acb1-e77b575c0006", - "name": "agent.llm-rag", - "start_time": "2025-10-16 08:49:05.153000000", - "end_time": "2025-10-16 08:49:10.187000000", - "duration_ms": 5034.282, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "rag_agent", - "agent.operation": "llm-rag", - "agent.capabilities": [ - "document_retrieval", - "context_generation", - "response_generation" - ], - "agent.id": "noveum_rag_agent", - "input_query": "What monitoring capabilities does Noveum provide?", - "query_length": 49, - "query_type": "rag_query", - "output_response": "RAG Answer: Noveum provides comprehensive monitoring capabilities for AI applications, including the ability to monitor, trace, and optimize AI agents across various frameworks such as LangChain, CrewAI, AutoGen,...", - "rag.context_length": 2740, - "rag.sources_count": 5, - "rag.answer_length": 552, - "rag.mode": "retrieval_augmented_generation" - }, - "events": [], - "links": [], - "trace_trace_id": "d5528586-c741-4703-8791-5fccf5c3e123", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:49:05.152000000", - "trace_end_time": "2025-10-16 08:49:10.187000000", - "trace_duration_ms": 5034.604, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:49:05.152000000", - "updated_at": "2025-10-16 08:49:05.152000000" - }, - { - "span_id": "26e510c0-441a-4f9f-9dce-31180f47d060", - "trace_id": "d5528586-c741-4703-8791-5fccf5c3e123", - "parent_span_id": "6a101d9d-de83-415a-8fb0-01a9fae5ffb9", - "name": "agent.llm_model_execution", - "start_time": "2025-10-16 08:49:05.787000000", - "end_time": "2025-10-16 08:49:09.274000000", - "duration_ms": 3486.699, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What monitoring capabilities does Noveum provide?", - "query_length": 49, - "query_type": "rag_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 3.4866299629211426, - "model.latency_ms": 3486.6299629211426, - "model.start_time": 1760604545.787763, - "model.end_time": 1760604549.274393, - "model.performance_tier": "medium", - "model.prompt_tokens": 821, - "model.completion_tokens": 138, - "model.total_tokens": 959, - "model.tokens_per_second": 275.0506965747916, - "model.estimated_cost": 0.02877, - "model.efficiency_score": 0.5755995828988529, - "model.response_length": 552, - "model.response_quality": "high", - "model.output_response": "Model Response: Noveum provides comprehensive monitoring capabilities for AI applications, including the ability to monitor, trace, and optimize AI agents across various frameworks such as LangChain, CrewAI, AutoGen,...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "d5528586-c741-4703-8791-5fccf5c3e123", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:49:05.152000000", - "trace_end_time": "2025-10-16 08:49:10.187000000", - "trace_duration_ms": 5034.604, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:49:05.152000000", - "updated_at": "2025-10-16 08:49:05.152000000" - }, - { - "span_id": "fcd89e38-3d21-468b-860a-ae2de0bbd119", - "trace_id": "d5528586-c741-4703-8791-5fccf5c3e123", - "parent_span_id": "6a101d9d-de83-415a-8fb0-01a9fae5ffb9", - "name": "agent.rag_evaluation_metrics", - "start_time": "2025-10-16 08:49:09.274000000", - "end_time": "2025-10-16 08:49:09.274000000", - "duration_ms": 0.054, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "rag_evaluation_metrics", - "agent.capabilities": [ - "retrieval_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What monitoring capabilities does Noveum provide?", - "query_length": 49, - "query_type": "rag_evaluation_query", - "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en):\nMonitor all your AI Agents\nimprove AI Agents today\nNoveum.ai helps you monitor, trace, and optimize your AI applications.\nNoveum.ai works with any AI framework \u2013 LangChain, CrewAI, AutoGen, custom implementations, or direct LLM calls. One dashboard shows everything.\n...", - "retrieval.context_length": 2740, - "retrieval.sources_count": 5, - "retrieval.context_quality": "high", - "retrieval.effectiveness": 1, - "retrieval.context_utilization": 2.74, - "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en):\nMonitor all your AI Agents\nimprove AI Agents today\nNoveum.ai helps you monitor, trace, and optimize your AI applications.\nNoveum.ai works with any AI framework \u2013 LangChain, CrewAI, AutoGen, custom implementations, or direct LLM calls. One dashboard shows everything.\nMonitor, Evaluate, Improve Your AI Agents\nThe control plane for AI agents.\nMonitor Everything, Miss Nothing\nOur lightweight SDKs capture every trace and span across your AI agent ecosystem\u2014from simple LLM calls to complex multi-agent...\n\nSource 2 (https://noveum.ai/):\nMonitor all your AI Agents\nimprove AI Agents today\nNoveum.ai helps you monitor, trace, and optimize your AI applications.\nNoveum.ai works with any AI framework \u2013 LangChain, CrewAI, AutoGen, custom implementations, or direct LLM calls. One dashboard shows everything.\nMonitor, Evaluate, Improve Your AI Agents\nThe control plane for AI agents.\nMonitor Everything, Miss Nothing\nOur lightweight SDKs capture every trace and span across your AI agent ecosystem\u2014from simple LLM calls to complex multi-agent...\n\nSource 3 (https://noveum.ai):\nMonitor all your AI Agents\nimprove AI Agents today\nNoveum.ai helps you monitor, trace, and optimize your AI applications.\nNoveum.ai works with any AI framework \u2013 LangChain, CrewAI, AutoGen, custom implementations, or direct LLM calls. One dashboard shows everything.\nMonitor, Evaluate, Improve Your AI Agents\nThe control plane for AI agents.\nMonitor Everything, Miss Nothing\nOur lightweight SDKs capture every trace and span across your AI agent ecosystem\u2014from simple LLM calls to complex multi-agent...\n\nSource 4 (https://noveum.ai/en/docs/getting-started/overview):\nNoveum.ai Overview\nComprehensive AI tracing and observability platform for LLM applications, RAG systems, and AI agents\nWelcome to Noveum.ai\u2014the comprehensive tracing and observability platform built specifically for AI applications. Whether you're building LLM-powered chatbots, RAG systems, multi-agent workflows, or any AI-driven application, Noveum provides the insights you need to understand, debug, and optimize your systems.\n\ud83c\udfaf Why AI Applications Need Specialized Observability\nTraditional mo...\n\nSource 5 (https://noveum.ai/docs/getting-started/overview):\nNoveum.ai Overview\nComprehensive AI tracing and observability platform for LLM applications, RAG systems, and AI agents\nWelcome to Noveum.ai\u2014the comprehensive tracing and observability platform built specifically for AI applications. Whether you're building LLM-powered chatbots, RAG systems, multi-agent workflows, or any AI-driven application, Noveum provides the insights you need to understand, debug, and optimize your systems.\n\ud83c\udfaf Why AI Applications Need Specialized Observability\nTraditional mo...\n\nUser Question: What monitoring capabilities does Noveum provide?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", - "prompt.prompt_length": 3285, - "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en):\nMonitor all your AI Agents\nimprove AI Agents today\nNoveum.ai helps you monitor, trace, and optimize your AI applications.\nNoveum.ai works with any AI framework \u2013 LangC...", - "prompt.instruction_following": "rag_optimized", - "response.answer_length": 552, - "response.answer_completeness": "complete", - "response.response_quality": "high", - "response.source_citation": 5, - "response.context_utilization": 0.20145985401459854, - "output_response": "RAG Answer: Noveum provides comprehensive monitoring capabilities for AI applications, including the ability to monitor, trace, and optimize AI agents across various frameworks such as LangChain, CrewAI, AutoGen,...", - "evaluation.retrieval_effectiveness": 1, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 5, - "evaluation.context_relevance": "high", - "evaluation.overall_quality": "high", - "evaluation.ready_for_production": true, - "rag.retrieval_strategy": "semantic_similarity", - "rag.vector_search_results": 5, - "rag.context_synthesis": "multi_source", - "rag.document_coverage": 1, - "rag.information_density": 0.20145985401459854 - }, - "events": [], - "links": [], - "trace_trace_id": "d5528586-c741-4703-8791-5fccf5c3e123", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:49:05.152000000", - "trace_end_time": "2025-10-16 08:49:10.187000000", - "trace_duration_ms": 5034.604, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:49:05.152000000", - "updated_at": "2025-10-16 08:49:05.152000000" - }, - { - "span_id": "bd5f205f-0329-4811-b883-129d69f12382", - "trace_id": "8100b869-07c0-4e37-b51e-092c9d20b4a7", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-16 08:48:57.574000000", - "end_time": "2025-10-16 08:49:04.149000000", - "duration_ms": 6574.424, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "How does Noveum handle agent tracing?", - "query.length": 37, - "processing.mode": "RAG", - "processing.time_seconds": 6.5743818283081055, - "processing.response_length": 643, - "processing.sources_count": 5, - "output_response": "Final Answer: Noveum.ai handles agent tracing through its specialized Multi-Agent Tracing capabilities, which are designed to observe complex workflows and inter-agent communications within multi-agent systems. The...", - "final_answer_mode": "RAG", - "query_processed.input_query": "How does Noveum handle agent tracing?", - "query_processed.output_response": "Successfully processed query using RAG, generated 643 character response", - "query_processed.mode": "RAG", - "query_processed.processing_time": 6.5743818283081055, - "query_processed.response_length": 643 - }, - "events": [], - "links": [], - "trace_trace_id": "8100b869-07c0-4e37-b51e-092c9d20b4a7", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:48:57.574000000", - "trace_end_time": "2025-10-16 08:49:04.149000000", - "trace_duration_ms": 6574.500999999999, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:48:57.574000000", - "updated_at": "2025-10-16 08:48:57.574000000" - }, - { - "span_id": "0801bcfe-8c09-4ea6-a754-5564395d6ded", - "trace_id": "8100b869-07c0-4e37-b51e-092c9d20b4a7", - "parent_span_id": "3c35d86a-0df9-42dc-bfe3-bbe978cf62b5", - "name": "agent.llm-rag", - "start_time": "2025-10-16 08:48:57.575000000", - "end_time": "2025-10-16 08:49:04.149000000", - "duration_ms": 6574.135, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "rag_agent", - "agent.operation": "llm-rag", - "agent.capabilities": [ - "document_retrieval", - "context_generation", - "response_generation" - ], - "agent.id": "noveum_rag_agent", - "input_query": "How does Noveum handle agent tracing?", - "query_length": 37, - "query_type": "rag_query", - "output_response": "RAG Answer: Noveum.ai handles agent tracing through its specialized Multi-Agent Tracing capabilities, which are designed to observe complex workflows and inter-agent communications within multi-agent systems. The...", - "rag.context_length": 2736, - "rag.sources_count": 5, - "rag.answer_length": 643, - "rag.mode": "retrieval_augmented_generation" - }, - "events": [], - "links": [], - "trace_trace_id": "8100b869-07c0-4e37-b51e-092c9d20b4a7", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:48:57.574000000", - "trace_end_time": "2025-10-16 08:49:04.149000000", - "trace_duration_ms": 6574.500999999999, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:48:57.574000000", - "updated_at": "2025-10-16 08:48:57.574000000" - }, - { - "span_id": "3c35d86a-0df9-42dc-bfe3-bbe978cf62b5", - "trace_id": "8100b869-07c0-4e37-b51e-092c9d20b4a7", - "parent_span_id": "bd5f205f-0329-4811-b883-129d69f12382", - "name": "agent.query_routing", - "start_time": "2025-10-16 08:48:57.575000000", - "end_time": "2025-10-16 08:49:04.149000000", - "duration_ms": 6574.342, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "How does Noveum handle agent tracing?", - "query_length": 37, - "query_type": "routing_query", - "output_response": "Routed to RAG for query processing", - "router.classification": "RAG", - "router.confidence_score": 1, - "router.rag_keyword_score": 2, - "router.web_keyword_score": 0, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "8100b869-07c0-4e37-b51e-092c9d20b4a7", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:48:57.574000000", - "trace_end_time": "2025-10-16 08:49:04.149000000", - "trace_duration_ms": 6574.500999999999, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:48:57.574000000", - "updated_at": "2025-10-16 08:48:57.574000000" - }, - { - "span_id": "b1914316-f545-4bc9-abf4-3494cf42c4a5", - "trace_id": "8100b869-07c0-4e37-b51e-092c9d20b4a7", - "parent_span_id": "3c35d86a-0df9-42dc-bfe3-bbe978cf62b5", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-16 08:48:57.575000000", - "end_time": "2025-10-16 08:48:57.575000000", - "duration_ms": 0.032, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: How does Noveum handle agent tracing?", - "query_length": 37, - "query_type": "routing_evaluation_query", - "classification.mode": "RAG", - "classification.rag_keyword_score": 2, - "classification.web_keyword_score": 0, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "noveum_specific", - "query.keyword_density": 0.3333333333333333, - "query.domain_affinity": "noveum", - "routing.decision": "Routed to RAG based on analysis", - "routing.rationale": "RAG score: 2, Web score: 0, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "Web Search", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.038461538461538464, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: RAG (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "8100b869-07c0-4e37-b51e-092c9d20b4a7", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:48:57.574000000", - "trace_end_time": "2025-10-16 08:49:04.149000000", - "trace_duration_ms": 6574.500999999999, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:48:57.574000000", - "updated_at": "2025-10-16 08:48:57.574000000" - }, - { - "span_id": "0906814b-6a80-40e1-9f00-5376b809703a", - "trace_id": "8100b869-07c0-4e37-b51e-092c9d20b4a7", - "parent_span_id": "0801bcfe-8c09-4ea6-a754-5564395d6ded", - "name": "agent.llm_model_execution", - "start_time": "2025-10-16 08:48:58.009000000", - "end_time": "2025-10-16 08:49:03.349000000", - "duration_ms": 5340.01, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: How does Noveum handle agent tracing?", - "query_length": 37, - "query_type": "rag_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 5.339954614639282, - "model.latency_ms": 5339.954614639282, - "model.start_time": 1760604538.0093303, - "model.end_time": 1760604543.349285, - "model.performance_tier": "slow", - "model.prompt_tokens": 817, - "model.completion_tokens": 160, - "model.total_tokens": 977, - "model.tokens_per_second": 182.9603565022054, - "model.estimated_cost": 0.02931, - "model.efficiency_score": 0.6581371545547595, - "model.response_length": 643, - "model.response_quality": "high", - "model.output_response": "Model Response: Noveum.ai handles agent tracing through its specialized Multi-Agent Tracing capabilities, which are designed to observe complex workflows and inter-agent communications within multi-agent systems. The...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "8100b869-07c0-4e37-b51e-092c9d20b4a7", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:48:57.574000000", - "trace_end_time": "2025-10-16 08:49:04.149000000", - "trace_duration_ms": 6574.500999999999, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:48:57.574000000", - "updated_at": "2025-10-16 08:48:57.574000000" - }, - { - "span_id": "29822edb-c15f-4baa-b6ec-84d85fcc926d", - "trace_id": "8100b869-07c0-4e37-b51e-092c9d20b4a7", - "parent_span_id": "0801bcfe-8c09-4ea6-a754-5564395d6ded", - "name": "agent.rag_evaluation_metrics", - "start_time": "2025-10-16 08:49:03.349000000", - "end_time": "2025-10-16 08:49:03.349000000", - "duration_ms": 0.034999999999999996, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "rag_evaluation_metrics", - "agent.capabilities": [ - "retrieval_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: How does Noveum handle agent tracing?", - "query_length": 37, - "query_type": "rag_evaluation_query", - "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, vector searches, data retrieval, agent reasoning, and more. Without proper observability, debugging ...", - "retrieval.context_length": 2736, - "retrieval.sources_count": 5, - "retrieval.context_quality": "high", - "retrieval.effectiveness": 1, - "retrieval.context_utilization": 2.736, - "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, vector searches, data retrieval, agent reasoning, and more. Without proper observability, debugging becomes a nightmare, optimization is guesswork, and understanding user interactions is nearly impossible.\nNoveum.ai solves this by providing comprehensive tracing and observability specifically designed for AI applications.\nKey benefits of using Noveum.ai:\n- Complete Visibility: Trace...\n\nSource 2 (https://noveum.ai/docs/advanced/multi-agent-tracing):\nMulti-Agent Tracing\nObserve complex agent workflows and inter-agent communications with comprehensive tracing\nMulti-agent systems represent some of the most complex AI applications, involving multiple agents that coordinate, communicate, and collaborate to achieve shared goals. Noveum.ai provides specialized tracing capabilities to help you understand and optimize these intricate workflows.\n\ud83c\udfaf Why Multi-Agent Tracing Matters\nMulti-agent systems introduce unique observability challenges:\n- Complex...\n\nSource 3 (https://noveum.ai/en/docs/advanced/multi-agent-tracing):\nMulti-Agent Tracing\nObserve complex agent workflows and inter-agent communications with comprehensive tracing\nMulti-agent systems represent some of the most complex AI applications, involving multiple agents that coordinate, communicate, and collaborate to achieve shared goals. Noveum.ai provides specialized tracing capabilities to help you understand and optimize these intricate workflows.\n\ud83c\udfaf Why Multi-Agent Tracing Matters\nMulti-agent systems introduce unique observability challenges:\n- Complex...\n\nSource 4 (https://noveum.ai/en/docs/advanced/performance-optimization):\n\ud83d\udd17 Next Steps\n- Multi-Agent Tracing - Observe agent workflows\n- RAG Pipeline Observability - Monitor retrieval and generation systems\n- Custom Instrumentation - Add domain-specific tracing\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nSource 5 (https://noveum.ai/docs/advanced/performance-optimization):\n\ud83d\udd17 Next Steps\n- Multi-Agent Tracing - Observe agent workflows\n- RAG Pipeline Observability - Monitor retrieval and generation systems\n- Custom Instrumentation - Add domain-specific tracing\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nUser Question: How does Noveum handle agent tracing?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", - "prompt.prompt_length": 3269, - "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, ...", - "prompt.instruction_following": "rag_optimized", - "response.answer_length": 643, - "response.answer_completeness": "complete", - "response.response_quality": "high", - "response.source_citation": 5, - "response.context_utilization": 0.23501461988304093, - "output_response": "RAG Answer: Noveum.ai handles agent tracing through its specialized Multi-Agent Tracing capabilities, which are designed to observe complex workflows and inter-agent communications within multi-agent systems. The...", - "evaluation.retrieval_effectiveness": 1, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 5, - "evaluation.context_relevance": "high", - "evaluation.overall_quality": "high", - "evaluation.ready_for_production": true, - "rag.retrieval_strategy": "semantic_similarity", - "rag.vector_search_results": 5, - "rag.context_synthesis": "multi_source", - "rag.document_coverage": 1, - "rag.information_density": 0.23501461988304093 - }, - "events": [], - "links": [], - "trace_trace_id": "8100b869-07c0-4e37-b51e-092c9d20b4a7", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:48:57.574000000", - "trace_end_time": "2025-10-16 08:49:04.149000000", - "trace_duration_ms": 6574.500999999999, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:48:57.574000000", - "updated_at": "2025-10-16 08:48:57.574000000" - }, - { - "span_id": "5f8ea007-edef-477c-b8f1-57c2da9e9258", - "trace_id": "c28bd31c-d200-4467-91fd-a3b80187c885", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-16 08:48:17.977000000", - "end_time": "2025-10-16 08:48:56.570000000", - "duration_ms": 38592.912, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What APIs are available in Noveum platform?", - "query.length": 43, - "processing.mode": "RAG", - "processing.time_seconds": 38.59272003173828, - "processing.response_length": 270, - "processing.sources_count": 5, - "output_response": "Final Answer: The provided context does not specify the APIs available in the Noveum platform. For detailed information about the APIs, I recommend checking the official Noveum documentation or the SDK Integration ...", - "final_answer_mode": "RAG", - "query_processed.input_query": "What APIs are available in Noveum platform?", - "query_processed.output_response": "Successfully processed query using RAG, generated 270 character response", - "query_processed.mode": "RAG", - "query_processed.processing_time": 38.59272003173828, - "query_processed.response_length": 270 - }, - "events": [], - "links": [], - "trace_trace_id": "c28bd31c-d200-4467-91fd-a3b80187c885", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:48:17.977000000", - "trace_end_time": "2025-10-16 08:48:56.570000000", - "trace_duration_ms": 38592.996, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:48:17.977000000", - "updated_at": "2025-10-16 08:48:17.977000000" - }, - { - "span_id": "db385f4d-beef-465b-9831-a65080537519", - "trace_id": "c28bd31c-d200-4467-91fd-a3b80187c885", - "parent_span_id": "5f8ea007-edef-477c-b8f1-57c2da9e9258", - "name": "agent.query_routing", - "start_time": "2025-10-16 08:48:17.977000000", - "end_time": "2025-10-16 08:48:56.570000000", - "duration_ms": 38592.66499999999, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What APIs are available in Noveum platform?", - "query_length": 43, - "query_type": "routing_query", - "output_response": "Routed to RAG for query processing", - "router.classification": "RAG", - "router.confidence_score": 1, - "router.rag_keyword_score": 3, - "router.web_keyword_score": 0, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "c28bd31c-d200-4467-91fd-a3b80187c885", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:48:17.977000000", - "trace_end_time": "2025-10-16 08:48:56.570000000", - "trace_duration_ms": 38592.996, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:48:17.977000000", - "updated_at": "2025-10-16 08:48:17.977000000" - }, - { - "span_id": "3ccebe1c-1f2a-433b-a36f-453a0b803511", - "trace_id": "c28bd31c-d200-4467-91fd-a3b80187c885", - "parent_span_id": "db385f4d-beef-465b-9831-a65080537519", - "name": "agent.llm-rag", - "start_time": "2025-10-16 08:48:17.978000000", - "end_time": "2025-10-16 08:48:56.570000000", - "duration_ms": 38592.416999999994, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "rag_agent", - "agent.operation": "llm-rag", - "agent.capabilities": [ - "document_retrieval", - "context_generation", - "response_generation" - ], - "agent.id": "noveum_rag_agent", - "input_query": "What APIs are available in Noveum platform?", - "query_length": 43, - "query_type": "rag_query", - "output_response": "RAG Answer: The provided context does not specify the APIs available in the Noveum platform. For detailed information about the APIs, I recommend checking the official Noveum documentation or the SDK Integration ...", - "rag.context_length": 2284, - "rag.sources_count": 5, - "rag.answer_length": 270, - "rag.mode": "retrieval_augmented_generation" - }, - "events": [], - "links": [], - "trace_trace_id": "c28bd31c-d200-4467-91fd-a3b80187c885", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:48:17.977000000", - "trace_end_time": "2025-10-16 08:48:56.570000000", - "trace_duration_ms": 38592.996, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:48:17.977000000", - "updated_at": "2025-10-16 08:48:17.977000000" - }, - { - "span_id": "6d1bb8e9-315d-433a-8df5-7f5fb2eeee16", - "trace_id": "c28bd31c-d200-4467-91fd-a3b80187c885", - "parent_span_id": "db385f4d-beef-465b-9831-a65080537519", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-16 08:48:17.978000000", - "end_time": "2025-10-16 08:48:17.978000000", - "duration_ms": 0.043000000000000003, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What APIs are available in Noveum platform?", - "query_length": 43, - "query_type": "routing_evaluation_query", - "classification.mode": "RAG", - "classification.rag_keyword_score": 3, - "classification.web_keyword_score": 0, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "noveum_specific", - "query.keyword_density": 0.42857142857142855, - "query.domain_affinity": "noveum", - "routing.decision": "Routed to RAG based on analysis", - "routing.rationale": "RAG score: 3, Web score: 0, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "Web Search", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.057692307692307696, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: RAG (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "c28bd31c-d200-4467-91fd-a3b80187c885", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:48:17.977000000", - "trace_end_time": "2025-10-16 08:48:56.570000000", - "trace_duration_ms": 38592.996, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:48:17.977000000", - "updated_at": "2025-10-16 08:48:17.977000000" - }, - { - "span_id": "2f11df8a-8f45-425d-a0ce-1eec7887b508", - "trace_id": "c28bd31c-d200-4467-91fd-a3b80187c885", - "parent_span_id": "3ccebe1c-1f2a-433b-a36f-453a0b803511", - "name": "agent.llm_model_execution", - "start_time": "2025-10-16 08:48:18.392000000", - "end_time": "2025-10-16 08:48:55.545000000", - "duration_ms": 37152.496999999996, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What APIs are available in Noveum platform?", - "query_length": 43, - "query_type": "rag_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 37.15243363380432, - "model.latency_ms": 37152.43363380432, - "model.start_time": 1760604498.3926067, - "model.end_time": 1760604535.5450404, - "model.performance_tier": "slow", - "model.prompt_tokens": 705, - "model.completion_tokens": 67, - "model.total_tokens": 772, - "model.tokens_per_second": 20.779257897592238, - "model.estimated_cost": 0.02316, - "model.efficiency_score": 0.34974093264248707, - "model.response_length": 270, - "model.response_quality": "high", - "model.output_response": "Model Response: The provided context does not specify the APIs available in the Noveum platform. For detailed information about the APIs, I recommend checking the official Noveum documentation or the SDK Integration ...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "c28bd31c-d200-4467-91fd-a3b80187c885", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:48:17.977000000", - "trace_end_time": "2025-10-16 08:48:56.570000000", - "trace_duration_ms": 38592.996, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:48:17.977000000", - "updated_at": "2025-10-16 08:48:17.977000000" - }, - { - "span_id": "d13e608c-06e4-47c4-b602-c40ac9560be2", - "trace_id": "c28bd31c-d200-4467-91fd-a3b80187c885", - "parent_span_id": "3ccebe1c-1f2a-433b-a36f-453a0b803511", - "name": "agent.rag_evaluation_metrics", - "start_time": "2025-10-16 08:48:55.545000000", - "end_time": "2025-10-16 08:48:55.545000000", - "duration_ms": 0.031, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "rag_evaluation_metrics", - "agent.capabilities": [ - "retrieval_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What APIs are available in Noveum platform?", - "query_length": 43, - "query_type": "rag_evaluation_query", - "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/docs):\n- \ud83c\udf0d Universal: Works across frameworks, clouds, and deployment models\n\ud83e\udd1d Community & Support\n- \ud83d\udcac Discord Community: Join our Discord\n- \ud83d\udce7 Email Support: [email protected]\n- \ud83d\udc1b Bug Reports: GitHub Issues\n- \ud83d\udcd6 Knowledge Base: Help Center\nReady to get started? Head to ...", - "retrieval.context_length": 2284, - "retrieval.sources_count": 5, - "retrieval.context_quality": "high", - "retrieval.effectiveness": 1, - "retrieval.context_utilization": 2.284, - "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/docs):\n- \ud83c\udf0d Universal: Works across frameworks, clouds, and deployment models\n\ud83e\udd1d Community & Support\n- \ud83d\udcac Discord Community: Join our Discord\n- \ud83d\udce7 Email Support: [email protected]\n- \ud83d\udc1b Bug Reports: GitHub Issues\n- \ud83d\udcd6 Knowledge Base: Help Center\nReady to get started? Head to our SDK Integration Guide to begin tracing your AI applications in under 5 minutes!\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability ...\n\nSource 2 (https://noveum.ai/docs):\n- \ud83c\udf0d Universal: Works across frameworks, clouds, and deployment models\n\ud83e\udd1d Community & Support\n- \ud83d\udcac Discord Community: Join our Discord\n- \ud83d\udce7 Email Support: [email protected]\n- \ud83d\udc1b Bug Reports: GitHub Issues\n- \ud83d\udcd6 Knowledge Base: Help Center\nReady to get started? Head to our SDK Integration Guide to begin tracing your AI applications in under 5 minutes!\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability ...\n\nSource 3 (https://noveum.ai/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nSource 4 (https://noveum.ai/en/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nSource 5 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, vector searches, data retrieval, agent reasoning, and more. Without proper observability, debugging becomes a nightmare, optimization is guesswork, and understanding user interactions is nearly impossible.\nNoveum.ai solves this by providing comprehensive tracing and observability specifically designed for AI applications.\nKey benefits of using Noveum.ai:\n- Complete Visibility: Trace...\n\nUser Question: What APIs are available in Noveum platform?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", - "prompt.prompt_length": 2823, - "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/docs):\n- \ud83c\udf0d Universal: Works across frameworks, clouds, and deployment models\n\ud83e\udd1d Community & Support\n- \ud83d\udcac Discord Community: Join our Discord\n- \ud83d\udce7 Email Support: [email prot...", - "prompt.instruction_following": "rag_optimized", - "response.answer_length": 270, - "response.answer_completeness": "complete", - "response.response_quality": "high", - "response.source_citation": 5, - "response.context_utilization": 0.11821366024518389, - "output_response": "RAG Answer: The provided context does not specify the APIs available in the Noveum platform. For detailed information about the APIs, I recommend checking the official Noveum documentation or the SDK Integration ...", - "evaluation.retrieval_effectiveness": 1, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 5, - "evaluation.context_relevance": "high", - "evaluation.overall_quality": "high", - "evaluation.ready_for_production": true, - "rag.retrieval_strategy": "semantic_similarity", - "rag.vector_search_results": 5, - "rag.context_synthesis": "multi_source", - "rag.document_coverage": 1, - "rag.information_density": 0.11821366024518389 - }, - "events": [], - "links": [], - "trace_trace_id": "c28bd31c-d200-4467-91fd-a3b80187c885", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:48:17.977000000", - "trace_end_time": "2025-10-16 08:48:56.570000000", - "trace_duration_ms": 38592.996, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:48:17.977000000", - "updated_at": "2025-10-16 08:48:17.977000000" - }, - { - "span_id": "3f3babbd-abf5-4f5a-b31c-fa5dcec9c360", - "trace_id": "b90ef3fc-11ea-4230-a59b-e2f2db8f2de0", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-16 08:48:08.007000000", - "end_time": "2025-10-16 08:48:16.975000000", - "duration_ms": 8967.201, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "How do I set up observability with Noveum?", - "query.length": 42, - "processing.mode": "RAG", - "processing.time_seconds": 8.96716833114624, - "processing.response_length": 838, - "processing.sources_count": 5, - "output_response": "Final Answer: To set up observability with Noveum, follow these steps:\n\n1. **Sign Up**: Go to [noveum.ai](https://noveum.ai) and create an account.\n2. **Create a Project**: Once signed in, create a new project to o...", - "final_answer_mode": "RAG", - "query_processed.input_query": "How do I set up observability with Noveum?", - "query_processed.output_response": "Successfully processed query using RAG, generated 838 character response", - "query_processed.mode": "RAG", - "query_processed.processing_time": 8.96716833114624, - "query_processed.response_length": 838 - }, - "events": [], - "links": [], - "trace_trace_id": "b90ef3fc-11ea-4230-a59b-e2f2db8f2de0", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:48:08.007000000", - "trace_end_time": "2025-10-16 08:48:16.975000000", - "trace_duration_ms": 8967.264, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:48:08.007000000", - "updated_at": "2025-10-16 08:48:08.007000000" - }, - { - "span_id": "41f6453c-4be6-4b66-8f9b-8e795f7b41b0", - "trace_id": "b90ef3fc-11ea-4230-a59b-e2f2db8f2de0", - "parent_span_id": "b0a76345-49e5-4c1b-915f-cec90ea2b1f7", - "name": "agent.llm-rag", - "start_time": "2025-10-16 08:48:08.007000000", - "end_time": "2025-10-16 08:48:16.974000000", - "duration_ms": 8966.964, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "rag_agent", - "agent.operation": "llm-rag", - "agent.capabilities": [ - "document_retrieval", - "context_generation", - "response_generation" - ], - "agent.id": "noveum_rag_agent", - "input_query": "How do I set up observability with Noveum?", - "query_length": 42, - "query_type": "rag_query", - "output_response": "RAG Answer: To set up observability with Noveum, follow these steps:\n\n1. **Sign Up**: Go to [noveum.ai](https://noveum.ai) and create an account.\n2. **Create a Project**: Once signed in, create a new project to o...", - "rag.context_length": 2870, - "rag.sources_count": 6, - "rag.answer_length": 838, - "rag.mode": "retrieval_augmented_generation" - }, - "events": [], - "links": [], - "trace_trace_id": "b90ef3fc-11ea-4230-a59b-e2f2db8f2de0", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:48:08.007000000", - "trace_end_time": "2025-10-16 08:48:16.975000000", - "trace_duration_ms": 8967.264, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:48:08.007000000", - "updated_at": "2025-10-16 08:48:08.007000000" - }, - { - "span_id": "b0a76345-49e5-4c1b-915f-cec90ea2b1f7", - "trace_id": "b90ef3fc-11ea-4230-a59b-e2f2db8f2de0", - "parent_span_id": "3f3babbd-abf5-4f5a-b31c-fa5dcec9c360", - "name": "agent.query_routing", - "start_time": "2025-10-16 08:48:08.007000000", - "end_time": "2025-10-16 08:48:16.974000000", - "duration_ms": 8967.129, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "How do I set up observability with Noveum?", - "query_length": 42, - "query_type": "routing_query", - "output_response": "Routed to RAG for query processing", - "router.classification": "RAG", - "router.confidence_score": 1, - "router.rag_keyword_score": 2, - "router.web_keyword_score": 0, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "b90ef3fc-11ea-4230-a59b-e2f2db8f2de0", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:48:08.007000000", - "trace_end_time": "2025-10-16 08:48:16.975000000", - "trace_duration_ms": 8967.264, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:48:08.007000000", - "updated_at": "2025-10-16 08:48:08.007000000" - }, - { - "span_id": "fe3074a7-f4c0-442e-9a5b-e2cc11d93785", - "trace_id": "b90ef3fc-11ea-4230-a59b-e2f2db8f2de0", - "parent_span_id": "b0a76345-49e5-4c1b-915f-cec90ea2b1f7", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-16 08:48:08.007000000", - "end_time": "2025-10-16 08:48:08.007000000", - "duration_ms": 0.032, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: How do I set up observability with Noveum?", - "query_length": 42, - "query_type": "routing_evaluation_query", - "classification.mode": "RAG", - "classification.rag_keyword_score": 2, - "classification.web_keyword_score": 0, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "noveum_specific", - "query.keyword_density": 0.25, - "query.domain_affinity": "noveum", - "routing.decision": "Routed to RAG based on analysis", - "routing.rationale": "RAG score: 2, Web score: 0, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "Web Search", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.038461538461538464, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: RAG (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "b90ef3fc-11ea-4230-a59b-e2f2db8f2de0", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:48:08.007000000", - "trace_end_time": "2025-10-16 08:48:16.975000000", - "trace_duration_ms": 8967.264, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:48:08.007000000", - "updated_at": "2025-10-16 08:48:08.007000000" - }, - { - "span_id": "e16139ee-048e-4c2f-87f8-90442f41a0ae", - "trace_id": "b90ef3fc-11ea-4230-a59b-e2f2db8f2de0", - "parent_span_id": "41f6453c-4be6-4b66-8f9b-8e795f7b41b0", - "name": "agent.llm_model_execution", - "start_time": "2025-10-16 08:48:08.644000000", - "end_time": "2025-10-16 08:48:15.709000000", - "duration_ms": 7064.898, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: How do I set up observability with Noveum?", - "query_length": 42, - "query_type": "rag_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 7.0648510456085205, - "model.latency_ms": 7064.8510456085205, - "model.start_time": 1760604488.6448367, - "model.end_time": 1760604495.7096877, - "model.performance_tier": "slow", - "model.prompt_tokens": 852, - "model.completion_tokens": 209, - "model.total_tokens": 1061, - "model.tokens_per_second": 150.1800948315128, - "model.estimated_cost": 0.031830000000000004, - "model.efficiency_score": 0.7898209236569275, - "model.response_length": 838, - "model.response_quality": "high", - "model.output_response": "Model Response: To set up observability with Noveum, follow these steps:\n\n1. **Sign Up**: Go to [noveum.ai](https://noveum.ai) and create an account.\n2. **Create a Project**: Once signed in, create a new project to o...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "b90ef3fc-11ea-4230-a59b-e2f2db8f2de0", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:48:08.007000000", - "trace_end_time": "2025-10-16 08:48:16.975000000", - "trace_duration_ms": 8967.264, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:48:08.007000000", - "updated_at": "2025-10-16 08:48:08.007000000" - }, - { - "span_id": "59e85f8e-c19b-4794-9a69-7b5bc90208f0", - "trace_id": "b90ef3fc-11ea-4230-a59b-e2f2db8f2de0", - "parent_span_id": "41f6453c-4be6-4b66-8f9b-8e795f7b41b0", - "name": "agent.rag_evaluation_metrics", - "start_time": "2025-10-16 08:48:15.709000000", - "end_time": "2025-10-16 08:48:15.709000000", - "duration_ms": 0.039, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "rag_evaluation_metrics", - "agent.capabilities": [ - "retrieval_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: How do I set up observability with Noveum?", - "query_length": 42, - "query_type": "rag_evaluation_query", - "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\n- Production Ready: Built for scale with intelligent sampling and batching\n- Privacy Focused: You control what data is captured and how it's stored\n- Open Source SDKs: Transparent, auditable, and extensible\nGetting ...", - "retrieval.context_length": 2870, - "retrieval.sources_count": 6, - "retrieval.context_quality": "high", - "retrieval.effectiveness": 1.2, - "retrieval.context_utilization": 2.87, - "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\n- Production Ready: Built for scale with intelligent sampling and batching\n- Privacy Focused: You control what data is captured and how it's stored\n- Open Source SDKs: Transparent, auditable, and extensible\nGetting Started\nReady to add comprehensive observability to your AI applications?\n- Sign up at noveum.ai\n- Create a project and get your API key\n- Install the SDK for your preferred language\n- Add tracing to your AI workflows\n- Explore insights in the Noveum.ai dashboard\nCheck out our integra...\n\nSource 2 (https://noveum.ai/en/docs/getting-started/tracing-concepts):\n- Explore Framework Integrations - Framework-specific guidance\n- Learn Advanced Patterns - Custom instrumentation techniques\n- Master the Dashboard - Analyze your traces effectively\nRemember: Good observability is not about collecting all possible data, but about collecting the right data that helps you understand, debug, and optimize your AI applications.\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to O...\n\nSource 3 (https://noveum.ai/docs/getting-started/tracing-concepts):\n- Explore Framework Integrations - Framework-specific guidance\n- Learn Advanced Patterns - Custom instrumentation techniques\n- Master the Dashboard - Analyze your traces effectively\nRemember: Good observability is not about collecting all possible data, but about collecting the right data that helps you understand, debug, and optimize your AI applications.\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to O...\n\nSource 4 (https://noveum.ai/en/docs/getting-started/overview):\nNoveum.ai Overview\nComprehensive AI tracing and observability platform for LLM applications, RAG systems, and AI agents\nWelcome to Noveum.ai\u2014the comprehensive tracing and observability platform built specifically for AI applications. Whether you're building LLM-powered chatbots, RAG systems, multi-agent workflows, or any AI-driven application, Noveum provides the insights you need to understand, debug, and optimize your systems.\n\ud83c\udfaf Why AI Applications Need Specialized Observability\nTraditional mo...\n\nSource 5 (https://noveum.ai/docs/getting-started/overview):\nNoveum.ai Overview\nComprehensive AI tracing and observability platform for LLM applications, RAG systems, and AI agents\nWelcome to Noveum.ai\u2014the comprehensive tracing and observability platform built specifically for AI applications. Whether you're building LLM-powered chatbots, RAG systems, multi-agent workflows, or any AI-driven application, Noveum provides the insights you need to understand, debug, and optimize your systems.\n\ud83c\udfaf Why AI Applications Need Specialized Observability\nTraditional mo...\n\nUser Question: How do I set up observability with Noveum?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", - "prompt.prompt_length": 3408, - "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\n- Production Ready: Built for scale with intelligent sampling and batching\n- Privacy Focused: You control what data...", - "prompt.instruction_following": "rag_optimized", - "response.answer_length": 838, - "response.answer_completeness": "complete", - "response.response_quality": "high", - "response.source_citation": 6, - "response.context_utilization": 0.29198606271777006, - "output_response": "RAG Answer: To set up observability with Noveum, follow these steps:\n\n1. **Sign Up**: Go to [noveum.ai](https://noveum.ai) and create an account.\n2. **Create a Project**: Once signed in, create a new project to o...", - "evaluation.retrieval_effectiveness": 1.2, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 6, - "evaluation.context_relevance": "high", - "evaluation.overall_quality": "high", - "evaluation.ready_for_production": true, - "rag.retrieval_strategy": "semantic_similarity", - "rag.vector_search_results": 6, - "rag.context_synthesis": "multi_source", - "rag.document_coverage": 1.2, - "rag.information_density": 0.29198606271777006 - }, - "events": [], - "links": [], - "trace_trace_id": "b90ef3fc-11ea-4230-a59b-e2f2db8f2de0", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:48:08.007000000", - "trace_end_time": "2025-10-16 08:48:16.975000000", - "trace_duration_ms": 8967.264, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:48:08.007000000", - "updated_at": "2025-10-16 08:48:08.007000000" - }, - { - "span_id": "f049ed2b-ff2f-4092-9974-acc66f81bd8d", - "trace_id": "b8abebac-a7dc-494c-a1eb-6470181f32d2", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-16 08:47:57.667000000", - "end_time": "2025-10-16 08:48:07.005000000", - "duration_ms": 9337.298, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What features does Noveum Trace offer?", - "query.length": 38, - "processing.mode": "RAG", - "processing.time_seconds": 9.33726453781128, - "processing.response_length": 1018, - "processing.sources_count": 5, - "output_response": "Final Answer: Noveum Trace offers several key features through its Python SDK (`noveum-trace`) and TypeScript SDK (`@noveum/trace`):\n\n1. **Decorator-Based Tracing**: It provides a decorator-based approach for traci...", - "final_answer_mode": "RAG", - "query_processed.input_query": "What features does Noveum Trace offer?", - "query_processed.output_response": "Successfully processed query using RAG, generated 1018 character response", - "query_processed.mode": "RAG", - "query_processed.processing_time": 9.33726453781128, - "query_processed.response_length": 1018 - }, - "events": [], - "links": [], - "trace_trace_id": "b8abebac-a7dc-494c-a1eb-6470181f32d2", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:47:57.667000000", - "trace_end_time": "2025-10-16 08:48:07.005000000", - "trace_duration_ms": 9337.405, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:47:57.667000000", - "updated_at": "2025-10-16 08:47:57.667000000" - }, - { - "span_id": "211f1fe7-4cfb-4c1c-9c72-62ae2825dda3", - "trace_id": "b8abebac-a7dc-494c-a1eb-6470181f32d2", - "parent_span_id": "f049ed2b-ff2f-4092-9974-acc66f81bd8d", - "name": "agent.query_routing", - "start_time": "2025-10-16 08:47:57.668000000", - "end_time": "2025-10-16 08:48:07.005000000", - "duration_ms": 9337.223, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What features does Noveum Trace offer?", - "query_length": 38, - "query_type": "routing_query", - "output_response": "Routed to RAG for query processing", - "router.classification": "RAG", - "router.confidence_score": 1, - "router.rag_keyword_score": 3, - "router.web_keyword_score": 0, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "b8abebac-a7dc-494c-a1eb-6470181f32d2", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:47:57.667000000", - "trace_end_time": "2025-10-16 08:48:07.005000000", - "trace_duration_ms": 9337.405, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:47:57.667000000", - "updated_at": "2025-10-16 08:47:57.667000000" - }, - { - "span_id": "976c0c4b-efc1-4b7d-bfad-350c7fd0f71e", - "trace_id": "b8abebac-a7dc-494c-a1eb-6470181f32d2", - "parent_span_id": "211f1fe7-4cfb-4c1c-9c72-62ae2825dda3", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-16 08:47:57.668000000", - "end_time": "2025-10-16 08:47:57.668000000", - "duration_ms": 0.028, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What features does Noveum Trace offer?", - "query_length": 38, - "query_type": "routing_evaluation_query", - "classification.mode": "RAG", - "classification.rag_keyword_score": 3, - "classification.web_keyword_score": 0, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "noveum_specific", - "query.keyword_density": 0.5, - "query.domain_affinity": "noveum", - "routing.decision": "Routed to RAG based on analysis", - "routing.rationale": "RAG score: 3, Web score: 0, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "Web Search", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.057692307692307696, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: RAG (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "b8abebac-a7dc-494c-a1eb-6470181f32d2", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:47:57.667000000", - "trace_end_time": "2025-10-16 08:48:07.005000000", - "trace_duration_ms": 9337.405, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:47:57.667000000", - "updated_at": "2025-10-16 08:47:57.667000000" - }, - { - "span_id": "d2afaf74-f63b-4141-9fa3-662162f51cc0", - "trace_id": "b8abebac-a7dc-494c-a1eb-6470181f32d2", - "parent_span_id": "211f1fe7-4cfb-4c1c-9c72-62ae2825dda3", - "name": "agent.llm-rag", - "start_time": "2025-10-16 08:47:57.668000000", - "end_time": "2025-10-16 08:48:07.005000000", - "duration_ms": 9337.056, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "rag_agent", - "agent.operation": "llm-rag", - "agent.capabilities": [ - "document_retrieval", - "context_generation", - "response_generation" - ], - "agent.id": "noveum_rag_agent", - "input_query": "What features does Noveum Trace offer?", - "query_length": 38, - "query_type": "rag_query", - "output_response": "RAG Answer: Noveum Trace offers several key features through its Python SDK (`noveum-trace`) and TypeScript SDK (`@noveum/trace`):\n\n1. **Decorator-Based Tracing**: It provides a decorator-based approach for traci...", - "rag.context_length": 2787, - "rag.sources_count": 5, - "rag.answer_length": 1018, - "rag.mode": "retrieval_augmented_generation" - }, - "events": [], - "links": [], - "trace_trace_id": "b8abebac-a7dc-494c-a1eb-6470181f32d2", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:47:57.667000000", - "trace_end_time": "2025-10-16 08:48:07.005000000", - "trace_duration_ms": 9337.405, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:47:57.667000000", - "updated_at": "2025-10-16 08:47:57.667000000" - }, - { - "span_id": "09811a73-405d-4c0d-ba5d-0472b3708062", - "trace_id": "b8abebac-a7dc-494c-a1eb-6470181f32d2", - "parent_span_id": "d2afaf74-f63b-4141-9fa3-662162f51cc0", - "name": "agent.llm_model_execution", - "start_time": "2025-10-16 08:47:58.659000000", - "end_time": "2025-10-16 08:48:05.878000000", - "duration_ms": 7218.618, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What features does Noveum Trace offer?", - "query_length": 38, - "query_type": "rag_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 7.218573331832886, - "model.latency_ms": 7218.573331832886, - "model.start_time": 1760604478.659586, - "model.end_time": 1760604485.8781593, - "model.performance_tier": "slow", - "model.prompt_tokens": 830, - "model.completion_tokens": 254, - "model.total_tokens": 1084, - "model.tokens_per_second": 150.16817730724068, - "model.estimated_cost": 0.03252, - "model.efficiency_score": 0.9391143911439115, - "model.response_length": 1018, - "model.response_quality": "high", - "model.output_response": "Model Response: Noveum Trace offers several key features through its Python SDK (`noveum-trace`) and TypeScript SDK (`@noveum/trace`):\n\n1. **Decorator-Based Tracing**: It provides a decorator-based approach for traci...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "b8abebac-a7dc-494c-a1eb-6470181f32d2", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:47:57.667000000", - "trace_end_time": "2025-10-16 08:48:07.005000000", - "trace_duration_ms": 9337.405, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:47:57.667000000", - "updated_at": "2025-10-16 08:47:57.667000000" - }, - { - "span_id": "015688fe-d287-48e8-ac59-63781ffd6f89", - "trace_id": "b8abebac-a7dc-494c-a1eb-6470181f32d2", - "parent_span_id": "d2afaf74-f63b-4141-9fa3-662162f51cc0", - "name": "agent.rag_evaluation_metrics", - "start_time": "2025-10-16 08:48:05.878000000", - "end_time": "2025-10-16 08:48:05.878000000", - "duration_ms": 0.024, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "rag_evaluation_metrics", - "agent.capabilities": [ - "retrieval_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What features does Noveum Trace offer?", - "query_length": 38, - "query_type": "rag_evaluation_query", - "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\n\ud83d\udcd8 TypeScript SDK (\n@noveum/trace\n)- Framework ...", - "retrieval.context_length": 2787, - "retrieval.sources_count": 5, - "retrieval.context_quality": "high", - "retrieval.effectiveness": 1, - "retrieval.context_utilization": 2.787, - "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\n\ud83d\udcd8 TypeScript SDK (\n@noveum/trace\n)- Framework integrations for Next.js, Express.js, and Hono\n- TypeScript-first design with full type safety\n- Universal compatibility (Node.js, Edge Runtime, browsers)\n-\n\ud83d\udcca Noveum Platform\n- Real-time dashboard for analyzing traces and performance\n- Ad...\n\nSource 2 (https://noveum.ai/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\n\ud83d\udcd8 TypeScript SDK (\n@noveum/trace\n)- Framework integrations for Next.js, Express.js, and Hono\n- TypeScript-first design with full type safety\n- Universal compatibility (Node.js, Edge Runtime, browsers)\n-\n\ud83d\udcca Noveum Platform\n- Real-time dashboard for analyzing traces and performance\n- Ad...\n\nSource 3 (https://noveum.ai/en/docs/getting-started/sdk-integration):\n- \ud83d\udc65 Collaborate with your team on insights\n\ud83d\udd12 Security & Privacy\n- \ud83d\udd10 Encryption: All data encrypted in transit and at rest\n- \ud83c\udf9b\ufe0f Configurable Capture: Control what data is collected\n- \ud83c\udfe0 Data Residency: Choose your data storage region\n- \u23f0 Retention Control: Set custom data retention policies\nNext Steps\n- Tracing Concepts - Learn about traces, spans, and observability best practices\n- Framework Integrations - Deep dive into specific framework setups\n- Multi-Agent Tracing - Observe complex agent work...\n\nSource 4 (https://noveum.ai/docs/getting-started/sdk-integration):\n- \ud83d\udc65 Collaborate with your team on insights\n\ud83d\udd12 Security & Privacy\n- \ud83d\udd10 Encryption: All data encrypted in transit and at rest\n- \ud83c\udf9b\ufe0f Configurable Capture: Control what data is collected\n- \ud83c\udfe0 Data Residency: Choose your data storage region\n- \u23f0 Retention Control: Set custom data retention policies\nNext Steps\n- Tracing Concepts - Learn about traces, spans, and observability best practices\n- Framework Integrations - Deep dive into specific framework setups\n- Multi-Agent Tracing - Observe complex agent work...\n\nSource 5 (https://noveum.ai/docs/platform/dashboard):\n- Bulk Export: Download multiple traces simultaneously\n- Comparative Analysis: Compare performance across multiple traces\nReal-Time Updates\n- Live Refresh: Automatic updates as new traces arrive\n- Connection Monitoring: Real-time status of your trace ingestion pipeline\n- Performance Indicators: Live metrics for system health monitoring\nReady to dive deeper? Explore Projects & Environments to organize your AI applications, or check out Team Collaboration to share insights with your team.\nGet Earl...\n\nUser Question: What features does Noveum Trace offer?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", - "prompt.prompt_length": 3321, - "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frame...", - "prompt.instruction_following": "rag_optimized", - "response.answer_length": 1018, - "response.answer_completeness": "complete", - "response.response_quality": "high", - "response.source_citation": 5, - "response.context_utilization": 0.36526731252242556, - "output_response": "RAG Answer: Noveum Trace offers several key features through its Python SDK (`noveum-trace`) and TypeScript SDK (`@noveum/trace`):\n\n1. **Decorator-Based Tracing**: It provides a decorator-based approach for traci...", - "evaluation.retrieval_effectiveness": 1, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 5, - "evaluation.context_relevance": "high", - "evaluation.overall_quality": "high", - "evaluation.ready_for_production": true, - "rag.retrieval_strategy": "semantic_similarity", - "rag.vector_search_results": 5, - "rag.context_synthesis": "multi_source", - "rag.document_coverage": 1, - "rag.information_density": 0.36526731252242556 - }, - "events": [], - "links": [], - "trace_trace_id": "b8abebac-a7dc-494c-a1eb-6470181f32d2", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:47:57.667000000", - "trace_end_time": "2025-10-16 08:48:07.005000000", - "trace_duration_ms": 9337.405, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:47:57.667000000", - "updated_at": "2025-10-16 08:47:57.667000000" - }, - { - "span_id": "46ee4d00-f914-41d8-9f91-c2db85f1c613", - "trace_id": "be9ccbaf-ccb4-4ebd-a380-4247f4d01868", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-16 08:47:52.037000000", - "end_time": "2025-10-16 08:47:56.662000000", - "duration_ms": 4625.395, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What are Noveum's pricing plans?", - "query.length": 32, - "processing.mode": "RAG", - "processing.time_seconds": 4.6253437995910645, - "processing.response_length": 471, - "processing.sources_count": 5, - "output_response": "Final Answer: The provided context does not contain specific information about Noveum's pricing plans. It mentions that all users will have access to the Observability suite for free, and early users will receive f...", - "final_answer_mode": "RAG", - "query_processed.input_query": "What are Noveum's pricing plans?", - "query_processed.output_response": "Successfully processed query using RAG, generated 471 character response", - "query_processed.mode": "RAG", - "query_processed.processing_time": 4.6253437995910645, - "query_processed.response_length": 471 - }, - "events": [], - "links": [], - "trace_trace_id": "be9ccbaf-ccb4-4ebd-a380-4247f4d01868", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:47:52.037000000", - "trace_end_time": "2025-10-16 08:47:56.662000000", - "trace_duration_ms": 4625.492, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:47:52.037000000", - "updated_at": "2025-10-16 08:47:52.037000000" - }, - { - "span_id": "6d15369a-ae35-45c7-84dd-bdf484f73f2d", - "trace_id": "be9ccbaf-ccb4-4ebd-a380-4247f4d01868", - "parent_span_id": "46ee4d00-f914-41d8-9f91-c2db85f1c613", - "name": "agent.query_routing", - "start_time": "2025-10-16 08:47:52.037000000", - "end_time": "2025-10-16 08:47:56.662000000", - "duration_ms": 4625.286, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What are Noveum's pricing plans?", - "query_length": 32, - "query_type": "routing_query", - "output_response": "Routed to RAG for query processing", - "router.classification": "RAG", - "router.confidence_score": 1, - "router.rag_keyword_score": 3, - "router.web_keyword_score": 0, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "be9ccbaf-ccb4-4ebd-a380-4247f4d01868", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:47:52.037000000", - "trace_end_time": "2025-10-16 08:47:56.662000000", - "trace_duration_ms": 4625.492, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:47:52.037000000", - "updated_at": "2025-10-16 08:47:52.037000000" - }, - { - "span_id": "bf201676-a3a9-433a-a8c6-d394b7aead3a", - "trace_id": "be9ccbaf-ccb4-4ebd-a380-4247f4d01868", - "parent_span_id": "6d15369a-ae35-45c7-84dd-bdf484f73f2d", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-16 08:47:52.037000000", - "end_time": "2025-10-16 08:47:52.037000000", - "duration_ms": 0.048, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What are Noveum's pricing plans?", - "query_length": 32, - "query_type": "routing_evaluation_query", - "classification.mode": "RAG", - "classification.rag_keyword_score": 3, - "classification.web_keyword_score": 0, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "noveum_specific", - "query.keyword_density": 0.6, - "query.domain_affinity": "noveum", - "routing.decision": "Routed to RAG based on analysis", - "routing.rationale": "RAG score: 3, Web score: 0, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "Web Search", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.057692307692307696, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: RAG (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "be9ccbaf-ccb4-4ebd-a380-4247f4d01868", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:47:52.037000000", - "trace_end_time": "2025-10-16 08:47:56.662000000", - "trace_duration_ms": 4625.492, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:47:52.037000000", - "updated_at": "2025-10-16 08:47:52.037000000" - }, - { - "span_id": "f8346f60-131b-4007-ba3e-7b0ff10cdfde", - "trace_id": "be9ccbaf-ccb4-4ebd-a380-4247f4d01868", - "parent_span_id": "6d15369a-ae35-45c7-84dd-bdf484f73f2d", - "name": "agent.llm-rag", - "start_time": "2025-10-16 08:47:52.037000000", - "end_time": "2025-10-16 08:47:56.662000000", - "duration_ms": 4624.998, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "rag_agent", - "agent.operation": "llm-rag", - "agent.capabilities": [ - "document_retrieval", - "context_generation", - "response_generation" - ], - "agent.id": "noveum_rag_agent", - "input_query": "What are Noveum's pricing plans?", - "query_length": 32, - "query_type": "rag_query", - "output_response": "RAG Answer: The provided context does not contain specific information about Noveum's pricing plans. It mentions that all users will have access to the Observability suite for free, and early users will receive f...", - "rag.context_length": 2105, - "rag.sources_count": 5, - "rag.answer_length": 471, - "rag.mode": "retrieval_augmented_generation" - }, - "events": [], - "links": [], - "trace_trace_id": "be9ccbaf-ccb4-4ebd-a380-4247f4d01868", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:47:52.037000000", - "trace_end_time": "2025-10-16 08:47:56.662000000", - "trace_duration_ms": 4625.492, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:47:52.037000000", - "updated_at": "2025-10-16 08:47:52.037000000" - }, - { - "span_id": "0186c28c-7596-4279-a4eb-7733cd665609", - "trace_id": "be9ccbaf-ccb4-4ebd-a380-4247f4d01868", - "parent_span_id": "f8346f60-131b-4007-ba3e-7b0ff10cdfde", - "name": "agent.llm_model_execution", - "start_time": "2025-10-16 08:47:52.579000000", - "end_time": "2025-10-16 08:47:55.842000000", - "duration_ms": 3262.5829999999996, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What are Noveum's pricing plans?", - "query_length": 32, - "query_type": "rag_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 3.2625203132629395, - "model.latency_ms": 3262.5203132629395, - "model.start_time": 1760604472.580008, - "model.end_time": 1760604475.8425283, - "model.performance_tier": "medium", - "model.prompt_tokens": 658, - "model.completion_tokens": 117, - "model.total_tokens": 775, - "model.tokens_per_second": 237.54641368804243, - "model.estimated_cost": 0.02325, - "model.efficiency_score": 0.607741935483871, - "model.response_length": 471, - "model.response_quality": "high", - "model.output_response": "Model Response: The provided context does not contain specific information about Noveum's pricing plans. It mentions that all users will have access to the Observability suite for free, and early users will receive f...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "be9ccbaf-ccb4-4ebd-a380-4247f4d01868", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:47:52.037000000", - "trace_end_time": "2025-10-16 08:47:56.662000000", - "trace_duration_ms": 4625.492, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:47:52.037000000", - "updated_at": "2025-10-16 08:47:52.037000000" - }, - { - "span_id": "3bdf8d5b-c10d-44f3-8514-8facc7f63db8", - "trace_id": "be9ccbaf-ccb4-4ebd-a380-4247f4d01868", - "parent_span_id": "f8346f60-131b-4007-ba3e-7b0ff10cdfde", - "name": "agent.rag_evaluation_metrics", - "start_time": "2025-10-16 08:47:55.842000000", - "end_time": "2025-10-16 08:47:55.842000000", - "duration_ms": 0.032, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "rag_evaluation_metrics", - "agent.capabilities": [ - "retrieval_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What are Noveum's pricing plans?", - "query_length": 32, - "query_type": "rag_evaluation_query", - "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....", - "retrieval.context_length": 2105, - "retrieval.sources_count": 5, - "retrieval.context_quality": "high", - "retrieval.effectiveness": 1, - "retrieval.context_utilization": 2.105, - "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nSource 2 (https://noveum.ai/en/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nSource 3 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nThe AI landscape is complex and moving fast. With Noveum.ai, you'll have the insights you need to build, debug, and optimize AI applications with confidence.\nReady to see what's really happening inside your AI applications? Try Noveum.ai today and transform how you build and monitor AI systems.\nLet's build more reliable, observable AI\u2014together.\nGet Early Access to Noveum.ai Platform\nJoin the select group of AI teams optimizing their models with our data-driven platform. We're onboarding users in...\n\nSource 4 (https://noveum.ai/en/blog/from-logs-to-intelligent-choices-inside-noveum-ais-evaluation-process):\nGet Early Access to Noveum.ai Platform\nJoin the select group of AI teams optimizing their models with our data-driven platform. We're onboarding users in limited batches to ensure a premium experience....\n\nSource 5 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, vector searches, data retrieval, agent reasoning, and more. Without proper observability, debugging becomes a nightmare, optimization is guesswork, and understanding user interactions is nearly impossible.\nNoveum.ai solves this by providing comprehensive tracing and observability specifically designed for AI applications.\nKey benefits of using Noveum.ai:\n- Complete Visibility: Trace...\n\nUser Question: What are Noveum's pricing plans?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", - "prompt.prompt_length": 2633, - "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to ...", - "prompt.instruction_following": "rag_optimized", - "response.answer_length": 471, - "response.answer_completeness": "complete", - "response.response_quality": "high", - "response.source_citation": 5, - "response.context_utilization": 0.22375296912114015, - "output_response": "RAG Answer: The provided context does not contain specific information about Noveum's pricing plans. It mentions that all users will have access to the Observability suite for free, and early users will receive f...", - "evaluation.retrieval_effectiveness": 1, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 5, - "evaluation.context_relevance": "high", - "evaluation.overall_quality": "high", - "evaluation.ready_for_production": true, - "rag.retrieval_strategy": "semantic_similarity", - "rag.vector_search_results": 5, - "rag.context_synthesis": "multi_source", - "rag.document_coverage": 1, - "rag.information_density": 0.22375296912114015 - }, - "events": [], - "links": [], - "trace_trace_id": "be9ccbaf-ccb4-4ebd-a380-4247f4d01868", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:47:52.037000000", - "trace_end_time": "2025-10-16 08:47:56.662000000", - "trace_duration_ms": 4625.492, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:47:52.037000000", - "updated_at": "2025-10-16 08:47:52.037000000" - }, - { - "span_id": "770ce93d-07ae-4b9f-ae15-c6e450634b9d", - "trace_id": "8d33a529-d9b1-4208-b528-762222d5ef8c", - "parent_span_id": "9977d5d6-a7ff-469e-bec7-5e4dfc38c149", - "name": "agent.llm-rag", - "start_time": "2025-10-16 08:47:42.103000000", - "end_time": "2025-10-16 08:47:51.032000000", - "duration_ms": 8928.644999999999, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "rag_agent", - "agent.operation": "llm-rag", - "agent.capabilities": [ - "document_retrieval", - "context_generation", - "response_generation" - ], - "agent.id": "noveum_rag_agent", - "input_query": "How do I integrate Noveum Trace in my application?", - "query_length": 50, - "query_type": "rag_query", - "output_response": "RAG Answer: To integrate Noveum Trace into your application, you can use either the Python SDK (`noveum-trace`) or the TypeScript SDK (`@noveum/trace`). Here\u2019s a quick start guide based on the documentation:\n\n1. ...", - "rag.context_length": 2803, - "rag.sources_count": 5, - "rag.answer_length": 1134, - "rag.mode": "retrieval_augmented_generation" - }, - "events": [], - "links": [], - "trace_trace_id": "8d33a529-d9b1-4208-b528-762222d5ef8c", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:47:42.103000000", - "trace_end_time": "2025-10-16 08:47:51.032000000", - "trace_duration_ms": 8928.948, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:47:42.103000000", - "updated_at": "2025-10-16 08:47:42.103000000" - }, - { - "span_id": "9977d5d6-a7ff-469e-bec7-5e4dfc38c149", - "trace_id": "8d33a529-d9b1-4208-b528-762222d5ef8c", - "parent_span_id": "f3a8fa53-5641-4a23-b4d3-762577a89151", - "name": "agent.query_routing", - "start_time": "2025-10-16 08:47:42.103000000", - "end_time": "2025-10-16 08:47:51.032000000", - "duration_ms": 8928.793000000001, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "How do I integrate Noveum Trace in my application?", - "query_length": 50, - "query_type": "routing_query", - "output_response": "Routed to RAG for query processing", - "router.classification": "RAG", - "router.confidence_score": 1, - "router.rag_keyword_score": 2, - "router.web_keyword_score": 0, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "8d33a529-d9b1-4208-b528-762222d5ef8c", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:47:42.103000000", - "trace_end_time": "2025-10-16 08:47:51.032000000", - "trace_duration_ms": 8928.948, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:47:42.103000000", - "updated_at": "2025-10-16 08:47:42.103000000" - }, - { - "span_id": "c678e222-ef74-48ee-8628-16461ab0f48f", - "trace_id": "8d33a529-d9b1-4208-b528-762222d5ef8c", - "parent_span_id": "9977d5d6-a7ff-469e-bec7-5e4dfc38c149", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-16 08:47:42.103000000", - "end_time": "2025-10-16 08:47:42.103000000", - "duration_ms": 0.027, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: How do I integrate Noveum Trace in my application?", - "query_length": 50, - "query_type": "routing_evaluation_query", - "classification.mode": "RAG", - "classification.rag_keyword_score": 2, - "classification.web_keyword_score": 0, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "noveum_specific", - "query.keyword_density": 0.2222222222222222, - "query.domain_affinity": "noveum", - "routing.decision": "Routed to RAG based on analysis", - "routing.rationale": "RAG score: 2, Web score: 0, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "Web Search", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.038461538461538464, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: RAG (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "8d33a529-d9b1-4208-b528-762222d5ef8c", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:47:42.103000000", - "trace_end_time": "2025-10-16 08:47:51.032000000", - "trace_duration_ms": 8928.948, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:47:42.103000000", - "updated_at": "2025-10-16 08:47:42.103000000" - }, - { - "span_id": "f3a8fa53-5641-4a23-b4d3-762577a89151", - "trace_id": "8d33a529-d9b1-4208-b528-762222d5ef8c", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-16 08:47:42.103000000", - "end_time": "2025-10-16 08:47:51.032000000", - "duration_ms": 8928.868, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "How do I integrate Noveum Trace in my application?", - "query.length": 50, - "processing.mode": "RAG", - "processing.time_seconds": 8.928829431533813, - "processing.response_length": 1134, - "processing.sources_count": 5, - "output_response": "Final Answer: To integrate Noveum Trace into your application, you can use either the Python SDK (`noveum-trace`) or the TypeScript SDK (`@noveum/trace`). Here\u2019s a quick start guide based on the documentation:\n\n1. ...", - "final_answer_mode": "RAG", - "query_processed.input_query": "How do I integrate Noveum Trace in my application?", - "query_processed.output_response": "Successfully processed query using RAG, generated 1134 character response", - "query_processed.mode": "RAG", - "query_processed.processing_time": 8.928829431533813, - "query_processed.response_length": 1134 - }, - "events": [], - "links": [], - "trace_trace_id": "8d33a529-d9b1-4208-b528-762222d5ef8c", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:47:42.103000000", - "trace_end_time": "2025-10-16 08:47:51.032000000", - "trace_duration_ms": 8928.948, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:47:42.103000000", - "updated_at": "2025-10-16 08:47:42.103000000" - }, - { - "span_id": "f459b71f-1d43-41b0-b9bf-d94a908b7afa", - "trace_id": "8d33a529-d9b1-4208-b528-762222d5ef8c", - "parent_span_id": "770ce93d-07ae-4b9f-ae15-c6e450634b9d", - "name": "agent.llm_model_execution", - "start_time": "2025-10-16 08:47:43.044000000", - "end_time": "2025-10-16 08:47:50.008000000", - "duration_ms": 6963.959, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: How do I integrate Noveum Trace in my application?", - "query_length": 50, - "query_type": "rag_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 6.9639081954956055, - "model.latency_ms": 6963.9081954956055, - "model.start_time": 1760604463.044064, - "model.end_time": 1760604470.0079722, - "model.performance_tier": "slow", - "model.prompt_tokens": 837, - "model.completion_tokens": 283, - "model.total_tokens": 1120, - "model.tokens_per_second": 160.8292310235276, - "model.estimated_cost": 0.0336, - "model.efficiency_score": 1.0125, - "model.response_length": 1134, - "model.response_quality": "high", - "model.output_response": "Model Response: To integrate Noveum Trace into your application, you can use either the Python SDK (`noveum-trace`) or the TypeScript SDK (`@noveum/trace`). Here\u2019s a quick start guide based on the documentation:\n\n1. ...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "8d33a529-d9b1-4208-b528-762222d5ef8c", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:47:42.103000000", - "trace_end_time": "2025-10-16 08:47:51.032000000", - "trace_duration_ms": 8928.948, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:47:42.103000000", - "updated_at": "2025-10-16 08:47:42.103000000" - }, - { - "span_id": "b8627171-b081-4b08-b900-c24375b73c3f", - "trace_id": "8d33a529-d9b1-4208-b528-762222d5ef8c", - "parent_span_id": "770ce93d-07ae-4b9f-ae15-c6e450634b9d", - "name": "agent.rag_evaluation_metrics", - "start_time": "2025-10-16 08:47:50.008000000", - "end_time": "2025-10-16 08:47:50.008000000", - "duration_ms": 0.041, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "rag_evaluation_metrics", - "agent.capabilities": [ - "retrieval_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: How do I integrate Noveum Trace in my application?", - "query_length": 50, - "query_type": "rag_evaluation_query", - "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\n\ud83d\udcd8 TypeScript SDK (\n@noveum/trace\n)- Framework ...", - "retrieval.context_length": 2803, - "retrieval.sources_count": 5, - "retrieval.context_quality": "high", - "retrieval.effectiveness": 1, - "retrieval.context_utilization": 2.803, - "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\n\ud83d\udcd8 TypeScript SDK (\n@noveum/trace\n)- Framework integrations for Next.js, Express.js, and Hono\n- TypeScript-first design with full type safety\n- Universal compatibility (Node.js, Edge Runtime, browsers)\n-\n\ud83d\udcca Noveum Platform\n- Real-time dashboard for analyzing traces and performance\n- Ad...\n\nSource 2 (https://noveum.ai/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\n\ud83d\udcd8 TypeScript SDK (\n@noveum/trace\n)- Framework integrations for Next.js, Express.js, and Hono\n- TypeScript-first design with full type safety\n- Universal compatibility (Node.js, Edge Runtime, browsers)\n-\n\ud83d\udcca Noveum Platform\n- Real-time dashboard for analyzing traces and performance\n- Ad...\n\nSource 3 (https://noveum.ai/docs/getting-started/sdk-integration):\nSDK Integration Guide\nIntegrate Noveum.ai tracing into your AI applications with Python or TypeScript SDKs\nThe Noveum.ai SDKs provide comprehensive tracing and observability for your AI applications with minimal code changes. Whether you're building LLM applications, RAG systems, or multi-agent workflows, our SDKs automatically capture essential metrics and traces.\n\ud83d\ude80 Quick Start\n1. Create Your Account & Get API Key\n- Sign up at noveum.ai\n- Create a project in your dashboard\n- Generate an API key...\n\nSource 4 (https://noveum.ai/en/docs/getting-started/sdk-integration):\nSDK Integration Guide\nIntegrate Noveum.ai tracing into your AI applications with Python or TypeScript SDKs\nThe Noveum.ai SDKs provide comprehensive tracing and observability for your AI applications with minimal code changes. Whether you're building LLM applications, RAG systems, or multi-agent workflows, our SDKs automatically capture essential metrics and traces.\n\ud83d\ude80 Quick Start\n1. Create Your Account & Get API Key\n- Sign up at noveum.ai\n- Create a project in your dashboard\n- Generate an API key...\n\nSource 5 (https://noveum.ai/en/docs/getting-started/sdk-integration):\n\ud83d\udd27 Framework Integrations\nNext.js Integration\nExpress.js Integration\nFastAPI Integration (Python)\n\ud83d\udcca Advanced Features\nCustom Attributes & Events\nSampling Configuration\n\ud83d\udd17 What's Captured Automatically\n- \ud83d\udcca Performance Metrics: Latency, throughput, error rates\n- \ud83d\udcb0 Cost Tracking: Token usage, API costs across providers\n- \ud83d\udd0d Request/Response: Configurable capture of inputs/outputs\n- \ud83c\udff7\ufe0f Metadata: Model names, parameters, user context\n- \ud83c\udf0a Context Flow: Trace relationships across services\n- \ud83d\udc1b Error Detail...\n\nUser Question: How do I integrate Noveum Trace in my application?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", - "prompt.prompt_length": 3349, - "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frame...", - "prompt.instruction_following": "rag_optimized", - "response.answer_length": 1134, - "response.answer_completeness": "complete", - "response.response_quality": "high", - "response.source_citation": 5, - "response.context_utilization": 0.4045665358544417, - "output_response": "RAG Answer: To integrate Noveum Trace into your application, you can use either the Python SDK (`noveum-trace`) or the TypeScript SDK (`@noveum/trace`). Here\u2019s a quick start guide based on the documentation:\n\n1. ...", - "evaluation.retrieval_effectiveness": 1, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 5, - "evaluation.context_relevance": "high", - "evaluation.overall_quality": "high", - "evaluation.ready_for_production": true, - "rag.retrieval_strategy": "semantic_similarity", - "rag.vector_search_results": 5, - "rag.context_synthesis": "multi_source", - "rag.document_coverage": 1, - "rag.information_density": 0.4045665358544417 - }, - "events": [], - "links": [], - "trace_trace_id": "8d33a529-d9b1-4208-b528-762222d5ef8c", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:47:42.103000000", - "trace_end_time": "2025-10-16 08:47:51.032000000", - "trace_duration_ms": 8928.948, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:47:42.103000000", - "updated_at": "2025-10-16 08:47:42.103000000" - }, - { - "span_id": "0b3d6893-776e-4118-b8b5-b1c5ea51def1", - "trace_id": "06192a09-b6bb-4548-8a94-e123d6caaffe", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-16 08:47:03.776000000", - "end_time": "2025-10-16 08:47:41.100000000", - "duration_ms": 37324.516, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What is Noveum and what does it do?", - "query.length": 35, - "processing.mode": "RAG", - "processing.time_seconds": 37.324477195739746, - "processing.response_length": 648, - "processing.sources_count": 5, - "output_response": "Final Answer: Noveum.ai is a comprehensive tracing and observability platform specifically designed for AI applications, including those powered by large language models (LLMs), retrieval-augmented generation (RAG)...", - "final_answer_mode": "RAG", - "query_processed.input_query": "What is Noveum and what does it do?", - "query_processed.output_response": "Successfully processed query using RAG, generated 648 character response", - "query_processed.mode": "RAG", - "query_processed.processing_time": 37.324477195739746, - "query_processed.response_length": 648 - }, - "events": [], - "links": [], - "trace_trace_id": "06192a09-b6bb-4548-8a94-e123d6caaffe", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:47:03.776000000", - "trace_end_time": "2025-10-16 08:47:41.100000000", - "trace_duration_ms": 37324.601, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:47:03.776000000", - "updated_at": "2025-10-16 08:47:03.776000000" - }, - { - "span_id": "137ad05b-bada-42be-abe5-be0c277f6d0e", - "trace_id": "06192a09-b6bb-4548-8a94-e123d6caaffe", - "parent_span_id": "c3664d20-1708-45f5-a051-b626b0ec77cd", - "name": "agent.llm-rag", - "start_time": "2025-10-16 08:47:03.776000000", - "end_time": "2025-10-16 08:47:41.100000000", - "duration_ms": 37324.042, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "rag_agent", - "agent.operation": "llm-rag", - "agent.capabilities": [ - "document_retrieval", - "context_generation", - "response_generation" - ], - "agent.id": "noveum_rag_agent", - "input_query": "What is Noveum and what does it do?", - "query_length": 35, - "query_type": "rag_query", - "output_response": "RAG Answer: Noveum.ai is a comprehensive tracing and observability platform specifically designed for AI applications, including those powered by large language models (LLMs), retrieval-augmented generation (RAG)...", - "rag.context_length": 2334, - "rag.sources_count": 5, - "rag.answer_length": 648, - "rag.mode": "retrieval_augmented_generation" - }, - "events": [], - "links": [], - "trace_trace_id": "06192a09-b6bb-4548-8a94-e123d6caaffe", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:47:03.776000000", - "trace_end_time": "2025-10-16 08:47:41.100000000", - "trace_duration_ms": 37324.601, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:47:03.776000000", - "updated_at": "2025-10-16 08:47:03.776000000" - }, - { - "span_id": "c3664d20-1708-45f5-a051-b626b0ec77cd", - "trace_id": "06192a09-b6bb-4548-8a94-e123d6caaffe", - "parent_span_id": "0b3d6893-776e-4118-b8b5-b1c5ea51def1", - "name": "agent.query_routing", - "start_time": "2025-10-16 08:47:03.776000000", - "end_time": "2025-10-16 08:47:41.100000000", - "duration_ms": 37324.382999999994, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What is Noveum and what does it do?", - "query_length": 35, - "query_type": "routing_query", - "output_response": "Routed to RAG for query processing", - "router.classification": "RAG", - "router.confidence_score": 1, - "router.rag_keyword_score": 1, - "router.web_keyword_score": 0, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "06192a09-b6bb-4548-8a94-e123d6caaffe", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:47:03.776000000", - "trace_end_time": "2025-10-16 08:47:41.100000000", - "trace_duration_ms": 37324.601, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:47:03.776000000", - "updated_at": "2025-10-16 08:47:03.776000000" - }, - { - "span_id": "ce9a94bf-1a72-4599-8561-11e66802b92d", - "trace_id": "06192a09-b6bb-4548-8a94-e123d6caaffe", - "parent_span_id": "c3664d20-1708-45f5-a051-b626b0ec77cd", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-16 08:47:03.776000000", - "end_time": "2025-10-16 08:47:03.776000000", - "duration_ms": 0.067, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What is Noveum and what does it do?", - "query_length": 35, - "query_type": "routing_evaluation_query", - "classification.mode": "RAG", - "classification.rag_keyword_score": 1, - "classification.web_keyword_score": 0, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "noveum_specific", - "query.keyword_density": 0.125, - "query.domain_affinity": "noveum", - "routing.decision": "Routed to RAG based on analysis", - "routing.rationale": "RAG score: 1, Web score: 0, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "Web Search", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.019230769230769232, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: RAG (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "06192a09-b6bb-4548-8a94-e123d6caaffe", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:47:03.776000000", - "trace_end_time": "2025-10-16 08:47:41.100000000", - "trace_duration_ms": 37324.601, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:47:03.776000000", - "updated_at": "2025-10-16 08:47:03.776000000" - }, - { - "span_id": "38414416-907a-4188-95fe-c0a5f900f72c", - "trace_id": "06192a09-b6bb-4548-8a94-e123d6caaffe", - "parent_span_id": "137ad05b-bada-42be-abe5-be0c277f6d0e", - "name": "agent.llm_model_execution", - "start_time": "2025-10-16 08:47:10.392000000", - "end_time": "2025-10-16 08:47:40.392000000", - "duration_ms": 30000.104, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What is Noveum and what does it do?", - "query_length": 35, - "query_type": "rag_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 30.000043392181396, - "model.latency_ms": 30000.043392181396, - "model.start_time": 1760604430.3921378, - "model.end_time": 1760604460.3921812, - "model.performance_tier": "slow", - "model.prompt_tokens": 716, - "model.completion_tokens": 162, - "model.total_tokens": 878, - "model.tokens_per_second": 29.266624335244266, - "model.estimated_cost": 0.026340000000000002, - "model.efficiency_score": 0.7380410022779044, - "model.response_length": 648, - "model.response_quality": "high", - "model.output_response": "Model Response: Noveum.ai is a comprehensive tracing and observability platform specifically designed for AI applications, including those powered by large language models (LLMs), retrieval-augmented generation (RAG)...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "06192a09-b6bb-4548-8a94-e123d6caaffe", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:47:03.776000000", - "trace_end_time": "2025-10-16 08:47:41.100000000", - "trace_duration_ms": 37324.601, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:47:03.776000000", - "updated_at": "2025-10-16 08:47:03.776000000" - }, - { - "span_id": "67e23649-b00b-4eb8-9a69-865e80f833b3", - "trace_id": "06192a09-b6bb-4548-8a94-e123d6caaffe", - "parent_span_id": "137ad05b-bada-42be-abe5-be0c277f6d0e", - "name": "agent.rag_evaluation_metrics", - "start_time": "2025-10-16 08:47:40.392000000", - "end_time": "2025-10-16 08:47:40.392000000", - "duration_ms": 0.029, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "rag_evaluation_metrics", - "agent.capabilities": [ - "retrieval_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What is Noveum and what does it do?", - "query_length": 35, - "query_type": "rag_evaluation_query", - "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, vector searches, data retrieval, agent reasoning, and more. Without proper observability, debugging ...", - "retrieval.context_length": 2334, - "retrieval.sources_count": 5, - "retrieval.context_quality": "high", - "retrieval.effectiveness": 1, - "retrieval.context_utilization": 2.334, - "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, vector searches, data retrieval, agent reasoning, and more. Without proper observability, debugging becomes a nightmare, optimization is guesswork, and understanding user interactions is nearly impossible.\nNoveum.ai solves this by providing comprehensive tracing and observability specifically designed for AI applications.\nKey benefits of using Noveum.ai:\n- Complete Visibility: Trace...\n\nSource 2 (https://noveum.ai/en/docs/getting-started/overview):\nNoveum.ai Overview\nComprehensive AI tracing and observability platform for LLM applications, RAG systems, and AI agents\nWelcome to Noveum.ai\u2014the comprehensive tracing and observability platform built specifically for AI applications. Whether you're building LLM-powered chatbots, RAG systems, multi-agent workflows, or any AI-driven application, Noveum provides the insights you need to understand, debug, and optimize your systems.\n\ud83c\udfaf Why AI Applications Need Specialized Observability\nTraditional mo...\n\nSource 3 (https://noveum.ai/docs/getting-started/overview):\nNoveum.ai Overview\nComprehensive AI tracing and observability platform for LLM applications, RAG systems, and AI agents\nWelcome to Noveum.ai\u2014the comprehensive tracing and observability platform built specifically for AI applications. Whether you're building LLM-powered chatbots, RAG systems, multi-agent workflows, or any AI-driven application, Noveum provides the insights you need to understand, debug, and optimize your systems.\n\ud83c\udfaf Why AI Applications Need Specialized Observability\nTraditional mo...\n\nSource 4 (https://noveum.ai/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nSource 5 (https://noveum.ai/en/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nUser Question: What is Noveum and what does it do?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", - "prompt.prompt_length": 2865, - "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, ...", - "prompt.instruction_following": "rag_optimized", - "response.answer_length": 648, - "response.answer_completeness": "complete", - "response.response_quality": "high", - "response.source_citation": 5, - "response.context_utilization": 0.2776349614395887, - "output_response": "RAG Answer: Noveum.ai is a comprehensive tracing and observability platform specifically designed for AI applications, including those powered by large language models (LLMs), retrieval-augmented generation (RAG)...", - "evaluation.retrieval_effectiveness": 1, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 5, - "evaluation.context_relevance": "high", - "evaluation.overall_quality": "high", - "evaluation.ready_for_production": true, - "rag.retrieval_strategy": "semantic_similarity", - "rag.vector_search_results": 5, - "rag.context_synthesis": "multi_source", - "rag.document_coverage": 1, - "rag.information_density": 0.2776349614395887 - }, - "events": [], - "links": [], - "trace_trace_id": "06192a09-b6bb-4548-8a94-e123d6caaffe", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:47:03.776000000", - "trace_end_time": "2025-10-16 08:47:41.100000000", - "trace_duration_ms": 37324.601, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:47:03.776000000", - "updated_at": "2025-10-16 08:47:03.776000000" - }, - { - "span_id": "050851c9-b2d3-4c82-accb-4085dc69f785", - "trace_id": "3552ee8e-eafb-499a-b109-a9fb851a41d2", - "parent_span_id": "", - "name": "system_initialization", - "start_time": "2025-10-16 08:47:03.695000000", - "end_time": "2025-10-16 08:47:03.770000000", - "duration_ms": 74.58, - "status": "ok", - "status_message": "", - "attributes": { - "system.force_scrape": false, - "system.config": { - "noveum_base_url": "https://noveum.ai", - "max_pages_to_scrape": 50, - "chunk_size": 1000, - "chunk_overlap": 200, - "max_search_results": 5, - "rag_threshold": 0.7, - "noveum_docs_file": "noveum_docs.json", - "vector_store_path": "noveum_vectorstore" - }, - "input_query": "Initialize system with force_scrape=False", - "output_response": "System initialization: RAG system loaded, vector store ready, agent operational", - "system.initialized": true, - "system.documents_loaded": 38, - "system.vectorstore_ready": true - }, - "events": [], - "links": [], - "trace_trace_id": "3552ee8e-eafb-499a-b109-a9fb851a41d2", - "trace_name": "auto_trace_system_initialization", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:47:03.695000000", - "trace_end_time": "2025-10-16 08:47:03.770000000", - "trace_duration_ms": 74.912, - "span_count": 1, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:47:03.695000000", - "updated_at": "2025-10-16 08:47:03.695000000" - }, - { - "span_id": "6b53235d-8b91-4d9e-8437-915828e84600", - "trace_id": "e6ef2449-7429-465b-9bff-f3def8de5bb0", - "parent_span_id": "e73ab184-50c1-4209-9276-be11e69e9513", - "name": "agent.query_routing", - "start_time": "2025-10-14 12:54:06.700000000", - "end_time": "2025-10-14 12:54:10.637000000", - "duration_ms": 3937.492, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What are the latest breakthroughs in artificial intelligence?", - "query_length": 61, - "query_type": "routing_query", - "output_response": "Routed to Web Search for query processing", - "router.classification": "Web Search", - "router.confidence_score": 1, - "router.rag_keyword_score": 0, - "router.web_keyword_score": 1, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "e6ef2449-7429-465b-9bff-f3def8de5bb0", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-14 12:54:06.700000000", - "trace_end_time": "2025-10-14 12:54:10.637000000", - "trace_duration_ms": 3937.729, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-14 12:54:06.700000000", - "updated_at": "2025-10-14 12:54:06.700000000" - }, - { - "span_id": "a7d1e33a-95a1-41a9-a117-62b4c373bd60", - "trace_id": "e6ef2449-7429-465b-9bff-f3def8de5bb0", - "parent_span_id": "6b53235d-8b91-4d9e-8437-915828e84600", - "name": "agent.web_search_generation", - "start_time": "2025-10-14 12:54:06.700000000", - "end_time": "2025-10-14 12:54:10.637000000", - "duration_ms": 3936.983, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "web_search_agent", - "agent.operation": "web_search_generation", - "agent.capabilities": [ - "web_search", - "content_synthesis", - "response_generation" - ], - "agent.id": "noveum_web_search_agent", - "input_query": "What are the latest breakthroughs in artificial intelligence?", - "query_length": 61, - "query_type": "web_search_query", - "output_response": "Web Search Answer: The web search did not yield specific results regarding the latest breakthroughs in artificial intelligence. However, I can provide a general overview based on knowledge up to October 2023.\n\nRecent br...", - "web_search.results_count": 1, - "web_search.context_length": 163, - "web_search.response_length": 1090, - "web_search.mode": "external_web_search" - }, - "events": [], - "links": [], - "trace_trace_id": "e6ef2449-7429-465b-9bff-f3def8de5bb0", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-14 12:54:06.700000000", - "trace_end_time": "2025-10-14 12:54:10.637000000", - "trace_duration_ms": 3937.729, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-14 12:54:06.700000000", - "updated_at": "2025-10-14 12:54:06.700000000" - }, - { - "span_id": "da630cab-2ba6-4f12-98b9-6ab1c0899c17", - "trace_id": "e6ef2449-7429-465b-9bff-f3def8de5bb0", - "parent_span_id": "6b53235d-8b91-4d9e-8437-915828e84600", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-14 12:54:06.700000000", - "end_time": "2025-10-14 12:54:06.700000000", - "duration_ms": 0.27999999999999997, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What are the latest breakthroughs in artificial intelligence?", - "query_length": 61, - "query_type": "routing_evaluation_query", - "classification.mode": "Web Search", - "classification.rag_keyword_score": 0, - "classification.web_keyword_score": 1, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "complex", - "query.intent": "general_knowledge", - "query.keyword_density": 0.125, - "query.domain_affinity": "general", - "routing.decision": "Routed to Web Search based on analysis", - "routing.rationale": "RAG score: 0, Web score: 1, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "RAG", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.019230769230769232, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: Web Search (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "e6ef2449-7429-465b-9bff-f3def8de5bb0", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-14 12:54:06.700000000", - "trace_end_time": "2025-10-14 12:54:10.637000000", - "trace_duration_ms": 3937.729, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-14 12:54:06.700000000", - "updated_at": "2025-10-14 12:54:06.700000000" - }, - { - "span_id": "e73ab184-50c1-4209-9276-be11e69e9513", - "trace_id": "e6ef2449-7429-465b-9bff-f3def8de5bb0", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-14 12:54:06.700000000", - "end_time": "2025-10-14 12:54:10.637000000", - "duration_ms": 3937.663, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What are the latest breakthroughs in artificial intelligence?", - "query.length": 61, - "processing.mode": "Web Search", - "processing.time_seconds": 3.93752384185791, - "processing.response_length": 1090, - "processing.sources_count": 1, - "output_response": "Final Answer: The web search did not yield specific results regarding the latest breakthroughs in artificial intelligence. However, I can provide a general overview based on knowledge up to October 2023.\n\nRecent br...", - "final_answer_mode": "Web Search", - "query_processed.input_query": "What are the latest breakthroughs in artificial intelligence?", - "query_processed.output_response": "Successfully processed query using Web Search, generated 1090 character response", - "query_processed.mode": "Web Search", - "query_processed.processing_time": 3.93752384185791, - "query_processed.response_length": 1090 - }, - "events": [], - "links": [], - "trace_trace_id": "e6ef2449-7429-465b-9bff-f3def8de5bb0", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-14 12:54:06.700000000", - "trace_end_time": "2025-10-14 12:54:10.637000000", - "trace_duration_ms": 3937.729, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-14 12:54:06.700000000", - "updated_at": "2025-10-14 12:54:06.700000000" - }, - { - "span_id": "bb47766c-afda-44fe-8838-7936c80cd4b6", - "trace_id": "e6ef2449-7429-465b-9bff-f3def8de5bb0", - "parent_span_id": "a7d1e33a-95a1-41a9-a117-62b4c373bd60", - "name": "agent.llm_model_execution", - "start_time": "2025-10-14 12:54:07.104000000", - "end_time": "2025-10-14 12:54:10.637000000", - "duration_ms": 3533.321, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What are the latest breakthroughs in artificial intelligence?", - "query_length": 61, - "query_type": "web_search_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 3.533268928527832, - "model.latency_ms": 3533.268928527832, - "model.start_time": 1760446447.104266, - "model.end_time": 1760446450.6375349, - "model.performance_tier": "medium", - "model.prompt_tokens": 167, - "model.completion_tokens": 272, - "model.total_tokens": 439, - "model.tokens_per_second": 124.2475477752307, - "model.estimated_cost": 0.013170000000000001, - "model.efficiency_score": 2.4829157175398633, - "model.response_length": 1090, - "model.response_quality": "high", - "model.output_response": "Model Response: The web search did not yield specific results regarding the latest breakthroughs in artificial intelligence. However, I can provide a general overview based on knowledge up to October 2023.\n\nRecent br...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "e6ef2449-7429-465b-9bff-f3def8de5bb0", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-14 12:54:06.700000000", - "trace_end_time": "2025-10-14 12:54:10.637000000", - "trace_duration_ms": 3937.729, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-14 12:54:06.700000000", - "updated_at": "2025-10-14 12:54:06.700000000" - }, - { - "span_id": "1d914a9d-b770-45dc-8308-6a36e9e0db89", - "trace_id": "e6ef2449-7429-465b-9bff-f3def8de5bb0", - "parent_span_id": "a7d1e33a-95a1-41a9-a117-62b4c373bd60", - "name": "agent.web_search_evaluation_metrics", - "start_time": "2025-10-14 12:54:10.637000000", - "end_time": "2025-10-14 12:54:10.637000000", - "duration_ms": 0.022, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "web_search_evaluation_metrics", - "agent.capabilities": [ - "web_search_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What are the latest breakthroughs in artificial intelligence?", - "query_length": 61, - "query_type": "web_search_evaluation_query", - "web_search.results_count": 1, - "web_search.context_length": 163, - "web_search.context_synthesized": "Context from 1 web sources", - "web_search.search_effectiveness": 0.2, - "web_search.context_quality": "low", - "web_search.source_diversity": 1, - "web_search.information_synthesis": "low", - "web_search.external_knowledge_utilization": 0.163, - "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What+are+the+latest+breakthroughs+in+artificial+intelligence?\n\nUser Question: What are the latest breakthroughs in artificial intelligence?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", - "prompt.prompt_length": 669, - "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What+are+the+latest+breakthroughs+in+artificial+intelligence?", - "prompt.instruction_following": "web_search_optimized", - "response.answer_length": 1090, - "response.answer_completeness": "complete", - "response.response_quality": "medium", - "response.source_citation": 1, - "response.context_utilization": 6.6871165644171775, - "output_response": "Web Search Answer: The web search did not yield specific results regarding the latest breakthroughs in artificial intelligence. However, I can provide a general overview based on knowledge up to October 2023.\n\nRecent br...", - "evaluation.search_effectiveness": 0.2, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 1, - "evaluation.context_relevance": "low", - "evaluation.overall_quality": "low", - "evaluation.ready_for_production": true, - "web_search.search_strategy": "duckduckgo_api", - "web_search.real_time_data": true, - "web_search.external_sources": 1, - "web_search.information_freshness": "current", - "web_search.knowledge_synthesis": "single_source" - }, - "events": [], - "links": [], - "trace_trace_id": "e6ef2449-7429-465b-9bff-f3def8de5bb0", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-14 12:54:06.700000000", - "trace_end_time": "2025-10-14 12:54:10.637000000", - "trace_duration_ms": 3937.729, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-14 12:54:06.700000000", - "updated_at": "2025-10-14 12:54:06.700000000" - }, - { - "span_id": "91f3999c-2edf-4c56-8f4d-ab43b6ebbaee", - "trace_id": "2ba8cab0-2366-4add-8ac3-b490a05e9a60", - "parent_span_id": "cbe51252-a323-4852-a90e-c907a0480fa3", - "name": "agent.web_search_generation", - "start_time": "2025-10-14 12:53:56.480000000", - "end_time": "2025-10-14 12:54:05.685000000", - "duration_ms": 9205.212, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "web_search_agent", - "agent.operation": "web_search_generation", - "agent.capabilities": [ - "web_search", - "content_synthesis", - "response_generation" - ], - "agent.id": "noveum_web_search_agent", - "input_query": "What's happening in the software development world today?", - "query_length": 57, - "query_type": "web_search_query", - "output_response": "Web Search Answer: The software development world is currently experiencing rapid changes driven by advancements in artificial intelligence (AI) and evolving programming practices. Key trends include:\n\n1. **AI Integrati...", - "web_search.results_count": 1, - "web_search.context_length": 975, - "web_search.response_length": 1636, - "web_search.mode": "external_web_search" - }, - "events": [], - "links": [], - "trace_trace_id": "2ba8cab0-2366-4add-8ac3-b490a05e9a60", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-14 12:53:56.480000000", - "trace_end_time": "2025-10-14 12:54:05.687000000", - "trace_duration_ms": 9207.67, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-14 12:53:56.480000000", - "updated_at": "2025-10-14 12:53:56.480000000" - }, - { - "span_id": "ea2aee75-4537-46ea-954b-2002fc0026c5", - "trace_id": "2ba8cab0-2366-4add-8ac3-b490a05e9a60", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-14 12:53:56.480000000", - "end_time": "2025-10-14 12:54:05.687000000", - "duration_ms": 9207.528999999999, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What's happening in the software development world today?", - "query.length": 57, - "processing.mode": "Web Search", - "processing.time_seconds": 9.206756830215454, - "processing.response_length": 1636, - "processing.sources_count": 1, - "output_response": "Final Answer: The software development world is currently experiencing rapid changes driven by advancements in artificial intelligence (AI) and evolving programming practices. Key trends include:\n\n1. **AI Integrati...", - "final_answer_mode": "Web Search", - "query_processed.input_query": "What's happening in the software development world today?", - "query_processed.output_response": "Successfully processed query using Web Search, generated 1636 character response", - "query_processed.mode": "Web Search", - "query_processed.processing_time": 9.206756830215454, - "query_processed.response_length": 1636 - }, - "events": [], - "links": [], - "trace_trace_id": "2ba8cab0-2366-4add-8ac3-b490a05e9a60", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-14 12:53:56.480000000", - "trace_end_time": "2025-10-14 12:54:05.687000000", - "trace_duration_ms": 9207.67, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-14 12:53:56.480000000", - "updated_at": "2025-10-14 12:53:56.480000000" - }, - { - "span_id": "cbe51252-a323-4852-a90e-c907a0480fa3", - "trace_id": "2ba8cab0-2366-4add-8ac3-b490a05e9a60", - "parent_span_id": "ea2aee75-4537-46ea-954b-2002fc0026c5", - "name": "agent.query_routing", - "start_time": "2025-10-14 12:53:56.480000000", - "end_time": "2025-10-14 12:54:05.686000000", - "duration_ms": 9205.569000000001, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What's happening in the software development world today?", - "query_length": 57, - "query_type": "routing_query", - "output_response": "Routed to Web Search for query processing", - "router.classification": "Web Search", - "router.confidence_score": 1, - "router.rag_keyword_score": 0, - "router.web_keyword_score": 1, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "2ba8cab0-2366-4add-8ac3-b490a05e9a60", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-14 12:53:56.480000000", - "trace_end_time": "2025-10-14 12:54:05.687000000", - "trace_duration_ms": 9207.67, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-14 12:53:56.480000000", - "updated_at": "2025-10-14 12:53:56.480000000" - }, - { - "span_id": "d850ecee-86ef-4818-9cec-638bfee26293", - "trace_id": "2ba8cab0-2366-4add-8ac3-b490a05e9a60", - "parent_span_id": "cbe51252-a323-4852-a90e-c907a0480fa3", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-14 12:53:56.480000000", - "end_time": "2025-10-14 12:53:56.480000000", - "duration_ms": 0.041999999999999996, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What's happening in the software development world today?", - "query_length": 57, - "query_type": "routing_evaluation_query", - "classification.mode": "Web Search", - "classification.rag_keyword_score": 0, - "classification.web_keyword_score": 1, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "complex", - "query.intent": "general_knowledge", - "query.keyword_density": 0.125, - "query.domain_affinity": "general", - "routing.decision": "Routed to Web Search based on analysis", - "routing.rationale": "RAG score: 0, Web score: 1, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "RAG", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.019230769230769232, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: Web Search (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "2ba8cab0-2366-4add-8ac3-b490a05e9a60", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-14 12:53:56.480000000", - "trace_end_time": "2025-10-14 12:54:05.687000000", - "trace_duration_ms": 9207.67, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-14 12:53:56.480000000", - "updated_at": "2025-10-14 12:53:56.480000000" - }, - { - "span_id": "0d4387a4-adf3-42c1-aca7-51453a90dd36", - "trace_id": "2ba8cab0-2366-4add-8ac3-b490a05e9a60", - "parent_span_id": "91f3999c-2edf-4c56-8f4d-ab43b6ebbaee", - "name": "agent.llm_model_execution", - "start_time": "2025-10-14 12:53:59.426000000", - "end_time": "2025-10-14 12:54:05.684000000", - "duration_ms": 6258.643, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What's happening in the software development world today?", - "query_length": 57, - "query_type": "web_search_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 6.25848388671875, - "model.latency_ms": 6258.48388671875, - "model.start_time": 1760446439.426367, - "model.end_time": 1760446445.684851, - "model.performance_tier": "slow", - "model.prompt_tokens": 369, - "model.completion_tokens": 409, - "model.total_tokens": 778, - "model.tokens_per_second": 124.31125718019486, - "model.estimated_cost": 0.02334, - "model.efficiency_score": 2.102827763496144, - "model.response_length": 1636, - "model.response_quality": "high", - "model.output_response": "Model Response: The software development world is currently experiencing rapid changes driven by advancements in artificial intelligence (AI) and evolving programming practices. Key trends include:\n\n1. **AI Integrati...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "2ba8cab0-2366-4add-8ac3-b490a05e9a60", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-14 12:53:56.480000000", - "trace_end_time": "2025-10-14 12:54:05.687000000", - "trace_duration_ms": 9207.67, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-14 12:53:56.480000000", - "updated_at": "2025-10-14 12:53:56.480000000" - }, - { - "span_id": "717e3c4a-6300-45e5-9456-caa3e773d758", - "trace_id": "2ba8cab0-2366-4add-8ac3-b490a05e9a60", - "parent_span_id": "91f3999c-2edf-4c56-8f4d-ab43b6ebbaee", - "name": "agent.web_search_evaluation_metrics", - "start_time": "2025-10-14 12:54:05.685000000", - "end_time": "2025-10-14 12:54:05.685000000", - "duration_ms": 0.044, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "web_search_evaluation_metrics", - "agent.capabilities": [ - "web_search_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What's happening in the software development world today?", - "query_length": 57, - "query_type": "web_search_evaluation_query", - "web_search.results_count": 1, - "web_search.context_length": 975, - "web_search.context_synthesized": "Context from 1 web sources", - "web_search.search_effectiveness": 0.2, - "web_search.context_quality": "high", - "web_search.source_diversity": 1, - "web_search.information_synthesis": "low", - "web_search.external_knowledge_utilization": 0.975, - "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nLog in to WhatsApp Web for simple, reliable and private messaging on your desktop. Send and receive messages and files with ease, all for free. Sep 23, 2025 \u00b7 Chief Economists' Outlook 2025 states that these under-the-radar trends are shaping the global economy, from the impact of AI on research productivity to a tightened labor \u2026 Sep 1, 2025 \u00b7 THE AI DEVELOPMENT REVOLUTION: WHAT'S REALLY HAPPENING? Let's face it - the software development world is changing faster than a React component re-rendering on \u2026 Oct 7, 2025 \u00b7 Computer Programming Research. Read current computer science articles on everything from computer programs to detect cancer genes and control vehicle maintenance to \u2026 Aug 24, 2025 \u00b7 Artificial Intelligence News. Everything on AI including futuristic robots with artificial intelligence, computer models of human intelligence and more.\nURL: https://duckduckgo.com/?q=What's+happening+in+the+software+development+world+today?\n\nUser Question: What's happening in the software development world today?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", - "prompt.prompt_length": 1477, - "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nLog in to WhatsApp Web for simple, reliable and private messaging on your desktop. Send and receive messages and files with ease, all for free. Sep 23, 2025 \u00b7 Chief Economi...", - "prompt.instruction_following": "web_search_optimized", - "response.answer_length": 1636, - "response.answer_completeness": "complete", - "response.response_quality": "medium", - "response.source_citation": 1, - "response.context_utilization": 1.677948717948718, - "output_response": "Web Search Answer: The software development world is currently experiencing rapid changes driven by advancements in artificial intelligence (AI) and evolving programming practices. Key trends include:\n\n1. **AI Integrati...", - "evaluation.search_effectiveness": 0.2, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 1, - "evaluation.context_relevance": "high", - "evaluation.overall_quality": "low", - "evaluation.ready_for_production": true, - "web_search.search_strategy": "duckduckgo_api", - "web_search.real_time_data": true, - "web_search.external_sources": 1, - "web_search.information_freshness": "current", - "web_search.knowledge_synthesis": "single_source" - }, - "events": [], - "links": [], - "trace_trace_id": "2ba8cab0-2366-4add-8ac3-b490a05e9a60", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-14 12:53:56.480000000", - "trace_end_time": "2025-10-14 12:54:05.687000000", - "trace_duration_ms": 9207.67, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-14 12:53:56.480000000", - "updated_at": "2025-10-14 12:53:56.480000000" - }, - { - "span_id": "3d39f522-8ae7-4988-aafd-58b09734eff0", - "trace_id": "a7c3399e-372f-4aae-baeb-05215cc4b85f", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-14 12:53:50.963000000", - "end_time": "2025-10-14 12:53:55.471000000", - "duration_ms": 4507.466, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What are the newest features in cloud computing?", - "query.length": 48, - "processing.mode": "RAG", - "processing.time_seconds": 4.507421255111694, - "processing.response_length": 870, - "processing.sources_count": 5, - "output_response": "Final Answer: The provided context does not specifically mention any new features in cloud computing as a general topic. However, it does highlight several recent enhancements related to the Noveum.ai platform, whi...", - "final_answer_mode": "RAG", - "query_processed.input_query": "What are the newest features in cloud computing?", - "query_processed.output_response": "Successfully processed query using RAG, generated 870 character response", - "query_processed.mode": "RAG", - "query_processed.processing_time": 4.507421255111694, - "query_processed.response_length": 870 - }, - "events": [], - "links": [], - "trace_trace_id": "a7c3399e-372f-4aae-baeb-05215cc4b85f", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-14 12:53:50.963000000", - "trace_end_time": "2025-10-14 12:53:55.471000000", - "trace_duration_ms": 4507.555, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-14 12:53:50.963000000", - "updated_at": "2025-10-14 12:53:50.963000000" - }, - { - "span_id": "54aa6700-f2de-4cf0-b1ad-a3d24ed692ed", - "trace_id": "a7c3399e-372f-4aae-baeb-05215cc4b85f", - "parent_span_id": "d17f31cf-c570-4604-8df3-532b2e530fce", - "name": "agent.llm-rag", - "start_time": "2025-10-14 12:53:50.963000000", - "end_time": "2025-10-14 12:53:55.471000000", - "duration_ms": 4507.083, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "rag_agent", - "agent.operation": "llm-rag", - "agent.capabilities": [ - "document_retrieval", - "context_generation", - "response_generation" - ], - "agent.id": "noveum_rag_agent", - "input_query": "What are the newest features in cloud computing?", - "query_length": 48, - "query_type": "rag_query", - "output_response": "RAG Answer: The provided context does not specifically mention any new features in cloud computing as a general topic. However, it does highlight several recent enhancements related to the Noveum.ai platform, whi...", - "rag.context_length": 2579, - "rag.sources_count": 5, - "rag.answer_length": 870, - "rag.mode": "retrieval_augmented_generation" - }, - "events": [], - "links": [], - "trace_trace_id": "a7c3399e-372f-4aae-baeb-05215cc4b85f", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-14 12:53:50.963000000", - "trace_end_time": "2025-10-14 12:53:55.471000000", - "trace_duration_ms": 4507.555, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-14 12:53:50.963000000", - "updated_at": "2025-10-14 12:53:50.963000000" - }, - { - "span_id": "8878096f-6cfa-4f70-9844-d7723a0875f0", - "trace_id": "a7c3399e-372f-4aae-baeb-05215cc4b85f", - "parent_span_id": "d17f31cf-c570-4604-8df3-532b2e530fce", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-14 12:53:50.963000000", - "end_time": "2025-10-14 12:53:50.963000000", - "duration_ms": 0.036000000000000004, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What are the newest features in cloud computing?", - "query_length": 48, - "query_type": "routing_evaluation_query", - "classification.mode": "RAG", - "classification.rag_keyword_score": 1, - "classification.web_keyword_score": 0, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "documentation", - "query.keyword_density": 0.125, - "query.domain_affinity": "noveum", - "routing.decision": "Routed to RAG based on analysis", - "routing.rationale": "RAG score: 1, Web score: 0, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "Web Search", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.019230769230769232, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: RAG (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "a7c3399e-372f-4aae-baeb-05215cc4b85f", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-14 12:53:50.963000000", - "trace_end_time": "2025-10-14 12:53:55.471000000", - "trace_duration_ms": 4507.555, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-14 12:53:50.963000000", - "updated_at": "2025-10-14 12:53:50.963000000" - }, - { - "span_id": "d17f31cf-c570-4604-8df3-532b2e530fce", - "trace_id": "a7c3399e-372f-4aae-baeb-05215cc4b85f", - "parent_span_id": "3d39f522-8ae7-4988-aafd-58b09734eff0", - "name": "agent.query_routing", - "start_time": "2025-10-14 12:53:50.963000000", - "end_time": "2025-10-14 12:53:55.471000000", - "duration_ms": 4507.3730000000005, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What are the newest features in cloud computing?", - "query_length": 48, - "query_type": "routing_query", - "output_response": "Routed to RAG for query processing", - "router.classification": "RAG", - "router.confidence_score": 1, - "router.rag_keyword_score": 1, - "router.web_keyword_score": 0, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "a7c3399e-372f-4aae-baeb-05215cc4b85f", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-14 12:53:50.963000000", - "trace_end_time": "2025-10-14 12:53:55.471000000", - "trace_duration_ms": 4507.555, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-14 12:53:50.963000000", - "updated_at": "2025-10-14 12:53:50.963000000" - }, - { - "span_id": "97ef01a8-5930-4902-909a-245c516a5871", - "trace_id": "a7c3399e-372f-4aae-baeb-05215cc4b85f", - "parent_span_id": "54aa6700-f2de-4cf0-b1ad-a3d24ed692ed", - "name": "agent.llm_model_execution", - "start_time": "2025-10-14 12:53:51.439000000", - "end_time": "2025-10-14 12:53:55.077000000", - "duration_ms": 3637.337, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What are the newest features in cloud computing?", - "query_length": 48, - "query_type": "rag_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 3.6372909545898438, - "model.latency_ms": 3637.2909545898438, - "model.start_time": 1760446431.439748, - "model.end_time": 1760446435.077039, - "model.performance_tier": "medium", - "model.prompt_tokens": 780, - "model.completion_tokens": 217, - "model.total_tokens": 997, - "model.tokens_per_second": 274.1050997698989, - "model.estimated_cost": 0.02991, - "model.efficiency_score": 0.872617853560682, - "model.response_length": 870, - "model.response_quality": "high", - "model.output_response": "Model Response: The provided context does not specifically mention any new features in cloud computing as a general topic. However, it does highlight several recent enhancements related to the Noveum.ai platform, whi...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "a7c3399e-372f-4aae-baeb-05215cc4b85f", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-14 12:53:50.963000000", - "trace_end_time": "2025-10-14 12:53:55.471000000", - "trace_duration_ms": 4507.555, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-14 12:53:50.963000000", - "updated_at": "2025-10-14 12:53:50.963000000" - }, - { - "span_id": "ea792ec6-64b4-4ed5-b418-ba902e90c57a", - "trace_id": "a7c3399e-372f-4aae-baeb-05215cc4b85f", - "parent_span_id": "54aa6700-f2de-4cf0-b1ad-a3d24ed692ed", - "name": "agent.rag_evaluation_metrics", - "start_time": "2025-10-14 12:53:55.077000000", - "end_time": "2025-10-14 12:53:55.077000000", - "duration_ms": 0.034, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "rag_evaluation_metrics", - "agent.capabilities": [ - "retrieval_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What are the newest features in cloud computing?", - "query_length": 48, - "query_type": "rag_evaluation_query", - "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/changelog):\n- \ud83d\udcc8 Enhanced Dashboard Analytics: Real-time request tracking, latency monitoring, and cost analysis\n- \ud83d\udd0d Improved Logs Interface: Better search functionality, detailed trace views, and enhanced debugging capabilities\n- \ud83d\udc65 Team Management: Advanced member mana...", - "retrieval.context_length": 2579, - "retrieval.sources_count": 5, - "retrieval.context_quality": "high", - "retrieval.effectiveness": 1, - "retrieval.context_utilization": 2.579, - "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/changelog):\n- \ud83d\udcc8 Enhanced Dashboard Analytics: Real-time request tracking, latency monitoring, and cost analysis\n- \ud83d\udd0d Improved Logs Interface: Better search functionality, detailed trace views, and enhanced debugging capabilities\n- \ud83d\udc65 Team Management: Advanced member management with role-based access control and invitation system\n- \ud83d\udee1\ufe0f Security Improvements: Enhanced API key management and secure credential storage\n4 months ago\n- \ud83d\udcca Advanced Metrics Collection: Comprehensive telemetry system with custom metrics ...\n\nSource 2 (https://noveum.ai/en/changelog):\n- \ud83d\udcdd Documentation Updates: Comprehensive guides for integration and best practices\n5 months ago\n- \ud83c\udf89 Platform Foundation: Initial release of Noveum.ai AI observability platform\n- \ud83d\ude80 Core Features: AI Gateway integration, basic evaluation metrics, and monitoring capabilities\n- \ud83d\udcc8 Evaluation Jobs: Automated model evaluation with configurable metrics and reporting\n- \ud83d\udd0c Provider Integrations: Support for major AI providers and custom model deployments...\n\nSource 3 (https://noveum.ai/docs/platform/dashboard):\n- Bulk Export: Download multiple traces simultaneously\n- Comparative Analysis: Compare performance across multiple traces\nReal-Time Updates\n- Live Refresh: Automatic updates as new traces arrive\n- Connection Monitoring: Real-time status of your trace ingestion pipeline\n- Performance Indicators: Live metrics for system health monitoring\nReady to dive deeper? Explore Projects & Environments to organize your AI applications, or check out Team Collaboration to share insights with your team.\nGet Earl...\n\nSource 4 (https://noveum.ai/en/changelog):\nChangelog\nStay up to date with the latest changes in our product.\n2 months ago\n- \ud83c\udfaf Major Platform Evolution: NovaEval Framework - Launched comprehensive AI model evaluation framework with 20+ built-in scorers including accuracy, RAG metrics, conversational scoring, and specialized agent evaluation tools. Features production-grade deployment on Docker and Kubernetes. (GitHub | PyPI Package)\n- \ud83d\ude80 Next-Generation Traces UI - Complete redesign featuring three-pane layout with directory tree navigatio...\n\nSource 5 (https://noveum.ai/en/blog):\n8/12/2025\nDiscover how Noveum.ai provides comprehensive tracing and observability for AI applications, from development debugging to production optimization.\nShashank Agarwal\n3/3/2025\nDiscover how Noveum.ai provides comprehensive tracing and observability for LLM applications, RAG systems, and multi-agent workflows with our powerful Python and TypeScript SDKs.\nShashank Agarwal\n3/2/2025...\n\nUser Question: What are the newest features in cloud computing?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", - "prompt.prompt_length": 3123, - "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/changelog):\n- \ud83d\udcc8 Enhanced Dashboard Analytics: Real-time request tracking, latency monitoring, and cost analysis\n- \ud83d\udd0d Improved Logs Interface: Better search functionality,...", - "prompt.instruction_following": "rag_optimized", - "response.answer_length": 870, - "response.answer_completeness": "complete", - "response.response_quality": "high", - "response.source_citation": 5, - "response.context_utilization": 0.33734005428460645, - "output_response": "RAG Answer: The provided context does not specifically mention any new features in cloud computing as a general topic. However, it does highlight several recent enhancements related to the Noveum.ai platform, whi...", - "evaluation.retrieval_effectiveness": 1, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 5, - "evaluation.context_relevance": "high", - "evaluation.overall_quality": "high", - "evaluation.ready_for_production": true, - "rag.retrieval_strategy": "semantic_similarity", - "rag.vector_search_results": 5, - "rag.context_synthesis": "multi_source", - "rag.document_coverage": 1, - "rag.information_density": 0.33734005428460645 - }, - "events": [], - "links": [], - "trace_trace_id": "a7c3399e-372f-4aae-baeb-05215cc4b85f", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-14 12:53:50.963000000", - "trace_end_time": "2025-10-14 12:53:55.471000000", - "trace_duration_ms": 4507.555, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-14 12:53:50.963000000", - "updated_at": "2025-10-14 12:53:50.963000000" - }, - { - "span_id": "e2d314c8-687f-4b8d-869c-8d7f2a22a4f3", - "trace_id": "d682797f-2ce0-448e-8f0d-891541430988", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-14 12:53:44.917000000", - "end_time": "2025-10-14 12:53:49.957000000", - "duration_ms": 5039.565, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What's the current status of cryptocurrency markets?", - "query.length": 52, - "processing.mode": "Web Search", - "processing.time_seconds": 5.0394370555877686, - "processing.response_length": 338, - "processing.sources_count": 1, - "output_response": "Final Answer: The web search results did not provide any specific information regarding the current status of cryptocurrency markets. Therefore, I cannot provide an update on market conditions, prices, or trends at...", - "final_answer_mode": "Web Search", - "query_processed.input_query": "What's the current status of cryptocurrency markets?", - "query_processed.output_response": "Successfully processed query using Web Search, generated 338 character response", - "query_processed.mode": "Web Search", - "query_processed.processing_time": 5.0394370555877686, - "query_processed.response_length": 338 - }, - "events": [], - "links": [], - "trace_trace_id": "d682797f-2ce0-448e-8f0d-891541430988", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-14 12:53:44.917000000", - "trace_end_time": "2025-10-14 12:53:49.957000000", - "trace_duration_ms": 5039.64, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-14 12:53:44.917000000", - "updated_at": "2025-10-14 12:53:44.917000000" - }, - { - "span_id": "d367a6bd-86af-41c5-96ca-5284425775ce", - "trace_id": "d682797f-2ce0-448e-8f0d-891541430988", - "parent_span_id": "f137a412-6ddf-4ace-908e-9edf73534aa7", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-14 12:53:44.917000000", - "end_time": "2025-10-14 12:53:44.918000000", - "duration_ms": 0.046, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What's the current status of cryptocurrency markets?", - "query_length": 52, - "query_type": "routing_evaluation_query", - "classification.mode": "Web Search", - "classification.rag_keyword_score": 0, - "classification.web_keyword_score": 3, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "complex", - "query.intent": "general_knowledge", - "query.keyword_density": 0.42857142857142855, - "query.domain_affinity": "general", - "routing.decision": "Routed to Web Search based on analysis", - "routing.rationale": "RAG score: 0, Web score: 3, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "RAG", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.057692307692307696, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: Web Search (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "d682797f-2ce0-448e-8f0d-891541430988", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-14 12:53:44.917000000", - "trace_end_time": "2025-10-14 12:53:49.957000000", - "trace_duration_ms": 5039.64, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-14 12:53:44.917000000", - "updated_at": "2025-10-14 12:53:44.917000000" - }, - { - "span_id": "f137a412-6ddf-4ace-908e-9edf73534aa7", - "trace_id": "d682797f-2ce0-448e-8f0d-891541430988", - "parent_span_id": "e2d314c8-687f-4b8d-869c-8d7f2a22a4f3", - "name": "agent.query_routing", - "start_time": "2025-10-14 12:53:44.917000000", - "end_time": "2025-10-14 12:53:49.957000000", - "duration_ms": 5039.404, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What's the current status of cryptocurrency markets?", - "query_length": 52, - "query_type": "routing_query", - "output_response": "Routed to Web Search for query processing", - "router.classification": "Web Search", - "router.confidence_score": 1, - "router.rag_keyword_score": 0, - "router.web_keyword_score": 3, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "d682797f-2ce0-448e-8f0d-891541430988", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-14 12:53:44.917000000", - "trace_end_time": "2025-10-14 12:53:49.957000000", - "trace_duration_ms": 5039.64, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-14 12:53:44.917000000", - "updated_at": "2025-10-14 12:53:44.917000000" - }, - { - "span_id": "987b6910-9b6d-4207-b0f3-8807a23e3ddb", - "trace_id": "d682797f-2ce0-448e-8f0d-891541430988", - "parent_span_id": "f137a412-6ddf-4ace-908e-9edf73534aa7", - "name": "agent.web_search_generation", - "start_time": "2025-10-14 12:53:44.918000000", - "end_time": "2025-10-14 12:53:49.957000000", - "duration_ms": 5039.248, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "web_search_agent", - "agent.operation": "web_search_generation", - "agent.capabilities": [ - "web_search", - "content_synthesis", - "response_generation" - ], - "agent.id": "noveum_web_search_agent", - "input_query": "What's the current status of cryptocurrency markets?", - "query_length": 52, - "query_type": "web_search_query", - "output_response": "Web Search Answer: The web search results did not provide any specific information regarding the current status of cryptocurrency markets. Therefore, I cannot provide an update on market conditions, prices, or trends at...", - "web_search.results_count": 1, - "web_search.context_length": 154, - "web_search.response_length": 338, - "web_search.mode": "external_web_search" - }, - "events": [], - "links": [], - "trace_trace_id": "d682797f-2ce0-448e-8f0d-891541430988", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-14 12:53:44.917000000", - "trace_end_time": "2025-10-14 12:53:49.957000000", - "trace_duration_ms": 5039.64, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-14 12:53:44.917000000", - "updated_at": "2025-10-14 12:53:44.917000000" - }, - { - "span_id": "b2614d0e-5fcd-46ec-8d25-f1b04eb142af", - "trace_id": "d682797f-2ce0-448e-8f0d-891541430988", - "parent_span_id": "987b6910-9b6d-4207-b0f3-8807a23e3ddb", - "name": "agent.llm_model_execution", - "start_time": "2025-10-14 12:53:45.255000000", - "end_time": "2025-10-14 12:53:49.957000000", - "duration_ms": 4701.744, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What's the current status of cryptocurrency markets?", - "query_length": 52, - "query_type": "web_search_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 4.701629161834717, - "model.latency_ms": 4701.629161834717, - "model.start_time": 1760446425.2554948, - "model.end_time": 1760446429.957124, - "model.performance_tier": "medium", - "model.prompt_tokens": 162, - "model.completion_tokens": 84, - "model.total_tokens": 246, - "model.tokens_per_second": 52.32228904756993, - "model.estimated_cost": 0.00738, - "model.efficiency_score": 1.3739837398373984, - "model.response_length": 338, - "model.response_quality": "high", - "model.output_response": "Model Response: The web search results did not provide any specific information regarding the current status of cryptocurrency markets. Therefore, I cannot provide an update on market conditions, prices, or trends at...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "d682797f-2ce0-448e-8f0d-891541430988", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-14 12:53:44.917000000", - "trace_end_time": "2025-10-14 12:53:49.957000000", - "trace_duration_ms": 5039.64, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-14 12:53:44.917000000", - "updated_at": "2025-10-14 12:53:44.917000000" - }, - { - "span_id": "663e8789-75d0-4c4d-b14d-4088022436ac", - "trace_id": "d682797f-2ce0-448e-8f0d-891541430988", - "parent_span_id": "987b6910-9b6d-4207-b0f3-8807a23e3ddb", - "name": "agent.web_search_evaluation_metrics", - "start_time": "2025-10-14 12:53:49.957000000", - "end_time": "2025-10-14 12:53:49.957000000", - "duration_ms": 0.032, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "web_search_evaluation_metrics", - "agent.capabilities": [ - "web_search_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What's the current status of cryptocurrency markets?", - "query_length": 52, - "query_type": "web_search_evaluation_query", - "web_search.results_count": 1, - "web_search.context_length": 154, - "web_search.context_synthesized": "Context from 1 web sources", - "web_search.search_effectiveness": 0.2, - "web_search.context_quality": "low", - "web_search.source_diversity": 1, - "web_search.information_synthesis": "low", - "web_search.external_knowledge_utilization": 0.154, - "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What's+the+current+status+of+cryptocurrency+markets?\n\nUser Question: What's the current status of cryptocurrency markets?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", - "prompt.prompt_length": 651, - "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What's+the+current+status+of+cryptocurrency+markets?", - "prompt.instruction_following": "web_search_optimized", - "response.answer_length": 338, - "response.answer_completeness": "complete", - "response.response_quality": "medium", - "response.source_citation": 1, - "response.context_utilization": 2.1948051948051948, - "output_response": "Web Search Answer: The web search results did not provide any specific information regarding the current status of cryptocurrency markets. Therefore, I cannot provide an update on market conditions, prices, or trends at...", - "evaluation.search_effectiveness": 0.2, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 1, - "evaluation.context_relevance": "low", - "evaluation.overall_quality": "low", - "evaluation.ready_for_production": true, - "web_search.search_strategy": "duckduckgo_api", - "web_search.real_time_data": true, - "web_search.external_sources": 1, - "web_search.information_freshness": "current", - "web_search.knowledge_synthesis": "single_source" - }, - "events": [], - "links": [], - "trace_trace_id": "d682797f-2ce0-448e-8f0d-891541430988", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-14 12:53:44.917000000", - "trace_end_time": "2025-10-14 12:53:49.957000000", - "trace_duration_ms": 5039.64, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-14 12:53:44.917000000", - "updated_at": "2025-10-14 12:53:44.917000000" - }, - { - "span_id": "3382fe7c-aaca-46e0-a8d2-dbee75ff3bed", - "trace_id": "b0858dca-4173-493b-a6c5-4e993f606477", - "parent_span_id": "6cf94049-87b0-4fc5-b37e-32d261543647", - "name": "agent.query_routing", - "start_time": "2025-10-14 12:53:37.818000000", - "end_time": "2025-10-14 12:53:43.908000000", - "duration_ms": 6089.22, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What are the latest updates in Python programming?", - "query_length": 50, - "query_type": "routing_query", - "output_response": "Routed to Web Search for query processing", - "router.classification": "Web Search", - "router.confidence_score": 1, - "router.rag_keyword_score": 0, - "router.web_keyword_score": 2, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "b0858dca-4173-493b-a6c5-4e993f606477", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-14 12:53:37.818000000", - "trace_end_time": "2025-10-14 12:53:43.908000000", - "trace_duration_ms": 6089.715, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-14 12:53:37.818000000", - "updated_at": "2025-10-14 12:53:37.818000000" - }, - { - "span_id": "6cf94049-87b0-4fc5-b37e-32d261543647", - "trace_id": "b0858dca-4173-493b-a6c5-4e993f606477", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-14 12:53:37.818000000", - "end_time": "2025-10-14 12:53:43.908000000", - "duration_ms": 6089.607, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What are the latest updates in Python programming?", - "query.length": 50, - "processing.mode": "Web Search", - "processing.time_seconds": 6.089300870895386, - "processing.response_length": 1324, - "processing.sources_count": 1, - "output_response": "Final Answer: The web search did not yield specific results regarding the latest updates in Python programming. However, I can provide a general overview based on my knowledge up to October 2023.\n\nAs of late 2023, ...", - "final_answer_mode": "Web Search", - "query_processed.input_query": "What are the latest updates in Python programming?", - "query_processed.output_response": "Successfully processed query using Web Search, generated 1324 character response", - "query_processed.mode": "Web Search", - "query_processed.processing_time": 6.089300870895386, - "query_processed.response_length": 1324 - }, - "events": [], - "links": [], - "trace_trace_id": "b0858dca-4173-493b-a6c5-4e993f606477", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-14 12:53:37.818000000", - "trace_end_time": "2025-10-14 12:53:43.908000000", - "trace_duration_ms": 6089.715, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-14 12:53:37.818000000", - "updated_at": "2025-10-14 12:53:37.818000000" - }, - { - "span_id": "21c8befa-356a-4034-8d62-792238cb9e6e", - "trace_id": "b0858dca-4173-493b-a6c5-4e993f606477", - "parent_span_id": "3382fe7c-aaca-46e0-a8d2-dbee75ff3bed", - "name": "agent.web_search_generation", - "start_time": "2025-10-14 12:53:37.819000000", - "end_time": "2025-10-14 12:53:43.908000000", - "duration_ms": 6088.807, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "web_search_agent", - "agent.operation": "web_search_generation", - "agent.capabilities": [ - "web_search", - "content_synthesis", - "response_generation" - ], - "agent.id": "noveum_web_search_agent", - "input_query": "What are the latest updates in Python programming?", - "query_length": 50, - "query_type": "web_search_query", - "output_response": "Web Search Answer: The web search did not yield specific results regarding the latest updates in Python programming. However, I can provide a general overview based on my knowledge up to October 2023.\n\nAs of late 2023, ...", - "web_search.results_count": 1, - "web_search.context_length": 152, - "web_search.response_length": 1324, - "web_search.mode": "external_web_search" - }, - "events": [], - "links": [], - "trace_trace_id": "b0858dca-4173-493b-a6c5-4e993f606477", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-14 12:53:37.818000000", - "trace_end_time": "2025-10-14 12:53:43.908000000", - "trace_duration_ms": 6089.715, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-14 12:53:37.818000000", - "updated_at": "2025-10-14 12:53:37.818000000" - }, - { - "span_id": "482feedf-52f4-4c6a-9186-d59898820caa", - "trace_id": "b0858dca-4173-493b-a6c5-4e993f606477", - "parent_span_id": "3382fe7c-aaca-46e0-a8d2-dbee75ff3bed", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-14 12:53:37.819000000", - "end_time": "2025-10-14 12:53:37.819000000", - "duration_ms": 0.039, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What are the latest updates in Python programming?", - "query_length": 50, - "query_type": "routing_evaluation_query", - "classification.mode": "Web Search", - "classification.rag_keyword_score": 0, - "classification.web_keyword_score": 2, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "general_knowledge", - "query.keyword_density": 0.25, - "query.domain_affinity": "general", - "routing.decision": "Routed to Web Search based on analysis", - "routing.rationale": "RAG score: 0, Web score: 2, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "RAG", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.038461538461538464, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: Web Search (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "b0858dca-4173-493b-a6c5-4e993f606477", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-14 12:53:37.818000000", - "trace_end_time": "2025-10-14 12:53:43.908000000", - "trace_duration_ms": 6089.715, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-14 12:53:37.818000000", - "updated_at": "2025-10-14 12:53:37.818000000" - }, - { - "span_id": "84f1f63f-ee05-411e-a7dd-4ee9e9826f5a", - "trace_id": "b0858dca-4173-493b-a6c5-4e993f606477", - "parent_span_id": "21c8befa-356a-4034-8d62-792238cb9e6e", - "name": "agent.llm_model_execution", - "start_time": "2025-10-14 12:53:38.191000000", - "end_time": "2025-10-14 12:53:43.907000000", - "duration_ms": 5716.078, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What are the latest updates in Python programming?", - "query_length": 50, - "query_type": "web_search_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 5.715963125228882, - "model.latency_ms": 5715.963125228882, - "model.start_time": 1760446418.191805, - "model.end_time": 1760446423.907768, - "model.performance_tier": "slow", - "model.prompt_tokens": 161, - "model.completion_tokens": 331, - "model.total_tokens": 492, - "model.tokens_per_second": 86.07473302765561, - "model.estimated_cost": 0.01476, - "model.efficiency_score": 2.6910569105691056, - "model.response_length": 1324, - "model.response_quality": "high", - "model.output_response": "Model Response: The web search did not yield specific results regarding the latest updates in Python programming. However, I can provide a general overview based on my knowledge up to October 2023.\n\nAs of late 2023, ...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "b0858dca-4173-493b-a6c5-4e993f606477", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-14 12:53:37.818000000", - "trace_end_time": "2025-10-14 12:53:43.908000000", - "trace_duration_ms": 6089.715, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-14 12:53:37.818000000", - "updated_at": "2025-10-14 12:53:37.818000000" - }, - { - "span_id": "2fb9ecf3-a497-4285-a694-1c95080d9e6a", - "trace_id": "b0858dca-4173-493b-a6c5-4e993f606477", - "parent_span_id": "21c8befa-356a-4034-8d62-792238cb9e6e", - "name": "agent.web_search_evaluation_metrics", - "start_time": "2025-10-14 12:53:43.908000000", - "end_time": "2025-10-14 12:53:43.908000000", - "duration_ms": 0.037, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "web_search_evaluation_metrics", - "agent.capabilities": [ - "web_search_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What are the latest updates in Python programming?", - "query_length": 50, - "query_type": "web_search_evaluation_query", - "web_search.results_count": 1, - "web_search.context_length": 152, - "web_search.context_synthesized": "Context from 1 web sources", - "web_search.search_effectiveness": 0.2, - "web_search.context_quality": "low", - "web_search.source_diversity": 1, - "web_search.information_synthesis": "low", - "web_search.external_knowledge_utilization": 0.152, - "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What+are+the+latest+updates+in+Python+programming?\n\nUser Question: What are the latest updates in Python programming?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", - "prompt.prompt_length": 647, - "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What+are+the+latest+updates+in+Python+programming?", - "prompt.instruction_following": "web_search_optimized", - "response.answer_length": 1324, - "response.answer_completeness": "complete", - "response.response_quality": "medium", - "response.source_citation": 1, - "response.context_utilization": 8.710526315789474, - "output_response": "Web Search Answer: The web search did not yield specific results regarding the latest updates in Python programming. However, I can provide a general overview based on my knowledge up to October 2023.\n\nAs of late 2023, ...", - "evaluation.search_effectiveness": 0.2, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 1, - "evaluation.context_relevance": "low", - "evaluation.overall_quality": "low", - "evaluation.ready_for_production": true, - "web_search.search_strategy": "duckduckgo_api", - "web_search.real_time_data": true, - "web_search.external_sources": 1, - "web_search.information_freshness": "current", - "web_search.knowledge_synthesis": "single_source" - }, - "events": [], - "links": [], - "trace_trace_id": "b0858dca-4173-493b-a6c5-4e993f606477", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-14 12:53:37.818000000", - "trace_end_time": "2025-10-14 12:53:43.908000000", - "trace_duration_ms": 6089.715, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-14 12:53:37.818000000", - "updated_at": "2025-10-14 12:53:37.818000000" - }, - { - "span_id": "9dd5ac2c-0deb-48e3-ac8b-0e6e31f7ff30", - "trace_id": "dc81be1a-e93d-4f60-90f3-ce1acaa8cb3e", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-14 12:53:34.584000000", - "end_time": "2025-10-14 12:53:36.780000000", - "duration_ms": 2196.351, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What happened in tech news this week?", - "query.length": 37, - "processing.mode": "Web Search", - "processing.time_seconds": 2.1948978900909424, - "processing.response_length": 324, - "processing.sources_count": 1, - "output_response": "Final Answer: I'm sorry, but I couldn't find any specific information regarding tech news from this week based on the search results. It appears that there were no relevant articles or updates available. If you're ...", - "final_answer_mode": "Web Search", - "query_processed.input_query": "What happened in tech news this week?", - "query_processed.output_response": "Successfully processed query using Web Search, generated 324 character response", - "query_processed.mode": "Web Search", - "query_processed.processing_time": 2.1948978900909424, - "query_processed.response_length": 324 - }, - "events": [], - "links": [], - "trace_trace_id": "dc81be1a-e93d-4f60-90f3-ce1acaa8cb3e", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-14 12:53:34.584000000", - "trace_end_time": "2025-10-14 12:53:36.780000000", - "trace_duration_ms": 2196.4579999999996, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-14 12:53:34.584000000", - "updated_at": "2025-10-14 12:53:34.584000000" - }, - { - "span_id": "b1cbafce-51fb-48fd-bc4a-d49081a2f374", - "trace_id": "dc81be1a-e93d-4f60-90f3-ce1acaa8cb3e", - "parent_span_id": "9dd5ac2c-0deb-48e3-ac8b-0e6e31f7ff30", - "name": "agent.query_routing", - "start_time": "2025-10-14 12:53:34.584000000", - "end_time": "2025-10-14 12:53:36.778000000", - "duration_ms": 2194.2200000000003, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What happened in tech news this week?", - "query_length": 37, - "query_type": "routing_query", - "output_response": "Routed to Web Search for query processing", - "router.classification": "Web Search", - "router.confidence_score": 1, - "router.rag_keyword_score": 0, - "router.web_keyword_score": 2, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "dc81be1a-e93d-4f60-90f3-ce1acaa8cb3e", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-14 12:53:34.584000000", - "trace_end_time": "2025-10-14 12:53:36.780000000", - "trace_duration_ms": 2196.4579999999996, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-14 12:53:34.584000000", - "updated_at": "2025-10-14 12:53:34.584000000" - }, - { - "span_id": "e3064687-1b18-478b-9e9d-4d081a6dac23", - "trace_id": "dc81be1a-e93d-4f60-90f3-ce1acaa8cb3e", - "parent_span_id": "b1cbafce-51fb-48fd-bc4a-d49081a2f374", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-14 12:53:34.584000000", - "end_time": "2025-10-14 12:53:34.584000000", - "duration_ms": 0.028, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What happened in tech news this week?", - "query_length": 37, - "query_type": "routing_evaluation_query", - "classification.mode": "Web Search", - "classification.rag_keyword_score": 0, - "classification.web_keyword_score": 2, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "general_knowledge", - "query.keyword_density": 0.2857142857142857, - "query.domain_affinity": "general", - "routing.decision": "Routed to Web Search based on analysis", - "routing.rationale": "RAG score: 0, Web score: 2, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "RAG", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.038461538461538464, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: Web Search (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "dc81be1a-e93d-4f60-90f3-ce1acaa8cb3e", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-14 12:53:34.584000000", - "trace_end_time": "2025-10-14 12:53:36.780000000", - "trace_duration_ms": 2196.4579999999996, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-14 12:53:34.584000000", - "updated_at": "2025-10-14 12:53:34.584000000" - }, - { - "span_id": "eb9dbc11-44b3-4041-8c83-0f33b8b761bb", - "trace_id": "dc81be1a-e93d-4f60-90f3-ce1acaa8cb3e", - "parent_span_id": "b1cbafce-51fb-48fd-bc4a-d49081a2f374", - "name": "agent.web_search_generation", - "start_time": "2025-10-14 12:53:34.584000000", - "end_time": "2025-10-14 12:53:36.774000000", - "duration_ms": 2189.866, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "web_search_agent", - "agent.operation": "web_search_generation", - "agent.capabilities": [ - "web_search", - "content_synthesis", - "response_generation" - ], - "agent.id": "noveum_web_search_agent", - "input_query": "What happened in tech news this week?", - "query_length": 37, - "query_type": "web_search_query", - "output_response": "Web Search Answer: I'm sorry, but I couldn't find any specific information regarding tech news from this week based on the search results. It appears that there were no relevant articles or updates available. If you're ...", - "web_search.results_count": 1, - "web_search.context_length": 139, - "web_search.response_length": 324, - "web_search.mode": "external_web_search" - }, - "events": [], - "links": [], - "trace_trace_id": "dc81be1a-e93d-4f60-90f3-ce1acaa8cb3e", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-14 12:53:34.584000000", - "trace_end_time": "2025-10-14 12:53:36.780000000", - "trace_duration_ms": 2196.4579999999996, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-14 12:53:34.584000000", - "updated_at": "2025-10-14 12:53:34.584000000" - }, - { - "span_id": "c8ee237b-ab5e-4b36-83f0-2f8478971898", - "trace_id": "dc81be1a-e93d-4f60-90f3-ce1acaa8cb3e", - "parent_span_id": "eb9dbc11-44b3-4041-8c83-0f33b8b761bb", - "name": "agent.llm_model_execution", - "start_time": "2025-10-14 12:53:34.981000000", - "end_time": "2025-10-14 12:53:36.768000000", - "duration_ms": 1786.467, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What happened in tech news this week?", - "query_length": 37, - "query_type": "web_search_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 1.785179853439331, - "model.latency_ms": 1785.179853439331, - "model.start_time": 1760446414.981964, - "model.end_time": 1760446416.767144, - "model.performance_tier": "fast", - "model.prompt_tokens": 155, - "model.completion_tokens": 81, - "model.total_tokens": 236, - "model.tokens_per_second": 132.19956495997977, - "model.estimated_cost": 0.00708, - "model.efficiency_score": 1.3728813559322033, - "model.response_length": 324, - "model.response_quality": "high", - "model.output_response": "Model Response: I'm sorry, but I couldn't find any specific information regarding tech news from this week based on the search results. It appears that there were no relevant articles or updates available. If you're ...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "dc81be1a-e93d-4f60-90f3-ce1acaa8cb3e", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-14 12:53:34.584000000", - "trace_end_time": "2025-10-14 12:53:36.780000000", - "trace_duration_ms": 2196.4579999999996, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-14 12:53:34.584000000", - "updated_at": "2025-10-14 12:53:34.584000000" - }, - { - "span_id": "9c8e1a08-61c8-409e-a79d-41c74a8d1505", - "trace_id": "dc81be1a-e93d-4f60-90f3-ce1acaa8cb3e", - "parent_span_id": "eb9dbc11-44b3-4041-8c83-0f33b8b761bb", - "name": "agent.web_search_evaluation_metrics", - "start_time": "2025-10-14 12:53:36.772000000", - "end_time": "2025-10-14 12:53:36.774000000", - "duration_ms": 1.5939999999999999, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "web_search_evaluation_metrics", - "agent.capabilities": [ - "web_search_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What happened in tech news this week?", - "query_length": 37, - "query_type": "web_search_evaluation_query", - "web_search.results_count": 1, - "web_search.context_length": 139, - "web_search.context_synthesized": "Context from 1 web sources", - "web_search.search_effectiveness": 0.2, - "web_search.context_quality": "low", - "web_search.source_diversity": 1, - "web_search.information_synthesis": "low", - "web_search.external_knowledge_utilization": 0.139, - "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What+happened+in+tech+news+this+week?\n\nUser Question: What happened in tech news this week?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", - "prompt.prompt_length": 621, - "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What+happened+in+tech+news+this+week?", - "prompt.instruction_following": "web_search_optimized", - "response.answer_length": 324, - "response.answer_completeness": "complete", - "response.response_quality": "medium", - "response.source_citation": 1, - "response.context_utilization": 2.3309352517985613, - "output_response": "Web Search Answer: I'm sorry, but I couldn't find any specific information regarding tech news from this week based on the search results. It appears that there were no relevant articles or updates available. If you're ...", - "evaluation.search_effectiveness": 0.2, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 1, - "evaluation.context_relevance": "low", - "evaluation.overall_quality": "low", - "evaluation.ready_for_production": true, - "web_search.search_strategy": "duckduckgo_api", - "web_search.real_time_data": true, - "web_search.external_sources": 1, - "web_search.information_freshness": "current", - "web_search.knowledge_synthesis": "single_source" - }, - "events": [], - "links": [], - "trace_trace_id": "dc81be1a-e93d-4f60-90f3-ce1acaa8cb3e", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-14 12:53:34.584000000", - "trace_end_time": "2025-10-14 12:53:36.780000000", - "trace_duration_ms": 2196.4579999999996, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-14 12:53:34.584000000", - "updated_at": "2025-10-14 12:53:34.584000000" - }, - { - "span_id": "59595232-8873-419c-9b6a-0727445a2afa", - "trace_id": "945b2757-8a5a-4aab-a869-8f44c02166a5", - "parent_span_id": "dfde2f79-d0c1-481e-819d-60730cb60fe5", - "name": "agent.query_routing", - "start_time": "2025-10-14 12:53:27.771000000", - "end_time": "2025-10-14 12:53:33.575000000", - "duration_ms": 5803.9800000000005, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What are the current trends in observability tools?", - "query_length": 51, - "query_type": "routing_query", - "output_response": "Routed to RAG for query processing", - "router.classification": "RAG", - "router.confidence_score": 0.3333333333333333, - "router.rag_keyword_score": 2, - "router.web_keyword_score": 1, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "945b2757-8a5a-4aab-a869-8f44c02166a5", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-14 12:53:27.771000000", - "trace_end_time": "2025-10-14 12:53:33.575000000", - "trace_duration_ms": 5804.15, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-14 12:53:27.771000000", - "updated_at": "2025-10-14 12:53:27.771000000" - }, - { - "span_id": "a59b3296-aaba-4642-b5a0-a271787a5607", - "trace_id": "945b2757-8a5a-4aab-a869-8f44c02166a5", - "parent_span_id": "59595232-8873-419c-9b6a-0727445a2afa", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-14 12:53:27.771000000", - "end_time": "2025-10-14 12:53:27.771000000", - "duration_ms": 0.031, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What are the current trends in observability tools?", - "query_length": 51, - "query_type": "routing_evaluation_query", - "classification.mode": "RAG", - "classification.rag_keyword_score": 2, - "classification.web_keyword_score": 1, - "classification.confidence_score": 0.3333333333333333, - "classification.confidence_level": "medium", - "classification.method": "keyword_based", - "query.complexity": "complex", - "query.intent": "documentation", - "query.keyword_density": 0.375, - "query.domain_affinity": "noveum", - "routing.decision": "Routed to RAG based on analysis", - "routing.rationale": "RAG score: 2, Web score: 1, Confidence: 0.33", - "routing.expected_performance": "medium", - "routing.alternative_mode": "Web Search", - "routing.decision_confidence": 0.3333333333333333, - "evaluation.routing_accuracy": "medium", - "evaluation.keyword_coverage": 0.057692307692307696, - "evaluation.query_understanding": "ambiguous", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: RAG (Confidence: 0.33)" - }, - "events": [], - "links": [], - "trace_trace_id": "945b2757-8a5a-4aab-a869-8f44c02166a5", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-14 12:53:27.771000000", - "trace_end_time": "2025-10-14 12:53:33.575000000", - "trace_duration_ms": 5804.15, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-14 12:53:27.771000000", - "updated_at": "2025-10-14 12:53:27.771000000" - }, - { - "span_id": "dfde2f79-d0c1-481e-819d-60730cb60fe5", - "trace_id": "945b2757-8a5a-4aab-a869-8f44c02166a5", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-14 12:53:27.771000000", - "end_time": "2025-10-14 12:53:33.575000000", - "duration_ms": 5804.074, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What are the current trends in observability tools?", - "query.length": 51, - "processing.mode": "RAG", - "processing.time_seconds": 5.804032802581787, - "processing.response_length": 997, - "processing.sources_count": 5, - "output_response": "Final Answer: The provided context does not specifically outline current trends in observability tools in general. However, it highlights the unique challenges faced by AI applications and how traditional monitorin...", - "final_answer_mode": "RAG", - "query_processed.input_query": "What are the current trends in observability tools?", - "query_processed.output_response": "Successfully processed query using RAG, generated 997 character response", - "query_processed.mode": "RAG", - "query_processed.processing_time": 5.804032802581787, - "query_processed.response_length": 997 - }, - "events": [], - "links": [], - "trace_trace_id": "945b2757-8a5a-4aab-a869-8f44c02166a5", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-14 12:53:27.771000000", - "trace_end_time": "2025-10-14 12:53:33.575000000", - "trace_duration_ms": 5804.15, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-14 12:53:27.771000000", - "updated_at": "2025-10-14 12:53:27.771000000" - }, - { - "span_id": "82489444-deba-46d2-b08b-c09754189566", - "trace_id": "945b2757-8a5a-4aab-a869-8f44c02166a5", - "parent_span_id": "59595232-8873-419c-9b6a-0727445a2afa", - "name": "agent.llm-rag", - "start_time": "2025-10-14 12:53:27.772000000", - "end_time": "2025-10-14 12:53:33.575000000", - "duration_ms": 5803.534, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "rag_agent", - "agent.operation": "llm-rag", - "agent.capabilities": [ - "document_retrieval", - "context_generation", - "response_generation" - ], - "agent.id": "noveum_rag_agent", - "input_query": "What are the current trends in observability tools?", - "query_length": 51, - "query_type": "rag_query", - "output_response": "RAG Answer: The provided context does not specifically outline current trends in observability tools in general. However, it highlights the unique challenges faced by AI applications and how traditional monitorin...", - "rag.context_length": 2943, - "rag.sources_count": 5, - "rag.answer_length": 997, - "rag.mode": "retrieval_augmented_generation" - }, - "events": [], - "links": [], - "trace_trace_id": "945b2757-8a5a-4aab-a869-8f44c02166a5", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-14 12:53:27.771000000", - "trace_end_time": "2025-10-14 12:53:33.575000000", - "trace_duration_ms": 5804.15, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-14 12:53:27.771000000", - "updated_at": "2025-10-14 12:53:27.771000000" - }, - { - "span_id": "34b1f14c-3fd3-4f89-b4dd-6150a52befd7", - "trace_id": "945b2757-8a5a-4aab-a869-8f44c02166a5", - "parent_span_id": "82489444-deba-46d2-b08b-c09754189566", - "name": "agent.llm_model_execution", - "start_time": "2025-10-14 12:53:28.257000000", - "end_time": "2025-10-14 12:53:33.162000000", - "duration_ms": 4905.2429999999995, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What are the current trends in observability tools?", - "query_length": 51, - "query_type": "rag_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 4.9052040576934814, - "model.latency_ms": 4905.204057693481, - "model.start_time": 1760446408.2575018, - "model.end_time": 1760446413.162706, - "model.performance_tier": "medium", - "model.prompt_tokens": 872, - "model.completion_tokens": 249, - "model.total_tokens": 1121, - "model.tokens_per_second": 228.53279635569638, - "model.estimated_cost": 0.03363, - "model.efficiency_score": 0.8893844781445138, - "model.response_length": 997, - "model.response_quality": "high", - "model.output_response": "Model Response: The provided context does not specifically outline current trends in observability tools in general. However, it highlights the unique challenges faced by AI applications and how traditional monitorin...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "945b2757-8a5a-4aab-a869-8f44c02166a5", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-14 12:53:27.771000000", - "trace_end_time": "2025-10-14 12:53:33.575000000", - "trace_duration_ms": 5804.15, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-14 12:53:27.771000000", - "updated_at": "2025-10-14 12:53:27.771000000" - }, - { - "span_id": "d5d2b966-3c37-4d6d-b7b8-c58c9f22aad9", - "trace_id": "945b2757-8a5a-4aab-a869-8f44c02166a5", - "parent_span_id": "82489444-deba-46d2-b08b-c09754189566", - "name": "agent.rag_evaluation_metrics", - "start_time": "2025-10-14 12:53:33.162000000", - "end_time": "2025-10-14 12:53:33.162000000", - "duration_ms": 0.036000000000000004, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "rag_evaluation_metrics", - "agent.capabilities": [ - "retrieval_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What are the current trends in observability tools?", - "query_length": 51, - "query_type": "rag_evaluation_query", - "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/blog/from-logs-to-intelligent-choices-inside-noveum-ais-evaluation-process):\nFrom Development to Production - Inside Noveum.ai's AI Observability Platform\nShashank Agarwal\n3/3/2025\nIntroduction\nEvery AI application tells a story\u2014but most developers never get to hear it...", - "retrieval.context_length": 2943, - "retrieval.sources_count": 5, - "retrieval.context_quality": "high", - "retrieval.effectiveness": 1, - "retrieval.context_utilization": 2.943, - "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/blog/from-logs-to-intelligent-choices-inside-noveum-ais-evaluation-process):\nFrom Development to Production - Inside Noveum.ai's AI Observability Platform\nShashank Agarwal\n3/3/2025\nIntroduction\nEvery AI application tells a story\u2014but most developers never get to hear it. When your RAG pipeline returns irrelevant results, when your multi-agent system gets stuck in loops, or when your LLM costs suddenly spike, you're left guessing what went wrong and where.\nThat's exactly the challenge Noveum.ai solves. Rather than flying blind, our platform provides comprehensive tracing a...\n\nSource 2 (https://noveum.ai/docs/getting-started/tracing-concepts):\n- Explore Framework Integrations - Framework-specific guidance\n- Learn Advanced Patterns - Custom instrumentation techniques\n- Master the Dashboard - Analyze your traces effectively\nRemember: Good observability is not about collecting all possible data, but about collecting the right data that helps you understand, debug, and optimize your AI applications.\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to O...\n\nSource 3 (https://noveum.ai/en/docs/getting-started/tracing-concepts):\n- Explore Framework Integrations - Framework-specific guidance\n- Learn Advanced Patterns - Custom instrumentation techniques\n- Master the Dashboard - Analyze your traces effectively\nRemember: Good observability is not about collecting all possible data, but about collecting the right data that helps you understand, debug, and optimize your AI applications.\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to O...\n\nSource 4 (https://noveum.ai/en/blog/from-logs-to-intelligent-choices-inside-noveum-ais-evaluation-process):\nThe AI Observability Challenge\nWhy Traditional Monitoring Falls Short\nTraditional application monitoring tools weren't built for AI applications. They can tell you if your API is responding, but they can't answer the questions that matter most for AI systems:\n- Why did my RAG pipeline retrieve irrelevant documents?\n- Which LLM calls are driving my costs?\n- How are my agents communicating with each other?\n- What's causing hallucinations in my responses?\n- Why is my embedding generation so slow?\nW...\n\nSource 5 (https://noveum.ai/en/docs/advanced/rag-observability):\n\ud83d\udd27 Best Practices for RAG Observability\n1. Comprehensive Pipeline Tracing\n2. Quality Monitoring\n3. A/B Testing for RAG Components\n\ud83c\udfaf Advanced RAG Patterns\nMulti-Modal RAG\nConversational RAG\nRAG observability with Noveum.ai provides the deep insights needed to build, optimize, and scale retrieval-augmented generation systems. By implementing comprehensive tracing across retrieval, context preparation, and generation stages, you can ensure your RAG pipeline delivers accurate, relevant, and cost-effe...\n\nUser Question: What are the current trends in observability tools?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", - "prompt.prompt_length": 3490, - "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/blog/from-logs-to-intelligent-choices-inside-noveum-ais-evaluation-process):\nFrom Development to Production - Inside Noveum.ai's AI Observability Platform\nShashank Agarw...", - "prompt.instruction_following": "rag_optimized", - "response.answer_length": 997, - "response.answer_completeness": "complete", - "response.response_quality": "high", - "response.source_citation": 5, - "response.context_utilization": 0.33876996262317366, - "output_response": "RAG Answer: The provided context does not specifically outline current trends in observability tools in general. However, it highlights the unique challenges faced by AI applications and how traditional monitorin...", - "evaluation.retrieval_effectiveness": 1, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 5, - "evaluation.context_relevance": "high", - "evaluation.overall_quality": "high", - "evaluation.ready_for_production": true, - "rag.retrieval_strategy": "semantic_similarity", - "rag.vector_search_results": 5, - "rag.context_synthesis": "multi_source", - "rag.document_coverage": 1, - "rag.information_density": 0.33876996262317366 - }, - "events": [], - "links": [], - "trace_trace_id": "945b2757-8a5a-4aab-a869-8f44c02166a5", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-14 12:53:27.771000000", - "trace_end_time": "2025-10-14 12:53:33.575000000", - "trace_duration_ms": 5804.15, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-14 12:53:27.771000000", - "updated_at": "2025-10-14 12:53:27.771000000" - }, - { - "span_id": "11c07411-df30-42a4-9062-90a032ef75ed", - "trace_id": "4bdd5814-acbb-4de7-a56b-543b87a1cfa5", - "parent_span_id": "ec530838-a7cf-477b-9186-a70114c8a12f", - "name": "agent.web_search_generation", - "start_time": "2025-10-14 12:53:20.120000000", - "end_time": "2025-10-14 12:53:26.750000000", - "duration_ms": 6629.273999999999, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "web_search_agent", - "agent.operation": "web_search_generation", - "agent.capabilities": [ - "web_search", - "content_synthesis", - "response_generation" - ], - "agent.id": "noveum_web_search_agent", - "input_query": "Tell me about recent developments in machine learning", - "query_length": 53, - "query_type": "web_search_query", - "output_response": "Web Search Answer: The web search did not yield specific results regarding recent developments in machine learning. However, I can provide a general overview based on my knowledge up to October 2023.\n\nRecent development...", - "web_search.results_count": 1, - "web_search.context_length": 155, - "web_search.response_length": 1849, - "web_search.mode": "external_web_search" - }, - "events": [], - "links": [], - "trace_trace_id": "4bdd5814-acbb-4de7-a56b-543b87a1cfa5", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-14 12:53:20.120000000", - "trace_end_time": "2025-10-14 12:53:26.750000000", - "trace_duration_ms": 6630.38, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-14 12:53:20.120000000", - "updated_at": "2025-10-14 12:53:20.120000000" - }, - { - "span_id": "282b121f-61dd-4dc5-9e55-cecc3f02f688", - "trace_id": "4bdd5814-acbb-4de7-a56b-543b87a1cfa5", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-14 12:53:20.120000000", - "end_time": "2025-10-14 12:53:26.750000000", - "duration_ms": 6630.264, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "Tell me about recent developments in machine learning", - "query.length": 53, - "processing.mode": "Web Search", - "processing.time_seconds": 6.629686117172241, - "processing.response_length": 1849, - "processing.sources_count": 1, - "output_response": "Final Answer: The web search did not yield specific results regarding recent developments in machine learning. However, I can provide a general overview based on my knowledge up to October 2023.\n\nRecent development...", - "final_answer_mode": "Web Search", - "query_processed.input_query": "Tell me about recent developments in machine learning", - "query_processed.output_response": "Successfully processed query using Web Search, generated 1849 character response", - "query_processed.mode": "Web Search", - "query_processed.processing_time": 6.629686117172241, - "query_processed.response_length": 1849 - }, - "events": [], - "links": [], - "trace_trace_id": "4bdd5814-acbb-4de7-a56b-543b87a1cfa5", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-14 12:53:20.120000000", - "trace_end_time": "2025-10-14 12:53:26.750000000", - "trace_duration_ms": 6630.38, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-14 12:53:20.120000000", - "updated_at": "2025-10-14 12:53:20.120000000" - }, - { - "span_id": "ec530838-a7cf-477b-9186-a70114c8a12f", - "trace_id": "4bdd5814-acbb-4de7-a56b-543b87a1cfa5", - "parent_span_id": "282b121f-61dd-4dc5-9e55-cecc3f02f688", - "name": "agent.query_routing", - "start_time": "2025-10-14 12:53:20.120000000", - "end_time": "2025-10-14 12:53:26.750000000", - "duration_ms": 6629.59, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "Tell me about recent developments in machine learning", - "query_length": 53, - "query_type": "routing_query", - "output_response": "Routed to Web Search for query processing", - "router.classification": "Web Search", - "router.confidence_score": 1, - "router.rag_keyword_score": 0, - "router.web_keyword_score": 1, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "4bdd5814-acbb-4de7-a56b-543b87a1cfa5", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-14 12:53:20.120000000", - "trace_end_time": "2025-10-14 12:53:26.750000000", - "trace_duration_ms": 6630.38, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-14 12:53:20.120000000", - "updated_at": "2025-10-14 12:53:20.120000000" - }, - { - "span_id": "df882f90-cf1a-42a6-a2ae-78b2a13f27a1", - "trace_id": "4bdd5814-acbb-4de7-a56b-543b87a1cfa5", - "parent_span_id": "ec530838-a7cf-477b-9186-a70114c8a12f", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-14 12:53:20.120000000", - "end_time": "2025-10-14 12:53:20.120000000", - "duration_ms": 0.082, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: Tell me about recent developments in machine learning", - "query_length": 53, - "query_type": "routing_evaluation_query", - "classification.mode": "Web Search", - "classification.rag_keyword_score": 0, - "classification.web_keyword_score": 1, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "complex", - "query.intent": "general_knowledge", - "query.keyword_density": 0.125, - "query.domain_affinity": "general", - "routing.decision": "Routed to Web Search based on analysis", - "routing.rationale": "RAG score: 0, Web score: 1, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "RAG", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.019230769230769232, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: Web Search (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "4bdd5814-acbb-4de7-a56b-543b87a1cfa5", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-14 12:53:20.120000000", - "trace_end_time": "2025-10-14 12:53:26.750000000", - "trace_duration_ms": 6630.38, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-14 12:53:20.120000000", - "updated_at": "2025-10-14 12:53:20.120000000" - }, - { - "span_id": "bea819fd-3673-480b-9f7b-a3a2efef4218", - "trace_id": "4bdd5814-acbb-4de7-a56b-543b87a1cfa5", - "parent_span_id": "11c07411-df30-42a4-9062-90a032ef75ed", - "name": "agent.llm_model_execution", - "start_time": "2025-10-14 12:53:20.455000000", - "end_time": "2025-10-14 12:53:26.746000000", - "duration_ms": 6291.401, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: Tell me about recent developments in machine learning", - "query_length": 53, - "query_type": "web_search_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 6.290812253952026, - "model.latency_ms": 6290.812253952026, - "model.start_time": 1760446400.4555368, - "model.end_time": 1760446406.746349, - "model.performance_tier": "slow", - "model.prompt_tokens": 163, - "model.completion_tokens": 462, - "model.total_tokens": 625, - "model.tokens_per_second": 99.35124031199013, - "model.estimated_cost": 0.01875, - "model.efficiency_score": 2.9584, - "model.response_length": 1849, - "model.response_quality": "high", - "model.output_response": "Model Response: The web search did not yield specific results regarding recent developments in machine learning. However, I can provide a general overview based on my knowledge up to October 2023.\n\nRecent development...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "4bdd5814-acbb-4de7-a56b-543b87a1cfa5", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-14 12:53:20.120000000", - "trace_end_time": "2025-10-14 12:53:26.750000000", - "trace_duration_ms": 6630.38, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-14 12:53:20.120000000", - "updated_at": "2025-10-14 12:53:20.120000000" - }, - { - "span_id": "6e62af18-569c-4783-b2ae-177c58542bd0", - "trace_id": "4bdd5814-acbb-4de7-a56b-543b87a1cfa5", - "parent_span_id": "11c07411-df30-42a4-9062-90a032ef75ed", - "name": "agent.web_search_evaluation_metrics", - "start_time": "2025-10-14 12:53:26.749000000", - "end_time": "2025-10-14 12:53:26.750000000", - "duration_ms": 0.217, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "web_search_evaluation_metrics", - "agent.capabilities": [ - "web_search_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: Tell me about recent developments in machine learning", - "query_length": 53, - "query_type": "web_search_evaluation_query", - "web_search.results_count": 1, - "web_search.context_length": 155, - "web_search.context_synthesized": "Context from 1 web sources", - "web_search.search_effectiveness": 0.2, - "web_search.context_quality": "low", - "web_search.source_diversity": 1, - "web_search.information_synthesis": "low", - "web_search.external_knowledge_utilization": 0.155, - "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=Tell+me+about+recent+developments+in+machine+learning\n\nUser Question: Tell me about recent developments in machine learning\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", - "prompt.prompt_length": 653, - "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=Tell+me+about+recent+developments+in+machine+learning", - "prompt.instruction_following": "web_search_optimized", - "response.answer_length": 1849, - "response.answer_completeness": "complete", - "response.response_quality": "medium", - "response.source_citation": 1, - "response.context_utilization": 11.929032258064517, - "output_response": "Web Search Answer: The web search did not yield specific results regarding recent developments in machine learning. However, I can provide a general overview based on my knowledge up to October 2023.\n\nRecent development...", - "evaluation.search_effectiveness": 0.2, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 1, - "evaluation.context_relevance": "low", - "evaluation.overall_quality": "low", - "evaluation.ready_for_production": true, - "web_search.search_strategy": "duckduckgo_api", - "web_search.real_time_data": true, - "web_search.external_sources": 1, - "web_search.information_freshness": "current", - "web_search.knowledge_synthesis": "single_source" - }, - "events": [], - "links": [], - "trace_trace_id": "4bdd5814-acbb-4de7-a56b-543b87a1cfa5", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-14 12:53:20.120000000", - "trace_end_time": "2025-10-14 12:53:26.750000000", - "trace_duration_ms": 6630.38, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-14 12:53:20.120000000", - "updated_at": "2025-10-14 12:53:20.120000000" - } -] \ No newline at end of file diff --git a/noveum_customer_support_bt/traces/traces/dataset_filtered.json b/noveum_customer_support_bt/traces/traces/dataset_filtered.json deleted file mode 100644 index 5578d29..0000000 --- a/noveum_customer_support_bt/traces/traces/dataset_filtered.json +++ /dev/null @@ -1,20904 +0,0 @@ -[ - { - "span_id": "4149abdc-acdc-448c-b9b0-b09e166e461a", - "trace_id": "6fe93abf-1413-49e8-8464-58558e25ab7b", - "parent_span_id": "817c9703-0062-4bd6-b3ee-b50a1121c2d2", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-22 16:58:42.084000000", - "end_time": "2025-10-22 16:58:42.084000000", - "duration_ms": 0.015000000000000001, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What are the latest breakthroughs in artificial intelligence?", - "query_length": 61, - "query_type": "routing_evaluation_query", - "classification.mode": "Web Search", - "classification.rag_keyword_score": 0, - "classification.web_keyword_score": 1, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "complex", - "query.intent": "general_knowledge", - "query.keyword_density": 0.125, - "query.domain_affinity": "general", - "routing.decision": "Routed to Web Search based on analysis", - "routing.rationale": "RAG score: 0, Web score: 1, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "RAG", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.019230769230769232, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: Web Search (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "6fe93abf-1413-49e8-8464-58558e25ab7b", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:58:42.084000000", - "trace_end_time": "2025-10-22 16:58:44.766000000", - "trace_duration_ms": 2681.9289999999996, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:58:42.084000000", - "updated_at": "2025-10-22 16:58:42.084000000" - }, - { - "span_id": "817c9703-0062-4bd6-b3ee-b50a1121c2d2", - "trace_id": "6fe93abf-1413-49e8-8464-58558e25ab7b", - "parent_span_id": "ff171ffc-dc6b-4cb4-9833-64c5622d9941", - "name": "agent.query_routing", - "start_time": "2025-10-22 16:58:42.084000000", - "end_time": "2025-10-22 16:58:44.766000000", - "duration_ms": 2681.799, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What are the latest breakthroughs in artificial intelligence?", - "query_length": 61, - "query_type": "routing_query", - "output_response": "Routed to Web Search for query processing", - "router.classification": "Web Search", - "router.confidence_score": 1, - "router.rag_keyword_score": 0, - "router.web_keyword_score": 1, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "6fe93abf-1413-49e8-8464-58558e25ab7b", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:58:42.084000000", - "trace_end_time": "2025-10-22 16:58:44.766000000", - "trace_duration_ms": 2681.9289999999996, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:58:42.084000000", - "updated_at": "2025-10-22 16:58:42.084000000" - }, - { - "span_id": "ad476d62-d6b5-4142-8771-29855db3f39e", - "trace_id": "6fe93abf-1413-49e8-8464-58558e25ab7b", - "parent_span_id": "817c9703-0062-4bd6-b3ee-b50a1121c2d2", - "name": "agent.web_search_generation", - "start_time": "2025-10-22 16:58:42.084000000", - "end_time": "2025-10-22 16:58:44.766000000", - "duration_ms": 2681.732, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "web_search_agent", - "agent.operation": "web_search_generation", - "agent.capabilities": [ - "web_search", - "content_synthesis", - "response_generation" - ], - "agent.id": "noveum_web_search_agent", - "input_query": "What are the latest breakthroughs in artificial intelligence?", - "query_length": 61, - "query_type": "web_search_query", - "output_response": "Web Search Answer: The web search did not yield specific results regarding the latest breakthroughs in artificial intelligence. However, I can provide a general overview based on knowledge up to October 2023.\n\nRecent br...", - "web_search.results_count": 1, - "web_search.context_length": 163, - "web_search.response_length": 1040, - "web_search.mode": "external_web_search" - }, - "events": [], - "links": [], - "trace_trace_id": "6fe93abf-1413-49e8-8464-58558e25ab7b", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:58:42.084000000", - "trace_end_time": "2025-10-22 16:58:44.766000000", - "trace_duration_ms": 2681.9289999999996, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:58:42.084000000", - "updated_at": "2025-10-22 16:58:42.084000000" - }, - { - "span_id": "ff171ffc-dc6b-4cb4-9833-64c5622d9941", - "trace_id": "6fe93abf-1413-49e8-8464-58558e25ab7b", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-22 16:58:42.084000000", - "end_time": "2025-10-22 16:58:44.766000000", - "duration_ms": 2681.902, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What are the latest breakthroughs in artificial intelligence?", - "query.length": 61, - "processing.mode": "Web Search", - "processing.time_seconds": 2.6818161010742188, - "processing.response_length": 1040, - "processing.sources_count": 1, - "output_response": "Final Answer: The web search did not yield specific results regarding the latest breakthroughs in artificial intelligence. However, I can provide a general overview based on knowledge up to October 2023.\n\nRecent br...", - "final_answer_mode": "Web Search", - "query_processed.input_query": "What are the latest breakthroughs in artificial intelligence?", - "query_processed.output_response": "Successfully processed query using Web Search, generated 1040 character response", - "query_processed.mode": "Web Search", - "query_processed.processing_time": 2.6818161010742188, - "query_processed.response_length": 1040 - }, - "events": [], - "links": [], - "trace_trace_id": "6fe93abf-1413-49e8-8464-58558e25ab7b", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:58:42.084000000", - "trace_end_time": "2025-10-22 16:58:44.766000000", - "trace_duration_ms": 2681.9289999999996, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:58:42.084000000", - "updated_at": "2025-10-22 16:58:42.084000000" - }, - { - "span_id": "467cb806-a6ef-43da-9739-570730dd244b", - "trace_id": "6fe93abf-1413-49e8-8464-58558e25ab7b", - "parent_span_id": "ad476d62-d6b5-4142-8771-29855db3f39e", - "name": "agent.llm_model_execution", - "start_time": "2025-10-22 16:58:42.438000000", - "end_time": "2025-10-22 16:58:44.766000000", - "duration_ms": 2327.706, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What are the latest breakthroughs in artificial intelligence?", - "query_length": 61, - "query_type": "web_search_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 2.3276820182800293, - "model.latency_ms": 2327.6820182800293, - "model.start_time": 1761152322.438459, - "model.end_time": 1761152324.766141, - "model.performance_tier": "medium", - "model.prompt_tokens": 167, - "model.completion_tokens": 260, - "model.total_tokens": 427, - "model.tokens_per_second": 183.44430065903882, - "model.estimated_cost": 0.01281, - "model.efficiency_score": 2.4355971896955504, - "model.response_length": 1040, - "model.response_quality": "high", - "model.output_response": "Model Response: The web search did not yield specific results regarding the latest breakthroughs in artificial intelligence. However, I can provide a general overview based on knowledge up to October 2023.\n\nRecent br...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "6fe93abf-1413-49e8-8464-58558e25ab7b", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:58:42.084000000", - "trace_end_time": "2025-10-22 16:58:44.766000000", - "trace_duration_ms": 2681.9289999999996, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:58:42.084000000", - "updated_at": "2025-10-22 16:58:42.084000000" - }, - { - "span_id": "a80e4596-01ea-44e0-a590-13e347857f2a", - "trace_id": "6fe93abf-1413-49e8-8464-58558e25ab7b", - "parent_span_id": "ad476d62-d6b5-4142-8771-29855db3f39e", - "name": "agent.web_search_evaluation_metrics", - "start_time": "2025-10-22 16:58:44.766000000", - "end_time": "2025-10-22 16:58:44.766000000", - "duration_ms": 0.008, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "web_search_evaluation_metrics", - "agent.capabilities": [ - "web_search_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What are the latest breakthroughs in artificial intelligence?", - "query_length": 61, - "query_type": "web_search_evaluation_query", - "web_search.results_count": 1, - "web_search.context_length": 163, - "web_search.context_synthesized": "Context from 1 web sources", - "web_search.search_effectiveness": 0.2, - "web_search.context_quality": "low", - "web_search.source_diversity": 1, - "web_search.information_synthesis": "low", - "web_search.external_knowledge_utilization": 0.163, - "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What+are+the+latest+breakthroughs+in+artificial+intelligence?\n\nUser Question: What are the latest breakthroughs in artificial intelligence?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", - "prompt.prompt_length": 669, - "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What+are+the+latest+breakthroughs+in+artificial+intelligence?", - "prompt.instruction_following": "web_search_optimized", - "response.answer_length": 1040, - "response.answer_completeness": "complete", - "response.response_quality": "medium", - "response.source_citation": 1, - "response.context_utilization": 6.380368098159509, - "output_response": "Web Search Answer: The web search did not yield specific results regarding the latest breakthroughs in artificial intelligence. However, I can provide a general overview based on knowledge up to October 2023.\n\nRecent br...", - "evaluation.search_effectiveness": 0.2, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 1, - "evaluation.context_relevance": "low", - "evaluation.overall_quality": "low", - "evaluation.ready_for_production": true, - "web_search.search_strategy": "duckduckgo_api", - "web_search.real_time_data": true, - "web_search.external_sources": 1, - "web_search.information_freshness": "current", - "web_search.knowledge_synthesis": "single_source" - }, - "events": [], - "links": [], - "trace_trace_id": "6fe93abf-1413-49e8-8464-58558e25ab7b", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:58:42.084000000", - "trace_end_time": "2025-10-22 16:58:44.766000000", - "trace_duration_ms": 2681.9289999999996, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:58:42.084000000", - "updated_at": "2025-10-22 16:58:42.084000000" - }, - { - "span_id": "409f29f6-21c7-400a-a9e0-11c87c4313cb", - "trace_id": "ec7c6cae-9bbd-47f4-a8ae-2d6d9b5011ba", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-22 16:58:32.801000000", - "end_time": "2025-10-22 16:58:41.077000000", - "duration_ms": 8276.701, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What's happening in the software development world today?", - "query.length": 57, - "processing.mode": "Web Search", - "processing.time_seconds": 8.276506900787354, - "processing.response_length": 1424, - "processing.sources_count": 1, - "output_response": "Final Answer: The web search did not yield any specific results regarding current events in the software development world. Therefore, I cannot provide detailed information on the latest trends, technologies, or ne...", - "final_answer_mode": "Web Search", - "query_processed.input_query": "What's happening in the software development world today?", - "query_processed.output_response": "Successfully processed query using Web Search, generated 1424 character response", - "query_processed.mode": "Web Search", - "query_processed.processing_time": 8.276506900787354, - "query_processed.response_length": 1424 - }, - "events": [], - "links": [], - "trace_trace_id": "ec7c6cae-9bbd-47f4-a8ae-2d6d9b5011ba", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:58:32.800000000", - "trace_end_time": "2025-10-22 16:58:41.077000000", - "trace_duration_ms": 8276.788, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:58:32.800000000", - "updated_at": "2025-10-22 16:58:32.800000000" - }, - { - "span_id": "43389a18-a566-41e7-8db5-f688f651a106", - "trace_id": "ec7c6cae-9bbd-47f4-a8ae-2d6d9b5011ba", - "parent_span_id": "af400e17-8b20-4218-834b-9330ccaa2635", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-22 16:58:32.801000000", - "end_time": "2025-10-22 16:58:32.801000000", - "duration_ms": 0.047, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What's happening in the software development world today?", - "query_length": 57, - "query_type": "routing_evaluation_query", - "classification.mode": "Web Search", - "classification.rag_keyword_score": 0, - "classification.web_keyword_score": 1, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "complex", - "query.intent": "general_knowledge", - "query.keyword_density": 0.125, - "query.domain_affinity": "general", - "routing.decision": "Routed to Web Search based on analysis", - "routing.rationale": "RAG score: 0, Web score: 1, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "RAG", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.019230769230769232, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: Web Search (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "ec7c6cae-9bbd-47f4-a8ae-2d6d9b5011ba", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:58:32.800000000", - "trace_end_time": "2025-10-22 16:58:41.077000000", - "trace_duration_ms": 8276.788, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:58:32.800000000", - "updated_at": "2025-10-22 16:58:32.800000000" - }, - { - "span_id": "6b431bca-af7e-4f7e-aedb-b103d7f47cae", - "trace_id": "ec7c6cae-9bbd-47f4-a8ae-2d6d9b5011ba", - "parent_span_id": "af400e17-8b20-4218-834b-9330ccaa2635", - "name": "agent.web_search_generation", - "start_time": "2025-10-22 16:58:32.801000000", - "end_time": "2025-10-22 16:58:41.077000000", - "duration_ms": 8276.225999999999, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "web_search_agent", - "agent.operation": "web_search_generation", - "agent.capabilities": [ - "web_search", - "content_synthesis", - "response_generation" - ], - "agent.id": "noveum_web_search_agent", - "input_query": "What's happening in the software development world today?", - "query_length": 57, - "query_type": "web_search_query", - "output_response": "Web Search Answer: The web search did not yield any specific results regarding current events in the software development world. Therefore, I cannot provide detailed information on the latest trends, technologies, or ne...", - "web_search.results_count": 1, - "web_search.context_length": 159, - "web_search.response_length": 1424, - "web_search.mode": "external_web_search" - }, - "events": [], - "links": [], - "trace_trace_id": "ec7c6cae-9bbd-47f4-a8ae-2d6d9b5011ba", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:58:32.800000000", - "trace_end_time": "2025-10-22 16:58:41.077000000", - "trace_duration_ms": 8276.788, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:58:32.800000000", - "updated_at": "2025-10-22 16:58:32.800000000" - }, - { - "span_id": "af400e17-8b20-4218-834b-9330ccaa2635", - "trace_id": "ec7c6cae-9bbd-47f4-a8ae-2d6d9b5011ba", - "parent_span_id": "409f29f6-21c7-400a-a9e0-11c87c4313cb", - "name": "agent.query_routing", - "start_time": "2025-10-22 16:58:32.801000000", - "end_time": "2025-10-22 16:58:41.077000000", - "duration_ms": 8276.465999999999, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What's happening in the software development world today?", - "query_length": 57, - "query_type": "routing_query", - "output_response": "Routed to Web Search for query processing", - "router.classification": "Web Search", - "router.confidence_score": 1, - "router.rag_keyword_score": 0, - "router.web_keyword_score": 1, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "ec7c6cae-9bbd-47f4-a8ae-2d6d9b5011ba", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:58:32.800000000", - "trace_end_time": "2025-10-22 16:58:41.077000000", - "trace_duration_ms": 8276.788, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:58:32.800000000", - "updated_at": "2025-10-22 16:58:32.800000000" - }, - { - "span_id": "37b153b2-d02b-4b7e-9229-1ba3eaae44ee", - "trace_id": "ec7c6cae-9bbd-47f4-a8ae-2d6d9b5011ba", - "parent_span_id": "6b431bca-af7e-4f7e-aedb-b103d7f47cae", - "name": "agent.llm_model_execution", - "start_time": "2025-10-22 16:58:33.172000000", - "end_time": "2025-10-22 16:58:41.077000000", - "duration_ms": 7904.722, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What's happening in the software development world today?", - "query_length": 57, - "query_type": "web_search_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 7.904651880264282, - "model.latency_ms": 7904.651880264282, - "model.start_time": 1761152313.172655, - "model.end_time": 1761152321.077307, - "model.performance_tier": "slow", - "model.prompt_tokens": 165, - "model.completion_tokens": 356, - "model.total_tokens": 521, - "model.tokens_per_second": 65.91055594754174, - "model.estimated_cost": 0.01563, - "model.efficiency_score": 2.7332053742802302, - "model.response_length": 1424, - "model.response_quality": "high", - "model.output_response": "Model Response: The web search did not yield any specific results regarding current events in the software development world. Therefore, I cannot provide detailed information on the latest trends, technologies, or ne...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "ec7c6cae-9bbd-47f4-a8ae-2d6d9b5011ba", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:58:32.800000000", - "trace_end_time": "2025-10-22 16:58:41.077000000", - "trace_duration_ms": 8276.788, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:58:32.800000000", - "updated_at": "2025-10-22 16:58:32.800000000" - }, - { - "span_id": "cc941b8e-6625-43e9-a0a9-7328af24f12f", - "trace_id": "ec7c6cae-9bbd-47f4-a8ae-2d6d9b5011ba", - "parent_span_id": "6b431bca-af7e-4f7e-aedb-b103d7f47cae", - "name": "agent.web_search_evaluation_metrics", - "start_time": "2025-10-22 16:58:41.077000000", - "end_time": "2025-10-22 16:58:41.077000000", - "duration_ms": 0.026, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "web_search_evaluation_metrics", - "agent.capabilities": [ - "web_search_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What's happening in the software development world today?", - "query_length": 57, - "query_type": "web_search_evaluation_query", - "web_search.results_count": 1, - "web_search.context_length": 159, - "web_search.context_synthesized": "Context from 1 web sources", - "web_search.search_effectiveness": 0.2, - "web_search.context_quality": "low", - "web_search.source_diversity": 1, - "web_search.information_synthesis": "low", - "web_search.external_knowledge_utilization": 0.159, - "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What's+happening+in+the+software+development+world+today?\n\nUser Question: What's happening in the software development world today?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", - "prompt.prompt_length": 661, - "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What's+happening+in+the+software+development+world+today?", - "prompt.instruction_following": "web_search_optimized", - "response.answer_length": 1424, - "response.answer_completeness": "complete", - "response.response_quality": "medium", - "response.source_citation": 1, - "response.context_utilization": 8.955974842767295, - "output_response": "Web Search Answer: The web search did not yield any specific results regarding current events in the software development world. Therefore, I cannot provide detailed information on the latest trends, technologies, or ne...", - "evaluation.search_effectiveness": 0.2, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 1, - "evaluation.context_relevance": "low", - "evaluation.overall_quality": "low", - "evaluation.ready_for_production": true, - "web_search.search_strategy": "duckduckgo_api", - "web_search.real_time_data": true, - "web_search.external_sources": 1, - "web_search.information_freshness": "current", - "web_search.knowledge_synthesis": "single_source" - }, - "events": [], - "links": [], - "trace_trace_id": "ec7c6cae-9bbd-47f4-a8ae-2d6d9b5011ba", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:58:32.800000000", - "trace_end_time": "2025-10-22 16:58:41.077000000", - "trace_duration_ms": 8276.788, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:58:32.800000000", - "updated_at": "2025-10-22 16:58:32.800000000" - }, - { - "span_id": "3b06d098-6eb7-4bb6-885e-443ddb28cc41", - "trace_id": "343764b8-7ac9-4549-93cf-8abc11ceba6f", - "parent_span_id": "5c2aa1cf-56a8-4fd5-af90-90a94e723bb5", - "name": "agent.query_routing", - "start_time": "2025-10-22 16:58:26.542000000", - "end_time": "2025-10-22 16:58:31.796000000", - "duration_ms": 5253.835, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What are the newest features in cloud computing?", - "query_length": 48, - "query_type": "routing_query", - "output_response": "Routed to RAG for query processing", - "router.classification": "RAG", - "router.confidence_score": 1, - "router.rag_keyword_score": 1, - "router.web_keyword_score": 0, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "343764b8-7ac9-4549-93cf-8abc11ceba6f", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:58:26.542000000", - "trace_end_time": "2025-10-22 16:58:31.796000000", - "trace_duration_ms": 5253.973, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:58:26.542000000", - "updated_at": "2025-10-22 16:58:26.542000000" - }, - { - "span_id": "5c2aa1cf-56a8-4fd5-af90-90a94e723bb5", - "trace_id": "343764b8-7ac9-4549-93cf-8abc11ceba6f", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-22 16:58:26.542000000", - "end_time": "2025-10-22 16:58:31.796000000", - "duration_ms": 5253.913, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What are the newest features in cloud computing?", - "query.length": 48, - "processing.mode": "RAG", - "processing.time_seconds": 5.253868818283081, - "processing.response_length": 837, - "processing.sources_count": 5, - "output_response": "Final Answer: The provided context does not specifically mention any new features in cloud computing as a general topic. However, it does highlight several recent enhancements related to the Noveum.ai platform, whi...", - "final_answer_mode": "RAG", - "query_processed.input_query": "What are the newest features in cloud computing?", - "query_processed.output_response": "Successfully processed query using RAG, generated 837 character response", - "query_processed.mode": "RAG", - "query_processed.processing_time": 5.253868818283081, - "query_processed.response_length": 837 - }, - "events": [], - "links": [], - "trace_trace_id": "343764b8-7ac9-4549-93cf-8abc11ceba6f", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:58:26.542000000", - "trace_end_time": "2025-10-22 16:58:31.796000000", - "trace_duration_ms": 5253.973, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:58:26.542000000", - "updated_at": "2025-10-22 16:58:26.542000000" - }, - { - "span_id": "6b838ed6-f5bb-4bcf-a76b-bac283bf7571", - "trace_id": "343764b8-7ac9-4549-93cf-8abc11ceba6f", - "parent_span_id": "3b06d098-6eb7-4bb6-885e-443ddb28cc41", - "name": "agent.llm-rag", - "start_time": "2025-10-22 16:58:26.542000000", - "end_time": "2025-10-22 16:58:31.796000000", - "duration_ms": 5253.519, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "rag_agent", - "agent.operation": "llm-rag", - "agent.capabilities": [ - "document_retrieval", - "context_generation", - "response_generation" - ], - "agent.id": "noveum_rag_agent", - "input_query": "What are the newest features in cloud computing?", - "query_length": 48, - "query_type": "rag_query", - "output_response": "RAG Answer: The provided context does not specifically mention any new features in cloud computing as a general topic. However, it does highlight several recent enhancements related to the Noveum.ai platform, whi...", - "rag.context_length": 2579, - "rag.sources_count": 5, - "rag.answer_length": 837, - "rag.mode": "retrieval_augmented_generation" - }, - "events": [], - "links": [], - "trace_trace_id": "343764b8-7ac9-4549-93cf-8abc11ceba6f", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:58:26.542000000", - "trace_end_time": "2025-10-22 16:58:31.796000000", - "trace_duration_ms": 5253.973, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:58:26.542000000", - "updated_at": "2025-10-22 16:58:26.542000000" - }, - { - "span_id": "f51d8f1e-eae3-488d-9a96-49ab97377851", - "trace_id": "343764b8-7ac9-4549-93cf-8abc11ceba6f", - "parent_span_id": "3b06d098-6eb7-4bb6-885e-443ddb28cc41", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-22 16:58:26.542000000", - "end_time": "2025-10-22 16:58:26.542000000", - "duration_ms": 0.082, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What are the newest features in cloud computing?", - "query_length": 48, - "query_type": "routing_evaluation_query", - "classification.mode": "RAG", - "classification.rag_keyword_score": 1, - "classification.web_keyword_score": 0, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "documentation", - "query.keyword_density": 0.125, - "query.domain_affinity": "noveum", - "routing.decision": "Routed to RAG based on analysis", - "routing.rationale": "RAG score: 1, Web score: 0, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "Web Search", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.019230769230769232, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: RAG (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "343764b8-7ac9-4549-93cf-8abc11ceba6f", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:58:26.542000000", - "trace_end_time": "2025-10-22 16:58:31.796000000", - "trace_duration_ms": 5253.973, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:58:26.542000000", - "updated_at": "2025-10-22 16:58:26.542000000" - }, - { - "span_id": "93a0fd69-47b6-4e7c-b8ab-8eaa5aeac240", - "trace_id": "343764b8-7ac9-4549-93cf-8abc11ceba6f", - "parent_span_id": "6b838ed6-f5bb-4bcf-a76b-bac283bf7571", - "name": "agent.llm_model_execution", - "start_time": "2025-10-22 16:58:27.411000000", - "end_time": "2025-10-22 16:58:31.373000000", - "duration_ms": 3961.481, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What are the newest features in cloud computing?", - "query_length": 48, - "query_type": "rag_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 3.9613850116729736, - "model.latency_ms": 3961.3850116729736, - "model.start_time": 1761152307.411936, - "model.end_time": 1761152311.373321, - "model.performance_tier": "medium", - "model.prompt_tokens": 780, - "model.completion_tokens": 209, - "model.total_tokens": 989, - "model.tokens_per_second": 249.66015600243944, - "model.estimated_cost": 0.029670000000000002, - "model.efficiency_score": 0.846309403437816, - "model.response_length": 837, - "model.response_quality": "high", - "model.output_response": "Model Response: The provided context does not specifically mention any new features in cloud computing as a general topic. However, it does highlight several recent enhancements related to the Noveum.ai platform, whi...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "343764b8-7ac9-4549-93cf-8abc11ceba6f", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:58:26.542000000", - "trace_end_time": "2025-10-22 16:58:31.796000000", - "trace_duration_ms": 5253.973, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:58:26.542000000", - "updated_at": "2025-10-22 16:58:26.542000000" - }, - { - "span_id": "48cbc1df-f03a-4c49-90e5-19a2047c1c7b", - "trace_id": "343764b8-7ac9-4549-93cf-8abc11ceba6f", - "parent_span_id": "6b838ed6-f5bb-4bcf-a76b-bac283bf7571", - "name": "agent.rag_evaluation_metrics", - "start_time": "2025-10-22 16:58:31.373000000", - "end_time": "2025-10-22 16:58:31.373000000", - "duration_ms": 0.061, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "rag_evaluation_metrics", - "agent.capabilities": [ - "retrieval_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What are the newest features in cloud computing?", - "query_length": 48, - "query_type": "rag_evaluation_query", - "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/changelog):\n- \ud83d\udcc8 Enhanced Dashboard Analytics: Real-time request tracking, latency monitoring, and cost analysis\n- \ud83d\udd0d Improved Logs Interface: Better search functionality, detailed trace views, and enhanced debugging capabilities\n- \ud83d\udc65 Team Management: Advanced member mana...", - "retrieval.context_length": 2579, - "retrieval.sources_count": 5, - "retrieval.context_quality": "high", - "retrieval.effectiveness": 1, - "retrieval.context_utilization": 2.579, - "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/changelog):\n- \ud83d\udcc8 Enhanced Dashboard Analytics: Real-time request tracking, latency monitoring, and cost analysis\n- \ud83d\udd0d Improved Logs Interface: Better search functionality, detailed trace views, and enhanced debugging capabilities\n- \ud83d\udc65 Team Management: Advanced member management with role-based access control and invitation system\n- \ud83d\udee1\ufe0f Security Improvements: Enhanced API key management and secure credential storage\n4 months ago\n- \ud83d\udcca Advanced Metrics Collection: Comprehensive telemetry system with custom metrics ...\n\nSource 2 (https://noveum.ai/en/changelog):\n- \ud83d\udcdd Documentation Updates: Comprehensive guides for integration and best practices\n5 months ago\n- \ud83c\udf89 Platform Foundation: Initial release of Noveum.ai AI observability platform\n- \ud83d\ude80 Core Features: AI Gateway integration, basic evaluation metrics, and monitoring capabilities\n- \ud83d\udcc8 Evaluation Jobs: Automated model evaluation with configurable metrics and reporting\n- \ud83d\udd0c Provider Integrations: Support for major AI providers and custom model deployments...\n\nSource 3 (https://noveum.ai/docs/platform/dashboard):\n- Bulk Export: Download multiple traces simultaneously\n- Comparative Analysis: Compare performance across multiple traces\nReal-Time Updates\n- Live Refresh: Automatic updates as new traces arrive\n- Connection Monitoring: Real-time status of your trace ingestion pipeline\n- Performance Indicators: Live metrics for system health monitoring\nReady to dive deeper? Explore Projects & Environments to organize your AI applications, or check out Team Collaboration to share insights with your team.\nGet Earl...\n\nSource 4 (https://noveum.ai/en/changelog):\nChangelog\nStay up to date with the latest changes in our product.\n2 months ago\n- \ud83c\udfaf Major Platform Evolution: NovaEval Framework - Launched comprehensive AI model evaluation framework with 20+ built-in scorers including accuracy, RAG metrics, conversational scoring, and specialized agent evaluation tools. Features production-grade deployment on Docker and Kubernetes. (GitHub | PyPI Package)\n- \ud83d\ude80 Next-Generation Traces UI - Complete redesign featuring three-pane layout with directory tree navigatio...\n\nSource 5 (https://noveum.ai/en/blog):\n8/12/2025\nDiscover how Noveum.ai provides comprehensive tracing and observability for AI applications, from development debugging to production optimization.\nShashank Agarwal\n3/3/2025\nDiscover how Noveum.ai provides comprehensive tracing and observability for LLM applications, RAG systems, and multi-agent workflows with our powerful Python and TypeScript SDKs.\nShashank Agarwal\n3/2/2025...\n\nUser Question: What are the newest features in cloud computing?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", - "prompt.prompt_length": 3123, - "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/changelog):\n- \ud83d\udcc8 Enhanced Dashboard Analytics: Real-time request tracking, latency monitoring, and cost analysis\n- \ud83d\udd0d Improved Logs Interface: Better search functionality,...", - "prompt.instruction_following": "rag_optimized", - "response.answer_length": 837, - "response.answer_completeness": "complete", - "response.response_quality": "high", - "response.source_citation": 5, - "response.context_utilization": 0.3245443970531214, - "output_response": "RAG Answer: The provided context does not specifically mention any new features in cloud computing as a general topic. However, it does highlight several recent enhancements related to the Noveum.ai platform, whi...", - "evaluation.retrieval_effectiveness": 1, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 5, - "evaluation.context_relevance": "high", - "evaluation.overall_quality": "high", - "evaluation.ready_for_production": true, - "rag.retrieval_strategy": "semantic_similarity", - "rag.vector_search_results": 5, - "rag.context_synthesis": "multi_source", - "rag.document_coverage": 1, - "rag.information_density": 0.3245443970531214 - }, - "events": [], - "links": [], - "trace_trace_id": "343764b8-7ac9-4549-93cf-8abc11ceba6f", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:58:26.542000000", - "trace_end_time": "2025-10-22 16:58:31.796000000", - "trace_duration_ms": 5253.973, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:58:26.542000000", - "updated_at": "2025-10-22 16:58:26.542000000" - }, - { - "span_id": "7384d697-d7bf-4709-8377-ec26b094b550", - "trace_id": "3f7b6a62-8888-4a5d-b12c-37a3829aea49", - "parent_span_id": "e631b22a-a621-4ecc-bb01-f731ca66f3d0", - "name": "agent.query_routing", - "start_time": "2025-10-22 16:58:19.885000000", - "end_time": "2025-10-22 16:58:25.536000000", - "duration_ms": 5651.335999999999, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What's the current status of cryptocurrency markets?", - "query_length": 52, - "query_type": "routing_query", - "output_response": "Routed to Web Search for query processing", - "router.classification": "Web Search", - "router.confidence_score": 1, - "router.rag_keyword_score": 0, - "router.web_keyword_score": 3, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "3f7b6a62-8888-4a5d-b12c-37a3829aea49", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:58:19.885000000", - "trace_end_time": "2025-10-22 16:58:25.536000000", - "trace_duration_ms": 5651.67, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:58:19.885000000", - "updated_at": "2025-10-22 16:58:19.885000000" - }, - { - "span_id": "7abf372d-d9f4-41d1-9e3a-d4f01173fa5e", - "trace_id": "3f7b6a62-8888-4a5d-b12c-37a3829aea49", - "parent_span_id": "7384d697-d7bf-4709-8377-ec26b094b550", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-22 16:58:19.885000000", - "end_time": "2025-10-22 16:58:19.885000000", - "duration_ms": 0.037, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What's the current status of cryptocurrency markets?", - "query_length": 52, - "query_type": "routing_evaluation_query", - "classification.mode": "Web Search", - "classification.rag_keyword_score": 0, - "classification.web_keyword_score": 3, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "complex", - "query.intent": "general_knowledge", - "query.keyword_density": 0.42857142857142855, - "query.domain_affinity": "general", - "routing.decision": "Routed to Web Search based on analysis", - "routing.rationale": "RAG score: 0, Web score: 3, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "RAG", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.057692307692307696, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: Web Search (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "3f7b6a62-8888-4a5d-b12c-37a3829aea49", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:58:19.885000000", - "trace_end_time": "2025-10-22 16:58:25.536000000", - "trace_duration_ms": 5651.67, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:58:19.885000000", - "updated_at": "2025-10-22 16:58:19.885000000" - }, - { - "span_id": "e5d9c3c3-6c18-45d2-a6b6-58bde4d4e461", - "trace_id": "3f7b6a62-8888-4a5d-b12c-37a3829aea49", - "parent_span_id": "7384d697-d7bf-4709-8377-ec26b094b550", - "name": "agent.web_search_generation", - "start_time": "2025-10-22 16:58:19.885000000", - "end_time": "2025-10-22 16:58:25.536000000", - "duration_ms": 5651.161, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "web_search_agent", - "agent.operation": "web_search_generation", - "agent.capabilities": [ - "web_search", - "content_synthesis", - "response_generation" - ], - "agent.id": "noveum_web_search_agent", - "input_query": "What's the current status of cryptocurrency markets?", - "query_length": 52, - "query_type": "web_search_query", - "output_response": "Web Search Answer: The provided web search results do not contain specific information regarding the current status of cryptocurrency markets. To get the latest updates on cryptocurrency prices, trends, and market analy...", - "web_search.results_count": 1, - "web_search.context_length": 251, - "web_search.response_length": 416, - "web_search.mode": "external_web_search" - }, - "events": [], - "links": [], - "trace_trace_id": "3f7b6a62-8888-4a5d-b12c-37a3829aea49", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:58:19.885000000", - "trace_end_time": "2025-10-22 16:58:25.536000000", - "trace_duration_ms": 5651.67, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:58:19.885000000", - "updated_at": "2025-10-22 16:58:19.885000000" - }, - { - "span_id": "e631b22a-a621-4ecc-bb01-f731ca66f3d0", - "trace_id": "3f7b6a62-8888-4a5d-b12c-37a3829aea49", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-22 16:58:19.885000000", - "end_time": "2025-10-22 16:58:25.536000000", - "duration_ms": 5651.588000000001, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What's the current status of cryptocurrency markets?", - "query.length": 52, - "processing.mode": "Web Search", - "processing.time_seconds": 5.651373863220215, - "processing.response_length": 416, - "processing.sources_count": 1, - "output_response": "Final Answer: The provided web search results do not contain specific information regarding the current status of cryptocurrency markets. To get the latest updates on cryptocurrency prices, trends, and market analy...", - "final_answer_mode": "Web Search", - "query_processed.input_query": "What's the current status of cryptocurrency markets?", - "query_processed.output_response": "Successfully processed query using Web Search, generated 416 character response", - "query_processed.mode": "Web Search", - "query_processed.processing_time": 5.651373863220215, - "query_processed.response_length": 416 - }, - "events": [], - "links": [], - "trace_trace_id": "3f7b6a62-8888-4a5d-b12c-37a3829aea49", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:58:19.885000000", - "trace_end_time": "2025-10-22 16:58:25.536000000", - "trace_duration_ms": 5651.67, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:58:19.885000000", - "updated_at": "2025-10-22 16:58:19.885000000" - }, - { - "span_id": "34e2f906-724e-4116-b61f-82eaa6b75959", - "trace_id": "3f7b6a62-8888-4a5d-b12c-37a3829aea49", - "parent_span_id": "e5d9c3c3-6c18-45d2-a6b6-58bde4d4e461", - "name": "agent.llm_model_execution", - "start_time": "2025-10-22 16:58:24.327000000", - "end_time": "2025-10-22 16:58:25.536000000", - "duration_ms": 1208.66, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What's the current status of cryptocurrency markets?", - "query_length": 52, - "query_type": "web_search_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 1.208608865737915, - "model.latency_ms": 1208.608865737915, - "model.start_time": 1761152304.327749, - "model.end_time": 1761152305.5363579, - "model.performance_tier": "fast", - "model.prompt_tokens": 187, - "model.completion_tokens": 104, - "model.total_tokens": 291, - "model.tokens_per_second": 240.77268357809888, - "model.estimated_cost": 0.00873, - "model.efficiency_score": 1.429553264604811, - "model.response_length": 416, - "model.response_quality": "high", - "model.output_response": "Model Response: The provided web search results do not contain specific information regarding the current status of cryptocurrency markets. To get the latest updates on cryptocurrency prices, trends, and market analy...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "3f7b6a62-8888-4a5d-b12c-37a3829aea49", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:58:19.885000000", - "trace_end_time": "2025-10-22 16:58:25.536000000", - "trace_duration_ms": 5651.67, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:58:19.885000000", - "updated_at": "2025-10-22 16:58:19.885000000" - }, - { - "span_id": "5548e02e-2ea8-472c-9c42-f811d6e4d30b", - "trace_id": "3f7b6a62-8888-4a5d-b12c-37a3829aea49", - "parent_span_id": "e5d9c3c3-6c18-45d2-a6b6-58bde4d4e461", - "name": "agent.web_search_evaluation_metrics", - "start_time": "2025-10-22 16:58:25.536000000", - "end_time": "2025-10-22 16:58:25.536000000", - "duration_ms": 0.044, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "web_search_evaluation_metrics", - "agent.capabilities": [ - "web_search_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What's the current status of cryptocurrency markets?", - "query_length": 52, - "query_type": "web_search_evaluation_query", - "web_search.results_count": 1, - "web_search.context_length": 251, - "web_search.context_synthesized": "Context from 1 web sources", - "web_search.search_effectiveness": 0.2, - "web_search.context_quality": "low", - "web_search.source_diversity": 1, - "web_search.information_synthesis": "low", - "web_search.external_knowledge_utilization": 0.251, - "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nLog in to WhatsApp Web for simple, reliable and private messaging on your desktop. Send and receive messages and files with ease, all for \u2026\nURL: https://duckduckgo.com/?q=What's+the+current+status+of+cryptocurrency+markets?\n\nUser Question: What's the current status of cryptocurrency markets?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", - "prompt.prompt_length": 748, - "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nLog in to WhatsApp Web for simple, reliable and private messaging on your desktop. Send and receive messages and files with ease, all for \u2026\nURL: https://duckduckgo.com/?q=W...", - "prompt.instruction_following": "web_search_optimized", - "response.answer_length": 416, - "response.answer_completeness": "complete", - "response.response_quality": "medium", - "response.source_citation": 1, - "response.context_utilization": 1.657370517928287, - "output_response": "Web Search Answer: The provided web search results do not contain specific information regarding the current status of cryptocurrency markets. To get the latest updates on cryptocurrency prices, trends, and market analy...", - "evaluation.search_effectiveness": 0.2, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 1, - "evaluation.context_relevance": "low", - "evaluation.overall_quality": "low", - "evaluation.ready_for_production": true, - "web_search.search_strategy": "duckduckgo_api", - "web_search.real_time_data": true, - "web_search.external_sources": 1, - "web_search.information_freshness": "current", - "web_search.knowledge_synthesis": "single_source" - }, - "events": [], - "links": [], - "trace_trace_id": "3f7b6a62-8888-4a5d-b12c-37a3829aea49", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:58:19.885000000", - "trace_end_time": "2025-10-22 16:58:25.536000000", - "trace_duration_ms": 5651.67, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:58:19.885000000", - "updated_at": "2025-10-22 16:58:19.885000000" - }, - { - "span_id": "a9e34cea-d4bd-478b-abf2-7a3a38dcd40e", - "trace_id": "51f4534f-23e4-4be7-8d6e-e93391d6e0e2", - "parent_span_id": "8e9b2d06-3e11-4d17-9a32-b1529a2c10a4", - "name": "agent.web_search_generation", - "start_time": "2025-10-22 16:58:15.254000000", - "end_time": "2025-10-22 16:58:18.876000000", - "duration_ms": 3621.674, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "web_search_agent", - "agent.operation": "web_search_generation", - "agent.capabilities": [ - "web_search", - "content_synthesis", - "response_generation" - ], - "agent.id": "noveum_web_search_agent", - "input_query": "What are the latest updates in Python programming?", - "query_length": 50, - "query_type": "web_search_query", - "output_response": "Web Search Answer: The provided web search results do not contain specific information about the latest updates in Python programming. They primarily focus on general news coverage and updates from India and around the ...", - "web_search.results_count": 1, - "web_search.context_length": 753, - "web_search.response_length": 580, - "web_search.mode": "external_web_search" - }, - "events": [], - "links": [], - "trace_trace_id": "51f4534f-23e4-4be7-8d6e-e93391d6e0e2", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:58:15.254000000", - "trace_end_time": "2025-10-22 16:58:18.877000000", - "trace_duration_ms": 3622.355, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:58:15.254000000", - "updated_at": "2025-10-22 16:58:15.254000000" - }, - { - "span_id": "8e9b2d06-3e11-4d17-9a32-b1529a2c10a4", - "trace_id": "51f4534f-23e4-4be7-8d6e-e93391d6e0e2", - "parent_span_id": "bda936a0-24f3-4841-af4f-227197e10b9d", - "name": "agent.query_routing", - "start_time": "2025-10-22 16:58:15.254000000", - "end_time": "2025-10-22 16:58:18.876000000", - "duration_ms": 3621.863, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What are the latest updates in Python programming?", - "query_length": 50, - "query_type": "routing_query", - "output_response": "Routed to Web Search for query processing", - "router.classification": "Web Search", - "router.confidence_score": 1, - "router.rag_keyword_score": 0, - "router.web_keyword_score": 2, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "51f4534f-23e4-4be7-8d6e-e93391d6e0e2", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:58:15.254000000", - "trace_end_time": "2025-10-22 16:58:18.877000000", - "trace_duration_ms": 3622.355, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:58:15.254000000", - "updated_at": "2025-10-22 16:58:15.254000000" - }, - { - "span_id": "bda936a0-24f3-4841-af4f-227197e10b9d", - "trace_id": "51f4534f-23e4-4be7-8d6e-e93391d6e0e2", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-22 16:58:15.254000000", - "end_time": "2025-10-22 16:58:18.877000000", - "duration_ms": 3622.271, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What are the latest updates in Python programming?", - "query.length": 50, - "processing.mode": "Web Search", - "processing.time_seconds": 3.6219100952148438, - "processing.response_length": 580, - "processing.sources_count": 1, - "output_response": "Final Answer: The provided web search results do not contain specific information about the latest updates in Python programming. They primarily focus on general news coverage and updates from India and around the ...", - "final_answer_mode": "Web Search", - "query_processed.input_query": "What are the latest updates in Python programming?", - "query_processed.output_response": "Successfully processed query using Web Search, generated 580 character response", - "query_processed.mode": "Web Search", - "query_processed.processing_time": 3.6219100952148438, - "query_processed.response_length": 580 - }, - "events": [], - "links": [], - "trace_trace_id": "51f4534f-23e4-4be7-8d6e-e93391d6e0e2", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:58:15.254000000", - "trace_end_time": "2025-10-22 16:58:18.877000000", - "trace_duration_ms": 3622.355, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:58:15.254000000", - "updated_at": "2025-10-22 16:58:15.254000000" - }, - { - "span_id": "e27e3812-d49d-4f4f-98b0-144cf0ceffc3", - "trace_id": "51f4534f-23e4-4be7-8d6e-e93391d6e0e2", - "parent_span_id": "8e9b2d06-3e11-4d17-9a32-b1529a2c10a4", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-22 16:58:15.254000000", - "end_time": "2025-10-22 16:58:15.254000000", - "duration_ms": 0.047, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What are the latest updates in Python programming?", - "query_length": 50, - "query_type": "routing_evaluation_query", - "classification.mode": "Web Search", - "classification.rag_keyword_score": 0, - "classification.web_keyword_score": 2, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "general_knowledge", - "query.keyword_density": 0.25, - "query.domain_affinity": "general", - "routing.decision": "Routed to Web Search based on analysis", - "routing.rationale": "RAG score: 0, Web score: 2, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "RAG", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.038461538461538464, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: Web Search (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "51f4534f-23e4-4be7-8d6e-e93391d6e0e2", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:58:15.254000000", - "trace_end_time": "2025-10-22 16:58:18.877000000", - "trace_duration_ms": 3622.355, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:58:15.254000000", - "updated_at": "2025-10-22 16:58:15.254000000" - }, - { - "span_id": "b0eea8ec-5cff-4667-bea7-aeb6d0f0daf3", - "trace_id": "51f4534f-23e4-4be7-8d6e-e93391d6e0e2", - "parent_span_id": "a9e34cea-d4bd-478b-abf2-7a3a38dcd40e", - "name": "agent.llm_model_execution", - "start_time": "2025-10-22 16:58:15.568000000", - "end_time": "2025-10-22 16:58:18.876000000", - "duration_ms": 3307.5739999999996, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What are the latest updates in Python programming?", - "query_length": 50, - "query_type": "web_search_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 3.307521104812622, - "model.latency_ms": 3307.521104812622, - "model.start_time": 1761152295.568949, - "model.end_time": 1761152298.87647, - "model.performance_tier": "medium", - "model.prompt_tokens": 312, - "model.completion_tokens": 145, - "model.total_tokens": 457, - "model.tokens_per_second": 138.16994223711538, - "model.estimated_cost": 0.01371, - "model.efficiency_score": 1.2691466083150984, - "model.response_length": 580, - "model.response_quality": "high", - "model.output_response": "Model Response: The provided web search results do not contain specific information about the latest updates in Python programming. They primarily focus on general news coverage and updates from India and around the ...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "51f4534f-23e4-4be7-8d6e-e93391d6e0e2", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:58:15.254000000", - "trace_end_time": "2025-10-22 16:58:18.877000000", - "trace_duration_ms": 3622.355, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:58:15.254000000", - "updated_at": "2025-10-22 16:58:15.254000000" - }, - { - "span_id": "d0bfcbe5-cb6c-4837-bdf3-f2c65be145cd", - "trace_id": "51f4534f-23e4-4be7-8d6e-e93391d6e0e2", - "parent_span_id": "a9e34cea-d4bd-478b-abf2-7a3a38dcd40e", - "name": "agent.web_search_evaluation_metrics", - "start_time": "2025-10-22 16:58:18.876000000", - "end_time": "2025-10-22 16:58:18.876000000", - "duration_ms": 0.04, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "web_search_evaluation_metrics", - "agent.capabilities": [ - "web_search_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What are the latest updates in Python programming?", - "query_length": 50, - "query_type": "web_search_evaluation_query", - "web_search.results_count": 1, - "web_search.context_length": 753, - "web_search.context_synthesized": "Context from 1 web sources", - "web_search.search_effectiveness": 0.2, - "web_search.context_quality": "medium", - "web_search.source_diversity": 1, - "web_search.information_synthesis": "low", - "web_search.external_knowledge_utilization": 0.753, - "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\n3 days ago \u00b7 Latest news headlines from India & around the world. Check out today\u2019s news coverage live with videos & photos on \u2026 6 days ago \u00b7 India News | Latest India News | Read latest and breaking news from India. Today's top India news headlines, news on \u2026 2 days ago \u00b7 Top News Stories of the day, Latest News Headlines, News Specials, Breaking News and Latest India News, World current affairs & \u2026 NDTV.com provides latest news from India and around the world. Get breaking news alerts from India and follow today\u2019s live news updates in \u2026 6 days ago \u00b7 Get Latest entertainment news from Bollywood, Hollywood and regional film industries.\nURL: https://duckduckgo.com/?q=What+are+the+latest+updates+in+Python+programming?\n\nUser Question: What are the latest updates in Python programming?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", - "prompt.prompt_length": 1248, - "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\n3 days ago \u00b7 Latest news headlines from India & around the world. Check out today\u2019s news coverage live with videos & photos on \u2026 6 days ago \u00b7 India News | Latest India News...", - "prompt.instruction_following": "web_search_optimized", - "response.answer_length": 580, - "response.answer_completeness": "complete", - "response.response_quality": "medium", - "response.source_citation": 1, - "response.context_utilization": 0.7702523240371846, - "output_response": "Web Search Answer: The provided web search results do not contain specific information about the latest updates in Python programming. They primarily focus on general news coverage and updates from India and around the ...", - "evaluation.search_effectiveness": 0.2, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 1, - "evaluation.context_relevance": "medium", - "evaluation.overall_quality": "low", - "evaluation.ready_for_production": true, - "web_search.search_strategy": "duckduckgo_api", - "web_search.real_time_data": true, - "web_search.external_sources": 1, - "web_search.information_freshness": "current", - "web_search.knowledge_synthesis": "single_source" - }, - "events": [], - "links": [], - "trace_trace_id": "51f4534f-23e4-4be7-8d6e-e93391d6e0e2", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:58:15.254000000", - "trace_end_time": "2025-10-22 16:58:18.877000000", - "trace_duration_ms": 3622.355, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:58:15.254000000", - "updated_at": "2025-10-22 16:58:15.254000000" - }, - { - "span_id": "5f5916a7-b9ea-4348-861d-db69b25d6b62", - "trace_id": "e30e9ba6-ffd1-4eb9-b7ae-6d19e04343d6", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-22 16:58:11.033000000", - "end_time": "2025-10-22 16:58:14.250000000", - "duration_ms": 3216.77, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What happened in tech news this week?", - "query.length": 37, - "processing.mode": "Web Search", - "processing.time_seconds": 3.2166178226470947, - "processing.response_length": 510, - "processing.sources_count": 1, - "output_response": "Final Answer: The web search results do not provide specific information about recent events in tech news for this week. They primarily discuss the grammatical usage of the word \"happened\" and its variations, such ...", - "final_answer_mode": "Web Search", - "query_processed.input_query": "What happened in tech news this week?", - "query_processed.output_response": "Successfully processed query using Web Search, generated 510 character response", - "query_processed.mode": "Web Search", - "query_processed.processing_time": 3.2166178226470947, - "query_processed.response_length": 510 - }, - "events": [], - "links": [], - "trace_trace_id": "e30e9ba6-ffd1-4eb9-b7ae-6d19e04343d6", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:58:11.033000000", - "trace_end_time": "2025-10-22 16:58:14.250000000", - "trace_duration_ms": 3216.801, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:58:11.033000000", - "updated_at": "2025-10-22 16:58:11.033000000" - }, - { - "span_id": "804613d0-a701-4c22-add1-d315d7a29813", - "trace_id": "e30e9ba6-ffd1-4eb9-b7ae-6d19e04343d6", - "parent_span_id": "5f5916a7-b9ea-4348-861d-db69b25d6b62", - "name": "agent.query_routing", - "start_time": "2025-10-22 16:58:11.033000000", - "end_time": "2025-10-22 16:58:14.250000000", - "duration_ms": 3216.594, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What happened in tech news this week?", - "query_length": 37, - "query_type": "routing_query", - "output_response": "Routed to Web Search for query processing", - "router.classification": "Web Search", - "router.confidence_score": 1, - "router.rag_keyword_score": 0, - "router.web_keyword_score": 2, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "e30e9ba6-ffd1-4eb9-b7ae-6d19e04343d6", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:58:11.033000000", - "trace_end_time": "2025-10-22 16:58:14.250000000", - "trace_duration_ms": 3216.801, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:58:11.033000000", - "updated_at": "2025-10-22 16:58:11.033000000" - }, - { - "span_id": "f6c8d34f-185f-49ba-a5a4-a9d25ccb4fc6", - "trace_id": "e30e9ba6-ffd1-4eb9-b7ae-6d19e04343d6", - "parent_span_id": "804613d0-a701-4c22-add1-d315d7a29813", - "name": "agent.web_search_generation", - "start_time": "2025-10-22 16:58:11.034000000", - "end_time": "2025-10-22 16:58:14.250000000", - "duration_ms": 3216.4959999999996, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "web_search_agent", - "agent.operation": "web_search_generation", - "agent.capabilities": [ - "web_search", - "content_synthesis", - "response_generation" - ], - "agent.id": "noveum_web_search_agent", - "input_query": "What happened in tech news this week?", - "query_length": 37, - "query_type": "web_search_query", - "output_response": "Web Search Answer: The web search results do not provide specific information about recent events in tech news for this week. They primarily discuss the grammatical usage of the word \"happened\" and its variations, such ...", - "web_search.results_count": 1, - "web_search.context_length": 724, - "web_search.response_length": 510, - "web_search.mode": "external_web_search" - }, - "events": [], - "links": [], - "trace_trace_id": "e30e9ba6-ffd1-4eb9-b7ae-6d19e04343d6", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:58:11.033000000", - "trace_end_time": "2025-10-22 16:58:14.250000000", - "trace_duration_ms": 3216.801, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:58:11.033000000", - "updated_at": "2025-10-22 16:58:11.033000000" - }, - { - "span_id": "11d68082-e7b8-4bbf-833a-79e296c31eda", - "trace_id": "e30e9ba6-ffd1-4eb9-b7ae-6d19e04343d6", - "parent_span_id": "804613d0-a701-4c22-add1-d315d7a29813", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-22 16:58:11.034000000", - "end_time": "2025-10-22 16:58:11.034000000", - "duration_ms": 0.017, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What happened in tech news this week?", - "query_length": 37, - "query_type": "routing_evaluation_query", - "classification.mode": "Web Search", - "classification.rag_keyword_score": 0, - "classification.web_keyword_score": 2, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "general_knowledge", - "query.keyword_density": 0.2857142857142857, - "query.domain_affinity": "general", - "routing.decision": "Routed to Web Search based on analysis", - "routing.rationale": "RAG score: 0, Web score: 2, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "RAG", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.038461538461538464, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: Web Search (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "e30e9ba6-ffd1-4eb9-b7ae-6d19e04343d6", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:58:11.033000000", - "trace_end_time": "2025-10-22 16:58:14.250000000", - "trace_duration_ms": 3216.801, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:58:11.033000000", - "updated_at": "2025-10-22 16:58:11.033000000" - }, - { - "span_id": "420b4ca1-c9f1-488f-ac2c-c53766717634", - "trace_id": "e30e9ba6-ffd1-4eb9-b7ae-6d19e04343d6", - "parent_span_id": "f6c8d34f-185f-49ba-a5a4-a9d25ccb4fc6", - "name": "agent.llm_model_execution", - "start_time": "2025-10-22 16:58:11.468000000", - "end_time": "2025-10-22 16:58:14.250000000", - "duration_ms": 2781.777, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What happened in tech news this week?", - "query_length": 37, - "query_type": "web_search_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 2.781723976135254, - "model.latency_ms": 2781.723976135254, - "model.start_time": 1761152291.468626, - "model.end_time": 1761152294.25035, - "model.performance_tier": "medium", - "model.prompt_tokens": 301, - "model.completion_tokens": 127, - "model.total_tokens": 428, - "model.tokens_per_second": 153.8614196346811, - "model.estimated_cost": 0.01284, - "model.efficiency_score": 1.191588785046729, - "model.response_length": 510, - "model.response_quality": "high", - "model.output_response": "Model Response: The web search results do not provide specific information about recent events in tech news for this week. They primarily discuss the grammatical usage of the word \"happened\" and its variations, such ...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "e30e9ba6-ffd1-4eb9-b7ae-6d19e04343d6", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:58:11.033000000", - "trace_end_time": "2025-10-22 16:58:14.250000000", - "trace_duration_ms": 3216.801, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:58:11.033000000", - "updated_at": "2025-10-22 16:58:11.033000000" - }, - { - "span_id": "56b8bf09-8ebf-44e1-b01f-daff23d3c42c", - "trace_id": "e30e9ba6-ffd1-4eb9-b7ae-6d19e04343d6", - "parent_span_id": "f6c8d34f-185f-49ba-a5a4-a9d25ccb4fc6", - "name": "agent.web_search_evaluation_metrics", - "start_time": "2025-10-22 16:58:14.250000000", - "end_time": "2025-10-22 16:58:14.250000000", - "duration_ms": 0.060000000000000005, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "web_search_evaluation_metrics", - "agent.capabilities": [ - "web_search_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What happened in tech news this week?", - "query_length": 37, - "query_type": "web_search_evaluation_query", - "web_search.results_count": 1, - "web_search.context_length": 724, - "web_search.context_synthesized": "Context from 1 web sources", - "web_search.search_effectiveness": 0.2, - "web_search.context_quality": "medium", - "web_search.source_diversity": 1, - "web_search.information_synthesis": "low", - "web_search.external_knowledge_utilization": 0.724, - "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nhappened\u7684\u7528\u6cd5\u5982\u4e0b\uff1a 1.happened\u662f \u4e0d\u53ca\u7269\u52a8\u8bcd,\u4f5c\u201c\u53d1\u751f\u201d\u89e3\u65f6,\u53ef\u6307\u5076\u7136\u7684\u4e8b\u6545,\u4e5f\u53ef\u6307\u6709\u8ba1\u5212\u6216\u65e0\u8ba1\u5212\u53d1\u751f\u7684\u4e8b\u3002\u4e00\u822c\u4ee5\u4e8b\u4ef6\u3001\u73af\u5883\u3001\u5f62\u52bf\u7b49\u4f5c\u4e3b\u8bed,\u8868\u793a\u201c\u67d0\u4eba\u53d1\u751f\u67d0\u4e8b\u201d\u65f6,\u987b\u4ee5\u4ecb\u8bcdto\u5f15\u51fa\u201c\u67d0\u4eba\u201d\u3002 2. \u2026 what happened\u8fd8\u662fwhat's happened\"What happened\"\u548c\"What's happened\"\u90fd\u7528\u4e8e\u8be2\u95ee\u8fc7\u53bb\u53d1\u751f\u4e86\u4ec0\u4e48\u4e8b\u60c5\uff0c\u4e3b\u8981\u7684\u533a\u522b\u5728\u4e8e\u8bed\u6cd5\u548c\u4f7f\u7528\u65b9\u5f0f\u3002\"What happened\"\u662f\u4e00\u4e2a\u8fc7\u53bb\u65f6\u6001\u7684\u7b80\u5355 \u63a5\u4e0b\u6765\u8ba9\u6211\u4eec\u770b\u4e0bwhat happen\u548cwhat happened\u7684\u7528\u6cd5\u533a\u522b\uff1a 1.\u8bcd\u6027\uff1awhat happen\u662f\u7528\u539f\u5f62\u52a8\u8bcdhappen\u6784\u6210\u7684\u73b0\u5728\u65f6\u77ed\u8bed\uff0c\u800cwhat happened\u662f\u7528\u52a8\u8bcdhappen\u7684\u8fc7\u53bb\u5f0fhappened\u6784\u6210\u7684\u8fc7\u53bb\u65f6\u77ed \u2026 What happened \u548c What's happened\u4e24\u8005\u7684\u533a\u522b\uff1a \u662f\u95ee What happened\uff1f\u53ea\u662f\u60f3\u77e5\u9053\u8fc7\u53bb\u6240\u53d1\u751f\u7684\u4e8b\u60c5\uff0c \u800c\u5982\u679c\u7528What's happened\uff1f\u5219\u5f3a\u8c03\u5bf9\u73b0\u5728\u7684\u5f71\u54cd\u3002\uff0c\u5373\u60f3\u77e5\u9053\u8fc7\u53bb\u6240\u53d1\u751f\u7684\u4ec0\u4e48\u4e8b\u5bfc\u81f4\u4e86\u8bf4 \u2026 it so happened that\u7684\u53e5\u5f0fhappen\u8868\u793a\u201c\u78b0\u5de7\u6216\u6070\u5de7\u53d1\u751f\u67d0\u4e8b\u201d\u65f6,\u53ef\u7528\u201cIt happens / happened that...\u201d\u8fd9\u4e00\u7ed3\u6784\u6765\u8868\u8fbe.\u4f8b\u5982\uff1a It happened that Brian and Peter were at home that day.\u78b0\u5de7\u90a3\u5929\u5e03\u83b1\u6069\u548c \u2026\nURL: https://duckduckgo.com/?q=What+happened+in+tech+news+this+week?\n\nUser Question: What happened in tech news this week?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", - "prompt.prompt_length": 1206, - "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nhappened\u7684\u7528\u6cd5\u5982\u4e0b\uff1a 1.happened\u662f \u4e0d\u53ca\u7269\u52a8\u8bcd,\u4f5c\u201c\u53d1\u751f\u201d\u89e3\u65f6,\u53ef\u6307\u5076\u7136\u7684\u4e8b\u6545,\u4e5f\u53ef\u6307\u6709\u8ba1\u5212\u6216\u65e0\u8ba1\u5212\u53d1\u751f\u7684\u4e8b\u3002\u4e00\u822c\u4ee5\u4e8b\u4ef6\u3001\u73af\u5883\u3001\u5f62\u52bf\u7b49\u4f5c\u4e3b\u8bed,\u8868\u793a\u201c\u67d0\u4eba\u53d1\u751f\u67d0\u4e8b\u201d\u65f6,\u987b\u4ee5\u4ecb\u8bcdto\u5f15\u51fa\u201c\u67d0\u4eba\u201d\u3002 2. \u2026 what happened\u8fd8\u662fwhat's happened\"What happened\"\u548c\"What's happene...", - "prompt.instruction_following": "web_search_optimized", - "response.answer_length": 510, - "response.answer_completeness": "complete", - "response.response_quality": "medium", - "response.source_citation": 1, - "response.context_utilization": 0.7044198895027625, - "output_response": "Web Search Answer: The web search results do not provide specific information about recent events in tech news for this week. They primarily discuss the grammatical usage of the word \"happened\" and its variations, such ...", - "evaluation.search_effectiveness": 0.2, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 1, - "evaluation.context_relevance": "medium", - "evaluation.overall_quality": "low", - "evaluation.ready_for_production": true, - "web_search.search_strategy": "duckduckgo_api", - "web_search.real_time_data": true, - "web_search.external_sources": 1, - "web_search.information_freshness": "current", - "web_search.knowledge_synthesis": "single_source" - }, - "events": [], - "links": [], - "trace_trace_id": "e30e9ba6-ffd1-4eb9-b7ae-6d19e04343d6", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:58:11.033000000", - "trace_end_time": "2025-10-22 16:58:14.250000000", - "trace_duration_ms": 3216.801, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:58:11.033000000", - "updated_at": "2025-10-22 16:58:11.033000000" - }, - { - "span_id": "82eb19e8-f6cf-43fd-ba56-640a6fd3ed0e", - "trace_id": "45efae74-ee25-4004-a509-6e0ff628a0cd", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-22 16:58:04.883000000", - "end_time": "2025-10-22 16:58:10.029000000", - "duration_ms": 5146.146, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What are the current trends in observability tools?", - "query.length": 51, - "processing.mode": "RAG", - "processing.time_seconds": 5.146108150482178, - "processing.response_length": 815, - "processing.sources_count": 5, - "output_response": "Final Answer: The provided context does not explicitly outline current trends in observability tools. However, it highlights the unique challenges faced by AI applications that traditional monitoring tools cannot a...", - "final_answer_mode": "RAG", - "query_processed.input_query": "What are the current trends in observability tools?", - "query_processed.output_response": "Successfully processed query using RAG, generated 815 character response", - "query_processed.mode": "RAG", - "query_processed.processing_time": 5.146108150482178, - "query_processed.response_length": 815 - }, - "events": [], - "links": [], - "trace_trace_id": "45efae74-ee25-4004-a509-6e0ff628a0cd", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:58:04.883000000", - "trace_end_time": "2025-10-22 16:58:10.029000000", - "trace_duration_ms": 5146.244000000001, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:58:04.883000000", - "updated_at": "2025-10-22 16:58:04.883000000" - }, - { - "span_id": "e81617c6-c2a1-4b08-a5fc-dde3439fe2f4", - "trace_id": "45efae74-ee25-4004-a509-6e0ff628a0cd", - "parent_span_id": "8001ba0e-95c7-42da-9087-603a38e1a500", - "name": "agent.llm-rag", - "start_time": "2025-10-22 16:58:04.883000000", - "end_time": "2025-10-22 16:58:10.029000000", - "duration_ms": 5145.655, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "rag_agent", - "agent.operation": "llm-rag", - "agent.capabilities": [ - "document_retrieval", - "context_generation", - "response_generation" - ], - "agent.id": "noveum_rag_agent", - "input_query": "What are the current trends in observability tools?", - "query_length": 51, - "query_type": "rag_query", - "output_response": "RAG Answer: The provided context does not explicitly outline current trends in observability tools. However, it highlights the unique challenges faced by AI applications that traditional monitoring tools cannot a...", - "rag.context_length": 2943, - "rag.sources_count": 5, - "rag.answer_length": 815, - "rag.mode": "retrieval_augmented_generation" - }, - "events": [], - "links": [], - "trace_trace_id": "45efae74-ee25-4004-a509-6e0ff628a0cd", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:58:04.883000000", - "trace_end_time": "2025-10-22 16:58:10.029000000", - "trace_duration_ms": 5146.244000000001, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:58:04.883000000", - "updated_at": "2025-10-22 16:58:04.883000000" - }, - { - "span_id": "8001ba0e-95c7-42da-9087-603a38e1a500", - "trace_id": "45efae74-ee25-4004-a509-6e0ff628a0cd", - "parent_span_id": "82eb19e8-f6cf-43fd-ba56-640a6fd3ed0e", - "name": "agent.query_routing", - "start_time": "2025-10-22 16:58:04.883000000", - "end_time": "2025-10-22 16:58:10.029000000", - "duration_ms": 5146.036999999999, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What are the current trends in observability tools?", - "query_length": 51, - "query_type": "routing_query", - "output_response": "Routed to RAG for query processing", - "router.classification": "RAG", - "router.confidence_score": 0.3333333333333333, - "router.rag_keyword_score": 2, - "router.web_keyword_score": 1, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "45efae74-ee25-4004-a509-6e0ff628a0cd", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:58:04.883000000", - "trace_end_time": "2025-10-22 16:58:10.029000000", - "trace_duration_ms": 5146.244000000001, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:58:04.883000000", - "updated_at": "2025-10-22 16:58:04.883000000" - }, - { - "span_id": "79b7d138-05e1-4f18-8b63-dcfe9732180b", - "trace_id": "45efae74-ee25-4004-a509-6e0ff628a0cd", - "parent_span_id": "8001ba0e-95c7-42da-9087-603a38e1a500", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-22 16:58:04.883000000", - "end_time": "2025-10-22 16:58:04.883000000", - "duration_ms": 0.051, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What are the current trends in observability tools?", - "query_length": 51, - "query_type": "routing_evaluation_query", - "classification.mode": "RAG", - "classification.rag_keyword_score": 2, - "classification.web_keyword_score": 1, - "classification.confidence_score": 0.3333333333333333, - "classification.confidence_level": "medium", - "classification.method": "keyword_based", - "query.complexity": "complex", - "query.intent": "documentation", - "query.keyword_density": 0.375, - "query.domain_affinity": "noveum", - "routing.decision": "Routed to RAG based on analysis", - "routing.rationale": "RAG score: 2, Web score: 1, Confidence: 0.33", - "routing.expected_performance": "medium", - "routing.alternative_mode": "Web Search", - "routing.decision_confidence": 0.3333333333333333, - "evaluation.routing_accuracy": "medium", - "evaluation.keyword_coverage": 0.057692307692307696, - "evaluation.query_understanding": "ambiguous", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: RAG (Confidence: 0.33)" - }, - "events": [], - "links": [], - "trace_trace_id": "45efae74-ee25-4004-a509-6e0ff628a0cd", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:58:04.883000000", - "trace_end_time": "2025-10-22 16:58:10.029000000", - "trace_duration_ms": 5146.244000000001, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:58:04.883000000", - "updated_at": "2025-10-22 16:58:04.883000000" - }, - { - "span_id": "28c233a0-d42a-4200-86b0-dfbd2d5be06e", - "trace_id": "45efae74-ee25-4004-a509-6e0ff628a0cd", - "parent_span_id": "e81617c6-c2a1-4b08-a5fc-dde3439fe2f4", - "name": "agent.llm_model_execution", - "start_time": "2025-10-22 16:58:05.635000000", - "end_time": "2025-10-22 16:58:09.222000000", - "duration_ms": 3586.924, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What are the current trends in observability tools?", - "query_length": 51, - "query_type": "rag_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 3.5868873596191406, - "model.latency_ms": 3586.8873596191406, - "model.start_time": 1761152285.6352398, - "model.end_time": 1761152289.2221272, - "model.performance_tier": "medium", - "model.prompt_tokens": 872, - "model.completion_tokens": 203, - "model.total_tokens": 1075, - "model.tokens_per_second": 299.7027484337129, - "model.estimated_cost": 0.03225, - "model.efficiency_score": 0.7581395348837209, - "model.response_length": 815, - "model.response_quality": "high", - "model.output_response": "Model Response: The provided context does not explicitly outline current trends in observability tools. However, it highlights the unique challenges faced by AI applications that traditional monitoring tools cannot a...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "45efae74-ee25-4004-a509-6e0ff628a0cd", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:58:04.883000000", - "trace_end_time": "2025-10-22 16:58:10.029000000", - "trace_duration_ms": 5146.244000000001, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:58:04.883000000", - "updated_at": "2025-10-22 16:58:04.883000000" - }, - { - "span_id": "024c05b6-edf9-40ef-931b-9660d48b4948", - "trace_id": "45efae74-ee25-4004-a509-6e0ff628a0cd", - "parent_span_id": "e81617c6-c2a1-4b08-a5fc-dde3439fe2f4", - "name": "agent.rag_evaluation_metrics", - "start_time": "2025-10-22 16:58:09.222000000", - "end_time": "2025-10-22 16:58:09.222000000", - "duration_ms": 0.020999999999999998, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "rag_evaluation_metrics", - "agent.capabilities": [ - "retrieval_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What are the current trends in observability tools?", - "query_length": 51, - "query_type": "rag_evaluation_query", - "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/blog/from-logs-to-intelligent-choices-inside-noveum-ais-evaluation-process):\nFrom Development to Production - Inside Noveum.ai's AI Observability Platform\nShashank Agarwal\n3/3/2025\nIntroduction\nEvery AI application tells a story\u2014but most developers never get to hear it...", - "retrieval.context_length": 2943, - "retrieval.sources_count": 5, - "retrieval.context_quality": "high", - "retrieval.effectiveness": 1, - "retrieval.context_utilization": 2.943, - "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/blog/from-logs-to-intelligent-choices-inside-noveum-ais-evaluation-process):\nFrom Development to Production - Inside Noveum.ai's AI Observability Platform\nShashank Agarwal\n3/3/2025\nIntroduction\nEvery AI application tells a story\u2014but most developers never get to hear it. When your RAG pipeline returns irrelevant results, when your multi-agent system gets stuck in loops, or when your LLM costs suddenly spike, you're left guessing what went wrong and where.\nThat's exactly the challenge Noveum.ai solves. Rather than flying blind, our platform provides comprehensive tracing a...\n\nSource 2 (https://noveum.ai/docs/getting-started/tracing-concepts):\n- Explore Framework Integrations - Framework-specific guidance\n- Learn Advanced Patterns - Custom instrumentation techniques\n- Master the Dashboard - Analyze your traces effectively\nRemember: Good observability is not about collecting all possible data, but about collecting the right data that helps you understand, debug, and optimize your AI applications.\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to O...\n\nSource 3 (https://noveum.ai/en/docs/getting-started/tracing-concepts):\n- Explore Framework Integrations - Framework-specific guidance\n- Learn Advanced Patterns - Custom instrumentation techniques\n- Master the Dashboard - Analyze your traces effectively\nRemember: Good observability is not about collecting all possible data, but about collecting the right data that helps you understand, debug, and optimize your AI applications.\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to O...\n\nSource 4 (https://noveum.ai/en/blog/from-logs-to-intelligent-choices-inside-noveum-ais-evaluation-process):\nThe AI Observability Challenge\nWhy Traditional Monitoring Falls Short\nTraditional application monitoring tools weren't built for AI applications. They can tell you if your API is responding, but they can't answer the questions that matter most for AI systems:\n- Why did my RAG pipeline retrieve irrelevant documents?\n- Which LLM calls are driving my costs?\n- How are my agents communicating with each other?\n- What's causing hallucinations in my responses?\n- Why is my embedding generation so slow?\nW...\n\nSource 5 (https://noveum.ai/en/docs/advanced/rag-observability):\n\ud83d\udd27 Best Practices for RAG Observability\n1. Comprehensive Pipeline Tracing\n2. Quality Monitoring\n3. A/B Testing for RAG Components\n\ud83c\udfaf Advanced RAG Patterns\nMulti-Modal RAG\nConversational RAG\nRAG observability with Noveum.ai provides the deep insights needed to build, optimize, and scale retrieval-augmented generation systems. By implementing comprehensive tracing across retrieval, context preparation, and generation stages, you can ensure your RAG pipeline delivers accurate, relevant, and cost-effe...\n\nUser Question: What are the current trends in observability tools?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", - "prompt.prompt_length": 3490, - "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/blog/from-logs-to-intelligent-choices-inside-noveum-ais-evaluation-process):\nFrom Development to Production - Inside Noveum.ai's AI Observability Platform\nShashank Agarw...", - "prompt.instruction_following": "rag_optimized", - "response.answer_length": 815, - "response.answer_completeness": "complete", - "response.response_quality": "high", - "response.source_citation": 5, - "response.context_utilization": 0.2769283044512402, - "output_response": "RAG Answer: The provided context does not explicitly outline current trends in observability tools. However, it highlights the unique challenges faced by AI applications that traditional monitoring tools cannot a...", - "evaluation.retrieval_effectiveness": 1, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 5, - "evaluation.context_relevance": "high", - "evaluation.overall_quality": "high", - "evaluation.ready_for_production": true, - "rag.retrieval_strategy": "semantic_similarity", - "rag.vector_search_results": 5, - "rag.context_synthesis": "multi_source", - "rag.document_coverage": 1, - "rag.information_density": 0.2769283044512402 - }, - "events": [], - "links": [], - "trace_trace_id": "45efae74-ee25-4004-a509-6e0ff628a0cd", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:58:04.883000000", - "trace_end_time": "2025-10-22 16:58:10.029000000", - "trace_duration_ms": 5146.244000000001, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:58:04.883000000", - "updated_at": "2025-10-22 16:58:04.883000000" - }, - { - "span_id": "99a5e8f3-a9f5-4953-94ea-ad0fa0cf6ec2", - "trace_id": "c5a64d08-d00f-46cf-ab3c-d8d473c10eca", - "parent_span_id": "2a351fd9-a7ee-42da-9e2e-86a643ceed0e", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-22 16:57:57.908000000", - "end_time": "2025-10-22 16:57:57.908000000", - "duration_ms": 0.026, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: Tell me about recent developments in machine learning", - "query_length": 53, - "query_type": "routing_evaluation_query", - "classification.mode": "Web Search", - "classification.rag_keyword_score": 0, - "classification.web_keyword_score": 1, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "complex", - "query.intent": "general_knowledge", - "query.keyword_density": 0.125, - "query.domain_affinity": "general", - "routing.decision": "Routed to Web Search based on analysis", - "routing.rationale": "RAG score: 0, Web score: 1, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "RAG", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.019230769230769232, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: Web Search (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "c5a64d08-d00f-46cf-ab3c-d8d473c10eca", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:57.908000000", - "trace_end_time": "2025-10-22 16:58:03.875000000", - "trace_duration_ms": 5967.0779999999995, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:57.908000000", - "updated_at": "2025-10-22 16:57:57.908000000" - }, - { - "span_id": "6e891e51-ca6d-4e6c-80d0-0791ee22d5b9", - "trace_id": "c5a64d08-d00f-46cf-ab3c-d8d473c10eca", - "parent_span_id": "2a351fd9-a7ee-42da-9e2e-86a643ceed0e", - "name": "agent.web_search_generation", - "start_time": "2025-10-22 16:57:57.908000000", - "end_time": "2025-10-22 16:58:03.874000000", - "duration_ms": 5966.728, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "web_search_agent", - "agent.operation": "web_search_generation", - "agent.capabilities": [ - "web_search", - "content_synthesis", - "response_generation" - ], - "agent.id": "noveum_web_search_agent", - "input_query": "Tell me about recent developments in machine learning", - "query_length": 53, - "query_type": "web_search_query", - "output_response": "Web Search Answer: The web search results did not provide specific information about recent developments in machine learning. Therefore, I cannot summarize any recent advancements based on the provided data.\n\nHowever, i...", - "web_search.results_count": 1, - "web_search.context_length": 768, - "web_search.response_length": 687, - "web_search.mode": "external_web_search" - }, - "events": [], - "links": [], - "trace_trace_id": "c5a64d08-d00f-46cf-ab3c-d8d473c10eca", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:57.908000000", - "trace_end_time": "2025-10-22 16:58:03.875000000", - "trace_duration_ms": 5967.0779999999995, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:57.908000000", - "updated_at": "2025-10-22 16:57:57.908000000" - }, - { - "span_id": "6d622376-d48b-4a64-b0a2-1402117febaf", - "trace_id": "c5a64d08-d00f-46cf-ab3c-d8d473c10eca", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-22 16:57:57.908000000", - "end_time": "2025-10-22 16:58:03.875000000", - "duration_ms": 5967.035, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "Tell me about recent developments in machine learning", - "query.length": 53, - "processing.mode": "Web Search", - "processing.time_seconds": 5.966889142990112, - "processing.response_length": 687, - "processing.sources_count": 1, - "output_response": "Final Answer: The web search results did not provide specific information about recent developments in machine learning. Therefore, I cannot summarize any recent advancements based on the provided data.\n\nHowever, i...", - "final_answer_mode": "Web Search", - "query_processed.input_query": "Tell me about recent developments in machine learning", - "query_processed.output_response": "Successfully processed query using Web Search, generated 687 character response", - "query_processed.mode": "Web Search", - "query_processed.processing_time": 5.966889142990112, - "query_processed.response_length": 687 - }, - "events": [], - "links": [], - "trace_trace_id": "c5a64d08-d00f-46cf-ab3c-d8d473c10eca", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:57.908000000", - "trace_end_time": "2025-10-22 16:58:03.875000000", - "trace_duration_ms": 5967.0779999999995, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:57.908000000", - "updated_at": "2025-10-22 16:57:57.908000000" - }, - { - "span_id": "2a351fd9-a7ee-42da-9e2e-86a643ceed0e", - "trace_id": "c5a64d08-d00f-46cf-ab3c-d8d473c10eca", - "parent_span_id": "6d622376-d48b-4a64-b0a2-1402117febaf", - "name": "agent.query_routing", - "start_time": "2025-10-22 16:57:57.908000000", - "end_time": "2025-10-22 16:58:03.875000000", - "duration_ms": 5966.8640000000005, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "Tell me about recent developments in machine learning", - "query_length": 53, - "query_type": "routing_query", - "output_response": "Routed to Web Search for query processing", - "router.classification": "Web Search", - "router.confidence_score": 1, - "router.rag_keyword_score": 0, - "router.web_keyword_score": 1, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "c5a64d08-d00f-46cf-ab3c-d8d473c10eca", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:57.908000000", - "trace_end_time": "2025-10-22 16:58:03.875000000", - "trace_duration_ms": 5967.0779999999995, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:57.908000000", - "updated_at": "2025-10-22 16:57:57.908000000" - }, - { - "span_id": "ba457b3b-9be8-4e24-af63-4118833711e6", - "trace_id": "c5a64d08-d00f-46cf-ab3c-d8d473c10eca", - "parent_span_id": "6e891e51-ca6d-4e6c-80d0-0791ee22d5b9", - "name": "agent.llm_model_execution", - "start_time": "2025-10-22 16:57:58.324000000", - "end_time": "2025-10-22 16:58:03.874000000", - "duration_ms": 5550.059, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: Tell me about recent developments in machine learning", - "query_length": 53, - "query_type": "web_search_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 5.550010919570923, - "model.latency_ms": 5550.010919570923, - "model.start_time": 1761152278.3248482, - "model.end_time": 1761152283.874859, - "model.performance_tier": "slow", - "model.prompt_tokens": 316, - "model.completion_tokens": 171, - "model.total_tokens": 487, - "model.tokens_per_second": 87.74757510524871, - "model.estimated_cost": 0.01461, - "model.efficiency_score": 1.4106776180698153, - "model.response_length": 687, - "model.response_quality": "high", - "model.output_response": "Model Response: The web search results did not provide specific information about recent developments in machine learning. Therefore, I cannot summarize any recent advancements based on the provided data.\n\nHowever, i...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "c5a64d08-d00f-46cf-ab3c-d8d473c10eca", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:57.908000000", - "trace_end_time": "2025-10-22 16:58:03.875000000", - "trace_duration_ms": 5967.0779999999995, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:57.908000000", - "updated_at": "2025-10-22 16:57:57.908000000" - }, - { - "span_id": "7b70a916-1c2b-4898-a052-cfd98a979167", - "trace_id": "c5a64d08-d00f-46cf-ab3c-d8d473c10eca", - "parent_span_id": "6e891e51-ca6d-4e6c-80d0-0791ee22d5b9", - "name": "agent.web_search_evaluation_metrics", - "start_time": "2025-10-22 16:58:03.874000000", - "end_time": "2025-10-22 16:58:03.874000000", - "duration_ms": 0.02, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "web_search_evaluation_metrics", - "agent.capabilities": [ - "web_search_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: Tell me about recent developments in machine learning", - "query_length": 53, - "query_type": "web_search_evaluation_query", - "web_search.results_count": 1, - "web_search.context_length": 768, - "web_search.context_synthesized": "Context from 1 web sources", - "web_search.search_effectiveness": 0.2, - "web_search.context_quality": "medium", - "web_search.source_diversity": 1, - "web_search.information_synthesis": "low", - "web_search.external_knowledge_utilization": 0.768, - "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nFeb 3, 2024 \u00b7 tell sb sth\u4e2d\u7684tell\u66f4\u591a\u662f\u544a\u77e5\u7684\u610f\u601d\uff0c\u5c31\u662f\u544a\u8bc9\u522b\u4eba\u6709\u8fd9\u4e48\u4e00\u4e2a\u4e8b\u7684\u5b58\u5728 tell sb about sth \u7684\u610f\u601d\u5305\u542b\u66f4\u591a\u4e00\u4e9b\uff0c\u9664\u4e86\u544a\u77e5\u6709\u8fd9\u4e48\u4e00\u4e2a\u4e8b\u60c5\u7684\u5b58\u5728\uff0c\u8fd8\u4f5c\u4e86\u4e00\u4e9b\u8bf4\u660e \u6bd4\u5982\uff1a tell me he \u2026 \u4e00\u3001Say\u548cTell\u7684\u7528\u6cd5 \u6211\u4eec\u5148\u5c06say\u3001tell\u3001speak\u3001talk\u8fd9\u56db\u4e2a\u8bcd\u5206\u6210\u4e24\u7ec4\uff0c\u5206\u522b\u4e3a\uff1a \u7b2c\u2460\u7ec4\uff1asay & tell\uff0c\u8fd9\u4e24\u4e2a\u8bcd\u4e4b\u540e\u53ef\u4ee5\u8868\u793a\u8c08\u8bdd\u7684 \u201c\u7ec6\u8282\u201d\u3002 \u7b2c\u2461\u7ec4\uff1aspeak & talk\uff0c\u4e0d\u80fd\u8868\u793a\u8c08\u8bdd\u7684\u201c\u7ec6\u8282\u201d\u3002 \u2026 tell of 1\u3001Tell of a turning point in your career and what you learned. \u8bb2\u8ff0\u4f60 \u804c\u4e1a\u751f\u6daf \u4e00\u4e2a\u8f6c\u6298\u70b9\uff0c\u4ee5\u53ca\u4f60\u5b66\u5230\u4e86\u4ec0\u4e48\u3002 2\u3001I have made the Sovereign Lord my refuge; I will tell of all your \u2026 tell sb to do sth \u548c tell sb do sth \u90fd\u8868\u793a\u544a\u8bc9\u67d0\u4eba\u505a\u67d0\u4e8b\uff0ctell sb to do sth \u5f3a\u8c03\u544a\u8bc9\u67d0\u4eba\u5e94\u8be5\u505a\u4ec0\u4e48\u5177\u4f53\u52a8\u4f5c\u6216\u884c\u4e3a\uff0c\u800c tell sb do sth \u5f3a\u8c03\u544a\u8bc9\u67d0\u4eba\u6b63\u5728\u8fdb\u884c\u7684\u5177\u4f53\u884c\u52a8\u6216\u72b6\u6001\u3002 \u5b83\u4eec\u5728\u91ca\u4e49\u3001 \u2026 Aug 23, 2023 \u00b7 tell\u7684\u8fc7\u53bb\u5f0f\u548c\u8fc7\u53bb\u5206\u8bcd\u90fd\u662ftold\u3002 \u8bfb\u97f3\uff1a\u82f1[tel]\uff0c\u7f8e[tel]\u3002 \u91ca\u4e49\uff1a vt.\u544a\u8bc9\uff0c\u8bf4\uff1b\u8fa8\u522b\uff1b\u5429\u5490\uff1b\u65ad\u5b9a\u3002 vi.\u8bb2\u8ff0\uff1b\u544a\u53d1\uff0c\u6cc4\u5bc6\uff1b\u8bc6\u522b\u3002 \u53d8\u5f62\uff1a\u8fc7\u53bb\u5f0ftold\uff0c\u8fc7\u53bb\u5206\u8bcdtold\uff0c\u73b0\u5728\u5206 \u2026\nURL: https://duckduckgo.com/?q=Tell+me+about+recent+developments+in+machine+learning\n\nUser Question: Tell me about recent developments in machine learning\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", - "prompt.prompt_length": 1266, - "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nFeb 3, 2024 \u00b7 tell sb sth\u4e2d\u7684tell\u66f4\u591a\u662f\u544a\u77e5\u7684\u610f\u601d\uff0c\u5c31\u662f\u544a\u8bc9\u522b\u4eba\u6709\u8fd9\u4e48\u4e00\u4e2a\u4e8b\u7684\u5b58\u5728 tell sb about sth \u7684\u610f\u601d\u5305\u542b\u66f4\u591a\u4e00\u4e9b\uff0c\u9664\u4e86\u544a\u77e5\u6709\u8fd9\u4e48\u4e00\u4e2a\u4e8b\u60c5\u7684\u5b58\u5728\uff0c\u8fd8\u4f5c\u4e86\u4e00\u4e9b\u8bf4\u660e \u6bd4\u5982\uff1a tell me he \u2026 \u4e00\u3001Say\u548cTell\u7684\u7528\u6cd5 \u6211\u4eec\u5148\u5c06say\u3001tell\u3001speak\u3001talk\u8fd9\u56db\u4e2a\u8bcd\u5206\u6210\u4e24\u7ec4\uff0c\u5206\u522b...", - "prompt.instruction_following": "web_search_optimized", - "response.answer_length": 687, - "response.answer_completeness": "complete", - "response.response_quality": "medium", - "response.source_citation": 1, - "response.context_utilization": 0.89453125, - "output_response": "Web Search Answer: The web search results did not provide specific information about recent developments in machine learning. Therefore, I cannot summarize any recent advancements based on the provided data.\n\nHowever, i...", - "evaluation.search_effectiveness": 0.2, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 1, - "evaluation.context_relevance": "medium", - "evaluation.overall_quality": "low", - "evaluation.ready_for_production": true, - "web_search.search_strategy": "duckduckgo_api", - "web_search.real_time_data": true, - "web_search.external_sources": 1, - "web_search.information_freshness": "current", - "web_search.knowledge_synthesis": "single_source" - }, - "events": [], - "links": [], - "trace_trace_id": "c5a64d08-d00f-46cf-ab3c-d8d473c10eca", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:57.908000000", - "trace_end_time": "2025-10-22 16:58:03.875000000", - "trace_duration_ms": 5967.0779999999995, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:57.908000000", - "updated_at": "2025-10-22 16:57:57.908000000" - }, - { - "span_id": "1e1a341f-66c1-48b0-95eb-fb092928c473", - "trace_id": "b1b13e60-5b32-44a0-9769-197a944989d0", - "parent_span_id": "1c5c4291-bbb5-4f85-b6d9-b6e8df66f742", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-22 16:57:51.967000000", - "end_time": "2025-10-22 16:57:51.967000000", - "duration_ms": 0.033, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What's the weather like today?", - "query_length": 30, - "query_type": "routing_evaluation_query", - "classification.mode": "Web Search", - "classification.rag_keyword_score": 0, - "classification.web_keyword_score": 2, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "general_knowledge", - "query.keyword_density": 0.4, - "query.domain_affinity": "general", - "routing.decision": "Routed to Web Search based on analysis", - "routing.rationale": "RAG score: 0, Web score: 2, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "RAG", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.038461538461538464, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: Web Search (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "b1b13e60-5b32-44a0-9769-197a944989d0", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:51.967000000", - "trace_end_time": "2025-10-22 16:57:56.899000000", - "trace_duration_ms": 4932.393, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:51.967000000", - "updated_at": "2025-10-22 16:57:51.967000000" - }, - { - "span_id": "1c5c4291-bbb5-4f85-b6d9-b6e8df66f742", - "trace_id": "b1b13e60-5b32-44a0-9769-197a944989d0", - "parent_span_id": "49e7cace-ba21-4807-a238-9a3880487583", - "name": "agent.query_routing", - "start_time": "2025-10-22 16:57:51.967000000", - "end_time": "2025-10-22 16:57:56.899000000", - "duration_ms": 4931.972, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What's the weather like today?", - "query_length": 30, - "query_type": "routing_query", - "output_response": "Routed to Web Search for query processing", - "router.classification": "Web Search", - "router.confidence_score": 1, - "router.rag_keyword_score": 0, - "router.web_keyword_score": 2, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "b1b13e60-5b32-44a0-9769-197a944989d0", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:51.967000000", - "trace_end_time": "2025-10-22 16:57:56.899000000", - "trace_duration_ms": 4932.393, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:51.967000000", - "updated_at": "2025-10-22 16:57:51.967000000" - }, - { - "span_id": "49e7cace-ba21-4807-a238-9a3880487583", - "trace_id": "b1b13e60-5b32-44a0-9769-197a944989d0", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-22 16:57:51.967000000", - "end_time": "2025-10-22 16:57:56.899000000", - "duration_ms": 4932.316, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What's the weather like today?", - "query.length": 30, - "processing.mode": "Web Search", - "processing.time_seconds": 4.932007789611816, - "processing.response_length": 260, - "processing.sources_count": 1, - "output_response": "Final Answer: The web search results do not provide specific information about today's weather. To find out the current weather conditions, I recommend checking a reliable weather website or app for detailed update...", - "final_answer_mode": "Web Search", - "query_processed.input_query": "What's the weather like today?", - "query_processed.output_response": "Successfully processed query using Web Search, generated 260 character response", - "query_processed.mode": "Web Search", - "query_processed.processing_time": 4.932007789611816, - "query_processed.response_length": 260 - }, - "events": [], - "links": [], - "trace_trace_id": "b1b13e60-5b32-44a0-9769-197a944989d0", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:51.967000000", - "trace_end_time": "2025-10-22 16:57:56.899000000", - "trace_duration_ms": 4932.393, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:51.967000000", - "updated_at": "2025-10-22 16:57:51.967000000" - }, - { - "span_id": "58d217aa-4350-43e8-8813-468149dc4107", - "trace_id": "b1b13e60-5b32-44a0-9769-197a944989d0", - "parent_span_id": "1c5c4291-bbb5-4f85-b6d9-b6e8df66f742", - "name": "agent.web_search_generation", - "start_time": "2025-10-22 16:57:51.967000000", - "end_time": "2025-10-22 16:57:56.899000000", - "duration_ms": 4931.796, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "web_search_agent", - "agent.operation": "web_search_generation", - "agent.capabilities": [ - "web_search", - "content_synthesis", - "response_generation" - ], - "agent.id": "noveum_web_search_agent", - "input_query": "What's the weather like today?", - "query_length": 30, - "query_type": "web_search_query", - "output_response": "Web Search Answer: The web search results do not provide specific information about today's weather. To find out the current weather conditions, I recommend checking a reliable weather website or app for detailed update...", - "web_search.results_count": 1, - "web_search.context_length": 995, - "web_search.response_length": 260, - "web_search.mode": "external_web_search" - }, - "events": [], - "links": [], - "trace_trace_id": "b1b13e60-5b32-44a0-9769-197a944989d0", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:51.967000000", - "trace_end_time": "2025-10-22 16:57:56.899000000", - "trace_duration_ms": 4932.393, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:51.967000000", - "updated_at": "2025-10-22 16:57:51.967000000" - }, - { - "span_id": "d89dd84e-0b5e-4c92-942b-18c4735713d4", - "trace_id": "b1b13e60-5b32-44a0-9769-197a944989d0", - "parent_span_id": "58d217aa-4350-43e8-8813-468149dc4107", - "name": "agent.llm_model_execution", - "start_time": "2025-10-22 16:57:55.269000000", - "end_time": "2025-10-22 16:57:56.899000000", - "duration_ms": 1629.7140000000002, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What's the weather like today?", - "query_length": 30, - "query_type": "web_search_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 1.629619836807251, - "model.latency_ms": 1629.619836807251, - "model.start_time": 1761152275.269459, - "model.end_time": 1761152276.8990788, - "model.performance_tier": "fast", - "model.prompt_tokens": 367, - "model.completion_tokens": 65, - "model.total_tokens": 432, - "model.tokens_per_second": 265.0925020932329, - "model.estimated_cost": 0.012960000000000001, - "model.efficiency_score": 0.6018518518518519, - "model.response_length": 260, - "model.response_quality": "high", - "model.output_response": "Model Response: The web search results do not provide specific information about today's weather. To find out the current weather conditions, I recommend checking a reliable weather website or app for detailed update...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "b1b13e60-5b32-44a0-9769-197a944989d0", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:51.967000000", - "trace_end_time": "2025-10-22 16:57:56.899000000", - "trace_duration_ms": 4932.393, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:51.967000000", - "updated_at": "2025-10-22 16:57:51.967000000" - }, - { - "span_id": "d076719d-da79-43fb-92ea-6015f856146b", - "trace_id": "b1b13e60-5b32-44a0-9769-197a944989d0", - "parent_span_id": "58d217aa-4350-43e8-8813-468149dc4107", - "name": "agent.web_search_evaluation_metrics", - "start_time": "2025-10-22 16:57:56.899000000", - "end_time": "2025-10-22 16:57:56.899000000", - "duration_ms": 0.049, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "web_search_evaluation_metrics", - "agent.capabilities": [ - "web_search_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What's the weather like today?", - "query_length": 30, - "query_type": "web_search_evaluation_query", - "web_search.results_count": 1, - "web_search.context_length": 995, - "web_search.context_synthesized": "Context from 1 web sources", - "web_search.search_effectiveness": 0.2, - "web_search.context_quality": "high", - "web_search.source_diversity": 1, - "web_search.information_synthesis": "low", - "web_search.external_knowledge_utilization": 0.995, - "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nLog in to WhatsApp Web for simple, reliable and private messaging on your desktop. Send and receive messages and files with ease, all for free. 2 days ago \u00b7 More videos you may like 00:13 Wanna be yours #kimpolin #colinbridgerton #benedictbri... 2 hours ago \u00b7 101 views 00:14 Viscount & Vicountess's Marital bliss #bridgerton \u2026 Nov 15, 2024 \u00b7 Learn the correct usage of \"what's the weather like today\" and \"how's the weather today\" in English. Discover differences, examples, alternatives and tips for choosing the right \u2026 Nov 15, 2024 \u00b7 Learn the correct usage of \"how's the weather today\" and \"what's the weather today\" in English. Discover differences, examples, alternatives and tips for choosing the right \u2026 2 days ago \u00b7 Get the latest hourly weather updates for Thane today. Detailed forecast including temperature, wind, rain, snow, and UV index. Stay informed about today's weather conditions in \u2026\nURL: https://duckduckgo.com/?q=What's+the+weather+like+today?\n\nUser Question: What's the weather like today?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", - "prompt.prompt_length": 1470, - "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nLog in to WhatsApp Web for simple, reliable and private messaging on your desktop. Send and receive messages and files with ease, all for free. 2 days ago \u00b7 More videos you...", - "prompt.instruction_following": "web_search_optimized", - "response.answer_length": 260, - "response.answer_completeness": "complete", - "response.response_quality": "medium", - "response.source_citation": 1, - "response.context_utilization": 0.2613065326633166, - "output_response": "Web Search Answer: The web search results do not provide specific information about today's weather. To find out the current weather conditions, I recommend checking a reliable weather website or app for detailed update...", - "evaluation.search_effectiveness": 0.2, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 1, - "evaluation.context_relevance": "high", - "evaluation.overall_quality": "low", - "evaluation.ready_for_production": true, - "web_search.search_strategy": "duckduckgo_api", - "web_search.real_time_data": true, - "web_search.external_sources": 1, - "web_search.information_freshness": "current", - "web_search.knowledge_synthesis": "single_source" - }, - "events": [], - "links": [], - "trace_trace_id": "b1b13e60-5b32-44a0-9769-197a944989d0", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:51.967000000", - "trace_end_time": "2025-10-22 16:57:56.899000000", - "trace_duration_ms": 4932.393, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:51.967000000", - "updated_at": "2025-10-22 16:57:51.967000000" - }, - { - "span_id": "41167ca4-e1ad-48a5-b5eb-aa272ad88a85", - "trace_id": "e4f3cc20-c137-4905-80fe-2d29b03c2231", - "parent_span_id": "56959f47-5219-4674-8c0d-e5fa928dcd45", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-22 16:57:47.477000000", - "end_time": "2025-10-22 16:57:47.477000000", - "duration_ms": 0.061, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What are the latest AI news today?", - "query_length": 34, - "query_type": "routing_evaluation_query", - "classification.mode": "Web Search", - "classification.rag_keyword_score": 0, - "classification.web_keyword_score": 3, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "general_knowledge", - "query.keyword_density": 0.42857142857142855, - "query.domain_affinity": "general", - "routing.decision": "Routed to Web Search based on analysis", - "routing.rationale": "RAG score: 0, Web score: 3, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "RAG", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.057692307692307696, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: Web Search (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "e4f3cc20-c137-4905-80fe-2d29b03c2231", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:47.477000000", - "trace_end_time": "2025-10-22 16:57:50.959000000", - "trace_duration_ms": 3481.856, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:47.477000000", - "updated_at": "2025-10-22 16:57:47.477000000" - }, - { - "span_id": "73005751-9eb1-4540-aa49-2ac580cd7418", - "trace_id": "e4f3cc20-c137-4905-80fe-2d29b03c2231", - "parent_span_id": "56959f47-5219-4674-8c0d-e5fa928dcd45", - "name": "agent.web_search_generation", - "start_time": "2025-10-22 16:57:47.477000000", - "end_time": "2025-10-22 16:57:50.958000000", - "duration_ms": 3480.8469999999998, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "web_search_agent", - "agent.operation": "web_search_generation", - "agent.capabilities": [ - "web_search", - "content_synthesis", - "response_generation" - ], - "agent.id": "noveum_web_search_agent", - "input_query": "What are the latest AI news today?", - "query_length": 34, - "query_type": "web_search_query", - "output_response": "Web Search Answer: The provided web search results do not contain specific information about the latest AI news today. They primarily focus on general news coverage from India and around the world, including politics, b...", - "web_search.results_count": 1, - "web_search.context_length": 887, - "web_search.response_length": 503, - "web_search.mode": "external_web_search" - }, - "events": [], - "links": [], - "trace_trace_id": "e4f3cc20-c137-4905-80fe-2d29b03c2231", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:47.477000000", - "trace_end_time": "2025-10-22 16:57:50.959000000", - "trace_duration_ms": 3481.856, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:47.477000000", - "updated_at": "2025-10-22 16:57:47.477000000" - }, - { - "span_id": "9e1b87be-bc29-4084-b37c-6e1220dde8eb", - "trace_id": "e4f3cc20-c137-4905-80fe-2d29b03c2231", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-22 16:57:47.477000000", - "end_time": "2025-10-22 16:57:50.959000000", - "duration_ms": 3481.7580000000003, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What are the latest AI news today?", - "query.length": 34, - "processing.mode": "Web Search", - "processing.time_seconds": 3.4813451766967773, - "processing.response_length": 503, - "processing.sources_count": 1, - "output_response": "Final Answer: The provided web search results do not contain specific information about the latest AI news today. They primarily focus on general news coverage from India and around the world, including politics, b...", - "final_answer_mode": "Web Search", - "query_processed.input_query": "What are the latest AI news today?", - "query_processed.output_response": "Successfully processed query using Web Search, generated 503 character response", - "query_processed.mode": "Web Search", - "query_processed.processing_time": 3.4813451766967773, - "query_processed.response_length": 503 - }, - "events": [], - "links": [], - "trace_trace_id": "e4f3cc20-c137-4905-80fe-2d29b03c2231", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:47.477000000", - "trace_end_time": "2025-10-22 16:57:50.959000000", - "trace_duration_ms": 3481.856, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:47.477000000", - "updated_at": "2025-10-22 16:57:47.477000000" - }, - { - "span_id": "56959f47-5219-4674-8c0d-e5fa928dcd45", - "trace_id": "e4f3cc20-c137-4905-80fe-2d29b03c2231", - "parent_span_id": "9e1b87be-bc29-4084-b37c-6e1220dde8eb", - "name": "agent.query_routing", - "start_time": "2025-10-22 16:57:47.477000000", - "end_time": "2025-10-22 16:57:50.958000000", - "duration_ms": 3481.254, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What are the latest AI news today?", - "query_length": 34, - "query_type": "routing_query", - "output_response": "Routed to Web Search for query processing", - "router.classification": "Web Search", - "router.confidence_score": 1, - "router.rag_keyword_score": 0, - "router.web_keyword_score": 3, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "e4f3cc20-c137-4905-80fe-2d29b03c2231", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:47.477000000", - "trace_end_time": "2025-10-22 16:57:50.959000000", - "trace_duration_ms": 3481.856, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:47.477000000", - "updated_at": "2025-10-22 16:57:47.477000000" - }, - { - "span_id": "3dbf95ba-3ef1-4294-93d4-217f4ada5ffe", - "trace_id": "e4f3cc20-c137-4905-80fe-2d29b03c2231", - "parent_span_id": "73005751-9eb1-4540-aa49-2ac580cd7418", - "name": "agent.llm_model_execution", - "start_time": "2025-10-22 16:57:47.864000000", - "end_time": "2025-10-22 16:57:50.958000000", - "duration_ms": 3093.442, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What are the latest AI news today?", - "query_length": 34, - "query_type": "web_search_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 3.0933361053466797, - "model.latency_ms": 3093.3361053466797, - "model.start_time": 1761152267.864976, - "model.end_time": 1761152270.958312, - "model.performance_tier": "medium", - "model.prompt_tokens": 341, - "model.completion_tokens": 125, - "model.total_tokens": 466, - "model.tokens_per_second": 150.6464167261171, - "model.estimated_cost": 0.013980000000000001, - "model.efficiency_score": 1.0793991416309012, - "model.response_length": 503, - "model.response_quality": "high", - "model.output_response": "Model Response: The provided web search results do not contain specific information about the latest AI news today. They primarily focus on general news coverage from India and around the world, including politics, b...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "e4f3cc20-c137-4905-80fe-2d29b03c2231", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:47.477000000", - "trace_end_time": "2025-10-22 16:57:50.959000000", - "trace_duration_ms": 3481.856, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:47.477000000", - "updated_at": "2025-10-22 16:57:47.477000000" - }, - { - "span_id": "f7fe795b-ce1b-4fcc-83f5-06a5f074f38d", - "trace_id": "e4f3cc20-c137-4905-80fe-2d29b03c2231", - "parent_span_id": "73005751-9eb1-4540-aa49-2ac580cd7418", - "name": "agent.web_search_evaluation_metrics", - "start_time": "2025-10-22 16:57:50.958000000", - "end_time": "2025-10-22 16:57:50.958000000", - "duration_ms": 0.026, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "web_search_evaluation_metrics", - "agent.capabilities": [ - "web_search_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What are the latest AI news today?", - "query_length": 34, - "query_type": "web_search_evaluation_query", - "web_search.results_count": 1, - "web_search.context_length": 887, - "web_search.context_synthesized": "Context from 1 web sources", - "web_search.search_effectiveness": 0.2, - "web_search.context_quality": "high", - "web_search.source_diversity": 1, - "web_search.information_synthesis": "low", - "web_search.external_knowledge_utilization": 0.887, - "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\n3 days ago \u00b7 Latest news headlines from India & around the world. Check out today\u2019s news coverage live with videos & photos on NDTV.com. 6 days ago \u00b7 India News | Latest India News | Read latest and breaking news from India. Today's top India news headlines, news on Indian politics, elections, government, business, \u2026 2 days ago \u00b7 Top News Stories of the day, Latest News Headlines, News Specials, Breaking News and Latest India News, World current affairs & Political News all around the clock at \u2026 NDTV.com provides latest news from India and around the world. Get breaking news alerts from India and follow today\u2019s live news updates in field of politics, business, technology, Bollywood ... 6 days ago \u00b7 Get Latest entertainment news from Bollywood, Hollywood and regional film industries.\nURL: https://duckduckgo.com/?q=What+are+the+latest+AI+news+today?\n\nUser Question: What are the latest AI news today?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", - "prompt.prompt_length": 1366, - "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\n3 days ago \u00b7 Latest news headlines from India & around the world. Check out today\u2019s news coverage live with videos & photos on NDTV.com. 6 days ago \u00b7 India News | Latest In...", - "prompt.instruction_following": "web_search_optimized", - "response.answer_length": 503, - "response.answer_completeness": "complete", - "response.response_quality": "medium", - "response.source_citation": 1, - "response.context_utilization": 0.5670800450958287, - "output_response": "Web Search Answer: The provided web search results do not contain specific information about the latest AI news today. They primarily focus on general news coverage from India and around the world, including politics, b...", - "evaluation.search_effectiveness": 0.2, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 1, - "evaluation.context_relevance": "high", - "evaluation.overall_quality": "low", - "evaluation.ready_for_production": true, - "web_search.search_strategy": "duckduckgo_api", - "web_search.real_time_data": true, - "web_search.external_sources": 1, - "web_search.information_freshness": "current", - "web_search.knowledge_synthesis": "single_source" - }, - "events": [], - "links": [], - "trace_trace_id": "e4f3cc20-c137-4905-80fe-2d29b03c2231", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:47.477000000", - "trace_end_time": "2025-10-22 16:57:50.959000000", - "trace_duration_ms": 3481.856, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:47.477000000", - "updated_at": "2025-10-22 16:57:47.477000000" - }, - { - "span_id": "b9c9d21e-5f76-4e5a-b99e-2b4ab821caee", - "trace_id": "823b7927-10bb-49ef-a94e-1b914c298bfb", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-22 16:57:38.501000000", - "end_time": "2025-10-22 16:57:46.469000000", - "duration_ms": 7968.089, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What are the benefits of using Noveum Trace?", - "query.length": 44, - "processing.mode": "RAG", - "processing.time_seconds": 7.968040943145752, - "processing.response_length": 1396, - "processing.sources_count": 5, - "output_response": "Final Answer: Using Noveum Trace offers several benefits, including:\n\n1. **Real-Time Monitoring**: The Noveum platform provides a real-time dashboard for analyzing traces and performance, with live updates as new t...", - "final_answer_mode": "RAG", - "query_processed.input_query": "What are the benefits of using Noveum Trace?", - "query_processed.output_response": "Successfully processed query using RAG, generated 1396 character response", - "query_processed.mode": "RAG", - "query_processed.processing_time": 7.968040943145752, - "query_processed.response_length": 1396 - }, - "events": [], - "links": [], - "trace_trace_id": "823b7927-10bb-49ef-a94e-1b914c298bfb", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:38.501000000", - "trace_end_time": "2025-10-22 16:57:46.469000000", - "trace_duration_ms": 7968.149, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:38.501000000", - "updated_at": "2025-10-22 16:57:38.501000000" - }, - { - "span_id": "bc85c745-4c27-4aa4-bc00-4c9f10a2f049", - "trace_id": "823b7927-10bb-49ef-a94e-1b914c298bfb", - "parent_span_id": "b9c9d21e-5f76-4e5a-b99e-2b4ab821caee", - "name": "agent.query_routing", - "start_time": "2025-10-22 16:57:38.501000000", - "end_time": "2025-10-22 16:57:46.469000000", - "duration_ms": 7968.001, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What are the benefits of using Noveum Trace?", - "query_length": 44, - "query_type": "routing_query", - "output_response": "Routed to RAG for query processing", - "router.classification": "RAG", - "router.confidence_score": 1, - "router.rag_keyword_score": 2, - "router.web_keyword_score": 0, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "823b7927-10bb-49ef-a94e-1b914c298bfb", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:38.501000000", - "trace_end_time": "2025-10-22 16:57:46.469000000", - "trace_duration_ms": 7968.149, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:38.501000000", - "updated_at": "2025-10-22 16:57:38.501000000" - }, - { - "span_id": "3ab8cc4c-f5ed-410a-b4ee-48d0954b756d", - "trace_id": "823b7927-10bb-49ef-a94e-1b914c298bfb", - "parent_span_id": "bc85c745-4c27-4aa4-bc00-4c9f10a2f049", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-22 16:57:38.501000000", - "end_time": "2025-10-22 16:57:38.501000000", - "duration_ms": 0.09899999999999999, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What are the benefits of using Noveum Trace?", - "query_length": 44, - "query_type": "routing_evaluation_query", - "classification.mode": "RAG", - "classification.rag_keyword_score": 2, - "classification.web_keyword_score": 0, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "noveum_specific", - "query.keyword_density": 0.25, - "query.domain_affinity": "noveum", - "routing.decision": "Routed to RAG based on analysis", - "routing.rationale": "RAG score: 2, Web score: 0, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "Web Search", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.038461538461538464, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: RAG (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "823b7927-10bb-49ef-a94e-1b914c298bfb", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:38.501000000", - "trace_end_time": "2025-10-22 16:57:46.469000000", - "trace_duration_ms": 7968.149, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:38.501000000", - "updated_at": "2025-10-22 16:57:38.501000000" - }, - { - "span_id": "67292076-ec8a-4856-9c5b-726868552157", - "trace_id": "823b7927-10bb-49ef-a94e-1b914c298bfb", - "parent_span_id": "bc85c745-4c27-4aa4-bc00-4c9f10a2f049", - "name": "agent.llm-rag", - "start_time": "2025-10-22 16:57:38.501000000", - "end_time": "2025-10-22 16:57:46.469000000", - "duration_ms": 7967.679, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "rag_agent", - "agent.operation": "llm-rag", - "agent.capabilities": [ - "document_retrieval", - "context_generation", - "response_generation" - ], - "agent.id": "noveum_rag_agent", - "input_query": "What are the benefits of using Noveum Trace?", - "query_length": 44, - "query_type": "rag_query", - "output_response": "RAG Answer: Using Noveum Trace offers several benefits, including:\n\n1. **Real-Time Monitoring**: The Noveum platform provides a real-time dashboard for analyzing traces and performance, with live updates as new t...", - "rag.context_length": 2773, - "rag.sources_count": 5, - "rag.answer_length": 1396, - "rag.mode": "retrieval_augmented_generation" - }, - "events": [], - "links": [], - "trace_trace_id": "823b7927-10bb-49ef-a94e-1b914c298bfb", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:38.501000000", - "trace_end_time": "2025-10-22 16:57:46.469000000", - "trace_duration_ms": 7968.149, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:38.501000000", - "updated_at": "2025-10-22 16:57:38.501000000" - }, - { - "span_id": "d044ae4a-c463-4c1b-abce-a9a18f045e75", - "trace_id": "823b7927-10bb-49ef-a94e-1b914c298bfb", - "parent_span_id": "67292076-ec8a-4856-9c5b-726868552157", - "name": "agent.llm_model_execution", - "start_time": "2025-10-22 16:57:38.901000000", - "end_time": "2025-10-22 16:57:45.885000000", - "duration_ms": 6984.242, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What are the benefits of using Noveum Trace?", - "query_length": 44, - "query_type": "rag_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 6.984165906906128, - "model.latency_ms": 6984.165906906128, - "model.start_time": 1761152258.9012191, - "model.end_time": 1761152265.885385, - "model.performance_tier": "slow", - "model.prompt_tokens": 828, - "model.completion_tokens": 349, - "model.total_tokens": 1177, - "model.tokens_per_second": 168.524060809631, - "model.estimated_cost": 0.03531, - "model.efficiency_score": 1.1860662701784197, - "model.response_length": 1396, - "model.response_quality": "high", - "model.output_response": "Model Response: Using Noveum Trace offers several benefits, including:\n\n1. **Real-Time Monitoring**: The Noveum platform provides a real-time dashboard for analyzing traces and performance, with live updates as new t...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "823b7927-10bb-49ef-a94e-1b914c298bfb", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:38.501000000", - "trace_end_time": "2025-10-22 16:57:46.469000000", - "trace_duration_ms": 7968.149, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:38.501000000", - "updated_at": "2025-10-22 16:57:38.501000000" - }, - { - "span_id": "5b1548cf-0de7-49d4-b9b8-9ad1f9f2abc9", - "trace_id": "823b7927-10bb-49ef-a94e-1b914c298bfb", - "parent_span_id": "67292076-ec8a-4856-9c5b-726868552157", - "name": "agent.rag_evaluation_metrics", - "start_time": "2025-10-22 16:57:45.885000000", - "end_time": "2025-10-22 16:57:45.885000000", - "duration_ms": 0.061, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "rag_evaluation_metrics", - "agent.capabilities": [ - "retrieval_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What are the benefits of using Noveum Trace?", - "query_length": 44, - "query_type": "rag_evaluation_query", - "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/docs/platform/dashboard):\n- Bulk Export: Download multiple traces simultaneously\n- Comparative Analysis: Compare performance across multiple traces\nReal-Time Updates\n- Live Refresh: Automatic updates as new traces arrive\n- Connection Monitoring: Real-time status of your t...", - "retrieval.context_length": 2773, - "retrieval.sources_count": 5, - "retrieval.context_quality": "high", - "retrieval.effectiveness": 1, - "retrieval.context_utilization": 2.773, - "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/docs/platform/dashboard):\n- Bulk Export: Download multiple traces simultaneously\n- Comparative Analysis: Compare performance across multiple traces\nReal-Time Updates\n- Live Refresh: Automatic updates as new traces arrive\n- Connection Monitoring: Real-time status of your trace ingestion pipeline\n- Performance Indicators: Live metrics for system health monitoring\nReady to dive deeper? Explore Projects & Environments to organize your AI applications, or check out Team Collaboration to share insights with your team.\nGet Earl...\n\nSource 2 (https://noveum.ai/en/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\n\ud83d\udcd8 TypeScript SDK (\n@noveum/trace\n)- Framework integrations for Next.js, Express.js, and Hono\n- TypeScript-first design with full type safety\n- Universal compatibility (Node.js, Edge Runtime, browsers)\n-\n\ud83d\udcca Noveum Platform\n- Real-time dashboard for analyzing traces and performance\n- Ad...\n\nSource 3 (https://noveum.ai/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\n\ud83d\udcd8 TypeScript SDK (\n@noveum/trace\n)- Framework integrations for Next.js, Express.js, and Hono\n- TypeScript-first design with full type safety\n- Universal compatibility (Node.js, Edge Runtime, browsers)\n-\n\ud83d\udcca Noveum Platform\n- Real-time dashboard for analyzing traces and performance\n- Ad...\n\nSource 4 (https://noveum.ai/en/docs/getting-started/overview):\n- Advanced search & filtering across traces and spans\n- Cost analysis and optimization recommendations\n- Team collaboration with shared insights and alerts\n\ud83d\udd0d What Noveum Traces\nLLM Operations\n- Model calls across all providers (OpenAI, Anthropic, Google, etc.)\n- Token usage and cost calculation\n- Prompt engineering effectiveness\n- Response quality metrics\nRAG Pipelines\n- Document retrieval performance and relevance\n- Embedding generation costs and latency\n- Context assembly and prompt constructi...\n\nSource 5 (https://noveum.ai/docs/getting-started/overview):\n- Advanced search & filtering across traces and spans\n- Cost analysis and optimization recommendations\n- Team collaboration with shared insights and alerts\n\ud83d\udd0d What Noveum Traces\nLLM Operations\n- Model calls across all providers (OpenAI, Anthropic, Google, etc.)\n- Token usage and cost calculation\n- Prompt engineering effectiveness\n- Response quality metrics\nRAG Pipelines\n- Document retrieval performance and relevance\n- Embedding generation costs and latency\n- Context assembly and prompt constructi...\n\nUser Question: What are the benefits of using Noveum Trace?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", - "prompt.prompt_length": 3313, - "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/docs/platform/dashboard):\n- Bulk Export: Download multiple traces simultaneously\n- Comparative Analysis: Compare performance across multiple traces\nReal-Time Updates\n- Live...", - "prompt.instruction_following": "rag_optimized", - "response.answer_length": 1396, - "response.answer_completeness": "complete", - "response.response_quality": "high", - "response.source_citation": 5, - "response.context_utilization": 0.5034258925351605, - "output_response": "RAG Answer: Using Noveum Trace offers several benefits, including:\n\n1. **Real-Time Monitoring**: The Noveum platform provides a real-time dashboard for analyzing traces and performance, with live updates as new t...", - "evaluation.retrieval_effectiveness": 1, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 5, - "evaluation.context_relevance": "high", - "evaluation.overall_quality": "high", - "evaluation.ready_for_production": true, - "rag.retrieval_strategy": "semantic_similarity", - "rag.vector_search_results": 5, - "rag.context_synthesis": "multi_source", - "rag.document_coverage": 1, - "rag.information_density": 0.5034258925351605 - }, - "events": [], - "links": [], - "trace_trace_id": "823b7927-10bb-49ef-a94e-1b914c298bfb", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:38.501000000", - "trace_end_time": "2025-10-22 16:57:46.469000000", - "trace_duration_ms": 7968.149, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:38.501000000", - "updated_at": "2025-10-22 16:57:38.501000000" - }, - { - "span_id": "aaba719b-36ec-46ab-af2e-3ea8c18e9872", - "trace_id": "1ae0d6c4-d1fb-4239-b536-3333187d44aa", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-22 16:57:32.800000000", - "end_time": "2025-10-22 16:57:37.494000000", - "duration_ms": 4694.408, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "How do I configure Noveum for my system?", - "query.length": 40, - "processing.mode": "RAG", - "processing.time_seconds": 4.694374084472656, - "processing.response_length": 553, - "processing.sources_count": 5, - "output_response": "Final Answer: To configure Noveum for your system, you can refer to the SDK Integration Guide available in the Noveum documentation. This guide will help you trace your AI applications with minimal code changes. Th...", - "final_answer_mode": "RAG", - "query_processed.input_query": "How do I configure Noveum for my system?", - "query_processed.output_response": "Successfully processed query using RAG, generated 553 character response", - "query_processed.mode": "RAG", - "query_processed.processing_time": 4.694374084472656, - "query_processed.response_length": 553 - }, - "events": [], - "links": [], - "trace_trace_id": "1ae0d6c4-d1fb-4239-b536-3333187d44aa", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:32.800000000", - "trace_end_time": "2025-10-22 16:57:37.494000000", - "trace_duration_ms": 4694.468, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:32.800000000", - "updated_at": "2025-10-22 16:57:32.800000000" - }, - { - "span_id": "4254e6f9-10b8-42ba-939d-d7c124f80f9a", - "trace_id": "1ae0d6c4-d1fb-4239-b536-3333187d44aa", - "parent_span_id": "6d5d074e-8fd3-4d33-88fb-7f54b0959030", - "name": "agent.llm-rag", - "start_time": "2025-10-22 16:57:32.800000000", - "end_time": "2025-10-22 16:57:37.494000000", - "duration_ms": 4694.155000000001, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "rag_agent", - "agent.operation": "llm-rag", - "agent.capabilities": [ - "document_retrieval", - "context_generation", - "response_generation" - ], - "agent.id": "noveum_rag_agent", - "input_query": "How do I configure Noveum for my system?", - "query_length": 40, - "query_type": "rag_query", - "output_response": "RAG Answer: To configure Noveum for your system, you can refer to the SDK Integration Guide available in the Noveum documentation. This guide will help you trace your AI applications with minimal code changes. Th...", - "rag.context_length": 2237, - "rag.sources_count": 5, - "rag.answer_length": 553, - "rag.mode": "retrieval_augmented_generation" - }, - "events": [], - "links": [], - "trace_trace_id": "1ae0d6c4-d1fb-4239-b536-3333187d44aa", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:32.800000000", - "trace_end_time": "2025-10-22 16:57:37.494000000", - "trace_duration_ms": 4694.468, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:32.800000000", - "updated_at": "2025-10-22 16:57:32.800000000" - }, - { - "span_id": "6d5d074e-8fd3-4d33-88fb-7f54b0959030", - "trace_id": "1ae0d6c4-d1fb-4239-b536-3333187d44aa", - "parent_span_id": "aaba719b-36ec-46ab-af2e-3ea8c18e9872", - "name": "agent.query_routing", - "start_time": "2025-10-22 16:57:32.800000000", - "end_time": "2025-10-22 16:57:37.494000000", - "duration_ms": 4694.318, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "How do I configure Noveum for my system?", - "query_length": 40, - "query_type": "routing_query", - "output_response": "Routed to RAG for query processing", - "router.classification": "RAG", - "router.confidence_score": 1, - "router.rag_keyword_score": 2, - "router.web_keyword_score": 0, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "1ae0d6c4-d1fb-4239-b536-3333187d44aa", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:32.800000000", - "trace_end_time": "2025-10-22 16:57:37.494000000", - "trace_duration_ms": 4694.468, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:32.800000000", - "updated_at": "2025-10-22 16:57:32.800000000" - }, - { - "span_id": "2cc2ab0b-2572-4a75-88a3-18b3a1eaca78", - "trace_id": "1ae0d6c4-d1fb-4239-b536-3333187d44aa", - "parent_span_id": "6d5d074e-8fd3-4d33-88fb-7f54b0959030", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-22 16:57:32.800000000", - "end_time": "2025-10-22 16:57:32.800000000", - "duration_ms": 0.04, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: How do I configure Noveum for my system?", - "query_length": 40, - "query_type": "routing_evaluation_query", - "classification.mode": "RAG", - "classification.rag_keyword_score": 2, - "classification.web_keyword_score": 0, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "noveum_specific", - "query.keyword_density": 0.25, - "query.domain_affinity": "noveum", - "routing.decision": "Routed to RAG based on analysis", - "routing.rationale": "RAG score: 2, Web score: 0, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "Web Search", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.038461538461538464, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: RAG (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "1ae0d6c4-d1fb-4239-b536-3333187d44aa", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:32.800000000", - "trace_end_time": "2025-10-22 16:57:37.494000000", - "trace_duration_ms": 4694.468, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:32.800000000", - "updated_at": "2025-10-22 16:57:32.800000000" - }, - { - "span_id": "f1d9f779-3312-43c1-b549-00fc874429f0", - "trace_id": "1ae0d6c4-d1fb-4239-b536-3333187d44aa", - "parent_span_id": "4254e6f9-10b8-42ba-939d-d7c124f80f9a", - "name": "agent.llm_model_execution", - "start_time": "2025-10-22 16:57:33.281000000", - "end_time": "2025-10-22 16:57:37.060000000", - "duration_ms": 3778.743, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: How do I configure Noveum for my system?", - "query_length": 40, - "query_type": "rag_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 3.7786097526550293, - "model.latency_ms": 3778.6097526550293, - "model.start_time": 1761152253.28163, - "model.end_time": 1761152257.0602398, - "model.performance_tier": "medium", - "model.prompt_tokens": 693, - "model.completion_tokens": 138, - "model.total_tokens": 831, - "model.tokens_per_second": 219.92215507730066, - "model.estimated_cost": 0.02493, - "model.efficiency_score": 0.6654632972322503, - "model.response_length": 553, - "model.response_quality": "high", - "model.output_response": "Model Response: To configure Noveum for your system, you can refer to the SDK Integration Guide available in the Noveum documentation. This guide will help you trace your AI applications with minimal code changes. Th...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "1ae0d6c4-d1fb-4239-b536-3333187d44aa", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:32.800000000", - "trace_end_time": "2025-10-22 16:57:37.494000000", - "trace_duration_ms": 4694.468, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:32.800000000", - "updated_at": "2025-10-22 16:57:32.800000000" - }, - { - "span_id": "5500be90-0cfa-44dd-a6e3-e72ac9f0a79e", - "trace_id": "1ae0d6c4-d1fb-4239-b536-3333187d44aa", - "parent_span_id": "4254e6f9-10b8-42ba-939d-d7c124f80f9a", - "name": "agent.rag_evaluation_metrics", - "start_time": "2025-10-22 16:57:37.060000000", - "end_time": "2025-10-22 16:57:37.060000000", - "duration_ms": 0.076, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "rag_evaluation_metrics", - "agent.capabilities": [ - "retrieval_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: How do I configure Noveum for my system?", - "query_length": 40, - "query_type": "rag_evaluation_query", - "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/docs):\nWelcome to Noveum.ai Docs\nComprehensive AI tracing and observability for LLM applications, RAG systems, and AI agents\nWelcome to the Noveum.ai documentation! Here you'll find everything you need to integrate, configure, and optimize your AI applications using ou...", - "retrieval.context_length": 2237, - "retrieval.sources_count": 5, - "retrieval.context_quality": "high", - "retrieval.effectiveness": 1, - "retrieval.context_utilization": 2.237, - "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/docs):\nWelcome to Noveum.ai Docs\nComprehensive AI tracing and observability for LLM applications, RAG systems, and AI agents\nWelcome to the Noveum.ai documentation! Here you'll find everything you need to integrate, configure, and optimize your AI applications using our comprehensive tracing and observability platform.\nWe provide powerful SDKs for Python and TypeScript that enable you to trace LLM calls, RAG pipelines, and multi-agent workflows with minimal code changes. Our documentation is organized ...\n\nSource 2 (https://noveum.ai/docs):\nWelcome to Noveum.ai Docs\nComprehensive AI tracing and observability for LLM applications, RAG systems, and AI agents\nWelcome to the Noveum.ai documentation! Here you'll find everything you need to integrate, configure, and optimize your AI applications using our comprehensive tracing and observability platform.\nWe provide powerful SDKs for Python and TypeScript that enable you to trace LLM calls, RAG pipelines, and multi-agent workflows with minimal code changes. Our documentation is organized ...\n\nSource 3 (https://noveum.ai/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nSource 4 (https://noveum.ai/en/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nSource 5 (https://noveum.ai/en/docs):\n- \ud83c\udf0d Universal: Works across frameworks, clouds, and deployment models\n\ud83e\udd1d Community & Support\n- \ud83d\udcac Discord Community: Join our Discord\n- \ud83d\udce7 Email Support: [email protected]\n- \ud83d\udc1b Bug Reports: GitHub Issues\n- \ud83d\udcd6 Knowledge Base: Help Center\nReady to get started? Head to our SDK Integration Guide to begin tracing your AI applications in under 5 minutes!\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability ...\n\nUser Question: How do I configure Noveum for my system?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", - "prompt.prompt_length": 2773, - "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/docs):\nWelcome to Noveum.ai Docs\nComprehensive AI tracing and observability for LLM applications, RAG systems, and AI agents\nWelcome to the Noveum.ai documentation! Here...", - "prompt.instruction_following": "rag_optimized", - "response.answer_length": 553, - "response.answer_completeness": "complete", - "response.response_quality": "high", - "response.source_citation": 5, - "response.context_utilization": 0.24720607957085383, - "output_response": "RAG Answer: To configure Noveum for your system, you can refer to the SDK Integration Guide available in the Noveum documentation. This guide will help you trace your AI applications with minimal code changes. Th...", - "evaluation.retrieval_effectiveness": 1, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 5, - "evaluation.context_relevance": "high", - "evaluation.overall_quality": "high", - "evaluation.ready_for_production": true, - "rag.retrieval_strategy": "semantic_similarity", - "rag.vector_search_results": 5, - "rag.context_synthesis": "multi_source", - "rag.document_coverage": 1, - "rag.information_density": 0.24720607957085383 - }, - "events": [], - "links": [], - "trace_trace_id": "1ae0d6c4-d1fb-4239-b536-3333187d44aa", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:32.800000000", - "trace_end_time": "2025-10-22 16:57:37.494000000", - "trace_duration_ms": 4694.468, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:32.800000000", - "updated_at": "2025-10-22 16:57:32.800000000" - }, - { - "span_id": "3068013a-6844-4f3c-8ec2-854a7353591e", - "trace_id": "fb0d31fc-2465-4ecd-973c-4cd443d9befa", - "parent_span_id": "a80d8fab-d5dc-4417-a5e5-9a2a76ed7419", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-22 16:57:24.828000000", - "end_time": "2025-10-22 16:57:24.828000000", - "duration_ms": 0.038, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What monitoring capabilities does Noveum provide?", - "query_length": 49, - "query_type": "routing_evaluation_query", - "classification.mode": "RAG", - "classification.rag_keyword_score": 2, - "classification.web_keyword_score": 0, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "noveum_specific", - "query.keyword_density": 0.3333333333333333, - "query.domain_affinity": "noveum", - "routing.decision": "Routed to RAG based on analysis", - "routing.rationale": "RAG score: 2, Web score: 0, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "Web Search", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.038461538461538464, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: RAG (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "fb0d31fc-2465-4ecd-973c-4cd443d9befa", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:24.828000000", - "trace_end_time": "2025-10-22 16:57:31.793000000", - "trace_duration_ms": 6965.302000000001, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:24.828000000", - "updated_at": "2025-10-22 16:57:24.828000000" - }, - { - "span_id": "189b89e3-fbbd-41de-8521-efb6a68f62f2", - "trace_id": "fb0d31fc-2465-4ecd-973c-4cd443d9befa", - "parent_span_id": "a80d8fab-d5dc-4417-a5e5-9a2a76ed7419", - "name": "agent.llm-rag", - "start_time": "2025-10-22 16:57:24.828000000", - "end_time": "2025-10-22 16:57:31.793000000", - "duration_ms": 6964.955, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "rag_agent", - "agent.operation": "llm-rag", - "agent.capabilities": [ - "document_retrieval", - "context_generation", - "response_generation" - ], - "agent.id": "noveum_rag_agent", - "input_query": "What monitoring capabilities does Noveum provide?", - "query_length": 49, - "query_type": "rag_query", - "output_response": "RAG Answer: Noveum provides comprehensive monitoring capabilities for AI applications, including the ability to monitor, trace, and optimize AI agents across various frameworks such as LangChain, CrewAI, AutoGen,...", - "rag.context_length": 2740, - "rag.sources_count": 5, - "rag.answer_length": 582, - "rag.mode": "retrieval_augmented_generation" - }, - "events": [], - "links": [], - "trace_trace_id": "fb0d31fc-2465-4ecd-973c-4cd443d9befa", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:24.828000000", - "trace_end_time": "2025-10-22 16:57:31.793000000", - "trace_duration_ms": 6965.302000000001, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:24.828000000", - "updated_at": "2025-10-22 16:57:24.828000000" - }, - { - "span_id": "f5cffb7d-341b-4cde-b162-02208042463d", - "trace_id": "fb0d31fc-2465-4ecd-973c-4cd443d9befa", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-22 16:57:24.828000000", - "end_time": "2025-10-22 16:57:31.793000000", - "duration_ms": 6965.236, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What monitoring capabilities does Noveum provide?", - "query.length": 49, - "processing.mode": "RAG", - "processing.time_seconds": 6.965206146240234, - "processing.response_length": 582, - "processing.sources_count": 5, - "output_response": "Final Answer: Noveum provides comprehensive monitoring capabilities for AI applications, including the ability to monitor, trace, and optimize AI agents across various frameworks such as LangChain, CrewAI, AutoGen,...", - "final_answer_mode": "RAG", - "query_processed.input_query": "What monitoring capabilities does Noveum provide?", - "query_processed.output_response": "Successfully processed query using RAG, generated 582 character response", - "query_processed.mode": "RAG", - "query_processed.processing_time": 6.965206146240234, - "query_processed.response_length": 582 - }, - "events": [], - "links": [], - "trace_trace_id": "fb0d31fc-2465-4ecd-973c-4cd443d9befa", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:24.828000000", - "trace_end_time": "2025-10-22 16:57:31.793000000", - "trace_duration_ms": 6965.302000000001, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:24.828000000", - "updated_at": "2025-10-22 16:57:24.828000000" - }, - { - "span_id": "a80d8fab-d5dc-4417-a5e5-9a2a76ed7419", - "trace_id": "fb0d31fc-2465-4ecd-973c-4cd443d9befa", - "parent_span_id": "f5cffb7d-341b-4cde-b162-02208042463d", - "name": "agent.query_routing", - "start_time": "2025-10-22 16:57:24.828000000", - "end_time": "2025-10-22 16:57:31.793000000", - "duration_ms": 6965.17, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What monitoring capabilities does Noveum provide?", - "query_length": 49, - "query_type": "routing_query", - "output_response": "Routed to RAG for query processing", - "router.classification": "RAG", - "router.confidence_score": 1, - "router.rag_keyword_score": 2, - "router.web_keyword_score": 0, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "fb0d31fc-2465-4ecd-973c-4cd443d9befa", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:24.828000000", - "trace_end_time": "2025-10-22 16:57:31.793000000", - "trace_duration_ms": 6965.302000000001, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:24.828000000", - "updated_at": "2025-10-22 16:57:24.828000000" - }, - { - "span_id": "a2b632b8-fbcb-480f-9292-9f513c64df38", - "trace_id": "fb0d31fc-2465-4ecd-973c-4cd443d9befa", - "parent_span_id": "189b89e3-fbbd-41de-8521-efb6a68f62f2", - "name": "agent.llm_model_execution", - "start_time": "2025-10-22 16:57:27.824000000", - "end_time": "2025-10-22 16:57:31.155000000", - "duration_ms": 3330.989, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What monitoring capabilities does Noveum provide?", - "query_length": 49, - "query_type": "rag_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 3.3309478759765625, - "model.latency_ms": 3330.9478759765625, - "model.start_time": 1761152247.824474, - "model.end_time": 1761152251.155422, - "model.performance_tier": "medium", - "model.prompt_tokens": 821, - "model.completion_tokens": 145, - "model.total_tokens": 966, - "model.tokens_per_second": 290.00754018607677, - "model.estimated_cost": 0.028980000000000002, - "model.efficiency_score": 0.6024844720496895, - "model.response_length": 582, - "model.response_quality": "high", - "model.output_response": "Model Response: Noveum provides comprehensive monitoring capabilities for AI applications, including the ability to monitor, trace, and optimize AI agents across various frameworks such as LangChain, CrewAI, AutoGen,...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "fb0d31fc-2465-4ecd-973c-4cd443d9befa", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:24.828000000", - "trace_end_time": "2025-10-22 16:57:31.793000000", - "trace_duration_ms": 6965.302000000001, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:24.828000000", - "updated_at": "2025-10-22 16:57:24.828000000" - }, - { - "span_id": "40ff54c4-07ca-4b2d-a9b4-7e3ae7adcd81", - "trace_id": "fb0d31fc-2465-4ecd-973c-4cd443d9befa", - "parent_span_id": "189b89e3-fbbd-41de-8521-efb6a68f62f2", - "name": "agent.rag_evaluation_metrics", - "start_time": "2025-10-22 16:57:31.155000000", - "end_time": "2025-10-22 16:57:31.155000000", - "duration_ms": 0.018000000000000002, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "rag_evaluation_metrics", - "agent.capabilities": [ - "retrieval_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What monitoring capabilities does Noveum provide?", - "query_length": 49, - "query_type": "rag_evaluation_query", - "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en):\nMonitor all your AI Agents\nimprove AI Agents today\nNoveum.ai helps you monitor, trace, and optimize your AI applications.\nNoveum.ai works with any AI framework \u2013 LangChain, CrewAI, AutoGen, custom implementations, or direct LLM calls. One dashboard shows everything.\n...", - "retrieval.context_length": 2740, - "retrieval.sources_count": 5, - "retrieval.context_quality": "high", - "retrieval.effectiveness": 1, - "retrieval.context_utilization": 2.74, - "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en):\nMonitor all your AI Agents\nimprove AI Agents today\nNoveum.ai helps you monitor, trace, and optimize your AI applications.\nNoveum.ai works with any AI framework \u2013 LangChain, CrewAI, AutoGen, custom implementations, or direct LLM calls. One dashboard shows everything.\nMonitor, Evaluate, Improve Your AI Agents\nThe control plane for AI agents.\nMonitor Everything, Miss Nothing\nOur lightweight SDKs capture every trace and span across your AI agent ecosystem\u2014from simple LLM calls to complex multi-agent...\n\nSource 2 (https://noveum.ai/):\nMonitor all your AI Agents\nimprove AI Agents today\nNoveum.ai helps you monitor, trace, and optimize your AI applications.\nNoveum.ai works with any AI framework \u2013 LangChain, CrewAI, AutoGen, custom implementations, or direct LLM calls. One dashboard shows everything.\nMonitor, Evaluate, Improve Your AI Agents\nThe control plane for AI agents.\nMonitor Everything, Miss Nothing\nOur lightweight SDKs capture every trace and span across your AI agent ecosystem\u2014from simple LLM calls to complex multi-agent...\n\nSource 3 (https://noveum.ai):\nMonitor all your AI Agents\nimprove AI Agents today\nNoveum.ai helps you monitor, trace, and optimize your AI applications.\nNoveum.ai works with any AI framework \u2013 LangChain, CrewAI, AutoGen, custom implementations, or direct LLM calls. One dashboard shows everything.\nMonitor, Evaluate, Improve Your AI Agents\nThe control plane for AI agents.\nMonitor Everything, Miss Nothing\nOur lightweight SDKs capture every trace and span across your AI agent ecosystem\u2014from simple LLM calls to complex multi-agent...\n\nSource 4 (https://noveum.ai/en/docs/getting-started/overview):\nNoveum.ai Overview\nComprehensive AI tracing and observability platform for LLM applications, RAG systems, and AI agents\nWelcome to Noveum.ai\u2014the comprehensive tracing and observability platform built specifically for AI applications. Whether you're building LLM-powered chatbots, RAG systems, multi-agent workflows, or any AI-driven application, Noveum provides the insights you need to understand, debug, and optimize your systems.\n\ud83c\udfaf Why AI Applications Need Specialized Observability\nTraditional mo...\n\nSource 5 (https://noveum.ai/docs/getting-started/overview):\nNoveum.ai Overview\nComprehensive AI tracing and observability platform for LLM applications, RAG systems, and AI agents\nWelcome to Noveum.ai\u2014the comprehensive tracing and observability platform built specifically for AI applications. Whether you're building LLM-powered chatbots, RAG systems, multi-agent workflows, or any AI-driven application, Noveum provides the insights you need to understand, debug, and optimize your systems.\n\ud83c\udfaf Why AI Applications Need Specialized Observability\nTraditional mo...\n\nUser Question: What monitoring capabilities does Noveum provide?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", - "prompt.prompt_length": 3285, - "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en):\nMonitor all your AI Agents\nimprove AI Agents today\nNoveum.ai helps you monitor, trace, and optimize your AI applications.\nNoveum.ai works with any AI framework \u2013 LangC...", - "prompt.instruction_following": "rag_optimized", - "response.answer_length": 582, - "response.answer_completeness": "complete", - "response.response_quality": "high", - "response.source_citation": 5, - "response.context_utilization": 0.2124087591240876, - "output_response": "RAG Answer: Noveum provides comprehensive monitoring capabilities for AI applications, including the ability to monitor, trace, and optimize AI agents across various frameworks such as LangChain, CrewAI, AutoGen,...", - "evaluation.retrieval_effectiveness": 1, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 5, - "evaluation.context_relevance": "high", - "evaluation.overall_quality": "high", - "evaluation.ready_for_production": true, - "rag.retrieval_strategy": "semantic_similarity", - "rag.vector_search_results": 5, - "rag.context_synthesis": "multi_source", - "rag.document_coverage": 1, - "rag.information_density": 0.2124087591240876 - }, - "events": [], - "links": [], - "trace_trace_id": "fb0d31fc-2465-4ecd-973c-4cd443d9befa", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:24.828000000", - "trace_end_time": "2025-10-22 16:57:31.793000000", - "trace_duration_ms": 6965.302000000001, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:24.828000000", - "updated_at": "2025-10-22 16:57:24.828000000" - }, - { - "span_id": "b0f0a2e0-c1f7-4ba4-8b75-7090e931a738", - "trace_id": "3bbd0b36-c30f-4126-b62a-dd0fae1c701d", - "parent_span_id": "4db80813-e62f-48ec-9091-62d569527caf", - "name": "agent.query_routing", - "start_time": "2025-10-22 16:57:19.339000000", - "end_time": "2025-10-22 16:57:23.822000000", - "duration_ms": 4482.909000000001, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "How does Noveum handle agent tracing?", - "query_length": 37, - "query_type": "routing_query", - "output_response": "Routed to RAG for query processing", - "router.classification": "RAG", - "router.confidence_score": 1, - "router.rag_keyword_score": 2, - "router.web_keyword_score": 0, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "3bbd0b36-c30f-4126-b62a-dd0fae1c701d", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:19.339000000", - "trace_end_time": "2025-10-22 16:57:23.822000000", - "trace_duration_ms": 4483.158, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:19.339000000", - "updated_at": "2025-10-22 16:57:19.339000000" - }, - { - "span_id": "24da2221-b0ab-4161-98ad-e7b5775fcb27", - "trace_id": "3bbd0b36-c30f-4126-b62a-dd0fae1c701d", - "parent_span_id": "b0f0a2e0-c1f7-4ba4-8b75-7090e931a738", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-22 16:57:19.339000000", - "end_time": "2025-10-22 16:57:19.339000000", - "duration_ms": 0.026, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: How does Noveum handle agent tracing?", - "query_length": 37, - "query_type": "routing_evaluation_query", - "classification.mode": "RAG", - "classification.rag_keyword_score": 2, - "classification.web_keyword_score": 0, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "noveum_specific", - "query.keyword_density": 0.3333333333333333, - "query.domain_affinity": "noveum", - "routing.decision": "Routed to RAG based on analysis", - "routing.rationale": "RAG score: 2, Web score: 0, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "Web Search", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.038461538461538464, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: RAG (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "3bbd0b36-c30f-4126-b62a-dd0fae1c701d", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:19.339000000", - "trace_end_time": "2025-10-22 16:57:23.822000000", - "trace_duration_ms": 4483.158, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:19.339000000", - "updated_at": "2025-10-22 16:57:19.339000000" - }, - { - "span_id": "d2b495eb-7e07-48a3-aa6d-1835aab18c11", - "trace_id": "3bbd0b36-c30f-4126-b62a-dd0fae1c701d", - "parent_span_id": "b0f0a2e0-c1f7-4ba4-8b75-7090e931a738", - "name": "agent.llm-rag", - "start_time": "2025-10-22 16:57:19.339000000", - "end_time": "2025-10-22 16:57:23.822000000", - "duration_ms": 4482.564, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "rag_agent", - "agent.operation": "llm-rag", - "agent.capabilities": [ - "document_retrieval", - "context_generation", - "response_generation" - ], - "agent.id": "noveum_rag_agent", - "input_query": "How does Noveum handle agent tracing?", - "query_length": 37, - "query_type": "rag_query", - "output_response": "RAG Answer: Noveum.ai handles agent tracing through its specialized Multi-Agent Tracing capabilities, which are designed to observe complex workflows and inter-agent communications within multi-agent systems. The...", - "rag.context_length": 2736, - "rag.sources_count": 5, - "rag.answer_length": 663, - "rag.mode": "retrieval_augmented_generation" - }, - "events": [], - "links": [], - "trace_trace_id": "3bbd0b36-c30f-4126-b62a-dd0fae1c701d", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:19.339000000", - "trace_end_time": "2025-10-22 16:57:23.822000000", - "trace_duration_ms": 4483.158, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:19.339000000", - "updated_at": "2025-10-22 16:57:19.339000000" - }, - { - "span_id": "4db80813-e62f-48ec-9091-62d569527caf", - "trace_id": "3bbd0b36-c30f-4126-b62a-dd0fae1c701d", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-22 16:57:19.339000000", - "end_time": "2025-10-22 16:57:23.822000000", - "duration_ms": 4483.031000000001, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "How does Noveum handle agent tracing?", - "query.length": 37, - "processing.mode": "RAG", - "processing.time_seconds": 4.482964038848877, - "processing.response_length": 663, - "processing.sources_count": 5, - "output_response": "Final Answer: Noveum.ai handles agent tracing through its specialized Multi-Agent Tracing capabilities, which are designed to observe complex workflows and inter-agent communications within multi-agent systems. The...", - "final_answer_mode": "RAG", - "query_processed.input_query": "How does Noveum handle agent tracing?", - "query_processed.output_response": "Successfully processed query using RAG, generated 663 character response", - "query_processed.mode": "RAG", - "query_processed.processing_time": 4.482964038848877, - "query_processed.response_length": 663 - }, - "events": [], - "links": [], - "trace_trace_id": "3bbd0b36-c30f-4126-b62a-dd0fae1c701d", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:19.339000000", - "trace_end_time": "2025-10-22 16:57:23.822000000", - "trace_duration_ms": 4483.158, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:19.339000000", - "updated_at": "2025-10-22 16:57:19.339000000" - }, - { - "span_id": "b80cb5cd-24f2-484d-bbff-9febe2aa3ffe", - "trace_id": "3bbd0b36-c30f-4126-b62a-dd0fae1c701d", - "parent_span_id": "d2b495eb-7e07-48a3-aa6d-1835aab18c11", - "name": "agent.llm_model_execution", - "start_time": "2025-10-22 16:57:19.958000000", - "end_time": "2025-10-22 16:57:22.709000000", - "duration_ms": 2750.9480000000003, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: How does Noveum handle agent tracing?", - "query_length": 37, - "query_type": "rag_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 2.750890016555786, - "model.latency_ms": 2750.890016555786, - "model.start_time": 1761152239.958973, - "model.end_time": 1761152242.709863, - "model.performance_tier": "medium", - "model.prompt_tokens": 817, - "model.completion_tokens": 165, - "model.total_tokens": 982, - "model.tokens_per_second": 356.975376728983, - "model.estimated_cost": 0.02946, - "model.efficiency_score": 0.675152749490835, - "model.response_length": 663, - "model.response_quality": "high", - "model.output_response": "Model Response: Noveum.ai handles agent tracing through its specialized Multi-Agent Tracing capabilities, which are designed to observe complex workflows and inter-agent communications within multi-agent systems. The...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "3bbd0b36-c30f-4126-b62a-dd0fae1c701d", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:19.339000000", - "trace_end_time": "2025-10-22 16:57:23.822000000", - "trace_duration_ms": 4483.158, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:19.339000000", - "updated_at": "2025-10-22 16:57:19.339000000" - }, - { - "span_id": "d85fcf7d-7eaa-4299-92fc-c969fab63a6e", - "trace_id": "3bbd0b36-c30f-4126-b62a-dd0fae1c701d", - "parent_span_id": "d2b495eb-7e07-48a3-aa6d-1835aab18c11", - "name": "agent.rag_evaluation_metrics", - "start_time": "2025-10-22 16:57:22.710000000", - "end_time": "2025-10-22 16:57:22.710000000", - "duration_ms": 0.036000000000000004, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "rag_evaluation_metrics", - "agent.capabilities": [ - "retrieval_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: How does Noveum handle agent tracing?", - "query_length": 37, - "query_type": "rag_evaluation_query", - "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, vector searches, data retrieval, agent reasoning, and more. Without proper observability, debugging ...", - "retrieval.context_length": 2736, - "retrieval.sources_count": 5, - "retrieval.context_quality": "high", - "retrieval.effectiveness": 1, - "retrieval.context_utilization": 2.736, - "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, vector searches, data retrieval, agent reasoning, and more. Without proper observability, debugging becomes a nightmare, optimization is guesswork, and understanding user interactions is nearly impossible.\nNoveum.ai solves this by providing comprehensive tracing and observability specifically designed for AI applications.\nKey benefits of using Noveum.ai:\n- Complete Visibility: Trace...\n\nSource 2 (https://noveum.ai/docs/advanced/multi-agent-tracing):\nMulti-Agent Tracing\nObserve complex agent workflows and inter-agent communications with comprehensive tracing\nMulti-agent systems represent some of the most complex AI applications, involving multiple agents that coordinate, communicate, and collaborate to achieve shared goals. Noveum.ai provides specialized tracing capabilities to help you understand and optimize these intricate workflows.\n\ud83c\udfaf Why Multi-Agent Tracing Matters\nMulti-agent systems introduce unique observability challenges:\n- Complex...\n\nSource 3 (https://noveum.ai/en/docs/advanced/multi-agent-tracing):\nMulti-Agent Tracing\nObserve complex agent workflows and inter-agent communications with comprehensive tracing\nMulti-agent systems represent some of the most complex AI applications, involving multiple agents that coordinate, communicate, and collaborate to achieve shared goals. Noveum.ai provides specialized tracing capabilities to help you understand and optimize these intricate workflows.\n\ud83c\udfaf Why Multi-Agent Tracing Matters\nMulti-agent systems introduce unique observability challenges:\n- Complex...\n\nSource 4 (https://noveum.ai/en/docs/advanced/performance-optimization):\n\ud83d\udd17 Next Steps\n- Multi-Agent Tracing - Observe agent workflows\n- RAG Pipeline Observability - Monitor retrieval and generation systems\n- Custom Instrumentation - Add domain-specific tracing\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nSource 5 (https://noveum.ai/docs/advanced/performance-optimization):\n\ud83d\udd17 Next Steps\n- Multi-Agent Tracing - Observe agent workflows\n- RAG Pipeline Observability - Monitor retrieval and generation systems\n- Custom Instrumentation - Add domain-specific tracing\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nUser Question: How does Noveum handle agent tracing?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", - "prompt.prompt_length": 3269, - "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, ...", - "prompt.instruction_following": "rag_optimized", - "response.answer_length": 663, - "response.answer_completeness": "complete", - "response.response_quality": "high", - "response.source_citation": 5, - "response.context_utilization": 0.24232456140350878, - "output_response": "RAG Answer: Noveum.ai handles agent tracing through its specialized Multi-Agent Tracing capabilities, which are designed to observe complex workflows and inter-agent communications within multi-agent systems. The...", - "evaluation.retrieval_effectiveness": 1, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 5, - "evaluation.context_relevance": "high", - "evaluation.overall_quality": "high", - "evaluation.ready_for_production": true, - "rag.retrieval_strategy": "semantic_similarity", - "rag.vector_search_results": 5, - "rag.context_synthesis": "multi_source", - "rag.document_coverage": 1, - "rag.information_density": 0.24232456140350878 - }, - "events": [], - "links": [], - "trace_trace_id": "3bbd0b36-c30f-4126-b62a-dd0fae1c701d", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:19.339000000", - "trace_end_time": "2025-10-22 16:57:23.822000000", - "trace_duration_ms": 4483.158, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:19.339000000", - "updated_at": "2025-10-22 16:57:19.339000000" - }, - { - "span_id": "93d82f68-6c62-40de-a675-c4d7c40c1d01", - "trace_id": "fee3e6d3-8da1-4ef3-8214-a7d7323d3a49", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-22 16:57:13.334000000", - "end_time": "2025-10-22 16:57:18.330000000", - "duration_ms": 4996.436, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What APIs are available in Noveum platform?", - "query.length": 43, - "processing.mode": "RAG", - "processing.time_seconds": 4.996389865875244, - "processing.response_length": 270, - "processing.sources_count": 5, - "output_response": "Final Answer: The provided context does not specify the APIs available in the Noveum platform. For detailed information about the APIs, I recommend checking the official Noveum documentation or the SDK Integration ...", - "final_answer_mode": "RAG", - "query_processed.input_query": "What APIs are available in Noveum platform?", - "query_processed.output_response": "Successfully processed query using RAG, generated 270 character response", - "query_processed.mode": "RAG", - "query_processed.processing_time": 4.996389865875244, - "query_processed.response_length": 270 - }, - "events": [], - "links": [], - "trace_trace_id": "fee3e6d3-8da1-4ef3-8214-a7d7323d3a49", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:13.334000000", - "trace_end_time": "2025-10-22 16:57:18.331000000", - "trace_duration_ms": 4996.502, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:13.334000000", - "updated_at": "2025-10-22 16:57:13.334000000" - }, - { - "span_id": "1be727c6-14e8-4b2e-b8df-a1cab902454f", - "trace_id": "fee3e6d3-8da1-4ef3-8214-a7d7323d3a49", - "parent_span_id": "93d82f68-6c62-40de-a675-c4d7c40c1d01", - "name": "agent.query_routing", - "start_time": "2025-10-22 16:57:13.334000000", - "end_time": "2025-10-22 16:57:18.330000000", - "duration_ms": 4996.354, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What APIs are available in Noveum platform?", - "query_length": 43, - "query_type": "routing_query", - "output_response": "Routed to RAG for query processing", - "router.classification": "RAG", - "router.confidence_score": 1, - "router.rag_keyword_score": 3, - "router.web_keyword_score": 0, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "fee3e6d3-8da1-4ef3-8214-a7d7323d3a49", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:13.334000000", - "trace_end_time": "2025-10-22 16:57:18.331000000", - "trace_duration_ms": 4996.502, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:13.334000000", - "updated_at": "2025-10-22 16:57:13.334000000" - }, - { - "span_id": "58ab1d39-d349-4164-8ff9-26ece4b1e1aa", - "trace_id": "fee3e6d3-8da1-4ef3-8214-a7d7323d3a49", - "parent_span_id": "1be727c6-14e8-4b2e-b8df-a1cab902454f", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-22 16:57:13.334000000", - "end_time": "2025-10-22 16:57:13.334000000", - "duration_ms": 0.041999999999999996, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What APIs are available in Noveum platform?", - "query_length": 43, - "query_type": "routing_evaluation_query", - "classification.mode": "RAG", - "classification.rag_keyword_score": 3, - "classification.web_keyword_score": 0, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "noveum_specific", - "query.keyword_density": 0.42857142857142855, - "query.domain_affinity": "noveum", - "routing.decision": "Routed to RAG based on analysis", - "routing.rationale": "RAG score: 3, Web score: 0, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "Web Search", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.057692307692307696, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: RAG (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "fee3e6d3-8da1-4ef3-8214-a7d7323d3a49", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:13.334000000", - "trace_end_time": "2025-10-22 16:57:18.331000000", - "trace_duration_ms": 4996.502, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:13.334000000", - "updated_at": "2025-10-22 16:57:13.334000000" - }, - { - "span_id": "f11ebf68-a416-484e-8e55-f77a11b9ea1c", - "trace_id": "fee3e6d3-8da1-4ef3-8214-a7d7323d3a49", - "parent_span_id": "1be727c6-14e8-4b2e-b8df-a1cab902454f", - "name": "agent.llm-rag", - "start_time": "2025-10-22 16:57:13.334000000", - "end_time": "2025-10-22 16:57:18.330000000", - "duration_ms": 4996.146000000001, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "rag_agent", - "agent.operation": "llm-rag", - "agent.capabilities": [ - "document_retrieval", - "context_generation", - "response_generation" - ], - "agent.id": "noveum_rag_agent", - "input_query": "What APIs are available in Noveum platform?", - "query_length": 43, - "query_type": "rag_query", - "output_response": "RAG Answer: The provided context does not specify the APIs available in the Noveum platform. For detailed information about the APIs, I recommend checking the official Noveum documentation or the SDK Integration ...", - "rag.context_length": 2284, - "rag.sources_count": 5, - "rag.answer_length": 270, - "rag.mode": "retrieval_augmented_generation" - }, - "events": [], - "links": [], - "trace_trace_id": "fee3e6d3-8da1-4ef3-8214-a7d7323d3a49", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:13.334000000", - "trace_end_time": "2025-10-22 16:57:18.331000000", - "trace_duration_ms": 4996.502, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:13.334000000", - "updated_at": "2025-10-22 16:57:13.334000000" - }, - { - "span_id": "11394488-8802-403d-a926-7fda3a510b61", - "trace_id": "fee3e6d3-8da1-4ef3-8214-a7d7323d3a49", - "parent_span_id": "f11ebf68-a416-484e-8e55-f77a11b9ea1c", - "name": "agent.llm_model_execution", - "start_time": "2025-10-22 16:57:13.862000000", - "end_time": "2025-10-22 16:57:16.049000000", - "duration_ms": 2187.29, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What APIs are available in Noveum platform?", - "query_length": 43, - "query_type": "rag_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 2.187239170074463, - "model.latency_ms": 2187.239170074463, - "model.start_time": 1761152233.862141, - "model.end_time": 1761152236.04938, - "model.performance_tier": "medium", - "model.prompt_tokens": 705, - "model.completion_tokens": 67, - "model.total_tokens": 772, - "model.tokens_per_second": 352.9563710098141, - "model.estimated_cost": 0.02316, - "model.efficiency_score": 0.34974093264248707, - "model.response_length": 270, - "model.response_quality": "high", - "model.output_response": "Model Response: The provided context does not specify the APIs available in the Noveum platform. For detailed information about the APIs, I recommend checking the official Noveum documentation or the SDK Integration ...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "fee3e6d3-8da1-4ef3-8214-a7d7323d3a49", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:13.334000000", - "trace_end_time": "2025-10-22 16:57:18.331000000", - "trace_duration_ms": 4996.502, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:13.334000000", - "updated_at": "2025-10-22 16:57:13.334000000" - }, - { - "span_id": "341cf8ab-cc33-4d57-952f-ba1065f0432e", - "trace_id": "fee3e6d3-8da1-4ef3-8214-a7d7323d3a49", - "parent_span_id": "f11ebf68-a416-484e-8e55-f77a11b9ea1c", - "name": "agent.rag_evaluation_metrics", - "start_time": "2025-10-22 16:57:16.049000000", - "end_time": "2025-10-22 16:57:16.049000000", - "duration_ms": 0.041, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "rag_evaluation_metrics", - "agent.capabilities": [ - "retrieval_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What APIs are available in Noveum platform?", - "query_length": 43, - "query_type": "rag_evaluation_query", - "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/docs):\n- \ud83c\udf0d Universal: Works across frameworks, clouds, and deployment models\n\ud83e\udd1d Community & Support\n- \ud83d\udcac Discord Community: Join our Discord\n- \ud83d\udce7 Email Support: [email protected]\n- \ud83d\udc1b Bug Reports: GitHub Issues\n- \ud83d\udcd6 Knowledge Base: Help Center\nReady to get started? Head to ...", - "retrieval.context_length": 2284, - "retrieval.sources_count": 5, - "retrieval.context_quality": "high", - "retrieval.effectiveness": 1, - "retrieval.context_utilization": 2.284, - "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/docs):\n- \ud83c\udf0d Universal: Works across frameworks, clouds, and deployment models\n\ud83e\udd1d Community & Support\n- \ud83d\udcac Discord Community: Join our Discord\n- \ud83d\udce7 Email Support: [email protected]\n- \ud83d\udc1b Bug Reports: GitHub Issues\n- \ud83d\udcd6 Knowledge Base: Help Center\nReady to get started? Head to our SDK Integration Guide to begin tracing your AI applications in under 5 minutes!\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability ...\n\nSource 2 (https://noveum.ai/docs):\n- \ud83c\udf0d Universal: Works across frameworks, clouds, and deployment models\n\ud83e\udd1d Community & Support\n- \ud83d\udcac Discord Community: Join our Discord\n- \ud83d\udce7 Email Support: [email protected]\n- \ud83d\udc1b Bug Reports: GitHub Issues\n- \ud83d\udcd6 Knowledge Base: Help Center\nReady to get started? Head to our SDK Integration Guide to begin tracing your AI applications in under 5 minutes!\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability ...\n\nSource 3 (https://noveum.ai/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nSource 4 (https://noveum.ai/en/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nSource 5 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, vector searches, data retrieval, agent reasoning, and more. Without proper observability, debugging becomes a nightmare, optimization is guesswork, and understanding user interactions is nearly impossible.\nNoveum.ai solves this by providing comprehensive tracing and observability specifically designed for AI applications.\nKey benefits of using Noveum.ai:\n- Complete Visibility: Trace...\n\nUser Question: What APIs are available in Noveum platform?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", - "prompt.prompt_length": 2823, - "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/docs):\n- \ud83c\udf0d Universal: Works across frameworks, clouds, and deployment models\n\ud83e\udd1d Community & Support\n- \ud83d\udcac Discord Community: Join our Discord\n- \ud83d\udce7 Email Support: [email prot...", - "prompt.instruction_following": "rag_optimized", - "response.answer_length": 270, - "response.answer_completeness": "complete", - "response.response_quality": "high", - "response.source_citation": 5, - "response.context_utilization": 0.11821366024518389, - "output_response": "RAG Answer: The provided context does not specify the APIs available in the Noveum platform. For detailed information about the APIs, I recommend checking the official Noveum documentation or the SDK Integration ...", - "evaluation.retrieval_effectiveness": 1, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 5, - "evaluation.context_relevance": "high", - "evaluation.overall_quality": "high", - "evaluation.ready_for_production": true, - "rag.retrieval_strategy": "semantic_similarity", - "rag.vector_search_results": 5, - "rag.context_synthesis": "multi_source", - "rag.document_coverage": 1, - "rag.information_density": 0.11821366024518389 - }, - "events": [], - "links": [], - "trace_trace_id": "fee3e6d3-8da1-4ef3-8214-a7d7323d3a49", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:13.334000000", - "trace_end_time": "2025-10-22 16:57:18.331000000", - "trace_duration_ms": 4996.502, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:13.334000000", - "updated_at": "2025-10-22 16:57:13.334000000" - }, - { - "span_id": "24930c28-7f93-426c-8914-bd0e0085589f", - "trace_id": "93bd37c4-4289-43c9-b34e-19eaaef610c2", - "parent_span_id": "bfc91518-f818-4f35-9113-70b04eb55edc", - "name": "agent.query_routing", - "start_time": "2025-10-22 16:57:06.605000000", - "end_time": "2025-10-22 16:57:12.327000000", - "duration_ms": 5721.561000000001, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "How do I set up observability with Noveum?", - "query_length": 42, - "query_type": "routing_query", - "output_response": "Routed to RAG for query processing", - "router.classification": "RAG", - "router.confidence_score": 1, - "router.rag_keyword_score": 2, - "router.web_keyword_score": 0, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "93bd37c4-4289-43c9-b34e-19eaaef610c2", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:06.605000000", - "trace_end_time": "2025-10-22 16:57:12.327000000", - "trace_duration_ms": 5721.707, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:06.605000000", - "updated_at": "2025-10-22 16:57:06.605000000" - }, - { - "span_id": "11fc18a8-0380-48ca-894e-cc26f648d290", - "trace_id": "93bd37c4-4289-43c9-b34e-19eaaef610c2", - "parent_span_id": "24930c28-7f93-426c-8914-bd0e0085589f", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-22 16:57:06.605000000", - "end_time": "2025-10-22 16:57:06.606000000", - "duration_ms": 0.047, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: How do I set up observability with Noveum?", - "query_length": 42, - "query_type": "routing_evaluation_query", - "classification.mode": "RAG", - "classification.rag_keyword_score": 2, - "classification.web_keyword_score": 0, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "noveum_specific", - "query.keyword_density": 0.25, - "query.domain_affinity": "noveum", - "routing.decision": "Routed to RAG based on analysis", - "routing.rationale": "RAG score: 2, Web score: 0, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "Web Search", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.038461538461538464, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: RAG (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "93bd37c4-4289-43c9-b34e-19eaaef610c2", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:06.605000000", - "trace_end_time": "2025-10-22 16:57:12.327000000", - "trace_duration_ms": 5721.707, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:06.605000000", - "updated_at": "2025-10-22 16:57:06.605000000" - }, - { - "span_id": "bfc91518-f818-4f35-9113-70b04eb55edc", - "trace_id": "93bd37c4-4289-43c9-b34e-19eaaef610c2", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-22 16:57:06.605000000", - "end_time": "2025-10-22 16:57:12.327000000", - "duration_ms": 5721.635, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "How do I set up observability with Noveum?", - "query.length": 42, - "processing.mode": "RAG", - "processing.time_seconds": 5.721601963043213, - "processing.response_length": 815, - "processing.sources_count": 5, - "output_response": "Final Answer: To set up observability with Noveum, follow these steps:\n\n1. **Sign Up**: Go to [noveum.ai](https://noveum.ai) and create an account.\n2. **Create a Project**: After signing up, create a project to obt...", - "final_answer_mode": "RAG", - "query_processed.input_query": "How do I set up observability with Noveum?", - "query_processed.output_response": "Successfully processed query using RAG, generated 815 character response", - "query_processed.mode": "RAG", - "query_processed.processing_time": 5.721601963043213, - "query_processed.response_length": 815 - }, - "events": [], - "links": [], - "trace_trace_id": "93bd37c4-4289-43c9-b34e-19eaaef610c2", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:06.605000000", - "trace_end_time": "2025-10-22 16:57:12.327000000", - "trace_duration_ms": 5721.707, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:06.605000000", - "updated_at": "2025-10-22 16:57:06.605000000" - }, - { - "span_id": "2a82e4b1-c93d-498f-9dd5-e971f228d642", - "trace_id": "93bd37c4-4289-43c9-b34e-19eaaef610c2", - "parent_span_id": "24930c28-7f93-426c-8914-bd0e0085589f", - "name": "agent.llm-rag", - "start_time": "2025-10-22 16:57:06.606000000", - "end_time": "2025-10-22 16:57:12.327000000", - "duration_ms": 5721.352000000001, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "rag_agent", - "agent.operation": "llm-rag", - "agent.capabilities": [ - "document_retrieval", - "context_generation", - "response_generation" - ], - "agent.id": "noveum_rag_agent", - "input_query": "How do I set up observability with Noveum?", - "query_length": 42, - "query_type": "rag_query", - "output_response": "RAG Answer: To set up observability with Noveum, follow these steps:\n\n1. **Sign Up**: Go to [noveum.ai](https://noveum.ai) and create an account.\n2. **Create a Project**: After signing up, create a project to obt...", - "rag.context_length": 2870, - "rag.sources_count": 6, - "rag.answer_length": 815, - "rag.mode": "retrieval_augmented_generation" - }, - "events": [], - "links": [], - "trace_trace_id": "93bd37c4-4289-43c9-b34e-19eaaef610c2", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:06.605000000", - "trace_end_time": "2025-10-22 16:57:12.327000000", - "trace_duration_ms": 5721.707, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:06.605000000", - "updated_at": "2025-10-22 16:57:06.605000000" - }, - { - "span_id": "520b4e59-da14-4db2-af02-0973cda9bd86", - "trace_id": "93bd37c4-4289-43c9-b34e-19eaaef610c2", - "parent_span_id": "2a82e4b1-c93d-498f-9dd5-e971f228d642", - "name": "agent.llm_model_execution", - "start_time": "2025-10-22 16:57:07.109000000", - "end_time": "2025-10-22 16:57:11.837000000", - "duration_ms": 4728.018999999999, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: How do I set up observability with Noveum?", - "query_length": 42, - "query_type": "rag_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 4.727963924407959, - "model.latency_ms": 4727.963924407959, - "model.start_time": 1761152227.109188, - "model.end_time": 1761152231.837152, - "model.performance_tier": "medium", - "model.prompt_tokens": 852, - "model.completion_tokens": 203, - "model.total_tokens": 1055, - "model.tokens_per_second": 223.1404504915101, - "model.estimated_cost": 0.03165, - "model.efficiency_score": 0.7725118483412322, - "model.response_length": 815, - "model.response_quality": "high", - "model.output_response": "Model Response: To set up observability with Noveum, follow these steps:\n\n1. **Sign Up**: Go to [noveum.ai](https://noveum.ai) and create an account.\n2. **Create a Project**: After signing up, create a project to obt...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "93bd37c4-4289-43c9-b34e-19eaaef610c2", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:06.605000000", - "trace_end_time": "2025-10-22 16:57:12.327000000", - "trace_duration_ms": 5721.707, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:06.605000000", - "updated_at": "2025-10-22 16:57:06.605000000" - }, - { - "span_id": "dff080ab-238c-4cca-a910-cf530f910bda", - "trace_id": "93bd37c4-4289-43c9-b34e-19eaaef610c2", - "parent_span_id": "2a82e4b1-c93d-498f-9dd5-e971f228d642", - "name": "agent.rag_evaluation_metrics", - "start_time": "2025-10-22 16:57:11.837000000", - "end_time": "2025-10-22 16:57:11.838000000", - "duration_ms": 0.259, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "rag_evaluation_metrics", - "agent.capabilities": [ - "retrieval_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: How do I set up observability with Noveum?", - "query_length": 42, - "query_type": "rag_evaluation_query", - "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\n- Production Ready: Built for scale with intelligent sampling and batching\n- Privacy Focused: You control what data is captured and how it's stored\n- Open Source SDKs: Transparent, auditable, and extensible\nGetting ...", - "retrieval.context_length": 2870, - "retrieval.sources_count": 6, - "retrieval.context_quality": "high", - "retrieval.effectiveness": 1.2, - "retrieval.context_utilization": 2.87, - "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\n- Production Ready: Built for scale with intelligent sampling and batching\n- Privacy Focused: You control what data is captured and how it's stored\n- Open Source SDKs: Transparent, auditable, and extensible\nGetting Started\nReady to add comprehensive observability to your AI applications?\n- Sign up at noveum.ai\n- Create a project and get your API key\n- Install the SDK for your preferred language\n- Add tracing to your AI workflows\n- Explore insights in the Noveum.ai dashboard\nCheck out our integra...\n\nSource 2 (https://noveum.ai/en/docs/getting-started/tracing-concepts):\n- Explore Framework Integrations - Framework-specific guidance\n- Learn Advanced Patterns - Custom instrumentation techniques\n- Master the Dashboard - Analyze your traces effectively\nRemember: Good observability is not about collecting all possible data, but about collecting the right data that helps you understand, debug, and optimize your AI applications.\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to O...\n\nSource 3 (https://noveum.ai/docs/getting-started/tracing-concepts):\n- Explore Framework Integrations - Framework-specific guidance\n- Learn Advanced Patterns - Custom instrumentation techniques\n- Master the Dashboard - Analyze your traces effectively\nRemember: Good observability is not about collecting all possible data, but about collecting the right data that helps you understand, debug, and optimize your AI applications.\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to O...\n\nSource 4 (https://noveum.ai/en/docs/getting-started/overview):\nNoveum.ai Overview\nComprehensive AI tracing and observability platform for LLM applications, RAG systems, and AI agents\nWelcome to Noveum.ai\u2014the comprehensive tracing and observability platform built specifically for AI applications. Whether you're building LLM-powered chatbots, RAG systems, multi-agent workflows, or any AI-driven application, Noveum provides the insights you need to understand, debug, and optimize your systems.\n\ud83c\udfaf Why AI Applications Need Specialized Observability\nTraditional mo...\n\nSource 5 (https://noveum.ai/docs/getting-started/overview):\nNoveum.ai Overview\nComprehensive AI tracing and observability platform for LLM applications, RAG systems, and AI agents\nWelcome to Noveum.ai\u2014the comprehensive tracing and observability platform built specifically for AI applications. Whether you're building LLM-powered chatbots, RAG systems, multi-agent workflows, or any AI-driven application, Noveum provides the insights you need to understand, debug, and optimize your systems.\n\ud83c\udfaf Why AI Applications Need Specialized Observability\nTraditional mo...\n\nUser Question: How do I set up observability with Noveum?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", - "prompt.prompt_length": 3408, - "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\n- Production Ready: Built for scale with intelligent sampling and batching\n- Privacy Focused: You control what data...", - "prompt.instruction_following": "rag_optimized", - "response.answer_length": 815, - "response.answer_completeness": "complete", - "response.response_quality": "high", - "response.source_citation": 6, - "response.context_utilization": 0.28397212543554007, - "output_response": "RAG Answer: To set up observability with Noveum, follow these steps:\n\n1. **Sign Up**: Go to [noveum.ai](https://noveum.ai) and create an account.\n2. **Create a Project**: After signing up, create a project to obt...", - "evaluation.retrieval_effectiveness": 1.2, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 6, - "evaluation.context_relevance": "high", - "evaluation.overall_quality": "high", - "evaluation.ready_for_production": true, - "rag.retrieval_strategy": "semantic_similarity", - "rag.vector_search_results": 6, - "rag.context_synthesis": "multi_source", - "rag.document_coverage": 1.2, - "rag.information_density": 0.28397212543554007 - }, - "events": [], - "links": [], - "trace_trace_id": "93bd37c4-4289-43c9-b34e-19eaaef610c2", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:06.605000000", - "trace_end_time": "2025-10-22 16:57:12.327000000", - "trace_duration_ms": 5721.707, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:06.605000000", - "updated_at": "2025-10-22 16:57:06.605000000" - }, - { - "span_id": "dad7df65-be49-4245-8721-c91da4962b9a", - "trace_id": "df0c6344-1a2e-4c53-bb7d-4e43b3bce1b8", - "parent_span_id": "9024b0d9-2b75-428f-af8f-75a921a7c69c", - "name": "agent.query_routing", - "start_time": "2025-10-22 16:56:59.213000000", - "end_time": "2025-10-22 16:57:05.598000000", - "duration_ms": 6384.249, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What features does Noveum Trace offer?", - "query_length": 38, - "query_type": "routing_query", - "output_response": "Routed to RAG for query processing", - "router.classification": "RAG", - "router.confidence_score": 1, - "router.rag_keyword_score": 3, - "router.web_keyword_score": 0, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "df0c6344-1a2e-4c53-bb7d-4e43b3bce1b8", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:56:59.213000000", - "trace_end_time": "2025-10-22 16:57:05.598000000", - "trace_duration_ms": 6384.386, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:56:59.213000000", - "updated_at": "2025-10-22 16:56:59.213000000" - }, - { - "span_id": "9024b0d9-2b75-428f-af8f-75a921a7c69c", - "trace_id": "df0c6344-1a2e-4c53-bb7d-4e43b3bce1b8", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-22 16:56:59.213000000", - "end_time": "2025-10-22 16:57:05.598000000", - "duration_ms": 6384.32, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What features does Noveum Trace offer?", - "query.length": 38, - "processing.mode": "RAG", - "processing.time_seconds": 6.384284973144531, - "processing.response_length": 1017, - "processing.sources_count": 5, - "output_response": "Final Answer: Noveum Trace offers several key features through its Python SDK (`noveum-trace`) and TypeScript SDK (`@noveum/trace`):\n\n1. **Decorator-based Tracing**: It provides a simple way to trace LLM calls, age...", - "final_answer_mode": "RAG", - "query_processed.input_query": "What features does Noveum Trace offer?", - "query_processed.output_response": "Successfully processed query using RAG, generated 1017 character response", - "query_processed.mode": "RAG", - "query_processed.processing_time": 6.384284973144531, - "query_processed.response_length": 1017 - }, - "events": [], - "links": [], - "trace_trace_id": "df0c6344-1a2e-4c53-bb7d-4e43b3bce1b8", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:56:59.213000000", - "trace_end_time": "2025-10-22 16:57:05.598000000", - "trace_duration_ms": 6384.386, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:56:59.213000000", - "updated_at": "2025-10-22 16:56:59.213000000" - }, - { - "span_id": "d07fdbdc-2b8f-4de9-ac32-988389701771", - "trace_id": "df0c6344-1a2e-4c53-bb7d-4e43b3bce1b8", - "parent_span_id": "dad7df65-be49-4245-8721-c91da4962b9a", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-22 16:56:59.214000000", - "end_time": "2025-10-22 16:56:59.214000000", - "duration_ms": 0.060000000000000005, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What features does Noveum Trace offer?", - "query_length": 38, - "query_type": "routing_evaluation_query", - "classification.mode": "RAG", - "classification.rag_keyword_score": 3, - "classification.web_keyword_score": 0, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "noveum_specific", - "query.keyword_density": 0.5, - "query.domain_affinity": "noveum", - "routing.decision": "Routed to RAG based on analysis", - "routing.rationale": "RAG score: 3, Web score: 0, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "Web Search", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.057692307692307696, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: RAG (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "df0c6344-1a2e-4c53-bb7d-4e43b3bce1b8", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:56:59.213000000", - "trace_end_time": "2025-10-22 16:57:05.598000000", - "trace_duration_ms": 6384.386, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:56:59.213000000", - "updated_at": "2025-10-22 16:56:59.213000000" - }, - { - "span_id": "83f86157-5af4-4c99-aad1-647866cd804e", - "trace_id": "df0c6344-1a2e-4c53-bb7d-4e43b3bce1b8", - "parent_span_id": "dad7df65-be49-4245-8721-c91da4962b9a", - "name": "agent.llm-rag", - "start_time": "2025-10-22 16:56:59.214000000", - "end_time": "2025-10-22 16:57:05.598000000", - "duration_ms": 6384.048, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "rag_agent", - "agent.operation": "llm-rag", - "agent.capabilities": [ - "document_retrieval", - "context_generation", - "response_generation" - ], - "agent.id": "noveum_rag_agent", - "input_query": "What features does Noveum Trace offer?", - "query_length": 38, - "query_type": "rag_query", - "output_response": "RAG Answer: Noveum Trace offers several key features through its Python SDK (`noveum-trace`) and TypeScript SDK (`@noveum/trace`):\n\n1. **Decorator-based Tracing**: It provides a simple way to trace LLM calls, age...", - "rag.context_length": 2787, - "rag.sources_count": 5, - "rag.answer_length": 1017, - "rag.mode": "retrieval_augmented_generation" - }, - "events": [], - "links": [], - "trace_trace_id": "df0c6344-1a2e-4c53-bb7d-4e43b3bce1b8", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:56:59.213000000", - "trace_end_time": "2025-10-22 16:57:05.598000000", - "trace_duration_ms": 6384.386, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:56:59.213000000", - "updated_at": "2025-10-22 16:56:59.213000000" - }, - { - "span_id": "3b4fa9a7-88bf-4ee1-aeb7-bfa9573f28f2", - "trace_id": "df0c6344-1a2e-4c53-bb7d-4e43b3bce1b8", - "parent_span_id": "83f86157-5af4-4c99-aad1-647866cd804e", - "name": "agent.llm_model_execution", - "start_time": "2025-10-22 16:56:59.591000000", - "end_time": "2025-10-22 16:57:04.869000000", - "duration_ms": 5278.925, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What features does Noveum Trace offer?", - "query_length": 38, - "query_type": "rag_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 5.278868913650513, - "model.latency_ms": 5278.868913650513, - "model.start_time": 1761152219.5910342, - "model.end_time": 1761152224.869903, - "model.performance_tier": "slow", - "model.prompt_tokens": 830, - "model.completion_tokens": 254, - "model.total_tokens": 1084, - "model.tokens_per_second": 205.3470199263535, - "model.estimated_cost": 0.03252, - "model.efficiency_score": 0.9381918819188192, - "model.response_length": 1017, - "model.response_quality": "high", - "model.output_response": "Model Response: Noveum Trace offers several key features through its Python SDK (`noveum-trace`) and TypeScript SDK (`@noveum/trace`):\n\n1. **Decorator-based Tracing**: It provides a simple way to trace LLM calls, age...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "df0c6344-1a2e-4c53-bb7d-4e43b3bce1b8", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:56:59.213000000", - "trace_end_time": "2025-10-22 16:57:05.598000000", - "trace_duration_ms": 6384.386, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:56:59.213000000", - "updated_at": "2025-10-22 16:56:59.213000000" - }, - { - "span_id": "f9214dbe-adb9-44ff-8361-70c872cef66d", - "trace_id": "df0c6344-1a2e-4c53-bb7d-4e43b3bce1b8", - "parent_span_id": "83f86157-5af4-4c99-aad1-647866cd804e", - "name": "agent.rag_evaluation_metrics", - "start_time": "2025-10-22 16:57:04.870000000", - "end_time": "2025-10-22 16:57:04.870000000", - "duration_ms": 0.043000000000000003, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "rag_evaluation_metrics", - "agent.capabilities": [ - "retrieval_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What features does Noveum Trace offer?", - "query_length": 38, - "query_type": "rag_evaluation_query", - "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\n\ud83d\udcd8 TypeScript SDK (\n@noveum/trace\n)- Framework ...", - "retrieval.context_length": 2787, - "retrieval.sources_count": 5, - "retrieval.context_quality": "high", - "retrieval.effectiveness": 1, - "retrieval.context_utilization": 2.787, - "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\n\ud83d\udcd8 TypeScript SDK (\n@noveum/trace\n)- Framework integrations for Next.js, Express.js, and Hono\n- TypeScript-first design with full type safety\n- Universal compatibility (Node.js, Edge Runtime, browsers)\n-\n\ud83d\udcca Noveum Platform\n- Real-time dashboard for analyzing traces and performance\n- Ad...\n\nSource 2 (https://noveum.ai/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\n\ud83d\udcd8 TypeScript SDK (\n@noveum/trace\n)- Framework integrations for Next.js, Express.js, and Hono\n- TypeScript-first design with full type safety\n- Universal compatibility (Node.js, Edge Runtime, browsers)\n-\n\ud83d\udcca Noveum Platform\n- Real-time dashboard for analyzing traces and performance\n- Ad...\n\nSource 3 (https://noveum.ai/en/docs/getting-started/sdk-integration):\n- \ud83d\udc65 Collaborate with your team on insights\n\ud83d\udd12 Security & Privacy\n- \ud83d\udd10 Encryption: All data encrypted in transit and at rest\n- \ud83c\udf9b\ufe0f Configurable Capture: Control what data is collected\n- \ud83c\udfe0 Data Residency: Choose your data storage region\n- \u23f0 Retention Control: Set custom data retention policies\nNext Steps\n- Tracing Concepts - Learn about traces, spans, and observability best practices\n- Framework Integrations - Deep dive into specific framework setups\n- Multi-Agent Tracing - Observe complex agent work...\n\nSource 4 (https://noveum.ai/docs/getting-started/sdk-integration):\n- \ud83d\udc65 Collaborate with your team on insights\n\ud83d\udd12 Security & Privacy\n- \ud83d\udd10 Encryption: All data encrypted in transit and at rest\n- \ud83c\udf9b\ufe0f Configurable Capture: Control what data is collected\n- \ud83c\udfe0 Data Residency: Choose your data storage region\n- \u23f0 Retention Control: Set custom data retention policies\nNext Steps\n- Tracing Concepts - Learn about traces, spans, and observability best practices\n- Framework Integrations - Deep dive into specific framework setups\n- Multi-Agent Tracing - Observe complex agent work...\n\nSource 5 (https://noveum.ai/docs/platform/dashboard):\n- Bulk Export: Download multiple traces simultaneously\n- Comparative Analysis: Compare performance across multiple traces\nReal-Time Updates\n- Live Refresh: Automatic updates as new traces arrive\n- Connection Monitoring: Real-time status of your trace ingestion pipeline\n- Performance Indicators: Live metrics for system health monitoring\nReady to dive deeper? Explore Projects & Environments to organize your AI applications, or check out Team Collaboration to share insights with your team.\nGet Earl...\n\nUser Question: What features does Noveum Trace offer?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", - "prompt.prompt_length": 3321, - "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frame...", - "prompt.instruction_following": "rag_optimized", - "response.answer_length": 1017, - "response.answer_completeness": "complete", - "response.response_quality": "high", - "response.source_citation": 5, - "response.context_utilization": 0.36490850376749195, - "output_response": "RAG Answer: Noveum Trace offers several key features through its Python SDK (`noveum-trace`) and TypeScript SDK (`@noveum/trace`):\n\n1. **Decorator-based Tracing**: It provides a simple way to trace LLM calls, age...", - "evaluation.retrieval_effectiveness": 1, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 5, - "evaluation.context_relevance": "high", - "evaluation.overall_quality": "high", - "evaluation.ready_for_production": true, - "rag.retrieval_strategy": "semantic_similarity", - "rag.vector_search_results": 5, - "rag.context_synthesis": "multi_source", - "rag.document_coverage": 1, - "rag.information_density": 0.36490850376749195 - }, - "events": [], - "links": [], - "trace_trace_id": "df0c6344-1a2e-4c53-bb7d-4e43b3bce1b8", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:56:59.213000000", - "trace_end_time": "2025-10-22 16:57:05.598000000", - "trace_duration_ms": 6384.386, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:56:59.213000000", - "updated_at": "2025-10-22 16:56:59.213000000" - }, - { - "span_id": "138a5089-5268-447a-afb5-4f14fb504281", - "trace_id": "29a862bf-d7aa-4cf9-963c-0a4244343bbe", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-22 16:56:54.719000000", - "end_time": "2025-10-22 16:56:58.207000000", - "duration_ms": 3487.959, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What are Noveum's pricing plans?", - "query.length": 32, - "processing.mode": "RAG", - "processing.time_seconds": 3.487928867340088, - "processing.response_length": 441, - "processing.sources_count": 5, - "output_response": "Final Answer: The provided context does not include specific information about Noveum's pricing plans. It mentions that all users will have access to the Observability suite for free, and early users will receive f...", - "final_answer_mode": "RAG", - "query_processed.input_query": "What are Noveum's pricing plans?", - "query_processed.output_response": "Successfully processed query using RAG, generated 441 character response", - "query_processed.mode": "RAG", - "query_processed.processing_time": 3.487928867340088, - "query_processed.response_length": 441 - }, - "events": [], - "links": [], - "trace_trace_id": "29a862bf-d7aa-4cf9-963c-0a4244343bbe", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:56:54.718000000", - "trace_end_time": "2025-10-22 16:56:58.207000000", - "trace_duration_ms": 3488.068, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:56:54.718000000", - "updated_at": "2025-10-22 16:56:54.718000000" - }, - { - "span_id": "7cda4dab-0fc5-4cae-8d8c-9445f661c67b", - "trace_id": "29a862bf-d7aa-4cf9-963c-0a4244343bbe", - "parent_span_id": "138a5089-5268-447a-afb5-4f14fb504281", - "name": "agent.query_routing", - "start_time": "2025-10-22 16:56:54.719000000", - "end_time": "2025-10-22 16:56:58.207000000", - "duration_ms": 3487.848, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What are Noveum's pricing plans?", - "query_length": 32, - "query_type": "routing_query", - "output_response": "Routed to RAG for query processing", - "router.classification": "RAG", - "router.confidence_score": 1, - "router.rag_keyword_score": 3, - "router.web_keyword_score": 0, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "29a862bf-d7aa-4cf9-963c-0a4244343bbe", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:56:54.718000000", - "trace_end_time": "2025-10-22 16:56:58.207000000", - "trace_duration_ms": 3488.068, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:56:54.718000000", - "updated_at": "2025-10-22 16:56:54.718000000" - }, - { - "span_id": "922c84a5-b14a-4b2f-8759-427d6e089812", - "trace_id": "29a862bf-d7aa-4cf9-963c-0a4244343bbe", - "parent_span_id": "7cda4dab-0fc5-4cae-8d8c-9445f661c67b", - "name": "agent.llm-rag", - "start_time": "2025-10-22 16:56:54.719000000", - "end_time": "2025-10-22 16:56:58.207000000", - "duration_ms": 3487.546, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "rag_agent", - "agent.operation": "llm-rag", - "agent.capabilities": [ - "document_retrieval", - "context_generation", - "response_generation" - ], - "agent.id": "noveum_rag_agent", - "input_query": "What are Noveum's pricing plans?", - "query_length": 32, - "query_type": "rag_query", - "output_response": "RAG Answer: The provided context does not include specific information about Noveum's pricing plans. It mentions that all users will have access to the Observability suite for free, and early users will receive f...", - "rag.context_length": 2105, - "rag.sources_count": 5, - "rag.answer_length": 441, - "rag.mode": "retrieval_augmented_generation" - }, - "events": [], - "links": [], - "trace_trace_id": "29a862bf-d7aa-4cf9-963c-0a4244343bbe", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:56:54.718000000", - "trace_end_time": "2025-10-22 16:56:58.207000000", - "trace_duration_ms": 3488.068, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:56:54.718000000", - "updated_at": "2025-10-22 16:56:54.718000000" - }, - { - "span_id": "c8d7bb28-e763-47c9-b3e9-77f6a91a22be", - "trace_id": "29a862bf-d7aa-4cf9-963c-0a4244343bbe", - "parent_span_id": "7cda4dab-0fc5-4cae-8d8c-9445f661c67b", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-22 16:56:54.719000000", - "end_time": "2025-10-22 16:56:54.719000000", - "duration_ms": 0.068, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What are Noveum's pricing plans?", - "query_length": 32, - "query_type": "routing_evaluation_query", - "classification.mode": "RAG", - "classification.rag_keyword_score": 3, - "classification.web_keyword_score": 0, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "noveum_specific", - "query.keyword_density": 0.6, - "query.domain_affinity": "noveum", - "routing.decision": "Routed to RAG based on analysis", - "routing.rationale": "RAG score: 3, Web score: 0, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "Web Search", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.057692307692307696, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: RAG (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "29a862bf-d7aa-4cf9-963c-0a4244343bbe", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:56:54.718000000", - "trace_end_time": "2025-10-22 16:56:58.207000000", - "trace_duration_ms": 3488.068, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:56:54.718000000", - "updated_at": "2025-10-22 16:56:54.718000000" - }, - { - "span_id": "a378123a-8a79-4bc5-8e20-8ec54c2f9783", - "trace_id": "29a862bf-d7aa-4cf9-963c-0a4244343bbe", - "parent_span_id": "922c84a5-b14a-4b2f-8759-427d6e089812", - "name": "agent.llm_model_execution", - "start_time": "2025-10-22 16:56:55.154000000", - "end_time": "2025-10-22 16:56:57.166000000", - "duration_ms": 2012.174, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What are Noveum's pricing plans?", - "query_length": 32, - "query_type": "rag_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 2.012089252471924, - "model.latency_ms": 2012.0892524719238, - "model.start_time": 1761152215.154625, - "model.end_time": 1761152217.1667142, - "model.performance_tier": "medium", - "model.prompt_tokens": 658, - "model.completion_tokens": 110, - "model.total_tokens": 768, - "model.tokens_per_second": 381.6928096288395, - "model.estimated_cost": 0.02304, - "model.efficiency_score": 0.57421875, - "model.response_length": 441, - "model.response_quality": "high", - "model.output_response": "Model Response: The provided context does not include specific information about Noveum's pricing plans. It mentions that all users will have access to the Observability suite for free, and early users will receive f...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "29a862bf-d7aa-4cf9-963c-0a4244343bbe", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:56:54.718000000", - "trace_end_time": "2025-10-22 16:56:58.207000000", - "trace_duration_ms": 3488.068, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:56:54.718000000", - "updated_at": "2025-10-22 16:56:54.718000000" - }, - { - "span_id": "5813673b-a1dc-46c7-9bea-92ce6aca2b5e", - "trace_id": "29a862bf-d7aa-4cf9-963c-0a4244343bbe", - "parent_span_id": "922c84a5-b14a-4b2f-8759-427d6e089812", - "name": "agent.rag_evaluation_metrics", - "start_time": "2025-10-22 16:56:57.166000000", - "end_time": "2025-10-22 16:56:57.167000000", - "duration_ms": 0.059, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "rag_evaluation_metrics", - "agent.capabilities": [ - "retrieval_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What are Noveum's pricing plans?", - "query_length": 32, - "query_type": "rag_evaluation_query", - "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....", - "retrieval.context_length": 2105, - "retrieval.sources_count": 5, - "retrieval.context_quality": "high", - "retrieval.effectiveness": 1, - "retrieval.context_utilization": 2.105, - "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nSource 2 (https://noveum.ai/en/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nSource 3 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nThe AI landscape is complex and moving fast. With Noveum.ai, you'll have the insights you need to build, debug, and optimize AI applications with confidence.\nReady to see what's really happening inside your AI applications? Try Noveum.ai today and transform how you build and monitor AI systems.\nLet's build more reliable, observable AI\u2014together.\nGet Early Access to Noveum.ai Platform\nJoin the select group of AI teams optimizing their models with our data-driven platform. We're onboarding users in...\n\nSource 4 (https://noveum.ai/en/blog/from-logs-to-intelligent-choices-inside-noveum-ais-evaluation-process):\nGet Early Access to Noveum.ai Platform\nJoin the select group of AI teams optimizing their models with our data-driven platform. We're onboarding users in limited batches to ensure a premium experience....\n\nSource 5 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, vector searches, data retrieval, agent reasoning, and more. Without proper observability, debugging becomes a nightmare, optimization is guesswork, and understanding user interactions is nearly impossible.\nNoveum.ai solves this by providing comprehensive tracing and observability specifically designed for AI applications.\nKey benefits of using Noveum.ai:\n- Complete Visibility: Trace...\n\nUser Question: What are Noveum's pricing plans?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", - "prompt.prompt_length": 2633, - "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to ...", - "prompt.instruction_following": "rag_optimized", - "response.answer_length": 441, - "response.answer_completeness": "complete", - "response.response_quality": "high", - "response.source_citation": 5, - "response.context_utilization": 0.20950118764845604, - "output_response": "RAG Answer: The provided context does not include specific information about Noveum's pricing plans. It mentions that all users will have access to the Observability suite for free, and early users will receive f...", - "evaluation.retrieval_effectiveness": 1, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 5, - "evaluation.context_relevance": "high", - "evaluation.overall_quality": "high", - "evaluation.ready_for_production": true, - "rag.retrieval_strategy": "semantic_similarity", - "rag.vector_search_results": 5, - "rag.context_synthesis": "multi_source", - "rag.document_coverage": 1, - "rag.information_density": 0.20950118764845604 - }, - "events": [], - "links": [], - "trace_trace_id": "29a862bf-d7aa-4cf9-963c-0a4244343bbe", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:56:54.718000000", - "trace_end_time": "2025-10-22 16:56:58.207000000", - "trace_duration_ms": 3488.068, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:56:54.718000000", - "updated_at": "2025-10-22 16:56:54.718000000" - }, - { - "span_id": "c5baf5c2-d18a-458a-b922-a19b96db53d9", - "trace_id": "bc60c122-f30b-4114-9f78-ee40a024b525", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-22 16:56:44.873000000", - "end_time": "2025-10-22 16:56:53.701000000", - "duration_ms": 8827.979, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "How do I integrate Noveum Trace in my application?", - "query.length": 50, - "processing.mode": "RAG", - "processing.time_seconds": 8.827935218811035, - "processing.response_length": 1145, - "processing.sources_count": 5, - "output_response": "Final Answer: To integrate Noveum Trace into your application, you can follow these steps based on the SDK Integration Guide:\n\n1. **Create Your Account & Get API Key**:\n - Sign up at [noveum.ai](https://noveum.ai...", - "final_answer_mode": "RAG", - "query_processed.input_query": "How do I integrate Noveum Trace in my application?", - "query_processed.output_response": "Successfully processed query using RAG, generated 1145 character response", - "query_processed.mode": "RAG", - "query_processed.processing_time": 8.827935218811035, - "query_processed.response_length": 1145 - }, - "events": [], - "links": [], - "trace_trace_id": "bc60c122-f30b-4114-9f78-ee40a024b525", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:56:44.873000000", - "trace_end_time": "2025-10-22 16:56:53.701000000", - "trace_duration_ms": 8828.055, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:56:44.873000000", - "updated_at": "2025-10-22 16:56:44.873000000" - }, - { - "span_id": "8819dde5-9cbe-4cdb-b919-9c5140582359", - "trace_id": "bc60c122-f30b-4114-9f78-ee40a024b525", - "parent_span_id": "c5baf5c2-d18a-458a-b922-a19b96db53d9", - "name": "agent.query_routing", - "start_time": "2025-10-22 16:56:44.873000000", - "end_time": "2025-10-22 16:56:53.701000000", - "duration_ms": 8827.883, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "How do I integrate Noveum Trace in my application?", - "query_length": 50, - "query_type": "routing_query", - "output_response": "Routed to RAG for query processing", - "router.classification": "RAG", - "router.confidence_score": 1, - "router.rag_keyword_score": 2, - "router.web_keyword_score": 0, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "bc60c122-f30b-4114-9f78-ee40a024b525", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:56:44.873000000", - "trace_end_time": "2025-10-22 16:56:53.701000000", - "trace_duration_ms": 8828.055, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:56:44.873000000", - "updated_at": "2025-10-22 16:56:44.873000000" - }, - { - "span_id": "932c4b0e-0f33-40c0-bb19-78b9f81e48a6", - "trace_id": "bc60c122-f30b-4114-9f78-ee40a024b525", - "parent_span_id": "8819dde5-9cbe-4cdb-b919-9c5140582359", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-22 16:56:44.873000000", - "end_time": "2025-10-22 16:56:44.873000000", - "duration_ms": 0.043000000000000003, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: How do I integrate Noveum Trace in my application?", - "query_length": 50, - "query_type": "routing_evaluation_query", - "classification.mode": "RAG", - "classification.rag_keyword_score": 2, - "classification.web_keyword_score": 0, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "noveum_specific", - "query.keyword_density": 0.2222222222222222, - "query.domain_affinity": "noveum", - "routing.decision": "Routed to RAG based on analysis", - "routing.rationale": "RAG score: 2, Web score: 0, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "Web Search", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.038461538461538464, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: RAG (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "bc60c122-f30b-4114-9f78-ee40a024b525", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:56:44.873000000", - "trace_end_time": "2025-10-22 16:56:53.701000000", - "trace_duration_ms": 8828.055, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:56:44.873000000", - "updated_at": "2025-10-22 16:56:44.873000000" - }, - { - "span_id": "2ece2bd6-e4c8-47bc-b3c8-26f82e9910f1", - "trace_id": "bc60c122-f30b-4114-9f78-ee40a024b525", - "parent_span_id": "8819dde5-9cbe-4cdb-b919-9c5140582359", - "name": "agent.llm-rag", - "start_time": "2025-10-22 16:56:44.873000000", - "end_time": "2025-10-22 16:56:53.701000000", - "duration_ms": 8827.666000000001, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "rag_agent", - "agent.operation": "llm-rag", - "agent.capabilities": [ - "document_retrieval", - "context_generation", - "response_generation" - ], - "agent.id": "noveum_rag_agent", - "input_query": "How do I integrate Noveum Trace in my application?", - "query_length": 50, - "query_type": "rag_query", - "output_response": "RAG Answer: To integrate Noveum Trace into your application, you can follow these steps based on the SDK Integration Guide:\n\n1. **Create Your Account & Get API Key**:\n - Sign up at [noveum.ai](https://noveum.ai...", - "rag.context_length": 2803, - "rag.sources_count": 5, - "rag.answer_length": 1145, - "rag.mode": "retrieval_augmented_generation" - }, - "events": [], - "links": [], - "trace_trace_id": "bc60c122-f30b-4114-9f78-ee40a024b525", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:56:44.873000000", - "trace_end_time": "2025-10-22 16:56:53.701000000", - "trace_duration_ms": 8828.055, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:56:44.873000000", - "updated_at": "2025-10-22 16:56:44.873000000" - }, - { - "span_id": "e6315110-36b1-4503-beaf-12d972e25a7d", - "trace_id": "bc60c122-f30b-4114-9f78-ee40a024b525", - "parent_span_id": "2ece2bd6-e4c8-47bc-b3c8-26f82e9910f1", - "name": "agent.llm_model_execution", - "start_time": "2025-10-22 16:56:45.576000000", - "end_time": "2025-10-22 16:56:53.143000000", - "duration_ms": 7566.311, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: How do I integrate Noveum Trace in my application?", - "query_length": 50, - "query_type": "rag_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 7.566190004348755, - "model.latency_ms": 7566.190004348755, - "model.start_time": 1761152205.576988, - "model.end_time": 1761152213.143178, - "model.performance_tier": "slow", - "model.prompt_tokens": 837, - "model.completion_tokens": 286, - "model.total_tokens": 1123, - "model.tokens_per_second": 148.4234468543009, - "model.estimated_cost": 0.03369, - "model.efficiency_score": 1.0195903829029385, - "model.response_length": 1145, - "model.response_quality": "high", - "model.output_response": "Model Response: To integrate Noveum Trace into your application, you can follow these steps based on the SDK Integration Guide:\n\n1. **Create Your Account & Get API Key**:\n - Sign up at [noveum.ai](https://noveum.ai...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "bc60c122-f30b-4114-9f78-ee40a024b525", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:56:44.873000000", - "trace_end_time": "2025-10-22 16:56:53.701000000", - "trace_duration_ms": 8828.055, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:56:44.873000000", - "updated_at": "2025-10-22 16:56:44.873000000" - }, - { - "span_id": "ae91d520-8442-48e5-aabf-fd58aed23694", - "trace_id": "bc60c122-f30b-4114-9f78-ee40a024b525", - "parent_span_id": "2ece2bd6-e4c8-47bc-b3c8-26f82e9910f1", - "name": "agent.rag_evaluation_metrics", - "start_time": "2025-10-22 16:56:53.143000000", - "end_time": "2025-10-22 16:56:53.143000000", - "duration_ms": 0.091, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "rag_evaluation_metrics", - "agent.capabilities": [ - "retrieval_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: How do I integrate Noveum Trace in my application?", - "query_length": 50, - "query_type": "rag_evaluation_query", - "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\n\ud83d\udcd8 TypeScript SDK (\n@noveum/trace\n)- Framework ...", - "retrieval.context_length": 2803, - "retrieval.sources_count": 5, - "retrieval.context_quality": "high", - "retrieval.effectiveness": 1, - "retrieval.context_utilization": 2.803, - "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\n\ud83d\udcd8 TypeScript SDK (\n@noveum/trace\n)- Framework integrations for Next.js, Express.js, and Hono\n- TypeScript-first design with full type safety\n- Universal compatibility (Node.js, Edge Runtime, browsers)\n-\n\ud83d\udcca Noveum Platform\n- Real-time dashboard for analyzing traces and performance\n- Ad...\n\nSource 2 (https://noveum.ai/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\n\ud83d\udcd8 TypeScript SDK (\n@noveum/trace\n)- Framework integrations for Next.js, Express.js, and Hono\n- TypeScript-first design with full type safety\n- Universal compatibility (Node.js, Edge Runtime, browsers)\n-\n\ud83d\udcca Noveum Platform\n- Real-time dashboard for analyzing traces and performance\n- Ad...\n\nSource 3 (https://noveum.ai/docs/getting-started/sdk-integration):\nSDK Integration Guide\nIntegrate Noveum.ai tracing into your AI applications with Python or TypeScript SDKs\nThe Noveum.ai SDKs provide comprehensive tracing and observability for your AI applications with minimal code changes. Whether you're building LLM applications, RAG systems, or multi-agent workflows, our SDKs automatically capture essential metrics and traces.\n\ud83d\ude80 Quick Start\n1. Create Your Account & Get API Key\n- Sign up at noveum.ai\n- Create a project in your dashboard\n- Generate an API key...\n\nSource 4 (https://noveum.ai/en/docs/getting-started/sdk-integration):\nSDK Integration Guide\nIntegrate Noveum.ai tracing into your AI applications with Python or TypeScript SDKs\nThe Noveum.ai SDKs provide comprehensive tracing and observability for your AI applications with minimal code changes. Whether you're building LLM applications, RAG systems, or multi-agent workflows, our SDKs automatically capture essential metrics and traces.\n\ud83d\ude80 Quick Start\n1. Create Your Account & Get API Key\n- Sign up at noveum.ai\n- Create a project in your dashboard\n- Generate an API key...\n\nSource 5 (https://noveum.ai/en/docs/getting-started/sdk-integration):\n\ud83d\udd27 Framework Integrations\nNext.js Integration\nExpress.js Integration\nFastAPI Integration (Python)\n\ud83d\udcca Advanced Features\nCustom Attributes & Events\nSampling Configuration\n\ud83d\udd17 What's Captured Automatically\n- \ud83d\udcca Performance Metrics: Latency, throughput, error rates\n- \ud83d\udcb0 Cost Tracking: Token usage, API costs across providers\n- \ud83d\udd0d Request/Response: Configurable capture of inputs/outputs\n- \ud83c\udff7\ufe0f Metadata: Model names, parameters, user context\n- \ud83c\udf0a Context Flow: Trace relationships across services\n- \ud83d\udc1b Error Detail...\n\nUser Question: How do I integrate Noveum Trace in my application?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", - "prompt.prompt_length": 3349, - "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frame...", - "prompt.instruction_following": "rag_optimized", - "response.answer_length": 1145, - "response.answer_completeness": "complete", - "response.response_quality": "high", - "response.source_citation": 5, - "response.context_utilization": 0.4084909026043525, - "output_response": "RAG Answer: To integrate Noveum Trace into your application, you can follow these steps based on the SDK Integration Guide:\n\n1. **Create Your Account & Get API Key**:\n - Sign up at [noveum.ai](https://noveum.ai...", - "evaluation.retrieval_effectiveness": 1, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 5, - "evaluation.context_relevance": "high", - "evaluation.overall_quality": "high", - "evaluation.ready_for_production": true, - "rag.retrieval_strategy": "semantic_similarity", - "rag.vector_search_results": 5, - "rag.context_synthesis": "multi_source", - "rag.document_coverage": 1, - "rag.information_density": 0.4084909026043525 - }, - "events": [], - "links": [], - "trace_trace_id": "bc60c122-f30b-4114-9f78-ee40a024b525", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:56:44.873000000", - "trace_end_time": "2025-10-22 16:56:53.701000000", - "trace_duration_ms": 8828.055, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:56:44.873000000", - "updated_at": "2025-10-22 16:56:44.873000000" - }, - { - "span_id": "9a9c611d-f552-4b06-a764-4463edb70e2c", - "trace_id": "3ad192e5-bbd4-4236-8144-019d53512769", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-22 16:56:35.188000000", - "end_time": "2025-10-22 16:56:43.863000000", - "duration_ms": 8675.25, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What is Noveum and what does it do?", - "query.length": 35, - "processing.mode": "RAG", - "processing.time_seconds": 8.67501425743103, - "processing.response_length": 527, - "processing.sources_count": 5, - "output_response": "Final Answer: Noveum.ai is a comprehensive tracing and observability platform specifically designed for AI applications, including LLM-powered chatbots, RAG systems, and multi-agent workflows. It provides the neces...", - "final_answer_mode": "RAG", - "query_processed.input_query": "What is Noveum and what does it do?", - "query_processed.output_response": "Successfully processed query using RAG, generated 527 character response", - "query_processed.mode": "RAG", - "query_processed.processing_time": 8.67501425743103, - "query_processed.response_length": 527 - }, - "events": [], - "links": [], - "trace_trace_id": "3ad192e5-bbd4-4236-8144-019d53512769", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:56:35.188000000", - "trace_end_time": "2025-10-22 16:56:43.863000000", - "trace_duration_ms": 8675.356000000002, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:56:35.188000000", - "updated_at": "2025-10-22 16:56:35.188000000" - }, - { - "span_id": "99eadeae-0733-4bf3-982e-e9f3014f5913", - "trace_id": "3ad192e5-bbd4-4236-8144-019d53512769", - "parent_span_id": "7e2c959c-157a-43d5-9870-32994cddd388", - "name": "agent.llm-rag", - "start_time": "2025-10-22 16:56:35.188000000", - "end_time": "2025-10-22 16:56:43.863000000", - "duration_ms": 8674.699999999999, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "rag_agent", - "agent.operation": "llm-rag", - "agent.capabilities": [ - "document_retrieval", - "context_generation", - "response_generation" - ], - "agent.id": "noveum_rag_agent", - "input_query": "What is Noveum and what does it do?", - "query_length": 35, - "query_type": "rag_query", - "output_response": "RAG Answer: Noveum.ai is a comprehensive tracing and observability platform specifically designed for AI applications, including LLM-powered chatbots, RAG systems, and multi-agent workflows. It provides the neces...", - "rag.context_length": 2334, - "rag.sources_count": 5, - "rag.answer_length": 527, - "rag.mode": "retrieval_augmented_generation" - }, - "events": [], - "links": [], - "trace_trace_id": "3ad192e5-bbd4-4236-8144-019d53512769", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:56:35.188000000", - "trace_end_time": "2025-10-22 16:56:43.863000000", - "trace_duration_ms": 8675.356000000002, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:56:35.188000000", - "updated_at": "2025-10-22 16:56:35.188000000" - }, - { - "span_id": "7e2c959c-157a-43d5-9870-32994cddd388", - "trace_id": "3ad192e5-bbd4-4236-8144-019d53512769", - "parent_span_id": "9a9c611d-f552-4b06-a764-4463edb70e2c", - "name": "agent.query_routing", - "start_time": "2025-10-22 16:56:35.188000000", - "end_time": "2025-10-22 16:56:43.863000000", - "duration_ms": 8674.961, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What is Noveum and what does it do?", - "query_length": 35, - "query_type": "routing_query", - "output_response": "Routed to RAG for query processing", - "router.classification": "RAG", - "router.confidence_score": 1, - "router.rag_keyword_score": 1, - "router.web_keyword_score": 0, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "3ad192e5-bbd4-4236-8144-019d53512769", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:56:35.188000000", - "trace_end_time": "2025-10-22 16:56:43.863000000", - "trace_duration_ms": 8675.356000000002, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:56:35.188000000", - "updated_at": "2025-10-22 16:56:35.188000000" - }, - { - "span_id": "6f81284c-a9aa-4d62-a402-1b474f2eba51", - "trace_id": "3ad192e5-bbd4-4236-8144-019d53512769", - "parent_span_id": "7e2c959c-157a-43d5-9870-32994cddd388", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-22 16:56:35.188000000", - "end_time": "2025-10-22 16:56:35.188000000", - "duration_ms": 0.054, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What is Noveum and what does it do?", - "query_length": 35, - "query_type": "routing_evaluation_query", - "classification.mode": "RAG", - "classification.rag_keyword_score": 1, - "classification.web_keyword_score": 0, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "noveum_specific", - "query.keyword_density": 0.125, - "query.domain_affinity": "noveum", - "routing.decision": "Routed to RAG based on analysis", - "routing.rationale": "RAG score: 1, Web score: 0, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "Web Search", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.019230769230769232, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: RAG (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "3ad192e5-bbd4-4236-8144-019d53512769", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:56:35.188000000", - "trace_end_time": "2025-10-22 16:56:43.863000000", - "trace_duration_ms": 8675.356000000002, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:56:35.188000000", - "updated_at": "2025-10-22 16:56:35.188000000" - }, - { - "span_id": "9f42d18a-0235-4ae2-838c-19a88ddc95e1", - "trace_id": "3ad192e5-bbd4-4236-8144-019d53512769", - "parent_span_id": "99eadeae-0733-4bf3-982e-e9f3014f5913", - "name": "agent.llm_model_execution", - "start_time": "2025-10-22 16:56:36.031000000", - "end_time": "2025-10-22 16:56:42.012000000", - "duration_ms": 5980.947999999999, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What is Noveum and what does it do?", - "query_length": 35, - "query_type": "rag_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 5.980915784835815, - "model.latency_ms": 5980.915784835815, - "model.start_time": 1761152196.031816, - "model.end_time": 1761152202.0127318, - "model.performance_tier": "slow", - "model.prompt_tokens": 716, - "model.completion_tokens": 131, - "model.total_tokens": 847, - "model.tokens_per_second": 141.61710856178715, - "model.estimated_cost": 0.025410000000000002, - "model.efficiency_score": 0.6221959858323495, - "model.response_length": 527, - "model.response_quality": "high", - "model.output_response": "Model Response: Noveum.ai is a comprehensive tracing and observability platform specifically designed for AI applications, including LLM-powered chatbots, RAG systems, and multi-agent workflows. It provides the neces...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "3ad192e5-bbd4-4236-8144-019d53512769", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:56:35.188000000", - "trace_end_time": "2025-10-22 16:56:43.863000000", - "trace_duration_ms": 8675.356000000002, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:56:35.188000000", - "updated_at": "2025-10-22 16:56:35.188000000" - }, - { - "span_id": "7bcbe4b7-585a-4c9d-add8-16e8aa62254f", - "trace_id": "3ad192e5-bbd4-4236-8144-019d53512769", - "parent_span_id": "99eadeae-0733-4bf3-982e-e9f3014f5913", - "name": "agent.rag_evaluation_metrics", - "start_time": "2025-10-22 16:56:42.012000000", - "end_time": "2025-10-22 16:56:42.012000000", - "duration_ms": 0.018000000000000002, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "rag_evaluation_metrics", - "agent.capabilities": [ - "retrieval_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What is Noveum and what does it do?", - "query_length": 35, - "query_type": "rag_evaluation_query", - "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, vector searches, data retrieval, agent reasoning, and more. Without proper observability, debugging ...", - "retrieval.context_length": 2334, - "retrieval.sources_count": 5, - "retrieval.context_quality": "high", - "retrieval.effectiveness": 1, - "retrieval.context_utilization": 2.334, - "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, vector searches, data retrieval, agent reasoning, and more. Without proper observability, debugging becomes a nightmare, optimization is guesswork, and understanding user interactions is nearly impossible.\nNoveum.ai solves this by providing comprehensive tracing and observability specifically designed for AI applications.\nKey benefits of using Noveum.ai:\n- Complete Visibility: Trace...\n\nSource 2 (https://noveum.ai/en/docs/getting-started/overview):\nNoveum.ai Overview\nComprehensive AI tracing and observability platform for LLM applications, RAG systems, and AI agents\nWelcome to Noveum.ai\u2014the comprehensive tracing and observability platform built specifically for AI applications. Whether you're building LLM-powered chatbots, RAG systems, multi-agent workflows, or any AI-driven application, Noveum provides the insights you need to understand, debug, and optimize your systems.\n\ud83c\udfaf Why AI Applications Need Specialized Observability\nTraditional mo...\n\nSource 3 (https://noveum.ai/docs/getting-started/overview):\nNoveum.ai Overview\nComprehensive AI tracing and observability platform for LLM applications, RAG systems, and AI agents\nWelcome to Noveum.ai\u2014the comprehensive tracing and observability platform built specifically for AI applications. Whether you're building LLM-powered chatbots, RAG systems, multi-agent workflows, or any AI-driven application, Noveum provides the insights you need to understand, debug, and optimize your systems.\n\ud83c\udfaf Why AI Applications Need Specialized Observability\nTraditional mo...\n\nSource 4 (https://noveum.ai/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nSource 5 (https://noveum.ai/en/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nUser Question: What is Noveum and what does it do?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", - "prompt.prompt_length": 2865, - "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, ...", - "prompt.instruction_following": "rag_optimized", - "response.answer_length": 527, - "response.answer_completeness": "complete", - "response.response_quality": "high", - "response.source_citation": 5, - "response.context_utilization": 0.22579263067694944, - "output_response": "RAG Answer: Noveum.ai is a comprehensive tracing and observability platform specifically designed for AI applications, including LLM-powered chatbots, RAG systems, and multi-agent workflows. It provides the neces...", - "evaluation.retrieval_effectiveness": 1, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 5, - "evaluation.context_relevance": "high", - "evaluation.overall_quality": "high", - "evaluation.ready_for_production": true, - "rag.retrieval_strategy": "semantic_similarity", - "rag.vector_search_results": 5, - "rag.context_synthesis": "multi_source", - "rag.document_coverage": 1, - "rag.information_density": 0.22579263067694944 - }, - "events": [], - "links": [], - "trace_trace_id": "3ad192e5-bbd4-4236-8144-019d53512769", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:56:35.188000000", - "trace_end_time": "2025-10-22 16:56:43.863000000", - "trace_duration_ms": 8675.356000000002, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:56:35.188000000", - "updated_at": "2025-10-22 16:56:35.188000000" - }, - { - "span_id": "d39297f4-b4fe-4f9c-b45b-854411ba770a", - "trace_id": "9af2b4b0-0c2f-4d2e-ae99-30df0d2f0fa6", - "parent_span_id": "", - "name": "system_initialization", - "start_time": "2025-10-22 16:56:34.565000000", - "end_time": "2025-10-22 16:56:35.186000000", - "duration_ms": 621.02, - "status": "ok", - "status_message": "", - "attributes": { - "system.force_scrape": false, - "system.config": { - "noveum_base_url": "https://noveum.ai", - "max_pages_to_scrape": 50, - "chunk_size": 1000, - "chunk_overlap": 200, - "max_search_results": 5, - "rag_threshold": 0.7, - "noveum_docs_file": "noveum_docs.json", - "vector_store_path": "noveum_vectorstore" - }, - "input_query": "Initialize system with force_scrape=False", - "output_response": "System initialization: RAG system loaded, vector store ready, agent operational", - "system.initialized": true, - "system.documents_loaded": 38, - "system.vectorstore_ready": true - }, - "events": [], - "links": [], - "trace_trace_id": "9af2b4b0-0c2f-4d2e-ae99-30df0d2f0fa6", - "trace_name": "auto_trace_system_initialization", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:56:34.565000000", - "trace_end_time": "2025-10-22 16:56:35.186000000", - "trace_duration_ms": 621.177, - "span_count": 1, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:56:34.565000000", - "updated_at": "2025-10-22 16:56:34.565000000" - }, - { - "span_id": "8c144ad8-a439-4986-b741-3bc4b7fc6110", - "trace_id": "bae13042-cf2e-4813-a673-7dc2a611725e", - "parent_span_id": "d37fe1a1-a126-4e4e-a914-93dd402fed3f", - "name": "agent.query_routing", - "start_time": "2025-10-16 08:50:56.153000000", - "end_time": "2025-10-16 08:51:01.908000000", - "duration_ms": 5754.43, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What are the latest breakthroughs in artificial intelligence?", - "query_length": 61, - "query_type": "routing_query", - "output_response": "Routed to Web Search for query processing", - "router.classification": "Web Search", - "router.confidence_score": 1, - "router.rag_keyword_score": 0, - "router.web_keyword_score": 1, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "bae13042-cf2e-4813-a673-7dc2a611725e", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:56.153000000", - "trace_end_time": "2025-10-16 08:51:01.908000000", - "trace_duration_ms": 5754.578, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:56.153000000", - "updated_at": "2025-10-16 08:50:56.153000000" - }, - { - "span_id": "b294f9b7-7762-47a9-b49d-00801a984af2", - "trace_id": "bae13042-cf2e-4813-a673-7dc2a611725e", - "parent_span_id": "8c144ad8-a439-4986-b741-3bc4b7fc6110", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-16 08:50:56.153000000", - "end_time": "2025-10-16 08:50:56.153000000", - "duration_ms": 0.024, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What are the latest breakthroughs in artificial intelligence?", - "query_length": 61, - "query_type": "routing_evaluation_query", - "classification.mode": "Web Search", - "classification.rag_keyword_score": 0, - "classification.web_keyword_score": 1, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "complex", - "query.intent": "general_knowledge", - "query.keyword_density": 0.125, - "query.domain_affinity": "general", - "routing.decision": "Routed to Web Search based on analysis", - "routing.rationale": "RAG score: 0, Web score: 1, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "RAG", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.019230769230769232, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: Web Search (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "bae13042-cf2e-4813-a673-7dc2a611725e", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:56.153000000", - "trace_end_time": "2025-10-16 08:51:01.908000000", - "trace_duration_ms": 5754.578, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:56.153000000", - "updated_at": "2025-10-16 08:50:56.153000000" - }, - { - "span_id": "ab6a281a-4989-40c8-9011-d0ec7dde7ea9", - "trace_id": "bae13042-cf2e-4813-a673-7dc2a611725e", - "parent_span_id": "8c144ad8-a439-4986-b741-3bc4b7fc6110", - "name": "agent.web_search_generation", - "start_time": "2025-10-16 08:50:56.153000000", - "end_time": "2025-10-16 08:51:01.908000000", - "duration_ms": 5754.317999999999, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "web_search_agent", - "agent.operation": "web_search_generation", - "agent.capabilities": [ - "web_search", - "content_synthesis", - "response_generation" - ], - "agent.id": "noveum_web_search_agent", - "input_query": "What are the latest breakthroughs in artificial intelligence?", - "query_length": 61, - "query_type": "web_search_query", - "output_response": "Web Search Answer: The web search did not yield specific results regarding the latest breakthroughs in artificial intelligence. However, I can provide a general overview based on knowledge up to October 2023.\n\nRecent br...", - "web_search.results_count": 1, - "web_search.context_length": 163, - "web_search.response_length": 1489, - "web_search.mode": "external_web_search" - }, - "events": [], - "links": [], - "trace_trace_id": "bae13042-cf2e-4813-a673-7dc2a611725e", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:56.153000000", - "trace_end_time": "2025-10-16 08:51:01.908000000", - "trace_duration_ms": 5754.578, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:56.153000000", - "updated_at": "2025-10-16 08:50:56.153000000" - }, - { - "span_id": "d37fe1a1-a126-4e4e-a914-93dd402fed3f", - "trace_id": "bae13042-cf2e-4813-a673-7dc2a611725e", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-16 08:50:56.153000000", - "end_time": "2025-10-16 08:51:01.908000000", - "duration_ms": 5754.527, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What are the latest breakthroughs in artificial intelligence?", - "query.length": 61, - "processing.mode": "Web Search", - "processing.time_seconds": 5.754457473754883, - "processing.response_length": 1489, - "processing.sources_count": 1, - "output_response": "Final Answer: The web search did not yield specific results regarding the latest breakthroughs in artificial intelligence. However, I can provide a general overview based on knowledge up to October 2023.\n\nRecent br...", - "final_answer_mode": "Web Search", - "query_processed.input_query": "What are the latest breakthroughs in artificial intelligence?", - "query_processed.output_response": "Successfully processed query using Web Search, generated 1489 character response", - "query_processed.mode": "Web Search", - "query_processed.processing_time": 5.754457473754883, - "query_processed.response_length": 1489 - }, - "events": [], - "links": [], - "trace_trace_id": "bae13042-cf2e-4813-a673-7dc2a611725e", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:56.153000000", - "trace_end_time": "2025-10-16 08:51:01.908000000", - "trace_duration_ms": 5754.578, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:56.153000000", - "updated_at": "2025-10-16 08:50:56.153000000" - }, - { - "span_id": "f61ef76d-4887-42da-9fd6-e2cdb25a9e09", - "trace_id": "bae13042-cf2e-4813-a673-7dc2a611725e", - "parent_span_id": "ab6a281a-4989-40c8-9011-d0ec7dde7ea9", - "name": "agent.llm_model_execution", - "start_time": "2025-10-16 08:50:56.502000000", - "end_time": "2025-10-16 08:51:01.908000000", - "duration_ms": 5405.44, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What are the latest breakthroughs in artificial intelligence?", - "query_length": 61, - "query_type": "web_search_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 5.405392408370972, - "model.latency_ms": 5405.392408370972, - "model.start_time": 1760604656.5026655, - "model.end_time": 1760604661.908058, - "model.performance_tier": "slow", - "model.prompt_tokens": 167, - "model.completion_tokens": 372, - "model.total_tokens": 539, - "model.tokens_per_second": 99.71523976044487, - "model.estimated_cost": 0.01617, - "model.efficiency_score": 2.7625231910946195, - "model.response_length": 1489, - "model.response_quality": "high", - "model.output_response": "Model Response: The web search did not yield specific results regarding the latest breakthroughs in artificial intelligence. However, I can provide a general overview based on knowledge up to October 2023.\n\nRecent br...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "bae13042-cf2e-4813-a673-7dc2a611725e", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:56.153000000", - "trace_end_time": "2025-10-16 08:51:01.908000000", - "trace_duration_ms": 5754.578, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:56.153000000", - "updated_at": "2025-10-16 08:50:56.153000000" - }, - { - "span_id": "550bf5ae-c816-4ef0-a6c1-1553b1c258b8", - "trace_id": "bae13042-cf2e-4813-a673-7dc2a611725e", - "parent_span_id": "ab6a281a-4989-40c8-9011-d0ec7dde7ea9", - "name": "agent.web_search_evaluation_metrics", - "start_time": "2025-10-16 08:51:01.908000000", - "end_time": "2025-10-16 08:51:01.908000000", - "duration_ms": 0.018000000000000002, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "web_search_evaluation_metrics", - "agent.capabilities": [ - "web_search_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What are the latest breakthroughs in artificial intelligence?", - "query_length": 61, - "query_type": "web_search_evaluation_query", - "web_search.results_count": 1, - "web_search.context_length": 163, - "web_search.context_synthesized": "Context from 1 web sources", - "web_search.search_effectiveness": 0.2, - "web_search.context_quality": "low", - "web_search.source_diversity": 1, - "web_search.information_synthesis": "low", - "web_search.external_knowledge_utilization": 0.163, - "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What+are+the+latest+breakthroughs+in+artificial+intelligence?\n\nUser Question: What are the latest breakthroughs in artificial intelligence?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", - "prompt.prompt_length": 669, - "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What+are+the+latest+breakthroughs+in+artificial+intelligence?", - "prompt.instruction_following": "web_search_optimized", - "response.answer_length": 1489, - "response.answer_completeness": "complete", - "response.response_quality": "medium", - "response.source_citation": 1, - "response.context_utilization": 9.134969325153374, - "output_response": "Web Search Answer: The web search did not yield specific results regarding the latest breakthroughs in artificial intelligence. However, I can provide a general overview based on knowledge up to October 2023.\n\nRecent br...", - "evaluation.search_effectiveness": 0.2, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 1, - "evaluation.context_relevance": "low", - "evaluation.overall_quality": "low", - "evaluation.ready_for_production": true, - "web_search.search_strategy": "duckduckgo_api", - "web_search.real_time_data": true, - "web_search.external_sources": 1, - "web_search.information_freshness": "current", - "web_search.knowledge_synthesis": "single_source" - }, - "events": [], - "links": [], - "trace_trace_id": "bae13042-cf2e-4813-a673-7dc2a611725e", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:56.153000000", - "trace_end_time": "2025-10-16 08:51:01.908000000", - "trace_duration_ms": 5754.578, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:56.153000000", - "updated_at": "2025-10-16 08:50:56.153000000" - }, - { - "span_id": "04cb7d29-9ea8-4ae0-9834-030cae944eca", - "trace_id": "79f9d235-d46e-4740-be35-ef0006071f6c", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-16 08:50:51.539000000", - "end_time": "2025-10-16 08:50:55.150000000", - "duration_ms": 3611.008, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What's happening in the software development world today?", - "query.length": 57, - "processing.mode": "Web Search", - "processing.time_seconds": 3.6109254360198975, - "processing.response_length": 526, - "processing.sources_count": 1, - "output_response": "Final Answer: The provided web search results do not contain specific information about current events or trends in the software development world. They primarily focus on unrelated topics, such as the meaning of \"...", - "final_answer_mode": "Web Search", - "query_processed.input_query": "What's happening in the software development world today?", - "query_processed.output_response": "Successfully processed query using Web Search, generated 526 character response", - "query_processed.mode": "Web Search", - "query_processed.processing_time": 3.6109254360198975, - "query_processed.response_length": 526 - }, - "events": [], - "links": [], - "trace_trace_id": "79f9d235-d46e-4740-be35-ef0006071f6c", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:51.539000000", - "trace_end_time": "2025-10-16 08:50:55.150000000", - "trace_duration_ms": 3611.084, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:51.539000000", - "updated_at": "2025-10-16 08:50:51.539000000" - }, - { - "span_id": "e5e3c927-783e-48c5-aec7-b32bae98c0a2", - "trace_id": "79f9d235-d46e-4740-be35-ef0006071f6c", - "parent_span_id": "f630c3e2-091e-4bd9-9556-e0ebc1c50291", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-16 08:50:51.539000000", - "end_time": "2025-10-16 08:50:51.540000000", - "duration_ms": 0.041, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What's happening in the software development world today?", - "query_length": 57, - "query_type": "routing_evaluation_query", - "classification.mode": "Web Search", - "classification.rag_keyword_score": 0, - "classification.web_keyword_score": 1, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "complex", - "query.intent": "general_knowledge", - "query.keyword_density": 0.125, - "query.domain_affinity": "general", - "routing.decision": "Routed to Web Search based on analysis", - "routing.rationale": "RAG score: 0, Web score: 1, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "RAG", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.019230769230769232, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: Web Search (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "79f9d235-d46e-4740-be35-ef0006071f6c", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:51.539000000", - "trace_end_time": "2025-10-16 08:50:55.150000000", - "trace_duration_ms": 3611.084, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:51.539000000", - "updated_at": "2025-10-16 08:50:51.539000000" - }, - { - "span_id": "f630c3e2-091e-4bd9-9556-e0ebc1c50291", - "trace_id": "79f9d235-d46e-4740-be35-ef0006071f6c", - "parent_span_id": "04cb7d29-9ea8-4ae0-9834-030cae944eca", - "name": "agent.query_routing", - "start_time": "2025-10-16 08:50:51.539000000", - "end_time": "2025-10-16 08:50:55.150000000", - "duration_ms": 3610.876, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What's happening in the software development world today?", - "query_length": 57, - "query_type": "routing_query", - "output_response": "Routed to Web Search for query processing", - "router.classification": "Web Search", - "router.confidence_score": 1, - "router.rag_keyword_score": 0, - "router.web_keyword_score": 1, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "79f9d235-d46e-4740-be35-ef0006071f6c", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:51.539000000", - "trace_end_time": "2025-10-16 08:50:55.150000000", - "trace_duration_ms": 3611.084, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:51.539000000", - "updated_at": "2025-10-16 08:50:51.539000000" - }, - { - "span_id": "d7b0f399-3b43-43bf-bac1-b2101268841c", - "trace_id": "79f9d235-d46e-4740-be35-ef0006071f6c", - "parent_span_id": "f630c3e2-091e-4bd9-9556-e0ebc1c50291", - "name": "agent.web_search_generation", - "start_time": "2025-10-16 08:50:51.540000000", - "end_time": "2025-10-16 08:50:55.150000000", - "duration_ms": 3610.679, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "web_search_agent", - "agent.operation": "web_search_generation", - "agent.capabilities": [ - "web_search", - "content_synthesis", - "response_generation" - ], - "agent.id": "noveum_web_search_agent", - "input_query": "What's happening in the software development world today?", - "query_length": 57, - "query_type": "web_search_query", - "output_response": "Web Search Answer: The provided web search results do not contain specific information about current events or trends in the software development world. They primarily focus on unrelated topics, such as the meaning of \"...", - "web_search.results_count": 1, - "web_search.context_length": 589, - "web_search.response_length": 526, - "web_search.mode": "external_web_search" - }, - "events": [], - "links": [], - "trace_trace_id": "79f9d235-d46e-4740-be35-ef0006071f6c", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:51.539000000", - "trace_end_time": "2025-10-16 08:50:55.150000000", - "trace_duration_ms": 3611.084, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:51.539000000", - "updated_at": "2025-10-16 08:50:51.539000000" - }, - { - "span_id": "4ed5dae4-f850-4644-aa55-2f1c00d11c22", - "trace_id": "79f9d235-d46e-4740-be35-ef0006071f6c", - "parent_span_id": "d7b0f399-3b43-43bf-bac1-b2101268841c", - "name": "agent.llm_model_execution", - "start_time": "2025-10-16 08:50:51.976000000", - "end_time": "2025-10-16 08:50:55.150000000", - "duration_ms": 3174.51, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What's happening in the software development world today?", - "query_length": 57, - "query_type": "web_search_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 3.174463987350464, - "model.latency_ms": 3174.463987350464, - "model.start_time": 1760604651.9761698, - "model.end_time": 1760604655.1506338, - "model.performance_tier": "medium", - "model.prompt_tokens": 272, - "model.completion_tokens": 131, - "model.total_tokens": 403, - "model.tokens_per_second": 126.95056601866197, - "model.estimated_cost": 0.01209, - "model.efficiency_score": 1.305210918114144, - "model.response_length": 526, - "model.response_quality": "high", - "model.output_response": "Model Response: The provided web search results do not contain specific information about current events or trends in the software development world. They primarily focus on unrelated topics, such as the meaning of \"...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "79f9d235-d46e-4740-be35-ef0006071f6c", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:51.539000000", - "trace_end_time": "2025-10-16 08:50:55.150000000", - "trace_duration_ms": 3611.084, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:51.539000000", - "updated_at": "2025-10-16 08:50:51.539000000" - }, - { - "span_id": "eb151834-9322-4f63-add0-dec9a2966367", - "trace_id": "79f9d235-d46e-4740-be35-ef0006071f6c", - "parent_span_id": "d7b0f399-3b43-43bf-bac1-b2101268841c", - "name": "agent.web_search_evaluation_metrics", - "start_time": "2025-10-16 08:50:55.150000000", - "end_time": "2025-10-16 08:50:55.150000000", - "duration_ms": 0.02, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "web_search_evaluation_metrics", - "agent.capabilities": [ - "web_search_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What's happening in the software development world today?", - "query_length": 57, - "query_type": "web_search_evaluation_query", - "web_search.results_count": 1, - "web_search.context_length": 589, - "web_search.context_synthesized": "Context from 1 web sources", - "web_search.search_effectiveness": 0.2, - "web_search.context_quality": "medium", - "web_search.source_diversity": 1, - "web_search.information_synthesis": "low", - "web_search.external_knowledge_utilization": 0.589, - "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\n\u66fc\u5df4\u7684\u610f\u601d\u662f\u4e00\u79cd\u6bd2\u86c7\uff0c\u8c61\u5f81\u7740\u79d1\u6bd4\u5728\u7403\u573a\u4e0a\u7684\u653b\u51fb\u6027\u548c\u575a\u97e7\u4e0d\u62d4 \u201cWhat can i say\uff0c Mamba out\uff01\u201d \u4f46\u662f\u7ecf\u8fc7\u4e92\u8054\u7f51\u7684\u89e3\u6784\uff0c\u8fd9\u53e5\u8bdd\u6210\u4e3a\u4e86\u4e00\u53e5\u4e07\u7528\u573a\u666f\u7684\u8bdd \u2026 Aprende ingl\u00e9s gratis con nuestros cursos. \u00a1Mas de 100 lecciones con ejemplos y audio! Distintas y divertidas formas de estudiar ingl\u00e9s: \u2026 \u77e5\u4e4e\uff0c\u4e2d\u6587\u4e92\u8054\u7f51\u9ad8\u8d28\u91cf\u7684\u95ee\u7b54\u793e\u533a\u548c\u521b\u4f5c\u8005\u805a\u96c6\u7684\u539f\u521b\u5185\u5bb9\u5e73\u53f0\uff0c\u4e8e 2011 \u5e74 1 \u6708\u6b63\u5f0f\u4e0a\u7ebf\uff0c\u4ee5\u300c\u8ba9\u4eba\u4eec\u66f4\u597d\u7684\u5206\u4eab\u77e5\u8bc6\u3001\u7ecf\u9a8c\u548c\u89c1\u89e3\uff0c\u627e\u5230\u81ea\u5df1\u7684\u89e3\u7b54\u300d\u4e3a\u54c1 \u2026 \u8f6f\u4ef6\u7279\u70b9 1\u3001Windows Desktop Runtime\u662f\u5fae\u8f6fWindows\u684c\u9762\u7a0b\u5e8f\u8fd0\u884c\u5e93\uff08\u542b\u5e38\u89c4\u8fd0\u884c\u5e93\uff09 2\u3001\u80fd\u5b8c\u7f8e\u517c\u5bb9\u5fae\u8f6f\u4e0d\u540c\u7248\u672c\u7684Windows\u7cfb\u7edf\uff0c\u89e3\u51b3\u5176\u7a0b\u5e8f\u7f3a \u2026 \u77e5\u4e4e\uff0c\u4e2d\u6587\u4e92\u8054\u7f51\u9ad8\u8d28\u91cf\u7684\u95ee\u7b54\u793e\u533a\u548c\u521b\u4f5c\u8005\u805a\u96c6\u7684\u539f\u521b\u5185\u5bb9\u5e73\u53f0\uff0c\u4e8e 2011 \u5e74 1 \u6708\u6b63\u5f0f\u4e0a\u7ebf\uff0c\u4ee5\u300c\u8ba9\u4eba\u4eec\u66f4\u597d\u7684\u5206\u4eab\u77e5\u8bc6\u3001\u7ecf\u9a8c\u548c\u89c1\u89e3\uff0c\u627e\u5230\u81ea\u5df1\u7684\u89e3\u7b54\u300d\u4e3a\u54c1 \u2026\nURL: https://duckduckgo.com/?q=What's+happening+in+the+software+development+world+today?\n\nUser Question: What's happening in the software development world today?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", - "prompt.prompt_length": 1091, - "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\n\u66fc\u5df4\u7684\u610f\u601d\u662f\u4e00\u79cd\u6bd2\u86c7\uff0c\u8c61\u5f81\u7740\u79d1\u6bd4\u5728\u7403\u573a\u4e0a\u7684\u653b\u51fb\u6027\u548c\u575a\u97e7\u4e0d\u62d4 \u201cWhat can i say\uff0c Mamba out\uff01\u201d \u4f46\u662f\u7ecf\u8fc7\u4e92\u8054\u7f51\u7684\u89e3\u6784\uff0c\u8fd9\u53e5\u8bdd\u6210\u4e3a\u4e86\u4e00\u53e5\u4e07\u7528\u573a\u666f\u7684\u8bdd \u2026 Aprende ingl\u00e9s gratis con nuestros cursos. \u00a1Mas de 100 lecciones con ejemplos y audio...", - "prompt.instruction_following": "web_search_optimized", - "response.answer_length": 526, - "response.answer_completeness": "complete", - "response.response_quality": "medium", - "response.source_citation": 1, - "response.context_utilization": 0.8930390492359932, - "output_response": "Web Search Answer: The provided web search results do not contain specific information about current events or trends in the software development world. They primarily focus on unrelated topics, such as the meaning of \"...", - "evaluation.search_effectiveness": 0.2, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 1, - "evaluation.context_relevance": "medium", - "evaluation.overall_quality": "low", - "evaluation.ready_for_production": true, - "web_search.search_strategy": "duckduckgo_api", - "web_search.real_time_data": true, - "web_search.external_sources": 1, - "web_search.information_freshness": "current", - "web_search.knowledge_synthesis": "single_source" - }, - "events": [], - "links": [], - "trace_trace_id": "79f9d235-d46e-4740-be35-ef0006071f6c", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:51.539000000", - "trace_end_time": "2025-10-16 08:50:55.150000000", - "trace_duration_ms": 3611.084, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:51.539000000", - "updated_at": "2025-10-16 08:50:51.539000000" - }, - { - "span_id": "888b4441-3678-42a5-a752-8daae2433482", - "trace_id": "77b81bd5-5a04-4fc0-9fc6-571f7907d6b1", - "parent_span_id": "ba6ffeeb-78b5-44dd-b460-7933ca6e7739", - "name": "agent.query_routing", - "start_time": "2025-10-16 08:50:44.888000000", - "end_time": "2025-10-16 08:50:50.529000000", - "duration_ms": 5641.062, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What are the newest features in cloud computing?", - "query_length": 48, - "query_type": "routing_query", - "output_response": "Routed to RAG for query processing", - "router.classification": "RAG", - "router.confidence_score": 1, - "router.rag_keyword_score": 1, - "router.web_keyword_score": 0, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "77b81bd5-5a04-4fc0-9fc6-571f7907d6b1", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:44.888000000", - "trace_end_time": "2025-10-16 08:50:50.529000000", - "trace_duration_ms": 5641.226, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:44.888000000", - "updated_at": "2025-10-16 08:50:44.888000000" - }, - { - "span_id": "b6bae768-93a6-47d9-8369-61897f9d7828", - "trace_id": "77b81bd5-5a04-4fc0-9fc6-571f7907d6b1", - "parent_span_id": "888b4441-3678-42a5-a752-8daae2433482", - "name": "agent.llm-rag", - "start_time": "2025-10-16 08:50:44.888000000", - "end_time": "2025-10-16 08:50:50.529000000", - "duration_ms": 5640.875999999999, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "rag_agent", - "agent.operation": "llm-rag", - "agent.capabilities": [ - "document_retrieval", - "context_generation", - "response_generation" - ], - "agent.id": "noveum_rag_agent", - "input_query": "What are the newest features in cloud computing?", - "query_length": 48, - "query_type": "rag_query", - "output_response": "RAG Answer: The provided context does not specifically mention any new features in cloud computing as a general topic. However, it does highlight several recent enhancements and features related to the Noveum.ai ...", - "rag.context_length": 2579, - "rag.sources_count": 5, - "rag.answer_length": 983, - "rag.mode": "retrieval_augmented_generation" - }, - "events": [], - "links": [], - "trace_trace_id": "77b81bd5-5a04-4fc0-9fc6-571f7907d6b1", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:44.888000000", - "trace_end_time": "2025-10-16 08:50:50.529000000", - "trace_duration_ms": 5641.226, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:44.888000000", - "updated_at": "2025-10-16 08:50:44.888000000" - }, - { - "span_id": "ba6ffeeb-78b5-44dd-b460-7933ca6e7739", - "trace_id": "77b81bd5-5a04-4fc0-9fc6-571f7907d6b1", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-16 08:50:44.888000000", - "end_time": "2025-10-16 08:50:50.529000000", - "duration_ms": 5641.156, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What are the newest features in cloud computing?", - "query.length": 48, - "processing.mode": "RAG", - "processing.time_seconds": 5.64110541343689, - "processing.response_length": 983, - "processing.sources_count": 5, - "output_response": "Final Answer: The provided context does not specifically mention any new features in cloud computing as a general topic. However, it does highlight several recent enhancements and features related to the Noveum.ai ...", - "final_answer_mode": "RAG", - "query_processed.input_query": "What are the newest features in cloud computing?", - "query_processed.output_response": "Successfully processed query using RAG, generated 983 character response", - "query_processed.mode": "RAG", - "query_processed.processing_time": 5.64110541343689, - "query_processed.response_length": 983 - }, - "events": [], - "links": [], - "trace_trace_id": "77b81bd5-5a04-4fc0-9fc6-571f7907d6b1", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:44.888000000", - "trace_end_time": "2025-10-16 08:50:50.529000000", - "trace_duration_ms": 5641.226, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:44.888000000", - "updated_at": "2025-10-16 08:50:44.888000000" - }, - { - "span_id": "f5538bff-8908-4a56-a622-4ba2eeedfd29", - "trace_id": "77b81bd5-5a04-4fc0-9fc6-571f7907d6b1", - "parent_span_id": "888b4441-3678-42a5-a752-8daae2433482", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-16 08:50:44.888000000", - "end_time": "2025-10-16 08:50:44.888000000", - "duration_ms": 0.032, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What are the newest features in cloud computing?", - "query_length": 48, - "query_type": "routing_evaluation_query", - "classification.mode": "RAG", - "classification.rag_keyword_score": 1, - "classification.web_keyword_score": 0, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "documentation", - "query.keyword_density": 0.125, - "query.domain_affinity": "noveum", - "routing.decision": "Routed to RAG based on analysis", - "routing.rationale": "RAG score: 1, Web score: 0, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "Web Search", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.019230769230769232, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: RAG (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "77b81bd5-5a04-4fc0-9fc6-571f7907d6b1", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:44.888000000", - "trace_end_time": "2025-10-16 08:50:50.529000000", - "trace_duration_ms": 5641.226, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:44.888000000", - "updated_at": "2025-10-16 08:50:44.888000000" - }, - { - "span_id": "856e51cf-fa4d-459b-a046-1ea7316b903e", - "trace_id": "77b81bd5-5a04-4fc0-9fc6-571f7907d6b1", - "parent_span_id": "b6bae768-93a6-47d9-8369-61897f9d7828", - "name": "agent.llm_model_execution", - "start_time": "2025-10-16 08:50:45.423000000", - "end_time": "2025-10-16 08:50:49.997000000", - "duration_ms": 4573.782999999999, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What are the newest features in cloud computing?", - "query_length": 48, - "query_type": "rag_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 4.573714733123779, - "model.latency_ms": 4573.714733123779, - "model.start_time": 1760604645.423261, - "model.end_time": 1760604649.9969757, - "model.performance_tier": "medium", - "model.prompt_tokens": 780, - "model.completion_tokens": 245, - "model.total_tokens": 1025, - "model.tokens_per_second": 224.1066747291299, - "model.estimated_cost": 0.03075, - "model.efficiency_score": 0.9590243902439024, - "model.response_length": 983, - "model.response_quality": "high", - "model.output_response": "Model Response: The provided context does not specifically mention any new features in cloud computing as a general topic. However, it does highlight several recent enhancements and features related to the Noveum.ai ...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "77b81bd5-5a04-4fc0-9fc6-571f7907d6b1", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:44.888000000", - "trace_end_time": "2025-10-16 08:50:50.529000000", - "trace_duration_ms": 5641.226, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:44.888000000", - "updated_at": "2025-10-16 08:50:44.888000000" - }, - { - "span_id": "a2fa7400-74c4-4718-a562-899b91503c95", - "trace_id": "77b81bd5-5a04-4fc0-9fc6-571f7907d6b1", - "parent_span_id": "b6bae768-93a6-47d9-8369-61897f9d7828", - "name": "agent.rag_evaluation_metrics", - "start_time": "2025-10-16 08:50:49.997000000", - "end_time": "2025-10-16 08:50:49.997000000", - "duration_ms": 0.048, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "rag_evaluation_metrics", - "agent.capabilities": [ - "retrieval_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What are the newest features in cloud computing?", - "query_length": 48, - "query_type": "rag_evaluation_query", - "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/changelog):\n- \ud83d\udcc8 Enhanced Dashboard Analytics: Real-time request tracking, latency monitoring, and cost analysis\n- \ud83d\udd0d Improved Logs Interface: Better search functionality, detailed trace views, and enhanced debugging capabilities\n- \ud83d\udc65 Team Management: Advanced member mana...", - "retrieval.context_length": 2579, - "retrieval.sources_count": 5, - "retrieval.context_quality": "high", - "retrieval.effectiveness": 1, - "retrieval.context_utilization": 2.579, - "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/changelog):\n- \ud83d\udcc8 Enhanced Dashboard Analytics: Real-time request tracking, latency monitoring, and cost analysis\n- \ud83d\udd0d Improved Logs Interface: Better search functionality, detailed trace views, and enhanced debugging capabilities\n- \ud83d\udc65 Team Management: Advanced member management with role-based access control and invitation system\n- \ud83d\udee1\ufe0f Security Improvements: Enhanced API key management and secure credential storage\n4 months ago\n- \ud83d\udcca Advanced Metrics Collection: Comprehensive telemetry system with custom metrics ...\n\nSource 2 (https://noveum.ai/en/changelog):\n- \ud83d\udcdd Documentation Updates: Comprehensive guides for integration and best practices\n5 months ago\n- \ud83c\udf89 Platform Foundation: Initial release of Noveum.ai AI observability platform\n- \ud83d\ude80 Core Features: AI Gateway integration, basic evaluation metrics, and monitoring capabilities\n- \ud83d\udcc8 Evaluation Jobs: Automated model evaluation with configurable metrics and reporting\n- \ud83d\udd0c Provider Integrations: Support for major AI providers and custom model deployments...\n\nSource 3 (https://noveum.ai/docs/platform/dashboard):\n- Bulk Export: Download multiple traces simultaneously\n- Comparative Analysis: Compare performance across multiple traces\nReal-Time Updates\n- Live Refresh: Automatic updates as new traces arrive\n- Connection Monitoring: Real-time status of your trace ingestion pipeline\n- Performance Indicators: Live metrics for system health monitoring\nReady to dive deeper? Explore Projects & Environments to organize your AI applications, or check out Team Collaboration to share insights with your team.\nGet Earl...\n\nSource 4 (https://noveum.ai/en/changelog):\nChangelog\nStay up to date with the latest changes in our product.\n2 months ago\n- \ud83c\udfaf Major Platform Evolution: NovaEval Framework - Launched comprehensive AI model evaluation framework with 20+ built-in scorers including accuracy, RAG metrics, conversational scoring, and specialized agent evaluation tools. Features production-grade deployment on Docker and Kubernetes. (GitHub | PyPI Package)\n- \ud83d\ude80 Next-Generation Traces UI - Complete redesign featuring three-pane layout with directory tree navigatio...\n\nSource 5 (https://noveum.ai/en/blog):\n8/12/2025\nDiscover how Noveum.ai provides comprehensive tracing and observability for AI applications, from development debugging to production optimization.\nShashank Agarwal\n3/3/2025\nDiscover how Noveum.ai provides comprehensive tracing and observability for LLM applications, RAG systems, and multi-agent workflows with our powerful Python and TypeScript SDKs.\nShashank Agarwal\n3/2/2025...\n\nUser Question: What are the newest features in cloud computing?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", - "prompt.prompt_length": 3123, - "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/changelog):\n- \ud83d\udcc8 Enhanced Dashboard Analytics: Real-time request tracking, latency monitoring, and cost analysis\n- \ud83d\udd0d Improved Logs Interface: Better search functionality,...", - "prompt.instruction_following": "rag_optimized", - "response.answer_length": 983, - "response.answer_completeness": "complete", - "response.response_quality": "high", - "response.source_citation": 5, - "response.context_utilization": 0.381155486622722, - "output_response": "RAG Answer: The provided context does not specifically mention any new features in cloud computing as a general topic. However, it does highlight several recent enhancements and features related to the Noveum.ai ...", - "evaluation.retrieval_effectiveness": 1, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 5, - "evaluation.context_relevance": "high", - "evaluation.overall_quality": "high", - "evaluation.ready_for_production": true, - "rag.retrieval_strategy": "semantic_similarity", - "rag.vector_search_results": 5, - "rag.context_synthesis": "multi_source", - "rag.document_coverage": 1, - "rag.information_density": 0.381155486622722 - }, - "events": [], - "links": [], - "trace_trace_id": "77b81bd5-5a04-4fc0-9fc6-571f7907d6b1", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:44.888000000", - "trace_end_time": "2025-10-16 08:50:50.529000000", - "trace_duration_ms": 5641.226, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:44.888000000", - "updated_at": "2025-10-16 08:50:44.888000000" - }, - { - "span_id": "4c391bb1-1a62-4e2a-913b-96f83feff494", - "trace_id": "32170d1a-d211-46d8-9cdc-5a4265f0c443", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-16 08:50:42.124000000", - "end_time": "2025-10-16 08:50:43.885000000", - "duration_ms": 1760.974, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What's the current status of cryptocurrency markets?", - "query.length": 52, - "processing.mode": "Web Search", - "processing.time_seconds": 1.7608659267425537, - "processing.response_length": 363, - "processing.sources_count": 1, - "output_response": "Final Answer: The web search did not yield any specific information regarding the current status of cryptocurrency markets. Therefore, I cannot provide an update on market trends, prices, or significant events affe...", - "final_answer_mode": "Web Search", - "query_processed.input_query": "What's the current status of cryptocurrency markets?", - "query_processed.output_response": "Successfully processed query using Web Search, generated 363 character response", - "query_processed.mode": "Web Search", - "query_processed.processing_time": 1.7608659267425537, - "query_processed.response_length": 363 - }, - "events": [], - "links": [], - "trace_trace_id": "32170d1a-d211-46d8-9cdc-5a4265f0c443", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:42.124000000", - "trace_end_time": "2025-10-16 08:50:43.885000000", - "trace_duration_ms": 1761.047, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:42.124000000", - "updated_at": "2025-10-16 08:50:42.124000000" - }, - { - "span_id": "7ab2ec7b-a617-4ebf-9bb1-82dc6b18e160", - "trace_id": "32170d1a-d211-46d8-9cdc-5a4265f0c443", - "parent_span_id": "4c391bb1-1a62-4e2a-913b-96f83feff494", - "name": "agent.query_routing", - "start_time": "2025-10-16 08:50:42.124000000", - "end_time": "2025-10-16 08:50:43.885000000", - "duration_ms": 1760.833, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What's the current status of cryptocurrency markets?", - "query_length": 52, - "query_type": "routing_query", - "output_response": "Routed to Web Search for query processing", - "router.classification": "Web Search", - "router.confidence_score": 1, - "router.rag_keyword_score": 0, - "router.web_keyword_score": 3, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "32170d1a-d211-46d8-9cdc-5a4265f0c443", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:42.124000000", - "trace_end_time": "2025-10-16 08:50:43.885000000", - "trace_duration_ms": 1761.047, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:42.124000000", - "updated_at": "2025-10-16 08:50:42.124000000" - }, - { - "span_id": "db01d228-7c63-4814-b2ea-bdf3898f9c0d", - "trace_id": "32170d1a-d211-46d8-9cdc-5a4265f0c443", - "parent_span_id": "7ab2ec7b-a617-4ebf-9bb1-82dc6b18e160", - "name": "agent.web_search_generation", - "start_time": "2025-10-16 08:50:42.124000000", - "end_time": "2025-10-16 08:50:43.885000000", - "duration_ms": 1760.701, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "web_search_agent", - "agent.operation": "web_search_generation", - "agent.capabilities": [ - "web_search", - "content_synthesis", - "response_generation" - ], - "agent.id": "noveum_web_search_agent", - "input_query": "What's the current status of cryptocurrency markets?", - "query_length": 52, - "query_type": "web_search_query", - "output_response": "Web Search Answer: The web search did not yield any specific information regarding the current status of cryptocurrency markets. Therefore, I cannot provide an update on market trends, prices, or significant events affe...", - "web_search.results_count": 1, - "web_search.context_length": 154, - "web_search.response_length": 363, - "web_search.mode": "external_web_search" - }, - "events": [], - "links": [], - "trace_trace_id": "32170d1a-d211-46d8-9cdc-5a4265f0c443", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:42.124000000", - "trace_end_time": "2025-10-16 08:50:43.885000000", - "trace_duration_ms": 1761.047, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:42.124000000", - "updated_at": "2025-10-16 08:50:42.124000000" - }, - { - "span_id": "da7b0550-b13c-4518-ad3b-050c666d198c", - "trace_id": "32170d1a-d211-46d8-9cdc-5a4265f0c443", - "parent_span_id": "7ab2ec7b-a617-4ebf-9bb1-82dc6b18e160", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-16 08:50:42.124000000", - "end_time": "2025-10-16 08:50:42.124000000", - "duration_ms": 0.029, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What's the current status of cryptocurrency markets?", - "query_length": 52, - "query_type": "routing_evaluation_query", - "classification.mode": "Web Search", - "classification.rag_keyword_score": 0, - "classification.web_keyword_score": 3, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "complex", - "query.intent": "general_knowledge", - "query.keyword_density": 0.42857142857142855, - "query.domain_affinity": "general", - "routing.decision": "Routed to Web Search based on analysis", - "routing.rationale": "RAG score: 0, Web score: 3, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "RAG", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.057692307692307696, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: Web Search (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "32170d1a-d211-46d8-9cdc-5a4265f0c443", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:42.124000000", - "trace_end_time": "2025-10-16 08:50:43.885000000", - "trace_duration_ms": 1761.047, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:42.124000000", - "updated_at": "2025-10-16 08:50:42.124000000" - }, - { - "span_id": "950e18b6-9f62-4ac4-8788-3cac08fb7899", - "trace_id": "32170d1a-d211-46d8-9cdc-5a4265f0c443", - "parent_span_id": "db01d228-7c63-4814-b2ea-bdf3898f9c0d", - "name": "agent.llm_model_execution", - "start_time": "2025-10-16 08:50:42.465000000", - "end_time": "2025-10-16 08:50:43.885000000", - "duration_ms": 1419.5690000000002, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What's the current status of cryptocurrency markets?", - "query_length": 52, - "query_type": "web_search_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 1.4195220470428467, - "model.latency_ms": 1419.5220470428467, - "model.start_time": 1760604642.4658613, - "model.end_time": 1760604643.8853834, - "model.performance_tier": "fast", - "model.prompt_tokens": 162, - "model.completion_tokens": 90, - "model.total_tokens": 252, - "model.tokens_per_second": 177.52454111224782, - "model.estimated_cost": 0.00756, - "model.efficiency_score": 1.4404761904761905, - "model.response_length": 363, - "model.response_quality": "high", - "model.output_response": "Model Response: The web search did not yield any specific information regarding the current status of cryptocurrency markets. Therefore, I cannot provide an update on market trends, prices, or significant events affe...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "32170d1a-d211-46d8-9cdc-5a4265f0c443", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:42.124000000", - "trace_end_time": "2025-10-16 08:50:43.885000000", - "trace_duration_ms": 1761.047, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:42.124000000", - "updated_at": "2025-10-16 08:50:42.124000000" - }, - { - "span_id": "e87846c9-ce82-42c2-a517-e0f67e9d14c1", - "trace_id": "32170d1a-d211-46d8-9cdc-5a4265f0c443", - "parent_span_id": "db01d228-7c63-4814-b2ea-bdf3898f9c0d", - "name": "agent.web_search_evaluation_metrics", - "start_time": "2025-10-16 08:50:43.885000000", - "end_time": "2025-10-16 08:50:43.885000000", - "duration_ms": 0.02, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "web_search_evaluation_metrics", - "agent.capabilities": [ - "web_search_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What's the current status of cryptocurrency markets?", - "query_length": 52, - "query_type": "web_search_evaluation_query", - "web_search.results_count": 1, - "web_search.context_length": 154, - "web_search.context_synthesized": "Context from 1 web sources", - "web_search.search_effectiveness": 0.2, - "web_search.context_quality": "low", - "web_search.source_diversity": 1, - "web_search.information_synthesis": "low", - "web_search.external_knowledge_utilization": 0.154, - "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What's+the+current+status+of+cryptocurrency+markets?\n\nUser Question: What's the current status of cryptocurrency markets?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", - "prompt.prompt_length": 651, - "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What's+the+current+status+of+cryptocurrency+markets?", - "prompt.instruction_following": "web_search_optimized", - "response.answer_length": 363, - "response.answer_completeness": "complete", - "response.response_quality": "medium", - "response.source_citation": 1, - "response.context_utilization": 2.357142857142857, - "output_response": "Web Search Answer: The web search did not yield any specific information regarding the current status of cryptocurrency markets. Therefore, I cannot provide an update on market trends, prices, or significant events affe...", - "evaluation.search_effectiveness": 0.2, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 1, - "evaluation.context_relevance": "low", - "evaluation.overall_quality": "low", - "evaluation.ready_for_production": true, - "web_search.search_strategy": "duckduckgo_api", - "web_search.real_time_data": true, - "web_search.external_sources": 1, - "web_search.information_freshness": "current", - "web_search.knowledge_synthesis": "single_source" - }, - "events": [], - "links": [], - "trace_trace_id": "32170d1a-d211-46d8-9cdc-5a4265f0c443", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:42.124000000", - "trace_end_time": "2025-10-16 08:50:43.885000000", - "trace_duration_ms": 1761.047, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:42.124000000", - "updated_at": "2025-10-16 08:50:42.124000000" - }, - { - "span_id": "305daacf-990b-4e23-8720-48900f75d9be", - "trace_id": "9424d420-deee-478b-b727-9dc791a7457c", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-16 08:50:31.476000000", - "end_time": "2025-10-16 08:50:41.121000000", - "duration_ms": 9644.752999999999, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What are the latest updates in Python programming?", - "query.length": 50, - "processing.mode": "Web Search", - "processing.time_seconds": 9.644662618637085, - "processing.response_length": 1408, - "processing.sources_count": 1, - "output_response": "Final Answer: The web search did not yield specific results regarding the latest updates in Python programming. However, I can provide some general information based on my knowledge up to October 2023.\n\nAs of late ...", - "final_answer_mode": "Web Search", - "query_processed.input_query": "What are the latest updates in Python programming?", - "query_processed.output_response": "Successfully processed query using Web Search, generated 1408 character response", - "query_processed.mode": "Web Search", - "query_processed.processing_time": 9.644662618637085, - "query_processed.response_length": 1408 - }, - "events": [], - "links": [], - "trace_trace_id": "9424d420-deee-478b-b727-9dc791a7457c", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:31.476000000", - "trace_end_time": "2025-10-16 08:50:41.121000000", - "trace_duration_ms": 9644.828, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:31.476000000", - "updated_at": "2025-10-16 08:50:31.476000000" - }, - { - "span_id": "56b4cc1b-0646-4dee-acb3-db98f3d44477", - "trace_id": "9424d420-deee-478b-b727-9dc791a7457c", - "parent_span_id": "fb234e30-69be-4d06-9248-c05bcc22f063", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-16 08:50:31.476000000", - "end_time": "2025-10-16 08:50:31.476000000", - "duration_ms": 0.036000000000000004, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What are the latest updates in Python programming?", - "query_length": 50, - "query_type": "routing_evaluation_query", - "classification.mode": "Web Search", - "classification.rag_keyword_score": 0, - "classification.web_keyword_score": 2, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "general_knowledge", - "query.keyword_density": 0.25, - "query.domain_affinity": "general", - "routing.decision": "Routed to Web Search based on analysis", - "routing.rationale": "RAG score: 0, Web score: 2, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "RAG", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.038461538461538464, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: Web Search (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "9424d420-deee-478b-b727-9dc791a7457c", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:31.476000000", - "trace_end_time": "2025-10-16 08:50:41.121000000", - "trace_duration_ms": 9644.828, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:31.476000000", - "updated_at": "2025-10-16 08:50:31.476000000" - }, - { - "span_id": "fb234e30-69be-4d06-9248-c05bcc22f063", - "trace_id": "9424d420-deee-478b-b727-9dc791a7457c", - "parent_span_id": "305daacf-990b-4e23-8720-48900f75d9be", - "name": "agent.query_routing", - "start_time": "2025-10-16 08:50:31.476000000", - "end_time": "2025-10-16 08:50:41.121000000", - "duration_ms": 9644.627, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What are the latest updates in Python programming?", - "query_length": 50, - "query_type": "routing_query", - "output_response": "Routed to Web Search for query processing", - "router.classification": "Web Search", - "router.confidence_score": 1, - "router.rag_keyword_score": 0, - "router.web_keyword_score": 2, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "9424d420-deee-478b-b727-9dc791a7457c", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:31.476000000", - "trace_end_time": "2025-10-16 08:50:41.121000000", - "trace_duration_ms": 9644.828, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:31.476000000", - "updated_at": "2025-10-16 08:50:31.476000000" - }, - { - "span_id": "f8a793a9-63ee-49b7-9931-d7dcc2bb0ca7", - "trace_id": "9424d420-deee-478b-b727-9dc791a7457c", - "parent_span_id": "fb234e30-69be-4d06-9248-c05bcc22f063", - "name": "agent.web_search_generation", - "start_time": "2025-10-16 08:50:31.477000000", - "end_time": "2025-10-16 08:50:41.121000000", - "duration_ms": 9644.478, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "web_search_agent", - "agent.operation": "web_search_generation", - "agent.capabilities": [ - "web_search", - "content_synthesis", - "response_generation" - ], - "agent.id": "noveum_web_search_agent", - "input_query": "What are the latest updates in Python programming?", - "query_length": 50, - "query_type": "web_search_query", - "output_response": "Web Search Answer: The web search did not yield specific results regarding the latest updates in Python programming. However, I can provide some general information based on my knowledge up to October 2023.\n\nAs of late ...", - "web_search.results_count": 1, - "web_search.context_length": 152, - "web_search.response_length": 1408, - "web_search.mode": "external_web_search" - }, - "events": [], - "links": [], - "trace_trace_id": "9424d420-deee-478b-b727-9dc791a7457c", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:31.476000000", - "trace_end_time": "2025-10-16 08:50:41.121000000", - "trace_duration_ms": 9644.828, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:31.476000000", - "updated_at": "2025-10-16 08:50:31.476000000" - }, - { - "span_id": "b968e3d3-d4aa-4357-85d8-34a612748fbd", - "trace_id": "9424d420-deee-478b-b727-9dc791a7457c", - "parent_span_id": "f8a793a9-63ee-49b7-9931-d7dcc2bb0ca7", - "name": "agent.llm_model_execution", - "start_time": "2025-10-16 08:50:32.016000000", - "end_time": "2025-10-16 08:50:41.121000000", - "duration_ms": 9105.344000000001, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What are the latest updates in Python programming?", - "query_length": 50, - "query_type": "web_search_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 9.105276584625244, - "model.latency_ms": 9105.276584625244, - "model.start_time": 1760604632.0161095, - "model.end_time": 1760604641.121386, - "model.performance_tier": "slow", - "model.prompt_tokens": 161, - "model.completion_tokens": 352, - "model.total_tokens": 513, - "model.tokens_per_second": 56.340957381374714, - "model.estimated_cost": 0.015390000000000001, - "model.efficiency_score": 2.7446393762183234, - "model.response_length": 1408, - "model.response_quality": "high", - "model.output_response": "Model Response: The web search did not yield specific results regarding the latest updates in Python programming. However, I can provide some general information based on my knowledge up to October 2023.\n\nAs of late ...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "9424d420-deee-478b-b727-9dc791a7457c", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:31.476000000", - "trace_end_time": "2025-10-16 08:50:41.121000000", - "trace_duration_ms": 9644.828, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:31.476000000", - "updated_at": "2025-10-16 08:50:31.476000000" - }, - { - "span_id": "80817711-e64c-4a9f-802e-2a9e36ff093d", - "trace_id": "9424d420-deee-478b-b727-9dc791a7457c", - "parent_span_id": "f8a793a9-63ee-49b7-9931-d7dcc2bb0ca7", - "name": "agent.web_search_evaluation_metrics", - "start_time": "2025-10-16 08:50:41.121000000", - "end_time": "2025-10-16 08:50:41.121000000", - "duration_ms": 0.02, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "web_search_evaluation_metrics", - "agent.capabilities": [ - "web_search_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What are the latest updates in Python programming?", - "query_length": 50, - "query_type": "web_search_evaluation_query", - "web_search.results_count": 1, - "web_search.context_length": 152, - "web_search.context_synthesized": "Context from 1 web sources", - "web_search.search_effectiveness": 0.2, - "web_search.context_quality": "low", - "web_search.source_diversity": 1, - "web_search.information_synthesis": "low", - "web_search.external_knowledge_utilization": 0.152, - "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What+are+the+latest+updates+in+Python+programming?\n\nUser Question: What are the latest updates in Python programming?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", - "prompt.prompt_length": 647, - "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What+are+the+latest+updates+in+Python+programming?", - "prompt.instruction_following": "web_search_optimized", - "response.answer_length": 1408, - "response.answer_completeness": "complete", - "response.response_quality": "medium", - "response.source_citation": 1, - "response.context_utilization": 9.263157894736842, - "output_response": "Web Search Answer: The web search did not yield specific results regarding the latest updates in Python programming. However, I can provide some general information based on my knowledge up to October 2023.\n\nAs of late ...", - "evaluation.search_effectiveness": 0.2, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 1, - "evaluation.context_relevance": "low", - "evaluation.overall_quality": "low", - "evaluation.ready_for_production": true, - "web_search.search_strategy": "duckduckgo_api", - "web_search.real_time_data": true, - "web_search.external_sources": 1, - "web_search.information_freshness": "current", - "web_search.knowledge_synthesis": "single_source" - }, - "events": [], - "links": [], - "trace_trace_id": "9424d420-deee-478b-b727-9dc791a7457c", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:31.476000000", - "trace_end_time": "2025-10-16 08:50:41.121000000", - "trace_duration_ms": 9644.828, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:31.476000000", - "updated_at": "2025-10-16 08:50:31.476000000" - }, - { - "span_id": "13af0f3c-bd5c-4024-8fa3-c9e347a1ef20", - "trace_id": "a3735d64-89bb-4fdf-a851-7a1f3515c52d", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-16 08:50:27.687000000", - "end_time": "2025-10-16 08:50:30.473000000", - "duration_ms": 2786.078, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What happened in tech news this week?", - "query.length": 37, - "processing.mode": "Web Search", - "processing.time_seconds": 2.7859997749328613, - "processing.response_length": 396, - "processing.sources_count": 1, - "output_response": "Final Answer: It appears that the web search did not yield any specific results regarding recent tech news. Unfortunately, I cannot provide a summary of this week's tech news based on the available information. For...", - "final_answer_mode": "Web Search", - "query_processed.input_query": "What happened in tech news this week?", - "query_processed.output_response": "Successfully processed query using Web Search, generated 396 character response", - "query_processed.mode": "Web Search", - "query_processed.processing_time": 2.7859997749328613, - "query_processed.response_length": 396 - }, - "events": [], - "links": [], - "trace_trace_id": "a3735d64-89bb-4fdf-a851-7a1f3515c52d", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:27.687000000", - "trace_end_time": "2025-10-16 08:50:30.473000000", - "trace_duration_ms": 2786.145, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:27.687000000", - "updated_at": "2025-10-16 08:50:27.687000000" - }, - { - "span_id": "47e8f375-35e6-4097-86ff-df73e8181c23", - "trace_id": "a3735d64-89bb-4fdf-a851-7a1f3515c52d", - "parent_span_id": "a4f1a7b3-dc35-4f13-96d3-b6dc8fb21a37", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-16 08:50:27.687000000", - "end_time": "2025-10-16 08:50:27.687000000", - "duration_ms": 0.038, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What happened in tech news this week?", - "query_length": 37, - "query_type": "routing_evaluation_query", - "classification.mode": "Web Search", - "classification.rag_keyword_score": 0, - "classification.web_keyword_score": 2, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "general_knowledge", - "query.keyword_density": 0.2857142857142857, - "query.domain_affinity": "general", - "routing.decision": "Routed to Web Search based on analysis", - "routing.rationale": "RAG score: 0, Web score: 2, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "RAG", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.038461538461538464, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: Web Search (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "a3735d64-89bb-4fdf-a851-7a1f3515c52d", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:27.687000000", - "trace_end_time": "2025-10-16 08:50:30.473000000", - "trace_duration_ms": 2786.145, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:27.687000000", - "updated_at": "2025-10-16 08:50:27.687000000" - }, - { - "span_id": "a4f1a7b3-dc35-4f13-96d3-b6dc8fb21a37", - "trace_id": "a3735d64-89bb-4fdf-a851-7a1f3515c52d", - "parent_span_id": "13af0f3c-bd5c-4024-8fa3-c9e347a1ef20", - "name": "agent.query_routing", - "start_time": "2025-10-16 08:50:27.687000000", - "end_time": "2025-10-16 08:50:30.473000000", - "duration_ms": 2785.964, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What happened in tech news this week?", - "query_length": 37, - "query_type": "routing_query", - "output_response": "Routed to Web Search for query processing", - "router.classification": "Web Search", - "router.confidence_score": 1, - "router.rag_keyword_score": 0, - "router.web_keyword_score": 2, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "a3735d64-89bb-4fdf-a851-7a1f3515c52d", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:27.687000000", - "trace_end_time": "2025-10-16 08:50:30.473000000", - "trace_duration_ms": 2786.145, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:27.687000000", - "updated_at": "2025-10-16 08:50:27.687000000" - }, - { - "span_id": "ebfff280-5eb6-4838-8de7-35c85e2e3eaa", - "trace_id": "a3735d64-89bb-4fdf-a851-7a1f3515c52d", - "parent_span_id": "a4f1a7b3-dc35-4f13-96d3-b6dc8fb21a37", - "name": "agent.web_search_generation", - "start_time": "2025-10-16 08:50:27.687000000", - "end_time": "2025-10-16 08:50:30.473000000", - "duration_ms": 2785.8230000000003, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "web_search_agent", - "agent.operation": "web_search_generation", - "agent.capabilities": [ - "web_search", - "content_synthesis", - "response_generation" - ], - "agent.id": "noveum_web_search_agent", - "input_query": "What happened in tech news this week?", - "query_length": 37, - "query_type": "web_search_query", - "output_response": "Web Search Answer: It appears that the web search did not yield any specific results regarding recent tech news. Unfortunately, I cannot provide a summary of this week's tech news based on the available information. For...", - "web_search.results_count": 1, - "web_search.context_length": 139, - "web_search.response_length": 396, - "web_search.mode": "external_web_search" - }, - "events": [], - "links": [], - "trace_trace_id": "a3735d64-89bb-4fdf-a851-7a1f3515c52d", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:27.687000000", - "trace_end_time": "2025-10-16 08:50:30.473000000", - "trace_duration_ms": 2786.145, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:27.687000000", - "updated_at": "2025-10-16 08:50:27.687000000" - }, - { - "span_id": "835b9351-d381-4785-8538-fbe1064fc8cd", - "trace_id": "a3735d64-89bb-4fdf-a851-7a1f3515c52d", - "parent_span_id": "ebfff280-5eb6-4838-8de7-35c85e2e3eaa", - "name": "agent.llm_model_execution", - "start_time": "2025-10-16 08:50:28.026000000", - "end_time": "2025-10-16 08:50:30.473000000", - "duration_ms": 2447.269, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What happened in tech news this week?", - "query_length": 37, - "query_type": "web_search_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 2.447227716445923, - "model.latency_ms": 2447.227716445923, - "model.start_time": 1760604628.0260897, - "model.end_time": 1760604630.4733174, - "model.performance_tier": "medium", - "model.prompt_tokens": 155, - "model.completion_tokens": 99, - "model.total_tokens": 254, - "model.tokens_per_second": 103.79091340501853, - "model.estimated_cost": 0.00762, - "model.efficiency_score": 1.5590551181102361, - "model.response_length": 396, - "model.response_quality": "high", - "model.output_response": "Model Response: It appears that the web search did not yield any specific results regarding recent tech news. Unfortunately, I cannot provide a summary of this week's tech news based on the available information. For...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "a3735d64-89bb-4fdf-a851-7a1f3515c52d", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:27.687000000", - "trace_end_time": "2025-10-16 08:50:30.473000000", - "trace_duration_ms": 2786.145, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:27.687000000", - "updated_at": "2025-10-16 08:50:27.687000000" - }, - { - "span_id": "93c6f427-6fcd-4318-aec8-095553fed469", - "trace_id": "a3735d64-89bb-4fdf-a851-7a1f3515c52d", - "parent_span_id": "ebfff280-5eb6-4838-8de7-35c85e2e3eaa", - "name": "agent.web_search_evaluation_metrics", - "start_time": "2025-10-16 08:50:30.473000000", - "end_time": "2025-10-16 08:50:30.473000000", - "duration_ms": 0.017, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "web_search_evaluation_metrics", - "agent.capabilities": [ - "web_search_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What happened in tech news this week?", - "query_length": 37, - "query_type": "web_search_evaluation_query", - "web_search.results_count": 1, - "web_search.context_length": 139, - "web_search.context_synthesized": "Context from 1 web sources", - "web_search.search_effectiveness": 0.2, - "web_search.context_quality": "low", - "web_search.source_diversity": 1, - "web_search.information_synthesis": "low", - "web_search.external_knowledge_utilization": 0.139, - "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What+happened+in+tech+news+this+week?\n\nUser Question: What happened in tech news this week?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", - "prompt.prompt_length": 621, - "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What+happened+in+tech+news+this+week?", - "prompt.instruction_following": "web_search_optimized", - "response.answer_length": 396, - "response.answer_completeness": "complete", - "response.response_quality": "medium", - "response.source_citation": 1, - "response.context_utilization": 2.8489208633093526, - "output_response": "Web Search Answer: It appears that the web search did not yield any specific results regarding recent tech news. Unfortunately, I cannot provide a summary of this week's tech news based on the available information. For...", - "evaluation.search_effectiveness": 0.2, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 1, - "evaluation.context_relevance": "low", - "evaluation.overall_quality": "low", - "evaluation.ready_for_production": true, - "web_search.search_strategy": "duckduckgo_api", - "web_search.real_time_data": true, - "web_search.external_sources": 1, - "web_search.information_freshness": "current", - "web_search.knowledge_synthesis": "single_source" - }, - "events": [], - "links": [], - "trace_trace_id": "a3735d64-89bb-4fdf-a851-7a1f3515c52d", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:27.687000000", - "trace_end_time": "2025-10-16 08:50:30.473000000", - "trace_duration_ms": 2786.145, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:27.687000000", - "updated_at": "2025-10-16 08:50:27.687000000" - }, - { - "span_id": "622e7353-4eeb-47d5-8558-4ecec28cdcfd", - "trace_id": "fa5bb641-f4b7-4741-b06c-392a5f2a9aea", - "parent_span_id": "c0af6629-48b1-45fc-9dd2-60f098ea140e", - "name": "agent.query_routing", - "start_time": "2025-10-16 08:50:21.645000000", - "end_time": "2025-10-16 08:50:26.683000000", - "duration_ms": 5037.965, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What are the current trends in observability tools?", - "query_length": 51, - "query_type": "routing_query", - "output_response": "Routed to RAG for query processing", - "router.classification": "RAG", - "router.confidence_score": 0.3333333333333333, - "router.rag_keyword_score": 2, - "router.web_keyword_score": 1, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "fa5bb641-f4b7-4741-b06c-392a5f2a9aea", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:21.645000000", - "trace_end_time": "2025-10-16 08:50:26.683000000", - "trace_duration_ms": 5038.112, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:21.645000000", - "updated_at": "2025-10-16 08:50:21.645000000" - }, - { - "span_id": "c0af6629-48b1-45fc-9dd2-60f098ea140e", - "trace_id": "fa5bb641-f4b7-4741-b06c-392a5f2a9aea", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-16 08:50:21.645000000", - "end_time": "2025-10-16 08:50:26.683000000", - "duration_ms": 5038.0419999999995, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What are the current trends in observability tools?", - "query.length": 51, - "processing.mode": "RAG", - "processing.time_seconds": 5.0380027294158936, - "processing.response_length": 897, - "processing.sources_count": 5, - "output_response": "Final Answer: The provided context does not specifically outline current trends in observability tools beyond the focus on AI applications. However, it highlights the limitations of traditional monitoring tools for...", - "final_answer_mode": "RAG", - "query_processed.input_query": "What are the current trends in observability tools?", - "query_processed.output_response": "Successfully processed query using RAG, generated 897 character response", - "query_processed.mode": "RAG", - "query_processed.processing_time": 5.0380027294158936, - "query_processed.response_length": 897 - }, - "events": [], - "links": [], - "trace_trace_id": "fa5bb641-f4b7-4741-b06c-392a5f2a9aea", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:21.645000000", - "trace_end_time": "2025-10-16 08:50:26.683000000", - "trace_duration_ms": 5038.112, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:21.645000000", - "updated_at": "2025-10-16 08:50:21.645000000" - }, - { - "span_id": "ee55906d-ffe3-42fd-9899-a476d4c33ff5", - "trace_id": "fa5bb641-f4b7-4741-b06c-392a5f2a9aea", - "parent_span_id": "622e7353-4eeb-47d5-8558-4ecec28cdcfd", - "name": "agent.llm-rag", - "start_time": "2025-10-16 08:50:21.645000000", - "end_time": "2025-10-16 08:50:26.683000000", - "duration_ms": 5037.803, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "rag_agent", - "agent.operation": "llm-rag", - "agent.capabilities": [ - "document_retrieval", - "context_generation", - "response_generation" - ], - "agent.id": "noveum_rag_agent", - "input_query": "What are the current trends in observability tools?", - "query_length": 51, - "query_type": "rag_query", - "output_response": "RAG Answer: The provided context does not specifically outline current trends in observability tools beyond the focus on AI applications. However, it highlights the limitations of traditional monitoring tools for...", - "rag.context_length": 2943, - "rag.sources_count": 5, - "rag.answer_length": 897, - "rag.mode": "retrieval_augmented_generation" - }, - "events": [], - "links": [], - "trace_trace_id": "fa5bb641-f4b7-4741-b06c-392a5f2a9aea", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:21.645000000", - "trace_end_time": "2025-10-16 08:50:26.683000000", - "trace_duration_ms": 5038.112, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:21.645000000", - "updated_at": "2025-10-16 08:50:21.645000000" - }, - { - "span_id": "fe610ecb-cc8e-4466-b719-34c6e0d45d87", - "trace_id": "fa5bb641-f4b7-4741-b06c-392a5f2a9aea", - "parent_span_id": "622e7353-4eeb-47d5-8558-4ecec28cdcfd", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-16 08:50:21.645000000", - "end_time": "2025-10-16 08:50:21.645000000", - "duration_ms": 0.032, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What are the current trends in observability tools?", - "query_length": 51, - "query_type": "routing_evaluation_query", - "classification.mode": "RAG", - "classification.rag_keyword_score": 2, - "classification.web_keyword_score": 1, - "classification.confidence_score": 0.3333333333333333, - "classification.confidence_level": "medium", - "classification.method": "keyword_based", - "query.complexity": "complex", - "query.intent": "documentation", - "query.keyword_density": 0.375, - "query.domain_affinity": "noveum", - "routing.decision": "Routed to RAG based on analysis", - "routing.rationale": "RAG score: 2, Web score: 1, Confidence: 0.33", - "routing.expected_performance": "medium", - "routing.alternative_mode": "Web Search", - "routing.decision_confidence": 0.3333333333333333, - "evaluation.routing_accuracy": "medium", - "evaluation.keyword_coverage": 0.057692307692307696, - "evaluation.query_understanding": "ambiguous", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: RAG (Confidence: 0.33)" - }, - "events": [], - "links": [], - "trace_trace_id": "fa5bb641-f4b7-4741-b06c-392a5f2a9aea", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:21.645000000", - "trace_end_time": "2025-10-16 08:50:26.683000000", - "trace_duration_ms": 5038.112, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:21.645000000", - "updated_at": "2025-10-16 08:50:21.645000000" - }, - { - "span_id": "69a4773d-c738-489e-87ed-999583e02187", - "trace_id": "fa5bb641-f4b7-4741-b06c-392a5f2a9aea", - "parent_span_id": "ee55906d-ffe3-42fd-9899-a476d4c33ff5", - "name": "agent.llm_model_execution", - "start_time": "2025-10-16 08:50:22.600000000", - "end_time": "2025-10-16 08:50:25.786000000", - "duration_ms": 3185.257, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What are the current trends in observability tools?", - "query_length": 51, - "query_type": "rag_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 3.185206174850464, - "model.latency_ms": 3185.206174850464, - "model.start_time": 1760604622.6008554, - "model.end_time": 1760604625.7860615, - "model.performance_tier": "medium", - "model.prompt_tokens": 872, - "model.completion_tokens": 224, - "model.total_tokens": 1096, - "model.tokens_per_second": 344.09075577390337, - "model.estimated_cost": 0.03288, - "model.efficiency_score": 0.8184306569343066, - "model.response_length": 897, - "model.response_quality": "high", - "model.output_response": "Model Response: The provided context does not specifically outline current trends in observability tools beyond the focus on AI applications. However, it highlights the limitations of traditional monitoring tools for...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "fa5bb641-f4b7-4741-b06c-392a5f2a9aea", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:21.645000000", - "trace_end_time": "2025-10-16 08:50:26.683000000", - "trace_duration_ms": 5038.112, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:21.645000000", - "updated_at": "2025-10-16 08:50:21.645000000" - }, - { - "span_id": "5e5a9d30-4cd0-45cd-9494-192c280727ac", - "trace_id": "fa5bb641-f4b7-4741-b06c-392a5f2a9aea", - "parent_span_id": "ee55906d-ffe3-42fd-9899-a476d4c33ff5", - "name": "agent.rag_evaluation_metrics", - "start_time": "2025-10-16 08:50:25.786000000", - "end_time": "2025-10-16 08:50:25.786000000", - "duration_ms": 0.066, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "rag_evaluation_metrics", - "agent.capabilities": [ - "retrieval_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What are the current trends in observability tools?", - "query_length": 51, - "query_type": "rag_evaluation_query", - "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/blog/from-logs-to-intelligent-choices-inside-noveum-ais-evaluation-process):\nFrom Development to Production - Inside Noveum.ai's AI Observability Platform\nShashank Agarwal\n3/3/2025\nIntroduction\nEvery AI application tells a story\u2014but most developers never get to hear it...", - "retrieval.context_length": 2943, - "retrieval.sources_count": 5, - "retrieval.context_quality": "high", - "retrieval.effectiveness": 1, - "retrieval.context_utilization": 2.943, - "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/blog/from-logs-to-intelligent-choices-inside-noveum-ais-evaluation-process):\nFrom Development to Production - Inside Noveum.ai's AI Observability Platform\nShashank Agarwal\n3/3/2025\nIntroduction\nEvery AI application tells a story\u2014but most developers never get to hear it. When your RAG pipeline returns irrelevant results, when your multi-agent system gets stuck in loops, or when your LLM costs suddenly spike, you're left guessing what went wrong and where.\nThat's exactly the challenge Noveum.ai solves. Rather than flying blind, our platform provides comprehensive tracing a...\n\nSource 2 (https://noveum.ai/docs/getting-started/tracing-concepts):\n- Explore Framework Integrations - Framework-specific guidance\n- Learn Advanced Patterns - Custom instrumentation techniques\n- Master the Dashboard - Analyze your traces effectively\nRemember: Good observability is not about collecting all possible data, but about collecting the right data that helps you understand, debug, and optimize your AI applications.\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to O...\n\nSource 3 (https://noveum.ai/en/docs/getting-started/tracing-concepts):\n- Explore Framework Integrations - Framework-specific guidance\n- Learn Advanced Patterns - Custom instrumentation techniques\n- Master the Dashboard - Analyze your traces effectively\nRemember: Good observability is not about collecting all possible data, but about collecting the right data that helps you understand, debug, and optimize your AI applications.\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to O...\n\nSource 4 (https://noveum.ai/en/blog/from-logs-to-intelligent-choices-inside-noveum-ais-evaluation-process):\nThe AI Observability Challenge\nWhy Traditional Monitoring Falls Short\nTraditional application monitoring tools weren't built for AI applications. They can tell you if your API is responding, but they can't answer the questions that matter most for AI systems:\n- Why did my RAG pipeline retrieve irrelevant documents?\n- Which LLM calls are driving my costs?\n- How are my agents communicating with each other?\n- What's causing hallucinations in my responses?\n- Why is my embedding generation so slow?\nW...\n\nSource 5 (https://noveum.ai/en/docs/advanced/rag-observability):\n\ud83d\udd27 Best Practices for RAG Observability\n1. Comprehensive Pipeline Tracing\n2. Quality Monitoring\n3. A/B Testing for RAG Components\n\ud83c\udfaf Advanced RAG Patterns\nMulti-Modal RAG\nConversational RAG\nRAG observability with Noveum.ai provides the deep insights needed to build, optimize, and scale retrieval-augmented generation systems. By implementing comprehensive tracing across retrieval, context preparation, and generation stages, you can ensure your RAG pipeline delivers accurate, relevant, and cost-effe...\n\nUser Question: What are the current trends in observability tools?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", - "prompt.prompt_length": 3490, - "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/blog/from-logs-to-intelligent-choices-inside-noveum-ais-evaluation-process):\nFrom Development to Production - Inside Noveum.ai's AI Observability Platform\nShashank Agarw...", - "prompt.instruction_following": "rag_optimized", - "response.answer_length": 897, - "response.answer_completeness": "complete", - "response.response_quality": "high", - "response.source_citation": 5, - "response.context_utilization": 0.30479102956167176, - "output_response": "RAG Answer: The provided context does not specifically outline current trends in observability tools beyond the focus on AI applications. However, it highlights the limitations of traditional monitoring tools for...", - "evaluation.retrieval_effectiveness": 1, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 5, - "evaluation.context_relevance": "high", - "evaluation.overall_quality": "high", - "evaluation.ready_for_production": true, - "rag.retrieval_strategy": "semantic_similarity", - "rag.vector_search_results": 5, - "rag.context_synthesis": "multi_source", - "rag.document_coverage": 1, - "rag.information_density": 0.30479102956167176 - }, - "events": [], - "links": [], - "trace_trace_id": "fa5bb641-f4b7-4741-b06c-392a5f2a9aea", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:21.645000000", - "trace_end_time": "2025-10-16 08:50:26.683000000", - "trace_duration_ms": 5038.112, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:21.645000000", - "updated_at": "2025-10-16 08:50:21.645000000" - }, - { - "span_id": "2e06953f-5cec-4873-92b5-91f7281f6d7c", - "trace_id": "c0a95e8e-513c-47eb-b746-4e6016aeaf7d", - "parent_span_id": "9e41eab5-cd8e-4e46-a15b-e5f52c9a4073", - "name": "agent.query_routing", - "start_time": "2025-10-16 08:50:08.944000000", - "end_time": "2025-10-16 08:50:20.641000000", - "duration_ms": 11696.524, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "Tell me about recent developments in machine learning", - "query_length": 53, - "query_type": "routing_query", - "output_response": "Routed to Web Search for query processing", - "router.classification": "Web Search", - "router.confidence_score": 1, - "router.rag_keyword_score": 0, - "router.web_keyword_score": 1, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "c0a95e8e-513c-47eb-b746-4e6016aeaf7d", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:08.944000000", - "trace_end_time": "2025-10-16 08:50:20.641000000", - "trace_duration_ms": 11696.768, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:08.944000000", - "updated_at": "2025-10-16 08:50:08.944000000" - }, - { - "span_id": "9e41eab5-cd8e-4e46-a15b-e5f52c9a4073", - "trace_id": "c0a95e8e-513c-47eb-b746-4e6016aeaf7d", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-16 08:50:08.944000000", - "end_time": "2025-10-16 08:50:20.641000000", - "duration_ms": 11696.671999999999, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "Tell me about recent developments in machine learning", - "query.length": 53, - "processing.mode": "Web Search", - "processing.time_seconds": 11.696574449539185, - "processing.response_length": 2001, - "processing.sources_count": 1, - "output_response": "Final Answer: The web search did not yield specific results regarding recent developments in machine learning. However, I can provide a general overview based on my knowledge up to October 2023.\n\nRecent development...", - "final_answer_mode": "Web Search", - "query_processed.input_query": "Tell me about recent developments in machine learning", - "query_processed.output_response": "Successfully processed query using Web Search, generated 2001 character response", - "query_processed.mode": "Web Search", - "query_processed.processing_time": 11.696574449539185, - "query_processed.response_length": 2001 - }, - "events": [], - "links": [], - "trace_trace_id": "c0a95e8e-513c-47eb-b746-4e6016aeaf7d", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:08.944000000", - "trace_end_time": "2025-10-16 08:50:20.641000000", - "trace_duration_ms": 11696.768, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:08.944000000", - "updated_at": "2025-10-16 08:50:08.944000000" - }, - { - "span_id": "7ddf938b-183a-427f-849b-b31b18a98d87", - "trace_id": "c0a95e8e-513c-47eb-b746-4e6016aeaf7d", - "parent_span_id": "2e06953f-5cec-4873-92b5-91f7281f6d7c", - "name": "agent.web_search_generation", - "start_time": "2025-10-16 08:50:08.945000000", - "end_time": "2025-10-16 08:50:20.641000000", - "duration_ms": 11696.328, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "web_search_agent", - "agent.operation": "web_search_generation", - "agent.capabilities": [ - "web_search", - "content_synthesis", - "response_generation" - ], - "agent.id": "noveum_web_search_agent", - "input_query": "Tell me about recent developments in machine learning", - "query_length": 53, - "query_type": "web_search_query", - "output_response": "Web Search Answer: The web search did not yield specific results regarding recent developments in machine learning. However, I can provide a general overview based on my knowledge up to October 2023.\n\nRecent development...", - "web_search.results_count": 1, - "web_search.context_length": 155, - "web_search.response_length": 2001, - "web_search.mode": "external_web_search" - }, - "events": [], - "links": [], - "trace_trace_id": "c0a95e8e-513c-47eb-b746-4e6016aeaf7d", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:08.944000000", - "trace_end_time": "2025-10-16 08:50:20.641000000", - "trace_duration_ms": 11696.768, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:08.944000000", - "updated_at": "2025-10-16 08:50:08.944000000" - }, - { - "span_id": "f3e4c594-f822-4f04-ab43-98fac161b9a3", - "trace_id": "c0a95e8e-513c-47eb-b746-4e6016aeaf7d", - "parent_span_id": "2e06953f-5cec-4873-92b5-91f7281f6d7c", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-16 08:50:08.945000000", - "end_time": "2025-10-16 08:50:08.945000000", - "duration_ms": 0.046, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: Tell me about recent developments in machine learning", - "query_length": 53, - "query_type": "routing_evaluation_query", - "classification.mode": "Web Search", - "classification.rag_keyword_score": 0, - "classification.web_keyword_score": 1, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "complex", - "query.intent": "general_knowledge", - "query.keyword_density": 0.125, - "query.domain_affinity": "general", - "routing.decision": "Routed to Web Search based on analysis", - "routing.rationale": "RAG score: 0, Web score: 1, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "RAG", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.019230769230769232, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: Web Search (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "c0a95e8e-513c-47eb-b746-4e6016aeaf7d", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:08.944000000", - "trace_end_time": "2025-10-16 08:50:20.641000000", - "trace_duration_ms": 11696.768, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:08.944000000", - "updated_at": "2025-10-16 08:50:08.944000000" - }, - { - "span_id": "b47d09be-a43b-4b0a-b7d3-d98bdf1773a3", - "trace_id": "c0a95e8e-513c-47eb-b746-4e6016aeaf7d", - "parent_span_id": "7ddf938b-183a-427f-849b-b31b18a98d87", - "name": "agent.llm_model_execution", - "start_time": "2025-10-16 08:50:09.490000000", - "end_time": "2025-10-16 08:50:20.641000000", - "duration_ms": 11151.141000000001, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: Tell me about recent developments in machine learning", - "query_length": 53, - "query_type": "web_search_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 11.151098012924194, - "model.latency_ms": 11151.098012924194, - "model.start_time": 1760604609.4902723, - "model.end_time": 1760604620.6413703, - "model.performance_tier": "slow", - "model.prompt_tokens": 163, - "model.completion_tokens": 500, - "model.total_tokens": 663, - "model.tokens_per_second": 59.456028386763236, - "model.estimated_cost": 0.01989, - "model.efficiency_score": 3.018099547511312, - "model.response_length": 2001, - "model.response_quality": "high", - "model.output_response": "Model Response: The web search did not yield specific results regarding recent developments in machine learning. However, I can provide a general overview based on my knowledge up to October 2023.\n\nRecent development...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "c0a95e8e-513c-47eb-b746-4e6016aeaf7d", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:08.944000000", - "trace_end_time": "2025-10-16 08:50:20.641000000", - "trace_duration_ms": 11696.768, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:08.944000000", - "updated_at": "2025-10-16 08:50:08.944000000" - }, - { - "span_id": "bfea8500-0e5c-4556-a014-5b2ec49b7ad5", - "trace_id": "c0a95e8e-513c-47eb-b746-4e6016aeaf7d", - "parent_span_id": "7ddf938b-183a-427f-849b-b31b18a98d87", - "name": "agent.web_search_evaluation_metrics", - "start_time": "2025-10-16 08:50:20.641000000", - "end_time": "2025-10-16 08:50:20.641000000", - "duration_ms": 0.018000000000000002, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "web_search_evaluation_metrics", - "agent.capabilities": [ - "web_search_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: Tell me about recent developments in machine learning", - "query_length": 53, - "query_type": "web_search_evaluation_query", - "web_search.results_count": 1, - "web_search.context_length": 155, - "web_search.context_synthesized": "Context from 1 web sources", - "web_search.search_effectiveness": 0.2, - "web_search.context_quality": "low", - "web_search.source_diversity": 1, - "web_search.information_synthesis": "low", - "web_search.external_knowledge_utilization": 0.155, - "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=Tell+me+about+recent+developments+in+machine+learning\n\nUser Question: Tell me about recent developments in machine learning\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", - "prompt.prompt_length": 653, - "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=Tell+me+about+recent+developments+in+machine+learning", - "prompt.instruction_following": "web_search_optimized", - "response.answer_length": 2001, - "response.answer_completeness": "complete", - "response.response_quality": "medium", - "response.source_citation": 1, - "response.context_utilization": 12.90967741935484, - "output_response": "Web Search Answer: The web search did not yield specific results regarding recent developments in machine learning. However, I can provide a general overview based on my knowledge up to October 2023.\n\nRecent development...", - "evaluation.search_effectiveness": 0.2, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 1, - "evaluation.context_relevance": "low", - "evaluation.overall_quality": "low", - "evaluation.ready_for_production": true, - "web_search.search_strategy": "duckduckgo_api", - "web_search.real_time_data": true, - "web_search.external_sources": 1, - "web_search.information_freshness": "current", - "web_search.knowledge_synthesis": "single_source" - }, - "events": [], - "links": [], - "trace_trace_id": "c0a95e8e-513c-47eb-b746-4e6016aeaf7d", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:08.944000000", - "trace_end_time": "2025-10-16 08:50:20.641000000", - "trace_duration_ms": 11696.768, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:08.944000000", - "updated_at": "2025-10-16 08:50:08.944000000" - }, - { - "span_id": "33297286-d440-446a-ba73-7cabd9d4b2e6", - "trace_id": "3af8c6e8-62d9-4893-92ba-cc265934ef03", - "parent_span_id": "4a3c230e-0dd5-4ba2-8e87-2a301f654fba", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-16 08:50:04.544000000", - "end_time": "2025-10-16 08:50:04.544000000", - "duration_ms": 0.038, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What's the weather like today?", - "query_length": 30, - "query_type": "routing_evaluation_query", - "classification.mode": "Web Search", - "classification.rag_keyword_score": 0, - "classification.web_keyword_score": 2, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "general_knowledge", - "query.keyword_density": 0.4, - "query.domain_affinity": "general", - "routing.decision": "Routed to Web Search based on analysis", - "routing.rationale": "RAG score: 0, Web score: 2, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "RAG", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.038461538461538464, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: Web Search (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "3af8c6e8-62d9-4893-92ba-cc265934ef03", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:04.544000000", - "trace_end_time": "2025-10-16 08:50:07.941000000", - "trace_duration_ms": 3397.228, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:04.544000000", - "updated_at": "2025-10-16 08:50:04.544000000" - }, - { - "span_id": "4a3c230e-0dd5-4ba2-8e87-2a301f654fba", - "trace_id": "3af8c6e8-62d9-4893-92ba-cc265934ef03", - "parent_span_id": "eefcbdc5-99fc-4f06-a9f4-f51ba50564db", - "name": "agent.query_routing", - "start_time": "2025-10-16 08:50:04.544000000", - "end_time": "2025-10-16 08:50:07.941000000", - "duration_ms": 3397.056, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What's the weather like today?", - "query_length": 30, - "query_type": "routing_query", - "output_response": "Routed to Web Search for query processing", - "router.classification": "Web Search", - "router.confidence_score": 1, - "router.rag_keyword_score": 0, - "router.web_keyword_score": 2, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "3af8c6e8-62d9-4893-92ba-cc265934ef03", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:04.544000000", - "trace_end_time": "2025-10-16 08:50:07.941000000", - "trace_duration_ms": 3397.228, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:04.544000000", - "updated_at": "2025-10-16 08:50:04.544000000" - }, - { - "span_id": "eefcbdc5-99fc-4f06-a9f4-f51ba50564db", - "trace_id": "3af8c6e8-62d9-4893-92ba-cc265934ef03", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-16 08:50:04.544000000", - "end_time": "2025-10-16 08:50:07.941000000", - "duration_ms": 3397.169, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What's the weather like today?", - "query.length": 30, - "processing.mode": "Web Search", - "processing.time_seconds": 3.3970956802368164, - "processing.response_length": 518, - "processing.sources_count": 1, - "output_response": "Final Answer: The web search results do not provide specific details about today's weather conditions in your location. However, they mention that you can get real-time updates, including temperature, humidity, and...", - "final_answer_mode": "Web Search", - "query_processed.input_query": "What's the weather like today?", - "query_processed.output_response": "Successfully processed query using Web Search, generated 518 character response", - "query_processed.mode": "Web Search", - "query_processed.processing_time": 3.3970956802368164, - "query_processed.response_length": 518 - }, - "events": [], - "links": [], - "trace_trace_id": "3af8c6e8-62d9-4893-92ba-cc265934ef03", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:04.544000000", - "trace_end_time": "2025-10-16 08:50:07.941000000", - "trace_duration_ms": 3397.228, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:04.544000000", - "updated_at": "2025-10-16 08:50:04.544000000" - }, - { - "span_id": "9683465c-e851-4004-a858-c8fdd1a2b3da", - "trace_id": "3af8c6e8-62d9-4893-92ba-cc265934ef03", - "parent_span_id": "4a3c230e-0dd5-4ba2-8e87-2a301f654fba", - "name": "agent.web_search_generation", - "start_time": "2025-10-16 08:50:04.545000000", - "end_time": "2025-10-16 08:50:07.941000000", - "duration_ms": 3396.871, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "web_search_agent", - "agent.operation": "web_search_generation", - "agent.capabilities": [ - "web_search", - "content_synthesis", - "response_generation" - ], - "agent.id": "noveum_web_search_agent", - "input_query": "What's the weather like today?", - "query_length": 30, - "query_type": "web_search_query", - "output_response": "Web Search Answer: The web search results do not provide specific details about today's weather conditions in your location. However, they mention that you can get real-time updates, including temperature, humidity, and...", - "web_search.results_count": 1, - "web_search.context_length": 1137, - "web_search.response_length": 518, - "web_search.mode": "external_web_search" - }, - "events": [], - "links": [], - "trace_trace_id": "3af8c6e8-62d9-4893-92ba-cc265934ef03", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:04.544000000", - "trace_end_time": "2025-10-16 08:50:07.941000000", - "trace_duration_ms": 3397.228, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:04.544000000", - "updated_at": "2025-10-16 08:50:04.544000000" - }, - { - "span_id": "6413eb7e-e5cd-4c46-9651-de14c72c260e", - "trace_id": "3af8c6e8-62d9-4893-92ba-cc265934ef03", - "parent_span_id": "9683465c-e851-4004-a858-c8fdd1a2b3da", - "name": "agent.llm_model_execution", - "start_time": "2025-10-16 08:50:04.986000000", - "end_time": "2025-10-16 08:50:07.941000000", - "duration_ms": 2955.55, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What's the weather like today?", - "query_length": 30, - "query_type": "web_search_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 2.955509662628174, - "model.latency_ms": 2955.509662628174, - "model.start_time": 1760604604.9862766, - "model.end_time": 1760604607.9417863, - "model.performance_tier": "medium", - "model.prompt_tokens": 403, - "model.completion_tokens": 129, - "model.total_tokens": 532, - "model.tokens_per_second": 180.0027950261957, - "model.estimated_cost": 0.015960000000000002, - "model.efficiency_score": 0.9736842105263158, - "model.response_length": 518, - "model.response_quality": "high", - "model.output_response": "Model Response: The web search results do not provide specific details about today's weather conditions in your location. However, they mention that you can get real-time updates, including temperature, humidity, and...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "3af8c6e8-62d9-4893-92ba-cc265934ef03", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:04.544000000", - "trace_end_time": "2025-10-16 08:50:07.941000000", - "trace_duration_ms": 3397.228, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:04.544000000", - "updated_at": "2025-10-16 08:50:04.544000000" - }, - { - "span_id": "64df69fb-0bcc-4845-8bfe-5a721c1ed912", - "trace_id": "3af8c6e8-62d9-4893-92ba-cc265934ef03", - "parent_span_id": "9683465c-e851-4004-a858-c8fdd1a2b3da", - "name": "agent.web_search_evaluation_metrics", - "start_time": "2025-10-16 08:50:07.941000000", - "end_time": "2025-10-16 08:50:07.941000000", - "duration_ms": 0.017, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "web_search_evaluation_metrics", - "agent.capabilities": [ - "web_search_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What's the weather like today?", - "query_length": 30, - "query_type": "web_search_evaluation_query", - "web_search.results_count": 1, - "web_search.context_length": 1137, - "web_search.context_synthesized": "Context from 1 web sources", - "web_search.search_effectiveness": 0.2, - "web_search.context_quality": "high", - "web_search.source_diversity": 1, - "web_search.information_synthesis": "low", - "web_search.external_knowledge_utilization": 1.137, - "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nStay updated with the latest weather conditions in Komna. Get real-time updates, temperature, humidity, AQI, and detailed forecast for today. Dec 31, 2024 \u00b7 Komna Weather Forecast: Check the latest temperature in Komna today along with details such as humidity, visibility, air quality, wind speed, pressure, and weekly weather report at Shuru. 2 days ago \u00b7 Latest weather forecast for India for today's, hourly weather forecast, including today's temperatures in India, wind, rain and more. 2 days ago \u00b7 Get the latest hourly weather updates for India today. Detailed forecast including temperature, wind, rain, snow, and UV index. Stay informed about today's weather conditions in India. 2 days ago \u00b7 Today's Weather Report ... Forecasts Urban Meteorological Services Short to Medium Range Model Guidance Extended Range Model Guidance Seasonal Forecast Quantitative Precipitation Forecast Latest CAP Alerts Specialised products & forecasts Cyclones | Monsoon | Satellite Radar | Rainfall | Agromet Advisories MAUSAM : | | Video Meghdoot Agro ...\nURL: https://duckduckgo.com/?q=What's+the+weather+like+today?\n\nUser Question: What's the weather like today?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", - "prompt.prompt_length": 1612, - "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nStay updated with the latest weather conditions in Komna. Get real-time updates, temperature, humidity, AQI, and detailed forecast for today. Dec 31, 2024 \u00b7 Komna Weather F...", - "prompt.instruction_following": "web_search_optimized", - "response.answer_length": 518, - "response.answer_completeness": "complete", - "response.response_quality": "medium", - "response.source_citation": 1, - "response.context_utilization": 0.455584872471416, - "output_response": "Web Search Answer: The web search results do not provide specific details about today's weather conditions in your location. However, they mention that you can get real-time updates, including temperature, humidity, and...", - "evaluation.search_effectiveness": 0.2, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 1, - "evaluation.context_relevance": "high", - "evaluation.overall_quality": "low", - "evaluation.ready_for_production": true, - "web_search.search_strategy": "duckduckgo_api", - "web_search.real_time_data": true, - "web_search.external_sources": 1, - "web_search.information_freshness": "current", - "web_search.knowledge_synthesis": "single_source" - }, - "events": [], - "links": [], - "trace_trace_id": "3af8c6e8-62d9-4893-92ba-cc265934ef03", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:04.544000000", - "trace_end_time": "2025-10-16 08:50:07.941000000", - "trace_duration_ms": 3397.228, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:04.544000000", - "updated_at": "2025-10-16 08:50:04.544000000" - }, - { - "span_id": "719a1a8f-fc50-439c-899b-436146accd23", - "trace_id": "d7b8b811-ad01-48f1-917f-ace7e8d289e6", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-16 08:50:01.375000000", - "end_time": "2025-10-16 08:50:03.540000000", - "duration_ms": 2164.4809999999998, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What are the latest AI news today?", - "query.length": 34, - "processing.mode": "Web Search", - "processing.time_seconds": 2.1643810272216797, - "processing.response_length": 298, - "processing.sources_count": 1, - "output_response": "Final Answer: The web search did not yield any specific results regarding the latest AI news today. Unfortunately, I cannot provide current updates or news articles on AI developments at this moment. For the latest...", - "final_answer_mode": "Web Search", - "query_processed.input_query": "What are the latest AI news today?", - "query_processed.output_response": "Successfully processed query using Web Search, generated 298 character response", - "query_processed.mode": "Web Search", - "query_processed.processing_time": 2.1643810272216797, - "query_processed.response_length": 298 - }, - "events": [], - "links": [], - "trace_trace_id": "d7b8b811-ad01-48f1-917f-ace7e8d289e6", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:01.375000000", - "trace_end_time": "2025-10-16 08:50:03.540000000", - "trace_duration_ms": 2164.576, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:01.375000000", - "updated_at": "2025-10-16 08:50:01.375000000" - }, - { - "span_id": "644a712f-1c9e-4e9a-9073-74621b754aa2", - "trace_id": "d7b8b811-ad01-48f1-917f-ace7e8d289e6", - "parent_span_id": "719a1a8f-fc50-439c-899b-436146accd23", - "name": "agent.query_routing", - "start_time": "2025-10-16 08:50:01.376000000", - "end_time": "2025-10-16 08:50:03.540000000", - "duration_ms": 2164.3219999999997, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What are the latest AI news today?", - "query_length": 34, - "query_type": "routing_query", - "output_response": "Routed to Web Search for query processing", - "router.classification": "Web Search", - "router.confidence_score": 1, - "router.rag_keyword_score": 0, - "router.web_keyword_score": 3, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "d7b8b811-ad01-48f1-917f-ace7e8d289e6", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:01.375000000", - "trace_end_time": "2025-10-16 08:50:03.540000000", - "trace_duration_ms": 2164.576, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:01.375000000", - "updated_at": "2025-10-16 08:50:01.375000000" - }, - { - "span_id": "8a4abdbe-6ad7-4d43-b234-81667c4abbc2", - "trace_id": "d7b8b811-ad01-48f1-917f-ace7e8d289e6", - "parent_span_id": "644a712f-1c9e-4e9a-9073-74621b754aa2", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-16 08:50:01.376000000", - "end_time": "2025-10-16 08:50:01.376000000", - "duration_ms": 0.051, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What are the latest AI news today?", - "query_length": 34, - "query_type": "routing_evaluation_query", - "classification.mode": "Web Search", - "classification.rag_keyword_score": 0, - "classification.web_keyword_score": 3, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "general_knowledge", - "query.keyword_density": 0.42857142857142855, - "query.domain_affinity": "general", - "routing.decision": "Routed to Web Search based on analysis", - "routing.rationale": "RAG score: 0, Web score: 3, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "RAG", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.057692307692307696, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: Web Search (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "d7b8b811-ad01-48f1-917f-ace7e8d289e6", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:01.375000000", - "trace_end_time": "2025-10-16 08:50:03.540000000", - "trace_duration_ms": 2164.576, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:01.375000000", - "updated_at": "2025-10-16 08:50:01.375000000" - }, - { - "span_id": "904621e8-c5fa-4b2d-a628-8f10eb9da51d", - "trace_id": "d7b8b811-ad01-48f1-917f-ace7e8d289e6", - "parent_span_id": "644a712f-1c9e-4e9a-9073-74621b754aa2", - "name": "agent.web_search_generation", - "start_time": "2025-10-16 08:50:01.376000000", - "end_time": "2025-10-16 08:50:03.540000000", - "duration_ms": 2164.076, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "web_search_agent", - "agent.operation": "web_search_generation", - "agent.capabilities": [ - "web_search", - "content_synthesis", - "response_generation" - ], - "agent.id": "noveum_web_search_agent", - "input_query": "What are the latest AI news today?", - "query_length": 34, - "query_type": "web_search_query", - "output_response": "Web Search Answer: The web search did not yield any specific results regarding the latest AI news today. Unfortunately, I cannot provide current updates or news articles on AI developments at this moment. For the latest...", - "web_search.results_count": 1, - "web_search.context_length": 136, - "web_search.response_length": 298, - "web_search.mode": "external_web_search" - }, - "events": [], - "links": [], - "trace_trace_id": "d7b8b811-ad01-48f1-917f-ace7e8d289e6", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:01.375000000", - "trace_end_time": "2025-10-16 08:50:03.540000000", - "trace_duration_ms": 2164.576, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:01.375000000", - "updated_at": "2025-10-16 08:50:01.375000000" - }, - { - "span_id": "421dd1af-2dbc-477f-bd59-b3194b14ae3a", - "trace_id": "d7b8b811-ad01-48f1-917f-ace7e8d289e6", - "parent_span_id": "904621e8-c5fa-4b2d-a628-8f10eb9da51d", - "name": "agent.llm_model_execution", - "start_time": "2025-10-16 08:50:01.708000000", - "end_time": "2025-10-16 08:50:03.540000000", - "duration_ms": 1831.237, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What are the latest AI news today?", - "query_length": 34, - "query_type": "web_search_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 1.8311803340911865, - "model.latency_ms": 1831.1803340911865, - "model.start_time": 1760604601.7090185, - "model.end_time": 1760604603.5401988, - "model.performance_tier": "fast", - "model.prompt_tokens": 153, - "model.completion_tokens": 74, - "model.total_tokens": 227, - "model.tokens_per_second": 123.96376029926071, - "model.estimated_cost": 0.00681, - "model.efficiency_score": 1.3127753303964758, - "model.response_length": 298, - "model.response_quality": "high", - "model.output_response": "Model Response: The web search did not yield any specific results regarding the latest AI news today. Unfortunately, I cannot provide current updates or news articles on AI developments at this moment. For the latest...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "d7b8b811-ad01-48f1-917f-ace7e8d289e6", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:01.375000000", - "trace_end_time": "2025-10-16 08:50:03.540000000", - "trace_duration_ms": 2164.576, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:01.375000000", - "updated_at": "2025-10-16 08:50:01.375000000" - }, - { - "span_id": "159aad6f-7174-406d-b7d9-eed9360d993c", - "trace_id": "d7b8b811-ad01-48f1-917f-ace7e8d289e6", - "parent_span_id": "904621e8-c5fa-4b2d-a628-8f10eb9da51d", - "name": "agent.web_search_evaluation_metrics", - "start_time": "2025-10-16 08:50:03.540000000", - "end_time": "2025-10-16 08:50:03.540000000", - "duration_ms": 0.028, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "web_search_evaluation_metrics", - "agent.capabilities": [ - "web_search_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What are the latest AI news today?", - "query_length": 34, - "query_type": "web_search_evaluation_query", - "web_search.results_count": 1, - "web_search.context_length": 136, - "web_search.context_synthesized": "Context from 1 web sources", - "web_search.search_effectiveness": 0.2, - "web_search.context_quality": "low", - "web_search.source_diversity": 1, - "web_search.information_synthesis": "low", - "web_search.external_knowledge_utilization": 0.136, - "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What+are+the+latest+AI+news+today?\n\nUser Question: What are the latest AI news today?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", - "prompt.prompt_length": 615, - "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What+are+the+latest+AI+news+today?", - "prompt.instruction_following": "web_search_optimized", - "response.answer_length": 298, - "response.answer_completeness": "complete", - "response.response_quality": "medium", - "response.source_citation": 1, - "response.context_utilization": 2.1911764705882355, - "output_response": "Web Search Answer: The web search did not yield any specific results regarding the latest AI news today. Unfortunately, I cannot provide current updates or news articles on AI developments at this moment. For the latest...", - "evaluation.search_effectiveness": 0.2, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 1, - "evaluation.context_relevance": "low", - "evaluation.overall_quality": "low", - "evaluation.ready_for_production": true, - "web_search.search_strategy": "duckduckgo_api", - "web_search.real_time_data": true, - "web_search.external_sources": 1, - "web_search.information_freshness": "current", - "web_search.knowledge_synthesis": "single_source" - }, - "events": [], - "links": [], - "trace_trace_id": "d7b8b811-ad01-48f1-917f-ace7e8d289e6", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:01.375000000", - "trace_end_time": "2025-10-16 08:50:03.540000000", - "trace_duration_ms": 2164.576, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:01.375000000", - "updated_at": "2025-10-16 08:50:01.375000000" - }, - { - "span_id": "dcf2b38e-ab90-45ee-9a65-e6fea47a3fd7", - "trace_id": "df8340b4-b75a-48e9-a2f3-99725b46a00d", - "parent_span_id": "d0e2b320-c18d-4ec6-98a1-74da5252107e", - "name": "agent.query_routing", - "start_time": "2025-10-16 08:49:17.236000000", - "end_time": "2025-10-16 08:50:00.368000000", - "duration_ms": 43131.384, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What are the benefits of using Noveum Trace?", - "query_length": 44, - "query_type": "routing_query", - "output_response": "Routed to RAG for query processing", - "router.classification": "RAG", - "router.confidence_score": 1, - "router.rag_keyword_score": 2, - "router.web_keyword_score": 0, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "df8340b4-b75a-48e9-a2f3-99725b46a00d", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:49:17.236000000", - "trace_end_time": "2025-10-16 08:50:00.368000000", - "trace_duration_ms": 43131.554000000004, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:49:17.236000000", - "updated_at": "2025-10-16 08:49:17.236000000" - }, - { - "span_id": "d0e2b320-c18d-4ec6-98a1-74da5252107e", - "trace_id": "df8340b4-b75a-48e9-a2f3-99725b46a00d", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-16 08:49:17.236000000", - "end_time": "2025-10-16 08:50:00.368000000", - "duration_ms": 43131.481, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What are the benefits of using Noveum Trace?", - "query.length": 44, - "processing.mode": "RAG", - "processing.time_seconds": 43.131428718566895, - "processing.response_length": 1415, - "processing.sources_count": 5, - "output_response": "Final Answer: The benefits of using Noveum Trace include:\n\n1. **Comprehensive Tracing**: Noveum Trace offers decorator-based tracing for LLM calls, agents, and RAG pipelines, allowing for detailed monitoring of AI ...", - "final_answer_mode": "RAG", - "query_processed.input_query": "What are the benefits of using Noveum Trace?", - "query_processed.output_response": "Successfully processed query using RAG, generated 1415 character response", - "query_processed.mode": "RAG", - "query_processed.processing_time": 43.131428718566895, - "query_processed.response_length": 1415 - }, - "events": [], - "links": [], - "trace_trace_id": "df8340b4-b75a-48e9-a2f3-99725b46a00d", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:49:17.236000000", - "trace_end_time": "2025-10-16 08:50:00.368000000", - "trace_duration_ms": 43131.554000000004, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:49:17.236000000", - "updated_at": "2025-10-16 08:49:17.236000000" - }, - { - "span_id": "fa93821f-c809-4071-87e1-a6c55014d37c", - "trace_id": "df8340b4-b75a-48e9-a2f3-99725b46a00d", - "parent_span_id": "dcf2b38e-ab90-45ee-9a65-e6fea47a3fd7", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-16 08:49:17.236000000", - "end_time": "2025-10-16 08:49:17.236000000", - "duration_ms": 0.034, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What are the benefits of using Noveum Trace?", - "query_length": 44, - "query_type": "routing_evaluation_query", - "classification.mode": "RAG", - "classification.rag_keyword_score": 2, - "classification.web_keyword_score": 0, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "noveum_specific", - "query.keyword_density": 0.25, - "query.domain_affinity": "noveum", - "routing.decision": "Routed to RAG based on analysis", - "routing.rationale": "RAG score: 2, Web score: 0, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "Web Search", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.038461538461538464, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: RAG (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "df8340b4-b75a-48e9-a2f3-99725b46a00d", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:49:17.236000000", - "trace_end_time": "2025-10-16 08:50:00.368000000", - "trace_duration_ms": 43131.554000000004, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:49:17.236000000", - "updated_at": "2025-10-16 08:49:17.236000000" - }, - { - "span_id": "024ad5aa-f394-4c26-af51-564f3f51c6f2", - "trace_id": "df8340b4-b75a-48e9-a2f3-99725b46a00d", - "parent_span_id": "dcf2b38e-ab90-45ee-9a65-e6fea47a3fd7", - "name": "agent.llm-rag", - "start_time": "2025-10-16 08:49:17.237000000", - "end_time": "2025-10-16 08:50:00.368000000", - "duration_ms": 43131.19300000001, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "rag_agent", - "agent.operation": "llm-rag", - "agent.capabilities": [ - "document_retrieval", - "context_generation", - "response_generation" - ], - "agent.id": "noveum_rag_agent", - "input_query": "What are the benefits of using Noveum Trace?", - "query_length": 44, - "query_type": "rag_query", - "output_response": "RAG Answer: The benefits of using Noveum Trace include:\n\n1. **Comprehensive Tracing**: Noveum Trace offers decorator-based tracing for LLM calls, agents, and RAG pipelines, allowing for detailed monitoring of AI ...", - "rag.context_length": 2773, - "rag.sources_count": 5, - "rag.answer_length": 1415, - "rag.mode": "retrieval_augmented_generation" - }, - "events": [], - "links": [], - "trace_trace_id": "df8340b4-b75a-48e9-a2f3-99725b46a00d", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:49:17.236000000", - "trace_end_time": "2025-10-16 08:50:00.368000000", - "trace_duration_ms": 43131.554000000004, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:49:17.236000000", - "updated_at": "2025-10-16 08:49:17.236000000" - }, - { - "span_id": "3d9db951-d8b0-4990-820d-61e4b1011709", - "trace_id": "df8340b4-b75a-48e9-a2f3-99725b46a00d", - "parent_span_id": "024ad5aa-f394-4c26-af51-564f3f51c6f2", - "name": "agent.llm_model_execution", - "start_time": "2025-10-16 08:49:20.224000000", - "end_time": "2025-10-16 08:49:58.420000000", - "duration_ms": 38196.111000000004, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What are the benefits of using Noveum Trace?", - "query_length": 44, - "query_type": "rag_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 38.19604015350342, - "model.latency_ms": 38196.04015350342, - "model.start_time": 1760604560.2242246, - "model.end_time": 1760604598.4202647, - "model.performance_tier": "slow", - "model.prompt_tokens": 828, - "model.completion_tokens": 353, - "model.total_tokens": 1181, - "model.tokens_per_second": 30.919435503098253, - "model.estimated_cost": 0.03543, - "model.efficiency_score": 1.1981371718882303, - "model.response_length": 1415, - "model.response_quality": "high", - "model.output_response": "Model Response: The benefits of using Noveum Trace include:\n\n1. **Comprehensive Tracing**: Noveum Trace offers decorator-based tracing for LLM calls, agents, and RAG pipelines, allowing for detailed monitoring of AI ...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "df8340b4-b75a-48e9-a2f3-99725b46a00d", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:49:17.236000000", - "trace_end_time": "2025-10-16 08:50:00.368000000", - "trace_duration_ms": 43131.554000000004, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:49:17.236000000", - "updated_at": "2025-10-16 08:49:17.236000000" - }, - { - "span_id": "8c33ffac-4d76-42ba-9737-89596e136b3d", - "trace_id": "df8340b4-b75a-48e9-a2f3-99725b46a00d", - "parent_span_id": "024ad5aa-f394-4c26-af51-564f3f51c6f2", - "name": "agent.rag_evaluation_metrics", - "start_time": "2025-10-16 08:49:58.420000000", - "end_time": "2025-10-16 08:49:58.420000000", - "duration_ms": 0.037, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "rag_evaluation_metrics", - "agent.capabilities": [ - "retrieval_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What are the benefits of using Noveum Trace?", - "query_length": 44, - "query_type": "rag_evaluation_query", - "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/docs/platform/dashboard):\n- Bulk Export: Download multiple traces simultaneously\n- Comparative Analysis: Compare performance across multiple traces\nReal-Time Updates\n- Live Refresh: Automatic updates as new traces arrive\n- Connection Monitoring: Real-time status of your t...", - "retrieval.context_length": 2773, - "retrieval.sources_count": 5, - "retrieval.context_quality": "high", - "retrieval.effectiveness": 1, - "retrieval.context_utilization": 2.773, - "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/docs/platform/dashboard):\n- Bulk Export: Download multiple traces simultaneously\n- Comparative Analysis: Compare performance across multiple traces\nReal-Time Updates\n- Live Refresh: Automatic updates as new traces arrive\n- Connection Monitoring: Real-time status of your trace ingestion pipeline\n- Performance Indicators: Live metrics for system health monitoring\nReady to dive deeper? Explore Projects & Environments to organize your AI applications, or check out Team Collaboration to share insights with your team.\nGet Earl...\n\nSource 2 (https://noveum.ai/en/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\n\ud83d\udcd8 TypeScript SDK (\n@noveum/trace\n)- Framework integrations for Next.js, Express.js, and Hono\n- TypeScript-first design with full type safety\n- Universal compatibility (Node.js, Edge Runtime, browsers)\n-\n\ud83d\udcca Noveum Platform\n- Real-time dashboard for analyzing traces and performance\n- Ad...\n\nSource 3 (https://noveum.ai/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\n\ud83d\udcd8 TypeScript SDK (\n@noveum/trace\n)- Framework integrations for Next.js, Express.js, and Hono\n- TypeScript-first design with full type safety\n- Universal compatibility (Node.js, Edge Runtime, browsers)\n-\n\ud83d\udcca Noveum Platform\n- Real-time dashboard for analyzing traces and performance\n- Ad...\n\nSource 4 (https://noveum.ai/en/docs/getting-started/overview):\n- Advanced search & filtering across traces and spans\n- Cost analysis and optimization recommendations\n- Team collaboration with shared insights and alerts\n\ud83d\udd0d What Noveum Traces\nLLM Operations\n- Model calls across all providers (OpenAI, Anthropic, Google, etc.)\n- Token usage and cost calculation\n- Prompt engineering effectiveness\n- Response quality metrics\nRAG Pipelines\n- Document retrieval performance and relevance\n- Embedding generation costs and latency\n- Context assembly and prompt constructi...\n\nSource 5 (https://noveum.ai/docs/getting-started/overview):\n- Advanced search & filtering across traces and spans\n- Cost analysis and optimization recommendations\n- Team collaboration with shared insights and alerts\n\ud83d\udd0d What Noveum Traces\nLLM Operations\n- Model calls across all providers (OpenAI, Anthropic, Google, etc.)\n- Token usage and cost calculation\n- Prompt engineering effectiveness\n- Response quality metrics\nRAG Pipelines\n- Document retrieval performance and relevance\n- Embedding generation costs and latency\n- Context assembly and prompt constructi...\n\nUser Question: What are the benefits of using Noveum Trace?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", - "prompt.prompt_length": 3313, - "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/docs/platform/dashboard):\n- Bulk Export: Download multiple traces simultaneously\n- Comparative Analysis: Compare performance across multiple traces\nReal-Time Updates\n- Live...", - "prompt.instruction_following": "rag_optimized", - "response.answer_length": 1415, - "response.answer_completeness": "complete", - "response.response_quality": "high", - "response.source_citation": 5, - "response.context_utilization": 0.5102776776054815, - "output_response": "RAG Answer: The benefits of using Noveum Trace include:\n\n1. **Comprehensive Tracing**: Noveum Trace offers decorator-based tracing for LLM calls, agents, and RAG pipelines, allowing for detailed monitoring of AI ...", - "evaluation.retrieval_effectiveness": 1, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 5, - "evaluation.context_relevance": "high", - "evaluation.overall_quality": "high", - "evaluation.ready_for_production": true, - "rag.retrieval_strategy": "semantic_similarity", - "rag.vector_search_results": 5, - "rag.context_synthesis": "multi_source", - "rag.document_coverage": 1, - "rag.information_density": 0.5102776776054815 - }, - "events": [], - "links": [], - "trace_trace_id": "df8340b4-b75a-48e9-a2f3-99725b46a00d", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:49:17.236000000", - "trace_end_time": "2025-10-16 08:50:00.368000000", - "trace_duration_ms": 43131.554000000004, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:49:17.236000000", - "updated_at": "2025-10-16 08:49:17.236000000" - }, - { - "span_id": "5b1a7121-d0bd-4525-821d-7721c6fbbd70", - "trace_id": "325ec189-2876-4ebe-b33c-649491a5053e", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-16 08:49:11.192000000", - "end_time": "2025-10-16 08:49:16.232000000", - "duration_ms": 5039.753, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "How do I configure Noveum for my system?", - "query.length": 40, - "processing.mode": "RAG", - "processing.time_seconds": 5.039698600769043, - "processing.response_length": 712, - "processing.sources_count": 5, - "output_response": "Final Answer: To configure Noveum for your system, you can refer to the SDK Integration Guide available in the Noveum documentation. This guide will help you trace your AI applications with minimal code changes. Th...", - "final_answer_mode": "RAG", - "query_processed.input_query": "How do I configure Noveum for my system?", - "query_processed.output_response": "Successfully processed query using RAG, generated 712 character response", - "query_processed.mode": "RAG", - "query_processed.processing_time": 5.039698600769043, - "query_processed.response_length": 712 - }, - "events": [], - "links": [], - "trace_trace_id": "325ec189-2876-4ebe-b33c-649491a5053e", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:49:11.192000000", - "trace_end_time": "2025-10-16 08:49:16.232000000", - "trace_duration_ms": 5039.841, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:49:11.192000000", - "updated_at": "2025-10-16 08:49:11.192000000" - }, - { - "span_id": "761c92c6-d15a-4a8e-a16a-a365922f8d7c", - "trace_id": "325ec189-2876-4ebe-b33c-649491a5053e", - "parent_span_id": "5b1a7121-d0bd-4525-821d-7721c6fbbd70", - "name": "agent.query_routing", - "start_time": "2025-10-16 08:49:11.192000000", - "end_time": "2025-10-16 08:49:16.232000000", - "duration_ms": 5039.642, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "How do I configure Noveum for my system?", - "query_length": 40, - "query_type": "routing_query", - "output_response": "Routed to RAG for query processing", - "router.classification": "RAG", - "router.confidence_score": 1, - "router.rag_keyword_score": 2, - "router.web_keyword_score": 0, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "325ec189-2876-4ebe-b33c-649491a5053e", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:49:11.192000000", - "trace_end_time": "2025-10-16 08:49:16.232000000", - "trace_duration_ms": 5039.841, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:49:11.192000000", - "updated_at": "2025-10-16 08:49:11.192000000" - }, - { - "span_id": "8fcf969e-a577-45aa-9fe6-1870d9665189", - "trace_id": "325ec189-2876-4ebe-b33c-649491a5053e", - "parent_span_id": "761c92c6-d15a-4a8e-a16a-a365922f8d7c", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-16 08:49:11.192000000", - "end_time": "2025-10-16 08:49:11.192000000", - "duration_ms": 0.045000000000000005, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: How do I configure Noveum for my system?", - "query_length": 40, - "query_type": "routing_evaluation_query", - "classification.mode": "RAG", - "classification.rag_keyword_score": 2, - "classification.web_keyword_score": 0, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "noveum_specific", - "query.keyword_density": 0.25, - "query.domain_affinity": "noveum", - "routing.decision": "Routed to RAG based on analysis", - "routing.rationale": "RAG score: 2, Web score: 0, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "Web Search", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.038461538461538464, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: RAG (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "325ec189-2876-4ebe-b33c-649491a5053e", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:49:11.192000000", - "trace_end_time": "2025-10-16 08:49:16.232000000", - "trace_duration_ms": 5039.841, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:49:11.192000000", - "updated_at": "2025-10-16 08:49:11.192000000" - }, - { - "span_id": "9e5bbea4-cc68-4fe6-b3d0-c87852635ec9", - "trace_id": "325ec189-2876-4ebe-b33c-649491a5053e", - "parent_span_id": "761c92c6-d15a-4a8e-a16a-a365922f8d7c", - "name": "agent.llm-rag", - "start_time": "2025-10-16 08:49:11.192000000", - "end_time": "2025-10-16 08:49:16.232000000", - "duration_ms": 5039.39, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "rag_agent", - "agent.operation": "llm-rag", - "agent.capabilities": [ - "document_retrieval", - "context_generation", - "response_generation" - ], - "agent.id": "noveum_rag_agent", - "input_query": "How do I configure Noveum for my system?", - "query_length": 40, - "query_type": "rag_query", - "output_response": "RAG Answer: To configure Noveum for your system, you can refer to the SDK Integration Guide available in the Noveum documentation. This guide will help you trace your AI applications with minimal code changes. Th...", - "rag.context_length": 2237, - "rag.sources_count": 5, - "rag.answer_length": 712, - "rag.mode": "retrieval_augmented_generation" - }, - "events": [], - "links": [], - "trace_trace_id": "325ec189-2876-4ebe-b33c-649491a5053e", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:49:11.192000000", - "trace_end_time": "2025-10-16 08:49:16.232000000", - "trace_duration_ms": 5039.841, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:49:11.192000000", - "updated_at": "2025-10-16 08:49:11.192000000" - }, - { - "span_id": "5ee706ad-c4ae-4c0b-94d1-6b9c43d173b4", - "trace_id": "325ec189-2876-4ebe-b33c-649491a5053e", - "parent_span_id": "9e5bbea4-cc68-4fe6-b3d0-c87852635ec9", - "name": "agent.llm_model_execution", - "start_time": "2025-10-16 08:49:11.607000000", - "end_time": "2025-10-16 08:49:15.718000000", - "duration_ms": 4110.915, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: How do I configure Noveum for my system?", - "query_length": 40, - "query_type": "rag_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 4.110846042633057, - "model.latency_ms": 4110.846042633057, - "model.start_time": 1760604551.6074958, - "model.end_time": 1760604555.7183418, - "model.performance_tier": "medium", - "model.prompt_tokens": 693, - "model.completion_tokens": 178, - "model.total_tokens": 871, - "model.tokens_per_second": 211.87852596934326, - "model.estimated_cost": 0.02613, - "model.efficiency_score": 0.817451205510907, - "model.response_length": 712, - "model.response_quality": "high", - "model.output_response": "Model Response: To configure Noveum for your system, you can refer to the SDK Integration Guide available in the Noveum documentation. This guide will help you trace your AI applications with minimal code changes. Th...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "325ec189-2876-4ebe-b33c-649491a5053e", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:49:11.192000000", - "trace_end_time": "2025-10-16 08:49:16.232000000", - "trace_duration_ms": 5039.841, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:49:11.192000000", - "updated_at": "2025-10-16 08:49:11.192000000" - }, - { - "span_id": "d13bdeda-81af-400a-82ca-a5ef7e1e8a6d", - "trace_id": "325ec189-2876-4ebe-b33c-649491a5053e", - "parent_span_id": "9e5bbea4-cc68-4fe6-b3d0-c87852635ec9", - "name": "agent.rag_evaluation_metrics", - "start_time": "2025-10-16 08:49:15.718000000", - "end_time": "2025-10-16 08:49:15.718000000", - "duration_ms": 0.118, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "rag_evaluation_metrics", - "agent.capabilities": [ - "retrieval_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: How do I configure Noveum for my system?", - "query_length": 40, - "query_type": "rag_evaluation_query", - "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/docs):\nWelcome to Noveum.ai Docs\nComprehensive AI tracing and observability for LLM applications, RAG systems, and AI agents\nWelcome to the Noveum.ai documentation! Here you'll find everything you need to integrate, configure, and optimize your AI applications using ou...", - "retrieval.context_length": 2237, - "retrieval.sources_count": 5, - "retrieval.context_quality": "high", - "retrieval.effectiveness": 1, - "retrieval.context_utilization": 2.237, - "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/docs):\nWelcome to Noveum.ai Docs\nComprehensive AI tracing and observability for LLM applications, RAG systems, and AI agents\nWelcome to the Noveum.ai documentation! Here you'll find everything you need to integrate, configure, and optimize your AI applications using our comprehensive tracing and observability platform.\nWe provide powerful SDKs for Python and TypeScript that enable you to trace LLM calls, RAG pipelines, and multi-agent workflows with minimal code changes. Our documentation is organized ...\n\nSource 2 (https://noveum.ai/docs):\nWelcome to Noveum.ai Docs\nComprehensive AI tracing and observability for LLM applications, RAG systems, and AI agents\nWelcome to the Noveum.ai documentation! Here you'll find everything you need to integrate, configure, and optimize your AI applications using our comprehensive tracing and observability platform.\nWe provide powerful SDKs for Python and TypeScript that enable you to trace LLM calls, RAG pipelines, and multi-agent workflows with minimal code changes. Our documentation is organized ...\n\nSource 3 (https://noveum.ai/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nSource 4 (https://noveum.ai/en/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nSource 5 (https://noveum.ai/en/docs):\n- \ud83c\udf0d Universal: Works across frameworks, clouds, and deployment models\n\ud83e\udd1d Community & Support\n- \ud83d\udcac Discord Community: Join our Discord\n- \ud83d\udce7 Email Support: [email protected]\n- \ud83d\udc1b Bug Reports: GitHub Issues\n- \ud83d\udcd6 Knowledge Base: Help Center\nReady to get started? Head to our SDK Integration Guide to begin tracing your AI applications in under 5 minutes!\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability ...\n\nUser Question: How do I configure Noveum for my system?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", - "prompt.prompt_length": 2773, - "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/docs):\nWelcome to Noveum.ai Docs\nComprehensive AI tracing and observability for LLM applications, RAG systems, and AI agents\nWelcome to the Noveum.ai documentation! Here...", - "prompt.instruction_following": "rag_optimized", - "response.answer_length": 712, - "response.answer_completeness": "complete", - "response.response_quality": "high", - "response.source_citation": 5, - "response.context_utilization": 0.3182834152883326, - "output_response": "RAG Answer: To configure Noveum for your system, you can refer to the SDK Integration Guide available in the Noveum documentation. This guide will help you trace your AI applications with minimal code changes. Th...", - "evaluation.retrieval_effectiveness": 1, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 5, - "evaluation.context_relevance": "high", - "evaluation.overall_quality": "high", - "evaluation.ready_for_production": true, - "rag.retrieval_strategy": "semantic_similarity", - "rag.vector_search_results": 5, - "rag.context_synthesis": "multi_source", - "rag.document_coverage": 1, - "rag.information_density": 0.3182834152883326 - }, - "events": [], - "links": [], - "trace_trace_id": "325ec189-2876-4ebe-b33c-649491a5053e", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:49:11.192000000", - "trace_end_time": "2025-10-16 08:49:16.232000000", - "trace_duration_ms": 5039.841, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:49:11.192000000", - "updated_at": "2025-10-16 08:49:11.192000000" - }, - { - "span_id": "7d6564fc-41b2-4d74-acb1-e77b575c0006", - "trace_id": "d5528586-c741-4703-8791-5fccf5c3e123", - "parent_span_id": "e79a037e-82b1-41e9-bdc7-42353e98ead7", - "name": "agent.query_routing", - "start_time": "2025-10-16 08:49:05.152000000", - "end_time": "2025-10-16 08:49:10.187000000", - "duration_ms": 5034.446, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What monitoring capabilities does Noveum provide?", - "query_length": 49, - "query_type": "routing_query", - "output_response": "Routed to RAG for query processing", - "router.classification": "RAG", - "router.confidence_score": 1, - "router.rag_keyword_score": 2, - "router.web_keyword_score": 0, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "d5528586-c741-4703-8791-5fccf5c3e123", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:49:05.152000000", - "trace_end_time": "2025-10-16 08:49:10.187000000", - "trace_duration_ms": 5034.604, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:49:05.152000000", - "updated_at": "2025-10-16 08:49:05.152000000" - }, - { - "span_id": "e79a037e-82b1-41e9-bdc7-42353e98ead7", - "trace_id": "d5528586-c741-4703-8791-5fccf5c3e123", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-16 08:49:05.152000000", - "end_time": "2025-10-16 08:49:10.187000000", - "duration_ms": 5034.531, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What monitoring capabilities does Noveum provide?", - "query.length": 49, - "processing.mode": "RAG", - "processing.time_seconds": 5.0344908237457275, - "processing.response_length": 552, - "processing.sources_count": 5, - "output_response": "Final Answer: Noveum provides comprehensive monitoring capabilities for AI applications, including the ability to monitor, trace, and optimize AI agents across various frameworks such as LangChain, CrewAI, AutoGen,...", - "final_answer_mode": "RAG", - "query_processed.input_query": "What monitoring capabilities does Noveum provide?", - "query_processed.output_response": "Successfully processed query using RAG, generated 552 character response", - "query_processed.mode": "RAG", - "query_processed.processing_time": 5.0344908237457275, - "query_processed.response_length": 552 - }, - "events": [], - "links": [], - "trace_trace_id": "d5528586-c741-4703-8791-5fccf5c3e123", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:49:05.152000000", - "trace_end_time": "2025-10-16 08:49:10.187000000", - "trace_duration_ms": 5034.604, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:49:05.152000000", - "updated_at": "2025-10-16 08:49:05.152000000" - }, - { - "span_id": "78297793-d73b-4311-b5ba-73509ae9f96e", - "trace_id": "d5528586-c741-4703-8791-5fccf5c3e123", - "parent_span_id": "7d6564fc-41b2-4d74-acb1-e77b575c0006", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-16 08:49:05.153000000", - "end_time": "2025-10-16 08:49:05.153000000", - "duration_ms": 0.029, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What monitoring capabilities does Noveum provide?", - "query_length": 49, - "query_type": "routing_evaluation_query", - "classification.mode": "RAG", - "classification.rag_keyword_score": 2, - "classification.web_keyword_score": 0, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "noveum_specific", - "query.keyword_density": 0.3333333333333333, - "query.domain_affinity": "noveum", - "routing.decision": "Routed to RAG based on analysis", - "routing.rationale": "RAG score: 2, Web score: 0, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "Web Search", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.038461538461538464, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: RAG (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "d5528586-c741-4703-8791-5fccf5c3e123", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:49:05.152000000", - "trace_end_time": "2025-10-16 08:49:10.187000000", - "trace_duration_ms": 5034.604, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:49:05.152000000", - "updated_at": "2025-10-16 08:49:05.152000000" - }, - { - "span_id": "6a101d9d-de83-415a-8fb0-01a9fae5ffb9", - "trace_id": "d5528586-c741-4703-8791-5fccf5c3e123", - "parent_span_id": "7d6564fc-41b2-4d74-acb1-e77b575c0006", - "name": "agent.llm-rag", - "start_time": "2025-10-16 08:49:05.153000000", - "end_time": "2025-10-16 08:49:10.187000000", - "duration_ms": 5034.282, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "rag_agent", - "agent.operation": "llm-rag", - "agent.capabilities": [ - "document_retrieval", - "context_generation", - "response_generation" - ], - "agent.id": "noveum_rag_agent", - "input_query": "What monitoring capabilities does Noveum provide?", - "query_length": 49, - "query_type": "rag_query", - "output_response": "RAG Answer: Noveum provides comprehensive monitoring capabilities for AI applications, including the ability to monitor, trace, and optimize AI agents across various frameworks such as LangChain, CrewAI, AutoGen,...", - "rag.context_length": 2740, - "rag.sources_count": 5, - "rag.answer_length": 552, - "rag.mode": "retrieval_augmented_generation" - }, - "events": [], - "links": [], - "trace_trace_id": "d5528586-c741-4703-8791-5fccf5c3e123", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:49:05.152000000", - "trace_end_time": "2025-10-16 08:49:10.187000000", - "trace_duration_ms": 5034.604, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:49:05.152000000", - "updated_at": "2025-10-16 08:49:05.152000000" - }, - { - "span_id": "26e510c0-441a-4f9f-9dce-31180f47d060", - "trace_id": "d5528586-c741-4703-8791-5fccf5c3e123", - "parent_span_id": "6a101d9d-de83-415a-8fb0-01a9fae5ffb9", - "name": "agent.llm_model_execution", - "start_time": "2025-10-16 08:49:05.787000000", - "end_time": "2025-10-16 08:49:09.274000000", - "duration_ms": 3486.699, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What monitoring capabilities does Noveum provide?", - "query_length": 49, - "query_type": "rag_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 3.4866299629211426, - "model.latency_ms": 3486.6299629211426, - "model.start_time": 1760604545.787763, - "model.end_time": 1760604549.274393, - "model.performance_tier": "medium", - "model.prompt_tokens": 821, - "model.completion_tokens": 138, - "model.total_tokens": 959, - "model.tokens_per_second": 275.0506965747916, - "model.estimated_cost": 0.02877, - "model.efficiency_score": 0.5755995828988529, - "model.response_length": 552, - "model.response_quality": "high", - "model.output_response": "Model Response: Noveum provides comprehensive monitoring capabilities for AI applications, including the ability to monitor, trace, and optimize AI agents across various frameworks such as LangChain, CrewAI, AutoGen,...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "d5528586-c741-4703-8791-5fccf5c3e123", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:49:05.152000000", - "trace_end_time": "2025-10-16 08:49:10.187000000", - "trace_duration_ms": 5034.604, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:49:05.152000000", - "updated_at": "2025-10-16 08:49:05.152000000" - }, - { - "span_id": "fcd89e38-3d21-468b-860a-ae2de0bbd119", - "trace_id": "d5528586-c741-4703-8791-5fccf5c3e123", - "parent_span_id": "6a101d9d-de83-415a-8fb0-01a9fae5ffb9", - "name": "agent.rag_evaluation_metrics", - "start_time": "2025-10-16 08:49:09.274000000", - "end_time": "2025-10-16 08:49:09.274000000", - "duration_ms": 0.054, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "rag_evaluation_metrics", - "agent.capabilities": [ - "retrieval_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What monitoring capabilities does Noveum provide?", - "query_length": 49, - "query_type": "rag_evaluation_query", - "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en):\nMonitor all your AI Agents\nimprove AI Agents today\nNoveum.ai helps you monitor, trace, and optimize your AI applications.\nNoveum.ai works with any AI framework \u2013 LangChain, CrewAI, AutoGen, custom implementations, or direct LLM calls. One dashboard shows everything.\n...", - "retrieval.context_length": 2740, - "retrieval.sources_count": 5, - "retrieval.context_quality": "high", - "retrieval.effectiveness": 1, - "retrieval.context_utilization": 2.74, - "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en):\nMonitor all your AI Agents\nimprove AI Agents today\nNoveum.ai helps you monitor, trace, and optimize your AI applications.\nNoveum.ai works with any AI framework \u2013 LangChain, CrewAI, AutoGen, custom implementations, or direct LLM calls. One dashboard shows everything.\nMonitor, Evaluate, Improve Your AI Agents\nThe control plane for AI agents.\nMonitor Everything, Miss Nothing\nOur lightweight SDKs capture every trace and span across your AI agent ecosystem\u2014from simple LLM calls to complex multi-agent...\n\nSource 2 (https://noveum.ai/):\nMonitor all your AI Agents\nimprove AI Agents today\nNoveum.ai helps you monitor, trace, and optimize your AI applications.\nNoveum.ai works with any AI framework \u2013 LangChain, CrewAI, AutoGen, custom implementations, or direct LLM calls. One dashboard shows everything.\nMonitor, Evaluate, Improve Your AI Agents\nThe control plane for AI agents.\nMonitor Everything, Miss Nothing\nOur lightweight SDKs capture every trace and span across your AI agent ecosystem\u2014from simple LLM calls to complex multi-agent...\n\nSource 3 (https://noveum.ai):\nMonitor all your AI Agents\nimprove AI Agents today\nNoveum.ai helps you monitor, trace, and optimize your AI applications.\nNoveum.ai works with any AI framework \u2013 LangChain, CrewAI, AutoGen, custom implementations, or direct LLM calls. One dashboard shows everything.\nMonitor, Evaluate, Improve Your AI Agents\nThe control plane for AI agents.\nMonitor Everything, Miss Nothing\nOur lightweight SDKs capture every trace and span across your AI agent ecosystem\u2014from simple LLM calls to complex multi-agent...\n\nSource 4 (https://noveum.ai/en/docs/getting-started/overview):\nNoveum.ai Overview\nComprehensive AI tracing and observability platform for LLM applications, RAG systems, and AI agents\nWelcome to Noveum.ai\u2014the comprehensive tracing and observability platform built specifically for AI applications. Whether you're building LLM-powered chatbots, RAG systems, multi-agent workflows, or any AI-driven application, Noveum provides the insights you need to understand, debug, and optimize your systems.\n\ud83c\udfaf Why AI Applications Need Specialized Observability\nTraditional mo...\n\nSource 5 (https://noveum.ai/docs/getting-started/overview):\nNoveum.ai Overview\nComprehensive AI tracing and observability platform for LLM applications, RAG systems, and AI agents\nWelcome to Noveum.ai\u2014the comprehensive tracing and observability platform built specifically for AI applications. Whether you're building LLM-powered chatbots, RAG systems, multi-agent workflows, or any AI-driven application, Noveum provides the insights you need to understand, debug, and optimize your systems.\n\ud83c\udfaf Why AI Applications Need Specialized Observability\nTraditional mo...\n\nUser Question: What monitoring capabilities does Noveum provide?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", - "prompt.prompt_length": 3285, - "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en):\nMonitor all your AI Agents\nimprove AI Agents today\nNoveum.ai helps you monitor, trace, and optimize your AI applications.\nNoveum.ai works with any AI framework \u2013 LangC...", - "prompt.instruction_following": "rag_optimized", - "response.answer_length": 552, - "response.answer_completeness": "complete", - "response.response_quality": "high", - "response.source_citation": 5, - "response.context_utilization": 0.20145985401459854, - "output_response": "RAG Answer: Noveum provides comprehensive monitoring capabilities for AI applications, including the ability to monitor, trace, and optimize AI agents across various frameworks such as LangChain, CrewAI, AutoGen,...", - "evaluation.retrieval_effectiveness": 1, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 5, - "evaluation.context_relevance": "high", - "evaluation.overall_quality": "high", - "evaluation.ready_for_production": true, - "rag.retrieval_strategy": "semantic_similarity", - "rag.vector_search_results": 5, - "rag.context_synthesis": "multi_source", - "rag.document_coverage": 1, - "rag.information_density": 0.20145985401459854 - }, - "events": [], - "links": [], - "trace_trace_id": "d5528586-c741-4703-8791-5fccf5c3e123", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:49:05.152000000", - "trace_end_time": "2025-10-16 08:49:10.187000000", - "trace_duration_ms": 5034.604, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:49:05.152000000", - "updated_at": "2025-10-16 08:49:05.152000000" - }, - { - "span_id": "bd5f205f-0329-4811-b883-129d69f12382", - "trace_id": "8100b869-07c0-4e37-b51e-092c9d20b4a7", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-16 08:48:57.574000000", - "end_time": "2025-10-16 08:49:04.149000000", - "duration_ms": 6574.424, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "How does Noveum handle agent tracing?", - "query.length": 37, - "processing.mode": "RAG", - "processing.time_seconds": 6.5743818283081055, - "processing.response_length": 643, - "processing.sources_count": 5, - "output_response": "Final Answer: Noveum.ai handles agent tracing through its specialized Multi-Agent Tracing capabilities, which are designed to observe complex workflows and inter-agent communications within multi-agent systems. The...", - "final_answer_mode": "RAG", - "query_processed.input_query": "How does Noveum handle agent tracing?", - "query_processed.output_response": "Successfully processed query using RAG, generated 643 character response", - "query_processed.mode": "RAG", - "query_processed.processing_time": 6.5743818283081055, - "query_processed.response_length": 643 - }, - "events": [], - "links": [], - "trace_trace_id": "8100b869-07c0-4e37-b51e-092c9d20b4a7", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:48:57.574000000", - "trace_end_time": "2025-10-16 08:49:04.149000000", - "trace_duration_ms": 6574.500999999999, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:48:57.574000000", - "updated_at": "2025-10-16 08:48:57.574000000" - }, - { - "span_id": "0801bcfe-8c09-4ea6-a754-5564395d6ded", - "trace_id": "8100b869-07c0-4e37-b51e-092c9d20b4a7", - "parent_span_id": "3c35d86a-0df9-42dc-bfe3-bbe978cf62b5", - "name": "agent.llm-rag", - "start_time": "2025-10-16 08:48:57.575000000", - "end_time": "2025-10-16 08:49:04.149000000", - "duration_ms": 6574.135, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "rag_agent", - "agent.operation": "llm-rag", - "agent.capabilities": [ - "document_retrieval", - "context_generation", - "response_generation" - ], - "agent.id": "noveum_rag_agent", - "input_query": "How does Noveum handle agent tracing?", - "query_length": 37, - "query_type": "rag_query", - "output_response": "RAG Answer: Noveum.ai handles agent tracing through its specialized Multi-Agent Tracing capabilities, which are designed to observe complex workflows and inter-agent communications within multi-agent systems. The...", - "rag.context_length": 2736, - "rag.sources_count": 5, - "rag.answer_length": 643, - "rag.mode": "retrieval_augmented_generation" - }, - "events": [], - "links": [], - "trace_trace_id": "8100b869-07c0-4e37-b51e-092c9d20b4a7", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:48:57.574000000", - "trace_end_time": "2025-10-16 08:49:04.149000000", - "trace_duration_ms": 6574.500999999999, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:48:57.574000000", - "updated_at": "2025-10-16 08:48:57.574000000" - }, - { - "span_id": "3c35d86a-0df9-42dc-bfe3-bbe978cf62b5", - "trace_id": "8100b869-07c0-4e37-b51e-092c9d20b4a7", - "parent_span_id": "bd5f205f-0329-4811-b883-129d69f12382", - "name": "agent.query_routing", - "start_time": "2025-10-16 08:48:57.575000000", - "end_time": "2025-10-16 08:49:04.149000000", - "duration_ms": 6574.342, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "How does Noveum handle agent tracing?", - "query_length": 37, - "query_type": "routing_query", - "output_response": "Routed to RAG for query processing", - "router.classification": "RAG", - "router.confidence_score": 1, - "router.rag_keyword_score": 2, - "router.web_keyword_score": 0, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "8100b869-07c0-4e37-b51e-092c9d20b4a7", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:48:57.574000000", - "trace_end_time": "2025-10-16 08:49:04.149000000", - "trace_duration_ms": 6574.500999999999, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:48:57.574000000", - "updated_at": "2025-10-16 08:48:57.574000000" - }, - { - "span_id": "b1914316-f545-4bc9-abf4-3494cf42c4a5", - "trace_id": "8100b869-07c0-4e37-b51e-092c9d20b4a7", - "parent_span_id": "3c35d86a-0df9-42dc-bfe3-bbe978cf62b5", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-16 08:48:57.575000000", - "end_time": "2025-10-16 08:48:57.575000000", - "duration_ms": 0.032, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: How does Noveum handle agent tracing?", - "query_length": 37, - "query_type": "routing_evaluation_query", - "classification.mode": "RAG", - "classification.rag_keyword_score": 2, - "classification.web_keyword_score": 0, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "noveum_specific", - "query.keyword_density": 0.3333333333333333, - "query.domain_affinity": "noveum", - "routing.decision": "Routed to RAG based on analysis", - "routing.rationale": "RAG score: 2, Web score: 0, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "Web Search", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.038461538461538464, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: RAG (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "8100b869-07c0-4e37-b51e-092c9d20b4a7", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:48:57.574000000", - "trace_end_time": "2025-10-16 08:49:04.149000000", - "trace_duration_ms": 6574.500999999999, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:48:57.574000000", - "updated_at": "2025-10-16 08:48:57.574000000" - }, - { - "span_id": "0906814b-6a80-40e1-9f00-5376b809703a", - "trace_id": "8100b869-07c0-4e37-b51e-092c9d20b4a7", - "parent_span_id": "0801bcfe-8c09-4ea6-a754-5564395d6ded", - "name": "agent.llm_model_execution", - "start_time": "2025-10-16 08:48:58.009000000", - "end_time": "2025-10-16 08:49:03.349000000", - "duration_ms": 5340.01, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: How does Noveum handle agent tracing?", - "query_length": 37, - "query_type": "rag_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 5.339954614639282, - "model.latency_ms": 5339.954614639282, - "model.start_time": 1760604538.0093303, - "model.end_time": 1760604543.349285, - "model.performance_tier": "slow", - "model.prompt_tokens": 817, - "model.completion_tokens": 160, - "model.total_tokens": 977, - "model.tokens_per_second": 182.9603565022054, - "model.estimated_cost": 0.02931, - "model.efficiency_score": 0.6581371545547595, - "model.response_length": 643, - "model.response_quality": "high", - "model.output_response": "Model Response: Noveum.ai handles agent tracing through its specialized Multi-Agent Tracing capabilities, which are designed to observe complex workflows and inter-agent communications within multi-agent systems. The...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "8100b869-07c0-4e37-b51e-092c9d20b4a7", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:48:57.574000000", - "trace_end_time": "2025-10-16 08:49:04.149000000", - "trace_duration_ms": 6574.500999999999, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:48:57.574000000", - "updated_at": "2025-10-16 08:48:57.574000000" - }, - { - "span_id": "29822edb-c15f-4baa-b6ec-84d85fcc926d", - "trace_id": "8100b869-07c0-4e37-b51e-092c9d20b4a7", - "parent_span_id": "0801bcfe-8c09-4ea6-a754-5564395d6ded", - "name": "agent.rag_evaluation_metrics", - "start_time": "2025-10-16 08:49:03.349000000", - "end_time": "2025-10-16 08:49:03.349000000", - "duration_ms": 0.034999999999999996, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "rag_evaluation_metrics", - "agent.capabilities": [ - "retrieval_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: How does Noveum handle agent tracing?", - "query_length": 37, - "query_type": "rag_evaluation_query", - "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, vector searches, data retrieval, agent reasoning, and more. Without proper observability, debugging ...", - "retrieval.context_length": 2736, - "retrieval.sources_count": 5, - "retrieval.context_quality": "high", - "retrieval.effectiveness": 1, - "retrieval.context_utilization": 2.736, - "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, vector searches, data retrieval, agent reasoning, and more. Without proper observability, debugging becomes a nightmare, optimization is guesswork, and understanding user interactions is nearly impossible.\nNoveum.ai solves this by providing comprehensive tracing and observability specifically designed for AI applications.\nKey benefits of using Noveum.ai:\n- Complete Visibility: Trace...\n\nSource 2 (https://noveum.ai/docs/advanced/multi-agent-tracing):\nMulti-Agent Tracing\nObserve complex agent workflows and inter-agent communications with comprehensive tracing\nMulti-agent systems represent some of the most complex AI applications, involving multiple agents that coordinate, communicate, and collaborate to achieve shared goals. Noveum.ai provides specialized tracing capabilities to help you understand and optimize these intricate workflows.\n\ud83c\udfaf Why Multi-Agent Tracing Matters\nMulti-agent systems introduce unique observability challenges:\n- Complex...\n\nSource 3 (https://noveum.ai/en/docs/advanced/multi-agent-tracing):\nMulti-Agent Tracing\nObserve complex agent workflows and inter-agent communications with comprehensive tracing\nMulti-agent systems represent some of the most complex AI applications, involving multiple agents that coordinate, communicate, and collaborate to achieve shared goals. Noveum.ai provides specialized tracing capabilities to help you understand and optimize these intricate workflows.\n\ud83c\udfaf Why Multi-Agent Tracing Matters\nMulti-agent systems introduce unique observability challenges:\n- Complex...\n\nSource 4 (https://noveum.ai/en/docs/advanced/performance-optimization):\n\ud83d\udd17 Next Steps\n- Multi-Agent Tracing - Observe agent workflows\n- RAG Pipeline Observability - Monitor retrieval and generation systems\n- Custom Instrumentation - Add domain-specific tracing\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nSource 5 (https://noveum.ai/docs/advanced/performance-optimization):\n\ud83d\udd17 Next Steps\n- Multi-Agent Tracing - Observe agent workflows\n- RAG Pipeline Observability - Monitor retrieval and generation systems\n- Custom Instrumentation - Add domain-specific tracing\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nUser Question: How does Noveum handle agent tracing?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", - "prompt.prompt_length": 3269, - "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, ...", - "prompt.instruction_following": "rag_optimized", - "response.answer_length": 643, - "response.answer_completeness": "complete", - "response.response_quality": "high", - "response.source_citation": 5, - "response.context_utilization": 0.23501461988304093, - "output_response": "RAG Answer: Noveum.ai handles agent tracing through its specialized Multi-Agent Tracing capabilities, which are designed to observe complex workflows and inter-agent communications within multi-agent systems. The...", - "evaluation.retrieval_effectiveness": 1, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 5, - "evaluation.context_relevance": "high", - "evaluation.overall_quality": "high", - "evaluation.ready_for_production": true, - "rag.retrieval_strategy": "semantic_similarity", - "rag.vector_search_results": 5, - "rag.context_synthesis": "multi_source", - "rag.document_coverage": 1, - "rag.information_density": 0.23501461988304093 - }, - "events": [], - "links": [], - "trace_trace_id": "8100b869-07c0-4e37-b51e-092c9d20b4a7", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:48:57.574000000", - "trace_end_time": "2025-10-16 08:49:04.149000000", - "trace_duration_ms": 6574.500999999999, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:48:57.574000000", - "updated_at": "2025-10-16 08:48:57.574000000" - }, - { - "span_id": "5f8ea007-edef-477c-b8f1-57c2da9e9258", - "trace_id": "c28bd31c-d200-4467-91fd-a3b80187c885", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-16 08:48:17.977000000", - "end_time": "2025-10-16 08:48:56.570000000", - "duration_ms": 38592.912, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What APIs are available in Noveum platform?", - "query.length": 43, - "processing.mode": "RAG", - "processing.time_seconds": 38.59272003173828, - "processing.response_length": 270, - "processing.sources_count": 5, - "output_response": "Final Answer: The provided context does not specify the APIs available in the Noveum platform. For detailed information about the APIs, I recommend checking the official Noveum documentation or the SDK Integration ...", - "final_answer_mode": "RAG", - "query_processed.input_query": "What APIs are available in Noveum platform?", - "query_processed.output_response": "Successfully processed query using RAG, generated 270 character response", - "query_processed.mode": "RAG", - "query_processed.processing_time": 38.59272003173828, - "query_processed.response_length": 270 - }, - "events": [], - "links": [], - "trace_trace_id": "c28bd31c-d200-4467-91fd-a3b80187c885", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:48:17.977000000", - "trace_end_time": "2025-10-16 08:48:56.570000000", - "trace_duration_ms": 38592.996, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:48:17.977000000", - "updated_at": "2025-10-16 08:48:17.977000000" - }, - { - "span_id": "db385f4d-beef-465b-9831-a65080537519", - "trace_id": "c28bd31c-d200-4467-91fd-a3b80187c885", - "parent_span_id": "5f8ea007-edef-477c-b8f1-57c2da9e9258", - "name": "agent.query_routing", - "start_time": "2025-10-16 08:48:17.977000000", - "end_time": "2025-10-16 08:48:56.570000000", - "duration_ms": 38592.66499999999, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What APIs are available in Noveum platform?", - "query_length": 43, - "query_type": "routing_query", - "output_response": "Routed to RAG for query processing", - "router.classification": "RAG", - "router.confidence_score": 1, - "router.rag_keyword_score": 3, - "router.web_keyword_score": 0, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "c28bd31c-d200-4467-91fd-a3b80187c885", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:48:17.977000000", - "trace_end_time": "2025-10-16 08:48:56.570000000", - "trace_duration_ms": 38592.996, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:48:17.977000000", - "updated_at": "2025-10-16 08:48:17.977000000" - }, - { - "span_id": "3ccebe1c-1f2a-433b-a36f-453a0b803511", - "trace_id": "c28bd31c-d200-4467-91fd-a3b80187c885", - "parent_span_id": "db385f4d-beef-465b-9831-a65080537519", - "name": "agent.llm-rag", - "start_time": "2025-10-16 08:48:17.978000000", - "end_time": "2025-10-16 08:48:56.570000000", - "duration_ms": 38592.416999999994, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "rag_agent", - "agent.operation": "llm-rag", - "agent.capabilities": [ - "document_retrieval", - "context_generation", - "response_generation" - ], - "agent.id": "noveum_rag_agent", - "input_query": "What APIs are available in Noveum platform?", - "query_length": 43, - "query_type": "rag_query", - "output_response": "RAG Answer: The provided context does not specify the APIs available in the Noveum platform. For detailed information about the APIs, I recommend checking the official Noveum documentation or the SDK Integration ...", - "rag.context_length": 2284, - "rag.sources_count": 5, - "rag.answer_length": 270, - "rag.mode": "retrieval_augmented_generation" - }, - "events": [], - "links": [], - "trace_trace_id": "c28bd31c-d200-4467-91fd-a3b80187c885", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:48:17.977000000", - "trace_end_time": "2025-10-16 08:48:56.570000000", - "trace_duration_ms": 38592.996, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:48:17.977000000", - "updated_at": "2025-10-16 08:48:17.977000000" - }, - { - "span_id": "6d1bb8e9-315d-433a-8df5-7f5fb2eeee16", - "trace_id": "c28bd31c-d200-4467-91fd-a3b80187c885", - "parent_span_id": "db385f4d-beef-465b-9831-a65080537519", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-16 08:48:17.978000000", - "end_time": "2025-10-16 08:48:17.978000000", - "duration_ms": 0.043000000000000003, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What APIs are available in Noveum platform?", - "query_length": 43, - "query_type": "routing_evaluation_query", - "classification.mode": "RAG", - "classification.rag_keyword_score": 3, - "classification.web_keyword_score": 0, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "noveum_specific", - "query.keyword_density": 0.42857142857142855, - "query.domain_affinity": "noveum", - "routing.decision": "Routed to RAG based on analysis", - "routing.rationale": "RAG score: 3, Web score: 0, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "Web Search", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.057692307692307696, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: RAG (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "c28bd31c-d200-4467-91fd-a3b80187c885", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:48:17.977000000", - "trace_end_time": "2025-10-16 08:48:56.570000000", - "trace_duration_ms": 38592.996, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:48:17.977000000", - "updated_at": "2025-10-16 08:48:17.977000000" - }, - { - "span_id": "2f11df8a-8f45-425d-a0ce-1eec7887b508", - "trace_id": "c28bd31c-d200-4467-91fd-a3b80187c885", - "parent_span_id": "3ccebe1c-1f2a-433b-a36f-453a0b803511", - "name": "agent.llm_model_execution", - "start_time": "2025-10-16 08:48:18.392000000", - "end_time": "2025-10-16 08:48:55.545000000", - "duration_ms": 37152.496999999996, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What APIs are available in Noveum platform?", - "query_length": 43, - "query_type": "rag_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 37.15243363380432, - "model.latency_ms": 37152.43363380432, - "model.start_time": 1760604498.3926067, - "model.end_time": 1760604535.5450404, - "model.performance_tier": "slow", - "model.prompt_tokens": 705, - "model.completion_tokens": 67, - "model.total_tokens": 772, - "model.tokens_per_second": 20.779257897592238, - "model.estimated_cost": 0.02316, - "model.efficiency_score": 0.34974093264248707, - "model.response_length": 270, - "model.response_quality": "high", - "model.output_response": "Model Response: The provided context does not specify the APIs available in the Noveum platform. For detailed information about the APIs, I recommend checking the official Noveum documentation or the SDK Integration ...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "c28bd31c-d200-4467-91fd-a3b80187c885", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:48:17.977000000", - "trace_end_time": "2025-10-16 08:48:56.570000000", - "trace_duration_ms": 38592.996, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:48:17.977000000", - "updated_at": "2025-10-16 08:48:17.977000000" - }, - { - "span_id": "d13e608c-06e4-47c4-b602-c40ac9560be2", - "trace_id": "c28bd31c-d200-4467-91fd-a3b80187c885", - "parent_span_id": "3ccebe1c-1f2a-433b-a36f-453a0b803511", - "name": "agent.rag_evaluation_metrics", - "start_time": "2025-10-16 08:48:55.545000000", - "end_time": "2025-10-16 08:48:55.545000000", - "duration_ms": 0.031, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "rag_evaluation_metrics", - "agent.capabilities": [ - "retrieval_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What APIs are available in Noveum platform?", - "query_length": 43, - "query_type": "rag_evaluation_query", - "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/docs):\n- \ud83c\udf0d Universal: Works across frameworks, clouds, and deployment models\n\ud83e\udd1d Community & Support\n- \ud83d\udcac Discord Community: Join our Discord\n- \ud83d\udce7 Email Support: [email protected]\n- \ud83d\udc1b Bug Reports: GitHub Issues\n- \ud83d\udcd6 Knowledge Base: Help Center\nReady to get started? Head to ...", - "retrieval.context_length": 2284, - "retrieval.sources_count": 5, - "retrieval.context_quality": "high", - "retrieval.effectiveness": 1, - "retrieval.context_utilization": 2.284, - "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/docs):\n- \ud83c\udf0d Universal: Works across frameworks, clouds, and deployment models\n\ud83e\udd1d Community & Support\n- \ud83d\udcac Discord Community: Join our Discord\n- \ud83d\udce7 Email Support: [email protected]\n- \ud83d\udc1b Bug Reports: GitHub Issues\n- \ud83d\udcd6 Knowledge Base: Help Center\nReady to get started? Head to our SDK Integration Guide to begin tracing your AI applications in under 5 minutes!\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability ...\n\nSource 2 (https://noveum.ai/docs):\n- \ud83c\udf0d Universal: Works across frameworks, clouds, and deployment models\n\ud83e\udd1d Community & Support\n- \ud83d\udcac Discord Community: Join our Discord\n- \ud83d\udce7 Email Support: [email protected]\n- \ud83d\udc1b Bug Reports: GitHub Issues\n- \ud83d\udcd6 Knowledge Base: Help Center\nReady to get started? Head to our SDK Integration Guide to begin tracing your AI applications in under 5 minutes!\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability ...\n\nSource 3 (https://noveum.ai/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nSource 4 (https://noveum.ai/en/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nSource 5 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, vector searches, data retrieval, agent reasoning, and more. Without proper observability, debugging becomes a nightmare, optimization is guesswork, and understanding user interactions is nearly impossible.\nNoveum.ai solves this by providing comprehensive tracing and observability specifically designed for AI applications.\nKey benefits of using Noveum.ai:\n- Complete Visibility: Trace...\n\nUser Question: What APIs are available in Noveum platform?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", - "prompt.prompt_length": 2823, - "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/docs):\n- \ud83c\udf0d Universal: Works across frameworks, clouds, and deployment models\n\ud83e\udd1d Community & Support\n- \ud83d\udcac Discord Community: Join our Discord\n- \ud83d\udce7 Email Support: [email prot...", - "prompt.instruction_following": "rag_optimized", - "response.answer_length": 270, - "response.answer_completeness": "complete", - "response.response_quality": "high", - "response.source_citation": 5, - "response.context_utilization": 0.11821366024518389, - "output_response": "RAG Answer: The provided context does not specify the APIs available in the Noveum platform. For detailed information about the APIs, I recommend checking the official Noveum documentation or the SDK Integration ...", - "evaluation.retrieval_effectiveness": 1, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 5, - "evaluation.context_relevance": "high", - "evaluation.overall_quality": "high", - "evaluation.ready_for_production": true, - "rag.retrieval_strategy": "semantic_similarity", - "rag.vector_search_results": 5, - "rag.context_synthesis": "multi_source", - "rag.document_coverage": 1, - "rag.information_density": 0.11821366024518389 - }, - "events": [], - "links": [], - "trace_trace_id": "c28bd31c-d200-4467-91fd-a3b80187c885", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:48:17.977000000", - "trace_end_time": "2025-10-16 08:48:56.570000000", - "trace_duration_ms": 38592.996, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:48:17.977000000", - "updated_at": "2025-10-16 08:48:17.977000000" - }, - { - "span_id": "3f3babbd-abf5-4f5a-b31c-fa5dcec9c360", - "trace_id": "b90ef3fc-11ea-4230-a59b-e2f2db8f2de0", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-16 08:48:08.007000000", - "end_time": "2025-10-16 08:48:16.975000000", - "duration_ms": 8967.201, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "How do I set up observability with Noveum?", - "query.length": 42, - "processing.mode": "RAG", - "processing.time_seconds": 8.96716833114624, - "processing.response_length": 838, - "processing.sources_count": 5, - "output_response": "Final Answer: To set up observability with Noveum, follow these steps:\n\n1. **Sign Up**: Go to [noveum.ai](https://noveum.ai) and create an account.\n2. **Create a Project**: Once signed in, create a new project to o...", - "final_answer_mode": "RAG", - "query_processed.input_query": "How do I set up observability with Noveum?", - "query_processed.output_response": "Successfully processed query using RAG, generated 838 character response", - "query_processed.mode": "RAG", - "query_processed.processing_time": 8.96716833114624, - "query_processed.response_length": 838 - }, - "events": [], - "links": [], - "trace_trace_id": "b90ef3fc-11ea-4230-a59b-e2f2db8f2de0", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:48:08.007000000", - "trace_end_time": "2025-10-16 08:48:16.975000000", - "trace_duration_ms": 8967.264, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:48:08.007000000", - "updated_at": "2025-10-16 08:48:08.007000000" - }, - { - "span_id": "41f6453c-4be6-4b66-8f9b-8e795f7b41b0", - "trace_id": "b90ef3fc-11ea-4230-a59b-e2f2db8f2de0", - "parent_span_id": "b0a76345-49e5-4c1b-915f-cec90ea2b1f7", - "name": "agent.llm-rag", - "start_time": "2025-10-16 08:48:08.007000000", - "end_time": "2025-10-16 08:48:16.974000000", - "duration_ms": 8966.964, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "rag_agent", - "agent.operation": "llm-rag", - "agent.capabilities": [ - "document_retrieval", - "context_generation", - "response_generation" - ], - "agent.id": "noveum_rag_agent", - "input_query": "How do I set up observability with Noveum?", - "query_length": 42, - "query_type": "rag_query", - "output_response": "RAG Answer: To set up observability with Noveum, follow these steps:\n\n1. **Sign Up**: Go to [noveum.ai](https://noveum.ai) and create an account.\n2. **Create a Project**: Once signed in, create a new project to o...", - "rag.context_length": 2870, - "rag.sources_count": 6, - "rag.answer_length": 838, - "rag.mode": "retrieval_augmented_generation" - }, - "events": [], - "links": [], - "trace_trace_id": "b90ef3fc-11ea-4230-a59b-e2f2db8f2de0", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:48:08.007000000", - "trace_end_time": "2025-10-16 08:48:16.975000000", - "trace_duration_ms": 8967.264, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:48:08.007000000", - "updated_at": "2025-10-16 08:48:08.007000000" - }, - { - "span_id": "b0a76345-49e5-4c1b-915f-cec90ea2b1f7", - "trace_id": "b90ef3fc-11ea-4230-a59b-e2f2db8f2de0", - "parent_span_id": "3f3babbd-abf5-4f5a-b31c-fa5dcec9c360", - "name": "agent.query_routing", - "start_time": "2025-10-16 08:48:08.007000000", - "end_time": "2025-10-16 08:48:16.974000000", - "duration_ms": 8967.129, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "How do I set up observability with Noveum?", - "query_length": 42, - "query_type": "routing_query", - "output_response": "Routed to RAG for query processing", - "router.classification": "RAG", - "router.confidence_score": 1, - "router.rag_keyword_score": 2, - "router.web_keyword_score": 0, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "b90ef3fc-11ea-4230-a59b-e2f2db8f2de0", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:48:08.007000000", - "trace_end_time": "2025-10-16 08:48:16.975000000", - "trace_duration_ms": 8967.264, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:48:08.007000000", - "updated_at": "2025-10-16 08:48:08.007000000" - }, - { - "span_id": "fe3074a7-f4c0-442e-9a5b-e2cc11d93785", - "trace_id": "b90ef3fc-11ea-4230-a59b-e2f2db8f2de0", - "parent_span_id": "b0a76345-49e5-4c1b-915f-cec90ea2b1f7", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-16 08:48:08.007000000", - "end_time": "2025-10-16 08:48:08.007000000", - "duration_ms": 0.032, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: How do I set up observability with Noveum?", - "query_length": 42, - "query_type": "routing_evaluation_query", - "classification.mode": "RAG", - "classification.rag_keyword_score": 2, - "classification.web_keyword_score": 0, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "noveum_specific", - "query.keyword_density": 0.25, - "query.domain_affinity": "noveum", - "routing.decision": "Routed to RAG based on analysis", - "routing.rationale": "RAG score: 2, Web score: 0, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "Web Search", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.038461538461538464, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: RAG (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "b90ef3fc-11ea-4230-a59b-e2f2db8f2de0", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:48:08.007000000", - "trace_end_time": "2025-10-16 08:48:16.975000000", - "trace_duration_ms": 8967.264, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:48:08.007000000", - "updated_at": "2025-10-16 08:48:08.007000000" - }, - { - "span_id": "e16139ee-048e-4c2f-87f8-90442f41a0ae", - "trace_id": "b90ef3fc-11ea-4230-a59b-e2f2db8f2de0", - "parent_span_id": "41f6453c-4be6-4b66-8f9b-8e795f7b41b0", - "name": "agent.llm_model_execution", - "start_time": "2025-10-16 08:48:08.644000000", - "end_time": "2025-10-16 08:48:15.709000000", - "duration_ms": 7064.898, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: How do I set up observability with Noveum?", - "query_length": 42, - "query_type": "rag_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 7.0648510456085205, - "model.latency_ms": 7064.8510456085205, - "model.start_time": 1760604488.6448367, - "model.end_time": 1760604495.7096877, - "model.performance_tier": "slow", - "model.prompt_tokens": 852, - "model.completion_tokens": 209, - "model.total_tokens": 1061, - "model.tokens_per_second": 150.1800948315128, - "model.estimated_cost": 0.031830000000000004, - "model.efficiency_score": 0.7898209236569275, - "model.response_length": 838, - "model.response_quality": "high", - "model.output_response": "Model Response: To set up observability with Noveum, follow these steps:\n\n1. **Sign Up**: Go to [noveum.ai](https://noveum.ai) and create an account.\n2. **Create a Project**: Once signed in, create a new project to o...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "b90ef3fc-11ea-4230-a59b-e2f2db8f2de0", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:48:08.007000000", - "trace_end_time": "2025-10-16 08:48:16.975000000", - "trace_duration_ms": 8967.264, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:48:08.007000000", - "updated_at": "2025-10-16 08:48:08.007000000" - }, - { - "span_id": "59e85f8e-c19b-4794-9a69-7b5bc90208f0", - "trace_id": "b90ef3fc-11ea-4230-a59b-e2f2db8f2de0", - "parent_span_id": "41f6453c-4be6-4b66-8f9b-8e795f7b41b0", - "name": "agent.rag_evaluation_metrics", - "start_time": "2025-10-16 08:48:15.709000000", - "end_time": "2025-10-16 08:48:15.709000000", - "duration_ms": 0.039, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "rag_evaluation_metrics", - "agent.capabilities": [ - "retrieval_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: How do I set up observability with Noveum?", - "query_length": 42, - "query_type": "rag_evaluation_query", - "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\n- Production Ready: Built for scale with intelligent sampling and batching\n- Privacy Focused: You control what data is captured and how it's stored\n- Open Source SDKs: Transparent, auditable, and extensible\nGetting ...", - "retrieval.context_length": 2870, - "retrieval.sources_count": 6, - "retrieval.context_quality": "high", - "retrieval.effectiveness": 1.2, - "retrieval.context_utilization": 2.87, - "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\n- Production Ready: Built for scale with intelligent sampling and batching\n- Privacy Focused: You control what data is captured and how it's stored\n- Open Source SDKs: Transparent, auditable, and extensible\nGetting Started\nReady to add comprehensive observability to your AI applications?\n- Sign up at noveum.ai\n- Create a project and get your API key\n- Install the SDK for your preferred language\n- Add tracing to your AI workflows\n- Explore insights in the Noveum.ai dashboard\nCheck out our integra...\n\nSource 2 (https://noveum.ai/en/docs/getting-started/tracing-concepts):\n- Explore Framework Integrations - Framework-specific guidance\n- Learn Advanced Patterns - Custom instrumentation techniques\n- Master the Dashboard - Analyze your traces effectively\nRemember: Good observability is not about collecting all possible data, but about collecting the right data that helps you understand, debug, and optimize your AI applications.\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to O...\n\nSource 3 (https://noveum.ai/docs/getting-started/tracing-concepts):\n- Explore Framework Integrations - Framework-specific guidance\n- Learn Advanced Patterns - Custom instrumentation techniques\n- Master the Dashboard - Analyze your traces effectively\nRemember: Good observability is not about collecting all possible data, but about collecting the right data that helps you understand, debug, and optimize your AI applications.\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to O...\n\nSource 4 (https://noveum.ai/en/docs/getting-started/overview):\nNoveum.ai Overview\nComprehensive AI tracing and observability platform for LLM applications, RAG systems, and AI agents\nWelcome to Noveum.ai\u2014the comprehensive tracing and observability platform built specifically for AI applications. Whether you're building LLM-powered chatbots, RAG systems, multi-agent workflows, or any AI-driven application, Noveum provides the insights you need to understand, debug, and optimize your systems.\n\ud83c\udfaf Why AI Applications Need Specialized Observability\nTraditional mo...\n\nSource 5 (https://noveum.ai/docs/getting-started/overview):\nNoveum.ai Overview\nComprehensive AI tracing and observability platform for LLM applications, RAG systems, and AI agents\nWelcome to Noveum.ai\u2014the comprehensive tracing and observability platform built specifically for AI applications. Whether you're building LLM-powered chatbots, RAG systems, multi-agent workflows, or any AI-driven application, Noveum provides the insights you need to understand, debug, and optimize your systems.\n\ud83c\udfaf Why AI Applications Need Specialized Observability\nTraditional mo...\n\nUser Question: How do I set up observability with Noveum?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", - "prompt.prompt_length": 3408, - "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\n- Production Ready: Built for scale with intelligent sampling and batching\n- Privacy Focused: You control what data...", - "prompt.instruction_following": "rag_optimized", - "response.answer_length": 838, - "response.answer_completeness": "complete", - "response.response_quality": "high", - "response.source_citation": 6, - "response.context_utilization": 0.29198606271777006, - "output_response": "RAG Answer: To set up observability with Noveum, follow these steps:\n\n1. **Sign Up**: Go to [noveum.ai](https://noveum.ai) and create an account.\n2. **Create a Project**: Once signed in, create a new project to o...", - "evaluation.retrieval_effectiveness": 1.2, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 6, - "evaluation.context_relevance": "high", - "evaluation.overall_quality": "high", - "evaluation.ready_for_production": true, - "rag.retrieval_strategy": "semantic_similarity", - "rag.vector_search_results": 6, - "rag.context_synthesis": "multi_source", - "rag.document_coverage": 1.2, - "rag.information_density": 0.29198606271777006 - }, - "events": [], - "links": [], - "trace_trace_id": "b90ef3fc-11ea-4230-a59b-e2f2db8f2de0", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:48:08.007000000", - "trace_end_time": "2025-10-16 08:48:16.975000000", - "trace_duration_ms": 8967.264, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:48:08.007000000", - "updated_at": "2025-10-16 08:48:08.007000000" - }, - { - "span_id": "f049ed2b-ff2f-4092-9974-acc66f81bd8d", - "trace_id": "b8abebac-a7dc-494c-a1eb-6470181f32d2", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-16 08:47:57.667000000", - "end_time": "2025-10-16 08:48:07.005000000", - "duration_ms": 9337.298, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What features does Noveum Trace offer?", - "query.length": 38, - "processing.mode": "RAG", - "processing.time_seconds": 9.33726453781128, - "processing.response_length": 1018, - "processing.sources_count": 5, - "output_response": "Final Answer: Noveum Trace offers several key features through its Python SDK (`noveum-trace`) and TypeScript SDK (`@noveum/trace`):\n\n1. **Decorator-Based Tracing**: It provides a decorator-based approach for traci...", - "final_answer_mode": "RAG", - "query_processed.input_query": "What features does Noveum Trace offer?", - "query_processed.output_response": "Successfully processed query using RAG, generated 1018 character response", - "query_processed.mode": "RAG", - "query_processed.processing_time": 9.33726453781128, - "query_processed.response_length": 1018 - }, - "events": [], - "links": [], - "trace_trace_id": "b8abebac-a7dc-494c-a1eb-6470181f32d2", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:47:57.667000000", - "trace_end_time": "2025-10-16 08:48:07.005000000", - "trace_duration_ms": 9337.405, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:47:57.667000000", - "updated_at": "2025-10-16 08:47:57.667000000" - }, - { - "span_id": "211f1fe7-4cfb-4c1c-9c72-62ae2825dda3", - "trace_id": "b8abebac-a7dc-494c-a1eb-6470181f32d2", - "parent_span_id": "f049ed2b-ff2f-4092-9974-acc66f81bd8d", - "name": "agent.query_routing", - "start_time": "2025-10-16 08:47:57.668000000", - "end_time": "2025-10-16 08:48:07.005000000", - "duration_ms": 9337.223, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What features does Noveum Trace offer?", - "query_length": 38, - "query_type": "routing_query", - "output_response": "Routed to RAG for query processing", - "router.classification": "RAG", - "router.confidence_score": 1, - "router.rag_keyword_score": 3, - "router.web_keyword_score": 0, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "b8abebac-a7dc-494c-a1eb-6470181f32d2", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:47:57.667000000", - "trace_end_time": "2025-10-16 08:48:07.005000000", - "trace_duration_ms": 9337.405, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:47:57.667000000", - "updated_at": "2025-10-16 08:47:57.667000000" - }, - { - "span_id": "976c0c4b-efc1-4b7d-bfad-350c7fd0f71e", - "trace_id": "b8abebac-a7dc-494c-a1eb-6470181f32d2", - "parent_span_id": "211f1fe7-4cfb-4c1c-9c72-62ae2825dda3", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-16 08:47:57.668000000", - "end_time": "2025-10-16 08:47:57.668000000", - "duration_ms": 0.028, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What features does Noveum Trace offer?", - "query_length": 38, - "query_type": "routing_evaluation_query", - "classification.mode": "RAG", - "classification.rag_keyword_score": 3, - "classification.web_keyword_score": 0, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "noveum_specific", - "query.keyword_density": 0.5, - "query.domain_affinity": "noveum", - "routing.decision": "Routed to RAG based on analysis", - "routing.rationale": "RAG score: 3, Web score: 0, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "Web Search", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.057692307692307696, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: RAG (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "b8abebac-a7dc-494c-a1eb-6470181f32d2", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:47:57.667000000", - "trace_end_time": "2025-10-16 08:48:07.005000000", - "trace_duration_ms": 9337.405, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:47:57.667000000", - "updated_at": "2025-10-16 08:47:57.667000000" - }, - { - "span_id": "d2afaf74-f63b-4141-9fa3-662162f51cc0", - "trace_id": "b8abebac-a7dc-494c-a1eb-6470181f32d2", - "parent_span_id": "211f1fe7-4cfb-4c1c-9c72-62ae2825dda3", - "name": "agent.llm-rag", - "start_time": "2025-10-16 08:47:57.668000000", - "end_time": "2025-10-16 08:48:07.005000000", - "duration_ms": 9337.056, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "rag_agent", - "agent.operation": "llm-rag", - "agent.capabilities": [ - "document_retrieval", - "context_generation", - "response_generation" - ], - "agent.id": "noveum_rag_agent", - "input_query": "What features does Noveum Trace offer?", - "query_length": 38, - "query_type": "rag_query", - "output_response": "RAG Answer: Noveum Trace offers several key features through its Python SDK (`noveum-trace`) and TypeScript SDK (`@noveum/trace`):\n\n1. **Decorator-Based Tracing**: It provides a decorator-based approach for traci...", - "rag.context_length": 2787, - "rag.sources_count": 5, - "rag.answer_length": 1018, - "rag.mode": "retrieval_augmented_generation" - }, - "events": [], - "links": [], - "trace_trace_id": "b8abebac-a7dc-494c-a1eb-6470181f32d2", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:47:57.667000000", - "trace_end_time": "2025-10-16 08:48:07.005000000", - "trace_duration_ms": 9337.405, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:47:57.667000000", - "updated_at": "2025-10-16 08:47:57.667000000" - }, - { - "span_id": "09811a73-405d-4c0d-ba5d-0472b3708062", - "trace_id": "b8abebac-a7dc-494c-a1eb-6470181f32d2", - "parent_span_id": "d2afaf74-f63b-4141-9fa3-662162f51cc0", - "name": "agent.llm_model_execution", - "start_time": "2025-10-16 08:47:58.659000000", - "end_time": "2025-10-16 08:48:05.878000000", - "duration_ms": 7218.618, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What features does Noveum Trace offer?", - "query_length": 38, - "query_type": "rag_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 7.218573331832886, - "model.latency_ms": 7218.573331832886, - "model.start_time": 1760604478.659586, - "model.end_time": 1760604485.8781593, - "model.performance_tier": "slow", - "model.prompt_tokens": 830, - "model.completion_tokens": 254, - "model.total_tokens": 1084, - "model.tokens_per_second": 150.16817730724068, - "model.estimated_cost": 0.03252, - "model.efficiency_score": 0.9391143911439115, - "model.response_length": 1018, - "model.response_quality": "high", - "model.output_response": "Model Response: Noveum Trace offers several key features through its Python SDK (`noveum-trace`) and TypeScript SDK (`@noveum/trace`):\n\n1. **Decorator-Based Tracing**: It provides a decorator-based approach for traci...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "b8abebac-a7dc-494c-a1eb-6470181f32d2", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:47:57.667000000", - "trace_end_time": "2025-10-16 08:48:07.005000000", - "trace_duration_ms": 9337.405, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:47:57.667000000", - "updated_at": "2025-10-16 08:47:57.667000000" - }, - { - "span_id": "015688fe-d287-48e8-ac59-63781ffd6f89", - "trace_id": "b8abebac-a7dc-494c-a1eb-6470181f32d2", - "parent_span_id": "d2afaf74-f63b-4141-9fa3-662162f51cc0", - "name": "agent.rag_evaluation_metrics", - "start_time": "2025-10-16 08:48:05.878000000", - "end_time": "2025-10-16 08:48:05.878000000", - "duration_ms": 0.024, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "rag_evaluation_metrics", - "agent.capabilities": [ - "retrieval_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What features does Noveum Trace offer?", - "query_length": 38, - "query_type": "rag_evaluation_query", - "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\n\ud83d\udcd8 TypeScript SDK (\n@noveum/trace\n)- Framework ...", - "retrieval.context_length": 2787, - "retrieval.sources_count": 5, - "retrieval.context_quality": "high", - "retrieval.effectiveness": 1, - "retrieval.context_utilization": 2.787, - "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\n\ud83d\udcd8 TypeScript SDK (\n@noveum/trace\n)- Framework integrations for Next.js, Express.js, and Hono\n- TypeScript-first design with full type safety\n- Universal compatibility (Node.js, Edge Runtime, browsers)\n-\n\ud83d\udcca Noveum Platform\n- Real-time dashboard for analyzing traces and performance\n- Ad...\n\nSource 2 (https://noveum.ai/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\n\ud83d\udcd8 TypeScript SDK (\n@noveum/trace\n)- Framework integrations for Next.js, Express.js, and Hono\n- TypeScript-first design with full type safety\n- Universal compatibility (Node.js, Edge Runtime, browsers)\n-\n\ud83d\udcca Noveum Platform\n- Real-time dashboard for analyzing traces and performance\n- Ad...\n\nSource 3 (https://noveum.ai/en/docs/getting-started/sdk-integration):\n- \ud83d\udc65 Collaborate with your team on insights\n\ud83d\udd12 Security & Privacy\n- \ud83d\udd10 Encryption: All data encrypted in transit and at rest\n- \ud83c\udf9b\ufe0f Configurable Capture: Control what data is collected\n- \ud83c\udfe0 Data Residency: Choose your data storage region\n- \u23f0 Retention Control: Set custom data retention policies\nNext Steps\n- Tracing Concepts - Learn about traces, spans, and observability best practices\n- Framework Integrations - Deep dive into specific framework setups\n- Multi-Agent Tracing - Observe complex agent work...\n\nSource 4 (https://noveum.ai/docs/getting-started/sdk-integration):\n- \ud83d\udc65 Collaborate with your team on insights\n\ud83d\udd12 Security & Privacy\n- \ud83d\udd10 Encryption: All data encrypted in transit and at rest\n- \ud83c\udf9b\ufe0f Configurable Capture: Control what data is collected\n- \ud83c\udfe0 Data Residency: Choose your data storage region\n- \u23f0 Retention Control: Set custom data retention policies\nNext Steps\n- Tracing Concepts - Learn about traces, spans, and observability best practices\n- Framework Integrations - Deep dive into specific framework setups\n- Multi-Agent Tracing - Observe complex agent work...\n\nSource 5 (https://noveum.ai/docs/platform/dashboard):\n- Bulk Export: Download multiple traces simultaneously\n- Comparative Analysis: Compare performance across multiple traces\nReal-Time Updates\n- Live Refresh: Automatic updates as new traces arrive\n- Connection Monitoring: Real-time status of your trace ingestion pipeline\n- Performance Indicators: Live metrics for system health monitoring\nReady to dive deeper? Explore Projects & Environments to organize your AI applications, or check out Team Collaboration to share insights with your team.\nGet Earl...\n\nUser Question: What features does Noveum Trace offer?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", - "prompt.prompt_length": 3321, - "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frame...", - "prompt.instruction_following": "rag_optimized", - "response.answer_length": 1018, - "response.answer_completeness": "complete", - "response.response_quality": "high", - "response.source_citation": 5, - "response.context_utilization": 0.36526731252242556, - "output_response": "RAG Answer: Noveum Trace offers several key features through its Python SDK (`noveum-trace`) and TypeScript SDK (`@noveum/trace`):\n\n1. **Decorator-Based Tracing**: It provides a decorator-based approach for traci...", - "evaluation.retrieval_effectiveness": 1, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 5, - "evaluation.context_relevance": "high", - "evaluation.overall_quality": "high", - "evaluation.ready_for_production": true, - "rag.retrieval_strategy": "semantic_similarity", - "rag.vector_search_results": 5, - "rag.context_synthesis": "multi_source", - "rag.document_coverage": 1, - "rag.information_density": 0.36526731252242556 - }, - "events": [], - "links": [], - "trace_trace_id": "b8abebac-a7dc-494c-a1eb-6470181f32d2", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:47:57.667000000", - "trace_end_time": "2025-10-16 08:48:07.005000000", - "trace_duration_ms": 9337.405, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:47:57.667000000", - "updated_at": "2025-10-16 08:47:57.667000000" - }, - { - "span_id": "46ee4d00-f914-41d8-9f91-c2db85f1c613", - "trace_id": "be9ccbaf-ccb4-4ebd-a380-4247f4d01868", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-16 08:47:52.037000000", - "end_time": "2025-10-16 08:47:56.662000000", - "duration_ms": 4625.395, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What are Noveum's pricing plans?", - "query.length": 32, - "processing.mode": "RAG", - "processing.time_seconds": 4.6253437995910645, - "processing.response_length": 471, - "processing.sources_count": 5, - "output_response": "Final Answer: The provided context does not contain specific information about Noveum's pricing plans. It mentions that all users will have access to the Observability suite for free, and early users will receive f...", - "final_answer_mode": "RAG", - "query_processed.input_query": "What are Noveum's pricing plans?", - "query_processed.output_response": "Successfully processed query using RAG, generated 471 character response", - "query_processed.mode": "RAG", - "query_processed.processing_time": 4.6253437995910645, - "query_processed.response_length": 471 - }, - "events": [], - "links": [], - "trace_trace_id": "be9ccbaf-ccb4-4ebd-a380-4247f4d01868", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:47:52.037000000", - "trace_end_time": "2025-10-16 08:47:56.662000000", - "trace_duration_ms": 4625.492, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:47:52.037000000", - "updated_at": "2025-10-16 08:47:52.037000000" - }, - { - "span_id": "6d15369a-ae35-45c7-84dd-bdf484f73f2d", - "trace_id": "be9ccbaf-ccb4-4ebd-a380-4247f4d01868", - "parent_span_id": "46ee4d00-f914-41d8-9f91-c2db85f1c613", - "name": "agent.query_routing", - "start_time": "2025-10-16 08:47:52.037000000", - "end_time": "2025-10-16 08:47:56.662000000", - "duration_ms": 4625.286, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What are Noveum's pricing plans?", - "query_length": 32, - "query_type": "routing_query", - "output_response": "Routed to RAG for query processing", - "router.classification": "RAG", - "router.confidence_score": 1, - "router.rag_keyword_score": 3, - "router.web_keyword_score": 0, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "be9ccbaf-ccb4-4ebd-a380-4247f4d01868", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:47:52.037000000", - "trace_end_time": "2025-10-16 08:47:56.662000000", - "trace_duration_ms": 4625.492, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:47:52.037000000", - "updated_at": "2025-10-16 08:47:52.037000000" - }, - { - "span_id": "bf201676-a3a9-433a-a8c6-d394b7aead3a", - "trace_id": "be9ccbaf-ccb4-4ebd-a380-4247f4d01868", - "parent_span_id": "6d15369a-ae35-45c7-84dd-bdf484f73f2d", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-16 08:47:52.037000000", - "end_time": "2025-10-16 08:47:52.037000000", - "duration_ms": 0.048, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What are Noveum's pricing plans?", - "query_length": 32, - "query_type": "routing_evaluation_query", - "classification.mode": "RAG", - "classification.rag_keyword_score": 3, - "classification.web_keyword_score": 0, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "noveum_specific", - "query.keyword_density": 0.6, - "query.domain_affinity": "noveum", - "routing.decision": "Routed to RAG based on analysis", - "routing.rationale": "RAG score: 3, Web score: 0, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "Web Search", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.057692307692307696, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: RAG (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "be9ccbaf-ccb4-4ebd-a380-4247f4d01868", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:47:52.037000000", - "trace_end_time": "2025-10-16 08:47:56.662000000", - "trace_duration_ms": 4625.492, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:47:52.037000000", - "updated_at": "2025-10-16 08:47:52.037000000" - }, - { - "span_id": "f8346f60-131b-4007-ba3e-7b0ff10cdfde", - "trace_id": "be9ccbaf-ccb4-4ebd-a380-4247f4d01868", - "parent_span_id": "6d15369a-ae35-45c7-84dd-bdf484f73f2d", - "name": "agent.llm-rag", - "start_time": "2025-10-16 08:47:52.037000000", - "end_time": "2025-10-16 08:47:56.662000000", - "duration_ms": 4624.998, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "rag_agent", - "agent.operation": "llm-rag", - "agent.capabilities": [ - "document_retrieval", - "context_generation", - "response_generation" - ], - "agent.id": "noveum_rag_agent", - "input_query": "What are Noveum's pricing plans?", - "query_length": 32, - "query_type": "rag_query", - "output_response": "RAG Answer: The provided context does not contain specific information about Noveum's pricing plans. It mentions that all users will have access to the Observability suite for free, and early users will receive f...", - "rag.context_length": 2105, - "rag.sources_count": 5, - "rag.answer_length": 471, - "rag.mode": "retrieval_augmented_generation" - }, - "events": [], - "links": [], - "trace_trace_id": "be9ccbaf-ccb4-4ebd-a380-4247f4d01868", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:47:52.037000000", - "trace_end_time": "2025-10-16 08:47:56.662000000", - "trace_duration_ms": 4625.492, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:47:52.037000000", - "updated_at": "2025-10-16 08:47:52.037000000" - }, - { - "span_id": "0186c28c-7596-4279-a4eb-7733cd665609", - "trace_id": "be9ccbaf-ccb4-4ebd-a380-4247f4d01868", - "parent_span_id": "f8346f60-131b-4007-ba3e-7b0ff10cdfde", - "name": "agent.llm_model_execution", - "start_time": "2025-10-16 08:47:52.579000000", - "end_time": "2025-10-16 08:47:55.842000000", - "duration_ms": 3262.5829999999996, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What are Noveum's pricing plans?", - "query_length": 32, - "query_type": "rag_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 3.2625203132629395, - "model.latency_ms": 3262.5203132629395, - "model.start_time": 1760604472.580008, - "model.end_time": 1760604475.8425283, - "model.performance_tier": "medium", - "model.prompt_tokens": 658, - "model.completion_tokens": 117, - "model.total_tokens": 775, - "model.tokens_per_second": 237.54641368804243, - "model.estimated_cost": 0.02325, - "model.efficiency_score": 0.607741935483871, - "model.response_length": 471, - "model.response_quality": "high", - "model.output_response": "Model Response: The provided context does not contain specific information about Noveum's pricing plans. It mentions that all users will have access to the Observability suite for free, and early users will receive f...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "be9ccbaf-ccb4-4ebd-a380-4247f4d01868", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:47:52.037000000", - "trace_end_time": "2025-10-16 08:47:56.662000000", - "trace_duration_ms": 4625.492, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:47:52.037000000", - "updated_at": "2025-10-16 08:47:52.037000000" - }, - { - "span_id": "3bdf8d5b-c10d-44f3-8514-8facc7f63db8", - "trace_id": "be9ccbaf-ccb4-4ebd-a380-4247f4d01868", - "parent_span_id": "f8346f60-131b-4007-ba3e-7b0ff10cdfde", - "name": "agent.rag_evaluation_metrics", - "start_time": "2025-10-16 08:47:55.842000000", - "end_time": "2025-10-16 08:47:55.842000000", - "duration_ms": 0.032, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "rag_evaluation_metrics", - "agent.capabilities": [ - "retrieval_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What are Noveum's pricing plans?", - "query_length": 32, - "query_type": "rag_evaluation_query", - "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....", - "retrieval.context_length": 2105, - "retrieval.sources_count": 5, - "retrieval.context_quality": "high", - "retrieval.effectiveness": 1, - "retrieval.context_utilization": 2.105, - "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nSource 2 (https://noveum.ai/en/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nSource 3 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nThe AI landscape is complex and moving fast. With Noveum.ai, you'll have the insights you need to build, debug, and optimize AI applications with confidence.\nReady to see what's really happening inside your AI applications? Try Noveum.ai today and transform how you build and monitor AI systems.\nLet's build more reliable, observable AI\u2014together.\nGet Early Access to Noveum.ai Platform\nJoin the select group of AI teams optimizing their models with our data-driven platform. We're onboarding users in...\n\nSource 4 (https://noveum.ai/en/blog/from-logs-to-intelligent-choices-inside-noveum-ais-evaluation-process):\nGet Early Access to Noveum.ai Platform\nJoin the select group of AI teams optimizing their models with our data-driven platform. We're onboarding users in limited batches to ensure a premium experience....\n\nSource 5 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, vector searches, data retrieval, agent reasoning, and more. Without proper observability, debugging becomes a nightmare, optimization is guesswork, and understanding user interactions is nearly impossible.\nNoveum.ai solves this by providing comprehensive tracing and observability specifically designed for AI applications.\nKey benefits of using Noveum.ai:\n- Complete Visibility: Trace...\n\nUser Question: What are Noveum's pricing plans?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", - "prompt.prompt_length": 2633, - "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to ...", - "prompt.instruction_following": "rag_optimized", - "response.answer_length": 471, - "response.answer_completeness": "complete", - "response.response_quality": "high", - "response.source_citation": 5, - "response.context_utilization": 0.22375296912114015, - "output_response": "RAG Answer: The provided context does not contain specific information about Noveum's pricing plans. It mentions that all users will have access to the Observability suite for free, and early users will receive f...", - "evaluation.retrieval_effectiveness": 1, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 5, - "evaluation.context_relevance": "high", - "evaluation.overall_quality": "high", - "evaluation.ready_for_production": true, - "rag.retrieval_strategy": "semantic_similarity", - "rag.vector_search_results": 5, - "rag.context_synthesis": "multi_source", - "rag.document_coverage": 1, - "rag.information_density": 0.22375296912114015 - }, - "events": [], - "links": [], - "trace_trace_id": "be9ccbaf-ccb4-4ebd-a380-4247f4d01868", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:47:52.037000000", - "trace_end_time": "2025-10-16 08:47:56.662000000", - "trace_duration_ms": 4625.492, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:47:52.037000000", - "updated_at": "2025-10-16 08:47:52.037000000" - }, - { - "span_id": "770ce93d-07ae-4b9f-ae15-c6e450634b9d", - "trace_id": "8d33a529-d9b1-4208-b528-762222d5ef8c", - "parent_span_id": "9977d5d6-a7ff-469e-bec7-5e4dfc38c149", - "name": "agent.llm-rag", - "start_time": "2025-10-16 08:47:42.103000000", - "end_time": "2025-10-16 08:47:51.032000000", - "duration_ms": 8928.644999999999, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "rag_agent", - "agent.operation": "llm-rag", - "agent.capabilities": [ - "document_retrieval", - "context_generation", - "response_generation" - ], - "agent.id": "noveum_rag_agent", - "input_query": "How do I integrate Noveum Trace in my application?", - "query_length": 50, - "query_type": "rag_query", - "output_response": "RAG Answer: To integrate Noveum Trace into your application, you can use either the Python SDK (`noveum-trace`) or the TypeScript SDK (`@noveum/trace`). Here\u2019s a quick start guide based on the documentation:\n\n1. ...", - "rag.context_length": 2803, - "rag.sources_count": 5, - "rag.answer_length": 1134, - "rag.mode": "retrieval_augmented_generation" - }, - "events": [], - "links": [], - "trace_trace_id": "8d33a529-d9b1-4208-b528-762222d5ef8c", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:47:42.103000000", - "trace_end_time": "2025-10-16 08:47:51.032000000", - "trace_duration_ms": 8928.948, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:47:42.103000000", - "updated_at": "2025-10-16 08:47:42.103000000" - }, - { - "span_id": "9977d5d6-a7ff-469e-bec7-5e4dfc38c149", - "trace_id": "8d33a529-d9b1-4208-b528-762222d5ef8c", - "parent_span_id": "f3a8fa53-5641-4a23-b4d3-762577a89151", - "name": "agent.query_routing", - "start_time": "2025-10-16 08:47:42.103000000", - "end_time": "2025-10-16 08:47:51.032000000", - "duration_ms": 8928.793000000001, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "How do I integrate Noveum Trace in my application?", - "query_length": 50, - "query_type": "routing_query", - "output_response": "Routed to RAG for query processing", - "router.classification": "RAG", - "router.confidence_score": 1, - "router.rag_keyword_score": 2, - "router.web_keyword_score": 0, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "8d33a529-d9b1-4208-b528-762222d5ef8c", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:47:42.103000000", - "trace_end_time": "2025-10-16 08:47:51.032000000", - "trace_duration_ms": 8928.948, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:47:42.103000000", - "updated_at": "2025-10-16 08:47:42.103000000" - }, - { - "span_id": "c678e222-ef74-48ee-8628-16461ab0f48f", - "trace_id": "8d33a529-d9b1-4208-b528-762222d5ef8c", - "parent_span_id": "9977d5d6-a7ff-469e-bec7-5e4dfc38c149", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-16 08:47:42.103000000", - "end_time": "2025-10-16 08:47:42.103000000", - "duration_ms": 0.027, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: How do I integrate Noveum Trace in my application?", - "query_length": 50, - "query_type": "routing_evaluation_query", - "classification.mode": "RAG", - "classification.rag_keyword_score": 2, - "classification.web_keyword_score": 0, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "noveum_specific", - "query.keyword_density": 0.2222222222222222, - "query.domain_affinity": "noveum", - "routing.decision": "Routed to RAG based on analysis", - "routing.rationale": "RAG score: 2, Web score: 0, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "Web Search", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.038461538461538464, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: RAG (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "8d33a529-d9b1-4208-b528-762222d5ef8c", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:47:42.103000000", - "trace_end_time": "2025-10-16 08:47:51.032000000", - "trace_duration_ms": 8928.948, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:47:42.103000000", - "updated_at": "2025-10-16 08:47:42.103000000" - }, - { - "span_id": "f3a8fa53-5641-4a23-b4d3-762577a89151", - "trace_id": "8d33a529-d9b1-4208-b528-762222d5ef8c", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-16 08:47:42.103000000", - "end_time": "2025-10-16 08:47:51.032000000", - "duration_ms": 8928.868, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "How do I integrate Noveum Trace in my application?", - "query.length": 50, - "processing.mode": "RAG", - "processing.time_seconds": 8.928829431533813, - "processing.response_length": 1134, - "processing.sources_count": 5, - "output_response": "Final Answer: To integrate Noveum Trace into your application, you can use either the Python SDK (`noveum-trace`) or the TypeScript SDK (`@noveum/trace`). Here\u2019s a quick start guide based on the documentation:\n\n1. ...", - "final_answer_mode": "RAG", - "query_processed.input_query": "How do I integrate Noveum Trace in my application?", - "query_processed.output_response": "Successfully processed query using RAG, generated 1134 character response", - "query_processed.mode": "RAG", - "query_processed.processing_time": 8.928829431533813, - "query_processed.response_length": 1134 - }, - "events": [], - "links": [], - "trace_trace_id": "8d33a529-d9b1-4208-b528-762222d5ef8c", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:47:42.103000000", - "trace_end_time": "2025-10-16 08:47:51.032000000", - "trace_duration_ms": 8928.948, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:47:42.103000000", - "updated_at": "2025-10-16 08:47:42.103000000" - }, - { - "span_id": "f459b71f-1d43-41b0-b9bf-d94a908b7afa", - "trace_id": "8d33a529-d9b1-4208-b528-762222d5ef8c", - "parent_span_id": "770ce93d-07ae-4b9f-ae15-c6e450634b9d", - "name": "agent.llm_model_execution", - "start_time": "2025-10-16 08:47:43.044000000", - "end_time": "2025-10-16 08:47:50.008000000", - "duration_ms": 6963.959, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: How do I integrate Noveum Trace in my application?", - "query_length": 50, - "query_type": "rag_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 6.9639081954956055, - "model.latency_ms": 6963.9081954956055, - "model.start_time": 1760604463.044064, - "model.end_time": 1760604470.0079722, - "model.performance_tier": "slow", - "model.prompt_tokens": 837, - "model.completion_tokens": 283, - "model.total_tokens": 1120, - "model.tokens_per_second": 160.8292310235276, - "model.estimated_cost": 0.0336, - "model.efficiency_score": 1.0125, - "model.response_length": 1134, - "model.response_quality": "high", - "model.output_response": "Model Response: To integrate Noveum Trace into your application, you can use either the Python SDK (`noveum-trace`) or the TypeScript SDK (`@noveum/trace`). Here\u2019s a quick start guide based on the documentation:\n\n1. ...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "8d33a529-d9b1-4208-b528-762222d5ef8c", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:47:42.103000000", - "trace_end_time": "2025-10-16 08:47:51.032000000", - "trace_duration_ms": 8928.948, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:47:42.103000000", - "updated_at": "2025-10-16 08:47:42.103000000" - }, - { - "span_id": "b8627171-b081-4b08-b900-c24375b73c3f", - "trace_id": "8d33a529-d9b1-4208-b528-762222d5ef8c", - "parent_span_id": "770ce93d-07ae-4b9f-ae15-c6e450634b9d", - "name": "agent.rag_evaluation_metrics", - "start_time": "2025-10-16 08:47:50.008000000", - "end_time": "2025-10-16 08:47:50.008000000", - "duration_ms": 0.041, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "rag_evaluation_metrics", - "agent.capabilities": [ - "retrieval_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: How do I integrate Noveum Trace in my application?", - "query_length": 50, - "query_type": "rag_evaluation_query", - "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\n\ud83d\udcd8 TypeScript SDK (\n@noveum/trace\n)- Framework ...", - "retrieval.context_length": 2803, - "retrieval.sources_count": 5, - "retrieval.context_quality": "high", - "retrieval.effectiveness": 1, - "retrieval.context_utilization": 2.803, - "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\n\ud83d\udcd8 TypeScript SDK (\n@noveum/trace\n)- Framework integrations for Next.js, Express.js, and Hono\n- TypeScript-first design with full type safety\n- Universal compatibility (Node.js, Edge Runtime, browsers)\n-\n\ud83d\udcca Noveum Platform\n- Real-time dashboard for analyzing traces and performance\n- Ad...\n\nSource 2 (https://noveum.ai/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\n\ud83d\udcd8 TypeScript SDK (\n@noveum/trace\n)- Framework integrations for Next.js, Express.js, and Hono\n- TypeScript-first design with full type safety\n- Universal compatibility (Node.js, Edge Runtime, browsers)\n-\n\ud83d\udcca Noveum Platform\n- Real-time dashboard for analyzing traces and performance\n- Ad...\n\nSource 3 (https://noveum.ai/docs/getting-started/sdk-integration):\nSDK Integration Guide\nIntegrate Noveum.ai tracing into your AI applications with Python or TypeScript SDKs\nThe Noveum.ai SDKs provide comprehensive tracing and observability for your AI applications with minimal code changes. Whether you're building LLM applications, RAG systems, or multi-agent workflows, our SDKs automatically capture essential metrics and traces.\n\ud83d\ude80 Quick Start\n1. Create Your Account & Get API Key\n- Sign up at noveum.ai\n- Create a project in your dashboard\n- Generate an API key...\n\nSource 4 (https://noveum.ai/en/docs/getting-started/sdk-integration):\nSDK Integration Guide\nIntegrate Noveum.ai tracing into your AI applications with Python or TypeScript SDKs\nThe Noveum.ai SDKs provide comprehensive tracing and observability for your AI applications with minimal code changes. Whether you're building LLM applications, RAG systems, or multi-agent workflows, our SDKs automatically capture essential metrics and traces.\n\ud83d\ude80 Quick Start\n1. Create Your Account & Get API Key\n- Sign up at noveum.ai\n- Create a project in your dashboard\n- Generate an API key...\n\nSource 5 (https://noveum.ai/en/docs/getting-started/sdk-integration):\n\ud83d\udd27 Framework Integrations\nNext.js Integration\nExpress.js Integration\nFastAPI Integration (Python)\n\ud83d\udcca Advanced Features\nCustom Attributes & Events\nSampling Configuration\n\ud83d\udd17 What's Captured Automatically\n- \ud83d\udcca Performance Metrics: Latency, throughput, error rates\n- \ud83d\udcb0 Cost Tracking: Token usage, API costs across providers\n- \ud83d\udd0d Request/Response: Configurable capture of inputs/outputs\n- \ud83c\udff7\ufe0f Metadata: Model names, parameters, user context\n- \ud83c\udf0a Context Flow: Trace relationships across services\n- \ud83d\udc1b Error Detail...\n\nUser Question: How do I integrate Noveum Trace in my application?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", - "prompt.prompt_length": 3349, - "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frame...", - "prompt.instruction_following": "rag_optimized", - "response.answer_length": 1134, - "response.answer_completeness": "complete", - "response.response_quality": "high", - "response.source_citation": 5, - "response.context_utilization": 0.4045665358544417, - "output_response": "RAG Answer: To integrate Noveum Trace into your application, you can use either the Python SDK (`noveum-trace`) or the TypeScript SDK (`@noveum/trace`). Here\u2019s a quick start guide based on the documentation:\n\n1. ...", - "evaluation.retrieval_effectiveness": 1, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 5, - "evaluation.context_relevance": "high", - "evaluation.overall_quality": "high", - "evaluation.ready_for_production": true, - "rag.retrieval_strategy": "semantic_similarity", - "rag.vector_search_results": 5, - "rag.context_synthesis": "multi_source", - "rag.document_coverage": 1, - "rag.information_density": 0.4045665358544417 - }, - "events": [], - "links": [], - "trace_trace_id": "8d33a529-d9b1-4208-b528-762222d5ef8c", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:47:42.103000000", - "trace_end_time": "2025-10-16 08:47:51.032000000", - "trace_duration_ms": 8928.948, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:47:42.103000000", - "updated_at": "2025-10-16 08:47:42.103000000" - }, - { - "span_id": "0b3d6893-776e-4118-b8b5-b1c5ea51def1", - "trace_id": "06192a09-b6bb-4548-8a94-e123d6caaffe", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-16 08:47:03.776000000", - "end_time": "2025-10-16 08:47:41.100000000", - "duration_ms": 37324.516, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What is Noveum and what does it do?", - "query.length": 35, - "processing.mode": "RAG", - "processing.time_seconds": 37.324477195739746, - "processing.response_length": 648, - "processing.sources_count": 5, - "output_response": "Final Answer: Noveum.ai is a comprehensive tracing and observability platform specifically designed for AI applications, including those powered by large language models (LLMs), retrieval-augmented generation (RAG)...", - "final_answer_mode": "RAG", - "query_processed.input_query": "What is Noveum and what does it do?", - "query_processed.output_response": "Successfully processed query using RAG, generated 648 character response", - "query_processed.mode": "RAG", - "query_processed.processing_time": 37.324477195739746, - "query_processed.response_length": 648 - }, - "events": [], - "links": [], - "trace_trace_id": "06192a09-b6bb-4548-8a94-e123d6caaffe", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:47:03.776000000", - "trace_end_time": "2025-10-16 08:47:41.100000000", - "trace_duration_ms": 37324.601, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:47:03.776000000", - "updated_at": "2025-10-16 08:47:03.776000000" - }, - { - "span_id": "137ad05b-bada-42be-abe5-be0c277f6d0e", - "trace_id": "06192a09-b6bb-4548-8a94-e123d6caaffe", - "parent_span_id": "c3664d20-1708-45f5-a051-b626b0ec77cd", - "name": "agent.llm-rag", - "start_time": "2025-10-16 08:47:03.776000000", - "end_time": "2025-10-16 08:47:41.100000000", - "duration_ms": 37324.042, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "rag_agent", - "agent.operation": "llm-rag", - "agent.capabilities": [ - "document_retrieval", - "context_generation", - "response_generation" - ], - "agent.id": "noveum_rag_agent", - "input_query": "What is Noveum and what does it do?", - "query_length": 35, - "query_type": "rag_query", - "output_response": "RAG Answer: Noveum.ai is a comprehensive tracing and observability platform specifically designed for AI applications, including those powered by large language models (LLMs), retrieval-augmented generation (RAG)...", - "rag.context_length": 2334, - "rag.sources_count": 5, - "rag.answer_length": 648, - "rag.mode": "retrieval_augmented_generation" - }, - "events": [], - "links": [], - "trace_trace_id": "06192a09-b6bb-4548-8a94-e123d6caaffe", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:47:03.776000000", - "trace_end_time": "2025-10-16 08:47:41.100000000", - "trace_duration_ms": 37324.601, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:47:03.776000000", - "updated_at": "2025-10-16 08:47:03.776000000" - }, - { - "span_id": "c3664d20-1708-45f5-a051-b626b0ec77cd", - "trace_id": "06192a09-b6bb-4548-8a94-e123d6caaffe", - "parent_span_id": "0b3d6893-776e-4118-b8b5-b1c5ea51def1", - "name": "agent.query_routing", - "start_time": "2025-10-16 08:47:03.776000000", - "end_time": "2025-10-16 08:47:41.100000000", - "duration_ms": 37324.382999999994, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What is Noveum and what does it do?", - "query_length": 35, - "query_type": "routing_query", - "output_response": "Routed to RAG for query processing", - "router.classification": "RAG", - "router.confidence_score": 1, - "router.rag_keyword_score": 1, - "router.web_keyword_score": 0, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "06192a09-b6bb-4548-8a94-e123d6caaffe", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:47:03.776000000", - "trace_end_time": "2025-10-16 08:47:41.100000000", - "trace_duration_ms": 37324.601, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:47:03.776000000", - "updated_at": "2025-10-16 08:47:03.776000000" - }, - { - "span_id": "ce9a94bf-1a72-4599-8561-11e66802b92d", - "trace_id": "06192a09-b6bb-4548-8a94-e123d6caaffe", - "parent_span_id": "c3664d20-1708-45f5-a051-b626b0ec77cd", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-16 08:47:03.776000000", - "end_time": "2025-10-16 08:47:03.776000000", - "duration_ms": 0.067, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What is Noveum and what does it do?", - "query_length": 35, - "query_type": "routing_evaluation_query", - "classification.mode": "RAG", - "classification.rag_keyword_score": 1, - "classification.web_keyword_score": 0, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "noveum_specific", - "query.keyword_density": 0.125, - "query.domain_affinity": "noveum", - "routing.decision": "Routed to RAG based on analysis", - "routing.rationale": "RAG score: 1, Web score: 0, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "Web Search", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.019230769230769232, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: RAG (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "06192a09-b6bb-4548-8a94-e123d6caaffe", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:47:03.776000000", - "trace_end_time": "2025-10-16 08:47:41.100000000", - "trace_duration_ms": 37324.601, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:47:03.776000000", - "updated_at": "2025-10-16 08:47:03.776000000" - }, - { - "span_id": "38414416-907a-4188-95fe-c0a5f900f72c", - "trace_id": "06192a09-b6bb-4548-8a94-e123d6caaffe", - "parent_span_id": "137ad05b-bada-42be-abe5-be0c277f6d0e", - "name": "agent.llm_model_execution", - "start_time": "2025-10-16 08:47:10.392000000", - "end_time": "2025-10-16 08:47:40.392000000", - "duration_ms": 30000.104, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What is Noveum and what does it do?", - "query_length": 35, - "query_type": "rag_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 30.000043392181396, - "model.latency_ms": 30000.043392181396, - "model.start_time": 1760604430.3921378, - "model.end_time": 1760604460.3921812, - "model.performance_tier": "slow", - "model.prompt_tokens": 716, - "model.completion_tokens": 162, - "model.total_tokens": 878, - "model.tokens_per_second": 29.266624335244266, - "model.estimated_cost": 0.026340000000000002, - "model.efficiency_score": 0.7380410022779044, - "model.response_length": 648, - "model.response_quality": "high", - "model.output_response": "Model Response: Noveum.ai is a comprehensive tracing and observability platform specifically designed for AI applications, including those powered by large language models (LLMs), retrieval-augmented generation (RAG)...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "06192a09-b6bb-4548-8a94-e123d6caaffe", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:47:03.776000000", - "trace_end_time": "2025-10-16 08:47:41.100000000", - "trace_duration_ms": 37324.601, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:47:03.776000000", - "updated_at": "2025-10-16 08:47:03.776000000" - }, - { - "span_id": "67e23649-b00b-4eb8-9a69-865e80f833b3", - "trace_id": "06192a09-b6bb-4548-8a94-e123d6caaffe", - "parent_span_id": "137ad05b-bada-42be-abe5-be0c277f6d0e", - "name": "agent.rag_evaluation_metrics", - "start_time": "2025-10-16 08:47:40.392000000", - "end_time": "2025-10-16 08:47:40.392000000", - "duration_ms": 0.029, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "rag_evaluation_metrics", - "agent.capabilities": [ - "retrieval_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What is Noveum and what does it do?", - "query_length": 35, - "query_type": "rag_evaluation_query", - "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, vector searches, data retrieval, agent reasoning, and more. Without proper observability, debugging ...", - "retrieval.context_length": 2334, - "retrieval.sources_count": 5, - "retrieval.context_quality": "high", - "retrieval.effectiveness": 1, - "retrieval.context_utilization": 2.334, - "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, vector searches, data retrieval, agent reasoning, and more. Without proper observability, debugging becomes a nightmare, optimization is guesswork, and understanding user interactions is nearly impossible.\nNoveum.ai solves this by providing comprehensive tracing and observability specifically designed for AI applications.\nKey benefits of using Noveum.ai:\n- Complete Visibility: Trace...\n\nSource 2 (https://noveum.ai/en/docs/getting-started/overview):\nNoveum.ai Overview\nComprehensive AI tracing and observability platform for LLM applications, RAG systems, and AI agents\nWelcome to Noveum.ai\u2014the comprehensive tracing and observability platform built specifically for AI applications. Whether you're building LLM-powered chatbots, RAG systems, multi-agent workflows, or any AI-driven application, Noveum provides the insights you need to understand, debug, and optimize your systems.\n\ud83c\udfaf Why AI Applications Need Specialized Observability\nTraditional mo...\n\nSource 3 (https://noveum.ai/docs/getting-started/overview):\nNoveum.ai Overview\nComprehensive AI tracing and observability platform for LLM applications, RAG systems, and AI agents\nWelcome to Noveum.ai\u2014the comprehensive tracing and observability platform built specifically for AI applications. Whether you're building LLM-powered chatbots, RAG systems, multi-agent workflows, or any AI-driven application, Noveum provides the insights you need to understand, debug, and optimize your systems.\n\ud83c\udfaf Why AI Applications Need Specialized Observability\nTraditional mo...\n\nSource 4 (https://noveum.ai/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nSource 5 (https://noveum.ai/en/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nUser Question: What is Noveum and what does it do?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", - "prompt.prompt_length": 2865, - "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, ...", - "prompt.instruction_following": "rag_optimized", - "response.answer_length": 648, - "response.answer_completeness": "complete", - "response.response_quality": "high", - "response.source_citation": 5, - "response.context_utilization": 0.2776349614395887, - "output_response": "RAG Answer: Noveum.ai is a comprehensive tracing and observability platform specifically designed for AI applications, including those powered by large language models (LLMs), retrieval-augmented generation (RAG)...", - "evaluation.retrieval_effectiveness": 1, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 5, - "evaluation.context_relevance": "high", - "evaluation.overall_quality": "high", - "evaluation.ready_for_production": true, - "rag.retrieval_strategy": "semantic_similarity", - "rag.vector_search_results": 5, - "rag.context_synthesis": "multi_source", - "rag.document_coverage": 1, - "rag.information_density": 0.2776349614395887 - }, - "events": [], - "links": [], - "trace_trace_id": "06192a09-b6bb-4548-8a94-e123d6caaffe", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:47:03.776000000", - "trace_end_time": "2025-10-16 08:47:41.100000000", - "trace_duration_ms": 37324.601, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:47:03.776000000", - "updated_at": "2025-10-16 08:47:03.776000000" - }, - { - "span_id": "050851c9-b2d3-4c82-accb-4085dc69f785", - "trace_id": "3552ee8e-eafb-499a-b109-a9fb851a41d2", - "parent_span_id": "", - "name": "system_initialization", - "start_time": "2025-10-16 08:47:03.695000000", - "end_time": "2025-10-16 08:47:03.770000000", - "duration_ms": 74.58, - "status": "ok", - "status_message": "", - "attributes": { - "system.force_scrape": false, - "system.config": { - "noveum_base_url": "https://noveum.ai", - "max_pages_to_scrape": 50, - "chunk_size": 1000, - "chunk_overlap": 200, - "max_search_results": 5, - "rag_threshold": 0.7, - "noveum_docs_file": "noveum_docs.json", - "vector_store_path": "noveum_vectorstore" - }, - "input_query": "Initialize system with force_scrape=False", - "output_response": "System initialization: RAG system loaded, vector store ready, agent operational", - "system.initialized": true, - "system.documents_loaded": 38, - "system.vectorstore_ready": true - }, - "events": [], - "links": [], - "trace_trace_id": "3552ee8e-eafb-499a-b109-a9fb851a41d2", - "trace_name": "auto_trace_system_initialization", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:47:03.695000000", - "trace_end_time": "2025-10-16 08:47:03.770000000", - "trace_duration_ms": 74.912, - "span_count": 1, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:47:03.695000000", - "updated_at": "2025-10-16 08:47:03.695000000" - }, - { - "span_id": "6b53235d-8b91-4d9e-8437-915828e84600", - "trace_id": "e6ef2449-7429-465b-9bff-f3def8de5bb0", - "parent_span_id": "e73ab184-50c1-4209-9276-be11e69e9513", - "name": "agent.query_routing", - "start_time": "2025-10-14 12:54:06.700000000", - "end_time": "2025-10-14 12:54:10.637000000", - "duration_ms": 3937.492, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What are the latest breakthroughs in artificial intelligence?", - "query_length": 61, - "query_type": "routing_query", - "output_response": "Routed to Web Search for query processing", - "router.classification": "Web Search", - "router.confidence_score": 1, - "router.rag_keyword_score": 0, - "router.web_keyword_score": 1, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "e6ef2449-7429-465b-9bff-f3def8de5bb0", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-14 12:54:06.700000000", - "trace_end_time": "2025-10-14 12:54:10.637000000", - "trace_duration_ms": 3937.729, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-14 12:54:06.700000000", - "updated_at": "2025-10-14 12:54:06.700000000" - }, - { - "span_id": "a7d1e33a-95a1-41a9-a117-62b4c373bd60", - "trace_id": "e6ef2449-7429-465b-9bff-f3def8de5bb0", - "parent_span_id": "6b53235d-8b91-4d9e-8437-915828e84600", - "name": "agent.web_search_generation", - "start_time": "2025-10-14 12:54:06.700000000", - "end_time": "2025-10-14 12:54:10.637000000", - "duration_ms": 3936.983, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "web_search_agent", - "agent.operation": "web_search_generation", - "agent.capabilities": [ - "web_search", - "content_synthesis", - "response_generation" - ], - "agent.id": "noveum_web_search_agent", - "input_query": "What are the latest breakthroughs in artificial intelligence?", - "query_length": 61, - "query_type": "web_search_query", - "output_response": "Web Search Answer: The web search did not yield specific results regarding the latest breakthroughs in artificial intelligence. However, I can provide a general overview based on knowledge up to October 2023.\n\nRecent br...", - "web_search.results_count": 1, - "web_search.context_length": 163, - "web_search.response_length": 1090, - "web_search.mode": "external_web_search" - }, - "events": [], - "links": [], - "trace_trace_id": "e6ef2449-7429-465b-9bff-f3def8de5bb0", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-14 12:54:06.700000000", - "trace_end_time": "2025-10-14 12:54:10.637000000", - "trace_duration_ms": 3937.729, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-14 12:54:06.700000000", - "updated_at": "2025-10-14 12:54:06.700000000" - }, - { - "span_id": "da630cab-2ba6-4f12-98b9-6ab1c0899c17", - "trace_id": "e6ef2449-7429-465b-9bff-f3def8de5bb0", - "parent_span_id": "6b53235d-8b91-4d9e-8437-915828e84600", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-14 12:54:06.700000000", - "end_time": "2025-10-14 12:54:06.700000000", - "duration_ms": 0.27999999999999997, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What are the latest breakthroughs in artificial intelligence?", - "query_length": 61, - "query_type": "routing_evaluation_query", - "classification.mode": "Web Search", - "classification.rag_keyword_score": 0, - "classification.web_keyword_score": 1, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "complex", - "query.intent": "general_knowledge", - "query.keyword_density": 0.125, - "query.domain_affinity": "general", - "routing.decision": "Routed to Web Search based on analysis", - "routing.rationale": "RAG score: 0, Web score: 1, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "RAG", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.019230769230769232, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: Web Search (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "e6ef2449-7429-465b-9bff-f3def8de5bb0", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-14 12:54:06.700000000", - "trace_end_time": "2025-10-14 12:54:10.637000000", - "trace_duration_ms": 3937.729, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-14 12:54:06.700000000", - "updated_at": "2025-10-14 12:54:06.700000000" - }, - { - "span_id": "e73ab184-50c1-4209-9276-be11e69e9513", - "trace_id": "e6ef2449-7429-465b-9bff-f3def8de5bb0", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-14 12:54:06.700000000", - "end_time": "2025-10-14 12:54:10.637000000", - "duration_ms": 3937.663, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What are the latest breakthroughs in artificial intelligence?", - "query.length": 61, - "processing.mode": "Web Search", - "processing.time_seconds": 3.93752384185791, - "processing.response_length": 1090, - "processing.sources_count": 1, - "output_response": "Final Answer: The web search did not yield specific results regarding the latest breakthroughs in artificial intelligence. However, I can provide a general overview based on knowledge up to October 2023.\n\nRecent br...", - "final_answer_mode": "Web Search", - "query_processed.input_query": "What are the latest breakthroughs in artificial intelligence?", - "query_processed.output_response": "Successfully processed query using Web Search, generated 1090 character response", - "query_processed.mode": "Web Search", - "query_processed.processing_time": 3.93752384185791, - "query_processed.response_length": 1090 - }, - "events": [], - "links": [], - "trace_trace_id": "e6ef2449-7429-465b-9bff-f3def8de5bb0", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-14 12:54:06.700000000", - "trace_end_time": "2025-10-14 12:54:10.637000000", - "trace_duration_ms": 3937.729, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-14 12:54:06.700000000", - "updated_at": "2025-10-14 12:54:06.700000000" - }, - { - "span_id": "bb47766c-afda-44fe-8838-7936c80cd4b6", - "trace_id": "e6ef2449-7429-465b-9bff-f3def8de5bb0", - "parent_span_id": "a7d1e33a-95a1-41a9-a117-62b4c373bd60", - "name": "agent.llm_model_execution", - "start_time": "2025-10-14 12:54:07.104000000", - "end_time": "2025-10-14 12:54:10.637000000", - "duration_ms": 3533.321, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What are the latest breakthroughs in artificial intelligence?", - "query_length": 61, - "query_type": "web_search_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 3.533268928527832, - "model.latency_ms": 3533.268928527832, - "model.start_time": 1760446447.104266, - "model.end_time": 1760446450.6375349, - "model.performance_tier": "medium", - "model.prompt_tokens": 167, - "model.completion_tokens": 272, - "model.total_tokens": 439, - "model.tokens_per_second": 124.2475477752307, - "model.estimated_cost": 0.013170000000000001, - "model.efficiency_score": 2.4829157175398633, - "model.response_length": 1090, - "model.response_quality": "high", - "model.output_response": "Model Response: The web search did not yield specific results regarding the latest breakthroughs in artificial intelligence. However, I can provide a general overview based on knowledge up to October 2023.\n\nRecent br...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "e6ef2449-7429-465b-9bff-f3def8de5bb0", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-14 12:54:06.700000000", - "trace_end_time": "2025-10-14 12:54:10.637000000", - "trace_duration_ms": 3937.729, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-14 12:54:06.700000000", - "updated_at": "2025-10-14 12:54:06.700000000" - }, - { - "span_id": "1d914a9d-b770-45dc-8308-6a36e9e0db89", - "trace_id": "e6ef2449-7429-465b-9bff-f3def8de5bb0", - "parent_span_id": "a7d1e33a-95a1-41a9-a117-62b4c373bd60", - "name": "agent.web_search_evaluation_metrics", - "start_time": "2025-10-14 12:54:10.637000000", - "end_time": "2025-10-14 12:54:10.637000000", - "duration_ms": 0.022, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "web_search_evaluation_metrics", - "agent.capabilities": [ - "web_search_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What are the latest breakthroughs in artificial intelligence?", - "query_length": 61, - "query_type": "web_search_evaluation_query", - "web_search.results_count": 1, - "web_search.context_length": 163, - "web_search.context_synthesized": "Context from 1 web sources", - "web_search.search_effectiveness": 0.2, - "web_search.context_quality": "low", - "web_search.source_diversity": 1, - "web_search.information_synthesis": "low", - "web_search.external_knowledge_utilization": 0.163, - "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What+are+the+latest+breakthroughs+in+artificial+intelligence?\n\nUser Question: What are the latest breakthroughs in artificial intelligence?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", - "prompt.prompt_length": 669, - "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What+are+the+latest+breakthroughs+in+artificial+intelligence?", - "prompt.instruction_following": "web_search_optimized", - "response.answer_length": 1090, - "response.answer_completeness": "complete", - "response.response_quality": "medium", - "response.source_citation": 1, - "response.context_utilization": 6.6871165644171775, - "output_response": "Web Search Answer: The web search did not yield specific results regarding the latest breakthroughs in artificial intelligence. However, I can provide a general overview based on knowledge up to October 2023.\n\nRecent br...", - "evaluation.search_effectiveness": 0.2, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 1, - "evaluation.context_relevance": "low", - "evaluation.overall_quality": "low", - "evaluation.ready_for_production": true, - "web_search.search_strategy": "duckduckgo_api", - "web_search.real_time_data": true, - "web_search.external_sources": 1, - "web_search.information_freshness": "current", - "web_search.knowledge_synthesis": "single_source" - }, - "events": [], - "links": [], - "trace_trace_id": "e6ef2449-7429-465b-9bff-f3def8de5bb0", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-14 12:54:06.700000000", - "trace_end_time": "2025-10-14 12:54:10.637000000", - "trace_duration_ms": 3937.729, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-14 12:54:06.700000000", - "updated_at": "2025-10-14 12:54:06.700000000" - }, - { - "span_id": "91f3999c-2edf-4c56-8f4d-ab43b6ebbaee", - "trace_id": "2ba8cab0-2366-4add-8ac3-b490a05e9a60", - "parent_span_id": "cbe51252-a323-4852-a90e-c907a0480fa3", - "name": "agent.web_search_generation", - "start_time": "2025-10-14 12:53:56.480000000", - "end_time": "2025-10-14 12:54:05.685000000", - "duration_ms": 9205.212, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "web_search_agent", - "agent.operation": "web_search_generation", - "agent.capabilities": [ - "web_search", - "content_synthesis", - "response_generation" - ], - "agent.id": "noveum_web_search_agent", - "input_query": "What's happening in the software development world today?", - "query_length": 57, - "query_type": "web_search_query", - "output_response": "Web Search Answer: The software development world is currently experiencing rapid changes driven by advancements in artificial intelligence (AI) and evolving programming practices. Key trends include:\n\n1. **AI Integrati...", - "web_search.results_count": 1, - "web_search.context_length": 975, - "web_search.response_length": 1636, - "web_search.mode": "external_web_search" - }, - "events": [], - "links": [], - "trace_trace_id": "2ba8cab0-2366-4add-8ac3-b490a05e9a60", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-14 12:53:56.480000000", - "trace_end_time": "2025-10-14 12:54:05.687000000", - "trace_duration_ms": 9207.67, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-14 12:53:56.480000000", - "updated_at": "2025-10-14 12:53:56.480000000" - }, - { - "span_id": "ea2aee75-4537-46ea-954b-2002fc0026c5", - "trace_id": "2ba8cab0-2366-4add-8ac3-b490a05e9a60", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-14 12:53:56.480000000", - "end_time": "2025-10-14 12:54:05.687000000", - "duration_ms": 9207.528999999999, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What's happening in the software development world today?", - "query.length": 57, - "processing.mode": "Web Search", - "processing.time_seconds": 9.206756830215454, - "processing.response_length": 1636, - "processing.sources_count": 1, - "output_response": "Final Answer: The software development world is currently experiencing rapid changes driven by advancements in artificial intelligence (AI) and evolving programming practices. Key trends include:\n\n1. **AI Integrati...", - "final_answer_mode": "Web Search", - "query_processed.input_query": "What's happening in the software development world today?", - "query_processed.output_response": "Successfully processed query using Web Search, generated 1636 character response", - "query_processed.mode": "Web Search", - "query_processed.processing_time": 9.206756830215454, - "query_processed.response_length": 1636 - }, - "events": [], - "links": [], - "trace_trace_id": "2ba8cab0-2366-4add-8ac3-b490a05e9a60", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-14 12:53:56.480000000", - "trace_end_time": "2025-10-14 12:54:05.687000000", - "trace_duration_ms": 9207.67, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-14 12:53:56.480000000", - "updated_at": "2025-10-14 12:53:56.480000000" - }, - { - "span_id": "cbe51252-a323-4852-a90e-c907a0480fa3", - "trace_id": "2ba8cab0-2366-4add-8ac3-b490a05e9a60", - "parent_span_id": "ea2aee75-4537-46ea-954b-2002fc0026c5", - "name": "agent.query_routing", - "start_time": "2025-10-14 12:53:56.480000000", - "end_time": "2025-10-14 12:54:05.686000000", - "duration_ms": 9205.569000000001, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What's happening in the software development world today?", - "query_length": 57, - "query_type": "routing_query", - "output_response": "Routed to Web Search for query processing", - "router.classification": "Web Search", - "router.confidence_score": 1, - "router.rag_keyword_score": 0, - "router.web_keyword_score": 1, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "2ba8cab0-2366-4add-8ac3-b490a05e9a60", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-14 12:53:56.480000000", - "trace_end_time": "2025-10-14 12:54:05.687000000", - "trace_duration_ms": 9207.67, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-14 12:53:56.480000000", - "updated_at": "2025-10-14 12:53:56.480000000" - }, - { - "span_id": "d850ecee-86ef-4818-9cec-638bfee26293", - "trace_id": "2ba8cab0-2366-4add-8ac3-b490a05e9a60", - "parent_span_id": "cbe51252-a323-4852-a90e-c907a0480fa3", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-14 12:53:56.480000000", - "end_time": "2025-10-14 12:53:56.480000000", - "duration_ms": 0.041999999999999996, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What's happening in the software development world today?", - "query_length": 57, - "query_type": "routing_evaluation_query", - "classification.mode": "Web Search", - "classification.rag_keyword_score": 0, - "classification.web_keyword_score": 1, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "complex", - "query.intent": "general_knowledge", - "query.keyword_density": 0.125, - "query.domain_affinity": "general", - "routing.decision": "Routed to Web Search based on analysis", - "routing.rationale": "RAG score: 0, Web score: 1, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "RAG", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.019230769230769232, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: Web Search (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "2ba8cab0-2366-4add-8ac3-b490a05e9a60", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-14 12:53:56.480000000", - "trace_end_time": "2025-10-14 12:54:05.687000000", - "trace_duration_ms": 9207.67, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-14 12:53:56.480000000", - "updated_at": "2025-10-14 12:53:56.480000000" - }, - { - "span_id": "0d4387a4-adf3-42c1-aca7-51453a90dd36", - "trace_id": "2ba8cab0-2366-4add-8ac3-b490a05e9a60", - "parent_span_id": "91f3999c-2edf-4c56-8f4d-ab43b6ebbaee", - "name": "agent.llm_model_execution", - "start_time": "2025-10-14 12:53:59.426000000", - "end_time": "2025-10-14 12:54:05.684000000", - "duration_ms": 6258.643, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What's happening in the software development world today?", - "query_length": 57, - "query_type": "web_search_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 6.25848388671875, - "model.latency_ms": 6258.48388671875, - "model.start_time": 1760446439.426367, - "model.end_time": 1760446445.684851, - "model.performance_tier": "slow", - "model.prompt_tokens": 369, - "model.completion_tokens": 409, - "model.total_tokens": 778, - "model.tokens_per_second": 124.31125718019486, - "model.estimated_cost": 0.02334, - "model.efficiency_score": 2.102827763496144, - "model.response_length": 1636, - "model.response_quality": "high", - "model.output_response": "Model Response: The software development world is currently experiencing rapid changes driven by advancements in artificial intelligence (AI) and evolving programming practices. Key trends include:\n\n1. **AI Integrati...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "2ba8cab0-2366-4add-8ac3-b490a05e9a60", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-14 12:53:56.480000000", - "trace_end_time": "2025-10-14 12:54:05.687000000", - "trace_duration_ms": 9207.67, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-14 12:53:56.480000000", - "updated_at": "2025-10-14 12:53:56.480000000" - }, - { - "span_id": "717e3c4a-6300-45e5-9456-caa3e773d758", - "trace_id": "2ba8cab0-2366-4add-8ac3-b490a05e9a60", - "parent_span_id": "91f3999c-2edf-4c56-8f4d-ab43b6ebbaee", - "name": "agent.web_search_evaluation_metrics", - "start_time": "2025-10-14 12:54:05.685000000", - "end_time": "2025-10-14 12:54:05.685000000", - "duration_ms": 0.044, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "web_search_evaluation_metrics", - "agent.capabilities": [ - "web_search_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What's happening in the software development world today?", - "query_length": 57, - "query_type": "web_search_evaluation_query", - "web_search.results_count": 1, - "web_search.context_length": 975, - "web_search.context_synthesized": "Context from 1 web sources", - "web_search.search_effectiveness": 0.2, - "web_search.context_quality": "high", - "web_search.source_diversity": 1, - "web_search.information_synthesis": "low", - "web_search.external_knowledge_utilization": 0.975, - "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nLog in to WhatsApp Web for simple, reliable and private messaging on your desktop. Send and receive messages and files with ease, all for free. Sep 23, 2025 \u00b7 Chief Economists' Outlook 2025 states that these under-the-radar trends are shaping the global economy, from the impact of AI on research productivity to a tightened labor \u2026 Sep 1, 2025 \u00b7 THE AI DEVELOPMENT REVOLUTION: WHAT'S REALLY HAPPENING? Let's face it - the software development world is changing faster than a React component re-rendering on \u2026 Oct 7, 2025 \u00b7 Computer Programming Research. Read current computer science articles on everything from computer programs to detect cancer genes and control vehicle maintenance to \u2026 Aug 24, 2025 \u00b7 Artificial Intelligence News. Everything on AI including futuristic robots with artificial intelligence, computer models of human intelligence and more.\nURL: https://duckduckgo.com/?q=What's+happening+in+the+software+development+world+today?\n\nUser Question: What's happening in the software development world today?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", - "prompt.prompt_length": 1477, - "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nLog in to WhatsApp Web for simple, reliable and private messaging on your desktop. Send and receive messages and files with ease, all for free. Sep 23, 2025 \u00b7 Chief Economi...", - "prompt.instruction_following": "web_search_optimized", - "response.answer_length": 1636, - "response.answer_completeness": "complete", - "response.response_quality": "medium", - "response.source_citation": 1, - "response.context_utilization": 1.677948717948718, - "output_response": "Web Search Answer: The software development world is currently experiencing rapid changes driven by advancements in artificial intelligence (AI) and evolving programming practices. Key trends include:\n\n1. **AI Integrati...", - "evaluation.search_effectiveness": 0.2, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 1, - "evaluation.context_relevance": "high", - "evaluation.overall_quality": "low", - "evaluation.ready_for_production": true, - "web_search.search_strategy": "duckduckgo_api", - "web_search.real_time_data": true, - "web_search.external_sources": 1, - "web_search.information_freshness": "current", - "web_search.knowledge_synthesis": "single_source" - }, - "events": [], - "links": [], - "trace_trace_id": "2ba8cab0-2366-4add-8ac3-b490a05e9a60", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-14 12:53:56.480000000", - "trace_end_time": "2025-10-14 12:54:05.687000000", - "trace_duration_ms": 9207.67, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-14 12:53:56.480000000", - "updated_at": "2025-10-14 12:53:56.480000000" - }, - { - "span_id": "3d39f522-8ae7-4988-aafd-58b09734eff0", - "trace_id": "a7c3399e-372f-4aae-baeb-05215cc4b85f", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-14 12:53:50.963000000", - "end_time": "2025-10-14 12:53:55.471000000", - "duration_ms": 4507.466, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What are the newest features in cloud computing?", - "query.length": 48, - "processing.mode": "RAG", - "processing.time_seconds": 4.507421255111694, - "processing.response_length": 870, - "processing.sources_count": 5, - "output_response": "Final Answer: The provided context does not specifically mention any new features in cloud computing as a general topic. However, it does highlight several recent enhancements related to the Noveum.ai platform, whi...", - "final_answer_mode": "RAG", - "query_processed.input_query": "What are the newest features in cloud computing?", - "query_processed.output_response": "Successfully processed query using RAG, generated 870 character response", - "query_processed.mode": "RAG", - "query_processed.processing_time": 4.507421255111694, - "query_processed.response_length": 870 - }, - "events": [], - "links": [], - "trace_trace_id": "a7c3399e-372f-4aae-baeb-05215cc4b85f", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-14 12:53:50.963000000", - "trace_end_time": "2025-10-14 12:53:55.471000000", - "trace_duration_ms": 4507.555, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-14 12:53:50.963000000", - "updated_at": "2025-10-14 12:53:50.963000000" - }, - { - "span_id": "54aa6700-f2de-4cf0-b1ad-a3d24ed692ed", - "trace_id": "a7c3399e-372f-4aae-baeb-05215cc4b85f", - "parent_span_id": "d17f31cf-c570-4604-8df3-532b2e530fce", - "name": "agent.llm-rag", - "start_time": "2025-10-14 12:53:50.963000000", - "end_time": "2025-10-14 12:53:55.471000000", - "duration_ms": 4507.083, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "rag_agent", - "agent.operation": "llm-rag", - "agent.capabilities": [ - "document_retrieval", - "context_generation", - "response_generation" - ], - "agent.id": "noveum_rag_agent", - "input_query": "What are the newest features in cloud computing?", - "query_length": 48, - "query_type": "rag_query", - "output_response": "RAG Answer: The provided context does not specifically mention any new features in cloud computing as a general topic. However, it does highlight several recent enhancements related to the Noveum.ai platform, whi...", - "rag.context_length": 2579, - "rag.sources_count": 5, - "rag.answer_length": 870, - "rag.mode": "retrieval_augmented_generation" - }, - "events": [], - "links": [], - "trace_trace_id": "a7c3399e-372f-4aae-baeb-05215cc4b85f", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-14 12:53:50.963000000", - "trace_end_time": "2025-10-14 12:53:55.471000000", - "trace_duration_ms": 4507.555, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-14 12:53:50.963000000", - "updated_at": "2025-10-14 12:53:50.963000000" - }, - { - "span_id": "8878096f-6cfa-4f70-9844-d7723a0875f0", - "trace_id": "a7c3399e-372f-4aae-baeb-05215cc4b85f", - "parent_span_id": "d17f31cf-c570-4604-8df3-532b2e530fce", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-14 12:53:50.963000000", - "end_time": "2025-10-14 12:53:50.963000000", - "duration_ms": 0.036000000000000004, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What are the newest features in cloud computing?", - "query_length": 48, - "query_type": "routing_evaluation_query", - "classification.mode": "RAG", - "classification.rag_keyword_score": 1, - "classification.web_keyword_score": 0, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "documentation", - "query.keyword_density": 0.125, - "query.domain_affinity": "noveum", - "routing.decision": "Routed to RAG based on analysis", - "routing.rationale": "RAG score: 1, Web score: 0, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "Web Search", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.019230769230769232, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: RAG (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "a7c3399e-372f-4aae-baeb-05215cc4b85f", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-14 12:53:50.963000000", - "trace_end_time": "2025-10-14 12:53:55.471000000", - "trace_duration_ms": 4507.555, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-14 12:53:50.963000000", - "updated_at": "2025-10-14 12:53:50.963000000" - }, - { - "span_id": "d17f31cf-c570-4604-8df3-532b2e530fce", - "trace_id": "a7c3399e-372f-4aae-baeb-05215cc4b85f", - "parent_span_id": "3d39f522-8ae7-4988-aafd-58b09734eff0", - "name": "agent.query_routing", - "start_time": "2025-10-14 12:53:50.963000000", - "end_time": "2025-10-14 12:53:55.471000000", - "duration_ms": 4507.3730000000005, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What are the newest features in cloud computing?", - "query_length": 48, - "query_type": "routing_query", - "output_response": "Routed to RAG for query processing", - "router.classification": "RAG", - "router.confidence_score": 1, - "router.rag_keyword_score": 1, - "router.web_keyword_score": 0, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "a7c3399e-372f-4aae-baeb-05215cc4b85f", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-14 12:53:50.963000000", - "trace_end_time": "2025-10-14 12:53:55.471000000", - "trace_duration_ms": 4507.555, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-14 12:53:50.963000000", - "updated_at": "2025-10-14 12:53:50.963000000" - }, - { - "span_id": "97ef01a8-5930-4902-909a-245c516a5871", - "trace_id": "a7c3399e-372f-4aae-baeb-05215cc4b85f", - "parent_span_id": "54aa6700-f2de-4cf0-b1ad-a3d24ed692ed", - "name": "agent.llm_model_execution", - "start_time": "2025-10-14 12:53:51.439000000", - "end_time": "2025-10-14 12:53:55.077000000", - "duration_ms": 3637.337, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What are the newest features in cloud computing?", - "query_length": 48, - "query_type": "rag_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 3.6372909545898438, - "model.latency_ms": 3637.2909545898438, - "model.start_time": 1760446431.439748, - "model.end_time": 1760446435.077039, - "model.performance_tier": "medium", - "model.prompt_tokens": 780, - "model.completion_tokens": 217, - "model.total_tokens": 997, - "model.tokens_per_second": 274.1050997698989, - "model.estimated_cost": 0.02991, - "model.efficiency_score": 0.872617853560682, - "model.response_length": 870, - "model.response_quality": "high", - "model.output_response": "Model Response: The provided context does not specifically mention any new features in cloud computing as a general topic. However, it does highlight several recent enhancements related to the Noveum.ai platform, whi...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "a7c3399e-372f-4aae-baeb-05215cc4b85f", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-14 12:53:50.963000000", - "trace_end_time": "2025-10-14 12:53:55.471000000", - "trace_duration_ms": 4507.555, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-14 12:53:50.963000000", - "updated_at": "2025-10-14 12:53:50.963000000" - }, - { - "span_id": "ea792ec6-64b4-4ed5-b418-ba902e90c57a", - "trace_id": "a7c3399e-372f-4aae-baeb-05215cc4b85f", - "parent_span_id": "54aa6700-f2de-4cf0-b1ad-a3d24ed692ed", - "name": "agent.rag_evaluation_metrics", - "start_time": "2025-10-14 12:53:55.077000000", - "end_time": "2025-10-14 12:53:55.077000000", - "duration_ms": 0.034, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "rag_evaluation_metrics", - "agent.capabilities": [ - "retrieval_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What are the newest features in cloud computing?", - "query_length": 48, - "query_type": "rag_evaluation_query", - "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/changelog):\n- \ud83d\udcc8 Enhanced Dashboard Analytics: Real-time request tracking, latency monitoring, and cost analysis\n- \ud83d\udd0d Improved Logs Interface: Better search functionality, detailed trace views, and enhanced debugging capabilities\n- \ud83d\udc65 Team Management: Advanced member mana...", - "retrieval.context_length": 2579, - "retrieval.sources_count": 5, - "retrieval.context_quality": "high", - "retrieval.effectiveness": 1, - "retrieval.context_utilization": 2.579, - "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/changelog):\n- \ud83d\udcc8 Enhanced Dashboard Analytics: Real-time request tracking, latency monitoring, and cost analysis\n- \ud83d\udd0d Improved Logs Interface: Better search functionality, detailed trace views, and enhanced debugging capabilities\n- \ud83d\udc65 Team Management: Advanced member management with role-based access control and invitation system\n- \ud83d\udee1\ufe0f Security Improvements: Enhanced API key management and secure credential storage\n4 months ago\n- \ud83d\udcca Advanced Metrics Collection: Comprehensive telemetry system with custom metrics ...\n\nSource 2 (https://noveum.ai/en/changelog):\n- \ud83d\udcdd Documentation Updates: Comprehensive guides for integration and best practices\n5 months ago\n- \ud83c\udf89 Platform Foundation: Initial release of Noveum.ai AI observability platform\n- \ud83d\ude80 Core Features: AI Gateway integration, basic evaluation metrics, and monitoring capabilities\n- \ud83d\udcc8 Evaluation Jobs: Automated model evaluation with configurable metrics and reporting\n- \ud83d\udd0c Provider Integrations: Support for major AI providers and custom model deployments...\n\nSource 3 (https://noveum.ai/docs/platform/dashboard):\n- Bulk Export: Download multiple traces simultaneously\n- Comparative Analysis: Compare performance across multiple traces\nReal-Time Updates\n- Live Refresh: Automatic updates as new traces arrive\n- Connection Monitoring: Real-time status of your trace ingestion pipeline\n- Performance Indicators: Live metrics for system health monitoring\nReady to dive deeper? Explore Projects & Environments to organize your AI applications, or check out Team Collaboration to share insights with your team.\nGet Earl...\n\nSource 4 (https://noveum.ai/en/changelog):\nChangelog\nStay up to date with the latest changes in our product.\n2 months ago\n- \ud83c\udfaf Major Platform Evolution: NovaEval Framework - Launched comprehensive AI model evaluation framework with 20+ built-in scorers including accuracy, RAG metrics, conversational scoring, and specialized agent evaluation tools. Features production-grade deployment on Docker and Kubernetes. (GitHub | PyPI Package)\n- \ud83d\ude80 Next-Generation Traces UI - Complete redesign featuring three-pane layout with directory tree navigatio...\n\nSource 5 (https://noveum.ai/en/blog):\n8/12/2025\nDiscover how Noveum.ai provides comprehensive tracing and observability for AI applications, from development debugging to production optimization.\nShashank Agarwal\n3/3/2025\nDiscover how Noveum.ai provides comprehensive tracing and observability for LLM applications, RAG systems, and multi-agent workflows with our powerful Python and TypeScript SDKs.\nShashank Agarwal\n3/2/2025...\n\nUser Question: What are the newest features in cloud computing?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", - "prompt.prompt_length": 3123, - "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/changelog):\n- \ud83d\udcc8 Enhanced Dashboard Analytics: Real-time request tracking, latency monitoring, and cost analysis\n- \ud83d\udd0d Improved Logs Interface: Better search functionality,...", - "prompt.instruction_following": "rag_optimized", - "response.answer_length": 870, - "response.answer_completeness": "complete", - "response.response_quality": "high", - "response.source_citation": 5, - "response.context_utilization": 0.33734005428460645, - "output_response": "RAG Answer: The provided context does not specifically mention any new features in cloud computing as a general topic. However, it does highlight several recent enhancements related to the Noveum.ai platform, whi...", - "evaluation.retrieval_effectiveness": 1, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 5, - "evaluation.context_relevance": "high", - "evaluation.overall_quality": "high", - "evaluation.ready_for_production": true, - "rag.retrieval_strategy": "semantic_similarity", - "rag.vector_search_results": 5, - "rag.context_synthesis": "multi_source", - "rag.document_coverage": 1, - "rag.information_density": 0.33734005428460645 - }, - "events": [], - "links": [], - "trace_trace_id": "a7c3399e-372f-4aae-baeb-05215cc4b85f", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-14 12:53:50.963000000", - "trace_end_time": "2025-10-14 12:53:55.471000000", - "trace_duration_ms": 4507.555, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-14 12:53:50.963000000", - "updated_at": "2025-10-14 12:53:50.963000000" - }, - { - "span_id": "e2d314c8-687f-4b8d-869c-8d7f2a22a4f3", - "trace_id": "d682797f-2ce0-448e-8f0d-891541430988", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-14 12:53:44.917000000", - "end_time": "2025-10-14 12:53:49.957000000", - "duration_ms": 5039.565, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What's the current status of cryptocurrency markets?", - "query.length": 52, - "processing.mode": "Web Search", - "processing.time_seconds": 5.0394370555877686, - "processing.response_length": 338, - "processing.sources_count": 1, - "output_response": "Final Answer: The web search results did not provide any specific information regarding the current status of cryptocurrency markets. Therefore, I cannot provide an update on market conditions, prices, or trends at...", - "final_answer_mode": "Web Search", - "query_processed.input_query": "What's the current status of cryptocurrency markets?", - "query_processed.output_response": "Successfully processed query using Web Search, generated 338 character response", - "query_processed.mode": "Web Search", - "query_processed.processing_time": 5.0394370555877686, - "query_processed.response_length": 338 - }, - "events": [], - "links": [], - "trace_trace_id": "d682797f-2ce0-448e-8f0d-891541430988", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-14 12:53:44.917000000", - "trace_end_time": "2025-10-14 12:53:49.957000000", - "trace_duration_ms": 5039.64, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-14 12:53:44.917000000", - "updated_at": "2025-10-14 12:53:44.917000000" - }, - { - "span_id": "d367a6bd-86af-41c5-96ca-5284425775ce", - "trace_id": "d682797f-2ce0-448e-8f0d-891541430988", - "parent_span_id": "f137a412-6ddf-4ace-908e-9edf73534aa7", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-14 12:53:44.917000000", - "end_time": "2025-10-14 12:53:44.918000000", - "duration_ms": 0.046, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What's the current status of cryptocurrency markets?", - "query_length": 52, - "query_type": "routing_evaluation_query", - "classification.mode": "Web Search", - "classification.rag_keyword_score": 0, - "classification.web_keyword_score": 3, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "complex", - "query.intent": "general_knowledge", - "query.keyword_density": 0.42857142857142855, - "query.domain_affinity": "general", - "routing.decision": "Routed to Web Search based on analysis", - "routing.rationale": "RAG score: 0, Web score: 3, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "RAG", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.057692307692307696, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: Web Search (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "d682797f-2ce0-448e-8f0d-891541430988", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-14 12:53:44.917000000", - "trace_end_time": "2025-10-14 12:53:49.957000000", - "trace_duration_ms": 5039.64, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-14 12:53:44.917000000", - "updated_at": "2025-10-14 12:53:44.917000000" - }, - { - "span_id": "f137a412-6ddf-4ace-908e-9edf73534aa7", - "trace_id": "d682797f-2ce0-448e-8f0d-891541430988", - "parent_span_id": "e2d314c8-687f-4b8d-869c-8d7f2a22a4f3", - "name": "agent.query_routing", - "start_time": "2025-10-14 12:53:44.917000000", - "end_time": "2025-10-14 12:53:49.957000000", - "duration_ms": 5039.404, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What's the current status of cryptocurrency markets?", - "query_length": 52, - "query_type": "routing_query", - "output_response": "Routed to Web Search for query processing", - "router.classification": "Web Search", - "router.confidence_score": 1, - "router.rag_keyword_score": 0, - "router.web_keyword_score": 3, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "d682797f-2ce0-448e-8f0d-891541430988", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-14 12:53:44.917000000", - "trace_end_time": "2025-10-14 12:53:49.957000000", - "trace_duration_ms": 5039.64, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-14 12:53:44.917000000", - "updated_at": "2025-10-14 12:53:44.917000000" - }, - { - "span_id": "987b6910-9b6d-4207-b0f3-8807a23e3ddb", - "trace_id": "d682797f-2ce0-448e-8f0d-891541430988", - "parent_span_id": "f137a412-6ddf-4ace-908e-9edf73534aa7", - "name": "agent.web_search_generation", - "start_time": "2025-10-14 12:53:44.918000000", - "end_time": "2025-10-14 12:53:49.957000000", - "duration_ms": 5039.248, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "web_search_agent", - "agent.operation": "web_search_generation", - "agent.capabilities": [ - "web_search", - "content_synthesis", - "response_generation" - ], - "agent.id": "noveum_web_search_agent", - "input_query": "What's the current status of cryptocurrency markets?", - "query_length": 52, - "query_type": "web_search_query", - "output_response": "Web Search Answer: The web search results did not provide any specific information regarding the current status of cryptocurrency markets. Therefore, I cannot provide an update on market conditions, prices, or trends at...", - "web_search.results_count": 1, - "web_search.context_length": 154, - "web_search.response_length": 338, - "web_search.mode": "external_web_search" - }, - "events": [], - "links": [], - "trace_trace_id": "d682797f-2ce0-448e-8f0d-891541430988", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-14 12:53:44.917000000", - "trace_end_time": "2025-10-14 12:53:49.957000000", - "trace_duration_ms": 5039.64, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-14 12:53:44.917000000", - "updated_at": "2025-10-14 12:53:44.917000000" - }, - { - "span_id": "b2614d0e-5fcd-46ec-8d25-f1b04eb142af", - "trace_id": "d682797f-2ce0-448e-8f0d-891541430988", - "parent_span_id": "987b6910-9b6d-4207-b0f3-8807a23e3ddb", - "name": "agent.llm_model_execution", - "start_time": "2025-10-14 12:53:45.255000000", - "end_time": "2025-10-14 12:53:49.957000000", - "duration_ms": 4701.744, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What's the current status of cryptocurrency markets?", - "query_length": 52, - "query_type": "web_search_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 4.701629161834717, - "model.latency_ms": 4701.629161834717, - "model.start_time": 1760446425.2554948, - "model.end_time": 1760446429.957124, - "model.performance_tier": "medium", - "model.prompt_tokens": 162, - "model.completion_tokens": 84, - "model.total_tokens": 246, - "model.tokens_per_second": 52.32228904756993, - "model.estimated_cost": 0.00738, - "model.efficiency_score": 1.3739837398373984, - "model.response_length": 338, - "model.response_quality": "high", - "model.output_response": "Model Response: The web search results did not provide any specific information regarding the current status of cryptocurrency markets. Therefore, I cannot provide an update on market conditions, prices, or trends at...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "d682797f-2ce0-448e-8f0d-891541430988", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-14 12:53:44.917000000", - "trace_end_time": "2025-10-14 12:53:49.957000000", - "trace_duration_ms": 5039.64, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-14 12:53:44.917000000", - "updated_at": "2025-10-14 12:53:44.917000000" - }, - { - "span_id": "663e8789-75d0-4c4d-b14d-4088022436ac", - "trace_id": "d682797f-2ce0-448e-8f0d-891541430988", - "parent_span_id": "987b6910-9b6d-4207-b0f3-8807a23e3ddb", - "name": "agent.web_search_evaluation_metrics", - "start_time": "2025-10-14 12:53:49.957000000", - "end_time": "2025-10-14 12:53:49.957000000", - "duration_ms": 0.032, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "web_search_evaluation_metrics", - "agent.capabilities": [ - "web_search_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What's the current status of cryptocurrency markets?", - "query_length": 52, - "query_type": "web_search_evaluation_query", - "web_search.results_count": 1, - "web_search.context_length": 154, - "web_search.context_synthesized": "Context from 1 web sources", - "web_search.search_effectiveness": 0.2, - "web_search.context_quality": "low", - "web_search.source_diversity": 1, - "web_search.information_synthesis": "low", - "web_search.external_knowledge_utilization": 0.154, - "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What's+the+current+status+of+cryptocurrency+markets?\n\nUser Question: What's the current status of cryptocurrency markets?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", - "prompt.prompt_length": 651, - "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What's+the+current+status+of+cryptocurrency+markets?", - "prompt.instruction_following": "web_search_optimized", - "response.answer_length": 338, - "response.answer_completeness": "complete", - "response.response_quality": "medium", - "response.source_citation": 1, - "response.context_utilization": 2.1948051948051948, - "output_response": "Web Search Answer: The web search results did not provide any specific information regarding the current status of cryptocurrency markets. Therefore, I cannot provide an update on market conditions, prices, or trends at...", - "evaluation.search_effectiveness": 0.2, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 1, - "evaluation.context_relevance": "low", - "evaluation.overall_quality": "low", - "evaluation.ready_for_production": true, - "web_search.search_strategy": "duckduckgo_api", - "web_search.real_time_data": true, - "web_search.external_sources": 1, - "web_search.information_freshness": "current", - "web_search.knowledge_synthesis": "single_source" - }, - "events": [], - "links": [], - "trace_trace_id": "d682797f-2ce0-448e-8f0d-891541430988", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-14 12:53:44.917000000", - "trace_end_time": "2025-10-14 12:53:49.957000000", - "trace_duration_ms": 5039.64, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-14 12:53:44.917000000", - "updated_at": "2025-10-14 12:53:44.917000000" - }, - { - "span_id": "3382fe7c-aaca-46e0-a8d2-dbee75ff3bed", - "trace_id": "b0858dca-4173-493b-a6c5-4e993f606477", - "parent_span_id": "6cf94049-87b0-4fc5-b37e-32d261543647", - "name": "agent.query_routing", - "start_time": "2025-10-14 12:53:37.818000000", - "end_time": "2025-10-14 12:53:43.908000000", - "duration_ms": 6089.22, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What are the latest updates in Python programming?", - "query_length": 50, - "query_type": "routing_query", - "output_response": "Routed to Web Search for query processing", - "router.classification": "Web Search", - "router.confidence_score": 1, - "router.rag_keyword_score": 0, - "router.web_keyword_score": 2, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "b0858dca-4173-493b-a6c5-4e993f606477", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-14 12:53:37.818000000", - "trace_end_time": "2025-10-14 12:53:43.908000000", - "trace_duration_ms": 6089.715, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-14 12:53:37.818000000", - "updated_at": "2025-10-14 12:53:37.818000000" - }, - { - "span_id": "6cf94049-87b0-4fc5-b37e-32d261543647", - "trace_id": "b0858dca-4173-493b-a6c5-4e993f606477", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-14 12:53:37.818000000", - "end_time": "2025-10-14 12:53:43.908000000", - "duration_ms": 6089.607, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What are the latest updates in Python programming?", - "query.length": 50, - "processing.mode": "Web Search", - "processing.time_seconds": 6.089300870895386, - "processing.response_length": 1324, - "processing.sources_count": 1, - "output_response": "Final Answer: The web search did not yield specific results regarding the latest updates in Python programming. However, I can provide a general overview based on my knowledge up to October 2023.\n\nAs of late 2023, ...", - "final_answer_mode": "Web Search", - "query_processed.input_query": "What are the latest updates in Python programming?", - "query_processed.output_response": "Successfully processed query using Web Search, generated 1324 character response", - "query_processed.mode": "Web Search", - "query_processed.processing_time": 6.089300870895386, - "query_processed.response_length": 1324 - }, - "events": [], - "links": [], - "trace_trace_id": "b0858dca-4173-493b-a6c5-4e993f606477", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-14 12:53:37.818000000", - "trace_end_time": "2025-10-14 12:53:43.908000000", - "trace_duration_ms": 6089.715, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-14 12:53:37.818000000", - "updated_at": "2025-10-14 12:53:37.818000000" - }, - { - "span_id": "21c8befa-356a-4034-8d62-792238cb9e6e", - "trace_id": "b0858dca-4173-493b-a6c5-4e993f606477", - "parent_span_id": "3382fe7c-aaca-46e0-a8d2-dbee75ff3bed", - "name": "agent.web_search_generation", - "start_time": "2025-10-14 12:53:37.819000000", - "end_time": "2025-10-14 12:53:43.908000000", - "duration_ms": 6088.807, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "web_search_agent", - "agent.operation": "web_search_generation", - "agent.capabilities": [ - "web_search", - "content_synthesis", - "response_generation" - ], - "agent.id": "noveum_web_search_agent", - "input_query": "What are the latest updates in Python programming?", - "query_length": 50, - "query_type": "web_search_query", - "output_response": "Web Search Answer: The web search did not yield specific results regarding the latest updates in Python programming. However, I can provide a general overview based on my knowledge up to October 2023.\n\nAs of late 2023, ...", - "web_search.results_count": 1, - "web_search.context_length": 152, - "web_search.response_length": 1324, - "web_search.mode": "external_web_search" - }, - "events": [], - "links": [], - "trace_trace_id": "b0858dca-4173-493b-a6c5-4e993f606477", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-14 12:53:37.818000000", - "trace_end_time": "2025-10-14 12:53:43.908000000", - "trace_duration_ms": 6089.715, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-14 12:53:37.818000000", - "updated_at": "2025-10-14 12:53:37.818000000" - }, - { - "span_id": "482feedf-52f4-4c6a-9186-d59898820caa", - "trace_id": "b0858dca-4173-493b-a6c5-4e993f606477", - "parent_span_id": "3382fe7c-aaca-46e0-a8d2-dbee75ff3bed", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-14 12:53:37.819000000", - "end_time": "2025-10-14 12:53:37.819000000", - "duration_ms": 0.039, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What are the latest updates in Python programming?", - "query_length": 50, - "query_type": "routing_evaluation_query", - "classification.mode": "Web Search", - "classification.rag_keyword_score": 0, - "classification.web_keyword_score": 2, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "general_knowledge", - "query.keyword_density": 0.25, - "query.domain_affinity": "general", - "routing.decision": "Routed to Web Search based on analysis", - "routing.rationale": "RAG score: 0, Web score: 2, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "RAG", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.038461538461538464, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: Web Search (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "b0858dca-4173-493b-a6c5-4e993f606477", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-14 12:53:37.818000000", - "trace_end_time": "2025-10-14 12:53:43.908000000", - "trace_duration_ms": 6089.715, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-14 12:53:37.818000000", - "updated_at": "2025-10-14 12:53:37.818000000" - }, - { - "span_id": "84f1f63f-ee05-411e-a7dd-4ee9e9826f5a", - "trace_id": "b0858dca-4173-493b-a6c5-4e993f606477", - "parent_span_id": "21c8befa-356a-4034-8d62-792238cb9e6e", - "name": "agent.llm_model_execution", - "start_time": "2025-10-14 12:53:38.191000000", - "end_time": "2025-10-14 12:53:43.907000000", - "duration_ms": 5716.078, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What are the latest updates in Python programming?", - "query_length": 50, - "query_type": "web_search_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 5.715963125228882, - "model.latency_ms": 5715.963125228882, - "model.start_time": 1760446418.191805, - "model.end_time": 1760446423.907768, - "model.performance_tier": "slow", - "model.prompt_tokens": 161, - "model.completion_tokens": 331, - "model.total_tokens": 492, - "model.tokens_per_second": 86.07473302765561, - "model.estimated_cost": 0.01476, - "model.efficiency_score": 2.6910569105691056, - "model.response_length": 1324, - "model.response_quality": "high", - "model.output_response": "Model Response: The web search did not yield specific results regarding the latest updates in Python programming. However, I can provide a general overview based on my knowledge up to October 2023.\n\nAs of late 2023, ...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "b0858dca-4173-493b-a6c5-4e993f606477", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-14 12:53:37.818000000", - "trace_end_time": "2025-10-14 12:53:43.908000000", - "trace_duration_ms": 6089.715, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-14 12:53:37.818000000", - "updated_at": "2025-10-14 12:53:37.818000000" - }, - { - "span_id": "2fb9ecf3-a497-4285-a694-1c95080d9e6a", - "trace_id": "b0858dca-4173-493b-a6c5-4e993f606477", - "parent_span_id": "21c8befa-356a-4034-8d62-792238cb9e6e", - "name": "agent.web_search_evaluation_metrics", - "start_time": "2025-10-14 12:53:43.908000000", - "end_time": "2025-10-14 12:53:43.908000000", - "duration_ms": 0.037, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "web_search_evaluation_metrics", - "agent.capabilities": [ - "web_search_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What are the latest updates in Python programming?", - "query_length": 50, - "query_type": "web_search_evaluation_query", - "web_search.results_count": 1, - "web_search.context_length": 152, - "web_search.context_synthesized": "Context from 1 web sources", - "web_search.search_effectiveness": 0.2, - "web_search.context_quality": "low", - "web_search.source_diversity": 1, - "web_search.information_synthesis": "low", - "web_search.external_knowledge_utilization": 0.152, - "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What+are+the+latest+updates+in+Python+programming?\n\nUser Question: What are the latest updates in Python programming?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", - "prompt.prompt_length": 647, - "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What+are+the+latest+updates+in+Python+programming?", - "prompt.instruction_following": "web_search_optimized", - "response.answer_length": 1324, - "response.answer_completeness": "complete", - "response.response_quality": "medium", - "response.source_citation": 1, - "response.context_utilization": 8.710526315789474, - "output_response": "Web Search Answer: The web search did not yield specific results regarding the latest updates in Python programming. However, I can provide a general overview based on my knowledge up to October 2023.\n\nAs of late 2023, ...", - "evaluation.search_effectiveness": 0.2, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 1, - "evaluation.context_relevance": "low", - "evaluation.overall_quality": "low", - "evaluation.ready_for_production": true, - "web_search.search_strategy": "duckduckgo_api", - "web_search.real_time_data": true, - "web_search.external_sources": 1, - "web_search.information_freshness": "current", - "web_search.knowledge_synthesis": "single_source" - }, - "events": [], - "links": [], - "trace_trace_id": "b0858dca-4173-493b-a6c5-4e993f606477", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-14 12:53:37.818000000", - "trace_end_time": "2025-10-14 12:53:43.908000000", - "trace_duration_ms": 6089.715, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-14 12:53:37.818000000", - "updated_at": "2025-10-14 12:53:37.818000000" - }, - { - "span_id": "9dd5ac2c-0deb-48e3-ac8b-0e6e31f7ff30", - "trace_id": "dc81be1a-e93d-4f60-90f3-ce1acaa8cb3e", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-14 12:53:34.584000000", - "end_time": "2025-10-14 12:53:36.780000000", - "duration_ms": 2196.351, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What happened in tech news this week?", - "query.length": 37, - "processing.mode": "Web Search", - "processing.time_seconds": 2.1948978900909424, - "processing.response_length": 324, - "processing.sources_count": 1, - "output_response": "Final Answer: I'm sorry, but I couldn't find any specific information regarding tech news from this week based on the search results. It appears that there were no relevant articles or updates available. If you're ...", - "final_answer_mode": "Web Search", - "query_processed.input_query": "What happened in tech news this week?", - "query_processed.output_response": "Successfully processed query using Web Search, generated 324 character response", - "query_processed.mode": "Web Search", - "query_processed.processing_time": 2.1948978900909424, - "query_processed.response_length": 324 - }, - "events": [], - "links": [], - "trace_trace_id": "dc81be1a-e93d-4f60-90f3-ce1acaa8cb3e", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-14 12:53:34.584000000", - "trace_end_time": "2025-10-14 12:53:36.780000000", - "trace_duration_ms": 2196.4579999999996, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-14 12:53:34.584000000", - "updated_at": "2025-10-14 12:53:34.584000000" - }, - { - "span_id": "b1cbafce-51fb-48fd-bc4a-d49081a2f374", - "trace_id": "dc81be1a-e93d-4f60-90f3-ce1acaa8cb3e", - "parent_span_id": "9dd5ac2c-0deb-48e3-ac8b-0e6e31f7ff30", - "name": "agent.query_routing", - "start_time": "2025-10-14 12:53:34.584000000", - "end_time": "2025-10-14 12:53:36.778000000", - "duration_ms": 2194.2200000000003, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What happened in tech news this week?", - "query_length": 37, - "query_type": "routing_query", - "output_response": "Routed to Web Search for query processing", - "router.classification": "Web Search", - "router.confidence_score": 1, - "router.rag_keyword_score": 0, - "router.web_keyword_score": 2, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "dc81be1a-e93d-4f60-90f3-ce1acaa8cb3e", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-14 12:53:34.584000000", - "trace_end_time": "2025-10-14 12:53:36.780000000", - "trace_duration_ms": 2196.4579999999996, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-14 12:53:34.584000000", - "updated_at": "2025-10-14 12:53:34.584000000" - }, - { - "span_id": "e3064687-1b18-478b-9e9d-4d081a6dac23", - "trace_id": "dc81be1a-e93d-4f60-90f3-ce1acaa8cb3e", - "parent_span_id": "b1cbafce-51fb-48fd-bc4a-d49081a2f374", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-14 12:53:34.584000000", - "end_time": "2025-10-14 12:53:34.584000000", - "duration_ms": 0.028, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What happened in tech news this week?", - "query_length": 37, - "query_type": "routing_evaluation_query", - "classification.mode": "Web Search", - "classification.rag_keyword_score": 0, - "classification.web_keyword_score": 2, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "general_knowledge", - "query.keyword_density": 0.2857142857142857, - "query.domain_affinity": "general", - "routing.decision": "Routed to Web Search based on analysis", - "routing.rationale": "RAG score: 0, Web score: 2, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "RAG", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.038461538461538464, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: Web Search (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "dc81be1a-e93d-4f60-90f3-ce1acaa8cb3e", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-14 12:53:34.584000000", - "trace_end_time": "2025-10-14 12:53:36.780000000", - "trace_duration_ms": 2196.4579999999996, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-14 12:53:34.584000000", - "updated_at": "2025-10-14 12:53:34.584000000" - }, - { - "span_id": "eb9dbc11-44b3-4041-8c83-0f33b8b761bb", - "trace_id": "dc81be1a-e93d-4f60-90f3-ce1acaa8cb3e", - "parent_span_id": "b1cbafce-51fb-48fd-bc4a-d49081a2f374", - "name": "agent.web_search_generation", - "start_time": "2025-10-14 12:53:34.584000000", - "end_time": "2025-10-14 12:53:36.774000000", - "duration_ms": 2189.866, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "web_search_agent", - "agent.operation": "web_search_generation", - "agent.capabilities": [ - "web_search", - "content_synthesis", - "response_generation" - ], - "agent.id": "noveum_web_search_agent", - "input_query": "What happened in tech news this week?", - "query_length": 37, - "query_type": "web_search_query", - "output_response": "Web Search Answer: I'm sorry, but I couldn't find any specific information regarding tech news from this week based on the search results. It appears that there were no relevant articles or updates available. If you're ...", - "web_search.results_count": 1, - "web_search.context_length": 139, - "web_search.response_length": 324, - "web_search.mode": "external_web_search" - }, - "events": [], - "links": [], - "trace_trace_id": "dc81be1a-e93d-4f60-90f3-ce1acaa8cb3e", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-14 12:53:34.584000000", - "trace_end_time": "2025-10-14 12:53:36.780000000", - "trace_duration_ms": 2196.4579999999996, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-14 12:53:34.584000000", - "updated_at": "2025-10-14 12:53:34.584000000" - }, - { - "span_id": "c8ee237b-ab5e-4b36-83f0-2f8478971898", - "trace_id": "dc81be1a-e93d-4f60-90f3-ce1acaa8cb3e", - "parent_span_id": "eb9dbc11-44b3-4041-8c83-0f33b8b761bb", - "name": "agent.llm_model_execution", - "start_time": "2025-10-14 12:53:34.981000000", - "end_time": "2025-10-14 12:53:36.768000000", - "duration_ms": 1786.467, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What happened in tech news this week?", - "query_length": 37, - "query_type": "web_search_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 1.785179853439331, - "model.latency_ms": 1785.179853439331, - "model.start_time": 1760446414.981964, - "model.end_time": 1760446416.767144, - "model.performance_tier": "fast", - "model.prompt_tokens": 155, - "model.completion_tokens": 81, - "model.total_tokens": 236, - "model.tokens_per_second": 132.19956495997977, - "model.estimated_cost": 0.00708, - "model.efficiency_score": 1.3728813559322033, - "model.response_length": 324, - "model.response_quality": "high", - "model.output_response": "Model Response: I'm sorry, but I couldn't find any specific information regarding tech news from this week based on the search results. It appears that there were no relevant articles or updates available. If you're ...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "dc81be1a-e93d-4f60-90f3-ce1acaa8cb3e", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-14 12:53:34.584000000", - "trace_end_time": "2025-10-14 12:53:36.780000000", - "trace_duration_ms": 2196.4579999999996, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-14 12:53:34.584000000", - "updated_at": "2025-10-14 12:53:34.584000000" - }, - { - "span_id": "9c8e1a08-61c8-409e-a79d-41c74a8d1505", - "trace_id": "dc81be1a-e93d-4f60-90f3-ce1acaa8cb3e", - "parent_span_id": "eb9dbc11-44b3-4041-8c83-0f33b8b761bb", - "name": "agent.web_search_evaluation_metrics", - "start_time": "2025-10-14 12:53:36.772000000", - "end_time": "2025-10-14 12:53:36.774000000", - "duration_ms": 1.5939999999999999, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "web_search_evaluation_metrics", - "agent.capabilities": [ - "web_search_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What happened in tech news this week?", - "query_length": 37, - "query_type": "web_search_evaluation_query", - "web_search.results_count": 1, - "web_search.context_length": 139, - "web_search.context_synthesized": "Context from 1 web sources", - "web_search.search_effectiveness": 0.2, - "web_search.context_quality": "low", - "web_search.source_diversity": 1, - "web_search.information_synthesis": "low", - "web_search.external_knowledge_utilization": 0.139, - "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What+happened+in+tech+news+this+week?\n\nUser Question: What happened in tech news this week?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", - "prompt.prompt_length": 621, - "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What+happened+in+tech+news+this+week?", - "prompt.instruction_following": "web_search_optimized", - "response.answer_length": 324, - "response.answer_completeness": "complete", - "response.response_quality": "medium", - "response.source_citation": 1, - "response.context_utilization": 2.3309352517985613, - "output_response": "Web Search Answer: I'm sorry, but I couldn't find any specific information regarding tech news from this week based on the search results. It appears that there were no relevant articles or updates available. If you're ...", - "evaluation.search_effectiveness": 0.2, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 1, - "evaluation.context_relevance": "low", - "evaluation.overall_quality": "low", - "evaluation.ready_for_production": true, - "web_search.search_strategy": "duckduckgo_api", - "web_search.real_time_data": true, - "web_search.external_sources": 1, - "web_search.information_freshness": "current", - "web_search.knowledge_synthesis": "single_source" - }, - "events": [], - "links": [], - "trace_trace_id": "dc81be1a-e93d-4f60-90f3-ce1acaa8cb3e", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-14 12:53:34.584000000", - "trace_end_time": "2025-10-14 12:53:36.780000000", - "trace_duration_ms": 2196.4579999999996, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-14 12:53:34.584000000", - "updated_at": "2025-10-14 12:53:34.584000000" - }, - { - "span_id": "59595232-8873-419c-9b6a-0727445a2afa", - "trace_id": "945b2757-8a5a-4aab-a869-8f44c02166a5", - "parent_span_id": "dfde2f79-d0c1-481e-819d-60730cb60fe5", - "name": "agent.query_routing", - "start_time": "2025-10-14 12:53:27.771000000", - "end_time": "2025-10-14 12:53:33.575000000", - "duration_ms": 5803.9800000000005, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What are the current trends in observability tools?", - "query_length": 51, - "query_type": "routing_query", - "output_response": "Routed to RAG for query processing", - "router.classification": "RAG", - "router.confidence_score": 0.3333333333333333, - "router.rag_keyword_score": 2, - "router.web_keyword_score": 1, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "945b2757-8a5a-4aab-a869-8f44c02166a5", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-14 12:53:27.771000000", - "trace_end_time": "2025-10-14 12:53:33.575000000", - "trace_duration_ms": 5804.15, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-14 12:53:27.771000000", - "updated_at": "2025-10-14 12:53:27.771000000" - }, - { - "span_id": "a59b3296-aaba-4642-b5a0-a271787a5607", - "trace_id": "945b2757-8a5a-4aab-a869-8f44c02166a5", - "parent_span_id": "59595232-8873-419c-9b6a-0727445a2afa", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-14 12:53:27.771000000", - "end_time": "2025-10-14 12:53:27.771000000", - "duration_ms": 0.031, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What are the current trends in observability tools?", - "query_length": 51, - "query_type": "routing_evaluation_query", - "classification.mode": "RAG", - "classification.rag_keyword_score": 2, - "classification.web_keyword_score": 1, - "classification.confidence_score": 0.3333333333333333, - "classification.confidence_level": "medium", - "classification.method": "keyword_based", - "query.complexity": "complex", - "query.intent": "documentation", - "query.keyword_density": 0.375, - "query.domain_affinity": "noveum", - "routing.decision": "Routed to RAG based on analysis", - "routing.rationale": "RAG score: 2, Web score: 1, Confidence: 0.33", - "routing.expected_performance": "medium", - "routing.alternative_mode": "Web Search", - "routing.decision_confidence": 0.3333333333333333, - "evaluation.routing_accuracy": "medium", - "evaluation.keyword_coverage": 0.057692307692307696, - "evaluation.query_understanding": "ambiguous", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: RAG (Confidence: 0.33)" - }, - "events": [], - "links": [], - "trace_trace_id": "945b2757-8a5a-4aab-a869-8f44c02166a5", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-14 12:53:27.771000000", - "trace_end_time": "2025-10-14 12:53:33.575000000", - "trace_duration_ms": 5804.15, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-14 12:53:27.771000000", - "updated_at": "2025-10-14 12:53:27.771000000" - }, - { - "span_id": "dfde2f79-d0c1-481e-819d-60730cb60fe5", - "trace_id": "945b2757-8a5a-4aab-a869-8f44c02166a5", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-14 12:53:27.771000000", - "end_time": "2025-10-14 12:53:33.575000000", - "duration_ms": 5804.074, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What are the current trends in observability tools?", - "query.length": 51, - "processing.mode": "RAG", - "processing.time_seconds": 5.804032802581787, - "processing.response_length": 997, - "processing.sources_count": 5, - "output_response": "Final Answer: The provided context does not specifically outline current trends in observability tools in general. However, it highlights the unique challenges faced by AI applications and how traditional monitorin...", - "final_answer_mode": "RAG", - "query_processed.input_query": "What are the current trends in observability tools?", - "query_processed.output_response": "Successfully processed query using RAG, generated 997 character response", - "query_processed.mode": "RAG", - "query_processed.processing_time": 5.804032802581787, - "query_processed.response_length": 997 - }, - "events": [], - "links": [], - "trace_trace_id": "945b2757-8a5a-4aab-a869-8f44c02166a5", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-14 12:53:27.771000000", - "trace_end_time": "2025-10-14 12:53:33.575000000", - "trace_duration_ms": 5804.15, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-14 12:53:27.771000000", - "updated_at": "2025-10-14 12:53:27.771000000" - }, - { - "span_id": "82489444-deba-46d2-b08b-c09754189566", - "trace_id": "945b2757-8a5a-4aab-a869-8f44c02166a5", - "parent_span_id": "59595232-8873-419c-9b6a-0727445a2afa", - "name": "agent.llm-rag", - "start_time": "2025-10-14 12:53:27.772000000", - "end_time": "2025-10-14 12:53:33.575000000", - "duration_ms": 5803.534, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "rag_agent", - "agent.operation": "llm-rag", - "agent.capabilities": [ - "document_retrieval", - "context_generation", - "response_generation" - ], - "agent.id": "noveum_rag_agent", - "input_query": "What are the current trends in observability tools?", - "query_length": 51, - "query_type": "rag_query", - "output_response": "RAG Answer: The provided context does not specifically outline current trends in observability tools in general. However, it highlights the unique challenges faced by AI applications and how traditional monitorin...", - "rag.context_length": 2943, - "rag.sources_count": 5, - "rag.answer_length": 997, - "rag.mode": "retrieval_augmented_generation" - }, - "events": [], - "links": [], - "trace_trace_id": "945b2757-8a5a-4aab-a869-8f44c02166a5", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-14 12:53:27.771000000", - "trace_end_time": "2025-10-14 12:53:33.575000000", - "trace_duration_ms": 5804.15, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-14 12:53:27.771000000", - "updated_at": "2025-10-14 12:53:27.771000000" - }, - { - "span_id": "34b1f14c-3fd3-4f89-b4dd-6150a52befd7", - "trace_id": "945b2757-8a5a-4aab-a869-8f44c02166a5", - "parent_span_id": "82489444-deba-46d2-b08b-c09754189566", - "name": "agent.llm_model_execution", - "start_time": "2025-10-14 12:53:28.257000000", - "end_time": "2025-10-14 12:53:33.162000000", - "duration_ms": 4905.2429999999995, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What are the current trends in observability tools?", - "query_length": 51, - "query_type": "rag_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 4.9052040576934814, - "model.latency_ms": 4905.204057693481, - "model.start_time": 1760446408.2575018, - "model.end_time": 1760446413.162706, - "model.performance_tier": "medium", - "model.prompt_tokens": 872, - "model.completion_tokens": 249, - "model.total_tokens": 1121, - "model.tokens_per_second": 228.53279635569638, - "model.estimated_cost": 0.03363, - "model.efficiency_score": 0.8893844781445138, - "model.response_length": 997, - "model.response_quality": "high", - "model.output_response": "Model Response: The provided context does not specifically outline current trends in observability tools in general. However, it highlights the unique challenges faced by AI applications and how traditional monitorin...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "945b2757-8a5a-4aab-a869-8f44c02166a5", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-14 12:53:27.771000000", - "trace_end_time": "2025-10-14 12:53:33.575000000", - "trace_duration_ms": 5804.15, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-14 12:53:27.771000000", - "updated_at": "2025-10-14 12:53:27.771000000" - }, - { - "span_id": "d5d2b966-3c37-4d6d-b7b8-c58c9f22aad9", - "trace_id": "945b2757-8a5a-4aab-a869-8f44c02166a5", - "parent_span_id": "82489444-deba-46d2-b08b-c09754189566", - "name": "agent.rag_evaluation_metrics", - "start_time": "2025-10-14 12:53:33.162000000", - "end_time": "2025-10-14 12:53:33.162000000", - "duration_ms": 0.036000000000000004, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "rag_evaluation_metrics", - "agent.capabilities": [ - "retrieval_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What are the current trends in observability tools?", - "query_length": 51, - "query_type": "rag_evaluation_query", - "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/blog/from-logs-to-intelligent-choices-inside-noveum-ais-evaluation-process):\nFrom Development to Production - Inside Noveum.ai's AI Observability Platform\nShashank Agarwal\n3/3/2025\nIntroduction\nEvery AI application tells a story\u2014but most developers never get to hear it...", - "retrieval.context_length": 2943, - "retrieval.sources_count": 5, - "retrieval.context_quality": "high", - "retrieval.effectiveness": 1, - "retrieval.context_utilization": 2.943, - "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/blog/from-logs-to-intelligent-choices-inside-noveum-ais-evaluation-process):\nFrom Development to Production - Inside Noveum.ai's AI Observability Platform\nShashank Agarwal\n3/3/2025\nIntroduction\nEvery AI application tells a story\u2014but most developers never get to hear it. When your RAG pipeline returns irrelevant results, when your multi-agent system gets stuck in loops, or when your LLM costs suddenly spike, you're left guessing what went wrong and where.\nThat's exactly the challenge Noveum.ai solves. Rather than flying blind, our platform provides comprehensive tracing a...\n\nSource 2 (https://noveum.ai/docs/getting-started/tracing-concepts):\n- Explore Framework Integrations - Framework-specific guidance\n- Learn Advanced Patterns - Custom instrumentation techniques\n- Master the Dashboard - Analyze your traces effectively\nRemember: Good observability is not about collecting all possible data, but about collecting the right data that helps you understand, debug, and optimize your AI applications.\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to O...\n\nSource 3 (https://noveum.ai/en/docs/getting-started/tracing-concepts):\n- Explore Framework Integrations - Framework-specific guidance\n- Learn Advanced Patterns - Custom instrumentation techniques\n- Master the Dashboard - Analyze your traces effectively\nRemember: Good observability is not about collecting all possible data, but about collecting the right data that helps you understand, debug, and optimize your AI applications.\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to O...\n\nSource 4 (https://noveum.ai/en/blog/from-logs-to-intelligent-choices-inside-noveum-ais-evaluation-process):\nThe AI Observability Challenge\nWhy Traditional Monitoring Falls Short\nTraditional application monitoring tools weren't built for AI applications. They can tell you if your API is responding, but they can't answer the questions that matter most for AI systems:\n- Why did my RAG pipeline retrieve irrelevant documents?\n- Which LLM calls are driving my costs?\n- How are my agents communicating with each other?\n- What's causing hallucinations in my responses?\n- Why is my embedding generation so slow?\nW...\n\nSource 5 (https://noveum.ai/en/docs/advanced/rag-observability):\n\ud83d\udd27 Best Practices for RAG Observability\n1. Comprehensive Pipeline Tracing\n2. Quality Monitoring\n3. A/B Testing for RAG Components\n\ud83c\udfaf Advanced RAG Patterns\nMulti-Modal RAG\nConversational RAG\nRAG observability with Noveum.ai provides the deep insights needed to build, optimize, and scale retrieval-augmented generation systems. By implementing comprehensive tracing across retrieval, context preparation, and generation stages, you can ensure your RAG pipeline delivers accurate, relevant, and cost-effe...\n\nUser Question: What are the current trends in observability tools?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", - "prompt.prompt_length": 3490, - "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/blog/from-logs-to-intelligent-choices-inside-noveum-ais-evaluation-process):\nFrom Development to Production - Inside Noveum.ai's AI Observability Platform\nShashank Agarw...", - "prompt.instruction_following": "rag_optimized", - "response.answer_length": 997, - "response.answer_completeness": "complete", - "response.response_quality": "high", - "response.source_citation": 5, - "response.context_utilization": 0.33876996262317366, - "output_response": "RAG Answer: The provided context does not specifically outline current trends in observability tools in general. However, it highlights the unique challenges faced by AI applications and how traditional monitorin...", - "evaluation.retrieval_effectiveness": 1, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 5, - "evaluation.context_relevance": "high", - "evaluation.overall_quality": "high", - "evaluation.ready_for_production": true, - "rag.retrieval_strategy": "semantic_similarity", - "rag.vector_search_results": 5, - "rag.context_synthesis": "multi_source", - "rag.document_coverage": 1, - "rag.information_density": 0.33876996262317366 - }, - "events": [], - "links": [], - "trace_trace_id": "945b2757-8a5a-4aab-a869-8f44c02166a5", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-14 12:53:27.771000000", - "trace_end_time": "2025-10-14 12:53:33.575000000", - "trace_duration_ms": 5804.15, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-14 12:53:27.771000000", - "updated_at": "2025-10-14 12:53:27.771000000" - }, - { - "span_id": "11c07411-df30-42a4-9062-90a032ef75ed", - "trace_id": "4bdd5814-acbb-4de7-a56b-543b87a1cfa5", - "parent_span_id": "ec530838-a7cf-477b-9186-a70114c8a12f", - "name": "agent.web_search_generation", - "start_time": "2025-10-14 12:53:20.120000000", - "end_time": "2025-10-14 12:53:26.750000000", - "duration_ms": 6629.273999999999, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "web_search_agent", - "agent.operation": "web_search_generation", - "agent.capabilities": [ - "web_search", - "content_synthesis", - "response_generation" - ], - "agent.id": "noveum_web_search_agent", - "input_query": "Tell me about recent developments in machine learning", - "query_length": 53, - "query_type": "web_search_query", - "output_response": "Web Search Answer: The web search did not yield specific results regarding recent developments in machine learning. However, I can provide a general overview based on my knowledge up to October 2023.\n\nRecent development...", - "web_search.results_count": 1, - "web_search.context_length": 155, - "web_search.response_length": 1849, - "web_search.mode": "external_web_search" - }, - "events": [], - "links": [], - "trace_trace_id": "4bdd5814-acbb-4de7-a56b-543b87a1cfa5", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-14 12:53:20.120000000", - "trace_end_time": "2025-10-14 12:53:26.750000000", - "trace_duration_ms": 6630.38, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-14 12:53:20.120000000", - "updated_at": "2025-10-14 12:53:20.120000000" - }, - { - "span_id": "282b121f-61dd-4dc5-9e55-cecc3f02f688", - "trace_id": "4bdd5814-acbb-4de7-a56b-543b87a1cfa5", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-14 12:53:20.120000000", - "end_time": "2025-10-14 12:53:26.750000000", - "duration_ms": 6630.264, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "Tell me about recent developments in machine learning", - "query.length": 53, - "processing.mode": "Web Search", - "processing.time_seconds": 6.629686117172241, - "processing.response_length": 1849, - "processing.sources_count": 1, - "output_response": "Final Answer: The web search did not yield specific results regarding recent developments in machine learning. However, I can provide a general overview based on my knowledge up to October 2023.\n\nRecent development...", - "final_answer_mode": "Web Search", - "query_processed.input_query": "Tell me about recent developments in machine learning", - "query_processed.output_response": "Successfully processed query using Web Search, generated 1849 character response", - "query_processed.mode": "Web Search", - "query_processed.processing_time": 6.629686117172241, - "query_processed.response_length": 1849 - }, - "events": [], - "links": [], - "trace_trace_id": "4bdd5814-acbb-4de7-a56b-543b87a1cfa5", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-14 12:53:20.120000000", - "trace_end_time": "2025-10-14 12:53:26.750000000", - "trace_duration_ms": 6630.38, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-14 12:53:20.120000000", - "updated_at": "2025-10-14 12:53:20.120000000" - }, - { - "span_id": "ec530838-a7cf-477b-9186-a70114c8a12f", - "trace_id": "4bdd5814-acbb-4de7-a56b-543b87a1cfa5", - "parent_span_id": "282b121f-61dd-4dc5-9e55-cecc3f02f688", - "name": "agent.query_routing", - "start_time": "2025-10-14 12:53:20.120000000", - "end_time": "2025-10-14 12:53:26.750000000", - "duration_ms": 6629.59, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "Tell me about recent developments in machine learning", - "query_length": 53, - "query_type": "routing_query", - "output_response": "Routed to Web Search for query processing", - "router.classification": "Web Search", - "router.confidence_score": 1, - "router.rag_keyword_score": 0, - "router.web_keyword_score": 1, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "4bdd5814-acbb-4de7-a56b-543b87a1cfa5", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-14 12:53:20.120000000", - "trace_end_time": "2025-10-14 12:53:26.750000000", - "trace_duration_ms": 6630.38, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-14 12:53:20.120000000", - "updated_at": "2025-10-14 12:53:20.120000000" - }, - { - "span_id": "df882f90-cf1a-42a6-a2ae-78b2a13f27a1", - "trace_id": "4bdd5814-acbb-4de7-a56b-543b87a1cfa5", - "parent_span_id": "ec530838-a7cf-477b-9186-a70114c8a12f", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-14 12:53:20.120000000", - "end_time": "2025-10-14 12:53:20.120000000", - "duration_ms": 0.082, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: Tell me about recent developments in machine learning", - "query_length": 53, - "query_type": "routing_evaluation_query", - "classification.mode": "Web Search", - "classification.rag_keyword_score": 0, - "classification.web_keyword_score": 1, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "complex", - "query.intent": "general_knowledge", - "query.keyword_density": 0.125, - "query.domain_affinity": "general", - "routing.decision": "Routed to Web Search based on analysis", - "routing.rationale": "RAG score: 0, Web score: 1, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "RAG", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.019230769230769232, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: Web Search (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "4bdd5814-acbb-4de7-a56b-543b87a1cfa5", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-14 12:53:20.120000000", - "trace_end_time": "2025-10-14 12:53:26.750000000", - "trace_duration_ms": 6630.38, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-14 12:53:20.120000000", - "updated_at": "2025-10-14 12:53:20.120000000" - }, - { - "span_id": "bea819fd-3673-480b-9f7b-a3a2efef4218", - "trace_id": "4bdd5814-acbb-4de7-a56b-543b87a1cfa5", - "parent_span_id": "11c07411-df30-42a4-9062-90a032ef75ed", - "name": "agent.llm_model_execution", - "start_time": "2025-10-14 12:53:20.455000000", - "end_time": "2025-10-14 12:53:26.746000000", - "duration_ms": 6291.401, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: Tell me about recent developments in machine learning", - "query_length": 53, - "query_type": "web_search_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 6.290812253952026, - "model.latency_ms": 6290.812253952026, - "model.start_time": 1760446400.4555368, - "model.end_time": 1760446406.746349, - "model.performance_tier": "slow", - "model.prompt_tokens": 163, - "model.completion_tokens": 462, - "model.total_tokens": 625, - "model.tokens_per_second": 99.35124031199013, - "model.estimated_cost": 0.01875, - "model.efficiency_score": 2.9584, - "model.response_length": 1849, - "model.response_quality": "high", - "model.output_response": "Model Response: The web search did not yield specific results regarding recent developments in machine learning. However, I can provide a general overview based on my knowledge up to October 2023.\n\nRecent development...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "4bdd5814-acbb-4de7-a56b-543b87a1cfa5", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-14 12:53:20.120000000", - "trace_end_time": "2025-10-14 12:53:26.750000000", - "trace_duration_ms": 6630.38, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-14 12:53:20.120000000", - "updated_at": "2025-10-14 12:53:20.120000000" - }, - { - "span_id": "6e62af18-569c-4783-b2ae-177c58542bd0", - "trace_id": "4bdd5814-acbb-4de7-a56b-543b87a1cfa5", - "parent_span_id": "11c07411-df30-42a4-9062-90a032ef75ed", - "name": "agent.web_search_evaluation_metrics", - "start_time": "2025-10-14 12:53:26.749000000", - "end_time": "2025-10-14 12:53:26.750000000", - "duration_ms": 0.217, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "web_search_evaluation_metrics", - "agent.capabilities": [ - "web_search_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: Tell me about recent developments in machine learning", - "query_length": 53, - "query_type": "web_search_evaluation_query", - "web_search.results_count": 1, - "web_search.context_length": 155, - "web_search.context_synthesized": "Context from 1 web sources", - "web_search.search_effectiveness": 0.2, - "web_search.context_quality": "low", - "web_search.source_diversity": 1, - "web_search.information_synthesis": "low", - "web_search.external_knowledge_utilization": 0.155, - "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=Tell+me+about+recent+developments+in+machine+learning\n\nUser Question: Tell me about recent developments in machine learning\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", - "prompt.prompt_length": 653, - "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=Tell+me+about+recent+developments+in+machine+learning", - "prompt.instruction_following": "web_search_optimized", - "response.answer_length": 1849, - "response.answer_completeness": "complete", - "response.response_quality": "medium", - "response.source_citation": 1, - "response.context_utilization": 11.929032258064517, - "output_response": "Web Search Answer: The web search did not yield specific results regarding recent developments in machine learning. However, I can provide a general overview based on my knowledge up to October 2023.\n\nRecent development...", - "evaluation.search_effectiveness": 0.2, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 1, - "evaluation.context_relevance": "low", - "evaluation.overall_quality": "low", - "evaluation.ready_for_production": true, - "web_search.search_strategy": "duckduckgo_api", - "web_search.real_time_data": true, - "web_search.external_sources": 1, - "web_search.information_freshness": "current", - "web_search.knowledge_synthesis": "single_source" - }, - "events": [], - "links": [], - "trace_trace_id": "4bdd5814-acbb-4de7-a56b-543b87a1cfa5", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-14 12:53:20.120000000", - "trace_end_time": "2025-10-14 12:53:26.750000000", - "trace_duration_ms": 6630.38, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-14 12:53:20.120000000", - "updated_at": "2025-10-14 12:53:20.120000000" - } -] \ No newline at end of file diff --git a/noveum_customer_support_bt/traces/traces/dataset_filtered_mapped.json b/noveum_customer_support_bt/traces/traces/dataset_filtered_mapped.json deleted file mode 100644 index 5578d29..0000000 --- a/noveum_customer_support_bt/traces/traces/dataset_filtered_mapped.json +++ /dev/null @@ -1,20904 +0,0 @@ -[ - { - "span_id": "4149abdc-acdc-448c-b9b0-b09e166e461a", - "trace_id": "6fe93abf-1413-49e8-8464-58558e25ab7b", - "parent_span_id": "817c9703-0062-4bd6-b3ee-b50a1121c2d2", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-22 16:58:42.084000000", - "end_time": "2025-10-22 16:58:42.084000000", - "duration_ms": 0.015000000000000001, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What are the latest breakthroughs in artificial intelligence?", - "query_length": 61, - "query_type": "routing_evaluation_query", - "classification.mode": "Web Search", - "classification.rag_keyword_score": 0, - "classification.web_keyword_score": 1, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "complex", - "query.intent": "general_knowledge", - "query.keyword_density": 0.125, - "query.domain_affinity": "general", - "routing.decision": "Routed to Web Search based on analysis", - "routing.rationale": "RAG score: 0, Web score: 1, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "RAG", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.019230769230769232, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: Web Search (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "6fe93abf-1413-49e8-8464-58558e25ab7b", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:58:42.084000000", - "trace_end_time": "2025-10-22 16:58:44.766000000", - "trace_duration_ms": 2681.9289999999996, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:58:42.084000000", - "updated_at": "2025-10-22 16:58:42.084000000" - }, - { - "span_id": "817c9703-0062-4bd6-b3ee-b50a1121c2d2", - "trace_id": "6fe93abf-1413-49e8-8464-58558e25ab7b", - "parent_span_id": "ff171ffc-dc6b-4cb4-9833-64c5622d9941", - "name": "agent.query_routing", - "start_time": "2025-10-22 16:58:42.084000000", - "end_time": "2025-10-22 16:58:44.766000000", - "duration_ms": 2681.799, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What are the latest breakthroughs in artificial intelligence?", - "query_length": 61, - "query_type": "routing_query", - "output_response": "Routed to Web Search for query processing", - "router.classification": "Web Search", - "router.confidence_score": 1, - "router.rag_keyword_score": 0, - "router.web_keyword_score": 1, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "6fe93abf-1413-49e8-8464-58558e25ab7b", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:58:42.084000000", - "trace_end_time": "2025-10-22 16:58:44.766000000", - "trace_duration_ms": 2681.9289999999996, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:58:42.084000000", - "updated_at": "2025-10-22 16:58:42.084000000" - }, - { - "span_id": "ad476d62-d6b5-4142-8771-29855db3f39e", - "trace_id": "6fe93abf-1413-49e8-8464-58558e25ab7b", - "parent_span_id": "817c9703-0062-4bd6-b3ee-b50a1121c2d2", - "name": "agent.web_search_generation", - "start_time": "2025-10-22 16:58:42.084000000", - "end_time": "2025-10-22 16:58:44.766000000", - "duration_ms": 2681.732, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "web_search_agent", - "agent.operation": "web_search_generation", - "agent.capabilities": [ - "web_search", - "content_synthesis", - "response_generation" - ], - "agent.id": "noveum_web_search_agent", - "input_query": "What are the latest breakthroughs in artificial intelligence?", - "query_length": 61, - "query_type": "web_search_query", - "output_response": "Web Search Answer: The web search did not yield specific results regarding the latest breakthroughs in artificial intelligence. However, I can provide a general overview based on knowledge up to October 2023.\n\nRecent br...", - "web_search.results_count": 1, - "web_search.context_length": 163, - "web_search.response_length": 1040, - "web_search.mode": "external_web_search" - }, - "events": [], - "links": [], - "trace_trace_id": "6fe93abf-1413-49e8-8464-58558e25ab7b", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:58:42.084000000", - "trace_end_time": "2025-10-22 16:58:44.766000000", - "trace_duration_ms": 2681.9289999999996, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:58:42.084000000", - "updated_at": "2025-10-22 16:58:42.084000000" - }, - { - "span_id": "ff171ffc-dc6b-4cb4-9833-64c5622d9941", - "trace_id": "6fe93abf-1413-49e8-8464-58558e25ab7b", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-22 16:58:42.084000000", - "end_time": "2025-10-22 16:58:44.766000000", - "duration_ms": 2681.902, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What are the latest breakthroughs in artificial intelligence?", - "query.length": 61, - "processing.mode": "Web Search", - "processing.time_seconds": 2.6818161010742188, - "processing.response_length": 1040, - "processing.sources_count": 1, - "output_response": "Final Answer: The web search did not yield specific results regarding the latest breakthroughs in artificial intelligence. However, I can provide a general overview based on knowledge up to October 2023.\n\nRecent br...", - "final_answer_mode": "Web Search", - "query_processed.input_query": "What are the latest breakthroughs in artificial intelligence?", - "query_processed.output_response": "Successfully processed query using Web Search, generated 1040 character response", - "query_processed.mode": "Web Search", - "query_processed.processing_time": 2.6818161010742188, - "query_processed.response_length": 1040 - }, - "events": [], - "links": [], - "trace_trace_id": "6fe93abf-1413-49e8-8464-58558e25ab7b", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:58:42.084000000", - "trace_end_time": "2025-10-22 16:58:44.766000000", - "trace_duration_ms": 2681.9289999999996, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:58:42.084000000", - "updated_at": "2025-10-22 16:58:42.084000000" - }, - { - "span_id": "467cb806-a6ef-43da-9739-570730dd244b", - "trace_id": "6fe93abf-1413-49e8-8464-58558e25ab7b", - "parent_span_id": "ad476d62-d6b5-4142-8771-29855db3f39e", - "name": "agent.llm_model_execution", - "start_time": "2025-10-22 16:58:42.438000000", - "end_time": "2025-10-22 16:58:44.766000000", - "duration_ms": 2327.706, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What are the latest breakthroughs in artificial intelligence?", - "query_length": 61, - "query_type": "web_search_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 2.3276820182800293, - "model.latency_ms": 2327.6820182800293, - "model.start_time": 1761152322.438459, - "model.end_time": 1761152324.766141, - "model.performance_tier": "medium", - "model.prompt_tokens": 167, - "model.completion_tokens": 260, - "model.total_tokens": 427, - "model.tokens_per_second": 183.44430065903882, - "model.estimated_cost": 0.01281, - "model.efficiency_score": 2.4355971896955504, - "model.response_length": 1040, - "model.response_quality": "high", - "model.output_response": "Model Response: The web search did not yield specific results regarding the latest breakthroughs in artificial intelligence. However, I can provide a general overview based on knowledge up to October 2023.\n\nRecent br...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "6fe93abf-1413-49e8-8464-58558e25ab7b", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:58:42.084000000", - "trace_end_time": "2025-10-22 16:58:44.766000000", - "trace_duration_ms": 2681.9289999999996, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:58:42.084000000", - "updated_at": "2025-10-22 16:58:42.084000000" - }, - { - "span_id": "a80e4596-01ea-44e0-a590-13e347857f2a", - "trace_id": "6fe93abf-1413-49e8-8464-58558e25ab7b", - "parent_span_id": "ad476d62-d6b5-4142-8771-29855db3f39e", - "name": "agent.web_search_evaluation_metrics", - "start_time": "2025-10-22 16:58:44.766000000", - "end_time": "2025-10-22 16:58:44.766000000", - "duration_ms": 0.008, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "web_search_evaluation_metrics", - "agent.capabilities": [ - "web_search_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What are the latest breakthroughs in artificial intelligence?", - "query_length": 61, - "query_type": "web_search_evaluation_query", - "web_search.results_count": 1, - "web_search.context_length": 163, - "web_search.context_synthesized": "Context from 1 web sources", - "web_search.search_effectiveness": 0.2, - "web_search.context_quality": "low", - "web_search.source_diversity": 1, - "web_search.information_synthesis": "low", - "web_search.external_knowledge_utilization": 0.163, - "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What+are+the+latest+breakthroughs+in+artificial+intelligence?\n\nUser Question: What are the latest breakthroughs in artificial intelligence?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", - "prompt.prompt_length": 669, - "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What+are+the+latest+breakthroughs+in+artificial+intelligence?", - "prompt.instruction_following": "web_search_optimized", - "response.answer_length": 1040, - "response.answer_completeness": "complete", - "response.response_quality": "medium", - "response.source_citation": 1, - "response.context_utilization": 6.380368098159509, - "output_response": "Web Search Answer: The web search did not yield specific results regarding the latest breakthroughs in artificial intelligence. However, I can provide a general overview based on knowledge up to October 2023.\n\nRecent br...", - "evaluation.search_effectiveness": 0.2, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 1, - "evaluation.context_relevance": "low", - "evaluation.overall_quality": "low", - "evaluation.ready_for_production": true, - "web_search.search_strategy": "duckduckgo_api", - "web_search.real_time_data": true, - "web_search.external_sources": 1, - "web_search.information_freshness": "current", - "web_search.knowledge_synthesis": "single_source" - }, - "events": [], - "links": [], - "trace_trace_id": "6fe93abf-1413-49e8-8464-58558e25ab7b", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:58:42.084000000", - "trace_end_time": "2025-10-22 16:58:44.766000000", - "trace_duration_ms": 2681.9289999999996, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:58:42.084000000", - "updated_at": "2025-10-22 16:58:42.084000000" - }, - { - "span_id": "409f29f6-21c7-400a-a9e0-11c87c4313cb", - "trace_id": "ec7c6cae-9bbd-47f4-a8ae-2d6d9b5011ba", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-22 16:58:32.801000000", - "end_time": "2025-10-22 16:58:41.077000000", - "duration_ms": 8276.701, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What's happening in the software development world today?", - "query.length": 57, - "processing.mode": "Web Search", - "processing.time_seconds": 8.276506900787354, - "processing.response_length": 1424, - "processing.sources_count": 1, - "output_response": "Final Answer: The web search did not yield any specific results regarding current events in the software development world. Therefore, I cannot provide detailed information on the latest trends, technologies, or ne...", - "final_answer_mode": "Web Search", - "query_processed.input_query": "What's happening in the software development world today?", - "query_processed.output_response": "Successfully processed query using Web Search, generated 1424 character response", - "query_processed.mode": "Web Search", - "query_processed.processing_time": 8.276506900787354, - "query_processed.response_length": 1424 - }, - "events": [], - "links": [], - "trace_trace_id": "ec7c6cae-9bbd-47f4-a8ae-2d6d9b5011ba", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:58:32.800000000", - "trace_end_time": "2025-10-22 16:58:41.077000000", - "trace_duration_ms": 8276.788, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:58:32.800000000", - "updated_at": "2025-10-22 16:58:32.800000000" - }, - { - "span_id": "43389a18-a566-41e7-8db5-f688f651a106", - "trace_id": "ec7c6cae-9bbd-47f4-a8ae-2d6d9b5011ba", - "parent_span_id": "af400e17-8b20-4218-834b-9330ccaa2635", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-22 16:58:32.801000000", - "end_time": "2025-10-22 16:58:32.801000000", - "duration_ms": 0.047, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What's happening in the software development world today?", - "query_length": 57, - "query_type": "routing_evaluation_query", - "classification.mode": "Web Search", - "classification.rag_keyword_score": 0, - "classification.web_keyword_score": 1, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "complex", - "query.intent": "general_knowledge", - "query.keyword_density": 0.125, - "query.domain_affinity": "general", - "routing.decision": "Routed to Web Search based on analysis", - "routing.rationale": "RAG score: 0, Web score: 1, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "RAG", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.019230769230769232, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: Web Search (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "ec7c6cae-9bbd-47f4-a8ae-2d6d9b5011ba", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:58:32.800000000", - "trace_end_time": "2025-10-22 16:58:41.077000000", - "trace_duration_ms": 8276.788, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:58:32.800000000", - "updated_at": "2025-10-22 16:58:32.800000000" - }, - { - "span_id": "6b431bca-af7e-4f7e-aedb-b103d7f47cae", - "trace_id": "ec7c6cae-9bbd-47f4-a8ae-2d6d9b5011ba", - "parent_span_id": "af400e17-8b20-4218-834b-9330ccaa2635", - "name": "agent.web_search_generation", - "start_time": "2025-10-22 16:58:32.801000000", - "end_time": "2025-10-22 16:58:41.077000000", - "duration_ms": 8276.225999999999, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "web_search_agent", - "agent.operation": "web_search_generation", - "agent.capabilities": [ - "web_search", - "content_synthesis", - "response_generation" - ], - "agent.id": "noveum_web_search_agent", - "input_query": "What's happening in the software development world today?", - "query_length": 57, - "query_type": "web_search_query", - "output_response": "Web Search Answer: The web search did not yield any specific results regarding current events in the software development world. Therefore, I cannot provide detailed information on the latest trends, technologies, or ne...", - "web_search.results_count": 1, - "web_search.context_length": 159, - "web_search.response_length": 1424, - "web_search.mode": "external_web_search" - }, - "events": [], - "links": [], - "trace_trace_id": "ec7c6cae-9bbd-47f4-a8ae-2d6d9b5011ba", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:58:32.800000000", - "trace_end_time": "2025-10-22 16:58:41.077000000", - "trace_duration_ms": 8276.788, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:58:32.800000000", - "updated_at": "2025-10-22 16:58:32.800000000" - }, - { - "span_id": "af400e17-8b20-4218-834b-9330ccaa2635", - "trace_id": "ec7c6cae-9bbd-47f4-a8ae-2d6d9b5011ba", - "parent_span_id": "409f29f6-21c7-400a-a9e0-11c87c4313cb", - "name": "agent.query_routing", - "start_time": "2025-10-22 16:58:32.801000000", - "end_time": "2025-10-22 16:58:41.077000000", - "duration_ms": 8276.465999999999, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What's happening in the software development world today?", - "query_length": 57, - "query_type": "routing_query", - "output_response": "Routed to Web Search for query processing", - "router.classification": "Web Search", - "router.confidence_score": 1, - "router.rag_keyword_score": 0, - "router.web_keyword_score": 1, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "ec7c6cae-9bbd-47f4-a8ae-2d6d9b5011ba", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:58:32.800000000", - "trace_end_time": "2025-10-22 16:58:41.077000000", - "trace_duration_ms": 8276.788, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:58:32.800000000", - "updated_at": "2025-10-22 16:58:32.800000000" - }, - { - "span_id": "37b153b2-d02b-4b7e-9229-1ba3eaae44ee", - "trace_id": "ec7c6cae-9bbd-47f4-a8ae-2d6d9b5011ba", - "parent_span_id": "6b431bca-af7e-4f7e-aedb-b103d7f47cae", - "name": "agent.llm_model_execution", - "start_time": "2025-10-22 16:58:33.172000000", - "end_time": "2025-10-22 16:58:41.077000000", - "duration_ms": 7904.722, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What's happening in the software development world today?", - "query_length": 57, - "query_type": "web_search_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 7.904651880264282, - "model.latency_ms": 7904.651880264282, - "model.start_time": 1761152313.172655, - "model.end_time": 1761152321.077307, - "model.performance_tier": "slow", - "model.prompt_tokens": 165, - "model.completion_tokens": 356, - "model.total_tokens": 521, - "model.tokens_per_second": 65.91055594754174, - "model.estimated_cost": 0.01563, - "model.efficiency_score": 2.7332053742802302, - "model.response_length": 1424, - "model.response_quality": "high", - "model.output_response": "Model Response: The web search did not yield any specific results regarding current events in the software development world. Therefore, I cannot provide detailed information on the latest trends, technologies, or ne...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "ec7c6cae-9bbd-47f4-a8ae-2d6d9b5011ba", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:58:32.800000000", - "trace_end_time": "2025-10-22 16:58:41.077000000", - "trace_duration_ms": 8276.788, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:58:32.800000000", - "updated_at": "2025-10-22 16:58:32.800000000" - }, - { - "span_id": "cc941b8e-6625-43e9-a0a9-7328af24f12f", - "trace_id": "ec7c6cae-9bbd-47f4-a8ae-2d6d9b5011ba", - "parent_span_id": "6b431bca-af7e-4f7e-aedb-b103d7f47cae", - "name": "agent.web_search_evaluation_metrics", - "start_time": "2025-10-22 16:58:41.077000000", - "end_time": "2025-10-22 16:58:41.077000000", - "duration_ms": 0.026, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "web_search_evaluation_metrics", - "agent.capabilities": [ - "web_search_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What's happening in the software development world today?", - "query_length": 57, - "query_type": "web_search_evaluation_query", - "web_search.results_count": 1, - "web_search.context_length": 159, - "web_search.context_synthesized": "Context from 1 web sources", - "web_search.search_effectiveness": 0.2, - "web_search.context_quality": "low", - "web_search.source_diversity": 1, - "web_search.information_synthesis": "low", - "web_search.external_knowledge_utilization": 0.159, - "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What's+happening+in+the+software+development+world+today?\n\nUser Question: What's happening in the software development world today?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", - "prompt.prompt_length": 661, - "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What's+happening+in+the+software+development+world+today?", - "prompt.instruction_following": "web_search_optimized", - "response.answer_length": 1424, - "response.answer_completeness": "complete", - "response.response_quality": "medium", - "response.source_citation": 1, - "response.context_utilization": 8.955974842767295, - "output_response": "Web Search Answer: The web search did not yield any specific results regarding current events in the software development world. Therefore, I cannot provide detailed information on the latest trends, technologies, or ne...", - "evaluation.search_effectiveness": 0.2, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 1, - "evaluation.context_relevance": "low", - "evaluation.overall_quality": "low", - "evaluation.ready_for_production": true, - "web_search.search_strategy": "duckduckgo_api", - "web_search.real_time_data": true, - "web_search.external_sources": 1, - "web_search.information_freshness": "current", - "web_search.knowledge_synthesis": "single_source" - }, - "events": [], - "links": [], - "trace_trace_id": "ec7c6cae-9bbd-47f4-a8ae-2d6d9b5011ba", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:58:32.800000000", - "trace_end_time": "2025-10-22 16:58:41.077000000", - "trace_duration_ms": 8276.788, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:58:32.800000000", - "updated_at": "2025-10-22 16:58:32.800000000" - }, - { - "span_id": "3b06d098-6eb7-4bb6-885e-443ddb28cc41", - "trace_id": "343764b8-7ac9-4549-93cf-8abc11ceba6f", - "parent_span_id": "5c2aa1cf-56a8-4fd5-af90-90a94e723bb5", - "name": "agent.query_routing", - "start_time": "2025-10-22 16:58:26.542000000", - "end_time": "2025-10-22 16:58:31.796000000", - "duration_ms": 5253.835, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What are the newest features in cloud computing?", - "query_length": 48, - "query_type": "routing_query", - "output_response": "Routed to RAG for query processing", - "router.classification": "RAG", - "router.confidence_score": 1, - "router.rag_keyword_score": 1, - "router.web_keyword_score": 0, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "343764b8-7ac9-4549-93cf-8abc11ceba6f", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:58:26.542000000", - "trace_end_time": "2025-10-22 16:58:31.796000000", - "trace_duration_ms": 5253.973, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:58:26.542000000", - "updated_at": "2025-10-22 16:58:26.542000000" - }, - { - "span_id": "5c2aa1cf-56a8-4fd5-af90-90a94e723bb5", - "trace_id": "343764b8-7ac9-4549-93cf-8abc11ceba6f", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-22 16:58:26.542000000", - "end_time": "2025-10-22 16:58:31.796000000", - "duration_ms": 5253.913, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What are the newest features in cloud computing?", - "query.length": 48, - "processing.mode": "RAG", - "processing.time_seconds": 5.253868818283081, - "processing.response_length": 837, - "processing.sources_count": 5, - "output_response": "Final Answer: The provided context does not specifically mention any new features in cloud computing as a general topic. However, it does highlight several recent enhancements related to the Noveum.ai platform, whi...", - "final_answer_mode": "RAG", - "query_processed.input_query": "What are the newest features in cloud computing?", - "query_processed.output_response": "Successfully processed query using RAG, generated 837 character response", - "query_processed.mode": "RAG", - "query_processed.processing_time": 5.253868818283081, - "query_processed.response_length": 837 - }, - "events": [], - "links": [], - "trace_trace_id": "343764b8-7ac9-4549-93cf-8abc11ceba6f", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:58:26.542000000", - "trace_end_time": "2025-10-22 16:58:31.796000000", - "trace_duration_ms": 5253.973, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:58:26.542000000", - "updated_at": "2025-10-22 16:58:26.542000000" - }, - { - "span_id": "6b838ed6-f5bb-4bcf-a76b-bac283bf7571", - "trace_id": "343764b8-7ac9-4549-93cf-8abc11ceba6f", - "parent_span_id": "3b06d098-6eb7-4bb6-885e-443ddb28cc41", - "name": "agent.llm-rag", - "start_time": "2025-10-22 16:58:26.542000000", - "end_time": "2025-10-22 16:58:31.796000000", - "duration_ms": 5253.519, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "rag_agent", - "agent.operation": "llm-rag", - "agent.capabilities": [ - "document_retrieval", - "context_generation", - "response_generation" - ], - "agent.id": "noveum_rag_agent", - "input_query": "What are the newest features in cloud computing?", - "query_length": 48, - "query_type": "rag_query", - "output_response": "RAG Answer: The provided context does not specifically mention any new features in cloud computing as a general topic. However, it does highlight several recent enhancements related to the Noveum.ai platform, whi...", - "rag.context_length": 2579, - "rag.sources_count": 5, - "rag.answer_length": 837, - "rag.mode": "retrieval_augmented_generation" - }, - "events": [], - "links": [], - "trace_trace_id": "343764b8-7ac9-4549-93cf-8abc11ceba6f", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:58:26.542000000", - "trace_end_time": "2025-10-22 16:58:31.796000000", - "trace_duration_ms": 5253.973, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:58:26.542000000", - "updated_at": "2025-10-22 16:58:26.542000000" - }, - { - "span_id": "f51d8f1e-eae3-488d-9a96-49ab97377851", - "trace_id": "343764b8-7ac9-4549-93cf-8abc11ceba6f", - "parent_span_id": "3b06d098-6eb7-4bb6-885e-443ddb28cc41", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-22 16:58:26.542000000", - "end_time": "2025-10-22 16:58:26.542000000", - "duration_ms": 0.082, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What are the newest features in cloud computing?", - "query_length": 48, - "query_type": "routing_evaluation_query", - "classification.mode": "RAG", - "classification.rag_keyword_score": 1, - "classification.web_keyword_score": 0, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "documentation", - "query.keyword_density": 0.125, - "query.domain_affinity": "noveum", - "routing.decision": "Routed to RAG based on analysis", - "routing.rationale": "RAG score: 1, Web score: 0, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "Web Search", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.019230769230769232, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: RAG (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "343764b8-7ac9-4549-93cf-8abc11ceba6f", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:58:26.542000000", - "trace_end_time": "2025-10-22 16:58:31.796000000", - "trace_duration_ms": 5253.973, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:58:26.542000000", - "updated_at": "2025-10-22 16:58:26.542000000" - }, - { - "span_id": "93a0fd69-47b6-4e7c-b8ab-8eaa5aeac240", - "trace_id": "343764b8-7ac9-4549-93cf-8abc11ceba6f", - "parent_span_id": "6b838ed6-f5bb-4bcf-a76b-bac283bf7571", - "name": "agent.llm_model_execution", - "start_time": "2025-10-22 16:58:27.411000000", - "end_time": "2025-10-22 16:58:31.373000000", - "duration_ms": 3961.481, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What are the newest features in cloud computing?", - "query_length": 48, - "query_type": "rag_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 3.9613850116729736, - "model.latency_ms": 3961.3850116729736, - "model.start_time": 1761152307.411936, - "model.end_time": 1761152311.373321, - "model.performance_tier": "medium", - "model.prompt_tokens": 780, - "model.completion_tokens": 209, - "model.total_tokens": 989, - "model.tokens_per_second": 249.66015600243944, - "model.estimated_cost": 0.029670000000000002, - "model.efficiency_score": 0.846309403437816, - "model.response_length": 837, - "model.response_quality": "high", - "model.output_response": "Model Response: The provided context does not specifically mention any new features in cloud computing as a general topic. However, it does highlight several recent enhancements related to the Noveum.ai platform, whi...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "343764b8-7ac9-4549-93cf-8abc11ceba6f", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:58:26.542000000", - "trace_end_time": "2025-10-22 16:58:31.796000000", - "trace_duration_ms": 5253.973, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:58:26.542000000", - "updated_at": "2025-10-22 16:58:26.542000000" - }, - { - "span_id": "48cbc1df-f03a-4c49-90e5-19a2047c1c7b", - "trace_id": "343764b8-7ac9-4549-93cf-8abc11ceba6f", - "parent_span_id": "6b838ed6-f5bb-4bcf-a76b-bac283bf7571", - "name": "agent.rag_evaluation_metrics", - "start_time": "2025-10-22 16:58:31.373000000", - "end_time": "2025-10-22 16:58:31.373000000", - "duration_ms": 0.061, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "rag_evaluation_metrics", - "agent.capabilities": [ - "retrieval_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What are the newest features in cloud computing?", - "query_length": 48, - "query_type": "rag_evaluation_query", - "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/changelog):\n- \ud83d\udcc8 Enhanced Dashboard Analytics: Real-time request tracking, latency monitoring, and cost analysis\n- \ud83d\udd0d Improved Logs Interface: Better search functionality, detailed trace views, and enhanced debugging capabilities\n- \ud83d\udc65 Team Management: Advanced member mana...", - "retrieval.context_length": 2579, - "retrieval.sources_count": 5, - "retrieval.context_quality": "high", - "retrieval.effectiveness": 1, - "retrieval.context_utilization": 2.579, - "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/changelog):\n- \ud83d\udcc8 Enhanced Dashboard Analytics: Real-time request tracking, latency monitoring, and cost analysis\n- \ud83d\udd0d Improved Logs Interface: Better search functionality, detailed trace views, and enhanced debugging capabilities\n- \ud83d\udc65 Team Management: Advanced member management with role-based access control and invitation system\n- \ud83d\udee1\ufe0f Security Improvements: Enhanced API key management and secure credential storage\n4 months ago\n- \ud83d\udcca Advanced Metrics Collection: Comprehensive telemetry system with custom metrics ...\n\nSource 2 (https://noveum.ai/en/changelog):\n- \ud83d\udcdd Documentation Updates: Comprehensive guides for integration and best practices\n5 months ago\n- \ud83c\udf89 Platform Foundation: Initial release of Noveum.ai AI observability platform\n- \ud83d\ude80 Core Features: AI Gateway integration, basic evaluation metrics, and monitoring capabilities\n- \ud83d\udcc8 Evaluation Jobs: Automated model evaluation with configurable metrics and reporting\n- \ud83d\udd0c Provider Integrations: Support for major AI providers and custom model deployments...\n\nSource 3 (https://noveum.ai/docs/platform/dashboard):\n- Bulk Export: Download multiple traces simultaneously\n- Comparative Analysis: Compare performance across multiple traces\nReal-Time Updates\n- Live Refresh: Automatic updates as new traces arrive\n- Connection Monitoring: Real-time status of your trace ingestion pipeline\n- Performance Indicators: Live metrics for system health monitoring\nReady to dive deeper? Explore Projects & Environments to organize your AI applications, or check out Team Collaboration to share insights with your team.\nGet Earl...\n\nSource 4 (https://noveum.ai/en/changelog):\nChangelog\nStay up to date with the latest changes in our product.\n2 months ago\n- \ud83c\udfaf Major Platform Evolution: NovaEval Framework - Launched comprehensive AI model evaluation framework with 20+ built-in scorers including accuracy, RAG metrics, conversational scoring, and specialized agent evaluation tools. Features production-grade deployment on Docker and Kubernetes. (GitHub | PyPI Package)\n- \ud83d\ude80 Next-Generation Traces UI - Complete redesign featuring three-pane layout with directory tree navigatio...\n\nSource 5 (https://noveum.ai/en/blog):\n8/12/2025\nDiscover how Noveum.ai provides comprehensive tracing and observability for AI applications, from development debugging to production optimization.\nShashank Agarwal\n3/3/2025\nDiscover how Noveum.ai provides comprehensive tracing and observability for LLM applications, RAG systems, and multi-agent workflows with our powerful Python and TypeScript SDKs.\nShashank Agarwal\n3/2/2025...\n\nUser Question: What are the newest features in cloud computing?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", - "prompt.prompt_length": 3123, - "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/changelog):\n- \ud83d\udcc8 Enhanced Dashboard Analytics: Real-time request tracking, latency monitoring, and cost analysis\n- \ud83d\udd0d Improved Logs Interface: Better search functionality,...", - "prompt.instruction_following": "rag_optimized", - "response.answer_length": 837, - "response.answer_completeness": "complete", - "response.response_quality": "high", - "response.source_citation": 5, - "response.context_utilization": 0.3245443970531214, - "output_response": "RAG Answer: The provided context does not specifically mention any new features in cloud computing as a general topic. However, it does highlight several recent enhancements related to the Noveum.ai platform, whi...", - "evaluation.retrieval_effectiveness": 1, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 5, - "evaluation.context_relevance": "high", - "evaluation.overall_quality": "high", - "evaluation.ready_for_production": true, - "rag.retrieval_strategy": "semantic_similarity", - "rag.vector_search_results": 5, - "rag.context_synthesis": "multi_source", - "rag.document_coverage": 1, - "rag.information_density": 0.3245443970531214 - }, - "events": [], - "links": [], - "trace_trace_id": "343764b8-7ac9-4549-93cf-8abc11ceba6f", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:58:26.542000000", - "trace_end_time": "2025-10-22 16:58:31.796000000", - "trace_duration_ms": 5253.973, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:58:26.542000000", - "updated_at": "2025-10-22 16:58:26.542000000" - }, - { - "span_id": "7384d697-d7bf-4709-8377-ec26b094b550", - "trace_id": "3f7b6a62-8888-4a5d-b12c-37a3829aea49", - "parent_span_id": "e631b22a-a621-4ecc-bb01-f731ca66f3d0", - "name": "agent.query_routing", - "start_time": "2025-10-22 16:58:19.885000000", - "end_time": "2025-10-22 16:58:25.536000000", - "duration_ms": 5651.335999999999, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What's the current status of cryptocurrency markets?", - "query_length": 52, - "query_type": "routing_query", - "output_response": "Routed to Web Search for query processing", - "router.classification": "Web Search", - "router.confidence_score": 1, - "router.rag_keyword_score": 0, - "router.web_keyword_score": 3, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "3f7b6a62-8888-4a5d-b12c-37a3829aea49", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:58:19.885000000", - "trace_end_time": "2025-10-22 16:58:25.536000000", - "trace_duration_ms": 5651.67, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:58:19.885000000", - "updated_at": "2025-10-22 16:58:19.885000000" - }, - { - "span_id": "7abf372d-d9f4-41d1-9e3a-d4f01173fa5e", - "trace_id": "3f7b6a62-8888-4a5d-b12c-37a3829aea49", - "parent_span_id": "7384d697-d7bf-4709-8377-ec26b094b550", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-22 16:58:19.885000000", - "end_time": "2025-10-22 16:58:19.885000000", - "duration_ms": 0.037, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What's the current status of cryptocurrency markets?", - "query_length": 52, - "query_type": "routing_evaluation_query", - "classification.mode": "Web Search", - "classification.rag_keyword_score": 0, - "classification.web_keyword_score": 3, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "complex", - "query.intent": "general_knowledge", - "query.keyword_density": 0.42857142857142855, - "query.domain_affinity": "general", - "routing.decision": "Routed to Web Search based on analysis", - "routing.rationale": "RAG score: 0, Web score: 3, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "RAG", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.057692307692307696, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: Web Search (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "3f7b6a62-8888-4a5d-b12c-37a3829aea49", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:58:19.885000000", - "trace_end_time": "2025-10-22 16:58:25.536000000", - "trace_duration_ms": 5651.67, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:58:19.885000000", - "updated_at": "2025-10-22 16:58:19.885000000" - }, - { - "span_id": "e5d9c3c3-6c18-45d2-a6b6-58bde4d4e461", - "trace_id": "3f7b6a62-8888-4a5d-b12c-37a3829aea49", - "parent_span_id": "7384d697-d7bf-4709-8377-ec26b094b550", - "name": "agent.web_search_generation", - "start_time": "2025-10-22 16:58:19.885000000", - "end_time": "2025-10-22 16:58:25.536000000", - "duration_ms": 5651.161, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "web_search_agent", - "agent.operation": "web_search_generation", - "agent.capabilities": [ - "web_search", - "content_synthesis", - "response_generation" - ], - "agent.id": "noveum_web_search_agent", - "input_query": "What's the current status of cryptocurrency markets?", - "query_length": 52, - "query_type": "web_search_query", - "output_response": "Web Search Answer: The provided web search results do not contain specific information regarding the current status of cryptocurrency markets. To get the latest updates on cryptocurrency prices, trends, and market analy...", - "web_search.results_count": 1, - "web_search.context_length": 251, - "web_search.response_length": 416, - "web_search.mode": "external_web_search" - }, - "events": [], - "links": [], - "trace_trace_id": "3f7b6a62-8888-4a5d-b12c-37a3829aea49", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:58:19.885000000", - "trace_end_time": "2025-10-22 16:58:25.536000000", - "trace_duration_ms": 5651.67, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:58:19.885000000", - "updated_at": "2025-10-22 16:58:19.885000000" - }, - { - "span_id": "e631b22a-a621-4ecc-bb01-f731ca66f3d0", - "trace_id": "3f7b6a62-8888-4a5d-b12c-37a3829aea49", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-22 16:58:19.885000000", - "end_time": "2025-10-22 16:58:25.536000000", - "duration_ms": 5651.588000000001, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What's the current status of cryptocurrency markets?", - "query.length": 52, - "processing.mode": "Web Search", - "processing.time_seconds": 5.651373863220215, - "processing.response_length": 416, - "processing.sources_count": 1, - "output_response": "Final Answer: The provided web search results do not contain specific information regarding the current status of cryptocurrency markets. To get the latest updates on cryptocurrency prices, trends, and market analy...", - "final_answer_mode": "Web Search", - "query_processed.input_query": "What's the current status of cryptocurrency markets?", - "query_processed.output_response": "Successfully processed query using Web Search, generated 416 character response", - "query_processed.mode": "Web Search", - "query_processed.processing_time": 5.651373863220215, - "query_processed.response_length": 416 - }, - "events": [], - "links": [], - "trace_trace_id": "3f7b6a62-8888-4a5d-b12c-37a3829aea49", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:58:19.885000000", - "trace_end_time": "2025-10-22 16:58:25.536000000", - "trace_duration_ms": 5651.67, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:58:19.885000000", - "updated_at": "2025-10-22 16:58:19.885000000" - }, - { - "span_id": "34e2f906-724e-4116-b61f-82eaa6b75959", - "trace_id": "3f7b6a62-8888-4a5d-b12c-37a3829aea49", - "parent_span_id": "e5d9c3c3-6c18-45d2-a6b6-58bde4d4e461", - "name": "agent.llm_model_execution", - "start_time": "2025-10-22 16:58:24.327000000", - "end_time": "2025-10-22 16:58:25.536000000", - "duration_ms": 1208.66, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What's the current status of cryptocurrency markets?", - "query_length": 52, - "query_type": "web_search_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 1.208608865737915, - "model.latency_ms": 1208.608865737915, - "model.start_time": 1761152304.327749, - "model.end_time": 1761152305.5363579, - "model.performance_tier": "fast", - "model.prompt_tokens": 187, - "model.completion_tokens": 104, - "model.total_tokens": 291, - "model.tokens_per_second": 240.77268357809888, - "model.estimated_cost": 0.00873, - "model.efficiency_score": 1.429553264604811, - "model.response_length": 416, - "model.response_quality": "high", - "model.output_response": "Model Response: The provided web search results do not contain specific information regarding the current status of cryptocurrency markets. To get the latest updates on cryptocurrency prices, trends, and market analy...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "3f7b6a62-8888-4a5d-b12c-37a3829aea49", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:58:19.885000000", - "trace_end_time": "2025-10-22 16:58:25.536000000", - "trace_duration_ms": 5651.67, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:58:19.885000000", - "updated_at": "2025-10-22 16:58:19.885000000" - }, - { - "span_id": "5548e02e-2ea8-472c-9c42-f811d6e4d30b", - "trace_id": "3f7b6a62-8888-4a5d-b12c-37a3829aea49", - "parent_span_id": "e5d9c3c3-6c18-45d2-a6b6-58bde4d4e461", - "name": "agent.web_search_evaluation_metrics", - "start_time": "2025-10-22 16:58:25.536000000", - "end_time": "2025-10-22 16:58:25.536000000", - "duration_ms": 0.044, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "web_search_evaluation_metrics", - "agent.capabilities": [ - "web_search_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What's the current status of cryptocurrency markets?", - "query_length": 52, - "query_type": "web_search_evaluation_query", - "web_search.results_count": 1, - "web_search.context_length": 251, - "web_search.context_synthesized": "Context from 1 web sources", - "web_search.search_effectiveness": 0.2, - "web_search.context_quality": "low", - "web_search.source_diversity": 1, - "web_search.information_synthesis": "low", - "web_search.external_knowledge_utilization": 0.251, - "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nLog in to WhatsApp Web for simple, reliable and private messaging on your desktop. Send and receive messages and files with ease, all for \u2026\nURL: https://duckduckgo.com/?q=What's+the+current+status+of+cryptocurrency+markets?\n\nUser Question: What's the current status of cryptocurrency markets?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", - "prompt.prompt_length": 748, - "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nLog in to WhatsApp Web for simple, reliable and private messaging on your desktop. Send and receive messages and files with ease, all for \u2026\nURL: https://duckduckgo.com/?q=W...", - "prompt.instruction_following": "web_search_optimized", - "response.answer_length": 416, - "response.answer_completeness": "complete", - "response.response_quality": "medium", - "response.source_citation": 1, - "response.context_utilization": 1.657370517928287, - "output_response": "Web Search Answer: The provided web search results do not contain specific information regarding the current status of cryptocurrency markets. To get the latest updates on cryptocurrency prices, trends, and market analy...", - "evaluation.search_effectiveness": 0.2, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 1, - "evaluation.context_relevance": "low", - "evaluation.overall_quality": "low", - "evaluation.ready_for_production": true, - "web_search.search_strategy": "duckduckgo_api", - "web_search.real_time_data": true, - "web_search.external_sources": 1, - "web_search.information_freshness": "current", - "web_search.knowledge_synthesis": "single_source" - }, - "events": [], - "links": [], - "trace_trace_id": "3f7b6a62-8888-4a5d-b12c-37a3829aea49", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:58:19.885000000", - "trace_end_time": "2025-10-22 16:58:25.536000000", - "trace_duration_ms": 5651.67, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:58:19.885000000", - "updated_at": "2025-10-22 16:58:19.885000000" - }, - { - "span_id": "a9e34cea-d4bd-478b-abf2-7a3a38dcd40e", - "trace_id": "51f4534f-23e4-4be7-8d6e-e93391d6e0e2", - "parent_span_id": "8e9b2d06-3e11-4d17-9a32-b1529a2c10a4", - "name": "agent.web_search_generation", - "start_time": "2025-10-22 16:58:15.254000000", - "end_time": "2025-10-22 16:58:18.876000000", - "duration_ms": 3621.674, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "web_search_agent", - "agent.operation": "web_search_generation", - "agent.capabilities": [ - "web_search", - "content_synthesis", - "response_generation" - ], - "agent.id": "noveum_web_search_agent", - "input_query": "What are the latest updates in Python programming?", - "query_length": 50, - "query_type": "web_search_query", - "output_response": "Web Search Answer: The provided web search results do not contain specific information about the latest updates in Python programming. They primarily focus on general news coverage and updates from India and around the ...", - "web_search.results_count": 1, - "web_search.context_length": 753, - "web_search.response_length": 580, - "web_search.mode": "external_web_search" - }, - "events": [], - "links": [], - "trace_trace_id": "51f4534f-23e4-4be7-8d6e-e93391d6e0e2", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:58:15.254000000", - "trace_end_time": "2025-10-22 16:58:18.877000000", - "trace_duration_ms": 3622.355, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:58:15.254000000", - "updated_at": "2025-10-22 16:58:15.254000000" - }, - { - "span_id": "8e9b2d06-3e11-4d17-9a32-b1529a2c10a4", - "trace_id": "51f4534f-23e4-4be7-8d6e-e93391d6e0e2", - "parent_span_id": "bda936a0-24f3-4841-af4f-227197e10b9d", - "name": "agent.query_routing", - "start_time": "2025-10-22 16:58:15.254000000", - "end_time": "2025-10-22 16:58:18.876000000", - "duration_ms": 3621.863, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What are the latest updates in Python programming?", - "query_length": 50, - "query_type": "routing_query", - "output_response": "Routed to Web Search for query processing", - "router.classification": "Web Search", - "router.confidence_score": 1, - "router.rag_keyword_score": 0, - "router.web_keyword_score": 2, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "51f4534f-23e4-4be7-8d6e-e93391d6e0e2", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:58:15.254000000", - "trace_end_time": "2025-10-22 16:58:18.877000000", - "trace_duration_ms": 3622.355, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:58:15.254000000", - "updated_at": "2025-10-22 16:58:15.254000000" - }, - { - "span_id": "bda936a0-24f3-4841-af4f-227197e10b9d", - "trace_id": "51f4534f-23e4-4be7-8d6e-e93391d6e0e2", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-22 16:58:15.254000000", - "end_time": "2025-10-22 16:58:18.877000000", - "duration_ms": 3622.271, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What are the latest updates in Python programming?", - "query.length": 50, - "processing.mode": "Web Search", - "processing.time_seconds": 3.6219100952148438, - "processing.response_length": 580, - "processing.sources_count": 1, - "output_response": "Final Answer: The provided web search results do not contain specific information about the latest updates in Python programming. They primarily focus on general news coverage and updates from India and around the ...", - "final_answer_mode": "Web Search", - "query_processed.input_query": "What are the latest updates in Python programming?", - "query_processed.output_response": "Successfully processed query using Web Search, generated 580 character response", - "query_processed.mode": "Web Search", - "query_processed.processing_time": 3.6219100952148438, - "query_processed.response_length": 580 - }, - "events": [], - "links": [], - "trace_trace_id": "51f4534f-23e4-4be7-8d6e-e93391d6e0e2", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:58:15.254000000", - "trace_end_time": "2025-10-22 16:58:18.877000000", - "trace_duration_ms": 3622.355, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:58:15.254000000", - "updated_at": "2025-10-22 16:58:15.254000000" - }, - { - "span_id": "e27e3812-d49d-4f4f-98b0-144cf0ceffc3", - "trace_id": "51f4534f-23e4-4be7-8d6e-e93391d6e0e2", - "parent_span_id": "8e9b2d06-3e11-4d17-9a32-b1529a2c10a4", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-22 16:58:15.254000000", - "end_time": "2025-10-22 16:58:15.254000000", - "duration_ms": 0.047, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What are the latest updates in Python programming?", - "query_length": 50, - "query_type": "routing_evaluation_query", - "classification.mode": "Web Search", - "classification.rag_keyword_score": 0, - "classification.web_keyword_score": 2, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "general_knowledge", - "query.keyword_density": 0.25, - "query.domain_affinity": "general", - "routing.decision": "Routed to Web Search based on analysis", - "routing.rationale": "RAG score: 0, Web score: 2, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "RAG", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.038461538461538464, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: Web Search (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "51f4534f-23e4-4be7-8d6e-e93391d6e0e2", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:58:15.254000000", - "trace_end_time": "2025-10-22 16:58:18.877000000", - "trace_duration_ms": 3622.355, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:58:15.254000000", - "updated_at": "2025-10-22 16:58:15.254000000" - }, - { - "span_id": "b0eea8ec-5cff-4667-bea7-aeb6d0f0daf3", - "trace_id": "51f4534f-23e4-4be7-8d6e-e93391d6e0e2", - "parent_span_id": "a9e34cea-d4bd-478b-abf2-7a3a38dcd40e", - "name": "agent.llm_model_execution", - "start_time": "2025-10-22 16:58:15.568000000", - "end_time": "2025-10-22 16:58:18.876000000", - "duration_ms": 3307.5739999999996, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What are the latest updates in Python programming?", - "query_length": 50, - "query_type": "web_search_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 3.307521104812622, - "model.latency_ms": 3307.521104812622, - "model.start_time": 1761152295.568949, - "model.end_time": 1761152298.87647, - "model.performance_tier": "medium", - "model.prompt_tokens": 312, - "model.completion_tokens": 145, - "model.total_tokens": 457, - "model.tokens_per_second": 138.16994223711538, - "model.estimated_cost": 0.01371, - "model.efficiency_score": 1.2691466083150984, - "model.response_length": 580, - "model.response_quality": "high", - "model.output_response": "Model Response: The provided web search results do not contain specific information about the latest updates in Python programming. They primarily focus on general news coverage and updates from India and around the ...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "51f4534f-23e4-4be7-8d6e-e93391d6e0e2", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:58:15.254000000", - "trace_end_time": "2025-10-22 16:58:18.877000000", - "trace_duration_ms": 3622.355, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:58:15.254000000", - "updated_at": "2025-10-22 16:58:15.254000000" - }, - { - "span_id": "d0bfcbe5-cb6c-4837-bdf3-f2c65be145cd", - "trace_id": "51f4534f-23e4-4be7-8d6e-e93391d6e0e2", - "parent_span_id": "a9e34cea-d4bd-478b-abf2-7a3a38dcd40e", - "name": "agent.web_search_evaluation_metrics", - "start_time": "2025-10-22 16:58:18.876000000", - "end_time": "2025-10-22 16:58:18.876000000", - "duration_ms": 0.04, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "web_search_evaluation_metrics", - "agent.capabilities": [ - "web_search_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What are the latest updates in Python programming?", - "query_length": 50, - "query_type": "web_search_evaluation_query", - "web_search.results_count": 1, - "web_search.context_length": 753, - "web_search.context_synthesized": "Context from 1 web sources", - "web_search.search_effectiveness": 0.2, - "web_search.context_quality": "medium", - "web_search.source_diversity": 1, - "web_search.information_synthesis": "low", - "web_search.external_knowledge_utilization": 0.753, - "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\n3 days ago \u00b7 Latest news headlines from India & around the world. Check out today\u2019s news coverage live with videos & photos on \u2026 6 days ago \u00b7 India News | Latest India News | Read latest and breaking news from India. Today's top India news headlines, news on \u2026 2 days ago \u00b7 Top News Stories of the day, Latest News Headlines, News Specials, Breaking News and Latest India News, World current affairs & \u2026 NDTV.com provides latest news from India and around the world. Get breaking news alerts from India and follow today\u2019s live news updates in \u2026 6 days ago \u00b7 Get Latest entertainment news from Bollywood, Hollywood and regional film industries.\nURL: https://duckduckgo.com/?q=What+are+the+latest+updates+in+Python+programming?\n\nUser Question: What are the latest updates in Python programming?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", - "prompt.prompt_length": 1248, - "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\n3 days ago \u00b7 Latest news headlines from India & around the world. Check out today\u2019s news coverage live with videos & photos on \u2026 6 days ago \u00b7 India News | Latest India News...", - "prompt.instruction_following": "web_search_optimized", - "response.answer_length": 580, - "response.answer_completeness": "complete", - "response.response_quality": "medium", - "response.source_citation": 1, - "response.context_utilization": 0.7702523240371846, - "output_response": "Web Search Answer: The provided web search results do not contain specific information about the latest updates in Python programming. They primarily focus on general news coverage and updates from India and around the ...", - "evaluation.search_effectiveness": 0.2, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 1, - "evaluation.context_relevance": "medium", - "evaluation.overall_quality": "low", - "evaluation.ready_for_production": true, - "web_search.search_strategy": "duckduckgo_api", - "web_search.real_time_data": true, - "web_search.external_sources": 1, - "web_search.information_freshness": "current", - "web_search.knowledge_synthesis": "single_source" - }, - "events": [], - "links": [], - "trace_trace_id": "51f4534f-23e4-4be7-8d6e-e93391d6e0e2", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:58:15.254000000", - "trace_end_time": "2025-10-22 16:58:18.877000000", - "trace_duration_ms": 3622.355, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:58:15.254000000", - "updated_at": "2025-10-22 16:58:15.254000000" - }, - { - "span_id": "5f5916a7-b9ea-4348-861d-db69b25d6b62", - "trace_id": "e30e9ba6-ffd1-4eb9-b7ae-6d19e04343d6", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-22 16:58:11.033000000", - "end_time": "2025-10-22 16:58:14.250000000", - "duration_ms": 3216.77, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What happened in tech news this week?", - "query.length": 37, - "processing.mode": "Web Search", - "processing.time_seconds": 3.2166178226470947, - "processing.response_length": 510, - "processing.sources_count": 1, - "output_response": "Final Answer: The web search results do not provide specific information about recent events in tech news for this week. They primarily discuss the grammatical usage of the word \"happened\" and its variations, such ...", - "final_answer_mode": "Web Search", - "query_processed.input_query": "What happened in tech news this week?", - "query_processed.output_response": "Successfully processed query using Web Search, generated 510 character response", - "query_processed.mode": "Web Search", - "query_processed.processing_time": 3.2166178226470947, - "query_processed.response_length": 510 - }, - "events": [], - "links": [], - "trace_trace_id": "e30e9ba6-ffd1-4eb9-b7ae-6d19e04343d6", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:58:11.033000000", - "trace_end_time": "2025-10-22 16:58:14.250000000", - "trace_duration_ms": 3216.801, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:58:11.033000000", - "updated_at": "2025-10-22 16:58:11.033000000" - }, - { - "span_id": "804613d0-a701-4c22-add1-d315d7a29813", - "trace_id": "e30e9ba6-ffd1-4eb9-b7ae-6d19e04343d6", - "parent_span_id": "5f5916a7-b9ea-4348-861d-db69b25d6b62", - "name": "agent.query_routing", - "start_time": "2025-10-22 16:58:11.033000000", - "end_time": "2025-10-22 16:58:14.250000000", - "duration_ms": 3216.594, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What happened in tech news this week?", - "query_length": 37, - "query_type": "routing_query", - "output_response": "Routed to Web Search for query processing", - "router.classification": "Web Search", - "router.confidence_score": 1, - "router.rag_keyword_score": 0, - "router.web_keyword_score": 2, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "e30e9ba6-ffd1-4eb9-b7ae-6d19e04343d6", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:58:11.033000000", - "trace_end_time": "2025-10-22 16:58:14.250000000", - "trace_duration_ms": 3216.801, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:58:11.033000000", - "updated_at": "2025-10-22 16:58:11.033000000" - }, - { - "span_id": "f6c8d34f-185f-49ba-a5a4-a9d25ccb4fc6", - "trace_id": "e30e9ba6-ffd1-4eb9-b7ae-6d19e04343d6", - "parent_span_id": "804613d0-a701-4c22-add1-d315d7a29813", - "name": "agent.web_search_generation", - "start_time": "2025-10-22 16:58:11.034000000", - "end_time": "2025-10-22 16:58:14.250000000", - "duration_ms": 3216.4959999999996, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "web_search_agent", - "agent.operation": "web_search_generation", - "agent.capabilities": [ - "web_search", - "content_synthesis", - "response_generation" - ], - "agent.id": "noveum_web_search_agent", - "input_query": "What happened in tech news this week?", - "query_length": 37, - "query_type": "web_search_query", - "output_response": "Web Search Answer: The web search results do not provide specific information about recent events in tech news for this week. They primarily discuss the grammatical usage of the word \"happened\" and its variations, such ...", - "web_search.results_count": 1, - "web_search.context_length": 724, - "web_search.response_length": 510, - "web_search.mode": "external_web_search" - }, - "events": [], - "links": [], - "trace_trace_id": "e30e9ba6-ffd1-4eb9-b7ae-6d19e04343d6", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:58:11.033000000", - "trace_end_time": "2025-10-22 16:58:14.250000000", - "trace_duration_ms": 3216.801, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:58:11.033000000", - "updated_at": "2025-10-22 16:58:11.033000000" - }, - { - "span_id": "11d68082-e7b8-4bbf-833a-79e296c31eda", - "trace_id": "e30e9ba6-ffd1-4eb9-b7ae-6d19e04343d6", - "parent_span_id": "804613d0-a701-4c22-add1-d315d7a29813", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-22 16:58:11.034000000", - "end_time": "2025-10-22 16:58:11.034000000", - "duration_ms": 0.017, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What happened in tech news this week?", - "query_length": 37, - "query_type": "routing_evaluation_query", - "classification.mode": "Web Search", - "classification.rag_keyword_score": 0, - "classification.web_keyword_score": 2, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "general_knowledge", - "query.keyword_density": 0.2857142857142857, - "query.domain_affinity": "general", - "routing.decision": "Routed to Web Search based on analysis", - "routing.rationale": "RAG score: 0, Web score: 2, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "RAG", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.038461538461538464, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: Web Search (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "e30e9ba6-ffd1-4eb9-b7ae-6d19e04343d6", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:58:11.033000000", - "trace_end_time": "2025-10-22 16:58:14.250000000", - "trace_duration_ms": 3216.801, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:58:11.033000000", - "updated_at": "2025-10-22 16:58:11.033000000" - }, - { - "span_id": "420b4ca1-c9f1-488f-ac2c-c53766717634", - "trace_id": "e30e9ba6-ffd1-4eb9-b7ae-6d19e04343d6", - "parent_span_id": "f6c8d34f-185f-49ba-a5a4-a9d25ccb4fc6", - "name": "agent.llm_model_execution", - "start_time": "2025-10-22 16:58:11.468000000", - "end_time": "2025-10-22 16:58:14.250000000", - "duration_ms": 2781.777, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What happened in tech news this week?", - "query_length": 37, - "query_type": "web_search_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 2.781723976135254, - "model.latency_ms": 2781.723976135254, - "model.start_time": 1761152291.468626, - "model.end_time": 1761152294.25035, - "model.performance_tier": "medium", - "model.prompt_tokens": 301, - "model.completion_tokens": 127, - "model.total_tokens": 428, - "model.tokens_per_second": 153.8614196346811, - "model.estimated_cost": 0.01284, - "model.efficiency_score": 1.191588785046729, - "model.response_length": 510, - "model.response_quality": "high", - "model.output_response": "Model Response: The web search results do not provide specific information about recent events in tech news for this week. They primarily discuss the grammatical usage of the word \"happened\" and its variations, such ...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "e30e9ba6-ffd1-4eb9-b7ae-6d19e04343d6", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:58:11.033000000", - "trace_end_time": "2025-10-22 16:58:14.250000000", - "trace_duration_ms": 3216.801, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:58:11.033000000", - "updated_at": "2025-10-22 16:58:11.033000000" - }, - { - "span_id": "56b8bf09-8ebf-44e1-b01f-daff23d3c42c", - "trace_id": "e30e9ba6-ffd1-4eb9-b7ae-6d19e04343d6", - "parent_span_id": "f6c8d34f-185f-49ba-a5a4-a9d25ccb4fc6", - "name": "agent.web_search_evaluation_metrics", - "start_time": "2025-10-22 16:58:14.250000000", - "end_time": "2025-10-22 16:58:14.250000000", - "duration_ms": 0.060000000000000005, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "web_search_evaluation_metrics", - "agent.capabilities": [ - "web_search_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What happened in tech news this week?", - "query_length": 37, - "query_type": "web_search_evaluation_query", - "web_search.results_count": 1, - "web_search.context_length": 724, - "web_search.context_synthesized": "Context from 1 web sources", - "web_search.search_effectiveness": 0.2, - "web_search.context_quality": "medium", - "web_search.source_diversity": 1, - "web_search.information_synthesis": "low", - "web_search.external_knowledge_utilization": 0.724, - "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nhappened\u7684\u7528\u6cd5\u5982\u4e0b\uff1a 1.happened\u662f \u4e0d\u53ca\u7269\u52a8\u8bcd,\u4f5c\u201c\u53d1\u751f\u201d\u89e3\u65f6,\u53ef\u6307\u5076\u7136\u7684\u4e8b\u6545,\u4e5f\u53ef\u6307\u6709\u8ba1\u5212\u6216\u65e0\u8ba1\u5212\u53d1\u751f\u7684\u4e8b\u3002\u4e00\u822c\u4ee5\u4e8b\u4ef6\u3001\u73af\u5883\u3001\u5f62\u52bf\u7b49\u4f5c\u4e3b\u8bed,\u8868\u793a\u201c\u67d0\u4eba\u53d1\u751f\u67d0\u4e8b\u201d\u65f6,\u987b\u4ee5\u4ecb\u8bcdto\u5f15\u51fa\u201c\u67d0\u4eba\u201d\u3002 2. \u2026 what happened\u8fd8\u662fwhat's happened\"What happened\"\u548c\"What's happened\"\u90fd\u7528\u4e8e\u8be2\u95ee\u8fc7\u53bb\u53d1\u751f\u4e86\u4ec0\u4e48\u4e8b\u60c5\uff0c\u4e3b\u8981\u7684\u533a\u522b\u5728\u4e8e\u8bed\u6cd5\u548c\u4f7f\u7528\u65b9\u5f0f\u3002\"What happened\"\u662f\u4e00\u4e2a\u8fc7\u53bb\u65f6\u6001\u7684\u7b80\u5355 \u63a5\u4e0b\u6765\u8ba9\u6211\u4eec\u770b\u4e0bwhat happen\u548cwhat happened\u7684\u7528\u6cd5\u533a\u522b\uff1a 1.\u8bcd\u6027\uff1awhat happen\u662f\u7528\u539f\u5f62\u52a8\u8bcdhappen\u6784\u6210\u7684\u73b0\u5728\u65f6\u77ed\u8bed\uff0c\u800cwhat happened\u662f\u7528\u52a8\u8bcdhappen\u7684\u8fc7\u53bb\u5f0fhappened\u6784\u6210\u7684\u8fc7\u53bb\u65f6\u77ed \u2026 What happened \u548c What's happened\u4e24\u8005\u7684\u533a\u522b\uff1a \u662f\u95ee What happened\uff1f\u53ea\u662f\u60f3\u77e5\u9053\u8fc7\u53bb\u6240\u53d1\u751f\u7684\u4e8b\u60c5\uff0c \u800c\u5982\u679c\u7528What's happened\uff1f\u5219\u5f3a\u8c03\u5bf9\u73b0\u5728\u7684\u5f71\u54cd\u3002\uff0c\u5373\u60f3\u77e5\u9053\u8fc7\u53bb\u6240\u53d1\u751f\u7684\u4ec0\u4e48\u4e8b\u5bfc\u81f4\u4e86\u8bf4 \u2026 it so happened that\u7684\u53e5\u5f0fhappen\u8868\u793a\u201c\u78b0\u5de7\u6216\u6070\u5de7\u53d1\u751f\u67d0\u4e8b\u201d\u65f6,\u53ef\u7528\u201cIt happens / happened that...\u201d\u8fd9\u4e00\u7ed3\u6784\u6765\u8868\u8fbe.\u4f8b\u5982\uff1a It happened that Brian and Peter were at home that day.\u78b0\u5de7\u90a3\u5929\u5e03\u83b1\u6069\u548c \u2026\nURL: https://duckduckgo.com/?q=What+happened+in+tech+news+this+week?\n\nUser Question: What happened in tech news this week?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", - "prompt.prompt_length": 1206, - "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nhappened\u7684\u7528\u6cd5\u5982\u4e0b\uff1a 1.happened\u662f \u4e0d\u53ca\u7269\u52a8\u8bcd,\u4f5c\u201c\u53d1\u751f\u201d\u89e3\u65f6,\u53ef\u6307\u5076\u7136\u7684\u4e8b\u6545,\u4e5f\u53ef\u6307\u6709\u8ba1\u5212\u6216\u65e0\u8ba1\u5212\u53d1\u751f\u7684\u4e8b\u3002\u4e00\u822c\u4ee5\u4e8b\u4ef6\u3001\u73af\u5883\u3001\u5f62\u52bf\u7b49\u4f5c\u4e3b\u8bed,\u8868\u793a\u201c\u67d0\u4eba\u53d1\u751f\u67d0\u4e8b\u201d\u65f6,\u987b\u4ee5\u4ecb\u8bcdto\u5f15\u51fa\u201c\u67d0\u4eba\u201d\u3002 2. \u2026 what happened\u8fd8\u662fwhat's happened\"What happened\"\u548c\"What's happene...", - "prompt.instruction_following": "web_search_optimized", - "response.answer_length": 510, - "response.answer_completeness": "complete", - "response.response_quality": "medium", - "response.source_citation": 1, - "response.context_utilization": 0.7044198895027625, - "output_response": "Web Search Answer: The web search results do not provide specific information about recent events in tech news for this week. They primarily discuss the grammatical usage of the word \"happened\" and its variations, such ...", - "evaluation.search_effectiveness": 0.2, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 1, - "evaluation.context_relevance": "medium", - "evaluation.overall_quality": "low", - "evaluation.ready_for_production": true, - "web_search.search_strategy": "duckduckgo_api", - "web_search.real_time_data": true, - "web_search.external_sources": 1, - "web_search.information_freshness": "current", - "web_search.knowledge_synthesis": "single_source" - }, - "events": [], - "links": [], - "trace_trace_id": "e30e9ba6-ffd1-4eb9-b7ae-6d19e04343d6", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:58:11.033000000", - "trace_end_time": "2025-10-22 16:58:14.250000000", - "trace_duration_ms": 3216.801, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:58:11.033000000", - "updated_at": "2025-10-22 16:58:11.033000000" - }, - { - "span_id": "82eb19e8-f6cf-43fd-ba56-640a6fd3ed0e", - "trace_id": "45efae74-ee25-4004-a509-6e0ff628a0cd", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-22 16:58:04.883000000", - "end_time": "2025-10-22 16:58:10.029000000", - "duration_ms": 5146.146, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What are the current trends in observability tools?", - "query.length": 51, - "processing.mode": "RAG", - "processing.time_seconds": 5.146108150482178, - "processing.response_length": 815, - "processing.sources_count": 5, - "output_response": "Final Answer: The provided context does not explicitly outline current trends in observability tools. However, it highlights the unique challenges faced by AI applications that traditional monitoring tools cannot a...", - "final_answer_mode": "RAG", - "query_processed.input_query": "What are the current trends in observability tools?", - "query_processed.output_response": "Successfully processed query using RAG, generated 815 character response", - "query_processed.mode": "RAG", - "query_processed.processing_time": 5.146108150482178, - "query_processed.response_length": 815 - }, - "events": [], - "links": [], - "trace_trace_id": "45efae74-ee25-4004-a509-6e0ff628a0cd", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:58:04.883000000", - "trace_end_time": "2025-10-22 16:58:10.029000000", - "trace_duration_ms": 5146.244000000001, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:58:04.883000000", - "updated_at": "2025-10-22 16:58:04.883000000" - }, - { - "span_id": "e81617c6-c2a1-4b08-a5fc-dde3439fe2f4", - "trace_id": "45efae74-ee25-4004-a509-6e0ff628a0cd", - "parent_span_id": "8001ba0e-95c7-42da-9087-603a38e1a500", - "name": "agent.llm-rag", - "start_time": "2025-10-22 16:58:04.883000000", - "end_time": "2025-10-22 16:58:10.029000000", - "duration_ms": 5145.655, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "rag_agent", - "agent.operation": "llm-rag", - "agent.capabilities": [ - "document_retrieval", - "context_generation", - "response_generation" - ], - "agent.id": "noveum_rag_agent", - "input_query": "What are the current trends in observability tools?", - "query_length": 51, - "query_type": "rag_query", - "output_response": "RAG Answer: The provided context does not explicitly outline current trends in observability tools. However, it highlights the unique challenges faced by AI applications that traditional monitoring tools cannot a...", - "rag.context_length": 2943, - "rag.sources_count": 5, - "rag.answer_length": 815, - "rag.mode": "retrieval_augmented_generation" - }, - "events": [], - "links": [], - "trace_trace_id": "45efae74-ee25-4004-a509-6e0ff628a0cd", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:58:04.883000000", - "trace_end_time": "2025-10-22 16:58:10.029000000", - "trace_duration_ms": 5146.244000000001, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:58:04.883000000", - "updated_at": "2025-10-22 16:58:04.883000000" - }, - { - "span_id": "8001ba0e-95c7-42da-9087-603a38e1a500", - "trace_id": "45efae74-ee25-4004-a509-6e0ff628a0cd", - "parent_span_id": "82eb19e8-f6cf-43fd-ba56-640a6fd3ed0e", - "name": "agent.query_routing", - "start_time": "2025-10-22 16:58:04.883000000", - "end_time": "2025-10-22 16:58:10.029000000", - "duration_ms": 5146.036999999999, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What are the current trends in observability tools?", - "query_length": 51, - "query_type": "routing_query", - "output_response": "Routed to RAG for query processing", - "router.classification": "RAG", - "router.confidence_score": 0.3333333333333333, - "router.rag_keyword_score": 2, - "router.web_keyword_score": 1, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "45efae74-ee25-4004-a509-6e0ff628a0cd", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:58:04.883000000", - "trace_end_time": "2025-10-22 16:58:10.029000000", - "trace_duration_ms": 5146.244000000001, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:58:04.883000000", - "updated_at": "2025-10-22 16:58:04.883000000" - }, - { - "span_id": "79b7d138-05e1-4f18-8b63-dcfe9732180b", - "trace_id": "45efae74-ee25-4004-a509-6e0ff628a0cd", - "parent_span_id": "8001ba0e-95c7-42da-9087-603a38e1a500", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-22 16:58:04.883000000", - "end_time": "2025-10-22 16:58:04.883000000", - "duration_ms": 0.051, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What are the current trends in observability tools?", - "query_length": 51, - "query_type": "routing_evaluation_query", - "classification.mode": "RAG", - "classification.rag_keyword_score": 2, - "classification.web_keyword_score": 1, - "classification.confidence_score": 0.3333333333333333, - "classification.confidence_level": "medium", - "classification.method": "keyword_based", - "query.complexity": "complex", - "query.intent": "documentation", - "query.keyword_density": 0.375, - "query.domain_affinity": "noveum", - "routing.decision": "Routed to RAG based on analysis", - "routing.rationale": "RAG score: 2, Web score: 1, Confidence: 0.33", - "routing.expected_performance": "medium", - "routing.alternative_mode": "Web Search", - "routing.decision_confidence": 0.3333333333333333, - "evaluation.routing_accuracy": "medium", - "evaluation.keyword_coverage": 0.057692307692307696, - "evaluation.query_understanding": "ambiguous", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: RAG (Confidence: 0.33)" - }, - "events": [], - "links": [], - "trace_trace_id": "45efae74-ee25-4004-a509-6e0ff628a0cd", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:58:04.883000000", - "trace_end_time": "2025-10-22 16:58:10.029000000", - "trace_duration_ms": 5146.244000000001, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:58:04.883000000", - "updated_at": "2025-10-22 16:58:04.883000000" - }, - { - "span_id": "28c233a0-d42a-4200-86b0-dfbd2d5be06e", - "trace_id": "45efae74-ee25-4004-a509-6e0ff628a0cd", - "parent_span_id": "e81617c6-c2a1-4b08-a5fc-dde3439fe2f4", - "name": "agent.llm_model_execution", - "start_time": "2025-10-22 16:58:05.635000000", - "end_time": "2025-10-22 16:58:09.222000000", - "duration_ms": 3586.924, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What are the current trends in observability tools?", - "query_length": 51, - "query_type": "rag_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 3.5868873596191406, - "model.latency_ms": 3586.8873596191406, - "model.start_time": 1761152285.6352398, - "model.end_time": 1761152289.2221272, - "model.performance_tier": "medium", - "model.prompt_tokens": 872, - "model.completion_tokens": 203, - "model.total_tokens": 1075, - "model.tokens_per_second": 299.7027484337129, - "model.estimated_cost": 0.03225, - "model.efficiency_score": 0.7581395348837209, - "model.response_length": 815, - "model.response_quality": "high", - "model.output_response": "Model Response: The provided context does not explicitly outline current trends in observability tools. However, it highlights the unique challenges faced by AI applications that traditional monitoring tools cannot a...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "45efae74-ee25-4004-a509-6e0ff628a0cd", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:58:04.883000000", - "trace_end_time": "2025-10-22 16:58:10.029000000", - "trace_duration_ms": 5146.244000000001, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:58:04.883000000", - "updated_at": "2025-10-22 16:58:04.883000000" - }, - { - "span_id": "024c05b6-edf9-40ef-931b-9660d48b4948", - "trace_id": "45efae74-ee25-4004-a509-6e0ff628a0cd", - "parent_span_id": "e81617c6-c2a1-4b08-a5fc-dde3439fe2f4", - "name": "agent.rag_evaluation_metrics", - "start_time": "2025-10-22 16:58:09.222000000", - "end_time": "2025-10-22 16:58:09.222000000", - "duration_ms": 0.020999999999999998, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "rag_evaluation_metrics", - "agent.capabilities": [ - "retrieval_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What are the current trends in observability tools?", - "query_length": 51, - "query_type": "rag_evaluation_query", - "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/blog/from-logs-to-intelligent-choices-inside-noveum-ais-evaluation-process):\nFrom Development to Production - Inside Noveum.ai's AI Observability Platform\nShashank Agarwal\n3/3/2025\nIntroduction\nEvery AI application tells a story\u2014but most developers never get to hear it...", - "retrieval.context_length": 2943, - "retrieval.sources_count": 5, - "retrieval.context_quality": "high", - "retrieval.effectiveness": 1, - "retrieval.context_utilization": 2.943, - "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/blog/from-logs-to-intelligent-choices-inside-noveum-ais-evaluation-process):\nFrom Development to Production - Inside Noveum.ai's AI Observability Platform\nShashank Agarwal\n3/3/2025\nIntroduction\nEvery AI application tells a story\u2014but most developers never get to hear it. When your RAG pipeline returns irrelevant results, when your multi-agent system gets stuck in loops, or when your LLM costs suddenly spike, you're left guessing what went wrong and where.\nThat's exactly the challenge Noveum.ai solves. Rather than flying blind, our platform provides comprehensive tracing a...\n\nSource 2 (https://noveum.ai/docs/getting-started/tracing-concepts):\n- Explore Framework Integrations - Framework-specific guidance\n- Learn Advanced Patterns - Custom instrumentation techniques\n- Master the Dashboard - Analyze your traces effectively\nRemember: Good observability is not about collecting all possible data, but about collecting the right data that helps you understand, debug, and optimize your AI applications.\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to O...\n\nSource 3 (https://noveum.ai/en/docs/getting-started/tracing-concepts):\n- Explore Framework Integrations - Framework-specific guidance\n- Learn Advanced Patterns - Custom instrumentation techniques\n- Master the Dashboard - Analyze your traces effectively\nRemember: Good observability is not about collecting all possible data, but about collecting the right data that helps you understand, debug, and optimize your AI applications.\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to O...\n\nSource 4 (https://noveum.ai/en/blog/from-logs-to-intelligent-choices-inside-noveum-ais-evaluation-process):\nThe AI Observability Challenge\nWhy Traditional Monitoring Falls Short\nTraditional application monitoring tools weren't built for AI applications. They can tell you if your API is responding, but they can't answer the questions that matter most for AI systems:\n- Why did my RAG pipeline retrieve irrelevant documents?\n- Which LLM calls are driving my costs?\n- How are my agents communicating with each other?\n- What's causing hallucinations in my responses?\n- Why is my embedding generation so slow?\nW...\n\nSource 5 (https://noveum.ai/en/docs/advanced/rag-observability):\n\ud83d\udd27 Best Practices for RAG Observability\n1. Comprehensive Pipeline Tracing\n2. Quality Monitoring\n3. A/B Testing for RAG Components\n\ud83c\udfaf Advanced RAG Patterns\nMulti-Modal RAG\nConversational RAG\nRAG observability with Noveum.ai provides the deep insights needed to build, optimize, and scale retrieval-augmented generation systems. By implementing comprehensive tracing across retrieval, context preparation, and generation stages, you can ensure your RAG pipeline delivers accurate, relevant, and cost-effe...\n\nUser Question: What are the current trends in observability tools?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", - "prompt.prompt_length": 3490, - "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/blog/from-logs-to-intelligent-choices-inside-noveum-ais-evaluation-process):\nFrom Development to Production - Inside Noveum.ai's AI Observability Platform\nShashank Agarw...", - "prompt.instruction_following": "rag_optimized", - "response.answer_length": 815, - "response.answer_completeness": "complete", - "response.response_quality": "high", - "response.source_citation": 5, - "response.context_utilization": 0.2769283044512402, - "output_response": "RAG Answer: The provided context does not explicitly outline current trends in observability tools. However, it highlights the unique challenges faced by AI applications that traditional monitoring tools cannot a...", - "evaluation.retrieval_effectiveness": 1, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 5, - "evaluation.context_relevance": "high", - "evaluation.overall_quality": "high", - "evaluation.ready_for_production": true, - "rag.retrieval_strategy": "semantic_similarity", - "rag.vector_search_results": 5, - "rag.context_synthesis": "multi_source", - "rag.document_coverage": 1, - "rag.information_density": 0.2769283044512402 - }, - "events": [], - "links": [], - "trace_trace_id": "45efae74-ee25-4004-a509-6e0ff628a0cd", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:58:04.883000000", - "trace_end_time": "2025-10-22 16:58:10.029000000", - "trace_duration_ms": 5146.244000000001, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:58:04.883000000", - "updated_at": "2025-10-22 16:58:04.883000000" - }, - { - "span_id": "99a5e8f3-a9f5-4953-94ea-ad0fa0cf6ec2", - "trace_id": "c5a64d08-d00f-46cf-ab3c-d8d473c10eca", - "parent_span_id": "2a351fd9-a7ee-42da-9e2e-86a643ceed0e", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-22 16:57:57.908000000", - "end_time": "2025-10-22 16:57:57.908000000", - "duration_ms": 0.026, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: Tell me about recent developments in machine learning", - "query_length": 53, - "query_type": "routing_evaluation_query", - "classification.mode": "Web Search", - "classification.rag_keyword_score": 0, - "classification.web_keyword_score": 1, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "complex", - "query.intent": "general_knowledge", - "query.keyword_density": 0.125, - "query.domain_affinity": "general", - "routing.decision": "Routed to Web Search based on analysis", - "routing.rationale": "RAG score: 0, Web score: 1, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "RAG", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.019230769230769232, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: Web Search (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "c5a64d08-d00f-46cf-ab3c-d8d473c10eca", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:57.908000000", - "trace_end_time": "2025-10-22 16:58:03.875000000", - "trace_duration_ms": 5967.0779999999995, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:57.908000000", - "updated_at": "2025-10-22 16:57:57.908000000" - }, - { - "span_id": "6e891e51-ca6d-4e6c-80d0-0791ee22d5b9", - "trace_id": "c5a64d08-d00f-46cf-ab3c-d8d473c10eca", - "parent_span_id": "2a351fd9-a7ee-42da-9e2e-86a643ceed0e", - "name": "agent.web_search_generation", - "start_time": "2025-10-22 16:57:57.908000000", - "end_time": "2025-10-22 16:58:03.874000000", - "duration_ms": 5966.728, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "web_search_agent", - "agent.operation": "web_search_generation", - "agent.capabilities": [ - "web_search", - "content_synthesis", - "response_generation" - ], - "agent.id": "noveum_web_search_agent", - "input_query": "Tell me about recent developments in machine learning", - "query_length": 53, - "query_type": "web_search_query", - "output_response": "Web Search Answer: The web search results did not provide specific information about recent developments in machine learning. Therefore, I cannot summarize any recent advancements based on the provided data.\n\nHowever, i...", - "web_search.results_count": 1, - "web_search.context_length": 768, - "web_search.response_length": 687, - "web_search.mode": "external_web_search" - }, - "events": [], - "links": [], - "trace_trace_id": "c5a64d08-d00f-46cf-ab3c-d8d473c10eca", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:57.908000000", - "trace_end_time": "2025-10-22 16:58:03.875000000", - "trace_duration_ms": 5967.0779999999995, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:57.908000000", - "updated_at": "2025-10-22 16:57:57.908000000" - }, - { - "span_id": "6d622376-d48b-4a64-b0a2-1402117febaf", - "trace_id": "c5a64d08-d00f-46cf-ab3c-d8d473c10eca", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-22 16:57:57.908000000", - "end_time": "2025-10-22 16:58:03.875000000", - "duration_ms": 5967.035, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "Tell me about recent developments in machine learning", - "query.length": 53, - "processing.mode": "Web Search", - "processing.time_seconds": 5.966889142990112, - "processing.response_length": 687, - "processing.sources_count": 1, - "output_response": "Final Answer: The web search results did not provide specific information about recent developments in machine learning. Therefore, I cannot summarize any recent advancements based on the provided data.\n\nHowever, i...", - "final_answer_mode": "Web Search", - "query_processed.input_query": "Tell me about recent developments in machine learning", - "query_processed.output_response": "Successfully processed query using Web Search, generated 687 character response", - "query_processed.mode": "Web Search", - "query_processed.processing_time": 5.966889142990112, - "query_processed.response_length": 687 - }, - "events": [], - "links": [], - "trace_trace_id": "c5a64d08-d00f-46cf-ab3c-d8d473c10eca", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:57.908000000", - "trace_end_time": "2025-10-22 16:58:03.875000000", - "trace_duration_ms": 5967.0779999999995, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:57.908000000", - "updated_at": "2025-10-22 16:57:57.908000000" - }, - { - "span_id": "2a351fd9-a7ee-42da-9e2e-86a643ceed0e", - "trace_id": "c5a64d08-d00f-46cf-ab3c-d8d473c10eca", - "parent_span_id": "6d622376-d48b-4a64-b0a2-1402117febaf", - "name": "agent.query_routing", - "start_time": "2025-10-22 16:57:57.908000000", - "end_time": "2025-10-22 16:58:03.875000000", - "duration_ms": 5966.8640000000005, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "Tell me about recent developments in machine learning", - "query_length": 53, - "query_type": "routing_query", - "output_response": "Routed to Web Search for query processing", - "router.classification": "Web Search", - "router.confidence_score": 1, - "router.rag_keyword_score": 0, - "router.web_keyword_score": 1, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "c5a64d08-d00f-46cf-ab3c-d8d473c10eca", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:57.908000000", - "trace_end_time": "2025-10-22 16:58:03.875000000", - "trace_duration_ms": 5967.0779999999995, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:57.908000000", - "updated_at": "2025-10-22 16:57:57.908000000" - }, - { - "span_id": "ba457b3b-9be8-4e24-af63-4118833711e6", - "trace_id": "c5a64d08-d00f-46cf-ab3c-d8d473c10eca", - "parent_span_id": "6e891e51-ca6d-4e6c-80d0-0791ee22d5b9", - "name": "agent.llm_model_execution", - "start_time": "2025-10-22 16:57:58.324000000", - "end_time": "2025-10-22 16:58:03.874000000", - "duration_ms": 5550.059, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: Tell me about recent developments in machine learning", - "query_length": 53, - "query_type": "web_search_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 5.550010919570923, - "model.latency_ms": 5550.010919570923, - "model.start_time": 1761152278.3248482, - "model.end_time": 1761152283.874859, - "model.performance_tier": "slow", - "model.prompt_tokens": 316, - "model.completion_tokens": 171, - "model.total_tokens": 487, - "model.tokens_per_second": 87.74757510524871, - "model.estimated_cost": 0.01461, - "model.efficiency_score": 1.4106776180698153, - "model.response_length": 687, - "model.response_quality": "high", - "model.output_response": "Model Response: The web search results did not provide specific information about recent developments in machine learning. Therefore, I cannot summarize any recent advancements based on the provided data.\n\nHowever, i...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "c5a64d08-d00f-46cf-ab3c-d8d473c10eca", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:57.908000000", - "trace_end_time": "2025-10-22 16:58:03.875000000", - "trace_duration_ms": 5967.0779999999995, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:57.908000000", - "updated_at": "2025-10-22 16:57:57.908000000" - }, - { - "span_id": "7b70a916-1c2b-4898-a052-cfd98a979167", - "trace_id": "c5a64d08-d00f-46cf-ab3c-d8d473c10eca", - "parent_span_id": "6e891e51-ca6d-4e6c-80d0-0791ee22d5b9", - "name": "agent.web_search_evaluation_metrics", - "start_time": "2025-10-22 16:58:03.874000000", - "end_time": "2025-10-22 16:58:03.874000000", - "duration_ms": 0.02, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "web_search_evaluation_metrics", - "agent.capabilities": [ - "web_search_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: Tell me about recent developments in machine learning", - "query_length": 53, - "query_type": "web_search_evaluation_query", - "web_search.results_count": 1, - "web_search.context_length": 768, - "web_search.context_synthesized": "Context from 1 web sources", - "web_search.search_effectiveness": 0.2, - "web_search.context_quality": "medium", - "web_search.source_diversity": 1, - "web_search.information_synthesis": "low", - "web_search.external_knowledge_utilization": 0.768, - "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nFeb 3, 2024 \u00b7 tell sb sth\u4e2d\u7684tell\u66f4\u591a\u662f\u544a\u77e5\u7684\u610f\u601d\uff0c\u5c31\u662f\u544a\u8bc9\u522b\u4eba\u6709\u8fd9\u4e48\u4e00\u4e2a\u4e8b\u7684\u5b58\u5728 tell sb about sth \u7684\u610f\u601d\u5305\u542b\u66f4\u591a\u4e00\u4e9b\uff0c\u9664\u4e86\u544a\u77e5\u6709\u8fd9\u4e48\u4e00\u4e2a\u4e8b\u60c5\u7684\u5b58\u5728\uff0c\u8fd8\u4f5c\u4e86\u4e00\u4e9b\u8bf4\u660e \u6bd4\u5982\uff1a tell me he \u2026 \u4e00\u3001Say\u548cTell\u7684\u7528\u6cd5 \u6211\u4eec\u5148\u5c06say\u3001tell\u3001speak\u3001talk\u8fd9\u56db\u4e2a\u8bcd\u5206\u6210\u4e24\u7ec4\uff0c\u5206\u522b\u4e3a\uff1a \u7b2c\u2460\u7ec4\uff1asay & tell\uff0c\u8fd9\u4e24\u4e2a\u8bcd\u4e4b\u540e\u53ef\u4ee5\u8868\u793a\u8c08\u8bdd\u7684 \u201c\u7ec6\u8282\u201d\u3002 \u7b2c\u2461\u7ec4\uff1aspeak & talk\uff0c\u4e0d\u80fd\u8868\u793a\u8c08\u8bdd\u7684\u201c\u7ec6\u8282\u201d\u3002 \u2026 tell of 1\u3001Tell of a turning point in your career and what you learned. \u8bb2\u8ff0\u4f60 \u804c\u4e1a\u751f\u6daf \u4e00\u4e2a\u8f6c\u6298\u70b9\uff0c\u4ee5\u53ca\u4f60\u5b66\u5230\u4e86\u4ec0\u4e48\u3002 2\u3001I have made the Sovereign Lord my refuge; I will tell of all your \u2026 tell sb to do sth \u548c tell sb do sth \u90fd\u8868\u793a\u544a\u8bc9\u67d0\u4eba\u505a\u67d0\u4e8b\uff0ctell sb to do sth \u5f3a\u8c03\u544a\u8bc9\u67d0\u4eba\u5e94\u8be5\u505a\u4ec0\u4e48\u5177\u4f53\u52a8\u4f5c\u6216\u884c\u4e3a\uff0c\u800c tell sb do sth \u5f3a\u8c03\u544a\u8bc9\u67d0\u4eba\u6b63\u5728\u8fdb\u884c\u7684\u5177\u4f53\u884c\u52a8\u6216\u72b6\u6001\u3002 \u5b83\u4eec\u5728\u91ca\u4e49\u3001 \u2026 Aug 23, 2023 \u00b7 tell\u7684\u8fc7\u53bb\u5f0f\u548c\u8fc7\u53bb\u5206\u8bcd\u90fd\u662ftold\u3002 \u8bfb\u97f3\uff1a\u82f1[tel]\uff0c\u7f8e[tel]\u3002 \u91ca\u4e49\uff1a vt.\u544a\u8bc9\uff0c\u8bf4\uff1b\u8fa8\u522b\uff1b\u5429\u5490\uff1b\u65ad\u5b9a\u3002 vi.\u8bb2\u8ff0\uff1b\u544a\u53d1\uff0c\u6cc4\u5bc6\uff1b\u8bc6\u522b\u3002 \u53d8\u5f62\uff1a\u8fc7\u53bb\u5f0ftold\uff0c\u8fc7\u53bb\u5206\u8bcdtold\uff0c\u73b0\u5728\u5206 \u2026\nURL: https://duckduckgo.com/?q=Tell+me+about+recent+developments+in+machine+learning\n\nUser Question: Tell me about recent developments in machine learning\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", - "prompt.prompt_length": 1266, - "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nFeb 3, 2024 \u00b7 tell sb sth\u4e2d\u7684tell\u66f4\u591a\u662f\u544a\u77e5\u7684\u610f\u601d\uff0c\u5c31\u662f\u544a\u8bc9\u522b\u4eba\u6709\u8fd9\u4e48\u4e00\u4e2a\u4e8b\u7684\u5b58\u5728 tell sb about sth \u7684\u610f\u601d\u5305\u542b\u66f4\u591a\u4e00\u4e9b\uff0c\u9664\u4e86\u544a\u77e5\u6709\u8fd9\u4e48\u4e00\u4e2a\u4e8b\u60c5\u7684\u5b58\u5728\uff0c\u8fd8\u4f5c\u4e86\u4e00\u4e9b\u8bf4\u660e \u6bd4\u5982\uff1a tell me he \u2026 \u4e00\u3001Say\u548cTell\u7684\u7528\u6cd5 \u6211\u4eec\u5148\u5c06say\u3001tell\u3001speak\u3001talk\u8fd9\u56db\u4e2a\u8bcd\u5206\u6210\u4e24\u7ec4\uff0c\u5206\u522b...", - "prompt.instruction_following": "web_search_optimized", - "response.answer_length": 687, - "response.answer_completeness": "complete", - "response.response_quality": "medium", - "response.source_citation": 1, - "response.context_utilization": 0.89453125, - "output_response": "Web Search Answer: The web search results did not provide specific information about recent developments in machine learning. Therefore, I cannot summarize any recent advancements based on the provided data.\n\nHowever, i...", - "evaluation.search_effectiveness": 0.2, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 1, - "evaluation.context_relevance": "medium", - "evaluation.overall_quality": "low", - "evaluation.ready_for_production": true, - "web_search.search_strategy": "duckduckgo_api", - "web_search.real_time_data": true, - "web_search.external_sources": 1, - "web_search.information_freshness": "current", - "web_search.knowledge_synthesis": "single_source" - }, - "events": [], - "links": [], - "trace_trace_id": "c5a64d08-d00f-46cf-ab3c-d8d473c10eca", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:57.908000000", - "trace_end_time": "2025-10-22 16:58:03.875000000", - "trace_duration_ms": 5967.0779999999995, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:57.908000000", - "updated_at": "2025-10-22 16:57:57.908000000" - }, - { - "span_id": "1e1a341f-66c1-48b0-95eb-fb092928c473", - "trace_id": "b1b13e60-5b32-44a0-9769-197a944989d0", - "parent_span_id": "1c5c4291-bbb5-4f85-b6d9-b6e8df66f742", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-22 16:57:51.967000000", - "end_time": "2025-10-22 16:57:51.967000000", - "duration_ms": 0.033, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What's the weather like today?", - "query_length": 30, - "query_type": "routing_evaluation_query", - "classification.mode": "Web Search", - "classification.rag_keyword_score": 0, - "classification.web_keyword_score": 2, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "general_knowledge", - "query.keyword_density": 0.4, - "query.domain_affinity": "general", - "routing.decision": "Routed to Web Search based on analysis", - "routing.rationale": "RAG score: 0, Web score: 2, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "RAG", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.038461538461538464, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: Web Search (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "b1b13e60-5b32-44a0-9769-197a944989d0", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:51.967000000", - "trace_end_time": "2025-10-22 16:57:56.899000000", - "trace_duration_ms": 4932.393, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:51.967000000", - "updated_at": "2025-10-22 16:57:51.967000000" - }, - { - "span_id": "1c5c4291-bbb5-4f85-b6d9-b6e8df66f742", - "trace_id": "b1b13e60-5b32-44a0-9769-197a944989d0", - "parent_span_id": "49e7cace-ba21-4807-a238-9a3880487583", - "name": "agent.query_routing", - "start_time": "2025-10-22 16:57:51.967000000", - "end_time": "2025-10-22 16:57:56.899000000", - "duration_ms": 4931.972, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What's the weather like today?", - "query_length": 30, - "query_type": "routing_query", - "output_response": "Routed to Web Search for query processing", - "router.classification": "Web Search", - "router.confidence_score": 1, - "router.rag_keyword_score": 0, - "router.web_keyword_score": 2, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "b1b13e60-5b32-44a0-9769-197a944989d0", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:51.967000000", - "trace_end_time": "2025-10-22 16:57:56.899000000", - "trace_duration_ms": 4932.393, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:51.967000000", - "updated_at": "2025-10-22 16:57:51.967000000" - }, - { - "span_id": "49e7cace-ba21-4807-a238-9a3880487583", - "trace_id": "b1b13e60-5b32-44a0-9769-197a944989d0", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-22 16:57:51.967000000", - "end_time": "2025-10-22 16:57:56.899000000", - "duration_ms": 4932.316, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What's the weather like today?", - "query.length": 30, - "processing.mode": "Web Search", - "processing.time_seconds": 4.932007789611816, - "processing.response_length": 260, - "processing.sources_count": 1, - "output_response": "Final Answer: The web search results do not provide specific information about today's weather. To find out the current weather conditions, I recommend checking a reliable weather website or app for detailed update...", - "final_answer_mode": "Web Search", - "query_processed.input_query": "What's the weather like today?", - "query_processed.output_response": "Successfully processed query using Web Search, generated 260 character response", - "query_processed.mode": "Web Search", - "query_processed.processing_time": 4.932007789611816, - "query_processed.response_length": 260 - }, - "events": [], - "links": [], - "trace_trace_id": "b1b13e60-5b32-44a0-9769-197a944989d0", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:51.967000000", - "trace_end_time": "2025-10-22 16:57:56.899000000", - "trace_duration_ms": 4932.393, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:51.967000000", - "updated_at": "2025-10-22 16:57:51.967000000" - }, - { - "span_id": "58d217aa-4350-43e8-8813-468149dc4107", - "trace_id": "b1b13e60-5b32-44a0-9769-197a944989d0", - "parent_span_id": "1c5c4291-bbb5-4f85-b6d9-b6e8df66f742", - "name": "agent.web_search_generation", - "start_time": "2025-10-22 16:57:51.967000000", - "end_time": "2025-10-22 16:57:56.899000000", - "duration_ms": 4931.796, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "web_search_agent", - "agent.operation": "web_search_generation", - "agent.capabilities": [ - "web_search", - "content_synthesis", - "response_generation" - ], - "agent.id": "noveum_web_search_agent", - "input_query": "What's the weather like today?", - "query_length": 30, - "query_type": "web_search_query", - "output_response": "Web Search Answer: The web search results do not provide specific information about today's weather. To find out the current weather conditions, I recommend checking a reliable weather website or app for detailed update...", - "web_search.results_count": 1, - "web_search.context_length": 995, - "web_search.response_length": 260, - "web_search.mode": "external_web_search" - }, - "events": [], - "links": [], - "trace_trace_id": "b1b13e60-5b32-44a0-9769-197a944989d0", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:51.967000000", - "trace_end_time": "2025-10-22 16:57:56.899000000", - "trace_duration_ms": 4932.393, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:51.967000000", - "updated_at": "2025-10-22 16:57:51.967000000" - }, - { - "span_id": "d89dd84e-0b5e-4c92-942b-18c4735713d4", - "trace_id": "b1b13e60-5b32-44a0-9769-197a944989d0", - "parent_span_id": "58d217aa-4350-43e8-8813-468149dc4107", - "name": "agent.llm_model_execution", - "start_time": "2025-10-22 16:57:55.269000000", - "end_time": "2025-10-22 16:57:56.899000000", - "duration_ms": 1629.7140000000002, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What's the weather like today?", - "query_length": 30, - "query_type": "web_search_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 1.629619836807251, - "model.latency_ms": 1629.619836807251, - "model.start_time": 1761152275.269459, - "model.end_time": 1761152276.8990788, - "model.performance_tier": "fast", - "model.prompt_tokens": 367, - "model.completion_tokens": 65, - "model.total_tokens": 432, - "model.tokens_per_second": 265.0925020932329, - "model.estimated_cost": 0.012960000000000001, - "model.efficiency_score": 0.6018518518518519, - "model.response_length": 260, - "model.response_quality": "high", - "model.output_response": "Model Response: The web search results do not provide specific information about today's weather. To find out the current weather conditions, I recommend checking a reliable weather website or app for detailed update...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "b1b13e60-5b32-44a0-9769-197a944989d0", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:51.967000000", - "trace_end_time": "2025-10-22 16:57:56.899000000", - "trace_duration_ms": 4932.393, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:51.967000000", - "updated_at": "2025-10-22 16:57:51.967000000" - }, - { - "span_id": "d076719d-da79-43fb-92ea-6015f856146b", - "trace_id": "b1b13e60-5b32-44a0-9769-197a944989d0", - "parent_span_id": "58d217aa-4350-43e8-8813-468149dc4107", - "name": "agent.web_search_evaluation_metrics", - "start_time": "2025-10-22 16:57:56.899000000", - "end_time": "2025-10-22 16:57:56.899000000", - "duration_ms": 0.049, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "web_search_evaluation_metrics", - "agent.capabilities": [ - "web_search_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What's the weather like today?", - "query_length": 30, - "query_type": "web_search_evaluation_query", - "web_search.results_count": 1, - "web_search.context_length": 995, - "web_search.context_synthesized": "Context from 1 web sources", - "web_search.search_effectiveness": 0.2, - "web_search.context_quality": "high", - "web_search.source_diversity": 1, - "web_search.information_synthesis": "low", - "web_search.external_knowledge_utilization": 0.995, - "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nLog in to WhatsApp Web for simple, reliable and private messaging on your desktop. Send and receive messages and files with ease, all for free. 2 days ago \u00b7 More videos you may like 00:13 Wanna be yours #kimpolin #colinbridgerton #benedictbri... 2 hours ago \u00b7 101 views 00:14 Viscount & Vicountess's Marital bliss #bridgerton \u2026 Nov 15, 2024 \u00b7 Learn the correct usage of \"what's the weather like today\" and \"how's the weather today\" in English. Discover differences, examples, alternatives and tips for choosing the right \u2026 Nov 15, 2024 \u00b7 Learn the correct usage of \"how's the weather today\" and \"what's the weather today\" in English. Discover differences, examples, alternatives and tips for choosing the right \u2026 2 days ago \u00b7 Get the latest hourly weather updates for Thane today. Detailed forecast including temperature, wind, rain, snow, and UV index. Stay informed about today's weather conditions in \u2026\nURL: https://duckduckgo.com/?q=What's+the+weather+like+today?\n\nUser Question: What's the weather like today?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", - "prompt.prompt_length": 1470, - "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nLog in to WhatsApp Web for simple, reliable and private messaging on your desktop. Send and receive messages and files with ease, all for free. 2 days ago \u00b7 More videos you...", - "prompt.instruction_following": "web_search_optimized", - "response.answer_length": 260, - "response.answer_completeness": "complete", - "response.response_quality": "medium", - "response.source_citation": 1, - "response.context_utilization": 0.2613065326633166, - "output_response": "Web Search Answer: The web search results do not provide specific information about today's weather. To find out the current weather conditions, I recommend checking a reliable weather website or app for detailed update...", - "evaluation.search_effectiveness": 0.2, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 1, - "evaluation.context_relevance": "high", - "evaluation.overall_quality": "low", - "evaluation.ready_for_production": true, - "web_search.search_strategy": "duckduckgo_api", - "web_search.real_time_data": true, - "web_search.external_sources": 1, - "web_search.information_freshness": "current", - "web_search.knowledge_synthesis": "single_source" - }, - "events": [], - "links": [], - "trace_trace_id": "b1b13e60-5b32-44a0-9769-197a944989d0", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:51.967000000", - "trace_end_time": "2025-10-22 16:57:56.899000000", - "trace_duration_ms": 4932.393, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:51.967000000", - "updated_at": "2025-10-22 16:57:51.967000000" - }, - { - "span_id": "41167ca4-e1ad-48a5-b5eb-aa272ad88a85", - "trace_id": "e4f3cc20-c137-4905-80fe-2d29b03c2231", - "parent_span_id": "56959f47-5219-4674-8c0d-e5fa928dcd45", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-22 16:57:47.477000000", - "end_time": "2025-10-22 16:57:47.477000000", - "duration_ms": 0.061, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What are the latest AI news today?", - "query_length": 34, - "query_type": "routing_evaluation_query", - "classification.mode": "Web Search", - "classification.rag_keyword_score": 0, - "classification.web_keyword_score": 3, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "general_knowledge", - "query.keyword_density": 0.42857142857142855, - "query.domain_affinity": "general", - "routing.decision": "Routed to Web Search based on analysis", - "routing.rationale": "RAG score: 0, Web score: 3, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "RAG", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.057692307692307696, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: Web Search (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "e4f3cc20-c137-4905-80fe-2d29b03c2231", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:47.477000000", - "trace_end_time": "2025-10-22 16:57:50.959000000", - "trace_duration_ms": 3481.856, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:47.477000000", - "updated_at": "2025-10-22 16:57:47.477000000" - }, - { - "span_id": "73005751-9eb1-4540-aa49-2ac580cd7418", - "trace_id": "e4f3cc20-c137-4905-80fe-2d29b03c2231", - "parent_span_id": "56959f47-5219-4674-8c0d-e5fa928dcd45", - "name": "agent.web_search_generation", - "start_time": "2025-10-22 16:57:47.477000000", - "end_time": "2025-10-22 16:57:50.958000000", - "duration_ms": 3480.8469999999998, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "web_search_agent", - "agent.operation": "web_search_generation", - "agent.capabilities": [ - "web_search", - "content_synthesis", - "response_generation" - ], - "agent.id": "noveum_web_search_agent", - "input_query": "What are the latest AI news today?", - "query_length": 34, - "query_type": "web_search_query", - "output_response": "Web Search Answer: The provided web search results do not contain specific information about the latest AI news today. They primarily focus on general news coverage from India and around the world, including politics, b...", - "web_search.results_count": 1, - "web_search.context_length": 887, - "web_search.response_length": 503, - "web_search.mode": "external_web_search" - }, - "events": [], - "links": [], - "trace_trace_id": "e4f3cc20-c137-4905-80fe-2d29b03c2231", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:47.477000000", - "trace_end_time": "2025-10-22 16:57:50.959000000", - "trace_duration_ms": 3481.856, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:47.477000000", - "updated_at": "2025-10-22 16:57:47.477000000" - }, - { - "span_id": "9e1b87be-bc29-4084-b37c-6e1220dde8eb", - "trace_id": "e4f3cc20-c137-4905-80fe-2d29b03c2231", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-22 16:57:47.477000000", - "end_time": "2025-10-22 16:57:50.959000000", - "duration_ms": 3481.7580000000003, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What are the latest AI news today?", - "query.length": 34, - "processing.mode": "Web Search", - "processing.time_seconds": 3.4813451766967773, - "processing.response_length": 503, - "processing.sources_count": 1, - "output_response": "Final Answer: The provided web search results do not contain specific information about the latest AI news today. They primarily focus on general news coverage from India and around the world, including politics, b...", - "final_answer_mode": "Web Search", - "query_processed.input_query": "What are the latest AI news today?", - "query_processed.output_response": "Successfully processed query using Web Search, generated 503 character response", - "query_processed.mode": "Web Search", - "query_processed.processing_time": 3.4813451766967773, - "query_processed.response_length": 503 - }, - "events": [], - "links": [], - "trace_trace_id": "e4f3cc20-c137-4905-80fe-2d29b03c2231", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:47.477000000", - "trace_end_time": "2025-10-22 16:57:50.959000000", - "trace_duration_ms": 3481.856, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:47.477000000", - "updated_at": "2025-10-22 16:57:47.477000000" - }, - { - "span_id": "56959f47-5219-4674-8c0d-e5fa928dcd45", - "trace_id": "e4f3cc20-c137-4905-80fe-2d29b03c2231", - "parent_span_id": "9e1b87be-bc29-4084-b37c-6e1220dde8eb", - "name": "agent.query_routing", - "start_time": "2025-10-22 16:57:47.477000000", - "end_time": "2025-10-22 16:57:50.958000000", - "duration_ms": 3481.254, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What are the latest AI news today?", - "query_length": 34, - "query_type": "routing_query", - "output_response": "Routed to Web Search for query processing", - "router.classification": "Web Search", - "router.confidence_score": 1, - "router.rag_keyword_score": 0, - "router.web_keyword_score": 3, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "e4f3cc20-c137-4905-80fe-2d29b03c2231", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:47.477000000", - "trace_end_time": "2025-10-22 16:57:50.959000000", - "trace_duration_ms": 3481.856, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:47.477000000", - "updated_at": "2025-10-22 16:57:47.477000000" - }, - { - "span_id": "3dbf95ba-3ef1-4294-93d4-217f4ada5ffe", - "trace_id": "e4f3cc20-c137-4905-80fe-2d29b03c2231", - "parent_span_id": "73005751-9eb1-4540-aa49-2ac580cd7418", - "name": "agent.llm_model_execution", - "start_time": "2025-10-22 16:57:47.864000000", - "end_time": "2025-10-22 16:57:50.958000000", - "duration_ms": 3093.442, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What are the latest AI news today?", - "query_length": 34, - "query_type": "web_search_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 3.0933361053466797, - "model.latency_ms": 3093.3361053466797, - "model.start_time": 1761152267.864976, - "model.end_time": 1761152270.958312, - "model.performance_tier": "medium", - "model.prompt_tokens": 341, - "model.completion_tokens": 125, - "model.total_tokens": 466, - "model.tokens_per_second": 150.6464167261171, - "model.estimated_cost": 0.013980000000000001, - "model.efficiency_score": 1.0793991416309012, - "model.response_length": 503, - "model.response_quality": "high", - "model.output_response": "Model Response: The provided web search results do not contain specific information about the latest AI news today. They primarily focus on general news coverage from India and around the world, including politics, b...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "e4f3cc20-c137-4905-80fe-2d29b03c2231", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:47.477000000", - "trace_end_time": "2025-10-22 16:57:50.959000000", - "trace_duration_ms": 3481.856, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:47.477000000", - "updated_at": "2025-10-22 16:57:47.477000000" - }, - { - "span_id": "f7fe795b-ce1b-4fcc-83f5-06a5f074f38d", - "trace_id": "e4f3cc20-c137-4905-80fe-2d29b03c2231", - "parent_span_id": "73005751-9eb1-4540-aa49-2ac580cd7418", - "name": "agent.web_search_evaluation_metrics", - "start_time": "2025-10-22 16:57:50.958000000", - "end_time": "2025-10-22 16:57:50.958000000", - "duration_ms": 0.026, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "web_search_evaluation_metrics", - "agent.capabilities": [ - "web_search_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What are the latest AI news today?", - "query_length": 34, - "query_type": "web_search_evaluation_query", - "web_search.results_count": 1, - "web_search.context_length": 887, - "web_search.context_synthesized": "Context from 1 web sources", - "web_search.search_effectiveness": 0.2, - "web_search.context_quality": "high", - "web_search.source_diversity": 1, - "web_search.information_synthesis": "low", - "web_search.external_knowledge_utilization": 0.887, - "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\n3 days ago \u00b7 Latest news headlines from India & around the world. Check out today\u2019s news coverage live with videos & photos on NDTV.com. 6 days ago \u00b7 India News | Latest India News | Read latest and breaking news from India. Today's top India news headlines, news on Indian politics, elections, government, business, \u2026 2 days ago \u00b7 Top News Stories of the day, Latest News Headlines, News Specials, Breaking News and Latest India News, World current affairs & Political News all around the clock at \u2026 NDTV.com provides latest news from India and around the world. Get breaking news alerts from India and follow today\u2019s live news updates in field of politics, business, technology, Bollywood ... 6 days ago \u00b7 Get Latest entertainment news from Bollywood, Hollywood and regional film industries.\nURL: https://duckduckgo.com/?q=What+are+the+latest+AI+news+today?\n\nUser Question: What are the latest AI news today?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", - "prompt.prompt_length": 1366, - "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\n3 days ago \u00b7 Latest news headlines from India & around the world. Check out today\u2019s news coverage live with videos & photos on NDTV.com. 6 days ago \u00b7 India News | Latest In...", - "prompt.instruction_following": "web_search_optimized", - "response.answer_length": 503, - "response.answer_completeness": "complete", - "response.response_quality": "medium", - "response.source_citation": 1, - "response.context_utilization": 0.5670800450958287, - "output_response": "Web Search Answer: The provided web search results do not contain specific information about the latest AI news today. They primarily focus on general news coverage from India and around the world, including politics, b...", - "evaluation.search_effectiveness": 0.2, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 1, - "evaluation.context_relevance": "high", - "evaluation.overall_quality": "low", - "evaluation.ready_for_production": true, - "web_search.search_strategy": "duckduckgo_api", - "web_search.real_time_data": true, - "web_search.external_sources": 1, - "web_search.information_freshness": "current", - "web_search.knowledge_synthesis": "single_source" - }, - "events": [], - "links": [], - "trace_trace_id": "e4f3cc20-c137-4905-80fe-2d29b03c2231", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:47.477000000", - "trace_end_time": "2025-10-22 16:57:50.959000000", - "trace_duration_ms": 3481.856, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:47.477000000", - "updated_at": "2025-10-22 16:57:47.477000000" - }, - { - "span_id": "b9c9d21e-5f76-4e5a-b99e-2b4ab821caee", - "trace_id": "823b7927-10bb-49ef-a94e-1b914c298bfb", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-22 16:57:38.501000000", - "end_time": "2025-10-22 16:57:46.469000000", - "duration_ms": 7968.089, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What are the benefits of using Noveum Trace?", - "query.length": 44, - "processing.mode": "RAG", - "processing.time_seconds": 7.968040943145752, - "processing.response_length": 1396, - "processing.sources_count": 5, - "output_response": "Final Answer: Using Noveum Trace offers several benefits, including:\n\n1. **Real-Time Monitoring**: The Noveum platform provides a real-time dashboard for analyzing traces and performance, with live updates as new t...", - "final_answer_mode": "RAG", - "query_processed.input_query": "What are the benefits of using Noveum Trace?", - "query_processed.output_response": "Successfully processed query using RAG, generated 1396 character response", - "query_processed.mode": "RAG", - "query_processed.processing_time": 7.968040943145752, - "query_processed.response_length": 1396 - }, - "events": [], - "links": [], - "trace_trace_id": "823b7927-10bb-49ef-a94e-1b914c298bfb", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:38.501000000", - "trace_end_time": "2025-10-22 16:57:46.469000000", - "trace_duration_ms": 7968.149, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:38.501000000", - "updated_at": "2025-10-22 16:57:38.501000000" - }, - { - "span_id": "bc85c745-4c27-4aa4-bc00-4c9f10a2f049", - "trace_id": "823b7927-10bb-49ef-a94e-1b914c298bfb", - "parent_span_id": "b9c9d21e-5f76-4e5a-b99e-2b4ab821caee", - "name": "agent.query_routing", - "start_time": "2025-10-22 16:57:38.501000000", - "end_time": "2025-10-22 16:57:46.469000000", - "duration_ms": 7968.001, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What are the benefits of using Noveum Trace?", - "query_length": 44, - "query_type": "routing_query", - "output_response": "Routed to RAG for query processing", - "router.classification": "RAG", - "router.confidence_score": 1, - "router.rag_keyword_score": 2, - "router.web_keyword_score": 0, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "823b7927-10bb-49ef-a94e-1b914c298bfb", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:38.501000000", - "trace_end_time": "2025-10-22 16:57:46.469000000", - "trace_duration_ms": 7968.149, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:38.501000000", - "updated_at": "2025-10-22 16:57:38.501000000" - }, - { - "span_id": "3ab8cc4c-f5ed-410a-b4ee-48d0954b756d", - "trace_id": "823b7927-10bb-49ef-a94e-1b914c298bfb", - "parent_span_id": "bc85c745-4c27-4aa4-bc00-4c9f10a2f049", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-22 16:57:38.501000000", - "end_time": "2025-10-22 16:57:38.501000000", - "duration_ms": 0.09899999999999999, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What are the benefits of using Noveum Trace?", - "query_length": 44, - "query_type": "routing_evaluation_query", - "classification.mode": "RAG", - "classification.rag_keyword_score": 2, - "classification.web_keyword_score": 0, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "noveum_specific", - "query.keyword_density": 0.25, - "query.domain_affinity": "noveum", - "routing.decision": "Routed to RAG based on analysis", - "routing.rationale": "RAG score: 2, Web score: 0, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "Web Search", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.038461538461538464, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: RAG (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "823b7927-10bb-49ef-a94e-1b914c298bfb", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:38.501000000", - "trace_end_time": "2025-10-22 16:57:46.469000000", - "trace_duration_ms": 7968.149, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:38.501000000", - "updated_at": "2025-10-22 16:57:38.501000000" - }, - { - "span_id": "67292076-ec8a-4856-9c5b-726868552157", - "trace_id": "823b7927-10bb-49ef-a94e-1b914c298bfb", - "parent_span_id": "bc85c745-4c27-4aa4-bc00-4c9f10a2f049", - "name": "agent.llm-rag", - "start_time": "2025-10-22 16:57:38.501000000", - "end_time": "2025-10-22 16:57:46.469000000", - "duration_ms": 7967.679, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "rag_agent", - "agent.operation": "llm-rag", - "agent.capabilities": [ - "document_retrieval", - "context_generation", - "response_generation" - ], - "agent.id": "noveum_rag_agent", - "input_query": "What are the benefits of using Noveum Trace?", - "query_length": 44, - "query_type": "rag_query", - "output_response": "RAG Answer: Using Noveum Trace offers several benefits, including:\n\n1. **Real-Time Monitoring**: The Noveum platform provides a real-time dashboard for analyzing traces and performance, with live updates as new t...", - "rag.context_length": 2773, - "rag.sources_count": 5, - "rag.answer_length": 1396, - "rag.mode": "retrieval_augmented_generation" - }, - "events": [], - "links": [], - "trace_trace_id": "823b7927-10bb-49ef-a94e-1b914c298bfb", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:38.501000000", - "trace_end_time": "2025-10-22 16:57:46.469000000", - "trace_duration_ms": 7968.149, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:38.501000000", - "updated_at": "2025-10-22 16:57:38.501000000" - }, - { - "span_id": "d044ae4a-c463-4c1b-abce-a9a18f045e75", - "trace_id": "823b7927-10bb-49ef-a94e-1b914c298bfb", - "parent_span_id": "67292076-ec8a-4856-9c5b-726868552157", - "name": "agent.llm_model_execution", - "start_time": "2025-10-22 16:57:38.901000000", - "end_time": "2025-10-22 16:57:45.885000000", - "duration_ms": 6984.242, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What are the benefits of using Noveum Trace?", - "query_length": 44, - "query_type": "rag_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 6.984165906906128, - "model.latency_ms": 6984.165906906128, - "model.start_time": 1761152258.9012191, - "model.end_time": 1761152265.885385, - "model.performance_tier": "slow", - "model.prompt_tokens": 828, - "model.completion_tokens": 349, - "model.total_tokens": 1177, - "model.tokens_per_second": 168.524060809631, - "model.estimated_cost": 0.03531, - "model.efficiency_score": 1.1860662701784197, - "model.response_length": 1396, - "model.response_quality": "high", - "model.output_response": "Model Response: Using Noveum Trace offers several benefits, including:\n\n1. **Real-Time Monitoring**: The Noveum platform provides a real-time dashboard for analyzing traces and performance, with live updates as new t...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "823b7927-10bb-49ef-a94e-1b914c298bfb", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:38.501000000", - "trace_end_time": "2025-10-22 16:57:46.469000000", - "trace_duration_ms": 7968.149, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:38.501000000", - "updated_at": "2025-10-22 16:57:38.501000000" - }, - { - "span_id": "5b1548cf-0de7-49d4-b9b8-9ad1f9f2abc9", - "trace_id": "823b7927-10bb-49ef-a94e-1b914c298bfb", - "parent_span_id": "67292076-ec8a-4856-9c5b-726868552157", - "name": "agent.rag_evaluation_metrics", - "start_time": "2025-10-22 16:57:45.885000000", - "end_time": "2025-10-22 16:57:45.885000000", - "duration_ms": 0.061, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "rag_evaluation_metrics", - "agent.capabilities": [ - "retrieval_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What are the benefits of using Noveum Trace?", - "query_length": 44, - "query_type": "rag_evaluation_query", - "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/docs/platform/dashboard):\n- Bulk Export: Download multiple traces simultaneously\n- Comparative Analysis: Compare performance across multiple traces\nReal-Time Updates\n- Live Refresh: Automatic updates as new traces arrive\n- Connection Monitoring: Real-time status of your t...", - "retrieval.context_length": 2773, - "retrieval.sources_count": 5, - "retrieval.context_quality": "high", - "retrieval.effectiveness": 1, - "retrieval.context_utilization": 2.773, - "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/docs/platform/dashboard):\n- Bulk Export: Download multiple traces simultaneously\n- Comparative Analysis: Compare performance across multiple traces\nReal-Time Updates\n- Live Refresh: Automatic updates as new traces arrive\n- Connection Monitoring: Real-time status of your trace ingestion pipeline\n- Performance Indicators: Live metrics for system health monitoring\nReady to dive deeper? Explore Projects & Environments to organize your AI applications, or check out Team Collaboration to share insights with your team.\nGet Earl...\n\nSource 2 (https://noveum.ai/en/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\n\ud83d\udcd8 TypeScript SDK (\n@noveum/trace\n)- Framework integrations for Next.js, Express.js, and Hono\n- TypeScript-first design with full type safety\n- Universal compatibility (Node.js, Edge Runtime, browsers)\n-\n\ud83d\udcca Noveum Platform\n- Real-time dashboard for analyzing traces and performance\n- Ad...\n\nSource 3 (https://noveum.ai/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\n\ud83d\udcd8 TypeScript SDK (\n@noveum/trace\n)- Framework integrations for Next.js, Express.js, and Hono\n- TypeScript-first design with full type safety\n- Universal compatibility (Node.js, Edge Runtime, browsers)\n-\n\ud83d\udcca Noveum Platform\n- Real-time dashboard for analyzing traces and performance\n- Ad...\n\nSource 4 (https://noveum.ai/en/docs/getting-started/overview):\n- Advanced search & filtering across traces and spans\n- Cost analysis and optimization recommendations\n- Team collaboration with shared insights and alerts\n\ud83d\udd0d What Noveum Traces\nLLM Operations\n- Model calls across all providers (OpenAI, Anthropic, Google, etc.)\n- Token usage and cost calculation\n- Prompt engineering effectiveness\n- Response quality metrics\nRAG Pipelines\n- Document retrieval performance and relevance\n- Embedding generation costs and latency\n- Context assembly and prompt constructi...\n\nSource 5 (https://noveum.ai/docs/getting-started/overview):\n- Advanced search & filtering across traces and spans\n- Cost analysis and optimization recommendations\n- Team collaboration with shared insights and alerts\n\ud83d\udd0d What Noveum Traces\nLLM Operations\n- Model calls across all providers (OpenAI, Anthropic, Google, etc.)\n- Token usage and cost calculation\n- Prompt engineering effectiveness\n- Response quality metrics\nRAG Pipelines\n- Document retrieval performance and relevance\n- Embedding generation costs and latency\n- Context assembly and prompt constructi...\n\nUser Question: What are the benefits of using Noveum Trace?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", - "prompt.prompt_length": 3313, - "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/docs/platform/dashboard):\n- Bulk Export: Download multiple traces simultaneously\n- Comparative Analysis: Compare performance across multiple traces\nReal-Time Updates\n- Live...", - "prompt.instruction_following": "rag_optimized", - "response.answer_length": 1396, - "response.answer_completeness": "complete", - "response.response_quality": "high", - "response.source_citation": 5, - "response.context_utilization": 0.5034258925351605, - "output_response": "RAG Answer: Using Noveum Trace offers several benefits, including:\n\n1. **Real-Time Monitoring**: The Noveum platform provides a real-time dashboard for analyzing traces and performance, with live updates as new t...", - "evaluation.retrieval_effectiveness": 1, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 5, - "evaluation.context_relevance": "high", - "evaluation.overall_quality": "high", - "evaluation.ready_for_production": true, - "rag.retrieval_strategy": "semantic_similarity", - "rag.vector_search_results": 5, - "rag.context_synthesis": "multi_source", - "rag.document_coverage": 1, - "rag.information_density": 0.5034258925351605 - }, - "events": [], - "links": [], - "trace_trace_id": "823b7927-10bb-49ef-a94e-1b914c298bfb", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:38.501000000", - "trace_end_time": "2025-10-22 16:57:46.469000000", - "trace_duration_ms": 7968.149, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:38.501000000", - "updated_at": "2025-10-22 16:57:38.501000000" - }, - { - "span_id": "aaba719b-36ec-46ab-af2e-3ea8c18e9872", - "trace_id": "1ae0d6c4-d1fb-4239-b536-3333187d44aa", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-22 16:57:32.800000000", - "end_time": "2025-10-22 16:57:37.494000000", - "duration_ms": 4694.408, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "How do I configure Noveum for my system?", - "query.length": 40, - "processing.mode": "RAG", - "processing.time_seconds": 4.694374084472656, - "processing.response_length": 553, - "processing.sources_count": 5, - "output_response": "Final Answer: To configure Noveum for your system, you can refer to the SDK Integration Guide available in the Noveum documentation. This guide will help you trace your AI applications with minimal code changes. Th...", - "final_answer_mode": "RAG", - "query_processed.input_query": "How do I configure Noveum for my system?", - "query_processed.output_response": "Successfully processed query using RAG, generated 553 character response", - "query_processed.mode": "RAG", - "query_processed.processing_time": 4.694374084472656, - "query_processed.response_length": 553 - }, - "events": [], - "links": [], - "trace_trace_id": "1ae0d6c4-d1fb-4239-b536-3333187d44aa", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:32.800000000", - "trace_end_time": "2025-10-22 16:57:37.494000000", - "trace_duration_ms": 4694.468, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:32.800000000", - "updated_at": "2025-10-22 16:57:32.800000000" - }, - { - "span_id": "4254e6f9-10b8-42ba-939d-d7c124f80f9a", - "trace_id": "1ae0d6c4-d1fb-4239-b536-3333187d44aa", - "parent_span_id": "6d5d074e-8fd3-4d33-88fb-7f54b0959030", - "name": "agent.llm-rag", - "start_time": "2025-10-22 16:57:32.800000000", - "end_time": "2025-10-22 16:57:37.494000000", - "duration_ms": 4694.155000000001, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "rag_agent", - "agent.operation": "llm-rag", - "agent.capabilities": [ - "document_retrieval", - "context_generation", - "response_generation" - ], - "agent.id": "noveum_rag_agent", - "input_query": "How do I configure Noveum for my system?", - "query_length": 40, - "query_type": "rag_query", - "output_response": "RAG Answer: To configure Noveum for your system, you can refer to the SDK Integration Guide available in the Noveum documentation. This guide will help you trace your AI applications with minimal code changes. Th...", - "rag.context_length": 2237, - "rag.sources_count": 5, - "rag.answer_length": 553, - "rag.mode": "retrieval_augmented_generation" - }, - "events": [], - "links": [], - "trace_trace_id": "1ae0d6c4-d1fb-4239-b536-3333187d44aa", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:32.800000000", - "trace_end_time": "2025-10-22 16:57:37.494000000", - "trace_duration_ms": 4694.468, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:32.800000000", - "updated_at": "2025-10-22 16:57:32.800000000" - }, - { - "span_id": "6d5d074e-8fd3-4d33-88fb-7f54b0959030", - "trace_id": "1ae0d6c4-d1fb-4239-b536-3333187d44aa", - "parent_span_id": "aaba719b-36ec-46ab-af2e-3ea8c18e9872", - "name": "agent.query_routing", - "start_time": "2025-10-22 16:57:32.800000000", - "end_time": "2025-10-22 16:57:37.494000000", - "duration_ms": 4694.318, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "How do I configure Noveum for my system?", - "query_length": 40, - "query_type": "routing_query", - "output_response": "Routed to RAG for query processing", - "router.classification": "RAG", - "router.confidence_score": 1, - "router.rag_keyword_score": 2, - "router.web_keyword_score": 0, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "1ae0d6c4-d1fb-4239-b536-3333187d44aa", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:32.800000000", - "trace_end_time": "2025-10-22 16:57:37.494000000", - "trace_duration_ms": 4694.468, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:32.800000000", - "updated_at": "2025-10-22 16:57:32.800000000" - }, - { - "span_id": "2cc2ab0b-2572-4a75-88a3-18b3a1eaca78", - "trace_id": "1ae0d6c4-d1fb-4239-b536-3333187d44aa", - "parent_span_id": "6d5d074e-8fd3-4d33-88fb-7f54b0959030", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-22 16:57:32.800000000", - "end_time": "2025-10-22 16:57:32.800000000", - "duration_ms": 0.04, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: How do I configure Noveum for my system?", - "query_length": 40, - "query_type": "routing_evaluation_query", - "classification.mode": "RAG", - "classification.rag_keyword_score": 2, - "classification.web_keyword_score": 0, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "noveum_specific", - "query.keyword_density": 0.25, - "query.domain_affinity": "noveum", - "routing.decision": "Routed to RAG based on analysis", - "routing.rationale": "RAG score: 2, Web score: 0, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "Web Search", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.038461538461538464, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: RAG (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "1ae0d6c4-d1fb-4239-b536-3333187d44aa", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:32.800000000", - "trace_end_time": "2025-10-22 16:57:37.494000000", - "trace_duration_ms": 4694.468, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:32.800000000", - "updated_at": "2025-10-22 16:57:32.800000000" - }, - { - "span_id": "f1d9f779-3312-43c1-b549-00fc874429f0", - "trace_id": "1ae0d6c4-d1fb-4239-b536-3333187d44aa", - "parent_span_id": "4254e6f9-10b8-42ba-939d-d7c124f80f9a", - "name": "agent.llm_model_execution", - "start_time": "2025-10-22 16:57:33.281000000", - "end_time": "2025-10-22 16:57:37.060000000", - "duration_ms": 3778.743, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: How do I configure Noveum for my system?", - "query_length": 40, - "query_type": "rag_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 3.7786097526550293, - "model.latency_ms": 3778.6097526550293, - "model.start_time": 1761152253.28163, - "model.end_time": 1761152257.0602398, - "model.performance_tier": "medium", - "model.prompt_tokens": 693, - "model.completion_tokens": 138, - "model.total_tokens": 831, - "model.tokens_per_second": 219.92215507730066, - "model.estimated_cost": 0.02493, - "model.efficiency_score": 0.6654632972322503, - "model.response_length": 553, - "model.response_quality": "high", - "model.output_response": "Model Response: To configure Noveum for your system, you can refer to the SDK Integration Guide available in the Noveum documentation. This guide will help you trace your AI applications with minimal code changes. Th...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "1ae0d6c4-d1fb-4239-b536-3333187d44aa", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:32.800000000", - "trace_end_time": "2025-10-22 16:57:37.494000000", - "trace_duration_ms": 4694.468, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:32.800000000", - "updated_at": "2025-10-22 16:57:32.800000000" - }, - { - "span_id": "5500be90-0cfa-44dd-a6e3-e72ac9f0a79e", - "trace_id": "1ae0d6c4-d1fb-4239-b536-3333187d44aa", - "parent_span_id": "4254e6f9-10b8-42ba-939d-d7c124f80f9a", - "name": "agent.rag_evaluation_metrics", - "start_time": "2025-10-22 16:57:37.060000000", - "end_time": "2025-10-22 16:57:37.060000000", - "duration_ms": 0.076, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "rag_evaluation_metrics", - "agent.capabilities": [ - "retrieval_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: How do I configure Noveum for my system?", - "query_length": 40, - "query_type": "rag_evaluation_query", - "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/docs):\nWelcome to Noveum.ai Docs\nComprehensive AI tracing and observability for LLM applications, RAG systems, and AI agents\nWelcome to the Noveum.ai documentation! Here you'll find everything you need to integrate, configure, and optimize your AI applications using ou...", - "retrieval.context_length": 2237, - "retrieval.sources_count": 5, - "retrieval.context_quality": "high", - "retrieval.effectiveness": 1, - "retrieval.context_utilization": 2.237, - "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/docs):\nWelcome to Noveum.ai Docs\nComprehensive AI tracing and observability for LLM applications, RAG systems, and AI agents\nWelcome to the Noveum.ai documentation! Here you'll find everything you need to integrate, configure, and optimize your AI applications using our comprehensive tracing and observability platform.\nWe provide powerful SDKs for Python and TypeScript that enable you to trace LLM calls, RAG pipelines, and multi-agent workflows with minimal code changes. Our documentation is organized ...\n\nSource 2 (https://noveum.ai/docs):\nWelcome to Noveum.ai Docs\nComprehensive AI tracing and observability for LLM applications, RAG systems, and AI agents\nWelcome to the Noveum.ai documentation! Here you'll find everything you need to integrate, configure, and optimize your AI applications using our comprehensive tracing and observability platform.\nWe provide powerful SDKs for Python and TypeScript that enable you to trace LLM calls, RAG pipelines, and multi-agent workflows with minimal code changes. Our documentation is organized ...\n\nSource 3 (https://noveum.ai/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nSource 4 (https://noveum.ai/en/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nSource 5 (https://noveum.ai/en/docs):\n- \ud83c\udf0d Universal: Works across frameworks, clouds, and deployment models\n\ud83e\udd1d Community & Support\n- \ud83d\udcac Discord Community: Join our Discord\n- \ud83d\udce7 Email Support: [email protected]\n- \ud83d\udc1b Bug Reports: GitHub Issues\n- \ud83d\udcd6 Knowledge Base: Help Center\nReady to get started? Head to our SDK Integration Guide to begin tracing your AI applications in under 5 minutes!\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability ...\n\nUser Question: How do I configure Noveum for my system?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", - "prompt.prompt_length": 2773, - "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/docs):\nWelcome to Noveum.ai Docs\nComprehensive AI tracing and observability for LLM applications, RAG systems, and AI agents\nWelcome to the Noveum.ai documentation! Here...", - "prompt.instruction_following": "rag_optimized", - "response.answer_length": 553, - "response.answer_completeness": "complete", - "response.response_quality": "high", - "response.source_citation": 5, - "response.context_utilization": 0.24720607957085383, - "output_response": "RAG Answer: To configure Noveum for your system, you can refer to the SDK Integration Guide available in the Noveum documentation. This guide will help you trace your AI applications with minimal code changes. Th...", - "evaluation.retrieval_effectiveness": 1, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 5, - "evaluation.context_relevance": "high", - "evaluation.overall_quality": "high", - "evaluation.ready_for_production": true, - "rag.retrieval_strategy": "semantic_similarity", - "rag.vector_search_results": 5, - "rag.context_synthesis": "multi_source", - "rag.document_coverage": 1, - "rag.information_density": 0.24720607957085383 - }, - "events": [], - "links": [], - "trace_trace_id": "1ae0d6c4-d1fb-4239-b536-3333187d44aa", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:32.800000000", - "trace_end_time": "2025-10-22 16:57:37.494000000", - "trace_duration_ms": 4694.468, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:32.800000000", - "updated_at": "2025-10-22 16:57:32.800000000" - }, - { - "span_id": "3068013a-6844-4f3c-8ec2-854a7353591e", - "trace_id": "fb0d31fc-2465-4ecd-973c-4cd443d9befa", - "parent_span_id": "a80d8fab-d5dc-4417-a5e5-9a2a76ed7419", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-22 16:57:24.828000000", - "end_time": "2025-10-22 16:57:24.828000000", - "duration_ms": 0.038, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What monitoring capabilities does Noveum provide?", - "query_length": 49, - "query_type": "routing_evaluation_query", - "classification.mode": "RAG", - "classification.rag_keyword_score": 2, - "classification.web_keyword_score": 0, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "noveum_specific", - "query.keyword_density": 0.3333333333333333, - "query.domain_affinity": "noveum", - "routing.decision": "Routed to RAG based on analysis", - "routing.rationale": "RAG score: 2, Web score: 0, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "Web Search", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.038461538461538464, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: RAG (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "fb0d31fc-2465-4ecd-973c-4cd443d9befa", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:24.828000000", - "trace_end_time": "2025-10-22 16:57:31.793000000", - "trace_duration_ms": 6965.302000000001, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:24.828000000", - "updated_at": "2025-10-22 16:57:24.828000000" - }, - { - "span_id": "189b89e3-fbbd-41de-8521-efb6a68f62f2", - "trace_id": "fb0d31fc-2465-4ecd-973c-4cd443d9befa", - "parent_span_id": "a80d8fab-d5dc-4417-a5e5-9a2a76ed7419", - "name": "agent.llm-rag", - "start_time": "2025-10-22 16:57:24.828000000", - "end_time": "2025-10-22 16:57:31.793000000", - "duration_ms": 6964.955, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "rag_agent", - "agent.operation": "llm-rag", - "agent.capabilities": [ - "document_retrieval", - "context_generation", - "response_generation" - ], - "agent.id": "noveum_rag_agent", - "input_query": "What monitoring capabilities does Noveum provide?", - "query_length": 49, - "query_type": "rag_query", - "output_response": "RAG Answer: Noveum provides comprehensive monitoring capabilities for AI applications, including the ability to monitor, trace, and optimize AI agents across various frameworks such as LangChain, CrewAI, AutoGen,...", - "rag.context_length": 2740, - "rag.sources_count": 5, - "rag.answer_length": 582, - "rag.mode": "retrieval_augmented_generation" - }, - "events": [], - "links": [], - "trace_trace_id": "fb0d31fc-2465-4ecd-973c-4cd443d9befa", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:24.828000000", - "trace_end_time": "2025-10-22 16:57:31.793000000", - "trace_duration_ms": 6965.302000000001, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:24.828000000", - "updated_at": "2025-10-22 16:57:24.828000000" - }, - { - "span_id": "f5cffb7d-341b-4cde-b162-02208042463d", - "trace_id": "fb0d31fc-2465-4ecd-973c-4cd443d9befa", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-22 16:57:24.828000000", - "end_time": "2025-10-22 16:57:31.793000000", - "duration_ms": 6965.236, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What monitoring capabilities does Noveum provide?", - "query.length": 49, - "processing.mode": "RAG", - "processing.time_seconds": 6.965206146240234, - "processing.response_length": 582, - "processing.sources_count": 5, - "output_response": "Final Answer: Noveum provides comprehensive monitoring capabilities for AI applications, including the ability to monitor, trace, and optimize AI agents across various frameworks such as LangChain, CrewAI, AutoGen,...", - "final_answer_mode": "RAG", - "query_processed.input_query": "What monitoring capabilities does Noveum provide?", - "query_processed.output_response": "Successfully processed query using RAG, generated 582 character response", - "query_processed.mode": "RAG", - "query_processed.processing_time": 6.965206146240234, - "query_processed.response_length": 582 - }, - "events": [], - "links": [], - "trace_trace_id": "fb0d31fc-2465-4ecd-973c-4cd443d9befa", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:24.828000000", - "trace_end_time": "2025-10-22 16:57:31.793000000", - "trace_duration_ms": 6965.302000000001, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:24.828000000", - "updated_at": "2025-10-22 16:57:24.828000000" - }, - { - "span_id": "a80d8fab-d5dc-4417-a5e5-9a2a76ed7419", - "trace_id": "fb0d31fc-2465-4ecd-973c-4cd443d9befa", - "parent_span_id": "f5cffb7d-341b-4cde-b162-02208042463d", - "name": "agent.query_routing", - "start_time": "2025-10-22 16:57:24.828000000", - "end_time": "2025-10-22 16:57:31.793000000", - "duration_ms": 6965.17, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What monitoring capabilities does Noveum provide?", - "query_length": 49, - "query_type": "routing_query", - "output_response": "Routed to RAG for query processing", - "router.classification": "RAG", - "router.confidence_score": 1, - "router.rag_keyword_score": 2, - "router.web_keyword_score": 0, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "fb0d31fc-2465-4ecd-973c-4cd443d9befa", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:24.828000000", - "trace_end_time": "2025-10-22 16:57:31.793000000", - "trace_duration_ms": 6965.302000000001, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:24.828000000", - "updated_at": "2025-10-22 16:57:24.828000000" - }, - { - "span_id": "a2b632b8-fbcb-480f-9292-9f513c64df38", - "trace_id": "fb0d31fc-2465-4ecd-973c-4cd443d9befa", - "parent_span_id": "189b89e3-fbbd-41de-8521-efb6a68f62f2", - "name": "agent.llm_model_execution", - "start_time": "2025-10-22 16:57:27.824000000", - "end_time": "2025-10-22 16:57:31.155000000", - "duration_ms": 3330.989, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What monitoring capabilities does Noveum provide?", - "query_length": 49, - "query_type": "rag_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 3.3309478759765625, - "model.latency_ms": 3330.9478759765625, - "model.start_time": 1761152247.824474, - "model.end_time": 1761152251.155422, - "model.performance_tier": "medium", - "model.prompt_tokens": 821, - "model.completion_tokens": 145, - "model.total_tokens": 966, - "model.tokens_per_second": 290.00754018607677, - "model.estimated_cost": 0.028980000000000002, - "model.efficiency_score": 0.6024844720496895, - "model.response_length": 582, - "model.response_quality": "high", - "model.output_response": "Model Response: Noveum provides comprehensive monitoring capabilities for AI applications, including the ability to monitor, trace, and optimize AI agents across various frameworks such as LangChain, CrewAI, AutoGen,...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "fb0d31fc-2465-4ecd-973c-4cd443d9befa", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:24.828000000", - "trace_end_time": "2025-10-22 16:57:31.793000000", - "trace_duration_ms": 6965.302000000001, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:24.828000000", - "updated_at": "2025-10-22 16:57:24.828000000" - }, - { - "span_id": "40ff54c4-07ca-4b2d-a9b4-7e3ae7adcd81", - "trace_id": "fb0d31fc-2465-4ecd-973c-4cd443d9befa", - "parent_span_id": "189b89e3-fbbd-41de-8521-efb6a68f62f2", - "name": "agent.rag_evaluation_metrics", - "start_time": "2025-10-22 16:57:31.155000000", - "end_time": "2025-10-22 16:57:31.155000000", - "duration_ms": 0.018000000000000002, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "rag_evaluation_metrics", - "agent.capabilities": [ - "retrieval_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What monitoring capabilities does Noveum provide?", - "query_length": 49, - "query_type": "rag_evaluation_query", - "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en):\nMonitor all your AI Agents\nimprove AI Agents today\nNoveum.ai helps you monitor, trace, and optimize your AI applications.\nNoveum.ai works with any AI framework \u2013 LangChain, CrewAI, AutoGen, custom implementations, or direct LLM calls. One dashboard shows everything.\n...", - "retrieval.context_length": 2740, - "retrieval.sources_count": 5, - "retrieval.context_quality": "high", - "retrieval.effectiveness": 1, - "retrieval.context_utilization": 2.74, - "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en):\nMonitor all your AI Agents\nimprove AI Agents today\nNoveum.ai helps you monitor, trace, and optimize your AI applications.\nNoveum.ai works with any AI framework \u2013 LangChain, CrewAI, AutoGen, custom implementations, or direct LLM calls. One dashboard shows everything.\nMonitor, Evaluate, Improve Your AI Agents\nThe control plane for AI agents.\nMonitor Everything, Miss Nothing\nOur lightweight SDKs capture every trace and span across your AI agent ecosystem\u2014from simple LLM calls to complex multi-agent...\n\nSource 2 (https://noveum.ai/):\nMonitor all your AI Agents\nimprove AI Agents today\nNoveum.ai helps you monitor, trace, and optimize your AI applications.\nNoveum.ai works with any AI framework \u2013 LangChain, CrewAI, AutoGen, custom implementations, or direct LLM calls. One dashboard shows everything.\nMonitor, Evaluate, Improve Your AI Agents\nThe control plane for AI agents.\nMonitor Everything, Miss Nothing\nOur lightweight SDKs capture every trace and span across your AI agent ecosystem\u2014from simple LLM calls to complex multi-agent...\n\nSource 3 (https://noveum.ai):\nMonitor all your AI Agents\nimprove AI Agents today\nNoveum.ai helps you monitor, trace, and optimize your AI applications.\nNoveum.ai works with any AI framework \u2013 LangChain, CrewAI, AutoGen, custom implementations, or direct LLM calls. One dashboard shows everything.\nMonitor, Evaluate, Improve Your AI Agents\nThe control plane for AI agents.\nMonitor Everything, Miss Nothing\nOur lightweight SDKs capture every trace and span across your AI agent ecosystem\u2014from simple LLM calls to complex multi-agent...\n\nSource 4 (https://noveum.ai/en/docs/getting-started/overview):\nNoveum.ai Overview\nComprehensive AI tracing and observability platform for LLM applications, RAG systems, and AI agents\nWelcome to Noveum.ai\u2014the comprehensive tracing and observability platform built specifically for AI applications. Whether you're building LLM-powered chatbots, RAG systems, multi-agent workflows, or any AI-driven application, Noveum provides the insights you need to understand, debug, and optimize your systems.\n\ud83c\udfaf Why AI Applications Need Specialized Observability\nTraditional mo...\n\nSource 5 (https://noveum.ai/docs/getting-started/overview):\nNoveum.ai Overview\nComprehensive AI tracing and observability platform for LLM applications, RAG systems, and AI agents\nWelcome to Noveum.ai\u2014the comprehensive tracing and observability platform built specifically for AI applications. Whether you're building LLM-powered chatbots, RAG systems, multi-agent workflows, or any AI-driven application, Noveum provides the insights you need to understand, debug, and optimize your systems.\n\ud83c\udfaf Why AI Applications Need Specialized Observability\nTraditional mo...\n\nUser Question: What monitoring capabilities does Noveum provide?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", - "prompt.prompt_length": 3285, - "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en):\nMonitor all your AI Agents\nimprove AI Agents today\nNoveum.ai helps you monitor, trace, and optimize your AI applications.\nNoveum.ai works with any AI framework \u2013 LangC...", - "prompt.instruction_following": "rag_optimized", - "response.answer_length": 582, - "response.answer_completeness": "complete", - "response.response_quality": "high", - "response.source_citation": 5, - "response.context_utilization": 0.2124087591240876, - "output_response": "RAG Answer: Noveum provides comprehensive monitoring capabilities for AI applications, including the ability to monitor, trace, and optimize AI agents across various frameworks such as LangChain, CrewAI, AutoGen,...", - "evaluation.retrieval_effectiveness": 1, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 5, - "evaluation.context_relevance": "high", - "evaluation.overall_quality": "high", - "evaluation.ready_for_production": true, - "rag.retrieval_strategy": "semantic_similarity", - "rag.vector_search_results": 5, - "rag.context_synthesis": "multi_source", - "rag.document_coverage": 1, - "rag.information_density": 0.2124087591240876 - }, - "events": [], - "links": [], - "trace_trace_id": "fb0d31fc-2465-4ecd-973c-4cd443d9befa", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:24.828000000", - "trace_end_time": "2025-10-22 16:57:31.793000000", - "trace_duration_ms": 6965.302000000001, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:24.828000000", - "updated_at": "2025-10-22 16:57:24.828000000" - }, - { - "span_id": "b0f0a2e0-c1f7-4ba4-8b75-7090e931a738", - "trace_id": "3bbd0b36-c30f-4126-b62a-dd0fae1c701d", - "parent_span_id": "4db80813-e62f-48ec-9091-62d569527caf", - "name": "agent.query_routing", - "start_time": "2025-10-22 16:57:19.339000000", - "end_time": "2025-10-22 16:57:23.822000000", - "duration_ms": 4482.909000000001, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "How does Noveum handle agent tracing?", - "query_length": 37, - "query_type": "routing_query", - "output_response": "Routed to RAG for query processing", - "router.classification": "RAG", - "router.confidence_score": 1, - "router.rag_keyword_score": 2, - "router.web_keyword_score": 0, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "3bbd0b36-c30f-4126-b62a-dd0fae1c701d", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:19.339000000", - "trace_end_time": "2025-10-22 16:57:23.822000000", - "trace_duration_ms": 4483.158, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:19.339000000", - "updated_at": "2025-10-22 16:57:19.339000000" - }, - { - "span_id": "24da2221-b0ab-4161-98ad-e7b5775fcb27", - "trace_id": "3bbd0b36-c30f-4126-b62a-dd0fae1c701d", - "parent_span_id": "b0f0a2e0-c1f7-4ba4-8b75-7090e931a738", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-22 16:57:19.339000000", - "end_time": "2025-10-22 16:57:19.339000000", - "duration_ms": 0.026, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: How does Noveum handle agent tracing?", - "query_length": 37, - "query_type": "routing_evaluation_query", - "classification.mode": "RAG", - "classification.rag_keyword_score": 2, - "classification.web_keyword_score": 0, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "noveum_specific", - "query.keyword_density": 0.3333333333333333, - "query.domain_affinity": "noveum", - "routing.decision": "Routed to RAG based on analysis", - "routing.rationale": "RAG score: 2, Web score: 0, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "Web Search", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.038461538461538464, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: RAG (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "3bbd0b36-c30f-4126-b62a-dd0fae1c701d", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:19.339000000", - "trace_end_time": "2025-10-22 16:57:23.822000000", - "trace_duration_ms": 4483.158, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:19.339000000", - "updated_at": "2025-10-22 16:57:19.339000000" - }, - { - "span_id": "d2b495eb-7e07-48a3-aa6d-1835aab18c11", - "trace_id": "3bbd0b36-c30f-4126-b62a-dd0fae1c701d", - "parent_span_id": "b0f0a2e0-c1f7-4ba4-8b75-7090e931a738", - "name": "agent.llm-rag", - "start_time": "2025-10-22 16:57:19.339000000", - "end_time": "2025-10-22 16:57:23.822000000", - "duration_ms": 4482.564, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "rag_agent", - "agent.operation": "llm-rag", - "agent.capabilities": [ - "document_retrieval", - "context_generation", - "response_generation" - ], - "agent.id": "noveum_rag_agent", - "input_query": "How does Noveum handle agent tracing?", - "query_length": 37, - "query_type": "rag_query", - "output_response": "RAG Answer: Noveum.ai handles agent tracing through its specialized Multi-Agent Tracing capabilities, which are designed to observe complex workflows and inter-agent communications within multi-agent systems. The...", - "rag.context_length": 2736, - "rag.sources_count": 5, - "rag.answer_length": 663, - "rag.mode": "retrieval_augmented_generation" - }, - "events": [], - "links": [], - "trace_trace_id": "3bbd0b36-c30f-4126-b62a-dd0fae1c701d", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:19.339000000", - "trace_end_time": "2025-10-22 16:57:23.822000000", - "trace_duration_ms": 4483.158, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:19.339000000", - "updated_at": "2025-10-22 16:57:19.339000000" - }, - { - "span_id": "4db80813-e62f-48ec-9091-62d569527caf", - "trace_id": "3bbd0b36-c30f-4126-b62a-dd0fae1c701d", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-22 16:57:19.339000000", - "end_time": "2025-10-22 16:57:23.822000000", - "duration_ms": 4483.031000000001, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "How does Noveum handle agent tracing?", - "query.length": 37, - "processing.mode": "RAG", - "processing.time_seconds": 4.482964038848877, - "processing.response_length": 663, - "processing.sources_count": 5, - "output_response": "Final Answer: Noveum.ai handles agent tracing through its specialized Multi-Agent Tracing capabilities, which are designed to observe complex workflows and inter-agent communications within multi-agent systems. The...", - "final_answer_mode": "RAG", - "query_processed.input_query": "How does Noveum handle agent tracing?", - "query_processed.output_response": "Successfully processed query using RAG, generated 663 character response", - "query_processed.mode": "RAG", - "query_processed.processing_time": 4.482964038848877, - "query_processed.response_length": 663 - }, - "events": [], - "links": [], - "trace_trace_id": "3bbd0b36-c30f-4126-b62a-dd0fae1c701d", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:19.339000000", - "trace_end_time": "2025-10-22 16:57:23.822000000", - "trace_duration_ms": 4483.158, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:19.339000000", - "updated_at": "2025-10-22 16:57:19.339000000" - }, - { - "span_id": "b80cb5cd-24f2-484d-bbff-9febe2aa3ffe", - "trace_id": "3bbd0b36-c30f-4126-b62a-dd0fae1c701d", - "parent_span_id": "d2b495eb-7e07-48a3-aa6d-1835aab18c11", - "name": "agent.llm_model_execution", - "start_time": "2025-10-22 16:57:19.958000000", - "end_time": "2025-10-22 16:57:22.709000000", - "duration_ms": 2750.9480000000003, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: How does Noveum handle agent tracing?", - "query_length": 37, - "query_type": "rag_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 2.750890016555786, - "model.latency_ms": 2750.890016555786, - "model.start_time": 1761152239.958973, - "model.end_time": 1761152242.709863, - "model.performance_tier": "medium", - "model.prompt_tokens": 817, - "model.completion_tokens": 165, - "model.total_tokens": 982, - "model.tokens_per_second": 356.975376728983, - "model.estimated_cost": 0.02946, - "model.efficiency_score": 0.675152749490835, - "model.response_length": 663, - "model.response_quality": "high", - "model.output_response": "Model Response: Noveum.ai handles agent tracing through its specialized Multi-Agent Tracing capabilities, which are designed to observe complex workflows and inter-agent communications within multi-agent systems. The...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "3bbd0b36-c30f-4126-b62a-dd0fae1c701d", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:19.339000000", - "trace_end_time": "2025-10-22 16:57:23.822000000", - "trace_duration_ms": 4483.158, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:19.339000000", - "updated_at": "2025-10-22 16:57:19.339000000" - }, - { - "span_id": "d85fcf7d-7eaa-4299-92fc-c969fab63a6e", - "trace_id": "3bbd0b36-c30f-4126-b62a-dd0fae1c701d", - "parent_span_id": "d2b495eb-7e07-48a3-aa6d-1835aab18c11", - "name": "agent.rag_evaluation_metrics", - "start_time": "2025-10-22 16:57:22.710000000", - "end_time": "2025-10-22 16:57:22.710000000", - "duration_ms": 0.036000000000000004, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "rag_evaluation_metrics", - "agent.capabilities": [ - "retrieval_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: How does Noveum handle agent tracing?", - "query_length": 37, - "query_type": "rag_evaluation_query", - "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, vector searches, data retrieval, agent reasoning, and more. Without proper observability, debugging ...", - "retrieval.context_length": 2736, - "retrieval.sources_count": 5, - "retrieval.context_quality": "high", - "retrieval.effectiveness": 1, - "retrieval.context_utilization": 2.736, - "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, vector searches, data retrieval, agent reasoning, and more. Without proper observability, debugging becomes a nightmare, optimization is guesswork, and understanding user interactions is nearly impossible.\nNoveum.ai solves this by providing comprehensive tracing and observability specifically designed for AI applications.\nKey benefits of using Noveum.ai:\n- Complete Visibility: Trace...\n\nSource 2 (https://noveum.ai/docs/advanced/multi-agent-tracing):\nMulti-Agent Tracing\nObserve complex agent workflows and inter-agent communications with comprehensive tracing\nMulti-agent systems represent some of the most complex AI applications, involving multiple agents that coordinate, communicate, and collaborate to achieve shared goals. Noveum.ai provides specialized tracing capabilities to help you understand and optimize these intricate workflows.\n\ud83c\udfaf Why Multi-Agent Tracing Matters\nMulti-agent systems introduce unique observability challenges:\n- Complex...\n\nSource 3 (https://noveum.ai/en/docs/advanced/multi-agent-tracing):\nMulti-Agent Tracing\nObserve complex agent workflows and inter-agent communications with comprehensive tracing\nMulti-agent systems represent some of the most complex AI applications, involving multiple agents that coordinate, communicate, and collaborate to achieve shared goals. Noveum.ai provides specialized tracing capabilities to help you understand and optimize these intricate workflows.\n\ud83c\udfaf Why Multi-Agent Tracing Matters\nMulti-agent systems introduce unique observability challenges:\n- Complex...\n\nSource 4 (https://noveum.ai/en/docs/advanced/performance-optimization):\n\ud83d\udd17 Next Steps\n- Multi-Agent Tracing - Observe agent workflows\n- RAG Pipeline Observability - Monitor retrieval and generation systems\n- Custom Instrumentation - Add domain-specific tracing\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nSource 5 (https://noveum.ai/docs/advanced/performance-optimization):\n\ud83d\udd17 Next Steps\n- Multi-Agent Tracing - Observe agent workflows\n- RAG Pipeline Observability - Monitor retrieval and generation systems\n- Custom Instrumentation - Add domain-specific tracing\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nUser Question: How does Noveum handle agent tracing?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", - "prompt.prompt_length": 3269, - "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, ...", - "prompt.instruction_following": "rag_optimized", - "response.answer_length": 663, - "response.answer_completeness": "complete", - "response.response_quality": "high", - "response.source_citation": 5, - "response.context_utilization": 0.24232456140350878, - "output_response": "RAG Answer: Noveum.ai handles agent tracing through its specialized Multi-Agent Tracing capabilities, which are designed to observe complex workflows and inter-agent communications within multi-agent systems. The...", - "evaluation.retrieval_effectiveness": 1, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 5, - "evaluation.context_relevance": "high", - "evaluation.overall_quality": "high", - "evaluation.ready_for_production": true, - "rag.retrieval_strategy": "semantic_similarity", - "rag.vector_search_results": 5, - "rag.context_synthesis": "multi_source", - "rag.document_coverage": 1, - "rag.information_density": 0.24232456140350878 - }, - "events": [], - "links": [], - "trace_trace_id": "3bbd0b36-c30f-4126-b62a-dd0fae1c701d", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:19.339000000", - "trace_end_time": "2025-10-22 16:57:23.822000000", - "trace_duration_ms": 4483.158, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:19.339000000", - "updated_at": "2025-10-22 16:57:19.339000000" - }, - { - "span_id": "93d82f68-6c62-40de-a675-c4d7c40c1d01", - "trace_id": "fee3e6d3-8da1-4ef3-8214-a7d7323d3a49", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-22 16:57:13.334000000", - "end_time": "2025-10-22 16:57:18.330000000", - "duration_ms": 4996.436, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What APIs are available in Noveum platform?", - "query.length": 43, - "processing.mode": "RAG", - "processing.time_seconds": 4.996389865875244, - "processing.response_length": 270, - "processing.sources_count": 5, - "output_response": "Final Answer: The provided context does not specify the APIs available in the Noveum platform. For detailed information about the APIs, I recommend checking the official Noveum documentation or the SDK Integration ...", - "final_answer_mode": "RAG", - "query_processed.input_query": "What APIs are available in Noveum platform?", - "query_processed.output_response": "Successfully processed query using RAG, generated 270 character response", - "query_processed.mode": "RAG", - "query_processed.processing_time": 4.996389865875244, - "query_processed.response_length": 270 - }, - "events": [], - "links": [], - "trace_trace_id": "fee3e6d3-8da1-4ef3-8214-a7d7323d3a49", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:13.334000000", - "trace_end_time": "2025-10-22 16:57:18.331000000", - "trace_duration_ms": 4996.502, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:13.334000000", - "updated_at": "2025-10-22 16:57:13.334000000" - }, - { - "span_id": "1be727c6-14e8-4b2e-b8df-a1cab902454f", - "trace_id": "fee3e6d3-8da1-4ef3-8214-a7d7323d3a49", - "parent_span_id": "93d82f68-6c62-40de-a675-c4d7c40c1d01", - "name": "agent.query_routing", - "start_time": "2025-10-22 16:57:13.334000000", - "end_time": "2025-10-22 16:57:18.330000000", - "duration_ms": 4996.354, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What APIs are available in Noveum platform?", - "query_length": 43, - "query_type": "routing_query", - "output_response": "Routed to RAG for query processing", - "router.classification": "RAG", - "router.confidence_score": 1, - "router.rag_keyword_score": 3, - "router.web_keyword_score": 0, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "fee3e6d3-8da1-4ef3-8214-a7d7323d3a49", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:13.334000000", - "trace_end_time": "2025-10-22 16:57:18.331000000", - "trace_duration_ms": 4996.502, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:13.334000000", - "updated_at": "2025-10-22 16:57:13.334000000" - }, - { - "span_id": "58ab1d39-d349-4164-8ff9-26ece4b1e1aa", - "trace_id": "fee3e6d3-8da1-4ef3-8214-a7d7323d3a49", - "parent_span_id": "1be727c6-14e8-4b2e-b8df-a1cab902454f", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-22 16:57:13.334000000", - "end_time": "2025-10-22 16:57:13.334000000", - "duration_ms": 0.041999999999999996, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What APIs are available in Noveum platform?", - "query_length": 43, - "query_type": "routing_evaluation_query", - "classification.mode": "RAG", - "classification.rag_keyword_score": 3, - "classification.web_keyword_score": 0, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "noveum_specific", - "query.keyword_density": 0.42857142857142855, - "query.domain_affinity": "noveum", - "routing.decision": "Routed to RAG based on analysis", - "routing.rationale": "RAG score: 3, Web score: 0, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "Web Search", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.057692307692307696, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: RAG (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "fee3e6d3-8da1-4ef3-8214-a7d7323d3a49", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:13.334000000", - "trace_end_time": "2025-10-22 16:57:18.331000000", - "trace_duration_ms": 4996.502, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:13.334000000", - "updated_at": "2025-10-22 16:57:13.334000000" - }, - { - "span_id": "f11ebf68-a416-484e-8e55-f77a11b9ea1c", - "trace_id": "fee3e6d3-8da1-4ef3-8214-a7d7323d3a49", - "parent_span_id": "1be727c6-14e8-4b2e-b8df-a1cab902454f", - "name": "agent.llm-rag", - "start_time": "2025-10-22 16:57:13.334000000", - "end_time": "2025-10-22 16:57:18.330000000", - "duration_ms": 4996.146000000001, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "rag_agent", - "agent.operation": "llm-rag", - "agent.capabilities": [ - "document_retrieval", - "context_generation", - "response_generation" - ], - "agent.id": "noveum_rag_agent", - "input_query": "What APIs are available in Noveum platform?", - "query_length": 43, - "query_type": "rag_query", - "output_response": "RAG Answer: The provided context does not specify the APIs available in the Noveum platform. For detailed information about the APIs, I recommend checking the official Noveum documentation or the SDK Integration ...", - "rag.context_length": 2284, - "rag.sources_count": 5, - "rag.answer_length": 270, - "rag.mode": "retrieval_augmented_generation" - }, - "events": [], - "links": [], - "trace_trace_id": "fee3e6d3-8da1-4ef3-8214-a7d7323d3a49", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:13.334000000", - "trace_end_time": "2025-10-22 16:57:18.331000000", - "trace_duration_ms": 4996.502, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:13.334000000", - "updated_at": "2025-10-22 16:57:13.334000000" - }, - { - "span_id": "11394488-8802-403d-a926-7fda3a510b61", - "trace_id": "fee3e6d3-8da1-4ef3-8214-a7d7323d3a49", - "parent_span_id": "f11ebf68-a416-484e-8e55-f77a11b9ea1c", - "name": "agent.llm_model_execution", - "start_time": "2025-10-22 16:57:13.862000000", - "end_time": "2025-10-22 16:57:16.049000000", - "duration_ms": 2187.29, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What APIs are available in Noveum platform?", - "query_length": 43, - "query_type": "rag_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 2.187239170074463, - "model.latency_ms": 2187.239170074463, - "model.start_time": 1761152233.862141, - "model.end_time": 1761152236.04938, - "model.performance_tier": "medium", - "model.prompt_tokens": 705, - "model.completion_tokens": 67, - "model.total_tokens": 772, - "model.tokens_per_second": 352.9563710098141, - "model.estimated_cost": 0.02316, - "model.efficiency_score": 0.34974093264248707, - "model.response_length": 270, - "model.response_quality": "high", - "model.output_response": "Model Response: The provided context does not specify the APIs available in the Noveum platform. For detailed information about the APIs, I recommend checking the official Noveum documentation or the SDK Integration ...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "fee3e6d3-8da1-4ef3-8214-a7d7323d3a49", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:13.334000000", - "trace_end_time": "2025-10-22 16:57:18.331000000", - "trace_duration_ms": 4996.502, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:13.334000000", - "updated_at": "2025-10-22 16:57:13.334000000" - }, - { - "span_id": "341cf8ab-cc33-4d57-952f-ba1065f0432e", - "trace_id": "fee3e6d3-8da1-4ef3-8214-a7d7323d3a49", - "parent_span_id": "f11ebf68-a416-484e-8e55-f77a11b9ea1c", - "name": "agent.rag_evaluation_metrics", - "start_time": "2025-10-22 16:57:16.049000000", - "end_time": "2025-10-22 16:57:16.049000000", - "duration_ms": 0.041, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "rag_evaluation_metrics", - "agent.capabilities": [ - "retrieval_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What APIs are available in Noveum platform?", - "query_length": 43, - "query_type": "rag_evaluation_query", - "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/docs):\n- \ud83c\udf0d Universal: Works across frameworks, clouds, and deployment models\n\ud83e\udd1d Community & Support\n- \ud83d\udcac Discord Community: Join our Discord\n- \ud83d\udce7 Email Support: [email protected]\n- \ud83d\udc1b Bug Reports: GitHub Issues\n- \ud83d\udcd6 Knowledge Base: Help Center\nReady to get started? Head to ...", - "retrieval.context_length": 2284, - "retrieval.sources_count": 5, - "retrieval.context_quality": "high", - "retrieval.effectiveness": 1, - "retrieval.context_utilization": 2.284, - "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/docs):\n- \ud83c\udf0d Universal: Works across frameworks, clouds, and deployment models\n\ud83e\udd1d Community & Support\n- \ud83d\udcac Discord Community: Join our Discord\n- \ud83d\udce7 Email Support: [email protected]\n- \ud83d\udc1b Bug Reports: GitHub Issues\n- \ud83d\udcd6 Knowledge Base: Help Center\nReady to get started? Head to our SDK Integration Guide to begin tracing your AI applications in under 5 minutes!\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability ...\n\nSource 2 (https://noveum.ai/docs):\n- \ud83c\udf0d Universal: Works across frameworks, clouds, and deployment models\n\ud83e\udd1d Community & Support\n- \ud83d\udcac Discord Community: Join our Discord\n- \ud83d\udce7 Email Support: [email protected]\n- \ud83d\udc1b Bug Reports: GitHub Issues\n- \ud83d\udcd6 Knowledge Base: Help Center\nReady to get started? Head to our SDK Integration Guide to begin tracing your AI applications in under 5 minutes!\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability ...\n\nSource 3 (https://noveum.ai/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nSource 4 (https://noveum.ai/en/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nSource 5 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, vector searches, data retrieval, agent reasoning, and more. Without proper observability, debugging becomes a nightmare, optimization is guesswork, and understanding user interactions is nearly impossible.\nNoveum.ai solves this by providing comprehensive tracing and observability specifically designed for AI applications.\nKey benefits of using Noveum.ai:\n- Complete Visibility: Trace...\n\nUser Question: What APIs are available in Noveum platform?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", - "prompt.prompt_length": 2823, - "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/docs):\n- \ud83c\udf0d Universal: Works across frameworks, clouds, and deployment models\n\ud83e\udd1d Community & Support\n- \ud83d\udcac Discord Community: Join our Discord\n- \ud83d\udce7 Email Support: [email prot...", - "prompt.instruction_following": "rag_optimized", - "response.answer_length": 270, - "response.answer_completeness": "complete", - "response.response_quality": "high", - "response.source_citation": 5, - "response.context_utilization": 0.11821366024518389, - "output_response": "RAG Answer: The provided context does not specify the APIs available in the Noveum platform. For detailed information about the APIs, I recommend checking the official Noveum documentation or the SDK Integration ...", - "evaluation.retrieval_effectiveness": 1, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 5, - "evaluation.context_relevance": "high", - "evaluation.overall_quality": "high", - "evaluation.ready_for_production": true, - "rag.retrieval_strategy": "semantic_similarity", - "rag.vector_search_results": 5, - "rag.context_synthesis": "multi_source", - "rag.document_coverage": 1, - "rag.information_density": 0.11821366024518389 - }, - "events": [], - "links": [], - "trace_trace_id": "fee3e6d3-8da1-4ef3-8214-a7d7323d3a49", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:13.334000000", - "trace_end_time": "2025-10-22 16:57:18.331000000", - "trace_duration_ms": 4996.502, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:13.334000000", - "updated_at": "2025-10-22 16:57:13.334000000" - }, - { - "span_id": "24930c28-7f93-426c-8914-bd0e0085589f", - "trace_id": "93bd37c4-4289-43c9-b34e-19eaaef610c2", - "parent_span_id": "bfc91518-f818-4f35-9113-70b04eb55edc", - "name": "agent.query_routing", - "start_time": "2025-10-22 16:57:06.605000000", - "end_time": "2025-10-22 16:57:12.327000000", - "duration_ms": 5721.561000000001, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "How do I set up observability with Noveum?", - "query_length": 42, - "query_type": "routing_query", - "output_response": "Routed to RAG for query processing", - "router.classification": "RAG", - "router.confidence_score": 1, - "router.rag_keyword_score": 2, - "router.web_keyword_score": 0, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "93bd37c4-4289-43c9-b34e-19eaaef610c2", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:06.605000000", - "trace_end_time": "2025-10-22 16:57:12.327000000", - "trace_duration_ms": 5721.707, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:06.605000000", - "updated_at": "2025-10-22 16:57:06.605000000" - }, - { - "span_id": "11fc18a8-0380-48ca-894e-cc26f648d290", - "trace_id": "93bd37c4-4289-43c9-b34e-19eaaef610c2", - "parent_span_id": "24930c28-7f93-426c-8914-bd0e0085589f", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-22 16:57:06.605000000", - "end_time": "2025-10-22 16:57:06.606000000", - "duration_ms": 0.047, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: How do I set up observability with Noveum?", - "query_length": 42, - "query_type": "routing_evaluation_query", - "classification.mode": "RAG", - "classification.rag_keyword_score": 2, - "classification.web_keyword_score": 0, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "noveum_specific", - "query.keyword_density": 0.25, - "query.domain_affinity": "noveum", - "routing.decision": "Routed to RAG based on analysis", - "routing.rationale": "RAG score: 2, Web score: 0, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "Web Search", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.038461538461538464, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: RAG (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "93bd37c4-4289-43c9-b34e-19eaaef610c2", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:06.605000000", - "trace_end_time": "2025-10-22 16:57:12.327000000", - "trace_duration_ms": 5721.707, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:06.605000000", - "updated_at": "2025-10-22 16:57:06.605000000" - }, - { - "span_id": "bfc91518-f818-4f35-9113-70b04eb55edc", - "trace_id": "93bd37c4-4289-43c9-b34e-19eaaef610c2", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-22 16:57:06.605000000", - "end_time": "2025-10-22 16:57:12.327000000", - "duration_ms": 5721.635, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "How do I set up observability with Noveum?", - "query.length": 42, - "processing.mode": "RAG", - "processing.time_seconds": 5.721601963043213, - "processing.response_length": 815, - "processing.sources_count": 5, - "output_response": "Final Answer: To set up observability with Noveum, follow these steps:\n\n1. **Sign Up**: Go to [noveum.ai](https://noveum.ai) and create an account.\n2. **Create a Project**: After signing up, create a project to obt...", - "final_answer_mode": "RAG", - "query_processed.input_query": "How do I set up observability with Noveum?", - "query_processed.output_response": "Successfully processed query using RAG, generated 815 character response", - "query_processed.mode": "RAG", - "query_processed.processing_time": 5.721601963043213, - "query_processed.response_length": 815 - }, - "events": [], - "links": [], - "trace_trace_id": "93bd37c4-4289-43c9-b34e-19eaaef610c2", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:06.605000000", - "trace_end_time": "2025-10-22 16:57:12.327000000", - "trace_duration_ms": 5721.707, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:06.605000000", - "updated_at": "2025-10-22 16:57:06.605000000" - }, - { - "span_id": "2a82e4b1-c93d-498f-9dd5-e971f228d642", - "trace_id": "93bd37c4-4289-43c9-b34e-19eaaef610c2", - "parent_span_id": "24930c28-7f93-426c-8914-bd0e0085589f", - "name": "agent.llm-rag", - "start_time": "2025-10-22 16:57:06.606000000", - "end_time": "2025-10-22 16:57:12.327000000", - "duration_ms": 5721.352000000001, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "rag_agent", - "agent.operation": "llm-rag", - "agent.capabilities": [ - "document_retrieval", - "context_generation", - "response_generation" - ], - "agent.id": "noveum_rag_agent", - "input_query": "How do I set up observability with Noveum?", - "query_length": 42, - "query_type": "rag_query", - "output_response": "RAG Answer: To set up observability with Noveum, follow these steps:\n\n1. **Sign Up**: Go to [noveum.ai](https://noveum.ai) and create an account.\n2. **Create a Project**: After signing up, create a project to obt...", - "rag.context_length": 2870, - "rag.sources_count": 6, - "rag.answer_length": 815, - "rag.mode": "retrieval_augmented_generation" - }, - "events": [], - "links": [], - "trace_trace_id": "93bd37c4-4289-43c9-b34e-19eaaef610c2", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:06.605000000", - "trace_end_time": "2025-10-22 16:57:12.327000000", - "trace_duration_ms": 5721.707, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:06.605000000", - "updated_at": "2025-10-22 16:57:06.605000000" - }, - { - "span_id": "520b4e59-da14-4db2-af02-0973cda9bd86", - "trace_id": "93bd37c4-4289-43c9-b34e-19eaaef610c2", - "parent_span_id": "2a82e4b1-c93d-498f-9dd5-e971f228d642", - "name": "agent.llm_model_execution", - "start_time": "2025-10-22 16:57:07.109000000", - "end_time": "2025-10-22 16:57:11.837000000", - "duration_ms": 4728.018999999999, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: How do I set up observability with Noveum?", - "query_length": 42, - "query_type": "rag_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 4.727963924407959, - "model.latency_ms": 4727.963924407959, - "model.start_time": 1761152227.109188, - "model.end_time": 1761152231.837152, - "model.performance_tier": "medium", - "model.prompt_tokens": 852, - "model.completion_tokens": 203, - "model.total_tokens": 1055, - "model.tokens_per_second": 223.1404504915101, - "model.estimated_cost": 0.03165, - "model.efficiency_score": 0.7725118483412322, - "model.response_length": 815, - "model.response_quality": "high", - "model.output_response": "Model Response: To set up observability with Noveum, follow these steps:\n\n1. **Sign Up**: Go to [noveum.ai](https://noveum.ai) and create an account.\n2. **Create a Project**: After signing up, create a project to obt...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "93bd37c4-4289-43c9-b34e-19eaaef610c2", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:06.605000000", - "trace_end_time": "2025-10-22 16:57:12.327000000", - "trace_duration_ms": 5721.707, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:06.605000000", - "updated_at": "2025-10-22 16:57:06.605000000" - }, - { - "span_id": "dff080ab-238c-4cca-a910-cf530f910bda", - "trace_id": "93bd37c4-4289-43c9-b34e-19eaaef610c2", - "parent_span_id": "2a82e4b1-c93d-498f-9dd5-e971f228d642", - "name": "agent.rag_evaluation_metrics", - "start_time": "2025-10-22 16:57:11.837000000", - "end_time": "2025-10-22 16:57:11.838000000", - "duration_ms": 0.259, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "rag_evaluation_metrics", - "agent.capabilities": [ - "retrieval_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: How do I set up observability with Noveum?", - "query_length": 42, - "query_type": "rag_evaluation_query", - "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\n- Production Ready: Built for scale with intelligent sampling and batching\n- Privacy Focused: You control what data is captured and how it's stored\n- Open Source SDKs: Transparent, auditable, and extensible\nGetting ...", - "retrieval.context_length": 2870, - "retrieval.sources_count": 6, - "retrieval.context_quality": "high", - "retrieval.effectiveness": 1.2, - "retrieval.context_utilization": 2.87, - "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\n- Production Ready: Built for scale with intelligent sampling and batching\n- Privacy Focused: You control what data is captured and how it's stored\n- Open Source SDKs: Transparent, auditable, and extensible\nGetting Started\nReady to add comprehensive observability to your AI applications?\n- Sign up at noveum.ai\n- Create a project and get your API key\n- Install the SDK for your preferred language\n- Add tracing to your AI workflows\n- Explore insights in the Noveum.ai dashboard\nCheck out our integra...\n\nSource 2 (https://noveum.ai/en/docs/getting-started/tracing-concepts):\n- Explore Framework Integrations - Framework-specific guidance\n- Learn Advanced Patterns - Custom instrumentation techniques\n- Master the Dashboard - Analyze your traces effectively\nRemember: Good observability is not about collecting all possible data, but about collecting the right data that helps you understand, debug, and optimize your AI applications.\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to O...\n\nSource 3 (https://noveum.ai/docs/getting-started/tracing-concepts):\n- Explore Framework Integrations - Framework-specific guidance\n- Learn Advanced Patterns - Custom instrumentation techniques\n- Master the Dashboard - Analyze your traces effectively\nRemember: Good observability is not about collecting all possible data, but about collecting the right data that helps you understand, debug, and optimize your AI applications.\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to O...\n\nSource 4 (https://noveum.ai/en/docs/getting-started/overview):\nNoveum.ai Overview\nComprehensive AI tracing and observability platform for LLM applications, RAG systems, and AI agents\nWelcome to Noveum.ai\u2014the comprehensive tracing and observability platform built specifically for AI applications. Whether you're building LLM-powered chatbots, RAG systems, multi-agent workflows, or any AI-driven application, Noveum provides the insights you need to understand, debug, and optimize your systems.\n\ud83c\udfaf Why AI Applications Need Specialized Observability\nTraditional mo...\n\nSource 5 (https://noveum.ai/docs/getting-started/overview):\nNoveum.ai Overview\nComprehensive AI tracing and observability platform for LLM applications, RAG systems, and AI agents\nWelcome to Noveum.ai\u2014the comprehensive tracing and observability platform built specifically for AI applications. Whether you're building LLM-powered chatbots, RAG systems, multi-agent workflows, or any AI-driven application, Noveum provides the insights you need to understand, debug, and optimize your systems.\n\ud83c\udfaf Why AI Applications Need Specialized Observability\nTraditional mo...\n\nUser Question: How do I set up observability with Noveum?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", - "prompt.prompt_length": 3408, - "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\n- Production Ready: Built for scale with intelligent sampling and batching\n- Privacy Focused: You control what data...", - "prompt.instruction_following": "rag_optimized", - "response.answer_length": 815, - "response.answer_completeness": "complete", - "response.response_quality": "high", - "response.source_citation": 6, - "response.context_utilization": 0.28397212543554007, - "output_response": "RAG Answer: To set up observability with Noveum, follow these steps:\n\n1. **Sign Up**: Go to [noveum.ai](https://noveum.ai) and create an account.\n2. **Create a Project**: After signing up, create a project to obt...", - "evaluation.retrieval_effectiveness": 1.2, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 6, - "evaluation.context_relevance": "high", - "evaluation.overall_quality": "high", - "evaluation.ready_for_production": true, - "rag.retrieval_strategy": "semantic_similarity", - "rag.vector_search_results": 6, - "rag.context_synthesis": "multi_source", - "rag.document_coverage": 1.2, - "rag.information_density": 0.28397212543554007 - }, - "events": [], - "links": [], - "trace_trace_id": "93bd37c4-4289-43c9-b34e-19eaaef610c2", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:57:06.605000000", - "trace_end_time": "2025-10-22 16:57:12.327000000", - "trace_duration_ms": 5721.707, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:57:06.605000000", - "updated_at": "2025-10-22 16:57:06.605000000" - }, - { - "span_id": "dad7df65-be49-4245-8721-c91da4962b9a", - "trace_id": "df0c6344-1a2e-4c53-bb7d-4e43b3bce1b8", - "parent_span_id": "9024b0d9-2b75-428f-af8f-75a921a7c69c", - "name": "agent.query_routing", - "start_time": "2025-10-22 16:56:59.213000000", - "end_time": "2025-10-22 16:57:05.598000000", - "duration_ms": 6384.249, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What features does Noveum Trace offer?", - "query_length": 38, - "query_type": "routing_query", - "output_response": "Routed to RAG for query processing", - "router.classification": "RAG", - "router.confidence_score": 1, - "router.rag_keyword_score": 3, - "router.web_keyword_score": 0, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "df0c6344-1a2e-4c53-bb7d-4e43b3bce1b8", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:56:59.213000000", - "trace_end_time": "2025-10-22 16:57:05.598000000", - "trace_duration_ms": 6384.386, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:56:59.213000000", - "updated_at": "2025-10-22 16:56:59.213000000" - }, - { - "span_id": "9024b0d9-2b75-428f-af8f-75a921a7c69c", - "trace_id": "df0c6344-1a2e-4c53-bb7d-4e43b3bce1b8", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-22 16:56:59.213000000", - "end_time": "2025-10-22 16:57:05.598000000", - "duration_ms": 6384.32, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What features does Noveum Trace offer?", - "query.length": 38, - "processing.mode": "RAG", - "processing.time_seconds": 6.384284973144531, - "processing.response_length": 1017, - "processing.sources_count": 5, - "output_response": "Final Answer: Noveum Trace offers several key features through its Python SDK (`noveum-trace`) and TypeScript SDK (`@noveum/trace`):\n\n1. **Decorator-based Tracing**: It provides a simple way to trace LLM calls, age...", - "final_answer_mode": "RAG", - "query_processed.input_query": "What features does Noveum Trace offer?", - "query_processed.output_response": "Successfully processed query using RAG, generated 1017 character response", - "query_processed.mode": "RAG", - "query_processed.processing_time": 6.384284973144531, - "query_processed.response_length": 1017 - }, - "events": [], - "links": [], - "trace_trace_id": "df0c6344-1a2e-4c53-bb7d-4e43b3bce1b8", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:56:59.213000000", - "trace_end_time": "2025-10-22 16:57:05.598000000", - "trace_duration_ms": 6384.386, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:56:59.213000000", - "updated_at": "2025-10-22 16:56:59.213000000" - }, - { - "span_id": "d07fdbdc-2b8f-4de9-ac32-988389701771", - "trace_id": "df0c6344-1a2e-4c53-bb7d-4e43b3bce1b8", - "parent_span_id": "dad7df65-be49-4245-8721-c91da4962b9a", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-22 16:56:59.214000000", - "end_time": "2025-10-22 16:56:59.214000000", - "duration_ms": 0.060000000000000005, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What features does Noveum Trace offer?", - "query_length": 38, - "query_type": "routing_evaluation_query", - "classification.mode": "RAG", - "classification.rag_keyword_score": 3, - "classification.web_keyword_score": 0, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "noveum_specific", - "query.keyword_density": 0.5, - "query.domain_affinity": "noveum", - "routing.decision": "Routed to RAG based on analysis", - "routing.rationale": "RAG score: 3, Web score: 0, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "Web Search", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.057692307692307696, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: RAG (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "df0c6344-1a2e-4c53-bb7d-4e43b3bce1b8", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:56:59.213000000", - "trace_end_time": "2025-10-22 16:57:05.598000000", - "trace_duration_ms": 6384.386, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:56:59.213000000", - "updated_at": "2025-10-22 16:56:59.213000000" - }, - { - "span_id": "83f86157-5af4-4c99-aad1-647866cd804e", - "trace_id": "df0c6344-1a2e-4c53-bb7d-4e43b3bce1b8", - "parent_span_id": "dad7df65-be49-4245-8721-c91da4962b9a", - "name": "agent.llm-rag", - "start_time": "2025-10-22 16:56:59.214000000", - "end_time": "2025-10-22 16:57:05.598000000", - "duration_ms": 6384.048, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "rag_agent", - "agent.operation": "llm-rag", - "agent.capabilities": [ - "document_retrieval", - "context_generation", - "response_generation" - ], - "agent.id": "noveum_rag_agent", - "input_query": "What features does Noveum Trace offer?", - "query_length": 38, - "query_type": "rag_query", - "output_response": "RAG Answer: Noveum Trace offers several key features through its Python SDK (`noveum-trace`) and TypeScript SDK (`@noveum/trace`):\n\n1. **Decorator-based Tracing**: It provides a simple way to trace LLM calls, age...", - "rag.context_length": 2787, - "rag.sources_count": 5, - "rag.answer_length": 1017, - "rag.mode": "retrieval_augmented_generation" - }, - "events": [], - "links": [], - "trace_trace_id": "df0c6344-1a2e-4c53-bb7d-4e43b3bce1b8", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:56:59.213000000", - "trace_end_time": "2025-10-22 16:57:05.598000000", - "trace_duration_ms": 6384.386, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:56:59.213000000", - "updated_at": "2025-10-22 16:56:59.213000000" - }, - { - "span_id": "3b4fa9a7-88bf-4ee1-aeb7-bfa9573f28f2", - "trace_id": "df0c6344-1a2e-4c53-bb7d-4e43b3bce1b8", - "parent_span_id": "83f86157-5af4-4c99-aad1-647866cd804e", - "name": "agent.llm_model_execution", - "start_time": "2025-10-22 16:56:59.591000000", - "end_time": "2025-10-22 16:57:04.869000000", - "duration_ms": 5278.925, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What features does Noveum Trace offer?", - "query_length": 38, - "query_type": "rag_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 5.278868913650513, - "model.latency_ms": 5278.868913650513, - "model.start_time": 1761152219.5910342, - "model.end_time": 1761152224.869903, - "model.performance_tier": "slow", - "model.prompt_tokens": 830, - "model.completion_tokens": 254, - "model.total_tokens": 1084, - "model.tokens_per_second": 205.3470199263535, - "model.estimated_cost": 0.03252, - "model.efficiency_score": 0.9381918819188192, - "model.response_length": 1017, - "model.response_quality": "high", - "model.output_response": "Model Response: Noveum Trace offers several key features through its Python SDK (`noveum-trace`) and TypeScript SDK (`@noveum/trace`):\n\n1. **Decorator-based Tracing**: It provides a simple way to trace LLM calls, age...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "df0c6344-1a2e-4c53-bb7d-4e43b3bce1b8", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:56:59.213000000", - "trace_end_time": "2025-10-22 16:57:05.598000000", - "trace_duration_ms": 6384.386, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:56:59.213000000", - "updated_at": "2025-10-22 16:56:59.213000000" - }, - { - "span_id": "f9214dbe-adb9-44ff-8361-70c872cef66d", - "trace_id": "df0c6344-1a2e-4c53-bb7d-4e43b3bce1b8", - "parent_span_id": "83f86157-5af4-4c99-aad1-647866cd804e", - "name": "agent.rag_evaluation_metrics", - "start_time": "2025-10-22 16:57:04.870000000", - "end_time": "2025-10-22 16:57:04.870000000", - "duration_ms": 0.043000000000000003, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "rag_evaluation_metrics", - "agent.capabilities": [ - "retrieval_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What features does Noveum Trace offer?", - "query_length": 38, - "query_type": "rag_evaluation_query", - "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\n\ud83d\udcd8 TypeScript SDK (\n@noveum/trace\n)- Framework ...", - "retrieval.context_length": 2787, - "retrieval.sources_count": 5, - "retrieval.context_quality": "high", - "retrieval.effectiveness": 1, - "retrieval.context_utilization": 2.787, - "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\n\ud83d\udcd8 TypeScript SDK (\n@noveum/trace\n)- Framework integrations for Next.js, Express.js, and Hono\n- TypeScript-first design with full type safety\n- Universal compatibility (Node.js, Edge Runtime, browsers)\n-\n\ud83d\udcca Noveum Platform\n- Real-time dashboard for analyzing traces and performance\n- Ad...\n\nSource 2 (https://noveum.ai/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\n\ud83d\udcd8 TypeScript SDK (\n@noveum/trace\n)- Framework integrations for Next.js, Express.js, and Hono\n- TypeScript-first design with full type safety\n- Universal compatibility (Node.js, Edge Runtime, browsers)\n-\n\ud83d\udcca Noveum Platform\n- Real-time dashboard for analyzing traces and performance\n- Ad...\n\nSource 3 (https://noveum.ai/en/docs/getting-started/sdk-integration):\n- \ud83d\udc65 Collaborate with your team on insights\n\ud83d\udd12 Security & Privacy\n- \ud83d\udd10 Encryption: All data encrypted in transit and at rest\n- \ud83c\udf9b\ufe0f Configurable Capture: Control what data is collected\n- \ud83c\udfe0 Data Residency: Choose your data storage region\n- \u23f0 Retention Control: Set custom data retention policies\nNext Steps\n- Tracing Concepts - Learn about traces, spans, and observability best practices\n- Framework Integrations - Deep dive into specific framework setups\n- Multi-Agent Tracing - Observe complex agent work...\n\nSource 4 (https://noveum.ai/docs/getting-started/sdk-integration):\n- \ud83d\udc65 Collaborate with your team on insights\n\ud83d\udd12 Security & Privacy\n- \ud83d\udd10 Encryption: All data encrypted in transit and at rest\n- \ud83c\udf9b\ufe0f Configurable Capture: Control what data is collected\n- \ud83c\udfe0 Data Residency: Choose your data storage region\n- \u23f0 Retention Control: Set custom data retention policies\nNext Steps\n- Tracing Concepts - Learn about traces, spans, and observability best practices\n- Framework Integrations - Deep dive into specific framework setups\n- Multi-Agent Tracing - Observe complex agent work...\n\nSource 5 (https://noveum.ai/docs/platform/dashboard):\n- Bulk Export: Download multiple traces simultaneously\n- Comparative Analysis: Compare performance across multiple traces\nReal-Time Updates\n- Live Refresh: Automatic updates as new traces arrive\n- Connection Monitoring: Real-time status of your trace ingestion pipeline\n- Performance Indicators: Live metrics for system health monitoring\nReady to dive deeper? Explore Projects & Environments to organize your AI applications, or check out Team Collaboration to share insights with your team.\nGet Earl...\n\nUser Question: What features does Noveum Trace offer?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", - "prompt.prompt_length": 3321, - "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frame...", - "prompt.instruction_following": "rag_optimized", - "response.answer_length": 1017, - "response.answer_completeness": "complete", - "response.response_quality": "high", - "response.source_citation": 5, - "response.context_utilization": 0.36490850376749195, - "output_response": "RAG Answer: Noveum Trace offers several key features through its Python SDK (`noveum-trace`) and TypeScript SDK (`@noveum/trace`):\n\n1. **Decorator-based Tracing**: It provides a simple way to trace LLM calls, age...", - "evaluation.retrieval_effectiveness": 1, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 5, - "evaluation.context_relevance": "high", - "evaluation.overall_quality": "high", - "evaluation.ready_for_production": true, - "rag.retrieval_strategy": "semantic_similarity", - "rag.vector_search_results": 5, - "rag.context_synthesis": "multi_source", - "rag.document_coverage": 1, - "rag.information_density": 0.36490850376749195 - }, - "events": [], - "links": [], - "trace_trace_id": "df0c6344-1a2e-4c53-bb7d-4e43b3bce1b8", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:56:59.213000000", - "trace_end_time": "2025-10-22 16:57:05.598000000", - "trace_duration_ms": 6384.386, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:56:59.213000000", - "updated_at": "2025-10-22 16:56:59.213000000" - }, - { - "span_id": "138a5089-5268-447a-afb5-4f14fb504281", - "trace_id": "29a862bf-d7aa-4cf9-963c-0a4244343bbe", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-22 16:56:54.719000000", - "end_time": "2025-10-22 16:56:58.207000000", - "duration_ms": 3487.959, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What are Noveum's pricing plans?", - "query.length": 32, - "processing.mode": "RAG", - "processing.time_seconds": 3.487928867340088, - "processing.response_length": 441, - "processing.sources_count": 5, - "output_response": "Final Answer: The provided context does not include specific information about Noveum's pricing plans. It mentions that all users will have access to the Observability suite for free, and early users will receive f...", - "final_answer_mode": "RAG", - "query_processed.input_query": "What are Noveum's pricing plans?", - "query_processed.output_response": "Successfully processed query using RAG, generated 441 character response", - "query_processed.mode": "RAG", - "query_processed.processing_time": 3.487928867340088, - "query_processed.response_length": 441 - }, - "events": [], - "links": [], - "trace_trace_id": "29a862bf-d7aa-4cf9-963c-0a4244343bbe", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:56:54.718000000", - "trace_end_time": "2025-10-22 16:56:58.207000000", - "trace_duration_ms": 3488.068, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:56:54.718000000", - "updated_at": "2025-10-22 16:56:54.718000000" - }, - { - "span_id": "7cda4dab-0fc5-4cae-8d8c-9445f661c67b", - "trace_id": "29a862bf-d7aa-4cf9-963c-0a4244343bbe", - "parent_span_id": "138a5089-5268-447a-afb5-4f14fb504281", - "name": "agent.query_routing", - "start_time": "2025-10-22 16:56:54.719000000", - "end_time": "2025-10-22 16:56:58.207000000", - "duration_ms": 3487.848, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What are Noveum's pricing plans?", - "query_length": 32, - "query_type": "routing_query", - "output_response": "Routed to RAG for query processing", - "router.classification": "RAG", - "router.confidence_score": 1, - "router.rag_keyword_score": 3, - "router.web_keyword_score": 0, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "29a862bf-d7aa-4cf9-963c-0a4244343bbe", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:56:54.718000000", - "trace_end_time": "2025-10-22 16:56:58.207000000", - "trace_duration_ms": 3488.068, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:56:54.718000000", - "updated_at": "2025-10-22 16:56:54.718000000" - }, - { - "span_id": "922c84a5-b14a-4b2f-8759-427d6e089812", - "trace_id": "29a862bf-d7aa-4cf9-963c-0a4244343bbe", - "parent_span_id": "7cda4dab-0fc5-4cae-8d8c-9445f661c67b", - "name": "agent.llm-rag", - "start_time": "2025-10-22 16:56:54.719000000", - "end_time": "2025-10-22 16:56:58.207000000", - "duration_ms": 3487.546, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "rag_agent", - "agent.operation": "llm-rag", - "agent.capabilities": [ - "document_retrieval", - "context_generation", - "response_generation" - ], - "agent.id": "noveum_rag_agent", - "input_query": "What are Noveum's pricing plans?", - "query_length": 32, - "query_type": "rag_query", - "output_response": "RAG Answer: The provided context does not include specific information about Noveum's pricing plans. It mentions that all users will have access to the Observability suite for free, and early users will receive f...", - "rag.context_length": 2105, - "rag.sources_count": 5, - "rag.answer_length": 441, - "rag.mode": "retrieval_augmented_generation" - }, - "events": [], - "links": [], - "trace_trace_id": "29a862bf-d7aa-4cf9-963c-0a4244343bbe", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:56:54.718000000", - "trace_end_time": "2025-10-22 16:56:58.207000000", - "trace_duration_ms": 3488.068, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:56:54.718000000", - "updated_at": "2025-10-22 16:56:54.718000000" - }, - { - "span_id": "c8d7bb28-e763-47c9-b3e9-77f6a91a22be", - "trace_id": "29a862bf-d7aa-4cf9-963c-0a4244343bbe", - "parent_span_id": "7cda4dab-0fc5-4cae-8d8c-9445f661c67b", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-22 16:56:54.719000000", - "end_time": "2025-10-22 16:56:54.719000000", - "duration_ms": 0.068, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What are Noveum's pricing plans?", - "query_length": 32, - "query_type": "routing_evaluation_query", - "classification.mode": "RAG", - "classification.rag_keyword_score": 3, - "classification.web_keyword_score": 0, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "noveum_specific", - "query.keyword_density": 0.6, - "query.domain_affinity": "noveum", - "routing.decision": "Routed to RAG based on analysis", - "routing.rationale": "RAG score: 3, Web score: 0, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "Web Search", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.057692307692307696, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: RAG (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "29a862bf-d7aa-4cf9-963c-0a4244343bbe", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:56:54.718000000", - "trace_end_time": "2025-10-22 16:56:58.207000000", - "trace_duration_ms": 3488.068, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:56:54.718000000", - "updated_at": "2025-10-22 16:56:54.718000000" - }, - { - "span_id": "a378123a-8a79-4bc5-8e20-8ec54c2f9783", - "trace_id": "29a862bf-d7aa-4cf9-963c-0a4244343bbe", - "parent_span_id": "922c84a5-b14a-4b2f-8759-427d6e089812", - "name": "agent.llm_model_execution", - "start_time": "2025-10-22 16:56:55.154000000", - "end_time": "2025-10-22 16:56:57.166000000", - "duration_ms": 2012.174, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What are Noveum's pricing plans?", - "query_length": 32, - "query_type": "rag_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 2.012089252471924, - "model.latency_ms": 2012.0892524719238, - "model.start_time": 1761152215.154625, - "model.end_time": 1761152217.1667142, - "model.performance_tier": "medium", - "model.prompt_tokens": 658, - "model.completion_tokens": 110, - "model.total_tokens": 768, - "model.tokens_per_second": 381.6928096288395, - "model.estimated_cost": 0.02304, - "model.efficiency_score": 0.57421875, - "model.response_length": 441, - "model.response_quality": "high", - "model.output_response": "Model Response: The provided context does not include specific information about Noveum's pricing plans. It mentions that all users will have access to the Observability suite for free, and early users will receive f...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "29a862bf-d7aa-4cf9-963c-0a4244343bbe", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:56:54.718000000", - "trace_end_time": "2025-10-22 16:56:58.207000000", - "trace_duration_ms": 3488.068, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:56:54.718000000", - "updated_at": "2025-10-22 16:56:54.718000000" - }, - { - "span_id": "5813673b-a1dc-46c7-9bea-92ce6aca2b5e", - "trace_id": "29a862bf-d7aa-4cf9-963c-0a4244343bbe", - "parent_span_id": "922c84a5-b14a-4b2f-8759-427d6e089812", - "name": "agent.rag_evaluation_metrics", - "start_time": "2025-10-22 16:56:57.166000000", - "end_time": "2025-10-22 16:56:57.167000000", - "duration_ms": 0.059, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "rag_evaluation_metrics", - "agent.capabilities": [ - "retrieval_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What are Noveum's pricing plans?", - "query_length": 32, - "query_type": "rag_evaluation_query", - "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....", - "retrieval.context_length": 2105, - "retrieval.sources_count": 5, - "retrieval.context_quality": "high", - "retrieval.effectiveness": 1, - "retrieval.context_utilization": 2.105, - "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nSource 2 (https://noveum.ai/en/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nSource 3 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nThe AI landscape is complex and moving fast. With Noveum.ai, you'll have the insights you need to build, debug, and optimize AI applications with confidence.\nReady to see what's really happening inside your AI applications? Try Noveum.ai today and transform how you build and monitor AI systems.\nLet's build more reliable, observable AI\u2014together.\nGet Early Access to Noveum.ai Platform\nJoin the select group of AI teams optimizing their models with our data-driven platform. We're onboarding users in...\n\nSource 4 (https://noveum.ai/en/blog/from-logs-to-intelligent-choices-inside-noveum-ais-evaluation-process):\nGet Early Access to Noveum.ai Platform\nJoin the select group of AI teams optimizing their models with our data-driven platform. We're onboarding users in limited batches to ensure a premium experience....\n\nSource 5 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, vector searches, data retrieval, agent reasoning, and more. Without proper observability, debugging becomes a nightmare, optimization is guesswork, and understanding user interactions is nearly impossible.\nNoveum.ai solves this by providing comprehensive tracing and observability specifically designed for AI applications.\nKey benefits of using Noveum.ai:\n- Complete Visibility: Trace...\n\nUser Question: What are Noveum's pricing plans?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", - "prompt.prompt_length": 2633, - "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to ...", - "prompt.instruction_following": "rag_optimized", - "response.answer_length": 441, - "response.answer_completeness": "complete", - "response.response_quality": "high", - "response.source_citation": 5, - "response.context_utilization": 0.20950118764845604, - "output_response": "RAG Answer: The provided context does not include specific information about Noveum's pricing plans. It mentions that all users will have access to the Observability suite for free, and early users will receive f...", - "evaluation.retrieval_effectiveness": 1, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 5, - "evaluation.context_relevance": "high", - "evaluation.overall_quality": "high", - "evaluation.ready_for_production": true, - "rag.retrieval_strategy": "semantic_similarity", - "rag.vector_search_results": 5, - "rag.context_synthesis": "multi_source", - "rag.document_coverage": 1, - "rag.information_density": 0.20950118764845604 - }, - "events": [], - "links": [], - "trace_trace_id": "29a862bf-d7aa-4cf9-963c-0a4244343bbe", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:56:54.718000000", - "trace_end_time": "2025-10-22 16:56:58.207000000", - "trace_duration_ms": 3488.068, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:56:54.718000000", - "updated_at": "2025-10-22 16:56:54.718000000" - }, - { - "span_id": "c5baf5c2-d18a-458a-b922-a19b96db53d9", - "trace_id": "bc60c122-f30b-4114-9f78-ee40a024b525", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-22 16:56:44.873000000", - "end_time": "2025-10-22 16:56:53.701000000", - "duration_ms": 8827.979, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "How do I integrate Noveum Trace in my application?", - "query.length": 50, - "processing.mode": "RAG", - "processing.time_seconds": 8.827935218811035, - "processing.response_length": 1145, - "processing.sources_count": 5, - "output_response": "Final Answer: To integrate Noveum Trace into your application, you can follow these steps based on the SDK Integration Guide:\n\n1. **Create Your Account & Get API Key**:\n - Sign up at [noveum.ai](https://noveum.ai...", - "final_answer_mode": "RAG", - "query_processed.input_query": "How do I integrate Noveum Trace in my application?", - "query_processed.output_response": "Successfully processed query using RAG, generated 1145 character response", - "query_processed.mode": "RAG", - "query_processed.processing_time": 8.827935218811035, - "query_processed.response_length": 1145 - }, - "events": [], - "links": [], - "trace_trace_id": "bc60c122-f30b-4114-9f78-ee40a024b525", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:56:44.873000000", - "trace_end_time": "2025-10-22 16:56:53.701000000", - "trace_duration_ms": 8828.055, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:56:44.873000000", - "updated_at": "2025-10-22 16:56:44.873000000" - }, - { - "span_id": "8819dde5-9cbe-4cdb-b919-9c5140582359", - "trace_id": "bc60c122-f30b-4114-9f78-ee40a024b525", - "parent_span_id": "c5baf5c2-d18a-458a-b922-a19b96db53d9", - "name": "agent.query_routing", - "start_time": "2025-10-22 16:56:44.873000000", - "end_time": "2025-10-22 16:56:53.701000000", - "duration_ms": 8827.883, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "How do I integrate Noveum Trace in my application?", - "query_length": 50, - "query_type": "routing_query", - "output_response": "Routed to RAG for query processing", - "router.classification": "RAG", - "router.confidence_score": 1, - "router.rag_keyword_score": 2, - "router.web_keyword_score": 0, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "bc60c122-f30b-4114-9f78-ee40a024b525", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:56:44.873000000", - "trace_end_time": "2025-10-22 16:56:53.701000000", - "trace_duration_ms": 8828.055, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:56:44.873000000", - "updated_at": "2025-10-22 16:56:44.873000000" - }, - { - "span_id": "932c4b0e-0f33-40c0-bb19-78b9f81e48a6", - "trace_id": "bc60c122-f30b-4114-9f78-ee40a024b525", - "parent_span_id": "8819dde5-9cbe-4cdb-b919-9c5140582359", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-22 16:56:44.873000000", - "end_time": "2025-10-22 16:56:44.873000000", - "duration_ms": 0.043000000000000003, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: How do I integrate Noveum Trace in my application?", - "query_length": 50, - "query_type": "routing_evaluation_query", - "classification.mode": "RAG", - "classification.rag_keyword_score": 2, - "classification.web_keyword_score": 0, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "noveum_specific", - "query.keyword_density": 0.2222222222222222, - "query.domain_affinity": "noveum", - "routing.decision": "Routed to RAG based on analysis", - "routing.rationale": "RAG score: 2, Web score: 0, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "Web Search", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.038461538461538464, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: RAG (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "bc60c122-f30b-4114-9f78-ee40a024b525", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:56:44.873000000", - "trace_end_time": "2025-10-22 16:56:53.701000000", - "trace_duration_ms": 8828.055, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:56:44.873000000", - "updated_at": "2025-10-22 16:56:44.873000000" - }, - { - "span_id": "2ece2bd6-e4c8-47bc-b3c8-26f82e9910f1", - "trace_id": "bc60c122-f30b-4114-9f78-ee40a024b525", - "parent_span_id": "8819dde5-9cbe-4cdb-b919-9c5140582359", - "name": "agent.llm-rag", - "start_time": "2025-10-22 16:56:44.873000000", - "end_time": "2025-10-22 16:56:53.701000000", - "duration_ms": 8827.666000000001, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "rag_agent", - "agent.operation": "llm-rag", - "agent.capabilities": [ - "document_retrieval", - "context_generation", - "response_generation" - ], - "agent.id": "noveum_rag_agent", - "input_query": "How do I integrate Noveum Trace in my application?", - "query_length": 50, - "query_type": "rag_query", - "output_response": "RAG Answer: To integrate Noveum Trace into your application, you can follow these steps based on the SDK Integration Guide:\n\n1. **Create Your Account & Get API Key**:\n - Sign up at [noveum.ai](https://noveum.ai...", - "rag.context_length": 2803, - "rag.sources_count": 5, - "rag.answer_length": 1145, - "rag.mode": "retrieval_augmented_generation" - }, - "events": [], - "links": [], - "trace_trace_id": "bc60c122-f30b-4114-9f78-ee40a024b525", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:56:44.873000000", - "trace_end_time": "2025-10-22 16:56:53.701000000", - "trace_duration_ms": 8828.055, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:56:44.873000000", - "updated_at": "2025-10-22 16:56:44.873000000" - }, - { - "span_id": "e6315110-36b1-4503-beaf-12d972e25a7d", - "trace_id": "bc60c122-f30b-4114-9f78-ee40a024b525", - "parent_span_id": "2ece2bd6-e4c8-47bc-b3c8-26f82e9910f1", - "name": "agent.llm_model_execution", - "start_time": "2025-10-22 16:56:45.576000000", - "end_time": "2025-10-22 16:56:53.143000000", - "duration_ms": 7566.311, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: How do I integrate Noveum Trace in my application?", - "query_length": 50, - "query_type": "rag_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 7.566190004348755, - "model.latency_ms": 7566.190004348755, - "model.start_time": 1761152205.576988, - "model.end_time": 1761152213.143178, - "model.performance_tier": "slow", - "model.prompt_tokens": 837, - "model.completion_tokens": 286, - "model.total_tokens": 1123, - "model.tokens_per_second": 148.4234468543009, - "model.estimated_cost": 0.03369, - "model.efficiency_score": 1.0195903829029385, - "model.response_length": 1145, - "model.response_quality": "high", - "model.output_response": "Model Response: To integrate Noveum Trace into your application, you can follow these steps based on the SDK Integration Guide:\n\n1. **Create Your Account & Get API Key**:\n - Sign up at [noveum.ai](https://noveum.ai...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "bc60c122-f30b-4114-9f78-ee40a024b525", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:56:44.873000000", - "trace_end_time": "2025-10-22 16:56:53.701000000", - "trace_duration_ms": 8828.055, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:56:44.873000000", - "updated_at": "2025-10-22 16:56:44.873000000" - }, - { - "span_id": "ae91d520-8442-48e5-aabf-fd58aed23694", - "trace_id": "bc60c122-f30b-4114-9f78-ee40a024b525", - "parent_span_id": "2ece2bd6-e4c8-47bc-b3c8-26f82e9910f1", - "name": "agent.rag_evaluation_metrics", - "start_time": "2025-10-22 16:56:53.143000000", - "end_time": "2025-10-22 16:56:53.143000000", - "duration_ms": 0.091, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "rag_evaluation_metrics", - "agent.capabilities": [ - "retrieval_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: How do I integrate Noveum Trace in my application?", - "query_length": 50, - "query_type": "rag_evaluation_query", - "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\n\ud83d\udcd8 TypeScript SDK (\n@noveum/trace\n)- Framework ...", - "retrieval.context_length": 2803, - "retrieval.sources_count": 5, - "retrieval.context_quality": "high", - "retrieval.effectiveness": 1, - "retrieval.context_utilization": 2.803, - "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\n\ud83d\udcd8 TypeScript SDK (\n@noveum/trace\n)- Framework integrations for Next.js, Express.js, and Hono\n- TypeScript-first design with full type safety\n- Universal compatibility (Node.js, Edge Runtime, browsers)\n-\n\ud83d\udcca Noveum Platform\n- Real-time dashboard for analyzing traces and performance\n- Ad...\n\nSource 2 (https://noveum.ai/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\n\ud83d\udcd8 TypeScript SDK (\n@noveum/trace\n)- Framework integrations for Next.js, Express.js, and Hono\n- TypeScript-first design with full type safety\n- Universal compatibility (Node.js, Edge Runtime, browsers)\n-\n\ud83d\udcca Noveum Platform\n- Real-time dashboard for analyzing traces and performance\n- Ad...\n\nSource 3 (https://noveum.ai/docs/getting-started/sdk-integration):\nSDK Integration Guide\nIntegrate Noveum.ai tracing into your AI applications with Python or TypeScript SDKs\nThe Noveum.ai SDKs provide comprehensive tracing and observability for your AI applications with minimal code changes. Whether you're building LLM applications, RAG systems, or multi-agent workflows, our SDKs automatically capture essential metrics and traces.\n\ud83d\ude80 Quick Start\n1. Create Your Account & Get API Key\n- Sign up at noveum.ai\n- Create a project in your dashboard\n- Generate an API key...\n\nSource 4 (https://noveum.ai/en/docs/getting-started/sdk-integration):\nSDK Integration Guide\nIntegrate Noveum.ai tracing into your AI applications with Python or TypeScript SDKs\nThe Noveum.ai SDKs provide comprehensive tracing and observability for your AI applications with minimal code changes. Whether you're building LLM applications, RAG systems, or multi-agent workflows, our SDKs automatically capture essential metrics and traces.\n\ud83d\ude80 Quick Start\n1. Create Your Account & Get API Key\n- Sign up at noveum.ai\n- Create a project in your dashboard\n- Generate an API key...\n\nSource 5 (https://noveum.ai/en/docs/getting-started/sdk-integration):\n\ud83d\udd27 Framework Integrations\nNext.js Integration\nExpress.js Integration\nFastAPI Integration (Python)\n\ud83d\udcca Advanced Features\nCustom Attributes & Events\nSampling Configuration\n\ud83d\udd17 What's Captured Automatically\n- \ud83d\udcca Performance Metrics: Latency, throughput, error rates\n- \ud83d\udcb0 Cost Tracking: Token usage, API costs across providers\n- \ud83d\udd0d Request/Response: Configurable capture of inputs/outputs\n- \ud83c\udff7\ufe0f Metadata: Model names, parameters, user context\n- \ud83c\udf0a Context Flow: Trace relationships across services\n- \ud83d\udc1b Error Detail...\n\nUser Question: How do I integrate Noveum Trace in my application?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", - "prompt.prompt_length": 3349, - "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frame...", - "prompt.instruction_following": "rag_optimized", - "response.answer_length": 1145, - "response.answer_completeness": "complete", - "response.response_quality": "high", - "response.source_citation": 5, - "response.context_utilization": 0.4084909026043525, - "output_response": "RAG Answer: To integrate Noveum Trace into your application, you can follow these steps based on the SDK Integration Guide:\n\n1. **Create Your Account & Get API Key**:\n - Sign up at [noveum.ai](https://noveum.ai...", - "evaluation.retrieval_effectiveness": 1, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 5, - "evaluation.context_relevance": "high", - "evaluation.overall_quality": "high", - "evaluation.ready_for_production": true, - "rag.retrieval_strategy": "semantic_similarity", - "rag.vector_search_results": 5, - "rag.context_synthesis": "multi_source", - "rag.document_coverage": 1, - "rag.information_density": 0.4084909026043525 - }, - "events": [], - "links": [], - "trace_trace_id": "bc60c122-f30b-4114-9f78-ee40a024b525", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:56:44.873000000", - "trace_end_time": "2025-10-22 16:56:53.701000000", - "trace_duration_ms": 8828.055, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:56:44.873000000", - "updated_at": "2025-10-22 16:56:44.873000000" - }, - { - "span_id": "9a9c611d-f552-4b06-a764-4463edb70e2c", - "trace_id": "3ad192e5-bbd4-4236-8144-019d53512769", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-22 16:56:35.188000000", - "end_time": "2025-10-22 16:56:43.863000000", - "duration_ms": 8675.25, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What is Noveum and what does it do?", - "query.length": 35, - "processing.mode": "RAG", - "processing.time_seconds": 8.67501425743103, - "processing.response_length": 527, - "processing.sources_count": 5, - "output_response": "Final Answer: Noveum.ai is a comprehensive tracing and observability platform specifically designed for AI applications, including LLM-powered chatbots, RAG systems, and multi-agent workflows. It provides the neces...", - "final_answer_mode": "RAG", - "query_processed.input_query": "What is Noveum and what does it do?", - "query_processed.output_response": "Successfully processed query using RAG, generated 527 character response", - "query_processed.mode": "RAG", - "query_processed.processing_time": 8.67501425743103, - "query_processed.response_length": 527 - }, - "events": [], - "links": [], - "trace_trace_id": "3ad192e5-bbd4-4236-8144-019d53512769", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:56:35.188000000", - "trace_end_time": "2025-10-22 16:56:43.863000000", - "trace_duration_ms": 8675.356000000002, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:56:35.188000000", - "updated_at": "2025-10-22 16:56:35.188000000" - }, - { - "span_id": "99eadeae-0733-4bf3-982e-e9f3014f5913", - "trace_id": "3ad192e5-bbd4-4236-8144-019d53512769", - "parent_span_id": "7e2c959c-157a-43d5-9870-32994cddd388", - "name": "agent.llm-rag", - "start_time": "2025-10-22 16:56:35.188000000", - "end_time": "2025-10-22 16:56:43.863000000", - "duration_ms": 8674.699999999999, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "rag_agent", - "agent.operation": "llm-rag", - "agent.capabilities": [ - "document_retrieval", - "context_generation", - "response_generation" - ], - "agent.id": "noveum_rag_agent", - "input_query": "What is Noveum and what does it do?", - "query_length": 35, - "query_type": "rag_query", - "output_response": "RAG Answer: Noveum.ai is a comprehensive tracing and observability platform specifically designed for AI applications, including LLM-powered chatbots, RAG systems, and multi-agent workflows. It provides the neces...", - "rag.context_length": 2334, - "rag.sources_count": 5, - "rag.answer_length": 527, - "rag.mode": "retrieval_augmented_generation" - }, - "events": [], - "links": [], - "trace_trace_id": "3ad192e5-bbd4-4236-8144-019d53512769", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:56:35.188000000", - "trace_end_time": "2025-10-22 16:56:43.863000000", - "trace_duration_ms": 8675.356000000002, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:56:35.188000000", - "updated_at": "2025-10-22 16:56:35.188000000" - }, - { - "span_id": "7e2c959c-157a-43d5-9870-32994cddd388", - "trace_id": "3ad192e5-bbd4-4236-8144-019d53512769", - "parent_span_id": "9a9c611d-f552-4b06-a764-4463edb70e2c", - "name": "agent.query_routing", - "start_time": "2025-10-22 16:56:35.188000000", - "end_time": "2025-10-22 16:56:43.863000000", - "duration_ms": 8674.961, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What is Noveum and what does it do?", - "query_length": 35, - "query_type": "routing_query", - "output_response": "Routed to RAG for query processing", - "router.classification": "RAG", - "router.confidence_score": 1, - "router.rag_keyword_score": 1, - "router.web_keyword_score": 0, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "3ad192e5-bbd4-4236-8144-019d53512769", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:56:35.188000000", - "trace_end_time": "2025-10-22 16:56:43.863000000", - "trace_duration_ms": 8675.356000000002, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:56:35.188000000", - "updated_at": "2025-10-22 16:56:35.188000000" - }, - { - "span_id": "6f81284c-a9aa-4d62-a402-1b474f2eba51", - "trace_id": "3ad192e5-bbd4-4236-8144-019d53512769", - "parent_span_id": "7e2c959c-157a-43d5-9870-32994cddd388", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-22 16:56:35.188000000", - "end_time": "2025-10-22 16:56:35.188000000", - "duration_ms": 0.054, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What is Noveum and what does it do?", - "query_length": 35, - "query_type": "routing_evaluation_query", - "classification.mode": "RAG", - "classification.rag_keyword_score": 1, - "classification.web_keyword_score": 0, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "noveum_specific", - "query.keyword_density": 0.125, - "query.domain_affinity": "noveum", - "routing.decision": "Routed to RAG based on analysis", - "routing.rationale": "RAG score: 1, Web score: 0, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "Web Search", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.019230769230769232, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: RAG (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "3ad192e5-bbd4-4236-8144-019d53512769", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:56:35.188000000", - "trace_end_time": "2025-10-22 16:56:43.863000000", - "trace_duration_ms": 8675.356000000002, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:56:35.188000000", - "updated_at": "2025-10-22 16:56:35.188000000" - }, - { - "span_id": "9f42d18a-0235-4ae2-838c-19a88ddc95e1", - "trace_id": "3ad192e5-bbd4-4236-8144-019d53512769", - "parent_span_id": "99eadeae-0733-4bf3-982e-e9f3014f5913", - "name": "agent.llm_model_execution", - "start_time": "2025-10-22 16:56:36.031000000", - "end_time": "2025-10-22 16:56:42.012000000", - "duration_ms": 5980.947999999999, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What is Noveum and what does it do?", - "query_length": 35, - "query_type": "rag_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 5.980915784835815, - "model.latency_ms": 5980.915784835815, - "model.start_time": 1761152196.031816, - "model.end_time": 1761152202.0127318, - "model.performance_tier": "slow", - "model.prompt_tokens": 716, - "model.completion_tokens": 131, - "model.total_tokens": 847, - "model.tokens_per_second": 141.61710856178715, - "model.estimated_cost": 0.025410000000000002, - "model.efficiency_score": 0.6221959858323495, - "model.response_length": 527, - "model.response_quality": "high", - "model.output_response": "Model Response: Noveum.ai is a comprehensive tracing and observability platform specifically designed for AI applications, including LLM-powered chatbots, RAG systems, and multi-agent workflows. It provides the neces...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "3ad192e5-bbd4-4236-8144-019d53512769", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:56:35.188000000", - "trace_end_time": "2025-10-22 16:56:43.863000000", - "trace_duration_ms": 8675.356000000002, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:56:35.188000000", - "updated_at": "2025-10-22 16:56:35.188000000" - }, - { - "span_id": "7bcbe4b7-585a-4c9d-add8-16e8aa62254f", - "trace_id": "3ad192e5-bbd4-4236-8144-019d53512769", - "parent_span_id": "99eadeae-0733-4bf3-982e-e9f3014f5913", - "name": "agent.rag_evaluation_metrics", - "start_time": "2025-10-22 16:56:42.012000000", - "end_time": "2025-10-22 16:56:42.012000000", - "duration_ms": 0.018000000000000002, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "rag_evaluation_metrics", - "agent.capabilities": [ - "retrieval_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What is Noveum and what does it do?", - "query_length": 35, - "query_type": "rag_evaluation_query", - "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, vector searches, data retrieval, agent reasoning, and more. Without proper observability, debugging ...", - "retrieval.context_length": 2334, - "retrieval.sources_count": 5, - "retrieval.context_quality": "high", - "retrieval.effectiveness": 1, - "retrieval.context_utilization": 2.334, - "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, vector searches, data retrieval, agent reasoning, and more. Without proper observability, debugging becomes a nightmare, optimization is guesswork, and understanding user interactions is nearly impossible.\nNoveum.ai solves this by providing comprehensive tracing and observability specifically designed for AI applications.\nKey benefits of using Noveum.ai:\n- Complete Visibility: Trace...\n\nSource 2 (https://noveum.ai/en/docs/getting-started/overview):\nNoveum.ai Overview\nComprehensive AI tracing and observability platform for LLM applications, RAG systems, and AI agents\nWelcome to Noveum.ai\u2014the comprehensive tracing and observability platform built specifically for AI applications. Whether you're building LLM-powered chatbots, RAG systems, multi-agent workflows, or any AI-driven application, Noveum provides the insights you need to understand, debug, and optimize your systems.\n\ud83c\udfaf Why AI Applications Need Specialized Observability\nTraditional mo...\n\nSource 3 (https://noveum.ai/docs/getting-started/overview):\nNoveum.ai Overview\nComprehensive AI tracing and observability platform for LLM applications, RAG systems, and AI agents\nWelcome to Noveum.ai\u2014the comprehensive tracing and observability platform built specifically for AI applications. Whether you're building LLM-powered chatbots, RAG systems, multi-agent workflows, or any AI-driven application, Noveum provides the insights you need to understand, debug, and optimize your systems.\n\ud83c\udfaf Why AI Applications Need Specialized Observability\nTraditional mo...\n\nSource 4 (https://noveum.ai/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nSource 5 (https://noveum.ai/en/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nUser Question: What is Noveum and what does it do?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", - "prompt.prompt_length": 2865, - "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, ...", - "prompt.instruction_following": "rag_optimized", - "response.answer_length": 527, - "response.answer_completeness": "complete", - "response.response_quality": "high", - "response.source_citation": 5, - "response.context_utilization": 0.22579263067694944, - "output_response": "RAG Answer: Noveum.ai is a comprehensive tracing and observability platform specifically designed for AI applications, including LLM-powered chatbots, RAG systems, and multi-agent workflows. It provides the neces...", - "evaluation.retrieval_effectiveness": 1, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 5, - "evaluation.context_relevance": "high", - "evaluation.overall_quality": "high", - "evaluation.ready_for_production": true, - "rag.retrieval_strategy": "semantic_similarity", - "rag.vector_search_results": 5, - "rag.context_synthesis": "multi_source", - "rag.document_coverage": 1, - "rag.information_density": 0.22579263067694944 - }, - "events": [], - "links": [], - "trace_trace_id": "3ad192e5-bbd4-4236-8144-019d53512769", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:56:35.188000000", - "trace_end_time": "2025-10-22 16:56:43.863000000", - "trace_duration_ms": 8675.356000000002, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:56:35.188000000", - "updated_at": "2025-10-22 16:56:35.188000000" - }, - { - "span_id": "d39297f4-b4fe-4f9c-b45b-854411ba770a", - "trace_id": "9af2b4b0-0c2f-4d2e-ae99-30df0d2f0fa6", - "parent_span_id": "", - "name": "system_initialization", - "start_time": "2025-10-22 16:56:34.565000000", - "end_time": "2025-10-22 16:56:35.186000000", - "duration_ms": 621.02, - "status": "ok", - "status_message": "", - "attributes": { - "system.force_scrape": false, - "system.config": { - "noveum_base_url": "https://noveum.ai", - "max_pages_to_scrape": 50, - "chunk_size": 1000, - "chunk_overlap": 200, - "max_search_results": 5, - "rag_threshold": 0.7, - "noveum_docs_file": "noveum_docs.json", - "vector_store_path": "noveum_vectorstore" - }, - "input_query": "Initialize system with force_scrape=False", - "output_response": "System initialization: RAG system loaded, vector store ready, agent operational", - "system.initialized": true, - "system.documents_loaded": 38, - "system.vectorstore_ready": true - }, - "events": [], - "links": [], - "trace_trace_id": "9af2b4b0-0c2f-4d2e-ae99-30df0d2f0fa6", - "trace_name": "auto_trace_system_initialization", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-22 16:56:34.565000000", - "trace_end_time": "2025-10-22 16:56:35.186000000", - "trace_duration_ms": 621.177, - "span_count": 1, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-22 16:56:34.565000000", - "updated_at": "2025-10-22 16:56:34.565000000" - }, - { - "span_id": "8c144ad8-a439-4986-b741-3bc4b7fc6110", - "trace_id": "bae13042-cf2e-4813-a673-7dc2a611725e", - "parent_span_id": "d37fe1a1-a126-4e4e-a914-93dd402fed3f", - "name": "agent.query_routing", - "start_time": "2025-10-16 08:50:56.153000000", - "end_time": "2025-10-16 08:51:01.908000000", - "duration_ms": 5754.43, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What are the latest breakthroughs in artificial intelligence?", - "query_length": 61, - "query_type": "routing_query", - "output_response": "Routed to Web Search for query processing", - "router.classification": "Web Search", - "router.confidence_score": 1, - "router.rag_keyword_score": 0, - "router.web_keyword_score": 1, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "bae13042-cf2e-4813-a673-7dc2a611725e", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:56.153000000", - "trace_end_time": "2025-10-16 08:51:01.908000000", - "trace_duration_ms": 5754.578, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:56.153000000", - "updated_at": "2025-10-16 08:50:56.153000000" - }, - { - "span_id": "b294f9b7-7762-47a9-b49d-00801a984af2", - "trace_id": "bae13042-cf2e-4813-a673-7dc2a611725e", - "parent_span_id": "8c144ad8-a439-4986-b741-3bc4b7fc6110", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-16 08:50:56.153000000", - "end_time": "2025-10-16 08:50:56.153000000", - "duration_ms": 0.024, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What are the latest breakthroughs in artificial intelligence?", - "query_length": 61, - "query_type": "routing_evaluation_query", - "classification.mode": "Web Search", - "classification.rag_keyword_score": 0, - "classification.web_keyword_score": 1, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "complex", - "query.intent": "general_knowledge", - "query.keyword_density": 0.125, - "query.domain_affinity": "general", - "routing.decision": "Routed to Web Search based on analysis", - "routing.rationale": "RAG score: 0, Web score: 1, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "RAG", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.019230769230769232, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: Web Search (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "bae13042-cf2e-4813-a673-7dc2a611725e", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:56.153000000", - "trace_end_time": "2025-10-16 08:51:01.908000000", - "trace_duration_ms": 5754.578, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:56.153000000", - "updated_at": "2025-10-16 08:50:56.153000000" - }, - { - "span_id": "ab6a281a-4989-40c8-9011-d0ec7dde7ea9", - "trace_id": "bae13042-cf2e-4813-a673-7dc2a611725e", - "parent_span_id": "8c144ad8-a439-4986-b741-3bc4b7fc6110", - "name": "agent.web_search_generation", - "start_time": "2025-10-16 08:50:56.153000000", - "end_time": "2025-10-16 08:51:01.908000000", - "duration_ms": 5754.317999999999, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "web_search_agent", - "agent.operation": "web_search_generation", - "agent.capabilities": [ - "web_search", - "content_synthesis", - "response_generation" - ], - "agent.id": "noveum_web_search_agent", - "input_query": "What are the latest breakthroughs in artificial intelligence?", - "query_length": 61, - "query_type": "web_search_query", - "output_response": "Web Search Answer: The web search did not yield specific results regarding the latest breakthroughs in artificial intelligence. However, I can provide a general overview based on knowledge up to October 2023.\n\nRecent br...", - "web_search.results_count": 1, - "web_search.context_length": 163, - "web_search.response_length": 1489, - "web_search.mode": "external_web_search" - }, - "events": [], - "links": [], - "trace_trace_id": "bae13042-cf2e-4813-a673-7dc2a611725e", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:56.153000000", - "trace_end_time": "2025-10-16 08:51:01.908000000", - "trace_duration_ms": 5754.578, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:56.153000000", - "updated_at": "2025-10-16 08:50:56.153000000" - }, - { - "span_id": "d37fe1a1-a126-4e4e-a914-93dd402fed3f", - "trace_id": "bae13042-cf2e-4813-a673-7dc2a611725e", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-16 08:50:56.153000000", - "end_time": "2025-10-16 08:51:01.908000000", - "duration_ms": 5754.527, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What are the latest breakthroughs in artificial intelligence?", - "query.length": 61, - "processing.mode": "Web Search", - "processing.time_seconds": 5.754457473754883, - "processing.response_length": 1489, - "processing.sources_count": 1, - "output_response": "Final Answer: The web search did not yield specific results regarding the latest breakthroughs in artificial intelligence. However, I can provide a general overview based on knowledge up to October 2023.\n\nRecent br...", - "final_answer_mode": "Web Search", - "query_processed.input_query": "What are the latest breakthroughs in artificial intelligence?", - "query_processed.output_response": "Successfully processed query using Web Search, generated 1489 character response", - "query_processed.mode": "Web Search", - "query_processed.processing_time": 5.754457473754883, - "query_processed.response_length": 1489 - }, - "events": [], - "links": [], - "trace_trace_id": "bae13042-cf2e-4813-a673-7dc2a611725e", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:56.153000000", - "trace_end_time": "2025-10-16 08:51:01.908000000", - "trace_duration_ms": 5754.578, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:56.153000000", - "updated_at": "2025-10-16 08:50:56.153000000" - }, - { - "span_id": "f61ef76d-4887-42da-9fd6-e2cdb25a9e09", - "trace_id": "bae13042-cf2e-4813-a673-7dc2a611725e", - "parent_span_id": "ab6a281a-4989-40c8-9011-d0ec7dde7ea9", - "name": "agent.llm_model_execution", - "start_time": "2025-10-16 08:50:56.502000000", - "end_time": "2025-10-16 08:51:01.908000000", - "duration_ms": 5405.44, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What are the latest breakthroughs in artificial intelligence?", - "query_length": 61, - "query_type": "web_search_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 5.405392408370972, - "model.latency_ms": 5405.392408370972, - "model.start_time": 1760604656.5026655, - "model.end_time": 1760604661.908058, - "model.performance_tier": "slow", - "model.prompt_tokens": 167, - "model.completion_tokens": 372, - "model.total_tokens": 539, - "model.tokens_per_second": 99.71523976044487, - "model.estimated_cost": 0.01617, - "model.efficiency_score": 2.7625231910946195, - "model.response_length": 1489, - "model.response_quality": "high", - "model.output_response": "Model Response: The web search did not yield specific results regarding the latest breakthroughs in artificial intelligence. However, I can provide a general overview based on knowledge up to October 2023.\n\nRecent br...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "bae13042-cf2e-4813-a673-7dc2a611725e", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:56.153000000", - "trace_end_time": "2025-10-16 08:51:01.908000000", - "trace_duration_ms": 5754.578, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:56.153000000", - "updated_at": "2025-10-16 08:50:56.153000000" - }, - { - "span_id": "550bf5ae-c816-4ef0-a6c1-1553b1c258b8", - "trace_id": "bae13042-cf2e-4813-a673-7dc2a611725e", - "parent_span_id": "ab6a281a-4989-40c8-9011-d0ec7dde7ea9", - "name": "agent.web_search_evaluation_metrics", - "start_time": "2025-10-16 08:51:01.908000000", - "end_time": "2025-10-16 08:51:01.908000000", - "duration_ms": 0.018000000000000002, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "web_search_evaluation_metrics", - "agent.capabilities": [ - "web_search_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What are the latest breakthroughs in artificial intelligence?", - "query_length": 61, - "query_type": "web_search_evaluation_query", - "web_search.results_count": 1, - "web_search.context_length": 163, - "web_search.context_synthesized": "Context from 1 web sources", - "web_search.search_effectiveness": 0.2, - "web_search.context_quality": "low", - "web_search.source_diversity": 1, - "web_search.information_synthesis": "low", - "web_search.external_knowledge_utilization": 0.163, - "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What+are+the+latest+breakthroughs+in+artificial+intelligence?\n\nUser Question: What are the latest breakthroughs in artificial intelligence?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", - "prompt.prompt_length": 669, - "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What+are+the+latest+breakthroughs+in+artificial+intelligence?", - "prompt.instruction_following": "web_search_optimized", - "response.answer_length": 1489, - "response.answer_completeness": "complete", - "response.response_quality": "medium", - "response.source_citation": 1, - "response.context_utilization": 9.134969325153374, - "output_response": "Web Search Answer: The web search did not yield specific results regarding the latest breakthroughs in artificial intelligence. However, I can provide a general overview based on knowledge up to October 2023.\n\nRecent br...", - "evaluation.search_effectiveness": 0.2, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 1, - "evaluation.context_relevance": "low", - "evaluation.overall_quality": "low", - "evaluation.ready_for_production": true, - "web_search.search_strategy": "duckduckgo_api", - "web_search.real_time_data": true, - "web_search.external_sources": 1, - "web_search.information_freshness": "current", - "web_search.knowledge_synthesis": "single_source" - }, - "events": [], - "links": [], - "trace_trace_id": "bae13042-cf2e-4813-a673-7dc2a611725e", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:56.153000000", - "trace_end_time": "2025-10-16 08:51:01.908000000", - "trace_duration_ms": 5754.578, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:56.153000000", - "updated_at": "2025-10-16 08:50:56.153000000" - }, - { - "span_id": "04cb7d29-9ea8-4ae0-9834-030cae944eca", - "trace_id": "79f9d235-d46e-4740-be35-ef0006071f6c", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-16 08:50:51.539000000", - "end_time": "2025-10-16 08:50:55.150000000", - "duration_ms": 3611.008, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What's happening in the software development world today?", - "query.length": 57, - "processing.mode": "Web Search", - "processing.time_seconds": 3.6109254360198975, - "processing.response_length": 526, - "processing.sources_count": 1, - "output_response": "Final Answer: The provided web search results do not contain specific information about current events or trends in the software development world. They primarily focus on unrelated topics, such as the meaning of \"...", - "final_answer_mode": "Web Search", - "query_processed.input_query": "What's happening in the software development world today?", - "query_processed.output_response": "Successfully processed query using Web Search, generated 526 character response", - "query_processed.mode": "Web Search", - "query_processed.processing_time": 3.6109254360198975, - "query_processed.response_length": 526 - }, - "events": [], - "links": [], - "trace_trace_id": "79f9d235-d46e-4740-be35-ef0006071f6c", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:51.539000000", - "trace_end_time": "2025-10-16 08:50:55.150000000", - "trace_duration_ms": 3611.084, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:51.539000000", - "updated_at": "2025-10-16 08:50:51.539000000" - }, - { - "span_id": "e5e3c927-783e-48c5-aec7-b32bae98c0a2", - "trace_id": "79f9d235-d46e-4740-be35-ef0006071f6c", - "parent_span_id": "f630c3e2-091e-4bd9-9556-e0ebc1c50291", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-16 08:50:51.539000000", - "end_time": "2025-10-16 08:50:51.540000000", - "duration_ms": 0.041, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What's happening in the software development world today?", - "query_length": 57, - "query_type": "routing_evaluation_query", - "classification.mode": "Web Search", - "classification.rag_keyword_score": 0, - "classification.web_keyword_score": 1, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "complex", - "query.intent": "general_knowledge", - "query.keyword_density": 0.125, - "query.domain_affinity": "general", - "routing.decision": "Routed to Web Search based on analysis", - "routing.rationale": "RAG score: 0, Web score: 1, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "RAG", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.019230769230769232, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: Web Search (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "79f9d235-d46e-4740-be35-ef0006071f6c", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:51.539000000", - "trace_end_time": "2025-10-16 08:50:55.150000000", - "trace_duration_ms": 3611.084, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:51.539000000", - "updated_at": "2025-10-16 08:50:51.539000000" - }, - { - "span_id": "f630c3e2-091e-4bd9-9556-e0ebc1c50291", - "trace_id": "79f9d235-d46e-4740-be35-ef0006071f6c", - "parent_span_id": "04cb7d29-9ea8-4ae0-9834-030cae944eca", - "name": "agent.query_routing", - "start_time": "2025-10-16 08:50:51.539000000", - "end_time": "2025-10-16 08:50:55.150000000", - "duration_ms": 3610.876, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What's happening in the software development world today?", - "query_length": 57, - "query_type": "routing_query", - "output_response": "Routed to Web Search for query processing", - "router.classification": "Web Search", - "router.confidence_score": 1, - "router.rag_keyword_score": 0, - "router.web_keyword_score": 1, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "79f9d235-d46e-4740-be35-ef0006071f6c", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:51.539000000", - "trace_end_time": "2025-10-16 08:50:55.150000000", - "trace_duration_ms": 3611.084, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:51.539000000", - "updated_at": "2025-10-16 08:50:51.539000000" - }, - { - "span_id": "d7b0f399-3b43-43bf-bac1-b2101268841c", - "trace_id": "79f9d235-d46e-4740-be35-ef0006071f6c", - "parent_span_id": "f630c3e2-091e-4bd9-9556-e0ebc1c50291", - "name": "agent.web_search_generation", - "start_time": "2025-10-16 08:50:51.540000000", - "end_time": "2025-10-16 08:50:55.150000000", - "duration_ms": 3610.679, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "web_search_agent", - "agent.operation": "web_search_generation", - "agent.capabilities": [ - "web_search", - "content_synthesis", - "response_generation" - ], - "agent.id": "noveum_web_search_agent", - "input_query": "What's happening in the software development world today?", - "query_length": 57, - "query_type": "web_search_query", - "output_response": "Web Search Answer: The provided web search results do not contain specific information about current events or trends in the software development world. They primarily focus on unrelated topics, such as the meaning of \"...", - "web_search.results_count": 1, - "web_search.context_length": 589, - "web_search.response_length": 526, - "web_search.mode": "external_web_search" - }, - "events": [], - "links": [], - "trace_trace_id": "79f9d235-d46e-4740-be35-ef0006071f6c", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:51.539000000", - "trace_end_time": "2025-10-16 08:50:55.150000000", - "trace_duration_ms": 3611.084, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:51.539000000", - "updated_at": "2025-10-16 08:50:51.539000000" - }, - { - "span_id": "4ed5dae4-f850-4644-aa55-2f1c00d11c22", - "trace_id": "79f9d235-d46e-4740-be35-ef0006071f6c", - "parent_span_id": "d7b0f399-3b43-43bf-bac1-b2101268841c", - "name": "agent.llm_model_execution", - "start_time": "2025-10-16 08:50:51.976000000", - "end_time": "2025-10-16 08:50:55.150000000", - "duration_ms": 3174.51, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What's happening in the software development world today?", - "query_length": 57, - "query_type": "web_search_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 3.174463987350464, - "model.latency_ms": 3174.463987350464, - "model.start_time": 1760604651.9761698, - "model.end_time": 1760604655.1506338, - "model.performance_tier": "medium", - "model.prompt_tokens": 272, - "model.completion_tokens": 131, - "model.total_tokens": 403, - "model.tokens_per_second": 126.95056601866197, - "model.estimated_cost": 0.01209, - "model.efficiency_score": 1.305210918114144, - "model.response_length": 526, - "model.response_quality": "high", - "model.output_response": "Model Response: The provided web search results do not contain specific information about current events or trends in the software development world. They primarily focus on unrelated topics, such as the meaning of \"...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "79f9d235-d46e-4740-be35-ef0006071f6c", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:51.539000000", - "trace_end_time": "2025-10-16 08:50:55.150000000", - "trace_duration_ms": 3611.084, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:51.539000000", - "updated_at": "2025-10-16 08:50:51.539000000" - }, - { - "span_id": "eb151834-9322-4f63-add0-dec9a2966367", - "trace_id": "79f9d235-d46e-4740-be35-ef0006071f6c", - "parent_span_id": "d7b0f399-3b43-43bf-bac1-b2101268841c", - "name": "agent.web_search_evaluation_metrics", - "start_time": "2025-10-16 08:50:55.150000000", - "end_time": "2025-10-16 08:50:55.150000000", - "duration_ms": 0.02, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "web_search_evaluation_metrics", - "agent.capabilities": [ - "web_search_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What's happening in the software development world today?", - "query_length": 57, - "query_type": "web_search_evaluation_query", - "web_search.results_count": 1, - "web_search.context_length": 589, - "web_search.context_synthesized": "Context from 1 web sources", - "web_search.search_effectiveness": 0.2, - "web_search.context_quality": "medium", - "web_search.source_diversity": 1, - "web_search.information_synthesis": "low", - "web_search.external_knowledge_utilization": 0.589, - "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\n\u66fc\u5df4\u7684\u610f\u601d\u662f\u4e00\u79cd\u6bd2\u86c7\uff0c\u8c61\u5f81\u7740\u79d1\u6bd4\u5728\u7403\u573a\u4e0a\u7684\u653b\u51fb\u6027\u548c\u575a\u97e7\u4e0d\u62d4 \u201cWhat can i say\uff0c Mamba out\uff01\u201d \u4f46\u662f\u7ecf\u8fc7\u4e92\u8054\u7f51\u7684\u89e3\u6784\uff0c\u8fd9\u53e5\u8bdd\u6210\u4e3a\u4e86\u4e00\u53e5\u4e07\u7528\u573a\u666f\u7684\u8bdd \u2026 Aprende ingl\u00e9s gratis con nuestros cursos. \u00a1Mas de 100 lecciones con ejemplos y audio! Distintas y divertidas formas de estudiar ingl\u00e9s: \u2026 \u77e5\u4e4e\uff0c\u4e2d\u6587\u4e92\u8054\u7f51\u9ad8\u8d28\u91cf\u7684\u95ee\u7b54\u793e\u533a\u548c\u521b\u4f5c\u8005\u805a\u96c6\u7684\u539f\u521b\u5185\u5bb9\u5e73\u53f0\uff0c\u4e8e 2011 \u5e74 1 \u6708\u6b63\u5f0f\u4e0a\u7ebf\uff0c\u4ee5\u300c\u8ba9\u4eba\u4eec\u66f4\u597d\u7684\u5206\u4eab\u77e5\u8bc6\u3001\u7ecf\u9a8c\u548c\u89c1\u89e3\uff0c\u627e\u5230\u81ea\u5df1\u7684\u89e3\u7b54\u300d\u4e3a\u54c1 \u2026 \u8f6f\u4ef6\u7279\u70b9 1\u3001Windows Desktop Runtime\u662f\u5fae\u8f6fWindows\u684c\u9762\u7a0b\u5e8f\u8fd0\u884c\u5e93\uff08\u542b\u5e38\u89c4\u8fd0\u884c\u5e93\uff09 2\u3001\u80fd\u5b8c\u7f8e\u517c\u5bb9\u5fae\u8f6f\u4e0d\u540c\u7248\u672c\u7684Windows\u7cfb\u7edf\uff0c\u89e3\u51b3\u5176\u7a0b\u5e8f\u7f3a \u2026 \u77e5\u4e4e\uff0c\u4e2d\u6587\u4e92\u8054\u7f51\u9ad8\u8d28\u91cf\u7684\u95ee\u7b54\u793e\u533a\u548c\u521b\u4f5c\u8005\u805a\u96c6\u7684\u539f\u521b\u5185\u5bb9\u5e73\u53f0\uff0c\u4e8e 2011 \u5e74 1 \u6708\u6b63\u5f0f\u4e0a\u7ebf\uff0c\u4ee5\u300c\u8ba9\u4eba\u4eec\u66f4\u597d\u7684\u5206\u4eab\u77e5\u8bc6\u3001\u7ecf\u9a8c\u548c\u89c1\u89e3\uff0c\u627e\u5230\u81ea\u5df1\u7684\u89e3\u7b54\u300d\u4e3a\u54c1 \u2026\nURL: https://duckduckgo.com/?q=What's+happening+in+the+software+development+world+today?\n\nUser Question: What's happening in the software development world today?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", - "prompt.prompt_length": 1091, - "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\n\u66fc\u5df4\u7684\u610f\u601d\u662f\u4e00\u79cd\u6bd2\u86c7\uff0c\u8c61\u5f81\u7740\u79d1\u6bd4\u5728\u7403\u573a\u4e0a\u7684\u653b\u51fb\u6027\u548c\u575a\u97e7\u4e0d\u62d4 \u201cWhat can i say\uff0c Mamba out\uff01\u201d \u4f46\u662f\u7ecf\u8fc7\u4e92\u8054\u7f51\u7684\u89e3\u6784\uff0c\u8fd9\u53e5\u8bdd\u6210\u4e3a\u4e86\u4e00\u53e5\u4e07\u7528\u573a\u666f\u7684\u8bdd \u2026 Aprende ingl\u00e9s gratis con nuestros cursos. \u00a1Mas de 100 lecciones con ejemplos y audio...", - "prompt.instruction_following": "web_search_optimized", - "response.answer_length": 526, - "response.answer_completeness": "complete", - "response.response_quality": "medium", - "response.source_citation": 1, - "response.context_utilization": 0.8930390492359932, - "output_response": "Web Search Answer: The provided web search results do not contain specific information about current events or trends in the software development world. They primarily focus on unrelated topics, such as the meaning of \"...", - "evaluation.search_effectiveness": 0.2, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 1, - "evaluation.context_relevance": "medium", - "evaluation.overall_quality": "low", - "evaluation.ready_for_production": true, - "web_search.search_strategy": "duckduckgo_api", - "web_search.real_time_data": true, - "web_search.external_sources": 1, - "web_search.information_freshness": "current", - "web_search.knowledge_synthesis": "single_source" - }, - "events": [], - "links": [], - "trace_trace_id": "79f9d235-d46e-4740-be35-ef0006071f6c", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:51.539000000", - "trace_end_time": "2025-10-16 08:50:55.150000000", - "trace_duration_ms": 3611.084, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:51.539000000", - "updated_at": "2025-10-16 08:50:51.539000000" - }, - { - "span_id": "888b4441-3678-42a5-a752-8daae2433482", - "trace_id": "77b81bd5-5a04-4fc0-9fc6-571f7907d6b1", - "parent_span_id": "ba6ffeeb-78b5-44dd-b460-7933ca6e7739", - "name": "agent.query_routing", - "start_time": "2025-10-16 08:50:44.888000000", - "end_time": "2025-10-16 08:50:50.529000000", - "duration_ms": 5641.062, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What are the newest features in cloud computing?", - "query_length": 48, - "query_type": "routing_query", - "output_response": "Routed to RAG for query processing", - "router.classification": "RAG", - "router.confidence_score": 1, - "router.rag_keyword_score": 1, - "router.web_keyword_score": 0, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "77b81bd5-5a04-4fc0-9fc6-571f7907d6b1", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:44.888000000", - "trace_end_time": "2025-10-16 08:50:50.529000000", - "trace_duration_ms": 5641.226, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:44.888000000", - "updated_at": "2025-10-16 08:50:44.888000000" - }, - { - "span_id": "b6bae768-93a6-47d9-8369-61897f9d7828", - "trace_id": "77b81bd5-5a04-4fc0-9fc6-571f7907d6b1", - "parent_span_id": "888b4441-3678-42a5-a752-8daae2433482", - "name": "agent.llm-rag", - "start_time": "2025-10-16 08:50:44.888000000", - "end_time": "2025-10-16 08:50:50.529000000", - "duration_ms": 5640.875999999999, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "rag_agent", - "agent.operation": "llm-rag", - "agent.capabilities": [ - "document_retrieval", - "context_generation", - "response_generation" - ], - "agent.id": "noveum_rag_agent", - "input_query": "What are the newest features in cloud computing?", - "query_length": 48, - "query_type": "rag_query", - "output_response": "RAG Answer: The provided context does not specifically mention any new features in cloud computing as a general topic. However, it does highlight several recent enhancements and features related to the Noveum.ai ...", - "rag.context_length": 2579, - "rag.sources_count": 5, - "rag.answer_length": 983, - "rag.mode": "retrieval_augmented_generation" - }, - "events": [], - "links": [], - "trace_trace_id": "77b81bd5-5a04-4fc0-9fc6-571f7907d6b1", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:44.888000000", - "trace_end_time": "2025-10-16 08:50:50.529000000", - "trace_duration_ms": 5641.226, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:44.888000000", - "updated_at": "2025-10-16 08:50:44.888000000" - }, - { - "span_id": "ba6ffeeb-78b5-44dd-b460-7933ca6e7739", - "trace_id": "77b81bd5-5a04-4fc0-9fc6-571f7907d6b1", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-16 08:50:44.888000000", - "end_time": "2025-10-16 08:50:50.529000000", - "duration_ms": 5641.156, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What are the newest features in cloud computing?", - "query.length": 48, - "processing.mode": "RAG", - "processing.time_seconds": 5.64110541343689, - "processing.response_length": 983, - "processing.sources_count": 5, - "output_response": "Final Answer: The provided context does not specifically mention any new features in cloud computing as a general topic. However, it does highlight several recent enhancements and features related to the Noveum.ai ...", - "final_answer_mode": "RAG", - "query_processed.input_query": "What are the newest features in cloud computing?", - "query_processed.output_response": "Successfully processed query using RAG, generated 983 character response", - "query_processed.mode": "RAG", - "query_processed.processing_time": 5.64110541343689, - "query_processed.response_length": 983 - }, - "events": [], - "links": [], - "trace_trace_id": "77b81bd5-5a04-4fc0-9fc6-571f7907d6b1", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:44.888000000", - "trace_end_time": "2025-10-16 08:50:50.529000000", - "trace_duration_ms": 5641.226, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:44.888000000", - "updated_at": "2025-10-16 08:50:44.888000000" - }, - { - "span_id": "f5538bff-8908-4a56-a622-4ba2eeedfd29", - "trace_id": "77b81bd5-5a04-4fc0-9fc6-571f7907d6b1", - "parent_span_id": "888b4441-3678-42a5-a752-8daae2433482", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-16 08:50:44.888000000", - "end_time": "2025-10-16 08:50:44.888000000", - "duration_ms": 0.032, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What are the newest features in cloud computing?", - "query_length": 48, - "query_type": "routing_evaluation_query", - "classification.mode": "RAG", - "classification.rag_keyword_score": 1, - "classification.web_keyword_score": 0, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "documentation", - "query.keyword_density": 0.125, - "query.domain_affinity": "noveum", - "routing.decision": "Routed to RAG based on analysis", - "routing.rationale": "RAG score: 1, Web score: 0, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "Web Search", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.019230769230769232, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: RAG (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "77b81bd5-5a04-4fc0-9fc6-571f7907d6b1", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:44.888000000", - "trace_end_time": "2025-10-16 08:50:50.529000000", - "trace_duration_ms": 5641.226, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:44.888000000", - "updated_at": "2025-10-16 08:50:44.888000000" - }, - { - "span_id": "856e51cf-fa4d-459b-a046-1ea7316b903e", - "trace_id": "77b81bd5-5a04-4fc0-9fc6-571f7907d6b1", - "parent_span_id": "b6bae768-93a6-47d9-8369-61897f9d7828", - "name": "agent.llm_model_execution", - "start_time": "2025-10-16 08:50:45.423000000", - "end_time": "2025-10-16 08:50:49.997000000", - "duration_ms": 4573.782999999999, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What are the newest features in cloud computing?", - "query_length": 48, - "query_type": "rag_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 4.573714733123779, - "model.latency_ms": 4573.714733123779, - "model.start_time": 1760604645.423261, - "model.end_time": 1760604649.9969757, - "model.performance_tier": "medium", - "model.prompt_tokens": 780, - "model.completion_tokens": 245, - "model.total_tokens": 1025, - "model.tokens_per_second": 224.1066747291299, - "model.estimated_cost": 0.03075, - "model.efficiency_score": 0.9590243902439024, - "model.response_length": 983, - "model.response_quality": "high", - "model.output_response": "Model Response: The provided context does not specifically mention any new features in cloud computing as a general topic. However, it does highlight several recent enhancements and features related to the Noveum.ai ...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "77b81bd5-5a04-4fc0-9fc6-571f7907d6b1", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:44.888000000", - "trace_end_time": "2025-10-16 08:50:50.529000000", - "trace_duration_ms": 5641.226, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:44.888000000", - "updated_at": "2025-10-16 08:50:44.888000000" - }, - { - "span_id": "a2fa7400-74c4-4718-a562-899b91503c95", - "trace_id": "77b81bd5-5a04-4fc0-9fc6-571f7907d6b1", - "parent_span_id": "b6bae768-93a6-47d9-8369-61897f9d7828", - "name": "agent.rag_evaluation_metrics", - "start_time": "2025-10-16 08:50:49.997000000", - "end_time": "2025-10-16 08:50:49.997000000", - "duration_ms": 0.048, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "rag_evaluation_metrics", - "agent.capabilities": [ - "retrieval_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What are the newest features in cloud computing?", - "query_length": 48, - "query_type": "rag_evaluation_query", - "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/changelog):\n- \ud83d\udcc8 Enhanced Dashboard Analytics: Real-time request tracking, latency monitoring, and cost analysis\n- \ud83d\udd0d Improved Logs Interface: Better search functionality, detailed trace views, and enhanced debugging capabilities\n- \ud83d\udc65 Team Management: Advanced member mana...", - "retrieval.context_length": 2579, - "retrieval.sources_count": 5, - "retrieval.context_quality": "high", - "retrieval.effectiveness": 1, - "retrieval.context_utilization": 2.579, - "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/changelog):\n- \ud83d\udcc8 Enhanced Dashboard Analytics: Real-time request tracking, latency monitoring, and cost analysis\n- \ud83d\udd0d Improved Logs Interface: Better search functionality, detailed trace views, and enhanced debugging capabilities\n- \ud83d\udc65 Team Management: Advanced member management with role-based access control and invitation system\n- \ud83d\udee1\ufe0f Security Improvements: Enhanced API key management and secure credential storage\n4 months ago\n- \ud83d\udcca Advanced Metrics Collection: Comprehensive telemetry system with custom metrics ...\n\nSource 2 (https://noveum.ai/en/changelog):\n- \ud83d\udcdd Documentation Updates: Comprehensive guides for integration and best practices\n5 months ago\n- \ud83c\udf89 Platform Foundation: Initial release of Noveum.ai AI observability platform\n- \ud83d\ude80 Core Features: AI Gateway integration, basic evaluation metrics, and monitoring capabilities\n- \ud83d\udcc8 Evaluation Jobs: Automated model evaluation with configurable metrics and reporting\n- \ud83d\udd0c Provider Integrations: Support for major AI providers and custom model deployments...\n\nSource 3 (https://noveum.ai/docs/platform/dashboard):\n- Bulk Export: Download multiple traces simultaneously\n- Comparative Analysis: Compare performance across multiple traces\nReal-Time Updates\n- Live Refresh: Automatic updates as new traces arrive\n- Connection Monitoring: Real-time status of your trace ingestion pipeline\n- Performance Indicators: Live metrics for system health monitoring\nReady to dive deeper? Explore Projects & Environments to organize your AI applications, or check out Team Collaboration to share insights with your team.\nGet Earl...\n\nSource 4 (https://noveum.ai/en/changelog):\nChangelog\nStay up to date with the latest changes in our product.\n2 months ago\n- \ud83c\udfaf Major Platform Evolution: NovaEval Framework - Launched comprehensive AI model evaluation framework with 20+ built-in scorers including accuracy, RAG metrics, conversational scoring, and specialized agent evaluation tools. Features production-grade deployment on Docker and Kubernetes. (GitHub | PyPI Package)\n- \ud83d\ude80 Next-Generation Traces UI - Complete redesign featuring three-pane layout with directory tree navigatio...\n\nSource 5 (https://noveum.ai/en/blog):\n8/12/2025\nDiscover how Noveum.ai provides comprehensive tracing and observability for AI applications, from development debugging to production optimization.\nShashank Agarwal\n3/3/2025\nDiscover how Noveum.ai provides comprehensive tracing and observability for LLM applications, RAG systems, and multi-agent workflows with our powerful Python and TypeScript SDKs.\nShashank Agarwal\n3/2/2025...\n\nUser Question: What are the newest features in cloud computing?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", - "prompt.prompt_length": 3123, - "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/changelog):\n- \ud83d\udcc8 Enhanced Dashboard Analytics: Real-time request tracking, latency monitoring, and cost analysis\n- \ud83d\udd0d Improved Logs Interface: Better search functionality,...", - "prompt.instruction_following": "rag_optimized", - "response.answer_length": 983, - "response.answer_completeness": "complete", - "response.response_quality": "high", - "response.source_citation": 5, - "response.context_utilization": 0.381155486622722, - "output_response": "RAG Answer: The provided context does not specifically mention any new features in cloud computing as a general topic. However, it does highlight several recent enhancements and features related to the Noveum.ai ...", - "evaluation.retrieval_effectiveness": 1, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 5, - "evaluation.context_relevance": "high", - "evaluation.overall_quality": "high", - "evaluation.ready_for_production": true, - "rag.retrieval_strategy": "semantic_similarity", - "rag.vector_search_results": 5, - "rag.context_synthesis": "multi_source", - "rag.document_coverage": 1, - "rag.information_density": 0.381155486622722 - }, - "events": [], - "links": [], - "trace_trace_id": "77b81bd5-5a04-4fc0-9fc6-571f7907d6b1", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:44.888000000", - "trace_end_time": "2025-10-16 08:50:50.529000000", - "trace_duration_ms": 5641.226, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:44.888000000", - "updated_at": "2025-10-16 08:50:44.888000000" - }, - { - "span_id": "4c391bb1-1a62-4e2a-913b-96f83feff494", - "trace_id": "32170d1a-d211-46d8-9cdc-5a4265f0c443", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-16 08:50:42.124000000", - "end_time": "2025-10-16 08:50:43.885000000", - "duration_ms": 1760.974, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What's the current status of cryptocurrency markets?", - "query.length": 52, - "processing.mode": "Web Search", - "processing.time_seconds": 1.7608659267425537, - "processing.response_length": 363, - "processing.sources_count": 1, - "output_response": "Final Answer: The web search did not yield any specific information regarding the current status of cryptocurrency markets. Therefore, I cannot provide an update on market trends, prices, or significant events affe...", - "final_answer_mode": "Web Search", - "query_processed.input_query": "What's the current status of cryptocurrency markets?", - "query_processed.output_response": "Successfully processed query using Web Search, generated 363 character response", - "query_processed.mode": "Web Search", - "query_processed.processing_time": 1.7608659267425537, - "query_processed.response_length": 363 - }, - "events": [], - "links": [], - "trace_trace_id": "32170d1a-d211-46d8-9cdc-5a4265f0c443", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:42.124000000", - "trace_end_time": "2025-10-16 08:50:43.885000000", - "trace_duration_ms": 1761.047, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:42.124000000", - "updated_at": "2025-10-16 08:50:42.124000000" - }, - { - "span_id": "7ab2ec7b-a617-4ebf-9bb1-82dc6b18e160", - "trace_id": "32170d1a-d211-46d8-9cdc-5a4265f0c443", - "parent_span_id": "4c391bb1-1a62-4e2a-913b-96f83feff494", - "name": "agent.query_routing", - "start_time": "2025-10-16 08:50:42.124000000", - "end_time": "2025-10-16 08:50:43.885000000", - "duration_ms": 1760.833, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What's the current status of cryptocurrency markets?", - "query_length": 52, - "query_type": "routing_query", - "output_response": "Routed to Web Search for query processing", - "router.classification": "Web Search", - "router.confidence_score": 1, - "router.rag_keyword_score": 0, - "router.web_keyword_score": 3, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "32170d1a-d211-46d8-9cdc-5a4265f0c443", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:42.124000000", - "trace_end_time": "2025-10-16 08:50:43.885000000", - "trace_duration_ms": 1761.047, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:42.124000000", - "updated_at": "2025-10-16 08:50:42.124000000" - }, - { - "span_id": "db01d228-7c63-4814-b2ea-bdf3898f9c0d", - "trace_id": "32170d1a-d211-46d8-9cdc-5a4265f0c443", - "parent_span_id": "7ab2ec7b-a617-4ebf-9bb1-82dc6b18e160", - "name": "agent.web_search_generation", - "start_time": "2025-10-16 08:50:42.124000000", - "end_time": "2025-10-16 08:50:43.885000000", - "duration_ms": 1760.701, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "web_search_agent", - "agent.operation": "web_search_generation", - "agent.capabilities": [ - "web_search", - "content_synthesis", - "response_generation" - ], - "agent.id": "noveum_web_search_agent", - "input_query": "What's the current status of cryptocurrency markets?", - "query_length": 52, - "query_type": "web_search_query", - "output_response": "Web Search Answer: The web search did not yield any specific information regarding the current status of cryptocurrency markets. Therefore, I cannot provide an update on market trends, prices, or significant events affe...", - "web_search.results_count": 1, - "web_search.context_length": 154, - "web_search.response_length": 363, - "web_search.mode": "external_web_search" - }, - "events": [], - "links": [], - "trace_trace_id": "32170d1a-d211-46d8-9cdc-5a4265f0c443", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:42.124000000", - "trace_end_time": "2025-10-16 08:50:43.885000000", - "trace_duration_ms": 1761.047, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:42.124000000", - "updated_at": "2025-10-16 08:50:42.124000000" - }, - { - "span_id": "da7b0550-b13c-4518-ad3b-050c666d198c", - "trace_id": "32170d1a-d211-46d8-9cdc-5a4265f0c443", - "parent_span_id": "7ab2ec7b-a617-4ebf-9bb1-82dc6b18e160", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-16 08:50:42.124000000", - "end_time": "2025-10-16 08:50:42.124000000", - "duration_ms": 0.029, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What's the current status of cryptocurrency markets?", - "query_length": 52, - "query_type": "routing_evaluation_query", - "classification.mode": "Web Search", - "classification.rag_keyword_score": 0, - "classification.web_keyword_score": 3, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "complex", - "query.intent": "general_knowledge", - "query.keyword_density": 0.42857142857142855, - "query.domain_affinity": "general", - "routing.decision": "Routed to Web Search based on analysis", - "routing.rationale": "RAG score: 0, Web score: 3, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "RAG", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.057692307692307696, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: Web Search (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "32170d1a-d211-46d8-9cdc-5a4265f0c443", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:42.124000000", - "trace_end_time": "2025-10-16 08:50:43.885000000", - "trace_duration_ms": 1761.047, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:42.124000000", - "updated_at": "2025-10-16 08:50:42.124000000" - }, - { - "span_id": "950e18b6-9f62-4ac4-8788-3cac08fb7899", - "trace_id": "32170d1a-d211-46d8-9cdc-5a4265f0c443", - "parent_span_id": "db01d228-7c63-4814-b2ea-bdf3898f9c0d", - "name": "agent.llm_model_execution", - "start_time": "2025-10-16 08:50:42.465000000", - "end_time": "2025-10-16 08:50:43.885000000", - "duration_ms": 1419.5690000000002, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What's the current status of cryptocurrency markets?", - "query_length": 52, - "query_type": "web_search_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 1.4195220470428467, - "model.latency_ms": 1419.5220470428467, - "model.start_time": 1760604642.4658613, - "model.end_time": 1760604643.8853834, - "model.performance_tier": "fast", - "model.prompt_tokens": 162, - "model.completion_tokens": 90, - "model.total_tokens": 252, - "model.tokens_per_second": 177.52454111224782, - "model.estimated_cost": 0.00756, - "model.efficiency_score": 1.4404761904761905, - "model.response_length": 363, - "model.response_quality": "high", - "model.output_response": "Model Response: The web search did not yield any specific information regarding the current status of cryptocurrency markets. Therefore, I cannot provide an update on market trends, prices, or significant events affe...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "32170d1a-d211-46d8-9cdc-5a4265f0c443", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:42.124000000", - "trace_end_time": "2025-10-16 08:50:43.885000000", - "trace_duration_ms": 1761.047, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:42.124000000", - "updated_at": "2025-10-16 08:50:42.124000000" - }, - { - "span_id": "e87846c9-ce82-42c2-a517-e0f67e9d14c1", - "trace_id": "32170d1a-d211-46d8-9cdc-5a4265f0c443", - "parent_span_id": "db01d228-7c63-4814-b2ea-bdf3898f9c0d", - "name": "agent.web_search_evaluation_metrics", - "start_time": "2025-10-16 08:50:43.885000000", - "end_time": "2025-10-16 08:50:43.885000000", - "duration_ms": 0.02, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "web_search_evaluation_metrics", - "agent.capabilities": [ - "web_search_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What's the current status of cryptocurrency markets?", - "query_length": 52, - "query_type": "web_search_evaluation_query", - "web_search.results_count": 1, - "web_search.context_length": 154, - "web_search.context_synthesized": "Context from 1 web sources", - "web_search.search_effectiveness": 0.2, - "web_search.context_quality": "low", - "web_search.source_diversity": 1, - "web_search.information_synthesis": "low", - "web_search.external_knowledge_utilization": 0.154, - "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What's+the+current+status+of+cryptocurrency+markets?\n\nUser Question: What's the current status of cryptocurrency markets?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", - "prompt.prompt_length": 651, - "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What's+the+current+status+of+cryptocurrency+markets?", - "prompt.instruction_following": "web_search_optimized", - "response.answer_length": 363, - "response.answer_completeness": "complete", - "response.response_quality": "medium", - "response.source_citation": 1, - "response.context_utilization": 2.357142857142857, - "output_response": "Web Search Answer: The web search did not yield any specific information regarding the current status of cryptocurrency markets. Therefore, I cannot provide an update on market trends, prices, or significant events affe...", - "evaluation.search_effectiveness": 0.2, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 1, - "evaluation.context_relevance": "low", - "evaluation.overall_quality": "low", - "evaluation.ready_for_production": true, - "web_search.search_strategy": "duckduckgo_api", - "web_search.real_time_data": true, - "web_search.external_sources": 1, - "web_search.information_freshness": "current", - "web_search.knowledge_synthesis": "single_source" - }, - "events": [], - "links": [], - "trace_trace_id": "32170d1a-d211-46d8-9cdc-5a4265f0c443", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:42.124000000", - "trace_end_time": "2025-10-16 08:50:43.885000000", - "trace_duration_ms": 1761.047, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:42.124000000", - "updated_at": "2025-10-16 08:50:42.124000000" - }, - { - "span_id": "305daacf-990b-4e23-8720-48900f75d9be", - "trace_id": "9424d420-deee-478b-b727-9dc791a7457c", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-16 08:50:31.476000000", - "end_time": "2025-10-16 08:50:41.121000000", - "duration_ms": 9644.752999999999, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What are the latest updates in Python programming?", - "query.length": 50, - "processing.mode": "Web Search", - "processing.time_seconds": 9.644662618637085, - "processing.response_length": 1408, - "processing.sources_count": 1, - "output_response": "Final Answer: The web search did not yield specific results regarding the latest updates in Python programming. However, I can provide some general information based on my knowledge up to October 2023.\n\nAs of late ...", - "final_answer_mode": "Web Search", - "query_processed.input_query": "What are the latest updates in Python programming?", - "query_processed.output_response": "Successfully processed query using Web Search, generated 1408 character response", - "query_processed.mode": "Web Search", - "query_processed.processing_time": 9.644662618637085, - "query_processed.response_length": 1408 - }, - "events": [], - "links": [], - "trace_trace_id": "9424d420-deee-478b-b727-9dc791a7457c", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:31.476000000", - "trace_end_time": "2025-10-16 08:50:41.121000000", - "trace_duration_ms": 9644.828, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:31.476000000", - "updated_at": "2025-10-16 08:50:31.476000000" - }, - { - "span_id": "56b4cc1b-0646-4dee-acb3-db98f3d44477", - "trace_id": "9424d420-deee-478b-b727-9dc791a7457c", - "parent_span_id": "fb234e30-69be-4d06-9248-c05bcc22f063", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-16 08:50:31.476000000", - "end_time": "2025-10-16 08:50:31.476000000", - "duration_ms": 0.036000000000000004, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What are the latest updates in Python programming?", - "query_length": 50, - "query_type": "routing_evaluation_query", - "classification.mode": "Web Search", - "classification.rag_keyword_score": 0, - "classification.web_keyword_score": 2, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "general_knowledge", - "query.keyword_density": 0.25, - "query.domain_affinity": "general", - "routing.decision": "Routed to Web Search based on analysis", - "routing.rationale": "RAG score: 0, Web score: 2, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "RAG", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.038461538461538464, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: Web Search (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "9424d420-deee-478b-b727-9dc791a7457c", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:31.476000000", - "trace_end_time": "2025-10-16 08:50:41.121000000", - "trace_duration_ms": 9644.828, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:31.476000000", - "updated_at": "2025-10-16 08:50:31.476000000" - }, - { - "span_id": "fb234e30-69be-4d06-9248-c05bcc22f063", - "trace_id": "9424d420-deee-478b-b727-9dc791a7457c", - "parent_span_id": "305daacf-990b-4e23-8720-48900f75d9be", - "name": "agent.query_routing", - "start_time": "2025-10-16 08:50:31.476000000", - "end_time": "2025-10-16 08:50:41.121000000", - "duration_ms": 9644.627, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What are the latest updates in Python programming?", - "query_length": 50, - "query_type": "routing_query", - "output_response": "Routed to Web Search for query processing", - "router.classification": "Web Search", - "router.confidence_score": 1, - "router.rag_keyword_score": 0, - "router.web_keyword_score": 2, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "9424d420-deee-478b-b727-9dc791a7457c", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:31.476000000", - "trace_end_time": "2025-10-16 08:50:41.121000000", - "trace_duration_ms": 9644.828, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:31.476000000", - "updated_at": "2025-10-16 08:50:31.476000000" - }, - { - "span_id": "f8a793a9-63ee-49b7-9931-d7dcc2bb0ca7", - "trace_id": "9424d420-deee-478b-b727-9dc791a7457c", - "parent_span_id": "fb234e30-69be-4d06-9248-c05bcc22f063", - "name": "agent.web_search_generation", - "start_time": "2025-10-16 08:50:31.477000000", - "end_time": "2025-10-16 08:50:41.121000000", - "duration_ms": 9644.478, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "web_search_agent", - "agent.operation": "web_search_generation", - "agent.capabilities": [ - "web_search", - "content_synthesis", - "response_generation" - ], - "agent.id": "noveum_web_search_agent", - "input_query": "What are the latest updates in Python programming?", - "query_length": 50, - "query_type": "web_search_query", - "output_response": "Web Search Answer: The web search did not yield specific results regarding the latest updates in Python programming. However, I can provide some general information based on my knowledge up to October 2023.\n\nAs of late ...", - "web_search.results_count": 1, - "web_search.context_length": 152, - "web_search.response_length": 1408, - "web_search.mode": "external_web_search" - }, - "events": [], - "links": [], - "trace_trace_id": "9424d420-deee-478b-b727-9dc791a7457c", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:31.476000000", - "trace_end_time": "2025-10-16 08:50:41.121000000", - "trace_duration_ms": 9644.828, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:31.476000000", - "updated_at": "2025-10-16 08:50:31.476000000" - }, - { - "span_id": "b968e3d3-d4aa-4357-85d8-34a612748fbd", - "trace_id": "9424d420-deee-478b-b727-9dc791a7457c", - "parent_span_id": "f8a793a9-63ee-49b7-9931-d7dcc2bb0ca7", - "name": "agent.llm_model_execution", - "start_time": "2025-10-16 08:50:32.016000000", - "end_time": "2025-10-16 08:50:41.121000000", - "duration_ms": 9105.344000000001, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What are the latest updates in Python programming?", - "query_length": 50, - "query_type": "web_search_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 9.105276584625244, - "model.latency_ms": 9105.276584625244, - "model.start_time": 1760604632.0161095, - "model.end_time": 1760604641.121386, - "model.performance_tier": "slow", - "model.prompt_tokens": 161, - "model.completion_tokens": 352, - "model.total_tokens": 513, - "model.tokens_per_second": 56.340957381374714, - "model.estimated_cost": 0.015390000000000001, - "model.efficiency_score": 2.7446393762183234, - "model.response_length": 1408, - "model.response_quality": "high", - "model.output_response": "Model Response: The web search did not yield specific results regarding the latest updates in Python programming. However, I can provide some general information based on my knowledge up to October 2023.\n\nAs of late ...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "9424d420-deee-478b-b727-9dc791a7457c", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:31.476000000", - "trace_end_time": "2025-10-16 08:50:41.121000000", - "trace_duration_ms": 9644.828, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:31.476000000", - "updated_at": "2025-10-16 08:50:31.476000000" - }, - { - "span_id": "80817711-e64c-4a9f-802e-2a9e36ff093d", - "trace_id": "9424d420-deee-478b-b727-9dc791a7457c", - "parent_span_id": "f8a793a9-63ee-49b7-9931-d7dcc2bb0ca7", - "name": "agent.web_search_evaluation_metrics", - "start_time": "2025-10-16 08:50:41.121000000", - "end_time": "2025-10-16 08:50:41.121000000", - "duration_ms": 0.02, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "web_search_evaluation_metrics", - "agent.capabilities": [ - "web_search_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What are the latest updates in Python programming?", - "query_length": 50, - "query_type": "web_search_evaluation_query", - "web_search.results_count": 1, - "web_search.context_length": 152, - "web_search.context_synthesized": "Context from 1 web sources", - "web_search.search_effectiveness": 0.2, - "web_search.context_quality": "low", - "web_search.source_diversity": 1, - "web_search.information_synthesis": "low", - "web_search.external_knowledge_utilization": 0.152, - "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What+are+the+latest+updates+in+Python+programming?\n\nUser Question: What are the latest updates in Python programming?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", - "prompt.prompt_length": 647, - "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What+are+the+latest+updates+in+Python+programming?", - "prompt.instruction_following": "web_search_optimized", - "response.answer_length": 1408, - "response.answer_completeness": "complete", - "response.response_quality": "medium", - "response.source_citation": 1, - "response.context_utilization": 9.263157894736842, - "output_response": "Web Search Answer: The web search did not yield specific results regarding the latest updates in Python programming. However, I can provide some general information based on my knowledge up to October 2023.\n\nAs of late ...", - "evaluation.search_effectiveness": 0.2, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 1, - "evaluation.context_relevance": "low", - "evaluation.overall_quality": "low", - "evaluation.ready_for_production": true, - "web_search.search_strategy": "duckduckgo_api", - "web_search.real_time_data": true, - "web_search.external_sources": 1, - "web_search.information_freshness": "current", - "web_search.knowledge_synthesis": "single_source" - }, - "events": [], - "links": [], - "trace_trace_id": "9424d420-deee-478b-b727-9dc791a7457c", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:31.476000000", - "trace_end_time": "2025-10-16 08:50:41.121000000", - "trace_duration_ms": 9644.828, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:31.476000000", - "updated_at": "2025-10-16 08:50:31.476000000" - }, - { - "span_id": "13af0f3c-bd5c-4024-8fa3-c9e347a1ef20", - "trace_id": "a3735d64-89bb-4fdf-a851-7a1f3515c52d", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-16 08:50:27.687000000", - "end_time": "2025-10-16 08:50:30.473000000", - "duration_ms": 2786.078, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What happened in tech news this week?", - "query.length": 37, - "processing.mode": "Web Search", - "processing.time_seconds": 2.7859997749328613, - "processing.response_length": 396, - "processing.sources_count": 1, - "output_response": "Final Answer: It appears that the web search did not yield any specific results regarding recent tech news. Unfortunately, I cannot provide a summary of this week's tech news based on the available information. For...", - "final_answer_mode": "Web Search", - "query_processed.input_query": "What happened in tech news this week?", - "query_processed.output_response": "Successfully processed query using Web Search, generated 396 character response", - "query_processed.mode": "Web Search", - "query_processed.processing_time": 2.7859997749328613, - "query_processed.response_length": 396 - }, - "events": [], - "links": [], - "trace_trace_id": "a3735d64-89bb-4fdf-a851-7a1f3515c52d", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:27.687000000", - "trace_end_time": "2025-10-16 08:50:30.473000000", - "trace_duration_ms": 2786.145, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:27.687000000", - "updated_at": "2025-10-16 08:50:27.687000000" - }, - { - "span_id": "47e8f375-35e6-4097-86ff-df73e8181c23", - "trace_id": "a3735d64-89bb-4fdf-a851-7a1f3515c52d", - "parent_span_id": "a4f1a7b3-dc35-4f13-96d3-b6dc8fb21a37", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-16 08:50:27.687000000", - "end_time": "2025-10-16 08:50:27.687000000", - "duration_ms": 0.038, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What happened in tech news this week?", - "query_length": 37, - "query_type": "routing_evaluation_query", - "classification.mode": "Web Search", - "classification.rag_keyword_score": 0, - "classification.web_keyword_score": 2, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "general_knowledge", - "query.keyword_density": 0.2857142857142857, - "query.domain_affinity": "general", - "routing.decision": "Routed to Web Search based on analysis", - "routing.rationale": "RAG score: 0, Web score: 2, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "RAG", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.038461538461538464, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: Web Search (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "a3735d64-89bb-4fdf-a851-7a1f3515c52d", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:27.687000000", - "trace_end_time": "2025-10-16 08:50:30.473000000", - "trace_duration_ms": 2786.145, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:27.687000000", - "updated_at": "2025-10-16 08:50:27.687000000" - }, - { - "span_id": "a4f1a7b3-dc35-4f13-96d3-b6dc8fb21a37", - "trace_id": "a3735d64-89bb-4fdf-a851-7a1f3515c52d", - "parent_span_id": "13af0f3c-bd5c-4024-8fa3-c9e347a1ef20", - "name": "agent.query_routing", - "start_time": "2025-10-16 08:50:27.687000000", - "end_time": "2025-10-16 08:50:30.473000000", - "duration_ms": 2785.964, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What happened in tech news this week?", - "query_length": 37, - "query_type": "routing_query", - "output_response": "Routed to Web Search for query processing", - "router.classification": "Web Search", - "router.confidence_score": 1, - "router.rag_keyword_score": 0, - "router.web_keyword_score": 2, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "a3735d64-89bb-4fdf-a851-7a1f3515c52d", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:27.687000000", - "trace_end_time": "2025-10-16 08:50:30.473000000", - "trace_duration_ms": 2786.145, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:27.687000000", - "updated_at": "2025-10-16 08:50:27.687000000" - }, - { - "span_id": "ebfff280-5eb6-4838-8de7-35c85e2e3eaa", - "trace_id": "a3735d64-89bb-4fdf-a851-7a1f3515c52d", - "parent_span_id": "a4f1a7b3-dc35-4f13-96d3-b6dc8fb21a37", - "name": "agent.web_search_generation", - "start_time": "2025-10-16 08:50:27.687000000", - "end_time": "2025-10-16 08:50:30.473000000", - "duration_ms": 2785.8230000000003, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "web_search_agent", - "agent.operation": "web_search_generation", - "agent.capabilities": [ - "web_search", - "content_synthesis", - "response_generation" - ], - "agent.id": "noveum_web_search_agent", - "input_query": "What happened in tech news this week?", - "query_length": 37, - "query_type": "web_search_query", - "output_response": "Web Search Answer: It appears that the web search did not yield any specific results regarding recent tech news. Unfortunately, I cannot provide a summary of this week's tech news based on the available information. For...", - "web_search.results_count": 1, - "web_search.context_length": 139, - "web_search.response_length": 396, - "web_search.mode": "external_web_search" - }, - "events": [], - "links": [], - "trace_trace_id": "a3735d64-89bb-4fdf-a851-7a1f3515c52d", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:27.687000000", - "trace_end_time": "2025-10-16 08:50:30.473000000", - "trace_duration_ms": 2786.145, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:27.687000000", - "updated_at": "2025-10-16 08:50:27.687000000" - }, - { - "span_id": "835b9351-d381-4785-8538-fbe1064fc8cd", - "trace_id": "a3735d64-89bb-4fdf-a851-7a1f3515c52d", - "parent_span_id": "ebfff280-5eb6-4838-8de7-35c85e2e3eaa", - "name": "agent.llm_model_execution", - "start_time": "2025-10-16 08:50:28.026000000", - "end_time": "2025-10-16 08:50:30.473000000", - "duration_ms": 2447.269, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What happened in tech news this week?", - "query_length": 37, - "query_type": "web_search_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 2.447227716445923, - "model.latency_ms": 2447.227716445923, - "model.start_time": 1760604628.0260897, - "model.end_time": 1760604630.4733174, - "model.performance_tier": "medium", - "model.prompt_tokens": 155, - "model.completion_tokens": 99, - "model.total_tokens": 254, - "model.tokens_per_second": 103.79091340501853, - "model.estimated_cost": 0.00762, - "model.efficiency_score": 1.5590551181102361, - "model.response_length": 396, - "model.response_quality": "high", - "model.output_response": "Model Response: It appears that the web search did not yield any specific results regarding recent tech news. Unfortunately, I cannot provide a summary of this week's tech news based on the available information. For...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "a3735d64-89bb-4fdf-a851-7a1f3515c52d", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:27.687000000", - "trace_end_time": "2025-10-16 08:50:30.473000000", - "trace_duration_ms": 2786.145, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:27.687000000", - "updated_at": "2025-10-16 08:50:27.687000000" - }, - { - "span_id": "93c6f427-6fcd-4318-aec8-095553fed469", - "trace_id": "a3735d64-89bb-4fdf-a851-7a1f3515c52d", - "parent_span_id": "ebfff280-5eb6-4838-8de7-35c85e2e3eaa", - "name": "agent.web_search_evaluation_metrics", - "start_time": "2025-10-16 08:50:30.473000000", - "end_time": "2025-10-16 08:50:30.473000000", - "duration_ms": 0.017, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "web_search_evaluation_metrics", - "agent.capabilities": [ - "web_search_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What happened in tech news this week?", - "query_length": 37, - "query_type": "web_search_evaluation_query", - "web_search.results_count": 1, - "web_search.context_length": 139, - "web_search.context_synthesized": "Context from 1 web sources", - "web_search.search_effectiveness": 0.2, - "web_search.context_quality": "low", - "web_search.source_diversity": 1, - "web_search.information_synthesis": "low", - "web_search.external_knowledge_utilization": 0.139, - "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What+happened+in+tech+news+this+week?\n\nUser Question: What happened in tech news this week?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", - "prompt.prompt_length": 621, - "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What+happened+in+tech+news+this+week?", - "prompt.instruction_following": "web_search_optimized", - "response.answer_length": 396, - "response.answer_completeness": "complete", - "response.response_quality": "medium", - "response.source_citation": 1, - "response.context_utilization": 2.8489208633093526, - "output_response": "Web Search Answer: It appears that the web search did not yield any specific results regarding recent tech news. Unfortunately, I cannot provide a summary of this week's tech news based on the available information. For...", - "evaluation.search_effectiveness": 0.2, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 1, - "evaluation.context_relevance": "low", - "evaluation.overall_quality": "low", - "evaluation.ready_for_production": true, - "web_search.search_strategy": "duckduckgo_api", - "web_search.real_time_data": true, - "web_search.external_sources": 1, - "web_search.information_freshness": "current", - "web_search.knowledge_synthesis": "single_source" - }, - "events": [], - "links": [], - "trace_trace_id": "a3735d64-89bb-4fdf-a851-7a1f3515c52d", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:27.687000000", - "trace_end_time": "2025-10-16 08:50:30.473000000", - "trace_duration_ms": 2786.145, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:27.687000000", - "updated_at": "2025-10-16 08:50:27.687000000" - }, - { - "span_id": "622e7353-4eeb-47d5-8558-4ecec28cdcfd", - "trace_id": "fa5bb641-f4b7-4741-b06c-392a5f2a9aea", - "parent_span_id": "c0af6629-48b1-45fc-9dd2-60f098ea140e", - "name": "agent.query_routing", - "start_time": "2025-10-16 08:50:21.645000000", - "end_time": "2025-10-16 08:50:26.683000000", - "duration_ms": 5037.965, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What are the current trends in observability tools?", - "query_length": 51, - "query_type": "routing_query", - "output_response": "Routed to RAG for query processing", - "router.classification": "RAG", - "router.confidence_score": 0.3333333333333333, - "router.rag_keyword_score": 2, - "router.web_keyword_score": 1, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "fa5bb641-f4b7-4741-b06c-392a5f2a9aea", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:21.645000000", - "trace_end_time": "2025-10-16 08:50:26.683000000", - "trace_duration_ms": 5038.112, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:21.645000000", - "updated_at": "2025-10-16 08:50:21.645000000" - }, - { - "span_id": "c0af6629-48b1-45fc-9dd2-60f098ea140e", - "trace_id": "fa5bb641-f4b7-4741-b06c-392a5f2a9aea", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-16 08:50:21.645000000", - "end_time": "2025-10-16 08:50:26.683000000", - "duration_ms": 5038.0419999999995, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What are the current trends in observability tools?", - "query.length": 51, - "processing.mode": "RAG", - "processing.time_seconds": 5.0380027294158936, - "processing.response_length": 897, - "processing.sources_count": 5, - "output_response": "Final Answer: The provided context does not specifically outline current trends in observability tools beyond the focus on AI applications. However, it highlights the limitations of traditional monitoring tools for...", - "final_answer_mode": "RAG", - "query_processed.input_query": "What are the current trends in observability tools?", - "query_processed.output_response": "Successfully processed query using RAG, generated 897 character response", - "query_processed.mode": "RAG", - "query_processed.processing_time": 5.0380027294158936, - "query_processed.response_length": 897 - }, - "events": [], - "links": [], - "trace_trace_id": "fa5bb641-f4b7-4741-b06c-392a5f2a9aea", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:21.645000000", - "trace_end_time": "2025-10-16 08:50:26.683000000", - "trace_duration_ms": 5038.112, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:21.645000000", - "updated_at": "2025-10-16 08:50:21.645000000" - }, - { - "span_id": "ee55906d-ffe3-42fd-9899-a476d4c33ff5", - "trace_id": "fa5bb641-f4b7-4741-b06c-392a5f2a9aea", - "parent_span_id": "622e7353-4eeb-47d5-8558-4ecec28cdcfd", - "name": "agent.llm-rag", - "start_time": "2025-10-16 08:50:21.645000000", - "end_time": "2025-10-16 08:50:26.683000000", - "duration_ms": 5037.803, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "rag_agent", - "agent.operation": "llm-rag", - "agent.capabilities": [ - "document_retrieval", - "context_generation", - "response_generation" - ], - "agent.id": "noveum_rag_agent", - "input_query": "What are the current trends in observability tools?", - "query_length": 51, - "query_type": "rag_query", - "output_response": "RAG Answer: The provided context does not specifically outline current trends in observability tools beyond the focus on AI applications. However, it highlights the limitations of traditional monitoring tools for...", - "rag.context_length": 2943, - "rag.sources_count": 5, - "rag.answer_length": 897, - "rag.mode": "retrieval_augmented_generation" - }, - "events": [], - "links": [], - "trace_trace_id": "fa5bb641-f4b7-4741-b06c-392a5f2a9aea", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:21.645000000", - "trace_end_time": "2025-10-16 08:50:26.683000000", - "trace_duration_ms": 5038.112, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:21.645000000", - "updated_at": "2025-10-16 08:50:21.645000000" - }, - { - "span_id": "fe610ecb-cc8e-4466-b719-34c6e0d45d87", - "trace_id": "fa5bb641-f4b7-4741-b06c-392a5f2a9aea", - "parent_span_id": "622e7353-4eeb-47d5-8558-4ecec28cdcfd", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-16 08:50:21.645000000", - "end_time": "2025-10-16 08:50:21.645000000", - "duration_ms": 0.032, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What are the current trends in observability tools?", - "query_length": 51, - "query_type": "routing_evaluation_query", - "classification.mode": "RAG", - "classification.rag_keyword_score": 2, - "classification.web_keyword_score": 1, - "classification.confidence_score": 0.3333333333333333, - "classification.confidence_level": "medium", - "classification.method": "keyword_based", - "query.complexity": "complex", - "query.intent": "documentation", - "query.keyword_density": 0.375, - "query.domain_affinity": "noveum", - "routing.decision": "Routed to RAG based on analysis", - "routing.rationale": "RAG score: 2, Web score: 1, Confidence: 0.33", - "routing.expected_performance": "medium", - "routing.alternative_mode": "Web Search", - "routing.decision_confidence": 0.3333333333333333, - "evaluation.routing_accuracy": "medium", - "evaluation.keyword_coverage": 0.057692307692307696, - "evaluation.query_understanding": "ambiguous", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: RAG (Confidence: 0.33)" - }, - "events": [], - "links": [], - "trace_trace_id": "fa5bb641-f4b7-4741-b06c-392a5f2a9aea", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:21.645000000", - "trace_end_time": "2025-10-16 08:50:26.683000000", - "trace_duration_ms": 5038.112, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:21.645000000", - "updated_at": "2025-10-16 08:50:21.645000000" - }, - { - "span_id": "69a4773d-c738-489e-87ed-999583e02187", - "trace_id": "fa5bb641-f4b7-4741-b06c-392a5f2a9aea", - "parent_span_id": "ee55906d-ffe3-42fd-9899-a476d4c33ff5", - "name": "agent.llm_model_execution", - "start_time": "2025-10-16 08:50:22.600000000", - "end_time": "2025-10-16 08:50:25.786000000", - "duration_ms": 3185.257, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What are the current trends in observability tools?", - "query_length": 51, - "query_type": "rag_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 3.185206174850464, - "model.latency_ms": 3185.206174850464, - "model.start_time": 1760604622.6008554, - "model.end_time": 1760604625.7860615, - "model.performance_tier": "medium", - "model.prompt_tokens": 872, - "model.completion_tokens": 224, - "model.total_tokens": 1096, - "model.tokens_per_second": 344.09075577390337, - "model.estimated_cost": 0.03288, - "model.efficiency_score": 0.8184306569343066, - "model.response_length": 897, - "model.response_quality": "high", - "model.output_response": "Model Response: The provided context does not specifically outline current trends in observability tools beyond the focus on AI applications. However, it highlights the limitations of traditional monitoring tools for...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "fa5bb641-f4b7-4741-b06c-392a5f2a9aea", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:21.645000000", - "trace_end_time": "2025-10-16 08:50:26.683000000", - "trace_duration_ms": 5038.112, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:21.645000000", - "updated_at": "2025-10-16 08:50:21.645000000" - }, - { - "span_id": "5e5a9d30-4cd0-45cd-9494-192c280727ac", - "trace_id": "fa5bb641-f4b7-4741-b06c-392a5f2a9aea", - "parent_span_id": "ee55906d-ffe3-42fd-9899-a476d4c33ff5", - "name": "agent.rag_evaluation_metrics", - "start_time": "2025-10-16 08:50:25.786000000", - "end_time": "2025-10-16 08:50:25.786000000", - "duration_ms": 0.066, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "rag_evaluation_metrics", - "agent.capabilities": [ - "retrieval_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What are the current trends in observability tools?", - "query_length": 51, - "query_type": "rag_evaluation_query", - "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/blog/from-logs-to-intelligent-choices-inside-noveum-ais-evaluation-process):\nFrom Development to Production - Inside Noveum.ai's AI Observability Platform\nShashank Agarwal\n3/3/2025\nIntroduction\nEvery AI application tells a story\u2014but most developers never get to hear it...", - "retrieval.context_length": 2943, - "retrieval.sources_count": 5, - "retrieval.context_quality": "high", - "retrieval.effectiveness": 1, - "retrieval.context_utilization": 2.943, - "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/blog/from-logs-to-intelligent-choices-inside-noveum-ais-evaluation-process):\nFrom Development to Production - Inside Noveum.ai's AI Observability Platform\nShashank Agarwal\n3/3/2025\nIntroduction\nEvery AI application tells a story\u2014but most developers never get to hear it. When your RAG pipeline returns irrelevant results, when your multi-agent system gets stuck in loops, or when your LLM costs suddenly spike, you're left guessing what went wrong and where.\nThat's exactly the challenge Noveum.ai solves. Rather than flying blind, our platform provides comprehensive tracing a...\n\nSource 2 (https://noveum.ai/docs/getting-started/tracing-concepts):\n- Explore Framework Integrations - Framework-specific guidance\n- Learn Advanced Patterns - Custom instrumentation techniques\n- Master the Dashboard - Analyze your traces effectively\nRemember: Good observability is not about collecting all possible data, but about collecting the right data that helps you understand, debug, and optimize your AI applications.\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to O...\n\nSource 3 (https://noveum.ai/en/docs/getting-started/tracing-concepts):\n- Explore Framework Integrations - Framework-specific guidance\n- Learn Advanced Patterns - Custom instrumentation techniques\n- Master the Dashboard - Analyze your traces effectively\nRemember: Good observability is not about collecting all possible data, but about collecting the right data that helps you understand, debug, and optimize your AI applications.\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to O...\n\nSource 4 (https://noveum.ai/en/blog/from-logs-to-intelligent-choices-inside-noveum-ais-evaluation-process):\nThe AI Observability Challenge\nWhy Traditional Monitoring Falls Short\nTraditional application monitoring tools weren't built for AI applications. They can tell you if your API is responding, but they can't answer the questions that matter most for AI systems:\n- Why did my RAG pipeline retrieve irrelevant documents?\n- Which LLM calls are driving my costs?\n- How are my agents communicating with each other?\n- What's causing hallucinations in my responses?\n- Why is my embedding generation so slow?\nW...\n\nSource 5 (https://noveum.ai/en/docs/advanced/rag-observability):\n\ud83d\udd27 Best Practices for RAG Observability\n1. Comprehensive Pipeline Tracing\n2. Quality Monitoring\n3. A/B Testing for RAG Components\n\ud83c\udfaf Advanced RAG Patterns\nMulti-Modal RAG\nConversational RAG\nRAG observability with Noveum.ai provides the deep insights needed to build, optimize, and scale retrieval-augmented generation systems. By implementing comprehensive tracing across retrieval, context preparation, and generation stages, you can ensure your RAG pipeline delivers accurate, relevant, and cost-effe...\n\nUser Question: What are the current trends in observability tools?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", - "prompt.prompt_length": 3490, - "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/blog/from-logs-to-intelligent-choices-inside-noveum-ais-evaluation-process):\nFrom Development to Production - Inside Noveum.ai's AI Observability Platform\nShashank Agarw...", - "prompt.instruction_following": "rag_optimized", - "response.answer_length": 897, - "response.answer_completeness": "complete", - "response.response_quality": "high", - "response.source_citation": 5, - "response.context_utilization": 0.30479102956167176, - "output_response": "RAG Answer: The provided context does not specifically outline current trends in observability tools beyond the focus on AI applications. However, it highlights the limitations of traditional monitoring tools for...", - "evaluation.retrieval_effectiveness": 1, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 5, - "evaluation.context_relevance": "high", - "evaluation.overall_quality": "high", - "evaluation.ready_for_production": true, - "rag.retrieval_strategy": "semantic_similarity", - "rag.vector_search_results": 5, - "rag.context_synthesis": "multi_source", - "rag.document_coverage": 1, - "rag.information_density": 0.30479102956167176 - }, - "events": [], - "links": [], - "trace_trace_id": "fa5bb641-f4b7-4741-b06c-392a5f2a9aea", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:21.645000000", - "trace_end_time": "2025-10-16 08:50:26.683000000", - "trace_duration_ms": 5038.112, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:21.645000000", - "updated_at": "2025-10-16 08:50:21.645000000" - }, - { - "span_id": "2e06953f-5cec-4873-92b5-91f7281f6d7c", - "trace_id": "c0a95e8e-513c-47eb-b746-4e6016aeaf7d", - "parent_span_id": "9e41eab5-cd8e-4e46-a15b-e5f52c9a4073", - "name": "agent.query_routing", - "start_time": "2025-10-16 08:50:08.944000000", - "end_time": "2025-10-16 08:50:20.641000000", - "duration_ms": 11696.524, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "Tell me about recent developments in machine learning", - "query_length": 53, - "query_type": "routing_query", - "output_response": "Routed to Web Search for query processing", - "router.classification": "Web Search", - "router.confidence_score": 1, - "router.rag_keyword_score": 0, - "router.web_keyword_score": 1, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "c0a95e8e-513c-47eb-b746-4e6016aeaf7d", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:08.944000000", - "trace_end_time": "2025-10-16 08:50:20.641000000", - "trace_duration_ms": 11696.768, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:08.944000000", - "updated_at": "2025-10-16 08:50:08.944000000" - }, - { - "span_id": "9e41eab5-cd8e-4e46-a15b-e5f52c9a4073", - "trace_id": "c0a95e8e-513c-47eb-b746-4e6016aeaf7d", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-16 08:50:08.944000000", - "end_time": "2025-10-16 08:50:20.641000000", - "duration_ms": 11696.671999999999, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "Tell me about recent developments in machine learning", - "query.length": 53, - "processing.mode": "Web Search", - "processing.time_seconds": 11.696574449539185, - "processing.response_length": 2001, - "processing.sources_count": 1, - "output_response": "Final Answer: The web search did not yield specific results regarding recent developments in machine learning. However, I can provide a general overview based on my knowledge up to October 2023.\n\nRecent development...", - "final_answer_mode": "Web Search", - "query_processed.input_query": "Tell me about recent developments in machine learning", - "query_processed.output_response": "Successfully processed query using Web Search, generated 2001 character response", - "query_processed.mode": "Web Search", - "query_processed.processing_time": 11.696574449539185, - "query_processed.response_length": 2001 - }, - "events": [], - "links": [], - "trace_trace_id": "c0a95e8e-513c-47eb-b746-4e6016aeaf7d", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:08.944000000", - "trace_end_time": "2025-10-16 08:50:20.641000000", - "trace_duration_ms": 11696.768, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:08.944000000", - "updated_at": "2025-10-16 08:50:08.944000000" - }, - { - "span_id": "7ddf938b-183a-427f-849b-b31b18a98d87", - "trace_id": "c0a95e8e-513c-47eb-b746-4e6016aeaf7d", - "parent_span_id": "2e06953f-5cec-4873-92b5-91f7281f6d7c", - "name": "agent.web_search_generation", - "start_time": "2025-10-16 08:50:08.945000000", - "end_time": "2025-10-16 08:50:20.641000000", - "duration_ms": 11696.328, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "web_search_agent", - "agent.operation": "web_search_generation", - "agent.capabilities": [ - "web_search", - "content_synthesis", - "response_generation" - ], - "agent.id": "noveum_web_search_agent", - "input_query": "Tell me about recent developments in machine learning", - "query_length": 53, - "query_type": "web_search_query", - "output_response": "Web Search Answer: The web search did not yield specific results regarding recent developments in machine learning. However, I can provide a general overview based on my knowledge up to October 2023.\n\nRecent development...", - "web_search.results_count": 1, - "web_search.context_length": 155, - "web_search.response_length": 2001, - "web_search.mode": "external_web_search" - }, - "events": [], - "links": [], - "trace_trace_id": "c0a95e8e-513c-47eb-b746-4e6016aeaf7d", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:08.944000000", - "trace_end_time": "2025-10-16 08:50:20.641000000", - "trace_duration_ms": 11696.768, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:08.944000000", - "updated_at": "2025-10-16 08:50:08.944000000" - }, - { - "span_id": "f3e4c594-f822-4f04-ab43-98fac161b9a3", - "trace_id": "c0a95e8e-513c-47eb-b746-4e6016aeaf7d", - "parent_span_id": "2e06953f-5cec-4873-92b5-91f7281f6d7c", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-16 08:50:08.945000000", - "end_time": "2025-10-16 08:50:08.945000000", - "duration_ms": 0.046, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: Tell me about recent developments in machine learning", - "query_length": 53, - "query_type": "routing_evaluation_query", - "classification.mode": "Web Search", - "classification.rag_keyword_score": 0, - "classification.web_keyword_score": 1, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "complex", - "query.intent": "general_knowledge", - "query.keyword_density": 0.125, - "query.domain_affinity": "general", - "routing.decision": "Routed to Web Search based on analysis", - "routing.rationale": "RAG score: 0, Web score: 1, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "RAG", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.019230769230769232, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: Web Search (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "c0a95e8e-513c-47eb-b746-4e6016aeaf7d", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:08.944000000", - "trace_end_time": "2025-10-16 08:50:20.641000000", - "trace_duration_ms": 11696.768, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:08.944000000", - "updated_at": "2025-10-16 08:50:08.944000000" - }, - { - "span_id": "b47d09be-a43b-4b0a-b7d3-d98bdf1773a3", - "trace_id": "c0a95e8e-513c-47eb-b746-4e6016aeaf7d", - "parent_span_id": "7ddf938b-183a-427f-849b-b31b18a98d87", - "name": "agent.llm_model_execution", - "start_time": "2025-10-16 08:50:09.490000000", - "end_time": "2025-10-16 08:50:20.641000000", - "duration_ms": 11151.141000000001, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: Tell me about recent developments in machine learning", - "query_length": 53, - "query_type": "web_search_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 11.151098012924194, - "model.latency_ms": 11151.098012924194, - "model.start_time": 1760604609.4902723, - "model.end_time": 1760604620.6413703, - "model.performance_tier": "slow", - "model.prompt_tokens": 163, - "model.completion_tokens": 500, - "model.total_tokens": 663, - "model.tokens_per_second": 59.456028386763236, - "model.estimated_cost": 0.01989, - "model.efficiency_score": 3.018099547511312, - "model.response_length": 2001, - "model.response_quality": "high", - "model.output_response": "Model Response: The web search did not yield specific results regarding recent developments in machine learning. However, I can provide a general overview based on my knowledge up to October 2023.\n\nRecent development...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "c0a95e8e-513c-47eb-b746-4e6016aeaf7d", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:08.944000000", - "trace_end_time": "2025-10-16 08:50:20.641000000", - "trace_duration_ms": 11696.768, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:08.944000000", - "updated_at": "2025-10-16 08:50:08.944000000" - }, - { - "span_id": "bfea8500-0e5c-4556-a014-5b2ec49b7ad5", - "trace_id": "c0a95e8e-513c-47eb-b746-4e6016aeaf7d", - "parent_span_id": "7ddf938b-183a-427f-849b-b31b18a98d87", - "name": "agent.web_search_evaluation_metrics", - "start_time": "2025-10-16 08:50:20.641000000", - "end_time": "2025-10-16 08:50:20.641000000", - "duration_ms": 0.018000000000000002, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "web_search_evaluation_metrics", - "agent.capabilities": [ - "web_search_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: Tell me about recent developments in machine learning", - "query_length": 53, - "query_type": "web_search_evaluation_query", - "web_search.results_count": 1, - "web_search.context_length": 155, - "web_search.context_synthesized": "Context from 1 web sources", - "web_search.search_effectiveness": 0.2, - "web_search.context_quality": "low", - "web_search.source_diversity": 1, - "web_search.information_synthesis": "low", - "web_search.external_knowledge_utilization": 0.155, - "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=Tell+me+about+recent+developments+in+machine+learning\n\nUser Question: Tell me about recent developments in machine learning\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", - "prompt.prompt_length": 653, - "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=Tell+me+about+recent+developments+in+machine+learning", - "prompt.instruction_following": "web_search_optimized", - "response.answer_length": 2001, - "response.answer_completeness": "complete", - "response.response_quality": "medium", - "response.source_citation": 1, - "response.context_utilization": 12.90967741935484, - "output_response": "Web Search Answer: The web search did not yield specific results regarding recent developments in machine learning. However, I can provide a general overview based on my knowledge up to October 2023.\n\nRecent development...", - "evaluation.search_effectiveness": 0.2, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 1, - "evaluation.context_relevance": "low", - "evaluation.overall_quality": "low", - "evaluation.ready_for_production": true, - "web_search.search_strategy": "duckduckgo_api", - "web_search.real_time_data": true, - "web_search.external_sources": 1, - "web_search.information_freshness": "current", - "web_search.knowledge_synthesis": "single_source" - }, - "events": [], - "links": [], - "trace_trace_id": "c0a95e8e-513c-47eb-b746-4e6016aeaf7d", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:08.944000000", - "trace_end_time": "2025-10-16 08:50:20.641000000", - "trace_duration_ms": 11696.768, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:08.944000000", - "updated_at": "2025-10-16 08:50:08.944000000" - }, - { - "span_id": "33297286-d440-446a-ba73-7cabd9d4b2e6", - "trace_id": "3af8c6e8-62d9-4893-92ba-cc265934ef03", - "parent_span_id": "4a3c230e-0dd5-4ba2-8e87-2a301f654fba", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-16 08:50:04.544000000", - "end_time": "2025-10-16 08:50:04.544000000", - "duration_ms": 0.038, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What's the weather like today?", - "query_length": 30, - "query_type": "routing_evaluation_query", - "classification.mode": "Web Search", - "classification.rag_keyword_score": 0, - "classification.web_keyword_score": 2, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "general_knowledge", - "query.keyword_density": 0.4, - "query.domain_affinity": "general", - "routing.decision": "Routed to Web Search based on analysis", - "routing.rationale": "RAG score: 0, Web score: 2, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "RAG", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.038461538461538464, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: Web Search (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "3af8c6e8-62d9-4893-92ba-cc265934ef03", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:04.544000000", - "trace_end_time": "2025-10-16 08:50:07.941000000", - "trace_duration_ms": 3397.228, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:04.544000000", - "updated_at": "2025-10-16 08:50:04.544000000" - }, - { - "span_id": "4a3c230e-0dd5-4ba2-8e87-2a301f654fba", - "trace_id": "3af8c6e8-62d9-4893-92ba-cc265934ef03", - "parent_span_id": "eefcbdc5-99fc-4f06-a9f4-f51ba50564db", - "name": "agent.query_routing", - "start_time": "2025-10-16 08:50:04.544000000", - "end_time": "2025-10-16 08:50:07.941000000", - "duration_ms": 3397.056, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What's the weather like today?", - "query_length": 30, - "query_type": "routing_query", - "output_response": "Routed to Web Search for query processing", - "router.classification": "Web Search", - "router.confidence_score": 1, - "router.rag_keyword_score": 0, - "router.web_keyword_score": 2, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "3af8c6e8-62d9-4893-92ba-cc265934ef03", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:04.544000000", - "trace_end_time": "2025-10-16 08:50:07.941000000", - "trace_duration_ms": 3397.228, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:04.544000000", - "updated_at": "2025-10-16 08:50:04.544000000" - }, - { - "span_id": "eefcbdc5-99fc-4f06-a9f4-f51ba50564db", - "trace_id": "3af8c6e8-62d9-4893-92ba-cc265934ef03", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-16 08:50:04.544000000", - "end_time": "2025-10-16 08:50:07.941000000", - "duration_ms": 3397.169, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What's the weather like today?", - "query.length": 30, - "processing.mode": "Web Search", - "processing.time_seconds": 3.3970956802368164, - "processing.response_length": 518, - "processing.sources_count": 1, - "output_response": "Final Answer: The web search results do not provide specific details about today's weather conditions in your location. However, they mention that you can get real-time updates, including temperature, humidity, and...", - "final_answer_mode": "Web Search", - "query_processed.input_query": "What's the weather like today?", - "query_processed.output_response": "Successfully processed query using Web Search, generated 518 character response", - "query_processed.mode": "Web Search", - "query_processed.processing_time": 3.3970956802368164, - "query_processed.response_length": 518 - }, - "events": [], - "links": [], - "trace_trace_id": "3af8c6e8-62d9-4893-92ba-cc265934ef03", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:04.544000000", - "trace_end_time": "2025-10-16 08:50:07.941000000", - "trace_duration_ms": 3397.228, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:04.544000000", - "updated_at": "2025-10-16 08:50:04.544000000" - }, - { - "span_id": "9683465c-e851-4004-a858-c8fdd1a2b3da", - "trace_id": "3af8c6e8-62d9-4893-92ba-cc265934ef03", - "parent_span_id": "4a3c230e-0dd5-4ba2-8e87-2a301f654fba", - "name": "agent.web_search_generation", - "start_time": "2025-10-16 08:50:04.545000000", - "end_time": "2025-10-16 08:50:07.941000000", - "duration_ms": 3396.871, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "web_search_agent", - "agent.operation": "web_search_generation", - "agent.capabilities": [ - "web_search", - "content_synthesis", - "response_generation" - ], - "agent.id": "noveum_web_search_agent", - "input_query": "What's the weather like today?", - "query_length": 30, - "query_type": "web_search_query", - "output_response": "Web Search Answer: The web search results do not provide specific details about today's weather conditions in your location. However, they mention that you can get real-time updates, including temperature, humidity, and...", - "web_search.results_count": 1, - "web_search.context_length": 1137, - "web_search.response_length": 518, - "web_search.mode": "external_web_search" - }, - "events": [], - "links": [], - "trace_trace_id": "3af8c6e8-62d9-4893-92ba-cc265934ef03", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:04.544000000", - "trace_end_time": "2025-10-16 08:50:07.941000000", - "trace_duration_ms": 3397.228, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:04.544000000", - "updated_at": "2025-10-16 08:50:04.544000000" - }, - { - "span_id": "6413eb7e-e5cd-4c46-9651-de14c72c260e", - "trace_id": "3af8c6e8-62d9-4893-92ba-cc265934ef03", - "parent_span_id": "9683465c-e851-4004-a858-c8fdd1a2b3da", - "name": "agent.llm_model_execution", - "start_time": "2025-10-16 08:50:04.986000000", - "end_time": "2025-10-16 08:50:07.941000000", - "duration_ms": 2955.55, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What's the weather like today?", - "query_length": 30, - "query_type": "web_search_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 2.955509662628174, - "model.latency_ms": 2955.509662628174, - "model.start_time": 1760604604.9862766, - "model.end_time": 1760604607.9417863, - "model.performance_tier": "medium", - "model.prompt_tokens": 403, - "model.completion_tokens": 129, - "model.total_tokens": 532, - "model.tokens_per_second": 180.0027950261957, - "model.estimated_cost": 0.015960000000000002, - "model.efficiency_score": 0.9736842105263158, - "model.response_length": 518, - "model.response_quality": "high", - "model.output_response": "Model Response: The web search results do not provide specific details about today's weather conditions in your location. However, they mention that you can get real-time updates, including temperature, humidity, and...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "3af8c6e8-62d9-4893-92ba-cc265934ef03", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:04.544000000", - "trace_end_time": "2025-10-16 08:50:07.941000000", - "trace_duration_ms": 3397.228, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:04.544000000", - "updated_at": "2025-10-16 08:50:04.544000000" - }, - { - "span_id": "64df69fb-0bcc-4845-8bfe-5a721c1ed912", - "trace_id": "3af8c6e8-62d9-4893-92ba-cc265934ef03", - "parent_span_id": "9683465c-e851-4004-a858-c8fdd1a2b3da", - "name": "agent.web_search_evaluation_metrics", - "start_time": "2025-10-16 08:50:07.941000000", - "end_time": "2025-10-16 08:50:07.941000000", - "duration_ms": 0.017, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "web_search_evaluation_metrics", - "agent.capabilities": [ - "web_search_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What's the weather like today?", - "query_length": 30, - "query_type": "web_search_evaluation_query", - "web_search.results_count": 1, - "web_search.context_length": 1137, - "web_search.context_synthesized": "Context from 1 web sources", - "web_search.search_effectiveness": 0.2, - "web_search.context_quality": "high", - "web_search.source_diversity": 1, - "web_search.information_synthesis": "low", - "web_search.external_knowledge_utilization": 1.137, - "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nStay updated with the latest weather conditions in Komna. Get real-time updates, temperature, humidity, AQI, and detailed forecast for today. Dec 31, 2024 \u00b7 Komna Weather Forecast: Check the latest temperature in Komna today along with details such as humidity, visibility, air quality, wind speed, pressure, and weekly weather report at Shuru. 2 days ago \u00b7 Latest weather forecast for India for today's, hourly weather forecast, including today's temperatures in India, wind, rain and more. 2 days ago \u00b7 Get the latest hourly weather updates for India today. Detailed forecast including temperature, wind, rain, snow, and UV index. Stay informed about today's weather conditions in India. 2 days ago \u00b7 Today's Weather Report ... Forecasts Urban Meteorological Services Short to Medium Range Model Guidance Extended Range Model Guidance Seasonal Forecast Quantitative Precipitation Forecast Latest CAP Alerts Specialised products & forecasts Cyclones | Monsoon | Satellite Radar | Rainfall | Agromet Advisories MAUSAM : | | Video Meghdoot Agro ...\nURL: https://duckduckgo.com/?q=What's+the+weather+like+today?\n\nUser Question: What's the weather like today?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", - "prompt.prompt_length": 1612, - "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nStay updated with the latest weather conditions in Komna. Get real-time updates, temperature, humidity, AQI, and detailed forecast for today. Dec 31, 2024 \u00b7 Komna Weather F...", - "prompt.instruction_following": "web_search_optimized", - "response.answer_length": 518, - "response.answer_completeness": "complete", - "response.response_quality": "medium", - "response.source_citation": 1, - "response.context_utilization": 0.455584872471416, - "output_response": "Web Search Answer: The web search results do not provide specific details about today's weather conditions in your location. However, they mention that you can get real-time updates, including temperature, humidity, and...", - "evaluation.search_effectiveness": 0.2, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 1, - "evaluation.context_relevance": "high", - "evaluation.overall_quality": "low", - "evaluation.ready_for_production": true, - "web_search.search_strategy": "duckduckgo_api", - "web_search.real_time_data": true, - "web_search.external_sources": 1, - "web_search.information_freshness": "current", - "web_search.knowledge_synthesis": "single_source" - }, - "events": [], - "links": [], - "trace_trace_id": "3af8c6e8-62d9-4893-92ba-cc265934ef03", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:04.544000000", - "trace_end_time": "2025-10-16 08:50:07.941000000", - "trace_duration_ms": 3397.228, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:04.544000000", - "updated_at": "2025-10-16 08:50:04.544000000" - }, - { - "span_id": "719a1a8f-fc50-439c-899b-436146accd23", - "trace_id": "d7b8b811-ad01-48f1-917f-ace7e8d289e6", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-16 08:50:01.375000000", - "end_time": "2025-10-16 08:50:03.540000000", - "duration_ms": 2164.4809999999998, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What are the latest AI news today?", - "query.length": 34, - "processing.mode": "Web Search", - "processing.time_seconds": 2.1643810272216797, - "processing.response_length": 298, - "processing.sources_count": 1, - "output_response": "Final Answer: The web search did not yield any specific results regarding the latest AI news today. Unfortunately, I cannot provide current updates or news articles on AI developments at this moment. For the latest...", - "final_answer_mode": "Web Search", - "query_processed.input_query": "What are the latest AI news today?", - "query_processed.output_response": "Successfully processed query using Web Search, generated 298 character response", - "query_processed.mode": "Web Search", - "query_processed.processing_time": 2.1643810272216797, - "query_processed.response_length": 298 - }, - "events": [], - "links": [], - "trace_trace_id": "d7b8b811-ad01-48f1-917f-ace7e8d289e6", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:01.375000000", - "trace_end_time": "2025-10-16 08:50:03.540000000", - "trace_duration_ms": 2164.576, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:01.375000000", - "updated_at": "2025-10-16 08:50:01.375000000" - }, - { - "span_id": "644a712f-1c9e-4e9a-9073-74621b754aa2", - "trace_id": "d7b8b811-ad01-48f1-917f-ace7e8d289e6", - "parent_span_id": "719a1a8f-fc50-439c-899b-436146accd23", - "name": "agent.query_routing", - "start_time": "2025-10-16 08:50:01.376000000", - "end_time": "2025-10-16 08:50:03.540000000", - "duration_ms": 2164.3219999999997, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What are the latest AI news today?", - "query_length": 34, - "query_type": "routing_query", - "output_response": "Routed to Web Search for query processing", - "router.classification": "Web Search", - "router.confidence_score": 1, - "router.rag_keyword_score": 0, - "router.web_keyword_score": 3, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "d7b8b811-ad01-48f1-917f-ace7e8d289e6", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:01.375000000", - "trace_end_time": "2025-10-16 08:50:03.540000000", - "trace_duration_ms": 2164.576, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:01.375000000", - "updated_at": "2025-10-16 08:50:01.375000000" - }, - { - "span_id": "8a4abdbe-6ad7-4d43-b234-81667c4abbc2", - "trace_id": "d7b8b811-ad01-48f1-917f-ace7e8d289e6", - "parent_span_id": "644a712f-1c9e-4e9a-9073-74621b754aa2", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-16 08:50:01.376000000", - "end_time": "2025-10-16 08:50:01.376000000", - "duration_ms": 0.051, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What are the latest AI news today?", - "query_length": 34, - "query_type": "routing_evaluation_query", - "classification.mode": "Web Search", - "classification.rag_keyword_score": 0, - "classification.web_keyword_score": 3, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "general_knowledge", - "query.keyword_density": 0.42857142857142855, - "query.domain_affinity": "general", - "routing.decision": "Routed to Web Search based on analysis", - "routing.rationale": "RAG score: 0, Web score: 3, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "RAG", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.057692307692307696, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: Web Search (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "d7b8b811-ad01-48f1-917f-ace7e8d289e6", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:01.375000000", - "trace_end_time": "2025-10-16 08:50:03.540000000", - "trace_duration_ms": 2164.576, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:01.375000000", - "updated_at": "2025-10-16 08:50:01.375000000" - }, - { - "span_id": "904621e8-c5fa-4b2d-a628-8f10eb9da51d", - "trace_id": "d7b8b811-ad01-48f1-917f-ace7e8d289e6", - "parent_span_id": "644a712f-1c9e-4e9a-9073-74621b754aa2", - "name": "agent.web_search_generation", - "start_time": "2025-10-16 08:50:01.376000000", - "end_time": "2025-10-16 08:50:03.540000000", - "duration_ms": 2164.076, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "web_search_agent", - "agent.operation": "web_search_generation", - "agent.capabilities": [ - "web_search", - "content_synthesis", - "response_generation" - ], - "agent.id": "noveum_web_search_agent", - "input_query": "What are the latest AI news today?", - "query_length": 34, - "query_type": "web_search_query", - "output_response": "Web Search Answer: The web search did not yield any specific results regarding the latest AI news today. Unfortunately, I cannot provide current updates or news articles on AI developments at this moment. For the latest...", - "web_search.results_count": 1, - "web_search.context_length": 136, - "web_search.response_length": 298, - "web_search.mode": "external_web_search" - }, - "events": [], - "links": [], - "trace_trace_id": "d7b8b811-ad01-48f1-917f-ace7e8d289e6", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:01.375000000", - "trace_end_time": "2025-10-16 08:50:03.540000000", - "trace_duration_ms": 2164.576, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:01.375000000", - "updated_at": "2025-10-16 08:50:01.375000000" - }, - { - "span_id": "421dd1af-2dbc-477f-bd59-b3194b14ae3a", - "trace_id": "d7b8b811-ad01-48f1-917f-ace7e8d289e6", - "parent_span_id": "904621e8-c5fa-4b2d-a628-8f10eb9da51d", - "name": "agent.llm_model_execution", - "start_time": "2025-10-16 08:50:01.708000000", - "end_time": "2025-10-16 08:50:03.540000000", - "duration_ms": 1831.237, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What are the latest AI news today?", - "query_length": 34, - "query_type": "web_search_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 1.8311803340911865, - "model.latency_ms": 1831.1803340911865, - "model.start_time": 1760604601.7090185, - "model.end_time": 1760604603.5401988, - "model.performance_tier": "fast", - "model.prompt_tokens": 153, - "model.completion_tokens": 74, - "model.total_tokens": 227, - "model.tokens_per_second": 123.96376029926071, - "model.estimated_cost": 0.00681, - "model.efficiency_score": 1.3127753303964758, - "model.response_length": 298, - "model.response_quality": "high", - "model.output_response": "Model Response: The web search did not yield any specific results regarding the latest AI news today. Unfortunately, I cannot provide current updates or news articles on AI developments at this moment. For the latest...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "d7b8b811-ad01-48f1-917f-ace7e8d289e6", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:01.375000000", - "trace_end_time": "2025-10-16 08:50:03.540000000", - "trace_duration_ms": 2164.576, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:01.375000000", - "updated_at": "2025-10-16 08:50:01.375000000" - }, - { - "span_id": "159aad6f-7174-406d-b7d9-eed9360d993c", - "trace_id": "d7b8b811-ad01-48f1-917f-ace7e8d289e6", - "parent_span_id": "904621e8-c5fa-4b2d-a628-8f10eb9da51d", - "name": "agent.web_search_evaluation_metrics", - "start_time": "2025-10-16 08:50:03.540000000", - "end_time": "2025-10-16 08:50:03.540000000", - "duration_ms": 0.028, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "web_search_evaluation_metrics", - "agent.capabilities": [ - "web_search_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What are the latest AI news today?", - "query_length": 34, - "query_type": "web_search_evaluation_query", - "web_search.results_count": 1, - "web_search.context_length": 136, - "web_search.context_synthesized": "Context from 1 web sources", - "web_search.search_effectiveness": 0.2, - "web_search.context_quality": "low", - "web_search.source_diversity": 1, - "web_search.information_synthesis": "low", - "web_search.external_knowledge_utilization": 0.136, - "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What+are+the+latest+AI+news+today?\n\nUser Question: What are the latest AI news today?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", - "prompt.prompt_length": 615, - "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What+are+the+latest+AI+news+today?", - "prompt.instruction_following": "web_search_optimized", - "response.answer_length": 298, - "response.answer_completeness": "complete", - "response.response_quality": "medium", - "response.source_citation": 1, - "response.context_utilization": 2.1911764705882355, - "output_response": "Web Search Answer: The web search did not yield any specific results regarding the latest AI news today. Unfortunately, I cannot provide current updates or news articles on AI developments at this moment. For the latest...", - "evaluation.search_effectiveness": 0.2, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 1, - "evaluation.context_relevance": "low", - "evaluation.overall_quality": "low", - "evaluation.ready_for_production": true, - "web_search.search_strategy": "duckduckgo_api", - "web_search.real_time_data": true, - "web_search.external_sources": 1, - "web_search.information_freshness": "current", - "web_search.knowledge_synthesis": "single_source" - }, - "events": [], - "links": [], - "trace_trace_id": "d7b8b811-ad01-48f1-917f-ace7e8d289e6", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:50:01.375000000", - "trace_end_time": "2025-10-16 08:50:03.540000000", - "trace_duration_ms": 2164.576, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:50:01.375000000", - "updated_at": "2025-10-16 08:50:01.375000000" - }, - { - "span_id": "dcf2b38e-ab90-45ee-9a65-e6fea47a3fd7", - "trace_id": "df8340b4-b75a-48e9-a2f3-99725b46a00d", - "parent_span_id": "d0e2b320-c18d-4ec6-98a1-74da5252107e", - "name": "agent.query_routing", - "start_time": "2025-10-16 08:49:17.236000000", - "end_time": "2025-10-16 08:50:00.368000000", - "duration_ms": 43131.384, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What are the benefits of using Noveum Trace?", - "query_length": 44, - "query_type": "routing_query", - "output_response": "Routed to RAG for query processing", - "router.classification": "RAG", - "router.confidence_score": 1, - "router.rag_keyword_score": 2, - "router.web_keyword_score": 0, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "df8340b4-b75a-48e9-a2f3-99725b46a00d", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:49:17.236000000", - "trace_end_time": "2025-10-16 08:50:00.368000000", - "trace_duration_ms": 43131.554000000004, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:49:17.236000000", - "updated_at": "2025-10-16 08:49:17.236000000" - }, - { - "span_id": "d0e2b320-c18d-4ec6-98a1-74da5252107e", - "trace_id": "df8340b4-b75a-48e9-a2f3-99725b46a00d", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-16 08:49:17.236000000", - "end_time": "2025-10-16 08:50:00.368000000", - "duration_ms": 43131.481, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What are the benefits of using Noveum Trace?", - "query.length": 44, - "processing.mode": "RAG", - "processing.time_seconds": 43.131428718566895, - "processing.response_length": 1415, - "processing.sources_count": 5, - "output_response": "Final Answer: The benefits of using Noveum Trace include:\n\n1. **Comprehensive Tracing**: Noveum Trace offers decorator-based tracing for LLM calls, agents, and RAG pipelines, allowing for detailed monitoring of AI ...", - "final_answer_mode": "RAG", - "query_processed.input_query": "What are the benefits of using Noveum Trace?", - "query_processed.output_response": "Successfully processed query using RAG, generated 1415 character response", - "query_processed.mode": "RAG", - "query_processed.processing_time": 43.131428718566895, - "query_processed.response_length": 1415 - }, - "events": [], - "links": [], - "trace_trace_id": "df8340b4-b75a-48e9-a2f3-99725b46a00d", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:49:17.236000000", - "trace_end_time": "2025-10-16 08:50:00.368000000", - "trace_duration_ms": 43131.554000000004, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:49:17.236000000", - "updated_at": "2025-10-16 08:49:17.236000000" - }, - { - "span_id": "fa93821f-c809-4071-87e1-a6c55014d37c", - "trace_id": "df8340b4-b75a-48e9-a2f3-99725b46a00d", - "parent_span_id": "dcf2b38e-ab90-45ee-9a65-e6fea47a3fd7", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-16 08:49:17.236000000", - "end_time": "2025-10-16 08:49:17.236000000", - "duration_ms": 0.034, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What are the benefits of using Noveum Trace?", - "query_length": 44, - "query_type": "routing_evaluation_query", - "classification.mode": "RAG", - "classification.rag_keyword_score": 2, - "classification.web_keyword_score": 0, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "noveum_specific", - "query.keyword_density": 0.25, - "query.domain_affinity": "noveum", - "routing.decision": "Routed to RAG based on analysis", - "routing.rationale": "RAG score: 2, Web score: 0, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "Web Search", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.038461538461538464, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: RAG (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "df8340b4-b75a-48e9-a2f3-99725b46a00d", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:49:17.236000000", - "trace_end_time": "2025-10-16 08:50:00.368000000", - "trace_duration_ms": 43131.554000000004, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:49:17.236000000", - "updated_at": "2025-10-16 08:49:17.236000000" - }, - { - "span_id": "024ad5aa-f394-4c26-af51-564f3f51c6f2", - "trace_id": "df8340b4-b75a-48e9-a2f3-99725b46a00d", - "parent_span_id": "dcf2b38e-ab90-45ee-9a65-e6fea47a3fd7", - "name": "agent.llm-rag", - "start_time": "2025-10-16 08:49:17.237000000", - "end_time": "2025-10-16 08:50:00.368000000", - "duration_ms": 43131.19300000001, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "rag_agent", - "agent.operation": "llm-rag", - "agent.capabilities": [ - "document_retrieval", - "context_generation", - "response_generation" - ], - "agent.id": "noveum_rag_agent", - "input_query": "What are the benefits of using Noveum Trace?", - "query_length": 44, - "query_type": "rag_query", - "output_response": "RAG Answer: The benefits of using Noveum Trace include:\n\n1. **Comprehensive Tracing**: Noveum Trace offers decorator-based tracing for LLM calls, agents, and RAG pipelines, allowing for detailed monitoring of AI ...", - "rag.context_length": 2773, - "rag.sources_count": 5, - "rag.answer_length": 1415, - "rag.mode": "retrieval_augmented_generation" - }, - "events": [], - "links": [], - "trace_trace_id": "df8340b4-b75a-48e9-a2f3-99725b46a00d", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:49:17.236000000", - "trace_end_time": "2025-10-16 08:50:00.368000000", - "trace_duration_ms": 43131.554000000004, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:49:17.236000000", - "updated_at": "2025-10-16 08:49:17.236000000" - }, - { - "span_id": "3d9db951-d8b0-4990-820d-61e4b1011709", - "trace_id": "df8340b4-b75a-48e9-a2f3-99725b46a00d", - "parent_span_id": "024ad5aa-f394-4c26-af51-564f3f51c6f2", - "name": "agent.llm_model_execution", - "start_time": "2025-10-16 08:49:20.224000000", - "end_time": "2025-10-16 08:49:58.420000000", - "duration_ms": 38196.111000000004, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What are the benefits of using Noveum Trace?", - "query_length": 44, - "query_type": "rag_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 38.19604015350342, - "model.latency_ms": 38196.04015350342, - "model.start_time": 1760604560.2242246, - "model.end_time": 1760604598.4202647, - "model.performance_tier": "slow", - "model.prompt_tokens": 828, - "model.completion_tokens": 353, - "model.total_tokens": 1181, - "model.tokens_per_second": 30.919435503098253, - "model.estimated_cost": 0.03543, - "model.efficiency_score": 1.1981371718882303, - "model.response_length": 1415, - "model.response_quality": "high", - "model.output_response": "Model Response: The benefits of using Noveum Trace include:\n\n1. **Comprehensive Tracing**: Noveum Trace offers decorator-based tracing for LLM calls, agents, and RAG pipelines, allowing for detailed monitoring of AI ...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "df8340b4-b75a-48e9-a2f3-99725b46a00d", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:49:17.236000000", - "trace_end_time": "2025-10-16 08:50:00.368000000", - "trace_duration_ms": 43131.554000000004, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:49:17.236000000", - "updated_at": "2025-10-16 08:49:17.236000000" - }, - { - "span_id": "8c33ffac-4d76-42ba-9737-89596e136b3d", - "trace_id": "df8340b4-b75a-48e9-a2f3-99725b46a00d", - "parent_span_id": "024ad5aa-f394-4c26-af51-564f3f51c6f2", - "name": "agent.rag_evaluation_metrics", - "start_time": "2025-10-16 08:49:58.420000000", - "end_time": "2025-10-16 08:49:58.420000000", - "duration_ms": 0.037, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "rag_evaluation_metrics", - "agent.capabilities": [ - "retrieval_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What are the benefits of using Noveum Trace?", - "query_length": 44, - "query_type": "rag_evaluation_query", - "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/docs/platform/dashboard):\n- Bulk Export: Download multiple traces simultaneously\n- Comparative Analysis: Compare performance across multiple traces\nReal-Time Updates\n- Live Refresh: Automatic updates as new traces arrive\n- Connection Monitoring: Real-time status of your t...", - "retrieval.context_length": 2773, - "retrieval.sources_count": 5, - "retrieval.context_quality": "high", - "retrieval.effectiveness": 1, - "retrieval.context_utilization": 2.773, - "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/docs/platform/dashboard):\n- Bulk Export: Download multiple traces simultaneously\n- Comparative Analysis: Compare performance across multiple traces\nReal-Time Updates\n- Live Refresh: Automatic updates as new traces arrive\n- Connection Monitoring: Real-time status of your trace ingestion pipeline\n- Performance Indicators: Live metrics for system health monitoring\nReady to dive deeper? Explore Projects & Environments to organize your AI applications, or check out Team Collaboration to share insights with your team.\nGet Earl...\n\nSource 2 (https://noveum.ai/en/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\n\ud83d\udcd8 TypeScript SDK (\n@noveum/trace\n)- Framework integrations for Next.js, Express.js, and Hono\n- TypeScript-first design with full type safety\n- Universal compatibility (Node.js, Edge Runtime, browsers)\n-\n\ud83d\udcca Noveum Platform\n- Real-time dashboard for analyzing traces and performance\n- Ad...\n\nSource 3 (https://noveum.ai/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\n\ud83d\udcd8 TypeScript SDK (\n@noveum/trace\n)- Framework integrations for Next.js, Express.js, and Hono\n- TypeScript-first design with full type safety\n- Universal compatibility (Node.js, Edge Runtime, browsers)\n-\n\ud83d\udcca Noveum Platform\n- Real-time dashboard for analyzing traces and performance\n- Ad...\n\nSource 4 (https://noveum.ai/en/docs/getting-started/overview):\n- Advanced search & filtering across traces and spans\n- Cost analysis and optimization recommendations\n- Team collaboration with shared insights and alerts\n\ud83d\udd0d What Noveum Traces\nLLM Operations\n- Model calls across all providers (OpenAI, Anthropic, Google, etc.)\n- Token usage and cost calculation\n- Prompt engineering effectiveness\n- Response quality metrics\nRAG Pipelines\n- Document retrieval performance and relevance\n- Embedding generation costs and latency\n- Context assembly and prompt constructi...\n\nSource 5 (https://noveum.ai/docs/getting-started/overview):\n- Advanced search & filtering across traces and spans\n- Cost analysis and optimization recommendations\n- Team collaboration with shared insights and alerts\n\ud83d\udd0d What Noveum Traces\nLLM Operations\n- Model calls across all providers (OpenAI, Anthropic, Google, etc.)\n- Token usage and cost calculation\n- Prompt engineering effectiveness\n- Response quality metrics\nRAG Pipelines\n- Document retrieval performance and relevance\n- Embedding generation costs and latency\n- Context assembly and prompt constructi...\n\nUser Question: What are the benefits of using Noveum Trace?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", - "prompt.prompt_length": 3313, - "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/docs/platform/dashboard):\n- Bulk Export: Download multiple traces simultaneously\n- Comparative Analysis: Compare performance across multiple traces\nReal-Time Updates\n- Live...", - "prompt.instruction_following": "rag_optimized", - "response.answer_length": 1415, - "response.answer_completeness": "complete", - "response.response_quality": "high", - "response.source_citation": 5, - "response.context_utilization": 0.5102776776054815, - "output_response": "RAG Answer: The benefits of using Noveum Trace include:\n\n1. **Comprehensive Tracing**: Noveum Trace offers decorator-based tracing for LLM calls, agents, and RAG pipelines, allowing for detailed monitoring of AI ...", - "evaluation.retrieval_effectiveness": 1, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 5, - "evaluation.context_relevance": "high", - "evaluation.overall_quality": "high", - "evaluation.ready_for_production": true, - "rag.retrieval_strategy": "semantic_similarity", - "rag.vector_search_results": 5, - "rag.context_synthesis": "multi_source", - "rag.document_coverage": 1, - "rag.information_density": 0.5102776776054815 - }, - "events": [], - "links": [], - "trace_trace_id": "df8340b4-b75a-48e9-a2f3-99725b46a00d", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:49:17.236000000", - "trace_end_time": "2025-10-16 08:50:00.368000000", - "trace_duration_ms": 43131.554000000004, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:49:17.236000000", - "updated_at": "2025-10-16 08:49:17.236000000" - }, - { - "span_id": "5b1a7121-d0bd-4525-821d-7721c6fbbd70", - "trace_id": "325ec189-2876-4ebe-b33c-649491a5053e", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-16 08:49:11.192000000", - "end_time": "2025-10-16 08:49:16.232000000", - "duration_ms": 5039.753, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "How do I configure Noveum for my system?", - "query.length": 40, - "processing.mode": "RAG", - "processing.time_seconds": 5.039698600769043, - "processing.response_length": 712, - "processing.sources_count": 5, - "output_response": "Final Answer: To configure Noveum for your system, you can refer to the SDK Integration Guide available in the Noveum documentation. This guide will help you trace your AI applications with minimal code changes. Th...", - "final_answer_mode": "RAG", - "query_processed.input_query": "How do I configure Noveum for my system?", - "query_processed.output_response": "Successfully processed query using RAG, generated 712 character response", - "query_processed.mode": "RAG", - "query_processed.processing_time": 5.039698600769043, - "query_processed.response_length": 712 - }, - "events": [], - "links": [], - "trace_trace_id": "325ec189-2876-4ebe-b33c-649491a5053e", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:49:11.192000000", - "trace_end_time": "2025-10-16 08:49:16.232000000", - "trace_duration_ms": 5039.841, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:49:11.192000000", - "updated_at": "2025-10-16 08:49:11.192000000" - }, - { - "span_id": "761c92c6-d15a-4a8e-a16a-a365922f8d7c", - "trace_id": "325ec189-2876-4ebe-b33c-649491a5053e", - "parent_span_id": "5b1a7121-d0bd-4525-821d-7721c6fbbd70", - "name": "agent.query_routing", - "start_time": "2025-10-16 08:49:11.192000000", - "end_time": "2025-10-16 08:49:16.232000000", - "duration_ms": 5039.642, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "How do I configure Noveum for my system?", - "query_length": 40, - "query_type": "routing_query", - "output_response": "Routed to RAG for query processing", - "router.classification": "RAG", - "router.confidence_score": 1, - "router.rag_keyword_score": 2, - "router.web_keyword_score": 0, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "325ec189-2876-4ebe-b33c-649491a5053e", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:49:11.192000000", - "trace_end_time": "2025-10-16 08:49:16.232000000", - "trace_duration_ms": 5039.841, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:49:11.192000000", - "updated_at": "2025-10-16 08:49:11.192000000" - }, - { - "span_id": "8fcf969e-a577-45aa-9fe6-1870d9665189", - "trace_id": "325ec189-2876-4ebe-b33c-649491a5053e", - "parent_span_id": "761c92c6-d15a-4a8e-a16a-a365922f8d7c", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-16 08:49:11.192000000", - "end_time": "2025-10-16 08:49:11.192000000", - "duration_ms": 0.045000000000000005, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: How do I configure Noveum for my system?", - "query_length": 40, - "query_type": "routing_evaluation_query", - "classification.mode": "RAG", - "classification.rag_keyword_score": 2, - "classification.web_keyword_score": 0, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "noveum_specific", - "query.keyword_density": 0.25, - "query.domain_affinity": "noveum", - "routing.decision": "Routed to RAG based on analysis", - "routing.rationale": "RAG score: 2, Web score: 0, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "Web Search", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.038461538461538464, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: RAG (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "325ec189-2876-4ebe-b33c-649491a5053e", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:49:11.192000000", - "trace_end_time": "2025-10-16 08:49:16.232000000", - "trace_duration_ms": 5039.841, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:49:11.192000000", - "updated_at": "2025-10-16 08:49:11.192000000" - }, - { - "span_id": "9e5bbea4-cc68-4fe6-b3d0-c87852635ec9", - "trace_id": "325ec189-2876-4ebe-b33c-649491a5053e", - "parent_span_id": "761c92c6-d15a-4a8e-a16a-a365922f8d7c", - "name": "agent.llm-rag", - "start_time": "2025-10-16 08:49:11.192000000", - "end_time": "2025-10-16 08:49:16.232000000", - "duration_ms": 5039.39, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "rag_agent", - "agent.operation": "llm-rag", - "agent.capabilities": [ - "document_retrieval", - "context_generation", - "response_generation" - ], - "agent.id": "noveum_rag_agent", - "input_query": "How do I configure Noveum for my system?", - "query_length": 40, - "query_type": "rag_query", - "output_response": "RAG Answer: To configure Noveum for your system, you can refer to the SDK Integration Guide available in the Noveum documentation. This guide will help you trace your AI applications with minimal code changes. Th...", - "rag.context_length": 2237, - "rag.sources_count": 5, - "rag.answer_length": 712, - "rag.mode": "retrieval_augmented_generation" - }, - "events": [], - "links": [], - "trace_trace_id": "325ec189-2876-4ebe-b33c-649491a5053e", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:49:11.192000000", - "trace_end_time": "2025-10-16 08:49:16.232000000", - "trace_duration_ms": 5039.841, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:49:11.192000000", - "updated_at": "2025-10-16 08:49:11.192000000" - }, - { - "span_id": "5ee706ad-c4ae-4c0b-94d1-6b9c43d173b4", - "trace_id": "325ec189-2876-4ebe-b33c-649491a5053e", - "parent_span_id": "9e5bbea4-cc68-4fe6-b3d0-c87852635ec9", - "name": "agent.llm_model_execution", - "start_time": "2025-10-16 08:49:11.607000000", - "end_time": "2025-10-16 08:49:15.718000000", - "duration_ms": 4110.915, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: How do I configure Noveum for my system?", - "query_length": 40, - "query_type": "rag_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 4.110846042633057, - "model.latency_ms": 4110.846042633057, - "model.start_time": 1760604551.6074958, - "model.end_time": 1760604555.7183418, - "model.performance_tier": "medium", - "model.prompt_tokens": 693, - "model.completion_tokens": 178, - "model.total_tokens": 871, - "model.tokens_per_second": 211.87852596934326, - "model.estimated_cost": 0.02613, - "model.efficiency_score": 0.817451205510907, - "model.response_length": 712, - "model.response_quality": "high", - "model.output_response": "Model Response: To configure Noveum for your system, you can refer to the SDK Integration Guide available in the Noveum documentation. This guide will help you trace your AI applications with minimal code changes. Th...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "325ec189-2876-4ebe-b33c-649491a5053e", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:49:11.192000000", - "trace_end_time": "2025-10-16 08:49:16.232000000", - "trace_duration_ms": 5039.841, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:49:11.192000000", - "updated_at": "2025-10-16 08:49:11.192000000" - }, - { - "span_id": "d13bdeda-81af-400a-82ca-a5ef7e1e8a6d", - "trace_id": "325ec189-2876-4ebe-b33c-649491a5053e", - "parent_span_id": "9e5bbea4-cc68-4fe6-b3d0-c87852635ec9", - "name": "agent.rag_evaluation_metrics", - "start_time": "2025-10-16 08:49:15.718000000", - "end_time": "2025-10-16 08:49:15.718000000", - "duration_ms": 0.118, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "rag_evaluation_metrics", - "agent.capabilities": [ - "retrieval_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: How do I configure Noveum for my system?", - "query_length": 40, - "query_type": "rag_evaluation_query", - "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/docs):\nWelcome to Noveum.ai Docs\nComprehensive AI tracing and observability for LLM applications, RAG systems, and AI agents\nWelcome to the Noveum.ai documentation! Here you'll find everything you need to integrate, configure, and optimize your AI applications using ou...", - "retrieval.context_length": 2237, - "retrieval.sources_count": 5, - "retrieval.context_quality": "high", - "retrieval.effectiveness": 1, - "retrieval.context_utilization": 2.237, - "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/docs):\nWelcome to Noveum.ai Docs\nComprehensive AI tracing and observability for LLM applications, RAG systems, and AI agents\nWelcome to the Noveum.ai documentation! Here you'll find everything you need to integrate, configure, and optimize your AI applications using our comprehensive tracing and observability platform.\nWe provide powerful SDKs for Python and TypeScript that enable you to trace LLM calls, RAG pipelines, and multi-agent workflows with minimal code changes. Our documentation is organized ...\n\nSource 2 (https://noveum.ai/docs):\nWelcome to Noveum.ai Docs\nComprehensive AI tracing and observability for LLM applications, RAG systems, and AI agents\nWelcome to the Noveum.ai documentation! Here you'll find everything you need to integrate, configure, and optimize your AI applications using our comprehensive tracing and observability platform.\nWe provide powerful SDKs for Python and TypeScript that enable you to trace LLM calls, RAG pipelines, and multi-agent workflows with minimal code changes. Our documentation is organized ...\n\nSource 3 (https://noveum.ai/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nSource 4 (https://noveum.ai/en/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nSource 5 (https://noveum.ai/en/docs):\n- \ud83c\udf0d Universal: Works across frameworks, clouds, and deployment models\n\ud83e\udd1d Community & Support\n- \ud83d\udcac Discord Community: Join our Discord\n- \ud83d\udce7 Email Support: [email protected]\n- \ud83d\udc1b Bug Reports: GitHub Issues\n- \ud83d\udcd6 Knowledge Base: Help Center\nReady to get started? Head to our SDK Integration Guide to begin tracing your AI applications in under 5 minutes!\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability ...\n\nUser Question: How do I configure Noveum for my system?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", - "prompt.prompt_length": 2773, - "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/docs):\nWelcome to Noveum.ai Docs\nComprehensive AI tracing and observability for LLM applications, RAG systems, and AI agents\nWelcome to the Noveum.ai documentation! Here...", - "prompt.instruction_following": "rag_optimized", - "response.answer_length": 712, - "response.answer_completeness": "complete", - "response.response_quality": "high", - "response.source_citation": 5, - "response.context_utilization": 0.3182834152883326, - "output_response": "RAG Answer: To configure Noveum for your system, you can refer to the SDK Integration Guide available in the Noveum documentation. This guide will help you trace your AI applications with minimal code changes. Th...", - "evaluation.retrieval_effectiveness": 1, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 5, - "evaluation.context_relevance": "high", - "evaluation.overall_quality": "high", - "evaluation.ready_for_production": true, - "rag.retrieval_strategy": "semantic_similarity", - "rag.vector_search_results": 5, - "rag.context_synthesis": "multi_source", - "rag.document_coverage": 1, - "rag.information_density": 0.3182834152883326 - }, - "events": [], - "links": [], - "trace_trace_id": "325ec189-2876-4ebe-b33c-649491a5053e", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:49:11.192000000", - "trace_end_time": "2025-10-16 08:49:16.232000000", - "trace_duration_ms": 5039.841, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:49:11.192000000", - "updated_at": "2025-10-16 08:49:11.192000000" - }, - { - "span_id": "7d6564fc-41b2-4d74-acb1-e77b575c0006", - "trace_id": "d5528586-c741-4703-8791-5fccf5c3e123", - "parent_span_id": "e79a037e-82b1-41e9-bdc7-42353e98ead7", - "name": "agent.query_routing", - "start_time": "2025-10-16 08:49:05.152000000", - "end_time": "2025-10-16 08:49:10.187000000", - "duration_ms": 5034.446, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What monitoring capabilities does Noveum provide?", - "query_length": 49, - "query_type": "routing_query", - "output_response": "Routed to RAG for query processing", - "router.classification": "RAG", - "router.confidence_score": 1, - "router.rag_keyword_score": 2, - "router.web_keyword_score": 0, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "d5528586-c741-4703-8791-5fccf5c3e123", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:49:05.152000000", - "trace_end_time": "2025-10-16 08:49:10.187000000", - "trace_duration_ms": 5034.604, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:49:05.152000000", - "updated_at": "2025-10-16 08:49:05.152000000" - }, - { - "span_id": "e79a037e-82b1-41e9-bdc7-42353e98ead7", - "trace_id": "d5528586-c741-4703-8791-5fccf5c3e123", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-16 08:49:05.152000000", - "end_time": "2025-10-16 08:49:10.187000000", - "duration_ms": 5034.531, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What monitoring capabilities does Noveum provide?", - "query.length": 49, - "processing.mode": "RAG", - "processing.time_seconds": 5.0344908237457275, - "processing.response_length": 552, - "processing.sources_count": 5, - "output_response": "Final Answer: Noveum provides comprehensive monitoring capabilities for AI applications, including the ability to monitor, trace, and optimize AI agents across various frameworks such as LangChain, CrewAI, AutoGen,...", - "final_answer_mode": "RAG", - "query_processed.input_query": "What monitoring capabilities does Noveum provide?", - "query_processed.output_response": "Successfully processed query using RAG, generated 552 character response", - "query_processed.mode": "RAG", - "query_processed.processing_time": 5.0344908237457275, - "query_processed.response_length": 552 - }, - "events": [], - "links": [], - "trace_trace_id": "d5528586-c741-4703-8791-5fccf5c3e123", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:49:05.152000000", - "trace_end_time": "2025-10-16 08:49:10.187000000", - "trace_duration_ms": 5034.604, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:49:05.152000000", - "updated_at": "2025-10-16 08:49:05.152000000" - }, - { - "span_id": "78297793-d73b-4311-b5ba-73509ae9f96e", - "trace_id": "d5528586-c741-4703-8791-5fccf5c3e123", - "parent_span_id": "7d6564fc-41b2-4d74-acb1-e77b575c0006", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-16 08:49:05.153000000", - "end_time": "2025-10-16 08:49:05.153000000", - "duration_ms": 0.029, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What monitoring capabilities does Noveum provide?", - "query_length": 49, - "query_type": "routing_evaluation_query", - "classification.mode": "RAG", - "classification.rag_keyword_score": 2, - "classification.web_keyword_score": 0, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "noveum_specific", - "query.keyword_density": 0.3333333333333333, - "query.domain_affinity": "noveum", - "routing.decision": "Routed to RAG based on analysis", - "routing.rationale": "RAG score: 2, Web score: 0, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "Web Search", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.038461538461538464, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: RAG (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "d5528586-c741-4703-8791-5fccf5c3e123", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:49:05.152000000", - "trace_end_time": "2025-10-16 08:49:10.187000000", - "trace_duration_ms": 5034.604, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:49:05.152000000", - "updated_at": "2025-10-16 08:49:05.152000000" - }, - { - "span_id": "6a101d9d-de83-415a-8fb0-01a9fae5ffb9", - "trace_id": "d5528586-c741-4703-8791-5fccf5c3e123", - "parent_span_id": "7d6564fc-41b2-4d74-acb1-e77b575c0006", - "name": "agent.llm-rag", - "start_time": "2025-10-16 08:49:05.153000000", - "end_time": "2025-10-16 08:49:10.187000000", - "duration_ms": 5034.282, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "rag_agent", - "agent.operation": "llm-rag", - "agent.capabilities": [ - "document_retrieval", - "context_generation", - "response_generation" - ], - "agent.id": "noveum_rag_agent", - "input_query": "What monitoring capabilities does Noveum provide?", - "query_length": 49, - "query_type": "rag_query", - "output_response": "RAG Answer: Noveum provides comprehensive monitoring capabilities for AI applications, including the ability to monitor, trace, and optimize AI agents across various frameworks such as LangChain, CrewAI, AutoGen,...", - "rag.context_length": 2740, - "rag.sources_count": 5, - "rag.answer_length": 552, - "rag.mode": "retrieval_augmented_generation" - }, - "events": [], - "links": [], - "trace_trace_id": "d5528586-c741-4703-8791-5fccf5c3e123", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:49:05.152000000", - "trace_end_time": "2025-10-16 08:49:10.187000000", - "trace_duration_ms": 5034.604, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:49:05.152000000", - "updated_at": "2025-10-16 08:49:05.152000000" - }, - { - "span_id": "26e510c0-441a-4f9f-9dce-31180f47d060", - "trace_id": "d5528586-c741-4703-8791-5fccf5c3e123", - "parent_span_id": "6a101d9d-de83-415a-8fb0-01a9fae5ffb9", - "name": "agent.llm_model_execution", - "start_time": "2025-10-16 08:49:05.787000000", - "end_time": "2025-10-16 08:49:09.274000000", - "duration_ms": 3486.699, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What monitoring capabilities does Noveum provide?", - "query_length": 49, - "query_type": "rag_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 3.4866299629211426, - "model.latency_ms": 3486.6299629211426, - "model.start_time": 1760604545.787763, - "model.end_time": 1760604549.274393, - "model.performance_tier": "medium", - "model.prompt_tokens": 821, - "model.completion_tokens": 138, - "model.total_tokens": 959, - "model.tokens_per_second": 275.0506965747916, - "model.estimated_cost": 0.02877, - "model.efficiency_score": 0.5755995828988529, - "model.response_length": 552, - "model.response_quality": "high", - "model.output_response": "Model Response: Noveum provides comprehensive monitoring capabilities for AI applications, including the ability to monitor, trace, and optimize AI agents across various frameworks such as LangChain, CrewAI, AutoGen,...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "d5528586-c741-4703-8791-5fccf5c3e123", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:49:05.152000000", - "trace_end_time": "2025-10-16 08:49:10.187000000", - "trace_duration_ms": 5034.604, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:49:05.152000000", - "updated_at": "2025-10-16 08:49:05.152000000" - }, - { - "span_id": "fcd89e38-3d21-468b-860a-ae2de0bbd119", - "trace_id": "d5528586-c741-4703-8791-5fccf5c3e123", - "parent_span_id": "6a101d9d-de83-415a-8fb0-01a9fae5ffb9", - "name": "agent.rag_evaluation_metrics", - "start_time": "2025-10-16 08:49:09.274000000", - "end_time": "2025-10-16 08:49:09.274000000", - "duration_ms": 0.054, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "rag_evaluation_metrics", - "agent.capabilities": [ - "retrieval_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What monitoring capabilities does Noveum provide?", - "query_length": 49, - "query_type": "rag_evaluation_query", - "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en):\nMonitor all your AI Agents\nimprove AI Agents today\nNoveum.ai helps you monitor, trace, and optimize your AI applications.\nNoveum.ai works with any AI framework \u2013 LangChain, CrewAI, AutoGen, custom implementations, or direct LLM calls. One dashboard shows everything.\n...", - "retrieval.context_length": 2740, - "retrieval.sources_count": 5, - "retrieval.context_quality": "high", - "retrieval.effectiveness": 1, - "retrieval.context_utilization": 2.74, - "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en):\nMonitor all your AI Agents\nimprove AI Agents today\nNoveum.ai helps you monitor, trace, and optimize your AI applications.\nNoveum.ai works with any AI framework \u2013 LangChain, CrewAI, AutoGen, custom implementations, or direct LLM calls. One dashboard shows everything.\nMonitor, Evaluate, Improve Your AI Agents\nThe control plane for AI agents.\nMonitor Everything, Miss Nothing\nOur lightweight SDKs capture every trace and span across your AI agent ecosystem\u2014from simple LLM calls to complex multi-agent...\n\nSource 2 (https://noveum.ai/):\nMonitor all your AI Agents\nimprove AI Agents today\nNoveum.ai helps you monitor, trace, and optimize your AI applications.\nNoveum.ai works with any AI framework \u2013 LangChain, CrewAI, AutoGen, custom implementations, or direct LLM calls. One dashboard shows everything.\nMonitor, Evaluate, Improve Your AI Agents\nThe control plane for AI agents.\nMonitor Everything, Miss Nothing\nOur lightweight SDKs capture every trace and span across your AI agent ecosystem\u2014from simple LLM calls to complex multi-agent...\n\nSource 3 (https://noveum.ai):\nMonitor all your AI Agents\nimprove AI Agents today\nNoveum.ai helps you monitor, trace, and optimize your AI applications.\nNoveum.ai works with any AI framework \u2013 LangChain, CrewAI, AutoGen, custom implementations, or direct LLM calls. One dashboard shows everything.\nMonitor, Evaluate, Improve Your AI Agents\nThe control plane for AI agents.\nMonitor Everything, Miss Nothing\nOur lightweight SDKs capture every trace and span across your AI agent ecosystem\u2014from simple LLM calls to complex multi-agent...\n\nSource 4 (https://noveum.ai/en/docs/getting-started/overview):\nNoveum.ai Overview\nComprehensive AI tracing and observability platform for LLM applications, RAG systems, and AI agents\nWelcome to Noveum.ai\u2014the comprehensive tracing and observability platform built specifically for AI applications. Whether you're building LLM-powered chatbots, RAG systems, multi-agent workflows, or any AI-driven application, Noveum provides the insights you need to understand, debug, and optimize your systems.\n\ud83c\udfaf Why AI Applications Need Specialized Observability\nTraditional mo...\n\nSource 5 (https://noveum.ai/docs/getting-started/overview):\nNoveum.ai Overview\nComprehensive AI tracing and observability platform for LLM applications, RAG systems, and AI agents\nWelcome to Noveum.ai\u2014the comprehensive tracing and observability platform built specifically for AI applications. Whether you're building LLM-powered chatbots, RAG systems, multi-agent workflows, or any AI-driven application, Noveum provides the insights you need to understand, debug, and optimize your systems.\n\ud83c\udfaf Why AI Applications Need Specialized Observability\nTraditional mo...\n\nUser Question: What monitoring capabilities does Noveum provide?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", - "prompt.prompt_length": 3285, - "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en):\nMonitor all your AI Agents\nimprove AI Agents today\nNoveum.ai helps you monitor, trace, and optimize your AI applications.\nNoveum.ai works with any AI framework \u2013 LangC...", - "prompt.instruction_following": "rag_optimized", - "response.answer_length": 552, - "response.answer_completeness": "complete", - "response.response_quality": "high", - "response.source_citation": 5, - "response.context_utilization": 0.20145985401459854, - "output_response": "RAG Answer: Noveum provides comprehensive monitoring capabilities for AI applications, including the ability to monitor, trace, and optimize AI agents across various frameworks such as LangChain, CrewAI, AutoGen,...", - "evaluation.retrieval_effectiveness": 1, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 5, - "evaluation.context_relevance": "high", - "evaluation.overall_quality": "high", - "evaluation.ready_for_production": true, - "rag.retrieval_strategy": "semantic_similarity", - "rag.vector_search_results": 5, - "rag.context_synthesis": "multi_source", - "rag.document_coverage": 1, - "rag.information_density": 0.20145985401459854 - }, - "events": [], - "links": [], - "trace_trace_id": "d5528586-c741-4703-8791-5fccf5c3e123", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:49:05.152000000", - "trace_end_time": "2025-10-16 08:49:10.187000000", - "trace_duration_ms": 5034.604, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:49:05.152000000", - "updated_at": "2025-10-16 08:49:05.152000000" - }, - { - "span_id": "bd5f205f-0329-4811-b883-129d69f12382", - "trace_id": "8100b869-07c0-4e37-b51e-092c9d20b4a7", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-16 08:48:57.574000000", - "end_time": "2025-10-16 08:49:04.149000000", - "duration_ms": 6574.424, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "How does Noveum handle agent tracing?", - "query.length": 37, - "processing.mode": "RAG", - "processing.time_seconds": 6.5743818283081055, - "processing.response_length": 643, - "processing.sources_count": 5, - "output_response": "Final Answer: Noveum.ai handles agent tracing through its specialized Multi-Agent Tracing capabilities, which are designed to observe complex workflows and inter-agent communications within multi-agent systems. The...", - "final_answer_mode": "RAG", - "query_processed.input_query": "How does Noveum handle agent tracing?", - "query_processed.output_response": "Successfully processed query using RAG, generated 643 character response", - "query_processed.mode": "RAG", - "query_processed.processing_time": 6.5743818283081055, - "query_processed.response_length": 643 - }, - "events": [], - "links": [], - "trace_trace_id": "8100b869-07c0-4e37-b51e-092c9d20b4a7", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:48:57.574000000", - "trace_end_time": "2025-10-16 08:49:04.149000000", - "trace_duration_ms": 6574.500999999999, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:48:57.574000000", - "updated_at": "2025-10-16 08:48:57.574000000" - }, - { - "span_id": "0801bcfe-8c09-4ea6-a754-5564395d6ded", - "trace_id": "8100b869-07c0-4e37-b51e-092c9d20b4a7", - "parent_span_id": "3c35d86a-0df9-42dc-bfe3-bbe978cf62b5", - "name": "agent.llm-rag", - "start_time": "2025-10-16 08:48:57.575000000", - "end_time": "2025-10-16 08:49:04.149000000", - "duration_ms": 6574.135, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "rag_agent", - "agent.operation": "llm-rag", - "agent.capabilities": [ - "document_retrieval", - "context_generation", - "response_generation" - ], - "agent.id": "noveum_rag_agent", - "input_query": "How does Noveum handle agent tracing?", - "query_length": 37, - "query_type": "rag_query", - "output_response": "RAG Answer: Noveum.ai handles agent tracing through its specialized Multi-Agent Tracing capabilities, which are designed to observe complex workflows and inter-agent communications within multi-agent systems. The...", - "rag.context_length": 2736, - "rag.sources_count": 5, - "rag.answer_length": 643, - "rag.mode": "retrieval_augmented_generation" - }, - "events": [], - "links": [], - "trace_trace_id": "8100b869-07c0-4e37-b51e-092c9d20b4a7", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:48:57.574000000", - "trace_end_time": "2025-10-16 08:49:04.149000000", - "trace_duration_ms": 6574.500999999999, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:48:57.574000000", - "updated_at": "2025-10-16 08:48:57.574000000" - }, - { - "span_id": "3c35d86a-0df9-42dc-bfe3-bbe978cf62b5", - "trace_id": "8100b869-07c0-4e37-b51e-092c9d20b4a7", - "parent_span_id": "bd5f205f-0329-4811-b883-129d69f12382", - "name": "agent.query_routing", - "start_time": "2025-10-16 08:48:57.575000000", - "end_time": "2025-10-16 08:49:04.149000000", - "duration_ms": 6574.342, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "How does Noveum handle agent tracing?", - "query_length": 37, - "query_type": "routing_query", - "output_response": "Routed to RAG for query processing", - "router.classification": "RAG", - "router.confidence_score": 1, - "router.rag_keyword_score": 2, - "router.web_keyword_score": 0, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "8100b869-07c0-4e37-b51e-092c9d20b4a7", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:48:57.574000000", - "trace_end_time": "2025-10-16 08:49:04.149000000", - "trace_duration_ms": 6574.500999999999, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:48:57.574000000", - "updated_at": "2025-10-16 08:48:57.574000000" - }, - { - "span_id": "b1914316-f545-4bc9-abf4-3494cf42c4a5", - "trace_id": "8100b869-07c0-4e37-b51e-092c9d20b4a7", - "parent_span_id": "3c35d86a-0df9-42dc-bfe3-bbe978cf62b5", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-16 08:48:57.575000000", - "end_time": "2025-10-16 08:48:57.575000000", - "duration_ms": 0.032, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: How does Noveum handle agent tracing?", - "query_length": 37, - "query_type": "routing_evaluation_query", - "classification.mode": "RAG", - "classification.rag_keyword_score": 2, - "classification.web_keyword_score": 0, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "noveum_specific", - "query.keyword_density": 0.3333333333333333, - "query.domain_affinity": "noveum", - "routing.decision": "Routed to RAG based on analysis", - "routing.rationale": "RAG score: 2, Web score: 0, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "Web Search", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.038461538461538464, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: RAG (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "8100b869-07c0-4e37-b51e-092c9d20b4a7", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:48:57.574000000", - "trace_end_time": "2025-10-16 08:49:04.149000000", - "trace_duration_ms": 6574.500999999999, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:48:57.574000000", - "updated_at": "2025-10-16 08:48:57.574000000" - }, - { - "span_id": "0906814b-6a80-40e1-9f00-5376b809703a", - "trace_id": "8100b869-07c0-4e37-b51e-092c9d20b4a7", - "parent_span_id": "0801bcfe-8c09-4ea6-a754-5564395d6ded", - "name": "agent.llm_model_execution", - "start_time": "2025-10-16 08:48:58.009000000", - "end_time": "2025-10-16 08:49:03.349000000", - "duration_ms": 5340.01, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: How does Noveum handle agent tracing?", - "query_length": 37, - "query_type": "rag_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 5.339954614639282, - "model.latency_ms": 5339.954614639282, - "model.start_time": 1760604538.0093303, - "model.end_time": 1760604543.349285, - "model.performance_tier": "slow", - "model.prompt_tokens": 817, - "model.completion_tokens": 160, - "model.total_tokens": 977, - "model.tokens_per_second": 182.9603565022054, - "model.estimated_cost": 0.02931, - "model.efficiency_score": 0.6581371545547595, - "model.response_length": 643, - "model.response_quality": "high", - "model.output_response": "Model Response: Noveum.ai handles agent tracing through its specialized Multi-Agent Tracing capabilities, which are designed to observe complex workflows and inter-agent communications within multi-agent systems. The...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "8100b869-07c0-4e37-b51e-092c9d20b4a7", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:48:57.574000000", - "trace_end_time": "2025-10-16 08:49:04.149000000", - "trace_duration_ms": 6574.500999999999, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:48:57.574000000", - "updated_at": "2025-10-16 08:48:57.574000000" - }, - { - "span_id": "29822edb-c15f-4baa-b6ec-84d85fcc926d", - "trace_id": "8100b869-07c0-4e37-b51e-092c9d20b4a7", - "parent_span_id": "0801bcfe-8c09-4ea6-a754-5564395d6ded", - "name": "agent.rag_evaluation_metrics", - "start_time": "2025-10-16 08:49:03.349000000", - "end_time": "2025-10-16 08:49:03.349000000", - "duration_ms": 0.034999999999999996, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "rag_evaluation_metrics", - "agent.capabilities": [ - "retrieval_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: How does Noveum handle agent tracing?", - "query_length": 37, - "query_type": "rag_evaluation_query", - "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, vector searches, data retrieval, agent reasoning, and more. Without proper observability, debugging ...", - "retrieval.context_length": 2736, - "retrieval.sources_count": 5, - "retrieval.context_quality": "high", - "retrieval.effectiveness": 1, - "retrieval.context_utilization": 2.736, - "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, vector searches, data retrieval, agent reasoning, and more. Without proper observability, debugging becomes a nightmare, optimization is guesswork, and understanding user interactions is nearly impossible.\nNoveum.ai solves this by providing comprehensive tracing and observability specifically designed for AI applications.\nKey benefits of using Noveum.ai:\n- Complete Visibility: Trace...\n\nSource 2 (https://noveum.ai/docs/advanced/multi-agent-tracing):\nMulti-Agent Tracing\nObserve complex agent workflows and inter-agent communications with comprehensive tracing\nMulti-agent systems represent some of the most complex AI applications, involving multiple agents that coordinate, communicate, and collaborate to achieve shared goals. Noveum.ai provides specialized tracing capabilities to help you understand and optimize these intricate workflows.\n\ud83c\udfaf Why Multi-Agent Tracing Matters\nMulti-agent systems introduce unique observability challenges:\n- Complex...\n\nSource 3 (https://noveum.ai/en/docs/advanced/multi-agent-tracing):\nMulti-Agent Tracing\nObserve complex agent workflows and inter-agent communications with comprehensive tracing\nMulti-agent systems represent some of the most complex AI applications, involving multiple agents that coordinate, communicate, and collaborate to achieve shared goals. Noveum.ai provides specialized tracing capabilities to help you understand and optimize these intricate workflows.\n\ud83c\udfaf Why Multi-Agent Tracing Matters\nMulti-agent systems introduce unique observability challenges:\n- Complex...\n\nSource 4 (https://noveum.ai/en/docs/advanced/performance-optimization):\n\ud83d\udd17 Next Steps\n- Multi-Agent Tracing - Observe agent workflows\n- RAG Pipeline Observability - Monitor retrieval and generation systems\n- Custom Instrumentation - Add domain-specific tracing\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nSource 5 (https://noveum.ai/docs/advanced/performance-optimization):\n\ud83d\udd17 Next Steps\n- Multi-Agent Tracing - Observe agent workflows\n- RAG Pipeline Observability - Monitor retrieval and generation systems\n- Custom Instrumentation - Add domain-specific tracing\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nUser Question: How does Noveum handle agent tracing?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", - "prompt.prompt_length": 3269, - "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, ...", - "prompt.instruction_following": "rag_optimized", - "response.answer_length": 643, - "response.answer_completeness": "complete", - "response.response_quality": "high", - "response.source_citation": 5, - "response.context_utilization": 0.23501461988304093, - "output_response": "RAG Answer: Noveum.ai handles agent tracing through its specialized Multi-Agent Tracing capabilities, which are designed to observe complex workflows and inter-agent communications within multi-agent systems. The...", - "evaluation.retrieval_effectiveness": 1, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 5, - "evaluation.context_relevance": "high", - "evaluation.overall_quality": "high", - "evaluation.ready_for_production": true, - "rag.retrieval_strategy": "semantic_similarity", - "rag.vector_search_results": 5, - "rag.context_synthesis": "multi_source", - "rag.document_coverage": 1, - "rag.information_density": 0.23501461988304093 - }, - "events": [], - "links": [], - "trace_trace_id": "8100b869-07c0-4e37-b51e-092c9d20b4a7", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:48:57.574000000", - "trace_end_time": "2025-10-16 08:49:04.149000000", - "trace_duration_ms": 6574.500999999999, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:48:57.574000000", - "updated_at": "2025-10-16 08:48:57.574000000" - }, - { - "span_id": "5f8ea007-edef-477c-b8f1-57c2da9e9258", - "trace_id": "c28bd31c-d200-4467-91fd-a3b80187c885", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-16 08:48:17.977000000", - "end_time": "2025-10-16 08:48:56.570000000", - "duration_ms": 38592.912, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What APIs are available in Noveum platform?", - "query.length": 43, - "processing.mode": "RAG", - "processing.time_seconds": 38.59272003173828, - "processing.response_length": 270, - "processing.sources_count": 5, - "output_response": "Final Answer: The provided context does not specify the APIs available in the Noveum platform. For detailed information about the APIs, I recommend checking the official Noveum documentation or the SDK Integration ...", - "final_answer_mode": "RAG", - "query_processed.input_query": "What APIs are available in Noveum platform?", - "query_processed.output_response": "Successfully processed query using RAG, generated 270 character response", - "query_processed.mode": "RAG", - "query_processed.processing_time": 38.59272003173828, - "query_processed.response_length": 270 - }, - "events": [], - "links": [], - "trace_trace_id": "c28bd31c-d200-4467-91fd-a3b80187c885", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:48:17.977000000", - "trace_end_time": "2025-10-16 08:48:56.570000000", - "trace_duration_ms": 38592.996, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:48:17.977000000", - "updated_at": "2025-10-16 08:48:17.977000000" - }, - { - "span_id": "db385f4d-beef-465b-9831-a65080537519", - "trace_id": "c28bd31c-d200-4467-91fd-a3b80187c885", - "parent_span_id": "5f8ea007-edef-477c-b8f1-57c2da9e9258", - "name": "agent.query_routing", - "start_time": "2025-10-16 08:48:17.977000000", - "end_time": "2025-10-16 08:48:56.570000000", - "duration_ms": 38592.66499999999, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What APIs are available in Noveum platform?", - "query_length": 43, - "query_type": "routing_query", - "output_response": "Routed to RAG for query processing", - "router.classification": "RAG", - "router.confidence_score": 1, - "router.rag_keyword_score": 3, - "router.web_keyword_score": 0, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "c28bd31c-d200-4467-91fd-a3b80187c885", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:48:17.977000000", - "trace_end_time": "2025-10-16 08:48:56.570000000", - "trace_duration_ms": 38592.996, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:48:17.977000000", - "updated_at": "2025-10-16 08:48:17.977000000" - }, - { - "span_id": "3ccebe1c-1f2a-433b-a36f-453a0b803511", - "trace_id": "c28bd31c-d200-4467-91fd-a3b80187c885", - "parent_span_id": "db385f4d-beef-465b-9831-a65080537519", - "name": "agent.llm-rag", - "start_time": "2025-10-16 08:48:17.978000000", - "end_time": "2025-10-16 08:48:56.570000000", - "duration_ms": 38592.416999999994, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "rag_agent", - "agent.operation": "llm-rag", - "agent.capabilities": [ - "document_retrieval", - "context_generation", - "response_generation" - ], - "agent.id": "noveum_rag_agent", - "input_query": "What APIs are available in Noveum platform?", - "query_length": 43, - "query_type": "rag_query", - "output_response": "RAG Answer: The provided context does not specify the APIs available in the Noveum platform. For detailed information about the APIs, I recommend checking the official Noveum documentation or the SDK Integration ...", - "rag.context_length": 2284, - "rag.sources_count": 5, - "rag.answer_length": 270, - "rag.mode": "retrieval_augmented_generation" - }, - "events": [], - "links": [], - "trace_trace_id": "c28bd31c-d200-4467-91fd-a3b80187c885", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:48:17.977000000", - "trace_end_time": "2025-10-16 08:48:56.570000000", - "trace_duration_ms": 38592.996, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:48:17.977000000", - "updated_at": "2025-10-16 08:48:17.977000000" - }, - { - "span_id": "6d1bb8e9-315d-433a-8df5-7f5fb2eeee16", - "trace_id": "c28bd31c-d200-4467-91fd-a3b80187c885", - "parent_span_id": "db385f4d-beef-465b-9831-a65080537519", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-16 08:48:17.978000000", - "end_time": "2025-10-16 08:48:17.978000000", - "duration_ms": 0.043000000000000003, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What APIs are available in Noveum platform?", - "query_length": 43, - "query_type": "routing_evaluation_query", - "classification.mode": "RAG", - "classification.rag_keyword_score": 3, - "classification.web_keyword_score": 0, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "noveum_specific", - "query.keyword_density": 0.42857142857142855, - "query.domain_affinity": "noveum", - "routing.decision": "Routed to RAG based on analysis", - "routing.rationale": "RAG score: 3, Web score: 0, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "Web Search", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.057692307692307696, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: RAG (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "c28bd31c-d200-4467-91fd-a3b80187c885", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:48:17.977000000", - "trace_end_time": "2025-10-16 08:48:56.570000000", - "trace_duration_ms": 38592.996, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:48:17.977000000", - "updated_at": "2025-10-16 08:48:17.977000000" - }, - { - "span_id": "2f11df8a-8f45-425d-a0ce-1eec7887b508", - "trace_id": "c28bd31c-d200-4467-91fd-a3b80187c885", - "parent_span_id": "3ccebe1c-1f2a-433b-a36f-453a0b803511", - "name": "agent.llm_model_execution", - "start_time": "2025-10-16 08:48:18.392000000", - "end_time": "2025-10-16 08:48:55.545000000", - "duration_ms": 37152.496999999996, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What APIs are available in Noveum platform?", - "query_length": 43, - "query_type": "rag_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 37.15243363380432, - "model.latency_ms": 37152.43363380432, - "model.start_time": 1760604498.3926067, - "model.end_time": 1760604535.5450404, - "model.performance_tier": "slow", - "model.prompt_tokens": 705, - "model.completion_tokens": 67, - "model.total_tokens": 772, - "model.tokens_per_second": 20.779257897592238, - "model.estimated_cost": 0.02316, - "model.efficiency_score": 0.34974093264248707, - "model.response_length": 270, - "model.response_quality": "high", - "model.output_response": "Model Response: The provided context does not specify the APIs available in the Noveum platform. For detailed information about the APIs, I recommend checking the official Noveum documentation or the SDK Integration ...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "c28bd31c-d200-4467-91fd-a3b80187c885", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:48:17.977000000", - "trace_end_time": "2025-10-16 08:48:56.570000000", - "trace_duration_ms": 38592.996, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:48:17.977000000", - "updated_at": "2025-10-16 08:48:17.977000000" - }, - { - "span_id": "d13e608c-06e4-47c4-b602-c40ac9560be2", - "trace_id": "c28bd31c-d200-4467-91fd-a3b80187c885", - "parent_span_id": "3ccebe1c-1f2a-433b-a36f-453a0b803511", - "name": "agent.rag_evaluation_metrics", - "start_time": "2025-10-16 08:48:55.545000000", - "end_time": "2025-10-16 08:48:55.545000000", - "duration_ms": 0.031, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "rag_evaluation_metrics", - "agent.capabilities": [ - "retrieval_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What APIs are available in Noveum platform?", - "query_length": 43, - "query_type": "rag_evaluation_query", - "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/docs):\n- \ud83c\udf0d Universal: Works across frameworks, clouds, and deployment models\n\ud83e\udd1d Community & Support\n- \ud83d\udcac Discord Community: Join our Discord\n- \ud83d\udce7 Email Support: [email protected]\n- \ud83d\udc1b Bug Reports: GitHub Issues\n- \ud83d\udcd6 Knowledge Base: Help Center\nReady to get started? Head to ...", - "retrieval.context_length": 2284, - "retrieval.sources_count": 5, - "retrieval.context_quality": "high", - "retrieval.effectiveness": 1, - "retrieval.context_utilization": 2.284, - "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/docs):\n- \ud83c\udf0d Universal: Works across frameworks, clouds, and deployment models\n\ud83e\udd1d Community & Support\n- \ud83d\udcac Discord Community: Join our Discord\n- \ud83d\udce7 Email Support: [email protected]\n- \ud83d\udc1b Bug Reports: GitHub Issues\n- \ud83d\udcd6 Knowledge Base: Help Center\nReady to get started? Head to our SDK Integration Guide to begin tracing your AI applications in under 5 minutes!\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability ...\n\nSource 2 (https://noveum.ai/docs):\n- \ud83c\udf0d Universal: Works across frameworks, clouds, and deployment models\n\ud83e\udd1d Community & Support\n- \ud83d\udcac Discord Community: Join our Discord\n- \ud83d\udce7 Email Support: [email protected]\n- \ud83d\udc1b Bug Reports: GitHub Issues\n- \ud83d\udcd6 Knowledge Base: Help Center\nReady to get started? Head to our SDK Integration Guide to begin tracing your AI applications in under 5 minutes!\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability ...\n\nSource 3 (https://noveum.ai/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nSource 4 (https://noveum.ai/en/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nSource 5 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, vector searches, data retrieval, agent reasoning, and more. Without proper observability, debugging becomes a nightmare, optimization is guesswork, and understanding user interactions is nearly impossible.\nNoveum.ai solves this by providing comprehensive tracing and observability specifically designed for AI applications.\nKey benefits of using Noveum.ai:\n- Complete Visibility: Trace...\n\nUser Question: What APIs are available in Noveum platform?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", - "prompt.prompt_length": 2823, - "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/docs):\n- \ud83c\udf0d Universal: Works across frameworks, clouds, and deployment models\n\ud83e\udd1d Community & Support\n- \ud83d\udcac Discord Community: Join our Discord\n- \ud83d\udce7 Email Support: [email prot...", - "prompt.instruction_following": "rag_optimized", - "response.answer_length": 270, - "response.answer_completeness": "complete", - "response.response_quality": "high", - "response.source_citation": 5, - "response.context_utilization": 0.11821366024518389, - "output_response": "RAG Answer: The provided context does not specify the APIs available in the Noveum platform. For detailed information about the APIs, I recommend checking the official Noveum documentation or the SDK Integration ...", - "evaluation.retrieval_effectiveness": 1, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 5, - "evaluation.context_relevance": "high", - "evaluation.overall_quality": "high", - "evaluation.ready_for_production": true, - "rag.retrieval_strategy": "semantic_similarity", - "rag.vector_search_results": 5, - "rag.context_synthesis": "multi_source", - "rag.document_coverage": 1, - "rag.information_density": 0.11821366024518389 - }, - "events": [], - "links": [], - "trace_trace_id": "c28bd31c-d200-4467-91fd-a3b80187c885", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:48:17.977000000", - "trace_end_time": "2025-10-16 08:48:56.570000000", - "trace_duration_ms": 38592.996, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:48:17.977000000", - "updated_at": "2025-10-16 08:48:17.977000000" - }, - { - "span_id": "3f3babbd-abf5-4f5a-b31c-fa5dcec9c360", - "trace_id": "b90ef3fc-11ea-4230-a59b-e2f2db8f2de0", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-16 08:48:08.007000000", - "end_time": "2025-10-16 08:48:16.975000000", - "duration_ms": 8967.201, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "How do I set up observability with Noveum?", - "query.length": 42, - "processing.mode": "RAG", - "processing.time_seconds": 8.96716833114624, - "processing.response_length": 838, - "processing.sources_count": 5, - "output_response": "Final Answer: To set up observability with Noveum, follow these steps:\n\n1. **Sign Up**: Go to [noveum.ai](https://noveum.ai) and create an account.\n2. **Create a Project**: Once signed in, create a new project to o...", - "final_answer_mode": "RAG", - "query_processed.input_query": "How do I set up observability with Noveum?", - "query_processed.output_response": "Successfully processed query using RAG, generated 838 character response", - "query_processed.mode": "RAG", - "query_processed.processing_time": 8.96716833114624, - "query_processed.response_length": 838 - }, - "events": [], - "links": [], - "trace_trace_id": "b90ef3fc-11ea-4230-a59b-e2f2db8f2de0", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:48:08.007000000", - "trace_end_time": "2025-10-16 08:48:16.975000000", - "trace_duration_ms": 8967.264, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:48:08.007000000", - "updated_at": "2025-10-16 08:48:08.007000000" - }, - { - "span_id": "41f6453c-4be6-4b66-8f9b-8e795f7b41b0", - "trace_id": "b90ef3fc-11ea-4230-a59b-e2f2db8f2de0", - "parent_span_id": "b0a76345-49e5-4c1b-915f-cec90ea2b1f7", - "name": "agent.llm-rag", - "start_time": "2025-10-16 08:48:08.007000000", - "end_time": "2025-10-16 08:48:16.974000000", - "duration_ms": 8966.964, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "rag_agent", - "agent.operation": "llm-rag", - "agent.capabilities": [ - "document_retrieval", - "context_generation", - "response_generation" - ], - "agent.id": "noveum_rag_agent", - "input_query": "How do I set up observability with Noveum?", - "query_length": 42, - "query_type": "rag_query", - "output_response": "RAG Answer: To set up observability with Noveum, follow these steps:\n\n1. **Sign Up**: Go to [noveum.ai](https://noveum.ai) and create an account.\n2. **Create a Project**: Once signed in, create a new project to o...", - "rag.context_length": 2870, - "rag.sources_count": 6, - "rag.answer_length": 838, - "rag.mode": "retrieval_augmented_generation" - }, - "events": [], - "links": [], - "trace_trace_id": "b90ef3fc-11ea-4230-a59b-e2f2db8f2de0", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:48:08.007000000", - "trace_end_time": "2025-10-16 08:48:16.975000000", - "trace_duration_ms": 8967.264, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:48:08.007000000", - "updated_at": "2025-10-16 08:48:08.007000000" - }, - { - "span_id": "b0a76345-49e5-4c1b-915f-cec90ea2b1f7", - "trace_id": "b90ef3fc-11ea-4230-a59b-e2f2db8f2de0", - "parent_span_id": "3f3babbd-abf5-4f5a-b31c-fa5dcec9c360", - "name": "agent.query_routing", - "start_time": "2025-10-16 08:48:08.007000000", - "end_time": "2025-10-16 08:48:16.974000000", - "duration_ms": 8967.129, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "How do I set up observability with Noveum?", - "query_length": 42, - "query_type": "routing_query", - "output_response": "Routed to RAG for query processing", - "router.classification": "RAG", - "router.confidence_score": 1, - "router.rag_keyword_score": 2, - "router.web_keyword_score": 0, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "b90ef3fc-11ea-4230-a59b-e2f2db8f2de0", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:48:08.007000000", - "trace_end_time": "2025-10-16 08:48:16.975000000", - "trace_duration_ms": 8967.264, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:48:08.007000000", - "updated_at": "2025-10-16 08:48:08.007000000" - }, - { - "span_id": "fe3074a7-f4c0-442e-9a5b-e2cc11d93785", - "trace_id": "b90ef3fc-11ea-4230-a59b-e2f2db8f2de0", - "parent_span_id": "b0a76345-49e5-4c1b-915f-cec90ea2b1f7", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-16 08:48:08.007000000", - "end_time": "2025-10-16 08:48:08.007000000", - "duration_ms": 0.032, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: How do I set up observability with Noveum?", - "query_length": 42, - "query_type": "routing_evaluation_query", - "classification.mode": "RAG", - "classification.rag_keyword_score": 2, - "classification.web_keyword_score": 0, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "noveum_specific", - "query.keyword_density": 0.25, - "query.domain_affinity": "noveum", - "routing.decision": "Routed to RAG based on analysis", - "routing.rationale": "RAG score: 2, Web score: 0, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "Web Search", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.038461538461538464, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: RAG (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "b90ef3fc-11ea-4230-a59b-e2f2db8f2de0", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:48:08.007000000", - "trace_end_time": "2025-10-16 08:48:16.975000000", - "trace_duration_ms": 8967.264, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:48:08.007000000", - "updated_at": "2025-10-16 08:48:08.007000000" - }, - { - "span_id": "e16139ee-048e-4c2f-87f8-90442f41a0ae", - "trace_id": "b90ef3fc-11ea-4230-a59b-e2f2db8f2de0", - "parent_span_id": "41f6453c-4be6-4b66-8f9b-8e795f7b41b0", - "name": "agent.llm_model_execution", - "start_time": "2025-10-16 08:48:08.644000000", - "end_time": "2025-10-16 08:48:15.709000000", - "duration_ms": 7064.898, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: How do I set up observability with Noveum?", - "query_length": 42, - "query_type": "rag_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 7.0648510456085205, - "model.latency_ms": 7064.8510456085205, - "model.start_time": 1760604488.6448367, - "model.end_time": 1760604495.7096877, - "model.performance_tier": "slow", - "model.prompt_tokens": 852, - "model.completion_tokens": 209, - "model.total_tokens": 1061, - "model.tokens_per_second": 150.1800948315128, - "model.estimated_cost": 0.031830000000000004, - "model.efficiency_score": 0.7898209236569275, - "model.response_length": 838, - "model.response_quality": "high", - "model.output_response": "Model Response: To set up observability with Noveum, follow these steps:\n\n1. **Sign Up**: Go to [noveum.ai](https://noveum.ai) and create an account.\n2. **Create a Project**: Once signed in, create a new project to o...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "b90ef3fc-11ea-4230-a59b-e2f2db8f2de0", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:48:08.007000000", - "trace_end_time": "2025-10-16 08:48:16.975000000", - "trace_duration_ms": 8967.264, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:48:08.007000000", - "updated_at": "2025-10-16 08:48:08.007000000" - }, - { - "span_id": "59e85f8e-c19b-4794-9a69-7b5bc90208f0", - "trace_id": "b90ef3fc-11ea-4230-a59b-e2f2db8f2de0", - "parent_span_id": "41f6453c-4be6-4b66-8f9b-8e795f7b41b0", - "name": "agent.rag_evaluation_metrics", - "start_time": "2025-10-16 08:48:15.709000000", - "end_time": "2025-10-16 08:48:15.709000000", - "duration_ms": 0.039, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "rag_evaluation_metrics", - "agent.capabilities": [ - "retrieval_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: How do I set up observability with Noveum?", - "query_length": 42, - "query_type": "rag_evaluation_query", - "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\n- Production Ready: Built for scale with intelligent sampling and batching\n- Privacy Focused: You control what data is captured and how it's stored\n- Open Source SDKs: Transparent, auditable, and extensible\nGetting ...", - "retrieval.context_length": 2870, - "retrieval.sources_count": 6, - "retrieval.context_quality": "high", - "retrieval.effectiveness": 1.2, - "retrieval.context_utilization": 2.87, - "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\n- Production Ready: Built for scale with intelligent sampling and batching\n- Privacy Focused: You control what data is captured and how it's stored\n- Open Source SDKs: Transparent, auditable, and extensible\nGetting Started\nReady to add comprehensive observability to your AI applications?\n- Sign up at noveum.ai\n- Create a project and get your API key\n- Install the SDK for your preferred language\n- Add tracing to your AI workflows\n- Explore insights in the Noveum.ai dashboard\nCheck out our integra...\n\nSource 2 (https://noveum.ai/en/docs/getting-started/tracing-concepts):\n- Explore Framework Integrations - Framework-specific guidance\n- Learn Advanced Patterns - Custom instrumentation techniques\n- Master the Dashboard - Analyze your traces effectively\nRemember: Good observability is not about collecting all possible data, but about collecting the right data that helps you understand, debug, and optimize your AI applications.\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to O...\n\nSource 3 (https://noveum.ai/docs/getting-started/tracing-concepts):\n- Explore Framework Integrations - Framework-specific guidance\n- Learn Advanced Patterns - Custom instrumentation techniques\n- Master the Dashboard - Analyze your traces effectively\nRemember: Good observability is not about collecting all possible data, but about collecting the right data that helps you understand, debug, and optimize your AI applications.\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to O...\n\nSource 4 (https://noveum.ai/en/docs/getting-started/overview):\nNoveum.ai Overview\nComprehensive AI tracing and observability platform for LLM applications, RAG systems, and AI agents\nWelcome to Noveum.ai\u2014the comprehensive tracing and observability platform built specifically for AI applications. Whether you're building LLM-powered chatbots, RAG systems, multi-agent workflows, or any AI-driven application, Noveum provides the insights you need to understand, debug, and optimize your systems.\n\ud83c\udfaf Why AI Applications Need Specialized Observability\nTraditional mo...\n\nSource 5 (https://noveum.ai/docs/getting-started/overview):\nNoveum.ai Overview\nComprehensive AI tracing and observability platform for LLM applications, RAG systems, and AI agents\nWelcome to Noveum.ai\u2014the comprehensive tracing and observability platform built specifically for AI applications. Whether you're building LLM-powered chatbots, RAG systems, multi-agent workflows, or any AI-driven application, Noveum provides the insights you need to understand, debug, and optimize your systems.\n\ud83c\udfaf Why AI Applications Need Specialized Observability\nTraditional mo...\n\nUser Question: How do I set up observability with Noveum?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", - "prompt.prompt_length": 3408, - "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\n- Production Ready: Built for scale with intelligent sampling and batching\n- Privacy Focused: You control what data...", - "prompt.instruction_following": "rag_optimized", - "response.answer_length": 838, - "response.answer_completeness": "complete", - "response.response_quality": "high", - "response.source_citation": 6, - "response.context_utilization": 0.29198606271777006, - "output_response": "RAG Answer: To set up observability with Noveum, follow these steps:\n\n1. **Sign Up**: Go to [noveum.ai](https://noveum.ai) and create an account.\n2. **Create a Project**: Once signed in, create a new project to o...", - "evaluation.retrieval_effectiveness": 1.2, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 6, - "evaluation.context_relevance": "high", - "evaluation.overall_quality": "high", - "evaluation.ready_for_production": true, - "rag.retrieval_strategy": "semantic_similarity", - "rag.vector_search_results": 6, - "rag.context_synthesis": "multi_source", - "rag.document_coverage": 1.2, - "rag.information_density": 0.29198606271777006 - }, - "events": [], - "links": [], - "trace_trace_id": "b90ef3fc-11ea-4230-a59b-e2f2db8f2de0", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:48:08.007000000", - "trace_end_time": "2025-10-16 08:48:16.975000000", - "trace_duration_ms": 8967.264, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:48:08.007000000", - "updated_at": "2025-10-16 08:48:08.007000000" - }, - { - "span_id": "f049ed2b-ff2f-4092-9974-acc66f81bd8d", - "trace_id": "b8abebac-a7dc-494c-a1eb-6470181f32d2", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-16 08:47:57.667000000", - "end_time": "2025-10-16 08:48:07.005000000", - "duration_ms": 9337.298, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What features does Noveum Trace offer?", - "query.length": 38, - "processing.mode": "RAG", - "processing.time_seconds": 9.33726453781128, - "processing.response_length": 1018, - "processing.sources_count": 5, - "output_response": "Final Answer: Noveum Trace offers several key features through its Python SDK (`noveum-trace`) and TypeScript SDK (`@noveum/trace`):\n\n1. **Decorator-Based Tracing**: It provides a decorator-based approach for traci...", - "final_answer_mode": "RAG", - "query_processed.input_query": "What features does Noveum Trace offer?", - "query_processed.output_response": "Successfully processed query using RAG, generated 1018 character response", - "query_processed.mode": "RAG", - "query_processed.processing_time": 9.33726453781128, - "query_processed.response_length": 1018 - }, - "events": [], - "links": [], - "trace_trace_id": "b8abebac-a7dc-494c-a1eb-6470181f32d2", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:47:57.667000000", - "trace_end_time": "2025-10-16 08:48:07.005000000", - "trace_duration_ms": 9337.405, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:47:57.667000000", - "updated_at": "2025-10-16 08:47:57.667000000" - }, - { - "span_id": "211f1fe7-4cfb-4c1c-9c72-62ae2825dda3", - "trace_id": "b8abebac-a7dc-494c-a1eb-6470181f32d2", - "parent_span_id": "f049ed2b-ff2f-4092-9974-acc66f81bd8d", - "name": "agent.query_routing", - "start_time": "2025-10-16 08:47:57.668000000", - "end_time": "2025-10-16 08:48:07.005000000", - "duration_ms": 9337.223, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What features does Noveum Trace offer?", - "query_length": 38, - "query_type": "routing_query", - "output_response": "Routed to RAG for query processing", - "router.classification": "RAG", - "router.confidence_score": 1, - "router.rag_keyword_score": 3, - "router.web_keyword_score": 0, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "b8abebac-a7dc-494c-a1eb-6470181f32d2", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:47:57.667000000", - "trace_end_time": "2025-10-16 08:48:07.005000000", - "trace_duration_ms": 9337.405, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:47:57.667000000", - "updated_at": "2025-10-16 08:47:57.667000000" - }, - { - "span_id": "976c0c4b-efc1-4b7d-bfad-350c7fd0f71e", - "trace_id": "b8abebac-a7dc-494c-a1eb-6470181f32d2", - "parent_span_id": "211f1fe7-4cfb-4c1c-9c72-62ae2825dda3", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-16 08:47:57.668000000", - "end_time": "2025-10-16 08:47:57.668000000", - "duration_ms": 0.028, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What features does Noveum Trace offer?", - "query_length": 38, - "query_type": "routing_evaluation_query", - "classification.mode": "RAG", - "classification.rag_keyword_score": 3, - "classification.web_keyword_score": 0, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "noveum_specific", - "query.keyword_density": 0.5, - "query.domain_affinity": "noveum", - "routing.decision": "Routed to RAG based on analysis", - "routing.rationale": "RAG score: 3, Web score: 0, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "Web Search", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.057692307692307696, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: RAG (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "b8abebac-a7dc-494c-a1eb-6470181f32d2", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:47:57.667000000", - "trace_end_time": "2025-10-16 08:48:07.005000000", - "trace_duration_ms": 9337.405, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:47:57.667000000", - "updated_at": "2025-10-16 08:47:57.667000000" - }, - { - "span_id": "d2afaf74-f63b-4141-9fa3-662162f51cc0", - "trace_id": "b8abebac-a7dc-494c-a1eb-6470181f32d2", - "parent_span_id": "211f1fe7-4cfb-4c1c-9c72-62ae2825dda3", - "name": "agent.llm-rag", - "start_time": "2025-10-16 08:47:57.668000000", - "end_time": "2025-10-16 08:48:07.005000000", - "duration_ms": 9337.056, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "rag_agent", - "agent.operation": "llm-rag", - "agent.capabilities": [ - "document_retrieval", - "context_generation", - "response_generation" - ], - "agent.id": "noveum_rag_agent", - "input_query": "What features does Noveum Trace offer?", - "query_length": 38, - "query_type": "rag_query", - "output_response": "RAG Answer: Noveum Trace offers several key features through its Python SDK (`noveum-trace`) and TypeScript SDK (`@noveum/trace`):\n\n1. **Decorator-Based Tracing**: It provides a decorator-based approach for traci...", - "rag.context_length": 2787, - "rag.sources_count": 5, - "rag.answer_length": 1018, - "rag.mode": "retrieval_augmented_generation" - }, - "events": [], - "links": [], - "trace_trace_id": "b8abebac-a7dc-494c-a1eb-6470181f32d2", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:47:57.667000000", - "trace_end_time": "2025-10-16 08:48:07.005000000", - "trace_duration_ms": 9337.405, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:47:57.667000000", - "updated_at": "2025-10-16 08:47:57.667000000" - }, - { - "span_id": "09811a73-405d-4c0d-ba5d-0472b3708062", - "trace_id": "b8abebac-a7dc-494c-a1eb-6470181f32d2", - "parent_span_id": "d2afaf74-f63b-4141-9fa3-662162f51cc0", - "name": "agent.llm_model_execution", - "start_time": "2025-10-16 08:47:58.659000000", - "end_time": "2025-10-16 08:48:05.878000000", - "duration_ms": 7218.618, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What features does Noveum Trace offer?", - "query_length": 38, - "query_type": "rag_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 7.218573331832886, - "model.latency_ms": 7218.573331832886, - "model.start_time": 1760604478.659586, - "model.end_time": 1760604485.8781593, - "model.performance_tier": "slow", - "model.prompt_tokens": 830, - "model.completion_tokens": 254, - "model.total_tokens": 1084, - "model.tokens_per_second": 150.16817730724068, - "model.estimated_cost": 0.03252, - "model.efficiency_score": 0.9391143911439115, - "model.response_length": 1018, - "model.response_quality": "high", - "model.output_response": "Model Response: Noveum Trace offers several key features through its Python SDK (`noveum-trace`) and TypeScript SDK (`@noveum/trace`):\n\n1. **Decorator-Based Tracing**: It provides a decorator-based approach for traci...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "b8abebac-a7dc-494c-a1eb-6470181f32d2", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:47:57.667000000", - "trace_end_time": "2025-10-16 08:48:07.005000000", - "trace_duration_ms": 9337.405, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:47:57.667000000", - "updated_at": "2025-10-16 08:47:57.667000000" - }, - { - "span_id": "015688fe-d287-48e8-ac59-63781ffd6f89", - "trace_id": "b8abebac-a7dc-494c-a1eb-6470181f32d2", - "parent_span_id": "d2afaf74-f63b-4141-9fa3-662162f51cc0", - "name": "agent.rag_evaluation_metrics", - "start_time": "2025-10-16 08:48:05.878000000", - "end_time": "2025-10-16 08:48:05.878000000", - "duration_ms": 0.024, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "rag_evaluation_metrics", - "agent.capabilities": [ - "retrieval_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What features does Noveum Trace offer?", - "query_length": 38, - "query_type": "rag_evaluation_query", - "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\n\ud83d\udcd8 TypeScript SDK (\n@noveum/trace\n)- Framework ...", - "retrieval.context_length": 2787, - "retrieval.sources_count": 5, - "retrieval.context_quality": "high", - "retrieval.effectiveness": 1, - "retrieval.context_utilization": 2.787, - "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\n\ud83d\udcd8 TypeScript SDK (\n@noveum/trace\n)- Framework integrations for Next.js, Express.js, and Hono\n- TypeScript-first design with full type safety\n- Universal compatibility (Node.js, Edge Runtime, browsers)\n-\n\ud83d\udcca Noveum Platform\n- Real-time dashboard for analyzing traces and performance\n- Ad...\n\nSource 2 (https://noveum.ai/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\n\ud83d\udcd8 TypeScript SDK (\n@noveum/trace\n)- Framework integrations for Next.js, Express.js, and Hono\n- TypeScript-first design with full type safety\n- Universal compatibility (Node.js, Edge Runtime, browsers)\n-\n\ud83d\udcca Noveum Platform\n- Real-time dashboard for analyzing traces and performance\n- Ad...\n\nSource 3 (https://noveum.ai/en/docs/getting-started/sdk-integration):\n- \ud83d\udc65 Collaborate with your team on insights\n\ud83d\udd12 Security & Privacy\n- \ud83d\udd10 Encryption: All data encrypted in transit and at rest\n- \ud83c\udf9b\ufe0f Configurable Capture: Control what data is collected\n- \ud83c\udfe0 Data Residency: Choose your data storage region\n- \u23f0 Retention Control: Set custom data retention policies\nNext Steps\n- Tracing Concepts - Learn about traces, spans, and observability best practices\n- Framework Integrations - Deep dive into specific framework setups\n- Multi-Agent Tracing - Observe complex agent work...\n\nSource 4 (https://noveum.ai/docs/getting-started/sdk-integration):\n- \ud83d\udc65 Collaborate with your team on insights\n\ud83d\udd12 Security & Privacy\n- \ud83d\udd10 Encryption: All data encrypted in transit and at rest\n- \ud83c\udf9b\ufe0f Configurable Capture: Control what data is collected\n- \ud83c\udfe0 Data Residency: Choose your data storage region\n- \u23f0 Retention Control: Set custom data retention policies\nNext Steps\n- Tracing Concepts - Learn about traces, spans, and observability best practices\n- Framework Integrations - Deep dive into specific framework setups\n- Multi-Agent Tracing - Observe complex agent work...\n\nSource 5 (https://noveum.ai/docs/platform/dashboard):\n- Bulk Export: Download multiple traces simultaneously\n- Comparative Analysis: Compare performance across multiple traces\nReal-Time Updates\n- Live Refresh: Automatic updates as new traces arrive\n- Connection Monitoring: Real-time status of your trace ingestion pipeline\n- Performance Indicators: Live metrics for system health monitoring\nReady to dive deeper? Explore Projects & Environments to organize your AI applications, or check out Team Collaboration to share insights with your team.\nGet Earl...\n\nUser Question: What features does Noveum Trace offer?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", - "prompt.prompt_length": 3321, - "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frame...", - "prompt.instruction_following": "rag_optimized", - "response.answer_length": 1018, - "response.answer_completeness": "complete", - "response.response_quality": "high", - "response.source_citation": 5, - "response.context_utilization": 0.36526731252242556, - "output_response": "RAG Answer: Noveum Trace offers several key features through its Python SDK (`noveum-trace`) and TypeScript SDK (`@noveum/trace`):\n\n1. **Decorator-Based Tracing**: It provides a decorator-based approach for traci...", - "evaluation.retrieval_effectiveness": 1, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 5, - "evaluation.context_relevance": "high", - "evaluation.overall_quality": "high", - "evaluation.ready_for_production": true, - "rag.retrieval_strategy": "semantic_similarity", - "rag.vector_search_results": 5, - "rag.context_synthesis": "multi_source", - "rag.document_coverage": 1, - "rag.information_density": 0.36526731252242556 - }, - "events": [], - "links": [], - "trace_trace_id": "b8abebac-a7dc-494c-a1eb-6470181f32d2", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:47:57.667000000", - "trace_end_time": "2025-10-16 08:48:07.005000000", - "trace_duration_ms": 9337.405, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:47:57.667000000", - "updated_at": "2025-10-16 08:47:57.667000000" - }, - { - "span_id": "46ee4d00-f914-41d8-9f91-c2db85f1c613", - "trace_id": "be9ccbaf-ccb4-4ebd-a380-4247f4d01868", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-16 08:47:52.037000000", - "end_time": "2025-10-16 08:47:56.662000000", - "duration_ms": 4625.395, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What are Noveum's pricing plans?", - "query.length": 32, - "processing.mode": "RAG", - "processing.time_seconds": 4.6253437995910645, - "processing.response_length": 471, - "processing.sources_count": 5, - "output_response": "Final Answer: The provided context does not contain specific information about Noveum's pricing plans. It mentions that all users will have access to the Observability suite for free, and early users will receive f...", - "final_answer_mode": "RAG", - "query_processed.input_query": "What are Noveum's pricing plans?", - "query_processed.output_response": "Successfully processed query using RAG, generated 471 character response", - "query_processed.mode": "RAG", - "query_processed.processing_time": 4.6253437995910645, - "query_processed.response_length": 471 - }, - "events": [], - "links": [], - "trace_trace_id": "be9ccbaf-ccb4-4ebd-a380-4247f4d01868", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:47:52.037000000", - "trace_end_time": "2025-10-16 08:47:56.662000000", - "trace_duration_ms": 4625.492, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:47:52.037000000", - "updated_at": "2025-10-16 08:47:52.037000000" - }, - { - "span_id": "6d15369a-ae35-45c7-84dd-bdf484f73f2d", - "trace_id": "be9ccbaf-ccb4-4ebd-a380-4247f4d01868", - "parent_span_id": "46ee4d00-f914-41d8-9f91-c2db85f1c613", - "name": "agent.query_routing", - "start_time": "2025-10-16 08:47:52.037000000", - "end_time": "2025-10-16 08:47:56.662000000", - "duration_ms": 4625.286, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What are Noveum's pricing plans?", - "query_length": 32, - "query_type": "routing_query", - "output_response": "Routed to RAG for query processing", - "router.classification": "RAG", - "router.confidence_score": 1, - "router.rag_keyword_score": 3, - "router.web_keyword_score": 0, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "be9ccbaf-ccb4-4ebd-a380-4247f4d01868", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:47:52.037000000", - "trace_end_time": "2025-10-16 08:47:56.662000000", - "trace_duration_ms": 4625.492, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:47:52.037000000", - "updated_at": "2025-10-16 08:47:52.037000000" - }, - { - "span_id": "bf201676-a3a9-433a-a8c6-d394b7aead3a", - "trace_id": "be9ccbaf-ccb4-4ebd-a380-4247f4d01868", - "parent_span_id": "6d15369a-ae35-45c7-84dd-bdf484f73f2d", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-16 08:47:52.037000000", - "end_time": "2025-10-16 08:47:52.037000000", - "duration_ms": 0.048, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What are Noveum's pricing plans?", - "query_length": 32, - "query_type": "routing_evaluation_query", - "classification.mode": "RAG", - "classification.rag_keyword_score": 3, - "classification.web_keyword_score": 0, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "noveum_specific", - "query.keyword_density": 0.6, - "query.domain_affinity": "noveum", - "routing.decision": "Routed to RAG based on analysis", - "routing.rationale": "RAG score: 3, Web score: 0, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "Web Search", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.057692307692307696, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: RAG (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "be9ccbaf-ccb4-4ebd-a380-4247f4d01868", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:47:52.037000000", - "trace_end_time": "2025-10-16 08:47:56.662000000", - "trace_duration_ms": 4625.492, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:47:52.037000000", - "updated_at": "2025-10-16 08:47:52.037000000" - }, - { - "span_id": "f8346f60-131b-4007-ba3e-7b0ff10cdfde", - "trace_id": "be9ccbaf-ccb4-4ebd-a380-4247f4d01868", - "parent_span_id": "6d15369a-ae35-45c7-84dd-bdf484f73f2d", - "name": "agent.llm-rag", - "start_time": "2025-10-16 08:47:52.037000000", - "end_time": "2025-10-16 08:47:56.662000000", - "duration_ms": 4624.998, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "rag_agent", - "agent.operation": "llm-rag", - "agent.capabilities": [ - "document_retrieval", - "context_generation", - "response_generation" - ], - "agent.id": "noveum_rag_agent", - "input_query": "What are Noveum's pricing plans?", - "query_length": 32, - "query_type": "rag_query", - "output_response": "RAG Answer: The provided context does not contain specific information about Noveum's pricing plans. It mentions that all users will have access to the Observability suite for free, and early users will receive f...", - "rag.context_length": 2105, - "rag.sources_count": 5, - "rag.answer_length": 471, - "rag.mode": "retrieval_augmented_generation" - }, - "events": [], - "links": [], - "trace_trace_id": "be9ccbaf-ccb4-4ebd-a380-4247f4d01868", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:47:52.037000000", - "trace_end_time": "2025-10-16 08:47:56.662000000", - "trace_duration_ms": 4625.492, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:47:52.037000000", - "updated_at": "2025-10-16 08:47:52.037000000" - }, - { - "span_id": "0186c28c-7596-4279-a4eb-7733cd665609", - "trace_id": "be9ccbaf-ccb4-4ebd-a380-4247f4d01868", - "parent_span_id": "f8346f60-131b-4007-ba3e-7b0ff10cdfde", - "name": "agent.llm_model_execution", - "start_time": "2025-10-16 08:47:52.579000000", - "end_time": "2025-10-16 08:47:55.842000000", - "duration_ms": 3262.5829999999996, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What are Noveum's pricing plans?", - "query_length": 32, - "query_type": "rag_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 3.2625203132629395, - "model.latency_ms": 3262.5203132629395, - "model.start_time": 1760604472.580008, - "model.end_time": 1760604475.8425283, - "model.performance_tier": "medium", - "model.prompt_tokens": 658, - "model.completion_tokens": 117, - "model.total_tokens": 775, - "model.tokens_per_second": 237.54641368804243, - "model.estimated_cost": 0.02325, - "model.efficiency_score": 0.607741935483871, - "model.response_length": 471, - "model.response_quality": "high", - "model.output_response": "Model Response: The provided context does not contain specific information about Noveum's pricing plans. It mentions that all users will have access to the Observability suite for free, and early users will receive f...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "be9ccbaf-ccb4-4ebd-a380-4247f4d01868", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:47:52.037000000", - "trace_end_time": "2025-10-16 08:47:56.662000000", - "trace_duration_ms": 4625.492, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:47:52.037000000", - "updated_at": "2025-10-16 08:47:52.037000000" - }, - { - "span_id": "3bdf8d5b-c10d-44f3-8514-8facc7f63db8", - "trace_id": "be9ccbaf-ccb4-4ebd-a380-4247f4d01868", - "parent_span_id": "f8346f60-131b-4007-ba3e-7b0ff10cdfde", - "name": "agent.rag_evaluation_metrics", - "start_time": "2025-10-16 08:47:55.842000000", - "end_time": "2025-10-16 08:47:55.842000000", - "duration_ms": 0.032, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "rag_evaluation_metrics", - "agent.capabilities": [ - "retrieval_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What are Noveum's pricing plans?", - "query_length": 32, - "query_type": "rag_evaluation_query", - "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....", - "retrieval.context_length": 2105, - "retrieval.sources_count": 5, - "retrieval.context_quality": "high", - "retrieval.effectiveness": 1, - "retrieval.context_utilization": 2.105, - "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nSource 2 (https://noveum.ai/en/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nSource 3 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nThe AI landscape is complex and moving fast. With Noveum.ai, you'll have the insights you need to build, debug, and optimize AI applications with confidence.\nReady to see what's really happening inside your AI applications? Try Noveum.ai today and transform how you build and monitor AI systems.\nLet's build more reliable, observable AI\u2014together.\nGet Early Access to Noveum.ai Platform\nJoin the select group of AI teams optimizing their models with our data-driven platform. We're onboarding users in...\n\nSource 4 (https://noveum.ai/en/blog/from-logs-to-intelligent-choices-inside-noveum-ais-evaluation-process):\nGet Early Access to Noveum.ai Platform\nJoin the select group of AI teams optimizing their models with our data-driven platform. We're onboarding users in limited batches to ensure a premium experience....\n\nSource 5 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, vector searches, data retrieval, agent reasoning, and more. Without proper observability, debugging becomes a nightmare, optimization is guesswork, and understanding user interactions is nearly impossible.\nNoveum.ai solves this by providing comprehensive tracing and observability specifically designed for AI applications.\nKey benefits of using Noveum.ai:\n- Complete Visibility: Trace...\n\nUser Question: What are Noveum's pricing plans?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", - "prompt.prompt_length": 2633, - "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to ...", - "prompt.instruction_following": "rag_optimized", - "response.answer_length": 471, - "response.answer_completeness": "complete", - "response.response_quality": "high", - "response.source_citation": 5, - "response.context_utilization": 0.22375296912114015, - "output_response": "RAG Answer: The provided context does not contain specific information about Noveum's pricing plans. It mentions that all users will have access to the Observability suite for free, and early users will receive f...", - "evaluation.retrieval_effectiveness": 1, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 5, - "evaluation.context_relevance": "high", - "evaluation.overall_quality": "high", - "evaluation.ready_for_production": true, - "rag.retrieval_strategy": "semantic_similarity", - "rag.vector_search_results": 5, - "rag.context_synthesis": "multi_source", - "rag.document_coverage": 1, - "rag.information_density": 0.22375296912114015 - }, - "events": [], - "links": [], - "trace_trace_id": "be9ccbaf-ccb4-4ebd-a380-4247f4d01868", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:47:52.037000000", - "trace_end_time": "2025-10-16 08:47:56.662000000", - "trace_duration_ms": 4625.492, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:47:52.037000000", - "updated_at": "2025-10-16 08:47:52.037000000" - }, - { - "span_id": "770ce93d-07ae-4b9f-ae15-c6e450634b9d", - "trace_id": "8d33a529-d9b1-4208-b528-762222d5ef8c", - "parent_span_id": "9977d5d6-a7ff-469e-bec7-5e4dfc38c149", - "name": "agent.llm-rag", - "start_time": "2025-10-16 08:47:42.103000000", - "end_time": "2025-10-16 08:47:51.032000000", - "duration_ms": 8928.644999999999, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "rag_agent", - "agent.operation": "llm-rag", - "agent.capabilities": [ - "document_retrieval", - "context_generation", - "response_generation" - ], - "agent.id": "noveum_rag_agent", - "input_query": "How do I integrate Noveum Trace in my application?", - "query_length": 50, - "query_type": "rag_query", - "output_response": "RAG Answer: To integrate Noveum Trace into your application, you can use either the Python SDK (`noveum-trace`) or the TypeScript SDK (`@noveum/trace`). Here\u2019s a quick start guide based on the documentation:\n\n1. ...", - "rag.context_length": 2803, - "rag.sources_count": 5, - "rag.answer_length": 1134, - "rag.mode": "retrieval_augmented_generation" - }, - "events": [], - "links": [], - "trace_trace_id": "8d33a529-d9b1-4208-b528-762222d5ef8c", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:47:42.103000000", - "trace_end_time": "2025-10-16 08:47:51.032000000", - "trace_duration_ms": 8928.948, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:47:42.103000000", - "updated_at": "2025-10-16 08:47:42.103000000" - }, - { - "span_id": "9977d5d6-a7ff-469e-bec7-5e4dfc38c149", - "trace_id": "8d33a529-d9b1-4208-b528-762222d5ef8c", - "parent_span_id": "f3a8fa53-5641-4a23-b4d3-762577a89151", - "name": "agent.query_routing", - "start_time": "2025-10-16 08:47:42.103000000", - "end_time": "2025-10-16 08:47:51.032000000", - "duration_ms": 8928.793000000001, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "How do I integrate Noveum Trace in my application?", - "query_length": 50, - "query_type": "routing_query", - "output_response": "Routed to RAG for query processing", - "router.classification": "RAG", - "router.confidence_score": 1, - "router.rag_keyword_score": 2, - "router.web_keyword_score": 0, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "8d33a529-d9b1-4208-b528-762222d5ef8c", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:47:42.103000000", - "trace_end_time": "2025-10-16 08:47:51.032000000", - "trace_duration_ms": 8928.948, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:47:42.103000000", - "updated_at": "2025-10-16 08:47:42.103000000" - }, - { - "span_id": "c678e222-ef74-48ee-8628-16461ab0f48f", - "trace_id": "8d33a529-d9b1-4208-b528-762222d5ef8c", - "parent_span_id": "9977d5d6-a7ff-469e-bec7-5e4dfc38c149", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-16 08:47:42.103000000", - "end_time": "2025-10-16 08:47:42.103000000", - "duration_ms": 0.027, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: How do I integrate Noveum Trace in my application?", - "query_length": 50, - "query_type": "routing_evaluation_query", - "classification.mode": "RAG", - "classification.rag_keyword_score": 2, - "classification.web_keyword_score": 0, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "noveum_specific", - "query.keyword_density": 0.2222222222222222, - "query.domain_affinity": "noveum", - "routing.decision": "Routed to RAG based on analysis", - "routing.rationale": "RAG score: 2, Web score: 0, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "Web Search", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.038461538461538464, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: RAG (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "8d33a529-d9b1-4208-b528-762222d5ef8c", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:47:42.103000000", - "trace_end_time": "2025-10-16 08:47:51.032000000", - "trace_duration_ms": 8928.948, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:47:42.103000000", - "updated_at": "2025-10-16 08:47:42.103000000" - }, - { - "span_id": "f3a8fa53-5641-4a23-b4d3-762577a89151", - "trace_id": "8d33a529-d9b1-4208-b528-762222d5ef8c", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-16 08:47:42.103000000", - "end_time": "2025-10-16 08:47:51.032000000", - "duration_ms": 8928.868, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "How do I integrate Noveum Trace in my application?", - "query.length": 50, - "processing.mode": "RAG", - "processing.time_seconds": 8.928829431533813, - "processing.response_length": 1134, - "processing.sources_count": 5, - "output_response": "Final Answer: To integrate Noveum Trace into your application, you can use either the Python SDK (`noveum-trace`) or the TypeScript SDK (`@noveum/trace`). Here\u2019s a quick start guide based on the documentation:\n\n1. ...", - "final_answer_mode": "RAG", - "query_processed.input_query": "How do I integrate Noveum Trace in my application?", - "query_processed.output_response": "Successfully processed query using RAG, generated 1134 character response", - "query_processed.mode": "RAG", - "query_processed.processing_time": 8.928829431533813, - "query_processed.response_length": 1134 - }, - "events": [], - "links": [], - "trace_trace_id": "8d33a529-d9b1-4208-b528-762222d5ef8c", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:47:42.103000000", - "trace_end_time": "2025-10-16 08:47:51.032000000", - "trace_duration_ms": 8928.948, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:47:42.103000000", - "updated_at": "2025-10-16 08:47:42.103000000" - }, - { - "span_id": "f459b71f-1d43-41b0-b9bf-d94a908b7afa", - "trace_id": "8d33a529-d9b1-4208-b528-762222d5ef8c", - "parent_span_id": "770ce93d-07ae-4b9f-ae15-c6e450634b9d", - "name": "agent.llm_model_execution", - "start_time": "2025-10-16 08:47:43.044000000", - "end_time": "2025-10-16 08:47:50.008000000", - "duration_ms": 6963.959, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: How do I integrate Noveum Trace in my application?", - "query_length": 50, - "query_type": "rag_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 6.9639081954956055, - "model.latency_ms": 6963.9081954956055, - "model.start_time": 1760604463.044064, - "model.end_time": 1760604470.0079722, - "model.performance_tier": "slow", - "model.prompt_tokens": 837, - "model.completion_tokens": 283, - "model.total_tokens": 1120, - "model.tokens_per_second": 160.8292310235276, - "model.estimated_cost": 0.0336, - "model.efficiency_score": 1.0125, - "model.response_length": 1134, - "model.response_quality": "high", - "model.output_response": "Model Response: To integrate Noveum Trace into your application, you can use either the Python SDK (`noveum-trace`) or the TypeScript SDK (`@noveum/trace`). Here\u2019s a quick start guide based on the documentation:\n\n1. ...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "8d33a529-d9b1-4208-b528-762222d5ef8c", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:47:42.103000000", - "trace_end_time": "2025-10-16 08:47:51.032000000", - "trace_duration_ms": 8928.948, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:47:42.103000000", - "updated_at": "2025-10-16 08:47:42.103000000" - }, - { - "span_id": "b8627171-b081-4b08-b900-c24375b73c3f", - "trace_id": "8d33a529-d9b1-4208-b528-762222d5ef8c", - "parent_span_id": "770ce93d-07ae-4b9f-ae15-c6e450634b9d", - "name": "agent.rag_evaluation_metrics", - "start_time": "2025-10-16 08:47:50.008000000", - "end_time": "2025-10-16 08:47:50.008000000", - "duration_ms": 0.041, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "rag_evaluation_metrics", - "agent.capabilities": [ - "retrieval_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: How do I integrate Noveum Trace in my application?", - "query_length": 50, - "query_type": "rag_evaluation_query", - "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\n\ud83d\udcd8 TypeScript SDK (\n@noveum/trace\n)- Framework ...", - "retrieval.context_length": 2803, - "retrieval.sources_count": 5, - "retrieval.context_quality": "high", - "retrieval.effectiveness": 1, - "retrieval.context_utilization": 2.803, - "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\n\ud83d\udcd8 TypeScript SDK (\n@noveum/trace\n)- Framework integrations for Next.js, Express.js, and Hono\n- TypeScript-first design with full type safety\n- Universal compatibility (Node.js, Edge Runtime, browsers)\n-\n\ud83d\udcca Noveum Platform\n- Real-time dashboard for analyzing traces and performance\n- Ad...\n\nSource 2 (https://noveum.ai/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\n\ud83d\udcd8 TypeScript SDK (\n@noveum/trace\n)- Framework integrations for Next.js, Express.js, and Hono\n- TypeScript-first design with full type safety\n- Universal compatibility (Node.js, Edge Runtime, browsers)\n-\n\ud83d\udcca Noveum Platform\n- Real-time dashboard for analyzing traces and performance\n- Ad...\n\nSource 3 (https://noveum.ai/docs/getting-started/sdk-integration):\nSDK Integration Guide\nIntegrate Noveum.ai tracing into your AI applications with Python or TypeScript SDKs\nThe Noveum.ai SDKs provide comprehensive tracing and observability for your AI applications with minimal code changes. Whether you're building LLM applications, RAG systems, or multi-agent workflows, our SDKs automatically capture essential metrics and traces.\n\ud83d\ude80 Quick Start\n1. Create Your Account & Get API Key\n- Sign up at noveum.ai\n- Create a project in your dashboard\n- Generate an API key...\n\nSource 4 (https://noveum.ai/en/docs/getting-started/sdk-integration):\nSDK Integration Guide\nIntegrate Noveum.ai tracing into your AI applications with Python or TypeScript SDKs\nThe Noveum.ai SDKs provide comprehensive tracing and observability for your AI applications with minimal code changes. Whether you're building LLM applications, RAG systems, or multi-agent workflows, our SDKs automatically capture essential metrics and traces.\n\ud83d\ude80 Quick Start\n1. Create Your Account & Get API Key\n- Sign up at noveum.ai\n- Create a project in your dashboard\n- Generate an API key...\n\nSource 5 (https://noveum.ai/en/docs/getting-started/sdk-integration):\n\ud83d\udd27 Framework Integrations\nNext.js Integration\nExpress.js Integration\nFastAPI Integration (Python)\n\ud83d\udcca Advanced Features\nCustom Attributes & Events\nSampling Configuration\n\ud83d\udd17 What's Captured Automatically\n- \ud83d\udcca Performance Metrics: Latency, throughput, error rates\n- \ud83d\udcb0 Cost Tracking: Token usage, API costs across providers\n- \ud83d\udd0d Request/Response: Configurable capture of inputs/outputs\n- \ud83c\udff7\ufe0f Metadata: Model names, parameters, user context\n- \ud83c\udf0a Context Flow: Trace relationships across services\n- \ud83d\udc1b Error Detail...\n\nUser Question: How do I integrate Noveum Trace in my application?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", - "prompt.prompt_length": 3349, - "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frame...", - "prompt.instruction_following": "rag_optimized", - "response.answer_length": 1134, - "response.answer_completeness": "complete", - "response.response_quality": "high", - "response.source_citation": 5, - "response.context_utilization": 0.4045665358544417, - "output_response": "RAG Answer: To integrate Noveum Trace into your application, you can use either the Python SDK (`noveum-trace`) or the TypeScript SDK (`@noveum/trace`). Here\u2019s a quick start guide based on the documentation:\n\n1. ...", - "evaluation.retrieval_effectiveness": 1, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 5, - "evaluation.context_relevance": "high", - "evaluation.overall_quality": "high", - "evaluation.ready_for_production": true, - "rag.retrieval_strategy": "semantic_similarity", - "rag.vector_search_results": 5, - "rag.context_synthesis": "multi_source", - "rag.document_coverage": 1, - "rag.information_density": 0.4045665358544417 - }, - "events": [], - "links": [], - "trace_trace_id": "8d33a529-d9b1-4208-b528-762222d5ef8c", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:47:42.103000000", - "trace_end_time": "2025-10-16 08:47:51.032000000", - "trace_duration_ms": 8928.948, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:47:42.103000000", - "updated_at": "2025-10-16 08:47:42.103000000" - }, - { - "span_id": "0b3d6893-776e-4118-b8b5-b1c5ea51def1", - "trace_id": "06192a09-b6bb-4548-8a94-e123d6caaffe", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-16 08:47:03.776000000", - "end_time": "2025-10-16 08:47:41.100000000", - "duration_ms": 37324.516, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What is Noveum and what does it do?", - "query.length": 35, - "processing.mode": "RAG", - "processing.time_seconds": 37.324477195739746, - "processing.response_length": 648, - "processing.sources_count": 5, - "output_response": "Final Answer: Noveum.ai is a comprehensive tracing and observability platform specifically designed for AI applications, including those powered by large language models (LLMs), retrieval-augmented generation (RAG)...", - "final_answer_mode": "RAG", - "query_processed.input_query": "What is Noveum and what does it do?", - "query_processed.output_response": "Successfully processed query using RAG, generated 648 character response", - "query_processed.mode": "RAG", - "query_processed.processing_time": 37.324477195739746, - "query_processed.response_length": 648 - }, - "events": [], - "links": [], - "trace_trace_id": "06192a09-b6bb-4548-8a94-e123d6caaffe", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:47:03.776000000", - "trace_end_time": "2025-10-16 08:47:41.100000000", - "trace_duration_ms": 37324.601, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:47:03.776000000", - "updated_at": "2025-10-16 08:47:03.776000000" - }, - { - "span_id": "137ad05b-bada-42be-abe5-be0c277f6d0e", - "trace_id": "06192a09-b6bb-4548-8a94-e123d6caaffe", - "parent_span_id": "c3664d20-1708-45f5-a051-b626b0ec77cd", - "name": "agent.llm-rag", - "start_time": "2025-10-16 08:47:03.776000000", - "end_time": "2025-10-16 08:47:41.100000000", - "duration_ms": 37324.042, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "rag_agent", - "agent.operation": "llm-rag", - "agent.capabilities": [ - "document_retrieval", - "context_generation", - "response_generation" - ], - "agent.id": "noveum_rag_agent", - "input_query": "What is Noveum and what does it do?", - "query_length": 35, - "query_type": "rag_query", - "output_response": "RAG Answer: Noveum.ai is a comprehensive tracing and observability platform specifically designed for AI applications, including those powered by large language models (LLMs), retrieval-augmented generation (RAG)...", - "rag.context_length": 2334, - "rag.sources_count": 5, - "rag.answer_length": 648, - "rag.mode": "retrieval_augmented_generation" - }, - "events": [], - "links": [], - "trace_trace_id": "06192a09-b6bb-4548-8a94-e123d6caaffe", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:47:03.776000000", - "trace_end_time": "2025-10-16 08:47:41.100000000", - "trace_duration_ms": 37324.601, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:47:03.776000000", - "updated_at": "2025-10-16 08:47:03.776000000" - }, - { - "span_id": "c3664d20-1708-45f5-a051-b626b0ec77cd", - "trace_id": "06192a09-b6bb-4548-8a94-e123d6caaffe", - "parent_span_id": "0b3d6893-776e-4118-b8b5-b1c5ea51def1", - "name": "agent.query_routing", - "start_time": "2025-10-16 08:47:03.776000000", - "end_time": "2025-10-16 08:47:41.100000000", - "duration_ms": 37324.382999999994, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What is Noveum and what does it do?", - "query_length": 35, - "query_type": "routing_query", - "output_response": "Routed to RAG for query processing", - "router.classification": "RAG", - "router.confidence_score": 1, - "router.rag_keyword_score": 1, - "router.web_keyword_score": 0, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "06192a09-b6bb-4548-8a94-e123d6caaffe", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:47:03.776000000", - "trace_end_time": "2025-10-16 08:47:41.100000000", - "trace_duration_ms": 37324.601, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:47:03.776000000", - "updated_at": "2025-10-16 08:47:03.776000000" - }, - { - "span_id": "ce9a94bf-1a72-4599-8561-11e66802b92d", - "trace_id": "06192a09-b6bb-4548-8a94-e123d6caaffe", - "parent_span_id": "c3664d20-1708-45f5-a051-b626b0ec77cd", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-16 08:47:03.776000000", - "end_time": "2025-10-16 08:47:03.776000000", - "duration_ms": 0.067, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What is Noveum and what does it do?", - "query_length": 35, - "query_type": "routing_evaluation_query", - "classification.mode": "RAG", - "classification.rag_keyword_score": 1, - "classification.web_keyword_score": 0, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "noveum_specific", - "query.keyword_density": 0.125, - "query.domain_affinity": "noveum", - "routing.decision": "Routed to RAG based on analysis", - "routing.rationale": "RAG score: 1, Web score: 0, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "Web Search", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.019230769230769232, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: RAG (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "06192a09-b6bb-4548-8a94-e123d6caaffe", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:47:03.776000000", - "trace_end_time": "2025-10-16 08:47:41.100000000", - "trace_duration_ms": 37324.601, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:47:03.776000000", - "updated_at": "2025-10-16 08:47:03.776000000" - }, - { - "span_id": "38414416-907a-4188-95fe-c0a5f900f72c", - "trace_id": "06192a09-b6bb-4548-8a94-e123d6caaffe", - "parent_span_id": "137ad05b-bada-42be-abe5-be0c277f6d0e", - "name": "agent.llm_model_execution", - "start_time": "2025-10-16 08:47:10.392000000", - "end_time": "2025-10-16 08:47:40.392000000", - "duration_ms": 30000.104, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What is Noveum and what does it do?", - "query_length": 35, - "query_type": "rag_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 30.000043392181396, - "model.latency_ms": 30000.043392181396, - "model.start_time": 1760604430.3921378, - "model.end_time": 1760604460.3921812, - "model.performance_tier": "slow", - "model.prompt_tokens": 716, - "model.completion_tokens": 162, - "model.total_tokens": 878, - "model.tokens_per_second": 29.266624335244266, - "model.estimated_cost": 0.026340000000000002, - "model.efficiency_score": 0.7380410022779044, - "model.response_length": 648, - "model.response_quality": "high", - "model.output_response": "Model Response: Noveum.ai is a comprehensive tracing and observability platform specifically designed for AI applications, including those powered by large language models (LLMs), retrieval-augmented generation (RAG)...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "06192a09-b6bb-4548-8a94-e123d6caaffe", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:47:03.776000000", - "trace_end_time": "2025-10-16 08:47:41.100000000", - "trace_duration_ms": 37324.601, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:47:03.776000000", - "updated_at": "2025-10-16 08:47:03.776000000" - }, - { - "span_id": "67e23649-b00b-4eb8-9a69-865e80f833b3", - "trace_id": "06192a09-b6bb-4548-8a94-e123d6caaffe", - "parent_span_id": "137ad05b-bada-42be-abe5-be0c277f6d0e", - "name": "agent.rag_evaluation_metrics", - "start_time": "2025-10-16 08:47:40.392000000", - "end_time": "2025-10-16 08:47:40.392000000", - "duration_ms": 0.029, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "rag_evaluation_metrics", - "agent.capabilities": [ - "retrieval_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What is Noveum and what does it do?", - "query_length": 35, - "query_type": "rag_evaluation_query", - "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, vector searches, data retrieval, agent reasoning, and more. Without proper observability, debugging ...", - "retrieval.context_length": 2334, - "retrieval.sources_count": 5, - "retrieval.context_quality": "high", - "retrieval.effectiveness": 1, - "retrieval.context_utilization": 2.334, - "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, vector searches, data retrieval, agent reasoning, and more. Without proper observability, debugging becomes a nightmare, optimization is guesswork, and understanding user interactions is nearly impossible.\nNoveum.ai solves this by providing comprehensive tracing and observability specifically designed for AI applications.\nKey benefits of using Noveum.ai:\n- Complete Visibility: Trace...\n\nSource 2 (https://noveum.ai/en/docs/getting-started/overview):\nNoveum.ai Overview\nComprehensive AI tracing and observability platform for LLM applications, RAG systems, and AI agents\nWelcome to Noveum.ai\u2014the comprehensive tracing and observability platform built specifically for AI applications. Whether you're building LLM-powered chatbots, RAG systems, multi-agent workflows, or any AI-driven application, Noveum provides the insights you need to understand, debug, and optimize your systems.\n\ud83c\udfaf Why AI Applications Need Specialized Observability\nTraditional mo...\n\nSource 3 (https://noveum.ai/docs/getting-started/overview):\nNoveum.ai Overview\nComprehensive AI tracing and observability platform for LLM applications, RAG systems, and AI agents\nWelcome to Noveum.ai\u2014the comprehensive tracing and observability platform built specifically for AI applications. Whether you're building LLM-powered chatbots, RAG systems, multi-agent workflows, or any AI-driven application, Noveum provides the insights you need to understand, debug, and optimize your systems.\n\ud83c\udfaf Why AI Applications Need Specialized Observability\nTraditional mo...\n\nSource 4 (https://noveum.ai/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nSource 5 (https://noveum.ai/en/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nUser Question: What is Noveum and what does it do?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", - "prompt.prompt_length": 2865, - "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, ...", - "prompt.instruction_following": "rag_optimized", - "response.answer_length": 648, - "response.answer_completeness": "complete", - "response.response_quality": "high", - "response.source_citation": 5, - "response.context_utilization": 0.2776349614395887, - "output_response": "RAG Answer: Noveum.ai is a comprehensive tracing and observability platform specifically designed for AI applications, including those powered by large language models (LLMs), retrieval-augmented generation (RAG)...", - "evaluation.retrieval_effectiveness": 1, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 5, - "evaluation.context_relevance": "high", - "evaluation.overall_quality": "high", - "evaluation.ready_for_production": true, - "rag.retrieval_strategy": "semantic_similarity", - "rag.vector_search_results": 5, - "rag.context_synthesis": "multi_source", - "rag.document_coverage": 1, - "rag.information_density": 0.2776349614395887 - }, - "events": [], - "links": [], - "trace_trace_id": "06192a09-b6bb-4548-8a94-e123d6caaffe", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:47:03.776000000", - "trace_end_time": "2025-10-16 08:47:41.100000000", - "trace_duration_ms": 37324.601, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:47:03.776000000", - "updated_at": "2025-10-16 08:47:03.776000000" - }, - { - "span_id": "050851c9-b2d3-4c82-accb-4085dc69f785", - "trace_id": "3552ee8e-eafb-499a-b109-a9fb851a41d2", - "parent_span_id": "", - "name": "system_initialization", - "start_time": "2025-10-16 08:47:03.695000000", - "end_time": "2025-10-16 08:47:03.770000000", - "duration_ms": 74.58, - "status": "ok", - "status_message": "", - "attributes": { - "system.force_scrape": false, - "system.config": { - "noveum_base_url": "https://noveum.ai", - "max_pages_to_scrape": 50, - "chunk_size": 1000, - "chunk_overlap": 200, - "max_search_results": 5, - "rag_threshold": 0.7, - "noveum_docs_file": "noveum_docs.json", - "vector_store_path": "noveum_vectorstore" - }, - "input_query": "Initialize system with force_scrape=False", - "output_response": "System initialization: RAG system loaded, vector store ready, agent operational", - "system.initialized": true, - "system.documents_loaded": 38, - "system.vectorstore_ready": true - }, - "events": [], - "links": [], - "trace_trace_id": "3552ee8e-eafb-499a-b109-a9fb851a41d2", - "trace_name": "auto_trace_system_initialization", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-16 08:47:03.695000000", - "trace_end_time": "2025-10-16 08:47:03.770000000", - "trace_duration_ms": 74.912, - "span_count": 1, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-16 08:47:03.695000000", - "updated_at": "2025-10-16 08:47:03.695000000" - }, - { - "span_id": "6b53235d-8b91-4d9e-8437-915828e84600", - "trace_id": "e6ef2449-7429-465b-9bff-f3def8de5bb0", - "parent_span_id": "e73ab184-50c1-4209-9276-be11e69e9513", - "name": "agent.query_routing", - "start_time": "2025-10-14 12:54:06.700000000", - "end_time": "2025-10-14 12:54:10.637000000", - "duration_ms": 3937.492, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What are the latest breakthroughs in artificial intelligence?", - "query_length": 61, - "query_type": "routing_query", - "output_response": "Routed to Web Search for query processing", - "router.classification": "Web Search", - "router.confidence_score": 1, - "router.rag_keyword_score": 0, - "router.web_keyword_score": 1, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "e6ef2449-7429-465b-9bff-f3def8de5bb0", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-14 12:54:06.700000000", - "trace_end_time": "2025-10-14 12:54:10.637000000", - "trace_duration_ms": 3937.729, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-14 12:54:06.700000000", - "updated_at": "2025-10-14 12:54:06.700000000" - }, - { - "span_id": "a7d1e33a-95a1-41a9-a117-62b4c373bd60", - "trace_id": "e6ef2449-7429-465b-9bff-f3def8de5bb0", - "parent_span_id": "6b53235d-8b91-4d9e-8437-915828e84600", - "name": "agent.web_search_generation", - "start_time": "2025-10-14 12:54:06.700000000", - "end_time": "2025-10-14 12:54:10.637000000", - "duration_ms": 3936.983, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "web_search_agent", - "agent.operation": "web_search_generation", - "agent.capabilities": [ - "web_search", - "content_synthesis", - "response_generation" - ], - "agent.id": "noveum_web_search_agent", - "input_query": "What are the latest breakthroughs in artificial intelligence?", - "query_length": 61, - "query_type": "web_search_query", - "output_response": "Web Search Answer: The web search did not yield specific results regarding the latest breakthroughs in artificial intelligence. However, I can provide a general overview based on knowledge up to October 2023.\n\nRecent br...", - "web_search.results_count": 1, - "web_search.context_length": 163, - "web_search.response_length": 1090, - "web_search.mode": "external_web_search" - }, - "events": [], - "links": [], - "trace_trace_id": "e6ef2449-7429-465b-9bff-f3def8de5bb0", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-14 12:54:06.700000000", - "trace_end_time": "2025-10-14 12:54:10.637000000", - "trace_duration_ms": 3937.729, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-14 12:54:06.700000000", - "updated_at": "2025-10-14 12:54:06.700000000" - }, - { - "span_id": "da630cab-2ba6-4f12-98b9-6ab1c0899c17", - "trace_id": "e6ef2449-7429-465b-9bff-f3def8de5bb0", - "parent_span_id": "6b53235d-8b91-4d9e-8437-915828e84600", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-14 12:54:06.700000000", - "end_time": "2025-10-14 12:54:06.700000000", - "duration_ms": 0.27999999999999997, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What are the latest breakthroughs in artificial intelligence?", - "query_length": 61, - "query_type": "routing_evaluation_query", - "classification.mode": "Web Search", - "classification.rag_keyword_score": 0, - "classification.web_keyword_score": 1, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "complex", - "query.intent": "general_knowledge", - "query.keyword_density": 0.125, - "query.domain_affinity": "general", - "routing.decision": "Routed to Web Search based on analysis", - "routing.rationale": "RAG score: 0, Web score: 1, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "RAG", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.019230769230769232, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: Web Search (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "e6ef2449-7429-465b-9bff-f3def8de5bb0", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-14 12:54:06.700000000", - "trace_end_time": "2025-10-14 12:54:10.637000000", - "trace_duration_ms": 3937.729, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-14 12:54:06.700000000", - "updated_at": "2025-10-14 12:54:06.700000000" - }, - { - "span_id": "e73ab184-50c1-4209-9276-be11e69e9513", - "trace_id": "e6ef2449-7429-465b-9bff-f3def8de5bb0", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-14 12:54:06.700000000", - "end_time": "2025-10-14 12:54:10.637000000", - "duration_ms": 3937.663, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What are the latest breakthroughs in artificial intelligence?", - "query.length": 61, - "processing.mode": "Web Search", - "processing.time_seconds": 3.93752384185791, - "processing.response_length": 1090, - "processing.sources_count": 1, - "output_response": "Final Answer: The web search did not yield specific results regarding the latest breakthroughs in artificial intelligence. However, I can provide a general overview based on knowledge up to October 2023.\n\nRecent br...", - "final_answer_mode": "Web Search", - "query_processed.input_query": "What are the latest breakthroughs in artificial intelligence?", - "query_processed.output_response": "Successfully processed query using Web Search, generated 1090 character response", - "query_processed.mode": "Web Search", - "query_processed.processing_time": 3.93752384185791, - "query_processed.response_length": 1090 - }, - "events": [], - "links": [], - "trace_trace_id": "e6ef2449-7429-465b-9bff-f3def8de5bb0", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-14 12:54:06.700000000", - "trace_end_time": "2025-10-14 12:54:10.637000000", - "trace_duration_ms": 3937.729, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-14 12:54:06.700000000", - "updated_at": "2025-10-14 12:54:06.700000000" - }, - { - "span_id": "bb47766c-afda-44fe-8838-7936c80cd4b6", - "trace_id": "e6ef2449-7429-465b-9bff-f3def8de5bb0", - "parent_span_id": "a7d1e33a-95a1-41a9-a117-62b4c373bd60", - "name": "agent.llm_model_execution", - "start_time": "2025-10-14 12:54:07.104000000", - "end_time": "2025-10-14 12:54:10.637000000", - "duration_ms": 3533.321, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What are the latest breakthroughs in artificial intelligence?", - "query_length": 61, - "query_type": "web_search_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 3.533268928527832, - "model.latency_ms": 3533.268928527832, - "model.start_time": 1760446447.104266, - "model.end_time": 1760446450.6375349, - "model.performance_tier": "medium", - "model.prompt_tokens": 167, - "model.completion_tokens": 272, - "model.total_tokens": 439, - "model.tokens_per_second": 124.2475477752307, - "model.estimated_cost": 0.013170000000000001, - "model.efficiency_score": 2.4829157175398633, - "model.response_length": 1090, - "model.response_quality": "high", - "model.output_response": "Model Response: The web search did not yield specific results regarding the latest breakthroughs in artificial intelligence. However, I can provide a general overview based on knowledge up to October 2023.\n\nRecent br...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "e6ef2449-7429-465b-9bff-f3def8de5bb0", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-14 12:54:06.700000000", - "trace_end_time": "2025-10-14 12:54:10.637000000", - "trace_duration_ms": 3937.729, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-14 12:54:06.700000000", - "updated_at": "2025-10-14 12:54:06.700000000" - }, - { - "span_id": "1d914a9d-b770-45dc-8308-6a36e9e0db89", - "trace_id": "e6ef2449-7429-465b-9bff-f3def8de5bb0", - "parent_span_id": "a7d1e33a-95a1-41a9-a117-62b4c373bd60", - "name": "agent.web_search_evaluation_metrics", - "start_time": "2025-10-14 12:54:10.637000000", - "end_time": "2025-10-14 12:54:10.637000000", - "duration_ms": 0.022, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "web_search_evaluation_metrics", - "agent.capabilities": [ - "web_search_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What are the latest breakthroughs in artificial intelligence?", - "query_length": 61, - "query_type": "web_search_evaluation_query", - "web_search.results_count": 1, - "web_search.context_length": 163, - "web_search.context_synthesized": "Context from 1 web sources", - "web_search.search_effectiveness": 0.2, - "web_search.context_quality": "low", - "web_search.source_diversity": 1, - "web_search.information_synthesis": "low", - "web_search.external_knowledge_utilization": 0.163, - "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What+are+the+latest+breakthroughs+in+artificial+intelligence?\n\nUser Question: What are the latest breakthroughs in artificial intelligence?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", - "prompt.prompt_length": 669, - "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What+are+the+latest+breakthroughs+in+artificial+intelligence?", - "prompt.instruction_following": "web_search_optimized", - "response.answer_length": 1090, - "response.answer_completeness": "complete", - "response.response_quality": "medium", - "response.source_citation": 1, - "response.context_utilization": 6.6871165644171775, - "output_response": "Web Search Answer: The web search did not yield specific results regarding the latest breakthroughs in artificial intelligence. However, I can provide a general overview based on knowledge up to October 2023.\n\nRecent br...", - "evaluation.search_effectiveness": 0.2, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 1, - "evaluation.context_relevance": "low", - "evaluation.overall_quality": "low", - "evaluation.ready_for_production": true, - "web_search.search_strategy": "duckduckgo_api", - "web_search.real_time_data": true, - "web_search.external_sources": 1, - "web_search.information_freshness": "current", - "web_search.knowledge_synthesis": "single_source" - }, - "events": [], - "links": [], - "trace_trace_id": "e6ef2449-7429-465b-9bff-f3def8de5bb0", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-14 12:54:06.700000000", - "trace_end_time": "2025-10-14 12:54:10.637000000", - "trace_duration_ms": 3937.729, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-14 12:54:06.700000000", - "updated_at": "2025-10-14 12:54:06.700000000" - }, - { - "span_id": "91f3999c-2edf-4c56-8f4d-ab43b6ebbaee", - "trace_id": "2ba8cab0-2366-4add-8ac3-b490a05e9a60", - "parent_span_id": "cbe51252-a323-4852-a90e-c907a0480fa3", - "name": "agent.web_search_generation", - "start_time": "2025-10-14 12:53:56.480000000", - "end_time": "2025-10-14 12:54:05.685000000", - "duration_ms": 9205.212, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "web_search_agent", - "agent.operation": "web_search_generation", - "agent.capabilities": [ - "web_search", - "content_synthesis", - "response_generation" - ], - "agent.id": "noveum_web_search_agent", - "input_query": "What's happening in the software development world today?", - "query_length": 57, - "query_type": "web_search_query", - "output_response": "Web Search Answer: The software development world is currently experiencing rapid changes driven by advancements in artificial intelligence (AI) and evolving programming practices. Key trends include:\n\n1. **AI Integrati...", - "web_search.results_count": 1, - "web_search.context_length": 975, - "web_search.response_length": 1636, - "web_search.mode": "external_web_search" - }, - "events": [], - "links": [], - "trace_trace_id": "2ba8cab0-2366-4add-8ac3-b490a05e9a60", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-14 12:53:56.480000000", - "trace_end_time": "2025-10-14 12:54:05.687000000", - "trace_duration_ms": 9207.67, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-14 12:53:56.480000000", - "updated_at": "2025-10-14 12:53:56.480000000" - }, - { - "span_id": "ea2aee75-4537-46ea-954b-2002fc0026c5", - "trace_id": "2ba8cab0-2366-4add-8ac3-b490a05e9a60", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-14 12:53:56.480000000", - "end_time": "2025-10-14 12:54:05.687000000", - "duration_ms": 9207.528999999999, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What's happening in the software development world today?", - "query.length": 57, - "processing.mode": "Web Search", - "processing.time_seconds": 9.206756830215454, - "processing.response_length": 1636, - "processing.sources_count": 1, - "output_response": "Final Answer: The software development world is currently experiencing rapid changes driven by advancements in artificial intelligence (AI) and evolving programming practices. Key trends include:\n\n1. **AI Integrati...", - "final_answer_mode": "Web Search", - "query_processed.input_query": "What's happening in the software development world today?", - "query_processed.output_response": "Successfully processed query using Web Search, generated 1636 character response", - "query_processed.mode": "Web Search", - "query_processed.processing_time": 9.206756830215454, - "query_processed.response_length": 1636 - }, - "events": [], - "links": [], - "trace_trace_id": "2ba8cab0-2366-4add-8ac3-b490a05e9a60", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-14 12:53:56.480000000", - "trace_end_time": "2025-10-14 12:54:05.687000000", - "trace_duration_ms": 9207.67, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-14 12:53:56.480000000", - "updated_at": "2025-10-14 12:53:56.480000000" - }, - { - "span_id": "cbe51252-a323-4852-a90e-c907a0480fa3", - "trace_id": "2ba8cab0-2366-4add-8ac3-b490a05e9a60", - "parent_span_id": "ea2aee75-4537-46ea-954b-2002fc0026c5", - "name": "agent.query_routing", - "start_time": "2025-10-14 12:53:56.480000000", - "end_time": "2025-10-14 12:54:05.686000000", - "duration_ms": 9205.569000000001, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What's happening in the software development world today?", - "query_length": 57, - "query_type": "routing_query", - "output_response": "Routed to Web Search for query processing", - "router.classification": "Web Search", - "router.confidence_score": 1, - "router.rag_keyword_score": 0, - "router.web_keyword_score": 1, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "2ba8cab0-2366-4add-8ac3-b490a05e9a60", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-14 12:53:56.480000000", - "trace_end_time": "2025-10-14 12:54:05.687000000", - "trace_duration_ms": 9207.67, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-14 12:53:56.480000000", - "updated_at": "2025-10-14 12:53:56.480000000" - }, - { - "span_id": "d850ecee-86ef-4818-9cec-638bfee26293", - "trace_id": "2ba8cab0-2366-4add-8ac3-b490a05e9a60", - "parent_span_id": "cbe51252-a323-4852-a90e-c907a0480fa3", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-14 12:53:56.480000000", - "end_time": "2025-10-14 12:53:56.480000000", - "duration_ms": 0.041999999999999996, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What's happening in the software development world today?", - "query_length": 57, - "query_type": "routing_evaluation_query", - "classification.mode": "Web Search", - "classification.rag_keyword_score": 0, - "classification.web_keyword_score": 1, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "complex", - "query.intent": "general_knowledge", - "query.keyword_density": 0.125, - "query.domain_affinity": "general", - "routing.decision": "Routed to Web Search based on analysis", - "routing.rationale": "RAG score: 0, Web score: 1, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "RAG", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.019230769230769232, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: Web Search (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "2ba8cab0-2366-4add-8ac3-b490a05e9a60", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-14 12:53:56.480000000", - "trace_end_time": "2025-10-14 12:54:05.687000000", - "trace_duration_ms": 9207.67, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-14 12:53:56.480000000", - "updated_at": "2025-10-14 12:53:56.480000000" - }, - { - "span_id": "0d4387a4-adf3-42c1-aca7-51453a90dd36", - "trace_id": "2ba8cab0-2366-4add-8ac3-b490a05e9a60", - "parent_span_id": "91f3999c-2edf-4c56-8f4d-ab43b6ebbaee", - "name": "agent.llm_model_execution", - "start_time": "2025-10-14 12:53:59.426000000", - "end_time": "2025-10-14 12:54:05.684000000", - "duration_ms": 6258.643, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What's happening in the software development world today?", - "query_length": 57, - "query_type": "web_search_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 6.25848388671875, - "model.latency_ms": 6258.48388671875, - "model.start_time": 1760446439.426367, - "model.end_time": 1760446445.684851, - "model.performance_tier": "slow", - "model.prompt_tokens": 369, - "model.completion_tokens": 409, - "model.total_tokens": 778, - "model.tokens_per_second": 124.31125718019486, - "model.estimated_cost": 0.02334, - "model.efficiency_score": 2.102827763496144, - "model.response_length": 1636, - "model.response_quality": "high", - "model.output_response": "Model Response: The software development world is currently experiencing rapid changes driven by advancements in artificial intelligence (AI) and evolving programming practices. Key trends include:\n\n1. **AI Integrati...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "2ba8cab0-2366-4add-8ac3-b490a05e9a60", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-14 12:53:56.480000000", - "trace_end_time": "2025-10-14 12:54:05.687000000", - "trace_duration_ms": 9207.67, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-14 12:53:56.480000000", - "updated_at": "2025-10-14 12:53:56.480000000" - }, - { - "span_id": "717e3c4a-6300-45e5-9456-caa3e773d758", - "trace_id": "2ba8cab0-2366-4add-8ac3-b490a05e9a60", - "parent_span_id": "91f3999c-2edf-4c56-8f4d-ab43b6ebbaee", - "name": "agent.web_search_evaluation_metrics", - "start_time": "2025-10-14 12:54:05.685000000", - "end_time": "2025-10-14 12:54:05.685000000", - "duration_ms": 0.044, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "web_search_evaluation_metrics", - "agent.capabilities": [ - "web_search_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What's happening in the software development world today?", - "query_length": 57, - "query_type": "web_search_evaluation_query", - "web_search.results_count": 1, - "web_search.context_length": 975, - "web_search.context_synthesized": "Context from 1 web sources", - "web_search.search_effectiveness": 0.2, - "web_search.context_quality": "high", - "web_search.source_diversity": 1, - "web_search.information_synthesis": "low", - "web_search.external_knowledge_utilization": 0.975, - "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nLog in to WhatsApp Web for simple, reliable and private messaging on your desktop. Send and receive messages and files with ease, all for free. Sep 23, 2025 \u00b7 Chief Economists' Outlook 2025 states that these under-the-radar trends are shaping the global economy, from the impact of AI on research productivity to a tightened labor \u2026 Sep 1, 2025 \u00b7 THE AI DEVELOPMENT REVOLUTION: WHAT'S REALLY HAPPENING? Let's face it - the software development world is changing faster than a React component re-rendering on \u2026 Oct 7, 2025 \u00b7 Computer Programming Research. Read current computer science articles on everything from computer programs to detect cancer genes and control vehicle maintenance to \u2026 Aug 24, 2025 \u00b7 Artificial Intelligence News. Everything on AI including futuristic robots with artificial intelligence, computer models of human intelligence and more.\nURL: https://duckduckgo.com/?q=What's+happening+in+the+software+development+world+today?\n\nUser Question: What's happening in the software development world today?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", - "prompt.prompt_length": 1477, - "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nLog in to WhatsApp Web for simple, reliable and private messaging on your desktop. Send and receive messages and files with ease, all for free. Sep 23, 2025 \u00b7 Chief Economi...", - "prompt.instruction_following": "web_search_optimized", - "response.answer_length": 1636, - "response.answer_completeness": "complete", - "response.response_quality": "medium", - "response.source_citation": 1, - "response.context_utilization": 1.677948717948718, - "output_response": "Web Search Answer: The software development world is currently experiencing rapid changes driven by advancements in artificial intelligence (AI) and evolving programming practices. Key trends include:\n\n1. **AI Integrati...", - "evaluation.search_effectiveness": 0.2, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 1, - "evaluation.context_relevance": "high", - "evaluation.overall_quality": "low", - "evaluation.ready_for_production": true, - "web_search.search_strategy": "duckduckgo_api", - "web_search.real_time_data": true, - "web_search.external_sources": 1, - "web_search.information_freshness": "current", - "web_search.knowledge_synthesis": "single_source" - }, - "events": [], - "links": [], - "trace_trace_id": "2ba8cab0-2366-4add-8ac3-b490a05e9a60", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-14 12:53:56.480000000", - "trace_end_time": "2025-10-14 12:54:05.687000000", - "trace_duration_ms": 9207.67, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-14 12:53:56.480000000", - "updated_at": "2025-10-14 12:53:56.480000000" - }, - { - "span_id": "3d39f522-8ae7-4988-aafd-58b09734eff0", - "trace_id": "a7c3399e-372f-4aae-baeb-05215cc4b85f", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-14 12:53:50.963000000", - "end_time": "2025-10-14 12:53:55.471000000", - "duration_ms": 4507.466, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What are the newest features in cloud computing?", - "query.length": 48, - "processing.mode": "RAG", - "processing.time_seconds": 4.507421255111694, - "processing.response_length": 870, - "processing.sources_count": 5, - "output_response": "Final Answer: The provided context does not specifically mention any new features in cloud computing as a general topic. However, it does highlight several recent enhancements related to the Noveum.ai platform, whi...", - "final_answer_mode": "RAG", - "query_processed.input_query": "What are the newest features in cloud computing?", - "query_processed.output_response": "Successfully processed query using RAG, generated 870 character response", - "query_processed.mode": "RAG", - "query_processed.processing_time": 4.507421255111694, - "query_processed.response_length": 870 - }, - "events": [], - "links": [], - "trace_trace_id": "a7c3399e-372f-4aae-baeb-05215cc4b85f", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-14 12:53:50.963000000", - "trace_end_time": "2025-10-14 12:53:55.471000000", - "trace_duration_ms": 4507.555, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-14 12:53:50.963000000", - "updated_at": "2025-10-14 12:53:50.963000000" - }, - { - "span_id": "54aa6700-f2de-4cf0-b1ad-a3d24ed692ed", - "trace_id": "a7c3399e-372f-4aae-baeb-05215cc4b85f", - "parent_span_id": "d17f31cf-c570-4604-8df3-532b2e530fce", - "name": "agent.llm-rag", - "start_time": "2025-10-14 12:53:50.963000000", - "end_time": "2025-10-14 12:53:55.471000000", - "duration_ms": 4507.083, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "rag_agent", - "agent.operation": "llm-rag", - "agent.capabilities": [ - "document_retrieval", - "context_generation", - "response_generation" - ], - "agent.id": "noveum_rag_agent", - "input_query": "What are the newest features in cloud computing?", - "query_length": 48, - "query_type": "rag_query", - "output_response": "RAG Answer: The provided context does not specifically mention any new features in cloud computing as a general topic. However, it does highlight several recent enhancements related to the Noveum.ai platform, whi...", - "rag.context_length": 2579, - "rag.sources_count": 5, - "rag.answer_length": 870, - "rag.mode": "retrieval_augmented_generation" - }, - "events": [], - "links": [], - "trace_trace_id": "a7c3399e-372f-4aae-baeb-05215cc4b85f", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-14 12:53:50.963000000", - "trace_end_time": "2025-10-14 12:53:55.471000000", - "trace_duration_ms": 4507.555, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-14 12:53:50.963000000", - "updated_at": "2025-10-14 12:53:50.963000000" - }, - { - "span_id": "8878096f-6cfa-4f70-9844-d7723a0875f0", - "trace_id": "a7c3399e-372f-4aae-baeb-05215cc4b85f", - "parent_span_id": "d17f31cf-c570-4604-8df3-532b2e530fce", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-14 12:53:50.963000000", - "end_time": "2025-10-14 12:53:50.963000000", - "duration_ms": 0.036000000000000004, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What are the newest features in cloud computing?", - "query_length": 48, - "query_type": "routing_evaluation_query", - "classification.mode": "RAG", - "classification.rag_keyword_score": 1, - "classification.web_keyword_score": 0, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "documentation", - "query.keyword_density": 0.125, - "query.domain_affinity": "noveum", - "routing.decision": "Routed to RAG based on analysis", - "routing.rationale": "RAG score: 1, Web score: 0, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "Web Search", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.019230769230769232, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: RAG (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "a7c3399e-372f-4aae-baeb-05215cc4b85f", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-14 12:53:50.963000000", - "trace_end_time": "2025-10-14 12:53:55.471000000", - "trace_duration_ms": 4507.555, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-14 12:53:50.963000000", - "updated_at": "2025-10-14 12:53:50.963000000" - }, - { - "span_id": "d17f31cf-c570-4604-8df3-532b2e530fce", - "trace_id": "a7c3399e-372f-4aae-baeb-05215cc4b85f", - "parent_span_id": "3d39f522-8ae7-4988-aafd-58b09734eff0", - "name": "agent.query_routing", - "start_time": "2025-10-14 12:53:50.963000000", - "end_time": "2025-10-14 12:53:55.471000000", - "duration_ms": 4507.3730000000005, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What are the newest features in cloud computing?", - "query_length": 48, - "query_type": "routing_query", - "output_response": "Routed to RAG for query processing", - "router.classification": "RAG", - "router.confidence_score": 1, - "router.rag_keyword_score": 1, - "router.web_keyword_score": 0, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "a7c3399e-372f-4aae-baeb-05215cc4b85f", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-14 12:53:50.963000000", - "trace_end_time": "2025-10-14 12:53:55.471000000", - "trace_duration_ms": 4507.555, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-14 12:53:50.963000000", - "updated_at": "2025-10-14 12:53:50.963000000" - }, - { - "span_id": "97ef01a8-5930-4902-909a-245c516a5871", - "trace_id": "a7c3399e-372f-4aae-baeb-05215cc4b85f", - "parent_span_id": "54aa6700-f2de-4cf0-b1ad-a3d24ed692ed", - "name": "agent.llm_model_execution", - "start_time": "2025-10-14 12:53:51.439000000", - "end_time": "2025-10-14 12:53:55.077000000", - "duration_ms": 3637.337, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What are the newest features in cloud computing?", - "query_length": 48, - "query_type": "rag_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 3.6372909545898438, - "model.latency_ms": 3637.2909545898438, - "model.start_time": 1760446431.439748, - "model.end_time": 1760446435.077039, - "model.performance_tier": "medium", - "model.prompt_tokens": 780, - "model.completion_tokens": 217, - "model.total_tokens": 997, - "model.tokens_per_second": 274.1050997698989, - "model.estimated_cost": 0.02991, - "model.efficiency_score": 0.872617853560682, - "model.response_length": 870, - "model.response_quality": "high", - "model.output_response": "Model Response: The provided context does not specifically mention any new features in cloud computing as a general topic. However, it does highlight several recent enhancements related to the Noveum.ai platform, whi...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "a7c3399e-372f-4aae-baeb-05215cc4b85f", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-14 12:53:50.963000000", - "trace_end_time": "2025-10-14 12:53:55.471000000", - "trace_duration_ms": 4507.555, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-14 12:53:50.963000000", - "updated_at": "2025-10-14 12:53:50.963000000" - }, - { - "span_id": "ea792ec6-64b4-4ed5-b418-ba902e90c57a", - "trace_id": "a7c3399e-372f-4aae-baeb-05215cc4b85f", - "parent_span_id": "54aa6700-f2de-4cf0-b1ad-a3d24ed692ed", - "name": "agent.rag_evaluation_metrics", - "start_time": "2025-10-14 12:53:55.077000000", - "end_time": "2025-10-14 12:53:55.077000000", - "duration_ms": 0.034, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "rag_evaluation_metrics", - "agent.capabilities": [ - "retrieval_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What are the newest features in cloud computing?", - "query_length": 48, - "query_type": "rag_evaluation_query", - "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/changelog):\n- \ud83d\udcc8 Enhanced Dashboard Analytics: Real-time request tracking, latency monitoring, and cost analysis\n- \ud83d\udd0d Improved Logs Interface: Better search functionality, detailed trace views, and enhanced debugging capabilities\n- \ud83d\udc65 Team Management: Advanced member mana...", - "retrieval.context_length": 2579, - "retrieval.sources_count": 5, - "retrieval.context_quality": "high", - "retrieval.effectiveness": 1, - "retrieval.context_utilization": 2.579, - "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/changelog):\n- \ud83d\udcc8 Enhanced Dashboard Analytics: Real-time request tracking, latency monitoring, and cost analysis\n- \ud83d\udd0d Improved Logs Interface: Better search functionality, detailed trace views, and enhanced debugging capabilities\n- \ud83d\udc65 Team Management: Advanced member management with role-based access control and invitation system\n- \ud83d\udee1\ufe0f Security Improvements: Enhanced API key management and secure credential storage\n4 months ago\n- \ud83d\udcca Advanced Metrics Collection: Comprehensive telemetry system with custom metrics ...\n\nSource 2 (https://noveum.ai/en/changelog):\n- \ud83d\udcdd Documentation Updates: Comprehensive guides for integration and best practices\n5 months ago\n- \ud83c\udf89 Platform Foundation: Initial release of Noveum.ai AI observability platform\n- \ud83d\ude80 Core Features: AI Gateway integration, basic evaluation metrics, and monitoring capabilities\n- \ud83d\udcc8 Evaluation Jobs: Automated model evaluation with configurable metrics and reporting\n- \ud83d\udd0c Provider Integrations: Support for major AI providers and custom model deployments...\n\nSource 3 (https://noveum.ai/docs/platform/dashboard):\n- Bulk Export: Download multiple traces simultaneously\n- Comparative Analysis: Compare performance across multiple traces\nReal-Time Updates\n- Live Refresh: Automatic updates as new traces arrive\n- Connection Monitoring: Real-time status of your trace ingestion pipeline\n- Performance Indicators: Live metrics for system health monitoring\nReady to dive deeper? Explore Projects & Environments to organize your AI applications, or check out Team Collaboration to share insights with your team.\nGet Earl...\n\nSource 4 (https://noveum.ai/en/changelog):\nChangelog\nStay up to date with the latest changes in our product.\n2 months ago\n- \ud83c\udfaf Major Platform Evolution: NovaEval Framework - Launched comprehensive AI model evaluation framework with 20+ built-in scorers including accuracy, RAG metrics, conversational scoring, and specialized agent evaluation tools. Features production-grade deployment on Docker and Kubernetes. (GitHub | PyPI Package)\n- \ud83d\ude80 Next-Generation Traces UI - Complete redesign featuring three-pane layout with directory tree navigatio...\n\nSource 5 (https://noveum.ai/en/blog):\n8/12/2025\nDiscover how Noveum.ai provides comprehensive tracing and observability for AI applications, from development debugging to production optimization.\nShashank Agarwal\n3/3/2025\nDiscover how Noveum.ai provides comprehensive tracing and observability for LLM applications, RAG systems, and multi-agent workflows with our powerful Python and TypeScript SDKs.\nShashank Agarwal\n3/2/2025...\n\nUser Question: What are the newest features in cloud computing?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", - "prompt.prompt_length": 3123, - "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/changelog):\n- \ud83d\udcc8 Enhanced Dashboard Analytics: Real-time request tracking, latency monitoring, and cost analysis\n- \ud83d\udd0d Improved Logs Interface: Better search functionality,...", - "prompt.instruction_following": "rag_optimized", - "response.answer_length": 870, - "response.answer_completeness": "complete", - "response.response_quality": "high", - "response.source_citation": 5, - "response.context_utilization": 0.33734005428460645, - "output_response": "RAG Answer: The provided context does not specifically mention any new features in cloud computing as a general topic. However, it does highlight several recent enhancements related to the Noveum.ai platform, whi...", - "evaluation.retrieval_effectiveness": 1, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 5, - "evaluation.context_relevance": "high", - "evaluation.overall_quality": "high", - "evaluation.ready_for_production": true, - "rag.retrieval_strategy": "semantic_similarity", - "rag.vector_search_results": 5, - "rag.context_synthesis": "multi_source", - "rag.document_coverage": 1, - "rag.information_density": 0.33734005428460645 - }, - "events": [], - "links": [], - "trace_trace_id": "a7c3399e-372f-4aae-baeb-05215cc4b85f", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-14 12:53:50.963000000", - "trace_end_time": "2025-10-14 12:53:55.471000000", - "trace_duration_ms": 4507.555, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-14 12:53:50.963000000", - "updated_at": "2025-10-14 12:53:50.963000000" - }, - { - "span_id": "e2d314c8-687f-4b8d-869c-8d7f2a22a4f3", - "trace_id": "d682797f-2ce0-448e-8f0d-891541430988", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-14 12:53:44.917000000", - "end_time": "2025-10-14 12:53:49.957000000", - "duration_ms": 5039.565, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What's the current status of cryptocurrency markets?", - "query.length": 52, - "processing.mode": "Web Search", - "processing.time_seconds": 5.0394370555877686, - "processing.response_length": 338, - "processing.sources_count": 1, - "output_response": "Final Answer: The web search results did not provide any specific information regarding the current status of cryptocurrency markets. Therefore, I cannot provide an update on market conditions, prices, or trends at...", - "final_answer_mode": "Web Search", - "query_processed.input_query": "What's the current status of cryptocurrency markets?", - "query_processed.output_response": "Successfully processed query using Web Search, generated 338 character response", - "query_processed.mode": "Web Search", - "query_processed.processing_time": 5.0394370555877686, - "query_processed.response_length": 338 - }, - "events": [], - "links": [], - "trace_trace_id": "d682797f-2ce0-448e-8f0d-891541430988", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-14 12:53:44.917000000", - "trace_end_time": "2025-10-14 12:53:49.957000000", - "trace_duration_ms": 5039.64, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-14 12:53:44.917000000", - "updated_at": "2025-10-14 12:53:44.917000000" - }, - { - "span_id": "d367a6bd-86af-41c5-96ca-5284425775ce", - "trace_id": "d682797f-2ce0-448e-8f0d-891541430988", - "parent_span_id": "f137a412-6ddf-4ace-908e-9edf73534aa7", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-14 12:53:44.917000000", - "end_time": "2025-10-14 12:53:44.918000000", - "duration_ms": 0.046, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What's the current status of cryptocurrency markets?", - "query_length": 52, - "query_type": "routing_evaluation_query", - "classification.mode": "Web Search", - "classification.rag_keyword_score": 0, - "classification.web_keyword_score": 3, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "complex", - "query.intent": "general_knowledge", - "query.keyword_density": 0.42857142857142855, - "query.domain_affinity": "general", - "routing.decision": "Routed to Web Search based on analysis", - "routing.rationale": "RAG score: 0, Web score: 3, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "RAG", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.057692307692307696, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: Web Search (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "d682797f-2ce0-448e-8f0d-891541430988", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-14 12:53:44.917000000", - "trace_end_time": "2025-10-14 12:53:49.957000000", - "trace_duration_ms": 5039.64, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-14 12:53:44.917000000", - "updated_at": "2025-10-14 12:53:44.917000000" - }, - { - "span_id": "f137a412-6ddf-4ace-908e-9edf73534aa7", - "trace_id": "d682797f-2ce0-448e-8f0d-891541430988", - "parent_span_id": "e2d314c8-687f-4b8d-869c-8d7f2a22a4f3", - "name": "agent.query_routing", - "start_time": "2025-10-14 12:53:44.917000000", - "end_time": "2025-10-14 12:53:49.957000000", - "duration_ms": 5039.404, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What's the current status of cryptocurrency markets?", - "query_length": 52, - "query_type": "routing_query", - "output_response": "Routed to Web Search for query processing", - "router.classification": "Web Search", - "router.confidence_score": 1, - "router.rag_keyword_score": 0, - "router.web_keyword_score": 3, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "d682797f-2ce0-448e-8f0d-891541430988", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-14 12:53:44.917000000", - "trace_end_time": "2025-10-14 12:53:49.957000000", - "trace_duration_ms": 5039.64, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-14 12:53:44.917000000", - "updated_at": "2025-10-14 12:53:44.917000000" - }, - { - "span_id": "987b6910-9b6d-4207-b0f3-8807a23e3ddb", - "trace_id": "d682797f-2ce0-448e-8f0d-891541430988", - "parent_span_id": "f137a412-6ddf-4ace-908e-9edf73534aa7", - "name": "agent.web_search_generation", - "start_time": "2025-10-14 12:53:44.918000000", - "end_time": "2025-10-14 12:53:49.957000000", - "duration_ms": 5039.248, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "web_search_agent", - "agent.operation": "web_search_generation", - "agent.capabilities": [ - "web_search", - "content_synthesis", - "response_generation" - ], - "agent.id": "noveum_web_search_agent", - "input_query": "What's the current status of cryptocurrency markets?", - "query_length": 52, - "query_type": "web_search_query", - "output_response": "Web Search Answer: The web search results did not provide any specific information regarding the current status of cryptocurrency markets. Therefore, I cannot provide an update on market conditions, prices, or trends at...", - "web_search.results_count": 1, - "web_search.context_length": 154, - "web_search.response_length": 338, - "web_search.mode": "external_web_search" - }, - "events": [], - "links": [], - "trace_trace_id": "d682797f-2ce0-448e-8f0d-891541430988", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-14 12:53:44.917000000", - "trace_end_time": "2025-10-14 12:53:49.957000000", - "trace_duration_ms": 5039.64, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-14 12:53:44.917000000", - "updated_at": "2025-10-14 12:53:44.917000000" - }, - { - "span_id": "b2614d0e-5fcd-46ec-8d25-f1b04eb142af", - "trace_id": "d682797f-2ce0-448e-8f0d-891541430988", - "parent_span_id": "987b6910-9b6d-4207-b0f3-8807a23e3ddb", - "name": "agent.llm_model_execution", - "start_time": "2025-10-14 12:53:45.255000000", - "end_time": "2025-10-14 12:53:49.957000000", - "duration_ms": 4701.744, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What's the current status of cryptocurrency markets?", - "query_length": 52, - "query_type": "web_search_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 4.701629161834717, - "model.latency_ms": 4701.629161834717, - "model.start_time": 1760446425.2554948, - "model.end_time": 1760446429.957124, - "model.performance_tier": "medium", - "model.prompt_tokens": 162, - "model.completion_tokens": 84, - "model.total_tokens": 246, - "model.tokens_per_second": 52.32228904756993, - "model.estimated_cost": 0.00738, - "model.efficiency_score": 1.3739837398373984, - "model.response_length": 338, - "model.response_quality": "high", - "model.output_response": "Model Response: The web search results did not provide any specific information regarding the current status of cryptocurrency markets. Therefore, I cannot provide an update on market conditions, prices, or trends at...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "d682797f-2ce0-448e-8f0d-891541430988", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-14 12:53:44.917000000", - "trace_end_time": "2025-10-14 12:53:49.957000000", - "trace_duration_ms": 5039.64, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-14 12:53:44.917000000", - "updated_at": "2025-10-14 12:53:44.917000000" - }, - { - "span_id": "663e8789-75d0-4c4d-b14d-4088022436ac", - "trace_id": "d682797f-2ce0-448e-8f0d-891541430988", - "parent_span_id": "987b6910-9b6d-4207-b0f3-8807a23e3ddb", - "name": "agent.web_search_evaluation_metrics", - "start_time": "2025-10-14 12:53:49.957000000", - "end_time": "2025-10-14 12:53:49.957000000", - "duration_ms": 0.032, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "web_search_evaluation_metrics", - "agent.capabilities": [ - "web_search_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What's the current status of cryptocurrency markets?", - "query_length": 52, - "query_type": "web_search_evaluation_query", - "web_search.results_count": 1, - "web_search.context_length": 154, - "web_search.context_synthesized": "Context from 1 web sources", - "web_search.search_effectiveness": 0.2, - "web_search.context_quality": "low", - "web_search.source_diversity": 1, - "web_search.information_synthesis": "low", - "web_search.external_knowledge_utilization": 0.154, - "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What's+the+current+status+of+cryptocurrency+markets?\n\nUser Question: What's the current status of cryptocurrency markets?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", - "prompt.prompt_length": 651, - "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What's+the+current+status+of+cryptocurrency+markets?", - "prompt.instruction_following": "web_search_optimized", - "response.answer_length": 338, - "response.answer_completeness": "complete", - "response.response_quality": "medium", - "response.source_citation": 1, - "response.context_utilization": 2.1948051948051948, - "output_response": "Web Search Answer: The web search results did not provide any specific information regarding the current status of cryptocurrency markets. Therefore, I cannot provide an update on market conditions, prices, or trends at...", - "evaluation.search_effectiveness": 0.2, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 1, - "evaluation.context_relevance": "low", - "evaluation.overall_quality": "low", - "evaluation.ready_for_production": true, - "web_search.search_strategy": "duckduckgo_api", - "web_search.real_time_data": true, - "web_search.external_sources": 1, - "web_search.information_freshness": "current", - "web_search.knowledge_synthesis": "single_source" - }, - "events": [], - "links": [], - "trace_trace_id": "d682797f-2ce0-448e-8f0d-891541430988", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-14 12:53:44.917000000", - "trace_end_time": "2025-10-14 12:53:49.957000000", - "trace_duration_ms": 5039.64, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-14 12:53:44.917000000", - "updated_at": "2025-10-14 12:53:44.917000000" - }, - { - "span_id": "3382fe7c-aaca-46e0-a8d2-dbee75ff3bed", - "trace_id": "b0858dca-4173-493b-a6c5-4e993f606477", - "parent_span_id": "6cf94049-87b0-4fc5-b37e-32d261543647", - "name": "agent.query_routing", - "start_time": "2025-10-14 12:53:37.818000000", - "end_time": "2025-10-14 12:53:43.908000000", - "duration_ms": 6089.22, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What are the latest updates in Python programming?", - "query_length": 50, - "query_type": "routing_query", - "output_response": "Routed to Web Search for query processing", - "router.classification": "Web Search", - "router.confidence_score": 1, - "router.rag_keyword_score": 0, - "router.web_keyword_score": 2, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "b0858dca-4173-493b-a6c5-4e993f606477", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-14 12:53:37.818000000", - "trace_end_time": "2025-10-14 12:53:43.908000000", - "trace_duration_ms": 6089.715, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-14 12:53:37.818000000", - "updated_at": "2025-10-14 12:53:37.818000000" - }, - { - "span_id": "6cf94049-87b0-4fc5-b37e-32d261543647", - "trace_id": "b0858dca-4173-493b-a6c5-4e993f606477", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-14 12:53:37.818000000", - "end_time": "2025-10-14 12:53:43.908000000", - "duration_ms": 6089.607, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What are the latest updates in Python programming?", - "query.length": 50, - "processing.mode": "Web Search", - "processing.time_seconds": 6.089300870895386, - "processing.response_length": 1324, - "processing.sources_count": 1, - "output_response": "Final Answer: The web search did not yield specific results regarding the latest updates in Python programming. However, I can provide a general overview based on my knowledge up to October 2023.\n\nAs of late 2023, ...", - "final_answer_mode": "Web Search", - "query_processed.input_query": "What are the latest updates in Python programming?", - "query_processed.output_response": "Successfully processed query using Web Search, generated 1324 character response", - "query_processed.mode": "Web Search", - "query_processed.processing_time": 6.089300870895386, - "query_processed.response_length": 1324 - }, - "events": [], - "links": [], - "trace_trace_id": "b0858dca-4173-493b-a6c5-4e993f606477", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-14 12:53:37.818000000", - "trace_end_time": "2025-10-14 12:53:43.908000000", - "trace_duration_ms": 6089.715, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-14 12:53:37.818000000", - "updated_at": "2025-10-14 12:53:37.818000000" - }, - { - "span_id": "21c8befa-356a-4034-8d62-792238cb9e6e", - "trace_id": "b0858dca-4173-493b-a6c5-4e993f606477", - "parent_span_id": "3382fe7c-aaca-46e0-a8d2-dbee75ff3bed", - "name": "agent.web_search_generation", - "start_time": "2025-10-14 12:53:37.819000000", - "end_time": "2025-10-14 12:53:43.908000000", - "duration_ms": 6088.807, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "web_search_agent", - "agent.operation": "web_search_generation", - "agent.capabilities": [ - "web_search", - "content_synthesis", - "response_generation" - ], - "agent.id": "noveum_web_search_agent", - "input_query": "What are the latest updates in Python programming?", - "query_length": 50, - "query_type": "web_search_query", - "output_response": "Web Search Answer: The web search did not yield specific results regarding the latest updates in Python programming. However, I can provide a general overview based on my knowledge up to October 2023.\n\nAs of late 2023, ...", - "web_search.results_count": 1, - "web_search.context_length": 152, - "web_search.response_length": 1324, - "web_search.mode": "external_web_search" - }, - "events": [], - "links": [], - "trace_trace_id": "b0858dca-4173-493b-a6c5-4e993f606477", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-14 12:53:37.818000000", - "trace_end_time": "2025-10-14 12:53:43.908000000", - "trace_duration_ms": 6089.715, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-14 12:53:37.818000000", - "updated_at": "2025-10-14 12:53:37.818000000" - }, - { - "span_id": "482feedf-52f4-4c6a-9186-d59898820caa", - "trace_id": "b0858dca-4173-493b-a6c5-4e993f606477", - "parent_span_id": "3382fe7c-aaca-46e0-a8d2-dbee75ff3bed", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-14 12:53:37.819000000", - "end_time": "2025-10-14 12:53:37.819000000", - "duration_ms": 0.039, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What are the latest updates in Python programming?", - "query_length": 50, - "query_type": "routing_evaluation_query", - "classification.mode": "Web Search", - "classification.rag_keyword_score": 0, - "classification.web_keyword_score": 2, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "general_knowledge", - "query.keyword_density": 0.25, - "query.domain_affinity": "general", - "routing.decision": "Routed to Web Search based on analysis", - "routing.rationale": "RAG score: 0, Web score: 2, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "RAG", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.038461538461538464, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: Web Search (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "b0858dca-4173-493b-a6c5-4e993f606477", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-14 12:53:37.818000000", - "trace_end_time": "2025-10-14 12:53:43.908000000", - "trace_duration_ms": 6089.715, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-14 12:53:37.818000000", - "updated_at": "2025-10-14 12:53:37.818000000" - }, - { - "span_id": "84f1f63f-ee05-411e-a7dd-4ee9e9826f5a", - "trace_id": "b0858dca-4173-493b-a6c5-4e993f606477", - "parent_span_id": "21c8befa-356a-4034-8d62-792238cb9e6e", - "name": "agent.llm_model_execution", - "start_time": "2025-10-14 12:53:38.191000000", - "end_time": "2025-10-14 12:53:43.907000000", - "duration_ms": 5716.078, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What are the latest updates in Python programming?", - "query_length": 50, - "query_type": "web_search_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 5.715963125228882, - "model.latency_ms": 5715.963125228882, - "model.start_time": 1760446418.191805, - "model.end_time": 1760446423.907768, - "model.performance_tier": "slow", - "model.prompt_tokens": 161, - "model.completion_tokens": 331, - "model.total_tokens": 492, - "model.tokens_per_second": 86.07473302765561, - "model.estimated_cost": 0.01476, - "model.efficiency_score": 2.6910569105691056, - "model.response_length": 1324, - "model.response_quality": "high", - "model.output_response": "Model Response: The web search did not yield specific results regarding the latest updates in Python programming. However, I can provide a general overview based on my knowledge up to October 2023.\n\nAs of late 2023, ...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "b0858dca-4173-493b-a6c5-4e993f606477", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-14 12:53:37.818000000", - "trace_end_time": "2025-10-14 12:53:43.908000000", - "trace_duration_ms": 6089.715, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-14 12:53:37.818000000", - "updated_at": "2025-10-14 12:53:37.818000000" - }, - { - "span_id": "2fb9ecf3-a497-4285-a694-1c95080d9e6a", - "trace_id": "b0858dca-4173-493b-a6c5-4e993f606477", - "parent_span_id": "21c8befa-356a-4034-8d62-792238cb9e6e", - "name": "agent.web_search_evaluation_metrics", - "start_time": "2025-10-14 12:53:43.908000000", - "end_time": "2025-10-14 12:53:43.908000000", - "duration_ms": 0.037, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "web_search_evaluation_metrics", - "agent.capabilities": [ - "web_search_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What are the latest updates in Python programming?", - "query_length": 50, - "query_type": "web_search_evaluation_query", - "web_search.results_count": 1, - "web_search.context_length": 152, - "web_search.context_synthesized": "Context from 1 web sources", - "web_search.search_effectiveness": 0.2, - "web_search.context_quality": "low", - "web_search.source_diversity": 1, - "web_search.information_synthesis": "low", - "web_search.external_knowledge_utilization": 0.152, - "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What+are+the+latest+updates+in+Python+programming?\n\nUser Question: What are the latest updates in Python programming?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", - "prompt.prompt_length": 647, - "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What+are+the+latest+updates+in+Python+programming?", - "prompt.instruction_following": "web_search_optimized", - "response.answer_length": 1324, - "response.answer_completeness": "complete", - "response.response_quality": "medium", - "response.source_citation": 1, - "response.context_utilization": 8.710526315789474, - "output_response": "Web Search Answer: The web search did not yield specific results regarding the latest updates in Python programming. However, I can provide a general overview based on my knowledge up to October 2023.\n\nAs of late 2023, ...", - "evaluation.search_effectiveness": 0.2, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 1, - "evaluation.context_relevance": "low", - "evaluation.overall_quality": "low", - "evaluation.ready_for_production": true, - "web_search.search_strategy": "duckduckgo_api", - "web_search.real_time_data": true, - "web_search.external_sources": 1, - "web_search.information_freshness": "current", - "web_search.knowledge_synthesis": "single_source" - }, - "events": [], - "links": [], - "trace_trace_id": "b0858dca-4173-493b-a6c5-4e993f606477", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-14 12:53:37.818000000", - "trace_end_time": "2025-10-14 12:53:43.908000000", - "trace_duration_ms": 6089.715, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-14 12:53:37.818000000", - "updated_at": "2025-10-14 12:53:37.818000000" - }, - { - "span_id": "9dd5ac2c-0deb-48e3-ac8b-0e6e31f7ff30", - "trace_id": "dc81be1a-e93d-4f60-90f3-ce1acaa8cb3e", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-14 12:53:34.584000000", - "end_time": "2025-10-14 12:53:36.780000000", - "duration_ms": 2196.351, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What happened in tech news this week?", - "query.length": 37, - "processing.mode": "Web Search", - "processing.time_seconds": 2.1948978900909424, - "processing.response_length": 324, - "processing.sources_count": 1, - "output_response": "Final Answer: I'm sorry, but I couldn't find any specific information regarding tech news from this week based on the search results. It appears that there were no relevant articles or updates available. If you're ...", - "final_answer_mode": "Web Search", - "query_processed.input_query": "What happened in tech news this week?", - "query_processed.output_response": "Successfully processed query using Web Search, generated 324 character response", - "query_processed.mode": "Web Search", - "query_processed.processing_time": 2.1948978900909424, - "query_processed.response_length": 324 - }, - "events": [], - "links": [], - "trace_trace_id": "dc81be1a-e93d-4f60-90f3-ce1acaa8cb3e", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-14 12:53:34.584000000", - "trace_end_time": "2025-10-14 12:53:36.780000000", - "trace_duration_ms": 2196.4579999999996, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-14 12:53:34.584000000", - "updated_at": "2025-10-14 12:53:34.584000000" - }, - { - "span_id": "b1cbafce-51fb-48fd-bc4a-d49081a2f374", - "trace_id": "dc81be1a-e93d-4f60-90f3-ce1acaa8cb3e", - "parent_span_id": "9dd5ac2c-0deb-48e3-ac8b-0e6e31f7ff30", - "name": "agent.query_routing", - "start_time": "2025-10-14 12:53:34.584000000", - "end_time": "2025-10-14 12:53:36.778000000", - "duration_ms": 2194.2200000000003, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What happened in tech news this week?", - "query_length": 37, - "query_type": "routing_query", - "output_response": "Routed to Web Search for query processing", - "router.classification": "Web Search", - "router.confidence_score": 1, - "router.rag_keyword_score": 0, - "router.web_keyword_score": 2, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "dc81be1a-e93d-4f60-90f3-ce1acaa8cb3e", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-14 12:53:34.584000000", - "trace_end_time": "2025-10-14 12:53:36.780000000", - "trace_duration_ms": 2196.4579999999996, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-14 12:53:34.584000000", - "updated_at": "2025-10-14 12:53:34.584000000" - }, - { - "span_id": "e3064687-1b18-478b-9e9d-4d081a6dac23", - "trace_id": "dc81be1a-e93d-4f60-90f3-ce1acaa8cb3e", - "parent_span_id": "b1cbafce-51fb-48fd-bc4a-d49081a2f374", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-14 12:53:34.584000000", - "end_time": "2025-10-14 12:53:34.584000000", - "duration_ms": 0.028, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What happened in tech news this week?", - "query_length": 37, - "query_type": "routing_evaluation_query", - "classification.mode": "Web Search", - "classification.rag_keyword_score": 0, - "classification.web_keyword_score": 2, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "general_knowledge", - "query.keyword_density": 0.2857142857142857, - "query.domain_affinity": "general", - "routing.decision": "Routed to Web Search based on analysis", - "routing.rationale": "RAG score: 0, Web score: 2, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "RAG", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.038461538461538464, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: Web Search (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "dc81be1a-e93d-4f60-90f3-ce1acaa8cb3e", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-14 12:53:34.584000000", - "trace_end_time": "2025-10-14 12:53:36.780000000", - "trace_duration_ms": 2196.4579999999996, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-14 12:53:34.584000000", - "updated_at": "2025-10-14 12:53:34.584000000" - }, - { - "span_id": "eb9dbc11-44b3-4041-8c83-0f33b8b761bb", - "trace_id": "dc81be1a-e93d-4f60-90f3-ce1acaa8cb3e", - "parent_span_id": "b1cbafce-51fb-48fd-bc4a-d49081a2f374", - "name": "agent.web_search_generation", - "start_time": "2025-10-14 12:53:34.584000000", - "end_time": "2025-10-14 12:53:36.774000000", - "duration_ms": 2189.866, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "web_search_agent", - "agent.operation": "web_search_generation", - "agent.capabilities": [ - "web_search", - "content_synthesis", - "response_generation" - ], - "agent.id": "noveum_web_search_agent", - "input_query": "What happened in tech news this week?", - "query_length": 37, - "query_type": "web_search_query", - "output_response": "Web Search Answer: I'm sorry, but I couldn't find any specific information regarding tech news from this week based on the search results. It appears that there were no relevant articles or updates available. If you're ...", - "web_search.results_count": 1, - "web_search.context_length": 139, - "web_search.response_length": 324, - "web_search.mode": "external_web_search" - }, - "events": [], - "links": [], - "trace_trace_id": "dc81be1a-e93d-4f60-90f3-ce1acaa8cb3e", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-14 12:53:34.584000000", - "trace_end_time": "2025-10-14 12:53:36.780000000", - "trace_duration_ms": 2196.4579999999996, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-14 12:53:34.584000000", - "updated_at": "2025-10-14 12:53:34.584000000" - }, - { - "span_id": "c8ee237b-ab5e-4b36-83f0-2f8478971898", - "trace_id": "dc81be1a-e93d-4f60-90f3-ce1acaa8cb3e", - "parent_span_id": "eb9dbc11-44b3-4041-8c83-0f33b8b761bb", - "name": "agent.llm_model_execution", - "start_time": "2025-10-14 12:53:34.981000000", - "end_time": "2025-10-14 12:53:36.768000000", - "duration_ms": 1786.467, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What happened in tech news this week?", - "query_length": 37, - "query_type": "web_search_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 1.785179853439331, - "model.latency_ms": 1785.179853439331, - "model.start_time": 1760446414.981964, - "model.end_time": 1760446416.767144, - "model.performance_tier": "fast", - "model.prompt_tokens": 155, - "model.completion_tokens": 81, - "model.total_tokens": 236, - "model.tokens_per_second": 132.19956495997977, - "model.estimated_cost": 0.00708, - "model.efficiency_score": 1.3728813559322033, - "model.response_length": 324, - "model.response_quality": "high", - "model.output_response": "Model Response: I'm sorry, but I couldn't find any specific information regarding tech news from this week based on the search results. It appears that there were no relevant articles or updates available. If you're ...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "dc81be1a-e93d-4f60-90f3-ce1acaa8cb3e", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-14 12:53:34.584000000", - "trace_end_time": "2025-10-14 12:53:36.780000000", - "trace_duration_ms": 2196.4579999999996, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-14 12:53:34.584000000", - "updated_at": "2025-10-14 12:53:34.584000000" - }, - { - "span_id": "9c8e1a08-61c8-409e-a79d-41c74a8d1505", - "trace_id": "dc81be1a-e93d-4f60-90f3-ce1acaa8cb3e", - "parent_span_id": "eb9dbc11-44b3-4041-8c83-0f33b8b761bb", - "name": "agent.web_search_evaluation_metrics", - "start_time": "2025-10-14 12:53:36.772000000", - "end_time": "2025-10-14 12:53:36.774000000", - "duration_ms": 1.5939999999999999, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "web_search_evaluation_metrics", - "agent.capabilities": [ - "web_search_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What happened in tech news this week?", - "query_length": 37, - "query_type": "web_search_evaluation_query", - "web_search.results_count": 1, - "web_search.context_length": 139, - "web_search.context_synthesized": "Context from 1 web sources", - "web_search.search_effectiveness": 0.2, - "web_search.context_quality": "low", - "web_search.source_diversity": 1, - "web_search.information_synthesis": "low", - "web_search.external_knowledge_utilization": 0.139, - "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What+happened+in+tech+news+this+week?\n\nUser Question: What happened in tech news this week?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", - "prompt.prompt_length": 621, - "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What+happened+in+tech+news+this+week?", - "prompt.instruction_following": "web_search_optimized", - "response.answer_length": 324, - "response.answer_completeness": "complete", - "response.response_quality": "medium", - "response.source_citation": 1, - "response.context_utilization": 2.3309352517985613, - "output_response": "Web Search Answer: I'm sorry, but I couldn't find any specific information regarding tech news from this week based on the search results. It appears that there were no relevant articles or updates available. If you're ...", - "evaluation.search_effectiveness": 0.2, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 1, - "evaluation.context_relevance": "low", - "evaluation.overall_quality": "low", - "evaluation.ready_for_production": true, - "web_search.search_strategy": "duckduckgo_api", - "web_search.real_time_data": true, - "web_search.external_sources": 1, - "web_search.information_freshness": "current", - "web_search.knowledge_synthesis": "single_source" - }, - "events": [], - "links": [], - "trace_trace_id": "dc81be1a-e93d-4f60-90f3-ce1acaa8cb3e", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-14 12:53:34.584000000", - "trace_end_time": "2025-10-14 12:53:36.780000000", - "trace_duration_ms": 2196.4579999999996, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-14 12:53:34.584000000", - "updated_at": "2025-10-14 12:53:34.584000000" - }, - { - "span_id": "59595232-8873-419c-9b6a-0727445a2afa", - "trace_id": "945b2757-8a5a-4aab-a869-8f44c02166a5", - "parent_span_id": "dfde2f79-d0c1-481e-819d-60730cb60fe5", - "name": "agent.query_routing", - "start_time": "2025-10-14 12:53:27.771000000", - "end_time": "2025-10-14 12:53:33.575000000", - "duration_ms": 5803.9800000000005, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What are the current trends in observability tools?", - "query_length": 51, - "query_type": "routing_query", - "output_response": "Routed to RAG for query processing", - "router.classification": "RAG", - "router.confidence_score": 0.3333333333333333, - "router.rag_keyword_score": 2, - "router.web_keyword_score": 1, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "945b2757-8a5a-4aab-a869-8f44c02166a5", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-14 12:53:27.771000000", - "trace_end_time": "2025-10-14 12:53:33.575000000", - "trace_duration_ms": 5804.15, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-14 12:53:27.771000000", - "updated_at": "2025-10-14 12:53:27.771000000" - }, - { - "span_id": "a59b3296-aaba-4642-b5a0-a271787a5607", - "trace_id": "945b2757-8a5a-4aab-a869-8f44c02166a5", - "parent_span_id": "59595232-8873-419c-9b6a-0727445a2afa", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-14 12:53:27.771000000", - "end_time": "2025-10-14 12:53:27.771000000", - "duration_ms": 0.031, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What are the current trends in observability tools?", - "query_length": 51, - "query_type": "routing_evaluation_query", - "classification.mode": "RAG", - "classification.rag_keyword_score": 2, - "classification.web_keyword_score": 1, - "classification.confidence_score": 0.3333333333333333, - "classification.confidence_level": "medium", - "classification.method": "keyword_based", - "query.complexity": "complex", - "query.intent": "documentation", - "query.keyword_density": 0.375, - "query.domain_affinity": "noveum", - "routing.decision": "Routed to RAG based on analysis", - "routing.rationale": "RAG score: 2, Web score: 1, Confidence: 0.33", - "routing.expected_performance": "medium", - "routing.alternative_mode": "Web Search", - "routing.decision_confidence": 0.3333333333333333, - "evaluation.routing_accuracy": "medium", - "evaluation.keyword_coverage": 0.057692307692307696, - "evaluation.query_understanding": "ambiguous", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: RAG (Confidence: 0.33)" - }, - "events": [], - "links": [], - "trace_trace_id": "945b2757-8a5a-4aab-a869-8f44c02166a5", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-14 12:53:27.771000000", - "trace_end_time": "2025-10-14 12:53:33.575000000", - "trace_duration_ms": 5804.15, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-14 12:53:27.771000000", - "updated_at": "2025-10-14 12:53:27.771000000" - }, - { - "span_id": "dfde2f79-d0c1-481e-819d-60730cb60fe5", - "trace_id": "945b2757-8a5a-4aab-a869-8f44c02166a5", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-14 12:53:27.771000000", - "end_time": "2025-10-14 12:53:33.575000000", - "duration_ms": 5804.074, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What are the current trends in observability tools?", - "query.length": 51, - "processing.mode": "RAG", - "processing.time_seconds": 5.804032802581787, - "processing.response_length": 997, - "processing.sources_count": 5, - "output_response": "Final Answer: The provided context does not specifically outline current trends in observability tools in general. However, it highlights the unique challenges faced by AI applications and how traditional monitorin...", - "final_answer_mode": "RAG", - "query_processed.input_query": "What are the current trends in observability tools?", - "query_processed.output_response": "Successfully processed query using RAG, generated 997 character response", - "query_processed.mode": "RAG", - "query_processed.processing_time": 5.804032802581787, - "query_processed.response_length": 997 - }, - "events": [], - "links": [], - "trace_trace_id": "945b2757-8a5a-4aab-a869-8f44c02166a5", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-14 12:53:27.771000000", - "trace_end_time": "2025-10-14 12:53:33.575000000", - "trace_duration_ms": 5804.15, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-14 12:53:27.771000000", - "updated_at": "2025-10-14 12:53:27.771000000" - }, - { - "span_id": "82489444-deba-46d2-b08b-c09754189566", - "trace_id": "945b2757-8a5a-4aab-a869-8f44c02166a5", - "parent_span_id": "59595232-8873-419c-9b6a-0727445a2afa", - "name": "agent.llm-rag", - "start_time": "2025-10-14 12:53:27.772000000", - "end_time": "2025-10-14 12:53:33.575000000", - "duration_ms": 5803.534, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "rag_agent", - "agent.operation": "llm-rag", - "agent.capabilities": [ - "document_retrieval", - "context_generation", - "response_generation" - ], - "agent.id": "noveum_rag_agent", - "input_query": "What are the current trends in observability tools?", - "query_length": 51, - "query_type": "rag_query", - "output_response": "RAG Answer: The provided context does not specifically outline current trends in observability tools in general. However, it highlights the unique challenges faced by AI applications and how traditional monitorin...", - "rag.context_length": 2943, - "rag.sources_count": 5, - "rag.answer_length": 997, - "rag.mode": "retrieval_augmented_generation" - }, - "events": [], - "links": [], - "trace_trace_id": "945b2757-8a5a-4aab-a869-8f44c02166a5", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-14 12:53:27.771000000", - "trace_end_time": "2025-10-14 12:53:33.575000000", - "trace_duration_ms": 5804.15, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-14 12:53:27.771000000", - "updated_at": "2025-10-14 12:53:27.771000000" - }, - { - "span_id": "34b1f14c-3fd3-4f89-b4dd-6150a52befd7", - "trace_id": "945b2757-8a5a-4aab-a869-8f44c02166a5", - "parent_span_id": "82489444-deba-46d2-b08b-c09754189566", - "name": "agent.llm_model_execution", - "start_time": "2025-10-14 12:53:28.257000000", - "end_time": "2025-10-14 12:53:33.162000000", - "duration_ms": 4905.2429999999995, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What are the current trends in observability tools?", - "query_length": 51, - "query_type": "rag_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 4.9052040576934814, - "model.latency_ms": 4905.204057693481, - "model.start_time": 1760446408.2575018, - "model.end_time": 1760446413.162706, - "model.performance_tier": "medium", - "model.prompt_tokens": 872, - "model.completion_tokens": 249, - "model.total_tokens": 1121, - "model.tokens_per_second": 228.53279635569638, - "model.estimated_cost": 0.03363, - "model.efficiency_score": 0.8893844781445138, - "model.response_length": 997, - "model.response_quality": "high", - "model.output_response": "Model Response: The provided context does not specifically outline current trends in observability tools in general. However, it highlights the unique challenges faced by AI applications and how traditional monitorin...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "945b2757-8a5a-4aab-a869-8f44c02166a5", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-14 12:53:27.771000000", - "trace_end_time": "2025-10-14 12:53:33.575000000", - "trace_duration_ms": 5804.15, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-14 12:53:27.771000000", - "updated_at": "2025-10-14 12:53:27.771000000" - }, - { - "span_id": "d5d2b966-3c37-4d6d-b7b8-c58c9f22aad9", - "trace_id": "945b2757-8a5a-4aab-a869-8f44c02166a5", - "parent_span_id": "82489444-deba-46d2-b08b-c09754189566", - "name": "agent.rag_evaluation_metrics", - "start_time": "2025-10-14 12:53:33.162000000", - "end_time": "2025-10-14 12:53:33.162000000", - "duration_ms": 0.036000000000000004, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "rag_evaluation_metrics", - "agent.capabilities": [ - "retrieval_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What are the current trends in observability tools?", - "query_length": 51, - "query_type": "rag_evaluation_query", - "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/blog/from-logs-to-intelligent-choices-inside-noveum-ais-evaluation-process):\nFrom Development to Production - Inside Noveum.ai's AI Observability Platform\nShashank Agarwal\n3/3/2025\nIntroduction\nEvery AI application tells a story\u2014but most developers never get to hear it...", - "retrieval.context_length": 2943, - "retrieval.sources_count": 5, - "retrieval.context_quality": "high", - "retrieval.effectiveness": 1, - "retrieval.context_utilization": 2.943, - "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/blog/from-logs-to-intelligent-choices-inside-noveum-ais-evaluation-process):\nFrom Development to Production - Inside Noveum.ai's AI Observability Platform\nShashank Agarwal\n3/3/2025\nIntroduction\nEvery AI application tells a story\u2014but most developers never get to hear it. When your RAG pipeline returns irrelevant results, when your multi-agent system gets stuck in loops, or when your LLM costs suddenly spike, you're left guessing what went wrong and where.\nThat's exactly the challenge Noveum.ai solves. Rather than flying blind, our platform provides comprehensive tracing a...\n\nSource 2 (https://noveum.ai/docs/getting-started/tracing-concepts):\n- Explore Framework Integrations - Framework-specific guidance\n- Learn Advanced Patterns - Custom instrumentation techniques\n- Master the Dashboard - Analyze your traces effectively\nRemember: Good observability is not about collecting all possible data, but about collecting the right data that helps you understand, debug, and optimize your AI applications.\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to O...\n\nSource 3 (https://noveum.ai/en/docs/getting-started/tracing-concepts):\n- Explore Framework Integrations - Framework-specific guidance\n- Learn Advanced Patterns - Custom instrumentation techniques\n- Master the Dashboard - Analyze your traces effectively\nRemember: Good observability is not about collecting all possible data, but about collecting the right data that helps you understand, debug, and optimize your AI applications.\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to O...\n\nSource 4 (https://noveum.ai/en/blog/from-logs-to-intelligent-choices-inside-noveum-ais-evaluation-process):\nThe AI Observability Challenge\nWhy Traditional Monitoring Falls Short\nTraditional application monitoring tools weren't built for AI applications. They can tell you if your API is responding, but they can't answer the questions that matter most for AI systems:\n- Why did my RAG pipeline retrieve irrelevant documents?\n- Which LLM calls are driving my costs?\n- How are my agents communicating with each other?\n- What's causing hallucinations in my responses?\n- Why is my embedding generation so slow?\nW...\n\nSource 5 (https://noveum.ai/en/docs/advanced/rag-observability):\n\ud83d\udd27 Best Practices for RAG Observability\n1. Comprehensive Pipeline Tracing\n2. Quality Monitoring\n3. A/B Testing for RAG Components\n\ud83c\udfaf Advanced RAG Patterns\nMulti-Modal RAG\nConversational RAG\nRAG observability with Noveum.ai provides the deep insights needed to build, optimize, and scale retrieval-augmented generation systems. By implementing comprehensive tracing across retrieval, context preparation, and generation stages, you can ensure your RAG pipeline delivers accurate, relevant, and cost-effe...\n\nUser Question: What are the current trends in observability tools?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", - "prompt.prompt_length": 3490, - "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/blog/from-logs-to-intelligent-choices-inside-noveum-ais-evaluation-process):\nFrom Development to Production - Inside Noveum.ai's AI Observability Platform\nShashank Agarw...", - "prompt.instruction_following": "rag_optimized", - "response.answer_length": 997, - "response.answer_completeness": "complete", - "response.response_quality": "high", - "response.source_citation": 5, - "response.context_utilization": 0.33876996262317366, - "output_response": "RAG Answer: The provided context does not specifically outline current trends in observability tools in general. However, it highlights the unique challenges faced by AI applications and how traditional monitorin...", - "evaluation.retrieval_effectiveness": 1, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 5, - "evaluation.context_relevance": "high", - "evaluation.overall_quality": "high", - "evaluation.ready_for_production": true, - "rag.retrieval_strategy": "semantic_similarity", - "rag.vector_search_results": 5, - "rag.context_synthesis": "multi_source", - "rag.document_coverage": 1, - "rag.information_density": 0.33876996262317366 - }, - "events": [], - "links": [], - "trace_trace_id": "945b2757-8a5a-4aab-a869-8f44c02166a5", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-14 12:53:27.771000000", - "trace_end_time": "2025-10-14 12:53:33.575000000", - "trace_duration_ms": 5804.15, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-14 12:53:27.771000000", - "updated_at": "2025-10-14 12:53:27.771000000" - }, - { - "span_id": "11c07411-df30-42a4-9062-90a032ef75ed", - "trace_id": "4bdd5814-acbb-4de7-a56b-543b87a1cfa5", - "parent_span_id": "ec530838-a7cf-477b-9186-a70114c8a12f", - "name": "agent.web_search_generation", - "start_time": "2025-10-14 12:53:20.120000000", - "end_time": "2025-10-14 12:53:26.750000000", - "duration_ms": 6629.273999999999, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "web_search_agent", - "agent.operation": "web_search_generation", - "agent.capabilities": [ - "web_search", - "content_synthesis", - "response_generation" - ], - "agent.id": "noveum_web_search_agent", - "input_query": "Tell me about recent developments in machine learning", - "query_length": 53, - "query_type": "web_search_query", - "output_response": "Web Search Answer: The web search did not yield specific results regarding recent developments in machine learning. However, I can provide a general overview based on my knowledge up to October 2023.\n\nRecent development...", - "web_search.results_count": 1, - "web_search.context_length": 155, - "web_search.response_length": 1849, - "web_search.mode": "external_web_search" - }, - "events": [], - "links": [], - "trace_trace_id": "4bdd5814-acbb-4de7-a56b-543b87a1cfa5", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-14 12:53:20.120000000", - "trace_end_time": "2025-10-14 12:53:26.750000000", - "trace_duration_ms": 6630.38, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-14 12:53:20.120000000", - "updated_at": "2025-10-14 12:53:20.120000000" - }, - { - "span_id": "282b121f-61dd-4dc5-9e55-cecc3f02f688", - "trace_id": "4bdd5814-acbb-4de7-a56b-543b87a1cfa5", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-14 12:53:20.120000000", - "end_time": "2025-10-14 12:53:26.750000000", - "duration_ms": 6630.264, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "Tell me about recent developments in machine learning", - "query.length": 53, - "processing.mode": "Web Search", - "processing.time_seconds": 6.629686117172241, - "processing.response_length": 1849, - "processing.sources_count": 1, - "output_response": "Final Answer: The web search did not yield specific results regarding recent developments in machine learning. However, I can provide a general overview based on my knowledge up to October 2023.\n\nRecent development...", - "final_answer_mode": "Web Search", - "query_processed.input_query": "Tell me about recent developments in machine learning", - "query_processed.output_response": "Successfully processed query using Web Search, generated 1849 character response", - "query_processed.mode": "Web Search", - "query_processed.processing_time": 6.629686117172241, - "query_processed.response_length": 1849 - }, - "events": [], - "links": [], - "trace_trace_id": "4bdd5814-acbb-4de7-a56b-543b87a1cfa5", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-14 12:53:20.120000000", - "trace_end_time": "2025-10-14 12:53:26.750000000", - "trace_duration_ms": 6630.38, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-14 12:53:20.120000000", - "updated_at": "2025-10-14 12:53:20.120000000" - }, - { - "span_id": "ec530838-a7cf-477b-9186-a70114c8a12f", - "trace_id": "4bdd5814-acbb-4de7-a56b-543b87a1cfa5", - "parent_span_id": "282b121f-61dd-4dc5-9e55-cecc3f02f688", - "name": "agent.query_routing", - "start_time": "2025-10-14 12:53:20.120000000", - "end_time": "2025-10-14 12:53:26.750000000", - "duration_ms": 6629.59, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "Tell me about recent developments in machine learning", - "query_length": 53, - "query_type": "routing_query", - "output_response": "Routed to Web Search for query processing", - "router.classification": "Web Search", - "router.confidence_score": 1, - "router.rag_keyword_score": 0, - "router.web_keyword_score": 1, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [], - "trace_trace_id": "4bdd5814-acbb-4de7-a56b-543b87a1cfa5", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-14 12:53:20.120000000", - "trace_end_time": "2025-10-14 12:53:26.750000000", - "trace_duration_ms": 6630.38, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-14 12:53:20.120000000", - "updated_at": "2025-10-14 12:53:20.120000000" - }, - { - "span_id": "df882f90-cf1a-42a6-a2ae-78b2a13f27a1", - "trace_id": "4bdd5814-acbb-4de7-a56b-543b87a1cfa5", - "parent_span_id": "ec530838-a7cf-477b-9186-a70114c8a12f", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-14 12:53:20.120000000", - "end_time": "2025-10-14 12:53:20.120000000", - "duration_ms": 0.082, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: Tell me about recent developments in machine learning", - "query_length": 53, - "query_type": "routing_evaluation_query", - "classification.mode": "Web Search", - "classification.rag_keyword_score": 0, - "classification.web_keyword_score": 1, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "complex", - "query.intent": "general_knowledge", - "query.keyword_density": 0.125, - "query.domain_affinity": "general", - "routing.decision": "Routed to Web Search based on analysis", - "routing.rationale": "RAG score: 0, Web score: 1, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "RAG", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.019230769230769232, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: Web Search (Confidence: 1.00)" - }, - "events": [], - "links": [], - "trace_trace_id": "4bdd5814-acbb-4de7-a56b-543b87a1cfa5", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-14 12:53:20.120000000", - "trace_end_time": "2025-10-14 12:53:26.750000000", - "trace_duration_ms": 6630.38, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-14 12:53:20.120000000", - "updated_at": "2025-10-14 12:53:20.120000000" - }, - { - "span_id": "bea819fd-3673-480b-9f7b-a3a2efef4218", - "trace_id": "4bdd5814-acbb-4de7-a56b-543b87a1cfa5", - "parent_span_id": "11c07411-df30-42a4-9062-90a032ef75ed", - "name": "agent.llm_model_execution", - "start_time": "2025-10-14 12:53:20.455000000", - "end_time": "2025-10-14 12:53:26.746000000", - "duration_ms": 6291.401, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: Tell me about recent developments in machine learning", - "query_length": 53, - "query_type": "web_search_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 6.290812253952026, - "model.latency_ms": 6290.812253952026, - "model.start_time": 1760446400.4555368, - "model.end_time": 1760446406.746349, - "model.performance_tier": "slow", - "model.prompt_tokens": 163, - "model.completion_tokens": 462, - "model.total_tokens": 625, - "model.tokens_per_second": 99.35124031199013, - "model.estimated_cost": 0.01875, - "model.efficiency_score": 2.9584, - "model.response_length": 1849, - "model.response_quality": "high", - "model.output_response": "Model Response: The web search did not yield specific results regarding recent developments in machine learning. However, I can provide a general overview based on my knowledge up to October 2023.\n\nRecent development...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [], - "trace_trace_id": "4bdd5814-acbb-4de7-a56b-543b87a1cfa5", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-14 12:53:20.120000000", - "trace_end_time": "2025-10-14 12:53:26.750000000", - "trace_duration_ms": 6630.38, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-14 12:53:20.120000000", - "updated_at": "2025-10-14 12:53:20.120000000" - }, - { - "span_id": "6e62af18-569c-4783-b2ae-177c58542bd0", - "trace_id": "4bdd5814-acbb-4de7-a56b-543b87a1cfa5", - "parent_span_id": "11c07411-df30-42a4-9062-90a032ef75ed", - "name": "agent.web_search_evaluation_metrics", - "start_time": "2025-10-14 12:53:26.749000000", - "end_time": "2025-10-14 12:53:26.750000000", - "duration_ms": 0.217, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "web_search_evaluation_metrics", - "agent.capabilities": [ - "web_search_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: Tell me about recent developments in machine learning", - "query_length": 53, - "query_type": "web_search_evaluation_query", - "web_search.results_count": 1, - "web_search.context_length": 155, - "web_search.context_synthesized": "Context from 1 web sources", - "web_search.search_effectiveness": 0.2, - "web_search.context_quality": "low", - "web_search.source_diversity": 1, - "web_search.information_synthesis": "low", - "web_search.external_knowledge_utilization": 0.155, - "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=Tell+me+about+recent+developments+in+machine+learning\n\nUser Question: Tell me about recent developments in machine learning\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", - "prompt.prompt_length": 653, - "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=Tell+me+about+recent+developments+in+machine+learning", - "prompt.instruction_following": "web_search_optimized", - "response.answer_length": 1849, - "response.answer_completeness": "complete", - "response.response_quality": "medium", - "response.source_citation": 1, - "response.context_utilization": 11.929032258064517, - "output_response": "Web Search Answer: The web search did not yield specific results regarding recent developments in machine learning. However, I can provide a general overview based on my knowledge up to October 2023.\n\nRecent development...", - "evaluation.search_effectiveness": 0.2, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 1, - "evaluation.context_relevance": "low", - "evaluation.overall_quality": "low", - "evaluation.ready_for_production": true, - "web_search.search_strategy": "duckduckgo_api", - "web_search.real_time_data": true, - "web_search.external_sources": 1, - "web_search.information_freshness": "current", - "web_search.knowledge_synthesis": "single_source" - }, - "events": [], - "links": [], - "trace_trace_id": "4bdd5814-acbb-4de7-a56b-543b87a1cfa5", - "trace_name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "trace_status": "ok", - "trace_status_message": "", - "trace_start_time": "2025-10-14 12:53:20.120000000", - "trace_end_time": "2025-10-14 12:53:26.750000000", - "trace_duration_ms": 6630.38, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "trace_attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "created_at": "2025-10-14 12:53:20.120000000", - "updated_at": "2025-10-14 12:53:20.120000000" - } -] \ No newline at end of file diff --git a/noveum_customer_support_bt/traces/traces/traces_batch_001.json b/noveum_customer_support_bt/traces/traces/traces_batch_001.json deleted file mode 100644 index c0b74d2..0000000 --- a/noveum_customer_support_bt/traces/traces/traces_batch_001.json +++ /dev/null @@ -1,13915 +0,0 @@ -{ - "success": true, - "traces": [ - { - "trace_id": "6fe93abf-1413-49e8-8464-58558e25ab7b", - "name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "status": "ok", - "status_message": "", - "start_time": "2025-10-22 16:58:42.084000000", - "end_time": "2025-10-22 16:58:44.766000000", - "duration_ms": 2681.9289999999996, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "spans": [ - { - "span_id": "4149abdc-acdc-448c-b9b0-b09e166e461a", - "trace_id": "6fe93abf-1413-49e8-8464-58558e25ab7b", - "parent_span_id": "817c9703-0062-4bd6-b3ee-b50a1121c2d2", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-22 16:58:42.084000000", - "end_time": "2025-10-22 16:58:42.084000000", - "duration_ms": 0.015000000000000001, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What are the latest breakthroughs in artificial intelligence?", - "query_length": 61, - "query_type": "routing_evaluation_query", - "classification.mode": "Web Search", - "classification.rag_keyword_score": 0, - "classification.web_keyword_score": 1, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "complex", - "query.intent": "general_knowledge", - "query.keyword_density": 0.125, - "query.domain_affinity": "general", - "routing.decision": "Routed to Web Search based on analysis", - "routing.rationale": "RAG score: 0, Web score: 1, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "RAG", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.019230769230769232, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: Web Search (Confidence: 1.00)" - }, - "events": [], - "links": [] - }, - { - "span_id": "817c9703-0062-4bd6-b3ee-b50a1121c2d2", - "trace_id": "6fe93abf-1413-49e8-8464-58558e25ab7b", - "parent_span_id": "ff171ffc-dc6b-4cb4-9833-64c5622d9941", - "name": "agent.query_routing", - "start_time": "2025-10-22 16:58:42.084000000", - "end_time": "2025-10-22 16:58:44.766000000", - "duration_ms": 2681.799, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What are the latest breakthroughs in artificial intelligence?", - "query_length": 61, - "query_type": "routing_query", - "output_response": "Routed to Web Search for query processing", - "router.classification": "Web Search", - "router.confidence_score": 1, - "router.rag_keyword_score": 0, - "router.web_keyword_score": 1, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [] - }, - { - "span_id": "ad476d62-d6b5-4142-8771-29855db3f39e", - "trace_id": "6fe93abf-1413-49e8-8464-58558e25ab7b", - "parent_span_id": "817c9703-0062-4bd6-b3ee-b50a1121c2d2", - "name": "agent.web_search_generation", - "start_time": "2025-10-22 16:58:42.084000000", - "end_time": "2025-10-22 16:58:44.766000000", - "duration_ms": 2681.732, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "web_search_agent", - "agent.operation": "web_search_generation", - "agent.capabilities": [ - "web_search", - "content_synthesis", - "response_generation" - ], - "agent.id": "noveum_web_search_agent", - "input_query": "What are the latest breakthroughs in artificial intelligence?", - "query_length": 61, - "query_type": "web_search_query", - "output_response": "Web Search Answer: The web search did not yield specific results regarding the latest breakthroughs in artificial intelligence. However, I can provide a general overview based on knowledge up to October 2023.\n\nRecent br...", - "web_search.results_count": 1, - "web_search.context_length": 163, - "web_search.response_length": 1040, - "web_search.mode": "external_web_search" - }, - "events": [], - "links": [] - }, - { - "span_id": "ff171ffc-dc6b-4cb4-9833-64c5622d9941", - "trace_id": "6fe93abf-1413-49e8-8464-58558e25ab7b", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-22 16:58:42.084000000", - "end_time": "2025-10-22 16:58:44.766000000", - "duration_ms": 2681.902, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What are the latest breakthroughs in artificial intelligence?", - "query.length": 61, - "processing.mode": "Web Search", - "processing.time_seconds": 2.6818161010742188, - "processing.response_length": 1040, - "processing.sources_count": 1, - "output_response": "Final Answer: The web search did not yield specific results regarding the latest breakthroughs in artificial intelligence. However, I can provide a general overview based on knowledge up to October 2023.\n\nRecent br...", - "final_answer_mode": "Web Search", - "query_processed.input_query": "What are the latest breakthroughs in artificial intelligence?", - "query_processed.output_response": "Successfully processed query using Web Search, generated 1040 character response", - "query_processed.mode": "Web Search", - "query_processed.processing_time": 2.6818161010742188, - "query_processed.response_length": 1040 - }, - "events": [], - "links": [] - }, - { - "span_id": "467cb806-a6ef-43da-9739-570730dd244b", - "trace_id": "6fe93abf-1413-49e8-8464-58558e25ab7b", - "parent_span_id": "ad476d62-d6b5-4142-8771-29855db3f39e", - "name": "agent.llm_model_execution", - "start_time": "2025-10-22 16:58:42.438000000", - "end_time": "2025-10-22 16:58:44.766000000", - "duration_ms": 2327.706, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What are the latest breakthroughs in artificial intelligence?", - "query_length": 61, - "query_type": "web_search_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 2.3276820182800293, - "model.latency_ms": 2327.6820182800293, - "model.start_time": 1761152322.438459, - "model.end_time": 1761152324.766141, - "model.performance_tier": "medium", - "model.prompt_tokens": 167, - "model.completion_tokens": 260, - "model.total_tokens": 427, - "model.tokens_per_second": 183.44430065903882, - "model.estimated_cost": 0.01281, - "model.efficiency_score": 2.4355971896955504, - "model.response_length": 1040, - "model.response_quality": "high", - "model.output_response": "Model Response: The web search did not yield specific results regarding the latest breakthroughs in artificial intelligence. However, I can provide a general overview based on knowledge up to October 2023.\n\nRecent br...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [] - }, - { - "span_id": "a80e4596-01ea-44e0-a590-13e347857f2a", - "trace_id": "6fe93abf-1413-49e8-8464-58558e25ab7b", - "parent_span_id": "ad476d62-d6b5-4142-8771-29855db3f39e", - "name": "agent.web_search_evaluation_metrics", - "start_time": "2025-10-22 16:58:44.766000000", - "end_time": "2025-10-22 16:58:44.766000000", - "duration_ms": 0.008, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "web_search_evaluation_metrics", - "agent.capabilities": [ - "web_search_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What are the latest breakthroughs in artificial intelligence?", - "query_length": 61, - "query_type": "web_search_evaluation_query", - "web_search.results_count": 1, - "web_search.context_length": 163, - "web_search.context_synthesized": "Context from 1 web sources", - "web_search.search_effectiveness": 0.2, - "web_search.context_quality": "low", - "web_search.source_diversity": 1, - "web_search.information_synthesis": "low", - "web_search.external_knowledge_utilization": 0.163, - "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What+are+the+latest+breakthroughs+in+artificial+intelligence?\n\nUser Question: What are the latest breakthroughs in artificial intelligence?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", - "prompt.prompt_length": 669, - "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What+are+the+latest+breakthroughs+in+artificial+intelligence?", - "prompt.instruction_following": "web_search_optimized", - "response.answer_length": 1040, - "response.answer_completeness": "complete", - "response.response_quality": "medium", - "response.source_citation": 1, - "response.context_utilization": 6.380368098159509, - "output_response": "Web Search Answer: The web search did not yield specific results regarding the latest breakthroughs in artificial intelligence. However, I can provide a general overview based on knowledge up to October 2023.\n\nRecent br...", - "evaluation.search_effectiveness": 0.2, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 1, - "evaluation.context_relevance": "low", - "evaluation.overall_quality": "low", - "evaluation.ready_for_production": true, - "web_search.search_strategy": "duckduckgo_api", - "web_search.real_time_data": true, - "web_search.external_sources": 1, - "web_search.information_freshness": "current", - "web_search.knowledge_synthesis": "single_source" - }, - "events": [], - "links": [] - } - ], - "created_at": "2025-10-22 16:58:42.084000000", - "updated_at": "2025-10-22 16:58:42.084000000" - }, - { - "trace_id": "ec7c6cae-9bbd-47f4-a8ae-2d6d9b5011ba", - "name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "status": "ok", - "status_message": "", - "start_time": "2025-10-22 16:58:32.800000000", - "end_time": "2025-10-22 16:58:41.077000000", - "duration_ms": 8276.788, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "spans": [ - { - "span_id": "409f29f6-21c7-400a-a9e0-11c87c4313cb", - "trace_id": "ec7c6cae-9bbd-47f4-a8ae-2d6d9b5011ba", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-22 16:58:32.801000000", - "end_time": "2025-10-22 16:58:41.077000000", - "duration_ms": 8276.701, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What's happening in the software development world today?", - "query.length": 57, - "processing.mode": "Web Search", - "processing.time_seconds": 8.276506900787354, - "processing.response_length": 1424, - "processing.sources_count": 1, - "output_response": "Final Answer: The web search did not yield any specific results regarding current events in the software development world. Therefore, I cannot provide detailed information on the latest trends, technologies, or ne...", - "final_answer_mode": "Web Search", - "query_processed.input_query": "What's happening in the software development world today?", - "query_processed.output_response": "Successfully processed query using Web Search, generated 1424 character response", - "query_processed.mode": "Web Search", - "query_processed.processing_time": 8.276506900787354, - "query_processed.response_length": 1424 - }, - "events": [], - "links": [] - }, - { - "span_id": "43389a18-a566-41e7-8db5-f688f651a106", - "trace_id": "ec7c6cae-9bbd-47f4-a8ae-2d6d9b5011ba", - "parent_span_id": "af400e17-8b20-4218-834b-9330ccaa2635", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-22 16:58:32.801000000", - "end_time": "2025-10-22 16:58:32.801000000", - "duration_ms": 0.047, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What's happening in the software development world today?", - "query_length": 57, - "query_type": "routing_evaluation_query", - "classification.mode": "Web Search", - "classification.rag_keyword_score": 0, - "classification.web_keyword_score": 1, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "complex", - "query.intent": "general_knowledge", - "query.keyword_density": 0.125, - "query.domain_affinity": "general", - "routing.decision": "Routed to Web Search based on analysis", - "routing.rationale": "RAG score: 0, Web score: 1, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "RAG", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.019230769230769232, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: Web Search (Confidence: 1.00)" - }, - "events": [], - "links": [] - }, - { - "span_id": "6b431bca-af7e-4f7e-aedb-b103d7f47cae", - "trace_id": "ec7c6cae-9bbd-47f4-a8ae-2d6d9b5011ba", - "parent_span_id": "af400e17-8b20-4218-834b-9330ccaa2635", - "name": "agent.web_search_generation", - "start_time": "2025-10-22 16:58:32.801000000", - "end_time": "2025-10-22 16:58:41.077000000", - "duration_ms": 8276.225999999999, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "web_search_agent", - "agent.operation": "web_search_generation", - "agent.capabilities": [ - "web_search", - "content_synthesis", - "response_generation" - ], - "agent.id": "noveum_web_search_agent", - "input_query": "What's happening in the software development world today?", - "query_length": 57, - "query_type": "web_search_query", - "output_response": "Web Search Answer: The web search did not yield any specific results regarding current events in the software development world. Therefore, I cannot provide detailed information on the latest trends, technologies, or ne...", - "web_search.results_count": 1, - "web_search.context_length": 159, - "web_search.response_length": 1424, - "web_search.mode": "external_web_search" - }, - "events": [], - "links": [] - }, - { - "span_id": "af400e17-8b20-4218-834b-9330ccaa2635", - "trace_id": "ec7c6cae-9bbd-47f4-a8ae-2d6d9b5011ba", - "parent_span_id": "409f29f6-21c7-400a-a9e0-11c87c4313cb", - "name": "agent.query_routing", - "start_time": "2025-10-22 16:58:32.801000000", - "end_time": "2025-10-22 16:58:41.077000000", - "duration_ms": 8276.465999999999, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What's happening in the software development world today?", - "query_length": 57, - "query_type": "routing_query", - "output_response": "Routed to Web Search for query processing", - "router.classification": "Web Search", - "router.confidence_score": 1, - "router.rag_keyword_score": 0, - "router.web_keyword_score": 1, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [] - }, - { - "span_id": "37b153b2-d02b-4b7e-9229-1ba3eaae44ee", - "trace_id": "ec7c6cae-9bbd-47f4-a8ae-2d6d9b5011ba", - "parent_span_id": "6b431bca-af7e-4f7e-aedb-b103d7f47cae", - "name": "agent.llm_model_execution", - "start_time": "2025-10-22 16:58:33.172000000", - "end_time": "2025-10-22 16:58:41.077000000", - "duration_ms": 7904.722, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What's happening in the software development world today?", - "query_length": 57, - "query_type": "web_search_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 7.904651880264282, - "model.latency_ms": 7904.651880264282, - "model.start_time": 1761152313.172655, - "model.end_time": 1761152321.077307, - "model.performance_tier": "slow", - "model.prompt_tokens": 165, - "model.completion_tokens": 356, - "model.total_tokens": 521, - "model.tokens_per_second": 65.91055594754174, - "model.estimated_cost": 0.01563, - "model.efficiency_score": 2.7332053742802302, - "model.response_length": 1424, - "model.response_quality": "high", - "model.output_response": "Model Response: The web search did not yield any specific results regarding current events in the software development world. Therefore, I cannot provide detailed information on the latest trends, technologies, or ne...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [] - }, - { - "span_id": "cc941b8e-6625-43e9-a0a9-7328af24f12f", - "trace_id": "ec7c6cae-9bbd-47f4-a8ae-2d6d9b5011ba", - "parent_span_id": "6b431bca-af7e-4f7e-aedb-b103d7f47cae", - "name": "agent.web_search_evaluation_metrics", - "start_time": "2025-10-22 16:58:41.077000000", - "end_time": "2025-10-22 16:58:41.077000000", - "duration_ms": 0.026, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "web_search_evaluation_metrics", - "agent.capabilities": [ - "web_search_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What's happening in the software development world today?", - "query_length": 57, - "query_type": "web_search_evaluation_query", - "web_search.results_count": 1, - "web_search.context_length": 159, - "web_search.context_synthesized": "Context from 1 web sources", - "web_search.search_effectiveness": 0.2, - "web_search.context_quality": "low", - "web_search.source_diversity": 1, - "web_search.information_synthesis": "low", - "web_search.external_knowledge_utilization": 0.159, - "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What's+happening+in+the+software+development+world+today?\n\nUser Question: What's happening in the software development world today?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", - "prompt.prompt_length": 661, - "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What's+happening+in+the+software+development+world+today?", - "prompt.instruction_following": "web_search_optimized", - "response.answer_length": 1424, - "response.answer_completeness": "complete", - "response.response_quality": "medium", - "response.source_citation": 1, - "response.context_utilization": 8.955974842767295, - "output_response": "Web Search Answer: The web search did not yield any specific results regarding current events in the software development world. Therefore, I cannot provide detailed information on the latest trends, technologies, or ne...", - "evaluation.search_effectiveness": 0.2, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 1, - "evaluation.context_relevance": "low", - "evaluation.overall_quality": "low", - "evaluation.ready_for_production": true, - "web_search.search_strategy": "duckduckgo_api", - "web_search.real_time_data": true, - "web_search.external_sources": 1, - "web_search.information_freshness": "current", - "web_search.knowledge_synthesis": "single_source" - }, - "events": [], - "links": [] - } - ], - "created_at": "2025-10-22 16:58:32.800000000", - "updated_at": "2025-10-22 16:58:32.800000000" - }, - { - "trace_id": "343764b8-7ac9-4549-93cf-8abc11ceba6f", - "name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "status": "ok", - "status_message": "", - "start_time": "2025-10-22 16:58:26.542000000", - "end_time": "2025-10-22 16:58:31.796000000", - "duration_ms": 5253.973, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "spans": [ - { - "span_id": "3b06d098-6eb7-4bb6-885e-443ddb28cc41", - "trace_id": "343764b8-7ac9-4549-93cf-8abc11ceba6f", - "parent_span_id": "5c2aa1cf-56a8-4fd5-af90-90a94e723bb5", - "name": "agent.query_routing", - "start_time": "2025-10-22 16:58:26.542000000", - "end_time": "2025-10-22 16:58:31.796000000", - "duration_ms": 5253.835, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What are the newest features in cloud computing?", - "query_length": 48, - "query_type": "routing_query", - "output_response": "Routed to RAG for query processing", - "router.classification": "RAG", - "router.confidence_score": 1, - "router.rag_keyword_score": 1, - "router.web_keyword_score": 0, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [] - }, - { - "span_id": "5c2aa1cf-56a8-4fd5-af90-90a94e723bb5", - "trace_id": "343764b8-7ac9-4549-93cf-8abc11ceba6f", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-22 16:58:26.542000000", - "end_time": "2025-10-22 16:58:31.796000000", - "duration_ms": 5253.913, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What are the newest features in cloud computing?", - "query.length": 48, - "processing.mode": "RAG", - "processing.time_seconds": 5.253868818283081, - "processing.response_length": 837, - "processing.sources_count": 5, - "output_response": "Final Answer: The provided context does not specifically mention any new features in cloud computing as a general topic. However, it does highlight several recent enhancements related to the Noveum.ai platform, whi...", - "final_answer_mode": "RAG", - "query_processed.input_query": "What are the newest features in cloud computing?", - "query_processed.output_response": "Successfully processed query using RAG, generated 837 character response", - "query_processed.mode": "RAG", - "query_processed.processing_time": 5.253868818283081, - "query_processed.response_length": 837 - }, - "events": [], - "links": [] - }, - { - "span_id": "6b838ed6-f5bb-4bcf-a76b-bac283bf7571", - "trace_id": "343764b8-7ac9-4549-93cf-8abc11ceba6f", - "parent_span_id": "3b06d098-6eb7-4bb6-885e-443ddb28cc41", - "name": "agent.llm-rag", - "start_time": "2025-10-22 16:58:26.542000000", - "end_time": "2025-10-22 16:58:31.796000000", - "duration_ms": 5253.519, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "rag_agent", - "agent.operation": "llm-rag", - "agent.capabilities": [ - "document_retrieval", - "context_generation", - "response_generation" - ], - "agent.id": "noveum_rag_agent", - "input_query": "What are the newest features in cloud computing?", - "query_length": 48, - "query_type": "rag_query", - "output_response": "RAG Answer: The provided context does not specifically mention any new features in cloud computing as a general topic. However, it does highlight several recent enhancements related to the Noveum.ai platform, whi...", - "rag.context_length": 2579, - "rag.sources_count": 5, - "rag.answer_length": 837, - "rag.mode": "retrieval_augmented_generation" - }, - "events": [], - "links": [] - }, - { - "span_id": "f51d8f1e-eae3-488d-9a96-49ab97377851", - "trace_id": "343764b8-7ac9-4549-93cf-8abc11ceba6f", - "parent_span_id": "3b06d098-6eb7-4bb6-885e-443ddb28cc41", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-22 16:58:26.542000000", - "end_time": "2025-10-22 16:58:26.542000000", - "duration_ms": 0.082, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What are the newest features in cloud computing?", - "query_length": 48, - "query_type": "routing_evaluation_query", - "classification.mode": "RAG", - "classification.rag_keyword_score": 1, - "classification.web_keyword_score": 0, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "documentation", - "query.keyword_density": 0.125, - "query.domain_affinity": "noveum", - "routing.decision": "Routed to RAG based on analysis", - "routing.rationale": "RAG score: 1, Web score: 0, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "Web Search", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.019230769230769232, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: RAG (Confidence: 1.00)" - }, - "events": [], - "links": [] - }, - { - "span_id": "93a0fd69-47b6-4e7c-b8ab-8eaa5aeac240", - "trace_id": "343764b8-7ac9-4549-93cf-8abc11ceba6f", - "parent_span_id": "6b838ed6-f5bb-4bcf-a76b-bac283bf7571", - "name": "agent.llm_model_execution", - "start_time": "2025-10-22 16:58:27.411000000", - "end_time": "2025-10-22 16:58:31.373000000", - "duration_ms": 3961.481, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What are the newest features in cloud computing?", - "query_length": 48, - "query_type": "rag_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 3.9613850116729736, - "model.latency_ms": 3961.3850116729736, - "model.start_time": 1761152307.411936, - "model.end_time": 1761152311.373321, - "model.performance_tier": "medium", - "model.prompt_tokens": 780, - "model.completion_tokens": 209, - "model.total_tokens": 989, - "model.tokens_per_second": 249.66015600243944, - "model.estimated_cost": 0.029670000000000002, - "model.efficiency_score": 0.846309403437816, - "model.response_length": 837, - "model.response_quality": "high", - "model.output_response": "Model Response: The provided context does not specifically mention any new features in cloud computing as a general topic. However, it does highlight several recent enhancements related to the Noveum.ai platform, whi...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [] - }, - { - "span_id": "48cbc1df-f03a-4c49-90e5-19a2047c1c7b", - "trace_id": "343764b8-7ac9-4549-93cf-8abc11ceba6f", - "parent_span_id": "6b838ed6-f5bb-4bcf-a76b-bac283bf7571", - "name": "agent.rag_evaluation_metrics", - "start_time": "2025-10-22 16:58:31.373000000", - "end_time": "2025-10-22 16:58:31.373000000", - "duration_ms": 0.061, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "rag_evaluation_metrics", - "agent.capabilities": [ - "retrieval_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What are the newest features in cloud computing?", - "query_length": 48, - "query_type": "rag_evaluation_query", - "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/changelog):\n- \ud83d\udcc8 Enhanced Dashboard Analytics: Real-time request tracking, latency monitoring, and cost analysis\n- \ud83d\udd0d Improved Logs Interface: Better search functionality, detailed trace views, and enhanced debugging capabilities\n- \ud83d\udc65 Team Management: Advanced member mana...", - "retrieval.context_length": 2579, - "retrieval.sources_count": 5, - "retrieval.context_quality": "high", - "retrieval.effectiveness": 1, - "retrieval.context_utilization": 2.579, - "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/changelog):\n- \ud83d\udcc8 Enhanced Dashboard Analytics: Real-time request tracking, latency monitoring, and cost analysis\n- \ud83d\udd0d Improved Logs Interface: Better search functionality, detailed trace views, and enhanced debugging capabilities\n- \ud83d\udc65 Team Management: Advanced member management with role-based access control and invitation system\n- \ud83d\udee1\ufe0f Security Improvements: Enhanced API key management and secure credential storage\n4 months ago\n- \ud83d\udcca Advanced Metrics Collection: Comprehensive telemetry system with custom metrics ...\n\nSource 2 (https://noveum.ai/en/changelog):\n- \ud83d\udcdd Documentation Updates: Comprehensive guides for integration and best practices\n5 months ago\n- \ud83c\udf89 Platform Foundation: Initial release of Noveum.ai AI observability platform\n- \ud83d\ude80 Core Features: AI Gateway integration, basic evaluation metrics, and monitoring capabilities\n- \ud83d\udcc8 Evaluation Jobs: Automated model evaluation with configurable metrics and reporting\n- \ud83d\udd0c Provider Integrations: Support for major AI providers and custom model deployments...\n\nSource 3 (https://noveum.ai/docs/platform/dashboard):\n- Bulk Export: Download multiple traces simultaneously\n- Comparative Analysis: Compare performance across multiple traces\nReal-Time Updates\n- Live Refresh: Automatic updates as new traces arrive\n- Connection Monitoring: Real-time status of your trace ingestion pipeline\n- Performance Indicators: Live metrics for system health monitoring\nReady to dive deeper? Explore Projects & Environments to organize your AI applications, or check out Team Collaboration to share insights with your team.\nGet Earl...\n\nSource 4 (https://noveum.ai/en/changelog):\nChangelog\nStay up to date with the latest changes in our product.\n2 months ago\n- \ud83c\udfaf Major Platform Evolution: NovaEval Framework - Launched comprehensive AI model evaluation framework with 20+ built-in scorers including accuracy, RAG metrics, conversational scoring, and specialized agent evaluation tools. Features production-grade deployment on Docker and Kubernetes. (GitHub | PyPI Package)\n- \ud83d\ude80 Next-Generation Traces UI - Complete redesign featuring three-pane layout with directory tree navigatio...\n\nSource 5 (https://noveum.ai/en/blog):\n8/12/2025\nDiscover how Noveum.ai provides comprehensive tracing and observability for AI applications, from development debugging to production optimization.\nShashank Agarwal\n3/3/2025\nDiscover how Noveum.ai provides comprehensive tracing and observability for LLM applications, RAG systems, and multi-agent workflows with our powerful Python and TypeScript SDKs.\nShashank Agarwal\n3/2/2025...\n\nUser Question: What are the newest features in cloud computing?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", - "prompt.prompt_length": 3123, - "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/changelog):\n- \ud83d\udcc8 Enhanced Dashboard Analytics: Real-time request tracking, latency monitoring, and cost analysis\n- \ud83d\udd0d Improved Logs Interface: Better search functionality,...", - "prompt.instruction_following": "rag_optimized", - "response.answer_length": 837, - "response.answer_completeness": "complete", - "response.response_quality": "high", - "response.source_citation": 5, - "response.context_utilization": 0.3245443970531214, - "output_response": "RAG Answer: The provided context does not specifically mention any new features in cloud computing as a general topic. However, it does highlight several recent enhancements related to the Noveum.ai platform, whi...", - "evaluation.retrieval_effectiveness": 1, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 5, - "evaluation.context_relevance": "high", - "evaluation.overall_quality": "high", - "evaluation.ready_for_production": true, - "rag.retrieval_strategy": "semantic_similarity", - "rag.vector_search_results": 5, - "rag.context_synthesis": "multi_source", - "rag.document_coverage": 1, - "rag.information_density": 0.3245443970531214 - }, - "events": [], - "links": [] - } - ], - "created_at": "2025-10-22 16:58:26.542000000", - "updated_at": "2025-10-22 16:58:26.542000000" - }, - { - "trace_id": "3f7b6a62-8888-4a5d-b12c-37a3829aea49", - "name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "status": "ok", - "status_message": "", - "start_time": "2025-10-22 16:58:19.885000000", - "end_time": "2025-10-22 16:58:25.536000000", - "duration_ms": 5651.67, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "spans": [ - { - "span_id": "7384d697-d7bf-4709-8377-ec26b094b550", - "trace_id": "3f7b6a62-8888-4a5d-b12c-37a3829aea49", - "parent_span_id": "e631b22a-a621-4ecc-bb01-f731ca66f3d0", - "name": "agent.query_routing", - "start_time": "2025-10-22 16:58:19.885000000", - "end_time": "2025-10-22 16:58:25.536000000", - "duration_ms": 5651.335999999999, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What's the current status of cryptocurrency markets?", - "query_length": 52, - "query_type": "routing_query", - "output_response": "Routed to Web Search for query processing", - "router.classification": "Web Search", - "router.confidence_score": 1, - "router.rag_keyword_score": 0, - "router.web_keyword_score": 3, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [] - }, - { - "span_id": "7abf372d-d9f4-41d1-9e3a-d4f01173fa5e", - "trace_id": "3f7b6a62-8888-4a5d-b12c-37a3829aea49", - "parent_span_id": "7384d697-d7bf-4709-8377-ec26b094b550", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-22 16:58:19.885000000", - "end_time": "2025-10-22 16:58:19.885000000", - "duration_ms": 0.037, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What's the current status of cryptocurrency markets?", - "query_length": 52, - "query_type": "routing_evaluation_query", - "classification.mode": "Web Search", - "classification.rag_keyword_score": 0, - "classification.web_keyword_score": 3, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "complex", - "query.intent": "general_knowledge", - "query.keyword_density": 0.42857142857142855, - "query.domain_affinity": "general", - "routing.decision": "Routed to Web Search based on analysis", - "routing.rationale": "RAG score: 0, Web score: 3, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "RAG", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.057692307692307696, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: Web Search (Confidence: 1.00)" - }, - "events": [], - "links": [] - }, - { - "span_id": "e5d9c3c3-6c18-45d2-a6b6-58bde4d4e461", - "trace_id": "3f7b6a62-8888-4a5d-b12c-37a3829aea49", - "parent_span_id": "7384d697-d7bf-4709-8377-ec26b094b550", - "name": "agent.web_search_generation", - "start_time": "2025-10-22 16:58:19.885000000", - "end_time": "2025-10-22 16:58:25.536000000", - "duration_ms": 5651.161, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "web_search_agent", - "agent.operation": "web_search_generation", - "agent.capabilities": [ - "web_search", - "content_synthesis", - "response_generation" - ], - "agent.id": "noveum_web_search_agent", - "input_query": "What's the current status of cryptocurrency markets?", - "query_length": 52, - "query_type": "web_search_query", - "output_response": "Web Search Answer: The provided web search results do not contain specific information regarding the current status of cryptocurrency markets. To get the latest updates on cryptocurrency prices, trends, and market analy...", - "web_search.results_count": 1, - "web_search.context_length": 251, - "web_search.response_length": 416, - "web_search.mode": "external_web_search" - }, - "events": [], - "links": [] - }, - { - "span_id": "e631b22a-a621-4ecc-bb01-f731ca66f3d0", - "trace_id": "3f7b6a62-8888-4a5d-b12c-37a3829aea49", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-22 16:58:19.885000000", - "end_time": "2025-10-22 16:58:25.536000000", - "duration_ms": 5651.588000000001, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What's the current status of cryptocurrency markets?", - "query.length": 52, - "processing.mode": "Web Search", - "processing.time_seconds": 5.651373863220215, - "processing.response_length": 416, - "processing.sources_count": 1, - "output_response": "Final Answer: The provided web search results do not contain specific information regarding the current status of cryptocurrency markets. To get the latest updates on cryptocurrency prices, trends, and market analy...", - "final_answer_mode": "Web Search", - "query_processed.input_query": "What's the current status of cryptocurrency markets?", - "query_processed.output_response": "Successfully processed query using Web Search, generated 416 character response", - "query_processed.mode": "Web Search", - "query_processed.processing_time": 5.651373863220215, - "query_processed.response_length": 416 - }, - "events": [], - "links": [] - }, - { - "span_id": "34e2f906-724e-4116-b61f-82eaa6b75959", - "trace_id": "3f7b6a62-8888-4a5d-b12c-37a3829aea49", - "parent_span_id": "e5d9c3c3-6c18-45d2-a6b6-58bde4d4e461", - "name": "agent.llm_model_execution", - "start_time": "2025-10-22 16:58:24.327000000", - "end_time": "2025-10-22 16:58:25.536000000", - "duration_ms": 1208.66, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What's the current status of cryptocurrency markets?", - "query_length": 52, - "query_type": "web_search_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 1.208608865737915, - "model.latency_ms": 1208.608865737915, - "model.start_time": 1761152304.327749, - "model.end_time": 1761152305.5363579, - "model.performance_tier": "fast", - "model.prompt_tokens": 187, - "model.completion_tokens": 104, - "model.total_tokens": 291, - "model.tokens_per_second": 240.77268357809888, - "model.estimated_cost": 0.00873, - "model.efficiency_score": 1.429553264604811, - "model.response_length": 416, - "model.response_quality": "high", - "model.output_response": "Model Response: The provided web search results do not contain specific information regarding the current status of cryptocurrency markets. To get the latest updates on cryptocurrency prices, trends, and market analy...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [] - }, - { - "span_id": "5548e02e-2ea8-472c-9c42-f811d6e4d30b", - "trace_id": "3f7b6a62-8888-4a5d-b12c-37a3829aea49", - "parent_span_id": "e5d9c3c3-6c18-45d2-a6b6-58bde4d4e461", - "name": "agent.web_search_evaluation_metrics", - "start_time": "2025-10-22 16:58:25.536000000", - "end_time": "2025-10-22 16:58:25.536000000", - "duration_ms": 0.044, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "web_search_evaluation_metrics", - "agent.capabilities": [ - "web_search_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What's the current status of cryptocurrency markets?", - "query_length": 52, - "query_type": "web_search_evaluation_query", - "web_search.results_count": 1, - "web_search.context_length": 251, - "web_search.context_synthesized": "Context from 1 web sources", - "web_search.search_effectiveness": 0.2, - "web_search.context_quality": "low", - "web_search.source_diversity": 1, - "web_search.information_synthesis": "low", - "web_search.external_knowledge_utilization": 0.251, - "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nLog in to WhatsApp Web for simple, reliable and private messaging on your desktop. Send and receive messages and files with ease, all for \u2026\nURL: https://duckduckgo.com/?q=What's+the+current+status+of+cryptocurrency+markets?\n\nUser Question: What's the current status of cryptocurrency markets?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", - "prompt.prompt_length": 748, - "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nLog in to WhatsApp Web for simple, reliable and private messaging on your desktop. Send and receive messages and files with ease, all for \u2026\nURL: https://duckduckgo.com/?q=W...", - "prompt.instruction_following": "web_search_optimized", - "response.answer_length": 416, - "response.answer_completeness": "complete", - "response.response_quality": "medium", - "response.source_citation": 1, - "response.context_utilization": 1.657370517928287, - "output_response": "Web Search Answer: The provided web search results do not contain specific information regarding the current status of cryptocurrency markets. To get the latest updates on cryptocurrency prices, trends, and market analy...", - "evaluation.search_effectiveness": 0.2, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 1, - "evaluation.context_relevance": "low", - "evaluation.overall_quality": "low", - "evaluation.ready_for_production": true, - "web_search.search_strategy": "duckduckgo_api", - "web_search.real_time_data": true, - "web_search.external_sources": 1, - "web_search.information_freshness": "current", - "web_search.knowledge_synthesis": "single_source" - }, - "events": [], - "links": [] - } - ], - "created_at": "2025-10-22 16:58:19.885000000", - "updated_at": "2025-10-22 16:58:19.885000000" - }, - { - "trace_id": "51f4534f-23e4-4be7-8d6e-e93391d6e0e2", - "name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "status": "ok", - "status_message": "", - "start_time": "2025-10-22 16:58:15.254000000", - "end_time": "2025-10-22 16:58:18.877000000", - "duration_ms": 3622.355, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "spans": [ - { - "span_id": "a9e34cea-d4bd-478b-abf2-7a3a38dcd40e", - "trace_id": "51f4534f-23e4-4be7-8d6e-e93391d6e0e2", - "parent_span_id": "8e9b2d06-3e11-4d17-9a32-b1529a2c10a4", - "name": "agent.web_search_generation", - "start_time": "2025-10-22 16:58:15.254000000", - "end_time": "2025-10-22 16:58:18.876000000", - "duration_ms": 3621.674, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "web_search_agent", - "agent.operation": "web_search_generation", - "agent.capabilities": [ - "web_search", - "content_synthesis", - "response_generation" - ], - "agent.id": "noveum_web_search_agent", - "input_query": "What are the latest updates in Python programming?", - "query_length": 50, - "query_type": "web_search_query", - "output_response": "Web Search Answer: The provided web search results do not contain specific information about the latest updates in Python programming. They primarily focus on general news coverage and updates from India and around the ...", - "web_search.results_count": 1, - "web_search.context_length": 753, - "web_search.response_length": 580, - "web_search.mode": "external_web_search" - }, - "events": [], - "links": [] - }, - { - "span_id": "8e9b2d06-3e11-4d17-9a32-b1529a2c10a4", - "trace_id": "51f4534f-23e4-4be7-8d6e-e93391d6e0e2", - "parent_span_id": "bda936a0-24f3-4841-af4f-227197e10b9d", - "name": "agent.query_routing", - "start_time": "2025-10-22 16:58:15.254000000", - "end_time": "2025-10-22 16:58:18.876000000", - "duration_ms": 3621.863, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What are the latest updates in Python programming?", - "query_length": 50, - "query_type": "routing_query", - "output_response": "Routed to Web Search for query processing", - "router.classification": "Web Search", - "router.confidence_score": 1, - "router.rag_keyword_score": 0, - "router.web_keyword_score": 2, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [] - }, - { - "span_id": "bda936a0-24f3-4841-af4f-227197e10b9d", - "trace_id": "51f4534f-23e4-4be7-8d6e-e93391d6e0e2", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-22 16:58:15.254000000", - "end_time": "2025-10-22 16:58:18.877000000", - "duration_ms": 3622.271, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What are the latest updates in Python programming?", - "query.length": 50, - "processing.mode": "Web Search", - "processing.time_seconds": 3.6219100952148438, - "processing.response_length": 580, - "processing.sources_count": 1, - "output_response": "Final Answer: The provided web search results do not contain specific information about the latest updates in Python programming. They primarily focus on general news coverage and updates from India and around the ...", - "final_answer_mode": "Web Search", - "query_processed.input_query": "What are the latest updates in Python programming?", - "query_processed.output_response": "Successfully processed query using Web Search, generated 580 character response", - "query_processed.mode": "Web Search", - "query_processed.processing_time": 3.6219100952148438, - "query_processed.response_length": 580 - }, - "events": [], - "links": [] - }, - { - "span_id": "e27e3812-d49d-4f4f-98b0-144cf0ceffc3", - "trace_id": "51f4534f-23e4-4be7-8d6e-e93391d6e0e2", - "parent_span_id": "8e9b2d06-3e11-4d17-9a32-b1529a2c10a4", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-22 16:58:15.254000000", - "end_time": "2025-10-22 16:58:15.254000000", - "duration_ms": 0.047, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What are the latest updates in Python programming?", - "query_length": 50, - "query_type": "routing_evaluation_query", - "classification.mode": "Web Search", - "classification.rag_keyword_score": 0, - "classification.web_keyword_score": 2, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "general_knowledge", - "query.keyword_density": 0.25, - "query.domain_affinity": "general", - "routing.decision": "Routed to Web Search based on analysis", - "routing.rationale": "RAG score: 0, Web score: 2, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "RAG", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.038461538461538464, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: Web Search (Confidence: 1.00)" - }, - "events": [], - "links": [] - }, - { - "span_id": "b0eea8ec-5cff-4667-bea7-aeb6d0f0daf3", - "trace_id": "51f4534f-23e4-4be7-8d6e-e93391d6e0e2", - "parent_span_id": "a9e34cea-d4bd-478b-abf2-7a3a38dcd40e", - "name": "agent.llm_model_execution", - "start_time": "2025-10-22 16:58:15.568000000", - "end_time": "2025-10-22 16:58:18.876000000", - "duration_ms": 3307.5739999999996, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What are the latest updates in Python programming?", - "query_length": 50, - "query_type": "web_search_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 3.307521104812622, - "model.latency_ms": 3307.521104812622, - "model.start_time": 1761152295.568949, - "model.end_time": 1761152298.87647, - "model.performance_tier": "medium", - "model.prompt_tokens": 312, - "model.completion_tokens": 145, - "model.total_tokens": 457, - "model.tokens_per_second": 138.16994223711538, - "model.estimated_cost": 0.01371, - "model.efficiency_score": 1.2691466083150984, - "model.response_length": 580, - "model.response_quality": "high", - "model.output_response": "Model Response: The provided web search results do not contain specific information about the latest updates in Python programming. They primarily focus on general news coverage and updates from India and around the ...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [] - }, - { - "span_id": "d0bfcbe5-cb6c-4837-bdf3-f2c65be145cd", - "trace_id": "51f4534f-23e4-4be7-8d6e-e93391d6e0e2", - "parent_span_id": "a9e34cea-d4bd-478b-abf2-7a3a38dcd40e", - "name": "agent.web_search_evaluation_metrics", - "start_time": "2025-10-22 16:58:18.876000000", - "end_time": "2025-10-22 16:58:18.876000000", - "duration_ms": 0.04, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "web_search_evaluation_metrics", - "agent.capabilities": [ - "web_search_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What are the latest updates in Python programming?", - "query_length": 50, - "query_type": "web_search_evaluation_query", - "web_search.results_count": 1, - "web_search.context_length": 753, - "web_search.context_synthesized": "Context from 1 web sources", - "web_search.search_effectiveness": 0.2, - "web_search.context_quality": "medium", - "web_search.source_diversity": 1, - "web_search.information_synthesis": "low", - "web_search.external_knowledge_utilization": 0.753, - "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\n3 days ago \u00b7 Latest news headlines from India & around the world. Check out today\u2019s news coverage live with videos & photos on \u2026 6 days ago \u00b7 India News | Latest India News | Read latest and breaking news from India. Today's top India news headlines, news on \u2026 2 days ago \u00b7 Top News Stories of the day, Latest News Headlines, News Specials, Breaking News and Latest India News, World current affairs & \u2026 NDTV.com provides latest news from India and around the world. Get breaking news alerts from India and follow today\u2019s live news updates in \u2026 6 days ago \u00b7 Get Latest entertainment news from Bollywood, Hollywood and regional film industries.\nURL: https://duckduckgo.com/?q=What+are+the+latest+updates+in+Python+programming?\n\nUser Question: What are the latest updates in Python programming?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", - "prompt.prompt_length": 1248, - "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\n3 days ago \u00b7 Latest news headlines from India & around the world. Check out today\u2019s news coverage live with videos & photos on \u2026 6 days ago \u00b7 India News | Latest India News...", - "prompt.instruction_following": "web_search_optimized", - "response.answer_length": 580, - "response.answer_completeness": "complete", - "response.response_quality": "medium", - "response.source_citation": 1, - "response.context_utilization": 0.7702523240371846, - "output_response": "Web Search Answer: The provided web search results do not contain specific information about the latest updates in Python programming. They primarily focus on general news coverage and updates from India and around the ...", - "evaluation.search_effectiveness": 0.2, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 1, - "evaluation.context_relevance": "medium", - "evaluation.overall_quality": "low", - "evaluation.ready_for_production": true, - "web_search.search_strategy": "duckduckgo_api", - "web_search.real_time_data": true, - "web_search.external_sources": 1, - "web_search.information_freshness": "current", - "web_search.knowledge_synthesis": "single_source" - }, - "events": [], - "links": [] - } - ], - "created_at": "2025-10-22 16:58:15.254000000", - "updated_at": "2025-10-22 16:58:15.254000000" - }, - { - "trace_id": "e30e9ba6-ffd1-4eb9-b7ae-6d19e04343d6", - "name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "status": "ok", - "status_message": "", - "start_time": "2025-10-22 16:58:11.033000000", - "end_time": "2025-10-22 16:58:14.250000000", - "duration_ms": 3216.801, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "spans": [ - { - "span_id": "5f5916a7-b9ea-4348-861d-db69b25d6b62", - "trace_id": "e30e9ba6-ffd1-4eb9-b7ae-6d19e04343d6", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-22 16:58:11.033000000", - "end_time": "2025-10-22 16:58:14.250000000", - "duration_ms": 3216.77, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What happened in tech news this week?", - "query.length": 37, - "processing.mode": "Web Search", - "processing.time_seconds": 3.2166178226470947, - "processing.response_length": 510, - "processing.sources_count": 1, - "output_response": "Final Answer: The web search results do not provide specific information about recent events in tech news for this week. They primarily discuss the grammatical usage of the word \"happened\" and its variations, such ...", - "final_answer_mode": "Web Search", - "query_processed.input_query": "What happened in tech news this week?", - "query_processed.output_response": "Successfully processed query using Web Search, generated 510 character response", - "query_processed.mode": "Web Search", - "query_processed.processing_time": 3.2166178226470947, - "query_processed.response_length": 510 - }, - "events": [], - "links": [] - }, - { - "span_id": "804613d0-a701-4c22-add1-d315d7a29813", - "trace_id": "e30e9ba6-ffd1-4eb9-b7ae-6d19e04343d6", - "parent_span_id": "5f5916a7-b9ea-4348-861d-db69b25d6b62", - "name": "agent.query_routing", - "start_time": "2025-10-22 16:58:11.033000000", - "end_time": "2025-10-22 16:58:14.250000000", - "duration_ms": 3216.594, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What happened in tech news this week?", - "query_length": 37, - "query_type": "routing_query", - "output_response": "Routed to Web Search for query processing", - "router.classification": "Web Search", - "router.confidence_score": 1, - "router.rag_keyword_score": 0, - "router.web_keyword_score": 2, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [] - }, - { - "span_id": "f6c8d34f-185f-49ba-a5a4-a9d25ccb4fc6", - "trace_id": "e30e9ba6-ffd1-4eb9-b7ae-6d19e04343d6", - "parent_span_id": "804613d0-a701-4c22-add1-d315d7a29813", - "name": "agent.web_search_generation", - "start_time": "2025-10-22 16:58:11.034000000", - "end_time": "2025-10-22 16:58:14.250000000", - "duration_ms": 3216.4959999999996, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "web_search_agent", - "agent.operation": "web_search_generation", - "agent.capabilities": [ - "web_search", - "content_synthesis", - "response_generation" - ], - "agent.id": "noveum_web_search_agent", - "input_query": "What happened in tech news this week?", - "query_length": 37, - "query_type": "web_search_query", - "output_response": "Web Search Answer: The web search results do not provide specific information about recent events in tech news for this week. They primarily discuss the grammatical usage of the word \"happened\" and its variations, such ...", - "web_search.results_count": 1, - "web_search.context_length": 724, - "web_search.response_length": 510, - "web_search.mode": "external_web_search" - }, - "events": [], - "links": [] - }, - { - "span_id": "11d68082-e7b8-4bbf-833a-79e296c31eda", - "trace_id": "e30e9ba6-ffd1-4eb9-b7ae-6d19e04343d6", - "parent_span_id": "804613d0-a701-4c22-add1-d315d7a29813", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-22 16:58:11.034000000", - "end_time": "2025-10-22 16:58:11.034000000", - "duration_ms": 0.017, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What happened in tech news this week?", - "query_length": 37, - "query_type": "routing_evaluation_query", - "classification.mode": "Web Search", - "classification.rag_keyword_score": 0, - "classification.web_keyword_score": 2, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "general_knowledge", - "query.keyword_density": 0.2857142857142857, - "query.domain_affinity": "general", - "routing.decision": "Routed to Web Search based on analysis", - "routing.rationale": "RAG score: 0, Web score: 2, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "RAG", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.038461538461538464, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: Web Search (Confidence: 1.00)" - }, - "events": [], - "links": [] - }, - { - "span_id": "420b4ca1-c9f1-488f-ac2c-c53766717634", - "trace_id": "e30e9ba6-ffd1-4eb9-b7ae-6d19e04343d6", - "parent_span_id": "f6c8d34f-185f-49ba-a5a4-a9d25ccb4fc6", - "name": "agent.llm_model_execution", - "start_time": "2025-10-22 16:58:11.468000000", - "end_time": "2025-10-22 16:58:14.250000000", - "duration_ms": 2781.777, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What happened in tech news this week?", - "query_length": 37, - "query_type": "web_search_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 2.781723976135254, - "model.latency_ms": 2781.723976135254, - "model.start_time": 1761152291.468626, - "model.end_time": 1761152294.25035, - "model.performance_tier": "medium", - "model.prompt_tokens": 301, - "model.completion_tokens": 127, - "model.total_tokens": 428, - "model.tokens_per_second": 153.8614196346811, - "model.estimated_cost": 0.01284, - "model.efficiency_score": 1.191588785046729, - "model.response_length": 510, - "model.response_quality": "high", - "model.output_response": "Model Response: The web search results do not provide specific information about recent events in tech news for this week. They primarily discuss the grammatical usage of the word \"happened\" and its variations, such ...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [] - }, - { - "span_id": "56b8bf09-8ebf-44e1-b01f-daff23d3c42c", - "trace_id": "e30e9ba6-ffd1-4eb9-b7ae-6d19e04343d6", - "parent_span_id": "f6c8d34f-185f-49ba-a5a4-a9d25ccb4fc6", - "name": "agent.web_search_evaluation_metrics", - "start_time": "2025-10-22 16:58:14.250000000", - "end_time": "2025-10-22 16:58:14.250000000", - "duration_ms": 0.060000000000000005, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "web_search_evaluation_metrics", - "agent.capabilities": [ - "web_search_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What happened in tech news this week?", - "query_length": 37, - "query_type": "web_search_evaluation_query", - "web_search.results_count": 1, - "web_search.context_length": 724, - "web_search.context_synthesized": "Context from 1 web sources", - "web_search.search_effectiveness": 0.2, - "web_search.context_quality": "medium", - "web_search.source_diversity": 1, - "web_search.information_synthesis": "low", - "web_search.external_knowledge_utilization": 0.724, - "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nhappened\u7684\u7528\u6cd5\u5982\u4e0b\uff1a 1.happened\u662f \u4e0d\u53ca\u7269\u52a8\u8bcd,\u4f5c\u201c\u53d1\u751f\u201d\u89e3\u65f6,\u53ef\u6307\u5076\u7136\u7684\u4e8b\u6545,\u4e5f\u53ef\u6307\u6709\u8ba1\u5212\u6216\u65e0\u8ba1\u5212\u53d1\u751f\u7684\u4e8b\u3002\u4e00\u822c\u4ee5\u4e8b\u4ef6\u3001\u73af\u5883\u3001\u5f62\u52bf\u7b49\u4f5c\u4e3b\u8bed,\u8868\u793a\u201c\u67d0\u4eba\u53d1\u751f\u67d0\u4e8b\u201d\u65f6,\u987b\u4ee5\u4ecb\u8bcdto\u5f15\u51fa\u201c\u67d0\u4eba\u201d\u3002 2. \u2026 what happened\u8fd8\u662fwhat's happened\"What happened\"\u548c\"What's happened\"\u90fd\u7528\u4e8e\u8be2\u95ee\u8fc7\u53bb\u53d1\u751f\u4e86\u4ec0\u4e48\u4e8b\u60c5\uff0c\u4e3b\u8981\u7684\u533a\u522b\u5728\u4e8e\u8bed\u6cd5\u548c\u4f7f\u7528\u65b9\u5f0f\u3002\"What happened\"\u662f\u4e00\u4e2a\u8fc7\u53bb\u65f6\u6001\u7684\u7b80\u5355 \u63a5\u4e0b\u6765\u8ba9\u6211\u4eec\u770b\u4e0bwhat happen\u548cwhat happened\u7684\u7528\u6cd5\u533a\u522b\uff1a 1.\u8bcd\u6027\uff1awhat happen\u662f\u7528\u539f\u5f62\u52a8\u8bcdhappen\u6784\u6210\u7684\u73b0\u5728\u65f6\u77ed\u8bed\uff0c\u800cwhat happened\u662f\u7528\u52a8\u8bcdhappen\u7684\u8fc7\u53bb\u5f0fhappened\u6784\u6210\u7684\u8fc7\u53bb\u65f6\u77ed \u2026 What happened \u548c What's happened\u4e24\u8005\u7684\u533a\u522b\uff1a \u662f\u95ee What happened\uff1f\u53ea\u662f\u60f3\u77e5\u9053\u8fc7\u53bb\u6240\u53d1\u751f\u7684\u4e8b\u60c5\uff0c \u800c\u5982\u679c\u7528What's happened\uff1f\u5219\u5f3a\u8c03\u5bf9\u73b0\u5728\u7684\u5f71\u54cd\u3002\uff0c\u5373\u60f3\u77e5\u9053\u8fc7\u53bb\u6240\u53d1\u751f\u7684\u4ec0\u4e48\u4e8b\u5bfc\u81f4\u4e86\u8bf4 \u2026 it so happened that\u7684\u53e5\u5f0fhappen\u8868\u793a\u201c\u78b0\u5de7\u6216\u6070\u5de7\u53d1\u751f\u67d0\u4e8b\u201d\u65f6,\u53ef\u7528\u201cIt happens / happened that...\u201d\u8fd9\u4e00\u7ed3\u6784\u6765\u8868\u8fbe.\u4f8b\u5982\uff1a It happened that Brian and Peter were at home that day.\u78b0\u5de7\u90a3\u5929\u5e03\u83b1\u6069\u548c \u2026\nURL: https://duckduckgo.com/?q=What+happened+in+tech+news+this+week?\n\nUser Question: What happened in tech news this week?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", - "prompt.prompt_length": 1206, - "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nhappened\u7684\u7528\u6cd5\u5982\u4e0b\uff1a 1.happened\u662f \u4e0d\u53ca\u7269\u52a8\u8bcd,\u4f5c\u201c\u53d1\u751f\u201d\u89e3\u65f6,\u53ef\u6307\u5076\u7136\u7684\u4e8b\u6545,\u4e5f\u53ef\u6307\u6709\u8ba1\u5212\u6216\u65e0\u8ba1\u5212\u53d1\u751f\u7684\u4e8b\u3002\u4e00\u822c\u4ee5\u4e8b\u4ef6\u3001\u73af\u5883\u3001\u5f62\u52bf\u7b49\u4f5c\u4e3b\u8bed,\u8868\u793a\u201c\u67d0\u4eba\u53d1\u751f\u67d0\u4e8b\u201d\u65f6,\u987b\u4ee5\u4ecb\u8bcdto\u5f15\u51fa\u201c\u67d0\u4eba\u201d\u3002 2. \u2026 what happened\u8fd8\u662fwhat's happened\"What happened\"\u548c\"What's happene...", - "prompt.instruction_following": "web_search_optimized", - "response.answer_length": 510, - "response.answer_completeness": "complete", - "response.response_quality": "medium", - "response.source_citation": 1, - "response.context_utilization": 0.7044198895027625, - "output_response": "Web Search Answer: The web search results do not provide specific information about recent events in tech news for this week. They primarily discuss the grammatical usage of the word \"happened\" and its variations, such ...", - "evaluation.search_effectiveness": 0.2, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 1, - "evaluation.context_relevance": "medium", - "evaluation.overall_quality": "low", - "evaluation.ready_for_production": true, - "web_search.search_strategy": "duckduckgo_api", - "web_search.real_time_data": true, - "web_search.external_sources": 1, - "web_search.information_freshness": "current", - "web_search.knowledge_synthesis": "single_source" - }, - "events": [], - "links": [] - } - ], - "created_at": "2025-10-22 16:58:11.033000000", - "updated_at": "2025-10-22 16:58:11.033000000" - }, - { - "trace_id": "45efae74-ee25-4004-a509-6e0ff628a0cd", - "name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "status": "ok", - "status_message": "", - "start_time": "2025-10-22 16:58:04.883000000", - "end_time": "2025-10-22 16:58:10.029000000", - "duration_ms": 5146.244000000001, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "spans": [ - { - "span_id": "82eb19e8-f6cf-43fd-ba56-640a6fd3ed0e", - "trace_id": "45efae74-ee25-4004-a509-6e0ff628a0cd", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-22 16:58:04.883000000", - "end_time": "2025-10-22 16:58:10.029000000", - "duration_ms": 5146.146, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What are the current trends in observability tools?", - "query.length": 51, - "processing.mode": "RAG", - "processing.time_seconds": 5.146108150482178, - "processing.response_length": 815, - "processing.sources_count": 5, - "output_response": "Final Answer: The provided context does not explicitly outline current trends in observability tools. However, it highlights the unique challenges faced by AI applications that traditional monitoring tools cannot a...", - "final_answer_mode": "RAG", - "query_processed.input_query": "What are the current trends in observability tools?", - "query_processed.output_response": "Successfully processed query using RAG, generated 815 character response", - "query_processed.mode": "RAG", - "query_processed.processing_time": 5.146108150482178, - "query_processed.response_length": 815 - }, - "events": [], - "links": [] - }, - { - "span_id": "e81617c6-c2a1-4b08-a5fc-dde3439fe2f4", - "trace_id": "45efae74-ee25-4004-a509-6e0ff628a0cd", - "parent_span_id": "8001ba0e-95c7-42da-9087-603a38e1a500", - "name": "agent.llm-rag", - "start_time": "2025-10-22 16:58:04.883000000", - "end_time": "2025-10-22 16:58:10.029000000", - "duration_ms": 5145.655, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "rag_agent", - "agent.operation": "llm-rag", - "agent.capabilities": [ - "document_retrieval", - "context_generation", - "response_generation" - ], - "agent.id": "noveum_rag_agent", - "input_query": "What are the current trends in observability tools?", - "query_length": 51, - "query_type": "rag_query", - "output_response": "RAG Answer: The provided context does not explicitly outline current trends in observability tools. However, it highlights the unique challenges faced by AI applications that traditional monitoring tools cannot a...", - "rag.context_length": 2943, - "rag.sources_count": 5, - "rag.answer_length": 815, - "rag.mode": "retrieval_augmented_generation" - }, - "events": [], - "links": [] - }, - { - "span_id": "8001ba0e-95c7-42da-9087-603a38e1a500", - "trace_id": "45efae74-ee25-4004-a509-6e0ff628a0cd", - "parent_span_id": "82eb19e8-f6cf-43fd-ba56-640a6fd3ed0e", - "name": "agent.query_routing", - "start_time": "2025-10-22 16:58:04.883000000", - "end_time": "2025-10-22 16:58:10.029000000", - "duration_ms": 5146.036999999999, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What are the current trends in observability tools?", - "query_length": 51, - "query_type": "routing_query", - "output_response": "Routed to RAG for query processing", - "router.classification": "RAG", - "router.confidence_score": 0.3333333333333333, - "router.rag_keyword_score": 2, - "router.web_keyword_score": 1, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [] - }, - { - "span_id": "79b7d138-05e1-4f18-8b63-dcfe9732180b", - "trace_id": "45efae74-ee25-4004-a509-6e0ff628a0cd", - "parent_span_id": "8001ba0e-95c7-42da-9087-603a38e1a500", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-22 16:58:04.883000000", - "end_time": "2025-10-22 16:58:04.883000000", - "duration_ms": 0.051, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What are the current trends in observability tools?", - "query_length": 51, - "query_type": "routing_evaluation_query", - "classification.mode": "RAG", - "classification.rag_keyword_score": 2, - "classification.web_keyword_score": 1, - "classification.confidence_score": 0.3333333333333333, - "classification.confidence_level": "medium", - "classification.method": "keyword_based", - "query.complexity": "complex", - "query.intent": "documentation", - "query.keyword_density": 0.375, - "query.domain_affinity": "noveum", - "routing.decision": "Routed to RAG based on analysis", - "routing.rationale": "RAG score: 2, Web score: 1, Confidence: 0.33", - "routing.expected_performance": "medium", - "routing.alternative_mode": "Web Search", - "routing.decision_confidence": 0.3333333333333333, - "evaluation.routing_accuracy": "medium", - "evaluation.keyword_coverage": 0.057692307692307696, - "evaluation.query_understanding": "ambiguous", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: RAG (Confidence: 0.33)" - }, - "events": [], - "links": [] - }, - { - "span_id": "28c233a0-d42a-4200-86b0-dfbd2d5be06e", - "trace_id": "45efae74-ee25-4004-a509-6e0ff628a0cd", - "parent_span_id": "e81617c6-c2a1-4b08-a5fc-dde3439fe2f4", - "name": "agent.llm_model_execution", - "start_time": "2025-10-22 16:58:05.635000000", - "end_time": "2025-10-22 16:58:09.222000000", - "duration_ms": 3586.924, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What are the current trends in observability tools?", - "query_length": 51, - "query_type": "rag_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 3.5868873596191406, - "model.latency_ms": 3586.8873596191406, - "model.start_time": 1761152285.6352398, - "model.end_time": 1761152289.2221272, - "model.performance_tier": "medium", - "model.prompt_tokens": 872, - "model.completion_tokens": 203, - "model.total_tokens": 1075, - "model.tokens_per_second": 299.7027484337129, - "model.estimated_cost": 0.03225, - "model.efficiency_score": 0.7581395348837209, - "model.response_length": 815, - "model.response_quality": "high", - "model.output_response": "Model Response: The provided context does not explicitly outline current trends in observability tools. However, it highlights the unique challenges faced by AI applications that traditional monitoring tools cannot a...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [] - }, - { - "span_id": "024c05b6-edf9-40ef-931b-9660d48b4948", - "trace_id": "45efae74-ee25-4004-a509-6e0ff628a0cd", - "parent_span_id": "e81617c6-c2a1-4b08-a5fc-dde3439fe2f4", - "name": "agent.rag_evaluation_metrics", - "start_time": "2025-10-22 16:58:09.222000000", - "end_time": "2025-10-22 16:58:09.222000000", - "duration_ms": 0.020999999999999998, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "rag_evaluation_metrics", - "agent.capabilities": [ - "retrieval_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What are the current trends in observability tools?", - "query_length": 51, - "query_type": "rag_evaluation_query", - "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/blog/from-logs-to-intelligent-choices-inside-noveum-ais-evaluation-process):\nFrom Development to Production - Inside Noveum.ai's AI Observability Platform\nShashank Agarwal\n3/3/2025\nIntroduction\nEvery AI application tells a story\u2014but most developers never get to hear it...", - "retrieval.context_length": 2943, - "retrieval.sources_count": 5, - "retrieval.context_quality": "high", - "retrieval.effectiveness": 1, - "retrieval.context_utilization": 2.943, - "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/blog/from-logs-to-intelligent-choices-inside-noveum-ais-evaluation-process):\nFrom Development to Production - Inside Noveum.ai's AI Observability Platform\nShashank Agarwal\n3/3/2025\nIntroduction\nEvery AI application tells a story\u2014but most developers never get to hear it. When your RAG pipeline returns irrelevant results, when your multi-agent system gets stuck in loops, or when your LLM costs suddenly spike, you're left guessing what went wrong and where.\nThat's exactly the challenge Noveum.ai solves. Rather than flying blind, our platform provides comprehensive tracing a...\n\nSource 2 (https://noveum.ai/docs/getting-started/tracing-concepts):\n- Explore Framework Integrations - Framework-specific guidance\n- Learn Advanced Patterns - Custom instrumentation techniques\n- Master the Dashboard - Analyze your traces effectively\nRemember: Good observability is not about collecting all possible data, but about collecting the right data that helps you understand, debug, and optimize your AI applications.\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to O...\n\nSource 3 (https://noveum.ai/en/docs/getting-started/tracing-concepts):\n- Explore Framework Integrations - Framework-specific guidance\n- Learn Advanced Patterns - Custom instrumentation techniques\n- Master the Dashboard - Analyze your traces effectively\nRemember: Good observability is not about collecting all possible data, but about collecting the right data that helps you understand, debug, and optimize your AI applications.\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to O...\n\nSource 4 (https://noveum.ai/en/blog/from-logs-to-intelligent-choices-inside-noveum-ais-evaluation-process):\nThe AI Observability Challenge\nWhy Traditional Monitoring Falls Short\nTraditional application monitoring tools weren't built for AI applications. They can tell you if your API is responding, but they can't answer the questions that matter most for AI systems:\n- Why did my RAG pipeline retrieve irrelevant documents?\n- Which LLM calls are driving my costs?\n- How are my agents communicating with each other?\n- What's causing hallucinations in my responses?\n- Why is my embedding generation so slow?\nW...\n\nSource 5 (https://noveum.ai/en/docs/advanced/rag-observability):\n\ud83d\udd27 Best Practices for RAG Observability\n1. Comprehensive Pipeline Tracing\n2. Quality Monitoring\n3. A/B Testing for RAG Components\n\ud83c\udfaf Advanced RAG Patterns\nMulti-Modal RAG\nConversational RAG\nRAG observability with Noveum.ai provides the deep insights needed to build, optimize, and scale retrieval-augmented generation systems. By implementing comprehensive tracing across retrieval, context preparation, and generation stages, you can ensure your RAG pipeline delivers accurate, relevant, and cost-effe...\n\nUser Question: What are the current trends in observability tools?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", - "prompt.prompt_length": 3490, - "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/blog/from-logs-to-intelligent-choices-inside-noveum-ais-evaluation-process):\nFrom Development to Production - Inside Noveum.ai's AI Observability Platform\nShashank Agarw...", - "prompt.instruction_following": "rag_optimized", - "response.answer_length": 815, - "response.answer_completeness": "complete", - "response.response_quality": "high", - "response.source_citation": 5, - "response.context_utilization": 0.2769283044512402, - "output_response": "RAG Answer: The provided context does not explicitly outline current trends in observability tools. However, it highlights the unique challenges faced by AI applications that traditional monitoring tools cannot a...", - "evaluation.retrieval_effectiveness": 1, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 5, - "evaluation.context_relevance": "high", - "evaluation.overall_quality": "high", - "evaluation.ready_for_production": true, - "rag.retrieval_strategy": "semantic_similarity", - "rag.vector_search_results": 5, - "rag.context_synthesis": "multi_source", - "rag.document_coverage": 1, - "rag.information_density": 0.2769283044512402 - }, - "events": [], - "links": [] - } - ], - "created_at": "2025-10-22 16:58:04.883000000", - "updated_at": "2025-10-22 16:58:04.883000000" - }, - { - "trace_id": "c5a64d08-d00f-46cf-ab3c-d8d473c10eca", - "name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "status": "ok", - "status_message": "", - "start_time": "2025-10-22 16:57:57.908000000", - "end_time": "2025-10-22 16:58:03.875000000", - "duration_ms": 5967.0779999999995, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "spans": [ - { - "span_id": "99a5e8f3-a9f5-4953-94ea-ad0fa0cf6ec2", - "trace_id": "c5a64d08-d00f-46cf-ab3c-d8d473c10eca", - "parent_span_id": "2a351fd9-a7ee-42da-9e2e-86a643ceed0e", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-22 16:57:57.908000000", - "end_time": "2025-10-22 16:57:57.908000000", - "duration_ms": 0.026, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: Tell me about recent developments in machine learning", - "query_length": 53, - "query_type": "routing_evaluation_query", - "classification.mode": "Web Search", - "classification.rag_keyword_score": 0, - "classification.web_keyword_score": 1, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "complex", - "query.intent": "general_knowledge", - "query.keyword_density": 0.125, - "query.domain_affinity": "general", - "routing.decision": "Routed to Web Search based on analysis", - "routing.rationale": "RAG score: 0, Web score: 1, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "RAG", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.019230769230769232, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: Web Search (Confidence: 1.00)" - }, - "events": [], - "links": [] - }, - { - "span_id": "6e891e51-ca6d-4e6c-80d0-0791ee22d5b9", - "trace_id": "c5a64d08-d00f-46cf-ab3c-d8d473c10eca", - "parent_span_id": "2a351fd9-a7ee-42da-9e2e-86a643ceed0e", - "name": "agent.web_search_generation", - "start_time": "2025-10-22 16:57:57.908000000", - "end_time": "2025-10-22 16:58:03.874000000", - "duration_ms": 5966.728, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "web_search_agent", - "agent.operation": "web_search_generation", - "agent.capabilities": [ - "web_search", - "content_synthesis", - "response_generation" - ], - "agent.id": "noveum_web_search_agent", - "input_query": "Tell me about recent developments in machine learning", - "query_length": 53, - "query_type": "web_search_query", - "output_response": "Web Search Answer: The web search results did not provide specific information about recent developments in machine learning. Therefore, I cannot summarize any recent advancements based on the provided data.\n\nHowever, i...", - "web_search.results_count": 1, - "web_search.context_length": 768, - "web_search.response_length": 687, - "web_search.mode": "external_web_search" - }, - "events": [], - "links": [] - }, - { - "span_id": "6d622376-d48b-4a64-b0a2-1402117febaf", - "trace_id": "c5a64d08-d00f-46cf-ab3c-d8d473c10eca", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-22 16:57:57.908000000", - "end_time": "2025-10-22 16:58:03.875000000", - "duration_ms": 5967.035, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "Tell me about recent developments in machine learning", - "query.length": 53, - "processing.mode": "Web Search", - "processing.time_seconds": 5.966889142990112, - "processing.response_length": 687, - "processing.sources_count": 1, - "output_response": "Final Answer: The web search results did not provide specific information about recent developments in machine learning. Therefore, I cannot summarize any recent advancements based on the provided data.\n\nHowever, i...", - "final_answer_mode": "Web Search", - "query_processed.input_query": "Tell me about recent developments in machine learning", - "query_processed.output_response": "Successfully processed query using Web Search, generated 687 character response", - "query_processed.mode": "Web Search", - "query_processed.processing_time": 5.966889142990112, - "query_processed.response_length": 687 - }, - "events": [], - "links": [] - }, - { - "span_id": "2a351fd9-a7ee-42da-9e2e-86a643ceed0e", - "trace_id": "c5a64d08-d00f-46cf-ab3c-d8d473c10eca", - "parent_span_id": "6d622376-d48b-4a64-b0a2-1402117febaf", - "name": "agent.query_routing", - "start_time": "2025-10-22 16:57:57.908000000", - "end_time": "2025-10-22 16:58:03.875000000", - "duration_ms": 5966.8640000000005, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "Tell me about recent developments in machine learning", - "query_length": 53, - "query_type": "routing_query", - "output_response": "Routed to Web Search for query processing", - "router.classification": "Web Search", - "router.confidence_score": 1, - "router.rag_keyword_score": 0, - "router.web_keyword_score": 1, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [] - }, - { - "span_id": "ba457b3b-9be8-4e24-af63-4118833711e6", - "trace_id": "c5a64d08-d00f-46cf-ab3c-d8d473c10eca", - "parent_span_id": "6e891e51-ca6d-4e6c-80d0-0791ee22d5b9", - "name": "agent.llm_model_execution", - "start_time": "2025-10-22 16:57:58.324000000", - "end_time": "2025-10-22 16:58:03.874000000", - "duration_ms": 5550.059, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: Tell me about recent developments in machine learning", - "query_length": 53, - "query_type": "web_search_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 5.550010919570923, - "model.latency_ms": 5550.010919570923, - "model.start_time": 1761152278.3248482, - "model.end_time": 1761152283.874859, - "model.performance_tier": "slow", - "model.prompt_tokens": 316, - "model.completion_tokens": 171, - "model.total_tokens": 487, - "model.tokens_per_second": 87.74757510524871, - "model.estimated_cost": 0.01461, - "model.efficiency_score": 1.4106776180698153, - "model.response_length": 687, - "model.response_quality": "high", - "model.output_response": "Model Response: The web search results did not provide specific information about recent developments in machine learning. Therefore, I cannot summarize any recent advancements based on the provided data.\n\nHowever, i...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [] - }, - { - "span_id": "7b70a916-1c2b-4898-a052-cfd98a979167", - "trace_id": "c5a64d08-d00f-46cf-ab3c-d8d473c10eca", - "parent_span_id": "6e891e51-ca6d-4e6c-80d0-0791ee22d5b9", - "name": "agent.web_search_evaluation_metrics", - "start_time": "2025-10-22 16:58:03.874000000", - "end_time": "2025-10-22 16:58:03.874000000", - "duration_ms": 0.02, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "web_search_evaluation_metrics", - "agent.capabilities": [ - "web_search_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: Tell me about recent developments in machine learning", - "query_length": 53, - "query_type": "web_search_evaluation_query", - "web_search.results_count": 1, - "web_search.context_length": 768, - "web_search.context_synthesized": "Context from 1 web sources", - "web_search.search_effectiveness": 0.2, - "web_search.context_quality": "medium", - "web_search.source_diversity": 1, - "web_search.information_synthesis": "low", - "web_search.external_knowledge_utilization": 0.768, - "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nFeb 3, 2024 \u00b7 tell sb sth\u4e2d\u7684tell\u66f4\u591a\u662f\u544a\u77e5\u7684\u610f\u601d\uff0c\u5c31\u662f\u544a\u8bc9\u522b\u4eba\u6709\u8fd9\u4e48\u4e00\u4e2a\u4e8b\u7684\u5b58\u5728 tell sb about sth \u7684\u610f\u601d\u5305\u542b\u66f4\u591a\u4e00\u4e9b\uff0c\u9664\u4e86\u544a\u77e5\u6709\u8fd9\u4e48\u4e00\u4e2a\u4e8b\u60c5\u7684\u5b58\u5728\uff0c\u8fd8\u4f5c\u4e86\u4e00\u4e9b\u8bf4\u660e \u6bd4\u5982\uff1a tell me he \u2026 \u4e00\u3001Say\u548cTell\u7684\u7528\u6cd5 \u6211\u4eec\u5148\u5c06say\u3001tell\u3001speak\u3001talk\u8fd9\u56db\u4e2a\u8bcd\u5206\u6210\u4e24\u7ec4\uff0c\u5206\u522b\u4e3a\uff1a \u7b2c\u2460\u7ec4\uff1asay & tell\uff0c\u8fd9\u4e24\u4e2a\u8bcd\u4e4b\u540e\u53ef\u4ee5\u8868\u793a\u8c08\u8bdd\u7684 \u201c\u7ec6\u8282\u201d\u3002 \u7b2c\u2461\u7ec4\uff1aspeak & talk\uff0c\u4e0d\u80fd\u8868\u793a\u8c08\u8bdd\u7684\u201c\u7ec6\u8282\u201d\u3002 \u2026 tell of 1\u3001Tell of a turning point in your career and what you learned. \u8bb2\u8ff0\u4f60 \u804c\u4e1a\u751f\u6daf \u4e00\u4e2a\u8f6c\u6298\u70b9\uff0c\u4ee5\u53ca\u4f60\u5b66\u5230\u4e86\u4ec0\u4e48\u3002 2\u3001I have made the Sovereign Lord my refuge; I will tell of all your \u2026 tell sb to do sth \u548c tell sb do sth \u90fd\u8868\u793a\u544a\u8bc9\u67d0\u4eba\u505a\u67d0\u4e8b\uff0ctell sb to do sth \u5f3a\u8c03\u544a\u8bc9\u67d0\u4eba\u5e94\u8be5\u505a\u4ec0\u4e48\u5177\u4f53\u52a8\u4f5c\u6216\u884c\u4e3a\uff0c\u800c tell sb do sth \u5f3a\u8c03\u544a\u8bc9\u67d0\u4eba\u6b63\u5728\u8fdb\u884c\u7684\u5177\u4f53\u884c\u52a8\u6216\u72b6\u6001\u3002 \u5b83\u4eec\u5728\u91ca\u4e49\u3001 \u2026 Aug 23, 2023 \u00b7 tell\u7684\u8fc7\u53bb\u5f0f\u548c\u8fc7\u53bb\u5206\u8bcd\u90fd\u662ftold\u3002 \u8bfb\u97f3\uff1a\u82f1[tel]\uff0c\u7f8e[tel]\u3002 \u91ca\u4e49\uff1a vt.\u544a\u8bc9\uff0c\u8bf4\uff1b\u8fa8\u522b\uff1b\u5429\u5490\uff1b\u65ad\u5b9a\u3002 vi.\u8bb2\u8ff0\uff1b\u544a\u53d1\uff0c\u6cc4\u5bc6\uff1b\u8bc6\u522b\u3002 \u53d8\u5f62\uff1a\u8fc7\u53bb\u5f0ftold\uff0c\u8fc7\u53bb\u5206\u8bcdtold\uff0c\u73b0\u5728\u5206 \u2026\nURL: https://duckduckgo.com/?q=Tell+me+about+recent+developments+in+machine+learning\n\nUser Question: Tell me about recent developments in machine learning\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", - "prompt.prompt_length": 1266, - "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nFeb 3, 2024 \u00b7 tell sb sth\u4e2d\u7684tell\u66f4\u591a\u662f\u544a\u77e5\u7684\u610f\u601d\uff0c\u5c31\u662f\u544a\u8bc9\u522b\u4eba\u6709\u8fd9\u4e48\u4e00\u4e2a\u4e8b\u7684\u5b58\u5728 tell sb about sth \u7684\u610f\u601d\u5305\u542b\u66f4\u591a\u4e00\u4e9b\uff0c\u9664\u4e86\u544a\u77e5\u6709\u8fd9\u4e48\u4e00\u4e2a\u4e8b\u60c5\u7684\u5b58\u5728\uff0c\u8fd8\u4f5c\u4e86\u4e00\u4e9b\u8bf4\u660e \u6bd4\u5982\uff1a tell me he \u2026 \u4e00\u3001Say\u548cTell\u7684\u7528\u6cd5 \u6211\u4eec\u5148\u5c06say\u3001tell\u3001speak\u3001talk\u8fd9\u56db\u4e2a\u8bcd\u5206\u6210\u4e24\u7ec4\uff0c\u5206\u522b...", - "prompt.instruction_following": "web_search_optimized", - "response.answer_length": 687, - "response.answer_completeness": "complete", - "response.response_quality": "medium", - "response.source_citation": 1, - "response.context_utilization": 0.89453125, - "output_response": "Web Search Answer: The web search results did not provide specific information about recent developments in machine learning. Therefore, I cannot summarize any recent advancements based on the provided data.\n\nHowever, i...", - "evaluation.search_effectiveness": 0.2, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 1, - "evaluation.context_relevance": "medium", - "evaluation.overall_quality": "low", - "evaluation.ready_for_production": true, - "web_search.search_strategy": "duckduckgo_api", - "web_search.real_time_data": true, - "web_search.external_sources": 1, - "web_search.information_freshness": "current", - "web_search.knowledge_synthesis": "single_source" - }, - "events": [], - "links": [] - } - ], - "created_at": "2025-10-22 16:57:57.908000000", - "updated_at": "2025-10-22 16:57:57.908000000" - }, - { - "trace_id": "b1b13e60-5b32-44a0-9769-197a944989d0", - "name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "status": "ok", - "status_message": "", - "start_time": "2025-10-22 16:57:51.967000000", - "end_time": "2025-10-22 16:57:56.899000000", - "duration_ms": 4932.393, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "spans": [ - { - "span_id": "1e1a341f-66c1-48b0-95eb-fb092928c473", - "trace_id": "b1b13e60-5b32-44a0-9769-197a944989d0", - "parent_span_id": "1c5c4291-bbb5-4f85-b6d9-b6e8df66f742", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-22 16:57:51.967000000", - "end_time": "2025-10-22 16:57:51.967000000", - "duration_ms": 0.033, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What's the weather like today?", - "query_length": 30, - "query_type": "routing_evaluation_query", - "classification.mode": "Web Search", - "classification.rag_keyword_score": 0, - "classification.web_keyword_score": 2, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "general_knowledge", - "query.keyword_density": 0.4, - "query.domain_affinity": "general", - "routing.decision": "Routed to Web Search based on analysis", - "routing.rationale": "RAG score: 0, Web score: 2, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "RAG", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.038461538461538464, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: Web Search (Confidence: 1.00)" - }, - "events": [], - "links": [] - }, - { - "span_id": "1c5c4291-bbb5-4f85-b6d9-b6e8df66f742", - "trace_id": "b1b13e60-5b32-44a0-9769-197a944989d0", - "parent_span_id": "49e7cace-ba21-4807-a238-9a3880487583", - "name": "agent.query_routing", - "start_time": "2025-10-22 16:57:51.967000000", - "end_time": "2025-10-22 16:57:56.899000000", - "duration_ms": 4931.972, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What's the weather like today?", - "query_length": 30, - "query_type": "routing_query", - "output_response": "Routed to Web Search for query processing", - "router.classification": "Web Search", - "router.confidence_score": 1, - "router.rag_keyword_score": 0, - "router.web_keyword_score": 2, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [] - }, - { - "span_id": "49e7cace-ba21-4807-a238-9a3880487583", - "trace_id": "b1b13e60-5b32-44a0-9769-197a944989d0", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-22 16:57:51.967000000", - "end_time": "2025-10-22 16:57:56.899000000", - "duration_ms": 4932.316, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What's the weather like today?", - "query.length": 30, - "processing.mode": "Web Search", - "processing.time_seconds": 4.932007789611816, - "processing.response_length": 260, - "processing.sources_count": 1, - "output_response": "Final Answer: The web search results do not provide specific information about today's weather. To find out the current weather conditions, I recommend checking a reliable weather website or app for detailed update...", - "final_answer_mode": "Web Search", - "query_processed.input_query": "What's the weather like today?", - "query_processed.output_response": "Successfully processed query using Web Search, generated 260 character response", - "query_processed.mode": "Web Search", - "query_processed.processing_time": 4.932007789611816, - "query_processed.response_length": 260 - }, - "events": [], - "links": [] - }, - { - "span_id": "58d217aa-4350-43e8-8813-468149dc4107", - "trace_id": "b1b13e60-5b32-44a0-9769-197a944989d0", - "parent_span_id": "1c5c4291-bbb5-4f85-b6d9-b6e8df66f742", - "name": "agent.web_search_generation", - "start_time": "2025-10-22 16:57:51.967000000", - "end_time": "2025-10-22 16:57:56.899000000", - "duration_ms": 4931.796, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "web_search_agent", - "agent.operation": "web_search_generation", - "agent.capabilities": [ - "web_search", - "content_synthesis", - "response_generation" - ], - "agent.id": "noveum_web_search_agent", - "input_query": "What's the weather like today?", - "query_length": 30, - "query_type": "web_search_query", - "output_response": "Web Search Answer: The web search results do not provide specific information about today's weather. To find out the current weather conditions, I recommend checking a reliable weather website or app for detailed update...", - "web_search.results_count": 1, - "web_search.context_length": 995, - "web_search.response_length": 260, - "web_search.mode": "external_web_search" - }, - "events": [], - "links": [] - }, - { - "span_id": "d89dd84e-0b5e-4c92-942b-18c4735713d4", - "trace_id": "b1b13e60-5b32-44a0-9769-197a944989d0", - "parent_span_id": "58d217aa-4350-43e8-8813-468149dc4107", - "name": "agent.llm_model_execution", - "start_time": "2025-10-22 16:57:55.269000000", - "end_time": "2025-10-22 16:57:56.899000000", - "duration_ms": 1629.7140000000002, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What's the weather like today?", - "query_length": 30, - "query_type": "web_search_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 1.629619836807251, - "model.latency_ms": 1629.619836807251, - "model.start_time": 1761152275.269459, - "model.end_time": 1761152276.8990788, - "model.performance_tier": "fast", - "model.prompt_tokens": 367, - "model.completion_tokens": 65, - "model.total_tokens": 432, - "model.tokens_per_second": 265.0925020932329, - "model.estimated_cost": 0.012960000000000001, - "model.efficiency_score": 0.6018518518518519, - "model.response_length": 260, - "model.response_quality": "high", - "model.output_response": "Model Response: The web search results do not provide specific information about today's weather. To find out the current weather conditions, I recommend checking a reliable weather website or app for detailed update...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [] - }, - { - "span_id": "d076719d-da79-43fb-92ea-6015f856146b", - "trace_id": "b1b13e60-5b32-44a0-9769-197a944989d0", - "parent_span_id": "58d217aa-4350-43e8-8813-468149dc4107", - "name": "agent.web_search_evaluation_metrics", - "start_time": "2025-10-22 16:57:56.899000000", - "end_time": "2025-10-22 16:57:56.899000000", - "duration_ms": 0.049, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "web_search_evaluation_metrics", - "agent.capabilities": [ - "web_search_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What's the weather like today?", - "query_length": 30, - "query_type": "web_search_evaluation_query", - "web_search.results_count": 1, - "web_search.context_length": 995, - "web_search.context_synthesized": "Context from 1 web sources", - "web_search.search_effectiveness": 0.2, - "web_search.context_quality": "high", - "web_search.source_diversity": 1, - "web_search.information_synthesis": "low", - "web_search.external_knowledge_utilization": 0.995, - "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nLog in to WhatsApp Web for simple, reliable and private messaging on your desktop. Send and receive messages and files with ease, all for free. 2 days ago \u00b7 More videos you may like 00:13 Wanna be yours #kimpolin #colinbridgerton #benedictbri... 2 hours ago \u00b7 101 views 00:14 Viscount & Vicountess's Marital bliss #bridgerton \u2026 Nov 15, 2024 \u00b7 Learn the correct usage of \"what's the weather like today\" and \"how's the weather today\" in English. Discover differences, examples, alternatives and tips for choosing the right \u2026 Nov 15, 2024 \u00b7 Learn the correct usage of \"how's the weather today\" and \"what's the weather today\" in English. Discover differences, examples, alternatives and tips for choosing the right \u2026 2 days ago \u00b7 Get the latest hourly weather updates for Thane today. Detailed forecast including temperature, wind, rain, snow, and UV index. Stay informed about today's weather conditions in \u2026\nURL: https://duckduckgo.com/?q=What's+the+weather+like+today?\n\nUser Question: What's the weather like today?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", - "prompt.prompt_length": 1470, - "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nLog in to WhatsApp Web for simple, reliable and private messaging on your desktop. Send and receive messages and files with ease, all for free. 2 days ago \u00b7 More videos you...", - "prompt.instruction_following": "web_search_optimized", - "response.answer_length": 260, - "response.answer_completeness": "complete", - "response.response_quality": "medium", - "response.source_citation": 1, - "response.context_utilization": 0.2613065326633166, - "output_response": "Web Search Answer: The web search results do not provide specific information about today's weather. To find out the current weather conditions, I recommend checking a reliable weather website or app for detailed update...", - "evaluation.search_effectiveness": 0.2, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 1, - "evaluation.context_relevance": "high", - "evaluation.overall_quality": "low", - "evaluation.ready_for_production": true, - "web_search.search_strategy": "duckduckgo_api", - "web_search.real_time_data": true, - "web_search.external_sources": 1, - "web_search.information_freshness": "current", - "web_search.knowledge_synthesis": "single_source" - }, - "events": [], - "links": [] - } - ], - "created_at": "2025-10-22 16:57:51.967000000", - "updated_at": "2025-10-22 16:57:51.967000000" - }, - { - "trace_id": "e4f3cc20-c137-4905-80fe-2d29b03c2231", - "name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "status": "ok", - "status_message": "", - "start_time": "2025-10-22 16:57:47.477000000", - "end_time": "2025-10-22 16:57:50.959000000", - "duration_ms": 3481.856, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "spans": [ - { - "span_id": "41167ca4-e1ad-48a5-b5eb-aa272ad88a85", - "trace_id": "e4f3cc20-c137-4905-80fe-2d29b03c2231", - "parent_span_id": "56959f47-5219-4674-8c0d-e5fa928dcd45", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-22 16:57:47.477000000", - "end_time": "2025-10-22 16:57:47.477000000", - "duration_ms": 0.061, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What are the latest AI news today?", - "query_length": 34, - "query_type": "routing_evaluation_query", - "classification.mode": "Web Search", - "classification.rag_keyword_score": 0, - "classification.web_keyword_score": 3, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "general_knowledge", - "query.keyword_density": 0.42857142857142855, - "query.domain_affinity": "general", - "routing.decision": "Routed to Web Search based on analysis", - "routing.rationale": "RAG score: 0, Web score: 3, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "RAG", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.057692307692307696, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: Web Search (Confidence: 1.00)" - }, - "events": [], - "links": [] - }, - { - "span_id": "73005751-9eb1-4540-aa49-2ac580cd7418", - "trace_id": "e4f3cc20-c137-4905-80fe-2d29b03c2231", - "parent_span_id": "56959f47-5219-4674-8c0d-e5fa928dcd45", - "name": "agent.web_search_generation", - "start_time": "2025-10-22 16:57:47.477000000", - "end_time": "2025-10-22 16:57:50.958000000", - "duration_ms": 3480.8469999999998, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "web_search_agent", - "agent.operation": "web_search_generation", - "agent.capabilities": [ - "web_search", - "content_synthesis", - "response_generation" - ], - "agent.id": "noveum_web_search_agent", - "input_query": "What are the latest AI news today?", - "query_length": 34, - "query_type": "web_search_query", - "output_response": "Web Search Answer: The provided web search results do not contain specific information about the latest AI news today. They primarily focus on general news coverage from India and around the world, including politics, b...", - "web_search.results_count": 1, - "web_search.context_length": 887, - "web_search.response_length": 503, - "web_search.mode": "external_web_search" - }, - "events": [], - "links": [] - }, - { - "span_id": "9e1b87be-bc29-4084-b37c-6e1220dde8eb", - "trace_id": "e4f3cc20-c137-4905-80fe-2d29b03c2231", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-22 16:57:47.477000000", - "end_time": "2025-10-22 16:57:50.959000000", - "duration_ms": 3481.7580000000003, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What are the latest AI news today?", - "query.length": 34, - "processing.mode": "Web Search", - "processing.time_seconds": 3.4813451766967773, - "processing.response_length": 503, - "processing.sources_count": 1, - "output_response": "Final Answer: The provided web search results do not contain specific information about the latest AI news today. They primarily focus on general news coverage from India and around the world, including politics, b...", - "final_answer_mode": "Web Search", - "query_processed.input_query": "What are the latest AI news today?", - "query_processed.output_response": "Successfully processed query using Web Search, generated 503 character response", - "query_processed.mode": "Web Search", - "query_processed.processing_time": 3.4813451766967773, - "query_processed.response_length": 503 - }, - "events": [], - "links": [] - }, - { - "span_id": "56959f47-5219-4674-8c0d-e5fa928dcd45", - "trace_id": "e4f3cc20-c137-4905-80fe-2d29b03c2231", - "parent_span_id": "9e1b87be-bc29-4084-b37c-6e1220dde8eb", - "name": "agent.query_routing", - "start_time": "2025-10-22 16:57:47.477000000", - "end_time": "2025-10-22 16:57:50.958000000", - "duration_ms": 3481.254, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What are the latest AI news today?", - "query_length": 34, - "query_type": "routing_query", - "output_response": "Routed to Web Search for query processing", - "router.classification": "Web Search", - "router.confidence_score": 1, - "router.rag_keyword_score": 0, - "router.web_keyword_score": 3, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [] - }, - { - "span_id": "3dbf95ba-3ef1-4294-93d4-217f4ada5ffe", - "trace_id": "e4f3cc20-c137-4905-80fe-2d29b03c2231", - "parent_span_id": "73005751-9eb1-4540-aa49-2ac580cd7418", - "name": "agent.llm_model_execution", - "start_time": "2025-10-22 16:57:47.864000000", - "end_time": "2025-10-22 16:57:50.958000000", - "duration_ms": 3093.442, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What are the latest AI news today?", - "query_length": 34, - "query_type": "web_search_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 3.0933361053466797, - "model.latency_ms": 3093.3361053466797, - "model.start_time": 1761152267.864976, - "model.end_time": 1761152270.958312, - "model.performance_tier": "medium", - "model.prompt_tokens": 341, - "model.completion_tokens": 125, - "model.total_tokens": 466, - "model.tokens_per_second": 150.6464167261171, - "model.estimated_cost": 0.013980000000000001, - "model.efficiency_score": 1.0793991416309012, - "model.response_length": 503, - "model.response_quality": "high", - "model.output_response": "Model Response: The provided web search results do not contain specific information about the latest AI news today. They primarily focus on general news coverage from India and around the world, including politics, b...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [] - }, - { - "span_id": "f7fe795b-ce1b-4fcc-83f5-06a5f074f38d", - "trace_id": "e4f3cc20-c137-4905-80fe-2d29b03c2231", - "parent_span_id": "73005751-9eb1-4540-aa49-2ac580cd7418", - "name": "agent.web_search_evaluation_metrics", - "start_time": "2025-10-22 16:57:50.958000000", - "end_time": "2025-10-22 16:57:50.958000000", - "duration_ms": 0.026, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "web_search_evaluation_metrics", - "agent.capabilities": [ - "web_search_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What are the latest AI news today?", - "query_length": 34, - "query_type": "web_search_evaluation_query", - "web_search.results_count": 1, - "web_search.context_length": 887, - "web_search.context_synthesized": "Context from 1 web sources", - "web_search.search_effectiveness": 0.2, - "web_search.context_quality": "high", - "web_search.source_diversity": 1, - "web_search.information_synthesis": "low", - "web_search.external_knowledge_utilization": 0.887, - "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\n3 days ago \u00b7 Latest news headlines from India & around the world. Check out today\u2019s news coverage live with videos & photos on NDTV.com. 6 days ago \u00b7 India News | Latest India News | Read latest and breaking news from India. Today's top India news headlines, news on Indian politics, elections, government, business, \u2026 2 days ago \u00b7 Top News Stories of the day, Latest News Headlines, News Specials, Breaking News and Latest India News, World current affairs & Political News all around the clock at \u2026 NDTV.com provides latest news from India and around the world. Get breaking news alerts from India and follow today\u2019s live news updates in field of politics, business, technology, Bollywood ... 6 days ago \u00b7 Get Latest entertainment news from Bollywood, Hollywood and regional film industries.\nURL: https://duckduckgo.com/?q=What+are+the+latest+AI+news+today?\n\nUser Question: What are the latest AI news today?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", - "prompt.prompt_length": 1366, - "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\n3 days ago \u00b7 Latest news headlines from India & around the world. Check out today\u2019s news coverage live with videos & photos on NDTV.com. 6 days ago \u00b7 India News | Latest In...", - "prompt.instruction_following": "web_search_optimized", - "response.answer_length": 503, - "response.answer_completeness": "complete", - "response.response_quality": "medium", - "response.source_citation": 1, - "response.context_utilization": 0.5670800450958287, - "output_response": "Web Search Answer: The provided web search results do not contain specific information about the latest AI news today. They primarily focus on general news coverage from India and around the world, including politics, b...", - "evaluation.search_effectiveness": 0.2, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 1, - "evaluation.context_relevance": "high", - "evaluation.overall_quality": "low", - "evaluation.ready_for_production": true, - "web_search.search_strategy": "duckduckgo_api", - "web_search.real_time_data": true, - "web_search.external_sources": 1, - "web_search.information_freshness": "current", - "web_search.knowledge_synthesis": "single_source" - }, - "events": [], - "links": [] - } - ], - "created_at": "2025-10-22 16:57:47.477000000", - "updated_at": "2025-10-22 16:57:47.477000000" - }, - { - "trace_id": "823b7927-10bb-49ef-a94e-1b914c298bfb", - "name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "status": "ok", - "status_message": "", - "start_time": "2025-10-22 16:57:38.501000000", - "end_time": "2025-10-22 16:57:46.469000000", - "duration_ms": 7968.149, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "spans": [ - { - "span_id": "b9c9d21e-5f76-4e5a-b99e-2b4ab821caee", - "trace_id": "823b7927-10bb-49ef-a94e-1b914c298bfb", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-22 16:57:38.501000000", - "end_time": "2025-10-22 16:57:46.469000000", - "duration_ms": 7968.089, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What are the benefits of using Noveum Trace?", - "query.length": 44, - "processing.mode": "RAG", - "processing.time_seconds": 7.968040943145752, - "processing.response_length": 1396, - "processing.sources_count": 5, - "output_response": "Final Answer: Using Noveum Trace offers several benefits, including:\n\n1. **Real-Time Monitoring**: The Noveum platform provides a real-time dashboard for analyzing traces and performance, with live updates as new t...", - "final_answer_mode": "RAG", - "query_processed.input_query": "What are the benefits of using Noveum Trace?", - "query_processed.output_response": "Successfully processed query using RAG, generated 1396 character response", - "query_processed.mode": "RAG", - "query_processed.processing_time": 7.968040943145752, - "query_processed.response_length": 1396 - }, - "events": [], - "links": [] - }, - { - "span_id": "bc85c745-4c27-4aa4-bc00-4c9f10a2f049", - "trace_id": "823b7927-10bb-49ef-a94e-1b914c298bfb", - "parent_span_id": "b9c9d21e-5f76-4e5a-b99e-2b4ab821caee", - "name": "agent.query_routing", - "start_time": "2025-10-22 16:57:38.501000000", - "end_time": "2025-10-22 16:57:46.469000000", - "duration_ms": 7968.001, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What are the benefits of using Noveum Trace?", - "query_length": 44, - "query_type": "routing_query", - "output_response": "Routed to RAG for query processing", - "router.classification": "RAG", - "router.confidence_score": 1, - "router.rag_keyword_score": 2, - "router.web_keyword_score": 0, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [] - }, - { - "span_id": "3ab8cc4c-f5ed-410a-b4ee-48d0954b756d", - "trace_id": "823b7927-10bb-49ef-a94e-1b914c298bfb", - "parent_span_id": "bc85c745-4c27-4aa4-bc00-4c9f10a2f049", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-22 16:57:38.501000000", - "end_time": "2025-10-22 16:57:38.501000000", - "duration_ms": 0.09899999999999999, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What are the benefits of using Noveum Trace?", - "query_length": 44, - "query_type": "routing_evaluation_query", - "classification.mode": "RAG", - "classification.rag_keyword_score": 2, - "classification.web_keyword_score": 0, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "noveum_specific", - "query.keyword_density": 0.25, - "query.domain_affinity": "noveum", - "routing.decision": "Routed to RAG based on analysis", - "routing.rationale": "RAG score: 2, Web score: 0, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "Web Search", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.038461538461538464, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: RAG (Confidence: 1.00)" - }, - "events": [], - "links": [] - }, - { - "span_id": "67292076-ec8a-4856-9c5b-726868552157", - "trace_id": "823b7927-10bb-49ef-a94e-1b914c298bfb", - "parent_span_id": "bc85c745-4c27-4aa4-bc00-4c9f10a2f049", - "name": "agent.llm-rag", - "start_time": "2025-10-22 16:57:38.501000000", - "end_time": "2025-10-22 16:57:46.469000000", - "duration_ms": 7967.679, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "rag_agent", - "agent.operation": "llm-rag", - "agent.capabilities": [ - "document_retrieval", - "context_generation", - "response_generation" - ], - "agent.id": "noveum_rag_agent", - "input_query": "What are the benefits of using Noveum Trace?", - "query_length": 44, - "query_type": "rag_query", - "output_response": "RAG Answer: Using Noveum Trace offers several benefits, including:\n\n1. **Real-Time Monitoring**: The Noveum platform provides a real-time dashboard for analyzing traces and performance, with live updates as new t...", - "rag.context_length": 2773, - "rag.sources_count": 5, - "rag.answer_length": 1396, - "rag.mode": "retrieval_augmented_generation" - }, - "events": [], - "links": [] - }, - { - "span_id": "d044ae4a-c463-4c1b-abce-a9a18f045e75", - "trace_id": "823b7927-10bb-49ef-a94e-1b914c298bfb", - "parent_span_id": "67292076-ec8a-4856-9c5b-726868552157", - "name": "agent.llm_model_execution", - "start_time": "2025-10-22 16:57:38.901000000", - "end_time": "2025-10-22 16:57:45.885000000", - "duration_ms": 6984.242, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What are the benefits of using Noveum Trace?", - "query_length": 44, - "query_type": "rag_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 6.984165906906128, - "model.latency_ms": 6984.165906906128, - "model.start_time": 1761152258.9012191, - "model.end_time": 1761152265.885385, - "model.performance_tier": "slow", - "model.prompt_tokens": 828, - "model.completion_tokens": 349, - "model.total_tokens": 1177, - "model.tokens_per_second": 168.524060809631, - "model.estimated_cost": 0.03531, - "model.efficiency_score": 1.1860662701784197, - "model.response_length": 1396, - "model.response_quality": "high", - "model.output_response": "Model Response: Using Noveum Trace offers several benefits, including:\n\n1. **Real-Time Monitoring**: The Noveum platform provides a real-time dashboard for analyzing traces and performance, with live updates as new t...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [] - }, - { - "span_id": "5b1548cf-0de7-49d4-b9b8-9ad1f9f2abc9", - "trace_id": "823b7927-10bb-49ef-a94e-1b914c298bfb", - "parent_span_id": "67292076-ec8a-4856-9c5b-726868552157", - "name": "agent.rag_evaluation_metrics", - "start_time": "2025-10-22 16:57:45.885000000", - "end_time": "2025-10-22 16:57:45.885000000", - "duration_ms": 0.061, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "rag_evaluation_metrics", - "agent.capabilities": [ - "retrieval_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What are the benefits of using Noveum Trace?", - "query_length": 44, - "query_type": "rag_evaluation_query", - "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/docs/platform/dashboard):\n- Bulk Export: Download multiple traces simultaneously\n- Comparative Analysis: Compare performance across multiple traces\nReal-Time Updates\n- Live Refresh: Automatic updates as new traces arrive\n- Connection Monitoring: Real-time status of your t...", - "retrieval.context_length": 2773, - "retrieval.sources_count": 5, - "retrieval.context_quality": "high", - "retrieval.effectiveness": 1, - "retrieval.context_utilization": 2.773, - "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/docs/platform/dashboard):\n- Bulk Export: Download multiple traces simultaneously\n- Comparative Analysis: Compare performance across multiple traces\nReal-Time Updates\n- Live Refresh: Automatic updates as new traces arrive\n- Connection Monitoring: Real-time status of your trace ingestion pipeline\n- Performance Indicators: Live metrics for system health monitoring\nReady to dive deeper? Explore Projects & Environments to organize your AI applications, or check out Team Collaboration to share insights with your team.\nGet Earl...\n\nSource 2 (https://noveum.ai/en/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\n\ud83d\udcd8 TypeScript SDK (\n@noveum/trace\n)- Framework integrations for Next.js, Express.js, and Hono\n- TypeScript-first design with full type safety\n- Universal compatibility (Node.js, Edge Runtime, browsers)\n-\n\ud83d\udcca Noveum Platform\n- Real-time dashboard for analyzing traces and performance\n- Ad...\n\nSource 3 (https://noveum.ai/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\n\ud83d\udcd8 TypeScript SDK (\n@noveum/trace\n)- Framework integrations for Next.js, Express.js, and Hono\n- TypeScript-first design with full type safety\n- Universal compatibility (Node.js, Edge Runtime, browsers)\n-\n\ud83d\udcca Noveum Platform\n- Real-time dashboard for analyzing traces and performance\n- Ad...\n\nSource 4 (https://noveum.ai/en/docs/getting-started/overview):\n- Advanced search & filtering across traces and spans\n- Cost analysis and optimization recommendations\n- Team collaboration with shared insights and alerts\n\ud83d\udd0d What Noveum Traces\nLLM Operations\n- Model calls across all providers (OpenAI, Anthropic, Google, etc.)\n- Token usage and cost calculation\n- Prompt engineering effectiveness\n- Response quality metrics\nRAG Pipelines\n- Document retrieval performance and relevance\n- Embedding generation costs and latency\n- Context assembly and prompt constructi...\n\nSource 5 (https://noveum.ai/docs/getting-started/overview):\n- Advanced search & filtering across traces and spans\n- Cost analysis and optimization recommendations\n- Team collaboration with shared insights and alerts\n\ud83d\udd0d What Noveum Traces\nLLM Operations\n- Model calls across all providers (OpenAI, Anthropic, Google, etc.)\n- Token usage and cost calculation\n- Prompt engineering effectiveness\n- Response quality metrics\nRAG Pipelines\n- Document retrieval performance and relevance\n- Embedding generation costs and latency\n- Context assembly and prompt constructi...\n\nUser Question: What are the benefits of using Noveum Trace?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", - "prompt.prompt_length": 3313, - "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/docs/platform/dashboard):\n- Bulk Export: Download multiple traces simultaneously\n- Comparative Analysis: Compare performance across multiple traces\nReal-Time Updates\n- Live...", - "prompt.instruction_following": "rag_optimized", - "response.answer_length": 1396, - "response.answer_completeness": "complete", - "response.response_quality": "high", - "response.source_citation": 5, - "response.context_utilization": 0.5034258925351605, - "output_response": "RAG Answer: Using Noveum Trace offers several benefits, including:\n\n1. **Real-Time Monitoring**: The Noveum platform provides a real-time dashboard for analyzing traces and performance, with live updates as new t...", - "evaluation.retrieval_effectiveness": 1, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 5, - "evaluation.context_relevance": "high", - "evaluation.overall_quality": "high", - "evaluation.ready_for_production": true, - "rag.retrieval_strategy": "semantic_similarity", - "rag.vector_search_results": 5, - "rag.context_synthesis": "multi_source", - "rag.document_coverage": 1, - "rag.information_density": 0.5034258925351605 - }, - "events": [], - "links": [] - } - ], - "created_at": "2025-10-22 16:57:38.501000000", - "updated_at": "2025-10-22 16:57:38.501000000" - }, - { - "trace_id": "1ae0d6c4-d1fb-4239-b536-3333187d44aa", - "name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "status": "ok", - "status_message": "", - "start_time": "2025-10-22 16:57:32.800000000", - "end_time": "2025-10-22 16:57:37.494000000", - "duration_ms": 4694.468, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "spans": [ - { - "span_id": "aaba719b-36ec-46ab-af2e-3ea8c18e9872", - "trace_id": "1ae0d6c4-d1fb-4239-b536-3333187d44aa", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-22 16:57:32.800000000", - "end_time": "2025-10-22 16:57:37.494000000", - "duration_ms": 4694.408, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "How do I configure Noveum for my system?", - "query.length": 40, - "processing.mode": "RAG", - "processing.time_seconds": 4.694374084472656, - "processing.response_length": 553, - "processing.sources_count": 5, - "output_response": "Final Answer: To configure Noveum for your system, you can refer to the SDK Integration Guide available in the Noveum documentation. This guide will help you trace your AI applications with minimal code changes. Th...", - "final_answer_mode": "RAG", - "query_processed.input_query": "How do I configure Noveum for my system?", - "query_processed.output_response": "Successfully processed query using RAG, generated 553 character response", - "query_processed.mode": "RAG", - "query_processed.processing_time": 4.694374084472656, - "query_processed.response_length": 553 - }, - "events": [], - "links": [] - }, - { - "span_id": "4254e6f9-10b8-42ba-939d-d7c124f80f9a", - "trace_id": "1ae0d6c4-d1fb-4239-b536-3333187d44aa", - "parent_span_id": "6d5d074e-8fd3-4d33-88fb-7f54b0959030", - "name": "agent.llm-rag", - "start_time": "2025-10-22 16:57:32.800000000", - "end_time": "2025-10-22 16:57:37.494000000", - "duration_ms": 4694.155000000001, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "rag_agent", - "agent.operation": "llm-rag", - "agent.capabilities": [ - "document_retrieval", - "context_generation", - "response_generation" - ], - "agent.id": "noveum_rag_agent", - "input_query": "How do I configure Noveum for my system?", - "query_length": 40, - "query_type": "rag_query", - "output_response": "RAG Answer: To configure Noveum for your system, you can refer to the SDK Integration Guide available in the Noveum documentation. This guide will help you trace your AI applications with minimal code changes. Th...", - "rag.context_length": 2237, - "rag.sources_count": 5, - "rag.answer_length": 553, - "rag.mode": "retrieval_augmented_generation" - }, - "events": [], - "links": [] - }, - { - "span_id": "6d5d074e-8fd3-4d33-88fb-7f54b0959030", - "trace_id": "1ae0d6c4-d1fb-4239-b536-3333187d44aa", - "parent_span_id": "aaba719b-36ec-46ab-af2e-3ea8c18e9872", - "name": "agent.query_routing", - "start_time": "2025-10-22 16:57:32.800000000", - "end_time": "2025-10-22 16:57:37.494000000", - "duration_ms": 4694.318, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "How do I configure Noveum for my system?", - "query_length": 40, - "query_type": "routing_query", - "output_response": "Routed to RAG for query processing", - "router.classification": "RAG", - "router.confidence_score": 1, - "router.rag_keyword_score": 2, - "router.web_keyword_score": 0, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [] - }, - { - "span_id": "2cc2ab0b-2572-4a75-88a3-18b3a1eaca78", - "trace_id": "1ae0d6c4-d1fb-4239-b536-3333187d44aa", - "parent_span_id": "6d5d074e-8fd3-4d33-88fb-7f54b0959030", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-22 16:57:32.800000000", - "end_time": "2025-10-22 16:57:32.800000000", - "duration_ms": 0.04, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: How do I configure Noveum for my system?", - "query_length": 40, - "query_type": "routing_evaluation_query", - "classification.mode": "RAG", - "classification.rag_keyword_score": 2, - "classification.web_keyword_score": 0, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "noveum_specific", - "query.keyword_density": 0.25, - "query.domain_affinity": "noveum", - "routing.decision": "Routed to RAG based on analysis", - "routing.rationale": "RAG score: 2, Web score: 0, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "Web Search", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.038461538461538464, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: RAG (Confidence: 1.00)" - }, - "events": [], - "links": [] - }, - { - "span_id": "f1d9f779-3312-43c1-b549-00fc874429f0", - "trace_id": "1ae0d6c4-d1fb-4239-b536-3333187d44aa", - "parent_span_id": "4254e6f9-10b8-42ba-939d-d7c124f80f9a", - "name": "agent.llm_model_execution", - "start_time": "2025-10-22 16:57:33.281000000", - "end_time": "2025-10-22 16:57:37.060000000", - "duration_ms": 3778.743, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: How do I configure Noveum for my system?", - "query_length": 40, - "query_type": "rag_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 3.7786097526550293, - "model.latency_ms": 3778.6097526550293, - "model.start_time": 1761152253.28163, - "model.end_time": 1761152257.0602398, - "model.performance_tier": "medium", - "model.prompt_tokens": 693, - "model.completion_tokens": 138, - "model.total_tokens": 831, - "model.tokens_per_second": 219.92215507730066, - "model.estimated_cost": 0.02493, - "model.efficiency_score": 0.6654632972322503, - "model.response_length": 553, - "model.response_quality": "high", - "model.output_response": "Model Response: To configure Noveum for your system, you can refer to the SDK Integration Guide available in the Noveum documentation. This guide will help you trace your AI applications with minimal code changes. Th...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [] - }, - { - "span_id": "5500be90-0cfa-44dd-a6e3-e72ac9f0a79e", - "trace_id": "1ae0d6c4-d1fb-4239-b536-3333187d44aa", - "parent_span_id": "4254e6f9-10b8-42ba-939d-d7c124f80f9a", - "name": "agent.rag_evaluation_metrics", - "start_time": "2025-10-22 16:57:37.060000000", - "end_time": "2025-10-22 16:57:37.060000000", - "duration_ms": 0.076, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "rag_evaluation_metrics", - "agent.capabilities": [ - "retrieval_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: How do I configure Noveum for my system?", - "query_length": 40, - "query_type": "rag_evaluation_query", - "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/docs):\nWelcome to Noveum.ai Docs\nComprehensive AI tracing and observability for LLM applications, RAG systems, and AI agents\nWelcome to the Noveum.ai documentation! Here you'll find everything you need to integrate, configure, and optimize your AI applications using ou...", - "retrieval.context_length": 2237, - "retrieval.sources_count": 5, - "retrieval.context_quality": "high", - "retrieval.effectiveness": 1, - "retrieval.context_utilization": 2.237, - "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/docs):\nWelcome to Noveum.ai Docs\nComprehensive AI tracing and observability for LLM applications, RAG systems, and AI agents\nWelcome to the Noveum.ai documentation! Here you'll find everything you need to integrate, configure, and optimize your AI applications using our comprehensive tracing and observability platform.\nWe provide powerful SDKs for Python and TypeScript that enable you to trace LLM calls, RAG pipelines, and multi-agent workflows with minimal code changes. Our documentation is organized ...\n\nSource 2 (https://noveum.ai/docs):\nWelcome to Noveum.ai Docs\nComprehensive AI tracing and observability for LLM applications, RAG systems, and AI agents\nWelcome to the Noveum.ai documentation! Here you'll find everything you need to integrate, configure, and optimize your AI applications using our comprehensive tracing and observability platform.\nWe provide powerful SDKs for Python and TypeScript that enable you to trace LLM calls, RAG pipelines, and multi-agent workflows with minimal code changes. Our documentation is organized ...\n\nSource 3 (https://noveum.ai/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nSource 4 (https://noveum.ai/en/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nSource 5 (https://noveum.ai/en/docs):\n- \ud83c\udf0d Universal: Works across frameworks, clouds, and deployment models\n\ud83e\udd1d Community & Support\n- \ud83d\udcac Discord Community: Join our Discord\n- \ud83d\udce7 Email Support: [email protected]\n- \ud83d\udc1b Bug Reports: GitHub Issues\n- \ud83d\udcd6 Knowledge Base: Help Center\nReady to get started? Head to our SDK Integration Guide to begin tracing your AI applications in under 5 minutes!\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability ...\n\nUser Question: How do I configure Noveum for my system?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", - "prompt.prompt_length": 2773, - "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/docs):\nWelcome to Noveum.ai Docs\nComprehensive AI tracing and observability for LLM applications, RAG systems, and AI agents\nWelcome to the Noveum.ai documentation! Here...", - "prompt.instruction_following": "rag_optimized", - "response.answer_length": 553, - "response.answer_completeness": "complete", - "response.response_quality": "high", - "response.source_citation": 5, - "response.context_utilization": 0.24720607957085383, - "output_response": "RAG Answer: To configure Noveum for your system, you can refer to the SDK Integration Guide available in the Noveum documentation. This guide will help you trace your AI applications with minimal code changes. Th...", - "evaluation.retrieval_effectiveness": 1, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 5, - "evaluation.context_relevance": "high", - "evaluation.overall_quality": "high", - "evaluation.ready_for_production": true, - "rag.retrieval_strategy": "semantic_similarity", - "rag.vector_search_results": 5, - "rag.context_synthesis": "multi_source", - "rag.document_coverage": 1, - "rag.information_density": 0.24720607957085383 - }, - "events": [], - "links": [] - } - ], - "created_at": "2025-10-22 16:57:32.800000000", - "updated_at": "2025-10-22 16:57:32.800000000" - }, - { - "trace_id": "fb0d31fc-2465-4ecd-973c-4cd443d9befa", - "name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "status": "ok", - "status_message": "", - "start_time": "2025-10-22 16:57:24.828000000", - "end_time": "2025-10-22 16:57:31.793000000", - "duration_ms": 6965.302000000001, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "spans": [ - { - "span_id": "3068013a-6844-4f3c-8ec2-854a7353591e", - "trace_id": "fb0d31fc-2465-4ecd-973c-4cd443d9befa", - "parent_span_id": "a80d8fab-d5dc-4417-a5e5-9a2a76ed7419", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-22 16:57:24.828000000", - "end_time": "2025-10-22 16:57:24.828000000", - "duration_ms": 0.038, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What monitoring capabilities does Noveum provide?", - "query_length": 49, - "query_type": "routing_evaluation_query", - "classification.mode": "RAG", - "classification.rag_keyword_score": 2, - "classification.web_keyword_score": 0, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "noveum_specific", - "query.keyword_density": 0.3333333333333333, - "query.domain_affinity": "noveum", - "routing.decision": "Routed to RAG based on analysis", - "routing.rationale": "RAG score: 2, Web score: 0, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "Web Search", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.038461538461538464, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: RAG (Confidence: 1.00)" - }, - "events": [], - "links": [] - }, - { - "span_id": "189b89e3-fbbd-41de-8521-efb6a68f62f2", - "trace_id": "fb0d31fc-2465-4ecd-973c-4cd443d9befa", - "parent_span_id": "a80d8fab-d5dc-4417-a5e5-9a2a76ed7419", - "name": "agent.llm-rag", - "start_time": "2025-10-22 16:57:24.828000000", - "end_time": "2025-10-22 16:57:31.793000000", - "duration_ms": 6964.955, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "rag_agent", - "agent.operation": "llm-rag", - "agent.capabilities": [ - "document_retrieval", - "context_generation", - "response_generation" - ], - "agent.id": "noveum_rag_agent", - "input_query": "What monitoring capabilities does Noveum provide?", - "query_length": 49, - "query_type": "rag_query", - "output_response": "RAG Answer: Noveum provides comprehensive monitoring capabilities for AI applications, including the ability to monitor, trace, and optimize AI agents across various frameworks such as LangChain, CrewAI, AutoGen,...", - "rag.context_length": 2740, - "rag.sources_count": 5, - "rag.answer_length": 582, - "rag.mode": "retrieval_augmented_generation" - }, - "events": [], - "links": [] - }, - { - "span_id": "f5cffb7d-341b-4cde-b162-02208042463d", - "trace_id": "fb0d31fc-2465-4ecd-973c-4cd443d9befa", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-22 16:57:24.828000000", - "end_time": "2025-10-22 16:57:31.793000000", - "duration_ms": 6965.236, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What monitoring capabilities does Noveum provide?", - "query.length": 49, - "processing.mode": "RAG", - "processing.time_seconds": 6.965206146240234, - "processing.response_length": 582, - "processing.sources_count": 5, - "output_response": "Final Answer: Noveum provides comprehensive monitoring capabilities for AI applications, including the ability to monitor, trace, and optimize AI agents across various frameworks such as LangChain, CrewAI, AutoGen,...", - "final_answer_mode": "RAG", - "query_processed.input_query": "What monitoring capabilities does Noveum provide?", - "query_processed.output_response": "Successfully processed query using RAG, generated 582 character response", - "query_processed.mode": "RAG", - "query_processed.processing_time": 6.965206146240234, - "query_processed.response_length": 582 - }, - "events": [], - "links": [] - }, - { - "span_id": "a80d8fab-d5dc-4417-a5e5-9a2a76ed7419", - "trace_id": "fb0d31fc-2465-4ecd-973c-4cd443d9befa", - "parent_span_id": "f5cffb7d-341b-4cde-b162-02208042463d", - "name": "agent.query_routing", - "start_time": "2025-10-22 16:57:24.828000000", - "end_time": "2025-10-22 16:57:31.793000000", - "duration_ms": 6965.17, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What monitoring capabilities does Noveum provide?", - "query_length": 49, - "query_type": "routing_query", - "output_response": "Routed to RAG for query processing", - "router.classification": "RAG", - "router.confidence_score": 1, - "router.rag_keyword_score": 2, - "router.web_keyword_score": 0, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [] - }, - { - "span_id": "a2b632b8-fbcb-480f-9292-9f513c64df38", - "trace_id": "fb0d31fc-2465-4ecd-973c-4cd443d9befa", - "parent_span_id": "189b89e3-fbbd-41de-8521-efb6a68f62f2", - "name": "agent.llm_model_execution", - "start_time": "2025-10-22 16:57:27.824000000", - "end_time": "2025-10-22 16:57:31.155000000", - "duration_ms": 3330.989, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What monitoring capabilities does Noveum provide?", - "query_length": 49, - "query_type": "rag_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 3.3309478759765625, - "model.latency_ms": 3330.9478759765625, - "model.start_time": 1761152247.824474, - "model.end_time": 1761152251.155422, - "model.performance_tier": "medium", - "model.prompt_tokens": 821, - "model.completion_tokens": 145, - "model.total_tokens": 966, - "model.tokens_per_second": 290.00754018607677, - "model.estimated_cost": 0.028980000000000002, - "model.efficiency_score": 0.6024844720496895, - "model.response_length": 582, - "model.response_quality": "high", - "model.output_response": "Model Response: Noveum provides comprehensive monitoring capabilities for AI applications, including the ability to monitor, trace, and optimize AI agents across various frameworks such as LangChain, CrewAI, AutoGen,...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [] - }, - { - "span_id": "40ff54c4-07ca-4b2d-a9b4-7e3ae7adcd81", - "trace_id": "fb0d31fc-2465-4ecd-973c-4cd443d9befa", - "parent_span_id": "189b89e3-fbbd-41de-8521-efb6a68f62f2", - "name": "agent.rag_evaluation_metrics", - "start_time": "2025-10-22 16:57:31.155000000", - "end_time": "2025-10-22 16:57:31.155000000", - "duration_ms": 0.018000000000000002, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "rag_evaluation_metrics", - "agent.capabilities": [ - "retrieval_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What monitoring capabilities does Noveum provide?", - "query_length": 49, - "query_type": "rag_evaluation_query", - "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en):\nMonitor all your AI Agents\nimprove AI Agents today\nNoveum.ai helps you monitor, trace, and optimize your AI applications.\nNoveum.ai works with any AI framework \u2013 LangChain, CrewAI, AutoGen, custom implementations, or direct LLM calls. One dashboard shows everything.\n...", - "retrieval.context_length": 2740, - "retrieval.sources_count": 5, - "retrieval.context_quality": "high", - "retrieval.effectiveness": 1, - "retrieval.context_utilization": 2.74, - "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en):\nMonitor all your AI Agents\nimprove AI Agents today\nNoveum.ai helps you monitor, trace, and optimize your AI applications.\nNoveum.ai works with any AI framework \u2013 LangChain, CrewAI, AutoGen, custom implementations, or direct LLM calls. One dashboard shows everything.\nMonitor, Evaluate, Improve Your AI Agents\nThe control plane for AI agents.\nMonitor Everything, Miss Nothing\nOur lightweight SDKs capture every trace and span across your AI agent ecosystem\u2014from simple LLM calls to complex multi-agent...\n\nSource 2 (https://noveum.ai/):\nMonitor all your AI Agents\nimprove AI Agents today\nNoveum.ai helps you monitor, trace, and optimize your AI applications.\nNoveum.ai works with any AI framework \u2013 LangChain, CrewAI, AutoGen, custom implementations, or direct LLM calls. One dashboard shows everything.\nMonitor, Evaluate, Improve Your AI Agents\nThe control plane for AI agents.\nMonitor Everything, Miss Nothing\nOur lightweight SDKs capture every trace and span across your AI agent ecosystem\u2014from simple LLM calls to complex multi-agent...\n\nSource 3 (https://noveum.ai):\nMonitor all your AI Agents\nimprove AI Agents today\nNoveum.ai helps you monitor, trace, and optimize your AI applications.\nNoveum.ai works with any AI framework \u2013 LangChain, CrewAI, AutoGen, custom implementations, or direct LLM calls. One dashboard shows everything.\nMonitor, Evaluate, Improve Your AI Agents\nThe control plane for AI agents.\nMonitor Everything, Miss Nothing\nOur lightweight SDKs capture every trace and span across your AI agent ecosystem\u2014from simple LLM calls to complex multi-agent...\n\nSource 4 (https://noveum.ai/en/docs/getting-started/overview):\nNoveum.ai Overview\nComprehensive AI tracing and observability platform for LLM applications, RAG systems, and AI agents\nWelcome to Noveum.ai\u2014the comprehensive tracing and observability platform built specifically for AI applications. Whether you're building LLM-powered chatbots, RAG systems, multi-agent workflows, or any AI-driven application, Noveum provides the insights you need to understand, debug, and optimize your systems.\n\ud83c\udfaf Why AI Applications Need Specialized Observability\nTraditional mo...\n\nSource 5 (https://noveum.ai/docs/getting-started/overview):\nNoveum.ai Overview\nComprehensive AI tracing and observability platform for LLM applications, RAG systems, and AI agents\nWelcome to Noveum.ai\u2014the comprehensive tracing and observability platform built specifically for AI applications. Whether you're building LLM-powered chatbots, RAG systems, multi-agent workflows, or any AI-driven application, Noveum provides the insights you need to understand, debug, and optimize your systems.\n\ud83c\udfaf Why AI Applications Need Specialized Observability\nTraditional mo...\n\nUser Question: What monitoring capabilities does Noveum provide?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", - "prompt.prompt_length": 3285, - "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en):\nMonitor all your AI Agents\nimprove AI Agents today\nNoveum.ai helps you monitor, trace, and optimize your AI applications.\nNoveum.ai works with any AI framework \u2013 LangC...", - "prompt.instruction_following": "rag_optimized", - "response.answer_length": 582, - "response.answer_completeness": "complete", - "response.response_quality": "high", - "response.source_citation": 5, - "response.context_utilization": 0.2124087591240876, - "output_response": "RAG Answer: Noveum provides comprehensive monitoring capabilities for AI applications, including the ability to monitor, trace, and optimize AI agents across various frameworks such as LangChain, CrewAI, AutoGen,...", - "evaluation.retrieval_effectiveness": 1, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 5, - "evaluation.context_relevance": "high", - "evaluation.overall_quality": "high", - "evaluation.ready_for_production": true, - "rag.retrieval_strategy": "semantic_similarity", - "rag.vector_search_results": 5, - "rag.context_synthesis": "multi_source", - "rag.document_coverage": 1, - "rag.information_density": 0.2124087591240876 - }, - "events": [], - "links": [] - } - ], - "created_at": "2025-10-22 16:57:24.828000000", - "updated_at": "2025-10-22 16:57:24.828000000" - }, - { - "trace_id": "3bbd0b36-c30f-4126-b62a-dd0fae1c701d", - "name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "status": "ok", - "status_message": "", - "start_time": "2025-10-22 16:57:19.339000000", - "end_time": "2025-10-22 16:57:23.822000000", - "duration_ms": 4483.158, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "spans": [ - { - "span_id": "b0f0a2e0-c1f7-4ba4-8b75-7090e931a738", - "trace_id": "3bbd0b36-c30f-4126-b62a-dd0fae1c701d", - "parent_span_id": "4db80813-e62f-48ec-9091-62d569527caf", - "name": "agent.query_routing", - "start_time": "2025-10-22 16:57:19.339000000", - "end_time": "2025-10-22 16:57:23.822000000", - "duration_ms": 4482.909000000001, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "How does Noveum handle agent tracing?", - "query_length": 37, - "query_type": "routing_query", - "output_response": "Routed to RAG for query processing", - "router.classification": "RAG", - "router.confidence_score": 1, - "router.rag_keyword_score": 2, - "router.web_keyword_score": 0, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [] - }, - { - "span_id": "24da2221-b0ab-4161-98ad-e7b5775fcb27", - "trace_id": "3bbd0b36-c30f-4126-b62a-dd0fae1c701d", - "parent_span_id": "b0f0a2e0-c1f7-4ba4-8b75-7090e931a738", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-22 16:57:19.339000000", - "end_time": "2025-10-22 16:57:19.339000000", - "duration_ms": 0.026, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: How does Noveum handle agent tracing?", - "query_length": 37, - "query_type": "routing_evaluation_query", - "classification.mode": "RAG", - "classification.rag_keyword_score": 2, - "classification.web_keyword_score": 0, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "noveum_specific", - "query.keyword_density": 0.3333333333333333, - "query.domain_affinity": "noveum", - "routing.decision": "Routed to RAG based on analysis", - "routing.rationale": "RAG score: 2, Web score: 0, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "Web Search", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.038461538461538464, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: RAG (Confidence: 1.00)" - }, - "events": [], - "links": [] - }, - { - "span_id": "d2b495eb-7e07-48a3-aa6d-1835aab18c11", - "trace_id": "3bbd0b36-c30f-4126-b62a-dd0fae1c701d", - "parent_span_id": "b0f0a2e0-c1f7-4ba4-8b75-7090e931a738", - "name": "agent.llm-rag", - "start_time": "2025-10-22 16:57:19.339000000", - "end_time": "2025-10-22 16:57:23.822000000", - "duration_ms": 4482.564, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "rag_agent", - "agent.operation": "llm-rag", - "agent.capabilities": [ - "document_retrieval", - "context_generation", - "response_generation" - ], - "agent.id": "noveum_rag_agent", - "input_query": "How does Noveum handle agent tracing?", - "query_length": 37, - "query_type": "rag_query", - "output_response": "RAG Answer: Noveum.ai handles agent tracing through its specialized Multi-Agent Tracing capabilities, which are designed to observe complex workflows and inter-agent communications within multi-agent systems. The...", - "rag.context_length": 2736, - "rag.sources_count": 5, - "rag.answer_length": 663, - "rag.mode": "retrieval_augmented_generation" - }, - "events": [], - "links": [] - }, - { - "span_id": "4db80813-e62f-48ec-9091-62d569527caf", - "trace_id": "3bbd0b36-c30f-4126-b62a-dd0fae1c701d", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-22 16:57:19.339000000", - "end_time": "2025-10-22 16:57:23.822000000", - "duration_ms": 4483.031000000001, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "How does Noveum handle agent tracing?", - "query.length": 37, - "processing.mode": "RAG", - "processing.time_seconds": 4.482964038848877, - "processing.response_length": 663, - "processing.sources_count": 5, - "output_response": "Final Answer: Noveum.ai handles agent tracing through its specialized Multi-Agent Tracing capabilities, which are designed to observe complex workflows and inter-agent communications within multi-agent systems. The...", - "final_answer_mode": "RAG", - "query_processed.input_query": "How does Noveum handle agent tracing?", - "query_processed.output_response": "Successfully processed query using RAG, generated 663 character response", - "query_processed.mode": "RAG", - "query_processed.processing_time": 4.482964038848877, - "query_processed.response_length": 663 - }, - "events": [], - "links": [] - }, - { - "span_id": "b80cb5cd-24f2-484d-bbff-9febe2aa3ffe", - "trace_id": "3bbd0b36-c30f-4126-b62a-dd0fae1c701d", - "parent_span_id": "d2b495eb-7e07-48a3-aa6d-1835aab18c11", - "name": "agent.llm_model_execution", - "start_time": "2025-10-22 16:57:19.958000000", - "end_time": "2025-10-22 16:57:22.709000000", - "duration_ms": 2750.9480000000003, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: How does Noveum handle agent tracing?", - "query_length": 37, - "query_type": "rag_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 2.750890016555786, - "model.latency_ms": 2750.890016555786, - "model.start_time": 1761152239.958973, - "model.end_time": 1761152242.709863, - "model.performance_tier": "medium", - "model.prompt_tokens": 817, - "model.completion_tokens": 165, - "model.total_tokens": 982, - "model.tokens_per_second": 356.975376728983, - "model.estimated_cost": 0.02946, - "model.efficiency_score": 0.675152749490835, - "model.response_length": 663, - "model.response_quality": "high", - "model.output_response": "Model Response: Noveum.ai handles agent tracing through its specialized Multi-Agent Tracing capabilities, which are designed to observe complex workflows and inter-agent communications within multi-agent systems. The...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [] - }, - { - "span_id": "d85fcf7d-7eaa-4299-92fc-c969fab63a6e", - "trace_id": "3bbd0b36-c30f-4126-b62a-dd0fae1c701d", - "parent_span_id": "d2b495eb-7e07-48a3-aa6d-1835aab18c11", - "name": "agent.rag_evaluation_metrics", - "start_time": "2025-10-22 16:57:22.710000000", - "end_time": "2025-10-22 16:57:22.710000000", - "duration_ms": 0.036000000000000004, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "rag_evaluation_metrics", - "agent.capabilities": [ - "retrieval_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: How does Noveum handle agent tracing?", - "query_length": 37, - "query_type": "rag_evaluation_query", - "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, vector searches, data retrieval, agent reasoning, and more. Without proper observability, debugging ...", - "retrieval.context_length": 2736, - "retrieval.sources_count": 5, - "retrieval.context_quality": "high", - "retrieval.effectiveness": 1, - "retrieval.context_utilization": 2.736, - "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, vector searches, data retrieval, agent reasoning, and more. Without proper observability, debugging becomes a nightmare, optimization is guesswork, and understanding user interactions is nearly impossible.\nNoveum.ai solves this by providing comprehensive tracing and observability specifically designed for AI applications.\nKey benefits of using Noveum.ai:\n- Complete Visibility: Trace...\n\nSource 2 (https://noveum.ai/docs/advanced/multi-agent-tracing):\nMulti-Agent Tracing\nObserve complex agent workflows and inter-agent communications with comprehensive tracing\nMulti-agent systems represent some of the most complex AI applications, involving multiple agents that coordinate, communicate, and collaborate to achieve shared goals. Noveum.ai provides specialized tracing capabilities to help you understand and optimize these intricate workflows.\n\ud83c\udfaf Why Multi-Agent Tracing Matters\nMulti-agent systems introduce unique observability challenges:\n- Complex...\n\nSource 3 (https://noveum.ai/en/docs/advanced/multi-agent-tracing):\nMulti-Agent Tracing\nObserve complex agent workflows and inter-agent communications with comprehensive tracing\nMulti-agent systems represent some of the most complex AI applications, involving multiple agents that coordinate, communicate, and collaborate to achieve shared goals. Noveum.ai provides specialized tracing capabilities to help you understand and optimize these intricate workflows.\n\ud83c\udfaf Why Multi-Agent Tracing Matters\nMulti-agent systems introduce unique observability challenges:\n- Complex...\n\nSource 4 (https://noveum.ai/en/docs/advanced/performance-optimization):\n\ud83d\udd17 Next Steps\n- Multi-Agent Tracing - Observe agent workflows\n- RAG Pipeline Observability - Monitor retrieval and generation systems\n- Custom Instrumentation - Add domain-specific tracing\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nSource 5 (https://noveum.ai/docs/advanced/performance-optimization):\n\ud83d\udd17 Next Steps\n- Multi-Agent Tracing - Observe agent workflows\n- RAG Pipeline Observability - Monitor retrieval and generation systems\n- Custom Instrumentation - Add domain-specific tracing\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nUser Question: How does Noveum handle agent tracing?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", - "prompt.prompt_length": 3269, - "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, ...", - "prompt.instruction_following": "rag_optimized", - "response.answer_length": 663, - "response.answer_completeness": "complete", - "response.response_quality": "high", - "response.source_citation": 5, - "response.context_utilization": 0.24232456140350878, - "output_response": "RAG Answer: Noveum.ai handles agent tracing through its specialized Multi-Agent Tracing capabilities, which are designed to observe complex workflows and inter-agent communications within multi-agent systems. The...", - "evaluation.retrieval_effectiveness": 1, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 5, - "evaluation.context_relevance": "high", - "evaluation.overall_quality": "high", - "evaluation.ready_for_production": true, - "rag.retrieval_strategy": "semantic_similarity", - "rag.vector_search_results": 5, - "rag.context_synthesis": "multi_source", - "rag.document_coverage": 1, - "rag.information_density": 0.24232456140350878 - }, - "events": [], - "links": [] - } - ], - "created_at": "2025-10-22 16:57:19.339000000", - "updated_at": "2025-10-22 16:57:19.339000000" - }, - { - "trace_id": "fee3e6d3-8da1-4ef3-8214-a7d7323d3a49", - "name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "status": "ok", - "status_message": "", - "start_time": "2025-10-22 16:57:13.334000000", - "end_time": "2025-10-22 16:57:18.331000000", - "duration_ms": 4996.502, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "spans": [ - { - "span_id": "93d82f68-6c62-40de-a675-c4d7c40c1d01", - "trace_id": "fee3e6d3-8da1-4ef3-8214-a7d7323d3a49", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-22 16:57:13.334000000", - "end_time": "2025-10-22 16:57:18.330000000", - "duration_ms": 4996.436, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What APIs are available in Noveum platform?", - "query.length": 43, - "processing.mode": "RAG", - "processing.time_seconds": 4.996389865875244, - "processing.response_length": 270, - "processing.sources_count": 5, - "output_response": "Final Answer: The provided context does not specify the APIs available in the Noveum platform. For detailed information about the APIs, I recommend checking the official Noveum documentation or the SDK Integration ...", - "final_answer_mode": "RAG", - "query_processed.input_query": "What APIs are available in Noveum platform?", - "query_processed.output_response": "Successfully processed query using RAG, generated 270 character response", - "query_processed.mode": "RAG", - "query_processed.processing_time": 4.996389865875244, - "query_processed.response_length": 270 - }, - "events": [], - "links": [] - }, - { - "span_id": "1be727c6-14e8-4b2e-b8df-a1cab902454f", - "trace_id": "fee3e6d3-8da1-4ef3-8214-a7d7323d3a49", - "parent_span_id": "93d82f68-6c62-40de-a675-c4d7c40c1d01", - "name": "agent.query_routing", - "start_time": "2025-10-22 16:57:13.334000000", - "end_time": "2025-10-22 16:57:18.330000000", - "duration_ms": 4996.354, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What APIs are available in Noveum platform?", - "query_length": 43, - "query_type": "routing_query", - "output_response": "Routed to RAG for query processing", - "router.classification": "RAG", - "router.confidence_score": 1, - "router.rag_keyword_score": 3, - "router.web_keyword_score": 0, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [] - }, - { - "span_id": "58ab1d39-d349-4164-8ff9-26ece4b1e1aa", - "trace_id": "fee3e6d3-8da1-4ef3-8214-a7d7323d3a49", - "parent_span_id": "1be727c6-14e8-4b2e-b8df-a1cab902454f", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-22 16:57:13.334000000", - "end_time": "2025-10-22 16:57:13.334000000", - "duration_ms": 0.041999999999999996, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What APIs are available in Noveum platform?", - "query_length": 43, - "query_type": "routing_evaluation_query", - "classification.mode": "RAG", - "classification.rag_keyword_score": 3, - "classification.web_keyword_score": 0, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "noveum_specific", - "query.keyword_density": 0.42857142857142855, - "query.domain_affinity": "noveum", - "routing.decision": "Routed to RAG based on analysis", - "routing.rationale": "RAG score: 3, Web score: 0, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "Web Search", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.057692307692307696, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: RAG (Confidence: 1.00)" - }, - "events": [], - "links": [] - }, - { - "span_id": "f11ebf68-a416-484e-8e55-f77a11b9ea1c", - "trace_id": "fee3e6d3-8da1-4ef3-8214-a7d7323d3a49", - "parent_span_id": "1be727c6-14e8-4b2e-b8df-a1cab902454f", - "name": "agent.llm-rag", - "start_time": "2025-10-22 16:57:13.334000000", - "end_time": "2025-10-22 16:57:18.330000000", - "duration_ms": 4996.146000000001, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "rag_agent", - "agent.operation": "llm-rag", - "agent.capabilities": [ - "document_retrieval", - "context_generation", - "response_generation" - ], - "agent.id": "noveum_rag_agent", - "input_query": "What APIs are available in Noveum platform?", - "query_length": 43, - "query_type": "rag_query", - "output_response": "RAG Answer: The provided context does not specify the APIs available in the Noveum platform. For detailed information about the APIs, I recommend checking the official Noveum documentation or the SDK Integration ...", - "rag.context_length": 2284, - "rag.sources_count": 5, - "rag.answer_length": 270, - "rag.mode": "retrieval_augmented_generation" - }, - "events": [], - "links": [] - }, - { - "span_id": "11394488-8802-403d-a926-7fda3a510b61", - "trace_id": "fee3e6d3-8da1-4ef3-8214-a7d7323d3a49", - "parent_span_id": "f11ebf68-a416-484e-8e55-f77a11b9ea1c", - "name": "agent.llm_model_execution", - "start_time": "2025-10-22 16:57:13.862000000", - "end_time": "2025-10-22 16:57:16.049000000", - "duration_ms": 2187.29, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What APIs are available in Noveum platform?", - "query_length": 43, - "query_type": "rag_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 2.187239170074463, - "model.latency_ms": 2187.239170074463, - "model.start_time": 1761152233.862141, - "model.end_time": 1761152236.04938, - "model.performance_tier": "medium", - "model.prompt_tokens": 705, - "model.completion_tokens": 67, - "model.total_tokens": 772, - "model.tokens_per_second": 352.9563710098141, - "model.estimated_cost": 0.02316, - "model.efficiency_score": 0.34974093264248707, - "model.response_length": 270, - "model.response_quality": "high", - "model.output_response": "Model Response: The provided context does not specify the APIs available in the Noveum platform. For detailed information about the APIs, I recommend checking the official Noveum documentation or the SDK Integration ...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [] - }, - { - "span_id": "341cf8ab-cc33-4d57-952f-ba1065f0432e", - "trace_id": "fee3e6d3-8da1-4ef3-8214-a7d7323d3a49", - "parent_span_id": "f11ebf68-a416-484e-8e55-f77a11b9ea1c", - "name": "agent.rag_evaluation_metrics", - "start_time": "2025-10-22 16:57:16.049000000", - "end_time": "2025-10-22 16:57:16.049000000", - "duration_ms": 0.041, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "rag_evaluation_metrics", - "agent.capabilities": [ - "retrieval_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What APIs are available in Noveum platform?", - "query_length": 43, - "query_type": "rag_evaluation_query", - "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/docs):\n- \ud83c\udf0d Universal: Works across frameworks, clouds, and deployment models\n\ud83e\udd1d Community & Support\n- \ud83d\udcac Discord Community: Join our Discord\n- \ud83d\udce7 Email Support: [email protected]\n- \ud83d\udc1b Bug Reports: GitHub Issues\n- \ud83d\udcd6 Knowledge Base: Help Center\nReady to get started? Head to ...", - "retrieval.context_length": 2284, - "retrieval.sources_count": 5, - "retrieval.context_quality": "high", - "retrieval.effectiveness": 1, - "retrieval.context_utilization": 2.284, - "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/docs):\n- \ud83c\udf0d Universal: Works across frameworks, clouds, and deployment models\n\ud83e\udd1d Community & Support\n- \ud83d\udcac Discord Community: Join our Discord\n- \ud83d\udce7 Email Support: [email protected]\n- \ud83d\udc1b Bug Reports: GitHub Issues\n- \ud83d\udcd6 Knowledge Base: Help Center\nReady to get started? Head to our SDK Integration Guide to begin tracing your AI applications in under 5 minutes!\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability ...\n\nSource 2 (https://noveum.ai/docs):\n- \ud83c\udf0d Universal: Works across frameworks, clouds, and deployment models\n\ud83e\udd1d Community & Support\n- \ud83d\udcac Discord Community: Join our Discord\n- \ud83d\udce7 Email Support: [email protected]\n- \ud83d\udc1b Bug Reports: GitHub Issues\n- \ud83d\udcd6 Knowledge Base: Help Center\nReady to get started? Head to our SDK Integration Guide to begin tracing your AI applications in under 5 minutes!\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability ...\n\nSource 3 (https://noveum.ai/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nSource 4 (https://noveum.ai/en/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nSource 5 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, vector searches, data retrieval, agent reasoning, and more. Without proper observability, debugging becomes a nightmare, optimization is guesswork, and understanding user interactions is nearly impossible.\nNoveum.ai solves this by providing comprehensive tracing and observability specifically designed for AI applications.\nKey benefits of using Noveum.ai:\n- Complete Visibility: Trace...\n\nUser Question: What APIs are available in Noveum platform?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", - "prompt.prompt_length": 2823, - "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/docs):\n- \ud83c\udf0d Universal: Works across frameworks, clouds, and deployment models\n\ud83e\udd1d Community & Support\n- \ud83d\udcac Discord Community: Join our Discord\n- \ud83d\udce7 Email Support: [email prot...", - "prompt.instruction_following": "rag_optimized", - "response.answer_length": 270, - "response.answer_completeness": "complete", - "response.response_quality": "high", - "response.source_citation": 5, - "response.context_utilization": 0.11821366024518389, - "output_response": "RAG Answer: The provided context does not specify the APIs available in the Noveum platform. For detailed information about the APIs, I recommend checking the official Noveum documentation or the SDK Integration ...", - "evaluation.retrieval_effectiveness": 1, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 5, - "evaluation.context_relevance": "high", - "evaluation.overall_quality": "high", - "evaluation.ready_for_production": true, - "rag.retrieval_strategy": "semantic_similarity", - "rag.vector_search_results": 5, - "rag.context_synthesis": "multi_source", - "rag.document_coverage": 1, - "rag.information_density": 0.11821366024518389 - }, - "events": [], - "links": [] - } - ], - "created_at": "2025-10-22 16:57:13.334000000", - "updated_at": "2025-10-22 16:57:13.334000000" - }, - { - "trace_id": "93bd37c4-4289-43c9-b34e-19eaaef610c2", - "name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "status": "ok", - "status_message": "", - "start_time": "2025-10-22 16:57:06.605000000", - "end_time": "2025-10-22 16:57:12.327000000", - "duration_ms": 5721.707, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "spans": [ - { - "span_id": "24930c28-7f93-426c-8914-bd0e0085589f", - "trace_id": "93bd37c4-4289-43c9-b34e-19eaaef610c2", - "parent_span_id": "bfc91518-f818-4f35-9113-70b04eb55edc", - "name": "agent.query_routing", - "start_time": "2025-10-22 16:57:06.605000000", - "end_time": "2025-10-22 16:57:12.327000000", - "duration_ms": 5721.561000000001, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "How do I set up observability with Noveum?", - "query_length": 42, - "query_type": "routing_query", - "output_response": "Routed to RAG for query processing", - "router.classification": "RAG", - "router.confidence_score": 1, - "router.rag_keyword_score": 2, - "router.web_keyword_score": 0, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [] - }, - { - "span_id": "11fc18a8-0380-48ca-894e-cc26f648d290", - "trace_id": "93bd37c4-4289-43c9-b34e-19eaaef610c2", - "parent_span_id": "24930c28-7f93-426c-8914-bd0e0085589f", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-22 16:57:06.605000000", - "end_time": "2025-10-22 16:57:06.606000000", - "duration_ms": 0.047, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: How do I set up observability with Noveum?", - "query_length": 42, - "query_type": "routing_evaluation_query", - "classification.mode": "RAG", - "classification.rag_keyword_score": 2, - "classification.web_keyword_score": 0, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "noveum_specific", - "query.keyword_density": 0.25, - "query.domain_affinity": "noveum", - "routing.decision": "Routed to RAG based on analysis", - "routing.rationale": "RAG score: 2, Web score: 0, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "Web Search", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.038461538461538464, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: RAG (Confidence: 1.00)" - }, - "events": [], - "links": [] - }, - { - "span_id": "bfc91518-f818-4f35-9113-70b04eb55edc", - "trace_id": "93bd37c4-4289-43c9-b34e-19eaaef610c2", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-22 16:57:06.605000000", - "end_time": "2025-10-22 16:57:12.327000000", - "duration_ms": 5721.635, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "How do I set up observability with Noveum?", - "query.length": 42, - "processing.mode": "RAG", - "processing.time_seconds": 5.721601963043213, - "processing.response_length": 815, - "processing.sources_count": 5, - "output_response": "Final Answer: To set up observability with Noveum, follow these steps:\n\n1. **Sign Up**: Go to [noveum.ai](https://noveum.ai) and create an account.\n2. **Create a Project**: After signing up, create a project to obt...", - "final_answer_mode": "RAG", - "query_processed.input_query": "How do I set up observability with Noveum?", - "query_processed.output_response": "Successfully processed query using RAG, generated 815 character response", - "query_processed.mode": "RAG", - "query_processed.processing_time": 5.721601963043213, - "query_processed.response_length": 815 - }, - "events": [], - "links": [] - }, - { - "span_id": "2a82e4b1-c93d-498f-9dd5-e971f228d642", - "trace_id": "93bd37c4-4289-43c9-b34e-19eaaef610c2", - "parent_span_id": "24930c28-7f93-426c-8914-bd0e0085589f", - "name": "agent.llm-rag", - "start_time": "2025-10-22 16:57:06.606000000", - "end_time": "2025-10-22 16:57:12.327000000", - "duration_ms": 5721.352000000001, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "rag_agent", - "agent.operation": "llm-rag", - "agent.capabilities": [ - "document_retrieval", - "context_generation", - "response_generation" - ], - "agent.id": "noveum_rag_agent", - "input_query": "How do I set up observability with Noveum?", - "query_length": 42, - "query_type": "rag_query", - "output_response": "RAG Answer: To set up observability with Noveum, follow these steps:\n\n1. **Sign Up**: Go to [noveum.ai](https://noveum.ai) and create an account.\n2. **Create a Project**: After signing up, create a project to obt...", - "rag.context_length": 2870, - "rag.sources_count": 6, - "rag.answer_length": 815, - "rag.mode": "retrieval_augmented_generation" - }, - "events": [], - "links": [] - }, - { - "span_id": "520b4e59-da14-4db2-af02-0973cda9bd86", - "trace_id": "93bd37c4-4289-43c9-b34e-19eaaef610c2", - "parent_span_id": "2a82e4b1-c93d-498f-9dd5-e971f228d642", - "name": "agent.llm_model_execution", - "start_time": "2025-10-22 16:57:07.109000000", - "end_time": "2025-10-22 16:57:11.837000000", - "duration_ms": 4728.018999999999, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: How do I set up observability with Noveum?", - "query_length": 42, - "query_type": "rag_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 4.727963924407959, - "model.latency_ms": 4727.963924407959, - "model.start_time": 1761152227.109188, - "model.end_time": 1761152231.837152, - "model.performance_tier": "medium", - "model.prompt_tokens": 852, - "model.completion_tokens": 203, - "model.total_tokens": 1055, - "model.tokens_per_second": 223.1404504915101, - "model.estimated_cost": 0.03165, - "model.efficiency_score": 0.7725118483412322, - "model.response_length": 815, - "model.response_quality": "high", - "model.output_response": "Model Response: To set up observability with Noveum, follow these steps:\n\n1. **Sign Up**: Go to [noveum.ai](https://noveum.ai) and create an account.\n2. **Create a Project**: After signing up, create a project to obt...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [] - }, - { - "span_id": "dff080ab-238c-4cca-a910-cf530f910bda", - "trace_id": "93bd37c4-4289-43c9-b34e-19eaaef610c2", - "parent_span_id": "2a82e4b1-c93d-498f-9dd5-e971f228d642", - "name": "agent.rag_evaluation_metrics", - "start_time": "2025-10-22 16:57:11.837000000", - "end_time": "2025-10-22 16:57:11.838000000", - "duration_ms": 0.259, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "rag_evaluation_metrics", - "agent.capabilities": [ - "retrieval_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: How do I set up observability with Noveum?", - "query_length": 42, - "query_type": "rag_evaluation_query", - "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\n- Production Ready: Built for scale with intelligent sampling and batching\n- Privacy Focused: You control what data is captured and how it's stored\n- Open Source SDKs: Transparent, auditable, and extensible\nGetting ...", - "retrieval.context_length": 2870, - "retrieval.sources_count": 6, - "retrieval.context_quality": "high", - "retrieval.effectiveness": 1.2, - "retrieval.context_utilization": 2.87, - "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\n- Production Ready: Built for scale with intelligent sampling and batching\n- Privacy Focused: You control what data is captured and how it's stored\n- Open Source SDKs: Transparent, auditable, and extensible\nGetting Started\nReady to add comprehensive observability to your AI applications?\n- Sign up at noveum.ai\n- Create a project and get your API key\n- Install the SDK for your preferred language\n- Add tracing to your AI workflows\n- Explore insights in the Noveum.ai dashboard\nCheck out our integra...\n\nSource 2 (https://noveum.ai/en/docs/getting-started/tracing-concepts):\n- Explore Framework Integrations - Framework-specific guidance\n- Learn Advanced Patterns - Custom instrumentation techniques\n- Master the Dashboard - Analyze your traces effectively\nRemember: Good observability is not about collecting all possible data, but about collecting the right data that helps you understand, debug, and optimize your AI applications.\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to O...\n\nSource 3 (https://noveum.ai/docs/getting-started/tracing-concepts):\n- Explore Framework Integrations - Framework-specific guidance\n- Learn Advanced Patterns - Custom instrumentation techniques\n- Master the Dashboard - Analyze your traces effectively\nRemember: Good observability is not about collecting all possible data, but about collecting the right data that helps you understand, debug, and optimize your AI applications.\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to O...\n\nSource 4 (https://noveum.ai/en/docs/getting-started/overview):\nNoveum.ai Overview\nComprehensive AI tracing and observability platform for LLM applications, RAG systems, and AI agents\nWelcome to Noveum.ai\u2014the comprehensive tracing and observability platform built specifically for AI applications. Whether you're building LLM-powered chatbots, RAG systems, multi-agent workflows, or any AI-driven application, Noveum provides the insights you need to understand, debug, and optimize your systems.\n\ud83c\udfaf Why AI Applications Need Specialized Observability\nTraditional mo...\n\nSource 5 (https://noveum.ai/docs/getting-started/overview):\nNoveum.ai Overview\nComprehensive AI tracing and observability platform for LLM applications, RAG systems, and AI agents\nWelcome to Noveum.ai\u2014the comprehensive tracing and observability platform built specifically for AI applications. Whether you're building LLM-powered chatbots, RAG systems, multi-agent workflows, or any AI-driven application, Noveum provides the insights you need to understand, debug, and optimize your systems.\n\ud83c\udfaf Why AI Applications Need Specialized Observability\nTraditional mo...\n\nUser Question: How do I set up observability with Noveum?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", - "prompt.prompt_length": 3408, - "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\n- Production Ready: Built for scale with intelligent sampling and batching\n- Privacy Focused: You control what data...", - "prompt.instruction_following": "rag_optimized", - "response.answer_length": 815, - "response.answer_completeness": "complete", - "response.response_quality": "high", - "response.source_citation": 6, - "response.context_utilization": 0.28397212543554007, - "output_response": "RAG Answer: To set up observability with Noveum, follow these steps:\n\n1. **Sign Up**: Go to [noveum.ai](https://noveum.ai) and create an account.\n2. **Create a Project**: After signing up, create a project to obt...", - "evaluation.retrieval_effectiveness": 1.2, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 6, - "evaluation.context_relevance": "high", - "evaluation.overall_quality": "high", - "evaluation.ready_for_production": true, - "rag.retrieval_strategy": "semantic_similarity", - "rag.vector_search_results": 6, - "rag.context_synthesis": "multi_source", - "rag.document_coverage": 1.2, - "rag.information_density": 0.28397212543554007 - }, - "events": [], - "links": [] - } - ], - "created_at": "2025-10-22 16:57:06.605000000", - "updated_at": "2025-10-22 16:57:06.605000000" - }, - { - "trace_id": "df0c6344-1a2e-4c53-bb7d-4e43b3bce1b8", - "name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "status": "ok", - "status_message": "", - "start_time": "2025-10-22 16:56:59.213000000", - "end_time": "2025-10-22 16:57:05.598000000", - "duration_ms": 6384.386, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "spans": [ - { - "span_id": "dad7df65-be49-4245-8721-c91da4962b9a", - "trace_id": "df0c6344-1a2e-4c53-bb7d-4e43b3bce1b8", - "parent_span_id": "9024b0d9-2b75-428f-af8f-75a921a7c69c", - "name": "agent.query_routing", - "start_time": "2025-10-22 16:56:59.213000000", - "end_time": "2025-10-22 16:57:05.598000000", - "duration_ms": 6384.249, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What features does Noveum Trace offer?", - "query_length": 38, - "query_type": "routing_query", - "output_response": "Routed to RAG for query processing", - "router.classification": "RAG", - "router.confidence_score": 1, - "router.rag_keyword_score": 3, - "router.web_keyword_score": 0, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [] - }, - { - "span_id": "9024b0d9-2b75-428f-af8f-75a921a7c69c", - "trace_id": "df0c6344-1a2e-4c53-bb7d-4e43b3bce1b8", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-22 16:56:59.213000000", - "end_time": "2025-10-22 16:57:05.598000000", - "duration_ms": 6384.32, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What features does Noveum Trace offer?", - "query.length": 38, - "processing.mode": "RAG", - "processing.time_seconds": 6.384284973144531, - "processing.response_length": 1017, - "processing.sources_count": 5, - "output_response": "Final Answer: Noveum Trace offers several key features through its Python SDK (`noveum-trace`) and TypeScript SDK (`@noveum/trace`):\n\n1. **Decorator-based Tracing**: It provides a simple way to trace LLM calls, age...", - "final_answer_mode": "RAG", - "query_processed.input_query": "What features does Noveum Trace offer?", - "query_processed.output_response": "Successfully processed query using RAG, generated 1017 character response", - "query_processed.mode": "RAG", - "query_processed.processing_time": 6.384284973144531, - "query_processed.response_length": 1017 - }, - "events": [], - "links": [] - }, - { - "span_id": "d07fdbdc-2b8f-4de9-ac32-988389701771", - "trace_id": "df0c6344-1a2e-4c53-bb7d-4e43b3bce1b8", - "parent_span_id": "dad7df65-be49-4245-8721-c91da4962b9a", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-22 16:56:59.214000000", - "end_time": "2025-10-22 16:56:59.214000000", - "duration_ms": 0.060000000000000005, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What features does Noveum Trace offer?", - "query_length": 38, - "query_type": "routing_evaluation_query", - "classification.mode": "RAG", - "classification.rag_keyword_score": 3, - "classification.web_keyword_score": 0, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "noveum_specific", - "query.keyword_density": 0.5, - "query.domain_affinity": "noveum", - "routing.decision": "Routed to RAG based on analysis", - "routing.rationale": "RAG score: 3, Web score: 0, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "Web Search", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.057692307692307696, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: RAG (Confidence: 1.00)" - }, - "events": [], - "links": [] - }, - { - "span_id": "83f86157-5af4-4c99-aad1-647866cd804e", - "trace_id": "df0c6344-1a2e-4c53-bb7d-4e43b3bce1b8", - "parent_span_id": "dad7df65-be49-4245-8721-c91da4962b9a", - "name": "agent.llm-rag", - "start_time": "2025-10-22 16:56:59.214000000", - "end_time": "2025-10-22 16:57:05.598000000", - "duration_ms": 6384.048, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "rag_agent", - "agent.operation": "llm-rag", - "agent.capabilities": [ - "document_retrieval", - "context_generation", - "response_generation" - ], - "agent.id": "noveum_rag_agent", - "input_query": "What features does Noveum Trace offer?", - "query_length": 38, - "query_type": "rag_query", - "output_response": "RAG Answer: Noveum Trace offers several key features through its Python SDK (`noveum-trace`) and TypeScript SDK (`@noveum/trace`):\n\n1. **Decorator-based Tracing**: It provides a simple way to trace LLM calls, age...", - "rag.context_length": 2787, - "rag.sources_count": 5, - "rag.answer_length": 1017, - "rag.mode": "retrieval_augmented_generation" - }, - "events": [], - "links": [] - }, - { - "span_id": "3b4fa9a7-88bf-4ee1-aeb7-bfa9573f28f2", - "trace_id": "df0c6344-1a2e-4c53-bb7d-4e43b3bce1b8", - "parent_span_id": "83f86157-5af4-4c99-aad1-647866cd804e", - "name": "agent.llm_model_execution", - "start_time": "2025-10-22 16:56:59.591000000", - "end_time": "2025-10-22 16:57:04.869000000", - "duration_ms": 5278.925, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What features does Noveum Trace offer?", - "query_length": 38, - "query_type": "rag_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 5.278868913650513, - "model.latency_ms": 5278.868913650513, - "model.start_time": 1761152219.5910342, - "model.end_time": 1761152224.869903, - "model.performance_tier": "slow", - "model.prompt_tokens": 830, - "model.completion_tokens": 254, - "model.total_tokens": 1084, - "model.tokens_per_second": 205.3470199263535, - "model.estimated_cost": 0.03252, - "model.efficiency_score": 0.9381918819188192, - "model.response_length": 1017, - "model.response_quality": "high", - "model.output_response": "Model Response: Noveum Trace offers several key features through its Python SDK (`noveum-trace`) and TypeScript SDK (`@noveum/trace`):\n\n1. **Decorator-based Tracing**: It provides a simple way to trace LLM calls, age...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [] - }, - { - "span_id": "f9214dbe-adb9-44ff-8361-70c872cef66d", - "trace_id": "df0c6344-1a2e-4c53-bb7d-4e43b3bce1b8", - "parent_span_id": "83f86157-5af4-4c99-aad1-647866cd804e", - "name": "agent.rag_evaluation_metrics", - "start_time": "2025-10-22 16:57:04.870000000", - "end_time": "2025-10-22 16:57:04.870000000", - "duration_ms": 0.043000000000000003, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "rag_evaluation_metrics", - "agent.capabilities": [ - "retrieval_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What features does Noveum Trace offer?", - "query_length": 38, - "query_type": "rag_evaluation_query", - "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\n\ud83d\udcd8 TypeScript SDK (\n@noveum/trace\n)- Framework ...", - "retrieval.context_length": 2787, - "retrieval.sources_count": 5, - "retrieval.context_quality": "high", - "retrieval.effectiveness": 1, - "retrieval.context_utilization": 2.787, - "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\n\ud83d\udcd8 TypeScript SDK (\n@noveum/trace\n)- Framework integrations for Next.js, Express.js, and Hono\n- TypeScript-first design with full type safety\n- Universal compatibility (Node.js, Edge Runtime, browsers)\n-\n\ud83d\udcca Noveum Platform\n- Real-time dashboard for analyzing traces and performance\n- Ad...\n\nSource 2 (https://noveum.ai/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\n\ud83d\udcd8 TypeScript SDK (\n@noveum/trace\n)- Framework integrations for Next.js, Express.js, and Hono\n- TypeScript-first design with full type safety\n- Universal compatibility (Node.js, Edge Runtime, browsers)\n-\n\ud83d\udcca Noveum Platform\n- Real-time dashboard for analyzing traces and performance\n- Ad...\n\nSource 3 (https://noveum.ai/en/docs/getting-started/sdk-integration):\n- \ud83d\udc65 Collaborate with your team on insights\n\ud83d\udd12 Security & Privacy\n- \ud83d\udd10 Encryption: All data encrypted in transit and at rest\n- \ud83c\udf9b\ufe0f Configurable Capture: Control what data is collected\n- \ud83c\udfe0 Data Residency: Choose your data storage region\n- \u23f0 Retention Control: Set custom data retention policies\nNext Steps\n- Tracing Concepts - Learn about traces, spans, and observability best practices\n- Framework Integrations - Deep dive into specific framework setups\n- Multi-Agent Tracing - Observe complex agent work...\n\nSource 4 (https://noveum.ai/docs/getting-started/sdk-integration):\n- \ud83d\udc65 Collaborate with your team on insights\n\ud83d\udd12 Security & Privacy\n- \ud83d\udd10 Encryption: All data encrypted in transit and at rest\n- \ud83c\udf9b\ufe0f Configurable Capture: Control what data is collected\n- \ud83c\udfe0 Data Residency: Choose your data storage region\n- \u23f0 Retention Control: Set custom data retention policies\nNext Steps\n- Tracing Concepts - Learn about traces, spans, and observability best practices\n- Framework Integrations - Deep dive into specific framework setups\n- Multi-Agent Tracing - Observe complex agent work...\n\nSource 5 (https://noveum.ai/docs/platform/dashboard):\n- Bulk Export: Download multiple traces simultaneously\n- Comparative Analysis: Compare performance across multiple traces\nReal-Time Updates\n- Live Refresh: Automatic updates as new traces arrive\n- Connection Monitoring: Real-time status of your trace ingestion pipeline\n- Performance Indicators: Live metrics for system health monitoring\nReady to dive deeper? Explore Projects & Environments to organize your AI applications, or check out Team Collaboration to share insights with your team.\nGet Earl...\n\nUser Question: What features does Noveum Trace offer?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", - "prompt.prompt_length": 3321, - "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frame...", - "prompt.instruction_following": "rag_optimized", - "response.answer_length": 1017, - "response.answer_completeness": "complete", - "response.response_quality": "high", - "response.source_citation": 5, - "response.context_utilization": 0.36490850376749195, - "output_response": "RAG Answer: Noveum Trace offers several key features through its Python SDK (`noveum-trace`) and TypeScript SDK (`@noveum/trace`):\n\n1. **Decorator-based Tracing**: It provides a simple way to trace LLM calls, age...", - "evaluation.retrieval_effectiveness": 1, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 5, - "evaluation.context_relevance": "high", - "evaluation.overall_quality": "high", - "evaluation.ready_for_production": true, - "rag.retrieval_strategy": "semantic_similarity", - "rag.vector_search_results": 5, - "rag.context_synthesis": "multi_source", - "rag.document_coverage": 1, - "rag.information_density": 0.36490850376749195 - }, - "events": [], - "links": [] - } - ], - "created_at": "2025-10-22 16:56:59.213000000", - "updated_at": "2025-10-22 16:56:59.213000000" - }, - { - "trace_id": "29a862bf-d7aa-4cf9-963c-0a4244343bbe", - "name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "status": "ok", - "status_message": "", - "start_time": "2025-10-22 16:56:54.718000000", - "end_time": "2025-10-22 16:56:58.207000000", - "duration_ms": 3488.068, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "spans": [ - { - "span_id": "138a5089-5268-447a-afb5-4f14fb504281", - "trace_id": "29a862bf-d7aa-4cf9-963c-0a4244343bbe", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-22 16:56:54.719000000", - "end_time": "2025-10-22 16:56:58.207000000", - "duration_ms": 3487.959, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What are Noveum's pricing plans?", - "query.length": 32, - "processing.mode": "RAG", - "processing.time_seconds": 3.487928867340088, - "processing.response_length": 441, - "processing.sources_count": 5, - "output_response": "Final Answer: The provided context does not include specific information about Noveum's pricing plans. It mentions that all users will have access to the Observability suite for free, and early users will receive f...", - "final_answer_mode": "RAG", - "query_processed.input_query": "What are Noveum's pricing plans?", - "query_processed.output_response": "Successfully processed query using RAG, generated 441 character response", - "query_processed.mode": "RAG", - "query_processed.processing_time": 3.487928867340088, - "query_processed.response_length": 441 - }, - "events": [], - "links": [] - }, - { - "span_id": "7cda4dab-0fc5-4cae-8d8c-9445f661c67b", - "trace_id": "29a862bf-d7aa-4cf9-963c-0a4244343bbe", - "parent_span_id": "138a5089-5268-447a-afb5-4f14fb504281", - "name": "agent.query_routing", - "start_time": "2025-10-22 16:56:54.719000000", - "end_time": "2025-10-22 16:56:58.207000000", - "duration_ms": 3487.848, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What are Noveum's pricing plans?", - "query_length": 32, - "query_type": "routing_query", - "output_response": "Routed to RAG for query processing", - "router.classification": "RAG", - "router.confidence_score": 1, - "router.rag_keyword_score": 3, - "router.web_keyword_score": 0, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [] - }, - { - "span_id": "922c84a5-b14a-4b2f-8759-427d6e089812", - "trace_id": "29a862bf-d7aa-4cf9-963c-0a4244343bbe", - "parent_span_id": "7cda4dab-0fc5-4cae-8d8c-9445f661c67b", - "name": "agent.llm-rag", - "start_time": "2025-10-22 16:56:54.719000000", - "end_time": "2025-10-22 16:56:58.207000000", - "duration_ms": 3487.546, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "rag_agent", - "agent.operation": "llm-rag", - "agent.capabilities": [ - "document_retrieval", - "context_generation", - "response_generation" - ], - "agent.id": "noveum_rag_agent", - "input_query": "What are Noveum's pricing plans?", - "query_length": 32, - "query_type": "rag_query", - "output_response": "RAG Answer: The provided context does not include specific information about Noveum's pricing plans. It mentions that all users will have access to the Observability suite for free, and early users will receive f...", - "rag.context_length": 2105, - "rag.sources_count": 5, - "rag.answer_length": 441, - "rag.mode": "retrieval_augmented_generation" - }, - "events": [], - "links": [] - }, - { - "span_id": "c8d7bb28-e763-47c9-b3e9-77f6a91a22be", - "trace_id": "29a862bf-d7aa-4cf9-963c-0a4244343bbe", - "parent_span_id": "7cda4dab-0fc5-4cae-8d8c-9445f661c67b", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-22 16:56:54.719000000", - "end_time": "2025-10-22 16:56:54.719000000", - "duration_ms": 0.068, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What are Noveum's pricing plans?", - "query_length": 32, - "query_type": "routing_evaluation_query", - "classification.mode": "RAG", - "classification.rag_keyword_score": 3, - "classification.web_keyword_score": 0, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "noveum_specific", - "query.keyword_density": 0.6, - "query.domain_affinity": "noveum", - "routing.decision": "Routed to RAG based on analysis", - "routing.rationale": "RAG score: 3, Web score: 0, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "Web Search", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.057692307692307696, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: RAG (Confidence: 1.00)" - }, - "events": [], - "links": [] - }, - { - "span_id": "a378123a-8a79-4bc5-8e20-8ec54c2f9783", - "trace_id": "29a862bf-d7aa-4cf9-963c-0a4244343bbe", - "parent_span_id": "922c84a5-b14a-4b2f-8759-427d6e089812", - "name": "agent.llm_model_execution", - "start_time": "2025-10-22 16:56:55.154000000", - "end_time": "2025-10-22 16:56:57.166000000", - "duration_ms": 2012.174, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What are Noveum's pricing plans?", - "query_length": 32, - "query_type": "rag_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 2.012089252471924, - "model.latency_ms": 2012.0892524719238, - "model.start_time": 1761152215.154625, - "model.end_time": 1761152217.1667142, - "model.performance_tier": "medium", - "model.prompt_tokens": 658, - "model.completion_tokens": 110, - "model.total_tokens": 768, - "model.tokens_per_second": 381.6928096288395, - "model.estimated_cost": 0.02304, - "model.efficiency_score": 0.57421875, - "model.response_length": 441, - "model.response_quality": "high", - "model.output_response": "Model Response: The provided context does not include specific information about Noveum's pricing plans. It mentions that all users will have access to the Observability suite for free, and early users will receive f...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [] - }, - { - "span_id": "5813673b-a1dc-46c7-9bea-92ce6aca2b5e", - "trace_id": "29a862bf-d7aa-4cf9-963c-0a4244343bbe", - "parent_span_id": "922c84a5-b14a-4b2f-8759-427d6e089812", - "name": "agent.rag_evaluation_metrics", - "start_time": "2025-10-22 16:56:57.166000000", - "end_time": "2025-10-22 16:56:57.167000000", - "duration_ms": 0.059, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "rag_evaluation_metrics", - "agent.capabilities": [ - "retrieval_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What are Noveum's pricing plans?", - "query_length": 32, - "query_type": "rag_evaluation_query", - "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....", - "retrieval.context_length": 2105, - "retrieval.sources_count": 5, - "retrieval.context_quality": "high", - "retrieval.effectiveness": 1, - "retrieval.context_utilization": 2.105, - "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nSource 2 (https://noveum.ai/en/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nSource 3 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nThe AI landscape is complex and moving fast. With Noveum.ai, you'll have the insights you need to build, debug, and optimize AI applications with confidence.\nReady to see what's really happening inside your AI applications? Try Noveum.ai today and transform how you build and monitor AI systems.\nLet's build more reliable, observable AI\u2014together.\nGet Early Access to Noveum.ai Platform\nJoin the select group of AI teams optimizing their models with our data-driven platform. We're onboarding users in...\n\nSource 4 (https://noveum.ai/en/blog/from-logs-to-intelligent-choices-inside-noveum-ais-evaluation-process):\nGet Early Access to Noveum.ai Platform\nJoin the select group of AI teams optimizing their models with our data-driven platform. We're onboarding users in limited batches to ensure a premium experience....\n\nSource 5 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, vector searches, data retrieval, agent reasoning, and more. Without proper observability, debugging becomes a nightmare, optimization is guesswork, and understanding user interactions is nearly impossible.\nNoveum.ai solves this by providing comprehensive tracing and observability specifically designed for AI applications.\nKey benefits of using Noveum.ai:\n- Complete Visibility: Trace...\n\nUser Question: What are Noveum's pricing plans?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", - "prompt.prompt_length": 2633, - "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to ...", - "prompt.instruction_following": "rag_optimized", - "response.answer_length": 441, - "response.answer_completeness": "complete", - "response.response_quality": "high", - "response.source_citation": 5, - "response.context_utilization": 0.20950118764845604, - "output_response": "RAG Answer: The provided context does not include specific information about Noveum's pricing plans. It mentions that all users will have access to the Observability suite for free, and early users will receive f...", - "evaluation.retrieval_effectiveness": 1, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 5, - "evaluation.context_relevance": "high", - "evaluation.overall_quality": "high", - "evaluation.ready_for_production": true, - "rag.retrieval_strategy": "semantic_similarity", - "rag.vector_search_results": 5, - "rag.context_synthesis": "multi_source", - "rag.document_coverage": 1, - "rag.information_density": 0.20950118764845604 - }, - "events": [], - "links": [] - } - ], - "created_at": "2025-10-22 16:56:54.718000000", - "updated_at": "2025-10-22 16:56:54.718000000" - }, - { - "trace_id": "bc60c122-f30b-4114-9f78-ee40a024b525", - "name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "status": "ok", - "status_message": "", - "start_time": "2025-10-22 16:56:44.873000000", - "end_time": "2025-10-22 16:56:53.701000000", - "duration_ms": 8828.055, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "spans": [ - { - "span_id": "c5baf5c2-d18a-458a-b922-a19b96db53d9", - "trace_id": "bc60c122-f30b-4114-9f78-ee40a024b525", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-22 16:56:44.873000000", - "end_time": "2025-10-22 16:56:53.701000000", - "duration_ms": 8827.979, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "How do I integrate Noveum Trace in my application?", - "query.length": 50, - "processing.mode": "RAG", - "processing.time_seconds": 8.827935218811035, - "processing.response_length": 1145, - "processing.sources_count": 5, - "output_response": "Final Answer: To integrate Noveum Trace into your application, you can follow these steps based on the SDK Integration Guide:\n\n1. **Create Your Account & Get API Key**:\n - Sign up at [noveum.ai](https://noveum.ai...", - "final_answer_mode": "RAG", - "query_processed.input_query": "How do I integrate Noveum Trace in my application?", - "query_processed.output_response": "Successfully processed query using RAG, generated 1145 character response", - "query_processed.mode": "RAG", - "query_processed.processing_time": 8.827935218811035, - "query_processed.response_length": 1145 - }, - "events": [], - "links": [] - }, - { - "span_id": "8819dde5-9cbe-4cdb-b919-9c5140582359", - "trace_id": "bc60c122-f30b-4114-9f78-ee40a024b525", - "parent_span_id": "c5baf5c2-d18a-458a-b922-a19b96db53d9", - "name": "agent.query_routing", - "start_time": "2025-10-22 16:56:44.873000000", - "end_time": "2025-10-22 16:56:53.701000000", - "duration_ms": 8827.883, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "How do I integrate Noveum Trace in my application?", - "query_length": 50, - "query_type": "routing_query", - "output_response": "Routed to RAG for query processing", - "router.classification": "RAG", - "router.confidence_score": 1, - "router.rag_keyword_score": 2, - "router.web_keyword_score": 0, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [] - }, - { - "span_id": "932c4b0e-0f33-40c0-bb19-78b9f81e48a6", - "trace_id": "bc60c122-f30b-4114-9f78-ee40a024b525", - "parent_span_id": "8819dde5-9cbe-4cdb-b919-9c5140582359", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-22 16:56:44.873000000", - "end_time": "2025-10-22 16:56:44.873000000", - "duration_ms": 0.043000000000000003, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: How do I integrate Noveum Trace in my application?", - "query_length": 50, - "query_type": "routing_evaluation_query", - "classification.mode": "RAG", - "classification.rag_keyword_score": 2, - "classification.web_keyword_score": 0, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "noveum_specific", - "query.keyword_density": 0.2222222222222222, - "query.domain_affinity": "noveum", - "routing.decision": "Routed to RAG based on analysis", - "routing.rationale": "RAG score: 2, Web score: 0, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "Web Search", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.038461538461538464, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: RAG (Confidence: 1.00)" - }, - "events": [], - "links": [] - }, - { - "span_id": "2ece2bd6-e4c8-47bc-b3c8-26f82e9910f1", - "trace_id": "bc60c122-f30b-4114-9f78-ee40a024b525", - "parent_span_id": "8819dde5-9cbe-4cdb-b919-9c5140582359", - "name": "agent.llm-rag", - "start_time": "2025-10-22 16:56:44.873000000", - "end_time": "2025-10-22 16:56:53.701000000", - "duration_ms": 8827.666000000001, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "rag_agent", - "agent.operation": "llm-rag", - "agent.capabilities": [ - "document_retrieval", - "context_generation", - "response_generation" - ], - "agent.id": "noveum_rag_agent", - "input_query": "How do I integrate Noveum Trace in my application?", - "query_length": 50, - "query_type": "rag_query", - "output_response": "RAG Answer: To integrate Noveum Trace into your application, you can follow these steps based on the SDK Integration Guide:\n\n1. **Create Your Account & Get API Key**:\n - Sign up at [noveum.ai](https://noveum.ai...", - "rag.context_length": 2803, - "rag.sources_count": 5, - "rag.answer_length": 1145, - "rag.mode": "retrieval_augmented_generation" - }, - "events": [], - "links": [] - }, - { - "span_id": "e6315110-36b1-4503-beaf-12d972e25a7d", - "trace_id": "bc60c122-f30b-4114-9f78-ee40a024b525", - "parent_span_id": "2ece2bd6-e4c8-47bc-b3c8-26f82e9910f1", - "name": "agent.llm_model_execution", - "start_time": "2025-10-22 16:56:45.576000000", - "end_time": "2025-10-22 16:56:53.143000000", - "duration_ms": 7566.311, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: How do I integrate Noveum Trace in my application?", - "query_length": 50, - "query_type": "rag_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 7.566190004348755, - "model.latency_ms": 7566.190004348755, - "model.start_time": 1761152205.576988, - "model.end_time": 1761152213.143178, - "model.performance_tier": "slow", - "model.prompt_tokens": 837, - "model.completion_tokens": 286, - "model.total_tokens": 1123, - "model.tokens_per_second": 148.4234468543009, - "model.estimated_cost": 0.03369, - "model.efficiency_score": 1.0195903829029385, - "model.response_length": 1145, - "model.response_quality": "high", - "model.output_response": "Model Response: To integrate Noveum Trace into your application, you can follow these steps based on the SDK Integration Guide:\n\n1. **Create Your Account & Get API Key**:\n - Sign up at [noveum.ai](https://noveum.ai...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [] - }, - { - "span_id": "ae91d520-8442-48e5-aabf-fd58aed23694", - "trace_id": "bc60c122-f30b-4114-9f78-ee40a024b525", - "parent_span_id": "2ece2bd6-e4c8-47bc-b3c8-26f82e9910f1", - "name": "agent.rag_evaluation_metrics", - "start_time": "2025-10-22 16:56:53.143000000", - "end_time": "2025-10-22 16:56:53.143000000", - "duration_ms": 0.091, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "rag_evaluation_metrics", - "agent.capabilities": [ - "retrieval_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: How do I integrate Noveum Trace in my application?", - "query_length": 50, - "query_type": "rag_evaluation_query", - "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\n\ud83d\udcd8 TypeScript SDK (\n@noveum/trace\n)- Framework ...", - "retrieval.context_length": 2803, - "retrieval.sources_count": 5, - "retrieval.context_quality": "high", - "retrieval.effectiveness": 1, - "retrieval.context_utilization": 2.803, - "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\n\ud83d\udcd8 TypeScript SDK (\n@noveum/trace\n)- Framework integrations for Next.js, Express.js, and Hono\n- TypeScript-first design with full type safety\n- Universal compatibility (Node.js, Edge Runtime, browsers)\n-\n\ud83d\udcca Noveum Platform\n- Real-time dashboard for analyzing traces and performance\n- Ad...\n\nSource 2 (https://noveum.ai/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\n\ud83d\udcd8 TypeScript SDK (\n@noveum/trace\n)- Framework integrations for Next.js, Express.js, and Hono\n- TypeScript-first design with full type safety\n- Universal compatibility (Node.js, Edge Runtime, browsers)\n-\n\ud83d\udcca Noveum Platform\n- Real-time dashboard for analyzing traces and performance\n- Ad...\n\nSource 3 (https://noveum.ai/docs/getting-started/sdk-integration):\nSDK Integration Guide\nIntegrate Noveum.ai tracing into your AI applications with Python or TypeScript SDKs\nThe Noveum.ai SDKs provide comprehensive tracing and observability for your AI applications with minimal code changes. Whether you're building LLM applications, RAG systems, or multi-agent workflows, our SDKs automatically capture essential metrics and traces.\n\ud83d\ude80 Quick Start\n1. Create Your Account & Get API Key\n- Sign up at noveum.ai\n- Create a project in your dashboard\n- Generate an API key...\n\nSource 4 (https://noveum.ai/en/docs/getting-started/sdk-integration):\nSDK Integration Guide\nIntegrate Noveum.ai tracing into your AI applications with Python or TypeScript SDKs\nThe Noveum.ai SDKs provide comprehensive tracing and observability for your AI applications with minimal code changes. Whether you're building LLM applications, RAG systems, or multi-agent workflows, our SDKs automatically capture essential metrics and traces.\n\ud83d\ude80 Quick Start\n1. Create Your Account & Get API Key\n- Sign up at noveum.ai\n- Create a project in your dashboard\n- Generate an API key...\n\nSource 5 (https://noveum.ai/en/docs/getting-started/sdk-integration):\n\ud83d\udd27 Framework Integrations\nNext.js Integration\nExpress.js Integration\nFastAPI Integration (Python)\n\ud83d\udcca Advanced Features\nCustom Attributes & Events\nSampling Configuration\n\ud83d\udd17 What's Captured Automatically\n- \ud83d\udcca Performance Metrics: Latency, throughput, error rates\n- \ud83d\udcb0 Cost Tracking: Token usage, API costs across providers\n- \ud83d\udd0d Request/Response: Configurable capture of inputs/outputs\n- \ud83c\udff7\ufe0f Metadata: Model names, parameters, user context\n- \ud83c\udf0a Context Flow: Trace relationships across services\n- \ud83d\udc1b Error Detail...\n\nUser Question: How do I integrate Noveum Trace in my application?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", - "prompt.prompt_length": 3349, - "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frame...", - "prompt.instruction_following": "rag_optimized", - "response.answer_length": 1145, - "response.answer_completeness": "complete", - "response.response_quality": "high", - "response.source_citation": 5, - "response.context_utilization": 0.4084909026043525, - "output_response": "RAG Answer: To integrate Noveum Trace into your application, you can follow these steps based on the SDK Integration Guide:\n\n1. **Create Your Account & Get API Key**:\n - Sign up at [noveum.ai](https://noveum.ai...", - "evaluation.retrieval_effectiveness": 1, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 5, - "evaluation.context_relevance": "high", - "evaluation.overall_quality": "high", - "evaluation.ready_for_production": true, - "rag.retrieval_strategy": "semantic_similarity", - "rag.vector_search_results": 5, - "rag.context_synthesis": "multi_source", - "rag.document_coverage": 1, - "rag.information_density": 0.4084909026043525 - }, - "events": [], - "links": [] - } - ], - "created_at": "2025-10-22 16:56:44.873000000", - "updated_at": "2025-10-22 16:56:44.873000000" - }, - { - "trace_id": "3ad192e5-bbd4-4236-8144-019d53512769", - "name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "status": "ok", - "status_message": "", - "start_time": "2025-10-22 16:56:35.188000000", - "end_time": "2025-10-22 16:56:43.863000000", - "duration_ms": 8675.356000000002, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "spans": [ - { - "span_id": "9a9c611d-f552-4b06-a764-4463edb70e2c", - "trace_id": "3ad192e5-bbd4-4236-8144-019d53512769", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-22 16:56:35.188000000", - "end_time": "2025-10-22 16:56:43.863000000", - "duration_ms": 8675.25, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What is Noveum and what does it do?", - "query.length": 35, - "processing.mode": "RAG", - "processing.time_seconds": 8.67501425743103, - "processing.response_length": 527, - "processing.sources_count": 5, - "output_response": "Final Answer: Noveum.ai is a comprehensive tracing and observability platform specifically designed for AI applications, including LLM-powered chatbots, RAG systems, and multi-agent workflows. It provides the neces...", - "final_answer_mode": "RAG", - "query_processed.input_query": "What is Noveum and what does it do?", - "query_processed.output_response": "Successfully processed query using RAG, generated 527 character response", - "query_processed.mode": "RAG", - "query_processed.processing_time": 8.67501425743103, - "query_processed.response_length": 527 - }, - "events": [], - "links": [] - }, - { - "span_id": "99eadeae-0733-4bf3-982e-e9f3014f5913", - "trace_id": "3ad192e5-bbd4-4236-8144-019d53512769", - "parent_span_id": "7e2c959c-157a-43d5-9870-32994cddd388", - "name": "agent.llm-rag", - "start_time": "2025-10-22 16:56:35.188000000", - "end_time": "2025-10-22 16:56:43.863000000", - "duration_ms": 8674.699999999999, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "rag_agent", - "agent.operation": "llm-rag", - "agent.capabilities": [ - "document_retrieval", - "context_generation", - "response_generation" - ], - "agent.id": "noveum_rag_agent", - "input_query": "What is Noveum and what does it do?", - "query_length": 35, - "query_type": "rag_query", - "output_response": "RAG Answer: Noveum.ai is a comprehensive tracing and observability platform specifically designed for AI applications, including LLM-powered chatbots, RAG systems, and multi-agent workflows. It provides the neces...", - "rag.context_length": 2334, - "rag.sources_count": 5, - "rag.answer_length": 527, - "rag.mode": "retrieval_augmented_generation" - }, - "events": [], - "links": [] - }, - { - "span_id": "7e2c959c-157a-43d5-9870-32994cddd388", - "trace_id": "3ad192e5-bbd4-4236-8144-019d53512769", - "parent_span_id": "9a9c611d-f552-4b06-a764-4463edb70e2c", - "name": "agent.query_routing", - "start_time": "2025-10-22 16:56:35.188000000", - "end_time": "2025-10-22 16:56:43.863000000", - "duration_ms": 8674.961, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What is Noveum and what does it do?", - "query_length": 35, - "query_type": "routing_query", - "output_response": "Routed to RAG for query processing", - "router.classification": "RAG", - "router.confidence_score": 1, - "router.rag_keyword_score": 1, - "router.web_keyword_score": 0, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [] - }, - { - "span_id": "6f81284c-a9aa-4d62-a402-1b474f2eba51", - "trace_id": "3ad192e5-bbd4-4236-8144-019d53512769", - "parent_span_id": "7e2c959c-157a-43d5-9870-32994cddd388", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-22 16:56:35.188000000", - "end_time": "2025-10-22 16:56:35.188000000", - "duration_ms": 0.054, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What is Noveum and what does it do?", - "query_length": 35, - "query_type": "routing_evaluation_query", - "classification.mode": "RAG", - "classification.rag_keyword_score": 1, - "classification.web_keyword_score": 0, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "noveum_specific", - "query.keyword_density": 0.125, - "query.domain_affinity": "noveum", - "routing.decision": "Routed to RAG based on analysis", - "routing.rationale": "RAG score: 1, Web score: 0, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "Web Search", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.019230769230769232, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: RAG (Confidence: 1.00)" - }, - "events": [], - "links": [] - }, - { - "span_id": "9f42d18a-0235-4ae2-838c-19a88ddc95e1", - "trace_id": "3ad192e5-bbd4-4236-8144-019d53512769", - "parent_span_id": "99eadeae-0733-4bf3-982e-e9f3014f5913", - "name": "agent.llm_model_execution", - "start_time": "2025-10-22 16:56:36.031000000", - "end_time": "2025-10-22 16:56:42.012000000", - "duration_ms": 5980.947999999999, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What is Noveum and what does it do?", - "query_length": 35, - "query_type": "rag_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 5.980915784835815, - "model.latency_ms": 5980.915784835815, - "model.start_time": 1761152196.031816, - "model.end_time": 1761152202.0127318, - "model.performance_tier": "slow", - "model.prompt_tokens": 716, - "model.completion_tokens": 131, - "model.total_tokens": 847, - "model.tokens_per_second": 141.61710856178715, - "model.estimated_cost": 0.025410000000000002, - "model.efficiency_score": 0.6221959858323495, - "model.response_length": 527, - "model.response_quality": "high", - "model.output_response": "Model Response: Noveum.ai is a comprehensive tracing and observability platform specifically designed for AI applications, including LLM-powered chatbots, RAG systems, and multi-agent workflows. It provides the neces...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [] - }, - { - "span_id": "7bcbe4b7-585a-4c9d-add8-16e8aa62254f", - "trace_id": "3ad192e5-bbd4-4236-8144-019d53512769", - "parent_span_id": "99eadeae-0733-4bf3-982e-e9f3014f5913", - "name": "agent.rag_evaluation_metrics", - "start_time": "2025-10-22 16:56:42.012000000", - "end_time": "2025-10-22 16:56:42.012000000", - "duration_ms": 0.018000000000000002, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "rag_evaluation_metrics", - "agent.capabilities": [ - "retrieval_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What is Noveum and what does it do?", - "query_length": 35, - "query_type": "rag_evaluation_query", - "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, vector searches, data retrieval, agent reasoning, and more. Without proper observability, debugging ...", - "retrieval.context_length": 2334, - "retrieval.sources_count": 5, - "retrieval.context_quality": "high", - "retrieval.effectiveness": 1, - "retrieval.context_utilization": 2.334, - "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, vector searches, data retrieval, agent reasoning, and more. Without proper observability, debugging becomes a nightmare, optimization is guesswork, and understanding user interactions is nearly impossible.\nNoveum.ai solves this by providing comprehensive tracing and observability specifically designed for AI applications.\nKey benefits of using Noveum.ai:\n- Complete Visibility: Trace...\n\nSource 2 (https://noveum.ai/en/docs/getting-started/overview):\nNoveum.ai Overview\nComprehensive AI tracing and observability platform for LLM applications, RAG systems, and AI agents\nWelcome to Noveum.ai\u2014the comprehensive tracing and observability platform built specifically for AI applications. Whether you're building LLM-powered chatbots, RAG systems, multi-agent workflows, or any AI-driven application, Noveum provides the insights you need to understand, debug, and optimize your systems.\n\ud83c\udfaf Why AI Applications Need Specialized Observability\nTraditional mo...\n\nSource 3 (https://noveum.ai/docs/getting-started/overview):\nNoveum.ai Overview\nComprehensive AI tracing and observability platform for LLM applications, RAG systems, and AI agents\nWelcome to Noveum.ai\u2014the comprehensive tracing and observability platform built specifically for AI applications. Whether you're building LLM-powered chatbots, RAG systems, multi-agent workflows, or any AI-driven application, Noveum provides the insights you need to understand, debug, and optimize your systems.\n\ud83c\udfaf Why AI Applications Need Specialized Observability\nTraditional mo...\n\nSource 4 (https://noveum.ai/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nSource 5 (https://noveum.ai/en/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nUser Question: What is Noveum and what does it do?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", - "prompt.prompt_length": 2865, - "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, ...", - "prompt.instruction_following": "rag_optimized", - "response.answer_length": 527, - "response.answer_completeness": "complete", - "response.response_quality": "high", - "response.source_citation": 5, - "response.context_utilization": 0.22579263067694944, - "output_response": "RAG Answer: Noveum.ai is a comprehensive tracing and observability platform specifically designed for AI applications, including LLM-powered chatbots, RAG systems, and multi-agent workflows. It provides the neces...", - "evaluation.retrieval_effectiveness": 1, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 5, - "evaluation.context_relevance": "high", - "evaluation.overall_quality": "high", - "evaluation.ready_for_production": true, - "rag.retrieval_strategy": "semantic_similarity", - "rag.vector_search_results": 5, - "rag.context_synthesis": "multi_source", - "rag.document_coverage": 1, - "rag.information_density": 0.22579263067694944 - }, - "events": [], - "links": [] - } - ], - "created_at": "2025-10-22 16:56:35.188000000", - "updated_at": "2025-10-22 16:56:35.188000000" - }, - { - "trace_id": "9af2b4b0-0c2f-4d2e-ae99-30df0d2f0fa6", - "name": "auto_trace_system_initialization", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "status": "ok", - "status_message": "", - "start_time": "2025-10-22 16:56:34.565000000", - "end_time": "2025-10-22 16:56:35.186000000", - "duration_ms": 621.177, - "span_count": 1, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "spans": [ - { - "span_id": "d39297f4-b4fe-4f9c-b45b-854411ba770a", - "trace_id": "9af2b4b0-0c2f-4d2e-ae99-30df0d2f0fa6", - "parent_span_id": "", - "name": "system_initialization", - "start_time": "2025-10-22 16:56:34.565000000", - "end_time": "2025-10-22 16:56:35.186000000", - "duration_ms": 621.02, - "status": "ok", - "status_message": "", - "attributes": { - "system.force_scrape": false, - "system.config": { - "noveum_base_url": "https://noveum.ai", - "max_pages_to_scrape": 50, - "chunk_size": 1000, - "chunk_overlap": 200, - "max_search_results": 5, - "rag_threshold": 0.7, - "noveum_docs_file": "noveum_docs.json", - "vector_store_path": "noveum_vectorstore" - }, - "input_query": "Initialize system with force_scrape=False", - "output_response": "System initialization: RAG system loaded, vector store ready, agent operational", - "system.initialized": true, - "system.documents_loaded": 38, - "system.vectorstore_ready": true - }, - "events": [], - "links": [] - } - ], - "created_at": "2025-10-22 16:56:34.565000000", - "updated_at": "2025-10-22 16:56:34.565000000" - }, - { - "trace_id": "bae13042-cf2e-4813-a673-7dc2a611725e", - "name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "status": "ok", - "status_message": "", - "start_time": "2025-10-16 08:50:56.153000000", - "end_time": "2025-10-16 08:51:01.908000000", - "duration_ms": 5754.578, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "spans": [ - { - "span_id": "8c144ad8-a439-4986-b741-3bc4b7fc6110", - "trace_id": "bae13042-cf2e-4813-a673-7dc2a611725e", - "parent_span_id": "d37fe1a1-a126-4e4e-a914-93dd402fed3f", - "name": "agent.query_routing", - "start_time": "2025-10-16 08:50:56.153000000", - "end_time": "2025-10-16 08:51:01.908000000", - "duration_ms": 5754.43, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What are the latest breakthroughs in artificial intelligence?", - "query_length": 61, - "query_type": "routing_query", - "output_response": "Routed to Web Search for query processing", - "router.classification": "Web Search", - "router.confidence_score": 1, - "router.rag_keyword_score": 0, - "router.web_keyword_score": 1, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [] - }, - { - "span_id": "b294f9b7-7762-47a9-b49d-00801a984af2", - "trace_id": "bae13042-cf2e-4813-a673-7dc2a611725e", - "parent_span_id": "8c144ad8-a439-4986-b741-3bc4b7fc6110", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-16 08:50:56.153000000", - "end_time": "2025-10-16 08:50:56.153000000", - "duration_ms": 0.024, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What are the latest breakthroughs in artificial intelligence?", - "query_length": 61, - "query_type": "routing_evaluation_query", - "classification.mode": "Web Search", - "classification.rag_keyword_score": 0, - "classification.web_keyword_score": 1, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "complex", - "query.intent": "general_knowledge", - "query.keyword_density": 0.125, - "query.domain_affinity": "general", - "routing.decision": "Routed to Web Search based on analysis", - "routing.rationale": "RAG score: 0, Web score: 1, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "RAG", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.019230769230769232, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: Web Search (Confidence: 1.00)" - }, - "events": [], - "links": [] - }, - { - "span_id": "ab6a281a-4989-40c8-9011-d0ec7dde7ea9", - "trace_id": "bae13042-cf2e-4813-a673-7dc2a611725e", - "parent_span_id": "8c144ad8-a439-4986-b741-3bc4b7fc6110", - "name": "agent.web_search_generation", - "start_time": "2025-10-16 08:50:56.153000000", - "end_time": "2025-10-16 08:51:01.908000000", - "duration_ms": 5754.317999999999, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "web_search_agent", - "agent.operation": "web_search_generation", - "agent.capabilities": [ - "web_search", - "content_synthesis", - "response_generation" - ], - "agent.id": "noveum_web_search_agent", - "input_query": "What are the latest breakthroughs in artificial intelligence?", - "query_length": 61, - "query_type": "web_search_query", - "output_response": "Web Search Answer: The web search did not yield specific results regarding the latest breakthroughs in artificial intelligence. However, I can provide a general overview based on knowledge up to October 2023.\n\nRecent br...", - "web_search.results_count": 1, - "web_search.context_length": 163, - "web_search.response_length": 1489, - "web_search.mode": "external_web_search" - }, - "events": [], - "links": [] - }, - { - "span_id": "d37fe1a1-a126-4e4e-a914-93dd402fed3f", - "trace_id": "bae13042-cf2e-4813-a673-7dc2a611725e", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-16 08:50:56.153000000", - "end_time": "2025-10-16 08:51:01.908000000", - "duration_ms": 5754.527, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What are the latest breakthroughs in artificial intelligence?", - "query.length": 61, - "processing.mode": "Web Search", - "processing.time_seconds": 5.754457473754883, - "processing.response_length": 1489, - "processing.sources_count": 1, - "output_response": "Final Answer: The web search did not yield specific results regarding the latest breakthroughs in artificial intelligence. However, I can provide a general overview based on knowledge up to October 2023.\n\nRecent br...", - "final_answer_mode": "Web Search", - "query_processed.input_query": "What are the latest breakthroughs in artificial intelligence?", - "query_processed.output_response": "Successfully processed query using Web Search, generated 1489 character response", - "query_processed.mode": "Web Search", - "query_processed.processing_time": 5.754457473754883, - "query_processed.response_length": 1489 - }, - "events": [], - "links": [] - }, - { - "span_id": "f61ef76d-4887-42da-9fd6-e2cdb25a9e09", - "trace_id": "bae13042-cf2e-4813-a673-7dc2a611725e", - "parent_span_id": "ab6a281a-4989-40c8-9011-d0ec7dde7ea9", - "name": "agent.llm_model_execution", - "start_time": "2025-10-16 08:50:56.502000000", - "end_time": "2025-10-16 08:51:01.908000000", - "duration_ms": 5405.44, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What are the latest breakthroughs in artificial intelligence?", - "query_length": 61, - "query_type": "web_search_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 5.405392408370972, - "model.latency_ms": 5405.392408370972, - "model.start_time": 1760604656.5026655, - "model.end_time": 1760604661.908058, - "model.performance_tier": "slow", - "model.prompt_tokens": 167, - "model.completion_tokens": 372, - "model.total_tokens": 539, - "model.tokens_per_second": 99.71523976044487, - "model.estimated_cost": 0.01617, - "model.efficiency_score": 2.7625231910946195, - "model.response_length": 1489, - "model.response_quality": "high", - "model.output_response": "Model Response: The web search did not yield specific results regarding the latest breakthroughs in artificial intelligence. However, I can provide a general overview based on knowledge up to October 2023.\n\nRecent br...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [] - }, - { - "span_id": "550bf5ae-c816-4ef0-a6c1-1553b1c258b8", - "trace_id": "bae13042-cf2e-4813-a673-7dc2a611725e", - "parent_span_id": "ab6a281a-4989-40c8-9011-d0ec7dde7ea9", - "name": "agent.web_search_evaluation_metrics", - "start_time": "2025-10-16 08:51:01.908000000", - "end_time": "2025-10-16 08:51:01.908000000", - "duration_ms": 0.018000000000000002, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "web_search_evaluation_metrics", - "agent.capabilities": [ - "web_search_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What are the latest breakthroughs in artificial intelligence?", - "query_length": 61, - "query_type": "web_search_evaluation_query", - "web_search.results_count": 1, - "web_search.context_length": 163, - "web_search.context_synthesized": "Context from 1 web sources", - "web_search.search_effectiveness": 0.2, - "web_search.context_quality": "low", - "web_search.source_diversity": 1, - "web_search.information_synthesis": "low", - "web_search.external_knowledge_utilization": 0.163, - "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What+are+the+latest+breakthroughs+in+artificial+intelligence?\n\nUser Question: What are the latest breakthroughs in artificial intelligence?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", - "prompt.prompt_length": 669, - "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What+are+the+latest+breakthroughs+in+artificial+intelligence?", - "prompt.instruction_following": "web_search_optimized", - "response.answer_length": 1489, - "response.answer_completeness": "complete", - "response.response_quality": "medium", - "response.source_citation": 1, - "response.context_utilization": 9.134969325153374, - "output_response": "Web Search Answer: The web search did not yield specific results regarding the latest breakthroughs in artificial intelligence. However, I can provide a general overview based on knowledge up to October 2023.\n\nRecent br...", - "evaluation.search_effectiveness": 0.2, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 1, - "evaluation.context_relevance": "low", - "evaluation.overall_quality": "low", - "evaluation.ready_for_production": true, - "web_search.search_strategy": "duckduckgo_api", - "web_search.real_time_data": true, - "web_search.external_sources": 1, - "web_search.information_freshness": "current", - "web_search.knowledge_synthesis": "single_source" - }, - "events": [], - "links": [] - } - ], - "created_at": "2025-10-16 08:50:56.153000000", - "updated_at": "2025-10-16 08:50:56.153000000" - }, - { - "trace_id": "79f9d235-d46e-4740-be35-ef0006071f6c", - "name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "status": "ok", - "status_message": "", - "start_time": "2025-10-16 08:50:51.539000000", - "end_time": "2025-10-16 08:50:55.150000000", - "duration_ms": 3611.084, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "spans": [ - { - "span_id": "04cb7d29-9ea8-4ae0-9834-030cae944eca", - "trace_id": "79f9d235-d46e-4740-be35-ef0006071f6c", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-16 08:50:51.539000000", - "end_time": "2025-10-16 08:50:55.150000000", - "duration_ms": 3611.008, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What's happening in the software development world today?", - "query.length": 57, - "processing.mode": "Web Search", - "processing.time_seconds": 3.6109254360198975, - "processing.response_length": 526, - "processing.sources_count": 1, - "output_response": "Final Answer: The provided web search results do not contain specific information about current events or trends in the software development world. They primarily focus on unrelated topics, such as the meaning of \"...", - "final_answer_mode": "Web Search", - "query_processed.input_query": "What's happening in the software development world today?", - "query_processed.output_response": "Successfully processed query using Web Search, generated 526 character response", - "query_processed.mode": "Web Search", - "query_processed.processing_time": 3.6109254360198975, - "query_processed.response_length": 526 - }, - "events": [], - "links": [] - }, - { - "span_id": "e5e3c927-783e-48c5-aec7-b32bae98c0a2", - "trace_id": "79f9d235-d46e-4740-be35-ef0006071f6c", - "parent_span_id": "f630c3e2-091e-4bd9-9556-e0ebc1c50291", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-16 08:50:51.539000000", - "end_time": "2025-10-16 08:50:51.540000000", - "duration_ms": 0.041, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What's happening in the software development world today?", - "query_length": 57, - "query_type": "routing_evaluation_query", - "classification.mode": "Web Search", - "classification.rag_keyword_score": 0, - "classification.web_keyword_score": 1, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "complex", - "query.intent": "general_knowledge", - "query.keyword_density": 0.125, - "query.domain_affinity": "general", - "routing.decision": "Routed to Web Search based on analysis", - "routing.rationale": "RAG score: 0, Web score: 1, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "RAG", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.019230769230769232, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: Web Search (Confidence: 1.00)" - }, - "events": [], - "links": [] - }, - { - "span_id": "f630c3e2-091e-4bd9-9556-e0ebc1c50291", - "trace_id": "79f9d235-d46e-4740-be35-ef0006071f6c", - "parent_span_id": "04cb7d29-9ea8-4ae0-9834-030cae944eca", - "name": "agent.query_routing", - "start_time": "2025-10-16 08:50:51.539000000", - "end_time": "2025-10-16 08:50:55.150000000", - "duration_ms": 3610.876, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What's happening in the software development world today?", - "query_length": 57, - "query_type": "routing_query", - "output_response": "Routed to Web Search for query processing", - "router.classification": "Web Search", - "router.confidence_score": 1, - "router.rag_keyword_score": 0, - "router.web_keyword_score": 1, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [] - }, - { - "span_id": "d7b0f399-3b43-43bf-bac1-b2101268841c", - "trace_id": "79f9d235-d46e-4740-be35-ef0006071f6c", - "parent_span_id": "f630c3e2-091e-4bd9-9556-e0ebc1c50291", - "name": "agent.web_search_generation", - "start_time": "2025-10-16 08:50:51.540000000", - "end_time": "2025-10-16 08:50:55.150000000", - "duration_ms": 3610.679, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "web_search_agent", - "agent.operation": "web_search_generation", - "agent.capabilities": [ - "web_search", - "content_synthesis", - "response_generation" - ], - "agent.id": "noveum_web_search_agent", - "input_query": "What's happening in the software development world today?", - "query_length": 57, - "query_type": "web_search_query", - "output_response": "Web Search Answer: The provided web search results do not contain specific information about current events or trends in the software development world. They primarily focus on unrelated topics, such as the meaning of \"...", - "web_search.results_count": 1, - "web_search.context_length": 589, - "web_search.response_length": 526, - "web_search.mode": "external_web_search" - }, - "events": [], - "links": [] - }, - { - "span_id": "4ed5dae4-f850-4644-aa55-2f1c00d11c22", - "trace_id": "79f9d235-d46e-4740-be35-ef0006071f6c", - "parent_span_id": "d7b0f399-3b43-43bf-bac1-b2101268841c", - "name": "agent.llm_model_execution", - "start_time": "2025-10-16 08:50:51.976000000", - "end_time": "2025-10-16 08:50:55.150000000", - "duration_ms": 3174.51, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What's happening in the software development world today?", - "query_length": 57, - "query_type": "web_search_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 3.174463987350464, - "model.latency_ms": 3174.463987350464, - "model.start_time": 1760604651.9761698, - "model.end_time": 1760604655.1506338, - "model.performance_tier": "medium", - "model.prompt_tokens": 272, - "model.completion_tokens": 131, - "model.total_tokens": 403, - "model.tokens_per_second": 126.95056601866197, - "model.estimated_cost": 0.01209, - "model.efficiency_score": 1.305210918114144, - "model.response_length": 526, - "model.response_quality": "high", - "model.output_response": "Model Response: The provided web search results do not contain specific information about current events or trends in the software development world. They primarily focus on unrelated topics, such as the meaning of \"...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [] - }, - { - "span_id": "eb151834-9322-4f63-add0-dec9a2966367", - "trace_id": "79f9d235-d46e-4740-be35-ef0006071f6c", - "parent_span_id": "d7b0f399-3b43-43bf-bac1-b2101268841c", - "name": "agent.web_search_evaluation_metrics", - "start_time": "2025-10-16 08:50:55.150000000", - "end_time": "2025-10-16 08:50:55.150000000", - "duration_ms": 0.02, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "web_search_evaluation_metrics", - "agent.capabilities": [ - "web_search_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What's happening in the software development world today?", - "query_length": 57, - "query_type": "web_search_evaluation_query", - "web_search.results_count": 1, - "web_search.context_length": 589, - "web_search.context_synthesized": "Context from 1 web sources", - "web_search.search_effectiveness": 0.2, - "web_search.context_quality": "medium", - "web_search.source_diversity": 1, - "web_search.information_synthesis": "low", - "web_search.external_knowledge_utilization": 0.589, - "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\n\u66fc\u5df4\u7684\u610f\u601d\u662f\u4e00\u79cd\u6bd2\u86c7\uff0c\u8c61\u5f81\u7740\u79d1\u6bd4\u5728\u7403\u573a\u4e0a\u7684\u653b\u51fb\u6027\u548c\u575a\u97e7\u4e0d\u62d4 \u201cWhat can i say\uff0c Mamba out\uff01\u201d \u4f46\u662f\u7ecf\u8fc7\u4e92\u8054\u7f51\u7684\u89e3\u6784\uff0c\u8fd9\u53e5\u8bdd\u6210\u4e3a\u4e86\u4e00\u53e5\u4e07\u7528\u573a\u666f\u7684\u8bdd \u2026 Aprende ingl\u00e9s gratis con nuestros cursos. \u00a1Mas de 100 lecciones con ejemplos y audio! Distintas y divertidas formas de estudiar ingl\u00e9s: \u2026 \u77e5\u4e4e\uff0c\u4e2d\u6587\u4e92\u8054\u7f51\u9ad8\u8d28\u91cf\u7684\u95ee\u7b54\u793e\u533a\u548c\u521b\u4f5c\u8005\u805a\u96c6\u7684\u539f\u521b\u5185\u5bb9\u5e73\u53f0\uff0c\u4e8e 2011 \u5e74 1 \u6708\u6b63\u5f0f\u4e0a\u7ebf\uff0c\u4ee5\u300c\u8ba9\u4eba\u4eec\u66f4\u597d\u7684\u5206\u4eab\u77e5\u8bc6\u3001\u7ecf\u9a8c\u548c\u89c1\u89e3\uff0c\u627e\u5230\u81ea\u5df1\u7684\u89e3\u7b54\u300d\u4e3a\u54c1 \u2026 \u8f6f\u4ef6\u7279\u70b9 1\u3001Windows Desktop Runtime\u662f\u5fae\u8f6fWindows\u684c\u9762\u7a0b\u5e8f\u8fd0\u884c\u5e93\uff08\u542b\u5e38\u89c4\u8fd0\u884c\u5e93\uff09 2\u3001\u80fd\u5b8c\u7f8e\u517c\u5bb9\u5fae\u8f6f\u4e0d\u540c\u7248\u672c\u7684Windows\u7cfb\u7edf\uff0c\u89e3\u51b3\u5176\u7a0b\u5e8f\u7f3a \u2026 \u77e5\u4e4e\uff0c\u4e2d\u6587\u4e92\u8054\u7f51\u9ad8\u8d28\u91cf\u7684\u95ee\u7b54\u793e\u533a\u548c\u521b\u4f5c\u8005\u805a\u96c6\u7684\u539f\u521b\u5185\u5bb9\u5e73\u53f0\uff0c\u4e8e 2011 \u5e74 1 \u6708\u6b63\u5f0f\u4e0a\u7ebf\uff0c\u4ee5\u300c\u8ba9\u4eba\u4eec\u66f4\u597d\u7684\u5206\u4eab\u77e5\u8bc6\u3001\u7ecf\u9a8c\u548c\u89c1\u89e3\uff0c\u627e\u5230\u81ea\u5df1\u7684\u89e3\u7b54\u300d\u4e3a\u54c1 \u2026\nURL: https://duckduckgo.com/?q=What's+happening+in+the+software+development+world+today?\n\nUser Question: What's happening in the software development world today?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", - "prompt.prompt_length": 1091, - "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\n\u66fc\u5df4\u7684\u610f\u601d\u662f\u4e00\u79cd\u6bd2\u86c7\uff0c\u8c61\u5f81\u7740\u79d1\u6bd4\u5728\u7403\u573a\u4e0a\u7684\u653b\u51fb\u6027\u548c\u575a\u97e7\u4e0d\u62d4 \u201cWhat can i say\uff0c Mamba out\uff01\u201d \u4f46\u662f\u7ecf\u8fc7\u4e92\u8054\u7f51\u7684\u89e3\u6784\uff0c\u8fd9\u53e5\u8bdd\u6210\u4e3a\u4e86\u4e00\u53e5\u4e07\u7528\u573a\u666f\u7684\u8bdd \u2026 Aprende ingl\u00e9s gratis con nuestros cursos. \u00a1Mas de 100 lecciones con ejemplos y audio...", - "prompt.instruction_following": "web_search_optimized", - "response.answer_length": 526, - "response.answer_completeness": "complete", - "response.response_quality": "medium", - "response.source_citation": 1, - "response.context_utilization": 0.8930390492359932, - "output_response": "Web Search Answer: The provided web search results do not contain specific information about current events or trends in the software development world. They primarily focus on unrelated topics, such as the meaning of \"...", - "evaluation.search_effectiveness": 0.2, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 1, - "evaluation.context_relevance": "medium", - "evaluation.overall_quality": "low", - "evaluation.ready_for_production": true, - "web_search.search_strategy": "duckduckgo_api", - "web_search.real_time_data": true, - "web_search.external_sources": 1, - "web_search.information_freshness": "current", - "web_search.knowledge_synthesis": "single_source" - }, - "events": [], - "links": [] - } - ], - "created_at": "2025-10-16 08:50:51.539000000", - "updated_at": "2025-10-16 08:50:51.539000000" - }, - { - "trace_id": "77b81bd5-5a04-4fc0-9fc6-571f7907d6b1", - "name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "status": "ok", - "status_message": "", - "start_time": "2025-10-16 08:50:44.888000000", - "end_time": "2025-10-16 08:50:50.529000000", - "duration_ms": 5641.226, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "spans": [ - { - "span_id": "888b4441-3678-42a5-a752-8daae2433482", - "trace_id": "77b81bd5-5a04-4fc0-9fc6-571f7907d6b1", - "parent_span_id": "ba6ffeeb-78b5-44dd-b460-7933ca6e7739", - "name": "agent.query_routing", - "start_time": "2025-10-16 08:50:44.888000000", - "end_time": "2025-10-16 08:50:50.529000000", - "duration_ms": 5641.062, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What are the newest features in cloud computing?", - "query_length": 48, - "query_type": "routing_query", - "output_response": "Routed to RAG for query processing", - "router.classification": "RAG", - "router.confidence_score": 1, - "router.rag_keyword_score": 1, - "router.web_keyword_score": 0, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [] - }, - { - "span_id": "b6bae768-93a6-47d9-8369-61897f9d7828", - "trace_id": "77b81bd5-5a04-4fc0-9fc6-571f7907d6b1", - "parent_span_id": "888b4441-3678-42a5-a752-8daae2433482", - "name": "agent.llm-rag", - "start_time": "2025-10-16 08:50:44.888000000", - "end_time": "2025-10-16 08:50:50.529000000", - "duration_ms": 5640.875999999999, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "rag_agent", - "agent.operation": "llm-rag", - "agent.capabilities": [ - "document_retrieval", - "context_generation", - "response_generation" - ], - "agent.id": "noveum_rag_agent", - "input_query": "What are the newest features in cloud computing?", - "query_length": 48, - "query_type": "rag_query", - "output_response": "RAG Answer: The provided context does not specifically mention any new features in cloud computing as a general topic. However, it does highlight several recent enhancements and features related to the Noveum.ai ...", - "rag.context_length": 2579, - "rag.sources_count": 5, - "rag.answer_length": 983, - "rag.mode": "retrieval_augmented_generation" - }, - "events": [], - "links": [] - }, - { - "span_id": "ba6ffeeb-78b5-44dd-b460-7933ca6e7739", - "trace_id": "77b81bd5-5a04-4fc0-9fc6-571f7907d6b1", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-16 08:50:44.888000000", - "end_time": "2025-10-16 08:50:50.529000000", - "duration_ms": 5641.156, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What are the newest features in cloud computing?", - "query.length": 48, - "processing.mode": "RAG", - "processing.time_seconds": 5.64110541343689, - "processing.response_length": 983, - "processing.sources_count": 5, - "output_response": "Final Answer: The provided context does not specifically mention any new features in cloud computing as a general topic. However, it does highlight several recent enhancements and features related to the Noveum.ai ...", - "final_answer_mode": "RAG", - "query_processed.input_query": "What are the newest features in cloud computing?", - "query_processed.output_response": "Successfully processed query using RAG, generated 983 character response", - "query_processed.mode": "RAG", - "query_processed.processing_time": 5.64110541343689, - "query_processed.response_length": 983 - }, - "events": [], - "links": [] - }, - { - "span_id": "f5538bff-8908-4a56-a622-4ba2eeedfd29", - "trace_id": "77b81bd5-5a04-4fc0-9fc6-571f7907d6b1", - "parent_span_id": "888b4441-3678-42a5-a752-8daae2433482", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-16 08:50:44.888000000", - "end_time": "2025-10-16 08:50:44.888000000", - "duration_ms": 0.032, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What are the newest features in cloud computing?", - "query_length": 48, - "query_type": "routing_evaluation_query", - "classification.mode": "RAG", - "classification.rag_keyword_score": 1, - "classification.web_keyword_score": 0, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "documentation", - "query.keyword_density": 0.125, - "query.domain_affinity": "noveum", - "routing.decision": "Routed to RAG based on analysis", - "routing.rationale": "RAG score: 1, Web score: 0, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "Web Search", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.019230769230769232, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: RAG (Confidence: 1.00)" - }, - "events": [], - "links": [] - }, - { - "span_id": "856e51cf-fa4d-459b-a046-1ea7316b903e", - "trace_id": "77b81bd5-5a04-4fc0-9fc6-571f7907d6b1", - "parent_span_id": "b6bae768-93a6-47d9-8369-61897f9d7828", - "name": "agent.llm_model_execution", - "start_time": "2025-10-16 08:50:45.423000000", - "end_time": "2025-10-16 08:50:49.997000000", - "duration_ms": 4573.782999999999, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What are the newest features in cloud computing?", - "query_length": 48, - "query_type": "rag_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 4.573714733123779, - "model.latency_ms": 4573.714733123779, - "model.start_time": 1760604645.423261, - "model.end_time": 1760604649.9969757, - "model.performance_tier": "medium", - "model.prompt_tokens": 780, - "model.completion_tokens": 245, - "model.total_tokens": 1025, - "model.tokens_per_second": 224.1066747291299, - "model.estimated_cost": 0.03075, - "model.efficiency_score": 0.9590243902439024, - "model.response_length": 983, - "model.response_quality": "high", - "model.output_response": "Model Response: The provided context does not specifically mention any new features in cloud computing as a general topic. However, it does highlight several recent enhancements and features related to the Noveum.ai ...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [] - }, - { - "span_id": "a2fa7400-74c4-4718-a562-899b91503c95", - "trace_id": "77b81bd5-5a04-4fc0-9fc6-571f7907d6b1", - "parent_span_id": "b6bae768-93a6-47d9-8369-61897f9d7828", - "name": "agent.rag_evaluation_metrics", - "start_time": "2025-10-16 08:50:49.997000000", - "end_time": "2025-10-16 08:50:49.997000000", - "duration_ms": 0.048, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "rag_evaluation_metrics", - "agent.capabilities": [ - "retrieval_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What are the newest features in cloud computing?", - "query_length": 48, - "query_type": "rag_evaluation_query", - "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/changelog):\n- \ud83d\udcc8 Enhanced Dashboard Analytics: Real-time request tracking, latency monitoring, and cost analysis\n- \ud83d\udd0d Improved Logs Interface: Better search functionality, detailed trace views, and enhanced debugging capabilities\n- \ud83d\udc65 Team Management: Advanced member mana...", - "retrieval.context_length": 2579, - "retrieval.sources_count": 5, - "retrieval.context_quality": "high", - "retrieval.effectiveness": 1, - "retrieval.context_utilization": 2.579, - "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/changelog):\n- \ud83d\udcc8 Enhanced Dashboard Analytics: Real-time request tracking, latency monitoring, and cost analysis\n- \ud83d\udd0d Improved Logs Interface: Better search functionality, detailed trace views, and enhanced debugging capabilities\n- \ud83d\udc65 Team Management: Advanced member management with role-based access control and invitation system\n- \ud83d\udee1\ufe0f Security Improvements: Enhanced API key management and secure credential storage\n4 months ago\n- \ud83d\udcca Advanced Metrics Collection: Comprehensive telemetry system with custom metrics ...\n\nSource 2 (https://noveum.ai/en/changelog):\n- \ud83d\udcdd Documentation Updates: Comprehensive guides for integration and best practices\n5 months ago\n- \ud83c\udf89 Platform Foundation: Initial release of Noveum.ai AI observability platform\n- \ud83d\ude80 Core Features: AI Gateway integration, basic evaluation metrics, and monitoring capabilities\n- \ud83d\udcc8 Evaluation Jobs: Automated model evaluation with configurable metrics and reporting\n- \ud83d\udd0c Provider Integrations: Support for major AI providers and custom model deployments...\n\nSource 3 (https://noveum.ai/docs/platform/dashboard):\n- Bulk Export: Download multiple traces simultaneously\n- Comparative Analysis: Compare performance across multiple traces\nReal-Time Updates\n- Live Refresh: Automatic updates as new traces arrive\n- Connection Monitoring: Real-time status of your trace ingestion pipeline\n- Performance Indicators: Live metrics for system health monitoring\nReady to dive deeper? Explore Projects & Environments to organize your AI applications, or check out Team Collaboration to share insights with your team.\nGet Earl...\n\nSource 4 (https://noveum.ai/en/changelog):\nChangelog\nStay up to date with the latest changes in our product.\n2 months ago\n- \ud83c\udfaf Major Platform Evolution: NovaEval Framework - Launched comprehensive AI model evaluation framework with 20+ built-in scorers including accuracy, RAG metrics, conversational scoring, and specialized agent evaluation tools. Features production-grade deployment on Docker and Kubernetes. (GitHub | PyPI Package)\n- \ud83d\ude80 Next-Generation Traces UI - Complete redesign featuring three-pane layout with directory tree navigatio...\n\nSource 5 (https://noveum.ai/en/blog):\n8/12/2025\nDiscover how Noveum.ai provides comprehensive tracing and observability for AI applications, from development debugging to production optimization.\nShashank Agarwal\n3/3/2025\nDiscover how Noveum.ai provides comprehensive tracing and observability for LLM applications, RAG systems, and multi-agent workflows with our powerful Python and TypeScript SDKs.\nShashank Agarwal\n3/2/2025...\n\nUser Question: What are the newest features in cloud computing?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", - "prompt.prompt_length": 3123, - "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/changelog):\n- \ud83d\udcc8 Enhanced Dashboard Analytics: Real-time request tracking, latency monitoring, and cost analysis\n- \ud83d\udd0d Improved Logs Interface: Better search functionality,...", - "prompt.instruction_following": "rag_optimized", - "response.answer_length": 983, - "response.answer_completeness": "complete", - "response.response_quality": "high", - "response.source_citation": 5, - "response.context_utilization": 0.381155486622722, - "output_response": "RAG Answer: The provided context does not specifically mention any new features in cloud computing as a general topic. However, it does highlight several recent enhancements and features related to the Noveum.ai ...", - "evaluation.retrieval_effectiveness": 1, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 5, - "evaluation.context_relevance": "high", - "evaluation.overall_quality": "high", - "evaluation.ready_for_production": true, - "rag.retrieval_strategy": "semantic_similarity", - "rag.vector_search_results": 5, - "rag.context_synthesis": "multi_source", - "rag.document_coverage": 1, - "rag.information_density": 0.381155486622722 - }, - "events": [], - "links": [] - } - ], - "created_at": "2025-10-16 08:50:44.888000000", - "updated_at": "2025-10-16 08:50:44.888000000" - }, - { - "trace_id": "32170d1a-d211-46d8-9cdc-5a4265f0c443", - "name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "status": "ok", - "status_message": "", - "start_time": "2025-10-16 08:50:42.124000000", - "end_time": "2025-10-16 08:50:43.885000000", - "duration_ms": 1761.047, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "spans": [ - { - "span_id": "4c391bb1-1a62-4e2a-913b-96f83feff494", - "trace_id": "32170d1a-d211-46d8-9cdc-5a4265f0c443", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-16 08:50:42.124000000", - "end_time": "2025-10-16 08:50:43.885000000", - "duration_ms": 1760.974, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What's the current status of cryptocurrency markets?", - "query.length": 52, - "processing.mode": "Web Search", - "processing.time_seconds": 1.7608659267425537, - "processing.response_length": 363, - "processing.sources_count": 1, - "output_response": "Final Answer: The web search did not yield any specific information regarding the current status of cryptocurrency markets. Therefore, I cannot provide an update on market trends, prices, or significant events affe...", - "final_answer_mode": "Web Search", - "query_processed.input_query": "What's the current status of cryptocurrency markets?", - "query_processed.output_response": "Successfully processed query using Web Search, generated 363 character response", - "query_processed.mode": "Web Search", - "query_processed.processing_time": 1.7608659267425537, - "query_processed.response_length": 363 - }, - "events": [], - "links": [] - }, - { - "span_id": "7ab2ec7b-a617-4ebf-9bb1-82dc6b18e160", - "trace_id": "32170d1a-d211-46d8-9cdc-5a4265f0c443", - "parent_span_id": "4c391bb1-1a62-4e2a-913b-96f83feff494", - "name": "agent.query_routing", - "start_time": "2025-10-16 08:50:42.124000000", - "end_time": "2025-10-16 08:50:43.885000000", - "duration_ms": 1760.833, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What's the current status of cryptocurrency markets?", - "query_length": 52, - "query_type": "routing_query", - "output_response": "Routed to Web Search for query processing", - "router.classification": "Web Search", - "router.confidence_score": 1, - "router.rag_keyword_score": 0, - "router.web_keyword_score": 3, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [] - }, - { - "span_id": "db01d228-7c63-4814-b2ea-bdf3898f9c0d", - "trace_id": "32170d1a-d211-46d8-9cdc-5a4265f0c443", - "parent_span_id": "7ab2ec7b-a617-4ebf-9bb1-82dc6b18e160", - "name": "agent.web_search_generation", - "start_time": "2025-10-16 08:50:42.124000000", - "end_time": "2025-10-16 08:50:43.885000000", - "duration_ms": 1760.701, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "web_search_agent", - "agent.operation": "web_search_generation", - "agent.capabilities": [ - "web_search", - "content_synthesis", - "response_generation" - ], - "agent.id": "noveum_web_search_agent", - "input_query": "What's the current status of cryptocurrency markets?", - "query_length": 52, - "query_type": "web_search_query", - "output_response": "Web Search Answer: The web search did not yield any specific information regarding the current status of cryptocurrency markets. Therefore, I cannot provide an update on market trends, prices, or significant events affe...", - "web_search.results_count": 1, - "web_search.context_length": 154, - "web_search.response_length": 363, - "web_search.mode": "external_web_search" - }, - "events": [], - "links": [] - }, - { - "span_id": "da7b0550-b13c-4518-ad3b-050c666d198c", - "trace_id": "32170d1a-d211-46d8-9cdc-5a4265f0c443", - "parent_span_id": "7ab2ec7b-a617-4ebf-9bb1-82dc6b18e160", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-16 08:50:42.124000000", - "end_time": "2025-10-16 08:50:42.124000000", - "duration_ms": 0.029, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What's the current status of cryptocurrency markets?", - "query_length": 52, - "query_type": "routing_evaluation_query", - "classification.mode": "Web Search", - "classification.rag_keyword_score": 0, - "classification.web_keyword_score": 3, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "complex", - "query.intent": "general_knowledge", - "query.keyword_density": 0.42857142857142855, - "query.domain_affinity": "general", - "routing.decision": "Routed to Web Search based on analysis", - "routing.rationale": "RAG score: 0, Web score: 3, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "RAG", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.057692307692307696, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: Web Search (Confidence: 1.00)" - }, - "events": [], - "links": [] - }, - { - "span_id": "950e18b6-9f62-4ac4-8788-3cac08fb7899", - "trace_id": "32170d1a-d211-46d8-9cdc-5a4265f0c443", - "parent_span_id": "db01d228-7c63-4814-b2ea-bdf3898f9c0d", - "name": "agent.llm_model_execution", - "start_time": "2025-10-16 08:50:42.465000000", - "end_time": "2025-10-16 08:50:43.885000000", - "duration_ms": 1419.5690000000002, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What's the current status of cryptocurrency markets?", - "query_length": 52, - "query_type": "web_search_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 1.4195220470428467, - "model.latency_ms": 1419.5220470428467, - "model.start_time": 1760604642.4658613, - "model.end_time": 1760604643.8853834, - "model.performance_tier": "fast", - "model.prompt_tokens": 162, - "model.completion_tokens": 90, - "model.total_tokens": 252, - "model.tokens_per_second": 177.52454111224782, - "model.estimated_cost": 0.00756, - "model.efficiency_score": 1.4404761904761905, - "model.response_length": 363, - "model.response_quality": "high", - "model.output_response": "Model Response: The web search did not yield any specific information regarding the current status of cryptocurrency markets. Therefore, I cannot provide an update on market trends, prices, or significant events affe...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [] - }, - { - "span_id": "e87846c9-ce82-42c2-a517-e0f67e9d14c1", - "trace_id": "32170d1a-d211-46d8-9cdc-5a4265f0c443", - "parent_span_id": "db01d228-7c63-4814-b2ea-bdf3898f9c0d", - "name": "agent.web_search_evaluation_metrics", - "start_time": "2025-10-16 08:50:43.885000000", - "end_time": "2025-10-16 08:50:43.885000000", - "duration_ms": 0.02, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "web_search_evaluation_metrics", - "agent.capabilities": [ - "web_search_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What's the current status of cryptocurrency markets?", - "query_length": 52, - "query_type": "web_search_evaluation_query", - "web_search.results_count": 1, - "web_search.context_length": 154, - "web_search.context_synthesized": "Context from 1 web sources", - "web_search.search_effectiveness": 0.2, - "web_search.context_quality": "low", - "web_search.source_diversity": 1, - "web_search.information_synthesis": "low", - "web_search.external_knowledge_utilization": 0.154, - "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What's+the+current+status+of+cryptocurrency+markets?\n\nUser Question: What's the current status of cryptocurrency markets?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", - "prompt.prompt_length": 651, - "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What's+the+current+status+of+cryptocurrency+markets?", - "prompt.instruction_following": "web_search_optimized", - "response.answer_length": 363, - "response.answer_completeness": "complete", - "response.response_quality": "medium", - "response.source_citation": 1, - "response.context_utilization": 2.357142857142857, - "output_response": "Web Search Answer: The web search did not yield any specific information regarding the current status of cryptocurrency markets. Therefore, I cannot provide an update on market trends, prices, or significant events affe...", - "evaluation.search_effectiveness": 0.2, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 1, - "evaluation.context_relevance": "low", - "evaluation.overall_quality": "low", - "evaluation.ready_for_production": true, - "web_search.search_strategy": "duckduckgo_api", - "web_search.real_time_data": true, - "web_search.external_sources": 1, - "web_search.information_freshness": "current", - "web_search.knowledge_synthesis": "single_source" - }, - "events": [], - "links": [] - } - ], - "created_at": "2025-10-16 08:50:42.124000000", - "updated_at": "2025-10-16 08:50:42.124000000" - }, - { - "trace_id": "9424d420-deee-478b-b727-9dc791a7457c", - "name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "status": "ok", - "status_message": "", - "start_time": "2025-10-16 08:50:31.476000000", - "end_time": "2025-10-16 08:50:41.121000000", - "duration_ms": 9644.828, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "spans": [ - { - "span_id": "305daacf-990b-4e23-8720-48900f75d9be", - "trace_id": "9424d420-deee-478b-b727-9dc791a7457c", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-16 08:50:31.476000000", - "end_time": "2025-10-16 08:50:41.121000000", - "duration_ms": 9644.752999999999, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What are the latest updates in Python programming?", - "query.length": 50, - "processing.mode": "Web Search", - "processing.time_seconds": 9.644662618637085, - "processing.response_length": 1408, - "processing.sources_count": 1, - "output_response": "Final Answer: The web search did not yield specific results regarding the latest updates in Python programming. However, I can provide some general information based on my knowledge up to October 2023.\n\nAs of late ...", - "final_answer_mode": "Web Search", - "query_processed.input_query": "What are the latest updates in Python programming?", - "query_processed.output_response": "Successfully processed query using Web Search, generated 1408 character response", - "query_processed.mode": "Web Search", - "query_processed.processing_time": 9.644662618637085, - "query_processed.response_length": 1408 - }, - "events": [], - "links": [] - }, - { - "span_id": "56b4cc1b-0646-4dee-acb3-db98f3d44477", - "trace_id": "9424d420-deee-478b-b727-9dc791a7457c", - "parent_span_id": "fb234e30-69be-4d06-9248-c05bcc22f063", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-16 08:50:31.476000000", - "end_time": "2025-10-16 08:50:31.476000000", - "duration_ms": 0.036000000000000004, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What are the latest updates in Python programming?", - "query_length": 50, - "query_type": "routing_evaluation_query", - "classification.mode": "Web Search", - "classification.rag_keyword_score": 0, - "classification.web_keyword_score": 2, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "general_knowledge", - "query.keyword_density": 0.25, - "query.domain_affinity": "general", - "routing.decision": "Routed to Web Search based on analysis", - "routing.rationale": "RAG score: 0, Web score: 2, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "RAG", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.038461538461538464, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: Web Search (Confidence: 1.00)" - }, - "events": [], - "links": [] - }, - { - "span_id": "fb234e30-69be-4d06-9248-c05bcc22f063", - "trace_id": "9424d420-deee-478b-b727-9dc791a7457c", - "parent_span_id": "305daacf-990b-4e23-8720-48900f75d9be", - "name": "agent.query_routing", - "start_time": "2025-10-16 08:50:31.476000000", - "end_time": "2025-10-16 08:50:41.121000000", - "duration_ms": 9644.627, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What are the latest updates in Python programming?", - "query_length": 50, - "query_type": "routing_query", - "output_response": "Routed to Web Search for query processing", - "router.classification": "Web Search", - "router.confidence_score": 1, - "router.rag_keyword_score": 0, - "router.web_keyword_score": 2, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [] - }, - { - "span_id": "f8a793a9-63ee-49b7-9931-d7dcc2bb0ca7", - "trace_id": "9424d420-deee-478b-b727-9dc791a7457c", - "parent_span_id": "fb234e30-69be-4d06-9248-c05bcc22f063", - "name": "agent.web_search_generation", - "start_time": "2025-10-16 08:50:31.477000000", - "end_time": "2025-10-16 08:50:41.121000000", - "duration_ms": 9644.478, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "web_search_agent", - "agent.operation": "web_search_generation", - "agent.capabilities": [ - "web_search", - "content_synthesis", - "response_generation" - ], - "agent.id": "noveum_web_search_agent", - "input_query": "What are the latest updates in Python programming?", - "query_length": 50, - "query_type": "web_search_query", - "output_response": "Web Search Answer: The web search did not yield specific results regarding the latest updates in Python programming. However, I can provide some general information based on my knowledge up to October 2023.\n\nAs of late ...", - "web_search.results_count": 1, - "web_search.context_length": 152, - "web_search.response_length": 1408, - "web_search.mode": "external_web_search" - }, - "events": [], - "links": [] - }, - { - "span_id": "b968e3d3-d4aa-4357-85d8-34a612748fbd", - "trace_id": "9424d420-deee-478b-b727-9dc791a7457c", - "parent_span_id": "f8a793a9-63ee-49b7-9931-d7dcc2bb0ca7", - "name": "agent.llm_model_execution", - "start_time": "2025-10-16 08:50:32.016000000", - "end_time": "2025-10-16 08:50:41.121000000", - "duration_ms": 9105.344000000001, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What are the latest updates in Python programming?", - "query_length": 50, - "query_type": "web_search_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 9.105276584625244, - "model.latency_ms": 9105.276584625244, - "model.start_time": 1760604632.0161095, - "model.end_time": 1760604641.121386, - "model.performance_tier": "slow", - "model.prompt_tokens": 161, - "model.completion_tokens": 352, - "model.total_tokens": 513, - "model.tokens_per_second": 56.340957381374714, - "model.estimated_cost": 0.015390000000000001, - "model.efficiency_score": 2.7446393762183234, - "model.response_length": 1408, - "model.response_quality": "high", - "model.output_response": "Model Response: The web search did not yield specific results regarding the latest updates in Python programming. However, I can provide some general information based on my knowledge up to October 2023.\n\nAs of late ...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [] - }, - { - "span_id": "80817711-e64c-4a9f-802e-2a9e36ff093d", - "trace_id": "9424d420-deee-478b-b727-9dc791a7457c", - "parent_span_id": "f8a793a9-63ee-49b7-9931-d7dcc2bb0ca7", - "name": "agent.web_search_evaluation_metrics", - "start_time": "2025-10-16 08:50:41.121000000", - "end_time": "2025-10-16 08:50:41.121000000", - "duration_ms": 0.02, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "web_search_evaluation_metrics", - "agent.capabilities": [ - "web_search_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What are the latest updates in Python programming?", - "query_length": 50, - "query_type": "web_search_evaluation_query", - "web_search.results_count": 1, - "web_search.context_length": 152, - "web_search.context_synthesized": "Context from 1 web sources", - "web_search.search_effectiveness": 0.2, - "web_search.context_quality": "low", - "web_search.source_diversity": 1, - "web_search.information_synthesis": "low", - "web_search.external_knowledge_utilization": 0.152, - "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What+are+the+latest+updates+in+Python+programming?\n\nUser Question: What are the latest updates in Python programming?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", - "prompt.prompt_length": 647, - "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What+are+the+latest+updates+in+Python+programming?", - "prompt.instruction_following": "web_search_optimized", - "response.answer_length": 1408, - "response.answer_completeness": "complete", - "response.response_quality": "medium", - "response.source_citation": 1, - "response.context_utilization": 9.263157894736842, - "output_response": "Web Search Answer: The web search did not yield specific results regarding the latest updates in Python programming. However, I can provide some general information based on my knowledge up to October 2023.\n\nAs of late ...", - "evaluation.search_effectiveness": 0.2, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 1, - "evaluation.context_relevance": "low", - "evaluation.overall_quality": "low", - "evaluation.ready_for_production": true, - "web_search.search_strategy": "duckduckgo_api", - "web_search.real_time_data": true, - "web_search.external_sources": 1, - "web_search.information_freshness": "current", - "web_search.knowledge_synthesis": "single_source" - }, - "events": [], - "links": [] - } - ], - "created_at": "2025-10-16 08:50:31.476000000", - "updated_at": "2025-10-16 08:50:31.476000000" - }, - { - "trace_id": "a3735d64-89bb-4fdf-a851-7a1f3515c52d", - "name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "status": "ok", - "status_message": "", - "start_time": "2025-10-16 08:50:27.687000000", - "end_time": "2025-10-16 08:50:30.473000000", - "duration_ms": 2786.145, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "spans": [ - { - "span_id": "13af0f3c-bd5c-4024-8fa3-c9e347a1ef20", - "trace_id": "a3735d64-89bb-4fdf-a851-7a1f3515c52d", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-16 08:50:27.687000000", - "end_time": "2025-10-16 08:50:30.473000000", - "duration_ms": 2786.078, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What happened in tech news this week?", - "query.length": 37, - "processing.mode": "Web Search", - "processing.time_seconds": 2.7859997749328613, - "processing.response_length": 396, - "processing.sources_count": 1, - "output_response": "Final Answer: It appears that the web search did not yield any specific results regarding recent tech news. Unfortunately, I cannot provide a summary of this week's tech news based on the available information. For...", - "final_answer_mode": "Web Search", - "query_processed.input_query": "What happened in tech news this week?", - "query_processed.output_response": "Successfully processed query using Web Search, generated 396 character response", - "query_processed.mode": "Web Search", - "query_processed.processing_time": 2.7859997749328613, - "query_processed.response_length": 396 - }, - "events": [], - "links": [] - }, - { - "span_id": "47e8f375-35e6-4097-86ff-df73e8181c23", - "trace_id": "a3735d64-89bb-4fdf-a851-7a1f3515c52d", - "parent_span_id": "a4f1a7b3-dc35-4f13-96d3-b6dc8fb21a37", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-16 08:50:27.687000000", - "end_time": "2025-10-16 08:50:27.687000000", - "duration_ms": 0.038, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What happened in tech news this week?", - "query_length": 37, - "query_type": "routing_evaluation_query", - "classification.mode": "Web Search", - "classification.rag_keyword_score": 0, - "classification.web_keyword_score": 2, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "general_knowledge", - "query.keyword_density": 0.2857142857142857, - "query.domain_affinity": "general", - "routing.decision": "Routed to Web Search based on analysis", - "routing.rationale": "RAG score: 0, Web score: 2, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "RAG", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.038461538461538464, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: Web Search (Confidence: 1.00)" - }, - "events": [], - "links": [] - }, - { - "span_id": "a4f1a7b3-dc35-4f13-96d3-b6dc8fb21a37", - "trace_id": "a3735d64-89bb-4fdf-a851-7a1f3515c52d", - "parent_span_id": "13af0f3c-bd5c-4024-8fa3-c9e347a1ef20", - "name": "agent.query_routing", - "start_time": "2025-10-16 08:50:27.687000000", - "end_time": "2025-10-16 08:50:30.473000000", - "duration_ms": 2785.964, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What happened in tech news this week?", - "query_length": 37, - "query_type": "routing_query", - "output_response": "Routed to Web Search for query processing", - "router.classification": "Web Search", - "router.confidence_score": 1, - "router.rag_keyword_score": 0, - "router.web_keyword_score": 2, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [] - }, - { - "span_id": "ebfff280-5eb6-4838-8de7-35c85e2e3eaa", - "trace_id": "a3735d64-89bb-4fdf-a851-7a1f3515c52d", - "parent_span_id": "a4f1a7b3-dc35-4f13-96d3-b6dc8fb21a37", - "name": "agent.web_search_generation", - "start_time": "2025-10-16 08:50:27.687000000", - "end_time": "2025-10-16 08:50:30.473000000", - "duration_ms": 2785.8230000000003, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "web_search_agent", - "agent.operation": "web_search_generation", - "agent.capabilities": [ - "web_search", - "content_synthesis", - "response_generation" - ], - "agent.id": "noveum_web_search_agent", - "input_query": "What happened in tech news this week?", - "query_length": 37, - "query_type": "web_search_query", - "output_response": "Web Search Answer: It appears that the web search did not yield any specific results regarding recent tech news. Unfortunately, I cannot provide a summary of this week's tech news based on the available information. For...", - "web_search.results_count": 1, - "web_search.context_length": 139, - "web_search.response_length": 396, - "web_search.mode": "external_web_search" - }, - "events": [], - "links": [] - }, - { - "span_id": "835b9351-d381-4785-8538-fbe1064fc8cd", - "trace_id": "a3735d64-89bb-4fdf-a851-7a1f3515c52d", - "parent_span_id": "ebfff280-5eb6-4838-8de7-35c85e2e3eaa", - "name": "agent.llm_model_execution", - "start_time": "2025-10-16 08:50:28.026000000", - "end_time": "2025-10-16 08:50:30.473000000", - "duration_ms": 2447.269, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What happened in tech news this week?", - "query_length": 37, - "query_type": "web_search_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 2.447227716445923, - "model.latency_ms": 2447.227716445923, - "model.start_time": 1760604628.0260897, - "model.end_time": 1760604630.4733174, - "model.performance_tier": "medium", - "model.prompt_tokens": 155, - "model.completion_tokens": 99, - "model.total_tokens": 254, - "model.tokens_per_second": 103.79091340501853, - "model.estimated_cost": 0.00762, - "model.efficiency_score": 1.5590551181102361, - "model.response_length": 396, - "model.response_quality": "high", - "model.output_response": "Model Response: It appears that the web search did not yield any specific results regarding recent tech news. Unfortunately, I cannot provide a summary of this week's tech news based on the available information. For...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [] - }, - { - "span_id": "93c6f427-6fcd-4318-aec8-095553fed469", - "trace_id": "a3735d64-89bb-4fdf-a851-7a1f3515c52d", - "parent_span_id": "ebfff280-5eb6-4838-8de7-35c85e2e3eaa", - "name": "agent.web_search_evaluation_metrics", - "start_time": "2025-10-16 08:50:30.473000000", - "end_time": "2025-10-16 08:50:30.473000000", - "duration_ms": 0.017, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "web_search_evaluation_metrics", - "agent.capabilities": [ - "web_search_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What happened in tech news this week?", - "query_length": 37, - "query_type": "web_search_evaluation_query", - "web_search.results_count": 1, - "web_search.context_length": 139, - "web_search.context_synthesized": "Context from 1 web sources", - "web_search.search_effectiveness": 0.2, - "web_search.context_quality": "low", - "web_search.source_diversity": 1, - "web_search.information_synthesis": "low", - "web_search.external_knowledge_utilization": 0.139, - "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What+happened+in+tech+news+this+week?\n\nUser Question: What happened in tech news this week?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", - "prompt.prompt_length": 621, - "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What+happened+in+tech+news+this+week?", - "prompt.instruction_following": "web_search_optimized", - "response.answer_length": 396, - "response.answer_completeness": "complete", - "response.response_quality": "medium", - "response.source_citation": 1, - "response.context_utilization": 2.8489208633093526, - "output_response": "Web Search Answer: It appears that the web search did not yield any specific results regarding recent tech news. Unfortunately, I cannot provide a summary of this week's tech news based on the available information. For...", - "evaluation.search_effectiveness": 0.2, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 1, - "evaluation.context_relevance": "low", - "evaluation.overall_quality": "low", - "evaluation.ready_for_production": true, - "web_search.search_strategy": "duckduckgo_api", - "web_search.real_time_data": true, - "web_search.external_sources": 1, - "web_search.information_freshness": "current", - "web_search.knowledge_synthesis": "single_source" - }, - "events": [], - "links": [] - } - ], - "created_at": "2025-10-16 08:50:27.687000000", - "updated_at": "2025-10-16 08:50:27.687000000" - }, - { - "trace_id": "fa5bb641-f4b7-4741-b06c-392a5f2a9aea", - "name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "status": "ok", - "status_message": "", - "start_time": "2025-10-16 08:50:21.645000000", - "end_time": "2025-10-16 08:50:26.683000000", - "duration_ms": 5038.112, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "spans": [ - { - "span_id": "622e7353-4eeb-47d5-8558-4ecec28cdcfd", - "trace_id": "fa5bb641-f4b7-4741-b06c-392a5f2a9aea", - "parent_span_id": "c0af6629-48b1-45fc-9dd2-60f098ea140e", - "name": "agent.query_routing", - "start_time": "2025-10-16 08:50:21.645000000", - "end_time": "2025-10-16 08:50:26.683000000", - "duration_ms": 5037.965, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What are the current trends in observability tools?", - "query_length": 51, - "query_type": "routing_query", - "output_response": "Routed to RAG for query processing", - "router.classification": "RAG", - "router.confidence_score": 0.3333333333333333, - "router.rag_keyword_score": 2, - "router.web_keyword_score": 1, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [] - }, - { - "span_id": "c0af6629-48b1-45fc-9dd2-60f098ea140e", - "trace_id": "fa5bb641-f4b7-4741-b06c-392a5f2a9aea", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-16 08:50:21.645000000", - "end_time": "2025-10-16 08:50:26.683000000", - "duration_ms": 5038.0419999999995, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What are the current trends in observability tools?", - "query.length": 51, - "processing.mode": "RAG", - "processing.time_seconds": 5.0380027294158936, - "processing.response_length": 897, - "processing.sources_count": 5, - "output_response": "Final Answer: The provided context does not specifically outline current trends in observability tools beyond the focus on AI applications. However, it highlights the limitations of traditional monitoring tools for...", - "final_answer_mode": "RAG", - "query_processed.input_query": "What are the current trends in observability tools?", - "query_processed.output_response": "Successfully processed query using RAG, generated 897 character response", - "query_processed.mode": "RAG", - "query_processed.processing_time": 5.0380027294158936, - "query_processed.response_length": 897 - }, - "events": [], - "links": [] - }, - { - "span_id": "ee55906d-ffe3-42fd-9899-a476d4c33ff5", - "trace_id": "fa5bb641-f4b7-4741-b06c-392a5f2a9aea", - "parent_span_id": "622e7353-4eeb-47d5-8558-4ecec28cdcfd", - "name": "agent.llm-rag", - "start_time": "2025-10-16 08:50:21.645000000", - "end_time": "2025-10-16 08:50:26.683000000", - "duration_ms": 5037.803, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "rag_agent", - "agent.operation": "llm-rag", - "agent.capabilities": [ - "document_retrieval", - "context_generation", - "response_generation" - ], - "agent.id": "noveum_rag_agent", - "input_query": "What are the current trends in observability tools?", - "query_length": 51, - "query_type": "rag_query", - "output_response": "RAG Answer: The provided context does not specifically outline current trends in observability tools beyond the focus on AI applications. However, it highlights the limitations of traditional monitoring tools for...", - "rag.context_length": 2943, - "rag.sources_count": 5, - "rag.answer_length": 897, - "rag.mode": "retrieval_augmented_generation" - }, - "events": [], - "links": [] - }, - { - "span_id": "fe610ecb-cc8e-4466-b719-34c6e0d45d87", - "trace_id": "fa5bb641-f4b7-4741-b06c-392a5f2a9aea", - "parent_span_id": "622e7353-4eeb-47d5-8558-4ecec28cdcfd", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-16 08:50:21.645000000", - "end_time": "2025-10-16 08:50:21.645000000", - "duration_ms": 0.032, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What are the current trends in observability tools?", - "query_length": 51, - "query_type": "routing_evaluation_query", - "classification.mode": "RAG", - "classification.rag_keyword_score": 2, - "classification.web_keyword_score": 1, - "classification.confidence_score": 0.3333333333333333, - "classification.confidence_level": "medium", - "classification.method": "keyword_based", - "query.complexity": "complex", - "query.intent": "documentation", - "query.keyword_density": 0.375, - "query.domain_affinity": "noveum", - "routing.decision": "Routed to RAG based on analysis", - "routing.rationale": "RAG score: 2, Web score: 1, Confidence: 0.33", - "routing.expected_performance": "medium", - "routing.alternative_mode": "Web Search", - "routing.decision_confidence": 0.3333333333333333, - "evaluation.routing_accuracy": "medium", - "evaluation.keyword_coverage": 0.057692307692307696, - "evaluation.query_understanding": "ambiguous", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: RAG (Confidence: 0.33)" - }, - "events": [], - "links": [] - }, - { - "span_id": "69a4773d-c738-489e-87ed-999583e02187", - "trace_id": "fa5bb641-f4b7-4741-b06c-392a5f2a9aea", - "parent_span_id": "ee55906d-ffe3-42fd-9899-a476d4c33ff5", - "name": "agent.llm_model_execution", - "start_time": "2025-10-16 08:50:22.600000000", - "end_time": "2025-10-16 08:50:25.786000000", - "duration_ms": 3185.257, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What are the current trends in observability tools?", - "query_length": 51, - "query_type": "rag_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 3.185206174850464, - "model.latency_ms": 3185.206174850464, - "model.start_time": 1760604622.6008554, - "model.end_time": 1760604625.7860615, - "model.performance_tier": "medium", - "model.prompt_tokens": 872, - "model.completion_tokens": 224, - "model.total_tokens": 1096, - "model.tokens_per_second": 344.09075577390337, - "model.estimated_cost": 0.03288, - "model.efficiency_score": 0.8184306569343066, - "model.response_length": 897, - "model.response_quality": "high", - "model.output_response": "Model Response: The provided context does not specifically outline current trends in observability tools beyond the focus on AI applications. However, it highlights the limitations of traditional monitoring tools for...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [] - }, - { - "span_id": "5e5a9d30-4cd0-45cd-9494-192c280727ac", - "trace_id": "fa5bb641-f4b7-4741-b06c-392a5f2a9aea", - "parent_span_id": "ee55906d-ffe3-42fd-9899-a476d4c33ff5", - "name": "agent.rag_evaluation_metrics", - "start_time": "2025-10-16 08:50:25.786000000", - "end_time": "2025-10-16 08:50:25.786000000", - "duration_ms": 0.066, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "rag_evaluation_metrics", - "agent.capabilities": [ - "retrieval_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What are the current trends in observability tools?", - "query_length": 51, - "query_type": "rag_evaluation_query", - "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/blog/from-logs-to-intelligent-choices-inside-noveum-ais-evaluation-process):\nFrom Development to Production - Inside Noveum.ai's AI Observability Platform\nShashank Agarwal\n3/3/2025\nIntroduction\nEvery AI application tells a story\u2014but most developers never get to hear it...", - "retrieval.context_length": 2943, - "retrieval.sources_count": 5, - "retrieval.context_quality": "high", - "retrieval.effectiveness": 1, - "retrieval.context_utilization": 2.943, - "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/blog/from-logs-to-intelligent-choices-inside-noveum-ais-evaluation-process):\nFrom Development to Production - Inside Noveum.ai's AI Observability Platform\nShashank Agarwal\n3/3/2025\nIntroduction\nEvery AI application tells a story\u2014but most developers never get to hear it. When your RAG pipeline returns irrelevant results, when your multi-agent system gets stuck in loops, or when your LLM costs suddenly spike, you're left guessing what went wrong and where.\nThat's exactly the challenge Noveum.ai solves. Rather than flying blind, our platform provides comprehensive tracing a...\n\nSource 2 (https://noveum.ai/docs/getting-started/tracing-concepts):\n- Explore Framework Integrations - Framework-specific guidance\n- Learn Advanced Patterns - Custom instrumentation techniques\n- Master the Dashboard - Analyze your traces effectively\nRemember: Good observability is not about collecting all possible data, but about collecting the right data that helps you understand, debug, and optimize your AI applications.\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to O...\n\nSource 3 (https://noveum.ai/en/docs/getting-started/tracing-concepts):\n- Explore Framework Integrations - Framework-specific guidance\n- Learn Advanced Patterns - Custom instrumentation techniques\n- Master the Dashboard - Analyze your traces effectively\nRemember: Good observability is not about collecting all possible data, but about collecting the right data that helps you understand, debug, and optimize your AI applications.\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to O...\n\nSource 4 (https://noveum.ai/en/blog/from-logs-to-intelligent-choices-inside-noveum-ais-evaluation-process):\nThe AI Observability Challenge\nWhy Traditional Monitoring Falls Short\nTraditional application monitoring tools weren't built for AI applications. They can tell you if your API is responding, but they can't answer the questions that matter most for AI systems:\n- Why did my RAG pipeline retrieve irrelevant documents?\n- Which LLM calls are driving my costs?\n- How are my agents communicating with each other?\n- What's causing hallucinations in my responses?\n- Why is my embedding generation so slow?\nW...\n\nSource 5 (https://noveum.ai/en/docs/advanced/rag-observability):\n\ud83d\udd27 Best Practices for RAG Observability\n1. Comprehensive Pipeline Tracing\n2. Quality Monitoring\n3. A/B Testing for RAG Components\n\ud83c\udfaf Advanced RAG Patterns\nMulti-Modal RAG\nConversational RAG\nRAG observability with Noveum.ai provides the deep insights needed to build, optimize, and scale retrieval-augmented generation systems. By implementing comprehensive tracing across retrieval, context preparation, and generation stages, you can ensure your RAG pipeline delivers accurate, relevant, and cost-effe...\n\nUser Question: What are the current trends in observability tools?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", - "prompt.prompt_length": 3490, - "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/blog/from-logs-to-intelligent-choices-inside-noveum-ais-evaluation-process):\nFrom Development to Production - Inside Noveum.ai's AI Observability Platform\nShashank Agarw...", - "prompt.instruction_following": "rag_optimized", - "response.answer_length": 897, - "response.answer_completeness": "complete", - "response.response_quality": "high", - "response.source_citation": 5, - "response.context_utilization": 0.30479102956167176, - "output_response": "RAG Answer: The provided context does not specifically outline current trends in observability tools beyond the focus on AI applications. However, it highlights the limitations of traditional monitoring tools for...", - "evaluation.retrieval_effectiveness": 1, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 5, - "evaluation.context_relevance": "high", - "evaluation.overall_quality": "high", - "evaluation.ready_for_production": true, - "rag.retrieval_strategy": "semantic_similarity", - "rag.vector_search_results": 5, - "rag.context_synthesis": "multi_source", - "rag.document_coverage": 1, - "rag.information_density": 0.30479102956167176 - }, - "events": [], - "links": [] - } - ], - "created_at": "2025-10-16 08:50:21.645000000", - "updated_at": "2025-10-16 08:50:21.645000000" - }, - { - "trace_id": "c0a95e8e-513c-47eb-b746-4e6016aeaf7d", - "name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "status": "ok", - "status_message": "", - "start_time": "2025-10-16 08:50:08.944000000", - "end_time": "2025-10-16 08:50:20.641000000", - "duration_ms": 11696.768, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "spans": [ - { - "span_id": "2e06953f-5cec-4873-92b5-91f7281f6d7c", - "trace_id": "c0a95e8e-513c-47eb-b746-4e6016aeaf7d", - "parent_span_id": "9e41eab5-cd8e-4e46-a15b-e5f52c9a4073", - "name": "agent.query_routing", - "start_time": "2025-10-16 08:50:08.944000000", - "end_time": "2025-10-16 08:50:20.641000000", - "duration_ms": 11696.524, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "Tell me about recent developments in machine learning", - "query_length": 53, - "query_type": "routing_query", - "output_response": "Routed to Web Search for query processing", - "router.classification": "Web Search", - "router.confidence_score": 1, - "router.rag_keyword_score": 0, - "router.web_keyword_score": 1, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [] - }, - { - "span_id": "9e41eab5-cd8e-4e46-a15b-e5f52c9a4073", - "trace_id": "c0a95e8e-513c-47eb-b746-4e6016aeaf7d", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-16 08:50:08.944000000", - "end_time": "2025-10-16 08:50:20.641000000", - "duration_ms": 11696.671999999999, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "Tell me about recent developments in machine learning", - "query.length": 53, - "processing.mode": "Web Search", - "processing.time_seconds": 11.696574449539185, - "processing.response_length": 2001, - "processing.sources_count": 1, - "output_response": "Final Answer: The web search did not yield specific results regarding recent developments in machine learning. However, I can provide a general overview based on my knowledge up to October 2023.\n\nRecent development...", - "final_answer_mode": "Web Search", - "query_processed.input_query": "Tell me about recent developments in machine learning", - "query_processed.output_response": "Successfully processed query using Web Search, generated 2001 character response", - "query_processed.mode": "Web Search", - "query_processed.processing_time": 11.696574449539185, - "query_processed.response_length": 2001 - }, - "events": [], - "links": [] - }, - { - "span_id": "7ddf938b-183a-427f-849b-b31b18a98d87", - "trace_id": "c0a95e8e-513c-47eb-b746-4e6016aeaf7d", - "parent_span_id": "2e06953f-5cec-4873-92b5-91f7281f6d7c", - "name": "agent.web_search_generation", - "start_time": "2025-10-16 08:50:08.945000000", - "end_time": "2025-10-16 08:50:20.641000000", - "duration_ms": 11696.328, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "web_search_agent", - "agent.operation": "web_search_generation", - "agent.capabilities": [ - "web_search", - "content_synthesis", - "response_generation" - ], - "agent.id": "noveum_web_search_agent", - "input_query": "Tell me about recent developments in machine learning", - "query_length": 53, - "query_type": "web_search_query", - "output_response": "Web Search Answer: The web search did not yield specific results regarding recent developments in machine learning. However, I can provide a general overview based on my knowledge up to October 2023.\n\nRecent development...", - "web_search.results_count": 1, - "web_search.context_length": 155, - "web_search.response_length": 2001, - "web_search.mode": "external_web_search" - }, - "events": [], - "links": [] - }, - { - "span_id": "f3e4c594-f822-4f04-ab43-98fac161b9a3", - "trace_id": "c0a95e8e-513c-47eb-b746-4e6016aeaf7d", - "parent_span_id": "2e06953f-5cec-4873-92b5-91f7281f6d7c", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-16 08:50:08.945000000", - "end_time": "2025-10-16 08:50:08.945000000", - "duration_ms": 0.046, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: Tell me about recent developments in machine learning", - "query_length": 53, - "query_type": "routing_evaluation_query", - "classification.mode": "Web Search", - "classification.rag_keyword_score": 0, - "classification.web_keyword_score": 1, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "complex", - "query.intent": "general_knowledge", - "query.keyword_density": 0.125, - "query.domain_affinity": "general", - "routing.decision": "Routed to Web Search based on analysis", - "routing.rationale": "RAG score: 0, Web score: 1, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "RAG", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.019230769230769232, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: Web Search (Confidence: 1.00)" - }, - "events": [], - "links": [] - }, - { - "span_id": "b47d09be-a43b-4b0a-b7d3-d98bdf1773a3", - "trace_id": "c0a95e8e-513c-47eb-b746-4e6016aeaf7d", - "parent_span_id": "7ddf938b-183a-427f-849b-b31b18a98d87", - "name": "agent.llm_model_execution", - "start_time": "2025-10-16 08:50:09.490000000", - "end_time": "2025-10-16 08:50:20.641000000", - "duration_ms": 11151.141000000001, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: Tell me about recent developments in machine learning", - "query_length": 53, - "query_type": "web_search_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 11.151098012924194, - "model.latency_ms": 11151.098012924194, - "model.start_time": 1760604609.4902723, - "model.end_time": 1760604620.6413703, - "model.performance_tier": "slow", - "model.prompt_tokens": 163, - "model.completion_tokens": 500, - "model.total_tokens": 663, - "model.tokens_per_second": 59.456028386763236, - "model.estimated_cost": 0.01989, - "model.efficiency_score": 3.018099547511312, - "model.response_length": 2001, - "model.response_quality": "high", - "model.output_response": "Model Response: The web search did not yield specific results regarding recent developments in machine learning. However, I can provide a general overview based on my knowledge up to October 2023.\n\nRecent development...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [] - }, - { - "span_id": "bfea8500-0e5c-4556-a014-5b2ec49b7ad5", - "trace_id": "c0a95e8e-513c-47eb-b746-4e6016aeaf7d", - "parent_span_id": "7ddf938b-183a-427f-849b-b31b18a98d87", - "name": "agent.web_search_evaluation_metrics", - "start_time": "2025-10-16 08:50:20.641000000", - "end_time": "2025-10-16 08:50:20.641000000", - "duration_ms": 0.018000000000000002, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "web_search_evaluation_metrics", - "agent.capabilities": [ - "web_search_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: Tell me about recent developments in machine learning", - "query_length": 53, - "query_type": "web_search_evaluation_query", - "web_search.results_count": 1, - "web_search.context_length": 155, - "web_search.context_synthesized": "Context from 1 web sources", - "web_search.search_effectiveness": 0.2, - "web_search.context_quality": "low", - "web_search.source_diversity": 1, - "web_search.information_synthesis": "low", - "web_search.external_knowledge_utilization": 0.155, - "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=Tell+me+about+recent+developments+in+machine+learning\n\nUser Question: Tell me about recent developments in machine learning\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", - "prompt.prompt_length": 653, - "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=Tell+me+about+recent+developments+in+machine+learning", - "prompt.instruction_following": "web_search_optimized", - "response.answer_length": 2001, - "response.answer_completeness": "complete", - "response.response_quality": "medium", - "response.source_citation": 1, - "response.context_utilization": 12.90967741935484, - "output_response": "Web Search Answer: The web search did not yield specific results regarding recent developments in machine learning. However, I can provide a general overview based on my knowledge up to October 2023.\n\nRecent development...", - "evaluation.search_effectiveness": 0.2, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 1, - "evaluation.context_relevance": "low", - "evaluation.overall_quality": "low", - "evaluation.ready_for_production": true, - "web_search.search_strategy": "duckduckgo_api", - "web_search.real_time_data": true, - "web_search.external_sources": 1, - "web_search.information_freshness": "current", - "web_search.knowledge_synthesis": "single_source" - }, - "events": [], - "links": [] - } - ], - "created_at": "2025-10-16 08:50:08.944000000", - "updated_at": "2025-10-16 08:50:08.944000000" - }, - { - "trace_id": "3af8c6e8-62d9-4893-92ba-cc265934ef03", - "name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "status": "ok", - "status_message": "", - "start_time": "2025-10-16 08:50:04.544000000", - "end_time": "2025-10-16 08:50:07.941000000", - "duration_ms": 3397.228, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "spans": [ - { - "span_id": "33297286-d440-446a-ba73-7cabd9d4b2e6", - "trace_id": "3af8c6e8-62d9-4893-92ba-cc265934ef03", - "parent_span_id": "4a3c230e-0dd5-4ba2-8e87-2a301f654fba", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-16 08:50:04.544000000", - "end_time": "2025-10-16 08:50:04.544000000", - "duration_ms": 0.038, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What's the weather like today?", - "query_length": 30, - "query_type": "routing_evaluation_query", - "classification.mode": "Web Search", - "classification.rag_keyword_score": 0, - "classification.web_keyword_score": 2, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "general_knowledge", - "query.keyword_density": 0.4, - "query.domain_affinity": "general", - "routing.decision": "Routed to Web Search based on analysis", - "routing.rationale": "RAG score: 0, Web score: 2, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "RAG", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.038461538461538464, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: Web Search (Confidence: 1.00)" - }, - "events": [], - "links": [] - }, - { - "span_id": "4a3c230e-0dd5-4ba2-8e87-2a301f654fba", - "trace_id": "3af8c6e8-62d9-4893-92ba-cc265934ef03", - "parent_span_id": "eefcbdc5-99fc-4f06-a9f4-f51ba50564db", - "name": "agent.query_routing", - "start_time": "2025-10-16 08:50:04.544000000", - "end_time": "2025-10-16 08:50:07.941000000", - "duration_ms": 3397.056, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What's the weather like today?", - "query_length": 30, - "query_type": "routing_query", - "output_response": "Routed to Web Search for query processing", - "router.classification": "Web Search", - "router.confidence_score": 1, - "router.rag_keyword_score": 0, - "router.web_keyword_score": 2, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [] - }, - { - "span_id": "eefcbdc5-99fc-4f06-a9f4-f51ba50564db", - "trace_id": "3af8c6e8-62d9-4893-92ba-cc265934ef03", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-16 08:50:04.544000000", - "end_time": "2025-10-16 08:50:07.941000000", - "duration_ms": 3397.169, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What's the weather like today?", - "query.length": 30, - "processing.mode": "Web Search", - "processing.time_seconds": 3.3970956802368164, - "processing.response_length": 518, - "processing.sources_count": 1, - "output_response": "Final Answer: The web search results do not provide specific details about today's weather conditions in your location. However, they mention that you can get real-time updates, including temperature, humidity, and...", - "final_answer_mode": "Web Search", - "query_processed.input_query": "What's the weather like today?", - "query_processed.output_response": "Successfully processed query using Web Search, generated 518 character response", - "query_processed.mode": "Web Search", - "query_processed.processing_time": 3.3970956802368164, - "query_processed.response_length": 518 - }, - "events": [], - "links": [] - }, - { - "span_id": "9683465c-e851-4004-a858-c8fdd1a2b3da", - "trace_id": "3af8c6e8-62d9-4893-92ba-cc265934ef03", - "parent_span_id": "4a3c230e-0dd5-4ba2-8e87-2a301f654fba", - "name": "agent.web_search_generation", - "start_time": "2025-10-16 08:50:04.545000000", - "end_time": "2025-10-16 08:50:07.941000000", - "duration_ms": 3396.871, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "web_search_agent", - "agent.operation": "web_search_generation", - "agent.capabilities": [ - "web_search", - "content_synthesis", - "response_generation" - ], - "agent.id": "noveum_web_search_agent", - "input_query": "What's the weather like today?", - "query_length": 30, - "query_type": "web_search_query", - "output_response": "Web Search Answer: The web search results do not provide specific details about today's weather conditions in your location. However, they mention that you can get real-time updates, including temperature, humidity, and...", - "web_search.results_count": 1, - "web_search.context_length": 1137, - "web_search.response_length": 518, - "web_search.mode": "external_web_search" - }, - "events": [], - "links": [] - }, - { - "span_id": "6413eb7e-e5cd-4c46-9651-de14c72c260e", - "trace_id": "3af8c6e8-62d9-4893-92ba-cc265934ef03", - "parent_span_id": "9683465c-e851-4004-a858-c8fdd1a2b3da", - "name": "agent.llm_model_execution", - "start_time": "2025-10-16 08:50:04.986000000", - "end_time": "2025-10-16 08:50:07.941000000", - "duration_ms": 2955.55, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What's the weather like today?", - "query_length": 30, - "query_type": "web_search_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 2.955509662628174, - "model.latency_ms": 2955.509662628174, - "model.start_time": 1760604604.9862766, - "model.end_time": 1760604607.9417863, - "model.performance_tier": "medium", - "model.prompt_tokens": 403, - "model.completion_tokens": 129, - "model.total_tokens": 532, - "model.tokens_per_second": 180.0027950261957, - "model.estimated_cost": 0.015960000000000002, - "model.efficiency_score": 0.9736842105263158, - "model.response_length": 518, - "model.response_quality": "high", - "model.output_response": "Model Response: The web search results do not provide specific details about today's weather conditions in your location. However, they mention that you can get real-time updates, including temperature, humidity, and...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [] - }, - { - "span_id": "64df69fb-0bcc-4845-8bfe-5a721c1ed912", - "trace_id": "3af8c6e8-62d9-4893-92ba-cc265934ef03", - "parent_span_id": "9683465c-e851-4004-a858-c8fdd1a2b3da", - "name": "agent.web_search_evaluation_metrics", - "start_time": "2025-10-16 08:50:07.941000000", - "end_time": "2025-10-16 08:50:07.941000000", - "duration_ms": 0.017, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "web_search_evaluation_metrics", - "agent.capabilities": [ - "web_search_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What's the weather like today?", - "query_length": 30, - "query_type": "web_search_evaluation_query", - "web_search.results_count": 1, - "web_search.context_length": 1137, - "web_search.context_synthesized": "Context from 1 web sources", - "web_search.search_effectiveness": 0.2, - "web_search.context_quality": "high", - "web_search.source_diversity": 1, - "web_search.information_synthesis": "low", - "web_search.external_knowledge_utilization": 1.137, - "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nStay updated with the latest weather conditions in Komna. Get real-time updates, temperature, humidity, AQI, and detailed forecast for today. Dec 31, 2024 \u00b7 Komna Weather Forecast: Check the latest temperature in Komna today along with details such as humidity, visibility, air quality, wind speed, pressure, and weekly weather report at Shuru. 2 days ago \u00b7 Latest weather forecast for India for today's, hourly weather forecast, including today's temperatures in India, wind, rain and more. 2 days ago \u00b7 Get the latest hourly weather updates for India today. Detailed forecast including temperature, wind, rain, snow, and UV index. Stay informed about today's weather conditions in India. 2 days ago \u00b7 Today's Weather Report ... Forecasts Urban Meteorological Services Short to Medium Range Model Guidance Extended Range Model Guidance Seasonal Forecast Quantitative Precipitation Forecast Latest CAP Alerts Specialised products & forecasts Cyclones | Monsoon | Satellite Radar | Rainfall | Agromet Advisories MAUSAM : | | Video Meghdoot Agro ...\nURL: https://duckduckgo.com/?q=What's+the+weather+like+today?\n\nUser Question: What's the weather like today?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", - "prompt.prompt_length": 1612, - "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nStay updated with the latest weather conditions in Komna. Get real-time updates, temperature, humidity, AQI, and detailed forecast for today. Dec 31, 2024 \u00b7 Komna Weather F...", - "prompt.instruction_following": "web_search_optimized", - "response.answer_length": 518, - "response.answer_completeness": "complete", - "response.response_quality": "medium", - "response.source_citation": 1, - "response.context_utilization": 0.455584872471416, - "output_response": "Web Search Answer: The web search results do not provide specific details about today's weather conditions in your location. However, they mention that you can get real-time updates, including temperature, humidity, and...", - "evaluation.search_effectiveness": 0.2, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 1, - "evaluation.context_relevance": "high", - "evaluation.overall_quality": "low", - "evaluation.ready_for_production": true, - "web_search.search_strategy": "duckduckgo_api", - "web_search.real_time_data": true, - "web_search.external_sources": 1, - "web_search.information_freshness": "current", - "web_search.knowledge_synthesis": "single_source" - }, - "events": [], - "links": [] - } - ], - "created_at": "2025-10-16 08:50:04.544000000", - "updated_at": "2025-10-16 08:50:04.544000000" - }, - { - "trace_id": "d7b8b811-ad01-48f1-917f-ace7e8d289e6", - "name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "status": "ok", - "status_message": "", - "start_time": "2025-10-16 08:50:01.375000000", - "end_time": "2025-10-16 08:50:03.540000000", - "duration_ms": 2164.576, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "spans": [ - { - "span_id": "719a1a8f-fc50-439c-899b-436146accd23", - "trace_id": "d7b8b811-ad01-48f1-917f-ace7e8d289e6", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-16 08:50:01.375000000", - "end_time": "2025-10-16 08:50:03.540000000", - "duration_ms": 2164.4809999999998, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What are the latest AI news today?", - "query.length": 34, - "processing.mode": "Web Search", - "processing.time_seconds": 2.1643810272216797, - "processing.response_length": 298, - "processing.sources_count": 1, - "output_response": "Final Answer: The web search did not yield any specific results regarding the latest AI news today. Unfortunately, I cannot provide current updates or news articles on AI developments at this moment. For the latest...", - "final_answer_mode": "Web Search", - "query_processed.input_query": "What are the latest AI news today?", - "query_processed.output_response": "Successfully processed query using Web Search, generated 298 character response", - "query_processed.mode": "Web Search", - "query_processed.processing_time": 2.1643810272216797, - "query_processed.response_length": 298 - }, - "events": [], - "links": [] - }, - { - "span_id": "644a712f-1c9e-4e9a-9073-74621b754aa2", - "trace_id": "d7b8b811-ad01-48f1-917f-ace7e8d289e6", - "parent_span_id": "719a1a8f-fc50-439c-899b-436146accd23", - "name": "agent.query_routing", - "start_time": "2025-10-16 08:50:01.376000000", - "end_time": "2025-10-16 08:50:03.540000000", - "duration_ms": 2164.3219999999997, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What are the latest AI news today?", - "query_length": 34, - "query_type": "routing_query", - "output_response": "Routed to Web Search for query processing", - "router.classification": "Web Search", - "router.confidence_score": 1, - "router.rag_keyword_score": 0, - "router.web_keyword_score": 3, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [] - }, - { - "span_id": "8a4abdbe-6ad7-4d43-b234-81667c4abbc2", - "trace_id": "d7b8b811-ad01-48f1-917f-ace7e8d289e6", - "parent_span_id": "644a712f-1c9e-4e9a-9073-74621b754aa2", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-16 08:50:01.376000000", - "end_time": "2025-10-16 08:50:01.376000000", - "duration_ms": 0.051, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What are the latest AI news today?", - "query_length": 34, - "query_type": "routing_evaluation_query", - "classification.mode": "Web Search", - "classification.rag_keyword_score": 0, - "classification.web_keyword_score": 3, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "general_knowledge", - "query.keyword_density": 0.42857142857142855, - "query.domain_affinity": "general", - "routing.decision": "Routed to Web Search based on analysis", - "routing.rationale": "RAG score: 0, Web score: 3, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "RAG", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.057692307692307696, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: Web Search (Confidence: 1.00)" - }, - "events": [], - "links": [] - }, - { - "span_id": "904621e8-c5fa-4b2d-a628-8f10eb9da51d", - "trace_id": "d7b8b811-ad01-48f1-917f-ace7e8d289e6", - "parent_span_id": "644a712f-1c9e-4e9a-9073-74621b754aa2", - "name": "agent.web_search_generation", - "start_time": "2025-10-16 08:50:01.376000000", - "end_time": "2025-10-16 08:50:03.540000000", - "duration_ms": 2164.076, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "web_search_agent", - "agent.operation": "web_search_generation", - "agent.capabilities": [ - "web_search", - "content_synthesis", - "response_generation" - ], - "agent.id": "noveum_web_search_agent", - "input_query": "What are the latest AI news today?", - "query_length": 34, - "query_type": "web_search_query", - "output_response": "Web Search Answer: The web search did not yield any specific results regarding the latest AI news today. Unfortunately, I cannot provide current updates or news articles on AI developments at this moment. For the latest...", - "web_search.results_count": 1, - "web_search.context_length": 136, - "web_search.response_length": 298, - "web_search.mode": "external_web_search" - }, - "events": [], - "links": [] - }, - { - "span_id": "421dd1af-2dbc-477f-bd59-b3194b14ae3a", - "trace_id": "d7b8b811-ad01-48f1-917f-ace7e8d289e6", - "parent_span_id": "904621e8-c5fa-4b2d-a628-8f10eb9da51d", - "name": "agent.llm_model_execution", - "start_time": "2025-10-16 08:50:01.708000000", - "end_time": "2025-10-16 08:50:03.540000000", - "duration_ms": 1831.237, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What are the latest AI news today?", - "query_length": 34, - "query_type": "web_search_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 1.8311803340911865, - "model.latency_ms": 1831.1803340911865, - "model.start_time": 1760604601.7090185, - "model.end_time": 1760604603.5401988, - "model.performance_tier": "fast", - "model.prompt_tokens": 153, - "model.completion_tokens": 74, - "model.total_tokens": 227, - "model.tokens_per_second": 123.96376029926071, - "model.estimated_cost": 0.00681, - "model.efficiency_score": 1.3127753303964758, - "model.response_length": 298, - "model.response_quality": "high", - "model.output_response": "Model Response: The web search did not yield any specific results regarding the latest AI news today. Unfortunately, I cannot provide current updates or news articles on AI developments at this moment. For the latest...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [] - }, - { - "span_id": "159aad6f-7174-406d-b7d9-eed9360d993c", - "trace_id": "d7b8b811-ad01-48f1-917f-ace7e8d289e6", - "parent_span_id": "904621e8-c5fa-4b2d-a628-8f10eb9da51d", - "name": "agent.web_search_evaluation_metrics", - "start_time": "2025-10-16 08:50:03.540000000", - "end_time": "2025-10-16 08:50:03.540000000", - "duration_ms": 0.028, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "web_search_evaluation_metrics", - "agent.capabilities": [ - "web_search_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What are the latest AI news today?", - "query_length": 34, - "query_type": "web_search_evaluation_query", - "web_search.results_count": 1, - "web_search.context_length": 136, - "web_search.context_synthesized": "Context from 1 web sources", - "web_search.search_effectiveness": 0.2, - "web_search.context_quality": "low", - "web_search.source_diversity": 1, - "web_search.information_synthesis": "low", - "web_search.external_knowledge_utilization": 0.136, - "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What+are+the+latest+AI+news+today?\n\nUser Question: What are the latest AI news today?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", - "prompt.prompt_length": 615, - "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What+are+the+latest+AI+news+today?", - "prompt.instruction_following": "web_search_optimized", - "response.answer_length": 298, - "response.answer_completeness": "complete", - "response.response_quality": "medium", - "response.source_citation": 1, - "response.context_utilization": 2.1911764705882355, - "output_response": "Web Search Answer: The web search did not yield any specific results regarding the latest AI news today. Unfortunately, I cannot provide current updates or news articles on AI developments at this moment. For the latest...", - "evaluation.search_effectiveness": 0.2, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 1, - "evaluation.context_relevance": "low", - "evaluation.overall_quality": "low", - "evaluation.ready_for_production": true, - "web_search.search_strategy": "duckduckgo_api", - "web_search.real_time_data": true, - "web_search.external_sources": 1, - "web_search.information_freshness": "current", - "web_search.knowledge_synthesis": "single_source" - }, - "events": [], - "links": [] - } - ], - "created_at": "2025-10-16 08:50:01.375000000", - "updated_at": "2025-10-16 08:50:01.375000000" - }, - { - "trace_id": "df8340b4-b75a-48e9-a2f3-99725b46a00d", - "name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "status": "ok", - "status_message": "", - "start_time": "2025-10-16 08:49:17.236000000", - "end_time": "2025-10-16 08:50:00.368000000", - "duration_ms": 43131.554000000004, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "spans": [ - { - "span_id": "dcf2b38e-ab90-45ee-9a65-e6fea47a3fd7", - "trace_id": "df8340b4-b75a-48e9-a2f3-99725b46a00d", - "parent_span_id": "d0e2b320-c18d-4ec6-98a1-74da5252107e", - "name": "agent.query_routing", - "start_time": "2025-10-16 08:49:17.236000000", - "end_time": "2025-10-16 08:50:00.368000000", - "duration_ms": 43131.384, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What are the benefits of using Noveum Trace?", - "query_length": 44, - "query_type": "routing_query", - "output_response": "Routed to RAG for query processing", - "router.classification": "RAG", - "router.confidence_score": 1, - "router.rag_keyword_score": 2, - "router.web_keyword_score": 0, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [] - }, - { - "span_id": "d0e2b320-c18d-4ec6-98a1-74da5252107e", - "trace_id": "df8340b4-b75a-48e9-a2f3-99725b46a00d", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-16 08:49:17.236000000", - "end_time": "2025-10-16 08:50:00.368000000", - "duration_ms": 43131.481, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What are the benefits of using Noveum Trace?", - "query.length": 44, - "processing.mode": "RAG", - "processing.time_seconds": 43.131428718566895, - "processing.response_length": 1415, - "processing.sources_count": 5, - "output_response": "Final Answer: The benefits of using Noveum Trace include:\n\n1. **Comprehensive Tracing**: Noveum Trace offers decorator-based tracing for LLM calls, agents, and RAG pipelines, allowing for detailed monitoring of AI ...", - "final_answer_mode": "RAG", - "query_processed.input_query": "What are the benefits of using Noveum Trace?", - "query_processed.output_response": "Successfully processed query using RAG, generated 1415 character response", - "query_processed.mode": "RAG", - "query_processed.processing_time": 43.131428718566895, - "query_processed.response_length": 1415 - }, - "events": [], - "links": [] - }, - { - "span_id": "fa93821f-c809-4071-87e1-a6c55014d37c", - "trace_id": "df8340b4-b75a-48e9-a2f3-99725b46a00d", - "parent_span_id": "dcf2b38e-ab90-45ee-9a65-e6fea47a3fd7", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-16 08:49:17.236000000", - "end_time": "2025-10-16 08:49:17.236000000", - "duration_ms": 0.034, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What are the benefits of using Noveum Trace?", - "query_length": 44, - "query_type": "routing_evaluation_query", - "classification.mode": "RAG", - "classification.rag_keyword_score": 2, - "classification.web_keyword_score": 0, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "noveum_specific", - "query.keyword_density": 0.25, - "query.domain_affinity": "noveum", - "routing.decision": "Routed to RAG based on analysis", - "routing.rationale": "RAG score: 2, Web score: 0, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "Web Search", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.038461538461538464, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: RAG (Confidence: 1.00)" - }, - "events": [], - "links": [] - }, - { - "span_id": "024ad5aa-f394-4c26-af51-564f3f51c6f2", - "trace_id": "df8340b4-b75a-48e9-a2f3-99725b46a00d", - "parent_span_id": "dcf2b38e-ab90-45ee-9a65-e6fea47a3fd7", - "name": "agent.llm-rag", - "start_time": "2025-10-16 08:49:17.237000000", - "end_time": "2025-10-16 08:50:00.368000000", - "duration_ms": 43131.19300000001, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "rag_agent", - "agent.operation": "llm-rag", - "agent.capabilities": [ - "document_retrieval", - "context_generation", - "response_generation" - ], - "agent.id": "noveum_rag_agent", - "input_query": "What are the benefits of using Noveum Trace?", - "query_length": 44, - "query_type": "rag_query", - "output_response": "RAG Answer: The benefits of using Noveum Trace include:\n\n1. **Comprehensive Tracing**: Noveum Trace offers decorator-based tracing for LLM calls, agents, and RAG pipelines, allowing for detailed monitoring of AI ...", - "rag.context_length": 2773, - "rag.sources_count": 5, - "rag.answer_length": 1415, - "rag.mode": "retrieval_augmented_generation" - }, - "events": [], - "links": [] - }, - { - "span_id": "3d9db951-d8b0-4990-820d-61e4b1011709", - "trace_id": "df8340b4-b75a-48e9-a2f3-99725b46a00d", - "parent_span_id": "024ad5aa-f394-4c26-af51-564f3f51c6f2", - "name": "agent.llm_model_execution", - "start_time": "2025-10-16 08:49:20.224000000", - "end_time": "2025-10-16 08:49:58.420000000", - "duration_ms": 38196.111000000004, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What are the benefits of using Noveum Trace?", - "query_length": 44, - "query_type": "rag_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 38.19604015350342, - "model.latency_ms": 38196.04015350342, - "model.start_time": 1760604560.2242246, - "model.end_time": 1760604598.4202647, - "model.performance_tier": "slow", - "model.prompt_tokens": 828, - "model.completion_tokens": 353, - "model.total_tokens": 1181, - "model.tokens_per_second": 30.919435503098253, - "model.estimated_cost": 0.03543, - "model.efficiency_score": 1.1981371718882303, - "model.response_length": 1415, - "model.response_quality": "high", - "model.output_response": "Model Response: The benefits of using Noveum Trace include:\n\n1. **Comprehensive Tracing**: Noveum Trace offers decorator-based tracing for LLM calls, agents, and RAG pipelines, allowing for detailed monitoring of AI ...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [] - }, - { - "span_id": "8c33ffac-4d76-42ba-9737-89596e136b3d", - "trace_id": "df8340b4-b75a-48e9-a2f3-99725b46a00d", - "parent_span_id": "024ad5aa-f394-4c26-af51-564f3f51c6f2", - "name": "agent.rag_evaluation_metrics", - "start_time": "2025-10-16 08:49:58.420000000", - "end_time": "2025-10-16 08:49:58.420000000", - "duration_ms": 0.037, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "rag_evaluation_metrics", - "agent.capabilities": [ - "retrieval_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What are the benefits of using Noveum Trace?", - "query_length": 44, - "query_type": "rag_evaluation_query", - "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/docs/platform/dashboard):\n- Bulk Export: Download multiple traces simultaneously\n- Comparative Analysis: Compare performance across multiple traces\nReal-Time Updates\n- Live Refresh: Automatic updates as new traces arrive\n- Connection Monitoring: Real-time status of your t...", - "retrieval.context_length": 2773, - "retrieval.sources_count": 5, - "retrieval.context_quality": "high", - "retrieval.effectiveness": 1, - "retrieval.context_utilization": 2.773, - "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/docs/platform/dashboard):\n- Bulk Export: Download multiple traces simultaneously\n- Comparative Analysis: Compare performance across multiple traces\nReal-Time Updates\n- Live Refresh: Automatic updates as new traces arrive\n- Connection Monitoring: Real-time status of your trace ingestion pipeline\n- Performance Indicators: Live metrics for system health monitoring\nReady to dive deeper? Explore Projects & Environments to organize your AI applications, or check out Team Collaboration to share insights with your team.\nGet Earl...\n\nSource 2 (https://noveum.ai/en/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\n\ud83d\udcd8 TypeScript SDK (\n@noveum/trace\n)- Framework integrations for Next.js, Express.js, and Hono\n- TypeScript-first design with full type safety\n- Universal compatibility (Node.js, Edge Runtime, browsers)\n-\n\ud83d\udcca Noveum Platform\n- Real-time dashboard for analyzing traces and performance\n- Ad...\n\nSource 3 (https://noveum.ai/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\n\ud83d\udcd8 TypeScript SDK (\n@noveum/trace\n)- Framework integrations for Next.js, Express.js, and Hono\n- TypeScript-first design with full type safety\n- Universal compatibility (Node.js, Edge Runtime, browsers)\n-\n\ud83d\udcca Noveum Platform\n- Real-time dashboard for analyzing traces and performance\n- Ad...\n\nSource 4 (https://noveum.ai/en/docs/getting-started/overview):\n- Advanced search & filtering across traces and spans\n- Cost analysis and optimization recommendations\n- Team collaboration with shared insights and alerts\n\ud83d\udd0d What Noveum Traces\nLLM Operations\n- Model calls across all providers (OpenAI, Anthropic, Google, etc.)\n- Token usage and cost calculation\n- Prompt engineering effectiveness\n- Response quality metrics\nRAG Pipelines\n- Document retrieval performance and relevance\n- Embedding generation costs and latency\n- Context assembly and prompt constructi...\n\nSource 5 (https://noveum.ai/docs/getting-started/overview):\n- Advanced search & filtering across traces and spans\n- Cost analysis and optimization recommendations\n- Team collaboration with shared insights and alerts\n\ud83d\udd0d What Noveum Traces\nLLM Operations\n- Model calls across all providers (OpenAI, Anthropic, Google, etc.)\n- Token usage and cost calculation\n- Prompt engineering effectiveness\n- Response quality metrics\nRAG Pipelines\n- Document retrieval performance and relevance\n- Embedding generation costs and latency\n- Context assembly and prompt constructi...\n\nUser Question: What are the benefits of using Noveum Trace?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", - "prompt.prompt_length": 3313, - "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/docs/platform/dashboard):\n- Bulk Export: Download multiple traces simultaneously\n- Comparative Analysis: Compare performance across multiple traces\nReal-Time Updates\n- Live...", - "prompt.instruction_following": "rag_optimized", - "response.answer_length": 1415, - "response.answer_completeness": "complete", - "response.response_quality": "high", - "response.source_citation": 5, - "response.context_utilization": 0.5102776776054815, - "output_response": "RAG Answer: The benefits of using Noveum Trace include:\n\n1. **Comprehensive Tracing**: Noveum Trace offers decorator-based tracing for LLM calls, agents, and RAG pipelines, allowing for detailed monitoring of AI ...", - "evaluation.retrieval_effectiveness": 1, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 5, - "evaluation.context_relevance": "high", - "evaluation.overall_quality": "high", - "evaluation.ready_for_production": true, - "rag.retrieval_strategy": "semantic_similarity", - "rag.vector_search_results": 5, - "rag.context_synthesis": "multi_source", - "rag.document_coverage": 1, - "rag.information_density": 0.5102776776054815 - }, - "events": [], - "links": [] - } - ], - "created_at": "2025-10-16 08:49:17.236000000", - "updated_at": "2025-10-16 08:49:17.236000000" - }, - { - "trace_id": "325ec189-2876-4ebe-b33c-649491a5053e", - "name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "status": "ok", - "status_message": "", - "start_time": "2025-10-16 08:49:11.192000000", - "end_time": "2025-10-16 08:49:16.232000000", - "duration_ms": 5039.841, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "spans": [ - { - "span_id": "5b1a7121-d0bd-4525-821d-7721c6fbbd70", - "trace_id": "325ec189-2876-4ebe-b33c-649491a5053e", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-16 08:49:11.192000000", - "end_time": "2025-10-16 08:49:16.232000000", - "duration_ms": 5039.753, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "How do I configure Noveum for my system?", - "query.length": 40, - "processing.mode": "RAG", - "processing.time_seconds": 5.039698600769043, - "processing.response_length": 712, - "processing.sources_count": 5, - "output_response": "Final Answer: To configure Noveum for your system, you can refer to the SDK Integration Guide available in the Noveum documentation. This guide will help you trace your AI applications with minimal code changes. Th...", - "final_answer_mode": "RAG", - "query_processed.input_query": "How do I configure Noveum for my system?", - "query_processed.output_response": "Successfully processed query using RAG, generated 712 character response", - "query_processed.mode": "RAG", - "query_processed.processing_time": 5.039698600769043, - "query_processed.response_length": 712 - }, - "events": [], - "links": [] - }, - { - "span_id": "761c92c6-d15a-4a8e-a16a-a365922f8d7c", - "trace_id": "325ec189-2876-4ebe-b33c-649491a5053e", - "parent_span_id": "5b1a7121-d0bd-4525-821d-7721c6fbbd70", - "name": "agent.query_routing", - "start_time": "2025-10-16 08:49:11.192000000", - "end_time": "2025-10-16 08:49:16.232000000", - "duration_ms": 5039.642, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "How do I configure Noveum for my system?", - "query_length": 40, - "query_type": "routing_query", - "output_response": "Routed to RAG for query processing", - "router.classification": "RAG", - "router.confidence_score": 1, - "router.rag_keyword_score": 2, - "router.web_keyword_score": 0, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [] - }, - { - "span_id": "8fcf969e-a577-45aa-9fe6-1870d9665189", - "trace_id": "325ec189-2876-4ebe-b33c-649491a5053e", - "parent_span_id": "761c92c6-d15a-4a8e-a16a-a365922f8d7c", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-16 08:49:11.192000000", - "end_time": "2025-10-16 08:49:11.192000000", - "duration_ms": 0.045000000000000005, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: How do I configure Noveum for my system?", - "query_length": 40, - "query_type": "routing_evaluation_query", - "classification.mode": "RAG", - "classification.rag_keyword_score": 2, - "classification.web_keyword_score": 0, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "noveum_specific", - "query.keyword_density": 0.25, - "query.domain_affinity": "noveum", - "routing.decision": "Routed to RAG based on analysis", - "routing.rationale": "RAG score: 2, Web score: 0, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "Web Search", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.038461538461538464, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: RAG (Confidence: 1.00)" - }, - "events": [], - "links": [] - }, - { - "span_id": "9e5bbea4-cc68-4fe6-b3d0-c87852635ec9", - "trace_id": "325ec189-2876-4ebe-b33c-649491a5053e", - "parent_span_id": "761c92c6-d15a-4a8e-a16a-a365922f8d7c", - "name": "agent.llm-rag", - "start_time": "2025-10-16 08:49:11.192000000", - "end_time": "2025-10-16 08:49:16.232000000", - "duration_ms": 5039.39, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "rag_agent", - "agent.operation": "llm-rag", - "agent.capabilities": [ - "document_retrieval", - "context_generation", - "response_generation" - ], - "agent.id": "noveum_rag_agent", - "input_query": "How do I configure Noveum for my system?", - "query_length": 40, - "query_type": "rag_query", - "output_response": "RAG Answer: To configure Noveum for your system, you can refer to the SDK Integration Guide available in the Noveum documentation. This guide will help you trace your AI applications with minimal code changes. Th...", - "rag.context_length": 2237, - "rag.sources_count": 5, - "rag.answer_length": 712, - "rag.mode": "retrieval_augmented_generation" - }, - "events": [], - "links": [] - }, - { - "span_id": "5ee706ad-c4ae-4c0b-94d1-6b9c43d173b4", - "trace_id": "325ec189-2876-4ebe-b33c-649491a5053e", - "parent_span_id": "9e5bbea4-cc68-4fe6-b3d0-c87852635ec9", - "name": "agent.llm_model_execution", - "start_time": "2025-10-16 08:49:11.607000000", - "end_time": "2025-10-16 08:49:15.718000000", - "duration_ms": 4110.915, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: How do I configure Noveum for my system?", - "query_length": 40, - "query_type": "rag_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 4.110846042633057, - "model.latency_ms": 4110.846042633057, - "model.start_time": 1760604551.6074958, - "model.end_time": 1760604555.7183418, - "model.performance_tier": "medium", - "model.prompt_tokens": 693, - "model.completion_tokens": 178, - "model.total_tokens": 871, - "model.tokens_per_second": 211.87852596934326, - "model.estimated_cost": 0.02613, - "model.efficiency_score": 0.817451205510907, - "model.response_length": 712, - "model.response_quality": "high", - "model.output_response": "Model Response: To configure Noveum for your system, you can refer to the SDK Integration Guide available in the Noveum documentation. This guide will help you trace your AI applications with minimal code changes. Th...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [] - }, - { - "span_id": "d13bdeda-81af-400a-82ca-a5ef7e1e8a6d", - "trace_id": "325ec189-2876-4ebe-b33c-649491a5053e", - "parent_span_id": "9e5bbea4-cc68-4fe6-b3d0-c87852635ec9", - "name": "agent.rag_evaluation_metrics", - "start_time": "2025-10-16 08:49:15.718000000", - "end_time": "2025-10-16 08:49:15.718000000", - "duration_ms": 0.118, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "rag_evaluation_metrics", - "agent.capabilities": [ - "retrieval_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: How do I configure Noveum for my system?", - "query_length": 40, - "query_type": "rag_evaluation_query", - "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/docs):\nWelcome to Noveum.ai Docs\nComprehensive AI tracing and observability for LLM applications, RAG systems, and AI agents\nWelcome to the Noveum.ai documentation! Here you'll find everything you need to integrate, configure, and optimize your AI applications using ou...", - "retrieval.context_length": 2237, - "retrieval.sources_count": 5, - "retrieval.context_quality": "high", - "retrieval.effectiveness": 1, - "retrieval.context_utilization": 2.237, - "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/docs):\nWelcome to Noveum.ai Docs\nComprehensive AI tracing and observability for LLM applications, RAG systems, and AI agents\nWelcome to the Noveum.ai documentation! Here you'll find everything you need to integrate, configure, and optimize your AI applications using our comprehensive tracing and observability platform.\nWe provide powerful SDKs for Python and TypeScript that enable you to trace LLM calls, RAG pipelines, and multi-agent workflows with minimal code changes. Our documentation is organized ...\n\nSource 2 (https://noveum.ai/docs):\nWelcome to Noveum.ai Docs\nComprehensive AI tracing and observability for LLM applications, RAG systems, and AI agents\nWelcome to the Noveum.ai documentation! Here you'll find everything you need to integrate, configure, and optimize your AI applications using our comprehensive tracing and observability platform.\nWe provide powerful SDKs for Python and TypeScript that enable you to trace LLM calls, RAG pipelines, and multi-agent workflows with minimal code changes. Our documentation is organized ...\n\nSource 3 (https://noveum.ai/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nSource 4 (https://noveum.ai/en/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nSource 5 (https://noveum.ai/en/docs):\n- \ud83c\udf0d Universal: Works across frameworks, clouds, and deployment models\n\ud83e\udd1d Community & Support\n- \ud83d\udcac Discord Community: Join our Discord\n- \ud83d\udce7 Email Support: [email protected]\n- \ud83d\udc1b Bug Reports: GitHub Issues\n- \ud83d\udcd6 Knowledge Base: Help Center\nReady to get started? Head to our SDK Integration Guide to begin tracing your AI applications in under 5 minutes!\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability ...\n\nUser Question: How do I configure Noveum for my system?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", - "prompt.prompt_length": 2773, - "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/docs):\nWelcome to Noveum.ai Docs\nComprehensive AI tracing and observability for LLM applications, RAG systems, and AI agents\nWelcome to the Noveum.ai documentation! Here...", - "prompt.instruction_following": "rag_optimized", - "response.answer_length": 712, - "response.answer_completeness": "complete", - "response.response_quality": "high", - "response.source_citation": 5, - "response.context_utilization": 0.3182834152883326, - "output_response": "RAG Answer: To configure Noveum for your system, you can refer to the SDK Integration Guide available in the Noveum documentation. This guide will help you trace your AI applications with minimal code changes. Th...", - "evaluation.retrieval_effectiveness": 1, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 5, - "evaluation.context_relevance": "high", - "evaluation.overall_quality": "high", - "evaluation.ready_for_production": true, - "rag.retrieval_strategy": "semantic_similarity", - "rag.vector_search_results": 5, - "rag.context_synthesis": "multi_source", - "rag.document_coverage": 1, - "rag.information_density": 0.3182834152883326 - }, - "events": [], - "links": [] - } - ], - "created_at": "2025-10-16 08:49:11.192000000", - "updated_at": "2025-10-16 08:49:11.192000000" - }, - { - "trace_id": "d5528586-c741-4703-8791-5fccf5c3e123", - "name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "status": "ok", - "status_message": "", - "start_time": "2025-10-16 08:49:05.152000000", - "end_time": "2025-10-16 08:49:10.187000000", - "duration_ms": 5034.604, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "spans": [ - { - "span_id": "7d6564fc-41b2-4d74-acb1-e77b575c0006", - "trace_id": "d5528586-c741-4703-8791-5fccf5c3e123", - "parent_span_id": "e79a037e-82b1-41e9-bdc7-42353e98ead7", - "name": "agent.query_routing", - "start_time": "2025-10-16 08:49:05.152000000", - "end_time": "2025-10-16 08:49:10.187000000", - "duration_ms": 5034.446, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What monitoring capabilities does Noveum provide?", - "query_length": 49, - "query_type": "routing_query", - "output_response": "Routed to RAG for query processing", - "router.classification": "RAG", - "router.confidence_score": 1, - "router.rag_keyword_score": 2, - "router.web_keyword_score": 0, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [] - }, - { - "span_id": "e79a037e-82b1-41e9-bdc7-42353e98ead7", - "trace_id": "d5528586-c741-4703-8791-5fccf5c3e123", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-16 08:49:05.152000000", - "end_time": "2025-10-16 08:49:10.187000000", - "duration_ms": 5034.531, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What monitoring capabilities does Noveum provide?", - "query.length": 49, - "processing.mode": "RAG", - "processing.time_seconds": 5.0344908237457275, - "processing.response_length": 552, - "processing.sources_count": 5, - "output_response": "Final Answer: Noveum provides comprehensive monitoring capabilities for AI applications, including the ability to monitor, trace, and optimize AI agents across various frameworks such as LangChain, CrewAI, AutoGen,...", - "final_answer_mode": "RAG", - "query_processed.input_query": "What monitoring capabilities does Noveum provide?", - "query_processed.output_response": "Successfully processed query using RAG, generated 552 character response", - "query_processed.mode": "RAG", - "query_processed.processing_time": 5.0344908237457275, - "query_processed.response_length": 552 - }, - "events": [], - "links": [] - }, - { - "span_id": "78297793-d73b-4311-b5ba-73509ae9f96e", - "trace_id": "d5528586-c741-4703-8791-5fccf5c3e123", - "parent_span_id": "7d6564fc-41b2-4d74-acb1-e77b575c0006", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-16 08:49:05.153000000", - "end_time": "2025-10-16 08:49:05.153000000", - "duration_ms": 0.029, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What monitoring capabilities does Noveum provide?", - "query_length": 49, - "query_type": "routing_evaluation_query", - "classification.mode": "RAG", - "classification.rag_keyword_score": 2, - "classification.web_keyword_score": 0, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "noveum_specific", - "query.keyword_density": 0.3333333333333333, - "query.domain_affinity": "noveum", - "routing.decision": "Routed to RAG based on analysis", - "routing.rationale": "RAG score: 2, Web score: 0, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "Web Search", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.038461538461538464, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: RAG (Confidence: 1.00)" - }, - "events": [], - "links": [] - }, - { - "span_id": "6a101d9d-de83-415a-8fb0-01a9fae5ffb9", - "trace_id": "d5528586-c741-4703-8791-5fccf5c3e123", - "parent_span_id": "7d6564fc-41b2-4d74-acb1-e77b575c0006", - "name": "agent.llm-rag", - "start_time": "2025-10-16 08:49:05.153000000", - "end_time": "2025-10-16 08:49:10.187000000", - "duration_ms": 5034.282, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "rag_agent", - "agent.operation": "llm-rag", - "agent.capabilities": [ - "document_retrieval", - "context_generation", - "response_generation" - ], - "agent.id": "noveum_rag_agent", - "input_query": "What monitoring capabilities does Noveum provide?", - "query_length": 49, - "query_type": "rag_query", - "output_response": "RAG Answer: Noveum provides comprehensive monitoring capabilities for AI applications, including the ability to monitor, trace, and optimize AI agents across various frameworks such as LangChain, CrewAI, AutoGen,...", - "rag.context_length": 2740, - "rag.sources_count": 5, - "rag.answer_length": 552, - "rag.mode": "retrieval_augmented_generation" - }, - "events": [], - "links": [] - }, - { - "span_id": "26e510c0-441a-4f9f-9dce-31180f47d060", - "trace_id": "d5528586-c741-4703-8791-5fccf5c3e123", - "parent_span_id": "6a101d9d-de83-415a-8fb0-01a9fae5ffb9", - "name": "agent.llm_model_execution", - "start_time": "2025-10-16 08:49:05.787000000", - "end_time": "2025-10-16 08:49:09.274000000", - "duration_ms": 3486.699, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What monitoring capabilities does Noveum provide?", - "query_length": 49, - "query_type": "rag_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 3.4866299629211426, - "model.latency_ms": 3486.6299629211426, - "model.start_time": 1760604545.787763, - "model.end_time": 1760604549.274393, - "model.performance_tier": "medium", - "model.prompt_tokens": 821, - "model.completion_tokens": 138, - "model.total_tokens": 959, - "model.tokens_per_second": 275.0506965747916, - "model.estimated_cost": 0.02877, - "model.efficiency_score": 0.5755995828988529, - "model.response_length": 552, - "model.response_quality": "high", - "model.output_response": "Model Response: Noveum provides comprehensive monitoring capabilities for AI applications, including the ability to monitor, trace, and optimize AI agents across various frameworks such as LangChain, CrewAI, AutoGen,...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [] - }, - { - "span_id": "fcd89e38-3d21-468b-860a-ae2de0bbd119", - "trace_id": "d5528586-c741-4703-8791-5fccf5c3e123", - "parent_span_id": "6a101d9d-de83-415a-8fb0-01a9fae5ffb9", - "name": "agent.rag_evaluation_metrics", - "start_time": "2025-10-16 08:49:09.274000000", - "end_time": "2025-10-16 08:49:09.274000000", - "duration_ms": 0.054, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "rag_evaluation_metrics", - "agent.capabilities": [ - "retrieval_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What monitoring capabilities does Noveum provide?", - "query_length": 49, - "query_type": "rag_evaluation_query", - "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en):\nMonitor all your AI Agents\nimprove AI Agents today\nNoveum.ai helps you monitor, trace, and optimize your AI applications.\nNoveum.ai works with any AI framework \u2013 LangChain, CrewAI, AutoGen, custom implementations, or direct LLM calls. One dashboard shows everything.\n...", - "retrieval.context_length": 2740, - "retrieval.sources_count": 5, - "retrieval.context_quality": "high", - "retrieval.effectiveness": 1, - "retrieval.context_utilization": 2.74, - "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en):\nMonitor all your AI Agents\nimprove AI Agents today\nNoveum.ai helps you monitor, trace, and optimize your AI applications.\nNoveum.ai works with any AI framework \u2013 LangChain, CrewAI, AutoGen, custom implementations, or direct LLM calls. One dashboard shows everything.\nMonitor, Evaluate, Improve Your AI Agents\nThe control plane for AI agents.\nMonitor Everything, Miss Nothing\nOur lightweight SDKs capture every trace and span across your AI agent ecosystem\u2014from simple LLM calls to complex multi-agent...\n\nSource 2 (https://noveum.ai/):\nMonitor all your AI Agents\nimprove AI Agents today\nNoveum.ai helps you monitor, trace, and optimize your AI applications.\nNoveum.ai works with any AI framework \u2013 LangChain, CrewAI, AutoGen, custom implementations, or direct LLM calls. One dashboard shows everything.\nMonitor, Evaluate, Improve Your AI Agents\nThe control plane for AI agents.\nMonitor Everything, Miss Nothing\nOur lightweight SDKs capture every trace and span across your AI agent ecosystem\u2014from simple LLM calls to complex multi-agent...\n\nSource 3 (https://noveum.ai):\nMonitor all your AI Agents\nimprove AI Agents today\nNoveum.ai helps you monitor, trace, and optimize your AI applications.\nNoveum.ai works with any AI framework \u2013 LangChain, CrewAI, AutoGen, custom implementations, or direct LLM calls. One dashboard shows everything.\nMonitor, Evaluate, Improve Your AI Agents\nThe control plane for AI agents.\nMonitor Everything, Miss Nothing\nOur lightweight SDKs capture every trace and span across your AI agent ecosystem\u2014from simple LLM calls to complex multi-agent...\n\nSource 4 (https://noveum.ai/en/docs/getting-started/overview):\nNoveum.ai Overview\nComprehensive AI tracing and observability platform for LLM applications, RAG systems, and AI agents\nWelcome to Noveum.ai\u2014the comprehensive tracing and observability platform built specifically for AI applications. Whether you're building LLM-powered chatbots, RAG systems, multi-agent workflows, or any AI-driven application, Noveum provides the insights you need to understand, debug, and optimize your systems.\n\ud83c\udfaf Why AI Applications Need Specialized Observability\nTraditional mo...\n\nSource 5 (https://noveum.ai/docs/getting-started/overview):\nNoveum.ai Overview\nComprehensive AI tracing and observability platform for LLM applications, RAG systems, and AI agents\nWelcome to Noveum.ai\u2014the comprehensive tracing and observability platform built specifically for AI applications. Whether you're building LLM-powered chatbots, RAG systems, multi-agent workflows, or any AI-driven application, Noveum provides the insights you need to understand, debug, and optimize your systems.\n\ud83c\udfaf Why AI Applications Need Specialized Observability\nTraditional mo...\n\nUser Question: What monitoring capabilities does Noveum provide?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", - "prompt.prompt_length": 3285, - "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en):\nMonitor all your AI Agents\nimprove AI Agents today\nNoveum.ai helps you monitor, trace, and optimize your AI applications.\nNoveum.ai works with any AI framework \u2013 LangC...", - "prompt.instruction_following": "rag_optimized", - "response.answer_length": 552, - "response.answer_completeness": "complete", - "response.response_quality": "high", - "response.source_citation": 5, - "response.context_utilization": 0.20145985401459854, - "output_response": "RAG Answer: Noveum provides comprehensive monitoring capabilities for AI applications, including the ability to monitor, trace, and optimize AI agents across various frameworks such as LangChain, CrewAI, AutoGen,...", - "evaluation.retrieval_effectiveness": 1, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 5, - "evaluation.context_relevance": "high", - "evaluation.overall_quality": "high", - "evaluation.ready_for_production": true, - "rag.retrieval_strategy": "semantic_similarity", - "rag.vector_search_results": 5, - "rag.context_synthesis": "multi_source", - "rag.document_coverage": 1, - "rag.information_density": 0.20145985401459854 - }, - "events": [], - "links": [] - } - ], - "created_at": "2025-10-16 08:49:05.152000000", - "updated_at": "2025-10-16 08:49:05.152000000" - }, - { - "trace_id": "8100b869-07c0-4e37-b51e-092c9d20b4a7", - "name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "status": "ok", - "status_message": "", - "start_time": "2025-10-16 08:48:57.574000000", - "end_time": "2025-10-16 08:49:04.149000000", - "duration_ms": 6574.500999999999, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "spans": [ - { - "span_id": "bd5f205f-0329-4811-b883-129d69f12382", - "trace_id": "8100b869-07c0-4e37-b51e-092c9d20b4a7", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-16 08:48:57.574000000", - "end_time": "2025-10-16 08:49:04.149000000", - "duration_ms": 6574.424, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "How does Noveum handle agent tracing?", - "query.length": 37, - "processing.mode": "RAG", - "processing.time_seconds": 6.5743818283081055, - "processing.response_length": 643, - "processing.sources_count": 5, - "output_response": "Final Answer: Noveum.ai handles agent tracing through its specialized Multi-Agent Tracing capabilities, which are designed to observe complex workflows and inter-agent communications within multi-agent systems. The...", - "final_answer_mode": "RAG", - "query_processed.input_query": "How does Noveum handle agent tracing?", - "query_processed.output_response": "Successfully processed query using RAG, generated 643 character response", - "query_processed.mode": "RAG", - "query_processed.processing_time": 6.5743818283081055, - "query_processed.response_length": 643 - }, - "events": [], - "links": [] - }, - { - "span_id": "0801bcfe-8c09-4ea6-a754-5564395d6ded", - "trace_id": "8100b869-07c0-4e37-b51e-092c9d20b4a7", - "parent_span_id": "3c35d86a-0df9-42dc-bfe3-bbe978cf62b5", - "name": "agent.llm-rag", - "start_time": "2025-10-16 08:48:57.575000000", - "end_time": "2025-10-16 08:49:04.149000000", - "duration_ms": 6574.135, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "rag_agent", - "agent.operation": "llm-rag", - "agent.capabilities": [ - "document_retrieval", - "context_generation", - "response_generation" - ], - "agent.id": "noveum_rag_agent", - "input_query": "How does Noveum handle agent tracing?", - "query_length": 37, - "query_type": "rag_query", - "output_response": "RAG Answer: Noveum.ai handles agent tracing through its specialized Multi-Agent Tracing capabilities, which are designed to observe complex workflows and inter-agent communications within multi-agent systems. The...", - "rag.context_length": 2736, - "rag.sources_count": 5, - "rag.answer_length": 643, - "rag.mode": "retrieval_augmented_generation" - }, - "events": [], - "links": [] - }, - { - "span_id": "3c35d86a-0df9-42dc-bfe3-bbe978cf62b5", - "trace_id": "8100b869-07c0-4e37-b51e-092c9d20b4a7", - "parent_span_id": "bd5f205f-0329-4811-b883-129d69f12382", - "name": "agent.query_routing", - "start_time": "2025-10-16 08:48:57.575000000", - "end_time": "2025-10-16 08:49:04.149000000", - "duration_ms": 6574.342, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "How does Noveum handle agent tracing?", - "query_length": 37, - "query_type": "routing_query", - "output_response": "Routed to RAG for query processing", - "router.classification": "RAG", - "router.confidence_score": 1, - "router.rag_keyword_score": 2, - "router.web_keyword_score": 0, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [] - }, - { - "span_id": "b1914316-f545-4bc9-abf4-3494cf42c4a5", - "trace_id": "8100b869-07c0-4e37-b51e-092c9d20b4a7", - "parent_span_id": "3c35d86a-0df9-42dc-bfe3-bbe978cf62b5", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-16 08:48:57.575000000", - "end_time": "2025-10-16 08:48:57.575000000", - "duration_ms": 0.032, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: How does Noveum handle agent tracing?", - "query_length": 37, - "query_type": "routing_evaluation_query", - "classification.mode": "RAG", - "classification.rag_keyword_score": 2, - "classification.web_keyword_score": 0, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "noveum_specific", - "query.keyword_density": 0.3333333333333333, - "query.domain_affinity": "noveum", - "routing.decision": "Routed to RAG based on analysis", - "routing.rationale": "RAG score: 2, Web score: 0, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "Web Search", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.038461538461538464, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: RAG (Confidence: 1.00)" - }, - "events": [], - "links": [] - }, - { - "span_id": "0906814b-6a80-40e1-9f00-5376b809703a", - "trace_id": "8100b869-07c0-4e37-b51e-092c9d20b4a7", - "parent_span_id": "0801bcfe-8c09-4ea6-a754-5564395d6ded", - "name": "agent.llm_model_execution", - "start_time": "2025-10-16 08:48:58.009000000", - "end_time": "2025-10-16 08:49:03.349000000", - "duration_ms": 5340.01, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: How does Noveum handle agent tracing?", - "query_length": 37, - "query_type": "rag_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 5.339954614639282, - "model.latency_ms": 5339.954614639282, - "model.start_time": 1760604538.0093303, - "model.end_time": 1760604543.349285, - "model.performance_tier": "slow", - "model.prompt_tokens": 817, - "model.completion_tokens": 160, - "model.total_tokens": 977, - "model.tokens_per_second": 182.9603565022054, - "model.estimated_cost": 0.02931, - "model.efficiency_score": 0.6581371545547595, - "model.response_length": 643, - "model.response_quality": "high", - "model.output_response": "Model Response: Noveum.ai handles agent tracing through its specialized Multi-Agent Tracing capabilities, which are designed to observe complex workflows and inter-agent communications within multi-agent systems. The...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [] - }, - { - "span_id": "29822edb-c15f-4baa-b6ec-84d85fcc926d", - "trace_id": "8100b869-07c0-4e37-b51e-092c9d20b4a7", - "parent_span_id": "0801bcfe-8c09-4ea6-a754-5564395d6ded", - "name": "agent.rag_evaluation_metrics", - "start_time": "2025-10-16 08:49:03.349000000", - "end_time": "2025-10-16 08:49:03.349000000", - "duration_ms": 0.034999999999999996, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "rag_evaluation_metrics", - "agent.capabilities": [ - "retrieval_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: How does Noveum handle agent tracing?", - "query_length": 37, - "query_type": "rag_evaluation_query", - "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, vector searches, data retrieval, agent reasoning, and more. Without proper observability, debugging ...", - "retrieval.context_length": 2736, - "retrieval.sources_count": 5, - "retrieval.context_quality": "high", - "retrieval.effectiveness": 1, - "retrieval.context_utilization": 2.736, - "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, vector searches, data retrieval, agent reasoning, and more. Without proper observability, debugging becomes a nightmare, optimization is guesswork, and understanding user interactions is nearly impossible.\nNoveum.ai solves this by providing comprehensive tracing and observability specifically designed for AI applications.\nKey benefits of using Noveum.ai:\n- Complete Visibility: Trace...\n\nSource 2 (https://noveum.ai/docs/advanced/multi-agent-tracing):\nMulti-Agent Tracing\nObserve complex agent workflows and inter-agent communications with comprehensive tracing\nMulti-agent systems represent some of the most complex AI applications, involving multiple agents that coordinate, communicate, and collaborate to achieve shared goals. Noveum.ai provides specialized tracing capabilities to help you understand and optimize these intricate workflows.\n\ud83c\udfaf Why Multi-Agent Tracing Matters\nMulti-agent systems introduce unique observability challenges:\n- Complex...\n\nSource 3 (https://noveum.ai/en/docs/advanced/multi-agent-tracing):\nMulti-Agent Tracing\nObserve complex agent workflows and inter-agent communications with comprehensive tracing\nMulti-agent systems represent some of the most complex AI applications, involving multiple agents that coordinate, communicate, and collaborate to achieve shared goals. Noveum.ai provides specialized tracing capabilities to help you understand and optimize these intricate workflows.\n\ud83c\udfaf Why Multi-Agent Tracing Matters\nMulti-agent systems introduce unique observability challenges:\n- Complex...\n\nSource 4 (https://noveum.ai/en/docs/advanced/performance-optimization):\n\ud83d\udd17 Next Steps\n- Multi-Agent Tracing - Observe agent workflows\n- RAG Pipeline Observability - Monitor retrieval and generation systems\n- Custom Instrumentation - Add domain-specific tracing\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nSource 5 (https://noveum.ai/docs/advanced/performance-optimization):\n\ud83d\udd17 Next Steps\n- Multi-Agent Tracing - Observe agent workflows\n- RAG Pipeline Observability - Monitor retrieval and generation systems\n- Custom Instrumentation - Add domain-specific tracing\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nUser Question: How does Noveum handle agent tracing?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", - "prompt.prompt_length": 3269, - "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, ...", - "prompt.instruction_following": "rag_optimized", - "response.answer_length": 643, - "response.answer_completeness": "complete", - "response.response_quality": "high", - "response.source_citation": 5, - "response.context_utilization": 0.23501461988304093, - "output_response": "RAG Answer: Noveum.ai handles agent tracing through its specialized Multi-Agent Tracing capabilities, which are designed to observe complex workflows and inter-agent communications within multi-agent systems. The...", - "evaluation.retrieval_effectiveness": 1, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 5, - "evaluation.context_relevance": "high", - "evaluation.overall_quality": "high", - "evaluation.ready_for_production": true, - "rag.retrieval_strategy": "semantic_similarity", - "rag.vector_search_results": 5, - "rag.context_synthesis": "multi_source", - "rag.document_coverage": 1, - "rag.information_density": 0.23501461988304093 - }, - "events": [], - "links": [] - } - ], - "created_at": "2025-10-16 08:48:57.574000000", - "updated_at": "2025-10-16 08:48:57.574000000" - }, - { - "trace_id": "c28bd31c-d200-4467-91fd-a3b80187c885", - "name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "status": "ok", - "status_message": "", - "start_time": "2025-10-16 08:48:17.977000000", - "end_time": "2025-10-16 08:48:56.570000000", - "duration_ms": 38592.996, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "spans": [ - { - "span_id": "5f8ea007-edef-477c-b8f1-57c2da9e9258", - "trace_id": "c28bd31c-d200-4467-91fd-a3b80187c885", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-16 08:48:17.977000000", - "end_time": "2025-10-16 08:48:56.570000000", - "duration_ms": 38592.912, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What APIs are available in Noveum platform?", - "query.length": 43, - "processing.mode": "RAG", - "processing.time_seconds": 38.59272003173828, - "processing.response_length": 270, - "processing.sources_count": 5, - "output_response": "Final Answer: The provided context does not specify the APIs available in the Noveum platform. For detailed information about the APIs, I recommend checking the official Noveum documentation or the SDK Integration ...", - "final_answer_mode": "RAG", - "query_processed.input_query": "What APIs are available in Noveum platform?", - "query_processed.output_response": "Successfully processed query using RAG, generated 270 character response", - "query_processed.mode": "RAG", - "query_processed.processing_time": 38.59272003173828, - "query_processed.response_length": 270 - }, - "events": [], - "links": [] - }, - { - "span_id": "db385f4d-beef-465b-9831-a65080537519", - "trace_id": "c28bd31c-d200-4467-91fd-a3b80187c885", - "parent_span_id": "5f8ea007-edef-477c-b8f1-57c2da9e9258", - "name": "agent.query_routing", - "start_time": "2025-10-16 08:48:17.977000000", - "end_time": "2025-10-16 08:48:56.570000000", - "duration_ms": 38592.66499999999, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What APIs are available in Noveum platform?", - "query_length": 43, - "query_type": "routing_query", - "output_response": "Routed to RAG for query processing", - "router.classification": "RAG", - "router.confidence_score": 1, - "router.rag_keyword_score": 3, - "router.web_keyword_score": 0, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [] - }, - { - "span_id": "3ccebe1c-1f2a-433b-a36f-453a0b803511", - "trace_id": "c28bd31c-d200-4467-91fd-a3b80187c885", - "parent_span_id": "db385f4d-beef-465b-9831-a65080537519", - "name": "agent.llm-rag", - "start_time": "2025-10-16 08:48:17.978000000", - "end_time": "2025-10-16 08:48:56.570000000", - "duration_ms": 38592.416999999994, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "rag_agent", - "agent.operation": "llm-rag", - "agent.capabilities": [ - "document_retrieval", - "context_generation", - "response_generation" - ], - "agent.id": "noveum_rag_agent", - "input_query": "What APIs are available in Noveum platform?", - "query_length": 43, - "query_type": "rag_query", - "output_response": "RAG Answer: The provided context does not specify the APIs available in the Noveum platform. For detailed information about the APIs, I recommend checking the official Noveum documentation or the SDK Integration ...", - "rag.context_length": 2284, - "rag.sources_count": 5, - "rag.answer_length": 270, - "rag.mode": "retrieval_augmented_generation" - }, - "events": [], - "links": [] - }, - { - "span_id": "6d1bb8e9-315d-433a-8df5-7f5fb2eeee16", - "trace_id": "c28bd31c-d200-4467-91fd-a3b80187c885", - "parent_span_id": "db385f4d-beef-465b-9831-a65080537519", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-16 08:48:17.978000000", - "end_time": "2025-10-16 08:48:17.978000000", - "duration_ms": 0.043000000000000003, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What APIs are available in Noveum platform?", - "query_length": 43, - "query_type": "routing_evaluation_query", - "classification.mode": "RAG", - "classification.rag_keyword_score": 3, - "classification.web_keyword_score": 0, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "noveum_specific", - "query.keyword_density": 0.42857142857142855, - "query.domain_affinity": "noveum", - "routing.decision": "Routed to RAG based on analysis", - "routing.rationale": "RAG score: 3, Web score: 0, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "Web Search", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.057692307692307696, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: RAG (Confidence: 1.00)" - }, - "events": [], - "links": [] - }, - { - "span_id": "2f11df8a-8f45-425d-a0ce-1eec7887b508", - "trace_id": "c28bd31c-d200-4467-91fd-a3b80187c885", - "parent_span_id": "3ccebe1c-1f2a-433b-a36f-453a0b803511", - "name": "agent.llm_model_execution", - "start_time": "2025-10-16 08:48:18.392000000", - "end_time": "2025-10-16 08:48:55.545000000", - "duration_ms": 37152.496999999996, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What APIs are available in Noveum platform?", - "query_length": 43, - "query_type": "rag_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 37.15243363380432, - "model.latency_ms": 37152.43363380432, - "model.start_time": 1760604498.3926067, - "model.end_time": 1760604535.5450404, - "model.performance_tier": "slow", - "model.prompt_tokens": 705, - "model.completion_tokens": 67, - "model.total_tokens": 772, - "model.tokens_per_second": 20.779257897592238, - "model.estimated_cost": 0.02316, - "model.efficiency_score": 0.34974093264248707, - "model.response_length": 270, - "model.response_quality": "high", - "model.output_response": "Model Response: The provided context does not specify the APIs available in the Noveum platform. For detailed information about the APIs, I recommend checking the official Noveum documentation or the SDK Integration ...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [] - }, - { - "span_id": "d13e608c-06e4-47c4-b602-c40ac9560be2", - "trace_id": "c28bd31c-d200-4467-91fd-a3b80187c885", - "parent_span_id": "3ccebe1c-1f2a-433b-a36f-453a0b803511", - "name": "agent.rag_evaluation_metrics", - "start_time": "2025-10-16 08:48:55.545000000", - "end_time": "2025-10-16 08:48:55.545000000", - "duration_ms": 0.031, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "rag_evaluation_metrics", - "agent.capabilities": [ - "retrieval_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What APIs are available in Noveum platform?", - "query_length": 43, - "query_type": "rag_evaluation_query", - "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/docs):\n- \ud83c\udf0d Universal: Works across frameworks, clouds, and deployment models\n\ud83e\udd1d Community & Support\n- \ud83d\udcac Discord Community: Join our Discord\n- \ud83d\udce7 Email Support: [email protected]\n- \ud83d\udc1b Bug Reports: GitHub Issues\n- \ud83d\udcd6 Knowledge Base: Help Center\nReady to get started? Head to ...", - "retrieval.context_length": 2284, - "retrieval.sources_count": 5, - "retrieval.context_quality": "high", - "retrieval.effectiveness": 1, - "retrieval.context_utilization": 2.284, - "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/docs):\n- \ud83c\udf0d Universal: Works across frameworks, clouds, and deployment models\n\ud83e\udd1d Community & Support\n- \ud83d\udcac Discord Community: Join our Discord\n- \ud83d\udce7 Email Support: [email protected]\n- \ud83d\udc1b Bug Reports: GitHub Issues\n- \ud83d\udcd6 Knowledge Base: Help Center\nReady to get started? Head to our SDK Integration Guide to begin tracing your AI applications in under 5 minutes!\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability ...\n\nSource 2 (https://noveum.ai/docs):\n- \ud83c\udf0d Universal: Works across frameworks, clouds, and deployment models\n\ud83e\udd1d Community & Support\n- \ud83d\udcac Discord Community: Join our Discord\n- \ud83d\udce7 Email Support: [email protected]\n- \ud83d\udc1b Bug Reports: GitHub Issues\n- \ud83d\udcd6 Knowledge Base: Help Center\nReady to get started? Head to our SDK Integration Guide to begin tracing your AI applications in under 5 minutes!\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability ...\n\nSource 3 (https://noveum.ai/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nSource 4 (https://noveum.ai/en/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nSource 5 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, vector searches, data retrieval, agent reasoning, and more. Without proper observability, debugging becomes a nightmare, optimization is guesswork, and understanding user interactions is nearly impossible.\nNoveum.ai solves this by providing comprehensive tracing and observability specifically designed for AI applications.\nKey benefits of using Noveum.ai:\n- Complete Visibility: Trace...\n\nUser Question: What APIs are available in Noveum platform?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", - "prompt.prompt_length": 2823, - "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/docs):\n- \ud83c\udf0d Universal: Works across frameworks, clouds, and deployment models\n\ud83e\udd1d Community & Support\n- \ud83d\udcac Discord Community: Join our Discord\n- \ud83d\udce7 Email Support: [email prot...", - "prompt.instruction_following": "rag_optimized", - "response.answer_length": 270, - "response.answer_completeness": "complete", - "response.response_quality": "high", - "response.source_citation": 5, - "response.context_utilization": 0.11821366024518389, - "output_response": "RAG Answer: The provided context does not specify the APIs available in the Noveum platform. For detailed information about the APIs, I recommend checking the official Noveum documentation or the SDK Integration ...", - "evaluation.retrieval_effectiveness": 1, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 5, - "evaluation.context_relevance": "high", - "evaluation.overall_quality": "high", - "evaluation.ready_for_production": true, - "rag.retrieval_strategy": "semantic_similarity", - "rag.vector_search_results": 5, - "rag.context_synthesis": "multi_source", - "rag.document_coverage": 1, - "rag.information_density": 0.11821366024518389 - }, - "events": [], - "links": [] - } - ], - "created_at": "2025-10-16 08:48:17.977000000", - "updated_at": "2025-10-16 08:48:17.977000000" - }, - { - "trace_id": "b90ef3fc-11ea-4230-a59b-e2f2db8f2de0", - "name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "status": "ok", - "status_message": "", - "start_time": "2025-10-16 08:48:08.007000000", - "end_time": "2025-10-16 08:48:16.975000000", - "duration_ms": 8967.264, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "spans": [ - { - "span_id": "3f3babbd-abf5-4f5a-b31c-fa5dcec9c360", - "trace_id": "b90ef3fc-11ea-4230-a59b-e2f2db8f2de0", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-16 08:48:08.007000000", - "end_time": "2025-10-16 08:48:16.975000000", - "duration_ms": 8967.201, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "How do I set up observability with Noveum?", - "query.length": 42, - "processing.mode": "RAG", - "processing.time_seconds": 8.96716833114624, - "processing.response_length": 838, - "processing.sources_count": 5, - "output_response": "Final Answer: To set up observability with Noveum, follow these steps:\n\n1. **Sign Up**: Go to [noveum.ai](https://noveum.ai) and create an account.\n2. **Create a Project**: Once signed in, create a new project to o...", - "final_answer_mode": "RAG", - "query_processed.input_query": "How do I set up observability with Noveum?", - "query_processed.output_response": "Successfully processed query using RAG, generated 838 character response", - "query_processed.mode": "RAG", - "query_processed.processing_time": 8.96716833114624, - "query_processed.response_length": 838 - }, - "events": [], - "links": [] - }, - { - "span_id": "41f6453c-4be6-4b66-8f9b-8e795f7b41b0", - "trace_id": "b90ef3fc-11ea-4230-a59b-e2f2db8f2de0", - "parent_span_id": "b0a76345-49e5-4c1b-915f-cec90ea2b1f7", - "name": "agent.llm-rag", - "start_time": "2025-10-16 08:48:08.007000000", - "end_time": "2025-10-16 08:48:16.974000000", - "duration_ms": 8966.964, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "rag_agent", - "agent.operation": "llm-rag", - "agent.capabilities": [ - "document_retrieval", - "context_generation", - "response_generation" - ], - "agent.id": "noveum_rag_agent", - "input_query": "How do I set up observability with Noveum?", - "query_length": 42, - "query_type": "rag_query", - "output_response": "RAG Answer: To set up observability with Noveum, follow these steps:\n\n1. **Sign Up**: Go to [noveum.ai](https://noveum.ai) and create an account.\n2. **Create a Project**: Once signed in, create a new project to o...", - "rag.context_length": 2870, - "rag.sources_count": 6, - "rag.answer_length": 838, - "rag.mode": "retrieval_augmented_generation" - }, - "events": [], - "links": [] - }, - { - "span_id": "b0a76345-49e5-4c1b-915f-cec90ea2b1f7", - "trace_id": "b90ef3fc-11ea-4230-a59b-e2f2db8f2de0", - "parent_span_id": "3f3babbd-abf5-4f5a-b31c-fa5dcec9c360", - "name": "agent.query_routing", - "start_time": "2025-10-16 08:48:08.007000000", - "end_time": "2025-10-16 08:48:16.974000000", - "duration_ms": 8967.129, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "How do I set up observability with Noveum?", - "query_length": 42, - "query_type": "routing_query", - "output_response": "Routed to RAG for query processing", - "router.classification": "RAG", - "router.confidence_score": 1, - "router.rag_keyword_score": 2, - "router.web_keyword_score": 0, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [] - }, - { - "span_id": "fe3074a7-f4c0-442e-9a5b-e2cc11d93785", - "trace_id": "b90ef3fc-11ea-4230-a59b-e2f2db8f2de0", - "parent_span_id": "b0a76345-49e5-4c1b-915f-cec90ea2b1f7", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-16 08:48:08.007000000", - "end_time": "2025-10-16 08:48:08.007000000", - "duration_ms": 0.032, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: How do I set up observability with Noveum?", - "query_length": 42, - "query_type": "routing_evaluation_query", - "classification.mode": "RAG", - "classification.rag_keyword_score": 2, - "classification.web_keyword_score": 0, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "noveum_specific", - "query.keyword_density": 0.25, - "query.domain_affinity": "noveum", - "routing.decision": "Routed to RAG based on analysis", - "routing.rationale": "RAG score: 2, Web score: 0, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "Web Search", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.038461538461538464, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: RAG (Confidence: 1.00)" - }, - "events": [], - "links": [] - }, - { - "span_id": "e16139ee-048e-4c2f-87f8-90442f41a0ae", - "trace_id": "b90ef3fc-11ea-4230-a59b-e2f2db8f2de0", - "parent_span_id": "41f6453c-4be6-4b66-8f9b-8e795f7b41b0", - "name": "agent.llm_model_execution", - "start_time": "2025-10-16 08:48:08.644000000", - "end_time": "2025-10-16 08:48:15.709000000", - "duration_ms": 7064.898, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: How do I set up observability with Noveum?", - "query_length": 42, - "query_type": "rag_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 7.0648510456085205, - "model.latency_ms": 7064.8510456085205, - "model.start_time": 1760604488.6448367, - "model.end_time": 1760604495.7096877, - "model.performance_tier": "slow", - "model.prompt_tokens": 852, - "model.completion_tokens": 209, - "model.total_tokens": 1061, - "model.tokens_per_second": 150.1800948315128, - "model.estimated_cost": 0.031830000000000004, - "model.efficiency_score": 0.7898209236569275, - "model.response_length": 838, - "model.response_quality": "high", - "model.output_response": "Model Response: To set up observability with Noveum, follow these steps:\n\n1. **Sign Up**: Go to [noveum.ai](https://noveum.ai) and create an account.\n2. **Create a Project**: Once signed in, create a new project to o...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [] - }, - { - "span_id": "59e85f8e-c19b-4794-9a69-7b5bc90208f0", - "trace_id": "b90ef3fc-11ea-4230-a59b-e2f2db8f2de0", - "parent_span_id": "41f6453c-4be6-4b66-8f9b-8e795f7b41b0", - "name": "agent.rag_evaluation_metrics", - "start_time": "2025-10-16 08:48:15.709000000", - "end_time": "2025-10-16 08:48:15.709000000", - "duration_ms": 0.039, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "rag_evaluation_metrics", - "agent.capabilities": [ - "retrieval_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: How do I set up observability with Noveum?", - "query_length": 42, - "query_type": "rag_evaluation_query", - "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\n- Production Ready: Built for scale with intelligent sampling and batching\n- Privacy Focused: You control what data is captured and how it's stored\n- Open Source SDKs: Transparent, auditable, and extensible\nGetting ...", - "retrieval.context_length": 2870, - "retrieval.sources_count": 6, - "retrieval.context_quality": "high", - "retrieval.effectiveness": 1.2, - "retrieval.context_utilization": 2.87, - "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\n- Production Ready: Built for scale with intelligent sampling and batching\n- Privacy Focused: You control what data is captured and how it's stored\n- Open Source SDKs: Transparent, auditable, and extensible\nGetting Started\nReady to add comprehensive observability to your AI applications?\n- Sign up at noveum.ai\n- Create a project and get your API key\n- Install the SDK for your preferred language\n- Add tracing to your AI workflows\n- Explore insights in the Noveum.ai dashboard\nCheck out our integra...\n\nSource 2 (https://noveum.ai/en/docs/getting-started/tracing-concepts):\n- Explore Framework Integrations - Framework-specific guidance\n- Learn Advanced Patterns - Custom instrumentation techniques\n- Master the Dashboard - Analyze your traces effectively\nRemember: Good observability is not about collecting all possible data, but about collecting the right data that helps you understand, debug, and optimize your AI applications.\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to O...\n\nSource 3 (https://noveum.ai/docs/getting-started/tracing-concepts):\n- Explore Framework Integrations - Framework-specific guidance\n- Learn Advanced Patterns - Custom instrumentation techniques\n- Master the Dashboard - Analyze your traces effectively\nRemember: Good observability is not about collecting all possible data, but about collecting the right data that helps you understand, debug, and optimize your AI applications.\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to O...\n\nSource 4 (https://noveum.ai/en/docs/getting-started/overview):\nNoveum.ai Overview\nComprehensive AI tracing and observability platform for LLM applications, RAG systems, and AI agents\nWelcome to Noveum.ai\u2014the comprehensive tracing and observability platform built specifically for AI applications. Whether you're building LLM-powered chatbots, RAG systems, multi-agent workflows, or any AI-driven application, Noveum provides the insights you need to understand, debug, and optimize your systems.\n\ud83c\udfaf Why AI Applications Need Specialized Observability\nTraditional mo...\n\nSource 5 (https://noveum.ai/docs/getting-started/overview):\nNoveum.ai Overview\nComprehensive AI tracing and observability platform for LLM applications, RAG systems, and AI agents\nWelcome to Noveum.ai\u2014the comprehensive tracing and observability platform built specifically for AI applications. Whether you're building LLM-powered chatbots, RAG systems, multi-agent workflows, or any AI-driven application, Noveum provides the insights you need to understand, debug, and optimize your systems.\n\ud83c\udfaf Why AI Applications Need Specialized Observability\nTraditional mo...\n\nUser Question: How do I set up observability with Noveum?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", - "prompt.prompt_length": 3408, - "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\n- Production Ready: Built for scale with intelligent sampling and batching\n- Privacy Focused: You control what data...", - "prompt.instruction_following": "rag_optimized", - "response.answer_length": 838, - "response.answer_completeness": "complete", - "response.response_quality": "high", - "response.source_citation": 6, - "response.context_utilization": 0.29198606271777006, - "output_response": "RAG Answer: To set up observability with Noveum, follow these steps:\n\n1. **Sign Up**: Go to [noveum.ai](https://noveum.ai) and create an account.\n2. **Create a Project**: Once signed in, create a new project to o...", - "evaluation.retrieval_effectiveness": 1.2, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 6, - "evaluation.context_relevance": "high", - "evaluation.overall_quality": "high", - "evaluation.ready_for_production": true, - "rag.retrieval_strategy": "semantic_similarity", - "rag.vector_search_results": 6, - "rag.context_synthesis": "multi_source", - "rag.document_coverage": 1.2, - "rag.information_density": 0.29198606271777006 - }, - "events": [], - "links": [] - } - ], - "created_at": "2025-10-16 08:48:08.007000000", - "updated_at": "2025-10-16 08:48:08.007000000" - }, - { - "trace_id": "b8abebac-a7dc-494c-a1eb-6470181f32d2", - "name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "status": "ok", - "status_message": "", - "start_time": "2025-10-16 08:47:57.667000000", - "end_time": "2025-10-16 08:48:07.005000000", - "duration_ms": 9337.405, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "spans": [ - { - "span_id": "f049ed2b-ff2f-4092-9974-acc66f81bd8d", - "trace_id": "b8abebac-a7dc-494c-a1eb-6470181f32d2", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-16 08:47:57.667000000", - "end_time": "2025-10-16 08:48:07.005000000", - "duration_ms": 9337.298, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What features does Noveum Trace offer?", - "query.length": 38, - "processing.mode": "RAG", - "processing.time_seconds": 9.33726453781128, - "processing.response_length": 1018, - "processing.sources_count": 5, - "output_response": "Final Answer: Noveum Trace offers several key features through its Python SDK (`noveum-trace`) and TypeScript SDK (`@noveum/trace`):\n\n1. **Decorator-Based Tracing**: It provides a decorator-based approach for traci...", - "final_answer_mode": "RAG", - "query_processed.input_query": "What features does Noveum Trace offer?", - "query_processed.output_response": "Successfully processed query using RAG, generated 1018 character response", - "query_processed.mode": "RAG", - "query_processed.processing_time": 9.33726453781128, - "query_processed.response_length": 1018 - }, - "events": [], - "links": [] - }, - { - "span_id": "211f1fe7-4cfb-4c1c-9c72-62ae2825dda3", - "trace_id": "b8abebac-a7dc-494c-a1eb-6470181f32d2", - "parent_span_id": "f049ed2b-ff2f-4092-9974-acc66f81bd8d", - "name": "agent.query_routing", - "start_time": "2025-10-16 08:47:57.668000000", - "end_time": "2025-10-16 08:48:07.005000000", - "duration_ms": 9337.223, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What features does Noveum Trace offer?", - "query_length": 38, - "query_type": "routing_query", - "output_response": "Routed to RAG for query processing", - "router.classification": "RAG", - "router.confidence_score": 1, - "router.rag_keyword_score": 3, - "router.web_keyword_score": 0, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [] - }, - { - "span_id": "976c0c4b-efc1-4b7d-bfad-350c7fd0f71e", - "trace_id": "b8abebac-a7dc-494c-a1eb-6470181f32d2", - "parent_span_id": "211f1fe7-4cfb-4c1c-9c72-62ae2825dda3", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-16 08:47:57.668000000", - "end_time": "2025-10-16 08:47:57.668000000", - "duration_ms": 0.028, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What features does Noveum Trace offer?", - "query_length": 38, - "query_type": "routing_evaluation_query", - "classification.mode": "RAG", - "classification.rag_keyword_score": 3, - "classification.web_keyword_score": 0, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "noveum_specific", - "query.keyword_density": 0.5, - "query.domain_affinity": "noveum", - "routing.decision": "Routed to RAG based on analysis", - "routing.rationale": "RAG score: 3, Web score: 0, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "Web Search", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.057692307692307696, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: RAG (Confidence: 1.00)" - }, - "events": [], - "links": [] - }, - { - "span_id": "d2afaf74-f63b-4141-9fa3-662162f51cc0", - "trace_id": "b8abebac-a7dc-494c-a1eb-6470181f32d2", - "parent_span_id": "211f1fe7-4cfb-4c1c-9c72-62ae2825dda3", - "name": "agent.llm-rag", - "start_time": "2025-10-16 08:47:57.668000000", - "end_time": "2025-10-16 08:48:07.005000000", - "duration_ms": 9337.056, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "rag_agent", - "agent.operation": "llm-rag", - "agent.capabilities": [ - "document_retrieval", - "context_generation", - "response_generation" - ], - "agent.id": "noveum_rag_agent", - "input_query": "What features does Noveum Trace offer?", - "query_length": 38, - "query_type": "rag_query", - "output_response": "RAG Answer: Noveum Trace offers several key features through its Python SDK (`noveum-trace`) and TypeScript SDK (`@noveum/trace`):\n\n1. **Decorator-Based Tracing**: It provides a decorator-based approach for traci...", - "rag.context_length": 2787, - "rag.sources_count": 5, - "rag.answer_length": 1018, - "rag.mode": "retrieval_augmented_generation" - }, - "events": [], - "links": [] - }, - { - "span_id": "09811a73-405d-4c0d-ba5d-0472b3708062", - "trace_id": "b8abebac-a7dc-494c-a1eb-6470181f32d2", - "parent_span_id": "d2afaf74-f63b-4141-9fa3-662162f51cc0", - "name": "agent.llm_model_execution", - "start_time": "2025-10-16 08:47:58.659000000", - "end_time": "2025-10-16 08:48:05.878000000", - "duration_ms": 7218.618, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What features does Noveum Trace offer?", - "query_length": 38, - "query_type": "rag_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 7.218573331832886, - "model.latency_ms": 7218.573331832886, - "model.start_time": 1760604478.659586, - "model.end_time": 1760604485.8781593, - "model.performance_tier": "slow", - "model.prompt_tokens": 830, - "model.completion_tokens": 254, - "model.total_tokens": 1084, - "model.tokens_per_second": 150.16817730724068, - "model.estimated_cost": 0.03252, - "model.efficiency_score": 0.9391143911439115, - "model.response_length": 1018, - "model.response_quality": "high", - "model.output_response": "Model Response: Noveum Trace offers several key features through its Python SDK (`noveum-trace`) and TypeScript SDK (`@noveum/trace`):\n\n1. **Decorator-Based Tracing**: It provides a decorator-based approach for traci...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [] - }, - { - "span_id": "015688fe-d287-48e8-ac59-63781ffd6f89", - "trace_id": "b8abebac-a7dc-494c-a1eb-6470181f32d2", - "parent_span_id": "d2afaf74-f63b-4141-9fa3-662162f51cc0", - "name": "agent.rag_evaluation_metrics", - "start_time": "2025-10-16 08:48:05.878000000", - "end_time": "2025-10-16 08:48:05.878000000", - "duration_ms": 0.024, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "rag_evaluation_metrics", - "agent.capabilities": [ - "retrieval_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What features does Noveum Trace offer?", - "query_length": 38, - "query_type": "rag_evaluation_query", - "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\n\ud83d\udcd8 TypeScript SDK (\n@noveum/trace\n)- Framework ...", - "retrieval.context_length": 2787, - "retrieval.sources_count": 5, - "retrieval.context_quality": "high", - "retrieval.effectiveness": 1, - "retrieval.context_utilization": 2.787, - "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\n\ud83d\udcd8 TypeScript SDK (\n@noveum/trace\n)- Framework integrations for Next.js, Express.js, and Hono\n- TypeScript-first design with full type safety\n- Universal compatibility (Node.js, Edge Runtime, browsers)\n-\n\ud83d\udcca Noveum Platform\n- Real-time dashboard for analyzing traces and performance\n- Ad...\n\nSource 2 (https://noveum.ai/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\n\ud83d\udcd8 TypeScript SDK (\n@noveum/trace\n)- Framework integrations for Next.js, Express.js, and Hono\n- TypeScript-first design with full type safety\n- Universal compatibility (Node.js, Edge Runtime, browsers)\n-\n\ud83d\udcca Noveum Platform\n- Real-time dashboard for analyzing traces and performance\n- Ad...\n\nSource 3 (https://noveum.ai/en/docs/getting-started/sdk-integration):\n- \ud83d\udc65 Collaborate with your team on insights\n\ud83d\udd12 Security & Privacy\n- \ud83d\udd10 Encryption: All data encrypted in transit and at rest\n- \ud83c\udf9b\ufe0f Configurable Capture: Control what data is collected\n- \ud83c\udfe0 Data Residency: Choose your data storage region\n- \u23f0 Retention Control: Set custom data retention policies\nNext Steps\n- Tracing Concepts - Learn about traces, spans, and observability best practices\n- Framework Integrations - Deep dive into specific framework setups\n- Multi-Agent Tracing - Observe complex agent work...\n\nSource 4 (https://noveum.ai/docs/getting-started/sdk-integration):\n- \ud83d\udc65 Collaborate with your team on insights\n\ud83d\udd12 Security & Privacy\n- \ud83d\udd10 Encryption: All data encrypted in transit and at rest\n- \ud83c\udf9b\ufe0f Configurable Capture: Control what data is collected\n- \ud83c\udfe0 Data Residency: Choose your data storage region\n- \u23f0 Retention Control: Set custom data retention policies\nNext Steps\n- Tracing Concepts - Learn about traces, spans, and observability best practices\n- Framework Integrations - Deep dive into specific framework setups\n- Multi-Agent Tracing - Observe complex agent work...\n\nSource 5 (https://noveum.ai/docs/platform/dashboard):\n- Bulk Export: Download multiple traces simultaneously\n- Comparative Analysis: Compare performance across multiple traces\nReal-Time Updates\n- Live Refresh: Automatic updates as new traces arrive\n- Connection Monitoring: Real-time status of your trace ingestion pipeline\n- Performance Indicators: Live metrics for system health monitoring\nReady to dive deeper? Explore Projects & Environments to organize your AI applications, or check out Team Collaboration to share insights with your team.\nGet Earl...\n\nUser Question: What features does Noveum Trace offer?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", - "prompt.prompt_length": 3321, - "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frame...", - "prompt.instruction_following": "rag_optimized", - "response.answer_length": 1018, - "response.answer_completeness": "complete", - "response.response_quality": "high", - "response.source_citation": 5, - "response.context_utilization": 0.36526731252242556, - "output_response": "RAG Answer: Noveum Trace offers several key features through its Python SDK (`noveum-trace`) and TypeScript SDK (`@noveum/trace`):\n\n1. **Decorator-Based Tracing**: It provides a decorator-based approach for traci...", - "evaluation.retrieval_effectiveness": 1, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 5, - "evaluation.context_relevance": "high", - "evaluation.overall_quality": "high", - "evaluation.ready_for_production": true, - "rag.retrieval_strategy": "semantic_similarity", - "rag.vector_search_results": 5, - "rag.context_synthesis": "multi_source", - "rag.document_coverage": 1, - "rag.information_density": 0.36526731252242556 - }, - "events": [], - "links": [] - } - ], - "created_at": "2025-10-16 08:47:57.667000000", - "updated_at": "2025-10-16 08:47:57.667000000" - }, - { - "trace_id": "be9ccbaf-ccb4-4ebd-a380-4247f4d01868", - "name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "status": "ok", - "status_message": "", - "start_time": "2025-10-16 08:47:52.037000000", - "end_time": "2025-10-16 08:47:56.662000000", - "duration_ms": 4625.492, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "spans": [ - { - "span_id": "46ee4d00-f914-41d8-9f91-c2db85f1c613", - "trace_id": "be9ccbaf-ccb4-4ebd-a380-4247f4d01868", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-16 08:47:52.037000000", - "end_time": "2025-10-16 08:47:56.662000000", - "duration_ms": 4625.395, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What are Noveum's pricing plans?", - "query.length": 32, - "processing.mode": "RAG", - "processing.time_seconds": 4.6253437995910645, - "processing.response_length": 471, - "processing.sources_count": 5, - "output_response": "Final Answer: The provided context does not contain specific information about Noveum's pricing plans. It mentions that all users will have access to the Observability suite for free, and early users will receive f...", - "final_answer_mode": "RAG", - "query_processed.input_query": "What are Noveum's pricing plans?", - "query_processed.output_response": "Successfully processed query using RAG, generated 471 character response", - "query_processed.mode": "RAG", - "query_processed.processing_time": 4.6253437995910645, - "query_processed.response_length": 471 - }, - "events": [], - "links": [] - }, - { - "span_id": "6d15369a-ae35-45c7-84dd-bdf484f73f2d", - "trace_id": "be9ccbaf-ccb4-4ebd-a380-4247f4d01868", - "parent_span_id": "46ee4d00-f914-41d8-9f91-c2db85f1c613", - "name": "agent.query_routing", - "start_time": "2025-10-16 08:47:52.037000000", - "end_time": "2025-10-16 08:47:56.662000000", - "duration_ms": 4625.286, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What are Noveum's pricing plans?", - "query_length": 32, - "query_type": "routing_query", - "output_response": "Routed to RAG for query processing", - "router.classification": "RAG", - "router.confidence_score": 1, - "router.rag_keyword_score": 3, - "router.web_keyword_score": 0, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [] - }, - { - "span_id": "bf201676-a3a9-433a-a8c6-d394b7aead3a", - "trace_id": "be9ccbaf-ccb4-4ebd-a380-4247f4d01868", - "parent_span_id": "6d15369a-ae35-45c7-84dd-bdf484f73f2d", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-16 08:47:52.037000000", - "end_time": "2025-10-16 08:47:52.037000000", - "duration_ms": 0.048, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What are Noveum's pricing plans?", - "query_length": 32, - "query_type": "routing_evaluation_query", - "classification.mode": "RAG", - "classification.rag_keyword_score": 3, - "classification.web_keyword_score": 0, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "noveum_specific", - "query.keyword_density": 0.6, - "query.domain_affinity": "noveum", - "routing.decision": "Routed to RAG based on analysis", - "routing.rationale": "RAG score: 3, Web score: 0, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "Web Search", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.057692307692307696, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: RAG (Confidence: 1.00)" - }, - "events": [], - "links": [] - }, - { - "span_id": "f8346f60-131b-4007-ba3e-7b0ff10cdfde", - "trace_id": "be9ccbaf-ccb4-4ebd-a380-4247f4d01868", - "parent_span_id": "6d15369a-ae35-45c7-84dd-bdf484f73f2d", - "name": "agent.llm-rag", - "start_time": "2025-10-16 08:47:52.037000000", - "end_time": "2025-10-16 08:47:56.662000000", - "duration_ms": 4624.998, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "rag_agent", - "agent.operation": "llm-rag", - "agent.capabilities": [ - "document_retrieval", - "context_generation", - "response_generation" - ], - "agent.id": "noveum_rag_agent", - "input_query": "What are Noveum's pricing plans?", - "query_length": 32, - "query_type": "rag_query", - "output_response": "RAG Answer: The provided context does not contain specific information about Noveum's pricing plans. It mentions that all users will have access to the Observability suite for free, and early users will receive f...", - "rag.context_length": 2105, - "rag.sources_count": 5, - "rag.answer_length": 471, - "rag.mode": "retrieval_augmented_generation" - }, - "events": [], - "links": [] - }, - { - "span_id": "0186c28c-7596-4279-a4eb-7733cd665609", - "trace_id": "be9ccbaf-ccb4-4ebd-a380-4247f4d01868", - "parent_span_id": "f8346f60-131b-4007-ba3e-7b0ff10cdfde", - "name": "agent.llm_model_execution", - "start_time": "2025-10-16 08:47:52.579000000", - "end_time": "2025-10-16 08:47:55.842000000", - "duration_ms": 3262.5829999999996, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What are Noveum's pricing plans?", - "query_length": 32, - "query_type": "rag_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 3.2625203132629395, - "model.latency_ms": 3262.5203132629395, - "model.start_time": 1760604472.580008, - "model.end_time": 1760604475.8425283, - "model.performance_tier": "medium", - "model.prompt_tokens": 658, - "model.completion_tokens": 117, - "model.total_tokens": 775, - "model.tokens_per_second": 237.54641368804243, - "model.estimated_cost": 0.02325, - "model.efficiency_score": 0.607741935483871, - "model.response_length": 471, - "model.response_quality": "high", - "model.output_response": "Model Response: The provided context does not contain specific information about Noveum's pricing plans. It mentions that all users will have access to the Observability suite for free, and early users will receive f...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [] - }, - { - "span_id": "3bdf8d5b-c10d-44f3-8514-8facc7f63db8", - "trace_id": "be9ccbaf-ccb4-4ebd-a380-4247f4d01868", - "parent_span_id": "f8346f60-131b-4007-ba3e-7b0ff10cdfde", - "name": "agent.rag_evaluation_metrics", - "start_time": "2025-10-16 08:47:55.842000000", - "end_time": "2025-10-16 08:47:55.842000000", - "duration_ms": 0.032, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "rag_evaluation_metrics", - "agent.capabilities": [ - "retrieval_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What are Noveum's pricing plans?", - "query_length": 32, - "query_type": "rag_evaluation_query", - "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....", - "retrieval.context_length": 2105, - "retrieval.sources_count": 5, - "retrieval.context_quality": "high", - "retrieval.effectiveness": 1, - "retrieval.context_utilization": 2.105, - "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nSource 2 (https://noveum.ai/en/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nSource 3 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nThe AI landscape is complex and moving fast. With Noveum.ai, you'll have the insights you need to build, debug, and optimize AI applications with confidence.\nReady to see what's really happening inside your AI applications? Try Noveum.ai today and transform how you build and monitor AI systems.\nLet's build more reliable, observable AI\u2014together.\nGet Early Access to Noveum.ai Platform\nJoin the select group of AI teams optimizing their models with our data-driven platform. We're onboarding users in...\n\nSource 4 (https://noveum.ai/en/blog/from-logs-to-intelligent-choices-inside-noveum-ais-evaluation-process):\nGet Early Access to Noveum.ai Platform\nJoin the select group of AI teams optimizing their models with our data-driven platform. We're onboarding users in limited batches to ensure a premium experience....\n\nSource 5 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, vector searches, data retrieval, agent reasoning, and more. Without proper observability, debugging becomes a nightmare, optimization is guesswork, and understanding user interactions is nearly impossible.\nNoveum.ai solves this by providing comprehensive tracing and observability specifically designed for AI applications.\nKey benefits of using Noveum.ai:\n- Complete Visibility: Trace...\n\nUser Question: What are Noveum's pricing plans?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", - "prompt.prompt_length": 2633, - "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to ...", - "prompt.instruction_following": "rag_optimized", - "response.answer_length": 471, - "response.answer_completeness": "complete", - "response.response_quality": "high", - "response.source_citation": 5, - "response.context_utilization": 0.22375296912114015, - "output_response": "RAG Answer: The provided context does not contain specific information about Noveum's pricing plans. It mentions that all users will have access to the Observability suite for free, and early users will receive f...", - "evaluation.retrieval_effectiveness": 1, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 5, - "evaluation.context_relevance": "high", - "evaluation.overall_quality": "high", - "evaluation.ready_for_production": true, - "rag.retrieval_strategy": "semantic_similarity", - "rag.vector_search_results": 5, - "rag.context_synthesis": "multi_source", - "rag.document_coverage": 1, - "rag.information_density": 0.22375296912114015 - }, - "events": [], - "links": [] - } - ], - "created_at": "2025-10-16 08:47:52.037000000", - "updated_at": "2025-10-16 08:47:52.037000000" - }, - { - "trace_id": "8d33a529-d9b1-4208-b528-762222d5ef8c", - "name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "status": "ok", - "status_message": "", - "start_time": "2025-10-16 08:47:42.103000000", - "end_time": "2025-10-16 08:47:51.032000000", - "duration_ms": 8928.948, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "spans": [ - { - "span_id": "770ce93d-07ae-4b9f-ae15-c6e450634b9d", - "trace_id": "8d33a529-d9b1-4208-b528-762222d5ef8c", - "parent_span_id": "9977d5d6-a7ff-469e-bec7-5e4dfc38c149", - "name": "agent.llm-rag", - "start_time": "2025-10-16 08:47:42.103000000", - "end_time": "2025-10-16 08:47:51.032000000", - "duration_ms": 8928.644999999999, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "rag_agent", - "agent.operation": "llm-rag", - "agent.capabilities": [ - "document_retrieval", - "context_generation", - "response_generation" - ], - "agent.id": "noveum_rag_agent", - "input_query": "How do I integrate Noveum Trace in my application?", - "query_length": 50, - "query_type": "rag_query", - "output_response": "RAG Answer: To integrate Noveum Trace into your application, you can use either the Python SDK (`noveum-trace`) or the TypeScript SDK (`@noveum/trace`). Here\u2019s a quick start guide based on the documentation:\n\n1. ...", - "rag.context_length": 2803, - "rag.sources_count": 5, - "rag.answer_length": 1134, - "rag.mode": "retrieval_augmented_generation" - }, - "events": [], - "links": [] - }, - { - "span_id": "9977d5d6-a7ff-469e-bec7-5e4dfc38c149", - "trace_id": "8d33a529-d9b1-4208-b528-762222d5ef8c", - "parent_span_id": "f3a8fa53-5641-4a23-b4d3-762577a89151", - "name": "agent.query_routing", - "start_time": "2025-10-16 08:47:42.103000000", - "end_time": "2025-10-16 08:47:51.032000000", - "duration_ms": 8928.793000000001, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "How do I integrate Noveum Trace in my application?", - "query_length": 50, - "query_type": "routing_query", - "output_response": "Routed to RAG for query processing", - "router.classification": "RAG", - "router.confidence_score": 1, - "router.rag_keyword_score": 2, - "router.web_keyword_score": 0, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [] - }, - { - "span_id": "c678e222-ef74-48ee-8628-16461ab0f48f", - "trace_id": "8d33a529-d9b1-4208-b528-762222d5ef8c", - "parent_span_id": "9977d5d6-a7ff-469e-bec7-5e4dfc38c149", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-16 08:47:42.103000000", - "end_time": "2025-10-16 08:47:42.103000000", - "duration_ms": 0.027, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: How do I integrate Noveum Trace in my application?", - "query_length": 50, - "query_type": "routing_evaluation_query", - "classification.mode": "RAG", - "classification.rag_keyword_score": 2, - "classification.web_keyword_score": 0, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "noveum_specific", - "query.keyword_density": 0.2222222222222222, - "query.domain_affinity": "noveum", - "routing.decision": "Routed to RAG based on analysis", - "routing.rationale": "RAG score: 2, Web score: 0, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "Web Search", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.038461538461538464, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: RAG (Confidence: 1.00)" - }, - "events": [], - "links": [] - }, - { - "span_id": "f3a8fa53-5641-4a23-b4d3-762577a89151", - "trace_id": "8d33a529-d9b1-4208-b528-762222d5ef8c", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-16 08:47:42.103000000", - "end_time": "2025-10-16 08:47:51.032000000", - "duration_ms": 8928.868, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "How do I integrate Noveum Trace in my application?", - "query.length": 50, - "processing.mode": "RAG", - "processing.time_seconds": 8.928829431533813, - "processing.response_length": 1134, - "processing.sources_count": 5, - "output_response": "Final Answer: To integrate Noveum Trace into your application, you can use either the Python SDK (`noveum-trace`) or the TypeScript SDK (`@noveum/trace`). Here\u2019s a quick start guide based on the documentation:\n\n1. ...", - "final_answer_mode": "RAG", - "query_processed.input_query": "How do I integrate Noveum Trace in my application?", - "query_processed.output_response": "Successfully processed query using RAG, generated 1134 character response", - "query_processed.mode": "RAG", - "query_processed.processing_time": 8.928829431533813, - "query_processed.response_length": 1134 - }, - "events": [], - "links": [] - }, - { - "span_id": "f459b71f-1d43-41b0-b9bf-d94a908b7afa", - "trace_id": "8d33a529-d9b1-4208-b528-762222d5ef8c", - "parent_span_id": "770ce93d-07ae-4b9f-ae15-c6e450634b9d", - "name": "agent.llm_model_execution", - "start_time": "2025-10-16 08:47:43.044000000", - "end_time": "2025-10-16 08:47:50.008000000", - "duration_ms": 6963.959, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: How do I integrate Noveum Trace in my application?", - "query_length": 50, - "query_type": "rag_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 6.9639081954956055, - "model.latency_ms": 6963.9081954956055, - "model.start_time": 1760604463.044064, - "model.end_time": 1760604470.0079722, - "model.performance_tier": "slow", - "model.prompt_tokens": 837, - "model.completion_tokens": 283, - "model.total_tokens": 1120, - "model.tokens_per_second": 160.8292310235276, - "model.estimated_cost": 0.0336, - "model.efficiency_score": 1.0125, - "model.response_length": 1134, - "model.response_quality": "high", - "model.output_response": "Model Response: To integrate Noveum Trace into your application, you can use either the Python SDK (`noveum-trace`) or the TypeScript SDK (`@noveum/trace`). Here\u2019s a quick start guide based on the documentation:\n\n1. ...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [] - }, - { - "span_id": "b8627171-b081-4b08-b900-c24375b73c3f", - "trace_id": "8d33a529-d9b1-4208-b528-762222d5ef8c", - "parent_span_id": "770ce93d-07ae-4b9f-ae15-c6e450634b9d", - "name": "agent.rag_evaluation_metrics", - "start_time": "2025-10-16 08:47:50.008000000", - "end_time": "2025-10-16 08:47:50.008000000", - "duration_ms": 0.041, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "rag_evaluation_metrics", - "agent.capabilities": [ - "retrieval_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: How do I integrate Noveum Trace in my application?", - "query_length": 50, - "query_type": "rag_evaluation_query", - "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\n\ud83d\udcd8 TypeScript SDK (\n@noveum/trace\n)- Framework ...", - "retrieval.context_length": 2803, - "retrieval.sources_count": 5, - "retrieval.context_quality": "high", - "retrieval.effectiveness": 1, - "retrieval.context_utilization": 2.803, - "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\n\ud83d\udcd8 TypeScript SDK (\n@noveum/trace\n)- Framework integrations for Next.js, Express.js, and Hono\n- TypeScript-first design with full type safety\n- Universal compatibility (Node.js, Edge Runtime, browsers)\n-\n\ud83d\udcca Noveum Platform\n- Real-time dashboard for analyzing traces and performance\n- Ad...\n\nSource 2 (https://noveum.ai/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\n\ud83d\udcd8 TypeScript SDK (\n@noveum/trace\n)- Framework integrations for Next.js, Express.js, and Hono\n- TypeScript-first design with full type safety\n- Universal compatibility (Node.js, Edge Runtime, browsers)\n-\n\ud83d\udcca Noveum Platform\n- Real-time dashboard for analyzing traces and performance\n- Ad...\n\nSource 3 (https://noveum.ai/docs/getting-started/sdk-integration):\nSDK Integration Guide\nIntegrate Noveum.ai tracing into your AI applications with Python or TypeScript SDKs\nThe Noveum.ai SDKs provide comprehensive tracing and observability for your AI applications with minimal code changes. Whether you're building LLM applications, RAG systems, or multi-agent workflows, our SDKs automatically capture essential metrics and traces.\n\ud83d\ude80 Quick Start\n1. Create Your Account & Get API Key\n- Sign up at noveum.ai\n- Create a project in your dashboard\n- Generate an API key...\n\nSource 4 (https://noveum.ai/en/docs/getting-started/sdk-integration):\nSDK Integration Guide\nIntegrate Noveum.ai tracing into your AI applications with Python or TypeScript SDKs\nThe Noveum.ai SDKs provide comprehensive tracing and observability for your AI applications with minimal code changes. Whether you're building LLM applications, RAG systems, or multi-agent workflows, our SDKs automatically capture essential metrics and traces.\n\ud83d\ude80 Quick Start\n1. Create Your Account & Get API Key\n- Sign up at noveum.ai\n- Create a project in your dashboard\n- Generate an API key...\n\nSource 5 (https://noveum.ai/en/docs/getting-started/sdk-integration):\n\ud83d\udd27 Framework Integrations\nNext.js Integration\nExpress.js Integration\nFastAPI Integration (Python)\n\ud83d\udcca Advanced Features\nCustom Attributes & Events\nSampling Configuration\n\ud83d\udd17 What's Captured Automatically\n- \ud83d\udcca Performance Metrics: Latency, throughput, error rates\n- \ud83d\udcb0 Cost Tracking: Token usage, API costs across providers\n- \ud83d\udd0d Request/Response: Configurable capture of inputs/outputs\n- \ud83c\udff7\ufe0f Metadata: Model names, parameters, user context\n- \ud83c\udf0a Context Flow: Trace relationships across services\n- \ud83d\udc1b Error Detail...\n\nUser Question: How do I integrate Noveum Trace in my application?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", - "prompt.prompt_length": 3349, - "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frame...", - "prompt.instruction_following": "rag_optimized", - "response.answer_length": 1134, - "response.answer_completeness": "complete", - "response.response_quality": "high", - "response.source_citation": 5, - "response.context_utilization": 0.4045665358544417, - "output_response": "RAG Answer: To integrate Noveum Trace into your application, you can use either the Python SDK (`noveum-trace`) or the TypeScript SDK (`@noveum/trace`). Here\u2019s a quick start guide based on the documentation:\n\n1. ...", - "evaluation.retrieval_effectiveness": 1, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 5, - "evaluation.context_relevance": "high", - "evaluation.overall_quality": "high", - "evaluation.ready_for_production": true, - "rag.retrieval_strategy": "semantic_similarity", - "rag.vector_search_results": 5, - "rag.context_synthesis": "multi_source", - "rag.document_coverage": 1, - "rag.information_density": 0.4045665358544417 - }, - "events": [], - "links": [] - } - ], - "created_at": "2025-10-16 08:47:42.103000000", - "updated_at": "2025-10-16 08:47:42.103000000" - }, - { - "trace_id": "06192a09-b6bb-4548-8a94-e123d6caaffe", - "name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "status": "ok", - "status_message": "", - "start_time": "2025-10-16 08:47:03.776000000", - "end_time": "2025-10-16 08:47:41.100000000", - "duration_ms": 37324.601, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "spans": [ - { - "span_id": "0b3d6893-776e-4118-b8b5-b1c5ea51def1", - "trace_id": "06192a09-b6bb-4548-8a94-e123d6caaffe", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-16 08:47:03.776000000", - "end_time": "2025-10-16 08:47:41.100000000", - "duration_ms": 37324.516, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What is Noveum and what does it do?", - "query.length": 35, - "processing.mode": "RAG", - "processing.time_seconds": 37.324477195739746, - "processing.response_length": 648, - "processing.sources_count": 5, - "output_response": "Final Answer: Noveum.ai is a comprehensive tracing and observability platform specifically designed for AI applications, including those powered by large language models (LLMs), retrieval-augmented generation (RAG)...", - "final_answer_mode": "RAG", - "query_processed.input_query": "What is Noveum and what does it do?", - "query_processed.output_response": "Successfully processed query using RAG, generated 648 character response", - "query_processed.mode": "RAG", - "query_processed.processing_time": 37.324477195739746, - "query_processed.response_length": 648 - }, - "events": [], - "links": [] - }, - { - "span_id": "137ad05b-bada-42be-abe5-be0c277f6d0e", - "trace_id": "06192a09-b6bb-4548-8a94-e123d6caaffe", - "parent_span_id": "c3664d20-1708-45f5-a051-b626b0ec77cd", - "name": "agent.llm-rag", - "start_time": "2025-10-16 08:47:03.776000000", - "end_time": "2025-10-16 08:47:41.100000000", - "duration_ms": 37324.042, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "rag_agent", - "agent.operation": "llm-rag", - "agent.capabilities": [ - "document_retrieval", - "context_generation", - "response_generation" - ], - "agent.id": "noveum_rag_agent", - "input_query": "What is Noveum and what does it do?", - "query_length": 35, - "query_type": "rag_query", - "output_response": "RAG Answer: Noveum.ai is a comprehensive tracing and observability platform specifically designed for AI applications, including those powered by large language models (LLMs), retrieval-augmented generation (RAG)...", - "rag.context_length": 2334, - "rag.sources_count": 5, - "rag.answer_length": 648, - "rag.mode": "retrieval_augmented_generation" - }, - "events": [], - "links": [] - }, - { - "span_id": "c3664d20-1708-45f5-a051-b626b0ec77cd", - "trace_id": "06192a09-b6bb-4548-8a94-e123d6caaffe", - "parent_span_id": "0b3d6893-776e-4118-b8b5-b1c5ea51def1", - "name": "agent.query_routing", - "start_time": "2025-10-16 08:47:03.776000000", - "end_time": "2025-10-16 08:47:41.100000000", - "duration_ms": 37324.382999999994, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What is Noveum and what does it do?", - "query_length": 35, - "query_type": "routing_query", - "output_response": "Routed to RAG for query processing", - "router.classification": "RAG", - "router.confidence_score": 1, - "router.rag_keyword_score": 1, - "router.web_keyword_score": 0, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [] - }, - { - "span_id": "ce9a94bf-1a72-4599-8561-11e66802b92d", - "trace_id": "06192a09-b6bb-4548-8a94-e123d6caaffe", - "parent_span_id": "c3664d20-1708-45f5-a051-b626b0ec77cd", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-16 08:47:03.776000000", - "end_time": "2025-10-16 08:47:03.776000000", - "duration_ms": 0.067, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What is Noveum and what does it do?", - "query_length": 35, - "query_type": "routing_evaluation_query", - "classification.mode": "RAG", - "classification.rag_keyword_score": 1, - "classification.web_keyword_score": 0, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "noveum_specific", - "query.keyword_density": 0.125, - "query.domain_affinity": "noveum", - "routing.decision": "Routed to RAG based on analysis", - "routing.rationale": "RAG score: 1, Web score: 0, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "Web Search", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.019230769230769232, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: RAG (Confidence: 1.00)" - }, - "events": [], - "links": [] - }, - { - "span_id": "38414416-907a-4188-95fe-c0a5f900f72c", - "trace_id": "06192a09-b6bb-4548-8a94-e123d6caaffe", - "parent_span_id": "137ad05b-bada-42be-abe5-be0c277f6d0e", - "name": "agent.llm_model_execution", - "start_time": "2025-10-16 08:47:10.392000000", - "end_time": "2025-10-16 08:47:40.392000000", - "duration_ms": 30000.104, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What is Noveum and what does it do?", - "query_length": 35, - "query_type": "rag_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 30.000043392181396, - "model.latency_ms": 30000.043392181396, - "model.start_time": 1760604430.3921378, - "model.end_time": 1760604460.3921812, - "model.performance_tier": "slow", - "model.prompt_tokens": 716, - "model.completion_tokens": 162, - "model.total_tokens": 878, - "model.tokens_per_second": 29.266624335244266, - "model.estimated_cost": 0.026340000000000002, - "model.efficiency_score": 0.7380410022779044, - "model.response_length": 648, - "model.response_quality": "high", - "model.output_response": "Model Response: Noveum.ai is a comprehensive tracing and observability platform specifically designed for AI applications, including those powered by large language models (LLMs), retrieval-augmented generation (RAG)...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [] - }, - { - "span_id": "67e23649-b00b-4eb8-9a69-865e80f833b3", - "trace_id": "06192a09-b6bb-4548-8a94-e123d6caaffe", - "parent_span_id": "137ad05b-bada-42be-abe5-be0c277f6d0e", - "name": "agent.rag_evaluation_metrics", - "start_time": "2025-10-16 08:47:40.392000000", - "end_time": "2025-10-16 08:47:40.392000000", - "duration_ms": 0.029, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "rag_evaluation_metrics", - "agent.capabilities": [ - "retrieval_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What is Noveum and what does it do?", - "query_length": 35, - "query_type": "rag_evaluation_query", - "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, vector searches, data retrieval, agent reasoning, and more. Without proper observability, debugging ...", - "retrieval.context_length": 2334, - "retrieval.sources_count": 5, - "retrieval.context_quality": "high", - "retrieval.effectiveness": 1, - "retrieval.context_utilization": 2.334, - "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, vector searches, data retrieval, agent reasoning, and more. Without proper observability, debugging becomes a nightmare, optimization is guesswork, and understanding user interactions is nearly impossible.\nNoveum.ai solves this by providing comprehensive tracing and observability specifically designed for AI applications.\nKey benefits of using Noveum.ai:\n- Complete Visibility: Trace...\n\nSource 2 (https://noveum.ai/en/docs/getting-started/overview):\nNoveum.ai Overview\nComprehensive AI tracing and observability platform for LLM applications, RAG systems, and AI agents\nWelcome to Noveum.ai\u2014the comprehensive tracing and observability platform built specifically for AI applications. Whether you're building LLM-powered chatbots, RAG systems, multi-agent workflows, or any AI-driven application, Noveum provides the insights you need to understand, debug, and optimize your systems.\n\ud83c\udfaf Why AI Applications Need Specialized Observability\nTraditional mo...\n\nSource 3 (https://noveum.ai/docs/getting-started/overview):\nNoveum.ai Overview\nComprehensive AI tracing and observability platform for LLM applications, RAG systems, and AI agents\nWelcome to Noveum.ai\u2014the comprehensive tracing and observability platform built specifically for AI applications. Whether you're building LLM-powered chatbots, RAG systems, multi-agent workflows, or any AI-driven application, Noveum provides the insights you need to understand, debug, and optimize your systems.\n\ud83c\udfaf Why AI Applications Need Specialized Observability\nTraditional mo...\n\nSource 4 (https://noveum.ai/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nSource 5 (https://noveum.ai/en/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nUser Question: What is Noveum and what does it do?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", - "prompt.prompt_length": 2865, - "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, ...", - "prompt.instruction_following": "rag_optimized", - "response.answer_length": 648, - "response.answer_completeness": "complete", - "response.response_quality": "high", - "response.source_citation": 5, - "response.context_utilization": 0.2776349614395887, - "output_response": "RAG Answer: Noveum.ai is a comprehensive tracing and observability platform specifically designed for AI applications, including those powered by large language models (LLMs), retrieval-augmented generation (RAG)...", - "evaluation.retrieval_effectiveness": 1, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 5, - "evaluation.context_relevance": "high", - "evaluation.overall_quality": "high", - "evaluation.ready_for_production": true, - "rag.retrieval_strategy": "semantic_similarity", - "rag.vector_search_results": 5, - "rag.context_synthesis": "multi_source", - "rag.document_coverage": 1, - "rag.information_density": 0.2776349614395887 - }, - "events": [], - "links": [] - } - ], - "created_at": "2025-10-16 08:47:03.776000000", - "updated_at": "2025-10-16 08:47:03.776000000" - }, - { - "trace_id": "3552ee8e-eafb-499a-b109-a9fb851a41d2", - "name": "auto_trace_system_initialization", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "status": "ok", - "status_message": "", - "start_time": "2025-10-16 08:47:03.695000000", - "end_time": "2025-10-16 08:47:03.770000000", - "duration_ms": 74.912, - "span_count": 1, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.9" - }, - "attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.9", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "spans": [ - { - "span_id": "050851c9-b2d3-4c82-accb-4085dc69f785", - "trace_id": "3552ee8e-eafb-499a-b109-a9fb851a41d2", - "parent_span_id": "", - "name": "system_initialization", - "start_time": "2025-10-16 08:47:03.695000000", - "end_time": "2025-10-16 08:47:03.770000000", - "duration_ms": 74.58, - "status": "ok", - "status_message": "", - "attributes": { - "system.force_scrape": false, - "system.config": { - "noveum_base_url": "https://noveum.ai", - "max_pages_to_scrape": 50, - "chunk_size": 1000, - "chunk_overlap": 200, - "max_search_results": 5, - "rag_threshold": 0.7, - "noveum_docs_file": "noveum_docs.json", - "vector_store_path": "noveum_vectorstore" - }, - "input_query": "Initialize system with force_scrape=False", - "output_response": "System initialization: RAG system loaded, vector store ready, agent operational", - "system.initialized": true, - "system.documents_loaded": 38, - "system.vectorstore_ready": true - }, - "events": [], - "links": [] - } - ], - "created_at": "2025-10-16 08:47:03.695000000", - "updated_at": "2025-10-16 08:47:03.695000000" - }, - { - "trace_id": "e6ef2449-7429-465b-9bff-f3def8de5bb0", - "name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "status": "ok", - "status_message": "", - "start_time": "2025-10-14 12:54:06.700000000", - "end_time": "2025-10-14 12:54:10.637000000", - "duration_ms": 3937.729, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "spans": [ - { - "span_id": "6b53235d-8b91-4d9e-8437-915828e84600", - "trace_id": "e6ef2449-7429-465b-9bff-f3def8de5bb0", - "parent_span_id": "e73ab184-50c1-4209-9276-be11e69e9513", - "name": "agent.query_routing", - "start_time": "2025-10-14 12:54:06.700000000", - "end_time": "2025-10-14 12:54:10.637000000", - "duration_ms": 3937.492, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What are the latest breakthroughs in artificial intelligence?", - "query_length": 61, - "query_type": "routing_query", - "output_response": "Routed to Web Search for query processing", - "router.classification": "Web Search", - "router.confidence_score": 1, - "router.rag_keyword_score": 0, - "router.web_keyword_score": 1, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [] - }, - { - "span_id": "a7d1e33a-95a1-41a9-a117-62b4c373bd60", - "trace_id": "e6ef2449-7429-465b-9bff-f3def8de5bb0", - "parent_span_id": "6b53235d-8b91-4d9e-8437-915828e84600", - "name": "agent.web_search_generation", - "start_time": "2025-10-14 12:54:06.700000000", - "end_time": "2025-10-14 12:54:10.637000000", - "duration_ms": 3936.983, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "web_search_agent", - "agent.operation": "web_search_generation", - "agent.capabilities": [ - "web_search", - "content_synthesis", - "response_generation" - ], - "agent.id": "noveum_web_search_agent", - "input_query": "What are the latest breakthroughs in artificial intelligence?", - "query_length": 61, - "query_type": "web_search_query", - "output_response": "Web Search Answer: The web search did not yield specific results regarding the latest breakthroughs in artificial intelligence. However, I can provide a general overview based on knowledge up to October 2023.\n\nRecent br...", - "web_search.results_count": 1, - "web_search.context_length": 163, - "web_search.response_length": 1090, - "web_search.mode": "external_web_search" - }, - "events": [], - "links": [] - }, - { - "span_id": "da630cab-2ba6-4f12-98b9-6ab1c0899c17", - "trace_id": "e6ef2449-7429-465b-9bff-f3def8de5bb0", - "parent_span_id": "6b53235d-8b91-4d9e-8437-915828e84600", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-14 12:54:06.700000000", - "end_time": "2025-10-14 12:54:06.700000000", - "duration_ms": 0.27999999999999997, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What are the latest breakthroughs in artificial intelligence?", - "query_length": 61, - "query_type": "routing_evaluation_query", - "classification.mode": "Web Search", - "classification.rag_keyword_score": 0, - "classification.web_keyword_score": 1, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "complex", - "query.intent": "general_knowledge", - "query.keyword_density": 0.125, - "query.domain_affinity": "general", - "routing.decision": "Routed to Web Search based on analysis", - "routing.rationale": "RAG score: 0, Web score: 1, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "RAG", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.019230769230769232, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: Web Search (Confidence: 1.00)" - }, - "events": [], - "links": [] - }, - { - "span_id": "e73ab184-50c1-4209-9276-be11e69e9513", - "trace_id": "e6ef2449-7429-465b-9bff-f3def8de5bb0", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-14 12:54:06.700000000", - "end_time": "2025-10-14 12:54:10.637000000", - "duration_ms": 3937.663, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What are the latest breakthroughs in artificial intelligence?", - "query.length": 61, - "processing.mode": "Web Search", - "processing.time_seconds": 3.93752384185791, - "processing.response_length": 1090, - "processing.sources_count": 1, - "output_response": "Final Answer: The web search did not yield specific results regarding the latest breakthroughs in artificial intelligence. However, I can provide a general overview based on knowledge up to October 2023.\n\nRecent br...", - "final_answer_mode": "Web Search", - "query_processed.input_query": "What are the latest breakthroughs in artificial intelligence?", - "query_processed.output_response": "Successfully processed query using Web Search, generated 1090 character response", - "query_processed.mode": "Web Search", - "query_processed.processing_time": 3.93752384185791, - "query_processed.response_length": 1090 - }, - "events": [], - "links": [] - }, - { - "span_id": "bb47766c-afda-44fe-8838-7936c80cd4b6", - "trace_id": "e6ef2449-7429-465b-9bff-f3def8de5bb0", - "parent_span_id": "a7d1e33a-95a1-41a9-a117-62b4c373bd60", - "name": "agent.llm_model_execution", - "start_time": "2025-10-14 12:54:07.104000000", - "end_time": "2025-10-14 12:54:10.637000000", - "duration_ms": 3533.321, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What are the latest breakthroughs in artificial intelligence?", - "query_length": 61, - "query_type": "web_search_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 3.533268928527832, - "model.latency_ms": 3533.268928527832, - "model.start_time": 1760446447.104266, - "model.end_time": 1760446450.6375349, - "model.performance_tier": "medium", - "model.prompt_tokens": 167, - "model.completion_tokens": 272, - "model.total_tokens": 439, - "model.tokens_per_second": 124.2475477752307, - "model.estimated_cost": 0.013170000000000001, - "model.efficiency_score": 2.4829157175398633, - "model.response_length": 1090, - "model.response_quality": "high", - "model.output_response": "Model Response: The web search did not yield specific results regarding the latest breakthroughs in artificial intelligence. However, I can provide a general overview based on knowledge up to October 2023.\n\nRecent br...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [] - }, - { - "span_id": "1d914a9d-b770-45dc-8308-6a36e9e0db89", - "trace_id": "e6ef2449-7429-465b-9bff-f3def8de5bb0", - "parent_span_id": "a7d1e33a-95a1-41a9-a117-62b4c373bd60", - "name": "agent.web_search_evaluation_metrics", - "start_time": "2025-10-14 12:54:10.637000000", - "end_time": "2025-10-14 12:54:10.637000000", - "duration_ms": 0.022, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "web_search_evaluation_metrics", - "agent.capabilities": [ - "web_search_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What are the latest breakthroughs in artificial intelligence?", - "query_length": 61, - "query_type": "web_search_evaluation_query", - "web_search.results_count": 1, - "web_search.context_length": 163, - "web_search.context_synthesized": "Context from 1 web sources", - "web_search.search_effectiveness": 0.2, - "web_search.context_quality": "low", - "web_search.source_diversity": 1, - "web_search.information_synthesis": "low", - "web_search.external_knowledge_utilization": 0.163, - "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What+are+the+latest+breakthroughs+in+artificial+intelligence?\n\nUser Question: What are the latest breakthroughs in artificial intelligence?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", - "prompt.prompt_length": 669, - "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What+are+the+latest+breakthroughs+in+artificial+intelligence?", - "prompt.instruction_following": "web_search_optimized", - "response.answer_length": 1090, - "response.answer_completeness": "complete", - "response.response_quality": "medium", - "response.source_citation": 1, - "response.context_utilization": 6.6871165644171775, - "output_response": "Web Search Answer: The web search did not yield specific results regarding the latest breakthroughs in artificial intelligence. However, I can provide a general overview based on knowledge up to October 2023.\n\nRecent br...", - "evaluation.search_effectiveness": 0.2, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 1, - "evaluation.context_relevance": "low", - "evaluation.overall_quality": "low", - "evaluation.ready_for_production": true, - "web_search.search_strategy": "duckduckgo_api", - "web_search.real_time_data": true, - "web_search.external_sources": 1, - "web_search.information_freshness": "current", - "web_search.knowledge_synthesis": "single_source" - }, - "events": [], - "links": [] - } - ], - "created_at": "2025-10-14 12:54:06.700000000", - "updated_at": "2025-10-14 12:54:06.700000000" - }, - { - "trace_id": "2ba8cab0-2366-4add-8ac3-b490a05e9a60", - "name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "status": "ok", - "status_message": "", - "start_time": "2025-10-14 12:53:56.480000000", - "end_time": "2025-10-14 12:54:05.687000000", - "duration_ms": 9207.67, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "spans": [ - { - "span_id": "91f3999c-2edf-4c56-8f4d-ab43b6ebbaee", - "trace_id": "2ba8cab0-2366-4add-8ac3-b490a05e9a60", - "parent_span_id": "cbe51252-a323-4852-a90e-c907a0480fa3", - "name": "agent.web_search_generation", - "start_time": "2025-10-14 12:53:56.480000000", - "end_time": "2025-10-14 12:54:05.685000000", - "duration_ms": 9205.212, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "web_search_agent", - "agent.operation": "web_search_generation", - "agent.capabilities": [ - "web_search", - "content_synthesis", - "response_generation" - ], - "agent.id": "noveum_web_search_agent", - "input_query": "What's happening in the software development world today?", - "query_length": 57, - "query_type": "web_search_query", - "output_response": "Web Search Answer: The software development world is currently experiencing rapid changes driven by advancements in artificial intelligence (AI) and evolving programming practices. Key trends include:\n\n1. **AI Integrati...", - "web_search.results_count": 1, - "web_search.context_length": 975, - "web_search.response_length": 1636, - "web_search.mode": "external_web_search" - }, - "events": [], - "links": [] - }, - { - "span_id": "ea2aee75-4537-46ea-954b-2002fc0026c5", - "trace_id": "2ba8cab0-2366-4add-8ac3-b490a05e9a60", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-14 12:53:56.480000000", - "end_time": "2025-10-14 12:54:05.687000000", - "duration_ms": 9207.528999999999, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What's happening in the software development world today?", - "query.length": 57, - "processing.mode": "Web Search", - "processing.time_seconds": 9.206756830215454, - "processing.response_length": 1636, - "processing.sources_count": 1, - "output_response": "Final Answer: The software development world is currently experiencing rapid changes driven by advancements in artificial intelligence (AI) and evolving programming practices. Key trends include:\n\n1. **AI Integrati...", - "final_answer_mode": "Web Search", - "query_processed.input_query": "What's happening in the software development world today?", - "query_processed.output_response": "Successfully processed query using Web Search, generated 1636 character response", - "query_processed.mode": "Web Search", - "query_processed.processing_time": 9.206756830215454, - "query_processed.response_length": 1636 - }, - "events": [], - "links": [] - }, - { - "span_id": "cbe51252-a323-4852-a90e-c907a0480fa3", - "trace_id": "2ba8cab0-2366-4add-8ac3-b490a05e9a60", - "parent_span_id": "ea2aee75-4537-46ea-954b-2002fc0026c5", - "name": "agent.query_routing", - "start_time": "2025-10-14 12:53:56.480000000", - "end_time": "2025-10-14 12:54:05.686000000", - "duration_ms": 9205.569000000001, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What's happening in the software development world today?", - "query_length": 57, - "query_type": "routing_query", - "output_response": "Routed to Web Search for query processing", - "router.classification": "Web Search", - "router.confidence_score": 1, - "router.rag_keyword_score": 0, - "router.web_keyword_score": 1, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [] - }, - { - "span_id": "d850ecee-86ef-4818-9cec-638bfee26293", - "trace_id": "2ba8cab0-2366-4add-8ac3-b490a05e9a60", - "parent_span_id": "cbe51252-a323-4852-a90e-c907a0480fa3", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-14 12:53:56.480000000", - "end_time": "2025-10-14 12:53:56.480000000", - "duration_ms": 0.041999999999999996, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What's happening in the software development world today?", - "query_length": 57, - "query_type": "routing_evaluation_query", - "classification.mode": "Web Search", - "classification.rag_keyword_score": 0, - "classification.web_keyword_score": 1, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "complex", - "query.intent": "general_knowledge", - "query.keyword_density": 0.125, - "query.domain_affinity": "general", - "routing.decision": "Routed to Web Search based on analysis", - "routing.rationale": "RAG score: 0, Web score: 1, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "RAG", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.019230769230769232, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: Web Search (Confidence: 1.00)" - }, - "events": [], - "links": [] - }, - { - "span_id": "0d4387a4-adf3-42c1-aca7-51453a90dd36", - "trace_id": "2ba8cab0-2366-4add-8ac3-b490a05e9a60", - "parent_span_id": "91f3999c-2edf-4c56-8f4d-ab43b6ebbaee", - "name": "agent.llm_model_execution", - "start_time": "2025-10-14 12:53:59.426000000", - "end_time": "2025-10-14 12:54:05.684000000", - "duration_ms": 6258.643, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What's happening in the software development world today?", - "query_length": 57, - "query_type": "web_search_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 6.25848388671875, - "model.latency_ms": 6258.48388671875, - "model.start_time": 1760446439.426367, - "model.end_time": 1760446445.684851, - "model.performance_tier": "slow", - "model.prompt_tokens": 369, - "model.completion_tokens": 409, - "model.total_tokens": 778, - "model.tokens_per_second": 124.31125718019486, - "model.estimated_cost": 0.02334, - "model.efficiency_score": 2.102827763496144, - "model.response_length": 1636, - "model.response_quality": "high", - "model.output_response": "Model Response: The software development world is currently experiencing rapid changes driven by advancements in artificial intelligence (AI) and evolving programming practices. Key trends include:\n\n1. **AI Integrati...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [] - }, - { - "span_id": "717e3c4a-6300-45e5-9456-caa3e773d758", - "trace_id": "2ba8cab0-2366-4add-8ac3-b490a05e9a60", - "parent_span_id": "91f3999c-2edf-4c56-8f4d-ab43b6ebbaee", - "name": "agent.web_search_evaluation_metrics", - "start_time": "2025-10-14 12:54:05.685000000", - "end_time": "2025-10-14 12:54:05.685000000", - "duration_ms": 0.044, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "web_search_evaluation_metrics", - "agent.capabilities": [ - "web_search_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What's happening in the software development world today?", - "query_length": 57, - "query_type": "web_search_evaluation_query", - "web_search.results_count": 1, - "web_search.context_length": 975, - "web_search.context_synthesized": "Context from 1 web sources", - "web_search.search_effectiveness": 0.2, - "web_search.context_quality": "high", - "web_search.source_diversity": 1, - "web_search.information_synthesis": "low", - "web_search.external_knowledge_utilization": 0.975, - "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nLog in to WhatsApp Web for simple, reliable and private messaging on your desktop. Send and receive messages and files with ease, all for free. Sep 23, 2025 \u00b7 Chief Economists' Outlook 2025 states that these under-the-radar trends are shaping the global economy, from the impact of AI on research productivity to a tightened labor \u2026 Sep 1, 2025 \u00b7 THE AI DEVELOPMENT REVOLUTION: WHAT'S REALLY HAPPENING? Let's face it - the software development world is changing faster than a React component re-rendering on \u2026 Oct 7, 2025 \u00b7 Computer Programming Research. Read current computer science articles on everything from computer programs to detect cancer genes and control vehicle maintenance to \u2026 Aug 24, 2025 \u00b7 Artificial Intelligence News. Everything on AI including futuristic robots with artificial intelligence, computer models of human intelligence and more.\nURL: https://duckduckgo.com/?q=What's+happening+in+the+software+development+world+today?\n\nUser Question: What's happening in the software development world today?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", - "prompt.prompt_length": 1477, - "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nLog in to WhatsApp Web for simple, reliable and private messaging on your desktop. Send and receive messages and files with ease, all for free. Sep 23, 2025 \u00b7 Chief Economi...", - "prompt.instruction_following": "web_search_optimized", - "response.answer_length": 1636, - "response.answer_completeness": "complete", - "response.response_quality": "medium", - "response.source_citation": 1, - "response.context_utilization": 1.677948717948718, - "output_response": "Web Search Answer: The software development world is currently experiencing rapid changes driven by advancements in artificial intelligence (AI) and evolving programming practices. Key trends include:\n\n1. **AI Integrati...", - "evaluation.search_effectiveness": 0.2, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 1, - "evaluation.context_relevance": "high", - "evaluation.overall_quality": "low", - "evaluation.ready_for_production": true, - "web_search.search_strategy": "duckduckgo_api", - "web_search.real_time_data": true, - "web_search.external_sources": 1, - "web_search.information_freshness": "current", - "web_search.knowledge_synthesis": "single_source" - }, - "events": [], - "links": [] - } - ], - "created_at": "2025-10-14 12:53:56.480000000", - "updated_at": "2025-10-14 12:53:56.480000000" - }, - { - "trace_id": "a7c3399e-372f-4aae-baeb-05215cc4b85f", - "name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "status": "ok", - "status_message": "", - "start_time": "2025-10-14 12:53:50.963000000", - "end_time": "2025-10-14 12:53:55.471000000", - "duration_ms": 4507.555, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "spans": [ - { - "span_id": "3d39f522-8ae7-4988-aafd-58b09734eff0", - "trace_id": "a7c3399e-372f-4aae-baeb-05215cc4b85f", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-14 12:53:50.963000000", - "end_time": "2025-10-14 12:53:55.471000000", - "duration_ms": 4507.466, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What are the newest features in cloud computing?", - "query.length": 48, - "processing.mode": "RAG", - "processing.time_seconds": 4.507421255111694, - "processing.response_length": 870, - "processing.sources_count": 5, - "output_response": "Final Answer: The provided context does not specifically mention any new features in cloud computing as a general topic. However, it does highlight several recent enhancements related to the Noveum.ai platform, whi...", - "final_answer_mode": "RAG", - "query_processed.input_query": "What are the newest features in cloud computing?", - "query_processed.output_response": "Successfully processed query using RAG, generated 870 character response", - "query_processed.mode": "RAG", - "query_processed.processing_time": 4.507421255111694, - "query_processed.response_length": 870 - }, - "events": [], - "links": [] - }, - { - "span_id": "54aa6700-f2de-4cf0-b1ad-a3d24ed692ed", - "trace_id": "a7c3399e-372f-4aae-baeb-05215cc4b85f", - "parent_span_id": "d17f31cf-c570-4604-8df3-532b2e530fce", - "name": "agent.llm-rag", - "start_time": "2025-10-14 12:53:50.963000000", - "end_time": "2025-10-14 12:53:55.471000000", - "duration_ms": 4507.083, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "rag_agent", - "agent.operation": "llm-rag", - "agent.capabilities": [ - "document_retrieval", - "context_generation", - "response_generation" - ], - "agent.id": "noveum_rag_agent", - "input_query": "What are the newest features in cloud computing?", - "query_length": 48, - "query_type": "rag_query", - "output_response": "RAG Answer: The provided context does not specifically mention any new features in cloud computing as a general topic. However, it does highlight several recent enhancements related to the Noveum.ai platform, whi...", - "rag.context_length": 2579, - "rag.sources_count": 5, - "rag.answer_length": 870, - "rag.mode": "retrieval_augmented_generation" - }, - "events": [], - "links": [] - }, - { - "span_id": "8878096f-6cfa-4f70-9844-d7723a0875f0", - "trace_id": "a7c3399e-372f-4aae-baeb-05215cc4b85f", - "parent_span_id": "d17f31cf-c570-4604-8df3-532b2e530fce", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-14 12:53:50.963000000", - "end_time": "2025-10-14 12:53:50.963000000", - "duration_ms": 0.036000000000000004, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What are the newest features in cloud computing?", - "query_length": 48, - "query_type": "routing_evaluation_query", - "classification.mode": "RAG", - "classification.rag_keyword_score": 1, - "classification.web_keyword_score": 0, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "documentation", - "query.keyword_density": 0.125, - "query.domain_affinity": "noveum", - "routing.decision": "Routed to RAG based on analysis", - "routing.rationale": "RAG score: 1, Web score: 0, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "Web Search", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.019230769230769232, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: RAG (Confidence: 1.00)" - }, - "events": [], - "links": [] - }, - { - "span_id": "d17f31cf-c570-4604-8df3-532b2e530fce", - "trace_id": "a7c3399e-372f-4aae-baeb-05215cc4b85f", - "parent_span_id": "3d39f522-8ae7-4988-aafd-58b09734eff0", - "name": "agent.query_routing", - "start_time": "2025-10-14 12:53:50.963000000", - "end_time": "2025-10-14 12:53:55.471000000", - "duration_ms": 4507.3730000000005, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What are the newest features in cloud computing?", - "query_length": 48, - "query_type": "routing_query", - "output_response": "Routed to RAG for query processing", - "router.classification": "RAG", - "router.confidence_score": 1, - "router.rag_keyword_score": 1, - "router.web_keyword_score": 0, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [] - }, - { - "span_id": "97ef01a8-5930-4902-909a-245c516a5871", - "trace_id": "a7c3399e-372f-4aae-baeb-05215cc4b85f", - "parent_span_id": "54aa6700-f2de-4cf0-b1ad-a3d24ed692ed", - "name": "agent.llm_model_execution", - "start_time": "2025-10-14 12:53:51.439000000", - "end_time": "2025-10-14 12:53:55.077000000", - "duration_ms": 3637.337, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What are the newest features in cloud computing?", - "query_length": 48, - "query_type": "rag_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 3.6372909545898438, - "model.latency_ms": 3637.2909545898438, - "model.start_time": 1760446431.439748, - "model.end_time": 1760446435.077039, - "model.performance_tier": "medium", - "model.prompt_tokens": 780, - "model.completion_tokens": 217, - "model.total_tokens": 997, - "model.tokens_per_second": 274.1050997698989, - "model.estimated_cost": 0.02991, - "model.efficiency_score": 0.872617853560682, - "model.response_length": 870, - "model.response_quality": "high", - "model.output_response": "Model Response: The provided context does not specifically mention any new features in cloud computing as a general topic. However, it does highlight several recent enhancements related to the Noveum.ai platform, whi...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [] - }, - { - "span_id": "ea792ec6-64b4-4ed5-b418-ba902e90c57a", - "trace_id": "a7c3399e-372f-4aae-baeb-05215cc4b85f", - "parent_span_id": "54aa6700-f2de-4cf0-b1ad-a3d24ed692ed", - "name": "agent.rag_evaluation_metrics", - "start_time": "2025-10-14 12:53:55.077000000", - "end_time": "2025-10-14 12:53:55.077000000", - "duration_ms": 0.034, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "rag_evaluation_metrics", - "agent.capabilities": [ - "retrieval_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What are the newest features in cloud computing?", - "query_length": 48, - "query_type": "rag_evaluation_query", - "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/changelog):\n- \ud83d\udcc8 Enhanced Dashboard Analytics: Real-time request tracking, latency monitoring, and cost analysis\n- \ud83d\udd0d Improved Logs Interface: Better search functionality, detailed trace views, and enhanced debugging capabilities\n- \ud83d\udc65 Team Management: Advanced member mana...", - "retrieval.context_length": 2579, - "retrieval.sources_count": 5, - "retrieval.context_quality": "high", - "retrieval.effectiveness": 1, - "retrieval.context_utilization": 2.579, - "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/changelog):\n- \ud83d\udcc8 Enhanced Dashboard Analytics: Real-time request tracking, latency monitoring, and cost analysis\n- \ud83d\udd0d Improved Logs Interface: Better search functionality, detailed trace views, and enhanced debugging capabilities\n- \ud83d\udc65 Team Management: Advanced member management with role-based access control and invitation system\n- \ud83d\udee1\ufe0f Security Improvements: Enhanced API key management and secure credential storage\n4 months ago\n- \ud83d\udcca Advanced Metrics Collection: Comprehensive telemetry system with custom metrics ...\n\nSource 2 (https://noveum.ai/en/changelog):\n- \ud83d\udcdd Documentation Updates: Comprehensive guides for integration and best practices\n5 months ago\n- \ud83c\udf89 Platform Foundation: Initial release of Noveum.ai AI observability platform\n- \ud83d\ude80 Core Features: AI Gateway integration, basic evaluation metrics, and monitoring capabilities\n- \ud83d\udcc8 Evaluation Jobs: Automated model evaluation with configurable metrics and reporting\n- \ud83d\udd0c Provider Integrations: Support for major AI providers and custom model deployments...\n\nSource 3 (https://noveum.ai/docs/platform/dashboard):\n- Bulk Export: Download multiple traces simultaneously\n- Comparative Analysis: Compare performance across multiple traces\nReal-Time Updates\n- Live Refresh: Automatic updates as new traces arrive\n- Connection Monitoring: Real-time status of your trace ingestion pipeline\n- Performance Indicators: Live metrics for system health monitoring\nReady to dive deeper? Explore Projects & Environments to organize your AI applications, or check out Team Collaboration to share insights with your team.\nGet Earl...\n\nSource 4 (https://noveum.ai/en/changelog):\nChangelog\nStay up to date with the latest changes in our product.\n2 months ago\n- \ud83c\udfaf Major Platform Evolution: NovaEval Framework - Launched comprehensive AI model evaluation framework with 20+ built-in scorers including accuracy, RAG metrics, conversational scoring, and specialized agent evaluation tools. Features production-grade deployment on Docker and Kubernetes. (GitHub | PyPI Package)\n- \ud83d\ude80 Next-Generation Traces UI - Complete redesign featuring three-pane layout with directory tree navigatio...\n\nSource 5 (https://noveum.ai/en/blog):\n8/12/2025\nDiscover how Noveum.ai provides comprehensive tracing and observability for AI applications, from development debugging to production optimization.\nShashank Agarwal\n3/3/2025\nDiscover how Noveum.ai provides comprehensive tracing and observability for LLM applications, RAG systems, and multi-agent workflows with our powerful Python and TypeScript SDKs.\nShashank Agarwal\n3/2/2025...\n\nUser Question: What are the newest features in cloud computing?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", - "prompt.prompt_length": 3123, - "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/changelog):\n- \ud83d\udcc8 Enhanced Dashboard Analytics: Real-time request tracking, latency monitoring, and cost analysis\n- \ud83d\udd0d Improved Logs Interface: Better search functionality,...", - "prompt.instruction_following": "rag_optimized", - "response.answer_length": 870, - "response.answer_completeness": "complete", - "response.response_quality": "high", - "response.source_citation": 5, - "response.context_utilization": 0.33734005428460645, - "output_response": "RAG Answer: The provided context does not specifically mention any new features in cloud computing as a general topic. However, it does highlight several recent enhancements related to the Noveum.ai platform, whi...", - "evaluation.retrieval_effectiveness": 1, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 5, - "evaluation.context_relevance": "high", - "evaluation.overall_quality": "high", - "evaluation.ready_for_production": true, - "rag.retrieval_strategy": "semantic_similarity", - "rag.vector_search_results": 5, - "rag.context_synthesis": "multi_source", - "rag.document_coverage": 1, - "rag.information_density": 0.33734005428460645 - }, - "events": [], - "links": [] - } - ], - "created_at": "2025-10-14 12:53:50.963000000", - "updated_at": "2025-10-14 12:53:50.963000000" - }, - { - "trace_id": "d682797f-2ce0-448e-8f0d-891541430988", - "name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "status": "ok", - "status_message": "", - "start_time": "2025-10-14 12:53:44.917000000", - "end_time": "2025-10-14 12:53:49.957000000", - "duration_ms": 5039.64, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "spans": [ - { - "span_id": "e2d314c8-687f-4b8d-869c-8d7f2a22a4f3", - "trace_id": "d682797f-2ce0-448e-8f0d-891541430988", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-14 12:53:44.917000000", - "end_time": "2025-10-14 12:53:49.957000000", - "duration_ms": 5039.565, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What's the current status of cryptocurrency markets?", - "query.length": 52, - "processing.mode": "Web Search", - "processing.time_seconds": 5.0394370555877686, - "processing.response_length": 338, - "processing.sources_count": 1, - "output_response": "Final Answer: The web search results did not provide any specific information regarding the current status of cryptocurrency markets. Therefore, I cannot provide an update on market conditions, prices, or trends at...", - "final_answer_mode": "Web Search", - "query_processed.input_query": "What's the current status of cryptocurrency markets?", - "query_processed.output_response": "Successfully processed query using Web Search, generated 338 character response", - "query_processed.mode": "Web Search", - "query_processed.processing_time": 5.0394370555877686, - "query_processed.response_length": 338 - }, - "events": [], - "links": [] - }, - { - "span_id": "d367a6bd-86af-41c5-96ca-5284425775ce", - "trace_id": "d682797f-2ce0-448e-8f0d-891541430988", - "parent_span_id": "f137a412-6ddf-4ace-908e-9edf73534aa7", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-14 12:53:44.917000000", - "end_time": "2025-10-14 12:53:44.918000000", - "duration_ms": 0.046, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What's the current status of cryptocurrency markets?", - "query_length": 52, - "query_type": "routing_evaluation_query", - "classification.mode": "Web Search", - "classification.rag_keyword_score": 0, - "classification.web_keyword_score": 3, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "complex", - "query.intent": "general_knowledge", - "query.keyword_density": 0.42857142857142855, - "query.domain_affinity": "general", - "routing.decision": "Routed to Web Search based on analysis", - "routing.rationale": "RAG score: 0, Web score: 3, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "RAG", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.057692307692307696, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: Web Search (Confidence: 1.00)" - }, - "events": [], - "links": [] - }, - { - "span_id": "f137a412-6ddf-4ace-908e-9edf73534aa7", - "trace_id": "d682797f-2ce0-448e-8f0d-891541430988", - "parent_span_id": "e2d314c8-687f-4b8d-869c-8d7f2a22a4f3", - "name": "agent.query_routing", - "start_time": "2025-10-14 12:53:44.917000000", - "end_time": "2025-10-14 12:53:49.957000000", - "duration_ms": 5039.404, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What's the current status of cryptocurrency markets?", - "query_length": 52, - "query_type": "routing_query", - "output_response": "Routed to Web Search for query processing", - "router.classification": "Web Search", - "router.confidence_score": 1, - "router.rag_keyword_score": 0, - "router.web_keyword_score": 3, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [] - }, - { - "span_id": "987b6910-9b6d-4207-b0f3-8807a23e3ddb", - "trace_id": "d682797f-2ce0-448e-8f0d-891541430988", - "parent_span_id": "f137a412-6ddf-4ace-908e-9edf73534aa7", - "name": "agent.web_search_generation", - "start_time": "2025-10-14 12:53:44.918000000", - "end_time": "2025-10-14 12:53:49.957000000", - "duration_ms": 5039.248, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "web_search_agent", - "agent.operation": "web_search_generation", - "agent.capabilities": [ - "web_search", - "content_synthesis", - "response_generation" - ], - "agent.id": "noveum_web_search_agent", - "input_query": "What's the current status of cryptocurrency markets?", - "query_length": 52, - "query_type": "web_search_query", - "output_response": "Web Search Answer: The web search results did not provide any specific information regarding the current status of cryptocurrency markets. Therefore, I cannot provide an update on market conditions, prices, or trends at...", - "web_search.results_count": 1, - "web_search.context_length": 154, - "web_search.response_length": 338, - "web_search.mode": "external_web_search" - }, - "events": [], - "links": [] - }, - { - "span_id": "b2614d0e-5fcd-46ec-8d25-f1b04eb142af", - "trace_id": "d682797f-2ce0-448e-8f0d-891541430988", - "parent_span_id": "987b6910-9b6d-4207-b0f3-8807a23e3ddb", - "name": "agent.llm_model_execution", - "start_time": "2025-10-14 12:53:45.255000000", - "end_time": "2025-10-14 12:53:49.957000000", - "duration_ms": 4701.744, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What's the current status of cryptocurrency markets?", - "query_length": 52, - "query_type": "web_search_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 4.701629161834717, - "model.latency_ms": 4701.629161834717, - "model.start_time": 1760446425.2554948, - "model.end_time": 1760446429.957124, - "model.performance_tier": "medium", - "model.prompt_tokens": 162, - "model.completion_tokens": 84, - "model.total_tokens": 246, - "model.tokens_per_second": 52.32228904756993, - "model.estimated_cost": 0.00738, - "model.efficiency_score": 1.3739837398373984, - "model.response_length": 338, - "model.response_quality": "high", - "model.output_response": "Model Response: The web search results did not provide any specific information regarding the current status of cryptocurrency markets. Therefore, I cannot provide an update on market conditions, prices, or trends at...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [] - }, - { - "span_id": "663e8789-75d0-4c4d-b14d-4088022436ac", - "trace_id": "d682797f-2ce0-448e-8f0d-891541430988", - "parent_span_id": "987b6910-9b6d-4207-b0f3-8807a23e3ddb", - "name": "agent.web_search_evaluation_metrics", - "start_time": "2025-10-14 12:53:49.957000000", - "end_time": "2025-10-14 12:53:49.957000000", - "duration_ms": 0.032, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "web_search_evaluation_metrics", - "agent.capabilities": [ - "web_search_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What's the current status of cryptocurrency markets?", - "query_length": 52, - "query_type": "web_search_evaluation_query", - "web_search.results_count": 1, - "web_search.context_length": 154, - "web_search.context_synthesized": "Context from 1 web sources", - "web_search.search_effectiveness": 0.2, - "web_search.context_quality": "low", - "web_search.source_diversity": 1, - "web_search.information_synthesis": "low", - "web_search.external_knowledge_utilization": 0.154, - "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What's+the+current+status+of+cryptocurrency+markets?\n\nUser Question: What's the current status of cryptocurrency markets?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", - "prompt.prompt_length": 651, - "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What's+the+current+status+of+cryptocurrency+markets?", - "prompt.instruction_following": "web_search_optimized", - "response.answer_length": 338, - "response.answer_completeness": "complete", - "response.response_quality": "medium", - "response.source_citation": 1, - "response.context_utilization": 2.1948051948051948, - "output_response": "Web Search Answer: The web search results did not provide any specific information regarding the current status of cryptocurrency markets. Therefore, I cannot provide an update on market conditions, prices, or trends at...", - "evaluation.search_effectiveness": 0.2, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 1, - "evaluation.context_relevance": "low", - "evaluation.overall_quality": "low", - "evaluation.ready_for_production": true, - "web_search.search_strategy": "duckduckgo_api", - "web_search.real_time_data": true, - "web_search.external_sources": 1, - "web_search.information_freshness": "current", - "web_search.knowledge_synthesis": "single_source" - }, - "events": [], - "links": [] - } - ], - "created_at": "2025-10-14 12:53:44.917000000", - "updated_at": "2025-10-14 12:53:44.917000000" - }, - { - "trace_id": "b0858dca-4173-493b-a6c5-4e993f606477", - "name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "status": "ok", - "status_message": "", - "start_time": "2025-10-14 12:53:37.818000000", - "end_time": "2025-10-14 12:53:43.908000000", - "duration_ms": 6089.715, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "spans": [ - { - "span_id": "3382fe7c-aaca-46e0-a8d2-dbee75ff3bed", - "trace_id": "b0858dca-4173-493b-a6c5-4e993f606477", - "parent_span_id": "6cf94049-87b0-4fc5-b37e-32d261543647", - "name": "agent.query_routing", - "start_time": "2025-10-14 12:53:37.818000000", - "end_time": "2025-10-14 12:53:43.908000000", - "duration_ms": 6089.22, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What are the latest updates in Python programming?", - "query_length": 50, - "query_type": "routing_query", - "output_response": "Routed to Web Search for query processing", - "router.classification": "Web Search", - "router.confidence_score": 1, - "router.rag_keyword_score": 0, - "router.web_keyword_score": 2, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [] - }, - { - "span_id": "6cf94049-87b0-4fc5-b37e-32d261543647", - "trace_id": "b0858dca-4173-493b-a6c5-4e993f606477", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-14 12:53:37.818000000", - "end_time": "2025-10-14 12:53:43.908000000", - "duration_ms": 6089.607, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What are the latest updates in Python programming?", - "query.length": 50, - "processing.mode": "Web Search", - "processing.time_seconds": 6.089300870895386, - "processing.response_length": 1324, - "processing.sources_count": 1, - "output_response": "Final Answer: The web search did not yield specific results regarding the latest updates in Python programming. However, I can provide a general overview based on my knowledge up to October 2023.\n\nAs of late 2023, ...", - "final_answer_mode": "Web Search", - "query_processed.input_query": "What are the latest updates in Python programming?", - "query_processed.output_response": "Successfully processed query using Web Search, generated 1324 character response", - "query_processed.mode": "Web Search", - "query_processed.processing_time": 6.089300870895386, - "query_processed.response_length": 1324 - }, - "events": [], - "links": [] - }, - { - "span_id": "21c8befa-356a-4034-8d62-792238cb9e6e", - "trace_id": "b0858dca-4173-493b-a6c5-4e993f606477", - "parent_span_id": "3382fe7c-aaca-46e0-a8d2-dbee75ff3bed", - "name": "agent.web_search_generation", - "start_time": "2025-10-14 12:53:37.819000000", - "end_time": "2025-10-14 12:53:43.908000000", - "duration_ms": 6088.807, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "web_search_agent", - "agent.operation": "web_search_generation", - "agent.capabilities": [ - "web_search", - "content_synthesis", - "response_generation" - ], - "agent.id": "noveum_web_search_agent", - "input_query": "What are the latest updates in Python programming?", - "query_length": 50, - "query_type": "web_search_query", - "output_response": "Web Search Answer: The web search did not yield specific results regarding the latest updates in Python programming. However, I can provide a general overview based on my knowledge up to October 2023.\n\nAs of late 2023, ...", - "web_search.results_count": 1, - "web_search.context_length": 152, - "web_search.response_length": 1324, - "web_search.mode": "external_web_search" - }, - "events": [], - "links": [] - }, - { - "span_id": "482feedf-52f4-4c6a-9186-d59898820caa", - "trace_id": "b0858dca-4173-493b-a6c5-4e993f606477", - "parent_span_id": "3382fe7c-aaca-46e0-a8d2-dbee75ff3bed", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-14 12:53:37.819000000", - "end_time": "2025-10-14 12:53:37.819000000", - "duration_ms": 0.039, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What are the latest updates in Python programming?", - "query_length": 50, - "query_type": "routing_evaluation_query", - "classification.mode": "Web Search", - "classification.rag_keyword_score": 0, - "classification.web_keyword_score": 2, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "general_knowledge", - "query.keyword_density": 0.25, - "query.domain_affinity": "general", - "routing.decision": "Routed to Web Search based on analysis", - "routing.rationale": "RAG score: 0, Web score: 2, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "RAG", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.038461538461538464, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: Web Search (Confidence: 1.00)" - }, - "events": [], - "links": [] - }, - { - "span_id": "84f1f63f-ee05-411e-a7dd-4ee9e9826f5a", - "trace_id": "b0858dca-4173-493b-a6c5-4e993f606477", - "parent_span_id": "21c8befa-356a-4034-8d62-792238cb9e6e", - "name": "agent.llm_model_execution", - "start_time": "2025-10-14 12:53:38.191000000", - "end_time": "2025-10-14 12:53:43.907000000", - "duration_ms": 5716.078, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What are the latest updates in Python programming?", - "query_length": 50, - "query_type": "web_search_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 5.715963125228882, - "model.latency_ms": 5715.963125228882, - "model.start_time": 1760446418.191805, - "model.end_time": 1760446423.907768, - "model.performance_tier": "slow", - "model.prompt_tokens": 161, - "model.completion_tokens": 331, - "model.total_tokens": 492, - "model.tokens_per_second": 86.07473302765561, - "model.estimated_cost": 0.01476, - "model.efficiency_score": 2.6910569105691056, - "model.response_length": 1324, - "model.response_quality": "high", - "model.output_response": "Model Response: The web search did not yield specific results regarding the latest updates in Python programming. However, I can provide a general overview based on my knowledge up to October 2023.\n\nAs of late 2023, ...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [] - }, - { - "span_id": "2fb9ecf3-a497-4285-a694-1c95080d9e6a", - "trace_id": "b0858dca-4173-493b-a6c5-4e993f606477", - "parent_span_id": "21c8befa-356a-4034-8d62-792238cb9e6e", - "name": "agent.web_search_evaluation_metrics", - "start_time": "2025-10-14 12:53:43.908000000", - "end_time": "2025-10-14 12:53:43.908000000", - "duration_ms": 0.037, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "web_search_evaluation_metrics", - "agent.capabilities": [ - "web_search_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What are the latest updates in Python programming?", - "query_length": 50, - "query_type": "web_search_evaluation_query", - "web_search.results_count": 1, - "web_search.context_length": 152, - "web_search.context_synthesized": "Context from 1 web sources", - "web_search.search_effectiveness": 0.2, - "web_search.context_quality": "low", - "web_search.source_diversity": 1, - "web_search.information_synthesis": "low", - "web_search.external_knowledge_utilization": 0.152, - "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What+are+the+latest+updates+in+Python+programming?\n\nUser Question: What are the latest updates in Python programming?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", - "prompt.prompt_length": 647, - "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What+are+the+latest+updates+in+Python+programming?", - "prompt.instruction_following": "web_search_optimized", - "response.answer_length": 1324, - "response.answer_completeness": "complete", - "response.response_quality": "medium", - "response.source_citation": 1, - "response.context_utilization": 8.710526315789474, - "output_response": "Web Search Answer: The web search did not yield specific results regarding the latest updates in Python programming. However, I can provide a general overview based on my knowledge up to October 2023.\n\nAs of late 2023, ...", - "evaluation.search_effectiveness": 0.2, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 1, - "evaluation.context_relevance": "low", - "evaluation.overall_quality": "low", - "evaluation.ready_for_production": true, - "web_search.search_strategy": "duckduckgo_api", - "web_search.real_time_data": true, - "web_search.external_sources": 1, - "web_search.information_freshness": "current", - "web_search.knowledge_synthesis": "single_source" - }, - "events": [], - "links": [] - } - ], - "created_at": "2025-10-14 12:53:37.818000000", - "updated_at": "2025-10-14 12:53:37.818000000" - }, - { - "trace_id": "dc81be1a-e93d-4f60-90f3-ce1acaa8cb3e", - "name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "status": "ok", - "status_message": "", - "start_time": "2025-10-14 12:53:34.584000000", - "end_time": "2025-10-14 12:53:36.780000000", - "duration_ms": 2196.4579999999996, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "spans": [ - { - "span_id": "9dd5ac2c-0deb-48e3-ac8b-0e6e31f7ff30", - "trace_id": "dc81be1a-e93d-4f60-90f3-ce1acaa8cb3e", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-14 12:53:34.584000000", - "end_time": "2025-10-14 12:53:36.780000000", - "duration_ms": 2196.351, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What happened in tech news this week?", - "query.length": 37, - "processing.mode": "Web Search", - "processing.time_seconds": 2.1948978900909424, - "processing.response_length": 324, - "processing.sources_count": 1, - "output_response": "Final Answer: I'm sorry, but I couldn't find any specific information regarding tech news from this week based on the search results. It appears that there were no relevant articles or updates available. If you're ...", - "final_answer_mode": "Web Search", - "query_processed.input_query": "What happened in tech news this week?", - "query_processed.output_response": "Successfully processed query using Web Search, generated 324 character response", - "query_processed.mode": "Web Search", - "query_processed.processing_time": 2.1948978900909424, - "query_processed.response_length": 324 - }, - "events": [], - "links": [] - }, - { - "span_id": "b1cbafce-51fb-48fd-bc4a-d49081a2f374", - "trace_id": "dc81be1a-e93d-4f60-90f3-ce1acaa8cb3e", - "parent_span_id": "9dd5ac2c-0deb-48e3-ac8b-0e6e31f7ff30", - "name": "agent.query_routing", - "start_time": "2025-10-14 12:53:34.584000000", - "end_time": "2025-10-14 12:53:36.778000000", - "duration_ms": 2194.2200000000003, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What happened in tech news this week?", - "query_length": 37, - "query_type": "routing_query", - "output_response": "Routed to Web Search for query processing", - "router.classification": "Web Search", - "router.confidence_score": 1, - "router.rag_keyword_score": 0, - "router.web_keyword_score": 2, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [] - }, - { - "span_id": "e3064687-1b18-478b-9e9d-4d081a6dac23", - "trace_id": "dc81be1a-e93d-4f60-90f3-ce1acaa8cb3e", - "parent_span_id": "b1cbafce-51fb-48fd-bc4a-d49081a2f374", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-14 12:53:34.584000000", - "end_time": "2025-10-14 12:53:34.584000000", - "duration_ms": 0.028, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What happened in tech news this week?", - "query_length": 37, - "query_type": "routing_evaluation_query", - "classification.mode": "Web Search", - "classification.rag_keyword_score": 0, - "classification.web_keyword_score": 2, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "medium", - "query.intent": "general_knowledge", - "query.keyword_density": 0.2857142857142857, - "query.domain_affinity": "general", - "routing.decision": "Routed to Web Search based on analysis", - "routing.rationale": "RAG score: 0, Web score: 2, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "RAG", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.038461538461538464, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: Web Search (Confidence: 1.00)" - }, - "events": [], - "links": [] - }, - { - "span_id": "eb9dbc11-44b3-4041-8c83-0f33b8b761bb", - "trace_id": "dc81be1a-e93d-4f60-90f3-ce1acaa8cb3e", - "parent_span_id": "b1cbafce-51fb-48fd-bc4a-d49081a2f374", - "name": "agent.web_search_generation", - "start_time": "2025-10-14 12:53:34.584000000", - "end_time": "2025-10-14 12:53:36.774000000", - "duration_ms": 2189.866, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "web_search_agent", - "agent.operation": "web_search_generation", - "agent.capabilities": [ - "web_search", - "content_synthesis", - "response_generation" - ], - "agent.id": "noveum_web_search_agent", - "input_query": "What happened in tech news this week?", - "query_length": 37, - "query_type": "web_search_query", - "output_response": "Web Search Answer: I'm sorry, but I couldn't find any specific information regarding tech news from this week based on the search results. It appears that there were no relevant articles or updates available. If you're ...", - "web_search.results_count": 1, - "web_search.context_length": 139, - "web_search.response_length": 324, - "web_search.mode": "external_web_search" - }, - "events": [], - "links": [] - }, - { - "span_id": "c8ee237b-ab5e-4b36-83f0-2f8478971898", - "trace_id": "dc81be1a-e93d-4f60-90f3-ce1acaa8cb3e", - "parent_span_id": "eb9dbc11-44b3-4041-8c83-0f33b8b761bb", - "name": "agent.llm_model_execution", - "start_time": "2025-10-14 12:53:34.981000000", - "end_time": "2025-10-14 12:53:36.768000000", - "duration_ms": 1786.467, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What happened in tech news this week?", - "query_length": 37, - "query_type": "web_search_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 1.785179853439331, - "model.latency_ms": 1785.179853439331, - "model.start_time": 1760446414.981964, - "model.end_time": 1760446416.767144, - "model.performance_tier": "fast", - "model.prompt_tokens": 155, - "model.completion_tokens": 81, - "model.total_tokens": 236, - "model.tokens_per_second": 132.19956495997977, - "model.estimated_cost": 0.00708, - "model.efficiency_score": 1.3728813559322033, - "model.response_length": 324, - "model.response_quality": "high", - "model.output_response": "Model Response: I'm sorry, but I couldn't find any specific information regarding tech news from this week based on the search results. It appears that there were no relevant articles or updates available. If you're ...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [] - }, - { - "span_id": "9c8e1a08-61c8-409e-a79d-41c74a8d1505", - "trace_id": "dc81be1a-e93d-4f60-90f3-ce1acaa8cb3e", - "parent_span_id": "eb9dbc11-44b3-4041-8c83-0f33b8b761bb", - "name": "agent.web_search_evaluation_metrics", - "start_time": "2025-10-14 12:53:36.772000000", - "end_time": "2025-10-14 12:53:36.774000000", - "duration_ms": 1.5939999999999999, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "web_search_evaluation_metrics", - "agent.capabilities": [ - "web_search_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What happened in tech news this week?", - "query_length": 37, - "query_type": "web_search_evaluation_query", - "web_search.results_count": 1, - "web_search.context_length": 139, - "web_search.context_synthesized": "Context from 1 web sources", - "web_search.search_effectiveness": 0.2, - "web_search.context_quality": "low", - "web_search.source_diversity": 1, - "web_search.information_synthesis": "low", - "web_search.external_knowledge_utilization": 0.139, - "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What+happened+in+tech+news+this+week?\n\nUser Question: What happened in tech news this week?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", - "prompt.prompt_length": 621, - "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What+happened+in+tech+news+this+week?", - "prompt.instruction_following": "web_search_optimized", - "response.answer_length": 324, - "response.answer_completeness": "complete", - "response.response_quality": "medium", - "response.source_citation": 1, - "response.context_utilization": 2.3309352517985613, - "output_response": "Web Search Answer: I'm sorry, but I couldn't find any specific information regarding tech news from this week based on the search results. It appears that there were no relevant articles or updates available. If you're ...", - "evaluation.search_effectiveness": 0.2, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 1, - "evaluation.context_relevance": "low", - "evaluation.overall_quality": "low", - "evaluation.ready_for_production": true, - "web_search.search_strategy": "duckduckgo_api", - "web_search.real_time_data": true, - "web_search.external_sources": 1, - "web_search.information_freshness": "current", - "web_search.knowledge_synthesis": "single_source" - }, - "events": [], - "links": [] - } - ], - "created_at": "2025-10-14 12:53:34.584000000", - "updated_at": "2025-10-14 12:53:34.584000000" - }, - { - "trace_id": "945b2757-8a5a-4aab-a869-8f44c02166a5", - "name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "status": "ok", - "status_message": "", - "start_time": "2025-10-14 12:53:27.771000000", - "end_time": "2025-10-14 12:53:33.575000000", - "duration_ms": 5804.15, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "spans": [ - { - "span_id": "59595232-8873-419c-9b6a-0727445a2afa", - "trace_id": "945b2757-8a5a-4aab-a869-8f44c02166a5", - "parent_span_id": "dfde2f79-d0c1-481e-819d-60730cb60fe5", - "name": "agent.query_routing", - "start_time": "2025-10-14 12:53:27.771000000", - "end_time": "2025-10-14 12:53:33.575000000", - "duration_ms": 5803.9800000000005, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "What are the current trends in observability tools?", - "query_length": 51, - "query_type": "routing_query", - "output_response": "Routed to RAG for query processing", - "router.classification": "RAG", - "router.confidence_score": 0.3333333333333333, - "router.rag_keyword_score": 2, - "router.web_keyword_score": 1, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [] - }, - { - "span_id": "a59b3296-aaba-4642-b5a0-a271787a5607", - "trace_id": "945b2757-8a5a-4aab-a869-8f44c02166a5", - "parent_span_id": "59595232-8873-419c-9b6a-0727445a2afa", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-14 12:53:27.771000000", - "end_time": "2025-10-14 12:53:27.771000000", - "duration_ms": 0.031, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: What are the current trends in observability tools?", - "query_length": 51, - "query_type": "routing_evaluation_query", - "classification.mode": "RAG", - "classification.rag_keyword_score": 2, - "classification.web_keyword_score": 1, - "classification.confidence_score": 0.3333333333333333, - "classification.confidence_level": "medium", - "classification.method": "keyword_based", - "query.complexity": "complex", - "query.intent": "documentation", - "query.keyword_density": 0.375, - "query.domain_affinity": "noveum", - "routing.decision": "Routed to RAG based on analysis", - "routing.rationale": "RAG score: 2, Web score: 1, Confidence: 0.33", - "routing.expected_performance": "medium", - "routing.alternative_mode": "Web Search", - "routing.decision_confidence": 0.3333333333333333, - "evaluation.routing_accuracy": "medium", - "evaluation.keyword_coverage": 0.057692307692307696, - "evaluation.query_understanding": "ambiguous", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: RAG (Confidence: 0.33)" - }, - "events": [], - "links": [] - }, - { - "span_id": "dfde2f79-d0c1-481e-819d-60730cb60fe5", - "trace_id": "945b2757-8a5a-4aab-a869-8f44c02166a5", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-14 12:53:27.771000000", - "end_time": "2025-10-14 12:53:33.575000000", - "duration_ms": 5804.074, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "What are the current trends in observability tools?", - "query.length": 51, - "processing.mode": "RAG", - "processing.time_seconds": 5.804032802581787, - "processing.response_length": 997, - "processing.sources_count": 5, - "output_response": "Final Answer: The provided context does not specifically outline current trends in observability tools in general. However, it highlights the unique challenges faced by AI applications and how traditional monitorin...", - "final_answer_mode": "RAG", - "query_processed.input_query": "What are the current trends in observability tools?", - "query_processed.output_response": "Successfully processed query using RAG, generated 997 character response", - "query_processed.mode": "RAG", - "query_processed.processing_time": 5.804032802581787, - "query_processed.response_length": 997 - }, - "events": [], - "links": [] - }, - { - "span_id": "82489444-deba-46d2-b08b-c09754189566", - "trace_id": "945b2757-8a5a-4aab-a869-8f44c02166a5", - "parent_span_id": "59595232-8873-419c-9b6a-0727445a2afa", - "name": "agent.llm-rag", - "start_time": "2025-10-14 12:53:27.772000000", - "end_time": "2025-10-14 12:53:33.575000000", - "duration_ms": 5803.534, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "rag_agent", - "agent.operation": "llm-rag", - "agent.capabilities": [ - "document_retrieval", - "context_generation", - "response_generation" - ], - "agent.id": "noveum_rag_agent", - "input_query": "What are the current trends in observability tools?", - "query_length": 51, - "query_type": "rag_query", - "output_response": "RAG Answer: The provided context does not specifically outline current trends in observability tools in general. However, it highlights the unique challenges faced by AI applications and how traditional monitorin...", - "rag.context_length": 2943, - "rag.sources_count": 5, - "rag.answer_length": 997, - "rag.mode": "retrieval_augmented_generation" - }, - "events": [], - "links": [] - }, - { - "span_id": "34b1f14c-3fd3-4f89-b4dd-6150a52befd7", - "trace_id": "945b2757-8a5a-4aab-a869-8f44c02166a5", - "parent_span_id": "82489444-deba-46d2-b08b-c09754189566", - "name": "agent.llm_model_execution", - "start_time": "2025-10-14 12:53:28.257000000", - "end_time": "2025-10-14 12:53:33.162000000", - "duration_ms": 4905.2429999999995, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: What are the current trends in observability tools?", - "query_length": 51, - "query_type": "rag_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 4.9052040576934814, - "model.latency_ms": 4905.204057693481, - "model.start_time": 1760446408.2575018, - "model.end_time": 1760446413.162706, - "model.performance_tier": "medium", - "model.prompt_tokens": 872, - "model.completion_tokens": 249, - "model.total_tokens": 1121, - "model.tokens_per_second": 228.53279635569638, - "model.estimated_cost": 0.03363, - "model.efficiency_score": 0.8893844781445138, - "model.response_length": 997, - "model.response_quality": "high", - "model.output_response": "Model Response: The provided context does not specifically outline current trends in observability tools in general. However, it highlights the unique challenges faced by AI applications and how traditional monitorin...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [] - }, - { - "span_id": "d5d2b966-3c37-4d6d-b7b8-c58c9f22aad9", - "trace_id": "945b2757-8a5a-4aab-a869-8f44c02166a5", - "parent_span_id": "82489444-deba-46d2-b08b-c09754189566", - "name": "agent.rag_evaluation_metrics", - "start_time": "2025-10-14 12:53:33.162000000", - "end_time": "2025-10-14 12:53:33.162000000", - "duration_ms": 0.036000000000000004, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "rag_evaluation_metrics", - "agent.capabilities": [ - "retrieval_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: What are the current trends in observability tools?", - "query_length": 51, - "query_type": "rag_evaluation_query", - "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/blog/from-logs-to-intelligent-choices-inside-noveum-ais-evaluation-process):\nFrom Development to Production - Inside Noveum.ai's AI Observability Platform\nShashank Agarwal\n3/3/2025\nIntroduction\nEvery AI application tells a story\u2014but most developers never get to hear it...", - "retrieval.context_length": 2943, - "retrieval.sources_count": 5, - "retrieval.context_quality": "high", - "retrieval.effectiveness": 1, - "retrieval.context_utilization": 2.943, - "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/blog/from-logs-to-intelligent-choices-inside-noveum-ais-evaluation-process):\nFrom Development to Production - Inside Noveum.ai's AI Observability Platform\nShashank Agarwal\n3/3/2025\nIntroduction\nEvery AI application tells a story\u2014but most developers never get to hear it. When your RAG pipeline returns irrelevant results, when your multi-agent system gets stuck in loops, or when your LLM costs suddenly spike, you're left guessing what went wrong and where.\nThat's exactly the challenge Noveum.ai solves. Rather than flying blind, our platform provides comprehensive tracing a...\n\nSource 2 (https://noveum.ai/docs/getting-started/tracing-concepts):\n- Explore Framework Integrations - Framework-specific guidance\n- Learn Advanced Patterns - Custom instrumentation techniques\n- Master the Dashboard - Analyze your traces effectively\nRemember: Good observability is not about collecting all possible data, but about collecting the right data that helps you understand, debug, and optimize your AI applications.\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to O...\n\nSource 3 (https://noveum.ai/en/docs/getting-started/tracing-concepts):\n- Explore Framework Integrations - Framework-specific guidance\n- Learn Advanced Patterns - Custom instrumentation techniques\n- Master the Dashboard - Analyze your traces effectively\nRemember: Good observability is not about collecting all possible data, but about collecting the right data that helps you understand, debug, and optimize your AI applications.\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to O...\n\nSource 4 (https://noveum.ai/en/blog/from-logs-to-intelligent-choices-inside-noveum-ais-evaluation-process):\nThe AI Observability Challenge\nWhy Traditional Monitoring Falls Short\nTraditional application monitoring tools weren't built for AI applications. They can tell you if your API is responding, but they can't answer the questions that matter most for AI systems:\n- Why did my RAG pipeline retrieve irrelevant documents?\n- Which LLM calls are driving my costs?\n- How are my agents communicating with each other?\n- What's causing hallucinations in my responses?\n- Why is my embedding generation so slow?\nW...\n\nSource 5 (https://noveum.ai/en/docs/advanced/rag-observability):\n\ud83d\udd27 Best Practices for RAG Observability\n1. Comprehensive Pipeline Tracing\n2. Quality Monitoring\n3. A/B Testing for RAG Components\n\ud83c\udfaf Advanced RAG Patterns\nMulti-Modal RAG\nConversational RAG\nRAG observability with Noveum.ai provides the deep insights needed to build, optimize, and scale retrieval-augmented generation systems. By implementing comprehensive tracing across retrieval, context preparation, and generation stages, you can ensure your RAG pipeline delivers accurate, relevant, and cost-effe...\n\nUser Question: What are the current trends in observability tools?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", - "prompt.prompt_length": 3490, - "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/blog/from-logs-to-intelligent-choices-inside-noveum-ais-evaluation-process):\nFrom Development to Production - Inside Noveum.ai's AI Observability Platform\nShashank Agarw...", - "prompt.instruction_following": "rag_optimized", - "response.answer_length": 997, - "response.answer_completeness": "complete", - "response.response_quality": "high", - "response.source_citation": 5, - "response.context_utilization": 0.33876996262317366, - "output_response": "RAG Answer: The provided context does not specifically outline current trends in observability tools in general. However, it highlights the unique challenges faced by AI applications and how traditional monitorin...", - "evaluation.retrieval_effectiveness": 1, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 5, - "evaluation.context_relevance": "high", - "evaluation.overall_quality": "high", - "evaluation.ready_for_production": true, - "rag.retrieval_strategy": "semantic_similarity", - "rag.vector_search_results": 5, - "rag.context_synthesis": "multi_source", - "rag.document_coverage": 1, - "rag.information_density": 0.33876996262317366 - }, - "events": [], - "links": [] - } - ], - "created_at": "2025-10-14 12:53:27.771000000", - "updated_at": "2025-10-14 12:53:27.771000000" - }, - { - "trace_id": "4bdd5814-acbb-4de7-a56b-543b87a1cfa5", - "name": "auto_trace_tool-orchestator", - "project": "noveum-ai-agent-rag-websearch", - "environment": "development", - "status": "ok", - "status_message": "", - "start_time": "2025-10-14 12:53:20.120000000", - "end_time": "2025-10-14 12:53:26.750000000", - "duration_ms": 6630.38, - "span_count": 6, - "error_count": 0, - "sdk": { - "name": "noveum-trace-python", - "version": "0.3.8" - }, - "attributes": { - "noveum.project": "noveum-ai-agent-rag-websearch", - "noveum.environment": "development", - "noveum.sdk.version": "0.3.8", - "noveum.sampling.decision": "record" - }, - "metadata": { - "user_id": null, - "session_id": null, - "request_id": null, - "tags": {}, - "custom_attributes": {} - }, - "spans": [ - { - "span_id": "11c07411-df30-42a4-9062-90a032ef75ed", - "trace_id": "4bdd5814-acbb-4de7-a56b-543b87a1cfa5", - "parent_span_id": "ec530838-a7cf-477b-9186-a70114c8a12f", - "name": "agent.web_search_generation", - "start_time": "2025-10-14 12:53:20.120000000", - "end_time": "2025-10-14 12:53:26.750000000", - "duration_ms": 6629.273999999999, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "web_search_agent", - "agent.operation": "web_search_generation", - "agent.capabilities": [ - "web_search", - "content_synthesis", - "response_generation" - ], - "agent.id": "noveum_web_search_agent", - "input_query": "Tell me about recent developments in machine learning", - "query_length": 53, - "query_type": "web_search_query", - "output_response": "Web Search Answer: The web search did not yield specific results regarding recent developments in machine learning. However, I can provide a general overview based on my knowledge up to October 2023.\n\nRecent development...", - "web_search.results_count": 1, - "web_search.context_length": 155, - "web_search.response_length": 1849, - "web_search.mode": "external_web_search" - }, - "events": [], - "links": [] - }, - { - "span_id": "282b121f-61dd-4dc5-9e55-cecc3f02f688", - "trace_id": "4bdd5814-acbb-4de7-a56b-543b87a1cfa5", - "parent_span_id": "", - "name": "tool-orchestator", - "start_time": "2025-10-14 12:53:20.120000000", - "end_time": "2025-10-14 12:53:26.750000000", - "duration_ms": 6630.264, - "status": "ok", - "status_message": "", - "attributes": { - "input_query": "Tell me about recent developments in machine learning", - "query.length": 53, - "processing.mode": "Web Search", - "processing.time_seconds": 6.629686117172241, - "processing.response_length": 1849, - "processing.sources_count": 1, - "output_response": "Final Answer: The web search did not yield specific results regarding recent developments in machine learning. However, I can provide a general overview based on my knowledge up to October 2023.\n\nRecent development...", - "final_answer_mode": "Web Search", - "query_processed.input_query": "Tell me about recent developments in machine learning", - "query_processed.output_response": "Successfully processed query using Web Search, generated 1849 character response", - "query_processed.mode": "Web Search", - "query_processed.processing_time": 6.629686117172241, - "query_processed.response_length": 1849 - }, - "events": [], - "links": [] - }, - { - "span_id": "ec530838-a7cf-477b-9186-a70114c8a12f", - "trace_id": "4bdd5814-acbb-4de7-a56b-543b87a1cfa5", - "parent_span_id": "282b121f-61dd-4dc5-9e55-cecc3f02f688", - "name": "agent.query_routing", - "start_time": "2025-10-14 12:53:20.120000000", - "end_time": "2025-10-14 12:53:26.750000000", - "duration_ms": 6629.59, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "query_router", - "agent.operation": "query_routing", - "agent.capabilities": [ - "query_classification", - "routing_decision" - ], - "agent.id": "noveum_query_router", - "input_query": "Tell me about recent developments in machine learning", - "query_length": 53, - "query_type": "routing_query", - "output_response": "Routed to Web Search for query processing", - "router.classification": "Web Search", - "router.confidence_score": 1, - "router.rag_keyword_score": 0, - "router.web_keyword_score": 1, - "router.mode": "intelligent_routing" - }, - "events": [], - "links": [] - }, - { - "span_id": "df882f90-cf1a-42a6-a2ae-78b2a13f27a1", - "trace_id": "4bdd5814-acbb-4de7-a56b-543b87a1cfa5", - "parent_span_id": "ec530838-a7cf-477b-9186-a70114c8a12f", - "name": "agent.routing_evaluation_metrics", - "start_time": "2025-10-14 12:53:20.120000000", - "end_time": "2025-10-14 12:53:20.120000000", - "duration_ms": 0.082, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "routing_evaluation_metrics", - "agent.capabilities": [ - "routing_analysis", - "decision_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Routing evaluation for query: Tell me about recent developments in machine learning", - "query_length": 53, - "query_type": "routing_evaluation_query", - "classification.mode": "Web Search", - "classification.rag_keyword_score": 0, - "classification.web_keyword_score": 1, - "classification.confidence_score": 1, - "classification.confidence_level": "high", - "classification.method": "keyword_based", - "query.complexity": "complex", - "query.intent": "general_knowledge", - "query.keyword_density": 0.125, - "query.domain_affinity": "general", - "routing.decision": "Routed to Web Search based on analysis", - "routing.rationale": "RAG score: 0, Web score: 1, Confidence: 1.00", - "routing.expected_performance": "high", - "routing.alternative_mode": "RAG", - "routing.decision_confidence": 1, - "evaluation.routing_accuracy": "high", - "evaluation.keyword_coverage": 0.019230769230769232, - "evaluation.query_understanding": "clear", - "evaluation.ready_for_production": true, - "router.classification_strategy": "hybrid_keyword_llm", - "router.keyword_matching": "used", - "router.llm_fallback": "not_needed", - "router.decision_time": "instant", - "output_response": "Routing Decision: Web Search (Confidence: 1.00)" - }, - "events": [], - "links": [] - }, - { - "span_id": "bea819fd-3673-480b-9f7b-a3a2efef4218", - "trace_id": "4bdd5814-acbb-4de7-a56b-543b87a1cfa5", - "parent_span_id": "11c07411-df30-42a4-9062-90a032ef75ed", - "name": "agent.llm_model_execution", - "start_time": "2025-10-14 12:53:20.455000000", - "end_time": "2025-10-14 12:53:26.746000000", - "duration_ms": 6291.401, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "model_details", - "agent.operation": "llm_model_execution", - "agent.capabilities": [ - "model_invocation", - "parameter_tracking", - "latency_measurement" - ], - "agent.id": "noveum_model_details", - "input_query": "Model execution for query: Tell me about recent developments in machine learning", - "query_length": 53, - "query_type": "web_search_model_query", - "model.name": "gpt-4o-mini", - "model.temperature": 0.1, - "model.max_tokens": null, - "model.top_p": null, - "model.frequency_penalty": null, - "model.presence_penalty": null, - "model.provider": "openai", - "model.type": "chat_completion", - "model.version": "gpt-4o-mini", - "model.latency_seconds": 6.290812253952026, - "model.latency_ms": 6290.812253952026, - "model.start_time": 1760446400.4555368, - "model.end_time": 1760446406.746349, - "model.performance_tier": "slow", - "model.prompt_tokens": 163, - "model.completion_tokens": 462, - "model.total_tokens": 625, - "model.tokens_per_second": 99.35124031199013, - "model.estimated_cost": 0.01875, - "model.efficiency_score": 2.9584, - "model.response_length": 1849, - "model.response_quality": "high", - "model.output_response": "Model Response: The web search did not yield specific results regarding recent developments in machine learning. However, I can provide a general overview based on my knowledge up to October 2023.\n\nRecent development...", - "model.config": { - "name": "gpt-4o-mini", - "temperature": 0.1, - "max_tokens": null, - "top_p": null, - "frequency_penalty": null, - "presence_penalty": null, - "provider": "openai", - "type": "chat_completion" - } - }, - "events": [], - "links": [] - }, - { - "span_id": "6e62af18-569c-4783-b2ae-177c58542bd0", - "trace_id": "4bdd5814-acbb-4de7-a56b-543b87a1cfa5", - "parent_span_id": "11c07411-df30-42a4-9062-90a032ef75ed", - "name": "agent.web_search_evaluation_metrics", - "start_time": "2025-10-14 12:53:26.749000000", - "end_time": "2025-10-14 12:53:26.750000000", - "duration_ms": 0.217, - "status": "ok", - "status_message": "", - "attributes": { - "agent.type": "other_details", - "agent.operation": "web_search_evaluation_metrics", - "agent.capabilities": [ - "web_search_analysis", - "response_evaluation", - "quality_assessment" - ], - "agent.id": "noveum_other_details", - "input_query": "Evaluation for query: Tell me about recent developments in machine learning", - "query_length": 53, - "query_type": "web_search_evaluation_query", - "web_search.results_count": 1, - "web_search.context_length": 155, - "web_search.context_synthesized": "Context from 1 web sources", - "web_search.search_effectiveness": 0.2, - "web_search.context_quality": "low", - "web_search.source_diversity": 1, - "web_search.information_synthesis": "low", - "web_search.external_knowledge_utilization": 0.155, - "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=Tell+me+about+recent+developments+in+machine+learning\n\nUser Question: Tell me about recent developments in machine learning\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", - "prompt.prompt_length": 653, - "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=Tell+me+about+recent+developments+in+machine+learning", - "prompt.instruction_following": "web_search_optimized", - "response.answer_length": 1849, - "response.answer_completeness": "complete", - "response.response_quality": "medium", - "response.source_citation": 1, - "response.context_utilization": 11.929032258064517, - "output_response": "Web Search Answer: The web search did not yield specific results regarding recent developments in machine learning. However, I can provide a general overview based on my knowledge up to October 2023.\n\nRecent development...", - "evaluation.search_effectiveness": 0.2, - "evaluation.response_completeness": "complete", - "evaluation.source_diversity": 1, - "evaluation.context_relevance": "low", - "evaluation.overall_quality": "low", - "evaluation.ready_for_production": true, - "web_search.search_strategy": "duckduckgo_api", - "web_search.real_time_data": true, - "web_search.external_sources": 1, - "web_search.information_freshness": "current", - "web_search.knowledge_synthesis": "single_source" - }, - "events": [], - "links": [] - } - ], - "created_at": "2025-10-14 12:53:20.120000000", - "updated_at": "2025-10-14 12:53:20.120000000" - } - ], - "pagination": { - "total": 321, - "limit": 50, - "offset": 0, - "has_more": true - }, - "timestamp": "2025-10-22T17:00:36.145Z", - "processing_time_ms": 137 -} \ No newline at end of file

Y00>0s9~p7Tm3d9^%K!LpX7L1F2$2=TrOx znkG+}tC3)jDpujrnZMzEoU?4d;E1@iORL5Yd&k+qEmw`C^FZ$5oviX-H$FAEF`rv= z2Bv@CkHqIiIafAo6(&00+JlN)Xio4;gHr8{;Z8vK4-uoSWJ=^OY_!z_I$d^@1Fmso z)%2y}1BL0267I3t+&H<}$?} zjZ4-*Xte-ez88r%Llg#4e)PmB0~`MB#S_vDC7|MO;ocq;7tq|&x*3zioKBr^sn-cO z6{>?0zC#>05?`ZLGd=O$@T^1c#P2f|suj zGWskJnHi0Qr;>h$^fD*RiMEji*PJEsvZPufjn}wJM|nNhhiG>zDYkIk>J z?M(+r!cRGCp(o~edWyWAzfoyL(vTQ3ZydM=){>+f#Es#*h(|e_)bOjq3I6%3G3rOA zfUT2`P`SGA&00ZxQ5Dn6@h7qSp2>5%08xL4&Mu(4dV7U3TI8(rudC>yA*~n83a@ zkJ--8*MKx7Z`1pw*5VR*TlyPuIH+7XYFP<&4F3okCw3f@h2zPmjQV8?PYUQ=(?5&k zOxv_0g=Muwzl;6Erx_dZ-lKMSu5CXkD_x1f4{xK+@-h~3b0ap)FJYC9vc>uNiQ?YT z$4Gnv^ga2#Sws1I!xHFeQUlK(ej*63B;^W-z>m1^d}|qUm>#o?bLJZc6hiau+(+dW zdu1~x?S!3^ zt^sKQ_;S}0#|F{;kBp%r@u9xx*0`>$o=+Z+#(c43CZum(1f;29UQPg;J9dYLcpfQ6 zcqK0m2FV=o@idm5-$je#BR*>=*NOKvg1#rtct(KoD)c=((OBS$-D`yYuN|Q3QQ_=@ z-pLR)yB6LYyBgTe|B8zKhQO{lo!Gsj!%=bEHDt&AM8#9;T8UHPw{Hk1eB{T|&!EyQ z9jhnNep+{kOP1rD8m0<|s(4#*F#VoUT#78#-U9L@NFEMIn}}`OpNX`KO9hQZ5SQ{` zSgAM)cev2r5WUBu+uM#voDS2bj;hkO`Fo$@iKCgAYlVz3O0>1I^d(Ip0$06JISj6^ z!ZqU)yzyo8H$w5{t#>a3)e4we??Te1aO%xv!6)cRm8WMu90uPZhZ*@$PW)2k6$brS z!uRJEBYB7_j!oZSCHIUtf(rMQrawBpq42EnNck7iRr-?n7HJGf`Gy5k>i}t4#gnM~ z(sa7F@4ZHgEtA$m>SJ?$V3Rv6Ol{30`7-5|A?5B~(z+3#v=wm=pXEQ46aI-_Uq7RD zqg1Bf_$8X@S#bkhW0+ss2*`UecS|GQByOp z)QYxkH=RxsbSxmAXXJCW#s`{_*8YLy!x;HN)LXX;j`XdsG%CA)Do8`x0GzBx;y=C7 zFcLbWcFCzKzE-$(e!dw-oN0k^XUf%BVS-mIsNB-}uVTtGkRJqv!{pHsD!#A<`PsC7 zjERigx*3$7f8~CSo^9R-amQmYD*F|vIwp-#6*I*t!w*g3iO(uv*t?_T=T->fEF{i` z*~5WfOZzT1H9I7Np(c{wWBg(&^WU%^iIW)V4Mutzrn$RH<=ftO3nf2lMRTR2JR-!` zUIk_yuK;;c(u0voQv>;YAkC*OUTwkOce_D)GoQ5n5yIpiNI5Qa&zaL2x(tb<6~>X4 z?S(%hBqq;jsq_F(x$8>DDG^Inns6FBI?$TVn#Wn}ZL~kATFMAKjCr)@&XLx|no;3R zq4u$RV0pa`K8}b2!U^TUapfv&Jf7ZJD!lqpaSo0HJulMGf!R7m0rfxVntOqNYbzvA zFSdk+h0Cs>(G}IpOf$VgN0?p)d34}NGBtncJ+0U!kap= zE334u@@Hxem0qB0Aq;6on&rBdc!5!TSojG`+}ya=pMf8skIOJ%6Eb^2$GJW zy6Zt)SjJ-i+|*J(h+_L0C`ODlwT5&7e5&_Kc$dzgen%alJYwThpV{f`_2dWZ3+kBw zaT&ik))WX&;ZcQ&(sEV(o#6T{U_rhMy17T9$Y{cMS+bod!9T{1yFh|uVWEJV4@_0oWN51NM zTO77}52HH7o0C2%?9oskLaUB5W@J1}-FF0j>^0*9*TzWlqiA@_N0RO$UEWhDEt+;w zkGw$vKC`1|Z@J=>6;0H>I1Tu*5#xNWtaX@ zN`EpMH!kyafq>(;C>IK$-k^y}N3w29vx$=~Y8*dq6;JOy)hPW!T1VVXO;^jal*~M@P1DWB<-;UujjNj(t>zU_?9sI2Meus0!Z((W%Hj>Z>kf{Zlp(EcR4(> zOUDnTods!Disfb?o&wUt@^!CVBtKOZ?=`GI_1>6e*&Nk}1LY%U)b-YqpBLd7g-F@} zly)ax(@+mVX*2vfWCg9M+=8B!U#uZqgL@5Q&7&nP03G~BQ4in=%#;%;W@j0B6X>p=E~u9X3ZSrtz zi(oKIq&-1GJT+(lXjL{A1@*koy2oS5|tz#J#k) zfDtV{)AGI^CcJ&DO?W$8_R%$y8qfOjspnG&pVbaG6u-umj3Vd~$-}R+TFAV0 zt6Akq;ETufkT!qzL*V{VD0{b-+h2G=htM|a_pmack^EY+6Sp)Sf!!_cv*K4xvDw~F zbZ^uiXAX8ha&vSnnuMo|vedVlvv0ESWansp&2uH5ZuJp|`y|2}XAg*OTtoKQa|xr9 za>2aCF6?DvBB$K4hwMI`rA<*U_?+C4V^pQNT32b-M|*J4@szdi{YBrG z8PH=-ciH#kB`no%4)um*!0;@MY}a`b_c*m&Mh^9o6JozJLz6kIxK@9j-RBIOZC{DE zyVsQGN4{lK3*)f(JiX&u>4G`mDlsSa1bM^%<{(G?q>0euL&iYs&!*7mBdeT1*MO zqoHf&o)S#rrqkNG#_AgQtx1ig{$GEnzORLRbUR;rZAU)7*?3p18SnuP zB>#YF@D6D1<=Xi5|d>u{XrZW*U}hwc{PGW^9j zF#YeTi0NC2HT&twi`VxF$~7E6&=oXw-_w2LwLG+298i4W)qz_Y&5lzzbGI{o7&nr4 z-E0Lqh6a30pWi_LP;a=m%xrmgt^+xrjUYlk%A!{`~thYihT^ig_-H1#dCdHe~| z|K>uWbwmCxY(J)rJIbP(d=OtV=Ht_2g{-oVxttxBh&R4;E=vFF&gq)C!?b(Ce)m%F zIAwsEMeEq#Y4v2M<({(HIXfQlKm&dIIIvNHr?H{=Km2po5&oq=1;og5N_Q=MGrvb{EGSj#0TKGQ(Gk+1#2NoA1KUQ_P`dkCuE}uM@BAG zml)F1M6$E-)W01C`VkYk(JWid>6#^&wl58u&Ygpc&UKImy+T2^b_UX%!PRpwkjBo< zTR6$1UR&Tr+a&h*QBNMwvbu~9W_)_hG|bY?2f_eu-)@+o+ODcuikr~v2o%H} z0!vHM3gu2{`l*(*Y7hu&%sJmC!*tpU<8Mlkq!S=3~nRzpCo$NV`%z zcj*sId)`;7*ey8jEW7sF3SDR|HSaqEP``_>su^(TnvI!sA9&NwCE9?kIq3AtTNc!b zmN}L-{Lq7C@N-l*cy4hKKg8COlvlpv<{pQ#YUZ-bqhWH_T6+KI+DCZZXagVy;6OY(kHEHC^y zO&Ddo#H9GixbFnr&-XroA6GZWBKtz|*D0Oy%mTKTEa6mdxUY39E~a&bK2JxGUBuOM&4>$dr@e;MdbJEDql_d$*7K4w?!3D z)~q8>{V*D7{)D=I{jxQhKZ*OX;It8DR9=SZ3r=gkY#yq8qTPbOZ>)sree3c=Jr7Wg zHIOwrq-zWu3?MSwmG>?Ghjv--SWI7<@74!^u7|bWZ;XVa;8w`t()fn>{Y(i?ZSDof zzTOcF-_2w-I@pOni7kYDY$WG&^pdu-W}r#d25msg7%{zXG9&zxqbICVc_zPl(0Z5O zDt(D(koguAP0qV5UVd&3FGoKVIm7CpQN~!`S)|kJ)=EH?m+4)a$#s4!V9}05ICdoq zLK^L+UX=k(d{>{3_>>1!>#%cW7*M{+7nP|>f8dBqQH=g8l57LSuG~g2Z^LP9H9r9< zM$o(aXZTdMop{U;>3ZQ~2M@SlRg<4SdIn!!jt2L>T~r;3Ta&iKr!L(z53i+S;D=+X zZm6D)g2LDJ-TjDTw&TRT?Ko);cKM$vfB(KArk>mg|HT(7+>_O)xlHqnqq{w(Ij=8R z_9K-e%<{Yk-?z9KC#?kUcJ5R-%UatNAmJvAh&BYmO8in(lYdS=2@#XcXm0Evyn1p` zX|TSEn~%cR+uc{f=HZ!|bt`pMz3@>5?{VDgNtoTj5`)Iomvl`)`hb_5Ta7>e`>Gvk z?5^^Vjy(~H8~D@N{$f+TeG2Ea6mLz7nUC0*X}g806E*I$?)uX7zmbG9Pgx`DmSSOK zFgu;P9!Ptzum)y|%g{3R1`bZ{i%EN|tLEF#r3qL1YN6>iAgsdt=Zz$B29o|^l*>?1 zr<&4hOpocwj=Ro@#a|pGX)#9FiK>XQ*XU;dE>j$f{mn;yp^ln%w)lQgHxkvgydw4+|RC#9IJ|3ss+6sFNOGR<5TY~wv zVHPLq$!)y?q)~}EQ0(x~gBqM-4=Sf#|F;DmHcS$?i}RszPC8TZcFrurmZ?q|ju9KmtD6LIH%*O;1PGu=S8t5H{YvvCkOAE&ho%m!l0)@#tNv4(q( z&w;yJw`kSa-VYCgu`SLc)tXQm{fEsBOfWH(w$-~x{~6uUznTNixgG~gI{RSo?5he> zHU7`ORbeyXAI$!tWw+h`U)!WhnQ_|zIHQga{x$rBI^LmzG$8ml>swVPgk^X!p(o9G zW70>K^4#tV3I{0PmV@WV8?;vB8Yo}u52l~i0DV`|^V2Y@uBO#7A$q1Ru0AJCpw(kbh3lx&|%gq>qq15*{iEV~)K8C2pw0Z|VI7w+HXwz}S6A`~Zv2 z9j7&q29o|Q5o4w~Yjt-;qS35P;={P7cpsWm4OmL6y|#R&b#JPhsYrUM3iEu{*VfYK z<=`nhh=0Bc;v^tU=5W5DbmXNSTkSCONrs=U#w?T@>;Jdjm zQoV7HB`$J$#e2nLjOu`?ye=6w0q&Zeg~E~R)wtv(M}I*z%dA%|l_^hik$eR%{<1@m zHb*rU#phOElVH^&S4p_YTg|U6$p;BKCQcl7n$6qLi75`+8#Y4WBV08-36$gXK2|wE z=`MwZ)W7WrU#jbIr3dPD_(poRx+L!4{i3Flw$Vqz6}-G*0!I0#ftt5Oi!9K)_7F*{(~?*c2Xa*1?;mg0^{qZvPOa4nF3WSX}Uy7i|%U5$5ijG;ij zixJlY=_cqiT`Nc@bIL`!mQ*x&*j(`g_Gr69aW3^)HnK)HuD~-B4?%nYm3@-5DaZGJ*b)sFuRT9`I!ABZ!9}v!wG|MRKZkK`Pp-6eYs)dks`&n*-z4N z(X!FngMS)GiWS)ToC5ODLgC-C`CdSI0TE3$va2uGA{<b}2!HXaN`EPSP@J#g zRVT$8noKyx_Oq^9@>b-B?&Au(JLGS6GvX-j?3KgLFR|gHCzyyOg^$Uj8A-}x=8&)) zC=VI&Y!$vA|1%3H|9~_jz4y?Fmrq=yatq0qqTA$CN+$z(2OPg8j{K(~*hNbuo&zh| zk7{*V4-rIbF)PhyI;x2DB(0@W*UIr(h-ocuqh}FTX*N1s_!WL z@)7NU`T+3k>~-yMo99?_;bD0Es8mcG)`?N9k-WUp?~q!1tw@TmBwf*fQ#|>l@mpZE zYg6T6wRFw$#JVw})3t2ae|{VJmQ~av*ao=`@_}?Q3@Is4o&a|T)&c80Z~5|bzEE+a zIg;&e9#i#)r0W@JL5<3%)^=@3D?by|S81!bv3>C=(PPMeuzXitNx8r$?kyHeZkpgQ z^I&$W`7JT`jsrZYQIFQP34?AgYHOn6>vA>Tv^CR!YL`>(A?YO*Q$f8I&3jf)Qz}}4 z!jB&nv4Z>@Dy~;OmJXo>%C!?{)-c z$C>dJ^mf9W!}qKF0OgG6oWB@UT?K1%81+u5mdb%}f@(%8?ZfH&z-_m}VcJrPYG3JF zs;$+G_#X)OwePP@$KC}gK)eCeKjPG9Vf8x9QCdw9rz;&x`x#skKKfcg{8^>B{qECq z;bL@QMr99SUi%l-`Fe5wVjW3biHinoB!AXKDlSsFMY$prHWJ_PCU@Tec>|RflJY^2 z<|i$?1*q@BHl{!9Jl zwfRcNFGzhU+2N@%{Ay=Q-?Q?i9vStvqUm|_MEqLfD9FQ#v)fv+=@nCD@}!$$RZ}BQ zwL|*yV%0Hwem;ey&oINoP1Rr#)dDO0NzaYh){?rr{#N;G>J^Auxd$=UEEA&+c2ao+ zT_Or_O9xX%HLq|E9BaK2og3}|@@>NJmjUh{zKHIY*Q@GD)HF7hzuIkLZk;Vy?8mv1 zuoHbF{|V~TD9`IlURCqP`58hiiba+nBs+4dQ=K`8&`Xt-hM8eud0) znV|W$Fo^ig7Kz{J92HcXGeoHLlvLluhf9UhH6Z==#GW6@q% zybcnozvi)fFj8+q@e>eNBlRrcN!oNg61@*d-%@^;&|D=j(se-itfkyX&yS(Lq}h-# z3aNJpq;a{@1a`CsGSw?r{Xo)#l6;rSWf{`QnJev0<3XiwRPSx*rs`C47pbQ-59{|l zR)t-ZKUL>_<<|nKj;k>7Md=i=;MG7*Isw~v-;DDf7chDa|fg-d~D?@^s|w;@0@_YAg(lwMXTfB7JqCG&mL)%*oZR zUz?^;d9He$gUURsJUw9mox z0o7wo?v$zY604Q#fl7Z3a@fnLmVk6O9{F@rVV&~jKz&EUcD(+t{B^_|w6x4-igyY} zTJrlN1}QwpoU7C$TX33Df0~c4bx2_cu6E57QNP=1sJ{T!!fYYiZHCNUY{GR1YVhyT zp3>4g6FOgLBqz4I%X-dgz<1jgz^Ue15G)X{1ydh*T|X=v)Luq4r~?lc+OdNtmdTNK zd!gT$V4+<*f|nS0@cG6%ylr5mxNze!>wfQ`_TP9c4^3+ z9;`y$fm-~$>M*bVc%SHapVq;e`wfN`JIGb7wPo^I8U+j&zk>_O!y~4P4||gJ3@Q_fY)fSOkmOEn)Q@>htn} z_2EO?_Zks30Shz2q#BP|@gAVDWAgNQ*m0vhT5tIabZmThwYL0~X3JZQn@aoloYkJ~ z7J*^CQ)y$Hwb039rEK_cD__@RBZ{bUJf2ue&vuj|#R|FS7VWklWxnQ*hse6|$M9fy z1KHnxBbMaR{y=w(`3R3G*nim)`J;FxVv4eGXJ zgEu6x$cC~gy8H$Ut*ECRawlHSvZH5sx5jA_hXH>a){@sMZ47-b{w?Ys`T?!xZNYz~ z^+lhHkq~>P5dMAojnxLM;qfhBqRU>|^J2q&tP!7}o!DxS#E6gL_0j60`pN-;iuHkTjU-kHyI55g>?cCuF8&XAY48q;q&$R3ScW&KAN zR1UJPS=HsaNz2*1H!s0&c0KMA$g1Z0;^cR@WuA*1U@I z@4SQGZ9K4SqABkB&kWlJPKGVh#-ORAAJl#1iEi`8@wW#S@yX>z(#D+D%V%zw)b65o zaBL@xIGYOnYj;D74=!Ad@BE+yxN*G)t{CRPSDVa{8Fu01odUA9(!jsbZP85CNY^=^VZqJ~Fy$QOzFsNZPI!x-p7dr^ z7w~v%AM|`U0ehWa%?JFwj+Bo;V}y!hUqmNy1J4|93h%qTqVJv(qwi#D=bE_5HEs!# z#)G=)4^e$5X6aE{SK%^H{-bT1*N{l}*bP68N5967_@!iomg-#0jG^ZhkAlJ&*d5_4 z_TB6&A06n*Cj8!yDL>z^mo@s~FuP?~>2wGwM%oXbf>oT+W3L3Wc`-#hy4!KP71iXw zPYXf2y``MED-m|QY|Rhss1$P=l74S8ftxHCtX1(^-{78D@0JBLM*QlyjVWv}PxIx; zEkcpb$%p?b5mX-{cgR^xOUg#dBl4#Xaz-zcBSn*SrM%%Tk9T$9C3$|Bb<&#u*Os28 zyxCWyAGR1gAC}|%_2(f!af?RP`=X^z_~^@2PMCrZ^vA1umi=1My}{LIVOXCbysTcD zZ$Lmx+4J8Rcv0^%(A+}c#cq=38O|D<7nB=#_`eQtWJ(LFt&Vca?K+%(uN8>}a{BP< zvee2QVyB;h%2f?;c;RM+!9wN6C+F{Y@{0*?%_)q1yp}BIN%JZ-%(w| z=QLfYzuAe0)H2{kTe>L>=3ArmdJK9#=i%#>9Gu^4 zCs5tPx6)`Z(klRkNi)lvaH|0WdDqJ3Xsc_6JuVbs{Oe;Hix1g&^Y~3Xe?SMR?s>r9 zIH*2xj^#BNYyH?PEc8S_*aJ+gIh>?3QVmP%scK=Y4>e3*t3x=^gIs zoi}L0wKFhs#0a@Ic|7(TJs!~Ik~nRB93sn7?QNQ<%u=Z@-cC09Nj~{ z4570hgL@Nue$dcWTu4CawxTuf)1fuIomeJj^jZbYn#97f!A;tp`x+vz)>{g69&HTy z0fm`Q7d_Q3ni0m=C)Hte&02+xQPZmNUVFO8=5^M{$uZkdVc*G6RQ=HS;OU}~yzLkdemC+f&Kus4KW*Diu3DE#m>nsftQ ze7p&@#~0#^^&Xt?L(;jbYPaBIFbu4NI zvVhb6$U!*o%~+b>0PeZzlyF;-uid%UO6&1{grxcx*J)3Kzw=#Ce6^R2(%!&?kgZ5u z%|q#-z!it5G(&}-gomIX_K)f!SeB=k0M#S>c6v^8Gn~6_+Q)|M zS)lMvD9vHh`7tA`!4*G?MAYy$*zx{Jme2NSsJ01TcH>WI*=IdPL}7Ql z6J`lJU&Mi`Yt6sV<97q zgOMBdVti&A4)2|(&c|b0G{MHY&Be-^v2b-zRuwN7zWk?s=eS%;pM@J;s&nJ3^Ekw5 zk$4`Wq%lYerv2Ej3@c7wx?Z2G<_$J6fkk-IaZyMmONkuq)c#O(RxTYC}rW2mA zBh3w2(}#sXIt^#|`@^yAXE6K?t*?9DiP!QruEM$SR?WFOK7B`wDS8B(b~LH_|J2u? z@aXlWxIe0pkxpWb$C`7h4|E7@0?UV&3gQrIs9%COV^4Wf<0)mN*yfTCWwKAMZk%KW+U-U2$tvEagQO zy$|hKrKxN`1aNos#`~-5301Si1)OkArVpD!m=!FE#~}~OaMi2zT8x;^W{vzJG_AjY zMW+TVW$Jdho!x(EEdVA1Igh;5$npd@gQ6 z(pg}JmDBz`XS=;CIC*j*B<>up>FQZZ z$Jwel0|(zdf~0ML^eQ-b_XE=T^22x?_)%O_Zfo}!X--H7JMxZ|iE#Pp5{PWE0!V{_ zvkYRSNAby;K}sXa&Cw6&nH?u-Ker}v`ZbmJ;%VDrsJ@E9oQ9c--(i+rFW!9f08DpX zFNlZL*pXt(D;8eSuJC(^UjL#Mm&od?1l7h^XcGU4S?pUz+}asCiBOu$J^23U77V<2 zkxi-fhLNtt!R;Hv#5af8_}vJX7g@8}re&D0L%BMdMSZLf=Lwf#zS-@{Y&VXXx4Il=p%-1Sf>p^K#nfny}rsd*fTM z^YC289RA?t$=z6iCJNL!ovTez9)pK#Ke44flfYtM4e4C%EgRwS7Pf#XBTV6>{W$p| zVR`f|n_RvEX3ngH;ijvx?v)o+Ii)nftdX;b`!f_r0V7)KVS&+9BwSXQ%!te6 zgd>Jkc$@vQyJqq99Fb*IEZU3>$Lr5J3DO!cD|s0skD#=k7~{SHNsqGaMvgqNL$q+d zRG0f-Fn~t=J5ekHfX=1G=evRCN< zY=B0z=D_!Xmjul>=GyGgk`91sLC4X`B47Kk-eH{o>k!PPb&r)M^|@%vNngOg%0UX} z=(%|tPM#TPZrIg9&3O2mrONNfq|C#(!T1^0Zfd~^8|Zg8LFtA1{g$ZtMDjoCoY3Jd zg3<+~4{%tmBGSpeXw$!ihO|1>ZV->jT&kftMTIj10)8m(L~FvkDXpwgJ|*RFC+yp( zBYxV_f_L1wl{opOco+6aVG@k4{g-@QEvn_eqLFk*uglX>)fo9rP<7a{|6Fz<*ou=L z!hw}{VcxFe!e!+nVbLiKNatxByl(>eMEtowiGIEf{JSnon^#t*^_My`fPCf@|!w4a7F|114)MUE|zrF9sBTdneRws-*g}$RnBc<;K#fy}O8t=LPY1Qv2H&~XvM~HG zGm*6jb=GwDY*$T{&qmSoO-1@3=Ki`8Yh%JPsb{zs95=yTqB}nMx-x;th7D z*&0R|i-eWTb(0$;6xO=96`9Piw2BoxPhU2$t zsV|^>kRZPFCEY4E7>{B!zhF4t9!6B0g~wqzK=n+Vb%)vRvqSPkg2teAn(Kgy$7bDj z!C`$KROv+09!&Xky@m81`K+Zt@kP=|jCe>Ir!~Uzft%sh!6AyH(0>0@2=E%g3tujV z`i>u}Vnkf3dTtlM$$ig zLH87tx2E}Kb+3$pO+5}O?Iwu36$g>V@h3kuQuPI7<9$AiJR_c5+#d&Dn26NNDk?mY z4lirmgrVowX?ISgb$0LXg$3nIJ81Vx%!%*^>Y>S3mQAX9VhVdH7diFGq=zs<@(P*~ zgM+BBQ+Z^+h0BPcCG-j*FTybpJBW2q1F0vG&Tu6!MPxxkpwSdn@yBEahj)%O=% zVw@^=yHq1x+Ds0<@J6HJNj(@rH7H11s~%aE#w6c`6JqBA`BHp3uo_QVI!#_W@mhI$ zOdZ->+d9>TN2dFk=(@pEpNUx1n zcf3CJ-Re|nJHm84*u57J_Th)R2|&FX^84kiOX?^{%Jf!x1e6{(PTPiSYuW>8NJg4c zD4rpXrQFJ9yIrDzaukl9IKmzsxB&JJsnq+pj*m*L_|K0%*#G=4^2xi9v=>Xwyevuw zhw=qK4RBsTEs?q80xmWQ1l98+kB-#)R2)>LqZCi5UZm1|DfJJ4S#fiDs9UD+>GvEe zADg1;ClC)Xw|5P(huKi-8}-D&UB>#SdL45!suK^JunUVf4p>^_F^h$ zXmz<39<^?c&w^@JdFO+Mdtus!aJch%FGQDz3aTwewMp7M5=l?vq8WFAiy+-840AdP zswqC8C`Uv3i#>Oy`yg=@m~h&WYSV(-wi%E6nT4$5vyd&leNW*NF1%O8HsiY+^{XR^Yf?~oe#(DVed0GQ^~AX9d6F(u+6f#jUMSB73il6ozXB#7 z){+*FqB%v1(){yBs5^KDzPRuT)p3()Z}~*CRO~=&0l818#{aJI6r09urgJ~QGYjY) zD5I`Cu;B^T{I3qS*v>U49}eRd%|4;^?zTAKp0RYJKE$0+7a3F;z>T~P4F1f_ z6XT{`V4jWiNl((*Usxb)h5`scOy_C*e?+S^B-C~nH5 zu3usE#+EYotRFygCudq-h8y`=Xmhs^&)jo_as4k~WnK;h`kdf&?K1fDKbY6OC!4;% zJ8wNjPX>hk#S^slP=(I~O=Vt7_ITY`ma@SX>@Lod9@nD9^jhXJ;I}@H_3O<$O*{eT zyN=Kd30m#DzV0Tju5ta#|6t*$WFC1j8NL~M;EV73gqruU4qaia(FLGmNHx~5j;qkN zHucV9-b43EMl$(PDq!y&_%rh$-W&9S_D=7vZ9a7__xe-}Mqb|he3FxV)2gLR8e+re zYc@ zyk{M)!Uq@YCFp0aFSpQ+JvWv=1L^lrBp>RFy4De}cd$MGWOo3*_*IAzHO|A_e?2JI zE8t#+3*N(QO=iXxZE#wgm^!@<@9-p6)blHZW3?aA{kR_NywwP#IfBqronYjcTC_&D z9jDw7()c0XK02CDd7p>|2lH|K&I@$kVkazNS1~xvP|SW{E*I-q^56nPl^?uS|6(;i z_{!@w{yUzB!39)ry$`^$NgE)7)?=A9@-jAU`%Pn+I0GpURZin?6HlO=7ONj#N6K9h za`zN_VZI-!{$Y+eJrD8t5JVTOm(8C=$tg|iP=1c#cQh`1X}KkQ)wW{y-)Cc!wAsFt zAFS8g6;*gLDxaQ@!f6olC6LZd53Sk}CNFA@*=NG=WvBgeYfKvSnR6PPT1G+hBcLJt zfwr#x(zWb6BW#w|!#?9x&n!@NA;_QaaK25~zgB8K$3R~vV>Wnk5U&+d0G$W?gLgXa zyz7pW_<>F2U9ZH6WxY2PxtaU`=Mhmb{t~LiM!2_D<%*x13|~bFLHq9bmjmdx*YbDTe5tgU&jJ z+-%i!WVLF@#Cz6~Y94;=trSJ$ny}&4Yox+Kg;hJ82g;r^PO;XHenPVYI&HIdrGu&+ zs~Lqdd2wG(bHSI-xCVo|25B$;+y%rJ&_9cw;XTof^H;xNpxG)mdr=R$rt=MKv&WAg zUzrKTT8FkC@x=-k*}LoI7-wiD6=!^^et;eJ35Qnqet_;!F9E;&1>ra&K9D}&juw^w z-mgi_4dpN0_6xscRM@cs8SI*@d_N9+7zAZqnMSL#3tyQ;qbqvP%)|{ahr|eElIhI1zW?VM`1LY z#C<@*P?cMF%3?UXee}6@VQ^jf_C-rZT#J4&w0~qr6G@y31HQ&%#Qko(%eXkCJb>^^ zL9iic6LarX4ND*sXC<0L+U5EX_&!qnJMJk}+$k@$Z!hKm;W=T}TP@AKc1)WDM*PEQ zKEx5*CRp0@DTK{6k<&`nvLQjHFg7MZ&84Js$!qqb!1H!Ht$*Pnw%NWv)?U_z&$)0} z{f>Qaa!^oCVOjiWc`$h!tUGOkr5kqRfnnQVk#cSkDO{62&3Ws zICqgWBo&FTVX@A7LB|ju(sJQRt$j#*FWN0?4`*{9LxZ(NSXQ_P12%U+b*)LR z4gh#)e-a^_%j*(yDnwAl{mb?g)pW&m*`kLqEGIS=g7%I!03 zj~{@PXQ)|M0(DBJLzi)ek~AD{uSmlu&GzAq%UP&$V(!1o^nQ|1dWOd80ed^x^M|f4 z8EG;N@phFa>o?PwKb!3a?RxzwI-TVRE>}Uk@8OO$8%=}oma8%9dM{)hYRE5>T1mmxb))r|RxG~lo`n57pAp_S=^jVt&T#VK zBdI&|5L>s;1_L7vq^f(;osu*jrfk^6mfxr+NBYwGM57Gwn*B5E;TNcM6V3l&-{YB0 zah_LCq?%E6!Y78BA?aU9ni(jrobXgSrZODStrlR%Mp%t|O8&>uRR%<%1YrwA!9)~9F~I;7 z!GPOYyA``zvAer501FTmMNu(OQ85tYcGke|E^K_Z*q!fNe(8H&Y|MOhce}BM*E#-M zSM$E|?fRj7v0nv^+E7OOdD=+PWA$lO^L?;dguA`tK6TMcJe-Y=BFsgD`?&V1qekzt zdA)|sac>;tx~o9={&iZ|6_4sal;zdtOjRd%dakis4(R|C*(Yv2m$vUxlx2 zP6f>(z^z8dDvzew7`mA2oU5mytp)sn85=)6zSGnga43!*luy8Yf|VGxB!QjWwo@3+ z2wG0yPtoCHUN!b}A!zvnVtDuSw4`pdLJwu}A+)>o(+7oXrbmSwWXW=zA z@PU102ppmgje7{KHxPV39$MjqwdnV8)>`FrN#H#@TsojmE_EmLBi~>4OrQ?+nO+{I z=K09o-bOnS@Y<}gf=5zK<~JatySf!SE#My{a7UKQ^^MV!mYk>j;G-7cwc^>$(IU{7 z+RI@Ix%LbotUuP>ANw6l3l^+o@TMh?%KKTN%0$TIVZeZ{)|| zG~V33E43+`msdN-h{kP?8GWt4+3gj1-i%hAr=0@E4hNRv&c31xsa4lTG_-U+cl)i7zybht9OqpZ$FVJdU<4fxBQjtx{MQ%3uDYfM3-m zd?wKL-sx;K@amLK(r9)(UczHir*4a~_!0bN4gVTVp`($br)`lJU4snQGxUZ3V0n%M z+vPDBD?&Tr4#s^^!ssuX7w@=f@euxbsG2xhppSTcX^5$Bxi_jbC3-#KK)opSBIXuYJ7?htV@3zspkto=2Z`(M?;d?30E zG~a6lf5HXUCo{AFop5|CfZZ0aq&ezM&R0e=nmx~|A?jg6i$AUs6;{r+LYG@nOIGX? z5v8pnN53(&?}6VtFJubo>u$^EQtLA275R8Qjiaek^xRTSN~%2gz)O z{3!U<3a(%JFI}h-FH*9_5j?MGG;^WY(D^p>;~O@A0&6L)!6V~=)x&SkECz#a4Xp%E zQiliDEWqG%g~w6UBNt)ztNO@bGMd9^e2)#2$Y?Wo4~CW#z<()A{87uMU57?1!O+Pn zc+(Z~Y&k~&L#!R~J5J0)9Dch=G+jK4`V2Y44Sp{W@c&l4FY0eD@BY4?uusH;&FN6aU>6Mz_wp)t2lIjz$I$ zq8eOEgZ~~t=VtXI;{gr+Za?BinbB3mWS7<=q5gUWZ!A!c%HSZoZK98_64_|#`Z&85 znTO(t zCCO-pwF^62t`6L$;LEc-3wp!?zsgU8UieoGW_~ksx5-S%O z*OI|pr{kYcm;Kb3DHnv93+jc9=H9+MJ?rnEHVvZ@%g!4;3*I|Q6|PSr|1P<#mh@b@ zLceShyb0dgqG}(X&&ZYt+Dg3~^pa{eou+H;`K3mdk1#&MYQ{6ZMaP6-$xnnbD65-q-@)#y4wLf@eywz#EzzLk0yjhwh9I zSn{yF@w41}`V#7#X0M^UglEg_$YSnq`jxLsHngWDMzmv)A)=baNvvGGY* zU*%o*2Tk{hVi|OlUuAfk#|z|$`u@!wgv^1rykBQQ1_m#C(s&jHt`d*aJvsY-BTb%? zg=vqrCh+dU8#wXYU1Vet;zZpoj9g3^oeNB2yoaiF|1dxPoWRK5=*#duzyo(W(QONU zo7A4_pI%_JBx64n=BmLkCY(CnpI9s!xAO8Avd zZr)jSL+wU~5&XOGUZ^?a)N?v=_aS4SQa{~#t-yK&oT9L>RTMKj#%NJx{4IKxD@@O3 zbfo%^CQcHAb2^-;9x-KJb#zt!52{(QhXIK&A|QOXHBTt7vws z6>4PO2twY)KMG{hZ-;%>;9Nx|c*y8d1{IRH(@PBmGM)^=rd#u#>cDnx5GrM4SA$Tm2A?z$ZSI92n_#+Uhb`+8G&yZ zG`)G0Fj=#YJY<0`pjFe?kmu~|5?+D&rZnUDF0o{4Anx#OOVi9_Cbv{oYR)w|d6r*- z<{@AK>EWB`Oy#X&V2NcSPX|*h`%gCy&F{N&ap?A)un(PKz zjg4OG-~OzEA5{(^W4O!{92xM+PR?&v6yEl{Fj#FoHE^1cYl&&U9T|Bc_WK;X)M6S~ zq5y+K6g#ckf-Sd^O`()Su|U4c>Li0qiJ(nd>7v#SnP^)lK1XarDDzM zOP}=t#2!CY=Sn_QrqN2(ut^b0hn5Z+VP^t9!`p zxTp1C(h|H+QT@A94ZYMkTyD3v=HO0$RFhUWIPaqB(h{?TDixf~*ER1`cmz-eKtR=os0H#Xsr*0wBo+*xu{g0XXhL|yz zG#)1hUMsDaPkbmQq+AuRr=_blXFl+V5%^T{K~LVCJ(C26qD6%vo4G>|AJKVaimG(PSHJmiPCZdI73M*IKc3}Zj~ZC7{5!?Hj$H(vTZ|Yb+D1c*>o9J$3l@-n3=X19qQLPmg?3NI!2-mEwcy z(X*0^uIj)vv5Vos5zvqmgr`H`8402;bawm+;?~?bb{5XW{yTAKo3%a_|zh zs+GI;>T#Mz{T(F-y`3q$f30fme5W6uyF650$qUX5UPQlQNcJevLH5WfjB^jR)Addl z)JLauVQMwr&7wyiufogilW6eaoP52|Nt*a6hu(53f%;F6ROnURzsx_L^79xsopqRO zzfYhgJ&Vc7%bYnH=O?uu;vn-j-bI%jd&q-7qNsw)EzY-X4u9*?LNd;(N3H8QC%m)z z$JBIPuPF4-esA(y%hI)AN@{CPuJJAosZ!0wsb1zFnPXl4?>;s zhj%q(5w8iDbGLvg?`DYc_4?AFW`((BZhxKozb-$QH2#0U3HZK|I>5o%k*2aJ8D^}SZA`b8_#)B*44Y3G(y?8j#s7~GzQT?x~l zaE`^omr6xcE+)~-G;vc6x$;6LmDKPz_Zz zX->|!kD}hRQJFoyiRo>xa^Tn}BIf2gF4nWjJ$ePD_a# z?y_R@dI~+j-F(I{`dWP5=8Vt(T)1_48)~pP27OSSu1?rvYRzDh>>M=4)SG4M=mTtR zvxy!C+tZ%jD=46NC;Ig|k3?_rh?3X1Qpemnv&SSZ_c(!Lai`dsIx%|h4O_i7cL-gS z^ZCe-*Jj-%dIMPhi}s$atHqfAw2rSPb1cXs`+QzaiB%G?AKAI9bru)-=qFB;Z6#wO zii$zG?A5oR2w+8`c^#tX9Zd}i?3pt2-8Vl~oq2bGyR^Q}>&EWqZ$X8~ae94l-d5gt zV6!#PWg#+m9_QgyQv&;G?eBUL_!)4w`(#q86?g5nhQPbrd{~@1GJda&Y$Nzqxj-sX zKf9hgp^vKg&`HNctW*b9)s_C6{{ue{=f+PeiBmaW)9nr^G$5`fQ>{&=o<%_Mn^yF< zeA+KkgsdKE$$xPR zAWt%1a*N}B;fZR)vf2E&NOuOF6V{)DzpmoSH8=C?Y27sDEl>l(^Vv-`VqZKrJ${nL zMO0Cd-LuOte1MnvjTFEq?(fxz3>NLSTgatnbtd20H>u;?>lVZBWBW9edk;?qcBIkE z^5?1DG)Eq^)P21OomE*z?OOF!Qsb`mYw6nx!F}VMG+7+0e zJU;o#9yn74^+;wN40d*0vWBNU^WcBua`4@RM&d`}9hx0F!(zC7R4YjtVJ&6R2P>ho zob~)jZ*9}`7`>_ILMw0Y7wuP$A=EhmTY2IORx3*?k@4jlqKJ{+#!q$ZT(E+Z|us-1P>nYVeStF=YvQrZ5H#oU&}rK0=pU zyP0|h)|Q{{Mq5lheRjyHhf}gzKCw5gtT=~Ty|~V0zMQ7xzbi?@F^{%aHCV{tc?z;E zL!0iNR#zwZ@b(ibqW;S;KH#cCXwhTqW^d`Qqx!t~FLTe)~- z$V|S#ff`t8>1zF_fYCIx`)b<%Ylj$FakWZ%o{xcRV%?xjLVeaj1|H(gJ+jGNw@UC*{ad+~u@e{OI%Mf~ zv)2Z9rfn=`G`tlaAYfi!>=$%)ZRfgzkn=)QT2Sw*M2;<1)HfZk`5GL5l~;70PVOPO zsr%PNhTf;Kdm>OvgXl_Y2bryTSBAa`fLGz&AAeGT&&_}pOGHH2Z?bf<6_uyqEXfu} z2pZkr)B*a_lCMj3Hk|dtT1qyKEo;5Msv@(;RCe06Q$btH>*o(CQwQy8I_N?-@ST^- zi|FO6FgE%JxMumV{zYm~@-+qgXYe;oFIEWq(vb#U+rV|2?XZq~5<#tdofPm*z>e*F zwzNO)x_eCUEd-2|n;rJk1AX3l)n~hS*EC%<4YR@7Nb!swvCML7r`5z$hATvL>SY?A zdxPcXpNd9h>#-9KmRcL`e8uKADHt-ok&vVxkC)g5xijB7P z-07r&ixU1wLDxZ3EwDg0@pF8J0WX3&?I_Zx-nHWW1=hg2;8-#0;P6E3w#&^OuWnSX zCqEl5kkGP(KBEg2ewcN&_WCp0_zoHJIlJK@%(E{yv-oS^m>Qg<33`g&&kKWA_111r zjs`Ts@4zr09_y;JxD48!;8)4`CjX)3bQHys`-P<{&xtPtFCh(=+P!m=&n`%6wk1el zJ<05C<{-(Yw-N^rm(Ux$vZT=Pi2OCP6z6d|MjMu8qk8Xc2s)d=JyO^PXlMs+)Nd@6 z9cEP+6Ic5A3HV$^2e_|g)Hqka)gQeuP^P?XMg40nvcL}s(^~_bD^jg0<5c?(^Z3k; z#_aBIMQ?H6^;NR4t zlI7%uxr=GQs?pF0ohW=%EQ4S8e!GnVkD=)DziC~HC9D3?bE$^kpcgd$9Uh^G0xxUm zb~z&cF~x3LCwi`1M1?!KVXvFW+#{b8u#~&}S5cpQ;Xv>TG^}W3YEqd?0v_g|w|r_f z`xtxw0dLQ{PGx(zOF>WbpZd{87Xm{Y(6Bj@&1+<=h=dQx5SqV0hW3=d~{d-VlnFX)W3Mkcez?gTbo z8`*;hOxq)F)c4Vb$G&E+H+WBNmhCY*n@}r^_k&*DpzPU=Mjzj;4F4KTI59RxK*KTS z%snz91KQfeF??x&$w7emG^KfjFkFZ=CwOF$(WsH^*8;zj+XIw@lSu>Z6)vK|Dhos})nMu9Gf`e`0MK9CH+vljDHH1nu@lxb&40x3}@p%Q5zOu-a&8x7h7E*D_b!*S2)H!OSe4qJH)f^ zH0Pmh+RLxL6$lv?8E-wq_nj5|FBjkIZLlq?U-q6o!oR{N@*3AI7I%Dp2VHMGhQTi~ znO3Xa%lUJ5t>P z^(;d-Y*6sUx_7`vRuh|8)@<3&XS+=SC*C#nrZ)GOO~|!m`)7x;yfS#-WHSoch_3ay z8kH*ICE*{qSb^-)bLR^5v%T(RS4-x(n_UI(EhtdSx@Tk~2`(2K=2-=LhtZGL`H{N_ zy&{diI`b}&i`0i!+r3<1T*x&7sm+-&Z18PE^-QC)1=fv#g92E|UZ+0Mvf9<5RVVOz zmjc@DZSE|rxO}I$G8tP$`kActXjA8ja;dRJoxFdNP-CgUJJM^1WxamQV@;IN4(LtMwm5^b{{?iMayphhW+)fYQCexj>+t}>C(VrcgE~lX6_F36f zs}~*kinW$Z*y>Bj2-x&(p3-^b!-|vnMd}8F*Oa~EdOmbJhK^R+?!y0Uiq}s0{)+jr^(kPiDut9Yeg+vn-nRc4>xGS1sbjbZ~RB5Mj}eJ z&GNp;9Jzd2Q$ohX8DZxwD@NEcuu7m`2^eYmP`=omZZbQX(=7)N=$R;hv&I8LYuIaW z53)a;ardlt05F?ye6({3+9blh8$KtuSgQrOq_E%H#ng!cwu!@k z57U62RTZ?8sbRwN=*s8G##5-GOPd*-C3t9tx60CxCg1BeyLMLg27HsXs(eyM@tIe} zWlaTe#5%9S+*J5=S~#vb*IO}GAn#>htTLQY^g+>7U@o$>+w3?!jq}?Lg7ztc?54a1 zZVF&(*4jsXD{4u3dl4D<8mid*D4!j0n7=tT1a~D{AHJdxaP$?Og>6wx9;Uq~t~cYD>>BY-VaZZyF;_Ip#m-W(+t_fHj1!<&m@?{Ox> zyC6Nrw+1<1%dJOtX`qJbo73L1 z;@iV)mvWEF4>)7Z*D084CcLHdHYFJI;%r}EaYeO>dyS44&7+4{Eqg9d>ckx!T)3<( zAG}TNDmI^cOztGd9!uv>#e4FbxLSDbS{kwakZ{W^r^|kvD|^XB^rBX2o!`bze&B1o zao`8B{!?YH>gFJ~7Qew4ee!6p#h-Y^Mn|gVSzMRyGgQ>=UyWb>>Z__nT%{VnhsdZ! zPeit!MP=u4-z))d?CDbB%Ce+=PL7{5TVLy)K`S;cr%lb~OS#RBA4V;q$fwy@pP$DS zLl#=j2Tsthb`0b#e9qdX>~E|0CpUTS-7vBn6|Dbk2o`Hy0_60J8$1KQKe#qH7mqK~ zAHSiplbLOaI}UuwbA9H}n>gHQo>YRvFSSwEe^ionTnDIydu~v(m;H6?TH$;(`yI~S z;j=Ym?*zUYvq8n@Qf}iyYk`&C!qh>%M=tVhVk7+40%kE- zJ5ru(+qtj|)-PK2zx<<`Q?5)(fRuvT6}hldgQ*}vfyHM4(#bBu{Wyts%CP@ zqm5i7dm~vjo3|X+;sAHdwwH3-b>R`)Ypd_oe{dwu<#)^6q&{$ax$ak2UR27lgITIowzMVHvhyZ=u+LOJTlcxzG;S7Es^y?j<7#%`2u^9C#qG0GENRa+aM=;xDd|gws&&*& z58FFJX4_pz4$VxUe=oZ@ke2RT{^ zJth5O`!MQ*{YRIRs5d^|u(Ist@5KcR7nOP3La?8Qc>B@H+Bct&pXP*cEWSULbKMcX z^Vg2L6~DlEUw&0n$UBztpuz2#BY6FAVJoJ{W#X$`DXN0&&esYADI#r+ti zWs`fIw82W#`{*U^wbYhNY{vOIjXTLvxldDI<+-Yqb0Iw~+W}qDKA4_Xj^w89Qbq=*2=b_9Jjh$D}l8V15 zwrda-+g4omTDpMf$7-?O;}p3+-pijS50rmi^x$2QQ`Mo}?u>q-zz)CZ?V1qk+G!DI z1QYj(i30BB(;m17bZyHU6!iWV@4dE)Mkk-wcN|X&!!y`p`ECAqHRHPt&i!)a&Yo^^ zL{xV6YKSvBV~S9FyGPV$<^_K8ue|$cp3TSh&(o_b=f&rVC+J--oO}Ia zok)<&Y3SNLS?lb!wj#Gu@!J33Z)(>tR{gqr5m;&?^Co&z;Z2Ii?k=f&-RvzhqJQ`w z?DLZafyoXl=H z`f}8g1}hS=|!_aD}ig;H(^SZcs@AHwo{!O{8*P z7PXGXa^d;+RA#n%vhR!U^PJxw40fB|qqFga<-f+ysC=8n{J_~=mP>A^ObvnmX~2O! z^buz1_FJ&)*jc~`OK&Qfk3 z{nC2qSO)hz8YgCjdde_cAL~10+2A>YTX_GRGFQtJs(1C5#BEAs_0NN!jkxQu(*hX9 z;00OL?+kUk5Jhc&A^*lXt#}>X-mZ_@-?$gIX!2H}H%ZK>MiZ-i6u$9uX^pBX&HjSh zxcYN1wWs`Y+TZvAeV+V`$5qDfuYYxwz+%Q)sh?hFfnS@r)tv=AsoxyY{!>{_-F{MR zj$cai+qIN4)3>~=#ISRgfOZY}`;$ME_N=s?9hgavUw$@PPA;60N=+K%kSp8Ztp93-H0n&= z*tVNj{VSnDylQL1H&tH!CNqa3PoC0epL|%4W|mTWpQ@k<3#`yChA;Tp+iME@#P+>> zX<_q{x{cRzHqHP-?`A|nyQZLQwC0m zCA;R+GY3C*w%yFJU7wi!<753o&0N)xF%1ZuO@nsLq43PAdd~Ce^gU)Xfp2MC&o~7R zBgazZmCK9coLb-?VN4vJnW*0;p5UMfwxaj^{}?(+k15iLlhRJ|hPF75=*oA_<7S6? zJ1%B4!(Dz6{qXtl2g_Ff>N43{4EVC#0^Q4%pM?w5BD?$UQqW`?bChn@+}hNFU#Hv> z`^(=`b22a+IQf@=2~_Cl8vhzA_X@M#Mz=hzbIIT>amj2NH7ULx^<}5#g}Br7)ynK2 zFdSnEp`>M%+4ODk{@!N%1nnlkcYNsaQHGWAFYb0U*aN03Ck(;*(XN;4}w{ehl z=7y_^cjof-{DY|3#zkbf@XxhU^s?U)=+2e&>C9vGeBEt=4x!L(9y&)zPI0c&DzV<~ zg(^I{6hUiI?>Ec1*!u6vXjI^&DA>qXI(hAbW;&s6&)vXLAM@*@A*)308%q6XzCBB) zjvL#Xi{4G`kAX%nr_p<=-`(2KKQExEL+P#4Ulp-7$vQf@rSTV8x;_lwJ1ALuJzsg_ z%8UC4$j097X#d|_)VxhWTGwwr8*K&s$EaO17Jkw4z7^Oi64Q`HOM zSvov5D%5Ip^}e|!^yc)+SsKb~@i^7L{33O`|52ll)dBnE>|8#lZ1K2;G zJ@1bcyPB8bbDfH5OH8U2t+MK05RuH0~f%CjxU(zXQ6MzF>?^-W>kk0{tYSr`cqA*CP&zJiO0G zg7#E_m2=3CHl=B9!dJp`rCCexJwXeIJbowm@2bcA@ZK|W$d@9nyosTD6~}4d55W(Z zy`bSg15|~vw+Mch7R`00$qnNKyoIhbYXD(gjaMV|k3^r5^<$*u;DeN}Q%>~fI{F+` z$>>ovUU^aj``kv)QMTO?%4n8r&E56&lS>HrDzIN_?6mB($2JL?{|SS4>Ae3zLj6eN zztIy}-o;=g{1``dIYsNtXS~zhv;6Uv&_q6V|F7}P{BHVo6*IItJ$P!TUFJGS_&d~H zXPWrTS;Oyw*N>9BuQ&W0ur0mFRSs5 z@8gRap0U*U{gU8$rNJuq$JOMz(^{qM{lcza+VPsVE%elR=LLGlcx!EZ4)lUFUg3yC zRpB@6xB$mX^qd3-=|2aKs3+&`G`CqRTyVxZ{4zcMxRP#NbEN?<&KvAD9)MGZlu&Jd zwdLBwAF&0$0W)~|V{iy{{+`{`20VQ;3Y`#U$+>R^w6dpq{IRIE*>RCCZ@MH>3(Vp! z>1DO?g4i#HjuA8C^I8XQ?oHqmb?X3gb^nTdyik6EMz-QPgmo0CS4-5QAg)`@NjSHf zp-|Hb9-C(7J0_Oi`X}IP1?q}>zd`;y@CP_2kxf1TZEEzoD(k-9)PVjrANSe;=~54ab(I}Q4hPVfnG(3fs%T%88O_(*7TwQm1x@OM7}zc1jk zDA%uM$}PuZ<(p4YwYr@xp*o0wttzHgG=o$4bmCEk{T1mw3Y%<1d?{QdD|><8;-L4_ zjIY$6=Gf7L7Dov^FR$&nkALsRNt>U;6Mw)NIV}qYuWvPe4SLnm=2;OP*fW;FM}%A@6`GlyynG3~fZx71lD%#|6g?VQvHvTPd;CYM z{;g$X0I4h6M3eEh;2!=`;kAJO2k%$skdyPX^mD^YO{KxMUlWR|G82nS+bWyxdr3N_6wiZ3K>pepl@Q*GOA z1Wth;S;KMD%V=I;w%yT;$)@ zcGj<>Ye5g^l+)ab$hJFf($XGb++<=bdUyxboidV;333bHz2e~0c?6wH(5wtjfxr6+ zUtS126hPl(4E_7Nn2k?r^%nW;nq1%_QseLB)zH1E4Ho5<9EqsMFhi7sOG_5z=OO5{@_kdBPlLXt8GDLl))@yhiapn zGl4I&coTePMa>epfn?`$W{i|pWh!Ts%q5`;0RkX}Wil#yp+A}mZoeSMh&|3=p!{}+AI%*kj>Xnw2L5BP& z!{8noQ{<+>GeY*MW^ofXe~xsIw}#|@$iNETU2hh(Xfl#tA1Ml7(VR`q#3f3CW>mHw zDe&yw6*!clcb74Fi85LUYatFz3nh>6n!3*1f)W{uh93~d$NgQ^mwn&eC1eI0SXjMFow0;WM6H%1km}=ald`%5Cj7W%5t=e+B9L#UmyUrr|HGO1XxUUCRbCWYu9lQfYzQy#nW4 z&Alid+)q*|2}@{P-#oJ4np~9kRZa3LkV;F&6xT1knd;md#D!Lt*6r?8l8*yZ`08TZ zGn0@{PF%Y_AbeMLs<|`=H3^L)*Jksx9(y{^BD&*$lAm|q#btYRkef4}sgP`^c-EX5 z*7YOo<^6*HEMK3;@Y9aLYI*BF8fF% zqHblLT3(+^l>LW0>H`xl<4mw`yk*)P@-CQHZ+TKnZs(&saNN?A^cye8?mq`~${TCa z=E8h-GIPtb{lt-;WyFVxr>Wo25BPs1^L?Dlzww*?rTL|FaXd|5@twv9r-RfOcLk5? zGTahXGMB!;?KDkqJcuHnec_f>BIuNzm30{lW8NHBu!{^1Tf@(Aw(0OW^Mv#EcWSI> zOP%U6QshRzJ#5U@z*W(FZpsq5vCbNCu*eFm&2wRMA9vf-nnKspR^vN~opfxjdDM4+ zvuqMtRwnk#PHqR2&G_ZMt2y;Fd_3OlM6 z25q*C?v`IZEL=tp$h**bBzzdRtM`Zp-CIF;J>GD$FeTYfR8@K*b zW>3q0UT1BS+f~+W^v1bZDU^k{G@t* zEQkl>NhaIIE*R%5Yt>>lGXK&Jv}pYzsyGZ|*j`ej)(Le(KNsiaHnwL|Tb|9JdX4MS zkHgpHsP_j1#wZ>o9ur&t3z2D6_b~QVzy8-;KMt&ge4~gq>-y-)1J2uSg#OgDGq>HW z#g%ud6m;!7GOK#B(~MK}e!?W$+<1lDb1sC-o%GV@oq9{tPknwD)K5QLr^^d`b->?T z!qnrB&Tcx^ANNq74QH&ip7*JYnS;u|^g30%)>fBnoyg^@R_CB$X(}VRHtHo#3>>TJ z-RM|0y?~mPy{mSY4FiSj_O`fQ@Uw(Qo$#E4_A;eOZ*93aChK{fnt93VX)fwg$pi`x zE6!#Ql5Ts;uO&j5Y_idV`_<+7j3}%B?+o<(2HigYN_A#zVG2G}Ll*dU*VKp(pX|iG z3u=<-39m_`7<+*=Y>wZF{1DFd!z6A@g$&a>0)lV*J1ifjSY*!%Om_ zQQztPoh?-6Q%gP;okATPN2!~qkEvcKuW^-Te=WL<73*$CrcajEzfCirCh)g;waI4J zOlj6(tk*i8a;t{v0n>Nn^P)amE`A`4+w?ted*!QFe_bLyF2wVq-1VhzJ6A?clG(2t zQQJhL*SmT6fl_=sXEz%0Y7xdWnfhOE#^^sb>vH4NSpIRi1DnT{tr(!Mz7M0!tChKQ z*#ca%atDqoIZpw<#K5gx^h}o~8koU)=J0lnVn1FNqTj!<5afC zb*_CYKm7?Bt891U_laAU%40s+E&GMNSwDSsaSvU-Y7#lGR%RsGTIjcqo33r>r{x=XLK@ zRHJKA={@;~WxLzf;%wjw){a z0#n=iN9S09brOrI3N?5-PqiJ@mT%#Vw+@p^N}EEvIQJ7p&%1le;6n$pxWM=A8$LgC zAi2$RMXfsN(e7n+`i;Tbxqe<*WXNn_Dz0d{+}BchOCSC5;UzK{`S-Vj-raW({j7M8 zG7gMj)1$x%n$^k=wUri-??W{SPNGs}YOA9^#%jX{;7_&ZoSnRPD=(SaG;@l^*Ha{Z zJGPj1i(i#?S>|-;sb>eTrw4`FaC$&7vOX(EQEe_B?D@q>{qE(0b*pD^-td+J=E@EY z=F;{K_S&rTwEMl~;3@umBK9$La*ANX)d}4Wu;JyGE5?%HoPuYD^0D*nxceeau1{~O zd53Lv(?j{CS=0YQ0{P&cfi!Q~C*p(q#8=l^Wca&oqfr{XEDFa=QD*V~ZxfR)#?_V?>^SDksKEAR9ir+zF7VC7nnz2bAof@fYS%#B=Ulos7mtLTff zLpXWjM%3|Z22b&Y=b2Pzd^K(zyIdaWxskvvl>b;=tPk_6DoYeFLfz=iVqd@z0&W1W zpKzhGiUy9$Z}bACy1XUBPhVz6sjW$0EJbnm`r_OF{O5-s=O$WHuDhccxJqAxGr`Z# z$#CUkm$f2n-4G5ux8Ka0T@ngu&-{aBLfSByJ8^>rJ;#_EzdlzL=bRrU^b|vL@R_Z< z_-gsBV(q|!{O!$n8dmxe(Sdz^rGdW;+=*!+ZX1B7=>wxpGObyCn&0 zLU~@|Jg(QjY0)(gG8_TD!6DJ%+^U9yEOb`rvklq^n?l|aT1RcVpO4RH6w$yz?Krrq zcEI;7(}G%%!Go!FHK8Y@U+2%JhPi5s4dTh`%8cIRdX0}L_f@YbeDVqgw{X>2!m=^` zGTptI%72#I$cIhxp%%j^yvQJ$S3!ciqj}@F0|uLE$(93rYhVi5m)b;U3ml+Eud8s` zN@3LedwbsH)exBHDN(O{xm~eTk0Dd^{EGP*cnIB6+{~ZMbJtyL%wxaLG1!MQ?T;E< z=Ql&j(bzr*8T%(HZ+XNG=X6phYI^dLEj6gA#|gT5rU>1wJz6KEy*8XDzqTnSQHR># zw3BT)v1d>^d|U)=n$jC}0=?4fxCQgoarbZ~=KX_|9Q9T3frp7f$oT%6E9Sz7A)*CcB}Zx?)Sh`YX=TkDF$V({E1962kT zj#RSc#iP&gfeCHYt19{Q={u9SSFd&8#{&e;vqmqm)lX8EF>Sm;;9tGpVx>kY%Pqho zZM0R!+;ZTrk~IJO&(w#7H?nn~9UmA!iyu6&(UV73;hc5c^tRW*3fj@=7rn5Gzxq)w zCnZ0)#bdtt=xwFeljqo#Tlz+{7TY=E`y81Z zt2W5V)OOSrHe7IKY^1?<2`-h`7X==`TD9e_2YXu9t?|^*k-%14r8C_y?_loQCZ3E( zDE|xhw0C(!d;IEf7gxng>vxeayk66gsi6X#EXJ)UZ?r8P{_JA*iPz=GFuZC7HiuQ{_zgZDuROASR0CvN_H~=hSCkv0u0&||Xi+j{Ko$o4`K1h|NvylL?pX$XGS>PM#0ne?U{a^5qmp25L-$(=T`LXIe^ z-z0*3-LlDcTk)=a%@1)4z zJum-?cp^@12v)}98IJ-DM33=V8nl_hdWzYt=cO9o0sU{)Wf!S1+YgL?SLjPiyLx>% zt=~@!e7XkiTTZU`RZTxuvYr^)7~Hdgus1Z>;z;Nh&J!3#ljEB4R_F3M!K1iNY4VW3 ziwb%Wwb$0*H#I+gFw4u~NOw;BdzJQ&^2p+5@QpGW-{2FxgLrwuM&fm(!PWA=oaO8D zv$00W0r$u6BC`e!T{aN>jObaZnk?7TT?Tu+5!Sh170Dlx(3^o%5;e#>yW?E@+Y8izdI!|JW#j0-(yLT5zKZq4Z2~-*V(g*9bBv~@ zwAXB@d?dUiP-|0i(1O4H_ zUtZOvxL)=7wOaLg82NP`uW$aTD%WrHkYhd13t%QSdeu`HKa4(>joUvp*l2LsXcuKX zJh)WAgNb*e-RMd=oT=2Tq6OZa-TQAq-)`m$ElTK!qIslSn`{EuL**iluzR$S6a1WI z(x#*8mhakB=o9?*!(aFQby(D&wo{l)0s39T%W`u41>00TcqR*jejahB2M^}+jozf-_fo$Ue?~n%jWfDmUTCp4HCMw)e74{M1}`gkHKYCI z2b>#vdr$~N>uKvg+;erWF2xSFGCYm(u(DK{L*mK8-^y_On99ExI*<)UL7(D|%+2Cc z(_8e?n`Fx>h4|b2W!C%KnrgES21ANZn@RA2s(t>V@<~boef!cO>-#NX)N1Vq__CbR zyWtjr%!c7{_f__GD|)}#qMb^L7#t=`Y< z4GnAQ%!4NnlEyDp&a;djw|qwaQ9Bu$pEW+W{_K=bf1Kok+HjT+3SDNbH35?e+y_6N z$LilFf>QUUu<^s-InrhA;gimA`yGR*`1DJ{-DiFlpA5LM27CNR7=KmkTPUwASA`dx z*ds=`#Vceh%J66FAr&m(JZt)heai4^2gi59ebrDayp_t^E|cF+_(s=z4U;-ER+UO! zi1j_k^&aHW7z;t85!Rle(Yef#91M?}x-8)#{LOL&d{J-3KHgNQ5^x}=(dU*KiEk}U zmrm0$r`K|y{!RG%+m%M!i!zIf>hPU=sBvjm`SbG;f+wTe;~o*XLSU`}7)O>tiy32P z=s1&i@D%$ij5SjA9)z;Vr+S@iuf|4Qv${63p^5%DWBtx$i_u8Nt9^A%5eM?E;ko)h zD>M&}eD|M#cF~&~JIU5V;!V9U>WX%G9j7VwZFR`GbgYvP!-vy{iA@E1TEbf!>@*&Z z40Zw!@tyQ|F>7usE|xD=K{x3ZKgx>}Lr=1)U3dgx^une_$0_@Ut`_6F|6MrCdFCVv zcwL_AfOBH+=Od5Ri@}+<37UwIg{amxQwZ3lbVqzoXmVYaab+cP6~1h}z|g|dtcB4- zX;pm;7Yk@;y4JNYf$R915h`6^F%x3?2=l}r&@Jww5- z(D%1R^|@UYc~{g_f)6s96#nJ{yo(2Pc0o1%Z$eg&816SUVmu`L7LQ!Go0e_`_9b29 zo;_R2FZ;7;U?v%@Sijgzqr(Mw7QffNskXEjNn2YU=0OFknYtzL0I~sl4Lk#Xmu8dY zoz7z~f@)R3-Kai#`hz`|wSRE7>3x5WnDJIcIQiqbdlhm$z0IwaGF&h=sy*)=?5X!% z4JPy6;4h4AvM^c9o9eZw#6AZZ=r>fEH7{_y5PbDt@##@%S+zxJs*(}S@Ero<6<>-E zB;+Gp?nHM1kHD*YH4K<==_S_Vrv*8I6&{r@rB1gDo9%~8;5_b7c9ih(@JzFe_fpm7 zKjr+#ju5;dO>c4#cj+?C4{u04d;DU9(|xD8SWZ6Uw__n80iBrbZ1#%EN8y z@9)I{r{z`)^j1K}N1Ihxti^&oR`B2Ic#Uq_W3>Y#N1;N#9|igknqj@o z{%{WTc}Z#N0Qv$tBah5AKCj7Dxy$qQWb~EO)!fwY;(B<1HNgHvtMMhs3Kia;@b>^@ zGdy!|xIk9L$gPaW6y;9brCbfaSxpXxb+jOZ%+gljnfh0pEm{69(89hdX>#fvo1P-$qIQ5hxCbI(OWMr0G&+4>40dz4X0L^498 zG$dO+_uMq>P0HSTm%aVY?e~Y*EA>2|=X38l=RMAIKlhSmZzq%vAx^<$kKKa25mJ8u zw@=kb)tjUKp&*}$j?SS#eORi8F{rp-dF`XK>njdcJ`kuE#GAI&k$IMNdAwgU(af-b zdh*Lq`D7nYXGqJnhiPq^VPS$VKi7E_#QR;NbF+f@qMwHHm+2bqH*0#{>Eu56d36g5 zir5QX7CzOCf7*yI*m)KW?m4p{O+MZ&v6ZLv9k^5QPYfN>l+Wp=g^y0V0VbEil3_h@ zXV;6OcW7^Xu+T1VRkRN8e7{_bn>b$dXc^0wwA+m%Oz7;8YXDDzD|nJ zoZkjre$YCV#aA^KmioOB^qgU2n#*-pqK48P0aYE zNY`V_*R_^kHHF~U;jE}h`yM;^MDo4i{rH<~XZ1R^$F2lo$p4P-(m|HZcm@wTG{xxu z?&6}Kt2oV-5IzTKKa_9i=#ve;C1Zrs97pLB_yB7ZyQB3k9Xa=fKXm^r#cgcky>jc`jy(^u8af8k8xDuRiK7{rq324%9JP@G5Gg38C38SjP7;g`pJA>NwU9sR5FzB8 zUOemj8*#<99J*U}kluBYRZa*GxP&w=98};6-){ASbxT8K=(jjF=Yh0y&F^&HRM zgl=u0!4+JxMNZA%!Y8h`!|>q*=^euD4`9EtO)S#std1>@2WAmux^ z3W;MKQ!Mb=j~{S$89gUJF{GHu_j-o;DN&oC!%w)Ph@n>#{Zy`?a4 zxS7Vab2Pke`4~dK8ABhJ3=Wg~!>dE{(PwgXZdTy|9duW)TkbE!`6r>^=;J0`Vsb_E zzcsmjdkYnNHnBk=OlmL@_qVt%Anqp){_Pge>-}HgS$YIB`7<^{w$W8k;;y*m& z#ILZXa(3okFc>`s<(!L9cw{0j@81zB#`TbdL2|(ZOF7oHC%>=ThB<6;;Jwz)58);Hch*EdM*|R*r8kf7>F^REI$?)!JKXy%JXj8xW}e)vsqnU zeC00QU;Y($@9^R?4?LiALmF_a?vr50x#w(WR7csaG(t9eGo7uUYtGHHYH{aJdK&sW zpBQ-?=sv*EZUghKV^481=E>)*z-_`1{^#msws7oUd{x^5dS!2h$wgmaL-J_G-oLGy zr^G=Pva9{aDqhL$JD*}M(Jpuy+qSx=RJC;Zt3D_G;T@Kr#-t4`(DCM4?E0ezFBAUedvoan+uvdUr50Ogxr@do9|@`}d9I&A1?-`;{~oc*p+^bXe{$ z>(6-)%MRC+ofdAuuEbNv8--mrt)*+ z?FEv?%qo{`1i2+#GxlsN_U(5#FuwUWo4-U`?Ew*rOP!AJeoKN;TV9WHpVfv4Sb!4qe6dAGLXLBDu_ zpwC09KTx%)c!uT`2M^CG`JQcM>-wKZ0$BHPsvoop<*M zhu^`h5A!5p5LP@g#2Pzl@aZ#8&~7-*f$E&rJj_Qm{z4BINq7u%ri_*wH>_Y+U?Dy! zuO=Jkw2+DH15g~%`sG*lphJen#KMF4?4js&sgQ*oh)2SD%nTXm7npAlXl-2-^f@aFjsXW-vhd+AlG!`phj!9bTKuz0f-&$#yu8h*MChnnof zxEgIGU0=l3h-U}8)3YyYzu>^(YkdYSc>vAnJg;NB24ih)rMA?o2b)AsxZEETsI(5y z&-~9@6R!UMhWZ_T9lB%uaWl+!Y+p708|NFbOMf3>p^1*@aVS8TmW{)ZF4kb2)Pvjq ziicmOCXiE+$4o1t(NFiU2##0)qyw1st|h3udMdo@QBIiRDJ%7Evh{!Z^U-HlDeWK) zcI2?agd*I0tvgS&IfGOq3QxgqR|IZ6R)*dB)s!yH5^?5*>XJBGEMIp>ah#aS?<$-G zsvjJ=zCvu-un3Os%0;Im>maw*azQ*q`fDC`@Lzz&LH~Tx?d*8TwAbjr=_bv$J8v97 zd+*h;2f`(GD&{`%zbEc^R@3Ld`QLoHA8G_?JA&2gLFs~>4V&)1umc}0^rM=yR5h-&l!h>u zzp2g8QN9)ZJK4(_=KYYknFYQ&Ojv6H`!23i`UaYv=|OdWPD`94Pd9tZtWLVX8b4P) z^Mfl8S0lX-(igPR42g)6gjHzS;)_fm|3i#;6A3qb|Ip|<)27A)?9kdD?Xobt-Ds3|Cv$;HWa*=3Je`vd&l^uw4^s#!>I?V0+OEdH+ zoteAi4>mFBt|1=AfqFl*q{l(kh{e=GmM0tXowQG_lTUw@pP=G|R<-2ceI@Yf(he=@ z4`JGOH$ELY6}q;yBAsZUp_ppxCzx~6>x2!jMP9HG9Gv<=)f?L|SIfM-wgP>g(x2#j zbCd|5vQB9Z-eHV0A5@Sf{KaDV*#0W%=;}z^%X8zq;)=`Zxb9Cj1|L0*jjl8Y(*80t zsTrOtbi~$)#!_K7;iA-C{Z>PGj5ULD#g~F0HrDk#5dKQSX;i$}Whf_J{00fh`6^HN zE&p14+oavXy1JbtZq{0^TwSG+NZ(4*mPovVbzi1qX`NI-`WB3W4Ai`Vu|>4PSfufC z6>q|2RJhp9(waHmG^_d@)eR%xAhVO^l4fd#dA{Qm_T_)x8bjv{o+Mlu0CQF>SNVab z^tV;j!ph6NL5=y>`pXI{313eGc@P$N`VAZM`y@7=CV+G%Qao`udCa%jXMpm7u)HR| zy5s~eE&l_GGuqM4An!u|K%PuhJcQmej^gw3YqWMx`V4jo#U(%Y%6CrstSXjnPl#*g^unBqcJTEUJx5lV#6s^*LRHU7 z8>S&&4|KK-;Ut8-UCb!2k$6+-9^R%j18%ubAZCFn-il7CNcRLv zn~-j&y0oKxN#;S*fTyskXB?yPilBmdKsp}=Jjz6Q#D>neXoV+Zye0W+H80xCvSfvY z8q!y+biW~Zl`g?GyE{wrHn?lFt$f{bjM!>6PLMw2JI;-V-@SAh|W5Bpf!gp(5RD4MO zh3bTQ6CQhD?pS@ks;4n&r0R0OxHmA~;4>)hQ=j`c>^qR=lnM)TFXpo!)~}HCtGqTN z5vlgH10P*gy$2A#3Si5-y6XM-ZQyD#P;V}KC`^RHPie7%gfnEn= zdwtCB_efV^gW@AO>aBsy3}40+x0BWe@}>N)t}$XDMk;G34z(q#g$5 z1xJ(j?4^7=Y3{>9+)PsVAyjRsIwxO+q%)NM(2!1GXVb0$VSzTSeJ98)v*%i$7*txt zt+R(De1=B`-XI*_k81$MCQT*r8}%tpt1)8tN^4HF zK)sB|*eGWOC@n=eD&xoh@FD)99ub|D_PY_+8*NUSYz{OD0His9^|$v5(q5pv7HKP} zeDH(@*pbbs&#cFjS_<* zI4F-eg_HinnAe?fPy5%XynFSiI#oJs_|^4DEba44hXlo>LWb*LFa1pqh~2~+d#;~ z0;oS=2daJ+zFGuc$D zLh**undIT2PDKy~RK5VxmF*thKceoJFP)tPyLAGk(lJ&c#!O)V`87tG0w#`X4a7;z zZr69X)2KEU9(f6>&qMVtGPGAP%fd{qS?WN#bu=SPhNo@2h?_<>*lONpP@3u6z}j-K zW*zBFS_j~sq%=Jk>V$#Ma-kyi# zRnkD!pz>ejciH^LW8vV*6|A0PJQ7C(^)0Bs(h-jM)&=U-;O~7~h4=oYjCxJfFA7(> zQB2?XM)l^9dPG3ngZs10rRr7C=Yd(@3rIML^BX@={SK%|m7?b*8|sH##!an}(VVGT z75z`$%P0RKDPD4HdNg5gwx-p{FXBN;vA8ighQ1@xN7D6d?TGElcL<8P#9!lqbd!eg zlqo!+YY6H?aN{yAY!~aI!RW`>Z=9bRhak@)loreXwGmW5fbdchjxo}EVuMWzl5b>X z?;9fZ@KE(JmHw^u9i;NG<$>)P=@V4`UiE4^HX4XuBVr|K3G6&H5s51aTbq!ED}{M0 z$^>C9@v^QQsu?XvXH@m_sMn{N_2H*zUf)yk8dLq$4*tIxc^*3Bdp}DLT7%Y2Izja7 zEm}3Nbbs(YVwl1bEqP|v^6x>7cj9nVJTb;K6$m?F;~)XVA!^=5+-`du(CLKG*Kvk` z+N-PhlRP9U{X#sA;cN1MUPt}GT-aYL2c=bO>f^+zK9Hv1Q!OAI7sPK!xWcISMOrBx z-}cx?{i6+_anDme1l>9xfD7B4;7ACt7i;y9G#w|8i_|w_)YIdc;nmo?(1nEe-{D?L zJ@TBLCH2}gst-c?lNpDnf~skH9jAJ1JMH=!R$DurQ+=VT(I;YrBn>B`J~n2g6I3qX z>ZbRRdb2RXlyu?L9dOfV6H;H48R}Gksv`%V>e?cYKs52bDc-#j%9jA)3e4_i&Q-tV zcmh3(KHHk~@iZV0Ey$BlZ^BkmzX(rvHdFnGeAR1iN9%d12i0zF<^tBD@Hm#o9l%FJ zliA1AXF%9n)i*lPESY+$ldJfgOP87$D3i?FWJ z!O8;vsJGWm(- z$II?e(WVh5YubVBk2PY%_@gj-i8tTy@DmvJoQNB~o`Oq9U1h@P-q7><3G7fwYkbPg zxjTJ_*L$%U_s$v0TYFiHb|Vb=pqzcQZ*G*h7dH}J&y5DxDURIszsBK_*pA2Udde0ZpCX>UNM&h@GO^ngYu=#FQ1u>oy&@X}p0?pH`+SAv zMjc>YKn$C0e@y5kj6(O=CH%!BUwPLt8-h;n!>d`(q2RKfJg3`-hyT7#=N%eju3IQK zT|7}dk_R!u?2|Y?U^SEu^c0u-*1-NPqO~qPK~6kpM6Vx!33KYhfk!(szj{iZ%_ z&aMw}dq1(*OEH*cw4TkZ)|SunJ_fHyEA;)|A42;V;H=uG#7NybGN-s68k|!5gx8kx z_6ygMm%=z?+Ts>Ths#R-qg|=%mPOttVss>f?)V8(`p}wJ>ny zWQ=hb&e)Bf{M+X^96r@X-Zy^%&7N;$6KsRwXXI`+;?4yepXCD8xVsFUv<@4mgu$BO zJGJ`0x7g#0snBhAB^t->74~Bu;Y$O3sBx_kx_(`WM{Yk8IvIy(uYextlC>CZMjVEd z0blT4N;ZsdQ%7FS8V{vY=$WMLv^OcuO~$5XeBd@Njc45w=aaM8u78mjzO9b@v~L1S zjjE;XaiYIG<3Q)b>A3P|!CGb$Zos=gG?bm|-l2P{z~SF?v2K13IeMd!7`0*({ROlD;w!6Mf81Esm-S6KNDoq zPIDY`j5A&9moU8JN6n7Qt$3??)|gYg1vf0+4+f_mifosTP*m!{Z!AvbgV+7Q#=gz@ z&&WnHrh6nr{TITWo104Okc}dF^L$S8CY=n}#SP_@jE|!0mN78@>N-q#dyJio z*vhD0<@h%1aDvtb?Cu4D>%jB)FzG$~H@6zR4jaPedAH@O`FY80g_*2f(HH-` zovpoXnaqUc89{R@Sy2R|+yRdr+ab}u7U1T~Fy*=pxIXj}GYmKL&M7Bg!tRs8r92I? z@?wEgI#qrX z|B^Zhg&&gw>(IIaTQ0q_pz&ZGIn;Zs?0LO2Z{%+(>m1nz>OOT69>THZ9N1ZvM0 zTB=#u?1~>yp25?1R9iE~$s_Z3RpF>hVLyDEZj63w`{TX*<(#mXcRp$@Prmb@xGj@3 zCcb0n798w(6nfJdRloFfzTikOtY}#s4-dQ#%Rfy)tFufKc(W4dzDRFJvbCWJ^19C- z7+Ms7yGESWZuYLl{M&H4PgD8rzD5QI?*Z%HB{b$pZa-!;%x_f&DFgBd``1ciD=Ua; zy9Ak=1KO*em3Uo-GIVd7Mw7V|A}iY4-Xu{3A6CT z>32YQg1XI_q*~|ja79lTg5lWkB7U9G$RWbRC|W}SNoRFnwK@D zI1+`Lew{UIA2-Dp=ikF@&zk(aizV+lavu_J$ZIX%i%Y(iU}3UYZce;}^F;xt{KuCI zlG&IV1uQcB6x95B2njRu2}eYw){RB|_Y_lq)Ym4IKLv%^FTPnyXKMtb--%i^W?k!v zu+{Q2t87_J_a2Jy$CSUA>LxF=U5+*U1=Rta1+}b%G(aA4&VQ|DAyjFJ6L)nHP+n zO#2zNABU#dCM^BkJ#2QRgelIanr9!+CCgTGhr+3=zrp0^L%0*!M>dqfF%-;QkY&{KpXLi~CA75Z*A!zk08 zKpId|EZK+!MsiZaH;{a?K5;^974{_eiN$uFV~Fc=z%w@cwRZ)kz-dJP4e-4~Mv4m#hB#Wb+c|L631}BKXBxSzA91tsa;Vo*!lQYgQuF zj(RU%qJ`0IzUNPGN#g~=ZJ8Ps3sk@O$nz|UBYLvgupH3zJAn#E9`9TT zYX=`!8j?l(BtV5zU(#HSNPk(u)JwD{Rryoq-_uNzKII?S3TRjAC`T-a7FP>>NOQSk zSmZjGt^X7uxW8ELWX?`qU4YjE`;eYF1vcR;`I*Z55U95a=LOWPx-RJ*RQKeS@5pa_ z_94Bt0)xeC%&9*D#+hEEJ(k`0;%B))Tm%Z6G95gCcv9s(e7|{9M znnk|4%L1nD5TteJeJz<&hJpMzP+yR4!%J>Ota|Yw=%4DOatcR(xd1NZjiurjHFv2$ zcH;N3#c=p*J?voMBnjI^!rQiLPSL@_6!)b+6AF_qdZdyzwm_rld&TlkZ;dd^xZNK-4{fvM{S92ZS+jD}Dop zp1DqV87X>pAI8XIAscf98y2s`;{$37x(6VxL+jptc%Ww>#uV7fS8uN1pSVLX>4XdK zx9|@6r8iI~dJbMKUrqd7T^Q?>(&UDST3X0KWCVs^&-G6PP-u!6W;xLU}6E z$o$7&d)B6DDQWgvyt&ybq}ov!Bwv;N(I}5ZUInp{k33^=FrT!J(!sF!*D_dP+e+RE zT?p%kO;!4e|4G%C>lRiQjtzCl8(xCTjUS+h9w8k!Sy8{rSnR_*7Tf$1yEXQ`82)W3 z`Ns2v|9XUj{xI#@6XNx;XwbKZ()Wb(he3G?9XXQWbG+p<%!FO~fMDF1ANO}P*8eL4@t36`NC*^ zRjqQ;{i2Sm7c4k(7Xqd^3B_#}Hl9JkEW(K9?7-VvNSc^Yz2c3UaX|b{>pt7V2d2-- zC-OBfW0Y1y(se+(1&FgnNXjOr{C8ZhR-EozX)pXn=baIURAE5Sfjd>4+WT`8^4_?POLF}4bDnxNzb_GNz&bl*Ku*!WHg1czBy9`6F1n=OTgoJz zLA-qVa$1+0O`0l1)a~pA$+?#V=~IrH+LH7R>1PLNU)>ScecTMBj~L~U8V4tRuO*Bj z&C77jwYK0gIvxq1@L@YQ?i*IAQTR@up&j#KGI&N#0d@W-F9?+c$o0d6Z)F$f!ooZ%F}7cZ;|(H2owWG zyp36TwNP<0<+1qO`U8^wf`Chpk>ZM^BlyrU(U^R4r_#|FZr+qrU23dvxyut1T{-zO zPC9~eZ!28w*${1BImoitS(rAl9IG8HMvABMH9RS{11J6!`}`X)!gzW4l$|8cCARm- z)R0!zE*g~qpIcAEA)Q=sbIoyZYSn5~T6jwLT#>OU6W`8zfs_~QPqiKplhGB4Q$&Xa zNAd_qmHt7hMNXVAm6kA|eUs1s3gvTl7Qo3H27z@Z7H2g8VHUe(28#1LVH+ zaaz(+xMp~LPWMilE>86xa8ko5H0}raI;|amv>=^}(;HsbsmpIiA0+P(!${vKZwOVf zU09umecyvwx(k5f%?od>XKKEdeJIHP8gxjcUYw7!J!7bM5v8Tx0wWDcx_G@$Ye!mp zJm{nkc`P=fa);6s%HMFiPwqV8^fI4oSF%LE%%$YZtYyT3T&VYiv7e78AbB%fclQ*& z4!bEX?A=<$C8Q~^e3~ax-m^KeZ+yr{Ak8%k>bI$?ein|8VQa1(6XgB0^*a5-w|2Yf zeMX#k7wCGLtL3%%R@c*lbeyovyv6Dbxu!G|s(YVF-?!2{Na9k;lUS&^bX!$^W$kK* z6-IxxeJrE3)W2ZW+x7s;W&XFR2@u~P;i|U7_{XXj#h&zY1Nxbf2W85eIvMl?($<2! zx28qSbn1y@Y02jS`Nk^lzOZB#X@XL*V)tg|XqQ9XNwNDI|P8!Nl zMp{8sr?ute4cO443+jG2&AZ|Osd$9O2k{=iU{jMV*e7}ikUxRVHJ@peW{JCLOMF_7 zdMe+<_1pO(Vb0E~K2G-z7r>9sR-s-UqyCZVUyvpU5xqaBLw-G1csag1M!pKJ(k1^I z?1gk&JtSNagaM#|N_Kn4a@6%Rkv(VW;ybrQM)xA9kH%N5IYl)!NK|%7gr#4cRZU@0 zw=-~fsU8OWG*r4=8{$??izd%$CH-m^_T7b(zrqQ&cd3?WU*iAr1of8qg7djr`b_F! zhAFH@l~YRR5I&;vq{Lmq$-%Ko&r?mX1&@{~&7yEzmUMb9#@MZ^(m3>agqztgG9pAFLQWJ>#IK!-`dydr0$zJs_mw}_F4#hPDR1NEmcaotWW;klrG z5{)5BL;uHCX-kN=qw(!pEjVSYWGy~l6F*H%__s-oVA{*0K>Agj+qIfm9hyk}!yC%W z$)N#9;iBCY;4jD*VB-8sP{;MK;vQT&`Iy2v;r_Y=LhtVZ-(+j{>9#SB z-WUxog|w#``Iw0vQqbfltpgfuiaR>{Ye;K}q7il)=hfF#4~y1)Hp4xCE|IP|gyg%K zZ=DuCbvqySpRqvyfS z59aXKoPpv8!f^0zcLK6@<$=jo2S(f~NpmVq3e8JYe?5?AmB<+LetQ#WrO-1LF&Or4qCT<@qjeggad5ODlRu1if*r7Ex@&MI4 z(JmZcQ|kJyp?NldbrV`C%uun%J~IPEw=Vm{iu@pK9R8J^owgEOUT(wDyN*J_PczJ0 zYlx|L#`6PH-+}d%El{xg5_sSHigVXD!ch&!;K;V)I2O5Z$4>^b!M+T*)1?(GGl=J} z&qu?bs9Vfo@hFXU;xh5@ODY>S^FP^tt)3jwb0YuLXa4PSKh6fy)!RyD4{>4HkDJN|vjgSk>aN;V z=8G|?p%XeTG~!_?<8b#CA31Bxa&WVi(63G_ezJ`@Pdw{|2WFLMieGHzhdWPT2Zv0S zb&be7H2$f^N^7KkiZ8ZZxLX6#k;jSz-7{p(S|TsBq5b{oIjDPk+~kai2iPNhD_pWl z6KRJYV$Oj8zQehz%+oiJy61k0vGew8i(ZBCNjBfbA8QY3bd}Sy9y3MzcP6r+&Nm!a zIuHgIrJ}pZTJ91MCcplE!{+H+Wah0GV2^Dlg^$-s%wd;s;Hzf5*JlfPdrUOY?{UC_ zrSkDb2l>g{5gY7ls;jYlN1A1Bl4uSMOZC)zt$o! z4Q)Z=*WsF-@6gk2FHU{G1=Dxj!#Lm5(4buwp6H$kL0iM|VtrZ@L30-0ytAlw{1VRU z6GVFgmg2!TMewyoCu!-h3lgHt*~MK8xp9mEr|Z)7|FRp^#^KIoMF?|Why_72Si;Y> zxRlP|8f$DQRo*|{L;JXfIB>nBZfv9Xa9Qy(4Oga>gM~+9d`;&Wn$dHkE6y*MuuS6a zKoeO={_jczosImjBY(DjirAO#gfz$e>->)5OS~T+x_KMy#ku^!;xue%ca?4J|3fVM z@>r9%@*q54)n5Mlt^>zSoyFDn+w-!X{$rk5OYnU6A<*{hCzeuovuN@1A&$?nXyCj{rWSX1}z8hz+U;78C{*e zx1G+cX8Uu?T6A90{C_wrs3oZQdk3G#`o(ULe5WBg2jmEdwUGZlxDp23!KLfW_?8I^ zXy1w)a0#fOcsz&xi-y43_zwJQ)JE9+vp-P1Gp9)_;BX1Uj$OTA^6DM<=#ew$|7mqm@5>nZi*^Hl}7!Z%8!lQvmBu!q?BVUidyd7z*i$5w^6 z;plEU|K`OI>>1rjwx~Rc#?|iO`*|rflVFp{U_qEF8?E|*bC*W&;9r}-@U{j1-g(fc*W(FoE6)d@Sx`16RBGT_ zze+eW@35$=)14oy<0YOY7s*+dn*+s5yjWNVUKp&0?_m}^b4iYHSXUc^a-2Eg3o5Mo zowbU)#|xbQwmW`K?JC=Z_0A6u3D!JPp@uOprP1`EOw(WB;naGgW@nVl7UpTJr5?DLw~m-TtQ{lGIew1PopO~A~5u}nV@U4?g| zEDH!LZ@3j#HOiKY^R0ME=|++HZz;%} zMik2-=oNKUP=3kE>kT;Nt+wCR1F-w_Si;OHI4AQj{0Lr)wwm^E*CR(`^~H|&u&;+d zg3V;qsKbJAL=Z;fh9097?h)VBl+NWX;Mb8ha!%$Vey6uSC(Kjzs5lSPGw8XrC@Z?R zO8n7ZgZ1`|=Z!!0z+ua`2#SpeykIPsU%X3oS*Qtr(uyz6-w5ZgF2gAi_Gr`Fm!CLF zOZJOD;<>xIxOP)rS?i!NZtXu8&JBxVr#6QOiV+sSu;ok50TW3t6LvuDrnS+{?1fO* zrJTXDKUN`e7Ga_dIZbF7}-6X!ZK!H{Wm*1eNAC|+`}p^?Pp zta!K${^wsPMzvXk+p|u?zq#&$at!j;u3~fzR#Iw(-=D98k^!CM%h&nZ6~21>qH9x5 z*M=8=K@J*p4TuXt&()d6eGcw5(UGR1`!H}&yuttx9_pzndR2iV1{8q)4?h_^(H9bC zJf|3#@hR1|%T6v)q#xp8M5#IEbl#o6xweihb1DGJ5x6$&w1`OjC20O}-?r{D?c{3M z=zWjYLw+Yr9fcohzvdGC~6`;7J-);j=SVrqbB7{X#I^*V6Ji0cTjE6hFU=(vk z7>mRg@bCK)q3TQL?j9I0Gn+UjK@esOo14XWS5uuQH5kgy_uGflS3d^gI~Z%c3Lg)4 zqBv|8s-{SDp@TuJ+%h)*^*?= z6fR6?(-*fqzXA#KQQapZpp?VD?JVQ&%3zDuU-xRmTa?3jM=C@<` zd2ce7Ikjhv!)JryGNlo_d4>z^!@-hpRjX?0#?pMaQSF(yGCmyF+7{7!*Fmqz=GdoG zZAto0@dKy#%A2!S!-iUOn6yWJK$DpL>wOHf51#JB`fa1%DJuaLyG#dHSqYqX) z0p?DC&2esYUbhujoHV1>F)YjMz*Jq4UKBKk=vUKCzG)c@3Uf{zw}9PO@&)NXywKiF zguh*Z;cxXxllsEb{4GM`+)2ClQe$v!UWlZxsxbWEWGm?(*b9@~_5<%mX-bOR_o-tKg=k72(_vU@8BNU8n~|J_?8jv3xZ zniulAO*q9KN=&<>bf#?DM1fmvg(3#vP%6=Bu7QsTP%nn_6)1nF`(_}Q9~CIHgCoN8VSDw)St z?%NB63(mli+&qZiy9t$MC#|oz3$g>RY6tgB!;U9cDU48BnG?1G=@0z7teR|k{zO$R z*(Li(@mGu4*f3XA9z{VDiDQ#O9x+4Ue^BI8q%(bkUz5}miDQ~{>0@%=l}lr z>`$=gzudfl>H%pSReAHX@G;PJxw=l5sp;5F>&R(Nr2nOz8sZ2f%$2D_8IVo}!ZR4( z@h>a-*j2LH$632>XOJ|j!e8wC-$I;}(twl3g}o;Ok+@sxmR=>k)WfFdJHw5oK0Nel z5k@^QL|0mC*LN4RI@=0dQ`@8RH>A%b=|mBxmbf$eBQXDh7zJ_}Wd0Y;)};KpX+2$AGkp=vPl)zTMYK+|@)f z;zjx{&WYEQ8q)N$m9K}IV9LoG0bvR%pW3Z+isC$_H5I=dqxs>cPxe-EMaRGx5wZO$ z(Dv(c!@Xl{T5HLI>Yb$=9>o=(7|hRd8Po^RI{9)uNMUq=9zU*&1?6=iJOS}xKZi2Rc zW?yR*i*2V^NsM|vK-dkZjcZfAyw~==^pvo+81$>v6XfBTSDcIL)5yM!TO;A1@MQ*g z+%!d91D>|D1=1E8!f?gGOzG=&D<;u?pBk1*U*o?w7{pI6e}rQr^&v9+7?Kyn4Z*8P zt3dw#C1F6Z1G*nMxMyRf`;~9PG(3vbSHRg_-0|h7cpx1k=$^>CM}c8TB7klJ`m&WU z{!9k(cea*12l=yM82rnI+dSHYGa^Qcn>uSyvoncf1mSE%Q^+UC}>QPc3iK(BdXU3Ek-{Z1M`6IESnED$_J@QoDP`se^rt7FUZS}x&OmTq9 zJ<{)3Q17BBE(<08JVrXx7dzD*A_!v`@hS1bU+PJe;hKHZwA3G^JgCc6{u%iLS9zv- zmEZ3%srZ7tl2m-&O}ibImwV#iqK5L(<#0j$JfXB4c|q;am6epMakTgGHy~fco~?fa z)Td-E!eSLaD(?tO?Gi}G*W%yr2_)ao$h)!CBb;&m?DP2Eb{YKM>B&i(!rZ0bsaO1* zIB^w{hthsEu~9fBR;CR?*VF=_ek$!X+gEWv;Y10#?zJQhWx#d&rK=vKHhah(>T{UF zqKmfp#rC8it30D5wCm&(hNYpAE*}w&kOqpLnJBh7}?^Q@|uHkCg>w_g*u4oR@e{2y`kc1Rs6#Q0%4g6**4)Dz1O zo}L6VzqJP9ZBG4A?cIYnnaAE6u-&UW&NE6ytHs-hTh}th=O>QSnSs=&P`xYtdDE1) zr#U|e)WcO?3F@EM<)x==xbh>^Q%2$tAm4+^BOAx8)PSzJ^0L~Rr`A#~{G#!9Q9O>M zjoBU10;b9KY96&plfDQSsCsskAF9`bR3lUi)39S;O-cEWPNViC;g`~LlI8%P=GWt0 zAMX}pe+}c_&o4mTm>#0|ehHF3pjYb`pAzen%*{xsLnK4adYRNMGZ=Qq9 zd(%N2a+|~dcpd~#*@1uR6_24MdNO&|6^sjhgu8krKx#=fUikYbey)rXPYdmN?{}^E zWQPVk#laap(oW&ofGu#VvIb1rb3lA|pMtf#GWZFb-qJF4C_cYkjgJUP!mJS&VbGQF ze6FJnTKs#8Z)Vb-z#D4gqiVi_K9f%@^M+Rzzaer?dzMq~3V-x+G#%FX!8vaexuy9y zyn1zrY`FUh?98;6Enf`=%;@zlwi>A-uP|REZoxk87uDmoHebei_Hw<`L$Pl@qBR%_FK48E-UTE zi`_%Dr@9ql`0Q>xcx|q>Z)HunR9wbqS!?moj-Qxl7%K;+d=_CtXUSyUi&*E_Tx^JE z(PYDFAIp|EXiuvzcr$yYG%+;ema(J3riMEokyRIZAJ5LGr9M9I{JU}dKx+qz%w^+< zJe+r1pTB>@`P>EZGU0tAPV)>)tcQxgu{(XXUL7h!cD@!z{@sU#JASjCEeGOi_Yc_p z-($wF4FEO1Q`bTId9T;%&6t7tWrv_R?-isSU5N2?x;(`MUtgZAHT@VV+s{6yowcS2 zwhq66wF7oSPtjN2%r=nf{#O}QLtp#m>_+AmIDG6Ad%i)Bd)-KrE`<$2XT6m+a9tA? z@Wzj4Uh9MEzC#0-Vi5X^xwK~eY@vg=_xHJ)7wy}BchRq61KN#S3;Pmm!KC*Zs5jqU z-hCb_?TuCd-Jhr!-9q;1m59a*Xq`@(CoZp%fweN+WT%^3H3lO}ao;>=S?`0Bwz3B8 z!AW^3ZaCwh&adgct>sDdLxXGO&?#XX_Od$)rD+}I+z=~x=4~wdwT%Y-T~F}(E(C!>=bm*m^ac7 zcC9y)4@dOme={0r>c1QfWp*BNXHg>B2S#ZrUvT)mhPc7ojF;DF%)1Q>m3v~w$r7$(k?Hxt+652N3LI&y5?_h2z|BL5KITEzp( zY2C@zF86V8@DpfU;0@zv_2aoA8~FIRp1iLUoi{h*Et|Jtm88$6{J#VrKP0hQW192d zSj0v;uMkuN{8y`!@Vq#hEqACdA9U@9Lq~D`VUCvkTs@UDklFP*jcXT9`Cu2-q&%1T zbXvp%9U8(w_v!M!)kADKd>H@G{Yblt(fe>krUMR(I12x~oS@lweR;FjTKr}?19sRQ z!XY;Ocu=eJIH%lQHpK2g_aJsBgF|L=M*5tQhrN$k2L?FeYY15S^Q<^H!O#Ugst+_brdv* zg5m=Eog;m=`tytNvh^w^8G4ryRMY#Lb2>4Y1{Q;#BK zGc@(K!g;Asc{=oZM|u|aA)Joz zW-jSoLBT#5!= zZzJI%@2ecMG7_}Ox`&H!exq+Fv`x0w=L0^l{I#2x7l{Zr-jL&duOS#Kk%57E^CqFMX z!uO?}#r_3xviIEPa=T>m()7=`V^2#}H<;^fDrvrsSjP*B-4!vVjh#H5-j+Wp&k&)3 zx-zo6kI(1q72urrfvxM&5~e!EbIL&+cl9oWj|gOxc z=br@e6sMe4aTkhn6(=cdp!{V2=4~Tf{U1kH8CKO2MNw1`5ye0R1G@`B<9$AndtzqqRrj2kk`(r4J6B99!oW)+ zm#18^&}7yn9W};@ntaTrTVM5&#cs`{Yk1Eh&La%D*IP>w+k^Z#rU>?c4Hr9a=D--_ zQwKiaoz6?Q)$%_dMQNK`^e{(td1`V34boD_9hpysubidOtKJ-U#8E@XP`5sn#r^VMOWya!}tGoft-c(hZ+5OU}O|MA_GZX3o1~>RBK9 zJ(;RL`NKgiYgr=RxdR{VO|3KLN;ug^3-F7|HqAi-FU0B0B~&N25bh_08|7Nd+x=J2 zP`f2Y*XThD>++hut9eihs~UQEsl{)xgN}H&LbP}`Sb>v8bn_$N={|hbJteKRt(1dW zSonOGDFj}niC*hfNVBtiYu_*NH@Xt79N(7WJaWiGqo&IV#j0rBujS#!I`rEujXoh` z_ZyH?Ufq<;58SL`!rO9u|79~apCA2q2zgIgLBJ-ep7unnUbxRP>gi|&ETP;{$JB>K zYv`l5i$*UA@R;f!nFV4Cx<8j!#|U^W>Te{5R)bC&56zHAMy7}IyftGqIGwIes)$-orlMQ02PnAhcZP*e zwoE3Bg~Yh>nZ-Hf?keA~mvbxX|Kh6J-|Vy~tlHBjyG3-eWi{!2csZ*@iTXm}Ax85W z%?aGL*AHKN>5o$i(NCN96w(ZP>A4J&-mRCg(E>)hK@&5uie|?yV)I#GL275$%T)Pe zcD-+25H@Cbs$w>mC!e@%8u*Ow`MMKy8JU`0Jk?1;yK=v@M}&T$W>uz;<@`^EhNMti zCDwhIofdpxw<=laC!Uciqx)5RmZdXLHw@fTHa8Yi!GHZy!NWZ4!yEzaO}HL8a^FAI zV`_eLeFAu;45!RbeMPY`@eF;!+a~Td$F@$0-mbb&UMHZxWX%QV_F3IMWk!pAs(<$` z1l+W|{@0C$#TL-ubD0<$&HoOqV7Jyq7@CYwr*h`Hjy%~TTz4q+U6>v>TKI6}H;!9u z$4SA*)sxIPt~4Nm2W~h@M)!5L>0)q$17l;e^zfORp&ZN@o7d^<`TNqYeQ+DEzUn1Uyxwc@O@LpKue!;G zeX7ve0W}l4cGuuNrkm-80?j`a4%ywpV?f)T1?fw|3DDJbklt z*v6;th^pVSlJ%Cr{Hx6Vx$$0qeSzaD%zGNfjytBlp;Ix-vgQd`DS^uxW5MBz%F^Dk z4;f=ajSiL83)lM-JcwH4S%*4}yKTk)gL9&Ij7vW`(Z4vE`uetJgwf`{&|vKSZZ(}w zayI&hd;&s6F6$k0Obv}gCp`C1&zp9#@1JozuE&Kek9OUADSe)nO;-IA%#Zq%)8_Yo z&JQqpSoHd6qZ0e)!ROZt@RjN_`a7*$aEgET+OIad^r6}P9m#k*qpe&w?GcyN;l_ly_e@08bRht>}P6Hv@Ft18P7Lu{}+6ha6ZeI zW%i%LIlBJH!p@!c4xDKhBW!%;qZb^c&5c4b{CXliJg|d3{9YTpvYNV_-G9EQm;D5R z2PF86;OiOwlJ-7(!ae#G(cE{Hf)3$UHLg;-fb4SZlpuod2&kRhnJ z#j^?ADvI{S9yczJ1vEFuT&p7Ct7&ufTMS+yT({BAWa&DAc4RJ9;3=Meu^(4zvp@mE zsC%w8Z1fB?1_Rq9=8%DXeBsY1`WwB44R5SDxLM_Nc`n{>+QtDnD|3OMcUhc_T2yUn z@8Ez|71{M*vhhkP_;sopxWSLW2Uhq-c024QQ?DFnaFPtlQCmUNptf5G)Tj)e=uPEv zcvH^5Z^_%cI2k?LaPtDK5g>1Iw(;v4KH8?2Ho5J{=I{ot@;}|JQR{aD%iB<5T z_<2JHzEhtrGmL*Rm_@w?(P~=@@(Ge}dmg5(Zfp%KnO}Of+Jb)UXmT>yvrQh$jr5tm-~IZL z$>xBG7L&bXOSk8RHBSM9XEU%Lm@{6RoUozSM$RlbPqn_fpYa(1EsIU$`!ME?!3E@) zxr&jiafO%?S(rBQ&>Cxcw|gq^!(2A~kD4TSBnd6!!+nc0G$H&~Xcn%8r~nR|v=#a_ zg`is*JVO`P4iLt#;6Heq9Rj>gTeCOjS+5QYqo=Z^M-nnRE9SxQKYwbSPl~xOxcuAA zG%s=z!&C9?t{)WcMI4xBt8=?GlI`MV@`7<)3A)zkiLCtaUoJNS-_qU0Zv1uBRB5eU zg(rHIC37725A#Ljhe&+i@>@U?i83|M8Sbz`pHZZb9TiKT&G7fg=$sfePeyMI?p>Rn zPMyalV}rjW_%Y*2ti~t817a`mzY4y^7uRb=|10<{;NKpB`=Z3HxlK-J?L7ag$x|&` ztk@H3%m|9kI6x70h4l$fFQW$;UWTBEtmlpAztZ(BZZgWI6CssL5=gjn5c=N6;Jlca&J7db}?v*B&m6O!_?Z-z*W>-$|mD30{=8W1T+uHi`TP zetIDpZZjUgYxAQtexsExk3Y{wHzGqaSuEc^|CyiKMyJ6S$L+30dF+v+{6tumZ-aCE4_MrW!AY3szpJmcnA`Q7V3Jz{s1YSk@PnxD5F z^M=}7{G}?D_SCXr7p>aa>4evL>Fk3o^wvF(wmI2MJ$(C!%J(fVbAS3n4T{fTN1xte z-H;75DX<1V>{v<+@K~Yttw@uP#tI(N|2O9t|CDNcEy`shr_;x&kuqP6b-Zat2v;w+ zmTu4T<;3ONR7{1kGAUmK)i@DDY4?f|u9IerPNf=Ax0Rn~5Esq} z3;wtzy!-fR?-@PliRTGExI2eTY@LT~9k;6c{vWC2TC5MW=`v-e{pDuAunrflwF%8t zyCUx3+I;!Qwxb;F>&y|)oVm-jE^=zM4jftUz9=zhydF>OWtl5$`S@WEo$S+E2Bl=9 z7qLfqbJ||kWv6o?&lq+{nWwju9!>3Mx6p^))YViL>sN>=8smz0aj|Yvs5>8cm`4vh z;wU{MR^z@3$keV)7e9>er8c5gMHr8h+4Kw4Ac4O{0%h*~Q*?DX(J?hO7y0^5j@4D=R$iHQI-LRuK z;ht!8$GEb5$UmdPp@9m_3wV$o13vSh|pu9ij2QNzW*F`q| zRF?{RP~m`WqW_9?vGwRfGuEO^o724N;av;HO^#YyiuUIiua+L^pzBRusGX8p(dXIT z>c+=CJTYe!j&qj3-pQ#`F4^Gw{!aROs*SZV)^rb!DWemo#)#mi*=1pma*{``RY{)) z)8`>8IHaZ@kAI&+1$!6equJwVN?ZXQ{&OmgEOwgxt2xSrHTtm4)bczfE{3%;o`Wv^ zL0#(EP^NAiz<0M~m$(*LzDT$(Q1KD<{Q8iKId3B0=0#~)Y#X)b2liELguSLGp5fbe z&T`z)uIizKjeOQ6zs@xyj{cmVZR#~``Pm$NGab*dV2#mc!S(dlJK@^av5Kzd{*iue zIzY2p_mJzaV9lSnTlB7OIf)))&&vntL`+e6=93HFHOo=l(348 z+A}*~(oNiN0h#-L0qz^^B6Hj+Bg4j3kO%*5;F^!CQq+P``cvy{`beSd634Lg{Mv^) zr`6{YgT4@rx=+iRB~!q^*8aBH)WS@fS2-&;B;0+fI zN#ymj57L}`UDR`@H!2|aE*1#`dEYQk;PGtem~3)-xuw+X-XsQ=l3)BX)7ONtwPOAy z>Xpu4ASv26m2NNZ@Q`!9a$d@5PTlyFfytC_%29!OP(Oo;YhbEG4^sU>rFFH}FGZ`* zSGmu@RoLIdn@}gR%xoJ`;-aVQdwaj-PT8#ld2jBS)2bY>9Coacm%e52O*b(vFK#3_}46%?}9UEWGLL~XaO z^R+#cn?#*$9BmV%J8X|py=-E5L!}h8{A`%%8I7L7ICsU^)>oI8f3fb*@r4@qen_(} z??oN2*Ofjc;k)l_^4R+da&Gg5RPL}B>Z>;o?|wip`EWZ8<3YWPSCxbAmFNC0!_<_c zMP<7uojBLZ9iqjH&77-9XM@MO;Ey2fy&dZlZ%rof3D!e73XW<<|898cLxXxVa9z*H z{fjGosw{U!be97q);qzz=xc7Sp&J_u%3T)h;fHxTwmt$J;fXpbA|_pn&}IyAOw_?H z9$wvtp9fdggMT*w|D^KMVZW@yCuefSRbBPI+P^6vYK{t=x{~|ksc+_%hEMKHp`8}c zwgLZ1*XFM|_^uzJ2i1ZvSc7a+J%a;0B*fL=BZKSt^2y%{bBB4@%_qI)@uTPt2BQTy zUhhY4g#NaGLnxwlZ7xu;FN3Er?zb46guPVj=&(c01vrTf27N1HrLZO*q`P}@sa`(R zB;za_4hEmmq-(i=4JqK%0@|xdk~;XWoWQ@+60(sPck>zjc3@Td{(Kib{}p01f$~Q- z+_CRS>#CM3scPsB`c-v_=>==~rt=MF)Aj|cxX@`li-T*6&y_~8rwWTR^;50GomYuY zUeEb<9aoL9lP0KN%_L!i|Rb+T%non5T*-5WeU?FU-tU?K# z%AfbK$T(C_S3X*Xo(`KL9a8d8$g&U>cIYIb{^;l2^<1ZIC-NwdHQ(P?pbMj`Yt%U> zo~%tPC*iyQZ|9*a7O4{d(tPdfl~QZUH-g3*Lj|JOFmS~(VTaV;u=h#%rg$$6y^nD= zpOaH}Ri&5S`xv-_-ifDtHOi1%nL={AeN*VVZl-T&M~@|3v*vjUJgL>WmQT3U!g{7g zb^n0UWN-_g;T)BYsVfld$IMHqc%bXI05;^;lTI_2>zPWtph zp+1DI@YEM{01uRV6>yzacFU~;){fIx?{wz8H`)<;gFBwtrp%aC3``RkKLH(VxCh$g zD7h=VdyIGYzW2Sw@BWP-bIf@!ZN%{|cy8%_0pUKb@8cn2(EXq0)ATOy_}f_ z-6#^5S5+Pk1ECALDw}U@DQ<4}NHk<3UvQOg%@kY-oaEvPET!E_VzSQ_s zN!jRodD(KFo33;%n)Z1(GwvDl`vrRQi)dHENr9K3S!(EULvIprlEZ)gCX6A055%=T zJE>9h3v>T^{Q77C{YGcwA{jNL{v^y~T#u;J_Bt1`ohM2^amt$4epsgzdPt3@D++&F z4;&v!&3ArOIo1!CrKbhbzS>i1U5Oml^PyO)wc}RxXsqIyCn9Mya+vg5#jMcDwCGZM zZL~(q>x!VG$RTc|Xt%tM(eG4w;w-haa;#|k&5xocW6kx(Pub|-lfN>otKAE8ixD%a z`PnFQOx{`W6B*1b`(B7C`4ty`SW+(F2?Xy&ODh+YQ~#a=zpg~KR*h%Rv6T%gRF{dp zg1O93RYEjcW*1^Jb+p zdYjvBEkm(>_4xbBMGT!Qf!*qv{H%b90)9@cm>11^el#Q8H}r*_x|5lkptaGH7gXBs zeMXy$7x$jfswI+arfyUpw&s)2Uc&veql6}9;Epa4=B&TGe{VEBK{NQ)O+U;=8-}ji z&R-n!lDYrqm2-3BQwd}|jL}_l?_c7hZT6`->H8J9l)$}Gt*$7cljO<Z4e3<3wT zdxPq

efR#xlQ$fB`%mZ5v;8CN^A5qPdp}s(?zp%f-vXki#bHj* zMsoDV4%}$05te?HS*v>PhfSZ4ahF3`SQs1tPkPcl;;!3e(c{P1Aj@2Rt{ib-51muh zLDJvBX{^8a-pNvIwC@hj-^cMzPt$Psb3J}`lc8)HU?2^T4pBeDdl@9+uSPDo=4}DK zuN(tE^CIZ(YaMA8eHF6$naYej3pJO{dwzF>yL1=dg>!4=+l7<(#~nwpq0>|u9%;q8 zeNF{y-FWTdS`P9=(KMD@;{-0BWI%U8ZfDL(OXdAjFJR3!XYSY|8#bIBq<#i0qfGd0 ze+_@w$CdY_EwaI`4fvo_bNEZW8)}>~p{t(kv?&tqRSbdv+n21S;|l)1@*%GLPz~J2 z-G$E+x5>=cjpQTCSf<9?qnWXcw26chMX%w4p9%EY&`W*3+-tZ6`_<`+&Hp5WdWLDN zE)1KOheIu!L5PeLQ%*LjTKl1Ag5b*VU0AYdCck{PuRQmtTy*-NFP-$S;Ny9n@zcl4 zIL6l+Z5}xte6Zj<-mc>g-T^bD z?xhv-YrSnUC2|E+-#!7KU5sJ&Mmli+6usZwtPXEKtTP(FSqBHxhsviD4&qp+r!=Mn z5izJ0tnVKv)L2s@f8qE&Nl-l`2s84cv-`XQ>E^Lf``c}zEM%vk%h@|{?e1OxXVFoQO7$JM&vxbGfLo!^G11Y66&TaWPUDY?w&gdUIi`xWPB^pIWo zA4coKr`~OXI`t9uZFFHtUxH+(sCm+>{|0$ghQhkVbUw!G-F)He{WvcC7ETx~U|>Zt z8`apGrIeZSX)axO=bi`gf?pVPPwh%`nIMuXpEBPT9VN{Lmn_(&y*_vf9CHEwI=nZY zZba`FLmRqFGD$d;HNd34%`ngQ2{isv8}D{FtKu*2?RJ%_Zgy|_$W*?WzPu`YukL`X z^5y8&U!NaLzm48kjuMY|<9>cCv=ndbx<$e&-vD?K=ZD-chW)qB9Ex9h;IBo6aC^o* zq1Nb|&0M@=6N(3%#vsK@_|;9(^e(C?os+(kzRhDbgG|}RD+htrQoCK}n7C{$9-N#yWY9UqbEO0b&r#S-Ug&e+V5r5XY z79Mt82_q+@fbQ-7oMI*^ulT-_LY#TYgnNDYsA4N->-PlfBWvMKP&K&Y_>L_)bP8hr zF6I{gG!8qNo>YbLd0xtXHi5qI0!T#>&F!MoApfS3?8Qo5f|JqaD zFlZ)8RB$*39LHZ~#^82YB1CZk6j{?dg*+(rgYsSX00# z|M98WWbAJA9MkI6mr)(g)RS{;IWTwuA0j`g{lnndr&VuN?F3z4)V=-KmxxS$~_I^5%cE zH~6s*5SIYesNC`_hOM}D51NI%WP~>?@VgyXvCubga_?M^$U%b*&-SOc(TL@4*^H;VdD%5L;*T0k_SASf9tGXz+M8 z@^f1t??R4n-Wv^k|Yl(~b*Rf*|;`I5gA2q~hk;;yn-$uekIqG6lPCo-DR`>|3xcB&YbuCU94n970 z<>^KpScgfqz-!la;^_)VtG5GO`ZeGei`~duwBUrZf@*D-jqO{o zFk6WAFFNqRfu2CQ0fafKmI!-lW6Nb`*xcubW@uVl__j=sm0!{maToUz*7QY(ve>E` zAUz^?U8eU$O!UBP({fbUXZI=_3%C0MaXt{fiAkQfK(tHJ@^i-0xq2uhlzz}gI(|aJ z2E15mz_+l@_-|)@e!EV#Cdc>_9tqqSUx0J4{S8LC1+C+7yFK6pt#6u2y zVN~lKIOoQ3$oh9xX(>4OXaP!#F5=0ML4;RfY{&?@=a;V%G#8rBXnwi2i7X7R6xV%% zvE7tDI6rR}Zp)0L+KbDc8a19lPIAvFpPlnvU-w*PDeoy5k4y+gr zgsu3}ZyxP|j;q3fpJVn4%55Or(=_l~1Iv1qu*%hQ*vpycQFAk{3g0rl1RSxd$%DEi z!;*q*F|&=XATGwn2m3<3I~q_}Xif(**yOrNszEt+X;XOL*A|J#;J|hIp3Urw=v}lK zBYqgO8{^%1!|OfdrNa%O-s(np?UFy<^^U?*o%X<<-=~!pBksxv>piWtcAg{AVvZr7 z=+IPb^{K^|P(SD4-%r%XG+;9(?v^p-J4uf;Bs|%H^lT(-lsCp_BVjTe&G%tCMx~v1TGD#ZS)S`~$-UbA^x7n*{@rHLSQxf?2hQq| z4l6$dvdZOaV#b)J@`y3*x3SL$&(ga1uso9T;DirO)+#4MECb%=BWoV9|Z{`jIj5iyb=eZNQX1t4Ov{ya7rdZ0g+y ze0~i;f4ZaWRD=+f3+kw`L!?bTPIV4VWB}a_wF*O0J*2`piV^p{T3=}rP51hlaA{W$ zNtnvjJSfM2G`J+qp)`-Un(8jeo4`PwR3vTBIv6h1tUc{2e|vQVmDi5R$v}DzPp@^X znq#lomYRg{0h&v@{=*)Z7SlQ9v5evi9VTt%G(TvbP#YUP+6CIxU&PN0U}rTKaphBI zEWflKJAK#yOE+JEvr$v|nOlR=ecW(PHBUOU13!P{zR=#=grtAj{7@H(#2ER>yMKmhHniX{qfEJqi^?lSly?j^D*XXp$36s8w{Rdo2c}z30O>dpTQi>#Pczc^?nPUklCzK#cw|p)C&Oe;Y@k8$(65^R@0DH0@5~66lq49tph4uL)eW9-|`pg$k!c@(fyO*(DRuM zkVk~BG20;g@g(f++!Bf3*w(=_aq^c<#3!Bk3*Sh>UOhfFzaLVJB=t|81Ux*r~&Sf3~A z?j+w*os+NEw0bj{&k$N{>qhnPHwFWbjK%w-x@#A@y;nFcg1R^Z^>;YMj!}II(%N`y z?`IKG-xa9-WY(b*)RQ|ThWz~n7rQu+Rw*Ftap7fF8Sgp2MvEjV_fTPST^}O`)u$>L-iqcKVJ{zoirVr+VTENI!f}>eCO6ghz!~Q z)F*&XLpi7Ms(EAdsW+n7V=VhLRFW1M2|GO|0nJ}i?~ax^1hcE~>_)SxcjJZ9S;W6`*gU!ea>IF4I%8Z{BdJ|rAr3j% zsJsL6*f@`P4)Ei%AFbknjt4-AM)8g$u-s?q|?-fgS1co1nRA%NzLp6Xc_W z@xv3?qK$#V2{wFF1r~ce7UW&D#NniCoT&~i$q!E^uC^Ac&)_q7E9_a`m?Pi`BV+9DH^!b zdmCfdBlQV!1f3;7emW+r(@nYR>-hXCBrG!4CUjj7Zu`ldrAIDGkfeS<#Tamls_u`Q@vpNJCH{rZL}9l_r!tn z_oOWt@wRs2wRNN??rLfMQPb)SsB!-3SCcf(T*$xD80)6B2kKKZsyDt;)0a;14)!P|?Kl!PF08?n-qP64 zWDi<6OV!h)-V}M{zM^$TfpF;;3&n1M5TCUfNjs9q?5TVvRDaeKbH`oPsNUGO^My$M zfF1T6K>FW}Qw>R_&1>m&;A`IY5u`DANd1#4UqE^9w}G8-QfRbh(Z8G8M#;nZjMEda z+=I@(Jm{`H`(q_;w%LNMbdTb+{$t?W;nQ?phZROmvf}{{&tUa$2iViuPBPhd17sUC z;lGUPz{AY&qWCSBMljb8r+tB3)-=i1jm|K?Up z)?(ky2?$dIS>{h``LL@E`lh#q>(SNu;XRj_-JG%VM^Y`#z`@RZ@br^-yIwd>duk$` zum56mHqMfZx*h`>JGQZUDA=6FS-cP z^AoaS=v;A!QS;ILLND68nUCiUHN0c>u5$06lQL^mV?5;DlTWtWB)9H+3;_>4WRr>J zFm_@uz{?JD%C7rjLi6j4;wXEZ+5xlD8puC0&7`~kB|&S(=yx(He)ziEY1A9uf?p}? zjYqStYjh`lqS!~v?8heHNxQuGBmBf6D==S<9&kS8GQ!(cfTP|f3X8cJQk?dY+qe>dM1bO z6(JBl;iVXC;U;@Gje|m_i!uFvXnzh36>ZL^qRQPyoeeQ>e|u^8Uu*4`m}_vf+gfqa z#*6n_ycZUAZAH)B3llPRrBnPNXdW|@<}jM8n5tatx9gem%p8AY zMg8Dl=>-To{!GP2e0A5C*V+fd+$Uw&)%Y>qnH2$*H?7#JYg*Be-Y4^1xe;o=$wckC zayW9dFZA!81k-g(*uzhI<-pl1IMpv2-2X}a&=;5xm;(*s2FqpR+>q)J?OW|dniG5W z^anKZy#rg8ZiV9`gE4J!xUgEH10IatX9@IT8*&RlJ!{S^TR!ro1V2L`%A4A<&WN*& z#;PGa;wrzwUPU2ck+APOTd48BSeL>+x%A?FT@Gjnm*nuf*_hR^OsG1ma1BDLYi$43 zgHLN8hG{!W@a8~639So(utTaj6u)UC$LIHlCU+*mw^2jk-6^R(`}jCS8O5Oo8_eVU zPD{!acI|1PY#u}X_+j6%@#siycWE5&=`a-^444KNCJOY79wNsKxqzQsF1z0vTT9Y& zRBfQDEkD`^@~mM?PV+#$`4yOaP)~HK*`4qqP`3A>bH0D>;Y$+RNUvUZF|b=V2;a2> zZ;joGgoi+}!5q&t@Mxkh>|5E;`95F8)_r5S$U8{;TZ>grV8>y3Y|!!Jc>Uvg`Sq2) zEPk^ax8I$^KP>LXRc=k*Iuv3LFXy#)WWnEp9q^^RP^~9xOy}4a4jqVl2DnPfDY}bQ zg73-zmfFY!S|&N-K(pnb>Sq3j`TXvR?O-$h0kmGfTDD3#DtzYb2Cs)knqSu+17VWX ziUhRTUmFb0w!+=JQxIAYko{vExEj;Zv4!w?b8oPIPkY@$S5Yn80O#u~VEw5+eCEs- znArXt&=_Upn7s-^(W&BKwrojvO!W$Y#;Ya@|MM?Z-blg*=^1^C&01Cp)dx10^t+nn zQ#(LHk*BEk9%Y-%Z!LwK_%gxokpOyoJP56D6 zAyatQ?8P+5JY*v%hG1*{2(I?*%jat5;?iR-Nb8DQ-i@X`9($_llQaRQHVVVZ3k>A_ zuTvyp5DR&9m(?AY%Ob>7EG%6IRBx=|k0D~SjWO;ykj4VLoyULsXQ2ItIMlcQ4qC_g ztkFxus@#ii_X!Ep+0q67;AKdfHgA7AyD@198ixK+7(sKWjzJ@PLwe<8thv?|?R4rg z!g0JDKzlSs>+vEzV=?b)d=)qCw-W5sW()cF!l$Y_o8#?W+5>2k$+ zgrU&paRAf`KTSMeLq?x(D$C~NvDWMVGPe(i*Nuv*@ImG24LMZmS$=^v+*^Bhg*hi& zK<|iJ?9m{4?{{-PI~J$QNE6W*^I`77{mjC#H%wj-4A0mvd~co$iZ3cks?)nbt8o0h zesGNTrro<`frR_eb6Nu#a?S`()ZQ#idJEBH;&zyQa|Y#lB_kdb_rJarq&=W@oi<|1 z@{sHnhvvW#>kjaH{Zp}P`*C<4z0y57xkeRd&}Yh#ksh*PyBboh-6Q{xK)57E|2hY^ z?284}yq3l$SGPR_1NJ%b=FJ<)80RFUHKTKmlfd;!XC67mM)bHknY4_nss;5dexQXX z8%}4Z_WeE&h@Zrui@!VHy&45N(feo~(LmS&YvV46C$$cV-f!N+B&!;7!KOn%>yEU> zc>Am= zTAh?K(W1l-jW@Mt0UdML!Lal0gpXKl=zC${wo+?1^5AMuKv{qb8vSnob9$q!0SHs_W0F`8pP zf|$ycb2?c7IXa@OQCIj~Cz5@wc1yeSW(%PIMlrb#x7Ivl{kuCdqb~p0?5sved`_|6 zBv#)>^!#`m?zok(FZXCqdP*6a(xN8V_A}v)N7s|pe$NzVeOF7WLG12ph8uN)*zffd zfcO)7b$FY7xz%|za%d+z9_a#)5*zcG-^`)^xFy)U`EI4>h022iSET#KA?vg)hdQx@ z)?K8++?X%l(Ds)J*ZsI39d|W{;KIQ$y);644==|4`it@Lh-UIs?`KGw83!IZhJ=|i zHuM;M555&B2Z8WVQ#Ajc!Zmu|X_X*-!wGAp--#YvW7kL9@W(CDZAyuFfqO z`Nllim)>zi@?GHB&TZw#I~Rd;sVH#21o~+XTB>oVXLi2R z-}x5Ub6TuB<+!}GG#3cBncH>-an&=}601s442{Ge%k5QsvG<4@C(VR;{W@Sp-y1^V z64f^n$IMLJ&R#y5>VU+>f-qdF*c1PVMLUkt9mJP0c~+57-YVo=2!3g$ z!%o+F1Apg7(=+OD@*23KATj%X8%t(gW2Dm85MCAy#BJ!6a8D!#=VklkHe^(LReDO* zHZE}#Z~kdW*V291Z9{Nn6D<&5f?9+05mN5b6Wp#WZ*T*8S%yr7`Z)h`q6mmtxd0 z&*pQS-!Z~o#ce+AJqZt{unVTB>Vy%G zfWqgt%iQ?%=(>Eym?whx16IuLgd2VZLf5zmB>ok_g+(ymA%QhEG-IzK+;EcX5FYMj zA@5!_W8}qwIFEJC?2aKpcaZWNre;!XnT4PQ%gFlDX2I60!@n{c* zb(0JKET%lDtu!}gr^jQP$CVg4v=I-B+RoM;yhxfmPGJEMu1MvZZFstiu3PHwU{LQEMj6?HeYw9NJ*PA$j*9hyC$(BAJZA>=TH%hD zZ&+in=B+adHq-wGq*CXH_nr zxqA#tP9_3rY4|N8wS>o0<_1sbEFSys`IsWbZ0*McyOGr&g+D3cI#T*ufJL29^W=};TkyuWji{a8oG@_&C+r2^38_$X?MX&H9~ADBH-pE=n)2CWcQINQ2sfCF zLoavW86H;PxGOQc=B+f+4zEFd){p5Ux#B+4MC?Qw1UESiPi(yEneva+ zZ)=J>V;2KyLJTe0rs@l=-)|OwJ5Ls~vv%Xs$`rAA+bC?Yb$|BheYmoK=mmU&Qe{0!K`G^VYWzmD0@2X=~q6^GUQLEOOuotX7HzRp8B=3NfZ=$&ROr|{A&7&bm9vw>T%C*!xA^r%X-o#a`VeG_- zgPHq<%c_Sarv`4}fuFlc;xzWH%s_k6sh%WkL*h9}8U%40bx((6Zw- z(AX7d>z?#dbxQs>A4uCFX?2*f$4q&Ap&Y@S0vs;#WCjBd<5xmx;t|)QvJD8-Vm;2pz6cT zsb?d`9PTD(-+ZGndcB&B#onlT8ub4mX+r2+_af*QEG7>%7^$u(PhU{%Pa^S&c)s$u zyTS(LtEit+rMWY6-mxw3is7gAL*m9fRMkGEodY?^m86C1 z0rhPNtLjsa)C(Wg{UE64t8|#s``YgnM*q*Hfpp)&m?xCKJ5?O9dxMEceMu~h>%>SG z09)Rdbvye6cD;7yXFG2kh zrPs5_E0M0Yl;kmiVu`Kg2}XGfwC2)iMXEcP>7eS{D4k21U+j$iN@GrDgh_Hbt0M@j z#KeFbm=o#3D@uY;X+PDcRX#HGX9*GxC||F%P8CK{pPzc#I^s#|CFpkj2 zI^7=yV*f?zj=YuEH`Jxx}!A0Micgd(LR>!6ZpWt$O?{XvlE+2zq)=ZUAr_H#n+XfcX%@x+Q zn84GLnz8HvFI=L(oCQAdlCOUL#B(DJF}!zuKB1s7FKKa99<_LYBfcDjHD~N(n8QZl zH$g}Cy~#y|U8r0g+m0KJ9>R9d42Pyqmg2}xr5XeC3;29n78s4b1@lMLkn~*k{N!7l zzHglHO1n+*i-!O5vhc}_)i@fW#9u3|im$X=)=^eA3KrFZ?&H@<1ElXntsFG|A;oJQ z_nfm1TxuB02iX$s9hb1ksOmDj_YQ#R)8#sYP+8!z9fu~@=Pvqtn|HK_r}@=WdbB&*{L*=&!hYs|#U-o}3=Ig)K9VVhdIr*7gday%}|G0$vz^ zw+!~9exVts=i-0+E#=p)TeWYy4#QCmf}n4aGX(6~58FxK&wTX(Gu#%#!C)ccVV(@m z_@J5V7K4RiikxlIQaTRE!?UKzm^fuuwt+=^KJ2)@OqpQLtxaoi-HnI9#`nI+>tH1E zh6$KxvPc@SZFqbP-R0VF1JKyShtxb~8Glr~-~1A19B}3oSL}1WgyLF=o$kG32RyE6 z+g3jVr^;@zLj|jaaeM{)I&nOt9vsG3?Ho+;7%uKbjhE5ZOJSYEKp8Qs6UH>F#qU8{H;%Lb4EodVK{4|0g zM|#P3J?p_8XA`*HGF7|I;X3%O?Zz+H>?A|9bWYsQ5YSXROz-D&qas3);zS^A1%dSws~@#)jBXq)q`vtTZ(qWqT$e{mbhW=HAds*PQIqn zcCe2cKl-gZB`&vmBliC|F7IupbDrru-RK9A_}}xlxc^6MIACim2ih3H)KMqInvxsh z&(|YpZ+Hqm-h2+G@2-Qzs7BoT?n%gTMOd)HLHdo4gcs>P@P5lJFke2Gw_I!o>7U~9 z?f_flGlnzsk;yo1^j?1P++p_l&ORLE)d2X+|KQtHYhGbj8!K6FKD#z!G*_|YRidD= z;^AW&`EAN^?ZPuN&?|B+FaMb$25-#9cs@=Vxu1g0#aW`i)mZ4?;3}>;vkxejShb0V zSw!dzw!&pCP+SDXn~!FRa@dCztVfe@$#nH;Jsj9^Zow;TY0tuiBbZaC5&wM0PGz*(}TM5s%byhim&FH?<>PPKidH+|O>OezzBL2G1 zg7Yh#WRty#Ea_UlPW&*3|4 zZ)1}^8F+nfAf5FR1;pQMTX;{xvN^b6fH6xR{Y#4nx61$GYhdiu!E8W7fF^r7Xo#~J z@d3X6(*wp`oQyO^=(r;o&(fZk7pF!_T2mJH^$G0!SB+B)sn)jRnRm|2aO4u<+V_|s zyoN5P-0(Wy0;*YT=hO&nTz5dVi4MZ#O}XfIdox7co+dvQ-K5|7#uBGA#6>whaZi|u z?Cx2iZI%5LGdu4GS~nz&hV+lFvS;@&G`Zx(=ufaZ5Gx+U7<0vQiVui;Fl$sUQapgP zLw0aVMDX?xeAsfWju?DQvaVOVYZ^X3x9v^>m(e)~GIcWx)?Y z^_e|=UkR;|0Y4bbVECsIoOl(77akHfpL+9SRu02n$7FCUUL3SXXX0TfS){{}%n%f_sE1)1)qKwKh6x%ZG(=e1DJ^@-Aj ze9OPyJhaCspqzqDr~Sa*@f&N=sw>#EJ1llKIe;q9S1vjyF1^ZRKep85Z>j~!!^uvJ zVuHl$F!Y!MX`NMqG#J2j&+zRFeKi z!U&wczdw?m!S1%+`0-?8OuE)vzUz?yzrPfaj(VbWJG(nDl(5%|=Nli=dgL85*k76L;I8$lYhQhNm>_4JPRv*4-gk7D0~F+A9Uz+E$Lt!2>zEsVE<2R(lw#v zIZDxD)NZ=tJP%Y&Y|3IHcW7U!v!RnTeK!!iwngG`Xpyy0{=yHb-f+O9KcBYR1LYf!7bK z%Tx3oVM*=gs<$de94N&@2 zLpY=D_SaM@-A!@enCuZWwtwr!soOY*m1K0>mUL+B3IU;)ivHU%0Zbp7ScF6?vU6bq(wKK;Dd zbK-`k+Pb$w<;J+fO_20EqdelIQLFk3Em0xvd5*7Cw3qhO(B4}^zvuY< z@qN8~>Z3L1JshT_Tps*(2S&+Vkfl*t8y5@7P zL&c7x#+r-sZbI?Xspw13rV|%4rT<8uXo(y1=k92Vr01C47jxB{qLoe=F55hwFmX>W4&dq2IgF8GSB1rLTTL5 zCC@N;{vV*VXUaQoom51c?2Qn+TB%$E$iK7T!TW&xF_8D9V`9d`s$i~amoI!-T?Q|h zhDuL|I(=rvPnJ^NGM0Q=y{a5#(_M2(7|uw;i02iHfjq0&cR3%3Q@N^ls#ElNSW|g4 zn36{K9frNG;?O2LHsVavTKu|xVun_Jf&$!Bxrft0@>oeJa|DR;@l9bacF?ZpV!047oXvu!zhJGBXW{%bN6 zwWE9d%hR+|cN+@QT1XfQo|_U0pSojvzi=dw!6`=1!NW;GP2t|q?{ z#wf>N5X$ge36Qsu{#-lK8i%7;%F;hkJI-vbWTKZ~`)>s0X}X;6IeH;4K& zM9{d|vz+5v!dU!dHJ30T9E0kN6U6J3b4=i*CBefVc>8HNg60FPO7uDTJ9fjRkyiOm z|I5LeX(iS$=4@wBx|w(a6PuXHyb*31|H~hNI2lMc(E1v2m2q`Cb_{O>^b^FHY+w|9 zPp~7H*2DwUJQ9cc81Ko_wtf>98H0Yz*mYF1bJ>%gMxApaPRq$eLNP=T0JTN$`ZKF_*HnKtu^fpy%86;Y%6xJ z8p%&BON6*?cY!d6v|&1s9;?dA)c=nRJ%wp@F?jF(Mc6jkT`1kv#~KB#3)Rd8pzH;d zCf^%fOX+J^-0KIDPDa1j?Lr%}L*!~yLG!RjOT8)L9P^Q`ODayJOpo%_ zCu&_d)wXm=Y%M8E(C&?nVubq|S_jqdBH@k$=6V+4+U{qCS}Wo!&7cExU)>A~I5BV} zJGt#IKCJEn(GT_``CRNfdIt5|03aDJyalFM&LHq>=Ez^-S38VQ2@^Gj;3}J)Pqd<76%^1}kEl+8DmM0c5ue-F5 z(TlxE*)MUdJ!QfRp=b1eaB)y|Zu;I9rr z^4Pq>63QfYVE-Tu{{G_6(l64p($|xq-JB@CrID|^R%1LPuctgUP_0v4(Vp;sih(@6 z(lDyeambZTf_Q-S?-T`u`&!CAFe&B)Y*_Dv%&{q_e3Gy#i4n$-uG}M3ohu#S(`YG@ zj$tZ~9bNoQLwbkK+G>Q9A^4~aT=~ykosz(A(B6E7#i_qyfc&#mbE$Ii_wlA&>66^L zsgwa4aOK5*(=#2!UwB!&fKhIYDl;FPu?nvg(0QIV`iwB1^u-+vxF$iIqI1s{e_&aA zPqX;gCH!3NdO+G*W#Y;sviB_vrPG{++>!Be6woFED+Z1Y3LySg)FnGUQVjr>7~v z-m9@3V_S=ty(`596F#x$^PO>-XKlXI>IOPDFpvhrzTow17BI)zTCVRuoeezfj?Jfy z5niLNfyM7B{OYgAtXp7xo;RUFv;LY9RIg~kC(NvYHMWJwtaVS|(wmJ?*<%%t3baF~ z;yAc9RtGK0BQV)8QnNi^zPz{|^ ztVUm5UG&Kv4|u?^iv<50Kz}Q@^jU|e ztKGrv0pGOrS!nN-BCajb5tg3S_;H_L9C&#$tzAE#r74j<6+hNM-DY2~t6LnzkRQ1l z+y@@7JpjF%-qUWjY$xxZ8LJs%8>O{$X~mBJ2!*I(EB?@>8$9?2@?v(LnqPkE`ekOk zq8wh-*O410n*z?$lP+D%<>b$HvL%kdiXR5Do!JIS>w=5Swn3vu#{8MBrTndTTZBeN z;{F!{!TaAn42E!7yMG`aNop%MANIkE{~5B9L!rcLvqjRsY)#U?wn+2IE2F=N#+!Tc z(~nZo>4h&JXxWmlqjS@vv$Np0-cStP*It}#Q-PNo*z(%$d&CV@{)`Q&G;o7@pY z^e6MVgLhG1E~VaiA(or01gbUJpz(bX{Bk-TEs?CW|2)m_@)WG_`3b9rJ{2?WP!9WK z2Je?(#t%jvgbQ}rNOggG-%k_QzxaWgC#p3%Q}G~cIQtefxwsp{u6O5d9e<#IS_kpV z!GhPg|DLrPc@!I0JI(eK1;O?UburJk2(7~l>FmoB+8@r&8cUbMn6YKCG+r@5bN$OX zDDbSy#~!QA$L!O>;BtDdZv8J<9v;9qeCZ6WzIfu(*|T6nWditnpJFoitLRkitQi0P zsVECdWmIe6>7=qx;XPyUalDUvmT$I8->eG6yW*K&a$f$;~kDg;rYO(yl!$K z`i0g4>LH}L!;3dQC|p7{X0>id-1FH;$3}8x-$U5JR+rbgxsdu#m%fc{AXm?fl%DCE zVR31?!YcHRzK7cvtzgM1MpB#JRKAO9BRxHO!l4#n@FORX7md)9G`BKu!eXHDLb1_7 z(QauRc6Pl+J)xueL6$-tPHT(Nc84&+$brX(uG5Zb-$8cR=P7qOrZa`j{N7+`(17m8 zGMo5d&{Ir$kzK1n$-0Y!Q_HJnP@)_%UU;+4?@RSRDHF&te zR@(3AA<$U4quUznS>_I1hTg*sE9T2rw@qaK!StQX&6O;y=O!5Y?H8=0b3cZpTXUSZ z8uH5%+4dnN(6j6a5_a*oF9I>Ho<`LSg7EOH}^UUr22E^o|7^=*z3 zsp<0T%Rk^%x>J0teOb*lANAZ78eOOROM`4>&l{upj-ajHs=n2nFAu6Gx0>1Up{0~< z>D|tc|8r5`9OSi@5EP) zYbPgm`>xS?48xF5)wyml?UhXYPJImz9@dm<{;6jWGJEmq8Jn~@gAZc2`wPJDO}GdQ zG83&UYw>BJv0{qdK0$Q_8E@*yO>Yn4hiQwn#0B`VfxXP@lfZ~uu*v&-iW3R*`+&l3 znnUz&`5C7U3P-C&@1b70Ie*giIpJF@oq_AfkB;mtR1GO?33;9c&!^s|_2>hgjFOOg zO=JBt4vCi;)wCo|2htQU+{F_}7s%$*%D_fz!yX#a`M6ErXh~z>+B<11MeD+sZfZxI zbQqg;xTQJkbROqB)Zp!sPYL2jr4_3B=W9tBnfCnvPR_ z@od&#pgNc8_k_Wqbk-rid`KNpy^2>Sx|zwJZ@u{EJBM+jvzW8ne5q>@{ zhpukMe8ZaVns*k@MGd!AT2(hcjXxmiAwfKfZRkFVQ$1~^Z^U>Ap!)#zbsob{<5_|@ zh^-&lUS1#B4Ty(X-!B@zVZ}Gpt@RVFhIk1F>vG(%ri6W2vI;EjdLZw+8;KvV#geJA&u}AgQv{L*fX9xRxMA`-BpkzOp~So8>jdEqp13y= zw&cga5VQKc|KLWnZgH%iGwsK7=CQb!d|UA!&O1xtP*73tz{d z!^swdrC)7^#v>0i`g~sb&_E`;6tF89schJkee89-A0J#E3r*gS;2+*JfSRApVcM42 zIJDG3Ds4dchjdLyd_%lbDTphy)DNI{;}#HC!+f8~FqH2^)3-Kw$j^rlInz)135X3f zg`Zg_@AumJFRocW2qRqYYJ-e+f`e`~I>UPpV)Ppzy`W9oSA;Hi>PymSvZr4L z4tVRp+nXQN5Ox813F=85)o-9UdYo5t298OLv>p;xLh0UUB&RNP15Nm=nHw2#vX*$v z=fr6pNVoo`)=aCg?faTit@1C_H*jfC9r70sh&$FH`6OJ>Css>iQ9n!4Dl{jxdAs#< zMA4^cq5FcqcgZRjYP}UsO)YE39lO03BiCAUg?%L#y|tu)nesKsjo;#yd?O&;2C2*P zK@()mi7(iE2lAT7>hky5v`2LEP?*uhn)_$0XT-5Q=kWt*U6=L->uHaw)&qmqvnPG? zL`%GvA3v*^B+LY*5vUH7j#L^CSB=Zj$Q0VIvEx4=uc2y3oQ*H0>ow%-`@a>WIc1Y^ zEnu@lq=v8%$pb;Fk_@yvJB2u41U&fq8kYyDvrT)T@*<>-q|#jvrsP%m!w$!4F&Yci z=uk{tx)n*2@X$7&G}FD$Fr|AGf2*1&?tszTCyTwEt8>ze(D%T1&@DFLYA#PdvWKAg z)~ue#X{36mUU*JgCYKr2Os3wRD__0;0;<2x%&Y^eD|`fT0g&H?%b}K_c!KyzzO$GJ z2|PP)bMA(pi?0v@G%Z3Fz(KQfWBU ziMU^-`)JHK%iUcZo17s2j(SZV&&vPC zd7+~V{k~9W=$ZE4eUe$1i~GoPJCpX@Lf+a2=Hy&Z*e%rfA6;lDdj0B$ZG6hGQ?;H| zc-nVz6lv+|_|hv6l&&To&>X9~0_nPidtrT9bN?keL;VViy1xPw3Y$oUMfC6ZvU4I`T2zD~4x7>(>zbKBZdkVVZt!LJV|IYu-t|GVBzhdxV?9RiD*|`_6Y19T zVX)pQHfhd3Hg(!$(!t*`FTW0XoeB`8nW*vs(x#X->!2V_4OHv$OvG1EctXDi!g3&= zAV|Z=fG+jr`wkzpW6pjO-q9m~UaK&dQy++K{e0P^;(C%i1o24-`N8o@udtKmML^gn z%rBRTzF)Mkc+_xua%3EkZWNC^6OlL-x1T5y5 zrOPN+(b}il2+}ahyD-OYGc^SlEEUFR7nYl1_@k|2o-y4=kiHqo(<)5Pr}yD9!cH(1MfK zgqIgVt2`%ZCk^Rjxwgv={5J9&kUq;NF2bpP?r4x*+YE1d3a) zU)z}f_h<+z4Q;k-2zircTxmwN&PpS7y#AjcZAJ5HAjyyVtofYAG(mAl-dJg6=IG>w z#7!!bW6CR2PNe)9Q<^IAH=RL7{+4tycHI`m3^LZR2VbYcsM(Liem5&lT&lGQ_Nwv_ zl=boc3vUAT6_VzJ^PdkW-TnAm<^#l+ln2Br z?4v1Y)Kv{ylP?%MItf9Eu#}d}Lw?pNDdhEZc zb%;M&r5PxTRP_XDXa7Hb`k9gERbGlT>;R#1$fH1K{#YInl(DJnBp+6j=4}aS5-rTnpz72E+4H~EkPN; z5V6N7T}!o(ezgS#?p~?Z1%Kr?P&pSqpE?wamT~BJ^Pkqy={OJu3G#J7ewb4}Clto1 z@mmG00pEB}N!p3iSR`eBO!>GMo5xo9Hp&V?X=loYmFLCHyC#9BT#9FP6WC6xLE`aR zXRzH~CP)X9AK!>eqMvG(|FnYsh1RI`xTAc4xN_+>Sf23Uv^IoW8$o|Tt18VBYOcZb znK4!PWq0-!>A`y%^5&WkWnKBEg`*&2kP#noA(r%~yZDM zReE4#*PDWJCNxTFOrF6IZ_#%MbRB%vMK@4hP|YoAL4KldTkaPc1(b_I>hrchwIPTT zw6@#Fu%0&_vDiV8K)MPG9S?$fKhgut?@bL(+CXHEzk`&uYROBhH6d<@N7DECqz6#x ziuENvRbHCbpOdz)(poC(BaZ`L_pU|CEb^(g1!)rYBj-NM3v-n-?(C?-u}1 zNVlmu6!g#*W~~d7Ru>`!U6+P(G$4(oO&HuvWr>p31(jEz+>_TzJ5G7pIM!ucMpb6g z=UNR;Izqe+JRm5yNAh-HaH$abPOb&hUulJ~SqHTOy6~ti|6yT^)%@_L^SCnX5_|X{ zTO2EG_dKps_Jjg7{tIG?_zv0A_JQQ4&=L^JQq>bN^oW9a_!C;M*M;4 zKh5_SNmxE8Tx{&~7re)q@nsga{8-vYXtZXYRIeTRZ4mw?pYd2&&(UjJx{&(emsE_=dL||_r^=8^{h5_$G6(--R5|15n{l( zAXYfio_$(y27;!3qJIN!{5cH&jM|E8O*pr*I0N7N?!xu;@}S#5Pd;ntbWMV-u6(fX z3@qz52{t_ch5@bCLrE7p&$3Gs(Ob!}X%AFtpELxkERC&Xu&}-PbhXnL#>W zmzS%qpC7$eLl&+_=6`yPOmZs~{caDHV|Avo-qjlO7tPGcGc1=I{2cLezi#~Bgu1fc z%sbfl^KUlgup|5&^as}7wU=9qf8nLf7*+ zT|cNf0lmY0bU%X_Sm1<{?P`M+k^ZD6iE=bKtpXPpjzhmhkMIs0Ti z`K2zh6-P`7AN#}n%Twg#SMfNaiKqMk;dpCXf6Y6+Zn*J9v3PNE98NIE1-))obe$`; zttm$}SvwG7cE&*SX><+yZ6(z@P~E_BR+!N^3ts_+Ycb(7rn8WBASy|>u zw)a^SFKV5Iw65~aKR@}nmls&Htt-{MmOXulhW%&AFFl4}zul5u{g4miN`H!v#egS?wWxDs`Ev$BH3#a));#oOf;= z(p=M?hW2Ux-p4JFkHhUOXZI-I9NA7==)? z8a#Z^mS1?1p!NRqgnB+j4lvm)&dxE1@Pc-{&X0SlM&!Ch_0VIrC#arqT{sJAEoct? zz&xpfS$2-*AIK|N*WHRE93o+|S#`d;!k^8X_=HiNFu!%S7&Gf9e7v&hL_zu3gR zL*)vq?vgMA|CXe%_D0V9M??W`iOo^h!+ym^v!p-A;ra0Unw0iu@!i!sjMhoU-d@Rm zeeH;k=y~^I{V7QO2Yq%}NQGh1JG5|;o<*=SUnsYZIt5JvuVZ_|iucESB~f z?NS~g`pxQxw<<$$;6hto(<}q^p9**}q9qS6$f@E2kN0o!!itGt7hyxST2BT$HIa93 zOqP~(KULi+A%v*_N3M+3IPV!FKR>z*3L`83&^>_kjPbAL_u)y-3{*WzH~}krx1ri9 z@-hD8&WV$tk$rn{qGFCzd_epuvd-7y_iq`3s!cVf#BU(^%t(lZ!|Zp93{B|3kuoHF zCpor+;Cky(arMFBIjDH~)uuV1ILGwtR6Kd1KaPlzU{g-}3IC@jdlZ;(HK)&_=)B;p zi4YP01u*X@`)=a}ioZK{oh6BvnR(J;q1JC_>fWk4bZhvKHP~HW`u46(Sb4o_eZSMW zGw-er!$U4}MMaDc3L8^&O{aS@&b}5jHn3UVm<`WwB`<|UVeW!ds^d3EJcSouyn_z& z+_BG&jl6JVD&6-{1|}yv()=L4ZWSj8BgBo-$d39<0m2Q!>!)aTjxwjYCQ2_rKAqdB z`03k<1hF^!o2LG*(-?dEpZBVDr?9(}pnGo?60XAc@OQ+|FF@bo9{-$Cd+)1!x|9)O-%^c?hXyFzeqgP}sObSC_lqvjv(HFnCoC;>{n3DQyahBS>#x zRN^}9-L5t_JICa?E`8}9ww9cP~`)W}LV?5mV>pC~StfJvn%5;&Cu)v&ZjUUu&w_oapbAE;@>_atwgzNZriUZ6UlZUi^%y~~4`i2e` zr#)RTZcjVt(eyOjJWFTFv=5dCi{2>CP8aJrtJ1IwCp7pb;k*W%GsC!|+K zozl>FW!im&jt}zSuFn=!9KG1Dxtf6n2e@i-#b(|wI?{koppd)Q< z67b~shMe>g27WyyNn4@+*n8Tu_qHniCsj?idR&4vjKYQDplHts!oE>Bc0wYSJsm~9 z)*7gPsyuyP#~w$Q`a;Io)Bzpr@RiuRScvk*s( z+D`nK1H^gS#5g_q^msf%>Qx~AmB%z%=z4skhVV!#ZS^F~f%wZ7R{i{y|HwQQwMT1V z&b*cQkiL6cG_w`f=wK)J(RXd69hH8Ok*&N%g+mRH#sV83;%t!QtO1>f?X#G``=N(j2woL- zNOi^5=p-`R;}fbd;O}ob?@7;()4a3mF80(XVOr8rg0PzImph5XG2;5DH=^B%08H<- z4!0IBp`R~6+rp)A_HGY4huNj7o{58H+?~F>S@i_y+q=DdZc{NGe zOwfF?kUkOl%6E|d*UBa_B3kbg((eWJrKW$UF<2Dzi!OYbB;3~&@AMEoKDW@Gp4yGL zCk1G2V3EcGs9(^aR{|_b>xp|U9p%BM*7A7wM?kzNNDor&^;7r_ey8%Pyj;R64Q5te zgJZLrR&g`&jZhk2;S+fW_Ud#^PVd7Nt~&m<ZFZJeZdOY7Ts@Qh@vn8`;90 zJB)PV;Z5H%ZP+&?Jf_Y3d!dWTOCbIh1u^D87|CB(F2-ZC_G(X8H|6y-?Rk9`Pv_u< zBH@C69ZP;^)G%##==00gzAa=94EANNXtQHA+)MZf89{Eg(^tw%jU;#+_GdCfr|H zNSa3^J%jq0jrsnly-;V&I90o>UvL_%JLCq$2l5l`8?3-ZVWYxjrCo&b@WiEzJQ({BYzJ2zZot=`%{AoR z(P?}N_3duF=-*JTuW!KZ`c`9vJ>amu5S8Zh0iI@1u` zvYUy<(Om^$A5i|Hv?$CwXDat*&&MV6Zo`_kL27MaT7V@^UNng-zvOHX503+$gLlFq zB+ZX9zXpD-XDmN2w4&Ou%;p)>_Y=&LBpln4b{_HcQnT9w|dw0fB zp?Gzm#RulJWvZ6El`!o84nNNS0oUnlKKX8kXp|pG^LrjUQN~Tz0;JJ^<{n9-qkUm@ zB#)aPfAA6cn~j3-MZ9N6ahF*O!hsq}pKF$9zSs0vyHqBfNyahbyW`LIo0Ya__Azwk z_@ON5*4T#s`dSF&$)Nq*G9a#CG*(6!2bxK-NY{*{<5b>(i;b=#&5`Kza1E+_YEtHaQL*K2R&=6$H`kO>{K2O6m}9G0`V_f8P*#c81E3Io1}k* zBRf5{7**|(M_@`X?CN?7bW(%oeLRjvdct8>w@)l)p&vL59s@7S8QPXzfD`1!uRJfG1VGZK~n`5>i9$=_Tg z@6wcXwyBs`d9bQS6!)tdQ67@~7S4=33~Toi_GMqfks|~6w`0{LVJ0Z8*tx+9rNbHV zEDr5+Q|$4d33~(n!%3#@s&0XJfU*H2Nq9#5-56DtcgDbojq_?Iw{3Kj%N}NFlK)0w z#|PfHe))T`zMeO|_K2WdPwunpEfg2bNf>}fCcDU^*TO-4Ht`o-+ajj2n73`cpyAQl zyjPp4Ld|)dQ)c9=E7_MadM3-iF|>M;h~zsMT`&7)JrO7$L5I^L7)2)IU|35n?w!y0=)dH zj>g#OBJO@NQRNNjS+t$0JdScRjjHF`QO=yaDEZ4*V%eim(&x5Z)dA@X(isL^cb=ij zRdMKxouKqpy=(dqbZIAffNg~RnOfybC@U27`9MGWP&R`rj;v#pRUze8%40Hz)AvBv z_lH*HU^I^!${?$>RnVnIoU|@{sQnQrD;Jd0;fLD6K9pU6;w`#PmA&BkLqkZjdkdvQ zh*L1xeYYTQgp?29j_DeyvN_5R1^G<0Ydixehoyc9LdE^cYv(SoR~)Q-AW#;>n|oE0 zd3EdY_%1C(i&_Pg$%mox$vz&9VNYg5nANTY?H|;MpYt69@m;Rr4wqqk#V`7fzRp_h zj|NsU`}AS>b#)sXw&)OaUH(in@tGrEmU(H^9_V2pQpzsogG$-ZOgRqvbYvQERGk--kEFgvk?yUu)XO%X>GDuG3oV8na|%+pOK%yOyYX;26Zztizu)>?gvqqh-pJuNZYG9eT_^0zR&$IL4zA z>Ktv)95!gVUbPgVIlmpcR-{40l~*+v!hWM}`WRllR;d`A6pklmZ3Cx(y0ZU-lNwq7 z7#6yxV_T~qn&^q4Nbkp1t#2d0YYL%DhqJ=a28FFpI6pM6KYw$uvHClfZgUc%$^TC5 z+Cko!Qx1#JK zx{?3+^ZI9van`OeV1{>stQQhEr=`FUj zVZh$L8QlVs$mkd5q2d% zdav9Xet62{Cq~q}2<-f6VcE$9W}6?t9kq^PQtvEqgT3f0TFXI?`?0qBSA89(CK z!WWHr;^3zut*8wI9Ud>wk9L)Z`nky8!E^(8+!7fS!)0>jXN_ZqUconwrSxlU1XeXp z!tv9UxNY_j`KMPX@a~5+TV_YXv6naS%$dco?r*p>y=X}LlPtg!?w4WX-%@+1z9YQ>Z@7}U*SND!Xy&u{hznp=%s{Rq4fNB1YSEXD@o+sVhn8}gL$qnb9~o5|_<+t}qLJ!HG@eMH_$ zH{QzrF3wqNDxZ?K+UFTBtGoQdIqepsYnUT{^J*jh`q~MeRO}ZS7G)Y*d!Ers7hZnq z#+R>1gSi*NK`Z@vb(bZ;rn>O#@dCB=ZbZ%R-eH^9^fLF(sqR-SC-1MV6bkJGN?zp}X z=bz31J0BO>EIL!PtTg0$?Jd;Wv&l^gVM>$9nAPgKfVf{ctIwa^XcN%$ba<26Tjv6Z|A$kX-htt{ms=&mZcv zW47CDd9QWzg-@I_+@5-rdN~l&ZMsOhCb4$+DN#N@53zm>Ok0{RE?Vh9lY!kuoA2q^ zEaN6BJmCgowY0t>wo7$Ey@k6XCS%8xjX-sTA9UR$)qtFL;UG9}2!eOTb$R>~FBx-a zA%y-KiVuS>i=lVy*p)nngm04YOZ#fwK-r~Xnt1U1rD#Qa2@@{j@<#|QCb^+@nt6MH61X=M>m^?m0`1=qSBQ7s(cHX0wfp z&AIsjBi{Inu7-ZkCx_n!x(=vilfXR98qzu$^W<~&!DZ4={`cxswtQSAzN&2jy$)`L zsVBZcLh@L~-oLGyr%i(`WVeQ&t9T`|-%?t0i8g^@YS?6c;QlT9A$xpX+;ltx4_SE2hYyYA?YLn; z*DGl*@LuAb&dU=p__}BQ*qBW)>4X!g+R1G?8^)fk#Y;;j%{we9wbs zc~XJ}q=$jCgFcr6uHj@YH$qorKvXnVfKKL?ZFNlZw7 z4BKlNsyP1Z670}@85$4!=d;(whM%1E z8oMXnrulZ|P5}w(wLrMUilZMA|9jxhazme)OMm$EJmv^{b_S}y2c-*A9h&movNX)F z7=z7P)Z;UUUJ`vo}uz!VY}AybJYQ zUDe}COKAvm`5U@#+)lm~13EX9bIkiAaWm`lDx0v@0uEnXuk;PHIO9ise@;uBBTHJm zW%W-tgSGVh*W3@zKwORVI@r6con|PVvqxBkbz6PYsGqM>kJ~sO&HXUy%J(irn?q2O>Z1Yj1U2S^3l50xVx@f2@G9x(8c5vBGvmAA>dSj^{ojKa zm|KF5S6TvTf4P543p`h7hix_)ONHHpi&7`{t%mRz4ToilZw15HIOp>~_$vvgQSoBe z;hcE!2W&{rS9QYgbT{JLr|cFDYS>8PW^LU_dIpFz66sq>+7gL(u-;4BqsMHQAbkr4 z!}Qg>fw9FJg|SHEzjkGc(*-EGsVf1|!(P$IJxH1qHu3n|;1B>;xSM@^V<=&vi{CneNg_VS_B|sj8#g)8aL;svcr&$6>XCkd9 zjwFxy_TU+ydLS$}#8;OZ!ArU~lGYh*WJ7p2<1ggNU5ba$b51Tkzj;j%r}+$NGy_9^ zW#PZG@ksoO-$zbjVOuAn>e>82cW|ixr>bw1hKYV%gjDaG^jXzfx;!PWSx9Gne)NX# zcbuT-r!8#8{VAyWS!tsib0%QKdL!O{*IMe`>vZ09XjKhT&kC9&Zew?x>5knX?$Gxb zq&Lus&SN3olCS+%Bk?gKogmtuG?%&&KG3@Gg&G?K&rODBH=Q+#H}CGdh(~&+aD!oy zKse356db|DPE5qmeo@5pXkFtC3t915t2D^@%Jun#->mkHdidfK-8))XtTnwD4GNP; zgEGQvdA)Q9e0&@Q9UNN%>2)B@kKqeCFf~8tM(gs*x4Cd_6C&{t+Ao|bNiS&#_n=y> z8nm9}nrgMemHvk2XPSsDv-QEctP6RC(%eBYlKridi-f_XyU&PgLuRvcg*#xwsA3=rl8QRC_@Gg?JpzzlkRiPnoW8Y2x22kDGhmr1ul>8wnnnvyiVs#WbN zlSSl}!X#-?-bLthr45X>q(OJD0vKHzHvLfM3SI-TdL$D{m>Nh8&e11G$JiTYnbai99!%HW?sno}w) z%)FS-DjK{((y#K`oJ~l*rycb8s>&Wf{4RiP@9L@7J8QB#X(_@{89%YYhxm&!qA2|O$C2xfH78BB5Sj)6 z(j36XyN3j6FHl~Kv=!Lr!kQyQ2i6unM;$S^@WkmGePeK2m6I9&w<2I^c_$f5N>mo zBRsAM1zJnmUZNe0pIiX;19zgzv#8$CVU~r=ZTd;+8*rGlu}X^*P9k}0Mmk!MUQyW? zP%gt--;04t=P*INDim)hok<=Z%svjofKM-gbftIM!yI+JJaoYp*j=rUR63^qcw?q8 zfczRGO#zd~v;pEIW|Q_4?m5=R!km|&avth;u}^!2?qfHVYeQ{Gw~l3m$?(jptGI1o ziEc}_g3?Ss2Gy2BH0w!c(tQA~Ta>1UTGfKVXKFZ-cLbV8)lJ*fN_`CYcuX&f9qy_#9u#UpVvP;No_N=L})R1YYt!9RzS zglASLqpXSYqIpU;irJgrsB8`?BLdertpN`Lr@OEjc;&awNeN5$3DUS6S}Bz2=XjKX|ep@bS?(v0fd*5 zaEy`O6A6|nNWPKXc<+Fe;i1YgmHsvQ2~v63n|VF9w6yn7yv9_X+M)X&MxF<1 zduFk{!`5Phrk!EU>up*!uXKIzbI}NeC0g>#taas4jpwG3sCXjCc^5qgQ3E#9zDmR) zYTiZM?uIz9^C_WMtuX}Dj;-QP@{p+X3-L70Tbl>;ca#riLY7em%KB|6$B9!pkfz{^ z5fF|G;x{B*VU&H5R@#ei{SH%plmHsnJmo{s#rFtY*xm?o#shn?P8Uhjaq_rGxe=pG zkMEyXoxPi}obdi9JV>!7&*>{EYtyJ4g!Csfp0^8BPt)IV>PN3x*VnSz+S#1?3ssLk z6{93+I1%yLiIGlFwSckBA0lP5Fscse!s$EVwm~9NF3M_E`v|I!Y<+5IPq_C%6VKb? z-7BGd2@tNpf*$5vpC-d1yhNgt7_Rzj&K1miU^gJR)CZ&E9lBFJ(~P zi}x>MtqT9cn{h|*@$h8!c~?0Q_EzOar&=UaraHBXzx{50p$uUUw6$6YD8oyWmfyOvg(9)N2WB5!Ln9N{#$~z zUwd+Irw-h5{9{c{a!(jh!x8;=dqZr%I#Dqx7ryoKY%)GLN{3C^o}@7}Q?Jo#xSIpEp>xSwzs%gUa^yVtrh&9*NeSa}_7 zI~wD~u`{?s&SVk1=P0gT@L7D+kA+S9+X(&CnpmD6qrE;6WYQ4>K5pV5G=6LkH-~KH z!yf3%4$btT`bK-YU;7Iy%!x+B6C0VDFYErtq3~T_q_KeAtO7K5JT3NCHCG#@_vf;eEZvsJ{P9!{e;_z#~UR+#`*?&u&C4#18G0Q?0w~8)Ba=d5&Ut? zqLrx4$cFd5zagy+WO$m%tBWSWPw!Qru2s#ALr4p*o6qH)y|+bh+Cg^ac{mp4nUT+& z$P%|0Xld@`H@j8X>QW27^m{(@o3GE8l+=Vpd`ZfPgEf5<~N4-dlQ zv<>p^OmnQy+OxX7UIKlFrsY*P9@VY^TDm4-)9qRC;N&AQvPDPQ3)r3a2;Ie_H&$R~ zU`sxGr=vXT7Y>oZ!#ZqZ;0$o$j;vDcb$HJ_`L7i>$$tnHNN9|?)#km6XdhX z|1hllthlkfKmMinnsuhd;DB50(PrTY7!?qX%YMb{DOLouyDq@Ht+a~!DrV$$5(+;K^fQIW*_OQPvf`I)1}Cabg=W10LXNBfT2gzQT6smH&6MUC5Xu(YsJyv>XPEm zsNbObXcJk+EoJH8{<7j|v}V!swJ;^M2meCPfjz@|q}z;S(WlXZ?w6k9VCSD}`Of$UBp(k>{Aub$y3-#89H zR@Vewe-5PkkKr|M_vTx!G?1MJwgswJNxjRzb>4^y!wJhl?A*aaUyp=?zSWWH6bNgP z`cpIW&Ifo9Qwxg^Mln^lb>I&)PV9r-lg?-HRC4NYjEm!xV>W-5BQUYDY1=s>Vim| z^DP_>&)K2!ikY8V;*$~&*89eLm=@EN?f7SiKT{r2FBTCF%#xR_^my>tvq1MO!VR9X z4uOMYj*B}|Ex^2c-n_L=ImS;Y72GpdIBx5}b*}vugdeP)b07Zv*%7wQT;Ph)TBLYG z(_MY!mTQe}wj!oqJ{}O?&iPWP=*u_e1YbiYCimzh){H0%7eLk|~S%o*aMgJxk z6`OIY6}~#uO%l#>pUyQ=t#_&JI?U_IB=IIM)70a}&-W?J#({c4K)IC1l3L;0CcVYC z4rVaVWHx>Ff%mCW+Ryk^GzO3Aq#>-V>bC=aM%=QwIlt07jWqOzxV&fVxX$ zP|zhLY~n{Z_+V$>&8*g2$$WFqK#t2{`p)2H!Mxk;gttu#7B}-L_B-OBB|e3}`=1NX zALdfKUx3CeO>5Jui8S|{2zd)Wv7K#C(7jK?Wq<4NNz+{hn^Zyzx4kH7>+pmw2pgHiEm|t1VPderZ%gyROt< z&R&~{kF)Ak;a&M=JxTq@+R(cQ^iSSi6a2I(fsq+Xu0J9dmhiBcf5E zF^+NTzPHj4 zmh#09bdh?&d*beYVt%zD*w*2jswqZ#1V$Jh6p9xqZ@k};ZQ6k9+j0KobTGI`vvc3i zf;bVPe5 zX5|**p97%QbjiUI$J2MIc;hDCIyhH*@_IwQ-SjZh@6evGm@D7DttHQVwBr}Yy#?2E z);y+OBwK9OnROg_24lNwMe}!GA!TV@{L~^7i07!c>ae6BLm4&vb(MxHwOt9sNl4la zDUZ0_WDh9Jr`q7eS(5Y};piQ0Y}jnEw5Fq0X(5jq2XMoe*IL2{>9lc|R(~fw!*tf2 z-dX>_!f&`RcN3bWJ<>?EKCfnH1mPDO;vBAe3~nsHqA6G$l|OZ5JGixAFM77VChwO1 zVk!r-*VYkPbVh9)%f_+hb>tVnqs*ss9O3R$^;sk>i8@)Qh!^A0FOhV~zrJ$k@5m~i zDW~@Z-}~G~;#5>RbKl54@MYjBR5(%x7D0Q*A~gNgn3ZXg;cVmn5L(a_68sy$j|rZ{ z=}%eG_`34tWj)?_Xe?w_Hj?YtEf*_BYcM&XCFO7p;l)$R-yrd8m>K)AHG=A}AL*Gh z;2Rppi#|Pob?NIc*V0H~rP4d7xNmJnBR(_78?78-aqfp#NHq*@rWbMOn-*Mm!#>3a z(6_KXZ*SKY0;dM4x(B8Ah!a@zcVS#{C23yLi1%pk)s&92^abfQ zjOkI#l>R$B(~dQrmx`n#VWV_Zu>-%vIMU6TnA`gVn0~iH>J#br-!;*`NqbayM4XJ- zot_DWNu*z8vo+?J(`=8(Z2krVbkbpK$#RGqK7|wi!Ooxsf;37MFDo6{JM#e6*xyX{ zESLmzFY>{$E=amltVt_o_3K#!aSEJvTS%NXnqAO|=Bma%4sH>XOVOtS2apK zi_3$1D*cUA_hSCVwL(94qwsRM2=D4LaOYoaR&@L0FCgSf)*UjbJ9r9p2t zBW#&ywt5vaYjg~^{wanWM<=AdP+F5kTd!j+v-QEFS}yx@G!(-2#Dmywhf13$J=DHQ zK5V}4##?5`3c^)5<2(kGz9fGF#JhsDIir5YYe7eF`29GAZ(^HOFHqcmGpY*`7vZxa zOL_e5Wn7bZ2%esH=AD<^Wz-wce&{?jxs`zDDyq|G&KK6;G5f>6Bjblu%yRoai#I;-o-M{A@Y^iZ_U3M3GgK{6iG`e{4WS< zyOuQhGV0+-sH@vX8W=2s!XHzWzM^&M%TpJslU~x1BV8}SrGSUZhsZ_GE!4HeD5nBO zab_PwD#hXl%h2dk2~-@_gRt@*swV{1B$f{Fm8Z%~fiOq;J<{DH;r8wdO=PQLB%h$& zKYWMsVVclG7NY5{3rKoYZYz&e`im1D3Z-L~_{_%A!B-%{Zj|a{*080uhOh&5FID2d z*(td2t2g{wRmRrd3MP*Efd~9ns2pm$t-Z$Rz7!6Mp%pf?Cr7r@?qGPfv3wApjN0-! zNO9Kmdmg|;MtJi@PwBjuX$^t+ol$&+;{2$ly9MPKLSNsboHr7R+mhCw!|JQd6#p~I zDJCo{Wxlg@kaQ>1saaU{I~otw=h?JBxbl~;{vA=Z3#41%avwSp;QD&@XBqAGYM2Q%+dW119?D!pAnQlDLakkBb5FdO#kmDi+fZTq2J& zkbh^!lJpL7m#vIz-w0pa-T*JJu4IIllmj|bF6*Y2JR}fLB4LK&1ti~yqwQU|-IjM6 zh419`v~xd9hE6e)K=Cc{J^op|4~`}6p&7Sxyt0-4 z`MJ2vlg?w?Z~`fASS^Y6mzmR%XB%u4q}7L4A-q(6TgkjW z(X%q=r*Y+Hm%AFu{<}@ce`I5a7AAsn4l%wx8TF|oAFU-VMI7FkclLAOEpt}kohd7E z#i}W&|L}IfDdwd&`3TBGIy*3MJx-dk9Z!|6P&h8g`)S99|HDA9?ZiQboN5)e=nvJ97F4>Q z_Ed2av~Ko!cp1h|9)ji7Bd~jl9|pSHO7f%hOw>xC`>m=k<-;#b_@$+_X;nYn$?%2d zS})!!+ypjiw^VVGr+c~5+$>~}3(&ta@}P|RM}D8#8@`1N7bP8rX>uHQ;HOqOTIHe0 zHv;)3AY4Y%7FWfyq75u<$8Ox?-4MthY37_L(5Qa(7YESzX)~C4XBMdcM>@J{7Ug^I zuOR$_ACsl>x9zXGN5h-1_V{VcSn`8T`s*TBNZz>d-@^ z*BU4vB}xB4y9RejYc!@ALITY(jx%FBdZy0Q9n@@zbTj$=dxCYl2)Pzf8q;5yNIpwx zBjsO7`@~jp4S9YZ=+zXceife!;(j$(1gd`|&0VF(a{FI~$r}fw(ke6d_7Dvw&%i0` zE|CT~q__tG^4Zsj355SA;rYXzB42h0yRYiV7T48yQ$2b+-daSS&_I%37E13LeI0;QGvtSB zshK46=}LPhMe1?FaJKkyJX8KQ+#pyeyl%Shj!-_E=D5tW^&r?6$k@oACfv=`uWHuN zdrJ!*+-#Ykc`18e(^%3R2o>I{d!kwgt6`^c*pvFap=%bXHLpmabC_waM|=!aD?pe6 ze-^vqaAR+*xjliC=K}isbkgd5rhG8Xhs2xr$v`*(T{o8l@f{=H6vton0?M(N9+|G8 zIU^5f_y5?E9?^aayIobEMqkTN%}PFacNH0~3rI_t0nG&9^$%xWRO|%br=21Vo)6O_ zZAnY&%KNJ`lqLn5BNY(d!;_LDNVvdy?YStXo(xyo3duhL%}Lc9o)H&H*VE=mIl%JG z(SmZKuvk#v(2S72>+F?sD=UaH-zY)T=T>|HwUop6O zQ#4u`h!K-R__sUGyvN6ya_i$9xVEPaII~1Ph_IK_GgHwAhG|l#M~R>PcCe3krpslC zda|8yymtBK?2jfO?q=$70*LN1Y#4p<$@LyB*YI3(&$(oZ)pfbH4*X?@|D^m_*xzJ+sYSUoc zty9_ z)Ras8E#&4YEts=Kb2%`;Up{EzuC3>^6xUpBRCNytS3^)=Pa`W9#sFOxpl%y}-<_V9 zz2}DEe@|(;AKbt{dxWrKO(sjjt&K3=_6IYaH-?8F{w{Rfy79mIX0pYEVo~qdMR+iG zsr>lw8ai0agbiO@r0KkeSZC!Xz$V*7=Yx-M&wqV*;~L%NzCktRw(5Vx-NQ$;U0%%O z`|f@f8Nca$aeCJAbep-NL79nEd95~SFx>mR4SUv3;(5L^r9qoFEPdPsmb@_nO*fyW z-+vl&2VKIRFPig4x6NeNh~*GscLw7YE|V`V*s4ExHMtqyvh2?c#=CPk6Xf9 zdiKS{sRzaGBrK_uBUTz4;N{MBas3gF_biQY>fkz{>ZDcvZOmV@S#xaaQ5@AH2G{JR zGn3j(mD>-d%e_W|>FqkH?LVWI44AVRqB2_3K3zJz*R{9UZFV01YrF}2Y`ITo?3{&R z{d3VeC=IS}n2QF3%OIAS$+rW$3$46_w*JGBa)k#s6~jqAAGz{*CYa1LV=QwKU%0I% z?-<)!vdlkh0PE?&DMLD z?)k>ly0*^#if0yd!do4V;n?P{*(@hLIly`x?a#ag@{d-N5tEIJ3&d^XDd-wV=kdOM4k2c$?L#vcIZ0(31obF#F?4|FCce=unH@ETgh!R|$;6mrZ zG(n%=yTyPNX7YVy3$Fk3E)EN;%T+Dj*{}!1q4(%Gs{-Db4TEv3JMunT)t^r7GPxC-;h!ih)pJ2$^5uN zJZBdr3WpuSJ9B%8@%1N)?5N>FCvZ8A{&WX!Z>uHgzwzocir4j{xKHOkPS~X*scz{S zCu#nR_@OCX#S7fVdRr*R5U zJb8-YK5^!hG3CXc7YzG}3PUHJisuW43*6?{3#nJ+9sh9!gjL!fE}gv1pZtTbNq15G z>9)?4>h~2`wu*tfhdp@Prav*H>uwmVQ%`1o93Uq4{w}T zX!S{4xGj!_-ySU~*WBo#u^eK(5xTE((^5ZRt)J$2AYh=v1l1pCXBmm!Nds}OYbLGL zQNBu9h_)?0L3pd>7_3(agEmR&Ze}PcPQ1tbZ;;ub02QViL|Xt|gRKiK)s*Ts;qUeW zA6ec8UOsn(h;B=yS?73)r-_{JkOyl=oRu5v9!6RxEL_}Jman}=`@4rSj`E=)EzB*E$zqpXj zSEOIZx8b|-Qn8y_7naz3K;K3hlqM7S(+|G1-i?BCBfkBsEuAmWK9f&RX_8XfRP}JSG3qEmwvT;&4B7^bHwFruel{2F+DDNN3F(2iD!X&MYNsX9X5r=vlZd**mo-n9ITTJy}f-T^`(~h znxAY=XA3Uc0B+}j>Z%N;t+WIE=JtH5Qwa>|q$8s;XwQC@NSIt9#?JB5sJ_Z+RH*n_ z(wY?xfceO}{D}1yNj-(_w-@%3LYkJf9d zc!Tr}5GM-NUk8uxgo``!pxe4NV(PucVoI%(Raon{Vg#G*wHG6Yl!MY+rB3nqwa5+? zmhXD^1BO)kekD-h_v8-0C7N@7{?we)N!a0rFng zzYs>9V1yy)>SL}j5nG-(2J}ZXe7hFTu6$LnW^X>T=nw{vEWQKj7nK{hwD}^uKi`#6 z?ZLm_?)=bQSE!)93Z=<&ZW!G-apO( zI7Jf&q=zu*C7nkeO0$abQuVc5Qo|QsI_&c{0AJ>=U+djGAty|$?9H+XvCsZe@~xKT8SKY?NA4ui!s z4MeT&holp|N$V09&cp_Lazxjm4t(alV@S1t=ie7r<=rmkrBFEgD=8Jyp#lybcjn6v ztiilgdi9}xCQ$!_?l2dm9^}-|+I~jI(Eb%?7w0_}?Kd6}v<95-;35~F4uFx;x2mRA zx7Ug}BhAs=tA%j3r?Wd=>ysw!45YzC_0>#UyNx6Ka5;{e%bk%pA5TrPkh*>SFezm} zP_DF3Zk)iuN4rT;a}y8;p5*i4fBTy3Fw zwYO;gWQn+3cAsYCPvOi~Cq8Na1)=cY@liuqyWLFIv^$|q3ZBZJ(K%*Hv)^=ZMbd8c zJ=P+w(@=`ci&GJMEXOPLE+9Me1V{t({`6f9^)B`F4`x<&U9-yeAX;0VNAfHMN}GTD zHwc&755O5EtKjtD5&TBs2>3KT4*f#r0_jh!>d!}PG8C|}%)Ft47j)Shk0NmnWk2kc?U&lmp$;t0=3dXFr!Z3N-^HR`h) zig!dKI`^G4ClE$4(+X4N$>4XyFd)w*jK=>4{l5A_bGewU`L2hAS@>e77eD;<7%DyO ze1V=h`t@4S_3%3F;bF5a3h7?)L5;puys+opK@csMkzTx}Y6Um%)uE$3;T5T!<*FgNh%>yH$C$ zy33^U)?#RYf&AFol9Shfjj#QH`b`if5k@S;d!HTf;p8mk1qmNJAbAn!kYr4r<&02% zcFp%_Mx4sYn~9&@&uQLNe!!-u>q^BJ0cUg6`suqecTkwpHhl#sOigK>#)foS3Y(Ma z0@X3aCsHi7)R#{!+KL>%NTz%~eV%)EWKh&~p7QmuRlg2bTzka2k@EKF+a&{*+Lxii zeCpx$r0Kb;*=`drLj2n0;_#I#Fz*S-W!>@_;R)O3cOHo|Iq!ZP6qYW!-GIDHApBpoB`Iq8Vl=M$l9p1rr>DLpoDsfU&{KHS+rXZBUTBt7d4X(xeqM932INwF8d9*EDiE|x33gR`O{uI=+q+=#B z@>`l^lN({RIfsPbiaTOKop(xG6EDmG`ujk=3WQaVp0}R8h`b`c*KIGUH_^MO0jD_u zC*1@YjT=D0qDLaxtTtD_`&Irps^LooN;^`mX$yue0McV9TIyog->HM!b)o``EK6N$V-ssgMO~*nvM~d6}M{&uB<>cZ!t>iBnXGuy_gST zMfeOj7M7~nQQI5|3;Dg6S)>KExNvc%NDOZ#2>UR_(3KIt(;Q_6{`~!mk@pcxTeRWi zqsdFP)G8mS`i3x=1!SEN)emI~<(&uL%@ZH*RBA^Dt`dZwVpNl@FzQcx^1q#d^o9tT zl#I0p*AnDAWpAUujQW$EY@P#FJy(e_i|TXox16*SY(*dJ=64#FB}QY&qvx_F`Q{D>?`{|>uA-R^em&B;%45f`HRR@Axj^%pD&0z03^cz0 z~te+S|SLiP;+Oi;xUSmrP2+W8Yb(hCLI`gXr&2dW&>}gW;|wPgLz=|W70-7A+_l- z^6j^%ClX+Mv{gZg%`ABBY!0NU7mO%O1iGG3`s4JjX!7!D zQ*`};a79^ix>^nPSdbckuf1G0R;uGB+XIXKg^q#iuFMwIF&u#1_5wsVW{W>0z_ z)a#4Vqxs}HCG{%281-5ZFABwxgsDRD8F>#*m@XC1K3;f%aLEX5hI;||PM}$omN=W{ zZ5kB|K{`?CH6$O5R;3YO_jwcCy}gO>OGnb|2B;rcti>Ki8dUCn5bw>bRs-p6Epd<1 ztKM{9?C-&eK(i+nR+bENdk%vuBiFM(uZ_sBu3-;a92WLuvT$|lk%W`amDd9D z@QnPJ;x3^1lpM1tjXd}#rOjZ_ae7D3yE~G`kXte}ocNhAXBHgroCY*k( z(SEh(nBo9cd!5eMpwcsxGgRD3nhH1R4^eX(Mtn-V^hYR9NL;Q}eo@VI67F{ahq1s_ zPSvckM-G$Z(-`STsr0An%VJLGsunLBje*+!9jj4NB z{MnB6pRp2XCZ~QsH1wfByG(}tJU0(*58e00xOX6$JCs2 z*0+wLOYNarm&o&|W;BTdeWdbqO8ZccC|`)=8D!Ip7WnPUKII*io|B|;knoGs{Rpai z{^98bARi(U?w?fJ2q_Pu-ClQWI<`m#cN>d(6<^r7KqtBFWF-H$z*gqJ?kEnq%;SIB z9fWqWBmd-=h|^B$$<3jcan;DjnB|uY+fG*F$AACC&mW@2)8jV0@7p$fvTYNdVQY_V zv(Mo9J{#fohZ-?_T;jlhaK)%fUOdX8!IMHq5Mrn~r-|IyDg>`w873XJ zUxv(GHq!OwP^f*vRHoh{9HhOgJLfd#KQ2G={!o;yb+_FPoqXHzfWpJjsnA1?IkB3~ zud?HPUZ2F_+xz0T_)y%~>KQxHwSqN!Umu$rB=WMC{juc43LLN~PDWns$xpPLraja1 zIL;00$wwyb)As*hB$tU&e3qMphqnI2G=r6LaK>jbb3~}zta}knkIhF1JdY-+3Et+e zH)&6+FL*02PMR3h<>o8LLBkp?`RLsG(6=bBfQ~fse%qrb$9I=5>!Hgt6{Y#MbFjr(Lm zZ_!`g%Bv~W{l^p zzNh&w!{OLN%nzph;g8#j`+qA`T(ocg-NP>LQ_(sk3HByif=S=iV71Uj-m91AuUJr?>4088l0Pb>*Yc zz4_mqCYr`C#=&*#wlcFg4Q>3RwNx)SYJmf$de-GPYB=(q0n_Bp6(RET2R-arIuo7s zoABFSdc0njFf5pdl#5BcU(E*6zjqE?E*Jw?(~MJ2S%B{Wc-F`qG!=6o@^xpfa@BDD zKGCARE=-!g8#lka3g>c~%AD2#;@pkq;#%Ti>=I!rgX&j;*_?^|U80-H2b~M!fSc(2 zouMP2fMcO2j1L{a_XVf&@vD0C{&qCCn*D|?NR5~DyQ%&!!N+&&SgpWT{5KY}G4`>7 zdVv3Ga~dj6EN3ycjpf7c18@Y%`*-uSkonF^)eP+FevQ_Zg;UQ8f$_R@7J4Mo{p09HmfT`YFM2P^68g^Hqk0i^KCHrce;wH~;4Pc zeiROOd5=`H*iII~iI#7*Kj#kNJ4SqhsKjG%qiBa%TOM<|v5$HCEav{U z4luaoEcw9V5xS0|GX#4bY5#uQUX0ykivi(B;h(!5G#{@oZ}}zRH}l!B)%p+)YdC-p zZ&QNvZnTsR*bC?$L{{E%_TT$NT=MS`BU}?xLu$*Jm%>HcwkHL}29k4j0QDHN@z;aO zF`ebJMyU{6ycVZ<#fxDHgTemt3y5}WrF9H%FJ($1dZwhpnX%sdbekT$#9%qH+3#_G zYAiIjkB6}EBe-hEcHww5pAmWWu7jdE_a)7baz zer$H4E+>3K*DDW@@`O?I3XyUtzhzV+#UHe9cf%pGzpNxR2Evm!$urkbPz>oj;8(EU zKFWJz!8o9L;v1IFK*D$a*TzYD8P<@5Z{Rk+H6-ouMr-2J*;!+`s&|7|rQ*Nf6i~hW zRNtCer=-E*(y7!JMIs>FlltWx-nyLtPkeWyrENCU3zz{G^L%ja_-e9Q@CIn+Y0iHp zlxlA;swNG>`e^^WIV5MTcnuWP6i0NLn9$UbGBrcE5v!tE^c> zJ#C79BNZQ*W*8;d?3wcJ(SkD`*dh<`WYtj2?m!Zzb>X`fLyxiUHh!|`h&i=G( z$8Rr+RGfs-*6!Tvq`v%6W4kupvO7LKz8Q4yE|X=tB}hH2Dd^?J=e{a|jhPc+o1rfA zv@?~WXs0GFZ-ty!nT>B<{W;YRr#MQtqLzGdmyhB>);%E3lrLREt2nvhgdtX5?JD+1 ztdf1_w~|{VJzGBOGj84Ks`>`^dDf8>uOkf-1?BFtnBrkA&+hQzPj2LhY5ux0s+YI- z=e$_3&wt0(_`1SWyF^Ykh#^<#45HEgjB1>o=kDB4=sal@n5i2 z{r5Q9Ga0Er;kH+_rpDjttmzvDOIO}k^(>G7UH~nh{lNg=wwl;?)`X7+s@9Qm#Y_y6 z81V~ze`>@DFW7&#)`Dk>Ijv_I`siok-aeaPO3k%O*T{*9?J*;03r_5%6;q1WX}YEw z$=G+BSg+@k1aUG;>Un~AIuNg0A1LsvPOoS6(%_}x3xRkQ!rixvWj!w7-J`$Q-=yZS zIkX?7+3WG+GZ*uDHZGFZt0~{%3BRoh;M47u*mZmjetqX2d~Bs<^C}(j(W@S)K0oX4 zWbjzH2?(2@b-^>XCGn_cX2mQ)SOSKtPm2$0GT@_!g`{3$#ACeA)&B^W4A@im3<$Y0 z5|RURaO90v#1m6-;rQx6`T$8g3F3BeUXZOc2bjAKMT!v+9;$UKEymXF*1}>*83C z|5$x75I)sh1Lu5OagX8+DAr`j()yE?=2e=LaN9tZzj2oz7n;LQoq@2dM<1+VF_C-t z(R0qE1(bFpO^k$95SOwB)n^F{@`p7ngBBmF$)hU+=wOBlk(}cMUTM0L@EL!M90jDy zK-Fw&1f2zI_8Wr>o&fa&^l(`SnoB>CG$ce-7i`;yxPq|4L&DDdCnG&2gIhhX(iv1Y zNVp|*Zl{9Tzu;I@u=3i9lzG=s0bM~90)fxZ~l#hsVVf`?K>-;m6C-2j&DLM zzXnK}45&`|(rrUkpIQ|=^rPjpUuruP>tu$;tJ{Birp8}=(hUpAf zxPkpsQmXWD$(<++^W87fEzjV8!Dgbyiq|0hOfhtIXE}84NH9t|h)QdV-c5l#hvFzk zIt4y^Rp&`#nNS*;bcs;;b-uM3cT~*4-#rV_xlaZZn|}tQaBtG<<1zngcQ!Jo87L0= zM|)ON9m`)0FR+Tf`uJw$a5>T1gDb9Xu{#>Q{Hx0`_WF`~9td;UtI}6c>&AYl)4m$6 zZg^RF4Wzo2UW*L8SN#6XtUnkd=?U2Sb`Q2%{s~A=QD5v4f5P|UpB*0XW8{57`W11P zwbXC@7^uGCmR&6o+-d?JS-Ar!PS9se52)=~p9K`Uqr%YQjQ0v>VNalQzEL}&A%2yY zTRl;H4wP3|aAFluuNM%G;pp5n;-UA<;FUike89=JSvYHUrP8m2nH>fB47T&{3r6uo z%CUxO0Yi6am|M|q+;ZKWpM7&g;Tt1<gag{^L ztE4)EQMat%=DUWx>DZ1kceEdR_*uwK?YygW*p7;K0IeTtL3@``{>7~jLwc{jBctmI z%6kFD4u|Bwh4abltMWux$qAPw#R6m2G=(E`%8_CNK6FNF;`Tm39z?8Z=LtjS-qX^* z6X&eP*%rY(ypJuY`udhJU1@V~(qQQEZVUXEYpV1Qc=<;$eeHd9O-ULD&bB)YWA4=C zL4W4r+|i{~9_^;bI{3Uum$&>AhED<=rTY6n6a1APW@A6zsi^03R#YNBYsq2=Y9htyj3%q>3AhjFBoy1 z+vYsxW)_qe?nC>oZxmi>Rb9q~B(M(E&I0iOCq4u6^+^5_jy!*cqk|hr3`!TIL$J5& zbr|4Zjb|=wfb~j0;?0HianFrh?ABg~N6&Y`YK?VdLru;ljyRLr&|v zNW21c-%2}!X2d+$cWa#>p2CF7!Pva-DnS?qqx3US=^4@(NZ7_Hhe+6mrGI9^-=tlr zcq5}IN7S%*!QSuOjs7)SV7-8MRh&$$jm|go z2hy1I?m{nAze8aHc|?e|J`69nepf!B%7@;ZJxQ()87(SK2b=)70ERdvx0%gsea$9}R)e7781$l|bCf)OAP~aq^M{ z`D0ENv~Kkhnl-Z48YgcR6i0k^tG>D}(A-G7;6!cZcOjr%d)6hF_M7YSnHlcrfIC+_ z(@>qU@F&_Okb=xgOet4#JR^gT3d|-V2KZUL$EX^%<%9 z*>%+2s=RHvepsVsb)zGas`%wvqC3!>3l9gp1oBVf_JVk>*iv0X^9kUU zV~Qcq3bp<1>T>tqyQm+Bs5u!Q)3=u9_Q|E*-@68bn#~a=YSipS_oM+PwmDB29EXJc zggK#7%?UfZ@4#DjD}~?nV@RK2q-BMQIpq$C3&8kRI?`MXos%0?VOsd{4DHF04@LhE zD^T@6)g+Kd;-qD~Fvu236O#Xms=~Ep!XKmO=aGIr0Hj-ycm^(Kj%UiR&>!+P`xx;$ zAR>sdq@>FTGD3_?^P3QPR1emdz#UiAk{o5y)|i^8&pKBL^We0e+lHrlqb{Uy6$7p_qihHO(y}OQ&QGS|cx>nLA ztr{qgzPNO{>M^D`lJJLShmBF!zlP!trgR};B%>JsM2~BSgv(WR{x@(t?zr?7Ztk+; zt96C~&E~ZCM$AHr3y)p69!l%KqTGBXZ#n~%9(od(ElQWa1)4Vr(gam=Kf*;#MsG*b z^?T?U=Ck5gn{MPmq6tf1LY>7+k>)@^n5gE+@UJohbkm~L%!agaiSlMr+0B z_hvk^xIWFKFOmL>Wc@=-IMp(c7lpmYw*dJzPV*n~(}|$CO?mtgJx@XC4lR_0UP7fC zX@;a`v3T#&XRI)Yr`eglngtBn)dPUBXfAQkedtX7bA9tBYU{~p}WHdZ$cNU8d>G3ST zT3FpUPdw^F-(zOx!SJ1BaI4@i_V`8naM8VXgB9Xn?0vd7Z#=R46ef9_V(fEM9DZXs zk7zX*<2yWJjV6Z5P#DM^u4dwyVtcvA%Z~>f(1n*Nr!c4B2+F35(Y#$U8XZ_EcQu&> z1LJzpd%Eoeki+%&7U9C)-Jzc#cwA6_R3w3hwhX1^}-!u+W!4h6SY zeACQny8}(<=*yXJPr$=<^}V-Mn+Sb;on+@B9zbh?SwsH9^g|}R>|BJlt!@v=Uai2l zqyC6-llWH}lMNs42mj2^TB-}|J<^OnTKEIkWc85^cKj4q%E#bJ6gNIIvoRdg3l-~543-@hBubOq-taljL)`v&7{hC@mSeRB% z#{7(c88-i6pH{{^(KQftBAjqxS~5ygGrp_I2XSRgXP&=fJU-Y}jnlnwrv@>yUb81K z=4(0HnC=Aco|dparLQ>rgU(g-p|hsLOYmMzQ$Baf2vKgN!=DeYEA^M8!JkWUs$L5= zT+qWiCtLAM+GDgwSO@v_evGVc>>^uPe}tbq3n9{PG~a&pk;qG-J)aBZ$FJkO^0CwE06I_RKF+Gf$cb`-07B^AfGTB7^?H*n%^2^{wx zp<+g{dqCgmH^!X)-{D6=9@e+dh66{QGKw$nd^bU(YTx!yL##J+GIzf4SnD|a8^+}u zK~uYv5LVnmD-nr_}%@Q zHl(_)_-I~2*L#KPT6Et~wjosW=XEC3Y#ayF!?Ln<7>F%7*tznX*zcO?t#j@ycwKLe z*Nn%?Gqq139g(YzZWaqGBfC(~ycMRgA8?~~9j5M?4|V!T!rZ|kH(;k`{>u-V53_Hh zufu&%|2&kGfIoM3kd#Zd?8qjCH)y#m3%AD=!SXu8#S4?S!r$;9vaJ*F=+t(E$8n@% zbopY(bJxA)g4Z3bzulgD&wpz+Rr+S4;rNG(1?%e9teVlyJo3A|ZALj3@9+#ybfHM-o62i-fSx!*Jo zNZzY8V}VPdrr78zs(xtc`vWRle+B9VJbi2y{=9J-Y|FPw>NCRN0eJ4hMK8i}ENs08 zE3UPdR3}In4(Yx&+|u%yNb`SLtvh}F-a#t4&IwA#_A+vcJl*b!-Z(o zGm?6S;yi+4+fkHl_@X_g_l4fyDTm0iBUH!RW%EzD^xd~EKmFdBujrWwj>Ya&U*mAv zs6x5+!@UBE2aL0A$tON^!Vrt8V&N%EKH%9<)Ze_1^?I3u`VEFDJeFoZ0;I=Y+MhUY z4-lWg=F_K$qk6!<+wSuC#BoTtE|(bo!see^@Po0V_yq1D6X;y@j5`@{dxsf6sG&3b zC{L%i#}Y@hqdIE9=HG~wDu#4Ts)K_#t+f}v2x}oH{q98klaCcse`%*JxP{HqN64dY zzrla?a?w9xGX~NgXOvr*wqO`U4P6Dh#}DESd%nT22VH@BP^|jWic|e59KgxT8Y+B5 z;(9DQ_ghfzC=ZA5ocl_Ak~COhG$W3e$7pUu{jDJmfmk;;tk*0EiKi&;cab=W&QkT1 z(Z_o-;v`fU^sRvwqPy9H9OaIU$9h4xYZp*)F!348zoAdqaDX_qo^74G#&Lp-uPQ3@xMf!LOEURWE3rx+f^khP|uO z(fq$bLfzX&{Q^iiU!Rq9%-2pcO=m;hU*J8@hLU2(EqF)i^L+=BCcz0K{5c(@CG$<1 zVx#CaXi{%E_1|rn;$XxnS1^7mii-Gkm~uW1ySGU94oX^!#xvJpIXz#eu<+{SO4ivZ z6&~BZMvo;A3LboM5L9zjc)9effw&VelYO3S%}Ik`Y{?h4oizfB54L#y{w+M;-;w*@ zX#p4io)jB`&jRTw?zgswY~kl6a78n3AD7?g>=UQ%#!~e@#aVrhM;vSkuRM+-;R^N6MyTWJ09Fp>e6K-w(sd(M z-@v}nYq3q6OVIzE6c>G;VdKOOszzmqe-lu+MV~>v=Ct==tPwR0XxItZY&xD-r?UV_U&(c0hXv&VUhD5;NBtYhzfr+5wj>n) zSnXm;pDCSnt1)2GVQ+ch>1)jW^ioiLu#%vGq4dw3E3e-lhP7`G z0O}1KQnFj9m^JHPz$ku3^OwT>m$L}xmx|E$Z6#?(P1kmJH55P5T0X(oH+SMq zy(WUN5eOTB_8mg!rN@!#7v5cd4zFIOf=-(lP#jOaq{&SD13eqI!4Kc|D{drg%w;p8u^5HFEt zag}qY+yTN#jQRN+D270Mz-|QY2cM*uYW*^FdlDo41{c!ek!nc%*}4?zewb&!n`mBr zC2R7`q$;n$i+3qKBtqe;pnqx$&&+`Tadh4RJ#S$c7a>A~2$_*RBIX=LJfG+7J6gtlQMOx0$nw)W z(&4((XiI5Z>%|Z}tF`NH^>nP_;M4Od2y>VdbxK*m$+YxJXKh%+_m(1X6t4XpB|5FC zZ}^?6PMWP&Rb4JR{Afyxrr??DJ}=nt@0s7%ts6=fybp0g6 z)Pjm#o|KUhJdu#SXjRq1a@yYu@Yi*it<~UpVL4=Dx0*7#-vq93t(*jJOVk&~j?Mw^ zZ$qd}Ij~teJ74f&llSWVvXh2W!EOTQWmq21)gT2nRdsAmbDcxm!1@DtjmNPpzD9OY zJtwsmv!0cg=Wb2mCVO(~`?;5@6~l5vPqzy2H;y4hh0i|0=xy$>y8E z3GG%dohYg#H*AJi5`jwKhP1~+M z?{<)2FH!PqJ_#me=#DNur;z^k>62l40yB7do;c2i4MVo<P9jd z#&FkyM_2e{yTdB%M1q2s61-Qc4V5K0Nv`vLPesmNCwL&2^sY(Ux*wr616#-uF@5E- zd)fKH#%o3!5)aKNqx|+d%60x>9JDx|8pZp7T?Z1fr7BjyQ}>B*6Zp9-PN=l31~mxF zFQL`cxa%P*^r0qqTUSvUCYdvChZQxght&o)gsfOnZZWZzB$ z<=yK4@tn$d-V1WH$;;3iEBcDZ)~Un=p4#Y-|2|;7w-EMFO$mQ$!P=R1rGhSp37*LpI02#t_QHEXCoyBZU?89#G7EBoZQ_LY|AU67R*F$dr!Lsu)*yZC~4 z86t0SL&vM)t0*EdpHRJ8ofOyyZDs*jb7`_Jd`Y2aBs9ycfoL%y3E8I~8=W&60gvQ9 z72l9Uwsn-Jmtc4lAt%zd@mH-SIz@?lgUaX*V@Al_BPtX6iL(bLQlDMNRq?nOaKZw5 zk~f&+ve{WbS2`l#u?z+j`}#N;pFz#GW_*^!jDSmDKEO?fEmiUNV~x(ypqB#$ye}&! z+)kXq3tBqJrCqD(xWZ%U!N$(0*9f(8{${i0oCNB@KPh5qaaKFxlk#maX{gnzW->Q@Y#hlj#8kZSjHz68PR?W(LqI>GpdlfATwu z{kmZt^W0Ybc|3+tFY+O-b^hen2y=!X7WdH7VC384zqm3~uUS=ka661pr}QVYfX4iW zR+b&eK|afAfwP@1&}Wy)x>o-K{}|pY=GpGCzLet>w3xv61ihff_wvd3{yC*Xwo;<& zvI#6>oQz*sK5c%4y!(l#`rj98idWJe15R2zCvH;bcc!!9QsjP-JF!((KE zfQJw=smYmO?b`}#t4(b-&2gRQk9ev6MHHsw#Z@J~o54#Wa>f^A&em$9U#c=3WM*D& z^)}LvbE~cQGW#>QO(Juc`388rieVyi{M7e!FTFc+HJf<_G!QJino!SNA9G!3G>&1l z74s!Fz1j28d@>pUy_L{3!n}bU+DP-+q`!R^YhG_K`%L@W#L$+*&AC(HKAQIUpnAU6 z*663ng$i>zv2G{Wbg+$V{nd|V-c47Sy>a1KE6uy%N$}6?4Z02O(Wv{9y@m%ZG*TuGc#l+auq5QSoOuk)?X^_)9 zGauq=k0qm?>7ipoVfYYU$=kN}pr$#xSdewq#&;0}=2nG20M$H=d|(NF2F?k=ek08cX4TIc&avIR>#nAQSi_t zF>l=m>-8iwE;IF-KmbmB?He|Z{ythS;Q^&TA|a$FlH zT3I(U@`w2E&oLTxc_$ghF#PuUM{UBqR6_@Z;enY!p0Y)Y1n`R;m|-yWDxpQ)ar?8l z6MWCeIM%!?@XQr3cdv^jjYi3`V_a$O0E?+R!`)=G*zlNHXOoM;To%vmZiclLv<-fI zT0z%n^A9Uyc7Qdz!JXO`HQB;yb-tpY!2)|t*Sm$F4jEdQ&9I&7l^IPX7q*wC#>{?W zkJXQ5Pek3y1qrjH#8IVuC2EbVvkFMm497OOK;RyWVRLw<;b?0OyWE5t8+5p32YC3tJVN{5ZTott3B4-CEsQ;^N|a_gSgzD?u;3y z6*ZvnbMU<~0$Pn3>vY1bO`e(`C?*9T7MKeYd{LXM3w=|rTi05_TwG;GSQal})~+z^ zBlr~N7*VRwhrI$>)y&e|dgGB({;CN?_81>$>3ziWAPP_Zc4 z=T@d@yC97NM`hE|elYH~`b^bQ;nsV6Zs^6U~uFL)n@akP5 zoKxrK(PwJsqVdCade!i~!?%Kk>~pn+Y-EWctf}UdAn`Dgqq zxu1&jo-G4UhG^GLLA2s;LHYRlA2sCcKEk=_o460~Y`s(D{j8QbA3ha+NIki9O0 zr;fZK>RrKJU!E)RF8Fu+VeAcYVqrDe*X+o5x#|2l zH;7$dk8%Bvy&3yKQwvp)*k9eg#^MG`yOq_A7FSn|y$&XxomWNp&AqQ&wrwJ(ZjDrW zAf7GJ>=xH;XhR=HZsc3nO6#cuLwRNABD(#&mioWbYk1F^rFv6_Ck2iQ=Ihss=ySCS zlEU~LPkM|O5mQ$w(y%snxvGw-=zH%e4>~{a3YSefpQu(}T9ZUQiJH~323jfoWE;!3|6JtFYM$~y(_}tfcO6{{Sh6`cPi`vm&`c zW%7DmM;?7MTGw4!LS{=!Q+I*VO)9p49;usOC9c? zqqN{roPFqX>Qfc>=ydeg_20P3Cua*%!-$1)*V^3jsO%!p&%CwWDY|~gSKpalQ8{(P zHPefV(cxyp`R<(=^5~KVJS*=kGChDgC)ACuzb=DLrL7hhZ)TJ2XFGCEhpx2u6XtNo zv+E|0*QncC zGl_M!KAKd4M~yivhL80&^+MnNThF^%Y)w3|a5s6rT&mXnT7-Ug=l(NB(olHll^U)b z-?%#e9W2!MLbh^tqLgDtCRtB3{A*cwFoj*Zjg=*1hqFs3ZE@RJnR5)nbB=N^Q1cGe zVf?widAkeQ=MSX+PCLs1laKJU#5~&c%NY+=BhNnN>QSZWv0FbbxMvZ6I{T9wd)4Lb zo9cQ#o7|Sm{5iq%2DIbCaThtTl#9N4YZsxGm05$M;j^Tv3Ew(VeCa_~z3Z#DcC;I+ ze(7DZP0gHgO@=LhJ?f+nFJHrz{`?T#ZsA%pr;)<+=$wG2bQI4-#raX0KR(#+VExJE zzW6!RjYcQ0(uH!J;a6u|*|pPW3QSF8Xd|IsI4qx^F5LZ)o|V^zi|*?#6;+kheR4=z zl%25NIMQSEhp(?lpRj@S6@22@FAXA5Uf2TYZ{-W{u7InbK&Ypti;X;vxSpiwm%I%Xx0{ck>@9#oDa2gGZaom^~BGQ+P)E-9~* z8|>s6fiF~~T_zn|lfuS3;v4S~) zg8B0(0k0zHtvdO5FV&afa+0I7o;DcoKCQl&Jl725Jo^h8J(bQe)fIG|XAQK_z|vt< zvhzV`j|${Y z!m~kw2|6L3PK;&KCu{a}!_V323R~*QutfasbjM~j{K8~14CR~FK>37alM%hDOMD(V zW#*J=7fb0sP9Mk|kEP)u`f$oqTzK?Ot;&9d;14P!exAO6bEp`f(x00}9OJ%jt@WU? zRrG|S`}n={I)Z2Mo((NzKx$4l-dOc>Q99TtA0;oaSADO)1glmRrY7N&G`UD3AAfn5 zPv4tOuf{hf@FCva`C7zodTCgdy3TGcqq?TiktI2@>clY1-K&+4oLqE{*LF!#pHB~< zR`u#|c)(Aq>$d9@mKem5G5K}##gFK$*B@}`KAIk|gtq)jree9KLH9hk%J__JK~ldP5n=DXC2?q?Z2OMkeYL|@`1!8S^=AhaFc|qkm zkc(65A3zQmXxLhX6gUnJ59dQGYr#8B=BjA?P=SRQY@@R!R_5_zW{3|ZS%B{pbWDSR z2wtp@MU>OwhwoVp`=SQ$e!Ujf1E2fQvVWoUjk8&5eSOF^-{(nuI3Wa?VIA%dsLtPt zl#t*~d0^Ex^vfPvyQ3W4s+3)WTNS+A?4L&8P_VUb7AxgW^anRjP)9O#3_j$9SL;Nb=53i?;XN+c!y7l_$<)5_#gk&F@V}T_f+vpD{+#OB zpwVus-ZIfzerKRov`fJ^2*;PzSDo+qk?FGn{U>q_-yXEJ)e<`JInArp;|mnmqC3N1 zgm@oBg?AUH65E{!?8=6@{{?KLwz{d~wdIz|nf@iEm)3 zPRMEnWlo2i9JSTnFtUOkEBBV~mFuPr481e{#XWC!Cgc*){my^rxd-66uU>K9*=2mx zb2h#dP_B%eov$1viFjy50Xg@`X=~+QkFhV;Rj6IjtQ;_{un(I|1!m^ZgG)tF5VBR> z4yc#BIKLl!xB7r1bsNr7?@}*XcW=pLWHO$1tgjq2b|G(T9mnuc+4@m81_S9Wo2$^H zbGh`Kp&8WtK?JY;PzGv)xm1|zhIxobY)&|h4tqB zD{@Nver*~4M({pywbn_7R}r$6gl>s>4sWSdSiD-;BOf;nFDsop;@JSb_Nd?iYuRK3 z?7KSr)}5Y}ix9&?|ExT1#2?GY(YTik?--*+v#qv{?|6N-K~{K^ zZtk(e>NnS4_MU!HG+2iB$4~W=_iw%;FrVQ#>07@eZQGIxJ?|<}@7&?m3N|&rz8&6g zd1hi3*Y~N4`xTFURmf13(=Ho0&!0z}IUx-1-MzY#A64`wa4Yq_P>A#9ipGbs&`LG{1ng&99NCM5_CCOM#x{WWhfD z`OK5s(3c1+Sc(1Ie{qCeE4e7|Yh|S0Ny+mfd*^)dwditB6`EYQd5)7eR4%<$5< z?z}ZwUsggd_JLQs8J?niaci@Bc5(NWR^)lY_gYLp`qX#H%K6w!5xp^wL_gu>DS1># ze^)u9U@WcaRN7iSYMrS+lX(Srg(i$R!>Cuw?aJjj=YT|ktU$e&#UN-->OD4JdfnpcbYJC8GkGJ zNp!st!!sQ(pts&wKOf5{z-J11f{+_%nDabQXH6WL`NX|Tjxr+uOl4*|$gh+)R}pDE zCHhBs?lL16_Nt0y={_%!e)2r!%3GF^wHSOPZUrkm(=ofr_g>+LduhiU2awG-7^Y^_ zDD|1Ni00bwCX?50+^i>eu3s-g`|TCSBY&vJ7b;=?vGs`m)U96NPrg2DzcTq1`b`s# z?WWE*H`2?p4f)=*eFDBGOuo21HK&}G7-nVw%FGWow>WGD?BUVl0>RGf7_3C528#9Q%V19#)o~{`_i*F#zuR)^f}_?i<7Se{J(w-}M7@6E#d2>q(W=FVYF+!iOKbSF#pckb# z@cH^Lgwgd@kK59fyT=GRYaQZJ7z~r0F>g^Z&F0XC4(_~hn};-WgJ~nCF=lksHpGt8 z1DBhhr!yWzQwQdv4O@z8u%CQ=DT9WU#%$TAfx!0|M!|DJ@=Ihp4V~m3ZpE~XI-|~o zwGw6y2d1PMx|QK=g*q3=0%RC|b95QPUP$ngf(L+kYichlD$G2jp#2OnHR&k0Bsy!} zgP9^Bqh@JZ=MbXhUGmD!-)0kf(}H;l@ALh@_-5 z!~I|f4UUo}vvpuIj~iLMkVO6CcZSupe~pP^L7qk;TT&l|Or$#0+DCu!9GS5t?!Nq7V!i?iY9FGZJl;oMbmo8}bR>rtZlUSvRI zsx;vr+*5A$4W+;eV=d4-`kPo9YXuMKVECWTJn1gMs@Ah!(+K)*I4z6ajoumdH0O!DN%?b@ zHf$vVDt{BFA9)hAkio%*6=hFX-1DF4PqmxI3gliIc{V_dI1wSt`F|_2R-so+UsBPX z*(K(q#z%$GeSTuAv7K8Zj7bu@DzGlx^2?^>vySJq=4R`Z*b0&KF| zkK%U;3~cnx3k*WwBh1Gla9?_L@~?axv$UHGE@0F!GU5&#LkUa3)Z54oAr|PmfOawF z>R?%6F|)OD+kMH*o{`(>`t?Z6ZX060>!)L*>hMlq3uQ0ZhF5R1!MwXB=I8SbhY;qQ z%IFGaObXwtC+ZdKiFYhK+FC&O4Q(nP+|D5#BNyXdzk%GZNmc1T@D8~|x2FO{XVbx1 z#bwp&QIz}48BQyasRGkavsXeZv1#l&RuejCxiEoOSGJ`)8SW~1M2s998ceT0R>J*` zfqdM$o!@?c#Yv01>mtiOP~Xs#d^tVdTG4M2zB7dDuSl~Fc{i8dos6WHRq;;Bwl5QB zOm^URYCq$1WaV6=rPaA6m9e{_&JAlS+gz+ng&yaVNgqn;JPmr&4`kBr* z-$c-@^#yhDKELSo&pg`h?jqc0HJ_^c%%KApi+T;~(U#u(f50(S(r0`8rS|hj$ir>7 zi0|hd)ex@`{{0o#KJPxjZEl5gxQ~bYv}B{md1MPkUrtJFwMSEG`cHxL)+ufusKDWd zi6tKH;11#wHRyPf*LSSLo9rj>zOHA<*~3o1cxa<-3(V2@Gu@(nNxgUNGgZFa1NxmS z)N;6MF4;6>i9Atu4?ldh6X)i~*;jhfSBDzXtbeC2KY884q1x%^Sc!F@(-C9&?&0FH zt9vVU4Bx{Q{R+_ZQw#OU`%8Jz$Q-K1_fT$WKTD+MJZdRcDjRR_UW*0|7>8?qi|DB| zmHXBnNPiPv()Z!*rRVRb)MnBdg>~V!3-Df=TBp?Ho8wiRQG-N|UN-#j!apG@9pJ`+ z^EubBN*s0PxjF>@Zn@7uJ(o=MiG!rMlW{?+Y>du{Qh9 z!4D(VqRe01=HgB+d(z5Sck71y#q{l0@5H;4MHqXbLq2+Gr+y`*S&xSe!*pb4CtlxZ zm_|+L^*wPPzvo>p7mRnBU#x8HUF^PE+5aU z;rT9e<&i68gO>gEqpgd$UacN9wcT;ic2+g6pJ}URjNZeQ?3U{2JLUBG(x*8`r_yrI z@@TFxce>o#asi=7Sgt4`@AW&v>%JD?)n#z~)wyM|m(@$Z*}ahBU#?dfopXrTmyfg7 zC3AQNoMz$dw3y;eCrmpmO(o1F9 z;#z>hie^?{Ca{h&{-p~o8Q)9~>v2J;372W$zR#kv!?mpMk7}HcU2F==O=C;RuygLAV^VJr~y*A89$-x}h3BW~sQ>&y&+;!(&Sp)na|lm3+s_riqo~DN4-kCT0rj|K4J|2ocP)Le*8pP|7$rt_{?NE zzwQz)aI_r91U1k{INW?T_g$S=w_UJ^_op1-(;E}T*nA85b#x5bc&()3mu4&f31y_& z^Y1fOaJJGT6xK(LICF@~f&K2rdeHs&9~@$XXOu_dGG*LTGEhkRC#xHJ`rW3_l>FKBPvC%Npu`sn=wAp8Ia(X+S z)-Sg-`&}|BznqqLheDqb&R<8wSkQCdsNcI(x;`eEm(B~PRpV#rN_K_l^%8Uc_COld zpfnY279=}-_(CnSZ%}!?Bk9V%wftc6F;2)=!rH*sX)~uT9xh+|YtD@K06g z;hSBq-%*GQ#NIROMk_KsbiN+z)W`7?=xRR6Rf->0&K~o1WBU~4nZK$`E-+YrZZnrh zRlIHav?e=sz`N?l#h$Sm4GYJ!AP%`_aKCr0bkc_?PCu}KkL*ez?}VIc)$k1(eW~C3 zkCZl7K63EQ{nj^rd#sBxC&SC;Nc=nZUA#k|{Ra^vz4}+v zha5Ghq@EL>T_kPZz`I^%=jUkXM!c6P_8u2)cb#5_mE$yz?FtMa?zS5v z{nGNl-)8ZGIv+(`wl{2ZWfcD#+g3*186-v&txJD;rihR=Jv4ky)bd}bpeg*|Qdx~P z60<2mffESzf*PNUHCk!?08skLD;v&j0Kw!=SGFCQWkGDxYd_Zm4Gj{`mXCv?X zim12MsPMyKqTkaOV#DtaYG330gw^T1OCSBhfLu zIy~mBGByACePh+`MxZ>_dk)=hctO2-@JJZHMx8Qh-RM4LpEE{4=hg3Axy0^AW#kU~ z1IlQQ47twGE_z#N8Li5HPZit!op9WgQoEyQA^oh6o1LRSiFW2Qc=s(_hWMr|ZK;1! z9e(}?$1_YGTD!@Dy<~8hm{4jzRUYXitmO+47>MBie7i$o)y4U^TDvHW^YtjDcVurb zqMn?g!VUKc_zjKS`rP{JK|coXaIsTm=-q*K`cL9~GW`m6Fj{B)fsXI&$DJqH>YfEl z&>{CJ{9@ug9-EQErY7OVw6S|St>3Yfd>v1Sc4-$>r-OpQ1B_#lFRS(ef9#_{d)uf*7?*))Q@Ig%{h*-l-GEhqCkRoa8@(X_;k_+*M;en?Gw^vXVTz{WsH{ z_$;r<@ZH1}-w#m(cGl9+G<>cER zB^ur_j00{^XN%=DH7~%U+VIQF5^cEhpo5s@o=vRfpHGTP?t1vJ|6H<98tlD|5{ za?4M*2wW<_T?GHJ8V|DlqeTVWcLdFotA>}hj61xWolnky-ri#9AuqdkRvAV`9q9fm z|Kka#E*r0*JacLi^i4u<39KUatjme>DNkh;o-gujEfu=d+zPGY?d{Jq^vC!N;X8C` zzpH9-+#m8ge}eFNvZn7T0)LCP_4XP@;whguaPHDE1Rc`Lp4BjV06*An0h3VuubbI; zDp*kDXr<`Ds7_+WkW>P@gU>A1fIlC+(mS`%LF+3>vmV0b5k{*PC<!%chfT zj;W=+jl%KZ{Tx4N;lH&!_g@+t<^qpcz-jXR$a;7_X?uF|vn|*r6q+6;47-6?6iDUoC;M<@gAGoSsriL5Bs9PVEnd_ zqZjy!9zJYFemAy*d!Gu_3BeDAIj8pyLE8AtsdtCb-x(C;Sq0C|Izm_FV>zqS=uA@HpUAg;@rG#V4%3CLjr9vOyP&-+O zaUOU_**r#0v_Mm>+t#(>nw@bUS&p9USp9VtD+ClcMA;jBGt9sa{rYI{=F^nX*xj$Q zlZP+fdzrtb)$VQq@WT@ZOB3{&P_v?F>?(6!Y|b-J#8OJ0uuwr8W%tc>z`KrF=e5DM zo)!9SNvnRsFs$zX*Fm4C;vh{;jK1AdvODfci_dO2h}MklM<&CTue&Ad+_Byka7E%< zyM5X<+Ftnton`PcMd7;m!VPQ5Ayfa5I$xWxe>|aL5%AM|LT=Sx7R?aoUoyREa+ArD z&~G-G75+lVJnG@A%HX_I)u82L-k5NhrWejDtG}-xZF82>?v+=WEU3Zr+$m+2(GyGR z!0d9mt|EI^DNi?QR#xAN7u1QP3ZO=q=9fE5!8hyZqg^(mU(2bdyE|kt=t4Y0+T^K* zkG>QB$v^KkV6abfA;t-U`F&@OtmXKmxBYK9k4v$iqE*uM&NxKjuGi72$bnIpghpF}2&) ztlCC~;c!bQ1^uMWmCEqkAvxvza=sS-nX?J~u8_BkmkVTb{*at5ifpZC*oW?)l6

ad!i|ujd-eQ%X2&2 zmxJxpW;r-R((@387pjEhzis1#+x`}nfZz<@6nZ%nER9myUfzI#&m;eIh8|0~Sh9QN zpIfgHjC;6LUr2ER6Qv6w-|=xwXGi}SHDsHezl~F;>-}YRV)Uw*Mr;%GnJzVl0n1|M z$2->kg;g0;P#Yc5M{CED6O}K87X!Q;!X@&n;a_Uc4jMCDrskcyGdMp zzJ9%9?nkvZp_K1&Fb@z3q}k9;k(~3BR&W8tJ!bR-^{}gKV&@`W=UatGL_&^4!FTuo z`mEnyJ)0(lDtP(K?Y#XpgYO9FGe2gq_K)gQ+fs*xFW39gxN-fFvS16x|0HjmS&pXZBVIs7+Bf`6<6 zfcsFpkNJk&_E4=7vBgvapAU(>4lpCxIY!!E?DuI5?H_TiFm%a4)KBqhTnU&6j8%D8&} zUKnFmHH55S<&!3`c)|brh23Dm$xyG5P^5^C8*&!aPvWk{=(ZSrUxQva0?^ZcVdm10 ztmFGou}9!+S+O$_6TI#$!NgF(M`x?#v>-^j|Kt&R!a#|}JQm0Oy+G_eSb$&4sfv|T zJG5qnf>LlN=(q0TO)GMUfVUBn^(G);7JvS9Si3Xjc8Z6#jG+cF0E*%vva_NRk*kLZ zt!Z~h>aBWo!62`_!ycCAM(&Hin611e)XO#o_3uoU|3HKPfgA<`sNo?g4(?-z1!ITE zqJ5$TE)~$+y2o>9Ds%)Z3$TKw)-~rR zMo*0~fWcAzzgkp1&v)qhCccE%yIgFab&%%>94&REc!=Gaw1Hfdf@_ zYJhp1|MnE^7al;$ zu^5W}yzMF18h#I|KFa47g5JJA+MRn2n8aO#ie43D$4`*KWR9b*)c@6R zr|M(W!BX|0eMj3oX}WqVN#}&G$lSy|tu{EjS2BtLDAcNkyFTZN`(Mogql7PsB!>P( z{4stdm5DGV)$o>|vsjOnF$Tyh`(fYGB?)6PaDHbB_9{TcMl}jNy-D zvAII8!EXZ6z<*G7%hJGFL+aZ^@)-WPm}mvxF+=k(a}dEqi@FLC^)o2=?zYE1jn8TwV^aNruGN0pdJ3L zsP`gDP#L4cL(t+iY1PvA=$?ceMOzq1WwO3Qt)o7`;hSz1PdpiSdV}M_;og_`2ZPD< zBKJp77jp2{P#(C_;Xd!~(x_&1^N&$H%yu~U!rQR(cCNu*s{X^fvX#}dbSGr9t)pL+ z29OaZ+J{JV911DO*4@TY-TQ-}vv|F$<~b>=CXhN4xFaSpcKvVR`4RUhwhh?tDFeWunj5Y6@t`7S(^oghk+Ww=1_#A=WAEfiW z=R0HCFc9sDpC7{%Bmu_Ne#K5L{3+__K*{5Dy?^;B$=)|$hp+aTO?*=og$kuW%PkcR zi5lSSP=4aGSt(T8!E%-OuzkWuE#f=W_7m9LM$lHU&9*mHp;BeUZwv0tyzKmIjv1~2 zK7y{pFBP-7-5Va!*@SOh@ABTyF?ilL+L z<-9ge?6_Pl9-3{z`H*?}evy65sm49v+l*s(qLvzmS|EF}(&XsFu(8B(uggy3Qsn3t z3EQvJewSIjJAlOmpYIT@$JH@`ZBBi&)1kit#@g8~{~mxLBcEKumm5C9Q`o3s-F){ubI@88-{3w=YWs!vB!oCESPA`6K4&@ib&*(Pss$c>;9aSiS9P!MG+m zBxe0m^oX5C;84m8>Q%iJTyMuW4vvmfg)iuV=gu!{g)RS$wybq1}+$P@40))XT z6UN6nopQqSA??J`qyl>Tlavi)Z!HkRm z8g;+Rebv?dV!RAy(=WcF4-7aoJSij`rs?NTkaJIdZ_$oW(TF5cN@UIbRROMc#R8vS z$a+f5ZtlNW@2CDxn|7n!=VRC~GB7EpAw?DLn6>1fpKj1&S(-3E}ve_Av-YRiiu zX0z-G6!AV#du))DQzywZV8dtaEHqptrGqY*R~)fzVpwOFb#mPuj{Q$#Cc@tNKz-Bi zFf03BHE%FBNuT<|%~Y&>SuWJvb=8b!dGsORSb)Q_%+TcdB~9yr*?*82Dl1!feDho$|i51=qUAN(31!w!>5m z7caB((rNAw=N{hY*<;63izE-{dQREQom_RiN(`3`n;Nkx2|5E{Fo3Z--)>y=Piy$0 z%P9^PU#tGpV84*NJ5jFmGcvK@4~4HsI{P_GBS#o|tUyJP!ozKRhCK$egWO^%h!|p9(4wzMerjDyYmVnruq7rQdJ5qNNN1^7I$69nr^j9 zi^tA(tBW`<404{#pRQ->n}A#E%$U7!vIZcR^wn#+966(}Y1V4Yjt<@R^~o;o5p3E3 zIJ{>E;c_#(oUUE#u{CJr1S)NmcsUvUsP3BGZ^5bFWM*LajDCLP*r!Kasseo8zr=6EP8&x-jO*eG0P z3l5A6d(Epufpv1d*xEXR90(m~KahCw>SJx)hqM8maxEWs7vRdA_@&r*wqV*amhmPK zF@^leb#=A($NU54Dc)QE;ee5MC2n#wcDg*`3i9#;dT%(!EOt&Dy#Zt$z3ScR^)P2% z(@(chlGFA8U5k8!a!a3Y!|+B?m><05mYq7CRH%)^HMz4oFTNRXW4L;WhY2EC51u9o}K4)uBcB7!3E#G zQfN75=PIOU83?`<(WYv3ULU&37lE;fHEnmI@J3s=kA_SdY*?j)BU!M*2dwa4Wm=3X z8snun@h}mu7VNIov%%b4{&al)`(C!kI>W*LbDX-vRG0sc@6``tYP@2 z8Y7rdNagbnVm1$ydZ)R%!tdWTgj+ud1!(g4#SM0$kHnI2NLLwX;3wPm4(D-S8jFiQ z%%}3XEJ_>nd8PUfTQc4}}mP|e>4K>HBV zU$Cs}v#p}x`OVnOM(YYclotO#+UnxUgBdlJ;0$-lH4yWQ!YYO{*jhApB(eW+jv$`r zhWOpY4nzk$ zQy60?!TfawN-_TjUFQMT)YA2P6r>{xM-UJML6j~nNJ#)es(?}rU6J0qgeD?QK$PBt z^d>FzDoPP4AynxoLg>90xf9R#-TS`ieV#n0FqzqF%}kQLe)C^zdo5#v;7KX(1u#-x zKEc$KH#wAr>T+*ifF@G6bvkUm$CzE&o(FYY6J_N#;MVDWY)nd@-dniQlkkIwgl0L- zK28!wlzVQP5ng0hROr{GIMz8M2VFp8&Sc@Ry)DHD=N@ukfefyF?Pfo zlUj@Fie=4X`m&$9DDAPOuC09mj)E|EwW@b#W2t`+gPN9s@;IID(Xz4 zz^CEbtuS=jU_n%$+3a#=oVYlOshe7mmRrA2fTQBxK^Yr$vKL{mDd9^3s!YG$`SDH_ zWtoSrsH#DEz7yQY(#Y39FN90oZW?&2>)rDS)|sZ;jon@rbQciv?&;a_ zlG|X6y{T+g+a*YsaXufLc>zJPnPblKcxSRFF?=OOi;lR{Q!moyClh&>dCiTD)K z6=hOEx1VEW=IEZi@N{&-H!N2oEN2GM^N}2#kWdmrO}d=hj&ySp`JnfWQ0*UPZ;NWM_w#m&FPr6IA1n)Ev;^6EH$;=yR)RIX?UkJ}xdkrv zt-}fM4sJ4_x0YdbnqOo6qoSZ|clOwxmd0^tyeiCfI=pn)=p?-b|FzI)cAW?M&g)U@ zjI-P>9QEO=RhuOiy>z&oXTDddqO zCp;KS$2wQeBiz(z;>0dFjx3=()3k-CEKh$oBRcgk7koXd68!GHV;fg#+z07!T za!gmu6f*R=WopX`mh%Equ#B>JLCg1bt56I5!&$f2Y-iiT+ZK`~^!$GJn;i=aRG$@5 zQ|2qC4@KQ(3S`2n2h^Lvl~TnM}=!Z33MN&u?^q0z?f-y ztExQ`*wyIExY?n?ZfN=7pIC7%AL+;Ca3~TUp^8ShL{B~k)kc)&qs3+nGP<2SX9ZuR zZZF}P<7fvSvA*Q_)q86@T;h#%1nLZaKZ&|Dl8A0o;+iZjtshgk?BUhQ!Eqq)H~BsD z&!Ss^eih&H_7cH$$h|D}(17A5`NT(pOWaLOeGO+^3Cm>UH24l|df@J`waG@cWj4WT=3fV`YChwsn+ATHpFehl$I z4J_g?`$_n0Gx00X;`i{A=vf-^n;(ea14r!xbps>LHoqkXE!2MgGG`n9?RN#&FMlx* zb+-BKTW~#+1moXR{qI`*zkh!0R>UDN)a3U+sOo>hsy}$^_)nm+ZR)VctEN0VFT`mx zOlKL?|11e5u+j0>|GWCX!+umz#o2I7_~PiS?Y~{)2bYSF{!?@9;0#v$WUkQt`QH#U z^Z(cHaZQ}EE)PazC|3P``M>P=ADg5cfH{dqnP5l@G zaeME9)~DaB_>Z-a2iRKte9J6uvJ362DF3$>!0*ZapA~pJgyUC3)|Y1g`f=*LF)jGQ z5Zq~g1-Bd@3{h`@GGXr6AVJVP8}cCu^ZJUlMSG6{POes0$zyr?%fH<2gb=9dBT?P2 zuY7&<>)69`XVKL~wiyeeokRgpJtxs^xp$)mr#e`lX3?#)V?FZ1j9@`ns}AZ-L1CbP ztqq@vEJ20+RN|6{uYd%lsLol42*v?pp%>rmU!Z>x_kuqcQ`14xujeM=1p z2K>^~$Gv&A$_Qe(;p6*W>i~2F0-5zdma}}3`L4&38vq??@P0X$!!A1AI&H5vPL=R7 zSw~KT+E+DIvlH{RuL1+rhsDJ)o3jNLp^<2nB%w?o4BiAI_oO)Q!B?$4B0lI=#9q9; ze~pUjp1W(q=kc+mDjVnf7DeER)36k*r|#mxa*cBjB=ic`y*@)aB;_?10IF7N>I8?k ztVZii3tl$>*o3)Y7dKpFfZkbIb;jHX$h)!Es{KJ%<|xLiaW9p3ZgR35-i`&}>pj2~ z37$t?38>cAp)79SG#QzHS7o9TLVm@-(P#gBIW*?0nm`Z(b}@nqBw zX!Z)e)n(c7MD7|>G<{Y@!d7fhQop=3QD(x8Rl^UTWxhcJ*f3FV54nAy53~a~9rb#O z-a>)8=u@xNe}Afv-E|wiJn~dm=;sW_8EW|2fILgQ&Ltg`-s(G~NWy^}uGuKqN}}-6 z!p}Sn9L9MoVsDqPmlNv|v@*d!SD0{j*mYst52VXJJ$~cN{3TqsB3}a#qX_gkk*fhf zcK3Je@=R7&pq?&XjUY;TFjnt#!d{Em$+y0oXBKY7Hsj|&n1%5wf9<();X+t#t%a-EoUQ9v37svkjhSzv2J8#Nm3yX7hEJxluu zL+drV6FsiNBpH5|T7yZ~kMlwwV?^yV}X*CBD zhZ##M!CgA3wJ#~2B(>q!psC^qt10=Qux`KJsZW5d{42oW`oJ-Lb?Ti!o z62zC0!Y`Xd%gi(OrjA4?o+3{HF!R~)AN7e^OA6<{7J{aFCGl=qLDxf0nP8Oy1KHTs; zmY~e3*`leYLF=Cu4;$X`2PKEIdpRzY>Wbm-z_)3U!iJyd1m${Zk={~GC%iDU&c?x@ z4x-nPhM`?8zVOC4p2kJ=aVIew`(y5x7>c||-m*@*9~Z6&#@ zOx%%ph^e(E0T?=2Sf>@sgqF0F<=KGC^cJ`)+fu!Iy1&kE4Ex55e&|NYMIw}#L({XXN8C@N+XT{l?Ps>O=}^L57(q3t85LU#mQ{Wt zYTe55#iN(RKTs}i!m;O0n&vz`Z>uB~&44GJ)?x0`gn^Tg+#A~pZ)bXXQodm^?A00H zopHLv^|zOWTt~SnoVQy{B6_b}d>YPpM8v3m2`anZbX3$MWb@5}mCEWL^~oon*3o%_ z;Hp#Cy}uCjwwhC%OA`p4+M61V2`obkd>eS&c)Clr+7yw);j21)M;;e<6GT;=iYCA~ zBRSg4O3t~^X+V#Ap+YY67S#IT{PXMIlO~aU)f-Vb0)J7vJ`T3I?#*^m$FGl$j9#f# zaMG)?qr8f|iI-KXAh{e7;!6)UR0k!@K?FzT+QJf~j{c#F`C4MQzCU$6n?*)G;gvE4 zvI(d~osE>C)&$zKBb^DXMB!#g;Yc^7QmOrCd-CBpZa(0U(71n$p9ryq3~y=1Fwa2U z-M_!kvztN5oeyQRhqa$1H_OOj`b@bry?!l6?O6;hH(3Do&rXLPoS#T0Pi&Hc(L)jE zR~{)CzmskA-G2Vsn@LeQme(D(V`m~JvplwFfU)Q8A~iKL9i-`7_bTO{`V*8H`fP~K zZ2h>|3C3Obud;0ZbQ7qWru!MMN!@-_e>MEat{Q8p^!eE0hEab)N+!in8a}O!1HhMm zdYT1zxLZFUfBTG@L`pHK`0`J5J9XCN^OKuH%X0aasI?O;1@*_MT_sk<#?Q94c@jiA zIQ@|O>UhokiinX3GAQZjQAYaxr5J*G66*U&u4<8SEo2E_*y-!pXlr;Q_t_F_=A*a3 zHzN)xLmblMqi=Qn{$Rs>Nl z$1Fs4tJC}VyLg*9Al*B@4PSH*REw}3oE`n(euYI*m4?EnQ+p3Df$hG}uLa+5pR6qT z)2%_@c*yS+FjFGAXfligHnARgVl<&v#fLo^@AHWmgGgIyER?Wab1ex7atyOFMPw6 zncc~_EOZ%ZN|?GX?lOOgvS%;NW&tO3ruZqGVA;Gp`WleE#4lfuUT(!2y)qw9azOk1mZ4ZBSf2?JGCIWxUO~ z4(b(@r~13-QubTwD?hRD86CHl)Mw*L##%mCq-6ey*^4X32>8#2-MbOfj(G{N{-o_(e7o%Xx*4GzHIhH{B}0^olRl6f$R>OVjTw_*kX&CQxsy1k`U{F(ZMz;GH2$c*1Hra49|W&W$pq zCe@kY=$6!DF%1ia+rwMuV(1_O1!^ztoa{TpE{12(?|r^;Fw9Zz6)Q$UMxn66edS64@#I?tD&_yd(Nu1st>dk6Bkom=?y6ZQ)+G zxoTq6$cB@>^?ow49lx(}-V&u9L4%yYCw8)DfF^>s3u}7KIU=`NGrNa-Il>{Ghou9k zM5gZ1!sD0e1E@e%4xmXM$jOq=35ImSkm^Gjs8FU4ncdmJ=&y)!jM>4kx-6={FJo3` z3EXe>nVFu?v#=%-9dZh3KLjUfLuSq;f#7R{!8Z1&oyr}+RMWZgW8KU|2*U@U+BE-% zT{18*Fy3gfe;{Kv&MM<^L@cc8HKY?!KEM7>Y1HN`xQTik0V~dy_beY7uSEzg>D0M_ zJIFhi6;llsnvhtd67BvB(aYlYjS`Lc^F@4Q`hTDX=C^Ld)AngODYf?$ak7X%Uh1(g z&~)=0M=>QA&teMZw}}A!q$XP7o1R5N{}Q8AT(qCMy)khXO1ve-btL6HcmI#j8W;L2 zTmc=`t_;4o_n+_p_un6PSYO~B?wCZ_15wDIeH9FF9SwH;`u^xB44}!Ou>2c7pn3UA z$1GgOtSkOc0S5c8z8@<9q&lp)5x(|U9~D+`9an^3ZJzBYMT%411f2UU@!tdfJHAvR z3t(S~d7S_Jb9U0zaqNNP^A+vCu16meU>*I2Deqg(PHGyEVgVV|9l}4>31Er0fjAlw z$VGYo4;n!MV21U>R}_Dl769S=a3JI!&nKd@%?`{!afUCUPx`mZImqCkUtqrKUHr4n z9bh4TtJ=;Eh+7puhA74*I0=ExUa9}B^=DyVR z^zTQ)0|&CUhM*FEEW_`L9K>g>_?IY6vjA9>$|8dxB_qq&!t(V3fkFyA0Kg6)r?}0z zC}B|l90TIxFUg{U28}KuJx0}1I>m;rl?g1+qiaPA&cPfRMd*YF-UBQFm!n*3v(U6N zF$NhUG%YBJj^nMVLbqXUmYt&;0c}ba*;zTfhyua9RxNC6tIL}>{H94;SPKPeA?l0s zN^PbxZ4cU+aH~-NN8_8e$nF7D85*3oUfAMw=E>NW7kH+y0+YWg^4)p8f@kMi^A5K+ zRfXqReUEGsWLUGv(N%Kdb3c!KDYl?$0t}QFB=3T{Yt(H@K{~xjGH)m@}$tn z^k->enE$p{kBv>1Hu63yY9lWv)wRthoq|Mi!QMK|y11kil?CV#+qOBGo*^Bslf<$(?Zb_yK1zCbBFmTkQ$#Kr#3ofVU#d zY#|2u!)6*)Og!G73nkjrs1ZVYEH17}-nQC@ZuDy<(fwvS-b4N1kU~WbhClD?`P1j$ zB;7q~pfc<5c%YGMd~R{7n09AzB8XijbB0x% zSO3Ej^wN;6OJ$s-ksGtc5@3T6!d)4rLbHLNK?v8nZjObkH zliGotHMMfH6@gmDk~35eM?nrYLX zNhNq9M{(M^FX48~hBFK3H*!>xs`?@%PvqJokPek22X8J8G#V(+Y`u=LPF_)L_ZW|F z9zbE;h0MlfaXr9odGGFu_=3DoY6Rf4HYk9Y21;cBI@!EZ0Hj)9#15~EVtuH za6nGS$Vb6*Pqq)Q2*k5*fJs2=7g_sj_2^u=&4H4+Csld!F81o-X2F`p!2AZT2Nrn} zck$X?1$>|5@Qc?$0vXD?PTOVA!2K#Rj@#h74WE$EE3@u+DWg&}t6aD zJhhzxV$l3=o<|>sZ@=Q%IsHyM%20zI$=()6dhU`O&W-)r9zMR0n+F@8An##wJPLY= zd##gAQ_n|&^-hS6?|D{tI~myJ>4h44eW%ml+xn$Dk5klnFFro>>)ab#`>eTqvV`P& z$gWDDyFO;W?bEn^H3=vdtcUYeiyQks4}idH%VfF6Yu|bEHi(UyaVem`RcWIIP{0uj z2Yf%G%bl^S@h8nbxsMu18#WG=_0cDmyU9&2g{iS(3qdC^i@PVAOP8)-W@c%TS^70j z>P^>uvKhtQS`UhhAwc^iyS6U=Bgy^<(*@sp$^ULTkz^*Yx8zC2g843-b^qvIM9yvB zUtR;nr&6PBCw9YG>RFrHzkGk$ref&?vDJx!!x0RE!E)jB0N*H-y>qvD_bK2T7HPRu zxzc0GZPZBaz1`%VyElAdaTqp&o2&VEO*rDPfCUMp1Y)Tkhy0u%@ZOn4OoALeXF9;w z?jwBPVftH6=uV%k}4EVScax2Nmg@PlQaDqo$hl&nCMt;&7vH8TNT=L z_;#s=4<`}AZHu?k_~hVxp2Vt#H$rmvj#lAAP&R-;#MV(*zV-E$_x7H?Z9C&Z*~g-v zaieeR`6dc`GEp3RTuKqAoweD**ik;L$d3tEt$Fg<%-l2JV4`>Lh%e85tx?GG^&801 z654ydxQV6|A|BjMxH`JbN;w7cREO^DaM#8!si#T#45>Fk#AaJM#738KRRUT=bn90Y_D`u^6U;)H#| zcACh>wIkLK@R=fc#5hbb$`nJZ1#t zJaz-&a@|hDwR~i)>EtB<+Ycm9wn1M1_oD8}ku2(vR93{c($KC!Tw(GH5yXH!6I53* z=_GqEr`&bPEu$NhM<1^Uc4-I6NFP@8sF|rM|dty@tlaV34J1 zQVyqp)|#}txp=ngjhJK1CUz`H9;+q@^;xWEL~qaYNuIvA!Mc}J{rSlS=;4sw*u!C7 z#&AfTO9C*Pnj%|`(L`t;L;z$!10${#INAFiW9rz@r4}d8ee8?RlI zgwI!OT2t8SRi&Bgqn#wJxVsY}KWN9FjCT3^hIe&eF{@Bos)~Sm(OR4|ykXG`ORSre1qRHn)K6hI2jewyp=8Z6zXFB_|N)Nc#_!IXut1H~Q~XX6gdRIl}} z!({0B&QxIGG|11St7j5jOnuH0u-sR}$}>yw%0LM=T&xq+Q>IWE(khYAy;l1Vum0Q2 z@L0r<0_5H1ct^_7O=X=;sSfBvwq4nu?+#-SLzjqej;J3ZZ?Ip3{YbU>u~8G1hfPs} zDKl&Oypxd~bH+wWa_M~i$Mdpuv3jdh@^~AJi|t$R3U7+I!%=-A{>#;7{!*PBHxtZg=E7ccQrh6*W)Gm(#g`_9MHO3IJK7a1v_yMv zD~8eI>u{hMLHc!!*VOX2?@X8e@dWs6##iQc|76Ml%7%BuG}ah@UZ648@B=BCDFUXP zp0;gz^WdRL9z25++`pD8R4d-3)1wNp9?EUoDbqGB;e{zlKA@_rvYmSWT|f)V+|=cz zip26VNRTZ?2dLRx zi#eT%LLErLeuLg@d}s9dZOEk@wK^hEwkP_dUW!UXn+HQ~A1nnU)&N+P4dIO8+FV`; z1NiczDv(rm$>QTQMGqHi3_ziz5E)qn1(M5Ij}w(L{9&ragktaGn`2U)&w)GDsX7G* zq{ z5reSnpw?u�qjoA$qvX?jfA4|?&+u01%xbe`!Z|MQ;4BG0xlSxl#g@!DJy46_eH8ZkA56d5{fO0%-mO{s z==@P`HACtYb@>S1uLzgR8ff{58nsJNMpqWUHD`2N9gGJb-Mw}9vYgO!aCj57E*liL zkq1<2QAu`dnFZZeOBgEWx8}OQrac?lA*a}2!s5(s#LoN&PVk*|Q`vLYnXvZm3s7kF z45*l%V}8-(=i@veSpW9`KM_Q8ih`f$&IVWxG9XMp2`n;Gj7e#SUW47srWAMvlqC0xXr_VpFlSyO(;da8gYxPJ-? zJsbQpui$XP$}o$>pY#I!r5E6Y+Kcj!&q4}6A&?BZEu`mq`|jC56&KUK5CF!+_{mRB*jhLYbz<1KQ`$hj*1+Xwu_ts3KTixd%L&{<+_Cp)) zd6Z&fHvP4tgN2W}_v=k@Dit`&>^-IA^Rayi^HndLS>OOkLA-J;9h}%;7W&PTfkqh< z4fOVkAUZUO9ZOy@?MV)So$N#DC7!W$r*6gJYEGete@WN5uVp#T+`@oDRVlU*`=R(n zP;9_!vfQHp)euID9@_vZRj}}F&0L1EXL}pPdmV!Yfikd7)1$NhJKe!aAndHE{r1?%6E^iShLcC*NH<%+ml6#yO zgaf@}58VtF2Z|IK#mY^Q?Y*FsmwtD`lJERf5-v$iBd1%lbZ=Y3H)jir?*CRwy|Df+ zqX%L=SSVLD<&(7G(Ve)1i;nO%5|$3YDX-Qtbfkd+RFY0;3p?@uH?~IXaJrbsskUJU z3x-8;%;9i1^T}tq{(m#Gj!Wf1lnRboDH+9O%cBp&II+_m0JzPT`TftXg`q7TTP;rV~dD!$y$y3%T3wEP`|Y`5GS1$cN}(OlQgb>Dqnp2sEeZ74J?pP@^ohk4u^&FD_i z*GQ~E-R|8nF3nuLhT|X3zw2*_ff+wNn�hdtUQ9TM7WAX`gfn{|veCoRm5@@iDH4 z%#sbeR+RK#6DL4^6|P#_Y2zt7k5OafP60Eq4svI9P_e+g1Yo9-GPg zcgGU;TD@i{(@v%9$9p_)#KYKlpt`A7@**6sx!-keGf29^n4|iEUg)LpJ`d@rO)NKj zeui?2Uo0z;=|pWP!+lktG*%;LqGE)mJLw^G$higE4c?ahfNlY$Hf5fot`-sNX1tc_ z&1Ab9T?ZK}x0JCONK4)Dhysq_M1KdfL2|>reHlg`3!cVPX89mli2I9kznFlr2P1C zJl#wvp`?Lg#4q`XTkt68lf{E?j=*P7m}4MtYJwH$e=_9v4!*W9W~&VMXQ+iRb#L6H zu&uP4&~@Re!T?rDvCn>6KbN`t`*eJuL$p7B0i~qZ^^=aLscrsjB3Z$DNf7&t`{Omk z^I@GT56dksQ(uz!$ai`8rruEDz=tt>xsLOr2%%~X zLmypvmB9@c=r@UgJgSMW8=9>c2@Y32-aGq|JL z>W-)@}M7DI6p|DNa8lB|}&b zFYI%_OeKgiozCaS%X^f;37AT1rt{@I=2%daF*VhDj${VKwratR6Qckk*aG9zRt(NF z{2-uJq#xp)stmG{$X+e<1@ud%`o0Eywx>f$%rC`rZQ&DRY(}!eO;{aeZN7G%EKmqO z!8r0-62Izu;t?XQ6~(TWF=!1mcSl9y4}$J{4PA8sBl=kl=Bf&|V4TH z>;l;I19jIT0=sfeO41UX!ILONjx9P#d9)RZvSZ|C4vw$|r)eQSwm4IawL!Tql@leX zHQPB{QKm)5re|$oqUF|Ab+Zq0>jPn53=O{NkVywrfJtOjZ_zNceedG`LEi3doR8mCNB}e?7PUVL8?1BS|l707|^#6(>WhXHS&|Ii^>MC0as7Tfd~Kl z%9y*Xp0i%V7ZS@G7p=gM?;f|%DY^_P>_$pyAMJj)%WqxzdY`8}^ExDAN<>-oVZmi7 z`~KPh#prkudlp7erDjNk0sk`ZKr?r`VVTvVT;gD9&s=I44-8kWj!p3$$X`mAosrw&kh~StHCHM1**szU zhRJ5$xP7036fF`t-6Es7e+8GzL=UMCAl>jZ-%UPndB~EdmKqfVw~uW6As0a4vT;x< ze_i01hKI&>2r8NhyA?*uqyGh*nM1fjOu&f)Y{<;KjHyBK^>V$$SKYN(o-xHp6`^el zk~>Oeeaj=h32Cv`=SQvOk<<>(u}S{S6C0u-gk0T+{@95*Q!-hj7( zBX||nNLpJH8nv$&g~_mK<4r*|Ni(Vk72ESft4*OxGcQt^xy1QB;r&*c9Ox;(FybOU-K-D5T@Eba> zlRmPrfJu2=AkO$fzV6n!j+PgCcY{|!VYqVwHSk%}JWNeok`$q%`Ca35kiYaX(ttfh z(6U;XaG2wVVB8}nB^4z2Fi|NCZHVFzJ6GF5QD~~G{quK|5OYU!KW^?Oh;wnmoJ|;d zo+c1j3a{Fdz&g4-RghUIF8l{P$eVgR&?7H!ss!?jF63eV*+>wl!|un?^Kx|ke2a*c zO%t?HGkNiI=b%h6vbVbG<`+Ix+w5d4U1U*cJ9c@l;lrFWO=DFJjE!IRv-yV3F{pGa z6Mfk%@&nJBEf~FYo|Fig-Xo8R;aWwt_l2P)tRGB1p}$wsjwQ$JRUUC?{ZnABh3tPM z8%Vr0samn;Hg}40XL8WJqQfA`z1&x~Jz2+WjK9EaEZtfZ!Y+66bim?47`xR2?vRGi zr_Gb|Dzz^!$X<9i>ABWWwI;D8pf@=YFp+{cv(yeKMaF}v`Q(Db6v`~vk?RHmm8uXSMdL6 zs*C1)F=NPIfV~%x>`Xx|J87c{!(a*tdofKPnKjB4FEW*T*vV|SA5GpX5AY8dFt?o) zO`_Qo^x`dqfO5H0eA?c;xiD-8A@ULWyEOGcs5eyJYm&&|v$34G5LG>ygJjZyzJ9T- zcX7PI*kOG~#$jip_xu_-b8W}dC^zTvmOG{UrAxr(FfmO>)=jd;84C01S+9BVGhCya zcg`C&y*Qe_7F7J{y1v`HzyMk+Ai1ov{>Fe3PNV^ zdH#ePxw2y+ZVTb%jc6mWserfebMMSA_nWN| z${6oNvkzNyc5;{rf$vQCF&}UImjnOnPYVH<0F!k3$HZqXz3Yne!FAcg>}(6pU%ziR zKH?#428onq&RTja1^94_QB+OgYzxliOBrC;2+ZpzrDxVqVfxu+;KQ?ba~@|~elPY@ z+_0sFGThhydnxcpo(TdUP;_^?qyE>c|8(`|RWRe&`245Re@u-hZd*Y0Eo1&RrOa4h z4iq~`r~KfoAKAbK5m5c>uS+Nz59}1J-hAwGcHT;uKjPJ1xIg;megS{^LIDOvQ&cEi zKkGe^!r;%JTYrB0tlu~o59`k+^J z0d-ry`mM`yueMoN{7q2VgS3(eKSjJMZf6PdrQU<==k$A-X2{ZD0jp_eG7AdZ6~&`h zNeDu+k~EjaaaHeB9*}BHj_V^<83m$<4t`4847kbHif;_6w|itkusN&&4(bVPa7CT75a(%NrRb@0cU(w*cHffxsm9I%vUu?Em(eUJRo3ZB| z2UE1N0hCW*^Wb=^Cl>$EP!76=IlO4Vsgr-(BD6bMSbnIuYU3cE%kxVyLXuCnoUQ!% z>i$-bEeAL$90DRB&zH8^g^#7_A-OqwTi4)KJ~AcHt1r;c19^lt55}E!zr16WPe#|d z2Oa6Zzb9%rp6geDbsf#Fc#HrxJI`H%S38sjh>9L9&-L^qi*A`#!J1^^KnmK|1Qq%k zfg*z{7|vzvmALA?kLOu&S^ezKuTPIoR&b4DolH9w`m#TMXr#G8Fw$d8p+OEv5j<$)jyEF?{I=uU|Iy%|tEuSmO@hH8zT} z*I?HS(c$GM5ASu7fTzV&t~8r+YZvA&6?Dgo>S-61Z`4>1tClGpbSG6k^oCv2ZwsVY zQp}6IX>fh&lexa&%4ntA@b2pvaF`xI>lQ6sFkEZ?jRilY45)F`55v^WMsaF`3kd2( zy5Wr0zWWvMa+CKOy8Fqw-TASoR;h?KaXj7pelwkK7N`{Qim07YzSauc^zNY#JfN_J zqgYk|lR=5`xrM$O|D_qd1q`EjWsC}gNYVnvwcOXJc>$B(ZAUU}hG}d*2|`r%Wr&yC z&-}ozuH-2n8O`t3?(>1UC^10Aq<1U^&A@dRFe{o7la<|;>olV_JcCPVSlRNCb;O&9 z2=^6sWm}pxzZG%K2<;QArT~auMxthc?rc2zJ3{EtBF-=!QQmK63$mhYHIpXtMhmUh zk2LGZ4ASlo=Nfh&gjH+JM~c`d@A_*d4O)AKyn}f9S)uB*s+{QS{UBse`e6|lB{C%H zeuSNCkyY7|K9rThAYc0twD!#+CUBH{n@b-Qk}ThaJ6z95-Mc!E3C8j~|&G?^~VQ zCAZN{wM0Zkwq)OFQZ4{d%^>!E=x6*}0Tul_$Eb+TcI0-{+O1#4Kei;9jzg&*O``&W zN2J9Au$EE2RUd1q>Pt!S>2L!G{52luPUV3eB z&~A2DQL8K-GiYcuk*Tz^%W>VamtfAZQ}}t@osdO7RNc&!vqiczjgdTD^*!RhsR5 z$vbxc_4Tf{w3NmZq8h27`z%vyTeWqSO^1hiWX?`l#;V#%Vz0_ec!}0{YXU25;IS3o zD3(#Te;#=w7Aj5x!j29Z$ecIZPmda?I|9iWEd}ltBoJ!x$Ip;8RPsA~e7dmJQ?NOL z#ecS3O+qvGb9c;cXu9nYoq)?L#r8{nDb_`e;|CHNt$Lvs(03%R11-P(6V+c1o%RNJ zj4(a&QOkkc#&L&}!OilgD!D?@Bn#gx*#E`O& zc3I&udgmwkc&2z2!7^lQO6xZcw*)lT8s25JgZC%d*BbUVGDtjoRjKlc2r{6xaTGf> zG4J(jXE=e~R94)kCAhT5Jk)s8-mYHm^%WjY|3GpPx-LJzj;aYnl-EY9)Pd3W#_8>U%Go|32Y9$%|~p<~0Q-f?at!PQBSJLT%derIh_# zRY7sBK0d#xY1)wBq0W1Fv*=JdDG3G9N=x!)Pxbznb&g`!bdY%f5@bS6M%9P1l2CvHa`iF&&`46UiPj4dg%|sj{!1CJij zZdOEjbDA4@u*dhBTPf%%Z~F{+$^DNFt10JG`tC)v5tI&Lkp(cy%c18cuU^~y+NqXg zh1xq^LEC-g^Qy#TpogWOe@I5t;Mem%OdqyJ*H-(-K#qowR$Zbr3b*Hn=#$$r(T(}t zy5#F^Y>@4Z_swc|`pnh_E4c|xrBSHf41Pyqf7D@~S&-lT_hj4i?o&SZU6lqiZh_y6 zz3~&@G8LjrpUugjhu2ab_BH6mmE+IH$LLo-WPzjHJ}8sSp zj&lUBctRg_g5MTb)(T8%Wjue}+e^i$_Ql$uk_5Ds{n+k`-O&41V#BUrj{F;>W{Qsq zer~+@yj(gfmf!!?TqwC}$>!VuDgZryhk1U*3le`b_yq%f?fl0l(W$NKY8UR-m-D9& zelqVm|7m`E5Mis=Mb#C@SDeHjRIK<|BJ3l1IxD?71H{HbkX}agDa|D@ic8_#UTvY| z{wfZ(Ne6}g(Y?g-P`$7Ln2dPf)oOn_DVlK4x4JbH|62yxgfVgdxG+ZXd2e0?}BEbG72BVbjgs9+OI}eDizy? zj-=gYHT%xf&zJT1AgtKXxaIod!F2OFU(2*%8_KhSG=2Z2Uu)lSVYc0iv>5cI0ZOL3 z^NS<*{^5bAzl!dM6UQ%@VjM3Bx(vnf=vfMUy1r{Q^bkpzhZdV@`D9N1Nk*@0+`OTZ z=lt;0IxRwbKm|h|2gU z8nWic+FEZ4T;{S{m+kKOmxR$B)1c^$DW#k19X&1>`If{#@lOb=Ho7A-e9udUj{ghc ztPE6-oL=h1Wp7%qljcw6UT{kE{jT1Mt(y0ZI*?ifWCLzq_1>t_Z9*&xe*aRzMvu{V zuZu{`8-!dFsW-F!7!6D(UsL+dKJAXp<&RvsM%0B`1+@j{q0&w3bVkV|xhN44n{kUD zf|FSnHfbHu#AL_~Vd?F`v=R6Lc?y9MwJcQ6!p%bP$uzZ)I0JI409`R&(O}>T3B3Tz zpr~yAYCdR22Dd^TXeA9MX!aJT@#u2*;_+)pxK)f0_!U{?%gZDU@=4xJ>6`9knJuT( z)Ev9qyUe{k9}LmcIU;bQmuP%b>0OrsNy?SC9&f(EeEm97ZD;qhPFqtvXSxg3t$9}+69sf`d~@4m;?K75hK%WbqUk10j|NIJj!UXD;-2LC!Os-%0{ zVyCLy3Y86Rh&EEtghnB4Ct>HF?|=G;2(Nf-ffBtJQIv-$_vsc*mk+XbGmKDmCKO|K zX89AxK#A8ydetHj*>Qi3O>eP(#5)Y3bEG^Qv40uF1?7W-%Gk3(`EMj4APpkr2OobF zogMWp0|_7t4AwEQcYi|?&jLWC41eEda&}Z^S{!qr)nYgJ(+Lo@6XRn2OE9KCfP^R# z3@F-kapBzApd7~tUQhr&Vtcob^lanL$2c_dE#IX1EE@UkUKnt~jy!tq4=ZzUR^|*| z?NmNXMsO4H)&MtYyk+How2HpMnVFgX_`%tU{*5GTaKNIHb&lkZ3swS4n8tQK_x*J` zlO=!~NQof{>az~Eu;J)dneOA;f4!iF6j(nEyp$ZaGh4>*F>a>02MIgx{mnxMPGFA@ z=fc+b*@eG^4}6dCQ!>7M)`x!?)=Mw}{EsNO3C>!YEQa$zY*#|?Y|C%TV}qOV%0>UI zv-`p?3Wq48-Xz~T>%(uCf**ll+bvp49-p<;p$xz;pE89liL))gL1-y%*vG{zu@PtS zP+0=-VT3$>?yq~si5$$h^8>BYw6nv$h2pjaQ&G9n-=_5K7I7w>R76BXT)oH4&yRlo z;6W?zjEg}%57rB0G!DBL8O8nqVYTtv`uERHpahL5V%wsm`x?l;+&TENFpK+Fa31<# zd$fn4f7lJ=`2+MWR8zS%+j(pMblMLeandq(l57jRu8E}7JJ5evoP_^$ha4Q%HyMt} zuPlvxF_+LJ17&7qykPT_M9q9Jm)Av`hERXQJ{)=IRaqvOlH z>!a=4q8aY;izmxVqK0eOyL%7Hd!4+X8GIgJa#!Fb#?r$%WpzkYhq)~mli0KVNyr|2 zEF_nq{K|*L8oX6=4Ww3+y3g&5)*nLR*2oIAH*P6f*Qb`Ed$8;5ua+nuCQ;NOa^=Ui z-9HG|?eF)iB|#azd*~1@HrA-o`~6q1R*aO0_$lmKuDcy-EnB17e<+U@p%#y%lP2u1 zT6ge5#k%nh*X$ald#r|JC%u-S3F7Y3^TM20xMJgAYxV4WO~=_`*Py)Ln8>}3pV%iz zc)YIPYra+HxtD63ByzGVW;wFSy)bbEO@NvkmNYoce*Gf9*2KZru$dipRTP5jwrN3>1-VhS=ACw-P_2jesg)x7>Dnt{TGqx)Z zri&h{S|hIX7>tQi#Hy|gVBJkY4n^}WsJqMQPvmohui+R3bPxexTK&k<1$c==Y5m@0 zdztIlu5pr%zE6nD(rCxvR&l>M;h4;&xy~m>UzGV8Sp0+D5ChAKn+^-P4Qj^+w_9KL2mUNMRcz3`kn`lfX$n3cK$^Rkky`!4S+P+~CMHCQ}p$n)eh;%7RNk9=0Q4plJ zsPryHN(dkzSm**G0to_&bfouy^cs5aE%X*b2=(2WdG7bSotfwT=UdC=Vy(1u_St)% zbN$-2^-HDFbTaWd>HYOsRMe~G+fC(O&`*y6NIA07pOtUu+&9;~9O8!e5*8RVHZbzTMqi6@j5>HKxQrm0EQd2gk&QK zVOh4WZv5%7r>5(Y^b&g(7Oy|?Ri*K~-JIKPSPm4;C3_?$4uCuFt(2cze&ZOgeVSW8 z2@^aV5fIOvrNYCAOaSZ+ngb2~T)*r=KT(F*$V)otM$U#(n|)jr`QsJH5z43R#JKdj z_YIr=Y^tK6c|Y7-*fGHqi%a43CSl5vRcrz|RXd*$JM|~%K}6C3c}K^{5^kx_#^0_8 z;AW-E9V77=dY(gOaBfxm=5IVFeD?Hw=sjs=w{No-M5>K@5V-V8tzYG~J9k)JA&l+w zc&y@!=l0)6`mfehzsY%9y5)7C3mUpNGM?Pf~!R> ze%+q1sK1|XqUbfGR<$7g0Qv+a;OFO4iapP(lbu zNALr;Scpvf;al26Lm-A*LljX?k9iHOYLsdy=wLvW>tkGete#$uF}uf}&#^YjvsDM? z6BjGrGv?NmsCL>5OA9?{Gu`M(@F~+u*U3o7A5`%h7F-*{nY+lZ4vWtPV3FZE#;XD| zReRjYZ`s|@zK60O)sq{V_m@N|65a?}%Jz0S?eyDt*7zxWolj8=u^!g()Hl{bH3qYB zf-AGQR}S-L@NJPps7@VS02S@fG%rQ=fP!%A$GE% zv^*{QdsV~bw{hg1NmP!l>kd((ablmwJI%2HHu>%n6^H$;TX{=S$%;MyreuyeKakuSB`FUjE(qvgV5}g1q;kKkCpNKWeYBL1c?LgV z#LH9w#q{L|NCfRXp1|yvz&VcQ7D#xnv_s|9e9hZ|5;k9zV9xlt?yp{K0}inwqK0t- zU*X%Yy*bkPrrq7R17Yy-v8s7x*-Ql<41jk`-%lbbAH0}qUCq611 zGQL&!AZ!i4hs)C)IsXF-_?G}ibwTvX)zTmP0jjC-J2oX8A!P2!irbM)UzOOb@^h-T zf-^ghXbbc}Y2B?-)@%kT%TOIs@{VQ9Zhw)8D-rRJr;vTQu`wy->l`X! zO$a5)`k;t%^2%=SS0gs!+yKkk3mU;n)`iiu%1r%AMaD(>j-+J09D6%nqp|`D4ED<| z^@a4OjYYS#$n{F@3NQ{DmcUa`IZGNnQA6D5n1iVruKg`a_nn)g*@ZY0&X9&xm4>~!8;PFpJRL;$*H>5Hd; zPK(1EaTzs>8x1GSDP7lx^>S7h1(~hYyC~x)Pg9@-P6d1sPgj|1VHmr7Hi&}mjms&U zRLgMJXy4E0%>gCU9oQnlekZr74!hKs@v!e^v=PK%h?OI}aYaPT@B30Bd(erK2*q4V z0y~Au$^@ril4gc>{d=>8mwIA*!a*dOqSW^xpHB9@2>+P=td3XgZN!_ldb@^&)@C($ z5|=VmaUDT-l9gY_(Df(IBOn4p8NS->Ye&V%{|D92%!hLiHOgISV#!c|u=vr-ms6tY z4OV6Shv=GD)a1;;fzNJ{)}U97#1;F#E;li$esieJjQ0l#rx-ROV!Twq=&)k|K3;x% z)PlMw6PA;W8+ws@_U6=0JPW=8;o+W0xkE<@`H>$r4>BvRe9-Q6bFQU$H8v^1hw=nl zeo$4$Cl(^YwRPU!WfRaXqvGe!Ur`}$nSFCirU1a3>ZKMxU(KF^u;&-(-c)|NIs9T? za>*mim(CDG{#zyaHY|#om!9YExKL)8jjnB<$U~}PAK;^teLR$W9T-ob7d-BcwV9|%tap$Ai|#R7%3dC{+f;}FTT!o`#OVZ#v|GR#$;+KH3CGJh`7Tc^ zcVA2Pir#Moor{}Z02qbhp_tX^6Lh>*=jjptlhVK|MqglY1(74le2o&-X*y? zsDJikmzw6~+`~R|{ICSbpA8etf{Oui1Hy`-pjHi3wfffXx;Ch)ozW3QCd~*P(1Mf& z7jc0#H(bJ&;SaZCldZa_xIBzp#G=`r@Ztm?9^Q;~7K(mrI&newk*v zZVk~kslB1GT;ddWr><50XVYRmp4W1z1prCBOvB1Z;$SZ5D(o}U-^@a&_5x0bm3qma z!{~A0!|gu)0zHJsx5n#Z2^0?B+Fw+kALj*>%=usb6a8>8F?T+@{^c%a%TkP-zfv0+ zK@3sU&HEMh+-dNfowOdmCE6sZFjgYt=kHWX` z;G`xUNj>reEBW`D?s6bWJt7}nll0$`ivRm}yauUi)i{um{Cm|J4=25{C-v+f#r$7| zus=A`pX#*yr~mQ6Tci)(+LjUd`zwDI!oj3MxZ&oT`hOO}k4c5_<9O`9DstXwkw(tC zJ=gH>Blq$nsSy4shZFnzD}Rouh#L6X&pB5a!~Z^Vc`>9yxHYE#?}PT|89JRO6~d}U zKU4nxwc!G!LYOgA?B5Gva?-^o=L=Eb{u?MkrUFjEpgTq2@3Z1h@Fbq3+K4ywY zxG{r^J9gsg=?ZILb@yk2_{5VOPnvKfE`q^y+8*vj#=_ z#Pgr@Q0;&3&hf8@;X;zsoM+Dd+rc*po}>=9U>eFCHah{ym{r|zq|QazPRsm5%M)I` zs02==C#Pfa_X!Yx|0MM%)crv{C=7wV{kmOW_vK3^Hb>Guk3@%JYdWqJiQlNf7dWc0{jLmne?Um#x?QU z>%J9G!>2%<>I2yVkM*8Ngyf6hm62khLG$A5gcLx{-*R7|)GKrG$gKo!%qHND7xL=E za&jME0*~4pM_8EI5lBtILOg5hGpWj5!W%-^S|+c+T0UNH^=s9wa2FwBl4+$-Jck=k zfsmv6Rla+d1{Svl0Xh%Yr9ss%;x^L=;|h1H+{z?{peFh-$hE^g+E0K6CKXv6CC40Y z6H`E)VdIwDetvc2K`UdsY3HPnS=aRSnjO&H;o9;3dNN#=_TzdLigZLrI|Jored&GK zND$S>V4H6@qU{78HOx8_h0Las>cMA#&Ibd*eKsEPpB-8ECe_Y_!#P#GV%G#}@HFz% z#3HwyN*~3OWq?{<@&F}9&CbP(7iUS;Z48n7Ck@ z-$%q!lB(xFYYY&MqXTs8R#@B{2W|>5y0R0m%58_r+KCl*q;kF_TiVrPzF zkNEKH6IR`zKr?iGkbAs6#<&|O5_S^mx3t$|MhhWp84mny&2~N|%AkidKkyKJ{Il4% zg`1m{9^?g?zBV4oH*co_W8km@ze_=V>845JW8t5KeZcFkOnNuqq*t%cTv$~MSyI99 z!J8&)YE8|ZP@?u%wJsQY%vrIduM!+*Y?p||=t}La`Sw)1i+B%tI5hR9Vi{j2x@2jo z6>lVNXx<(lx#k@^w9y@O>^#RN?RwAryo%cdn|+D?xmk>m%|KRkIzvm#l9o-1ZRxE% z+c|wA{NDV;+gHo|pO$mVyNNO;4J$5~xL6vDO`K`lLXIieo<*H?%l3^^=j7uU2X^*e zv2~9LC9_!%r17Flr&zJL?Zy0C@IglcsVmkTpO<$C2QK~3dZAd|T#5Yk;lgzkjM*H$eZV{RZ}n?e{@!tH)Yrhy3N zgT*kP@S`PMR2rHzjZ%*YKOS{g;V82#sQb?v!Hpf$G=Dr%4O}RH77az;ivit_TL8Je zhB)$QDIm$rj-i|H4t9GqSwx&NFUhzI56;iU81diW8TaAS&OTqH^YxwwF=(gn2}btt zKn1K%cf(g7fx#pVLxNYyYa~2@LJ8F9OJd~=eA=f+_+Z(Y2lyN}<>MoH!?9Crn&e@J z`<%B{2V%34g!Ktd+pXy!!KmRv8>vHU;#PFcPRm06R1JC}d9RO7)NZHC2QoQ)8@5zv z_qE-1yP@9;YT=5Li4YHpahKP&YR^fk|+@P!W9bi$RVt&ag9efH72*u zcbXOByf%D26SU9Uxfbu><(U@lw8x>+4li5gPr+S)+3WN=V;y@H$}+kA=J!=x%h&A+hR|bfF`35s{rIzh7L#C#7s6&} zOX}`>Q*9&98g1Ed(yW?T=1+i9-lvqHmIpRkY<>U-jX>GsWt@?$WtA^SNX_WNx)elz zq=~T=;&YVT^AeEUIE=Kxa4}kL=KP=M>Vcb&9x(P$6feJfYl;n&y4lhWk*n$Q@}!4U z3?wxG^tkIF$_zXR^Pa(D6EHI*uG?+gF=J)4u+_?y5$@9Z$>QvImlZ|lg{U8zX-VZG zj`9J5?j92dBg>}SgPYubvMso`2HwjY_hqf+#*;)ssnDRQQ8uouquQdP!a=?30wFF?xA(FXr!pR=i%|+=5ptd%em8+GUxe$Wvj=eY zVZCnlz7@1s4hTC>>1Sdi?#@y+Id|AN;@@5YtG zYh<=pE-~~z+d{4F`r|zMZP{u#{bpFRXm=|Hp1o|+vq<2jVe5@N%Sv}&{)S64O19?x zjbzP&1o2kIgQ}^$jDngxb+033eyPh~&-%nIb(!LBXJ1BbK=SS`LU0md-K)boo4ZK_ zID)mmSrBe0iM%^SSspOH?{O^T!uOcObdTT@2`5n&Z3Pz&aw_-EG07=4z9Ot`kcot_ ziFc~%JjD~G^{g&`y7D?ywfn)YOs(PuT0dZ9RqBRmH?8a!m$H{a$vxu6{$qUFfk9>^ zs+}x>7?3(eU1>YIsgwS)Tt&KXV<@Rd3+>I$e#)hYY9mMYg5amG;m5sXR&>HX;#rOBkBDBDs%$uy<_K824r9^!cjUjUL+0iQW%8JM4`z|D(EAZu*zPhX->` z-)2|G*U2*svMaMwgZJmp3<%-_qKOc>uX@aC*7GpGjO6V z)BAkpA1Ml?eVJkSEy<-kH_|w0B5k*lpX;+~2?n)ZWz$vnU(|KvW%VhMg zylEYNVcqX1>~XjP#XVt}5`!}azhI{X1Sn|a5{0t$Pa(!lg{!j=#Jq0YmP!JK5~<7Y zw)jkiTJAL{$|;1ORivmsd!1`5Qk`WxBs#mqC*7s6=-d%SD(^RfGktdQqq(|M*U}Zl*Cc%7g4h0JKL_5QuFHYkCdMdyE zne8qj!}|5)8Pwx~AsprthzChim25uk{xUTDG`*L%8;<6dV&NsK8cws)IRe^o{lp`> z{0$cW*$k}=@IYme*#07kR?K-X09fRkjbk9HyEb|2n;^(;euHG?EQnHC#3zG`x^jEO}Kp$EQ!b;vn=a$QS zi6r?RG0eE|1hjdeS*A%g2!gR=SALCZVIt(GLat^LB7Ff z&u%(l#(Z3$piF08sJMIPLvnj%<1^BGzwu{`9@TF#lu_vK(`V&Wkz&)%cH3Iv=vrAtmj>0~ZIe|MLDKpbfTQ`_yoAJ(w{ z#~uR$X`iA5du2vD^p#%J=ksjRv|n&v?S2LnG0ZRh^1k|-JIb0cYWY!A zZc>Zl^b}YsUcTtF2!41!5s1kI)A5;pT~%J5T9<<_`sHsZ=I^j5-1v6y+{rOU|EOoQ z2|A}x8`*hu3hIp;fYWn+gV_^5B~!l1a4~3~1u8FGE83Qa46}sHl-j9dc2AxfzHWJY zh-kP3#HjvV)ww??MnW%Xq41^b`OxKHdYNK@`y9K$m#N3qU7?NT&lsh$&d>&8s^wX@ z26i}1PzqiR&eBBN&PUMNu+`A6+0(B!t{|YMv(C*L*SC#__|8z9!+oDmT@5HePUROV zYwQMR#lp7IFahTno}dV6n5q2p>L+bbRA~#k^8VvAE{ynhQ_1uRwgXj#nd7;7(wnht z!tak1rtSr7mu9UB^PvjaDb%T8Rk^z77zA%v_N?{6l|Ae%+Tq5l(o9`8X9kj@G1nh5 zx(z%JM0Ci>u}|mW^dvc;@y(uhX!tv+HVo^^TdqEN^Yk+12W&`vwbfwltItaNHYF$l zmyi{l2g-&osqL%E6+DZVCg5bPAxuBBELK&L+D$KnpAe$94hw2s&SnpSHzev8ObO)J zJ-S)*s?tYxul#G_rA^74yytc2lAhkSRVZFt%x!qG8*!eddH^CW;lrcMkTwH%+|5{ClT8+Y5$l z-${SA$u|y@=;g&r2KgTROn(X<1qx~S4qFzxdd=lIFRYDOq}3SMgNF)jD4_EHL|d+SBoaE6s3VXOHqm*Jk`aCVg>Sqbp z#h4Mg*?T@)Hm@^>3vV~kSIR1HuZ>S^M~%@pwCuc^g4sP3Ymv z5qno>Q(nW>fi{5U@Ngn)k8Q}H*cLDN`5Q9h_5zMEtvEK)egBrh_gB;BmGdjkqy$s+ zG;*{EO>jPJdhOAB;@^(#G8w>(T-5w{e;c%LIo4C5oN^8sH8@pPnPdFR3WU!cbjJoX zT{IULFC5Z#T5TAERZH!4{?Mj}VZObEzp^!(hPLgYs#hxF*;I{HlWKv59o=kmA2$fT z#A1XDoG9r4?1 zdG{xGYj6}Ew9T(JJ#Q({foBpufrO7O6_v-ze`HOhEAhmeMy!b$Rn2v%V%#2`xk3m}-W z>I_xON^N7+xd=gW-MN3&)|>#?4Ni&#Y5mtfV2IrG$5PskKYdnw)ZxLw;{7AhT=cg8 zuu{sQBWOkUH0*X|r-i9!H}ztZHx%Wf*N3!b#_e&ov=*DVIf1lHTVOWhn3AhTt~rf{EOOM-{Dz|a35s3=_#nXf z;;bX9e%qepgvn0j_UL`IgO^OI5*J^3$GYwgW@`JFxJ^@UnK<4D`@vza<|Vn5@=IH+~82$0G^{S?~Y#`xL26J4b)pG@N(F0J33FnCr>Eu;CQ|B~d+KGuqT zo-5kQuC|QcPHnx!WI36Z;1RTaDitaq;Tbxv2L`yZeZf5!<|74W7@-du;nm+|Vp4z|=X%fF4KD$mL0tFvA9N_G!fP{TOC~EEfS1NMavHdkB|=2Q^?fpmt+gC zEAnGk+)-|U^lI)a_gUKa4-p{077_fds~+bO)LYKVa%0k2Po;H{XS65Ycdh+vwUywa z{R3x*?0*s0`HtQLK=r6bSAlzhP)+8bv@>zt zn8CO!tF25%#;7pWWCCVoXW`He*Y%C`^D(_JpigsNe1DF)5)x9e|Hcz8+Bqfs`uqQD z0-2H~(Co0u&~GF}^_vq(_E!8>nGNY0i=eYxm@WJ%Pg5KwydU@d9gumC1-rtK3u0x6 zt`TQF=&%tA^KhvvWpY&q`8^H0Fb1Yn??~IZ^;yx*+l#N6qImi~&Ocmb?rv+Cv>>3yhl)8PyUfS%@?Jln)7uEbWt!4UG*f0n6qLhQL z&mY5D1rv%KGgKyGE3a8qhE?y0|3nhzwyj(|4)Sx@h(JcGwNT2b?PcvI7C$r}NaYqKyrtF2)amY|3<=AMX(mf?*#njsRGQ zG>Vo(hIz#~VOSi52IplXi*4UinqN_*`lCq*Sm|P7fsIuslUgL&-FDxVscF95!1Ty3 zz-dzMFR&A|`?G;yBHJmTSPbei5OKZRtr%^<0^!@DSMgXM&X@emMLkn|*cb5(mAQuu zrbWgh&sfT?U8_?WppuqE#R$MISa+~bd~YY_dF`K%%^~0biUb|_q5e2sGkbQ)BhnWG z2owaGN2kT>V}=kRBkLLG3>+q+ z(v#3K>Olo?s;>%BH??jmXMGVjgn%A=yo1IZAt{Fa$>n=8N+8yV5zq=9((ATEIryMI zu2Y6LC0@^gd#IYQD-F5rd6n1ceOoQA?V|2IZ@N0xAjlgMoC_b)59a4 z+r*Ii^;ob!);T1`+|<$l{nD?LoWJd$IMVa>Vx?9*nQz(bvKPllZc1CRji)caQJ$uD zrat@7r9ue&n10TYy>L_WDTByUcX}{$P%^@%NN%>xo;Nr*N+8k$VAD*XB1HrJb)1bPBP391bm(NT1sw z_PpWc-3jT4;#W6Z>dWE^DlqPJYi|2kCo-qisXg)m__e#e`&z_uLJLB1SiDXohAS)M zaLnCqLO=n6TJTnFm2)yFt1Q_w1VEZzo_lNw!CB)pkK}aN`=S`tBfoxZb9!`!8Ot8C z@6LfkVpxuI(IdSw4cE2NgkLE_`onxb#XZQbddBqIlq&>%Ut=jUZ!fz~f5pepMV-kD z${z{D#EWV|CmsztnMW#Wm#9YgJA{h6hC@b+OKVC<|ku((_9 zT=Pp|jh)`Qz;T8`Bjx3D;IZ8(ePcX=wURzl*09tk0d-p*R$j+O4>!KWDBUyq1i}Cx zbR{X6W-sY1JPi~CQjJJlAtVTHTnK+mnkB#@>p~E938{SxIG_Em?mu62VC^WNOal6v zf|g#{8Cb0hTf`dhx*Chx7(zIqh&!K6$h~A|INqq1k~k($g%5vRSfK0V)!{C>@~aJ; zyK(;#jt-Alcv6DFFZV@gd{b#LT!;c+DVRBIJJO$x)LGiEf3ev6Ga=IeZCn_ypg3%6 z$7}LKiRq^1lf-Ypa?z8lwLF&lJTe38f2J0{060bE(mb6vyBtOApb#co>dn)u*LQ(} zLp$eY=S_7?90_=@;0OU0n)(-aIZd#Fz+i&T+bI3A)YbPGzi_8?b~+pUVh>QE1R|fO zDGg+*k1gr|CkjVja(Vf_$NLD5kRpB;%U+x@6-0l1PCtFFEs&({NcRrV4NBk`>#mOm z_OxN!Zyz*tCP`w%1&zEc<2mV1bLaq@hI^&*EF$c{(I0|4YImFpBnesr&3KHqOqd?X?i-dEPQ& z*pHild+)_@?o{$$vSw*FYaWY?&Yqm(aDK|AG&Pv+=~yPD=P%;H7(q5={0f!L5pmKh zyXLm-?gNHxSF@UXE%H*9Z@>3>V1HozwSx8IolF>==*EqK36r=mQ?<3~7Mb_#3jPY{ z9GCOqhK=cuJ=-hxUme2&IT3-(3}uNTD~A$X|tXNfm^17l=NRyqb2sr79e9f zUW#-=E_Y8pK4f{RsjiPmuh z%~iU=VldEUA(Ms>OiF11xmNMf6t`r$Kg9<1NP(mCi;YVcYCd{|evkJ>cCcJ<#q}@u zrtun;yF>vYs;k|_Ou|V1$LXl!ZfL{jpEG88>YU`{w$od6rG(c3iEnG-#;**2r#gQIK^HrSOMCl}5tJLS2kSCx{HV~}i~ zs+6g41j(5w5&M;$+PN9b^T{25KPzX zJ+hgb-jP{tu$+vCp<;XKw+Z$BKIj3wP!%f0i`s{tn4pnzM2Wujs>*Qm!eXpLPUiY1 z8_RCH=>SLV_zl>H>wV|Y;&=+`hN{*6xH+5H%WizyU@$8qM`VhsNbbN&Om;uB55^8XYC8xY7HI7;M0r0CI z-J?6ijTN`1lDN?x#83ZjCX|oHdgc4AxpyMK#tV$j_k?eNZ+*_~`-EDza(!G@5@<)a zRKjzPCb38_!>VKNJlxRLU6;OSh~bkLB+LBWyVT<(7ca`1Z8{Z|y7%JDH;fFPGxhi@ zQsATrGqt`vIX!u=MbKHc67!?YN!Tnsp4iA4CM(RJd>Og7p6q91z8hom$nRndzT|G&_S!n1f}Dr}W`m<- z{iVkAr$@p=ig5w`L9Y+Ma`&H;XL{6cnMv-umw^F25vjznCC4 z%um4pddBi6wF`EvJj3=L$%lEbu`F~Q)&T-`7vHAeR?EHYa^x|xzGsbEWVO(+p zXo^$l#bEF$v}#yUxA`VVbzLLKENu==nDQ48xj+~BHnGO!rRjoS^M{04*aITwk35%wP|(Lk|=jWfZ#$*)zGp>DoM8Iv%fqhb-~TQhgX2Y;04`1@Bo{j z`o0Rb;HPZsK=5uRQN2k5LCwkXZ__2p1rUB>C%2gChIPcHr6GIbOm4Zr27HL)G^%+1 zZZDYZ$=B9WDM!k6QW6v}>qLSG{&Mv`#t^c$?ku1(wVBlq{v!Yv!rHGxH*!kUm0wXH zW}v}2;_6a^Z;f_!z>`_QdMOIbdYNQ?Vhiij+Ppur9`uX;tv zxf|M0tQLxm9VdD4{0%%i6}sNt<*SbKiJxjKN z=#N#qF(L)zorne&v!%gE;6#S zo{EeEp(Jd5R&%1}swQ=uigO0RszOHTQ6G{npxjtork@CbOcQ7SISGfeoa|Jd%#TCW zGl2!umyW#q@=j)V>=j1;4^I?b3`O7rS2w6UDZ`&@?n$vJKwvdJjHKDV*t?*9ACFR$D*65a6DrpSY?U8o4W8Q99R^P>olB@MF;UvhgvF=Depm!n6}Nm=e)B{wx(`RJ z=5N+SccrD~}NUEE%6qmtPv#+1VHx1HNPV>J{l-+pj|)%i0)yE6s_xgQ79t1m5s0CHbBkU*QD_AV8(;YtPEZVg~h{OhI>OBJ6`V z#a`uC6}>)lm8d)4+mQ=FYAknH3Wh|(tnv+L{7mj#A08<$Ku;qic7KUfC8}$P^Ml9H zq;?5c){1D=_imi#CL>E--$EC=ru-l0$%iSjov}LuoXxB<+m^lJV$wlXu$_f2aU)pu z!!Y1c9`AW44dleCVf{I523D?x_*S2NTIMAXKl7=Npat}J)H3-cRKH^Ya-lroGR(1U zv0~av+6qT_(atlA4v9Ja6LJG04_GO?KAUoDa5@kgiHJwvIcQ$IlcP)c54a8k;QCXY zk{*iLgvXF7BFptT{Eon^(W%Z&F@ez!|H%SK+j$VJsJeLgkiYsTy^Wscsl(< zj!R;kTu2zxHmfP|H;n0W>#2`QYLP7O^%J5(4j70u+mKtr^k`qC>+H}~msriR-UvGf z+37fx`WgY}f!292j;g_w>=3DyevFY-U5iNelJpXM#aijJ1W6j$osbJ_lf&$SEXeXj zWzo*$tnUe7f(d|kutJGY3`}pmsa+K48WD|fyA-*)jM>FsSF%dF;ZXu$ps+9$rhWi?|}If>$I~PT8?>Z$b|9}Vc_5W?=C^Ad ze)^;`+2$W-l(JM?xDV&Z7nISyYXT`uWx?Y}h=(kfRC5_>a|Nph_Id)&s{3f1`$h^2 zY+Jg{-I~a@P*qgfN&@!dYAQ&e{(f4^K5kEw>`o{&$12^IsZ9QZFXQ5D&k>CaT?5Mj zdJWSg8`0OSy(`-@HtF{4-?*DnlIy!PiFNUJ*SG1X6IY$B_|-Ea^OSvvC*af6F5>=K zlsVsG_C$%RP#>vOWQ@fmSi8k6?QCQ&2S}n2TXBonAt(BM=pU^@Y(-`Gqrq7#)dLzA zOxOMt>5Z}5%|2*usgP{nNR-o9=Co9E|^hj(uFx|M11TnF}cwLd&(??Z>M~XY)OS2i5#-anqZeUC`g+ zuhMCZ#{0~_?oWKA{fRFzD$b{y};4Zk^XJNIT2I;8{2I-jCJ+6Ua3RH2Krt@b|wqizWsViNnzMfamu;CI1v% zvx>=kefH+>hKuA2%|7H18kMw}cB{20XPvUw7J1YA6$(r3eidee-59xJ&{piNfYFRv zB$o!JkEx%;Y4v<5E{^ktCy9bdK_@`=#k4FBPP0Xu{UiLhZ# zqh$P@uu&BxJ2jp3>4%o`Pzw6x8`?zF!tKsOYDZ3G5)Xp$=tw|YA%BMFVn4G~+1*5b zMZF}5b`C7m%_R%Ug|Y7Jgnyq6(r=s|P;&YiLzvt8lPSpjHvUCnNri4|&3P#DJGfQc z;jH-oX0Wepk0DDTZT@e&*8kk0a3uQ&ZaJLe_Zh%QnL_?+A_r1L|y^!a1r%}KP+4sqLuoZa(<&~eVb=$4x4t2~BX(e`Ndec^v3d4MDT zUouBbedV^A7Jt?zYhO2uJo<%%zD91dEgWq_O=Esmpf=+R+`1h;&IEgx7sID3P`49I zyD+&sHmX4OfP->z65o`gd^w0TE@+s!hJ&bv8Z;(g`#wcdaig_y>)e?qyLa+uVr?rY zkkMlGuY=n(qmak<(Y~Sj{>>y7X7I~=%O7t={*iT>PDeHT`npYJ|E< zZ+ePtH|)-@KX3g?vCUa}oW$z>d=)M;jsxSL)7>B=O>$tvudR2DptI*VZ3ddp!1;Inb4)r^j$PD%c)h zFft^K3pRTkF)yYD(Fv=;COuQGam!OCTp_i0-Kbf`J zwp4EYxD_2vM{>q!$Gv@O-}XWf;K;46_w6`Zk^-XM&S{}$R1HwH(w}(Wn?hqC_)F{A zcMXH+c;uOSosF}w@^JR)y|YGn1K8Rn=aQZdY{gRkxDO+AWLraqnnu)K^D64lal5;)UDrb*o1g14EWb93Y0+LC>EK| zjj`Lab(Q6s{a_>F2KwX@Eb{t@(>O_2O9!R>74n5H&@ZflrN?7y;V@n(=aMUR*rk1C zYzj-~okCeC?}^U^5RC|Lm1|5nUMB~oo$3f88unp`&r-2Z*W~*av5xCsk+husTcxhL z1(i)6qI*JX*b2ub(P5-g?FjOY!@Y{V%$@TYm@1bmyvU!HJ=4R1>k;s#A8*qL@(^qf zJam-L6{&VjAFo!j?O=Wno_}QE&AZa~ zA!^?OQ-0IQPAOZ#rz$hG1jRMrx@4iNoe4ygK;TzK;c_v~M%w3SaEImKrN9>;xhsOA zUlbyZIFCPL-i9m<>7(HS*+93j z0dtdLWEXeMegNF1e~DS?dSCuG3eK*Tm&bbi%6Qx4r{<}c>@3Td$2HiBz+u}u>?y({ z7dr+24|i86920f%EQ0zr`DZO1Yz~pC(5I>QE%kj{Y;LEhOzhtvW} zulxb9ggzdlanjBgoogRoZw7r`YqM`a%3joOZGmRAttvw3D_lw;RKO&ILTrC}ggnUK z_Bf^=s`OOLZ3khAS@B-3R35l?=q=BVif5j4EzuESvMT~YmD}GDl1`5R$-BF8c-yq4 z7xoR~iE7F=D)+M+cdza1UOI4|4H@?`1?&?@C%Kaxg*dX}1nDH`OYDH89QP!M$3ne| z`f-hUCXF4>;s%UHo(>n<2{>3nFrOu*2aSQe*i`PAwg5R1{`G(2NhaWWpo4b~{@|zW zL>1TIe4h?T@duQ!u|ox4k44+LBAo(Dbry-;rylC-*emi%B--)zg!^4aLEV+?)vS@C z(<3V$_1C+zY%CV>^W|I1Iimin3_!P`NWIP@RMo3OK}#fMnry>YC4ox3v?2pv0l1%M zaS*sm3`dNMGyb5pfQ97CY7#P0qW#EZymH;2f|eGH9e@xEGW81;$5(L?fN;D%I1j>Y zZ$LnFK_$GBj|2}78dp$!;6^|UjoqhSu?So0k}NEX0mm(6c(&OwH`$LNe+HF|lY&a{ z4EmV$Cd||CAeDvg;IKH^5JKP-z6K6WhOjb|XNJ@EP+In=4GgJ~5L)TJyHN$N9FX4X z!JtWJ+#K0Z245TT$$ej$irOpoRH{tIR~Ro#y8yZ@>tJo3ZMJY=;uELO7gj)YANSDC z%o3SkF+zLiK}rdl=lHtk&Pue~_~DcOYy(c)e53Myrx8wjC;Wkb(K`Pp6t@qEX<6PP zGLZ|kHIpoGpb2VJaUTd2?mss&*0YUFfq1QCvJQGUQSl(MeWYLinbfbt%fCEw{@ zyu)p9xKZmPFQYY~h5CW9(HPt&+pc3-;4S)|Ac<>JbgC?>hYl>i+}qxeINQ;UP0;xU zJHDc~W^!5abDLQyK>f;ZaY3Tn!>J2r9SSH)3wRnAJ7) zk0IHf{^<1eBgcE8?TvOtGoSmHIpH1y5u|zIqaF!mrz*XvcQc+g29ATd>Le&W?c~AW z`SMsjAXvw1{CN4CMaHruQ_nMLpLpNm&Q1JMUxe(z(d{)BHmk^MLh9R_EUn2|cQlsW zNlBIq$XQtT<^?3wM^kc_jOT&Z=%M8-{Th%cbs4Nj69R)jnel0J$4V;-dAuKM(BwO~ z5}`Z>Le52z%J zJ?B_Fh|#yr+6Fg4stssaMr#_scfuTg170AKCzDVGX3@DJ#cBQjj`F{7`<}D`ikfahDu5H!`l3*gko4k49xqfVrWs9 zx$OQdd+Z%YK#$yut0aD3{$>_ruA2Hrkg-hj222(DM1TQapjE!e)d%_eE!C{`#UbVlI;g?sLdYHH>{i{%K;SRfWf#a71knfyy z-_ffWks|eNsjF6zLiCMqabcK<+xWdyC5MqB$l!c5#FKJ6{&r$J>-OOXMuDrffrzyN zJwC`@h%u&hT0td2qWl#IF!3UqF;NvfwA|?NrWr7&V&K1#thf`y^tGLJu?zTqGBb6n zx~*Kf;oINF=5F<+z;sYPL^JrtD}OR~SQz_Np&t~ooe$!?h~H98+m{Nnk+)ea^M9Ba zPIQ`I$e=P37T${6tOcP)oOi@XEF}oEi2`e?6t2ix=CV56Y$e%gP9j52`gxyjKMxPB z-x?JSJG*9*O0u`TOPyj*01gG?{Z;V_JYKzQTLe`Ly9hM+E_YQZZ)D&p3U#I2tg)v?PE*+-vJHxhqWS_EmWHP5 zbra>t2ruqKa*Pu+Oxx3sDX+>k`D5s^8rZ<3l;l`9wCTtC)}waRn)7J^}J@(nz<r?Wwj^(0>~LNS$&z0Kq1 zmhP@$7H9^I4*_JA|6NMU-0h0#?spj8y6sFIk7aFa3&_FO8q`mDF;tbwCgo}xeKR^< z?v7Vr2u9cwkbkovmZ#Z?3nju#Q)5bsYXzacQSevRtu9#>K@AW@)m}^MEKg-oj1bJ% zcEjn&!$MlbW&GfDo|}U^M5mje>1F5hA~7ZwBlVgZU(wctC6Q(0t#rZy_q%{_N=2Tg zq1poYOm5J(0|UG=Sa zU@66$s8i;`d$-)Qs%pL1PQjCnM_yQPnRkyFr@8R{)ZKjh#u#$4TWuA(WwrsvGaHhb zh;M~z0Xsayc?JcKr^(|1ux7!%+Ie)+PL}_Zmeh|ci+2Rl4L-W3&VDI?^2@bcBFk*E+ibY8DIQa0*xcjf z5&8hLTeYyB?Ej(ctHYw|y7m=B6r`k-76gPL zq+w{0E)@i6VMt*>xQ% zEWi@Zu%;Ofw!1tg(?B*O$Io?!u_FPwK)wFU&lT6H)_GF&$ zR8=H`m7PGXXw}V-8ERQHbok4N7*5IiEqgO=7BKYkF?#U(d(eO!ZGewu`v~cmd+O3^ zL`ZmhedgTgt1S7T8K#5u(C#}P(VF`FS1{+aFZ?>2mt(@XN33DYVFdDbZdJ{0Y;0(x zepUZgTO0Sv*}3YnvGB6;MNlp9b=&!F1mOkGY3Zgu2WO{y_07mF9WmqSjdjGj$Dx&( z)8;5YNX3R7Wu>HK4dEH7=yjU0Y*yJYD^YVZ_^c1)*s(SdPqauDK*AByb{D;n(_b2l z?&Tp|^T#sXH0)7j#?SoO-Xcq6Ws%Yzu$U5fTUom9Kno4~+$%VAxE8MgC<&yV6_O&V ztudtkvyadydkAPPt~x*6j4NS0kPBP4ur<84{#ssQnZwe2a_uXMrk1VYvIgt z`TFcm3P8#jYl&VZ2Fk2eCC4y{@4(!tqYqa5qL&}kCum3et5ml8ah(DwDBf!-@9DuV zDr0Q3*tIMSZ)!Y4(^lM4$(YTcfvr=hz$84@UTRI#$akc)w`;dnX%Si48)TD1F(PZ`z#WnZmv@a&pi9614|@}5vne8 zC(TMVi(o{cX(dhzS*geGR5h~b|DZMxRW4(d^p=1!Neph(J+NYhq$r7D@z`(h;du4c z@y@kp{pqqy7g2xrhfhlVDf#%duE7$77@gF!7Z`9`~Ta?4bUF%Zb7d zD(ui*=vs6YG(xmV8l7QStN^U&&swoKIS-V*^YWqhYvFl14ZZ<>T&t zd+?1<;glueoxiubm9SV}=^v?kKZh7=3JveI{PL;uOqzP$ODhUY24fFb#}YUdS?Y zl3*WTO1aQ0Q%Vqy(oMN$#~Mj_>r`n|^wzI{y?$tMyrU3C83B)8G0i1GaM~^aj;Zn(Z@| zYj)KkbE{?b!Q=0xu*_Y>cgNIA8>CB;&)qspFP?##udGCkm)P|V3;BfQdwhQzb@j*6 zGS6T9%y|xWT#0@a_g&}x#-LaV;sQF}urjS^m0f)6^b>|gDu`Nv@NX^7>E2Hfe!x7h zTYmTCy2-?T5ZwNc#T3lr@7+NRI8<~saQlm__Lr}hEN?+pr^-P))aPpgy}@Lxaw7(V z%3HVbyzwcO-4AGIp+2653!|)N;B}Ph?~ljJZTg~XD(Bndy*lRe^9dIYZsHRK=VxEV zh6=S9D0ofp*Po45rG#PB+PL$MIm80pYk$;>%29y&%XEt}n*Qz-(DA^yp}%7myI&hP z!P8qU-DLk6I7a7GS?x=^pi!@=3AblZ@LK#K3;UgXboY$M6kXUG;t85Dw&+6bG`9WA zS!kYo&{3jnO7<^G{_GBd9yw*bBw7`#1&R&+FC?0*IzhKR&Qm~f=4<6+Lx-?FwR!)< z?F8FY0bcw)O|essp?78l$qQ!A=2s^8Fr;cScHp~93yTM)XvWqHg1;HzOx@!f*e9-I z>n|w&a&6{}iO#x)t|?~yG0fmq_of8F(QP+Ei!rfouu4HXKkSR3&6LPcrL89q=S-Xa zIUdQkCuglgIVQxpTGbc@x{ZNk1-eq?&gN3n7orjO@2kAb82^<6 z31$-Mq{9wPh=fgoI#D32<`;<4?Cg`)uK7sH`NhLK140?! zJdrX(->vZ&?59d&cwG;kS*uB0?)Cxu7qU?ve~+NyG}LKR`0f1WXjvQ?yMYwJln3cT zXs2>UHXcDm zTy>zW0Cn#8>F~&)A(j3!?jT~)U0Ha&jk@;mMV~M{^?9u0OMi{Mth`=qeCjmZATp-{ z3FpI^1O=UH{>p;N<*m6bgZ<6A*=eutiM9iV8yn@ls-QN3ey9Qc?VHz@1>k6I`1IfT z;X^~g9F6hUYd%)bC6aw4KY~#5;Kt0@X&(frAEIjZ*(Oy-uK;(HTn9V5>=*cHfiU-+ z1zah+LHYEtu=~AOuk20VoA?gsn59jz{RrDs#dTz0V7Zqf+vMOOY2p@2lZ2XDISJz2 z|6zbQ=XG<<(8o)~;=Jlcmt*-kD)Q`ua7@2f!cNgu7K0xu*GHx6fqdA;Mz2SrPN)0j zn?EGoPQ@EvG{iq}yYNAUK@5ja#KeH@A1N?*1$K&;~XP zORO5lnd8Z!vu3Xh-N&EGCd)Ze8A(_Q(FM-3XEf`(dsE>6(X z$a(QgDc9YWe)r4smZ^yxI_echLMPA}xw6s=^ap;< z8GiIV^D@EyzE*+$Nq=8u%a&M}5P{Xt78d(RXys<&p;=9LHoOhTL0shQ%E12X;}daZ zZ}*u+B9nJm@VHYjFW)D^gabCFOW_P(bF$=R*8QtL89Zi zeZ0LcxA=0Kr)2;-UFV^xuz<_&pkI3f;g`GQ1iC(6&2yZCO!~(hk2a3(Du~3$&3Ww? zNO5T1;!-% zOQeB?iK615y#%UMi%$iLRdas(6}<= zjMb^I(*-#6&(K85&Hec?T=TKSeXnF6P`~@GK)=8(4@^CrG14WQKupgT7q!hfG%XLl zgKe1ig`#-bGkt!!B=I@^DjakB9nz%@8gX}v>21qH=IIYrMe}i^CkvCuX#2Kb} ztK9MV0DG~&GfQyi!;Sy`d{VlBzZdZ`jqch7V{(kyi@yeompBf}n$>F{$WFd5a=6N3 zv&3XU@I?V8@49oFj%e~pTC}PSR!yG|hl)M?Ox3{SBbe95%E>N>diKkgFHZg!HmQdj z!(a5>jyR$?L9GxwG|o30L};{h|*kU!MH1!%saJ=O*({MzQE)8Z*O$rb0*LBq? z9CMeuqVOpMAo%C~Hg~8OY;qO9X=dPvdY$g8#~Cbez)FmNX}g^V)0)p!uAA1L6Mx5; zT$C4a8n;?GMWB~~!P`s<#xxk8sIH^hvp^%6?B=vGl~Deq}uRF2}kUi}2cE>tEI+5z7p?>e$U^ zPyTgs{LlB__X(MGB4)arA`zT93(l#hnyJ`dVz5eENS`EHv}ovo`m&Ir$h?`U6dACY zI*ICALFK!3IFqy=qx)dOiD7WTWzcl^W3DGP5NH!;0= zE-ho*gsclXS6?B1`X<8N$PXIVTeZB)QYo^2(+auq0MvhOhUxWpoBtpocMCXXyE|+u z{K|cu)g&d;4?u&@?ni5+8A%8CtEq}AT%Pg7oA#!}X$(YEiC*f`YBzY0hCUFA4Wm6Q zh`&>97x8hDW9>PBBMN{4NuH2g;8rP6?e95I6Z+P^8O#ikNU6V|t{{HeGgqFd>2uEv zHa9!_iiOn8X@rA1#~PN%8*mD6Im&68N?(neK4L9>B|o8nne`+`DdcRG!Z%Kz6zDYi z_=P`EY!9QxEwYOk1URpZ+juL3sZ9*#=KyV>@F^~a(?HB^J;qE6Ey?TIwzkEZ{pzGs zj9}NbYHY~^=*6h6@Hu5NO^Om|68~sbBzd?1Md;S5uCUliEM?QMO~1b>9+0IBkNNXN zm$RGWHGHu$@5O1w>>k&3Nh(xXqJp$}ZJuFjp(`w5n~t|xC|+r7OjIvVRvBb@8G{R)We;uraR9?{D6vl$@<$4RaF*~AGNFASv%;Lnd;RnAVGvcTtZuSm^k3t zHS}>6;0D6 zikP@*OG`~1%wReXT%2Q@3$A+1%%nxEe${vpwi-r_8cdaJwphsJ%OS-TGSH7YJvQ5< zfH5nlklEFj<4k!^Q)hWgqtCJNvtILNLIJ%geYF7|6>o?}p)U0+*dC7OaVvKIu0*1s z#XC1?sd2B($*O^u4YGlV)9;P-er=%?FTywUmJEhnTZ3NfwP(s$yPHE6la2;z*{r@cDlFum*Ig8$P0! z-Kl?fI91Zr{TdZBSh;t5mfJ%>E zfV-SYCN}kr^O$yhXI(gWpDoehA|O!o0p11PbuDBV?+=cgJ&fRp*rNV>Ja+y;?bLQRBBZCnZ_U7#fJ^H6Cayp*3 zqPRW2a0+W_%=mSkbzSP%RQnVeN|`tP{3_Y?8NLsY+li~q-v;JbxVX4HZWdB#8aO`m zR!Dt{_YfRuRJHjYiYKo1r4L=Z@jaoaOv?P?W9x6+JFn(oRr*;hYNq`i19Ls1Ombg} zOWh7Pq+5eYdB8To$`^|P#kUwqIr`cU>JKMmE&~v|_eXy#hp^<4eZs!%~BQd_|B`a}P zBDeKuM$UCdq8N6aM8A6v<#j5p6}9Mkl3#sQ=m$!fDq`NgmnLLS{z*&NWn0t0%WrX; zi{ad3$Jk@*nc=+tV3{z`gbcA%=h{8|^xbVvZGe7qJ1aR2MF8VTp#P+2BF?}gh>Sz~ zokJ|zxac=f{4f*H22;QUpa<4T^&h5p2CMoQ_WEv!qSkQx#O1q>goe$MM1Q|FS(z%} z`Sndd23p2o!GLP5^P`u>K?3ChGS-8ox;;Q!^pLVH7J4901F@Q_i#f-K zEOiiV9xNTK>!_76g@zz3N!JE|S>HQIvI;jf2nKH5Zl_6oxdyZ&{h$zXp-MLFbgis9p7?8a!N=Q);dG&ht&q8qS9) zkJ4OmDZ5Q~pWodC3!tOw-vN?l+gWu|h9JSH&Bj9&$X0};ugv%T#Z~j&SSW8)sEI93 zlvE$_9#U zQZ%pgkAP%(24{4u)5lWZ(}vD>f)t}28USTo@BV9t;drXSVo6b+8${Wk{C+<%LnQCr z5|`_eaM_MDdDs-2jsW^g$O9P)b6zQ5xo;4m)tdeGL(*P1zkf`?=IM$Ntn^B|?oB(P z$2wnqud+K>m0u|R>7y*`No~_-JuVw)b{szj+f?yp;tAu(L3&2mRarK$%Mzh#aY8Qv z#}qc5zwUHwbhf=q;H4XZ z9!elLB@T-u$Zd4S@kO}~f{9GXgs~uy)&$ZwQ~Zp4LV#?s$wF}&0SQT16EWz7+=KI; zhT4Hd_sTn>}=id{5V{M2l*X41AT#&T#n zVe`pcEa~lc=hFFQ6J;TdsEtuQu}=EVgHaMle9C5N~C%IL=k8W6vQmH|8(SO=-oW}bK8`+ ztbJ|Vwh~jBGkhKMqzPMWZ33Zzs$Ui39#@lxJmeV=ykA$Tx+31C(yg^t7bP!25p}zT z^0Ua z+6f|W{7_hZVn@La20f7eDmohxhel|Xp>vfemO7&5gGoVM{KCVohcrd3lv>;x#4t@3 z;5Ny_X?JY)#gbjNN(06Yl#+d_VG%S?{*sB2+V z`jkYrN^qWUZJ_k21e1RN0!7O7eW5qWf1?)d1j!yMm7R9#-WMU8e&;a!=KDwp`E8*4 zM$Gzv+R)kn!9@Fkc}XwpSjNH$U2sy%?IJFrO!>LnrX}g4Gki?@o%199v~~V6PsTHB zYPKonh`$7Swv((4aLcFTb83)Kp7DQ{_Z%1yS}u{@5UA8*ci+PsF=#y;5IyVZ1%uc$ zqFV{^M#Y3v9H(-h=BT!rd%EnDXNjKFSQha@ZnC?VE8Te4;|aAY!vV`~p?i!CKP=^& zB-J0RfO^uIu2z~73|tX%Je|*V@RJ92G1R(TN1oEV&Lqc&L0O>!{P$FR*I_ zu@>IOM9D#9-Wwa=B4vBi?-1)(rt^V}Lp^|2T_qm3rfo*sl&kX|d4hcQ_IpljFT)hU z_wi;{7RzkJM9*UhkXKS>w%@Q(6R6k^Eq;KowJQR^@N4zJ-B0E)=>S%h8_0RU_O*35!`Le%5|^H zQK$I{t0z@+dsQ9mXPt&7#X48OwlWP{`N>$ez0qLFy61kp0UDCeW5CS&Y;i;=alX1` z&))QxkhhnD2+TYsi8l=Dy82nfyX!dL@O|{$^ONH{w5ax&LX=TRqaAfz3{~} zkMtc`XKr+)N<5SW-ifa;(v;*rEFsN3#87BFYuv%Dq+GSDWB>P1&huTAU3p2&G7#D_ zi}ga#$&ab#!my}N7Z}!e)4qO%x4vDi6<4%`$`v&9R+Xqtd10Q{b{Lhjb4)Fi?{(r& z_~sUy{X|=uP zriqx-VF)0Wv?BWXQW1@jzf060ZaKz>2tjGV3^uvCL*kO0_L!8+NoHE zI4|fk62s^5H}*N-d8!+0}V&?@*gc_dL4+h37qFcllb^uq)}|)*beIhLM}V; zh0w}~Qqhy1G1Gmktv3F=cj3cTy(d;%WH4_kIpz+m1(up081inJLV|1 zg`aZBXG8kw%E#+Skg*fStnqcgXOFHF@htZH4u*~?K(9a>Iu5JvK~I-N;*N64SHOjV(kEa#U2*|FsnmVpnh@D6#s*iu;QgCqv-+I>;u&=hIwCWn#d&n ziA2>(0oWz22Hdpkul7k%qAvZf*rmP?0+nf;8r%zuJwaO%a>V+NjB=OTS)Hu4Z;&PD z?OBLzcoeNw$)<)^McGuDJ+2gu8?Z@4ouKJCnx-WhKQ{W{W1ozP3m41&9X`#M2jH;$ zF&lHfYj~Io9i0l|FBB25FZab;j)J*8@l`p)R90ilr$wB{=j>gy^`Ks+$|$F?(MkOz zT5TZJ_-gEYs2 zW)>!TZK8e%G12M&_{)o+NtaN7bXanj$I_l#XS}kzZR23BT-*o2 z<-c+}jToWP>SZKg1QTesZGz2RlnOTBrq>p{>~O% z=>e^Uj@w%iOsNY3L^0N<{~PcRzWx7!f43h3{l0R-o2B`egRIt9{2z-gBH)hOcS*je z{q8Qja^{W9RtqIU+17+timC?@EnG;!7XtjDsjg%795crjSo{`|*QcUNfn?x=0l zYc{4t>%y0T934@?S*;sswmMGz+iHO~Tcf&z!+In~G>hcyoi@HMoZ{P7BT1+q%4=${ z$nxQ!vpw5Y`$yS@T~U%#i=m&UK$kGh^V&v>4He9O~7)%|F44MI1l#r=nAM4l;+x?U(nY47z8R*-ZmCenV>3>DKYj?8M=P+6>` zah5haarw;}r0S47%Tu|T=F2=63OOIKF8DS*-dk>W`KTrcHd8xx^WxVmUrz31_DkiO z??x77(_WVRNp?WKmt6LJRb=Wu>YGx-gPJlYQa7U&4{bxLHIA2d=64r|KDwV)E9`o$ zE7+p+?Hz8-qt??}ob;QQGf((EnT+x!RM%&oiT%y$Z``?c1tq@c=)k%*CRR!kbrJtF zCXT5DO4>d&&m%cO_9t*D>q4lPj8QG|Bj)|S=MS7L+}#`oJ?11fJ5|=8vUBFYH_lBp zOP}c_j~cLRDXuFRyw&)yDe4v3W-t)QK_il&83R`s$i(fA%Uom(>QAzdO@g|n)91*3 ze#QHCQiC;7nEV&Vs~>{%(AG+}!{J0yi^(*G@-W!Z`LGoi;xtgxDXZIG<@;DO~E?}FpNphFy4O{ zgG^<>2bCYc1zo#lEQK+f`9B^yAM<>g1k~@_aj&BkUoFYx&jMg4b=6|Z9C>?q@L_LD zQMjKWYVjd<%`Z*;KK&@_Nlt+x#iBm_F8qDWJZh9H9Q`__7|sw$>&w zuT>SV))V}>(S}C-*FmE9WlQ<%tp`5xI-|uBJBmK43c}>5709!b-o4JOZA0*bN`hPG z(=Q#~nNl5%jC_-zB%tO4jY#)<*YakkQC1ERUKr4sshrf<^PIfBP)KZahtKs@x*7i- zE#wuzrdl-wUV52OEP_wt8{iB?h5r#-HV3kN2ttV!uIIAw)6?m9|IH2h-sz&z8ZNP^ zD6t3~?IJ0q=VW;F9;>E6yx2^KHHBHgv%wti+-9E%q0)e@3j<-9o0g29z80+w)oS!I9wjp2gx@gWsGbda~;X z0(R&=A`D<0Uj0b{FoGBIX;jy)8B=1+rri7gNj!U*q)Wpu zZ&}5b)1SDE93J?Gz*@^`6)l_w7)kYU}d(XN?;Xf# zON?{f5S$q3QbaW0n+O4UaYM!UdnD5m0yOP65`Jf4<|e{7PBzw;GbevR0fM8JjGKN_ z_tOv3r5(IIA>qTp!tcacERoh~H4j7uJWCoazO4O@*^E^UoKz0nv3`%Q`I=pJ|HyHj zvrk*0J}>$!kd-j#kE?OKdgl1j(hYLJtzCMU+DZJ7*v&c1bM1wA%FJn)+n0#}5_r6v z)UZuPAj%0zS+Fwjx;gups$X|CsLSNr>l~xKy>jVCr%_N$gvj*X!OWD`!C=GO&<1CD z%%M}O=iB^4yghrzOjO2Au4@zjFQZ7N#(19!r$&t8Ub|?_jWHWY`7d67nI>))F_dWh+#;=Q2$ z)jyGMBw|hsy0>D5|;Q>Sq7TcnS*uUvtP0MN558+j*}Kj)Onc_ z8&*aEJ$g6(t|)en;_>}GhMFv?G?5e~4;eCsdlDWOH^k1DT$D(T#*Z~Xa4^X5Fm9VSSL9azh^ubIe zRW#BK)ZYx=%_j@^d%z+=*L_9sb~`?#xE@$i+-gBC;S{o6$4=9y52`R^3Wy~X4OhiKU1 zfP(CMaJM)7)2DcO+_x3w#T=veQg>G1TRl0NRKKuj?Ga- zPCq!cr4Xf-R>qcHeT^@=B^=$4?e>thD&YuOS1N~O8N^@!k_pjqPv=3M{8d^`rtBhA=M_U-kqwqm273IM3 zskW}SkD{b@pMRY%b|fFAS&n=6n8WogIK$y%*B2=%iZz^dk>xD(}{8R-ZVSE~4Nu+Z(spnNr_i;yy^qRa1jdpPklSLlRy3mZfgMS*@cpZZ| zrOC58B7REp=30RRUcz zo=FFjr2)DBNI-=n6%{R$BjRfG4Nz*8TaDoW)E)m7x3DsrhPOUm;jdAoB?~lQ|HCf+ z%8gn6Xe6zL<}1Ja;d8|aQWHldfy-^Ax?_tn*>x&E>vM%huA{25>fH|>!sd>3{Pd*& z4Z^qQ+qmXfSEW#=91$>Ka>td?uv1q?=v3)1%Uz{&r*&16z zA77={?mDmYYL+2&RZikf|D<*|3m#4#yvtSw6!lckka>;l+wsUI3PE^G>Nk32_z42M z?oCFf6#n!4bo;$j)qVvAje&qp(JV5gvkcVp)oWFw*L<~d zy*vS41)KmNr+9lMuPKlp@k$(ymWbTn$5$#?s(o_c4klqy0@^aop^IOad!8ale(^*O zz~G_+TlZ)AK&%DSX+I{jTp0y@?f8+3LlzWzd6V!jqlaj+Cb+O3u__;tuZjQGI3nt*B%&A5yvX2uW` zifO_x`f;#iXB58Z1^lRtv~;OxO>nMg4QV)67|lQ#@XKm2E)5lsSVb{qMys1yN>(4~uT@TmemPe*9 zvT~57Zn4Z5`oHI7ZeRNPhdd1vD0;S#h5T;oxiVrySfE!ml_%H*7Z>+Dy0DZ%v#6Q5 zT;1;Wn+3@~02HXnNHGcWNp~${+5?)^TZL$&wNmcY0L|Q?!^fOJZbglmEB(wd`lP=t zROdU~Kkq>mAkORa{k^z%IE~rcJ7CGsx^)1gR15$mJ*A_VRCiesL*54o?Y*gv{WHH= z{b&-@9}~x9!WWXaDdr8&l8VGK4c?#wqq5vc%~r%G4_}TqK$rs&6fy+1i5+yyoWoj7 z9C~A%zJ57Mi2IN!Y0;WNxHsNYe3MX_iyh?19?zT;>)z%793Y!#ptK%0&D48^t1*psR=LTNu31b?~>R(0V z+~jjiCdAV|2wCG3P_Hr>kTX_*@O$@*6)EaGE#rN0OwgOu*g>{M;=SLB%?k*&Dfm6g zgQB*xD2oiyYL8CLeh?16{}j}8*@V53C4MmiUqtB4$ZP5x@2XCJ<$^F%!1$6wMhu)8 zNzG1!m-2s;kj%|woF(sKPwOo90L&ROe0tGrJ%`s}^hr!B;fOvmNg;-{2~412OV*h= zFQzJDzL5N z-sDTp z+*ntwa{#HSCh{=|L#Y?&H`3k>)oVBJOy043?v*g<5HO zxjf>ZfSqoM&89y>z~AbNmxjytxQE_AC0T)}Z{-fLfm(K+q|7yb2<`_Q8s*=X(F{1? z?p_^@cijaGIqfn;sKMl*uI(13MaI2Z37bXhC99&CMJDN)#ef+(Fzfk0I_YFK+qZqseSq zPe)wUf%)=8bU2i&*srkcJDu;ROe@#m(ERA&{lhY~DR4hh&&QfNu#Qq|oe^C&vAKRO z^tQ|*K-r&Sr0_LN^Wz3DCZmPkThY7(ATP9hc5v;vV6P1Yp&td>e#*o3oALH+r-O>+ zWtvr#nY3a27c{iHeM5He>yfhbGwZ&T$us`YPZ8(k8h%MO`ux^0#_gXbPOZdIr9gnJ z^>S*6P}j^Ecf#G)MjWbfBXC6bWXxF%_b%nD*h6Zq9GkeiSzi@mdHWn?dseLBBD908Is-W!TAVvyQH_<|zZ+iY+&=jLd(< z-(v*_V@jqZ0BHsVX=Qb3@@)&HJDSZ2jBW3#(UvO{AKD}B@Q{OAQ~l8(a9qOUv~-@L zHw9h9&*znCu}MSn2m4CjOERSgvdd<`O-NsG@aJ}y6l@K)xWF-lBF{@2?PkS1 z);sca*(~(+;ZD!vyuhl~w%ny;VQ1^zMMBrPkVJ%|Wl~i{z!!ZfGpzzvo9KutjHOXi z`-kZA(Y101GRoAtebq219?Zl@s-aH)&-$L`2k{F7ieA7^{1AjVSd}rl*oI|^f|*g_ zKq{KmwAlNZh!oL8XxUCBfvCF!$*+Z7)i%i{IO#$ziT=0NgC(iN19KY8PMt)6B6SQw zP}sa71&g`w5djk7G1o(360`sah(nq7EJE(K@S{FJ%LeRX@F#r{8<4orTbpsWB2mI zDsvL{p9kh**?iD4qsGWiJ8dwW$es6(rWcDF;~JXqLBffN4!`=}81}Fa&36pF$`dL- zi&{$GHLi6wkYL5Djde6D^w1L0-JB&98-utYMbaRLkc z@EiQiu_FpPLi|qMRmX(@=6kvK2WO#wn|DXftKoLv)9NOLalC(o&UCJzGaX0f=<7Vx zi@O2X|IyJpre@ayT%QZ4*08}hB9v2rhL}KluWkFr{RBQJDrWnQ1~_TtGqdZ*E1(sW z#Z&rmd_?Nu2&inR3ZMb4#)VQm&tL!%viC~$G)&v9R#~xkR>%d*4I%gubJEkA0q>eD z<6n}ECP2kXlZ<*=`Q%oM`!Y_BsSss8rf59?y$@I&zWqbJ6x?^cBae^1K}eB2=`m?h%MX5ARS4gp zxSGJE?a{(}Rhd^~L)F)U12~jaY*FWDeGdD4k$=}%(nW{@ykLsx36b`{K;M6VzY;d{ z43G}?>_FL~=XgcGoIb$|=~FJrX3)dn)*3ufRI`kAcc-wJi@+bXMw))+?LLfDDKQ$a z9NE|u30d#5r+u^7{b4<5=lvXs2cPSB12^^uQ9=x(!vmg}N;^H<&Vc7N$p*hK?s{W- z!Y@OK27#X6tE@CCra`$K|E)5*VlUSu0FZ30Ke+$qH3*XXxkO#SAFA%eYhj+1vyCdP zZpHTV_T0^QW!w3zjyn@^=b`^Z zCYK6aVWG~K&xEgGp>n<-zQ6pp@<%lC^ufAZNNQ)c9oC(WcwbJ553AWaDVFx}@>RS5 zgbr2v6w%Sm-&F7d&>F{dLq3bq;>9KzS!vEEVIu7C?^gwalc_8n5skY8TmH@R>mcIaeJv*z?Ie z5&neJQI%0{TVh)0Pm+UR&e(4+OW}$gC#=#*Yb4s9Y6a(=4oC8+9besg8cTcHUJ;&| zF!azp*63xCm$692Ea?(Fr9;u3cs+0_7}_h2NE`b&)`_rOqpgN@hzkrGv#Gl?mnW17{L%IhJqLs5l*x64*kN1#b z2`a8h_+D+JWZxl&4`V&tOQ-@Jq!EINaS>5|-!_@igX@rZ{?cR9asCq0h2Z!AdiT|m}}7@Dz(SSXZz3Z zG1@&e*rKkN4F8>)@Cx(`PoA6Tnlt;icISW|AlwgDRL!qz)D7!|w{txn;U$;xomk_Cuk<(1D>f)~}cTyfmvqXj;_1frFo z*7Yw}(rnS6Jaahb<3x9dxL0nitA?abt1M<=b~06V43M@kZgf}n>)9q!>Y04`Kac<- z_X)F&BM2X(I_5~LiGDYf>DKsPNpOIEMN!X?bNQ~p(N zWBN$U!Dg)YC~2LH(gO>IK^%q1%@O9Ia(3sGmYmLuR^Z0dWs7 z_PWABu2WTAh+vdCfJuk?6Sf%GNJC38vr^aX=|E`83F{ z+>!*t$6lF^ZQ9|@XT8L)zeVCAbAnVHeHF$2af7SB@;db=#N*e!&h5LIztiwbSzI-C z)NfP%_*Fh{T!;1TE=GdyhuGcmFLP%&{ZGxaCQ}w=bng5UZDTBq0i@9r5fMc*$l@y~ zDPnTf~YXF&_j42A4&#_zwaRD`A7IaW1G_|0toQ0J&=wZt2<tQ|b+t0tqQT1#p!#Gi%RYr_oVaT2SuFnRH;g(lq>6*S<{L?|5h=uXU{aEb@F^X=NBhE zDAFXQ>xDSB^DEhH7tJs!p#~lrsxc=j2k@5Gz!;S=xrH_F*}TUWcldY-bt`yxzby$7 z#;Rd9UC(;N*TMq3;6(O$=!vvC;VmVL6%8poPlKyT5!-ob2v%0KOZE%d>o;?IiwCpE zz)4k0pkxP25k*`Z9~0`~U+d9DFrYZc;WmW|c$ErO-0O!d+9*9a_qZ6F>3txMJDTAd zk!78Pw<6IvcR-o*viW3hUqHNnhcs`ZU3YXDpn3-S#J%Kcy?6NVXt|iyPMTQHwoj{g z?c1nMF50J;mj*JLfT-)9U$ZZ=-TIbH<$;n7m( z%Z<7=UYz0e!m1*?j(!V(J?7;`_a{lM>wdp`hst8~+}}SeH(s_oTr5Vm29bVkygVCN z5;0!tC|i;45mvfR>Qr4RxSA0p!*tASK0!g3nABeoy&G<@yczG?)UePPO2`8 z_J|I5XC5(pB)u!Vd1@n`VbZNRa43GJ0MtIjscoqJ$){5J1`?s*YAz-B!E2uO@0#X7hDk4n*=|!YN0O^E45K-7j34(ys2uM*n zfzUxgIw5ojp-BrZv_PnVyRy%H?--uF@7d?e9pnA*-ya-Xv!3T!PfOOEzd7e`3J9|w zF)ga`dheX4Nx!mc7yV^{r1b<8n$Y;RK-q1iiuOxO?b&XDl1Q2o!qChmvHHcc_vBv| z6Kx4W^&#vJc#*4D88_S?m-P5W?Tum9t>$`lwy4|2R*S>l_gdaY5I`xad;7goaGR6e zDu>flaB|aGfA$2>_+kEe{MS8TUg-)=D4#B0L?#U(sGd>R8`g^Y4DK0wZF1vcq(?9n zfBeBE<~?iGyU8%y=MzR}j8mA0M|X!Dw%~m^Ly6{YbG4>vhrAhfKEM1FO1;lY#prv2wI|&Mle`4ch2@vM{$|wJ=x?oT^W%U4Fj3 zL#Yq1Dr3`=#1+;tULk!kh?EZn&D*mDZ*AP~6#~5ev$@W)bQW9u`@>GP>Kzo;j>yx$ zbCPC1@Z~ggr+sLZw-;X4nGbc+k=xoN$xZwC%uNp#<7=<1)6hCoELCa#;EnV;_NY)&_zcR%Cwv7Yv(G*8#*uT{)l5__A0X{vE_ z6Cs=EH1pP6t`BZA{+4C-_`YakHJRND=dbI5{4!1n7`3S5`j`^`cHGpXZ^qS3m0K&f zR|%xO)Z=H>Pux8C4MK1WX8lH$K|#pAU3tn5~!w`LB^T9(DR zK3}2;F7oRnzr8O%{1ni0;!SO@I&p^k}-~=aN{Y;z$R=Chz%38x}Q@U|iSrmHu->%6+yK_JD?^Z=CgarG!E+3Uk=^!pUX|ASJAbN zXZlmDy){(@ig^}r}sGSjJi9^ zGOHKhkn1COtGec%dLHl5@AvR_6|>DsU!{r8RTD^~601N<_{zL2tkJk$T>4e}P2ewm zJ$SS?$It+J4ttN54{ZwHUsUgPTT3UsM-Fk%8|-;fFYpGG_0x_B@)M zI1jp)%1Aup+ouvEHrRiCdoRT_jy91}Y}Jd{ylp_Bg-cx#=uMB1m7yMB8bu2x$pdQiP%~*VvV=W8mu@Bz(Gc*zlaQIhm75 zxj?VPKe7zrGzIR#=`kKdYbNPM?8jH(pSJvPr%gNCNSjM_sdUG-%oyhS$E=*C1XHEmd&Yvoq0(HE6S8mrx`xUo2caycJx!G$0;X+dzQCW z8<&L+3Nt25`c8fujxK1USVx4niQZB=dHiL+6mqA;(tolg5_05zjB=4>?z-Z!&d;_v zW$_k)ZE@jT$9VJ!X|4?8%0^8zuJq5&pv`8+qmXYhjb;hwc@{$2WMZCd!N(tQC+Gtk zcyN;F_uiYQqONPax9Q0;;qwHyp7zVgr3yHDleBm!NLV41CEDzqN}XJyj5NI2?e(34 zpZkkEmYotczjWcFDTfLUd*xUq;8i#zZVJ56rar7NaxTRl(Js!!sNAj}o|C|Vy zA83vo#5YuM?gX6|y3Yt7`~DKRLT0><&sD_p_vDP^-8yJEcBh&wVr_g*t}AEs_=-Z^ zsDs3zNU(6#kaw}~&AKJ@V*GRc#I4ge&fYh_v|Q#o_gu?E!bS&`oZ_6{8o2`-8yXc6 zKNEB8%8?Fk=?XQTvkB%0vIdpvJmT^L&s?fI&N$nb`svdkxXb(j%52eysynJ!ff>OD zta2ht0%TbHLLXOv?R zdqhxz`aF!|SMq|P^+A+~*`)ygwjzSK0O>QR4cLhTNkYA^1d6Op{OGbUoE?(O3*P{biorsDILgSh8~3hw23XYI@tg8Q#STv!l=GSW>od_ZiV zd{U7!6H1!yY7BxW_0y z;gHhK)O!cAAXhP=P^p1CHl+6n>P#3TG@GZQzWO&QoSY zm^;s+uqFoQHrpRXR)zt2Big_1`3-H5KcNdsSU|U6x+aR$`gp6V{)_R9UnrTFm%Z~k z7Fu=&(88C-FXjJOB$A5aaGak)ps}oscEeeqa%Ra@iUNnO3*i$EZGZ!Mt;j?ZP3s zzf(M-a7-@DW+VV!3aMH^B}*AEE$KK{)v^Z#)i z^yoog6t_C^8^}FzsMFCS9iz(;=Juhm_{Oz;r!R!n9sjaGQ}tGQ-8{qcr1*@i`6Q!>j4O+FTBr5!*4{op$0YdJFsQ>H8OD(kMnx3>7o<_5YoB(7O66s$$gAT0*yo zwS7VG!D!Yd20sv5aLP;U{BQU7f4>`i;AtB0%KNv^E|?uydKD0S;_$uE;4`nscY?>> zz52@}d|`BN2W+VFDbsIzZacjHPAC4b>-^b6e~jK~c=lH|X5RrLK*uaZUYPy21fDPV z_O-2wjlMX0?AocCb=1(qN_MKfJhC57T25(jM&c=~x?<&pu={Iu#iLjj6#8q5OHLsuRU#sh?# zrVZHc&qmxFHFU8iXra;?Eh196GuHKp^uN(*qR0Gx=i$IM1r!h1w|9AOcgZvF6bk71 z0BvcB;AaL-W{lX2@2BXZF*=a$P*_L-LWek{EYRjy#?W&rz|yrV?BZoBav5Xd{yM=C>#jE=*?Ld6p@nRuiT~ztTM0D zQ+nuU09s+Qk_nh-(9&25c%-R<81taCZvSn)E%;ijYvSa?M~^xK7OU-D5=CA8>2uoM zUib_M3$_Gi9+?TCk^fYSKtU06e}>NfPg>?!+Hu z;#w8ND31Ggr#d4zu1MwTjnflw=DXZy&^|}HzBF}dP7HbX2~BwlNh$dt7=bh)-F`@_ zWAw*mI*OrZ>!5u=L)VGbw`C;qjfMtXnywCov;md#DtJ^H^P#UspK{2V zynkPzZ(r5ccsgNniwx<`G4vCzF63wAkclsJq053iT1B6zkT@MmVkeL{-=N4t<&|>= zKz&>GZ9P@&sf$rs7aoLTaD;G4hSrCmCyJDmX>+;d;Z6?~;wXR$fhL-30j!%-Idvk1 z(wQnizh8@0loKKBw?EVaq1h$1foOiUU`e$5!XR1K0tlyP>N#_g@A>(60lYxJSBn*E zXV#a`rBG(NhDRD-0W9?l@B_hZD<@|mnmL-j_EWY$LEe$oujEwv1#`f(MewPlbeQrz zefcukGDe%a@K98kSKLd?ZltOUQ*q*aa|d_v&|{F}beJKU&Ylc=(GF9xfP4j7Z(B09 zlBk)bbE_AAuC9Jz=|(8Xhmm(5fiutqq(pS)!QJy9_mHgkPzeescD3Wjhh3{cYwoXR zfn#DHU88QL_B~B=xn8z7OblrQy&YEQ%D~A&ZL#v!~YVwoZzBbth%IRHiq zzw#tX@E$YNgB6fmY`D1McKYks$)MJ;b2uotjbE)E5p^9O=)}|v^=XN+OPq|nhTIRW zaXSXLkG<{gOMfumk_ zQtmJ!T%wsYGc8={FZeUC*NhkUJb~I#MGhaEZ5l`3o~SK!BOj5)Hhk^30KMxKT;YOZ zW{r;r@;b`xv=SYRypnQfNHG|oUhq!z2otyOZ$$<;-oO%=xCfrr9$5=+o0EX8dgV;C z#6=syH?6k<@FeypSV9AVrxK8!SX}63AgC?JW@jiy_&*}K&HC2bXA$^Gwre09Q zYULDMv0f_xziiqx+pmLfLYJCqARN48?jREQ*)p& z^LR3a$~}Cv@9>EUiG#S)$(ZnGt2xpR;%(pAZ%-}NmaHGuaFlJTEn$l)c377!VX<8C zULR~@`r!z>(0wav$Zpa-p|Fpm2sPyuVQzZT0dj()J{c$P1d?b%P0 zOMY{|Y8%?=AvM+1Cj$*k&q3V9zDrESX)^mxCqRO)jopp@Q8Zog>O<&7o%l1Nx-SL_ zb#ELz6*GO6IeoldWvc0+z8KTP+XP#-xr!G_p%*u1GleB+0?g|H=Z{>6qRV}^0D zj+5yu}ZCIoP^JxKh=GcYtv=HHl9yU%Gh!9 z>P{=W14!yQrI8Tg_wq5jwEY>or=Pm{rCJ_5g7d{l7&mMPsRsVk2(&nrerx)hUg{;+ zvE%XMI+h`U!O0Ym993Vx8E7MA3?i$Pa3(mr0v#d*tKjmaAXfky)089Fb-2df=92!@eUj+Yy-z z?|jX6O^k1Uq$U4&U901Kf8dP>iHFvXPRcP-T|g*$msKH{jfYy%Zz(A#BvU^X}2a2L^BW!SJU*HkE`251lR78@;ny;0K9m zT$x4M+^b?;@!*(O_+038g~ktYfwS*_4qw6Q>V|zeZML%79BJDP!`SH+j|NJchqOtb zgJGPY6_n7pkTyLKDI-&9f%EKzW1AlYf(6C1)D!2eYQ9{*0o0f=UUD zOi60}Dcom)p>p&}&xD|_Ng;|ANamIWf<~tTM%Aw(R>?9Zs`X>ZnED#1?#X853j@P4Sh)-fRE~anly26whz*dU19M6up=ac49|fnqdvZ1+7i0xVc}*WvG3VR? z>0f&Q4t>FY6SSsQTJ>(}K+9qPlm5r`vV^c+0zz*YP;mTw&woS4WuTJ0y#I%$?VD z@bnaCrW<-+j^Wk2q_O7VA+0}ALDKb^DiyhTr}?W}JAt4tKDf@~wYLQ*TI-rz&YYMu zQRg`%rUhY3AcQ%6(tgjY{VDIVW#%e|%Ma-PZl0Q*Q?5UCmFZ~+f%3?bCy#&0<`Oi) z`I+(pn=|;h`7x(R$>&dwTQ)O-k51^`OHI3tZn@=!ilXQ$k;bwx@pz7dCMMHek{?{7 zY(K4^cXpzMlfsp=Z&$lMa*5m@9op8B<(#OTjZqH$ND_f-X?Db*6;>v298$cd32N85 zh$|5EJimt~khj|0nGqQVA8TK=r!L}1eM zH?eIpY0V9y&t0c1&%lv=pA2r4y>FA;KpWJGZRys3xTKLoBnxf8E^c#`+Bv>D1t_6B}AHmPql?<^xz=m4r2N$7zcOx&y|Uva$LpHbJzDXh9&osv@Lh+lfz~a1mjB z)%{zb!tZLDe^+<;W<3TVHLKs&>h@p2;J@JHU;lh91!$k15BD$p*IV4x0%81~TJw-X z_Wv3h-c0~7cJ}eb-)Wcs1ST&C(1`;9D#yTIRepa5vg!28XVra7F0NnxrnScZZHut; zbg9I71f$*GM8N-FF7{K9bWtXye(Y~FFMqeihL5y6JFMpe&D-l&&b>FFm(H<;{bvq1 zy1QRoX!`l?nD3!69`LWy_k>y9-PbL%RYmb}2D{y>$?x^o%Pqf&MmVeW>7s`%7Z;4z zWshA5<2tN!S1q)%-ty7D{fAE7{f~dVVtuiHDBxAo1Tptv$iWm-?&)n zObRy-6fgrd`3_g}ws=Jjv69E2*1^XqVIOLb=dkL|tAdT>YiMQZTm^>KKkPA@O=IEw z_`^w4Pz3c+arAZ5N7ns$iM{dHIK}PX)3@+&gc3@fqqB5MpM>(Fa!btuUxrH<8z~xK z4cw%=<1fk|vF&})BC=v3i<6+U{ zci)4*24y|Ps-{4;sN#p3clYHG-SK=X9M(Mr*3$s@c-er?a8X=R9@kbW9VrSa+zhP3 z*LwR}qOydXY{ZlpDz>j%70L8^b1CkIg+`AL!!^Y~~`y;X2 z_vSy`cz(m0nD%fQL>YVAfPSU^Ts?Z^ZLf*O5B;cXhZ&5nT~#{A3|Va7u4?F52-iwN z_hlK9I^w&(=^8G!G!#umI&Hl2!W|ZF6-jjwwCJfs$m&|d-0LrVL$pU^@l$~(JNDeV@Yf~#*Z@kke^2YhA zcCm#6QzxQhMM^Fd10vz#Dy27D7E;{1O=&)#X+HU_M;HPSjxr}2f*By_WAO=-h zJ6rv_*B;h>hy|QT`?u$}b+f%IUZ$$yfJHp|KC>DNh$S_g zkn6nl`qxN3Kw+e(a0_T%pH7hXbM_nUa{>J zOhT`dM|g*furR`x&`?VzU&Zq^{ z*A+Zm+#jXk*=%(ET9^D286stJOz&hb_oC;btVOU=*u4gkx8wX569j>2g zp7yz`O({V(JlXJ2bGLH021^W(UiHZ6VF;PASVVqs@O%U~OL$ z)4#pjcX%~4x+ublV|s1Z^tIAHOA~@^u8e@(&PKU%g7{_6Q5a*z8amo+_l=zS=w}mQ zleJXt0eVe)a$Rn#{OvOquFB2&pcNW(b<@YMU4}Nh_&JeAAm}px{y9nbV%p)B=b2jJ zBZzh8h`xSiYu_4>$Ni1A6Q>+2BZMtoLP|RbzAB7`-dASevP&8-9;Vg2kf6#Bd|C5b zA3g?Aix-1{QMG%ZZ z{EH@1)&8(I%ZML@i#s0o1bOLP;$$5h3r`&nONHz9s(1FLHoKgO6MZPihJU~|LFK)9 z1;+D-UH6eT5Rc1DgGns)*NO*t5ymfXE^Hf{4{j*t)6 z9o|*1=R9(X`V;w>cy0MPUhO{4v_T~pyytsg`FUCNPl7WOw{rk?XnZCyoX+x`rXJR9MR%wK#llZ96_!J1@qhqJv|D)w^no+Ud}-!nOD2}{8q|Mg8k+CP?cxAe}R zx24#w0{U`-EtGjfYJr%0C0g~rt_YE8veHaJ{S44I? zBNhe^3!|4?cnn>0qz(K}a{nsG^8`x8cZVf138ucSEEvFkyJeRax5nHaNu63i`&z%h)|Gu{&_YkQ2ZA zg(hP<#T6Y}j@(F7ju%q_c1)i7Gx;G#-aV5h6c-1wB?{Muy|o#W&J)S&vt^iu|HQq$ zT(d0)^fkB|3x6=ZCgYAx(d*BWts1XG-)Ko&8hI=PKBdRFln@a*35=R8(snm5=Ka_ zQ~xO&PWBNG^5#>@$FH*RLi0)n7U5lKROpB%G0B_rYu0$(a1>4ocCK)HSl^+F%O+8# zjvs|IWQ?9R@B%%KV-*@4ZdOi&N2#aiJ;$X$;(Bnb`3h}Zk`tBg#*C7w+B~#xQythP z>GlHM09+I`?-5xKavz7);Y66VL_L??8pVZaludSs$flA;IcDU(uS{>KcmS~jJx74LJ{9zH2zS^8(?6KRF=OJ;|mf8<;-3|#AV`MyaUwVn#a=t8c-@ahx zW&~fw;AL3q>#z%>A8_J!PWfUTD$do`$jJzW^e&k)+YEvL3R%{2x!}13M*qm-2i$;P zCG~THxf}MEC^avjzp6vBYVxBI+$EZy=Y(!T&wRdh15c=ZW?#NB#IKlc(-`yp02Az8dQ?JTo?$AF;Mh7$GHS|`Q#e* z?iq`^!%*3CcenaQJ?sz6keB$hsue0$=xqVN$oyCS+}P|&pz?hCqQ%t7POXG+V9C>4 zZ_sp~nVMt&SwZZn*?bK6-4q9`hw;5U?p+=T(Y~6itiAdkI$n1A<}_x=;d!Pe6K#IE zt}vFYc%1mq9)f<=|209*$EYdtwxv$iTL0S@LBTzceZ@_|+PL$D7Q|&%8@G*zfnHL# z3i*9u3S;*Id~=2x=dI5ok{w7n8|lbwH2Js;@j^ZU{9s!!Hp9l-q|#&UmcJ*8=Od~8 zDj>C@>*CS1uPTT$KT<4+sR;^m*w3w|n82Y5+c|sis*q<+AufT9P5Yd9$fnZOVWfKzrkaQd~o3=z< zxDy?!aAKy-ocO$QqQzZ7MQfzV2!tQ(yKYf^1uEV5J*znPcWg) zpWYzuB!7~1d`Y6p2BaS%V+b@t{T(B zz9X~2?Lo{oAq{^d3Pb0Y<`l4{X|R~iBsBDX?^8@0+D;=1nlS$iBjjP(Z45mLnKQUd zZTxsWqQ9c94J@n4tcLTl=HSul_BP?ZO&}GrKXUM!9gBE2J(6hn)^=|zn{!}#zd$WFXZgcs7@<8llUHKpVa8Z6hs zj49JFFqr~fglMT#>U$r{rDZjmo*Xf^zrW~5&JYj8IKz2}s}1e~zGt%0l}=AuQpiRQ zx}gfVDC>SdOYpi!;s#uMd$}uYnBawNN_))~<35r<#mW6LBZ*<_8Jl3u_TGVqoPe8Z zNrAiAH^@j8O?C7bP^MQV!S@z_yWQD*CGpJCEc3{y9|)MZBrDuS5|_fP&r@0F)VyKr zipA+-O?o6yy)Z4G=Xe1>W_V#}P&RsqzOJ;DlZNHxBb31RE5!b>=iQVu2^3h+AY}gk7;#LAo}P4$PGH-im?M-QZy|yTNR?9)?8) zx3_IQF(d-?#CX}hV~faTdlVA55c)Ddt(Q&@88t(TH3AUM&1G`vMF^2#HO_~g!x@vP zn}(RE%smwoh`tCh{(BJ`zqssn2U9E4$ak2!H-@5khnk$ky>Dn0K1=(K&Z)n`_T| zxoXOzxM4kJk+SC*g+K@G6|RHVw|K-Yr5msavWv@+g1X^L)yS#6s@$hP4y+6n4^5_^ zWTP+EKH6O$Y1ly-`|LhRQ;EA?S+#qQiAUyKm!`*JcCmp7h_L#IFZa0`#3B-6F9Lc2 zthA_APSm2xnt+QXnTR=p-1?;y13@Hut@SV$fGD_Wae8=vX~~a=o!JPHO$z>M2YIr| z*>`o9tWPG{i@sV~0x){Nv;#})-tAQipms>T1rZO!)c&w8PE_tVONe6_v;&BTftlU;8wq$gn zEvXRlO|_><*+i^DF;Gf7=Wq9JqRW0GzTFOrT>yd#E5%zO&Tj0H(6CayuN*dt{P>8R$vY0eY-wU}ZLc$`9b7W@BHh>ATk8V`L`%0>-J? z&fy>tmk-LGBgIIga(i!=xFXxWFj691aFqfQpy*F1UnuV-V;@?I*EUT=m<(NS9S?^h z#AlZ4Bnm9ZWz(RQP8Oet?8rme%`XALaW^H)ptdgtKiymLqb^H~(B=zfCT4_Df8Y!~ zr%Nk6*9NBXI)EXAtK^>Mwfz$-gXyzD3#xavYQ{s2 z49MP_8Mf~1x15yetO9L&_uAAtu-i3%xU2wPP|^bNi+Y_?tOio(`0>CF@_7NR2cF1| z=akF^QP|IE`pL*vR!2CF^R;N}QGV41fNekp_={)Jjf?391unWSh<6OEA04V^`FP!E z2Ka_&^qE!Dwu+QvWDKi15vxDsmkIOa;vt|X&|-#&xHYCQoNE-3L+-t^aEn!7UTaG6 z+09RH{d7xVd-JH|tI-#Q<2VUJfwj4wVZ zZgGScjT@%pVCN+JSc1^bZP~cW(Kmxzv~5{4%)z6KK?`SXN6Q^#nM?DtrRRG}k@F^! zsuYV5c^m+f@9k6pt(dePg@$0}_yA1Mw-jWd$jP)EDL3 z+Y^R_40ct#(TJ^4>FqKc6PeifT-zvzw$rC)2=j}Q>*v~|G0R2{`)o{f@Y?cJ+RahJ zaQ4oPDI(THbwH{W5MnT66f6DsL`l2CigXBKvV*s3kC(=?7$t&0@`DLyYT+hWDi!wT z4tUm`nEYuk%i#LVHMjOaqL6G+=S`2^_|1Zji(Z(B0=qbt0#*6!2ML1l?Kf{zGd`b{ z*Wb76+;Jv4?yceIK%osFt)K@KGVue%&&JfBKPRO+f^$o~N=3ufj%r~&st{^B$*G4O z?ABV$f_6B?e;y6ha?XrWk-zB>aUa#>s+(ZOQgYLW?K6Ck?e^@$N)S$JaCUFVc|q=oZk9K)hRM@2qt zq)OvVlrC=B_x_Cf)h~}f+)uJV)miK7kTZh1-28M%n$DQC5VLgQcoF@mYYM3PK3I}M zNAkm(eLetC*3F1ncTVris5OR2zn$srz{2ZtAjgll6iscMN_`Ock|XMT7t&}#bQ)7P zAs}UZyH2tzKD}?q}zn6N?A|zVMLKMKQz>s!&)Pi6STsPOqg>^5b#$M&V1kw>-A z?!cSSch~@~`yB^B$>!Vh8#ATZ5IS^Vh8L+CSk7^oMrz<#d$O+r&17%vSp)Hzfic6i z+2>++moqn;1*H6`plU$w#A5^dL+g~HtZ~ct@^B(HGj9rdDqL-M(k^Yo+*crZO4}}<+Q9V# z4&S+93*Hj75u!N;#mE|Mr_22v@;%CTQzzS_q@^u z+ul(#J1L-5)(z|9jEff0iytj${2-v+_6lDdE8#Hwa_F45L2r_s&Z=P!&8yiY<*nUR zWB65Pk5e_KU6n#l#L7yL$9yFudtmo6d&&)1H=s5-da22P9}=3-=7EqJKN!YNzwCtp zvP|WTTX;59UQ0(I52%RD19^0m?*zZ01&NCkYd|gXu>A%(F2Q1(4O0=ae9J^ z6?iTvLgfZG}l1i^`qMiAnW zMMh5Xcc2o$W5*e}GIOVfB^=5kv=^r^x6v;UXO|sxj8nqyI0>sDuCIzAFbKsJ#dlsr zVUOMgX(#ZrS&O4&G>rflv%s328_)n(YvN6m^s;~>D#hbO%oI>7uK~c{nT#f6I z8JODa#h1&h0Nf}-la5@|1F3{8 zDh3?B;DO`y98Pvt!FE{hb=1O+I}bU(lvsa7g~K?D`T7C_=aP&K)>9||?EoIL`*|{3 zPrTV)`R7P)m}3~t&}}}jt|SFOo-u0FuSg%Omv^*M=Q+ft7OuQvO!~JpD zU?+xWp6J1x(s%AWiJPg zd8D3nzw{|37BUyUQ;zYzh@iq&=5M3kAro&ow-Kty!9MF7rp%4sp4;A+K;w$zmmLbsaW_O{_P(rkAroaysr0ECd#!%hUPTle0f&MB zh**#!7GK?T2mXL)kaXM(`~ha5pPXWF>c7#=tJOF9{0@~TKe;!oSqV)n$)mE&|N$3DqJ-ogeN@V5r8WkoKn8)?X zJfbejy}~*utt`&=WnN}A)RgY7cscp)%H9IZ_%W!q0wAT7gicpZaKt)DkB(`oWKe)u zA8w~7lyA%J?e1+Il+KM4v$^0`IQ5{C_b1O+jr1tyVT9q8ReL`2Det}bX_tCf()o#j_?wO$M*I)sB{i>9U1riIn(4~lgw_0H4v z6d95xgcbVQA0_9hWXp5qEZenk{z#?SCK{W8#Lo8OYwAF$H*s?-*GDQ2a|H&Ei{=8=e zcH5!T;w>MqU)45-!>380?nnpM9;9q-v+ij}QP0*x(Nqxc>Xd&g8?0N1RDfp}yoQRj z8^8P3n*Blt(HH2$o|LQX(r)3sGDd(?;vSlc)HdJ)j|2n6flww?RtfS)0u3)`O@R{R zLjdh}NNcQS$;0W7tZ`uN^xhtWL9PSf!EQ5LVQ%Y!sC&=y0W>8RHn1d#EPbVCJfG#X zC2o3yj=ebW%PImF^I`u+qNd^WlWjr6&9FfrS|7m6n-A~E(I*4^0PHbI&X$>kh0Od0 z^Jr}3&L*X)Jt@U(JPntRQ~dFUwHfL@#IXfmsCY-mWg?XuA6t6`TWw^KI}cpq$i$*3 zps;@%D={r2MZNaQ1X&8K8S*&iDPem3cf=_iK5Gf#m39MoRA~VFG5H}wKy05wKiiNI zXbwA>baXLl_n?0x@RrkOlZc=?>Xy86%-Edn0ALhrJ!CMiXG@}%tt}H^i88E>rm+%h zU9n`Z)jNss*=M>1V)sbp<28d9uqtr}BSDeSx%qTK??qHw0+0Al^5N4 zAfI2_{(`#}ARyrF@R+vDg;Dbr7Mam@U(7L5eJCi0ZKy`J-sDrT5MGA6L<#Dy4tHLi z&^Cg|tt?d*AZhhsI35``G)}ytBvB{Plm;Up=RaK6;DfJ4T7uOYym08KxWmAi6#)J$_y8+|{#U%?xU}LrKn9-#c5yh@N zX7pmD_ad~oNf`G^5JPE9nd|*}osqM*tJVIkpnIc*5Q<(;y^oxXQPAhJg&-z6fGj^c ztt`g^!VlykU)58?ndLT6t7Pd$AygGW=5(PHKt3{MkJi3*-tPHCg`9C+tv5{aB0R+isW%S3u+pO^dYHk$%Ml zbGc}NvS-Pok0SmY5D(D7@%X;+?gRdvAmNgbkEIl^12wievsboVDR{G?w|NdeJ5#6h z%?fcBQxs6q5ZiHy5?vh2;J{DU9F1$9J?FV0GLp#$hrSA*4O{U<7c8fPn(hyr4N_ME zL72*yboRdAAs}UVL&3GxI!;A~?*i5lqZ{GRQ4s1*7`1>9OQEoSs~Bhl|2gs(hFknD z_ZmPk^ME>#8d(!60)xA64qbkbAB*-`2s|}~izQ=BdRR8dc@Haw5MvTQKMLv^(DAzs z_1Lp{48i|-`8wRLv$9ZLKOZsrGf=}tUKvREf>*&#%`}t>0h1ubJUZATOiBb8iz4tX}4Q$dZUmDPp|d=bg3+F zAm6n7RDtIq1uaShM?2BLeJfa37(yJ2z&*>hApT@WoV%fOSIWIgtb6`67lD*7S-;Be zd>_XXcVyn=k9h*7*!*KBk6uN2*OL-k6^n5A_YWMZLz0h zsEu(FsQx3OnFJoqH&UjBv;9U46SVyRp2Xau7Re(Z}6#9V#vhdWLA&_o?L7x*W_B@|4Nv@J`9XN>tT9(`58O9Q@k9oMyaopmvvmg6c5EO3{o-~G*(g5tq<(NsRfwm0Cg40!$R6ZF(` z46KK^Bs$&ZgQObx1C6ZT-3;D6rg$G-&$&uEOLR0?2feCeE4;`VuR_cR#AbDF>%TsD8nR{ zh+IB-i9||e0nl}Qgh~~o&jF>y(e-cBa6<(NCLN}HE#94As90zeYa&A3x%oqxVjn*W z@}+(eikbF72CAlg$QZ)oKtxNT~Bxd7~RNeb_{Af z(knDu)ocifnyt%-njUSZGNg=z9WUBFau3-`+8i4wMOzOake{T$v z^%HjCdL>!B_m#k+EOX=ob?6wx_O1_D-^oob6)I-vvc9%E8ropL{?s|Ky3MXQ*7cIR zGXE!Wo#kVpf?S2Zk)T$7R-1U`?!!{+aD~joOTebX8k9UY8k5VuqRNt#WO+ zOzOEdJ8uqL4v{CWBMZtfs;Z9*U@DWjY6Y?Ei3J*0Bal9JdRrDD=+w{Tsoz?#h-o;| z-;VcW#5jJ^mz%5UvZkT5aV(x@EmQg!D8$mA=SLK~Ub{`hrlB)W_U312+}9dN>-Zqn zcp*Q%9E!K&jOf?t9aj{YGv4;XI=0=v41^6LjY*_7*@i_-NHMu;?DUJ>=gwOBr9qNq zJz47q5KeV-98nUBhXLPwlRpp7h3H?aw~@8d4Xqt>DRZ%2+l`jT+r^DHh@8{)hJ(Vk z)S|SDj>PGPy^{MSxnC4hAakYT%MZ#A7mH&lS%k!$BFX?b-BN!cmQ 0: + # Process each trace in the traces array + for trace in data['traces']: + # Process each span in this trace + for span in trace.get('spans', []): + # Create a new object that combines span and trace data + combined_span = {} + + # Add all span fields first + for key, value in span.items(): + combined_span[key] = value + + # Add trace fields with prefix to avoid clashes + for key, value in trace.items(): + if key != 'spans': # Skip the spans array itself + # Check if key already exists in span + if key in span: + # Prefix with 'trace_' to avoid clash + combined_span[f'trace_{key}'] = value + else: + # No clash, add as is + combined_span[key] = value + + combined_spans.append(combined_span) + + except Exception as e: + print(f"Error processing {trace_file}: {e}") + continue + + return combined_spans + + +def main(): + # Get the directory where this script is located + script_dir = Path(__file__).parent + traces_dir = str(script_dir / "traces") + + print(f"Processing traces from: {traces_dir}") + + # Combine all spans + combined_spans = combine_spans_from_traces(traces_dir) + + print(f"Combined {len(combined_spans)} spans total") + + # Save to dataset.json + output_file = os.path.join(traces_dir, 'dataset.json') + + with open(output_file, 'w') as f: + json.dump(combined_spans, f, indent=2) + + print(f"Saved combined spans to: {output_file}") + + # Print some statistics + if combined_spans: + print(f"\nSample of first span keys: {list(combined_spans[0].keys())}") + print(f"Total spans: {len(combined_spans)}") + + # Count spans by type + span_types = {} + for span in combined_spans: + span_name = span.get('name', 'unknown') + span_types[span_name] = span_types.get(span_name, 0) + 1 + + print(f"\nSpan types distribution:") + for span_type, count in sorted(span_types.items()): + print(f" {span_type}: {count}") + + +if __name__ == "__main__": + main() diff --git a/noveum_customer_support_bt/traces/fetch_traces_api.py b/noveum_customer_support_bt/traces/fetch_traces_api.py new file mode 100755 index 0000000..5333af9 --- /dev/null +++ b/noveum_customer_support_bt/traces/fetch_traces_api.py @@ -0,0 +1,161 @@ +#!/usr/bin/env python3 +""" +Script to fetch traces from Noveum API and save them to a traces directory. +Supports batch fetching with pagination for large numbers of traces. +""" + +import os +import json +import requests +import argparse +import shutil +from dotenv import load_dotenv +from typing import List, Dict, Any + +# Load environment variables +load_dotenv() + +# Get API key and project from environment +api_key = os.getenv('NOVEUM_API_KEY') +project = 'noveum-ai-agent-rag-websearch' + +if not api_key: + print('Error: NOVEUM_API_KEY environment variable not found') + exit(1) + +# Common headers +headers = { + 'Authorization': f'Bearer {api_key}', + 'Content-Type': 'application/json' +} + +def clean_and_create_traces_dir(): + """Clean existing traces directory and create a new one""" + # Get the directory where this script is located + script_dir = os.path.dirname(os.path.abspath(__file__)) + traces_dir = os.path.join(script_dir, 'traces') + + if os.path.exists(traces_dir): + print(f"Cleaning existing traces directory: {traces_dir}") + shutil.rmtree(traces_dir) + + os.makedirs(traces_dir) + print(f"Created traces directory: {traces_dir}") + return traces_dir + +def fetch_traces_batch(size: int, from_offset: int = 0) -> Dict[str, Any]: + """Fetch a batch of traces from the API""" + traces_url = 'https://api.noveum.ai/api/v1/traces' + params = { + 'project': project, + 'size': size, + 'from': from_offset, + 'includeSpans': True + } + + print(f"Fetching traces: size={size}, from={from_offset}") + + try: + response = requests.get(traces_url, headers=headers, params=params) + response.raise_for_status() + + data = response.json() + print(f"Successfully fetched {len(data.get('traces', []))} traces") + return data + + except requests.exceptions.RequestException as e: + print(f'Error fetching traces: {e}') + if hasattr(e, 'response') and e.response is not None: + print(f'Response status: {e.response.status_code}') + print(f'Response text: {e.response.text}') + return None + +def save_traces_batch(traces_dir: str, batch_data: Dict[str, Any], batch_number: int): + """Save a batch of traces to a JSON file""" + filename = f"traces_batch_{batch_number:03d}.json" + filepath = os.path.join(traces_dir, filename) + + with open(filepath, 'w') as f: + json.dump(batch_data, f, indent=2) + + print(f"Saved batch {batch_number} to: {filepath}") + return filepath + +def main(): + global project + + parser = argparse.ArgumentParser(description='Fetch traces from Noveum API and save to traces directory') + parser.add_argument('count', type=int, help='Number of traces to fetch') + parser.add_argument('--project', type=str, default=project, help=f'Project name (default: {project})') + + args = parser.parse_args() + + # Update project if specified + project = args.project + + print(f"Fetching {args.count} traces for project: {project}") + + # Clean and create traces directory + traces_dir = clean_and_create_traces_dir() + + # Calculate number of batches needed + max_per_batch = 100 + num_batches = (args.count + max_per_batch - 1) // max_per_batch # Ceiling division + + print(f"Will fetch in {num_batches} batch(es) of up to {max_per_batch} traces each") + + total_fetched = 0 + batch_number = 1 + + for batch in range(num_batches): + # Calculate size and from_offset for this batch + remaining_traces = args.count - total_fetched + current_size = min(max_per_batch, remaining_traces) + current_from = batch * max_per_batch + + print(f"\n--- Batch {batch_number}/{num_batches} ---") + + # Fetch this batch + batch_data = fetch_traces_batch(current_size, current_from) + + if batch_data is None: + print(f"Failed to fetch batch {batch_number}") + break + + # Save this batch + save_traces_batch(traces_dir, batch_data, batch_number) + + # Update counters + traces_in_batch = len(batch_data.get('traces', [])) + total_fetched += traces_in_batch + + print(f"Batch {batch_number} complete: {traces_in_batch} traces") + print(f"Total fetched so far: {total_fetched}/{args.count}") + + # Check if we've fetched enough traces + if total_fetched >= args.count: + print(f"Reached target of {args.count} traces") + break + + # Check if there are more traces available + pagination = batch_data.get('pagination', {}) + has_more = pagination.get('has_more', False) + + if not has_more: + print("No more traces available from API") + break + + batch_number += 1 + + print(f"\n=== Summary ===") + print(f"Total traces fetched: {total_fetched}") + print(f"Batches created: {batch_number}") + print(f"Traces directory: {traces_dir}") + + # List all created files + files = [f for f in os.listdir(traces_dir) if f.endswith('.json')] + files.sort() + print(f"Created files: {', '.join(files)}") + +if __name__ == "__main__": + main() diff --git a/noveum_customer_support_bt/traces/traces/dataset.json b/noveum_customer_support_bt/traces/traces/dataset.json new file mode 100644 index 0000000..5578d29 --- /dev/null +++ b/noveum_customer_support_bt/traces/traces/dataset.json @@ -0,0 +1,20904 @@ +[ + { + "span_id": "4149abdc-acdc-448c-b9b0-b09e166e461a", + "trace_id": "6fe93abf-1413-49e8-8464-58558e25ab7b", + "parent_span_id": "817c9703-0062-4bd6-b3ee-b50a1121c2d2", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-22 16:58:42.084000000", + "end_time": "2025-10-22 16:58:42.084000000", + "duration_ms": 0.015000000000000001, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What are the latest breakthroughs in artificial intelligence?", + "query_length": 61, + "query_type": "routing_evaluation_query", + "classification.mode": "Web Search", + "classification.rag_keyword_score": 0, + "classification.web_keyword_score": 1, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "complex", + "query.intent": "general_knowledge", + "query.keyword_density": 0.125, + "query.domain_affinity": "general", + "routing.decision": "Routed to Web Search based on analysis", + "routing.rationale": "RAG score: 0, Web score: 1, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "RAG", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.019230769230769232, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: Web Search (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "6fe93abf-1413-49e8-8464-58558e25ab7b", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:58:42.084000000", + "trace_end_time": "2025-10-22 16:58:44.766000000", + "trace_duration_ms": 2681.9289999999996, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:58:42.084000000", + "updated_at": "2025-10-22 16:58:42.084000000" + }, + { + "span_id": "817c9703-0062-4bd6-b3ee-b50a1121c2d2", + "trace_id": "6fe93abf-1413-49e8-8464-58558e25ab7b", + "parent_span_id": "ff171ffc-dc6b-4cb4-9833-64c5622d9941", + "name": "agent.query_routing", + "start_time": "2025-10-22 16:58:42.084000000", + "end_time": "2025-10-22 16:58:44.766000000", + "duration_ms": 2681.799, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What are the latest breakthroughs in artificial intelligence?", + "query_length": 61, + "query_type": "routing_query", + "output_response": "Routed to Web Search for query processing", + "router.classification": "Web Search", + "router.confidence_score": 1, + "router.rag_keyword_score": 0, + "router.web_keyword_score": 1, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "6fe93abf-1413-49e8-8464-58558e25ab7b", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:58:42.084000000", + "trace_end_time": "2025-10-22 16:58:44.766000000", + "trace_duration_ms": 2681.9289999999996, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:58:42.084000000", + "updated_at": "2025-10-22 16:58:42.084000000" + }, + { + "span_id": "ad476d62-d6b5-4142-8771-29855db3f39e", + "trace_id": "6fe93abf-1413-49e8-8464-58558e25ab7b", + "parent_span_id": "817c9703-0062-4bd6-b3ee-b50a1121c2d2", + "name": "agent.web_search_generation", + "start_time": "2025-10-22 16:58:42.084000000", + "end_time": "2025-10-22 16:58:44.766000000", + "duration_ms": 2681.732, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "web_search_agent", + "agent.operation": "web_search_generation", + "agent.capabilities": [ + "web_search", + "content_synthesis", + "response_generation" + ], + "agent.id": "noveum_web_search_agent", + "input_query": "What are the latest breakthroughs in artificial intelligence?", + "query_length": 61, + "query_type": "web_search_query", + "output_response": "Web Search Answer: The web search did not yield specific results regarding the latest breakthroughs in artificial intelligence. However, I can provide a general overview based on knowledge up to October 2023.\n\nRecent br...", + "web_search.results_count": 1, + "web_search.context_length": 163, + "web_search.response_length": 1040, + "web_search.mode": "external_web_search" + }, + "events": [], + "links": [], + "trace_trace_id": "6fe93abf-1413-49e8-8464-58558e25ab7b", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:58:42.084000000", + "trace_end_time": "2025-10-22 16:58:44.766000000", + "trace_duration_ms": 2681.9289999999996, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:58:42.084000000", + "updated_at": "2025-10-22 16:58:42.084000000" + }, + { + "span_id": "ff171ffc-dc6b-4cb4-9833-64c5622d9941", + "trace_id": "6fe93abf-1413-49e8-8464-58558e25ab7b", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-22 16:58:42.084000000", + "end_time": "2025-10-22 16:58:44.766000000", + "duration_ms": 2681.902, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What are the latest breakthroughs in artificial intelligence?", + "query.length": 61, + "processing.mode": "Web Search", + "processing.time_seconds": 2.6818161010742188, + "processing.response_length": 1040, + "processing.sources_count": 1, + "output_response": "Final Answer: The web search did not yield specific results regarding the latest breakthroughs in artificial intelligence. However, I can provide a general overview based on knowledge up to October 2023.\n\nRecent br...", + "final_answer_mode": "Web Search", + "query_processed.input_query": "What are the latest breakthroughs in artificial intelligence?", + "query_processed.output_response": "Successfully processed query using Web Search, generated 1040 character response", + "query_processed.mode": "Web Search", + "query_processed.processing_time": 2.6818161010742188, + "query_processed.response_length": 1040 + }, + "events": [], + "links": [], + "trace_trace_id": "6fe93abf-1413-49e8-8464-58558e25ab7b", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:58:42.084000000", + "trace_end_time": "2025-10-22 16:58:44.766000000", + "trace_duration_ms": 2681.9289999999996, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:58:42.084000000", + "updated_at": "2025-10-22 16:58:42.084000000" + }, + { + "span_id": "467cb806-a6ef-43da-9739-570730dd244b", + "trace_id": "6fe93abf-1413-49e8-8464-58558e25ab7b", + "parent_span_id": "ad476d62-d6b5-4142-8771-29855db3f39e", + "name": "agent.llm_model_execution", + "start_time": "2025-10-22 16:58:42.438000000", + "end_time": "2025-10-22 16:58:44.766000000", + "duration_ms": 2327.706, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What are the latest breakthroughs in artificial intelligence?", + "query_length": 61, + "query_type": "web_search_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 2.3276820182800293, + "model.latency_ms": 2327.6820182800293, + "model.start_time": 1761152322.438459, + "model.end_time": 1761152324.766141, + "model.performance_tier": "medium", + "model.prompt_tokens": 167, + "model.completion_tokens": 260, + "model.total_tokens": 427, + "model.tokens_per_second": 183.44430065903882, + "model.estimated_cost": 0.01281, + "model.efficiency_score": 2.4355971896955504, + "model.response_length": 1040, + "model.response_quality": "high", + "model.output_response": "Model Response: The web search did not yield specific results regarding the latest breakthroughs in artificial intelligence. However, I can provide a general overview based on knowledge up to October 2023.\n\nRecent br...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "6fe93abf-1413-49e8-8464-58558e25ab7b", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:58:42.084000000", + "trace_end_time": "2025-10-22 16:58:44.766000000", + "trace_duration_ms": 2681.9289999999996, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:58:42.084000000", + "updated_at": "2025-10-22 16:58:42.084000000" + }, + { + "span_id": "a80e4596-01ea-44e0-a590-13e347857f2a", + "trace_id": "6fe93abf-1413-49e8-8464-58558e25ab7b", + "parent_span_id": "ad476d62-d6b5-4142-8771-29855db3f39e", + "name": "agent.web_search_evaluation_metrics", + "start_time": "2025-10-22 16:58:44.766000000", + "end_time": "2025-10-22 16:58:44.766000000", + "duration_ms": 0.008, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "web_search_evaluation_metrics", + "agent.capabilities": [ + "web_search_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What are the latest breakthroughs in artificial intelligence?", + "query_length": 61, + "query_type": "web_search_evaluation_query", + "web_search.results_count": 1, + "web_search.context_length": 163, + "web_search.context_synthesized": "Context from 1 web sources", + "web_search.search_effectiveness": 0.2, + "web_search.context_quality": "low", + "web_search.source_diversity": 1, + "web_search.information_synthesis": "low", + "web_search.external_knowledge_utilization": 0.163, + "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What+are+the+latest+breakthroughs+in+artificial+intelligence?\n\nUser Question: What are the latest breakthroughs in artificial intelligence?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", + "prompt.prompt_length": 669, + "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What+are+the+latest+breakthroughs+in+artificial+intelligence?", + "prompt.instruction_following": "web_search_optimized", + "response.answer_length": 1040, + "response.answer_completeness": "complete", + "response.response_quality": "medium", + "response.source_citation": 1, + "response.context_utilization": 6.380368098159509, + "output_response": "Web Search Answer: The web search did not yield specific results regarding the latest breakthroughs in artificial intelligence. However, I can provide a general overview based on knowledge up to October 2023.\n\nRecent br...", + "evaluation.search_effectiveness": 0.2, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 1, + "evaluation.context_relevance": "low", + "evaluation.overall_quality": "low", + "evaluation.ready_for_production": true, + "web_search.search_strategy": "duckduckgo_api", + "web_search.real_time_data": true, + "web_search.external_sources": 1, + "web_search.information_freshness": "current", + "web_search.knowledge_synthesis": "single_source" + }, + "events": [], + "links": [], + "trace_trace_id": "6fe93abf-1413-49e8-8464-58558e25ab7b", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:58:42.084000000", + "trace_end_time": "2025-10-22 16:58:44.766000000", + "trace_duration_ms": 2681.9289999999996, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:58:42.084000000", + "updated_at": "2025-10-22 16:58:42.084000000" + }, + { + "span_id": "409f29f6-21c7-400a-a9e0-11c87c4313cb", + "trace_id": "ec7c6cae-9bbd-47f4-a8ae-2d6d9b5011ba", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-22 16:58:32.801000000", + "end_time": "2025-10-22 16:58:41.077000000", + "duration_ms": 8276.701, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What's happening in the software development world today?", + "query.length": 57, + "processing.mode": "Web Search", + "processing.time_seconds": 8.276506900787354, + "processing.response_length": 1424, + "processing.sources_count": 1, + "output_response": "Final Answer: The web search did not yield any specific results regarding current events in the software development world. Therefore, I cannot provide detailed information on the latest trends, technologies, or ne...", + "final_answer_mode": "Web Search", + "query_processed.input_query": "What's happening in the software development world today?", + "query_processed.output_response": "Successfully processed query using Web Search, generated 1424 character response", + "query_processed.mode": "Web Search", + "query_processed.processing_time": 8.276506900787354, + "query_processed.response_length": 1424 + }, + "events": [], + "links": [], + "trace_trace_id": "ec7c6cae-9bbd-47f4-a8ae-2d6d9b5011ba", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:58:32.800000000", + "trace_end_time": "2025-10-22 16:58:41.077000000", + "trace_duration_ms": 8276.788, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:58:32.800000000", + "updated_at": "2025-10-22 16:58:32.800000000" + }, + { + "span_id": "43389a18-a566-41e7-8db5-f688f651a106", + "trace_id": "ec7c6cae-9bbd-47f4-a8ae-2d6d9b5011ba", + "parent_span_id": "af400e17-8b20-4218-834b-9330ccaa2635", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-22 16:58:32.801000000", + "end_time": "2025-10-22 16:58:32.801000000", + "duration_ms": 0.047, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What's happening in the software development world today?", + "query_length": 57, + "query_type": "routing_evaluation_query", + "classification.mode": "Web Search", + "classification.rag_keyword_score": 0, + "classification.web_keyword_score": 1, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "complex", + "query.intent": "general_knowledge", + "query.keyword_density": 0.125, + "query.domain_affinity": "general", + "routing.decision": "Routed to Web Search based on analysis", + "routing.rationale": "RAG score: 0, Web score: 1, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "RAG", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.019230769230769232, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: Web Search (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "ec7c6cae-9bbd-47f4-a8ae-2d6d9b5011ba", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:58:32.800000000", + "trace_end_time": "2025-10-22 16:58:41.077000000", + "trace_duration_ms": 8276.788, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:58:32.800000000", + "updated_at": "2025-10-22 16:58:32.800000000" + }, + { + "span_id": "6b431bca-af7e-4f7e-aedb-b103d7f47cae", + "trace_id": "ec7c6cae-9bbd-47f4-a8ae-2d6d9b5011ba", + "parent_span_id": "af400e17-8b20-4218-834b-9330ccaa2635", + "name": "agent.web_search_generation", + "start_time": "2025-10-22 16:58:32.801000000", + "end_time": "2025-10-22 16:58:41.077000000", + "duration_ms": 8276.225999999999, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "web_search_agent", + "agent.operation": "web_search_generation", + "agent.capabilities": [ + "web_search", + "content_synthesis", + "response_generation" + ], + "agent.id": "noveum_web_search_agent", + "input_query": "What's happening in the software development world today?", + "query_length": 57, + "query_type": "web_search_query", + "output_response": "Web Search Answer: The web search did not yield any specific results regarding current events in the software development world. Therefore, I cannot provide detailed information on the latest trends, technologies, or ne...", + "web_search.results_count": 1, + "web_search.context_length": 159, + "web_search.response_length": 1424, + "web_search.mode": "external_web_search" + }, + "events": [], + "links": [], + "trace_trace_id": "ec7c6cae-9bbd-47f4-a8ae-2d6d9b5011ba", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:58:32.800000000", + "trace_end_time": "2025-10-22 16:58:41.077000000", + "trace_duration_ms": 8276.788, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:58:32.800000000", + "updated_at": "2025-10-22 16:58:32.800000000" + }, + { + "span_id": "af400e17-8b20-4218-834b-9330ccaa2635", + "trace_id": "ec7c6cae-9bbd-47f4-a8ae-2d6d9b5011ba", + "parent_span_id": "409f29f6-21c7-400a-a9e0-11c87c4313cb", + "name": "agent.query_routing", + "start_time": "2025-10-22 16:58:32.801000000", + "end_time": "2025-10-22 16:58:41.077000000", + "duration_ms": 8276.465999999999, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What's happening in the software development world today?", + "query_length": 57, + "query_type": "routing_query", + "output_response": "Routed to Web Search for query processing", + "router.classification": "Web Search", + "router.confidence_score": 1, + "router.rag_keyword_score": 0, + "router.web_keyword_score": 1, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "ec7c6cae-9bbd-47f4-a8ae-2d6d9b5011ba", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:58:32.800000000", + "trace_end_time": "2025-10-22 16:58:41.077000000", + "trace_duration_ms": 8276.788, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:58:32.800000000", + "updated_at": "2025-10-22 16:58:32.800000000" + }, + { + "span_id": "37b153b2-d02b-4b7e-9229-1ba3eaae44ee", + "trace_id": "ec7c6cae-9bbd-47f4-a8ae-2d6d9b5011ba", + "parent_span_id": "6b431bca-af7e-4f7e-aedb-b103d7f47cae", + "name": "agent.llm_model_execution", + "start_time": "2025-10-22 16:58:33.172000000", + "end_time": "2025-10-22 16:58:41.077000000", + "duration_ms": 7904.722, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What's happening in the software development world today?", + "query_length": 57, + "query_type": "web_search_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 7.904651880264282, + "model.latency_ms": 7904.651880264282, + "model.start_time": 1761152313.172655, + "model.end_time": 1761152321.077307, + "model.performance_tier": "slow", + "model.prompt_tokens": 165, + "model.completion_tokens": 356, + "model.total_tokens": 521, + "model.tokens_per_second": 65.91055594754174, + "model.estimated_cost": 0.01563, + "model.efficiency_score": 2.7332053742802302, + "model.response_length": 1424, + "model.response_quality": "high", + "model.output_response": "Model Response: The web search did not yield any specific results regarding current events in the software development world. Therefore, I cannot provide detailed information on the latest trends, technologies, or ne...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "ec7c6cae-9bbd-47f4-a8ae-2d6d9b5011ba", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:58:32.800000000", + "trace_end_time": "2025-10-22 16:58:41.077000000", + "trace_duration_ms": 8276.788, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:58:32.800000000", + "updated_at": "2025-10-22 16:58:32.800000000" + }, + { + "span_id": "cc941b8e-6625-43e9-a0a9-7328af24f12f", + "trace_id": "ec7c6cae-9bbd-47f4-a8ae-2d6d9b5011ba", + "parent_span_id": "6b431bca-af7e-4f7e-aedb-b103d7f47cae", + "name": "agent.web_search_evaluation_metrics", + "start_time": "2025-10-22 16:58:41.077000000", + "end_time": "2025-10-22 16:58:41.077000000", + "duration_ms": 0.026, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "web_search_evaluation_metrics", + "agent.capabilities": [ + "web_search_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What's happening in the software development world today?", + "query_length": 57, + "query_type": "web_search_evaluation_query", + "web_search.results_count": 1, + "web_search.context_length": 159, + "web_search.context_synthesized": "Context from 1 web sources", + "web_search.search_effectiveness": 0.2, + "web_search.context_quality": "low", + "web_search.source_diversity": 1, + "web_search.information_synthesis": "low", + "web_search.external_knowledge_utilization": 0.159, + "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What's+happening+in+the+software+development+world+today?\n\nUser Question: What's happening in the software development world today?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", + "prompt.prompt_length": 661, + "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What's+happening+in+the+software+development+world+today?", + "prompt.instruction_following": "web_search_optimized", + "response.answer_length": 1424, + "response.answer_completeness": "complete", + "response.response_quality": "medium", + "response.source_citation": 1, + "response.context_utilization": 8.955974842767295, + "output_response": "Web Search Answer: The web search did not yield any specific results regarding current events in the software development world. Therefore, I cannot provide detailed information on the latest trends, technologies, or ne...", + "evaluation.search_effectiveness": 0.2, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 1, + "evaluation.context_relevance": "low", + "evaluation.overall_quality": "low", + "evaluation.ready_for_production": true, + "web_search.search_strategy": "duckduckgo_api", + "web_search.real_time_data": true, + "web_search.external_sources": 1, + "web_search.information_freshness": "current", + "web_search.knowledge_synthesis": "single_source" + }, + "events": [], + "links": [], + "trace_trace_id": "ec7c6cae-9bbd-47f4-a8ae-2d6d9b5011ba", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:58:32.800000000", + "trace_end_time": "2025-10-22 16:58:41.077000000", + "trace_duration_ms": 8276.788, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:58:32.800000000", + "updated_at": "2025-10-22 16:58:32.800000000" + }, + { + "span_id": "3b06d098-6eb7-4bb6-885e-443ddb28cc41", + "trace_id": "343764b8-7ac9-4549-93cf-8abc11ceba6f", + "parent_span_id": "5c2aa1cf-56a8-4fd5-af90-90a94e723bb5", + "name": "agent.query_routing", + "start_time": "2025-10-22 16:58:26.542000000", + "end_time": "2025-10-22 16:58:31.796000000", + "duration_ms": 5253.835, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What are the newest features in cloud computing?", + "query_length": 48, + "query_type": "routing_query", + "output_response": "Routed to RAG for query processing", + "router.classification": "RAG", + "router.confidence_score": 1, + "router.rag_keyword_score": 1, + "router.web_keyword_score": 0, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "343764b8-7ac9-4549-93cf-8abc11ceba6f", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:58:26.542000000", + "trace_end_time": "2025-10-22 16:58:31.796000000", + "trace_duration_ms": 5253.973, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:58:26.542000000", + "updated_at": "2025-10-22 16:58:26.542000000" + }, + { + "span_id": "5c2aa1cf-56a8-4fd5-af90-90a94e723bb5", + "trace_id": "343764b8-7ac9-4549-93cf-8abc11ceba6f", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-22 16:58:26.542000000", + "end_time": "2025-10-22 16:58:31.796000000", + "duration_ms": 5253.913, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What are the newest features in cloud computing?", + "query.length": 48, + "processing.mode": "RAG", + "processing.time_seconds": 5.253868818283081, + "processing.response_length": 837, + "processing.sources_count": 5, + "output_response": "Final Answer: The provided context does not specifically mention any new features in cloud computing as a general topic. However, it does highlight several recent enhancements related to the Noveum.ai platform, whi...", + "final_answer_mode": "RAG", + "query_processed.input_query": "What are the newest features in cloud computing?", + "query_processed.output_response": "Successfully processed query using RAG, generated 837 character response", + "query_processed.mode": "RAG", + "query_processed.processing_time": 5.253868818283081, + "query_processed.response_length": 837 + }, + "events": [], + "links": [], + "trace_trace_id": "343764b8-7ac9-4549-93cf-8abc11ceba6f", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:58:26.542000000", + "trace_end_time": "2025-10-22 16:58:31.796000000", + "trace_duration_ms": 5253.973, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:58:26.542000000", + "updated_at": "2025-10-22 16:58:26.542000000" + }, + { + "span_id": "6b838ed6-f5bb-4bcf-a76b-bac283bf7571", + "trace_id": "343764b8-7ac9-4549-93cf-8abc11ceba6f", + "parent_span_id": "3b06d098-6eb7-4bb6-885e-443ddb28cc41", + "name": "agent.llm-rag", + "start_time": "2025-10-22 16:58:26.542000000", + "end_time": "2025-10-22 16:58:31.796000000", + "duration_ms": 5253.519, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "rag_agent", + "agent.operation": "llm-rag", + "agent.capabilities": [ + "document_retrieval", + "context_generation", + "response_generation" + ], + "agent.id": "noveum_rag_agent", + "input_query": "What are the newest features in cloud computing?", + "query_length": 48, + "query_type": "rag_query", + "output_response": "RAG Answer: The provided context does not specifically mention any new features in cloud computing as a general topic. However, it does highlight several recent enhancements related to the Noveum.ai platform, whi...", + "rag.context_length": 2579, + "rag.sources_count": 5, + "rag.answer_length": 837, + "rag.mode": "retrieval_augmented_generation" + }, + "events": [], + "links": [], + "trace_trace_id": "343764b8-7ac9-4549-93cf-8abc11ceba6f", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:58:26.542000000", + "trace_end_time": "2025-10-22 16:58:31.796000000", + "trace_duration_ms": 5253.973, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:58:26.542000000", + "updated_at": "2025-10-22 16:58:26.542000000" + }, + { + "span_id": "f51d8f1e-eae3-488d-9a96-49ab97377851", + "trace_id": "343764b8-7ac9-4549-93cf-8abc11ceba6f", + "parent_span_id": "3b06d098-6eb7-4bb6-885e-443ddb28cc41", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-22 16:58:26.542000000", + "end_time": "2025-10-22 16:58:26.542000000", + "duration_ms": 0.082, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What are the newest features in cloud computing?", + "query_length": 48, + "query_type": "routing_evaluation_query", + "classification.mode": "RAG", + "classification.rag_keyword_score": 1, + "classification.web_keyword_score": 0, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "documentation", + "query.keyword_density": 0.125, + "query.domain_affinity": "noveum", + "routing.decision": "Routed to RAG based on analysis", + "routing.rationale": "RAG score: 1, Web score: 0, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "Web Search", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.019230769230769232, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: RAG (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "343764b8-7ac9-4549-93cf-8abc11ceba6f", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:58:26.542000000", + "trace_end_time": "2025-10-22 16:58:31.796000000", + "trace_duration_ms": 5253.973, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:58:26.542000000", + "updated_at": "2025-10-22 16:58:26.542000000" + }, + { + "span_id": "93a0fd69-47b6-4e7c-b8ab-8eaa5aeac240", + "trace_id": "343764b8-7ac9-4549-93cf-8abc11ceba6f", + "parent_span_id": "6b838ed6-f5bb-4bcf-a76b-bac283bf7571", + "name": "agent.llm_model_execution", + "start_time": "2025-10-22 16:58:27.411000000", + "end_time": "2025-10-22 16:58:31.373000000", + "duration_ms": 3961.481, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What are the newest features in cloud computing?", + "query_length": 48, + "query_type": "rag_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 3.9613850116729736, + "model.latency_ms": 3961.3850116729736, + "model.start_time": 1761152307.411936, + "model.end_time": 1761152311.373321, + "model.performance_tier": "medium", + "model.prompt_tokens": 780, + "model.completion_tokens": 209, + "model.total_tokens": 989, + "model.tokens_per_second": 249.66015600243944, + "model.estimated_cost": 0.029670000000000002, + "model.efficiency_score": 0.846309403437816, + "model.response_length": 837, + "model.response_quality": "high", + "model.output_response": "Model Response: The provided context does not specifically mention any new features in cloud computing as a general topic. However, it does highlight several recent enhancements related to the Noveum.ai platform, whi...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "343764b8-7ac9-4549-93cf-8abc11ceba6f", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:58:26.542000000", + "trace_end_time": "2025-10-22 16:58:31.796000000", + "trace_duration_ms": 5253.973, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:58:26.542000000", + "updated_at": "2025-10-22 16:58:26.542000000" + }, + { + "span_id": "48cbc1df-f03a-4c49-90e5-19a2047c1c7b", + "trace_id": "343764b8-7ac9-4549-93cf-8abc11ceba6f", + "parent_span_id": "6b838ed6-f5bb-4bcf-a76b-bac283bf7571", + "name": "agent.rag_evaluation_metrics", + "start_time": "2025-10-22 16:58:31.373000000", + "end_time": "2025-10-22 16:58:31.373000000", + "duration_ms": 0.061, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "rag_evaluation_metrics", + "agent.capabilities": [ + "retrieval_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What are the newest features in cloud computing?", + "query_length": 48, + "query_type": "rag_evaluation_query", + "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/changelog):\n- \ud83d\udcc8 Enhanced Dashboard Analytics: Real-time request tracking, latency monitoring, and cost analysis\n- \ud83d\udd0d Improved Logs Interface: Better search functionality, detailed trace views, and enhanced debugging capabilities\n- \ud83d\udc65 Team Management: Advanced member mana...", + "retrieval.context_length": 2579, + "retrieval.sources_count": 5, + "retrieval.context_quality": "high", + "retrieval.effectiveness": 1, + "retrieval.context_utilization": 2.579, + "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/changelog):\n- \ud83d\udcc8 Enhanced Dashboard Analytics: Real-time request tracking, latency monitoring, and cost analysis\n- \ud83d\udd0d Improved Logs Interface: Better search functionality, detailed trace views, and enhanced debugging capabilities\n- \ud83d\udc65 Team Management: Advanced member management with role-based access control and invitation system\n- \ud83d\udee1\ufe0f Security Improvements: Enhanced API key management and secure credential storage\n4 months ago\n- \ud83d\udcca Advanced Metrics Collection: Comprehensive telemetry system with custom metrics ...\n\nSource 2 (https://noveum.ai/en/changelog):\n- \ud83d\udcdd Documentation Updates: Comprehensive guides for integration and best practices\n5 months ago\n- \ud83c\udf89 Platform Foundation: Initial release of Noveum.ai AI observability platform\n- \ud83d\ude80 Core Features: AI Gateway integration, basic evaluation metrics, and monitoring capabilities\n- \ud83d\udcc8 Evaluation Jobs: Automated model evaluation with configurable metrics and reporting\n- \ud83d\udd0c Provider Integrations: Support for major AI providers and custom model deployments...\n\nSource 3 (https://noveum.ai/docs/platform/dashboard):\n- Bulk Export: Download multiple traces simultaneously\n- Comparative Analysis: Compare performance across multiple traces\nReal-Time Updates\n- Live Refresh: Automatic updates as new traces arrive\n- Connection Monitoring: Real-time status of your trace ingestion pipeline\n- Performance Indicators: Live metrics for system health monitoring\nReady to dive deeper? Explore Projects & Environments to organize your AI applications, or check out Team Collaboration to share insights with your team.\nGet Earl...\n\nSource 4 (https://noveum.ai/en/changelog):\nChangelog\nStay up to date with the latest changes in our product.\n2 months ago\n- \ud83c\udfaf Major Platform Evolution: NovaEval Framework - Launched comprehensive AI model evaluation framework with 20+ built-in scorers including accuracy, RAG metrics, conversational scoring, and specialized agent evaluation tools. Features production-grade deployment on Docker and Kubernetes. (GitHub | PyPI Package)\n- \ud83d\ude80 Next-Generation Traces UI - Complete redesign featuring three-pane layout with directory tree navigatio...\n\nSource 5 (https://noveum.ai/en/blog):\n8/12/2025\nDiscover how Noveum.ai provides comprehensive tracing and observability for AI applications, from development debugging to production optimization.\nShashank Agarwal\n3/3/2025\nDiscover how Noveum.ai provides comprehensive tracing and observability for LLM applications, RAG systems, and multi-agent workflows with our powerful Python and TypeScript SDKs.\nShashank Agarwal\n3/2/2025...\n\nUser Question: What are the newest features in cloud computing?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", + "prompt.prompt_length": 3123, + "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/changelog):\n- \ud83d\udcc8 Enhanced Dashboard Analytics: Real-time request tracking, latency monitoring, and cost analysis\n- \ud83d\udd0d Improved Logs Interface: Better search functionality,...", + "prompt.instruction_following": "rag_optimized", + "response.answer_length": 837, + "response.answer_completeness": "complete", + "response.response_quality": "high", + "response.source_citation": 5, + "response.context_utilization": 0.3245443970531214, + "output_response": "RAG Answer: The provided context does not specifically mention any new features in cloud computing as a general topic. However, it does highlight several recent enhancements related to the Noveum.ai platform, whi...", + "evaluation.retrieval_effectiveness": 1, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 5, + "evaluation.context_relevance": "high", + "evaluation.overall_quality": "high", + "evaluation.ready_for_production": true, + "rag.retrieval_strategy": "semantic_similarity", + "rag.vector_search_results": 5, + "rag.context_synthesis": "multi_source", + "rag.document_coverage": 1, + "rag.information_density": 0.3245443970531214 + }, + "events": [], + "links": [], + "trace_trace_id": "343764b8-7ac9-4549-93cf-8abc11ceba6f", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:58:26.542000000", + "trace_end_time": "2025-10-22 16:58:31.796000000", + "trace_duration_ms": 5253.973, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:58:26.542000000", + "updated_at": "2025-10-22 16:58:26.542000000" + }, + { + "span_id": "7384d697-d7bf-4709-8377-ec26b094b550", + "trace_id": "3f7b6a62-8888-4a5d-b12c-37a3829aea49", + "parent_span_id": "e631b22a-a621-4ecc-bb01-f731ca66f3d0", + "name": "agent.query_routing", + "start_time": "2025-10-22 16:58:19.885000000", + "end_time": "2025-10-22 16:58:25.536000000", + "duration_ms": 5651.335999999999, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What's the current status of cryptocurrency markets?", + "query_length": 52, + "query_type": "routing_query", + "output_response": "Routed to Web Search for query processing", + "router.classification": "Web Search", + "router.confidence_score": 1, + "router.rag_keyword_score": 0, + "router.web_keyword_score": 3, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "3f7b6a62-8888-4a5d-b12c-37a3829aea49", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:58:19.885000000", + "trace_end_time": "2025-10-22 16:58:25.536000000", + "trace_duration_ms": 5651.67, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:58:19.885000000", + "updated_at": "2025-10-22 16:58:19.885000000" + }, + { + "span_id": "7abf372d-d9f4-41d1-9e3a-d4f01173fa5e", + "trace_id": "3f7b6a62-8888-4a5d-b12c-37a3829aea49", + "parent_span_id": "7384d697-d7bf-4709-8377-ec26b094b550", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-22 16:58:19.885000000", + "end_time": "2025-10-22 16:58:19.885000000", + "duration_ms": 0.037, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What's the current status of cryptocurrency markets?", + "query_length": 52, + "query_type": "routing_evaluation_query", + "classification.mode": "Web Search", + "classification.rag_keyword_score": 0, + "classification.web_keyword_score": 3, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "complex", + "query.intent": "general_knowledge", + "query.keyword_density": 0.42857142857142855, + "query.domain_affinity": "general", + "routing.decision": "Routed to Web Search based on analysis", + "routing.rationale": "RAG score: 0, Web score: 3, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "RAG", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.057692307692307696, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: Web Search (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "3f7b6a62-8888-4a5d-b12c-37a3829aea49", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:58:19.885000000", + "trace_end_time": "2025-10-22 16:58:25.536000000", + "trace_duration_ms": 5651.67, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:58:19.885000000", + "updated_at": "2025-10-22 16:58:19.885000000" + }, + { + "span_id": "e5d9c3c3-6c18-45d2-a6b6-58bde4d4e461", + "trace_id": "3f7b6a62-8888-4a5d-b12c-37a3829aea49", + "parent_span_id": "7384d697-d7bf-4709-8377-ec26b094b550", + "name": "agent.web_search_generation", + "start_time": "2025-10-22 16:58:19.885000000", + "end_time": "2025-10-22 16:58:25.536000000", + "duration_ms": 5651.161, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "web_search_agent", + "agent.operation": "web_search_generation", + "agent.capabilities": [ + "web_search", + "content_synthesis", + "response_generation" + ], + "agent.id": "noveum_web_search_agent", + "input_query": "What's the current status of cryptocurrency markets?", + "query_length": 52, + "query_type": "web_search_query", + "output_response": "Web Search Answer: The provided web search results do not contain specific information regarding the current status of cryptocurrency markets. To get the latest updates on cryptocurrency prices, trends, and market analy...", + "web_search.results_count": 1, + "web_search.context_length": 251, + "web_search.response_length": 416, + "web_search.mode": "external_web_search" + }, + "events": [], + "links": [], + "trace_trace_id": "3f7b6a62-8888-4a5d-b12c-37a3829aea49", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:58:19.885000000", + "trace_end_time": "2025-10-22 16:58:25.536000000", + "trace_duration_ms": 5651.67, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:58:19.885000000", + "updated_at": "2025-10-22 16:58:19.885000000" + }, + { + "span_id": "e631b22a-a621-4ecc-bb01-f731ca66f3d0", + "trace_id": "3f7b6a62-8888-4a5d-b12c-37a3829aea49", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-22 16:58:19.885000000", + "end_time": "2025-10-22 16:58:25.536000000", + "duration_ms": 5651.588000000001, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What's the current status of cryptocurrency markets?", + "query.length": 52, + "processing.mode": "Web Search", + "processing.time_seconds": 5.651373863220215, + "processing.response_length": 416, + "processing.sources_count": 1, + "output_response": "Final Answer: The provided web search results do not contain specific information regarding the current status of cryptocurrency markets. To get the latest updates on cryptocurrency prices, trends, and market analy...", + "final_answer_mode": "Web Search", + "query_processed.input_query": "What's the current status of cryptocurrency markets?", + "query_processed.output_response": "Successfully processed query using Web Search, generated 416 character response", + "query_processed.mode": "Web Search", + "query_processed.processing_time": 5.651373863220215, + "query_processed.response_length": 416 + }, + "events": [], + "links": [], + "trace_trace_id": "3f7b6a62-8888-4a5d-b12c-37a3829aea49", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:58:19.885000000", + "trace_end_time": "2025-10-22 16:58:25.536000000", + "trace_duration_ms": 5651.67, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:58:19.885000000", + "updated_at": "2025-10-22 16:58:19.885000000" + }, + { + "span_id": "34e2f906-724e-4116-b61f-82eaa6b75959", + "trace_id": "3f7b6a62-8888-4a5d-b12c-37a3829aea49", + "parent_span_id": "e5d9c3c3-6c18-45d2-a6b6-58bde4d4e461", + "name": "agent.llm_model_execution", + "start_time": "2025-10-22 16:58:24.327000000", + "end_time": "2025-10-22 16:58:25.536000000", + "duration_ms": 1208.66, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What's the current status of cryptocurrency markets?", + "query_length": 52, + "query_type": "web_search_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 1.208608865737915, + "model.latency_ms": 1208.608865737915, + "model.start_time": 1761152304.327749, + "model.end_time": 1761152305.5363579, + "model.performance_tier": "fast", + "model.prompt_tokens": 187, + "model.completion_tokens": 104, + "model.total_tokens": 291, + "model.tokens_per_second": 240.77268357809888, + "model.estimated_cost": 0.00873, + "model.efficiency_score": 1.429553264604811, + "model.response_length": 416, + "model.response_quality": "high", + "model.output_response": "Model Response: The provided web search results do not contain specific information regarding the current status of cryptocurrency markets. To get the latest updates on cryptocurrency prices, trends, and market analy...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "3f7b6a62-8888-4a5d-b12c-37a3829aea49", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:58:19.885000000", + "trace_end_time": "2025-10-22 16:58:25.536000000", + "trace_duration_ms": 5651.67, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:58:19.885000000", + "updated_at": "2025-10-22 16:58:19.885000000" + }, + { + "span_id": "5548e02e-2ea8-472c-9c42-f811d6e4d30b", + "trace_id": "3f7b6a62-8888-4a5d-b12c-37a3829aea49", + "parent_span_id": "e5d9c3c3-6c18-45d2-a6b6-58bde4d4e461", + "name": "agent.web_search_evaluation_metrics", + "start_time": "2025-10-22 16:58:25.536000000", + "end_time": "2025-10-22 16:58:25.536000000", + "duration_ms": 0.044, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "web_search_evaluation_metrics", + "agent.capabilities": [ + "web_search_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What's the current status of cryptocurrency markets?", + "query_length": 52, + "query_type": "web_search_evaluation_query", + "web_search.results_count": 1, + "web_search.context_length": 251, + "web_search.context_synthesized": "Context from 1 web sources", + "web_search.search_effectiveness": 0.2, + "web_search.context_quality": "low", + "web_search.source_diversity": 1, + "web_search.information_synthesis": "low", + "web_search.external_knowledge_utilization": 0.251, + "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nLog in to WhatsApp Web for simple, reliable and private messaging on your desktop. Send and receive messages and files with ease, all for \u2026\nURL: https://duckduckgo.com/?q=What's+the+current+status+of+cryptocurrency+markets?\n\nUser Question: What's the current status of cryptocurrency markets?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", + "prompt.prompt_length": 748, + "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nLog in to WhatsApp Web for simple, reliable and private messaging on your desktop. Send and receive messages and files with ease, all for \u2026\nURL: https://duckduckgo.com/?q=W...", + "prompt.instruction_following": "web_search_optimized", + "response.answer_length": 416, + "response.answer_completeness": "complete", + "response.response_quality": "medium", + "response.source_citation": 1, + "response.context_utilization": 1.657370517928287, + "output_response": "Web Search Answer: The provided web search results do not contain specific information regarding the current status of cryptocurrency markets. To get the latest updates on cryptocurrency prices, trends, and market analy...", + "evaluation.search_effectiveness": 0.2, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 1, + "evaluation.context_relevance": "low", + "evaluation.overall_quality": "low", + "evaluation.ready_for_production": true, + "web_search.search_strategy": "duckduckgo_api", + "web_search.real_time_data": true, + "web_search.external_sources": 1, + "web_search.information_freshness": "current", + "web_search.knowledge_synthesis": "single_source" + }, + "events": [], + "links": [], + "trace_trace_id": "3f7b6a62-8888-4a5d-b12c-37a3829aea49", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:58:19.885000000", + "trace_end_time": "2025-10-22 16:58:25.536000000", + "trace_duration_ms": 5651.67, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:58:19.885000000", + "updated_at": "2025-10-22 16:58:19.885000000" + }, + { + "span_id": "a9e34cea-d4bd-478b-abf2-7a3a38dcd40e", + "trace_id": "51f4534f-23e4-4be7-8d6e-e93391d6e0e2", + "parent_span_id": "8e9b2d06-3e11-4d17-9a32-b1529a2c10a4", + "name": "agent.web_search_generation", + "start_time": "2025-10-22 16:58:15.254000000", + "end_time": "2025-10-22 16:58:18.876000000", + "duration_ms": 3621.674, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "web_search_agent", + "agent.operation": "web_search_generation", + "agent.capabilities": [ + "web_search", + "content_synthesis", + "response_generation" + ], + "agent.id": "noveum_web_search_agent", + "input_query": "What are the latest updates in Python programming?", + "query_length": 50, + "query_type": "web_search_query", + "output_response": "Web Search Answer: The provided web search results do not contain specific information about the latest updates in Python programming. They primarily focus on general news coverage and updates from India and around the ...", + "web_search.results_count": 1, + "web_search.context_length": 753, + "web_search.response_length": 580, + "web_search.mode": "external_web_search" + }, + "events": [], + "links": [], + "trace_trace_id": "51f4534f-23e4-4be7-8d6e-e93391d6e0e2", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:58:15.254000000", + "trace_end_time": "2025-10-22 16:58:18.877000000", + "trace_duration_ms": 3622.355, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:58:15.254000000", + "updated_at": "2025-10-22 16:58:15.254000000" + }, + { + "span_id": "8e9b2d06-3e11-4d17-9a32-b1529a2c10a4", + "trace_id": "51f4534f-23e4-4be7-8d6e-e93391d6e0e2", + "parent_span_id": "bda936a0-24f3-4841-af4f-227197e10b9d", + "name": "agent.query_routing", + "start_time": "2025-10-22 16:58:15.254000000", + "end_time": "2025-10-22 16:58:18.876000000", + "duration_ms": 3621.863, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What are the latest updates in Python programming?", + "query_length": 50, + "query_type": "routing_query", + "output_response": "Routed to Web Search for query processing", + "router.classification": "Web Search", + "router.confidence_score": 1, + "router.rag_keyword_score": 0, + "router.web_keyword_score": 2, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "51f4534f-23e4-4be7-8d6e-e93391d6e0e2", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:58:15.254000000", + "trace_end_time": "2025-10-22 16:58:18.877000000", + "trace_duration_ms": 3622.355, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:58:15.254000000", + "updated_at": "2025-10-22 16:58:15.254000000" + }, + { + "span_id": "bda936a0-24f3-4841-af4f-227197e10b9d", + "trace_id": "51f4534f-23e4-4be7-8d6e-e93391d6e0e2", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-22 16:58:15.254000000", + "end_time": "2025-10-22 16:58:18.877000000", + "duration_ms": 3622.271, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What are the latest updates in Python programming?", + "query.length": 50, + "processing.mode": "Web Search", + "processing.time_seconds": 3.6219100952148438, + "processing.response_length": 580, + "processing.sources_count": 1, + "output_response": "Final Answer: The provided web search results do not contain specific information about the latest updates in Python programming. They primarily focus on general news coverage and updates from India and around the ...", + "final_answer_mode": "Web Search", + "query_processed.input_query": "What are the latest updates in Python programming?", + "query_processed.output_response": "Successfully processed query using Web Search, generated 580 character response", + "query_processed.mode": "Web Search", + "query_processed.processing_time": 3.6219100952148438, + "query_processed.response_length": 580 + }, + "events": [], + "links": [], + "trace_trace_id": "51f4534f-23e4-4be7-8d6e-e93391d6e0e2", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:58:15.254000000", + "trace_end_time": "2025-10-22 16:58:18.877000000", + "trace_duration_ms": 3622.355, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:58:15.254000000", + "updated_at": "2025-10-22 16:58:15.254000000" + }, + { + "span_id": "e27e3812-d49d-4f4f-98b0-144cf0ceffc3", + "trace_id": "51f4534f-23e4-4be7-8d6e-e93391d6e0e2", + "parent_span_id": "8e9b2d06-3e11-4d17-9a32-b1529a2c10a4", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-22 16:58:15.254000000", + "end_time": "2025-10-22 16:58:15.254000000", + "duration_ms": 0.047, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What are the latest updates in Python programming?", + "query_length": 50, + "query_type": "routing_evaluation_query", + "classification.mode": "Web Search", + "classification.rag_keyword_score": 0, + "classification.web_keyword_score": 2, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "general_knowledge", + "query.keyword_density": 0.25, + "query.domain_affinity": "general", + "routing.decision": "Routed to Web Search based on analysis", + "routing.rationale": "RAG score: 0, Web score: 2, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "RAG", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.038461538461538464, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: Web Search (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "51f4534f-23e4-4be7-8d6e-e93391d6e0e2", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:58:15.254000000", + "trace_end_time": "2025-10-22 16:58:18.877000000", + "trace_duration_ms": 3622.355, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:58:15.254000000", + "updated_at": "2025-10-22 16:58:15.254000000" + }, + { + "span_id": "b0eea8ec-5cff-4667-bea7-aeb6d0f0daf3", + "trace_id": "51f4534f-23e4-4be7-8d6e-e93391d6e0e2", + "parent_span_id": "a9e34cea-d4bd-478b-abf2-7a3a38dcd40e", + "name": "agent.llm_model_execution", + "start_time": "2025-10-22 16:58:15.568000000", + "end_time": "2025-10-22 16:58:18.876000000", + "duration_ms": 3307.5739999999996, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What are the latest updates in Python programming?", + "query_length": 50, + "query_type": "web_search_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 3.307521104812622, + "model.latency_ms": 3307.521104812622, + "model.start_time": 1761152295.568949, + "model.end_time": 1761152298.87647, + "model.performance_tier": "medium", + "model.prompt_tokens": 312, + "model.completion_tokens": 145, + "model.total_tokens": 457, + "model.tokens_per_second": 138.16994223711538, + "model.estimated_cost": 0.01371, + "model.efficiency_score": 1.2691466083150984, + "model.response_length": 580, + "model.response_quality": "high", + "model.output_response": "Model Response: The provided web search results do not contain specific information about the latest updates in Python programming. They primarily focus on general news coverage and updates from India and around the ...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "51f4534f-23e4-4be7-8d6e-e93391d6e0e2", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:58:15.254000000", + "trace_end_time": "2025-10-22 16:58:18.877000000", + "trace_duration_ms": 3622.355, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:58:15.254000000", + "updated_at": "2025-10-22 16:58:15.254000000" + }, + { + "span_id": "d0bfcbe5-cb6c-4837-bdf3-f2c65be145cd", + "trace_id": "51f4534f-23e4-4be7-8d6e-e93391d6e0e2", + "parent_span_id": "a9e34cea-d4bd-478b-abf2-7a3a38dcd40e", + "name": "agent.web_search_evaluation_metrics", + "start_time": "2025-10-22 16:58:18.876000000", + "end_time": "2025-10-22 16:58:18.876000000", + "duration_ms": 0.04, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "web_search_evaluation_metrics", + "agent.capabilities": [ + "web_search_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What are the latest updates in Python programming?", + "query_length": 50, + "query_type": "web_search_evaluation_query", + "web_search.results_count": 1, + "web_search.context_length": 753, + "web_search.context_synthesized": "Context from 1 web sources", + "web_search.search_effectiveness": 0.2, + "web_search.context_quality": "medium", + "web_search.source_diversity": 1, + "web_search.information_synthesis": "low", + "web_search.external_knowledge_utilization": 0.753, + "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\n3 days ago \u00b7 Latest news headlines from India & around the world. Check out today\u2019s news coverage live with videos & photos on \u2026 6 days ago \u00b7 India News | Latest India News | Read latest and breaking news from India. Today's top India news headlines, news on \u2026 2 days ago \u00b7 Top News Stories of the day, Latest News Headlines, News Specials, Breaking News and Latest India News, World current affairs & \u2026 NDTV.com provides latest news from India and around the world. Get breaking news alerts from India and follow today\u2019s live news updates in \u2026 6 days ago \u00b7 Get Latest entertainment news from Bollywood, Hollywood and regional film industries.\nURL: https://duckduckgo.com/?q=What+are+the+latest+updates+in+Python+programming?\n\nUser Question: What are the latest updates in Python programming?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", + "prompt.prompt_length": 1248, + "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\n3 days ago \u00b7 Latest news headlines from India & around the world. Check out today\u2019s news coverage live with videos & photos on \u2026 6 days ago \u00b7 India News | Latest India News...", + "prompt.instruction_following": "web_search_optimized", + "response.answer_length": 580, + "response.answer_completeness": "complete", + "response.response_quality": "medium", + "response.source_citation": 1, + "response.context_utilization": 0.7702523240371846, + "output_response": "Web Search Answer: The provided web search results do not contain specific information about the latest updates in Python programming. They primarily focus on general news coverage and updates from India and around the ...", + "evaluation.search_effectiveness": 0.2, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 1, + "evaluation.context_relevance": "medium", + "evaluation.overall_quality": "low", + "evaluation.ready_for_production": true, + "web_search.search_strategy": "duckduckgo_api", + "web_search.real_time_data": true, + "web_search.external_sources": 1, + "web_search.information_freshness": "current", + "web_search.knowledge_synthesis": "single_source" + }, + "events": [], + "links": [], + "trace_trace_id": "51f4534f-23e4-4be7-8d6e-e93391d6e0e2", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:58:15.254000000", + "trace_end_time": "2025-10-22 16:58:18.877000000", + "trace_duration_ms": 3622.355, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:58:15.254000000", + "updated_at": "2025-10-22 16:58:15.254000000" + }, + { + "span_id": "5f5916a7-b9ea-4348-861d-db69b25d6b62", + "trace_id": "e30e9ba6-ffd1-4eb9-b7ae-6d19e04343d6", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-22 16:58:11.033000000", + "end_time": "2025-10-22 16:58:14.250000000", + "duration_ms": 3216.77, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What happened in tech news this week?", + "query.length": 37, + "processing.mode": "Web Search", + "processing.time_seconds": 3.2166178226470947, + "processing.response_length": 510, + "processing.sources_count": 1, + "output_response": "Final Answer: The web search results do not provide specific information about recent events in tech news for this week. They primarily discuss the grammatical usage of the word \"happened\" and its variations, such ...", + "final_answer_mode": "Web Search", + "query_processed.input_query": "What happened in tech news this week?", + "query_processed.output_response": "Successfully processed query using Web Search, generated 510 character response", + "query_processed.mode": "Web Search", + "query_processed.processing_time": 3.2166178226470947, + "query_processed.response_length": 510 + }, + "events": [], + "links": [], + "trace_trace_id": "e30e9ba6-ffd1-4eb9-b7ae-6d19e04343d6", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:58:11.033000000", + "trace_end_time": "2025-10-22 16:58:14.250000000", + "trace_duration_ms": 3216.801, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:58:11.033000000", + "updated_at": "2025-10-22 16:58:11.033000000" + }, + { + "span_id": "804613d0-a701-4c22-add1-d315d7a29813", + "trace_id": "e30e9ba6-ffd1-4eb9-b7ae-6d19e04343d6", + "parent_span_id": "5f5916a7-b9ea-4348-861d-db69b25d6b62", + "name": "agent.query_routing", + "start_time": "2025-10-22 16:58:11.033000000", + "end_time": "2025-10-22 16:58:14.250000000", + "duration_ms": 3216.594, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What happened in tech news this week?", + "query_length": 37, + "query_type": "routing_query", + "output_response": "Routed to Web Search for query processing", + "router.classification": "Web Search", + "router.confidence_score": 1, + "router.rag_keyword_score": 0, + "router.web_keyword_score": 2, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "e30e9ba6-ffd1-4eb9-b7ae-6d19e04343d6", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:58:11.033000000", + "trace_end_time": "2025-10-22 16:58:14.250000000", + "trace_duration_ms": 3216.801, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:58:11.033000000", + "updated_at": "2025-10-22 16:58:11.033000000" + }, + { + "span_id": "f6c8d34f-185f-49ba-a5a4-a9d25ccb4fc6", + "trace_id": "e30e9ba6-ffd1-4eb9-b7ae-6d19e04343d6", + "parent_span_id": "804613d0-a701-4c22-add1-d315d7a29813", + "name": "agent.web_search_generation", + "start_time": "2025-10-22 16:58:11.034000000", + "end_time": "2025-10-22 16:58:14.250000000", + "duration_ms": 3216.4959999999996, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "web_search_agent", + "agent.operation": "web_search_generation", + "agent.capabilities": [ + "web_search", + "content_synthesis", + "response_generation" + ], + "agent.id": "noveum_web_search_agent", + "input_query": "What happened in tech news this week?", + "query_length": 37, + "query_type": "web_search_query", + "output_response": "Web Search Answer: The web search results do not provide specific information about recent events in tech news for this week. They primarily discuss the grammatical usage of the word \"happened\" and its variations, such ...", + "web_search.results_count": 1, + "web_search.context_length": 724, + "web_search.response_length": 510, + "web_search.mode": "external_web_search" + }, + "events": [], + "links": [], + "trace_trace_id": "e30e9ba6-ffd1-4eb9-b7ae-6d19e04343d6", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:58:11.033000000", + "trace_end_time": "2025-10-22 16:58:14.250000000", + "trace_duration_ms": 3216.801, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:58:11.033000000", + "updated_at": "2025-10-22 16:58:11.033000000" + }, + { + "span_id": "11d68082-e7b8-4bbf-833a-79e296c31eda", + "trace_id": "e30e9ba6-ffd1-4eb9-b7ae-6d19e04343d6", + "parent_span_id": "804613d0-a701-4c22-add1-d315d7a29813", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-22 16:58:11.034000000", + "end_time": "2025-10-22 16:58:11.034000000", + "duration_ms": 0.017, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What happened in tech news this week?", + "query_length": 37, + "query_type": "routing_evaluation_query", + "classification.mode": "Web Search", + "classification.rag_keyword_score": 0, + "classification.web_keyword_score": 2, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "general_knowledge", + "query.keyword_density": 0.2857142857142857, + "query.domain_affinity": "general", + "routing.decision": "Routed to Web Search based on analysis", + "routing.rationale": "RAG score: 0, Web score: 2, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "RAG", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.038461538461538464, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: Web Search (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "e30e9ba6-ffd1-4eb9-b7ae-6d19e04343d6", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:58:11.033000000", + "trace_end_time": "2025-10-22 16:58:14.250000000", + "trace_duration_ms": 3216.801, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:58:11.033000000", + "updated_at": "2025-10-22 16:58:11.033000000" + }, + { + "span_id": "420b4ca1-c9f1-488f-ac2c-c53766717634", + "trace_id": "e30e9ba6-ffd1-4eb9-b7ae-6d19e04343d6", + "parent_span_id": "f6c8d34f-185f-49ba-a5a4-a9d25ccb4fc6", + "name": "agent.llm_model_execution", + "start_time": "2025-10-22 16:58:11.468000000", + "end_time": "2025-10-22 16:58:14.250000000", + "duration_ms": 2781.777, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What happened in tech news this week?", + "query_length": 37, + "query_type": "web_search_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 2.781723976135254, + "model.latency_ms": 2781.723976135254, + "model.start_time": 1761152291.468626, + "model.end_time": 1761152294.25035, + "model.performance_tier": "medium", + "model.prompt_tokens": 301, + "model.completion_tokens": 127, + "model.total_tokens": 428, + "model.tokens_per_second": 153.8614196346811, + "model.estimated_cost": 0.01284, + "model.efficiency_score": 1.191588785046729, + "model.response_length": 510, + "model.response_quality": "high", + "model.output_response": "Model Response: The web search results do not provide specific information about recent events in tech news for this week. They primarily discuss the grammatical usage of the word \"happened\" and its variations, such ...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "e30e9ba6-ffd1-4eb9-b7ae-6d19e04343d6", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:58:11.033000000", + "trace_end_time": "2025-10-22 16:58:14.250000000", + "trace_duration_ms": 3216.801, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:58:11.033000000", + "updated_at": "2025-10-22 16:58:11.033000000" + }, + { + "span_id": "56b8bf09-8ebf-44e1-b01f-daff23d3c42c", + "trace_id": "e30e9ba6-ffd1-4eb9-b7ae-6d19e04343d6", + "parent_span_id": "f6c8d34f-185f-49ba-a5a4-a9d25ccb4fc6", + "name": "agent.web_search_evaluation_metrics", + "start_time": "2025-10-22 16:58:14.250000000", + "end_time": "2025-10-22 16:58:14.250000000", + "duration_ms": 0.060000000000000005, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "web_search_evaluation_metrics", + "agent.capabilities": [ + "web_search_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What happened in tech news this week?", + "query_length": 37, + "query_type": "web_search_evaluation_query", + "web_search.results_count": 1, + "web_search.context_length": 724, + "web_search.context_synthesized": "Context from 1 web sources", + "web_search.search_effectiveness": 0.2, + "web_search.context_quality": "medium", + "web_search.source_diversity": 1, + "web_search.information_synthesis": "low", + "web_search.external_knowledge_utilization": 0.724, + "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nhappened\u7684\u7528\u6cd5\u5982\u4e0b\uff1a 1.happened\u662f \u4e0d\u53ca\u7269\u52a8\u8bcd,\u4f5c\u201c\u53d1\u751f\u201d\u89e3\u65f6,\u53ef\u6307\u5076\u7136\u7684\u4e8b\u6545,\u4e5f\u53ef\u6307\u6709\u8ba1\u5212\u6216\u65e0\u8ba1\u5212\u53d1\u751f\u7684\u4e8b\u3002\u4e00\u822c\u4ee5\u4e8b\u4ef6\u3001\u73af\u5883\u3001\u5f62\u52bf\u7b49\u4f5c\u4e3b\u8bed,\u8868\u793a\u201c\u67d0\u4eba\u53d1\u751f\u67d0\u4e8b\u201d\u65f6,\u987b\u4ee5\u4ecb\u8bcdto\u5f15\u51fa\u201c\u67d0\u4eba\u201d\u3002 2. \u2026 what happened\u8fd8\u662fwhat's happened\"What happened\"\u548c\"What's happened\"\u90fd\u7528\u4e8e\u8be2\u95ee\u8fc7\u53bb\u53d1\u751f\u4e86\u4ec0\u4e48\u4e8b\u60c5\uff0c\u4e3b\u8981\u7684\u533a\u522b\u5728\u4e8e\u8bed\u6cd5\u548c\u4f7f\u7528\u65b9\u5f0f\u3002\"What happened\"\u662f\u4e00\u4e2a\u8fc7\u53bb\u65f6\u6001\u7684\u7b80\u5355 \u63a5\u4e0b\u6765\u8ba9\u6211\u4eec\u770b\u4e0bwhat happen\u548cwhat happened\u7684\u7528\u6cd5\u533a\u522b\uff1a 1.\u8bcd\u6027\uff1awhat happen\u662f\u7528\u539f\u5f62\u52a8\u8bcdhappen\u6784\u6210\u7684\u73b0\u5728\u65f6\u77ed\u8bed\uff0c\u800cwhat happened\u662f\u7528\u52a8\u8bcdhappen\u7684\u8fc7\u53bb\u5f0fhappened\u6784\u6210\u7684\u8fc7\u53bb\u65f6\u77ed \u2026 What happened \u548c What's happened\u4e24\u8005\u7684\u533a\u522b\uff1a \u662f\u95ee What happened\uff1f\u53ea\u662f\u60f3\u77e5\u9053\u8fc7\u53bb\u6240\u53d1\u751f\u7684\u4e8b\u60c5\uff0c \u800c\u5982\u679c\u7528What's happened\uff1f\u5219\u5f3a\u8c03\u5bf9\u73b0\u5728\u7684\u5f71\u54cd\u3002\uff0c\u5373\u60f3\u77e5\u9053\u8fc7\u53bb\u6240\u53d1\u751f\u7684\u4ec0\u4e48\u4e8b\u5bfc\u81f4\u4e86\u8bf4 \u2026 it so happened that\u7684\u53e5\u5f0fhappen\u8868\u793a\u201c\u78b0\u5de7\u6216\u6070\u5de7\u53d1\u751f\u67d0\u4e8b\u201d\u65f6,\u53ef\u7528\u201cIt happens / happened that...\u201d\u8fd9\u4e00\u7ed3\u6784\u6765\u8868\u8fbe.\u4f8b\u5982\uff1a It happened that Brian and Peter were at home that day.\u78b0\u5de7\u90a3\u5929\u5e03\u83b1\u6069\u548c \u2026\nURL: https://duckduckgo.com/?q=What+happened+in+tech+news+this+week?\n\nUser Question: What happened in tech news this week?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", + "prompt.prompt_length": 1206, + "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nhappened\u7684\u7528\u6cd5\u5982\u4e0b\uff1a 1.happened\u662f \u4e0d\u53ca\u7269\u52a8\u8bcd,\u4f5c\u201c\u53d1\u751f\u201d\u89e3\u65f6,\u53ef\u6307\u5076\u7136\u7684\u4e8b\u6545,\u4e5f\u53ef\u6307\u6709\u8ba1\u5212\u6216\u65e0\u8ba1\u5212\u53d1\u751f\u7684\u4e8b\u3002\u4e00\u822c\u4ee5\u4e8b\u4ef6\u3001\u73af\u5883\u3001\u5f62\u52bf\u7b49\u4f5c\u4e3b\u8bed,\u8868\u793a\u201c\u67d0\u4eba\u53d1\u751f\u67d0\u4e8b\u201d\u65f6,\u987b\u4ee5\u4ecb\u8bcdto\u5f15\u51fa\u201c\u67d0\u4eba\u201d\u3002 2. \u2026 what happened\u8fd8\u662fwhat's happened\"What happened\"\u548c\"What's happene...", + "prompt.instruction_following": "web_search_optimized", + "response.answer_length": 510, + "response.answer_completeness": "complete", + "response.response_quality": "medium", + "response.source_citation": 1, + "response.context_utilization": 0.7044198895027625, + "output_response": "Web Search Answer: The web search results do not provide specific information about recent events in tech news for this week. They primarily discuss the grammatical usage of the word \"happened\" and its variations, such ...", + "evaluation.search_effectiveness": 0.2, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 1, + "evaluation.context_relevance": "medium", + "evaluation.overall_quality": "low", + "evaluation.ready_for_production": true, + "web_search.search_strategy": "duckduckgo_api", + "web_search.real_time_data": true, + "web_search.external_sources": 1, + "web_search.information_freshness": "current", + "web_search.knowledge_synthesis": "single_source" + }, + "events": [], + "links": [], + "trace_trace_id": "e30e9ba6-ffd1-4eb9-b7ae-6d19e04343d6", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:58:11.033000000", + "trace_end_time": "2025-10-22 16:58:14.250000000", + "trace_duration_ms": 3216.801, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:58:11.033000000", + "updated_at": "2025-10-22 16:58:11.033000000" + }, + { + "span_id": "82eb19e8-f6cf-43fd-ba56-640a6fd3ed0e", + "trace_id": "45efae74-ee25-4004-a509-6e0ff628a0cd", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-22 16:58:04.883000000", + "end_time": "2025-10-22 16:58:10.029000000", + "duration_ms": 5146.146, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What are the current trends in observability tools?", + "query.length": 51, + "processing.mode": "RAG", + "processing.time_seconds": 5.146108150482178, + "processing.response_length": 815, + "processing.sources_count": 5, + "output_response": "Final Answer: The provided context does not explicitly outline current trends in observability tools. However, it highlights the unique challenges faced by AI applications that traditional monitoring tools cannot a...", + "final_answer_mode": "RAG", + "query_processed.input_query": "What are the current trends in observability tools?", + "query_processed.output_response": "Successfully processed query using RAG, generated 815 character response", + "query_processed.mode": "RAG", + "query_processed.processing_time": 5.146108150482178, + "query_processed.response_length": 815 + }, + "events": [], + "links": [], + "trace_trace_id": "45efae74-ee25-4004-a509-6e0ff628a0cd", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:58:04.883000000", + "trace_end_time": "2025-10-22 16:58:10.029000000", + "trace_duration_ms": 5146.244000000001, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:58:04.883000000", + "updated_at": "2025-10-22 16:58:04.883000000" + }, + { + "span_id": "e81617c6-c2a1-4b08-a5fc-dde3439fe2f4", + "trace_id": "45efae74-ee25-4004-a509-6e0ff628a0cd", + "parent_span_id": "8001ba0e-95c7-42da-9087-603a38e1a500", + "name": "agent.llm-rag", + "start_time": "2025-10-22 16:58:04.883000000", + "end_time": "2025-10-22 16:58:10.029000000", + "duration_ms": 5145.655, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "rag_agent", + "agent.operation": "llm-rag", + "agent.capabilities": [ + "document_retrieval", + "context_generation", + "response_generation" + ], + "agent.id": "noveum_rag_agent", + "input_query": "What are the current trends in observability tools?", + "query_length": 51, + "query_type": "rag_query", + "output_response": "RAG Answer: The provided context does not explicitly outline current trends in observability tools. However, it highlights the unique challenges faced by AI applications that traditional monitoring tools cannot a...", + "rag.context_length": 2943, + "rag.sources_count": 5, + "rag.answer_length": 815, + "rag.mode": "retrieval_augmented_generation" + }, + "events": [], + "links": [], + "trace_trace_id": "45efae74-ee25-4004-a509-6e0ff628a0cd", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:58:04.883000000", + "trace_end_time": "2025-10-22 16:58:10.029000000", + "trace_duration_ms": 5146.244000000001, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:58:04.883000000", + "updated_at": "2025-10-22 16:58:04.883000000" + }, + { + "span_id": "8001ba0e-95c7-42da-9087-603a38e1a500", + "trace_id": "45efae74-ee25-4004-a509-6e0ff628a0cd", + "parent_span_id": "82eb19e8-f6cf-43fd-ba56-640a6fd3ed0e", + "name": "agent.query_routing", + "start_time": "2025-10-22 16:58:04.883000000", + "end_time": "2025-10-22 16:58:10.029000000", + "duration_ms": 5146.036999999999, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What are the current trends in observability tools?", + "query_length": 51, + "query_type": "routing_query", + "output_response": "Routed to RAG for query processing", + "router.classification": "RAG", + "router.confidence_score": 0.3333333333333333, + "router.rag_keyword_score": 2, + "router.web_keyword_score": 1, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "45efae74-ee25-4004-a509-6e0ff628a0cd", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:58:04.883000000", + "trace_end_time": "2025-10-22 16:58:10.029000000", + "trace_duration_ms": 5146.244000000001, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:58:04.883000000", + "updated_at": "2025-10-22 16:58:04.883000000" + }, + { + "span_id": "79b7d138-05e1-4f18-8b63-dcfe9732180b", + "trace_id": "45efae74-ee25-4004-a509-6e0ff628a0cd", + "parent_span_id": "8001ba0e-95c7-42da-9087-603a38e1a500", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-22 16:58:04.883000000", + "end_time": "2025-10-22 16:58:04.883000000", + "duration_ms": 0.051, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What are the current trends in observability tools?", + "query_length": 51, + "query_type": "routing_evaluation_query", + "classification.mode": "RAG", + "classification.rag_keyword_score": 2, + "classification.web_keyword_score": 1, + "classification.confidence_score": 0.3333333333333333, + "classification.confidence_level": "medium", + "classification.method": "keyword_based", + "query.complexity": "complex", + "query.intent": "documentation", + "query.keyword_density": 0.375, + "query.domain_affinity": "noveum", + "routing.decision": "Routed to RAG based on analysis", + "routing.rationale": "RAG score: 2, Web score: 1, Confidence: 0.33", + "routing.expected_performance": "medium", + "routing.alternative_mode": "Web Search", + "routing.decision_confidence": 0.3333333333333333, + "evaluation.routing_accuracy": "medium", + "evaluation.keyword_coverage": 0.057692307692307696, + "evaluation.query_understanding": "ambiguous", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: RAG (Confidence: 0.33)" + }, + "events": [], + "links": [], + "trace_trace_id": "45efae74-ee25-4004-a509-6e0ff628a0cd", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:58:04.883000000", + "trace_end_time": "2025-10-22 16:58:10.029000000", + "trace_duration_ms": 5146.244000000001, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:58:04.883000000", + "updated_at": "2025-10-22 16:58:04.883000000" + }, + { + "span_id": "28c233a0-d42a-4200-86b0-dfbd2d5be06e", + "trace_id": "45efae74-ee25-4004-a509-6e0ff628a0cd", + "parent_span_id": "e81617c6-c2a1-4b08-a5fc-dde3439fe2f4", + "name": "agent.llm_model_execution", + "start_time": "2025-10-22 16:58:05.635000000", + "end_time": "2025-10-22 16:58:09.222000000", + "duration_ms": 3586.924, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What are the current trends in observability tools?", + "query_length": 51, + "query_type": "rag_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 3.5868873596191406, + "model.latency_ms": 3586.8873596191406, + "model.start_time": 1761152285.6352398, + "model.end_time": 1761152289.2221272, + "model.performance_tier": "medium", + "model.prompt_tokens": 872, + "model.completion_tokens": 203, + "model.total_tokens": 1075, + "model.tokens_per_second": 299.7027484337129, + "model.estimated_cost": 0.03225, + "model.efficiency_score": 0.7581395348837209, + "model.response_length": 815, + "model.response_quality": "high", + "model.output_response": "Model Response: The provided context does not explicitly outline current trends in observability tools. However, it highlights the unique challenges faced by AI applications that traditional monitoring tools cannot a...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "45efae74-ee25-4004-a509-6e0ff628a0cd", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:58:04.883000000", + "trace_end_time": "2025-10-22 16:58:10.029000000", + "trace_duration_ms": 5146.244000000001, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:58:04.883000000", + "updated_at": "2025-10-22 16:58:04.883000000" + }, + { + "span_id": "024c05b6-edf9-40ef-931b-9660d48b4948", + "trace_id": "45efae74-ee25-4004-a509-6e0ff628a0cd", + "parent_span_id": "e81617c6-c2a1-4b08-a5fc-dde3439fe2f4", + "name": "agent.rag_evaluation_metrics", + "start_time": "2025-10-22 16:58:09.222000000", + "end_time": "2025-10-22 16:58:09.222000000", + "duration_ms": 0.020999999999999998, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "rag_evaluation_metrics", + "agent.capabilities": [ + "retrieval_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What are the current trends in observability tools?", + "query_length": 51, + "query_type": "rag_evaluation_query", + "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/blog/from-logs-to-intelligent-choices-inside-noveum-ais-evaluation-process):\nFrom Development to Production - Inside Noveum.ai's AI Observability Platform\nShashank Agarwal\n3/3/2025\nIntroduction\nEvery AI application tells a story\u2014but most developers never get to hear it...", + "retrieval.context_length": 2943, + "retrieval.sources_count": 5, + "retrieval.context_quality": "high", + "retrieval.effectiveness": 1, + "retrieval.context_utilization": 2.943, + "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/blog/from-logs-to-intelligent-choices-inside-noveum-ais-evaluation-process):\nFrom Development to Production - Inside Noveum.ai's AI Observability Platform\nShashank Agarwal\n3/3/2025\nIntroduction\nEvery AI application tells a story\u2014but most developers never get to hear it. When your RAG pipeline returns irrelevant results, when your multi-agent system gets stuck in loops, or when your LLM costs suddenly spike, you're left guessing what went wrong and where.\nThat's exactly the challenge Noveum.ai solves. Rather than flying blind, our platform provides comprehensive tracing a...\n\nSource 2 (https://noveum.ai/docs/getting-started/tracing-concepts):\n- Explore Framework Integrations - Framework-specific guidance\n- Learn Advanced Patterns - Custom instrumentation techniques\n- Master the Dashboard - Analyze your traces effectively\nRemember: Good observability is not about collecting all possible data, but about collecting the right data that helps you understand, debug, and optimize your AI applications.\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to O...\n\nSource 3 (https://noveum.ai/en/docs/getting-started/tracing-concepts):\n- Explore Framework Integrations - Framework-specific guidance\n- Learn Advanced Patterns - Custom instrumentation techniques\n- Master the Dashboard - Analyze your traces effectively\nRemember: Good observability is not about collecting all possible data, but about collecting the right data that helps you understand, debug, and optimize your AI applications.\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to O...\n\nSource 4 (https://noveum.ai/en/blog/from-logs-to-intelligent-choices-inside-noveum-ais-evaluation-process):\nThe AI Observability Challenge\nWhy Traditional Monitoring Falls Short\nTraditional application monitoring tools weren't built for AI applications. They can tell you if your API is responding, but they can't answer the questions that matter most for AI systems:\n- Why did my RAG pipeline retrieve irrelevant documents?\n- Which LLM calls are driving my costs?\n- How are my agents communicating with each other?\n- What's causing hallucinations in my responses?\n- Why is my embedding generation so slow?\nW...\n\nSource 5 (https://noveum.ai/en/docs/advanced/rag-observability):\n\ud83d\udd27 Best Practices for RAG Observability\n1. Comprehensive Pipeline Tracing\n2. Quality Monitoring\n3. A/B Testing for RAG Components\n\ud83c\udfaf Advanced RAG Patterns\nMulti-Modal RAG\nConversational RAG\nRAG observability with Noveum.ai provides the deep insights needed to build, optimize, and scale retrieval-augmented generation systems. By implementing comprehensive tracing across retrieval, context preparation, and generation stages, you can ensure your RAG pipeline delivers accurate, relevant, and cost-effe...\n\nUser Question: What are the current trends in observability tools?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", + "prompt.prompt_length": 3490, + "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/blog/from-logs-to-intelligent-choices-inside-noveum-ais-evaluation-process):\nFrom Development to Production - Inside Noveum.ai's AI Observability Platform\nShashank Agarw...", + "prompt.instruction_following": "rag_optimized", + "response.answer_length": 815, + "response.answer_completeness": "complete", + "response.response_quality": "high", + "response.source_citation": 5, + "response.context_utilization": 0.2769283044512402, + "output_response": "RAG Answer: The provided context does not explicitly outline current trends in observability tools. However, it highlights the unique challenges faced by AI applications that traditional monitoring tools cannot a...", + "evaluation.retrieval_effectiveness": 1, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 5, + "evaluation.context_relevance": "high", + "evaluation.overall_quality": "high", + "evaluation.ready_for_production": true, + "rag.retrieval_strategy": "semantic_similarity", + "rag.vector_search_results": 5, + "rag.context_synthesis": "multi_source", + "rag.document_coverage": 1, + "rag.information_density": 0.2769283044512402 + }, + "events": [], + "links": [], + "trace_trace_id": "45efae74-ee25-4004-a509-6e0ff628a0cd", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:58:04.883000000", + "trace_end_time": "2025-10-22 16:58:10.029000000", + "trace_duration_ms": 5146.244000000001, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:58:04.883000000", + "updated_at": "2025-10-22 16:58:04.883000000" + }, + { + "span_id": "99a5e8f3-a9f5-4953-94ea-ad0fa0cf6ec2", + "trace_id": "c5a64d08-d00f-46cf-ab3c-d8d473c10eca", + "parent_span_id": "2a351fd9-a7ee-42da-9e2e-86a643ceed0e", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-22 16:57:57.908000000", + "end_time": "2025-10-22 16:57:57.908000000", + "duration_ms": 0.026, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: Tell me about recent developments in machine learning", + "query_length": 53, + "query_type": "routing_evaluation_query", + "classification.mode": "Web Search", + "classification.rag_keyword_score": 0, + "classification.web_keyword_score": 1, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "complex", + "query.intent": "general_knowledge", + "query.keyword_density": 0.125, + "query.domain_affinity": "general", + "routing.decision": "Routed to Web Search based on analysis", + "routing.rationale": "RAG score: 0, Web score: 1, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "RAG", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.019230769230769232, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: Web Search (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "c5a64d08-d00f-46cf-ab3c-d8d473c10eca", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:57.908000000", + "trace_end_time": "2025-10-22 16:58:03.875000000", + "trace_duration_ms": 5967.0779999999995, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:57.908000000", + "updated_at": "2025-10-22 16:57:57.908000000" + }, + { + "span_id": "6e891e51-ca6d-4e6c-80d0-0791ee22d5b9", + "trace_id": "c5a64d08-d00f-46cf-ab3c-d8d473c10eca", + "parent_span_id": "2a351fd9-a7ee-42da-9e2e-86a643ceed0e", + "name": "agent.web_search_generation", + "start_time": "2025-10-22 16:57:57.908000000", + "end_time": "2025-10-22 16:58:03.874000000", + "duration_ms": 5966.728, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "web_search_agent", + "agent.operation": "web_search_generation", + "agent.capabilities": [ + "web_search", + "content_synthesis", + "response_generation" + ], + "agent.id": "noveum_web_search_agent", + "input_query": "Tell me about recent developments in machine learning", + "query_length": 53, + "query_type": "web_search_query", + "output_response": "Web Search Answer: The web search results did not provide specific information about recent developments in machine learning. Therefore, I cannot summarize any recent advancements based on the provided data.\n\nHowever, i...", + "web_search.results_count": 1, + "web_search.context_length": 768, + "web_search.response_length": 687, + "web_search.mode": "external_web_search" + }, + "events": [], + "links": [], + "trace_trace_id": "c5a64d08-d00f-46cf-ab3c-d8d473c10eca", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:57.908000000", + "trace_end_time": "2025-10-22 16:58:03.875000000", + "trace_duration_ms": 5967.0779999999995, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:57.908000000", + "updated_at": "2025-10-22 16:57:57.908000000" + }, + { + "span_id": "6d622376-d48b-4a64-b0a2-1402117febaf", + "trace_id": "c5a64d08-d00f-46cf-ab3c-d8d473c10eca", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-22 16:57:57.908000000", + "end_time": "2025-10-22 16:58:03.875000000", + "duration_ms": 5967.035, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "Tell me about recent developments in machine learning", + "query.length": 53, + "processing.mode": "Web Search", + "processing.time_seconds": 5.966889142990112, + "processing.response_length": 687, + "processing.sources_count": 1, + "output_response": "Final Answer: The web search results did not provide specific information about recent developments in machine learning. Therefore, I cannot summarize any recent advancements based on the provided data.\n\nHowever, i...", + "final_answer_mode": "Web Search", + "query_processed.input_query": "Tell me about recent developments in machine learning", + "query_processed.output_response": "Successfully processed query using Web Search, generated 687 character response", + "query_processed.mode": "Web Search", + "query_processed.processing_time": 5.966889142990112, + "query_processed.response_length": 687 + }, + "events": [], + "links": [], + "trace_trace_id": "c5a64d08-d00f-46cf-ab3c-d8d473c10eca", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:57.908000000", + "trace_end_time": "2025-10-22 16:58:03.875000000", + "trace_duration_ms": 5967.0779999999995, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:57.908000000", + "updated_at": "2025-10-22 16:57:57.908000000" + }, + { + "span_id": "2a351fd9-a7ee-42da-9e2e-86a643ceed0e", + "trace_id": "c5a64d08-d00f-46cf-ab3c-d8d473c10eca", + "parent_span_id": "6d622376-d48b-4a64-b0a2-1402117febaf", + "name": "agent.query_routing", + "start_time": "2025-10-22 16:57:57.908000000", + "end_time": "2025-10-22 16:58:03.875000000", + "duration_ms": 5966.8640000000005, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "Tell me about recent developments in machine learning", + "query_length": 53, + "query_type": "routing_query", + "output_response": "Routed to Web Search for query processing", + "router.classification": "Web Search", + "router.confidence_score": 1, + "router.rag_keyword_score": 0, + "router.web_keyword_score": 1, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "c5a64d08-d00f-46cf-ab3c-d8d473c10eca", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:57.908000000", + "trace_end_time": "2025-10-22 16:58:03.875000000", + "trace_duration_ms": 5967.0779999999995, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:57.908000000", + "updated_at": "2025-10-22 16:57:57.908000000" + }, + { + "span_id": "ba457b3b-9be8-4e24-af63-4118833711e6", + "trace_id": "c5a64d08-d00f-46cf-ab3c-d8d473c10eca", + "parent_span_id": "6e891e51-ca6d-4e6c-80d0-0791ee22d5b9", + "name": "agent.llm_model_execution", + "start_time": "2025-10-22 16:57:58.324000000", + "end_time": "2025-10-22 16:58:03.874000000", + "duration_ms": 5550.059, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: Tell me about recent developments in machine learning", + "query_length": 53, + "query_type": "web_search_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 5.550010919570923, + "model.latency_ms": 5550.010919570923, + "model.start_time": 1761152278.3248482, + "model.end_time": 1761152283.874859, + "model.performance_tier": "slow", + "model.prompt_tokens": 316, + "model.completion_tokens": 171, + "model.total_tokens": 487, + "model.tokens_per_second": 87.74757510524871, + "model.estimated_cost": 0.01461, + "model.efficiency_score": 1.4106776180698153, + "model.response_length": 687, + "model.response_quality": "high", + "model.output_response": "Model Response: The web search results did not provide specific information about recent developments in machine learning. Therefore, I cannot summarize any recent advancements based on the provided data.\n\nHowever, i...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "c5a64d08-d00f-46cf-ab3c-d8d473c10eca", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:57.908000000", + "trace_end_time": "2025-10-22 16:58:03.875000000", + "trace_duration_ms": 5967.0779999999995, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:57.908000000", + "updated_at": "2025-10-22 16:57:57.908000000" + }, + { + "span_id": "7b70a916-1c2b-4898-a052-cfd98a979167", + "trace_id": "c5a64d08-d00f-46cf-ab3c-d8d473c10eca", + "parent_span_id": "6e891e51-ca6d-4e6c-80d0-0791ee22d5b9", + "name": "agent.web_search_evaluation_metrics", + "start_time": "2025-10-22 16:58:03.874000000", + "end_time": "2025-10-22 16:58:03.874000000", + "duration_ms": 0.02, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "web_search_evaluation_metrics", + "agent.capabilities": [ + "web_search_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: Tell me about recent developments in machine learning", + "query_length": 53, + "query_type": "web_search_evaluation_query", + "web_search.results_count": 1, + "web_search.context_length": 768, + "web_search.context_synthesized": "Context from 1 web sources", + "web_search.search_effectiveness": 0.2, + "web_search.context_quality": "medium", + "web_search.source_diversity": 1, + "web_search.information_synthesis": "low", + "web_search.external_knowledge_utilization": 0.768, + "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nFeb 3, 2024 \u00b7 tell sb sth\u4e2d\u7684tell\u66f4\u591a\u662f\u544a\u77e5\u7684\u610f\u601d\uff0c\u5c31\u662f\u544a\u8bc9\u522b\u4eba\u6709\u8fd9\u4e48\u4e00\u4e2a\u4e8b\u7684\u5b58\u5728 tell sb about sth \u7684\u610f\u601d\u5305\u542b\u66f4\u591a\u4e00\u4e9b\uff0c\u9664\u4e86\u544a\u77e5\u6709\u8fd9\u4e48\u4e00\u4e2a\u4e8b\u60c5\u7684\u5b58\u5728\uff0c\u8fd8\u4f5c\u4e86\u4e00\u4e9b\u8bf4\u660e \u6bd4\u5982\uff1a tell me he \u2026 \u4e00\u3001Say\u548cTell\u7684\u7528\u6cd5 \u6211\u4eec\u5148\u5c06say\u3001tell\u3001speak\u3001talk\u8fd9\u56db\u4e2a\u8bcd\u5206\u6210\u4e24\u7ec4\uff0c\u5206\u522b\u4e3a\uff1a \u7b2c\u2460\u7ec4\uff1asay & tell\uff0c\u8fd9\u4e24\u4e2a\u8bcd\u4e4b\u540e\u53ef\u4ee5\u8868\u793a\u8c08\u8bdd\u7684 \u201c\u7ec6\u8282\u201d\u3002 \u7b2c\u2461\u7ec4\uff1aspeak & talk\uff0c\u4e0d\u80fd\u8868\u793a\u8c08\u8bdd\u7684\u201c\u7ec6\u8282\u201d\u3002 \u2026 tell of 1\u3001Tell of a turning point in your career and what you learned. \u8bb2\u8ff0\u4f60 \u804c\u4e1a\u751f\u6daf \u4e00\u4e2a\u8f6c\u6298\u70b9\uff0c\u4ee5\u53ca\u4f60\u5b66\u5230\u4e86\u4ec0\u4e48\u3002 2\u3001I have made the Sovereign Lord my refuge; I will tell of all your \u2026 tell sb to do sth \u548c tell sb do sth \u90fd\u8868\u793a\u544a\u8bc9\u67d0\u4eba\u505a\u67d0\u4e8b\uff0ctell sb to do sth \u5f3a\u8c03\u544a\u8bc9\u67d0\u4eba\u5e94\u8be5\u505a\u4ec0\u4e48\u5177\u4f53\u52a8\u4f5c\u6216\u884c\u4e3a\uff0c\u800c tell sb do sth \u5f3a\u8c03\u544a\u8bc9\u67d0\u4eba\u6b63\u5728\u8fdb\u884c\u7684\u5177\u4f53\u884c\u52a8\u6216\u72b6\u6001\u3002 \u5b83\u4eec\u5728\u91ca\u4e49\u3001 \u2026 Aug 23, 2023 \u00b7 tell\u7684\u8fc7\u53bb\u5f0f\u548c\u8fc7\u53bb\u5206\u8bcd\u90fd\u662ftold\u3002 \u8bfb\u97f3\uff1a\u82f1[tel]\uff0c\u7f8e[tel]\u3002 \u91ca\u4e49\uff1a vt.\u544a\u8bc9\uff0c\u8bf4\uff1b\u8fa8\u522b\uff1b\u5429\u5490\uff1b\u65ad\u5b9a\u3002 vi.\u8bb2\u8ff0\uff1b\u544a\u53d1\uff0c\u6cc4\u5bc6\uff1b\u8bc6\u522b\u3002 \u53d8\u5f62\uff1a\u8fc7\u53bb\u5f0ftold\uff0c\u8fc7\u53bb\u5206\u8bcdtold\uff0c\u73b0\u5728\u5206 \u2026\nURL: https://duckduckgo.com/?q=Tell+me+about+recent+developments+in+machine+learning\n\nUser Question: Tell me about recent developments in machine learning\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", + "prompt.prompt_length": 1266, + "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nFeb 3, 2024 \u00b7 tell sb sth\u4e2d\u7684tell\u66f4\u591a\u662f\u544a\u77e5\u7684\u610f\u601d\uff0c\u5c31\u662f\u544a\u8bc9\u522b\u4eba\u6709\u8fd9\u4e48\u4e00\u4e2a\u4e8b\u7684\u5b58\u5728 tell sb about sth \u7684\u610f\u601d\u5305\u542b\u66f4\u591a\u4e00\u4e9b\uff0c\u9664\u4e86\u544a\u77e5\u6709\u8fd9\u4e48\u4e00\u4e2a\u4e8b\u60c5\u7684\u5b58\u5728\uff0c\u8fd8\u4f5c\u4e86\u4e00\u4e9b\u8bf4\u660e \u6bd4\u5982\uff1a tell me he \u2026 \u4e00\u3001Say\u548cTell\u7684\u7528\u6cd5 \u6211\u4eec\u5148\u5c06say\u3001tell\u3001speak\u3001talk\u8fd9\u56db\u4e2a\u8bcd\u5206\u6210\u4e24\u7ec4\uff0c\u5206\u522b...", + "prompt.instruction_following": "web_search_optimized", + "response.answer_length": 687, + "response.answer_completeness": "complete", + "response.response_quality": "medium", + "response.source_citation": 1, + "response.context_utilization": 0.89453125, + "output_response": "Web Search Answer: The web search results did not provide specific information about recent developments in machine learning. Therefore, I cannot summarize any recent advancements based on the provided data.\n\nHowever, i...", + "evaluation.search_effectiveness": 0.2, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 1, + "evaluation.context_relevance": "medium", + "evaluation.overall_quality": "low", + "evaluation.ready_for_production": true, + "web_search.search_strategy": "duckduckgo_api", + "web_search.real_time_data": true, + "web_search.external_sources": 1, + "web_search.information_freshness": "current", + "web_search.knowledge_synthesis": "single_source" + }, + "events": [], + "links": [], + "trace_trace_id": "c5a64d08-d00f-46cf-ab3c-d8d473c10eca", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:57.908000000", + "trace_end_time": "2025-10-22 16:58:03.875000000", + "trace_duration_ms": 5967.0779999999995, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:57.908000000", + "updated_at": "2025-10-22 16:57:57.908000000" + }, + { + "span_id": "1e1a341f-66c1-48b0-95eb-fb092928c473", + "trace_id": "b1b13e60-5b32-44a0-9769-197a944989d0", + "parent_span_id": "1c5c4291-bbb5-4f85-b6d9-b6e8df66f742", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-22 16:57:51.967000000", + "end_time": "2025-10-22 16:57:51.967000000", + "duration_ms": 0.033, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What's the weather like today?", + "query_length": 30, + "query_type": "routing_evaluation_query", + "classification.mode": "Web Search", + "classification.rag_keyword_score": 0, + "classification.web_keyword_score": 2, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "general_knowledge", + "query.keyword_density": 0.4, + "query.domain_affinity": "general", + "routing.decision": "Routed to Web Search based on analysis", + "routing.rationale": "RAG score: 0, Web score: 2, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "RAG", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.038461538461538464, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: Web Search (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "b1b13e60-5b32-44a0-9769-197a944989d0", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:51.967000000", + "trace_end_time": "2025-10-22 16:57:56.899000000", + "trace_duration_ms": 4932.393, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:51.967000000", + "updated_at": "2025-10-22 16:57:51.967000000" + }, + { + "span_id": "1c5c4291-bbb5-4f85-b6d9-b6e8df66f742", + "trace_id": "b1b13e60-5b32-44a0-9769-197a944989d0", + "parent_span_id": "49e7cace-ba21-4807-a238-9a3880487583", + "name": "agent.query_routing", + "start_time": "2025-10-22 16:57:51.967000000", + "end_time": "2025-10-22 16:57:56.899000000", + "duration_ms": 4931.972, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What's the weather like today?", + "query_length": 30, + "query_type": "routing_query", + "output_response": "Routed to Web Search for query processing", + "router.classification": "Web Search", + "router.confidence_score": 1, + "router.rag_keyword_score": 0, + "router.web_keyword_score": 2, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "b1b13e60-5b32-44a0-9769-197a944989d0", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:51.967000000", + "trace_end_time": "2025-10-22 16:57:56.899000000", + "trace_duration_ms": 4932.393, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:51.967000000", + "updated_at": "2025-10-22 16:57:51.967000000" + }, + { + "span_id": "49e7cace-ba21-4807-a238-9a3880487583", + "trace_id": "b1b13e60-5b32-44a0-9769-197a944989d0", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-22 16:57:51.967000000", + "end_time": "2025-10-22 16:57:56.899000000", + "duration_ms": 4932.316, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What's the weather like today?", + "query.length": 30, + "processing.mode": "Web Search", + "processing.time_seconds": 4.932007789611816, + "processing.response_length": 260, + "processing.sources_count": 1, + "output_response": "Final Answer: The web search results do not provide specific information about today's weather. To find out the current weather conditions, I recommend checking a reliable weather website or app for detailed update...", + "final_answer_mode": "Web Search", + "query_processed.input_query": "What's the weather like today?", + "query_processed.output_response": "Successfully processed query using Web Search, generated 260 character response", + "query_processed.mode": "Web Search", + "query_processed.processing_time": 4.932007789611816, + "query_processed.response_length": 260 + }, + "events": [], + "links": [], + "trace_trace_id": "b1b13e60-5b32-44a0-9769-197a944989d0", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:51.967000000", + "trace_end_time": "2025-10-22 16:57:56.899000000", + "trace_duration_ms": 4932.393, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:51.967000000", + "updated_at": "2025-10-22 16:57:51.967000000" + }, + { + "span_id": "58d217aa-4350-43e8-8813-468149dc4107", + "trace_id": "b1b13e60-5b32-44a0-9769-197a944989d0", + "parent_span_id": "1c5c4291-bbb5-4f85-b6d9-b6e8df66f742", + "name": "agent.web_search_generation", + "start_time": "2025-10-22 16:57:51.967000000", + "end_time": "2025-10-22 16:57:56.899000000", + "duration_ms": 4931.796, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "web_search_agent", + "agent.operation": "web_search_generation", + "agent.capabilities": [ + "web_search", + "content_synthesis", + "response_generation" + ], + "agent.id": "noveum_web_search_agent", + "input_query": "What's the weather like today?", + "query_length": 30, + "query_type": "web_search_query", + "output_response": "Web Search Answer: The web search results do not provide specific information about today's weather. To find out the current weather conditions, I recommend checking a reliable weather website or app for detailed update...", + "web_search.results_count": 1, + "web_search.context_length": 995, + "web_search.response_length": 260, + "web_search.mode": "external_web_search" + }, + "events": [], + "links": [], + "trace_trace_id": "b1b13e60-5b32-44a0-9769-197a944989d0", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:51.967000000", + "trace_end_time": "2025-10-22 16:57:56.899000000", + "trace_duration_ms": 4932.393, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:51.967000000", + "updated_at": "2025-10-22 16:57:51.967000000" + }, + { + "span_id": "d89dd84e-0b5e-4c92-942b-18c4735713d4", + "trace_id": "b1b13e60-5b32-44a0-9769-197a944989d0", + "parent_span_id": "58d217aa-4350-43e8-8813-468149dc4107", + "name": "agent.llm_model_execution", + "start_time": "2025-10-22 16:57:55.269000000", + "end_time": "2025-10-22 16:57:56.899000000", + "duration_ms": 1629.7140000000002, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What's the weather like today?", + "query_length": 30, + "query_type": "web_search_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 1.629619836807251, + "model.latency_ms": 1629.619836807251, + "model.start_time": 1761152275.269459, + "model.end_time": 1761152276.8990788, + "model.performance_tier": "fast", + "model.prompt_tokens": 367, + "model.completion_tokens": 65, + "model.total_tokens": 432, + "model.tokens_per_second": 265.0925020932329, + "model.estimated_cost": 0.012960000000000001, + "model.efficiency_score": 0.6018518518518519, + "model.response_length": 260, + "model.response_quality": "high", + "model.output_response": "Model Response: The web search results do not provide specific information about today's weather. To find out the current weather conditions, I recommend checking a reliable weather website or app for detailed update...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "b1b13e60-5b32-44a0-9769-197a944989d0", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:51.967000000", + "trace_end_time": "2025-10-22 16:57:56.899000000", + "trace_duration_ms": 4932.393, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:51.967000000", + "updated_at": "2025-10-22 16:57:51.967000000" + }, + { + "span_id": "d076719d-da79-43fb-92ea-6015f856146b", + "trace_id": "b1b13e60-5b32-44a0-9769-197a944989d0", + "parent_span_id": "58d217aa-4350-43e8-8813-468149dc4107", + "name": "agent.web_search_evaluation_metrics", + "start_time": "2025-10-22 16:57:56.899000000", + "end_time": "2025-10-22 16:57:56.899000000", + "duration_ms": 0.049, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "web_search_evaluation_metrics", + "agent.capabilities": [ + "web_search_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What's the weather like today?", + "query_length": 30, + "query_type": "web_search_evaluation_query", + "web_search.results_count": 1, + "web_search.context_length": 995, + "web_search.context_synthesized": "Context from 1 web sources", + "web_search.search_effectiveness": 0.2, + "web_search.context_quality": "high", + "web_search.source_diversity": 1, + "web_search.information_synthesis": "low", + "web_search.external_knowledge_utilization": 0.995, + "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nLog in to WhatsApp Web for simple, reliable and private messaging on your desktop. Send and receive messages and files with ease, all for free. 2 days ago \u00b7 More videos you may like 00:13 Wanna be yours #kimpolin #colinbridgerton #benedictbri... 2 hours ago \u00b7 101 views 00:14 Viscount & Vicountess's Marital bliss #bridgerton \u2026 Nov 15, 2024 \u00b7 Learn the correct usage of \"what's the weather like today\" and \"how's the weather today\" in English. Discover differences, examples, alternatives and tips for choosing the right \u2026 Nov 15, 2024 \u00b7 Learn the correct usage of \"how's the weather today\" and \"what's the weather today\" in English. Discover differences, examples, alternatives and tips for choosing the right \u2026 2 days ago \u00b7 Get the latest hourly weather updates for Thane today. Detailed forecast including temperature, wind, rain, snow, and UV index. Stay informed about today's weather conditions in \u2026\nURL: https://duckduckgo.com/?q=What's+the+weather+like+today?\n\nUser Question: What's the weather like today?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", + "prompt.prompt_length": 1470, + "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nLog in to WhatsApp Web for simple, reliable and private messaging on your desktop. Send and receive messages and files with ease, all for free. 2 days ago \u00b7 More videos you...", + "prompt.instruction_following": "web_search_optimized", + "response.answer_length": 260, + "response.answer_completeness": "complete", + "response.response_quality": "medium", + "response.source_citation": 1, + "response.context_utilization": 0.2613065326633166, + "output_response": "Web Search Answer: The web search results do not provide specific information about today's weather. To find out the current weather conditions, I recommend checking a reliable weather website or app for detailed update...", + "evaluation.search_effectiveness": 0.2, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 1, + "evaluation.context_relevance": "high", + "evaluation.overall_quality": "low", + "evaluation.ready_for_production": true, + "web_search.search_strategy": "duckduckgo_api", + "web_search.real_time_data": true, + "web_search.external_sources": 1, + "web_search.information_freshness": "current", + "web_search.knowledge_synthesis": "single_source" + }, + "events": [], + "links": [], + "trace_trace_id": "b1b13e60-5b32-44a0-9769-197a944989d0", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:51.967000000", + "trace_end_time": "2025-10-22 16:57:56.899000000", + "trace_duration_ms": 4932.393, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:51.967000000", + "updated_at": "2025-10-22 16:57:51.967000000" + }, + { + "span_id": "41167ca4-e1ad-48a5-b5eb-aa272ad88a85", + "trace_id": "e4f3cc20-c137-4905-80fe-2d29b03c2231", + "parent_span_id": "56959f47-5219-4674-8c0d-e5fa928dcd45", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-22 16:57:47.477000000", + "end_time": "2025-10-22 16:57:47.477000000", + "duration_ms": 0.061, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What are the latest AI news today?", + "query_length": 34, + "query_type": "routing_evaluation_query", + "classification.mode": "Web Search", + "classification.rag_keyword_score": 0, + "classification.web_keyword_score": 3, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "general_knowledge", + "query.keyword_density": 0.42857142857142855, + "query.domain_affinity": "general", + "routing.decision": "Routed to Web Search based on analysis", + "routing.rationale": "RAG score: 0, Web score: 3, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "RAG", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.057692307692307696, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: Web Search (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "e4f3cc20-c137-4905-80fe-2d29b03c2231", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:47.477000000", + "trace_end_time": "2025-10-22 16:57:50.959000000", + "trace_duration_ms": 3481.856, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:47.477000000", + "updated_at": "2025-10-22 16:57:47.477000000" + }, + { + "span_id": "73005751-9eb1-4540-aa49-2ac580cd7418", + "trace_id": "e4f3cc20-c137-4905-80fe-2d29b03c2231", + "parent_span_id": "56959f47-5219-4674-8c0d-e5fa928dcd45", + "name": "agent.web_search_generation", + "start_time": "2025-10-22 16:57:47.477000000", + "end_time": "2025-10-22 16:57:50.958000000", + "duration_ms": 3480.8469999999998, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "web_search_agent", + "agent.operation": "web_search_generation", + "agent.capabilities": [ + "web_search", + "content_synthesis", + "response_generation" + ], + "agent.id": "noveum_web_search_agent", + "input_query": "What are the latest AI news today?", + "query_length": 34, + "query_type": "web_search_query", + "output_response": "Web Search Answer: The provided web search results do not contain specific information about the latest AI news today. They primarily focus on general news coverage from India and around the world, including politics, b...", + "web_search.results_count": 1, + "web_search.context_length": 887, + "web_search.response_length": 503, + "web_search.mode": "external_web_search" + }, + "events": [], + "links": [], + "trace_trace_id": "e4f3cc20-c137-4905-80fe-2d29b03c2231", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:47.477000000", + "trace_end_time": "2025-10-22 16:57:50.959000000", + "trace_duration_ms": 3481.856, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:47.477000000", + "updated_at": "2025-10-22 16:57:47.477000000" + }, + { + "span_id": "9e1b87be-bc29-4084-b37c-6e1220dde8eb", + "trace_id": "e4f3cc20-c137-4905-80fe-2d29b03c2231", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-22 16:57:47.477000000", + "end_time": "2025-10-22 16:57:50.959000000", + "duration_ms": 3481.7580000000003, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What are the latest AI news today?", + "query.length": 34, + "processing.mode": "Web Search", + "processing.time_seconds": 3.4813451766967773, + "processing.response_length": 503, + "processing.sources_count": 1, + "output_response": "Final Answer: The provided web search results do not contain specific information about the latest AI news today. They primarily focus on general news coverage from India and around the world, including politics, b...", + "final_answer_mode": "Web Search", + "query_processed.input_query": "What are the latest AI news today?", + "query_processed.output_response": "Successfully processed query using Web Search, generated 503 character response", + "query_processed.mode": "Web Search", + "query_processed.processing_time": 3.4813451766967773, + "query_processed.response_length": 503 + }, + "events": [], + "links": [], + "trace_trace_id": "e4f3cc20-c137-4905-80fe-2d29b03c2231", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:47.477000000", + "trace_end_time": "2025-10-22 16:57:50.959000000", + "trace_duration_ms": 3481.856, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:47.477000000", + "updated_at": "2025-10-22 16:57:47.477000000" + }, + { + "span_id": "56959f47-5219-4674-8c0d-e5fa928dcd45", + "trace_id": "e4f3cc20-c137-4905-80fe-2d29b03c2231", + "parent_span_id": "9e1b87be-bc29-4084-b37c-6e1220dde8eb", + "name": "agent.query_routing", + "start_time": "2025-10-22 16:57:47.477000000", + "end_time": "2025-10-22 16:57:50.958000000", + "duration_ms": 3481.254, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What are the latest AI news today?", + "query_length": 34, + "query_type": "routing_query", + "output_response": "Routed to Web Search for query processing", + "router.classification": "Web Search", + "router.confidence_score": 1, + "router.rag_keyword_score": 0, + "router.web_keyword_score": 3, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "e4f3cc20-c137-4905-80fe-2d29b03c2231", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:47.477000000", + "trace_end_time": "2025-10-22 16:57:50.959000000", + "trace_duration_ms": 3481.856, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:47.477000000", + "updated_at": "2025-10-22 16:57:47.477000000" + }, + { + "span_id": "3dbf95ba-3ef1-4294-93d4-217f4ada5ffe", + "trace_id": "e4f3cc20-c137-4905-80fe-2d29b03c2231", + "parent_span_id": "73005751-9eb1-4540-aa49-2ac580cd7418", + "name": "agent.llm_model_execution", + "start_time": "2025-10-22 16:57:47.864000000", + "end_time": "2025-10-22 16:57:50.958000000", + "duration_ms": 3093.442, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What are the latest AI news today?", + "query_length": 34, + "query_type": "web_search_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 3.0933361053466797, + "model.latency_ms": 3093.3361053466797, + "model.start_time": 1761152267.864976, + "model.end_time": 1761152270.958312, + "model.performance_tier": "medium", + "model.prompt_tokens": 341, + "model.completion_tokens": 125, + "model.total_tokens": 466, + "model.tokens_per_second": 150.6464167261171, + "model.estimated_cost": 0.013980000000000001, + "model.efficiency_score": 1.0793991416309012, + "model.response_length": 503, + "model.response_quality": "high", + "model.output_response": "Model Response: The provided web search results do not contain specific information about the latest AI news today. They primarily focus on general news coverage from India and around the world, including politics, b...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "e4f3cc20-c137-4905-80fe-2d29b03c2231", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:47.477000000", + "trace_end_time": "2025-10-22 16:57:50.959000000", + "trace_duration_ms": 3481.856, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:47.477000000", + "updated_at": "2025-10-22 16:57:47.477000000" + }, + { + "span_id": "f7fe795b-ce1b-4fcc-83f5-06a5f074f38d", + "trace_id": "e4f3cc20-c137-4905-80fe-2d29b03c2231", + "parent_span_id": "73005751-9eb1-4540-aa49-2ac580cd7418", + "name": "agent.web_search_evaluation_metrics", + "start_time": "2025-10-22 16:57:50.958000000", + "end_time": "2025-10-22 16:57:50.958000000", + "duration_ms": 0.026, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "web_search_evaluation_metrics", + "agent.capabilities": [ + "web_search_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What are the latest AI news today?", + "query_length": 34, + "query_type": "web_search_evaluation_query", + "web_search.results_count": 1, + "web_search.context_length": 887, + "web_search.context_synthesized": "Context from 1 web sources", + "web_search.search_effectiveness": 0.2, + "web_search.context_quality": "high", + "web_search.source_diversity": 1, + "web_search.information_synthesis": "low", + "web_search.external_knowledge_utilization": 0.887, + "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\n3 days ago \u00b7 Latest news headlines from India & around the world. Check out today\u2019s news coverage live with videos & photos on NDTV.com. 6 days ago \u00b7 India News | Latest India News | Read latest and breaking news from India. Today's top India news headlines, news on Indian politics, elections, government, business, \u2026 2 days ago \u00b7 Top News Stories of the day, Latest News Headlines, News Specials, Breaking News and Latest India News, World current affairs & Political News all around the clock at \u2026 NDTV.com provides latest news from India and around the world. Get breaking news alerts from India and follow today\u2019s live news updates in field of politics, business, technology, Bollywood ... 6 days ago \u00b7 Get Latest entertainment news from Bollywood, Hollywood and regional film industries.\nURL: https://duckduckgo.com/?q=What+are+the+latest+AI+news+today?\n\nUser Question: What are the latest AI news today?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", + "prompt.prompt_length": 1366, + "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\n3 days ago \u00b7 Latest news headlines from India & around the world. Check out today\u2019s news coverage live with videos & photos on NDTV.com. 6 days ago \u00b7 India News | Latest In...", + "prompt.instruction_following": "web_search_optimized", + "response.answer_length": 503, + "response.answer_completeness": "complete", + "response.response_quality": "medium", + "response.source_citation": 1, + "response.context_utilization": 0.5670800450958287, + "output_response": "Web Search Answer: The provided web search results do not contain specific information about the latest AI news today. They primarily focus on general news coverage from India and around the world, including politics, b...", + "evaluation.search_effectiveness": 0.2, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 1, + "evaluation.context_relevance": "high", + "evaluation.overall_quality": "low", + "evaluation.ready_for_production": true, + "web_search.search_strategy": "duckduckgo_api", + "web_search.real_time_data": true, + "web_search.external_sources": 1, + "web_search.information_freshness": "current", + "web_search.knowledge_synthesis": "single_source" + }, + "events": [], + "links": [], + "trace_trace_id": "e4f3cc20-c137-4905-80fe-2d29b03c2231", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:47.477000000", + "trace_end_time": "2025-10-22 16:57:50.959000000", + "trace_duration_ms": 3481.856, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:47.477000000", + "updated_at": "2025-10-22 16:57:47.477000000" + }, + { + "span_id": "b9c9d21e-5f76-4e5a-b99e-2b4ab821caee", + "trace_id": "823b7927-10bb-49ef-a94e-1b914c298bfb", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-22 16:57:38.501000000", + "end_time": "2025-10-22 16:57:46.469000000", + "duration_ms": 7968.089, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What are the benefits of using Noveum Trace?", + "query.length": 44, + "processing.mode": "RAG", + "processing.time_seconds": 7.968040943145752, + "processing.response_length": 1396, + "processing.sources_count": 5, + "output_response": "Final Answer: Using Noveum Trace offers several benefits, including:\n\n1. **Real-Time Monitoring**: The Noveum platform provides a real-time dashboard for analyzing traces and performance, with live updates as new t...", + "final_answer_mode": "RAG", + "query_processed.input_query": "What are the benefits of using Noveum Trace?", + "query_processed.output_response": "Successfully processed query using RAG, generated 1396 character response", + "query_processed.mode": "RAG", + "query_processed.processing_time": 7.968040943145752, + "query_processed.response_length": 1396 + }, + "events": [], + "links": [], + "trace_trace_id": "823b7927-10bb-49ef-a94e-1b914c298bfb", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:38.501000000", + "trace_end_time": "2025-10-22 16:57:46.469000000", + "trace_duration_ms": 7968.149, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:38.501000000", + "updated_at": "2025-10-22 16:57:38.501000000" + }, + { + "span_id": "bc85c745-4c27-4aa4-bc00-4c9f10a2f049", + "trace_id": "823b7927-10bb-49ef-a94e-1b914c298bfb", + "parent_span_id": "b9c9d21e-5f76-4e5a-b99e-2b4ab821caee", + "name": "agent.query_routing", + "start_time": "2025-10-22 16:57:38.501000000", + "end_time": "2025-10-22 16:57:46.469000000", + "duration_ms": 7968.001, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What are the benefits of using Noveum Trace?", + "query_length": 44, + "query_type": "routing_query", + "output_response": "Routed to RAG for query processing", + "router.classification": "RAG", + "router.confidence_score": 1, + "router.rag_keyword_score": 2, + "router.web_keyword_score": 0, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "823b7927-10bb-49ef-a94e-1b914c298bfb", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:38.501000000", + "trace_end_time": "2025-10-22 16:57:46.469000000", + "trace_duration_ms": 7968.149, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:38.501000000", + "updated_at": "2025-10-22 16:57:38.501000000" + }, + { + "span_id": "3ab8cc4c-f5ed-410a-b4ee-48d0954b756d", + "trace_id": "823b7927-10bb-49ef-a94e-1b914c298bfb", + "parent_span_id": "bc85c745-4c27-4aa4-bc00-4c9f10a2f049", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-22 16:57:38.501000000", + "end_time": "2025-10-22 16:57:38.501000000", + "duration_ms": 0.09899999999999999, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What are the benefits of using Noveum Trace?", + "query_length": 44, + "query_type": "routing_evaluation_query", + "classification.mode": "RAG", + "classification.rag_keyword_score": 2, + "classification.web_keyword_score": 0, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "noveum_specific", + "query.keyword_density": 0.25, + "query.domain_affinity": "noveum", + "routing.decision": "Routed to RAG based on analysis", + "routing.rationale": "RAG score: 2, Web score: 0, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "Web Search", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.038461538461538464, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: RAG (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "823b7927-10bb-49ef-a94e-1b914c298bfb", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:38.501000000", + "trace_end_time": "2025-10-22 16:57:46.469000000", + "trace_duration_ms": 7968.149, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:38.501000000", + "updated_at": "2025-10-22 16:57:38.501000000" + }, + { + "span_id": "67292076-ec8a-4856-9c5b-726868552157", + "trace_id": "823b7927-10bb-49ef-a94e-1b914c298bfb", + "parent_span_id": "bc85c745-4c27-4aa4-bc00-4c9f10a2f049", + "name": "agent.llm-rag", + "start_time": "2025-10-22 16:57:38.501000000", + "end_time": "2025-10-22 16:57:46.469000000", + "duration_ms": 7967.679, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "rag_agent", + "agent.operation": "llm-rag", + "agent.capabilities": [ + "document_retrieval", + "context_generation", + "response_generation" + ], + "agent.id": "noveum_rag_agent", + "input_query": "What are the benefits of using Noveum Trace?", + "query_length": 44, + "query_type": "rag_query", + "output_response": "RAG Answer: Using Noveum Trace offers several benefits, including:\n\n1. **Real-Time Monitoring**: The Noveum platform provides a real-time dashboard for analyzing traces and performance, with live updates as new t...", + "rag.context_length": 2773, + "rag.sources_count": 5, + "rag.answer_length": 1396, + "rag.mode": "retrieval_augmented_generation" + }, + "events": [], + "links": [], + "trace_trace_id": "823b7927-10bb-49ef-a94e-1b914c298bfb", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:38.501000000", + "trace_end_time": "2025-10-22 16:57:46.469000000", + "trace_duration_ms": 7968.149, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:38.501000000", + "updated_at": "2025-10-22 16:57:38.501000000" + }, + { + "span_id": "d044ae4a-c463-4c1b-abce-a9a18f045e75", + "trace_id": "823b7927-10bb-49ef-a94e-1b914c298bfb", + "parent_span_id": "67292076-ec8a-4856-9c5b-726868552157", + "name": "agent.llm_model_execution", + "start_time": "2025-10-22 16:57:38.901000000", + "end_time": "2025-10-22 16:57:45.885000000", + "duration_ms": 6984.242, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What are the benefits of using Noveum Trace?", + "query_length": 44, + "query_type": "rag_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 6.984165906906128, + "model.latency_ms": 6984.165906906128, + "model.start_time": 1761152258.9012191, + "model.end_time": 1761152265.885385, + "model.performance_tier": "slow", + "model.prompt_tokens": 828, + "model.completion_tokens": 349, + "model.total_tokens": 1177, + "model.tokens_per_second": 168.524060809631, + "model.estimated_cost": 0.03531, + "model.efficiency_score": 1.1860662701784197, + "model.response_length": 1396, + "model.response_quality": "high", + "model.output_response": "Model Response: Using Noveum Trace offers several benefits, including:\n\n1. **Real-Time Monitoring**: The Noveum platform provides a real-time dashboard for analyzing traces and performance, with live updates as new t...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "823b7927-10bb-49ef-a94e-1b914c298bfb", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:38.501000000", + "trace_end_time": "2025-10-22 16:57:46.469000000", + "trace_duration_ms": 7968.149, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:38.501000000", + "updated_at": "2025-10-22 16:57:38.501000000" + }, + { + "span_id": "5b1548cf-0de7-49d4-b9b8-9ad1f9f2abc9", + "trace_id": "823b7927-10bb-49ef-a94e-1b914c298bfb", + "parent_span_id": "67292076-ec8a-4856-9c5b-726868552157", + "name": "agent.rag_evaluation_metrics", + "start_time": "2025-10-22 16:57:45.885000000", + "end_time": "2025-10-22 16:57:45.885000000", + "duration_ms": 0.061, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "rag_evaluation_metrics", + "agent.capabilities": [ + "retrieval_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What are the benefits of using Noveum Trace?", + "query_length": 44, + "query_type": "rag_evaluation_query", + "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/docs/platform/dashboard):\n- Bulk Export: Download multiple traces simultaneously\n- Comparative Analysis: Compare performance across multiple traces\nReal-Time Updates\n- Live Refresh: Automatic updates as new traces arrive\n- Connection Monitoring: Real-time status of your t...", + "retrieval.context_length": 2773, + "retrieval.sources_count": 5, + "retrieval.context_quality": "high", + "retrieval.effectiveness": 1, + "retrieval.context_utilization": 2.773, + "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/docs/platform/dashboard):\n- Bulk Export: Download multiple traces simultaneously\n- Comparative Analysis: Compare performance across multiple traces\nReal-Time Updates\n- Live Refresh: Automatic updates as new traces arrive\n- Connection Monitoring: Real-time status of your trace ingestion pipeline\n- Performance Indicators: Live metrics for system health monitoring\nReady to dive deeper? Explore Projects & Environments to organize your AI applications, or check out Team Collaboration to share insights with your team.\nGet Earl...\n\nSource 2 (https://noveum.ai/en/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\n\ud83d\udcd8 TypeScript SDK (\n@noveum/trace\n)- Framework integrations for Next.js, Express.js, and Hono\n- TypeScript-first design with full type safety\n- Universal compatibility (Node.js, Edge Runtime, browsers)\n-\n\ud83d\udcca Noveum Platform\n- Real-time dashboard for analyzing traces and performance\n- Ad...\n\nSource 3 (https://noveum.ai/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\n\ud83d\udcd8 TypeScript SDK (\n@noveum/trace\n)- Framework integrations for Next.js, Express.js, and Hono\n- TypeScript-first design with full type safety\n- Universal compatibility (Node.js, Edge Runtime, browsers)\n-\n\ud83d\udcca Noveum Platform\n- Real-time dashboard for analyzing traces and performance\n- Ad...\n\nSource 4 (https://noveum.ai/en/docs/getting-started/overview):\n- Advanced search & filtering across traces and spans\n- Cost analysis and optimization recommendations\n- Team collaboration with shared insights and alerts\n\ud83d\udd0d What Noveum Traces\nLLM Operations\n- Model calls across all providers (OpenAI, Anthropic, Google, etc.)\n- Token usage and cost calculation\n- Prompt engineering effectiveness\n- Response quality metrics\nRAG Pipelines\n- Document retrieval performance and relevance\n- Embedding generation costs and latency\n- Context assembly and prompt constructi...\n\nSource 5 (https://noveum.ai/docs/getting-started/overview):\n- Advanced search & filtering across traces and spans\n- Cost analysis and optimization recommendations\n- Team collaboration with shared insights and alerts\n\ud83d\udd0d What Noveum Traces\nLLM Operations\n- Model calls across all providers (OpenAI, Anthropic, Google, etc.)\n- Token usage and cost calculation\n- Prompt engineering effectiveness\n- Response quality metrics\nRAG Pipelines\n- Document retrieval performance and relevance\n- Embedding generation costs and latency\n- Context assembly and prompt constructi...\n\nUser Question: What are the benefits of using Noveum Trace?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", + "prompt.prompt_length": 3313, + "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/docs/platform/dashboard):\n- Bulk Export: Download multiple traces simultaneously\n- Comparative Analysis: Compare performance across multiple traces\nReal-Time Updates\n- Live...", + "prompt.instruction_following": "rag_optimized", + "response.answer_length": 1396, + "response.answer_completeness": "complete", + "response.response_quality": "high", + "response.source_citation": 5, + "response.context_utilization": 0.5034258925351605, + "output_response": "RAG Answer: Using Noveum Trace offers several benefits, including:\n\n1. **Real-Time Monitoring**: The Noveum platform provides a real-time dashboard for analyzing traces and performance, with live updates as new t...", + "evaluation.retrieval_effectiveness": 1, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 5, + "evaluation.context_relevance": "high", + "evaluation.overall_quality": "high", + "evaluation.ready_for_production": true, + "rag.retrieval_strategy": "semantic_similarity", + "rag.vector_search_results": 5, + "rag.context_synthesis": "multi_source", + "rag.document_coverage": 1, + "rag.information_density": 0.5034258925351605 + }, + "events": [], + "links": [], + "trace_trace_id": "823b7927-10bb-49ef-a94e-1b914c298bfb", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:38.501000000", + "trace_end_time": "2025-10-22 16:57:46.469000000", + "trace_duration_ms": 7968.149, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:38.501000000", + "updated_at": "2025-10-22 16:57:38.501000000" + }, + { + "span_id": "aaba719b-36ec-46ab-af2e-3ea8c18e9872", + "trace_id": "1ae0d6c4-d1fb-4239-b536-3333187d44aa", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-22 16:57:32.800000000", + "end_time": "2025-10-22 16:57:37.494000000", + "duration_ms": 4694.408, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "How do I configure Noveum for my system?", + "query.length": 40, + "processing.mode": "RAG", + "processing.time_seconds": 4.694374084472656, + "processing.response_length": 553, + "processing.sources_count": 5, + "output_response": "Final Answer: To configure Noveum for your system, you can refer to the SDK Integration Guide available in the Noveum documentation. This guide will help you trace your AI applications with minimal code changes. Th...", + "final_answer_mode": "RAG", + "query_processed.input_query": "How do I configure Noveum for my system?", + "query_processed.output_response": "Successfully processed query using RAG, generated 553 character response", + "query_processed.mode": "RAG", + "query_processed.processing_time": 4.694374084472656, + "query_processed.response_length": 553 + }, + "events": [], + "links": [], + "trace_trace_id": "1ae0d6c4-d1fb-4239-b536-3333187d44aa", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:32.800000000", + "trace_end_time": "2025-10-22 16:57:37.494000000", + "trace_duration_ms": 4694.468, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:32.800000000", + "updated_at": "2025-10-22 16:57:32.800000000" + }, + { + "span_id": "4254e6f9-10b8-42ba-939d-d7c124f80f9a", + "trace_id": "1ae0d6c4-d1fb-4239-b536-3333187d44aa", + "parent_span_id": "6d5d074e-8fd3-4d33-88fb-7f54b0959030", + "name": "agent.llm-rag", + "start_time": "2025-10-22 16:57:32.800000000", + "end_time": "2025-10-22 16:57:37.494000000", + "duration_ms": 4694.155000000001, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "rag_agent", + "agent.operation": "llm-rag", + "agent.capabilities": [ + "document_retrieval", + "context_generation", + "response_generation" + ], + "agent.id": "noveum_rag_agent", + "input_query": "How do I configure Noveum for my system?", + "query_length": 40, + "query_type": "rag_query", + "output_response": "RAG Answer: To configure Noveum for your system, you can refer to the SDK Integration Guide available in the Noveum documentation. This guide will help you trace your AI applications with minimal code changes. Th...", + "rag.context_length": 2237, + "rag.sources_count": 5, + "rag.answer_length": 553, + "rag.mode": "retrieval_augmented_generation" + }, + "events": [], + "links": [], + "trace_trace_id": "1ae0d6c4-d1fb-4239-b536-3333187d44aa", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:32.800000000", + "trace_end_time": "2025-10-22 16:57:37.494000000", + "trace_duration_ms": 4694.468, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:32.800000000", + "updated_at": "2025-10-22 16:57:32.800000000" + }, + { + "span_id": "6d5d074e-8fd3-4d33-88fb-7f54b0959030", + "trace_id": "1ae0d6c4-d1fb-4239-b536-3333187d44aa", + "parent_span_id": "aaba719b-36ec-46ab-af2e-3ea8c18e9872", + "name": "agent.query_routing", + "start_time": "2025-10-22 16:57:32.800000000", + "end_time": "2025-10-22 16:57:37.494000000", + "duration_ms": 4694.318, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "How do I configure Noveum for my system?", + "query_length": 40, + "query_type": "routing_query", + "output_response": "Routed to RAG for query processing", + "router.classification": "RAG", + "router.confidence_score": 1, + "router.rag_keyword_score": 2, + "router.web_keyword_score": 0, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "1ae0d6c4-d1fb-4239-b536-3333187d44aa", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:32.800000000", + "trace_end_time": "2025-10-22 16:57:37.494000000", + "trace_duration_ms": 4694.468, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:32.800000000", + "updated_at": "2025-10-22 16:57:32.800000000" + }, + { + "span_id": "2cc2ab0b-2572-4a75-88a3-18b3a1eaca78", + "trace_id": "1ae0d6c4-d1fb-4239-b536-3333187d44aa", + "parent_span_id": "6d5d074e-8fd3-4d33-88fb-7f54b0959030", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-22 16:57:32.800000000", + "end_time": "2025-10-22 16:57:32.800000000", + "duration_ms": 0.04, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: How do I configure Noveum for my system?", + "query_length": 40, + "query_type": "routing_evaluation_query", + "classification.mode": "RAG", + "classification.rag_keyword_score": 2, + "classification.web_keyword_score": 0, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "noveum_specific", + "query.keyword_density": 0.25, + "query.domain_affinity": "noveum", + "routing.decision": "Routed to RAG based on analysis", + "routing.rationale": "RAG score: 2, Web score: 0, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "Web Search", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.038461538461538464, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: RAG (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "1ae0d6c4-d1fb-4239-b536-3333187d44aa", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:32.800000000", + "trace_end_time": "2025-10-22 16:57:37.494000000", + "trace_duration_ms": 4694.468, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:32.800000000", + "updated_at": "2025-10-22 16:57:32.800000000" + }, + { + "span_id": "f1d9f779-3312-43c1-b549-00fc874429f0", + "trace_id": "1ae0d6c4-d1fb-4239-b536-3333187d44aa", + "parent_span_id": "4254e6f9-10b8-42ba-939d-d7c124f80f9a", + "name": "agent.llm_model_execution", + "start_time": "2025-10-22 16:57:33.281000000", + "end_time": "2025-10-22 16:57:37.060000000", + "duration_ms": 3778.743, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: How do I configure Noveum for my system?", + "query_length": 40, + "query_type": "rag_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 3.7786097526550293, + "model.latency_ms": 3778.6097526550293, + "model.start_time": 1761152253.28163, + "model.end_time": 1761152257.0602398, + "model.performance_tier": "medium", + "model.prompt_tokens": 693, + "model.completion_tokens": 138, + "model.total_tokens": 831, + "model.tokens_per_second": 219.92215507730066, + "model.estimated_cost": 0.02493, + "model.efficiency_score": 0.6654632972322503, + "model.response_length": 553, + "model.response_quality": "high", + "model.output_response": "Model Response: To configure Noveum for your system, you can refer to the SDK Integration Guide available in the Noveum documentation. This guide will help you trace your AI applications with minimal code changes. Th...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "1ae0d6c4-d1fb-4239-b536-3333187d44aa", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:32.800000000", + "trace_end_time": "2025-10-22 16:57:37.494000000", + "trace_duration_ms": 4694.468, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:32.800000000", + "updated_at": "2025-10-22 16:57:32.800000000" + }, + { + "span_id": "5500be90-0cfa-44dd-a6e3-e72ac9f0a79e", + "trace_id": "1ae0d6c4-d1fb-4239-b536-3333187d44aa", + "parent_span_id": "4254e6f9-10b8-42ba-939d-d7c124f80f9a", + "name": "agent.rag_evaluation_metrics", + "start_time": "2025-10-22 16:57:37.060000000", + "end_time": "2025-10-22 16:57:37.060000000", + "duration_ms": 0.076, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "rag_evaluation_metrics", + "agent.capabilities": [ + "retrieval_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: How do I configure Noveum for my system?", + "query_length": 40, + "query_type": "rag_evaluation_query", + "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/docs):\nWelcome to Noveum.ai Docs\nComprehensive AI tracing and observability for LLM applications, RAG systems, and AI agents\nWelcome to the Noveum.ai documentation! Here you'll find everything you need to integrate, configure, and optimize your AI applications using ou...", + "retrieval.context_length": 2237, + "retrieval.sources_count": 5, + "retrieval.context_quality": "high", + "retrieval.effectiveness": 1, + "retrieval.context_utilization": 2.237, + "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/docs):\nWelcome to Noveum.ai Docs\nComprehensive AI tracing and observability for LLM applications, RAG systems, and AI agents\nWelcome to the Noveum.ai documentation! Here you'll find everything you need to integrate, configure, and optimize your AI applications using our comprehensive tracing and observability platform.\nWe provide powerful SDKs for Python and TypeScript that enable you to trace LLM calls, RAG pipelines, and multi-agent workflows with minimal code changes. Our documentation is organized ...\n\nSource 2 (https://noveum.ai/docs):\nWelcome to Noveum.ai Docs\nComprehensive AI tracing and observability for LLM applications, RAG systems, and AI agents\nWelcome to the Noveum.ai documentation! Here you'll find everything you need to integrate, configure, and optimize your AI applications using our comprehensive tracing and observability platform.\nWe provide powerful SDKs for Python and TypeScript that enable you to trace LLM calls, RAG pipelines, and multi-agent workflows with minimal code changes. Our documentation is organized ...\n\nSource 3 (https://noveum.ai/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nSource 4 (https://noveum.ai/en/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nSource 5 (https://noveum.ai/en/docs):\n- \ud83c\udf0d Universal: Works across frameworks, clouds, and deployment models\n\ud83e\udd1d Community & Support\n- \ud83d\udcac Discord Community: Join our Discord\n- \ud83d\udce7 Email Support: [email protected]\n- \ud83d\udc1b Bug Reports: GitHub Issues\n- \ud83d\udcd6 Knowledge Base: Help Center\nReady to get started? Head to our SDK Integration Guide to begin tracing your AI applications in under 5 minutes!\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability ...\n\nUser Question: How do I configure Noveum for my system?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", + "prompt.prompt_length": 2773, + "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/docs):\nWelcome to Noveum.ai Docs\nComprehensive AI tracing and observability for LLM applications, RAG systems, and AI agents\nWelcome to the Noveum.ai documentation! Here...", + "prompt.instruction_following": "rag_optimized", + "response.answer_length": 553, + "response.answer_completeness": "complete", + "response.response_quality": "high", + "response.source_citation": 5, + "response.context_utilization": 0.24720607957085383, + "output_response": "RAG Answer: To configure Noveum for your system, you can refer to the SDK Integration Guide available in the Noveum documentation. This guide will help you trace your AI applications with minimal code changes. Th...", + "evaluation.retrieval_effectiveness": 1, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 5, + "evaluation.context_relevance": "high", + "evaluation.overall_quality": "high", + "evaluation.ready_for_production": true, + "rag.retrieval_strategy": "semantic_similarity", + "rag.vector_search_results": 5, + "rag.context_synthesis": "multi_source", + "rag.document_coverage": 1, + "rag.information_density": 0.24720607957085383 + }, + "events": [], + "links": [], + "trace_trace_id": "1ae0d6c4-d1fb-4239-b536-3333187d44aa", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:32.800000000", + "trace_end_time": "2025-10-22 16:57:37.494000000", + "trace_duration_ms": 4694.468, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:32.800000000", + "updated_at": "2025-10-22 16:57:32.800000000" + }, + { + "span_id": "3068013a-6844-4f3c-8ec2-854a7353591e", + "trace_id": "fb0d31fc-2465-4ecd-973c-4cd443d9befa", + "parent_span_id": "a80d8fab-d5dc-4417-a5e5-9a2a76ed7419", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-22 16:57:24.828000000", + "end_time": "2025-10-22 16:57:24.828000000", + "duration_ms": 0.038, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What monitoring capabilities does Noveum provide?", + "query_length": 49, + "query_type": "routing_evaluation_query", + "classification.mode": "RAG", + "classification.rag_keyword_score": 2, + "classification.web_keyword_score": 0, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "noveum_specific", + "query.keyword_density": 0.3333333333333333, + "query.domain_affinity": "noveum", + "routing.decision": "Routed to RAG based on analysis", + "routing.rationale": "RAG score: 2, Web score: 0, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "Web Search", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.038461538461538464, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: RAG (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "fb0d31fc-2465-4ecd-973c-4cd443d9befa", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:24.828000000", + "trace_end_time": "2025-10-22 16:57:31.793000000", + "trace_duration_ms": 6965.302000000001, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:24.828000000", + "updated_at": "2025-10-22 16:57:24.828000000" + }, + { + "span_id": "189b89e3-fbbd-41de-8521-efb6a68f62f2", + "trace_id": "fb0d31fc-2465-4ecd-973c-4cd443d9befa", + "parent_span_id": "a80d8fab-d5dc-4417-a5e5-9a2a76ed7419", + "name": "agent.llm-rag", + "start_time": "2025-10-22 16:57:24.828000000", + "end_time": "2025-10-22 16:57:31.793000000", + "duration_ms": 6964.955, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "rag_agent", + "agent.operation": "llm-rag", + "agent.capabilities": [ + "document_retrieval", + "context_generation", + "response_generation" + ], + "agent.id": "noveum_rag_agent", + "input_query": "What monitoring capabilities does Noveum provide?", + "query_length": 49, + "query_type": "rag_query", + "output_response": "RAG Answer: Noveum provides comprehensive monitoring capabilities for AI applications, including the ability to monitor, trace, and optimize AI agents across various frameworks such as LangChain, CrewAI, AutoGen,...", + "rag.context_length": 2740, + "rag.sources_count": 5, + "rag.answer_length": 582, + "rag.mode": "retrieval_augmented_generation" + }, + "events": [], + "links": [], + "trace_trace_id": "fb0d31fc-2465-4ecd-973c-4cd443d9befa", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:24.828000000", + "trace_end_time": "2025-10-22 16:57:31.793000000", + "trace_duration_ms": 6965.302000000001, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:24.828000000", + "updated_at": "2025-10-22 16:57:24.828000000" + }, + { + "span_id": "f5cffb7d-341b-4cde-b162-02208042463d", + "trace_id": "fb0d31fc-2465-4ecd-973c-4cd443d9befa", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-22 16:57:24.828000000", + "end_time": "2025-10-22 16:57:31.793000000", + "duration_ms": 6965.236, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What monitoring capabilities does Noveum provide?", + "query.length": 49, + "processing.mode": "RAG", + "processing.time_seconds": 6.965206146240234, + "processing.response_length": 582, + "processing.sources_count": 5, + "output_response": "Final Answer: Noveum provides comprehensive monitoring capabilities for AI applications, including the ability to monitor, trace, and optimize AI agents across various frameworks such as LangChain, CrewAI, AutoGen,...", + "final_answer_mode": "RAG", + "query_processed.input_query": "What monitoring capabilities does Noveum provide?", + "query_processed.output_response": "Successfully processed query using RAG, generated 582 character response", + "query_processed.mode": "RAG", + "query_processed.processing_time": 6.965206146240234, + "query_processed.response_length": 582 + }, + "events": [], + "links": [], + "trace_trace_id": "fb0d31fc-2465-4ecd-973c-4cd443d9befa", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:24.828000000", + "trace_end_time": "2025-10-22 16:57:31.793000000", + "trace_duration_ms": 6965.302000000001, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:24.828000000", + "updated_at": "2025-10-22 16:57:24.828000000" + }, + { + "span_id": "a80d8fab-d5dc-4417-a5e5-9a2a76ed7419", + "trace_id": "fb0d31fc-2465-4ecd-973c-4cd443d9befa", + "parent_span_id": "f5cffb7d-341b-4cde-b162-02208042463d", + "name": "agent.query_routing", + "start_time": "2025-10-22 16:57:24.828000000", + "end_time": "2025-10-22 16:57:31.793000000", + "duration_ms": 6965.17, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What monitoring capabilities does Noveum provide?", + "query_length": 49, + "query_type": "routing_query", + "output_response": "Routed to RAG for query processing", + "router.classification": "RAG", + "router.confidence_score": 1, + "router.rag_keyword_score": 2, + "router.web_keyword_score": 0, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "fb0d31fc-2465-4ecd-973c-4cd443d9befa", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:24.828000000", + "trace_end_time": "2025-10-22 16:57:31.793000000", + "trace_duration_ms": 6965.302000000001, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:24.828000000", + "updated_at": "2025-10-22 16:57:24.828000000" + }, + { + "span_id": "a2b632b8-fbcb-480f-9292-9f513c64df38", + "trace_id": "fb0d31fc-2465-4ecd-973c-4cd443d9befa", + "parent_span_id": "189b89e3-fbbd-41de-8521-efb6a68f62f2", + "name": "agent.llm_model_execution", + "start_time": "2025-10-22 16:57:27.824000000", + "end_time": "2025-10-22 16:57:31.155000000", + "duration_ms": 3330.989, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What monitoring capabilities does Noveum provide?", + "query_length": 49, + "query_type": "rag_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 3.3309478759765625, + "model.latency_ms": 3330.9478759765625, + "model.start_time": 1761152247.824474, + "model.end_time": 1761152251.155422, + "model.performance_tier": "medium", + "model.prompt_tokens": 821, + "model.completion_tokens": 145, + "model.total_tokens": 966, + "model.tokens_per_second": 290.00754018607677, + "model.estimated_cost": 0.028980000000000002, + "model.efficiency_score": 0.6024844720496895, + "model.response_length": 582, + "model.response_quality": "high", + "model.output_response": "Model Response: Noveum provides comprehensive monitoring capabilities for AI applications, including the ability to monitor, trace, and optimize AI agents across various frameworks such as LangChain, CrewAI, AutoGen,...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "fb0d31fc-2465-4ecd-973c-4cd443d9befa", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:24.828000000", + "trace_end_time": "2025-10-22 16:57:31.793000000", + "trace_duration_ms": 6965.302000000001, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:24.828000000", + "updated_at": "2025-10-22 16:57:24.828000000" + }, + { + "span_id": "40ff54c4-07ca-4b2d-a9b4-7e3ae7adcd81", + "trace_id": "fb0d31fc-2465-4ecd-973c-4cd443d9befa", + "parent_span_id": "189b89e3-fbbd-41de-8521-efb6a68f62f2", + "name": "agent.rag_evaluation_metrics", + "start_time": "2025-10-22 16:57:31.155000000", + "end_time": "2025-10-22 16:57:31.155000000", + "duration_ms": 0.018000000000000002, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "rag_evaluation_metrics", + "agent.capabilities": [ + "retrieval_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What monitoring capabilities does Noveum provide?", + "query_length": 49, + "query_type": "rag_evaluation_query", + "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en):\nMonitor all your AI Agents\nimprove AI Agents today\nNoveum.ai helps you monitor, trace, and optimize your AI applications.\nNoveum.ai works with any AI framework \u2013 LangChain, CrewAI, AutoGen, custom implementations, or direct LLM calls. One dashboard shows everything.\n...", + "retrieval.context_length": 2740, + "retrieval.sources_count": 5, + "retrieval.context_quality": "high", + "retrieval.effectiveness": 1, + "retrieval.context_utilization": 2.74, + "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en):\nMonitor all your AI Agents\nimprove AI Agents today\nNoveum.ai helps you monitor, trace, and optimize your AI applications.\nNoveum.ai works with any AI framework \u2013 LangChain, CrewAI, AutoGen, custom implementations, or direct LLM calls. One dashboard shows everything.\nMonitor, Evaluate, Improve Your AI Agents\nThe control plane for AI agents.\nMonitor Everything, Miss Nothing\nOur lightweight SDKs capture every trace and span across your AI agent ecosystem\u2014from simple LLM calls to complex multi-agent...\n\nSource 2 (https://noveum.ai/):\nMonitor all your AI Agents\nimprove AI Agents today\nNoveum.ai helps you monitor, trace, and optimize your AI applications.\nNoveum.ai works with any AI framework \u2013 LangChain, CrewAI, AutoGen, custom implementations, or direct LLM calls. One dashboard shows everything.\nMonitor, Evaluate, Improve Your AI Agents\nThe control plane for AI agents.\nMonitor Everything, Miss Nothing\nOur lightweight SDKs capture every trace and span across your AI agent ecosystem\u2014from simple LLM calls to complex multi-agent...\n\nSource 3 (https://noveum.ai):\nMonitor all your AI Agents\nimprove AI Agents today\nNoveum.ai helps you monitor, trace, and optimize your AI applications.\nNoveum.ai works with any AI framework \u2013 LangChain, CrewAI, AutoGen, custom implementations, or direct LLM calls. One dashboard shows everything.\nMonitor, Evaluate, Improve Your AI Agents\nThe control plane for AI agents.\nMonitor Everything, Miss Nothing\nOur lightweight SDKs capture every trace and span across your AI agent ecosystem\u2014from simple LLM calls to complex multi-agent...\n\nSource 4 (https://noveum.ai/en/docs/getting-started/overview):\nNoveum.ai Overview\nComprehensive AI tracing and observability platform for LLM applications, RAG systems, and AI agents\nWelcome to Noveum.ai\u2014the comprehensive tracing and observability platform built specifically for AI applications. Whether you're building LLM-powered chatbots, RAG systems, multi-agent workflows, or any AI-driven application, Noveum provides the insights you need to understand, debug, and optimize your systems.\n\ud83c\udfaf Why AI Applications Need Specialized Observability\nTraditional mo...\n\nSource 5 (https://noveum.ai/docs/getting-started/overview):\nNoveum.ai Overview\nComprehensive AI tracing and observability platform for LLM applications, RAG systems, and AI agents\nWelcome to Noveum.ai\u2014the comprehensive tracing and observability platform built specifically for AI applications. Whether you're building LLM-powered chatbots, RAG systems, multi-agent workflows, or any AI-driven application, Noveum provides the insights you need to understand, debug, and optimize your systems.\n\ud83c\udfaf Why AI Applications Need Specialized Observability\nTraditional mo...\n\nUser Question: What monitoring capabilities does Noveum provide?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", + "prompt.prompt_length": 3285, + "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en):\nMonitor all your AI Agents\nimprove AI Agents today\nNoveum.ai helps you monitor, trace, and optimize your AI applications.\nNoveum.ai works with any AI framework \u2013 LangC...", + "prompt.instruction_following": "rag_optimized", + "response.answer_length": 582, + "response.answer_completeness": "complete", + "response.response_quality": "high", + "response.source_citation": 5, + "response.context_utilization": 0.2124087591240876, + "output_response": "RAG Answer: Noveum provides comprehensive monitoring capabilities for AI applications, including the ability to monitor, trace, and optimize AI agents across various frameworks such as LangChain, CrewAI, AutoGen,...", + "evaluation.retrieval_effectiveness": 1, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 5, + "evaluation.context_relevance": "high", + "evaluation.overall_quality": "high", + "evaluation.ready_for_production": true, + "rag.retrieval_strategy": "semantic_similarity", + "rag.vector_search_results": 5, + "rag.context_synthesis": "multi_source", + "rag.document_coverage": 1, + "rag.information_density": 0.2124087591240876 + }, + "events": [], + "links": [], + "trace_trace_id": "fb0d31fc-2465-4ecd-973c-4cd443d9befa", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:24.828000000", + "trace_end_time": "2025-10-22 16:57:31.793000000", + "trace_duration_ms": 6965.302000000001, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:24.828000000", + "updated_at": "2025-10-22 16:57:24.828000000" + }, + { + "span_id": "b0f0a2e0-c1f7-4ba4-8b75-7090e931a738", + "trace_id": "3bbd0b36-c30f-4126-b62a-dd0fae1c701d", + "parent_span_id": "4db80813-e62f-48ec-9091-62d569527caf", + "name": "agent.query_routing", + "start_time": "2025-10-22 16:57:19.339000000", + "end_time": "2025-10-22 16:57:23.822000000", + "duration_ms": 4482.909000000001, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "How does Noveum handle agent tracing?", + "query_length": 37, + "query_type": "routing_query", + "output_response": "Routed to RAG for query processing", + "router.classification": "RAG", + "router.confidence_score": 1, + "router.rag_keyword_score": 2, + "router.web_keyword_score": 0, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "3bbd0b36-c30f-4126-b62a-dd0fae1c701d", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:19.339000000", + "trace_end_time": "2025-10-22 16:57:23.822000000", + "trace_duration_ms": 4483.158, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:19.339000000", + "updated_at": "2025-10-22 16:57:19.339000000" + }, + { + "span_id": "24da2221-b0ab-4161-98ad-e7b5775fcb27", + "trace_id": "3bbd0b36-c30f-4126-b62a-dd0fae1c701d", + "parent_span_id": "b0f0a2e0-c1f7-4ba4-8b75-7090e931a738", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-22 16:57:19.339000000", + "end_time": "2025-10-22 16:57:19.339000000", + "duration_ms": 0.026, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: How does Noveum handle agent tracing?", + "query_length": 37, + "query_type": "routing_evaluation_query", + "classification.mode": "RAG", + "classification.rag_keyword_score": 2, + "classification.web_keyword_score": 0, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "noveum_specific", + "query.keyword_density": 0.3333333333333333, + "query.domain_affinity": "noveum", + "routing.decision": "Routed to RAG based on analysis", + "routing.rationale": "RAG score: 2, Web score: 0, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "Web Search", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.038461538461538464, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: RAG (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "3bbd0b36-c30f-4126-b62a-dd0fae1c701d", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:19.339000000", + "trace_end_time": "2025-10-22 16:57:23.822000000", + "trace_duration_ms": 4483.158, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:19.339000000", + "updated_at": "2025-10-22 16:57:19.339000000" + }, + { + "span_id": "d2b495eb-7e07-48a3-aa6d-1835aab18c11", + "trace_id": "3bbd0b36-c30f-4126-b62a-dd0fae1c701d", + "parent_span_id": "b0f0a2e0-c1f7-4ba4-8b75-7090e931a738", + "name": "agent.llm-rag", + "start_time": "2025-10-22 16:57:19.339000000", + "end_time": "2025-10-22 16:57:23.822000000", + "duration_ms": 4482.564, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "rag_agent", + "agent.operation": "llm-rag", + "agent.capabilities": [ + "document_retrieval", + "context_generation", + "response_generation" + ], + "agent.id": "noveum_rag_agent", + "input_query": "How does Noveum handle agent tracing?", + "query_length": 37, + "query_type": "rag_query", + "output_response": "RAG Answer: Noveum.ai handles agent tracing through its specialized Multi-Agent Tracing capabilities, which are designed to observe complex workflows and inter-agent communications within multi-agent systems. The...", + "rag.context_length": 2736, + "rag.sources_count": 5, + "rag.answer_length": 663, + "rag.mode": "retrieval_augmented_generation" + }, + "events": [], + "links": [], + "trace_trace_id": "3bbd0b36-c30f-4126-b62a-dd0fae1c701d", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:19.339000000", + "trace_end_time": "2025-10-22 16:57:23.822000000", + "trace_duration_ms": 4483.158, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:19.339000000", + "updated_at": "2025-10-22 16:57:19.339000000" + }, + { + "span_id": "4db80813-e62f-48ec-9091-62d569527caf", + "trace_id": "3bbd0b36-c30f-4126-b62a-dd0fae1c701d", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-22 16:57:19.339000000", + "end_time": "2025-10-22 16:57:23.822000000", + "duration_ms": 4483.031000000001, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "How does Noveum handle agent tracing?", + "query.length": 37, + "processing.mode": "RAG", + "processing.time_seconds": 4.482964038848877, + "processing.response_length": 663, + "processing.sources_count": 5, + "output_response": "Final Answer: Noveum.ai handles agent tracing through its specialized Multi-Agent Tracing capabilities, which are designed to observe complex workflows and inter-agent communications within multi-agent systems. The...", + "final_answer_mode": "RAG", + "query_processed.input_query": "How does Noveum handle agent tracing?", + "query_processed.output_response": "Successfully processed query using RAG, generated 663 character response", + "query_processed.mode": "RAG", + "query_processed.processing_time": 4.482964038848877, + "query_processed.response_length": 663 + }, + "events": [], + "links": [], + "trace_trace_id": "3bbd0b36-c30f-4126-b62a-dd0fae1c701d", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:19.339000000", + "trace_end_time": "2025-10-22 16:57:23.822000000", + "trace_duration_ms": 4483.158, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:19.339000000", + "updated_at": "2025-10-22 16:57:19.339000000" + }, + { + "span_id": "b80cb5cd-24f2-484d-bbff-9febe2aa3ffe", + "trace_id": "3bbd0b36-c30f-4126-b62a-dd0fae1c701d", + "parent_span_id": "d2b495eb-7e07-48a3-aa6d-1835aab18c11", + "name": "agent.llm_model_execution", + "start_time": "2025-10-22 16:57:19.958000000", + "end_time": "2025-10-22 16:57:22.709000000", + "duration_ms": 2750.9480000000003, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: How does Noveum handle agent tracing?", + "query_length": 37, + "query_type": "rag_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 2.750890016555786, + "model.latency_ms": 2750.890016555786, + "model.start_time": 1761152239.958973, + "model.end_time": 1761152242.709863, + "model.performance_tier": "medium", + "model.prompt_tokens": 817, + "model.completion_tokens": 165, + "model.total_tokens": 982, + "model.tokens_per_second": 356.975376728983, + "model.estimated_cost": 0.02946, + "model.efficiency_score": 0.675152749490835, + "model.response_length": 663, + "model.response_quality": "high", + "model.output_response": "Model Response: Noveum.ai handles agent tracing through its specialized Multi-Agent Tracing capabilities, which are designed to observe complex workflows and inter-agent communications within multi-agent systems. The...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "3bbd0b36-c30f-4126-b62a-dd0fae1c701d", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:19.339000000", + "trace_end_time": "2025-10-22 16:57:23.822000000", + "trace_duration_ms": 4483.158, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:19.339000000", + "updated_at": "2025-10-22 16:57:19.339000000" + }, + { + "span_id": "d85fcf7d-7eaa-4299-92fc-c969fab63a6e", + "trace_id": "3bbd0b36-c30f-4126-b62a-dd0fae1c701d", + "parent_span_id": "d2b495eb-7e07-48a3-aa6d-1835aab18c11", + "name": "agent.rag_evaluation_metrics", + "start_time": "2025-10-22 16:57:22.710000000", + "end_time": "2025-10-22 16:57:22.710000000", + "duration_ms": 0.036000000000000004, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "rag_evaluation_metrics", + "agent.capabilities": [ + "retrieval_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: How does Noveum handle agent tracing?", + "query_length": 37, + "query_type": "rag_evaluation_query", + "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, vector searches, data retrieval, agent reasoning, and more. Without proper observability, debugging ...", + "retrieval.context_length": 2736, + "retrieval.sources_count": 5, + "retrieval.context_quality": "high", + "retrieval.effectiveness": 1, + "retrieval.context_utilization": 2.736, + "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, vector searches, data retrieval, agent reasoning, and more. Without proper observability, debugging becomes a nightmare, optimization is guesswork, and understanding user interactions is nearly impossible.\nNoveum.ai solves this by providing comprehensive tracing and observability specifically designed for AI applications.\nKey benefits of using Noveum.ai:\n- Complete Visibility: Trace...\n\nSource 2 (https://noveum.ai/docs/advanced/multi-agent-tracing):\nMulti-Agent Tracing\nObserve complex agent workflows and inter-agent communications with comprehensive tracing\nMulti-agent systems represent some of the most complex AI applications, involving multiple agents that coordinate, communicate, and collaborate to achieve shared goals. Noveum.ai provides specialized tracing capabilities to help you understand and optimize these intricate workflows.\n\ud83c\udfaf Why Multi-Agent Tracing Matters\nMulti-agent systems introduce unique observability challenges:\n- Complex...\n\nSource 3 (https://noveum.ai/en/docs/advanced/multi-agent-tracing):\nMulti-Agent Tracing\nObserve complex agent workflows and inter-agent communications with comprehensive tracing\nMulti-agent systems represent some of the most complex AI applications, involving multiple agents that coordinate, communicate, and collaborate to achieve shared goals. Noveum.ai provides specialized tracing capabilities to help you understand and optimize these intricate workflows.\n\ud83c\udfaf Why Multi-Agent Tracing Matters\nMulti-agent systems introduce unique observability challenges:\n- Complex...\n\nSource 4 (https://noveum.ai/en/docs/advanced/performance-optimization):\n\ud83d\udd17 Next Steps\n- Multi-Agent Tracing - Observe agent workflows\n- RAG Pipeline Observability - Monitor retrieval and generation systems\n- Custom Instrumentation - Add domain-specific tracing\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nSource 5 (https://noveum.ai/docs/advanced/performance-optimization):\n\ud83d\udd17 Next Steps\n- Multi-Agent Tracing - Observe agent workflows\n- RAG Pipeline Observability - Monitor retrieval and generation systems\n- Custom Instrumentation - Add domain-specific tracing\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nUser Question: How does Noveum handle agent tracing?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", + "prompt.prompt_length": 3269, + "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, ...", + "prompt.instruction_following": "rag_optimized", + "response.answer_length": 663, + "response.answer_completeness": "complete", + "response.response_quality": "high", + "response.source_citation": 5, + "response.context_utilization": 0.24232456140350878, + "output_response": "RAG Answer: Noveum.ai handles agent tracing through its specialized Multi-Agent Tracing capabilities, which are designed to observe complex workflows and inter-agent communications within multi-agent systems. The...", + "evaluation.retrieval_effectiveness": 1, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 5, + "evaluation.context_relevance": "high", + "evaluation.overall_quality": "high", + "evaluation.ready_for_production": true, + "rag.retrieval_strategy": "semantic_similarity", + "rag.vector_search_results": 5, + "rag.context_synthesis": "multi_source", + "rag.document_coverage": 1, + "rag.information_density": 0.24232456140350878 + }, + "events": [], + "links": [], + "trace_trace_id": "3bbd0b36-c30f-4126-b62a-dd0fae1c701d", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:19.339000000", + "trace_end_time": "2025-10-22 16:57:23.822000000", + "trace_duration_ms": 4483.158, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:19.339000000", + "updated_at": "2025-10-22 16:57:19.339000000" + }, + { + "span_id": "93d82f68-6c62-40de-a675-c4d7c40c1d01", + "trace_id": "fee3e6d3-8da1-4ef3-8214-a7d7323d3a49", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-22 16:57:13.334000000", + "end_time": "2025-10-22 16:57:18.330000000", + "duration_ms": 4996.436, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What APIs are available in Noveum platform?", + "query.length": 43, + "processing.mode": "RAG", + "processing.time_seconds": 4.996389865875244, + "processing.response_length": 270, + "processing.sources_count": 5, + "output_response": "Final Answer: The provided context does not specify the APIs available in the Noveum platform. For detailed information about the APIs, I recommend checking the official Noveum documentation or the SDK Integration ...", + "final_answer_mode": "RAG", + "query_processed.input_query": "What APIs are available in Noveum platform?", + "query_processed.output_response": "Successfully processed query using RAG, generated 270 character response", + "query_processed.mode": "RAG", + "query_processed.processing_time": 4.996389865875244, + "query_processed.response_length": 270 + }, + "events": [], + "links": [], + "trace_trace_id": "fee3e6d3-8da1-4ef3-8214-a7d7323d3a49", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:13.334000000", + "trace_end_time": "2025-10-22 16:57:18.331000000", + "trace_duration_ms": 4996.502, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:13.334000000", + "updated_at": "2025-10-22 16:57:13.334000000" + }, + { + "span_id": "1be727c6-14e8-4b2e-b8df-a1cab902454f", + "trace_id": "fee3e6d3-8da1-4ef3-8214-a7d7323d3a49", + "parent_span_id": "93d82f68-6c62-40de-a675-c4d7c40c1d01", + "name": "agent.query_routing", + "start_time": "2025-10-22 16:57:13.334000000", + "end_time": "2025-10-22 16:57:18.330000000", + "duration_ms": 4996.354, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What APIs are available in Noveum platform?", + "query_length": 43, + "query_type": "routing_query", + "output_response": "Routed to RAG for query processing", + "router.classification": "RAG", + "router.confidence_score": 1, + "router.rag_keyword_score": 3, + "router.web_keyword_score": 0, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "fee3e6d3-8da1-4ef3-8214-a7d7323d3a49", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:13.334000000", + "trace_end_time": "2025-10-22 16:57:18.331000000", + "trace_duration_ms": 4996.502, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:13.334000000", + "updated_at": "2025-10-22 16:57:13.334000000" + }, + { + "span_id": "58ab1d39-d349-4164-8ff9-26ece4b1e1aa", + "trace_id": "fee3e6d3-8da1-4ef3-8214-a7d7323d3a49", + "parent_span_id": "1be727c6-14e8-4b2e-b8df-a1cab902454f", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-22 16:57:13.334000000", + "end_time": "2025-10-22 16:57:13.334000000", + "duration_ms": 0.041999999999999996, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What APIs are available in Noveum platform?", + "query_length": 43, + "query_type": "routing_evaluation_query", + "classification.mode": "RAG", + "classification.rag_keyword_score": 3, + "classification.web_keyword_score": 0, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "noveum_specific", + "query.keyword_density": 0.42857142857142855, + "query.domain_affinity": "noveum", + "routing.decision": "Routed to RAG based on analysis", + "routing.rationale": "RAG score: 3, Web score: 0, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "Web Search", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.057692307692307696, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: RAG (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "fee3e6d3-8da1-4ef3-8214-a7d7323d3a49", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:13.334000000", + "trace_end_time": "2025-10-22 16:57:18.331000000", + "trace_duration_ms": 4996.502, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:13.334000000", + "updated_at": "2025-10-22 16:57:13.334000000" + }, + { + "span_id": "f11ebf68-a416-484e-8e55-f77a11b9ea1c", + "trace_id": "fee3e6d3-8da1-4ef3-8214-a7d7323d3a49", + "parent_span_id": "1be727c6-14e8-4b2e-b8df-a1cab902454f", + "name": "agent.llm-rag", + "start_time": "2025-10-22 16:57:13.334000000", + "end_time": "2025-10-22 16:57:18.330000000", + "duration_ms": 4996.146000000001, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "rag_agent", + "agent.operation": "llm-rag", + "agent.capabilities": [ + "document_retrieval", + "context_generation", + "response_generation" + ], + "agent.id": "noveum_rag_agent", + "input_query": "What APIs are available in Noveum platform?", + "query_length": 43, + "query_type": "rag_query", + "output_response": "RAG Answer: The provided context does not specify the APIs available in the Noveum platform. For detailed information about the APIs, I recommend checking the official Noveum documentation or the SDK Integration ...", + "rag.context_length": 2284, + "rag.sources_count": 5, + "rag.answer_length": 270, + "rag.mode": "retrieval_augmented_generation" + }, + "events": [], + "links": [], + "trace_trace_id": "fee3e6d3-8da1-4ef3-8214-a7d7323d3a49", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:13.334000000", + "trace_end_time": "2025-10-22 16:57:18.331000000", + "trace_duration_ms": 4996.502, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:13.334000000", + "updated_at": "2025-10-22 16:57:13.334000000" + }, + { + "span_id": "11394488-8802-403d-a926-7fda3a510b61", + "trace_id": "fee3e6d3-8da1-4ef3-8214-a7d7323d3a49", + "parent_span_id": "f11ebf68-a416-484e-8e55-f77a11b9ea1c", + "name": "agent.llm_model_execution", + "start_time": "2025-10-22 16:57:13.862000000", + "end_time": "2025-10-22 16:57:16.049000000", + "duration_ms": 2187.29, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What APIs are available in Noveum platform?", + "query_length": 43, + "query_type": "rag_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 2.187239170074463, + "model.latency_ms": 2187.239170074463, + "model.start_time": 1761152233.862141, + "model.end_time": 1761152236.04938, + "model.performance_tier": "medium", + "model.prompt_tokens": 705, + "model.completion_tokens": 67, + "model.total_tokens": 772, + "model.tokens_per_second": 352.9563710098141, + "model.estimated_cost": 0.02316, + "model.efficiency_score": 0.34974093264248707, + "model.response_length": 270, + "model.response_quality": "high", + "model.output_response": "Model Response: The provided context does not specify the APIs available in the Noveum platform. For detailed information about the APIs, I recommend checking the official Noveum documentation or the SDK Integration ...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "fee3e6d3-8da1-4ef3-8214-a7d7323d3a49", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:13.334000000", + "trace_end_time": "2025-10-22 16:57:18.331000000", + "trace_duration_ms": 4996.502, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:13.334000000", + "updated_at": "2025-10-22 16:57:13.334000000" + }, + { + "span_id": "341cf8ab-cc33-4d57-952f-ba1065f0432e", + "trace_id": "fee3e6d3-8da1-4ef3-8214-a7d7323d3a49", + "parent_span_id": "f11ebf68-a416-484e-8e55-f77a11b9ea1c", + "name": "agent.rag_evaluation_metrics", + "start_time": "2025-10-22 16:57:16.049000000", + "end_time": "2025-10-22 16:57:16.049000000", + "duration_ms": 0.041, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "rag_evaluation_metrics", + "agent.capabilities": [ + "retrieval_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What APIs are available in Noveum platform?", + "query_length": 43, + "query_type": "rag_evaluation_query", + "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/docs):\n- \ud83c\udf0d Universal: Works across frameworks, clouds, and deployment models\n\ud83e\udd1d Community & Support\n- \ud83d\udcac Discord Community: Join our Discord\n- \ud83d\udce7 Email Support: [email protected]\n- \ud83d\udc1b Bug Reports: GitHub Issues\n- \ud83d\udcd6 Knowledge Base: Help Center\nReady to get started? Head to ...", + "retrieval.context_length": 2284, + "retrieval.sources_count": 5, + "retrieval.context_quality": "high", + "retrieval.effectiveness": 1, + "retrieval.context_utilization": 2.284, + "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/docs):\n- \ud83c\udf0d Universal: Works across frameworks, clouds, and deployment models\n\ud83e\udd1d Community & Support\n- \ud83d\udcac Discord Community: Join our Discord\n- \ud83d\udce7 Email Support: [email protected]\n- \ud83d\udc1b Bug Reports: GitHub Issues\n- \ud83d\udcd6 Knowledge Base: Help Center\nReady to get started? Head to our SDK Integration Guide to begin tracing your AI applications in under 5 minutes!\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability ...\n\nSource 2 (https://noveum.ai/docs):\n- \ud83c\udf0d Universal: Works across frameworks, clouds, and deployment models\n\ud83e\udd1d Community & Support\n- \ud83d\udcac Discord Community: Join our Discord\n- \ud83d\udce7 Email Support: [email protected]\n- \ud83d\udc1b Bug Reports: GitHub Issues\n- \ud83d\udcd6 Knowledge Base: Help Center\nReady to get started? Head to our SDK Integration Guide to begin tracing your AI applications in under 5 minutes!\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability ...\n\nSource 3 (https://noveum.ai/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nSource 4 (https://noveum.ai/en/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nSource 5 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, vector searches, data retrieval, agent reasoning, and more. Without proper observability, debugging becomes a nightmare, optimization is guesswork, and understanding user interactions is nearly impossible.\nNoveum.ai solves this by providing comprehensive tracing and observability specifically designed for AI applications.\nKey benefits of using Noveum.ai:\n- Complete Visibility: Trace...\n\nUser Question: What APIs are available in Noveum platform?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", + "prompt.prompt_length": 2823, + "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/docs):\n- \ud83c\udf0d Universal: Works across frameworks, clouds, and deployment models\n\ud83e\udd1d Community & Support\n- \ud83d\udcac Discord Community: Join our Discord\n- \ud83d\udce7 Email Support: [email prot...", + "prompt.instruction_following": "rag_optimized", + "response.answer_length": 270, + "response.answer_completeness": "complete", + "response.response_quality": "high", + "response.source_citation": 5, + "response.context_utilization": 0.11821366024518389, + "output_response": "RAG Answer: The provided context does not specify the APIs available in the Noveum platform. For detailed information about the APIs, I recommend checking the official Noveum documentation or the SDK Integration ...", + "evaluation.retrieval_effectiveness": 1, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 5, + "evaluation.context_relevance": "high", + "evaluation.overall_quality": "high", + "evaluation.ready_for_production": true, + "rag.retrieval_strategy": "semantic_similarity", + "rag.vector_search_results": 5, + "rag.context_synthesis": "multi_source", + "rag.document_coverage": 1, + "rag.information_density": 0.11821366024518389 + }, + "events": [], + "links": [], + "trace_trace_id": "fee3e6d3-8da1-4ef3-8214-a7d7323d3a49", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:13.334000000", + "trace_end_time": "2025-10-22 16:57:18.331000000", + "trace_duration_ms": 4996.502, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:13.334000000", + "updated_at": "2025-10-22 16:57:13.334000000" + }, + { + "span_id": "24930c28-7f93-426c-8914-bd0e0085589f", + "trace_id": "93bd37c4-4289-43c9-b34e-19eaaef610c2", + "parent_span_id": "bfc91518-f818-4f35-9113-70b04eb55edc", + "name": "agent.query_routing", + "start_time": "2025-10-22 16:57:06.605000000", + "end_time": "2025-10-22 16:57:12.327000000", + "duration_ms": 5721.561000000001, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "How do I set up observability with Noveum?", + "query_length": 42, + "query_type": "routing_query", + "output_response": "Routed to RAG for query processing", + "router.classification": "RAG", + "router.confidence_score": 1, + "router.rag_keyword_score": 2, + "router.web_keyword_score": 0, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "93bd37c4-4289-43c9-b34e-19eaaef610c2", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:06.605000000", + "trace_end_time": "2025-10-22 16:57:12.327000000", + "trace_duration_ms": 5721.707, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:06.605000000", + "updated_at": "2025-10-22 16:57:06.605000000" + }, + { + "span_id": "11fc18a8-0380-48ca-894e-cc26f648d290", + "trace_id": "93bd37c4-4289-43c9-b34e-19eaaef610c2", + "parent_span_id": "24930c28-7f93-426c-8914-bd0e0085589f", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-22 16:57:06.605000000", + "end_time": "2025-10-22 16:57:06.606000000", + "duration_ms": 0.047, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: How do I set up observability with Noveum?", + "query_length": 42, + "query_type": "routing_evaluation_query", + "classification.mode": "RAG", + "classification.rag_keyword_score": 2, + "classification.web_keyword_score": 0, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "noveum_specific", + "query.keyword_density": 0.25, + "query.domain_affinity": "noveum", + "routing.decision": "Routed to RAG based on analysis", + "routing.rationale": "RAG score: 2, Web score: 0, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "Web Search", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.038461538461538464, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: RAG (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "93bd37c4-4289-43c9-b34e-19eaaef610c2", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:06.605000000", + "trace_end_time": "2025-10-22 16:57:12.327000000", + "trace_duration_ms": 5721.707, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:06.605000000", + "updated_at": "2025-10-22 16:57:06.605000000" + }, + { + "span_id": "bfc91518-f818-4f35-9113-70b04eb55edc", + "trace_id": "93bd37c4-4289-43c9-b34e-19eaaef610c2", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-22 16:57:06.605000000", + "end_time": "2025-10-22 16:57:12.327000000", + "duration_ms": 5721.635, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "How do I set up observability with Noveum?", + "query.length": 42, + "processing.mode": "RAG", + "processing.time_seconds": 5.721601963043213, + "processing.response_length": 815, + "processing.sources_count": 5, + "output_response": "Final Answer: To set up observability with Noveum, follow these steps:\n\n1. **Sign Up**: Go to [noveum.ai](https://noveum.ai) and create an account.\n2. **Create a Project**: After signing up, create a project to obt...", + "final_answer_mode": "RAG", + "query_processed.input_query": "How do I set up observability with Noveum?", + "query_processed.output_response": "Successfully processed query using RAG, generated 815 character response", + "query_processed.mode": "RAG", + "query_processed.processing_time": 5.721601963043213, + "query_processed.response_length": 815 + }, + "events": [], + "links": [], + "trace_trace_id": "93bd37c4-4289-43c9-b34e-19eaaef610c2", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:06.605000000", + "trace_end_time": "2025-10-22 16:57:12.327000000", + "trace_duration_ms": 5721.707, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:06.605000000", + "updated_at": "2025-10-22 16:57:06.605000000" + }, + { + "span_id": "2a82e4b1-c93d-498f-9dd5-e971f228d642", + "trace_id": "93bd37c4-4289-43c9-b34e-19eaaef610c2", + "parent_span_id": "24930c28-7f93-426c-8914-bd0e0085589f", + "name": "agent.llm-rag", + "start_time": "2025-10-22 16:57:06.606000000", + "end_time": "2025-10-22 16:57:12.327000000", + "duration_ms": 5721.352000000001, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "rag_agent", + "agent.operation": "llm-rag", + "agent.capabilities": [ + "document_retrieval", + "context_generation", + "response_generation" + ], + "agent.id": "noveum_rag_agent", + "input_query": "How do I set up observability with Noveum?", + "query_length": 42, + "query_type": "rag_query", + "output_response": "RAG Answer: To set up observability with Noveum, follow these steps:\n\n1. **Sign Up**: Go to [noveum.ai](https://noveum.ai) and create an account.\n2. **Create a Project**: After signing up, create a project to obt...", + "rag.context_length": 2870, + "rag.sources_count": 6, + "rag.answer_length": 815, + "rag.mode": "retrieval_augmented_generation" + }, + "events": [], + "links": [], + "trace_trace_id": "93bd37c4-4289-43c9-b34e-19eaaef610c2", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:06.605000000", + "trace_end_time": "2025-10-22 16:57:12.327000000", + "trace_duration_ms": 5721.707, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:06.605000000", + "updated_at": "2025-10-22 16:57:06.605000000" + }, + { + "span_id": "520b4e59-da14-4db2-af02-0973cda9bd86", + "trace_id": "93bd37c4-4289-43c9-b34e-19eaaef610c2", + "parent_span_id": "2a82e4b1-c93d-498f-9dd5-e971f228d642", + "name": "agent.llm_model_execution", + "start_time": "2025-10-22 16:57:07.109000000", + "end_time": "2025-10-22 16:57:11.837000000", + "duration_ms": 4728.018999999999, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: How do I set up observability with Noveum?", + "query_length": 42, + "query_type": "rag_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 4.727963924407959, + "model.latency_ms": 4727.963924407959, + "model.start_time": 1761152227.109188, + "model.end_time": 1761152231.837152, + "model.performance_tier": "medium", + "model.prompt_tokens": 852, + "model.completion_tokens": 203, + "model.total_tokens": 1055, + "model.tokens_per_second": 223.1404504915101, + "model.estimated_cost": 0.03165, + "model.efficiency_score": 0.7725118483412322, + "model.response_length": 815, + "model.response_quality": "high", + "model.output_response": "Model Response: To set up observability with Noveum, follow these steps:\n\n1. **Sign Up**: Go to [noveum.ai](https://noveum.ai) and create an account.\n2. **Create a Project**: After signing up, create a project to obt...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "93bd37c4-4289-43c9-b34e-19eaaef610c2", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:06.605000000", + "trace_end_time": "2025-10-22 16:57:12.327000000", + "trace_duration_ms": 5721.707, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:06.605000000", + "updated_at": "2025-10-22 16:57:06.605000000" + }, + { + "span_id": "dff080ab-238c-4cca-a910-cf530f910bda", + "trace_id": "93bd37c4-4289-43c9-b34e-19eaaef610c2", + "parent_span_id": "2a82e4b1-c93d-498f-9dd5-e971f228d642", + "name": "agent.rag_evaluation_metrics", + "start_time": "2025-10-22 16:57:11.837000000", + "end_time": "2025-10-22 16:57:11.838000000", + "duration_ms": 0.259, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "rag_evaluation_metrics", + "agent.capabilities": [ + "retrieval_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: How do I set up observability with Noveum?", + "query_length": 42, + "query_type": "rag_evaluation_query", + "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\n- Production Ready: Built for scale with intelligent sampling and batching\n- Privacy Focused: You control what data is captured and how it's stored\n- Open Source SDKs: Transparent, auditable, and extensible\nGetting ...", + "retrieval.context_length": 2870, + "retrieval.sources_count": 6, + "retrieval.context_quality": "high", + "retrieval.effectiveness": 1.2, + "retrieval.context_utilization": 2.87, + "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\n- Production Ready: Built for scale with intelligent sampling and batching\n- Privacy Focused: You control what data is captured and how it's stored\n- Open Source SDKs: Transparent, auditable, and extensible\nGetting Started\nReady to add comprehensive observability to your AI applications?\n- Sign up at noveum.ai\n- Create a project and get your API key\n- Install the SDK for your preferred language\n- Add tracing to your AI workflows\n- Explore insights in the Noveum.ai dashboard\nCheck out our integra...\n\nSource 2 (https://noveum.ai/en/docs/getting-started/tracing-concepts):\n- Explore Framework Integrations - Framework-specific guidance\n- Learn Advanced Patterns - Custom instrumentation techniques\n- Master the Dashboard - Analyze your traces effectively\nRemember: Good observability is not about collecting all possible data, but about collecting the right data that helps you understand, debug, and optimize your AI applications.\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to O...\n\nSource 3 (https://noveum.ai/docs/getting-started/tracing-concepts):\n- Explore Framework Integrations - Framework-specific guidance\n- Learn Advanced Patterns - Custom instrumentation techniques\n- Master the Dashboard - Analyze your traces effectively\nRemember: Good observability is not about collecting all possible data, but about collecting the right data that helps you understand, debug, and optimize your AI applications.\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to O...\n\nSource 4 (https://noveum.ai/en/docs/getting-started/overview):\nNoveum.ai Overview\nComprehensive AI tracing and observability platform for LLM applications, RAG systems, and AI agents\nWelcome to Noveum.ai\u2014the comprehensive tracing and observability platform built specifically for AI applications. Whether you're building LLM-powered chatbots, RAG systems, multi-agent workflows, or any AI-driven application, Noveum provides the insights you need to understand, debug, and optimize your systems.\n\ud83c\udfaf Why AI Applications Need Specialized Observability\nTraditional mo...\n\nSource 5 (https://noveum.ai/docs/getting-started/overview):\nNoveum.ai Overview\nComprehensive AI tracing and observability platform for LLM applications, RAG systems, and AI agents\nWelcome to Noveum.ai\u2014the comprehensive tracing and observability platform built specifically for AI applications. Whether you're building LLM-powered chatbots, RAG systems, multi-agent workflows, or any AI-driven application, Noveum provides the insights you need to understand, debug, and optimize your systems.\n\ud83c\udfaf Why AI Applications Need Specialized Observability\nTraditional mo...\n\nUser Question: How do I set up observability with Noveum?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", + "prompt.prompt_length": 3408, + "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\n- Production Ready: Built for scale with intelligent sampling and batching\n- Privacy Focused: You control what data...", + "prompt.instruction_following": "rag_optimized", + "response.answer_length": 815, + "response.answer_completeness": "complete", + "response.response_quality": "high", + "response.source_citation": 6, + "response.context_utilization": 0.28397212543554007, + "output_response": "RAG Answer: To set up observability with Noveum, follow these steps:\n\n1. **Sign Up**: Go to [noveum.ai](https://noveum.ai) and create an account.\n2. **Create a Project**: After signing up, create a project to obt...", + "evaluation.retrieval_effectiveness": 1.2, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 6, + "evaluation.context_relevance": "high", + "evaluation.overall_quality": "high", + "evaluation.ready_for_production": true, + "rag.retrieval_strategy": "semantic_similarity", + "rag.vector_search_results": 6, + "rag.context_synthesis": "multi_source", + "rag.document_coverage": 1.2, + "rag.information_density": 0.28397212543554007 + }, + "events": [], + "links": [], + "trace_trace_id": "93bd37c4-4289-43c9-b34e-19eaaef610c2", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:06.605000000", + "trace_end_time": "2025-10-22 16:57:12.327000000", + "trace_duration_ms": 5721.707, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:06.605000000", + "updated_at": "2025-10-22 16:57:06.605000000" + }, + { + "span_id": "dad7df65-be49-4245-8721-c91da4962b9a", + "trace_id": "df0c6344-1a2e-4c53-bb7d-4e43b3bce1b8", + "parent_span_id": "9024b0d9-2b75-428f-af8f-75a921a7c69c", + "name": "agent.query_routing", + "start_time": "2025-10-22 16:56:59.213000000", + "end_time": "2025-10-22 16:57:05.598000000", + "duration_ms": 6384.249, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What features does Noveum Trace offer?", + "query_length": 38, + "query_type": "routing_query", + "output_response": "Routed to RAG for query processing", + "router.classification": "RAG", + "router.confidence_score": 1, + "router.rag_keyword_score": 3, + "router.web_keyword_score": 0, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "df0c6344-1a2e-4c53-bb7d-4e43b3bce1b8", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:56:59.213000000", + "trace_end_time": "2025-10-22 16:57:05.598000000", + "trace_duration_ms": 6384.386, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:56:59.213000000", + "updated_at": "2025-10-22 16:56:59.213000000" + }, + { + "span_id": "9024b0d9-2b75-428f-af8f-75a921a7c69c", + "trace_id": "df0c6344-1a2e-4c53-bb7d-4e43b3bce1b8", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-22 16:56:59.213000000", + "end_time": "2025-10-22 16:57:05.598000000", + "duration_ms": 6384.32, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What features does Noveum Trace offer?", + "query.length": 38, + "processing.mode": "RAG", + "processing.time_seconds": 6.384284973144531, + "processing.response_length": 1017, + "processing.sources_count": 5, + "output_response": "Final Answer: Noveum Trace offers several key features through its Python SDK (`noveum-trace`) and TypeScript SDK (`@noveum/trace`):\n\n1. **Decorator-based Tracing**: It provides a simple way to trace LLM calls, age...", + "final_answer_mode": "RAG", + "query_processed.input_query": "What features does Noveum Trace offer?", + "query_processed.output_response": "Successfully processed query using RAG, generated 1017 character response", + "query_processed.mode": "RAG", + "query_processed.processing_time": 6.384284973144531, + "query_processed.response_length": 1017 + }, + "events": [], + "links": [], + "trace_trace_id": "df0c6344-1a2e-4c53-bb7d-4e43b3bce1b8", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:56:59.213000000", + "trace_end_time": "2025-10-22 16:57:05.598000000", + "trace_duration_ms": 6384.386, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:56:59.213000000", + "updated_at": "2025-10-22 16:56:59.213000000" + }, + { + "span_id": "d07fdbdc-2b8f-4de9-ac32-988389701771", + "trace_id": "df0c6344-1a2e-4c53-bb7d-4e43b3bce1b8", + "parent_span_id": "dad7df65-be49-4245-8721-c91da4962b9a", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-22 16:56:59.214000000", + "end_time": "2025-10-22 16:56:59.214000000", + "duration_ms": 0.060000000000000005, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What features does Noveum Trace offer?", + "query_length": 38, + "query_type": "routing_evaluation_query", + "classification.mode": "RAG", + "classification.rag_keyword_score": 3, + "classification.web_keyword_score": 0, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "noveum_specific", + "query.keyword_density": 0.5, + "query.domain_affinity": "noveum", + "routing.decision": "Routed to RAG based on analysis", + "routing.rationale": "RAG score: 3, Web score: 0, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "Web Search", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.057692307692307696, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: RAG (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "df0c6344-1a2e-4c53-bb7d-4e43b3bce1b8", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:56:59.213000000", + "trace_end_time": "2025-10-22 16:57:05.598000000", + "trace_duration_ms": 6384.386, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:56:59.213000000", + "updated_at": "2025-10-22 16:56:59.213000000" + }, + { + "span_id": "83f86157-5af4-4c99-aad1-647866cd804e", + "trace_id": "df0c6344-1a2e-4c53-bb7d-4e43b3bce1b8", + "parent_span_id": "dad7df65-be49-4245-8721-c91da4962b9a", + "name": "agent.llm-rag", + "start_time": "2025-10-22 16:56:59.214000000", + "end_time": "2025-10-22 16:57:05.598000000", + "duration_ms": 6384.048, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "rag_agent", + "agent.operation": "llm-rag", + "agent.capabilities": [ + "document_retrieval", + "context_generation", + "response_generation" + ], + "agent.id": "noveum_rag_agent", + "input_query": "What features does Noveum Trace offer?", + "query_length": 38, + "query_type": "rag_query", + "output_response": "RAG Answer: Noveum Trace offers several key features through its Python SDK (`noveum-trace`) and TypeScript SDK (`@noveum/trace`):\n\n1. **Decorator-based Tracing**: It provides a simple way to trace LLM calls, age...", + "rag.context_length": 2787, + "rag.sources_count": 5, + "rag.answer_length": 1017, + "rag.mode": "retrieval_augmented_generation" + }, + "events": [], + "links": [], + "trace_trace_id": "df0c6344-1a2e-4c53-bb7d-4e43b3bce1b8", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:56:59.213000000", + "trace_end_time": "2025-10-22 16:57:05.598000000", + "trace_duration_ms": 6384.386, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:56:59.213000000", + "updated_at": "2025-10-22 16:56:59.213000000" + }, + { + "span_id": "3b4fa9a7-88bf-4ee1-aeb7-bfa9573f28f2", + "trace_id": "df0c6344-1a2e-4c53-bb7d-4e43b3bce1b8", + "parent_span_id": "83f86157-5af4-4c99-aad1-647866cd804e", + "name": "agent.llm_model_execution", + "start_time": "2025-10-22 16:56:59.591000000", + "end_time": "2025-10-22 16:57:04.869000000", + "duration_ms": 5278.925, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What features does Noveum Trace offer?", + "query_length": 38, + "query_type": "rag_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 5.278868913650513, + "model.latency_ms": 5278.868913650513, + "model.start_time": 1761152219.5910342, + "model.end_time": 1761152224.869903, + "model.performance_tier": "slow", + "model.prompt_tokens": 830, + "model.completion_tokens": 254, + "model.total_tokens": 1084, + "model.tokens_per_second": 205.3470199263535, + "model.estimated_cost": 0.03252, + "model.efficiency_score": 0.9381918819188192, + "model.response_length": 1017, + "model.response_quality": "high", + "model.output_response": "Model Response: Noveum Trace offers several key features through its Python SDK (`noveum-trace`) and TypeScript SDK (`@noveum/trace`):\n\n1. **Decorator-based Tracing**: It provides a simple way to trace LLM calls, age...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "df0c6344-1a2e-4c53-bb7d-4e43b3bce1b8", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:56:59.213000000", + "trace_end_time": "2025-10-22 16:57:05.598000000", + "trace_duration_ms": 6384.386, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:56:59.213000000", + "updated_at": "2025-10-22 16:56:59.213000000" + }, + { + "span_id": "f9214dbe-adb9-44ff-8361-70c872cef66d", + "trace_id": "df0c6344-1a2e-4c53-bb7d-4e43b3bce1b8", + "parent_span_id": "83f86157-5af4-4c99-aad1-647866cd804e", + "name": "agent.rag_evaluation_metrics", + "start_time": "2025-10-22 16:57:04.870000000", + "end_time": "2025-10-22 16:57:04.870000000", + "duration_ms": 0.043000000000000003, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "rag_evaluation_metrics", + "agent.capabilities": [ + "retrieval_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What features does Noveum Trace offer?", + "query_length": 38, + "query_type": "rag_evaluation_query", + "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\n\ud83d\udcd8 TypeScript SDK (\n@noveum/trace\n)- Framework ...", + "retrieval.context_length": 2787, + "retrieval.sources_count": 5, + "retrieval.context_quality": "high", + "retrieval.effectiveness": 1, + "retrieval.context_utilization": 2.787, + "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\n\ud83d\udcd8 TypeScript SDK (\n@noveum/trace\n)- Framework integrations for Next.js, Express.js, and Hono\n- TypeScript-first design with full type safety\n- Universal compatibility (Node.js, Edge Runtime, browsers)\n-\n\ud83d\udcca Noveum Platform\n- Real-time dashboard for analyzing traces and performance\n- Ad...\n\nSource 2 (https://noveum.ai/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\n\ud83d\udcd8 TypeScript SDK (\n@noveum/trace\n)- Framework integrations for Next.js, Express.js, and Hono\n- TypeScript-first design with full type safety\n- Universal compatibility (Node.js, Edge Runtime, browsers)\n-\n\ud83d\udcca Noveum Platform\n- Real-time dashboard for analyzing traces and performance\n- Ad...\n\nSource 3 (https://noveum.ai/en/docs/getting-started/sdk-integration):\n- \ud83d\udc65 Collaborate with your team on insights\n\ud83d\udd12 Security & Privacy\n- \ud83d\udd10 Encryption: All data encrypted in transit and at rest\n- \ud83c\udf9b\ufe0f Configurable Capture: Control what data is collected\n- \ud83c\udfe0 Data Residency: Choose your data storage region\n- \u23f0 Retention Control: Set custom data retention policies\nNext Steps\n- Tracing Concepts - Learn about traces, spans, and observability best practices\n- Framework Integrations - Deep dive into specific framework setups\n- Multi-Agent Tracing - Observe complex agent work...\n\nSource 4 (https://noveum.ai/docs/getting-started/sdk-integration):\n- \ud83d\udc65 Collaborate with your team on insights\n\ud83d\udd12 Security & Privacy\n- \ud83d\udd10 Encryption: All data encrypted in transit and at rest\n- \ud83c\udf9b\ufe0f Configurable Capture: Control what data is collected\n- \ud83c\udfe0 Data Residency: Choose your data storage region\n- \u23f0 Retention Control: Set custom data retention policies\nNext Steps\n- Tracing Concepts - Learn about traces, spans, and observability best practices\n- Framework Integrations - Deep dive into specific framework setups\n- Multi-Agent Tracing - Observe complex agent work...\n\nSource 5 (https://noveum.ai/docs/platform/dashboard):\n- Bulk Export: Download multiple traces simultaneously\n- Comparative Analysis: Compare performance across multiple traces\nReal-Time Updates\n- Live Refresh: Automatic updates as new traces arrive\n- Connection Monitoring: Real-time status of your trace ingestion pipeline\n- Performance Indicators: Live metrics for system health monitoring\nReady to dive deeper? Explore Projects & Environments to organize your AI applications, or check out Team Collaboration to share insights with your team.\nGet Earl...\n\nUser Question: What features does Noveum Trace offer?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", + "prompt.prompt_length": 3321, + "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frame...", + "prompt.instruction_following": "rag_optimized", + "response.answer_length": 1017, + "response.answer_completeness": "complete", + "response.response_quality": "high", + "response.source_citation": 5, + "response.context_utilization": 0.36490850376749195, + "output_response": "RAG Answer: Noveum Trace offers several key features through its Python SDK (`noveum-trace`) and TypeScript SDK (`@noveum/trace`):\n\n1. **Decorator-based Tracing**: It provides a simple way to trace LLM calls, age...", + "evaluation.retrieval_effectiveness": 1, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 5, + "evaluation.context_relevance": "high", + "evaluation.overall_quality": "high", + "evaluation.ready_for_production": true, + "rag.retrieval_strategy": "semantic_similarity", + "rag.vector_search_results": 5, + "rag.context_synthesis": "multi_source", + "rag.document_coverage": 1, + "rag.information_density": 0.36490850376749195 + }, + "events": [], + "links": [], + "trace_trace_id": "df0c6344-1a2e-4c53-bb7d-4e43b3bce1b8", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:56:59.213000000", + "trace_end_time": "2025-10-22 16:57:05.598000000", + "trace_duration_ms": 6384.386, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:56:59.213000000", + "updated_at": "2025-10-22 16:56:59.213000000" + }, + { + "span_id": "138a5089-5268-447a-afb5-4f14fb504281", + "trace_id": "29a862bf-d7aa-4cf9-963c-0a4244343bbe", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-22 16:56:54.719000000", + "end_time": "2025-10-22 16:56:58.207000000", + "duration_ms": 3487.959, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What are Noveum's pricing plans?", + "query.length": 32, + "processing.mode": "RAG", + "processing.time_seconds": 3.487928867340088, + "processing.response_length": 441, + "processing.sources_count": 5, + "output_response": "Final Answer: The provided context does not include specific information about Noveum's pricing plans. It mentions that all users will have access to the Observability suite for free, and early users will receive f...", + "final_answer_mode": "RAG", + "query_processed.input_query": "What are Noveum's pricing plans?", + "query_processed.output_response": "Successfully processed query using RAG, generated 441 character response", + "query_processed.mode": "RAG", + "query_processed.processing_time": 3.487928867340088, + "query_processed.response_length": 441 + }, + "events": [], + "links": [], + "trace_trace_id": "29a862bf-d7aa-4cf9-963c-0a4244343bbe", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:56:54.718000000", + "trace_end_time": "2025-10-22 16:56:58.207000000", + "trace_duration_ms": 3488.068, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:56:54.718000000", + "updated_at": "2025-10-22 16:56:54.718000000" + }, + { + "span_id": "7cda4dab-0fc5-4cae-8d8c-9445f661c67b", + "trace_id": "29a862bf-d7aa-4cf9-963c-0a4244343bbe", + "parent_span_id": "138a5089-5268-447a-afb5-4f14fb504281", + "name": "agent.query_routing", + "start_time": "2025-10-22 16:56:54.719000000", + "end_time": "2025-10-22 16:56:58.207000000", + "duration_ms": 3487.848, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What are Noveum's pricing plans?", + "query_length": 32, + "query_type": "routing_query", + "output_response": "Routed to RAG for query processing", + "router.classification": "RAG", + "router.confidence_score": 1, + "router.rag_keyword_score": 3, + "router.web_keyword_score": 0, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "29a862bf-d7aa-4cf9-963c-0a4244343bbe", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:56:54.718000000", + "trace_end_time": "2025-10-22 16:56:58.207000000", + "trace_duration_ms": 3488.068, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:56:54.718000000", + "updated_at": "2025-10-22 16:56:54.718000000" + }, + { + "span_id": "922c84a5-b14a-4b2f-8759-427d6e089812", + "trace_id": "29a862bf-d7aa-4cf9-963c-0a4244343bbe", + "parent_span_id": "7cda4dab-0fc5-4cae-8d8c-9445f661c67b", + "name": "agent.llm-rag", + "start_time": "2025-10-22 16:56:54.719000000", + "end_time": "2025-10-22 16:56:58.207000000", + "duration_ms": 3487.546, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "rag_agent", + "agent.operation": "llm-rag", + "agent.capabilities": [ + "document_retrieval", + "context_generation", + "response_generation" + ], + "agent.id": "noveum_rag_agent", + "input_query": "What are Noveum's pricing plans?", + "query_length": 32, + "query_type": "rag_query", + "output_response": "RAG Answer: The provided context does not include specific information about Noveum's pricing plans. It mentions that all users will have access to the Observability suite for free, and early users will receive f...", + "rag.context_length": 2105, + "rag.sources_count": 5, + "rag.answer_length": 441, + "rag.mode": "retrieval_augmented_generation" + }, + "events": [], + "links": [], + "trace_trace_id": "29a862bf-d7aa-4cf9-963c-0a4244343bbe", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:56:54.718000000", + "trace_end_time": "2025-10-22 16:56:58.207000000", + "trace_duration_ms": 3488.068, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:56:54.718000000", + "updated_at": "2025-10-22 16:56:54.718000000" + }, + { + "span_id": "c8d7bb28-e763-47c9-b3e9-77f6a91a22be", + "trace_id": "29a862bf-d7aa-4cf9-963c-0a4244343bbe", + "parent_span_id": "7cda4dab-0fc5-4cae-8d8c-9445f661c67b", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-22 16:56:54.719000000", + "end_time": "2025-10-22 16:56:54.719000000", + "duration_ms": 0.068, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What are Noveum's pricing plans?", + "query_length": 32, + "query_type": "routing_evaluation_query", + "classification.mode": "RAG", + "classification.rag_keyword_score": 3, + "classification.web_keyword_score": 0, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "noveum_specific", + "query.keyword_density": 0.6, + "query.domain_affinity": "noveum", + "routing.decision": "Routed to RAG based on analysis", + "routing.rationale": "RAG score: 3, Web score: 0, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "Web Search", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.057692307692307696, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: RAG (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "29a862bf-d7aa-4cf9-963c-0a4244343bbe", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:56:54.718000000", + "trace_end_time": "2025-10-22 16:56:58.207000000", + "trace_duration_ms": 3488.068, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:56:54.718000000", + "updated_at": "2025-10-22 16:56:54.718000000" + }, + { + "span_id": "a378123a-8a79-4bc5-8e20-8ec54c2f9783", + "trace_id": "29a862bf-d7aa-4cf9-963c-0a4244343bbe", + "parent_span_id": "922c84a5-b14a-4b2f-8759-427d6e089812", + "name": "agent.llm_model_execution", + "start_time": "2025-10-22 16:56:55.154000000", + "end_time": "2025-10-22 16:56:57.166000000", + "duration_ms": 2012.174, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What are Noveum's pricing plans?", + "query_length": 32, + "query_type": "rag_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 2.012089252471924, + "model.latency_ms": 2012.0892524719238, + "model.start_time": 1761152215.154625, + "model.end_time": 1761152217.1667142, + "model.performance_tier": "medium", + "model.prompt_tokens": 658, + "model.completion_tokens": 110, + "model.total_tokens": 768, + "model.tokens_per_second": 381.6928096288395, + "model.estimated_cost": 0.02304, + "model.efficiency_score": 0.57421875, + "model.response_length": 441, + "model.response_quality": "high", + "model.output_response": "Model Response: The provided context does not include specific information about Noveum's pricing plans. It mentions that all users will have access to the Observability suite for free, and early users will receive f...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "29a862bf-d7aa-4cf9-963c-0a4244343bbe", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:56:54.718000000", + "trace_end_time": "2025-10-22 16:56:58.207000000", + "trace_duration_ms": 3488.068, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:56:54.718000000", + "updated_at": "2025-10-22 16:56:54.718000000" + }, + { + "span_id": "5813673b-a1dc-46c7-9bea-92ce6aca2b5e", + "trace_id": "29a862bf-d7aa-4cf9-963c-0a4244343bbe", + "parent_span_id": "922c84a5-b14a-4b2f-8759-427d6e089812", + "name": "agent.rag_evaluation_metrics", + "start_time": "2025-10-22 16:56:57.166000000", + "end_time": "2025-10-22 16:56:57.167000000", + "duration_ms": 0.059, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "rag_evaluation_metrics", + "agent.capabilities": [ + "retrieval_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What are Noveum's pricing plans?", + "query_length": 32, + "query_type": "rag_evaluation_query", + "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....", + "retrieval.context_length": 2105, + "retrieval.sources_count": 5, + "retrieval.context_quality": "high", + "retrieval.effectiveness": 1, + "retrieval.context_utilization": 2.105, + "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nSource 2 (https://noveum.ai/en/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nSource 3 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nThe AI landscape is complex and moving fast. With Noveum.ai, you'll have the insights you need to build, debug, and optimize AI applications with confidence.\nReady to see what's really happening inside your AI applications? Try Noveum.ai today and transform how you build and monitor AI systems.\nLet's build more reliable, observable AI\u2014together.\nGet Early Access to Noveum.ai Platform\nJoin the select group of AI teams optimizing their models with our data-driven platform. We're onboarding users in...\n\nSource 4 (https://noveum.ai/en/blog/from-logs-to-intelligent-choices-inside-noveum-ais-evaluation-process):\nGet Early Access to Noveum.ai Platform\nJoin the select group of AI teams optimizing their models with our data-driven platform. We're onboarding users in limited batches to ensure a premium experience....\n\nSource 5 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, vector searches, data retrieval, agent reasoning, and more. Without proper observability, debugging becomes a nightmare, optimization is guesswork, and understanding user interactions is nearly impossible.\nNoveum.ai solves this by providing comprehensive tracing and observability specifically designed for AI applications.\nKey benefits of using Noveum.ai:\n- Complete Visibility: Trace...\n\nUser Question: What are Noveum's pricing plans?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", + "prompt.prompt_length": 2633, + "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to ...", + "prompt.instruction_following": "rag_optimized", + "response.answer_length": 441, + "response.answer_completeness": "complete", + "response.response_quality": "high", + "response.source_citation": 5, + "response.context_utilization": 0.20950118764845604, + "output_response": "RAG Answer: The provided context does not include specific information about Noveum's pricing plans. It mentions that all users will have access to the Observability suite for free, and early users will receive f...", + "evaluation.retrieval_effectiveness": 1, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 5, + "evaluation.context_relevance": "high", + "evaluation.overall_quality": "high", + "evaluation.ready_for_production": true, + "rag.retrieval_strategy": "semantic_similarity", + "rag.vector_search_results": 5, + "rag.context_synthesis": "multi_source", + "rag.document_coverage": 1, + "rag.information_density": 0.20950118764845604 + }, + "events": [], + "links": [], + "trace_trace_id": "29a862bf-d7aa-4cf9-963c-0a4244343bbe", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:56:54.718000000", + "trace_end_time": "2025-10-22 16:56:58.207000000", + "trace_duration_ms": 3488.068, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:56:54.718000000", + "updated_at": "2025-10-22 16:56:54.718000000" + }, + { + "span_id": "c5baf5c2-d18a-458a-b922-a19b96db53d9", + "trace_id": "bc60c122-f30b-4114-9f78-ee40a024b525", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-22 16:56:44.873000000", + "end_time": "2025-10-22 16:56:53.701000000", + "duration_ms": 8827.979, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "How do I integrate Noveum Trace in my application?", + "query.length": 50, + "processing.mode": "RAG", + "processing.time_seconds": 8.827935218811035, + "processing.response_length": 1145, + "processing.sources_count": 5, + "output_response": "Final Answer: To integrate Noveum Trace into your application, you can follow these steps based on the SDK Integration Guide:\n\n1. **Create Your Account & Get API Key**:\n - Sign up at [noveum.ai](https://noveum.ai...", + "final_answer_mode": "RAG", + "query_processed.input_query": "How do I integrate Noveum Trace in my application?", + "query_processed.output_response": "Successfully processed query using RAG, generated 1145 character response", + "query_processed.mode": "RAG", + "query_processed.processing_time": 8.827935218811035, + "query_processed.response_length": 1145 + }, + "events": [], + "links": [], + "trace_trace_id": "bc60c122-f30b-4114-9f78-ee40a024b525", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:56:44.873000000", + "trace_end_time": "2025-10-22 16:56:53.701000000", + "trace_duration_ms": 8828.055, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:56:44.873000000", + "updated_at": "2025-10-22 16:56:44.873000000" + }, + { + "span_id": "8819dde5-9cbe-4cdb-b919-9c5140582359", + "trace_id": "bc60c122-f30b-4114-9f78-ee40a024b525", + "parent_span_id": "c5baf5c2-d18a-458a-b922-a19b96db53d9", + "name": "agent.query_routing", + "start_time": "2025-10-22 16:56:44.873000000", + "end_time": "2025-10-22 16:56:53.701000000", + "duration_ms": 8827.883, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "How do I integrate Noveum Trace in my application?", + "query_length": 50, + "query_type": "routing_query", + "output_response": "Routed to RAG for query processing", + "router.classification": "RAG", + "router.confidence_score": 1, + "router.rag_keyword_score": 2, + "router.web_keyword_score": 0, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "bc60c122-f30b-4114-9f78-ee40a024b525", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:56:44.873000000", + "trace_end_time": "2025-10-22 16:56:53.701000000", + "trace_duration_ms": 8828.055, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:56:44.873000000", + "updated_at": "2025-10-22 16:56:44.873000000" + }, + { + "span_id": "932c4b0e-0f33-40c0-bb19-78b9f81e48a6", + "trace_id": "bc60c122-f30b-4114-9f78-ee40a024b525", + "parent_span_id": "8819dde5-9cbe-4cdb-b919-9c5140582359", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-22 16:56:44.873000000", + "end_time": "2025-10-22 16:56:44.873000000", + "duration_ms": 0.043000000000000003, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: How do I integrate Noveum Trace in my application?", + "query_length": 50, + "query_type": "routing_evaluation_query", + "classification.mode": "RAG", + "classification.rag_keyword_score": 2, + "classification.web_keyword_score": 0, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "noveum_specific", + "query.keyword_density": 0.2222222222222222, + "query.domain_affinity": "noveum", + "routing.decision": "Routed to RAG based on analysis", + "routing.rationale": "RAG score: 2, Web score: 0, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "Web Search", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.038461538461538464, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: RAG (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "bc60c122-f30b-4114-9f78-ee40a024b525", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:56:44.873000000", + "trace_end_time": "2025-10-22 16:56:53.701000000", + "trace_duration_ms": 8828.055, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:56:44.873000000", + "updated_at": "2025-10-22 16:56:44.873000000" + }, + { + "span_id": "2ece2bd6-e4c8-47bc-b3c8-26f82e9910f1", + "trace_id": "bc60c122-f30b-4114-9f78-ee40a024b525", + "parent_span_id": "8819dde5-9cbe-4cdb-b919-9c5140582359", + "name": "agent.llm-rag", + "start_time": "2025-10-22 16:56:44.873000000", + "end_time": "2025-10-22 16:56:53.701000000", + "duration_ms": 8827.666000000001, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "rag_agent", + "agent.operation": "llm-rag", + "agent.capabilities": [ + "document_retrieval", + "context_generation", + "response_generation" + ], + "agent.id": "noveum_rag_agent", + "input_query": "How do I integrate Noveum Trace in my application?", + "query_length": 50, + "query_type": "rag_query", + "output_response": "RAG Answer: To integrate Noveum Trace into your application, you can follow these steps based on the SDK Integration Guide:\n\n1. **Create Your Account & Get API Key**:\n - Sign up at [noveum.ai](https://noveum.ai...", + "rag.context_length": 2803, + "rag.sources_count": 5, + "rag.answer_length": 1145, + "rag.mode": "retrieval_augmented_generation" + }, + "events": [], + "links": [], + "trace_trace_id": "bc60c122-f30b-4114-9f78-ee40a024b525", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:56:44.873000000", + "trace_end_time": "2025-10-22 16:56:53.701000000", + "trace_duration_ms": 8828.055, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:56:44.873000000", + "updated_at": "2025-10-22 16:56:44.873000000" + }, + { + "span_id": "e6315110-36b1-4503-beaf-12d972e25a7d", + "trace_id": "bc60c122-f30b-4114-9f78-ee40a024b525", + "parent_span_id": "2ece2bd6-e4c8-47bc-b3c8-26f82e9910f1", + "name": "agent.llm_model_execution", + "start_time": "2025-10-22 16:56:45.576000000", + "end_time": "2025-10-22 16:56:53.143000000", + "duration_ms": 7566.311, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: How do I integrate Noveum Trace in my application?", + "query_length": 50, + "query_type": "rag_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 7.566190004348755, + "model.latency_ms": 7566.190004348755, + "model.start_time": 1761152205.576988, + "model.end_time": 1761152213.143178, + "model.performance_tier": "slow", + "model.prompt_tokens": 837, + "model.completion_tokens": 286, + "model.total_tokens": 1123, + "model.tokens_per_second": 148.4234468543009, + "model.estimated_cost": 0.03369, + "model.efficiency_score": 1.0195903829029385, + "model.response_length": 1145, + "model.response_quality": "high", + "model.output_response": "Model Response: To integrate Noveum Trace into your application, you can follow these steps based on the SDK Integration Guide:\n\n1. **Create Your Account & Get API Key**:\n - Sign up at [noveum.ai](https://noveum.ai...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "bc60c122-f30b-4114-9f78-ee40a024b525", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:56:44.873000000", + "trace_end_time": "2025-10-22 16:56:53.701000000", + "trace_duration_ms": 8828.055, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:56:44.873000000", + "updated_at": "2025-10-22 16:56:44.873000000" + }, + { + "span_id": "ae91d520-8442-48e5-aabf-fd58aed23694", + "trace_id": "bc60c122-f30b-4114-9f78-ee40a024b525", + "parent_span_id": "2ece2bd6-e4c8-47bc-b3c8-26f82e9910f1", + "name": "agent.rag_evaluation_metrics", + "start_time": "2025-10-22 16:56:53.143000000", + "end_time": "2025-10-22 16:56:53.143000000", + "duration_ms": 0.091, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "rag_evaluation_metrics", + "agent.capabilities": [ + "retrieval_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: How do I integrate Noveum Trace in my application?", + "query_length": 50, + "query_type": "rag_evaluation_query", + "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\n\ud83d\udcd8 TypeScript SDK (\n@noveum/trace\n)- Framework ...", + "retrieval.context_length": 2803, + "retrieval.sources_count": 5, + "retrieval.context_quality": "high", + "retrieval.effectiveness": 1, + "retrieval.context_utilization": 2.803, + "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\n\ud83d\udcd8 TypeScript SDK (\n@noveum/trace\n)- Framework integrations for Next.js, Express.js, and Hono\n- TypeScript-first design with full type safety\n- Universal compatibility (Node.js, Edge Runtime, browsers)\n-\n\ud83d\udcca Noveum Platform\n- Real-time dashboard for analyzing traces and performance\n- Ad...\n\nSource 2 (https://noveum.ai/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\n\ud83d\udcd8 TypeScript SDK (\n@noveum/trace\n)- Framework integrations for Next.js, Express.js, and Hono\n- TypeScript-first design with full type safety\n- Universal compatibility (Node.js, Edge Runtime, browsers)\n-\n\ud83d\udcca Noveum Platform\n- Real-time dashboard for analyzing traces and performance\n- Ad...\n\nSource 3 (https://noveum.ai/docs/getting-started/sdk-integration):\nSDK Integration Guide\nIntegrate Noveum.ai tracing into your AI applications with Python or TypeScript SDKs\nThe Noveum.ai SDKs provide comprehensive tracing and observability for your AI applications with minimal code changes. Whether you're building LLM applications, RAG systems, or multi-agent workflows, our SDKs automatically capture essential metrics and traces.\n\ud83d\ude80 Quick Start\n1. Create Your Account & Get API Key\n- Sign up at noveum.ai\n- Create a project in your dashboard\n- Generate an API key...\n\nSource 4 (https://noveum.ai/en/docs/getting-started/sdk-integration):\nSDK Integration Guide\nIntegrate Noveum.ai tracing into your AI applications with Python or TypeScript SDKs\nThe Noveum.ai SDKs provide comprehensive tracing and observability for your AI applications with minimal code changes. Whether you're building LLM applications, RAG systems, or multi-agent workflows, our SDKs automatically capture essential metrics and traces.\n\ud83d\ude80 Quick Start\n1. Create Your Account & Get API Key\n- Sign up at noveum.ai\n- Create a project in your dashboard\n- Generate an API key...\n\nSource 5 (https://noveum.ai/en/docs/getting-started/sdk-integration):\n\ud83d\udd27 Framework Integrations\nNext.js Integration\nExpress.js Integration\nFastAPI Integration (Python)\n\ud83d\udcca Advanced Features\nCustom Attributes & Events\nSampling Configuration\n\ud83d\udd17 What's Captured Automatically\n- \ud83d\udcca Performance Metrics: Latency, throughput, error rates\n- \ud83d\udcb0 Cost Tracking: Token usage, API costs across providers\n- \ud83d\udd0d Request/Response: Configurable capture of inputs/outputs\n- \ud83c\udff7\ufe0f Metadata: Model names, parameters, user context\n- \ud83c\udf0a Context Flow: Trace relationships across services\n- \ud83d\udc1b Error Detail...\n\nUser Question: How do I integrate Noveum Trace in my application?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", + "prompt.prompt_length": 3349, + "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frame...", + "prompt.instruction_following": "rag_optimized", + "response.answer_length": 1145, + "response.answer_completeness": "complete", + "response.response_quality": "high", + "response.source_citation": 5, + "response.context_utilization": 0.4084909026043525, + "output_response": "RAG Answer: To integrate Noveum Trace into your application, you can follow these steps based on the SDK Integration Guide:\n\n1. **Create Your Account & Get API Key**:\n - Sign up at [noveum.ai](https://noveum.ai...", + "evaluation.retrieval_effectiveness": 1, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 5, + "evaluation.context_relevance": "high", + "evaluation.overall_quality": "high", + "evaluation.ready_for_production": true, + "rag.retrieval_strategy": "semantic_similarity", + "rag.vector_search_results": 5, + "rag.context_synthesis": "multi_source", + "rag.document_coverage": 1, + "rag.information_density": 0.4084909026043525 + }, + "events": [], + "links": [], + "trace_trace_id": "bc60c122-f30b-4114-9f78-ee40a024b525", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:56:44.873000000", + "trace_end_time": "2025-10-22 16:56:53.701000000", + "trace_duration_ms": 8828.055, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:56:44.873000000", + "updated_at": "2025-10-22 16:56:44.873000000" + }, + { + "span_id": "9a9c611d-f552-4b06-a764-4463edb70e2c", + "trace_id": "3ad192e5-bbd4-4236-8144-019d53512769", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-22 16:56:35.188000000", + "end_time": "2025-10-22 16:56:43.863000000", + "duration_ms": 8675.25, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What is Noveum and what does it do?", + "query.length": 35, + "processing.mode": "RAG", + "processing.time_seconds": 8.67501425743103, + "processing.response_length": 527, + "processing.sources_count": 5, + "output_response": "Final Answer: Noveum.ai is a comprehensive tracing and observability platform specifically designed for AI applications, including LLM-powered chatbots, RAG systems, and multi-agent workflows. It provides the neces...", + "final_answer_mode": "RAG", + "query_processed.input_query": "What is Noveum and what does it do?", + "query_processed.output_response": "Successfully processed query using RAG, generated 527 character response", + "query_processed.mode": "RAG", + "query_processed.processing_time": 8.67501425743103, + "query_processed.response_length": 527 + }, + "events": [], + "links": [], + "trace_trace_id": "3ad192e5-bbd4-4236-8144-019d53512769", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:56:35.188000000", + "trace_end_time": "2025-10-22 16:56:43.863000000", + "trace_duration_ms": 8675.356000000002, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:56:35.188000000", + "updated_at": "2025-10-22 16:56:35.188000000" + }, + { + "span_id": "99eadeae-0733-4bf3-982e-e9f3014f5913", + "trace_id": "3ad192e5-bbd4-4236-8144-019d53512769", + "parent_span_id": "7e2c959c-157a-43d5-9870-32994cddd388", + "name": "agent.llm-rag", + "start_time": "2025-10-22 16:56:35.188000000", + "end_time": "2025-10-22 16:56:43.863000000", + "duration_ms": 8674.699999999999, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "rag_agent", + "agent.operation": "llm-rag", + "agent.capabilities": [ + "document_retrieval", + "context_generation", + "response_generation" + ], + "agent.id": "noveum_rag_agent", + "input_query": "What is Noveum and what does it do?", + "query_length": 35, + "query_type": "rag_query", + "output_response": "RAG Answer: Noveum.ai is a comprehensive tracing and observability platform specifically designed for AI applications, including LLM-powered chatbots, RAG systems, and multi-agent workflows. It provides the neces...", + "rag.context_length": 2334, + "rag.sources_count": 5, + "rag.answer_length": 527, + "rag.mode": "retrieval_augmented_generation" + }, + "events": [], + "links": [], + "trace_trace_id": "3ad192e5-bbd4-4236-8144-019d53512769", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:56:35.188000000", + "trace_end_time": "2025-10-22 16:56:43.863000000", + "trace_duration_ms": 8675.356000000002, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:56:35.188000000", + "updated_at": "2025-10-22 16:56:35.188000000" + }, + { + "span_id": "7e2c959c-157a-43d5-9870-32994cddd388", + "trace_id": "3ad192e5-bbd4-4236-8144-019d53512769", + "parent_span_id": "9a9c611d-f552-4b06-a764-4463edb70e2c", + "name": "agent.query_routing", + "start_time": "2025-10-22 16:56:35.188000000", + "end_time": "2025-10-22 16:56:43.863000000", + "duration_ms": 8674.961, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What is Noveum and what does it do?", + "query_length": 35, + "query_type": "routing_query", + "output_response": "Routed to RAG for query processing", + "router.classification": "RAG", + "router.confidence_score": 1, + "router.rag_keyword_score": 1, + "router.web_keyword_score": 0, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "3ad192e5-bbd4-4236-8144-019d53512769", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:56:35.188000000", + "trace_end_time": "2025-10-22 16:56:43.863000000", + "trace_duration_ms": 8675.356000000002, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:56:35.188000000", + "updated_at": "2025-10-22 16:56:35.188000000" + }, + { + "span_id": "6f81284c-a9aa-4d62-a402-1b474f2eba51", + "trace_id": "3ad192e5-bbd4-4236-8144-019d53512769", + "parent_span_id": "7e2c959c-157a-43d5-9870-32994cddd388", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-22 16:56:35.188000000", + "end_time": "2025-10-22 16:56:35.188000000", + "duration_ms": 0.054, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What is Noveum and what does it do?", + "query_length": 35, + "query_type": "routing_evaluation_query", + "classification.mode": "RAG", + "classification.rag_keyword_score": 1, + "classification.web_keyword_score": 0, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "noveum_specific", + "query.keyword_density": 0.125, + "query.domain_affinity": "noveum", + "routing.decision": "Routed to RAG based on analysis", + "routing.rationale": "RAG score: 1, Web score: 0, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "Web Search", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.019230769230769232, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: RAG (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "3ad192e5-bbd4-4236-8144-019d53512769", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:56:35.188000000", + "trace_end_time": "2025-10-22 16:56:43.863000000", + "trace_duration_ms": 8675.356000000002, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:56:35.188000000", + "updated_at": "2025-10-22 16:56:35.188000000" + }, + { + "span_id": "9f42d18a-0235-4ae2-838c-19a88ddc95e1", + "trace_id": "3ad192e5-bbd4-4236-8144-019d53512769", + "parent_span_id": "99eadeae-0733-4bf3-982e-e9f3014f5913", + "name": "agent.llm_model_execution", + "start_time": "2025-10-22 16:56:36.031000000", + "end_time": "2025-10-22 16:56:42.012000000", + "duration_ms": 5980.947999999999, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What is Noveum and what does it do?", + "query_length": 35, + "query_type": "rag_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 5.980915784835815, + "model.latency_ms": 5980.915784835815, + "model.start_time": 1761152196.031816, + "model.end_time": 1761152202.0127318, + "model.performance_tier": "slow", + "model.prompt_tokens": 716, + "model.completion_tokens": 131, + "model.total_tokens": 847, + "model.tokens_per_second": 141.61710856178715, + "model.estimated_cost": 0.025410000000000002, + "model.efficiency_score": 0.6221959858323495, + "model.response_length": 527, + "model.response_quality": "high", + "model.output_response": "Model Response: Noveum.ai is a comprehensive tracing and observability platform specifically designed for AI applications, including LLM-powered chatbots, RAG systems, and multi-agent workflows. It provides the neces...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "3ad192e5-bbd4-4236-8144-019d53512769", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:56:35.188000000", + "trace_end_time": "2025-10-22 16:56:43.863000000", + "trace_duration_ms": 8675.356000000002, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:56:35.188000000", + "updated_at": "2025-10-22 16:56:35.188000000" + }, + { + "span_id": "7bcbe4b7-585a-4c9d-add8-16e8aa62254f", + "trace_id": "3ad192e5-bbd4-4236-8144-019d53512769", + "parent_span_id": "99eadeae-0733-4bf3-982e-e9f3014f5913", + "name": "agent.rag_evaluation_metrics", + "start_time": "2025-10-22 16:56:42.012000000", + "end_time": "2025-10-22 16:56:42.012000000", + "duration_ms": 0.018000000000000002, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "rag_evaluation_metrics", + "agent.capabilities": [ + "retrieval_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What is Noveum and what does it do?", + "query_length": 35, + "query_type": "rag_evaluation_query", + "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, vector searches, data retrieval, agent reasoning, and more. Without proper observability, debugging ...", + "retrieval.context_length": 2334, + "retrieval.sources_count": 5, + "retrieval.context_quality": "high", + "retrieval.effectiveness": 1, + "retrieval.context_utilization": 2.334, + "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, vector searches, data retrieval, agent reasoning, and more. Without proper observability, debugging becomes a nightmare, optimization is guesswork, and understanding user interactions is nearly impossible.\nNoveum.ai solves this by providing comprehensive tracing and observability specifically designed for AI applications.\nKey benefits of using Noveum.ai:\n- Complete Visibility: Trace...\n\nSource 2 (https://noveum.ai/en/docs/getting-started/overview):\nNoveum.ai Overview\nComprehensive AI tracing and observability platform for LLM applications, RAG systems, and AI agents\nWelcome to Noveum.ai\u2014the comprehensive tracing and observability platform built specifically for AI applications. Whether you're building LLM-powered chatbots, RAG systems, multi-agent workflows, or any AI-driven application, Noveum provides the insights you need to understand, debug, and optimize your systems.\n\ud83c\udfaf Why AI Applications Need Specialized Observability\nTraditional mo...\n\nSource 3 (https://noveum.ai/docs/getting-started/overview):\nNoveum.ai Overview\nComprehensive AI tracing and observability platform for LLM applications, RAG systems, and AI agents\nWelcome to Noveum.ai\u2014the comprehensive tracing and observability platform built specifically for AI applications. Whether you're building LLM-powered chatbots, RAG systems, multi-agent workflows, or any AI-driven application, Noveum provides the insights you need to understand, debug, and optimize your systems.\n\ud83c\udfaf Why AI Applications Need Specialized Observability\nTraditional mo...\n\nSource 4 (https://noveum.ai/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nSource 5 (https://noveum.ai/en/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nUser Question: What is Noveum and what does it do?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", + "prompt.prompt_length": 2865, + "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, ...", + "prompt.instruction_following": "rag_optimized", + "response.answer_length": 527, + "response.answer_completeness": "complete", + "response.response_quality": "high", + "response.source_citation": 5, + "response.context_utilization": 0.22579263067694944, + "output_response": "RAG Answer: Noveum.ai is a comprehensive tracing and observability platform specifically designed for AI applications, including LLM-powered chatbots, RAG systems, and multi-agent workflows. It provides the neces...", + "evaluation.retrieval_effectiveness": 1, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 5, + "evaluation.context_relevance": "high", + "evaluation.overall_quality": "high", + "evaluation.ready_for_production": true, + "rag.retrieval_strategy": "semantic_similarity", + "rag.vector_search_results": 5, + "rag.context_synthesis": "multi_source", + "rag.document_coverage": 1, + "rag.information_density": 0.22579263067694944 + }, + "events": [], + "links": [], + "trace_trace_id": "3ad192e5-bbd4-4236-8144-019d53512769", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:56:35.188000000", + "trace_end_time": "2025-10-22 16:56:43.863000000", + "trace_duration_ms": 8675.356000000002, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:56:35.188000000", + "updated_at": "2025-10-22 16:56:35.188000000" + }, + { + "span_id": "d39297f4-b4fe-4f9c-b45b-854411ba770a", + "trace_id": "9af2b4b0-0c2f-4d2e-ae99-30df0d2f0fa6", + "parent_span_id": "", + "name": "system_initialization", + "start_time": "2025-10-22 16:56:34.565000000", + "end_time": "2025-10-22 16:56:35.186000000", + "duration_ms": 621.02, + "status": "ok", + "status_message": "", + "attributes": { + "system.force_scrape": false, + "system.config": { + "noveum_base_url": "https://noveum.ai", + "max_pages_to_scrape": 50, + "chunk_size": 1000, + "chunk_overlap": 200, + "max_search_results": 5, + "rag_threshold": 0.7, + "noveum_docs_file": "noveum_docs.json", + "vector_store_path": "noveum_vectorstore" + }, + "input_query": "Initialize system with force_scrape=False", + "output_response": "System initialization: RAG system loaded, vector store ready, agent operational", + "system.initialized": true, + "system.documents_loaded": 38, + "system.vectorstore_ready": true + }, + "events": [], + "links": [], + "trace_trace_id": "9af2b4b0-0c2f-4d2e-ae99-30df0d2f0fa6", + "trace_name": "auto_trace_system_initialization", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:56:34.565000000", + "trace_end_time": "2025-10-22 16:56:35.186000000", + "trace_duration_ms": 621.177, + "span_count": 1, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:56:34.565000000", + "updated_at": "2025-10-22 16:56:34.565000000" + }, + { + "span_id": "8c144ad8-a439-4986-b741-3bc4b7fc6110", + "trace_id": "bae13042-cf2e-4813-a673-7dc2a611725e", + "parent_span_id": "d37fe1a1-a126-4e4e-a914-93dd402fed3f", + "name": "agent.query_routing", + "start_time": "2025-10-16 08:50:56.153000000", + "end_time": "2025-10-16 08:51:01.908000000", + "duration_ms": 5754.43, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What are the latest breakthroughs in artificial intelligence?", + "query_length": 61, + "query_type": "routing_query", + "output_response": "Routed to Web Search for query processing", + "router.classification": "Web Search", + "router.confidence_score": 1, + "router.rag_keyword_score": 0, + "router.web_keyword_score": 1, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "bae13042-cf2e-4813-a673-7dc2a611725e", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:56.153000000", + "trace_end_time": "2025-10-16 08:51:01.908000000", + "trace_duration_ms": 5754.578, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:56.153000000", + "updated_at": "2025-10-16 08:50:56.153000000" + }, + { + "span_id": "b294f9b7-7762-47a9-b49d-00801a984af2", + "trace_id": "bae13042-cf2e-4813-a673-7dc2a611725e", + "parent_span_id": "8c144ad8-a439-4986-b741-3bc4b7fc6110", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-16 08:50:56.153000000", + "end_time": "2025-10-16 08:50:56.153000000", + "duration_ms": 0.024, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What are the latest breakthroughs in artificial intelligence?", + "query_length": 61, + "query_type": "routing_evaluation_query", + "classification.mode": "Web Search", + "classification.rag_keyword_score": 0, + "classification.web_keyword_score": 1, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "complex", + "query.intent": "general_knowledge", + "query.keyword_density": 0.125, + "query.domain_affinity": "general", + "routing.decision": "Routed to Web Search based on analysis", + "routing.rationale": "RAG score: 0, Web score: 1, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "RAG", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.019230769230769232, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: Web Search (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "bae13042-cf2e-4813-a673-7dc2a611725e", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:56.153000000", + "trace_end_time": "2025-10-16 08:51:01.908000000", + "trace_duration_ms": 5754.578, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:56.153000000", + "updated_at": "2025-10-16 08:50:56.153000000" + }, + { + "span_id": "ab6a281a-4989-40c8-9011-d0ec7dde7ea9", + "trace_id": "bae13042-cf2e-4813-a673-7dc2a611725e", + "parent_span_id": "8c144ad8-a439-4986-b741-3bc4b7fc6110", + "name": "agent.web_search_generation", + "start_time": "2025-10-16 08:50:56.153000000", + "end_time": "2025-10-16 08:51:01.908000000", + "duration_ms": 5754.317999999999, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "web_search_agent", + "agent.operation": "web_search_generation", + "agent.capabilities": [ + "web_search", + "content_synthesis", + "response_generation" + ], + "agent.id": "noveum_web_search_agent", + "input_query": "What are the latest breakthroughs in artificial intelligence?", + "query_length": 61, + "query_type": "web_search_query", + "output_response": "Web Search Answer: The web search did not yield specific results regarding the latest breakthroughs in artificial intelligence. However, I can provide a general overview based on knowledge up to October 2023.\n\nRecent br...", + "web_search.results_count": 1, + "web_search.context_length": 163, + "web_search.response_length": 1489, + "web_search.mode": "external_web_search" + }, + "events": [], + "links": [], + "trace_trace_id": "bae13042-cf2e-4813-a673-7dc2a611725e", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:56.153000000", + "trace_end_time": "2025-10-16 08:51:01.908000000", + "trace_duration_ms": 5754.578, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:56.153000000", + "updated_at": "2025-10-16 08:50:56.153000000" + }, + { + "span_id": "d37fe1a1-a126-4e4e-a914-93dd402fed3f", + "trace_id": "bae13042-cf2e-4813-a673-7dc2a611725e", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-16 08:50:56.153000000", + "end_time": "2025-10-16 08:51:01.908000000", + "duration_ms": 5754.527, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What are the latest breakthroughs in artificial intelligence?", + "query.length": 61, + "processing.mode": "Web Search", + "processing.time_seconds": 5.754457473754883, + "processing.response_length": 1489, + "processing.sources_count": 1, + "output_response": "Final Answer: The web search did not yield specific results regarding the latest breakthroughs in artificial intelligence. However, I can provide a general overview based on knowledge up to October 2023.\n\nRecent br...", + "final_answer_mode": "Web Search", + "query_processed.input_query": "What are the latest breakthroughs in artificial intelligence?", + "query_processed.output_response": "Successfully processed query using Web Search, generated 1489 character response", + "query_processed.mode": "Web Search", + "query_processed.processing_time": 5.754457473754883, + "query_processed.response_length": 1489 + }, + "events": [], + "links": [], + "trace_trace_id": "bae13042-cf2e-4813-a673-7dc2a611725e", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:56.153000000", + "trace_end_time": "2025-10-16 08:51:01.908000000", + "trace_duration_ms": 5754.578, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:56.153000000", + "updated_at": "2025-10-16 08:50:56.153000000" + }, + { + "span_id": "f61ef76d-4887-42da-9fd6-e2cdb25a9e09", + "trace_id": "bae13042-cf2e-4813-a673-7dc2a611725e", + "parent_span_id": "ab6a281a-4989-40c8-9011-d0ec7dde7ea9", + "name": "agent.llm_model_execution", + "start_time": "2025-10-16 08:50:56.502000000", + "end_time": "2025-10-16 08:51:01.908000000", + "duration_ms": 5405.44, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What are the latest breakthroughs in artificial intelligence?", + "query_length": 61, + "query_type": "web_search_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 5.405392408370972, + "model.latency_ms": 5405.392408370972, + "model.start_time": 1760604656.5026655, + "model.end_time": 1760604661.908058, + "model.performance_tier": "slow", + "model.prompt_tokens": 167, + "model.completion_tokens": 372, + "model.total_tokens": 539, + "model.tokens_per_second": 99.71523976044487, + "model.estimated_cost": 0.01617, + "model.efficiency_score": 2.7625231910946195, + "model.response_length": 1489, + "model.response_quality": "high", + "model.output_response": "Model Response: The web search did not yield specific results regarding the latest breakthroughs in artificial intelligence. However, I can provide a general overview based on knowledge up to October 2023.\n\nRecent br...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "bae13042-cf2e-4813-a673-7dc2a611725e", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:56.153000000", + "trace_end_time": "2025-10-16 08:51:01.908000000", + "trace_duration_ms": 5754.578, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:56.153000000", + "updated_at": "2025-10-16 08:50:56.153000000" + }, + { + "span_id": "550bf5ae-c816-4ef0-a6c1-1553b1c258b8", + "trace_id": "bae13042-cf2e-4813-a673-7dc2a611725e", + "parent_span_id": "ab6a281a-4989-40c8-9011-d0ec7dde7ea9", + "name": "agent.web_search_evaluation_metrics", + "start_time": "2025-10-16 08:51:01.908000000", + "end_time": "2025-10-16 08:51:01.908000000", + "duration_ms": 0.018000000000000002, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "web_search_evaluation_metrics", + "agent.capabilities": [ + "web_search_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What are the latest breakthroughs in artificial intelligence?", + "query_length": 61, + "query_type": "web_search_evaluation_query", + "web_search.results_count": 1, + "web_search.context_length": 163, + "web_search.context_synthesized": "Context from 1 web sources", + "web_search.search_effectiveness": 0.2, + "web_search.context_quality": "low", + "web_search.source_diversity": 1, + "web_search.information_synthesis": "low", + "web_search.external_knowledge_utilization": 0.163, + "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What+are+the+latest+breakthroughs+in+artificial+intelligence?\n\nUser Question: What are the latest breakthroughs in artificial intelligence?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", + "prompt.prompt_length": 669, + "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What+are+the+latest+breakthroughs+in+artificial+intelligence?", + "prompt.instruction_following": "web_search_optimized", + "response.answer_length": 1489, + "response.answer_completeness": "complete", + "response.response_quality": "medium", + "response.source_citation": 1, + "response.context_utilization": 9.134969325153374, + "output_response": "Web Search Answer: The web search did not yield specific results regarding the latest breakthroughs in artificial intelligence. However, I can provide a general overview based on knowledge up to October 2023.\n\nRecent br...", + "evaluation.search_effectiveness": 0.2, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 1, + "evaluation.context_relevance": "low", + "evaluation.overall_quality": "low", + "evaluation.ready_for_production": true, + "web_search.search_strategy": "duckduckgo_api", + "web_search.real_time_data": true, + "web_search.external_sources": 1, + "web_search.information_freshness": "current", + "web_search.knowledge_synthesis": "single_source" + }, + "events": [], + "links": [], + "trace_trace_id": "bae13042-cf2e-4813-a673-7dc2a611725e", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:56.153000000", + "trace_end_time": "2025-10-16 08:51:01.908000000", + "trace_duration_ms": 5754.578, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:56.153000000", + "updated_at": "2025-10-16 08:50:56.153000000" + }, + { + "span_id": "04cb7d29-9ea8-4ae0-9834-030cae944eca", + "trace_id": "79f9d235-d46e-4740-be35-ef0006071f6c", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-16 08:50:51.539000000", + "end_time": "2025-10-16 08:50:55.150000000", + "duration_ms": 3611.008, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What's happening in the software development world today?", + "query.length": 57, + "processing.mode": "Web Search", + "processing.time_seconds": 3.6109254360198975, + "processing.response_length": 526, + "processing.sources_count": 1, + "output_response": "Final Answer: The provided web search results do not contain specific information about current events or trends in the software development world. They primarily focus on unrelated topics, such as the meaning of \"...", + "final_answer_mode": "Web Search", + "query_processed.input_query": "What's happening in the software development world today?", + "query_processed.output_response": "Successfully processed query using Web Search, generated 526 character response", + "query_processed.mode": "Web Search", + "query_processed.processing_time": 3.6109254360198975, + "query_processed.response_length": 526 + }, + "events": [], + "links": [], + "trace_trace_id": "79f9d235-d46e-4740-be35-ef0006071f6c", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:51.539000000", + "trace_end_time": "2025-10-16 08:50:55.150000000", + "trace_duration_ms": 3611.084, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:51.539000000", + "updated_at": "2025-10-16 08:50:51.539000000" + }, + { + "span_id": "e5e3c927-783e-48c5-aec7-b32bae98c0a2", + "trace_id": "79f9d235-d46e-4740-be35-ef0006071f6c", + "parent_span_id": "f630c3e2-091e-4bd9-9556-e0ebc1c50291", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-16 08:50:51.539000000", + "end_time": "2025-10-16 08:50:51.540000000", + "duration_ms": 0.041, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What's happening in the software development world today?", + "query_length": 57, + "query_type": "routing_evaluation_query", + "classification.mode": "Web Search", + "classification.rag_keyword_score": 0, + "classification.web_keyword_score": 1, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "complex", + "query.intent": "general_knowledge", + "query.keyword_density": 0.125, + "query.domain_affinity": "general", + "routing.decision": "Routed to Web Search based on analysis", + "routing.rationale": "RAG score: 0, Web score: 1, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "RAG", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.019230769230769232, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: Web Search (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "79f9d235-d46e-4740-be35-ef0006071f6c", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:51.539000000", + "trace_end_time": "2025-10-16 08:50:55.150000000", + "trace_duration_ms": 3611.084, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:51.539000000", + "updated_at": "2025-10-16 08:50:51.539000000" + }, + { + "span_id": "f630c3e2-091e-4bd9-9556-e0ebc1c50291", + "trace_id": "79f9d235-d46e-4740-be35-ef0006071f6c", + "parent_span_id": "04cb7d29-9ea8-4ae0-9834-030cae944eca", + "name": "agent.query_routing", + "start_time": "2025-10-16 08:50:51.539000000", + "end_time": "2025-10-16 08:50:55.150000000", + "duration_ms": 3610.876, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What's happening in the software development world today?", + "query_length": 57, + "query_type": "routing_query", + "output_response": "Routed to Web Search for query processing", + "router.classification": "Web Search", + "router.confidence_score": 1, + "router.rag_keyword_score": 0, + "router.web_keyword_score": 1, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "79f9d235-d46e-4740-be35-ef0006071f6c", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:51.539000000", + "trace_end_time": "2025-10-16 08:50:55.150000000", + "trace_duration_ms": 3611.084, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:51.539000000", + "updated_at": "2025-10-16 08:50:51.539000000" + }, + { + "span_id": "d7b0f399-3b43-43bf-bac1-b2101268841c", + "trace_id": "79f9d235-d46e-4740-be35-ef0006071f6c", + "parent_span_id": "f630c3e2-091e-4bd9-9556-e0ebc1c50291", + "name": "agent.web_search_generation", + "start_time": "2025-10-16 08:50:51.540000000", + "end_time": "2025-10-16 08:50:55.150000000", + "duration_ms": 3610.679, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "web_search_agent", + "agent.operation": "web_search_generation", + "agent.capabilities": [ + "web_search", + "content_synthesis", + "response_generation" + ], + "agent.id": "noveum_web_search_agent", + "input_query": "What's happening in the software development world today?", + "query_length": 57, + "query_type": "web_search_query", + "output_response": "Web Search Answer: The provided web search results do not contain specific information about current events or trends in the software development world. They primarily focus on unrelated topics, such as the meaning of \"...", + "web_search.results_count": 1, + "web_search.context_length": 589, + "web_search.response_length": 526, + "web_search.mode": "external_web_search" + }, + "events": [], + "links": [], + "trace_trace_id": "79f9d235-d46e-4740-be35-ef0006071f6c", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:51.539000000", + "trace_end_time": "2025-10-16 08:50:55.150000000", + "trace_duration_ms": 3611.084, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:51.539000000", + "updated_at": "2025-10-16 08:50:51.539000000" + }, + { + "span_id": "4ed5dae4-f850-4644-aa55-2f1c00d11c22", + "trace_id": "79f9d235-d46e-4740-be35-ef0006071f6c", + "parent_span_id": "d7b0f399-3b43-43bf-bac1-b2101268841c", + "name": "agent.llm_model_execution", + "start_time": "2025-10-16 08:50:51.976000000", + "end_time": "2025-10-16 08:50:55.150000000", + "duration_ms": 3174.51, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What's happening in the software development world today?", + "query_length": 57, + "query_type": "web_search_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 3.174463987350464, + "model.latency_ms": 3174.463987350464, + "model.start_time": 1760604651.9761698, + "model.end_time": 1760604655.1506338, + "model.performance_tier": "medium", + "model.prompt_tokens": 272, + "model.completion_tokens": 131, + "model.total_tokens": 403, + "model.tokens_per_second": 126.95056601866197, + "model.estimated_cost": 0.01209, + "model.efficiency_score": 1.305210918114144, + "model.response_length": 526, + "model.response_quality": "high", + "model.output_response": "Model Response: The provided web search results do not contain specific information about current events or trends in the software development world. They primarily focus on unrelated topics, such as the meaning of \"...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "79f9d235-d46e-4740-be35-ef0006071f6c", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:51.539000000", + "trace_end_time": "2025-10-16 08:50:55.150000000", + "trace_duration_ms": 3611.084, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:51.539000000", + "updated_at": "2025-10-16 08:50:51.539000000" + }, + { + "span_id": "eb151834-9322-4f63-add0-dec9a2966367", + "trace_id": "79f9d235-d46e-4740-be35-ef0006071f6c", + "parent_span_id": "d7b0f399-3b43-43bf-bac1-b2101268841c", + "name": "agent.web_search_evaluation_metrics", + "start_time": "2025-10-16 08:50:55.150000000", + "end_time": "2025-10-16 08:50:55.150000000", + "duration_ms": 0.02, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "web_search_evaluation_metrics", + "agent.capabilities": [ + "web_search_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What's happening in the software development world today?", + "query_length": 57, + "query_type": "web_search_evaluation_query", + "web_search.results_count": 1, + "web_search.context_length": 589, + "web_search.context_synthesized": "Context from 1 web sources", + "web_search.search_effectiveness": 0.2, + "web_search.context_quality": "medium", + "web_search.source_diversity": 1, + "web_search.information_synthesis": "low", + "web_search.external_knowledge_utilization": 0.589, + "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\n\u66fc\u5df4\u7684\u610f\u601d\u662f\u4e00\u79cd\u6bd2\u86c7\uff0c\u8c61\u5f81\u7740\u79d1\u6bd4\u5728\u7403\u573a\u4e0a\u7684\u653b\u51fb\u6027\u548c\u575a\u97e7\u4e0d\u62d4 \u201cWhat can i say\uff0c Mamba out\uff01\u201d \u4f46\u662f\u7ecf\u8fc7\u4e92\u8054\u7f51\u7684\u89e3\u6784\uff0c\u8fd9\u53e5\u8bdd\u6210\u4e3a\u4e86\u4e00\u53e5\u4e07\u7528\u573a\u666f\u7684\u8bdd \u2026 Aprende ingl\u00e9s gratis con nuestros cursos. \u00a1Mas de 100 lecciones con ejemplos y audio! Distintas y divertidas formas de estudiar ingl\u00e9s: \u2026 \u77e5\u4e4e\uff0c\u4e2d\u6587\u4e92\u8054\u7f51\u9ad8\u8d28\u91cf\u7684\u95ee\u7b54\u793e\u533a\u548c\u521b\u4f5c\u8005\u805a\u96c6\u7684\u539f\u521b\u5185\u5bb9\u5e73\u53f0\uff0c\u4e8e 2011 \u5e74 1 \u6708\u6b63\u5f0f\u4e0a\u7ebf\uff0c\u4ee5\u300c\u8ba9\u4eba\u4eec\u66f4\u597d\u7684\u5206\u4eab\u77e5\u8bc6\u3001\u7ecf\u9a8c\u548c\u89c1\u89e3\uff0c\u627e\u5230\u81ea\u5df1\u7684\u89e3\u7b54\u300d\u4e3a\u54c1 \u2026 \u8f6f\u4ef6\u7279\u70b9 1\u3001Windows Desktop Runtime\u662f\u5fae\u8f6fWindows\u684c\u9762\u7a0b\u5e8f\u8fd0\u884c\u5e93\uff08\u542b\u5e38\u89c4\u8fd0\u884c\u5e93\uff09 2\u3001\u80fd\u5b8c\u7f8e\u517c\u5bb9\u5fae\u8f6f\u4e0d\u540c\u7248\u672c\u7684Windows\u7cfb\u7edf\uff0c\u89e3\u51b3\u5176\u7a0b\u5e8f\u7f3a \u2026 \u77e5\u4e4e\uff0c\u4e2d\u6587\u4e92\u8054\u7f51\u9ad8\u8d28\u91cf\u7684\u95ee\u7b54\u793e\u533a\u548c\u521b\u4f5c\u8005\u805a\u96c6\u7684\u539f\u521b\u5185\u5bb9\u5e73\u53f0\uff0c\u4e8e 2011 \u5e74 1 \u6708\u6b63\u5f0f\u4e0a\u7ebf\uff0c\u4ee5\u300c\u8ba9\u4eba\u4eec\u66f4\u597d\u7684\u5206\u4eab\u77e5\u8bc6\u3001\u7ecf\u9a8c\u548c\u89c1\u89e3\uff0c\u627e\u5230\u81ea\u5df1\u7684\u89e3\u7b54\u300d\u4e3a\u54c1 \u2026\nURL: https://duckduckgo.com/?q=What's+happening+in+the+software+development+world+today?\n\nUser Question: What's happening in the software development world today?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", + "prompt.prompt_length": 1091, + "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\n\u66fc\u5df4\u7684\u610f\u601d\u662f\u4e00\u79cd\u6bd2\u86c7\uff0c\u8c61\u5f81\u7740\u79d1\u6bd4\u5728\u7403\u573a\u4e0a\u7684\u653b\u51fb\u6027\u548c\u575a\u97e7\u4e0d\u62d4 \u201cWhat can i say\uff0c Mamba out\uff01\u201d \u4f46\u662f\u7ecf\u8fc7\u4e92\u8054\u7f51\u7684\u89e3\u6784\uff0c\u8fd9\u53e5\u8bdd\u6210\u4e3a\u4e86\u4e00\u53e5\u4e07\u7528\u573a\u666f\u7684\u8bdd \u2026 Aprende ingl\u00e9s gratis con nuestros cursos. \u00a1Mas de 100 lecciones con ejemplos y audio...", + "prompt.instruction_following": "web_search_optimized", + "response.answer_length": 526, + "response.answer_completeness": "complete", + "response.response_quality": "medium", + "response.source_citation": 1, + "response.context_utilization": 0.8930390492359932, + "output_response": "Web Search Answer: The provided web search results do not contain specific information about current events or trends in the software development world. They primarily focus on unrelated topics, such as the meaning of \"...", + "evaluation.search_effectiveness": 0.2, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 1, + "evaluation.context_relevance": "medium", + "evaluation.overall_quality": "low", + "evaluation.ready_for_production": true, + "web_search.search_strategy": "duckduckgo_api", + "web_search.real_time_data": true, + "web_search.external_sources": 1, + "web_search.information_freshness": "current", + "web_search.knowledge_synthesis": "single_source" + }, + "events": [], + "links": [], + "trace_trace_id": "79f9d235-d46e-4740-be35-ef0006071f6c", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:51.539000000", + "trace_end_time": "2025-10-16 08:50:55.150000000", + "trace_duration_ms": 3611.084, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:51.539000000", + "updated_at": "2025-10-16 08:50:51.539000000" + }, + { + "span_id": "888b4441-3678-42a5-a752-8daae2433482", + "trace_id": "77b81bd5-5a04-4fc0-9fc6-571f7907d6b1", + "parent_span_id": "ba6ffeeb-78b5-44dd-b460-7933ca6e7739", + "name": "agent.query_routing", + "start_time": "2025-10-16 08:50:44.888000000", + "end_time": "2025-10-16 08:50:50.529000000", + "duration_ms": 5641.062, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What are the newest features in cloud computing?", + "query_length": 48, + "query_type": "routing_query", + "output_response": "Routed to RAG for query processing", + "router.classification": "RAG", + "router.confidence_score": 1, + "router.rag_keyword_score": 1, + "router.web_keyword_score": 0, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "77b81bd5-5a04-4fc0-9fc6-571f7907d6b1", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:44.888000000", + "trace_end_time": "2025-10-16 08:50:50.529000000", + "trace_duration_ms": 5641.226, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:44.888000000", + "updated_at": "2025-10-16 08:50:44.888000000" + }, + { + "span_id": "b6bae768-93a6-47d9-8369-61897f9d7828", + "trace_id": "77b81bd5-5a04-4fc0-9fc6-571f7907d6b1", + "parent_span_id": "888b4441-3678-42a5-a752-8daae2433482", + "name": "agent.llm-rag", + "start_time": "2025-10-16 08:50:44.888000000", + "end_time": "2025-10-16 08:50:50.529000000", + "duration_ms": 5640.875999999999, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "rag_agent", + "agent.operation": "llm-rag", + "agent.capabilities": [ + "document_retrieval", + "context_generation", + "response_generation" + ], + "agent.id": "noveum_rag_agent", + "input_query": "What are the newest features in cloud computing?", + "query_length": 48, + "query_type": "rag_query", + "output_response": "RAG Answer: The provided context does not specifically mention any new features in cloud computing as a general topic. However, it does highlight several recent enhancements and features related to the Noveum.ai ...", + "rag.context_length": 2579, + "rag.sources_count": 5, + "rag.answer_length": 983, + "rag.mode": "retrieval_augmented_generation" + }, + "events": [], + "links": [], + "trace_trace_id": "77b81bd5-5a04-4fc0-9fc6-571f7907d6b1", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:44.888000000", + "trace_end_time": "2025-10-16 08:50:50.529000000", + "trace_duration_ms": 5641.226, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:44.888000000", + "updated_at": "2025-10-16 08:50:44.888000000" + }, + { + "span_id": "ba6ffeeb-78b5-44dd-b460-7933ca6e7739", + "trace_id": "77b81bd5-5a04-4fc0-9fc6-571f7907d6b1", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-16 08:50:44.888000000", + "end_time": "2025-10-16 08:50:50.529000000", + "duration_ms": 5641.156, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What are the newest features in cloud computing?", + "query.length": 48, + "processing.mode": "RAG", + "processing.time_seconds": 5.64110541343689, + "processing.response_length": 983, + "processing.sources_count": 5, + "output_response": "Final Answer: The provided context does not specifically mention any new features in cloud computing as a general topic. However, it does highlight several recent enhancements and features related to the Noveum.ai ...", + "final_answer_mode": "RAG", + "query_processed.input_query": "What are the newest features in cloud computing?", + "query_processed.output_response": "Successfully processed query using RAG, generated 983 character response", + "query_processed.mode": "RAG", + "query_processed.processing_time": 5.64110541343689, + "query_processed.response_length": 983 + }, + "events": [], + "links": [], + "trace_trace_id": "77b81bd5-5a04-4fc0-9fc6-571f7907d6b1", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:44.888000000", + "trace_end_time": "2025-10-16 08:50:50.529000000", + "trace_duration_ms": 5641.226, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:44.888000000", + "updated_at": "2025-10-16 08:50:44.888000000" + }, + { + "span_id": "f5538bff-8908-4a56-a622-4ba2eeedfd29", + "trace_id": "77b81bd5-5a04-4fc0-9fc6-571f7907d6b1", + "parent_span_id": "888b4441-3678-42a5-a752-8daae2433482", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-16 08:50:44.888000000", + "end_time": "2025-10-16 08:50:44.888000000", + "duration_ms": 0.032, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What are the newest features in cloud computing?", + "query_length": 48, + "query_type": "routing_evaluation_query", + "classification.mode": "RAG", + "classification.rag_keyword_score": 1, + "classification.web_keyword_score": 0, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "documentation", + "query.keyword_density": 0.125, + "query.domain_affinity": "noveum", + "routing.decision": "Routed to RAG based on analysis", + "routing.rationale": "RAG score: 1, Web score: 0, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "Web Search", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.019230769230769232, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: RAG (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "77b81bd5-5a04-4fc0-9fc6-571f7907d6b1", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:44.888000000", + "trace_end_time": "2025-10-16 08:50:50.529000000", + "trace_duration_ms": 5641.226, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:44.888000000", + "updated_at": "2025-10-16 08:50:44.888000000" + }, + { + "span_id": "856e51cf-fa4d-459b-a046-1ea7316b903e", + "trace_id": "77b81bd5-5a04-4fc0-9fc6-571f7907d6b1", + "parent_span_id": "b6bae768-93a6-47d9-8369-61897f9d7828", + "name": "agent.llm_model_execution", + "start_time": "2025-10-16 08:50:45.423000000", + "end_time": "2025-10-16 08:50:49.997000000", + "duration_ms": 4573.782999999999, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What are the newest features in cloud computing?", + "query_length": 48, + "query_type": "rag_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 4.573714733123779, + "model.latency_ms": 4573.714733123779, + "model.start_time": 1760604645.423261, + "model.end_time": 1760604649.9969757, + "model.performance_tier": "medium", + "model.prompt_tokens": 780, + "model.completion_tokens": 245, + "model.total_tokens": 1025, + "model.tokens_per_second": 224.1066747291299, + "model.estimated_cost": 0.03075, + "model.efficiency_score": 0.9590243902439024, + "model.response_length": 983, + "model.response_quality": "high", + "model.output_response": "Model Response: The provided context does not specifically mention any new features in cloud computing as a general topic. However, it does highlight several recent enhancements and features related to the Noveum.ai ...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "77b81bd5-5a04-4fc0-9fc6-571f7907d6b1", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:44.888000000", + "trace_end_time": "2025-10-16 08:50:50.529000000", + "trace_duration_ms": 5641.226, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:44.888000000", + "updated_at": "2025-10-16 08:50:44.888000000" + }, + { + "span_id": "a2fa7400-74c4-4718-a562-899b91503c95", + "trace_id": "77b81bd5-5a04-4fc0-9fc6-571f7907d6b1", + "parent_span_id": "b6bae768-93a6-47d9-8369-61897f9d7828", + "name": "agent.rag_evaluation_metrics", + "start_time": "2025-10-16 08:50:49.997000000", + "end_time": "2025-10-16 08:50:49.997000000", + "duration_ms": 0.048, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "rag_evaluation_metrics", + "agent.capabilities": [ + "retrieval_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What are the newest features in cloud computing?", + "query_length": 48, + "query_type": "rag_evaluation_query", + "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/changelog):\n- \ud83d\udcc8 Enhanced Dashboard Analytics: Real-time request tracking, latency monitoring, and cost analysis\n- \ud83d\udd0d Improved Logs Interface: Better search functionality, detailed trace views, and enhanced debugging capabilities\n- \ud83d\udc65 Team Management: Advanced member mana...", + "retrieval.context_length": 2579, + "retrieval.sources_count": 5, + "retrieval.context_quality": "high", + "retrieval.effectiveness": 1, + "retrieval.context_utilization": 2.579, + "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/changelog):\n- \ud83d\udcc8 Enhanced Dashboard Analytics: Real-time request tracking, latency monitoring, and cost analysis\n- \ud83d\udd0d Improved Logs Interface: Better search functionality, detailed trace views, and enhanced debugging capabilities\n- \ud83d\udc65 Team Management: Advanced member management with role-based access control and invitation system\n- \ud83d\udee1\ufe0f Security Improvements: Enhanced API key management and secure credential storage\n4 months ago\n- \ud83d\udcca Advanced Metrics Collection: Comprehensive telemetry system with custom metrics ...\n\nSource 2 (https://noveum.ai/en/changelog):\n- \ud83d\udcdd Documentation Updates: Comprehensive guides for integration and best practices\n5 months ago\n- \ud83c\udf89 Platform Foundation: Initial release of Noveum.ai AI observability platform\n- \ud83d\ude80 Core Features: AI Gateway integration, basic evaluation metrics, and monitoring capabilities\n- \ud83d\udcc8 Evaluation Jobs: Automated model evaluation with configurable metrics and reporting\n- \ud83d\udd0c Provider Integrations: Support for major AI providers and custom model deployments...\n\nSource 3 (https://noveum.ai/docs/platform/dashboard):\n- Bulk Export: Download multiple traces simultaneously\n- Comparative Analysis: Compare performance across multiple traces\nReal-Time Updates\n- Live Refresh: Automatic updates as new traces arrive\n- Connection Monitoring: Real-time status of your trace ingestion pipeline\n- Performance Indicators: Live metrics for system health monitoring\nReady to dive deeper? Explore Projects & Environments to organize your AI applications, or check out Team Collaboration to share insights with your team.\nGet Earl...\n\nSource 4 (https://noveum.ai/en/changelog):\nChangelog\nStay up to date with the latest changes in our product.\n2 months ago\n- \ud83c\udfaf Major Platform Evolution: NovaEval Framework - Launched comprehensive AI model evaluation framework with 20+ built-in scorers including accuracy, RAG metrics, conversational scoring, and specialized agent evaluation tools. Features production-grade deployment on Docker and Kubernetes. (GitHub | PyPI Package)\n- \ud83d\ude80 Next-Generation Traces UI - Complete redesign featuring three-pane layout with directory tree navigatio...\n\nSource 5 (https://noveum.ai/en/blog):\n8/12/2025\nDiscover how Noveum.ai provides comprehensive tracing and observability for AI applications, from development debugging to production optimization.\nShashank Agarwal\n3/3/2025\nDiscover how Noveum.ai provides comprehensive tracing and observability for LLM applications, RAG systems, and multi-agent workflows with our powerful Python and TypeScript SDKs.\nShashank Agarwal\n3/2/2025...\n\nUser Question: What are the newest features in cloud computing?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", + "prompt.prompt_length": 3123, + "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/changelog):\n- \ud83d\udcc8 Enhanced Dashboard Analytics: Real-time request tracking, latency monitoring, and cost analysis\n- \ud83d\udd0d Improved Logs Interface: Better search functionality,...", + "prompt.instruction_following": "rag_optimized", + "response.answer_length": 983, + "response.answer_completeness": "complete", + "response.response_quality": "high", + "response.source_citation": 5, + "response.context_utilization": 0.381155486622722, + "output_response": "RAG Answer: The provided context does not specifically mention any new features in cloud computing as a general topic. However, it does highlight several recent enhancements and features related to the Noveum.ai ...", + "evaluation.retrieval_effectiveness": 1, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 5, + "evaluation.context_relevance": "high", + "evaluation.overall_quality": "high", + "evaluation.ready_for_production": true, + "rag.retrieval_strategy": "semantic_similarity", + "rag.vector_search_results": 5, + "rag.context_synthesis": "multi_source", + "rag.document_coverage": 1, + "rag.information_density": 0.381155486622722 + }, + "events": [], + "links": [], + "trace_trace_id": "77b81bd5-5a04-4fc0-9fc6-571f7907d6b1", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:44.888000000", + "trace_end_time": "2025-10-16 08:50:50.529000000", + "trace_duration_ms": 5641.226, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:44.888000000", + "updated_at": "2025-10-16 08:50:44.888000000" + }, + { + "span_id": "4c391bb1-1a62-4e2a-913b-96f83feff494", + "trace_id": "32170d1a-d211-46d8-9cdc-5a4265f0c443", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-16 08:50:42.124000000", + "end_time": "2025-10-16 08:50:43.885000000", + "duration_ms": 1760.974, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What's the current status of cryptocurrency markets?", + "query.length": 52, + "processing.mode": "Web Search", + "processing.time_seconds": 1.7608659267425537, + "processing.response_length": 363, + "processing.sources_count": 1, + "output_response": "Final Answer: The web search did not yield any specific information regarding the current status of cryptocurrency markets. Therefore, I cannot provide an update on market trends, prices, or significant events affe...", + "final_answer_mode": "Web Search", + "query_processed.input_query": "What's the current status of cryptocurrency markets?", + "query_processed.output_response": "Successfully processed query using Web Search, generated 363 character response", + "query_processed.mode": "Web Search", + "query_processed.processing_time": 1.7608659267425537, + "query_processed.response_length": 363 + }, + "events": [], + "links": [], + "trace_trace_id": "32170d1a-d211-46d8-9cdc-5a4265f0c443", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:42.124000000", + "trace_end_time": "2025-10-16 08:50:43.885000000", + "trace_duration_ms": 1761.047, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:42.124000000", + "updated_at": "2025-10-16 08:50:42.124000000" + }, + { + "span_id": "7ab2ec7b-a617-4ebf-9bb1-82dc6b18e160", + "trace_id": "32170d1a-d211-46d8-9cdc-5a4265f0c443", + "parent_span_id": "4c391bb1-1a62-4e2a-913b-96f83feff494", + "name": "agent.query_routing", + "start_time": "2025-10-16 08:50:42.124000000", + "end_time": "2025-10-16 08:50:43.885000000", + "duration_ms": 1760.833, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What's the current status of cryptocurrency markets?", + "query_length": 52, + "query_type": "routing_query", + "output_response": "Routed to Web Search for query processing", + "router.classification": "Web Search", + "router.confidence_score": 1, + "router.rag_keyword_score": 0, + "router.web_keyword_score": 3, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "32170d1a-d211-46d8-9cdc-5a4265f0c443", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:42.124000000", + "trace_end_time": "2025-10-16 08:50:43.885000000", + "trace_duration_ms": 1761.047, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:42.124000000", + "updated_at": "2025-10-16 08:50:42.124000000" + }, + { + "span_id": "db01d228-7c63-4814-b2ea-bdf3898f9c0d", + "trace_id": "32170d1a-d211-46d8-9cdc-5a4265f0c443", + "parent_span_id": "7ab2ec7b-a617-4ebf-9bb1-82dc6b18e160", + "name": "agent.web_search_generation", + "start_time": "2025-10-16 08:50:42.124000000", + "end_time": "2025-10-16 08:50:43.885000000", + "duration_ms": 1760.701, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "web_search_agent", + "agent.operation": "web_search_generation", + "agent.capabilities": [ + "web_search", + "content_synthesis", + "response_generation" + ], + "agent.id": "noveum_web_search_agent", + "input_query": "What's the current status of cryptocurrency markets?", + "query_length": 52, + "query_type": "web_search_query", + "output_response": "Web Search Answer: The web search did not yield any specific information regarding the current status of cryptocurrency markets. Therefore, I cannot provide an update on market trends, prices, or significant events affe...", + "web_search.results_count": 1, + "web_search.context_length": 154, + "web_search.response_length": 363, + "web_search.mode": "external_web_search" + }, + "events": [], + "links": [], + "trace_trace_id": "32170d1a-d211-46d8-9cdc-5a4265f0c443", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:42.124000000", + "trace_end_time": "2025-10-16 08:50:43.885000000", + "trace_duration_ms": 1761.047, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:42.124000000", + "updated_at": "2025-10-16 08:50:42.124000000" + }, + { + "span_id": "da7b0550-b13c-4518-ad3b-050c666d198c", + "trace_id": "32170d1a-d211-46d8-9cdc-5a4265f0c443", + "parent_span_id": "7ab2ec7b-a617-4ebf-9bb1-82dc6b18e160", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-16 08:50:42.124000000", + "end_time": "2025-10-16 08:50:42.124000000", + "duration_ms": 0.029, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What's the current status of cryptocurrency markets?", + "query_length": 52, + "query_type": "routing_evaluation_query", + "classification.mode": "Web Search", + "classification.rag_keyword_score": 0, + "classification.web_keyword_score": 3, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "complex", + "query.intent": "general_knowledge", + "query.keyword_density": 0.42857142857142855, + "query.domain_affinity": "general", + "routing.decision": "Routed to Web Search based on analysis", + "routing.rationale": "RAG score: 0, Web score: 3, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "RAG", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.057692307692307696, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: Web Search (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "32170d1a-d211-46d8-9cdc-5a4265f0c443", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:42.124000000", + "trace_end_time": "2025-10-16 08:50:43.885000000", + "trace_duration_ms": 1761.047, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:42.124000000", + "updated_at": "2025-10-16 08:50:42.124000000" + }, + { + "span_id": "950e18b6-9f62-4ac4-8788-3cac08fb7899", + "trace_id": "32170d1a-d211-46d8-9cdc-5a4265f0c443", + "parent_span_id": "db01d228-7c63-4814-b2ea-bdf3898f9c0d", + "name": "agent.llm_model_execution", + "start_time": "2025-10-16 08:50:42.465000000", + "end_time": "2025-10-16 08:50:43.885000000", + "duration_ms": 1419.5690000000002, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What's the current status of cryptocurrency markets?", + "query_length": 52, + "query_type": "web_search_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 1.4195220470428467, + "model.latency_ms": 1419.5220470428467, + "model.start_time": 1760604642.4658613, + "model.end_time": 1760604643.8853834, + "model.performance_tier": "fast", + "model.prompt_tokens": 162, + "model.completion_tokens": 90, + "model.total_tokens": 252, + "model.tokens_per_second": 177.52454111224782, + "model.estimated_cost": 0.00756, + "model.efficiency_score": 1.4404761904761905, + "model.response_length": 363, + "model.response_quality": "high", + "model.output_response": "Model Response: The web search did not yield any specific information regarding the current status of cryptocurrency markets. Therefore, I cannot provide an update on market trends, prices, or significant events affe...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "32170d1a-d211-46d8-9cdc-5a4265f0c443", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:42.124000000", + "trace_end_time": "2025-10-16 08:50:43.885000000", + "trace_duration_ms": 1761.047, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:42.124000000", + "updated_at": "2025-10-16 08:50:42.124000000" + }, + { + "span_id": "e87846c9-ce82-42c2-a517-e0f67e9d14c1", + "trace_id": "32170d1a-d211-46d8-9cdc-5a4265f0c443", + "parent_span_id": "db01d228-7c63-4814-b2ea-bdf3898f9c0d", + "name": "agent.web_search_evaluation_metrics", + "start_time": "2025-10-16 08:50:43.885000000", + "end_time": "2025-10-16 08:50:43.885000000", + "duration_ms": 0.02, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "web_search_evaluation_metrics", + "agent.capabilities": [ + "web_search_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What's the current status of cryptocurrency markets?", + "query_length": 52, + "query_type": "web_search_evaluation_query", + "web_search.results_count": 1, + "web_search.context_length": 154, + "web_search.context_synthesized": "Context from 1 web sources", + "web_search.search_effectiveness": 0.2, + "web_search.context_quality": "low", + "web_search.source_diversity": 1, + "web_search.information_synthesis": "low", + "web_search.external_knowledge_utilization": 0.154, + "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What's+the+current+status+of+cryptocurrency+markets?\n\nUser Question: What's the current status of cryptocurrency markets?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", + "prompt.prompt_length": 651, + "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What's+the+current+status+of+cryptocurrency+markets?", + "prompt.instruction_following": "web_search_optimized", + "response.answer_length": 363, + "response.answer_completeness": "complete", + "response.response_quality": "medium", + "response.source_citation": 1, + "response.context_utilization": 2.357142857142857, + "output_response": "Web Search Answer: The web search did not yield any specific information regarding the current status of cryptocurrency markets. Therefore, I cannot provide an update on market trends, prices, or significant events affe...", + "evaluation.search_effectiveness": 0.2, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 1, + "evaluation.context_relevance": "low", + "evaluation.overall_quality": "low", + "evaluation.ready_for_production": true, + "web_search.search_strategy": "duckduckgo_api", + "web_search.real_time_data": true, + "web_search.external_sources": 1, + "web_search.information_freshness": "current", + "web_search.knowledge_synthesis": "single_source" + }, + "events": [], + "links": [], + "trace_trace_id": "32170d1a-d211-46d8-9cdc-5a4265f0c443", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:42.124000000", + "trace_end_time": "2025-10-16 08:50:43.885000000", + "trace_duration_ms": 1761.047, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:42.124000000", + "updated_at": "2025-10-16 08:50:42.124000000" + }, + { + "span_id": "305daacf-990b-4e23-8720-48900f75d9be", + "trace_id": "9424d420-deee-478b-b727-9dc791a7457c", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-16 08:50:31.476000000", + "end_time": "2025-10-16 08:50:41.121000000", + "duration_ms": 9644.752999999999, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What are the latest updates in Python programming?", + "query.length": 50, + "processing.mode": "Web Search", + "processing.time_seconds": 9.644662618637085, + "processing.response_length": 1408, + "processing.sources_count": 1, + "output_response": "Final Answer: The web search did not yield specific results regarding the latest updates in Python programming. However, I can provide some general information based on my knowledge up to October 2023.\n\nAs of late ...", + "final_answer_mode": "Web Search", + "query_processed.input_query": "What are the latest updates in Python programming?", + "query_processed.output_response": "Successfully processed query using Web Search, generated 1408 character response", + "query_processed.mode": "Web Search", + "query_processed.processing_time": 9.644662618637085, + "query_processed.response_length": 1408 + }, + "events": [], + "links": [], + "trace_trace_id": "9424d420-deee-478b-b727-9dc791a7457c", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:31.476000000", + "trace_end_time": "2025-10-16 08:50:41.121000000", + "trace_duration_ms": 9644.828, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:31.476000000", + "updated_at": "2025-10-16 08:50:31.476000000" + }, + { + "span_id": "56b4cc1b-0646-4dee-acb3-db98f3d44477", + "trace_id": "9424d420-deee-478b-b727-9dc791a7457c", + "parent_span_id": "fb234e30-69be-4d06-9248-c05bcc22f063", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-16 08:50:31.476000000", + "end_time": "2025-10-16 08:50:31.476000000", + "duration_ms": 0.036000000000000004, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What are the latest updates in Python programming?", + "query_length": 50, + "query_type": "routing_evaluation_query", + "classification.mode": "Web Search", + "classification.rag_keyword_score": 0, + "classification.web_keyword_score": 2, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "general_knowledge", + "query.keyword_density": 0.25, + "query.domain_affinity": "general", + "routing.decision": "Routed to Web Search based on analysis", + "routing.rationale": "RAG score: 0, Web score: 2, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "RAG", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.038461538461538464, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: Web Search (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "9424d420-deee-478b-b727-9dc791a7457c", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:31.476000000", + "trace_end_time": "2025-10-16 08:50:41.121000000", + "trace_duration_ms": 9644.828, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:31.476000000", + "updated_at": "2025-10-16 08:50:31.476000000" + }, + { + "span_id": "fb234e30-69be-4d06-9248-c05bcc22f063", + "trace_id": "9424d420-deee-478b-b727-9dc791a7457c", + "parent_span_id": "305daacf-990b-4e23-8720-48900f75d9be", + "name": "agent.query_routing", + "start_time": "2025-10-16 08:50:31.476000000", + "end_time": "2025-10-16 08:50:41.121000000", + "duration_ms": 9644.627, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What are the latest updates in Python programming?", + "query_length": 50, + "query_type": "routing_query", + "output_response": "Routed to Web Search for query processing", + "router.classification": "Web Search", + "router.confidence_score": 1, + "router.rag_keyword_score": 0, + "router.web_keyword_score": 2, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "9424d420-deee-478b-b727-9dc791a7457c", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:31.476000000", + "trace_end_time": "2025-10-16 08:50:41.121000000", + "trace_duration_ms": 9644.828, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:31.476000000", + "updated_at": "2025-10-16 08:50:31.476000000" + }, + { + "span_id": "f8a793a9-63ee-49b7-9931-d7dcc2bb0ca7", + "trace_id": "9424d420-deee-478b-b727-9dc791a7457c", + "parent_span_id": "fb234e30-69be-4d06-9248-c05bcc22f063", + "name": "agent.web_search_generation", + "start_time": "2025-10-16 08:50:31.477000000", + "end_time": "2025-10-16 08:50:41.121000000", + "duration_ms": 9644.478, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "web_search_agent", + "agent.operation": "web_search_generation", + "agent.capabilities": [ + "web_search", + "content_synthesis", + "response_generation" + ], + "agent.id": "noveum_web_search_agent", + "input_query": "What are the latest updates in Python programming?", + "query_length": 50, + "query_type": "web_search_query", + "output_response": "Web Search Answer: The web search did not yield specific results regarding the latest updates in Python programming. However, I can provide some general information based on my knowledge up to October 2023.\n\nAs of late ...", + "web_search.results_count": 1, + "web_search.context_length": 152, + "web_search.response_length": 1408, + "web_search.mode": "external_web_search" + }, + "events": [], + "links": [], + "trace_trace_id": "9424d420-deee-478b-b727-9dc791a7457c", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:31.476000000", + "trace_end_time": "2025-10-16 08:50:41.121000000", + "trace_duration_ms": 9644.828, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:31.476000000", + "updated_at": "2025-10-16 08:50:31.476000000" + }, + { + "span_id": "b968e3d3-d4aa-4357-85d8-34a612748fbd", + "trace_id": "9424d420-deee-478b-b727-9dc791a7457c", + "parent_span_id": "f8a793a9-63ee-49b7-9931-d7dcc2bb0ca7", + "name": "agent.llm_model_execution", + "start_time": "2025-10-16 08:50:32.016000000", + "end_time": "2025-10-16 08:50:41.121000000", + "duration_ms": 9105.344000000001, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What are the latest updates in Python programming?", + "query_length": 50, + "query_type": "web_search_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 9.105276584625244, + "model.latency_ms": 9105.276584625244, + "model.start_time": 1760604632.0161095, + "model.end_time": 1760604641.121386, + "model.performance_tier": "slow", + "model.prompt_tokens": 161, + "model.completion_tokens": 352, + "model.total_tokens": 513, + "model.tokens_per_second": 56.340957381374714, + "model.estimated_cost": 0.015390000000000001, + "model.efficiency_score": 2.7446393762183234, + "model.response_length": 1408, + "model.response_quality": "high", + "model.output_response": "Model Response: The web search did not yield specific results regarding the latest updates in Python programming. However, I can provide some general information based on my knowledge up to October 2023.\n\nAs of late ...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "9424d420-deee-478b-b727-9dc791a7457c", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:31.476000000", + "trace_end_time": "2025-10-16 08:50:41.121000000", + "trace_duration_ms": 9644.828, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:31.476000000", + "updated_at": "2025-10-16 08:50:31.476000000" + }, + { + "span_id": "80817711-e64c-4a9f-802e-2a9e36ff093d", + "trace_id": "9424d420-deee-478b-b727-9dc791a7457c", + "parent_span_id": "f8a793a9-63ee-49b7-9931-d7dcc2bb0ca7", + "name": "agent.web_search_evaluation_metrics", + "start_time": "2025-10-16 08:50:41.121000000", + "end_time": "2025-10-16 08:50:41.121000000", + "duration_ms": 0.02, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "web_search_evaluation_metrics", + "agent.capabilities": [ + "web_search_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What are the latest updates in Python programming?", + "query_length": 50, + "query_type": "web_search_evaluation_query", + "web_search.results_count": 1, + "web_search.context_length": 152, + "web_search.context_synthesized": "Context from 1 web sources", + "web_search.search_effectiveness": 0.2, + "web_search.context_quality": "low", + "web_search.source_diversity": 1, + "web_search.information_synthesis": "low", + "web_search.external_knowledge_utilization": 0.152, + "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What+are+the+latest+updates+in+Python+programming?\n\nUser Question: What are the latest updates in Python programming?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", + "prompt.prompt_length": 647, + "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What+are+the+latest+updates+in+Python+programming?", + "prompt.instruction_following": "web_search_optimized", + "response.answer_length": 1408, + "response.answer_completeness": "complete", + "response.response_quality": "medium", + "response.source_citation": 1, + "response.context_utilization": 9.263157894736842, + "output_response": "Web Search Answer: The web search did not yield specific results regarding the latest updates in Python programming. However, I can provide some general information based on my knowledge up to October 2023.\n\nAs of late ...", + "evaluation.search_effectiveness": 0.2, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 1, + "evaluation.context_relevance": "low", + "evaluation.overall_quality": "low", + "evaluation.ready_for_production": true, + "web_search.search_strategy": "duckduckgo_api", + "web_search.real_time_data": true, + "web_search.external_sources": 1, + "web_search.information_freshness": "current", + "web_search.knowledge_synthesis": "single_source" + }, + "events": [], + "links": [], + "trace_trace_id": "9424d420-deee-478b-b727-9dc791a7457c", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:31.476000000", + "trace_end_time": "2025-10-16 08:50:41.121000000", + "trace_duration_ms": 9644.828, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:31.476000000", + "updated_at": "2025-10-16 08:50:31.476000000" + }, + { + "span_id": "13af0f3c-bd5c-4024-8fa3-c9e347a1ef20", + "trace_id": "a3735d64-89bb-4fdf-a851-7a1f3515c52d", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-16 08:50:27.687000000", + "end_time": "2025-10-16 08:50:30.473000000", + "duration_ms": 2786.078, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What happened in tech news this week?", + "query.length": 37, + "processing.mode": "Web Search", + "processing.time_seconds": 2.7859997749328613, + "processing.response_length": 396, + "processing.sources_count": 1, + "output_response": "Final Answer: It appears that the web search did not yield any specific results regarding recent tech news. Unfortunately, I cannot provide a summary of this week's tech news based on the available information. For...", + "final_answer_mode": "Web Search", + "query_processed.input_query": "What happened in tech news this week?", + "query_processed.output_response": "Successfully processed query using Web Search, generated 396 character response", + "query_processed.mode": "Web Search", + "query_processed.processing_time": 2.7859997749328613, + "query_processed.response_length": 396 + }, + "events": [], + "links": [], + "trace_trace_id": "a3735d64-89bb-4fdf-a851-7a1f3515c52d", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:27.687000000", + "trace_end_time": "2025-10-16 08:50:30.473000000", + "trace_duration_ms": 2786.145, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:27.687000000", + "updated_at": "2025-10-16 08:50:27.687000000" + }, + { + "span_id": "47e8f375-35e6-4097-86ff-df73e8181c23", + "trace_id": "a3735d64-89bb-4fdf-a851-7a1f3515c52d", + "parent_span_id": "a4f1a7b3-dc35-4f13-96d3-b6dc8fb21a37", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-16 08:50:27.687000000", + "end_time": "2025-10-16 08:50:27.687000000", + "duration_ms": 0.038, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What happened in tech news this week?", + "query_length": 37, + "query_type": "routing_evaluation_query", + "classification.mode": "Web Search", + "classification.rag_keyword_score": 0, + "classification.web_keyword_score": 2, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "general_knowledge", + "query.keyword_density": 0.2857142857142857, + "query.domain_affinity": "general", + "routing.decision": "Routed to Web Search based on analysis", + "routing.rationale": "RAG score: 0, Web score: 2, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "RAG", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.038461538461538464, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: Web Search (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "a3735d64-89bb-4fdf-a851-7a1f3515c52d", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:27.687000000", + "trace_end_time": "2025-10-16 08:50:30.473000000", + "trace_duration_ms": 2786.145, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:27.687000000", + "updated_at": "2025-10-16 08:50:27.687000000" + }, + { + "span_id": "a4f1a7b3-dc35-4f13-96d3-b6dc8fb21a37", + "trace_id": "a3735d64-89bb-4fdf-a851-7a1f3515c52d", + "parent_span_id": "13af0f3c-bd5c-4024-8fa3-c9e347a1ef20", + "name": "agent.query_routing", + "start_time": "2025-10-16 08:50:27.687000000", + "end_time": "2025-10-16 08:50:30.473000000", + "duration_ms": 2785.964, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What happened in tech news this week?", + "query_length": 37, + "query_type": "routing_query", + "output_response": "Routed to Web Search for query processing", + "router.classification": "Web Search", + "router.confidence_score": 1, + "router.rag_keyword_score": 0, + "router.web_keyword_score": 2, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "a3735d64-89bb-4fdf-a851-7a1f3515c52d", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:27.687000000", + "trace_end_time": "2025-10-16 08:50:30.473000000", + "trace_duration_ms": 2786.145, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:27.687000000", + "updated_at": "2025-10-16 08:50:27.687000000" + }, + { + "span_id": "ebfff280-5eb6-4838-8de7-35c85e2e3eaa", + "trace_id": "a3735d64-89bb-4fdf-a851-7a1f3515c52d", + "parent_span_id": "a4f1a7b3-dc35-4f13-96d3-b6dc8fb21a37", + "name": "agent.web_search_generation", + "start_time": "2025-10-16 08:50:27.687000000", + "end_time": "2025-10-16 08:50:30.473000000", + "duration_ms": 2785.8230000000003, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "web_search_agent", + "agent.operation": "web_search_generation", + "agent.capabilities": [ + "web_search", + "content_synthesis", + "response_generation" + ], + "agent.id": "noveum_web_search_agent", + "input_query": "What happened in tech news this week?", + "query_length": 37, + "query_type": "web_search_query", + "output_response": "Web Search Answer: It appears that the web search did not yield any specific results regarding recent tech news. Unfortunately, I cannot provide a summary of this week's tech news based on the available information. For...", + "web_search.results_count": 1, + "web_search.context_length": 139, + "web_search.response_length": 396, + "web_search.mode": "external_web_search" + }, + "events": [], + "links": [], + "trace_trace_id": "a3735d64-89bb-4fdf-a851-7a1f3515c52d", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:27.687000000", + "trace_end_time": "2025-10-16 08:50:30.473000000", + "trace_duration_ms": 2786.145, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:27.687000000", + "updated_at": "2025-10-16 08:50:27.687000000" + }, + { + "span_id": "835b9351-d381-4785-8538-fbe1064fc8cd", + "trace_id": "a3735d64-89bb-4fdf-a851-7a1f3515c52d", + "parent_span_id": "ebfff280-5eb6-4838-8de7-35c85e2e3eaa", + "name": "agent.llm_model_execution", + "start_time": "2025-10-16 08:50:28.026000000", + "end_time": "2025-10-16 08:50:30.473000000", + "duration_ms": 2447.269, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What happened in tech news this week?", + "query_length": 37, + "query_type": "web_search_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 2.447227716445923, + "model.latency_ms": 2447.227716445923, + "model.start_time": 1760604628.0260897, + "model.end_time": 1760604630.4733174, + "model.performance_tier": "medium", + "model.prompt_tokens": 155, + "model.completion_tokens": 99, + "model.total_tokens": 254, + "model.tokens_per_second": 103.79091340501853, + "model.estimated_cost": 0.00762, + "model.efficiency_score": 1.5590551181102361, + "model.response_length": 396, + "model.response_quality": "high", + "model.output_response": "Model Response: It appears that the web search did not yield any specific results regarding recent tech news. Unfortunately, I cannot provide a summary of this week's tech news based on the available information. For...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "a3735d64-89bb-4fdf-a851-7a1f3515c52d", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:27.687000000", + "trace_end_time": "2025-10-16 08:50:30.473000000", + "trace_duration_ms": 2786.145, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:27.687000000", + "updated_at": "2025-10-16 08:50:27.687000000" + }, + { + "span_id": "93c6f427-6fcd-4318-aec8-095553fed469", + "trace_id": "a3735d64-89bb-4fdf-a851-7a1f3515c52d", + "parent_span_id": "ebfff280-5eb6-4838-8de7-35c85e2e3eaa", + "name": "agent.web_search_evaluation_metrics", + "start_time": "2025-10-16 08:50:30.473000000", + "end_time": "2025-10-16 08:50:30.473000000", + "duration_ms": 0.017, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "web_search_evaluation_metrics", + "agent.capabilities": [ + "web_search_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What happened in tech news this week?", + "query_length": 37, + "query_type": "web_search_evaluation_query", + "web_search.results_count": 1, + "web_search.context_length": 139, + "web_search.context_synthesized": "Context from 1 web sources", + "web_search.search_effectiveness": 0.2, + "web_search.context_quality": "low", + "web_search.source_diversity": 1, + "web_search.information_synthesis": "low", + "web_search.external_knowledge_utilization": 0.139, + "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What+happened+in+tech+news+this+week?\n\nUser Question: What happened in tech news this week?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", + "prompt.prompt_length": 621, + "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What+happened+in+tech+news+this+week?", + "prompt.instruction_following": "web_search_optimized", + "response.answer_length": 396, + "response.answer_completeness": "complete", + "response.response_quality": "medium", + "response.source_citation": 1, + "response.context_utilization": 2.8489208633093526, + "output_response": "Web Search Answer: It appears that the web search did not yield any specific results regarding recent tech news. Unfortunately, I cannot provide a summary of this week's tech news based on the available information. For...", + "evaluation.search_effectiveness": 0.2, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 1, + "evaluation.context_relevance": "low", + "evaluation.overall_quality": "low", + "evaluation.ready_for_production": true, + "web_search.search_strategy": "duckduckgo_api", + "web_search.real_time_data": true, + "web_search.external_sources": 1, + "web_search.information_freshness": "current", + "web_search.knowledge_synthesis": "single_source" + }, + "events": [], + "links": [], + "trace_trace_id": "a3735d64-89bb-4fdf-a851-7a1f3515c52d", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:27.687000000", + "trace_end_time": "2025-10-16 08:50:30.473000000", + "trace_duration_ms": 2786.145, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:27.687000000", + "updated_at": "2025-10-16 08:50:27.687000000" + }, + { + "span_id": "622e7353-4eeb-47d5-8558-4ecec28cdcfd", + "trace_id": "fa5bb641-f4b7-4741-b06c-392a5f2a9aea", + "parent_span_id": "c0af6629-48b1-45fc-9dd2-60f098ea140e", + "name": "agent.query_routing", + "start_time": "2025-10-16 08:50:21.645000000", + "end_time": "2025-10-16 08:50:26.683000000", + "duration_ms": 5037.965, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What are the current trends in observability tools?", + "query_length": 51, + "query_type": "routing_query", + "output_response": "Routed to RAG for query processing", + "router.classification": "RAG", + "router.confidence_score": 0.3333333333333333, + "router.rag_keyword_score": 2, + "router.web_keyword_score": 1, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "fa5bb641-f4b7-4741-b06c-392a5f2a9aea", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:21.645000000", + "trace_end_time": "2025-10-16 08:50:26.683000000", + "trace_duration_ms": 5038.112, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:21.645000000", + "updated_at": "2025-10-16 08:50:21.645000000" + }, + { + "span_id": "c0af6629-48b1-45fc-9dd2-60f098ea140e", + "trace_id": "fa5bb641-f4b7-4741-b06c-392a5f2a9aea", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-16 08:50:21.645000000", + "end_time": "2025-10-16 08:50:26.683000000", + "duration_ms": 5038.0419999999995, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What are the current trends in observability tools?", + "query.length": 51, + "processing.mode": "RAG", + "processing.time_seconds": 5.0380027294158936, + "processing.response_length": 897, + "processing.sources_count": 5, + "output_response": "Final Answer: The provided context does not specifically outline current trends in observability tools beyond the focus on AI applications. However, it highlights the limitations of traditional monitoring tools for...", + "final_answer_mode": "RAG", + "query_processed.input_query": "What are the current trends in observability tools?", + "query_processed.output_response": "Successfully processed query using RAG, generated 897 character response", + "query_processed.mode": "RAG", + "query_processed.processing_time": 5.0380027294158936, + "query_processed.response_length": 897 + }, + "events": [], + "links": [], + "trace_trace_id": "fa5bb641-f4b7-4741-b06c-392a5f2a9aea", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:21.645000000", + "trace_end_time": "2025-10-16 08:50:26.683000000", + "trace_duration_ms": 5038.112, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:21.645000000", + "updated_at": "2025-10-16 08:50:21.645000000" + }, + { + "span_id": "ee55906d-ffe3-42fd-9899-a476d4c33ff5", + "trace_id": "fa5bb641-f4b7-4741-b06c-392a5f2a9aea", + "parent_span_id": "622e7353-4eeb-47d5-8558-4ecec28cdcfd", + "name": "agent.llm-rag", + "start_time": "2025-10-16 08:50:21.645000000", + "end_time": "2025-10-16 08:50:26.683000000", + "duration_ms": 5037.803, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "rag_agent", + "agent.operation": "llm-rag", + "agent.capabilities": [ + "document_retrieval", + "context_generation", + "response_generation" + ], + "agent.id": "noveum_rag_agent", + "input_query": "What are the current trends in observability tools?", + "query_length": 51, + "query_type": "rag_query", + "output_response": "RAG Answer: The provided context does not specifically outline current trends in observability tools beyond the focus on AI applications. However, it highlights the limitations of traditional monitoring tools for...", + "rag.context_length": 2943, + "rag.sources_count": 5, + "rag.answer_length": 897, + "rag.mode": "retrieval_augmented_generation" + }, + "events": [], + "links": [], + "trace_trace_id": "fa5bb641-f4b7-4741-b06c-392a5f2a9aea", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:21.645000000", + "trace_end_time": "2025-10-16 08:50:26.683000000", + "trace_duration_ms": 5038.112, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:21.645000000", + "updated_at": "2025-10-16 08:50:21.645000000" + }, + { + "span_id": "fe610ecb-cc8e-4466-b719-34c6e0d45d87", + "trace_id": "fa5bb641-f4b7-4741-b06c-392a5f2a9aea", + "parent_span_id": "622e7353-4eeb-47d5-8558-4ecec28cdcfd", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-16 08:50:21.645000000", + "end_time": "2025-10-16 08:50:21.645000000", + "duration_ms": 0.032, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What are the current trends in observability tools?", + "query_length": 51, + "query_type": "routing_evaluation_query", + "classification.mode": "RAG", + "classification.rag_keyword_score": 2, + "classification.web_keyword_score": 1, + "classification.confidence_score": 0.3333333333333333, + "classification.confidence_level": "medium", + "classification.method": "keyword_based", + "query.complexity": "complex", + "query.intent": "documentation", + "query.keyword_density": 0.375, + "query.domain_affinity": "noveum", + "routing.decision": "Routed to RAG based on analysis", + "routing.rationale": "RAG score: 2, Web score: 1, Confidence: 0.33", + "routing.expected_performance": "medium", + "routing.alternative_mode": "Web Search", + "routing.decision_confidence": 0.3333333333333333, + "evaluation.routing_accuracy": "medium", + "evaluation.keyword_coverage": 0.057692307692307696, + "evaluation.query_understanding": "ambiguous", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: RAG (Confidence: 0.33)" + }, + "events": [], + "links": [], + "trace_trace_id": "fa5bb641-f4b7-4741-b06c-392a5f2a9aea", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:21.645000000", + "trace_end_time": "2025-10-16 08:50:26.683000000", + "trace_duration_ms": 5038.112, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:21.645000000", + "updated_at": "2025-10-16 08:50:21.645000000" + }, + { + "span_id": "69a4773d-c738-489e-87ed-999583e02187", + "trace_id": "fa5bb641-f4b7-4741-b06c-392a5f2a9aea", + "parent_span_id": "ee55906d-ffe3-42fd-9899-a476d4c33ff5", + "name": "agent.llm_model_execution", + "start_time": "2025-10-16 08:50:22.600000000", + "end_time": "2025-10-16 08:50:25.786000000", + "duration_ms": 3185.257, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What are the current trends in observability tools?", + "query_length": 51, + "query_type": "rag_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 3.185206174850464, + "model.latency_ms": 3185.206174850464, + "model.start_time": 1760604622.6008554, + "model.end_time": 1760604625.7860615, + "model.performance_tier": "medium", + "model.prompt_tokens": 872, + "model.completion_tokens": 224, + "model.total_tokens": 1096, + "model.tokens_per_second": 344.09075577390337, + "model.estimated_cost": 0.03288, + "model.efficiency_score": 0.8184306569343066, + "model.response_length": 897, + "model.response_quality": "high", + "model.output_response": "Model Response: The provided context does not specifically outline current trends in observability tools beyond the focus on AI applications. However, it highlights the limitations of traditional monitoring tools for...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "fa5bb641-f4b7-4741-b06c-392a5f2a9aea", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:21.645000000", + "trace_end_time": "2025-10-16 08:50:26.683000000", + "trace_duration_ms": 5038.112, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:21.645000000", + "updated_at": "2025-10-16 08:50:21.645000000" + }, + { + "span_id": "5e5a9d30-4cd0-45cd-9494-192c280727ac", + "trace_id": "fa5bb641-f4b7-4741-b06c-392a5f2a9aea", + "parent_span_id": "ee55906d-ffe3-42fd-9899-a476d4c33ff5", + "name": "agent.rag_evaluation_metrics", + "start_time": "2025-10-16 08:50:25.786000000", + "end_time": "2025-10-16 08:50:25.786000000", + "duration_ms": 0.066, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "rag_evaluation_metrics", + "agent.capabilities": [ + "retrieval_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What are the current trends in observability tools?", + "query_length": 51, + "query_type": "rag_evaluation_query", + "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/blog/from-logs-to-intelligent-choices-inside-noveum-ais-evaluation-process):\nFrom Development to Production - Inside Noveum.ai's AI Observability Platform\nShashank Agarwal\n3/3/2025\nIntroduction\nEvery AI application tells a story\u2014but most developers never get to hear it...", + "retrieval.context_length": 2943, + "retrieval.sources_count": 5, + "retrieval.context_quality": "high", + "retrieval.effectiveness": 1, + "retrieval.context_utilization": 2.943, + "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/blog/from-logs-to-intelligent-choices-inside-noveum-ais-evaluation-process):\nFrom Development to Production - Inside Noveum.ai's AI Observability Platform\nShashank Agarwal\n3/3/2025\nIntroduction\nEvery AI application tells a story\u2014but most developers never get to hear it. When your RAG pipeline returns irrelevant results, when your multi-agent system gets stuck in loops, or when your LLM costs suddenly spike, you're left guessing what went wrong and where.\nThat's exactly the challenge Noveum.ai solves. Rather than flying blind, our platform provides comprehensive tracing a...\n\nSource 2 (https://noveum.ai/docs/getting-started/tracing-concepts):\n- Explore Framework Integrations - Framework-specific guidance\n- Learn Advanced Patterns - Custom instrumentation techniques\n- Master the Dashboard - Analyze your traces effectively\nRemember: Good observability is not about collecting all possible data, but about collecting the right data that helps you understand, debug, and optimize your AI applications.\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to O...\n\nSource 3 (https://noveum.ai/en/docs/getting-started/tracing-concepts):\n- Explore Framework Integrations - Framework-specific guidance\n- Learn Advanced Patterns - Custom instrumentation techniques\n- Master the Dashboard - Analyze your traces effectively\nRemember: Good observability is not about collecting all possible data, but about collecting the right data that helps you understand, debug, and optimize your AI applications.\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to O...\n\nSource 4 (https://noveum.ai/en/blog/from-logs-to-intelligent-choices-inside-noveum-ais-evaluation-process):\nThe AI Observability Challenge\nWhy Traditional Monitoring Falls Short\nTraditional application monitoring tools weren't built for AI applications. They can tell you if your API is responding, but they can't answer the questions that matter most for AI systems:\n- Why did my RAG pipeline retrieve irrelevant documents?\n- Which LLM calls are driving my costs?\n- How are my agents communicating with each other?\n- What's causing hallucinations in my responses?\n- Why is my embedding generation so slow?\nW...\n\nSource 5 (https://noveum.ai/en/docs/advanced/rag-observability):\n\ud83d\udd27 Best Practices for RAG Observability\n1. Comprehensive Pipeline Tracing\n2. Quality Monitoring\n3. A/B Testing for RAG Components\n\ud83c\udfaf Advanced RAG Patterns\nMulti-Modal RAG\nConversational RAG\nRAG observability with Noveum.ai provides the deep insights needed to build, optimize, and scale retrieval-augmented generation systems. By implementing comprehensive tracing across retrieval, context preparation, and generation stages, you can ensure your RAG pipeline delivers accurate, relevant, and cost-effe...\n\nUser Question: What are the current trends in observability tools?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", + "prompt.prompt_length": 3490, + "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/blog/from-logs-to-intelligent-choices-inside-noveum-ais-evaluation-process):\nFrom Development to Production - Inside Noveum.ai's AI Observability Platform\nShashank Agarw...", + "prompt.instruction_following": "rag_optimized", + "response.answer_length": 897, + "response.answer_completeness": "complete", + "response.response_quality": "high", + "response.source_citation": 5, + "response.context_utilization": 0.30479102956167176, + "output_response": "RAG Answer: The provided context does not specifically outline current trends in observability tools beyond the focus on AI applications. However, it highlights the limitations of traditional monitoring tools for...", + "evaluation.retrieval_effectiveness": 1, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 5, + "evaluation.context_relevance": "high", + "evaluation.overall_quality": "high", + "evaluation.ready_for_production": true, + "rag.retrieval_strategy": "semantic_similarity", + "rag.vector_search_results": 5, + "rag.context_synthesis": "multi_source", + "rag.document_coverage": 1, + "rag.information_density": 0.30479102956167176 + }, + "events": [], + "links": [], + "trace_trace_id": "fa5bb641-f4b7-4741-b06c-392a5f2a9aea", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:21.645000000", + "trace_end_time": "2025-10-16 08:50:26.683000000", + "trace_duration_ms": 5038.112, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:21.645000000", + "updated_at": "2025-10-16 08:50:21.645000000" + }, + { + "span_id": "2e06953f-5cec-4873-92b5-91f7281f6d7c", + "trace_id": "c0a95e8e-513c-47eb-b746-4e6016aeaf7d", + "parent_span_id": "9e41eab5-cd8e-4e46-a15b-e5f52c9a4073", + "name": "agent.query_routing", + "start_time": "2025-10-16 08:50:08.944000000", + "end_time": "2025-10-16 08:50:20.641000000", + "duration_ms": 11696.524, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "Tell me about recent developments in machine learning", + "query_length": 53, + "query_type": "routing_query", + "output_response": "Routed to Web Search for query processing", + "router.classification": "Web Search", + "router.confidence_score": 1, + "router.rag_keyword_score": 0, + "router.web_keyword_score": 1, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "c0a95e8e-513c-47eb-b746-4e6016aeaf7d", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:08.944000000", + "trace_end_time": "2025-10-16 08:50:20.641000000", + "trace_duration_ms": 11696.768, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:08.944000000", + "updated_at": "2025-10-16 08:50:08.944000000" + }, + { + "span_id": "9e41eab5-cd8e-4e46-a15b-e5f52c9a4073", + "trace_id": "c0a95e8e-513c-47eb-b746-4e6016aeaf7d", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-16 08:50:08.944000000", + "end_time": "2025-10-16 08:50:20.641000000", + "duration_ms": 11696.671999999999, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "Tell me about recent developments in machine learning", + "query.length": 53, + "processing.mode": "Web Search", + "processing.time_seconds": 11.696574449539185, + "processing.response_length": 2001, + "processing.sources_count": 1, + "output_response": "Final Answer: The web search did not yield specific results regarding recent developments in machine learning. However, I can provide a general overview based on my knowledge up to October 2023.\n\nRecent development...", + "final_answer_mode": "Web Search", + "query_processed.input_query": "Tell me about recent developments in machine learning", + "query_processed.output_response": "Successfully processed query using Web Search, generated 2001 character response", + "query_processed.mode": "Web Search", + "query_processed.processing_time": 11.696574449539185, + "query_processed.response_length": 2001 + }, + "events": [], + "links": [], + "trace_trace_id": "c0a95e8e-513c-47eb-b746-4e6016aeaf7d", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:08.944000000", + "trace_end_time": "2025-10-16 08:50:20.641000000", + "trace_duration_ms": 11696.768, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:08.944000000", + "updated_at": "2025-10-16 08:50:08.944000000" + }, + { + "span_id": "7ddf938b-183a-427f-849b-b31b18a98d87", + "trace_id": "c0a95e8e-513c-47eb-b746-4e6016aeaf7d", + "parent_span_id": "2e06953f-5cec-4873-92b5-91f7281f6d7c", + "name": "agent.web_search_generation", + "start_time": "2025-10-16 08:50:08.945000000", + "end_time": "2025-10-16 08:50:20.641000000", + "duration_ms": 11696.328, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "web_search_agent", + "agent.operation": "web_search_generation", + "agent.capabilities": [ + "web_search", + "content_synthesis", + "response_generation" + ], + "agent.id": "noveum_web_search_agent", + "input_query": "Tell me about recent developments in machine learning", + "query_length": 53, + "query_type": "web_search_query", + "output_response": "Web Search Answer: The web search did not yield specific results regarding recent developments in machine learning. However, I can provide a general overview based on my knowledge up to October 2023.\n\nRecent development...", + "web_search.results_count": 1, + "web_search.context_length": 155, + "web_search.response_length": 2001, + "web_search.mode": "external_web_search" + }, + "events": [], + "links": [], + "trace_trace_id": "c0a95e8e-513c-47eb-b746-4e6016aeaf7d", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:08.944000000", + "trace_end_time": "2025-10-16 08:50:20.641000000", + "trace_duration_ms": 11696.768, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:08.944000000", + "updated_at": "2025-10-16 08:50:08.944000000" + }, + { + "span_id": "f3e4c594-f822-4f04-ab43-98fac161b9a3", + "trace_id": "c0a95e8e-513c-47eb-b746-4e6016aeaf7d", + "parent_span_id": "2e06953f-5cec-4873-92b5-91f7281f6d7c", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-16 08:50:08.945000000", + "end_time": "2025-10-16 08:50:08.945000000", + "duration_ms": 0.046, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: Tell me about recent developments in machine learning", + "query_length": 53, + "query_type": "routing_evaluation_query", + "classification.mode": "Web Search", + "classification.rag_keyword_score": 0, + "classification.web_keyword_score": 1, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "complex", + "query.intent": "general_knowledge", + "query.keyword_density": 0.125, + "query.domain_affinity": "general", + "routing.decision": "Routed to Web Search based on analysis", + "routing.rationale": "RAG score: 0, Web score: 1, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "RAG", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.019230769230769232, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: Web Search (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "c0a95e8e-513c-47eb-b746-4e6016aeaf7d", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:08.944000000", + "trace_end_time": "2025-10-16 08:50:20.641000000", + "trace_duration_ms": 11696.768, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:08.944000000", + "updated_at": "2025-10-16 08:50:08.944000000" + }, + { + "span_id": "b47d09be-a43b-4b0a-b7d3-d98bdf1773a3", + "trace_id": "c0a95e8e-513c-47eb-b746-4e6016aeaf7d", + "parent_span_id": "7ddf938b-183a-427f-849b-b31b18a98d87", + "name": "agent.llm_model_execution", + "start_time": "2025-10-16 08:50:09.490000000", + "end_time": "2025-10-16 08:50:20.641000000", + "duration_ms": 11151.141000000001, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: Tell me about recent developments in machine learning", + "query_length": 53, + "query_type": "web_search_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 11.151098012924194, + "model.latency_ms": 11151.098012924194, + "model.start_time": 1760604609.4902723, + "model.end_time": 1760604620.6413703, + "model.performance_tier": "slow", + "model.prompt_tokens": 163, + "model.completion_tokens": 500, + "model.total_tokens": 663, + "model.tokens_per_second": 59.456028386763236, + "model.estimated_cost": 0.01989, + "model.efficiency_score": 3.018099547511312, + "model.response_length": 2001, + "model.response_quality": "high", + "model.output_response": "Model Response: The web search did not yield specific results regarding recent developments in machine learning. However, I can provide a general overview based on my knowledge up to October 2023.\n\nRecent development...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "c0a95e8e-513c-47eb-b746-4e6016aeaf7d", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:08.944000000", + "trace_end_time": "2025-10-16 08:50:20.641000000", + "trace_duration_ms": 11696.768, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:08.944000000", + "updated_at": "2025-10-16 08:50:08.944000000" + }, + { + "span_id": "bfea8500-0e5c-4556-a014-5b2ec49b7ad5", + "trace_id": "c0a95e8e-513c-47eb-b746-4e6016aeaf7d", + "parent_span_id": "7ddf938b-183a-427f-849b-b31b18a98d87", + "name": "agent.web_search_evaluation_metrics", + "start_time": "2025-10-16 08:50:20.641000000", + "end_time": "2025-10-16 08:50:20.641000000", + "duration_ms": 0.018000000000000002, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "web_search_evaluation_metrics", + "agent.capabilities": [ + "web_search_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: Tell me about recent developments in machine learning", + "query_length": 53, + "query_type": "web_search_evaluation_query", + "web_search.results_count": 1, + "web_search.context_length": 155, + "web_search.context_synthesized": "Context from 1 web sources", + "web_search.search_effectiveness": 0.2, + "web_search.context_quality": "low", + "web_search.source_diversity": 1, + "web_search.information_synthesis": "low", + "web_search.external_knowledge_utilization": 0.155, + "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=Tell+me+about+recent+developments+in+machine+learning\n\nUser Question: Tell me about recent developments in machine learning\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", + "prompt.prompt_length": 653, + "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=Tell+me+about+recent+developments+in+machine+learning", + "prompt.instruction_following": "web_search_optimized", + "response.answer_length": 2001, + "response.answer_completeness": "complete", + "response.response_quality": "medium", + "response.source_citation": 1, + "response.context_utilization": 12.90967741935484, + "output_response": "Web Search Answer: The web search did not yield specific results regarding recent developments in machine learning. However, I can provide a general overview based on my knowledge up to October 2023.\n\nRecent development...", + "evaluation.search_effectiveness": 0.2, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 1, + "evaluation.context_relevance": "low", + "evaluation.overall_quality": "low", + "evaluation.ready_for_production": true, + "web_search.search_strategy": "duckduckgo_api", + "web_search.real_time_data": true, + "web_search.external_sources": 1, + "web_search.information_freshness": "current", + "web_search.knowledge_synthesis": "single_source" + }, + "events": [], + "links": [], + "trace_trace_id": "c0a95e8e-513c-47eb-b746-4e6016aeaf7d", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:08.944000000", + "trace_end_time": "2025-10-16 08:50:20.641000000", + "trace_duration_ms": 11696.768, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:08.944000000", + "updated_at": "2025-10-16 08:50:08.944000000" + }, + { + "span_id": "33297286-d440-446a-ba73-7cabd9d4b2e6", + "trace_id": "3af8c6e8-62d9-4893-92ba-cc265934ef03", + "parent_span_id": "4a3c230e-0dd5-4ba2-8e87-2a301f654fba", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-16 08:50:04.544000000", + "end_time": "2025-10-16 08:50:04.544000000", + "duration_ms": 0.038, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What's the weather like today?", + "query_length": 30, + "query_type": "routing_evaluation_query", + "classification.mode": "Web Search", + "classification.rag_keyword_score": 0, + "classification.web_keyword_score": 2, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "general_knowledge", + "query.keyword_density": 0.4, + "query.domain_affinity": "general", + "routing.decision": "Routed to Web Search based on analysis", + "routing.rationale": "RAG score: 0, Web score: 2, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "RAG", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.038461538461538464, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: Web Search (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "3af8c6e8-62d9-4893-92ba-cc265934ef03", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:04.544000000", + "trace_end_time": "2025-10-16 08:50:07.941000000", + "trace_duration_ms": 3397.228, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:04.544000000", + "updated_at": "2025-10-16 08:50:04.544000000" + }, + { + "span_id": "4a3c230e-0dd5-4ba2-8e87-2a301f654fba", + "trace_id": "3af8c6e8-62d9-4893-92ba-cc265934ef03", + "parent_span_id": "eefcbdc5-99fc-4f06-a9f4-f51ba50564db", + "name": "agent.query_routing", + "start_time": "2025-10-16 08:50:04.544000000", + "end_time": "2025-10-16 08:50:07.941000000", + "duration_ms": 3397.056, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What's the weather like today?", + "query_length": 30, + "query_type": "routing_query", + "output_response": "Routed to Web Search for query processing", + "router.classification": "Web Search", + "router.confidence_score": 1, + "router.rag_keyword_score": 0, + "router.web_keyword_score": 2, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "3af8c6e8-62d9-4893-92ba-cc265934ef03", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:04.544000000", + "trace_end_time": "2025-10-16 08:50:07.941000000", + "trace_duration_ms": 3397.228, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:04.544000000", + "updated_at": "2025-10-16 08:50:04.544000000" + }, + { + "span_id": "eefcbdc5-99fc-4f06-a9f4-f51ba50564db", + "trace_id": "3af8c6e8-62d9-4893-92ba-cc265934ef03", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-16 08:50:04.544000000", + "end_time": "2025-10-16 08:50:07.941000000", + "duration_ms": 3397.169, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What's the weather like today?", + "query.length": 30, + "processing.mode": "Web Search", + "processing.time_seconds": 3.3970956802368164, + "processing.response_length": 518, + "processing.sources_count": 1, + "output_response": "Final Answer: The web search results do not provide specific details about today's weather conditions in your location. However, they mention that you can get real-time updates, including temperature, humidity, and...", + "final_answer_mode": "Web Search", + "query_processed.input_query": "What's the weather like today?", + "query_processed.output_response": "Successfully processed query using Web Search, generated 518 character response", + "query_processed.mode": "Web Search", + "query_processed.processing_time": 3.3970956802368164, + "query_processed.response_length": 518 + }, + "events": [], + "links": [], + "trace_trace_id": "3af8c6e8-62d9-4893-92ba-cc265934ef03", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:04.544000000", + "trace_end_time": "2025-10-16 08:50:07.941000000", + "trace_duration_ms": 3397.228, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:04.544000000", + "updated_at": "2025-10-16 08:50:04.544000000" + }, + { + "span_id": "9683465c-e851-4004-a858-c8fdd1a2b3da", + "trace_id": "3af8c6e8-62d9-4893-92ba-cc265934ef03", + "parent_span_id": "4a3c230e-0dd5-4ba2-8e87-2a301f654fba", + "name": "agent.web_search_generation", + "start_time": "2025-10-16 08:50:04.545000000", + "end_time": "2025-10-16 08:50:07.941000000", + "duration_ms": 3396.871, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "web_search_agent", + "agent.operation": "web_search_generation", + "agent.capabilities": [ + "web_search", + "content_synthesis", + "response_generation" + ], + "agent.id": "noveum_web_search_agent", + "input_query": "What's the weather like today?", + "query_length": 30, + "query_type": "web_search_query", + "output_response": "Web Search Answer: The web search results do not provide specific details about today's weather conditions in your location. However, they mention that you can get real-time updates, including temperature, humidity, and...", + "web_search.results_count": 1, + "web_search.context_length": 1137, + "web_search.response_length": 518, + "web_search.mode": "external_web_search" + }, + "events": [], + "links": [], + "trace_trace_id": "3af8c6e8-62d9-4893-92ba-cc265934ef03", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:04.544000000", + "trace_end_time": "2025-10-16 08:50:07.941000000", + "trace_duration_ms": 3397.228, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:04.544000000", + "updated_at": "2025-10-16 08:50:04.544000000" + }, + { + "span_id": "6413eb7e-e5cd-4c46-9651-de14c72c260e", + "trace_id": "3af8c6e8-62d9-4893-92ba-cc265934ef03", + "parent_span_id": "9683465c-e851-4004-a858-c8fdd1a2b3da", + "name": "agent.llm_model_execution", + "start_time": "2025-10-16 08:50:04.986000000", + "end_time": "2025-10-16 08:50:07.941000000", + "duration_ms": 2955.55, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What's the weather like today?", + "query_length": 30, + "query_type": "web_search_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 2.955509662628174, + "model.latency_ms": 2955.509662628174, + "model.start_time": 1760604604.9862766, + "model.end_time": 1760604607.9417863, + "model.performance_tier": "medium", + "model.prompt_tokens": 403, + "model.completion_tokens": 129, + "model.total_tokens": 532, + "model.tokens_per_second": 180.0027950261957, + "model.estimated_cost": 0.015960000000000002, + "model.efficiency_score": 0.9736842105263158, + "model.response_length": 518, + "model.response_quality": "high", + "model.output_response": "Model Response: The web search results do not provide specific details about today's weather conditions in your location. However, they mention that you can get real-time updates, including temperature, humidity, and...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "3af8c6e8-62d9-4893-92ba-cc265934ef03", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:04.544000000", + "trace_end_time": "2025-10-16 08:50:07.941000000", + "trace_duration_ms": 3397.228, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:04.544000000", + "updated_at": "2025-10-16 08:50:04.544000000" + }, + { + "span_id": "64df69fb-0bcc-4845-8bfe-5a721c1ed912", + "trace_id": "3af8c6e8-62d9-4893-92ba-cc265934ef03", + "parent_span_id": "9683465c-e851-4004-a858-c8fdd1a2b3da", + "name": "agent.web_search_evaluation_metrics", + "start_time": "2025-10-16 08:50:07.941000000", + "end_time": "2025-10-16 08:50:07.941000000", + "duration_ms": 0.017, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "web_search_evaluation_metrics", + "agent.capabilities": [ + "web_search_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What's the weather like today?", + "query_length": 30, + "query_type": "web_search_evaluation_query", + "web_search.results_count": 1, + "web_search.context_length": 1137, + "web_search.context_synthesized": "Context from 1 web sources", + "web_search.search_effectiveness": 0.2, + "web_search.context_quality": "high", + "web_search.source_diversity": 1, + "web_search.information_synthesis": "low", + "web_search.external_knowledge_utilization": 1.137, + "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nStay updated with the latest weather conditions in Komna. Get real-time updates, temperature, humidity, AQI, and detailed forecast for today. Dec 31, 2024 \u00b7 Komna Weather Forecast: Check the latest temperature in Komna today along with details such as humidity, visibility, air quality, wind speed, pressure, and weekly weather report at Shuru. 2 days ago \u00b7 Latest weather forecast for India for today's, hourly weather forecast, including today's temperatures in India, wind, rain and more. 2 days ago \u00b7 Get the latest hourly weather updates for India today. Detailed forecast including temperature, wind, rain, snow, and UV index. Stay informed about today's weather conditions in India. 2 days ago \u00b7 Today's Weather Report ... Forecasts Urban Meteorological Services Short to Medium Range Model Guidance Extended Range Model Guidance Seasonal Forecast Quantitative Precipitation Forecast Latest CAP Alerts Specialised products & forecasts Cyclones | Monsoon | Satellite Radar | Rainfall | Agromet Advisories MAUSAM : | | Video Meghdoot Agro ...\nURL: https://duckduckgo.com/?q=What's+the+weather+like+today?\n\nUser Question: What's the weather like today?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", + "prompt.prompt_length": 1612, + "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nStay updated with the latest weather conditions in Komna. Get real-time updates, temperature, humidity, AQI, and detailed forecast for today. Dec 31, 2024 \u00b7 Komna Weather F...", + "prompt.instruction_following": "web_search_optimized", + "response.answer_length": 518, + "response.answer_completeness": "complete", + "response.response_quality": "medium", + "response.source_citation": 1, + "response.context_utilization": 0.455584872471416, + "output_response": "Web Search Answer: The web search results do not provide specific details about today's weather conditions in your location. However, they mention that you can get real-time updates, including temperature, humidity, and...", + "evaluation.search_effectiveness": 0.2, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 1, + "evaluation.context_relevance": "high", + "evaluation.overall_quality": "low", + "evaluation.ready_for_production": true, + "web_search.search_strategy": "duckduckgo_api", + "web_search.real_time_data": true, + "web_search.external_sources": 1, + "web_search.information_freshness": "current", + "web_search.knowledge_synthesis": "single_source" + }, + "events": [], + "links": [], + "trace_trace_id": "3af8c6e8-62d9-4893-92ba-cc265934ef03", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:04.544000000", + "trace_end_time": "2025-10-16 08:50:07.941000000", + "trace_duration_ms": 3397.228, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:04.544000000", + "updated_at": "2025-10-16 08:50:04.544000000" + }, + { + "span_id": "719a1a8f-fc50-439c-899b-436146accd23", + "trace_id": "d7b8b811-ad01-48f1-917f-ace7e8d289e6", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-16 08:50:01.375000000", + "end_time": "2025-10-16 08:50:03.540000000", + "duration_ms": 2164.4809999999998, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What are the latest AI news today?", + "query.length": 34, + "processing.mode": "Web Search", + "processing.time_seconds": 2.1643810272216797, + "processing.response_length": 298, + "processing.sources_count": 1, + "output_response": "Final Answer: The web search did not yield any specific results regarding the latest AI news today. Unfortunately, I cannot provide current updates or news articles on AI developments at this moment. For the latest...", + "final_answer_mode": "Web Search", + "query_processed.input_query": "What are the latest AI news today?", + "query_processed.output_response": "Successfully processed query using Web Search, generated 298 character response", + "query_processed.mode": "Web Search", + "query_processed.processing_time": 2.1643810272216797, + "query_processed.response_length": 298 + }, + "events": [], + "links": [], + "trace_trace_id": "d7b8b811-ad01-48f1-917f-ace7e8d289e6", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:01.375000000", + "trace_end_time": "2025-10-16 08:50:03.540000000", + "trace_duration_ms": 2164.576, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:01.375000000", + "updated_at": "2025-10-16 08:50:01.375000000" + }, + { + "span_id": "644a712f-1c9e-4e9a-9073-74621b754aa2", + "trace_id": "d7b8b811-ad01-48f1-917f-ace7e8d289e6", + "parent_span_id": "719a1a8f-fc50-439c-899b-436146accd23", + "name": "agent.query_routing", + "start_time": "2025-10-16 08:50:01.376000000", + "end_time": "2025-10-16 08:50:03.540000000", + "duration_ms": 2164.3219999999997, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What are the latest AI news today?", + "query_length": 34, + "query_type": "routing_query", + "output_response": "Routed to Web Search for query processing", + "router.classification": "Web Search", + "router.confidence_score": 1, + "router.rag_keyword_score": 0, + "router.web_keyword_score": 3, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "d7b8b811-ad01-48f1-917f-ace7e8d289e6", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:01.375000000", + "trace_end_time": "2025-10-16 08:50:03.540000000", + "trace_duration_ms": 2164.576, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:01.375000000", + "updated_at": "2025-10-16 08:50:01.375000000" + }, + { + "span_id": "8a4abdbe-6ad7-4d43-b234-81667c4abbc2", + "trace_id": "d7b8b811-ad01-48f1-917f-ace7e8d289e6", + "parent_span_id": "644a712f-1c9e-4e9a-9073-74621b754aa2", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-16 08:50:01.376000000", + "end_time": "2025-10-16 08:50:01.376000000", + "duration_ms": 0.051, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What are the latest AI news today?", + "query_length": 34, + "query_type": "routing_evaluation_query", + "classification.mode": "Web Search", + "classification.rag_keyword_score": 0, + "classification.web_keyword_score": 3, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "general_knowledge", + "query.keyword_density": 0.42857142857142855, + "query.domain_affinity": "general", + "routing.decision": "Routed to Web Search based on analysis", + "routing.rationale": "RAG score: 0, Web score: 3, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "RAG", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.057692307692307696, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: Web Search (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "d7b8b811-ad01-48f1-917f-ace7e8d289e6", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:01.375000000", + "trace_end_time": "2025-10-16 08:50:03.540000000", + "trace_duration_ms": 2164.576, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:01.375000000", + "updated_at": "2025-10-16 08:50:01.375000000" + }, + { + "span_id": "904621e8-c5fa-4b2d-a628-8f10eb9da51d", + "trace_id": "d7b8b811-ad01-48f1-917f-ace7e8d289e6", + "parent_span_id": "644a712f-1c9e-4e9a-9073-74621b754aa2", + "name": "agent.web_search_generation", + "start_time": "2025-10-16 08:50:01.376000000", + "end_time": "2025-10-16 08:50:03.540000000", + "duration_ms": 2164.076, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "web_search_agent", + "agent.operation": "web_search_generation", + "agent.capabilities": [ + "web_search", + "content_synthesis", + "response_generation" + ], + "agent.id": "noveum_web_search_agent", + "input_query": "What are the latest AI news today?", + "query_length": 34, + "query_type": "web_search_query", + "output_response": "Web Search Answer: The web search did not yield any specific results regarding the latest AI news today. Unfortunately, I cannot provide current updates or news articles on AI developments at this moment. For the latest...", + "web_search.results_count": 1, + "web_search.context_length": 136, + "web_search.response_length": 298, + "web_search.mode": "external_web_search" + }, + "events": [], + "links": [], + "trace_trace_id": "d7b8b811-ad01-48f1-917f-ace7e8d289e6", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:01.375000000", + "trace_end_time": "2025-10-16 08:50:03.540000000", + "trace_duration_ms": 2164.576, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:01.375000000", + "updated_at": "2025-10-16 08:50:01.375000000" + }, + { + "span_id": "421dd1af-2dbc-477f-bd59-b3194b14ae3a", + "trace_id": "d7b8b811-ad01-48f1-917f-ace7e8d289e6", + "parent_span_id": "904621e8-c5fa-4b2d-a628-8f10eb9da51d", + "name": "agent.llm_model_execution", + "start_time": "2025-10-16 08:50:01.708000000", + "end_time": "2025-10-16 08:50:03.540000000", + "duration_ms": 1831.237, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What are the latest AI news today?", + "query_length": 34, + "query_type": "web_search_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 1.8311803340911865, + "model.latency_ms": 1831.1803340911865, + "model.start_time": 1760604601.7090185, + "model.end_time": 1760604603.5401988, + "model.performance_tier": "fast", + "model.prompt_tokens": 153, + "model.completion_tokens": 74, + "model.total_tokens": 227, + "model.tokens_per_second": 123.96376029926071, + "model.estimated_cost": 0.00681, + "model.efficiency_score": 1.3127753303964758, + "model.response_length": 298, + "model.response_quality": "high", + "model.output_response": "Model Response: The web search did not yield any specific results regarding the latest AI news today. Unfortunately, I cannot provide current updates or news articles on AI developments at this moment. For the latest...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "d7b8b811-ad01-48f1-917f-ace7e8d289e6", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:01.375000000", + "trace_end_time": "2025-10-16 08:50:03.540000000", + "trace_duration_ms": 2164.576, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:01.375000000", + "updated_at": "2025-10-16 08:50:01.375000000" + }, + { + "span_id": "159aad6f-7174-406d-b7d9-eed9360d993c", + "trace_id": "d7b8b811-ad01-48f1-917f-ace7e8d289e6", + "parent_span_id": "904621e8-c5fa-4b2d-a628-8f10eb9da51d", + "name": "agent.web_search_evaluation_metrics", + "start_time": "2025-10-16 08:50:03.540000000", + "end_time": "2025-10-16 08:50:03.540000000", + "duration_ms": 0.028, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "web_search_evaluation_metrics", + "agent.capabilities": [ + "web_search_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What are the latest AI news today?", + "query_length": 34, + "query_type": "web_search_evaluation_query", + "web_search.results_count": 1, + "web_search.context_length": 136, + "web_search.context_synthesized": "Context from 1 web sources", + "web_search.search_effectiveness": 0.2, + "web_search.context_quality": "low", + "web_search.source_diversity": 1, + "web_search.information_synthesis": "low", + "web_search.external_knowledge_utilization": 0.136, + "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What+are+the+latest+AI+news+today?\n\nUser Question: What are the latest AI news today?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", + "prompt.prompt_length": 615, + "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What+are+the+latest+AI+news+today?", + "prompt.instruction_following": "web_search_optimized", + "response.answer_length": 298, + "response.answer_completeness": "complete", + "response.response_quality": "medium", + "response.source_citation": 1, + "response.context_utilization": 2.1911764705882355, + "output_response": "Web Search Answer: The web search did not yield any specific results regarding the latest AI news today. Unfortunately, I cannot provide current updates or news articles on AI developments at this moment. For the latest...", + "evaluation.search_effectiveness": 0.2, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 1, + "evaluation.context_relevance": "low", + "evaluation.overall_quality": "low", + "evaluation.ready_for_production": true, + "web_search.search_strategy": "duckduckgo_api", + "web_search.real_time_data": true, + "web_search.external_sources": 1, + "web_search.information_freshness": "current", + "web_search.knowledge_synthesis": "single_source" + }, + "events": [], + "links": [], + "trace_trace_id": "d7b8b811-ad01-48f1-917f-ace7e8d289e6", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:01.375000000", + "trace_end_time": "2025-10-16 08:50:03.540000000", + "trace_duration_ms": 2164.576, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:01.375000000", + "updated_at": "2025-10-16 08:50:01.375000000" + }, + { + "span_id": "dcf2b38e-ab90-45ee-9a65-e6fea47a3fd7", + "trace_id": "df8340b4-b75a-48e9-a2f3-99725b46a00d", + "parent_span_id": "d0e2b320-c18d-4ec6-98a1-74da5252107e", + "name": "agent.query_routing", + "start_time": "2025-10-16 08:49:17.236000000", + "end_time": "2025-10-16 08:50:00.368000000", + "duration_ms": 43131.384, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What are the benefits of using Noveum Trace?", + "query_length": 44, + "query_type": "routing_query", + "output_response": "Routed to RAG for query processing", + "router.classification": "RAG", + "router.confidence_score": 1, + "router.rag_keyword_score": 2, + "router.web_keyword_score": 0, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "df8340b4-b75a-48e9-a2f3-99725b46a00d", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:49:17.236000000", + "trace_end_time": "2025-10-16 08:50:00.368000000", + "trace_duration_ms": 43131.554000000004, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:49:17.236000000", + "updated_at": "2025-10-16 08:49:17.236000000" + }, + { + "span_id": "d0e2b320-c18d-4ec6-98a1-74da5252107e", + "trace_id": "df8340b4-b75a-48e9-a2f3-99725b46a00d", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-16 08:49:17.236000000", + "end_time": "2025-10-16 08:50:00.368000000", + "duration_ms": 43131.481, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What are the benefits of using Noveum Trace?", + "query.length": 44, + "processing.mode": "RAG", + "processing.time_seconds": 43.131428718566895, + "processing.response_length": 1415, + "processing.sources_count": 5, + "output_response": "Final Answer: The benefits of using Noveum Trace include:\n\n1. **Comprehensive Tracing**: Noveum Trace offers decorator-based tracing for LLM calls, agents, and RAG pipelines, allowing for detailed monitoring of AI ...", + "final_answer_mode": "RAG", + "query_processed.input_query": "What are the benefits of using Noveum Trace?", + "query_processed.output_response": "Successfully processed query using RAG, generated 1415 character response", + "query_processed.mode": "RAG", + "query_processed.processing_time": 43.131428718566895, + "query_processed.response_length": 1415 + }, + "events": [], + "links": [], + "trace_trace_id": "df8340b4-b75a-48e9-a2f3-99725b46a00d", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:49:17.236000000", + "trace_end_time": "2025-10-16 08:50:00.368000000", + "trace_duration_ms": 43131.554000000004, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:49:17.236000000", + "updated_at": "2025-10-16 08:49:17.236000000" + }, + { + "span_id": "fa93821f-c809-4071-87e1-a6c55014d37c", + "trace_id": "df8340b4-b75a-48e9-a2f3-99725b46a00d", + "parent_span_id": "dcf2b38e-ab90-45ee-9a65-e6fea47a3fd7", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-16 08:49:17.236000000", + "end_time": "2025-10-16 08:49:17.236000000", + "duration_ms": 0.034, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What are the benefits of using Noveum Trace?", + "query_length": 44, + "query_type": "routing_evaluation_query", + "classification.mode": "RAG", + "classification.rag_keyword_score": 2, + "classification.web_keyword_score": 0, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "noveum_specific", + "query.keyword_density": 0.25, + "query.domain_affinity": "noveum", + "routing.decision": "Routed to RAG based on analysis", + "routing.rationale": "RAG score: 2, Web score: 0, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "Web Search", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.038461538461538464, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: RAG (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "df8340b4-b75a-48e9-a2f3-99725b46a00d", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:49:17.236000000", + "trace_end_time": "2025-10-16 08:50:00.368000000", + "trace_duration_ms": 43131.554000000004, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:49:17.236000000", + "updated_at": "2025-10-16 08:49:17.236000000" + }, + { + "span_id": "024ad5aa-f394-4c26-af51-564f3f51c6f2", + "trace_id": "df8340b4-b75a-48e9-a2f3-99725b46a00d", + "parent_span_id": "dcf2b38e-ab90-45ee-9a65-e6fea47a3fd7", + "name": "agent.llm-rag", + "start_time": "2025-10-16 08:49:17.237000000", + "end_time": "2025-10-16 08:50:00.368000000", + "duration_ms": 43131.19300000001, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "rag_agent", + "agent.operation": "llm-rag", + "agent.capabilities": [ + "document_retrieval", + "context_generation", + "response_generation" + ], + "agent.id": "noveum_rag_agent", + "input_query": "What are the benefits of using Noveum Trace?", + "query_length": 44, + "query_type": "rag_query", + "output_response": "RAG Answer: The benefits of using Noveum Trace include:\n\n1. **Comprehensive Tracing**: Noveum Trace offers decorator-based tracing for LLM calls, agents, and RAG pipelines, allowing for detailed monitoring of AI ...", + "rag.context_length": 2773, + "rag.sources_count": 5, + "rag.answer_length": 1415, + "rag.mode": "retrieval_augmented_generation" + }, + "events": [], + "links": [], + "trace_trace_id": "df8340b4-b75a-48e9-a2f3-99725b46a00d", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:49:17.236000000", + "trace_end_time": "2025-10-16 08:50:00.368000000", + "trace_duration_ms": 43131.554000000004, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:49:17.236000000", + "updated_at": "2025-10-16 08:49:17.236000000" + }, + { + "span_id": "3d9db951-d8b0-4990-820d-61e4b1011709", + "trace_id": "df8340b4-b75a-48e9-a2f3-99725b46a00d", + "parent_span_id": "024ad5aa-f394-4c26-af51-564f3f51c6f2", + "name": "agent.llm_model_execution", + "start_time": "2025-10-16 08:49:20.224000000", + "end_time": "2025-10-16 08:49:58.420000000", + "duration_ms": 38196.111000000004, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What are the benefits of using Noveum Trace?", + "query_length": 44, + "query_type": "rag_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 38.19604015350342, + "model.latency_ms": 38196.04015350342, + "model.start_time": 1760604560.2242246, + "model.end_time": 1760604598.4202647, + "model.performance_tier": "slow", + "model.prompt_tokens": 828, + "model.completion_tokens": 353, + "model.total_tokens": 1181, + "model.tokens_per_second": 30.919435503098253, + "model.estimated_cost": 0.03543, + "model.efficiency_score": 1.1981371718882303, + "model.response_length": 1415, + "model.response_quality": "high", + "model.output_response": "Model Response: The benefits of using Noveum Trace include:\n\n1. **Comprehensive Tracing**: Noveum Trace offers decorator-based tracing for LLM calls, agents, and RAG pipelines, allowing for detailed monitoring of AI ...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "df8340b4-b75a-48e9-a2f3-99725b46a00d", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:49:17.236000000", + "trace_end_time": "2025-10-16 08:50:00.368000000", + "trace_duration_ms": 43131.554000000004, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:49:17.236000000", + "updated_at": "2025-10-16 08:49:17.236000000" + }, + { + "span_id": "8c33ffac-4d76-42ba-9737-89596e136b3d", + "trace_id": "df8340b4-b75a-48e9-a2f3-99725b46a00d", + "parent_span_id": "024ad5aa-f394-4c26-af51-564f3f51c6f2", + "name": "agent.rag_evaluation_metrics", + "start_time": "2025-10-16 08:49:58.420000000", + "end_time": "2025-10-16 08:49:58.420000000", + "duration_ms": 0.037, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "rag_evaluation_metrics", + "agent.capabilities": [ + "retrieval_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What are the benefits of using Noveum Trace?", + "query_length": 44, + "query_type": "rag_evaluation_query", + "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/docs/platform/dashboard):\n- Bulk Export: Download multiple traces simultaneously\n- Comparative Analysis: Compare performance across multiple traces\nReal-Time Updates\n- Live Refresh: Automatic updates as new traces arrive\n- Connection Monitoring: Real-time status of your t...", + "retrieval.context_length": 2773, + "retrieval.sources_count": 5, + "retrieval.context_quality": "high", + "retrieval.effectiveness": 1, + "retrieval.context_utilization": 2.773, + "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/docs/platform/dashboard):\n- Bulk Export: Download multiple traces simultaneously\n- Comparative Analysis: Compare performance across multiple traces\nReal-Time Updates\n- Live Refresh: Automatic updates as new traces arrive\n- Connection Monitoring: Real-time status of your trace ingestion pipeline\n- Performance Indicators: Live metrics for system health monitoring\nReady to dive deeper? Explore Projects & Environments to organize your AI applications, or check out Team Collaboration to share insights with your team.\nGet Earl...\n\nSource 2 (https://noveum.ai/en/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\n\ud83d\udcd8 TypeScript SDK (\n@noveum/trace\n)- Framework integrations for Next.js, Express.js, and Hono\n- TypeScript-first design with full type safety\n- Universal compatibility (Node.js, Edge Runtime, browsers)\n-\n\ud83d\udcca Noveum Platform\n- Real-time dashboard for analyzing traces and performance\n- Ad...\n\nSource 3 (https://noveum.ai/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\n\ud83d\udcd8 TypeScript SDK (\n@noveum/trace\n)- Framework integrations for Next.js, Express.js, and Hono\n- TypeScript-first design with full type safety\n- Universal compatibility (Node.js, Edge Runtime, browsers)\n-\n\ud83d\udcca Noveum Platform\n- Real-time dashboard for analyzing traces and performance\n- Ad...\n\nSource 4 (https://noveum.ai/en/docs/getting-started/overview):\n- Advanced search & filtering across traces and spans\n- Cost analysis and optimization recommendations\n- Team collaboration with shared insights and alerts\n\ud83d\udd0d What Noveum Traces\nLLM Operations\n- Model calls across all providers (OpenAI, Anthropic, Google, etc.)\n- Token usage and cost calculation\n- Prompt engineering effectiveness\n- Response quality metrics\nRAG Pipelines\n- Document retrieval performance and relevance\n- Embedding generation costs and latency\n- Context assembly and prompt constructi...\n\nSource 5 (https://noveum.ai/docs/getting-started/overview):\n- Advanced search & filtering across traces and spans\n- Cost analysis and optimization recommendations\n- Team collaboration with shared insights and alerts\n\ud83d\udd0d What Noveum Traces\nLLM Operations\n- Model calls across all providers (OpenAI, Anthropic, Google, etc.)\n- Token usage and cost calculation\n- Prompt engineering effectiveness\n- Response quality metrics\nRAG Pipelines\n- Document retrieval performance and relevance\n- Embedding generation costs and latency\n- Context assembly and prompt constructi...\n\nUser Question: What are the benefits of using Noveum Trace?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", + "prompt.prompt_length": 3313, + "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/docs/platform/dashboard):\n- Bulk Export: Download multiple traces simultaneously\n- Comparative Analysis: Compare performance across multiple traces\nReal-Time Updates\n- Live...", + "prompt.instruction_following": "rag_optimized", + "response.answer_length": 1415, + "response.answer_completeness": "complete", + "response.response_quality": "high", + "response.source_citation": 5, + "response.context_utilization": 0.5102776776054815, + "output_response": "RAG Answer: The benefits of using Noveum Trace include:\n\n1. **Comprehensive Tracing**: Noveum Trace offers decorator-based tracing for LLM calls, agents, and RAG pipelines, allowing for detailed monitoring of AI ...", + "evaluation.retrieval_effectiveness": 1, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 5, + "evaluation.context_relevance": "high", + "evaluation.overall_quality": "high", + "evaluation.ready_for_production": true, + "rag.retrieval_strategy": "semantic_similarity", + "rag.vector_search_results": 5, + "rag.context_synthesis": "multi_source", + "rag.document_coverage": 1, + "rag.information_density": 0.5102776776054815 + }, + "events": [], + "links": [], + "trace_trace_id": "df8340b4-b75a-48e9-a2f3-99725b46a00d", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:49:17.236000000", + "trace_end_time": "2025-10-16 08:50:00.368000000", + "trace_duration_ms": 43131.554000000004, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:49:17.236000000", + "updated_at": "2025-10-16 08:49:17.236000000" + }, + { + "span_id": "5b1a7121-d0bd-4525-821d-7721c6fbbd70", + "trace_id": "325ec189-2876-4ebe-b33c-649491a5053e", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-16 08:49:11.192000000", + "end_time": "2025-10-16 08:49:16.232000000", + "duration_ms": 5039.753, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "How do I configure Noveum for my system?", + "query.length": 40, + "processing.mode": "RAG", + "processing.time_seconds": 5.039698600769043, + "processing.response_length": 712, + "processing.sources_count": 5, + "output_response": "Final Answer: To configure Noveum for your system, you can refer to the SDK Integration Guide available in the Noveum documentation. This guide will help you trace your AI applications with minimal code changes. Th...", + "final_answer_mode": "RAG", + "query_processed.input_query": "How do I configure Noveum for my system?", + "query_processed.output_response": "Successfully processed query using RAG, generated 712 character response", + "query_processed.mode": "RAG", + "query_processed.processing_time": 5.039698600769043, + "query_processed.response_length": 712 + }, + "events": [], + "links": [], + "trace_trace_id": "325ec189-2876-4ebe-b33c-649491a5053e", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:49:11.192000000", + "trace_end_time": "2025-10-16 08:49:16.232000000", + "trace_duration_ms": 5039.841, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:49:11.192000000", + "updated_at": "2025-10-16 08:49:11.192000000" + }, + { + "span_id": "761c92c6-d15a-4a8e-a16a-a365922f8d7c", + "trace_id": "325ec189-2876-4ebe-b33c-649491a5053e", + "parent_span_id": "5b1a7121-d0bd-4525-821d-7721c6fbbd70", + "name": "agent.query_routing", + "start_time": "2025-10-16 08:49:11.192000000", + "end_time": "2025-10-16 08:49:16.232000000", + "duration_ms": 5039.642, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "How do I configure Noveum for my system?", + "query_length": 40, + "query_type": "routing_query", + "output_response": "Routed to RAG for query processing", + "router.classification": "RAG", + "router.confidence_score": 1, + "router.rag_keyword_score": 2, + "router.web_keyword_score": 0, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "325ec189-2876-4ebe-b33c-649491a5053e", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:49:11.192000000", + "trace_end_time": "2025-10-16 08:49:16.232000000", + "trace_duration_ms": 5039.841, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:49:11.192000000", + "updated_at": "2025-10-16 08:49:11.192000000" + }, + { + "span_id": "8fcf969e-a577-45aa-9fe6-1870d9665189", + "trace_id": "325ec189-2876-4ebe-b33c-649491a5053e", + "parent_span_id": "761c92c6-d15a-4a8e-a16a-a365922f8d7c", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-16 08:49:11.192000000", + "end_time": "2025-10-16 08:49:11.192000000", + "duration_ms": 0.045000000000000005, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: How do I configure Noveum for my system?", + "query_length": 40, + "query_type": "routing_evaluation_query", + "classification.mode": "RAG", + "classification.rag_keyword_score": 2, + "classification.web_keyword_score": 0, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "noveum_specific", + "query.keyword_density": 0.25, + "query.domain_affinity": "noveum", + "routing.decision": "Routed to RAG based on analysis", + "routing.rationale": "RAG score: 2, Web score: 0, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "Web Search", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.038461538461538464, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: RAG (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "325ec189-2876-4ebe-b33c-649491a5053e", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:49:11.192000000", + "trace_end_time": "2025-10-16 08:49:16.232000000", + "trace_duration_ms": 5039.841, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:49:11.192000000", + "updated_at": "2025-10-16 08:49:11.192000000" + }, + { + "span_id": "9e5bbea4-cc68-4fe6-b3d0-c87852635ec9", + "trace_id": "325ec189-2876-4ebe-b33c-649491a5053e", + "parent_span_id": "761c92c6-d15a-4a8e-a16a-a365922f8d7c", + "name": "agent.llm-rag", + "start_time": "2025-10-16 08:49:11.192000000", + "end_time": "2025-10-16 08:49:16.232000000", + "duration_ms": 5039.39, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "rag_agent", + "agent.operation": "llm-rag", + "agent.capabilities": [ + "document_retrieval", + "context_generation", + "response_generation" + ], + "agent.id": "noveum_rag_agent", + "input_query": "How do I configure Noveum for my system?", + "query_length": 40, + "query_type": "rag_query", + "output_response": "RAG Answer: To configure Noveum for your system, you can refer to the SDK Integration Guide available in the Noveum documentation. This guide will help you trace your AI applications with minimal code changes. Th...", + "rag.context_length": 2237, + "rag.sources_count": 5, + "rag.answer_length": 712, + "rag.mode": "retrieval_augmented_generation" + }, + "events": [], + "links": [], + "trace_trace_id": "325ec189-2876-4ebe-b33c-649491a5053e", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:49:11.192000000", + "trace_end_time": "2025-10-16 08:49:16.232000000", + "trace_duration_ms": 5039.841, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:49:11.192000000", + "updated_at": "2025-10-16 08:49:11.192000000" + }, + { + "span_id": "5ee706ad-c4ae-4c0b-94d1-6b9c43d173b4", + "trace_id": "325ec189-2876-4ebe-b33c-649491a5053e", + "parent_span_id": "9e5bbea4-cc68-4fe6-b3d0-c87852635ec9", + "name": "agent.llm_model_execution", + "start_time": "2025-10-16 08:49:11.607000000", + "end_time": "2025-10-16 08:49:15.718000000", + "duration_ms": 4110.915, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: How do I configure Noveum for my system?", + "query_length": 40, + "query_type": "rag_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 4.110846042633057, + "model.latency_ms": 4110.846042633057, + "model.start_time": 1760604551.6074958, + "model.end_time": 1760604555.7183418, + "model.performance_tier": "medium", + "model.prompt_tokens": 693, + "model.completion_tokens": 178, + "model.total_tokens": 871, + "model.tokens_per_second": 211.87852596934326, + "model.estimated_cost": 0.02613, + "model.efficiency_score": 0.817451205510907, + "model.response_length": 712, + "model.response_quality": "high", + "model.output_response": "Model Response: To configure Noveum for your system, you can refer to the SDK Integration Guide available in the Noveum documentation. This guide will help you trace your AI applications with minimal code changes. Th...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "325ec189-2876-4ebe-b33c-649491a5053e", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:49:11.192000000", + "trace_end_time": "2025-10-16 08:49:16.232000000", + "trace_duration_ms": 5039.841, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:49:11.192000000", + "updated_at": "2025-10-16 08:49:11.192000000" + }, + { + "span_id": "d13bdeda-81af-400a-82ca-a5ef7e1e8a6d", + "trace_id": "325ec189-2876-4ebe-b33c-649491a5053e", + "parent_span_id": "9e5bbea4-cc68-4fe6-b3d0-c87852635ec9", + "name": "agent.rag_evaluation_metrics", + "start_time": "2025-10-16 08:49:15.718000000", + "end_time": "2025-10-16 08:49:15.718000000", + "duration_ms": 0.118, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "rag_evaluation_metrics", + "agent.capabilities": [ + "retrieval_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: How do I configure Noveum for my system?", + "query_length": 40, + "query_type": "rag_evaluation_query", + "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/docs):\nWelcome to Noveum.ai Docs\nComprehensive AI tracing and observability for LLM applications, RAG systems, and AI agents\nWelcome to the Noveum.ai documentation! Here you'll find everything you need to integrate, configure, and optimize your AI applications using ou...", + "retrieval.context_length": 2237, + "retrieval.sources_count": 5, + "retrieval.context_quality": "high", + "retrieval.effectiveness": 1, + "retrieval.context_utilization": 2.237, + "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/docs):\nWelcome to Noveum.ai Docs\nComprehensive AI tracing and observability for LLM applications, RAG systems, and AI agents\nWelcome to the Noveum.ai documentation! Here you'll find everything you need to integrate, configure, and optimize your AI applications using our comprehensive tracing and observability platform.\nWe provide powerful SDKs for Python and TypeScript that enable you to trace LLM calls, RAG pipelines, and multi-agent workflows with minimal code changes. Our documentation is organized ...\n\nSource 2 (https://noveum.ai/docs):\nWelcome to Noveum.ai Docs\nComprehensive AI tracing and observability for LLM applications, RAG systems, and AI agents\nWelcome to the Noveum.ai documentation! Here you'll find everything you need to integrate, configure, and optimize your AI applications using our comprehensive tracing and observability platform.\nWe provide powerful SDKs for Python and TypeScript that enable you to trace LLM calls, RAG pipelines, and multi-agent workflows with minimal code changes. Our documentation is organized ...\n\nSource 3 (https://noveum.ai/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nSource 4 (https://noveum.ai/en/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nSource 5 (https://noveum.ai/en/docs):\n- \ud83c\udf0d Universal: Works across frameworks, clouds, and deployment models\n\ud83e\udd1d Community & Support\n- \ud83d\udcac Discord Community: Join our Discord\n- \ud83d\udce7 Email Support: [email protected]\n- \ud83d\udc1b Bug Reports: GitHub Issues\n- \ud83d\udcd6 Knowledge Base: Help Center\nReady to get started? Head to our SDK Integration Guide to begin tracing your AI applications in under 5 minutes!\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability ...\n\nUser Question: How do I configure Noveum for my system?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", + "prompt.prompt_length": 2773, + "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/docs):\nWelcome to Noveum.ai Docs\nComprehensive AI tracing and observability for LLM applications, RAG systems, and AI agents\nWelcome to the Noveum.ai documentation! Here...", + "prompt.instruction_following": "rag_optimized", + "response.answer_length": 712, + "response.answer_completeness": "complete", + "response.response_quality": "high", + "response.source_citation": 5, + "response.context_utilization": 0.3182834152883326, + "output_response": "RAG Answer: To configure Noveum for your system, you can refer to the SDK Integration Guide available in the Noveum documentation. This guide will help you trace your AI applications with minimal code changes. Th...", + "evaluation.retrieval_effectiveness": 1, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 5, + "evaluation.context_relevance": "high", + "evaluation.overall_quality": "high", + "evaluation.ready_for_production": true, + "rag.retrieval_strategy": "semantic_similarity", + "rag.vector_search_results": 5, + "rag.context_synthesis": "multi_source", + "rag.document_coverage": 1, + "rag.information_density": 0.3182834152883326 + }, + "events": [], + "links": [], + "trace_trace_id": "325ec189-2876-4ebe-b33c-649491a5053e", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:49:11.192000000", + "trace_end_time": "2025-10-16 08:49:16.232000000", + "trace_duration_ms": 5039.841, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:49:11.192000000", + "updated_at": "2025-10-16 08:49:11.192000000" + }, + { + "span_id": "7d6564fc-41b2-4d74-acb1-e77b575c0006", + "trace_id": "d5528586-c741-4703-8791-5fccf5c3e123", + "parent_span_id": "e79a037e-82b1-41e9-bdc7-42353e98ead7", + "name": "agent.query_routing", + "start_time": "2025-10-16 08:49:05.152000000", + "end_time": "2025-10-16 08:49:10.187000000", + "duration_ms": 5034.446, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What monitoring capabilities does Noveum provide?", + "query_length": 49, + "query_type": "routing_query", + "output_response": "Routed to RAG for query processing", + "router.classification": "RAG", + "router.confidence_score": 1, + "router.rag_keyword_score": 2, + "router.web_keyword_score": 0, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "d5528586-c741-4703-8791-5fccf5c3e123", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:49:05.152000000", + "trace_end_time": "2025-10-16 08:49:10.187000000", + "trace_duration_ms": 5034.604, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:49:05.152000000", + "updated_at": "2025-10-16 08:49:05.152000000" + }, + { + "span_id": "e79a037e-82b1-41e9-bdc7-42353e98ead7", + "trace_id": "d5528586-c741-4703-8791-5fccf5c3e123", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-16 08:49:05.152000000", + "end_time": "2025-10-16 08:49:10.187000000", + "duration_ms": 5034.531, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What monitoring capabilities does Noveum provide?", + "query.length": 49, + "processing.mode": "RAG", + "processing.time_seconds": 5.0344908237457275, + "processing.response_length": 552, + "processing.sources_count": 5, + "output_response": "Final Answer: Noveum provides comprehensive monitoring capabilities for AI applications, including the ability to monitor, trace, and optimize AI agents across various frameworks such as LangChain, CrewAI, AutoGen,...", + "final_answer_mode": "RAG", + "query_processed.input_query": "What monitoring capabilities does Noveum provide?", + "query_processed.output_response": "Successfully processed query using RAG, generated 552 character response", + "query_processed.mode": "RAG", + "query_processed.processing_time": 5.0344908237457275, + "query_processed.response_length": 552 + }, + "events": [], + "links": [], + "trace_trace_id": "d5528586-c741-4703-8791-5fccf5c3e123", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:49:05.152000000", + "trace_end_time": "2025-10-16 08:49:10.187000000", + "trace_duration_ms": 5034.604, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:49:05.152000000", + "updated_at": "2025-10-16 08:49:05.152000000" + }, + { + "span_id": "78297793-d73b-4311-b5ba-73509ae9f96e", + "trace_id": "d5528586-c741-4703-8791-5fccf5c3e123", + "parent_span_id": "7d6564fc-41b2-4d74-acb1-e77b575c0006", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-16 08:49:05.153000000", + "end_time": "2025-10-16 08:49:05.153000000", + "duration_ms": 0.029, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What monitoring capabilities does Noveum provide?", + "query_length": 49, + "query_type": "routing_evaluation_query", + "classification.mode": "RAG", + "classification.rag_keyword_score": 2, + "classification.web_keyword_score": 0, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "noveum_specific", + "query.keyword_density": 0.3333333333333333, + "query.domain_affinity": "noveum", + "routing.decision": "Routed to RAG based on analysis", + "routing.rationale": "RAG score: 2, Web score: 0, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "Web Search", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.038461538461538464, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: RAG (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "d5528586-c741-4703-8791-5fccf5c3e123", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:49:05.152000000", + "trace_end_time": "2025-10-16 08:49:10.187000000", + "trace_duration_ms": 5034.604, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:49:05.152000000", + "updated_at": "2025-10-16 08:49:05.152000000" + }, + { + "span_id": "6a101d9d-de83-415a-8fb0-01a9fae5ffb9", + "trace_id": "d5528586-c741-4703-8791-5fccf5c3e123", + "parent_span_id": "7d6564fc-41b2-4d74-acb1-e77b575c0006", + "name": "agent.llm-rag", + "start_time": "2025-10-16 08:49:05.153000000", + "end_time": "2025-10-16 08:49:10.187000000", + "duration_ms": 5034.282, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "rag_agent", + "agent.operation": "llm-rag", + "agent.capabilities": [ + "document_retrieval", + "context_generation", + "response_generation" + ], + "agent.id": "noveum_rag_agent", + "input_query": "What monitoring capabilities does Noveum provide?", + "query_length": 49, + "query_type": "rag_query", + "output_response": "RAG Answer: Noveum provides comprehensive monitoring capabilities for AI applications, including the ability to monitor, trace, and optimize AI agents across various frameworks such as LangChain, CrewAI, AutoGen,...", + "rag.context_length": 2740, + "rag.sources_count": 5, + "rag.answer_length": 552, + "rag.mode": "retrieval_augmented_generation" + }, + "events": [], + "links": [], + "trace_trace_id": "d5528586-c741-4703-8791-5fccf5c3e123", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:49:05.152000000", + "trace_end_time": "2025-10-16 08:49:10.187000000", + "trace_duration_ms": 5034.604, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:49:05.152000000", + "updated_at": "2025-10-16 08:49:05.152000000" + }, + { + "span_id": "26e510c0-441a-4f9f-9dce-31180f47d060", + "trace_id": "d5528586-c741-4703-8791-5fccf5c3e123", + "parent_span_id": "6a101d9d-de83-415a-8fb0-01a9fae5ffb9", + "name": "agent.llm_model_execution", + "start_time": "2025-10-16 08:49:05.787000000", + "end_time": "2025-10-16 08:49:09.274000000", + "duration_ms": 3486.699, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What monitoring capabilities does Noveum provide?", + "query_length": 49, + "query_type": "rag_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 3.4866299629211426, + "model.latency_ms": 3486.6299629211426, + "model.start_time": 1760604545.787763, + "model.end_time": 1760604549.274393, + "model.performance_tier": "medium", + "model.prompt_tokens": 821, + "model.completion_tokens": 138, + "model.total_tokens": 959, + "model.tokens_per_second": 275.0506965747916, + "model.estimated_cost": 0.02877, + "model.efficiency_score": 0.5755995828988529, + "model.response_length": 552, + "model.response_quality": "high", + "model.output_response": "Model Response: Noveum provides comprehensive monitoring capabilities for AI applications, including the ability to monitor, trace, and optimize AI agents across various frameworks such as LangChain, CrewAI, AutoGen,...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "d5528586-c741-4703-8791-5fccf5c3e123", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:49:05.152000000", + "trace_end_time": "2025-10-16 08:49:10.187000000", + "trace_duration_ms": 5034.604, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:49:05.152000000", + "updated_at": "2025-10-16 08:49:05.152000000" + }, + { + "span_id": "fcd89e38-3d21-468b-860a-ae2de0bbd119", + "trace_id": "d5528586-c741-4703-8791-5fccf5c3e123", + "parent_span_id": "6a101d9d-de83-415a-8fb0-01a9fae5ffb9", + "name": "agent.rag_evaluation_metrics", + "start_time": "2025-10-16 08:49:09.274000000", + "end_time": "2025-10-16 08:49:09.274000000", + "duration_ms": 0.054, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "rag_evaluation_metrics", + "agent.capabilities": [ + "retrieval_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What monitoring capabilities does Noveum provide?", + "query_length": 49, + "query_type": "rag_evaluation_query", + "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en):\nMonitor all your AI Agents\nimprove AI Agents today\nNoveum.ai helps you monitor, trace, and optimize your AI applications.\nNoveum.ai works with any AI framework \u2013 LangChain, CrewAI, AutoGen, custom implementations, or direct LLM calls. One dashboard shows everything.\n...", + "retrieval.context_length": 2740, + "retrieval.sources_count": 5, + "retrieval.context_quality": "high", + "retrieval.effectiveness": 1, + "retrieval.context_utilization": 2.74, + "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en):\nMonitor all your AI Agents\nimprove AI Agents today\nNoveum.ai helps you monitor, trace, and optimize your AI applications.\nNoveum.ai works with any AI framework \u2013 LangChain, CrewAI, AutoGen, custom implementations, or direct LLM calls. One dashboard shows everything.\nMonitor, Evaluate, Improve Your AI Agents\nThe control plane for AI agents.\nMonitor Everything, Miss Nothing\nOur lightweight SDKs capture every trace and span across your AI agent ecosystem\u2014from simple LLM calls to complex multi-agent...\n\nSource 2 (https://noveum.ai/):\nMonitor all your AI Agents\nimprove AI Agents today\nNoveum.ai helps you monitor, trace, and optimize your AI applications.\nNoveum.ai works with any AI framework \u2013 LangChain, CrewAI, AutoGen, custom implementations, or direct LLM calls. One dashboard shows everything.\nMonitor, Evaluate, Improve Your AI Agents\nThe control plane for AI agents.\nMonitor Everything, Miss Nothing\nOur lightweight SDKs capture every trace and span across your AI agent ecosystem\u2014from simple LLM calls to complex multi-agent...\n\nSource 3 (https://noveum.ai):\nMonitor all your AI Agents\nimprove AI Agents today\nNoveum.ai helps you monitor, trace, and optimize your AI applications.\nNoveum.ai works with any AI framework \u2013 LangChain, CrewAI, AutoGen, custom implementations, or direct LLM calls. One dashboard shows everything.\nMonitor, Evaluate, Improve Your AI Agents\nThe control plane for AI agents.\nMonitor Everything, Miss Nothing\nOur lightweight SDKs capture every trace and span across your AI agent ecosystem\u2014from simple LLM calls to complex multi-agent...\n\nSource 4 (https://noveum.ai/en/docs/getting-started/overview):\nNoveum.ai Overview\nComprehensive AI tracing and observability platform for LLM applications, RAG systems, and AI agents\nWelcome to Noveum.ai\u2014the comprehensive tracing and observability platform built specifically for AI applications. Whether you're building LLM-powered chatbots, RAG systems, multi-agent workflows, or any AI-driven application, Noveum provides the insights you need to understand, debug, and optimize your systems.\n\ud83c\udfaf Why AI Applications Need Specialized Observability\nTraditional mo...\n\nSource 5 (https://noveum.ai/docs/getting-started/overview):\nNoveum.ai Overview\nComprehensive AI tracing and observability platform for LLM applications, RAG systems, and AI agents\nWelcome to Noveum.ai\u2014the comprehensive tracing and observability platform built specifically for AI applications. Whether you're building LLM-powered chatbots, RAG systems, multi-agent workflows, or any AI-driven application, Noveum provides the insights you need to understand, debug, and optimize your systems.\n\ud83c\udfaf Why AI Applications Need Specialized Observability\nTraditional mo...\n\nUser Question: What monitoring capabilities does Noveum provide?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", + "prompt.prompt_length": 3285, + "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en):\nMonitor all your AI Agents\nimprove AI Agents today\nNoveum.ai helps you monitor, trace, and optimize your AI applications.\nNoveum.ai works with any AI framework \u2013 LangC...", + "prompt.instruction_following": "rag_optimized", + "response.answer_length": 552, + "response.answer_completeness": "complete", + "response.response_quality": "high", + "response.source_citation": 5, + "response.context_utilization": 0.20145985401459854, + "output_response": "RAG Answer: Noveum provides comprehensive monitoring capabilities for AI applications, including the ability to monitor, trace, and optimize AI agents across various frameworks such as LangChain, CrewAI, AutoGen,...", + "evaluation.retrieval_effectiveness": 1, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 5, + "evaluation.context_relevance": "high", + "evaluation.overall_quality": "high", + "evaluation.ready_for_production": true, + "rag.retrieval_strategy": "semantic_similarity", + "rag.vector_search_results": 5, + "rag.context_synthesis": "multi_source", + "rag.document_coverage": 1, + "rag.information_density": 0.20145985401459854 + }, + "events": [], + "links": [], + "trace_trace_id": "d5528586-c741-4703-8791-5fccf5c3e123", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:49:05.152000000", + "trace_end_time": "2025-10-16 08:49:10.187000000", + "trace_duration_ms": 5034.604, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:49:05.152000000", + "updated_at": "2025-10-16 08:49:05.152000000" + }, + { + "span_id": "bd5f205f-0329-4811-b883-129d69f12382", + "trace_id": "8100b869-07c0-4e37-b51e-092c9d20b4a7", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-16 08:48:57.574000000", + "end_time": "2025-10-16 08:49:04.149000000", + "duration_ms": 6574.424, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "How does Noveum handle agent tracing?", + "query.length": 37, + "processing.mode": "RAG", + "processing.time_seconds": 6.5743818283081055, + "processing.response_length": 643, + "processing.sources_count": 5, + "output_response": "Final Answer: Noveum.ai handles agent tracing through its specialized Multi-Agent Tracing capabilities, which are designed to observe complex workflows and inter-agent communications within multi-agent systems. The...", + "final_answer_mode": "RAG", + "query_processed.input_query": "How does Noveum handle agent tracing?", + "query_processed.output_response": "Successfully processed query using RAG, generated 643 character response", + "query_processed.mode": "RAG", + "query_processed.processing_time": 6.5743818283081055, + "query_processed.response_length": 643 + }, + "events": [], + "links": [], + "trace_trace_id": "8100b869-07c0-4e37-b51e-092c9d20b4a7", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:48:57.574000000", + "trace_end_time": "2025-10-16 08:49:04.149000000", + "trace_duration_ms": 6574.500999999999, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:48:57.574000000", + "updated_at": "2025-10-16 08:48:57.574000000" + }, + { + "span_id": "0801bcfe-8c09-4ea6-a754-5564395d6ded", + "trace_id": "8100b869-07c0-4e37-b51e-092c9d20b4a7", + "parent_span_id": "3c35d86a-0df9-42dc-bfe3-bbe978cf62b5", + "name": "agent.llm-rag", + "start_time": "2025-10-16 08:48:57.575000000", + "end_time": "2025-10-16 08:49:04.149000000", + "duration_ms": 6574.135, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "rag_agent", + "agent.operation": "llm-rag", + "agent.capabilities": [ + "document_retrieval", + "context_generation", + "response_generation" + ], + "agent.id": "noveum_rag_agent", + "input_query": "How does Noveum handle agent tracing?", + "query_length": 37, + "query_type": "rag_query", + "output_response": "RAG Answer: Noveum.ai handles agent tracing through its specialized Multi-Agent Tracing capabilities, which are designed to observe complex workflows and inter-agent communications within multi-agent systems. The...", + "rag.context_length": 2736, + "rag.sources_count": 5, + "rag.answer_length": 643, + "rag.mode": "retrieval_augmented_generation" + }, + "events": [], + "links": [], + "trace_trace_id": "8100b869-07c0-4e37-b51e-092c9d20b4a7", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:48:57.574000000", + "trace_end_time": "2025-10-16 08:49:04.149000000", + "trace_duration_ms": 6574.500999999999, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:48:57.574000000", + "updated_at": "2025-10-16 08:48:57.574000000" + }, + { + "span_id": "3c35d86a-0df9-42dc-bfe3-bbe978cf62b5", + "trace_id": "8100b869-07c0-4e37-b51e-092c9d20b4a7", + "parent_span_id": "bd5f205f-0329-4811-b883-129d69f12382", + "name": "agent.query_routing", + "start_time": "2025-10-16 08:48:57.575000000", + "end_time": "2025-10-16 08:49:04.149000000", + "duration_ms": 6574.342, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "How does Noveum handle agent tracing?", + "query_length": 37, + "query_type": "routing_query", + "output_response": "Routed to RAG for query processing", + "router.classification": "RAG", + "router.confidence_score": 1, + "router.rag_keyword_score": 2, + "router.web_keyword_score": 0, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "8100b869-07c0-4e37-b51e-092c9d20b4a7", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:48:57.574000000", + "trace_end_time": "2025-10-16 08:49:04.149000000", + "trace_duration_ms": 6574.500999999999, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:48:57.574000000", + "updated_at": "2025-10-16 08:48:57.574000000" + }, + { + "span_id": "b1914316-f545-4bc9-abf4-3494cf42c4a5", + "trace_id": "8100b869-07c0-4e37-b51e-092c9d20b4a7", + "parent_span_id": "3c35d86a-0df9-42dc-bfe3-bbe978cf62b5", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-16 08:48:57.575000000", + "end_time": "2025-10-16 08:48:57.575000000", + "duration_ms": 0.032, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: How does Noveum handle agent tracing?", + "query_length": 37, + "query_type": "routing_evaluation_query", + "classification.mode": "RAG", + "classification.rag_keyword_score": 2, + "classification.web_keyword_score": 0, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "noveum_specific", + "query.keyword_density": 0.3333333333333333, + "query.domain_affinity": "noveum", + "routing.decision": "Routed to RAG based on analysis", + "routing.rationale": "RAG score: 2, Web score: 0, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "Web Search", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.038461538461538464, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: RAG (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "8100b869-07c0-4e37-b51e-092c9d20b4a7", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:48:57.574000000", + "trace_end_time": "2025-10-16 08:49:04.149000000", + "trace_duration_ms": 6574.500999999999, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:48:57.574000000", + "updated_at": "2025-10-16 08:48:57.574000000" + }, + { + "span_id": "0906814b-6a80-40e1-9f00-5376b809703a", + "trace_id": "8100b869-07c0-4e37-b51e-092c9d20b4a7", + "parent_span_id": "0801bcfe-8c09-4ea6-a754-5564395d6ded", + "name": "agent.llm_model_execution", + "start_time": "2025-10-16 08:48:58.009000000", + "end_time": "2025-10-16 08:49:03.349000000", + "duration_ms": 5340.01, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: How does Noveum handle agent tracing?", + "query_length": 37, + "query_type": "rag_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 5.339954614639282, + "model.latency_ms": 5339.954614639282, + "model.start_time": 1760604538.0093303, + "model.end_time": 1760604543.349285, + "model.performance_tier": "slow", + "model.prompt_tokens": 817, + "model.completion_tokens": 160, + "model.total_tokens": 977, + "model.tokens_per_second": 182.9603565022054, + "model.estimated_cost": 0.02931, + "model.efficiency_score": 0.6581371545547595, + "model.response_length": 643, + "model.response_quality": "high", + "model.output_response": "Model Response: Noveum.ai handles agent tracing through its specialized Multi-Agent Tracing capabilities, which are designed to observe complex workflows and inter-agent communications within multi-agent systems. The...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "8100b869-07c0-4e37-b51e-092c9d20b4a7", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:48:57.574000000", + "trace_end_time": "2025-10-16 08:49:04.149000000", + "trace_duration_ms": 6574.500999999999, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:48:57.574000000", + "updated_at": "2025-10-16 08:48:57.574000000" + }, + { + "span_id": "29822edb-c15f-4baa-b6ec-84d85fcc926d", + "trace_id": "8100b869-07c0-4e37-b51e-092c9d20b4a7", + "parent_span_id": "0801bcfe-8c09-4ea6-a754-5564395d6ded", + "name": "agent.rag_evaluation_metrics", + "start_time": "2025-10-16 08:49:03.349000000", + "end_time": "2025-10-16 08:49:03.349000000", + "duration_ms": 0.034999999999999996, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "rag_evaluation_metrics", + "agent.capabilities": [ + "retrieval_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: How does Noveum handle agent tracing?", + "query_length": 37, + "query_type": "rag_evaluation_query", + "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, vector searches, data retrieval, agent reasoning, and more. Without proper observability, debugging ...", + "retrieval.context_length": 2736, + "retrieval.sources_count": 5, + "retrieval.context_quality": "high", + "retrieval.effectiveness": 1, + "retrieval.context_utilization": 2.736, + "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, vector searches, data retrieval, agent reasoning, and more. Without proper observability, debugging becomes a nightmare, optimization is guesswork, and understanding user interactions is nearly impossible.\nNoveum.ai solves this by providing comprehensive tracing and observability specifically designed for AI applications.\nKey benefits of using Noveum.ai:\n- Complete Visibility: Trace...\n\nSource 2 (https://noveum.ai/docs/advanced/multi-agent-tracing):\nMulti-Agent Tracing\nObserve complex agent workflows and inter-agent communications with comprehensive tracing\nMulti-agent systems represent some of the most complex AI applications, involving multiple agents that coordinate, communicate, and collaborate to achieve shared goals. Noveum.ai provides specialized tracing capabilities to help you understand and optimize these intricate workflows.\n\ud83c\udfaf Why Multi-Agent Tracing Matters\nMulti-agent systems introduce unique observability challenges:\n- Complex...\n\nSource 3 (https://noveum.ai/en/docs/advanced/multi-agent-tracing):\nMulti-Agent Tracing\nObserve complex agent workflows and inter-agent communications with comprehensive tracing\nMulti-agent systems represent some of the most complex AI applications, involving multiple agents that coordinate, communicate, and collaborate to achieve shared goals. Noveum.ai provides specialized tracing capabilities to help you understand and optimize these intricate workflows.\n\ud83c\udfaf Why Multi-Agent Tracing Matters\nMulti-agent systems introduce unique observability challenges:\n- Complex...\n\nSource 4 (https://noveum.ai/en/docs/advanced/performance-optimization):\n\ud83d\udd17 Next Steps\n- Multi-Agent Tracing - Observe agent workflows\n- RAG Pipeline Observability - Monitor retrieval and generation systems\n- Custom Instrumentation - Add domain-specific tracing\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nSource 5 (https://noveum.ai/docs/advanced/performance-optimization):\n\ud83d\udd17 Next Steps\n- Multi-Agent Tracing - Observe agent workflows\n- RAG Pipeline Observability - Monitor retrieval and generation systems\n- Custom Instrumentation - Add domain-specific tracing\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nUser Question: How does Noveum handle agent tracing?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", + "prompt.prompt_length": 3269, + "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, ...", + "prompt.instruction_following": "rag_optimized", + "response.answer_length": 643, + "response.answer_completeness": "complete", + "response.response_quality": "high", + "response.source_citation": 5, + "response.context_utilization": 0.23501461988304093, + "output_response": "RAG Answer: Noveum.ai handles agent tracing through its specialized Multi-Agent Tracing capabilities, which are designed to observe complex workflows and inter-agent communications within multi-agent systems. The...", + "evaluation.retrieval_effectiveness": 1, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 5, + "evaluation.context_relevance": "high", + "evaluation.overall_quality": "high", + "evaluation.ready_for_production": true, + "rag.retrieval_strategy": "semantic_similarity", + "rag.vector_search_results": 5, + "rag.context_synthesis": "multi_source", + "rag.document_coverage": 1, + "rag.information_density": 0.23501461988304093 + }, + "events": [], + "links": [], + "trace_trace_id": "8100b869-07c0-4e37-b51e-092c9d20b4a7", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:48:57.574000000", + "trace_end_time": "2025-10-16 08:49:04.149000000", + "trace_duration_ms": 6574.500999999999, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:48:57.574000000", + "updated_at": "2025-10-16 08:48:57.574000000" + }, + { + "span_id": "5f8ea007-edef-477c-b8f1-57c2da9e9258", + "trace_id": "c28bd31c-d200-4467-91fd-a3b80187c885", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-16 08:48:17.977000000", + "end_time": "2025-10-16 08:48:56.570000000", + "duration_ms": 38592.912, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What APIs are available in Noveum platform?", + "query.length": 43, + "processing.mode": "RAG", + "processing.time_seconds": 38.59272003173828, + "processing.response_length": 270, + "processing.sources_count": 5, + "output_response": "Final Answer: The provided context does not specify the APIs available in the Noveum platform. For detailed information about the APIs, I recommend checking the official Noveum documentation or the SDK Integration ...", + "final_answer_mode": "RAG", + "query_processed.input_query": "What APIs are available in Noveum platform?", + "query_processed.output_response": "Successfully processed query using RAG, generated 270 character response", + "query_processed.mode": "RAG", + "query_processed.processing_time": 38.59272003173828, + "query_processed.response_length": 270 + }, + "events": [], + "links": [], + "trace_trace_id": "c28bd31c-d200-4467-91fd-a3b80187c885", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:48:17.977000000", + "trace_end_time": "2025-10-16 08:48:56.570000000", + "trace_duration_ms": 38592.996, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:48:17.977000000", + "updated_at": "2025-10-16 08:48:17.977000000" + }, + { + "span_id": "db385f4d-beef-465b-9831-a65080537519", + "trace_id": "c28bd31c-d200-4467-91fd-a3b80187c885", + "parent_span_id": "5f8ea007-edef-477c-b8f1-57c2da9e9258", + "name": "agent.query_routing", + "start_time": "2025-10-16 08:48:17.977000000", + "end_time": "2025-10-16 08:48:56.570000000", + "duration_ms": 38592.66499999999, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What APIs are available in Noveum platform?", + "query_length": 43, + "query_type": "routing_query", + "output_response": "Routed to RAG for query processing", + "router.classification": "RAG", + "router.confidence_score": 1, + "router.rag_keyword_score": 3, + "router.web_keyword_score": 0, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "c28bd31c-d200-4467-91fd-a3b80187c885", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:48:17.977000000", + "trace_end_time": "2025-10-16 08:48:56.570000000", + "trace_duration_ms": 38592.996, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:48:17.977000000", + "updated_at": "2025-10-16 08:48:17.977000000" + }, + { + "span_id": "3ccebe1c-1f2a-433b-a36f-453a0b803511", + "trace_id": "c28bd31c-d200-4467-91fd-a3b80187c885", + "parent_span_id": "db385f4d-beef-465b-9831-a65080537519", + "name": "agent.llm-rag", + "start_time": "2025-10-16 08:48:17.978000000", + "end_time": "2025-10-16 08:48:56.570000000", + "duration_ms": 38592.416999999994, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "rag_agent", + "agent.operation": "llm-rag", + "agent.capabilities": [ + "document_retrieval", + "context_generation", + "response_generation" + ], + "agent.id": "noveum_rag_agent", + "input_query": "What APIs are available in Noveum platform?", + "query_length": 43, + "query_type": "rag_query", + "output_response": "RAG Answer: The provided context does not specify the APIs available in the Noveum platform. For detailed information about the APIs, I recommend checking the official Noveum documentation or the SDK Integration ...", + "rag.context_length": 2284, + "rag.sources_count": 5, + "rag.answer_length": 270, + "rag.mode": "retrieval_augmented_generation" + }, + "events": [], + "links": [], + "trace_trace_id": "c28bd31c-d200-4467-91fd-a3b80187c885", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:48:17.977000000", + "trace_end_time": "2025-10-16 08:48:56.570000000", + "trace_duration_ms": 38592.996, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:48:17.977000000", + "updated_at": "2025-10-16 08:48:17.977000000" + }, + { + "span_id": "6d1bb8e9-315d-433a-8df5-7f5fb2eeee16", + "trace_id": "c28bd31c-d200-4467-91fd-a3b80187c885", + "parent_span_id": "db385f4d-beef-465b-9831-a65080537519", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-16 08:48:17.978000000", + "end_time": "2025-10-16 08:48:17.978000000", + "duration_ms": 0.043000000000000003, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What APIs are available in Noveum platform?", + "query_length": 43, + "query_type": "routing_evaluation_query", + "classification.mode": "RAG", + "classification.rag_keyword_score": 3, + "classification.web_keyword_score": 0, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "noveum_specific", + "query.keyword_density": 0.42857142857142855, + "query.domain_affinity": "noveum", + "routing.decision": "Routed to RAG based on analysis", + "routing.rationale": "RAG score: 3, Web score: 0, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "Web Search", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.057692307692307696, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: RAG (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "c28bd31c-d200-4467-91fd-a3b80187c885", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:48:17.977000000", + "trace_end_time": "2025-10-16 08:48:56.570000000", + "trace_duration_ms": 38592.996, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:48:17.977000000", + "updated_at": "2025-10-16 08:48:17.977000000" + }, + { + "span_id": "2f11df8a-8f45-425d-a0ce-1eec7887b508", + "trace_id": "c28bd31c-d200-4467-91fd-a3b80187c885", + "parent_span_id": "3ccebe1c-1f2a-433b-a36f-453a0b803511", + "name": "agent.llm_model_execution", + "start_time": "2025-10-16 08:48:18.392000000", + "end_time": "2025-10-16 08:48:55.545000000", + "duration_ms": 37152.496999999996, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What APIs are available in Noveum platform?", + "query_length": 43, + "query_type": "rag_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 37.15243363380432, + "model.latency_ms": 37152.43363380432, + "model.start_time": 1760604498.3926067, + "model.end_time": 1760604535.5450404, + "model.performance_tier": "slow", + "model.prompt_tokens": 705, + "model.completion_tokens": 67, + "model.total_tokens": 772, + "model.tokens_per_second": 20.779257897592238, + "model.estimated_cost": 0.02316, + "model.efficiency_score": 0.34974093264248707, + "model.response_length": 270, + "model.response_quality": "high", + "model.output_response": "Model Response: The provided context does not specify the APIs available in the Noveum platform. For detailed information about the APIs, I recommend checking the official Noveum documentation or the SDK Integration ...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "c28bd31c-d200-4467-91fd-a3b80187c885", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:48:17.977000000", + "trace_end_time": "2025-10-16 08:48:56.570000000", + "trace_duration_ms": 38592.996, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:48:17.977000000", + "updated_at": "2025-10-16 08:48:17.977000000" + }, + { + "span_id": "d13e608c-06e4-47c4-b602-c40ac9560be2", + "trace_id": "c28bd31c-d200-4467-91fd-a3b80187c885", + "parent_span_id": "3ccebe1c-1f2a-433b-a36f-453a0b803511", + "name": "agent.rag_evaluation_metrics", + "start_time": "2025-10-16 08:48:55.545000000", + "end_time": "2025-10-16 08:48:55.545000000", + "duration_ms": 0.031, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "rag_evaluation_metrics", + "agent.capabilities": [ + "retrieval_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What APIs are available in Noveum platform?", + "query_length": 43, + "query_type": "rag_evaluation_query", + "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/docs):\n- \ud83c\udf0d Universal: Works across frameworks, clouds, and deployment models\n\ud83e\udd1d Community & Support\n- \ud83d\udcac Discord Community: Join our Discord\n- \ud83d\udce7 Email Support: [email protected]\n- \ud83d\udc1b Bug Reports: GitHub Issues\n- \ud83d\udcd6 Knowledge Base: Help Center\nReady to get started? Head to ...", + "retrieval.context_length": 2284, + "retrieval.sources_count": 5, + "retrieval.context_quality": "high", + "retrieval.effectiveness": 1, + "retrieval.context_utilization": 2.284, + "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/docs):\n- \ud83c\udf0d Universal: Works across frameworks, clouds, and deployment models\n\ud83e\udd1d Community & Support\n- \ud83d\udcac Discord Community: Join our Discord\n- \ud83d\udce7 Email Support: [email protected]\n- \ud83d\udc1b Bug Reports: GitHub Issues\n- \ud83d\udcd6 Knowledge Base: Help Center\nReady to get started? Head to our SDK Integration Guide to begin tracing your AI applications in under 5 minutes!\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability ...\n\nSource 2 (https://noveum.ai/docs):\n- \ud83c\udf0d Universal: Works across frameworks, clouds, and deployment models\n\ud83e\udd1d Community & Support\n- \ud83d\udcac Discord Community: Join our Discord\n- \ud83d\udce7 Email Support: [email protected]\n- \ud83d\udc1b Bug Reports: GitHub Issues\n- \ud83d\udcd6 Knowledge Base: Help Center\nReady to get started? Head to our SDK Integration Guide to begin tracing your AI applications in under 5 minutes!\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability ...\n\nSource 3 (https://noveum.ai/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nSource 4 (https://noveum.ai/en/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nSource 5 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, vector searches, data retrieval, agent reasoning, and more. Without proper observability, debugging becomes a nightmare, optimization is guesswork, and understanding user interactions is nearly impossible.\nNoveum.ai solves this by providing comprehensive tracing and observability specifically designed for AI applications.\nKey benefits of using Noveum.ai:\n- Complete Visibility: Trace...\n\nUser Question: What APIs are available in Noveum platform?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", + "prompt.prompt_length": 2823, + "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/docs):\n- \ud83c\udf0d Universal: Works across frameworks, clouds, and deployment models\n\ud83e\udd1d Community & Support\n- \ud83d\udcac Discord Community: Join our Discord\n- \ud83d\udce7 Email Support: [email prot...", + "prompt.instruction_following": "rag_optimized", + "response.answer_length": 270, + "response.answer_completeness": "complete", + "response.response_quality": "high", + "response.source_citation": 5, + "response.context_utilization": 0.11821366024518389, + "output_response": "RAG Answer: The provided context does not specify the APIs available in the Noveum platform. For detailed information about the APIs, I recommend checking the official Noveum documentation or the SDK Integration ...", + "evaluation.retrieval_effectiveness": 1, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 5, + "evaluation.context_relevance": "high", + "evaluation.overall_quality": "high", + "evaluation.ready_for_production": true, + "rag.retrieval_strategy": "semantic_similarity", + "rag.vector_search_results": 5, + "rag.context_synthesis": "multi_source", + "rag.document_coverage": 1, + "rag.information_density": 0.11821366024518389 + }, + "events": [], + "links": [], + "trace_trace_id": "c28bd31c-d200-4467-91fd-a3b80187c885", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:48:17.977000000", + "trace_end_time": "2025-10-16 08:48:56.570000000", + "trace_duration_ms": 38592.996, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:48:17.977000000", + "updated_at": "2025-10-16 08:48:17.977000000" + }, + { + "span_id": "3f3babbd-abf5-4f5a-b31c-fa5dcec9c360", + "trace_id": "b90ef3fc-11ea-4230-a59b-e2f2db8f2de0", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-16 08:48:08.007000000", + "end_time": "2025-10-16 08:48:16.975000000", + "duration_ms": 8967.201, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "How do I set up observability with Noveum?", + "query.length": 42, + "processing.mode": "RAG", + "processing.time_seconds": 8.96716833114624, + "processing.response_length": 838, + "processing.sources_count": 5, + "output_response": "Final Answer: To set up observability with Noveum, follow these steps:\n\n1. **Sign Up**: Go to [noveum.ai](https://noveum.ai) and create an account.\n2. **Create a Project**: Once signed in, create a new project to o...", + "final_answer_mode": "RAG", + "query_processed.input_query": "How do I set up observability with Noveum?", + "query_processed.output_response": "Successfully processed query using RAG, generated 838 character response", + "query_processed.mode": "RAG", + "query_processed.processing_time": 8.96716833114624, + "query_processed.response_length": 838 + }, + "events": [], + "links": [], + "trace_trace_id": "b90ef3fc-11ea-4230-a59b-e2f2db8f2de0", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:48:08.007000000", + "trace_end_time": "2025-10-16 08:48:16.975000000", + "trace_duration_ms": 8967.264, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:48:08.007000000", + "updated_at": "2025-10-16 08:48:08.007000000" + }, + { + "span_id": "41f6453c-4be6-4b66-8f9b-8e795f7b41b0", + "trace_id": "b90ef3fc-11ea-4230-a59b-e2f2db8f2de0", + "parent_span_id": "b0a76345-49e5-4c1b-915f-cec90ea2b1f7", + "name": "agent.llm-rag", + "start_time": "2025-10-16 08:48:08.007000000", + "end_time": "2025-10-16 08:48:16.974000000", + "duration_ms": 8966.964, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "rag_agent", + "agent.operation": "llm-rag", + "agent.capabilities": [ + "document_retrieval", + "context_generation", + "response_generation" + ], + "agent.id": "noveum_rag_agent", + "input_query": "How do I set up observability with Noveum?", + "query_length": 42, + "query_type": "rag_query", + "output_response": "RAG Answer: To set up observability with Noveum, follow these steps:\n\n1. **Sign Up**: Go to [noveum.ai](https://noveum.ai) and create an account.\n2. **Create a Project**: Once signed in, create a new project to o...", + "rag.context_length": 2870, + "rag.sources_count": 6, + "rag.answer_length": 838, + "rag.mode": "retrieval_augmented_generation" + }, + "events": [], + "links": [], + "trace_trace_id": "b90ef3fc-11ea-4230-a59b-e2f2db8f2de0", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:48:08.007000000", + "trace_end_time": "2025-10-16 08:48:16.975000000", + "trace_duration_ms": 8967.264, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:48:08.007000000", + "updated_at": "2025-10-16 08:48:08.007000000" + }, + { + "span_id": "b0a76345-49e5-4c1b-915f-cec90ea2b1f7", + "trace_id": "b90ef3fc-11ea-4230-a59b-e2f2db8f2de0", + "parent_span_id": "3f3babbd-abf5-4f5a-b31c-fa5dcec9c360", + "name": "agent.query_routing", + "start_time": "2025-10-16 08:48:08.007000000", + "end_time": "2025-10-16 08:48:16.974000000", + "duration_ms": 8967.129, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "How do I set up observability with Noveum?", + "query_length": 42, + "query_type": "routing_query", + "output_response": "Routed to RAG for query processing", + "router.classification": "RAG", + "router.confidence_score": 1, + "router.rag_keyword_score": 2, + "router.web_keyword_score": 0, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "b90ef3fc-11ea-4230-a59b-e2f2db8f2de0", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:48:08.007000000", + "trace_end_time": "2025-10-16 08:48:16.975000000", + "trace_duration_ms": 8967.264, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:48:08.007000000", + "updated_at": "2025-10-16 08:48:08.007000000" + }, + { + "span_id": "fe3074a7-f4c0-442e-9a5b-e2cc11d93785", + "trace_id": "b90ef3fc-11ea-4230-a59b-e2f2db8f2de0", + "parent_span_id": "b0a76345-49e5-4c1b-915f-cec90ea2b1f7", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-16 08:48:08.007000000", + "end_time": "2025-10-16 08:48:08.007000000", + "duration_ms": 0.032, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: How do I set up observability with Noveum?", + "query_length": 42, + "query_type": "routing_evaluation_query", + "classification.mode": "RAG", + "classification.rag_keyword_score": 2, + "classification.web_keyword_score": 0, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "noveum_specific", + "query.keyword_density": 0.25, + "query.domain_affinity": "noveum", + "routing.decision": "Routed to RAG based on analysis", + "routing.rationale": "RAG score: 2, Web score: 0, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "Web Search", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.038461538461538464, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: RAG (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "b90ef3fc-11ea-4230-a59b-e2f2db8f2de0", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:48:08.007000000", + "trace_end_time": "2025-10-16 08:48:16.975000000", + "trace_duration_ms": 8967.264, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:48:08.007000000", + "updated_at": "2025-10-16 08:48:08.007000000" + }, + { + "span_id": "e16139ee-048e-4c2f-87f8-90442f41a0ae", + "trace_id": "b90ef3fc-11ea-4230-a59b-e2f2db8f2de0", + "parent_span_id": "41f6453c-4be6-4b66-8f9b-8e795f7b41b0", + "name": "agent.llm_model_execution", + "start_time": "2025-10-16 08:48:08.644000000", + "end_time": "2025-10-16 08:48:15.709000000", + "duration_ms": 7064.898, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: How do I set up observability with Noveum?", + "query_length": 42, + "query_type": "rag_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 7.0648510456085205, + "model.latency_ms": 7064.8510456085205, + "model.start_time": 1760604488.6448367, + "model.end_time": 1760604495.7096877, + "model.performance_tier": "slow", + "model.prompt_tokens": 852, + "model.completion_tokens": 209, + "model.total_tokens": 1061, + "model.tokens_per_second": 150.1800948315128, + "model.estimated_cost": 0.031830000000000004, + "model.efficiency_score": 0.7898209236569275, + "model.response_length": 838, + "model.response_quality": "high", + "model.output_response": "Model Response: To set up observability with Noveum, follow these steps:\n\n1. **Sign Up**: Go to [noveum.ai](https://noveum.ai) and create an account.\n2. **Create a Project**: Once signed in, create a new project to o...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "b90ef3fc-11ea-4230-a59b-e2f2db8f2de0", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:48:08.007000000", + "trace_end_time": "2025-10-16 08:48:16.975000000", + "trace_duration_ms": 8967.264, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:48:08.007000000", + "updated_at": "2025-10-16 08:48:08.007000000" + }, + { + "span_id": "59e85f8e-c19b-4794-9a69-7b5bc90208f0", + "trace_id": "b90ef3fc-11ea-4230-a59b-e2f2db8f2de0", + "parent_span_id": "41f6453c-4be6-4b66-8f9b-8e795f7b41b0", + "name": "agent.rag_evaluation_metrics", + "start_time": "2025-10-16 08:48:15.709000000", + "end_time": "2025-10-16 08:48:15.709000000", + "duration_ms": 0.039, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "rag_evaluation_metrics", + "agent.capabilities": [ + "retrieval_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: How do I set up observability with Noveum?", + "query_length": 42, + "query_type": "rag_evaluation_query", + "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\n- Production Ready: Built for scale with intelligent sampling and batching\n- Privacy Focused: You control what data is captured and how it's stored\n- Open Source SDKs: Transparent, auditable, and extensible\nGetting ...", + "retrieval.context_length": 2870, + "retrieval.sources_count": 6, + "retrieval.context_quality": "high", + "retrieval.effectiveness": 1.2, + "retrieval.context_utilization": 2.87, + "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\n- Production Ready: Built for scale with intelligent sampling and batching\n- Privacy Focused: You control what data is captured and how it's stored\n- Open Source SDKs: Transparent, auditable, and extensible\nGetting Started\nReady to add comprehensive observability to your AI applications?\n- Sign up at noveum.ai\n- Create a project and get your API key\n- Install the SDK for your preferred language\n- Add tracing to your AI workflows\n- Explore insights in the Noveum.ai dashboard\nCheck out our integra...\n\nSource 2 (https://noveum.ai/en/docs/getting-started/tracing-concepts):\n- Explore Framework Integrations - Framework-specific guidance\n- Learn Advanced Patterns - Custom instrumentation techniques\n- Master the Dashboard - Analyze your traces effectively\nRemember: Good observability is not about collecting all possible data, but about collecting the right data that helps you understand, debug, and optimize your AI applications.\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to O...\n\nSource 3 (https://noveum.ai/docs/getting-started/tracing-concepts):\n- Explore Framework Integrations - Framework-specific guidance\n- Learn Advanced Patterns - Custom instrumentation techniques\n- Master the Dashboard - Analyze your traces effectively\nRemember: Good observability is not about collecting all possible data, but about collecting the right data that helps you understand, debug, and optimize your AI applications.\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to O...\n\nSource 4 (https://noveum.ai/en/docs/getting-started/overview):\nNoveum.ai Overview\nComprehensive AI tracing and observability platform for LLM applications, RAG systems, and AI agents\nWelcome to Noveum.ai\u2014the comprehensive tracing and observability platform built specifically for AI applications. Whether you're building LLM-powered chatbots, RAG systems, multi-agent workflows, or any AI-driven application, Noveum provides the insights you need to understand, debug, and optimize your systems.\n\ud83c\udfaf Why AI Applications Need Specialized Observability\nTraditional mo...\n\nSource 5 (https://noveum.ai/docs/getting-started/overview):\nNoveum.ai Overview\nComprehensive AI tracing and observability platform for LLM applications, RAG systems, and AI agents\nWelcome to Noveum.ai\u2014the comprehensive tracing and observability platform built specifically for AI applications. Whether you're building LLM-powered chatbots, RAG systems, multi-agent workflows, or any AI-driven application, Noveum provides the insights you need to understand, debug, and optimize your systems.\n\ud83c\udfaf Why AI Applications Need Specialized Observability\nTraditional mo...\n\nUser Question: How do I set up observability with Noveum?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", + "prompt.prompt_length": 3408, + "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\n- Production Ready: Built for scale with intelligent sampling and batching\n- Privacy Focused: You control what data...", + "prompt.instruction_following": "rag_optimized", + "response.answer_length": 838, + "response.answer_completeness": "complete", + "response.response_quality": "high", + "response.source_citation": 6, + "response.context_utilization": 0.29198606271777006, + "output_response": "RAG Answer: To set up observability with Noveum, follow these steps:\n\n1. **Sign Up**: Go to [noveum.ai](https://noveum.ai) and create an account.\n2. **Create a Project**: Once signed in, create a new project to o...", + "evaluation.retrieval_effectiveness": 1.2, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 6, + "evaluation.context_relevance": "high", + "evaluation.overall_quality": "high", + "evaluation.ready_for_production": true, + "rag.retrieval_strategy": "semantic_similarity", + "rag.vector_search_results": 6, + "rag.context_synthesis": "multi_source", + "rag.document_coverage": 1.2, + "rag.information_density": 0.29198606271777006 + }, + "events": [], + "links": [], + "trace_trace_id": "b90ef3fc-11ea-4230-a59b-e2f2db8f2de0", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:48:08.007000000", + "trace_end_time": "2025-10-16 08:48:16.975000000", + "trace_duration_ms": 8967.264, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:48:08.007000000", + "updated_at": "2025-10-16 08:48:08.007000000" + }, + { + "span_id": "f049ed2b-ff2f-4092-9974-acc66f81bd8d", + "trace_id": "b8abebac-a7dc-494c-a1eb-6470181f32d2", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-16 08:47:57.667000000", + "end_time": "2025-10-16 08:48:07.005000000", + "duration_ms": 9337.298, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What features does Noveum Trace offer?", + "query.length": 38, + "processing.mode": "RAG", + "processing.time_seconds": 9.33726453781128, + "processing.response_length": 1018, + "processing.sources_count": 5, + "output_response": "Final Answer: Noveum Trace offers several key features through its Python SDK (`noveum-trace`) and TypeScript SDK (`@noveum/trace`):\n\n1. **Decorator-Based Tracing**: It provides a decorator-based approach for traci...", + "final_answer_mode": "RAG", + "query_processed.input_query": "What features does Noveum Trace offer?", + "query_processed.output_response": "Successfully processed query using RAG, generated 1018 character response", + "query_processed.mode": "RAG", + "query_processed.processing_time": 9.33726453781128, + "query_processed.response_length": 1018 + }, + "events": [], + "links": [], + "trace_trace_id": "b8abebac-a7dc-494c-a1eb-6470181f32d2", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:47:57.667000000", + "trace_end_time": "2025-10-16 08:48:07.005000000", + "trace_duration_ms": 9337.405, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:47:57.667000000", + "updated_at": "2025-10-16 08:47:57.667000000" + }, + { + "span_id": "211f1fe7-4cfb-4c1c-9c72-62ae2825dda3", + "trace_id": "b8abebac-a7dc-494c-a1eb-6470181f32d2", + "parent_span_id": "f049ed2b-ff2f-4092-9974-acc66f81bd8d", + "name": "agent.query_routing", + "start_time": "2025-10-16 08:47:57.668000000", + "end_time": "2025-10-16 08:48:07.005000000", + "duration_ms": 9337.223, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What features does Noveum Trace offer?", + "query_length": 38, + "query_type": "routing_query", + "output_response": "Routed to RAG for query processing", + "router.classification": "RAG", + "router.confidence_score": 1, + "router.rag_keyword_score": 3, + "router.web_keyword_score": 0, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "b8abebac-a7dc-494c-a1eb-6470181f32d2", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:47:57.667000000", + "trace_end_time": "2025-10-16 08:48:07.005000000", + "trace_duration_ms": 9337.405, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:47:57.667000000", + "updated_at": "2025-10-16 08:47:57.667000000" + }, + { + "span_id": "976c0c4b-efc1-4b7d-bfad-350c7fd0f71e", + "trace_id": "b8abebac-a7dc-494c-a1eb-6470181f32d2", + "parent_span_id": "211f1fe7-4cfb-4c1c-9c72-62ae2825dda3", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-16 08:47:57.668000000", + "end_time": "2025-10-16 08:47:57.668000000", + "duration_ms": 0.028, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What features does Noveum Trace offer?", + "query_length": 38, + "query_type": "routing_evaluation_query", + "classification.mode": "RAG", + "classification.rag_keyword_score": 3, + "classification.web_keyword_score": 0, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "noveum_specific", + "query.keyword_density": 0.5, + "query.domain_affinity": "noveum", + "routing.decision": "Routed to RAG based on analysis", + "routing.rationale": "RAG score: 3, Web score: 0, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "Web Search", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.057692307692307696, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: RAG (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "b8abebac-a7dc-494c-a1eb-6470181f32d2", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:47:57.667000000", + "trace_end_time": "2025-10-16 08:48:07.005000000", + "trace_duration_ms": 9337.405, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:47:57.667000000", + "updated_at": "2025-10-16 08:47:57.667000000" + }, + { + "span_id": "d2afaf74-f63b-4141-9fa3-662162f51cc0", + "trace_id": "b8abebac-a7dc-494c-a1eb-6470181f32d2", + "parent_span_id": "211f1fe7-4cfb-4c1c-9c72-62ae2825dda3", + "name": "agent.llm-rag", + "start_time": "2025-10-16 08:47:57.668000000", + "end_time": "2025-10-16 08:48:07.005000000", + "duration_ms": 9337.056, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "rag_agent", + "agent.operation": "llm-rag", + "agent.capabilities": [ + "document_retrieval", + "context_generation", + "response_generation" + ], + "agent.id": "noveum_rag_agent", + "input_query": "What features does Noveum Trace offer?", + "query_length": 38, + "query_type": "rag_query", + "output_response": "RAG Answer: Noveum Trace offers several key features through its Python SDK (`noveum-trace`) and TypeScript SDK (`@noveum/trace`):\n\n1. **Decorator-Based Tracing**: It provides a decorator-based approach for traci...", + "rag.context_length": 2787, + "rag.sources_count": 5, + "rag.answer_length": 1018, + "rag.mode": "retrieval_augmented_generation" + }, + "events": [], + "links": [], + "trace_trace_id": "b8abebac-a7dc-494c-a1eb-6470181f32d2", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:47:57.667000000", + "trace_end_time": "2025-10-16 08:48:07.005000000", + "trace_duration_ms": 9337.405, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:47:57.667000000", + "updated_at": "2025-10-16 08:47:57.667000000" + }, + { + "span_id": "09811a73-405d-4c0d-ba5d-0472b3708062", + "trace_id": "b8abebac-a7dc-494c-a1eb-6470181f32d2", + "parent_span_id": "d2afaf74-f63b-4141-9fa3-662162f51cc0", + "name": "agent.llm_model_execution", + "start_time": "2025-10-16 08:47:58.659000000", + "end_time": "2025-10-16 08:48:05.878000000", + "duration_ms": 7218.618, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What features does Noveum Trace offer?", + "query_length": 38, + "query_type": "rag_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 7.218573331832886, + "model.latency_ms": 7218.573331832886, + "model.start_time": 1760604478.659586, + "model.end_time": 1760604485.8781593, + "model.performance_tier": "slow", + "model.prompt_tokens": 830, + "model.completion_tokens": 254, + "model.total_tokens": 1084, + "model.tokens_per_second": 150.16817730724068, + "model.estimated_cost": 0.03252, + "model.efficiency_score": 0.9391143911439115, + "model.response_length": 1018, + "model.response_quality": "high", + "model.output_response": "Model Response: Noveum Trace offers several key features through its Python SDK (`noveum-trace`) and TypeScript SDK (`@noveum/trace`):\n\n1. **Decorator-Based Tracing**: It provides a decorator-based approach for traci...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "b8abebac-a7dc-494c-a1eb-6470181f32d2", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:47:57.667000000", + "trace_end_time": "2025-10-16 08:48:07.005000000", + "trace_duration_ms": 9337.405, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:47:57.667000000", + "updated_at": "2025-10-16 08:47:57.667000000" + }, + { + "span_id": "015688fe-d287-48e8-ac59-63781ffd6f89", + "trace_id": "b8abebac-a7dc-494c-a1eb-6470181f32d2", + "parent_span_id": "d2afaf74-f63b-4141-9fa3-662162f51cc0", + "name": "agent.rag_evaluation_metrics", + "start_time": "2025-10-16 08:48:05.878000000", + "end_time": "2025-10-16 08:48:05.878000000", + "duration_ms": 0.024, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "rag_evaluation_metrics", + "agent.capabilities": [ + "retrieval_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What features does Noveum Trace offer?", + "query_length": 38, + "query_type": "rag_evaluation_query", + "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\n\ud83d\udcd8 TypeScript SDK (\n@noveum/trace\n)- Framework ...", + "retrieval.context_length": 2787, + "retrieval.sources_count": 5, + "retrieval.context_quality": "high", + "retrieval.effectiveness": 1, + "retrieval.context_utilization": 2.787, + "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\n\ud83d\udcd8 TypeScript SDK (\n@noveum/trace\n)- Framework integrations for Next.js, Express.js, and Hono\n- TypeScript-first design with full type safety\n- Universal compatibility (Node.js, Edge Runtime, browsers)\n-\n\ud83d\udcca Noveum Platform\n- Real-time dashboard for analyzing traces and performance\n- Ad...\n\nSource 2 (https://noveum.ai/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\n\ud83d\udcd8 TypeScript SDK (\n@noveum/trace\n)- Framework integrations for Next.js, Express.js, and Hono\n- TypeScript-first design with full type safety\n- Universal compatibility (Node.js, Edge Runtime, browsers)\n-\n\ud83d\udcca Noveum Platform\n- Real-time dashboard for analyzing traces and performance\n- Ad...\n\nSource 3 (https://noveum.ai/en/docs/getting-started/sdk-integration):\n- \ud83d\udc65 Collaborate with your team on insights\n\ud83d\udd12 Security & Privacy\n- \ud83d\udd10 Encryption: All data encrypted in transit and at rest\n- \ud83c\udf9b\ufe0f Configurable Capture: Control what data is collected\n- \ud83c\udfe0 Data Residency: Choose your data storage region\n- \u23f0 Retention Control: Set custom data retention policies\nNext Steps\n- Tracing Concepts - Learn about traces, spans, and observability best practices\n- Framework Integrations - Deep dive into specific framework setups\n- Multi-Agent Tracing - Observe complex agent work...\n\nSource 4 (https://noveum.ai/docs/getting-started/sdk-integration):\n- \ud83d\udc65 Collaborate with your team on insights\n\ud83d\udd12 Security & Privacy\n- \ud83d\udd10 Encryption: All data encrypted in transit and at rest\n- \ud83c\udf9b\ufe0f Configurable Capture: Control what data is collected\n- \ud83c\udfe0 Data Residency: Choose your data storage region\n- \u23f0 Retention Control: Set custom data retention policies\nNext Steps\n- Tracing Concepts - Learn about traces, spans, and observability best practices\n- Framework Integrations - Deep dive into specific framework setups\n- Multi-Agent Tracing - Observe complex agent work...\n\nSource 5 (https://noveum.ai/docs/platform/dashboard):\n- Bulk Export: Download multiple traces simultaneously\n- Comparative Analysis: Compare performance across multiple traces\nReal-Time Updates\n- Live Refresh: Automatic updates as new traces arrive\n- Connection Monitoring: Real-time status of your trace ingestion pipeline\n- Performance Indicators: Live metrics for system health monitoring\nReady to dive deeper? Explore Projects & Environments to organize your AI applications, or check out Team Collaboration to share insights with your team.\nGet Earl...\n\nUser Question: What features does Noveum Trace offer?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", + "prompt.prompt_length": 3321, + "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frame...", + "prompt.instruction_following": "rag_optimized", + "response.answer_length": 1018, + "response.answer_completeness": "complete", + "response.response_quality": "high", + "response.source_citation": 5, + "response.context_utilization": 0.36526731252242556, + "output_response": "RAG Answer: Noveum Trace offers several key features through its Python SDK (`noveum-trace`) and TypeScript SDK (`@noveum/trace`):\n\n1. **Decorator-Based Tracing**: It provides a decorator-based approach for traci...", + "evaluation.retrieval_effectiveness": 1, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 5, + "evaluation.context_relevance": "high", + "evaluation.overall_quality": "high", + "evaluation.ready_for_production": true, + "rag.retrieval_strategy": "semantic_similarity", + "rag.vector_search_results": 5, + "rag.context_synthesis": "multi_source", + "rag.document_coverage": 1, + "rag.information_density": 0.36526731252242556 + }, + "events": [], + "links": [], + "trace_trace_id": "b8abebac-a7dc-494c-a1eb-6470181f32d2", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:47:57.667000000", + "trace_end_time": "2025-10-16 08:48:07.005000000", + "trace_duration_ms": 9337.405, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:47:57.667000000", + "updated_at": "2025-10-16 08:47:57.667000000" + }, + { + "span_id": "46ee4d00-f914-41d8-9f91-c2db85f1c613", + "trace_id": "be9ccbaf-ccb4-4ebd-a380-4247f4d01868", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-16 08:47:52.037000000", + "end_time": "2025-10-16 08:47:56.662000000", + "duration_ms": 4625.395, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What are Noveum's pricing plans?", + "query.length": 32, + "processing.mode": "RAG", + "processing.time_seconds": 4.6253437995910645, + "processing.response_length": 471, + "processing.sources_count": 5, + "output_response": "Final Answer: The provided context does not contain specific information about Noveum's pricing plans. It mentions that all users will have access to the Observability suite for free, and early users will receive f...", + "final_answer_mode": "RAG", + "query_processed.input_query": "What are Noveum's pricing plans?", + "query_processed.output_response": "Successfully processed query using RAG, generated 471 character response", + "query_processed.mode": "RAG", + "query_processed.processing_time": 4.6253437995910645, + "query_processed.response_length": 471 + }, + "events": [], + "links": [], + "trace_trace_id": "be9ccbaf-ccb4-4ebd-a380-4247f4d01868", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:47:52.037000000", + "trace_end_time": "2025-10-16 08:47:56.662000000", + "trace_duration_ms": 4625.492, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:47:52.037000000", + "updated_at": "2025-10-16 08:47:52.037000000" + }, + { + "span_id": "6d15369a-ae35-45c7-84dd-bdf484f73f2d", + "trace_id": "be9ccbaf-ccb4-4ebd-a380-4247f4d01868", + "parent_span_id": "46ee4d00-f914-41d8-9f91-c2db85f1c613", + "name": "agent.query_routing", + "start_time": "2025-10-16 08:47:52.037000000", + "end_time": "2025-10-16 08:47:56.662000000", + "duration_ms": 4625.286, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What are Noveum's pricing plans?", + "query_length": 32, + "query_type": "routing_query", + "output_response": "Routed to RAG for query processing", + "router.classification": "RAG", + "router.confidence_score": 1, + "router.rag_keyword_score": 3, + "router.web_keyword_score": 0, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "be9ccbaf-ccb4-4ebd-a380-4247f4d01868", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:47:52.037000000", + "trace_end_time": "2025-10-16 08:47:56.662000000", + "trace_duration_ms": 4625.492, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:47:52.037000000", + "updated_at": "2025-10-16 08:47:52.037000000" + }, + { + "span_id": "bf201676-a3a9-433a-a8c6-d394b7aead3a", + "trace_id": "be9ccbaf-ccb4-4ebd-a380-4247f4d01868", + "parent_span_id": "6d15369a-ae35-45c7-84dd-bdf484f73f2d", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-16 08:47:52.037000000", + "end_time": "2025-10-16 08:47:52.037000000", + "duration_ms": 0.048, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What are Noveum's pricing plans?", + "query_length": 32, + "query_type": "routing_evaluation_query", + "classification.mode": "RAG", + "classification.rag_keyword_score": 3, + "classification.web_keyword_score": 0, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "noveum_specific", + "query.keyword_density": 0.6, + "query.domain_affinity": "noveum", + "routing.decision": "Routed to RAG based on analysis", + "routing.rationale": "RAG score: 3, Web score: 0, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "Web Search", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.057692307692307696, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: RAG (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "be9ccbaf-ccb4-4ebd-a380-4247f4d01868", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:47:52.037000000", + "trace_end_time": "2025-10-16 08:47:56.662000000", + "trace_duration_ms": 4625.492, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:47:52.037000000", + "updated_at": "2025-10-16 08:47:52.037000000" + }, + { + "span_id": "f8346f60-131b-4007-ba3e-7b0ff10cdfde", + "trace_id": "be9ccbaf-ccb4-4ebd-a380-4247f4d01868", + "parent_span_id": "6d15369a-ae35-45c7-84dd-bdf484f73f2d", + "name": "agent.llm-rag", + "start_time": "2025-10-16 08:47:52.037000000", + "end_time": "2025-10-16 08:47:56.662000000", + "duration_ms": 4624.998, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "rag_agent", + "agent.operation": "llm-rag", + "agent.capabilities": [ + "document_retrieval", + "context_generation", + "response_generation" + ], + "agent.id": "noveum_rag_agent", + "input_query": "What are Noveum's pricing plans?", + "query_length": 32, + "query_type": "rag_query", + "output_response": "RAG Answer: The provided context does not contain specific information about Noveum's pricing plans. It mentions that all users will have access to the Observability suite for free, and early users will receive f...", + "rag.context_length": 2105, + "rag.sources_count": 5, + "rag.answer_length": 471, + "rag.mode": "retrieval_augmented_generation" + }, + "events": [], + "links": [], + "trace_trace_id": "be9ccbaf-ccb4-4ebd-a380-4247f4d01868", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:47:52.037000000", + "trace_end_time": "2025-10-16 08:47:56.662000000", + "trace_duration_ms": 4625.492, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:47:52.037000000", + "updated_at": "2025-10-16 08:47:52.037000000" + }, + { + "span_id": "0186c28c-7596-4279-a4eb-7733cd665609", + "trace_id": "be9ccbaf-ccb4-4ebd-a380-4247f4d01868", + "parent_span_id": "f8346f60-131b-4007-ba3e-7b0ff10cdfde", + "name": "agent.llm_model_execution", + "start_time": "2025-10-16 08:47:52.579000000", + "end_time": "2025-10-16 08:47:55.842000000", + "duration_ms": 3262.5829999999996, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What are Noveum's pricing plans?", + "query_length": 32, + "query_type": "rag_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 3.2625203132629395, + "model.latency_ms": 3262.5203132629395, + "model.start_time": 1760604472.580008, + "model.end_time": 1760604475.8425283, + "model.performance_tier": "medium", + "model.prompt_tokens": 658, + "model.completion_tokens": 117, + "model.total_tokens": 775, + "model.tokens_per_second": 237.54641368804243, + "model.estimated_cost": 0.02325, + "model.efficiency_score": 0.607741935483871, + "model.response_length": 471, + "model.response_quality": "high", + "model.output_response": "Model Response: The provided context does not contain specific information about Noveum's pricing plans. It mentions that all users will have access to the Observability suite for free, and early users will receive f...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "be9ccbaf-ccb4-4ebd-a380-4247f4d01868", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:47:52.037000000", + "trace_end_time": "2025-10-16 08:47:56.662000000", + "trace_duration_ms": 4625.492, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:47:52.037000000", + "updated_at": "2025-10-16 08:47:52.037000000" + }, + { + "span_id": "3bdf8d5b-c10d-44f3-8514-8facc7f63db8", + "trace_id": "be9ccbaf-ccb4-4ebd-a380-4247f4d01868", + "parent_span_id": "f8346f60-131b-4007-ba3e-7b0ff10cdfde", + "name": "agent.rag_evaluation_metrics", + "start_time": "2025-10-16 08:47:55.842000000", + "end_time": "2025-10-16 08:47:55.842000000", + "duration_ms": 0.032, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "rag_evaluation_metrics", + "agent.capabilities": [ + "retrieval_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What are Noveum's pricing plans?", + "query_length": 32, + "query_type": "rag_evaluation_query", + "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....", + "retrieval.context_length": 2105, + "retrieval.sources_count": 5, + "retrieval.context_quality": "high", + "retrieval.effectiveness": 1, + "retrieval.context_utilization": 2.105, + "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nSource 2 (https://noveum.ai/en/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nSource 3 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nThe AI landscape is complex and moving fast. With Noveum.ai, you'll have the insights you need to build, debug, and optimize AI applications with confidence.\nReady to see what's really happening inside your AI applications? Try Noveum.ai today and transform how you build and monitor AI systems.\nLet's build more reliable, observable AI\u2014together.\nGet Early Access to Noveum.ai Platform\nJoin the select group of AI teams optimizing their models with our data-driven platform. We're onboarding users in...\n\nSource 4 (https://noveum.ai/en/blog/from-logs-to-intelligent-choices-inside-noveum-ais-evaluation-process):\nGet Early Access to Noveum.ai Platform\nJoin the select group of AI teams optimizing their models with our data-driven platform. We're onboarding users in limited batches to ensure a premium experience....\n\nSource 5 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, vector searches, data retrieval, agent reasoning, and more. Without proper observability, debugging becomes a nightmare, optimization is guesswork, and understanding user interactions is nearly impossible.\nNoveum.ai solves this by providing comprehensive tracing and observability specifically designed for AI applications.\nKey benefits of using Noveum.ai:\n- Complete Visibility: Trace...\n\nUser Question: What are Noveum's pricing plans?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", + "prompt.prompt_length": 2633, + "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to ...", + "prompt.instruction_following": "rag_optimized", + "response.answer_length": 471, + "response.answer_completeness": "complete", + "response.response_quality": "high", + "response.source_citation": 5, + "response.context_utilization": 0.22375296912114015, + "output_response": "RAG Answer: The provided context does not contain specific information about Noveum's pricing plans. It mentions that all users will have access to the Observability suite for free, and early users will receive f...", + "evaluation.retrieval_effectiveness": 1, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 5, + "evaluation.context_relevance": "high", + "evaluation.overall_quality": "high", + "evaluation.ready_for_production": true, + "rag.retrieval_strategy": "semantic_similarity", + "rag.vector_search_results": 5, + "rag.context_synthesis": "multi_source", + "rag.document_coverage": 1, + "rag.information_density": 0.22375296912114015 + }, + "events": [], + "links": [], + "trace_trace_id": "be9ccbaf-ccb4-4ebd-a380-4247f4d01868", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:47:52.037000000", + "trace_end_time": "2025-10-16 08:47:56.662000000", + "trace_duration_ms": 4625.492, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:47:52.037000000", + "updated_at": "2025-10-16 08:47:52.037000000" + }, + { + "span_id": "770ce93d-07ae-4b9f-ae15-c6e450634b9d", + "trace_id": "8d33a529-d9b1-4208-b528-762222d5ef8c", + "parent_span_id": "9977d5d6-a7ff-469e-bec7-5e4dfc38c149", + "name": "agent.llm-rag", + "start_time": "2025-10-16 08:47:42.103000000", + "end_time": "2025-10-16 08:47:51.032000000", + "duration_ms": 8928.644999999999, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "rag_agent", + "agent.operation": "llm-rag", + "agent.capabilities": [ + "document_retrieval", + "context_generation", + "response_generation" + ], + "agent.id": "noveum_rag_agent", + "input_query": "How do I integrate Noveum Trace in my application?", + "query_length": 50, + "query_type": "rag_query", + "output_response": "RAG Answer: To integrate Noveum Trace into your application, you can use either the Python SDK (`noveum-trace`) or the TypeScript SDK (`@noveum/trace`). Here\u2019s a quick start guide based on the documentation:\n\n1. ...", + "rag.context_length": 2803, + "rag.sources_count": 5, + "rag.answer_length": 1134, + "rag.mode": "retrieval_augmented_generation" + }, + "events": [], + "links": [], + "trace_trace_id": "8d33a529-d9b1-4208-b528-762222d5ef8c", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:47:42.103000000", + "trace_end_time": "2025-10-16 08:47:51.032000000", + "trace_duration_ms": 8928.948, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:47:42.103000000", + "updated_at": "2025-10-16 08:47:42.103000000" + }, + { + "span_id": "9977d5d6-a7ff-469e-bec7-5e4dfc38c149", + "trace_id": "8d33a529-d9b1-4208-b528-762222d5ef8c", + "parent_span_id": "f3a8fa53-5641-4a23-b4d3-762577a89151", + "name": "agent.query_routing", + "start_time": "2025-10-16 08:47:42.103000000", + "end_time": "2025-10-16 08:47:51.032000000", + "duration_ms": 8928.793000000001, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "How do I integrate Noveum Trace in my application?", + "query_length": 50, + "query_type": "routing_query", + "output_response": "Routed to RAG for query processing", + "router.classification": "RAG", + "router.confidence_score": 1, + "router.rag_keyword_score": 2, + "router.web_keyword_score": 0, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "8d33a529-d9b1-4208-b528-762222d5ef8c", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:47:42.103000000", + "trace_end_time": "2025-10-16 08:47:51.032000000", + "trace_duration_ms": 8928.948, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:47:42.103000000", + "updated_at": "2025-10-16 08:47:42.103000000" + }, + { + "span_id": "c678e222-ef74-48ee-8628-16461ab0f48f", + "trace_id": "8d33a529-d9b1-4208-b528-762222d5ef8c", + "parent_span_id": "9977d5d6-a7ff-469e-bec7-5e4dfc38c149", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-16 08:47:42.103000000", + "end_time": "2025-10-16 08:47:42.103000000", + "duration_ms": 0.027, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: How do I integrate Noveum Trace in my application?", + "query_length": 50, + "query_type": "routing_evaluation_query", + "classification.mode": "RAG", + "classification.rag_keyword_score": 2, + "classification.web_keyword_score": 0, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "noveum_specific", + "query.keyword_density": 0.2222222222222222, + "query.domain_affinity": "noveum", + "routing.decision": "Routed to RAG based on analysis", + "routing.rationale": "RAG score: 2, Web score: 0, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "Web Search", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.038461538461538464, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: RAG (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "8d33a529-d9b1-4208-b528-762222d5ef8c", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:47:42.103000000", + "trace_end_time": "2025-10-16 08:47:51.032000000", + "trace_duration_ms": 8928.948, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:47:42.103000000", + "updated_at": "2025-10-16 08:47:42.103000000" + }, + { + "span_id": "f3a8fa53-5641-4a23-b4d3-762577a89151", + "trace_id": "8d33a529-d9b1-4208-b528-762222d5ef8c", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-16 08:47:42.103000000", + "end_time": "2025-10-16 08:47:51.032000000", + "duration_ms": 8928.868, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "How do I integrate Noveum Trace in my application?", + "query.length": 50, + "processing.mode": "RAG", + "processing.time_seconds": 8.928829431533813, + "processing.response_length": 1134, + "processing.sources_count": 5, + "output_response": "Final Answer: To integrate Noveum Trace into your application, you can use either the Python SDK (`noveum-trace`) or the TypeScript SDK (`@noveum/trace`). Here\u2019s a quick start guide based on the documentation:\n\n1. ...", + "final_answer_mode": "RAG", + "query_processed.input_query": "How do I integrate Noveum Trace in my application?", + "query_processed.output_response": "Successfully processed query using RAG, generated 1134 character response", + "query_processed.mode": "RAG", + "query_processed.processing_time": 8.928829431533813, + "query_processed.response_length": 1134 + }, + "events": [], + "links": [], + "trace_trace_id": "8d33a529-d9b1-4208-b528-762222d5ef8c", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:47:42.103000000", + "trace_end_time": "2025-10-16 08:47:51.032000000", + "trace_duration_ms": 8928.948, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:47:42.103000000", + "updated_at": "2025-10-16 08:47:42.103000000" + }, + { + "span_id": "f459b71f-1d43-41b0-b9bf-d94a908b7afa", + "trace_id": "8d33a529-d9b1-4208-b528-762222d5ef8c", + "parent_span_id": "770ce93d-07ae-4b9f-ae15-c6e450634b9d", + "name": "agent.llm_model_execution", + "start_time": "2025-10-16 08:47:43.044000000", + "end_time": "2025-10-16 08:47:50.008000000", + "duration_ms": 6963.959, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: How do I integrate Noveum Trace in my application?", + "query_length": 50, + "query_type": "rag_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 6.9639081954956055, + "model.latency_ms": 6963.9081954956055, + "model.start_time": 1760604463.044064, + "model.end_time": 1760604470.0079722, + "model.performance_tier": "slow", + "model.prompt_tokens": 837, + "model.completion_tokens": 283, + "model.total_tokens": 1120, + "model.tokens_per_second": 160.8292310235276, + "model.estimated_cost": 0.0336, + "model.efficiency_score": 1.0125, + "model.response_length": 1134, + "model.response_quality": "high", + "model.output_response": "Model Response: To integrate Noveum Trace into your application, you can use either the Python SDK (`noveum-trace`) or the TypeScript SDK (`@noveum/trace`). Here\u2019s a quick start guide based on the documentation:\n\n1. ...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "8d33a529-d9b1-4208-b528-762222d5ef8c", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:47:42.103000000", + "trace_end_time": "2025-10-16 08:47:51.032000000", + "trace_duration_ms": 8928.948, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:47:42.103000000", + "updated_at": "2025-10-16 08:47:42.103000000" + }, + { + "span_id": "b8627171-b081-4b08-b900-c24375b73c3f", + "trace_id": "8d33a529-d9b1-4208-b528-762222d5ef8c", + "parent_span_id": "770ce93d-07ae-4b9f-ae15-c6e450634b9d", + "name": "agent.rag_evaluation_metrics", + "start_time": "2025-10-16 08:47:50.008000000", + "end_time": "2025-10-16 08:47:50.008000000", + "duration_ms": 0.041, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "rag_evaluation_metrics", + "agent.capabilities": [ + "retrieval_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: How do I integrate Noveum Trace in my application?", + "query_length": 50, + "query_type": "rag_evaluation_query", + "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\n\ud83d\udcd8 TypeScript SDK (\n@noveum/trace\n)- Framework ...", + "retrieval.context_length": 2803, + "retrieval.sources_count": 5, + "retrieval.context_quality": "high", + "retrieval.effectiveness": 1, + "retrieval.context_utilization": 2.803, + "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\n\ud83d\udcd8 TypeScript SDK (\n@noveum/trace\n)- Framework integrations for Next.js, Express.js, and Hono\n- TypeScript-first design with full type safety\n- Universal compatibility (Node.js, Edge Runtime, browsers)\n-\n\ud83d\udcca Noveum Platform\n- Real-time dashboard for analyzing traces and performance\n- Ad...\n\nSource 2 (https://noveum.ai/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\n\ud83d\udcd8 TypeScript SDK (\n@noveum/trace\n)- Framework integrations for Next.js, Express.js, and Hono\n- TypeScript-first design with full type safety\n- Universal compatibility (Node.js, Edge Runtime, browsers)\n-\n\ud83d\udcca Noveum Platform\n- Real-time dashboard for analyzing traces and performance\n- Ad...\n\nSource 3 (https://noveum.ai/docs/getting-started/sdk-integration):\nSDK Integration Guide\nIntegrate Noveum.ai tracing into your AI applications with Python or TypeScript SDKs\nThe Noveum.ai SDKs provide comprehensive tracing and observability for your AI applications with minimal code changes. Whether you're building LLM applications, RAG systems, or multi-agent workflows, our SDKs automatically capture essential metrics and traces.\n\ud83d\ude80 Quick Start\n1. Create Your Account & Get API Key\n- Sign up at noveum.ai\n- Create a project in your dashboard\n- Generate an API key...\n\nSource 4 (https://noveum.ai/en/docs/getting-started/sdk-integration):\nSDK Integration Guide\nIntegrate Noveum.ai tracing into your AI applications with Python or TypeScript SDKs\nThe Noveum.ai SDKs provide comprehensive tracing and observability for your AI applications with minimal code changes. Whether you're building LLM applications, RAG systems, or multi-agent workflows, our SDKs automatically capture essential metrics and traces.\n\ud83d\ude80 Quick Start\n1. Create Your Account & Get API Key\n- Sign up at noveum.ai\n- Create a project in your dashboard\n- Generate an API key...\n\nSource 5 (https://noveum.ai/en/docs/getting-started/sdk-integration):\n\ud83d\udd27 Framework Integrations\nNext.js Integration\nExpress.js Integration\nFastAPI Integration (Python)\n\ud83d\udcca Advanced Features\nCustom Attributes & Events\nSampling Configuration\n\ud83d\udd17 What's Captured Automatically\n- \ud83d\udcca Performance Metrics: Latency, throughput, error rates\n- \ud83d\udcb0 Cost Tracking: Token usage, API costs across providers\n- \ud83d\udd0d Request/Response: Configurable capture of inputs/outputs\n- \ud83c\udff7\ufe0f Metadata: Model names, parameters, user context\n- \ud83c\udf0a Context Flow: Trace relationships across services\n- \ud83d\udc1b Error Detail...\n\nUser Question: How do I integrate Noveum Trace in my application?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", + "prompt.prompt_length": 3349, + "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frame...", + "prompt.instruction_following": "rag_optimized", + "response.answer_length": 1134, + "response.answer_completeness": "complete", + "response.response_quality": "high", + "response.source_citation": 5, + "response.context_utilization": 0.4045665358544417, + "output_response": "RAG Answer: To integrate Noveum Trace into your application, you can use either the Python SDK (`noveum-trace`) or the TypeScript SDK (`@noveum/trace`). Here\u2019s a quick start guide based on the documentation:\n\n1. ...", + "evaluation.retrieval_effectiveness": 1, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 5, + "evaluation.context_relevance": "high", + "evaluation.overall_quality": "high", + "evaluation.ready_for_production": true, + "rag.retrieval_strategy": "semantic_similarity", + "rag.vector_search_results": 5, + "rag.context_synthesis": "multi_source", + "rag.document_coverage": 1, + "rag.information_density": 0.4045665358544417 + }, + "events": [], + "links": [], + "trace_trace_id": "8d33a529-d9b1-4208-b528-762222d5ef8c", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:47:42.103000000", + "trace_end_time": "2025-10-16 08:47:51.032000000", + "trace_duration_ms": 8928.948, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:47:42.103000000", + "updated_at": "2025-10-16 08:47:42.103000000" + }, + { + "span_id": "0b3d6893-776e-4118-b8b5-b1c5ea51def1", + "trace_id": "06192a09-b6bb-4548-8a94-e123d6caaffe", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-16 08:47:03.776000000", + "end_time": "2025-10-16 08:47:41.100000000", + "duration_ms": 37324.516, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What is Noveum and what does it do?", + "query.length": 35, + "processing.mode": "RAG", + "processing.time_seconds": 37.324477195739746, + "processing.response_length": 648, + "processing.sources_count": 5, + "output_response": "Final Answer: Noveum.ai is a comprehensive tracing and observability platform specifically designed for AI applications, including those powered by large language models (LLMs), retrieval-augmented generation (RAG)...", + "final_answer_mode": "RAG", + "query_processed.input_query": "What is Noveum and what does it do?", + "query_processed.output_response": "Successfully processed query using RAG, generated 648 character response", + "query_processed.mode": "RAG", + "query_processed.processing_time": 37.324477195739746, + "query_processed.response_length": 648 + }, + "events": [], + "links": [], + "trace_trace_id": "06192a09-b6bb-4548-8a94-e123d6caaffe", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:47:03.776000000", + "trace_end_time": "2025-10-16 08:47:41.100000000", + "trace_duration_ms": 37324.601, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:47:03.776000000", + "updated_at": "2025-10-16 08:47:03.776000000" + }, + { + "span_id": "137ad05b-bada-42be-abe5-be0c277f6d0e", + "trace_id": "06192a09-b6bb-4548-8a94-e123d6caaffe", + "parent_span_id": "c3664d20-1708-45f5-a051-b626b0ec77cd", + "name": "agent.llm-rag", + "start_time": "2025-10-16 08:47:03.776000000", + "end_time": "2025-10-16 08:47:41.100000000", + "duration_ms": 37324.042, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "rag_agent", + "agent.operation": "llm-rag", + "agent.capabilities": [ + "document_retrieval", + "context_generation", + "response_generation" + ], + "agent.id": "noveum_rag_agent", + "input_query": "What is Noveum and what does it do?", + "query_length": 35, + "query_type": "rag_query", + "output_response": "RAG Answer: Noveum.ai is a comprehensive tracing and observability platform specifically designed for AI applications, including those powered by large language models (LLMs), retrieval-augmented generation (RAG)...", + "rag.context_length": 2334, + "rag.sources_count": 5, + "rag.answer_length": 648, + "rag.mode": "retrieval_augmented_generation" + }, + "events": [], + "links": [], + "trace_trace_id": "06192a09-b6bb-4548-8a94-e123d6caaffe", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:47:03.776000000", + "trace_end_time": "2025-10-16 08:47:41.100000000", + "trace_duration_ms": 37324.601, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:47:03.776000000", + "updated_at": "2025-10-16 08:47:03.776000000" + }, + { + "span_id": "c3664d20-1708-45f5-a051-b626b0ec77cd", + "trace_id": "06192a09-b6bb-4548-8a94-e123d6caaffe", + "parent_span_id": "0b3d6893-776e-4118-b8b5-b1c5ea51def1", + "name": "agent.query_routing", + "start_time": "2025-10-16 08:47:03.776000000", + "end_time": "2025-10-16 08:47:41.100000000", + "duration_ms": 37324.382999999994, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What is Noveum and what does it do?", + "query_length": 35, + "query_type": "routing_query", + "output_response": "Routed to RAG for query processing", + "router.classification": "RAG", + "router.confidence_score": 1, + "router.rag_keyword_score": 1, + "router.web_keyword_score": 0, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "06192a09-b6bb-4548-8a94-e123d6caaffe", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:47:03.776000000", + "trace_end_time": "2025-10-16 08:47:41.100000000", + "trace_duration_ms": 37324.601, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:47:03.776000000", + "updated_at": "2025-10-16 08:47:03.776000000" + }, + { + "span_id": "ce9a94bf-1a72-4599-8561-11e66802b92d", + "trace_id": "06192a09-b6bb-4548-8a94-e123d6caaffe", + "parent_span_id": "c3664d20-1708-45f5-a051-b626b0ec77cd", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-16 08:47:03.776000000", + "end_time": "2025-10-16 08:47:03.776000000", + "duration_ms": 0.067, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What is Noveum and what does it do?", + "query_length": 35, + "query_type": "routing_evaluation_query", + "classification.mode": "RAG", + "classification.rag_keyword_score": 1, + "classification.web_keyword_score": 0, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "noveum_specific", + "query.keyword_density": 0.125, + "query.domain_affinity": "noveum", + "routing.decision": "Routed to RAG based on analysis", + "routing.rationale": "RAG score: 1, Web score: 0, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "Web Search", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.019230769230769232, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: RAG (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "06192a09-b6bb-4548-8a94-e123d6caaffe", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:47:03.776000000", + "trace_end_time": "2025-10-16 08:47:41.100000000", + "trace_duration_ms": 37324.601, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:47:03.776000000", + "updated_at": "2025-10-16 08:47:03.776000000" + }, + { + "span_id": "38414416-907a-4188-95fe-c0a5f900f72c", + "trace_id": "06192a09-b6bb-4548-8a94-e123d6caaffe", + "parent_span_id": "137ad05b-bada-42be-abe5-be0c277f6d0e", + "name": "agent.llm_model_execution", + "start_time": "2025-10-16 08:47:10.392000000", + "end_time": "2025-10-16 08:47:40.392000000", + "duration_ms": 30000.104, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What is Noveum and what does it do?", + "query_length": 35, + "query_type": "rag_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 30.000043392181396, + "model.latency_ms": 30000.043392181396, + "model.start_time": 1760604430.3921378, + "model.end_time": 1760604460.3921812, + "model.performance_tier": "slow", + "model.prompt_tokens": 716, + "model.completion_tokens": 162, + "model.total_tokens": 878, + "model.tokens_per_second": 29.266624335244266, + "model.estimated_cost": 0.026340000000000002, + "model.efficiency_score": 0.7380410022779044, + "model.response_length": 648, + "model.response_quality": "high", + "model.output_response": "Model Response: Noveum.ai is a comprehensive tracing and observability platform specifically designed for AI applications, including those powered by large language models (LLMs), retrieval-augmented generation (RAG)...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "06192a09-b6bb-4548-8a94-e123d6caaffe", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:47:03.776000000", + "trace_end_time": "2025-10-16 08:47:41.100000000", + "trace_duration_ms": 37324.601, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:47:03.776000000", + "updated_at": "2025-10-16 08:47:03.776000000" + }, + { + "span_id": "67e23649-b00b-4eb8-9a69-865e80f833b3", + "trace_id": "06192a09-b6bb-4548-8a94-e123d6caaffe", + "parent_span_id": "137ad05b-bada-42be-abe5-be0c277f6d0e", + "name": "agent.rag_evaluation_metrics", + "start_time": "2025-10-16 08:47:40.392000000", + "end_time": "2025-10-16 08:47:40.392000000", + "duration_ms": 0.029, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "rag_evaluation_metrics", + "agent.capabilities": [ + "retrieval_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What is Noveum and what does it do?", + "query_length": 35, + "query_type": "rag_evaluation_query", + "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, vector searches, data retrieval, agent reasoning, and more. Without proper observability, debugging ...", + "retrieval.context_length": 2334, + "retrieval.sources_count": 5, + "retrieval.context_quality": "high", + "retrieval.effectiveness": 1, + "retrieval.context_utilization": 2.334, + "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, vector searches, data retrieval, agent reasoning, and more. Without proper observability, debugging becomes a nightmare, optimization is guesswork, and understanding user interactions is nearly impossible.\nNoveum.ai solves this by providing comprehensive tracing and observability specifically designed for AI applications.\nKey benefits of using Noveum.ai:\n- Complete Visibility: Trace...\n\nSource 2 (https://noveum.ai/en/docs/getting-started/overview):\nNoveum.ai Overview\nComprehensive AI tracing and observability platform for LLM applications, RAG systems, and AI agents\nWelcome to Noveum.ai\u2014the comprehensive tracing and observability platform built specifically for AI applications. Whether you're building LLM-powered chatbots, RAG systems, multi-agent workflows, or any AI-driven application, Noveum provides the insights you need to understand, debug, and optimize your systems.\n\ud83c\udfaf Why AI Applications Need Specialized Observability\nTraditional mo...\n\nSource 3 (https://noveum.ai/docs/getting-started/overview):\nNoveum.ai Overview\nComprehensive AI tracing and observability platform for LLM applications, RAG systems, and AI agents\nWelcome to Noveum.ai\u2014the comprehensive tracing and observability platform built specifically for AI applications. Whether you're building LLM-powered chatbots, RAG systems, multi-agent workflows, or any AI-driven application, Noveum provides the insights you need to understand, debug, and optimize your systems.\n\ud83c\udfaf Why AI Applications Need Specialized Observability\nTraditional mo...\n\nSource 4 (https://noveum.ai/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nSource 5 (https://noveum.ai/en/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nUser Question: What is Noveum and what does it do?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", + "prompt.prompt_length": 2865, + "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, ...", + "prompt.instruction_following": "rag_optimized", + "response.answer_length": 648, + "response.answer_completeness": "complete", + "response.response_quality": "high", + "response.source_citation": 5, + "response.context_utilization": 0.2776349614395887, + "output_response": "RAG Answer: Noveum.ai is a comprehensive tracing and observability platform specifically designed for AI applications, including those powered by large language models (LLMs), retrieval-augmented generation (RAG)...", + "evaluation.retrieval_effectiveness": 1, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 5, + "evaluation.context_relevance": "high", + "evaluation.overall_quality": "high", + "evaluation.ready_for_production": true, + "rag.retrieval_strategy": "semantic_similarity", + "rag.vector_search_results": 5, + "rag.context_synthesis": "multi_source", + "rag.document_coverage": 1, + "rag.information_density": 0.2776349614395887 + }, + "events": [], + "links": [], + "trace_trace_id": "06192a09-b6bb-4548-8a94-e123d6caaffe", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:47:03.776000000", + "trace_end_time": "2025-10-16 08:47:41.100000000", + "trace_duration_ms": 37324.601, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:47:03.776000000", + "updated_at": "2025-10-16 08:47:03.776000000" + }, + { + "span_id": "050851c9-b2d3-4c82-accb-4085dc69f785", + "trace_id": "3552ee8e-eafb-499a-b109-a9fb851a41d2", + "parent_span_id": "", + "name": "system_initialization", + "start_time": "2025-10-16 08:47:03.695000000", + "end_time": "2025-10-16 08:47:03.770000000", + "duration_ms": 74.58, + "status": "ok", + "status_message": "", + "attributes": { + "system.force_scrape": false, + "system.config": { + "noveum_base_url": "https://noveum.ai", + "max_pages_to_scrape": 50, + "chunk_size": 1000, + "chunk_overlap": 200, + "max_search_results": 5, + "rag_threshold": 0.7, + "noveum_docs_file": "noveum_docs.json", + "vector_store_path": "noveum_vectorstore" + }, + "input_query": "Initialize system with force_scrape=False", + "output_response": "System initialization: RAG system loaded, vector store ready, agent operational", + "system.initialized": true, + "system.documents_loaded": 38, + "system.vectorstore_ready": true + }, + "events": [], + "links": [], + "trace_trace_id": "3552ee8e-eafb-499a-b109-a9fb851a41d2", + "trace_name": "auto_trace_system_initialization", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:47:03.695000000", + "trace_end_time": "2025-10-16 08:47:03.770000000", + "trace_duration_ms": 74.912, + "span_count": 1, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:47:03.695000000", + "updated_at": "2025-10-16 08:47:03.695000000" + }, + { + "span_id": "6b53235d-8b91-4d9e-8437-915828e84600", + "trace_id": "e6ef2449-7429-465b-9bff-f3def8de5bb0", + "parent_span_id": "e73ab184-50c1-4209-9276-be11e69e9513", + "name": "agent.query_routing", + "start_time": "2025-10-14 12:54:06.700000000", + "end_time": "2025-10-14 12:54:10.637000000", + "duration_ms": 3937.492, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What are the latest breakthroughs in artificial intelligence?", + "query_length": 61, + "query_type": "routing_query", + "output_response": "Routed to Web Search for query processing", + "router.classification": "Web Search", + "router.confidence_score": 1, + "router.rag_keyword_score": 0, + "router.web_keyword_score": 1, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "e6ef2449-7429-465b-9bff-f3def8de5bb0", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-14 12:54:06.700000000", + "trace_end_time": "2025-10-14 12:54:10.637000000", + "trace_duration_ms": 3937.729, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-14 12:54:06.700000000", + "updated_at": "2025-10-14 12:54:06.700000000" + }, + { + "span_id": "a7d1e33a-95a1-41a9-a117-62b4c373bd60", + "trace_id": "e6ef2449-7429-465b-9bff-f3def8de5bb0", + "parent_span_id": "6b53235d-8b91-4d9e-8437-915828e84600", + "name": "agent.web_search_generation", + "start_time": "2025-10-14 12:54:06.700000000", + "end_time": "2025-10-14 12:54:10.637000000", + "duration_ms": 3936.983, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "web_search_agent", + "agent.operation": "web_search_generation", + "agent.capabilities": [ + "web_search", + "content_synthesis", + "response_generation" + ], + "agent.id": "noveum_web_search_agent", + "input_query": "What are the latest breakthroughs in artificial intelligence?", + "query_length": 61, + "query_type": "web_search_query", + "output_response": "Web Search Answer: The web search did not yield specific results regarding the latest breakthroughs in artificial intelligence. However, I can provide a general overview based on knowledge up to October 2023.\n\nRecent br...", + "web_search.results_count": 1, + "web_search.context_length": 163, + "web_search.response_length": 1090, + "web_search.mode": "external_web_search" + }, + "events": [], + "links": [], + "trace_trace_id": "e6ef2449-7429-465b-9bff-f3def8de5bb0", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-14 12:54:06.700000000", + "trace_end_time": "2025-10-14 12:54:10.637000000", + "trace_duration_ms": 3937.729, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-14 12:54:06.700000000", + "updated_at": "2025-10-14 12:54:06.700000000" + }, + { + "span_id": "da630cab-2ba6-4f12-98b9-6ab1c0899c17", + "trace_id": "e6ef2449-7429-465b-9bff-f3def8de5bb0", + "parent_span_id": "6b53235d-8b91-4d9e-8437-915828e84600", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-14 12:54:06.700000000", + "end_time": "2025-10-14 12:54:06.700000000", + "duration_ms": 0.27999999999999997, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What are the latest breakthroughs in artificial intelligence?", + "query_length": 61, + "query_type": "routing_evaluation_query", + "classification.mode": "Web Search", + "classification.rag_keyword_score": 0, + "classification.web_keyword_score": 1, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "complex", + "query.intent": "general_knowledge", + "query.keyword_density": 0.125, + "query.domain_affinity": "general", + "routing.decision": "Routed to Web Search based on analysis", + "routing.rationale": "RAG score: 0, Web score: 1, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "RAG", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.019230769230769232, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: Web Search (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "e6ef2449-7429-465b-9bff-f3def8de5bb0", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-14 12:54:06.700000000", + "trace_end_time": "2025-10-14 12:54:10.637000000", + "trace_duration_ms": 3937.729, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-14 12:54:06.700000000", + "updated_at": "2025-10-14 12:54:06.700000000" + }, + { + "span_id": "e73ab184-50c1-4209-9276-be11e69e9513", + "trace_id": "e6ef2449-7429-465b-9bff-f3def8de5bb0", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-14 12:54:06.700000000", + "end_time": "2025-10-14 12:54:10.637000000", + "duration_ms": 3937.663, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What are the latest breakthroughs in artificial intelligence?", + "query.length": 61, + "processing.mode": "Web Search", + "processing.time_seconds": 3.93752384185791, + "processing.response_length": 1090, + "processing.sources_count": 1, + "output_response": "Final Answer: The web search did not yield specific results regarding the latest breakthroughs in artificial intelligence. However, I can provide a general overview based on knowledge up to October 2023.\n\nRecent br...", + "final_answer_mode": "Web Search", + "query_processed.input_query": "What are the latest breakthroughs in artificial intelligence?", + "query_processed.output_response": "Successfully processed query using Web Search, generated 1090 character response", + "query_processed.mode": "Web Search", + "query_processed.processing_time": 3.93752384185791, + "query_processed.response_length": 1090 + }, + "events": [], + "links": [], + "trace_trace_id": "e6ef2449-7429-465b-9bff-f3def8de5bb0", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-14 12:54:06.700000000", + "trace_end_time": "2025-10-14 12:54:10.637000000", + "trace_duration_ms": 3937.729, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-14 12:54:06.700000000", + "updated_at": "2025-10-14 12:54:06.700000000" + }, + { + "span_id": "bb47766c-afda-44fe-8838-7936c80cd4b6", + "trace_id": "e6ef2449-7429-465b-9bff-f3def8de5bb0", + "parent_span_id": "a7d1e33a-95a1-41a9-a117-62b4c373bd60", + "name": "agent.llm_model_execution", + "start_time": "2025-10-14 12:54:07.104000000", + "end_time": "2025-10-14 12:54:10.637000000", + "duration_ms": 3533.321, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What are the latest breakthroughs in artificial intelligence?", + "query_length": 61, + "query_type": "web_search_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 3.533268928527832, + "model.latency_ms": 3533.268928527832, + "model.start_time": 1760446447.104266, + "model.end_time": 1760446450.6375349, + "model.performance_tier": "medium", + "model.prompt_tokens": 167, + "model.completion_tokens": 272, + "model.total_tokens": 439, + "model.tokens_per_second": 124.2475477752307, + "model.estimated_cost": 0.013170000000000001, + "model.efficiency_score": 2.4829157175398633, + "model.response_length": 1090, + "model.response_quality": "high", + "model.output_response": "Model Response: The web search did not yield specific results regarding the latest breakthroughs in artificial intelligence. However, I can provide a general overview based on knowledge up to October 2023.\n\nRecent br...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "e6ef2449-7429-465b-9bff-f3def8de5bb0", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-14 12:54:06.700000000", + "trace_end_time": "2025-10-14 12:54:10.637000000", + "trace_duration_ms": 3937.729, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-14 12:54:06.700000000", + "updated_at": "2025-10-14 12:54:06.700000000" + }, + { + "span_id": "1d914a9d-b770-45dc-8308-6a36e9e0db89", + "trace_id": "e6ef2449-7429-465b-9bff-f3def8de5bb0", + "parent_span_id": "a7d1e33a-95a1-41a9-a117-62b4c373bd60", + "name": "agent.web_search_evaluation_metrics", + "start_time": "2025-10-14 12:54:10.637000000", + "end_time": "2025-10-14 12:54:10.637000000", + "duration_ms": 0.022, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "web_search_evaluation_metrics", + "agent.capabilities": [ + "web_search_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What are the latest breakthroughs in artificial intelligence?", + "query_length": 61, + "query_type": "web_search_evaluation_query", + "web_search.results_count": 1, + "web_search.context_length": 163, + "web_search.context_synthesized": "Context from 1 web sources", + "web_search.search_effectiveness": 0.2, + "web_search.context_quality": "low", + "web_search.source_diversity": 1, + "web_search.information_synthesis": "low", + "web_search.external_knowledge_utilization": 0.163, + "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What+are+the+latest+breakthroughs+in+artificial+intelligence?\n\nUser Question: What are the latest breakthroughs in artificial intelligence?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", + "prompt.prompt_length": 669, + "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What+are+the+latest+breakthroughs+in+artificial+intelligence?", + "prompt.instruction_following": "web_search_optimized", + "response.answer_length": 1090, + "response.answer_completeness": "complete", + "response.response_quality": "medium", + "response.source_citation": 1, + "response.context_utilization": 6.6871165644171775, + "output_response": "Web Search Answer: The web search did not yield specific results regarding the latest breakthroughs in artificial intelligence. However, I can provide a general overview based on knowledge up to October 2023.\n\nRecent br...", + "evaluation.search_effectiveness": 0.2, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 1, + "evaluation.context_relevance": "low", + "evaluation.overall_quality": "low", + "evaluation.ready_for_production": true, + "web_search.search_strategy": "duckduckgo_api", + "web_search.real_time_data": true, + "web_search.external_sources": 1, + "web_search.information_freshness": "current", + "web_search.knowledge_synthesis": "single_source" + }, + "events": [], + "links": [], + "trace_trace_id": "e6ef2449-7429-465b-9bff-f3def8de5bb0", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-14 12:54:06.700000000", + "trace_end_time": "2025-10-14 12:54:10.637000000", + "trace_duration_ms": 3937.729, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-14 12:54:06.700000000", + "updated_at": "2025-10-14 12:54:06.700000000" + }, + { + "span_id": "91f3999c-2edf-4c56-8f4d-ab43b6ebbaee", + "trace_id": "2ba8cab0-2366-4add-8ac3-b490a05e9a60", + "parent_span_id": "cbe51252-a323-4852-a90e-c907a0480fa3", + "name": "agent.web_search_generation", + "start_time": "2025-10-14 12:53:56.480000000", + "end_time": "2025-10-14 12:54:05.685000000", + "duration_ms": 9205.212, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "web_search_agent", + "agent.operation": "web_search_generation", + "agent.capabilities": [ + "web_search", + "content_synthesis", + "response_generation" + ], + "agent.id": "noveum_web_search_agent", + "input_query": "What's happening in the software development world today?", + "query_length": 57, + "query_type": "web_search_query", + "output_response": "Web Search Answer: The software development world is currently experiencing rapid changes driven by advancements in artificial intelligence (AI) and evolving programming practices. Key trends include:\n\n1. **AI Integrati...", + "web_search.results_count": 1, + "web_search.context_length": 975, + "web_search.response_length": 1636, + "web_search.mode": "external_web_search" + }, + "events": [], + "links": [], + "trace_trace_id": "2ba8cab0-2366-4add-8ac3-b490a05e9a60", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-14 12:53:56.480000000", + "trace_end_time": "2025-10-14 12:54:05.687000000", + "trace_duration_ms": 9207.67, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-14 12:53:56.480000000", + "updated_at": "2025-10-14 12:53:56.480000000" + }, + { + "span_id": "ea2aee75-4537-46ea-954b-2002fc0026c5", + "trace_id": "2ba8cab0-2366-4add-8ac3-b490a05e9a60", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-14 12:53:56.480000000", + "end_time": "2025-10-14 12:54:05.687000000", + "duration_ms": 9207.528999999999, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What's happening in the software development world today?", + "query.length": 57, + "processing.mode": "Web Search", + "processing.time_seconds": 9.206756830215454, + "processing.response_length": 1636, + "processing.sources_count": 1, + "output_response": "Final Answer: The software development world is currently experiencing rapid changes driven by advancements in artificial intelligence (AI) and evolving programming practices. Key trends include:\n\n1. **AI Integrati...", + "final_answer_mode": "Web Search", + "query_processed.input_query": "What's happening in the software development world today?", + "query_processed.output_response": "Successfully processed query using Web Search, generated 1636 character response", + "query_processed.mode": "Web Search", + "query_processed.processing_time": 9.206756830215454, + "query_processed.response_length": 1636 + }, + "events": [], + "links": [], + "trace_trace_id": "2ba8cab0-2366-4add-8ac3-b490a05e9a60", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-14 12:53:56.480000000", + "trace_end_time": "2025-10-14 12:54:05.687000000", + "trace_duration_ms": 9207.67, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-14 12:53:56.480000000", + "updated_at": "2025-10-14 12:53:56.480000000" + }, + { + "span_id": "cbe51252-a323-4852-a90e-c907a0480fa3", + "trace_id": "2ba8cab0-2366-4add-8ac3-b490a05e9a60", + "parent_span_id": "ea2aee75-4537-46ea-954b-2002fc0026c5", + "name": "agent.query_routing", + "start_time": "2025-10-14 12:53:56.480000000", + "end_time": "2025-10-14 12:54:05.686000000", + "duration_ms": 9205.569000000001, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What's happening in the software development world today?", + "query_length": 57, + "query_type": "routing_query", + "output_response": "Routed to Web Search for query processing", + "router.classification": "Web Search", + "router.confidence_score": 1, + "router.rag_keyword_score": 0, + "router.web_keyword_score": 1, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "2ba8cab0-2366-4add-8ac3-b490a05e9a60", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-14 12:53:56.480000000", + "trace_end_time": "2025-10-14 12:54:05.687000000", + "trace_duration_ms": 9207.67, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-14 12:53:56.480000000", + "updated_at": "2025-10-14 12:53:56.480000000" + }, + { + "span_id": "d850ecee-86ef-4818-9cec-638bfee26293", + "trace_id": "2ba8cab0-2366-4add-8ac3-b490a05e9a60", + "parent_span_id": "cbe51252-a323-4852-a90e-c907a0480fa3", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-14 12:53:56.480000000", + "end_time": "2025-10-14 12:53:56.480000000", + "duration_ms": 0.041999999999999996, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What's happening in the software development world today?", + "query_length": 57, + "query_type": "routing_evaluation_query", + "classification.mode": "Web Search", + "classification.rag_keyword_score": 0, + "classification.web_keyword_score": 1, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "complex", + "query.intent": "general_knowledge", + "query.keyword_density": 0.125, + "query.domain_affinity": "general", + "routing.decision": "Routed to Web Search based on analysis", + "routing.rationale": "RAG score: 0, Web score: 1, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "RAG", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.019230769230769232, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: Web Search (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "2ba8cab0-2366-4add-8ac3-b490a05e9a60", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-14 12:53:56.480000000", + "trace_end_time": "2025-10-14 12:54:05.687000000", + "trace_duration_ms": 9207.67, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-14 12:53:56.480000000", + "updated_at": "2025-10-14 12:53:56.480000000" + }, + { + "span_id": "0d4387a4-adf3-42c1-aca7-51453a90dd36", + "trace_id": "2ba8cab0-2366-4add-8ac3-b490a05e9a60", + "parent_span_id": "91f3999c-2edf-4c56-8f4d-ab43b6ebbaee", + "name": "agent.llm_model_execution", + "start_time": "2025-10-14 12:53:59.426000000", + "end_time": "2025-10-14 12:54:05.684000000", + "duration_ms": 6258.643, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What's happening in the software development world today?", + "query_length": 57, + "query_type": "web_search_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 6.25848388671875, + "model.latency_ms": 6258.48388671875, + "model.start_time": 1760446439.426367, + "model.end_time": 1760446445.684851, + "model.performance_tier": "slow", + "model.prompt_tokens": 369, + "model.completion_tokens": 409, + "model.total_tokens": 778, + "model.tokens_per_second": 124.31125718019486, + "model.estimated_cost": 0.02334, + "model.efficiency_score": 2.102827763496144, + "model.response_length": 1636, + "model.response_quality": "high", + "model.output_response": "Model Response: The software development world is currently experiencing rapid changes driven by advancements in artificial intelligence (AI) and evolving programming practices. Key trends include:\n\n1. **AI Integrati...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "2ba8cab0-2366-4add-8ac3-b490a05e9a60", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-14 12:53:56.480000000", + "trace_end_time": "2025-10-14 12:54:05.687000000", + "trace_duration_ms": 9207.67, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-14 12:53:56.480000000", + "updated_at": "2025-10-14 12:53:56.480000000" + }, + { + "span_id": "717e3c4a-6300-45e5-9456-caa3e773d758", + "trace_id": "2ba8cab0-2366-4add-8ac3-b490a05e9a60", + "parent_span_id": "91f3999c-2edf-4c56-8f4d-ab43b6ebbaee", + "name": "agent.web_search_evaluation_metrics", + "start_time": "2025-10-14 12:54:05.685000000", + "end_time": "2025-10-14 12:54:05.685000000", + "duration_ms": 0.044, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "web_search_evaluation_metrics", + "agent.capabilities": [ + "web_search_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What's happening in the software development world today?", + "query_length": 57, + "query_type": "web_search_evaluation_query", + "web_search.results_count": 1, + "web_search.context_length": 975, + "web_search.context_synthesized": "Context from 1 web sources", + "web_search.search_effectiveness": 0.2, + "web_search.context_quality": "high", + "web_search.source_diversity": 1, + "web_search.information_synthesis": "low", + "web_search.external_knowledge_utilization": 0.975, + "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nLog in to WhatsApp Web for simple, reliable and private messaging on your desktop. Send and receive messages and files with ease, all for free. Sep 23, 2025 \u00b7 Chief Economists' Outlook 2025 states that these under-the-radar trends are shaping the global economy, from the impact of AI on research productivity to a tightened labor \u2026 Sep 1, 2025 \u00b7 THE AI DEVELOPMENT REVOLUTION: WHAT'S REALLY HAPPENING? Let's face it - the software development world is changing faster than a React component re-rendering on \u2026 Oct 7, 2025 \u00b7 Computer Programming Research. Read current computer science articles on everything from computer programs to detect cancer genes and control vehicle maintenance to \u2026 Aug 24, 2025 \u00b7 Artificial Intelligence News. Everything on AI including futuristic robots with artificial intelligence, computer models of human intelligence and more.\nURL: https://duckduckgo.com/?q=What's+happening+in+the+software+development+world+today?\n\nUser Question: What's happening in the software development world today?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", + "prompt.prompt_length": 1477, + "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nLog in to WhatsApp Web for simple, reliable and private messaging on your desktop. Send and receive messages and files with ease, all for free. Sep 23, 2025 \u00b7 Chief Economi...", + "prompt.instruction_following": "web_search_optimized", + "response.answer_length": 1636, + "response.answer_completeness": "complete", + "response.response_quality": "medium", + "response.source_citation": 1, + "response.context_utilization": 1.677948717948718, + "output_response": "Web Search Answer: The software development world is currently experiencing rapid changes driven by advancements in artificial intelligence (AI) and evolving programming practices. Key trends include:\n\n1. **AI Integrati...", + "evaluation.search_effectiveness": 0.2, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 1, + "evaluation.context_relevance": "high", + "evaluation.overall_quality": "low", + "evaluation.ready_for_production": true, + "web_search.search_strategy": "duckduckgo_api", + "web_search.real_time_data": true, + "web_search.external_sources": 1, + "web_search.information_freshness": "current", + "web_search.knowledge_synthesis": "single_source" + }, + "events": [], + "links": [], + "trace_trace_id": "2ba8cab0-2366-4add-8ac3-b490a05e9a60", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-14 12:53:56.480000000", + "trace_end_time": "2025-10-14 12:54:05.687000000", + "trace_duration_ms": 9207.67, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-14 12:53:56.480000000", + "updated_at": "2025-10-14 12:53:56.480000000" + }, + { + "span_id": "3d39f522-8ae7-4988-aafd-58b09734eff0", + "trace_id": "a7c3399e-372f-4aae-baeb-05215cc4b85f", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-14 12:53:50.963000000", + "end_time": "2025-10-14 12:53:55.471000000", + "duration_ms": 4507.466, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What are the newest features in cloud computing?", + "query.length": 48, + "processing.mode": "RAG", + "processing.time_seconds": 4.507421255111694, + "processing.response_length": 870, + "processing.sources_count": 5, + "output_response": "Final Answer: The provided context does not specifically mention any new features in cloud computing as a general topic. However, it does highlight several recent enhancements related to the Noveum.ai platform, whi...", + "final_answer_mode": "RAG", + "query_processed.input_query": "What are the newest features in cloud computing?", + "query_processed.output_response": "Successfully processed query using RAG, generated 870 character response", + "query_processed.mode": "RAG", + "query_processed.processing_time": 4.507421255111694, + "query_processed.response_length": 870 + }, + "events": [], + "links": [], + "trace_trace_id": "a7c3399e-372f-4aae-baeb-05215cc4b85f", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-14 12:53:50.963000000", + "trace_end_time": "2025-10-14 12:53:55.471000000", + "trace_duration_ms": 4507.555, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-14 12:53:50.963000000", + "updated_at": "2025-10-14 12:53:50.963000000" + }, + { + "span_id": "54aa6700-f2de-4cf0-b1ad-a3d24ed692ed", + "trace_id": "a7c3399e-372f-4aae-baeb-05215cc4b85f", + "parent_span_id": "d17f31cf-c570-4604-8df3-532b2e530fce", + "name": "agent.llm-rag", + "start_time": "2025-10-14 12:53:50.963000000", + "end_time": "2025-10-14 12:53:55.471000000", + "duration_ms": 4507.083, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "rag_agent", + "agent.operation": "llm-rag", + "agent.capabilities": [ + "document_retrieval", + "context_generation", + "response_generation" + ], + "agent.id": "noveum_rag_agent", + "input_query": "What are the newest features in cloud computing?", + "query_length": 48, + "query_type": "rag_query", + "output_response": "RAG Answer: The provided context does not specifically mention any new features in cloud computing as a general topic. However, it does highlight several recent enhancements related to the Noveum.ai platform, whi...", + "rag.context_length": 2579, + "rag.sources_count": 5, + "rag.answer_length": 870, + "rag.mode": "retrieval_augmented_generation" + }, + "events": [], + "links": [], + "trace_trace_id": "a7c3399e-372f-4aae-baeb-05215cc4b85f", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-14 12:53:50.963000000", + "trace_end_time": "2025-10-14 12:53:55.471000000", + "trace_duration_ms": 4507.555, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-14 12:53:50.963000000", + "updated_at": "2025-10-14 12:53:50.963000000" + }, + { + "span_id": "8878096f-6cfa-4f70-9844-d7723a0875f0", + "trace_id": "a7c3399e-372f-4aae-baeb-05215cc4b85f", + "parent_span_id": "d17f31cf-c570-4604-8df3-532b2e530fce", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-14 12:53:50.963000000", + "end_time": "2025-10-14 12:53:50.963000000", + "duration_ms": 0.036000000000000004, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What are the newest features in cloud computing?", + "query_length": 48, + "query_type": "routing_evaluation_query", + "classification.mode": "RAG", + "classification.rag_keyword_score": 1, + "classification.web_keyword_score": 0, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "documentation", + "query.keyword_density": 0.125, + "query.domain_affinity": "noveum", + "routing.decision": "Routed to RAG based on analysis", + "routing.rationale": "RAG score: 1, Web score: 0, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "Web Search", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.019230769230769232, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: RAG (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "a7c3399e-372f-4aae-baeb-05215cc4b85f", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-14 12:53:50.963000000", + "trace_end_time": "2025-10-14 12:53:55.471000000", + "trace_duration_ms": 4507.555, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-14 12:53:50.963000000", + "updated_at": "2025-10-14 12:53:50.963000000" + }, + { + "span_id": "d17f31cf-c570-4604-8df3-532b2e530fce", + "trace_id": "a7c3399e-372f-4aae-baeb-05215cc4b85f", + "parent_span_id": "3d39f522-8ae7-4988-aafd-58b09734eff0", + "name": "agent.query_routing", + "start_time": "2025-10-14 12:53:50.963000000", + "end_time": "2025-10-14 12:53:55.471000000", + "duration_ms": 4507.3730000000005, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What are the newest features in cloud computing?", + "query_length": 48, + "query_type": "routing_query", + "output_response": "Routed to RAG for query processing", + "router.classification": "RAG", + "router.confidence_score": 1, + "router.rag_keyword_score": 1, + "router.web_keyword_score": 0, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "a7c3399e-372f-4aae-baeb-05215cc4b85f", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-14 12:53:50.963000000", + "trace_end_time": "2025-10-14 12:53:55.471000000", + "trace_duration_ms": 4507.555, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-14 12:53:50.963000000", + "updated_at": "2025-10-14 12:53:50.963000000" + }, + { + "span_id": "97ef01a8-5930-4902-909a-245c516a5871", + "trace_id": "a7c3399e-372f-4aae-baeb-05215cc4b85f", + "parent_span_id": "54aa6700-f2de-4cf0-b1ad-a3d24ed692ed", + "name": "agent.llm_model_execution", + "start_time": "2025-10-14 12:53:51.439000000", + "end_time": "2025-10-14 12:53:55.077000000", + "duration_ms": 3637.337, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What are the newest features in cloud computing?", + "query_length": 48, + "query_type": "rag_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 3.6372909545898438, + "model.latency_ms": 3637.2909545898438, + "model.start_time": 1760446431.439748, + "model.end_time": 1760446435.077039, + "model.performance_tier": "medium", + "model.prompt_tokens": 780, + "model.completion_tokens": 217, + "model.total_tokens": 997, + "model.tokens_per_second": 274.1050997698989, + "model.estimated_cost": 0.02991, + "model.efficiency_score": 0.872617853560682, + "model.response_length": 870, + "model.response_quality": "high", + "model.output_response": "Model Response: The provided context does not specifically mention any new features in cloud computing as a general topic. However, it does highlight several recent enhancements related to the Noveum.ai platform, whi...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "a7c3399e-372f-4aae-baeb-05215cc4b85f", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-14 12:53:50.963000000", + "trace_end_time": "2025-10-14 12:53:55.471000000", + "trace_duration_ms": 4507.555, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-14 12:53:50.963000000", + "updated_at": "2025-10-14 12:53:50.963000000" + }, + { + "span_id": "ea792ec6-64b4-4ed5-b418-ba902e90c57a", + "trace_id": "a7c3399e-372f-4aae-baeb-05215cc4b85f", + "parent_span_id": "54aa6700-f2de-4cf0-b1ad-a3d24ed692ed", + "name": "agent.rag_evaluation_metrics", + "start_time": "2025-10-14 12:53:55.077000000", + "end_time": "2025-10-14 12:53:55.077000000", + "duration_ms": 0.034, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "rag_evaluation_metrics", + "agent.capabilities": [ + "retrieval_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What are the newest features in cloud computing?", + "query_length": 48, + "query_type": "rag_evaluation_query", + "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/changelog):\n- \ud83d\udcc8 Enhanced Dashboard Analytics: Real-time request tracking, latency monitoring, and cost analysis\n- \ud83d\udd0d Improved Logs Interface: Better search functionality, detailed trace views, and enhanced debugging capabilities\n- \ud83d\udc65 Team Management: Advanced member mana...", + "retrieval.context_length": 2579, + "retrieval.sources_count": 5, + "retrieval.context_quality": "high", + "retrieval.effectiveness": 1, + "retrieval.context_utilization": 2.579, + "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/changelog):\n- \ud83d\udcc8 Enhanced Dashboard Analytics: Real-time request tracking, latency monitoring, and cost analysis\n- \ud83d\udd0d Improved Logs Interface: Better search functionality, detailed trace views, and enhanced debugging capabilities\n- \ud83d\udc65 Team Management: Advanced member management with role-based access control and invitation system\n- \ud83d\udee1\ufe0f Security Improvements: Enhanced API key management and secure credential storage\n4 months ago\n- \ud83d\udcca Advanced Metrics Collection: Comprehensive telemetry system with custom metrics ...\n\nSource 2 (https://noveum.ai/en/changelog):\n- \ud83d\udcdd Documentation Updates: Comprehensive guides for integration and best practices\n5 months ago\n- \ud83c\udf89 Platform Foundation: Initial release of Noveum.ai AI observability platform\n- \ud83d\ude80 Core Features: AI Gateway integration, basic evaluation metrics, and monitoring capabilities\n- \ud83d\udcc8 Evaluation Jobs: Automated model evaluation with configurable metrics and reporting\n- \ud83d\udd0c Provider Integrations: Support for major AI providers and custom model deployments...\n\nSource 3 (https://noveum.ai/docs/platform/dashboard):\n- Bulk Export: Download multiple traces simultaneously\n- Comparative Analysis: Compare performance across multiple traces\nReal-Time Updates\n- Live Refresh: Automatic updates as new traces arrive\n- Connection Monitoring: Real-time status of your trace ingestion pipeline\n- Performance Indicators: Live metrics for system health monitoring\nReady to dive deeper? Explore Projects & Environments to organize your AI applications, or check out Team Collaboration to share insights with your team.\nGet Earl...\n\nSource 4 (https://noveum.ai/en/changelog):\nChangelog\nStay up to date with the latest changes in our product.\n2 months ago\n- \ud83c\udfaf Major Platform Evolution: NovaEval Framework - Launched comprehensive AI model evaluation framework with 20+ built-in scorers including accuracy, RAG metrics, conversational scoring, and specialized agent evaluation tools. Features production-grade deployment on Docker and Kubernetes. (GitHub | PyPI Package)\n- \ud83d\ude80 Next-Generation Traces UI - Complete redesign featuring three-pane layout with directory tree navigatio...\n\nSource 5 (https://noveum.ai/en/blog):\n8/12/2025\nDiscover how Noveum.ai provides comprehensive tracing and observability for AI applications, from development debugging to production optimization.\nShashank Agarwal\n3/3/2025\nDiscover how Noveum.ai provides comprehensive tracing and observability for LLM applications, RAG systems, and multi-agent workflows with our powerful Python and TypeScript SDKs.\nShashank Agarwal\n3/2/2025...\n\nUser Question: What are the newest features in cloud computing?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", + "prompt.prompt_length": 3123, + "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/changelog):\n- \ud83d\udcc8 Enhanced Dashboard Analytics: Real-time request tracking, latency monitoring, and cost analysis\n- \ud83d\udd0d Improved Logs Interface: Better search functionality,...", + "prompt.instruction_following": "rag_optimized", + "response.answer_length": 870, + "response.answer_completeness": "complete", + "response.response_quality": "high", + "response.source_citation": 5, + "response.context_utilization": 0.33734005428460645, + "output_response": "RAG Answer: The provided context does not specifically mention any new features in cloud computing as a general topic. However, it does highlight several recent enhancements related to the Noveum.ai platform, whi...", + "evaluation.retrieval_effectiveness": 1, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 5, + "evaluation.context_relevance": "high", + "evaluation.overall_quality": "high", + "evaluation.ready_for_production": true, + "rag.retrieval_strategy": "semantic_similarity", + "rag.vector_search_results": 5, + "rag.context_synthesis": "multi_source", + "rag.document_coverage": 1, + "rag.information_density": 0.33734005428460645 + }, + "events": [], + "links": [], + "trace_trace_id": "a7c3399e-372f-4aae-baeb-05215cc4b85f", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-14 12:53:50.963000000", + "trace_end_time": "2025-10-14 12:53:55.471000000", + "trace_duration_ms": 4507.555, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-14 12:53:50.963000000", + "updated_at": "2025-10-14 12:53:50.963000000" + }, + { + "span_id": "e2d314c8-687f-4b8d-869c-8d7f2a22a4f3", + "trace_id": "d682797f-2ce0-448e-8f0d-891541430988", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-14 12:53:44.917000000", + "end_time": "2025-10-14 12:53:49.957000000", + "duration_ms": 5039.565, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What's the current status of cryptocurrency markets?", + "query.length": 52, + "processing.mode": "Web Search", + "processing.time_seconds": 5.0394370555877686, + "processing.response_length": 338, + "processing.sources_count": 1, + "output_response": "Final Answer: The web search results did not provide any specific information regarding the current status of cryptocurrency markets. Therefore, I cannot provide an update on market conditions, prices, or trends at...", + "final_answer_mode": "Web Search", + "query_processed.input_query": "What's the current status of cryptocurrency markets?", + "query_processed.output_response": "Successfully processed query using Web Search, generated 338 character response", + "query_processed.mode": "Web Search", + "query_processed.processing_time": 5.0394370555877686, + "query_processed.response_length": 338 + }, + "events": [], + "links": [], + "trace_trace_id": "d682797f-2ce0-448e-8f0d-891541430988", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-14 12:53:44.917000000", + "trace_end_time": "2025-10-14 12:53:49.957000000", + "trace_duration_ms": 5039.64, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-14 12:53:44.917000000", + "updated_at": "2025-10-14 12:53:44.917000000" + }, + { + "span_id": "d367a6bd-86af-41c5-96ca-5284425775ce", + "trace_id": "d682797f-2ce0-448e-8f0d-891541430988", + "parent_span_id": "f137a412-6ddf-4ace-908e-9edf73534aa7", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-14 12:53:44.917000000", + "end_time": "2025-10-14 12:53:44.918000000", + "duration_ms": 0.046, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What's the current status of cryptocurrency markets?", + "query_length": 52, + "query_type": "routing_evaluation_query", + "classification.mode": "Web Search", + "classification.rag_keyword_score": 0, + "classification.web_keyword_score": 3, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "complex", + "query.intent": "general_knowledge", + "query.keyword_density": 0.42857142857142855, + "query.domain_affinity": "general", + "routing.decision": "Routed to Web Search based on analysis", + "routing.rationale": "RAG score: 0, Web score: 3, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "RAG", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.057692307692307696, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: Web Search (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "d682797f-2ce0-448e-8f0d-891541430988", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-14 12:53:44.917000000", + "trace_end_time": "2025-10-14 12:53:49.957000000", + "trace_duration_ms": 5039.64, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-14 12:53:44.917000000", + "updated_at": "2025-10-14 12:53:44.917000000" + }, + { + "span_id": "f137a412-6ddf-4ace-908e-9edf73534aa7", + "trace_id": "d682797f-2ce0-448e-8f0d-891541430988", + "parent_span_id": "e2d314c8-687f-4b8d-869c-8d7f2a22a4f3", + "name": "agent.query_routing", + "start_time": "2025-10-14 12:53:44.917000000", + "end_time": "2025-10-14 12:53:49.957000000", + "duration_ms": 5039.404, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What's the current status of cryptocurrency markets?", + "query_length": 52, + "query_type": "routing_query", + "output_response": "Routed to Web Search for query processing", + "router.classification": "Web Search", + "router.confidence_score": 1, + "router.rag_keyword_score": 0, + "router.web_keyword_score": 3, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "d682797f-2ce0-448e-8f0d-891541430988", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-14 12:53:44.917000000", + "trace_end_time": "2025-10-14 12:53:49.957000000", + "trace_duration_ms": 5039.64, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-14 12:53:44.917000000", + "updated_at": "2025-10-14 12:53:44.917000000" + }, + { + "span_id": "987b6910-9b6d-4207-b0f3-8807a23e3ddb", + "trace_id": "d682797f-2ce0-448e-8f0d-891541430988", + "parent_span_id": "f137a412-6ddf-4ace-908e-9edf73534aa7", + "name": "agent.web_search_generation", + "start_time": "2025-10-14 12:53:44.918000000", + "end_time": "2025-10-14 12:53:49.957000000", + "duration_ms": 5039.248, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "web_search_agent", + "agent.operation": "web_search_generation", + "agent.capabilities": [ + "web_search", + "content_synthesis", + "response_generation" + ], + "agent.id": "noveum_web_search_agent", + "input_query": "What's the current status of cryptocurrency markets?", + "query_length": 52, + "query_type": "web_search_query", + "output_response": "Web Search Answer: The web search results did not provide any specific information regarding the current status of cryptocurrency markets. Therefore, I cannot provide an update on market conditions, prices, or trends at...", + "web_search.results_count": 1, + "web_search.context_length": 154, + "web_search.response_length": 338, + "web_search.mode": "external_web_search" + }, + "events": [], + "links": [], + "trace_trace_id": "d682797f-2ce0-448e-8f0d-891541430988", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-14 12:53:44.917000000", + "trace_end_time": "2025-10-14 12:53:49.957000000", + "trace_duration_ms": 5039.64, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-14 12:53:44.917000000", + "updated_at": "2025-10-14 12:53:44.917000000" + }, + { + "span_id": "b2614d0e-5fcd-46ec-8d25-f1b04eb142af", + "trace_id": "d682797f-2ce0-448e-8f0d-891541430988", + "parent_span_id": "987b6910-9b6d-4207-b0f3-8807a23e3ddb", + "name": "agent.llm_model_execution", + "start_time": "2025-10-14 12:53:45.255000000", + "end_time": "2025-10-14 12:53:49.957000000", + "duration_ms": 4701.744, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What's the current status of cryptocurrency markets?", + "query_length": 52, + "query_type": "web_search_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 4.701629161834717, + "model.latency_ms": 4701.629161834717, + "model.start_time": 1760446425.2554948, + "model.end_time": 1760446429.957124, + "model.performance_tier": "medium", + "model.prompt_tokens": 162, + "model.completion_tokens": 84, + "model.total_tokens": 246, + "model.tokens_per_second": 52.32228904756993, + "model.estimated_cost": 0.00738, + "model.efficiency_score": 1.3739837398373984, + "model.response_length": 338, + "model.response_quality": "high", + "model.output_response": "Model Response: The web search results did not provide any specific information regarding the current status of cryptocurrency markets. Therefore, I cannot provide an update on market conditions, prices, or trends at...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "d682797f-2ce0-448e-8f0d-891541430988", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-14 12:53:44.917000000", + "trace_end_time": "2025-10-14 12:53:49.957000000", + "trace_duration_ms": 5039.64, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-14 12:53:44.917000000", + "updated_at": "2025-10-14 12:53:44.917000000" + }, + { + "span_id": "663e8789-75d0-4c4d-b14d-4088022436ac", + "trace_id": "d682797f-2ce0-448e-8f0d-891541430988", + "parent_span_id": "987b6910-9b6d-4207-b0f3-8807a23e3ddb", + "name": "agent.web_search_evaluation_metrics", + "start_time": "2025-10-14 12:53:49.957000000", + "end_time": "2025-10-14 12:53:49.957000000", + "duration_ms": 0.032, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "web_search_evaluation_metrics", + "agent.capabilities": [ + "web_search_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What's the current status of cryptocurrency markets?", + "query_length": 52, + "query_type": "web_search_evaluation_query", + "web_search.results_count": 1, + "web_search.context_length": 154, + "web_search.context_synthesized": "Context from 1 web sources", + "web_search.search_effectiveness": 0.2, + "web_search.context_quality": "low", + "web_search.source_diversity": 1, + "web_search.information_synthesis": "low", + "web_search.external_knowledge_utilization": 0.154, + "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What's+the+current+status+of+cryptocurrency+markets?\n\nUser Question: What's the current status of cryptocurrency markets?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", + "prompt.prompt_length": 651, + "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What's+the+current+status+of+cryptocurrency+markets?", + "prompt.instruction_following": "web_search_optimized", + "response.answer_length": 338, + "response.answer_completeness": "complete", + "response.response_quality": "medium", + "response.source_citation": 1, + "response.context_utilization": 2.1948051948051948, + "output_response": "Web Search Answer: The web search results did not provide any specific information regarding the current status of cryptocurrency markets. Therefore, I cannot provide an update on market conditions, prices, or trends at...", + "evaluation.search_effectiveness": 0.2, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 1, + "evaluation.context_relevance": "low", + "evaluation.overall_quality": "low", + "evaluation.ready_for_production": true, + "web_search.search_strategy": "duckduckgo_api", + "web_search.real_time_data": true, + "web_search.external_sources": 1, + "web_search.information_freshness": "current", + "web_search.knowledge_synthesis": "single_source" + }, + "events": [], + "links": [], + "trace_trace_id": "d682797f-2ce0-448e-8f0d-891541430988", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-14 12:53:44.917000000", + "trace_end_time": "2025-10-14 12:53:49.957000000", + "trace_duration_ms": 5039.64, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-14 12:53:44.917000000", + "updated_at": "2025-10-14 12:53:44.917000000" + }, + { + "span_id": "3382fe7c-aaca-46e0-a8d2-dbee75ff3bed", + "trace_id": "b0858dca-4173-493b-a6c5-4e993f606477", + "parent_span_id": "6cf94049-87b0-4fc5-b37e-32d261543647", + "name": "agent.query_routing", + "start_time": "2025-10-14 12:53:37.818000000", + "end_time": "2025-10-14 12:53:43.908000000", + "duration_ms": 6089.22, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What are the latest updates in Python programming?", + "query_length": 50, + "query_type": "routing_query", + "output_response": "Routed to Web Search for query processing", + "router.classification": "Web Search", + "router.confidence_score": 1, + "router.rag_keyword_score": 0, + "router.web_keyword_score": 2, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "b0858dca-4173-493b-a6c5-4e993f606477", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-14 12:53:37.818000000", + "trace_end_time": "2025-10-14 12:53:43.908000000", + "trace_duration_ms": 6089.715, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-14 12:53:37.818000000", + "updated_at": "2025-10-14 12:53:37.818000000" + }, + { + "span_id": "6cf94049-87b0-4fc5-b37e-32d261543647", + "trace_id": "b0858dca-4173-493b-a6c5-4e993f606477", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-14 12:53:37.818000000", + "end_time": "2025-10-14 12:53:43.908000000", + "duration_ms": 6089.607, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What are the latest updates in Python programming?", + "query.length": 50, + "processing.mode": "Web Search", + "processing.time_seconds": 6.089300870895386, + "processing.response_length": 1324, + "processing.sources_count": 1, + "output_response": "Final Answer: The web search did not yield specific results regarding the latest updates in Python programming. However, I can provide a general overview based on my knowledge up to October 2023.\n\nAs of late 2023, ...", + "final_answer_mode": "Web Search", + "query_processed.input_query": "What are the latest updates in Python programming?", + "query_processed.output_response": "Successfully processed query using Web Search, generated 1324 character response", + "query_processed.mode": "Web Search", + "query_processed.processing_time": 6.089300870895386, + "query_processed.response_length": 1324 + }, + "events": [], + "links": [], + "trace_trace_id": "b0858dca-4173-493b-a6c5-4e993f606477", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-14 12:53:37.818000000", + "trace_end_time": "2025-10-14 12:53:43.908000000", + "trace_duration_ms": 6089.715, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-14 12:53:37.818000000", + "updated_at": "2025-10-14 12:53:37.818000000" + }, + { + "span_id": "21c8befa-356a-4034-8d62-792238cb9e6e", + "trace_id": "b0858dca-4173-493b-a6c5-4e993f606477", + "parent_span_id": "3382fe7c-aaca-46e0-a8d2-dbee75ff3bed", + "name": "agent.web_search_generation", + "start_time": "2025-10-14 12:53:37.819000000", + "end_time": "2025-10-14 12:53:43.908000000", + "duration_ms": 6088.807, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "web_search_agent", + "agent.operation": "web_search_generation", + "agent.capabilities": [ + "web_search", + "content_synthesis", + "response_generation" + ], + "agent.id": "noveum_web_search_agent", + "input_query": "What are the latest updates in Python programming?", + "query_length": 50, + "query_type": "web_search_query", + "output_response": "Web Search Answer: The web search did not yield specific results regarding the latest updates in Python programming. However, I can provide a general overview based on my knowledge up to October 2023.\n\nAs of late 2023, ...", + "web_search.results_count": 1, + "web_search.context_length": 152, + "web_search.response_length": 1324, + "web_search.mode": "external_web_search" + }, + "events": [], + "links": [], + "trace_trace_id": "b0858dca-4173-493b-a6c5-4e993f606477", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-14 12:53:37.818000000", + "trace_end_time": "2025-10-14 12:53:43.908000000", + "trace_duration_ms": 6089.715, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-14 12:53:37.818000000", + "updated_at": "2025-10-14 12:53:37.818000000" + }, + { + "span_id": "482feedf-52f4-4c6a-9186-d59898820caa", + "trace_id": "b0858dca-4173-493b-a6c5-4e993f606477", + "parent_span_id": "3382fe7c-aaca-46e0-a8d2-dbee75ff3bed", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-14 12:53:37.819000000", + "end_time": "2025-10-14 12:53:37.819000000", + "duration_ms": 0.039, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What are the latest updates in Python programming?", + "query_length": 50, + "query_type": "routing_evaluation_query", + "classification.mode": "Web Search", + "classification.rag_keyword_score": 0, + "classification.web_keyword_score": 2, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "general_knowledge", + "query.keyword_density": 0.25, + "query.domain_affinity": "general", + "routing.decision": "Routed to Web Search based on analysis", + "routing.rationale": "RAG score: 0, Web score: 2, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "RAG", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.038461538461538464, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: Web Search (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "b0858dca-4173-493b-a6c5-4e993f606477", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-14 12:53:37.818000000", + "trace_end_time": "2025-10-14 12:53:43.908000000", + "trace_duration_ms": 6089.715, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-14 12:53:37.818000000", + "updated_at": "2025-10-14 12:53:37.818000000" + }, + { + "span_id": "84f1f63f-ee05-411e-a7dd-4ee9e9826f5a", + "trace_id": "b0858dca-4173-493b-a6c5-4e993f606477", + "parent_span_id": "21c8befa-356a-4034-8d62-792238cb9e6e", + "name": "agent.llm_model_execution", + "start_time": "2025-10-14 12:53:38.191000000", + "end_time": "2025-10-14 12:53:43.907000000", + "duration_ms": 5716.078, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What are the latest updates in Python programming?", + "query_length": 50, + "query_type": "web_search_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 5.715963125228882, + "model.latency_ms": 5715.963125228882, + "model.start_time": 1760446418.191805, + "model.end_time": 1760446423.907768, + "model.performance_tier": "slow", + "model.prompt_tokens": 161, + "model.completion_tokens": 331, + "model.total_tokens": 492, + "model.tokens_per_second": 86.07473302765561, + "model.estimated_cost": 0.01476, + "model.efficiency_score": 2.6910569105691056, + "model.response_length": 1324, + "model.response_quality": "high", + "model.output_response": "Model Response: The web search did not yield specific results regarding the latest updates in Python programming. However, I can provide a general overview based on my knowledge up to October 2023.\n\nAs of late 2023, ...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "b0858dca-4173-493b-a6c5-4e993f606477", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-14 12:53:37.818000000", + "trace_end_time": "2025-10-14 12:53:43.908000000", + "trace_duration_ms": 6089.715, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-14 12:53:37.818000000", + "updated_at": "2025-10-14 12:53:37.818000000" + }, + { + "span_id": "2fb9ecf3-a497-4285-a694-1c95080d9e6a", + "trace_id": "b0858dca-4173-493b-a6c5-4e993f606477", + "parent_span_id": "21c8befa-356a-4034-8d62-792238cb9e6e", + "name": "agent.web_search_evaluation_metrics", + "start_time": "2025-10-14 12:53:43.908000000", + "end_time": "2025-10-14 12:53:43.908000000", + "duration_ms": 0.037, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "web_search_evaluation_metrics", + "agent.capabilities": [ + "web_search_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What are the latest updates in Python programming?", + "query_length": 50, + "query_type": "web_search_evaluation_query", + "web_search.results_count": 1, + "web_search.context_length": 152, + "web_search.context_synthesized": "Context from 1 web sources", + "web_search.search_effectiveness": 0.2, + "web_search.context_quality": "low", + "web_search.source_diversity": 1, + "web_search.information_synthesis": "low", + "web_search.external_knowledge_utilization": 0.152, + "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What+are+the+latest+updates+in+Python+programming?\n\nUser Question: What are the latest updates in Python programming?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", + "prompt.prompt_length": 647, + "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What+are+the+latest+updates+in+Python+programming?", + "prompt.instruction_following": "web_search_optimized", + "response.answer_length": 1324, + "response.answer_completeness": "complete", + "response.response_quality": "medium", + "response.source_citation": 1, + "response.context_utilization": 8.710526315789474, + "output_response": "Web Search Answer: The web search did not yield specific results regarding the latest updates in Python programming. However, I can provide a general overview based on my knowledge up to October 2023.\n\nAs of late 2023, ...", + "evaluation.search_effectiveness": 0.2, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 1, + "evaluation.context_relevance": "low", + "evaluation.overall_quality": "low", + "evaluation.ready_for_production": true, + "web_search.search_strategy": "duckduckgo_api", + "web_search.real_time_data": true, + "web_search.external_sources": 1, + "web_search.information_freshness": "current", + "web_search.knowledge_synthesis": "single_source" + }, + "events": [], + "links": [], + "trace_trace_id": "b0858dca-4173-493b-a6c5-4e993f606477", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-14 12:53:37.818000000", + "trace_end_time": "2025-10-14 12:53:43.908000000", + "trace_duration_ms": 6089.715, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-14 12:53:37.818000000", + "updated_at": "2025-10-14 12:53:37.818000000" + }, + { + "span_id": "9dd5ac2c-0deb-48e3-ac8b-0e6e31f7ff30", + "trace_id": "dc81be1a-e93d-4f60-90f3-ce1acaa8cb3e", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-14 12:53:34.584000000", + "end_time": "2025-10-14 12:53:36.780000000", + "duration_ms": 2196.351, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What happened in tech news this week?", + "query.length": 37, + "processing.mode": "Web Search", + "processing.time_seconds": 2.1948978900909424, + "processing.response_length": 324, + "processing.sources_count": 1, + "output_response": "Final Answer: I'm sorry, but I couldn't find any specific information regarding tech news from this week based on the search results. It appears that there were no relevant articles or updates available. If you're ...", + "final_answer_mode": "Web Search", + "query_processed.input_query": "What happened in tech news this week?", + "query_processed.output_response": "Successfully processed query using Web Search, generated 324 character response", + "query_processed.mode": "Web Search", + "query_processed.processing_time": 2.1948978900909424, + "query_processed.response_length": 324 + }, + "events": [], + "links": [], + "trace_trace_id": "dc81be1a-e93d-4f60-90f3-ce1acaa8cb3e", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-14 12:53:34.584000000", + "trace_end_time": "2025-10-14 12:53:36.780000000", + "trace_duration_ms": 2196.4579999999996, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-14 12:53:34.584000000", + "updated_at": "2025-10-14 12:53:34.584000000" + }, + { + "span_id": "b1cbafce-51fb-48fd-bc4a-d49081a2f374", + "trace_id": "dc81be1a-e93d-4f60-90f3-ce1acaa8cb3e", + "parent_span_id": "9dd5ac2c-0deb-48e3-ac8b-0e6e31f7ff30", + "name": "agent.query_routing", + "start_time": "2025-10-14 12:53:34.584000000", + "end_time": "2025-10-14 12:53:36.778000000", + "duration_ms": 2194.2200000000003, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What happened in tech news this week?", + "query_length": 37, + "query_type": "routing_query", + "output_response": "Routed to Web Search for query processing", + "router.classification": "Web Search", + "router.confidence_score": 1, + "router.rag_keyword_score": 0, + "router.web_keyword_score": 2, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "dc81be1a-e93d-4f60-90f3-ce1acaa8cb3e", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-14 12:53:34.584000000", + "trace_end_time": "2025-10-14 12:53:36.780000000", + "trace_duration_ms": 2196.4579999999996, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-14 12:53:34.584000000", + "updated_at": "2025-10-14 12:53:34.584000000" + }, + { + "span_id": "e3064687-1b18-478b-9e9d-4d081a6dac23", + "trace_id": "dc81be1a-e93d-4f60-90f3-ce1acaa8cb3e", + "parent_span_id": "b1cbafce-51fb-48fd-bc4a-d49081a2f374", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-14 12:53:34.584000000", + "end_time": "2025-10-14 12:53:34.584000000", + "duration_ms": 0.028, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What happened in tech news this week?", + "query_length": 37, + "query_type": "routing_evaluation_query", + "classification.mode": "Web Search", + "classification.rag_keyword_score": 0, + "classification.web_keyword_score": 2, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "general_knowledge", + "query.keyword_density": 0.2857142857142857, + "query.domain_affinity": "general", + "routing.decision": "Routed to Web Search based on analysis", + "routing.rationale": "RAG score: 0, Web score: 2, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "RAG", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.038461538461538464, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: Web Search (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "dc81be1a-e93d-4f60-90f3-ce1acaa8cb3e", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-14 12:53:34.584000000", + "trace_end_time": "2025-10-14 12:53:36.780000000", + "trace_duration_ms": 2196.4579999999996, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-14 12:53:34.584000000", + "updated_at": "2025-10-14 12:53:34.584000000" + }, + { + "span_id": "eb9dbc11-44b3-4041-8c83-0f33b8b761bb", + "trace_id": "dc81be1a-e93d-4f60-90f3-ce1acaa8cb3e", + "parent_span_id": "b1cbafce-51fb-48fd-bc4a-d49081a2f374", + "name": "agent.web_search_generation", + "start_time": "2025-10-14 12:53:34.584000000", + "end_time": "2025-10-14 12:53:36.774000000", + "duration_ms": 2189.866, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "web_search_agent", + "agent.operation": "web_search_generation", + "agent.capabilities": [ + "web_search", + "content_synthesis", + "response_generation" + ], + "agent.id": "noveum_web_search_agent", + "input_query": "What happened in tech news this week?", + "query_length": 37, + "query_type": "web_search_query", + "output_response": "Web Search Answer: I'm sorry, but I couldn't find any specific information regarding tech news from this week based on the search results. It appears that there were no relevant articles or updates available. If you're ...", + "web_search.results_count": 1, + "web_search.context_length": 139, + "web_search.response_length": 324, + "web_search.mode": "external_web_search" + }, + "events": [], + "links": [], + "trace_trace_id": "dc81be1a-e93d-4f60-90f3-ce1acaa8cb3e", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-14 12:53:34.584000000", + "trace_end_time": "2025-10-14 12:53:36.780000000", + "trace_duration_ms": 2196.4579999999996, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-14 12:53:34.584000000", + "updated_at": "2025-10-14 12:53:34.584000000" + }, + { + "span_id": "c8ee237b-ab5e-4b36-83f0-2f8478971898", + "trace_id": "dc81be1a-e93d-4f60-90f3-ce1acaa8cb3e", + "parent_span_id": "eb9dbc11-44b3-4041-8c83-0f33b8b761bb", + "name": "agent.llm_model_execution", + "start_time": "2025-10-14 12:53:34.981000000", + "end_time": "2025-10-14 12:53:36.768000000", + "duration_ms": 1786.467, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What happened in tech news this week?", + "query_length": 37, + "query_type": "web_search_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 1.785179853439331, + "model.latency_ms": 1785.179853439331, + "model.start_time": 1760446414.981964, + "model.end_time": 1760446416.767144, + "model.performance_tier": "fast", + "model.prompt_tokens": 155, + "model.completion_tokens": 81, + "model.total_tokens": 236, + "model.tokens_per_second": 132.19956495997977, + "model.estimated_cost": 0.00708, + "model.efficiency_score": 1.3728813559322033, + "model.response_length": 324, + "model.response_quality": "high", + "model.output_response": "Model Response: I'm sorry, but I couldn't find any specific information regarding tech news from this week based on the search results. It appears that there were no relevant articles or updates available. If you're ...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "dc81be1a-e93d-4f60-90f3-ce1acaa8cb3e", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-14 12:53:34.584000000", + "trace_end_time": "2025-10-14 12:53:36.780000000", + "trace_duration_ms": 2196.4579999999996, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-14 12:53:34.584000000", + "updated_at": "2025-10-14 12:53:34.584000000" + }, + { + "span_id": "9c8e1a08-61c8-409e-a79d-41c74a8d1505", + "trace_id": "dc81be1a-e93d-4f60-90f3-ce1acaa8cb3e", + "parent_span_id": "eb9dbc11-44b3-4041-8c83-0f33b8b761bb", + "name": "agent.web_search_evaluation_metrics", + "start_time": "2025-10-14 12:53:36.772000000", + "end_time": "2025-10-14 12:53:36.774000000", + "duration_ms": 1.5939999999999999, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "web_search_evaluation_metrics", + "agent.capabilities": [ + "web_search_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What happened in tech news this week?", + "query_length": 37, + "query_type": "web_search_evaluation_query", + "web_search.results_count": 1, + "web_search.context_length": 139, + "web_search.context_synthesized": "Context from 1 web sources", + "web_search.search_effectiveness": 0.2, + "web_search.context_quality": "low", + "web_search.source_diversity": 1, + "web_search.information_synthesis": "low", + "web_search.external_knowledge_utilization": 0.139, + "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What+happened+in+tech+news+this+week?\n\nUser Question: What happened in tech news this week?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", + "prompt.prompt_length": 621, + "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What+happened+in+tech+news+this+week?", + "prompt.instruction_following": "web_search_optimized", + "response.answer_length": 324, + "response.answer_completeness": "complete", + "response.response_quality": "medium", + "response.source_citation": 1, + "response.context_utilization": 2.3309352517985613, + "output_response": "Web Search Answer: I'm sorry, but I couldn't find any specific information regarding tech news from this week based on the search results. It appears that there were no relevant articles or updates available. If you're ...", + "evaluation.search_effectiveness": 0.2, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 1, + "evaluation.context_relevance": "low", + "evaluation.overall_quality": "low", + "evaluation.ready_for_production": true, + "web_search.search_strategy": "duckduckgo_api", + "web_search.real_time_data": true, + "web_search.external_sources": 1, + "web_search.information_freshness": "current", + "web_search.knowledge_synthesis": "single_source" + }, + "events": [], + "links": [], + "trace_trace_id": "dc81be1a-e93d-4f60-90f3-ce1acaa8cb3e", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-14 12:53:34.584000000", + "trace_end_time": "2025-10-14 12:53:36.780000000", + "trace_duration_ms": 2196.4579999999996, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-14 12:53:34.584000000", + "updated_at": "2025-10-14 12:53:34.584000000" + }, + { + "span_id": "59595232-8873-419c-9b6a-0727445a2afa", + "trace_id": "945b2757-8a5a-4aab-a869-8f44c02166a5", + "parent_span_id": "dfde2f79-d0c1-481e-819d-60730cb60fe5", + "name": "agent.query_routing", + "start_time": "2025-10-14 12:53:27.771000000", + "end_time": "2025-10-14 12:53:33.575000000", + "duration_ms": 5803.9800000000005, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What are the current trends in observability tools?", + "query_length": 51, + "query_type": "routing_query", + "output_response": "Routed to RAG for query processing", + "router.classification": "RAG", + "router.confidence_score": 0.3333333333333333, + "router.rag_keyword_score": 2, + "router.web_keyword_score": 1, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "945b2757-8a5a-4aab-a869-8f44c02166a5", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-14 12:53:27.771000000", + "trace_end_time": "2025-10-14 12:53:33.575000000", + "trace_duration_ms": 5804.15, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-14 12:53:27.771000000", + "updated_at": "2025-10-14 12:53:27.771000000" + }, + { + "span_id": "a59b3296-aaba-4642-b5a0-a271787a5607", + "trace_id": "945b2757-8a5a-4aab-a869-8f44c02166a5", + "parent_span_id": "59595232-8873-419c-9b6a-0727445a2afa", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-14 12:53:27.771000000", + "end_time": "2025-10-14 12:53:27.771000000", + "duration_ms": 0.031, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What are the current trends in observability tools?", + "query_length": 51, + "query_type": "routing_evaluation_query", + "classification.mode": "RAG", + "classification.rag_keyword_score": 2, + "classification.web_keyword_score": 1, + "classification.confidence_score": 0.3333333333333333, + "classification.confidence_level": "medium", + "classification.method": "keyword_based", + "query.complexity": "complex", + "query.intent": "documentation", + "query.keyword_density": 0.375, + "query.domain_affinity": "noveum", + "routing.decision": "Routed to RAG based on analysis", + "routing.rationale": "RAG score: 2, Web score: 1, Confidence: 0.33", + "routing.expected_performance": "medium", + "routing.alternative_mode": "Web Search", + "routing.decision_confidence": 0.3333333333333333, + "evaluation.routing_accuracy": "medium", + "evaluation.keyword_coverage": 0.057692307692307696, + "evaluation.query_understanding": "ambiguous", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: RAG (Confidence: 0.33)" + }, + "events": [], + "links": [], + "trace_trace_id": "945b2757-8a5a-4aab-a869-8f44c02166a5", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-14 12:53:27.771000000", + "trace_end_time": "2025-10-14 12:53:33.575000000", + "trace_duration_ms": 5804.15, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-14 12:53:27.771000000", + "updated_at": "2025-10-14 12:53:27.771000000" + }, + { + "span_id": "dfde2f79-d0c1-481e-819d-60730cb60fe5", + "trace_id": "945b2757-8a5a-4aab-a869-8f44c02166a5", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-14 12:53:27.771000000", + "end_time": "2025-10-14 12:53:33.575000000", + "duration_ms": 5804.074, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What are the current trends in observability tools?", + "query.length": 51, + "processing.mode": "RAG", + "processing.time_seconds": 5.804032802581787, + "processing.response_length": 997, + "processing.sources_count": 5, + "output_response": "Final Answer: The provided context does not specifically outline current trends in observability tools in general. However, it highlights the unique challenges faced by AI applications and how traditional monitorin...", + "final_answer_mode": "RAG", + "query_processed.input_query": "What are the current trends in observability tools?", + "query_processed.output_response": "Successfully processed query using RAG, generated 997 character response", + "query_processed.mode": "RAG", + "query_processed.processing_time": 5.804032802581787, + "query_processed.response_length": 997 + }, + "events": [], + "links": [], + "trace_trace_id": "945b2757-8a5a-4aab-a869-8f44c02166a5", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-14 12:53:27.771000000", + "trace_end_time": "2025-10-14 12:53:33.575000000", + "trace_duration_ms": 5804.15, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-14 12:53:27.771000000", + "updated_at": "2025-10-14 12:53:27.771000000" + }, + { + "span_id": "82489444-deba-46d2-b08b-c09754189566", + "trace_id": "945b2757-8a5a-4aab-a869-8f44c02166a5", + "parent_span_id": "59595232-8873-419c-9b6a-0727445a2afa", + "name": "agent.llm-rag", + "start_time": "2025-10-14 12:53:27.772000000", + "end_time": "2025-10-14 12:53:33.575000000", + "duration_ms": 5803.534, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "rag_agent", + "agent.operation": "llm-rag", + "agent.capabilities": [ + "document_retrieval", + "context_generation", + "response_generation" + ], + "agent.id": "noveum_rag_agent", + "input_query": "What are the current trends in observability tools?", + "query_length": 51, + "query_type": "rag_query", + "output_response": "RAG Answer: The provided context does not specifically outline current trends in observability tools in general. However, it highlights the unique challenges faced by AI applications and how traditional monitorin...", + "rag.context_length": 2943, + "rag.sources_count": 5, + "rag.answer_length": 997, + "rag.mode": "retrieval_augmented_generation" + }, + "events": [], + "links": [], + "trace_trace_id": "945b2757-8a5a-4aab-a869-8f44c02166a5", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-14 12:53:27.771000000", + "trace_end_time": "2025-10-14 12:53:33.575000000", + "trace_duration_ms": 5804.15, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-14 12:53:27.771000000", + "updated_at": "2025-10-14 12:53:27.771000000" + }, + { + "span_id": "34b1f14c-3fd3-4f89-b4dd-6150a52befd7", + "trace_id": "945b2757-8a5a-4aab-a869-8f44c02166a5", + "parent_span_id": "82489444-deba-46d2-b08b-c09754189566", + "name": "agent.llm_model_execution", + "start_time": "2025-10-14 12:53:28.257000000", + "end_time": "2025-10-14 12:53:33.162000000", + "duration_ms": 4905.2429999999995, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What are the current trends in observability tools?", + "query_length": 51, + "query_type": "rag_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 4.9052040576934814, + "model.latency_ms": 4905.204057693481, + "model.start_time": 1760446408.2575018, + "model.end_time": 1760446413.162706, + "model.performance_tier": "medium", + "model.prompt_tokens": 872, + "model.completion_tokens": 249, + "model.total_tokens": 1121, + "model.tokens_per_second": 228.53279635569638, + "model.estimated_cost": 0.03363, + "model.efficiency_score": 0.8893844781445138, + "model.response_length": 997, + "model.response_quality": "high", + "model.output_response": "Model Response: The provided context does not specifically outline current trends in observability tools in general. However, it highlights the unique challenges faced by AI applications and how traditional monitorin...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "945b2757-8a5a-4aab-a869-8f44c02166a5", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-14 12:53:27.771000000", + "trace_end_time": "2025-10-14 12:53:33.575000000", + "trace_duration_ms": 5804.15, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-14 12:53:27.771000000", + "updated_at": "2025-10-14 12:53:27.771000000" + }, + { + "span_id": "d5d2b966-3c37-4d6d-b7b8-c58c9f22aad9", + "trace_id": "945b2757-8a5a-4aab-a869-8f44c02166a5", + "parent_span_id": "82489444-deba-46d2-b08b-c09754189566", + "name": "agent.rag_evaluation_metrics", + "start_time": "2025-10-14 12:53:33.162000000", + "end_time": "2025-10-14 12:53:33.162000000", + "duration_ms": 0.036000000000000004, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "rag_evaluation_metrics", + "agent.capabilities": [ + "retrieval_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What are the current trends in observability tools?", + "query_length": 51, + "query_type": "rag_evaluation_query", + "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/blog/from-logs-to-intelligent-choices-inside-noveum-ais-evaluation-process):\nFrom Development to Production - Inside Noveum.ai's AI Observability Platform\nShashank Agarwal\n3/3/2025\nIntroduction\nEvery AI application tells a story\u2014but most developers never get to hear it...", + "retrieval.context_length": 2943, + "retrieval.sources_count": 5, + "retrieval.context_quality": "high", + "retrieval.effectiveness": 1, + "retrieval.context_utilization": 2.943, + "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/blog/from-logs-to-intelligent-choices-inside-noveum-ais-evaluation-process):\nFrom Development to Production - Inside Noveum.ai's AI Observability Platform\nShashank Agarwal\n3/3/2025\nIntroduction\nEvery AI application tells a story\u2014but most developers never get to hear it. When your RAG pipeline returns irrelevant results, when your multi-agent system gets stuck in loops, or when your LLM costs suddenly spike, you're left guessing what went wrong and where.\nThat's exactly the challenge Noveum.ai solves. Rather than flying blind, our platform provides comprehensive tracing a...\n\nSource 2 (https://noveum.ai/docs/getting-started/tracing-concepts):\n- Explore Framework Integrations - Framework-specific guidance\n- Learn Advanced Patterns - Custom instrumentation techniques\n- Master the Dashboard - Analyze your traces effectively\nRemember: Good observability is not about collecting all possible data, but about collecting the right data that helps you understand, debug, and optimize your AI applications.\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to O...\n\nSource 3 (https://noveum.ai/en/docs/getting-started/tracing-concepts):\n- Explore Framework Integrations - Framework-specific guidance\n- Learn Advanced Patterns - Custom instrumentation techniques\n- Master the Dashboard - Analyze your traces effectively\nRemember: Good observability is not about collecting all possible data, but about collecting the right data that helps you understand, debug, and optimize your AI applications.\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to O...\n\nSource 4 (https://noveum.ai/en/blog/from-logs-to-intelligent-choices-inside-noveum-ais-evaluation-process):\nThe AI Observability Challenge\nWhy Traditional Monitoring Falls Short\nTraditional application monitoring tools weren't built for AI applications. They can tell you if your API is responding, but they can't answer the questions that matter most for AI systems:\n- Why did my RAG pipeline retrieve irrelevant documents?\n- Which LLM calls are driving my costs?\n- How are my agents communicating with each other?\n- What's causing hallucinations in my responses?\n- Why is my embedding generation so slow?\nW...\n\nSource 5 (https://noveum.ai/en/docs/advanced/rag-observability):\n\ud83d\udd27 Best Practices for RAG Observability\n1. Comprehensive Pipeline Tracing\n2. Quality Monitoring\n3. A/B Testing for RAG Components\n\ud83c\udfaf Advanced RAG Patterns\nMulti-Modal RAG\nConversational RAG\nRAG observability with Noveum.ai provides the deep insights needed to build, optimize, and scale retrieval-augmented generation systems. By implementing comprehensive tracing across retrieval, context preparation, and generation stages, you can ensure your RAG pipeline delivers accurate, relevant, and cost-effe...\n\nUser Question: What are the current trends in observability tools?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", + "prompt.prompt_length": 3490, + "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/blog/from-logs-to-intelligent-choices-inside-noveum-ais-evaluation-process):\nFrom Development to Production - Inside Noveum.ai's AI Observability Platform\nShashank Agarw...", + "prompt.instruction_following": "rag_optimized", + "response.answer_length": 997, + "response.answer_completeness": "complete", + "response.response_quality": "high", + "response.source_citation": 5, + "response.context_utilization": 0.33876996262317366, + "output_response": "RAG Answer: The provided context does not specifically outline current trends in observability tools in general. However, it highlights the unique challenges faced by AI applications and how traditional monitorin...", + "evaluation.retrieval_effectiveness": 1, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 5, + "evaluation.context_relevance": "high", + "evaluation.overall_quality": "high", + "evaluation.ready_for_production": true, + "rag.retrieval_strategy": "semantic_similarity", + "rag.vector_search_results": 5, + "rag.context_synthesis": "multi_source", + "rag.document_coverage": 1, + "rag.information_density": 0.33876996262317366 + }, + "events": [], + "links": [], + "trace_trace_id": "945b2757-8a5a-4aab-a869-8f44c02166a5", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-14 12:53:27.771000000", + "trace_end_time": "2025-10-14 12:53:33.575000000", + "trace_duration_ms": 5804.15, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-14 12:53:27.771000000", + "updated_at": "2025-10-14 12:53:27.771000000" + }, + { + "span_id": "11c07411-df30-42a4-9062-90a032ef75ed", + "trace_id": "4bdd5814-acbb-4de7-a56b-543b87a1cfa5", + "parent_span_id": "ec530838-a7cf-477b-9186-a70114c8a12f", + "name": "agent.web_search_generation", + "start_time": "2025-10-14 12:53:20.120000000", + "end_time": "2025-10-14 12:53:26.750000000", + "duration_ms": 6629.273999999999, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "web_search_agent", + "agent.operation": "web_search_generation", + "agent.capabilities": [ + "web_search", + "content_synthesis", + "response_generation" + ], + "agent.id": "noveum_web_search_agent", + "input_query": "Tell me about recent developments in machine learning", + "query_length": 53, + "query_type": "web_search_query", + "output_response": "Web Search Answer: The web search did not yield specific results regarding recent developments in machine learning. However, I can provide a general overview based on my knowledge up to October 2023.\n\nRecent development...", + "web_search.results_count": 1, + "web_search.context_length": 155, + "web_search.response_length": 1849, + "web_search.mode": "external_web_search" + }, + "events": [], + "links": [], + "trace_trace_id": "4bdd5814-acbb-4de7-a56b-543b87a1cfa5", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-14 12:53:20.120000000", + "trace_end_time": "2025-10-14 12:53:26.750000000", + "trace_duration_ms": 6630.38, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-14 12:53:20.120000000", + "updated_at": "2025-10-14 12:53:20.120000000" + }, + { + "span_id": "282b121f-61dd-4dc5-9e55-cecc3f02f688", + "trace_id": "4bdd5814-acbb-4de7-a56b-543b87a1cfa5", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-14 12:53:20.120000000", + "end_time": "2025-10-14 12:53:26.750000000", + "duration_ms": 6630.264, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "Tell me about recent developments in machine learning", + "query.length": 53, + "processing.mode": "Web Search", + "processing.time_seconds": 6.629686117172241, + "processing.response_length": 1849, + "processing.sources_count": 1, + "output_response": "Final Answer: The web search did not yield specific results regarding recent developments in machine learning. However, I can provide a general overview based on my knowledge up to October 2023.\n\nRecent development...", + "final_answer_mode": "Web Search", + "query_processed.input_query": "Tell me about recent developments in machine learning", + "query_processed.output_response": "Successfully processed query using Web Search, generated 1849 character response", + "query_processed.mode": "Web Search", + "query_processed.processing_time": 6.629686117172241, + "query_processed.response_length": 1849 + }, + "events": [], + "links": [], + "trace_trace_id": "4bdd5814-acbb-4de7-a56b-543b87a1cfa5", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-14 12:53:20.120000000", + "trace_end_time": "2025-10-14 12:53:26.750000000", + "trace_duration_ms": 6630.38, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-14 12:53:20.120000000", + "updated_at": "2025-10-14 12:53:20.120000000" + }, + { + "span_id": "ec530838-a7cf-477b-9186-a70114c8a12f", + "trace_id": "4bdd5814-acbb-4de7-a56b-543b87a1cfa5", + "parent_span_id": "282b121f-61dd-4dc5-9e55-cecc3f02f688", + "name": "agent.query_routing", + "start_time": "2025-10-14 12:53:20.120000000", + "end_time": "2025-10-14 12:53:26.750000000", + "duration_ms": 6629.59, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "Tell me about recent developments in machine learning", + "query_length": 53, + "query_type": "routing_query", + "output_response": "Routed to Web Search for query processing", + "router.classification": "Web Search", + "router.confidence_score": 1, + "router.rag_keyword_score": 0, + "router.web_keyword_score": 1, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "4bdd5814-acbb-4de7-a56b-543b87a1cfa5", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-14 12:53:20.120000000", + "trace_end_time": "2025-10-14 12:53:26.750000000", + "trace_duration_ms": 6630.38, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-14 12:53:20.120000000", + "updated_at": "2025-10-14 12:53:20.120000000" + }, + { + "span_id": "df882f90-cf1a-42a6-a2ae-78b2a13f27a1", + "trace_id": "4bdd5814-acbb-4de7-a56b-543b87a1cfa5", + "parent_span_id": "ec530838-a7cf-477b-9186-a70114c8a12f", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-14 12:53:20.120000000", + "end_time": "2025-10-14 12:53:20.120000000", + "duration_ms": 0.082, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: Tell me about recent developments in machine learning", + "query_length": 53, + "query_type": "routing_evaluation_query", + "classification.mode": "Web Search", + "classification.rag_keyword_score": 0, + "classification.web_keyword_score": 1, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "complex", + "query.intent": "general_knowledge", + "query.keyword_density": 0.125, + "query.domain_affinity": "general", + "routing.decision": "Routed to Web Search based on analysis", + "routing.rationale": "RAG score: 0, Web score: 1, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "RAG", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.019230769230769232, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: Web Search (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "4bdd5814-acbb-4de7-a56b-543b87a1cfa5", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-14 12:53:20.120000000", + "trace_end_time": "2025-10-14 12:53:26.750000000", + "trace_duration_ms": 6630.38, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-14 12:53:20.120000000", + "updated_at": "2025-10-14 12:53:20.120000000" + }, + { + "span_id": "bea819fd-3673-480b-9f7b-a3a2efef4218", + "trace_id": "4bdd5814-acbb-4de7-a56b-543b87a1cfa5", + "parent_span_id": "11c07411-df30-42a4-9062-90a032ef75ed", + "name": "agent.llm_model_execution", + "start_time": "2025-10-14 12:53:20.455000000", + "end_time": "2025-10-14 12:53:26.746000000", + "duration_ms": 6291.401, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: Tell me about recent developments in machine learning", + "query_length": 53, + "query_type": "web_search_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 6.290812253952026, + "model.latency_ms": 6290.812253952026, + "model.start_time": 1760446400.4555368, + "model.end_time": 1760446406.746349, + "model.performance_tier": "slow", + "model.prompt_tokens": 163, + "model.completion_tokens": 462, + "model.total_tokens": 625, + "model.tokens_per_second": 99.35124031199013, + "model.estimated_cost": 0.01875, + "model.efficiency_score": 2.9584, + "model.response_length": 1849, + "model.response_quality": "high", + "model.output_response": "Model Response: The web search did not yield specific results regarding recent developments in machine learning. However, I can provide a general overview based on my knowledge up to October 2023.\n\nRecent development...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "4bdd5814-acbb-4de7-a56b-543b87a1cfa5", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-14 12:53:20.120000000", + "trace_end_time": "2025-10-14 12:53:26.750000000", + "trace_duration_ms": 6630.38, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-14 12:53:20.120000000", + "updated_at": "2025-10-14 12:53:20.120000000" + }, + { + "span_id": "6e62af18-569c-4783-b2ae-177c58542bd0", + "trace_id": "4bdd5814-acbb-4de7-a56b-543b87a1cfa5", + "parent_span_id": "11c07411-df30-42a4-9062-90a032ef75ed", + "name": "agent.web_search_evaluation_metrics", + "start_time": "2025-10-14 12:53:26.749000000", + "end_time": "2025-10-14 12:53:26.750000000", + "duration_ms": 0.217, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "web_search_evaluation_metrics", + "agent.capabilities": [ + "web_search_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: Tell me about recent developments in machine learning", + "query_length": 53, + "query_type": "web_search_evaluation_query", + "web_search.results_count": 1, + "web_search.context_length": 155, + "web_search.context_synthesized": "Context from 1 web sources", + "web_search.search_effectiveness": 0.2, + "web_search.context_quality": "low", + "web_search.source_diversity": 1, + "web_search.information_synthesis": "low", + "web_search.external_knowledge_utilization": 0.155, + "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=Tell+me+about+recent+developments+in+machine+learning\n\nUser Question: Tell me about recent developments in machine learning\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", + "prompt.prompt_length": 653, + "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=Tell+me+about+recent+developments+in+machine+learning", + "prompt.instruction_following": "web_search_optimized", + "response.answer_length": 1849, + "response.answer_completeness": "complete", + "response.response_quality": "medium", + "response.source_citation": 1, + "response.context_utilization": 11.929032258064517, + "output_response": "Web Search Answer: The web search did not yield specific results regarding recent developments in machine learning. However, I can provide a general overview based on my knowledge up to October 2023.\n\nRecent development...", + "evaluation.search_effectiveness": 0.2, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 1, + "evaluation.context_relevance": "low", + "evaluation.overall_quality": "low", + "evaluation.ready_for_production": true, + "web_search.search_strategy": "duckduckgo_api", + "web_search.real_time_data": true, + "web_search.external_sources": 1, + "web_search.information_freshness": "current", + "web_search.knowledge_synthesis": "single_source" + }, + "events": [], + "links": [], + "trace_trace_id": "4bdd5814-acbb-4de7-a56b-543b87a1cfa5", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-14 12:53:20.120000000", + "trace_end_time": "2025-10-14 12:53:26.750000000", + "trace_duration_ms": 6630.38, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-14 12:53:20.120000000", + "updated_at": "2025-10-14 12:53:20.120000000" + } +] \ No newline at end of file diff --git a/noveum_customer_support_bt/traces/traces/dataset_filtered.json b/noveum_customer_support_bt/traces/traces/dataset_filtered.json new file mode 100644 index 0000000..5578d29 --- /dev/null +++ b/noveum_customer_support_bt/traces/traces/dataset_filtered.json @@ -0,0 +1,20904 @@ +[ + { + "span_id": "4149abdc-acdc-448c-b9b0-b09e166e461a", + "trace_id": "6fe93abf-1413-49e8-8464-58558e25ab7b", + "parent_span_id": "817c9703-0062-4bd6-b3ee-b50a1121c2d2", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-22 16:58:42.084000000", + "end_time": "2025-10-22 16:58:42.084000000", + "duration_ms": 0.015000000000000001, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What are the latest breakthroughs in artificial intelligence?", + "query_length": 61, + "query_type": "routing_evaluation_query", + "classification.mode": "Web Search", + "classification.rag_keyword_score": 0, + "classification.web_keyword_score": 1, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "complex", + "query.intent": "general_knowledge", + "query.keyword_density": 0.125, + "query.domain_affinity": "general", + "routing.decision": "Routed to Web Search based on analysis", + "routing.rationale": "RAG score: 0, Web score: 1, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "RAG", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.019230769230769232, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: Web Search (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "6fe93abf-1413-49e8-8464-58558e25ab7b", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:58:42.084000000", + "trace_end_time": "2025-10-22 16:58:44.766000000", + "trace_duration_ms": 2681.9289999999996, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:58:42.084000000", + "updated_at": "2025-10-22 16:58:42.084000000" + }, + { + "span_id": "817c9703-0062-4bd6-b3ee-b50a1121c2d2", + "trace_id": "6fe93abf-1413-49e8-8464-58558e25ab7b", + "parent_span_id": "ff171ffc-dc6b-4cb4-9833-64c5622d9941", + "name": "agent.query_routing", + "start_time": "2025-10-22 16:58:42.084000000", + "end_time": "2025-10-22 16:58:44.766000000", + "duration_ms": 2681.799, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What are the latest breakthroughs in artificial intelligence?", + "query_length": 61, + "query_type": "routing_query", + "output_response": "Routed to Web Search for query processing", + "router.classification": "Web Search", + "router.confidence_score": 1, + "router.rag_keyword_score": 0, + "router.web_keyword_score": 1, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "6fe93abf-1413-49e8-8464-58558e25ab7b", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:58:42.084000000", + "trace_end_time": "2025-10-22 16:58:44.766000000", + "trace_duration_ms": 2681.9289999999996, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:58:42.084000000", + "updated_at": "2025-10-22 16:58:42.084000000" + }, + { + "span_id": "ad476d62-d6b5-4142-8771-29855db3f39e", + "trace_id": "6fe93abf-1413-49e8-8464-58558e25ab7b", + "parent_span_id": "817c9703-0062-4bd6-b3ee-b50a1121c2d2", + "name": "agent.web_search_generation", + "start_time": "2025-10-22 16:58:42.084000000", + "end_time": "2025-10-22 16:58:44.766000000", + "duration_ms": 2681.732, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "web_search_agent", + "agent.operation": "web_search_generation", + "agent.capabilities": [ + "web_search", + "content_synthesis", + "response_generation" + ], + "agent.id": "noveum_web_search_agent", + "input_query": "What are the latest breakthroughs in artificial intelligence?", + "query_length": 61, + "query_type": "web_search_query", + "output_response": "Web Search Answer: The web search did not yield specific results regarding the latest breakthroughs in artificial intelligence. However, I can provide a general overview based on knowledge up to October 2023.\n\nRecent br...", + "web_search.results_count": 1, + "web_search.context_length": 163, + "web_search.response_length": 1040, + "web_search.mode": "external_web_search" + }, + "events": [], + "links": [], + "trace_trace_id": "6fe93abf-1413-49e8-8464-58558e25ab7b", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:58:42.084000000", + "trace_end_time": "2025-10-22 16:58:44.766000000", + "trace_duration_ms": 2681.9289999999996, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:58:42.084000000", + "updated_at": "2025-10-22 16:58:42.084000000" + }, + { + "span_id": "ff171ffc-dc6b-4cb4-9833-64c5622d9941", + "trace_id": "6fe93abf-1413-49e8-8464-58558e25ab7b", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-22 16:58:42.084000000", + "end_time": "2025-10-22 16:58:44.766000000", + "duration_ms": 2681.902, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What are the latest breakthroughs in artificial intelligence?", + "query.length": 61, + "processing.mode": "Web Search", + "processing.time_seconds": 2.6818161010742188, + "processing.response_length": 1040, + "processing.sources_count": 1, + "output_response": "Final Answer: The web search did not yield specific results regarding the latest breakthroughs in artificial intelligence. However, I can provide a general overview based on knowledge up to October 2023.\n\nRecent br...", + "final_answer_mode": "Web Search", + "query_processed.input_query": "What are the latest breakthroughs in artificial intelligence?", + "query_processed.output_response": "Successfully processed query using Web Search, generated 1040 character response", + "query_processed.mode": "Web Search", + "query_processed.processing_time": 2.6818161010742188, + "query_processed.response_length": 1040 + }, + "events": [], + "links": [], + "trace_trace_id": "6fe93abf-1413-49e8-8464-58558e25ab7b", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:58:42.084000000", + "trace_end_time": "2025-10-22 16:58:44.766000000", + "trace_duration_ms": 2681.9289999999996, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:58:42.084000000", + "updated_at": "2025-10-22 16:58:42.084000000" + }, + { + "span_id": "467cb806-a6ef-43da-9739-570730dd244b", + "trace_id": "6fe93abf-1413-49e8-8464-58558e25ab7b", + "parent_span_id": "ad476d62-d6b5-4142-8771-29855db3f39e", + "name": "agent.llm_model_execution", + "start_time": "2025-10-22 16:58:42.438000000", + "end_time": "2025-10-22 16:58:44.766000000", + "duration_ms": 2327.706, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What are the latest breakthroughs in artificial intelligence?", + "query_length": 61, + "query_type": "web_search_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 2.3276820182800293, + "model.latency_ms": 2327.6820182800293, + "model.start_time": 1761152322.438459, + "model.end_time": 1761152324.766141, + "model.performance_tier": "medium", + "model.prompt_tokens": 167, + "model.completion_tokens": 260, + "model.total_tokens": 427, + "model.tokens_per_second": 183.44430065903882, + "model.estimated_cost": 0.01281, + "model.efficiency_score": 2.4355971896955504, + "model.response_length": 1040, + "model.response_quality": "high", + "model.output_response": "Model Response: The web search did not yield specific results regarding the latest breakthroughs in artificial intelligence. However, I can provide a general overview based on knowledge up to October 2023.\n\nRecent br...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "6fe93abf-1413-49e8-8464-58558e25ab7b", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:58:42.084000000", + "trace_end_time": "2025-10-22 16:58:44.766000000", + "trace_duration_ms": 2681.9289999999996, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:58:42.084000000", + "updated_at": "2025-10-22 16:58:42.084000000" + }, + { + "span_id": "a80e4596-01ea-44e0-a590-13e347857f2a", + "trace_id": "6fe93abf-1413-49e8-8464-58558e25ab7b", + "parent_span_id": "ad476d62-d6b5-4142-8771-29855db3f39e", + "name": "agent.web_search_evaluation_metrics", + "start_time": "2025-10-22 16:58:44.766000000", + "end_time": "2025-10-22 16:58:44.766000000", + "duration_ms": 0.008, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "web_search_evaluation_metrics", + "agent.capabilities": [ + "web_search_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What are the latest breakthroughs in artificial intelligence?", + "query_length": 61, + "query_type": "web_search_evaluation_query", + "web_search.results_count": 1, + "web_search.context_length": 163, + "web_search.context_synthesized": "Context from 1 web sources", + "web_search.search_effectiveness": 0.2, + "web_search.context_quality": "low", + "web_search.source_diversity": 1, + "web_search.information_synthesis": "low", + "web_search.external_knowledge_utilization": 0.163, + "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What+are+the+latest+breakthroughs+in+artificial+intelligence?\n\nUser Question: What are the latest breakthroughs in artificial intelligence?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", + "prompt.prompt_length": 669, + "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What+are+the+latest+breakthroughs+in+artificial+intelligence?", + "prompt.instruction_following": "web_search_optimized", + "response.answer_length": 1040, + "response.answer_completeness": "complete", + "response.response_quality": "medium", + "response.source_citation": 1, + "response.context_utilization": 6.380368098159509, + "output_response": "Web Search Answer: The web search did not yield specific results regarding the latest breakthroughs in artificial intelligence. However, I can provide a general overview based on knowledge up to October 2023.\n\nRecent br...", + "evaluation.search_effectiveness": 0.2, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 1, + "evaluation.context_relevance": "low", + "evaluation.overall_quality": "low", + "evaluation.ready_for_production": true, + "web_search.search_strategy": "duckduckgo_api", + "web_search.real_time_data": true, + "web_search.external_sources": 1, + "web_search.information_freshness": "current", + "web_search.knowledge_synthesis": "single_source" + }, + "events": [], + "links": [], + "trace_trace_id": "6fe93abf-1413-49e8-8464-58558e25ab7b", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:58:42.084000000", + "trace_end_time": "2025-10-22 16:58:44.766000000", + "trace_duration_ms": 2681.9289999999996, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:58:42.084000000", + "updated_at": "2025-10-22 16:58:42.084000000" + }, + { + "span_id": "409f29f6-21c7-400a-a9e0-11c87c4313cb", + "trace_id": "ec7c6cae-9bbd-47f4-a8ae-2d6d9b5011ba", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-22 16:58:32.801000000", + "end_time": "2025-10-22 16:58:41.077000000", + "duration_ms": 8276.701, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What's happening in the software development world today?", + "query.length": 57, + "processing.mode": "Web Search", + "processing.time_seconds": 8.276506900787354, + "processing.response_length": 1424, + "processing.sources_count": 1, + "output_response": "Final Answer: The web search did not yield any specific results regarding current events in the software development world. Therefore, I cannot provide detailed information on the latest trends, technologies, or ne...", + "final_answer_mode": "Web Search", + "query_processed.input_query": "What's happening in the software development world today?", + "query_processed.output_response": "Successfully processed query using Web Search, generated 1424 character response", + "query_processed.mode": "Web Search", + "query_processed.processing_time": 8.276506900787354, + "query_processed.response_length": 1424 + }, + "events": [], + "links": [], + "trace_trace_id": "ec7c6cae-9bbd-47f4-a8ae-2d6d9b5011ba", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:58:32.800000000", + "trace_end_time": "2025-10-22 16:58:41.077000000", + "trace_duration_ms": 8276.788, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:58:32.800000000", + "updated_at": "2025-10-22 16:58:32.800000000" + }, + { + "span_id": "43389a18-a566-41e7-8db5-f688f651a106", + "trace_id": "ec7c6cae-9bbd-47f4-a8ae-2d6d9b5011ba", + "parent_span_id": "af400e17-8b20-4218-834b-9330ccaa2635", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-22 16:58:32.801000000", + "end_time": "2025-10-22 16:58:32.801000000", + "duration_ms": 0.047, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What's happening in the software development world today?", + "query_length": 57, + "query_type": "routing_evaluation_query", + "classification.mode": "Web Search", + "classification.rag_keyword_score": 0, + "classification.web_keyword_score": 1, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "complex", + "query.intent": "general_knowledge", + "query.keyword_density": 0.125, + "query.domain_affinity": "general", + "routing.decision": "Routed to Web Search based on analysis", + "routing.rationale": "RAG score: 0, Web score: 1, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "RAG", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.019230769230769232, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: Web Search (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "ec7c6cae-9bbd-47f4-a8ae-2d6d9b5011ba", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:58:32.800000000", + "trace_end_time": "2025-10-22 16:58:41.077000000", + "trace_duration_ms": 8276.788, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:58:32.800000000", + "updated_at": "2025-10-22 16:58:32.800000000" + }, + { + "span_id": "6b431bca-af7e-4f7e-aedb-b103d7f47cae", + "trace_id": "ec7c6cae-9bbd-47f4-a8ae-2d6d9b5011ba", + "parent_span_id": "af400e17-8b20-4218-834b-9330ccaa2635", + "name": "agent.web_search_generation", + "start_time": "2025-10-22 16:58:32.801000000", + "end_time": "2025-10-22 16:58:41.077000000", + "duration_ms": 8276.225999999999, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "web_search_agent", + "agent.operation": "web_search_generation", + "agent.capabilities": [ + "web_search", + "content_synthesis", + "response_generation" + ], + "agent.id": "noveum_web_search_agent", + "input_query": "What's happening in the software development world today?", + "query_length": 57, + "query_type": "web_search_query", + "output_response": "Web Search Answer: The web search did not yield any specific results regarding current events in the software development world. Therefore, I cannot provide detailed information on the latest trends, technologies, or ne...", + "web_search.results_count": 1, + "web_search.context_length": 159, + "web_search.response_length": 1424, + "web_search.mode": "external_web_search" + }, + "events": [], + "links": [], + "trace_trace_id": "ec7c6cae-9bbd-47f4-a8ae-2d6d9b5011ba", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:58:32.800000000", + "trace_end_time": "2025-10-22 16:58:41.077000000", + "trace_duration_ms": 8276.788, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:58:32.800000000", + "updated_at": "2025-10-22 16:58:32.800000000" + }, + { + "span_id": "af400e17-8b20-4218-834b-9330ccaa2635", + "trace_id": "ec7c6cae-9bbd-47f4-a8ae-2d6d9b5011ba", + "parent_span_id": "409f29f6-21c7-400a-a9e0-11c87c4313cb", + "name": "agent.query_routing", + "start_time": "2025-10-22 16:58:32.801000000", + "end_time": "2025-10-22 16:58:41.077000000", + "duration_ms": 8276.465999999999, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What's happening in the software development world today?", + "query_length": 57, + "query_type": "routing_query", + "output_response": "Routed to Web Search for query processing", + "router.classification": "Web Search", + "router.confidence_score": 1, + "router.rag_keyword_score": 0, + "router.web_keyword_score": 1, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "ec7c6cae-9bbd-47f4-a8ae-2d6d9b5011ba", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:58:32.800000000", + "trace_end_time": "2025-10-22 16:58:41.077000000", + "trace_duration_ms": 8276.788, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:58:32.800000000", + "updated_at": "2025-10-22 16:58:32.800000000" + }, + { + "span_id": "37b153b2-d02b-4b7e-9229-1ba3eaae44ee", + "trace_id": "ec7c6cae-9bbd-47f4-a8ae-2d6d9b5011ba", + "parent_span_id": "6b431bca-af7e-4f7e-aedb-b103d7f47cae", + "name": "agent.llm_model_execution", + "start_time": "2025-10-22 16:58:33.172000000", + "end_time": "2025-10-22 16:58:41.077000000", + "duration_ms": 7904.722, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What's happening in the software development world today?", + "query_length": 57, + "query_type": "web_search_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 7.904651880264282, + "model.latency_ms": 7904.651880264282, + "model.start_time": 1761152313.172655, + "model.end_time": 1761152321.077307, + "model.performance_tier": "slow", + "model.prompt_tokens": 165, + "model.completion_tokens": 356, + "model.total_tokens": 521, + "model.tokens_per_second": 65.91055594754174, + "model.estimated_cost": 0.01563, + "model.efficiency_score": 2.7332053742802302, + "model.response_length": 1424, + "model.response_quality": "high", + "model.output_response": "Model Response: The web search did not yield any specific results regarding current events in the software development world. Therefore, I cannot provide detailed information on the latest trends, technologies, or ne...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "ec7c6cae-9bbd-47f4-a8ae-2d6d9b5011ba", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:58:32.800000000", + "trace_end_time": "2025-10-22 16:58:41.077000000", + "trace_duration_ms": 8276.788, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:58:32.800000000", + "updated_at": "2025-10-22 16:58:32.800000000" + }, + { + "span_id": "cc941b8e-6625-43e9-a0a9-7328af24f12f", + "trace_id": "ec7c6cae-9bbd-47f4-a8ae-2d6d9b5011ba", + "parent_span_id": "6b431bca-af7e-4f7e-aedb-b103d7f47cae", + "name": "agent.web_search_evaluation_metrics", + "start_time": "2025-10-22 16:58:41.077000000", + "end_time": "2025-10-22 16:58:41.077000000", + "duration_ms": 0.026, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "web_search_evaluation_metrics", + "agent.capabilities": [ + "web_search_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What's happening in the software development world today?", + "query_length": 57, + "query_type": "web_search_evaluation_query", + "web_search.results_count": 1, + "web_search.context_length": 159, + "web_search.context_synthesized": "Context from 1 web sources", + "web_search.search_effectiveness": 0.2, + "web_search.context_quality": "low", + "web_search.source_diversity": 1, + "web_search.information_synthesis": "low", + "web_search.external_knowledge_utilization": 0.159, + "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What's+happening+in+the+software+development+world+today?\n\nUser Question: What's happening in the software development world today?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", + "prompt.prompt_length": 661, + "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What's+happening+in+the+software+development+world+today?", + "prompt.instruction_following": "web_search_optimized", + "response.answer_length": 1424, + "response.answer_completeness": "complete", + "response.response_quality": "medium", + "response.source_citation": 1, + "response.context_utilization": 8.955974842767295, + "output_response": "Web Search Answer: The web search did not yield any specific results regarding current events in the software development world. Therefore, I cannot provide detailed information on the latest trends, technologies, or ne...", + "evaluation.search_effectiveness": 0.2, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 1, + "evaluation.context_relevance": "low", + "evaluation.overall_quality": "low", + "evaluation.ready_for_production": true, + "web_search.search_strategy": "duckduckgo_api", + "web_search.real_time_data": true, + "web_search.external_sources": 1, + "web_search.information_freshness": "current", + "web_search.knowledge_synthesis": "single_source" + }, + "events": [], + "links": [], + "trace_trace_id": "ec7c6cae-9bbd-47f4-a8ae-2d6d9b5011ba", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:58:32.800000000", + "trace_end_time": "2025-10-22 16:58:41.077000000", + "trace_duration_ms": 8276.788, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:58:32.800000000", + "updated_at": "2025-10-22 16:58:32.800000000" + }, + { + "span_id": "3b06d098-6eb7-4bb6-885e-443ddb28cc41", + "trace_id": "343764b8-7ac9-4549-93cf-8abc11ceba6f", + "parent_span_id": "5c2aa1cf-56a8-4fd5-af90-90a94e723bb5", + "name": "agent.query_routing", + "start_time": "2025-10-22 16:58:26.542000000", + "end_time": "2025-10-22 16:58:31.796000000", + "duration_ms": 5253.835, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What are the newest features in cloud computing?", + "query_length": 48, + "query_type": "routing_query", + "output_response": "Routed to RAG for query processing", + "router.classification": "RAG", + "router.confidence_score": 1, + "router.rag_keyword_score": 1, + "router.web_keyword_score": 0, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "343764b8-7ac9-4549-93cf-8abc11ceba6f", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:58:26.542000000", + "trace_end_time": "2025-10-22 16:58:31.796000000", + "trace_duration_ms": 5253.973, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:58:26.542000000", + "updated_at": "2025-10-22 16:58:26.542000000" + }, + { + "span_id": "5c2aa1cf-56a8-4fd5-af90-90a94e723bb5", + "trace_id": "343764b8-7ac9-4549-93cf-8abc11ceba6f", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-22 16:58:26.542000000", + "end_time": "2025-10-22 16:58:31.796000000", + "duration_ms": 5253.913, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What are the newest features in cloud computing?", + "query.length": 48, + "processing.mode": "RAG", + "processing.time_seconds": 5.253868818283081, + "processing.response_length": 837, + "processing.sources_count": 5, + "output_response": "Final Answer: The provided context does not specifically mention any new features in cloud computing as a general topic. However, it does highlight several recent enhancements related to the Noveum.ai platform, whi...", + "final_answer_mode": "RAG", + "query_processed.input_query": "What are the newest features in cloud computing?", + "query_processed.output_response": "Successfully processed query using RAG, generated 837 character response", + "query_processed.mode": "RAG", + "query_processed.processing_time": 5.253868818283081, + "query_processed.response_length": 837 + }, + "events": [], + "links": [], + "trace_trace_id": "343764b8-7ac9-4549-93cf-8abc11ceba6f", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:58:26.542000000", + "trace_end_time": "2025-10-22 16:58:31.796000000", + "trace_duration_ms": 5253.973, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:58:26.542000000", + "updated_at": "2025-10-22 16:58:26.542000000" + }, + { + "span_id": "6b838ed6-f5bb-4bcf-a76b-bac283bf7571", + "trace_id": "343764b8-7ac9-4549-93cf-8abc11ceba6f", + "parent_span_id": "3b06d098-6eb7-4bb6-885e-443ddb28cc41", + "name": "agent.llm-rag", + "start_time": "2025-10-22 16:58:26.542000000", + "end_time": "2025-10-22 16:58:31.796000000", + "duration_ms": 5253.519, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "rag_agent", + "agent.operation": "llm-rag", + "agent.capabilities": [ + "document_retrieval", + "context_generation", + "response_generation" + ], + "agent.id": "noveum_rag_agent", + "input_query": "What are the newest features in cloud computing?", + "query_length": 48, + "query_type": "rag_query", + "output_response": "RAG Answer: The provided context does not specifically mention any new features in cloud computing as a general topic. However, it does highlight several recent enhancements related to the Noveum.ai platform, whi...", + "rag.context_length": 2579, + "rag.sources_count": 5, + "rag.answer_length": 837, + "rag.mode": "retrieval_augmented_generation" + }, + "events": [], + "links": [], + "trace_trace_id": "343764b8-7ac9-4549-93cf-8abc11ceba6f", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:58:26.542000000", + "trace_end_time": "2025-10-22 16:58:31.796000000", + "trace_duration_ms": 5253.973, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:58:26.542000000", + "updated_at": "2025-10-22 16:58:26.542000000" + }, + { + "span_id": "f51d8f1e-eae3-488d-9a96-49ab97377851", + "trace_id": "343764b8-7ac9-4549-93cf-8abc11ceba6f", + "parent_span_id": "3b06d098-6eb7-4bb6-885e-443ddb28cc41", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-22 16:58:26.542000000", + "end_time": "2025-10-22 16:58:26.542000000", + "duration_ms": 0.082, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What are the newest features in cloud computing?", + "query_length": 48, + "query_type": "routing_evaluation_query", + "classification.mode": "RAG", + "classification.rag_keyword_score": 1, + "classification.web_keyword_score": 0, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "documentation", + "query.keyword_density": 0.125, + "query.domain_affinity": "noveum", + "routing.decision": "Routed to RAG based on analysis", + "routing.rationale": "RAG score: 1, Web score: 0, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "Web Search", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.019230769230769232, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: RAG (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "343764b8-7ac9-4549-93cf-8abc11ceba6f", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:58:26.542000000", + "trace_end_time": "2025-10-22 16:58:31.796000000", + "trace_duration_ms": 5253.973, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:58:26.542000000", + "updated_at": "2025-10-22 16:58:26.542000000" + }, + { + "span_id": "93a0fd69-47b6-4e7c-b8ab-8eaa5aeac240", + "trace_id": "343764b8-7ac9-4549-93cf-8abc11ceba6f", + "parent_span_id": "6b838ed6-f5bb-4bcf-a76b-bac283bf7571", + "name": "agent.llm_model_execution", + "start_time": "2025-10-22 16:58:27.411000000", + "end_time": "2025-10-22 16:58:31.373000000", + "duration_ms": 3961.481, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What are the newest features in cloud computing?", + "query_length": 48, + "query_type": "rag_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 3.9613850116729736, + "model.latency_ms": 3961.3850116729736, + "model.start_time": 1761152307.411936, + "model.end_time": 1761152311.373321, + "model.performance_tier": "medium", + "model.prompt_tokens": 780, + "model.completion_tokens": 209, + "model.total_tokens": 989, + "model.tokens_per_second": 249.66015600243944, + "model.estimated_cost": 0.029670000000000002, + "model.efficiency_score": 0.846309403437816, + "model.response_length": 837, + "model.response_quality": "high", + "model.output_response": "Model Response: The provided context does not specifically mention any new features in cloud computing as a general topic. However, it does highlight several recent enhancements related to the Noveum.ai platform, whi...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "343764b8-7ac9-4549-93cf-8abc11ceba6f", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:58:26.542000000", + "trace_end_time": "2025-10-22 16:58:31.796000000", + "trace_duration_ms": 5253.973, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:58:26.542000000", + "updated_at": "2025-10-22 16:58:26.542000000" + }, + { + "span_id": "48cbc1df-f03a-4c49-90e5-19a2047c1c7b", + "trace_id": "343764b8-7ac9-4549-93cf-8abc11ceba6f", + "parent_span_id": "6b838ed6-f5bb-4bcf-a76b-bac283bf7571", + "name": "agent.rag_evaluation_metrics", + "start_time": "2025-10-22 16:58:31.373000000", + "end_time": "2025-10-22 16:58:31.373000000", + "duration_ms": 0.061, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "rag_evaluation_metrics", + "agent.capabilities": [ + "retrieval_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What are the newest features in cloud computing?", + "query_length": 48, + "query_type": "rag_evaluation_query", + "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/changelog):\n- \ud83d\udcc8 Enhanced Dashboard Analytics: Real-time request tracking, latency monitoring, and cost analysis\n- \ud83d\udd0d Improved Logs Interface: Better search functionality, detailed trace views, and enhanced debugging capabilities\n- \ud83d\udc65 Team Management: Advanced member mana...", + "retrieval.context_length": 2579, + "retrieval.sources_count": 5, + "retrieval.context_quality": "high", + "retrieval.effectiveness": 1, + "retrieval.context_utilization": 2.579, + "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/changelog):\n- \ud83d\udcc8 Enhanced Dashboard Analytics: Real-time request tracking, latency monitoring, and cost analysis\n- \ud83d\udd0d Improved Logs Interface: Better search functionality, detailed trace views, and enhanced debugging capabilities\n- \ud83d\udc65 Team Management: Advanced member management with role-based access control and invitation system\n- \ud83d\udee1\ufe0f Security Improvements: Enhanced API key management and secure credential storage\n4 months ago\n- \ud83d\udcca Advanced Metrics Collection: Comprehensive telemetry system with custom metrics ...\n\nSource 2 (https://noveum.ai/en/changelog):\n- \ud83d\udcdd Documentation Updates: Comprehensive guides for integration and best practices\n5 months ago\n- \ud83c\udf89 Platform Foundation: Initial release of Noveum.ai AI observability platform\n- \ud83d\ude80 Core Features: AI Gateway integration, basic evaluation metrics, and monitoring capabilities\n- \ud83d\udcc8 Evaluation Jobs: Automated model evaluation with configurable metrics and reporting\n- \ud83d\udd0c Provider Integrations: Support for major AI providers and custom model deployments...\n\nSource 3 (https://noveum.ai/docs/platform/dashboard):\n- Bulk Export: Download multiple traces simultaneously\n- Comparative Analysis: Compare performance across multiple traces\nReal-Time Updates\n- Live Refresh: Automatic updates as new traces arrive\n- Connection Monitoring: Real-time status of your trace ingestion pipeline\n- Performance Indicators: Live metrics for system health monitoring\nReady to dive deeper? Explore Projects & Environments to organize your AI applications, or check out Team Collaboration to share insights with your team.\nGet Earl...\n\nSource 4 (https://noveum.ai/en/changelog):\nChangelog\nStay up to date with the latest changes in our product.\n2 months ago\n- \ud83c\udfaf Major Platform Evolution: NovaEval Framework - Launched comprehensive AI model evaluation framework with 20+ built-in scorers including accuracy, RAG metrics, conversational scoring, and specialized agent evaluation tools. Features production-grade deployment on Docker and Kubernetes. (GitHub | PyPI Package)\n- \ud83d\ude80 Next-Generation Traces UI - Complete redesign featuring three-pane layout with directory tree navigatio...\n\nSource 5 (https://noveum.ai/en/blog):\n8/12/2025\nDiscover how Noveum.ai provides comprehensive tracing and observability for AI applications, from development debugging to production optimization.\nShashank Agarwal\n3/3/2025\nDiscover how Noveum.ai provides comprehensive tracing and observability for LLM applications, RAG systems, and multi-agent workflows with our powerful Python and TypeScript SDKs.\nShashank Agarwal\n3/2/2025...\n\nUser Question: What are the newest features in cloud computing?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", + "prompt.prompt_length": 3123, + "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/changelog):\n- \ud83d\udcc8 Enhanced Dashboard Analytics: Real-time request tracking, latency monitoring, and cost analysis\n- \ud83d\udd0d Improved Logs Interface: Better search functionality,...", + "prompt.instruction_following": "rag_optimized", + "response.answer_length": 837, + "response.answer_completeness": "complete", + "response.response_quality": "high", + "response.source_citation": 5, + "response.context_utilization": 0.3245443970531214, + "output_response": "RAG Answer: The provided context does not specifically mention any new features in cloud computing as a general topic. However, it does highlight several recent enhancements related to the Noveum.ai platform, whi...", + "evaluation.retrieval_effectiveness": 1, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 5, + "evaluation.context_relevance": "high", + "evaluation.overall_quality": "high", + "evaluation.ready_for_production": true, + "rag.retrieval_strategy": "semantic_similarity", + "rag.vector_search_results": 5, + "rag.context_synthesis": "multi_source", + "rag.document_coverage": 1, + "rag.information_density": 0.3245443970531214 + }, + "events": [], + "links": [], + "trace_trace_id": "343764b8-7ac9-4549-93cf-8abc11ceba6f", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:58:26.542000000", + "trace_end_time": "2025-10-22 16:58:31.796000000", + "trace_duration_ms": 5253.973, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:58:26.542000000", + "updated_at": "2025-10-22 16:58:26.542000000" + }, + { + "span_id": "7384d697-d7bf-4709-8377-ec26b094b550", + "trace_id": "3f7b6a62-8888-4a5d-b12c-37a3829aea49", + "parent_span_id": "e631b22a-a621-4ecc-bb01-f731ca66f3d0", + "name": "agent.query_routing", + "start_time": "2025-10-22 16:58:19.885000000", + "end_time": "2025-10-22 16:58:25.536000000", + "duration_ms": 5651.335999999999, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What's the current status of cryptocurrency markets?", + "query_length": 52, + "query_type": "routing_query", + "output_response": "Routed to Web Search for query processing", + "router.classification": "Web Search", + "router.confidence_score": 1, + "router.rag_keyword_score": 0, + "router.web_keyword_score": 3, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "3f7b6a62-8888-4a5d-b12c-37a3829aea49", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:58:19.885000000", + "trace_end_time": "2025-10-22 16:58:25.536000000", + "trace_duration_ms": 5651.67, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:58:19.885000000", + "updated_at": "2025-10-22 16:58:19.885000000" + }, + { + "span_id": "7abf372d-d9f4-41d1-9e3a-d4f01173fa5e", + "trace_id": "3f7b6a62-8888-4a5d-b12c-37a3829aea49", + "parent_span_id": "7384d697-d7bf-4709-8377-ec26b094b550", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-22 16:58:19.885000000", + "end_time": "2025-10-22 16:58:19.885000000", + "duration_ms": 0.037, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What's the current status of cryptocurrency markets?", + "query_length": 52, + "query_type": "routing_evaluation_query", + "classification.mode": "Web Search", + "classification.rag_keyword_score": 0, + "classification.web_keyword_score": 3, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "complex", + "query.intent": "general_knowledge", + "query.keyword_density": 0.42857142857142855, + "query.domain_affinity": "general", + "routing.decision": "Routed to Web Search based on analysis", + "routing.rationale": "RAG score: 0, Web score: 3, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "RAG", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.057692307692307696, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: Web Search (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "3f7b6a62-8888-4a5d-b12c-37a3829aea49", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:58:19.885000000", + "trace_end_time": "2025-10-22 16:58:25.536000000", + "trace_duration_ms": 5651.67, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:58:19.885000000", + "updated_at": "2025-10-22 16:58:19.885000000" + }, + { + "span_id": "e5d9c3c3-6c18-45d2-a6b6-58bde4d4e461", + "trace_id": "3f7b6a62-8888-4a5d-b12c-37a3829aea49", + "parent_span_id": "7384d697-d7bf-4709-8377-ec26b094b550", + "name": "agent.web_search_generation", + "start_time": "2025-10-22 16:58:19.885000000", + "end_time": "2025-10-22 16:58:25.536000000", + "duration_ms": 5651.161, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "web_search_agent", + "agent.operation": "web_search_generation", + "agent.capabilities": [ + "web_search", + "content_synthesis", + "response_generation" + ], + "agent.id": "noveum_web_search_agent", + "input_query": "What's the current status of cryptocurrency markets?", + "query_length": 52, + "query_type": "web_search_query", + "output_response": "Web Search Answer: The provided web search results do not contain specific information regarding the current status of cryptocurrency markets. To get the latest updates on cryptocurrency prices, trends, and market analy...", + "web_search.results_count": 1, + "web_search.context_length": 251, + "web_search.response_length": 416, + "web_search.mode": "external_web_search" + }, + "events": [], + "links": [], + "trace_trace_id": "3f7b6a62-8888-4a5d-b12c-37a3829aea49", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:58:19.885000000", + "trace_end_time": "2025-10-22 16:58:25.536000000", + "trace_duration_ms": 5651.67, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:58:19.885000000", + "updated_at": "2025-10-22 16:58:19.885000000" + }, + { + "span_id": "e631b22a-a621-4ecc-bb01-f731ca66f3d0", + "trace_id": "3f7b6a62-8888-4a5d-b12c-37a3829aea49", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-22 16:58:19.885000000", + "end_time": "2025-10-22 16:58:25.536000000", + "duration_ms": 5651.588000000001, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What's the current status of cryptocurrency markets?", + "query.length": 52, + "processing.mode": "Web Search", + "processing.time_seconds": 5.651373863220215, + "processing.response_length": 416, + "processing.sources_count": 1, + "output_response": "Final Answer: The provided web search results do not contain specific information regarding the current status of cryptocurrency markets. To get the latest updates on cryptocurrency prices, trends, and market analy...", + "final_answer_mode": "Web Search", + "query_processed.input_query": "What's the current status of cryptocurrency markets?", + "query_processed.output_response": "Successfully processed query using Web Search, generated 416 character response", + "query_processed.mode": "Web Search", + "query_processed.processing_time": 5.651373863220215, + "query_processed.response_length": 416 + }, + "events": [], + "links": [], + "trace_trace_id": "3f7b6a62-8888-4a5d-b12c-37a3829aea49", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:58:19.885000000", + "trace_end_time": "2025-10-22 16:58:25.536000000", + "trace_duration_ms": 5651.67, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:58:19.885000000", + "updated_at": "2025-10-22 16:58:19.885000000" + }, + { + "span_id": "34e2f906-724e-4116-b61f-82eaa6b75959", + "trace_id": "3f7b6a62-8888-4a5d-b12c-37a3829aea49", + "parent_span_id": "e5d9c3c3-6c18-45d2-a6b6-58bde4d4e461", + "name": "agent.llm_model_execution", + "start_time": "2025-10-22 16:58:24.327000000", + "end_time": "2025-10-22 16:58:25.536000000", + "duration_ms": 1208.66, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What's the current status of cryptocurrency markets?", + "query_length": 52, + "query_type": "web_search_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 1.208608865737915, + "model.latency_ms": 1208.608865737915, + "model.start_time": 1761152304.327749, + "model.end_time": 1761152305.5363579, + "model.performance_tier": "fast", + "model.prompt_tokens": 187, + "model.completion_tokens": 104, + "model.total_tokens": 291, + "model.tokens_per_second": 240.77268357809888, + "model.estimated_cost": 0.00873, + "model.efficiency_score": 1.429553264604811, + "model.response_length": 416, + "model.response_quality": "high", + "model.output_response": "Model Response: The provided web search results do not contain specific information regarding the current status of cryptocurrency markets. To get the latest updates on cryptocurrency prices, trends, and market analy...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "3f7b6a62-8888-4a5d-b12c-37a3829aea49", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:58:19.885000000", + "trace_end_time": "2025-10-22 16:58:25.536000000", + "trace_duration_ms": 5651.67, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:58:19.885000000", + "updated_at": "2025-10-22 16:58:19.885000000" + }, + { + "span_id": "5548e02e-2ea8-472c-9c42-f811d6e4d30b", + "trace_id": "3f7b6a62-8888-4a5d-b12c-37a3829aea49", + "parent_span_id": "e5d9c3c3-6c18-45d2-a6b6-58bde4d4e461", + "name": "agent.web_search_evaluation_metrics", + "start_time": "2025-10-22 16:58:25.536000000", + "end_time": "2025-10-22 16:58:25.536000000", + "duration_ms": 0.044, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "web_search_evaluation_metrics", + "agent.capabilities": [ + "web_search_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What's the current status of cryptocurrency markets?", + "query_length": 52, + "query_type": "web_search_evaluation_query", + "web_search.results_count": 1, + "web_search.context_length": 251, + "web_search.context_synthesized": "Context from 1 web sources", + "web_search.search_effectiveness": 0.2, + "web_search.context_quality": "low", + "web_search.source_diversity": 1, + "web_search.information_synthesis": "low", + "web_search.external_knowledge_utilization": 0.251, + "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nLog in to WhatsApp Web for simple, reliable and private messaging on your desktop. Send and receive messages and files with ease, all for \u2026\nURL: https://duckduckgo.com/?q=What's+the+current+status+of+cryptocurrency+markets?\n\nUser Question: What's the current status of cryptocurrency markets?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", + "prompt.prompt_length": 748, + "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nLog in to WhatsApp Web for simple, reliable and private messaging on your desktop. Send and receive messages and files with ease, all for \u2026\nURL: https://duckduckgo.com/?q=W...", + "prompt.instruction_following": "web_search_optimized", + "response.answer_length": 416, + "response.answer_completeness": "complete", + "response.response_quality": "medium", + "response.source_citation": 1, + "response.context_utilization": 1.657370517928287, + "output_response": "Web Search Answer: The provided web search results do not contain specific information regarding the current status of cryptocurrency markets. To get the latest updates on cryptocurrency prices, trends, and market analy...", + "evaluation.search_effectiveness": 0.2, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 1, + "evaluation.context_relevance": "low", + "evaluation.overall_quality": "low", + "evaluation.ready_for_production": true, + "web_search.search_strategy": "duckduckgo_api", + "web_search.real_time_data": true, + "web_search.external_sources": 1, + "web_search.information_freshness": "current", + "web_search.knowledge_synthesis": "single_source" + }, + "events": [], + "links": [], + "trace_trace_id": "3f7b6a62-8888-4a5d-b12c-37a3829aea49", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:58:19.885000000", + "trace_end_time": "2025-10-22 16:58:25.536000000", + "trace_duration_ms": 5651.67, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:58:19.885000000", + "updated_at": "2025-10-22 16:58:19.885000000" + }, + { + "span_id": "a9e34cea-d4bd-478b-abf2-7a3a38dcd40e", + "trace_id": "51f4534f-23e4-4be7-8d6e-e93391d6e0e2", + "parent_span_id": "8e9b2d06-3e11-4d17-9a32-b1529a2c10a4", + "name": "agent.web_search_generation", + "start_time": "2025-10-22 16:58:15.254000000", + "end_time": "2025-10-22 16:58:18.876000000", + "duration_ms": 3621.674, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "web_search_agent", + "agent.operation": "web_search_generation", + "agent.capabilities": [ + "web_search", + "content_synthesis", + "response_generation" + ], + "agent.id": "noveum_web_search_agent", + "input_query": "What are the latest updates in Python programming?", + "query_length": 50, + "query_type": "web_search_query", + "output_response": "Web Search Answer: The provided web search results do not contain specific information about the latest updates in Python programming. They primarily focus on general news coverage and updates from India and around the ...", + "web_search.results_count": 1, + "web_search.context_length": 753, + "web_search.response_length": 580, + "web_search.mode": "external_web_search" + }, + "events": [], + "links": [], + "trace_trace_id": "51f4534f-23e4-4be7-8d6e-e93391d6e0e2", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:58:15.254000000", + "trace_end_time": "2025-10-22 16:58:18.877000000", + "trace_duration_ms": 3622.355, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:58:15.254000000", + "updated_at": "2025-10-22 16:58:15.254000000" + }, + { + "span_id": "8e9b2d06-3e11-4d17-9a32-b1529a2c10a4", + "trace_id": "51f4534f-23e4-4be7-8d6e-e93391d6e0e2", + "parent_span_id": "bda936a0-24f3-4841-af4f-227197e10b9d", + "name": "agent.query_routing", + "start_time": "2025-10-22 16:58:15.254000000", + "end_time": "2025-10-22 16:58:18.876000000", + "duration_ms": 3621.863, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What are the latest updates in Python programming?", + "query_length": 50, + "query_type": "routing_query", + "output_response": "Routed to Web Search for query processing", + "router.classification": "Web Search", + "router.confidence_score": 1, + "router.rag_keyword_score": 0, + "router.web_keyword_score": 2, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "51f4534f-23e4-4be7-8d6e-e93391d6e0e2", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:58:15.254000000", + "trace_end_time": "2025-10-22 16:58:18.877000000", + "trace_duration_ms": 3622.355, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:58:15.254000000", + "updated_at": "2025-10-22 16:58:15.254000000" + }, + { + "span_id": "bda936a0-24f3-4841-af4f-227197e10b9d", + "trace_id": "51f4534f-23e4-4be7-8d6e-e93391d6e0e2", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-22 16:58:15.254000000", + "end_time": "2025-10-22 16:58:18.877000000", + "duration_ms": 3622.271, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What are the latest updates in Python programming?", + "query.length": 50, + "processing.mode": "Web Search", + "processing.time_seconds": 3.6219100952148438, + "processing.response_length": 580, + "processing.sources_count": 1, + "output_response": "Final Answer: The provided web search results do not contain specific information about the latest updates in Python programming. They primarily focus on general news coverage and updates from India and around the ...", + "final_answer_mode": "Web Search", + "query_processed.input_query": "What are the latest updates in Python programming?", + "query_processed.output_response": "Successfully processed query using Web Search, generated 580 character response", + "query_processed.mode": "Web Search", + "query_processed.processing_time": 3.6219100952148438, + "query_processed.response_length": 580 + }, + "events": [], + "links": [], + "trace_trace_id": "51f4534f-23e4-4be7-8d6e-e93391d6e0e2", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:58:15.254000000", + "trace_end_time": "2025-10-22 16:58:18.877000000", + "trace_duration_ms": 3622.355, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:58:15.254000000", + "updated_at": "2025-10-22 16:58:15.254000000" + }, + { + "span_id": "e27e3812-d49d-4f4f-98b0-144cf0ceffc3", + "trace_id": "51f4534f-23e4-4be7-8d6e-e93391d6e0e2", + "parent_span_id": "8e9b2d06-3e11-4d17-9a32-b1529a2c10a4", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-22 16:58:15.254000000", + "end_time": "2025-10-22 16:58:15.254000000", + "duration_ms": 0.047, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What are the latest updates in Python programming?", + "query_length": 50, + "query_type": "routing_evaluation_query", + "classification.mode": "Web Search", + "classification.rag_keyword_score": 0, + "classification.web_keyword_score": 2, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "general_knowledge", + "query.keyword_density": 0.25, + "query.domain_affinity": "general", + "routing.decision": "Routed to Web Search based on analysis", + "routing.rationale": "RAG score: 0, Web score: 2, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "RAG", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.038461538461538464, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: Web Search (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "51f4534f-23e4-4be7-8d6e-e93391d6e0e2", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:58:15.254000000", + "trace_end_time": "2025-10-22 16:58:18.877000000", + "trace_duration_ms": 3622.355, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:58:15.254000000", + "updated_at": "2025-10-22 16:58:15.254000000" + }, + { + "span_id": "b0eea8ec-5cff-4667-bea7-aeb6d0f0daf3", + "trace_id": "51f4534f-23e4-4be7-8d6e-e93391d6e0e2", + "parent_span_id": "a9e34cea-d4bd-478b-abf2-7a3a38dcd40e", + "name": "agent.llm_model_execution", + "start_time": "2025-10-22 16:58:15.568000000", + "end_time": "2025-10-22 16:58:18.876000000", + "duration_ms": 3307.5739999999996, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What are the latest updates in Python programming?", + "query_length": 50, + "query_type": "web_search_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 3.307521104812622, + "model.latency_ms": 3307.521104812622, + "model.start_time": 1761152295.568949, + "model.end_time": 1761152298.87647, + "model.performance_tier": "medium", + "model.prompt_tokens": 312, + "model.completion_tokens": 145, + "model.total_tokens": 457, + "model.tokens_per_second": 138.16994223711538, + "model.estimated_cost": 0.01371, + "model.efficiency_score": 1.2691466083150984, + "model.response_length": 580, + "model.response_quality": "high", + "model.output_response": "Model Response: The provided web search results do not contain specific information about the latest updates in Python programming. They primarily focus on general news coverage and updates from India and around the ...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "51f4534f-23e4-4be7-8d6e-e93391d6e0e2", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:58:15.254000000", + "trace_end_time": "2025-10-22 16:58:18.877000000", + "trace_duration_ms": 3622.355, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:58:15.254000000", + "updated_at": "2025-10-22 16:58:15.254000000" + }, + { + "span_id": "d0bfcbe5-cb6c-4837-bdf3-f2c65be145cd", + "trace_id": "51f4534f-23e4-4be7-8d6e-e93391d6e0e2", + "parent_span_id": "a9e34cea-d4bd-478b-abf2-7a3a38dcd40e", + "name": "agent.web_search_evaluation_metrics", + "start_time": "2025-10-22 16:58:18.876000000", + "end_time": "2025-10-22 16:58:18.876000000", + "duration_ms": 0.04, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "web_search_evaluation_metrics", + "agent.capabilities": [ + "web_search_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What are the latest updates in Python programming?", + "query_length": 50, + "query_type": "web_search_evaluation_query", + "web_search.results_count": 1, + "web_search.context_length": 753, + "web_search.context_synthesized": "Context from 1 web sources", + "web_search.search_effectiveness": 0.2, + "web_search.context_quality": "medium", + "web_search.source_diversity": 1, + "web_search.information_synthesis": "low", + "web_search.external_knowledge_utilization": 0.753, + "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\n3 days ago \u00b7 Latest news headlines from India & around the world. Check out today\u2019s news coverage live with videos & photos on \u2026 6 days ago \u00b7 India News | Latest India News | Read latest and breaking news from India. Today's top India news headlines, news on \u2026 2 days ago \u00b7 Top News Stories of the day, Latest News Headlines, News Specials, Breaking News and Latest India News, World current affairs & \u2026 NDTV.com provides latest news from India and around the world. Get breaking news alerts from India and follow today\u2019s live news updates in \u2026 6 days ago \u00b7 Get Latest entertainment news from Bollywood, Hollywood and regional film industries.\nURL: https://duckduckgo.com/?q=What+are+the+latest+updates+in+Python+programming?\n\nUser Question: What are the latest updates in Python programming?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", + "prompt.prompt_length": 1248, + "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\n3 days ago \u00b7 Latest news headlines from India & around the world. Check out today\u2019s news coverage live with videos & photos on \u2026 6 days ago \u00b7 India News | Latest India News...", + "prompt.instruction_following": "web_search_optimized", + "response.answer_length": 580, + "response.answer_completeness": "complete", + "response.response_quality": "medium", + "response.source_citation": 1, + "response.context_utilization": 0.7702523240371846, + "output_response": "Web Search Answer: The provided web search results do not contain specific information about the latest updates in Python programming. They primarily focus on general news coverage and updates from India and around the ...", + "evaluation.search_effectiveness": 0.2, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 1, + "evaluation.context_relevance": "medium", + "evaluation.overall_quality": "low", + "evaluation.ready_for_production": true, + "web_search.search_strategy": "duckduckgo_api", + "web_search.real_time_data": true, + "web_search.external_sources": 1, + "web_search.information_freshness": "current", + "web_search.knowledge_synthesis": "single_source" + }, + "events": [], + "links": [], + "trace_trace_id": "51f4534f-23e4-4be7-8d6e-e93391d6e0e2", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:58:15.254000000", + "trace_end_time": "2025-10-22 16:58:18.877000000", + "trace_duration_ms": 3622.355, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:58:15.254000000", + "updated_at": "2025-10-22 16:58:15.254000000" + }, + { + "span_id": "5f5916a7-b9ea-4348-861d-db69b25d6b62", + "trace_id": "e30e9ba6-ffd1-4eb9-b7ae-6d19e04343d6", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-22 16:58:11.033000000", + "end_time": "2025-10-22 16:58:14.250000000", + "duration_ms": 3216.77, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What happened in tech news this week?", + "query.length": 37, + "processing.mode": "Web Search", + "processing.time_seconds": 3.2166178226470947, + "processing.response_length": 510, + "processing.sources_count": 1, + "output_response": "Final Answer: The web search results do not provide specific information about recent events in tech news for this week. They primarily discuss the grammatical usage of the word \"happened\" and its variations, such ...", + "final_answer_mode": "Web Search", + "query_processed.input_query": "What happened in tech news this week?", + "query_processed.output_response": "Successfully processed query using Web Search, generated 510 character response", + "query_processed.mode": "Web Search", + "query_processed.processing_time": 3.2166178226470947, + "query_processed.response_length": 510 + }, + "events": [], + "links": [], + "trace_trace_id": "e30e9ba6-ffd1-4eb9-b7ae-6d19e04343d6", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:58:11.033000000", + "trace_end_time": "2025-10-22 16:58:14.250000000", + "trace_duration_ms": 3216.801, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:58:11.033000000", + "updated_at": "2025-10-22 16:58:11.033000000" + }, + { + "span_id": "804613d0-a701-4c22-add1-d315d7a29813", + "trace_id": "e30e9ba6-ffd1-4eb9-b7ae-6d19e04343d6", + "parent_span_id": "5f5916a7-b9ea-4348-861d-db69b25d6b62", + "name": "agent.query_routing", + "start_time": "2025-10-22 16:58:11.033000000", + "end_time": "2025-10-22 16:58:14.250000000", + "duration_ms": 3216.594, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What happened in tech news this week?", + "query_length": 37, + "query_type": "routing_query", + "output_response": "Routed to Web Search for query processing", + "router.classification": "Web Search", + "router.confidence_score": 1, + "router.rag_keyword_score": 0, + "router.web_keyword_score": 2, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "e30e9ba6-ffd1-4eb9-b7ae-6d19e04343d6", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:58:11.033000000", + "trace_end_time": "2025-10-22 16:58:14.250000000", + "trace_duration_ms": 3216.801, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:58:11.033000000", + "updated_at": "2025-10-22 16:58:11.033000000" + }, + { + "span_id": "f6c8d34f-185f-49ba-a5a4-a9d25ccb4fc6", + "trace_id": "e30e9ba6-ffd1-4eb9-b7ae-6d19e04343d6", + "parent_span_id": "804613d0-a701-4c22-add1-d315d7a29813", + "name": "agent.web_search_generation", + "start_time": "2025-10-22 16:58:11.034000000", + "end_time": "2025-10-22 16:58:14.250000000", + "duration_ms": 3216.4959999999996, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "web_search_agent", + "agent.operation": "web_search_generation", + "agent.capabilities": [ + "web_search", + "content_synthesis", + "response_generation" + ], + "agent.id": "noveum_web_search_agent", + "input_query": "What happened in tech news this week?", + "query_length": 37, + "query_type": "web_search_query", + "output_response": "Web Search Answer: The web search results do not provide specific information about recent events in tech news for this week. They primarily discuss the grammatical usage of the word \"happened\" and its variations, such ...", + "web_search.results_count": 1, + "web_search.context_length": 724, + "web_search.response_length": 510, + "web_search.mode": "external_web_search" + }, + "events": [], + "links": [], + "trace_trace_id": "e30e9ba6-ffd1-4eb9-b7ae-6d19e04343d6", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:58:11.033000000", + "trace_end_time": "2025-10-22 16:58:14.250000000", + "trace_duration_ms": 3216.801, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:58:11.033000000", + "updated_at": "2025-10-22 16:58:11.033000000" + }, + { + "span_id": "11d68082-e7b8-4bbf-833a-79e296c31eda", + "trace_id": "e30e9ba6-ffd1-4eb9-b7ae-6d19e04343d6", + "parent_span_id": "804613d0-a701-4c22-add1-d315d7a29813", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-22 16:58:11.034000000", + "end_time": "2025-10-22 16:58:11.034000000", + "duration_ms": 0.017, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What happened in tech news this week?", + "query_length": 37, + "query_type": "routing_evaluation_query", + "classification.mode": "Web Search", + "classification.rag_keyword_score": 0, + "classification.web_keyword_score": 2, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "general_knowledge", + "query.keyword_density": 0.2857142857142857, + "query.domain_affinity": "general", + "routing.decision": "Routed to Web Search based on analysis", + "routing.rationale": "RAG score: 0, Web score: 2, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "RAG", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.038461538461538464, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: Web Search (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "e30e9ba6-ffd1-4eb9-b7ae-6d19e04343d6", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:58:11.033000000", + "trace_end_time": "2025-10-22 16:58:14.250000000", + "trace_duration_ms": 3216.801, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:58:11.033000000", + "updated_at": "2025-10-22 16:58:11.033000000" + }, + { + "span_id": "420b4ca1-c9f1-488f-ac2c-c53766717634", + "trace_id": "e30e9ba6-ffd1-4eb9-b7ae-6d19e04343d6", + "parent_span_id": "f6c8d34f-185f-49ba-a5a4-a9d25ccb4fc6", + "name": "agent.llm_model_execution", + "start_time": "2025-10-22 16:58:11.468000000", + "end_time": "2025-10-22 16:58:14.250000000", + "duration_ms": 2781.777, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What happened in tech news this week?", + "query_length": 37, + "query_type": "web_search_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 2.781723976135254, + "model.latency_ms": 2781.723976135254, + "model.start_time": 1761152291.468626, + "model.end_time": 1761152294.25035, + "model.performance_tier": "medium", + "model.prompt_tokens": 301, + "model.completion_tokens": 127, + "model.total_tokens": 428, + "model.tokens_per_second": 153.8614196346811, + "model.estimated_cost": 0.01284, + "model.efficiency_score": 1.191588785046729, + "model.response_length": 510, + "model.response_quality": "high", + "model.output_response": "Model Response: The web search results do not provide specific information about recent events in tech news for this week. They primarily discuss the grammatical usage of the word \"happened\" and its variations, such ...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "e30e9ba6-ffd1-4eb9-b7ae-6d19e04343d6", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:58:11.033000000", + "trace_end_time": "2025-10-22 16:58:14.250000000", + "trace_duration_ms": 3216.801, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:58:11.033000000", + "updated_at": "2025-10-22 16:58:11.033000000" + }, + { + "span_id": "56b8bf09-8ebf-44e1-b01f-daff23d3c42c", + "trace_id": "e30e9ba6-ffd1-4eb9-b7ae-6d19e04343d6", + "parent_span_id": "f6c8d34f-185f-49ba-a5a4-a9d25ccb4fc6", + "name": "agent.web_search_evaluation_metrics", + "start_time": "2025-10-22 16:58:14.250000000", + "end_time": "2025-10-22 16:58:14.250000000", + "duration_ms": 0.060000000000000005, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "web_search_evaluation_metrics", + "agent.capabilities": [ + "web_search_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What happened in tech news this week?", + "query_length": 37, + "query_type": "web_search_evaluation_query", + "web_search.results_count": 1, + "web_search.context_length": 724, + "web_search.context_synthesized": "Context from 1 web sources", + "web_search.search_effectiveness": 0.2, + "web_search.context_quality": "medium", + "web_search.source_diversity": 1, + "web_search.information_synthesis": "low", + "web_search.external_knowledge_utilization": 0.724, + "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nhappened\u7684\u7528\u6cd5\u5982\u4e0b\uff1a 1.happened\u662f \u4e0d\u53ca\u7269\u52a8\u8bcd,\u4f5c\u201c\u53d1\u751f\u201d\u89e3\u65f6,\u53ef\u6307\u5076\u7136\u7684\u4e8b\u6545,\u4e5f\u53ef\u6307\u6709\u8ba1\u5212\u6216\u65e0\u8ba1\u5212\u53d1\u751f\u7684\u4e8b\u3002\u4e00\u822c\u4ee5\u4e8b\u4ef6\u3001\u73af\u5883\u3001\u5f62\u52bf\u7b49\u4f5c\u4e3b\u8bed,\u8868\u793a\u201c\u67d0\u4eba\u53d1\u751f\u67d0\u4e8b\u201d\u65f6,\u987b\u4ee5\u4ecb\u8bcdto\u5f15\u51fa\u201c\u67d0\u4eba\u201d\u3002 2. \u2026 what happened\u8fd8\u662fwhat's happened\"What happened\"\u548c\"What's happened\"\u90fd\u7528\u4e8e\u8be2\u95ee\u8fc7\u53bb\u53d1\u751f\u4e86\u4ec0\u4e48\u4e8b\u60c5\uff0c\u4e3b\u8981\u7684\u533a\u522b\u5728\u4e8e\u8bed\u6cd5\u548c\u4f7f\u7528\u65b9\u5f0f\u3002\"What happened\"\u662f\u4e00\u4e2a\u8fc7\u53bb\u65f6\u6001\u7684\u7b80\u5355 \u63a5\u4e0b\u6765\u8ba9\u6211\u4eec\u770b\u4e0bwhat happen\u548cwhat happened\u7684\u7528\u6cd5\u533a\u522b\uff1a 1.\u8bcd\u6027\uff1awhat happen\u662f\u7528\u539f\u5f62\u52a8\u8bcdhappen\u6784\u6210\u7684\u73b0\u5728\u65f6\u77ed\u8bed\uff0c\u800cwhat happened\u662f\u7528\u52a8\u8bcdhappen\u7684\u8fc7\u53bb\u5f0fhappened\u6784\u6210\u7684\u8fc7\u53bb\u65f6\u77ed \u2026 What happened \u548c What's happened\u4e24\u8005\u7684\u533a\u522b\uff1a \u662f\u95ee What happened\uff1f\u53ea\u662f\u60f3\u77e5\u9053\u8fc7\u53bb\u6240\u53d1\u751f\u7684\u4e8b\u60c5\uff0c \u800c\u5982\u679c\u7528What's happened\uff1f\u5219\u5f3a\u8c03\u5bf9\u73b0\u5728\u7684\u5f71\u54cd\u3002\uff0c\u5373\u60f3\u77e5\u9053\u8fc7\u53bb\u6240\u53d1\u751f\u7684\u4ec0\u4e48\u4e8b\u5bfc\u81f4\u4e86\u8bf4 \u2026 it so happened that\u7684\u53e5\u5f0fhappen\u8868\u793a\u201c\u78b0\u5de7\u6216\u6070\u5de7\u53d1\u751f\u67d0\u4e8b\u201d\u65f6,\u53ef\u7528\u201cIt happens / happened that...\u201d\u8fd9\u4e00\u7ed3\u6784\u6765\u8868\u8fbe.\u4f8b\u5982\uff1a It happened that Brian and Peter were at home that day.\u78b0\u5de7\u90a3\u5929\u5e03\u83b1\u6069\u548c \u2026\nURL: https://duckduckgo.com/?q=What+happened+in+tech+news+this+week?\n\nUser Question: What happened in tech news this week?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", + "prompt.prompt_length": 1206, + "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nhappened\u7684\u7528\u6cd5\u5982\u4e0b\uff1a 1.happened\u662f \u4e0d\u53ca\u7269\u52a8\u8bcd,\u4f5c\u201c\u53d1\u751f\u201d\u89e3\u65f6,\u53ef\u6307\u5076\u7136\u7684\u4e8b\u6545,\u4e5f\u53ef\u6307\u6709\u8ba1\u5212\u6216\u65e0\u8ba1\u5212\u53d1\u751f\u7684\u4e8b\u3002\u4e00\u822c\u4ee5\u4e8b\u4ef6\u3001\u73af\u5883\u3001\u5f62\u52bf\u7b49\u4f5c\u4e3b\u8bed,\u8868\u793a\u201c\u67d0\u4eba\u53d1\u751f\u67d0\u4e8b\u201d\u65f6,\u987b\u4ee5\u4ecb\u8bcdto\u5f15\u51fa\u201c\u67d0\u4eba\u201d\u3002 2. \u2026 what happened\u8fd8\u662fwhat's happened\"What happened\"\u548c\"What's happene...", + "prompt.instruction_following": "web_search_optimized", + "response.answer_length": 510, + "response.answer_completeness": "complete", + "response.response_quality": "medium", + "response.source_citation": 1, + "response.context_utilization": 0.7044198895027625, + "output_response": "Web Search Answer: The web search results do not provide specific information about recent events in tech news for this week. They primarily discuss the grammatical usage of the word \"happened\" and its variations, such ...", + "evaluation.search_effectiveness": 0.2, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 1, + "evaluation.context_relevance": "medium", + "evaluation.overall_quality": "low", + "evaluation.ready_for_production": true, + "web_search.search_strategy": "duckduckgo_api", + "web_search.real_time_data": true, + "web_search.external_sources": 1, + "web_search.information_freshness": "current", + "web_search.knowledge_synthesis": "single_source" + }, + "events": [], + "links": [], + "trace_trace_id": "e30e9ba6-ffd1-4eb9-b7ae-6d19e04343d6", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:58:11.033000000", + "trace_end_time": "2025-10-22 16:58:14.250000000", + "trace_duration_ms": 3216.801, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:58:11.033000000", + "updated_at": "2025-10-22 16:58:11.033000000" + }, + { + "span_id": "82eb19e8-f6cf-43fd-ba56-640a6fd3ed0e", + "trace_id": "45efae74-ee25-4004-a509-6e0ff628a0cd", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-22 16:58:04.883000000", + "end_time": "2025-10-22 16:58:10.029000000", + "duration_ms": 5146.146, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What are the current trends in observability tools?", + "query.length": 51, + "processing.mode": "RAG", + "processing.time_seconds": 5.146108150482178, + "processing.response_length": 815, + "processing.sources_count": 5, + "output_response": "Final Answer: The provided context does not explicitly outline current trends in observability tools. However, it highlights the unique challenges faced by AI applications that traditional monitoring tools cannot a...", + "final_answer_mode": "RAG", + "query_processed.input_query": "What are the current trends in observability tools?", + "query_processed.output_response": "Successfully processed query using RAG, generated 815 character response", + "query_processed.mode": "RAG", + "query_processed.processing_time": 5.146108150482178, + "query_processed.response_length": 815 + }, + "events": [], + "links": [], + "trace_trace_id": "45efae74-ee25-4004-a509-6e0ff628a0cd", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:58:04.883000000", + "trace_end_time": "2025-10-22 16:58:10.029000000", + "trace_duration_ms": 5146.244000000001, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:58:04.883000000", + "updated_at": "2025-10-22 16:58:04.883000000" + }, + { + "span_id": "e81617c6-c2a1-4b08-a5fc-dde3439fe2f4", + "trace_id": "45efae74-ee25-4004-a509-6e0ff628a0cd", + "parent_span_id": "8001ba0e-95c7-42da-9087-603a38e1a500", + "name": "agent.llm-rag", + "start_time": "2025-10-22 16:58:04.883000000", + "end_time": "2025-10-22 16:58:10.029000000", + "duration_ms": 5145.655, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "rag_agent", + "agent.operation": "llm-rag", + "agent.capabilities": [ + "document_retrieval", + "context_generation", + "response_generation" + ], + "agent.id": "noveum_rag_agent", + "input_query": "What are the current trends in observability tools?", + "query_length": 51, + "query_type": "rag_query", + "output_response": "RAG Answer: The provided context does not explicitly outline current trends in observability tools. However, it highlights the unique challenges faced by AI applications that traditional monitoring tools cannot a...", + "rag.context_length": 2943, + "rag.sources_count": 5, + "rag.answer_length": 815, + "rag.mode": "retrieval_augmented_generation" + }, + "events": [], + "links": [], + "trace_trace_id": "45efae74-ee25-4004-a509-6e0ff628a0cd", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:58:04.883000000", + "trace_end_time": "2025-10-22 16:58:10.029000000", + "trace_duration_ms": 5146.244000000001, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:58:04.883000000", + "updated_at": "2025-10-22 16:58:04.883000000" + }, + { + "span_id": "8001ba0e-95c7-42da-9087-603a38e1a500", + "trace_id": "45efae74-ee25-4004-a509-6e0ff628a0cd", + "parent_span_id": "82eb19e8-f6cf-43fd-ba56-640a6fd3ed0e", + "name": "agent.query_routing", + "start_time": "2025-10-22 16:58:04.883000000", + "end_time": "2025-10-22 16:58:10.029000000", + "duration_ms": 5146.036999999999, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What are the current trends in observability tools?", + "query_length": 51, + "query_type": "routing_query", + "output_response": "Routed to RAG for query processing", + "router.classification": "RAG", + "router.confidence_score": 0.3333333333333333, + "router.rag_keyword_score": 2, + "router.web_keyword_score": 1, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "45efae74-ee25-4004-a509-6e0ff628a0cd", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:58:04.883000000", + "trace_end_time": "2025-10-22 16:58:10.029000000", + "trace_duration_ms": 5146.244000000001, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:58:04.883000000", + "updated_at": "2025-10-22 16:58:04.883000000" + }, + { + "span_id": "79b7d138-05e1-4f18-8b63-dcfe9732180b", + "trace_id": "45efae74-ee25-4004-a509-6e0ff628a0cd", + "parent_span_id": "8001ba0e-95c7-42da-9087-603a38e1a500", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-22 16:58:04.883000000", + "end_time": "2025-10-22 16:58:04.883000000", + "duration_ms": 0.051, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What are the current trends in observability tools?", + "query_length": 51, + "query_type": "routing_evaluation_query", + "classification.mode": "RAG", + "classification.rag_keyword_score": 2, + "classification.web_keyword_score": 1, + "classification.confidence_score": 0.3333333333333333, + "classification.confidence_level": "medium", + "classification.method": "keyword_based", + "query.complexity": "complex", + "query.intent": "documentation", + "query.keyword_density": 0.375, + "query.domain_affinity": "noveum", + "routing.decision": "Routed to RAG based on analysis", + "routing.rationale": "RAG score: 2, Web score: 1, Confidence: 0.33", + "routing.expected_performance": "medium", + "routing.alternative_mode": "Web Search", + "routing.decision_confidence": 0.3333333333333333, + "evaluation.routing_accuracy": "medium", + "evaluation.keyword_coverage": 0.057692307692307696, + "evaluation.query_understanding": "ambiguous", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: RAG (Confidence: 0.33)" + }, + "events": [], + "links": [], + "trace_trace_id": "45efae74-ee25-4004-a509-6e0ff628a0cd", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:58:04.883000000", + "trace_end_time": "2025-10-22 16:58:10.029000000", + "trace_duration_ms": 5146.244000000001, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:58:04.883000000", + "updated_at": "2025-10-22 16:58:04.883000000" + }, + { + "span_id": "28c233a0-d42a-4200-86b0-dfbd2d5be06e", + "trace_id": "45efae74-ee25-4004-a509-6e0ff628a0cd", + "parent_span_id": "e81617c6-c2a1-4b08-a5fc-dde3439fe2f4", + "name": "agent.llm_model_execution", + "start_time": "2025-10-22 16:58:05.635000000", + "end_time": "2025-10-22 16:58:09.222000000", + "duration_ms": 3586.924, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What are the current trends in observability tools?", + "query_length": 51, + "query_type": "rag_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 3.5868873596191406, + "model.latency_ms": 3586.8873596191406, + "model.start_time": 1761152285.6352398, + "model.end_time": 1761152289.2221272, + "model.performance_tier": "medium", + "model.prompt_tokens": 872, + "model.completion_tokens": 203, + "model.total_tokens": 1075, + "model.tokens_per_second": 299.7027484337129, + "model.estimated_cost": 0.03225, + "model.efficiency_score": 0.7581395348837209, + "model.response_length": 815, + "model.response_quality": "high", + "model.output_response": "Model Response: The provided context does not explicitly outline current trends in observability tools. However, it highlights the unique challenges faced by AI applications that traditional monitoring tools cannot a...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "45efae74-ee25-4004-a509-6e0ff628a0cd", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:58:04.883000000", + "trace_end_time": "2025-10-22 16:58:10.029000000", + "trace_duration_ms": 5146.244000000001, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:58:04.883000000", + "updated_at": "2025-10-22 16:58:04.883000000" + }, + { + "span_id": "024c05b6-edf9-40ef-931b-9660d48b4948", + "trace_id": "45efae74-ee25-4004-a509-6e0ff628a0cd", + "parent_span_id": "e81617c6-c2a1-4b08-a5fc-dde3439fe2f4", + "name": "agent.rag_evaluation_metrics", + "start_time": "2025-10-22 16:58:09.222000000", + "end_time": "2025-10-22 16:58:09.222000000", + "duration_ms": 0.020999999999999998, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "rag_evaluation_metrics", + "agent.capabilities": [ + "retrieval_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What are the current trends in observability tools?", + "query_length": 51, + "query_type": "rag_evaluation_query", + "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/blog/from-logs-to-intelligent-choices-inside-noveum-ais-evaluation-process):\nFrom Development to Production - Inside Noveum.ai's AI Observability Platform\nShashank Agarwal\n3/3/2025\nIntroduction\nEvery AI application tells a story\u2014but most developers never get to hear it...", + "retrieval.context_length": 2943, + "retrieval.sources_count": 5, + "retrieval.context_quality": "high", + "retrieval.effectiveness": 1, + "retrieval.context_utilization": 2.943, + "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/blog/from-logs-to-intelligent-choices-inside-noveum-ais-evaluation-process):\nFrom Development to Production - Inside Noveum.ai's AI Observability Platform\nShashank Agarwal\n3/3/2025\nIntroduction\nEvery AI application tells a story\u2014but most developers never get to hear it. When your RAG pipeline returns irrelevant results, when your multi-agent system gets stuck in loops, or when your LLM costs suddenly spike, you're left guessing what went wrong and where.\nThat's exactly the challenge Noveum.ai solves. Rather than flying blind, our platform provides comprehensive tracing a...\n\nSource 2 (https://noveum.ai/docs/getting-started/tracing-concepts):\n- Explore Framework Integrations - Framework-specific guidance\n- Learn Advanced Patterns - Custom instrumentation techniques\n- Master the Dashboard - Analyze your traces effectively\nRemember: Good observability is not about collecting all possible data, but about collecting the right data that helps you understand, debug, and optimize your AI applications.\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to O...\n\nSource 3 (https://noveum.ai/en/docs/getting-started/tracing-concepts):\n- Explore Framework Integrations - Framework-specific guidance\n- Learn Advanced Patterns - Custom instrumentation techniques\n- Master the Dashboard - Analyze your traces effectively\nRemember: Good observability is not about collecting all possible data, but about collecting the right data that helps you understand, debug, and optimize your AI applications.\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to O...\n\nSource 4 (https://noveum.ai/en/blog/from-logs-to-intelligent-choices-inside-noveum-ais-evaluation-process):\nThe AI Observability Challenge\nWhy Traditional Monitoring Falls Short\nTraditional application monitoring tools weren't built for AI applications. They can tell you if your API is responding, but they can't answer the questions that matter most for AI systems:\n- Why did my RAG pipeline retrieve irrelevant documents?\n- Which LLM calls are driving my costs?\n- How are my agents communicating with each other?\n- What's causing hallucinations in my responses?\n- Why is my embedding generation so slow?\nW...\n\nSource 5 (https://noveum.ai/en/docs/advanced/rag-observability):\n\ud83d\udd27 Best Practices for RAG Observability\n1. Comprehensive Pipeline Tracing\n2. Quality Monitoring\n3. A/B Testing for RAG Components\n\ud83c\udfaf Advanced RAG Patterns\nMulti-Modal RAG\nConversational RAG\nRAG observability with Noveum.ai provides the deep insights needed to build, optimize, and scale retrieval-augmented generation systems. By implementing comprehensive tracing across retrieval, context preparation, and generation stages, you can ensure your RAG pipeline delivers accurate, relevant, and cost-effe...\n\nUser Question: What are the current trends in observability tools?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", + "prompt.prompt_length": 3490, + "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/blog/from-logs-to-intelligent-choices-inside-noveum-ais-evaluation-process):\nFrom Development to Production - Inside Noveum.ai's AI Observability Platform\nShashank Agarw...", + "prompt.instruction_following": "rag_optimized", + "response.answer_length": 815, + "response.answer_completeness": "complete", + "response.response_quality": "high", + "response.source_citation": 5, + "response.context_utilization": 0.2769283044512402, + "output_response": "RAG Answer: The provided context does not explicitly outline current trends in observability tools. However, it highlights the unique challenges faced by AI applications that traditional monitoring tools cannot a...", + "evaluation.retrieval_effectiveness": 1, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 5, + "evaluation.context_relevance": "high", + "evaluation.overall_quality": "high", + "evaluation.ready_for_production": true, + "rag.retrieval_strategy": "semantic_similarity", + "rag.vector_search_results": 5, + "rag.context_synthesis": "multi_source", + "rag.document_coverage": 1, + "rag.information_density": 0.2769283044512402 + }, + "events": [], + "links": [], + "trace_trace_id": "45efae74-ee25-4004-a509-6e0ff628a0cd", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:58:04.883000000", + "trace_end_time": "2025-10-22 16:58:10.029000000", + "trace_duration_ms": 5146.244000000001, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:58:04.883000000", + "updated_at": "2025-10-22 16:58:04.883000000" + }, + { + "span_id": "99a5e8f3-a9f5-4953-94ea-ad0fa0cf6ec2", + "trace_id": "c5a64d08-d00f-46cf-ab3c-d8d473c10eca", + "parent_span_id": "2a351fd9-a7ee-42da-9e2e-86a643ceed0e", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-22 16:57:57.908000000", + "end_time": "2025-10-22 16:57:57.908000000", + "duration_ms": 0.026, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: Tell me about recent developments in machine learning", + "query_length": 53, + "query_type": "routing_evaluation_query", + "classification.mode": "Web Search", + "classification.rag_keyword_score": 0, + "classification.web_keyword_score": 1, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "complex", + "query.intent": "general_knowledge", + "query.keyword_density": 0.125, + "query.domain_affinity": "general", + "routing.decision": "Routed to Web Search based on analysis", + "routing.rationale": "RAG score: 0, Web score: 1, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "RAG", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.019230769230769232, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: Web Search (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "c5a64d08-d00f-46cf-ab3c-d8d473c10eca", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:57.908000000", + "trace_end_time": "2025-10-22 16:58:03.875000000", + "trace_duration_ms": 5967.0779999999995, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:57.908000000", + "updated_at": "2025-10-22 16:57:57.908000000" + }, + { + "span_id": "6e891e51-ca6d-4e6c-80d0-0791ee22d5b9", + "trace_id": "c5a64d08-d00f-46cf-ab3c-d8d473c10eca", + "parent_span_id": "2a351fd9-a7ee-42da-9e2e-86a643ceed0e", + "name": "agent.web_search_generation", + "start_time": "2025-10-22 16:57:57.908000000", + "end_time": "2025-10-22 16:58:03.874000000", + "duration_ms": 5966.728, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "web_search_agent", + "agent.operation": "web_search_generation", + "agent.capabilities": [ + "web_search", + "content_synthesis", + "response_generation" + ], + "agent.id": "noveum_web_search_agent", + "input_query": "Tell me about recent developments in machine learning", + "query_length": 53, + "query_type": "web_search_query", + "output_response": "Web Search Answer: The web search results did not provide specific information about recent developments in machine learning. Therefore, I cannot summarize any recent advancements based on the provided data.\n\nHowever, i...", + "web_search.results_count": 1, + "web_search.context_length": 768, + "web_search.response_length": 687, + "web_search.mode": "external_web_search" + }, + "events": [], + "links": [], + "trace_trace_id": "c5a64d08-d00f-46cf-ab3c-d8d473c10eca", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:57.908000000", + "trace_end_time": "2025-10-22 16:58:03.875000000", + "trace_duration_ms": 5967.0779999999995, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:57.908000000", + "updated_at": "2025-10-22 16:57:57.908000000" + }, + { + "span_id": "6d622376-d48b-4a64-b0a2-1402117febaf", + "trace_id": "c5a64d08-d00f-46cf-ab3c-d8d473c10eca", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-22 16:57:57.908000000", + "end_time": "2025-10-22 16:58:03.875000000", + "duration_ms": 5967.035, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "Tell me about recent developments in machine learning", + "query.length": 53, + "processing.mode": "Web Search", + "processing.time_seconds": 5.966889142990112, + "processing.response_length": 687, + "processing.sources_count": 1, + "output_response": "Final Answer: The web search results did not provide specific information about recent developments in machine learning. Therefore, I cannot summarize any recent advancements based on the provided data.\n\nHowever, i...", + "final_answer_mode": "Web Search", + "query_processed.input_query": "Tell me about recent developments in machine learning", + "query_processed.output_response": "Successfully processed query using Web Search, generated 687 character response", + "query_processed.mode": "Web Search", + "query_processed.processing_time": 5.966889142990112, + "query_processed.response_length": 687 + }, + "events": [], + "links": [], + "trace_trace_id": "c5a64d08-d00f-46cf-ab3c-d8d473c10eca", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:57.908000000", + "trace_end_time": "2025-10-22 16:58:03.875000000", + "trace_duration_ms": 5967.0779999999995, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:57.908000000", + "updated_at": "2025-10-22 16:57:57.908000000" + }, + { + "span_id": "2a351fd9-a7ee-42da-9e2e-86a643ceed0e", + "trace_id": "c5a64d08-d00f-46cf-ab3c-d8d473c10eca", + "parent_span_id": "6d622376-d48b-4a64-b0a2-1402117febaf", + "name": "agent.query_routing", + "start_time": "2025-10-22 16:57:57.908000000", + "end_time": "2025-10-22 16:58:03.875000000", + "duration_ms": 5966.8640000000005, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "Tell me about recent developments in machine learning", + "query_length": 53, + "query_type": "routing_query", + "output_response": "Routed to Web Search for query processing", + "router.classification": "Web Search", + "router.confidence_score": 1, + "router.rag_keyword_score": 0, + "router.web_keyword_score": 1, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "c5a64d08-d00f-46cf-ab3c-d8d473c10eca", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:57.908000000", + "trace_end_time": "2025-10-22 16:58:03.875000000", + "trace_duration_ms": 5967.0779999999995, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:57.908000000", + "updated_at": "2025-10-22 16:57:57.908000000" + }, + { + "span_id": "ba457b3b-9be8-4e24-af63-4118833711e6", + "trace_id": "c5a64d08-d00f-46cf-ab3c-d8d473c10eca", + "parent_span_id": "6e891e51-ca6d-4e6c-80d0-0791ee22d5b9", + "name": "agent.llm_model_execution", + "start_time": "2025-10-22 16:57:58.324000000", + "end_time": "2025-10-22 16:58:03.874000000", + "duration_ms": 5550.059, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: Tell me about recent developments in machine learning", + "query_length": 53, + "query_type": "web_search_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 5.550010919570923, + "model.latency_ms": 5550.010919570923, + "model.start_time": 1761152278.3248482, + "model.end_time": 1761152283.874859, + "model.performance_tier": "slow", + "model.prompt_tokens": 316, + "model.completion_tokens": 171, + "model.total_tokens": 487, + "model.tokens_per_second": 87.74757510524871, + "model.estimated_cost": 0.01461, + "model.efficiency_score": 1.4106776180698153, + "model.response_length": 687, + "model.response_quality": "high", + "model.output_response": "Model Response: The web search results did not provide specific information about recent developments in machine learning. Therefore, I cannot summarize any recent advancements based on the provided data.\n\nHowever, i...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "c5a64d08-d00f-46cf-ab3c-d8d473c10eca", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:57.908000000", + "trace_end_time": "2025-10-22 16:58:03.875000000", + "trace_duration_ms": 5967.0779999999995, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:57.908000000", + "updated_at": "2025-10-22 16:57:57.908000000" + }, + { + "span_id": "7b70a916-1c2b-4898-a052-cfd98a979167", + "trace_id": "c5a64d08-d00f-46cf-ab3c-d8d473c10eca", + "parent_span_id": "6e891e51-ca6d-4e6c-80d0-0791ee22d5b9", + "name": "agent.web_search_evaluation_metrics", + "start_time": "2025-10-22 16:58:03.874000000", + "end_time": "2025-10-22 16:58:03.874000000", + "duration_ms": 0.02, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "web_search_evaluation_metrics", + "agent.capabilities": [ + "web_search_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: Tell me about recent developments in machine learning", + "query_length": 53, + "query_type": "web_search_evaluation_query", + "web_search.results_count": 1, + "web_search.context_length": 768, + "web_search.context_synthesized": "Context from 1 web sources", + "web_search.search_effectiveness": 0.2, + "web_search.context_quality": "medium", + "web_search.source_diversity": 1, + "web_search.information_synthesis": "low", + "web_search.external_knowledge_utilization": 0.768, + "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nFeb 3, 2024 \u00b7 tell sb sth\u4e2d\u7684tell\u66f4\u591a\u662f\u544a\u77e5\u7684\u610f\u601d\uff0c\u5c31\u662f\u544a\u8bc9\u522b\u4eba\u6709\u8fd9\u4e48\u4e00\u4e2a\u4e8b\u7684\u5b58\u5728 tell sb about sth \u7684\u610f\u601d\u5305\u542b\u66f4\u591a\u4e00\u4e9b\uff0c\u9664\u4e86\u544a\u77e5\u6709\u8fd9\u4e48\u4e00\u4e2a\u4e8b\u60c5\u7684\u5b58\u5728\uff0c\u8fd8\u4f5c\u4e86\u4e00\u4e9b\u8bf4\u660e \u6bd4\u5982\uff1a tell me he \u2026 \u4e00\u3001Say\u548cTell\u7684\u7528\u6cd5 \u6211\u4eec\u5148\u5c06say\u3001tell\u3001speak\u3001talk\u8fd9\u56db\u4e2a\u8bcd\u5206\u6210\u4e24\u7ec4\uff0c\u5206\u522b\u4e3a\uff1a \u7b2c\u2460\u7ec4\uff1asay & tell\uff0c\u8fd9\u4e24\u4e2a\u8bcd\u4e4b\u540e\u53ef\u4ee5\u8868\u793a\u8c08\u8bdd\u7684 \u201c\u7ec6\u8282\u201d\u3002 \u7b2c\u2461\u7ec4\uff1aspeak & talk\uff0c\u4e0d\u80fd\u8868\u793a\u8c08\u8bdd\u7684\u201c\u7ec6\u8282\u201d\u3002 \u2026 tell of 1\u3001Tell of a turning point in your career and what you learned. \u8bb2\u8ff0\u4f60 \u804c\u4e1a\u751f\u6daf \u4e00\u4e2a\u8f6c\u6298\u70b9\uff0c\u4ee5\u53ca\u4f60\u5b66\u5230\u4e86\u4ec0\u4e48\u3002 2\u3001I have made the Sovereign Lord my refuge; I will tell of all your \u2026 tell sb to do sth \u548c tell sb do sth \u90fd\u8868\u793a\u544a\u8bc9\u67d0\u4eba\u505a\u67d0\u4e8b\uff0ctell sb to do sth \u5f3a\u8c03\u544a\u8bc9\u67d0\u4eba\u5e94\u8be5\u505a\u4ec0\u4e48\u5177\u4f53\u52a8\u4f5c\u6216\u884c\u4e3a\uff0c\u800c tell sb do sth \u5f3a\u8c03\u544a\u8bc9\u67d0\u4eba\u6b63\u5728\u8fdb\u884c\u7684\u5177\u4f53\u884c\u52a8\u6216\u72b6\u6001\u3002 \u5b83\u4eec\u5728\u91ca\u4e49\u3001 \u2026 Aug 23, 2023 \u00b7 tell\u7684\u8fc7\u53bb\u5f0f\u548c\u8fc7\u53bb\u5206\u8bcd\u90fd\u662ftold\u3002 \u8bfb\u97f3\uff1a\u82f1[tel]\uff0c\u7f8e[tel]\u3002 \u91ca\u4e49\uff1a vt.\u544a\u8bc9\uff0c\u8bf4\uff1b\u8fa8\u522b\uff1b\u5429\u5490\uff1b\u65ad\u5b9a\u3002 vi.\u8bb2\u8ff0\uff1b\u544a\u53d1\uff0c\u6cc4\u5bc6\uff1b\u8bc6\u522b\u3002 \u53d8\u5f62\uff1a\u8fc7\u53bb\u5f0ftold\uff0c\u8fc7\u53bb\u5206\u8bcdtold\uff0c\u73b0\u5728\u5206 \u2026\nURL: https://duckduckgo.com/?q=Tell+me+about+recent+developments+in+machine+learning\n\nUser Question: Tell me about recent developments in machine learning\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", + "prompt.prompt_length": 1266, + "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nFeb 3, 2024 \u00b7 tell sb sth\u4e2d\u7684tell\u66f4\u591a\u662f\u544a\u77e5\u7684\u610f\u601d\uff0c\u5c31\u662f\u544a\u8bc9\u522b\u4eba\u6709\u8fd9\u4e48\u4e00\u4e2a\u4e8b\u7684\u5b58\u5728 tell sb about sth \u7684\u610f\u601d\u5305\u542b\u66f4\u591a\u4e00\u4e9b\uff0c\u9664\u4e86\u544a\u77e5\u6709\u8fd9\u4e48\u4e00\u4e2a\u4e8b\u60c5\u7684\u5b58\u5728\uff0c\u8fd8\u4f5c\u4e86\u4e00\u4e9b\u8bf4\u660e \u6bd4\u5982\uff1a tell me he \u2026 \u4e00\u3001Say\u548cTell\u7684\u7528\u6cd5 \u6211\u4eec\u5148\u5c06say\u3001tell\u3001speak\u3001talk\u8fd9\u56db\u4e2a\u8bcd\u5206\u6210\u4e24\u7ec4\uff0c\u5206\u522b...", + "prompt.instruction_following": "web_search_optimized", + "response.answer_length": 687, + "response.answer_completeness": "complete", + "response.response_quality": "medium", + "response.source_citation": 1, + "response.context_utilization": 0.89453125, + "output_response": "Web Search Answer: The web search results did not provide specific information about recent developments in machine learning. Therefore, I cannot summarize any recent advancements based on the provided data.\n\nHowever, i...", + "evaluation.search_effectiveness": 0.2, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 1, + "evaluation.context_relevance": "medium", + "evaluation.overall_quality": "low", + "evaluation.ready_for_production": true, + "web_search.search_strategy": "duckduckgo_api", + "web_search.real_time_data": true, + "web_search.external_sources": 1, + "web_search.information_freshness": "current", + "web_search.knowledge_synthesis": "single_source" + }, + "events": [], + "links": [], + "trace_trace_id": "c5a64d08-d00f-46cf-ab3c-d8d473c10eca", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:57.908000000", + "trace_end_time": "2025-10-22 16:58:03.875000000", + "trace_duration_ms": 5967.0779999999995, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:57.908000000", + "updated_at": "2025-10-22 16:57:57.908000000" + }, + { + "span_id": "1e1a341f-66c1-48b0-95eb-fb092928c473", + "trace_id": "b1b13e60-5b32-44a0-9769-197a944989d0", + "parent_span_id": "1c5c4291-bbb5-4f85-b6d9-b6e8df66f742", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-22 16:57:51.967000000", + "end_time": "2025-10-22 16:57:51.967000000", + "duration_ms": 0.033, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What's the weather like today?", + "query_length": 30, + "query_type": "routing_evaluation_query", + "classification.mode": "Web Search", + "classification.rag_keyword_score": 0, + "classification.web_keyword_score": 2, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "general_knowledge", + "query.keyword_density": 0.4, + "query.domain_affinity": "general", + "routing.decision": "Routed to Web Search based on analysis", + "routing.rationale": "RAG score: 0, Web score: 2, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "RAG", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.038461538461538464, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: Web Search (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "b1b13e60-5b32-44a0-9769-197a944989d0", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:51.967000000", + "trace_end_time": "2025-10-22 16:57:56.899000000", + "trace_duration_ms": 4932.393, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:51.967000000", + "updated_at": "2025-10-22 16:57:51.967000000" + }, + { + "span_id": "1c5c4291-bbb5-4f85-b6d9-b6e8df66f742", + "trace_id": "b1b13e60-5b32-44a0-9769-197a944989d0", + "parent_span_id": "49e7cace-ba21-4807-a238-9a3880487583", + "name": "agent.query_routing", + "start_time": "2025-10-22 16:57:51.967000000", + "end_time": "2025-10-22 16:57:56.899000000", + "duration_ms": 4931.972, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What's the weather like today?", + "query_length": 30, + "query_type": "routing_query", + "output_response": "Routed to Web Search for query processing", + "router.classification": "Web Search", + "router.confidence_score": 1, + "router.rag_keyword_score": 0, + "router.web_keyword_score": 2, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "b1b13e60-5b32-44a0-9769-197a944989d0", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:51.967000000", + "trace_end_time": "2025-10-22 16:57:56.899000000", + "trace_duration_ms": 4932.393, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:51.967000000", + "updated_at": "2025-10-22 16:57:51.967000000" + }, + { + "span_id": "49e7cace-ba21-4807-a238-9a3880487583", + "trace_id": "b1b13e60-5b32-44a0-9769-197a944989d0", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-22 16:57:51.967000000", + "end_time": "2025-10-22 16:57:56.899000000", + "duration_ms": 4932.316, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What's the weather like today?", + "query.length": 30, + "processing.mode": "Web Search", + "processing.time_seconds": 4.932007789611816, + "processing.response_length": 260, + "processing.sources_count": 1, + "output_response": "Final Answer: The web search results do not provide specific information about today's weather. To find out the current weather conditions, I recommend checking a reliable weather website or app for detailed update...", + "final_answer_mode": "Web Search", + "query_processed.input_query": "What's the weather like today?", + "query_processed.output_response": "Successfully processed query using Web Search, generated 260 character response", + "query_processed.mode": "Web Search", + "query_processed.processing_time": 4.932007789611816, + "query_processed.response_length": 260 + }, + "events": [], + "links": [], + "trace_trace_id": "b1b13e60-5b32-44a0-9769-197a944989d0", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:51.967000000", + "trace_end_time": "2025-10-22 16:57:56.899000000", + "trace_duration_ms": 4932.393, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:51.967000000", + "updated_at": "2025-10-22 16:57:51.967000000" + }, + { + "span_id": "58d217aa-4350-43e8-8813-468149dc4107", + "trace_id": "b1b13e60-5b32-44a0-9769-197a944989d0", + "parent_span_id": "1c5c4291-bbb5-4f85-b6d9-b6e8df66f742", + "name": "agent.web_search_generation", + "start_time": "2025-10-22 16:57:51.967000000", + "end_time": "2025-10-22 16:57:56.899000000", + "duration_ms": 4931.796, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "web_search_agent", + "agent.operation": "web_search_generation", + "agent.capabilities": [ + "web_search", + "content_synthesis", + "response_generation" + ], + "agent.id": "noveum_web_search_agent", + "input_query": "What's the weather like today?", + "query_length": 30, + "query_type": "web_search_query", + "output_response": "Web Search Answer: The web search results do not provide specific information about today's weather. To find out the current weather conditions, I recommend checking a reliable weather website or app for detailed update...", + "web_search.results_count": 1, + "web_search.context_length": 995, + "web_search.response_length": 260, + "web_search.mode": "external_web_search" + }, + "events": [], + "links": [], + "trace_trace_id": "b1b13e60-5b32-44a0-9769-197a944989d0", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:51.967000000", + "trace_end_time": "2025-10-22 16:57:56.899000000", + "trace_duration_ms": 4932.393, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:51.967000000", + "updated_at": "2025-10-22 16:57:51.967000000" + }, + { + "span_id": "d89dd84e-0b5e-4c92-942b-18c4735713d4", + "trace_id": "b1b13e60-5b32-44a0-9769-197a944989d0", + "parent_span_id": "58d217aa-4350-43e8-8813-468149dc4107", + "name": "agent.llm_model_execution", + "start_time": "2025-10-22 16:57:55.269000000", + "end_time": "2025-10-22 16:57:56.899000000", + "duration_ms": 1629.7140000000002, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What's the weather like today?", + "query_length": 30, + "query_type": "web_search_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 1.629619836807251, + "model.latency_ms": 1629.619836807251, + "model.start_time": 1761152275.269459, + "model.end_time": 1761152276.8990788, + "model.performance_tier": "fast", + "model.prompt_tokens": 367, + "model.completion_tokens": 65, + "model.total_tokens": 432, + "model.tokens_per_second": 265.0925020932329, + "model.estimated_cost": 0.012960000000000001, + "model.efficiency_score": 0.6018518518518519, + "model.response_length": 260, + "model.response_quality": "high", + "model.output_response": "Model Response: The web search results do not provide specific information about today's weather. To find out the current weather conditions, I recommend checking a reliable weather website or app for detailed update...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "b1b13e60-5b32-44a0-9769-197a944989d0", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:51.967000000", + "trace_end_time": "2025-10-22 16:57:56.899000000", + "trace_duration_ms": 4932.393, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:51.967000000", + "updated_at": "2025-10-22 16:57:51.967000000" + }, + { + "span_id": "d076719d-da79-43fb-92ea-6015f856146b", + "trace_id": "b1b13e60-5b32-44a0-9769-197a944989d0", + "parent_span_id": "58d217aa-4350-43e8-8813-468149dc4107", + "name": "agent.web_search_evaluation_metrics", + "start_time": "2025-10-22 16:57:56.899000000", + "end_time": "2025-10-22 16:57:56.899000000", + "duration_ms": 0.049, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "web_search_evaluation_metrics", + "agent.capabilities": [ + "web_search_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What's the weather like today?", + "query_length": 30, + "query_type": "web_search_evaluation_query", + "web_search.results_count": 1, + "web_search.context_length": 995, + "web_search.context_synthesized": "Context from 1 web sources", + "web_search.search_effectiveness": 0.2, + "web_search.context_quality": "high", + "web_search.source_diversity": 1, + "web_search.information_synthesis": "low", + "web_search.external_knowledge_utilization": 0.995, + "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nLog in to WhatsApp Web for simple, reliable and private messaging on your desktop. Send and receive messages and files with ease, all for free. 2 days ago \u00b7 More videos you may like 00:13 Wanna be yours #kimpolin #colinbridgerton #benedictbri... 2 hours ago \u00b7 101 views 00:14 Viscount & Vicountess's Marital bliss #bridgerton \u2026 Nov 15, 2024 \u00b7 Learn the correct usage of \"what's the weather like today\" and \"how's the weather today\" in English. Discover differences, examples, alternatives and tips for choosing the right \u2026 Nov 15, 2024 \u00b7 Learn the correct usage of \"how's the weather today\" and \"what's the weather today\" in English. Discover differences, examples, alternatives and tips for choosing the right \u2026 2 days ago \u00b7 Get the latest hourly weather updates for Thane today. Detailed forecast including temperature, wind, rain, snow, and UV index. Stay informed about today's weather conditions in \u2026\nURL: https://duckduckgo.com/?q=What's+the+weather+like+today?\n\nUser Question: What's the weather like today?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", + "prompt.prompt_length": 1470, + "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nLog in to WhatsApp Web for simple, reliable and private messaging on your desktop. Send and receive messages and files with ease, all for free. 2 days ago \u00b7 More videos you...", + "prompt.instruction_following": "web_search_optimized", + "response.answer_length": 260, + "response.answer_completeness": "complete", + "response.response_quality": "medium", + "response.source_citation": 1, + "response.context_utilization": 0.2613065326633166, + "output_response": "Web Search Answer: The web search results do not provide specific information about today's weather. To find out the current weather conditions, I recommend checking a reliable weather website or app for detailed update...", + "evaluation.search_effectiveness": 0.2, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 1, + "evaluation.context_relevance": "high", + "evaluation.overall_quality": "low", + "evaluation.ready_for_production": true, + "web_search.search_strategy": "duckduckgo_api", + "web_search.real_time_data": true, + "web_search.external_sources": 1, + "web_search.information_freshness": "current", + "web_search.knowledge_synthesis": "single_source" + }, + "events": [], + "links": [], + "trace_trace_id": "b1b13e60-5b32-44a0-9769-197a944989d0", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:51.967000000", + "trace_end_time": "2025-10-22 16:57:56.899000000", + "trace_duration_ms": 4932.393, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:51.967000000", + "updated_at": "2025-10-22 16:57:51.967000000" + }, + { + "span_id": "41167ca4-e1ad-48a5-b5eb-aa272ad88a85", + "trace_id": "e4f3cc20-c137-4905-80fe-2d29b03c2231", + "parent_span_id": "56959f47-5219-4674-8c0d-e5fa928dcd45", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-22 16:57:47.477000000", + "end_time": "2025-10-22 16:57:47.477000000", + "duration_ms": 0.061, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What are the latest AI news today?", + "query_length": 34, + "query_type": "routing_evaluation_query", + "classification.mode": "Web Search", + "classification.rag_keyword_score": 0, + "classification.web_keyword_score": 3, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "general_knowledge", + "query.keyword_density": 0.42857142857142855, + "query.domain_affinity": "general", + "routing.decision": "Routed to Web Search based on analysis", + "routing.rationale": "RAG score: 0, Web score: 3, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "RAG", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.057692307692307696, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: Web Search (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "e4f3cc20-c137-4905-80fe-2d29b03c2231", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:47.477000000", + "trace_end_time": "2025-10-22 16:57:50.959000000", + "trace_duration_ms": 3481.856, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:47.477000000", + "updated_at": "2025-10-22 16:57:47.477000000" + }, + { + "span_id": "73005751-9eb1-4540-aa49-2ac580cd7418", + "trace_id": "e4f3cc20-c137-4905-80fe-2d29b03c2231", + "parent_span_id": "56959f47-5219-4674-8c0d-e5fa928dcd45", + "name": "agent.web_search_generation", + "start_time": "2025-10-22 16:57:47.477000000", + "end_time": "2025-10-22 16:57:50.958000000", + "duration_ms": 3480.8469999999998, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "web_search_agent", + "agent.operation": "web_search_generation", + "agent.capabilities": [ + "web_search", + "content_synthesis", + "response_generation" + ], + "agent.id": "noveum_web_search_agent", + "input_query": "What are the latest AI news today?", + "query_length": 34, + "query_type": "web_search_query", + "output_response": "Web Search Answer: The provided web search results do not contain specific information about the latest AI news today. They primarily focus on general news coverage from India and around the world, including politics, b...", + "web_search.results_count": 1, + "web_search.context_length": 887, + "web_search.response_length": 503, + "web_search.mode": "external_web_search" + }, + "events": [], + "links": [], + "trace_trace_id": "e4f3cc20-c137-4905-80fe-2d29b03c2231", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:47.477000000", + "trace_end_time": "2025-10-22 16:57:50.959000000", + "trace_duration_ms": 3481.856, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:47.477000000", + "updated_at": "2025-10-22 16:57:47.477000000" + }, + { + "span_id": "9e1b87be-bc29-4084-b37c-6e1220dde8eb", + "trace_id": "e4f3cc20-c137-4905-80fe-2d29b03c2231", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-22 16:57:47.477000000", + "end_time": "2025-10-22 16:57:50.959000000", + "duration_ms": 3481.7580000000003, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What are the latest AI news today?", + "query.length": 34, + "processing.mode": "Web Search", + "processing.time_seconds": 3.4813451766967773, + "processing.response_length": 503, + "processing.sources_count": 1, + "output_response": "Final Answer: The provided web search results do not contain specific information about the latest AI news today. They primarily focus on general news coverage from India and around the world, including politics, b...", + "final_answer_mode": "Web Search", + "query_processed.input_query": "What are the latest AI news today?", + "query_processed.output_response": "Successfully processed query using Web Search, generated 503 character response", + "query_processed.mode": "Web Search", + "query_processed.processing_time": 3.4813451766967773, + "query_processed.response_length": 503 + }, + "events": [], + "links": [], + "trace_trace_id": "e4f3cc20-c137-4905-80fe-2d29b03c2231", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:47.477000000", + "trace_end_time": "2025-10-22 16:57:50.959000000", + "trace_duration_ms": 3481.856, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:47.477000000", + "updated_at": "2025-10-22 16:57:47.477000000" + }, + { + "span_id": "56959f47-5219-4674-8c0d-e5fa928dcd45", + "trace_id": "e4f3cc20-c137-4905-80fe-2d29b03c2231", + "parent_span_id": "9e1b87be-bc29-4084-b37c-6e1220dde8eb", + "name": "agent.query_routing", + "start_time": "2025-10-22 16:57:47.477000000", + "end_time": "2025-10-22 16:57:50.958000000", + "duration_ms": 3481.254, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What are the latest AI news today?", + "query_length": 34, + "query_type": "routing_query", + "output_response": "Routed to Web Search for query processing", + "router.classification": "Web Search", + "router.confidence_score": 1, + "router.rag_keyword_score": 0, + "router.web_keyword_score": 3, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "e4f3cc20-c137-4905-80fe-2d29b03c2231", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:47.477000000", + "trace_end_time": "2025-10-22 16:57:50.959000000", + "trace_duration_ms": 3481.856, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:47.477000000", + "updated_at": "2025-10-22 16:57:47.477000000" + }, + { + "span_id": "3dbf95ba-3ef1-4294-93d4-217f4ada5ffe", + "trace_id": "e4f3cc20-c137-4905-80fe-2d29b03c2231", + "parent_span_id": "73005751-9eb1-4540-aa49-2ac580cd7418", + "name": "agent.llm_model_execution", + "start_time": "2025-10-22 16:57:47.864000000", + "end_time": "2025-10-22 16:57:50.958000000", + "duration_ms": 3093.442, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What are the latest AI news today?", + "query_length": 34, + "query_type": "web_search_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 3.0933361053466797, + "model.latency_ms": 3093.3361053466797, + "model.start_time": 1761152267.864976, + "model.end_time": 1761152270.958312, + "model.performance_tier": "medium", + "model.prompt_tokens": 341, + "model.completion_tokens": 125, + "model.total_tokens": 466, + "model.tokens_per_second": 150.6464167261171, + "model.estimated_cost": 0.013980000000000001, + "model.efficiency_score": 1.0793991416309012, + "model.response_length": 503, + "model.response_quality": "high", + "model.output_response": "Model Response: The provided web search results do not contain specific information about the latest AI news today. They primarily focus on general news coverage from India and around the world, including politics, b...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "e4f3cc20-c137-4905-80fe-2d29b03c2231", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:47.477000000", + "trace_end_time": "2025-10-22 16:57:50.959000000", + "trace_duration_ms": 3481.856, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:47.477000000", + "updated_at": "2025-10-22 16:57:47.477000000" + }, + { + "span_id": "f7fe795b-ce1b-4fcc-83f5-06a5f074f38d", + "trace_id": "e4f3cc20-c137-4905-80fe-2d29b03c2231", + "parent_span_id": "73005751-9eb1-4540-aa49-2ac580cd7418", + "name": "agent.web_search_evaluation_metrics", + "start_time": "2025-10-22 16:57:50.958000000", + "end_time": "2025-10-22 16:57:50.958000000", + "duration_ms": 0.026, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "web_search_evaluation_metrics", + "agent.capabilities": [ + "web_search_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What are the latest AI news today?", + "query_length": 34, + "query_type": "web_search_evaluation_query", + "web_search.results_count": 1, + "web_search.context_length": 887, + "web_search.context_synthesized": "Context from 1 web sources", + "web_search.search_effectiveness": 0.2, + "web_search.context_quality": "high", + "web_search.source_diversity": 1, + "web_search.information_synthesis": "low", + "web_search.external_knowledge_utilization": 0.887, + "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\n3 days ago \u00b7 Latest news headlines from India & around the world. Check out today\u2019s news coverage live with videos & photos on NDTV.com. 6 days ago \u00b7 India News | Latest India News | Read latest and breaking news from India. Today's top India news headlines, news on Indian politics, elections, government, business, \u2026 2 days ago \u00b7 Top News Stories of the day, Latest News Headlines, News Specials, Breaking News and Latest India News, World current affairs & Political News all around the clock at \u2026 NDTV.com provides latest news from India and around the world. Get breaking news alerts from India and follow today\u2019s live news updates in field of politics, business, technology, Bollywood ... 6 days ago \u00b7 Get Latest entertainment news from Bollywood, Hollywood and regional film industries.\nURL: https://duckduckgo.com/?q=What+are+the+latest+AI+news+today?\n\nUser Question: What are the latest AI news today?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", + "prompt.prompt_length": 1366, + "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\n3 days ago \u00b7 Latest news headlines from India & around the world. Check out today\u2019s news coverage live with videos & photos on NDTV.com. 6 days ago \u00b7 India News | Latest In...", + "prompt.instruction_following": "web_search_optimized", + "response.answer_length": 503, + "response.answer_completeness": "complete", + "response.response_quality": "medium", + "response.source_citation": 1, + "response.context_utilization": 0.5670800450958287, + "output_response": "Web Search Answer: The provided web search results do not contain specific information about the latest AI news today. They primarily focus on general news coverage from India and around the world, including politics, b...", + "evaluation.search_effectiveness": 0.2, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 1, + "evaluation.context_relevance": "high", + "evaluation.overall_quality": "low", + "evaluation.ready_for_production": true, + "web_search.search_strategy": "duckduckgo_api", + "web_search.real_time_data": true, + "web_search.external_sources": 1, + "web_search.information_freshness": "current", + "web_search.knowledge_synthesis": "single_source" + }, + "events": [], + "links": [], + "trace_trace_id": "e4f3cc20-c137-4905-80fe-2d29b03c2231", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:47.477000000", + "trace_end_time": "2025-10-22 16:57:50.959000000", + "trace_duration_ms": 3481.856, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:47.477000000", + "updated_at": "2025-10-22 16:57:47.477000000" + }, + { + "span_id": "b9c9d21e-5f76-4e5a-b99e-2b4ab821caee", + "trace_id": "823b7927-10bb-49ef-a94e-1b914c298bfb", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-22 16:57:38.501000000", + "end_time": "2025-10-22 16:57:46.469000000", + "duration_ms": 7968.089, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What are the benefits of using Noveum Trace?", + "query.length": 44, + "processing.mode": "RAG", + "processing.time_seconds": 7.968040943145752, + "processing.response_length": 1396, + "processing.sources_count": 5, + "output_response": "Final Answer: Using Noveum Trace offers several benefits, including:\n\n1. **Real-Time Monitoring**: The Noveum platform provides a real-time dashboard for analyzing traces and performance, with live updates as new t...", + "final_answer_mode": "RAG", + "query_processed.input_query": "What are the benefits of using Noveum Trace?", + "query_processed.output_response": "Successfully processed query using RAG, generated 1396 character response", + "query_processed.mode": "RAG", + "query_processed.processing_time": 7.968040943145752, + "query_processed.response_length": 1396 + }, + "events": [], + "links": [], + "trace_trace_id": "823b7927-10bb-49ef-a94e-1b914c298bfb", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:38.501000000", + "trace_end_time": "2025-10-22 16:57:46.469000000", + "trace_duration_ms": 7968.149, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:38.501000000", + "updated_at": "2025-10-22 16:57:38.501000000" + }, + { + "span_id": "bc85c745-4c27-4aa4-bc00-4c9f10a2f049", + "trace_id": "823b7927-10bb-49ef-a94e-1b914c298bfb", + "parent_span_id": "b9c9d21e-5f76-4e5a-b99e-2b4ab821caee", + "name": "agent.query_routing", + "start_time": "2025-10-22 16:57:38.501000000", + "end_time": "2025-10-22 16:57:46.469000000", + "duration_ms": 7968.001, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What are the benefits of using Noveum Trace?", + "query_length": 44, + "query_type": "routing_query", + "output_response": "Routed to RAG for query processing", + "router.classification": "RAG", + "router.confidence_score": 1, + "router.rag_keyword_score": 2, + "router.web_keyword_score": 0, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "823b7927-10bb-49ef-a94e-1b914c298bfb", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:38.501000000", + "trace_end_time": "2025-10-22 16:57:46.469000000", + "trace_duration_ms": 7968.149, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:38.501000000", + "updated_at": "2025-10-22 16:57:38.501000000" + }, + { + "span_id": "3ab8cc4c-f5ed-410a-b4ee-48d0954b756d", + "trace_id": "823b7927-10bb-49ef-a94e-1b914c298bfb", + "parent_span_id": "bc85c745-4c27-4aa4-bc00-4c9f10a2f049", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-22 16:57:38.501000000", + "end_time": "2025-10-22 16:57:38.501000000", + "duration_ms": 0.09899999999999999, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What are the benefits of using Noveum Trace?", + "query_length": 44, + "query_type": "routing_evaluation_query", + "classification.mode": "RAG", + "classification.rag_keyword_score": 2, + "classification.web_keyword_score": 0, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "noveum_specific", + "query.keyword_density": 0.25, + "query.domain_affinity": "noveum", + "routing.decision": "Routed to RAG based on analysis", + "routing.rationale": "RAG score: 2, Web score: 0, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "Web Search", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.038461538461538464, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: RAG (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "823b7927-10bb-49ef-a94e-1b914c298bfb", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:38.501000000", + "trace_end_time": "2025-10-22 16:57:46.469000000", + "trace_duration_ms": 7968.149, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:38.501000000", + "updated_at": "2025-10-22 16:57:38.501000000" + }, + { + "span_id": "67292076-ec8a-4856-9c5b-726868552157", + "trace_id": "823b7927-10bb-49ef-a94e-1b914c298bfb", + "parent_span_id": "bc85c745-4c27-4aa4-bc00-4c9f10a2f049", + "name": "agent.llm-rag", + "start_time": "2025-10-22 16:57:38.501000000", + "end_time": "2025-10-22 16:57:46.469000000", + "duration_ms": 7967.679, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "rag_agent", + "agent.operation": "llm-rag", + "agent.capabilities": [ + "document_retrieval", + "context_generation", + "response_generation" + ], + "agent.id": "noveum_rag_agent", + "input_query": "What are the benefits of using Noveum Trace?", + "query_length": 44, + "query_type": "rag_query", + "output_response": "RAG Answer: Using Noveum Trace offers several benefits, including:\n\n1. **Real-Time Monitoring**: The Noveum platform provides a real-time dashboard for analyzing traces and performance, with live updates as new t...", + "rag.context_length": 2773, + "rag.sources_count": 5, + "rag.answer_length": 1396, + "rag.mode": "retrieval_augmented_generation" + }, + "events": [], + "links": [], + "trace_trace_id": "823b7927-10bb-49ef-a94e-1b914c298bfb", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:38.501000000", + "trace_end_time": "2025-10-22 16:57:46.469000000", + "trace_duration_ms": 7968.149, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:38.501000000", + "updated_at": "2025-10-22 16:57:38.501000000" + }, + { + "span_id": "d044ae4a-c463-4c1b-abce-a9a18f045e75", + "trace_id": "823b7927-10bb-49ef-a94e-1b914c298bfb", + "parent_span_id": "67292076-ec8a-4856-9c5b-726868552157", + "name": "agent.llm_model_execution", + "start_time": "2025-10-22 16:57:38.901000000", + "end_time": "2025-10-22 16:57:45.885000000", + "duration_ms": 6984.242, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What are the benefits of using Noveum Trace?", + "query_length": 44, + "query_type": "rag_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 6.984165906906128, + "model.latency_ms": 6984.165906906128, + "model.start_time": 1761152258.9012191, + "model.end_time": 1761152265.885385, + "model.performance_tier": "slow", + "model.prompt_tokens": 828, + "model.completion_tokens": 349, + "model.total_tokens": 1177, + "model.tokens_per_second": 168.524060809631, + "model.estimated_cost": 0.03531, + "model.efficiency_score": 1.1860662701784197, + "model.response_length": 1396, + "model.response_quality": "high", + "model.output_response": "Model Response: Using Noveum Trace offers several benefits, including:\n\n1. **Real-Time Monitoring**: The Noveum platform provides a real-time dashboard for analyzing traces and performance, with live updates as new t...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "823b7927-10bb-49ef-a94e-1b914c298bfb", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:38.501000000", + "trace_end_time": "2025-10-22 16:57:46.469000000", + "trace_duration_ms": 7968.149, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:38.501000000", + "updated_at": "2025-10-22 16:57:38.501000000" + }, + { + "span_id": "5b1548cf-0de7-49d4-b9b8-9ad1f9f2abc9", + "trace_id": "823b7927-10bb-49ef-a94e-1b914c298bfb", + "parent_span_id": "67292076-ec8a-4856-9c5b-726868552157", + "name": "agent.rag_evaluation_metrics", + "start_time": "2025-10-22 16:57:45.885000000", + "end_time": "2025-10-22 16:57:45.885000000", + "duration_ms": 0.061, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "rag_evaluation_metrics", + "agent.capabilities": [ + "retrieval_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What are the benefits of using Noveum Trace?", + "query_length": 44, + "query_type": "rag_evaluation_query", + "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/docs/platform/dashboard):\n- Bulk Export: Download multiple traces simultaneously\n- Comparative Analysis: Compare performance across multiple traces\nReal-Time Updates\n- Live Refresh: Automatic updates as new traces arrive\n- Connection Monitoring: Real-time status of your t...", + "retrieval.context_length": 2773, + "retrieval.sources_count": 5, + "retrieval.context_quality": "high", + "retrieval.effectiveness": 1, + "retrieval.context_utilization": 2.773, + "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/docs/platform/dashboard):\n- Bulk Export: Download multiple traces simultaneously\n- Comparative Analysis: Compare performance across multiple traces\nReal-Time Updates\n- Live Refresh: Automatic updates as new traces arrive\n- Connection Monitoring: Real-time status of your trace ingestion pipeline\n- Performance Indicators: Live metrics for system health monitoring\nReady to dive deeper? Explore Projects & Environments to organize your AI applications, or check out Team Collaboration to share insights with your team.\nGet Earl...\n\nSource 2 (https://noveum.ai/en/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\n\ud83d\udcd8 TypeScript SDK (\n@noveum/trace\n)- Framework integrations for Next.js, Express.js, and Hono\n- TypeScript-first design with full type safety\n- Universal compatibility (Node.js, Edge Runtime, browsers)\n-\n\ud83d\udcca Noveum Platform\n- Real-time dashboard for analyzing traces and performance\n- Ad...\n\nSource 3 (https://noveum.ai/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\n\ud83d\udcd8 TypeScript SDK (\n@noveum/trace\n)- Framework integrations for Next.js, Express.js, and Hono\n- TypeScript-first design with full type safety\n- Universal compatibility (Node.js, Edge Runtime, browsers)\n-\n\ud83d\udcca Noveum Platform\n- Real-time dashboard for analyzing traces and performance\n- Ad...\n\nSource 4 (https://noveum.ai/en/docs/getting-started/overview):\n- Advanced search & filtering across traces and spans\n- Cost analysis and optimization recommendations\n- Team collaboration with shared insights and alerts\n\ud83d\udd0d What Noveum Traces\nLLM Operations\n- Model calls across all providers (OpenAI, Anthropic, Google, etc.)\n- Token usage and cost calculation\n- Prompt engineering effectiveness\n- Response quality metrics\nRAG Pipelines\n- Document retrieval performance and relevance\n- Embedding generation costs and latency\n- Context assembly and prompt constructi...\n\nSource 5 (https://noveum.ai/docs/getting-started/overview):\n- Advanced search & filtering across traces and spans\n- Cost analysis and optimization recommendations\n- Team collaboration with shared insights and alerts\n\ud83d\udd0d What Noveum Traces\nLLM Operations\n- Model calls across all providers (OpenAI, Anthropic, Google, etc.)\n- Token usage and cost calculation\n- Prompt engineering effectiveness\n- Response quality metrics\nRAG Pipelines\n- Document retrieval performance and relevance\n- Embedding generation costs and latency\n- Context assembly and prompt constructi...\n\nUser Question: What are the benefits of using Noveum Trace?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", + "prompt.prompt_length": 3313, + "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/docs/platform/dashboard):\n- Bulk Export: Download multiple traces simultaneously\n- Comparative Analysis: Compare performance across multiple traces\nReal-Time Updates\n- Live...", + "prompt.instruction_following": "rag_optimized", + "response.answer_length": 1396, + "response.answer_completeness": "complete", + "response.response_quality": "high", + "response.source_citation": 5, + "response.context_utilization": 0.5034258925351605, + "output_response": "RAG Answer: Using Noveum Trace offers several benefits, including:\n\n1. **Real-Time Monitoring**: The Noveum platform provides a real-time dashboard for analyzing traces and performance, with live updates as new t...", + "evaluation.retrieval_effectiveness": 1, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 5, + "evaluation.context_relevance": "high", + "evaluation.overall_quality": "high", + "evaluation.ready_for_production": true, + "rag.retrieval_strategy": "semantic_similarity", + "rag.vector_search_results": 5, + "rag.context_synthesis": "multi_source", + "rag.document_coverage": 1, + "rag.information_density": 0.5034258925351605 + }, + "events": [], + "links": [], + "trace_trace_id": "823b7927-10bb-49ef-a94e-1b914c298bfb", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:38.501000000", + "trace_end_time": "2025-10-22 16:57:46.469000000", + "trace_duration_ms": 7968.149, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:38.501000000", + "updated_at": "2025-10-22 16:57:38.501000000" + }, + { + "span_id": "aaba719b-36ec-46ab-af2e-3ea8c18e9872", + "trace_id": "1ae0d6c4-d1fb-4239-b536-3333187d44aa", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-22 16:57:32.800000000", + "end_time": "2025-10-22 16:57:37.494000000", + "duration_ms": 4694.408, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "How do I configure Noveum for my system?", + "query.length": 40, + "processing.mode": "RAG", + "processing.time_seconds": 4.694374084472656, + "processing.response_length": 553, + "processing.sources_count": 5, + "output_response": "Final Answer: To configure Noveum for your system, you can refer to the SDK Integration Guide available in the Noveum documentation. This guide will help you trace your AI applications with minimal code changes. Th...", + "final_answer_mode": "RAG", + "query_processed.input_query": "How do I configure Noveum for my system?", + "query_processed.output_response": "Successfully processed query using RAG, generated 553 character response", + "query_processed.mode": "RAG", + "query_processed.processing_time": 4.694374084472656, + "query_processed.response_length": 553 + }, + "events": [], + "links": [], + "trace_trace_id": "1ae0d6c4-d1fb-4239-b536-3333187d44aa", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:32.800000000", + "trace_end_time": "2025-10-22 16:57:37.494000000", + "trace_duration_ms": 4694.468, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:32.800000000", + "updated_at": "2025-10-22 16:57:32.800000000" + }, + { + "span_id": "4254e6f9-10b8-42ba-939d-d7c124f80f9a", + "trace_id": "1ae0d6c4-d1fb-4239-b536-3333187d44aa", + "parent_span_id": "6d5d074e-8fd3-4d33-88fb-7f54b0959030", + "name": "agent.llm-rag", + "start_time": "2025-10-22 16:57:32.800000000", + "end_time": "2025-10-22 16:57:37.494000000", + "duration_ms": 4694.155000000001, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "rag_agent", + "agent.operation": "llm-rag", + "agent.capabilities": [ + "document_retrieval", + "context_generation", + "response_generation" + ], + "agent.id": "noveum_rag_agent", + "input_query": "How do I configure Noveum for my system?", + "query_length": 40, + "query_type": "rag_query", + "output_response": "RAG Answer: To configure Noveum for your system, you can refer to the SDK Integration Guide available in the Noveum documentation. This guide will help you trace your AI applications with minimal code changes. Th...", + "rag.context_length": 2237, + "rag.sources_count": 5, + "rag.answer_length": 553, + "rag.mode": "retrieval_augmented_generation" + }, + "events": [], + "links": [], + "trace_trace_id": "1ae0d6c4-d1fb-4239-b536-3333187d44aa", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:32.800000000", + "trace_end_time": "2025-10-22 16:57:37.494000000", + "trace_duration_ms": 4694.468, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:32.800000000", + "updated_at": "2025-10-22 16:57:32.800000000" + }, + { + "span_id": "6d5d074e-8fd3-4d33-88fb-7f54b0959030", + "trace_id": "1ae0d6c4-d1fb-4239-b536-3333187d44aa", + "parent_span_id": "aaba719b-36ec-46ab-af2e-3ea8c18e9872", + "name": "agent.query_routing", + "start_time": "2025-10-22 16:57:32.800000000", + "end_time": "2025-10-22 16:57:37.494000000", + "duration_ms": 4694.318, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "How do I configure Noveum for my system?", + "query_length": 40, + "query_type": "routing_query", + "output_response": "Routed to RAG for query processing", + "router.classification": "RAG", + "router.confidence_score": 1, + "router.rag_keyword_score": 2, + "router.web_keyword_score": 0, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "1ae0d6c4-d1fb-4239-b536-3333187d44aa", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:32.800000000", + "trace_end_time": "2025-10-22 16:57:37.494000000", + "trace_duration_ms": 4694.468, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:32.800000000", + "updated_at": "2025-10-22 16:57:32.800000000" + }, + { + "span_id": "2cc2ab0b-2572-4a75-88a3-18b3a1eaca78", + "trace_id": "1ae0d6c4-d1fb-4239-b536-3333187d44aa", + "parent_span_id": "6d5d074e-8fd3-4d33-88fb-7f54b0959030", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-22 16:57:32.800000000", + "end_time": "2025-10-22 16:57:32.800000000", + "duration_ms": 0.04, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: How do I configure Noveum for my system?", + "query_length": 40, + "query_type": "routing_evaluation_query", + "classification.mode": "RAG", + "classification.rag_keyword_score": 2, + "classification.web_keyword_score": 0, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "noveum_specific", + "query.keyword_density": 0.25, + "query.domain_affinity": "noveum", + "routing.decision": "Routed to RAG based on analysis", + "routing.rationale": "RAG score: 2, Web score: 0, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "Web Search", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.038461538461538464, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: RAG (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "1ae0d6c4-d1fb-4239-b536-3333187d44aa", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:32.800000000", + "trace_end_time": "2025-10-22 16:57:37.494000000", + "trace_duration_ms": 4694.468, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:32.800000000", + "updated_at": "2025-10-22 16:57:32.800000000" + }, + { + "span_id": "f1d9f779-3312-43c1-b549-00fc874429f0", + "trace_id": "1ae0d6c4-d1fb-4239-b536-3333187d44aa", + "parent_span_id": "4254e6f9-10b8-42ba-939d-d7c124f80f9a", + "name": "agent.llm_model_execution", + "start_time": "2025-10-22 16:57:33.281000000", + "end_time": "2025-10-22 16:57:37.060000000", + "duration_ms": 3778.743, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: How do I configure Noveum for my system?", + "query_length": 40, + "query_type": "rag_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 3.7786097526550293, + "model.latency_ms": 3778.6097526550293, + "model.start_time": 1761152253.28163, + "model.end_time": 1761152257.0602398, + "model.performance_tier": "medium", + "model.prompt_tokens": 693, + "model.completion_tokens": 138, + "model.total_tokens": 831, + "model.tokens_per_second": 219.92215507730066, + "model.estimated_cost": 0.02493, + "model.efficiency_score": 0.6654632972322503, + "model.response_length": 553, + "model.response_quality": "high", + "model.output_response": "Model Response: To configure Noveum for your system, you can refer to the SDK Integration Guide available in the Noveum documentation. This guide will help you trace your AI applications with minimal code changes. Th...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "1ae0d6c4-d1fb-4239-b536-3333187d44aa", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:32.800000000", + "trace_end_time": "2025-10-22 16:57:37.494000000", + "trace_duration_ms": 4694.468, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:32.800000000", + "updated_at": "2025-10-22 16:57:32.800000000" + }, + { + "span_id": "5500be90-0cfa-44dd-a6e3-e72ac9f0a79e", + "trace_id": "1ae0d6c4-d1fb-4239-b536-3333187d44aa", + "parent_span_id": "4254e6f9-10b8-42ba-939d-d7c124f80f9a", + "name": "agent.rag_evaluation_metrics", + "start_time": "2025-10-22 16:57:37.060000000", + "end_time": "2025-10-22 16:57:37.060000000", + "duration_ms": 0.076, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "rag_evaluation_metrics", + "agent.capabilities": [ + "retrieval_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: How do I configure Noveum for my system?", + "query_length": 40, + "query_type": "rag_evaluation_query", + "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/docs):\nWelcome to Noveum.ai Docs\nComprehensive AI tracing and observability for LLM applications, RAG systems, and AI agents\nWelcome to the Noveum.ai documentation! Here you'll find everything you need to integrate, configure, and optimize your AI applications using ou...", + "retrieval.context_length": 2237, + "retrieval.sources_count": 5, + "retrieval.context_quality": "high", + "retrieval.effectiveness": 1, + "retrieval.context_utilization": 2.237, + "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/docs):\nWelcome to Noveum.ai Docs\nComprehensive AI tracing and observability for LLM applications, RAG systems, and AI agents\nWelcome to the Noveum.ai documentation! Here you'll find everything you need to integrate, configure, and optimize your AI applications using our comprehensive tracing and observability platform.\nWe provide powerful SDKs for Python and TypeScript that enable you to trace LLM calls, RAG pipelines, and multi-agent workflows with minimal code changes. Our documentation is organized ...\n\nSource 2 (https://noveum.ai/docs):\nWelcome to Noveum.ai Docs\nComprehensive AI tracing and observability for LLM applications, RAG systems, and AI agents\nWelcome to the Noveum.ai documentation! Here you'll find everything you need to integrate, configure, and optimize your AI applications using our comprehensive tracing and observability platform.\nWe provide powerful SDKs for Python and TypeScript that enable you to trace LLM calls, RAG pipelines, and multi-agent workflows with minimal code changes. Our documentation is organized ...\n\nSource 3 (https://noveum.ai/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nSource 4 (https://noveum.ai/en/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nSource 5 (https://noveum.ai/en/docs):\n- \ud83c\udf0d Universal: Works across frameworks, clouds, and deployment models\n\ud83e\udd1d Community & Support\n- \ud83d\udcac Discord Community: Join our Discord\n- \ud83d\udce7 Email Support: [email protected]\n- \ud83d\udc1b Bug Reports: GitHub Issues\n- \ud83d\udcd6 Knowledge Base: Help Center\nReady to get started? Head to our SDK Integration Guide to begin tracing your AI applications in under 5 minutes!\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability ...\n\nUser Question: How do I configure Noveum for my system?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", + "prompt.prompt_length": 2773, + "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/docs):\nWelcome to Noveum.ai Docs\nComprehensive AI tracing and observability for LLM applications, RAG systems, and AI agents\nWelcome to the Noveum.ai documentation! Here...", + "prompt.instruction_following": "rag_optimized", + "response.answer_length": 553, + "response.answer_completeness": "complete", + "response.response_quality": "high", + "response.source_citation": 5, + "response.context_utilization": 0.24720607957085383, + "output_response": "RAG Answer: To configure Noveum for your system, you can refer to the SDK Integration Guide available in the Noveum documentation. This guide will help you trace your AI applications with minimal code changes. Th...", + "evaluation.retrieval_effectiveness": 1, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 5, + "evaluation.context_relevance": "high", + "evaluation.overall_quality": "high", + "evaluation.ready_for_production": true, + "rag.retrieval_strategy": "semantic_similarity", + "rag.vector_search_results": 5, + "rag.context_synthesis": "multi_source", + "rag.document_coverage": 1, + "rag.information_density": 0.24720607957085383 + }, + "events": [], + "links": [], + "trace_trace_id": "1ae0d6c4-d1fb-4239-b536-3333187d44aa", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:32.800000000", + "trace_end_time": "2025-10-22 16:57:37.494000000", + "trace_duration_ms": 4694.468, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:32.800000000", + "updated_at": "2025-10-22 16:57:32.800000000" + }, + { + "span_id": "3068013a-6844-4f3c-8ec2-854a7353591e", + "trace_id": "fb0d31fc-2465-4ecd-973c-4cd443d9befa", + "parent_span_id": "a80d8fab-d5dc-4417-a5e5-9a2a76ed7419", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-22 16:57:24.828000000", + "end_time": "2025-10-22 16:57:24.828000000", + "duration_ms": 0.038, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What monitoring capabilities does Noveum provide?", + "query_length": 49, + "query_type": "routing_evaluation_query", + "classification.mode": "RAG", + "classification.rag_keyword_score": 2, + "classification.web_keyword_score": 0, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "noveum_specific", + "query.keyword_density": 0.3333333333333333, + "query.domain_affinity": "noveum", + "routing.decision": "Routed to RAG based on analysis", + "routing.rationale": "RAG score: 2, Web score: 0, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "Web Search", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.038461538461538464, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: RAG (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "fb0d31fc-2465-4ecd-973c-4cd443d9befa", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:24.828000000", + "trace_end_time": "2025-10-22 16:57:31.793000000", + "trace_duration_ms": 6965.302000000001, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:24.828000000", + "updated_at": "2025-10-22 16:57:24.828000000" + }, + { + "span_id": "189b89e3-fbbd-41de-8521-efb6a68f62f2", + "trace_id": "fb0d31fc-2465-4ecd-973c-4cd443d9befa", + "parent_span_id": "a80d8fab-d5dc-4417-a5e5-9a2a76ed7419", + "name": "agent.llm-rag", + "start_time": "2025-10-22 16:57:24.828000000", + "end_time": "2025-10-22 16:57:31.793000000", + "duration_ms": 6964.955, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "rag_agent", + "agent.operation": "llm-rag", + "agent.capabilities": [ + "document_retrieval", + "context_generation", + "response_generation" + ], + "agent.id": "noveum_rag_agent", + "input_query": "What monitoring capabilities does Noveum provide?", + "query_length": 49, + "query_type": "rag_query", + "output_response": "RAG Answer: Noveum provides comprehensive monitoring capabilities for AI applications, including the ability to monitor, trace, and optimize AI agents across various frameworks such as LangChain, CrewAI, AutoGen,...", + "rag.context_length": 2740, + "rag.sources_count": 5, + "rag.answer_length": 582, + "rag.mode": "retrieval_augmented_generation" + }, + "events": [], + "links": [], + "trace_trace_id": "fb0d31fc-2465-4ecd-973c-4cd443d9befa", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:24.828000000", + "trace_end_time": "2025-10-22 16:57:31.793000000", + "trace_duration_ms": 6965.302000000001, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:24.828000000", + "updated_at": "2025-10-22 16:57:24.828000000" + }, + { + "span_id": "f5cffb7d-341b-4cde-b162-02208042463d", + "trace_id": "fb0d31fc-2465-4ecd-973c-4cd443d9befa", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-22 16:57:24.828000000", + "end_time": "2025-10-22 16:57:31.793000000", + "duration_ms": 6965.236, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What monitoring capabilities does Noveum provide?", + "query.length": 49, + "processing.mode": "RAG", + "processing.time_seconds": 6.965206146240234, + "processing.response_length": 582, + "processing.sources_count": 5, + "output_response": "Final Answer: Noveum provides comprehensive monitoring capabilities for AI applications, including the ability to monitor, trace, and optimize AI agents across various frameworks such as LangChain, CrewAI, AutoGen,...", + "final_answer_mode": "RAG", + "query_processed.input_query": "What monitoring capabilities does Noveum provide?", + "query_processed.output_response": "Successfully processed query using RAG, generated 582 character response", + "query_processed.mode": "RAG", + "query_processed.processing_time": 6.965206146240234, + "query_processed.response_length": 582 + }, + "events": [], + "links": [], + "trace_trace_id": "fb0d31fc-2465-4ecd-973c-4cd443d9befa", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:24.828000000", + "trace_end_time": "2025-10-22 16:57:31.793000000", + "trace_duration_ms": 6965.302000000001, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:24.828000000", + "updated_at": "2025-10-22 16:57:24.828000000" + }, + { + "span_id": "a80d8fab-d5dc-4417-a5e5-9a2a76ed7419", + "trace_id": "fb0d31fc-2465-4ecd-973c-4cd443d9befa", + "parent_span_id": "f5cffb7d-341b-4cde-b162-02208042463d", + "name": "agent.query_routing", + "start_time": "2025-10-22 16:57:24.828000000", + "end_time": "2025-10-22 16:57:31.793000000", + "duration_ms": 6965.17, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What monitoring capabilities does Noveum provide?", + "query_length": 49, + "query_type": "routing_query", + "output_response": "Routed to RAG for query processing", + "router.classification": "RAG", + "router.confidence_score": 1, + "router.rag_keyword_score": 2, + "router.web_keyword_score": 0, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "fb0d31fc-2465-4ecd-973c-4cd443d9befa", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:24.828000000", + "trace_end_time": "2025-10-22 16:57:31.793000000", + "trace_duration_ms": 6965.302000000001, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:24.828000000", + "updated_at": "2025-10-22 16:57:24.828000000" + }, + { + "span_id": "a2b632b8-fbcb-480f-9292-9f513c64df38", + "trace_id": "fb0d31fc-2465-4ecd-973c-4cd443d9befa", + "parent_span_id": "189b89e3-fbbd-41de-8521-efb6a68f62f2", + "name": "agent.llm_model_execution", + "start_time": "2025-10-22 16:57:27.824000000", + "end_time": "2025-10-22 16:57:31.155000000", + "duration_ms": 3330.989, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What monitoring capabilities does Noveum provide?", + "query_length": 49, + "query_type": "rag_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 3.3309478759765625, + "model.latency_ms": 3330.9478759765625, + "model.start_time": 1761152247.824474, + "model.end_time": 1761152251.155422, + "model.performance_tier": "medium", + "model.prompt_tokens": 821, + "model.completion_tokens": 145, + "model.total_tokens": 966, + "model.tokens_per_second": 290.00754018607677, + "model.estimated_cost": 0.028980000000000002, + "model.efficiency_score": 0.6024844720496895, + "model.response_length": 582, + "model.response_quality": "high", + "model.output_response": "Model Response: Noveum provides comprehensive monitoring capabilities for AI applications, including the ability to monitor, trace, and optimize AI agents across various frameworks such as LangChain, CrewAI, AutoGen,...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "fb0d31fc-2465-4ecd-973c-4cd443d9befa", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:24.828000000", + "trace_end_time": "2025-10-22 16:57:31.793000000", + "trace_duration_ms": 6965.302000000001, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:24.828000000", + "updated_at": "2025-10-22 16:57:24.828000000" + }, + { + "span_id": "40ff54c4-07ca-4b2d-a9b4-7e3ae7adcd81", + "trace_id": "fb0d31fc-2465-4ecd-973c-4cd443d9befa", + "parent_span_id": "189b89e3-fbbd-41de-8521-efb6a68f62f2", + "name": "agent.rag_evaluation_metrics", + "start_time": "2025-10-22 16:57:31.155000000", + "end_time": "2025-10-22 16:57:31.155000000", + "duration_ms": 0.018000000000000002, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "rag_evaluation_metrics", + "agent.capabilities": [ + "retrieval_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What monitoring capabilities does Noveum provide?", + "query_length": 49, + "query_type": "rag_evaluation_query", + "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en):\nMonitor all your AI Agents\nimprove AI Agents today\nNoveum.ai helps you monitor, trace, and optimize your AI applications.\nNoveum.ai works with any AI framework \u2013 LangChain, CrewAI, AutoGen, custom implementations, or direct LLM calls. One dashboard shows everything.\n...", + "retrieval.context_length": 2740, + "retrieval.sources_count": 5, + "retrieval.context_quality": "high", + "retrieval.effectiveness": 1, + "retrieval.context_utilization": 2.74, + "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en):\nMonitor all your AI Agents\nimprove AI Agents today\nNoveum.ai helps you monitor, trace, and optimize your AI applications.\nNoveum.ai works with any AI framework \u2013 LangChain, CrewAI, AutoGen, custom implementations, or direct LLM calls. One dashboard shows everything.\nMonitor, Evaluate, Improve Your AI Agents\nThe control plane for AI agents.\nMonitor Everything, Miss Nothing\nOur lightweight SDKs capture every trace and span across your AI agent ecosystem\u2014from simple LLM calls to complex multi-agent...\n\nSource 2 (https://noveum.ai/):\nMonitor all your AI Agents\nimprove AI Agents today\nNoveum.ai helps you monitor, trace, and optimize your AI applications.\nNoveum.ai works with any AI framework \u2013 LangChain, CrewAI, AutoGen, custom implementations, or direct LLM calls. One dashboard shows everything.\nMonitor, Evaluate, Improve Your AI Agents\nThe control plane for AI agents.\nMonitor Everything, Miss Nothing\nOur lightweight SDKs capture every trace and span across your AI agent ecosystem\u2014from simple LLM calls to complex multi-agent...\n\nSource 3 (https://noveum.ai):\nMonitor all your AI Agents\nimprove AI Agents today\nNoveum.ai helps you monitor, trace, and optimize your AI applications.\nNoveum.ai works with any AI framework \u2013 LangChain, CrewAI, AutoGen, custom implementations, or direct LLM calls. One dashboard shows everything.\nMonitor, Evaluate, Improve Your AI Agents\nThe control plane for AI agents.\nMonitor Everything, Miss Nothing\nOur lightweight SDKs capture every trace and span across your AI agent ecosystem\u2014from simple LLM calls to complex multi-agent...\n\nSource 4 (https://noveum.ai/en/docs/getting-started/overview):\nNoveum.ai Overview\nComprehensive AI tracing and observability platform for LLM applications, RAG systems, and AI agents\nWelcome to Noveum.ai\u2014the comprehensive tracing and observability platform built specifically for AI applications. Whether you're building LLM-powered chatbots, RAG systems, multi-agent workflows, or any AI-driven application, Noveum provides the insights you need to understand, debug, and optimize your systems.\n\ud83c\udfaf Why AI Applications Need Specialized Observability\nTraditional mo...\n\nSource 5 (https://noveum.ai/docs/getting-started/overview):\nNoveum.ai Overview\nComprehensive AI tracing and observability platform for LLM applications, RAG systems, and AI agents\nWelcome to Noveum.ai\u2014the comprehensive tracing and observability platform built specifically for AI applications. Whether you're building LLM-powered chatbots, RAG systems, multi-agent workflows, or any AI-driven application, Noveum provides the insights you need to understand, debug, and optimize your systems.\n\ud83c\udfaf Why AI Applications Need Specialized Observability\nTraditional mo...\n\nUser Question: What monitoring capabilities does Noveum provide?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", + "prompt.prompt_length": 3285, + "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en):\nMonitor all your AI Agents\nimprove AI Agents today\nNoveum.ai helps you monitor, trace, and optimize your AI applications.\nNoveum.ai works with any AI framework \u2013 LangC...", + "prompt.instruction_following": "rag_optimized", + "response.answer_length": 582, + "response.answer_completeness": "complete", + "response.response_quality": "high", + "response.source_citation": 5, + "response.context_utilization": 0.2124087591240876, + "output_response": "RAG Answer: Noveum provides comprehensive monitoring capabilities for AI applications, including the ability to monitor, trace, and optimize AI agents across various frameworks such as LangChain, CrewAI, AutoGen,...", + "evaluation.retrieval_effectiveness": 1, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 5, + "evaluation.context_relevance": "high", + "evaluation.overall_quality": "high", + "evaluation.ready_for_production": true, + "rag.retrieval_strategy": "semantic_similarity", + "rag.vector_search_results": 5, + "rag.context_synthesis": "multi_source", + "rag.document_coverage": 1, + "rag.information_density": 0.2124087591240876 + }, + "events": [], + "links": [], + "trace_trace_id": "fb0d31fc-2465-4ecd-973c-4cd443d9befa", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:24.828000000", + "trace_end_time": "2025-10-22 16:57:31.793000000", + "trace_duration_ms": 6965.302000000001, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:24.828000000", + "updated_at": "2025-10-22 16:57:24.828000000" + }, + { + "span_id": "b0f0a2e0-c1f7-4ba4-8b75-7090e931a738", + "trace_id": "3bbd0b36-c30f-4126-b62a-dd0fae1c701d", + "parent_span_id": "4db80813-e62f-48ec-9091-62d569527caf", + "name": "agent.query_routing", + "start_time": "2025-10-22 16:57:19.339000000", + "end_time": "2025-10-22 16:57:23.822000000", + "duration_ms": 4482.909000000001, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "How does Noveum handle agent tracing?", + "query_length": 37, + "query_type": "routing_query", + "output_response": "Routed to RAG for query processing", + "router.classification": "RAG", + "router.confidence_score": 1, + "router.rag_keyword_score": 2, + "router.web_keyword_score": 0, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "3bbd0b36-c30f-4126-b62a-dd0fae1c701d", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:19.339000000", + "trace_end_time": "2025-10-22 16:57:23.822000000", + "trace_duration_ms": 4483.158, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:19.339000000", + "updated_at": "2025-10-22 16:57:19.339000000" + }, + { + "span_id": "24da2221-b0ab-4161-98ad-e7b5775fcb27", + "trace_id": "3bbd0b36-c30f-4126-b62a-dd0fae1c701d", + "parent_span_id": "b0f0a2e0-c1f7-4ba4-8b75-7090e931a738", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-22 16:57:19.339000000", + "end_time": "2025-10-22 16:57:19.339000000", + "duration_ms": 0.026, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: How does Noveum handle agent tracing?", + "query_length": 37, + "query_type": "routing_evaluation_query", + "classification.mode": "RAG", + "classification.rag_keyword_score": 2, + "classification.web_keyword_score": 0, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "noveum_specific", + "query.keyword_density": 0.3333333333333333, + "query.domain_affinity": "noveum", + "routing.decision": "Routed to RAG based on analysis", + "routing.rationale": "RAG score: 2, Web score: 0, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "Web Search", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.038461538461538464, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: RAG (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "3bbd0b36-c30f-4126-b62a-dd0fae1c701d", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:19.339000000", + "trace_end_time": "2025-10-22 16:57:23.822000000", + "trace_duration_ms": 4483.158, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:19.339000000", + "updated_at": "2025-10-22 16:57:19.339000000" + }, + { + "span_id": "d2b495eb-7e07-48a3-aa6d-1835aab18c11", + "trace_id": "3bbd0b36-c30f-4126-b62a-dd0fae1c701d", + "parent_span_id": "b0f0a2e0-c1f7-4ba4-8b75-7090e931a738", + "name": "agent.llm-rag", + "start_time": "2025-10-22 16:57:19.339000000", + "end_time": "2025-10-22 16:57:23.822000000", + "duration_ms": 4482.564, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "rag_agent", + "agent.operation": "llm-rag", + "agent.capabilities": [ + "document_retrieval", + "context_generation", + "response_generation" + ], + "agent.id": "noveum_rag_agent", + "input_query": "How does Noveum handle agent tracing?", + "query_length": 37, + "query_type": "rag_query", + "output_response": "RAG Answer: Noveum.ai handles agent tracing through its specialized Multi-Agent Tracing capabilities, which are designed to observe complex workflows and inter-agent communications within multi-agent systems. The...", + "rag.context_length": 2736, + "rag.sources_count": 5, + "rag.answer_length": 663, + "rag.mode": "retrieval_augmented_generation" + }, + "events": [], + "links": [], + "trace_trace_id": "3bbd0b36-c30f-4126-b62a-dd0fae1c701d", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:19.339000000", + "trace_end_time": "2025-10-22 16:57:23.822000000", + "trace_duration_ms": 4483.158, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:19.339000000", + "updated_at": "2025-10-22 16:57:19.339000000" + }, + { + "span_id": "4db80813-e62f-48ec-9091-62d569527caf", + "trace_id": "3bbd0b36-c30f-4126-b62a-dd0fae1c701d", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-22 16:57:19.339000000", + "end_time": "2025-10-22 16:57:23.822000000", + "duration_ms": 4483.031000000001, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "How does Noveum handle agent tracing?", + "query.length": 37, + "processing.mode": "RAG", + "processing.time_seconds": 4.482964038848877, + "processing.response_length": 663, + "processing.sources_count": 5, + "output_response": "Final Answer: Noveum.ai handles agent tracing through its specialized Multi-Agent Tracing capabilities, which are designed to observe complex workflows and inter-agent communications within multi-agent systems. The...", + "final_answer_mode": "RAG", + "query_processed.input_query": "How does Noveum handle agent tracing?", + "query_processed.output_response": "Successfully processed query using RAG, generated 663 character response", + "query_processed.mode": "RAG", + "query_processed.processing_time": 4.482964038848877, + "query_processed.response_length": 663 + }, + "events": [], + "links": [], + "trace_trace_id": "3bbd0b36-c30f-4126-b62a-dd0fae1c701d", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:19.339000000", + "trace_end_time": "2025-10-22 16:57:23.822000000", + "trace_duration_ms": 4483.158, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:19.339000000", + "updated_at": "2025-10-22 16:57:19.339000000" + }, + { + "span_id": "b80cb5cd-24f2-484d-bbff-9febe2aa3ffe", + "trace_id": "3bbd0b36-c30f-4126-b62a-dd0fae1c701d", + "parent_span_id": "d2b495eb-7e07-48a3-aa6d-1835aab18c11", + "name": "agent.llm_model_execution", + "start_time": "2025-10-22 16:57:19.958000000", + "end_time": "2025-10-22 16:57:22.709000000", + "duration_ms": 2750.9480000000003, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: How does Noveum handle agent tracing?", + "query_length": 37, + "query_type": "rag_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 2.750890016555786, + "model.latency_ms": 2750.890016555786, + "model.start_time": 1761152239.958973, + "model.end_time": 1761152242.709863, + "model.performance_tier": "medium", + "model.prompt_tokens": 817, + "model.completion_tokens": 165, + "model.total_tokens": 982, + "model.tokens_per_second": 356.975376728983, + "model.estimated_cost": 0.02946, + "model.efficiency_score": 0.675152749490835, + "model.response_length": 663, + "model.response_quality": "high", + "model.output_response": "Model Response: Noveum.ai handles agent tracing through its specialized Multi-Agent Tracing capabilities, which are designed to observe complex workflows and inter-agent communications within multi-agent systems. The...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "3bbd0b36-c30f-4126-b62a-dd0fae1c701d", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:19.339000000", + "trace_end_time": "2025-10-22 16:57:23.822000000", + "trace_duration_ms": 4483.158, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:19.339000000", + "updated_at": "2025-10-22 16:57:19.339000000" + }, + { + "span_id": "d85fcf7d-7eaa-4299-92fc-c969fab63a6e", + "trace_id": "3bbd0b36-c30f-4126-b62a-dd0fae1c701d", + "parent_span_id": "d2b495eb-7e07-48a3-aa6d-1835aab18c11", + "name": "agent.rag_evaluation_metrics", + "start_time": "2025-10-22 16:57:22.710000000", + "end_time": "2025-10-22 16:57:22.710000000", + "duration_ms": 0.036000000000000004, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "rag_evaluation_metrics", + "agent.capabilities": [ + "retrieval_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: How does Noveum handle agent tracing?", + "query_length": 37, + "query_type": "rag_evaluation_query", + "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, vector searches, data retrieval, agent reasoning, and more. Without proper observability, debugging ...", + "retrieval.context_length": 2736, + "retrieval.sources_count": 5, + "retrieval.context_quality": "high", + "retrieval.effectiveness": 1, + "retrieval.context_utilization": 2.736, + "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, vector searches, data retrieval, agent reasoning, and more. Without proper observability, debugging becomes a nightmare, optimization is guesswork, and understanding user interactions is nearly impossible.\nNoveum.ai solves this by providing comprehensive tracing and observability specifically designed for AI applications.\nKey benefits of using Noveum.ai:\n- Complete Visibility: Trace...\n\nSource 2 (https://noveum.ai/docs/advanced/multi-agent-tracing):\nMulti-Agent Tracing\nObserve complex agent workflows and inter-agent communications with comprehensive tracing\nMulti-agent systems represent some of the most complex AI applications, involving multiple agents that coordinate, communicate, and collaborate to achieve shared goals. Noveum.ai provides specialized tracing capabilities to help you understand and optimize these intricate workflows.\n\ud83c\udfaf Why Multi-Agent Tracing Matters\nMulti-agent systems introduce unique observability challenges:\n- Complex...\n\nSource 3 (https://noveum.ai/en/docs/advanced/multi-agent-tracing):\nMulti-Agent Tracing\nObserve complex agent workflows and inter-agent communications with comprehensive tracing\nMulti-agent systems represent some of the most complex AI applications, involving multiple agents that coordinate, communicate, and collaborate to achieve shared goals. Noveum.ai provides specialized tracing capabilities to help you understand and optimize these intricate workflows.\n\ud83c\udfaf Why Multi-Agent Tracing Matters\nMulti-agent systems introduce unique observability challenges:\n- Complex...\n\nSource 4 (https://noveum.ai/en/docs/advanced/performance-optimization):\n\ud83d\udd17 Next Steps\n- Multi-Agent Tracing - Observe agent workflows\n- RAG Pipeline Observability - Monitor retrieval and generation systems\n- Custom Instrumentation - Add domain-specific tracing\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nSource 5 (https://noveum.ai/docs/advanced/performance-optimization):\n\ud83d\udd17 Next Steps\n- Multi-Agent Tracing - Observe agent workflows\n- RAG Pipeline Observability - Monitor retrieval and generation systems\n- Custom Instrumentation - Add domain-specific tracing\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nUser Question: How does Noveum handle agent tracing?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", + "prompt.prompt_length": 3269, + "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, ...", + "prompt.instruction_following": "rag_optimized", + "response.answer_length": 663, + "response.answer_completeness": "complete", + "response.response_quality": "high", + "response.source_citation": 5, + "response.context_utilization": 0.24232456140350878, + "output_response": "RAG Answer: Noveum.ai handles agent tracing through its specialized Multi-Agent Tracing capabilities, which are designed to observe complex workflows and inter-agent communications within multi-agent systems. The...", + "evaluation.retrieval_effectiveness": 1, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 5, + "evaluation.context_relevance": "high", + "evaluation.overall_quality": "high", + "evaluation.ready_for_production": true, + "rag.retrieval_strategy": "semantic_similarity", + "rag.vector_search_results": 5, + "rag.context_synthesis": "multi_source", + "rag.document_coverage": 1, + "rag.information_density": 0.24232456140350878 + }, + "events": [], + "links": [], + "trace_trace_id": "3bbd0b36-c30f-4126-b62a-dd0fae1c701d", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:19.339000000", + "trace_end_time": "2025-10-22 16:57:23.822000000", + "trace_duration_ms": 4483.158, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:19.339000000", + "updated_at": "2025-10-22 16:57:19.339000000" + }, + { + "span_id": "93d82f68-6c62-40de-a675-c4d7c40c1d01", + "trace_id": "fee3e6d3-8da1-4ef3-8214-a7d7323d3a49", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-22 16:57:13.334000000", + "end_time": "2025-10-22 16:57:18.330000000", + "duration_ms": 4996.436, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What APIs are available in Noveum platform?", + "query.length": 43, + "processing.mode": "RAG", + "processing.time_seconds": 4.996389865875244, + "processing.response_length": 270, + "processing.sources_count": 5, + "output_response": "Final Answer: The provided context does not specify the APIs available in the Noveum platform. For detailed information about the APIs, I recommend checking the official Noveum documentation or the SDK Integration ...", + "final_answer_mode": "RAG", + "query_processed.input_query": "What APIs are available in Noveum platform?", + "query_processed.output_response": "Successfully processed query using RAG, generated 270 character response", + "query_processed.mode": "RAG", + "query_processed.processing_time": 4.996389865875244, + "query_processed.response_length": 270 + }, + "events": [], + "links": [], + "trace_trace_id": "fee3e6d3-8da1-4ef3-8214-a7d7323d3a49", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:13.334000000", + "trace_end_time": "2025-10-22 16:57:18.331000000", + "trace_duration_ms": 4996.502, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:13.334000000", + "updated_at": "2025-10-22 16:57:13.334000000" + }, + { + "span_id": "1be727c6-14e8-4b2e-b8df-a1cab902454f", + "trace_id": "fee3e6d3-8da1-4ef3-8214-a7d7323d3a49", + "parent_span_id": "93d82f68-6c62-40de-a675-c4d7c40c1d01", + "name": "agent.query_routing", + "start_time": "2025-10-22 16:57:13.334000000", + "end_time": "2025-10-22 16:57:18.330000000", + "duration_ms": 4996.354, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What APIs are available in Noveum platform?", + "query_length": 43, + "query_type": "routing_query", + "output_response": "Routed to RAG for query processing", + "router.classification": "RAG", + "router.confidence_score": 1, + "router.rag_keyword_score": 3, + "router.web_keyword_score": 0, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "fee3e6d3-8da1-4ef3-8214-a7d7323d3a49", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:13.334000000", + "trace_end_time": "2025-10-22 16:57:18.331000000", + "trace_duration_ms": 4996.502, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:13.334000000", + "updated_at": "2025-10-22 16:57:13.334000000" + }, + { + "span_id": "58ab1d39-d349-4164-8ff9-26ece4b1e1aa", + "trace_id": "fee3e6d3-8da1-4ef3-8214-a7d7323d3a49", + "parent_span_id": "1be727c6-14e8-4b2e-b8df-a1cab902454f", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-22 16:57:13.334000000", + "end_time": "2025-10-22 16:57:13.334000000", + "duration_ms": 0.041999999999999996, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What APIs are available in Noveum platform?", + "query_length": 43, + "query_type": "routing_evaluation_query", + "classification.mode": "RAG", + "classification.rag_keyword_score": 3, + "classification.web_keyword_score": 0, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "noveum_specific", + "query.keyword_density": 0.42857142857142855, + "query.domain_affinity": "noveum", + "routing.decision": "Routed to RAG based on analysis", + "routing.rationale": "RAG score: 3, Web score: 0, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "Web Search", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.057692307692307696, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: RAG (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "fee3e6d3-8da1-4ef3-8214-a7d7323d3a49", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:13.334000000", + "trace_end_time": "2025-10-22 16:57:18.331000000", + "trace_duration_ms": 4996.502, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:13.334000000", + "updated_at": "2025-10-22 16:57:13.334000000" + }, + { + "span_id": "f11ebf68-a416-484e-8e55-f77a11b9ea1c", + "trace_id": "fee3e6d3-8da1-4ef3-8214-a7d7323d3a49", + "parent_span_id": "1be727c6-14e8-4b2e-b8df-a1cab902454f", + "name": "agent.llm-rag", + "start_time": "2025-10-22 16:57:13.334000000", + "end_time": "2025-10-22 16:57:18.330000000", + "duration_ms": 4996.146000000001, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "rag_agent", + "agent.operation": "llm-rag", + "agent.capabilities": [ + "document_retrieval", + "context_generation", + "response_generation" + ], + "agent.id": "noveum_rag_agent", + "input_query": "What APIs are available in Noveum platform?", + "query_length": 43, + "query_type": "rag_query", + "output_response": "RAG Answer: The provided context does not specify the APIs available in the Noveum platform. For detailed information about the APIs, I recommend checking the official Noveum documentation or the SDK Integration ...", + "rag.context_length": 2284, + "rag.sources_count": 5, + "rag.answer_length": 270, + "rag.mode": "retrieval_augmented_generation" + }, + "events": [], + "links": [], + "trace_trace_id": "fee3e6d3-8da1-4ef3-8214-a7d7323d3a49", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:13.334000000", + "trace_end_time": "2025-10-22 16:57:18.331000000", + "trace_duration_ms": 4996.502, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:13.334000000", + "updated_at": "2025-10-22 16:57:13.334000000" + }, + { + "span_id": "11394488-8802-403d-a926-7fda3a510b61", + "trace_id": "fee3e6d3-8da1-4ef3-8214-a7d7323d3a49", + "parent_span_id": "f11ebf68-a416-484e-8e55-f77a11b9ea1c", + "name": "agent.llm_model_execution", + "start_time": "2025-10-22 16:57:13.862000000", + "end_time": "2025-10-22 16:57:16.049000000", + "duration_ms": 2187.29, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What APIs are available in Noveum platform?", + "query_length": 43, + "query_type": "rag_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 2.187239170074463, + "model.latency_ms": 2187.239170074463, + "model.start_time": 1761152233.862141, + "model.end_time": 1761152236.04938, + "model.performance_tier": "medium", + "model.prompt_tokens": 705, + "model.completion_tokens": 67, + "model.total_tokens": 772, + "model.tokens_per_second": 352.9563710098141, + "model.estimated_cost": 0.02316, + "model.efficiency_score": 0.34974093264248707, + "model.response_length": 270, + "model.response_quality": "high", + "model.output_response": "Model Response: The provided context does not specify the APIs available in the Noveum platform. For detailed information about the APIs, I recommend checking the official Noveum documentation or the SDK Integration ...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "fee3e6d3-8da1-4ef3-8214-a7d7323d3a49", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:13.334000000", + "trace_end_time": "2025-10-22 16:57:18.331000000", + "trace_duration_ms": 4996.502, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:13.334000000", + "updated_at": "2025-10-22 16:57:13.334000000" + }, + { + "span_id": "341cf8ab-cc33-4d57-952f-ba1065f0432e", + "trace_id": "fee3e6d3-8da1-4ef3-8214-a7d7323d3a49", + "parent_span_id": "f11ebf68-a416-484e-8e55-f77a11b9ea1c", + "name": "agent.rag_evaluation_metrics", + "start_time": "2025-10-22 16:57:16.049000000", + "end_time": "2025-10-22 16:57:16.049000000", + "duration_ms": 0.041, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "rag_evaluation_metrics", + "agent.capabilities": [ + "retrieval_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What APIs are available in Noveum platform?", + "query_length": 43, + "query_type": "rag_evaluation_query", + "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/docs):\n- \ud83c\udf0d Universal: Works across frameworks, clouds, and deployment models\n\ud83e\udd1d Community & Support\n- \ud83d\udcac Discord Community: Join our Discord\n- \ud83d\udce7 Email Support: [email protected]\n- \ud83d\udc1b Bug Reports: GitHub Issues\n- \ud83d\udcd6 Knowledge Base: Help Center\nReady to get started? Head to ...", + "retrieval.context_length": 2284, + "retrieval.sources_count": 5, + "retrieval.context_quality": "high", + "retrieval.effectiveness": 1, + "retrieval.context_utilization": 2.284, + "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/docs):\n- \ud83c\udf0d Universal: Works across frameworks, clouds, and deployment models\n\ud83e\udd1d Community & Support\n- \ud83d\udcac Discord Community: Join our Discord\n- \ud83d\udce7 Email Support: [email protected]\n- \ud83d\udc1b Bug Reports: GitHub Issues\n- \ud83d\udcd6 Knowledge Base: Help Center\nReady to get started? Head to our SDK Integration Guide to begin tracing your AI applications in under 5 minutes!\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability ...\n\nSource 2 (https://noveum.ai/docs):\n- \ud83c\udf0d Universal: Works across frameworks, clouds, and deployment models\n\ud83e\udd1d Community & Support\n- \ud83d\udcac Discord Community: Join our Discord\n- \ud83d\udce7 Email Support: [email protected]\n- \ud83d\udc1b Bug Reports: GitHub Issues\n- \ud83d\udcd6 Knowledge Base: Help Center\nReady to get started? Head to our SDK Integration Guide to begin tracing your AI applications in under 5 minutes!\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability ...\n\nSource 3 (https://noveum.ai/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nSource 4 (https://noveum.ai/en/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nSource 5 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, vector searches, data retrieval, agent reasoning, and more. Without proper observability, debugging becomes a nightmare, optimization is guesswork, and understanding user interactions is nearly impossible.\nNoveum.ai solves this by providing comprehensive tracing and observability specifically designed for AI applications.\nKey benefits of using Noveum.ai:\n- Complete Visibility: Trace...\n\nUser Question: What APIs are available in Noveum platform?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", + "prompt.prompt_length": 2823, + "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/docs):\n- \ud83c\udf0d Universal: Works across frameworks, clouds, and deployment models\n\ud83e\udd1d Community & Support\n- \ud83d\udcac Discord Community: Join our Discord\n- \ud83d\udce7 Email Support: [email prot...", + "prompt.instruction_following": "rag_optimized", + "response.answer_length": 270, + "response.answer_completeness": "complete", + "response.response_quality": "high", + "response.source_citation": 5, + "response.context_utilization": 0.11821366024518389, + "output_response": "RAG Answer: The provided context does not specify the APIs available in the Noveum platform. For detailed information about the APIs, I recommend checking the official Noveum documentation or the SDK Integration ...", + "evaluation.retrieval_effectiveness": 1, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 5, + "evaluation.context_relevance": "high", + "evaluation.overall_quality": "high", + "evaluation.ready_for_production": true, + "rag.retrieval_strategy": "semantic_similarity", + "rag.vector_search_results": 5, + "rag.context_synthesis": "multi_source", + "rag.document_coverage": 1, + "rag.information_density": 0.11821366024518389 + }, + "events": [], + "links": [], + "trace_trace_id": "fee3e6d3-8da1-4ef3-8214-a7d7323d3a49", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:13.334000000", + "trace_end_time": "2025-10-22 16:57:18.331000000", + "trace_duration_ms": 4996.502, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:13.334000000", + "updated_at": "2025-10-22 16:57:13.334000000" + }, + { + "span_id": "24930c28-7f93-426c-8914-bd0e0085589f", + "trace_id": "93bd37c4-4289-43c9-b34e-19eaaef610c2", + "parent_span_id": "bfc91518-f818-4f35-9113-70b04eb55edc", + "name": "agent.query_routing", + "start_time": "2025-10-22 16:57:06.605000000", + "end_time": "2025-10-22 16:57:12.327000000", + "duration_ms": 5721.561000000001, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "How do I set up observability with Noveum?", + "query_length": 42, + "query_type": "routing_query", + "output_response": "Routed to RAG for query processing", + "router.classification": "RAG", + "router.confidence_score": 1, + "router.rag_keyword_score": 2, + "router.web_keyword_score": 0, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "93bd37c4-4289-43c9-b34e-19eaaef610c2", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:06.605000000", + "trace_end_time": "2025-10-22 16:57:12.327000000", + "trace_duration_ms": 5721.707, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:06.605000000", + "updated_at": "2025-10-22 16:57:06.605000000" + }, + { + "span_id": "11fc18a8-0380-48ca-894e-cc26f648d290", + "trace_id": "93bd37c4-4289-43c9-b34e-19eaaef610c2", + "parent_span_id": "24930c28-7f93-426c-8914-bd0e0085589f", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-22 16:57:06.605000000", + "end_time": "2025-10-22 16:57:06.606000000", + "duration_ms": 0.047, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: How do I set up observability with Noveum?", + "query_length": 42, + "query_type": "routing_evaluation_query", + "classification.mode": "RAG", + "classification.rag_keyword_score": 2, + "classification.web_keyword_score": 0, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "noveum_specific", + "query.keyword_density": 0.25, + "query.domain_affinity": "noveum", + "routing.decision": "Routed to RAG based on analysis", + "routing.rationale": "RAG score: 2, Web score: 0, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "Web Search", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.038461538461538464, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: RAG (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "93bd37c4-4289-43c9-b34e-19eaaef610c2", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:06.605000000", + "trace_end_time": "2025-10-22 16:57:12.327000000", + "trace_duration_ms": 5721.707, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:06.605000000", + "updated_at": "2025-10-22 16:57:06.605000000" + }, + { + "span_id": "bfc91518-f818-4f35-9113-70b04eb55edc", + "trace_id": "93bd37c4-4289-43c9-b34e-19eaaef610c2", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-22 16:57:06.605000000", + "end_time": "2025-10-22 16:57:12.327000000", + "duration_ms": 5721.635, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "How do I set up observability with Noveum?", + "query.length": 42, + "processing.mode": "RAG", + "processing.time_seconds": 5.721601963043213, + "processing.response_length": 815, + "processing.sources_count": 5, + "output_response": "Final Answer: To set up observability with Noveum, follow these steps:\n\n1. **Sign Up**: Go to [noveum.ai](https://noveum.ai) and create an account.\n2. **Create a Project**: After signing up, create a project to obt...", + "final_answer_mode": "RAG", + "query_processed.input_query": "How do I set up observability with Noveum?", + "query_processed.output_response": "Successfully processed query using RAG, generated 815 character response", + "query_processed.mode": "RAG", + "query_processed.processing_time": 5.721601963043213, + "query_processed.response_length": 815 + }, + "events": [], + "links": [], + "trace_trace_id": "93bd37c4-4289-43c9-b34e-19eaaef610c2", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:06.605000000", + "trace_end_time": "2025-10-22 16:57:12.327000000", + "trace_duration_ms": 5721.707, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:06.605000000", + "updated_at": "2025-10-22 16:57:06.605000000" + }, + { + "span_id": "2a82e4b1-c93d-498f-9dd5-e971f228d642", + "trace_id": "93bd37c4-4289-43c9-b34e-19eaaef610c2", + "parent_span_id": "24930c28-7f93-426c-8914-bd0e0085589f", + "name": "agent.llm-rag", + "start_time": "2025-10-22 16:57:06.606000000", + "end_time": "2025-10-22 16:57:12.327000000", + "duration_ms": 5721.352000000001, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "rag_agent", + "agent.operation": "llm-rag", + "agent.capabilities": [ + "document_retrieval", + "context_generation", + "response_generation" + ], + "agent.id": "noveum_rag_agent", + "input_query": "How do I set up observability with Noveum?", + "query_length": 42, + "query_type": "rag_query", + "output_response": "RAG Answer: To set up observability with Noveum, follow these steps:\n\n1. **Sign Up**: Go to [noveum.ai](https://noveum.ai) and create an account.\n2. **Create a Project**: After signing up, create a project to obt...", + "rag.context_length": 2870, + "rag.sources_count": 6, + "rag.answer_length": 815, + "rag.mode": "retrieval_augmented_generation" + }, + "events": [], + "links": [], + "trace_trace_id": "93bd37c4-4289-43c9-b34e-19eaaef610c2", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:06.605000000", + "trace_end_time": "2025-10-22 16:57:12.327000000", + "trace_duration_ms": 5721.707, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:06.605000000", + "updated_at": "2025-10-22 16:57:06.605000000" + }, + { + "span_id": "520b4e59-da14-4db2-af02-0973cda9bd86", + "trace_id": "93bd37c4-4289-43c9-b34e-19eaaef610c2", + "parent_span_id": "2a82e4b1-c93d-498f-9dd5-e971f228d642", + "name": "agent.llm_model_execution", + "start_time": "2025-10-22 16:57:07.109000000", + "end_time": "2025-10-22 16:57:11.837000000", + "duration_ms": 4728.018999999999, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: How do I set up observability with Noveum?", + "query_length": 42, + "query_type": "rag_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 4.727963924407959, + "model.latency_ms": 4727.963924407959, + "model.start_time": 1761152227.109188, + "model.end_time": 1761152231.837152, + "model.performance_tier": "medium", + "model.prompt_tokens": 852, + "model.completion_tokens": 203, + "model.total_tokens": 1055, + "model.tokens_per_second": 223.1404504915101, + "model.estimated_cost": 0.03165, + "model.efficiency_score": 0.7725118483412322, + "model.response_length": 815, + "model.response_quality": "high", + "model.output_response": "Model Response: To set up observability with Noveum, follow these steps:\n\n1. **Sign Up**: Go to [noveum.ai](https://noveum.ai) and create an account.\n2. **Create a Project**: After signing up, create a project to obt...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "93bd37c4-4289-43c9-b34e-19eaaef610c2", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:06.605000000", + "trace_end_time": "2025-10-22 16:57:12.327000000", + "trace_duration_ms": 5721.707, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:06.605000000", + "updated_at": "2025-10-22 16:57:06.605000000" + }, + { + "span_id": "dff080ab-238c-4cca-a910-cf530f910bda", + "trace_id": "93bd37c4-4289-43c9-b34e-19eaaef610c2", + "parent_span_id": "2a82e4b1-c93d-498f-9dd5-e971f228d642", + "name": "agent.rag_evaluation_metrics", + "start_time": "2025-10-22 16:57:11.837000000", + "end_time": "2025-10-22 16:57:11.838000000", + "duration_ms": 0.259, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "rag_evaluation_metrics", + "agent.capabilities": [ + "retrieval_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: How do I set up observability with Noveum?", + "query_length": 42, + "query_type": "rag_evaluation_query", + "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\n- Production Ready: Built for scale with intelligent sampling and batching\n- Privacy Focused: You control what data is captured and how it's stored\n- Open Source SDKs: Transparent, auditable, and extensible\nGetting ...", + "retrieval.context_length": 2870, + "retrieval.sources_count": 6, + "retrieval.context_quality": "high", + "retrieval.effectiveness": 1.2, + "retrieval.context_utilization": 2.87, + "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\n- Production Ready: Built for scale with intelligent sampling and batching\n- Privacy Focused: You control what data is captured and how it's stored\n- Open Source SDKs: Transparent, auditable, and extensible\nGetting Started\nReady to add comprehensive observability to your AI applications?\n- Sign up at noveum.ai\n- Create a project and get your API key\n- Install the SDK for your preferred language\n- Add tracing to your AI workflows\n- Explore insights in the Noveum.ai dashboard\nCheck out our integra...\n\nSource 2 (https://noveum.ai/en/docs/getting-started/tracing-concepts):\n- Explore Framework Integrations - Framework-specific guidance\n- Learn Advanced Patterns - Custom instrumentation techniques\n- Master the Dashboard - Analyze your traces effectively\nRemember: Good observability is not about collecting all possible data, but about collecting the right data that helps you understand, debug, and optimize your AI applications.\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to O...\n\nSource 3 (https://noveum.ai/docs/getting-started/tracing-concepts):\n- Explore Framework Integrations - Framework-specific guidance\n- Learn Advanced Patterns - Custom instrumentation techniques\n- Master the Dashboard - Analyze your traces effectively\nRemember: Good observability is not about collecting all possible data, but about collecting the right data that helps you understand, debug, and optimize your AI applications.\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to O...\n\nSource 4 (https://noveum.ai/en/docs/getting-started/overview):\nNoveum.ai Overview\nComprehensive AI tracing and observability platform for LLM applications, RAG systems, and AI agents\nWelcome to Noveum.ai\u2014the comprehensive tracing and observability platform built specifically for AI applications. Whether you're building LLM-powered chatbots, RAG systems, multi-agent workflows, or any AI-driven application, Noveum provides the insights you need to understand, debug, and optimize your systems.\n\ud83c\udfaf Why AI Applications Need Specialized Observability\nTraditional mo...\n\nSource 5 (https://noveum.ai/docs/getting-started/overview):\nNoveum.ai Overview\nComprehensive AI tracing and observability platform for LLM applications, RAG systems, and AI agents\nWelcome to Noveum.ai\u2014the comprehensive tracing and observability platform built specifically for AI applications. Whether you're building LLM-powered chatbots, RAG systems, multi-agent workflows, or any AI-driven application, Noveum provides the insights you need to understand, debug, and optimize your systems.\n\ud83c\udfaf Why AI Applications Need Specialized Observability\nTraditional mo...\n\nUser Question: How do I set up observability with Noveum?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", + "prompt.prompt_length": 3408, + "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\n- Production Ready: Built for scale with intelligent sampling and batching\n- Privacy Focused: You control what data...", + "prompt.instruction_following": "rag_optimized", + "response.answer_length": 815, + "response.answer_completeness": "complete", + "response.response_quality": "high", + "response.source_citation": 6, + "response.context_utilization": 0.28397212543554007, + "output_response": "RAG Answer: To set up observability with Noveum, follow these steps:\n\n1. **Sign Up**: Go to [noveum.ai](https://noveum.ai) and create an account.\n2. **Create a Project**: After signing up, create a project to obt...", + "evaluation.retrieval_effectiveness": 1.2, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 6, + "evaluation.context_relevance": "high", + "evaluation.overall_quality": "high", + "evaluation.ready_for_production": true, + "rag.retrieval_strategy": "semantic_similarity", + "rag.vector_search_results": 6, + "rag.context_synthesis": "multi_source", + "rag.document_coverage": 1.2, + "rag.information_density": 0.28397212543554007 + }, + "events": [], + "links": [], + "trace_trace_id": "93bd37c4-4289-43c9-b34e-19eaaef610c2", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:06.605000000", + "trace_end_time": "2025-10-22 16:57:12.327000000", + "trace_duration_ms": 5721.707, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:06.605000000", + "updated_at": "2025-10-22 16:57:06.605000000" + }, + { + "span_id": "dad7df65-be49-4245-8721-c91da4962b9a", + "trace_id": "df0c6344-1a2e-4c53-bb7d-4e43b3bce1b8", + "parent_span_id": "9024b0d9-2b75-428f-af8f-75a921a7c69c", + "name": "agent.query_routing", + "start_time": "2025-10-22 16:56:59.213000000", + "end_time": "2025-10-22 16:57:05.598000000", + "duration_ms": 6384.249, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What features does Noveum Trace offer?", + "query_length": 38, + "query_type": "routing_query", + "output_response": "Routed to RAG for query processing", + "router.classification": "RAG", + "router.confidence_score": 1, + "router.rag_keyword_score": 3, + "router.web_keyword_score": 0, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "df0c6344-1a2e-4c53-bb7d-4e43b3bce1b8", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:56:59.213000000", + "trace_end_time": "2025-10-22 16:57:05.598000000", + "trace_duration_ms": 6384.386, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:56:59.213000000", + "updated_at": "2025-10-22 16:56:59.213000000" + }, + { + "span_id": "9024b0d9-2b75-428f-af8f-75a921a7c69c", + "trace_id": "df0c6344-1a2e-4c53-bb7d-4e43b3bce1b8", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-22 16:56:59.213000000", + "end_time": "2025-10-22 16:57:05.598000000", + "duration_ms": 6384.32, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What features does Noveum Trace offer?", + "query.length": 38, + "processing.mode": "RAG", + "processing.time_seconds": 6.384284973144531, + "processing.response_length": 1017, + "processing.sources_count": 5, + "output_response": "Final Answer: Noveum Trace offers several key features through its Python SDK (`noveum-trace`) and TypeScript SDK (`@noveum/trace`):\n\n1. **Decorator-based Tracing**: It provides a simple way to trace LLM calls, age...", + "final_answer_mode": "RAG", + "query_processed.input_query": "What features does Noveum Trace offer?", + "query_processed.output_response": "Successfully processed query using RAG, generated 1017 character response", + "query_processed.mode": "RAG", + "query_processed.processing_time": 6.384284973144531, + "query_processed.response_length": 1017 + }, + "events": [], + "links": [], + "trace_trace_id": "df0c6344-1a2e-4c53-bb7d-4e43b3bce1b8", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:56:59.213000000", + "trace_end_time": "2025-10-22 16:57:05.598000000", + "trace_duration_ms": 6384.386, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:56:59.213000000", + "updated_at": "2025-10-22 16:56:59.213000000" + }, + { + "span_id": "d07fdbdc-2b8f-4de9-ac32-988389701771", + "trace_id": "df0c6344-1a2e-4c53-bb7d-4e43b3bce1b8", + "parent_span_id": "dad7df65-be49-4245-8721-c91da4962b9a", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-22 16:56:59.214000000", + "end_time": "2025-10-22 16:56:59.214000000", + "duration_ms": 0.060000000000000005, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What features does Noveum Trace offer?", + "query_length": 38, + "query_type": "routing_evaluation_query", + "classification.mode": "RAG", + "classification.rag_keyword_score": 3, + "classification.web_keyword_score": 0, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "noveum_specific", + "query.keyword_density": 0.5, + "query.domain_affinity": "noveum", + "routing.decision": "Routed to RAG based on analysis", + "routing.rationale": "RAG score: 3, Web score: 0, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "Web Search", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.057692307692307696, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: RAG (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "df0c6344-1a2e-4c53-bb7d-4e43b3bce1b8", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:56:59.213000000", + "trace_end_time": "2025-10-22 16:57:05.598000000", + "trace_duration_ms": 6384.386, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:56:59.213000000", + "updated_at": "2025-10-22 16:56:59.213000000" + }, + { + "span_id": "83f86157-5af4-4c99-aad1-647866cd804e", + "trace_id": "df0c6344-1a2e-4c53-bb7d-4e43b3bce1b8", + "parent_span_id": "dad7df65-be49-4245-8721-c91da4962b9a", + "name": "agent.llm-rag", + "start_time": "2025-10-22 16:56:59.214000000", + "end_time": "2025-10-22 16:57:05.598000000", + "duration_ms": 6384.048, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "rag_agent", + "agent.operation": "llm-rag", + "agent.capabilities": [ + "document_retrieval", + "context_generation", + "response_generation" + ], + "agent.id": "noveum_rag_agent", + "input_query": "What features does Noveum Trace offer?", + "query_length": 38, + "query_type": "rag_query", + "output_response": "RAG Answer: Noveum Trace offers several key features through its Python SDK (`noveum-trace`) and TypeScript SDK (`@noveum/trace`):\n\n1. **Decorator-based Tracing**: It provides a simple way to trace LLM calls, age...", + "rag.context_length": 2787, + "rag.sources_count": 5, + "rag.answer_length": 1017, + "rag.mode": "retrieval_augmented_generation" + }, + "events": [], + "links": [], + "trace_trace_id": "df0c6344-1a2e-4c53-bb7d-4e43b3bce1b8", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:56:59.213000000", + "trace_end_time": "2025-10-22 16:57:05.598000000", + "trace_duration_ms": 6384.386, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:56:59.213000000", + "updated_at": "2025-10-22 16:56:59.213000000" + }, + { + "span_id": "3b4fa9a7-88bf-4ee1-aeb7-bfa9573f28f2", + "trace_id": "df0c6344-1a2e-4c53-bb7d-4e43b3bce1b8", + "parent_span_id": "83f86157-5af4-4c99-aad1-647866cd804e", + "name": "agent.llm_model_execution", + "start_time": "2025-10-22 16:56:59.591000000", + "end_time": "2025-10-22 16:57:04.869000000", + "duration_ms": 5278.925, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What features does Noveum Trace offer?", + "query_length": 38, + "query_type": "rag_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 5.278868913650513, + "model.latency_ms": 5278.868913650513, + "model.start_time": 1761152219.5910342, + "model.end_time": 1761152224.869903, + "model.performance_tier": "slow", + "model.prompt_tokens": 830, + "model.completion_tokens": 254, + "model.total_tokens": 1084, + "model.tokens_per_second": 205.3470199263535, + "model.estimated_cost": 0.03252, + "model.efficiency_score": 0.9381918819188192, + "model.response_length": 1017, + "model.response_quality": "high", + "model.output_response": "Model Response: Noveum Trace offers several key features through its Python SDK (`noveum-trace`) and TypeScript SDK (`@noveum/trace`):\n\n1. **Decorator-based Tracing**: It provides a simple way to trace LLM calls, age...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "df0c6344-1a2e-4c53-bb7d-4e43b3bce1b8", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:56:59.213000000", + "trace_end_time": "2025-10-22 16:57:05.598000000", + "trace_duration_ms": 6384.386, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:56:59.213000000", + "updated_at": "2025-10-22 16:56:59.213000000" + }, + { + "span_id": "f9214dbe-adb9-44ff-8361-70c872cef66d", + "trace_id": "df0c6344-1a2e-4c53-bb7d-4e43b3bce1b8", + "parent_span_id": "83f86157-5af4-4c99-aad1-647866cd804e", + "name": "agent.rag_evaluation_metrics", + "start_time": "2025-10-22 16:57:04.870000000", + "end_time": "2025-10-22 16:57:04.870000000", + "duration_ms": 0.043000000000000003, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "rag_evaluation_metrics", + "agent.capabilities": [ + "retrieval_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What features does Noveum Trace offer?", + "query_length": 38, + "query_type": "rag_evaluation_query", + "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\n\ud83d\udcd8 TypeScript SDK (\n@noveum/trace\n)- Framework ...", + "retrieval.context_length": 2787, + "retrieval.sources_count": 5, + "retrieval.context_quality": "high", + "retrieval.effectiveness": 1, + "retrieval.context_utilization": 2.787, + "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\n\ud83d\udcd8 TypeScript SDK (\n@noveum/trace\n)- Framework integrations for Next.js, Express.js, and Hono\n- TypeScript-first design with full type safety\n- Universal compatibility (Node.js, Edge Runtime, browsers)\n-\n\ud83d\udcca Noveum Platform\n- Real-time dashboard for analyzing traces and performance\n- Ad...\n\nSource 2 (https://noveum.ai/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\n\ud83d\udcd8 TypeScript SDK (\n@noveum/trace\n)- Framework integrations for Next.js, Express.js, and Hono\n- TypeScript-first design with full type safety\n- Universal compatibility (Node.js, Edge Runtime, browsers)\n-\n\ud83d\udcca Noveum Platform\n- Real-time dashboard for analyzing traces and performance\n- Ad...\n\nSource 3 (https://noveum.ai/en/docs/getting-started/sdk-integration):\n- \ud83d\udc65 Collaborate with your team on insights\n\ud83d\udd12 Security & Privacy\n- \ud83d\udd10 Encryption: All data encrypted in transit and at rest\n- \ud83c\udf9b\ufe0f Configurable Capture: Control what data is collected\n- \ud83c\udfe0 Data Residency: Choose your data storage region\n- \u23f0 Retention Control: Set custom data retention policies\nNext Steps\n- Tracing Concepts - Learn about traces, spans, and observability best practices\n- Framework Integrations - Deep dive into specific framework setups\n- Multi-Agent Tracing - Observe complex agent work...\n\nSource 4 (https://noveum.ai/docs/getting-started/sdk-integration):\n- \ud83d\udc65 Collaborate with your team on insights\n\ud83d\udd12 Security & Privacy\n- \ud83d\udd10 Encryption: All data encrypted in transit and at rest\n- \ud83c\udf9b\ufe0f Configurable Capture: Control what data is collected\n- \ud83c\udfe0 Data Residency: Choose your data storage region\n- \u23f0 Retention Control: Set custom data retention policies\nNext Steps\n- Tracing Concepts - Learn about traces, spans, and observability best practices\n- Framework Integrations - Deep dive into specific framework setups\n- Multi-Agent Tracing - Observe complex agent work...\n\nSource 5 (https://noveum.ai/docs/platform/dashboard):\n- Bulk Export: Download multiple traces simultaneously\n- Comparative Analysis: Compare performance across multiple traces\nReal-Time Updates\n- Live Refresh: Automatic updates as new traces arrive\n- Connection Monitoring: Real-time status of your trace ingestion pipeline\n- Performance Indicators: Live metrics for system health monitoring\nReady to dive deeper? Explore Projects & Environments to organize your AI applications, or check out Team Collaboration to share insights with your team.\nGet Earl...\n\nUser Question: What features does Noveum Trace offer?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", + "prompt.prompt_length": 3321, + "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frame...", + "prompt.instruction_following": "rag_optimized", + "response.answer_length": 1017, + "response.answer_completeness": "complete", + "response.response_quality": "high", + "response.source_citation": 5, + "response.context_utilization": 0.36490850376749195, + "output_response": "RAG Answer: Noveum Trace offers several key features through its Python SDK (`noveum-trace`) and TypeScript SDK (`@noveum/trace`):\n\n1. **Decorator-based Tracing**: It provides a simple way to trace LLM calls, age...", + "evaluation.retrieval_effectiveness": 1, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 5, + "evaluation.context_relevance": "high", + "evaluation.overall_quality": "high", + "evaluation.ready_for_production": true, + "rag.retrieval_strategy": "semantic_similarity", + "rag.vector_search_results": 5, + "rag.context_synthesis": "multi_source", + "rag.document_coverage": 1, + "rag.information_density": 0.36490850376749195 + }, + "events": [], + "links": [], + "trace_trace_id": "df0c6344-1a2e-4c53-bb7d-4e43b3bce1b8", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:56:59.213000000", + "trace_end_time": "2025-10-22 16:57:05.598000000", + "trace_duration_ms": 6384.386, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:56:59.213000000", + "updated_at": "2025-10-22 16:56:59.213000000" + }, + { + "span_id": "138a5089-5268-447a-afb5-4f14fb504281", + "trace_id": "29a862bf-d7aa-4cf9-963c-0a4244343bbe", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-22 16:56:54.719000000", + "end_time": "2025-10-22 16:56:58.207000000", + "duration_ms": 3487.959, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What are Noveum's pricing plans?", + "query.length": 32, + "processing.mode": "RAG", + "processing.time_seconds": 3.487928867340088, + "processing.response_length": 441, + "processing.sources_count": 5, + "output_response": "Final Answer: The provided context does not include specific information about Noveum's pricing plans. It mentions that all users will have access to the Observability suite for free, and early users will receive f...", + "final_answer_mode": "RAG", + "query_processed.input_query": "What are Noveum's pricing plans?", + "query_processed.output_response": "Successfully processed query using RAG, generated 441 character response", + "query_processed.mode": "RAG", + "query_processed.processing_time": 3.487928867340088, + "query_processed.response_length": 441 + }, + "events": [], + "links": [], + "trace_trace_id": "29a862bf-d7aa-4cf9-963c-0a4244343bbe", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:56:54.718000000", + "trace_end_time": "2025-10-22 16:56:58.207000000", + "trace_duration_ms": 3488.068, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:56:54.718000000", + "updated_at": "2025-10-22 16:56:54.718000000" + }, + { + "span_id": "7cda4dab-0fc5-4cae-8d8c-9445f661c67b", + "trace_id": "29a862bf-d7aa-4cf9-963c-0a4244343bbe", + "parent_span_id": "138a5089-5268-447a-afb5-4f14fb504281", + "name": "agent.query_routing", + "start_time": "2025-10-22 16:56:54.719000000", + "end_time": "2025-10-22 16:56:58.207000000", + "duration_ms": 3487.848, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What are Noveum's pricing plans?", + "query_length": 32, + "query_type": "routing_query", + "output_response": "Routed to RAG for query processing", + "router.classification": "RAG", + "router.confidence_score": 1, + "router.rag_keyword_score": 3, + "router.web_keyword_score": 0, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "29a862bf-d7aa-4cf9-963c-0a4244343bbe", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:56:54.718000000", + "trace_end_time": "2025-10-22 16:56:58.207000000", + "trace_duration_ms": 3488.068, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:56:54.718000000", + "updated_at": "2025-10-22 16:56:54.718000000" + }, + { + "span_id": "922c84a5-b14a-4b2f-8759-427d6e089812", + "trace_id": "29a862bf-d7aa-4cf9-963c-0a4244343bbe", + "parent_span_id": "7cda4dab-0fc5-4cae-8d8c-9445f661c67b", + "name": "agent.llm-rag", + "start_time": "2025-10-22 16:56:54.719000000", + "end_time": "2025-10-22 16:56:58.207000000", + "duration_ms": 3487.546, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "rag_agent", + "agent.operation": "llm-rag", + "agent.capabilities": [ + "document_retrieval", + "context_generation", + "response_generation" + ], + "agent.id": "noveum_rag_agent", + "input_query": "What are Noveum's pricing plans?", + "query_length": 32, + "query_type": "rag_query", + "output_response": "RAG Answer: The provided context does not include specific information about Noveum's pricing plans. It mentions that all users will have access to the Observability suite for free, and early users will receive f...", + "rag.context_length": 2105, + "rag.sources_count": 5, + "rag.answer_length": 441, + "rag.mode": "retrieval_augmented_generation" + }, + "events": [], + "links": [], + "trace_trace_id": "29a862bf-d7aa-4cf9-963c-0a4244343bbe", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:56:54.718000000", + "trace_end_time": "2025-10-22 16:56:58.207000000", + "trace_duration_ms": 3488.068, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:56:54.718000000", + "updated_at": "2025-10-22 16:56:54.718000000" + }, + { + "span_id": "c8d7bb28-e763-47c9-b3e9-77f6a91a22be", + "trace_id": "29a862bf-d7aa-4cf9-963c-0a4244343bbe", + "parent_span_id": "7cda4dab-0fc5-4cae-8d8c-9445f661c67b", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-22 16:56:54.719000000", + "end_time": "2025-10-22 16:56:54.719000000", + "duration_ms": 0.068, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What are Noveum's pricing plans?", + "query_length": 32, + "query_type": "routing_evaluation_query", + "classification.mode": "RAG", + "classification.rag_keyword_score": 3, + "classification.web_keyword_score": 0, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "noveum_specific", + "query.keyword_density": 0.6, + "query.domain_affinity": "noveum", + "routing.decision": "Routed to RAG based on analysis", + "routing.rationale": "RAG score: 3, Web score: 0, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "Web Search", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.057692307692307696, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: RAG (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "29a862bf-d7aa-4cf9-963c-0a4244343bbe", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:56:54.718000000", + "trace_end_time": "2025-10-22 16:56:58.207000000", + "trace_duration_ms": 3488.068, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:56:54.718000000", + "updated_at": "2025-10-22 16:56:54.718000000" + }, + { + "span_id": "a378123a-8a79-4bc5-8e20-8ec54c2f9783", + "trace_id": "29a862bf-d7aa-4cf9-963c-0a4244343bbe", + "parent_span_id": "922c84a5-b14a-4b2f-8759-427d6e089812", + "name": "agent.llm_model_execution", + "start_time": "2025-10-22 16:56:55.154000000", + "end_time": "2025-10-22 16:56:57.166000000", + "duration_ms": 2012.174, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What are Noveum's pricing plans?", + "query_length": 32, + "query_type": "rag_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 2.012089252471924, + "model.latency_ms": 2012.0892524719238, + "model.start_time": 1761152215.154625, + "model.end_time": 1761152217.1667142, + "model.performance_tier": "medium", + "model.prompt_tokens": 658, + "model.completion_tokens": 110, + "model.total_tokens": 768, + "model.tokens_per_second": 381.6928096288395, + "model.estimated_cost": 0.02304, + "model.efficiency_score": 0.57421875, + "model.response_length": 441, + "model.response_quality": "high", + "model.output_response": "Model Response: The provided context does not include specific information about Noveum's pricing plans. It mentions that all users will have access to the Observability suite for free, and early users will receive f...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "29a862bf-d7aa-4cf9-963c-0a4244343bbe", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:56:54.718000000", + "trace_end_time": "2025-10-22 16:56:58.207000000", + "trace_duration_ms": 3488.068, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:56:54.718000000", + "updated_at": "2025-10-22 16:56:54.718000000" + }, + { + "span_id": "5813673b-a1dc-46c7-9bea-92ce6aca2b5e", + "trace_id": "29a862bf-d7aa-4cf9-963c-0a4244343bbe", + "parent_span_id": "922c84a5-b14a-4b2f-8759-427d6e089812", + "name": "agent.rag_evaluation_metrics", + "start_time": "2025-10-22 16:56:57.166000000", + "end_time": "2025-10-22 16:56:57.167000000", + "duration_ms": 0.059, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "rag_evaluation_metrics", + "agent.capabilities": [ + "retrieval_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What are Noveum's pricing plans?", + "query_length": 32, + "query_type": "rag_evaluation_query", + "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....", + "retrieval.context_length": 2105, + "retrieval.sources_count": 5, + "retrieval.context_quality": "high", + "retrieval.effectiveness": 1, + "retrieval.context_utilization": 2.105, + "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nSource 2 (https://noveum.ai/en/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nSource 3 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nThe AI landscape is complex and moving fast. With Noveum.ai, you'll have the insights you need to build, debug, and optimize AI applications with confidence.\nReady to see what's really happening inside your AI applications? Try Noveum.ai today and transform how you build and monitor AI systems.\nLet's build more reliable, observable AI\u2014together.\nGet Early Access to Noveum.ai Platform\nJoin the select group of AI teams optimizing their models with our data-driven platform. We're onboarding users in...\n\nSource 4 (https://noveum.ai/en/blog/from-logs-to-intelligent-choices-inside-noveum-ais-evaluation-process):\nGet Early Access to Noveum.ai Platform\nJoin the select group of AI teams optimizing their models with our data-driven platform. We're onboarding users in limited batches to ensure a premium experience....\n\nSource 5 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, vector searches, data retrieval, agent reasoning, and more. Without proper observability, debugging becomes a nightmare, optimization is guesswork, and understanding user interactions is nearly impossible.\nNoveum.ai solves this by providing comprehensive tracing and observability specifically designed for AI applications.\nKey benefits of using Noveum.ai:\n- Complete Visibility: Trace...\n\nUser Question: What are Noveum's pricing plans?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", + "prompt.prompt_length": 2633, + "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to ...", + "prompt.instruction_following": "rag_optimized", + "response.answer_length": 441, + "response.answer_completeness": "complete", + "response.response_quality": "high", + "response.source_citation": 5, + "response.context_utilization": 0.20950118764845604, + "output_response": "RAG Answer: The provided context does not include specific information about Noveum's pricing plans. It mentions that all users will have access to the Observability suite for free, and early users will receive f...", + "evaluation.retrieval_effectiveness": 1, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 5, + "evaluation.context_relevance": "high", + "evaluation.overall_quality": "high", + "evaluation.ready_for_production": true, + "rag.retrieval_strategy": "semantic_similarity", + "rag.vector_search_results": 5, + "rag.context_synthesis": "multi_source", + "rag.document_coverage": 1, + "rag.information_density": 0.20950118764845604 + }, + "events": [], + "links": [], + "trace_trace_id": "29a862bf-d7aa-4cf9-963c-0a4244343bbe", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:56:54.718000000", + "trace_end_time": "2025-10-22 16:56:58.207000000", + "trace_duration_ms": 3488.068, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:56:54.718000000", + "updated_at": "2025-10-22 16:56:54.718000000" + }, + { + "span_id": "c5baf5c2-d18a-458a-b922-a19b96db53d9", + "trace_id": "bc60c122-f30b-4114-9f78-ee40a024b525", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-22 16:56:44.873000000", + "end_time": "2025-10-22 16:56:53.701000000", + "duration_ms": 8827.979, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "How do I integrate Noveum Trace in my application?", + "query.length": 50, + "processing.mode": "RAG", + "processing.time_seconds": 8.827935218811035, + "processing.response_length": 1145, + "processing.sources_count": 5, + "output_response": "Final Answer: To integrate Noveum Trace into your application, you can follow these steps based on the SDK Integration Guide:\n\n1. **Create Your Account & Get API Key**:\n - Sign up at [noveum.ai](https://noveum.ai...", + "final_answer_mode": "RAG", + "query_processed.input_query": "How do I integrate Noveum Trace in my application?", + "query_processed.output_response": "Successfully processed query using RAG, generated 1145 character response", + "query_processed.mode": "RAG", + "query_processed.processing_time": 8.827935218811035, + "query_processed.response_length": 1145 + }, + "events": [], + "links": [], + "trace_trace_id": "bc60c122-f30b-4114-9f78-ee40a024b525", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:56:44.873000000", + "trace_end_time": "2025-10-22 16:56:53.701000000", + "trace_duration_ms": 8828.055, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:56:44.873000000", + "updated_at": "2025-10-22 16:56:44.873000000" + }, + { + "span_id": "8819dde5-9cbe-4cdb-b919-9c5140582359", + "trace_id": "bc60c122-f30b-4114-9f78-ee40a024b525", + "parent_span_id": "c5baf5c2-d18a-458a-b922-a19b96db53d9", + "name": "agent.query_routing", + "start_time": "2025-10-22 16:56:44.873000000", + "end_time": "2025-10-22 16:56:53.701000000", + "duration_ms": 8827.883, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "How do I integrate Noveum Trace in my application?", + "query_length": 50, + "query_type": "routing_query", + "output_response": "Routed to RAG for query processing", + "router.classification": "RAG", + "router.confidence_score": 1, + "router.rag_keyword_score": 2, + "router.web_keyword_score": 0, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "bc60c122-f30b-4114-9f78-ee40a024b525", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:56:44.873000000", + "trace_end_time": "2025-10-22 16:56:53.701000000", + "trace_duration_ms": 8828.055, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:56:44.873000000", + "updated_at": "2025-10-22 16:56:44.873000000" + }, + { + "span_id": "932c4b0e-0f33-40c0-bb19-78b9f81e48a6", + "trace_id": "bc60c122-f30b-4114-9f78-ee40a024b525", + "parent_span_id": "8819dde5-9cbe-4cdb-b919-9c5140582359", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-22 16:56:44.873000000", + "end_time": "2025-10-22 16:56:44.873000000", + "duration_ms": 0.043000000000000003, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: How do I integrate Noveum Trace in my application?", + "query_length": 50, + "query_type": "routing_evaluation_query", + "classification.mode": "RAG", + "classification.rag_keyword_score": 2, + "classification.web_keyword_score": 0, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "noveum_specific", + "query.keyword_density": 0.2222222222222222, + "query.domain_affinity": "noveum", + "routing.decision": "Routed to RAG based on analysis", + "routing.rationale": "RAG score: 2, Web score: 0, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "Web Search", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.038461538461538464, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: RAG (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "bc60c122-f30b-4114-9f78-ee40a024b525", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:56:44.873000000", + "trace_end_time": "2025-10-22 16:56:53.701000000", + "trace_duration_ms": 8828.055, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:56:44.873000000", + "updated_at": "2025-10-22 16:56:44.873000000" + }, + { + "span_id": "2ece2bd6-e4c8-47bc-b3c8-26f82e9910f1", + "trace_id": "bc60c122-f30b-4114-9f78-ee40a024b525", + "parent_span_id": "8819dde5-9cbe-4cdb-b919-9c5140582359", + "name": "agent.llm-rag", + "start_time": "2025-10-22 16:56:44.873000000", + "end_time": "2025-10-22 16:56:53.701000000", + "duration_ms": 8827.666000000001, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "rag_agent", + "agent.operation": "llm-rag", + "agent.capabilities": [ + "document_retrieval", + "context_generation", + "response_generation" + ], + "agent.id": "noveum_rag_agent", + "input_query": "How do I integrate Noveum Trace in my application?", + "query_length": 50, + "query_type": "rag_query", + "output_response": "RAG Answer: To integrate Noveum Trace into your application, you can follow these steps based on the SDK Integration Guide:\n\n1. **Create Your Account & Get API Key**:\n - Sign up at [noveum.ai](https://noveum.ai...", + "rag.context_length": 2803, + "rag.sources_count": 5, + "rag.answer_length": 1145, + "rag.mode": "retrieval_augmented_generation" + }, + "events": [], + "links": [], + "trace_trace_id": "bc60c122-f30b-4114-9f78-ee40a024b525", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:56:44.873000000", + "trace_end_time": "2025-10-22 16:56:53.701000000", + "trace_duration_ms": 8828.055, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:56:44.873000000", + "updated_at": "2025-10-22 16:56:44.873000000" + }, + { + "span_id": "e6315110-36b1-4503-beaf-12d972e25a7d", + "trace_id": "bc60c122-f30b-4114-9f78-ee40a024b525", + "parent_span_id": "2ece2bd6-e4c8-47bc-b3c8-26f82e9910f1", + "name": "agent.llm_model_execution", + "start_time": "2025-10-22 16:56:45.576000000", + "end_time": "2025-10-22 16:56:53.143000000", + "duration_ms": 7566.311, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: How do I integrate Noveum Trace in my application?", + "query_length": 50, + "query_type": "rag_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 7.566190004348755, + "model.latency_ms": 7566.190004348755, + "model.start_time": 1761152205.576988, + "model.end_time": 1761152213.143178, + "model.performance_tier": "slow", + "model.prompt_tokens": 837, + "model.completion_tokens": 286, + "model.total_tokens": 1123, + "model.tokens_per_second": 148.4234468543009, + "model.estimated_cost": 0.03369, + "model.efficiency_score": 1.0195903829029385, + "model.response_length": 1145, + "model.response_quality": "high", + "model.output_response": "Model Response: To integrate Noveum Trace into your application, you can follow these steps based on the SDK Integration Guide:\n\n1. **Create Your Account & Get API Key**:\n - Sign up at [noveum.ai](https://noveum.ai...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "bc60c122-f30b-4114-9f78-ee40a024b525", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:56:44.873000000", + "trace_end_time": "2025-10-22 16:56:53.701000000", + "trace_duration_ms": 8828.055, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:56:44.873000000", + "updated_at": "2025-10-22 16:56:44.873000000" + }, + { + "span_id": "ae91d520-8442-48e5-aabf-fd58aed23694", + "trace_id": "bc60c122-f30b-4114-9f78-ee40a024b525", + "parent_span_id": "2ece2bd6-e4c8-47bc-b3c8-26f82e9910f1", + "name": "agent.rag_evaluation_metrics", + "start_time": "2025-10-22 16:56:53.143000000", + "end_time": "2025-10-22 16:56:53.143000000", + "duration_ms": 0.091, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "rag_evaluation_metrics", + "agent.capabilities": [ + "retrieval_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: How do I integrate Noveum Trace in my application?", + "query_length": 50, + "query_type": "rag_evaluation_query", + "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\n\ud83d\udcd8 TypeScript SDK (\n@noveum/trace\n)- Framework ...", + "retrieval.context_length": 2803, + "retrieval.sources_count": 5, + "retrieval.context_quality": "high", + "retrieval.effectiveness": 1, + "retrieval.context_utilization": 2.803, + "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\n\ud83d\udcd8 TypeScript SDK (\n@noveum/trace\n)- Framework integrations for Next.js, Express.js, and Hono\n- TypeScript-first design with full type safety\n- Universal compatibility (Node.js, Edge Runtime, browsers)\n-\n\ud83d\udcca Noveum Platform\n- Real-time dashboard for analyzing traces and performance\n- Ad...\n\nSource 2 (https://noveum.ai/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\n\ud83d\udcd8 TypeScript SDK (\n@noveum/trace\n)- Framework integrations for Next.js, Express.js, and Hono\n- TypeScript-first design with full type safety\n- Universal compatibility (Node.js, Edge Runtime, browsers)\n-\n\ud83d\udcca Noveum Platform\n- Real-time dashboard for analyzing traces and performance\n- Ad...\n\nSource 3 (https://noveum.ai/docs/getting-started/sdk-integration):\nSDK Integration Guide\nIntegrate Noveum.ai tracing into your AI applications with Python or TypeScript SDKs\nThe Noveum.ai SDKs provide comprehensive tracing and observability for your AI applications with minimal code changes. Whether you're building LLM applications, RAG systems, or multi-agent workflows, our SDKs automatically capture essential metrics and traces.\n\ud83d\ude80 Quick Start\n1. Create Your Account & Get API Key\n- Sign up at noveum.ai\n- Create a project in your dashboard\n- Generate an API key...\n\nSource 4 (https://noveum.ai/en/docs/getting-started/sdk-integration):\nSDK Integration Guide\nIntegrate Noveum.ai tracing into your AI applications with Python or TypeScript SDKs\nThe Noveum.ai SDKs provide comprehensive tracing and observability for your AI applications with minimal code changes. Whether you're building LLM applications, RAG systems, or multi-agent workflows, our SDKs automatically capture essential metrics and traces.\n\ud83d\ude80 Quick Start\n1. Create Your Account & Get API Key\n- Sign up at noveum.ai\n- Create a project in your dashboard\n- Generate an API key...\n\nSource 5 (https://noveum.ai/en/docs/getting-started/sdk-integration):\n\ud83d\udd27 Framework Integrations\nNext.js Integration\nExpress.js Integration\nFastAPI Integration (Python)\n\ud83d\udcca Advanced Features\nCustom Attributes & Events\nSampling Configuration\n\ud83d\udd17 What's Captured Automatically\n- \ud83d\udcca Performance Metrics: Latency, throughput, error rates\n- \ud83d\udcb0 Cost Tracking: Token usage, API costs across providers\n- \ud83d\udd0d Request/Response: Configurable capture of inputs/outputs\n- \ud83c\udff7\ufe0f Metadata: Model names, parameters, user context\n- \ud83c\udf0a Context Flow: Trace relationships across services\n- \ud83d\udc1b Error Detail...\n\nUser Question: How do I integrate Noveum Trace in my application?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", + "prompt.prompt_length": 3349, + "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frame...", + "prompt.instruction_following": "rag_optimized", + "response.answer_length": 1145, + "response.answer_completeness": "complete", + "response.response_quality": "high", + "response.source_citation": 5, + "response.context_utilization": 0.4084909026043525, + "output_response": "RAG Answer: To integrate Noveum Trace into your application, you can follow these steps based on the SDK Integration Guide:\n\n1. **Create Your Account & Get API Key**:\n - Sign up at [noveum.ai](https://noveum.ai...", + "evaluation.retrieval_effectiveness": 1, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 5, + "evaluation.context_relevance": "high", + "evaluation.overall_quality": "high", + "evaluation.ready_for_production": true, + "rag.retrieval_strategy": "semantic_similarity", + "rag.vector_search_results": 5, + "rag.context_synthesis": "multi_source", + "rag.document_coverage": 1, + "rag.information_density": 0.4084909026043525 + }, + "events": [], + "links": [], + "trace_trace_id": "bc60c122-f30b-4114-9f78-ee40a024b525", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:56:44.873000000", + "trace_end_time": "2025-10-22 16:56:53.701000000", + "trace_duration_ms": 8828.055, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:56:44.873000000", + "updated_at": "2025-10-22 16:56:44.873000000" + }, + { + "span_id": "9a9c611d-f552-4b06-a764-4463edb70e2c", + "trace_id": "3ad192e5-bbd4-4236-8144-019d53512769", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-22 16:56:35.188000000", + "end_time": "2025-10-22 16:56:43.863000000", + "duration_ms": 8675.25, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What is Noveum and what does it do?", + "query.length": 35, + "processing.mode": "RAG", + "processing.time_seconds": 8.67501425743103, + "processing.response_length": 527, + "processing.sources_count": 5, + "output_response": "Final Answer: Noveum.ai is a comprehensive tracing and observability platform specifically designed for AI applications, including LLM-powered chatbots, RAG systems, and multi-agent workflows. It provides the neces...", + "final_answer_mode": "RAG", + "query_processed.input_query": "What is Noveum and what does it do?", + "query_processed.output_response": "Successfully processed query using RAG, generated 527 character response", + "query_processed.mode": "RAG", + "query_processed.processing_time": 8.67501425743103, + "query_processed.response_length": 527 + }, + "events": [], + "links": [], + "trace_trace_id": "3ad192e5-bbd4-4236-8144-019d53512769", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:56:35.188000000", + "trace_end_time": "2025-10-22 16:56:43.863000000", + "trace_duration_ms": 8675.356000000002, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:56:35.188000000", + "updated_at": "2025-10-22 16:56:35.188000000" + }, + { + "span_id": "99eadeae-0733-4bf3-982e-e9f3014f5913", + "trace_id": "3ad192e5-bbd4-4236-8144-019d53512769", + "parent_span_id": "7e2c959c-157a-43d5-9870-32994cddd388", + "name": "agent.llm-rag", + "start_time": "2025-10-22 16:56:35.188000000", + "end_time": "2025-10-22 16:56:43.863000000", + "duration_ms": 8674.699999999999, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "rag_agent", + "agent.operation": "llm-rag", + "agent.capabilities": [ + "document_retrieval", + "context_generation", + "response_generation" + ], + "agent.id": "noveum_rag_agent", + "input_query": "What is Noveum and what does it do?", + "query_length": 35, + "query_type": "rag_query", + "output_response": "RAG Answer: Noveum.ai is a comprehensive tracing and observability platform specifically designed for AI applications, including LLM-powered chatbots, RAG systems, and multi-agent workflows. It provides the neces...", + "rag.context_length": 2334, + "rag.sources_count": 5, + "rag.answer_length": 527, + "rag.mode": "retrieval_augmented_generation" + }, + "events": [], + "links": [], + "trace_trace_id": "3ad192e5-bbd4-4236-8144-019d53512769", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:56:35.188000000", + "trace_end_time": "2025-10-22 16:56:43.863000000", + "trace_duration_ms": 8675.356000000002, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:56:35.188000000", + "updated_at": "2025-10-22 16:56:35.188000000" + }, + { + "span_id": "7e2c959c-157a-43d5-9870-32994cddd388", + "trace_id": "3ad192e5-bbd4-4236-8144-019d53512769", + "parent_span_id": "9a9c611d-f552-4b06-a764-4463edb70e2c", + "name": "agent.query_routing", + "start_time": "2025-10-22 16:56:35.188000000", + "end_time": "2025-10-22 16:56:43.863000000", + "duration_ms": 8674.961, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What is Noveum and what does it do?", + "query_length": 35, + "query_type": "routing_query", + "output_response": "Routed to RAG for query processing", + "router.classification": "RAG", + "router.confidence_score": 1, + "router.rag_keyword_score": 1, + "router.web_keyword_score": 0, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "3ad192e5-bbd4-4236-8144-019d53512769", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:56:35.188000000", + "trace_end_time": "2025-10-22 16:56:43.863000000", + "trace_duration_ms": 8675.356000000002, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:56:35.188000000", + "updated_at": "2025-10-22 16:56:35.188000000" + }, + { + "span_id": "6f81284c-a9aa-4d62-a402-1b474f2eba51", + "trace_id": "3ad192e5-bbd4-4236-8144-019d53512769", + "parent_span_id": "7e2c959c-157a-43d5-9870-32994cddd388", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-22 16:56:35.188000000", + "end_time": "2025-10-22 16:56:35.188000000", + "duration_ms": 0.054, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What is Noveum and what does it do?", + "query_length": 35, + "query_type": "routing_evaluation_query", + "classification.mode": "RAG", + "classification.rag_keyword_score": 1, + "classification.web_keyword_score": 0, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "noveum_specific", + "query.keyword_density": 0.125, + "query.domain_affinity": "noveum", + "routing.decision": "Routed to RAG based on analysis", + "routing.rationale": "RAG score: 1, Web score: 0, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "Web Search", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.019230769230769232, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: RAG (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "3ad192e5-bbd4-4236-8144-019d53512769", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:56:35.188000000", + "trace_end_time": "2025-10-22 16:56:43.863000000", + "trace_duration_ms": 8675.356000000002, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:56:35.188000000", + "updated_at": "2025-10-22 16:56:35.188000000" + }, + { + "span_id": "9f42d18a-0235-4ae2-838c-19a88ddc95e1", + "trace_id": "3ad192e5-bbd4-4236-8144-019d53512769", + "parent_span_id": "99eadeae-0733-4bf3-982e-e9f3014f5913", + "name": "agent.llm_model_execution", + "start_time": "2025-10-22 16:56:36.031000000", + "end_time": "2025-10-22 16:56:42.012000000", + "duration_ms": 5980.947999999999, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What is Noveum and what does it do?", + "query_length": 35, + "query_type": "rag_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 5.980915784835815, + "model.latency_ms": 5980.915784835815, + "model.start_time": 1761152196.031816, + "model.end_time": 1761152202.0127318, + "model.performance_tier": "slow", + "model.prompt_tokens": 716, + "model.completion_tokens": 131, + "model.total_tokens": 847, + "model.tokens_per_second": 141.61710856178715, + "model.estimated_cost": 0.025410000000000002, + "model.efficiency_score": 0.6221959858323495, + "model.response_length": 527, + "model.response_quality": "high", + "model.output_response": "Model Response: Noveum.ai is a comprehensive tracing and observability platform specifically designed for AI applications, including LLM-powered chatbots, RAG systems, and multi-agent workflows. It provides the neces...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "3ad192e5-bbd4-4236-8144-019d53512769", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:56:35.188000000", + "trace_end_time": "2025-10-22 16:56:43.863000000", + "trace_duration_ms": 8675.356000000002, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:56:35.188000000", + "updated_at": "2025-10-22 16:56:35.188000000" + }, + { + "span_id": "7bcbe4b7-585a-4c9d-add8-16e8aa62254f", + "trace_id": "3ad192e5-bbd4-4236-8144-019d53512769", + "parent_span_id": "99eadeae-0733-4bf3-982e-e9f3014f5913", + "name": "agent.rag_evaluation_metrics", + "start_time": "2025-10-22 16:56:42.012000000", + "end_time": "2025-10-22 16:56:42.012000000", + "duration_ms": 0.018000000000000002, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "rag_evaluation_metrics", + "agent.capabilities": [ + "retrieval_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What is Noveum and what does it do?", + "query_length": 35, + "query_type": "rag_evaluation_query", + "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, vector searches, data retrieval, agent reasoning, and more. Without proper observability, debugging ...", + "retrieval.context_length": 2334, + "retrieval.sources_count": 5, + "retrieval.context_quality": "high", + "retrieval.effectiveness": 1, + "retrieval.context_utilization": 2.334, + "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, vector searches, data retrieval, agent reasoning, and more. Without proper observability, debugging becomes a nightmare, optimization is guesswork, and understanding user interactions is nearly impossible.\nNoveum.ai solves this by providing comprehensive tracing and observability specifically designed for AI applications.\nKey benefits of using Noveum.ai:\n- Complete Visibility: Trace...\n\nSource 2 (https://noveum.ai/en/docs/getting-started/overview):\nNoveum.ai Overview\nComprehensive AI tracing and observability platform for LLM applications, RAG systems, and AI agents\nWelcome to Noveum.ai\u2014the comprehensive tracing and observability platform built specifically for AI applications. Whether you're building LLM-powered chatbots, RAG systems, multi-agent workflows, or any AI-driven application, Noveum provides the insights you need to understand, debug, and optimize your systems.\n\ud83c\udfaf Why AI Applications Need Specialized Observability\nTraditional mo...\n\nSource 3 (https://noveum.ai/docs/getting-started/overview):\nNoveum.ai Overview\nComprehensive AI tracing and observability platform for LLM applications, RAG systems, and AI agents\nWelcome to Noveum.ai\u2014the comprehensive tracing and observability platform built specifically for AI applications. Whether you're building LLM-powered chatbots, RAG systems, multi-agent workflows, or any AI-driven application, Noveum provides the insights you need to understand, debug, and optimize your systems.\n\ud83c\udfaf Why AI Applications Need Specialized Observability\nTraditional mo...\n\nSource 4 (https://noveum.ai/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nSource 5 (https://noveum.ai/en/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nUser Question: What is Noveum and what does it do?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", + "prompt.prompt_length": 2865, + "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, ...", + "prompt.instruction_following": "rag_optimized", + "response.answer_length": 527, + "response.answer_completeness": "complete", + "response.response_quality": "high", + "response.source_citation": 5, + "response.context_utilization": 0.22579263067694944, + "output_response": "RAG Answer: Noveum.ai is a comprehensive tracing and observability platform specifically designed for AI applications, including LLM-powered chatbots, RAG systems, and multi-agent workflows. It provides the neces...", + "evaluation.retrieval_effectiveness": 1, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 5, + "evaluation.context_relevance": "high", + "evaluation.overall_quality": "high", + "evaluation.ready_for_production": true, + "rag.retrieval_strategy": "semantic_similarity", + "rag.vector_search_results": 5, + "rag.context_synthesis": "multi_source", + "rag.document_coverage": 1, + "rag.information_density": 0.22579263067694944 + }, + "events": [], + "links": [], + "trace_trace_id": "3ad192e5-bbd4-4236-8144-019d53512769", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:56:35.188000000", + "trace_end_time": "2025-10-22 16:56:43.863000000", + "trace_duration_ms": 8675.356000000002, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:56:35.188000000", + "updated_at": "2025-10-22 16:56:35.188000000" + }, + { + "span_id": "d39297f4-b4fe-4f9c-b45b-854411ba770a", + "trace_id": "9af2b4b0-0c2f-4d2e-ae99-30df0d2f0fa6", + "parent_span_id": "", + "name": "system_initialization", + "start_time": "2025-10-22 16:56:34.565000000", + "end_time": "2025-10-22 16:56:35.186000000", + "duration_ms": 621.02, + "status": "ok", + "status_message": "", + "attributes": { + "system.force_scrape": false, + "system.config": { + "noveum_base_url": "https://noveum.ai", + "max_pages_to_scrape": 50, + "chunk_size": 1000, + "chunk_overlap": 200, + "max_search_results": 5, + "rag_threshold": 0.7, + "noveum_docs_file": "noveum_docs.json", + "vector_store_path": "noveum_vectorstore" + }, + "input_query": "Initialize system with force_scrape=False", + "output_response": "System initialization: RAG system loaded, vector store ready, agent operational", + "system.initialized": true, + "system.documents_loaded": 38, + "system.vectorstore_ready": true + }, + "events": [], + "links": [], + "trace_trace_id": "9af2b4b0-0c2f-4d2e-ae99-30df0d2f0fa6", + "trace_name": "auto_trace_system_initialization", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:56:34.565000000", + "trace_end_time": "2025-10-22 16:56:35.186000000", + "trace_duration_ms": 621.177, + "span_count": 1, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:56:34.565000000", + "updated_at": "2025-10-22 16:56:34.565000000" + }, + { + "span_id": "8c144ad8-a439-4986-b741-3bc4b7fc6110", + "trace_id": "bae13042-cf2e-4813-a673-7dc2a611725e", + "parent_span_id": "d37fe1a1-a126-4e4e-a914-93dd402fed3f", + "name": "agent.query_routing", + "start_time": "2025-10-16 08:50:56.153000000", + "end_time": "2025-10-16 08:51:01.908000000", + "duration_ms": 5754.43, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What are the latest breakthroughs in artificial intelligence?", + "query_length": 61, + "query_type": "routing_query", + "output_response": "Routed to Web Search for query processing", + "router.classification": "Web Search", + "router.confidence_score": 1, + "router.rag_keyword_score": 0, + "router.web_keyword_score": 1, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "bae13042-cf2e-4813-a673-7dc2a611725e", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:56.153000000", + "trace_end_time": "2025-10-16 08:51:01.908000000", + "trace_duration_ms": 5754.578, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:56.153000000", + "updated_at": "2025-10-16 08:50:56.153000000" + }, + { + "span_id": "b294f9b7-7762-47a9-b49d-00801a984af2", + "trace_id": "bae13042-cf2e-4813-a673-7dc2a611725e", + "parent_span_id": "8c144ad8-a439-4986-b741-3bc4b7fc6110", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-16 08:50:56.153000000", + "end_time": "2025-10-16 08:50:56.153000000", + "duration_ms": 0.024, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What are the latest breakthroughs in artificial intelligence?", + "query_length": 61, + "query_type": "routing_evaluation_query", + "classification.mode": "Web Search", + "classification.rag_keyword_score": 0, + "classification.web_keyword_score": 1, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "complex", + "query.intent": "general_knowledge", + "query.keyword_density": 0.125, + "query.domain_affinity": "general", + "routing.decision": "Routed to Web Search based on analysis", + "routing.rationale": "RAG score: 0, Web score: 1, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "RAG", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.019230769230769232, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: Web Search (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "bae13042-cf2e-4813-a673-7dc2a611725e", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:56.153000000", + "trace_end_time": "2025-10-16 08:51:01.908000000", + "trace_duration_ms": 5754.578, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:56.153000000", + "updated_at": "2025-10-16 08:50:56.153000000" + }, + { + "span_id": "ab6a281a-4989-40c8-9011-d0ec7dde7ea9", + "trace_id": "bae13042-cf2e-4813-a673-7dc2a611725e", + "parent_span_id": "8c144ad8-a439-4986-b741-3bc4b7fc6110", + "name": "agent.web_search_generation", + "start_time": "2025-10-16 08:50:56.153000000", + "end_time": "2025-10-16 08:51:01.908000000", + "duration_ms": 5754.317999999999, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "web_search_agent", + "agent.operation": "web_search_generation", + "agent.capabilities": [ + "web_search", + "content_synthesis", + "response_generation" + ], + "agent.id": "noveum_web_search_agent", + "input_query": "What are the latest breakthroughs in artificial intelligence?", + "query_length": 61, + "query_type": "web_search_query", + "output_response": "Web Search Answer: The web search did not yield specific results regarding the latest breakthroughs in artificial intelligence. However, I can provide a general overview based on knowledge up to October 2023.\n\nRecent br...", + "web_search.results_count": 1, + "web_search.context_length": 163, + "web_search.response_length": 1489, + "web_search.mode": "external_web_search" + }, + "events": [], + "links": [], + "trace_trace_id": "bae13042-cf2e-4813-a673-7dc2a611725e", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:56.153000000", + "trace_end_time": "2025-10-16 08:51:01.908000000", + "trace_duration_ms": 5754.578, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:56.153000000", + "updated_at": "2025-10-16 08:50:56.153000000" + }, + { + "span_id": "d37fe1a1-a126-4e4e-a914-93dd402fed3f", + "trace_id": "bae13042-cf2e-4813-a673-7dc2a611725e", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-16 08:50:56.153000000", + "end_time": "2025-10-16 08:51:01.908000000", + "duration_ms": 5754.527, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What are the latest breakthroughs in artificial intelligence?", + "query.length": 61, + "processing.mode": "Web Search", + "processing.time_seconds": 5.754457473754883, + "processing.response_length": 1489, + "processing.sources_count": 1, + "output_response": "Final Answer: The web search did not yield specific results regarding the latest breakthroughs in artificial intelligence. However, I can provide a general overview based on knowledge up to October 2023.\n\nRecent br...", + "final_answer_mode": "Web Search", + "query_processed.input_query": "What are the latest breakthroughs in artificial intelligence?", + "query_processed.output_response": "Successfully processed query using Web Search, generated 1489 character response", + "query_processed.mode": "Web Search", + "query_processed.processing_time": 5.754457473754883, + "query_processed.response_length": 1489 + }, + "events": [], + "links": [], + "trace_trace_id": "bae13042-cf2e-4813-a673-7dc2a611725e", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:56.153000000", + "trace_end_time": "2025-10-16 08:51:01.908000000", + "trace_duration_ms": 5754.578, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:56.153000000", + "updated_at": "2025-10-16 08:50:56.153000000" + }, + { + "span_id": "f61ef76d-4887-42da-9fd6-e2cdb25a9e09", + "trace_id": "bae13042-cf2e-4813-a673-7dc2a611725e", + "parent_span_id": "ab6a281a-4989-40c8-9011-d0ec7dde7ea9", + "name": "agent.llm_model_execution", + "start_time": "2025-10-16 08:50:56.502000000", + "end_time": "2025-10-16 08:51:01.908000000", + "duration_ms": 5405.44, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What are the latest breakthroughs in artificial intelligence?", + "query_length": 61, + "query_type": "web_search_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 5.405392408370972, + "model.latency_ms": 5405.392408370972, + "model.start_time": 1760604656.5026655, + "model.end_time": 1760604661.908058, + "model.performance_tier": "slow", + "model.prompt_tokens": 167, + "model.completion_tokens": 372, + "model.total_tokens": 539, + "model.tokens_per_second": 99.71523976044487, + "model.estimated_cost": 0.01617, + "model.efficiency_score": 2.7625231910946195, + "model.response_length": 1489, + "model.response_quality": "high", + "model.output_response": "Model Response: The web search did not yield specific results regarding the latest breakthroughs in artificial intelligence. However, I can provide a general overview based on knowledge up to October 2023.\n\nRecent br...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "bae13042-cf2e-4813-a673-7dc2a611725e", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:56.153000000", + "trace_end_time": "2025-10-16 08:51:01.908000000", + "trace_duration_ms": 5754.578, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:56.153000000", + "updated_at": "2025-10-16 08:50:56.153000000" + }, + { + "span_id": "550bf5ae-c816-4ef0-a6c1-1553b1c258b8", + "trace_id": "bae13042-cf2e-4813-a673-7dc2a611725e", + "parent_span_id": "ab6a281a-4989-40c8-9011-d0ec7dde7ea9", + "name": "agent.web_search_evaluation_metrics", + "start_time": "2025-10-16 08:51:01.908000000", + "end_time": "2025-10-16 08:51:01.908000000", + "duration_ms": 0.018000000000000002, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "web_search_evaluation_metrics", + "agent.capabilities": [ + "web_search_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What are the latest breakthroughs in artificial intelligence?", + "query_length": 61, + "query_type": "web_search_evaluation_query", + "web_search.results_count": 1, + "web_search.context_length": 163, + "web_search.context_synthesized": "Context from 1 web sources", + "web_search.search_effectiveness": 0.2, + "web_search.context_quality": "low", + "web_search.source_diversity": 1, + "web_search.information_synthesis": "low", + "web_search.external_knowledge_utilization": 0.163, + "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What+are+the+latest+breakthroughs+in+artificial+intelligence?\n\nUser Question: What are the latest breakthroughs in artificial intelligence?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", + "prompt.prompt_length": 669, + "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What+are+the+latest+breakthroughs+in+artificial+intelligence?", + "prompt.instruction_following": "web_search_optimized", + "response.answer_length": 1489, + "response.answer_completeness": "complete", + "response.response_quality": "medium", + "response.source_citation": 1, + "response.context_utilization": 9.134969325153374, + "output_response": "Web Search Answer: The web search did not yield specific results regarding the latest breakthroughs in artificial intelligence. However, I can provide a general overview based on knowledge up to October 2023.\n\nRecent br...", + "evaluation.search_effectiveness": 0.2, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 1, + "evaluation.context_relevance": "low", + "evaluation.overall_quality": "low", + "evaluation.ready_for_production": true, + "web_search.search_strategy": "duckduckgo_api", + "web_search.real_time_data": true, + "web_search.external_sources": 1, + "web_search.information_freshness": "current", + "web_search.knowledge_synthesis": "single_source" + }, + "events": [], + "links": [], + "trace_trace_id": "bae13042-cf2e-4813-a673-7dc2a611725e", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:56.153000000", + "trace_end_time": "2025-10-16 08:51:01.908000000", + "trace_duration_ms": 5754.578, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:56.153000000", + "updated_at": "2025-10-16 08:50:56.153000000" + }, + { + "span_id": "04cb7d29-9ea8-4ae0-9834-030cae944eca", + "trace_id": "79f9d235-d46e-4740-be35-ef0006071f6c", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-16 08:50:51.539000000", + "end_time": "2025-10-16 08:50:55.150000000", + "duration_ms": 3611.008, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What's happening in the software development world today?", + "query.length": 57, + "processing.mode": "Web Search", + "processing.time_seconds": 3.6109254360198975, + "processing.response_length": 526, + "processing.sources_count": 1, + "output_response": "Final Answer: The provided web search results do not contain specific information about current events or trends in the software development world. They primarily focus on unrelated topics, such as the meaning of \"...", + "final_answer_mode": "Web Search", + "query_processed.input_query": "What's happening in the software development world today?", + "query_processed.output_response": "Successfully processed query using Web Search, generated 526 character response", + "query_processed.mode": "Web Search", + "query_processed.processing_time": 3.6109254360198975, + "query_processed.response_length": 526 + }, + "events": [], + "links": [], + "trace_trace_id": "79f9d235-d46e-4740-be35-ef0006071f6c", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:51.539000000", + "trace_end_time": "2025-10-16 08:50:55.150000000", + "trace_duration_ms": 3611.084, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:51.539000000", + "updated_at": "2025-10-16 08:50:51.539000000" + }, + { + "span_id": "e5e3c927-783e-48c5-aec7-b32bae98c0a2", + "trace_id": "79f9d235-d46e-4740-be35-ef0006071f6c", + "parent_span_id": "f630c3e2-091e-4bd9-9556-e0ebc1c50291", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-16 08:50:51.539000000", + "end_time": "2025-10-16 08:50:51.540000000", + "duration_ms": 0.041, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What's happening in the software development world today?", + "query_length": 57, + "query_type": "routing_evaluation_query", + "classification.mode": "Web Search", + "classification.rag_keyword_score": 0, + "classification.web_keyword_score": 1, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "complex", + "query.intent": "general_knowledge", + "query.keyword_density": 0.125, + "query.domain_affinity": "general", + "routing.decision": "Routed to Web Search based on analysis", + "routing.rationale": "RAG score: 0, Web score: 1, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "RAG", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.019230769230769232, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: Web Search (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "79f9d235-d46e-4740-be35-ef0006071f6c", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:51.539000000", + "trace_end_time": "2025-10-16 08:50:55.150000000", + "trace_duration_ms": 3611.084, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:51.539000000", + "updated_at": "2025-10-16 08:50:51.539000000" + }, + { + "span_id": "f630c3e2-091e-4bd9-9556-e0ebc1c50291", + "trace_id": "79f9d235-d46e-4740-be35-ef0006071f6c", + "parent_span_id": "04cb7d29-9ea8-4ae0-9834-030cae944eca", + "name": "agent.query_routing", + "start_time": "2025-10-16 08:50:51.539000000", + "end_time": "2025-10-16 08:50:55.150000000", + "duration_ms": 3610.876, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What's happening in the software development world today?", + "query_length": 57, + "query_type": "routing_query", + "output_response": "Routed to Web Search for query processing", + "router.classification": "Web Search", + "router.confidence_score": 1, + "router.rag_keyword_score": 0, + "router.web_keyword_score": 1, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "79f9d235-d46e-4740-be35-ef0006071f6c", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:51.539000000", + "trace_end_time": "2025-10-16 08:50:55.150000000", + "trace_duration_ms": 3611.084, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:51.539000000", + "updated_at": "2025-10-16 08:50:51.539000000" + }, + { + "span_id": "d7b0f399-3b43-43bf-bac1-b2101268841c", + "trace_id": "79f9d235-d46e-4740-be35-ef0006071f6c", + "parent_span_id": "f630c3e2-091e-4bd9-9556-e0ebc1c50291", + "name": "agent.web_search_generation", + "start_time": "2025-10-16 08:50:51.540000000", + "end_time": "2025-10-16 08:50:55.150000000", + "duration_ms": 3610.679, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "web_search_agent", + "agent.operation": "web_search_generation", + "agent.capabilities": [ + "web_search", + "content_synthesis", + "response_generation" + ], + "agent.id": "noveum_web_search_agent", + "input_query": "What's happening in the software development world today?", + "query_length": 57, + "query_type": "web_search_query", + "output_response": "Web Search Answer: The provided web search results do not contain specific information about current events or trends in the software development world. They primarily focus on unrelated topics, such as the meaning of \"...", + "web_search.results_count": 1, + "web_search.context_length": 589, + "web_search.response_length": 526, + "web_search.mode": "external_web_search" + }, + "events": [], + "links": [], + "trace_trace_id": "79f9d235-d46e-4740-be35-ef0006071f6c", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:51.539000000", + "trace_end_time": "2025-10-16 08:50:55.150000000", + "trace_duration_ms": 3611.084, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:51.539000000", + "updated_at": "2025-10-16 08:50:51.539000000" + }, + { + "span_id": "4ed5dae4-f850-4644-aa55-2f1c00d11c22", + "trace_id": "79f9d235-d46e-4740-be35-ef0006071f6c", + "parent_span_id": "d7b0f399-3b43-43bf-bac1-b2101268841c", + "name": "agent.llm_model_execution", + "start_time": "2025-10-16 08:50:51.976000000", + "end_time": "2025-10-16 08:50:55.150000000", + "duration_ms": 3174.51, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What's happening in the software development world today?", + "query_length": 57, + "query_type": "web_search_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 3.174463987350464, + "model.latency_ms": 3174.463987350464, + "model.start_time": 1760604651.9761698, + "model.end_time": 1760604655.1506338, + "model.performance_tier": "medium", + "model.prompt_tokens": 272, + "model.completion_tokens": 131, + "model.total_tokens": 403, + "model.tokens_per_second": 126.95056601866197, + "model.estimated_cost": 0.01209, + "model.efficiency_score": 1.305210918114144, + "model.response_length": 526, + "model.response_quality": "high", + "model.output_response": "Model Response: The provided web search results do not contain specific information about current events or trends in the software development world. They primarily focus on unrelated topics, such as the meaning of \"...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "79f9d235-d46e-4740-be35-ef0006071f6c", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:51.539000000", + "trace_end_time": "2025-10-16 08:50:55.150000000", + "trace_duration_ms": 3611.084, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:51.539000000", + "updated_at": "2025-10-16 08:50:51.539000000" + }, + { + "span_id": "eb151834-9322-4f63-add0-dec9a2966367", + "trace_id": "79f9d235-d46e-4740-be35-ef0006071f6c", + "parent_span_id": "d7b0f399-3b43-43bf-bac1-b2101268841c", + "name": "agent.web_search_evaluation_metrics", + "start_time": "2025-10-16 08:50:55.150000000", + "end_time": "2025-10-16 08:50:55.150000000", + "duration_ms": 0.02, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "web_search_evaluation_metrics", + "agent.capabilities": [ + "web_search_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What's happening in the software development world today?", + "query_length": 57, + "query_type": "web_search_evaluation_query", + "web_search.results_count": 1, + "web_search.context_length": 589, + "web_search.context_synthesized": "Context from 1 web sources", + "web_search.search_effectiveness": 0.2, + "web_search.context_quality": "medium", + "web_search.source_diversity": 1, + "web_search.information_synthesis": "low", + "web_search.external_knowledge_utilization": 0.589, + "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\n\u66fc\u5df4\u7684\u610f\u601d\u662f\u4e00\u79cd\u6bd2\u86c7\uff0c\u8c61\u5f81\u7740\u79d1\u6bd4\u5728\u7403\u573a\u4e0a\u7684\u653b\u51fb\u6027\u548c\u575a\u97e7\u4e0d\u62d4 \u201cWhat can i say\uff0c Mamba out\uff01\u201d \u4f46\u662f\u7ecf\u8fc7\u4e92\u8054\u7f51\u7684\u89e3\u6784\uff0c\u8fd9\u53e5\u8bdd\u6210\u4e3a\u4e86\u4e00\u53e5\u4e07\u7528\u573a\u666f\u7684\u8bdd \u2026 Aprende ingl\u00e9s gratis con nuestros cursos. \u00a1Mas de 100 lecciones con ejemplos y audio! Distintas y divertidas formas de estudiar ingl\u00e9s: \u2026 \u77e5\u4e4e\uff0c\u4e2d\u6587\u4e92\u8054\u7f51\u9ad8\u8d28\u91cf\u7684\u95ee\u7b54\u793e\u533a\u548c\u521b\u4f5c\u8005\u805a\u96c6\u7684\u539f\u521b\u5185\u5bb9\u5e73\u53f0\uff0c\u4e8e 2011 \u5e74 1 \u6708\u6b63\u5f0f\u4e0a\u7ebf\uff0c\u4ee5\u300c\u8ba9\u4eba\u4eec\u66f4\u597d\u7684\u5206\u4eab\u77e5\u8bc6\u3001\u7ecf\u9a8c\u548c\u89c1\u89e3\uff0c\u627e\u5230\u81ea\u5df1\u7684\u89e3\u7b54\u300d\u4e3a\u54c1 \u2026 \u8f6f\u4ef6\u7279\u70b9 1\u3001Windows Desktop Runtime\u662f\u5fae\u8f6fWindows\u684c\u9762\u7a0b\u5e8f\u8fd0\u884c\u5e93\uff08\u542b\u5e38\u89c4\u8fd0\u884c\u5e93\uff09 2\u3001\u80fd\u5b8c\u7f8e\u517c\u5bb9\u5fae\u8f6f\u4e0d\u540c\u7248\u672c\u7684Windows\u7cfb\u7edf\uff0c\u89e3\u51b3\u5176\u7a0b\u5e8f\u7f3a \u2026 \u77e5\u4e4e\uff0c\u4e2d\u6587\u4e92\u8054\u7f51\u9ad8\u8d28\u91cf\u7684\u95ee\u7b54\u793e\u533a\u548c\u521b\u4f5c\u8005\u805a\u96c6\u7684\u539f\u521b\u5185\u5bb9\u5e73\u53f0\uff0c\u4e8e 2011 \u5e74 1 \u6708\u6b63\u5f0f\u4e0a\u7ebf\uff0c\u4ee5\u300c\u8ba9\u4eba\u4eec\u66f4\u597d\u7684\u5206\u4eab\u77e5\u8bc6\u3001\u7ecf\u9a8c\u548c\u89c1\u89e3\uff0c\u627e\u5230\u81ea\u5df1\u7684\u89e3\u7b54\u300d\u4e3a\u54c1 \u2026\nURL: https://duckduckgo.com/?q=What's+happening+in+the+software+development+world+today?\n\nUser Question: What's happening in the software development world today?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", + "prompt.prompt_length": 1091, + "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\n\u66fc\u5df4\u7684\u610f\u601d\u662f\u4e00\u79cd\u6bd2\u86c7\uff0c\u8c61\u5f81\u7740\u79d1\u6bd4\u5728\u7403\u573a\u4e0a\u7684\u653b\u51fb\u6027\u548c\u575a\u97e7\u4e0d\u62d4 \u201cWhat can i say\uff0c Mamba out\uff01\u201d \u4f46\u662f\u7ecf\u8fc7\u4e92\u8054\u7f51\u7684\u89e3\u6784\uff0c\u8fd9\u53e5\u8bdd\u6210\u4e3a\u4e86\u4e00\u53e5\u4e07\u7528\u573a\u666f\u7684\u8bdd \u2026 Aprende ingl\u00e9s gratis con nuestros cursos. \u00a1Mas de 100 lecciones con ejemplos y audio...", + "prompt.instruction_following": "web_search_optimized", + "response.answer_length": 526, + "response.answer_completeness": "complete", + "response.response_quality": "medium", + "response.source_citation": 1, + "response.context_utilization": 0.8930390492359932, + "output_response": "Web Search Answer: The provided web search results do not contain specific information about current events or trends in the software development world. They primarily focus on unrelated topics, such as the meaning of \"...", + "evaluation.search_effectiveness": 0.2, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 1, + "evaluation.context_relevance": "medium", + "evaluation.overall_quality": "low", + "evaluation.ready_for_production": true, + "web_search.search_strategy": "duckduckgo_api", + "web_search.real_time_data": true, + "web_search.external_sources": 1, + "web_search.information_freshness": "current", + "web_search.knowledge_synthesis": "single_source" + }, + "events": [], + "links": [], + "trace_trace_id": "79f9d235-d46e-4740-be35-ef0006071f6c", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:51.539000000", + "trace_end_time": "2025-10-16 08:50:55.150000000", + "trace_duration_ms": 3611.084, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:51.539000000", + "updated_at": "2025-10-16 08:50:51.539000000" + }, + { + "span_id": "888b4441-3678-42a5-a752-8daae2433482", + "trace_id": "77b81bd5-5a04-4fc0-9fc6-571f7907d6b1", + "parent_span_id": "ba6ffeeb-78b5-44dd-b460-7933ca6e7739", + "name": "agent.query_routing", + "start_time": "2025-10-16 08:50:44.888000000", + "end_time": "2025-10-16 08:50:50.529000000", + "duration_ms": 5641.062, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What are the newest features in cloud computing?", + "query_length": 48, + "query_type": "routing_query", + "output_response": "Routed to RAG for query processing", + "router.classification": "RAG", + "router.confidence_score": 1, + "router.rag_keyword_score": 1, + "router.web_keyword_score": 0, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "77b81bd5-5a04-4fc0-9fc6-571f7907d6b1", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:44.888000000", + "trace_end_time": "2025-10-16 08:50:50.529000000", + "trace_duration_ms": 5641.226, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:44.888000000", + "updated_at": "2025-10-16 08:50:44.888000000" + }, + { + "span_id": "b6bae768-93a6-47d9-8369-61897f9d7828", + "trace_id": "77b81bd5-5a04-4fc0-9fc6-571f7907d6b1", + "parent_span_id": "888b4441-3678-42a5-a752-8daae2433482", + "name": "agent.llm-rag", + "start_time": "2025-10-16 08:50:44.888000000", + "end_time": "2025-10-16 08:50:50.529000000", + "duration_ms": 5640.875999999999, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "rag_agent", + "agent.operation": "llm-rag", + "agent.capabilities": [ + "document_retrieval", + "context_generation", + "response_generation" + ], + "agent.id": "noveum_rag_agent", + "input_query": "What are the newest features in cloud computing?", + "query_length": 48, + "query_type": "rag_query", + "output_response": "RAG Answer: The provided context does not specifically mention any new features in cloud computing as a general topic. However, it does highlight several recent enhancements and features related to the Noveum.ai ...", + "rag.context_length": 2579, + "rag.sources_count": 5, + "rag.answer_length": 983, + "rag.mode": "retrieval_augmented_generation" + }, + "events": [], + "links": [], + "trace_trace_id": "77b81bd5-5a04-4fc0-9fc6-571f7907d6b1", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:44.888000000", + "trace_end_time": "2025-10-16 08:50:50.529000000", + "trace_duration_ms": 5641.226, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:44.888000000", + "updated_at": "2025-10-16 08:50:44.888000000" + }, + { + "span_id": "ba6ffeeb-78b5-44dd-b460-7933ca6e7739", + "trace_id": "77b81bd5-5a04-4fc0-9fc6-571f7907d6b1", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-16 08:50:44.888000000", + "end_time": "2025-10-16 08:50:50.529000000", + "duration_ms": 5641.156, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What are the newest features in cloud computing?", + "query.length": 48, + "processing.mode": "RAG", + "processing.time_seconds": 5.64110541343689, + "processing.response_length": 983, + "processing.sources_count": 5, + "output_response": "Final Answer: The provided context does not specifically mention any new features in cloud computing as a general topic. However, it does highlight several recent enhancements and features related to the Noveum.ai ...", + "final_answer_mode": "RAG", + "query_processed.input_query": "What are the newest features in cloud computing?", + "query_processed.output_response": "Successfully processed query using RAG, generated 983 character response", + "query_processed.mode": "RAG", + "query_processed.processing_time": 5.64110541343689, + "query_processed.response_length": 983 + }, + "events": [], + "links": [], + "trace_trace_id": "77b81bd5-5a04-4fc0-9fc6-571f7907d6b1", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:44.888000000", + "trace_end_time": "2025-10-16 08:50:50.529000000", + "trace_duration_ms": 5641.226, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:44.888000000", + "updated_at": "2025-10-16 08:50:44.888000000" + }, + { + "span_id": "f5538bff-8908-4a56-a622-4ba2eeedfd29", + "trace_id": "77b81bd5-5a04-4fc0-9fc6-571f7907d6b1", + "parent_span_id": "888b4441-3678-42a5-a752-8daae2433482", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-16 08:50:44.888000000", + "end_time": "2025-10-16 08:50:44.888000000", + "duration_ms": 0.032, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What are the newest features in cloud computing?", + "query_length": 48, + "query_type": "routing_evaluation_query", + "classification.mode": "RAG", + "classification.rag_keyword_score": 1, + "classification.web_keyword_score": 0, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "documentation", + "query.keyword_density": 0.125, + "query.domain_affinity": "noveum", + "routing.decision": "Routed to RAG based on analysis", + "routing.rationale": "RAG score: 1, Web score: 0, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "Web Search", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.019230769230769232, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: RAG (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "77b81bd5-5a04-4fc0-9fc6-571f7907d6b1", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:44.888000000", + "trace_end_time": "2025-10-16 08:50:50.529000000", + "trace_duration_ms": 5641.226, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:44.888000000", + "updated_at": "2025-10-16 08:50:44.888000000" + }, + { + "span_id": "856e51cf-fa4d-459b-a046-1ea7316b903e", + "trace_id": "77b81bd5-5a04-4fc0-9fc6-571f7907d6b1", + "parent_span_id": "b6bae768-93a6-47d9-8369-61897f9d7828", + "name": "agent.llm_model_execution", + "start_time": "2025-10-16 08:50:45.423000000", + "end_time": "2025-10-16 08:50:49.997000000", + "duration_ms": 4573.782999999999, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What are the newest features in cloud computing?", + "query_length": 48, + "query_type": "rag_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 4.573714733123779, + "model.latency_ms": 4573.714733123779, + "model.start_time": 1760604645.423261, + "model.end_time": 1760604649.9969757, + "model.performance_tier": "medium", + "model.prompt_tokens": 780, + "model.completion_tokens": 245, + "model.total_tokens": 1025, + "model.tokens_per_second": 224.1066747291299, + "model.estimated_cost": 0.03075, + "model.efficiency_score": 0.9590243902439024, + "model.response_length": 983, + "model.response_quality": "high", + "model.output_response": "Model Response: The provided context does not specifically mention any new features in cloud computing as a general topic. However, it does highlight several recent enhancements and features related to the Noveum.ai ...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "77b81bd5-5a04-4fc0-9fc6-571f7907d6b1", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:44.888000000", + "trace_end_time": "2025-10-16 08:50:50.529000000", + "trace_duration_ms": 5641.226, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:44.888000000", + "updated_at": "2025-10-16 08:50:44.888000000" + }, + { + "span_id": "a2fa7400-74c4-4718-a562-899b91503c95", + "trace_id": "77b81bd5-5a04-4fc0-9fc6-571f7907d6b1", + "parent_span_id": "b6bae768-93a6-47d9-8369-61897f9d7828", + "name": "agent.rag_evaluation_metrics", + "start_time": "2025-10-16 08:50:49.997000000", + "end_time": "2025-10-16 08:50:49.997000000", + "duration_ms": 0.048, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "rag_evaluation_metrics", + "agent.capabilities": [ + "retrieval_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What are the newest features in cloud computing?", + "query_length": 48, + "query_type": "rag_evaluation_query", + "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/changelog):\n- \ud83d\udcc8 Enhanced Dashboard Analytics: Real-time request tracking, latency monitoring, and cost analysis\n- \ud83d\udd0d Improved Logs Interface: Better search functionality, detailed trace views, and enhanced debugging capabilities\n- \ud83d\udc65 Team Management: Advanced member mana...", + "retrieval.context_length": 2579, + "retrieval.sources_count": 5, + "retrieval.context_quality": "high", + "retrieval.effectiveness": 1, + "retrieval.context_utilization": 2.579, + "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/changelog):\n- \ud83d\udcc8 Enhanced Dashboard Analytics: Real-time request tracking, latency monitoring, and cost analysis\n- \ud83d\udd0d Improved Logs Interface: Better search functionality, detailed trace views, and enhanced debugging capabilities\n- \ud83d\udc65 Team Management: Advanced member management with role-based access control and invitation system\n- \ud83d\udee1\ufe0f Security Improvements: Enhanced API key management and secure credential storage\n4 months ago\n- \ud83d\udcca Advanced Metrics Collection: Comprehensive telemetry system with custom metrics ...\n\nSource 2 (https://noveum.ai/en/changelog):\n- \ud83d\udcdd Documentation Updates: Comprehensive guides for integration and best practices\n5 months ago\n- \ud83c\udf89 Platform Foundation: Initial release of Noveum.ai AI observability platform\n- \ud83d\ude80 Core Features: AI Gateway integration, basic evaluation metrics, and monitoring capabilities\n- \ud83d\udcc8 Evaluation Jobs: Automated model evaluation with configurable metrics and reporting\n- \ud83d\udd0c Provider Integrations: Support for major AI providers and custom model deployments...\n\nSource 3 (https://noveum.ai/docs/platform/dashboard):\n- Bulk Export: Download multiple traces simultaneously\n- Comparative Analysis: Compare performance across multiple traces\nReal-Time Updates\n- Live Refresh: Automatic updates as new traces arrive\n- Connection Monitoring: Real-time status of your trace ingestion pipeline\n- Performance Indicators: Live metrics for system health monitoring\nReady to dive deeper? Explore Projects & Environments to organize your AI applications, or check out Team Collaboration to share insights with your team.\nGet Earl...\n\nSource 4 (https://noveum.ai/en/changelog):\nChangelog\nStay up to date with the latest changes in our product.\n2 months ago\n- \ud83c\udfaf Major Platform Evolution: NovaEval Framework - Launched comprehensive AI model evaluation framework with 20+ built-in scorers including accuracy, RAG metrics, conversational scoring, and specialized agent evaluation tools. Features production-grade deployment on Docker and Kubernetes. (GitHub | PyPI Package)\n- \ud83d\ude80 Next-Generation Traces UI - Complete redesign featuring three-pane layout with directory tree navigatio...\n\nSource 5 (https://noveum.ai/en/blog):\n8/12/2025\nDiscover how Noveum.ai provides comprehensive tracing and observability for AI applications, from development debugging to production optimization.\nShashank Agarwal\n3/3/2025\nDiscover how Noveum.ai provides comprehensive tracing and observability for LLM applications, RAG systems, and multi-agent workflows with our powerful Python and TypeScript SDKs.\nShashank Agarwal\n3/2/2025...\n\nUser Question: What are the newest features in cloud computing?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", + "prompt.prompt_length": 3123, + "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/changelog):\n- \ud83d\udcc8 Enhanced Dashboard Analytics: Real-time request tracking, latency monitoring, and cost analysis\n- \ud83d\udd0d Improved Logs Interface: Better search functionality,...", + "prompt.instruction_following": "rag_optimized", + "response.answer_length": 983, + "response.answer_completeness": "complete", + "response.response_quality": "high", + "response.source_citation": 5, + "response.context_utilization": 0.381155486622722, + "output_response": "RAG Answer: The provided context does not specifically mention any new features in cloud computing as a general topic. However, it does highlight several recent enhancements and features related to the Noveum.ai ...", + "evaluation.retrieval_effectiveness": 1, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 5, + "evaluation.context_relevance": "high", + "evaluation.overall_quality": "high", + "evaluation.ready_for_production": true, + "rag.retrieval_strategy": "semantic_similarity", + "rag.vector_search_results": 5, + "rag.context_synthesis": "multi_source", + "rag.document_coverage": 1, + "rag.information_density": 0.381155486622722 + }, + "events": [], + "links": [], + "trace_trace_id": "77b81bd5-5a04-4fc0-9fc6-571f7907d6b1", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:44.888000000", + "trace_end_time": "2025-10-16 08:50:50.529000000", + "trace_duration_ms": 5641.226, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:44.888000000", + "updated_at": "2025-10-16 08:50:44.888000000" + }, + { + "span_id": "4c391bb1-1a62-4e2a-913b-96f83feff494", + "trace_id": "32170d1a-d211-46d8-9cdc-5a4265f0c443", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-16 08:50:42.124000000", + "end_time": "2025-10-16 08:50:43.885000000", + "duration_ms": 1760.974, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What's the current status of cryptocurrency markets?", + "query.length": 52, + "processing.mode": "Web Search", + "processing.time_seconds": 1.7608659267425537, + "processing.response_length": 363, + "processing.sources_count": 1, + "output_response": "Final Answer: The web search did not yield any specific information regarding the current status of cryptocurrency markets. Therefore, I cannot provide an update on market trends, prices, or significant events affe...", + "final_answer_mode": "Web Search", + "query_processed.input_query": "What's the current status of cryptocurrency markets?", + "query_processed.output_response": "Successfully processed query using Web Search, generated 363 character response", + "query_processed.mode": "Web Search", + "query_processed.processing_time": 1.7608659267425537, + "query_processed.response_length": 363 + }, + "events": [], + "links": [], + "trace_trace_id": "32170d1a-d211-46d8-9cdc-5a4265f0c443", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:42.124000000", + "trace_end_time": "2025-10-16 08:50:43.885000000", + "trace_duration_ms": 1761.047, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:42.124000000", + "updated_at": "2025-10-16 08:50:42.124000000" + }, + { + "span_id": "7ab2ec7b-a617-4ebf-9bb1-82dc6b18e160", + "trace_id": "32170d1a-d211-46d8-9cdc-5a4265f0c443", + "parent_span_id": "4c391bb1-1a62-4e2a-913b-96f83feff494", + "name": "agent.query_routing", + "start_time": "2025-10-16 08:50:42.124000000", + "end_time": "2025-10-16 08:50:43.885000000", + "duration_ms": 1760.833, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What's the current status of cryptocurrency markets?", + "query_length": 52, + "query_type": "routing_query", + "output_response": "Routed to Web Search for query processing", + "router.classification": "Web Search", + "router.confidence_score": 1, + "router.rag_keyword_score": 0, + "router.web_keyword_score": 3, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "32170d1a-d211-46d8-9cdc-5a4265f0c443", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:42.124000000", + "trace_end_time": "2025-10-16 08:50:43.885000000", + "trace_duration_ms": 1761.047, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:42.124000000", + "updated_at": "2025-10-16 08:50:42.124000000" + }, + { + "span_id": "db01d228-7c63-4814-b2ea-bdf3898f9c0d", + "trace_id": "32170d1a-d211-46d8-9cdc-5a4265f0c443", + "parent_span_id": "7ab2ec7b-a617-4ebf-9bb1-82dc6b18e160", + "name": "agent.web_search_generation", + "start_time": "2025-10-16 08:50:42.124000000", + "end_time": "2025-10-16 08:50:43.885000000", + "duration_ms": 1760.701, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "web_search_agent", + "agent.operation": "web_search_generation", + "agent.capabilities": [ + "web_search", + "content_synthesis", + "response_generation" + ], + "agent.id": "noveum_web_search_agent", + "input_query": "What's the current status of cryptocurrency markets?", + "query_length": 52, + "query_type": "web_search_query", + "output_response": "Web Search Answer: The web search did not yield any specific information regarding the current status of cryptocurrency markets. Therefore, I cannot provide an update on market trends, prices, or significant events affe...", + "web_search.results_count": 1, + "web_search.context_length": 154, + "web_search.response_length": 363, + "web_search.mode": "external_web_search" + }, + "events": [], + "links": [], + "trace_trace_id": "32170d1a-d211-46d8-9cdc-5a4265f0c443", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:42.124000000", + "trace_end_time": "2025-10-16 08:50:43.885000000", + "trace_duration_ms": 1761.047, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:42.124000000", + "updated_at": "2025-10-16 08:50:42.124000000" + }, + { + "span_id": "da7b0550-b13c-4518-ad3b-050c666d198c", + "trace_id": "32170d1a-d211-46d8-9cdc-5a4265f0c443", + "parent_span_id": "7ab2ec7b-a617-4ebf-9bb1-82dc6b18e160", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-16 08:50:42.124000000", + "end_time": "2025-10-16 08:50:42.124000000", + "duration_ms": 0.029, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What's the current status of cryptocurrency markets?", + "query_length": 52, + "query_type": "routing_evaluation_query", + "classification.mode": "Web Search", + "classification.rag_keyword_score": 0, + "classification.web_keyword_score": 3, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "complex", + "query.intent": "general_knowledge", + "query.keyword_density": 0.42857142857142855, + "query.domain_affinity": "general", + "routing.decision": "Routed to Web Search based on analysis", + "routing.rationale": "RAG score: 0, Web score: 3, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "RAG", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.057692307692307696, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: Web Search (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "32170d1a-d211-46d8-9cdc-5a4265f0c443", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:42.124000000", + "trace_end_time": "2025-10-16 08:50:43.885000000", + "trace_duration_ms": 1761.047, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:42.124000000", + "updated_at": "2025-10-16 08:50:42.124000000" + }, + { + "span_id": "950e18b6-9f62-4ac4-8788-3cac08fb7899", + "trace_id": "32170d1a-d211-46d8-9cdc-5a4265f0c443", + "parent_span_id": "db01d228-7c63-4814-b2ea-bdf3898f9c0d", + "name": "agent.llm_model_execution", + "start_time": "2025-10-16 08:50:42.465000000", + "end_time": "2025-10-16 08:50:43.885000000", + "duration_ms": 1419.5690000000002, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What's the current status of cryptocurrency markets?", + "query_length": 52, + "query_type": "web_search_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 1.4195220470428467, + "model.latency_ms": 1419.5220470428467, + "model.start_time": 1760604642.4658613, + "model.end_time": 1760604643.8853834, + "model.performance_tier": "fast", + "model.prompt_tokens": 162, + "model.completion_tokens": 90, + "model.total_tokens": 252, + "model.tokens_per_second": 177.52454111224782, + "model.estimated_cost": 0.00756, + "model.efficiency_score": 1.4404761904761905, + "model.response_length": 363, + "model.response_quality": "high", + "model.output_response": "Model Response: The web search did not yield any specific information regarding the current status of cryptocurrency markets. Therefore, I cannot provide an update on market trends, prices, or significant events affe...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "32170d1a-d211-46d8-9cdc-5a4265f0c443", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:42.124000000", + "trace_end_time": "2025-10-16 08:50:43.885000000", + "trace_duration_ms": 1761.047, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:42.124000000", + "updated_at": "2025-10-16 08:50:42.124000000" + }, + { + "span_id": "e87846c9-ce82-42c2-a517-e0f67e9d14c1", + "trace_id": "32170d1a-d211-46d8-9cdc-5a4265f0c443", + "parent_span_id": "db01d228-7c63-4814-b2ea-bdf3898f9c0d", + "name": "agent.web_search_evaluation_metrics", + "start_time": "2025-10-16 08:50:43.885000000", + "end_time": "2025-10-16 08:50:43.885000000", + "duration_ms": 0.02, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "web_search_evaluation_metrics", + "agent.capabilities": [ + "web_search_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What's the current status of cryptocurrency markets?", + "query_length": 52, + "query_type": "web_search_evaluation_query", + "web_search.results_count": 1, + "web_search.context_length": 154, + "web_search.context_synthesized": "Context from 1 web sources", + "web_search.search_effectiveness": 0.2, + "web_search.context_quality": "low", + "web_search.source_diversity": 1, + "web_search.information_synthesis": "low", + "web_search.external_knowledge_utilization": 0.154, + "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What's+the+current+status+of+cryptocurrency+markets?\n\nUser Question: What's the current status of cryptocurrency markets?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", + "prompt.prompt_length": 651, + "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What's+the+current+status+of+cryptocurrency+markets?", + "prompt.instruction_following": "web_search_optimized", + "response.answer_length": 363, + "response.answer_completeness": "complete", + "response.response_quality": "medium", + "response.source_citation": 1, + "response.context_utilization": 2.357142857142857, + "output_response": "Web Search Answer: The web search did not yield any specific information regarding the current status of cryptocurrency markets. Therefore, I cannot provide an update on market trends, prices, or significant events affe...", + "evaluation.search_effectiveness": 0.2, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 1, + "evaluation.context_relevance": "low", + "evaluation.overall_quality": "low", + "evaluation.ready_for_production": true, + "web_search.search_strategy": "duckduckgo_api", + "web_search.real_time_data": true, + "web_search.external_sources": 1, + "web_search.information_freshness": "current", + "web_search.knowledge_synthesis": "single_source" + }, + "events": [], + "links": [], + "trace_trace_id": "32170d1a-d211-46d8-9cdc-5a4265f0c443", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:42.124000000", + "trace_end_time": "2025-10-16 08:50:43.885000000", + "trace_duration_ms": 1761.047, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:42.124000000", + "updated_at": "2025-10-16 08:50:42.124000000" + }, + { + "span_id": "305daacf-990b-4e23-8720-48900f75d9be", + "trace_id": "9424d420-deee-478b-b727-9dc791a7457c", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-16 08:50:31.476000000", + "end_time": "2025-10-16 08:50:41.121000000", + "duration_ms": 9644.752999999999, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What are the latest updates in Python programming?", + "query.length": 50, + "processing.mode": "Web Search", + "processing.time_seconds": 9.644662618637085, + "processing.response_length": 1408, + "processing.sources_count": 1, + "output_response": "Final Answer: The web search did not yield specific results regarding the latest updates in Python programming. However, I can provide some general information based on my knowledge up to October 2023.\n\nAs of late ...", + "final_answer_mode": "Web Search", + "query_processed.input_query": "What are the latest updates in Python programming?", + "query_processed.output_response": "Successfully processed query using Web Search, generated 1408 character response", + "query_processed.mode": "Web Search", + "query_processed.processing_time": 9.644662618637085, + "query_processed.response_length": 1408 + }, + "events": [], + "links": [], + "trace_trace_id": "9424d420-deee-478b-b727-9dc791a7457c", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:31.476000000", + "trace_end_time": "2025-10-16 08:50:41.121000000", + "trace_duration_ms": 9644.828, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:31.476000000", + "updated_at": "2025-10-16 08:50:31.476000000" + }, + { + "span_id": "56b4cc1b-0646-4dee-acb3-db98f3d44477", + "trace_id": "9424d420-deee-478b-b727-9dc791a7457c", + "parent_span_id": "fb234e30-69be-4d06-9248-c05bcc22f063", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-16 08:50:31.476000000", + "end_time": "2025-10-16 08:50:31.476000000", + "duration_ms": 0.036000000000000004, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What are the latest updates in Python programming?", + "query_length": 50, + "query_type": "routing_evaluation_query", + "classification.mode": "Web Search", + "classification.rag_keyword_score": 0, + "classification.web_keyword_score": 2, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "general_knowledge", + "query.keyword_density": 0.25, + "query.domain_affinity": "general", + "routing.decision": "Routed to Web Search based on analysis", + "routing.rationale": "RAG score: 0, Web score: 2, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "RAG", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.038461538461538464, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: Web Search (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "9424d420-deee-478b-b727-9dc791a7457c", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:31.476000000", + "trace_end_time": "2025-10-16 08:50:41.121000000", + "trace_duration_ms": 9644.828, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:31.476000000", + "updated_at": "2025-10-16 08:50:31.476000000" + }, + { + "span_id": "fb234e30-69be-4d06-9248-c05bcc22f063", + "trace_id": "9424d420-deee-478b-b727-9dc791a7457c", + "parent_span_id": "305daacf-990b-4e23-8720-48900f75d9be", + "name": "agent.query_routing", + "start_time": "2025-10-16 08:50:31.476000000", + "end_time": "2025-10-16 08:50:41.121000000", + "duration_ms": 9644.627, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What are the latest updates in Python programming?", + "query_length": 50, + "query_type": "routing_query", + "output_response": "Routed to Web Search for query processing", + "router.classification": "Web Search", + "router.confidence_score": 1, + "router.rag_keyword_score": 0, + "router.web_keyword_score": 2, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "9424d420-deee-478b-b727-9dc791a7457c", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:31.476000000", + "trace_end_time": "2025-10-16 08:50:41.121000000", + "trace_duration_ms": 9644.828, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:31.476000000", + "updated_at": "2025-10-16 08:50:31.476000000" + }, + { + "span_id": "f8a793a9-63ee-49b7-9931-d7dcc2bb0ca7", + "trace_id": "9424d420-deee-478b-b727-9dc791a7457c", + "parent_span_id": "fb234e30-69be-4d06-9248-c05bcc22f063", + "name": "agent.web_search_generation", + "start_time": "2025-10-16 08:50:31.477000000", + "end_time": "2025-10-16 08:50:41.121000000", + "duration_ms": 9644.478, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "web_search_agent", + "agent.operation": "web_search_generation", + "agent.capabilities": [ + "web_search", + "content_synthesis", + "response_generation" + ], + "agent.id": "noveum_web_search_agent", + "input_query": "What are the latest updates in Python programming?", + "query_length": 50, + "query_type": "web_search_query", + "output_response": "Web Search Answer: The web search did not yield specific results regarding the latest updates in Python programming. However, I can provide some general information based on my knowledge up to October 2023.\n\nAs of late ...", + "web_search.results_count": 1, + "web_search.context_length": 152, + "web_search.response_length": 1408, + "web_search.mode": "external_web_search" + }, + "events": [], + "links": [], + "trace_trace_id": "9424d420-deee-478b-b727-9dc791a7457c", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:31.476000000", + "trace_end_time": "2025-10-16 08:50:41.121000000", + "trace_duration_ms": 9644.828, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:31.476000000", + "updated_at": "2025-10-16 08:50:31.476000000" + }, + { + "span_id": "b968e3d3-d4aa-4357-85d8-34a612748fbd", + "trace_id": "9424d420-deee-478b-b727-9dc791a7457c", + "parent_span_id": "f8a793a9-63ee-49b7-9931-d7dcc2bb0ca7", + "name": "agent.llm_model_execution", + "start_time": "2025-10-16 08:50:32.016000000", + "end_time": "2025-10-16 08:50:41.121000000", + "duration_ms": 9105.344000000001, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What are the latest updates in Python programming?", + "query_length": 50, + "query_type": "web_search_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 9.105276584625244, + "model.latency_ms": 9105.276584625244, + "model.start_time": 1760604632.0161095, + "model.end_time": 1760604641.121386, + "model.performance_tier": "slow", + "model.prompt_tokens": 161, + "model.completion_tokens": 352, + "model.total_tokens": 513, + "model.tokens_per_second": 56.340957381374714, + "model.estimated_cost": 0.015390000000000001, + "model.efficiency_score": 2.7446393762183234, + "model.response_length": 1408, + "model.response_quality": "high", + "model.output_response": "Model Response: The web search did not yield specific results regarding the latest updates in Python programming. However, I can provide some general information based on my knowledge up to October 2023.\n\nAs of late ...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "9424d420-deee-478b-b727-9dc791a7457c", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:31.476000000", + "trace_end_time": "2025-10-16 08:50:41.121000000", + "trace_duration_ms": 9644.828, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:31.476000000", + "updated_at": "2025-10-16 08:50:31.476000000" + }, + { + "span_id": "80817711-e64c-4a9f-802e-2a9e36ff093d", + "trace_id": "9424d420-deee-478b-b727-9dc791a7457c", + "parent_span_id": "f8a793a9-63ee-49b7-9931-d7dcc2bb0ca7", + "name": "agent.web_search_evaluation_metrics", + "start_time": "2025-10-16 08:50:41.121000000", + "end_time": "2025-10-16 08:50:41.121000000", + "duration_ms": 0.02, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "web_search_evaluation_metrics", + "agent.capabilities": [ + "web_search_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What are the latest updates in Python programming?", + "query_length": 50, + "query_type": "web_search_evaluation_query", + "web_search.results_count": 1, + "web_search.context_length": 152, + "web_search.context_synthesized": "Context from 1 web sources", + "web_search.search_effectiveness": 0.2, + "web_search.context_quality": "low", + "web_search.source_diversity": 1, + "web_search.information_synthesis": "low", + "web_search.external_knowledge_utilization": 0.152, + "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What+are+the+latest+updates+in+Python+programming?\n\nUser Question: What are the latest updates in Python programming?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", + "prompt.prompt_length": 647, + "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What+are+the+latest+updates+in+Python+programming?", + "prompt.instruction_following": "web_search_optimized", + "response.answer_length": 1408, + "response.answer_completeness": "complete", + "response.response_quality": "medium", + "response.source_citation": 1, + "response.context_utilization": 9.263157894736842, + "output_response": "Web Search Answer: The web search did not yield specific results regarding the latest updates in Python programming. However, I can provide some general information based on my knowledge up to October 2023.\n\nAs of late ...", + "evaluation.search_effectiveness": 0.2, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 1, + "evaluation.context_relevance": "low", + "evaluation.overall_quality": "low", + "evaluation.ready_for_production": true, + "web_search.search_strategy": "duckduckgo_api", + "web_search.real_time_data": true, + "web_search.external_sources": 1, + "web_search.information_freshness": "current", + "web_search.knowledge_synthesis": "single_source" + }, + "events": [], + "links": [], + "trace_trace_id": "9424d420-deee-478b-b727-9dc791a7457c", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:31.476000000", + "trace_end_time": "2025-10-16 08:50:41.121000000", + "trace_duration_ms": 9644.828, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:31.476000000", + "updated_at": "2025-10-16 08:50:31.476000000" + }, + { + "span_id": "13af0f3c-bd5c-4024-8fa3-c9e347a1ef20", + "trace_id": "a3735d64-89bb-4fdf-a851-7a1f3515c52d", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-16 08:50:27.687000000", + "end_time": "2025-10-16 08:50:30.473000000", + "duration_ms": 2786.078, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What happened in tech news this week?", + "query.length": 37, + "processing.mode": "Web Search", + "processing.time_seconds": 2.7859997749328613, + "processing.response_length": 396, + "processing.sources_count": 1, + "output_response": "Final Answer: It appears that the web search did not yield any specific results regarding recent tech news. Unfortunately, I cannot provide a summary of this week's tech news based on the available information. For...", + "final_answer_mode": "Web Search", + "query_processed.input_query": "What happened in tech news this week?", + "query_processed.output_response": "Successfully processed query using Web Search, generated 396 character response", + "query_processed.mode": "Web Search", + "query_processed.processing_time": 2.7859997749328613, + "query_processed.response_length": 396 + }, + "events": [], + "links": [], + "trace_trace_id": "a3735d64-89bb-4fdf-a851-7a1f3515c52d", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:27.687000000", + "trace_end_time": "2025-10-16 08:50:30.473000000", + "trace_duration_ms": 2786.145, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:27.687000000", + "updated_at": "2025-10-16 08:50:27.687000000" + }, + { + "span_id": "47e8f375-35e6-4097-86ff-df73e8181c23", + "trace_id": "a3735d64-89bb-4fdf-a851-7a1f3515c52d", + "parent_span_id": "a4f1a7b3-dc35-4f13-96d3-b6dc8fb21a37", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-16 08:50:27.687000000", + "end_time": "2025-10-16 08:50:27.687000000", + "duration_ms": 0.038, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What happened in tech news this week?", + "query_length": 37, + "query_type": "routing_evaluation_query", + "classification.mode": "Web Search", + "classification.rag_keyword_score": 0, + "classification.web_keyword_score": 2, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "general_knowledge", + "query.keyword_density": 0.2857142857142857, + "query.domain_affinity": "general", + "routing.decision": "Routed to Web Search based on analysis", + "routing.rationale": "RAG score: 0, Web score: 2, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "RAG", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.038461538461538464, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: Web Search (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "a3735d64-89bb-4fdf-a851-7a1f3515c52d", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:27.687000000", + "trace_end_time": "2025-10-16 08:50:30.473000000", + "trace_duration_ms": 2786.145, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:27.687000000", + "updated_at": "2025-10-16 08:50:27.687000000" + }, + { + "span_id": "a4f1a7b3-dc35-4f13-96d3-b6dc8fb21a37", + "trace_id": "a3735d64-89bb-4fdf-a851-7a1f3515c52d", + "parent_span_id": "13af0f3c-bd5c-4024-8fa3-c9e347a1ef20", + "name": "agent.query_routing", + "start_time": "2025-10-16 08:50:27.687000000", + "end_time": "2025-10-16 08:50:30.473000000", + "duration_ms": 2785.964, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What happened in tech news this week?", + "query_length": 37, + "query_type": "routing_query", + "output_response": "Routed to Web Search for query processing", + "router.classification": "Web Search", + "router.confidence_score": 1, + "router.rag_keyword_score": 0, + "router.web_keyword_score": 2, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "a3735d64-89bb-4fdf-a851-7a1f3515c52d", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:27.687000000", + "trace_end_time": "2025-10-16 08:50:30.473000000", + "trace_duration_ms": 2786.145, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:27.687000000", + "updated_at": "2025-10-16 08:50:27.687000000" + }, + { + "span_id": "ebfff280-5eb6-4838-8de7-35c85e2e3eaa", + "trace_id": "a3735d64-89bb-4fdf-a851-7a1f3515c52d", + "parent_span_id": "a4f1a7b3-dc35-4f13-96d3-b6dc8fb21a37", + "name": "agent.web_search_generation", + "start_time": "2025-10-16 08:50:27.687000000", + "end_time": "2025-10-16 08:50:30.473000000", + "duration_ms": 2785.8230000000003, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "web_search_agent", + "agent.operation": "web_search_generation", + "agent.capabilities": [ + "web_search", + "content_synthesis", + "response_generation" + ], + "agent.id": "noveum_web_search_agent", + "input_query": "What happened in tech news this week?", + "query_length": 37, + "query_type": "web_search_query", + "output_response": "Web Search Answer: It appears that the web search did not yield any specific results regarding recent tech news. Unfortunately, I cannot provide a summary of this week's tech news based on the available information. For...", + "web_search.results_count": 1, + "web_search.context_length": 139, + "web_search.response_length": 396, + "web_search.mode": "external_web_search" + }, + "events": [], + "links": [], + "trace_trace_id": "a3735d64-89bb-4fdf-a851-7a1f3515c52d", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:27.687000000", + "trace_end_time": "2025-10-16 08:50:30.473000000", + "trace_duration_ms": 2786.145, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:27.687000000", + "updated_at": "2025-10-16 08:50:27.687000000" + }, + { + "span_id": "835b9351-d381-4785-8538-fbe1064fc8cd", + "trace_id": "a3735d64-89bb-4fdf-a851-7a1f3515c52d", + "parent_span_id": "ebfff280-5eb6-4838-8de7-35c85e2e3eaa", + "name": "agent.llm_model_execution", + "start_time": "2025-10-16 08:50:28.026000000", + "end_time": "2025-10-16 08:50:30.473000000", + "duration_ms": 2447.269, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What happened in tech news this week?", + "query_length": 37, + "query_type": "web_search_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 2.447227716445923, + "model.latency_ms": 2447.227716445923, + "model.start_time": 1760604628.0260897, + "model.end_time": 1760604630.4733174, + "model.performance_tier": "medium", + "model.prompt_tokens": 155, + "model.completion_tokens": 99, + "model.total_tokens": 254, + "model.tokens_per_second": 103.79091340501853, + "model.estimated_cost": 0.00762, + "model.efficiency_score": 1.5590551181102361, + "model.response_length": 396, + "model.response_quality": "high", + "model.output_response": "Model Response: It appears that the web search did not yield any specific results regarding recent tech news. Unfortunately, I cannot provide a summary of this week's tech news based on the available information. For...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "a3735d64-89bb-4fdf-a851-7a1f3515c52d", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:27.687000000", + "trace_end_time": "2025-10-16 08:50:30.473000000", + "trace_duration_ms": 2786.145, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:27.687000000", + "updated_at": "2025-10-16 08:50:27.687000000" + }, + { + "span_id": "93c6f427-6fcd-4318-aec8-095553fed469", + "trace_id": "a3735d64-89bb-4fdf-a851-7a1f3515c52d", + "parent_span_id": "ebfff280-5eb6-4838-8de7-35c85e2e3eaa", + "name": "agent.web_search_evaluation_metrics", + "start_time": "2025-10-16 08:50:30.473000000", + "end_time": "2025-10-16 08:50:30.473000000", + "duration_ms": 0.017, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "web_search_evaluation_metrics", + "agent.capabilities": [ + "web_search_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What happened in tech news this week?", + "query_length": 37, + "query_type": "web_search_evaluation_query", + "web_search.results_count": 1, + "web_search.context_length": 139, + "web_search.context_synthesized": "Context from 1 web sources", + "web_search.search_effectiveness": 0.2, + "web_search.context_quality": "low", + "web_search.source_diversity": 1, + "web_search.information_synthesis": "low", + "web_search.external_knowledge_utilization": 0.139, + "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What+happened+in+tech+news+this+week?\n\nUser Question: What happened in tech news this week?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", + "prompt.prompt_length": 621, + "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What+happened+in+tech+news+this+week?", + "prompt.instruction_following": "web_search_optimized", + "response.answer_length": 396, + "response.answer_completeness": "complete", + "response.response_quality": "medium", + "response.source_citation": 1, + "response.context_utilization": 2.8489208633093526, + "output_response": "Web Search Answer: It appears that the web search did not yield any specific results regarding recent tech news. Unfortunately, I cannot provide a summary of this week's tech news based on the available information. For...", + "evaluation.search_effectiveness": 0.2, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 1, + "evaluation.context_relevance": "low", + "evaluation.overall_quality": "low", + "evaluation.ready_for_production": true, + "web_search.search_strategy": "duckduckgo_api", + "web_search.real_time_data": true, + "web_search.external_sources": 1, + "web_search.information_freshness": "current", + "web_search.knowledge_synthesis": "single_source" + }, + "events": [], + "links": [], + "trace_trace_id": "a3735d64-89bb-4fdf-a851-7a1f3515c52d", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:27.687000000", + "trace_end_time": "2025-10-16 08:50:30.473000000", + "trace_duration_ms": 2786.145, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:27.687000000", + "updated_at": "2025-10-16 08:50:27.687000000" + }, + { + "span_id": "622e7353-4eeb-47d5-8558-4ecec28cdcfd", + "trace_id": "fa5bb641-f4b7-4741-b06c-392a5f2a9aea", + "parent_span_id": "c0af6629-48b1-45fc-9dd2-60f098ea140e", + "name": "agent.query_routing", + "start_time": "2025-10-16 08:50:21.645000000", + "end_time": "2025-10-16 08:50:26.683000000", + "duration_ms": 5037.965, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What are the current trends in observability tools?", + "query_length": 51, + "query_type": "routing_query", + "output_response": "Routed to RAG for query processing", + "router.classification": "RAG", + "router.confidence_score": 0.3333333333333333, + "router.rag_keyword_score": 2, + "router.web_keyword_score": 1, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "fa5bb641-f4b7-4741-b06c-392a5f2a9aea", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:21.645000000", + "trace_end_time": "2025-10-16 08:50:26.683000000", + "trace_duration_ms": 5038.112, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:21.645000000", + "updated_at": "2025-10-16 08:50:21.645000000" + }, + { + "span_id": "c0af6629-48b1-45fc-9dd2-60f098ea140e", + "trace_id": "fa5bb641-f4b7-4741-b06c-392a5f2a9aea", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-16 08:50:21.645000000", + "end_time": "2025-10-16 08:50:26.683000000", + "duration_ms": 5038.0419999999995, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What are the current trends in observability tools?", + "query.length": 51, + "processing.mode": "RAG", + "processing.time_seconds": 5.0380027294158936, + "processing.response_length": 897, + "processing.sources_count": 5, + "output_response": "Final Answer: The provided context does not specifically outline current trends in observability tools beyond the focus on AI applications. However, it highlights the limitations of traditional monitoring tools for...", + "final_answer_mode": "RAG", + "query_processed.input_query": "What are the current trends in observability tools?", + "query_processed.output_response": "Successfully processed query using RAG, generated 897 character response", + "query_processed.mode": "RAG", + "query_processed.processing_time": 5.0380027294158936, + "query_processed.response_length": 897 + }, + "events": [], + "links": [], + "trace_trace_id": "fa5bb641-f4b7-4741-b06c-392a5f2a9aea", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:21.645000000", + "trace_end_time": "2025-10-16 08:50:26.683000000", + "trace_duration_ms": 5038.112, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:21.645000000", + "updated_at": "2025-10-16 08:50:21.645000000" + }, + { + "span_id": "ee55906d-ffe3-42fd-9899-a476d4c33ff5", + "trace_id": "fa5bb641-f4b7-4741-b06c-392a5f2a9aea", + "parent_span_id": "622e7353-4eeb-47d5-8558-4ecec28cdcfd", + "name": "agent.llm-rag", + "start_time": "2025-10-16 08:50:21.645000000", + "end_time": "2025-10-16 08:50:26.683000000", + "duration_ms": 5037.803, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "rag_agent", + "agent.operation": "llm-rag", + "agent.capabilities": [ + "document_retrieval", + "context_generation", + "response_generation" + ], + "agent.id": "noveum_rag_agent", + "input_query": "What are the current trends in observability tools?", + "query_length": 51, + "query_type": "rag_query", + "output_response": "RAG Answer: The provided context does not specifically outline current trends in observability tools beyond the focus on AI applications. However, it highlights the limitations of traditional monitoring tools for...", + "rag.context_length": 2943, + "rag.sources_count": 5, + "rag.answer_length": 897, + "rag.mode": "retrieval_augmented_generation" + }, + "events": [], + "links": [], + "trace_trace_id": "fa5bb641-f4b7-4741-b06c-392a5f2a9aea", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:21.645000000", + "trace_end_time": "2025-10-16 08:50:26.683000000", + "trace_duration_ms": 5038.112, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:21.645000000", + "updated_at": "2025-10-16 08:50:21.645000000" + }, + { + "span_id": "fe610ecb-cc8e-4466-b719-34c6e0d45d87", + "trace_id": "fa5bb641-f4b7-4741-b06c-392a5f2a9aea", + "parent_span_id": "622e7353-4eeb-47d5-8558-4ecec28cdcfd", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-16 08:50:21.645000000", + "end_time": "2025-10-16 08:50:21.645000000", + "duration_ms": 0.032, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What are the current trends in observability tools?", + "query_length": 51, + "query_type": "routing_evaluation_query", + "classification.mode": "RAG", + "classification.rag_keyword_score": 2, + "classification.web_keyword_score": 1, + "classification.confidence_score": 0.3333333333333333, + "classification.confidence_level": "medium", + "classification.method": "keyword_based", + "query.complexity": "complex", + "query.intent": "documentation", + "query.keyword_density": 0.375, + "query.domain_affinity": "noveum", + "routing.decision": "Routed to RAG based on analysis", + "routing.rationale": "RAG score: 2, Web score: 1, Confidence: 0.33", + "routing.expected_performance": "medium", + "routing.alternative_mode": "Web Search", + "routing.decision_confidence": 0.3333333333333333, + "evaluation.routing_accuracy": "medium", + "evaluation.keyword_coverage": 0.057692307692307696, + "evaluation.query_understanding": "ambiguous", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: RAG (Confidence: 0.33)" + }, + "events": [], + "links": [], + "trace_trace_id": "fa5bb641-f4b7-4741-b06c-392a5f2a9aea", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:21.645000000", + "trace_end_time": "2025-10-16 08:50:26.683000000", + "trace_duration_ms": 5038.112, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:21.645000000", + "updated_at": "2025-10-16 08:50:21.645000000" + }, + { + "span_id": "69a4773d-c738-489e-87ed-999583e02187", + "trace_id": "fa5bb641-f4b7-4741-b06c-392a5f2a9aea", + "parent_span_id": "ee55906d-ffe3-42fd-9899-a476d4c33ff5", + "name": "agent.llm_model_execution", + "start_time": "2025-10-16 08:50:22.600000000", + "end_time": "2025-10-16 08:50:25.786000000", + "duration_ms": 3185.257, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What are the current trends in observability tools?", + "query_length": 51, + "query_type": "rag_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 3.185206174850464, + "model.latency_ms": 3185.206174850464, + "model.start_time": 1760604622.6008554, + "model.end_time": 1760604625.7860615, + "model.performance_tier": "medium", + "model.prompt_tokens": 872, + "model.completion_tokens": 224, + "model.total_tokens": 1096, + "model.tokens_per_second": 344.09075577390337, + "model.estimated_cost": 0.03288, + "model.efficiency_score": 0.8184306569343066, + "model.response_length": 897, + "model.response_quality": "high", + "model.output_response": "Model Response: The provided context does not specifically outline current trends in observability tools beyond the focus on AI applications. However, it highlights the limitations of traditional monitoring tools for...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "fa5bb641-f4b7-4741-b06c-392a5f2a9aea", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:21.645000000", + "trace_end_time": "2025-10-16 08:50:26.683000000", + "trace_duration_ms": 5038.112, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:21.645000000", + "updated_at": "2025-10-16 08:50:21.645000000" + }, + { + "span_id": "5e5a9d30-4cd0-45cd-9494-192c280727ac", + "trace_id": "fa5bb641-f4b7-4741-b06c-392a5f2a9aea", + "parent_span_id": "ee55906d-ffe3-42fd-9899-a476d4c33ff5", + "name": "agent.rag_evaluation_metrics", + "start_time": "2025-10-16 08:50:25.786000000", + "end_time": "2025-10-16 08:50:25.786000000", + "duration_ms": 0.066, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "rag_evaluation_metrics", + "agent.capabilities": [ + "retrieval_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What are the current trends in observability tools?", + "query_length": 51, + "query_type": "rag_evaluation_query", + "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/blog/from-logs-to-intelligent-choices-inside-noveum-ais-evaluation-process):\nFrom Development to Production - Inside Noveum.ai's AI Observability Platform\nShashank Agarwal\n3/3/2025\nIntroduction\nEvery AI application tells a story\u2014but most developers never get to hear it...", + "retrieval.context_length": 2943, + "retrieval.sources_count": 5, + "retrieval.context_quality": "high", + "retrieval.effectiveness": 1, + "retrieval.context_utilization": 2.943, + "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/blog/from-logs-to-intelligent-choices-inside-noveum-ais-evaluation-process):\nFrom Development to Production - Inside Noveum.ai's AI Observability Platform\nShashank Agarwal\n3/3/2025\nIntroduction\nEvery AI application tells a story\u2014but most developers never get to hear it. When your RAG pipeline returns irrelevant results, when your multi-agent system gets stuck in loops, or when your LLM costs suddenly spike, you're left guessing what went wrong and where.\nThat's exactly the challenge Noveum.ai solves. Rather than flying blind, our platform provides comprehensive tracing a...\n\nSource 2 (https://noveum.ai/docs/getting-started/tracing-concepts):\n- Explore Framework Integrations - Framework-specific guidance\n- Learn Advanced Patterns - Custom instrumentation techniques\n- Master the Dashboard - Analyze your traces effectively\nRemember: Good observability is not about collecting all possible data, but about collecting the right data that helps you understand, debug, and optimize your AI applications.\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to O...\n\nSource 3 (https://noveum.ai/en/docs/getting-started/tracing-concepts):\n- Explore Framework Integrations - Framework-specific guidance\n- Learn Advanced Patterns - Custom instrumentation techniques\n- Master the Dashboard - Analyze your traces effectively\nRemember: Good observability is not about collecting all possible data, but about collecting the right data that helps you understand, debug, and optimize your AI applications.\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to O...\n\nSource 4 (https://noveum.ai/en/blog/from-logs-to-intelligent-choices-inside-noveum-ais-evaluation-process):\nThe AI Observability Challenge\nWhy Traditional Monitoring Falls Short\nTraditional application monitoring tools weren't built for AI applications. They can tell you if your API is responding, but they can't answer the questions that matter most for AI systems:\n- Why did my RAG pipeline retrieve irrelevant documents?\n- Which LLM calls are driving my costs?\n- How are my agents communicating with each other?\n- What's causing hallucinations in my responses?\n- Why is my embedding generation so slow?\nW...\n\nSource 5 (https://noveum.ai/en/docs/advanced/rag-observability):\n\ud83d\udd27 Best Practices for RAG Observability\n1. Comprehensive Pipeline Tracing\n2. Quality Monitoring\n3. A/B Testing for RAG Components\n\ud83c\udfaf Advanced RAG Patterns\nMulti-Modal RAG\nConversational RAG\nRAG observability with Noveum.ai provides the deep insights needed to build, optimize, and scale retrieval-augmented generation systems. By implementing comprehensive tracing across retrieval, context preparation, and generation stages, you can ensure your RAG pipeline delivers accurate, relevant, and cost-effe...\n\nUser Question: What are the current trends in observability tools?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", + "prompt.prompt_length": 3490, + "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/blog/from-logs-to-intelligent-choices-inside-noveum-ais-evaluation-process):\nFrom Development to Production - Inside Noveum.ai's AI Observability Platform\nShashank Agarw...", + "prompt.instruction_following": "rag_optimized", + "response.answer_length": 897, + "response.answer_completeness": "complete", + "response.response_quality": "high", + "response.source_citation": 5, + "response.context_utilization": 0.30479102956167176, + "output_response": "RAG Answer: The provided context does not specifically outline current trends in observability tools beyond the focus on AI applications. However, it highlights the limitations of traditional monitoring tools for...", + "evaluation.retrieval_effectiveness": 1, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 5, + "evaluation.context_relevance": "high", + "evaluation.overall_quality": "high", + "evaluation.ready_for_production": true, + "rag.retrieval_strategy": "semantic_similarity", + "rag.vector_search_results": 5, + "rag.context_synthesis": "multi_source", + "rag.document_coverage": 1, + "rag.information_density": 0.30479102956167176 + }, + "events": [], + "links": [], + "trace_trace_id": "fa5bb641-f4b7-4741-b06c-392a5f2a9aea", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:21.645000000", + "trace_end_time": "2025-10-16 08:50:26.683000000", + "trace_duration_ms": 5038.112, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:21.645000000", + "updated_at": "2025-10-16 08:50:21.645000000" + }, + { + "span_id": "2e06953f-5cec-4873-92b5-91f7281f6d7c", + "trace_id": "c0a95e8e-513c-47eb-b746-4e6016aeaf7d", + "parent_span_id": "9e41eab5-cd8e-4e46-a15b-e5f52c9a4073", + "name": "agent.query_routing", + "start_time": "2025-10-16 08:50:08.944000000", + "end_time": "2025-10-16 08:50:20.641000000", + "duration_ms": 11696.524, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "Tell me about recent developments in machine learning", + "query_length": 53, + "query_type": "routing_query", + "output_response": "Routed to Web Search for query processing", + "router.classification": "Web Search", + "router.confidence_score": 1, + "router.rag_keyword_score": 0, + "router.web_keyword_score": 1, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "c0a95e8e-513c-47eb-b746-4e6016aeaf7d", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:08.944000000", + "trace_end_time": "2025-10-16 08:50:20.641000000", + "trace_duration_ms": 11696.768, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:08.944000000", + "updated_at": "2025-10-16 08:50:08.944000000" + }, + { + "span_id": "9e41eab5-cd8e-4e46-a15b-e5f52c9a4073", + "trace_id": "c0a95e8e-513c-47eb-b746-4e6016aeaf7d", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-16 08:50:08.944000000", + "end_time": "2025-10-16 08:50:20.641000000", + "duration_ms": 11696.671999999999, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "Tell me about recent developments in machine learning", + "query.length": 53, + "processing.mode": "Web Search", + "processing.time_seconds": 11.696574449539185, + "processing.response_length": 2001, + "processing.sources_count": 1, + "output_response": "Final Answer: The web search did not yield specific results regarding recent developments in machine learning. However, I can provide a general overview based on my knowledge up to October 2023.\n\nRecent development...", + "final_answer_mode": "Web Search", + "query_processed.input_query": "Tell me about recent developments in machine learning", + "query_processed.output_response": "Successfully processed query using Web Search, generated 2001 character response", + "query_processed.mode": "Web Search", + "query_processed.processing_time": 11.696574449539185, + "query_processed.response_length": 2001 + }, + "events": [], + "links": [], + "trace_trace_id": "c0a95e8e-513c-47eb-b746-4e6016aeaf7d", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:08.944000000", + "trace_end_time": "2025-10-16 08:50:20.641000000", + "trace_duration_ms": 11696.768, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:08.944000000", + "updated_at": "2025-10-16 08:50:08.944000000" + }, + { + "span_id": "7ddf938b-183a-427f-849b-b31b18a98d87", + "trace_id": "c0a95e8e-513c-47eb-b746-4e6016aeaf7d", + "parent_span_id": "2e06953f-5cec-4873-92b5-91f7281f6d7c", + "name": "agent.web_search_generation", + "start_time": "2025-10-16 08:50:08.945000000", + "end_time": "2025-10-16 08:50:20.641000000", + "duration_ms": 11696.328, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "web_search_agent", + "agent.operation": "web_search_generation", + "agent.capabilities": [ + "web_search", + "content_synthesis", + "response_generation" + ], + "agent.id": "noveum_web_search_agent", + "input_query": "Tell me about recent developments in machine learning", + "query_length": 53, + "query_type": "web_search_query", + "output_response": "Web Search Answer: The web search did not yield specific results regarding recent developments in machine learning. However, I can provide a general overview based on my knowledge up to October 2023.\n\nRecent development...", + "web_search.results_count": 1, + "web_search.context_length": 155, + "web_search.response_length": 2001, + "web_search.mode": "external_web_search" + }, + "events": [], + "links": [], + "trace_trace_id": "c0a95e8e-513c-47eb-b746-4e6016aeaf7d", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:08.944000000", + "trace_end_time": "2025-10-16 08:50:20.641000000", + "trace_duration_ms": 11696.768, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:08.944000000", + "updated_at": "2025-10-16 08:50:08.944000000" + }, + { + "span_id": "f3e4c594-f822-4f04-ab43-98fac161b9a3", + "trace_id": "c0a95e8e-513c-47eb-b746-4e6016aeaf7d", + "parent_span_id": "2e06953f-5cec-4873-92b5-91f7281f6d7c", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-16 08:50:08.945000000", + "end_time": "2025-10-16 08:50:08.945000000", + "duration_ms": 0.046, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: Tell me about recent developments in machine learning", + "query_length": 53, + "query_type": "routing_evaluation_query", + "classification.mode": "Web Search", + "classification.rag_keyword_score": 0, + "classification.web_keyword_score": 1, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "complex", + "query.intent": "general_knowledge", + "query.keyword_density": 0.125, + "query.domain_affinity": "general", + "routing.decision": "Routed to Web Search based on analysis", + "routing.rationale": "RAG score: 0, Web score: 1, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "RAG", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.019230769230769232, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: Web Search (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "c0a95e8e-513c-47eb-b746-4e6016aeaf7d", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:08.944000000", + "trace_end_time": "2025-10-16 08:50:20.641000000", + "trace_duration_ms": 11696.768, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:08.944000000", + "updated_at": "2025-10-16 08:50:08.944000000" + }, + { + "span_id": "b47d09be-a43b-4b0a-b7d3-d98bdf1773a3", + "trace_id": "c0a95e8e-513c-47eb-b746-4e6016aeaf7d", + "parent_span_id": "7ddf938b-183a-427f-849b-b31b18a98d87", + "name": "agent.llm_model_execution", + "start_time": "2025-10-16 08:50:09.490000000", + "end_time": "2025-10-16 08:50:20.641000000", + "duration_ms": 11151.141000000001, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: Tell me about recent developments in machine learning", + "query_length": 53, + "query_type": "web_search_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 11.151098012924194, + "model.latency_ms": 11151.098012924194, + "model.start_time": 1760604609.4902723, + "model.end_time": 1760604620.6413703, + "model.performance_tier": "slow", + "model.prompt_tokens": 163, + "model.completion_tokens": 500, + "model.total_tokens": 663, + "model.tokens_per_second": 59.456028386763236, + "model.estimated_cost": 0.01989, + "model.efficiency_score": 3.018099547511312, + "model.response_length": 2001, + "model.response_quality": "high", + "model.output_response": "Model Response: The web search did not yield specific results regarding recent developments in machine learning. However, I can provide a general overview based on my knowledge up to October 2023.\n\nRecent development...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "c0a95e8e-513c-47eb-b746-4e6016aeaf7d", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:08.944000000", + "trace_end_time": "2025-10-16 08:50:20.641000000", + "trace_duration_ms": 11696.768, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:08.944000000", + "updated_at": "2025-10-16 08:50:08.944000000" + }, + { + "span_id": "bfea8500-0e5c-4556-a014-5b2ec49b7ad5", + "trace_id": "c0a95e8e-513c-47eb-b746-4e6016aeaf7d", + "parent_span_id": "7ddf938b-183a-427f-849b-b31b18a98d87", + "name": "agent.web_search_evaluation_metrics", + "start_time": "2025-10-16 08:50:20.641000000", + "end_time": "2025-10-16 08:50:20.641000000", + "duration_ms": 0.018000000000000002, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "web_search_evaluation_metrics", + "agent.capabilities": [ + "web_search_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: Tell me about recent developments in machine learning", + "query_length": 53, + "query_type": "web_search_evaluation_query", + "web_search.results_count": 1, + "web_search.context_length": 155, + "web_search.context_synthesized": "Context from 1 web sources", + "web_search.search_effectiveness": 0.2, + "web_search.context_quality": "low", + "web_search.source_diversity": 1, + "web_search.information_synthesis": "low", + "web_search.external_knowledge_utilization": 0.155, + "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=Tell+me+about+recent+developments+in+machine+learning\n\nUser Question: Tell me about recent developments in machine learning\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", + "prompt.prompt_length": 653, + "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=Tell+me+about+recent+developments+in+machine+learning", + "prompt.instruction_following": "web_search_optimized", + "response.answer_length": 2001, + "response.answer_completeness": "complete", + "response.response_quality": "medium", + "response.source_citation": 1, + "response.context_utilization": 12.90967741935484, + "output_response": "Web Search Answer: The web search did not yield specific results regarding recent developments in machine learning. However, I can provide a general overview based on my knowledge up to October 2023.\n\nRecent development...", + "evaluation.search_effectiveness": 0.2, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 1, + "evaluation.context_relevance": "low", + "evaluation.overall_quality": "low", + "evaluation.ready_for_production": true, + "web_search.search_strategy": "duckduckgo_api", + "web_search.real_time_data": true, + "web_search.external_sources": 1, + "web_search.information_freshness": "current", + "web_search.knowledge_synthesis": "single_source" + }, + "events": [], + "links": [], + "trace_trace_id": "c0a95e8e-513c-47eb-b746-4e6016aeaf7d", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:08.944000000", + "trace_end_time": "2025-10-16 08:50:20.641000000", + "trace_duration_ms": 11696.768, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:08.944000000", + "updated_at": "2025-10-16 08:50:08.944000000" + }, + { + "span_id": "33297286-d440-446a-ba73-7cabd9d4b2e6", + "trace_id": "3af8c6e8-62d9-4893-92ba-cc265934ef03", + "parent_span_id": "4a3c230e-0dd5-4ba2-8e87-2a301f654fba", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-16 08:50:04.544000000", + "end_time": "2025-10-16 08:50:04.544000000", + "duration_ms": 0.038, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What's the weather like today?", + "query_length": 30, + "query_type": "routing_evaluation_query", + "classification.mode": "Web Search", + "classification.rag_keyword_score": 0, + "classification.web_keyword_score": 2, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "general_knowledge", + "query.keyword_density": 0.4, + "query.domain_affinity": "general", + "routing.decision": "Routed to Web Search based on analysis", + "routing.rationale": "RAG score: 0, Web score: 2, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "RAG", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.038461538461538464, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: Web Search (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "3af8c6e8-62d9-4893-92ba-cc265934ef03", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:04.544000000", + "trace_end_time": "2025-10-16 08:50:07.941000000", + "trace_duration_ms": 3397.228, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:04.544000000", + "updated_at": "2025-10-16 08:50:04.544000000" + }, + { + "span_id": "4a3c230e-0dd5-4ba2-8e87-2a301f654fba", + "trace_id": "3af8c6e8-62d9-4893-92ba-cc265934ef03", + "parent_span_id": "eefcbdc5-99fc-4f06-a9f4-f51ba50564db", + "name": "agent.query_routing", + "start_time": "2025-10-16 08:50:04.544000000", + "end_time": "2025-10-16 08:50:07.941000000", + "duration_ms": 3397.056, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What's the weather like today?", + "query_length": 30, + "query_type": "routing_query", + "output_response": "Routed to Web Search for query processing", + "router.classification": "Web Search", + "router.confidence_score": 1, + "router.rag_keyword_score": 0, + "router.web_keyword_score": 2, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "3af8c6e8-62d9-4893-92ba-cc265934ef03", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:04.544000000", + "trace_end_time": "2025-10-16 08:50:07.941000000", + "trace_duration_ms": 3397.228, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:04.544000000", + "updated_at": "2025-10-16 08:50:04.544000000" + }, + { + "span_id": "eefcbdc5-99fc-4f06-a9f4-f51ba50564db", + "trace_id": "3af8c6e8-62d9-4893-92ba-cc265934ef03", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-16 08:50:04.544000000", + "end_time": "2025-10-16 08:50:07.941000000", + "duration_ms": 3397.169, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What's the weather like today?", + "query.length": 30, + "processing.mode": "Web Search", + "processing.time_seconds": 3.3970956802368164, + "processing.response_length": 518, + "processing.sources_count": 1, + "output_response": "Final Answer: The web search results do not provide specific details about today's weather conditions in your location. However, they mention that you can get real-time updates, including temperature, humidity, and...", + "final_answer_mode": "Web Search", + "query_processed.input_query": "What's the weather like today?", + "query_processed.output_response": "Successfully processed query using Web Search, generated 518 character response", + "query_processed.mode": "Web Search", + "query_processed.processing_time": 3.3970956802368164, + "query_processed.response_length": 518 + }, + "events": [], + "links": [], + "trace_trace_id": "3af8c6e8-62d9-4893-92ba-cc265934ef03", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:04.544000000", + "trace_end_time": "2025-10-16 08:50:07.941000000", + "trace_duration_ms": 3397.228, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:04.544000000", + "updated_at": "2025-10-16 08:50:04.544000000" + }, + { + "span_id": "9683465c-e851-4004-a858-c8fdd1a2b3da", + "trace_id": "3af8c6e8-62d9-4893-92ba-cc265934ef03", + "parent_span_id": "4a3c230e-0dd5-4ba2-8e87-2a301f654fba", + "name": "agent.web_search_generation", + "start_time": "2025-10-16 08:50:04.545000000", + "end_time": "2025-10-16 08:50:07.941000000", + "duration_ms": 3396.871, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "web_search_agent", + "agent.operation": "web_search_generation", + "agent.capabilities": [ + "web_search", + "content_synthesis", + "response_generation" + ], + "agent.id": "noveum_web_search_agent", + "input_query": "What's the weather like today?", + "query_length": 30, + "query_type": "web_search_query", + "output_response": "Web Search Answer: The web search results do not provide specific details about today's weather conditions in your location. However, they mention that you can get real-time updates, including temperature, humidity, and...", + "web_search.results_count": 1, + "web_search.context_length": 1137, + "web_search.response_length": 518, + "web_search.mode": "external_web_search" + }, + "events": [], + "links": [], + "trace_trace_id": "3af8c6e8-62d9-4893-92ba-cc265934ef03", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:04.544000000", + "trace_end_time": "2025-10-16 08:50:07.941000000", + "trace_duration_ms": 3397.228, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:04.544000000", + "updated_at": "2025-10-16 08:50:04.544000000" + }, + { + "span_id": "6413eb7e-e5cd-4c46-9651-de14c72c260e", + "trace_id": "3af8c6e8-62d9-4893-92ba-cc265934ef03", + "parent_span_id": "9683465c-e851-4004-a858-c8fdd1a2b3da", + "name": "agent.llm_model_execution", + "start_time": "2025-10-16 08:50:04.986000000", + "end_time": "2025-10-16 08:50:07.941000000", + "duration_ms": 2955.55, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What's the weather like today?", + "query_length": 30, + "query_type": "web_search_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 2.955509662628174, + "model.latency_ms": 2955.509662628174, + "model.start_time": 1760604604.9862766, + "model.end_time": 1760604607.9417863, + "model.performance_tier": "medium", + "model.prompt_tokens": 403, + "model.completion_tokens": 129, + "model.total_tokens": 532, + "model.tokens_per_second": 180.0027950261957, + "model.estimated_cost": 0.015960000000000002, + "model.efficiency_score": 0.9736842105263158, + "model.response_length": 518, + "model.response_quality": "high", + "model.output_response": "Model Response: The web search results do not provide specific details about today's weather conditions in your location. However, they mention that you can get real-time updates, including temperature, humidity, and...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "3af8c6e8-62d9-4893-92ba-cc265934ef03", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:04.544000000", + "trace_end_time": "2025-10-16 08:50:07.941000000", + "trace_duration_ms": 3397.228, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:04.544000000", + "updated_at": "2025-10-16 08:50:04.544000000" + }, + { + "span_id": "64df69fb-0bcc-4845-8bfe-5a721c1ed912", + "trace_id": "3af8c6e8-62d9-4893-92ba-cc265934ef03", + "parent_span_id": "9683465c-e851-4004-a858-c8fdd1a2b3da", + "name": "agent.web_search_evaluation_metrics", + "start_time": "2025-10-16 08:50:07.941000000", + "end_time": "2025-10-16 08:50:07.941000000", + "duration_ms": 0.017, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "web_search_evaluation_metrics", + "agent.capabilities": [ + "web_search_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What's the weather like today?", + "query_length": 30, + "query_type": "web_search_evaluation_query", + "web_search.results_count": 1, + "web_search.context_length": 1137, + "web_search.context_synthesized": "Context from 1 web sources", + "web_search.search_effectiveness": 0.2, + "web_search.context_quality": "high", + "web_search.source_diversity": 1, + "web_search.information_synthesis": "low", + "web_search.external_knowledge_utilization": 1.137, + "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nStay updated with the latest weather conditions in Komna. Get real-time updates, temperature, humidity, AQI, and detailed forecast for today. Dec 31, 2024 \u00b7 Komna Weather Forecast: Check the latest temperature in Komna today along with details such as humidity, visibility, air quality, wind speed, pressure, and weekly weather report at Shuru. 2 days ago \u00b7 Latest weather forecast for India for today's, hourly weather forecast, including today's temperatures in India, wind, rain and more. 2 days ago \u00b7 Get the latest hourly weather updates for India today. Detailed forecast including temperature, wind, rain, snow, and UV index. Stay informed about today's weather conditions in India. 2 days ago \u00b7 Today's Weather Report ... Forecasts Urban Meteorological Services Short to Medium Range Model Guidance Extended Range Model Guidance Seasonal Forecast Quantitative Precipitation Forecast Latest CAP Alerts Specialised products & forecasts Cyclones | Monsoon | Satellite Radar | Rainfall | Agromet Advisories MAUSAM : | | Video Meghdoot Agro ...\nURL: https://duckduckgo.com/?q=What's+the+weather+like+today?\n\nUser Question: What's the weather like today?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", + "prompt.prompt_length": 1612, + "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nStay updated with the latest weather conditions in Komna. Get real-time updates, temperature, humidity, AQI, and detailed forecast for today. Dec 31, 2024 \u00b7 Komna Weather F...", + "prompt.instruction_following": "web_search_optimized", + "response.answer_length": 518, + "response.answer_completeness": "complete", + "response.response_quality": "medium", + "response.source_citation": 1, + "response.context_utilization": 0.455584872471416, + "output_response": "Web Search Answer: The web search results do not provide specific details about today's weather conditions in your location. However, they mention that you can get real-time updates, including temperature, humidity, and...", + "evaluation.search_effectiveness": 0.2, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 1, + "evaluation.context_relevance": "high", + "evaluation.overall_quality": "low", + "evaluation.ready_for_production": true, + "web_search.search_strategy": "duckduckgo_api", + "web_search.real_time_data": true, + "web_search.external_sources": 1, + "web_search.information_freshness": "current", + "web_search.knowledge_synthesis": "single_source" + }, + "events": [], + "links": [], + "trace_trace_id": "3af8c6e8-62d9-4893-92ba-cc265934ef03", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:04.544000000", + "trace_end_time": "2025-10-16 08:50:07.941000000", + "trace_duration_ms": 3397.228, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:04.544000000", + "updated_at": "2025-10-16 08:50:04.544000000" + }, + { + "span_id": "719a1a8f-fc50-439c-899b-436146accd23", + "trace_id": "d7b8b811-ad01-48f1-917f-ace7e8d289e6", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-16 08:50:01.375000000", + "end_time": "2025-10-16 08:50:03.540000000", + "duration_ms": 2164.4809999999998, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What are the latest AI news today?", + "query.length": 34, + "processing.mode": "Web Search", + "processing.time_seconds": 2.1643810272216797, + "processing.response_length": 298, + "processing.sources_count": 1, + "output_response": "Final Answer: The web search did not yield any specific results regarding the latest AI news today. Unfortunately, I cannot provide current updates or news articles on AI developments at this moment. For the latest...", + "final_answer_mode": "Web Search", + "query_processed.input_query": "What are the latest AI news today?", + "query_processed.output_response": "Successfully processed query using Web Search, generated 298 character response", + "query_processed.mode": "Web Search", + "query_processed.processing_time": 2.1643810272216797, + "query_processed.response_length": 298 + }, + "events": [], + "links": [], + "trace_trace_id": "d7b8b811-ad01-48f1-917f-ace7e8d289e6", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:01.375000000", + "trace_end_time": "2025-10-16 08:50:03.540000000", + "trace_duration_ms": 2164.576, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:01.375000000", + "updated_at": "2025-10-16 08:50:01.375000000" + }, + { + "span_id": "644a712f-1c9e-4e9a-9073-74621b754aa2", + "trace_id": "d7b8b811-ad01-48f1-917f-ace7e8d289e6", + "parent_span_id": "719a1a8f-fc50-439c-899b-436146accd23", + "name": "agent.query_routing", + "start_time": "2025-10-16 08:50:01.376000000", + "end_time": "2025-10-16 08:50:03.540000000", + "duration_ms": 2164.3219999999997, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What are the latest AI news today?", + "query_length": 34, + "query_type": "routing_query", + "output_response": "Routed to Web Search for query processing", + "router.classification": "Web Search", + "router.confidence_score": 1, + "router.rag_keyword_score": 0, + "router.web_keyword_score": 3, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "d7b8b811-ad01-48f1-917f-ace7e8d289e6", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:01.375000000", + "trace_end_time": "2025-10-16 08:50:03.540000000", + "trace_duration_ms": 2164.576, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:01.375000000", + "updated_at": "2025-10-16 08:50:01.375000000" + }, + { + "span_id": "8a4abdbe-6ad7-4d43-b234-81667c4abbc2", + "trace_id": "d7b8b811-ad01-48f1-917f-ace7e8d289e6", + "parent_span_id": "644a712f-1c9e-4e9a-9073-74621b754aa2", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-16 08:50:01.376000000", + "end_time": "2025-10-16 08:50:01.376000000", + "duration_ms": 0.051, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What are the latest AI news today?", + "query_length": 34, + "query_type": "routing_evaluation_query", + "classification.mode": "Web Search", + "classification.rag_keyword_score": 0, + "classification.web_keyword_score": 3, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "general_knowledge", + "query.keyword_density": 0.42857142857142855, + "query.domain_affinity": "general", + "routing.decision": "Routed to Web Search based on analysis", + "routing.rationale": "RAG score: 0, Web score: 3, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "RAG", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.057692307692307696, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: Web Search (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "d7b8b811-ad01-48f1-917f-ace7e8d289e6", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:01.375000000", + "trace_end_time": "2025-10-16 08:50:03.540000000", + "trace_duration_ms": 2164.576, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:01.375000000", + "updated_at": "2025-10-16 08:50:01.375000000" + }, + { + "span_id": "904621e8-c5fa-4b2d-a628-8f10eb9da51d", + "trace_id": "d7b8b811-ad01-48f1-917f-ace7e8d289e6", + "parent_span_id": "644a712f-1c9e-4e9a-9073-74621b754aa2", + "name": "agent.web_search_generation", + "start_time": "2025-10-16 08:50:01.376000000", + "end_time": "2025-10-16 08:50:03.540000000", + "duration_ms": 2164.076, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "web_search_agent", + "agent.operation": "web_search_generation", + "agent.capabilities": [ + "web_search", + "content_synthesis", + "response_generation" + ], + "agent.id": "noveum_web_search_agent", + "input_query": "What are the latest AI news today?", + "query_length": 34, + "query_type": "web_search_query", + "output_response": "Web Search Answer: The web search did not yield any specific results regarding the latest AI news today. Unfortunately, I cannot provide current updates or news articles on AI developments at this moment. For the latest...", + "web_search.results_count": 1, + "web_search.context_length": 136, + "web_search.response_length": 298, + "web_search.mode": "external_web_search" + }, + "events": [], + "links": [], + "trace_trace_id": "d7b8b811-ad01-48f1-917f-ace7e8d289e6", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:01.375000000", + "trace_end_time": "2025-10-16 08:50:03.540000000", + "trace_duration_ms": 2164.576, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:01.375000000", + "updated_at": "2025-10-16 08:50:01.375000000" + }, + { + "span_id": "421dd1af-2dbc-477f-bd59-b3194b14ae3a", + "trace_id": "d7b8b811-ad01-48f1-917f-ace7e8d289e6", + "parent_span_id": "904621e8-c5fa-4b2d-a628-8f10eb9da51d", + "name": "agent.llm_model_execution", + "start_time": "2025-10-16 08:50:01.708000000", + "end_time": "2025-10-16 08:50:03.540000000", + "duration_ms": 1831.237, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What are the latest AI news today?", + "query_length": 34, + "query_type": "web_search_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 1.8311803340911865, + "model.latency_ms": 1831.1803340911865, + "model.start_time": 1760604601.7090185, + "model.end_time": 1760604603.5401988, + "model.performance_tier": "fast", + "model.prompt_tokens": 153, + "model.completion_tokens": 74, + "model.total_tokens": 227, + "model.tokens_per_second": 123.96376029926071, + "model.estimated_cost": 0.00681, + "model.efficiency_score": 1.3127753303964758, + "model.response_length": 298, + "model.response_quality": "high", + "model.output_response": "Model Response: The web search did not yield any specific results regarding the latest AI news today. Unfortunately, I cannot provide current updates or news articles on AI developments at this moment. For the latest...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "d7b8b811-ad01-48f1-917f-ace7e8d289e6", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:01.375000000", + "trace_end_time": "2025-10-16 08:50:03.540000000", + "trace_duration_ms": 2164.576, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:01.375000000", + "updated_at": "2025-10-16 08:50:01.375000000" + }, + { + "span_id": "159aad6f-7174-406d-b7d9-eed9360d993c", + "trace_id": "d7b8b811-ad01-48f1-917f-ace7e8d289e6", + "parent_span_id": "904621e8-c5fa-4b2d-a628-8f10eb9da51d", + "name": "agent.web_search_evaluation_metrics", + "start_time": "2025-10-16 08:50:03.540000000", + "end_time": "2025-10-16 08:50:03.540000000", + "duration_ms": 0.028, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "web_search_evaluation_metrics", + "agent.capabilities": [ + "web_search_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What are the latest AI news today?", + "query_length": 34, + "query_type": "web_search_evaluation_query", + "web_search.results_count": 1, + "web_search.context_length": 136, + "web_search.context_synthesized": "Context from 1 web sources", + "web_search.search_effectiveness": 0.2, + "web_search.context_quality": "low", + "web_search.source_diversity": 1, + "web_search.information_synthesis": "low", + "web_search.external_knowledge_utilization": 0.136, + "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What+are+the+latest+AI+news+today?\n\nUser Question: What are the latest AI news today?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", + "prompt.prompt_length": 615, + "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What+are+the+latest+AI+news+today?", + "prompt.instruction_following": "web_search_optimized", + "response.answer_length": 298, + "response.answer_completeness": "complete", + "response.response_quality": "medium", + "response.source_citation": 1, + "response.context_utilization": 2.1911764705882355, + "output_response": "Web Search Answer: The web search did not yield any specific results regarding the latest AI news today. Unfortunately, I cannot provide current updates or news articles on AI developments at this moment. For the latest...", + "evaluation.search_effectiveness": 0.2, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 1, + "evaluation.context_relevance": "low", + "evaluation.overall_quality": "low", + "evaluation.ready_for_production": true, + "web_search.search_strategy": "duckduckgo_api", + "web_search.real_time_data": true, + "web_search.external_sources": 1, + "web_search.information_freshness": "current", + "web_search.knowledge_synthesis": "single_source" + }, + "events": [], + "links": [], + "trace_trace_id": "d7b8b811-ad01-48f1-917f-ace7e8d289e6", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:01.375000000", + "trace_end_time": "2025-10-16 08:50:03.540000000", + "trace_duration_ms": 2164.576, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:01.375000000", + "updated_at": "2025-10-16 08:50:01.375000000" + }, + { + "span_id": "dcf2b38e-ab90-45ee-9a65-e6fea47a3fd7", + "trace_id": "df8340b4-b75a-48e9-a2f3-99725b46a00d", + "parent_span_id": "d0e2b320-c18d-4ec6-98a1-74da5252107e", + "name": "agent.query_routing", + "start_time": "2025-10-16 08:49:17.236000000", + "end_time": "2025-10-16 08:50:00.368000000", + "duration_ms": 43131.384, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What are the benefits of using Noveum Trace?", + "query_length": 44, + "query_type": "routing_query", + "output_response": "Routed to RAG for query processing", + "router.classification": "RAG", + "router.confidence_score": 1, + "router.rag_keyword_score": 2, + "router.web_keyword_score": 0, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "df8340b4-b75a-48e9-a2f3-99725b46a00d", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:49:17.236000000", + "trace_end_time": "2025-10-16 08:50:00.368000000", + "trace_duration_ms": 43131.554000000004, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:49:17.236000000", + "updated_at": "2025-10-16 08:49:17.236000000" + }, + { + "span_id": "d0e2b320-c18d-4ec6-98a1-74da5252107e", + "trace_id": "df8340b4-b75a-48e9-a2f3-99725b46a00d", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-16 08:49:17.236000000", + "end_time": "2025-10-16 08:50:00.368000000", + "duration_ms": 43131.481, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What are the benefits of using Noveum Trace?", + "query.length": 44, + "processing.mode": "RAG", + "processing.time_seconds": 43.131428718566895, + "processing.response_length": 1415, + "processing.sources_count": 5, + "output_response": "Final Answer: The benefits of using Noveum Trace include:\n\n1. **Comprehensive Tracing**: Noveum Trace offers decorator-based tracing for LLM calls, agents, and RAG pipelines, allowing for detailed monitoring of AI ...", + "final_answer_mode": "RAG", + "query_processed.input_query": "What are the benefits of using Noveum Trace?", + "query_processed.output_response": "Successfully processed query using RAG, generated 1415 character response", + "query_processed.mode": "RAG", + "query_processed.processing_time": 43.131428718566895, + "query_processed.response_length": 1415 + }, + "events": [], + "links": [], + "trace_trace_id": "df8340b4-b75a-48e9-a2f3-99725b46a00d", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:49:17.236000000", + "trace_end_time": "2025-10-16 08:50:00.368000000", + "trace_duration_ms": 43131.554000000004, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:49:17.236000000", + "updated_at": "2025-10-16 08:49:17.236000000" + }, + { + "span_id": "fa93821f-c809-4071-87e1-a6c55014d37c", + "trace_id": "df8340b4-b75a-48e9-a2f3-99725b46a00d", + "parent_span_id": "dcf2b38e-ab90-45ee-9a65-e6fea47a3fd7", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-16 08:49:17.236000000", + "end_time": "2025-10-16 08:49:17.236000000", + "duration_ms": 0.034, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What are the benefits of using Noveum Trace?", + "query_length": 44, + "query_type": "routing_evaluation_query", + "classification.mode": "RAG", + "classification.rag_keyword_score": 2, + "classification.web_keyword_score": 0, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "noveum_specific", + "query.keyword_density": 0.25, + "query.domain_affinity": "noveum", + "routing.decision": "Routed to RAG based on analysis", + "routing.rationale": "RAG score: 2, Web score: 0, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "Web Search", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.038461538461538464, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: RAG (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "df8340b4-b75a-48e9-a2f3-99725b46a00d", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:49:17.236000000", + "trace_end_time": "2025-10-16 08:50:00.368000000", + "trace_duration_ms": 43131.554000000004, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:49:17.236000000", + "updated_at": "2025-10-16 08:49:17.236000000" + }, + { + "span_id": "024ad5aa-f394-4c26-af51-564f3f51c6f2", + "trace_id": "df8340b4-b75a-48e9-a2f3-99725b46a00d", + "parent_span_id": "dcf2b38e-ab90-45ee-9a65-e6fea47a3fd7", + "name": "agent.llm-rag", + "start_time": "2025-10-16 08:49:17.237000000", + "end_time": "2025-10-16 08:50:00.368000000", + "duration_ms": 43131.19300000001, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "rag_agent", + "agent.operation": "llm-rag", + "agent.capabilities": [ + "document_retrieval", + "context_generation", + "response_generation" + ], + "agent.id": "noveum_rag_agent", + "input_query": "What are the benefits of using Noveum Trace?", + "query_length": 44, + "query_type": "rag_query", + "output_response": "RAG Answer: The benefits of using Noveum Trace include:\n\n1. **Comprehensive Tracing**: Noveum Trace offers decorator-based tracing for LLM calls, agents, and RAG pipelines, allowing for detailed monitoring of AI ...", + "rag.context_length": 2773, + "rag.sources_count": 5, + "rag.answer_length": 1415, + "rag.mode": "retrieval_augmented_generation" + }, + "events": [], + "links": [], + "trace_trace_id": "df8340b4-b75a-48e9-a2f3-99725b46a00d", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:49:17.236000000", + "trace_end_time": "2025-10-16 08:50:00.368000000", + "trace_duration_ms": 43131.554000000004, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:49:17.236000000", + "updated_at": "2025-10-16 08:49:17.236000000" + }, + { + "span_id": "3d9db951-d8b0-4990-820d-61e4b1011709", + "trace_id": "df8340b4-b75a-48e9-a2f3-99725b46a00d", + "parent_span_id": "024ad5aa-f394-4c26-af51-564f3f51c6f2", + "name": "agent.llm_model_execution", + "start_time": "2025-10-16 08:49:20.224000000", + "end_time": "2025-10-16 08:49:58.420000000", + "duration_ms": 38196.111000000004, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What are the benefits of using Noveum Trace?", + "query_length": 44, + "query_type": "rag_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 38.19604015350342, + "model.latency_ms": 38196.04015350342, + "model.start_time": 1760604560.2242246, + "model.end_time": 1760604598.4202647, + "model.performance_tier": "slow", + "model.prompt_tokens": 828, + "model.completion_tokens": 353, + "model.total_tokens": 1181, + "model.tokens_per_second": 30.919435503098253, + "model.estimated_cost": 0.03543, + "model.efficiency_score": 1.1981371718882303, + "model.response_length": 1415, + "model.response_quality": "high", + "model.output_response": "Model Response: The benefits of using Noveum Trace include:\n\n1. **Comprehensive Tracing**: Noveum Trace offers decorator-based tracing for LLM calls, agents, and RAG pipelines, allowing for detailed monitoring of AI ...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "df8340b4-b75a-48e9-a2f3-99725b46a00d", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:49:17.236000000", + "trace_end_time": "2025-10-16 08:50:00.368000000", + "trace_duration_ms": 43131.554000000004, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:49:17.236000000", + "updated_at": "2025-10-16 08:49:17.236000000" + }, + { + "span_id": "8c33ffac-4d76-42ba-9737-89596e136b3d", + "trace_id": "df8340b4-b75a-48e9-a2f3-99725b46a00d", + "parent_span_id": "024ad5aa-f394-4c26-af51-564f3f51c6f2", + "name": "agent.rag_evaluation_metrics", + "start_time": "2025-10-16 08:49:58.420000000", + "end_time": "2025-10-16 08:49:58.420000000", + "duration_ms": 0.037, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "rag_evaluation_metrics", + "agent.capabilities": [ + "retrieval_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What are the benefits of using Noveum Trace?", + "query_length": 44, + "query_type": "rag_evaluation_query", + "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/docs/platform/dashboard):\n- Bulk Export: Download multiple traces simultaneously\n- Comparative Analysis: Compare performance across multiple traces\nReal-Time Updates\n- Live Refresh: Automatic updates as new traces arrive\n- Connection Monitoring: Real-time status of your t...", + "retrieval.context_length": 2773, + "retrieval.sources_count": 5, + "retrieval.context_quality": "high", + "retrieval.effectiveness": 1, + "retrieval.context_utilization": 2.773, + "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/docs/platform/dashboard):\n- Bulk Export: Download multiple traces simultaneously\n- Comparative Analysis: Compare performance across multiple traces\nReal-Time Updates\n- Live Refresh: Automatic updates as new traces arrive\n- Connection Monitoring: Real-time status of your trace ingestion pipeline\n- Performance Indicators: Live metrics for system health monitoring\nReady to dive deeper? Explore Projects & Environments to organize your AI applications, or check out Team Collaboration to share insights with your team.\nGet Earl...\n\nSource 2 (https://noveum.ai/en/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\n\ud83d\udcd8 TypeScript SDK (\n@noveum/trace\n)- Framework integrations for Next.js, Express.js, and Hono\n- TypeScript-first design with full type safety\n- Universal compatibility (Node.js, Edge Runtime, browsers)\n-\n\ud83d\udcca Noveum Platform\n- Real-time dashboard for analyzing traces and performance\n- Ad...\n\nSource 3 (https://noveum.ai/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\n\ud83d\udcd8 TypeScript SDK (\n@noveum/trace\n)- Framework integrations for Next.js, Express.js, and Hono\n- TypeScript-first design with full type safety\n- Universal compatibility (Node.js, Edge Runtime, browsers)\n-\n\ud83d\udcca Noveum Platform\n- Real-time dashboard for analyzing traces and performance\n- Ad...\n\nSource 4 (https://noveum.ai/en/docs/getting-started/overview):\n- Advanced search & filtering across traces and spans\n- Cost analysis and optimization recommendations\n- Team collaboration with shared insights and alerts\n\ud83d\udd0d What Noveum Traces\nLLM Operations\n- Model calls across all providers (OpenAI, Anthropic, Google, etc.)\n- Token usage and cost calculation\n- Prompt engineering effectiveness\n- Response quality metrics\nRAG Pipelines\n- Document retrieval performance and relevance\n- Embedding generation costs and latency\n- Context assembly and prompt constructi...\n\nSource 5 (https://noveum.ai/docs/getting-started/overview):\n- Advanced search & filtering across traces and spans\n- Cost analysis and optimization recommendations\n- Team collaboration with shared insights and alerts\n\ud83d\udd0d What Noveum Traces\nLLM Operations\n- Model calls across all providers (OpenAI, Anthropic, Google, etc.)\n- Token usage and cost calculation\n- Prompt engineering effectiveness\n- Response quality metrics\nRAG Pipelines\n- Document retrieval performance and relevance\n- Embedding generation costs and latency\n- Context assembly and prompt constructi...\n\nUser Question: What are the benefits of using Noveum Trace?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", + "prompt.prompt_length": 3313, + "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/docs/platform/dashboard):\n- Bulk Export: Download multiple traces simultaneously\n- Comparative Analysis: Compare performance across multiple traces\nReal-Time Updates\n- Live...", + "prompt.instruction_following": "rag_optimized", + "response.answer_length": 1415, + "response.answer_completeness": "complete", + "response.response_quality": "high", + "response.source_citation": 5, + "response.context_utilization": 0.5102776776054815, + "output_response": "RAG Answer: The benefits of using Noveum Trace include:\n\n1. **Comprehensive Tracing**: Noveum Trace offers decorator-based tracing for LLM calls, agents, and RAG pipelines, allowing for detailed monitoring of AI ...", + "evaluation.retrieval_effectiveness": 1, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 5, + "evaluation.context_relevance": "high", + "evaluation.overall_quality": "high", + "evaluation.ready_for_production": true, + "rag.retrieval_strategy": "semantic_similarity", + "rag.vector_search_results": 5, + "rag.context_synthesis": "multi_source", + "rag.document_coverage": 1, + "rag.information_density": 0.5102776776054815 + }, + "events": [], + "links": [], + "trace_trace_id": "df8340b4-b75a-48e9-a2f3-99725b46a00d", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:49:17.236000000", + "trace_end_time": "2025-10-16 08:50:00.368000000", + "trace_duration_ms": 43131.554000000004, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:49:17.236000000", + "updated_at": "2025-10-16 08:49:17.236000000" + }, + { + "span_id": "5b1a7121-d0bd-4525-821d-7721c6fbbd70", + "trace_id": "325ec189-2876-4ebe-b33c-649491a5053e", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-16 08:49:11.192000000", + "end_time": "2025-10-16 08:49:16.232000000", + "duration_ms": 5039.753, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "How do I configure Noveum for my system?", + "query.length": 40, + "processing.mode": "RAG", + "processing.time_seconds": 5.039698600769043, + "processing.response_length": 712, + "processing.sources_count": 5, + "output_response": "Final Answer: To configure Noveum for your system, you can refer to the SDK Integration Guide available in the Noveum documentation. This guide will help you trace your AI applications with minimal code changes. Th...", + "final_answer_mode": "RAG", + "query_processed.input_query": "How do I configure Noveum for my system?", + "query_processed.output_response": "Successfully processed query using RAG, generated 712 character response", + "query_processed.mode": "RAG", + "query_processed.processing_time": 5.039698600769043, + "query_processed.response_length": 712 + }, + "events": [], + "links": [], + "trace_trace_id": "325ec189-2876-4ebe-b33c-649491a5053e", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:49:11.192000000", + "trace_end_time": "2025-10-16 08:49:16.232000000", + "trace_duration_ms": 5039.841, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:49:11.192000000", + "updated_at": "2025-10-16 08:49:11.192000000" + }, + { + "span_id": "761c92c6-d15a-4a8e-a16a-a365922f8d7c", + "trace_id": "325ec189-2876-4ebe-b33c-649491a5053e", + "parent_span_id": "5b1a7121-d0bd-4525-821d-7721c6fbbd70", + "name": "agent.query_routing", + "start_time": "2025-10-16 08:49:11.192000000", + "end_time": "2025-10-16 08:49:16.232000000", + "duration_ms": 5039.642, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "How do I configure Noveum for my system?", + "query_length": 40, + "query_type": "routing_query", + "output_response": "Routed to RAG for query processing", + "router.classification": "RAG", + "router.confidence_score": 1, + "router.rag_keyword_score": 2, + "router.web_keyword_score": 0, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "325ec189-2876-4ebe-b33c-649491a5053e", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:49:11.192000000", + "trace_end_time": "2025-10-16 08:49:16.232000000", + "trace_duration_ms": 5039.841, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:49:11.192000000", + "updated_at": "2025-10-16 08:49:11.192000000" + }, + { + "span_id": "8fcf969e-a577-45aa-9fe6-1870d9665189", + "trace_id": "325ec189-2876-4ebe-b33c-649491a5053e", + "parent_span_id": "761c92c6-d15a-4a8e-a16a-a365922f8d7c", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-16 08:49:11.192000000", + "end_time": "2025-10-16 08:49:11.192000000", + "duration_ms": 0.045000000000000005, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: How do I configure Noveum for my system?", + "query_length": 40, + "query_type": "routing_evaluation_query", + "classification.mode": "RAG", + "classification.rag_keyword_score": 2, + "classification.web_keyword_score": 0, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "noveum_specific", + "query.keyword_density": 0.25, + "query.domain_affinity": "noveum", + "routing.decision": "Routed to RAG based on analysis", + "routing.rationale": "RAG score: 2, Web score: 0, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "Web Search", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.038461538461538464, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: RAG (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "325ec189-2876-4ebe-b33c-649491a5053e", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:49:11.192000000", + "trace_end_time": "2025-10-16 08:49:16.232000000", + "trace_duration_ms": 5039.841, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:49:11.192000000", + "updated_at": "2025-10-16 08:49:11.192000000" + }, + { + "span_id": "9e5bbea4-cc68-4fe6-b3d0-c87852635ec9", + "trace_id": "325ec189-2876-4ebe-b33c-649491a5053e", + "parent_span_id": "761c92c6-d15a-4a8e-a16a-a365922f8d7c", + "name": "agent.llm-rag", + "start_time": "2025-10-16 08:49:11.192000000", + "end_time": "2025-10-16 08:49:16.232000000", + "duration_ms": 5039.39, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "rag_agent", + "agent.operation": "llm-rag", + "agent.capabilities": [ + "document_retrieval", + "context_generation", + "response_generation" + ], + "agent.id": "noveum_rag_agent", + "input_query": "How do I configure Noveum for my system?", + "query_length": 40, + "query_type": "rag_query", + "output_response": "RAG Answer: To configure Noveum for your system, you can refer to the SDK Integration Guide available in the Noveum documentation. This guide will help you trace your AI applications with minimal code changes. Th...", + "rag.context_length": 2237, + "rag.sources_count": 5, + "rag.answer_length": 712, + "rag.mode": "retrieval_augmented_generation" + }, + "events": [], + "links": [], + "trace_trace_id": "325ec189-2876-4ebe-b33c-649491a5053e", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:49:11.192000000", + "trace_end_time": "2025-10-16 08:49:16.232000000", + "trace_duration_ms": 5039.841, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:49:11.192000000", + "updated_at": "2025-10-16 08:49:11.192000000" + }, + { + "span_id": "5ee706ad-c4ae-4c0b-94d1-6b9c43d173b4", + "trace_id": "325ec189-2876-4ebe-b33c-649491a5053e", + "parent_span_id": "9e5bbea4-cc68-4fe6-b3d0-c87852635ec9", + "name": "agent.llm_model_execution", + "start_time": "2025-10-16 08:49:11.607000000", + "end_time": "2025-10-16 08:49:15.718000000", + "duration_ms": 4110.915, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: How do I configure Noveum for my system?", + "query_length": 40, + "query_type": "rag_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 4.110846042633057, + "model.latency_ms": 4110.846042633057, + "model.start_time": 1760604551.6074958, + "model.end_time": 1760604555.7183418, + "model.performance_tier": "medium", + "model.prompt_tokens": 693, + "model.completion_tokens": 178, + "model.total_tokens": 871, + "model.tokens_per_second": 211.87852596934326, + "model.estimated_cost": 0.02613, + "model.efficiency_score": 0.817451205510907, + "model.response_length": 712, + "model.response_quality": "high", + "model.output_response": "Model Response: To configure Noveum for your system, you can refer to the SDK Integration Guide available in the Noveum documentation. This guide will help you trace your AI applications with minimal code changes. Th...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "325ec189-2876-4ebe-b33c-649491a5053e", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:49:11.192000000", + "trace_end_time": "2025-10-16 08:49:16.232000000", + "trace_duration_ms": 5039.841, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:49:11.192000000", + "updated_at": "2025-10-16 08:49:11.192000000" + }, + { + "span_id": "d13bdeda-81af-400a-82ca-a5ef7e1e8a6d", + "trace_id": "325ec189-2876-4ebe-b33c-649491a5053e", + "parent_span_id": "9e5bbea4-cc68-4fe6-b3d0-c87852635ec9", + "name": "agent.rag_evaluation_metrics", + "start_time": "2025-10-16 08:49:15.718000000", + "end_time": "2025-10-16 08:49:15.718000000", + "duration_ms": 0.118, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "rag_evaluation_metrics", + "agent.capabilities": [ + "retrieval_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: How do I configure Noveum for my system?", + "query_length": 40, + "query_type": "rag_evaluation_query", + "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/docs):\nWelcome to Noveum.ai Docs\nComprehensive AI tracing and observability for LLM applications, RAG systems, and AI agents\nWelcome to the Noveum.ai documentation! Here you'll find everything you need to integrate, configure, and optimize your AI applications using ou...", + "retrieval.context_length": 2237, + "retrieval.sources_count": 5, + "retrieval.context_quality": "high", + "retrieval.effectiveness": 1, + "retrieval.context_utilization": 2.237, + "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/docs):\nWelcome to Noveum.ai Docs\nComprehensive AI tracing and observability for LLM applications, RAG systems, and AI agents\nWelcome to the Noveum.ai documentation! Here you'll find everything you need to integrate, configure, and optimize your AI applications using our comprehensive tracing and observability platform.\nWe provide powerful SDKs for Python and TypeScript that enable you to trace LLM calls, RAG pipelines, and multi-agent workflows with minimal code changes. Our documentation is organized ...\n\nSource 2 (https://noveum.ai/docs):\nWelcome to Noveum.ai Docs\nComprehensive AI tracing and observability for LLM applications, RAG systems, and AI agents\nWelcome to the Noveum.ai documentation! Here you'll find everything you need to integrate, configure, and optimize your AI applications using our comprehensive tracing and observability platform.\nWe provide powerful SDKs for Python and TypeScript that enable you to trace LLM calls, RAG pipelines, and multi-agent workflows with minimal code changes. Our documentation is organized ...\n\nSource 3 (https://noveum.ai/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nSource 4 (https://noveum.ai/en/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nSource 5 (https://noveum.ai/en/docs):\n- \ud83c\udf0d Universal: Works across frameworks, clouds, and deployment models\n\ud83e\udd1d Community & Support\n- \ud83d\udcac Discord Community: Join our Discord\n- \ud83d\udce7 Email Support: [email protected]\n- \ud83d\udc1b Bug Reports: GitHub Issues\n- \ud83d\udcd6 Knowledge Base: Help Center\nReady to get started? Head to our SDK Integration Guide to begin tracing your AI applications in under 5 minutes!\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability ...\n\nUser Question: How do I configure Noveum for my system?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", + "prompt.prompt_length": 2773, + "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/docs):\nWelcome to Noveum.ai Docs\nComprehensive AI tracing and observability for LLM applications, RAG systems, and AI agents\nWelcome to the Noveum.ai documentation! Here...", + "prompt.instruction_following": "rag_optimized", + "response.answer_length": 712, + "response.answer_completeness": "complete", + "response.response_quality": "high", + "response.source_citation": 5, + "response.context_utilization": 0.3182834152883326, + "output_response": "RAG Answer: To configure Noveum for your system, you can refer to the SDK Integration Guide available in the Noveum documentation. This guide will help you trace your AI applications with minimal code changes. Th...", + "evaluation.retrieval_effectiveness": 1, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 5, + "evaluation.context_relevance": "high", + "evaluation.overall_quality": "high", + "evaluation.ready_for_production": true, + "rag.retrieval_strategy": "semantic_similarity", + "rag.vector_search_results": 5, + "rag.context_synthesis": "multi_source", + "rag.document_coverage": 1, + "rag.information_density": 0.3182834152883326 + }, + "events": [], + "links": [], + "trace_trace_id": "325ec189-2876-4ebe-b33c-649491a5053e", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:49:11.192000000", + "trace_end_time": "2025-10-16 08:49:16.232000000", + "trace_duration_ms": 5039.841, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:49:11.192000000", + "updated_at": "2025-10-16 08:49:11.192000000" + }, + { + "span_id": "7d6564fc-41b2-4d74-acb1-e77b575c0006", + "trace_id": "d5528586-c741-4703-8791-5fccf5c3e123", + "parent_span_id": "e79a037e-82b1-41e9-bdc7-42353e98ead7", + "name": "agent.query_routing", + "start_time": "2025-10-16 08:49:05.152000000", + "end_time": "2025-10-16 08:49:10.187000000", + "duration_ms": 5034.446, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What monitoring capabilities does Noveum provide?", + "query_length": 49, + "query_type": "routing_query", + "output_response": "Routed to RAG for query processing", + "router.classification": "RAG", + "router.confidence_score": 1, + "router.rag_keyword_score": 2, + "router.web_keyword_score": 0, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "d5528586-c741-4703-8791-5fccf5c3e123", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:49:05.152000000", + "trace_end_time": "2025-10-16 08:49:10.187000000", + "trace_duration_ms": 5034.604, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:49:05.152000000", + "updated_at": "2025-10-16 08:49:05.152000000" + }, + { + "span_id": "e79a037e-82b1-41e9-bdc7-42353e98ead7", + "trace_id": "d5528586-c741-4703-8791-5fccf5c3e123", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-16 08:49:05.152000000", + "end_time": "2025-10-16 08:49:10.187000000", + "duration_ms": 5034.531, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What monitoring capabilities does Noveum provide?", + "query.length": 49, + "processing.mode": "RAG", + "processing.time_seconds": 5.0344908237457275, + "processing.response_length": 552, + "processing.sources_count": 5, + "output_response": "Final Answer: Noveum provides comprehensive monitoring capabilities for AI applications, including the ability to monitor, trace, and optimize AI agents across various frameworks such as LangChain, CrewAI, AutoGen,...", + "final_answer_mode": "RAG", + "query_processed.input_query": "What monitoring capabilities does Noveum provide?", + "query_processed.output_response": "Successfully processed query using RAG, generated 552 character response", + "query_processed.mode": "RAG", + "query_processed.processing_time": 5.0344908237457275, + "query_processed.response_length": 552 + }, + "events": [], + "links": [], + "trace_trace_id": "d5528586-c741-4703-8791-5fccf5c3e123", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:49:05.152000000", + "trace_end_time": "2025-10-16 08:49:10.187000000", + "trace_duration_ms": 5034.604, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:49:05.152000000", + "updated_at": "2025-10-16 08:49:05.152000000" + }, + { + "span_id": "78297793-d73b-4311-b5ba-73509ae9f96e", + "trace_id": "d5528586-c741-4703-8791-5fccf5c3e123", + "parent_span_id": "7d6564fc-41b2-4d74-acb1-e77b575c0006", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-16 08:49:05.153000000", + "end_time": "2025-10-16 08:49:05.153000000", + "duration_ms": 0.029, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What monitoring capabilities does Noveum provide?", + "query_length": 49, + "query_type": "routing_evaluation_query", + "classification.mode": "RAG", + "classification.rag_keyword_score": 2, + "classification.web_keyword_score": 0, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "noveum_specific", + "query.keyword_density": 0.3333333333333333, + "query.domain_affinity": "noveum", + "routing.decision": "Routed to RAG based on analysis", + "routing.rationale": "RAG score: 2, Web score: 0, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "Web Search", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.038461538461538464, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: RAG (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "d5528586-c741-4703-8791-5fccf5c3e123", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:49:05.152000000", + "trace_end_time": "2025-10-16 08:49:10.187000000", + "trace_duration_ms": 5034.604, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:49:05.152000000", + "updated_at": "2025-10-16 08:49:05.152000000" + }, + { + "span_id": "6a101d9d-de83-415a-8fb0-01a9fae5ffb9", + "trace_id": "d5528586-c741-4703-8791-5fccf5c3e123", + "parent_span_id": "7d6564fc-41b2-4d74-acb1-e77b575c0006", + "name": "agent.llm-rag", + "start_time": "2025-10-16 08:49:05.153000000", + "end_time": "2025-10-16 08:49:10.187000000", + "duration_ms": 5034.282, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "rag_agent", + "agent.operation": "llm-rag", + "agent.capabilities": [ + "document_retrieval", + "context_generation", + "response_generation" + ], + "agent.id": "noveum_rag_agent", + "input_query": "What monitoring capabilities does Noveum provide?", + "query_length": 49, + "query_type": "rag_query", + "output_response": "RAG Answer: Noveum provides comprehensive monitoring capabilities for AI applications, including the ability to monitor, trace, and optimize AI agents across various frameworks such as LangChain, CrewAI, AutoGen,...", + "rag.context_length": 2740, + "rag.sources_count": 5, + "rag.answer_length": 552, + "rag.mode": "retrieval_augmented_generation" + }, + "events": [], + "links": [], + "trace_trace_id": "d5528586-c741-4703-8791-5fccf5c3e123", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:49:05.152000000", + "trace_end_time": "2025-10-16 08:49:10.187000000", + "trace_duration_ms": 5034.604, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:49:05.152000000", + "updated_at": "2025-10-16 08:49:05.152000000" + }, + { + "span_id": "26e510c0-441a-4f9f-9dce-31180f47d060", + "trace_id": "d5528586-c741-4703-8791-5fccf5c3e123", + "parent_span_id": "6a101d9d-de83-415a-8fb0-01a9fae5ffb9", + "name": "agent.llm_model_execution", + "start_time": "2025-10-16 08:49:05.787000000", + "end_time": "2025-10-16 08:49:09.274000000", + "duration_ms": 3486.699, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What monitoring capabilities does Noveum provide?", + "query_length": 49, + "query_type": "rag_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 3.4866299629211426, + "model.latency_ms": 3486.6299629211426, + "model.start_time": 1760604545.787763, + "model.end_time": 1760604549.274393, + "model.performance_tier": "medium", + "model.prompt_tokens": 821, + "model.completion_tokens": 138, + "model.total_tokens": 959, + "model.tokens_per_second": 275.0506965747916, + "model.estimated_cost": 0.02877, + "model.efficiency_score": 0.5755995828988529, + "model.response_length": 552, + "model.response_quality": "high", + "model.output_response": "Model Response: Noveum provides comprehensive monitoring capabilities for AI applications, including the ability to monitor, trace, and optimize AI agents across various frameworks such as LangChain, CrewAI, AutoGen,...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "d5528586-c741-4703-8791-5fccf5c3e123", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:49:05.152000000", + "trace_end_time": "2025-10-16 08:49:10.187000000", + "trace_duration_ms": 5034.604, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:49:05.152000000", + "updated_at": "2025-10-16 08:49:05.152000000" + }, + { + "span_id": "fcd89e38-3d21-468b-860a-ae2de0bbd119", + "trace_id": "d5528586-c741-4703-8791-5fccf5c3e123", + "parent_span_id": "6a101d9d-de83-415a-8fb0-01a9fae5ffb9", + "name": "agent.rag_evaluation_metrics", + "start_time": "2025-10-16 08:49:09.274000000", + "end_time": "2025-10-16 08:49:09.274000000", + "duration_ms": 0.054, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "rag_evaluation_metrics", + "agent.capabilities": [ + "retrieval_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What monitoring capabilities does Noveum provide?", + "query_length": 49, + "query_type": "rag_evaluation_query", + "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en):\nMonitor all your AI Agents\nimprove AI Agents today\nNoveum.ai helps you monitor, trace, and optimize your AI applications.\nNoveum.ai works with any AI framework \u2013 LangChain, CrewAI, AutoGen, custom implementations, or direct LLM calls. One dashboard shows everything.\n...", + "retrieval.context_length": 2740, + "retrieval.sources_count": 5, + "retrieval.context_quality": "high", + "retrieval.effectiveness": 1, + "retrieval.context_utilization": 2.74, + "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en):\nMonitor all your AI Agents\nimprove AI Agents today\nNoveum.ai helps you monitor, trace, and optimize your AI applications.\nNoveum.ai works with any AI framework \u2013 LangChain, CrewAI, AutoGen, custom implementations, or direct LLM calls. One dashboard shows everything.\nMonitor, Evaluate, Improve Your AI Agents\nThe control plane for AI agents.\nMonitor Everything, Miss Nothing\nOur lightweight SDKs capture every trace and span across your AI agent ecosystem\u2014from simple LLM calls to complex multi-agent...\n\nSource 2 (https://noveum.ai/):\nMonitor all your AI Agents\nimprove AI Agents today\nNoveum.ai helps you monitor, trace, and optimize your AI applications.\nNoveum.ai works with any AI framework \u2013 LangChain, CrewAI, AutoGen, custom implementations, or direct LLM calls. One dashboard shows everything.\nMonitor, Evaluate, Improve Your AI Agents\nThe control plane for AI agents.\nMonitor Everything, Miss Nothing\nOur lightweight SDKs capture every trace and span across your AI agent ecosystem\u2014from simple LLM calls to complex multi-agent...\n\nSource 3 (https://noveum.ai):\nMonitor all your AI Agents\nimprove AI Agents today\nNoveum.ai helps you monitor, trace, and optimize your AI applications.\nNoveum.ai works with any AI framework \u2013 LangChain, CrewAI, AutoGen, custom implementations, or direct LLM calls. One dashboard shows everything.\nMonitor, Evaluate, Improve Your AI Agents\nThe control plane for AI agents.\nMonitor Everything, Miss Nothing\nOur lightweight SDKs capture every trace and span across your AI agent ecosystem\u2014from simple LLM calls to complex multi-agent...\n\nSource 4 (https://noveum.ai/en/docs/getting-started/overview):\nNoveum.ai Overview\nComprehensive AI tracing and observability platform for LLM applications, RAG systems, and AI agents\nWelcome to Noveum.ai\u2014the comprehensive tracing and observability platform built specifically for AI applications. Whether you're building LLM-powered chatbots, RAG systems, multi-agent workflows, or any AI-driven application, Noveum provides the insights you need to understand, debug, and optimize your systems.\n\ud83c\udfaf Why AI Applications Need Specialized Observability\nTraditional mo...\n\nSource 5 (https://noveum.ai/docs/getting-started/overview):\nNoveum.ai Overview\nComprehensive AI tracing and observability platform for LLM applications, RAG systems, and AI agents\nWelcome to Noveum.ai\u2014the comprehensive tracing and observability platform built specifically for AI applications. Whether you're building LLM-powered chatbots, RAG systems, multi-agent workflows, or any AI-driven application, Noveum provides the insights you need to understand, debug, and optimize your systems.\n\ud83c\udfaf Why AI Applications Need Specialized Observability\nTraditional mo...\n\nUser Question: What monitoring capabilities does Noveum provide?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", + "prompt.prompt_length": 3285, + "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en):\nMonitor all your AI Agents\nimprove AI Agents today\nNoveum.ai helps you monitor, trace, and optimize your AI applications.\nNoveum.ai works with any AI framework \u2013 LangC...", + "prompt.instruction_following": "rag_optimized", + "response.answer_length": 552, + "response.answer_completeness": "complete", + "response.response_quality": "high", + "response.source_citation": 5, + "response.context_utilization": 0.20145985401459854, + "output_response": "RAG Answer: Noveum provides comprehensive monitoring capabilities for AI applications, including the ability to monitor, trace, and optimize AI agents across various frameworks such as LangChain, CrewAI, AutoGen,...", + "evaluation.retrieval_effectiveness": 1, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 5, + "evaluation.context_relevance": "high", + "evaluation.overall_quality": "high", + "evaluation.ready_for_production": true, + "rag.retrieval_strategy": "semantic_similarity", + "rag.vector_search_results": 5, + "rag.context_synthesis": "multi_source", + "rag.document_coverage": 1, + "rag.information_density": 0.20145985401459854 + }, + "events": [], + "links": [], + "trace_trace_id": "d5528586-c741-4703-8791-5fccf5c3e123", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:49:05.152000000", + "trace_end_time": "2025-10-16 08:49:10.187000000", + "trace_duration_ms": 5034.604, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:49:05.152000000", + "updated_at": "2025-10-16 08:49:05.152000000" + }, + { + "span_id": "bd5f205f-0329-4811-b883-129d69f12382", + "trace_id": "8100b869-07c0-4e37-b51e-092c9d20b4a7", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-16 08:48:57.574000000", + "end_time": "2025-10-16 08:49:04.149000000", + "duration_ms": 6574.424, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "How does Noveum handle agent tracing?", + "query.length": 37, + "processing.mode": "RAG", + "processing.time_seconds": 6.5743818283081055, + "processing.response_length": 643, + "processing.sources_count": 5, + "output_response": "Final Answer: Noveum.ai handles agent tracing through its specialized Multi-Agent Tracing capabilities, which are designed to observe complex workflows and inter-agent communications within multi-agent systems. The...", + "final_answer_mode": "RAG", + "query_processed.input_query": "How does Noveum handle agent tracing?", + "query_processed.output_response": "Successfully processed query using RAG, generated 643 character response", + "query_processed.mode": "RAG", + "query_processed.processing_time": 6.5743818283081055, + "query_processed.response_length": 643 + }, + "events": [], + "links": [], + "trace_trace_id": "8100b869-07c0-4e37-b51e-092c9d20b4a7", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:48:57.574000000", + "trace_end_time": "2025-10-16 08:49:04.149000000", + "trace_duration_ms": 6574.500999999999, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:48:57.574000000", + "updated_at": "2025-10-16 08:48:57.574000000" + }, + { + "span_id": "0801bcfe-8c09-4ea6-a754-5564395d6ded", + "trace_id": "8100b869-07c0-4e37-b51e-092c9d20b4a7", + "parent_span_id": "3c35d86a-0df9-42dc-bfe3-bbe978cf62b5", + "name": "agent.llm-rag", + "start_time": "2025-10-16 08:48:57.575000000", + "end_time": "2025-10-16 08:49:04.149000000", + "duration_ms": 6574.135, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "rag_agent", + "agent.operation": "llm-rag", + "agent.capabilities": [ + "document_retrieval", + "context_generation", + "response_generation" + ], + "agent.id": "noveum_rag_agent", + "input_query": "How does Noveum handle agent tracing?", + "query_length": 37, + "query_type": "rag_query", + "output_response": "RAG Answer: Noveum.ai handles agent tracing through its specialized Multi-Agent Tracing capabilities, which are designed to observe complex workflows and inter-agent communications within multi-agent systems. The...", + "rag.context_length": 2736, + "rag.sources_count": 5, + "rag.answer_length": 643, + "rag.mode": "retrieval_augmented_generation" + }, + "events": [], + "links": [], + "trace_trace_id": "8100b869-07c0-4e37-b51e-092c9d20b4a7", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:48:57.574000000", + "trace_end_time": "2025-10-16 08:49:04.149000000", + "trace_duration_ms": 6574.500999999999, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:48:57.574000000", + "updated_at": "2025-10-16 08:48:57.574000000" + }, + { + "span_id": "3c35d86a-0df9-42dc-bfe3-bbe978cf62b5", + "trace_id": "8100b869-07c0-4e37-b51e-092c9d20b4a7", + "parent_span_id": "bd5f205f-0329-4811-b883-129d69f12382", + "name": "agent.query_routing", + "start_time": "2025-10-16 08:48:57.575000000", + "end_time": "2025-10-16 08:49:04.149000000", + "duration_ms": 6574.342, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "How does Noveum handle agent tracing?", + "query_length": 37, + "query_type": "routing_query", + "output_response": "Routed to RAG for query processing", + "router.classification": "RAG", + "router.confidence_score": 1, + "router.rag_keyword_score": 2, + "router.web_keyword_score": 0, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "8100b869-07c0-4e37-b51e-092c9d20b4a7", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:48:57.574000000", + "trace_end_time": "2025-10-16 08:49:04.149000000", + "trace_duration_ms": 6574.500999999999, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:48:57.574000000", + "updated_at": "2025-10-16 08:48:57.574000000" + }, + { + "span_id": "b1914316-f545-4bc9-abf4-3494cf42c4a5", + "trace_id": "8100b869-07c0-4e37-b51e-092c9d20b4a7", + "parent_span_id": "3c35d86a-0df9-42dc-bfe3-bbe978cf62b5", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-16 08:48:57.575000000", + "end_time": "2025-10-16 08:48:57.575000000", + "duration_ms": 0.032, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: How does Noveum handle agent tracing?", + "query_length": 37, + "query_type": "routing_evaluation_query", + "classification.mode": "RAG", + "classification.rag_keyword_score": 2, + "classification.web_keyword_score": 0, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "noveum_specific", + "query.keyword_density": 0.3333333333333333, + "query.domain_affinity": "noveum", + "routing.decision": "Routed to RAG based on analysis", + "routing.rationale": "RAG score: 2, Web score: 0, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "Web Search", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.038461538461538464, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: RAG (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "8100b869-07c0-4e37-b51e-092c9d20b4a7", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:48:57.574000000", + "trace_end_time": "2025-10-16 08:49:04.149000000", + "trace_duration_ms": 6574.500999999999, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:48:57.574000000", + "updated_at": "2025-10-16 08:48:57.574000000" + }, + { + "span_id": "0906814b-6a80-40e1-9f00-5376b809703a", + "trace_id": "8100b869-07c0-4e37-b51e-092c9d20b4a7", + "parent_span_id": "0801bcfe-8c09-4ea6-a754-5564395d6ded", + "name": "agent.llm_model_execution", + "start_time": "2025-10-16 08:48:58.009000000", + "end_time": "2025-10-16 08:49:03.349000000", + "duration_ms": 5340.01, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: How does Noveum handle agent tracing?", + "query_length": 37, + "query_type": "rag_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 5.339954614639282, + "model.latency_ms": 5339.954614639282, + "model.start_time": 1760604538.0093303, + "model.end_time": 1760604543.349285, + "model.performance_tier": "slow", + "model.prompt_tokens": 817, + "model.completion_tokens": 160, + "model.total_tokens": 977, + "model.tokens_per_second": 182.9603565022054, + "model.estimated_cost": 0.02931, + "model.efficiency_score": 0.6581371545547595, + "model.response_length": 643, + "model.response_quality": "high", + "model.output_response": "Model Response: Noveum.ai handles agent tracing through its specialized Multi-Agent Tracing capabilities, which are designed to observe complex workflows and inter-agent communications within multi-agent systems. The...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "8100b869-07c0-4e37-b51e-092c9d20b4a7", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:48:57.574000000", + "trace_end_time": "2025-10-16 08:49:04.149000000", + "trace_duration_ms": 6574.500999999999, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:48:57.574000000", + "updated_at": "2025-10-16 08:48:57.574000000" + }, + { + "span_id": "29822edb-c15f-4baa-b6ec-84d85fcc926d", + "trace_id": "8100b869-07c0-4e37-b51e-092c9d20b4a7", + "parent_span_id": "0801bcfe-8c09-4ea6-a754-5564395d6ded", + "name": "agent.rag_evaluation_metrics", + "start_time": "2025-10-16 08:49:03.349000000", + "end_time": "2025-10-16 08:49:03.349000000", + "duration_ms": 0.034999999999999996, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "rag_evaluation_metrics", + "agent.capabilities": [ + "retrieval_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: How does Noveum handle agent tracing?", + "query_length": 37, + "query_type": "rag_evaluation_query", + "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, vector searches, data retrieval, agent reasoning, and more. Without proper observability, debugging ...", + "retrieval.context_length": 2736, + "retrieval.sources_count": 5, + "retrieval.context_quality": "high", + "retrieval.effectiveness": 1, + "retrieval.context_utilization": 2.736, + "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, vector searches, data retrieval, agent reasoning, and more. Without proper observability, debugging becomes a nightmare, optimization is guesswork, and understanding user interactions is nearly impossible.\nNoveum.ai solves this by providing comprehensive tracing and observability specifically designed for AI applications.\nKey benefits of using Noveum.ai:\n- Complete Visibility: Trace...\n\nSource 2 (https://noveum.ai/docs/advanced/multi-agent-tracing):\nMulti-Agent Tracing\nObserve complex agent workflows and inter-agent communications with comprehensive tracing\nMulti-agent systems represent some of the most complex AI applications, involving multiple agents that coordinate, communicate, and collaborate to achieve shared goals. Noveum.ai provides specialized tracing capabilities to help you understand and optimize these intricate workflows.\n\ud83c\udfaf Why Multi-Agent Tracing Matters\nMulti-agent systems introduce unique observability challenges:\n- Complex...\n\nSource 3 (https://noveum.ai/en/docs/advanced/multi-agent-tracing):\nMulti-Agent Tracing\nObserve complex agent workflows and inter-agent communications with comprehensive tracing\nMulti-agent systems represent some of the most complex AI applications, involving multiple agents that coordinate, communicate, and collaborate to achieve shared goals. Noveum.ai provides specialized tracing capabilities to help you understand and optimize these intricate workflows.\n\ud83c\udfaf Why Multi-Agent Tracing Matters\nMulti-agent systems introduce unique observability challenges:\n- Complex...\n\nSource 4 (https://noveum.ai/en/docs/advanced/performance-optimization):\n\ud83d\udd17 Next Steps\n- Multi-Agent Tracing - Observe agent workflows\n- RAG Pipeline Observability - Monitor retrieval and generation systems\n- Custom Instrumentation - Add domain-specific tracing\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nSource 5 (https://noveum.ai/docs/advanced/performance-optimization):\n\ud83d\udd17 Next Steps\n- Multi-Agent Tracing - Observe agent workflows\n- RAG Pipeline Observability - Monitor retrieval and generation systems\n- Custom Instrumentation - Add domain-specific tracing\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nUser Question: How does Noveum handle agent tracing?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", + "prompt.prompt_length": 3269, + "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, ...", + "prompt.instruction_following": "rag_optimized", + "response.answer_length": 643, + "response.answer_completeness": "complete", + "response.response_quality": "high", + "response.source_citation": 5, + "response.context_utilization": 0.23501461988304093, + "output_response": "RAG Answer: Noveum.ai handles agent tracing through its specialized Multi-Agent Tracing capabilities, which are designed to observe complex workflows and inter-agent communications within multi-agent systems. The...", + "evaluation.retrieval_effectiveness": 1, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 5, + "evaluation.context_relevance": "high", + "evaluation.overall_quality": "high", + "evaluation.ready_for_production": true, + "rag.retrieval_strategy": "semantic_similarity", + "rag.vector_search_results": 5, + "rag.context_synthesis": "multi_source", + "rag.document_coverage": 1, + "rag.information_density": 0.23501461988304093 + }, + "events": [], + "links": [], + "trace_trace_id": "8100b869-07c0-4e37-b51e-092c9d20b4a7", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:48:57.574000000", + "trace_end_time": "2025-10-16 08:49:04.149000000", + "trace_duration_ms": 6574.500999999999, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:48:57.574000000", + "updated_at": "2025-10-16 08:48:57.574000000" + }, + { + "span_id": "5f8ea007-edef-477c-b8f1-57c2da9e9258", + "trace_id": "c28bd31c-d200-4467-91fd-a3b80187c885", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-16 08:48:17.977000000", + "end_time": "2025-10-16 08:48:56.570000000", + "duration_ms": 38592.912, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What APIs are available in Noveum platform?", + "query.length": 43, + "processing.mode": "RAG", + "processing.time_seconds": 38.59272003173828, + "processing.response_length": 270, + "processing.sources_count": 5, + "output_response": "Final Answer: The provided context does not specify the APIs available in the Noveum platform. For detailed information about the APIs, I recommend checking the official Noveum documentation or the SDK Integration ...", + "final_answer_mode": "RAG", + "query_processed.input_query": "What APIs are available in Noveum platform?", + "query_processed.output_response": "Successfully processed query using RAG, generated 270 character response", + "query_processed.mode": "RAG", + "query_processed.processing_time": 38.59272003173828, + "query_processed.response_length": 270 + }, + "events": [], + "links": [], + "trace_trace_id": "c28bd31c-d200-4467-91fd-a3b80187c885", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:48:17.977000000", + "trace_end_time": "2025-10-16 08:48:56.570000000", + "trace_duration_ms": 38592.996, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:48:17.977000000", + "updated_at": "2025-10-16 08:48:17.977000000" + }, + { + "span_id": "db385f4d-beef-465b-9831-a65080537519", + "trace_id": "c28bd31c-d200-4467-91fd-a3b80187c885", + "parent_span_id": "5f8ea007-edef-477c-b8f1-57c2da9e9258", + "name": "agent.query_routing", + "start_time": "2025-10-16 08:48:17.977000000", + "end_time": "2025-10-16 08:48:56.570000000", + "duration_ms": 38592.66499999999, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What APIs are available in Noveum platform?", + "query_length": 43, + "query_type": "routing_query", + "output_response": "Routed to RAG for query processing", + "router.classification": "RAG", + "router.confidence_score": 1, + "router.rag_keyword_score": 3, + "router.web_keyword_score": 0, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "c28bd31c-d200-4467-91fd-a3b80187c885", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:48:17.977000000", + "trace_end_time": "2025-10-16 08:48:56.570000000", + "trace_duration_ms": 38592.996, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:48:17.977000000", + "updated_at": "2025-10-16 08:48:17.977000000" + }, + { + "span_id": "3ccebe1c-1f2a-433b-a36f-453a0b803511", + "trace_id": "c28bd31c-d200-4467-91fd-a3b80187c885", + "parent_span_id": "db385f4d-beef-465b-9831-a65080537519", + "name": "agent.llm-rag", + "start_time": "2025-10-16 08:48:17.978000000", + "end_time": "2025-10-16 08:48:56.570000000", + "duration_ms": 38592.416999999994, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "rag_agent", + "agent.operation": "llm-rag", + "agent.capabilities": [ + "document_retrieval", + "context_generation", + "response_generation" + ], + "agent.id": "noveum_rag_agent", + "input_query": "What APIs are available in Noveum platform?", + "query_length": 43, + "query_type": "rag_query", + "output_response": "RAG Answer: The provided context does not specify the APIs available in the Noveum platform. For detailed information about the APIs, I recommend checking the official Noveum documentation or the SDK Integration ...", + "rag.context_length": 2284, + "rag.sources_count": 5, + "rag.answer_length": 270, + "rag.mode": "retrieval_augmented_generation" + }, + "events": [], + "links": [], + "trace_trace_id": "c28bd31c-d200-4467-91fd-a3b80187c885", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:48:17.977000000", + "trace_end_time": "2025-10-16 08:48:56.570000000", + "trace_duration_ms": 38592.996, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:48:17.977000000", + "updated_at": "2025-10-16 08:48:17.977000000" + }, + { + "span_id": "6d1bb8e9-315d-433a-8df5-7f5fb2eeee16", + "trace_id": "c28bd31c-d200-4467-91fd-a3b80187c885", + "parent_span_id": "db385f4d-beef-465b-9831-a65080537519", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-16 08:48:17.978000000", + "end_time": "2025-10-16 08:48:17.978000000", + "duration_ms": 0.043000000000000003, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What APIs are available in Noveum platform?", + "query_length": 43, + "query_type": "routing_evaluation_query", + "classification.mode": "RAG", + "classification.rag_keyword_score": 3, + "classification.web_keyword_score": 0, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "noveum_specific", + "query.keyword_density": 0.42857142857142855, + "query.domain_affinity": "noveum", + "routing.decision": "Routed to RAG based on analysis", + "routing.rationale": "RAG score: 3, Web score: 0, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "Web Search", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.057692307692307696, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: RAG (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "c28bd31c-d200-4467-91fd-a3b80187c885", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:48:17.977000000", + "trace_end_time": "2025-10-16 08:48:56.570000000", + "trace_duration_ms": 38592.996, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:48:17.977000000", + "updated_at": "2025-10-16 08:48:17.977000000" + }, + { + "span_id": "2f11df8a-8f45-425d-a0ce-1eec7887b508", + "trace_id": "c28bd31c-d200-4467-91fd-a3b80187c885", + "parent_span_id": "3ccebe1c-1f2a-433b-a36f-453a0b803511", + "name": "agent.llm_model_execution", + "start_time": "2025-10-16 08:48:18.392000000", + "end_time": "2025-10-16 08:48:55.545000000", + "duration_ms": 37152.496999999996, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What APIs are available in Noveum platform?", + "query_length": 43, + "query_type": "rag_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 37.15243363380432, + "model.latency_ms": 37152.43363380432, + "model.start_time": 1760604498.3926067, + "model.end_time": 1760604535.5450404, + "model.performance_tier": "slow", + "model.prompt_tokens": 705, + "model.completion_tokens": 67, + "model.total_tokens": 772, + "model.tokens_per_second": 20.779257897592238, + "model.estimated_cost": 0.02316, + "model.efficiency_score": 0.34974093264248707, + "model.response_length": 270, + "model.response_quality": "high", + "model.output_response": "Model Response: The provided context does not specify the APIs available in the Noveum platform. For detailed information about the APIs, I recommend checking the official Noveum documentation or the SDK Integration ...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "c28bd31c-d200-4467-91fd-a3b80187c885", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:48:17.977000000", + "trace_end_time": "2025-10-16 08:48:56.570000000", + "trace_duration_ms": 38592.996, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:48:17.977000000", + "updated_at": "2025-10-16 08:48:17.977000000" + }, + { + "span_id": "d13e608c-06e4-47c4-b602-c40ac9560be2", + "trace_id": "c28bd31c-d200-4467-91fd-a3b80187c885", + "parent_span_id": "3ccebe1c-1f2a-433b-a36f-453a0b803511", + "name": "agent.rag_evaluation_metrics", + "start_time": "2025-10-16 08:48:55.545000000", + "end_time": "2025-10-16 08:48:55.545000000", + "duration_ms": 0.031, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "rag_evaluation_metrics", + "agent.capabilities": [ + "retrieval_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What APIs are available in Noveum platform?", + "query_length": 43, + "query_type": "rag_evaluation_query", + "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/docs):\n- \ud83c\udf0d Universal: Works across frameworks, clouds, and deployment models\n\ud83e\udd1d Community & Support\n- \ud83d\udcac Discord Community: Join our Discord\n- \ud83d\udce7 Email Support: [email protected]\n- \ud83d\udc1b Bug Reports: GitHub Issues\n- \ud83d\udcd6 Knowledge Base: Help Center\nReady to get started? Head to ...", + "retrieval.context_length": 2284, + "retrieval.sources_count": 5, + "retrieval.context_quality": "high", + "retrieval.effectiveness": 1, + "retrieval.context_utilization": 2.284, + "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/docs):\n- \ud83c\udf0d Universal: Works across frameworks, clouds, and deployment models\n\ud83e\udd1d Community & Support\n- \ud83d\udcac Discord Community: Join our Discord\n- \ud83d\udce7 Email Support: [email protected]\n- \ud83d\udc1b Bug Reports: GitHub Issues\n- \ud83d\udcd6 Knowledge Base: Help Center\nReady to get started? Head to our SDK Integration Guide to begin tracing your AI applications in under 5 minutes!\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability ...\n\nSource 2 (https://noveum.ai/docs):\n- \ud83c\udf0d Universal: Works across frameworks, clouds, and deployment models\n\ud83e\udd1d Community & Support\n- \ud83d\udcac Discord Community: Join our Discord\n- \ud83d\udce7 Email Support: [email protected]\n- \ud83d\udc1b Bug Reports: GitHub Issues\n- \ud83d\udcd6 Knowledge Base: Help Center\nReady to get started? Head to our SDK Integration Guide to begin tracing your AI applications in under 5 minutes!\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability ...\n\nSource 3 (https://noveum.ai/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nSource 4 (https://noveum.ai/en/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nSource 5 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, vector searches, data retrieval, agent reasoning, and more. Without proper observability, debugging becomes a nightmare, optimization is guesswork, and understanding user interactions is nearly impossible.\nNoveum.ai solves this by providing comprehensive tracing and observability specifically designed for AI applications.\nKey benefits of using Noveum.ai:\n- Complete Visibility: Trace...\n\nUser Question: What APIs are available in Noveum platform?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", + "prompt.prompt_length": 2823, + "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/docs):\n- \ud83c\udf0d Universal: Works across frameworks, clouds, and deployment models\n\ud83e\udd1d Community & Support\n- \ud83d\udcac Discord Community: Join our Discord\n- \ud83d\udce7 Email Support: [email prot...", + "prompt.instruction_following": "rag_optimized", + "response.answer_length": 270, + "response.answer_completeness": "complete", + "response.response_quality": "high", + "response.source_citation": 5, + "response.context_utilization": 0.11821366024518389, + "output_response": "RAG Answer: The provided context does not specify the APIs available in the Noveum platform. For detailed information about the APIs, I recommend checking the official Noveum documentation or the SDK Integration ...", + "evaluation.retrieval_effectiveness": 1, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 5, + "evaluation.context_relevance": "high", + "evaluation.overall_quality": "high", + "evaluation.ready_for_production": true, + "rag.retrieval_strategy": "semantic_similarity", + "rag.vector_search_results": 5, + "rag.context_synthesis": "multi_source", + "rag.document_coverage": 1, + "rag.information_density": 0.11821366024518389 + }, + "events": [], + "links": [], + "trace_trace_id": "c28bd31c-d200-4467-91fd-a3b80187c885", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:48:17.977000000", + "trace_end_time": "2025-10-16 08:48:56.570000000", + "trace_duration_ms": 38592.996, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:48:17.977000000", + "updated_at": "2025-10-16 08:48:17.977000000" + }, + { + "span_id": "3f3babbd-abf5-4f5a-b31c-fa5dcec9c360", + "trace_id": "b90ef3fc-11ea-4230-a59b-e2f2db8f2de0", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-16 08:48:08.007000000", + "end_time": "2025-10-16 08:48:16.975000000", + "duration_ms": 8967.201, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "How do I set up observability with Noveum?", + "query.length": 42, + "processing.mode": "RAG", + "processing.time_seconds": 8.96716833114624, + "processing.response_length": 838, + "processing.sources_count": 5, + "output_response": "Final Answer: To set up observability with Noveum, follow these steps:\n\n1. **Sign Up**: Go to [noveum.ai](https://noveum.ai) and create an account.\n2. **Create a Project**: Once signed in, create a new project to o...", + "final_answer_mode": "RAG", + "query_processed.input_query": "How do I set up observability with Noveum?", + "query_processed.output_response": "Successfully processed query using RAG, generated 838 character response", + "query_processed.mode": "RAG", + "query_processed.processing_time": 8.96716833114624, + "query_processed.response_length": 838 + }, + "events": [], + "links": [], + "trace_trace_id": "b90ef3fc-11ea-4230-a59b-e2f2db8f2de0", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:48:08.007000000", + "trace_end_time": "2025-10-16 08:48:16.975000000", + "trace_duration_ms": 8967.264, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:48:08.007000000", + "updated_at": "2025-10-16 08:48:08.007000000" + }, + { + "span_id": "41f6453c-4be6-4b66-8f9b-8e795f7b41b0", + "trace_id": "b90ef3fc-11ea-4230-a59b-e2f2db8f2de0", + "parent_span_id": "b0a76345-49e5-4c1b-915f-cec90ea2b1f7", + "name": "agent.llm-rag", + "start_time": "2025-10-16 08:48:08.007000000", + "end_time": "2025-10-16 08:48:16.974000000", + "duration_ms": 8966.964, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "rag_agent", + "agent.operation": "llm-rag", + "agent.capabilities": [ + "document_retrieval", + "context_generation", + "response_generation" + ], + "agent.id": "noveum_rag_agent", + "input_query": "How do I set up observability with Noveum?", + "query_length": 42, + "query_type": "rag_query", + "output_response": "RAG Answer: To set up observability with Noveum, follow these steps:\n\n1. **Sign Up**: Go to [noveum.ai](https://noveum.ai) and create an account.\n2. **Create a Project**: Once signed in, create a new project to o...", + "rag.context_length": 2870, + "rag.sources_count": 6, + "rag.answer_length": 838, + "rag.mode": "retrieval_augmented_generation" + }, + "events": [], + "links": [], + "trace_trace_id": "b90ef3fc-11ea-4230-a59b-e2f2db8f2de0", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:48:08.007000000", + "trace_end_time": "2025-10-16 08:48:16.975000000", + "trace_duration_ms": 8967.264, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:48:08.007000000", + "updated_at": "2025-10-16 08:48:08.007000000" + }, + { + "span_id": "b0a76345-49e5-4c1b-915f-cec90ea2b1f7", + "trace_id": "b90ef3fc-11ea-4230-a59b-e2f2db8f2de0", + "parent_span_id": "3f3babbd-abf5-4f5a-b31c-fa5dcec9c360", + "name": "agent.query_routing", + "start_time": "2025-10-16 08:48:08.007000000", + "end_time": "2025-10-16 08:48:16.974000000", + "duration_ms": 8967.129, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "How do I set up observability with Noveum?", + "query_length": 42, + "query_type": "routing_query", + "output_response": "Routed to RAG for query processing", + "router.classification": "RAG", + "router.confidence_score": 1, + "router.rag_keyword_score": 2, + "router.web_keyword_score": 0, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "b90ef3fc-11ea-4230-a59b-e2f2db8f2de0", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:48:08.007000000", + "trace_end_time": "2025-10-16 08:48:16.975000000", + "trace_duration_ms": 8967.264, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:48:08.007000000", + "updated_at": "2025-10-16 08:48:08.007000000" + }, + { + "span_id": "fe3074a7-f4c0-442e-9a5b-e2cc11d93785", + "trace_id": "b90ef3fc-11ea-4230-a59b-e2f2db8f2de0", + "parent_span_id": "b0a76345-49e5-4c1b-915f-cec90ea2b1f7", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-16 08:48:08.007000000", + "end_time": "2025-10-16 08:48:08.007000000", + "duration_ms": 0.032, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: How do I set up observability with Noveum?", + "query_length": 42, + "query_type": "routing_evaluation_query", + "classification.mode": "RAG", + "classification.rag_keyword_score": 2, + "classification.web_keyword_score": 0, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "noveum_specific", + "query.keyword_density": 0.25, + "query.domain_affinity": "noveum", + "routing.decision": "Routed to RAG based on analysis", + "routing.rationale": "RAG score: 2, Web score: 0, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "Web Search", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.038461538461538464, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: RAG (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "b90ef3fc-11ea-4230-a59b-e2f2db8f2de0", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:48:08.007000000", + "trace_end_time": "2025-10-16 08:48:16.975000000", + "trace_duration_ms": 8967.264, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:48:08.007000000", + "updated_at": "2025-10-16 08:48:08.007000000" + }, + { + "span_id": "e16139ee-048e-4c2f-87f8-90442f41a0ae", + "trace_id": "b90ef3fc-11ea-4230-a59b-e2f2db8f2de0", + "parent_span_id": "41f6453c-4be6-4b66-8f9b-8e795f7b41b0", + "name": "agent.llm_model_execution", + "start_time": "2025-10-16 08:48:08.644000000", + "end_time": "2025-10-16 08:48:15.709000000", + "duration_ms": 7064.898, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: How do I set up observability with Noveum?", + "query_length": 42, + "query_type": "rag_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 7.0648510456085205, + "model.latency_ms": 7064.8510456085205, + "model.start_time": 1760604488.6448367, + "model.end_time": 1760604495.7096877, + "model.performance_tier": "slow", + "model.prompt_tokens": 852, + "model.completion_tokens": 209, + "model.total_tokens": 1061, + "model.tokens_per_second": 150.1800948315128, + "model.estimated_cost": 0.031830000000000004, + "model.efficiency_score": 0.7898209236569275, + "model.response_length": 838, + "model.response_quality": "high", + "model.output_response": "Model Response: To set up observability with Noveum, follow these steps:\n\n1. **Sign Up**: Go to [noveum.ai](https://noveum.ai) and create an account.\n2. **Create a Project**: Once signed in, create a new project to o...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "b90ef3fc-11ea-4230-a59b-e2f2db8f2de0", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:48:08.007000000", + "trace_end_time": "2025-10-16 08:48:16.975000000", + "trace_duration_ms": 8967.264, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:48:08.007000000", + "updated_at": "2025-10-16 08:48:08.007000000" + }, + { + "span_id": "59e85f8e-c19b-4794-9a69-7b5bc90208f0", + "trace_id": "b90ef3fc-11ea-4230-a59b-e2f2db8f2de0", + "parent_span_id": "41f6453c-4be6-4b66-8f9b-8e795f7b41b0", + "name": "agent.rag_evaluation_metrics", + "start_time": "2025-10-16 08:48:15.709000000", + "end_time": "2025-10-16 08:48:15.709000000", + "duration_ms": 0.039, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "rag_evaluation_metrics", + "agent.capabilities": [ + "retrieval_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: How do I set up observability with Noveum?", + "query_length": 42, + "query_type": "rag_evaluation_query", + "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\n- Production Ready: Built for scale with intelligent sampling and batching\n- Privacy Focused: You control what data is captured and how it's stored\n- Open Source SDKs: Transparent, auditable, and extensible\nGetting ...", + "retrieval.context_length": 2870, + "retrieval.sources_count": 6, + "retrieval.context_quality": "high", + "retrieval.effectiveness": 1.2, + "retrieval.context_utilization": 2.87, + "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\n- Production Ready: Built for scale with intelligent sampling and batching\n- Privacy Focused: You control what data is captured and how it's stored\n- Open Source SDKs: Transparent, auditable, and extensible\nGetting Started\nReady to add comprehensive observability to your AI applications?\n- Sign up at noveum.ai\n- Create a project and get your API key\n- Install the SDK for your preferred language\n- Add tracing to your AI workflows\n- Explore insights in the Noveum.ai dashboard\nCheck out our integra...\n\nSource 2 (https://noveum.ai/en/docs/getting-started/tracing-concepts):\n- Explore Framework Integrations - Framework-specific guidance\n- Learn Advanced Patterns - Custom instrumentation techniques\n- Master the Dashboard - Analyze your traces effectively\nRemember: Good observability is not about collecting all possible data, but about collecting the right data that helps you understand, debug, and optimize your AI applications.\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to O...\n\nSource 3 (https://noveum.ai/docs/getting-started/tracing-concepts):\n- Explore Framework Integrations - Framework-specific guidance\n- Learn Advanced Patterns - Custom instrumentation techniques\n- Master the Dashboard - Analyze your traces effectively\nRemember: Good observability is not about collecting all possible data, but about collecting the right data that helps you understand, debug, and optimize your AI applications.\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to O...\n\nSource 4 (https://noveum.ai/en/docs/getting-started/overview):\nNoveum.ai Overview\nComprehensive AI tracing and observability platform for LLM applications, RAG systems, and AI agents\nWelcome to Noveum.ai\u2014the comprehensive tracing and observability platform built specifically for AI applications. Whether you're building LLM-powered chatbots, RAG systems, multi-agent workflows, or any AI-driven application, Noveum provides the insights you need to understand, debug, and optimize your systems.\n\ud83c\udfaf Why AI Applications Need Specialized Observability\nTraditional mo...\n\nSource 5 (https://noveum.ai/docs/getting-started/overview):\nNoveum.ai Overview\nComprehensive AI tracing and observability platform for LLM applications, RAG systems, and AI agents\nWelcome to Noveum.ai\u2014the comprehensive tracing and observability platform built specifically for AI applications. Whether you're building LLM-powered chatbots, RAG systems, multi-agent workflows, or any AI-driven application, Noveum provides the insights you need to understand, debug, and optimize your systems.\n\ud83c\udfaf Why AI Applications Need Specialized Observability\nTraditional mo...\n\nUser Question: How do I set up observability with Noveum?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", + "prompt.prompt_length": 3408, + "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\n- Production Ready: Built for scale with intelligent sampling and batching\n- Privacy Focused: You control what data...", + "prompt.instruction_following": "rag_optimized", + "response.answer_length": 838, + "response.answer_completeness": "complete", + "response.response_quality": "high", + "response.source_citation": 6, + "response.context_utilization": 0.29198606271777006, + "output_response": "RAG Answer: To set up observability with Noveum, follow these steps:\n\n1. **Sign Up**: Go to [noveum.ai](https://noveum.ai) and create an account.\n2. **Create a Project**: Once signed in, create a new project to o...", + "evaluation.retrieval_effectiveness": 1.2, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 6, + "evaluation.context_relevance": "high", + "evaluation.overall_quality": "high", + "evaluation.ready_for_production": true, + "rag.retrieval_strategy": "semantic_similarity", + "rag.vector_search_results": 6, + "rag.context_synthesis": "multi_source", + "rag.document_coverage": 1.2, + "rag.information_density": 0.29198606271777006 + }, + "events": [], + "links": [], + "trace_trace_id": "b90ef3fc-11ea-4230-a59b-e2f2db8f2de0", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:48:08.007000000", + "trace_end_time": "2025-10-16 08:48:16.975000000", + "trace_duration_ms": 8967.264, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:48:08.007000000", + "updated_at": "2025-10-16 08:48:08.007000000" + }, + { + "span_id": "f049ed2b-ff2f-4092-9974-acc66f81bd8d", + "trace_id": "b8abebac-a7dc-494c-a1eb-6470181f32d2", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-16 08:47:57.667000000", + "end_time": "2025-10-16 08:48:07.005000000", + "duration_ms": 9337.298, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What features does Noveum Trace offer?", + "query.length": 38, + "processing.mode": "RAG", + "processing.time_seconds": 9.33726453781128, + "processing.response_length": 1018, + "processing.sources_count": 5, + "output_response": "Final Answer: Noveum Trace offers several key features through its Python SDK (`noveum-trace`) and TypeScript SDK (`@noveum/trace`):\n\n1. **Decorator-Based Tracing**: It provides a decorator-based approach for traci...", + "final_answer_mode": "RAG", + "query_processed.input_query": "What features does Noveum Trace offer?", + "query_processed.output_response": "Successfully processed query using RAG, generated 1018 character response", + "query_processed.mode": "RAG", + "query_processed.processing_time": 9.33726453781128, + "query_processed.response_length": 1018 + }, + "events": [], + "links": [], + "trace_trace_id": "b8abebac-a7dc-494c-a1eb-6470181f32d2", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:47:57.667000000", + "trace_end_time": "2025-10-16 08:48:07.005000000", + "trace_duration_ms": 9337.405, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:47:57.667000000", + "updated_at": "2025-10-16 08:47:57.667000000" + }, + { + "span_id": "211f1fe7-4cfb-4c1c-9c72-62ae2825dda3", + "trace_id": "b8abebac-a7dc-494c-a1eb-6470181f32d2", + "parent_span_id": "f049ed2b-ff2f-4092-9974-acc66f81bd8d", + "name": "agent.query_routing", + "start_time": "2025-10-16 08:47:57.668000000", + "end_time": "2025-10-16 08:48:07.005000000", + "duration_ms": 9337.223, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What features does Noveum Trace offer?", + "query_length": 38, + "query_type": "routing_query", + "output_response": "Routed to RAG for query processing", + "router.classification": "RAG", + "router.confidence_score": 1, + "router.rag_keyword_score": 3, + "router.web_keyword_score": 0, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "b8abebac-a7dc-494c-a1eb-6470181f32d2", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:47:57.667000000", + "trace_end_time": "2025-10-16 08:48:07.005000000", + "trace_duration_ms": 9337.405, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:47:57.667000000", + "updated_at": "2025-10-16 08:47:57.667000000" + }, + { + "span_id": "976c0c4b-efc1-4b7d-bfad-350c7fd0f71e", + "trace_id": "b8abebac-a7dc-494c-a1eb-6470181f32d2", + "parent_span_id": "211f1fe7-4cfb-4c1c-9c72-62ae2825dda3", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-16 08:47:57.668000000", + "end_time": "2025-10-16 08:47:57.668000000", + "duration_ms": 0.028, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What features does Noveum Trace offer?", + "query_length": 38, + "query_type": "routing_evaluation_query", + "classification.mode": "RAG", + "classification.rag_keyword_score": 3, + "classification.web_keyword_score": 0, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "noveum_specific", + "query.keyword_density": 0.5, + "query.domain_affinity": "noveum", + "routing.decision": "Routed to RAG based on analysis", + "routing.rationale": "RAG score: 3, Web score: 0, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "Web Search", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.057692307692307696, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: RAG (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "b8abebac-a7dc-494c-a1eb-6470181f32d2", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:47:57.667000000", + "trace_end_time": "2025-10-16 08:48:07.005000000", + "trace_duration_ms": 9337.405, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:47:57.667000000", + "updated_at": "2025-10-16 08:47:57.667000000" + }, + { + "span_id": "d2afaf74-f63b-4141-9fa3-662162f51cc0", + "trace_id": "b8abebac-a7dc-494c-a1eb-6470181f32d2", + "parent_span_id": "211f1fe7-4cfb-4c1c-9c72-62ae2825dda3", + "name": "agent.llm-rag", + "start_time": "2025-10-16 08:47:57.668000000", + "end_time": "2025-10-16 08:48:07.005000000", + "duration_ms": 9337.056, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "rag_agent", + "agent.operation": "llm-rag", + "agent.capabilities": [ + "document_retrieval", + "context_generation", + "response_generation" + ], + "agent.id": "noveum_rag_agent", + "input_query": "What features does Noveum Trace offer?", + "query_length": 38, + "query_type": "rag_query", + "output_response": "RAG Answer: Noveum Trace offers several key features through its Python SDK (`noveum-trace`) and TypeScript SDK (`@noveum/trace`):\n\n1. **Decorator-Based Tracing**: It provides a decorator-based approach for traci...", + "rag.context_length": 2787, + "rag.sources_count": 5, + "rag.answer_length": 1018, + "rag.mode": "retrieval_augmented_generation" + }, + "events": [], + "links": [], + "trace_trace_id": "b8abebac-a7dc-494c-a1eb-6470181f32d2", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:47:57.667000000", + "trace_end_time": "2025-10-16 08:48:07.005000000", + "trace_duration_ms": 9337.405, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:47:57.667000000", + "updated_at": "2025-10-16 08:47:57.667000000" + }, + { + "span_id": "09811a73-405d-4c0d-ba5d-0472b3708062", + "trace_id": "b8abebac-a7dc-494c-a1eb-6470181f32d2", + "parent_span_id": "d2afaf74-f63b-4141-9fa3-662162f51cc0", + "name": "agent.llm_model_execution", + "start_time": "2025-10-16 08:47:58.659000000", + "end_time": "2025-10-16 08:48:05.878000000", + "duration_ms": 7218.618, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What features does Noveum Trace offer?", + "query_length": 38, + "query_type": "rag_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 7.218573331832886, + "model.latency_ms": 7218.573331832886, + "model.start_time": 1760604478.659586, + "model.end_time": 1760604485.8781593, + "model.performance_tier": "slow", + "model.prompt_tokens": 830, + "model.completion_tokens": 254, + "model.total_tokens": 1084, + "model.tokens_per_second": 150.16817730724068, + "model.estimated_cost": 0.03252, + "model.efficiency_score": 0.9391143911439115, + "model.response_length": 1018, + "model.response_quality": "high", + "model.output_response": "Model Response: Noveum Trace offers several key features through its Python SDK (`noveum-trace`) and TypeScript SDK (`@noveum/trace`):\n\n1. **Decorator-Based Tracing**: It provides a decorator-based approach for traci...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "b8abebac-a7dc-494c-a1eb-6470181f32d2", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:47:57.667000000", + "trace_end_time": "2025-10-16 08:48:07.005000000", + "trace_duration_ms": 9337.405, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:47:57.667000000", + "updated_at": "2025-10-16 08:47:57.667000000" + }, + { + "span_id": "015688fe-d287-48e8-ac59-63781ffd6f89", + "trace_id": "b8abebac-a7dc-494c-a1eb-6470181f32d2", + "parent_span_id": "d2afaf74-f63b-4141-9fa3-662162f51cc0", + "name": "agent.rag_evaluation_metrics", + "start_time": "2025-10-16 08:48:05.878000000", + "end_time": "2025-10-16 08:48:05.878000000", + "duration_ms": 0.024, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "rag_evaluation_metrics", + "agent.capabilities": [ + "retrieval_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What features does Noveum Trace offer?", + "query_length": 38, + "query_type": "rag_evaluation_query", + "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\n\ud83d\udcd8 TypeScript SDK (\n@noveum/trace\n)- Framework ...", + "retrieval.context_length": 2787, + "retrieval.sources_count": 5, + "retrieval.context_quality": "high", + "retrieval.effectiveness": 1, + "retrieval.context_utilization": 2.787, + "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\n\ud83d\udcd8 TypeScript SDK (\n@noveum/trace\n)- Framework integrations for Next.js, Express.js, and Hono\n- TypeScript-first design with full type safety\n- Universal compatibility (Node.js, Edge Runtime, browsers)\n-\n\ud83d\udcca Noveum Platform\n- Real-time dashboard for analyzing traces and performance\n- Ad...\n\nSource 2 (https://noveum.ai/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\n\ud83d\udcd8 TypeScript SDK (\n@noveum/trace\n)- Framework integrations for Next.js, Express.js, and Hono\n- TypeScript-first design with full type safety\n- Universal compatibility (Node.js, Edge Runtime, browsers)\n-\n\ud83d\udcca Noveum Platform\n- Real-time dashboard for analyzing traces and performance\n- Ad...\n\nSource 3 (https://noveum.ai/en/docs/getting-started/sdk-integration):\n- \ud83d\udc65 Collaborate with your team on insights\n\ud83d\udd12 Security & Privacy\n- \ud83d\udd10 Encryption: All data encrypted in transit and at rest\n- \ud83c\udf9b\ufe0f Configurable Capture: Control what data is collected\n- \ud83c\udfe0 Data Residency: Choose your data storage region\n- \u23f0 Retention Control: Set custom data retention policies\nNext Steps\n- Tracing Concepts - Learn about traces, spans, and observability best practices\n- Framework Integrations - Deep dive into specific framework setups\n- Multi-Agent Tracing - Observe complex agent work...\n\nSource 4 (https://noveum.ai/docs/getting-started/sdk-integration):\n- \ud83d\udc65 Collaborate with your team on insights\n\ud83d\udd12 Security & Privacy\n- \ud83d\udd10 Encryption: All data encrypted in transit and at rest\n- \ud83c\udf9b\ufe0f Configurable Capture: Control what data is collected\n- \ud83c\udfe0 Data Residency: Choose your data storage region\n- \u23f0 Retention Control: Set custom data retention policies\nNext Steps\n- Tracing Concepts - Learn about traces, spans, and observability best practices\n- Framework Integrations - Deep dive into specific framework setups\n- Multi-Agent Tracing - Observe complex agent work...\n\nSource 5 (https://noveum.ai/docs/platform/dashboard):\n- Bulk Export: Download multiple traces simultaneously\n- Comparative Analysis: Compare performance across multiple traces\nReal-Time Updates\n- Live Refresh: Automatic updates as new traces arrive\n- Connection Monitoring: Real-time status of your trace ingestion pipeline\n- Performance Indicators: Live metrics for system health monitoring\nReady to dive deeper? Explore Projects & Environments to organize your AI applications, or check out Team Collaboration to share insights with your team.\nGet Earl...\n\nUser Question: What features does Noveum Trace offer?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", + "prompt.prompt_length": 3321, + "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frame...", + "prompt.instruction_following": "rag_optimized", + "response.answer_length": 1018, + "response.answer_completeness": "complete", + "response.response_quality": "high", + "response.source_citation": 5, + "response.context_utilization": 0.36526731252242556, + "output_response": "RAG Answer: Noveum Trace offers several key features through its Python SDK (`noveum-trace`) and TypeScript SDK (`@noveum/trace`):\n\n1. **Decorator-Based Tracing**: It provides a decorator-based approach for traci...", + "evaluation.retrieval_effectiveness": 1, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 5, + "evaluation.context_relevance": "high", + "evaluation.overall_quality": "high", + "evaluation.ready_for_production": true, + "rag.retrieval_strategy": "semantic_similarity", + "rag.vector_search_results": 5, + "rag.context_synthesis": "multi_source", + "rag.document_coverage": 1, + "rag.information_density": 0.36526731252242556 + }, + "events": [], + "links": [], + "trace_trace_id": "b8abebac-a7dc-494c-a1eb-6470181f32d2", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:47:57.667000000", + "trace_end_time": "2025-10-16 08:48:07.005000000", + "trace_duration_ms": 9337.405, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:47:57.667000000", + "updated_at": "2025-10-16 08:47:57.667000000" + }, + { + "span_id": "46ee4d00-f914-41d8-9f91-c2db85f1c613", + "trace_id": "be9ccbaf-ccb4-4ebd-a380-4247f4d01868", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-16 08:47:52.037000000", + "end_time": "2025-10-16 08:47:56.662000000", + "duration_ms": 4625.395, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What are Noveum's pricing plans?", + "query.length": 32, + "processing.mode": "RAG", + "processing.time_seconds": 4.6253437995910645, + "processing.response_length": 471, + "processing.sources_count": 5, + "output_response": "Final Answer: The provided context does not contain specific information about Noveum's pricing plans. It mentions that all users will have access to the Observability suite for free, and early users will receive f...", + "final_answer_mode": "RAG", + "query_processed.input_query": "What are Noveum's pricing plans?", + "query_processed.output_response": "Successfully processed query using RAG, generated 471 character response", + "query_processed.mode": "RAG", + "query_processed.processing_time": 4.6253437995910645, + "query_processed.response_length": 471 + }, + "events": [], + "links": [], + "trace_trace_id": "be9ccbaf-ccb4-4ebd-a380-4247f4d01868", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:47:52.037000000", + "trace_end_time": "2025-10-16 08:47:56.662000000", + "trace_duration_ms": 4625.492, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:47:52.037000000", + "updated_at": "2025-10-16 08:47:52.037000000" + }, + { + "span_id": "6d15369a-ae35-45c7-84dd-bdf484f73f2d", + "trace_id": "be9ccbaf-ccb4-4ebd-a380-4247f4d01868", + "parent_span_id": "46ee4d00-f914-41d8-9f91-c2db85f1c613", + "name": "agent.query_routing", + "start_time": "2025-10-16 08:47:52.037000000", + "end_time": "2025-10-16 08:47:56.662000000", + "duration_ms": 4625.286, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What are Noveum's pricing plans?", + "query_length": 32, + "query_type": "routing_query", + "output_response": "Routed to RAG for query processing", + "router.classification": "RAG", + "router.confidence_score": 1, + "router.rag_keyword_score": 3, + "router.web_keyword_score": 0, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "be9ccbaf-ccb4-4ebd-a380-4247f4d01868", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:47:52.037000000", + "trace_end_time": "2025-10-16 08:47:56.662000000", + "trace_duration_ms": 4625.492, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:47:52.037000000", + "updated_at": "2025-10-16 08:47:52.037000000" + }, + { + "span_id": "bf201676-a3a9-433a-a8c6-d394b7aead3a", + "trace_id": "be9ccbaf-ccb4-4ebd-a380-4247f4d01868", + "parent_span_id": "6d15369a-ae35-45c7-84dd-bdf484f73f2d", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-16 08:47:52.037000000", + "end_time": "2025-10-16 08:47:52.037000000", + "duration_ms": 0.048, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What are Noveum's pricing plans?", + "query_length": 32, + "query_type": "routing_evaluation_query", + "classification.mode": "RAG", + "classification.rag_keyword_score": 3, + "classification.web_keyword_score": 0, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "noveum_specific", + "query.keyword_density": 0.6, + "query.domain_affinity": "noveum", + "routing.decision": "Routed to RAG based on analysis", + "routing.rationale": "RAG score: 3, Web score: 0, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "Web Search", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.057692307692307696, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: RAG (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "be9ccbaf-ccb4-4ebd-a380-4247f4d01868", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:47:52.037000000", + "trace_end_time": "2025-10-16 08:47:56.662000000", + "trace_duration_ms": 4625.492, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:47:52.037000000", + "updated_at": "2025-10-16 08:47:52.037000000" + }, + { + "span_id": "f8346f60-131b-4007-ba3e-7b0ff10cdfde", + "trace_id": "be9ccbaf-ccb4-4ebd-a380-4247f4d01868", + "parent_span_id": "6d15369a-ae35-45c7-84dd-bdf484f73f2d", + "name": "agent.llm-rag", + "start_time": "2025-10-16 08:47:52.037000000", + "end_time": "2025-10-16 08:47:56.662000000", + "duration_ms": 4624.998, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "rag_agent", + "agent.operation": "llm-rag", + "agent.capabilities": [ + "document_retrieval", + "context_generation", + "response_generation" + ], + "agent.id": "noveum_rag_agent", + "input_query": "What are Noveum's pricing plans?", + "query_length": 32, + "query_type": "rag_query", + "output_response": "RAG Answer: The provided context does not contain specific information about Noveum's pricing plans. It mentions that all users will have access to the Observability suite for free, and early users will receive f...", + "rag.context_length": 2105, + "rag.sources_count": 5, + "rag.answer_length": 471, + "rag.mode": "retrieval_augmented_generation" + }, + "events": [], + "links": [], + "trace_trace_id": "be9ccbaf-ccb4-4ebd-a380-4247f4d01868", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:47:52.037000000", + "trace_end_time": "2025-10-16 08:47:56.662000000", + "trace_duration_ms": 4625.492, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:47:52.037000000", + "updated_at": "2025-10-16 08:47:52.037000000" + }, + { + "span_id": "0186c28c-7596-4279-a4eb-7733cd665609", + "trace_id": "be9ccbaf-ccb4-4ebd-a380-4247f4d01868", + "parent_span_id": "f8346f60-131b-4007-ba3e-7b0ff10cdfde", + "name": "agent.llm_model_execution", + "start_time": "2025-10-16 08:47:52.579000000", + "end_time": "2025-10-16 08:47:55.842000000", + "duration_ms": 3262.5829999999996, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What are Noveum's pricing plans?", + "query_length": 32, + "query_type": "rag_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 3.2625203132629395, + "model.latency_ms": 3262.5203132629395, + "model.start_time": 1760604472.580008, + "model.end_time": 1760604475.8425283, + "model.performance_tier": "medium", + "model.prompt_tokens": 658, + "model.completion_tokens": 117, + "model.total_tokens": 775, + "model.tokens_per_second": 237.54641368804243, + "model.estimated_cost": 0.02325, + "model.efficiency_score": 0.607741935483871, + "model.response_length": 471, + "model.response_quality": "high", + "model.output_response": "Model Response: The provided context does not contain specific information about Noveum's pricing plans. It mentions that all users will have access to the Observability suite for free, and early users will receive f...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "be9ccbaf-ccb4-4ebd-a380-4247f4d01868", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:47:52.037000000", + "trace_end_time": "2025-10-16 08:47:56.662000000", + "trace_duration_ms": 4625.492, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:47:52.037000000", + "updated_at": "2025-10-16 08:47:52.037000000" + }, + { + "span_id": "3bdf8d5b-c10d-44f3-8514-8facc7f63db8", + "trace_id": "be9ccbaf-ccb4-4ebd-a380-4247f4d01868", + "parent_span_id": "f8346f60-131b-4007-ba3e-7b0ff10cdfde", + "name": "agent.rag_evaluation_metrics", + "start_time": "2025-10-16 08:47:55.842000000", + "end_time": "2025-10-16 08:47:55.842000000", + "duration_ms": 0.032, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "rag_evaluation_metrics", + "agent.capabilities": [ + "retrieval_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What are Noveum's pricing plans?", + "query_length": 32, + "query_type": "rag_evaluation_query", + "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....", + "retrieval.context_length": 2105, + "retrieval.sources_count": 5, + "retrieval.context_quality": "high", + "retrieval.effectiveness": 1, + "retrieval.context_utilization": 2.105, + "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nSource 2 (https://noveum.ai/en/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nSource 3 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nThe AI landscape is complex and moving fast. With Noveum.ai, you'll have the insights you need to build, debug, and optimize AI applications with confidence.\nReady to see what's really happening inside your AI applications? Try Noveum.ai today and transform how you build and monitor AI systems.\nLet's build more reliable, observable AI\u2014together.\nGet Early Access to Noveum.ai Platform\nJoin the select group of AI teams optimizing their models with our data-driven platform. We're onboarding users in...\n\nSource 4 (https://noveum.ai/en/blog/from-logs-to-intelligent-choices-inside-noveum-ais-evaluation-process):\nGet Early Access to Noveum.ai Platform\nJoin the select group of AI teams optimizing their models with our data-driven platform. We're onboarding users in limited batches to ensure a premium experience....\n\nSource 5 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, vector searches, data retrieval, agent reasoning, and more. Without proper observability, debugging becomes a nightmare, optimization is guesswork, and understanding user interactions is nearly impossible.\nNoveum.ai solves this by providing comprehensive tracing and observability specifically designed for AI applications.\nKey benefits of using Noveum.ai:\n- Complete Visibility: Trace...\n\nUser Question: What are Noveum's pricing plans?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", + "prompt.prompt_length": 2633, + "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to ...", + "prompt.instruction_following": "rag_optimized", + "response.answer_length": 471, + "response.answer_completeness": "complete", + "response.response_quality": "high", + "response.source_citation": 5, + "response.context_utilization": 0.22375296912114015, + "output_response": "RAG Answer: The provided context does not contain specific information about Noveum's pricing plans. It mentions that all users will have access to the Observability suite for free, and early users will receive f...", + "evaluation.retrieval_effectiveness": 1, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 5, + "evaluation.context_relevance": "high", + "evaluation.overall_quality": "high", + "evaluation.ready_for_production": true, + "rag.retrieval_strategy": "semantic_similarity", + "rag.vector_search_results": 5, + "rag.context_synthesis": "multi_source", + "rag.document_coverage": 1, + "rag.information_density": 0.22375296912114015 + }, + "events": [], + "links": [], + "trace_trace_id": "be9ccbaf-ccb4-4ebd-a380-4247f4d01868", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:47:52.037000000", + "trace_end_time": "2025-10-16 08:47:56.662000000", + "trace_duration_ms": 4625.492, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:47:52.037000000", + "updated_at": "2025-10-16 08:47:52.037000000" + }, + { + "span_id": "770ce93d-07ae-4b9f-ae15-c6e450634b9d", + "trace_id": "8d33a529-d9b1-4208-b528-762222d5ef8c", + "parent_span_id": "9977d5d6-a7ff-469e-bec7-5e4dfc38c149", + "name": "agent.llm-rag", + "start_time": "2025-10-16 08:47:42.103000000", + "end_time": "2025-10-16 08:47:51.032000000", + "duration_ms": 8928.644999999999, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "rag_agent", + "agent.operation": "llm-rag", + "agent.capabilities": [ + "document_retrieval", + "context_generation", + "response_generation" + ], + "agent.id": "noveum_rag_agent", + "input_query": "How do I integrate Noveum Trace in my application?", + "query_length": 50, + "query_type": "rag_query", + "output_response": "RAG Answer: To integrate Noveum Trace into your application, you can use either the Python SDK (`noveum-trace`) or the TypeScript SDK (`@noveum/trace`). Here\u2019s a quick start guide based on the documentation:\n\n1. ...", + "rag.context_length": 2803, + "rag.sources_count": 5, + "rag.answer_length": 1134, + "rag.mode": "retrieval_augmented_generation" + }, + "events": [], + "links": [], + "trace_trace_id": "8d33a529-d9b1-4208-b528-762222d5ef8c", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:47:42.103000000", + "trace_end_time": "2025-10-16 08:47:51.032000000", + "trace_duration_ms": 8928.948, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:47:42.103000000", + "updated_at": "2025-10-16 08:47:42.103000000" + }, + { + "span_id": "9977d5d6-a7ff-469e-bec7-5e4dfc38c149", + "trace_id": "8d33a529-d9b1-4208-b528-762222d5ef8c", + "parent_span_id": "f3a8fa53-5641-4a23-b4d3-762577a89151", + "name": "agent.query_routing", + "start_time": "2025-10-16 08:47:42.103000000", + "end_time": "2025-10-16 08:47:51.032000000", + "duration_ms": 8928.793000000001, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "How do I integrate Noveum Trace in my application?", + "query_length": 50, + "query_type": "routing_query", + "output_response": "Routed to RAG for query processing", + "router.classification": "RAG", + "router.confidence_score": 1, + "router.rag_keyword_score": 2, + "router.web_keyword_score": 0, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "8d33a529-d9b1-4208-b528-762222d5ef8c", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:47:42.103000000", + "trace_end_time": "2025-10-16 08:47:51.032000000", + "trace_duration_ms": 8928.948, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:47:42.103000000", + "updated_at": "2025-10-16 08:47:42.103000000" + }, + { + "span_id": "c678e222-ef74-48ee-8628-16461ab0f48f", + "trace_id": "8d33a529-d9b1-4208-b528-762222d5ef8c", + "parent_span_id": "9977d5d6-a7ff-469e-bec7-5e4dfc38c149", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-16 08:47:42.103000000", + "end_time": "2025-10-16 08:47:42.103000000", + "duration_ms": 0.027, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: How do I integrate Noveum Trace in my application?", + "query_length": 50, + "query_type": "routing_evaluation_query", + "classification.mode": "RAG", + "classification.rag_keyword_score": 2, + "classification.web_keyword_score": 0, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "noveum_specific", + "query.keyword_density": 0.2222222222222222, + "query.domain_affinity": "noveum", + "routing.decision": "Routed to RAG based on analysis", + "routing.rationale": "RAG score: 2, Web score: 0, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "Web Search", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.038461538461538464, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: RAG (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "8d33a529-d9b1-4208-b528-762222d5ef8c", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:47:42.103000000", + "trace_end_time": "2025-10-16 08:47:51.032000000", + "trace_duration_ms": 8928.948, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:47:42.103000000", + "updated_at": "2025-10-16 08:47:42.103000000" + }, + { + "span_id": "f3a8fa53-5641-4a23-b4d3-762577a89151", + "trace_id": "8d33a529-d9b1-4208-b528-762222d5ef8c", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-16 08:47:42.103000000", + "end_time": "2025-10-16 08:47:51.032000000", + "duration_ms": 8928.868, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "How do I integrate Noveum Trace in my application?", + "query.length": 50, + "processing.mode": "RAG", + "processing.time_seconds": 8.928829431533813, + "processing.response_length": 1134, + "processing.sources_count": 5, + "output_response": "Final Answer: To integrate Noveum Trace into your application, you can use either the Python SDK (`noveum-trace`) or the TypeScript SDK (`@noveum/trace`). Here\u2019s a quick start guide based on the documentation:\n\n1. ...", + "final_answer_mode": "RAG", + "query_processed.input_query": "How do I integrate Noveum Trace in my application?", + "query_processed.output_response": "Successfully processed query using RAG, generated 1134 character response", + "query_processed.mode": "RAG", + "query_processed.processing_time": 8.928829431533813, + "query_processed.response_length": 1134 + }, + "events": [], + "links": [], + "trace_trace_id": "8d33a529-d9b1-4208-b528-762222d5ef8c", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:47:42.103000000", + "trace_end_time": "2025-10-16 08:47:51.032000000", + "trace_duration_ms": 8928.948, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:47:42.103000000", + "updated_at": "2025-10-16 08:47:42.103000000" + }, + { + "span_id": "f459b71f-1d43-41b0-b9bf-d94a908b7afa", + "trace_id": "8d33a529-d9b1-4208-b528-762222d5ef8c", + "parent_span_id": "770ce93d-07ae-4b9f-ae15-c6e450634b9d", + "name": "agent.llm_model_execution", + "start_time": "2025-10-16 08:47:43.044000000", + "end_time": "2025-10-16 08:47:50.008000000", + "duration_ms": 6963.959, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: How do I integrate Noveum Trace in my application?", + "query_length": 50, + "query_type": "rag_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 6.9639081954956055, + "model.latency_ms": 6963.9081954956055, + "model.start_time": 1760604463.044064, + "model.end_time": 1760604470.0079722, + "model.performance_tier": "slow", + "model.prompt_tokens": 837, + "model.completion_tokens": 283, + "model.total_tokens": 1120, + "model.tokens_per_second": 160.8292310235276, + "model.estimated_cost": 0.0336, + "model.efficiency_score": 1.0125, + "model.response_length": 1134, + "model.response_quality": "high", + "model.output_response": "Model Response: To integrate Noveum Trace into your application, you can use either the Python SDK (`noveum-trace`) or the TypeScript SDK (`@noveum/trace`). Here\u2019s a quick start guide based on the documentation:\n\n1. ...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "8d33a529-d9b1-4208-b528-762222d5ef8c", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:47:42.103000000", + "trace_end_time": "2025-10-16 08:47:51.032000000", + "trace_duration_ms": 8928.948, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:47:42.103000000", + "updated_at": "2025-10-16 08:47:42.103000000" + }, + { + "span_id": "b8627171-b081-4b08-b900-c24375b73c3f", + "trace_id": "8d33a529-d9b1-4208-b528-762222d5ef8c", + "parent_span_id": "770ce93d-07ae-4b9f-ae15-c6e450634b9d", + "name": "agent.rag_evaluation_metrics", + "start_time": "2025-10-16 08:47:50.008000000", + "end_time": "2025-10-16 08:47:50.008000000", + "duration_ms": 0.041, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "rag_evaluation_metrics", + "agent.capabilities": [ + "retrieval_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: How do I integrate Noveum Trace in my application?", + "query_length": 50, + "query_type": "rag_evaluation_query", + "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\n\ud83d\udcd8 TypeScript SDK (\n@noveum/trace\n)- Framework ...", + "retrieval.context_length": 2803, + "retrieval.sources_count": 5, + "retrieval.context_quality": "high", + "retrieval.effectiveness": 1, + "retrieval.context_utilization": 2.803, + "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\n\ud83d\udcd8 TypeScript SDK (\n@noveum/trace\n)- Framework integrations for Next.js, Express.js, and Hono\n- TypeScript-first design with full type safety\n- Universal compatibility (Node.js, Edge Runtime, browsers)\n-\n\ud83d\udcca Noveum Platform\n- Real-time dashboard for analyzing traces and performance\n- Ad...\n\nSource 2 (https://noveum.ai/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\n\ud83d\udcd8 TypeScript SDK (\n@noveum/trace\n)- Framework integrations for Next.js, Express.js, and Hono\n- TypeScript-first design with full type safety\n- Universal compatibility (Node.js, Edge Runtime, browsers)\n-\n\ud83d\udcca Noveum Platform\n- Real-time dashboard for analyzing traces and performance\n- Ad...\n\nSource 3 (https://noveum.ai/docs/getting-started/sdk-integration):\nSDK Integration Guide\nIntegrate Noveum.ai tracing into your AI applications with Python or TypeScript SDKs\nThe Noveum.ai SDKs provide comprehensive tracing and observability for your AI applications with minimal code changes. Whether you're building LLM applications, RAG systems, or multi-agent workflows, our SDKs automatically capture essential metrics and traces.\n\ud83d\ude80 Quick Start\n1. Create Your Account & Get API Key\n- Sign up at noveum.ai\n- Create a project in your dashboard\n- Generate an API key...\n\nSource 4 (https://noveum.ai/en/docs/getting-started/sdk-integration):\nSDK Integration Guide\nIntegrate Noveum.ai tracing into your AI applications with Python or TypeScript SDKs\nThe Noveum.ai SDKs provide comprehensive tracing and observability for your AI applications with minimal code changes. Whether you're building LLM applications, RAG systems, or multi-agent workflows, our SDKs automatically capture essential metrics and traces.\n\ud83d\ude80 Quick Start\n1. Create Your Account & Get API Key\n- Sign up at noveum.ai\n- Create a project in your dashboard\n- Generate an API key...\n\nSource 5 (https://noveum.ai/en/docs/getting-started/sdk-integration):\n\ud83d\udd27 Framework Integrations\nNext.js Integration\nExpress.js Integration\nFastAPI Integration (Python)\n\ud83d\udcca Advanced Features\nCustom Attributes & Events\nSampling Configuration\n\ud83d\udd17 What's Captured Automatically\n- \ud83d\udcca Performance Metrics: Latency, throughput, error rates\n- \ud83d\udcb0 Cost Tracking: Token usage, API costs across providers\n- \ud83d\udd0d Request/Response: Configurable capture of inputs/outputs\n- \ud83c\udff7\ufe0f Metadata: Model names, parameters, user context\n- \ud83c\udf0a Context Flow: Trace relationships across services\n- \ud83d\udc1b Error Detail...\n\nUser Question: How do I integrate Noveum Trace in my application?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", + "prompt.prompt_length": 3349, + "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frame...", + "prompt.instruction_following": "rag_optimized", + "response.answer_length": 1134, + "response.answer_completeness": "complete", + "response.response_quality": "high", + "response.source_citation": 5, + "response.context_utilization": 0.4045665358544417, + "output_response": "RAG Answer: To integrate Noveum Trace into your application, you can use either the Python SDK (`noveum-trace`) or the TypeScript SDK (`@noveum/trace`). Here\u2019s a quick start guide based on the documentation:\n\n1. ...", + "evaluation.retrieval_effectiveness": 1, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 5, + "evaluation.context_relevance": "high", + "evaluation.overall_quality": "high", + "evaluation.ready_for_production": true, + "rag.retrieval_strategy": "semantic_similarity", + "rag.vector_search_results": 5, + "rag.context_synthesis": "multi_source", + "rag.document_coverage": 1, + "rag.information_density": 0.4045665358544417 + }, + "events": [], + "links": [], + "trace_trace_id": "8d33a529-d9b1-4208-b528-762222d5ef8c", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:47:42.103000000", + "trace_end_time": "2025-10-16 08:47:51.032000000", + "trace_duration_ms": 8928.948, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:47:42.103000000", + "updated_at": "2025-10-16 08:47:42.103000000" + }, + { + "span_id": "0b3d6893-776e-4118-b8b5-b1c5ea51def1", + "trace_id": "06192a09-b6bb-4548-8a94-e123d6caaffe", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-16 08:47:03.776000000", + "end_time": "2025-10-16 08:47:41.100000000", + "duration_ms": 37324.516, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What is Noveum and what does it do?", + "query.length": 35, + "processing.mode": "RAG", + "processing.time_seconds": 37.324477195739746, + "processing.response_length": 648, + "processing.sources_count": 5, + "output_response": "Final Answer: Noveum.ai is a comprehensive tracing and observability platform specifically designed for AI applications, including those powered by large language models (LLMs), retrieval-augmented generation (RAG)...", + "final_answer_mode": "RAG", + "query_processed.input_query": "What is Noveum and what does it do?", + "query_processed.output_response": "Successfully processed query using RAG, generated 648 character response", + "query_processed.mode": "RAG", + "query_processed.processing_time": 37.324477195739746, + "query_processed.response_length": 648 + }, + "events": [], + "links": [], + "trace_trace_id": "06192a09-b6bb-4548-8a94-e123d6caaffe", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:47:03.776000000", + "trace_end_time": "2025-10-16 08:47:41.100000000", + "trace_duration_ms": 37324.601, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:47:03.776000000", + "updated_at": "2025-10-16 08:47:03.776000000" + }, + { + "span_id": "137ad05b-bada-42be-abe5-be0c277f6d0e", + "trace_id": "06192a09-b6bb-4548-8a94-e123d6caaffe", + "parent_span_id": "c3664d20-1708-45f5-a051-b626b0ec77cd", + "name": "agent.llm-rag", + "start_time": "2025-10-16 08:47:03.776000000", + "end_time": "2025-10-16 08:47:41.100000000", + "duration_ms": 37324.042, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "rag_agent", + "agent.operation": "llm-rag", + "agent.capabilities": [ + "document_retrieval", + "context_generation", + "response_generation" + ], + "agent.id": "noveum_rag_agent", + "input_query": "What is Noveum and what does it do?", + "query_length": 35, + "query_type": "rag_query", + "output_response": "RAG Answer: Noveum.ai is a comprehensive tracing and observability platform specifically designed for AI applications, including those powered by large language models (LLMs), retrieval-augmented generation (RAG)...", + "rag.context_length": 2334, + "rag.sources_count": 5, + "rag.answer_length": 648, + "rag.mode": "retrieval_augmented_generation" + }, + "events": [], + "links": [], + "trace_trace_id": "06192a09-b6bb-4548-8a94-e123d6caaffe", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:47:03.776000000", + "trace_end_time": "2025-10-16 08:47:41.100000000", + "trace_duration_ms": 37324.601, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:47:03.776000000", + "updated_at": "2025-10-16 08:47:03.776000000" + }, + { + "span_id": "c3664d20-1708-45f5-a051-b626b0ec77cd", + "trace_id": "06192a09-b6bb-4548-8a94-e123d6caaffe", + "parent_span_id": "0b3d6893-776e-4118-b8b5-b1c5ea51def1", + "name": "agent.query_routing", + "start_time": "2025-10-16 08:47:03.776000000", + "end_time": "2025-10-16 08:47:41.100000000", + "duration_ms": 37324.382999999994, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What is Noveum and what does it do?", + "query_length": 35, + "query_type": "routing_query", + "output_response": "Routed to RAG for query processing", + "router.classification": "RAG", + "router.confidence_score": 1, + "router.rag_keyword_score": 1, + "router.web_keyword_score": 0, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "06192a09-b6bb-4548-8a94-e123d6caaffe", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:47:03.776000000", + "trace_end_time": "2025-10-16 08:47:41.100000000", + "trace_duration_ms": 37324.601, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:47:03.776000000", + "updated_at": "2025-10-16 08:47:03.776000000" + }, + { + "span_id": "ce9a94bf-1a72-4599-8561-11e66802b92d", + "trace_id": "06192a09-b6bb-4548-8a94-e123d6caaffe", + "parent_span_id": "c3664d20-1708-45f5-a051-b626b0ec77cd", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-16 08:47:03.776000000", + "end_time": "2025-10-16 08:47:03.776000000", + "duration_ms": 0.067, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What is Noveum and what does it do?", + "query_length": 35, + "query_type": "routing_evaluation_query", + "classification.mode": "RAG", + "classification.rag_keyword_score": 1, + "classification.web_keyword_score": 0, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "noveum_specific", + "query.keyword_density": 0.125, + "query.domain_affinity": "noveum", + "routing.decision": "Routed to RAG based on analysis", + "routing.rationale": "RAG score: 1, Web score: 0, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "Web Search", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.019230769230769232, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: RAG (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "06192a09-b6bb-4548-8a94-e123d6caaffe", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:47:03.776000000", + "trace_end_time": "2025-10-16 08:47:41.100000000", + "trace_duration_ms": 37324.601, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:47:03.776000000", + "updated_at": "2025-10-16 08:47:03.776000000" + }, + { + "span_id": "38414416-907a-4188-95fe-c0a5f900f72c", + "trace_id": "06192a09-b6bb-4548-8a94-e123d6caaffe", + "parent_span_id": "137ad05b-bada-42be-abe5-be0c277f6d0e", + "name": "agent.llm_model_execution", + "start_time": "2025-10-16 08:47:10.392000000", + "end_time": "2025-10-16 08:47:40.392000000", + "duration_ms": 30000.104, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What is Noveum and what does it do?", + "query_length": 35, + "query_type": "rag_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 30.000043392181396, + "model.latency_ms": 30000.043392181396, + "model.start_time": 1760604430.3921378, + "model.end_time": 1760604460.3921812, + "model.performance_tier": "slow", + "model.prompt_tokens": 716, + "model.completion_tokens": 162, + "model.total_tokens": 878, + "model.tokens_per_second": 29.266624335244266, + "model.estimated_cost": 0.026340000000000002, + "model.efficiency_score": 0.7380410022779044, + "model.response_length": 648, + "model.response_quality": "high", + "model.output_response": "Model Response: Noveum.ai is a comprehensive tracing and observability platform specifically designed for AI applications, including those powered by large language models (LLMs), retrieval-augmented generation (RAG)...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "06192a09-b6bb-4548-8a94-e123d6caaffe", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:47:03.776000000", + "trace_end_time": "2025-10-16 08:47:41.100000000", + "trace_duration_ms": 37324.601, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:47:03.776000000", + "updated_at": "2025-10-16 08:47:03.776000000" + }, + { + "span_id": "67e23649-b00b-4eb8-9a69-865e80f833b3", + "trace_id": "06192a09-b6bb-4548-8a94-e123d6caaffe", + "parent_span_id": "137ad05b-bada-42be-abe5-be0c277f6d0e", + "name": "agent.rag_evaluation_metrics", + "start_time": "2025-10-16 08:47:40.392000000", + "end_time": "2025-10-16 08:47:40.392000000", + "duration_ms": 0.029, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "rag_evaluation_metrics", + "agent.capabilities": [ + "retrieval_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What is Noveum and what does it do?", + "query_length": 35, + "query_type": "rag_evaluation_query", + "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, vector searches, data retrieval, agent reasoning, and more. Without proper observability, debugging ...", + "retrieval.context_length": 2334, + "retrieval.sources_count": 5, + "retrieval.context_quality": "high", + "retrieval.effectiveness": 1, + "retrieval.context_utilization": 2.334, + "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, vector searches, data retrieval, agent reasoning, and more. Without proper observability, debugging becomes a nightmare, optimization is guesswork, and understanding user interactions is nearly impossible.\nNoveum.ai solves this by providing comprehensive tracing and observability specifically designed for AI applications.\nKey benefits of using Noveum.ai:\n- Complete Visibility: Trace...\n\nSource 2 (https://noveum.ai/en/docs/getting-started/overview):\nNoveum.ai Overview\nComprehensive AI tracing and observability platform for LLM applications, RAG systems, and AI agents\nWelcome to Noveum.ai\u2014the comprehensive tracing and observability platform built specifically for AI applications. Whether you're building LLM-powered chatbots, RAG systems, multi-agent workflows, or any AI-driven application, Noveum provides the insights you need to understand, debug, and optimize your systems.\n\ud83c\udfaf Why AI Applications Need Specialized Observability\nTraditional mo...\n\nSource 3 (https://noveum.ai/docs/getting-started/overview):\nNoveum.ai Overview\nComprehensive AI tracing and observability platform for LLM applications, RAG systems, and AI agents\nWelcome to Noveum.ai\u2014the comprehensive tracing and observability platform built specifically for AI applications. Whether you're building LLM-powered chatbots, RAG systems, multi-agent workflows, or any AI-driven application, Noveum provides the insights you need to understand, debug, and optimize your systems.\n\ud83c\udfaf Why AI Applications Need Specialized Observability\nTraditional mo...\n\nSource 4 (https://noveum.ai/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nSource 5 (https://noveum.ai/en/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nUser Question: What is Noveum and what does it do?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", + "prompt.prompt_length": 2865, + "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, ...", + "prompt.instruction_following": "rag_optimized", + "response.answer_length": 648, + "response.answer_completeness": "complete", + "response.response_quality": "high", + "response.source_citation": 5, + "response.context_utilization": 0.2776349614395887, + "output_response": "RAG Answer: Noveum.ai is a comprehensive tracing and observability platform specifically designed for AI applications, including those powered by large language models (LLMs), retrieval-augmented generation (RAG)...", + "evaluation.retrieval_effectiveness": 1, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 5, + "evaluation.context_relevance": "high", + "evaluation.overall_quality": "high", + "evaluation.ready_for_production": true, + "rag.retrieval_strategy": "semantic_similarity", + "rag.vector_search_results": 5, + "rag.context_synthesis": "multi_source", + "rag.document_coverage": 1, + "rag.information_density": 0.2776349614395887 + }, + "events": [], + "links": [], + "trace_trace_id": "06192a09-b6bb-4548-8a94-e123d6caaffe", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:47:03.776000000", + "trace_end_time": "2025-10-16 08:47:41.100000000", + "trace_duration_ms": 37324.601, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:47:03.776000000", + "updated_at": "2025-10-16 08:47:03.776000000" + }, + { + "span_id": "050851c9-b2d3-4c82-accb-4085dc69f785", + "trace_id": "3552ee8e-eafb-499a-b109-a9fb851a41d2", + "parent_span_id": "", + "name": "system_initialization", + "start_time": "2025-10-16 08:47:03.695000000", + "end_time": "2025-10-16 08:47:03.770000000", + "duration_ms": 74.58, + "status": "ok", + "status_message": "", + "attributes": { + "system.force_scrape": false, + "system.config": { + "noveum_base_url": "https://noveum.ai", + "max_pages_to_scrape": 50, + "chunk_size": 1000, + "chunk_overlap": 200, + "max_search_results": 5, + "rag_threshold": 0.7, + "noveum_docs_file": "noveum_docs.json", + "vector_store_path": "noveum_vectorstore" + }, + "input_query": "Initialize system with force_scrape=False", + "output_response": "System initialization: RAG system loaded, vector store ready, agent operational", + "system.initialized": true, + "system.documents_loaded": 38, + "system.vectorstore_ready": true + }, + "events": [], + "links": [], + "trace_trace_id": "3552ee8e-eafb-499a-b109-a9fb851a41d2", + "trace_name": "auto_trace_system_initialization", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:47:03.695000000", + "trace_end_time": "2025-10-16 08:47:03.770000000", + "trace_duration_ms": 74.912, + "span_count": 1, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:47:03.695000000", + "updated_at": "2025-10-16 08:47:03.695000000" + }, + { + "span_id": "6b53235d-8b91-4d9e-8437-915828e84600", + "trace_id": "e6ef2449-7429-465b-9bff-f3def8de5bb0", + "parent_span_id": "e73ab184-50c1-4209-9276-be11e69e9513", + "name": "agent.query_routing", + "start_time": "2025-10-14 12:54:06.700000000", + "end_time": "2025-10-14 12:54:10.637000000", + "duration_ms": 3937.492, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What are the latest breakthroughs in artificial intelligence?", + "query_length": 61, + "query_type": "routing_query", + "output_response": "Routed to Web Search for query processing", + "router.classification": "Web Search", + "router.confidence_score": 1, + "router.rag_keyword_score": 0, + "router.web_keyword_score": 1, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "e6ef2449-7429-465b-9bff-f3def8de5bb0", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-14 12:54:06.700000000", + "trace_end_time": "2025-10-14 12:54:10.637000000", + "trace_duration_ms": 3937.729, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-14 12:54:06.700000000", + "updated_at": "2025-10-14 12:54:06.700000000" + }, + { + "span_id": "a7d1e33a-95a1-41a9-a117-62b4c373bd60", + "trace_id": "e6ef2449-7429-465b-9bff-f3def8de5bb0", + "parent_span_id": "6b53235d-8b91-4d9e-8437-915828e84600", + "name": "agent.web_search_generation", + "start_time": "2025-10-14 12:54:06.700000000", + "end_time": "2025-10-14 12:54:10.637000000", + "duration_ms": 3936.983, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "web_search_agent", + "agent.operation": "web_search_generation", + "agent.capabilities": [ + "web_search", + "content_synthesis", + "response_generation" + ], + "agent.id": "noveum_web_search_agent", + "input_query": "What are the latest breakthroughs in artificial intelligence?", + "query_length": 61, + "query_type": "web_search_query", + "output_response": "Web Search Answer: The web search did not yield specific results regarding the latest breakthroughs in artificial intelligence. However, I can provide a general overview based on knowledge up to October 2023.\n\nRecent br...", + "web_search.results_count": 1, + "web_search.context_length": 163, + "web_search.response_length": 1090, + "web_search.mode": "external_web_search" + }, + "events": [], + "links": [], + "trace_trace_id": "e6ef2449-7429-465b-9bff-f3def8de5bb0", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-14 12:54:06.700000000", + "trace_end_time": "2025-10-14 12:54:10.637000000", + "trace_duration_ms": 3937.729, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-14 12:54:06.700000000", + "updated_at": "2025-10-14 12:54:06.700000000" + }, + { + "span_id": "da630cab-2ba6-4f12-98b9-6ab1c0899c17", + "trace_id": "e6ef2449-7429-465b-9bff-f3def8de5bb0", + "parent_span_id": "6b53235d-8b91-4d9e-8437-915828e84600", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-14 12:54:06.700000000", + "end_time": "2025-10-14 12:54:06.700000000", + "duration_ms": 0.27999999999999997, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What are the latest breakthroughs in artificial intelligence?", + "query_length": 61, + "query_type": "routing_evaluation_query", + "classification.mode": "Web Search", + "classification.rag_keyword_score": 0, + "classification.web_keyword_score": 1, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "complex", + "query.intent": "general_knowledge", + "query.keyword_density": 0.125, + "query.domain_affinity": "general", + "routing.decision": "Routed to Web Search based on analysis", + "routing.rationale": "RAG score: 0, Web score: 1, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "RAG", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.019230769230769232, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: Web Search (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "e6ef2449-7429-465b-9bff-f3def8de5bb0", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-14 12:54:06.700000000", + "trace_end_time": "2025-10-14 12:54:10.637000000", + "trace_duration_ms": 3937.729, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-14 12:54:06.700000000", + "updated_at": "2025-10-14 12:54:06.700000000" + }, + { + "span_id": "e73ab184-50c1-4209-9276-be11e69e9513", + "trace_id": "e6ef2449-7429-465b-9bff-f3def8de5bb0", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-14 12:54:06.700000000", + "end_time": "2025-10-14 12:54:10.637000000", + "duration_ms": 3937.663, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What are the latest breakthroughs in artificial intelligence?", + "query.length": 61, + "processing.mode": "Web Search", + "processing.time_seconds": 3.93752384185791, + "processing.response_length": 1090, + "processing.sources_count": 1, + "output_response": "Final Answer: The web search did not yield specific results regarding the latest breakthroughs in artificial intelligence. However, I can provide a general overview based on knowledge up to October 2023.\n\nRecent br...", + "final_answer_mode": "Web Search", + "query_processed.input_query": "What are the latest breakthroughs in artificial intelligence?", + "query_processed.output_response": "Successfully processed query using Web Search, generated 1090 character response", + "query_processed.mode": "Web Search", + "query_processed.processing_time": 3.93752384185791, + "query_processed.response_length": 1090 + }, + "events": [], + "links": [], + "trace_trace_id": "e6ef2449-7429-465b-9bff-f3def8de5bb0", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-14 12:54:06.700000000", + "trace_end_time": "2025-10-14 12:54:10.637000000", + "trace_duration_ms": 3937.729, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-14 12:54:06.700000000", + "updated_at": "2025-10-14 12:54:06.700000000" + }, + { + "span_id": "bb47766c-afda-44fe-8838-7936c80cd4b6", + "trace_id": "e6ef2449-7429-465b-9bff-f3def8de5bb0", + "parent_span_id": "a7d1e33a-95a1-41a9-a117-62b4c373bd60", + "name": "agent.llm_model_execution", + "start_time": "2025-10-14 12:54:07.104000000", + "end_time": "2025-10-14 12:54:10.637000000", + "duration_ms": 3533.321, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What are the latest breakthroughs in artificial intelligence?", + "query_length": 61, + "query_type": "web_search_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 3.533268928527832, + "model.latency_ms": 3533.268928527832, + "model.start_time": 1760446447.104266, + "model.end_time": 1760446450.6375349, + "model.performance_tier": "medium", + "model.prompt_tokens": 167, + "model.completion_tokens": 272, + "model.total_tokens": 439, + "model.tokens_per_second": 124.2475477752307, + "model.estimated_cost": 0.013170000000000001, + "model.efficiency_score": 2.4829157175398633, + "model.response_length": 1090, + "model.response_quality": "high", + "model.output_response": "Model Response: The web search did not yield specific results regarding the latest breakthroughs in artificial intelligence. However, I can provide a general overview based on knowledge up to October 2023.\n\nRecent br...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "e6ef2449-7429-465b-9bff-f3def8de5bb0", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-14 12:54:06.700000000", + "trace_end_time": "2025-10-14 12:54:10.637000000", + "trace_duration_ms": 3937.729, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-14 12:54:06.700000000", + "updated_at": "2025-10-14 12:54:06.700000000" + }, + { + "span_id": "1d914a9d-b770-45dc-8308-6a36e9e0db89", + "trace_id": "e6ef2449-7429-465b-9bff-f3def8de5bb0", + "parent_span_id": "a7d1e33a-95a1-41a9-a117-62b4c373bd60", + "name": "agent.web_search_evaluation_metrics", + "start_time": "2025-10-14 12:54:10.637000000", + "end_time": "2025-10-14 12:54:10.637000000", + "duration_ms": 0.022, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "web_search_evaluation_metrics", + "agent.capabilities": [ + "web_search_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What are the latest breakthroughs in artificial intelligence?", + "query_length": 61, + "query_type": "web_search_evaluation_query", + "web_search.results_count": 1, + "web_search.context_length": 163, + "web_search.context_synthesized": "Context from 1 web sources", + "web_search.search_effectiveness": 0.2, + "web_search.context_quality": "low", + "web_search.source_diversity": 1, + "web_search.information_synthesis": "low", + "web_search.external_knowledge_utilization": 0.163, + "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What+are+the+latest+breakthroughs+in+artificial+intelligence?\n\nUser Question: What are the latest breakthroughs in artificial intelligence?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", + "prompt.prompt_length": 669, + "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What+are+the+latest+breakthroughs+in+artificial+intelligence?", + "prompt.instruction_following": "web_search_optimized", + "response.answer_length": 1090, + "response.answer_completeness": "complete", + "response.response_quality": "medium", + "response.source_citation": 1, + "response.context_utilization": 6.6871165644171775, + "output_response": "Web Search Answer: The web search did not yield specific results regarding the latest breakthroughs in artificial intelligence. However, I can provide a general overview based on knowledge up to October 2023.\n\nRecent br...", + "evaluation.search_effectiveness": 0.2, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 1, + "evaluation.context_relevance": "low", + "evaluation.overall_quality": "low", + "evaluation.ready_for_production": true, + "web_search.search_strategy": "duckduckgo_api", + "web_search.real_time_data": true, + "web_search.external_sources": 1, + "web_search.information_freshness": "current", + "web_search.knowledge_synthesis": "single_source" + }, + "events": [], + "links": [], + "trace_trace_id": "e6ef2449-7429-465b-9bff-f3def8de5bb0", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-14 12:54:06.700000000", + "trace_end_time": "2025-10-14 12:54:10.637000000", + "trace_duration_ms": 3937.729, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-14 12:54:06.700000000", + "updated_at": "2025-10-14 12:54:06.700000000" + }, + { + "span_id": "91f3999c-2edf-4c56-8f4d-ab43b6ebbaee", + "trace_id": "2ba8cab0-2366-4add-8ac3-b490a05e9a60", + "parent_span_id": "cbe51252-a323-4852-a90e-c907a0480fa3", + "name": "agent.web_search_generation", + "start_time": "2025-10-14 12:53:56.480000000", + "end_time": "2025-10-14 12:54:05.685000000", + "duration_ms": 9205.212, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "web_search_agent", + "agent.operation": "web_search_generation", + "agent.capabilities": [ + "web_search", + "content_synthesis", + "response_generation" + ], + "agent.id": "noveum_web_search_agent", + "input_query": "What's happening in the software development world today?", + "query_length": 57, + "query_type": "web_search_query", + "output_response": "Web Search Answer: The software development world is currently experiencing rapid changes driven by advancements in artificial intelligence (AI) and evolving programming practices. Key trends include:\n\n1. **AI Integrati...", + "web_search.results_count": 1, + "web_search.context_length": 975, + "web_search.response_length": 1636, + "web_search.mode": "external_web_search" + }, + "events": [], + "links": [], + "trace_trace_id": "2ba8cab0-2366-4add-8ac3-b490a05e9a60", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-14 12:53:56.480000000", + "trace_end_time": "2025-10-14 12:54:05.687000000", + "trace_duration_ms": 9207.67, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-14 12:53:56.480000000", + "updated_at": "2025-10-14 12:53:56.480000000" + }, + { + "span_id": "ea2aee75-4537-46ea-954b-2002fc0026c5", + "trace_id": "2ba8cab0-2366-4add-8ac3-b490a05e9a60", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-14 12:53:56.480000000", + "end_time": "2025-10-14 12:54:05.687000000", + "duration_ms": 9207.528999999999, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What's happening in the software development world today?", + "query.length": 57, + "processing.mode": "Web Search", + "processing.time_seconds": 9.206756830215454, + "processing.response_length": 1636, + "processing.sources_count": 1, + "output_response": "Final Answer: The software development world is currently experiencing rapid changes driven by advancements in artificial intelligence (AI) and evolving programming practices. Key trends include:\n\n1. **AI Integrati...", + "final_answer_mode": "Web Search", + "query_processed.input_query": "What's happening in the software development world today?", + "query_processed.output_response": "Successfully processed query using Web Search, generated 1636 character response", + "query_processed.mode": "Web Search", + "query_processed.processing_time": 9.206756830215454, + "query_processed.response_length": 1636 + }, + "events": [], + "links": [], + "trace_trace_id": "2ba8cab0-2366-4add-8ac3-b490a05e9a60", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-14 12:53:56.480000000", + "trace_end_time": "2025-10-14 12:54:05.687000000", + "trace_duration_ms": 9207.67, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-14 12:53:56.480000000", + "updated_at": "2025-10-14 12:53:56.480000000" + }, + { + "span_id": "cbe51252-a323-4852-a90e-c907a0480fa3", + "trace_id": "2ba8cab0-2366-4add-8ac3-b490a05e9a60", + "parent_span_id": "ea2aee75-4537-46ea-954b-2002fc0026c5", + "name": "agent.query_routing", + "start_time": "2025-10-14 12:53:56.480000000", + "end_time": "2025-10-14 12:54:05.686000000", + "duration_ms": 9205.569000000001, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What's happening in the software development world today?", + "query_length": 57, + "query_type": "routing_query", + "output_response": "Routed to Web Search for query processing", + "router.classification": "Web Search", + "router.confidence_score": 1, + "router.rag_keyword_score": 0, + "router.web_keyword_score": 1, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "2ba8cab0-2366-4add-8ac3-b490a05e9a60", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-14 12:53:56.480000000", + "trace_end_time": "2025-10-14 12:54:05.687000000", + "trace_duration_ms": 9207.67, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-14 12:53:56.480000000", + "updated_at": "2025-10-14 12:53:56.480000000" + }, + { + "span_id": "d850ecee-86ef-4818-9cec-638bfee26293", + "trace_id": "2ba8cab0-2366-4add-8ac3-b490a05e9a60", + "parent_span_id": "cbe51252-a323-4852-a90e-c907a0480fa3", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-14 12:53:56.480000000", + "end_time": "2025-10-14 12:53:56.480000000", + "duration_ms": 0.041999999999999996, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What's happening in the software development world today?", + "query_length": 57, + "query_type": "routing_evaluation_query", + "classification.mode": "Web Search", + "classification.rag_keyword_score": 0, + "classification.web_keyword_score": 1, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "complex", + "query.intent": "general_knowledge", + "query.keyword_density": 0.125, + "query.domain_affinity": "general", + "routing.decision": "Routed to Web Search based on analysis", + "routing.rationale": "RAG score: 0, Web score: 1, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "RAG", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.019230769230769232, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: Web Search (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "2ba8cab0-2366-4add-8ac3-b490a05e9a60", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-14 12:53:56.480000000", + "trace_end_time": "2025-10-14 12:54:05.687000000", + "trace_duration_ms": 9207.67, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-14 12:53:56.480000000", + "updated_at": "2025-10-14 12:53:56.480000000" + }, + { + "span_id": "0d4387a4-adf3-42c1-aca7-51453a90dd36", + "trace_id": "2ba8cab0-2366-4add-8ac3-b490a05e9a60", + "parent_span_id": "91f3999c-2edf-4c56-8f4d-ab43b6ebbaee", + "name": "agent.llm_model_execution", + "start_time": "2025-10-14 12:53:59.426000000", + "end_time": "2025-10-14 12:54:05.684000000", + "duration_ms": 6258.643, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What's happening in the software development world today?", + "query_length": 57, + "query_type": "web_search_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 6.25848388671875, + "model.latency_ms": 6258.48388671875, + "model.start_time": 1760446439.426367, + "model.end_time": 1760446445.684851, + "model.performance_tier": "slow", + "model.prompt_tokens": 369, + "model.completion_tokens": 409, + "model.total_tokens": 778, + "model.tokens_per_second": 124.31125718019486, + "model.estimated_cost": 0.02334, + "model.efficiency_score": 2.102827763496144, + "model.response_length": 1636, + "model.response_quality": "high", + "model.output_response": "Model Response: The software development world is currently experiencing rapid changes driven by advancements in artificial intelligence (AI) and evolving programming practices. Key trends include:\n\n1. **AI Integrati...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "2ba8cab0-2366-4add-8ac3-b490a05e9a60", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-14 12:53:56.480000000", + "trace_end_time": "2025-10-14 12:54:05.687000000", + "trace_duration_ms": 9207.67, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-14 12:53:56.480000000", + "updated_at": "2025-10-14 12:53:56.480000000" + }, + { + "span_id": "717e3c4a-6300-45e5-9456-caa3e773d758", + "trace_id": "2ba8cab0-2366-4add-8ac3-b490a05e9a60", + "parent_span_id": "91f3999c-2edf-4c56-8f4d-ab43b6ebbaee", + "name": "agent.web_search_evaluation_metrics", + "start_time": "2025-10-14 12:54:05.685000000", + "end_time": "2025-10-14 12:54:05.685000000", + "duration_ms": 0.044, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "web_search_evaluation_metrics", + "agent.capabilities": [ + "web_search_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What's happening in the software development world today?", + "query_length": 57, + "query_type": "web_search_evaluation_query", + "web_search.results_count": 1, + "web_search.context_length": 975, + "web_search.context_synthesized": "Context from 1 web sources", + "web_search.search_effectiveness": 0.2, + "web_search.context_quality": "high", + "web_search.source_diversity": 1, + "web_search.information_synthesis": "low", + "web_search.external_knowledge_utilization": 0.975, + "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nLog in to WhatsApp Web for simple, reliable and private messaging on your desktop. Send and receive messages and files with ease, all for free. Sep 23, 2025 \u00b7 Chief Economists' Outlook 2025 states that these under-the-radar trends are shaping the global economy, from the impact of AI on research productivity to a tightened labor \u2026 Sep 1, 2025 \u00b7 THE AI DEVELOPMENT REVOLUTION: WHAT'S REALLY HAPPENING? Let's face it - the software development world is changing faster than a React component re-rendering on \u2026 Oct 7, 2025 \u00b7 Computer Programming Research. Read current computer science articles on everything from computer programs to detect cancer genes and control vehicle maintenance to \u2026 Aug 24, 2025 \u00b7 Artificial Intelligence News. Everything on AI including futuristic robots with artificial intelligence, computer models of human intelligence and more.\nURL: https://duckduckgo.com/?q=What's+happening+in+the+software+development+world+today?\n\nUser Question: What's happening in the software development world today?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", + "prompt.prompt_length": 1477, + "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nLog in to WhatsApp Web for simple, reliable and private messaging on your desktop. Send and receive messages and files with ease, all for free. Sep 23, 2025 \u00b7 Chief Economi...", + "prompt.instruction_following": "web_search_optimized", + "response.answer_length": 1636, + "response.answer_completeness": "complete", + "response.response_quality": "medium", + "response.source_citation": 1, + "response.context_utilization": 1.677948717948718, + "output_response": "Web Search Answer: The software development world is currently experiencing rapid changes driven by advancements in artificial intelligence (AI) and evolving programming practices. Key trends include:\n\n1. **AI Integrati...", + "evaluation.search_effectiveness": 0.2, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 1, + "evaluation.context_relevance": "high", + "evaluation.overall_quality": "low", + "evaluation.ready_for_production": true, + "web_search.search_strategy": "duckduckgo_api", + "web_search.real_time_data": true, + "web_search.external_sources": 1, + "web_search.information_freshness": "current", + "web_search.knowledge_synthesis": "single_source" + }, + "events": [], + "links": [], + "trace_trace_id": "2ba8cab0-2366-4add-8ac3-b490a05e9a60", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-14 12:53:56.480000000", + "trace_end_time": "2025-10-14 12:54:05.687000000", + "trace_duration_ms": 9207.67, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-14 12:53:56.480000000", + "updated_at": "2025-10-14 12:53:56.480000000" + }, + { + "span_id": "3d39f522-8ae7-4988-aafd-58b09734eff0", + "trace_id": "a7c3399e-372f-4aae-baeb-05215cc4b85f", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-14 12:53:50.963000000", + "end_time": "2025-10-14 12:53:55.471000000", + "duration_ms": 4507.466, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What are the newest features in cloud computing?", + "query.length": 48, + "processing.mode": "RAG", + "processing.time_seconds": 4.507421255111694, + "processing.response_length": 870, + "processing.sources_count": 5, + "output_response": "Final Answer: The provided context does not specifically mention any new features in cloud computing as a general topic. However, it does highlight several recent enhancements related to the Noveum.ai platform, whi...", + "final_answer_mode": "RAG", + "query_processed.input_query": "What are the newest features in cloud computing?", + "query_processed.output_response": "Successfully processed query using RAG, generated 870 character response", + "query_processed.mode": "RAG", + "query_processed.processing_time": 4.507421255111694, + "query_processed.response_length": 870 + }, + "events": [], + "links": [], + "trace_trace_id": "a7c3399e-372f-4aae-baeb-05215cc4b85f", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-14 12:53:50.963000000", + "trace_end_time": "2025-10-14 12:53:55.471000000", + "trace_duration_ms": 4507.555, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-14 12:53:50.963000000", + "updated_at": "2025-10-14 12:53:50.963000000" + }, + { + "span_id": "54aa6700-f2de-4cf0-b1ad-a3d24ed692ed", + "trace_id": "a7c3399e-372f-4aae-baeb-05215cc4b85f", + "parent_span_id": "d17f31cf-c570-4604-8df3-532b2e530fce", + "name": "agent.llm-rag", + "start_time": "2025-10-14 12:53:50.963000000", + "end_time": "2025-10-14 12:53:55.471000000", + "duration_ms": 4507.083, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "rag_agent", + "agent.operation": "llm-rag", + "agent.capabilities": [ + "document_retrieval", + "context_generation", + "response_generation" + ], + "agent.id": "noveum_rag_agent", + "input_query": "What are the newest features in cloud computing?", + "query_length": 48, + "query_type": "rag_query", + "output_response": "RAG Answer: The provided context does not specifically mention any new features in cloud computing as a general topic. However, it does highlight several recent enhancements related to the Noveum.ai platform, whi...", + "rag.context_length": 2579, + "rag.sources_count": 5, + "rag.answer_length": 870, + "rag.mode": "retrieval_augmented_generation" + }, + "events": [], + "links": [], + "trace_trace_id": "a7c3399e-372f-4aae-baeb-05215cc4b85f", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-14 12:53:50.963000000", + "trace_end_time": "2025-10-14 12:53:55.471000000", + "trace_duration_ms": 4507.555, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-14 12:53:50.963000000", + "updated_at": "2025-10-14 12:53:50.963000000" + }, + { + "span_id": "8878096f-6cfa-4f70-9844-d7723a0875f0", + "trace_id": "a7c3399e-372f-4aae-baeb-05215cc4b85f", + "parent_span_id": "d17f31cf-c570-4604-8df3-532b2e530fce", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-14 12:53:50.963000000", + "end_time": "2025-10-14 12:53:50.963000000", + "duration_ms": 0.036000000000000004, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What are the newest features in cloud computing?", + "query_length": 48, + "query_type": "routing_evaluation_query", + "classification.mode": "RAG", + "classification.rag_keyword_score": 1, + "classification.web_keyword_score": 0, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "documentation", + "query.keyword_density": 0.125, + "query.domain_affinity": "noveum", + "routing.decision": "Routed to RAG based on analysis", + "routing.rationale": "RAG score: 1, Web score: 0, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "Web Search", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.019230769230769232, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: RAG (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "a7c3399e-372f-4aae-baeb-05215cc4b85f", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-14 12:53:50.963000000", + "trace_end_time": "2025-10-14 12:53:55.471000000", + "trace_duration_ms": 4507.555, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-14 12:53:50.963000000", + "updated_at": "2025-10-14 12:53:50.963000000" + }, + { + "span_id": "d17f31cf-c570-4604-8df3-532b2e530fce", + "trace_id": "a7c3399e-372f-4aae-baeb-05215cc4b85f", + "parent_span_id": "3d39f522-8ae7-4988-aafd-58b09734eff0", + "name": "agent.query_routing", + "start_time": "2025-10-14 12:53:50.963000000", + "end_time": "2025-10-14 12:53:55.471000000", + "duration_ms": 4507.3730000000005, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What are the newest features in cloud computing?", + "query_length": 48, + "query_type": "routing_query", + "output_response": "Routed to RAG for query processing", + "router.classification": "RAG", + "router.confidence_score": 1, + "router.rag_keyword_score": 1, + "router.web_keyword_score": 0, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "a7c3399e-372f-4aae-baeb-05215cc4b85f", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-14 12:53:50.963000000", + "trace_end_time": "2025-10-14 12:53:55.471000000", + "trace_duration_ms": 4507.555, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-14 12:53:50.963000000", + "updated_at": "2025-10-14 12:53:50.963000000" + }, + { + "span_id": "97ef01a8-5930-4902-909a-245c516a5871", + "trace_id": "a7c3399e-372f-4aae-baeb-05215cc4b85f", + "parent_span_id": "54aa6700-f2de-4cf0-b1ad-a3d24ed692ed", + "name": "agent.llm_model_execution", + "start_time": "2025-10-14 12:53:51.439000000", + "end_time": "2025-10-14 12:53:55.077000000", + "duration_ms": 3637.337, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What are the newest features in cloud computing?", + "query_length": 48, + "query_type": "rag_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 3.6372909545898438, + "model.latency_ms": 3637.2909545898438, + "model.start_time": 1760446431.439748, + "model.end_time": 1760446435.077039, + "model.performance_tier": "medium", + "model.prompt_tokens": 780, + "model.completion_tokens": 217, + "model.total_tokens": 997, + "model.tokens_per_second": 274.1050997698989, + "model.estimated_cost": 0.02991, + "model.efficiency_score": 0.872617853560682, + "model.response_length": 870, + "model.response_quality": "high", + "model.output_response": "Model Response: The provided context does not specifically mention any new features in cloud computing as a general topic. However, it does highlight several recent enhancements related to the Noveum.ai platform, whi...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "a7c3399e-372f-4aae-baeb-05215cc4b85f", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-14 12:53:50.963000000", + "trace_end_time": "2025-10-14 12:53:55.471000000", + "trace_duration_ms": 4507.555, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-14 12:53:50.963000000", + "updated_at": "2025-10-14 12:53:50.963000000" + }, + { + "span_id": "ea792ec6-64b4-4ed5-b418-ba902e90c57a", + "trace_id": "a7c3399e-372f-4aae-baeb-05215cc4b85f", + "parent_span_id": "54aa6700-f2de-4cf0-b1ad-a3d24ed692ed", + "name": "agent.rag_evaluation_metrics", + "start_time": "2025-10-14 12:53:55.077000000", + "end_time": "2025-10-14 12:53:55.077000000", + "duration_ms": 0.034, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "rag_evaluation_metrics", + "agent.capabilities": [ + "retrieval_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What are the newest features in cloud computing?", + "query_length": 48, + "query_type": "rag_evaluation_query", + "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/changelog):\n- \ud83d\udcc8 Enhanced Dashboard Analytics: Real-time request tracking, latency monitoring, and cost analysis\n- \ud83d\udd0d Improved Logs Interface: Better search functionality, detailed trace views, and enhanced debugging capabilities\n- \ud83d\udc65 Team Management: Advanced member mana...", + "retrieval.context_length": 2579, + "retrieval.sources_count": 5, + "retrieval.context_quality": "high", + "retrieval.effectiveness": 1, + "retrieval.context_utilization": 2.579, + "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/changelog):\n- \ud83d\udcc8 Enhanced Dashboard Analytics: Real-time request tracking, latency monitoring, and cost analysis\n- \ud83d\udd0d Improved Logs Interface: Better search functionality, detailed trace views, and enhanced debugging capabilities\n- \ud83d\udc65 Team Management: Advanced member management with role-based access control and invitation system\n- \ud83d\udee1\ufe0f Security Improvements: Enhanced API key management and secure credential storage\n4 months ago\n- \ud83d\udcca Advanced Metrics Collection: Comprehensive telemetry system with custom metrics ...\n\nSource 2 (https://noveum.ai/en/changelog):\n- \ud83d\udcdd Documentation Updates: Comprehensive guides for integration and best practices\n5 months ago\n- \ud83c\udf89 Platform Foundation: Initial release of Noveum.ai AI observability platform\n- \ud83d\ude80 Core Features: AI Gateway integration, basic evaluation metrics, and monitoring capabilities\n- \ud83d\udcc8 Evaluation Jobs: Automated model evaluation with configurable metrics and reporting\n- \ud83d\udd0c Provider Integrations: Support for major AI providers and custom model deployments...\n\nSource 3 (https://noveum.ai/docs/platform/dashboard):\n- Bulk Export: Download multiple traces simultaneously\n- Comparative Analysis: Compare performance across multiple traces\nReal-Time Updates\n- Live Refresh: Automatic updates as new traces arrive\n- Connection Monitoring: Real-time status of your trace ingestion pipeline\n- Performance Indicators: Live metrics for system health monitoring\nReady to dive deeper? Explore Projects & Environments to organize your AI applications, or check out Team Collaboration to share insights with your team.\nGet Earl...\n\nSource 4 (https://noveum.ai/en/changelog):\nChangelog\nStay up to date with the latest changes in our product.\n2 months ago\n- \ud83c\udfaf Major Platform Evolution: NovaEval Framework - Launched comprehensive AI model evaluation framework with 20+ built-in scorers including accuracy, RAG metrics, conversational scoring, and specialized agent evaluation tools. Features production-grade deployment on Docker and Kubernetes. (GitHub | PyPI Package)\n- \ud83d\ude80 Next-Generation Traces UI - Complete redesign featuring three-pane layout with directory tree navigatio...\n\nSource 5 (https://noveum.ai/en/blog):\n8/12/2025\nDiscover how Noveum.ai provides comprehensive tracing and observability for AI applications, from development debugging to production optimization.\nShashank Agarwal\n3/3/2025\nDiscover how Noveum.ai provides comprehensive tracing and observability for LLM applications, RAG systems, and multi-agent workflows with our powerful Python and TypeScript SDKs.\nShashank Agarwal\n3/2/2025...\n\nUser Question: What are the newest features in cloud computing?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", + "prompt.prompt_length": 3123, + "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/changelog):\n- \ud83d\udcc8 Enhanced Dashboard Analytics: Real-time request tracking, latency monitoring, and cost analysis\n- \ud83d\udd0d Improved Logs Interface: Better search functionality,...", + "prompt.instruction_following": "rag_optimized", + "response.answer_length": 870, + "response.answer_completeness": "complete", + "response.response_quality": "high", + "response.source_citation": 5, + "response.context_utilization": 0.33734005428460645, + "output_response": "RAG Answer: The provided context does not specifically mention any new features in cloud computing as a general topic. However, it does highlight several recent enhancements related to the Noveum.ai platform, whi...", + "evaluation.retrieval_effectiveness": 1, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 5, + "evaluation.context_relevance": "high", + "evaluation.overall_quality": "high", + "evaluation.ready_for_production": true, + "rag.retrieval_strategy": "semantic_similarity", + "rag.vector_search_results": 5, + "rag.context_synthesis": "multi_source", + "rag.document_coverage": 1, + "rag.information_density": 0.33734005428460645 + }, + "events": [], + "links": [], + "trace_trace_id": "a7c3399e-372f-4aae-baeb-05215cc4b85f", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-14 12:53:50.963000000", + "trace_end_time": "2025-10-14 12:53:55.471000000", + "trace_duration_ms": 4507.555, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-14 12:53:50.963000000", + "updated_at": "2025-10-14 12:53:50.963000000" + }, + { + "span_id": "e2d314c8-687f-4b8d-869c-8d7f2a22a4f3", + "trace_id": "d682797f-2ce0-448e-8f0d-891541430988", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-14 12:53:44.917000000", + "end_time": "2025-10-14 12:53:49.957000000", + "duration_ms": 5039.565, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What's the current status of cryptocurrency markets?", + "query.length": 52, + "processing.mode": "Web Search", + "processing.time_seconds": 5.0394370555877686, + "processing.response_length": 338, + "processing.sources_count": 1, + "output_response": "Final Answer: The web search results did not provide any specific information regarding the current status of cryptocurrency markets. Therefore, I cannot provide an update on market conditions, prices, or trends at...", + "final_answer_mode": "Web Search", + "query_processed.input_query": "What's the current status of cryptocurrency markets?", + "query_processed.output_response": "Successfully processed query using Web Search, generated 338 character response", + "query_processed.mode": "Web Search", + "query_processed.processing_time": 5.0394370555877686, + "query_processed.response_length": 338 + }, + "events": [], + "links": [], + "trace_trace_id": "d682797f-2ce0-448e-8f0d-891541430988", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-14 12:53:44.917000000", + "trace_end_time": "2025-10-14 12:53:49.957000000", + "trace_duration_ms": 5039.64, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-14 12:53:44.917000000", + "updated_at": "2025-10-14 12:53:44.917000000" + }, + { + "span_id": "d367a6bd-86af-41c5-96ca-5284425775ce", + "trace_id": "d682797f-2ce0-448e-8f0d-891541430988", + "parent_span_id": "f137a412-6ddf-4ace-908e-9edf73534aa7", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-14 12:53:44.917000000", + "end_time": "2025-10-14 12:53:44.918000000", + "duration_ms": 0.046, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What's the current status of cryptocurrency markets?", + "query_length": 52, + "query_type": "routing_evaluation_query", + "classification.mode": "Web Search", + "classification.rag_keyword_score": 0, + "classification.web_keyword_score": 3, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "complex", + "query.intent": "general_knowledge", + "query.keyword_density": 0.42857142857142855, + "query.domain_affinity": "general", + "routing.decision": "Routed to Web Search based on analysis", + "routing.rationale": "RAG score: 0, Web score: 3, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "RAG", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.057692307692307696, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: Web Search (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "d682797f-2ce0-448e-8f0d-891541430988", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-14 12:53:44.917000000", + "trace_end_time": "2025-10-14 12:53:49.957000000", + "trace_duration_ms": 5039.64, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-14 12:53:44.917000000", + "updated_at": "2025-10-14 12:53:44.917000000" + }, + { + "span_id": "f137a412-6ddf-4ace-908e-9edf73534aa7", + "trace_id": "d682797f-2ce0-448e-8f0d-891541430988", + "parent_span_id": "e2d314c8-687f-4b8d-869c-8d7f2a22a4f3", + "name": "agent.query_routing", + "start_time": "2025-10-14 12:53:44.917000000", + "end_time": "2025-10-14 12:53:49.957000000", + "duration_ms": 5039.404, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What's the current status of cryptocurrency markets?", + "query_length": 52, + "query_type": "routing_query", + "output_response": "Routed to Web Search for query processing", + "router.classification": "Web Search", + "router.confidence_score": 1, + "router.rag_keyword_score": 0, + "router.web_keyword_score": 3, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "d682797f-2ce0-448e-8f0d-891541430988", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-14 12:53:44.917000000", + "trace_end_time": "2025-10-14 12:53:49.957000000", + "trace_duration_ms": 5039.64, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-14 12:53:44.917000000", + "updated_at": "2025-10-14 12:53:44.917000000" + }, + { + "span_id": "987b6910-9b6d-4207-b0f3-8807a23e3ddb", + "trace_id": "d682797f-2ce0-448e-8f0d-891541430988", + "parent_span_id": "f137a412-6ddf-4ace-908e-9edf73534aa7", + "name": "agent.web_search_generation", + "start_time": "2025-10-14 12:53:44.918000000", + "end_time": "2025-10-14 12:53:49.957000000", + "duration_ms": 5039.248, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "web_search_agent", + "agent.operation": "web_search_generation", + "agent.capabilities": [ + "web_search", + "content_synthesis", + "response_generation" + ], + "agent.id": "noveum_web_search_agent", + "input_query": "What's the current status of cryptocurrency markets?", + "query_length": 52, + "query_type": "web_search_query", + "output_response": "Web Search Answer: The web search results did not provide any specific information regarding the current status of cryptocurrency markets. Therefore, I cannot provide an update on market conditions, prices, or trends at...", + "web_search.results_count": 1, + "web_search.context_length": 154, + "web_search.response_length": 338, + "web_search.mode": "external_web_search" + }, + "events": [], + "links": [], + "trace_trace_id": "d682797f-2ce0-448e-8f0d-891541430988", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-14 12:53:44.917000000", + "trace_end_time": "2025-10-14 12:53:49.957000000", + "trace_duration_ms": 5039.64, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-14 12:53:44.917000000", + "updated_at": "2025-10-14 12:53:44.917000000" + }, + { + "span_id": "b2614d0e-5fcd-46ec-8d25-f1b04eb142af", + "trace_id": "d682797f-2ce0-448e-8f0d-891541430988", + "parent_span_id": "987b6910-9b6d-4207-b0f3-8807a23e3ddb", + "name": "agent.llm_model_execution", + "start_time": "2025-10-14 12:53:45.255000000", + "end_time": "2025-10-14 12:53:49.957000000", + "duration_ms": 4701.744, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What's the current status of cryptocurrency markets?", + "query_length": 52, + "query_type": "web_search_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 4.701629161834717, + "model.latency_ms": 4701.629161834717, + "model.start_time": 1760446425.2554948, + "model.end_time": 1760446429.957124, + "model.performance_tier": "medium", + "model.prompt_tokens": 162, + "model.completion_tokens": 84, + "model.total_tokens": 246, + "model.tokens_per_second": 52.32228904756993, + "model.estimated_cost": 0.00738, + "model.efficiency_score": 1.3739837398373984, + "model.response_length": 338, + "model.response_quality": "high", + "model.output_response": "Model Response: The web search results did not provide any specific information regarding the current status of cryptocurrency markets. Therefore, I cannot provide an update on market conditions, prices, or trends at...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "d682797f-2ce0-448e-8f0d-891541430988", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-14 12:53:44.917000000", + "trace_end_time": "2025-10-14 12:53:49.957000000", + "trace_duration_ms": 5039.64, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-14 12:53:44.917000000", + "updated_at": "2025-10-14 12:53:44.917000000" + }, + { + "span_id": "663e8789-75d0-4c4d-b14d-4088022436ac", + "trace_id": "d682797f-2ce0-448e-8f0d-891541430988", + "parent_span_id": "987b6910-9b6d-4207-b0f3-8807a23e3ddb", + "name": "agent.web_search_evaluation_metrics", + "start_time": "2025-10-14 12:53:49.957000000", + "end_time": "2025-10-14 12:53:49.957000000", + "duration_ms": 0.032, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "web_search_evaluation_metrics", + "agent.capabilities": [ + "web_search_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What's the current status of cryptocurrency markets?", + "query_length": 52, + "query_type": "web_search_evaluation_query", + "web_search.results_count": 1, + "web_search.context_length": 154, + "web_search.context_synthesized": "Context from 1 web sources", + "web_search.search_effectiveness": 0.2, + "web_search.context_quality": "low", + "web_search.source_diversity": 1, + "web_search.information_synthesis": "low", + "web_search.external_knowledge_utilization": 0.154, + "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What's+the+current+status+of+cryptocurrency+markets?\n\nUser Question: What's the current status of cryptocurrency markets?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", + "prompt.prompt_length": 651, + "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What's+the+current+status+of+cryptocurrency+markets?", + "prompt.instruction_following": "web_search_optimized", + "response.answer_length": 338, + "response.answer_completeness": "complete", + "response.response_quality": "medium", + "response.source_citation": 1, + "response.context_utilization": 2.1948051948051948, + "output_response": "Web Search Answer: The web search results did not provide any specific information regarding the current status of cryptocurrency markets. Therefore, I cannot provide an update on market conditions, prices, or trends at...", + "evaluation.search_effectiveness": 0.2, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 1, + "evaluation.context_relevance": "low", + "evaluation.overall_quality": "low", + "evaluation.ready_for_production": true, + "web_search.search_strategy": "duckduckgo_api", + "web_search.real_time_data": true, + "web_search.external_sources": 1, + "web_search.information_freshness": "current", + "web_search.knowledge_synthesis": "single_source" + }, + "events": [], + "links": [], + "trace_trace_id": "d682797f-2ce0-448e-8f0d-891541430988", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-14 12:53:44.917000000", + "trace_end_time": "2025-10-14 12:53:49.957000000", + "trace_duration_ms": 5039.64, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-14 12:53:44.917000000", + "updated_at": "2025-10-14 12:53:44.917000000" + }, + { + "span_id": "3382fe7c-aaca-46e0-a8d2-dbee75ff3bed", + "trace_id": "b0858dca-4173-493b-a6c5-4e993f606477", + "parent_span_id": "6cf94049-87b0-4fc5-b37e-32d261543647", + "name": "agent.query_routing", + "start_time": "2025-10-14 12:53:37.818000000", + "end_time": "2025-10-14 12:53:43.908000000", + "duration_ms": 6089.22, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What are the latest updates in Python programming?", + "query_length": 50, + "query_type": "routing_query", + "output_response": "Routed to Web Search for query processing", + "router.classification": "Web Search", + "router.confidence_score": 1, + "router.rag_keyword_score": 0, + "router.web_keyword_score": 2, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "b0858dca-4173-493b-a6c5-4e993f606477", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-14 12:53:37.818000000", + "trace_end_time": "2025-10-14 12:53:43.908000000", + "trace_duration_ms": 6089.715, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-14 12:53:37.818000000", + "updated_at": "2025-10-14 12:53:37.818000000" + }, + { + "span_id": "6cf94049-87b0-4fc5-b37e-32d261543647", + "trace_id": "b0858dca-4173-493b-a6c5-4e993f606477", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-14 12:53:37.818000000", + "end_time": "2025-10-14 12:53:43.908000000", + "duration_ms": 6089.607, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What are the latest updates in Python programming?", + "query.length": 50, + "processing.mode": "Web Search", + "processing.time_seconds": 6.089300870895386, + "processing.response_length": 1324, + "processing.sources_count": 1, + "output_response": "Final Answer: The web search did not yield specific results regarding the latest updates in Python programming. However, I can provide a general overview based on my knowledge up to October 2023.\n\nAs of late 2023, ...", + "final_answer_mode": "Web Search", + "query_processed.input_query": "What are the latest updates in Python programming?", + "query_processed.output_response": "Successfully processed query using Web Search, generated 1324 character response", + "query_processed.mode": "Web Search", + "query_processed.processing_time": 6.089300870895386, + "query_processed.response_length": 1324 + }, + "events": [], + "links": [], + "trace_trace_id": "b0858dca-4173-493b-a6c5-4e993f606477", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-14 12:53:37.818000000", + "trace_end_time": "2025-10-14 12:53:43.908000000", + "trace_duration_ms": 6089.715, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-14 12:53:37.818000000", + "updated_at": "2025-10-14 12:53:37.818000000" + }, + { + "span_id": "21c8befa-356a-4034-8d62-792238cb9e6e", + "trace_id": "b0858dca-4173-493b-a6c5-4e993f606477", + "parent_span_id": "3382fe7c-aaca-46e0-a8d2-dbee75ff3bed", + "name": "agent.web_search_generation", + "start_time": "2025-10-14 12:53:37.819000000", + "end_time": "2025-10-14 12:53:43.908000000", + "duration_ms": 6088.807, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "web_search_agent", + "agent.operation": "web_search_generation", + "agent.capabilities": [ + "web_search", + "content_synthesis", + "response_generation" + ], + "agent.id": "noveum_web_search_agent", + "input_query": "What are the latest updates in Python programming?", + "query_length": 50, + "query_type": "web_search_query", + "output_response": "Web Search Answer: The web search did not yield specific results regarding the latest updates in Python programming. However, I can provide a general overview based on my knowledge up to October 2023.\n\nAs of late 2023, ...", + "web_search.results_count": 1, + "web_search.context_length": 152, + "web_search.response_length": 1324, + "web_search.mode": "external_web_search" + }, + "events": [], + "links": [], + "trace_trace_id": "b0858dca-4173-493b-a6c5-4e993f606477", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-14 12:53:37.818000000", + "trace_end_time": "2025-10-14 12:53:43.908000000", + "trace_duration_ms": 6089.715, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-14 12:53:37.818000000", + "updated_at": "2025-10-14 12:53:37.818000000" + }, + { + "span_id": "482feedf-52f4-4c6a-9186-d59898820caa", + "trace_id": "b0858dca-4173-493b-a6c5-4e993f606477", + "parent_span_id": "3382fe7c-aaca-46e0-a8d2-dbee75ff3bed", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-14 12:53:37.819000000", + "end_time": "2025-10-14 12:53:37.819000000", + "duration_ms": 0.039, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What are the latest updates in Python programming?", + "query_length": 50, + "query_type": "routing_evaluation_query", + "classification.mode": "Web Search", + "classification.rag_keyword_score": 0, + "classification.web_keyword_score": 2, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "general_knowledge", + "query.keyword_density": 0.25, + "query.domain_affinity": "general", + "routing.decision": "Routed to Web Search based on analysis", + "routing.rationale": "RAG score: 0, Web score: 2, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "RAG", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.038461538461538464, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: Web Search (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "b0858dca-4173-493b-a6c5-4e993f606477", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-14 12:53:37.818000000", + "trace_end_time": "2025-10-14 12:53:43.908000000", + "trace_duration_ms": 6089.715, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-14 12:53:37.818000000", + "updated_at": "2025-10-14 12:53:37.818000000" + }, + { + "span_id": "84f1f63f-ee05-411e-a7dd-4ee9e9826f5a", + "trace_id": "b0858dca-4173-493b-a6c5-4e993f606477", + "parent_span_id": "21c8befa-356a-4034-8d62-792238cb9e6e", + "name": "agent.llm_model_execution", + "start_time": "2025-10-14 12:53:38.191000000", + "end_time": "2025-10-14 12:53:43.907000000", + "duration_ms": 5716.078, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What are the latest updates in Python programming?", + "query_length": 50, + "query_type": "web_search_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 5.715963125228882, + "model.latency_ms": 5715.963125228882, + "model.start_time": 1760446418.191805, + "model.end_time": 1760446423.907768, + "model.performance_tier": "slow", + "model.prompt_tokens": 161, + "model.completion_tokens": 331, + "model.total_tokens": 492, + "model.tokens_per_second": 86.07473302765561, + "model.estimated_cost": 0.01476, + "model.efficiency_score": 2.6910569105691056, + "model.response_length": 1324, + "model.response_quality": "high", + "model.output_response": "Model Response: The web search did not yield specific results regarding the latest updates in Python programming. However, I can provide a general overview based on my knowledge up to October 2023.\n\nAs of late 2023, ...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "b0858dca-4173-493b-a6c5-4e993f606477", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-14 12:53:37.818000000", + "trace_end_time": "2025-10-14 12:53:43.908000000", + "trace_duration_ms": 6089.715, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-14 12:53:37.818000000", + "updated_at": "2025-10-14 12:53:37.818000000" + }, + { + "span_id": "2fb9ecf3-a497-4285-a694-1c95080d9e6a", + "trace_id": "b0858dca-4173-493b-a6c5-4e993f606477", + "parent_span_id": "21c8befa-356a-4034-8d62-792238cb9e6e", + "name": "agent.web_search_evaluation_metrics", + "start_time": "2025-10-14 12:53:43.908000000", + "end_time": "2025-10-14 12:53:43.908000000", + "duration_ms": 0.037, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "web_search_evaluation_metrics", + "agent.capabilities": [ + "web_search_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What are the latest updates in Python programming?", + "query_length": 50, + "query_type": "web_search_evaluation_query", + "web_search.results_count": 1, + "web_search.context_length": 152, + "web_search.context_synthesized": "Context from 1 web sources", + "web_search.search_effectiveness": 0.2, + "web_search.context_quality": "low", + "web_search.source_diversity": 1, + "web_search.information_synthesis": "low", + "web_search.external_knowledge_utilization": 0.152, + "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What+are+the+latest+updates+in+Python+programming?\n\nUser Question: What are the latest updates in Python programming?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", + "prompt.prompt_length": 647, + "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What+are+the+latest+updates+in+Python+programming?", + "prompt.instruction_following": "web_search_optimized", + "response.answer_length": 1324, + "response.answer_completeness": "complete", + "response.response_quality": "medium", + "response.source_citation": 1, + "response.context_utilization": 8.710526315789474, + "output_response": "Web Search Answer: The web search did not yield specific results regarding the latest updates in Python programming. However, I can provide a general overview based on my knowledge up to October 2023.\n\nAs of late 2023, ...", + "evaluation.search_effectiveness": 0.2, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 1, + "evaluation.context_relevance": "low", + "evaluation.overall_quality": "low", + "evaluation.ready_for_production": true, + "web_search.search_strategy": "duckduckgo_api", + "web_search.real_time_data": true, + "web_search.external_sources": 1, + "web_search.information_freshness": "current", + "web_search.knowledge_synthesis": "single_source" + }, + "events": [], + "links": [], + "trace_trace_id": "b0858dca-4173-493b-a6c5-4e993f606477", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-14 12:53:37.818000000", + "trace_end_time": "2025-10-14 12:53:43.908000000", + "trace_duration_ms": 6089.715, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-14 12:53:37.818000000", + "updated_at": "2025-10-14 12:53:37.818000000" + }, + { + "span_id": "9dd5ac2c-0deb-48e3-ac8b-0e6e31f7ff30", + "trace_id": "dc81be1a-e93d-4f60-90f3-ce1acaa8cb3e", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-14 12:53:34.584000000", + "end_time": "2025-10-14 12:53:36.780000000", + "duration_ms": 2196.351, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What happened in tech news this week?", + "query.length": 37, + "processing.mode": "Web Search", + "processing.time_seconds": 2.1948978900909424, + "processing.response_length": 324, + "processing.sources_count": 1, + "output_response": "Final Answer: I'm sorry, but I couldn't find any specific information regarding tech news from this week based on the search results. It appears that there were no relevant articles or updates available. If you're ...", + "final_answer_mode": "Web Search", + "query_processed.input_query": "What happened in tech news this week?", + "query_processed.output_response": "Successfully processed query using Web Search, generated 324 character response", + "query_processed.mode": "Web Search", + "query_processed.processing_time": 2.1948978900909424, + "query_processed.response_length": 324 + }, + "events": [], + "links": [], + "trace_trace_id": "dc81be1a-e93d-4f60-90f3-ce1acaa8cb3e", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-14 12:53:34.584000000", + "trace_end_time": "2025-10-14 12:53:36.780000000", + "trace_duration_ms": 2196.4579999999996, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-14 12:53:34.584000000", + "updated_at": "2025-10-14 12:53:34.584000000" + }, + { + "span_id": "b1cbafce-51fb-48fd-bc4a-d49081a2f374", + "trace_id": "dc81be1a-e93d-4f60-90f3-ce1acaa8cb3e", + "parent_span_id": "9dd5ac2c-0deb-48e3-ac8b-0e6e31f7ff30", + "name": "agent.query_routing", + "start_time": "2025-10-14 12:53:34.584000000", + "end_time": "2025-10-14 12:53:36.778000000", + "duration_ms": 2194.2200000000003, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What happened in tech news this week?", + "query_length": 37, + "query_type": "routing_query", + "output_response": "Routed to Web Search for query processing", + "router.classification": "Web Search", + "router.confidence_score": 1, + "router.rag_keyword_score": 0, + "router.web_keyword_score": 2, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "dc81be1a-e93d-4f60-90f3-ce1acaa8cb3e", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-14 12:53:34.584000000", + "trace_end_time": "2025-10-14 12:53:36.780000000", + "trace_duration_ms": 2196.4579999999996, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-14 12:53:34.584000000", + "updated_at": "2025-10-14 12:53:34.584000000" + }, + { + "span_id": "e3064687-1b18-478b-9e9d-4d081a6dac23", + "trace_id": "dc81be1a-e93d-4f60-90f3-ce1acaa8cb3e", + "parent_span_id": "b1cbafce-51fb-48fd-bc4a-d49081a2f374", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-14 12:53:34.584000000", + "end_time": "2025-10-14 12:53:34.584000000", + "duration_ms": 0.028, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What happened in tech news this week?", + "query_length": 37, + "query_type": "routing_evaluation_query", + "classification.mode": "Web Search", + "classification.rag_keyword_score": 0, + "classification.web_keyword_score": 2, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "general_knowledge", + "query.keyword_density": 0.2857142857142857, + "query.domain_affinity": "general", + "routing.decision": "Routed to Web Search based on analysis", + "routing.rationale": "RAG score: 0, Web score: 2, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "RAG", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.038461538461538464, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: Web Search (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "dc81be1a-e93d-4f60-90f3-ce1acaa8cb3e", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-14 12:53:34.584000000", + "trace_end_time": "2025-10-14 12:53:36.780000000", + "trace_duration_ms": 2196.4579999999996, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-14 12:53:34.584000000", + "updated_at": "2025-10-14 12:53:34.584000000" + }, + { + "span_id": "eb9dbc11-44b3-4041-8c83-0f33b8b761bb", + "trace_id": "dc81be1a-e93d-4f60-90f3-ce1acaa8cb3e", + "parent_span_id": "b1cbafce-51fb-48fd-bc4a-d49081a2f374", + "name": "agent.web_search_generation", + "start_time": "2025-10-14 12:53:34.584000000", + "end_time": "2025-10-14 12:53:36.774000000", + "duration_ms": 2189.866, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "web_search_agent", + "agent.operation": "web_search_generation", + "agent.capabilities": [ + "web_search", + "content_synthesis", + "response_generation" + ], + "agent.id": "noveum_web_search_agent", + "input_query": "What happened in tech news this week?", + "query_length": 37, + "query_type": "web_search_query", + "output_response": "Web Search Answer: I'm sorry, but I couldn't find any specific information regarding tech news from this week based on the search results. It appears that there were no relevant articles or updates available. If you're ...", + "web_search.results_count": 1, + "web_search.context_length": 139, + "web_search.response_length": 324, + "web_search.mode": "external_web_search" + }, + "events": [], + "links": [], + "trace_trace_id": "dc81be1a-e93d-4f60-90f3-ce1acaa8cb3e", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-14 12:53:34.584000000", + "trace_end_time": "2025-10-14 12:53:36.780000000", + "trace_duration_ms": 2196.4579999999996, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-14 12:53:34.584000000", + "updated_at": "2025-10-14 12:53:34.584000000" + }, + { + "span_id": "c8ee237b-ab5e-4b36-83f0-2f8478971898", + "trace_id": "dc81be1a-e93d-4f60-90f3-ce1acaa8cb3e", + "parent_span_id": "eb9dbc11-44b3-4041-8c83-0f33b8b761bb", + "name": "agent.llm_model_execution", + "start_time": "2025-10-14 12:53:34.981000000", + "end_time": "2025-10-14 12:53:36.768000000", + "duration_ms": 1786.467, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What happened in tech news this week?", + "query_length": 37, + "query_type": "web_search_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 1.785179853439331, + "model.latency_ms": 1785.179853439331, + "model.start_time": 1760446414.981964, + "model.end_time": 1760446416.767144, + "model.performance_tier": "fast", + "model.prompt_tokens": 155, + "model.completion_tokens": 81, + "model.total_tokens": 236, + "model.tokens_per_second": 132.19956495997977, + "model.estimated_cost": 0.00708, + "model.efficiency_score": 1.3728813559322033, + "model.response_length": 324, + "model.response_quality": "high", + "model.output_response": "Model Response: I'm sorry, but I couldn't find any specific information regarding tech news from this week based on the search results. It appears that there were no relevant articles or updates available. If you're ...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "dc81be1a-e93d-4f60-90f3-ce1acaa8cb3e", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-14 12:53:34.584000000", + "trace_end_time": "2025-10-14 12:53:36.780000000", + "trace_duration_ms": 2196.4579999999996, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-14 12:53:34.584000000", + "updated_at": "2025-10-14 12:53:34.584000000" + }, + { + "span_id": "9c8e1a08-61c8-409e-a79d-41c74a8d1505", + "trace_id": "dc81be1a-e93d-4f60-90f3-ce1acaa8cb3e", + "parent_span_id": "eb9dbc11-44b3-4041-8c83-0f33b8b761bb", + "name": "agent.web_search_evaluation_metrics", + "start_time": "2025-10-14 12:53:36.772000000", + "end_time": "2025-10-14 12:53:36.774000000", + "duration_ms": 1.5939999999999999, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "web_search_evaluation_metrics", + "agent.capabilities": [ + "web_search_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What happened in tech news this week?", + "query_length": 37, + "query_type": "web_search_evaluation_query", + "web_search.results_count": 1, + "web_search.context_length": 139, + "web_search.context_synthesized": "Context from 1 web sources", + "web_search.search_effectiveness": 0.2, + "web_search.context_quality": "low", + "web_search.source_diversity": 1, + "web_search.information_synthesis": "low", + "web_search.external_knowledge_utilization": 0.139, + "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What+happened+in+tech+news+this+week?\n\nUser Question: What happened in tech news this week?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", + "prompt.prompt_length": 621, + "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What+happened+in+tech+news+this+week?", + "prompt.instruction_following": "web_search_optimized", + "response.answer_length": 324, + "response.answer_completeness": "complete", + "response.response_quality": "medium", + "response.source_citation": 1, + "response.context_utilization": 2.3309352517985613, + "output_response": "Web Search Answer: I'm sorry, but I couldn't find any specific information regarding tech news from this week based on the search results. It appears that there were no relevant articles or updates available. If you're ...", + "evaluation.search_effectiveness": 0.2, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 1, + "evaluation.context_relevance": "low", + "evaluation.overall_quality": "low", + "evaluation.ready_for_production": true, + "web_search.search_strategy": "duckduckgo_api", + "web_search.real_time_data": true, + "web_search.external_sources": 1, + "web_search.information_freshness": "current", + "web_search.knowledge_synthesis": "single_source" + }, + "events": [], + "links": [], + "trace_trace_id": "dc81be1a-e93d-4f60-90f3-ce1acaa8cb3e", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-14 12:53:34.584000000", + "trace_end_time": "2025-10-14 12:53:36.780000000", + "trace_duration_ms": 2196.4579999999996, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-14 12:53:34.584000000", + "updated_at": "2025-10-14 12:53:34.584000000" + }, + { + "span_id": "59595232-8873-419c-9b6a-0727445a2afa", + "trace_id": "945b2757-8a5a-4aab-a869-8f44c02166a5", + "parent_span_id": "dfde2f79-d0c1-481e-819d-60730cb60fe5", + "name": "agent.query_routing", + "start_time": "2025-10-14 12:53:27.771000000", + "end_time": "2025-10-14 12:53:33.575000000", + "duration_ms": 5803.9800000000005, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What are the current trends in observability tools?", + "query_length": 51, + "query_type": "routing_query", + "output_response": "Routed to RAG for query processing", + "router.classification": "RAG", + "router.confidence_score": 0.3333333333333333, + "router.rag_keyword_score": 2, + "router.web_keyword_score": 1, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "945b2757-8a5a-4aab-a869-8f44c02166a5", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-14 12:53:27.771000000", + "trace_end_time": "2025-10-14 12:53:33.575000000", + "trace_duration_ms": 5804.15, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-14 12:53:27.771000000", + "updated_at": "2025-10-14 12:53:27.771000000" + }, + { + "span_id": "a59b3296-aaba-4642-b5a0-a271787a5607", + "trace_id": "945b2757-8a5a-4aab-a869-8f44c02166a5", + "parent_span_id": "59595232-8873-419c-9b6a-0727445a2afa", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-14 12:53:27.771000000", + "end_time": "2025-10-14 12:53:27.771000000", + "duration_ms": 0.031, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What are the current trends in observability tools?", + "query_length": 51, + "query_type": "routing_evaluation_query", + "classification.mode": "RAG", + "classification.rag_keyword_score": 2, + "classification.web_keyword_score": 1, + "classification.confidence_score": 0.3333333333333333, + "classification.confidence_level": "medium", + "classification.method": "keyword_based", + "query.complexity": "complex", + "query.intent": "documentation", + "query.keyword_density": 0.375, + "query.domain_affinity": "noveum", + "routing.decision": "Routed to RAG based on analysis", + "routing.rationale": "RAG score: 2, Web score: 1, Confidence: 0.33", + "routing.expected_performance": "medium", + "routing.alternative_mode": "Web Search", + "routing.decision_confidence": 0.3333333333333333, + "evaluation.routing_accuracy": "medium", + "evaluation.keyword_coverage": 0.057692307692307696, + "evaluation.query_understanding": "ambiguous", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: RAG (Confidence: 0.33)" + }, + "events": [], + "links": [], + "trace_trace_id": "945b2757-8a5a-4aab-a869-8f44c02166a5", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-14 12:53:27.771000000", + "trace_end_time": "2025-10-14 12:53:33.575000000", + "trace_duration_ms": 5804.15, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-14 12:53:27.771000000", + "updated_at": "2025-10-14 12:53:27.771000000" + }, + { + "span_id": "dfde2f79-d0c1-481e-819d-60730cb60fe5", + "trace_id": "945b2757-8a5a-4aab-a869-8f44c02166a5", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-14 12:53:27.771000000", + "end_time": "2025-10-14 12:53:33.575000000", + "duration_ms": 5804.074, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What are the current trends in observability tools?", + "query.length": 51, + "processing.mode": "RAG", + "processing.time_seconds": 5.804032802581787, + "processing.response_length": 997, + "processing.sources_count": 5, + "output_response": "Final Answer: The provided context does not specifically outline current trends in observability tools in general. However, it highlights the unique challenges faced by AI applications and how traditional monitorin...", + "final_answer_mode": "RAG", + "query_processed.input_query": "What are the current trends in observability tools?", + "query_processed.output_response": "Successfully processed query using RAG, generated 997 character response", + "query_processed.mode": "RAG", + "query_processed.processing_time": 5.804032802581787, + "query_processed.response_length": 997 + }, + "events": [], + "links": [], + "trace_trace_id": "945b2757-8a5a-4aab-a869-8f44c02166a5", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-14 12:53:27.771000000", + "trace_end_time": "2025-10-14 12:53:33.575000000", + "trace_duration_ms": 5804.15, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-14 12:53:27.771000000", + "updated_at": "2025-10-14 12:53:27.771000000" + }, + { + "span_id": "82489444-deba-46d2-b08b-c09754189566", + "trace_id": "945b2757-8a5a-4aab-a869-8f44c02166a5", + "parent_span_id": "59595232-8873-419c-9b6a-0727445a2afa", + "name": "agent.llm-rag", + "start_time": "2025-10-14 12:53:27.772000000", + "end_time": "2025-10-14 12:53:33.575000000", + "duration_ms": 5803.534, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "rag_agent", + "agent.operation": "llm-rag", + "agent.capabilities": [ + "document_retrieval", + "context_generation", + "response_generation" + ], + "agent.id": "noveum_rag_agent", + "input_query": "What are the current trends in observability tools?", + "query_length": 51, + "query_type": "rag_query", + "output_response": "RAG Answer: The provided context does not specifically outline current trends in observability tools in general. However, it highlights the unique challenges faced by AI applications and how traditional monitorin...", + "rag.context_length": 2943, + "rag.sources_count": 5, + "rag.answer_length": 997, + "rag.mode": "retrieval_augmented_generation" + }, + "events": [], + "links": [], + "trace_trace_id": "945b2757-8a5a-4aab-a869-8f44c02166a5", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-14 12:53:27.771000000", + "trace_end_time": "2025-10-14 12:53:33.575000000", + "trace_duration_ms": 5804.15, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-14 12:53:27.771000000", + "updated_at": "2025-10-14 12:53:27.771000000" + }, + { + "span_id": "34b1f14c-3fd3-4f89-b4dd-6150a52befd7", + "trace_id": "945b2757-8a5a-4aab-a869-8f44c02166a5", + "parent_span_id": "82489444-deba-46d2-b08b-c09754189566", + "name": "agent.llm_model_execution", + "start_time": "2025-10-14 12:53:28.257000000", + "end_time": "2025-10-14 12:53:33.162000000", + "duration_ms": 4905.2429999999995, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What are the current trends in observability tools?", + "query_length": 51, + "query_type": "rag_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 4.9052040576934814, + "model.latency_ms": 4905.204057693481, + "model.start_time": 1760446408.2575018, + "model.end_time": 1760446413.162706, + "model.performance_tier": "medium", + "model.prompt_tokens": 872, + "model.completion_tokens": 249, + "model.total_tokens": 1121, + "model.tokens_per_second": 228.53279635569638, + "model.estimated_cost": 0.03363, + "model.efficiency_score": 0.8893844781445138, + "model.response_length": 997, + "model.response_quality": "high", + "model.output_response": "Model Response: The provided context does not specifically outline current trends in observability tools in general. However, it highlights the unique challenges faced by AI applications and how traditional monitorin...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "945b2757-8a5a-4aab-a869-8f44c02166a5", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-14 12:53:27.771000000", + "trace_end_time": "2025-10-14 12:53:33.575000000", + "trace_duration_ms": 5804.15, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-14 12:53:27.771000000", + "updated_at": "2025-10-14 12:53:27.771000000" + }, + { + "span_id": "d5d2b966-3c37-4d6d-b7b8-c58c9f22aad9", + "trace_id": "945b2757-8a5a-4aab-a869-8f44c02166a5", + "parent_span_id": "82489444-deba-46d2-b08b-c09754189566", + "name": "agent.rag_evaluation_metrics", + "start_time": "2025-10-14 12:53:33.162000000", + "end_time": "2025-10-14 12:53:33.162000000", + "duration_ms": 0.036000000000000004, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "rag_evaluation_metrics", + "agent.capabilities": [ + "retrieval_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What are the current trends in observability tools?", + "query_length": 51, + "query_type": "rag_evaluation_query", + "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/blog/from-logs-to-intelligent-choices-inside-noveum-ais-evaluation-process):\nFrom Development to Production - Inside Noveum.ai's AI Observability Platform\nShashank Agarwal\n3/3/2025\nIntroduction\nEvery AI application tells a story\u2014but most developers never get to hear it...", + "retrieval.context_length": 2943, + "retrieval.sources_count": 5, + "retrieval.context_quality": "high", + "retrieval.effectiveness": 1, + "retrieval.context_utilization": 2.943, + "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/blog/from-logs-to-intelligent-choices-inside-noveum-ais-evaluation-process):\nFrom Development to Production - Inside Noveum.ai's AI Observability Platform\nShashank Agarwal\n3/3/2025\nIntroduction\nEvery AI application tells a story\u2014but most developers never get to hear it. When your RAG pipeline returns irrelevant results, when your multi-agent system gets stuck in loops, or when your LLM costs suddenly spike, you're left guessing what went wrong and where.\nThat's exactly the challenge Noveum.ai solves. Rather than flying blind, our platform provides comprehensive tracing a...\n\nSource 2 (https://noveum.ai/docs/getting-started/tracing-concepts):\n- Explore Framework Integrations - Framework-specific guidance\n- Learn Advanced Patterns - Custom instrumentation techniques\n- Master the Dashboard - Analyze your traces effectively\nRemember: Good observability is not about collecting all possible data, but about collecting the right data that helps you understand, debug, and optimize your AI applications.\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to O...\n\nSource 3 (https://noveum.ai/en/docs/getting-started/tracing-concepts):\n- Explore Framework Integrations - Framework-specific guidance\n- Learn Advanced Patterns - Custom instrumentation techniques\n- Master the Dashboard - Analyze your traces effectively\nRemember: Good observability is not about collecting all possible data, but about collecting the right data that helps you understand, debug, and optimize your AI applications.\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to O...\n\nSource 4 (https://noveum.ai/en/blog/from-logs-to-intelligent-choices-inside-noveum-ais-evaluation-process):\nThe AI Observability Challenge\nWhy Traditional Monitoring Falls Short\nTraditional application monitoring tools weren't built for AI applications. They can tell you if your API is responding, but they can't answer the questions that matter most for AI systems:\n- Why did my RAG pipeline retrieve irrelevant documents?\n- Which LLM calls are driving my costs?\n- How are my agents communicating with each other?\n- What's causing hallucinations in my responses?\n- Why is my embedding generation so slow?\nW...\n\nSource 5 (https://noveum.ai/en/docs/advanced/rag-observability):\n\ud83d\udd27 Best Practices for RAG Observability\n1. Comprehensive Pipeline Tracing\n2. Quality Monitoring\n3. A/B Testing for RAG Components\n\ud83c\udfaf Advanced RAG Patterns\nMulti-Modal RAG\nConversational RAG\nRAG observability with Noveum.ai provides the deep insights needed to build, optimize, and scale retrieval-augmented generation systems. By implementing comprehensive tracing across retrieval, context preparation, and generation stages, you can ensure your RAG pipeline delivers accurate, relevant, and cost-effe...\n\nUser Question: What are the current trends in observability tools?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", + "prompt.prompt_length": 3490, + "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/blog/from-logs-to-intelligent-choices-inside-noveum-ais-evaluation-process):\nFrom Development to Production - Inside Noveum.ai's AI Observability Platform\nShashank Agarw...", + "prompt.instruction_following": "rag_optimized", + "response.answer_length": 997, + "response.answer_completeness": "complete", + "response.response_quality": "high", + "response.source_citation": 5, + "response.context_utilization": 0.33876996262317366, + "output_response": "RAG Answer: The provided context does not specifically outline current trends in observability tools in general. However, it highlights the unique challenges faced by AI applications and how traditional monitorin...", + "evaluation.retrieval_effectiveness": 1, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 5, + "evaluation.context_relevance": "high", + "evaluation.overall_quality": "high", + "evaluation.ready_for_production": true, + "rag.retrieval_strategy": "semantic_similarity", + "rag.vector_search_results": 5, + "rag.context_synthesis": "multi_source", + "rag.document_coverage": 1, + "rag.information_density": 0.33876996262317366 + }, + "events": [], + "links": [], + "trace_trace_id": "945b2757-8a5a-4aab-a869-8f44c02166a5", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-14 12:53:27.771000000", + "trace_end_time": "2025-10-14 12:53:33.575000000", + "trace_duration_ms": 5804.15, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-14 12:53:27.771000000", + "updated_at": "2025-10-14 12:53:27.771000000" + }, + { + "span_id": "11c07411-df30-42a4-9062-90a032ef75ed", + "trace_id": "4bdd5814-acbb-4de7-a56b-543b87a1cfa5", + "parent_span_id": "ec530838-a7cf-477b-9186-a70114c8a12f", + "name": "agent.web_search_generation", + "start_time": "2025-10-14 12:53:20.120000000", + "end_time": "2025-10-14 12:53:26.750000000", + "duration_ms": 6629.273999999999, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "web_search_agent", + "agent.operation": "web_search_generation", + "agent.capabilities": [ + "web_search", + "content_synthesis", + "response_generation" + ], + "agent.id": "noveum_web_search_agent", + "input_query": "Tell me about recent developments in machine learning", + "query_length": 53, + "query_type": "web_search_query", + "output_response": "Web Search Answer: The web search did not yield specific results regarding recent developments in machine learning. However, I can provide a general overview based on my knowledge up to October 2023.\n\nRecent development...", + "web_search.results_count": 1, + "web_search.context_length": 155, + "web_search.response_length": 1849, + "web_search.mode": "external_web_search" + }, + "events": [], + "links": [], + "trace_trace_id": "4bdd5814-acbb-4de7-a56b-543b87a1cfa5", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-14 12:53:20.120000000", + "trace_end_time": "2025-10-14 12:53:26.750000000", + "trace_duration_ms": 6630.38, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-14 12:53:20.120000000", + "updated_at": "2025-10-14 12:53:20.120000000" + }, + { + "span_id": "282b121f-61dd-4dc5-9e55-cecc3f02f688", + "trace_id": "4bdd5814-acbb-4de7-a56b-543b87a1cfa5", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-14 12:53:20.120000000", + "end_time": "2025-10-14 12:53:26.750000000", + "duration_ms": 6630.264, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "Tell me about recent developments in machine learning", + "query.length": 53, + "processing.mode": "Web Search", + "processing.time_seconds": 6.629686117172241, + "processing.response_length": 1849, + "processing.sources_count": 1, + "output_response": "Final Answer: The web search did not yield specific results regarding recent developments in machine learning. However, I can provide a general overview based on my knowledge up to October 2023.\n\nRecent development...", + "final_answer_mode": "Web Search", + "query_processed.input_query": "Tell me about recent developments in machine learning", + "query_processed.output_response": "Successfully processed query using Web Search, generated 1849 character response", + "query_processed.mode": "Web Search", + "query_processed.processing_time": 6.629686117172241, + "query_processed.response_length": 1849 + }, + "events": [], + "links": [], + "trace_trace_id": "4bdd5814-acbb-4de7-a56b-543b87a1cfa5", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-14 12:53:20.120000000", + "trace_end_time": "2025-10-14 12:53:26.750000000", + "trace_duration_ms": 6630.38, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-14 12:53:20.120000000", + "updated_at": "2025-10-14 12:53:20.120000000" + }, + { + "span_id": "ec530838-a7cf-477b-9186-a70114c8a12f", + "trace_id": "4bdd5814-acbb-4de7-a56b-543b87a1cfa5", + "parent_span_id": "282b121f-61dd-4dc5-9e55-cecc3f02f688", + "name": "agent.query_routing", + "start_time": "2025-10-14 12:53:20.120000000", + "end_time": "2025-10-14 12:53:26.750000000", + "duration_ms": 6629.59, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "Tell me about recent developments in machine learning", + "query_length": 53, + "query_type": "routing_query", + "output_response": "Routed to Web Search for query processing", + "router.classification": "Web Search", + "router.confidence_score": 1, + "router.rag_keyword_score": 0, + "router.web_keyword_score": 1, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "4bdd5814-acbb-4de7-a56b-543b87a1cfa5", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-14 12:53:20.120000000", + "trace_end_time": "2025-10-14 12:53:26.750000000", + "trace_duration_ms": 6630.38, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-14 12:53:20.120000000", + "updated_at": "2025-10-14 12:53:20.120000000" + }, + { + "span_id": "df882f90-cf1a-42a6-a2ae-78b2a13f27a1", + "trace_id": "4bdd5814-acbb-4de7-a56b-543b87a1cfa5", + "parent_span_id": "ec530838-a7cf-477b-9186-a70114c8a12f", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-14 12:53:20.120000000", + "end_time": "2025-10-14 12:53:20.120000000", + "duration_ms": 0.082, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: Tell me about recent developments in machine learning", + "query_length": 53, + "query_type": "routing_evaluation_query", + "classification.mode": "Web Search", + "classification.rag_keyword_score": 0, + "classification.web_keyword_score": 1, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "complex", + "query.intent": "general_knowledge", + "query.keyword_density": 0.125, + "query.domain_affinity": "general", + "routing.decision": "Routed to Web Search based on analysis", + "routing.rationale": "RAG score: 0, Web score: 1, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "RAG", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.019230769230769232, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: Web Search (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "4bdd5814-acbb-4de7-a56b-543b87a1cfa5", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-14 12:53:20.120000000", + "trace_end_time": "2025-10-14 12:53:26.750000000", + "trace_duration_ms": 6630.38, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-14 12:53:20.120000000", + "updated_at": "2025-10-14 12:53:20.120000000" + }, + { + "span_id": "bea819fd-3673-480b-9f7b-a3a2efef4218", + "trace_id": "4bdd5814-acbb-4de7-a56b-543b87a1cfa5", + "parent_span_id": "11c07411-df30-42a4-9062-90a032ef75ed", + "name": "agent.llm_model_execution", + "start_time": "2025-10-14 12:53:20.455000000", + "end_time": "2025-10-14 12:53:26.746000000", + "duration_ms": 6291.401, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: Tell me about recent developments in machine learning", + "query_length": 53, + "query_type": "web_search_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 6.290812253952026, + "model.latency_ms": 6290.812253952026, + "model.start_time": 1760446400.4555368, + "model.end_time": 1760446406.746349, + "model.performance_tier": "slow", + "model.prompt_tokens": 163, + "model.completion_tokens": 462, + "model.total_tokens": 625, + "model.tokens_per_second": 99.35124031199013, + "model.estimated_cost": 0.01875, + "model.efficiency_score": 2.9584, + "model.response_length": 1849, + "model.response_quality": "high", + "model.output_response": "Model Response: The web search did not yield specific results regarding recent developments in machine learning. However, I can provide a general overview based on my knowledge up to October 2023.\n\nRecent development...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "4bdd5814-acbb-4de7-a56b-543b87a1cfa5", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-14 12:53:20.120000000", + "trace_end_time": "2025-10-14 12:53:26.750000000", + "trace_duration_ms": 6630.38, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-14 12:53:20.120000000", + "updated_at": "2025-10-14 12:53:20.120000000" + }, + { + "span_id": "6e62af18-569c-4783-b2ae-177c58542bd0", + "trace_id": "4bdd5814-acbb-4de7-a56b-543b87a1cfa5", + "parent_span_id": "11c07411-df30-42a4-9062-90a032ef75ed", + "name": "agent.web_search_evaluation_metrics", + "start_time": "2025-10-14 12:53:26.749000000", + "end_time": "2025-10-14 12:53:26.750000000", + "duration_ms": 0.217, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "web_search_evaluation_metrics", + "agent.capabilities": [ + "web_search_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: Tell me about recent developments in machine learning", + "query_length": 53, + "query_type": "web_search_evaluation_query", + "web_search.results_count": 1, + "web_search.context_length": 155, + "web_search.context_synthesized": "Context from 1 web sources", + "web_search.search_effectiveness": 0.2, + "web_search.context_quality": "low", + "web_search.source_diversity": 1, + "web_search.information_synthesis": "low", + "web_search.external_knowledge_utilization": 0.155, + "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=Tell+me+about+recent+developments+in+machine+learning\n\nUser Question: Tell me about recent developments in machine learning\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", + "prompt.prompt_length": 653, + "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=Tell+me+about+recent+developments+in+machine+learning", + "prompt.instruction_following": "web_search_optimized", + "response.answer_length": 1849, + "response.answer_completeness": "complete", + "response.response_quality": "medium", + "response.source_citation": 1, + "response.context_utilization": 11.929032258064517, + "output_response": "Web Search Answer: The web search did not yield specific results regarding recent developments in machine learning. However, I can provide a general overview based on my knowledge up to October 2023.\n\nRecent development...", + "evaluation.search_effectiveness": 0.2, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 1, + "evaluation.context_relevance": "low", + "evaluation.overall_quality": "low", + "evaluation.ready_for_production": true, + "web_search.search_strategy": "duckduckgo_api", + "web_search.real_time_data": true, + "web_search.external_sources": 1, + "web_search.information_freshness": "current", + "web_search.knowledge_synthesis": "single_source" + }, + "events": [], + "links": [], + "trace_trace_id": "4bdd5814-acbb-4de7-a56b-543b87a1cfa5", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-14 12:53:20.120000000", + "trace_end_time": "2025-10-14 12:53:26.750000000", + "trace_duration_ms": 6630.38, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-14 12:53:20.120000000", + "updated_at": "2025-10-14 12:53:20.120000000" + } +] \ No newline at end of file diff --git a/noveum_customer_support_bt/traces/traces/dataset_filtered_mapped.json b/noveum_customer_support_bt/traces/traces/dataset_filtered_mapped.json new file mode 100644 index 0000000..5578d29 --- /dev/null +++ b/noveum_customer_support_bt/traces/traces/dataset_filtered_mapped.json @@ -0,0 +1,20904 @@ +[ + { + "span_id": "4149abdc-acdc-448c-b9b0-b09e166e461a", + "trace_id": "6fe93abf-1413-49e8-8464-58558e25ab7b", + "parent_span_id": "817c9703-0062-4bd6-b3ee-b50a1121c2d2", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-22 16:58:42.084000000", + "end_time": "2025-10-22 16:58:42.084000000", + "duration_ms": 0.015000000000000001, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What are the latest breakthroughs in artificial intelligence?", + "query_length": 61, + "query_type": "routing_evaluation_query", + "classification.mode": "Web Search", + "classification.rag_keyword_score": 0, + "classification.web_keyword_score": 1, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "complex", + "query.intent": "general_knowledge", + "query.keyword_density": 0.125, + "query.domain_affinity": "general", + "routing.decision": "Routed to Web Search based on analysis", + "routing.rationale": "RAG score: 0, Web score: 1, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "RAG", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.019230769230769232, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: Web Search (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "6fe93abf-1413-49e8-8464-58558e25ab7b", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:58:42.084000000", + "trace_end_time": "2025-10-22 16:58:44.766000000", + "trace_duration_ms": 2681.9289999999996, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:58:42.084000000", + "updated_at": "2025-10-22 16:58:42.084000000" + }, + { + "span_id": "817c9703-0062-4bd6-b3ee-b50a1121c2d2", + "trace_id": "6fe93abf-1413-49e8-8464-58558e25ab7b", + "parent_span_id": "ff171ffc-dc6b-4cb4-9833-64c5622d9941", + "name": "agent.query_routing", + "start_time": "2025-10-22 16:58:42.084000000", + "end_time": "2025-10-22 16:58:44.766000000", + "duration_ms": 2681.799, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What are the latest breakthroughs in artificial intelligence?", + "query_length": 61, + "query_type": "routing_query", + "output_response": "Routed to Web Search for query processing", + "router.classification": "Web Search", + "router.confidence_score": 1, + "router.rag_keyword_score": 0, + "router.web_keyword_score": 1, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "6fe93abf-1413-49e8-8464-58558e25ab7b", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:58:42.084000000", + "trace_end_time": "2025-10-22 16:58:44.766000000", + "trace_duration_ms": 2681.9289999999996, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:58:42.084000000", + "updated_at": "2025-10-22 16:58:42.084000000" + }, + { + "span_id": "ad476d62-d6b5-4142-8771-29855db3f39e", + "trace_id": "6fe93abf-1413-49e8-8464-58558e25ab7b", + "parent_span_id": "817c9703-0062-4bd6-b3ee-b50a1121c2d2", + "name": "agent.web_search_generation", + "start_time": "2025-10-22 16:58:42.084000000", + "end_time": "2025-10-22 16:58:44.766000000", + "duration_ms": 2681.732, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "web_search_agent", + "agent.operation": "web_search_generation", + "agent.capabilities": [ + "web_search", + "content_synthesis", + "response_generation" + ], + "agent.id": "noveum_web_search_agent", + "input_query": "What are the latest breakthroughs in artificial intelligence?", + "query_length": 61, + "query_type": "web_search_query", + "output_response": "Web Search Answer: The web search did not yield specific results regarding the latest breakthroughs in artificial intelligence. However, I can provide a general overview based on knowledge up to October 2023.\n\nRecent br...", + "web_search.results_count": 1, + "web_search.context_length": 163, + "web_search.response_length": 1040, + "web_search.mode": "external_web_search" + }, + "events": [], + "links": [], + "trace_trace_id": "6fe93abf-1413-49e8-8464-58558e25ab7b", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:58:42.084000000", + "trace_end_time": "2025-10-22 16:58:44.766000000", + "trace_duration_ms": 2681.9289999999996, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:58:42.084000000", + "updated_at": "2025-10-22 16:58:42.084000000" + }, + { + "span_id": "ff171ffc-dc6b-4cb4-9833-64c5622d9941", + "trace_id": "6fe93abf-1413-49e8-8464-58558e25ab7b", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-22 16:58:42.084000000", + "end_time": "2025-10-22 16:58:44.766000000", + "duration_ms": 2681.902, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What are the latest breakthroughs in artificial intelligence?", + "query.length": 61, + "processing.mode": "Web Search", + "processing.time_seconds": 2.6818161010742188, + "processing.response_length": 1040, + "processing.sources_count": 1, + "output_response": "Final Answer: The web search did not yield specific results regarding the latest breakthroughs in artificial intelligence. However, I can provide a general overview based on knowledge up to October 2023.\n\nRecent br...", + "final_answer_mode": "Web Search", + "query_processed.input_query": "What are the latest breakthroughs in artificial intelligence?", + "query_processed.output_response": "Successfully processed query using Web Search, generated 1040 character response", + "query_processed.mode": "Web Search", + "query_processed.processing_time": 2.6818161010742188, + "query_processed.response_length": 1040 + }, + "events": [], + "links": [], + "trace_trace_id": "6fe93abf-1413-49e8-8464-58558e25ab7b", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:58:42.084000000", + "trace_end_time": "2025-10-22 16:58:44.766000000", + "trace_duration_ms": 2681.9289999999996, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:58:42.084000000", + "updated_at": "2025-10-22 16:58:42.084000000" + }, + { + "span_id": "467cb806-a6ef-43da-9739-570730dd244b", + "trace_id": "6fe93abf-1413-49e8-8464-58558e25ab7b", + "parent_span_id": "ad476d62-d6b5-4142-8771-29855db3f39e", + "name": "agent.llm_model_execution", + "start_time": "2025-10-22 16:58:42.438000000", + "end_time": "2025-10-22 16:58:44.766000000", + "duration_ms": 2327.706, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What are the latest breakthroughs in artificial intelligence?", + "query_length": 61, + "query_type": "web_search_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 2.3276820182800293, + "model.latency_ms": 2327.6820182800293, + "model.start_time": 1761152322.438459, + "model.end_time": 1761152324.766141, + "model.performance_tier": "medium", + "model.prompt_tokens": 167, + "model.completion_tokens": 260, + "model.total_tokens": 427, + "model.tokens_per_second": 183.44430065903882, + "model.estimated_cost": 0.01281, + "model.efficiency_score": 2.4355971896955504, + "model.response_length": 1040, + "model.response_quality": "high", + "model.output_response": "Model Response: The web search did not yield specific results regarding the latest breakthroughs in artificial intelligence. However, I can provide a general overview based on knowledge up to October 2023.\n\nRecent br...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "6fe93abf-1413-49e8-8464-58558e25ab7b", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:58:42.084000000", + "trace_end_time": "2025-10-22 16:58:44.766000000", + "trace_duration_ms": 2681.9289999999996, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:58:42.084000000", + "updated_at": "2025-10-22 16:58:42.084000000" + }, + { + "span_id": "a80e4596-01ea-44e0-a590-13e347857f2a", + "trace_id": "6fe93abf-1413-49e8-8464-58558e25ab7b", + "parent_span_id": "ad476d62-d6b5-4142-8771-29855db3f39e", + "name": "agent.web_search_evaluation_metrics", + "start_time": "2025-10-22 16:58:44.766000000", + "end_time": "2025-10-22 16:58:44.766000000", + "duration_ms": 0.008, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "web_search_evaluation_metrics", + "agent.capabilities": [ + "web_search_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What are the latest breakthroughs in artificial intelligence?", + "query_length": 61, + "query_type": "web_search_evaluation_query", + "web_search.results_count": 1, + "web_search.context_length": 163, + "web_search.context_synthesized": "Context from 1 web sources", + "web_search.search_effectiveness": 0.2, + "web_search.context_quality": "low", + "web_search.source_diversity": 1, + "web_search.information_synthesis": "low", + "web_search.external_knowledge_utilization": 0.163, + "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What+are+the+latest+breakthroughs+in+artificial+intelligence?\n\nUser Question: What are the latest breakthroughs in artificial intelligence?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", + "prompt.prompt_length": 669, + "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What+are+the+latest+breakthroughs+in+artificial+intelligence?", + "prompt.instruction_following": "web_search_optimized", + "response.answer_length": 1040, + "response.answer_completeness": "complete", + "response.response_quality": "medium", + "response.source_citation": 1, + "response.context_utilization": 6.380368098159509, + "output_response": "Web Search Answer: The web search did not yield specific results regarding the latest breakthroughs in artificial intelligence. However, I can provide a general overview based on knowledge up to October 2023.\n\nRecent br...", + "evaluation.search_effectiveness": 0.2, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 1, + "evaluation.context_relevance": "low", + "evaluation.overall_quality": "low", + "evaluation.ready_for_production": true, + "web_search.search_strategy": "duckduckgo_api", + "web_search.real_time_data": true, + "web_search.external_sources": 1, + "web_search.information_freshness": "current", + "web_search.knowledge_synthesis": "single_source" + }, + "events": [], + "links": [], + "trace_trace_id": "6fe93abf-1413-49e8-8464-58558e25ab7b", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:58:42.084000000", + "trace_end_time": "2025-10-22 16:58:44.766000000", + "trace_duration_ms": 2681.9289999999996, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:58:42.084000000", + "updated_at": "2025-10-22 16:58:42.084000000" + }, + { + "span_id": "409f29f6-21c7-400a-a9e0-11c87c4313cb", + "trace_id": "ec7c6cae-9bbd-47f4-a8ae-2d6d9b5011ba", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-22 16:58:32.801000000", + "end_time": "2025-10-22 16:58:41.077000000", + "duration_ms": 8276.701, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What's happening in the software development world today?", + "query.length": 57, + "processing.mode": "Web Search", + "processing.time_seconds": 8.276506900787354, + "processing.response_length": 1424, + "processing.sources_count": 1, + "output_response": "Final Answer: The web search did not yield any specific results regarding current events in the software development world. Therefore, I cannot provide detailed information on the latest trends, technologies, or ne...", + "final_answer_mode": "Web Search", + "query_processed.input_query": "What's happening in the software development world today?", + "query_processed.output_response": "Successfully processed query using Web Search, generated 1424 character response", + "query_processed.mode": "Web Search", + "query_processed.processing_time": 8.276506900787354, + "query_processed.response_length": 1424 + }, + "events": [], + "links": [], + "trace_trace_id": "ec7c6cae-9bbd-47f4-a8ae-2d6d9b5011ba", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:58:32.800000000", + "trace_end_time": "2025-10-22 16:58:41.077000000", + "trace_duration_ms": 8276.788, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:58:32.800000000", + "updated_at": "2025-10-22 16:58:32.800000000" + }, + { + "span_id": "43389a18-a566-41e7-8db5-f688f651a106", + "trace_id": "ec7c6cae-9bbd-47f4-a8ae-2d6d9b5011ba", + "parent_span_id": "af400e17-8b20-4218-834b-9330ccaa2635", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-22 16:58:32.801000000", + "end_time": "2025-10-22 16:58:32.801000000", + "duration_ms": 0.047, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What's happening in the software development world today?", + "query_length": 57, + "query_type": "routing_evaluation_query", + "classification.mode": "Web Search", + "classification.rag_keyword_score": 0, + "classification.web_keyword_score": 1, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "complex", + "query.intent": "general_knowledge", + "query.keyword_density": 0.125, + "query.domain_affinity": "general", + "routing.decision": "Routed to Web Search based on analysis", + "routing.rationale": "RAG score: 0, Web score: 1, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "RAG", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.019230769230769232, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: Web Search (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "ec7c6cae-9bbd-47f4-a8ae-2d6d9b5011ba", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:58:32.800000000", + "trace_end_time": "2025-10-22 16:58:41.077000000", + "trace_duration_ms": 8276.788, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:58:32.800000000", + "updated_at": "2025-10-22 16:58:32.800000000" + }, + { + "span_id": "6b431bca-af7e-4f7e-aedb-b103d7f47cae", + "trace_id": "ec7c6cae-9bbd-47f4-a8ae-2d6d9b5011ba", + "parent_span_id": "af400e17-8b20-4218-834b-9330ccaa2635", + "name": "agent.web_search_generation", + "start_time": "2025-10-22 16:58:32.801000000", + "end_time": "2025-10-22 16:58:41.077000000", + "duration_ms": 8276.225999999999, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "web_search_agent", + "agent.operation": "web_search_generation", + "agent.capabilities": [ + "web_search", + "content_synthesis", + "response_generation" + ], + "agent.id": "noveum_web_search_agent", + "input_query": "What's happening in the software development world today?", + "query_length": 57, + "query_type": "web_search_query", + "output_response": "Web Search Answer: The web search did not yield any specific results regarding current events in the software development world. Therefore, I cannot provide detailed information on the latest trends, technologies, or ne...", + "web_search.results_count": 1, + "web_search.context_length": 159, + "web_search.response_length": 1424, + "web_search.mode": "external_web_search" + }, + "events": [], + "links": [], + "trace_trace_id": "ec7c6cae-9bbd-47f4-a8ae-2d6d9b5011ba", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:58:32.800000000", + "trace_end_time": "2025-10-22 16:58:41.077000000", + "trace_duration_ms": 8276.788, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:58:32.800000000", + "updated_at": "2025-10-22 16:58:32.800000000" + }, + { + "span_id": "af400e17-8b20-4218-834b-9330ccaa2635", + "trace_id": "ec7c6cae-9bbd-47f4-a8ae-2d6d9b5011ba", + "parent_span_id": "409f29f6-21c7-400a-a9e0-11c87c4313cb", + "name": "agent.query_routing", + "start_time": "2025-10-22 16:58:32.801000000", + "end_time": "2025-10-22 16:58:41.077000000", + "duration_ms": 8276.465999999999, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What's happening in the software development world today?", + "query_length": 57, + "query_type": "routing_query", + "output_response": "Routed to Web Search for query processing", + "router.classification": "Web Search", + "router.confidence_score": 1, + "router.rag_keyword_score": 0, + "router.web_keyword_score": 1, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "ec7c6cae-9bbd-47f4-a8ae-2d6d9b5011ba", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:58:32.800000000", + "trace_end_time": "2025-10-22 16:58:41.077000000", + "trace_duration_ms": 8276.788, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:58:32.800000000", + "updated_at": "2025-10-22 16:58:32.800000000" + }, + { + "span_id": "37b153b2-d02b-4b7e-9229-1ba3eaae44ee", + "trace_id": "ec7c6cae-9bbd-47f4-a8ae-2d6d9b5011ba", + "parent_span_id": "6b431bca-af7e-4f7e-aedb-b103d7f47cae", + "name": "agent.llm_model_execution", + "start_time": "2025-10-22 16:58:33.172000000", + "end_time": "2025-10-22 16:58:41.077000000", + "duration_ms": 7904.722, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What's happening in the software development world today?", + "query_length": 57, + "query_type": "web_search_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 7.904651880264282, + "model.latency_ms": 7904.651880264282, + "model.start_time": 1761152313.172655, + "model.end_time": 1761152321.077307, + "model.performance_tier": "slow", + "model.prompt_tokens": 165, + "model.completion_tokens": 356, + "model.total_tokens": 521, + "model.tokens_per_second": 65.91055594754174, + "model.estimated_cost": 0.01563, + "model.efficiency_score": 2.7332053742802302, + "model.response_length": 1424, + "model.response_quality": "high", + "model.output_response": "Model Response: The web search did not yield any specific results regarding current events in the software development world. Therefore, I cannot provide detailed information on the latest trends, technologies, or ne...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "ec7c6cae-9bbd-47f4-a8ae-2d6d9b5011ba", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:58:32.800000000", + "trace_end_time": "2025-10-22 16:58:41.077000000", + "trace_duration_ms": 8276.788, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:58:32.800000000", + "updated_at": "2025-10-22 16:58:32.800000000" + }, + { + "span_id": "cc941b8e-6625-43e9-a0a9-7328af24f12f", + "trace_id": "ec7c6cae-9bbd-47f4-a8ae-2d6d9b5011ba", + "parent_span_id": "6b431bca-af7e-4f7e-aedb-b103d7f47cae", + "name": "agent.web_search_evaluation_metrics", + "start_time": "2025-10-22 16:58:41.077000000", + "end_time": "2025-10-22 16:58:41.077000000", + "duration_ms": 0.026, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "web_search_evaluation_metrics", + "agent.capabilities": [ + "web_search_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What's happening in the software development world today?", + "query_length": 57, + "query_type": "web_search_evaluation_query", + "web_search.results_count": 1, + "web_search.context_length": 159, + "web_search.context_synthesized": "Context from 1 web sources", + "web_search.search_effectiveness": 0.2, + "web_search.context_quality": "low", + "web_search.source_diversity": 1, + "web_search.information_synthesis": "low", + "web_search.external_knowledge_utilization": 0.159, + "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What's+happening+in+the+software+development+world+today?\n\nUser Question: What's happening in the software development world today?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", + "prompt.prompt_length": 661, + "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What's+happening+in+the+software+development+world+today?", + "prompt.instruction_following": "web_search_optimized", + "response.answer_length": 1424, + "response.answer_completeness": "complete", + "response.response_quality": "medium", + "response.source_citation": 1, + "response.context_utilization": 8.955974842767295, + "output_response": "Web Search Answer: The web search did not yield any specific results regarding current events in the software development world. Therefore, I cannot provide detailed information on the latest trends, technologies, or ne...", + "evaluation.search_effectiveness": 0.2, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 1, + "evaluation.context_relevance": "low", + "evaluation.overall_quality": "low", + "evaluation.ready_for_production": true, + "web_search.search_strategy": "duckduckgo_api", + "web_search.real_time_data": true, + "web_search.external_sources": 1, + "web_search.information_freshness": "current", + "web_search.knowledge_synthesis": "single_source" + }, + "events": [], + "links": [], + "trace_trace_id": "ec7c6cae-9bbd-47f4-a8ae-2d6d9b5011ba", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:58:32.800000000", + "trace_end_time": "2025-10-22 16:58:41.077000000", + "trace_duration_ms": 8276.788, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:58:32.800000000", + "updated_at": "2025-10-22 16:58:32.800000000" + }, + { + "span_id": "3b06d098-6eb7-4bb6-885e-443ddb28cc41", + "trace_id": "343764b8-7ac9-4549-93cf-8abc11ceba6f", + "parent_span_id": "5c2aa1cf-56a8-4fd5-af90-90a94e723bb5", + "name": "agent.query_routing", + "start_time": "2025-10-22 16:58:26.542000000", + "end_time": "2025-10-22 16:58:31.796000000", + "duration_ms": 5253.835, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What are the newest features in cloud computing?", + "query_length": 48, + "query_type": "routing_query", + "output_response": "Routed to RAG for query processing", + "router.classification": "RAG", + "router.confidence_score": 1, + "router.rag_keyword_score": 1, + "router.web_keyword_score": 0, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "343764b8-7ac9-4549-93cf-8abc11ceba6f", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:58:26.542000000", + "trace_end_time": "2025-10-22 16:58:31.796000000", + "trace_duration_ms": 5253.973, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:58:26.542000000", + "updated_at": "2025-10-22 16:58:26.542000000" + }, + { + "span_id": "5c2aa1cf-56a8-4fd5-af90-90a94e723bb5", + "trace_id": "343764b8-7ac9-4549-93cf-8abc11ceba6f", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-22 16:58:26.542000000", + "end_time": "2025-10-22 16:58:31.796000000", + "duration_ms": 5253.913, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What are the newest features in cloud computing?", + "query.length": 48, + "processing.mode": "RAG", + "processing.time_seconds": 5.253868818283081, + "processing.response_length": 837, + "processing.sources_count": 5, + "output_response": "Final Answer: The provided context does not specifically mention any new features in cloud computing as a general topic. However, it does highlight several recent enhancements related to the Noveum.ai platform, whi...", + "final_answer_mode": "RAG", + "query_processed.input_query": "What are the newest features in cloud computing?", + "query_processed.output_response": "Successfully processed query using RAG, generated 837 character response", + "query_processed.mode": "RAG", + "query_processed.processing_time": 5.253868818283081, + "query_processed.response_length": 837 + }, + "events": [], + "links": [], + "trace_trace_id": "343764b8-7ac9-4549-93cf-8abc11ceba6f", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:58:26.542000000", + "trace_end_time": "2025-10-22 16:58:31.796000000", + "trace_duration_ms": 5253.973, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:58:26.542000000", + "updated_at": "2025-10-22 16:58:26.542000000" + }, + { + "span_id": "6b838ed6-f5bb-4bcf-a76b-bac283bf7571", + "trace_id": "343764b8-7ac9-4549-93cf-8abc11ceba6f", + "parent_span_id": "3b06d098-6eb7-4bb6-885e-443ddb28cc41", + "name": "agent.llm-rag", + "start_time": "2025-10-22 16:58:26.542000000", + "end_time": "2025-10-22 16:58:31.796000000", + "duration_ms": 5253.519, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "rag_agent", + "agent.operation": "llm-rag", + "agent.capabilities": [ + "document_retrieval", + "context_generation", + "response_generation" + ], + "agent.id": "noveum_rag_agent", + "input_query": "What are the newest features in cloud computing?", + "query_length": 48, + "query_type": "rag_query", + "output_response": "RAG Answer: The provided context does not specifically mention any new features in cloud computing as a general topic. However, it does highlight several recent enhancements related to the Noveum.ai platform, whi...", + "rag.context_length": 2579, + "rag.sources_count": 5, + "rag.answer_length": 837, + "rag.mode": "retrieval_augmented_generation" + }, + "events": [], + "links": [], + "trace_trace_id": "343764b8-7ac9-4549-93cf-8abc11ceba6f", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:58:26.542000000", + "trace_end_time": "2025-10-22 16:58:31.796000000", + "trace_duration_ms": 5253.973, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:58:26.542000000", + "updated_at": "2025-10-22 16:58:26.542000000" + }, + { + "span_id": "f51d8f1e-eae3-488d-9a96-49ab97377851", + "trace_id": "343764b8-7ac9-4549-93cf-8abc11ceba6f", + "parent_span_id": "3b06d098-6eb7-4bb6-885e-443ddb28cc41", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-22 16:58:26.542000000", + "end_time": "2025-10-22 16:58:26.542000000", + "duration_ms": 0.082, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What are the newest features in cloud computing?", + "query_length": 48, + "query_type": "routing_evaluation_query", + "classification.mode": "RAG", + "classification.rag_keyword_score": 1, + "classification.web_keyword_score": 0, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "documentation", + "query.keyword_density": 0.125, + "query.domain_affinity": "noveum", + "routing.decision": "Routed to RAG based on analysis", + "routing.rationale": "RAG score: 1, Web score: 0, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "Web Search", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.019230769230769232, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: RAG (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "343764b8-7ac9-4549-93cf-8abc11ceba6f", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:58:26.542000000", + "trace_end_time": "2025-10-22 16:58:31.796000000", + "trace_duration_ms": 5253.973, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:58:26.542000000", + "updated_at": "2025-10-22 16:58:26.542000000" + }, + { + "span_id": "93a0fd69-47b6-4e7c-b8ab-8eaa5aeac240", + "trace_id": "343764b8-7ac9-4549-93cf-8abc11ceba6f", + "parent_span_id": "6b838ed6-f5bb-4bcf-a76b-bac283bf7571", + "name": "agent.llm_model_execution", + "start_time": "2025-10-22 16:58:27.411000000", + "end_time": "2025-10-22 16:58:31.373000000", + "duration_ms": 3961.481, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What are the newest features in cloud computing?", + "query_length": 48, + "query_type": "rag_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 3.9613850116729736, + "model.latency_ms": 3961.3850116729736, + "model.start_time": 1761152307.411936, + "model.end_time": 1761152311.373321, + "model.performance_tier": "medium", + "model.prompt_tokens": 780, + "model.completion_tokens": 209, + "model.total_tokens": 989, + "model.tokens_per_second": 249.66015600243944, + "model.estimated_cost": 0.029670000000000002, + "model.efficiency_score": 0.846309403437816, + "model.response_length": 837, + "model.response_quality": "high", + "model.output_response": "Model Response: The provided context does not specifically mention any new features in cloud computing as a general topic. However, it does highlight several recent enhancements related to the Noveum.ai platform, whi...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "343764b8-7ac9-4549-93cf-8abc11ceba6f", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:58:26.542000000", + "trace_end_time": "2025-10-22 16:58:31.796000000", + "trace_duration_ms": 5253.973, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:58:26.542000000", + "updated_at": "2025-10-22 16:58:26.542000000" + }, + { + "span_id": "48cbc1df-f03a-4c49-90e5-19a2047c1c7b", + "trace_id": "343764b8-7ac9-4549-93cf-8abc11ceba6f", + "parent_span_id": "6b838ed6-f5bb-4bcf-a76b-bac283bf7571", + "name": "agent.rag_evaluation_metrics", + "start_time": "2025-10-22 16:58:31.373000000", + "end_time": "2025-10-22 16:58:31.373000000", + "duration_ms": 0.061, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "rag_evaluation_metrics", + "agent.capabilities": [ + "retrieval_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What are the newest features in cloud computing?", + "query_length": 48, + "query_type": "rag_evaluation_query", + "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/changelog):\n- \ud83d\udcc8 Enhanced Dashboard Analytics: Real-time request tracking, latency monitoring, and cost analysis\n- \ud83d\udd0d Improved Logs Interface: Better search functionality, detailed trace views, and enhanced debugging capabilities\n- \ud83d\udc65 Team Management: Advanced member mana...", + "retrieval.context_length": 2579, + "retrieval.sources_count": 5, + "retrieval.context_quality": "high", + "retrieval.effectiveness": 1, + "retrieval.context_utilization": 2.579, + "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/changelog):\n- \ud83d\udcc8 Enhanced Dashboard Analytics: Real-time request tracking, latency monitoring, and cost analysis\n- \ud83d\udd0d Improved Logs Interface: Better search functionality, detailed trace views, and enhanced debugging capabilities\n- \ud83d\udc65 Team Management: Advanced member management with role-based access control and invitation system\n- \ud83d\udee1\ufe0f Security Improvements: Enhanced API key management and secure credential storage\n4 months ago\n- \ud83d\udcca Advanced Metrics Collection: Comprehensive telemetry system with custom metrics ...\n\nSource 2 (https://noveum.ai/en/changelog):\n- \ud83d\udcdd Documentation Updates: Comprehensive guides for integration and best practices\n5 months ago\n- \ud83c\udf89 Platform Foundation: Initial release of Noveum.ai AI observability platform\n- \ud83d\ude80 Core Features: AI Gateway integration, basic evaluation metrics, and monitoring capabilities\n- \ud83d\udcc8 Evaluation Jobs: Automated model evaluation with configurable metrics and reporting\n- \ud83d\udd0c Provider Integrations: Support for major AI providers and custom model deployments...\n\nSource 3 (https://noveum.ai/docs/platform/dashboard):\n- Bulk Export: Download multiple traces simultaneously\n- Comparative Analysis: Compare performance across multiple traces\nReal-Time Updates\n- Live Refresh: Automatic updates as new traces arrive\n- Connection Monitoring: Real-time status of your trace ingestion pipeline\n- Performance Indicators: Live metrics for system health monitoring\nReady to dive deeper? Explore Projects & Environments to organize your AI applications, or check out Team Collaboration to share insights with your team.\nGet Earl...\n\nSource 4 (https://noveum.ai/en/changelog):\nChangelog\nStay up to date with the latest changes in our product.\n2 months ago\n- \ud83c\udfaf Major Platform Evolution: NovaEval Framework - Launched comprehensive AI model evaluation framework with 20+ built-in scorers including accuracy, RAG metrics, conversational scoring, and specialized agent evaluation tools. Features production-grade deployment on Docker and Kubernetes. (GitHub | PyPI Package)\n- \ud83d\ude80 Next-Generation Traces UI - Complete redesign featuring three-pane layout with directory tree navigatio...\n\nSource 5 (https://noveum.ai/en/blog):\n8/12/2025\nDiscover how Noveum.ai provides comprehensive tracing and observability for AI applications, from development debugging to production optimization.\nShashank Agarwal\n3/3/2025\nDiscover how Noveum.ai provides comprehensive tracing and observability for LLM applications, RAG systems, and multi-agent workflows with our powerful Python and TypeScript SDKs.\nShashank Agarwal\n3/2/2025...\n\nUser Question: What are the newest features in cloud computing?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", + "prompt.prompt_length": 3123, + "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/changelog):\n- \ud83d\udcc8 Enhanced Dashboard Analytics: Real-time request tracking, latency monitoring, and cost analysis\n- \ud83d\udd0d Improved Logs Interface: Better search functionality,...", + "prompt.instruction_following": "rag_optimized", + "response.answer_length": 837, + "response.answer_completeness": "complete", + "response.response_quality": "high", + "response.source_citation": 5, + "response.context_utilization": 0.3245443970531214, + "output_response": "RAG Answer: The provided context does not specifically mention any new features in cloud computing as a general topic. However, it does highlight several recent enhancements related to the Noveum.ai platform, whi...", + "evaluation.retrieval_effectiveness": 1, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 5, + "evaluation.context_relevance": "high", + "evaluation.overall_quality": "high", + "evaluation.ready_for_production": true, + "rag.retrieval_strategy": "semantic_similarity", + "rag.vector_search_results": 5, + "rag.context_synthesis": "multi_source", + "rag.document_coverage": 1, + "rag.information_density": 0.3245443970531214 + }, + "events": [], + "links": [], + "trace_trace_id": "343764b8-7ac9-4549-93cf-8abc11ceba6f", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:58:26.542000000", + "trace_end_time": "2025-10-22 16:58:31.796000000", + "trace_duration_ms": 5253.973, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:58:26.542000000", + "updated_at": "2025-10-22 16:58:26.542000000" + }, + { + "span_id": "7384d697-d7bf-4709-8377-ec26b094b550", + "trace_id": "3f7b6a62-8888-4a5d-b12c-37a3829aea49", + "parent_span_id": "e631b22a-a621-4ecc-bb01-f731ca66f3d0", + "name": "agent.query_routing", + "start_time": "2025-10-22 16:58:19.885000000", + "end_time": "2025-10-22 16:58:25.536000000", + "duration_ms": 5651.335999999999, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What's the current status of cryptocurrency markets?", + "query_length": 52, + "query_type": "routing_query", + "output_response": "Routed to Web Search for query processing", + "router.classification": "Web Search", + "router.confidence_score": 1, + "router.rag_keyword_score": 0, + "router.web_keyword_score": 3, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "3f7b6a62-8888-4a5d-b12c-37a3829aea49", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:58:19.885000000", + "trace_end_time": "2025-10-22 16:58:25.536000000", + "trace_duration_ms": 5651.67, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:58:19.885000000", + "updated_at": "2025-10-22 16:58:19.885000000" + }, + { + "span_id": "7abf372d-d9f4-41d1-9e3a-d4f01173fa5e", + "trace_id": "3f7b6a62-8888-4a5d-b12c-37a3829aea49", + "parent_span_id": "7384d697-d7bf-4709-8377-ec26b094b550", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-22 16:58:19.885000000", + "end_time": "2025-10-22 16:58:19.885000000", + "duration_ms": 0.037, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What's the current status of cryptocurrency markets?", + "query_length": 52, + "query_type": "routing_evaluation_query", + "classification.mode": "Web Search", + "classification.rag_keyword_score": 0, + "classification.web_keyword_score": 3, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "complex", + "query.intent": "general_knowledge", + "query.keyword_density": 0.42857142857142855, + "query.domain_affinity": "general", + "routing.decision": "Routed to Web Search based on analysis", + "routing.rationale": "RAG score: 0, Web score: 3, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "RAG", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.057692307692307696, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: Web Search (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "3f7b6a62-8888-4a5d-b12c-37a3829aea49", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:58:19.885000000", + "trace_end_time": "2025-10-22 16:58:25.536000000", + "trace_duration_ms": 5651.67, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:58:19.885000000", + "updated_at": "2025-10-22 16:58:19.885000000" + }, + { + "span_id": "e5d9c3c3-6c18-45d2-a6b6-58bde4d4e461", + "trace_id": "3f7b6a62-8888-4a5d-b12c-37a3829aea49", + "parent_span_id": "7384d697-d7bf-4709-8377-ec26b094b550", + "name": "agent.web_search_generation", + "start_time": "2025-10-22 16:58:19.885000000", + "end_time": "2025-10-22 16:58:25.536000000", + "duration_ms": 5651.161, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "web_search_agent", + "agent.operation": "web_search_generation", + "agent.capabilities": [ + "web_search", + "content_synthesis", + "response_generation" + ], + "agent.id": "noveum_web_search_agent", + "input_query": "What's the current status of cryptocurrency markets?", + "query_length": 52, + "query_type": "web_search_query", + "output_response": "Web Search Answer: The provided web search results do not contain specific information regarding the current status of cryptocurrency markets. To get the latest updates on cryptocurrency prices, trends, and market analy...", + "web_search.results_count": 1, + "web_search.context_length": 251, + "web_search.response_length": 416, + "web_search.mode": "external_web_search" + }, + "events": [], + "links": [], + "trace_trace_id": "3f7b6a62-8888-4a5d-b12c-37a3829aea49", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:58:19.885000000", + "trace_end_time": "2025-10-22 16:58:25.536000000", + "trace_duration_ms": 5651.67, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:58:19.885000000", + "updated_at": "2025-10-22 16:58:19.885000000" + }, + { + "span_id": "e631b22a-a621-4ecc-bb01-f731ca66f3d0", + "trace_id": "3f7b6a62-8888-4a5d-b12c-37a3829aea49", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-22 16:58:19.885000000", + "end_time": "2025-10-22 16:58:25.536000000", + "duration_ms": 5651.588000000001, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What's the current status of cryptocurrency markets?", + "query.length": 52, + "processing.mode": "Web Search", + "processing.time_seconds": 5.651373863220215, + "processing.response_length": 416, + "processing.sources_count": 1, + "output_response": "Final Answer: The provided web search results do not contain specific information regarding the current status of cryptocurrency markets. To get the latest updates on cryptocurrency prices, trends, and market analy...", + "final_answer_mode": "Web Search", + "query_processed.input_query": "What's the current status of cryptocurrency markets?", + "query_processed.output_response": "Successfully processed query using Web Search, generated 416 character response", + "query_processed.mode": "Web Search", + "query_processed.processing_time": 5.651373863220215, + "query_processed.response_length": 416 + }, + "events": [], + "links": [], + "trace_trace_id": "3f7b6a62-8888-4a5d-b12c-37a3829aea49", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:58:19.885000000", + "trace_end_time": "2025-10-22 16:58:25.536000000", + "trace_duration_ms": 5651.67, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:58:19.885000000", + "updated_at": "2025-10-22 16:58:19.885000000" + }, + { + "span_id": "34e2f906-724e-4116-b61f-82eaa6b75959", + "trace_id": "3f7b6a62-8888-4a5d-b12c-37a3829aea49", + "parent_span_id": "e5d9c3c3-6c18-45d2-a6b6-58bde4d4e461", + "name": "agent.llm_model_execution", + "start_time": "2025-10-22 16:58:24.327000000", + "end_time": "2025-10-22 16:58:25.536000000", + "duration_ms": 1208.66, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What's the current status of cryptocurrency markets?", + "query_length": 52, + "query_type": "web_search_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 1.208608865737915, + "model.latency_ms": 1208.608865737915, + "model.start_time": 1761152304.327749, + "model.end_time": 1761152305.5363579, + "model.performance_tier": "fast", + "model.prompt_tokens": 187, + "model.completion_tokens": 104, + "model.total_tokens": 291, + "model.tokens_per_second": 240.77268357809888, + "model.estimated_cost": 0.00873, + "model.efficiency_score": 1.429553264604811, + "model.response_length": 416, + "model.response_quality": "high", + "model.output_response": "Model Response: The provided web search results do not contain specific information regarding the current status of cryptocurrency markets. To get the latest updates on cryptocurrency prices, trends, and market analy...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "3f7b6a62-8888-4a5d-b12c-37a3829aea49", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:58:19.885000000", + "trace_end_time": "2025-10-22 16:58:25.536000000", + "trace_duration_ms": 5651.67, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:58:19.885000000", + "updated_at": "2025-10-22 16:58:19.885000000" + }, + { + "span_id": "5548e02e-2ea8-472c-9c42-f811d6e4d30b", + "trace_id": "3f7b6a62-8888-4a5d-b12c-37a3829aea49", + "parent_span_id": "e5d9c3c3-6c18-45d2-a6b6-58bde4d4e461", + "name": "agent.web_search_evaluation_metrics", + "start_time": "2025-10-22 16:58:25.536000000", + "end_time": "2025-10-22 16:58:25.536000000", + "duration_ms": 0.044, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "web_search_evaluation_metrics", + "agent.capabilities": [ + "web_search_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What's the current status of cryptocurrency markets?", + "query_length": 52, + "query_type": "web_search_evaluation_query", + "web_search.results_count": 1, + "web_search.context_length": 251, + "web_search.context_synthesized": "Context from 1 web sources", + "web_search.search_effectiveness": 0.2, + "web_search.context_quality": "low", + "web_search.source_diversity": 1, + "web_search.information_synthesis": "low", + "web_search.external_knowledge_utilization": 0.251, + "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nLog in to WhatsApp Web for simple, reliable and private messaging on your desktop. Send and receive messages and files with ease, all for \u2026\nURL: https://duckduckgo.com/?q=What's+the+current+status+of+cryptocurrency+markets?\n\nUser Question: What's the current status of cryptocurrency markets?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", + "prompt.prompt_length": 748, + "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nLog in to WhatsApp Web for simple, reliable and private messaging on your desktop. Send and receive messages and files with ease, all for \u2026\nURL: https://duckduckgo.com/?q=W...", + "prompt.instruction_following": "web_search_optimized", + "response.answer_length": 416, + "response.answer_completeness": "complete", + "response.response_quality": "medium", + "response.source_citation": 1, + "response.context_utilization": 1.657370517928287, + "output_response": "Web Search Answer: The provided web search results do not contain specific information regarding the current status of cryptocurrency markets. To get the latest updates on cryptocurrency prices, trends, and market analy...", + "evaluation.search_effectiveness": 0.2, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 1, + "evaluation.context_relevance": "low", + "evaluation.overall_quality": "low", + "evaluation.ready_for_production": true, + "web_search.search_strategy": "duckduckgo_api", + "web_search.real_time_data": true, + "web_search.external_sources": 1, + "web_search.information_freshness": "current", + "web_search.knowledge_synthesis": "single_source" + }, + "events": [], + "links": [], + "trace_trace_id": "3f7b6a62-8888-4a5d-b12c-37a3829aea49", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:58:19.885000000", + "trace_end_time": "2025-10-22 16:58:25.536000000", + "trace_duration_ms": 5651.67, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:58:19.885000000", + "updated_at": "2025-10-22 16:58:19.885000000" + }, + { + "span_id": "a9e34cea-d4bd-478b-abf2-7a3a38dcd40e", + "trace_id": "51f4534f-23e4-4be7-8d6e-e93391d6e0e2", + "parent_span_id": "8e9b2d06-3e11-4d17-9a32-b1529a2c10a4", + "name": "agent.web_search_generation", + "start_time": "2025-10-22 16:58:15.254000000", + "end_time": "2025-10-22 16:58:18.876000000", + "duration_ms": 3621.674, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "web_search_agent", + "agent.operation": "web_search_generation", + "agent.capabilities": [ + "web_search", + "content_synthesis", + "response_generation" + ], + "agent.id": "noveum_web_search_agent", + "input_query": "What are the latest updates in Python programming?", + "query_length": 50, + "query_type": "web_search_query", + "output_response": "Web Search Answer: The provided web search results do not contain specific information about the latest updates in Python programming. They primarily focus on general news coverage and updates from India and around the ...", + "web_search.results_count": 1, + "web_search.context_length": 753, + "web_search.response_length": 580, + "web_search.mode": "external_web_search" + }, + "events": [], + "links": [], + "trace_trace_id": "51f4534f-23e4-4be7-8d6e-e93391d6e0e2", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:58:15.254000000", + "trace_end_time": "2025-10-22 16:58:18.877000000", + "trace_duration_ms": 3622.355, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:58:15.254000000", + "updated_at": "2025-10-22 16:58:15.254000000" + }, + { + "span_id": "8e9b2d06-3e11-4d17-9a32-b1529a2c10a4", + "trace_id": "51f4534f-23e4-4be7-8d6e-e93391d6e0e2", + "parent_span_id": "bda936a0-24f3-4841-af4f-227197e10b9d", + "name": "agent.query_routing", + "start_time": "2025-10-22 16:58:15.254000000", + "end_time": "2025-10-22 16:58:18.876000000", + "duration_ms": 3621.863, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What are the latest updates in Python programming?", + "query_length": 50, + "query_type": "routing_query", + "output_response": "Routed to Web Search for query processing", + "router.classification": "Web Search", + "router.confidence_score": 1, + "router.rag_keyword_score": 0, + "router.web_keyword_score": 2, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "51f4534f-23e4-4be7-8d6e-e93391d6e0e2", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:58:15.254000000", + "trace_end_time": "2025-10-22 16:58:18.877000000", + "trace_duration_ms": 3622.355, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:58:15.254000000", + "updated_at": "2025-10-22 16:58:15.254000000" + }, + { + "span_id": "bda936a0-24f3-4841-af4f-227197e10b9d", + "trace_id": "51f4534f-23e4-4be7-8d6e-e93391d6e0e2", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-22 16:58:15.254000000", + "end_time": "2025-10-22 16:58:18.877000000", + "duration_ms": 3622.271, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What are the latest updates in Python programming?", + "query.length": 50, + "processing.mode": "Web Search", + "processing.time_seconds": 3.6219100952148438, + "processing.response_length": 580, + "processing.sources_count": 1, + "output_response": "Final Answer: The provided web search results do not contain specific information about the latest updates in Python programming. They primarily focus on general news coverage and updates from India and around the ...", + "final_answer_mode": "Web Search", + "query_processed.input_query": "What are the latest updates in Python programming?", + "query_processed.output_response": "Successfully processed query using Web Search, generated 580 character response", + "query_processed.mode": "Web Search", + "query_processed.processing_time": 3.6219100952148438, + "query_processed.response_length": 580 + }, + "events": [], + "links": [], + "trace_trace_id": "51f4534f-23e4-4be7-8d6e-e93391d6e0e2", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:58:15.254000000", + "trace_end_time": "2025-10-22 16:58:18.877000000", + "trace_duration_ms": 3622.355, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:58:15.254000000", + "updated_at": "2025-10-22 16:58:15.254000000" + }, + { + "span_id": "e27e3812-d49d-4f4f-98b0-144cf0ceffc3", + "trace_id": "51f4534f-23e4-4be7-8d6e-e93391d6e0e2", + "parent_span_id": "8e9b2d06-3e11-4d17-9a32-b1529a2c10a4", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-22 16:58:15.254000000", + "end_time": "2025-10-22 16:58:15.254000000", + "duration_ms": 0.047, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What are the latest updates in Python programming?", + "query_length": 50, + "query_type": "routing_evaluation_query", + "classification.mode": "Web Search", + "classification.rag_keyword_score": 0, + "classification.web_keyword_score": 2, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "general_knowledge", + "query.keyword_density": 0.25, + "query.domain_affinity": "general", + "routing.decision": "Routed to Web Search based on analysis", + "routing.rationale": "RAG score: 0, Web score: 2, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "RAG", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.038461538461538464, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: Web Search (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "51f4534f-23e4-4be7-8d6e-e93391d6e0e2", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:58:15.254000000", + "trace_end_time": "2025-10-22 16:58:18.877000000", + "trace_duration_ms": 3622.355, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:58:15.254000000", + "updated_at": "2025-10-22 16:58:15.254000000" + }, + { + "span_id": "b0eea8ec-5cff-4667-bea7-aeb6d0f0daf3", + "trace_id": "51f4534f-23e4-4be7-8d6e-e93391d6e0e2", + "parent_span_id": "a9e34cea-d4bd-478b-abf2-7a3a38dcd40e", + "name": "agent.llm_model_execution", + "start_time": "2025-10-22 16:58:15.568000000", + "end_time": "2025-10-22 16:58:18.876000000", + "duration_ms": 3307.5739999999996, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What are the latest updates in Python programming?", + "query_length": 50, + "query_type": "web_search_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 3.307521104812622, + "model.latency_ms": 3307.521104812622, + "model.start_time": 1761152295.568949, + "model.end_time": 1761152298.87647, + "model.performance_tier": "medium", + "model.prompt_tokens": 312, + "model.completion_tokens": 145, + "model.total_tokens": 457, + "model.tokens_per_second": 138.16994223711538, + "model.estimated_cost": 0.01371, + "model.efficiency_score": 1.2691466083150984, + "model.response_length": 580, + "model.response_quality": "high", + "model.output_response": "Model Response: The provided web search results do not contain specific information about the latest updates in Python programming. They primarily focus on general news coverage and updates from India and around the ...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "51f4534f-23e4-4be7-8d6e-e93391d6e0e2", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:58:15.254000000", + "trace_end_time": "2025-10-22 16:58:18.877000000", + "trace_duration_ms": 3622.355, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:58:15.254000000", + "updated_at": "2025-10-22 16:58:15.254000000" + }, + { + "span_id": "d0bfcbe5-cb6c-4837-bdf3-f2c65be145cd", + "trace_id": "51f4534f-23e4-4be7-8d6e-e93391d6e0e2", + "parent_span_id": "a9e34cea-d4bd-478b-abf2-7a3a38dcd40e", + "name": "agent.web_search_evaluation_metrics", + "start_time": "2025-10-22 16:58:18.876000000", + "end_time": "2025-10-22 16:58:18.876000000", + "duration_ms": 0.04, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "web_search_evaluation_metrics", + "agent.capabilities": [ + "web_search_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What are the latest updates in Python programming?", + "query_length": 50, + "query_type": "web_search_evaluation_query", + "web_search.results_count": 1, + "web_search.context_length": 753, + "web_search.context_synthesized": "Context from 1 web sources", + "web_search.search_effectiveness": 0.2, + "web_search.context_quality": "medium", + "web_search.source_diversity": 1, + "web_search.information_synthesis": "low", + "web_search.external_knowledge_utilization": 0.753, + "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\n3 days ago \u00b7 Latest news headlines from India & around the world. Check out today\u2019s news coverage live with videos & photos on \u2026 6 days ago \u00b7 India News | Latest India News | Read latest and breaking news from India. Today's top India news headlines, news on \u2026 2 days ago \u00b7 Top News Stories of the day, Latest News Headlines, News Specials, Breaking News and Latest India News, World current affairs & \u2026 NDTV.com provides latest news from India and around the world. Get breaking news alerts from India and follow today\u2019s live news updates in \u2026 6 days ago \u00b7 Get Latest entertainment news from Bollywood, Hollywood and regional film industries.\nURL: https://duckduckgo.com/?q=What+are+the+latest+updates+in+Python+programming?\n\nUser Question: What are the latest updates in Python programming?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", + "prompt.prompt_length": 1248, + "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\n3 days ago \u00b7 Latest news headlines from India & around the world. Check out today\u2019s news coverage live with videos & photos on \u2026 6 days ago \u00b7 India News | Latest India News...", + "prompt.instruction_following": "web_search_optimized", + "response.answer_length": 580, + "response.answer_completeness": "complete", + "response.response_quality": "medium", + "response.source_citation": 1, + "response.context_utilization": 0.7702523240371846, + "output_response": "Web Search Answer: The provided web search results do not contain specific information about the latest updates in Python programming. They primarily focus on general news coverage and updates from India and around the ...", + "evaluation.search_effectiveness": 0.2, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 1, + "evaluation.context_relevance": "medium", + "evaluation.overall_quality": "low", + "evaluation.ready_for_production": true, + "web_search.search_strategy": "duckduckgo_api", + "web_search.real_time_data": true, + "web_search.external_sources": 1, + "web_search.information_freshness": "current", + "web_search.knowledge_synthesis": "single_source" + }, + "events": [], + "links": [], + "trace_trace_id": "51f4534f-23e4-4be7-8d6e-e93391d6e0e2", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:58:15.254000000", + "trace_end_time": "2025-10-22 16:58:18.877000000", + "trace_duration_ms": 3622.355, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:58:15.254000000", + "updated_at": "2025-10-22 16:58:15.254000000" + }, + { + "span_id": "5f5916a7-b9ea-4348-861d-db69b25d6b62", + "trace_id": "e30e9ba6-ffd1-4eb9-b7ae-6d19e04343d6", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-22 16:58:11.033000000", + "end_time": "2025-10-22 16:58:14.250000000", + "duration_ms": 3216.77, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What happened in tech news this week?", + "query.length": 37, + "processing.mode": "Web Search", + "processing.time_seconds": 3.2166178226470947, + "processing.response_length": 510, + "processing.sources_count": 1, + "output_response": "Final Answer: The web search results do not provide specific information about recent events in tech news for this week. They primarily discuss the grammatical usage of the word \"happened\" and its variations, such ...", + "final_answer_mode": "Web Search", + "query_processed.input_query": "What happened in tech news this week?", + "query_processed.output_response": "Successfully processed query using Web Search, generated 510 character response", + "query_processed.mode": "Web Search", + "query_processed.processing_time": 3.2166178226470947, + "query_processed.response_length": 510 + }, + "events": [], + "links": [], + "trace_trace_id": "e30e9ba6-ffd1-4eb9-b7ae-6d19e04343d6", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:58:11.033000000", + "trace_end_time": "2025-10-22 16:58:14.250000000", + "trace_duration_ms": 3216.801, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:58:11.033000000", + "updated_at": "2025-10-22 16:58:11.033000000" + }, + { + "span_id": "804613d0-a701-4c22-add1-d315d7a29813", + "trace_id": "e30e9ba6-ffd1-4eb9-b7ae-6d19e04343d6", + "parent_span_id": "5f5916a7-b9ea-4348-861d-db69b25d6b62", + "name": "agent.query_routing", + "start_time": "2025-10-22 16:58:11.033000000", + "end_time": "2025-10-22 16:58:14.250000000", + "duration_ms": 3216.594, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What happened in tech news this week?", + "query_length": 37, + "query_type": "routing_query", + "output_response": "Routed to Web Search for query processing", + "router.classification": "Web Search", + "router.confidence_score": 1, + "router.rag_keyword_score": 0, + "router.web_keyword_score": 2, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "e30e9ba6-ffd1-4eb9-b7ae-6d19e04343d6", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:58:11.033000000", + "trace_end_time": "2025-10-22 16:58:14.250000000", + "trace_duration_ms": 3216.801, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:58:11.033000000", + "updated_at": "2025-10-22 16:58:11.033000000" + }, + { + "span_id": "f6c8d34f-185f-49ba-a5a4-a9d25ccb4fc6", + "trace_id": "e30e9ba6-ffd1-4eb9-b7ae-6d19e04343d6", + "parent_span_id": "804613d0-a701-4c22-add1-d315d7a29813", + "name": "agent.web_search_generation", + "start_time": "2025-10-22 16:58:11.034000000", + "end_time": "2025-10-22 16:58:14.250000000", + "duration_ms": 3216.4959999999996, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "web_search_agent", + "agent.operation": "web_search_generation", + "agent.capabilities": [ + "web_search", + "content_synthesis", + "response_generation" + ], + "agent.id": "noveum_web_search_agent", + "input_query": "What happened in tech news this week?", + "query_length": 37, + "query_type": "web_search_query", + "output_response": "Web Search Answer: The web search results do not provide specific information about recent events in tech news for this week. They primarily discuss the grammatical usage of the word \"happened\" and its variations, such ...", + "web_search.results_count": 1, + "web_search.context_length": 724, + "web_search.response_length": 510, + "web_search.mode": "external_web_search" + }, + "events": [], + "links": [], + "trace_trace_id": "e30e9ba6-ffd1-4eb9-b7ae-6d19e04343d6", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:58:11.033000000", + "trace_end_time": "2025-10-22 16:58:14.250000000", + "trace_duration_ms": 3216.801, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:58:11.033000000", + "updated_at": "2025-10-22 16:58:11.033000000" + }, + { + "span_id": "11d68082-e7b8-4bbf-833a-79e296c31eda", + "trace_id": "e30e9ba6-ffd1-4eb9-b7ae-6d19e04343d6", + "parent_span_id": "804613d0-a701-4c22-add1-d315d7a29813", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-22 16:58:11.034000000", + "end_time": "2025-10-22 16:58:11.034000000", + "duration_ms": 0.017, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What happened in tech news this week?", + "query_length": 37, + "query_type": "routing_evaluation_query", + "classification.mode": "Web Search", + "classification.rag_keyword_score": 0, + "classification.web_keyword_score": 2, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "general_knowledge", + "query.keyword_density": 0.2857142857142857, + "query.domain_affinity": "general", + "routing.decision": "Routed to Web Search based on analysis", + "routing.rationale": "RAG score: 0, Web score: 2, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "RAG", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.038461538461538464, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: Web Search (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "e30e9ba6-ffd1-4eb9-b7ae-6d19e04343d6", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:58:11.033000000", + "trace_end_time": "2025-10-22 16:58:14.250000000", + "trace_duration_ms": 3216.801, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:58:11.033000000", + "updated_at": "2025-10-22 16:58:11.033000000" + }, + { + "span_id": "420b4ca1-c9f1-488f-ac2c-c53766717634", + "trace_id": "e30e9ba6-ffd1-4eb9-b7ae-6d19e04343d6", + "parent_span_id": "f6c8d34f-185f-49ba-a5a4-a9d25ccb4fc6", + "name": "agent.llm_model_execution", + "start_time": "2025-10-22 16:58:11.468000000", + "end_time": "2025-10-22 16:58:14.250000000", + "duration_ms": 2781.777, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What happened in tech news this week?", + "query_length": 37, + "query_type": "web_search_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 2.781723976135254, + "model.latency_ms": 2781.723976135254, + "model.start_time": 1761152291.468626, + "model.end_time": 1761152294.25035, + "model.performance_tier": "medium", + "model.prompt_tokens": 301, + "model.completion_tokens": 127, + "model.total_tokens": 428, + "model.tokens_per_second": 153.8614196346811, + "model.estimated_cost": 0.01284, + "model.efficiency_score": 1.191588785046729, + "model.response_length": 510, + "model.response_quality": "high", + "model.output_response": "Model Response: The web search results do not provide specific information about recent events in tech news for this week. They primarily discuss the grammatical usage of the word \"happened\" and its variations, such ...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "e30e9ba6-ffd1-4eb9-b7ae-6d19e04343d6", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:58:11.033000000", + "trace_end_time": "2025-10-22 16:58:14.250000000", + "trace_duration_ms": 3216.801, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:58:11.033000000", + "updated_at": "2025-10-22 16:58:11.033000000" + }, + { + "span_id": "56b8bf09-8ebf-44e1-b01f-daff23d3c42c", + "trace_id": "e30e9ba6-ffd1-4eb9-b7ae-6d19e04343d6", + "parent_span_id": "f6c8d34f-185f-49ba-a5a4-a9d25ccb4fc6", + "name": "agent.web_search_evaluation_metrics", + "start_time": "2025-10-22 16:58:14.250000000", + "end_time": "2025-10-22 16:58:14.250000000", + "duration_ms": 0.060000000000000005, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "web_search_evaluation_metrics", + "agent.capabilities": [ + "web_search_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What happened in tech news this week?", + "query_length": 37, + "query_type": "web_search_evaluation_query", + "web_search.results_count": 1, + "web_search.context_length": 724, + "web_search.context_synthesized": "Context from 1 web sources", + "web_search.search_effectiveness": 0.2, + "web_search.context_quality": "medium", + "web_search.source_diversity": 1, + "web_search.information_synthesis": "low", + "web_search.external_knowledge_utilization": 0.724, + "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nhappened\u7684\u7528\u6cd5\u5982\u4e0b\uff1a 1.happened\u662f \u4e0d\u53ca\u7269\u52a8\u8bcd,\u4f5c\u201c\u53d1\u751f\u201d\u89e3\u65f6,\u53ef\u6307\u5076\u7136\u7684\u4e8b\u6545,\u4e5f\u53ef\u6307\u6709\u8ba1\u5212\u6216\u65e0\u8ba1\u5212\u53d1\u751f\u7684\u4e8b\u3002\u4e00\u822c\u4ee5\u4e8b\u4ef6\u3001\u73af\u5883\u3001\u5f62\u52bf\u7b49\u4f5c\u4e3b\u8bed,\u8868\u793a\u201c\u67d0\u4eba\u53d1\u751f\u67d0\u4e8b\u201d\u65f6,\u987b\u4ee5\u4ecb\u8bcdto\u5f15\u51fa\u201c\u67d0\u4eba\u201d\u3002 2. \u2026 what happened\u8fd8\u662fwhat's happened\"What happened\"\u548c\"What's happened\"\u90fd\u7528\u4e8e\u8be2\u95ee\u8fc7\u53bb\u53d1\u751f\u4e86\u4ec0\u4e48\u4e8b\u60c5\uff0c\u4e3b\u8981\u7684\u533a\u522b\u5728\u4e8e\u8bed\u6cd5\u548c\u4f7f\u7528\u65b9\u5f0f\u3002\"What happened\"\u662f\u4e00\u4e2a\u8fc7\u53bb\u65f6\u6001\u7684\u7b80\u5355 \u63a5\u4e0b\u6765\u8ba9\u6211\u4eec\u770b\u4e0bwhat happen\u548cwhat happened\u7684\u7528\u6cd5\u533a\u522b\uff1a 1.\u8bcd\u6027\uff1awhat happen\u662f\u7528\u539f\u5f62\u52a8\u8bcdhappen\u6784\u6210\u7684\u73b0\u5728\u65f6\u77ed\u8bed\uff0c\u800cwhat happened\u662f\u7528\u52a8\u8bcdhappen\u7684\u8fc7\u53bb\u5f0fhappened\u6784\u6210\u7684\u8fc7\u53bb\u65f6\u77ed \u2026 What happened \u548c What's happened\u4e24\u8005\u7684\u533a\u522b\uff1a \u662f\u95ee What happened\uff1f\u53ea\u662f\u60f3\u77e5\u9053\u8fc7\u53bb\u6240\u53d1\u751f\u7684\u4e8b\u60c5\uff0c \u800c\u5982\u679c\u7528What's happened\uff1f\u5219\u5f3a\u8c03\u5bf9\u73b0\u5728\u7684\u5f71\u54cd\u3002\uff0c\u5373\u60f3\u77e5\u9053\u8fc7\u53bb\u6240\u53d1\u751f\u7684\u4ec0\u4e48\u4e8b\u5bfc\u81f4\u4e86\u8bf4 \u2026 it so happened that\u7684\u53e5\u5f0fhappen\u8868\u793a\u201c\u78b0\u5de7\u6216\u6070\u5de7\u53d1\u751f\u67d0\u4e8b\u201d\u65f6,\u53ef\u7528\u201cIt happens / happened that...\u201d\u8fd9\u4e00\u7ed3\u6784\u6765\u8868\u8fbe.\u4f8b\u5982\uff1a It happened that Brian and Peter were at home that day.\u78b0\u5de7\u90a3\u5929\u5e03\u83b1\u6069\u548c \u2026\nURL: https://duckduckgo.com/?q=What+happened+in+tech+news+this+week?\n\nUser Question: What happened in tech news this week?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", + "prompt.prompt_length": 1206, + "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nhappened\u7684\u7528\u6cd5\u5982\u4e0b\uff1a 1.happened\u662f \u4e0d\u53ca\u7269\u52a8\u8bcd,\u4f5c\u201c\u53d1\u751f\u201d\u89e3\u65f6,\u53ef\u6307\u5076\u7136\u7684\u4e8b\u6545,\u4e5f\u53ef\u6307\u6709\u8ba1\u5212\u6216\u65e0\u8ba1\u5212\u53d1\u751f\u7684\u4e8b\u3002\u4e00\u822c\u4ee5\u4e8b\u4ef6\u3001\u73af\u5883\u3001\u5f62\u52bf\u7b49\u4f5c\u4e3b\u8bed,\u8868\u793a\u201c\u67d0\u4eba\u53d1\u751f\u67d0\u4e8b\u201d\u65f6,\u987b\u4ee5\u4ecb\u8bcdto\u5f15\u51fa\u201c\u67d0\u4eba\u201d\u3002 2. \u2026 what happened\u8fd8\u662fwhat's happened\"What happened\"\u548c\"What's happene...", + "prompt.instruction_following": "web_search_optimized", + "response.answer_length": 510, + "response.answer_completeness": "complete", + "response.response_quality": "medium", + "response.source_citation": 1, + "response.context_utilization": 0.7044198895027625, + "output_response": "Web Search Answer: The web search results do not provide specific information about recent events in tech news for this week. They primarily discuss the grammatical usage of the word \"happened\" and its variations, such ...", + "evaluation.search_effectiveness": 0.2, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 1, + "evaluation.context_relevance": "medium", + "evaluation.overall_quality": "low", + "evaluation.ready_for_production": true, + "web_search.search_strategy": "duckduckgo_api", + "web_search.real_time_data": true, + "web_search.external_sources": 1, + "web_search.information_freshness": "current", + "web_search.knowledge_synthesis": "single_source" + }, + "events": [], + "links": [], + "trace_trace_id": "e30e9ba6-ffd1-4eb9-b7ae-6d19e04343d6", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:58:11.033000000", + "trace_end_time": "2025-10-22 16:58:14.250000000", + "trace_duration_ms": 3216.801, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:58:11.033000000", + "updated_at": "2025-10-22 16:58:11.033000000" + }, + { + "span_id": "82eb19e8-f6cf-43fd-ba56-640a6fd3ed0e", + "trace_id": "45efae74-ee25-4004-a509-6e0ff628a0cd", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-22 16:58:04.883000000", + "end_time": "2025-10-22 16:58:10.029000000", + "duration_ms": 5146.146, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What are the current trends in observability tools?", + "query.length": 51, + "processing.mode": "RAG", + "processing.time_seconds": 5.146108150482178, + "processing.response_length": 815, + "processing.sources_count": 5, + "output_response": "Final Answer: The provided context does not explicitly outline current trends in observability tools. However, it highlights the unique challenges faced by AI applications that traditional monitoring tools cannot a...", + "final_answer_mode": "RAG", + "query_processed.input_query": "What are the current trends in observability tools?", + "query_processed.output_response": "Successfully processed query using RAG, generated 815 character response", + "query_processed.mode": "RAG", + "query_processed.processing_time": 5.146108150482178, + "query_processed.response_length": 815 + }, + "events": [], + "links": [], + "trace_trace_id": "45efae74-ee25-4004-a509-6e0ff628a0cd", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:58:04.883000000", + "trace_end_time": "2025-10-22 16:58:10.029000000", + "trace_duration_ms": 5146.244000000001, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:58:04.883000000", + "updated_at": "2025-10-22 16:58:04.883000000" + }, + { + "span_id": "e81617c6-c2a1-4b08-a5fc-dde3439fe2f4", + "trace_id": "45efae74-ee25-4004-a509-6e0ff628a0cd", + "parent_span_id": "8001ba0e-95c7-42da-9087-603a38e1a500", + "name": "agent.llm-rag", + "start_time": "2025-10-22 16:58:04.883000000", + "end_time": "2025-10-22 16:58:10.029000000", + "duration_ms": 5145.655, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "rag_agent", + "agent.operation": "llm-rag", + "agent.capabilities": [ + "document_retrieval", + "context_generation", + "response_generation" + ], + "agent.id": "noveum_rag_agent", + "input_query": "What are the current trends in observability tools?", + "query_length": 51, + "query_type": "rag_query", + "output_response": "RAG Answer: The provided context does not explicitly outline current trends in observability tools. However, it highlights the unique challenges faced by AI applications that traditional monitoring tools cannot a...", + "rag.context_length": 2943, + "rag.sources_count": 5, + "rag.answer_length": 815, + "rag.mode": "retrieval_augmented_generation" + }, + "events": [], + "links": [], + "trace_trace_id": "45efae74-ee25-4004-a509-6e0ff628a0cd", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:58:04.883000000", + "trace_end_time": "2025-10-22 16:58:10.029000000", + "trace_duration_ms": 5146.244000000001, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:58:04.883000000", + "updated_at": "2025-10-22 16:58:04.883000000" + }, + { + "span_id": "8001ba0e-95c7-42da-9087-603a38e1a500", + "trace_id": "45efae74-ee25-4004-a509-6e0ff628a0cd", + "parent_span_id": "82eb19e8-f6cf-43fd-ba56-640a6fd3ed0e", + "name": "agent.query_routing", + "start_time": "2025-10-22 16:58:04.883000000", + "end_time": "2025-10-22 16:58:10.029000000", + "duration_ms": 5146.036999999999, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What are the current trends in observability tools?", + "query_length": 51, + "query_type": "routing_query", + "output_response": "Routed to RAG for query processing", + "router.classification": "RAG", + "router.confidence_score": 0.3333333333333333, + "router.rag_keyword_score": 2, + "router.web_keyword_score": 1, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "45efae74-ee25-4004-a509-6e0ff628a0cd", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:58:04.883000000", + "trace_end_time": "2025-10-22 16:58:10.029000000", + "trace_duration_ms": 5146.244000000001, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:58:04.883000000", + "updated_at": "2025-10-22 16:58:04.883000000" + }, + { + "span_id": "79b7d138-05e1-4f18-8b63-dcfe9732180b", + "trace_id": "45efae74-ee25-4004-a509-6e0ff628a0cd", + "parent_span_id": "8001ba0e-95c7-42da-9087-603a38e1a500", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-22 16:58:04.883000000", + "end_time": "2025-10-22 16:58:04.883000000", + "duration_ms": 0.051, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What are the current trends in observability tools?", + "query_length": 51, + "query_type": "routing_evaluation_query", + "classification.mode": "RAG", + "classification.rag_keyword_score": 2, + "classification.web_keyword_score": 1, + "classification.confidence_score": 0.3333333333333333, + "classification.confidence_level": "medium", + "classification.method": "keyword_based", + "query.complexity": "complex", + "query.intent": "documentation", + "query.keyword_density": 0.375, + "query.domain_affinity": "noveum", + "routing.decision": "Routed to RAG based on analysis", + "routing.rationale": "RAG score: 2, Web score: 1, Confidence: 0.33", + "routing.expected_performance": "medium", + "routing.alternative_mode": "Web Search", + "routing.decision_confidence": 0.3333333333333333, + "evaluation.routing_accuracy": "medium", + "evaluation.keyword_coverage": 0.057692307692307696, + "evaluation.query_understanding": "ambiguous", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: RAG (Confidence: 0.33)" + }, + "events": [], + "links": [], + "trace_trace_id": "45efae74-ee25-4004-a509-6e0ff628a0cd", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:58:04.883000000", + "trace_end_time": "2025-10-22 16:58:10.029000000", + "trace_duration_ms": 5146.244000000001, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:58:04.883000000", + "updated_at": "2025-10-22 16:58:04.883000000" + }, + { + "span_id": "28c233a0-d42a-4200-86b0-dfbd2d5be06e", + "trace_id": "45efae74-ee25-4004-a509-6e0ff628a0cd", + "parent_span_id": "e81617c6-c2a1-4b08-a5fc-dde3439fe2f4", + "name": "agent.llm_model_execution", + "start_time": "2025-10-22 16:58:05.635000000", + "end_time": "2025-10-22 16:58:09.222000000", + "duration_ms": 3586.924, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What are the current trends in observability tools?", + "query_length": 51, + "query_type": "rag_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 3.5868873596191406, + "model.latency_ms": 3586.8873596191406, + "model.start_time": 1761152285.6352398, + "model.end_time": 1761152289.2221272, + "model.performance_tier": "medium", + "model.prompt_tokens": 872, + "model.completion_tokens": 203, + "model.total_tokens": 1075, + "model.tokens_per_second": 299.7027484337129, + "model.estimated_cost": 0.03225, + "model.efficiency_score": 0.7581395348837209, + "model.response_length": 815, + "model.response_quality": "high", + "model.output_response": "Model Response: The provided context does not explicitly outline current trends in observability tools. However, it highlights the unique challenges faced by AI applications that traditional monitoring tools cannot a...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "45efae74-ee25-4004-a509-6e0ff628a0cd", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:58:04.883000000", + "trace_end_time": "2025-10-22 16:58:10.029000000", + "trace_duration_ms": 5146.244000000001, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:58:04.883000000", + "updated_at": "2025-10-22 16:58:04.883000000" + }, + { + "span_id": "024c05b6-edf9-40ef-931b-9660d48b4948", + "trace_id": "45efae74-ee25-4004-a509-6e0ff628a0cd", + "parent_span_id": "e81617c6-c2a1-4b08-a5fc-dde3439fe2f4", + "name": "agent.rag_evaluation_metrics", + "start_time": "2025-10-22 16:58:09.222000000", + "end_time": "2025-10-22 16:58:09.222000000", + "duration_ms": 0.020999999999999998, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "rag_evaluation_metrics", + "agent.capabilities": [ + "retrieval_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What are the current trends in observability tools?", + "query_length": 51, + "query_type": "rag_evaluation_query", + "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/blog/from-logs-to-intelligent-choices-inside-noveum-ais-evaluation-process):\nFrom Development to Production - Inside Noveum.ai's AI Observability Platform\nShashank Agarwal\n3/3/2025\nIntroduction\nEvery AI application tells a story\u2014but most developers never get to hear it...", + "retrieval.context_length": 2943, + "retrieval.sources_count": 5, + "retrieval.context_quality": "high", + "retrieval.effectiveness": 1, + "retrieval.context_utilization": 2.943, + "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/blog/from-logs-to-intelligent-choices-inside-noveum-ais-evaluation-process):\nFrom Development to Production - Inside Noveum.ai's AI Observability Platform\nShashank Agarwal\n3/3/2025\nIntroduction\nEvery AI application tells a story\u2014but most developers never get to hear it. When your RAG pipeline returns irrelevant results, when your multi-agent system gets stuck in loops, or when your LLM costs suddenly spike, you're left guessing what went wrong and where.\nThat's exactly the challenge Noveum.ai solves. Rather than flying blind, our platform provides comprehensive tracing a...\n\nSource 2 (https://noveum.ai/docs/getting-started/tracing-concepts):\n- Explore Framework Integrations - Framework-specific guidance\n- Learn Advanced Patterns - Custom instrumentation techniques\n- Master the Dashboard - Analyze your traces effectively\nRemember: Good observability is not about collecting all possible data, but about collecting the right data that helps you understand, debug, and optimize your AI applications.\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to O...\n\nSource 3 (https://noveum.ai/en/docs/getting-started/tracing-concepts):\n- Explore Framework Integrations - Framework-specific guidance\n- Learn Advanced Patterns - Custom instrumentation techniques\n- Master the Dashboard - Analyze your traces effectively\nRemember: Good observability is not about collecting all possible data, but about collecting the right data that helps you understand, debug, and optimize your AI applications.\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to O...\n\nSource 4 (https://noveum.ai/en/blog/from-logs-to-intelligent-choices-inside-noveum-ais-evaluation-process):\nThe AI Observability Challenge\nWhy Traditional Monitoring Falls Short\nTraditional application monitoring tools weren't built for AI applications. They can tell you if your API is responding, but they can't answer the questions that matter most for AI systems:\n- Why did my RAG pipeline retrieve irrelevant documents?\n- Which LLM calls are driving my costs?\n- How are my agents communicating with each other?\n- What's causing hallucinations in my responses?\n- Why is my embedding generation so slow?\nW...\n\nSource 5 (https://noveum.ai/en/docs/advanced/rag-observability):\n\ud83d\udd27 Best Practices for RAG Observability\n1. Comprehensive Pipeline Tracing\n2. Quality Monitoring\n3. A/B Testing for RAG Components\n\ud83c\udfaf Advanced RAG Patterns\nMulti-Modal RAG\nConversational RAG\nRAG observability with Noveum.ai provides the deep insights needed to build, optimize, and scale retrieval-augmented generation systems. By implementing comprehensive tracing across retrieval, context preparation, and generation stages, you can ensure your RAG pipeline delivers accurate, relevant, and cost-effe...\n\nUser Question: What are the current trends in observability tools?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", + "prompt.prompt_length": 3490, + "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/blog/from-logs-to-intelligent-choices-inside-noveum-ais-evaluation-process):\nFrom Development to Production - Inside Noveum.ai's AI Observability Platform\nShashank Agarw...", + "prompt.instruction_following": "rag_optimized", + "response.answer_length": 815, + "response.answer_completeness": "complete", + "response.response_quality": "high", + "response.source_citation": 5, + "response.context_utilization": 0.2769283044512402, + "output_response": "RAG Answer: The provided context does not explicitly outline current trends in observability tools. However, it highlights the unique challenges faced by AI applications that traditional monitoring tools cannot a...", + "evaluation.retrieval_effectiveness": 1, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 5, + "evaluation.context_relevance": "high", + "evaluation.overall_quality": "high", + "evaluation.ready_for_production": true, + "rag.retrieval_strategy": "semantic_similarity", + "rag.vector_search_results": 5, + "rag.context_synthesis": "multi_source", + "rag.document_coverage": 1, + "rag.information_density": 0.2769283044512402 + }, + "events": [], + "links": [], + "trace_trace_id": "45efae74-ee25-4004-a509-6e0ff628a0cd", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:58:04.883000000", + "trace_end_time": "2025-10-22 16:58:10.029000000", + "trace_duration_ms": 5146.244000000001, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:58:04.883000000", + "updated_at": "2025-10-22 16:58:04.883000000" + }, + { + "span_id": "99a5e8f3-a9f5-4953-94ea-ad0fa0cf6ec2", + "trace_id": "c5a64d08-d00f-46cf-ab3c-d8d473c10eca", + "parent_span_id": "2a351fd9-a7ee-42da-9e2e-86a643ceed0e", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-22 16:57:57.908000000", + "end_time": "2025-10-22 16:57:57.908000000", + "duration_ms": 0.026, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: Tell me about recent developments in machine learning", + "query_length": 53, + "query_type": "routing_evaluation_query", + "classification.mode": "Web Search", + "classification.rag_keyword_score": 0, + "classification.web_keyword_score": 1, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "complex", + "query.intent": "general_knowledge", + "query.keyword_density": 0.125, + "query.domain_affinity": "general", + "routing.decision": "Routed to Web Search based on analysis", + "routing.rationale": "RAG score: 0, Web score: 1, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "RAG", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.019230769230769232, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: Web Search (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "c5a64d08-d00f-46cf-ab3c-d8d473c10eca", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:57.908000000", + "trace_end_time": "2025-10-22 16:58:03.875000000", + "trace_duration_ms": 5967.0779999999995, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:57.908000000", + "updated_at": "2025-10-22 16:57:57.908000000" + }, + { + "span_id": "6e891e51-ca6d-4e6c-80d0-0791ee22d5b9", + "trace_id": "c5a64d08-d00f-46cf-ab3c-d8d473c10eca", + "parent_span_id": "2a351fd9-a7ee-42da-9e2e-86a643ceed0e", + "name": "agent.web_search_generation", + "start_time": "2025-10-22 16:57:57.908000000", + "end_time": "2025-10-22 16:58:03.874000000", + "duration_ms": 5966.728, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "web_search_agent", + "agent.operation": "web_search_generation", + "agent.capabilities": [ + "web_search", + "content_synthesis", + "response_generation" + ], + "agent.id": "noveum_web_search_agent", + "input_query": "Tell me about recent developments in machine learning", + "query_length": 53, + "query_type": "web_search_query", + "output_response": "Web Search Answer: The web search results did not provide specific information about recent developments in machine learning. Therefore, I cannot summarize any recent advancements based on the provided data.\n\nHowever, i...", + "web_search.results_count": 1, + "web_search.context_length": 768, + "web_search.response_length": 687, + "web_search.mode": "external_web_search" + }, + "events": [], + "links": [], + "trace_trace_id": "c5a64d08-d00f-46cf-ab3c-d8d473c10eca", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:57.908000000", + "trace_end_time": "2025-10-22 16:58:03.875000000", + "trace_duration_ms": 5967.0779999999995, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:57.908000000", + "updated_at": "2025-10-22 16:57:57.908000000" + }, + { + "span_id": "6d622376-d48b-4a64-b0a2-1402117febaf", + "trace_id": "c5a64d08-d00f-46cf-ab3c-d8d473c10eca", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-22 16:57:57.908000000", + "end_time": "2025-10-22 16:58:03.875000000", + "duration_ms": 5967.035, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "Tell me about recent developments in machine learning", + "query.length": 53, + "processing.mode": "Web Search", + "processing.time_seconds": 5.966889142990112, + "processing.response_length": 687, + "processing.sources_count": 1, + "output_response": "Final Answer: The web search results did not provide specific information about recent developments in machine learning. Therefore, I cannot summarize any recent advancements based on the provided data.\n\nHowever, i...", + "final_answer_mode": "Web Search", + "query_processed.input_query": "Tell me about recent developments in machine learning", + "query_processed.output_response": "Successfully processed query using Web Search, generated 687 character response", + "query_processed.mode": "Web Search", + "query_processed.processing_time": 5.966889142990112, + "query_processed.response_length": 687 + }, + "events": [], + "links": [], + "trace_trace_id": "c5a64d08-d00f-46cf-ab3c-d8d473c10eca", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:57.908000000", + "trace_end_time": "2025-10-22 16:58:03.875000000", + "trace_duration_ms": 5967.0779999999995, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:57.908000000", + "updated_at": "2025-10-22 16:57:57.908000000" + }, + { + "span_id": "2a351fd9-a7ee-42da-9e2e-86a643ceed0e", + "trace_id": "c5a64d08-d00f-46cf-ab3c-d8d473c10eca", + "parent_span_id": "6d622376-d48b-4a64-b0a2-1402117febaf", + "name": "agent.query_routing", + "start_time": "2025-10-22 16:57:57.908000000", + "end_time": "2025-10-22 16:58:03.875000000", + "duration_ms": 5966.8640000000005, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "Tell me about recent developments in machine learning", + "query_length": 53, + "query_type": "routing_query", + "output_response": "Routed to Web Search for query processing", + "router.classification": "Web Search", + "router.confidence_score": 1, + "router.rag_keyword_score": 0, + "router.web_keyword_score": 1, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "c5a64d08-d00f-46cf-ab3c-d8d473c10eca", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:57.908000000", + "trace_end_time": "2025-10-22 16:58:03.875000000", + "trace_duration_ms": 5967.0779999999995, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:57.908000000", + "updated_at": "2025-10-22 16:57:57.908000000" + }, + { + "span_id": "ba457b3b-9be8-4e24-af63-4118833711e6", + "trace_id": "c5a64d08-d00f-46cf-ab3c-d8d473c10eca", + "parent_span_id": "6e891e51-ca6d-4e6c-80d0-0791ee22d5b9", + "name": "agent.llm_model_execution", + "start_time": "2025-10-22 16:57:58.324000000", + "end_time": "2025-10-22 16:58:03.874000000", + "duration_ms": 5550.059, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: Tell me about recent developments in machine learning", + "query_length": 53, + "query_type": "web_search_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 5.550010919570923, + "model.latency_ms": 5550.010919570923, + "model.start_time": 1761152278.3248482, + "model.end_time": 1761152283.874859, + "model.performance_tier": "slow", + "model.prompt_tokens": 316, + "model.completion_tokens": 171, + "model.total_tokens": 487, + "model.tokens_per_second": 87.74757510524871, + "model.estimated_cost": 0.01461, + "model.efficiency_score": 1.4106776180698153, + "model.response_length": 687, + "model.response_quality": "high", + "model.output_response": "Model Response: The web search results did not provide specific information about recent developments in machine learning. Therefore, I cannot summarize any recent advancements based on the provided data.\n\nHowever, i...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "c5a64d08-d00f-46cf-ab3c-d8d473c10eca", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:57.908000000", + "trace_end_time": "2025-10-22 16:58:03.875000000", + "trace_duration_ms": 5967.0779999999995, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:57.908000000", + "updated_at": "2025-10-22 16:57:57.908000000" + }, + { + "span_id": "7b70a916-1c2b-4898-a052-cfd98a979167", + "trace_id": "c5a64d08-d00f-46cf-ab3c-d8d473c10eca", + "parent_span_id": "6e891e51-ca6d-4e6c-80d0-0791ee22d5b9", + "name": "agent.web_search_evaluation_metrics", + "start_time": "2025-10-22 16:58:03.874000000", + "end_time": "2025-10-22 16:58:03.874000000", + "duration_ms": 0.02, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "web_search_evaluation_metrics", + "agent.capabilities": [ + "web_search_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: Tell me about recent developments in machine learning", + "query_length": 53, + "query_type": "web_search_evaluation_query", + "web_search.results_count": 1, + "web_search.context_length": 768, + "web_search.context_synthesized": "Context from 1 web sources", + "web_search.search_effectiveness": 0.2, + "web_search.context_quality": "medium", + "web_search.source_diversity": 1, + "web_search.information_synthesis": "low", + "web_search.external_knowledge_utilization": 0.768, + "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nFeb 3, 2024 \u00b7 tell sb sth\u4e2d\u7684tell\u66f4\u591a\u662f\u544a\u77e5\u7684\u610f\u601d\uff0c\u5c31\u662f\u544a\u8bc9\u522b\u4eba\u6709\u8fd9\u4e48\u4e00\u4e2a\u4e8b\u7684\u5b58\u5728 tell sb about sth \u7684\u610f\u601d\u5305\u542b\u66f4\u591a\u4e00\u4e9b\uff0c\u9664\u4e86\u544a\u77e5\u6709\u8fd9\u4e48\u4e00\u4e2a\u4e8b\u60c5\u7684\u5b58\u5728\uff0c\u8fd8\u4f5c\u4e86\u4e00\u4e9b\u8bf4\u660e \u6bd4\u5982\uff1a tell me he \u2026 \u4e00\u3001Say\u548cTell\u7684\u7528\u6cd5 \u6211\u4eec\u5148\u5c06say\u3001tell\u3001speak\u3001talk\u8fd9\u56db\u4e2a\u8bcd\u5206\u6210\u4e24\u7ec4\uff0c\u5206\u522b\u4e3a\uff1a \u7b2c\u2460\u7ec4\uff1asay & tell\uff0c\u8fd9\u4e24\u4e2a\u8bcd\u4e4b\u540e\u53ef\u4ee5\u8868\u793a\u8c08\u8bdd\u7684 \u201c\u7ec6\u8282\u201d\u3002 \u7b2c\u2461\u7ec4\uff1aspeak & talk\uff0c\u4e0d\u80fd\u8868\u793a\u8c08\u8bdd\u7684\u201c\u7ec6\u8282\u201d\u3002 \u2026 tell of 1\u3001Tell of a turning point in your career and what you learned. \u8bb2\u8ff0\u4f60 \u804c\u4e1a\u751f\u6daf \u4e00\u4e2a\u8f6c\u6298\u70b9\uff0c\u4ee5\u53ca\u4f60\u5b66\u5230\u4e86\u4ec0\u4e48\u3002 2\u3001I have made the Sovereign Lord my refuge; I will tell of all your \u2026 tell sb to do sth \u548c tell sb do sth \u90fd\u8868\u793a\u544a\u8bc9\u67d0\u4eba\u505a\u67d0\u4e8b\uff0ctell sb to do sth \u5f3a\u8c03\u544a\u8bc9\u67d0\u4eba\u5e94\u8be5\u505a\u4ec0\u4e48\u5177\u4f53\u52a8\u4f5c\u6216\u884c\u4e3a\uff0c\u800c tell sb do sth \u5f3a\u8c03\u544a\u8bc9\u67d0\u4eba\u6b63\u5728\u8fdb\u884c\u7684\u5177\u4f53\u884c\u52a8\u6216\u72b6\u6001\u3002 \u5b83\u4eec\u5728\u91ca\u4e49\u3001 \u2026 Aug 23, 2023 \u00b7 tell\u7684\u8fc7\u53bb\u5f0f\u548c\u8fc7\u53bb\u5206\u8bcd\u90fd\u662ftold\u3002 \u8bfb\u97f3\uff1a\u82f1[tel]\uff0c\u7f8e[tel]\u3002 \u91ca\u4e49\uff1a vt.\u544a\u8bc9\uff0c\u8bf4\uff1b\u8fa8\u522b\uff1b\u5429\u5490\uff1b\u65ad\u5b9a\u3002 vi.\u8bb2\u8ff0\uff1b\u544a\u53d1\uff0c\u6cc4\u5bc6\uff1b\u8bc6\u522b\u3002 \u53d8\u5f62\uff1a\u8fc7\u53bb\u5f0ftold\uff0c\u8fc7\u53bb\u5206\u8bcdtold\uff0c\u73b0\u5728\u5206 \u2026\nURL: https://duckduckgo.com/?q=Tell+me+about+recent+developments+in+machine+learning\n\nUser Question: Tell me about recent developments in machine learning\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", + "prompt.prompt_length": 1266, + "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nFeb 3, 2024 \u00b7 tell sb sth\u4e2d\u7684tell\u66f4\u591a\u662f\u544a\u77e5\u7684\u610f\u601d\uff0c\u5c31\u662f\u544a\u8bc9\u522b\u4eba\u6709\u8fd9\u4e48\u4e00\u4e2a\u4e8b\u7684\u5b58\u5728 tell sb about sth \u7684\u610f\u601d\u5305\u542b\u66f4\u591a\u4e00\u4e9b\uff0c\u9664\u4e86\u544a\u77e5\u6709\u8fd9\u4e48\u4e00\u4e2a\u4e8b\u60c5\u7684\u5b58\u5728\uff0c\u8fd8\u4f5c\u4e86\u4e00\u4e9b\u8bf4\u660e \u6bd4\u5982\uff1a tell me he \u2026 \u4e00\u3001Say\u548cTell\u7684\u7528\u6cd5 \u6211\u4eec\u5148\u5c06say\u3001tell\u3001speak\u3001talk\u8fd9\u56db\u4e2a\u8bcd\u5206\u6210\u4e24\u7ec4\uff0c\u5206\u522b...", + "prompt.instruction_following": "web_search_optimized", + "response.answer_length": 687, + "response.answer_completeness": "complete", + "response.response_quality": "medium", + "response.source_citation": 1, + "response.context_utilization": 0.89453125, + "output_response": "Web Search Answer: The web search results did not provide specific information about recent developments in machine learning. Therefore, I cannot summarize any recent advancements based on the provided data.\n\nHowever, i...", + "evaluation.search_effectiveness": 0.2, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 1, + "evaluation.context_relevance": "medium", + "evaluation.overall_quality": "low", + "evaluation.ready_for_production": true, + "web_search.search_strategy": "duckduckgo_api", + "web_search.real_time_data": true, + "web_search.external_sources": 1, + "web_search.information_freshness": "current", + "web_search.knowledge_synthesis": "single_source" + }, + "events": [], + "links": [], + "trace_trace_id": "c5a64d08-d00f-46cf-ab3c-d8d473c10eca", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:57.908000000", + "trace_end_time": "2025-10-22 16:58:03.875000000", + "trace_duration_ms": 5967.0779999999995, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:57.908000000", + "updated_at": "2025-10-22 16:57:57.908000000" + }, + { + "span_id": "1e1a341f-66c1-48b0-95eb-fb092928c473", + "trace_id": "b1b13e60-5b32-44a0-9769-197a944989d0", + "parent_span_id": "1c5c4291-bbb5-4f85-b6d9-b6e8df66f742", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-22 16:57:51.967000000", + "end_time": "2025-10-22 16:57:51.967000000", + "duration_ms": 0.033, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What's the weather like today?", + "query_length": 30, + "query_type": "routing_evaluation_query", + "classification.mode": "Web Search", + "classification.rag_keyword_score": 0, + "classification.web_keyword_score": 2, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "general_knowledge", + "query.keyword_density": 0.4, + "query.domain_affinity": "general", + "routing.decision": "Routed to Web Search based on analysis", + "routing.rationale": "RAG score: 0, Web score: 2, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "RAG", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.038461538461538464, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: Web Search (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "b1b13e60-5b32-44a0-9769-197a944989d0", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:51.967000000", + "trace_end_time": "2025-10-22 16:57:56.899000000", + "trace_duration_ms": 4932.393, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:51.967000000", + "updated_at": "2025-10-22 16:57:51.967000000" + }, + { + "span_id": "1c5c4291-bbb5-4f85-b6d9-b6e8df66f742", + "trace_id": "b1b13e60-5b32-44a0-9769-197a944989d0", + "parent_span_id": "49e7cace-ba21-4807-a238-9a3880487583", + "name": "agent.query_routing", + "start_time": "2025-10-22 16:57:51.967000000", + "end_time": "2025-10-22 16:57:56.899000000", + "duration_ms": 4931.972, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What's the weather like today?", + "query_length": 30, + "query_type": "routing_query", + "output_response": "Routed to Web Search for query processing", + "router.classification": "Web Search", + "router.confidence_score": 1, + "router.rag_keyword_score": 0, + "router.web_keyword_score": 2, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "b1b13e60-5b32-44a0-9769-197a944989d0", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:51.967000000", + "trace_end_time": "2025-10-22 16:57:56.899000000", + "trace_duration_ms": 4932.393, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:51.967000000", + "updated_at": "2025-10-22 16:57:51.967000000" + }, + { + "span_id": "49e7cace-ba21-4807-a238-9a3880487583", + "trace_id": "b1b13e60-5b32-44a0-9769-197a944989d0", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-22 16:57:51.967000000", + "end_time": "2025-10-22 16:57:56.899000000", + "duration_ms": 4932.316, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What's the weather like today?", + "query.length": 30, + "processing.mode": "Web Search", + "processing.time_seconds": 4.932007789611816, + "processing.response_length": 260, + "processing.sources_count": 1, + "output_response": "Final Answer: The web search results do not provide specific information about today's weather. To find out the current weather conditions, I recommend checking a reliable weather website or app for detailed update...", + "final_answer_mode": "Web Search", + "query_processed.input_query": "What's the weather like today?", + "query_processed.output_response": "Successfully processed query using Web Search, generated 260 character response", + "query_processed.mode": "Web Search", + "query_processed.processing_time": 4.932007789611816, + "query_processed.response_length": 260 + }, + "events": [], + "links": [], + "trace_trace_id": "b1b13e60-5b32-44a0-9769-197a944989d0", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:51.967000000", + "trace_end_time": "2025-10-22 16:57:56.899000000", + "trace_duration_ms": 4932.393, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:51.967000000", + "updated_at": "2025-10-22 16:57:51.967000000" + }, + { + "span_id": "58d217aa-4350-43e8-8813-468149dc4107", + "trace_id": "b1b13e60-5b32-44a0-9769-197a944989d0", + "parent_span_id": "1c5c4291-bbb5-4f85-b6d9-b6e8df66f742", + "name": "agent.web_search_generation", + "start_time": "2025-10-22 16:57:51.967000000", + "end_time": "2025-10-22 16:57:56.899000000", + "duration_ms": 4931.796, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "web_search_agent", + "agent.operation": "web_search_generation", + "agent.capabilities": [ + "web_search", + "content_synthesis", + "response_generation" + ], + "agent.id": "noveum_web_search_agent", + "input_query": "What's the weather like today?", + "query_length": 30, + "query_type": "web_search_query", + "output_response": "Web Search Answer: The web search results do not provide specific information about today's weather. To find out the current weather conditions, I recommend checking a reliable weather website or app for detailed update...", + "web_search.results_count": 1, + "web_search.context_length": 995, + "web_search.response_length": 260, + "web_search.mode": "external_web_search" + }, + "events": [], + "links": [], + "trace_trace_id": "b1b13e60-5b32-44a0-9769-197a944989d0", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:51.967000000", + "trace_end_time": "2025-10-22 16:57:56.899000000", + "trace_duration_ms": 4932.393, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:51.967000000", + "updated_at": "2025-10-22 16:57:51.967000000" + }, + { + "span_id": "d89dd84e-0b5e-4c92-942b-18c4735713d4", + "trace_id": "b1b13e60-5b32-44a0-9769-197a944989d0", + "parent_span_id": "58d217aa-4350-43e8-8813-468149dc4107", + "name": "agent.llm_model_execution", + "start_time": "2025-10-22 16:57:55.269000000", + "end_time": "2025-10-22 16:57:56.899000000", + "duration_ms": 1629.7140000000002, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What's the weather like today?", + "query_length": 30, + "query_type": "web_search_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 1.629619836807251, + "model.latency_ms": 1629.619836807251, + "model.start_time": 1761152275.269459, + "model.end_time": 1761152276.8990788, + "model.performance_tier": "fast", + "model.prompt_tokens": 367, + "model.completion_tokens": 65, + "model.total_tokens": 432, + "model.tokens_per_second": 265.0925020932329, + "model.estimated_cost": 0.012960000000000001, + "model.efficiency_score": 0.6018518518518519, + "model.response_length": 260, + "model.response_quality": "high", + "model.output_response": "Model Response: The web search results do not provide specific information about today's weather. To find out the current weather conditions, I recommend checking a reliable weather website or app for detailed update...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "b1b13e60-5b32-44a0-9769-197a944989d0", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:51.967000000", + "trace_end_time": "2025-10-22 16:57:56.899000000", + "trace_duration_ms": 4932.393, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:51.967000000", + "updated_at": "2025-10-22 16:57:51.967000000" + }, + { + "span_id": "d076719d-da79-43fb-92ea-6015f856146b", + "trace_id": "b1b13e60-5b32-44a0-9769-197a944989d0", + "parent_span_id": "58d217aa-4350-43e8-8813-468149dc4107", + "name": "agent.web_search_evaluation_metrics", + "start_time": "2025-10-22 16:57:56.899000000", + "end_time": "2025-10-22 16:57:56.899000000", + "duration_ms": 0.049, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "web_search_evaluation_metrics", + "agent.capabilities": [ + "web_search_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What's the weather like today?", + "query_length": 30, + "query_type": "web_search_evaluation_query", + "web_search.results_count": 1, + "web_search.context_length": 995, + "web_search.context_synthesized": "Context from 1 web sources", + "web_search.search_effectiveness": 0.2, + "web_search.context_quality": "high", + "web_search.source_diversity": 1, + "web_search.information_synthesis": "low", + "web_search.external_knowledge_utilization": 0.995, + "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nLog in to WhatsApp Web for simple, reliable and private messaging on your desktop. Send and receive messages and files with ease, all for free. 2 days ago \u00b7 More videos you may like 00:13 Wanna be yours #kimpolin #colinbridgerton #benedictbri... 2 hours ago \u00b7 101 views 00:14 Viscount & Vicountess's Marital bliss #bridgerton \u2026 Nov 15, 2024 \u00b7 Learn the correct usage of \"what's the weather like today\" and \"how's the weather today\" in English. Discover differences, examples, alternatives and tips for choosing the right \u2026 Nov 15, 2024 \u00b7 Learn the correct usage of \"how's the weather today\" and \"what's the weather today\" in English. Discover differences, examples, alternatives and tips for choosing the right \u2026 2 days ago \u00b7 Get the latest hourly weather updates for Thane today. Detailed forecast including temperature, wind, rain, snow, and UV index. Stay informed about today's weather conditions in \u2026\nURL: https://duckduckgo.com/?q=What's+the+weather+like+today?\n\nUser Question: What's the weather like today?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", + "prompt.prompt_length": 1470, + "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nLog in to WhatsApp Web for simple, reliable and private messaging on your desktop. Send and receive messages and files with ease, all for free. 2 days ago \u00b7 More videos you...", + "prompt.instruction_following": "web_search_optimized", + "response.answer_length": 260, + "response.answer_completeness": "complete", + "response.response_quality": "medium", + "response.source_citation": 1, + "response.context_utilization": 0.2613065326633166, + "output_response": "Web Search Answer: The web search results do not provide specific information about today's weather. To find out the current weather conditions, I recommend checking a reliable weather website or app for detailed update...", + "evaluation.search_effectiveness": 0.2, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 1, + "evaluation.context_relevance": "high", + "evaluation.overall_quality": "low", + "evaluation.ready_for_production": true, + "web_search.search_strategy": "duckduckgo_api", + "web_search.real_time_data": true, + "web_search.external_sources": 1, + "web_search.information_freshness": "current", + "web_search.knowledge_synthesis": "single_source" + }, + "events": [], + "links": [], + "trace_trace_id": "b1b13e60-5b32-44a0-9769-197a944989d0", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:51.967000000", + "trace_end_time": "2025-10-22 16:57:56.899000000", + "trace_duration_ms": 4932.393, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:51.967000000", + "updated_at": "2025-10-22 16:57:51.967000000" + }, + { + "span_id": "41167ca4-e1ad-48a5-b5eb-aa272ad88a85", + "trace_id": "e4f3cc20-c137-4905-80fe-2d29b03c2231", + "parent_span_id": "56959f47-5219-4674-8c0d-e5fa928dcd45", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-22 16:57:47.477000000", + "end_time": "2025-10-22 16:57:47.477000000", + "duration_ms": 0.061, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What are the latest AI news today?", + "query_length": 34, + "query_type": "routing_evaluation_query", + "classification.mode": "Web Search", + "classification.rag_keyword_score": 0, + "classification.web_keyword_score": 3, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "general_knowledge", + "query.keyword_density": 0.42857142857142855, + "query.domain_affinity": "general", + "routing.decision": "Routed to Web Search based on analysis", + "routing.rationale": "RAG score: 0, Web score: 3, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "RAG", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.057692307692307696, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: Web Search (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "e4f3cc20-c137-4905-80fe-2d29b03c2231", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:47.477000000", + "trace_end_time": "2025-10-22 16:57:50.959000000", + "trace_duration_ms": 3481.856, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:47.477000000", + "updated_at": "2025-10-22 16:57:47.477000000" + }, + { + "span_id": "73005751-9eb1-4540-aa49-2ac580cd7418", + "trace_id": "e4f3cc20-c137-4905-80fe-2d29b03c2231", + "parent_span_id": "56959f47-5219-4674-8c0d-e5fa928dcd45", + "name": "agent.web_search_generation", + "start_time": "2025-10-22 16:57:47.477000000", + "end_time": "2025-10-22 16:57:50.958000000", + "duration_ms": 3480.8469999999998, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "web_search_agent", + "agent.operation": "web_search_generation", + "agent.capabilities": [ + "web_search", + "content_synthesis", + "response_generation" + ], + "agent.id": "noveum_web_search_agent", + "input_query": "What are the latest AI news today?", + "query_length": 34, + "query_type": "web_search_query", + "output_response": "Web Search Answer: The provided web search results do not contain specific information about the latest AI news today. They primarily focus on general news coverage from India and around the world, including politics, b...", + "web_search.results_count": 1, + "web_search.context_length": 887, + "web_search.response_length": 503, + "web_search.mode": "external_web_search" + }, + "events": [], + "links": [], + "trace_trace_id": "e4f3cc20-c137-4905-80fe-2d29b03c2231", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:47.477000000", + "trace_end_time": "2025-10-22 16:57:50.959000000", + "trace_duration_ms": 3481.856, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:47.477000000", + "updated_at": "2025-10-22 16:57:47.477000000" + }, + { + "span_id": "9e1b87be-bc29-4084-b37c-6e1220dde8eb", + "trace_id": "e4f3cc20-c137-4905-80fe-2d29b03c2231", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-22 16:57:47.477000000", + "end_time": "2025-10-22 16:57:50.959000000", + "duration_ms": 3481.7580000000003, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What are the latest AI news today?", + "query.length": 34, + "processing.mode": "Web Search", + "processing.time_seconds": 3.4813451766967773, + "processing.response_length": 503, + "processing.sources_count": 1, + "output_response": "Final Answer: The provided web search results do not contain specific information about the latest AI news today. They primarily focus on general news coverage from India and around the world, including politics, b...", + "final_answer_mode": "Web Search", + "query_processed.input_query": "What are the latest AI news today?", + "query_processed.output_response": "Successfully processed query using Web Search, generated 503 character response", + "query_processed.mode": "Web Search", + "query_processed.processing_time": 3.4813451766967773, + "query_processed.response_length": 503 + }, + "events": [], + "links": [], + "trace_trace_id": "e4f3cc20-c137-4905-80fe-2d29b03c2231", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:47.477000000", + "trace_end_time": "2025-10-22 16:57:50.959000000", + "trace_duration_ms": 3481.856, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:47.477000000", + "updated_at": "2025-10-22 16:57:47.477000000" + }, + { + "span_id": "56959f47-5219-4674-8c0d-e5fa928dcd45", + "trace_id": "e4f3cc20-c137-4905-80fe-2d29b03c2231", + "parent_span_id": "9e1b87be-bc29-4084-b37c-6e1220dde8eb", + "name": "agent.query_routing", + "start_time": "2025-10-22 16:57:47.477000000", + "end_time": "2025-10-22 16:57:50.958000000", + "duration_ms": 3481.254, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What are the latest AI news today?", + "query_length": 34, + "query_type": "routing_query", + "output_response": "Routed to Web Search for query processing", + "router.classification": "Web Search", + "router.confidence_score": 1, + "router.rag_keyword_score": 0, + "router.web_keyword_score": 3, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "e4f3cc20-c137-4905-80fe-2d29b03c2231", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:47.477000000", + "trace_end_time": "2025-10-22 16:57:50.959000000", + "trace_duration_ms": 3481.856, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:47.477000000", + "updated_at": "2025-10-22 16:57:47.477000000" + }, + { + "span_id": "3dbf95ba-3ef1-4294-93d4-217f4ada5ffe", + "trace_id": "e4f3cc20-c137-4905-80fe-2d29b03c2231", + "parent_span_id": "73005751-9eb1-4540-aa49-2ac580cd7418", + "name": "agent.llm_model_execution", + "start_time": "2025-10-22 16:57:47.864000000", + "end_time": "2025-10-22 16:57:50.958000000", + "duration_ms": 3093.442, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What are the latest AI news today?", + "query_length": 34, + "query_type": "web_search_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 3.0933361053466797, + "model.latency_ms": 3093.3361053466797, + "model.start_time": 1761152267.864976, + "model.end_time": 1761152270.958312, + "model.performance_tier": "medium", + "model.prompt_tokens": 341, + "model.completion_tokens": 125, + "model.total_tokens": 466, + "model.tokens_per_second": 150.6464167261171, + "model.estimated_cost": 0.013980000000000001, + "model.efficiency_score": 1.0793991416309012, + "model.response_length": 503, + "model.response_quality": "high", + "model.output_response": "Model Response: The provided web search results do not contain specific information about the latest AI news today. They primarily focus on general news coverage from India and around the world, including politics, b...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "e4f3cc20-c137-4905-80fe-2d29b03c2231", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:47.477000000", + "trace_end_time": "2025-10-22 16:57:50.959000000", + "trace_duration_ms": 3481.856, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:47.477000000", + "updated_at": "2025-10-22 16:57:47.477000000" + }, + { + "span_id": "f7fe795b-ce1b-4fcc-83f5-06a5f074f38d", + "trace_id": "e4f3cc20-c137-4905-80fe-2d29b03c2231", + "parent_span_id": "73005751-9eb1-4540-aa49-2ac580cd7418", + "name": "agent.web_search_evaluation_metrics", + "start_time": "2025-10-22 16:57:50.958000000", + "end_time": "2025-10-22 16:57:50.958000000", + "duration_ms": 0.026, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "web_search_evaluation_metrics", + "agent.capabilities": [ + "web_search_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What are the latest AI news today?", + "query_length": 34, + "query_type": "web_search_evaluation_query", + "web_search.results_count": 1, + "web_search.context_length": 887, + "web_search.context_synthesized": "Context from 1 web sources", + "web_search.search_effectiveness": 0.2, + "web_search.context_quality": "high", + "web_search.source_diversity": 1, + "web_search.information_synthesis": "low", + "web_search.external_knowledge_utilization": 0.887, + "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\n3 days ago \u00b7 Latest news headlines from India & around the world. Check out today\u2019s news coverage live with videos & photos on NDTV.com. 6 days ago \u00b7 India News | Latest India News | Read latest and breaking news from India. Today's top India news headlines, news on Indian politics, elections, government, business, \u2026 2 days ago \u00b7 Top News Stories of the day, Latest News Headlines, News Specials, Breaking News and Latest India News, World current affairs & Political News all around the clock at \u2026 NDTV.com provides latest news from India and around the world. Get breaking news alerts from India and follow today\u2019s live news updates in field of politics, business, technology, Bollywood ... 6 days ago \u00b7 Get Latest entertainment news from Bollywood, Hollywood and regional film industries.\nURL: https://duckduckgo.com/?q=What+are+the+latest+AI+news+today?\n\nUser Question: What are the latest AI news today?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", + "prompt.prompt_length": 1366, + "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\n3 days ago \u00b7 Latest news headlines from India & around the world. Check out today\u2019s news coverage live with videos & photos on NDTV.com. 6 days ago \u00b7 India News | Latest In...", + "prompt.instruction_following": "web_search_optimized", + "response.answer_length": 503, + "response.answer_completeness": "complete", + "response.response_quality": "medium", + "response.source_citation": 1, + "response.context_utilization": 0.5670800450958287, + "output_response": "Web Search Answer: The provided web search results do not contain specific information about the latest AI news today. They primarily focus on general news coverage from India and around the world, including politics, b...", + "evaluation.search_effectiveness": 0.2, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 1, + "evaluation.context_relevance": "high", + "evaluation.overall_quality": "low", + "evaluation.ready_for_production": true, + "web_search.search_strategy": "duckduckgo_api", + "web_search.real_time_data": true, + "web_search.external_sources": 1, + "web_search.information_freshness": "current", + "web_search.knowledge_synthesis": "single_source" + }, + "events": [], + "links": [], + "trace_trace_id": "e4f3cc20-c137-4905-80fe-2d29b03c2231", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:47.477000000", + "trace_end_time": "2025-10-22 16:57:50.959000000", + "trace_duration_ms": 3481.856, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:47.477000000", + "updated_at": "2025-10-22 16:57:47.477000000" + }, + { + "span_id": "b9c9d21e-5f76-4e5a-b99e-2b4ab821caee", + "trace_id": "823b7927-10bb-49ef-a94e-1b914c298bfb", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-22 16:57:38.501000000", + "end_time": "2025-10-22 16:57:46.469000000", + "duration_ms": 7968.089, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What are the benefits of using Noveum Trace?", + "query.length": 44, + "processing.mode": "RAG", + "processing.time_seconds": 7.968040943145752, + "processing.response_length": 1396, + "processing.sources_count": 5, + "output_response": "Final Answer: Using Noveum Trace offers several benefits, including:\n\n1. **Real-Time Monitoring**: The Noveum platform provides a real-time dashboard for analyzing traces and performance, with live updates as new t...", + "final_answer_mode": "RAG", + "query_processed.input_query": "What are the benefits of using Noveum Trace?", + "query_processed.output_response": "Successfully processed query using RAG, generated 1396 character response", + "query_processed.mode": "RAG", + "query_processed.processing_time": 7.968040943145752, + "query_processed.response_length": 1396 + }, + "events": [], + "links": [], + "trace_trace_id": "823b7927-10bb-49ef-a94e-1b914c298bfb", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:38.501000000", + "trace_end_time": "2025-10-22 16:57:46.469000000", + "trace_duration_ms": 7968.149, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:38.501000000", + "updated_at": "2025-10-22 16:57:38.501000000" + }, + { + "span_id": "bc85c745-4c27-4aa4-bc00-4c9f10a2f049", + "trace_id": "823b7927-10bb-49ef-a94e-1b914c298bfb", + "parent_span_id": "b9c9d21e-5f76-4e5a-b99e-2b4ab821caee", + "name": "agent.query_routing", + "start_time": "2025-10-22 16:57:38.501000000", + "end_time": "2025-10-22 16:57:46.469000000", + "duration_ms": 7968.001, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What are the benefits of using Noveum Trace?", + "query_length": 44, + "query_type": "routing_query", + "output_response": "Routed to RAG for query processing", + "router.classification": "RAG", + "router.confidence_score": 1, + "router.rag_keyword_score": 2, + "router.web_keyword_score": 0, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "823b7927-10bb-49ef-a94e-1b914c298bfb", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:38.501000000", + "trace_end_time": "2025-10-22 16:57:46.469000000", + "trace_duration_ms": 7968.149, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:38.501000000", + "updated_at": "2025-10-22 16:57:38.501000000" + }, + { + "span_id": "3ab8cc4c-f5ed-410a-b4ee-48d0954b756d", + "trace_id": "823b7927-10bb-49ef-a94e-1b914c298bfb", + "parent_span_id": "bc85c745-4c27-4aa4-bc00-4c9f10a2f049", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-22 16:57:38.501000000", + "end_time": "2025-10-22 16:57:38.501000000", + "duration_ms": 0.09899999999999999, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What are the benefits of using Noveum Trace?", + "query_length": 44, + "query_type": "routing_evaluation_query", + "classification.mode": "RAG", + "classification.rag_keyword_score": 2, + "classification.web_keyword_score": 0, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "noveum_specific", + "query.keyword_density": 0.25, + "query.domain_affinity": "noveum", + "routing.decision": "Routed to RAG based on analysis", + "routing.rationale": "RAG score: 2, Web score: 0, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "Web Search", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.038461538461538464, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: RAG (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "823b7927-10bb-49ef-a94e-1b914c298bfb", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:38.501000000", + "trace_end_time": "2025-10-22 16:57:46.469000000", + "trace_duration_ms": 7968.149, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:38.501000000", + "updated_at": "2025-10-22 16:57:38.501000000" + }, + { + "span_id": "67292076-ec8a-4856-9c5b-726868552157", + "trace_id": "823b7927-10bb-49ef-a94e-1b914c298bfb", + "parent_span_id": "bc85c745-4c27-4aa4-bc00-4c9f10a2f049", + "name": "agent.llm-rag", + "start_time": "2025-10-22 16:57:38.501000000", + "end_time": "2025-10-22 16:57:46.469000000", + "duration_ms": 7967.679, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "rag_agent", + "agent.operation": "llm-rag", + "agent.capabilities": [ + "document_retrieval", + "context_generation", + "response_generation" + ], + "agent.id": "noveum_rag_agent", + "input_query": "What are the benefits of using Noveum Trace?", + "query_length": 44, + "query_type": "rag_query", + "output_response": "RAG Answer: Using Noveum Trace offers several benefits, including:\n\n1. **Real-Time Monitoring**: The Noveum platform provides a real-time dashboard for analyzing traces and performance, with live updates as new t...", + "rag.context_length": 2773, + "rag.sources_count": 5, + "rag.answer_length": 1396, + "rag.mode": "retrieval_augmented_generation" + }, + "events": [], + "links": [], + "trace_trace_id": "823b7927-10bb-49ef-a94e-1b914c298bfb", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:38.501000000", + "trace_end_time": "2025-10-22 16:57:46.469000000", + "trace_duration_ms": 7968.149, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:38.501000000", + "updated_at": "2025-10-22 16:57:38.501000000" + }, + { + "span_id": "d044ae4a-c463-4c1b-abce-a9a18f045e75", + "trace_id": "823b7927-10bb-49ef-a94e-1b914c298bfb", + "parent_span_id": "67292076-ec8a-4856-9c5b-726868552157", + "name": "agent.llm_model_execution", + "start_time": "2025-10-22 16:57:38.901000000", + "end_time": "2025-10-22 16:57:45.885000000", + "duration_ms": 6984.242, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What are the benefits of using Noveum Trace?", + "query_length": 44, + "query_type": "rag_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 6.984165906906128, + "model.latency_ms": 6984.165906906128, + "model.start_time": 1761152258.9012191, + "model.end_time": 1761152265.885385, + "model.performance_tier": "slow", + "model.prompt_tokens": 828, + "model.completion_tokens": 349, + "model.total_tokens": 1177, + "model.tokens_per_second": 168.524060809631, + "model.estimated_cost": 0.03531, + "model.efficiency_score": 1.1860662701784197, + "model.response_length": 1396, + "model.response_quality": "high", + "model.output_response": "Model Response: Using Noveum Trace offers several benefits, including:\n\n1. **Real-Time Monitoring**: The Noveum platform provides a real-time dashboard for analyzing traces and performance, with live updates as new t...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "823b7927-10bb-49ef-a94e-1b914c298bfb", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:38.501000000", + "trace_end_time": "2025-10-22 16:57:46.469000000", + "trace_duration_ms": 7968.149, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:38.501000000", + "updated_at": "2025-10-22 16:57:38.501000000" + }, + { + "span_id": "5b1548cf-0de7-49d4-b9b8-9ad1f9f2abc9", + "trace_id": "823b7927-10bb-49ef-a94e-1b914c298bfb", + "parent_span_id": "67292076-ec8a-4856-9c5b-726868552157", + "name": "agent.rag_evaluation_metrics", + "start_time": "2025-10-22 16:57:45.885000000", + "end_time": "2025-10-22 16:57:45.885000000", + "duration_ms": 0.061, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "rag_evaluation_metrics", + "agent.capabilities": [ + "retrieval_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What are the benefits of using Noveum Trace?", + "query_length": 44, + "query_type": "rag_evaluation_query", + "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/docs/platform/dashboard):\n- Bulk Export: Download multiple traces simultaneously\n- Comparative Analysis: Compare performance across multiple traces\nReal-Time Updates\n- Live Refresh: Automatic updates as new traces arrive\n- Connection Monitoring: Real-time status of your t...", + "retrieval.context_length": 2773, + "retrieval.sources_count": 5, + "retrieval.context_quality": "high", + "retrieval.effectiveness": 1, + "retrieval.context_utilization": 2.773, + "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/docs/platform/dashboard):\n- Bulk Export: Download multiple traces simultaneously\n- Comparative Analysis: Compare performance across multiple traces\nReal-Time Updates\n- Live Refresh: Automatic updates as new traces arrive\n- Connection Monitoring: Real-time status of your trace ingestion pipeline\n- Performance Indicators: Live metrics for system health monitoring\nReady to dive deeper? Explore Projects & Environments to organize your AI applications, or check out Team Collaboration to share insights with your team.\nGet Earl...\n\nSource 2 (https://noveum.ai/en/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\n\ud83d\udcd8 TypeScript SDK (\n@noveum/trace\n)- Framework integrations for Next.js, Express.js, and Hono\n- TypeScript-first design with full type safety\n- Universal compatibility (Node.js, Edge Runtime, browsers)\n-\n\ud83d\udcca Noveum Platform\n- Real-time dashboard for analyzing traces and performance\n- Ad...\n\nSource 3 (https://noveum.ai/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\n\ud83d\udcd8 TypeScript SDK (\n@noveum/trace\n)- Framework integrations for Next.js, Express.js, and Hono\n- TypeScript-first design with full type safety\n- Universal compatibility (Node.js, Edge Runtime, browsers)\n-\n\ud83d\udcca Noveum Platform\n- Real-time dashboard for analyzing traces and performance\n- Ad...\n\nSource 4 (https://noveum.ai/en/docs/getting-started/overview):\n- Advanced search & filtering across traces and spans\n- Cost analysis and optimization recommendations\n- Team collaboration with shared insights and alerts\n\ud83d\udd0d What Noveum Traces\nLLM Operations\n- Model calls across all providers (OpenAI, Anthropic, Google, etc.)\n- Token usage and cost calculation\n- Prompt engineering effectiveness\n- Response quality metrics\nRAG Pipelines\n- Document retrieval performance and relevance\n- Embedding generation costs and latency\n- Context assembly and prompt constructi...\n\nSource 5 (https://noveum.ai/docs/getting-started/overview):\n- Advanced search & filtering across traces and spans\n- Cost analysis and optimization recommendations\n- Team collaboration with shared insights and alerts\n\ud83d\udd0d What Noveum Traces\nLLM Operations\n- Model calls across all providers (OpenAI, Anthropic, Google, etc.)\n- Token usage and cost calculation\n- Prompt engineering effectiveness\n- Response quality metrics\nRAG Pipelines\n- Document retrieval performance and relevance\n- Embedding generation costs and latency\n- Context assembly and prompt constructi...\n\nUser Question: What are the benefits of using Noveum Trace?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", + "prompt.prompt_length": 3313, + "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/docs/platform/dashboard):\n- Bulk Export: Download multiple traces simultaneously\n- Comparative Analysis: Compare performance across multiple traces\nReal-Time Updates\n- Live...", + "prompt.instruction_following": "rag_optimized", + "response.answer_length": 1396, + "response.answer_completeness": "complete", + "response.response_quality": "high", + "response.source_citation": 5, + "response.context_utilization": 0.5034258925351605, + "output_response": "RAG Answer: Using Noveum Trace offers several benefits, including:\n\n1. **Real-Time Monitoring**: The Noveum platform provides a real-time dashboard for analyzing traces and performance, with live updates as new t...", + "evaluation.retrieval_effectiveness": 1, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 5, + "evaluation.context_relevance": "high", + "evaluation.overall_quality": "high", + "evaluation.ready_for_production": true, + "rag.retrieval_strategy": "semantic_similarity", + "rag.vector_search_results": 5, + "rag.context_synthesis": "multi_source", + "rag.document_coverage": 1, + "rag.information_density": 0.5034258925351605 + }, + "events": [], + "links": [], + "trace_trace_id": "823b7927-10bb-49ef-a94e-1b914c298bfb", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:38.501000000", + "trace_end_time": "2025-10-22 16:57:46.469000000", + "trace_duration_ms": 7968.149, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:38.501000000", + "updated_at": "2025-10-22 16:57:38.501000000" + }, + { + "span_id": "aaba719b-36ec-46ab-af2e-3ea8c18e9872", + "trace_id": "1ae0d6c4-d1fb-4239-b536-3333187d44aa", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-22 16:57:32.800000000", + "end_time": "2025-10-22 16:57:37.494000000", + "duration_ms": 4694.408, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "How do I configure Noveum for my system?", + "query.length": 40, + "processing.mode": "RAG", + "processing.time_seconds": 4.694374084472656, + "processing.response_length": 553, + "processing.sources_count": 5, + "output_response": "Final Answer: To configure Noveum for your system, you can refer to the SDK Integration Guide available in the Noveum documentation. This guide will help you trace your AI applications with minimal code changes. Th...", + "final_answer_mode": "RAG", + "query_processed.input_query": "How do I configure Noveum for my system?", + "query_processed.output_response": "Successfully processed query using RAG, generated 553 character response", + "query_processed.mode": "RAG", + "query_processed.processing_time": 4.694374084472656, + "query_processed.response_length": 553 + }, + "events": [], + "links": [], + "trace_trace_id": "1ae0d6c4-d1fb-4239-b536-3333187d44aa", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:32.800000000", + "trace_end_time": "2025-10-22 16:57:37.494000000", + "trace_duration_ms": 4694.468, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:32.800000000", + "updated_at": "2025-10-22 16:57:32.800000000" + }, + { + "span_id": "4254e6f9-10b8-42ba-939d-d7c124f80f9a", + "trace_id": "1ae0d6c4-d1fb-4239-b536-3333187d44aa", + "parent_span_id": "6d5d074e-8fd3-4d33-88fb-7f54b0959030", + "name": "agent.llm-rag", + "start_time": "2025-10-22 16:57:32.800000000", + "end_time": "2025-10-22 16:57:37.494000000", + "duration_ms": 4694.155000000001, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "rag_agent", + "agent.operation": "llm-rag", + "agent.capabilities": [ + "document_retrieval", + "context_generation", + "response_generation" + ], + "agent.id": "noveum_rag_agent", + "input_query": "How do I configure Noveum for my system?", + "query_length": 40, + "query_type": "rag_query", + "output_response": "RAG Answer: To configure Noveum for your system, you can refer to the SDK Integration Guide available in the Noveum documentation. This guide will help you trace your AI applications with minimal code changes. Th...", + "rag.context_length": 2237, + "rag.sources_count": 5, + "rag.answer_length": 553, + "rag.mode": "retrieval_augmented_generation" + }, + "events": [], + "links": [], + "trace_trace_id": "1ae0d6c4-d1fb-4239-b536-3333187d44aa", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:32.800000000", + "trace_end_time": "2025-10-22 16:57:37.494000000", + "trace_duration_ms": 4694.468, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:32.800000000", + "updated_at": "2025-10-22 16:57:32.800000000" + }, + { + "span_id": "6d5d074e-8fd3-4d33-88fb-7f54b0959030", + "trace_id": "1ae0d6c4-d1fb-4239-b536-3333187d44aa", + "parent_span_id": "aaba719b-36ec-46ab-af2e-3ea8c18e9872", + "name": "agent.query_routing", + "start_time": "2025-10-22 16:57:32.800000000", + "end_time": "2025-10-22 16:57:37.494000000", + "duration_ms": 4694.318, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "How do I configure Noveum for my system?", + "query_length": 40, + "query_type": "routing_query", + "output_response": "Routed to RAG for query processing", + "router.classification": "RAG", + "router.confidence_score": 1, + "router.rag_keyword_score": 2, + "router.web_keyword_score": 0, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "1ae0d6c4-d1fb-4239-b536-3333187d44aa", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:32.800000000", + "trace_end_time": "2025-10-22 16:57:37.494000000", + "trace_duration_ms": 4694.468, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:32.800000000", + "updated_at": "2025-10-22 16:57:32.800000000" + }, + { + "span_id": "2cc2ab0b-2572-4a75-88a3-18b3a1eaca78", + "trace_id": "1ae0d6c4-d1fb-4239-b536-3333187d44aa", + "parent_span_id": "6d5d074e-8fd3-4d33-88fb-7f54b0959030", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-22 16:57:32.800000000", + "end_time": "2025-10-22 16:57:32.800000000", + "duration_ms": 0.04, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: How do I configure Noveum for my system?", + "query_length": 40, + "query_type": "routing_evaluation_query", + "classification.mode": "RAG", + "classification.rag_keyword_score": 2, + "classification.web_keyword_score": 0, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "noveum_specific", + "query.keyword_density": 0.25, + "query.domain_affinity": "noveum", + "routing.decision": "Routed to RAG based on analysis", + "routing.rationale": "RAG score: 2, Web score: 0, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "Web Search", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.038461538461538464, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: RAG (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "1ae0d6c4-d1fb-4239-b536-3333187d44aa", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:32.800000000", + "trace_end_time": "2025-10-22 16:57:37.494000000", + "trace_duration_ms": 4694.468, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:32.800000000", + "updated_at": "2025-10-22 16:57:32.800000000" + }, + { + "span_id": "f1d9f779-3312-43c1-b549-00fc874429f0", + "trace_id": "1ae0d6c4-d1fb-4239-b536-3333187d44aa", + "parent_span_id": "4254e6f9-10b8-42ba-939d-d7c124f80f9a", + "name": "agent.llm_model_execution", + "start_time": "2025-10-22 16:57:33.281000000", + "end_time": "2025-10-22 16:57:37.060000000", + "duration_ms": 3778.743, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: How do I configure Noveum for my system?", + "query_length": 40, + "query_type": "rag_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 3.7786097526550293, + "model.latency_ms": 3778.6097526550293, + "model.start_time": 1761152253.28163, + "model.end_time": 1761152257.0602398, + "model.performance_tier": "medium", + "model.prompt_tokens": 693, + "model.completion_tokens": 138, + "model.total_tokens": 831, + "model.tokens_per_second": 219.92215507730066, + "model.estimated_cost": 0.02493, + "model.efficiency_score": 0.6654632972322503, + "model.response_length": 553, + "model.response_quality": "high", + "model.output_response": "Model Response: To configure Noveum for your system, you can refer to the SDK Integration Guide available in the Noveum documentation. This guide will help you trace your AI applications with minimal code changes. Th...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "1ae0d6c4-d1fb-4239-b536-3333187d44aa", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:32.800000000", + "trace_end_time": "2025-10-22 16:57:37.494000000", + "trace_duration_ms": 4694.468, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:32.800000000", + "updated_at": "2025-10-22 16:57:32.800000000" + }, + { + "span_id": "5500be90-0cfa-44dd-a6e3-e72ac9f0a79e", + "trace_id": "1ae0d6c4-d1fb-4239-b536-3333187d44aa", + "parent_span_id": "4254e6f9-10b8-42ba-939d-d7c124f80f9a", + "name": "agent.rag_evaluation_metrics", + "start_time": "2025-10-22 16:57:37.060000000", + "end_time": "2025-10-22 16:57:37.060000000", + "duration_ms": 0.076, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "rag_evaluation_metrics", + "agent.capabilities": [ + "retrieval_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: How do I configure Noveum for my system?", + "query_length": 40, + "query_type": "rag_evaluation_query", + "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/docs):\nWelcome to Noveum.ai Docs\nComprehensive AI tracing and observability for LLM applications, RAG systems, and AI agents\nWelcome to the Noveum.ai documentation! Here you'll find everything you need to integrate, configure, and optimize your AI applications using ou...", + "retrieval.context_length": 2237, + "retrieval.sources_count": 5, + "retrieval.context_quality": "high", + "retrieval.effectiveness": 1, + "retrieval.context_utilization": 2.237, + "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/docs):\nWelcome to Noveum.ai Docs\nComprehensive AI tracing and observability for LLM applications, RAG systems, and AI agents\nWelcome to the Noveum.ai documentation! Here you'll find everything you need to integrate, configure, and optimize your AI applications using our comprehensive tracing and observability platform.\nWe provide powerful SDKs for Python and TypeScript that enable you to trace LLM calls, RAG pipelines, and multi-agent workflows with minimal code changes. Our documentation is organized ...\n\nSource 2 (https://noveum.ai/docs):\nWelcome to Noveum.ai Docs\nComprehensive AI tracing and observability for LLM applications, RAG systems, and AI agents\nWelcome to the Noveum.ai documentation! Here you'll find everything you need to integrate, configure, and optimize your AI applications using our comprehensive tracing and observability platform.\nWe provide powerful SDKs for Python and TypeScript that enable you to trace LLM calls, RAG pipelines, and multi-agent workflows with minimal code changes. Our documentation is organized ...\n\nSource 3 (https://noveum.ai/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nSource 4 (https://noveum.ai/en/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nSource 5 (https://noveum.ai/en/docs):\n- \ud83c\udf0d Universal: Works across frameworks, clouds, and deployment models\n\ud83e\udd1d Community & Support\n- \ud83d\udcac Discord Community: Join our Discord\n- \ud83d\udce7 Email Support: [email protected]\n- \ud83d\udc1b Bug Reports: GitHub Issues\n- \ud83d\udcd6 Knowledge Base: Help Center\nReady to get started? Head to our SDK Integration Guide to begin tracing your AI applications in under 5 minutes!\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability ...\n\nUser Question: How do I configure Noveum for my system?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", + "prompt.prompt_length": 2773, + "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/docs):\nWelcome to Noveum.ai Docs\nComprehensive AI tracing and observability for LLM applications, RAG systems, and AI agents\nWelcome to the Noveum.ai documentation! Here...", + "prompt.instruction_following": "rag_optimized", + "response.answer_length": 553, + "response.answer_completeness": "complete", + "response.response_quality": "high", + "response.source_citation": 5, + "response.context_utilization": 0.24720607957085383, + "output_response": "RAG Answer: To configure Noveum for your system, you can refer to the SDK Integration Guide available in the Noveum documentation. This guide will help you trace your AI applications with minimal code changes. Th...", + "evaluation.retrieval_effectiveness": 1, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 5, + "evaluation.context_relevance": "high", + "evaluation.overall_quality": "high", + "evaluation.ready_for_production": true, + "rag.retrieval_strategy": "semantic_similarity", + "rag.vector_search_results": 5, + "rag.context_synthesis": "multi_source", + "rag.document_coverage": 1, + "rag.information_density": 0.24720607957085383 + }, + "events": [], + "links": [], + "trace_trace_id": "1ae0d6c4-d1fb-4239-b536-3333187d44aa", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:32.800000000", + "trace_end_time": "2025-10-22 16:57:37.494000000", + "trace_duration_ms": 4694.468, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:32.800000000", + "updated_at": "2025-10-22 16:57:32.800000000" + }, + { + "span_id": "3068013a-6844-4f3c-8ec2-854a7353591e", + "trace_id": "fb0d31fc-2465-4ecd-973c-4cd443d9befa", + "parent_span_id": "a80d8fab-d5dc-4417-a5e5-9a2a76ed7419", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-22 16:57:24.828000000", + "end_time": "2025-10-22 16:57:24.828000000", + "duration_ms": 0.038, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What monitoring capabilities does Noveum provide?", + "query_length": 49, + "query_type": "routing_evaluation_query", + "classification.mode": "RAG", + "classification.rag_keyword_score": 2, + "classification.web_keyword_score": 0, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "noveum_specific", + "query.keyword_density": 0.3333333333333333, + "query.domain_affinity": "noveum", + "routing.decision": "Routed to RAG based on analysis", + "routing.rationale": "RAG score: 2, Web score: 0, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "Web Search", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.038461538461538464, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: RAG (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "fb0d31fc-2465-4ecd-973c-4cd443d9befa", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:24.828000000", + "trace_end_time": "2025-10-22 16:57:31.793000000", + "trace_duration_ms": 6965.302000000001, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:24.828000000", + "updated_at": "2025-10-22 16:57:24.828000000" + }, + { + "span_id": "189b89e3-fbbd-41de-8521-efb6a68f62f2", + "trace_id": "fb0d31fc-2465-4ecd-973c-4cd443d9befa", + "parent_span_id": "a80d8fab-d5dc-4417-a5e5-9a2a76ed7419", + "name": "agent.llm-rag", + "start_time": "2025-10-22 16:57:24.828000000", + "end_time": "2025-10-22 16:57:31.793000000", + "duration_ms": 6964.955, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "rag_agent", + "agent.operation": "llm-rag", + "agent.capabilities": [ + "document_retrieval", + "context_generation", + "response_generation" + ], + "agent.id": "noveum_rag_agent", + "input_query": "What monitoring capabilities does Noveum provide?", + "query_length": 49, + "query_type": "rag_query", + "output_response": "RAG Answer: Noveum provides comprehensive monitoring capabilities for AI applications, including the ability to monitor, trace, and optimize AI agents across various frameworks such as LangChain, CrewAI, AutoGen,...", + "rag.context_length": 2740, + "rag.sources_count": 5, + "rag.answer_length": 582, + "rag.mode": "retrieval_augmented_generation" + }, + "events": [], + "links": [], + "trace_trace_id": "fb0d31fc-2465-4ecd-973c-4cd443d9befa", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:24.828000000", + "trace_end_time": "2025-10-22 16:57:31.793000000", + "trace_duration_ms": 6965.302000000001, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:24.828000000", + "updated_at": "2025-10-22 16:57:24.828000000" + }, + { + "span_id": "f5cffb7d-341b-4cde-b162-02208042463d", + "trace_id": "fb0d31fc-2465-4ecd-973c-4cd443d9befa", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-22 16:57:24.828000000", + "end_time": "2025-10-22 16:57:31.793000000", + "duration_ms": 6965.236, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What monitoring capabilities does Noveum provide?", + "query.length": 49, + "processing.mode": "RAG", + "processing.time_seconds": 6.965206146240234, + "processing.response_length": 582, + "processing.sources_count": 5, + "output_response": "Final Answer: Noveum provides comprehensive monitoring capabilities for AI applications, including the ability to monitor, trace, and optimize AI agents across various frameworks such as LangChain, CrewAI, AutoGen,...", + "final_answer_mode": "RAG", + "query_processed.input_query": "What monitoring capabilities does Noveum provide?", + "query_processed.output_response": "Successfully processed query using RAG, generated 582 character response", + "query_processed.mode": "RAG", + "query_processed.processing_time": 6.965206146240234, + "query_processed.response_length": 582 + }, + "events": [], + "links": [], + "trace_trace_id": "fb0d31fc-2465-4ecd-973c-4cd443d9befa", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:24.828000000", + "trace_end_time": "2025-10-22 16:57:31.793000000", + "trace_duration_ms": 6965.302000000001, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:24.828000000", + "updated_at": "2025-10-22 16:57:24.828000000" + }, + { + "span_id": "a80d8fab-d5dc-4417-a5e5-9a2a76ed7419", + "trace_id": "fb0d31fc-2465-4ecd-973c-4cd443d9befa", + "parent_span_id": "f5cffb7d-341b-4cde-b162-02208042463d", + "name": "agent.query_routing", + "start_time": "2025-10-22 16:57:24.828000000", + "end_time": "2025-10-22 16:57:31.793000000", + "duration_ms": 6965.17, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What monitoring capabilities does Noveum provide?", + "query_length": 49, + "query_type": "routing_query", + "output_response": "Routed to RAG for query processing", + "router.classification": "RAG", + "router.confidence_score": 1, + "router.rag_keyword_score": 2, + "router.web_keyword_score": 0, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "fb0d31fc-2465-4ecd-973c-4cd443d9befa", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:24.828000000", + "trace_end_time": "2025-10-22 16:57:31.793000000", + "trace_duration_ms": 6965.302000000001, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:24.828000000", + "updated_at": "2025-10-22 16:57:24.828000000" + }, + { + "span_id": "a2b632b8-fbcb-480f-9292-9f513c64df38", + "trace_id": "fb0d31fc-2465-4ecd-973c-4cd443d9befa", + "parent_span_id": "189b89e3-fbbd-41de-8521-efb6a68f62f2", + "name": "agent.llm_model_execution", + "start_time": "2025-10-22 16:57:27.824000000", + "end_time": "2025-10-22 16:57:31.155000000", + "duration_ms": 3330.989, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What monitoring capabilities does Noveum provide?", + "query_length": 49, + "query_type": "rag_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 3.3309478759765625, + "model.latency_ms": 3330.9478759765625, + "model.start_time": 1761152247.824474, + "model.end_time": 1761152251.155422, + "model.performance_tier": "medium", + "model.prompt_tokens": 821, + "model.completion_tokens": 145, + "model.total_tokens": 966, + "model.tokens_per_second": 290.00754018607677, + "model.estimated_cost": 0.028980000000000002, + "model.efficiency_score": 0.6024844720496895, + "model.response_length": 582, + "model.response_quality": "high", + "model.output_response": "Model Response: Noveum provides comprehensive monitoring capabilities for AI applications, including the ability to monitor, trace, and optimize AI agents across various frameworks such as LangChain, CrewAI, AutoGen,...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "fb0d31fc-2465-4ecd-973c-4cd443d9befa", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:24.828000000", + "trace_end_time": "2025-10-22 16:57:31.793000000", + "trace_duration_ms": 6965.302000000001, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:24.828000000", + "updated_at": "2025-10-22 16:57:24.828000000" + }, + { + "span_id": "40ff54c4-07ca-4b2d-a9b4-7e3ae7adcd81", + "trace_id": "fb0d31fc-2465-4ecd-973c-4cd443d9befa", + "parent_span_id": "189b89e3-fbbd-41de-8521-efb6a68f62f2", + "name": "agent.rag_evaluation_metrics", + "start_time": "2025-10-22 16:57:31.155000000", + "end_time": "2025-10-22 16:57:31.155000000", + "duration_ms": 0.018000000000000002, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "rag_evaluation_metrics", + "agent.capabilities": [ + "retrieval_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What monitoring capabilities does Noveum provide?", + "query_length": 49, + "query_type": "rag_evaluation_query", + "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en):\nMonitor all your AI Agents\nimprove AI Agents today\nNoveum.ai helps you monitor, trace, and optimize your AI applications.\nNoveum.ai works with any AI framework \u2013 LangChain, CrewAI, AutoGen, custom implementations, or direct LLM calls. One dashboard shows everything.\n...", + "retrieval.context_length": 2740, + "retrieval.sources_count": 5, + "retrieval.context_quality": "high", + "retrieval.effectiveness": 1, + "retrieval.context_utilization": 2.74, + "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en):\nMonitor all your AI Agents\nimprove AI Agents today\nNoveum.ai helps you monitor, trace, and optimize your AI applications.\nNoveum.ai works with any AI framework \u2013 LangChain, CrewAI, AutoGen, custom implementations, or direct LLM calls. One dashboard shows everything.\nMonitor, Evaluate, Improve Your AI Agents\nThe control plane for AI agents.\nMonitor Everything, Miss Nothing\nOur lightweight SDKs capture every trace and span across your AI agent ecosystem\u2014from simple LLM calls to complex multi-agent...\n\nSource 2 (https://noveum.ai/):\nMonitor all your AI Agents\nimprove AI Agents today\nNoveum.ai helps you monitor, trace, and optimize your AI applications.\nNoveum.ai works with any AI framework \u2013 LangChain, CrewAI, AutoGen, custom implementations, or direct LLM calls. One dashboard shows everything.\nMonitor, Evaluate, Improve Your AI Agents\nThe control plane for AI agents.\nMonitor Everything, Miss Nothing\nOur lightweight SDKs capture every trace and span across your AI agent ecosystem\u2014from simple LLM calls to complex multi-agent...\n\nSource 3 (https://noveum.ai):\nMonitor all your AI Agents\nimprove AI Agents today\nNoveum.ai helps you monitor, trace, and optimize your AI applications.\nNoveum.ai works with any AI framework \u2013 LangChain, CrewAI, AutoGen, custom implementations, or direct LLM calls. One dashboard shows everything.\nMonitor, Evaluate, Improve Your AI Agents\nThe control plane for AI agents.\nMonitor Everything, Miss Nothing\nOur lightweight SDKs capture every trace and span across your AI agent ecosystem\u2014from simple LLM calls to complex multi-agent...\n\nSource 4 (https://noveum.ai/en/docs/getting-started/overview):\nNoveum.ai Overview\nComprehensive AI tracing and observability platform for LLM applications, RAG systems, and AI agents\nWelcome to Noveum.ai\u2014the comprehensive tracing and observability platform built specifically for AI applications. Whether you're building LLM-powered chatbots, RAG systems, multi-agent workflows, or any AI-driven application, Noveum provides the insights you need to understand, debug, and optimize your systems.\n\ud83c\udfaf Why AI Applications Need Specialized Observability\nTraditional mo...\n\nSource 5 (https://noveum.ai/docs/getting-started/overview):\nNoveum.ai Overview\nComprehensive AI tracing and observability platform for LLM applications, RAG systems, and AI agents\nWelcome to Noveum.ai\u2014the comprehensive tracing and observability platform built specifically for AI applications. Whether you're building LLM-powered chatbots, RAG systems, multi-agent workflows, or any AI-driven application, Noveum provides the insights you need to understand, debug, and optimize your systems.\n\ud83c\udfaf Why AI Applications Need Specialized Observability\nTraditional mo...\n\nUser Question: What monitoring capabilities does Noveum provide?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", + "prompt.prompt_length": 3285, + "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en):\nMonitor all your AI Agents\nimprove AI Agents today\nNoveum.ai helps you monitor, trace, and optimize your AI applications.\nNoveum.ai works with any AI framework \u2013 LangC...", + "prompt.instruction_following": "rag_optimized", + "response.answer_length": 582, + "response.answer_completeness": "complete", + "response.response_quality": "high", + "response.source_citation": 5, + "response.context_utilization": 0.2124087591240876, + "output_response": "RAG Answer: Noveum provides comprehensive monitoring capabilities for AI applications, including the ability to monitor, trace, and optimize AI agents across various frameworks such as LangChain, CrewAI, AutoGen,...", + "evaluation.retrieval_effectiveness": 1, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 5, + "evaluation.context_relevance": "high", + "evaluation.overall_quality": "high", + "evaluation.ready_for_production": true, + "rag.retrieval_strategy": "semantic_similarity", + "rag.vector_search_results": 5, + "rag.context_synthesis": "multi_source", + "rag.document_coverage": 1, + "rag.information_density": 0.2124087591240876 + }, + "events": [], + "links": [], + "trace_trace_id": "fb0d31fc-2465-4ecd-973c-4cd443d9befa", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:24.828000000", + "trace_end_time": "2025-10-22 16:57:31.793000000", + "trace_duration_ms": 6965.302000000001, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:24.828000000", + "updated_at": "2025-10-22 16:57:24.828000000" + }, + { + "span_id": "b0f0a2e0-c1f7-4ba4-8b75-7090e931a738", + "trace_id": "3bbd0b36-c30f-4126-b62a-dd0fae1c701d", + "parent_span_id": "4db80813-e62f-48ec-9091-62d569527caf", + "name": "agent.query_routing", + "start_time": "2025-10-22 16:57:19.339000000", + "end_time": "2025-10-22 16:57:23.822000000", + "duration_ms": 4482.909000000001, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "How does Noveum handle agent tracing?", + "query_length": 37, + "query_type": "routing_query", + "output_response": "Routed to RAG for query processing", + "router.classification": "RAG", + "router.confidence_score": 1, + "router.rag_keyword_score": 2, + "router.web_keyword_score": 0, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "3bbd0b36-c30f-4126-b62a-dd0fae1c701d", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:19.339000000", + "trace_end_time": "2025-10-22 16:57:23.822000000", + "trace_duration_ms": 4483.158, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:19.339000000", + "updated_at": "2025-10-22 16:57:19.339000000" + }, + { + "span_id": "24da2221-b0ab-4161-98ad-e7b5775fcb27", + "trace_id": "3bbd0b36-c30f-4126-b62a-dd0fae1c701d", + "parent_span_id": "b0f0a2e0-c1f7-4ba4-8b75-7090e931a738", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-22 16:57:19.339000000", + "end_time": "2025-10-22 16:57:19.339000000", + "duration_ms": 0.026, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: How does Noveum handle agent tracing?", + "query_length": 37, + "query_type": "routing_evaluation_query", + "classification.mode": "RAG", + "classification.rag_keyword_score": 2, + "classification.web_keyword_score": 0, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "noveum_specific", + "query.keyword_density": 0.3333333333333333, + "query.domain_affinity": "noveum", + "routing.decision": "Routed to RAG based on analysis", + "routing.rationale": "RAG score: 2, Web score: 0, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "Web Search", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.038461538461538464, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: RAG (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "3bbd0b36-c30f-4126-b62a-dd0fae1c701d", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:19.339000000", + "trace_end_time": "2025-10-22 16:57:23.822000000", + "trace_duration_ms": 4483.158, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:19.339000000", + "updated_at": "2025-10-22 16:57:19.339000000" + }, + { + "span_id": "d2b495eb-7e07-48a3-aa6d-1835aab18c11", + "trace_id": "3bbd0b36-c30f-4126-b62a-dd0fae1c701d", + "parent_span_id": "b0f0a2e0-c1f7-4ba4-8b75-7090e931a738", + "name": "agent.llm-rag", + "start_time": "2025-10-22 16:57:19.339000000", + "end_time": "2025-10-22 16:57:23.822000000", + "duration_ms": 4482.564, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "rag_agent", + "agent.operation": "llm-rag", + "agent.capabilities": [ + "document_retrieval", + "context_generation", + "response_generation" + ], + "agent.id": "noveum_rag_agent", + "input_query": "How does Noveum handle agent tracing?", + "query_length": 37, + "query_type": "rag_query", + "output_response": "RAG Answer: Noveum.ai handles agent tracing through its specialized Multi-Agent Tracing capabilities, which are designed to observe complex workflows and inter-agent communications within multi-agent systems. The...", + "rag.context_length": 2736, + "rag.sources_count": 5, + "rag.answer_length": 663, + "rag.mode": "retrieval_augmented_generation" + }, + "events": [], + "links": [], + "trace_trace_id": "3bbd0b36-c30f-4126-b62a-dd0fae1c701d", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:19.339000000", + "trace_end_time": "2025-10-22 16:57:23.822000000", + "trace_duration_ms": 4483.158, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:19.339000000", + "updated_at": "2025-10-22 16:57:19.339000000" + }, + { + "span_id": "4db80813-e62f-48ec-9091-62d569527caf", + "trace_id": "3bbd0b36-c30f-4126-b62a-dd0fae1c701d", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-22 16:57:19.339000000", + "end_time": "2025-10-22 16:57:23.822000000", + "duration_ms": 4483.031000000001, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "How does Noveum handle agent tracing?", + "query.length": 37, + "processing.mode": "RAG", + "processing.time_seconds": 4.482964038848877, + "processing.response_length": 663, + "processing.sources_count": 5, + "output_response": "Final Answer: Noveum.ai handles agent tracing through its specialized Multi-Agent Tracing capabilities, which are designed to observe complex workflows and inter-agent communications within multi-agent systems. The...", + "final_answer_mode": "RAG", + "query_processed.input_query": "How does Noveum handle agent tracing?", + "query_processed.output_response": "Successfully processed query using RAG, generated 663 character response", + "query_processed.mode": "RAG", + "query_processed.processing_time": 4.482964038848877, + "query_processed.response_length": 663 + }, + "events": [], + "links": [], + "trace_trace_id": "3bbd0b36-c30f-4126-b62a-dd0fae1c701d", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:19.339000000", + "trace_end_time": "2025-10-22 16:57:23.822000000", + "trace_duration_ms": 4483.158, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:19.339000000", + "updated_at": "2025-10-22 16:57:19.339000000" + }, + { + "span_id": "b80cb5cd-24f2-484d-bbff-9febe2aa3ffe", + "trace_id": "3bbd0b36-c30f-4126-b62a-dd0fae1c701d", + "parent_span_id": "d2b495eb-7e07-48a3-aa6d-1835aab18c11", + "name": "agent.llm_model_execution", + "start_time": "2025-10-22 16:57:19.958000000", + "end_time": "2025-10-22 16:57:22.709000000", + "duration_ms": 2750.9480000000003, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: How does Noveum handle agent tracing?", + "query_length": 37, + "query_type": "rag_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 2.750890016555786, + "model.latency_ms": 2750.890016555786, + "model.start_time": 1761152239.958973, + "model.end_time": 1761152242.709863, + "model.performance_tier": "medium", + "model.prompt_tokens": 817, + "model.completion_tokens": 165, + "model.total_tokens": 982, + "model.tokens_per_second": 356.975376728983, + "model.estimated_cost": 0.02946, + "model.efficiency_score": 0.675152749490835, + "model.response_length": 663, + "model.response_quality": "high", + "model.output_response": "Model Response: Noveum.ai handles agent tracing through its specialized Multi-Agent Tracing capabilities, which are designed to observe complex workflows and inter-agent communications within multi-agent systems. The...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "3bbd0b36-c30f-4126-b62a-dd0fae1c701d", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:19.339000000", + "trace_end_time": "2025-10-22 16:57:23.822000000", + "trace_duration_ms": 4483.158, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:19.339000000", + "updated_at": "2025-10-22 16:57:19.339000000" + }, + { + "span_id": "d85fcf7d-7eaa-4299-92fc-c969fab63a6e", + "trace_id": "3bbd0b36-c30f-4126-b62a-dd0fae1c701d", + "parent_span_id": "d2b495eb-7e07-48a3-aa6d-1835aab18c11", + "name": "agent.rag_evaluation_metrics", + "start_time": "2025-10-22 16:57:22.710000000", + "end_time": "2025-10-22 16:57:22.710000000", + "duration_ms": 0.036000000000000004, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "rag_evaluation_metrics", + "agent.capabilities": [ + "retrieval_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: How does Noveum handle agent tracing?", + "query_length": 37, + "query_type": "rag_evaluation_query", + "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, vector searches, data retrieval, agent reasoning, and more. Without proper observability, debugging ...", + "retrieval.context_length": 2736, + "retrieval.sources_count": 5, + "retrieval.context_quality": "high", + "retrieval.effectiveness": 1, + "retrieval.context_utilization": 2.736, + "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, vector searches, data retrieval, agent reasoning, and more. Without proper observability, debugging becomes a nightmare, optimization is guesswork, and understanding user interactions is nearly impossible.\nNoveum.ai solves this by providing comprehensive tracing and observability specifically designed for AI applications.\nKey benefits of using Noveum.ai:\n- Complete Visibility: Trace...\n\nSource 2 (https://noveum.ai/docs/advanced/multi-agent-tracing):\nMulti-Agent Tracing\nObserve complex agent workflows and inter-agent communications with comprehensive tracing\nMulti-agent systems represent some of the most complex AI applications, involving multiple agents that coordinate, communicate, and collaborate to achieve shared goals. Noveum.ai provides specialized tracing capabilities to help you understand and optimize these intricate workflows.\n\ud83c\udfaf Why Multi-Agent Tracing Matters\nMulti-agent systems introduce unique observability challenges:\n- Complex...\n\nSource 3 (https://noveum.ai/en/docs/advanced/multi-agent-tracing):\nMulti-Agent Tracing\nObserve complex agent workflows and inter-agent communications with comprehensive tracing\nMulti-agent systems represent some of the most complex AI applications, involving multiple agents that coordinate, communicate, and collaborate to achieve shared goals. Noveum.ai provides specialized tracing capabilities to help you understand and optimize these intricate workflows.\n\ud83c\udfaf Why Multi-Agent Tracing Matters\nMulti-agent systems introduce unique observability challenges:\n- Complex...\n\nSource 4 (https://noveum.ai/en/docs/advanced/performance-optimization):\n\ud83d\udd17 Next Steps\n- Multi-Agent Tracing - Observe agent workflows\n- RAG Pipeline Observability - Monitor retrieval and generation systems\n- Custom Instrumentation - Add domain-specific tracing\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nSource 5 (https://noveum.ai/docs/advanced/performance-optimization):\n\ud83d\udd17 Next Steps\n- Multi-Agent Tracing - Observe agent workflows\n- RAG Pipeline Observability - Monitor retrieval and generation systems\n- Custom Instrumentation - Add domain-specific tracing\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nUser Question: How does Noveum handle agent tracing?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", + "prompt.prompt_length": 3269, + "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, ...", + "prompt.instruction_following": "rag_optimized", + "response.answer_length": 663, + "response.answer_completeness": "complete", + "response.response_quality": "high", + "response.source_citation": 5, + "response.context_utilization": 0.24232456140350878, + "output_response": "RAG Answer: Noveum.ai handles agent tracing through its specialized Multi-Agent Tracing capabilities, which are designed to observe complex workflows and inter-agent communications within multi-agent systems. The...", + "evaluation.retrieval_effectiveness": 1, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 5, + "evaluation.context_relevance": "high", + "evaluation.overall_quality": "high", + "evaluation.ready_for_production": true, + "rag.retrieval_strategy": "semantic_similarity", + "rag.vector_search_results": 5, + "rag.context_synthesis": "multi_source", + "rag.document_coverage": 1, + "rag.information_density": 0.24232456140350878 + }, + "events": [], + "links": [], + "trace_trace_id": "3bbd0b36-c30f-4126-b62a-dd0fae1c701d", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:19.339000000", + "trace_end_time": "2025-10-22 16:57:23.822000000", + "trace_duration_ms": 4483.158, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:19.339000000", + "updated_at": "2025-10-22 16:57:19.339000000" + }, + { + "span_id": "93d82f68-6c62-40de-a675-c4d7c40c1d01", + "trace_id": "fee3e6d3-8da1-4ef3-8214-a7d7323d3a49", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-22 16:57:13.334000000", + "end_time": "2025-10-22 16:57:18.330000000", + "duration_ms": 4996.436, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What APIs are available in Noveum platform?", + "query.length": 43, + "processing.mode": "RAG", + "processing.time_seconds": 4.996389865875244, + "processing.response_length": 270, + "processing.sources_count": 5, + "output_response": "Final Answer: The provided context does not specify the APIs available in the Noveum platform. For detailed information about the APIs, I recommend checking the official Noveum documentation or the SDK Integration ...", + "final_answer_mode": "RAG", + "query_processed.input_query": "What APIs are available in Noveum platform?", + "query_processed.output_response": "Successfully processed query using RAG, generated 270 character response", + "query_processed.mode": "RAG", + "query_processed.processing_time": 4.996389865875244, + "query_processed.response_length": 270 + }, + "events": [], + "links": [], + "trace_trace_id": "fee3e6d3-8da1-4ef3-8214-a7d7323d3a49", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:13.334000000", + "trace_end_time": "2025-10-22 16:57:18.331000000", + "trace_duration_ms": 4996.502, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:13.334000000", + "updated_at": "2025-10-22 16:57:13.334000000" + }, + { + "span_id": "1be727c6-14e8-4b2e-b8df-a1cab902454f", + "trace_id": "fee3e6d3-8da1-4ef3-8214-a7d7323d3a49", + "parent_span_id": "93d82f68-6c62-40de-a675-c4d7c40c1d01", + "name": "agent.query_routing", + "start_time": "2025-10-22 16:57:13.334000000", + "end_time": "2025-10-22 16:57:18.330000000", + "duration_ms": 4996.354, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What APIs are available in Noveum platform?", + "query_length": 43, + "query_type": "routing_query", + "output_response": "Routed to RAG for query processing", + "router.classification": "RAG", + "router.confidence_score": 1, + "router.rag_keyword_score": 3, + "router.web_keyword_score": 0, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "fee3e6d3-8da1-4ef3-8214-a7d7323d3a49", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:13.334000000", + "trace_end_time": "2025-10-22 16:57:18.331000000", + "trace_duration_ms": 4996.502, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:13.334000000", + "updated_at": "2025-10-22 16:57:13.334000000" + }, + { + "span_id": "58ab1d39-d349-4164-8ff9-26ece4b1e1aa", + "trace_id": "fee3e6d3-8da1-4ef3-8214-a7d7323d3a49", + "parent_span_id": "1be727c6-14e8-4b2e-b8df-a1cab902454f", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-22 16:57:13.334000000", + "end_time": "2025-10-22 16:57:13.334000000", + "duration_ms": 0.041999999999999996, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What APIs are available in Noveum platform?", + "query_length": 43, + "query_type": "routing_evaluation_query", + "classification.mode": "RAG", + "classification.rag_keyword_score": 3, + "classification.web_keyword_score": 0, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "noveum_specific", + "query.keyword_density": 0.42857142857142855, + "query.domain_affinity": "noveum", + "routing.decision": "Routed to RAG based on analysis", + "routing.rationale": "RAG score: 3, Web score: 0, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "Web Search", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.057692307692307696, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: RAG (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "fee3e6d3-8da1-4ef3-8214-a7d7323d3a49", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:13.334000000", + "trace_end_time": "2025-10-22 16:57:18.331000000", + "trace_duration_ms": 4996.502, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:13.334000000", + "updated_at": "2025-10-22 16:57:13.334000000" + }, + { + "span_id": "f11ebf68-a416-484e-8e55-f77a11b9ea1c", + "trace_id": "fee3e6d3-8da1-4ef3-8214-a7d7323d3a49", + "parent_span_id": "1be727c6-14e8-4b2e-b8df-a1cab902454f", + "name": "agent.llm-rag", + "start_time": "2025-10-22 16:57:13.334000000", + "end_time": "2025-10-22 16:57:18.330000000", + "duration_ms": 4996.146000000001, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "rag_agent", + "agent.operation": "llm-rag", + "agent.capabilities": [ + "document_retrieval", + "context_generation", + "response_generation" + ], + "agent.id": "noveum_rag_agent", + "input_query": "What APIs are available in Noveum platform?", + "query_length": 43, + "query_type": "rag_query", + "output_response": "RAG Answer: The provided context does not specify the APIs available in the Noveum platform. For detailed information about the APIs, I recommend checking the official Noveum documentation or the SDK Integration ...", + "rag.context_length": 2284, + "rag.sources_count": 5, + "rag.answer_length": 270, + "rag.mode": "retrieval_augmented_generation" + }, + "events": [], + "links": [], + "trace_trace_id": "fee3e6d3-8da1-4ef3-8214-a7d7323d3a49", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:13.334000000", + "trace_end_time": "2025-10-22 16:57:18.331000000", + "trace_duration_ms": 4996.502, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:13.334000000", + "updated_at": "2025-10-22 16:57:13.334000000" + }, + { + "span_id": "11394488-8802-403d-a926-7fda3a510b61", + "trace_id": "fee3e6d3-8da1-4ef3-8214-a7d7323d3a49", + "parent_span_id": "f11ebf68-a416-484e-8e55-f77a11b9ea1c", + "name": "agent.llm_model_execution", + "start_time": "2025-10-22 16:57:13.862000000", + "end_time": "2025-10-22 16:57:16.049000000", + "duration_ms": 2187.29, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What APIs are available in Noveum platform?", + "query_length": 43, + "query_type": "rag_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 2.187239170074463, + "model.latency_ms": 2187.239170074463, + "model.start_time": 1761152233.862141, + "model.end_time": 1761152236.04938, + "model.performance_tier": "medium", + "model.prompt_tokens": 705, + "model.completion_tokens": 67, + "model.total_tokens": 772, + "model.tokens_per_second": 352.9563710098141, + "model.estimated_cost": 0.02316, + "model.efficiency_score": 0.34974093264248707, + "model.response_length": 270, + "model.response_quality": "high", + "model.output_response": "Model Response: The provided context does not specify the APIs available in the Noveum platform. For detailed information about the APIs, I recommend checking the official Noveum documentation or the SDK Integration ...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "fee3e6d3-8da1-4ef3-8214-a7d7323d3a49", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:13.334000000", + "trace_end_time": "2025-10-22 16:57:18.331000000", + "trace_duration_ms": 4996.502, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:13.334000000", + "updated_at": "2025-10-22 16:57:13.334000000" + }, + { + "span_id": "341cf8ab-cc33-4d57-952f-ba1065f0432e", + "trace_id": "fee3e6d3-8da1-4ef3-8214-a7d7323d3a49", + "parent_span_id": "f11ebf68-a416-484e-8e55-f77a11b9ea1c", + "name": "agent.rag_evaluation_metrics", + "start_time": "2025-10-22 16:57:16.049000000", + "end_time": "2025-10-22 16:57:16.049000000", + "duration_ms": 0.041, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "rag_evaluation_metrics", + "agent.capabilities": [ + "retrieval_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What APIs are available in Noveum platform?", + "query_length": 43, + "query_type": "rag_evaluation_query", + "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/docs):\n- \ud83c\udf0d Universal: Works across frameworks, clouds, and deployment models\n\ud83e\udd1d Community & Support\n- \ud83d\udcac Discord Community: Join our Discord\n- \ud83d\udce7 Email Support: [email protected]\n- \ud83d\udc1b Bug Reports: GitHub Issues\n- \ud83d\udcd6 Knowledge Base: Help Center\nReady to get started? Head to ...", + "retrieval.context_length": 2284, + "retrieval.sources_count": 5, + "retrieval.context_quality": "high", + "retrieval.effectiveness": 1, + "retrieval.context_utilization": 2.284, + "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/docs):\n- \ud83c\udf0d Universal: Works across frameworks, clouds, and deployment models\n\ud83e\udd1d Community & Support\n- \ud83d\udcac Discord Community: Join our Discord\n- \ud83d\udce7 Email Support: [email protected]\n- \ud83d\udc1b Bug Reports: GitHub Issues\n- \ud83d\udcd6 Knowledge Base: Help Center\nReady to get started? Head to our SDK Integration Guide to begin tracing your AI applications in under 5 minutes!\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability ...\n\nSource 2 (https://noveum.ai/docs):\n- \ud83c\udf0d Universal: Works across frameworks, clouds, and deployment models\n\ud83e\udd1d Community & Support\n- \ud83d\udcac Discord Community: Join our Discord\n- \ud83d\udce7 Email Support: [email protected]\n- \ud83d\udc1b Bug Reports: GitHub Issues\n- \ud83d\udcd6 Knowledge Base: Help Center\nReady to get started? Head to our SDK Integration Guide to begin tracing your AI applications in under 5 minutes!\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability ...\n\nSource 3 (https://noveum.ai/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nSource 4 (https://noveum.ai/en/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nSource 5 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, vector searches, data retrieval, agent reasoning, and more. Without proper observability, debugging becomes a nightmare, optimization is guesswork, and understanding user interactions is nearly impossible.\nNoveum.ai solves this by providing comprehensive tracing and observability specifically designed for AI applications.\nKey benefits of using Noveum.ai:\n- Complete Visibility: Trace...\n\nUser Question: What APIs are available in Noveum platform?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", + "prompt.prompt_length": 2823, + "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/docs):\n- \ud83c\udf0d Universal: Works across frameworks, clouds, and deployment models\n\ud83e\udd1d Community & Support\n- \ud83d\udcac Discord Community: Join our Discord\n- \ud83d\udce7 Email Support: [email prot...", + "prompt.instruction_following": "rag_optimized", + "response.answer_length": 270, + "response.answer_completeness": "complete", + "response.response_quality": "high", + "response.source_citation": 5, + "response.context_utilization": 0.11821366024518389, + "output_response": "RAG Answer: The provided context does not specify the APIs available in the Noveum platform. For detailed information about the APIs, I recommend checking the official Noveum documentation or the SDK Integration ...", + "evaluation.retrieval_effectiveness": 1, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 5, + "evaluation.context_relevance": "high", + "evaluation.overall_quality": "high", + "evaluation.ready_for_production": true, + "rag.retrieval_strategy": "semantic_similarity", + "rag.vector_search_results": 5, + "rag.context_synthesis": "multi_source", + "rag.document_coverage": 1, + "rag.information_density": 0.11821366024518389 + }, + "events": [], + "links": [], + "trace_trace_id": "fee3e6d3-8da1-4ef3-8214-a7d7323d3a49", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:13.334000000", + "trace_end_time": "2025-10-22 16:57:18.331000000", + "trace_duration_ms": 4996.502, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:13.334000000", + "updated_at": "2025-10-22 16:57:13.334000000" + }, + { + "span_id": "24930c28-7f93-426c-8914-bd0e0085589f", + "trace_id": "93bd37c4-4289-43c9-b34e-19eaaef610c2", + "parent_span_id": "bfc91518-f818-4f35-9113-70b04eb55edc", + "name": "agent.query_routing", + "start_time": "2025-10-22 16:57:06.605000000", + "end_time": "2025-10-22 16:57:12.327000000", + "duration_ms": 5721.561000000001, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "How do I set up observability with Noveum?", + "query_length": 42, + "query_type": "routing_query", + "output_response": "Routed to RAG for query processing", + "router.classification": "RAG", + "router.confidence_score": 1, + "router.rag_keyword_score": 2, + "router.web_keyword_score": 0, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "93bd37c4-4289-43c9-b34e-19eaaef610c2", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:06.605000000", + "trace_end_time": "2025-10-22 16:57:12.327000000", + "trace_duration_ms": 5721.707, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:06.605000000", + "updated_at": "2025-10-22 16:57:06.605000000" + }, + { + "span_id": "11fc18a8-0380-48ca-894e-cc26f648d290", + "trace_id": "93bd37c4-4289-43c9-b34e-19eaaef610c2", + "parent_span_id": "24930c28-7f93-426c-8914-bd0e0085589f", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-22 16:57:06.605000000", + "end_time": "2025-10-22 16:57:06.606000000", + "duration_ms": 0.047, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: How do I set up observability with Noveum?", + "query_length": 42, + "query_type": "routing_evaluation_query", + "classification.mode": "RAG", + "classification.rag_keyword_score": 2, + "classification.web_keyword_score": 0, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "noveum_specific", + "query.keyword_density": 0.25, + "query.domain_affinity": "noveum", + "routing.decision": "Routed to RAG based on analysis", + "routing.rationale": "RAG score: 2, Web score: 0, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "Web Search", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.038461538461538464, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: RAG (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "93bd37c4-4289-43c9-b34e-19eaaef610c2", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:06.605000000", + "trace_end_time": "2025-10-22 16:57:12.327000000", + "trace_duration_ms": 5721.707, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:06.605000000", + "updated_at": "2025-10-22 16:57:06.605000000" + }, + { + "span_id": "bfc91518-f818-4f35-9113-70b04eb55edc", + "trace_id": "93bd37c4-4289-43c9-b34e-19eaaef610c2", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-22 16:57:06.605000000", + "end_time": "2025-10-22 16:57:12.327000000", + "duration_ms": 5721.635, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "How do I set up observability with Noveum?", + "query.length": 42, + "processing.mode": "RAG", + "processing.time_seconds": 5.721601963043213, + "processing.response_length": 815, + "processing.sources_count": 5, + "output_response": "Final Answer: To set up observability with Noveum, follow these steps:\n\n1. **Sign Up**: Go to [noveum.ai](https://noveum.ai) and create an account.\n2. **Create a Project**: After signing up, create a project to obt...", + "final_answer_mode": "RAG", + "query_processed.input_query": "How do I set up observability with Noveum?", + "query_processed.output_response": "Successfully processed query using RAG, generated 815 character response", + "query_processed.mode": "RAG", + "query_processed.processing_time": 5.721601963043213, + "query_processed.response_length": 815 + }, + "events": [], + "links": [], + "trace_trace_id": "93bd37c4-4289-43c9-b34e-19eaaef610c2", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:06.605000000", + "trace_end_time": "2025-10-22 16:57:12.327000000", + "trace_duration_ms": 5721.707, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:06.605000000", + "updated_at": "2025-10-22 16:57:06.605000000" + }, + { + "span_id": "2a82e4b1-c93d-498f-9dd5-e971f228d642", + "trace_id": "93bd37c4-4289-43c9-b34e-19eaaef610c2", + "parent_span_id": "24930c28-7f93-426c-8914-bd0e0085589f", + "name": "agent.llm-rag", + "start_time": "2025-10-22 16:57:06.606000000", + "end_time": "2025-10-22 16:57:12.327000000", + "duration_ms": 5721.352000000001, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "rag_agent", + "agent.operation": "llm-rag", + "agent.capabilities": [ + "document_retrieval", + "context_generation", + "response_generation" + ], + "agent.id": "noveum_rag_agent", + "input_query": "How do I set up observability with Noveum?", + "query_length": 42, + "query_type": "rag_query", + "output_response": "RAG Answer: To set up observability with Noveum, follow these steps:\n\n1. **Sign Up**: Go to [noveum.ai](https://noveum.ai) and create an account.\n2. **Create a Project**: After signing up, create a project to obt...", + "rag.context_length": 2870, + "rag.sources_count": 6, + "rag.answer_length": 815, + "rag.mode": "retrieval_augmented_generation" + }, + "events": [], + "links": [], + "trace_trace_id": "93bd37c4-4289-43c9-b34e-19eaaef610c2", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:06.605000000", + "trace_end_time": "2025-10-22 16:57:12.327000000", + "trace_duration_ms": 5721.707, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:06.605000000", + "updated_at": "2025-10-22 16:57:06.605000000" + }, + { + "span_id": "520b4e59-da14-4db2-af02-0973cda9bd86", + "trace_id": "93bd37c4-4289-43c9-b34e-19eaaef610c2", + "parent_span_id": "2a82e4b1-c93d-498f-9dd5-e971f228d642", + "name": "agent.llm_model_execution", + "start_time": "2025-10-22 16:57:07.109000000", + "end_time": "2025-10-22 16:57:11.837000000", + "duration_ms": 4728.018999999999, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: How do I set up observability with Noveum?", + "query_length": 42, + "query_type": "rag_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 4.727963924407959, + "model.latency_ms": 4727.963924407959, + "model.start_time": 1761152227.109188, + "model.end_time": 1761152231.837152, + "model.performance_tier": "medium", + "model.prompt_tokens": 852, + "model.completion_tokens": 203, + "model.total_tokens": 1055, + "model.tokens_per_second": 223.1404504915101, + "model.estimated_cost": 0.03165, + "model.efficiency_score": 0.7725118483412322, + "model.response_length": 815, + "model.response_quality": "high", + "model.output_response": "Model Response: To set up observability with Noveum, follow these steps:\n\n1. **Sign Up**: Go to [noveum.ai](https://noveum.ai) and create an account.\n2. **Create a Project**: After signing up, create a project to obt...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "93bd37c4-4289-43c9-b34e-19eaaef610c2", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:06.605000000", + "trace_end_time": "2025-10-22 16:57:12.327000000", + "trace_duration_ms": 5721.707, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:06.605000000", + "updated_at": "2025-10-22 16:57:06.605000000" + }, + { + "span_id": "dff080ab-238c-4cca-a910-cf530f910bda", + "trace_id": "93bd37c4-4289-43c9-b34e-19eaaef610c2", + "parent_span_id": "2a82e4b1-c93d-498f-9dd5-e971f228d642", + "name": "agent.rag_evaluation_metrics", + "start_time": "2025-10-22 16:57:11.837000000", + "end_time": "2025-10-22 16:57:11.838000000", + "duration_ms": 0.259, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "rag_evaluation_metrics", + "agent.capabilities": [ + "retrieval_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: How do I set up observability with Noveum?", + "query_length": 42, + "query_type": "rag_evaluation_query", + "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\n- Production Ready: Built for scale with intelligent sampling and batching\n- Privacy Focused: You control what data is captured and how it's stored\n- Open Source SDKs: Transparent, auditable, and extensible\nGetting ...", + "retrieval.context_length": 2870, + "retrieval.sources_count": 6, + "retrieval.context_quality": "high", + "retrieval.effectiveness": 1.2, + "retrieval.context_utilization": 2.87, + "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\n- Production Ready: Built for scale with intelligent sampling and batching\n- Privacy Focused: You control what data is captured and how it's stored\n- Open Source SDKs: Transparent, auditable, and extensible\nGetting Started\nReady to add comprehensive observability to your AI applications?\n- Sign up at noveum.ai\n- Create a project and get your API key\n- Install the SDK for your preferred language\n- Add tracing to your AI workflows\n- Explore insights in the Noveum.ai dashboard\nCheck out our integra...\n\nSource 2 (https://noveum.ai/en/docs/getting-started/tracing-concepts):\n- Explore Framework Integrations - Framework-specific guidance\n- Learn Advanced Patterns - Custom instrumentation techniques\n- Master the Dashboard - Analyze your traces effectively\nRemember: Good observability is not about collecting all possible data, but about collecting the right data that helps you understand, debug, and optimize your AI applications.\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to O...\n\nSource 3 (https://noveum.ai/docs/getting-started/tracing-concepts):\n- Explore Framework Integrations - Framework-specific guidance\n- Learn Advanced Patterns - Custom instrumentation techniques\n- Master the Dashboard - Analyze your traces effectively\nRemember: Good observability is not about collecting all possible data, but about collecting the right data that helps you understand, debug, and optimize your AI applications.\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to O...\n\nSource 4 (https://noveum.ai/en/docs/getting-started/overview):\nNoveum.ai Overview\nComprehensive AI tracing and observability platform for LLM applications, RAG systems, and AI agents\nWelcome to Noveum.ai\u2014the comprehensive tracing and observability platform built specifically for AI applications. Whether you're building LLM-powered chatbots, RAG systems, multi-agent workflows, or any AI-driven application, Noveum provides the insights you need to understand, debug, and optimize your systems.\n\ud83c\udfaf Why AI Applications Need Specialized Observability\nTraditional mo...\n\nSource 5 (https://noveum.ai/docs/getting-started/overview):\nNoveum.ai Overview\nComprehensive AI tracing and observability platform for LLM applications, RAG systems, and AI agents\nWelcome to Noveum.ai\u2014the comprehensive tracing and observability platform built specifically for AI applications. Whether you're building LLM-powered chatbots, RAG systems, multi-agent workflows, or any AI-driven application, Noveum provides the insights you need to understand, debug, and optimize your systems.\n\ud83c\udfaf Why AI Applications Need Specialized Observability\nTraditional mo...\n\nUser Question: How do I set up observability with Noveum?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", + "prompt.prompt_length": 3408, + "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\n- Production Ready: Built for scale with intelligent sampling and batching\n- Privacy Focused: You control what data...", + "prompt.instruction_following": "rag_optimized", + "response.answer_length": 815, + "response.answer_completeness": "complete", + "response.response_quality": "high", + "response.source_citation": 6, + "response.context_utilization": 0.28397212543554007, + "output_response": "RAG Answer: To set up observability with Noveum, follow these steps:\n\n1. **Sign Up**: Go to [noveum.ai](https://noveum.ai) and create an account.\n2. **Create a Project**: After signing up, create a project to obt...", + "evaluation.retrieval_effectiveness": 1.2, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 6, + "evaluation.context_relevance": "high", + "evaluation.overall_quality": "high", + "evaluation.ready_for_production": true, + "rag.retrieval_strategy": "semantic_similarity", + "rag.vector_search_results": 6, + "rag.context_synthesis": "multi_source", + "rag.document_coverage": 1.2, + "rag.information_density": 0.28397212543554007 + }, + "events": [], + "links": [], + "trace_trace_id": "93bd37c4-4289-43c9-b34e-19eaaef610c2", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:57:06.605000000", + "trace_end_time": "2025-10-22 16:57:12.327000000", + "trace_duration_ms": 5721.707, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:57:06.605000000", + "updated_at": "2025-10-22 16:57:06.605000000" + }, + { + "span_id": "dad7df65-be49-4245-8721-c91da4962b9a", + "trace_id": "df0c6344-1a2e-4c53-bb7d-4e43b3bce1b8", + "parent_span_id": "9024b0d9-2b75-428f-af8f-75a921a7c69c", + "name": "agent.query_routing", + "start_time": "2025-10-22 16:56:59.213000000", + "end_time": "2025-10-22 16:57:05.598000000", + "duration_ms": 6384.249, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What features does Noveum Trace offer?", + "query_length": 38, + "query_type": "routing_query", + "output_response": "Routed to RAG for query processing", + "router.classification": "RAG", + "router.confidence_score": 1, + "router.rag_keyword_score": 3, + "router.web_keyword_score": 0, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "df0c6344-1a2e-4c53-bb7d-4e43b3bce1b8", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:56:59.213000000", + "trace_end_time": "2025-10-22 16:57:05.598000000", + "trace_duration_ms": 6384.386, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:56:59.213000000", + "updated_at": "2025-10-22 16:56:59.213000000" + }, + { + "span_id": "9024b0d9-2b75-428f-af8f-75a921a7c69c", + "trace_id": "df0c6344-1a2e-4c53-bb7d-4e43b3bce1b8", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-22 16:56:59.213000000", + "end_time": "2025-10-22 16:57:05.598000000", + "duration_ms": 6384.32, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What features does Noveum Trace offer?", + "query.length": 38, + "processing.mode": "RAG", + "processing.time_seconds": 6.384284973144531, + "processing.response_length": 1017, + "processing.sources_count": 5, + "output_response": "Final Answer: Noveum Trace offers several key features through its Python SDK (`noveum-trace`) and TypeScript SDK (`@noveum/trace`):\n\n1. **Decorator-based Tracing**: It provides a simple way to trace LLM calls, age...", + "final_answer_mode": "RAG", + "query_processed.input_query": "What features does Noveum Trace offer?", + "query_processed.output_response": "Successfully processed query using RAG, generated 1017 character response", + "query_processed.mode": "RAG", + "query_processed.processing_time": 6.384284973144531, + "query_processed.response_length": 1017 + }, + "events": [], + "links": [], + "trace_trace_id": "df0c6344-1a2e-4c53-bb7d-4e43b3bce1b8", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:56:59.213000000", + "trace_end_time": "2025-10-22 16:57:05.598000000", + "trace_duration_ms": 6384.386, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:56:59.213000000", + "updated_at": "2025-10-22 16:56:59.213000000" + }, + { + "span_id": "d07fdbdc-2b8f-4de9-ac32-988389701771", + "trace_id": "df0c6344-1a2e-4c53-bb7d-4e43b3bce1b8", + "parent_span_id": "dad7df65-be49-4245-8721-c91da4962b9a", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-22 16:56:59.214000000", + "end_time": "2025-10-22 16:56:59.214000000", + "duration_ms": 0.060000000000000005, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What features does Noveum Trace offer?", + "query_length": 38, + "query_type": "routing_evaluation_query", + "classification.mode": "RAG", + "classification.rag_keyword_score": 3, + "classification.web_keyword_score": 0, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "noveum_specific", + "query.keyword_density": 0.5, + "query.domain_affinity": "noveum", + "routing.decision": "Routed to RAG based on analysis", + "routing.rationale": "RAG score: 3, Web score: 0, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "Web Search", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.057692307692307696, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: RAG (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "df0c6344-1a2e-4c53-bb7d-4e43b3bce1b8", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:56:59.213000000", + "trace_end_time": "2025-10-22 16:57:05.598000000", + "trace_duration_ms": 6384.386, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:56:59.213000000", + "updated_at": "2025-10-22 16:56:59.213000000" + }, + { + "span_id": "83f86157-5af4-4c99-aad1-647866cd804e", + "trace_id": "df0c6344-1a2e-4c53-bb7d-4e43b3bce1b8", + "parent_span_id": "dad7df65-be49-4245-8721-c91da4962b9a", + "name": "agent.llm-rag", + "start_time": "2025-10-22 16:56:59.214000000", + "end_time": "2025-10-22 16:57:05.598000000", + "duration_ms": 6384.048, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "rag_agent", + "agent.operation": "llm-rag", + "agent.capabilities": [ + "document_retrieval", + "context_generation", + "response_generation" + ], + "agent.id": "noveum_rag_agent", + "input_query": "What features does Noveum Trace offer?", + "query_length": 38, + "query_type": "rag_query", + "output_response": "RAG Answer: Noveum Trace offers several key features through its Python SDK (`noveum-trace`) and TypeScript SDK (`@noveum/trace`):\n\n1. **Decorator-based Tracing**: It provides a simple way to trace LLM calls, age...", + "rag.context_length": 2787, + "rag.sources_count": 5, + "rag.answer_length": 1017, + "rag.mode": "retrieval_augmented_generation" + }, + "events": [], + "links": [], + "trace_trace_id": "df0c6344-1a2e-4c53-bb7d-4e43b3bce1b8", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:56:59.213000000", + "trace_end_time": "2025-10-22 16:57:05.598000000", + "trace_duration_ms": 6384.386, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:56:59.213000000", + "updated_at": "2025-10-22 16:56:59.213000000" + }, + { + "span_id": "3b4fa9a7-88bf-4ee1-aeb7-bfa9573f28f2", + "trace_id": "df0c6344-1a2e-4c53-bb7d-4e43b3bce1b8", + "parent_span_id": "83f86157-5af4-4c99-aad1-647866cd804e", + "name": "agent.llm_model_execution", + "start_time": "2025-10-22 16:56:59.591000000", + "end_time": "2025-10-22 16:57:04.869000000", + "duration_ms": 5278.925, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What features does Noveum Trace offer?", + "query_length": 38, + "query_type": "rag_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 5.278868913650513, + "model.latency_ms": 5278.868913650513, + "model.start_time": 1761152219.5910342, + "model.end_time": 1761152224.869903, + "model.performance_tier": "slow", + "model.prompt_tokens": 830, + "model.completion_tokens": 254, + "model.total_tokens": 1084, + "model.tokens_per_second": 205.3470199263535, + "model.estimated_cost": 0.03252, + "model.efficiency_score": 0.9381918819188192, + "model.response_length": 1017, + "model.response_quality": "high", + "model.output_response": "Model Response: Noveum Trace offers several key features through its Python SDK (`noveum-trace`) and TypeScript SDK (`@noveum/trace`):\n\n1. **Decorator-based Tracing**: It provides a simple way to trace LLM calls, age...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "df0c6344-1a2e-4c53-bb7d-4e43b3bce1b8", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:56:59.213000000", + "trace_end_time": "2025-10-22 16:57:05.598000000", + "trace_duration_ms": 6384.386, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:56:59.213000000", + "updated_at": "2025-10-22 16:56:59.213000000" + }, + { + "span_id": "f9214dbe-adb9-44ff-8361-70c872cef66d", + "trace_id": "df0c6344-1a2e-4c53-bb7d-4e43b3bce1b8", + "parent_span_id": "83f86157-5af4-4c99-aad1-647866cd804e", + "name": "agent.rag_evaluation_metrics", + "start_time": "2025-10-22 16:57:04.870000000", + "end_time": "2025-10-22 16:57:04.870000000", + "duration_ms": 0.043000000000000003, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "rag_evaluation_metrics", + "agent.capabilities": [ + "retrieval_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What features does Noveum Trace offer?", + "query_length": 38, + "query_type": "rag_evaluation_query", + "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\n\ud83d\udcd8 TypeScript SDK (\n@noveum/trace\n)- Framework ...", + "retrieval.context_length": 2787, + "retrieval.sources_count": 5, + "retrieval.context_quality": "high", + "retrieval.effectiveness": 1, + "retrieval.context_utilization": 2.787, + "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\n\ud83d\udcd8 TypeScript SDK (\n@noveum/trace\n)- Framework integrations for Next.js, Express.js, and Hono\n- TypeScript-first design with full type safety\n- Universal compatibility (Node.js, Edge Runtime, browsers)\n-\n\ud83d\udcca Noveum Platform\n- Real-time dashboard for analyzing traces and performance\n- Ad...\n\nSource 2 (https://noveum.ai/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\n\ud83d\udcd8 TypeScript SDK (\n@noveum/trace\n)- Framework integrations for Next.js, Express.js, and Hono\n- TypeScript-first design with full type safety\n- Universal compatibility (Node.js, Edge Runtime, browsers)\n-\n\ud83d\udcca Noveum Platform\n- Real-time dashboard for analyzing traces and performance\n- Ad...\n\nSource 3 (https://noveum.ai/en/docs/getting-started/sdk-integration):\n- \ud83d\udc65 Collaborate with your team on insights\n\ud83d\udd12 Security & Privacy\n- \ud83d\udd10 Encryption: All data encrypted in transit and at rest\n- \ud83c\udf9b\ufe0f Configurable Capture: Control what data is collected\n- \ud83c\udfe0 Data Residency: Choose your data storage region\n- \u23f0 Retention Control: Set custom data retention policies\nNext Steps\n- Tracing Concepts - Learn about traces, spans, and observability best practices\n- Framework Integrations - Deep dive into specific framework setups\n- Multi-Agent Tracing - Observe complex agent work...\n\nSource 4 (https://noveum.ai/docs/getting-started/sdk-integration):\n- \ud83d\udc65 Collaborate with your team on insights\n\ud83d\udd12 Security & Privacy\n- \ud83d\udd10 Encryption: All data encrypted in transit and at rest\n- \ud83c\udf9b\ufe0f Configurable Capture: Control what data is collected\n- \ud83c\udfe0 Data Residency: Choose your data storage region\n- \u23f0 Retention Control: Set custom data retention policies\nNext Steps\n- Tracing Concepts - Learn about traces, spans, and observability best practices\n- Framework Integrations - Deep dive into specific framework setups\n- Multi-Agent Tracing - Observe complex agent work...\n\nSource 5 (https://noveum.ai/docs/platform/dashboard):\n- Bulk Export: Download multiple traces simultaneously\n- Comparative Analysis: Compare performance across multiple traces\nReal-Time Updates\n- Live Refresh: Automatic updates as new traces arrive\n- Connection Monitoring: Real-time status of your trace ingestion pipeline\n- Performance Indicators: Live metrics for system health monitoring\nReady to dive deeper? Explore Projects & Environments to organize your AI applications, or check out Team Collaboration to share insights with your team.\nGet Earl...\n\nUser Question: What features does Noveum Trace offer?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", + "prompt.prompt_length": 3321, + "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frame...", + "prompt.instruction_following": "rag_optimized", + "response.answer_length": 1017, + "response.answer_completeness": "complete", + "response.response_quality": "high", + "response.source_citation": 5, + "response.context_utilization": 0.36490850376749195, + "output_response": "RAG Answer: Noveum Trace offers several key features through its Python SDK (`noveum-trace`) and TypeScript SDK (`@noveum/trace`):\n\n1. **Decorator-based Tracing**: It provides a simple way to trace LLM calls, age...", + "evaluation.retrieval_effectiveness": 1, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 5, + "evaluation.context_relevance": "high", + "evaluation.overall_quality": "high", + "evaluation.ready_for_production": true, + "rag.retrieval_strategy": "semantic_similarity", + "rag.vector_search_results": 5, + "rag.context_synthesis": "multi_source", + "rag.document_coverage": 1, + "rag.information_density": 0.36490850376749195 + }, + "events": [], + "links": [], + "trace_trace_id": "df0c6344-1a2e-4c53-bb7d-4e43b3bce1b8", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:56:59.213000000", + "trace_end_time": "2025-10-22 16:57:05.598000000", + "trace_duration_ms": 6384.386, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:56:59.213000000", + "updated_at": "2025-10-22 16:56:59.213000000" + }, + { + "span_id": "138a5089-5268-447a-afb5-4f14fb504281", + "trace_id": "29a862bf-d7aa-4cf9-963c-0a4244343bbe", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-22 16:56:54.719000000", + "end_time": "2025-10-22 16:56:58.207000000", + "duration_ms": 3487.959, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What are Noveum's pricing plans?", + "query.length": 32, + "processing.mode": "RAG", + "processing.time_seconds": 3.487928867340088, + "processing.response_length": 441, + "processing.sources_count": 5, + "output_response": "Final Answer: The provided context does not include specific information about Noveum's pricing plans. It mentions that all users will have access to the Observability suite for free, and early users will receive f...", + "final_answer_mode": "RAG", + "query_processed.input_query": "What are Noveum's pricing plans?", + "query_processed.output_response": "Successfully processed query using RAG, generated 441 character response", + "query_processed.mode": "RAG", + "query_processed.processing_time": 3.487928867340088, + "query_processed.response_length": 441 + }, + "events": [], + "links": [], + "trace_trace_id": "29a862bf-d7aa-4cf9-963c-0a4244343bbe", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:56:54.718000000", + "trace_end_time": "2025-10-22 16:56:58.207000000", + "trace_duration_ms": 3488.068, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:56:54.718000000", + "updated_at": "2025-10-22 16:56:54.718000000" + }, + { + "span_id": "7cda4dab-0fc5-4cae-8d8c-9445f661c67b", + "trace_id": "29a862bf-d7aa-4cf9-963c-0a4244343bbe", + "parent_span_id": "138a5089-5268-447a-afb5-4f14fb504281", + "name": "agent.query_routing", + "start_time": "2025-10-22 16:56:54.719000000", + "end_time": "2025-10-22 16:56:58.207000000", + "duration_ms": 3487.848, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What are Noveum's pricing plans?", + "query_length": 32, + "query_type": "routing_query", + "output_response": "Routed to RAG for query processing", + "router.classification": "RAG", + "router.confidence_score": 1, + "router.rag_keyword_score": 3, + "router.web_keyword_score": 0, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "29a862bf-d7aa-4cf9-963c-0a4244343bbe", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:56:54.718000000", + "trace_end_time": "2025-10-22 16:56:58.207000000", + "trace_duration_ms": 3488.068, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:56:54.718000000", + "updated_at": "2025-10-22 16:56:54.718000000" + }, + { + "span_id": "922c84a5-b14a-4b2f-8759-427d6e089812", + "trace_id": "29a862bf-d7aa-4cf9-963c-0a4244343bbe", + "parent_span_id": "7cda4dab-0fc5-4cae-8d8c-9445f661c67b", + "name": "agent.llm-rag", + "start_time": "2025-10-22 16:56:54.719000000", + "end_time": "2025-10-22 16:56:58.207000000", + "duration_ms": 3487.546, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "rag_agent", + "agent.operation": "llm-rag", + "agent.capabilities": [ + "document_retrieval", + "context_generation", + "response_generation" + ], + "agent.id": "noveum_rag_agent", + "input_query": "What are Noveum's pricing plans?", + "query_length": 32, + "query_type": "rag_query", + "output_response": "RAG Answer: The provided context does not include specific information about Noveum's pricing plans. It mentions that all users will have access to the Observability suite for free, and early users will receive f...", + "rag.context_length": 2105, + "rag.sources_count": 5, + "rag.answer_length": 441, + "rag.mode": "retrieval_augmented_generation" + }, + "events": [], + "links": [], + "trace_trace_id": "29a862bf-d7aa-4cf9-963c-0a4244343bbe", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:56:54.718000000", + "trace_end_time": "2025-10-22 16:56:58.207000000", + "trace_duration_ms": 3488.068, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:56:54.718000000", + "updated_at": "2025-10-22 16:56:54.718000000" + }, + { + "span_id": "c8d7bb28-e763-47c9-b3e9-77f6a91a22be", + "trace_id": "29a862bf-d7aa-4cf9-963c-0a4244343bbe", + "parent_span_id": "7cda4dab-0fc5-4cae-8d8c-9445f661c67b", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-22 16:56:54.719000000", + "end_time": "2025-10-22 16:56:54.719000000", + "duration_ms": 0.068, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What are Noveum's pricing plans?", + "query_length": 32, + "query_type": "routing_evaluation_query", + "classification.mode": "RAG", + "classification.rag_keyword_score": 3, + "classification.web_keyword_score": 0, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "noveum_specific", + "query.keyword_density": 0.6, + "query.domain_affinity": "noveum", + "routing.decision": "Routed to RAG based on analysis", + "routing.rationale": "RAG score: 3, Web score: 0, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "Web Search", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.057692307692307696, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: RAG (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "29a862bf-d7aa-4cf9-963c-0a4244343bbe", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:56:54.718000000", + "trace_end_time": "2025-10-22 16:56:58.207000000", + "trace_duration_ms": 3488.068, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:56:54.718000000", + "updated_at": "2025-10-22 16:56:54.718000000" + }, + { + "span_id": "a378123a-8a79-4bc5-8e20-8ec54c2f9783", + "trace_id": "29a862bf-d7aa-4cf9-963c-0a4244343bbe", + "parent_span_id": "922c84a5-b14a-4b2f-8759-427d6e089812", + "name": "agent.llm_model_execution", + "start_time": "2025-10-22 16:56:55.154000000", + "end_time": "2025-10-22 16:56:57.166000000", + "duration_ms": 2012.174, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What are Noveum's pricing plans?", + "query_length": 32, + "query_type": "rag_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 2.012089252471924, + "model.latency_ms": 2012.0892524719238, + "model.start_time": 1761152215.154625, + "model.end_time": 1761152217.1667142, + "model.performance_tier": "medium", + "model.prompt_tokens": 658, + "model.completion_tokens": 110, + "model.total_tokens": 768, + "model.tokens_per_second": 381.6928096288395, + "model.estimated_cost": 0.02304, + "model.efficiency_score": 0.57421875, + "model.response_length": 441, + "model.response_quality": "high", + "model.output_response": "Model Response: The provided context does not include specific information about Noveum's pricing plans. It mentions that all users will have access to the Observability suite for free, and early users will receive f...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "29a862bf-d7aa-4cf9-963c-0a4244343bbe", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:56:54.718000000", + "trace_end_time": "2025-10-22 16:56:58.207000000", + "trace_duration_ms": 3488.068, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:56:54.718000000", + "updated_at": "2025-10-22 16:56:54.718000000" + }, + { + "span_id": "5813673b-a1dc-46c7-9bea-92ce6aca2b5e", + "trace_id": "29a862bf-d7aa-4cf9-963c-0a4244343bbe", + "parent_span_id": "922c84a5-b14a-4b2f-8759-427d6e089812", + "name": "agent.rag_evaluation_metrics", + "start_time": "2025-10-22 16:56:57.166000000", + "end_time": "2025-10-22 16:56:57.167000000", + "duration_ms": 0.059, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "rag_evaluation_metrics", + "agent.capabilities": [ + "retrieval_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What are Noveum's pricing plans?", + "query_length": 32, + "query_type": "rag_evaluation_query", + "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....", + "retrieval.context_length": 2105, + "retrieval.sources_count": 5, + "retrieval.context_quality": "high", + "retrieval.effectiveness": 1, + "retrieval.context_utilization": 2.105, + "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nSource 2 (https://noveum.ai/en/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nSource 3 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nThe AI landscape is complex and moving fast. With Noveum.ai, you'll have the insights you need to build, debug, and optimize AI applications with confidence.\nReady to see what's really happening inside your AI applications? Try Noveum.ai today and transform how you build and monitor AI systems.\nLet's build more reliable, observable AI\u2014together.\nGet Early Access to Noveum.ai Platform\nJoin the select group of AI teams optimizing their models with our data-driven platform. We're onboarding users in...\n\nSource 4 (https://noveum.ai/en/blog/from-logs-to-intelligent-choices-inside-noveum-ais-evaluation-process):\nGet Early Access to Noveum.ai Platform\nJoin the select group of AI teams optimizing their models with our data-driven platform. We're onboarding users in limited batches to ensure a premium experience....\n\nSource 5 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, vector searches, data retrieval, agent reasoning, and more. Without proper observability, debugging becomes a nightmare, optimization is guesswork, and understanding user interactions is nearly impossible.\nNoveum.ai solves this by providing comprehensive tracing and observability specifically designed for AI applications.\nKey benefits of using Noveum.ai:\n- Complete Visibility: Trace...\n\nUser Question: What are Noveum's pricing plans?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", + "prompt.prompt_length": 2633, + "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to ...", + "prompt.instruction_following": "rag_optimized", + "response.answer_length": 441, + "response.answer_completeness": "complete", + "response.response_quality": "high", + "response.source_citation": 5, + "response.context_utilization": 0.20950118764845604, + "output_response": "RAG Answer: The provided context does not include specific information about Noveum's pricing plans. It mentions that all users will have access to the Observability suite for free, and early users will receive f...", + "evaluation.retrieval_effectiveness": 1, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 5, + "evaluation.context_relevance": "high", + "evaluation.overall_quality": "high", + "evaluation.ready_for_production": true, + "rag.retrieval_strategy": "semantic_similarity", + "rag.vector_search_results": 5, + "rag.context_synthesis": "multi_source", + "rag.document_coverage": 1, + "rag.information_density": 0.20950118764845604 + }, + "events": [], + "links": [], + "trace_trace_id": "29a862bf-d7aa-4cf9-963c-0a4244343bbe", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:56:54.718000000", + "trace_end_time": "2025-10-22 16:56:58.207000000", + "trace_duration_ms": 3488.068, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:56:54.718000000", + "updated_at": "2025-10-22 16:56:54.718000000" + }, + { + "span_id": "c5baf5c2-d18a-458a-b922-a19b96db53d9", + "trace_id": "bc60c122-f30b-4114-9f78-ee40a024b525", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-22 16:56:44.873000000", + "end_time": "2025-10-22 16:56:53.701000000", + "duration_ms": 8827.979, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "How do I integrate Noveum Trace in my application?", + "query.length": 50, + "processing.mode": "RAG", + "processing.time_seconds": 8.827935218811035, + "processing.response_length": 1145, + "processing.sources_count": 5, + "output_response": "Final Answer: To integrate Noveum Trace into your application, you can follow these steps based on the SDK Integration Guide:\n\n1. **Create Your Account & Get API Key**:\n - Sign up at [noveum.ai](https://noveum.ai...", + "final_answer_mode": "RAG", + "query_processed.input_query": "How do I integrate Noveum Trace in my application?", + "query_processed.output_response": "Successfully processed query using RAG, generated 1145 character response", + "query_processed.mode": "RAG", + "query_processed.processing_time": 8.827935218811035, + "query_processed.response_length": 1145 + }, + "events": [], + "links": [], + "trace_trace_id": "bc60c122-f30b-4114-9f78-ee40a024b525", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:56:44.873000000", + "trace_end_time": "2025-10-22 16:56:53.701000000", + "trace_duration_ms": 8828.055, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:56:44.873000000", + "updated_at": "2025-10-22 16:56:44.873000000" + }, + { + "span_id": "8819dde5-9cbe-4cdb-b919-9c5140582359", + "trace_id": "bc60c122-f30b-4114-9f78-ee40a024b525", + "parent_span_id": "c5baf5c2-d18a-458a-b922-a19b96db53d9", + "name": "agent.query_routing", + "start_time": "2025-10-22 16:56:44.873000000", + "end_time": "2025-10-22 16:56:53.701000000", + "duration_ms": 8827.883, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "How do I integrate Noveum Trace in my application?", + "query_length": 50, + "query_type": "routing_query", + "output_response": "Routed to RAG for query processing", + "router.classification": "RAG", + "router.confidence_score": 1, + "router.rag_keyword_score": 2, + "router.web_keyword_score": 0, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "bc60c122-f30b-4114-9f78-ee40a024b525", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:56:44.873000000", + "trace_end_time": "2025-10-22 16:56:53.701000000", + "trace_duration_ms": 8828.055, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:56:44.873000000", + "updated_at": "2025-10-22 16:56:44.873000000" + }, + { + "span_id": "932c4b0e-0f33-40c0-bb19-78b9f81e48a6", + "trace_id": "bc60c122-f30b-4114-9f78-ee40a024b525", + "parent_span_id": "8819dde5-9cbe-4cdb-b919-9c5140582359", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-22 16:56:44.873000000", + "end_time": "2025-10-22 16:56:44.873000000", + "duration_ms": 0.043000000000000003, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: How do I integrate Noveum Trace in my application?", + "query_length": 50, + "query_type": "routing_evaluation_query", + "classification.mode": "RAG", + "classification.rag_keyword_score": 2, + "classification.web_keyword_score": 0, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "noveum_specific", + "query.keyword_density": 0.2222222222222222, + "query.domain_affinity": "noveum", + "routing.decision": "Routed to RAG based on analysis", + "routing.rationale": "RAG score: 2, Web score: 0, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "Web Search", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.038461538461538464, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: RAG (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "bc60c122-f30b-4114-9f78-ee40a024b525", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:56:44.873000000", + "trace_end_time": "2025-10-22 16:56:53.701000000", + "trace_duration_ms": 8828.055, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:56:44.873000000", + "updated_at": "2025-10-22 16:56:44.873000000" + }, + { + "span_id": "2ece2bd6-e4c8-47bc-b3c8-26f82e9910f1", + "trace_id": "bc60c122-f30b-4114-9f78-ee40a024b525", + "parent_span_id": "8819dde5-9cbe-4cdb-b919-9c5140582359", + "name": "agent.llm-rag", + "start_time": "2025-10-22 16:56:44.873000000", + "end_time": "2025-10-22 16:56:53.701000000", + "duration_ms": 8827.666000000001, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "rag_agent", + "agent.operation": "llm-rag", + "agent.capabilities": [ + "document_retrieval", + "context_generation", + "response_generation" + ], + "agent.id": "noveum_rag_agent", + "input_query": "How do I integrate Noveum Trace in my application?", + "query_length": 50, + "query_type": "rag_query", + "output_response": "RAG Answer: To integrate Noveum Trace into your application, you can follow these steps based on the SDK Integration Guide:\n\n1. **Create Your Account & Get API Key**:\n - Sign up at [noveum.ai](https://noveum.ai...", + "rag.context_length": 2803, + "rag.sources_count": 5, + "rag.answer_length": 1145, + "rag.mode": "retrieval_augmented_generation" + }, + "events": [], + "links": [], + "trace_trace_id": "bc60c122-f30b-4114-9f78-ee40a024b525", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:56:44.873000000", + "trace_end_time": "2025-10-22 16:56:53.701000000", + "trace_duration_ms": 8828.055, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:56:44.873000000", + "updated_at": "2025-10-22 16:56:44.873000000" + }, + { + "span_id": "e6315110-36b1-4503-beaf-12d972e25a7d", + "trace_id": "bc60c122-f30b-4114-9f78-ee40a024b525", + "parent_span_id": "2ece2bd6-e4c8-47bc-b3c8-26f82e9910f1", + "name": "agent.llm_model_execution", + "start_time": "2025-10-22 16:56:45.576000000", + "end_time": "2025-10-22 16:56:53.143000000", + "duration_ms": 7566.311, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: How do I integrate Noveum Trace in my application?", + "query_length": 50, + "query_type": "rag_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 7.566190004348755, + "model.latency_ms": 7566.190004348755, + "model.start_time": 1761152205.576988, + "model.end_time": 1761152213.143178, + "model.performance_tier": "slow", + "model.prompt_tokens": 837, + "model.completion_tokens": 286, + "model.total_tokens": 1123, + "model.tokens_per_second": 148.4234468543009, + "model.estimated_cost": 0.03369, + "model.efficiency_score": 1.0195903829029385, + "model.response_length": 1145, + "model.response_quality": "high", + "model.output_response": "Model Response: To integrate Noveum Trace into your application, you can follow these steps based on the SDK Integration Guide:\n\n1. **Create Your Account & Get API Key**:\n - Sign up at [noveum.ai](https://noveum.ai...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "bc60c122-f30b-4114-9f78-ee40a024b525", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:56:44.873000000", + "trace_end_time": "2025-10-22 16:56:53.701000000", + "trace_duration_ms": 8828.055, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:56:44.873000000", + "updated_at": "2025-10-22 16:56:44.873000000" + }, + { + "span_id": "ae91d520-8442-48e5-aabf-fd58aed23694", + "trace_id": "bc60c122-f30b-4114-9f78-ee40a024b525", + "parent_span_id": "2ece2bd6-e4c8-47bc-b3c8-26f82e9910f1", + "name": "agent.rag_evaluation_metrics", + "start_time": "2025-10-22 16:56:53.143000000", + "end_time": "2025-10-22 16:56:53.143000000", + "duration_ms": 0.091, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "rag_evaluation_metrics", + "agent.capabilities": [ + "retrieval_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: How do I integrate Noveum Trace in my application?", + "query_length": 50, + "query_type": "rag_evaluation_query", + "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\n\ud83d\udcd8 TypeScript SDK (\n@noveum/trace\n)- Framework ...", + "retrieval.context_length": 2803, + "retrieval.sources_count": 5, + "retrieval.context_quality": "high", + "retrieval.effectiveness": 1, + "retrieval.context_utilization": 2.803, + "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\n\ud83d\udcd8 TypeScript SDK (\n@noveum/trace\n)- Framework integrations for Next.js, Express.js, and Hono\n- TypeScript-first design with full type safety\n- Universal compatibility (Node.js, Edge Runtime, browsers)\n-\n\ud83d\udcca Noveum Platform\n- Real-time dashboard for analyzing traces and performance\n- Ad...\n\nSource 2 (https://noveum.ai/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\n\ud83d\udcd8 TypeScript SDK (\n@noveum/trace\n)- Framework integrations for Next.js, Express.js, and Hono\n- TypeScript-first design with full type safety\n- Universal compatibility (Node.js, Edge Runtime, browsers)\n-\n\ud83d\udcca Noveum Platform\n- Real-time dashboard for analyzing traces and performance\n- Ad...\n\nSource 3 (https://noveum.ai/docs/getting-started/sdk-integration):\nSDK Integration Guide\nIntegrate Noveum.ai tracing into your AI applications with Python or TypeScript SDKs\nThe Noveum.ai SDKs provide comprehensive tracing and observability for your AI applications with minimal code changes. Whether you're building LLM applications, RAG systems, or multi-agent workflows, our SDKs automatically capture essential metrics and traces.\n\ud83d\ude80 Quick Start\n1. Create Your Account & Get API Key\n- Sign up at noveum.ai\n- Create a project in your dashboard\n- Generate an API key...\n\nSource 4 (https://noveum.ai/en/docs/getting-started/sdk-integration):\nSDK Integration Guide\nIntegrate Noveum.ai tracing into your AI applications with Python or TypeScript SDKs\nThe Noveum.ai SDKs provide comprehensive tracing and observability for your AI applications with minimal code changes. Whether you're building LLM applications, RAG systems, or multi-agent workflows, our SDKs automatically capture essential metrics and traces.\n\ud83d\ude80 Quick Start\n1. Create Your Account & Get API Key\n- Sign up at noveum.ai\n- Create a project in your dashboard\n- Generate an API key...\n\nSource 5 (https://noveum.ai/en/docs/getting-started/sdk-integration):\n\ud83d\udd27 Framework Integrations\nNext.js Integration\nExpress.js Integration\nFastAPI Integration (Python)\n\ud83d\udcca Advanced Features\nCustom Attributes & Events\nSampling Configuration\n\ud83d\udd17 What's Captured Automatically\n- \ud83d\udcca Performance Metrics: Latency, throughput, error rates\n- \ud83d\udcb0 Cost Tracking: Token usage, API costs across providers\n- \ud83d\udd0d Request/Response: Configurable capture of inputs/outputs\n- \ud83c\udff7\ufe0f Metadata: Model names, parameters, user context\n- \ud83c\udf0a Context Flow: Trace relationships across services\n- \ud83d\udc1b Error Detail...\n\nUser Question: How do I integrate Noveum Trace in my application?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", + "prompt.prompt_length": 3349, + "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frame...", + "prompt.instruction_following": "rag_optimized", + "response.answer_length": 1145, + "response.answer_completeness": "complete", + "response.response_quality": "high", + "response.source_citation": 5, + "response.context_utilization": 0.4084909026043525, + "output_response": "RAG Answer: To integrate Noveum Trace into your application, you can follow these steps based on the SDK Integration Guide:\n\n1. **Create Your Account & Get API Key**:\n - Sign up at [noveum.ai](https://noveum.ai...", + "evaluation.retrieval_effectiveness": 1, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 5, + "evaluation.context_relevance": "high", + "evaluation.overall_quality": "high", + "evaluation.ready_for_production": true, + "rag.retrieval_strategy": "semantic_similarity", + "rag.vector_search_results": 5, + "rag.context_synthesis": "multi_source", + "rag.document_coverage": 1, + "rag.information_density": 0.4084909026043525 + }, + "events": [], + "links": [], + "trace_trace_id": "bc60c122-f30b-4114-9f78-ee40a024b525", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:56:44.873000000", + "trace_end_time": "2025-10-22 16:56:53.701000000", + "trace_duration_ms": 8828.055, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:56:44.873000000", + "updated_at": "2025-10-22 16:56:44.873000000" + }, + { + "span_id": "9a9c611d-f552-4b06-a764-4463edb70e2c", + "trace_id": "3ad192e5-bbd4-4236-8144-019d53512769", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-22 16:56:35.188000000", + "end_time": "2025-10-22 16:56:43.863000000", + "duration_ms": 8675.25, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What is Noveum and what does it do?", + "query.length": 35, + "processing.mode": "RAG", + "processing.time_seconds": 8.67501425743103, + "processing.response_length": 527, + "processing.sources_count": 5, + "output_response": "Final Answer: Noveum.ai is a comprehensive tracing and observability platform specifically designed for AI applications, including LLM-powered chatbots, RAG systems, and multi-agent workflows. It provides the neces...", + "final_answer_mode": "RAG", + "query_processed.input_query": "What is Noveum and what does it do?", + "query_processed.output_response": "Successfully processed query using RAG, generated 527 character response", + "query_processed.mode": "RAG", + "query_processed.processing_time": 8.67501425743103, + "query_processed.response_length": 527 + }, + "events": [], + "links": [], + "trace_trace_id": "3ad192e5-bbd4-4236-8144-019d53512769", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:56:35.188000000", + "trace_end_time": "2025-10-22 16:56:43.863000000", + "trace_duration_ms": 8675.356000000002, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:56:35.188000000", + "updated_at": "2025-10-22 16:56:35.188000000" + }, + { + "span_id": "99eadeae-0733-4bf3-982e-e9f3014f5913", + "trace_id": "3ad192e5-bbd4-4236-8144-019d53512769", + "parent_span_id": "7e2c959c-157a-43d5-9870-32994cddd388", + "name": "agent.llm-rag", + "start_time": "2025-10-22 16:56:35.188000000", + "end_time": "2025-10-22 16:56:43.863000000", + "duration_ms": 8674.699999999999, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "rag_agent", + "agent.operation": "llm-rag", + "agent.capabilities": [ + "document_retrieval", + "context_generation", + "response_generation" + ], + "agent.id": "noveum_rag_agent", + "input_query": "What is Noveum and what does it do?", + "query_length": 35, + "query_type": "rag_query", + "output_response": "RAG Answer: Noveum.ai is a comprehensive tracing and observability platform specifically designed for AI applications, including LLM-powered chatbots, RAG systems, and multi-agent workflows. It provides the neces...", + "rag.context_length": 2334, + "rag.sources_count": 5, + "rag.answer_length": 527, + "rag.mode": "retrieval_augmented_generation" + }, + "events": [], + "links": [], + "trace_trace_id": "3ad192e5-bbd4-4236-8144-019d53512769", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:56:35.188000000", + "trace_end_time": "2025-10-22 16:56:43.863000000", + "trace_duration_ms": 8675.356000000002, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:56:35.188000000", + "updated_at": "2025-10-22 16:56:35.188000000" + }, + { + "span_id": "7e2c959c-157a-43d5-9870-32994cddd388", + "trace_id": "3ad192e5-bbd4-4236-8144-019d53512769", + "parent_span_id": "9a9c611d-f552-4b06-a764-4463edb70e2c", + "name": "agent.query_routing", + "start_time": "2025-10-22 16:56:35.188000000", + "end_time": "2025-10-22 16:56:43.863000000", + "duration_ms": 8674.961, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What is Noveum and what does it do?", + "query_length": 35, + "query_type": "routing_query", + "output_response": "Routed to RAG for query processing", + "router.classification": "RAG", + "router.confidence_score": 1, + "router.rag_keyword_score": 1, + "router.web_keyword_score": 0, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "3ad192e5-bbd4-4236-8144-019d53512769", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:56:35.188000000", + "trace_end_time": "2025-10-22 16:56:43.863000000", + "trace_duration_ms": 8675.356000000002, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:56:35.188000000", + "updated_at": "2025-10-22 16:56:35.188000000" + }, + { + "span_id": "6f81284c-a9aa-4d62-a402-1b474f2eba51", + "trace_id": "3ad192e5-bbd4-4236-8144-019d53512769", + "parent_span_id": "7e2c959c-157a-43d5-9870-32994cddd388", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-22 16:56:35.188000000", + "end_time": "2025-10-22 16:56:35.188000000", + "duration_ms": 0.054, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What is Noveum and what does it do?", + "query_length": 35, + "query_type": "routing_evaluation_query", + "classification.mode": "RAG", + "classification.rag_keyword_score": 1, + "classification.web_keyword_score": 0, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "noveum_specific", + "query.keyword_density": 0.125, + "query.domain_affinity": "noveum", + "routing.decision": "Routed to RAG based on analysis", + "routing.rationale": "RAG score: 1, Web score: 0, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "Web Search", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.019230769230769232, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: RAG (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "3ad192e5-bbd4-4236-8144-019d53512769", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:56:35.188000000", + "trace_end_time": "2025-10-22 16:56:43.863000000", + "trace_duration_ms": 8675.356000000002, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:56:35.188000000", + "updated_at": "2025-10-22 16:56:35.188000000" + }, + { + "span_id": "9f42d18a-0235-4ae2-838c-19a88ddc95e1", + "trace_id": "3ad192e5-bbd4-4236-8144-019d53512769", + "parent_span_id": "99eadeae-0733-4bf3-982e-e9f3014f5913", + "name": "agent.llm_model_execution", + "start_time": "2025-10-22 16:56:36.031000000", + "end_time": "2025-10-22 16:56:42.012000000", + "duration_ms": 5980.947999999999, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What is Noveum and what does it do?", + "query_length": 35, + "query_type": "rag_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 5.980915784835815, + "model.latency_ms": 5980.915784835815, + "model.start_time": 1761152196.031816, + "model.end_time": 1761152202.0127318, + "model.performance_tier": "slow", + "model.prompt_tokens": 716, + "model.completion_tokens": 131, + "model.total_tokens": 847, + "model.tokens_per_second": 141.61710856178715, + "model.estimated_cost": 0.025410000000000002, + "model.efficiency_score": 0.6221959858323495, + "model.response_length": 527, + "model.response_quality": "high", + "model.output_response": "Model Response: Noveum.ai is a comprehensive tracing and observability platform specifically designed for AI applications, including LLM-powered chatbots, RAG systems, and multi-agent workflows. It provides the neces...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "3ad192e5-bbd4-4236-8144-019d53512769", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:56:35.188000000", + "trace_end_time": "2025-10-22 16:56:43.863000000", + "trace_duration_ms": 8675.356000000002, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:56:35.188000000", + "updated_at": "2025-10-22 16:56:35.188000000" + }, + { + "span_id": "7bcbe4b7-585a-4c9d-add8-16e8aa62254f", + "trace_id": "3ad192e5-bbd4-4236-8144-019d53512769", + "parent_span_id": "99eadeae-0733-4bf3-982e-e9f3014f5913", + "name": "agent.rag_evaluation_metrics", + "start_time": "2025-10-22 16:56:42.012000000", + "end_time": "2025-10-22 16:56:42.012000000", + "duration_ms": 0.018000000000000002, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "rag_evaluation_metrics", + "agent.capabilities": [ + "retrieval_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What is Noveum and what does it do?", + "query_length": 35, + "query_type": "rag_evaluation_query", + "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, vector searches, data retrieval, agent reasoning, and more. Without proper observability, debugging ...", + "retrieval.context_length": 2334, + "retrieval.sources_count": 5, + "retrieval.context_quality": "high", + "retrieval.effectiveness": 1, + "retrieval.context_utilization": 2.334, + "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, vector searches, data retrieval, agent reasoning, and more. Without proper observability, debugging becomes a nightmare, optimization is guesswork, and understanding user interactions is nearly impossible.\nNoveum.ai solves this by providing comprehensive tracing and observability specifically designed for AI applications.\nKey benefits of using Noveum.ai:\n- Complete Visibility: Trace...\n\nSource 2 (https://noveum.ai/en/docs/getting-started/overview):\nNoveum.ai Overview\nComprehensive AI tracing and observability platform for LLM applications, RAG systems, and AI agents\nWelcome to Noveum.ai\u2014the comprehensive tracing and observability platform built specifically for AI applications. Whether you're building LLM-powered chatbots, RAG systems, multi-agent workflows, or any AI-driven application, Noveum provides the insights you need to understand, debug, and optimize your systems.\n\ud83c\udfaf Why AI Applications Need Specialized Observability\nTraditional mo...\n\nSource 3 (https://noveum.ai/docs/getting-started/overview):\nNoveum.ai Overview\nComprehensive AI tracing and observability platform for LLM applications, RAG systems, and AI agents\nWelcome to Noveum.ai\u2014the comprehensive tracing and observability platform built specifically for AI applications. Whether you're building LLM-powered chatbots, RAG systems, multi-agent workflows, or any AI-driven application, Noveum provides the insights you need to understand, debug, and optimize your systems.\n\ud83c\udfaf Why AI Applications Need Specialized Observability\nTraditional mo...\n\nSource 4 (https://noveum.ai/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nSource 5 (https://noveum.ai/en/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nUser Question: What is Noveum and what does it do?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", + "prompt.prompt_length": 2865, + "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, ...", + "prompt.instruction_following": "rag_optimized", + "response.answer_length": 527, + "response.answer_completeness": "complete", + "response.response_quality": "high", + "response.source_citation": 5, + "response.context_utilization": 0.22579263067694944, + "output_response": "RAG Answer: Noveum.ai is a comprehensive tracing and observability platform specifically designed for AI applications, including LLM-powered chatbots, RAG systems, and multi-agent workflows. It provides the neces...", + "evaluation.retrieval_effectiveness": 1, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 5, + "evaluation.context_relevance": "high", + "evaluation.overall_quality": "high", + "evaluation.ready_for_production": true, + "rag.retrieval_strategy": "semantic_similarity", + "rag.vector_search_results": 5, + "rag.context_synthesis": "multi_source", + "rag.document_coverage": 1, + "rag.information_density": 0.22579263067694944 + }, + "events": [], + "links": [], + "trace_trace_id": "3ad192e5-bbd4-4236-8144-019d53512769", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:56:35.188000000", + "trace_end_time": "2025-10-22 16:56:43.863000000", + "trace_duration_ms": 8675.356000000002, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:56:35.188000000", + "updated_at": "2025-10-22 16:56:35.188000000" + }, + { + "span_id": "d39297f4-b4fe-4f9c-b45b-854411ba770a", + "trace_id": "9af2b4b0-0c2f-4d2e-ae99-30df0d2f0fa6", + "parent_span_id": "", + "name": "system_initialization", + "start_time": "2025-10-22 16:56:34.565000000", + "end_time": "2025-10-22 16:56:35.186000000", + "duration_ms": 621.02, + "status": "ok", + "status_message": "", + "attributes": { + "system.force_scrape": false, + "system.config": { + "noveum_base_url": "https://noveum.ai", + "max_pages_to_scrape": 50, + "chunk_size": 1000, + "chunk_overlap": 200, + "max_search_results": 5, + "rag_threshold": 0.7, + "noveum_docs_file": "noveum_docs.json", + "vector_store_path": "noveum_vectorstore" + }, + "input_query": "Initialize system with force_scrape=False", + "output_response": "System initialization: RAG system loaded, vector store ready, agent operational", + "system.initialized": true, + "system.documents_loaded": 38, + "system.vectorstore_ready": true + }, + "events": [], + "links": [], + "trace_trace_id": "9af2b4b0-0c2f-4d2e-ae99-30df0d2f0fa6", + "trace_name": "auto_trace_system_initialization", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-22 16:56:34.565000000", + "trace_end_time": "2025-10-22 16:56:35.186000000", + "trace_duration_ms": 621.177, + "span_count": 1, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-22 16:56:34.565000000", + "updated_at": "2025-10-22 16:56:34.565000000" + }, + { + "span_id": "8c144ad8-a439-4986-b741-3bc4b7fc6110", + "trace_id": "bae13042-cf2e-4813-a673-7dc2a611725e", + "parent_span_id": "d37fe1a1-a126-4e4e-a914-93dd402fed3f", + "name": "agent.query_routing", + "start_time": "2025-10-16 08:50:56.153000000", + "end_time": "2025-10-16 08:51:01.908000000", + "duration_ms": 5754.43, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What are the latest breakthroughs in artificial intelligence?", + "query_length": 61, + "query_type": "routing_query", + "output_response": "Routed to Web Search for query processing", + "router.classification": "Web Search", + "router.confidence_score": 1, + "router.rag_keyword_score": 0, + "router.web_keyword_score": 1, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "bae13042-cf2e-4813-a673-7dc2a611725e", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:56.153000000", + "trace_end_time": "2025-10-16 08:51:01.908000000", + "trace_duration_ms": 5754.578, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:56.153000000", + "updated_at": "2025-10-16 08:50:56.153000000" + }, + { + "span_id": "b294f9b7-7762-47a9-b49d-00801a984af2", + "trace_id": "bae13042-cf2e-4813-a673-7dc2a611725e", + "parent_span_id": "8c144ad8-a439-4986-b741-3bc4b7fc6110", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-16 08:50:56.153000000", + "end_time": "2025-10-16 08:50:56.153000000", + "duration_ms": 0.024, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What are the latest breakthroughs in artificial intelligence?", + "query_length": 61, + "query_type": "routing_evaluation_query", + "classification.mode": "Web Search", + "classification.rag_keyword_score": 0, + "classification.web_keyword_score": 1, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "complex", + "query.intent": "general_knowledge", + "query.keyword_density": 0.125, + "query.domain_affinity": "general", + "routing.decision": "Routed to Web Search based on analysis", + "routing.rationale": "RAG score: 0, Web score: 1, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "RAG", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.019230769230769232, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: Web Search (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "bae13042-cf2e-4813-a673-7dc2a611725e", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:56.153000000", + "trace_end_time": "2025-10-16 08:51:01.908000000", + "trace_duration_ms": 5754.578, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:56.153000000", + "updated_at": "2025-10-16 08:50:56.153000000" + }, + { + "span_id": "ab6a281a-4989-40c8-9011-d0ec7dde7ea9", + "trace_id": "bae13042-cf2e-4813-a673-7dc2a611725e", + "parent_span_id": "8c144ad8-a439-4986-b741-3bc4b7fc6110", + "name": "agent.web_search_generation", + "start_time": "2025-10-16 08:50:56.153000000", + "end_time": "2025-10-16 08:51:01.908000000", + "duration_ms": 5754.317999999999, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "web_search_agent", + "agent.operation": "web_search_generation", + "agent.capabilities": [ + "web_search", + "content_synthesis", + "response_generation" + ], + "agent.id": "noveum_web_search_agent", + "input_query": "What are the latest breakthroughs in artificial intelligence?", + "query_length": 61, + "query_type": "web_search_query", + "output_response": "Web Search Answer: The web search did not yield specific results regarding the latest breakthroughs in artificial intelligence. However, I can provide a general overview based on knowledge up to October 2023.\n\nRecent br...", + "web_search.results_count": 1, + "web_search.context_length": 163, + "web_search.response_length": 1489, + "web_search.mode": "external_web_search" + }, + "events": [], + "links": [], + "trace_trace_id": "bae13042-cf2e-4813-a673-7dc2a611725e", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:56.153000000", + "trace_end_time": "2025-10-16 08:51:01.908000000", + "trace_duration_ms": 5754.578, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:56.153000000", + "updated_at": "2025-10-16 08:50:56.153000000" + }, + { + "span_id": "d37fe1a1-a126-4e4e-a914-93dd402fed3f", + "trace_id": "bae13042-cf2e-4813-a673-7dc2a611725e", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-16 08:50:56.153000000", + "end_time": "2025-10-16 08:51:01.908000000", + "duration_ms": 5754.527, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What are the latest breakthroughs in artificial intelligence?", + "query.length": 61, + "processing.mode": "Web Search", + "processing.time_seconds": 5.754457473754883, + "processing.response_length": 1489, + "processing.sources_count": 1, + "output_response": "Final Answer: The web search did not yield specific results regarding the latest breakthroughs in artificial intelligence. However, I can provide a general overview based on knowledge up to October 2023.\n\nRecent br...", + "final_answer_mode": "Web Search", + "query_processed.input_query": "What are the latest breakthroughs in artificial intelligence?", + "query_processed.output_response": "Successfully processed query using Web Search, generated 1489 character response", + "query_processed.mode": "Web Search", + "query_processed.processing_time": 5.754457473754883, + "query_processed.response_length": 1489 + }, + "events": [], + "links": [], + "trace_trace_id": "bae13042-cf2e-4813-a673-7dc2a611725e", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:56.153000000", + "trace_end_time": "2025-10-16 08:51:01.908000000", + "trace_duration_ms": 5754.578, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:56.153000000", + "updated_at": "2025-10-16 08:50:56.153000000" + }, + { + "span_id": "f61ef76d-4887-42da-9fd6-e2cdb25a9e09", + "trace_id": "bae13042-cf2e-4813-a673-7dc2a611725e", + "parent_span_id": "ab6a281a-4989-40c8-9011-d0ec7dde7ea9", + "name": "agent.llm_model_execution", + "start_time": "2025-10-16 08:50:56.502000000", + "end_time": "2025-10-16 08:51:01.908000000", + "duration_ms": 5405.44, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What are the latest breakthroughs in artificial intelligence?", + "query_length": 61, + "query_type": "web_search_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 5.405392408370972, + "model.latency_ms": 5405.392408370972, + "model.start_time": 1760604656.5026655, + "model.end_time": 1760604661.908058, + "model.performance_tier": "slow", + "model.prompt_tokens": 167, + "model.completion_tokens": 372, + "model.total_tokens": 539, + "model.tokens_per_second": 99.71523976044487, + "model.estimated_cost": 0.01617, + "model.efficiency_score": 2.7625231910946195, + "model.response_length": 1489, + "model.response_quality": "high", + "model.output_response": "Model Response: The web search did not yield specific results regarding the latest breakthroughs in artificial intelligence. However, I can provide a general overview based on knowledge up to October 2023.\n\nRecent br...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "bae13042-cf2e-4813-a673-7dc2a611725e", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:56.153000000", + "trace_end_time": "2025-10-16 08:51:01.908000000", + "trace_duration_ms": 5754.578, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:56.153000000", + "updated_at": "2025-10-16 08:50:56.153000000" + }, + { + "span_id": "550bf5ae-c816-4ef0-a6c1-1553b1c258b8", + "trace_id": "bae13042-cf2e-4813-a673-7dc2a611725e", + "parent_span_id": "ab6a281a-4989-40c8-9011-d0ec7dde7ea9", + "name": "agent.web_search_evaluation_metrics", + "start_time": "2025-10-16 08:51:01.908000000", + "end_time": "2025-10-16 08:51:01.908000000", + "duration_ms": 0.018000000000000002, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "web_search_evaluation_metrics", + "agent.capabilities": [ + "web_search_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What are the latest breakthroughs in artificial intelligence?", + "query_length": 61, + "query_type": "web_search_evaluation_query", + "web_search.results_count": 1, + "web_search.context_length": 163, + "web_search.context_synthesized": "Context from 1 web sources", + "web_search.search_effectiveness": 0.2, + "web_search.context_quality": "low", + "web_search.source_diversity": 1, + "web_search.information_synthesis": "low", + "web_search.external_knowledge_utilization": 0.163, + "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What+are+the+latest+breakthroughs+in+artificial+intelligence?\n\nUser Question: What are the latest breakthroughs in artificial intelligence?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", + "prompt.prompt_length": 669, + "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What+are+the+latest+breakthroughs+in+artificial+intelligence?", + "prompt.instruction_following": "web_search_optimized", + "response.answer_length": 1489, + "response.answer_completeness": "complete", + "response.response_quality": "medium", + "response.source_citation": 1, + "response.context_utilization": 9.134969325153374, + "output_response": "Web Search Answer: The web search did not yield specific results regarding the latest breakthroughs in artificial intelligence. However, I can provide a general overview based on knowledge up to October 2023.\n\nRecent br...", + "evaluation.search_effectiveness": 0.2, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 1, + "evaluation.context_relevance": "low", + "evaluation.overall_quality": "low", + "evaluation.ready_for_production": true, + "web_search.search_strategy": "duckduckgo_api", + "web_search.real_time_data": true, + "web_search.external_sources": 1, + "web_search.information_freshness": "current", + "web_search.knowledge_synthesis": "single_source" + }, + "events": [], + "links": [], + "trace_trace_id": "bae13042-cf2e-4813-a673-7dc2a611725e", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:56.153000000", + "trace_end_time": "2025-10-16 08:51:01.908000000", + "trace_duration_ms": 5754.578, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:56.153000000", + "updated_at": "2025-10-16 08:50:56.153000000" + }, + { + "span_id": "04cb7d29-9ea8-4ae0-9834-030cae944eca", + "trace_id": "79f9d235-d46e-4740-be35-ef0006071f6c", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-16 08:50:51.539000000", + "end_time": "2025-10-16 08:50:55.150000000", + "duration_ms": 3611.008, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What's happening in the software development world today?", + "query.length": 57, + "processing.mode": "Web Search", + "processing.time_seconds": 3.6109254360198975, + "processing.response_length": 526, + "processing.sources_count": 1, + "output_response": "Final Answer: The provided web search results do not contain specific information about current events or trends in the software development world. They primarily focus on unrelated topics, such as the meaning of \"...", + "final_answer_mode": "Web Search", + "query_processed.input_query": "What's happening in the software development world today?", + "query_processed.output_response": "Successfully processed query using Web Search, generated 526 character response", + "query_processed.mode": "Web Search", + "query_processed.processing_time": 3.6109254360198975, + "query_processed.response_length": 526 + }, + "events": [], + "links": [], + "trace_trace_id": "79f9d235-d46e-4740-be35-ef0006071f6c", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:51.539000000", + "trace_end_time": "2025-10-16 08:50:55.150000000", + "trace_duration_ms": 3611.084, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:51.539000000", + "updated_at": "2025-10-16 08:50:51.539000000" + }, + { + "span_id": "e5e3c927-783e-48c5-aec7-b32bae98c0a2", + "trace_id": "79f9d235-d46e-4740-be35-ef0006071f6c", + "parent_span_id": "f630c3e2-091e-4bd9-9556-e0ebc1c50291", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-16 08:50:51.539000000", + "end_time": "2025-10-16 08:50:51.540000000", + "duration_ms": 0.041, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What's happening in the software development world today?", + "query_length": 57, + "query_type": "routing_evaluation_query", + "classification.mode": "Web Search", + "classification.rag_keyword_score": 0, + "classification.web_keyword_score": 1, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "complex", + "query.intent": "general_knowledge", + "query.keyword_density": 0.125, + "query.domain_affinity": "general", + "routing.decision": "Routed to Web Search based on analysis", + "routing.rationale": "RAG score: 0, Web score: 1, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "RAG", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.019230769230769232, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: Web Search (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "79f9d235-d46e-4740-be35-ef0006071f6c", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:51.539000000", + "trace_end_time": "2025-10-16 08:50:55.150000000", + "trace_duration_ms": 3611.084, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:51.539000000", + "updated_at": "2025-10-16 08:50:51.539000000" + }, + { + "span_id": "f630c3e2-091e-4bd9-9556-e0ebc1c50291", + "trace_id": "79f9d235-d46e-4740-be35-ef0006071f6c", + "parent_span_id": "04cb7d29-9ea8-4ae0-9834-030cae944eca", + "name": "agent.query_routing", + "start_time": "2025-10-16 08:50:51.539000000", + "end_time": "2025-10-16 08:50:55.150000000", + "duration_ms": 3610.876, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What's happening in the software development world today?", + "query_length": 57, + "query_type": "routing_query", + "output_response": "Routed to Web Search for query processing", + "router.classification": "Web Search", + "router.confidence_score": 1, + "router.rag_keyword_score": 0, + "router.web_keyword_score": 1, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "79f9d235-d46e-4740-be35-ef0006071f6c", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:51.539000000", + "trace_end_time": "2025-10-16 08:50:55.150000000", + "trace_duration_ms": 3611.084, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:51.539000000", + "updated_at": "2025-10-16 08:50:51.539000000" + }, + { + "span_id": "d7b0f399-3b43-43bf-bac1-b2101268841c", + "trace_id": "79f9d235-d46e-4740-be35-ef0006071f6c", + "parent_span_id": "f630c3e2-091e-4bd9-9556-e0ebc1c50291", + "name": "agent.web_search_generation", + "start_time": "2025-10-16 08:50:51.540000000", + "end_time": "2025-10-16 08:50:55.150000000", + "duration_ms": 3610.679, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "web_search_agent", + "agent.operation": "web_search_generation", + "agent.capabilities": [ + "web_search", + "content_synthesis", + "response_generation" + ], + "agent.id": "noveum_web_search_agent", + "input_query": "What's happening in the software development world today?", + "query_length": 57, + "query_type": "web_search_query", + "output_response": "Web Search Answer: The provided web search results do not contain specific information about current events or trends in the software development world. They primarily focus on unrelated topics, such as the meaning of \"...", + "web_search.results_count": 1, + "web_search.context_length": 589, + "web_search.response_length": 526, + "web_search.mode": "external_web_search" + }, + "events": [], + "links": [], + "trace_trace_id": "79f9d235-d46e-4740-be35-ef0006071f6c", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:51.539000000", + "trace_end_time": "2025-10-16 08:50:55.150000000", + "trace_duration_ms": 3611.084, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:51.539000000", + "updated_at": "2025-10-16 08:50:51.539000000" + }, + { + "span_id": "4ed5dae4-f850-4644-aa55-2f1c00d11c22", + "trace_id": "79f9d235-d46e-4740-be35-ef0006071f6c", + "parent_span_id": "d7b0f399-3b43-43bf-bac1-b2101268841c", + "name": "agent.llm_model_execution", + "start_time": "2025-10-16 08:50:51.976000000", + "end_time": "2025-10-16 08:50:55.150000000", + "duration_ms": 3174.51, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What's happening in the software development world today?", + "query_length": 57, + "query_type": "web_search_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 3.174463987350464, + "model.latency_ms": 3174.463987350464, + "model.start_time": 1760604651.9761698, + "model.end_time": 1760604655.1506338, + "model.performance_tier": "medium", + "model.prompt_tokens": 272, + "model.completion_tokens": 131, + "model.total_tokens": 403, + "model.tokens_per_second": 126.95056601866197, + "model.estimated_cost": 0.01209, + "model.efficiency_score": 1.305210918114144, + "model.response_length": 526, + "model.response_quality": "high", + "model.output_response": "Model Response: The provided web search results do not contain specific information about current events or trends in the software development world. They primarily focus on unrelated topics, such as the meaning of \"...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "79f9d235-d46e-4740-be35-ef0006071f6c", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:51.539000000", + "trace_end_time": "2025-10-16 08:50:55.150000000", + "trace_duration_ms": 3611.084, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:51.539000000", + "updated_at": "2025-10-16 08:50:51.539000000" + }, + { + "span_id": "eb151834-9322-4f63-add0-dec9a2966367", + "trace_id": "79f9d235-d46e-4740-be35-ef0006071f6c", + "parent_span_id": "d7b0f399-3b43-43bf-bac1-b2101268841c", + "name": "agent.web_search_evaluation_metrics", + "start_time": "2025-10-16 08:50:55.150000000", + "end_time": "2025-10-16 08:50:55.150000000", + "duration_ms": 0.02, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "web_search_evaluation_metrics", + "agent.capabilities": [ + "web_search_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What's happening in the software development world today?", + "query_length": 57, + "query_type": "web_search_evaluation_query", + "web_search.results_count": 1, + "web_search.context_length": 589, + "web_search.context_synthesized": "Context from 1 web sources", + "web_search.search_effectiveness": 0.2, + "web_search.context_quality": "medium", + "web_search.source_diversity": 1, + "web_search.information_synthesis": "low", + "web_search.external_knowledge_utilization": 0.589, + "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\n\u66fc\u5df4\u7684\u610f\u601d\u662f\u4e00\u79cd\u6bd2\u86c7\uff0c\u8c61\u5f81\u7740\u79d1\u6bd4\u5728\u7403\u573a\u4e0a\u7684\u653b\u51fb\u6027\u548c\u575a\u97e7\u4e0d\u62d4 \u201cWhat can i say\uff0c Mamba out\uff01\u201d \u4f46\u662f\u7ecf\u8fc7\u4e92\u8054\u7f51\u7684\u89e3\u6784\uff0c\u8fd9\u53e5\u8bdd\u6210\u4e3a\u4e86\u4e00\u53e5\u4e07\u7528\u573a\u666f\u7684\u8bdd \u2026 Aprende ingl\u00e9s gratis con nuestros cursos. \u00a1Mas de 100 lecciones con ejemplos y audio! Distintas y divertidas formas de estudiar ingl\u00e9s: \u2026 \u77e5\u4e4e\uff0c\u4e2d\u6587\u4e92\u8054\u7f51\u9ad8\u8d28\u91cf\u7684\u95ee\u7b54\u793e\u533a\u548c\u521b\u4f5c\u8005\u805a\u96c6\u7684\u539f\u521b\u5185\u5bb9\u5e73\u53f0\uff0c\u4e8e 2011 \u5e74 1 \u6708\u6b63\u5f0f\u4e0a\u7ebf\uff0c\u4ee5\u300c\u8ba9\u4eba\u4eec\u66f4\u597d\u7684\u5206\u4eab\u77e5\u8bc6\u3001\u7ecf\u9a8c\u548c\u89c1\u89e3\uff0c\u627e\u5230\u81ea\u5df1\u7684\u89e3\u7b54\u300d\u4e3a\u54c1 \u2026 \u8f6f\u4ef6\u7279\u70b9 1\u3001Windows Desktop Runtime\u662f\u5fae\u8f6fWindows\u684c\u9762\u7a0b\u5e8f\u8fd0\u884c\u5e93\uff08\u542b\u5e38\u89c4\u8fd0\u884c\u5e93\uff09 2\u3001\u80fd\u5b8c\u7f8e\u517c\u5bb9\u5fae\u8f6f\u4e0d\u540c\u7248\u672c\u7684Windows\u7cfb\u7edf\uff0c\u89e3\u51b3\u5176\u7a0b\u5e8f\u7f3a \u2026 \u77e5\u4e4e\uff0c\u4e2d\u6587\u4e92\u8054\u7f51\u9ad8\u8d28\u91cf\u7684\u95ee\u7b54\u793e\u533a\u548c\u521b\u4f5c\u8005\u805a\u96c6\u7684\u539f\u521b\u5185\u5bb9\u5e73\u53f0\uff0c\u4e8e 2011 \u5e74 1 \u6708\u6b63\u5f0f\u4e0a\u7ebf\uff0c\u4ee5\u300c\u8ba9\u4eba\u4eec\u66f4\u597d\u7684\u5206\u4eab\u77e5\u8bc6\u3001\u7ecf\u9a8c\u548c\u89c1\u89e3\uff0c\u627e\u5230\u81ea\u5df1\u7684\u89e3\u7b54\u300d\u4e3a\u54c1 \u2026\nURL: https://duckduckgo.com/?q=What's+happening+in+the+software+development+world+today?\n\nUser Question: What's happening in the software development world today?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", + "prompt.prompt_length": 1091, + "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\n\u66fc\u5df4\u7684\u610f\u601d\u662f\u4e00\u79cd\u6bd2\u86c7\uff0c\u8c61\u5f81\u7740\u79d1\u6bd4\u5728\u7403\u573a\u4e0a\u7684\u653b\u51fb\u6027\u548c\u575a\u97e7\u4e0d\u62d4 \u201cWhat can i say\uff0c Mamba out\uff01\u201d \u4f46\u662f\u7ecf\u8fc7\u4e92\u8054\u7f51\u7684\u89e3\u6784\uff0c\u8fd9\u53e5\u8bdd\u6210\u4e3a\u4e86\u4e00\u53e5\u4e07\u7528\u573a\u666f\u7684\u8bdd \u2026 Aprende ingl\u00e9s gratis con nuestros cursos. \u00a1Mas de 100 lecciones con ejemplos y audio...", + "prompt.instruction_following": "web_search_optimized", + "response.answer_length": 526, + "response.answer_completeness": "complete", + "response.response_quality": "medium", + "response.source_citation": 1, + "response.context_utilization": 0.8930390492359932, + "output_response": "Web Search Answer: The provided web search results do not contain specific information about current events or trends in the software development world. They primarily focus on unrelated topics, such as the meaning of \"...", + "evaluation.search_effectiveness": 0.2, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 1, + "evaluation.context_relevance": "medium", + "evaluation.overall_quality": "low", + "evaluation.ready_for_production": true, + "web_search.search_strategy": "duckduckgo_api", + "web_search.real_time_data": true, + "web_search.external_sources": 1, + "web_search.information_freshness": "current", + "web_search.knowledge_synthesis": "single_source" + }, + "events": [], + "links": [], + "trace_trace_id": "79f9d235-d46e-4740-be35-ef0006071f6c", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:51.539000000", + "trace_end_time": "2025-10-16 08:50:55.150000000", + "trace_duration_ms": 3611.084, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:51.539000000", + "updated_at": "2025-10-16 08:50:51.539000000" + }, + { + "span_id": "888b4441-3678-42a5-a752-8daae2433482", + "trace_id": "77b81bd5-5a04-4fc0-9fc6-571f7907d6b1", + "parent_span_id": "ba6ffeeb-78b5-44dd-b460-7933ca6e7739", + "name": "agent.query_routing", + "start_time": "2025-10-16 08:50:44.888000000", + "end_time": "2025-10-16 08:50:50.529000000", + "duration_ms": 5641.062, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What are the newest features in cloud computing?", + "query_length": 48, + "query_type": "routing_query", + "output_response": "Routed to RAG for query processing", + "router.classification": "RAG", + "router.confidence_score": 1, + "router.rag_keyword_score": 1, + "router.web_keyword_score": 0, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "77b81bd5-5a04-4fc0-9fc6-571f7907d6b1", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:44.888000000", + "trace_end_time": "2025-10-16 08:50:50.529000000", + "trace_duration_ms": 5641.226, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:44.888000000", + "updated_at": "2025-10-16 08:50:44.888000000" + }, + { + "span_id": "b6bae768-93a6-47d9-8369-61897f9d7828", + "trace_id": "77b81bd5-5a04-4fc0-9fc6-571f7907d6b1", + "parent_span_id": "888b4441-3678-42a5-a752-8daae2433482", + "name": "agent.llm-rag", + "start_time": "2025-10-16 08:50:44.888000000", + "end_time": "2025-10-16 08:50:50.529000000", + "duration_ms": 5640.875999999999, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "rag_agent", + "agent.operation": "llm-rag", + "agent.capabilities": [ + "document_retrieval", + "context_generation", + "response_generation" + ], + "agent.id": "noveum_rag_agent", + "input_query": "What are the newest features in cloud computing?", + "query_length": 48, + "query_type": "rag_query", + "output_response": "RAG Answer: The provided context does not specifically mention any new features in cloud computing as a general topic. However, it does highlight several recent enhancements and features related to the Noveum.ai ...", + "rag.context_length": 2579, + "rag.sources_count": 5, + "rag.answer_length": 983, + "rag.mode": "retrieval_augmented_generation" + }, + "events": [], + "links": [], + "trace_trace_id": "77b81bd5-5a04-4fc0-9fc6-571f7907d6b1", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:44.888000000", + "trace_end_time": "2025-10-16 08:50:50.529000000", + "trace_duration_ms": 5641.226, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:44.888000000", + "updated_at": "2025-10-16 08:50:44.888000000" + }, + { + "span_id": "ba6ffeeb-78b5-44dd-b460-7933ca6e7739", + "trace_id": "77b81bd5-5a04-4fc0-9fc6-571f7907d6b1", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-16 08:50:44.888000000", + "end_time": "2025-10-16 08:50:50.529000000", + "duration_ms": 5641.156, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What are the newest features in cloud computing?", + "query.length": 48, + "processing.mode": "RAG", + "processing.time_seconds": 5.64110541343689, + "processing.response_length": 983, + "processing.sources_count": 5, + "output_response": "Final Answer: The provided context does not specifically mention any new features in cloud computing as a general topic. However, it does highlight several recent enhancements and features related to the Noveum.ai ...", + "final_answer_mode": "RAG", + "query_processed.input_query": "What are the newest features in cloud computing?", + "query_processed.output_response": "Successfully processed query using RAG, generated 983 character response", + "query_processed.mode": "RAG", + "query_processed.processing_time": 5.64110541343689, + "query_processed.response_length": 983 + }, + "events": [], + "links": [], + "trace_trace_id": "77b81bd5-5a04-4fc0-9fc6-571f7907d6b1", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:44.888000000", + "trace_end_time": "2025-10-16 08:50:50.529000000", + "trace_duration_ms": 5641.226, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:44.888000000", + "updated_at": "2025-10-16 08:50:44.888000000" + }, + { + "span_id": "f5538bff-8908-4a56-a622-4ba2eeedfd29", + "trace_id": "77b81bd5-5a04-4fc0-9fc6-571f7907d6b1", + "parent_span_id": "888b4441-3678-42a5-a752-8daae2433482", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-16 08:50:44.888000000", + "end_time": "2025-10-16 08:50:44.888000000", + "duration_ms": 0.032, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What are the newest features in cloud computing?", + "query_length": 48, + "query_type": "routing_evaluation_query", + "classification.mode": "RAG", + "classification.rag_keyword_score": 1, + "classification.web_keyword_score": 0, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "documentation", + "query.keyword_density": 0.125, + "query.domain_affinity": "noveum", + "routing.decision": "Routed to RAG based on analysis", + "routing.rationale": "RAG score: 1, Web score: 0, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "Web Search", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.019230769230769232, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: RAG (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "77b81bd5-5a04-4fc0-9fc6-571f7907d6b1", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:44.888000000", + "trace_end_time": "2025-10-16 08:50:50.529000000", + "trace_duration_ms": 5641.226, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:44.888000000", + "updated_at": "2025-10-16 08:50:44.888000000" + }, + { + "span_id": "856e51cf-fa4d-459b-a046-1ea7316b903e", + "trace_id": "77b81bd5-5a04-4fc0-9fc6-571f7907d6b1", + "parent_span_id": "b6bae768-93a6-47d9-8369-61897f9d7828", + "name": "agent.llm_model_execution", + "start_time": "2025-10-16 08:50:45.423000000", + "end_time": "2025-10-16 08:50:49.997000000", + "duration_ms": 4573.782999999999, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What are the newest features in cloud computing?", + "query_length": 48, + "query_type": "rag_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 4.573714733123779, + "model.latency_ms": 4573.714733123779, + "model.start_time": 1760604645.423261, + "model.end_time": 1760604649.9969757, + "model.performance_tier": "medium", + "model.prompt_tokens": 780, + "model.completion_tokens": 245, + "model.total_tokens": 1025, + "model.tokens_per_second": 224.1066747291299, + "model.estimated_cost": 0.03075, + "model.efficiency_score": 0.9590243902439024, + "model.response_length": 983, + "model.response_quality": "high", + "model.output_response": "Model Response: The provided context does not specifically mention any new features in cloud computing as a general topic. However, it does highlight several recent enhancements and features related to the Noveum.ai ...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "77b81bd5-5a04-4fc0-9fc6-571f7907d6b1", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:44.888000000", + "trace_end_time": "2025-10-16 08:50:50.529000000", + "trace_duration_ms": 5641.226, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:44.888000000", + "updated_at": "2025-10-16 08:50:44.888000000" + }, + { + "span_id": "a2fa7400-74c4-4718-a562-899b91503c95", + "trace_id": "77b81bd5-5a04-4fc0-9fc6-571f7907d6b1", + "parent_span_id": "b6bae768-93a6-47d9-8369-61897f9d7828", + "name": "agent.rag_evaluation_metrics", + "start_time": "2025-10-16 08:50:49.997000000", + "end_time": "2025-10-16 08:50:49.997000000", + "duration_ms": 0.048, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "rag_evaluation_metrics", + "agent.capabilities": [ + "retrieval_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What are the newest features in cloud computing?", + "query_length": 48, + "query_type": "rag_evaluation_query", + "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/changelog):\n- \ud83d\udcc8 Enhanced Dashboard Analytics: Real-time request tracking, latency monitoring, and cost analysis\n- \ud83d\udd0d Improved Logs Interface: Better search functionality, detailed trace views, and enhanced debugging capabilities\n- \ud83d\udc65 Team Management: Advanced member mana...", + "retrieval.context_length": 2579, + "retrieval.sources_count": 5, + "retrieval.context_quality": "high", + "retrieval.effectiveness": 1, + "retrieval.context_utilization": 2.579, + "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/changelog):\n- \ud83d\udcc8 Enhanced Dashboard Analytics: Real-time request tracking, latency monitoring, and cost analysis\n- \ud83d\udd0d Improved Logs Interface: Better search functionality, detailed trace views, and enhanced debugging capabilities\n- \ud83d\udc65 Team Management: Advanced member management with role-based access control and invitation system\n- \ud83d\udee1\ufe0f Security Improvements: Enhanced API key management and secure credential storage\n4 months ago\n- \ud83d\udcca Advanced Metrics Collection: Comprehensive telemetry system with custom metrics ...\n\nSource 2 (https://noveum.ai/en/changelog):\n- \ud83d\udcdd Documentation Updates: Comprehensive guides for integration and best practices\n5 months ago\n- \ud83c\udf89 Platform Foundation: Initial release of Noveum.ai AI observability platform\n- \ud83d\ude80 Core Features: AI Gateway integration, basic evaluation metrics, and monitoring capabilities\n- \ud83d\udcc8 Evaluation Jobs: Automated model evaluation with configurable metrics and reporting\n- \ud83d\udd0c Provider Integrations: Support for major AI providers and custom model deployments...\n\nSource 3 (https://noveum.ai/docs/platform/dashboard):\n- Bulk Export: Download multiple traces simultaneously\n- Comparative Analysis: Compare performance across multiple traces\nReal-Time Updates\n- Live Refresh: Automatic updates as new traces arrive\n- Connection Monitoring: Real-time status of your trace ingestion pipeline\n- Performance Indicators: Live metrics for system health monitoring\nReady to dive deeper? Explore Projects & Environments to organize your AI applications, or check out Team Collaboration to share insights with your team.\nGet Earl...\n\nSource 4 (https://noveum.ai/en/changelog):\nChangelog\nStay up to date with the latest changes in our product.\n2 months ago\n- \ud83c\udfaf Major Platform Evolution: NovaEval Framework - Launched comprehensive AI model evaluation framework with 20+ built-in scorers including accuracy, RAG metrics, conversational scoring, and specialized agent evaluation tools. Features production-grade deployment on Docker and Kubernetes. (GitHub | PyPI Package)\n- \ud83d\ude80 Next-Generation Traces UI - Complete redesign featuring three-pane layout with directory tree navigatio...\n\nSource 5 (https://noveum.ai/en/blog):\n8/12/2025\nDiscover how Noveum.ai provides comprehensive tracing and observability for AI applications, from development debugging to production optimization.\nShashank Agarwal\n3/3/2025\nDiscover how Noveum.ai provides comprehensive tracing and observability for LLM applications, RAG systems, and multi-agent workflows with our powerful Python and TypeScript SDKs.\nShashank Agarwal\n3/2/2025...\n\nUser Question: What are the newest features in cloud computing?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", + "prompt.prompt_length": 3123, + "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/changelog):\n- \ud83d\udcc8 Enhanced Dashboard Analytics: Real-time request tracking, latency monitoring, and cost analysis\n- \ud83d\udd0d Improved Logs Interface: Better search functionality,...", + "prompt.instruction_following": "rag_optimized", + "response.answer_length": 983, + "response.answer_completeness": "complete", + "response.response_quality": "high", + "response.source_citation": 5, + "response.context_utilization": 0.381155486622722, + "output_response": "RAG Answer: The provided context does not specifically mention any new features in cloud computing as a general topic. However, it does highlight several recent enhancements and features related to the Noveum.ai ...", + "evaluation.retrieval_effectiveness": 1, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 5, + "evaluation.context_relevance": "high", + "evaluation.overall_quality": "high", + "evaluation.ready_for_production": true, + "rag.retrieval_strategy": "semantic_similarity", + "rag.vector_search_results": 5, + "rag.context_synthesis": "multi_source", + "rag.document_coverage": 1, + "rag.information_density": 0.381155486622722 + }, + "events": [], + "links": [], + "trace_trace_id": "77b81bd5-5a04-4fc0-9fc6-571f7907d6b1", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:44.888000000", + "trace_end_time": "2025-10-16 08:50:50.529000000", + "trace_duration_ms": 5641.226, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:44.888000000", + "updated_at": "2025-10-16 08:50:44.888000000" + }, + { + "span_id": "4c391bb1-1a62-4e2a-913b-96f83feff494", + "trace_id": "32170d1a-d211-46d8-9cdc-5a4265f0c443", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-16 08:50:42.124000000", + "end_time": "2025-10-16 08:50:43.885000000", + "duration_ms": 1760.974, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What's the current status of cryptocurrency markets?", + "query.length": 52, + "processing.mode": "Web Search", + "processing.time_seconds": 1.7608659267425537, + "processing.response_length": 363, + "processing.sources_count": 1, + "output_response": "Final Answer: The web search did not yield any specific information regarding the current status of cryptocurrency markets. Therefore, I cannot provide an update on market trends, prices, or significant events affe...", + "final_answer_mode": "Web Search", + "query_processed.input_query": "What's the current status of cryptocurrency markets?", + "query_processed.output_response": "Successfully processed query using Web Search, generated 363 character response", + "query_processed.mode": "Web Search", + "query_processed.processing_time": 1.7608659267425537, + "query_processed.response_length": 363 + }, + "events": [], + "links": [], + "trace_trace_id": "32170d1a-d211-46d8-9cdc-5a4265f0c443", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:42.124000000", + "trace_end_time": "2025-10-16 08:50:43.885000000", + "trace_duration_ms": 1761.047, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:42.124000000", + "updated_at": "2025-10-16 08:50:42.124000000" + }, + { + "span_id": "7ab2ec7b-a617-4ebf-9bb1-82dc6b18e160", + "trace_id": "32170d1a-d211-46d8-9cdc-5a4265f0c443", + "parent_span_id": "4c391bb1-1a62-4e2a-913b-96f83feff494", + "name": "agent.query_routing", + "start_time": "2025-10-16 08:50:42.124000000", + "end_time": "2025-10-16 08:50:43.885000000", + "duration_ms": 1760.833, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What's the current status of cryptocurrency markets?", + "query_length": 52, + "query_type": "routing_query", + "output_response": "Routed to Web Search for query processing", + "router.classification": "Web Search", + "router.confidence_score": 1, + "router.rag_keyword_score": 0, + "router.web_keyword_score": 3, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "32170d1a-d211-46d8-9cdc-5a4265f0c443", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:42.124000000", + "trace_end_time": "2025-10-16 08:50:43.885000000", + "trace_duration_ms": 1761.047, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:42.124000000", + "updated_at": "2025-10-16 08:50:42.124000000" + }, + { + "span_id": "db01d228-7c63-4814-b2ea-bdf3898f9c0d", + "trace_id": "32170d1a-d211-46d8-9cdc-5a4265f0c443", + "parent_span_id": "7ab2ec7b-a617-4ebf-9bb1-82dc6b18e160", + "name": "agent.web_search_generation", + "start_time": "2025-10-16 08:50:42.124000000", + "end_time": "2025-10-16 08:50:43.885000000", + "duration_ms": 1760.701, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "web_search_agent", + "agent.operation": "web_search_generation", + "agent.capabilities": [ + "web_search", + "content_synthesis", + "response_generation" + ], + "agent.id": "noveum_web_search_agent", + "input_query": "What's the current status of cryptocurrency markets?", + "query_length": 52, + "query_type": "web_search_query", + "output_response": "Web Search Answer: The web search did not yield any specific information regarding the current status of cryptocurrency markets. Therefore, I cannot provide an update on market trends, prices, or significant events affe...", + "web_search.results_count": 1, + "web_search.context_length": 154, + "web_search.response_length": 363, + "web_search.mode": "external_web_search" + }, + "events": [], + "links": [], + "trace_trace_id": "32170d1a-d211-46d8-9cdc-5a4265f0c443", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:42.124000000", + "trace_end_time": "2025-10-16 08:50:43.885000000", + "trace_duration_ms": 1761.047, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:42.124000000", + "updated_at": "2025-10-16 08:50:42.124000000" + }, + { + "span_id": "da7b0550-b13c-4518-ad3b-050c666d198c", + "trace_id": "32170d1a-d211-46d8-9cdc-5a4265f0c443", + "parent_span_id": "7ab2ec7b-a617-4ebf-9bb1-82dc6b18e160", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-16 08:50:42.124000000", + "end_time": "2025-10-16 08:50:42.124000000", + "duration_ms": 0.029, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What's the current status of cryptocurrency markets?", + "query_length": 52, + "query_type": "routing_evaluation_query", + "classification.mode": "Web Search", + "classification.rag_keyword_score": 0, + "classification.web_keyword_score": 3, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "complex", + "query.intent": "general_knowledge", + "query.keyword_density": 0.42857142857142855, + "query.domain_affinity": "general", + "routing.decision": "Routed to Web Search based on analysis", + "routing.rationale": "RAG score: 0, Web score: 3, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "RAG", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.057692307692307696, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: Web Search (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "32170d1a-d211-46d8-9cdc-5a4265f0c443", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:42.124000000", + "trace_end_time": "2025-10-16 08:50:43.885000000", + "trace_duration_ms": 1761.047, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:42.124000000", + "updated_at": "2025-10-16 08:50:42.124000000" + }, + { + "span_id": "950e18b6-9f62-4ac4-8788-3cac08fb7899", + "trace_id": "32170d1a-d211-46d8-9cdc-5a4265f0c443", + "parent_span_id": "db01d228-7c63-4814-b2ea-bdf3898f9c0d", + "name": "agent.llm_model_execution", + "start_time": "2025-10-16 08:50:42.465000000", + "end_time": "2025-10-16 08:50:43.885000000", + "duration_ms": 1419.5690000000002, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What's the current status of cryptocurrency markets?", + "query_length": 52, + "query_type": "web_search_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 1.4195220470428467, + "model.latency_ms": 1419.5220470428467, + "model.start_time": 1760604642.4658613, + "model.end_time": 1760604643.8853834, + "model.performance_tier": "fast", + "model.prompt_tokens": 162, + "model.completion_tokens": 90, + "model.total_tokens": 252, + "model.tokens_per_second": 177.52454111224782, + "model.estimated_cost": 0.00756, + "model.efficiency_score": 1.4404761904761905, + "model.response_length": 363, + "model.response_quality": "high", + "model.output_response": "Model Response: The web search did not yield any specific information regarding the current status of cryptocurrency markets. Therefore, I cannot provide an update on market trends, prices, or significant events affe...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "32170d1a-d211-46d8-9cdc-5a4265f0c443", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:42.124000000", + "trace_end_time": "2025-10-16 08:50:43.885000000", + "trace_duration_ms": 1761.047, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:42.124000000", + "updated_at": "2025-10-16 08:50:42.124000000" + }, + { + "span_id": "e87846c9-ce82-42c2-a517-e0f67e9d14c1", + "trace_id": "32170d1a-d211-46d8-9cdc-5a4265f0c443", + "parent_span_id": "db01d228-7c63-4814-b2ea-bdf3898f9c0d", + "name": "agent.web_search_evaluation_metrics", + "start_time": "2025-10-16 08:50:43.885000000", + "end_time": "2025-10-16 08:50:43.885000000", + "duration_ms": 0.02, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "web_search_evaluation_metrics", + "agent.capabilities": [ + "web_search_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What's the current status of cryptocurrency markets?", + "query_length": 52, + "query_type": "web_search_evaluation_query", + "web_search.results_count": 1, + "web_search.context_length": 154, + "web_search.context_synthesized": "Context from 1 web sources", + "web_search.search_effectiveness": 0.2, + "web_search.context_quality": "low", + "web_search.source_diversity": 1, + "web_search.information_synthesis": "low", + "web_search.external_knowledge_utilization": 0.154, + "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What's+the+current+status+of+cryptocurrency+markets?\n\nUser Question: What's the current status of cryptocurrency markets?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", + "prompt.prompt_length": 651, + "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What's+the+current+status+of+cryptocurrency+markets?", + "prompt.instruction_following": "web_search_optimized", + "response.answer_length": 363, + "response.answer_completeness": "complete", + "response.response_quality": "medium", + "response.source_citation": 1, + "response.context_utilization": 2.357142857142857, + "output_response": "Web Search Answer: The web search did not yield any specific information regarding the current status of cryptocurrency markets. Therefore, I cannot provide an update on market trends, prices, or significant events affe...", + "evaluation.search_effectiveness": 0.2, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 1, + "evaluation.context_relevance": "low", + "evaluation.overall_quality": "low", + "evaluation.ready_for_production": true, + "web_search.search_strategy": "duckduckgo_api", + "web_search.real_time_data": true, + "web_search.external_sources": 1, + "web_search.information_freshness": "current", + "web_search.knowledge_synthesis": "single_source" + }, + "events": [], + "links": [], + "trace_trace_id": "32170d1a-d211-46d8-9cdc-5a4265f0c443", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:42.124000000", + "trace_end_time": "2025-10-16 08:50:43.885000000", + "trace_duration_ms": 1761.047, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:42.124000000", + "updated_at": "2025-10-16 08:50:42.124000000" + }, + { + "span_id": "305daacf-990b-4e23-8720-48900f75d9be", + "trace_id": "9424d420-deee-478b-b727-9dc791a7457c", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-16 08:50:31.476000000", + "end_time": "2025-10-16 08:50:41.121000000", + "duration_ms": 9644.752999999999, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What are the latest updates in Python programming?", + "query.length": 50, + "processing.mode": "Web Search", + "processing.time_seconds": 9.644662618637085, + "processing.response_length": 1408, + "processing.sources_count": 1, + "output_response": "Final Answer: The web search did not yield specific results regarding the latest updates in Python programming. However, I can provide some general information based on my knowledge up to October 2023.\n\nAs of late ...", + "final_answer_mode": "Web Search", + "query_processed.input_query": "What are the latest updates in Python programming?", + "query_processed.output_response": "Successfully processed query using Web Search, generated 1408 character response", + "query_processed.mode": "Web Search", + "query_processed.processing_time": 9.644662618637085, + "query_processed.response_length": 1408 + }, + "events": [], + "links": [], + "trace_trace_id": "9424d420-deee-478b-b727-9dc791a7457c", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:31.476000000", + "trace_end_time": "2025-10-16 08:50:41.121000000", + "trace_duration_ms": 9644.828, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:31.476000000", + "updated_at": "2025-10-16 08:50:31.476000000" + }, + { + "span_id": "56b4cc1b-0646-4dee-acb3-db98f3d44477", + "trace_id": "9424d420-deee-478b-b727-9dc791a7457c", + "parent_span_id": "fb234e30-69be-4d06-9248-c05bcc22f063", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-16 08:50:31.476000000", + "end_time": "2025-10-16 08:50:31.476000000", + "duration_ms": 0.036000000000000004, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What are the latest updates in Python programming?", + "query_length": 50, + "query_type": "routing_evaluation_query", + "classification.mode": "Web Search", + "classification.rag_keyword_score": 0, + "classification.web_keyword_score": 2, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "general_knowledge", + "query.keyword_density": 0.25, + "query.domain_affinity": "general", + "routing.decision": "Routed to Web Search based on analysis", + "routing.rationale": "RAG score: 0, Web score: 2, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "RAG", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.038461538461538464, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: Web Search (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "9424d420-deee-478b-b727-9dc791a7457c", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:31.476000000", + "trace_end_time": "2025-10-16 08:50:41.121000000", + "trace_duration_ms": 9644.828, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:31.476000000", + "updated_at": "2025-10-16 08:50:31.476000000" + }, + { + "span_id": "fb234e30-69be-4d06-9248-c05bcc22f063", + "trace_id": "9424d420-deee-478b-b727-9dc791a7457c", + "parent_span_id": "305daacf-990b-4e23-8720-48900f75d9be", + "name": "agent.query_routing", + "start_time": "2025-10-16 08:50:31.476000000", + "end_time": "2025-10-16 08:50:41.121000000", + "duration_ms": 9644.627, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What are the latest updates in Python programming?", + "query_length": 50, + "query_type": "routing_query", + "output_response": "Routed to Web Search for query processing", + "router.classification": "Web Search", + "router.confidence_score": 1, + "router.rag_keyword_score": 0, + "router.web_keyword_score": 2, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "9424d420-deee-478b-b727-9dc791a7457c", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:31.476000000", + "trace_end_time": "2025-10-16 08:50:41.121000000", + "trace_duration_ms": 9644.828, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:31.476000000", + "updated_at": "2025-10-16 08:50:31.476000000" + }, + { + "span_id": "f8a793a9-63ee-49b7-9931-d7dcc2bb0ca7", + "trace_id": "9424d420-deee-478b-b727-9dc791a7457c", + "parent_span_id": "fb234e30-69be-4d06-9248-c05bcc22f063", + "name": "agent.web_search_generation", + "start_time": "2025-10-16 08:50:31.477000000", + "end_time": "2025-10-16 08:50:41.121000000", + "duration_ms": 9644.478, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "web_search_agent", + "agent.operation": "web_search_generation", + "agent.capabilities": [ + "web_search", + "content_synthesis", + "response_generation" + ], + "agent.id": "noveum_web_search_agent", + "input_query": "What are the latest updates in Python programming?", + "query_length": 50, + "query_type": "web_search_query", + "output_response": "Web Search Answer: The web search did not yield specific results regarding the latest updates in Python programming. However, I can provide some general information based on my knowledge up to October 2023.\n\nAs of late ...", + "web_search.results_count": 1, + "web_search.context_length": 152, + "web_search.response_length": 1408, + "web_search.mode": "external_web_search" + }, + "events": [], + "links": [], + "trace_trace_id": "9424d420-deee-478b-b727-9dc791a7457c", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:31.476000000", + "trace_end_time": "2025-10-16 08:50:41.121000000", + "trace_duration_ms": 9644.828, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:31.476000000", + "updated_at": "2025-10-16 08:50:31.476000000" + }, + { + "span_id": "b968e3d3-d4aa-4357-85d8-34a612748fbd", + "trace_id": "9424d420-deee-478b-b727-9dc791a7457c", + "parent_span_id": "f8a793a9-63ee-49b7-9931-d7dcc2bb0ca7", + "name": "agent.llm_model_execution", + "start_time": "2025-10-16 08:50:32.016000000", + "end_time": "2025-10-16 08:50:41.121000000", + "duration_ms": 9105.344000000001, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What are the latest updates in Python programming?", + "query_length": 50, + "query_type": "web_search_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 9.105276584625244, + "model.latency_ms": 9105.276584625244, + "model.start_time": 1760604632.0161095, + "model.end_time": 1760604641.121386, + "model.performance_tier": "slow", + "model.prompt_tokens": 161, + "model.completion_tokens": 352, + "model.total_tokens": 513, + "model.tokens_per_second": 56.340957381374714, + "model.estimated_cost": 0.015390000000000001, + "model.efficiency_score": 2.7446393762183234, + "model.response_length": 1408, + "model.response_quality": "high", + "model.output_response": "Model Response: The web search did not yield specific results regarding the latest updates in Python programming. However, I can provide some general information based on my knowledge up to October 2023.\n\nAs of late ...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "9424d420-deee-478b-b727-9dc791a7457c", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:31.476000000", + "trace_end_time": "2025-10-16 08:50:41.121000000", + "trace_duration_ms": 9644.828, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:31.476000000", + "updated_at": "2025-10-16 08:50:31.476000000" + }, + { + "span_id": "80817711-e64c-4a9f-802e-2a9e36ff093d", + "trace_id": "9424d420-deee-478b-b727-9dc791a7457c", + "parent_span_id": "f8a793a9-63ee-49b7-9931-d7dcc2bb0ca7", + "name": "agent.web_search_evaluation_metrics", + "start_time": "2025-10-16 08:50:41.121000000", + "end_time": "2025-10-16 08:50:41.121000000", + "duration_ms": 0.02, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "web_search_evaluation_metrics", + "agent.capabilities": [ + "web_search_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What are the latest updates in Python programming?", + "query_length": 50, + "query_type": "web_search_evaluation_query", + "web_search.results_count": 1, + "web_search.context_length": 152, + "web_search.context_synthesized": "Context from 1 web sources", + "web_search.search_effectiveness": 0.2, + "web_search.context_quality": "low", + "web_search.source_diversity": 1, + "web_search.information_synthesis": "low", + "web_search.external_knowledge_utilization": 0.152, + "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What+are+the+latest+updates+in+Python+programming?\n\nUser Question: What are the latest updates in Python programming?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", + "prompt.prompt_length": 647, + "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What+are+the+latest+updates+in+Python+programming?", + "prompt.instruction_following": "web_search_optimized", + "response.answer_length": 1408, + "response.answer_completeness": "complete", + "response.response_quality": "medium", + "response.source_citation": 1, + "response.context_utilization": 9.263157894736842, + "output_response": "Web Search Answer: The web search did not yield specific results regarding the latest updates in Python programming. However, I can provide some general information based on my knowledge up to October 2023.\n\nAs of late ...", + "evaluation.search_effectiveness": 0.2, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 1, + "evaluation.context_relevance": "low", + "evaluation.overall_quality": "low", + "evaluation.ready_for_production": true, + "web_search.search_strategy": "duckduckgo_api", + "web_search.real_time_data": true, + "web_search.external_sources": 1, + "web_search.information_freshness": "current", + "web_search.knowledge_synthesis": "single_source" + }, + "events": [], + "links": [], + "trace_trace_id": "9424d420-deee-478b-b727-9dc791a7457c", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:31.476000000", + "trace_end_time": "2025-10-16 08:50:41.121000000", + "trace_duration_ms": 9644.828, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:31.476000000", + "updated_at": "2025-10-16 08:50:31.476000000" + }, + { + "span_id": "13af0f3c-bd5c-4024-8fa3-c9e347a1ef20", + "trace_id": "a3735d64-89bb-4fdf-a851-7a1f3515c52d", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-16 08:50:27.687000000", + "end_time": "2025-10-16 08:50:30.473000000", + "duration_ms": 2786.078, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What happened in tech news this week?", + "query.length": 37, + "processing.mode": "Web Search", + "processing.time_seconds": 2.7859997749328613, + "processing.response_length": 396, + "processing.sources_count": 1, + "output_response": "Final Answer: It appears that the web search did not yield any specific results regarding recent tech news. Unfortunately, I cannot provide a summary of this week's tech news based on the available information. For...", + "final_answer_mode": "Web Search", + "query_processed.input_query": "What happened in tech news this week?", + "query_processed.output_response": "Successfully processed query using Web Search, generated 396 character response", + "query_processed.mode": "Web Search", + "query_processed.processing_time": 2.7859997749328613, + "query_processed.response_length": 396 + }, + "events": [], + "links": [], + "trace_trace_id": "a3735d64-89bb-4fdf-a851-7a1f3515c52d", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:27.687000000", + "trace_end_time": "2025-10-16 08:50:30.473000000", + "trace_duration_ms": 2786.145, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:27.687000000", + "updated_at": "2025-10-16 08:50:27.687000000" + }, + { + "span_id": "47e8f375-35e6-4097-86ff-df73e8181c23", + "trace_id": "a3735d64-89bb-4fdf-a851-7a1f3515c52d", + "parent_span_id": "a4f1a7b3-dc35-4f13-96d3-b6dc8fb21a37", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-16 08:50:27.687000000", + "end_time": "2025-10-16 08:50:27.687000000", + "duration_ms": 0.038, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What happened in tech news this week?", + "query_length": 37, + "query_type": "routing_evaluation_query", + "classification.mode": "Web Search", + "classification.rag_keyword_score": 0, + "classification.web_keyword_score": 2, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "general_knowledge", + "query.keyword_density": 0.2857142857142857, + "query.domain_affinity": "general", + "routing.decision": "Routed to Web Search based on analysis", + "routing.rationale": "RAG score: 0, Web score: 2, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "RAG", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.038461538461538464, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: Web Search (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "a3735d64-89bb-4fdf-a851-7a1f3515c52d", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:27.687000000", + "trace_end_time": "2025-10-16 08:50:30.473000000", + "trace_duration_ms": 2786.145, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:27.687000000", + "updated_at": "2025-10-16 08:50:27.687000000" + }, + { + "span_id": "a4f1a7b3-dc35-4f13-96d3-b6dc8fb21a37", + "trace_id": "a3735d64-89bb-4fdf-a851-7a1f3515c52d", + "parent_span_id": "13af0f3c-bd5c-4024-8fa3-c9e347a1ef20", + "name": "agent.query_routing", + "start_time": "2025-10-16 08:50:27.687000000", + "end_time": "2025-10-16 08:50:30.473000000", + "duration_ms": 2785.964, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What happened in tech news this week?", + "query_length": 37, + "query_type": "routing_query", + "output_response": "Routed to Web Search for query processing", + "router.classification": "Web Search", + "router.confidence_score": 1, + "router.rag_keyword_score": 0, + "router.web_keyword_score": 2, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "a3735d64-89bb-4fdf-a851-7a1f3515c52d", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:27.687000000", + "trace_end_time": "2025-10-16 08:50:30.473000000", + "trace_duration_ms": 2786.145, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:27.687000000", + "updated_at": "2025-10-16 08:50:27.687000000" + }, + { + "span_id": "ebfff280-5eb6-4838-8de7-35c85e2e3eaa", + "trace_id": "a3735d64-89bb-4fdf-a851-7a1f3515c52d", + "parent_span_id": "a4f1a7b3-dc35-4f13-96d3-b6dc8fb21a37", + "name": "agent.web_search_generation", + "start_time": "2025-10-16 08:50:27.687000000", + "end_time": "2025-10-16 08:50:30.473000000", + "duration_ms": 2785.8230000000003, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "web_search_agent", + "agent.operation": "web_search_generation", + "agent.capabilities": [ + "web_search", + "content_synthesis", + "response_generation" + ], + "agent.id": "noveum_web_search_agent", + "input_query": "What happened in tech news this week?", + "query_length": 37, + "query_type": "web_search_query", + "output_response": "Web Search Answer: It appears that the web search did not yield any specific results regarding recent tech news. Unfortunately, I cannot provide a summary of this week's tech news based on the available information. For...", + "web_search.results_count": 1, + "web_search.context_length": 139, + "web_search.response_length": 396, + "web_search.mode": "external_web_search" + }, + "events": [], + "links": [], + "trace_trace_id": "a3735d64-89bb-4fdf-a851-7a1f3515c52d", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:27.687000000", + "trace_end_time": "2025-10-16 08:50:30.473000000", + "trace_duration_ms": 2786.145, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:27.687000000", + "updated_at": "2025-10-16 08:50:27.687000000" + }, + { + "span_id": "835b9351-d381-4785-8538-fbe1064fc8cd", + "trace_id": "a3735d64-89bb-4fdf-a851-7a1f3515c52d", + "parent_span_id": "ebfff280-5eb6-4838-8de7-35c85e2e3eaa", + "name": "agent.llm_model_execution", + "start_time": "2025-10-16 08:50:28.026000000", + "end_time": "2025-10-16 08:50:30.473000000", + "duration_ms": 2447.269, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What happened in tech news this week?", + "query_length": 37, + "query_type": "web_search_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 2.447227716445923, + "model.latency_ms": 2447.227716445923, + "model.start_time": 1760604628.0260897, + "model.end_time": 1760604630.4733174, + "model.performance_tier": "medium", + "model.prompt_tokens": 155, + "model.completion_tokens": 99, + "model.total_tokens": 254, + "model.tokens_per_second": 103.79091340501853, + "model.estimated_cost": 0.00762, + "model.efficiency_score": 1.5590551181102361, + "model.response_length": 396, + "model.response_quality": "high", + "model.output_response": "Model Response: It appears that the web search did not yield any specific results regarding recent tech news. Unfortunately, I cannot provide a summary of this week's tech news based on the available information. For...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "a3735d64-89bb-4fdf-a851-7a1f3515c52d", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:27.687000000", + "trace_end_time": "2025-10-16 08:50:30.473000000", + "trace_duration_ms": 2786.145, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:27.687000000", + "updated_at": "2025-10-16 08:50:27.687000000" + }, + { + "span_id": "93c6f427-6fcd-4318-aec8-095553fed469", + "trace_id": "a3735d64-89bb-4fdf-a851-7a1f3515c52d", + "parent_span_id": "ebfff280-5eb6-4838-8de7-35c85e2e3eaa", + "name": "agent.web_search_evaluation_metrics", + "start_time": "2025-10-16 08:50:30.473000000", + "end_time": "2025-10-16 08:50:30.473000000", + "duration_ms": 0.017, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "web_search_evaluation_metrics", + "agent.capabilities": [ + "web_search_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What happened in tech news this week?", + "query_length": 37, + "query_type": "web_search_evaluation_query", + "web_search.results_count": 1, + "web_search.context_length": 139, + "web_search.context_synthesized": "Context from 1 web sources", + "web_search.search_effectiveness": 0.2, + "web_search.context_quality": "low", + "web_search.source_diversity": 1, + "web_search.information_synthesis": "low", + "web_search.external_knowledge_utilization": 0.139, + "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What+happened+in+tech+news+this+week?\n\nUser Question: What happened in tech news this week?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", + "prompt.prompt_length": 621, + "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What+happened+in+tech+news+this+week?", + "prompt.instruction_following": "web_search_optimized", + "response.answer_length": 396, + "response.answer_completeness": "complete", + "response.response_quality": "medium", + "response.source_citation": 1, + "response.context_utilization": 2.8489208633093526, + "output_response": "Web Search Answer: It appears that the web search did not yield any specific results regarding recent tech news. Unfortunately, I cannot provide a summary of this week's tech news based on the available information. For...", + "evaluation.search_effectiveness": 0.2, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 1, + "evaluation.context_relevance": "low", + "evaluation.overall_quality": "low", + "evaluation.ready_for_production": true, + "web_search.search_strategy": "duckduckgo_api", + "web_search.real_time_data": true, + "web_search.external_sources": 1, + "web_search.information_freshness": "current", + "web_search.knowledge_synthesis": "single_source" + }, + "events": [], + "links": [], + "trace_trace_id": "a3735d64-89bb-4fdf-a851-7a1f3515c52d", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:27.687000000", + "trace_end_time": "2025-10-16 08:50:30.473000000", + "trace_duration_ms": 2786.145, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:27.687000000", + "updated_at": "2025-10-16 08:50:27.687000000" + }, + { + "span_id": "622e7353-4eeb-47d5-8558-4ecec28cdcfd", + "trace_id": "fa5bb641-f4b7-4741-b06c-392a5f2a9aea", + "parent_span_id": "c0af6629-48b1-45fc-9dd2-60f098ea140e", + "name": "agent.query_routing", + "start_time": "2025-10-16 08:50:21.645000000", + "end_time": "2025-10-16 08:50:26.683000000", + "duration_ms": 5037.965, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What are the current trends in observability tools?", + "query_length": 51, + "query_type": "routing_query", + "output_response": "Routed to RAG for query processing", + "router.classification": "RAG", + "router.confidence_score": 0.3333333333333333, + "router.rag_keyword_score": 2, + "router.web_keyword_score": 1, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "fa5bb641-f4b7-4741-b06c-392a5f2a9aea", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:21.645000000", + "trace_end_time": "2025-10-16 08:50:26.683000000", + "trace_duration_ms": 5038.112, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:21.645000000", + "updated_at": "2025-10-16 08:50:21.645000000" + }, + { + "span_id": "c0af6629-48b1-45fc-9dd2-60f098ea140e", + "trace_id": "fa5bb641-f4b7-4741-b06c-392a5f2a9aea", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-16 08:50:21.645000000", + "end_time": "2025-10-16 08:50:26.683000000", + "duration_ms": 5038.0419999999995, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What are the current trends in observability tools?", + "query.length": 51, + "processing.mode": "RAG", + "processing.time_seconds": 5.0380027294158936, + "processing.response_length": 897, + "processing.sources_count": 5, + "output_response": "Final Answer: The provided context does not specifically outline current trends in observability tools beyond the focus on AI applications. However, it highlights the limitations of traditional monitoring tools for...", + "final_answer_mode": "RAG", + "query_processed.input_query": "What are the current trends in observability tools?", + "query_processed.output_response": "Successfully processed query using RAG, generated 897 character response", + "query_processed.mode": "RAG", + "query_processed.processing_time": 5.0380027294158936, + "query_processed.response_length": 897 + }, + "events": [], + "links": [], + "trace_trace_id": "fa5bb641-f4b7-4741-b06c-392a5f2a9aea", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:21.645000000", + "trace_end_time": "2025-10-16 08:50:26.683000000", + "trace_duration_ms": 5038.112, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:21.645000000", + "updated_at": "2025-10-16 08:50:21.645000000" + }, + { + "span_id": "ee55906d-ffe3-42fd-9899-a476d4c33ff5", + "trace_id": "fa5bb641-f4b7-4741-b06c-392a5f2a9aea", + "parent_span_id": "622e7353-4eeb-47d5-8558-4ecec28cdcfd", + "name": "agent.llm-rag", + "start_time": "2025-10-16 08:50:21.645000000", + "end_time": "2025-10-16 08:50:26.683000000", + "duration_ms": 5037.803, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "rag_agent", + "agent.operation": "llm-rag", + "agent.capabilities": [ + "document_retrieval", + "context_generation", + "response_generation" + ], + "agent.id": "noveum_rag_agent", + "input_query": "What are the current trends in observability tools?", + "query_length": 51, + "query_type": "rag_query", + "output_response": "RAG Answer: The provided context does not specifically outline current trends in observability tools beyond the focus on AI applications. However, it highlights the limitations of traditional monitoring tools for...", + "rag.context_length": 2943, + "rag.sources_count": 5, + "rag.answer_length": 897, + "rag.mode": "retrieval_augmented_generation" + }, + "events": [], + "links": [], + "trace_trace_id": "fa5bb641-f4b7-4741-b06c-392a5f2a9aea", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:21.645000000", + "trace_end_time": "2025-10-16 08:50:26.683000000", + "trace_duration_ms": 5038.112, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:21.645000000", + "updated_at": "2025-10-16 08:50:21.645000000" + }, + { + "span_id": "fe610ecb-cc8e-4466-b719-34c6e0d45d87", + "trace_id": "fa5bb641-f4b7-4741-b06c-392a5f2a9aea", + "parent_span_id": "622e7353-4eeb-47d5-8558-4ecec28cdcfd", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-16 08:50:21.645000000", + "end_time": "2025-10-16 08:50:21.645000000", + "duration_ms": 0.032, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What are the current trends in observability tools?", + "query_length": 51, + "query_type": "routing_evaluation_query", + "classification.mode": "RAG", + "classification.rag_keyword_score": 2, + "classification.web_keyword_score": 1, + "classification.confidence_score": 0.3333333333333333, + "classification.confidence_level": "medium", + "classification.method": "keyword_based", + "query.complexity": "complex", + "query.intent": "documentation", + "query.keyword_density": 0.375, + "query.domain_affinity": "noveum", + "routing.decision": "Routed to RAG based on analysis", + "routing.rationale": "RAG score: 2, Web score: 1, Confidence: 0.33", + "routing.expected_performance": "medium", + "routing.alternative_mode": "Web Search", + "routing.decision_confidence": 0.3333333333333333, + "evaluation.routing_accuracy": "medium", + "evaluation.keyword_coverage": 0.057692307692307696, + "evaluation.query_understanding": "ambiguous", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: RAG (Confidence: 0.33)" + }, + "events": [], + "links": [], + "trace_trace_id": "fa5bb641-f4b7-4741-b06c-392a5f2a9aea", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:21.645000000", + "trace_end_time": "2025-10-16 08:50:26.683000000", + "trace_duration_ms": 5038.112, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:21.645000000", + "updated_at": "2025-10-16 08:50:21.645000000" + }, + { + "span_id": "69a4773d-c738-489e-87ed-999583e02187", + "trace_id": "fa5bb641-f4b7-4741-b06c-392a5f2a9aea", + "parent_span_id": "ee55906d-ffe3-42fd-9899-a476d4c33ff5", + "name": "agent.llm_model_execution", + "start_time": "2025-10-16 08:50:22.600000000", + "end_time": "2025-10-16 08:50:25.786000000", + "duration_ms": 3185.257, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What are the current trends in observability tools?", + "query_length": 51, + "query_type": "rag_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 3.185206174850464, + "model.latency_ms": 3185.206174850464, + "model.start_time": 1760604622.6008554, + "model.end_time": 1760604625.7860615, + "model.performance_tier": "medium", + "model.prompt_tokens": 872, + "model.completion_tokens": 224, + "model.total_tokens": 1096, + "model.tokens_per_second": 344.09075577390337, + "model.estimated_cost": 0.03288, + "model.efficiency_score": 0.8184306569343066, + "model.response_length": 897, + "model.response_quality": "high", + "model.output_response": "Model Response: The provided context does not specifically outline current trends in observability tools beyond the focus on AI applications. However, it highlights the limitations of traditional monitoring tools for...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "fa5bb641-f4b7-4741-b06c-392a5f2a9aea", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:21.645000000", + "trace_end_time": "2025-10-16 08:50:26.683000000", + "trace_duration_ms": 5038.112, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:21.645000000", + "updated_at": "2025-10-16 08:50:21.645000000" + }, + { + "span_id": "5e5a9d30-4cd0-45cd-9494-192c280727ac", + "trace_id": "fa5bb641-f4b7-4741-b06c-392a5f2a9aea", + "parent_span_id": "ee55906d-ffe3-42fd-9899-a476d4c33ff5", + "name": "agent.rag_evaluation_metrics", + "start_time": "2025-10-16 08:50:25.786000000", + "end_time": "2025-10-16 08:50:25.786000000", + "duration_ms": 0.066, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "rag_evaluation_metrics", + "agent.capabilities": [ + "retrieval_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What are the current trends in observability tools?", + "query_length": 51, + "query_type": "rag_evaluation_query", + "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/blog/from-logs-to-intelligent-choices-inside-noveum-ais-evaluation-process):\nFrom Development to Production - Inside Noveum.ai's AI Observability Platform\nShashank Agarwal\n3/3/2025\nIntroduction\nEvery AI application tells a story\u2014but most developers never get to hear it...", + "retrieval.context_length": 2943, + "retrieval.sources_count": 5, + "retrieval.context_quality": "high", + "retrieval.effectiveness": 1, + "retrieval.context_utilization": 2.943, + "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/blog/from-logs-to-intelligent-choices-inside-noveum-ais-evaluation-process):\nFrom Development to Production - Inside Noveum.ai's AI Observability Platform\nShashank Agarwal\n3/3/2025\nIntroduction\nEvery AI application tells a story\u2014but most developers never get to hear it. When your RAG pipeline returns irrelevant results, when your multi-agent system gets stuck in loops, or when your LLM costs suddenly spike, you're left guessing what went wrong and where.\nThat's exactly the challenge Noveum.ai solves. Rather than flying blind, our platform provides comprehensive tracing a...\n\nSource 2 (https://noveum.ai/docs/getting-started/tracing-concepts):\n- Explore Framework Integrations - Framework-specific guidance\n- Learn Advanced Patterns - Custom instrumentation techniques\n- Master the Dashboard - Analyze your traces effectively\nRemember: Good observability is not about collecting all possible data, but about collecting the right data that helps you understand, debug, and optimize your AI applications.\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to O...\n\nSource 3 (https://noveum.ai/en/docs/getting-started/tracing-concepts):\n- Explore Framework Integrations - Framework-specific guidance\n- Learn Advanced Patterns - Custom instrumentation techniques\n- Master the Dashboard - Analyze your traces effectively\nRemember: Good observability is not about collecting all possible data, but about collecting the right data that helps you understand, debug, and optimize your AI applications.\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to O...\n\nSource 4 (https://noveum.ai/en/blog/from-logs-to-intelligent-choices-inside-noveum-ais-evaluation-process):\nThe AI Observability Challenge\nWhy Traditional Monitoring Falls Short\nTraditional application monitoring tools weren't built for AI applications. They can tell you if your API is responding, but they can't answer the questions that matter most for AI systems:\n- Why did my RAG pipeline retrieve irrelevant documents?\n- Which LLM calls are driving my costs?\n- How are my agents communicating with each other?\n- What's causing hallucinations in my responses?\n- Why is my embedding generation so slow?\nW...\n\nSource 5 (https://noveum.ai/en/docs/advanced/rag-observability):\n\ud83d\udd27 Best Practices for RAG Observability\n1. Comprehensive Pipeline Tracing\n2. Quality Monitoring\n3. A/B Testing for RAG Components\n\ud83c\udfaf Advanced RAG Patterns\nMulti-Modal RAG\nConversational RAG\nRAG observability with Noveum.ai provides the deep insights needed to build, optimize, and scale retrieval-augmented generation systems. By implementing comprehensive tracing across retrieval, context preparation, and generation stages, you can ensure your RAG pipeline delivers accurate, relevant, and cost-effe...\n\nUser Question: What are the current trends in observability tools?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", + "prompt.prompt_length": 3490, + "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/blog/from-logs-to-intelligent-choices-inside-noveum-ais-evaluation-process):\nFrom Development to Production - Inside Noveum.ai's AI Observability Platform\nShashank Agarw...", + "prompt.instruction_following": "rag_optimized", + "response.answer_length": 897, + "response.answer_completeness": "complete", + "response.response_quality": "high", + "response.source_citation": 5, + "response.context_utilization": 0.30479102956167176, + "output_response": "RAG Answer: The provided context does not specifically outline current trends in observability tools beyond the focus on AI applications. However, it highlights the limitations of traditional monitoring tools for...", + "evaluation.retrieval_effectiveness": 1, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 5, + "evaluation.context_relevance": "high", + "evaluation.overall_quality": "high", + "evaluation.ready_for_production": true, + "rag.retrieval_strategy": "semantic_similarity", + "rag.vector_search_results": 5, + "rag.context_synthesis": "multi_source", + "rag.document_coverage": 1, + "rag.information_density": 0.30479102956167176 + }, + "events": [], + "links": [], + "trace_trace_id": "fa5bb641-f4b7-4741-b06c-392a5f2a9aea", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:21.645000000", + "trace_end_time": "2025-10-16 08:50:26.683000000", + "trace_duration_ms": 5038.112, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:21.645000000", + "updated_at": "2025-10-16 08:50:21.645000000" + }, + { + "span_id": "2e06953f-5cec-4873-92b5-91f7281f6d7c", + "trace_id": "c0a95e8e-513c-47eb-b746-4e6016aeaf7d", + "parent_span_id": "9e41eab5-cd8e-4e46-a15b-e5f52c9a4073", + "name": "agent.query_routing", + "start_time": "2025-10-16 08:50:08.944000000", + "end_time": "2025-10-16 08:50:20.641000000", + "duration_ms": 11696.524, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "Tell me about recent developments in machine learning", + "query_length": 53, + "query_type": "routing_query", + "output_response": "Routed to Web Search for query processing", + "router.classification": "Web Search", + "router.confidence_score": 1, + "router.rag_keyword_score": 0, + "router.web_keyword_score": 1, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "c0a95e8e-513c-47eb-b746-4e6016aeaf7d", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:08.944000000", + "trace_end_time": "2025-10-16 08:50:20.641000000", + "trace_duration_ms": 11696.768, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:08.944000000", + "updated_at": "2025-10-16 08:50:08.944000000" + }, + { + "span_id": "9e41eab5-cd8e-4e46-a15b-e5f52c9a4073", + "trace_id": "c0a95e8e-513c-47eb-b746-4e6016aeaf7d", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-16 08:50:08.944000000", + "end_time": "2025-10-16 08:50:20.641000000", + "duration_ms": 11696.671999999999, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "Tell me about recent developments in machine learning", + "query.length": 53, + "processing.mode": "Web Search", + "processing.time_seconds": 11.696574449539185, + "processing.response_length": 2001, + "processing.sources_count": 1, + "output_response": "Final Answer: The web search did not yield specific results regarding recent developments in machine learning. However, I can provide a general overview based on my knowledge up to October 2023.\n\nRecent development...", + "final_answer_mode": "Web Search", + "query_processed.input_query": "Tell me about recent developments in machine learning", + "query_processed.output_response": "Successfully processed query using Web Search, generated 2001 character response", + "query_processed.mode": "Web Search", + "query_processed.processing_time": 11.696574449539185, + "query_processed.response_length": 2001 + }, + "events": [], + "links": [], + "trace_trace_id": "c0a95e8e-513c-47eb-b746-4e6016aeaf7d", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:08.944000000", + "trace_end_time": "2025-10-16 08:50:20.641000000", + "trace_duration_ms": 11696.768, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:08.944000000", + "updated_at": "2025-10-16 08:50:08.944000000" + }, + { + "span_id": "7ddf938b-183a-427f-849b-b31b18a98d87", + "trace_id": "c0a95e8e-513c-47eb-b746-4e6016aeaf7d", + "parent_span_id": "2e06953f-5cec-4873-92b5-91f7281f6d7c", + "name": "agent.web_search_generation", + "start_time": "2025-10-16 08:50:08.945000000", + "end_time": "2025-10-16 08:50:20.641000000", + "duration_ms": 11696.328, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "web_search_agent", + "agent.operation": "web_search_generation", + "agent.capabilities": [ + "web_search", + "content_synthesis", + "response_generation" + ], + "agent.id": "noveum_web_search_agent", + "input_query": "Tell me about recent developments in machine learning", + "query_length": 53, + "query_type": "web_search_query", + "output_response": "Web Search Answer: The web search did not yield specific results regarding recent developments in machine learning. However, I can provide a general overview based on my knowledge up to October 2023.\n\nRecent development...", + "web_search.results_count": 1, + "web_search.context_length": 155, + "web_search.response_length": 2001, + "web_search.mode": "external_web_search" + }, + "events": [], + "links": [], + "trace_trace_id": "c0a95e8e-513c-47eb-b746-4e6016aeaf7d", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:08.944000000", + "trace_end_time": "2025-10-16 08:50:20.641000000", + "trace_duration_ms": 11696.768, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:08.944000000", + "updated_at": "2025-10-16 08:50:08.944000000" + }, + { + "span_id": "f3e4c594-f822-4f04-ab43-98fac161b9a3", + "trace_id": "c0a95e8e-513c-47eb-b746-4e6016aeaf7d", + "parent_span_id": "2e06953f-5cec-4873-92b5-91f7281f6d7c", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-16 08:50:08.945000000", + "end_time": "2025-10-16 08:50:08.945000000", + "duration_ms": 0.046, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: Tell me about recent developments in machine learning", + "query_length": 53, + "query_type": "routing_evaluation_query", + "classification.mode": "Web Search", + "classification.rag_keyword_score": 0, + "classification.web_keyword_score": 1, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "complex", + "query.intent": "general_knowledge", + "query.keyword_density": 0.125, + "query.domain_affinity": "general", + "routing.decision": "Routed to Web Search based on analysis", + "routing.rationale": "RAG score: 0, Web score: 1, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "RAG", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.019230769230769232, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: Web Search (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "c0a95e8e-513c-47eb-b746-4e6016aeaf7d", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:08.944000000", + "trace_end_time": "2025-10-16 08:50:20.641000000", + "trace_duration_ms": 11696.768, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:08.944000000", + "updated_at": "2025-10-16 08:50:08.944000000" + }, + { + "span_id": "b47d09be-a43b-4b0a-b7d3-d98bdf1773a3", + "trace_id": "c0a95e8e-513c-47eb-b746-4e6016aeaf7d", + "parent_span_id": "7ddf938b-183a-427f-849b-b31b18a98d87", + "name": "agent.llm_model_execution", + "start_time": "2025-10-16 08:50:09.490000000", + "end_time": "2025-10-16 08:50:20.641000000", + "duration_ms": 11151.141000000001, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: Tell me about recent developments in machine learning", + "query_length": 53, + "query_type": "web_search_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 11.151098012924194, + "model.latency_ms": 11151.098012924194, + "model.start_time": 1760604609.4902723, + "model.end_time": 1760604620.6413703, + "model.performance_tier": "slow", + "model.prompt_tokens": 163, + "model.completion_tokens": 500, + "model.total_tokens": 663, + "model.tokens_per_second": 59.456028386763236, + "model.estimated_cost": 0.01989, + "model.efficiency_score": 3.018099547511312, + "model.response_length": 2001, + "model.response_quality": "high", + "model.output_response": "Model Response: The web search did not yield specific results regarding recent developments in machine learning. However, I can provide a general overview based on my knowledge up to October 2023.\n\nRecent development...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "c0a95e8e-513c-47eb-b746-4e6016aeaf7d", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:08.944000000", + "trace_end_time": "2025-10-16 08:50:20.641000000", + "trace_duration_ms": 11696.768, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:08.944000000", + "updated_at": "2025-10-16 08:50:08.944000000" + }, + { + "span_id": "bfea8500-0e5c-4556-a014-5b2ec49b7ad5", + "trace_id": "c0a95e8e-513c-47eb-b746-4e6016aeaf7d", + "parent_span_id": "7ddf938b-183a-427f-849b-b31b18a98d87", + "name": "agent.web_search_evaluation_metrics", + "start_time": "2025-10-16 08:50:20.641000000", + "end_time": "2025-10-16 08:50:20.641000000", + "duration_ms": 0.018000000000000002, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "web_search_evaluation_metrics", + "agent.capabilities": [ + "web_search_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: Tell me about recent developments in machine learning", + "query_length": 53, + "query_type": "web_search_evaluation_query", + "web_search.results_count": 1, + "web_search.context_length": 155, + "web_search.context_synthesized": "Context from 1 web sources", + "web_search.search_effectiveness": 0.2, + "web_search.context_quality": "low", + "web_search.source_diversity": 1, + "web_search.information_synthesis": "low", + "web_search.external_knowledge_utilization": 0.155, + "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=Tell+me+about+recent+developments+in+machine+learning\n\nUser Question: Tell me about recent developments in machine learning\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", + "prompt.prompt_length": 653, + "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=Tell+me+about+recent+developments+in+machine+learning", + "prompt.instruction_following": "web_search_optimized", + "response.answer_length": 2001, + "response.answer_completeness": "complete", + "response.response_quality": "medium", + "response.source_citation": 1, + "response.context_utilization": 12.90967741935484, + "output_response": "Web Search Answer: The web search did not yield specific results regarding recent developments in machine learning. However, I can provide a general overview based on my knowledge up to October 2023.\n\nRecent development...", + "evaluation.search_effectiveness": 0.2, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 1, + "evaluation.context_relevance": "low", + "evaluation.overall_quality": "low", + "evaluation.ready_for_production": true, + "web_search.search_strategy": "duckduckgo_api", + "web_search.real_time_data": true, + "web_search.external_sources": 1, + "web_search.information_freshness": "current", + "web_search.knowledge_synthesis": "single_source" + }, + "events": [], + "links": [], + "trace_trace_id": "c0a95e8e-513c-47eb-b746-4e6016aeaf7d", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:08.944000000", + "trace_end_time": "2025-10-16 08:50:20.641000000", + "trace_duration_ms": 11696.768, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:08.944000000", + "updated_at": "2025-10-16 08:50:08.944000000" + }, + { + "span_id": "33297286-d440-446a-ba73-7cabd9d4b2e6", + "trace_id": "3af8c6e8-62d9-4893-92ba-cc265934ef03", + "parent_span_id": "4a3c230e-0dd5-4ba2-8e87-2a301f654fba", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-16 08:50:04.544000000", + "end_time": "2025-10-16 08:50:04.544000000", + "duration_ms": 0.038, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What's the weather like today?", + "query_length": 30, + "query_type": "routing_evaluation_query", + "classification.mode": "Web Search", + "classification.rag_keyword_score": 0, + "classification.web_keyword_score": 2, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "general_knowledge", + "query.keyword_density": 0.4, + "query.domain_affinity": "general", + "routing.decision": "Routed to Web Search based on analysis", + "routing.rationale": "RAG score: 0, Web score: 2, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "RAG", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.038461538461538464, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: Web Search (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "3af8c6e8-62d9-4893-92ba-cc265934ef03", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:04.544000000", + "trace_end_time": "2025-10-16 08:50:07.941000000", + "trace_duration_ms": 3397.228, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:04.544000000", + "updated_at": "2025-10-16 08:50:04.544000000" + }, + { + "span_id": "4a3c230e-0dd5-4ba2-8e87-2a301f654fba", + "trace_id": "3af8c6e8-62d9-4893-92ba-cc265934ef03", + "parent_span_id": "eefcbdc5-99fc-4f06-a9f4-f51ba50564db", + "name": "agent.query_routing", + "start_time": "2025-10-16 08:50:04.544000000", + "end_time": "2025-10-16 08:50:07.941000000", + "duration_ms": 3397.056, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What's the weather like today?", + "query_length": 30, + "query_type": "routing_query", + "output_response": "Routed to Web Search for query processing", + "router.classification": "Web Search", + "router.confidence_score": 1, + "router.rag_keyword_score": 0, + "router.web_keyword_score": 2, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "3af8c6e8-62d9-4893-92ba-cc265934ef03", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:04.544000000", + "trace_end_time": "2025-10-16 08:50:07.941000000", + "trace_duration_ms": 3397.228, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:04.544000000", + "updated_at": "2025-10-16 08:50:04.544000000" + }, + { + "span_id": "eefcbdc5-99fc-4f06-a9f4-f51ba50564db", + "trace_id": "3af8c6e8-62d9-4893-92ba-cc265934ef03", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-16 08:50:04.544000000", + "end_time": "2025-10-16 08:50:07.941000000", + "duration_ms": 3397.169, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What's the weather like today?", + "query.length": 30, + "processing.mode": "Web Search", + "processing.time_seconds": 3.3970956802368164, + "processing.response_length": 518, + "processing.sources_count": 1, + "output_response": "Final Answer: The web search results do not provide specific details about today's weather conditions in your location. However, they mention that you can get real-time updates, including temperature, humidity, and...", + "final_answer_mode": "Web Search", + "query_processed.input_query": "What's the weather like today?", + "query_processed.output_response": "Successfully processed query using Web Search, generated 518 character response", + "query_processed.mode": "Web Search", + "query_processed.processing_time": 3.3970956802368164, + "query_processed.response_length": 518 + }, + "events": [], + "links": [], + "trace_trace_id": "3af8c6e8-62d9-4893-92ba-cc265934ef03", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:04.544000000", + "trace_end_time": "2025-10-16 08:50:07.941000000", + "trace_duration_ms": 3397.228, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:04.544000000", + "updated_at": "2025-10-16 08:50:04.544000000" + }, + { + "span_id": "9683465c-e851-4004-a858-c8fdd1a2b3da", + "trace_id": "3af8c6e8-62d9-4893-92ba-cc265934ef03", + "parent_span_id": "4a3c230e-0dd5-4ba2-8e87-2a301f654fba", + "name": "agent.web_search_generation", + "start_time": "2025-10-16 08:50:04.545000000", + "end_time": "2025-10-16 08:50:07.941000000", + "duration_ms": 3396.871, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "web_search_agent", + "agent.operation": "web_search_generation", + "agent.capabilities": [ + "web_search", + "content_synthesis", + "response_generation" + ], + "agent.id": "noveum_web_search_agent", + "input_query": "What's the weather like today?", + "query_length": 30, + "query_type": "web_search_query", + "output_response": "Web Search Answer: The web search results do not provide specific details about today's weather conditions in your location. However, they mention that you can get real-time updates, including temperature, humidity, and...", + "web_search.results_count": 1, + "web_search.context_length": 1137, + "web_search.response_length": 518, + "web_search.mode": "external_web_search" + }, + "events": [], + "links": [], + "trace_trace_id": "3af8c6e8-62d9-4893-92ba-cc265934ef03", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:04.544000000", + "trace_end_time": "2025-10-16 08:50:07.941000000", + "trace_duration_ms": 3397.228, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:04.544000000", + "updated_at": "2025-10-16 08:50:04.544000000" + }, + { + "span_id": "6413eb7e-e5cd-4c46-9651-de14c72c260e", + "trace_id": "3af8c6e8-62d9-4893-92ba-cc265934ef03", + "parent_span_id": "9683465c-e851-4004-a858-c8fdd1a2b3da", + "name": "agent.llm_model_execution", + "start_time": "2025-10-16 08:50:04.986000000", + "end_time": "2025-10-16 08:50:07.941000000", + "duration_ms": 2955.55, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What's the weather like today?", + "query_length": 30, + "query_type": "web_search_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 2.955509662628174, + "model.latency_ms": 2955.509662628174, + "model.start_time": 1760604604.9862766, + "model.end_time": 1760604607.9417863, + "model.performance_tier": "medium", + "model.prompt_tokens": 403, + "model.completion_tokens": 129, + "model.total_tokens": 532, + "model.tokens_per_second": 180.0027950261957, + "model.estimated_cost": 0.015960000000000002, + "model.efficiency_score": 0.9736842105263158, + "model.response_length": 518, + "model.response_quality": "high", + "model.output_response": "Model Response: The web search results do not provide specific details about today's weather conditions in your location. However, they mention that you can get real-time updates, including temperature, humidity, and...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "3af8c6e8-62d9-4893-92ba-cc265934ef03", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:04.544000000", + "trace_end_time": "2025-10-16 08:50:07.941000000", + "trace_duration_ms": 3397.228, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:04.544000000", + "updated_at": "2025-10-16 08:50:04.544000000" + }, + { + "span_id": "64df69fb-0bcc-4845-8bfe-5a721c1ed912", + "trace_id": "3af8c6e8-62d9-4893-92ba-cc265934ef03", + "parent_span_id": "9683465c-e851-4004-a858-c8fdd1a2b3da", + "name": "agent.web_search_evaluation_metrics", + "start_time": "2025-10-16 08:50:07.941000000", + "end_time": "2025-10-16 08:50:07.941000000", + "duration_ms": 0.017, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "web_search_evaluation_metrics", + "agent.capabilities": [ + "web_search_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What's the weather like today?", + "query_length": 30, + "query_type": "web_search_evaluation_query", + "web_search.results_count": 1, + "web_search.context_length": 1137, + "web_search.context_synthesized": "Context from 1 web sources", + "web_search.search_effectiveness": 0.2, + "web_search.context_quality": "high", + "web_search.source_diversity": 1, + "web_search.information_synthesis": "low", + "web_search.external_knowledge_utilization": 1.137, + "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nStay updated with the latest weather conditions in Komna. Get real-time updates, temperature, humidity, AQI, and detailed forecast for today. Dec 31, 2024 \u00b7 Komna Weather Forecast: Check the latest temperature in Komna today along with details such as humidity, visibility, air quality, wind speed, pressure, and weekly weather report at Shuru. 2 days ago \u00b7 Latest weather forecast for India for today's, hourly weather forecast, including today's temperatures in India, wind, rain and more. 2 days ago \u00b7 Get the latest hourly weather updates for India today. Detailed forecast including temperature, wind, rain, snow, and UV index. Stay informed about today's weather conditions in India. 2 days ago \u00b7 Today's Weather Report ... Forecasts Urban Meteorological Services Short to Medium Range Model Guidance Extended Range Model Guidance Seasonal Forecast Quantitative Precipitation Forecast Latest CAP Alerts Specialised products & forecasts Cyclones | Monsoon | Satellite Radar | Rainfall | Agromet Advisories MAUSAM : | | Video Meghdoot Agro ...\nURL: https://duckduckgo.com/?q=What's+the+weather+like+today?\n\nUser Question: What's the weather like today?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", + "prompt.prompt_length": 1612, + "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nStay updated with the latest weather conditions in Komna. Get real-time updates, temperature, humidity, AQI, and detailed forecast for today. Dec 31, 2024 \u00b7 Komna Weather F...", + "prompt.instruction_following": "web_search_optimized", + "response.answer_length": 518, + "response.answer_completeness": "complete", + "response.response_quality": "medium", + "response.source_citation": 1, + "response.context_utilization": 0.455584872471416, + "output_response": "Web Search Answer: The web search results do not provide specific details about today's weather conditions in your location. However, they mention that you can get real-time updates, including temperature, humidity, and...", + "evaluation.search_effectiveness": 0.2, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 1, + "evaluation.context_relevance": "high", + "evaluation.overall_quality": "low", + "evaluation.ready_for_production": true, + "web_search.search_strategy": "duckduckgo_api", + "web_search.real_time_data": true, + "web_search.external_sources": 1, + "web_search.information_freshness": "current", + "web_search.knowledge_synthesis": "single_source" + }, + "events": [], + "links": [], + "trace_trace_id": "3af8c6e8-62d9-4893-92ba-cc265934ef03", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:04.544000000", + "trace_end_time": "2025-10-16 08:50:07.941000000", + "trace_duration_ms": 3397.228, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:04.544000000", + "updated_at": "2025-10-16 08:50:04.544000000" + }, + { + "span_id": "719a1a8f-fc50-439c-899b-436146accd23", + "trace_id": "d7b8b811-ad01-48f1-917f-ace7e8d289e6", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-16 08:50:01.375000000", + "end_time": "2025-10-16 08:50:03.540000000", + "duration_ms": 2164.4809999999998, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What are the latest AI news today?", + "query.length": 34, + "processing.mode": "Web Search", + "processing.time_seconds": 2.1643810272216797, + "processing.response_length": 298, + "processing.sources_count": 1, + "output_response": "Final Answer: The web search did not yield any specific results regarding the latest AI news today. Unfortunately, I cannot provide current updates or news articles on AI developments at this moment. For the latest...", + "final_answer_mode": "Web Search", + "query_processed.input_query": "What are the latest AI news today?", + "query_processed.output_response": "Successfully processed query using Web Search, generated 298 character response", + "query_processed.mode": "Web Search", + "query_processed.processing_time": 2.1643810272216797, + "query_processed.response_length": 298 + }, + "events": [], + "links": [], + "trace_trace_id": "d7b8b811-ad01-48f1-917f-ace7e8d289e6", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:01.375000000", + "trace_end_time": "2025-10-16 08:50:03.540000000", + "trace_duration_ms": 2164.576, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:01.375000000", + "updated_at": "2025-10-16 08:50:01.375000000" + }, + { + "span_id": "644a712f-1c9e-4e9a-9073-74621b754aa2", + "trace_id": "d7b8b811-ad01-48f1-917f-ace7e8d289e6", + "parent_span_id": "719a1a8f-fc50-439c-899b-436146accd23", + "name": "agent.query_routing", + "start_time": "2025-10-16 08:50:01.376000000", + "end_time": "2025-10-16 08:50:03.540000000", + "duration_ms": 2164.3219999999997, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What are the latest AI news today?", + "query_length": 34, + "query_type": "routing_query", + "output_response": "Routed to Web Search for query processing", + "router.classification": "Web Search", + "router.confidence_score": 1, + "router.rag_keyword_score": 0, + "router.web_keyword_score": 3, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "d7b8b811-ad01-48f1-917f-ace7e8d289e6", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:01.375000000", + "trace_end_time": "2025-10-16 08:50:03.540000000", + "trace_duration_ms": 2164.576, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:01.375000000", + "updated_at": "2025-10-16 08:50:01.375000000" + }, + { + "span_id": "8a4abdbe-6ad7-4d43-b234-81667c4abbc2", + "trace_id": "d7b8b811-ad01-48f1-917f-ace7e8d289e6", + "parent_span_id": "644a712f-1c9e-4e9a-9073-74621b754aa2", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-16 08:50:01.376000000", + "end_time": "2025-10-16 08:50:01.376000000", + "duration_ms": 0.051, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What are the latest AI news today?", + "query_length": 34, + "query_type": "routing_evaluation_query", + "classification.mode": "Web Search", + "classification.rag_keyword_score": 0, + "classification.web_keyword_score": 3, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "general_knowledge", + "query.keyword_density": 0.42857142857142855, + "query.domain_affinity": "general", + "routing.decision": "Routed to Web Search based on analysis", + "routing.rationale": "RAG score: 0, Web score: 3, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "RAG", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.057692307692307696, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: Web Search (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "d7b8b811-ad01-48f1-917f-ace7e8d289e6", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:01.375000000", + "trace_end_time": "2025-10-16 08:50:03.540000000", + "trace_duration_ms": 2164.576, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:01.375000000", + "updated_at": "2025-10-16 08:50:01.375000000" + }, + { + "span_id": "904621e8-c5fa-4b2d-a628-8f10eb9da51d", + "trace_id": "d7b8b811-ad01-48f1-917f-ace7e8d289e6", + "parent_span_id": "644a712f-1c9e-4e9a-9073-74621b754aa2", + "name": "agent.web_search_generation", + "start_time": "2025-10-16 08:50:01.376000000", + "end_time": "2025-10-16 08:50:03.540000000", + "duration_ms": 2164.076, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "web_search_agent", + "agent.operation": "web_search_generation", + "agent.capabilities": [ + "web_search", + "content_synthesis", + "response_generation" + ], + "agent.id": "noveum_web_search_agent", + "input_query": "What are the latest AI news today?", + "query_length": 34, + "query_type": "web_search_query", + "output_response": "Web Search Answer: The web search did not yield any specific results regarding the latest AI news today. Unfortunately, I cannot provide current updates or news articles on AI developments at this moment. For the latest...", + "web_search.results_count": 1, + "web_search.context_length": 136, + "web_search.response_length": 298, + "web_search.mode": "external_web_search" + }, + "events": [], + "links": [], + "trace_trace_id": "d7b8b811-ad01-48f1-917f-ace7e8d289e6", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:01.375000000", + "trace_end_time": "2025-10-16 08:50:03.540000000", + "trace_duration_ms": 2164.576, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:01.375000000", + "updated_at": "2025-10-16 08:50:01.375000000" + }, + { + "span_id": "421dd1af-2dbc-477f-bd59-b3194b14ae3a", + "trace_id": "d7b8b811-ad01-48f1-917f-ace7e8d289e6", + "parent_span_id": "904621e8-c5fa-4b2d-a628-8f10eb9da51d", + "name": "agent.llm_model_execution", + "start_time": "2025-10-16 08:50:01.708000000", + "end_time": "2025-10-16 08:50:03.540000000", + "duration_ms": 1831.237, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What are the latest AI news today?", + "query_length": 34, + "query_type": "web_search_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 1.8311803340911865, + "model.latency_ms": 1831.1803340911865, + "model.start_time": 1760604601.7090185, + "model.end_time": 1760604603.5401988, + "model.performance_tier": "fast", + "model.prompt_tokens": 153, + "model.completion_tokens": 74, + "model.total_tokens": 227, + "model.tokens_per_second": 123.96376029926071, + "model.estimated_cost": 0.00681, + "model.efficiency_score": 1.3127753303964758, + "model.response_length": 298, + "model.response_quality": "high", + "model.output_response": "Model Response: The web search did not yield any specific results regarding the latest AI news today. Unfortunately, I cannot provide current updates or news articles on AI developments at this moment. For the latest...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "d7b8b811-ad01-48f1-917f-ace7e8d289e6", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:01.375000000", + "trace_end_time": "2025-10-16 08:50:03.540000000", + "trace_duration_ms": 2164.576, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:01.375000000", + "updated_at": "2025-10-16 08:50:01.375000000" + }, + { + "span_id": "159aad6f-7174-406d-b7d9-eed9360d993c", + "trace_id": "d7b8b811-ad01-48f1-917f-ace7e8d289e6", + "parent_span_id": "904621e8-c5fa-4b2d-a628-8f10eb9da51d", + "name": "agent.web_search_evaluation_metrics", + "start_time": "2025-10-16 08:50:03.540000000", + "end_time": "2025-10-16 08:50:03.540000000", + "duration_ms": 0.028, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "web_search_evaluation_metrics", + "agent.capabilities": [ + "web_search_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What are the latest AI news today?", + "query_length": 34, + "query_type": "web_search_evaluation_query", + "web_search.results_count": 1, + "web_search.context_length": 136, + "web_search.context_synthesized": "Context from 1 web sources", + "web_search.search_effectiveness": 0.2, + "web_search.context_quality": "low", + "web_search.source_diversity": 1, + "web_search.information_synthesis": "low", + "web_search.external_knowledge_utilization": 0.136, + "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What+are+the+latest+AI+news+today?\n\nUser Question: What are the latest AI news today?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", + "prompt.prompt_length": 615, + "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What+are+the+latest+AI+news+today?", + "prompt.instruction_following": "web_search_optimized", + "response.answer_length": 298, + "response.answer_completeness": "complete", + "response.response_quality": "medium", + "response.source_citation": 1, + "response.context_utilization": 2.1911764705882355, + "output_response": "Web Search Answer: The web search did not yield any specific results regarding the latest AI news today. Unfortunately, I cannot provide current updates or news articles on AI developments at this moment. For the latest...", + "evaluation.search_effectiveness": 0.2, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 1, + "evaluation.context_relevance": "low", + "evaluation.overall_quality": "low", + "evaluation.ready_for_production": true, + "web_search.search_strategy": "duckduckgo_api", + "web_search.real_time_data": true, + "web_search.external_sources": 1, + "web_search.information_freshness": "current", + "web_search.knowledge_synthesis": "single_source" + }, + "events": [], + "links": [], + "trace_trace_id": "d7b8b811-ad01-48f1-917f-ace7e8d289e6", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:50:01.375000000", + "trace_end_time": "2025-10-16 08:50:03.540000000", + "trace_duration_ms": 2164.576, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:50:01.375000000", + "updated_at": "2025-10-16 08:50:01.375000000" + }, + { + "span_id": "dcf2b38e-ab90-45ee-9a65-e6fea47a3fd7", + "trace_id": "df8340b4-b75a-48e9-a2f3-99725b46a00d", + "parent_span_id": "d0e2b320-c18d-4ec6-98a1-74da5252107e", + "name": "agent.query_routing", + "start_time": "2025-10-16 08:49:17.236000000", + "end_time": "2025-10-16 08:50:00.368000000", + "duration_ms": 43131.384, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What are the benefits of using Noveum Trace?", + "query_length": 44, + "query_type": "routing_query", + "output_response": "Routed to RAG for query processing", + "router.classification": "RAG", + "router.confidence_score": 1, + "router.rag_keyword_score": 2, + "router.web_keyword_score": 0, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "df8340b4-b75a-48e9-a2f3-99725b46a00d", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:49:17.236000000", + "trace_end_time": "2025-10-16 08:50:00.368000000", + "trace_duration_ms": 43131.554000000004, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:49:17.236000000", + "updated_at": "2025-10-16 08:49:17.236000000" + }, + { + "span_id": "d0e2b320-c18d-4ec6-98a1-74da5252107e", + "trace_id": "df8340b4-b75a-48e9-a2f3-99725b46a00d", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-16 08:49:17.236000000", + "end_time": "2025-10-16 08:50:00.368000000", + "duration_ms": 43131.481, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What are the benefits of using Noveum Trace?", + "query.length": 44, + "processing.mode": "RAG", + "processing.time_seconds": 43.131428718566895, + "processing.response_length": 1415, + "processing.sources_count": 5, + "output_response": "Final Answer: The benefits of using Noveum Trace include:\n\n1. **Comprehensive Tracing**: Noveum Trace offers decorator-based tracing for LLM calls, agents, and RAG pipelines, allowing for detailed monitoring of AI ...", + "final_answer_mode": "RAG", + "query_processed.input_query": "What are the benefits of using Noveum Trace?", + "query_processed.output_response": "Successfully processed query using RAG, generated 1415 character response", + "query_processed.mode": "RAG", + "query_processed.processing_time": 43.131428718566895, + "query_processed.response_length": 1415 + }, + "events": [], + "links": [], + "trace_trace_id": "df8340b4-b75a-48e9-a2f3-99725b46a00d", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:49:17.236000000", + "trace_end_time": "2025-10-16 08:50:00.368000000", + "trace_duration_ms": 43131.554000000004, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:49:17.236000000", + "updated_at": "2025-10-16 08:49:17.236000000" + }, + { + "span_id": "fa93821f-c809-4071-87e1-a6c55014d37c", + "trace_id": "df8340b4-b75a-48e9-a2f3-99725b46a00d", + "parent_span_id": "dcf2b38e-ab90-45ee-9a65-e6fea47a3fd7", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-16 08:49:17.236000000", + "end_time": "2025-10-16 08:49:17.236000000", + "duration_ms": 0.034, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What are the benefits of using Noveum Trace?", + "query_length": 44, + "query_type": "routing_evaluation_query", + "classification.mode": "RAG", + "classification.rag_keyword_score": 2, + "classification.web_keyword_score": 0, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "noveum_specific", + "query.keyword_density": 0.25, + "query.domain_affinity": "noveum", + "routing.decision": "Routed to RAG based on analysis", + "routing.rationale": "RAG score: 2, Web score: 0, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "Web Search", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.038461538461538464, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: RAG (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "df8340b4-b75a-48e9-a2f3-99725b46a00d", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:49:17.236000000", + "trace_end_time": "2025-10-16 08:50:00.368000000", + "trace_duration_ms": 43131.554000000004, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:49:17.236000000", + "updated_at": "2025-10-16 08:49:17.236000000" + }, + { + "span_id": "024ad5aa-f394-4c26-af51-564f3f51c6f2", + "trace_id": "df8340b4-b75a-48e9-a2f3-99725b46a00d", + "parent_span_id": "dcf2b38e-ab90-45ee-9a65-e6fea47a3fd7", + "name": "agent.llm-rag", + "start_time": "2025-10-16 08:49:17.237000000", + "end_time": "2025-10-16 08:50:00.368000000", + "duration_ms": 43131.19300000001, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "rag_agent", + "agent.operation": "llm-rag", + "agent.capabilities": [ + "document_retrieval", + "context_generation", + "response_generation" + ], + "agent.id": "noveum_rag_agent", + "input_query": "What are the benefits of using Noveum Trace?", + "query_length": 44, + "query_type": "rag_query", + "output_response": "RAG Answer: The benefits of using Noveum Trace include:\n\n1. **Comprehensive Tracing**: Noveum Trace offers decorator-based tracing for LLM calls, agents, and RAG pipelines, allowing for detailed monitoring of AI ...", + "rag.context_length": 2773, + "rag.sources_count": 5, + "rag.answer_length": 1415, + "rag.mode": "retrieval_augmented_generation" + }, + "events": [], + "links": [], + "trace_trace_id": "df8340b4-b75a-48e9-a2f3-99725b46a00d", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:49:17.236000000", + "trace_end_time": "2025-10-16 08:50:00.368000000", + "trace_duration_ms": 43131.554000000004, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:49:17.236000000", + "updated_at": "2025-10-16 08:49:17.236000000" + }, + { + "span_id": "3d9db951-d8b0-4990-820d-61e4b1011709", + "trace_id": "df8340b4-b75a-48e9-a2f3-99725b46a00d", + "parent_span_id": "024ad5aa-f394-4c26-af51-564f3f51c6f2", + "name": "agent.llm_model_execution", + "start_time": "2025-10-16 08:49:20.224000000", + "end_time": "2025-10-16 08:49:58.420000000", + "duration_ms": 38196.111000000004, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What are the benefits of using Noveum Trace?", + "query_length": 44, + "query_type": "rag_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 38.19604015350342, + "model.latency_ms": 38196.04015350342, + "model.start_time": 1760604560.2242246, + "model.end_time": 1760604598.4202647, + "model.performance_tier": "slow", + "model.prompt_tokens": 828, + "model.completion_tokens": 353, + "model.total_tokens": 1181, + "model.tokens_per_second": 30.919435503098253, + "model.estimated_cost": 0.03543, + "model.efficiency_score": 1.1981371718882303, + "model.response_length": 1415, + "model.response_quality": "high", + "model.output_response": "Model Response: The benefits of using Noveum Trace include:\n\n1. **Comprehensive Tracing**: Noveum Trace offers decorator-based tracing for LLM calls, agents, and RAG pipelines, allowing for detailed monitoring of AI ...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "df8340b4-b75a-48e9-a2f3-99725b46a00d", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:49:17.236000000", + "trace_end_time": "2025-10-16 08:50:00.368000000", + "trace_duration_ms": 43131.554000000004, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:49:17.236000000", + "updated_at": "2025-10-16 08:49:17.236000000" + }, + { + "span_id": "8c33ffac-4d76-42ba-9737-89596e136b3d", + "trace_id": "df8340b4-b75a-48e9-a2f3-99725b46a00d", + "parent_span_id": "024ad5aa-f394-4c26-af51-564f3f51c6f2", + "name": "agent.rag_evaluation_metrics", + "start_time": "2025-10-16 08:49:58.420000000", + "end_time": "2025-10-16 08:49:58.420000000", + "duration_ms": 0.037, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "rag_evaluation_metrics", + "agent.capabilities": [ + "retrieval_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What are the benefits of using Noveum Trace?", + "query_length": 44, + "query_type": "rag_evaluation_query", + "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/docs/platform/dashboard):\n- Bulk Export: Download multiple traces simultaneously\n- Comparative Analysis: Compare performance across multiple traces\nReal-Time Updates\n- Live Refresh: Automatic updates as new traces arrive\n- Connection Monitoring: Real-time status of your t...", + "retrieval.context_length": 2773, + "retrieval.sources_count": 5, + "retrieval.context_quality": "high", + "retrieval.effectiveness": 1, + "retrieval.context_utilization": 2.773, + "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/docs/platform/dashboard):\n- Bulk Export: Download multiple traces simultaneously\n- Comparative Analysis: Compare performance across multiple traces\nReal-Time Updates\n- Live Refresh: Automatic updates as new traces arrive\n- Connection Monitoring: Real-time status of your trace ingestion pipeline\n- Performance Indicators: Live metrics for system health monitoring\nReady to dive deeper? Explore Projects & Environments to organize your AI applications, or check out Team Collaboration to share insights with your team.\nGet Earl...\n\nSource 2 (https://noveum.ai/en/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\n\ud83d\udcd8 TypeScript SDK (\n@noveum/trace\n)- Framework integrations for Next.js, Express.js, and Hono\n- TypeScript-first design with full type safety\n- Universal compatibility (Node.js, Edge Runtime, browsers)\n-\n\ud83d\udcca Noveum Platform\n- Real-time dashboard for analyzing traces and performance\n- Ad...\n\nSource 3 (https://noveum.ai/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\n\ud83d\udcd8 TypeScript SDK (\n@noveum/trace\n)- Framework integrations for Next.js, Express.js, and Hono\n- TypeScript-first design with full type safety\n- Universal compatibility (Node.js, Edge Runtime, browsers)\n-\n\ud83d\udcca Noveum Platform\n- Real-time dashboard for analyzing traces and performance\n- Ad...\n\nSource 4 (https://noveum.ai/en/docs/getting-started/overview):\n- Advanced search & filtering across traces and spans\n- Cost analysis and optimization recommendations\n- Team collaboration with shared insights and alerts\n\ud83d\udd0d What Noveum Traces\nLLM Operations\n- Model calls across all providers (OpenAI, Anthropic, Google, etc.)\n- Token usage and cost calculation\n- Prompt engineering effectiveness\n- Response quality metrics\nRAG Pipelines\n- Document retrieval performance and relevance\n- Embedding generation costs and latency\n- Context assembly and prompt constructi...\n\nSource 5 (https://noveum.ai/docs/getting-started/overview):\n- Advanced search & filtering across traces and spans\n- Cost analysis and optimization recommendations\n- Team collaboration with shared insights and alerts\n\ud83d\udd0d What Noveum Traces\nLLM Operations\n- Model calls across all providers (OpenAI, Anthropic, Google, etc.)\n- Token usage and cost calculation\n- Prompt engineering effectiveness\n- Response quality metrics\nRAG Pipelines\n- Document retrieval performance and relevance\n- Embedding generation costs and latency\n- Context assembly and prompt constructi...\n\nUser Question: What are the benefits of using Noveum Trace?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", + "prompt.prompt_length": 3313, + "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/docs/platform/dashboard):\n- Bulk Export: Download multiple traces simultaneously\n- Comparative Analysis: Compare performance across multiple traces\nReal-Time Updates\n- Live...", + "prompt.instruction_following": "rag_optimized", + "response.answer_length": 1415, + "response.answer_completeness": "complete", + "response.response_quality": "high", + "response.source_citation": 5, + "response.context_utilization": 0.5102776776054815, + "output_response": "RAG Answer: The benefits of using Noveum Trace include:\n\n1. **Comprehensive Tracing**: Noveum Trace offers decorator-based tracing for LLM calls, agents, and RAG pipelines, allowing for detailed monitoring of AI ...", + "evaluation.retrieval_effectiveness": 1, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 5, + "evaluation.context_relevance": "high", + "evaluation.overall_quality": "high", + "evaluation.ready_for_production": true, + "rag.retrieval_strategy": "semantic_similarity", + "rag.vector_search_results": 5, + "rag.context_synthesis": "multi_source", + "rag.document_coverage": 1, + "rag.information_density": 0.5102776776054815 + }, + "events": [], + "links": [], + "trace_trace_id": "df8340b4-b75a-48e9-a2f3-99725b46a00d", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:49:17.236000000", + "trace_end_time": "2025-10-16 08:50:00.368000000", + "trace_duration_ms": 43131.554000000004, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:49:17.236000000", + "updated_at": "2025-10-16 08:49:17.236000000" + }, + { + "span_id": "5b1a7121-d0bd-4525-821d-7721c6fbbd70", + "trace_id": "325ec189-2876-4ebe-b33c-649491a5053e", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-16 08:49:11.192000000", + "end_time": "2025-10-16 08:49:16.232000000", + "duration_ms": 5039.753, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "How do I configure Noveum for my system?", + "query.length": 40, + "processing.mode": "RAG", + "processing.time_seconds": 5.039698600769043, + "processing.response_length": 712, + "processing.sources_count": 5, + "output_response": "Final Answer: To configure Noveum for your system, you can refer to the SDK Integration Guide available in the Noveum documentation. This guide will help you trace your AI applications with minimal code changes. Th...", + "final_answer_mode": "RAG", + "query_processed.input_query": "How do I configure Noveum for my system?", + "query_processed.output_response": "Successfully processed query using RAG, generated 712 character response", + "query_processed.mode": "RAG", + "query_processed.processing_time": 5.039698600769043, + "query_processed.response_length": 712 + }, + "events": [], + "links": [], + "trace_trace_id": "325ec189-2876-4ebe-b33c-649491a5053e", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:49:11.192000000", + "trace_end_time": "2025-10-16 08:49:16.232000000", + "trace_duration_ms": 5039.841, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:49:11.192000000", + "updated_at": "2025-10-16 08:49:11.192000000" + }, + { + "span_id": "761c92c6-d15a-4a8e-a16a-a365922f8d7c", + "trace_id": "325ec189-2876-4ebe-b33c-649491a5053e", + "parent_span_id": "5b1a7121-d0bd-4525-821d-7721c6fbbd70", + "name": "agent.query_routing", + "start_time": "2025-10-16 08:49:11.192000000", + "end_time": "2025-10-16 08:49:16.232000000", + "duration_ms": 5039.642, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "How do I configure Noveum for my system?", + "query_length": 40, + "query_type": "routing_query", + "output_response": "Routed to RAG for query processing", + "router.classification": "RAG", + "router.confidence_score": 1, + "router.rag_keyword_score": 2, + "router.web_keyword_score": 0, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "325ec189-2876-4ebe-b33c-649491a5053e", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:49:11.192000000", + "trace_end_time": "2025-10-16 08:49:16.232000000", + "trace_duration_ms": 5039.841, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:49:11.192000000", + "updated_at": "2025-10-16 08:49:11.192000000" + }, + { + "span_id": "8fcf969e-a577-45aa-9fe6-1870d9665189", + "trace_id": "325ec189-2876-4ebe-b33c-649491a5053e", + "parent_span_id": "761c92c6-d15a-4a8e-a16a-a365922f8d7c", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-16 08:49:11.192000000", + "end_time": "2025-10-16 08:49:11.192000000", + "duration_ms": 0.045000000000000005, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: How do I configure Noveum for my system?", + "query_length": 40, + "query_type": "routing_evaluation_query", + "classification.mode": "RAG", + "classification.rag_keyword_score": 2, + "classification.web_keyword_score": 0, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "noveum_specific", + "query.keyword_density": 0.25, + "query.domain_affinity": "noveum", + "routing.decision": "Routed to RAG based on analysis", + "routing.rationale": "RAG score: 2, Web score: 0, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "Web Search", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.038461538461538464, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: RAG (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "325ec189-2876-4ebe-b33c-649491a5053e", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:49:11.192000000", + "trace_end_time": "2025-10-16 08:49:16.232000000", + "trace_duration_ms": 5039.841, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:49:11.192000000", + "updated_at": "2025-10-16 08:49:11.192000000" + }, + { + "span_id": "9e5bbea4-cc68-4fe6-b3d0-c87852635ec9", + "trace_id": "325ec189-2876-4ebe-b33c-649491a5053e", + "parent_span_id": "761c92c6-d15a-4a8e-a16a-a365922f8d7c", + "name": "agent.llm-rag", + "start_time": "2025-10-16 08:49:11.192000000", + "end_time": "2025-10-16 08:49:16.232000000", + "duration_ms": 5039.39, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "rag_agent", + "agent.operation": "llm-rag", + "agent.capabilities": [ + "document_retrieval", + "context_generation", + "response_generation" + ], + "agent.id": "noveum_rag_agent", + "input_query": "How do I configure Noveum for my system?", + "query_length": 40, + "query_type": "rag_query", + "output_response": "RAG Answer: To configure Noveum for your system, you can refer to the SDK Integration Guide available in the Noveum documentation. This guide will help you trace your AI applications with minimal code changes. Th...", + "rag.context_length": 2237, + "rag.sources_count": 5, + "rag.answer_length": 712, + "rag.mode": "retrieval_augmented_generation" + }, + "events": [], + "links": [], + "trace_trace_id": "325ec189-2876-4ebe-b33c-649491a5053e", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:49:11.192000000", + "trace_end_time": "2025-10-16 08:49:16.232000000", + "trace_duration_ms": 5039.841, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:49:11.192000000", + "updated_at": "2025-10-16 08:49:11.192000000" + }, + { + "span_id": "5ee706ad-c4ae-4c0b-94d1-6b9c43d173b4", + "trace_id": "325ec189-2876-4ebe-b33c-649491a5053e", + "parent_span_id": "9e5bbea4-cc68-4fe6-b3d0-c87852635ec9", + "name": "agent.llm_model_execution", + "start_time": "2025-10-16 08:49:11.607000000", + "end_time": "2025-10-16 08:49:15.718000000", + "duration_ms": 4110.915, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: How do I configure Noveum for my system?", + "query_length": 40, + "query_type": "rag_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 4.110846042633057, + "model.latency_ms": 4110.846042633057, + "model.start_time": 1760604551.6074958, + "model.end_time": 1760604555.7183418, + "model.performance_tier": "medium", + "model.prompt_tokens": 693, + "model.completion_tokens": 178, + "model.total_tokens": 871, + "model.tokens_per_second": 211.87852596934326, + "model.estimated_cost": 0.02613, + "model.efficiency_score": 0.817451205510907, + "model.response_length": 712, + "model.response_quality": "high", + "model.output_response": "Model Response: To configure Noveum for your system, you can refer to the SDK Integration Guide available in the Noveum documentation. This guide will help you trace your AI applications with minimal code changes. Th...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "325ec189-2876-4ebe-b33c-649491a5053e", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:49:11.192000000", + "trace_end_time": "2025-10-16 08:49:16.232000000", + "trace_duration_ms": 5039.841, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:49:11.192000000", + "updated_at": "2025-10-16 08:49:11.192000000" + }, + { + "span_id": "d13bdeda-81af-400a-82ca-a5ef7e1e8a6d", + "trace_id": "325ec189-2876-4ebe-b33c-649491a5053e", + "parent_span_id": "9e5bbea4-cc68-4fe6-b3d0-c87852635ec9", + "name": "agent.rag_evaluation_metrics", + "start_time": "2025-10-16 08:49:15.718000000", + "end_time": "2025-10-16 08:49:15.718000000", + "duration_ms": 0.118, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "rag_evaluation_metrics", + "agent.capabilities": [ + "retrieval_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: How do I configure Noveum for my system?", + "query_length": 40, + "query_type": "rag_evaluation_query", + "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/docs):\nWelcome to Noveum.ai Docs\nComprehensive AI tracing and observability for LLM applications, RAG systems, and AI agents\nWelcome to the Noveum.ai documentation! Here you'll find everything you need to integrate, configure, and optimize your AI applications using ou...", + "retrieval.context_length": 2237, + "retrieval.sources_count": 5, + "retrieval.context_quality": "high", + "retrieval.effectiveness": 1, + "retrieval.context_utilization": 2.237, + "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/docs):\nWelcome to Noveum.ai Docs\nComprehensive AI tracing and observability for LLM applications, RAG systems, and AI agents\nWelcome to the Noveum.ai documentation! Here you'll find everything you need to integrate, configure, and optimize your AI applications using our comprehensive tracing and observability platform.\nWe provide powerful SDKs for Python and TypeScript that enable you to trace LLM calls, RAG pipelines, and multi-agent workflows with minimal code changes. Our documentation is organized ...\n\nSource 2 (https://noveum.ai/docs):\nWelcome to Noveum.ai Docs\nComprehensive AI tracing and observability for LLM applications, RAG systems, and AI agents\nWelcome to the Noveum.ai documentation! Here you'll find everything you need to integrate, configure, and optimize your AI applications using our comprehensive tracing and observability platform.\nWe provide powerful SDKs for Python and TypeScript that enable you to trace LLM calls, RAG pipelines, and multi-agent workflows with minimal code changes. Our documentation is organized ...\n\nSource 3 (https://noveum.ai/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nSource 4 (https://noveum.ai/en/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nSource 5 (https://noveum.ai/en/docs):\n- \ud83c\udf0d Universal: Works across frameworks, clouds, and deployment models\n\ud83e\udd1d Community & Support\n- \ud83d\udcac Discord Community: Join our Discord\n- \ud83d\udce7 Email Support: [email protected]\n- \ud83d\udc1b Bug Reports: GitHub Issues\n- \ud83d\udcd6 Knowledge Base: Help Center\nReady to get started? Head to our SDK Integration Guide to begin tracing your AI applications in under 5 minutes!\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability ...\n\nUser Question: How do I configure Noveum for my system?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", + "prompt.prompt_length": 2773, + "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/docs):\nWelcome to Noveum.ai Docs\nComprehensive AI tracing and observability for LLM applications, RAG systems, and AI agents\nWelcome to the Noveum.ai documentation! Here...", + "prompt.instruction_following": "rag_optimized", + "response.answer_length": 712, + "response.answer_completeness": "complete", + "response.response_quality": "high", + "response.source_citation": 5, + "response.context_utilization": 0.3182834152883326, + "output_response": "RAG Answer: To configure Noveum for your system, you can refer to the SDK Integration Guide available in the Noveum documentation. This guide will help you trace your AI applications with minimal code changes. Th...", + "evaluation.retrieval_effectiveness": 1, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 5, + "evaluation.context_relevance": "high", + "evaluation.overall_quality": "high", + "evaluation.ready_for_production": true, + "rag.retrieval_strategy": "semantic_similarity", + "rag.vector_search_results": 5, + "rag.context_synthesis": "multi_source", + "rag.document_coverage": 1, + "rag.information_density": 0.3182834152883326 + }, + "events": [], + "links": [], + "trace_trace_id": "325ec189-2876-4ebe-b33c-649491a5053e", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:49:11.192000000", + "trace_end_time": "2025-10-16 08:49:16.232000000", + "trace_duration_ms": 5039.841, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:49:11.192000000", + "updated_at": "2025-10-16 08:49:11.192000000" + }, + { + "span_id": "7d6564fc-41b2-4d74-acb1-e77b575c0006", + "trace_id": "d5528586-c741-4703-8791-5fccf5c3e123", + "parent_span_id": "e79a037e-82b1-41e9-bdc7-42353e98ead7", + "name": "agent.query_routing", + "start_time": "2025-10-16 08:49:05.152000000", + "end_time": "2025-10-16 08:49:10.187000000", + "duration_ms": 5034.446, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What monitoring capabilities does Noveum provide?", + "query_length": 49, + "query_type": "routing_query", + "output_response": "Routed to RAG for query processing", + "router.classification": "RAG", + "router.confidence_score": 1, + "router.rag_keyword_score": 2, + "router.web_keyword_score": 0, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "d5528586-c741-4703-8791-5fccf5c3e123", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:49:05.152000000", + "trace_end_time": "2025-10-16 08:49:10.187000000", + "trace_duration_ms": 5034.604, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:49:05.152000000", + "updated_at": "2025-10-16 08:49:05.152000000" + }, + { + "span_id": "e79a037e-82b1-41e9-bdc7-42353e98ead7", + "trace_id": "d5528586-c741-4703-8791-5fccf5c3e123", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-16 08:49:05.152000000", + "end_time": "2025-10-16 08:49:10.187000000", + "duration_ms": 5034.531, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What monitoring capabilities does Noveum provide?", + "query.length": 49, + "processing.mode": "RAG", + "processing.time_seconds": 5.0344908237457275, + "processing.response_length": 552, + "processing.sources_count": 5, + "output_response": "Final Answer: Noveum provides comprehensive monitoring capabilities for AI applications, including the ability to monitor, trace, and optimize AI agents across various frameworks such as LangChain, CrewAI, AutoGen,...", + "final_answer_mode": "RAG", + "query_processed.input_query": "What monitoring capabilities does Noveum provide?", + "query_processed.output_response": "Successfully processed query using RAG, generated 552 character response", + "query_processed.mode": "RAG", + "query_processed.processing_time": 5.0344908237457275, + "query_processed.response_length": 552 + }, + "events": [], + "links": [], + "trace_trace_id": "d5528586-c741-4703-8791-5fccf5c3e123", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:49:05.152000000", + "trace_end_time": "2025-10-16 08:49:10.187000000", + "trace_duration_ms": 5034.604, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:49:05.152000000", + "updated_at": "2025-10-16 08:49:05.152000000" + }, + { + "span_id": "78297793-d73b-4311-b5ba-73509ae9f96e", + "trace_id": "d5528586-c741-4703-8791-5fccf5c3e123", + "parent_span_id": "7d6564fc-41b2-4d74-acb1-e77b575c0006", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-16 08:49:05.153000000", + "end_time": "2025-10-16 08:49:05.153000000", + "duration_ms": 0.029, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What monitoring capabilities does Noveum provide?", + "query_length": 49, + "query_type": "routing_evaluation_query", + "classification.mode": "RAG", + "classification.rag_keyword_score": 2, + "classification.web_keyword_score": 0, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "noveum_specific", + "query.keyword_density": 0.3333333333333333, + "query.domain_affinity": "noveum", + "routing.decision": "Routed to RAG based on analysis", + "routing.rationale": "RAG score: 2, Web score: 0, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "Web Search", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.038461538461538464, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: RAG (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "d5528586-c741-4703-8791-5fccf5c3e123", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:49:05.152000000", + "trace_end_time": "2025-10-16 08:49:10.187000000", + "trace_duration_ms": 5034.604, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:49:05.152000000", + "updated_at": "2025-10-16 08:49:05.152000000" + }, + { + "span_id": "6a101d9d-de83-415a-8fb0-01a9fae5ffb9", + "trace_id": "d5528586-c741-4703-8791-5fccf5c3e123", + "parent_span_id": "7d6564fc-41b2-4d74-acb1-e77b575c0006", + "name": "agent.llm-rag", + "start_time": "2025-10-16 08:49:05.153000000", + "end_time": "2025-10-16 08:49:10.187000000", + "duration_ms": 5034.282, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "rag_agent", + "agent.operation": "llm-rag", + "agent.capabilities": [ + "document_retrieval", + "context_generation", + "response_generation" + ], + "agent.id": "noveum_rag_agent", + "input_query": "What monitoring capabilities does Noveum provide?", + "query_length": 49, + "query_type": "rag_query", + "output_response": "RAG Answer: Noveum provides comprehensive monitoring capabilities for AI applications, including the ability to monitor, trace, and optimize AI agents across various frameworks such as LangChain, CrewAI, AutoGen,...", + "rag.context_length": 2740, + "rag.sources_count": 5, + "rag.answer_length": 552, + "rag.mode": "retrieval_augmented_generation" + }, + "events": [], + "links": [], + "trace_trace_id": "d5528586-c741-4703-8791-5fccf5c3e123", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:49:05.152000000", + "trace_end_time": "2025-10-16 08:49:10.187000000", + "trace_duration_ms": 5034.604, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:49:05.152000000", + "updated_at": "2025-10-16 08:49:05.152000000" + }, + { + "span_id": "26e510c0-441a-4f9f-9dce-31180f47d060", + "trace_id": "d5528586-c741-4703-8791-5fccf5c3e123", + "parent_span_id": "6a101d9d-de83-415a-8fb0-01a9fae5ffb9", + "name": "agent.llm_model_execution", + "start_time": "2025-10-16 08:49:05.787000000", + "end_time": "2025-10-16 08:49:09.274000000", + "duration_ms": 3486.699, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What monitoring capabilities does Noveum provide?", + "query_length": 49, + "query_type": "rag_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 3.4866299629211426, + "model.latency_ms": 3486.6299629211426, + "model.start_time": 1760604545.787763, + "model.end_time": 1760604549.274393, + "model.performance_tier": "medium", + "model.prompt_tokens": 821, + "model.completion_tokens": 138, + "model.total_tokens": 959, + "model.tokens_per_second": 275.0506965747916, + "model.estimated_cost": 0.02877, + "model.efficiency_score": 0.5755995828988529, + "model.response_length": 552, + "model.response_quality": "high", + "model.output_response": "Model Response: Noveum provides comprehensive monitoring capabilities for AI applications, including the ability to monitor, trace, and optimize AI agents across various frameworks such as LangChain, CrewAI, AutoGen,...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "d5528586-c741-4703-8791-5fccf5c3e123", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:49:05.152000000", + "trace_end_time": "2025-10-16 08:49:10.187000000", + "trace_duration_ms": 5034.604, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:49:05.152000000", + "updated_at": "2025-10-16 08:49:05.152000000" + }, + { + "span_id": "fcd89e38-3d21-468b-860a-ae2de0bbd119", + "trace_id": "d5528586-c741-4703-8791-5fccf5c3e123", + "parent_span_id": "6a101d9d-de83-415a-8fb0-01a9fae5ffb9", + "name": "agent.rag_evaluation_metrics", + "start_time": "2025-10-16 08:49:09.274000000", + "end_time": "2025-10-16 08:49:09.274000000", + "duration_ms": 0.054, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "rag_evaluation_metrics", + "agent.capabilities": [ + "retrieval_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What monitoring capabilities does Noveum provide?", + "query_length": 49, + "query_type": "rag_evaluation_query", + "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en):\nMonitor all your AI Agents\nimprove AI Agents today\nNoveum.ai helps you monitor, trace, and optimize your AI applications.\nNoveum.ai works with any AI framework \u2013 LangChain, CrewAI, AutoGen, custom implementations, or direct LLM calls. One dashboard shows everything.\n...", + "retrieval.context_length": 2740, + "retrieval.sources_count": 5, + "retrieval.context_quality": "high", + "retrieval.effectiveness": 1, + "retrieval.context_utilization": 2.74, + "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en):\nMonitor all your AI Agents\nimprove AI Agents today\nNoveum.ai helps you monitor, trace, and optimize your AI applications.\nNoveum.ai works with any AI framework \u2013 LangChain, CrewAI, AutoGen, custom implementations, or direct LLM calls. One dashboard shows everything.\nMonitor, Evaluate, Improve Your AI Agents\nThe control plane for AI agents.\nMonitor Everything, Miss Nothing\nOur lightweight SDKs capture every trace and span across your AI agent ecosystem\u2014from simple LLM calls to complex multi-agent...\n\nSource 2 (https://noveum.ai/):\nMonitor all your AI Agents\nimprove AI Agents today\nNoveum.ai helps you monitor, trace, and optimize your AI applications.\nNoveum.ai works with any AI framework \u2013 LangChain, CrewAI, AutoGen, custom implementations, or direct LLM calls. One dashboard shows everything.\nMonitor, Evaluate, Improve Your AI Agents\nThe control plane for AI agents.\nMonitor Everything, Miss Nothing\nOur lightweight SDKs capture every trace and span across your AI agent ecosystem\u2014from simple LLM calls to complex multi-agent...\n\nSource 3 (https://noveum.ai):\nMonitor all your AI Agents\nimprove AI Agents today\nNoveum.ai helps you monitor, trace, and optimize your AI applications.\nNoveum.ai works with any AI framework \u2013 LangChain, CrewAI, AutoGen, custom implementations, or direct LLM calls. One dashboard shows everything.\nMonitor, Evaluate, Improve Your AI Agents\nThe control plane for AI agents.\nMonitor Everything, Miss Nothing\nOur lightweight SDKs capture every trace and span across your AI agent ecosystem\u2014from simple LLM calls to complex multi-agent...\n\nSource 4 (https://noveum.ai/en/docs/getting-started/overview):\nNoveum.ai Overview\nComprehensive AI tracing and observability platform for LLM applications, RAG systems, and AI agents\nWelcome to Noveum.ai\u2014the comprehensive tracing and observability platform built specifically for AI applications. Whether you're building LLM-powered chatbots, RAG systems, multi-agent workflows, or any AI-driven application, Noveum provides the insights you need to understand, debug, and optimize your systems.\n\ud83c\udfaf Why AI Applications Need Specialized Observability\nTraditional mo...\n\nSource 5 (https://noveum.ai/docs/getting-started/overview):\nNoveum.ai Overview\nComprehensive AI tracing and observability platform for LLM applications, RAG systems, and AI agents\nWelcome to Noveum.ai\u2014the comprehensive tracing and observability platform built specifically for AI applications. Whether you're building LLM-powered chatbots, RAG systems, multi-agent workflows, or any AI-driven application, Noveum provides the insights you need to understand, debug, and optimize your systems.\n\ud83c\udfaf Why AI Applications Need Specialized Observability\nTraditional mo...\n\nUser Question: What monitoring capabilities does Noveum provide?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", + "prompt.prompt_length": 3285, + "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en):\nMonitor all your AI Agents\nimprove AI Agents today\nNoveum.ai helps you monitor, trace, and optimize your AI applications.\nNoveum.ai works with any AI framework \u2013 LangC...", + "prompt.instruction_following": "rag_optimized", + "response.answer_length": 552, + "response.answer_completeness": "complete", + "response.response_quality": "high", + "response.source_citation": 5, + "response.context_utilization": 0.20145985401459854, + "output_response": "RAG Answer: Noveum provides comprehensive monitoring capabilities for AI applications, including the ability to monitor, trace, and optimize AI agents across various frameworks such as LangChain, CrewAI, AutoGen,...", + "evaluation.retrieval_effectiveness": 1, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 5, + "evaluation.context_relevance": "high", + "evaluation.overall_quality": "high", + "evaluation.ready_for_production": true, + "rag.retrieval_strategy": "semantic_similarity", + "rag.vector_search_results": 5, + "rag.context_synthesis": "multi_source", + "rag.document_coverage": 1, + "rag.information_density": 0.20145985401459854 + }, + "events": [], + "links": [], + "trace_trace_id": "d5528586-c741-4703-8791-5fccf5c3e123", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:49:05.152000000", + "trace_end_time": "2025-10-16 08:49:10.187000000", + "trace_duration_ms": 5034.604, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:49:05.152000000", + "updated_at": "2025-10-16 08:49:05.152000000" + }, + { + "span_id": "bd5f205f-0329-4811-b883-129d69f12382", + "trace_id": "8100b869-07c0-4e37-b51e-092c9d20b4a7", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-16 08:48:57.574000000", + "end_time": "2025-10-16 08:49:04.149000000", + "duration_ms": 6574.424, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "How does Noveum handle agent tracing?", + "query.length": 37, + "processing.mode": "RAG", + "processing.time_seconds": 6.5743818283081055, + "processing.response_length": 643, + "processing.sources_count": 5, + "output_response": "Final Answer: Noveum.ai handles agent tracing through its specialized Multi-Agent Tracing capabilities, which are designed to observe complex workflows and inter-agent communications within multi-agent systems. The...", + "final_answer_mode": "RAG", + "query_processed.input_query": "How does Noveum handle agent tracing?", + "query_processed.output_response": "Successfully processed query using RAG, generated 643 character response", + "query_processed.mode": "RAG", + "query_processed.processing_time": 6.5743818283081055, + "query_processed.response_length": 643 + }, + "events": [], + "links": [], + "trace_trace_id": "8100b869-07c0-4e37-b51e-092c9d20b4a7", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:48:57.574000000", + "trace_end_time": "2025-10-16 08:49:04.149000000", + "trace_duration_ms": 6574.500999999999, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:48:57.574000000", + "updated_at": "2025-10-16 08:48:57.574000000" + }, + { + "span_id": "0801bcfe-8c09-4ea6-a754-5564395d6ded", + "trace_id": "8100b869-07c0-4e37-b51e-092c9d20b4a7", + "parent_span_id": "3c35d86a-0df9-42dc-bfe3-bbe978cf62b5", + "name": "agent.llm-rag", + "start_time": "2025-10-16 08:48:57.575000000", + "end_time": "2025-10-16 08:49:04.149000000", + "duration_ms": 6574.135, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "rag_agent", + "agent.operation": "llm-rag", + "agent.capabilities": [ + "document_retrieval", + "context_generation", + "response_generation" + ], + "agent.id": "noveum_rag_agent", + "input_query": "How does Noveum handle agent tracing?", + "query_length": 37, + "query_type": "rag_query", + "output_response": "RAG Answer: Noveum.ai handles agent tracing through its specialized Multi-Agent Tracing capabilities, which are designed to observe complex workflows and inter-agent communications within multi-agent systems. The...", + "rag.context_length": 2736, + "rag.sources_count": 5, + "rag.answer_length": 643, + "rag.mode": "retrieval_augmented_generation" + }, + "events": [], + "links": [], + "trace_trace_id": "8100b869-07c0-4e37-b51e-092c9d20b4a7", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:48:57.574000000", + "trace_end_time": "2025-10-16 08:49:04.149000000", + "trace_duration_ms": 6574.500999999999, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:48:57.574000000", + "updated_at": "2025-10-16 08:48:57.574000000" + }, + { + "span_id": "3c35d86a-0df9-42dc-bfe3-bbe978cf62b5", + "trace_id": "8100b869-07c0-4e37-b51e-092c9d20b4a7", + "parent_span_id": "bd5f205f-0329-4811-b883-129d69f12382", + "name": "agent.query_routing", + "start_time": "2025-10-16 08:48:57.575000000", + "end_time": "2025-10-16 08:49:04.149000000", + "duration_ms": 6574.342, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "How does Noveum handle agent tracing?", + "query_length": 37, + "query_type": "routing_query", + "output_response": "Routed to RAG for query processing", + "router.classification": "RAG", + "router.confidence_score": 1, + "router.rag_keyword_score": 2, + "router.web_keyword_score": 0, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "8100b869-07c0-4e37-b51e-092c9d20b4a7", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:48:57.574000000", + "trace_end_time": "2025-10-16 08:49:04.149000000", + "trace_duration_ms": 6574.500999999999, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:48:57.574000000", + "updated_at": "2025-10-16 08:48:57.574000000" + }, + { + "span_id": "b1914316-f545-4bc9-abf4-3494cf42c4a5", + "trace_id": "8100b869-07c0-4e37-b51e-092c9d20b4a7", + "parent_span_id": "3c35d86a-0df9-42dc-bfe3-bbe978cf62b5", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-16 08:48:57.575000000", + "end_time": "2025-10-16 08:48:57.575000000", + "duration_ms": 0.032, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: How does Noveum handle agent tracing?", + "query_length": 37, + "query_type": "routing_evaluation_query", + "classification.mode": "RAG", + "classification.rag_keyword_score": 2, + "classification.web_keyword_score": 0, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "noveum_specific", + "query.keyword_density": 0.3333333333333333, + "query.domain_affinity": "noveum", + "routing.decision": "Routed to RAG based on analysis", + "routing.rationale": "RAG score: 2, Web score: 0, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "Web Search", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.038461538461538464, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: RAG (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "8100b869-07c0-4e37-b51e-092c9d20b4a7", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:48:57.574000000", + "trace_end_time": "2025-10-16 08:49:04.149000000", + "trace_duration_ms": 6574.500999999999, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:48:57.574000000", + "updated_at": "2025-10-16 08:48:57.574000000" + }, + { + "span_id": "0906814b-6a80-40e1-9f00-5376b809703a", + "trace_id": "8100b869-07c0-4e37-b51e-092c9d20b4a7", + "parent_span_id": "0801bcfe-8c09-4ea6-a754-5564395d6ded", + "name": "agent.llm_model_execution", + "start_time": "2025-10-16 08:48:58.009000000", + "end_time": "2025-10-16 08:49:03.349000000", + "duration_ms": 5340.01, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: How does Noveum handle agent tracing?", + "query_length": 37, + "query_type": "rag_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 5.339954614639282, + "model.latency_ms": 5339.954614639282, + "model.start_time": 1760604538.0093303, + "model.end_time": 1760604543.349285, + "model.performance_tier": "slow", + "model.prompt_tokens": 817, + "model.completion_tokens": 160, + "model.total_tokens": 977, + "model.tokens_per_second": 182.9603565022054, + "model.estimated_cost": 0.02931, + "model.efficiency_score": 0.6581371545547595, + "model.response_length": 643, + "model.response_quality": "high", + "model.output_response": "Model Response: Noveum.ai handles agent tracing through its specialized Multi-Agent Tracing capabilities, which are designed to observe complex workflows and inter-agent communications within multi-agent systems. The...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "8100b869-07c0-4e37-b51e-092c9d20b4a7", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:48:57.574000000", + "trace_end_time": "2025-10-16 08:49:04.149000000", + "trace_duration_ms": 6574.500999999999, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:48:57.574000000", + "updated_at": "2025-10-16 08:48:57.574000000" + }, + { + "span_id": "29822edb-c15f-4baa-b6ec-84d85fcc926d", + "trace_id": "8100b869-07c0-4e37-b51e-092c9d20b4a7", + "parent_span_id": "0801bcfe-8c09-4ea6-a754-5564395d6ded", + "name": "agent.rag_evaluation_metrics", + "start_time": "2025-10-16 08:49:03.349000000", + "end_time": "2025-10-16 08:49:03.349000000", + "duration_ms": 0.034999999999999996, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "rag_evaluation_metrics", + "agent.capabilities": [ + "retrieval_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: How does Noveum handle agent tracing?", + "query_length": 37, + "query_type": "rag_evaluation_query", + "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, vector searches, data retrieval, agent reasoning, and more. Without proper observability, debugging ...", + "retrieval.context_length": 2736, + "retrieval.sources_count": 5, + "retrieval.context_quality": "high", + "retrieval.effectiveness": 1, + "retrieval.context_utilization": 2.736, + "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, vector searches, data retrieval, agent reasoning, and more. Without proper observability, debugging becomes a nightmare, optimization is guesswork, and understanding user interactions is nearly impossible.\nNoveum.ai solves this by providing comprehensive tracing and observability specifically designed for AI applications.\nKey benefits of using Noveum.ai:\n- Complete Visibility: Trace...\n\nSource 2 (https://noveum.ai/docs/advanced/multi-agent-tracing):\nMulti-Agent Tracing\nObserve complex agent workflows and inter-agent communications with comprehensive tracing\nMulti-agent systems represent some of the most complex AI applications, involving multiple agents that coordinate, communicate, and collaborate to achieve shared goals. Noveum.ai provides specialized tracing capabilities to help you understand and optimize these intricate workflows.\n\ud83c\udfaf Why Multi-Agent Tracing Matters\nMulti-agent systems introduce unique observability challenges:\n- Complex...\n\nSource 3 (https://noveum.ai/en/docs/advanced/multi-agent-tracing):\nMulti-Agent Tracing\nObserve complex agent workflows and inter-agent communications with comprehensive tracing\nMulti-agent systems represent some of the most complex AI applications, involving multiple agents that coordinate, communicate, and collaborate to achieve shared goals. Noveum.ai provides specialized tracing capabilities to help you understand and optimize these intricate workflows.\n\ud83c\udfaf Why Multi-Agent Tracing Matters\nMulti-agent systems introduce unique observability challenges:\n- Complex...\n\nSource 4 (https://noveum.ai/en/docs/advanced/performance-optimization):\n\ud83d\udd17 Next Steps\n- Multi-Agent Tracing - Observe agent workflows\n- RAG Pipeline Observability - Monitor retrieval and generation systems\n- Custom Instrumentation - Add domain-specific tracing\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nSource 5 (https://noveum.ai/docs/advanced/performance-optimization):\n\ud83d\udd17 Next Steps\n- Multi-Agent Tracing - Observe agent workflows\n- RAG Pipeline Observability - Monitor retrieval and generation systems\n- Custom Instrumentation - Add domain-specific tracing\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nUser Question: How does Noveum handle agent tracing?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", + "prompt.prompt_length": 3269, + "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, ...", + "prompt.instruction_following": "rag_optimized", + "response.answer_length": 643, + "response.answer_completeness": "complete", + "response.response_quality": "high", + "response.source_citation": 5, + "response.context_utilization": 0.23501461988304093, + "output_response": "RAG Answer: Noveum.ai handles agent tracing through its specialized Multi-Agent Tracing capabilities, which are designed to observe complex workflows and inter-agent communications within multi-agent systems. The...", + "evaluation.retrieval_effectiveness": 1, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 5, + "evaluation.context_relevance": "high", + "evaluation.overall_quality": "high", + "evaluation.ready_for_production": true, + "rag.retrieval_strategy": "semantic_similarity", + "rag.vector_search_results": 5, + "rag.context_synthesis": "multi_source", + "rag.document_coverage": 1, + "rag.information_density": 0.23501461988304093 + }, + "events": [], + "links": [], + "trace_trace_id": "8100b869-07c0-4e37-b51e-092c9d20b4a7", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:48:57.574000000", + "trace_end_time": "2025-10-16 08:49:04.149000000", + "trace_duration_ms": 6574.500999999999, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:48:57.574000000", + "updated_at": "2025-10-16 08:48:57.574000000" + }, + { + "span_id": "5f8ea007-edef-477c-b8f1-57c2da9e9258", + "trace_id": "c28bd31c-d200-4467-91fd-a3b80187c885", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-16 08:48:17.977000000", + "end_time": "2025-10-16 08:48:56.570000000", + "duration_ms": 38592.912, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What APIs are available in Noveum platform?", + "query.length": 43, + "processing.mode": "RAG", + "processing.time_seconds": 38.59272003173828, + "processing.response_length": 270, + "processing.sources_count": 5, + "output_response": "Final Answer: The provided context does not specify the APIs available in the Noveum platform. For detailed information about the APIs, I recommend checking the official Noveum documentation or the SDK Integration ...", + "final_answer_mode": "RAG", + "query_processed.input_query": "What APIs are available in Noveum platform?", + "query_processed.output_response": "Successfully processed query using RAG, generated 270 character response", + "query_processed.mode": "RAG", + "query_processed.processing_time": 38.59272003173828, + "query_processed.response_length": 270 + }, + "events": [], + "links": [], + "trace_trace_id": "c28bd31c-d200-4467-91fd-a3b80187c885", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:48:17.977000000", + "trace_end_time": "2025-10-16 08:48:56.570000000", + "trace_duration_ms": 38592.996, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:48:17.977000000", + "updated_at": "2025-10-16 08:48:17.977000000" + }, + { + "span_id": "db385f4d-beef-465b-9831-a65080537519", + "trace_id": "c28bd31c-d200-4467-91fd-a3b80187c885", + "parent_span_id": "5f8ea007-edef-477c-b8f1-57c2da9e9258", + "name": "agent.query_routing", + "start_time": "2025-10-16 08:48:17.977000000", + "end_time": "2025-10-16 08:48:56.570000000", + "duration_ms": 38592.66499999999, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What APIs are available in Noveum platform?", + "query_length": 43, + "query_type": "routing_query", + "output_response": "Routed to RAG for query processing", + "router.classification": "RAG", + "router.confidence_score": 1, + "router.rag_keyword_score": 3, + "router.web_keyword_score": 0, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "c28bd31c-d200-4467-91fd-a3b80187c885", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:48:17.977000000", + "trace_end_time": "2025-10-16 08:48:56.570000000", + "trace_duration_ms": 38592.996, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:48:17.977000000", + "updated_at": "2025-10-16 08:48:17.977000000" + }, + { + "span_id": "3ccebe1c-1f2a-433b-a36f-453a0b803511", + "trace_id": "c28bd31c-d200-4467-91fd-a3b80187c885", + "parent_span_id": "db385f4d-beef-465b-9831-a65080537519", + "name": "agent.llm-rag", + "start_time": "2025-10-16 08:48:17.978000000", + "end_time": "2025-10-16 08:48:56.570000000", + "duration_ms": 38592.416999999994, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "rag_agent", + "agent.operation": "llm-rag", + "agent.capabilities": [ + "document_retrieval", + "context_generation", + "response_generation" + ], + "agent.id": "noveum_rag_agent", + "input_query": "What APIs are available in Noveum platform?", + "query_length": 43, + "query_type": "rag_query", + "output_response": "RAG Answer: The provided context does not specify the APIs available in the Noveum platform. For detailed information about the APIs, I recommend checking the official Noveum documentation or the SDK Integration ...", + "rag.context_length": 2284, + "rag.sources_count": 5, + "rag.answer_length": 270, + "rag.mode": "retrieval_augmented_generation" + }, + "events": [], + "links": [], + "trace_trace_id": "c28bd31c-d200-4467-91fd-a3b80187c885", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:48:17.977000000", + "trace_end_time": "2025-10-16 08:48:56.570000000", + "trace_duration_ms": 38592.996, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:48:17.977000000", + "updated_at": "2025-10-16 08:48:17.977000000" + }, + { + "span_id": "6d1bb8e9-315d-433a-8df5-7f5fb2eeee16", + "trace_id": "c28bd31c-d200-4467-91fd-a3b80187c885", + "parent_span_id": "db385f4d-beef-465b-9831-a65080537519", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-16 08:48:17.978000000", + "end_time": "2025-10-16 08:48:17.978000000", + "duration_ms": 0.043000000000000003, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What APIs are available in Noveum platform?", + "query_length": 43, + "query_type": "routing_evaluation_query", + "classification.mode": "RAG", + "classification.rag_keyword_score": 3, + "classification.web_keyword_score": 0, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "noveum_specific", + "query.keyword_density": 0.42857142857142855, + "query.domain_affinity": "noveum", + "routing.decision": "Routed to RAG based on analysis", + "routing.rationale": "RAG score: 3, Web score: 0, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "Web Search", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.057692307692307696, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: RAG (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "c28bd31c-d200-4467-91fd-a3b80187c885", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:48:17.977000000", + "trace_end_time": "2025-10-16 08:48:56.570000000", + "trace_duration_ms": 38592.996, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:48:17.977000000", + "updated_at": "2025-10-16 08:48:17.977000000" + }, + { + "span_id": "2f11df8a-8f45-425d-a0ce-1eec7887b508", + "trace_id": "c28bd31c-d200-4467-91fd-a3b80187c885", + "parent_span_id": "3ccebe1c-1f2a-433b-a36f-453a0b803511", + "name": "agent.llm_model_execution", + "start_time": "2025-10-16 08:48:18.392000000", + "end_time": "2025-10-16 08:48:55.545000000", + "duration_ms": 37152.496999999996, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What APIs are available in Noveum platform?", + "query_length": 43, + "query_type": "rag_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 37.15243363380432, + "model.latency_ms": 37152.43363380432, + "model.start_time": 1760604498.3926067, + "model.end_time": 1760604535.5450404, + "model.performance_tier": "slow", + "model.prompt_tokens": 705, + "model.completion_tokens": 67, + "model.total_tokens": 772, + "model.tokens_per_second": 20.779257897592238, + "model.estimated_cost": 0.02316, + "model.efficiency_score": 0.34974093264248707, + "model.response_length": 270, + "model.response_quality": "high", + "model.output_response": "Model Response: The provided context does not specify the APIs available in the Noveum platform. For detailed information about the APIs, I recommend checking the official Noveum documentation or the SDK Integration ...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "c28bd31c-d200-4467-91fd-a3b80187c885", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:48:17.977000000", + "trace_end_time": "2025-10-16 08:48:56.570000000", + "trace_duration_ms": 38592.996, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:48:17.977000000", + "updated_at": "2025-10-16 08:48:17.977000000" + }, + { + "span_id": "d13e608c-06e4-47c4-b602-c40ac9560be2", + "trace_id": "c28bd31c-d200-4467-91fd-a3b80187c885", + "parent_span_id": "3ccebe1c-1f2a-433b-a36f-453a0b803511", + "name": "agent.rag_evaluation_metrics", + "start_time": "2025-10-16 08:48:55.545000000", + "end_time": "2025-10-16 08:48:55.545000000", + "duration_ms": 0.031, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "rag_evaluation_metrics", + "agent.capabilities": [ + "retrieval_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What APIs are available in Noveum platform?", + "query_length": 43, + "query_type": "rag_evaluation_query", + "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/docs):\n- \ud83c\udf0d Universal: Works across frameworks, clouds, and deployment models\n\ud83e\udd1d Community & Support\n- \ud83d\udcac Discord Community: Join our Discord\n- \ud83d\udce7 Email Support: [email protected]\n- \ud83d\udc1b Bug Reports: GitHub Issues\n- \ud83d\udcd6 Knowledge Base: Help Center\nReady to get started? Head to ...", + "retrieval.context_length": 2284, + "retrieval.sources_count": 5, + "retrieval.context_quality": "high", + "retrieval.effectiveness": 1, + "retrieval.context_utilization": 2.284, + "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/docs):\n- \ud83c\udf0d Universal: Works across frameworks, clouds, and deployment models\n\ud83e\udd1d Community & Support\n- \ud83d\udcac Discord Community: Join our Discord\n- \ud83d\udce7 Email Support: [email protected]\n- \ud83d\udc1b Bug Reports: GitHub Issues\n- \ud83d\udcd6 Knowledge Base: Help Center\nReady to get started? Head to our SDK Integration Guide to begin tracing your AI applications in under 5 minutes!\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability ...\n\nSource 2 (https://noveum.ai/docs):\n- \ud83c\udf0d Universal: Works across frameworks, clouds, and deployment models\n\ud83e\udd1d Community & Support\n- \ud83d\udcac Discord Community: Join our Discord\n- \ud83d\udce7 Email Support: [email protected]\n- \ud83d\udc1b Bug Reports: GitHub Issues\n- \ud83d\udcd6 Knowledge Base: Help Center\nReady to get started? Head to our SDK Integration Guide to begin tracing your AI applications in under 5 minutes!\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability ...\n\nSource 3 (https://noveum.ai/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nSource 4 (https://noveum.ai/en/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nSource 5 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, vector searches, data retrieval, agent reasoning, and more. Without proper observability, debugging becomes a nightmare, optimization is guesswork, and understanding user interactions is nearly impossible.\nNoveum.ai solves this by providing comprehensive tracing and observability specifically designed for AI applications.\nKey benefits of using Noveum.ai:\n- Complete Visibility: Trace...\n\nUser Question: What APIs are available in Noveum platform?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", + "prompt.prompt_length": 2823, + "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/docs):\n- \ud83c\udf0d Universal: Works across frameworks, clouds, and deployment models\n\ud83e\udd1d Community & Support\n- \ud83d\udcac Discord Community: Join our Discord\n- \ud83d\udce7 Email Support: [email prot...", + "prompt.instruction_following": "rag_optimized", + "response.answer_length": 270, + "response.answer_completeness": "complete", + "response.response_quality": "high", + "response.source_citation": 5, + "response.context_utilization": 0.11821366024518389, + "output_response": "RAG Answer: The provided context does not specify the APIs available in the Noveum platform. For detailed information about the APIs, I recommend checking the official Noveum documentation or the SDK Integration ...", + "evaluation.retrieval_effectiveness": 1, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 5, + "evaluation.context_relevance": "high", + "evaluation.overall_quality": "high", + "evaluation.ready_for_production": true, + "rag.retrieval_strategy": "semantic_similarity", + "rag.vector_search_results": 5, + "rag.context_synthesis": "multi_source", + "rag.document_coverage": 1, + "rag.information_density": 0.11821366024518389 + }, + "events": [], + "links": [], + "trace_trace_id": "c28bd31c-d200-4467-91fd-a3b80187c885", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:48:17.977000000", + "trace_end_time": "2025-10-16 08:48:56.570000000", + "trace_duration_ms": 38592.996, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:48:17.977000000", + "updated_at": "2025-10-16 08:48:17.977000000" + }, + { + "span_id": "3f3babbd-abf5-4f5a-b31c-fa5dcec9c360", + "trace_id": "b90ef3fc-11ea-4230-a59b-e2f2db8f2de0", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-16 08:48:08.007000000", + "end_time": "2025-10-16 08:48:16.975000000", + "duration_ms": 8967.201, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "How do I set up observability with Noveum?", + "query.length": 42, + "processing.mode": "RAG", + "processing.time_seconds": 8.96716833114624, + "processing.response_length": 838, + "processing.sources_count": 5, + "output_response": "Final Answer: To set up observability with Noveum, follow these steps:\n\n1. **Sign Up**: Go to [noveum.ai](https://noveum.ai) and create an account.\n2. **Create a Project**: Once signed in, create a new project to o...", + "final_answer_mode": "RAG", + "query_processed.input_query": "How do I set up observability with Noveum?", + "query_processed.output_response": "Successfully processed query using RAG, generated 838 character response", + "query_processed.mode": "RAG", + "query_processed.processing_time": 8.96716833114624, + "query_processed.response_length": 838 + }, + "events": [], + "links": [], + "trace_trace_id": "b90ef3fc-11ea-4230-a59b-e2f2db8f2de0", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:48:08.007000000", + "trace_end_time": "2025-10-16 08:48:16.975000000", + "trace_duration_ms": 8967.264, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:48:08.007000000", + "updated_at": "2025-10-16 08:48:08.007000000" + }, + { + "span_id": "41f6453c-4be6-4b66-8f9b-8e795f7b41b0", + "trace_id": "b90ef3fc-11ea-4230-a59b-e2f2db8f2de0", + "parent_span_id": "b0a76345-49e5-4c1b-915f-cec90ea2b1f7", + "name": "agent.llm-rag", + "start_time": "2025-10-16 08:48:08.007000000", + "end_time": "2025-10-16 08:48:16.974000000", + "duration_ms": 8966.964, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "rag_agent", + "agent.operation": "llm-rag", + "agent.capabilities": [ + "document_retrieval", + "context_generation", + "response_generation" + ], + "agent.id": "noveum_rag_agent", + "input_query": "How do I set up observability with Noveum?", + "query_length": 42, + "query_type": "rag_query", + "output_response": "RAG Answer: To set up observability with Noveum, follow these steps:\n\n1. **Sign Up**: Go to [noveum.ai](https://noveum.ai) and create an account.\n2. **Create a Project**: Once signed in, create a new project to o...", + "rag.context_length": 2870, + "rag.sources_count": 6, + "rag.answer_length": 838, + "rag.mode": "retrieval_augmented_generation" + }, + "events": [], + "links": [], + "trace_trace_id": "b90ef3fc-11ea-4230-a59b-e2f2db8f2de0", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:48:08.007000000", + "trace_end_time": "2025-10-16 08:48:16.975000000", + "trace_duration_ms": 8967.264, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:48:08.007000000", + "updated_at": "2025-10-16 08:48:08.007000000" + }, + { + "span_id": "b0a76345-49e5-4c1b-915f-cec90ea2b1f7", + "trace_id": "b90ef3fc-11ea-4230-a59b-e2f2db8f2de0", + "parent_span_id": "3f3babbd-abf5-4f5a-b31c-fa5dcec9c360", + "name": "agent.query_routing", + "start_time": "2025-10-16 08:48:08.007000000", + "end_time": "2025-10-16 08:48:16.974000000", + "duration_ms": 8967.129, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "How do I set up observability with Noveum?", + "query_length": 42, + "query_type": "routing_query", + "output_response": "Routed to RAG for query processing", + "router.classification": "RAG", + "router.confidence_score": 1, + "router.rag_keyword_score": 2, + "router.web_keyword_score": 0, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "b90ef3fc-11ea-4230-a59b-e2f2db8f2de0", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:48:08.007000000", + "trace_end_time": "2025-10-16 08:48:16.975000000", + "trace_duration_ms": 8967.264, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:48:08.007000000", + "updated_at": "2025-10-16 08:48:08.007000000" + }, + { + "span_id": "fe3074a7-f4c0-442e-9a5b-e2cc11d93785", + "trace_id": "b90ef3fc-11ea-4230-a59b-e2f2db8f2de0", + "parent_span_id": "b0a76345-49e5-4c1b-915f-cec90ea2b1f7", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-16 08:48:08.007000000", + "end_time": "2025-10-16 08:48:08.007000000", + "duration_ms": 0.032, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: How do I set up observability with Noveum?", + "query_length": 42, + "query_type": "routing_evaluation_query", + "classification.mode": "RAG", + "classification.rag_keyword_score": 2, + "classification.web_keyword_score": 0, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "noveum_specific", + "query.keyword_density": 0.25, + "query.domain_affinity": "noveum", + "routing.decision": "Routed to RAG based on analysis", + "routing.rationale": "RAG score: 2, Web score: 0, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "Web Search", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.038461538461538464, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: RAG (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "b90ef3fc-11ea-4230-a59b-e2f2db8f2de0", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:48:08.007000000", + "trace_end_time": "2025-10-16 08:48:16.975000000", + "trace_duration_ms": 8967.264, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:48:08.007000000", + "updated_at": "2025-10-16 08:48:08.007000000" + }, + { + "span_id": "e16139ee-048e-4c2f-87f8-90442f41a0ae", + "trace_id": "b90ef3fc-11ea-4230-a59b-e2f2db8f2de0", + "parent_span_id": "41f6453c-4be6-4b66-8f9b-8e795f7b41b0", + "name": "agent.llm_model_execution", + "start_time": "2025-10-16 08:48:08.644000000", + "end_time": "2025-10-16 08:48:15.709000000", + "duration_ms": 7064.898, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: How do I set up observability with Noveum?", + "query_length": 42, + "query_type": "rag_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 7.0648510456085205, + "model.latency_ms": 7064.8510456085205, + "model.start_time": 1760604488.6448367, + "model.end_time": 1760604495.7096877, + "model.performance_tier": "slow", + "model.prompt_tokens": 852, + "model.completion_tokens": 209, + "model.total_tokens": 1061, + "model.tokens_per_second": 150.1800948315128, + "model.estimated_cost": 0.031830000000000004, + "model.efficiency_score": 0.7898209236569275, + "model.response_length": 838, + "model.response_quality": "high", + "model.output_response": "Model Response: To set up observability with Noveum, follow these steps:\n\n1. **Sign Up**: Go to [noveum.ai](https://noveum.ai) and create an account.\n2. **Create a Project**: Once signed in, create a new project to o...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "b90ef3fc-11ea-4230-a59b-e2f2db8f2de0", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:48:08.007000000", + "trace_end_time": "2025-10-16 08:48:16.975000000", + "trace_duration_ms": 8967.264, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:48:08.007000000", + "updated_at": "2025-10-16 08:48:08.007000000" + }, + { + "span_id": "59e85f8e-c19b-4794-9a69-7b5bc90208f0", + "trace_id": "b90ef3fc-11ea-4230-a59b-e2f2db8f2de0", + "parent_span_id": "41f6453c-4be6-4b66-8f9b-8e795f7b41b0", + "name": "agent.rag_evaluation_metrics", + "start_time": "2025-10-16 08:48:15.709000000", + "end_time": "2025-10-16 08:48:15.709000000", + "duration_ms": 0.039, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "rag_evaluation_metrics", + "agent.capabilities": [ + "retrieval_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: How do I set up observability with Noveum?", + "query_length": 42, + "query_type": "rag_evaluation_query", + "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\n- Production Ready: Built for scale with intelligent sampling and batching\n- Privacy Focused: You control what data is captured and how it's stored\n- Open Source SDKs: Transparent, auditable, and extensible\nGetting ...", + "retrieval.context_length": 2870, + "retrieval.sources_count": 6, + "retrieval.context_quality": "high", + "retrieval.effectiveness": 1.2, + "retrieval.context_utilization": 2.87, + "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\n- Production Ready: Built for scale with intelligent sampling and batching\n- Privacy Focused: You control what data is captured and how it's stored\n- Open Source SDKs: Transparent, auditable, and extensible\nGetting Started\nReady to add comprehensive observability to your AI applications?\n- Sign up at noveum.ai\n- Create a project and get your API key\n- Install the SDK for your preferred language\n- Add tracing to your AI workflows\n- Explore insights in the Noveum.ai dashboard\nCheck out our integra...\n\nSource 2 (https://noveum.ai/en/docs/getting-started/tracing-concepts):\n- Explore Framework Integrations - Framework-specific guidance\n- Learn Advanced Patterns - Custom instrumentation techniques\n- Master the Dashboard - Analyze your traces effectively\nRemember: Good observability is not about collecting all possible data, but about collecting the right data that helps you understand, debug, and optimize your AI applications.\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to O...\n\nSource 3 (https://noveum.ai/docs/getting-started/tracing-concepts):\n- Explore Framework Integrations - Framework-specific guidance\n- Learn Advanced Patterns - Custom instrumentation techniques\n- Master the Dashboard - Analyze your traces effectively\nRemember: Good observability is not about collecting all possible data, but about collecting the right data that helps you understand, debug, and optimize your AI applications.\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to O...\n\nSource 4 (https://noveum.ai/en/docs/getting-started/overview):\nNoveum.ai Overview\nComprehensive AI tracing and observability platform for LLM applications, RAG systems, and AI agents\nWelcome to Noveum.ai\u2014the comprehensive tracing and observability platform built specifically for AI applications. Whether you're building LLM-powered chatbots, RAG systems, multi-agent workflows, or any AI-driven application, Noveum provides the insights you need to understand, debug, and optimize your systems.\n\ud83c\udfaf Why AI Applications Need Specialized Observability\nTraditional mo...\n\nSource 5 (https://noveum.ai/docs/getting-started/overview):\nNoveum.ai Overview\nComprehensive AI tracing and observability platform for LLM applications, RAG systems, and AI agents\nWelcome to Noveum.ai\u2014the comprehensive tracing and observability platform built specifically for AI applications. Whether you're building LLM-powered chatbots, RAG systems, multi-agent workflows, or any AI-driven application, Noveum provides the insights you need to understand, debug, and optimize your systems.\n\ud83c\udfaf Why AI Applications Need Specialized Observability\nTraditional mo...\n\nUser Question: How do I set up observability with Noveum?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", + "prompt.prompt_length": 3408, + "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\n- Production Ready: Built for scale with intelligent sampling and batching\n- Privacy Focused: You control what data...", + "prompt.instruction_following": "rag_optimized", + "response.answer_length": 838, + "response.answer_completeness": "complete", + "response.response_quality": "high", + "response.source_citation": 6, + "response.context_utilization": 0.29198606271777006, + "output_response": "RAG Answer: To set up observability with Noveum, follow these steps:\n\n1. **Sign Up**: Go to [noveum.ai](https://noveum.ai) and create an account.\n2. **Create a Project**: Once signed in, create a new project to o...", + "evaluation.retrieval_effectiveness": 1.2, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 6, + "evaluation.context_relevance": "high", + "evaluation.overall_quality": "high", + "evaluation.ready_for_production": true, + "rag.retrieval_strategy": "semantic_similarity", + "rag.vector_search_results": 6, + "rag.context_synthesis": "multi_source", + "rag.document_coverage": 1.2, + "rag.information_density": 0.29198606271777006 + }, + "events": [], + "links": [], + "trace_trace_id": "b90ef3fc-11ea-4230-a59b-e2f2db8f2de0", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:48:08.007000000", + "trace_end_time": "2025-10-16 08:48:16.975000000", + "trace_duration_ms": 8967.264, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:48:08.007000000", + "updated_at": "2025-10-16 08:48:08.007000000" + }, + { + "span_id": "f049ed2b-ff2f-4092-9974-acc66f81bd8d", + "trace_id": "b8abebac-a7dc-494c-a1eb-6470181f32d2", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-16 08:47:57.667000000", + "end_time": "2025-10-16 08:48:07.005000000", + "duration_ms": 9337.298, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What features does Noveum Trace offer?", + "query.length": 38, + "processing.mode": "RAG", + "processing.time_seconds": 9.33726453781128, + "processing.response_length": 1018, + "processing.sources_count": 5, + "output_response": "Final Answer: Noveum Trace offers several key features through its Python SDK (`noveum-trace`) and TypeScript SDK (`@noveum/trace`):\n\n1. **Decorator-Based Tracing**: It provides a decorator-based approach for traci...", + "final_answer_mode": "RAG", + "query_processed.input_query": "What features does Noveum Trace offer?", + "query_processed.output_response": "Successfully processed query using RAG, generated 1018 character response", + "query_processed.mode": "RAG", + "query_processed.processing_time": 9.33726453781128, + "query_processed.response_length": 1018 + }, + "events": [], + "links": [], + "trace_trace_id": "b8abebac-a7dc-494c-a1eb-6470181f32d2", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:47:57.667000000", + "trace_end_time": "2025-10-16 08:48:07.005000000", + "trace_duration_ms": 9337.405, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:47:57.667000000", + "updated_at": "2025-10-16 08:47:57.667000000" + }, + { + "span_id": "211f1fe7-4cfb-4c1c-9c72-62ae2825dda3", + "trace_id": "b8abebac-a7dc-494c-a1eb-6470181f32d2", + "parent_span_id": "f049ed2b-ff2f-4092-9974-acc66f81bd8d", + "name": "agent.query_routing", + "start_time": "2025-10-16 08:47:57.668000000", + "end_time": "2025-10-16 08:48:07.005000000", + "duration_ms": 9337.223, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What features does Noveum Trace offer?", + "query_length": 38, + "query_type": "routing_query", + "output_response": "Routed to RAG for query processing", + "router.classification": "RAG", + "router.confidence_score": 1, + "router.rag_keyword_score": 3, + "router.web_keyword_score": 0, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "b8abebac-a7dc-494c-a1eb-6470181f32d2", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:47:57.667000000", + "trace_end_time": "2025-10-16 08:48:07.005000000", + "trace_duration_ms": 9337.405, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:47:57.667000000", + "updated_at": "2025-10-16 08:47:57.667000000" + }, + { + "span_id": "976c0c4b-efc1-4b7d-bfad-350c7fd0f71e", + "trace_id": "b8abebac-a7dc-494c-a1eb-6470181f32d2", + "parent_span_id": "211f1fe7-4cfb-4c1c-9c72-62ae2825dda3", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-16 08:47:57.668000000", + "end_time": "2025-10-16 08:47:57.668000000", + "duration_ms": 0.028, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What features does Noveum Trace offer?", + "query_length": 38, + "query_type": "routing_evaluation_query", + "classification.mode": "RAG", + "classification.rag_keyword_score": 3, + "classification.web_keyword_score": 0, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "noveum_specific", + "query.keyword_density": 0.5, + "query.domain_affinity": "noveum", + "routing.decision": "Routed to RAG based on analysis", + "routing.rationale": "RAG score: 3, Web score: 0, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "Web Search", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.057692307692307696, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: RAG (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "b8abebac-a7dc-494c-a1eb-6470181f32d2", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:47:57.667000000", + "trace_end_time": "2025-10-16 08:48:07.005000000", + "trace_duration_ms": 9337.405, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:47:57.667000000", + "updated_at": "2025-10-16 08:47:57.667000000" + }, + { + "span_id": "d2afaf74-f63b-4141-9fa3-662162f51cc0", + "trace_id": "b8abebac-a7dc-494c-a1eb-6470181f32d2", + "parent_span_id": "211f1fe7-4cfb-4c1c-9c72-62ae2825dda3", + "name": "agent.llm-rag", + "start_time": "2025-10-16 08:47:57.668000000", + "end_time": "2025-10-16 08:48:07.005000000", + "duration_ms": 9337.056, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "rag_agent", + "agent.operation": "llm-rag", + "agent.capabilities": [ + "document_retrieval", + "context_generation", + "response_generation" + ], + "agent.id": "noveum_rag_agent", + "input_query": "What features does Noveum Trace offer?", + "query_length": 38, + "query_type": "rag_query", + "output_response": "RAG Answer: Noveum Trace offers several key features through its Python SDK (`noveum-trace`) and TypeScript SDK (`@noveum/trace`):\n\n1. **Decorator-Based Tracing**: It provides a decorator-based approach for traci...", + "rag.context_length": 2787, + "rag.sources_count": 5, + "rag.answer_length": 1018, + "rag.mode": "retrieval_augmented_generation" + }, + "events": [], + "links": [], + "trace_trace_id": "b8abebac-a7dc-494c-a1eb-6470181f32d2", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:47:57.667000000", + "trace_end_time": "2025-10-16 08:48:07.005000000", + "trace_duration_ms": 9337.405, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:47:57.667000000", + "updated_at": "2025-10-16 08:47:57.667000000" + }, + { + "span_id": "09811a73-405d-4c0d-ba5d-0472b3708062", + "trace_id": "b8abebac-a7dc-494c-a1eb-6470181f32d2", + "parent_span_id": "d2afaf74-f63b-4141-9fa3-662162f51cc0", + "name": "agent.llm_model_execution", + "start_time": "2025-10-16 08:47:58.659000000", + "end_time": "2025-10-16 08:48:05.878000000", + "duration_ms": 7218.618, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What features does Noveum Trace offer?", + "query_length": 38, + "query_type": "rag_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 7.218573331832886, + "model.latency_ms": 7218.573331832886, + "model.start_time": 1760604478.659586, + "model.end_time": 1760604485.8781593, + "model.performance_tier": "slow", + "model.prompt_tokens": 830, + "model.completion_tokens": 254, + "model.total_tokens": 1084, + "model.tokens_per_second": 150.16817730724068, + "model.estimated_cost": 0.03252, + "model.efficiency_score": 0.9391143911439115, + "model.response_length": 1018, + "model.response_quality": "high", + "model.output_response": "Model Response: Noveum Trace offers several key features through its Python SDK (`noveum-trace`) and TypeScript SDK (`@noveum/trace`):\n\n1. **Decorator-Based Tracing**: It provides a decorator-based approach for traci...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "b8abebac-a7dc-494c-a1eb-6470181f32d2", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:47:57.667000000", + "trace_end_time": "2025-10-16 08:48:07.005000000", + "trace_duration_ms": 9337.405, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:47:57.667000000", + "updated_at": "2025-10-16 08:47:57.667000000" + }, + { + "span_id": "015688fe-d287-48e8-ac59-63781ffd6f89", + "trace_id": "b8abebac-a7dc-494c-a1eb-6470181f32d2", + "parent_span_id": "d2afaf74-f63b-4141-9fa3-662162f51cc0", + "name": "agent.rag_evaluation_metrics", + "start_time": "2025-10-16 08:48:05.878000000", + "end_time": "2025-10-16 08:48:05.878000000", + "duration_ms": 0.024, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "rag_evaluation_metrics", + "agent.capabilities": [ + "retrieval_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What features does Noveum Trace offer?", + "query_length": 38, + "query_type": "rag_evaluation_query", + "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\n\ud83d\udcd8 TypeScript SDK (\n@noveum/trace\n)- Framework ...", + "retrieval.context_length": 2787, + "retrieval.sources_count": 5, + "retrieval.context_quality": "high", + "retrieval.effectiveness": 1, + "retrieval.context_utilization": 2.787, + "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\n\ud83d\udcd8 TypeScript SDK (\n@noveum/trace\n)- Framework integrations for Next.js, Express.js, and Hono\n- TypeScript-first design with full type safety\n- Universal compatibility (Node.js, Edge Runtime, browsers)\n-\n\ud83d\udcca Noveum Platform\n- Real-time dashboard for analyzing traces and performance\n- Ad...\n\nSource 2 (https://noveum.ai/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\n\ud83d\udcd8 TypeScript SDK (\n@noveum/trace\n)- Framework integrations for Next.js, Express.js, and Hono\n- TypeScript-first design with full type safety\n- Universal compatibility (Node.js, Edge Runtime, browsers)\n-\n\ud83d\udcca Noveum Platform\n- Real-time dashboard for analyzing traces and performance\n- Ad...\n\nSource 3 (https://noveum.ai/en/docs/getting-started/sdk-integration):\n- \ud83d\udc65 Collaborate with your team on insights\n\ud83d\udd12 Security & Privacy\n- \ud83d\udd10 Encryption: All data encrypted in transit and at rest\n- \ud83c\udf9b\ufe0f Configurable Capture: Control what data is collected\n- \ud83c\udfe0 Data Residency: Choose your data storage region\n- \u23f0 Retention Control: Set custom data retention policies\nNext Steps\n- Tracing Concepts - Learn about traces, spans, and observability best practices\n- Framework Integrations - Deep dive into specific framework setups\n- Multi-Agent Tracing - Observe complex agent work...\n\nSource 4 (https://noveum.ai/docs/getting-started/sdk-integration):\n- \ud83d\udc65 Collaborate with your team on insights\n\ud83d\udd12 Security & Privacy\n- \ud83d\udd10 Encryption: All data encrypted in transit and at rest\n- \ud83c\udf9b\ufe0f Configurable Capture: Control what data is collected\n- \ud83c\udfe0 Data Residency: Choose your data storage region\n- \u23f0 Retention Control: Set custom data retention policies\nNext Steps\n- Tracing Concepts - Learn about traces, spans, and observability best practices\n- Framework Integrations - Deep dive into specific framework setups\n- Multi-Agent Tracing - Observe complex agent work...\n\nSource 5 (https://noveum.ai/docs/platform/dashboard):\n- Bulk Export: Download multiple traces simultaneously\n- Comparative Analysis: Compare performance across multiple traces\nReal-Time Updates\n- Live Refresh: Automatic updates as new traces arrive\n- Connection Monitoring: Real-time status of your trace ingestion pipeline\n- Performance Indicators: Live metrics for system health monitoring\nReady to dive deeper? Explore Projects & Environments to organize your AI applications, or check out Team Collaboration to share insights with your team.\nGet Earl...\n\nUser Question: What features does Noveum Trace offer?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", + "prompt.prompt_length": 3321, + "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frame...", + "prompt.instruction_following": "rag_optimized", + "response.answer_length": 1018, + "response.answer_completeness": "complete", + "response.response_quality": "high", + "response.source_citation": 5, + "response.context_utilization": 0.36526731252242556, + "output_response": "RAG Answer: Noveum Trace offers several key features through its Python SDK (`noveum-trace`) and TypeScript SDK (`@noveum/trace`):\n\n1. **Decorator-Based Tracing**: It provides a decorator-based approach for traci...", + "evaluation.retrieval_effectiveness": 1, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 5, + "evaluation.context_relevance": "high", + "evaluation.overall_quality": "high", + "evaluation.ready_for_production": true, + "rag.retrieval_strategy": "semantic_similarity", + "rag.vector_search_results": 5, + "rag.context_synthesis": "multi_source", + "rag.document_coverage": 1, + "rag.information_density": 0.36526731252242556 + }, + "events": [], + "links": [], + "trace_trace_id": "b8abebac-a7dc-494c-a1eb-6470181f32d2", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:47:57.667000000", + "trace_end_time": "2025-10-16 08:48:07.005000000", + "trace_duration_ms": 9337.405, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:47:57.667000000", + "updated_at": "2025-10-16 08:47:57.667000000" + }, + { + "span_id": "46ee4d00-f914-41d8-9f91-c2db85f1c613", + "trace_id": "be9ccbaf-ccb4-4ebd-a380-4247f4d01868", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-16 08:47:52.037000000", + "end_time": "2025-10-16 08:47:56.662000000", + "duration_ms": 4625.395, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What are Noveum's pricing plans?", + "query.length": 32, + "processing.mode": "RAG", + "processing.time_seconds": 4.6253437995910645, + "processing.response_length": 471, + "processing.sources_count": 5, + "output_response": "Final Answer: The provided context does not contain specific information about Noveum's pricing plans. It mentions that all users will have access to the Observability suite for free, and early users will receive f...", + "final_answer_mode": "RAG", + "query_processed.input_query": "What are Noveum's pricing plans?", + "query_processed.output_response": "Successfully processed query using RAG, generated 471 character response", + "query_processed.mode": "RAG", + "query_processed.processing_time": 4.6253437995910645, + "query_processed.response_length": 471 + }, + "events": [], + "links": [], + "trace_trace_id": "be9ccbaf-ccb4-4ebd-a380-4247f4d01868", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:47:52.037000000", + "trace_end_time": "2025-10-16 08:47:56.662000000", + "trace_duration_ms": 4625.492, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:47:52.037000000", + "updated_at": "2025-10-16 08:47:52.037000000" + }, + { + "span_id": "6d15369a-ae35-45c7-84dd-bdf484f73f2d", + "trace_id": "be9ccbaf-ccb4-4ebd-a380-4247f4d01868", + "parent_span_id": "46ee4d00-f914-41d8-9f91-c2db85f1c613", + "name": "agent.query_routing", + "start_time": "2025-10-16 08:47:52.037000000", + "end_time": "2025-10-16 08:47:56.662000000", + "duration_ms": 4625.286, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What are Noveum's pricing plans?", + "query_length": 32, + "query_type": "routing_query", + "output_response": "Routed to RAG for query processing", + "router.classification": "RAG", + "router.confidence_score": 1, + "router.rag_keyword_score": 3, + "router.web_keyword_score": 0, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "be9ccbaf-ccb4-4ebd-a380-4247f4d01868", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:47:52.037000000", + "trace_end_time": "2025-10-16 08:47:56.662000000", + "trace_duration_ms": 4625.492, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:47:52.037000000", + "updated_at": "2025-10-16 08:47:52.037000000" + }, + { + "span_id": "bf201676-a3a9-433a-a8c6-d394b7aead3a", + "trace_id": "be9ccbaf-ccb4-4ebd-a380-4247f4d01868", + "parent_span_id": "6d15369a-ae35-45c7-84dd-bdf484f73f2d", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-16 08:47:52.037000000", + "end_time": "2025-10-16 08:47:52.037000000", + "duration_ms": 0.048, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What are Noveum's pricing plans?", + "query_length": 32, + "query_type": "routing_evaluation_query", + "classification.mode": "RAG", + "classification.rag_keyword_score": 3, + "classification.web_keyword_score": 0, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "noveum_specific", + "query.keyword_density": 0.6, + "query.domain_affinity": "noveum", + "routing.decision": "Routed to RAG based on analysis", + "routing.rationale": "RAG score: 3, Web score: 0, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "Web Search", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.057692307692307696, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: RAG (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "be9ccbaf-ccb4-4ebd-a380-4247f4d01868", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:47:52.037000000", + "trace_end_time": "2025-10-16 08:47:56.662000000", + "trace_duration_ms": 4625.492, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:47:52.037000000", + "updated_at": "2025-10-16 08:47:52.037000000" + }, + { + "span_id": "f8346f60-131b-4007-ba3e-7b0ff10cdfde", + "trace_id": "be9ccbaf-ccb4-4ebd-a380-4247f4d01868", + "parent_span_id": "6d15369a-ae35-45c7-84dd-bdf484f73f2d", + "name": "agent.llm-rag", + "start_time": "2025-10-16 08:47:52.037000000", + "end_time": "2025-10-16 08:47:56.662000000", + "duration_ms": 4624.998, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "rag_agent", + "agent.operation": "llm-rag", + "agent.capabilities": [ + "document_retrieval", + "context_generation", + "response_generation" + ], + "agent.id": "noveum_rag_agent", + "input_query": "What are Noveum's pricing plans?", + "query_length": 32, + "query_type": "rag_query", + "output_response": "RAG Answer: The provided context does not contain specific information about Noveum's pricing plans. It mentions that all users will have access to the Observability suite for free, and early users will receive f...", + "rag.context_length": 2105, + "rag.sources_count": 5, + "rag.answer_length": 471, + "rag.mode": "retrieval_augmented_generation" + }, + "events": [], + "links": [], + "trace_trace_id": "be9ccbaf-ccb4-4ebd-a380-4247f4d01868", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:47:52.037000000", + "trace_end_time": "2025-10-16 08:47:56.662000000", + "trace_duration_ms": 4625.492, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:47:52.037000000", + "updated_at": "2025-10-16 08:47:52.037000000" + }, + { + "span_id": "0186c28c-7596-4279-a4eb-7733cd665609", + "trace_id": "be9ccbaf-ccb4-4ebd-a380-4247f4d01868", + "parent_span_id": "f8346f60-131b-4007-ba3e-7b0ff10cdfde", + "name": "agent.llm_model_execution", + "start_time": "2025-10-16 08:47:52.579000000", + "end_time": "2025-10-16 08:47:55.842000000", + "duration_ms": 3262.5829999999996, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What are Noveum's pricing plans?", + "query_length": 32, + "query_type": "rag_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 3.2625203132629395, + "model.latency_ms": 3262.5203132629395, + "model.start_time": 1760604472.580008, + "model.end_time": 1760604475.8425283, + "model.performance_tier": "medium", + "model.prompt_tokens": 658, + "model.completion_tokens": 117, + "model.total_tokens": 775, + "model.tokens_per_second": 237.54641368804243, + "model.estimated_cost": 0.02325, + "model.efficiency_score": 0.607741935483871, + "model.response_length": 471, + "model.response_quality": "high", + "model.output_response": "Model Response: The provided context does not contain specific information about Noveum's pricing plans. It mentions that all users will have access to the Observability suite for free, and early users will receive f...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "be9ccbaf-ccb4-4ebd-a380-4247f4d01868", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:47:52.037000000", + "trace_end_time": "2025-10-16 08:47:56.662000000", + "trace_duration_ms": 4625.492, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:47:52.037000000", + "updated_at": "2025-10-16 08:47:52.037000000" + }, + { + "span_id": "3bdf8d5b-c10d-44f3-8514-8facc7f63db8", + "trace_id": "be9ccbaf-ccb4-4ebd-a380-4247f4d01868", + "parent_span_id": "f8346f60-131b-4007-ba3e-7b0ff10cdfde", + "name": "agent.rag_evaluation_metrics", + "start_time": "2025-10-16 08:47:55.842000000", + "end_time": "2025-10-16 08:47:55.842000000", + "duration_ms": 0.032, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "rag_evaluation_metrics", + "agent.capabilities": [ + "retrieval_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What are Noveum's pricing plans?", + "query_length": 32, + "query_type": "rag_evaluation_query", + "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....", + "retrieval.context_length": 2105, + "retrieval.sources_count": 5, + "retrieval.context_quality": "high", + "retrieval.effectiveness": 1, + "retrieval.context_utilization": 2.105, + "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nSource 2 (https://noveum.ai/en/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nSource 3 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nThe AI landscape is complex and moving fast. With Noveum.ai, you'll have the insights you need to build, debug, and optimize AI applications with confidence.\nReady to see what's really happening inside your AI applications? Try Noveum.ai today and transform how you build and monitor AI systems.\nLet's build more reliable, observable AI\u2014together.\nGet Early Access to Noveum.ai Platform\nJoin the select group of AI teams optimizing their models with our data-driven platform. We're onboarding users in...\n\nSource 4 (https://noveum.ai/en/blog/from-logs-to-intelligent-choices-inside-noveum-ais-evaluation-process):\nGet Early Access to Noveum.ai Platform\nJoin the select group of AI teams optimizing their models with our data-driven platform. We're onboarding users in limited batches to ensure a premium experience....\n\nSource 5 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, vector searches, data retrieval, agent reasoning, and more. Without proper observability, debugging becomes a nightmare, optimization is guesswork, and understanding user interactions is nearly impossible.\nNoveum.ai solves this by providing comprehensive tracing and observability specifically designed for AI applications.\nKey benefits of using Noveum.ai:\n- Complete Visibility: Trace...\n\nUser Question: What are Noveum's pricing plans?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", + "prompt.prompt_length": 2633, + "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to ...", + "prompt.instruction_following": "rag_optimized", + "response.answer_length": 471, + "response.answer_completeness": "complete", + "response.response_quality": "high", + "response.source_citation": 5, + "response.context_utilization": 0.22375296912114015, + "output_response": "RAG Answer: The provided context does not contain specific information about Noveum's pricing plans. It mentions that all users will have access to the Observability suite for free, and early users will receive f...", + "evaluation.retrieval_effectiveness": 1, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 5, + "evaluation.context_relevance": "high", + "evaluation.overall_quality": "high", + "evaluation.ready_for_production": true, + "rag.retrieval_strategy": "semantic_similarity", + "rag.vector_search_results": 5, + "rag.context_synthesis": "multi_source", + "rag.document_coverage": 1, + "rag.information_density": 0.22375296912114015 + }, + "events": [], + "links": [], + "trace_trace_id": "be9ccbaf-ccb4-4ebd-a380-4247f4d01868", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:47:52.037000000", + "trace_end_time": "2025-10-16 08:47:56.662000000", + "trace_duration_ms": 4625.492, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:47:52.037000000", + "updated_at": "2025-10-16 08:47:52.037000000" + }, + { + "span_id": "770ce93d-07ae-4b9f-ae15-c6e450634b9d", + "trace_id": "8d33a529-d9b1-4208-b528-762222d5ef8c", + "parent_span_id": "9977d5d6-a7ff-469e-bec7-5e4dfc38c149", + "name": "agent.llm-rag", + "start_time": "2025-10-16 08:47:42.103000000", + "end_time": "2025-10-16 08:47:51.032000000", + "duration_ms": 8928.644999999999, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "rag_agent", + "agent.operation": "llm-rag", + "agent.capabilities": [ + "document_retrieval", + "context_generation", + "response_generation" + ], + "agent.id": "noveum_rag_agent", + "input_query": "How do I integrate Noveum Trace in my application?", + "query_length": 50, + "query_type": "rag_query", + "output_response": "RAG Answer: To integrate Noveum Trace into your application, you can use either the Python SDK (`noveum-trace`) or the TypeScript SDK (`@noveum/trace`). Here\u2019s a quick start guide based on the documentation:\n\n1. ...", + "rag.context_length": 2803, + "rag.sources_count": 5, + "rag.answer_length": 1134, + "rag.mode": "retrieval_augmented_generation" + }, + "events": [], + "links": [], + "trace_trace_id": "8d33a529-d9b1-4208-b528-762222d5ef8c", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:47:42.103000000", + "trace_end_time": "2025-10-16 08:47:51.032000000", + "trace_duration_ms": 8928.948, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:47:42.103000000", + "updated_at": "2025-10-16 08:47:42.103000000" + }, + { + "span_id": "9977d5d6-a7ff-469e-bec7-5e4dfc38c149", + "trace_id": "8d33a529-d9b1-4208-b528-762222d5ef8c", + "parent_span_id": "f3a8fa53-5641-4a23-b4d3-762577a89151", + "name": "agent.query_routing", + "start_time": "2025-10-16 08:47:42.103000000", + "end_time": "2025-10-16 08:47:51.032000000", + "duration_ms": 8928.793000000001, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "How do I integrate Noveum Trace in my application?", + "query_length": 50, + "query_type": "routing_query", + "output_response": "Routed to RAG for query processing", + "router.classification": "RAG", + "router.confidence_score": 1, + "router.rag_keyword_score": 2, + "router.web_keyword_score": 0, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "8d33a529-d9b1-4208-b528-762222d5ef8c", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:47:42.103000000", + "trace_end_time": "2025-10-16 08:47:51.032000000", + "trace_duration_ms": 8928.948, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:47:42.103000000", + "updated_at": "2025-10-16 08:47:42.103000000" + }, + { + "span_id": "c678e222-ef74-48ee-8628-16461ab0f48f", + "trace_id": "8d33a529-d9b1-4208-b528-762222d5ef8c", + "parent_span_id": "9977d5d6-a7ff-469e-bec7-5e4dfc38c149", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-16 08:47:42.103000000", + "end_time": "2025-10-16 08:47:42.103000000", + "duration_ms": 0.027, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: How do I integrate Noveum Trace in my application?", + "query_length": 50, + "query_type": "routing_evaluation_query", + "classification.mode": "RAG", + "classification.rag_keyword_score": 2, + "classification.web_keyword_score": 0, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "noveum_specific", + "query.keyword_density": 0.2222222222222222, + "query.domain_affinity": "noveum", + "routing.decision": "Routed to RAG based on analysis", + "routing.rationale": "RAG score: 2, Web score: 0, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "Web Search", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.038461538461538464, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: RAG (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "8d33a529-d9b1-4208-b528-762222d5ef8c", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:47:42.103000000", + "trace_end_time": "2025-10-16 08:47:51.032000000", + "trace_duration_ms": 8928.948, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:47:42.103000000", + "updated_at": "2025-10-16 08:47:42.103000000" + }, + { + "span_id": "f3a8fa53-5641-4a23-b4d3-762577a89151", + "trace_id": "8d33a529-d9b1-4208-b528-762222d5ef8c", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-16 08:47:42.103000000", + "end_time": "2025-10-16 08:47:51.032000000", + "duration_ms": 8928.868, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "How do I integrate Noveum Trace in my application?", + "query.length": 50, + "processing.mode": "RAG", + "processing.time_seconds": 8.928829431533813, + "processing.response_length": 1134, + "processing.sources_count": 5, + "output_response": "Final Answer: To integrate Noveum Trace into your application, you can use either the Python SDK (`noveum-trace`) or the TypeScript SDK (`@noveum/trace`). Here\u2019s a quick start guide based on the documentation:\n\n1. ...", + "final_answer_mode": "RAG", + "query_processed.input_query": "How do I integrate Noveum Trace in my application?", + "query_processed.output_response": "Successfully processed query using RAG, generated 1134 character response", + "query_processed.mode": "RAG", + "query_processed.processing_time": 8.928829431533813, + "query_processed.response_length": 1134 + }, + "events": [], + "links": [], + "trace_trace_id": "8d33a529-d9b1-4208-b528-762222d5ef8c", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:47:42.103000000", + "trace_end_time": "2025-10-16 08:47:51.032000000", + "trace_duration_ms": 8928.948, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:47:42.103000000", + "updated_at": "2025-10-16 08:47:42.103000000" + }, + { + "span_id": "f459b71f-1d43-41b0-b9bf-d94a908b7afa", + "trace_id": "8d33a529-d9b1-4208-b528-762222d5ef8c", + "parent_span_id": "770ce93d-07ae-4b9f-ae15-c6e450634b9d", + "name": "agent.llm_model_execution", + "start_time": "2025-10-16 08:47:43.044000000", + "end_time": "2025-10-16 08:47:50.008000000", + "duration_ms": 6963.959, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: How do I integrate Noveum Trace in my application?", + "query_length": 50, + "query_type": "rag_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 6.9639081954956055, + "model.latency_ms": 6963.9081954956055, + "model.start_time": 1760604463.044064, + "model.end_time": 1760604470.0079722, + "model.performance_tier": "slow", + "model.prompt_tokens": 837, + "model.completion_tokens": 283, + "model.total_tokens": 1120, + "model.tokens_per_second": 160.8292310235276, + "model.estimated_cost": 0.0336, + "model.efficiency_score": 1.0125, + "model.response_length": 1134, + "model.response_quality": "high", + "model.output_response": "Model Response: To integrate Noveum Trace into your application, you can use either the Python SDK (`noveum-trace`) or the TypeScript SDK (`@noveum/trace`). Here\u2019s a quick start guide based on the documentation:\n\n1. ...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "8d33a529-d9b1-4208-b528-762222d5ef8c", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:47:42.103000000", + "trace_end_time": "2025-10-16 08:47:51.032000000", + "trace_duration_ms": 8928.948, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:47:42.103000000", + "updated_at": "2025-10-16 08:47:42.103000000" + }, + { + "span_id": "b8627171-b081-4b08-b900-c24375b73c3f", + "trace_id": "8d33a529-d9b1-4208-b528-762222d5ef8c", + "parent_span_id": "770ce93d-07ae-4b9f-ae15-c6e450634b9d", + "name": "agent.rag_evaluation_metrics", + "start_time": "2025-10-16 08:47:50.008000000", + "end_time": "2025-10-16 08:47:50.008000000", + "duration_ms": 0.041, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "rag_evaluation_metrics", + "agent.capabilities": [ + "retrieval_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: How do I integrate Noveum Trace in my application?", + "query_length": 50, + "query_type": "rag_evaluation_query", + "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\n\ud83d\udcd8 TypeScript SDK (\n@noveum/trace\n)- Framework ...", + "retrieval.context_length": 2803, + "retrieval.sources_count": 5, + "retrieval.context_quality": "high", + "retrieval.effectiveness": 1, + "retrieval.context_utilization": 2.803, + "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\n\ud83d\udcd8 TypeScript SDK (\n@noveum/trace\n)- Framework integrations for Next.js, Express.js, and Hono\n- TypeScript-first design with full type safety\n- Universal compatibility (Node.js, Edge Runtime, browsers)\n-\n\ud83d\udcca Noveum Platform\n- Real-time dashboard for analyzing traces and performance\n- Ad...\n\nSource 2 (https://noveum.ai/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\n\ud83d\udcd8 TypeScript SDK (\n@noveum/trace\n)- Framework integrations for Next.js, Express.js, and Hono\n- TypeScript-first design with full type safety\n- Universal compatibility (Node.js, Edge Runtime, browsers)\n-\n\ud83d\udcca Noveum Platform\n- Real-time dashboard for analyzing traces and performance\n- Ad...\n\nSource 3 (https://noveum.ai/docs/getting-started/sdk-integration):\nSDK Integration Guide\nIntegrate Noveum.ai tracing into your AI applications with Python or TypeScript SDKs\nThe Noveum.ai SDKs provide comprehensive tracing and observability for your AI applications with minimal code changes. Whether you're building LLM applications, RAG systems, or multi-agent workflows, our SDKs automatically capture essential metrics and traces.\n\ud83d\ude80 Quick Start\n1. Create Your Account & Get API Key\n- Sign up at noveum.ai\n- Create a project in your dashboard\n- Generate an API key...\n\nSource 4 (https://noveum.ai/en/docs/getting-started/sdk-integration):\nSDK Integration Guide\nIntegrate Noveum.ai tracing into your AI applications with Python or TypeScript SDKs\nThe Noveum.ai SDKs provide comprehensive tracing and observability for your AI applications with minimal code changes. Whether you're building LLM applications, RAG systems, or multi-agent workflows, our SDKs automatically capture essential metrics and traces.\n\ud83d\ude80 Quick Start\n1. Create Your Account & Get API Key\n- Sign up at noveum.ai\n- Create a project in your dashboard\n- Generate an API key...\n\nSource 5 (https://noveum.ai/en/docs/getting-started/sdk-integration):\n\ud83d\udd27 Framework Integrations\nNext.js Integration\nExpress.js Integration\nFastAPI Integration (Python)\n\ud83d\udcca Advanced Features\nCustom Attributes & Events\nSampling Configuration\n\ud83d\udd17 What's Captured Automatically\n- \ud83d\udcca Performance Metrics: Latency, throughput, error rates\n- \ud83d\udcb0 Cost Tracking: Token usage, API costs across providers\n- \ud83d\udd0d Request/Response: Configurable capture of inputs/outputs\n- \ud83c\udff7\ufe0f Metadata: Model names, parameters, user context\n- \ud83c\udf0a Context Flow: Trace relationships across services\n- \ud83d\udc1b Error Detail...\n\nUser Question: How do I integrate Noveum Trace in my application?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", + "prompt.prompt_length": 3349, + "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frame...", + "prompt.instruction_following": "rag_optimized", + "response.answer_length": 1134, + "response.answer_completeness": "complete", + "response.response_quality": "high", + "response.source_citation": 5, + "response.context_utilization": 0.4045665358544417, + "output_response": "RAG Answer: To integrate Noveum Trace into your application, you can use either the Python SDK (`noveum-trace`) or the TypeScript SDK (`@noveum/trace`). Here\u2019s a quick start guide based on the documentation:\n\n1. ...", + "evaluation.retrieval_effectiveness": 1, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 5, + "evaluation.context_relevance": "high", + "evaluation.overall_quality": "high", + "evaluation.ready_for_production": true, + "rag.retrieval_strategy": "semantic_similarity", + "rag.vector_search_results": 5, + "rag.context_synthesis": "multi_source", + "rag.document_coverage": 1, + "rag.information_density": 0.4045665358544417 + }, + "events": [], + "links": [], + "trace_trace_id": "8d33a529-d9b1-4208-b528-762222d5ef8c", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:47:42.103000000", + "trace_end_time": "2025-10-16 08:47:51.032000000", + "trace_duration_ms": 8928.948, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:47:42.103000000", + "updated_at": "2025-10-16 08:47:42.103000000" + }, + { + "span_id": "0b3d6893-776e-4118-b8b5-b1c5ea51def1", + "trace_id": "06192a09-b6bb-4548-8a94-e123d6caaffe", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-16 08:47:03.776000000", + "end_time": "2025-10-16 08:47:41.100000000", + "duration_ms": 37324.516, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What is Noveum and what does it do?", + "query.length": 35, + "processing.mode": "RAG", + "processing.time_seconds": 37.324477195739746, + "processing.response_length": 648, + "processing.sources_count": 5, + "output_response": "Final Answer: Noveum.ai is a comprehensive tracing and observability platform specifically designed for AI applications, including those powered by large language models (LLMs), retrieval-augmented generation (RAG)...", + "final_answer_mode": "RAG", + "query_processed.input_query": "What is Noveum and what does it do?", + "query_processed.output_response": "Successfully processed query using RAG, generated 648 character response", + "query_processed.mode": "RAG", + "query_processed.processing_time": 37.324477195739746, + "query_processed.response_length": 648 + }, + "events": [], + "links": [], + "trace_trace_id": "06192a09-b6bb-4548-8a94-e123d6caaffe", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:47:03.776000000", + "trace_end_time": "2025-10-16 08:47:41.100000000", + "trace_duration_ms": 37324.601, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:47:03.776000000", + "updated_at": "2025-10-16 08:47:03.776000000" + }, + { + "span_id": "137ad05b-bada-42be-abe5-be0c277f6d0e", + "trace_id": "06192a09-b6bb-4548-8a94-e123d6caaffe", + "parent_span_id": "c3664d20-1708-45f5-a051-b626b0ec77cd", + "name": "agent.llm-rag", + "start_time": "2025-10-16 08:47:03.776000000", + "end_time": "2025-10-16 08:47:41.100000000", + "duration_ms": 37324.042, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "rag_agent", + "agent.operation": "llm-rag", + "agent.capabilities": [ + "document_retrieval", + "context_generation", + "response_generation" + ], + "agent.id": "noveum_rag_agent", + "input_query": "What is Noveum and what does it do?", + "query_length": 35, + "query_type": "rag_query", + "output_response": "RAG Answer: Noveum.ai is a comprehensive tracing and observability platform specifically designed for AI applications, including those powered by large language models (LLMs), retrieval-augmented generation (RAG)...", + "rag.context_length": 2334, + "rag.sources_count": 5, + "rag.answer_length": 648, + "rag.mode": "retrieval_augmented_generation" + }, + "events": [], + "links": [], + "trace_trace_id": "06192a09-b6bb-4548-8a94-e123d6caaffe", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:47:03.776000000", + "trace_end_time": "2025-10-16 08:47:41.100000000", + "trace_duration_ms": 37324.601, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:47:03.776000000", + "updated_at": "2025-10-16 08:47:03.776000000" + }, + { + "span_id": "c3664d20-1708-45f5-a051-b626b0ec77cd", + "trace_id": "06192a09-b6bb-4548-8a94-e123d6caaffe", + "parent_span_id": "0b3d6893-776e-4118-b8b5-b1c5ea51def1", + "name": "agent.query_routing", + "start_time": "2025-10-16 08:47:03.776000000", + "end_time": "2025-10-16 08:47:41.100000000", + "duration_ms": 37324.382999999994, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What is Noveum and what does it do?", + "query_length": 35, + "query_type": "routing_query", + "output_response": "Routed to RAG for query processing", + "router.classification": "RAG", + "router.confidence_score": 1, + "router.rag_keyword_score": 1, + "router.web_keyword_score": 0, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "06192a09-b6bb-4548-8a94-e123d6caaffe", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:47:03.776000000", + "trace_end_time": "2025-10-16 08:47:41.100000000", + "trace_duration_ms": 37324.601, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:47:03.776000000", + "updated_at": "2025-10-16 08:47:03.776000000" + }, + { + "span_id": "ce9a94bf-1a72-4599-8561-11e66802b92d", + "trace_id": "06192a09-b6bb-4548-8a94-e123d6caaffe", + "parent_span_id": "c3664d20-1708-45f5-a051-b626b0ec77cd", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-16 08:47:03.776000000", + "end_time": "2025-10-16 08:47:03.776000000", + "duration_ms": 0.067, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What is Noveum and what does it do?", + "query_length": 35, + "query_type": "routing_evaluation_query", + "classification.mode": "RAG", + "classification.rag_keyword_score": 1, + "classification.web_keyword_score": 0, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "noveum_specific", + "query.keyword_density": 0.125, + "query.domain_affinity": "noveum", + "routing.decision": "Routed to RAG based on analysis", + "routing.rationale": "RAG score: 1, Web score: 0, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "Web Search", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.019230769230769232, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: RAG (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "06192a09-b6bb-4548-8a94-e123d6caaffe", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:47:03.776000000", + "trace_end_time": "2025-10-16 08:47:41.100000000", + "trace_duration_ms": 37324.601, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:47:03.776000000", + "updated_at": "2025-10-16 08:47:03.776000000" + }, + { + "span_id": "38414416-907a-4188-95fe-c0a5f900f72c", + "trace_id": "06192a09-b6bb-4548-8a94-e123d6caaffe", + "parent_span_id": "137ad05b-bada-42be-abe5-be0c277f6d0e", + "name": "agent.llm_model_execution", + "start_time": "2025-10-16 08:47:10.392000000", + "end_time": "2025-10-16 08:47:40.392000000", + "duration_ms": 30000.104, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What is Noveum and what does it do?", + "query_length": 35, + "query_type": "rag_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 30.000043392181396, + "model.latency_ms": 30000.043392181396, + "model.start_time": 1760604430.3921378, + "model.end_time": 1760604460.3921812, + "model.performance_tier": "slow", + "model.prompt_tokens": 716, + "model.completion_tokens": 162, + "model.total_tokens": 878, + "model.tokens_per_second": 29.266624335244266, + "model.estimated_cost": 0.026340000000000002, + "model.efficiency_score": 0.7380410022779044, + "model.response_length": 648, + "model.response_quality": "high", + "model.output_response": "Model Response: Noveum.ai is a comprehensive tracing and observability platform specifically designed for AI applications, including those powered by large language models (LLMs), retrieval-augmented generation (RAG)...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "06192a09-b6bb-4548-8a94-e123d6caaffe", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:47:03.776000000", + "trace_end_time": "2025-10-16 08:47:41.100000000", + "trace_duration_ms": 37324.601, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:47:03.776000000", + "updated_at": "2025-10-16 08:47:03.776000000" + }, + { + "span_id": "67e23649-b00b-4eb8-9a69-865e80f833b3", + "trace_id": "06192a09-b6bb-4548-8a94-e123d6caaffe", + "parent_span_id": "137ad05b-bada-42be-abe5-be0c277f6d0e", + "name": "agent.rag_evaluation_metrics", + "start_time": "2025-10-16 08:47:40.392000000", + "end_time": "2025-10-16 08:47:40.392000000", + "duration_ms": 0.029, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "rag_evaluation_metrics", + "agent.capabilities": [ + "retrieval_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What is Noveum and what does it do?", + "query_length": 35, + "query_type": "rag_evaluation_query", + "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, vector searches, data retrieval, agent reasoning, and more. Without proper observability, debugging ...", + "retrieval.context_length": 2334, + "retrieval.sources_count": 5, + "retrieval.context_quality": "high", + "retrieval.effectiveness": 1, + "retrieval.context_utilization": 2.334, + "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, vector searches, data retrieval, agent reasoning, and more. Without proper observability, debugging becomes a nightmare, optimization is guesswork, and understanding user interactions is nearly impossible.\nNoveum.ai solves this by providing comprehensive tracing and observability specifically designed for AI applications.\nKey benefits of using Noveum.ai:\n- Complete Visibility: Trace...\n\nSource 2 (https://noveum.ai/en/docs/getting-started/overview):\nNoveum.ai Overview\nComprehensive AI tracing and observability platform for LLM applications, RAG systems, and AI agents\nWelcome to Noveum.ai\u2014the comprehensive tracing and observability platform built specifically for AI applications. Whether you're building LLM-powered chatbots, RAG systems, multi-agent workflows, or any AI-driven application, Noveum provides the insights you need to understand, debug, and optimize your systems.\n\ud83c\udfaf Why AI Applications Need Specialized Observability\nTraditional mo...\n\nSource 3 (https://noveum.ai/docs/getting-started/overview):\nNoveum.ai Overview\nComprehensive AI tracing and observability platform for LLM applications, RAG systems, and AI agents\nWelcome to Noveum.ai\u2014the comprehensive tracing and observability platform built specifically for AI applications. Whether you're building LLM-powered chatbots, RAG systems, multi-agent workflows, or any AI-driven application, Noveum provides the insights you need to understand, debug, and optimize your systems.\n\ud83c\udfaf Why AI Applications Need Specialized Observability\nTraditional mo...\n\nSource 4 (https://noveum.ai/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nSource 5 (https://noveum.ai/en/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nUser Question: What is Noveum and what does it do?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", + "prompt.prompt_length": 2865, + "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, ...", + "prompt.instruction_following": "rag_optimized", + "response.answer_length": 648, + "response.answer_completeness": "complete", + "response.response_quality": "high", + "response.source_citation": 5, + "response.context_utilization": 0.2776349614395887, + "output_response": "RAG Answer: Noveum.ai is a comprehensive tracing and observability platform specifically designed for AI applications, including those powered by large language models (LLMs), retrieval-augmented generation (RAG)...", + "evaluation.retrieval_effectiveness": 1, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 5, + "evaluation.context_relevance": "high", + "evaluation.overall_quality": "high", + "evaluation.ready_for_production": true, + "rag.retrieval_strategy": "semantic_similarity", + "rag.vector_search_results": 5, + "rag.context_synthesis": "multi_source", + "rag.document_coverage": 1, + "rag.information_density": 0.2776349614395887 + }, + "events": [], + "links": [], + "trace_trace_id": "06192a09-b6bb-4548-8a94-e123d6caaffe", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:47:03.776000000", + "trace_end_time": "2025-10-16 08:47:41.100000000", + "trace_duration_ms": 37324.601, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:47:03.776000000", + "updated_at": "2025-10-16 08:47:03.776000000" + }, + { + "span_id": "050851c9-b2d3-4c82-accb-4085dc69f785", + "trace_id": "3552ee8e-eafb-499a-b109-a9fb851a41d2", + "parent_span_id": "", + "name": "system_initialization", + "start_time": "2025-10-16 08:47:03.695000000", + "end_time": "2025-10-16 08:47:03.770000000", + "duration_ms": 74.58, + "status": "ok", + "status_message": "", + "attributes": { + "system.force_scrape": false, + "system.config": { + "noveum_base_url": "https://noveum.ai", + "max_pages_to_scrape": 50, + "chunk_size": 1000, + "chunk_overlap": 200, + "max_search_results": 5, + "rag_threshold": 0.7, + "noveum_docs_file": "noveum_docs.json", + "vector_store_path": "noveum_vectorstore" + }, + "input_query": "Initialize system with force_scrape=False", + "output_response": "System initialization: RAG system loaded, vector store ready, agent operational", + "system.initialized": true, + "system.documents_loaded": 38, + "system.vectorstore_ready": true + }, + "events": [], + "links": [], + "trace_trace_id": "3552ee8e-eafb-499a-b109-a9fb851a41d2", + "trace_name": "auto_trace_system_initialization", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-16 08:47:03.695000000", + "trace_end_time": "2025-10-16 08:47:03.770000000", + "trace_duration_ms": 74.912, + "span_count": 1, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-16 08:47:03.695000000", + "updated_at": "2025-10-16 08:47:03.695000000" + }, + { + "span_id": "6b53235d-8b91-4d9e-8437-915828e84600", + "trace_id": "e6ef2449-7429-465b-9bff-f3def8de5bb0", + "parent_span_id": "e73ab184-50c1-4209-9276-be11e69e9513", + "name": "agent.query_routing", + "start_time": "2025-10-14 12:54:06.700000000", + "end_time": "2025-10-14 12:54:10.637000000", + "duration_ms": 3937.492, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What are the latest breakthroughs in artificial intelligence?", + "query_length": 61, + "query_type": "routing_query", + "output_response": "Routed to Web Search for query processing", + "router.classification": "Web Search", + "router.confidence_score": 1, + "router.rag_keyword_score": 0, + "router.web_keyword_score": 1, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "e6ef2449-7429-465b-9bff-f3def8de5bb0", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-14 12:54:06.700000000", + "trace_end_time": "2025-10-14 12:54:10.637000000", + "trace_duration_ms": 3937.729, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-14 12:54:06.700000000", + "updated_at": "2025-10-14 12:54:06.700000000" + }, + { + "span_id": "a7d1e33a-95a1-41a9-a117-62b4c373bd60", + "trace_id": "e6ef2449-7429-465b-9bff-f3def8de5bb0", + "parent_span_id": "6b53235d-8b91-4d9e-8437-915828e84600", + "name": "agent.web_search_generation", + "start_time": "2025-10-14 12:54:06.700000000", + "end_time": "2025-10-14 12:54:10.637000000", + "duration_ms": 3936.983, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "web_search_agent", + "agent.operation": "web_search_generation", + "agent.capabilities": [ + "web_search", + "content_synthesis", + "response_generation" + ], + "agent.id": "noveum_web_search_agent", + "input_query": "What are the latest breakthroughs in artificial intelligence?", + "query_length": 61, + "query_type": "web_search_query", + "output_response": "Web Search Answer: The web search did not yield specific results regarding the latest breakthroughs in artificial intelligence. However, I can provide a general overview based on knowledge up to October 2023.\n\nRecent br...", + "web_search.results_count": 1, + "web_search.context_length": 163, + "web_search.response_length": 1090, + "web_search.mode": "external_web_search" + }, + "events": [], + "links": [], + "trace_trace_id": "e6ef2449-7429-465b-9bff-f3def8de5bb0", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-14 12:54:06.700000000", + "trace_end_time": "2025-10-14 12:54:10.637000000", + "trace_duration_ms": 3937.729, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-14 12:54:06.700000000", + "updated_at": "2025-10-14 12:54:06.700000000" + }, + { + "span_id": "da630cab-2ba6-4f12-98b9-6ab1c0899c17", + "trace_id": "e6ef2449-7429-465b-9bff-f3def8de5bb0", + "parent_span_id": "6b53235d-8b91-4d9e-8437-915828e84600", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-14 12:54:06.700000000", + "end_time": "2025-10-14 12:54:06.700000000", + "duration_ms": 0.27999999999999997, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What are the latest breakthroughs in artificial intelligence?", + "query_length": 61, + "query_type": "routing_evaluation_query", + "classification.mode": "Web Search", + "classification.rag_keyword_score": 0, + "classification.web_keyword_score": 1, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "complex", + "query.intent": "general_knowledge", + "query.keyword_density": 0.125, + "query.domain_affinity": "general", + "routing.decision": "Routed to Web Search based on analysis", + "routing.rationale": "RAG score: 0, Web score: 1, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "RAG", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.019230769230769232, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: Web Search (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "e6ef2449-7429-465b-9bff-f3def8de5bb0", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-14 12:54:06.700000000", + "trace_end_time": "2025-10-14 12:54:10.637000000", + "trace_duration_ms": 3937.729, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-14 12:54:06.700000000", + "updated_at": "2025-10-14 12:54:06.700000000" + }, + { + "span_id": "e73ab184-50c1-4209-9276-be11e69e9513", + "trace_id": "e6ef2449-7429-465b-9bff-f3def8de5bb0", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-14 12:54:06.700000000", + "end_time": "2025-10-14 12:54:10.637000000", + "duration_ms": 3937.663, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What are the latest breakthroughs in artificial intelligence?", + "query.length": 61, + "processing.mode": "Web Search", + "processing.time_seconds": 3.93752384185791, + "processing.response_length": 1090, + "processing.sources_count": 1, + "output_response": "Final Answer: The web search did not yield specific results regarding the latest breakthroughs in artificial intelligence. However, I can provide a general overview based on knowledge up to October 2023.\n\nRecent br...", + "final_answer_mode": "Web Search", + "query_processed.input_query": "What are the latest breakthroughs in artificial intelligence?", + "query_processed.output_response": "Successfully processed query using Web Search, generated 1090 character response", + "query_processed.mode": "Web Search", + "query_processed.processing_time": 3.93752384185791, + "query_processed.response_length": 1090 + }, + "events": [], + "links": [], + "trace_trace_id": "e6ef2449-7429-465b-9bff-f3def8de5bb0", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-14 12:54:06.700000000", + "trace_end_time": "2025-10-14 12:54:10.637000000", + "trace_duration_ms": 3937.729, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-14 12:54:06.700000000", + "updated_at": "2025-10-14 12:54:06.700000000" + }, + { + "span_id": "bb47766c-afda-44fe-8838-7936c80cd4b6", + "trace_id": "e6ef2449-7429-465b-9bff-f3def8de5bb0", + "parent_span_id": "a7d1e33a-95a1-41a9-a117-62b4c373bd60", + "name": "agent.llm_model_execution", + "start_time": "2025-10-14 12:54:07.104000000", + "end_time": "2025-10-14 12:54:10.637000000", + "duration_ms": 3533.321, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What are the latest breakthroughs in artificial intelligence?", + "query_length": 61, + "query_type": "web_search_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 3.533268928527832, + "model.latency_ms": 3533.268928527832, + "model.start_time": 1760446447.104266, + "model.end_time": 1760446450.6375349, + "model.performance_tier": "medium", + "model.prompt_tokens": 167, + "model.completion_tokens": 272, + "model.total_tokens": 439, + "model.tokens_per_second": 124.2475477752307, + "model.estimated_cost": 0.013170000000000001, + "model.efficiency_score": 2.4829157175398633, + "model.response_length": 1090, + "model.response_quality": "high", + "model.output_response": "Model Response: The web search did not yield specific results regarding the latest breakthroughs in artificial intelligence. However, I can provide a general overview based on knowledge up to October 2023.\n\nRecent br...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "e6ef2449-7429-465b-9bff-f3def8de5bb0", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-14 12:54:06.700000000", + "trace_end_time": "2025-10-14 12:54:10.637000000", + "trace_duration_ms": 3937.729, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-14 12:54:06.700000000", + "updated_at": "2025-10-14 12:54:06.700000000" + }, + { + "span_id": "1d914a9d-b770-45dc-8308-6a36e9e0db89", + "trace_id": "e6ef2449-7429-465b-9bff-f3def8de5bb0", + "parent_span_id": "a7d1e33a-95a1-41a9-a117-62b4c373bd60", + "name": "agent.web_search_evaluation_metrics", + "start_time": "2025-10-14 12:54:10.637000000", + "end_time": "2025-10-14 12:54:10.637000000", + "duration_ms": 0.022, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "web_search_evaluation_metrics", + "agent.capabilities": [ + "web_search_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What are the latest breakthroughs in artificial intelligence?", + "query_length": 61, + "query_type": "web_search_evaluation_query", + "web_search.results_count": 1, + "web_search.context_length": 163, + "web_search.context_synthesized": "Context from 1 web sources", + "web_search.search_effectiveness": 0.2, + "web_search.context_quality": "low", + "web_search.source_diversity": 1, + "web_search.information_synthesis": "low", + "web_search.external_knowledge_utilization": 0.163, + "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What+are+the+latest+breakthroughs+in+artificial+intelligence?\n\nUser Question: What are the latest breakthroughs in artificial intelligence?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", + "prompt.prompt_length": 669, + "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What+are+the+latest+breakthroughs+in+artificial+intelligence?", + "prompt.instruction_following": "web_search_optimized", + "response.answer_length": 1090, + "response.answer_completeness": "complete", + "response.response_quality": "medium", + "response.source_citation": 1, + "response.context_utilization": 6.6871165644171775, + "output_response": "Web Search Answer: The web search did not yield specific results regarding the latest breakthroughs in artificial intelligence. However, I can provide a general overview based on knowledge up to October 2023.\n\nRecent br...", + "evaluation.search_effectiveness": 0.2, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 1, + "evaluation.context_relevance": "low", + "evaluation.overall_quality": "low", + "evaluation.ready_for_production": true, + "web_search.search_strategy": "duckduckgo_api", + "web_search.real_time_data": true, + "web_search.external_sources": 1, + "web_search.information_freshness": "current", + "web_search.knowledge_synthesis": "single_source" + }, + "events": [], + "links": [], + "trace_trace_id": "e6ef2449-7429-465b-9bff-f3def8de5bb0", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-14 12:54:06.700000000", + "trace_end_time": "2025-10-14 12:54:10.637000000", + "trace_duration_ms": 3937.729, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-14 12:54:06.700000000", + "updated_at": "2025-10-14 12:54:06.700000000" + }, + { + "span_id": "91f3999c-2edf-4c56-8f4d-ab43b6ebbaee", + "trace_id": "2ba8cab0-2366-4add-8ac3-b490a05e9a60", + "parent_span_id": "cbe51252-a323-4852-a90e-c907a0480fa3", + "name": "agent.web_search_generation", + "start_time": "2025-10-14 12:53:56.480000000", + "end_time": "2025-10-14 12:54:05.685000000", + "duration_ms": 9205.212, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "web_search_agent", + "agent.operation": "web_search_generation", + "agent.capabilities": [ + "web_search", + "content_synthesis", + "response_generation" + ], + "agent.id": "noveum_web_search_agent", + "input_query": "What's happening in the software development world today?", + "query_length": 57, + "query_type": "web_search_query", + "output_response": "Web Search Answer: The software development world is currently experiencing rapid changes driven by advancements in artificial intelligence (AI) and evolving programming practices. Key trends include:\n\n1. **AI Integrati...", + "web_search.results_count": 1, + "web_search.context_length": 975, + "web_search.response_length": 1636, + "web_search.mode": "external_web_search" + }, + "events": [], + "links": [], + "trace_trace_id": "2ba8cab0-2366-4add-8ac3-b490a05e9a60", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-14 12:53:56.480000000", + "trace_end_time": "2025-10-14 12:54:05.687000000", + "trace_duration_ms": 9207.67, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-14 12:53:56.480000000", + "updated_at": "2025-10-14 12:53:56.480000000" + }, + { + "span_id": "ea2aee75-4537-46ea-954b-2002fc0026c5", + "trace_id": "2ba8cab0-2366-4add-8ac3-b490a05e9a60", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-14 12:53:56.480000000", + "end_time": "2025-10-14 12:54:05.687000000", + "duration_ms": 9207.528999999999, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What's happening in the software development world today?", + "query.length": 57, + "processing.mode": "Web Search", + "processing.time_seconds": 9.206756830215454, + "processing.response_length": 1636, + "processing.sources_count": 1, + "output_response": "Final Answer: The software development world is currently experiencing rapid changes driven by advancements in artificial intelligence (AI) and evolving programming practices. Key trends include:\n\n1. **AI Integrati...", + "final_answer_mode": "Web Search", + "query_processed.input_query": "What's happening in the software development world today?", + "query_processed.output_response": "Successfully processed query using Web Search, generated 1636 character response", + "query_processed.mode": "Web Search", + "query_processed.processing_time": 9.206756830215454, + "query_processed.response_length": 1636 + }, + "events": [], + "links": [], + "trace_trace_id": "2ba8cab0-2366-4add-8ac3-b490a05e9a60", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-14 12:53:56.480000000", + "trace_end_time": "2025-10-14 12:54:05.687000000", + "trace_duration_ms": 9207.67, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-14 12:53:56.480000000", + "updated_at": "2025-10-14 12:53:56.480000000" + }, + { + "span_id": "cbe51252-a323-4852-a90e-c907a0480fa3", + "trace_id": "2ba8cab0-2366-4add-8ac3-b490a05e9a60", + "parent_span_id": "ea2aee75-4537-46ea-954b-2002fc0026c5", + "name": "agent.query_routing", + "start_time": "2025-10-14 12:53:56.480000000", + "end_time": "2025-10-14 12:54:05.686000000", + "duration_ms": 9205.569000000001, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What's happening in the software development world today?", + "query_length": 57, + "query_type": "routing_query", + "output_response": "Routed to Web Search for query processing", + "router.classification": "Web Search", + "router.confidence_score": 1, + "router.rag_keyword_score": 0, + "router.web_keyword_score": 1, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "2ba8cab0-2366-4add-8ac3-b490a05e9a60", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-14 12:53:56.480000000", + "trace_end_time": "2025-10-14 12:54:05.687000000", + "trace_duration_ms": 9207.67, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-14 12:53:56.480000000", + "updated_at": "2025-10-14 12:53:56.480000000" + }, + { + "span_id": "d850ecee-86ef-4818-9cec-638bfee26293", + "trace_id": "2ba8cab0-2366-4add-8ac3-b490a05e9a60", + "parent_span_id": "cbe51252-a323-4852-a90e-c907a0480fa3", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-14 12:53:56.480000000", + "end_time": "2025-10-14 12:53:56.480000000", + "duration_ms": 0.041999999999999996, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What's happening in the software development world today?", + "query_length": 57, + "query_type": "routing_evaluation_query", + "classification.mode": "Web Search", + "classification.rag_keyword_score": 0, + "classification.web_keyword_score": 1, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "complex", + "query.intent": "general_knowledge", + "query.keyword_density": 0.125, + "query.domain_affinity": "general", + "routing.decision": "Routed to Web Search based on analysis", + "routing.rationale": "RAG score: 0, Web score: 1, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "RAG", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.019230769230769232, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: Web Search (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "2ba8cab0-2366-4add-8ac3-b490a05e9a60", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-14 12:53:56.480000000", + "trace_end_time": "2025-10-14 12:54:05.687000000", + "trace_duration_ms": 9207.67, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-14 12:53:56.480000000", + "updated_at": "2025-10-14 12:53:56.480000000" + }, + { + "span_id": "0d4387a4-adf3-42c1-aca7-51453a90dd36", + "trace_id": "2ba8cab0-2366-4add-8ac3-b490a05e9a60", + "parent_span_id": "91f3999c-2edf-4c56-8f4d-ab43b6ebbaee", + "name": "agent.llm_model_execution", + "start_time": "2025-10-14 12:53:59.426000000", + "end_time": "2025-10-14 12:54:05.684000000", + "duration_ms": 6258.643, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What's happening in the software development world today?", + "query_length": 57, + "query_type": "web_search_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 6.25848388671875, + "model.latency_ms": 6258.48388671875, + "model.start_time": 1760446439.426367, + "model.end_time": 1760446445.684851, + "model.performance_tier": "slow", + "model.prompt_tokens": 369, + "model.completion_tokens": 409, + "model.total_tokens": 778, + "model.tokens_per_second": 124.31125718019486, + "model.estimated_cost": 0.02334, + "model.efficiency_score": 2.102827763496144, + "model.response_length": 1636, + "model.response_quality": "high", + "model.output_response": "Model Response: The software development world is currently experiencing rapid changes driven by advancements in artificial intelligence (AI) and evolving programming practices. Key trends include:\n\n1. **AI Integrati...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "2ba8cab0-2366-4add-8ac3-b490a05e9a60", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-14 12:53:56.480000000", + "trace_end_time": "2025-10-14 12:54:05.687000000", + "trace_duration_ms": 9207.67, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-14 12:53:56.480000000", + "updated_at": "2025-10-14 12:53:56.480000000" + }, + { + "span_id": "717e3c4a-6300-45e5-9456-caa3e773d758", + "trace_id": "2ba8cab0-2366-4add-8ac3-b490a05e9a60", + "parent_span_id": "91f3999c-2edf-4c56-8f4d-ab43b6ebbaee", + "name": "agent.web_search_evaluation_metrics", + "start_time": "2025-10-14 12:54:05.685000000", + "end_time": "2025-10-14 12:54:05.685000000", + "duration_ms": 0.044, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "web_search_evaluation_metrics", + "agent.capabilities": [ + "web_search_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What's happening in the software development world today?", + "query_length": 57, + "query_type": "web_search_evaluation_query", + "web_search.results_count": 1, + "web_search.context_length": 975, + "web_search.context_synthesized": "Context from 1 web sources", + "web_search.search_effectiveness": 0.2, + "web_search.context_quality": "high", + "web_search.source_diversity": 1, + "web_search.information_synthesis": "low", + "web_search.external_knowledge_utilization": 0.975, + "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nLog in to WhatsApp Web for simple, reliable and private messaging on your desktop. Send and receive messages and files with ease, all for free. Sep 23, 2025 \u00b7 Chief Economists' Outlook 2025 states that these under-the-radar trends are shaping the global economy, from the impact of AI on research productivity to a tightened labor \u2026 Sep 1, 2025 \u00b7 THE AI DEVELOPMENT REVOLUTION: WHAT'S REALLY HAPPENING? Let's face it - the software development world is changing faster than a React component re-rendering on \u2026 Oct 7, 2025 \u00b7 Computer Programming Research. Read current computer science articles on everything from computer programs to detect cancer genes and control vehicle maintenance to \u2026 Aug 24, 2025 \u00b7 Artificial Intelligence News. Everything on AI including futuristic robots with artificial intelligence, computer models of human intelligence and more.\nURL: https://duckduckgo.com/?q=What's+happening+in+the+software+development+world+today?\n\nUser Question: What's happening in the software development world today?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", + "prompt.prompt_length": 1477, + "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nLog in to WhatsApp Web for simple, reliable and private messaging on your desktop. Send and receive messages and files with ease, all for free. Sep 23, 2025 \u00b7 Chief Economi...", + "prompt.instruction_following": "web_search_optimized", + "response.answer_length": 1636, + "response.answer_completeness": "complete", + "response.response_quality": "medium", + "response.source_citation": 1, + "response.context_utilization": 1.677948717948718, + "output_response": "Web Search Answer: The software development world is currently experiencing rapid changes driven by advancements in artificial intelligence (AI) and evolving programming practices. Key trends include:\n\n1. **AI Integrati...", + "evaluation.search_effectiveness": 0.2, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 1, + "evaluation.context_relevance": "high", + "evaluation.overall_quality": "low", + "evaluation.ready_for_production": true, + "web_search.search_strategy": "duckduckgo_api", + "web_search.real_time_data": true, + "web_search.external_sources": 1, + "web_search.information_freshness": "current", + "web_search.knowledge_synthesis": "single_source" + }, + "events": [], + "links": [], + "trace_trace_id": "2ba8cab0-2366-4add-8ac3-b490a05e9a60", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-14 12:53:56.480000000", + "trace_end_time": "2025-10-14 12:54:05.687000000", + "trace_duration_ms": 9207.67, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-14 12:53:56.480000000", + "updated_at": "2025-10-14 12:53:56.480000000" + }, + { + "span_id": "3d39f522-8ae7-4988-aafd-58b09734eff0", + "trace_id": "a7c3399e-372f-4aae-baeb-05215cc4b85f", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-14 12:53:50.963000000", + "end_time": "2025-10-14 12:53:55.471000000", + "duration_ms": 4507.466, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What are the newest features in cloud computing?", + "query.length": 48, + "processing.mode": "RAG", + "processing.time_seconds": 4.507421255111694, + "processing.response_length": 870, + "processing.sources_count": 5, + "output_response": "Final Answer: The provided context does not specifically mention any new features in cloud computing as a general topic. However, it does highlight several recent enhancements related to the Noveum.ai platform, whi...", + "final_answer_mode": "RAG", + "query_processed.input_query": "What are the newest features in cloud computing?", + "query_processed.output_response": "Successfully processed query using RAG, generated 870 character response", + "query_processed.mode": "RAG", + "query_processed.processing_time": 4.507421255111694, + "query_processed.response_length": 870 + }, + "events": [], + "links": [], + "trace_trace_id": "a7c3399e-372f-4aae-baeb-05215cc4b85f", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-14 12:53:50.963000000", + "trace_end_time": "2025-10-14 12:53:55.471000000", + "trace_duration_ms": 4507.555, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-14 12:53:50.963000000", + "updated_at": "2025-10-14 12:53:50.963000000" + }, + { + "span_id": "54aa6700-f2de-4cf0-b1ad-a3d24ed692ed", + "trace_id": "a7c3399e-372f-4aae-baeb-05215cc4b85f", + "parent_span_id": "d17f31cf-c570-4604-8df3-532b2e530fce", + "name": "agent.llm-rag", + "start_time": "2025-10-14 12:53:50.963000000", + "end_time": "2025-10-14 12:53:55.471000000", + "duration_ms": 4507.083, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "rag_agent", + "agent.operation": "llm-rag", + "agent.capabilities": [ + "document_retrieval", + "context_generation", + "response_generation" + ], + "agent.id": "noveum_rag_agent", + "input_query": "What are the newest features in cloud computing?", + "query_length": 48, + "query_type": "rag_query", + "output_response": "RAG Answer: The provided context does not specifically mention any new features in cloud computing as a general topic. However, it does highlight several recent enhancements related to the Noveum.ai platform, whi...", + "rag.context_length": 2579, + "rag.sources_count": 5, + "rag.answer_length": 870, + "rag.mode": "retrieval_augmented_generation" + }, + "events": [], + "links": [], + "trace_trace_id": "a7c3399e-372f-4aae-baeb-05215cc4b85f", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-14 12:53:50.963000000", + "trace_end_time": "2025-10-14 12:53:55.471000000", + "trace_duration_ms": 4507.555, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-14 12:53:50.963000000", + "updated_at": "2025-10-14 12:53:50.963000000" + }, + { + "span_id": "8878096f-6cfa-4f70-9844-d7723a0875f0", + "trace_id": "a7c3399e-372f-4aae-baeb-05215cc4b85f", + "parent_span_id": "d17f31cf-c570-4604-8df3-532b2e530fce", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-14 12:53:50.963000000", + "end_time": "2025-10-14 12:53:50.963000000", + "duration_ms": 0.036000000000000004, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What are the newest features in cloud computing?", + "query_length": 48, + "query_type": "routing_evaluation_query", + "classification.mode": "RAG", + "classification.rag_keyword_score": 1, + "classification.web_keyword_score": 0, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "documentation", + "query.keyword_density": 0.125, + "query.domain_affinity": "noveum", + "routing.decision": "Routed to RAG based on analysis", + "routing.rationale": "RAG score: 1, Web score: 0, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "Web Search", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.019230769230769232, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: RAG (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "a7c3399e-372f-4aae-baeb-05215cc4b85f", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-14 12:53:50.963000000", + "trace_end_time": "2025-10-14 12:53:55.471000000", + "trace_duration_ms": 4507.555, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-14 12:53:50.963000000", + "updated_at": "2025-10-14 12:53:50.963000000" + }, + { + "span_id": "d17f31cf-c570-4604-8df3-532b2e530fce", + "trace_id": "a7c3399e-372f-4aae-baeb-05215cc4b85f", + "parent_span_id": "3d39f522-8ae7-4988-aafd-58b09734eff0", + "name": "agent.query_routing", + "start_time": "2025-10-14 12:53:50.963000000", + "end_time": "2025-10-14 12:53:55.471000000", + "duration_ms": 4507.3730000000005, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What are the newest features in cloud computing?", + "query_length": 48, + "query_type": "routing_query", + "output_response": "Routed to RAG for query processing", + "router.classification": "RAG", + "router.confidence_score": 1, + "router.rag_keyword_score": 1, + "router.web_keyword_score": 0, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "a7c3399e-372f-4aae-baeb-05215cc4b85f", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-14 12:53:50.963000000", + "trace_end_time": "2025-10-14 12:53:55.471000000", + "trace_duration_ms": 4507.555, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-14 12:53:50.963000000", + "updated_at": "2025-10-14 12:53:50.963000000" + }, + { + "span_id": "97ef01a8-5930-4902-909a-245c516a5871", + "trace_id": "a7c3399e-372f-4aae-baeb-05215cc4b85f", + "parent_span_id": "54aa6700-f2de-4cf0-b1ad-a3d24ed692ed", + "name": "agent.llm_model_execution", + "start_time": "2025-10-14 12:53:51.439000000", + "end_time": "2025-10-14 12:53:55.077000000", + "duration_ms": 3637.337, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What are the newest features in cloud computing?", + "query_length": 48, + "query_type": "rag_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 3.6372909545898438, + "model.latency_ms": 3637.2909545898438, + "model.start_time": 1760446431.439748, + "model.end_time": 1760446435.077039, + "model.performance_tier": "medium", + "model.prompt_tokens": 780, + "model.completion_tokens": 217, + "model.total_tokens": 997, + "model.tokens_per_second": 274.1050997698989, + "model.estimated_cost": 0.02991, + "model.efficiency_score": 0.872617853560682, + "model.response_length": 870, + "model.response_quality": "high", + "model.output_response": "Model Response: The provided context does not specifically mention any new features in cloud computing as a general topic. However, it does highlight several recent enhancements related to the Noveum.ai platform, whi...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "a7c3399e-372f-4aae-baeb-05215cc4b85f", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-14 12:53:50.963000000", + "trace_end_time": "2025-10-14 12:53:55.471000000", + "trace_duration_ms": 4507.555, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-14 12:53:50.963000000", + "updated_at": "2025-10-14 12:53:50.963000000" + }, + { + "span_id": "ea792ec6-64b4-4ed5-b418-ba902e90c57a", + "trace_id": "a7c3399e-372f-4aae-baeb-05215cc4b85f", + "parent_span_id": "54aa6700-f2de-4cf0-b1ad-a3d24ed692ed", + "name": "agent.rag_evaluation_metrics", + "start_time": "2025-10-14 12:53:55.077000000", + "end_time": "2025-10-14 12:53:55.077000000", + "duration_ms": 0.034, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "rag_evaluation_metrics", + "agent.capabilities": [ + "retrieval_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What are the newest features in cloud computing?", + "query_length": 48, + "query_type": "rag_evaluation_query", + "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/changelog):\n- \ud83d\udcc8 Enhanced Dashboard Analytics: Real-time request tracking, latency monitoring, and cost analysis\n- \ud83d\udd0d Improved Logs Interface: Better search functionality, detailed trace views, and enhanced debugging capabilities\n- \ud83d\udc65 Team Management: Advanced member mana...", + "retrieval.context_length": 2579, + "retrieval.sources_count": 5, + "retrieval.context_quality": "high", + "retrieval.effectiveness": 1, + "retrieval.context_utilization": 2.579, + "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/changelog):\n- \ud83d\udcc8 Enhanced Dashboard Analytics: Real-time request tracking, latency monitoring, and cost analysis\n- \ud83d\udd0d Improved Logs Interface: Better search functionality, detailed trace views, and enhanced debugging capabilities\n- \ud83d\udc65 Team Management: Advanced member management with role-based access control and invitation system\n- \ud83d\udee1\ufe0f Security Improvements: Enhanced API key management and secure credential storage\n4 months ago\n- \ud83d\udcca Advanced Metrics Collection: Comprehensive telemetry system with custom metrics ...\n\nSource 2 (https://noveum.ai/en/changelog):\n- \ud83d\udcdd Documentation Updates: Comprehensive guides for integration and best practices\n5 months ago\n- \ud83c\udf89 Platform Foundation: Initial release of Noveum.ai AI observability platform\n- \ud83d\ude80 Core Features: AI Gateway integration, basic evaluation metrics, and monitoring capabilities\n- \ud83d\udcc8 Evaluation Jobs: Automated model evaluation with configurable metrics and reporting\n- \ud83d\udd0c Provider Integrations: Support for major AI providers and custom model deployments...\n\nSource 3 (https://noveum.ai/docs/platform/dashboard):\n- Bulk Export: Download multiple traces simultaneously\n- Comparative Analysis: Compare performance across multiple traces\nReal-Time Updates\n- Live Refresh: Automatic updates as new traces arrive\n- Connection Monitoring: Real-time status of your trace ingestion pipeline\n- Performance Indicators: Live metrics for system health monitoring\nReady to dive deeper? Explore Projects & Environments to organize your AI applications, or check out Team Collaboration to share insights with your team.\nGet Earl...\n\nSource 4 (https://noveum.ai/en/changelog):\nChangelog\nStay up to date with the latest changes in our product.\n2 months ago\n- \ud83c\udfaf Major Platform Evolution: NovaEval Framework - Launched comprehensive AI model evaluation framework with 20+ built-in scorers including accuracy, RAG metrics, conversational scoring, and specialized agent evaluation tools. Features production-grade deployment on Docker and Kubernetes. (GitHub | PyPI Package)\n- \ud83d\ude80 Next-Generation Traces UI - Complete redesign featuring three-pane layout with directory tree navigatio...\n\nSource 5 (https://noveum.ai/en/blog):\n8/12/2025\nDiscover how Noveum.ai provides comprehensive tracing and observability for AI applications, from development debugging to production optimization.\nShashank Agarwal\n3/3/2025\nDiscover how Noveum.ai provides comprehensive tracing and observability for LLM applications, RAG systems, and multi-agent workflows with our powerful Python and TypeScript SDKs.\nShashank Agarwal\n3/2/2025...\n\nUser Question: What are the newest features in cloud computing?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", + "prompt.prompt_length": 3123, + "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/changelog):\n- \ud83d\udcc8 Enhanced Dashboard Analytics: Real-time request tracking, latency monitoring, and cost analysis\n- \ud83d\udd0d Improved Logs Interface: Better search functionality,...", + "prompt.instruction_following": "rag_optimized", + "response.answer_length": 870, + "response.answer_completeness": "complete", + "response.response_quality": "high", + "response.source_citation": 5, + "response.context_utilization": 0.33734005428460645, + "output_response": "RAG Answer: The provided context does not specifically mention any new features in cloud computing as a general topic. However, it does highlight several recent enhancements related to the Noveum.ai platform, whi...", + "evaluation.retrieval_effectiveness": 1, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 5, + "evaluation.context_relevance": "high", + "evaluation.overall_quality": "high", + "evaluation.ready_for_production": true, + "rag.retrieval_strategy": "semantic_similarity", + "rag.vector_search_results": 5, + "rag.context_synthesis": "multi_source", + "rag.document_coverage": 1, + "rag.information_density": 0.33734005428460645 + }, + "events": [], + "links": [], + "trace_trace_id": "a7c3399e-372f-4aae-baeb-05215cc4b85f", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-14 12:53:50.963000000", + "trace_end_time": "2025-10-14 12:53:55.471000000", + "trace_duration_ms": 4507.555, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-14 12:53:50.963000000", + "updated_at": "2025-10-14 12:53:50.963000000" + }, + { + "span_id": "e2d314c8-687f-4b8d-869c-8d7f2a22a4f3", + "trace_id": "d682797f-2ce0-448e-8f0d-891541430988", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-14 12:53:44.917000000", + "end_time": "2025-10-14 12:53:49.957000000", + "duration_ms": 5039.565, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What's the current status of cryptocurrency markets?", + "query.length": 52, + "processing.mode": "Web Search", + "processing.time_seconds": 5.0394370555877686, + "processing.response_length": 338, + "processing.sources_count": 1, + "output_response": "Final Answer: The web search results did not provide any specific information regarding the current status of cryptocurrency markets. Therefore, I cannot provide an update on market conditions, prices, or trends at...", + "final_answer_mode": "Web Search", + "query_processed.input_query": "What's the current status of cryptocurrency markets?", + "query_processed.output_response": "Successfully processed query using Web Search, generated 338 character response", + "query_processed.mode": "Web Search", + "query_processed.processing_time": 5.0394370555877686, + "query_processed.response_length": 338 + }, + "events": [], + "links": [], + "trace_trace_id": "d682797f-2ce0-448e-8f0d-891541430988", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-14 12:53:44.917000000", + "trace_end_time": "2025-10-14 12:53:49.957000000", + "trace_duration_ms": 5039.64, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-14 12:53:44.917000000", + "updated_at": "2025-10-14 12:53:44.917000000" + }, + { + "span_id": "d367a6bd-86af-41c5-96ca-5284425775ce", + "trace_id": "d682797f-2ce0-448e-8f0d-891541430988", + "parent_span_id": "f137a412-6ddf-4ace-908e-9edf73534aa7", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-14 12:53:44.917000000", + "end_time": "2025-10-14 12:53:44.918000000", + "duration_ms": 0.046, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What's the current status of cryptocurrency markets?", + "query_length": 52, + "query_type": "routing_evaluation_query", + "classification.mode": "Web Search", + "classification.rag_keyword_score": 0, + "classification.web_keyword_score": 3, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "complex", + "query.intent": "general_knowledge", + "query.keyword_density": 0.42857142857142855, + "query.domain_affinity": "general", + "routing.decision": "Routed to Web Search based on analysis", + "routing.rationale": "RAG score: 0, Web score: 3, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "RAG", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.057692307692307696, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: Web Search (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "d682797f-2ce0-448e-8f0d-891541430988", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-14 12:53:44.917000000", + "trace_end_time": "2025-10-14 12:53:49.957000000", + "trace_duration_ms": 5039.64, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-14 12:53:44.917000000", + "updated_at": "2025-10-14 12:53:44.917000000" + }, + { + "span_id": "f137a412-6ddf-4ace-908e-9edf73534aa7", + "trace_id": "d682797f-2ce0-448e-8f0d-891541430988", + "parent_span_id": "e2d314c8-687f-4b8d-869c-8d7f2a22a4f3", + "name": "agent.query_routing", + "start_time": "2025-10-14 12:53:44.917000000", + "end_time": "2025-10-14 12:53:49.957000000", + "duration_ms": 5039.404, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What's the current status of cryptocurrency markets?", + "query_length": 52, + "query_type": "routing_query", + "output_response": "Routed to Web Search for query processing", + "router.classification": "Web Search", + "router.confidence_score": 1, + "router.rag_keyword_score": 0, + "router.web_keyword_score": 3, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "d682797f-2ce0-448e-8f0d-891541430988", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-14 12:53:44.917000000", + "trace_end_time": "2025-10-14 12:53:49.957000000", + "trace_duration_ms": 5039.64, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-14 12:53:44.917000000", + "updated_at": "2025-10-14 12:53:44.917000000" + }, + { + "span_id": "987b6910-9b6d-4207-b0f3-8807a23e3ddb", + "trace_id": "d682797f-2ce0-448e-8f0d-891541430988", + "parent_span_id": "f137a412-6ddf-4ace-908e-9edf73534aa7", + "name": "agent.web_search_generation", + "start_time": "2025-10-14 12:53:44.918000000", + "end_time": "2025-10-14 12:53:49.957000000", + "duration_ms": 5039.248, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "web_search_agent", + "agent.operation": "web_search_generation", + "agent.capabilities": [ + "web_search", + "content_synthesis", + "response_generation" + ], + "agent.id": "noveum_web_search_agent", + "input_query": "What's the current status of cryptocurrency markets?", + "query_length": 52, + "query_type": "web_search_query", + "output_response": "Web Search Answer: The web search results did not provide any specific information regarding the current status of cryptocurrency markets. Therefore, I cannot provide an update on market conditions, prices, or trends at...", + "web_search.results_count": 1, + "web_search.context_length": 154, + "web_search.response_length": 338, + "web_search.mode": "external_web_search" + }, + "events": [], + "links": [], + "trace_trace_id": "d682797f-2ce0-448e-8f0d-891541430988", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-14 12:53:44.917000000", + "trace_end_time": "2025-10-14 12:53:49.957000000", + "trace_duration_ms": 5039.64, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-14 12:53:44.917000000", + "updated_at": "2025-10-14 12:53:44.917000000" + }, + { + "span_id": "b2614d0e-5fcd-46ec-8d25-f1b04eb142af", + "trace_id": "d682797f-2ce0-448e-8f0d-891541430988", + "parent_span_id": "987b6910-9b6d-4207-b0f3-8807a23e3ddb", + "name": "agent.llm_model_execution", + "start_time": "2025-10-14 12:53:45.255000000", + "end_time": "2025-10-14 12:53:49.957000000", + "duration_ms": 4701.744, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What's the current status of cryptocurrency markets?", + "query_length": 52, + "query_type": "web_search_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 4.701629161834717, + "model.latency_ms": 4701.629161834717, + "model.start_time": 1760446425.2554948, + "model.end_time": 1760446429.957124, + "model.performance_tier": "medium", + "model.prompt_tokens": 162, + "model.completion_tokens": 84, + "model.total_tokens": 246, + "model.tokens_per_second": 52.32228904756993, + "model.estimated_cost": 0.00738, + "model.efficiency_score": 1.3739837398373984, + "model.response_length": 338, + "model.response_quality": "high", + "model.output_response": "Model Response: The web search results did not provide any specific information regarding the current status of cryptocurrency markets. Therefore, I cannot provide an update on market conditions, prices, or trends at...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "d682797f-2ce0-448e-8f0d-891541430988", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-14 12:53:44.917000000", + "trace_end_time": "2025-10-14 12:53:49.957000000", + "trace_duration_ms": 5039.64, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-14 12:53:44.917000000", + "updated_at": "2025-10-14 12:53:44.917000000" + }, + { + "span_id": "663e8789-75d0-4c4d-b14d-4088022436ac", + "trace_id": "d682797f-2ce0-448e-8f0d-891541430988", + "parent_span_id": "987b6910-9b6d-4207-b0f3-8807a23e3ddb", + "name": "agent.web_search_evaluation_metrics", + "start_time": "2025-10-14 12:53:49.957000000", + "end_time": "2025-10-14 12:53:49.957000000", + "duration_ms": 0.032, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "web_search_evaluation_metrics", + "agent.capabilities": [ + "web_search_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What's the current status of cryptocurrency markets?", + "query_length": 52, + "query_type": "web_search_evaluation_query", + "web_search.results_count": 1, + "web_search.context_length": 154, + "web_search.context_synthesized": "Context from 1 web sources", + "web_search.search_effectiveness": 0.2, + "web_search.context_quality": "low", + "web_search.source_diversity": 1, + "web_search.information_synthesis": "low", + "web_search.external_knowledge_utilization": 0.154, + "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What's+the+current+status+of+cryptocurrency+markets?\n\nUser Question: What's the current status of cryptocurrency markets?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", + "prompt.prompt_length": 651, + "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What's+the+current+status+of+cryptocurrency+markets?", + "prompt.instruction_following": "web_search_optimized", + "response.answer_length": 338, + "response.answer_completeness": "complete", + "response.response_quality": "medium", + "response.source_citation": 1, + "response.context_utilization": 2.1948051948051948, + "output_response": "Web Search Answer: The web search results did not provide any specific information regarding the current status of cryptocurrency markets. Therefore, I cannot provide an update on market conditions, prices, or trends at...", + "evaluation.search_effectiveness": 0.2, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 1, + "evaluation.context_relevance": "low", + "evaluation.overall_quality": "low", + "evaluation.ready_for_production": true, + "web_search.search_strategy": "duckduckgo_api", + "web_search.real_time_data": true, + "web_search.external_sources": 1, + "web_search.information_freshness": "current", + "web_search.knowledge_synthesis": "single_source" + }, + "events": [], + "links": [], + "trace_trace_id": "d682797f-2ce0-448e-8f0d-891541430988", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-14 12:53:44.917000000", + "trace_end_time": "2025-10-14 12:53:49.957000000", + "trace_duration_ms": 5039.64, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-14 12:53:44.917000000", + "updated_at": "2025-10-14 12:53:44.917000000" + }, + { + "span_id": "3382fe7c-aaca-46e0-a8d2-dbee75ff3bed", + "trace_id": "b0858dca-4173-493b-a6c5-4e993f606477", + "parent_span_id": "6cf94049-87b0-4fc5-b37e-32d261543647", + "name": "agent.query_routing", + "start_time": "2025-10-14 12:53:37.818000000", + "end_time": "2025-10-14 12:53:43.908000000", + "duration_ms": 6089.22, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What are the latest updates in Python programming?", + "query_length": 50, + "query_type": "routing_query", + "output_response": "Routed to Web Search for query processing", + "router.classification": "Web Search", + "router.confidence_score": 1, + "router.rag_keyword_score": 0, + "router.web_keyword_score": 2, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "b0858dca-4173-493b-a6c5-4e993f606477", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-14 12:53:37.818000000", + "trace_end_time": "2025-10-14 12:53:43.908000000", + "trace_duration_ms": 6089.715, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-14 12:53:37.818000000", + "updated_at": "2025-10-14 12:53:37.818000000" + }, + { + "span_id": "6cf94049-87b0-4fc5-b37e-32d261543647", + "trace_id": "b0858dca-4173-493b-a6c5-4e993f606477", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-14 12:53:37.818000000", + "end_time": "2025-10-14 12:53:43.908000000", + "duration_ms": 6089.607, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What are the latest updates in Python programming?", + "query.length": 50, + "processing.mode": "Web Search", + "processing.time_seconds": 6.089300870895386, + "processing.response_length": 1324, + "processing.sources_count": 1, + "output_response": "Final Answer: The web search did not yield specific results regarding the latest updates in Python programming. However, I can provide a general overview based on my knowledge up to October 2023.\n\nAs of late 2023, ...", + "final_answer_mode": "Web Search", + "query_processed.input_query": "What are the latest updates in Python programming?", + "query_processed.output_response": "Successfully processed query using Web Search, generated 1324 character response", + "query_processed.mode": "Web Search", + "query_processed.processing_time": 6.089300870895386, + "query_processed.response_length": 1324 + }, + "events": [], + "links": [], + "trace_trace_id": "b0858dca-4173-493b-a6c5-4e993f606477", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-14 12:53:37.818000000", + "trace_end_time": "2025-10-14 12:53:43.908000000", + "trace_duration_ms": 6089.715, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-14 12:53:37.818000000", + "updated_at": "2025-10-14 12:53:37.818000000" + }, + { + "span_id": "21c8befa-356a-4034-8d62-792238cb9e6e", + "trace_id": "b0858dca-4173-493b-a6c5-4e993f606477", + "parent_span_id": "3382fe7c-aaca-46e0-a8d2-dbee75ff3bed", + "name": "agent.web_search_generation", + "start_time": "2025-10-14 12:53:37.819000000", + "end_time": "2025-10-14 12:53:43.908000000", + "duration_ms": 6088.807, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "web_search_agent", + "agent.operation": "web_search_generation", + "agent.capabilities": [ + "web_search", + "content_synthesis", + "response_generation" + ], + "agent.id": "noveum_web_search_agent", + "input_query": "What are the latest updates in Python programming?", + "query_length": 50, + "query_type": "web_search_query", + "output_response": "Web Search Answer: The web search did not yield specific results regarding the latest updates in Python programming. However, I can provide a general overview based on my knowledge up to October 2023.\n\nAs of late 2023, ...", + "web_search.results_count": 1, + "web_search.context_length": 152, + "web_search.response_length": 1324, + "web_search.mode": "external_web_search" + }, + "events": [], + "links": [], + "trace_trace_id": "b0858dca-4173-493b-a6c5-4e993f606477", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-14 12:53:37.818000000", + "trace_end_time": "2025-10-14 12:53:43.908000000", + "trace_duration_ms": 6089.715, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-14 12:53:37.818000000", + "updated_at": "2025-10-14 12:53:37.818000000" + }, + { + "span_id": "482feedf-52f4-4c6a-9186-d59898820caa", + "trace_id": "b0858dca-4173-493b-a6c5-4e993f606477", + "parent_span_id": "3382fe7c-aaca-46e0-a8d2-dbee75ff3bed", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-14 12:53:37.819000000", + "end_time": "2025-10-14 12:53:37.819000000", + "duration_ms": 0.039, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What are the latest updates in Python programming?", + "query_length": 50, + "query_type": "routing_evaluation_query", + "classification.mode": "Web Search", + "classification.rag_keyword_score": 0, + "classification.web_keyword_score": 2, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "general_knowledge", + "query.keyword_density": 0.25, + "query.domain_affinity": "general", + "routing.decision": "Routed to Web Search based on analysis", + "routing.rationale": "RAG score: 0, Web score: 2, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "RAG", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.038461538461538464, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: Web Search (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "b0858dca-4173-493b-a6c5-4e993f606477", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-14 12:53:37.818000000", + "trace_end_time": "2025-10-14 12:53:43.908000000", + "trace_duration_ms": 6089.715, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-14 12:53:37.818000000", + "updated_at": "2025-10-14 12:53:37.818000000" + }, + { + "span_id": "84f1f63f-ee05-411e-a7dd-4ee9e9826f5a", + "trace_id": "b0858dca-4173-493b-a6c5-4e993f606477", + "parent_span_id": "21c8befa-356a-4034-8d62-792238cb9e6e", + "name": "agent.llm_model_execution", + "start_time": "2025-10-14 12:53:38.191000000", + "end_time": "2025-10-14 12:53:43.907000000", + "duration_ms": 5716.078, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What are the latest updates in Python programming?", + "query_length": 50, + "query_type": "web_search_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 5.715963125228882, + "model.latency_ms": 5715.963125228882, + "model.start_time": 1760446418.191805, + "model.end_time": 1760446423.907768, + "model.performance_tier": "slow", + "model.prompt_tokens": 161, + "model.completion_tokens": 331, + "model.total_tokens": 492, + "model.tokens_per_second": 86.07473302765561, + "model.estimated_cost": 0.01476, + "model.efficiency_score": 2.6910569105691056, + "model.response_length": 1324, + "model.response_quality": "high", + "model.output_response": "Model Response: The web search did not yield specific results regarding the latest updates in Python programming. However, I can provide a general overview based on my knowledge up to October 2023.\n\nAs of late 2023, ...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "b0858dca-4173-493b-a6c5-4e993f606477", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-14 12:53:37.818000000", + "trace_end_time": "2025-10-14 12:53:43.908000000", + "trace_duration_ms": 6089.715, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-14 12:53:37.818000000", + "updated_at": "2025-10-14 12:53:37.818000000" + }, + { + "span_id": "2fb9ecf3-a497-4285-a694-1c95080d9e6a", + "trace_id": "b0858dca-4173-493b-a6c5-4e993f606477", + "parent_span_id": "21c8befa-356a-4034-8d62-792238cb9e6e", + "name": "agent.web_search_evaluation_metrics", + "start_time": "2025-10-14 12:53:43.908000000", + "end_time": "2025-10-14 12:53:43.908000000", + "duration_ms": 0.037, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "web_search_evaluation_metrics", + "agent.capabilities": [ + "web_search_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What are the latest updates in Python programming?", + "query_length": 50, + "query_type": "web_search_evaluation_query", + "web_search.results_count": 1, + "web_search.context_length": 152, + "web_search.context_synthesized": "Context from 1 web sources", + "web_search.search_effectiveness": 0.2, + "web_search.context_quality": "low", + "web_search.source_diversity": 1, + "web_search.information_synthesis": "low", + "web_search.external_knowledge_utilization": 0.152, + "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What+are+the+latest+updates+in+Python+programming?\n\nUser Question: What are the latest updates in Python programming?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", + "prompt.prompt_length": 647, + "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What+are+the+latest+updates+in+Python+programming?", + "prompt.instruction_following": "web_search_optimized", + "response.answer_length": 1324, + "response.answer_completeness": "complete", + "response.response_quality": "medium", + "response.source_citation": 1, + "response.context_utilization": 8.710526315789474, + "output_response": "Web Search Answer: The web search did not yield specific results regarding the latest updates in Python programming. However, I can provide a general overview based on my knowledge up to October 2023.\n\nAs of late 2023, ...", + "evaluation.search_effectiveness": 0.2, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 1, + "evaluation.context_relevance": "low", + "evaluation.overall_quality": "low", + "evaluation.ready_for_production": true, + "web_search.search_strategy": "duckduckgo_api", + "web_search.real_time_data": true, + "web_search.external_sources": 1, + "web_search.information_freshness": "current", + "web_search.knowledge_synthesis": "single_source" + }, + "events": [], + "links": [], + "trace_trace_id": "b0858dca-4173-493b-a6c5-4e993f606477", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-14 12:53:37.818000000", + "trace_end_time": "2025-10-14 12:53:43.908000000", + "trace_duration_ms": 6089.715, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-14 12:53:37.818000000", + "updated_at": "2025-10-14 12:53:37.818000000" + }, + { + "span_id": "9dd5ac2c-0deb-48e3-ac8b-0e6e31f7ff30", + "trace_id": "dc81be1a-e93d-4f60-90f3-ce1acaa8cb3e", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-14 12:53:34.584000000", + "end_time": "2025-10-14 12:53:36.780000000", + "duration_ms": 2196.351, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What happened in tech news this week?", + "query.length": 37, + "processing.mode": "Web Search", + "processing.time_seconds": 2.1948978900909424, + "processing.response_length": 324, + "processing.sources_count": 1, + "output_response": "Final Answer: I'm sorry, but I couldn't find any specific information regarding tech news from this week based on the search results. It appears that there were no relevant articles or updates available. If you're ...", + "final_answer_mode": "Web Search", + "query_processed.input_query": "What happened in tech news this week?", + "query_processed.output_response": "Successfully processed query using Web Search, generated 324 character response", + "query_processed.mode": "Web Search", + "query_processed.processing_time": 2.1948978900909424, + "query_processed.response_length": 324 + }, + "events": [], + "links": [], + "trace_trace_id": "dc81be1a-e93d-4f60-90f3-ce1acaa8cb3e", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-14 12:53:34.584000000", + "trace_end_time": "2025-10-14 12:53:36.780000000", + "trace_duration_ms": 2196.4579999999996, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-14 12:53:34.584000000", + "updated_at": "2025-10-14 12:53:34.584000000" + }, + { + "span_id": "b1cbafce-51fb-48fd-bc4a-d49081a2f374", + "trace_id": "dc81be1a-e93d-4f60-90f3-ce1acaa8cb3e", + "parent_span_id": "9dd5ac2c-0deb-48e3-ac8b-0e6e31f7ff30", + "name": "agent.query_routing", + "start_time": "2025-10-14 12:53:34.584000000", + "end_time": "2025-10-14 12:53:36.778000000", + "duration_ms": 2194.2200000000003, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What happened in tech news this week?", + "query_length": 37, + "query_type": "routing_query", + "output_response": "Routed to Web Search for query processing", + "router.classification": "Web Search", + "router.confidence_score": 1, + "router.rag_keyword_score": 0, + "router.web_keyword_score": 2, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "dc81be1a-e93d-4f60-90f3-ce1acaa8cb3e", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-14 12:53:34.584000000", + "trace_end_time": "2025-10-14 12:53:36.780000000", + "trace_duration_ms": 2196.4579999999996, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-14 12:53:34.584000000", + "updated_at": "2025-10-14 12:53:34.584000000" + }, + { + "span_id": "e3064687-1b18-478b-9e9d-4d081a6dac23", + "trace_id": "dc81be1a-e93d-4f60-90f3-ce1acaa8cb3e", + "parent_span_id": "b1cbafce-51fb-48fd-bc4a-d49081a2f374", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-14 12:53:34.584000000", + "end_time": "2025-10-14 12:53:34.584000000", + "duration_ms": 0.028, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What happened in tech news this week?", + "query_length": 37, + "query_type": "routing_evaluation_query", + "classification.mode": "Web Search", + "classification.rag_keyword_score": 0, + "classification.web_keyword_score": 2, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "general_knowledge", + "query.keyword_density": 0.2857142857142857, + "query.domain_affinity": "general", + "routing.decision": "Routed to Web Search based on analysis", + "routing.rationale": "RAG score: 0, Web score: 2, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "RAG", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.038461538461538464, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: Web Search (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "dc81be1a-e93d-4f60-90f3-ce1acaa8cb3e", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-14 12:53:34.584000000", + "trace_end_time": "2025-10-14 12:53:36.780000000", + "trace_duration_ms": 2196.4579999999996, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-14 12:53:34.584000000", + "updated_at": "2025-10-14 12:53:34.584000000" + }, + { + "span_id": "eb9dbc11-44b3-4041-8c83-0f33b8b761bb", + "trace_id": "dc81be1a-e93d-4f60-90f3-ce1acaa8cb3e", + "parent_span_id": "b1cbafce-51fb-48fd-bc4a-d49081a2f374", + "name": "agent.web_search_generation", + "start_time": "2025-10-14 12:53:34.584000000", + "end_time": "2025-10-14 12:53:36.774000000", + "duration_ms": 2189.866, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "web_search_agent", + "agent.operation": "web_search_generation", + "agent.capabilities": [ + "web_search", + "content_synthesis", + "response_generation" + ], + "agent.id": "noveum_web_search_agent", + "input_query": "What happened in tech news this week?", + "query_length": 37, + "query_type": "web_search_query", + "output_response": "Web Search Answer: I'm sorry, but I couldn't find any specific information regarding tech news from this week based on the search results. It appears that there were no relevant articles or updates available. If you're ...", + "web_search.results_count": 1, + "web_search.context_length": 139, + "web_search.response_length": 324, + "web_search.mode": "external_web_search" + }, + "events": [], + "links": [], + "trace_trace_id": "dc81be1a-e93d-4f60-90f3-ce1acaa8cb3e", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-14 12:53:34.584000000", + "trace_end_time": "2025-10-14 12:53:36.780000000", + "trace_duration_ms": 2196.4579999999996, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-14 12:53:34.584000000", + "updated_at": "2025-10-14 12:53:34.584000000" + }, + { + "span_id": "c8ee237b-ab5e-4b36-83f0-2f8478971898", + "trace_id": "dc81be1a-e93d-4f60-90f3-ce1acaa8cb3e", + "parent_span_id": "eb9dbc11-44b3-4041-8c83-0f33b8b761bb", + "name": "agent.llm_model_execution", + "start_time": "2025-10-14 12:53:34.981000000", + "end_time": "2025-10-14 12:53:36.768000000", + "duration_ms": 1786.467, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What happened in tech news this week?", + "query_length": 37, + "query_type": "web_search_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 1.785179853439331, + "model.latency_ms": 1785.179853439331, + "model.start_time": 1760446414.981964, + "model.end_time": 1760446416.767144, + "model.performance_tier": "fast", + "model.prompt_tokens": 155, + "model.completion_tokens": 81, + "model.total_tokens": 236, + "model.tokens_per_second": 132.19956495997977, + "model.estimated_cost": 0.00708, + "model.efficiency_score": 1.3728813559322033, + "model.response_length": 324, + "model.response_quality": "high", + "model.output_response": "Model Response: I'm sorry, but I couldn't find any specific information regarding tech news from this week based on the search results. It appears that there were no relevant articles or updates available. If you're ...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "dc81be1a-e93d-4f60-90f3-ce1acaa8cb3e", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-14 12:53:34.584000000", + "trace_end_time": "2025-10-14 12:53:36.780000000", + "trace_duration_ms": 2196.4579999999996, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-14 12:53:34.584000000", + "updated_at": "2025-10-14 12:53:34.584000000" + }, + { + "span_id": "9c8e1a08-61c8-409e-a79d-41c74a8d1505", + "trace_id": "dc81be1a-e93d-4f60-90f3-ce1acaa8cb3e", + "parent_span_id": "eb9dbc11-44b3-4041-8c83-0f33b8b761bb", + "name": "agent.web_search_evaluation_metrics", + "start_time": "2025-10-14 12:53:36.772000000", + "end_time": "2025-10-14 12:53:36.774000000", + "duration_ms": 1.5939999999999999, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "web_search_evaluation_metrics", + "agent.capabilities": [ + "web_search_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What happened in tech news this week?", + "query_length": 37, + "query_type": "web_search_evaluation_query", + "web_search.results_count": 1, + "web_search.context_length": 139, + "web_search.context_synthesized": "Context from 1 web sources", + "web_search.search_effectiveness": 0.2, + "web_search.context_quality": "low", + "web_search.source_diversity": 1, + "web_search.information_synthesis": "low", + "web_search.external_knowledge_utilization": 0.139, + "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What+happened+in+tech+news+this+week?\n\nUser Question: What happened in tech news this week?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", + "prompt.prompt_length": 621, + "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What+happened+in+tech+news+this+week?", + "prompt.instruction_following": "web_search_optimized", + "response.answer_length": 324, + "response.answer_completeness": "complete", + "response.response_quality": "medium", + "response.source_citation": 1, + "response.context_utilization": 2.3309352517985613, + "output_response": "Web Search Answer: I'm sorry, but I couldn't find any specific information regarding tech news from this week based on the search results. It appears that there were no relevant articles or updates available. If you're ...", + "evaluation.search_effectiveness": 0.2, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 1, + "evaluation.context_relevance": "low", + "evaluation.overall_quality": "low", + "evaluation.ready_for_production": true, + "web_search.search_strategy": "duckduckgo_api", + "web_search.real_time_data": true, + "web_search.external_sources": 1, + "web_search.information_freshness": "current", + "web_search.knowledge_synthesis": "single_source" + }, + "events": [], + "links": [], + "trace_trace_id": "dc81be1a-e93d-4f60-90f3-ce1acaa8cb3e", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-14 12:53:34.584000000", + "trace_end_time": "2025-10-14 12:53:36.780000000", + "trace_duration_ms": 2196.4579999999996, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-14 12:53:34.584000000", + "updated_at": "2025-10-14 12:53:34.584000000" + }, + { + "span_id": "59595232-8873-419c-9b6a-0727445a2afa", + "trace_id": "945b2757-8a5a-4aab-a869-8f44c02166a5", + "parent_span_id": "dfde2f79-d0c1-481e-819d-60730cb60fe5", + "name": "agent.query_routing", + "start_time": "2025-10-14 12:53:27.771000000", + "end_time": "2025-10-14 12:53:33.575000000", + "duration_ms": 5803.9800000000005, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What are the current trends in observability tools?", + "query_length": 51, + "query_type": "routing_query", + "output_response": "Routed to RAG for query processing", + "router.classification": "RAG", + "router.confidence_score": 0.3333333333333333, + "router.rag_keyword_score": 2, + "router.web_keyword_score": 1, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "945b2757-8a5a-4aab-a869-8f44c02166a5", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-14 12:53:27.771000000", + "trace_end_time": "2025-10-14 12:53:33.575000000", + "trace_duration_ms": 5804.15, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-14 12:53:27.771000000", + "updated_at": "2025-10-14 12:53:27.771000000" + }, + { + "span_id": "a59b3296-aaba-4642-b5a0-a271787a5607", + "trace_id": "945b2757-8a5a-4aab-a869-8f44c02166a5", + "parent_span_id": "59595232-8873-419c-9b6a-0727445a2afa", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-14 12:53:27.771000000", + "end_time": "2025-10-14 12:53:27.771000000", + "duration_ms": 0.031, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What are the current trends in observability tools?", + "query_length": 51, + "query_type": "routing_evaluation_query", + "classification.mode": "RAG", + "classification.rag_keyword_score": 2, + "classification.web_keyword_score": 1, + "classification.confidence_score": 0.3333333333333333, + "classification.confidence_level": "medium", + "classification.method": "keyword_based", + "query.complexity": "complex", + "query.intent": "documentation", + "query.keyword_density": 0.375, + "query.domain_affinity": "noveum", + "routing.decision": "Routed to RAG based on analysis", + "routing.rationale": "RAG score: 2, Web score: 1, Confidence: 0.33", + "routing.expected_performance": "medium", + "routing.alternative_mode": "Web Search", + "routing.decision_confidence": 0.3333333333333333, + "evaluation.routing_accuracy": "medium", + "evaluation.keyword_coverage": 0.057692307692307696, + "evaluation.query_understanding": "ambiguous", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: RAG (Confidence: 0.33)" + }, + "events": [], + "links": [], + "trace_trace_id": "945b2757-8a5a-4aab-a869-8f44c02166a5", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-14 12:53:27.771000000", + "trace_end_time": "2025-10-14 12:53:33.575000000", + "trace_duration_ms": 5804.15, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-14 12:53:27.771000000", + "updated_at": "2025-10-14 12:53:27.771000000" + }, + { + "span_id": "dfde2f79-d0c1-481e-819d-60730cb60fe5", + "trace_id": "945b2757-8a5a-4aab-a869-8f44c02166a5", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-14 12:53:27.771000000", + "end_time": "2025-10-14 12:53:33.575000000", + "duration_ms": 5804.074, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What are the current trends in observability tools?", + "query.length": 51, + "processing.mode": "RAG", + "processing.time_seconds": 5.804032802581787, + "processing.response_length": 997, + "processing.sources_count": 5, + "output_response": "Final Answer: The provided context does not specifically outline current trends in observability tools in general. However, it highlights the unique challenges faced by AI applications and how traditional monitorin...", + "final_answer_mode": "RAG", + "query_processed.input_query": "What are the current trends in observability tools?", + "query_processed.output_response": "Successfully processed query using RAG, generated 997 character response", + "query_processed.mode": "RAG", + "query_processed.processing_time": 5.804032802581787, + "query_processed.response_length": 997 + }, + "events": [], + "links": [], + "trace_trace_id": "945b2757-8a5a-4aab-a869-8f44c02166a5", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-14 12:53:27.771000000", + "trace_end_time": "2025-10-14 12:53:33.575000000", + "trace_duration_ms": 5804.15, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-14 12:53:27.771000000", + "updated_at": "2025-10-14 12:53:27.771000000" + }, + { + "span_id": "82489444-deba-46d2-b08b-c09754189566", + "trace_id": "945b2757-8a5a-4aab-a869-8f44c02166a5", + "parent_span_id": "59595232-8873-419c-9b6a-0727445a2afa", + "name": "agent.llm-rag", + "start_time": "2025-10-14 12:53:27.772000000", + "end_time": "2025-10-14 12:53:33.575000000", + "duration_ms": 5803.534, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "rag_agent", + "agent.operation": "llm-rag", + "agent.capabilities": [ + "document_retrieval", + "context_generation", + "response_generation" + ], + "agent.id": "noveum_rag_agent", + "input_query": "What are the current trends in observability tools?", + "query_length": 51, + "query_type": "rag_query", + "output_response": "RAG Answer: The provided context does not specifically outline current trends in observability tools in general. However, it highlights the unique challenges faced by AI applications and how traditional monitorin...", + "rag.context_length": 2943, + "rag.sources_count": 5, + "rag.answer_length": 997, + "rag.mode": "retrieval_augmented_generation" + }, + "events": [], + "links": [], + "trace_trace_id": "945b2757-8a5a-4aab-a869-8f44c02166a5", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-14 12:53:27.771000000", + "trace_end_time": "2025-10-14 12:53:33.575000000", + "trace_duration_ms": 5804.15, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-14 12:53:27.771000000", + "updated_at": "2025-10-14 12:53:27.771000000" + }, + { + "span_id": "34b1f14c-3fd3-4f89-b4dd-6150a52befd7", + "trace_id": "945b2757-8a5a-4aab-a869-8f44c02166a5", + "parent_span_id": "82489444-deba-46d2-b08b-c09754189566", + "name": "agent.llm_model_execution", + "start_time": "2025-10-14 12:53:28.257000000", + "end_time": "2025-10-14 12:53:33.162000000", + "duration_ms": 4905.2429999999995, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What are the current trends in observability tools?", + "query_length": 51, + "query_type": "rag_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 4.9052040576934814, + "model.latency_ms": 4905.204057693481, + "model.start_time": 1760446408.2575018, + "model.end_time": 1760446413.162706, + "model.performance_tier": "medium", + "model.prompt_tokens": 872, + "model.completion_tokens": 249, + "model.total_tokens": 1121, + "model.tokens_per_second": 228.53279635569638, + "model.estimated_cost": 0.03363, + "model.efficiency_score": 0.8893844781445138, + "model.response_length": 997, + "model.response_quality": "high", + "model.output_response": "Model Response: The provided context does not specifically outline current trends in observability tools in general. However, it highlights the unique challenges faced by AI applications and how traditional monitorin...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "945b2757-8a5a-4aab-a869-8f44c02166a5", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-14 12:53:27.771000000", + "trace_end_time": "2025-10-14 12:53:33.575000000", + "trace_duration_ms": 5804.15, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-14 12:53:27.771000000", + "updated_at": "2025-10-14 12:53:27.771000000" + }, + { + "span_id": "d5d2b966-3c37-4d6d-b7b8-c58c9f22aad9", + "trace_id": "945b2757-8a5a-4aab-a869-8f44c02166a5", + "parent_span_id": "82489444-deba-46d2-b08b-c09754189566", + "name": "agent.rag_evaluation_metrics", + "start_time": "2025-10-14 12:53:33.162000000", + "end_time": "2025-10-14 12:53:33.162000000", + "duration_ms": 0.036000000000000004, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "rag_evaluation_metrics", + "agent.capabilities": [ + "retrieval_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What are the current trends in observability tools?", + "query_length": 51, + "query_type": "rag_evaluation_query", + "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/blog/from-logs-to-intelligent-choices-inside-noveum-ais-evaluation-process):\nFrom Development to Production - Inside Noveum.ai's AI Observability Platform\nShashank Agarwal\n3/3/2025\nIntroduction\nEvery AI application tells a story\u2014but most developers never get to hear it...", + "retrieval.context_length": 2943, + "retrieval.sources_count": 5, + "retrieval.context_quality": "high", + "retrieval.effectiveness": 1, + "retrieval.context_utilization": 2.943, + "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/blog/from-logs-to-intelligent-choices-inside-noveum-ais-evaluation-process):\nFrom Development to Production - Inside Noveum.ai's AI Observability Platform\nShashank Agarwal\n3/3/2025\nIntroduction\nEvery AI application tells a story\u2014but most developers never get to hear it. When your RAG pipeline returns irrelevant results, when your multi-agent system gets stuck in loops, or when your LLM costs suddenly spike, you're left guessing what went wrong and where.\nThat's exactly the challenge Noveum.ai solves. Rather than flying blind, our platform provides comprehensive tracing a...\n\nSource 2 (https://noveum.ai/docs/getting-started/tracing-concepts):\n- Explore Framework Integrations - Framework-specific guidance\n- Learn Advanced Patterns - Custom instrumentation techniques\n- Master the Dashboard - Analyze your traces effectively\nRemember: Good observability is not about collecting all possible data, but about collecting the right data that helps you understand, debug, and optimize your AI applications.\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to O...\n\nSource 3 (https://noveum.ai/en/docs/getting-started/tracing-concepts):\n- Explore Framework Integrations - Framework-specific guidance\n- Learn Advanced Patterns - Custom instrumentation techniques\n- Master the Dashboard - Analyze your traces effectively\nRemember: Good observability is not about collecting all possible data, but about collecting the right data that helps you understand, debug, and optimize your AI applications.\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to O...\n\nSource 4 (https://noveum.ai/en/blog/from-logs-to-intelligent-choices-inside-noveum-ais-evaluation-process):\nThe AI Observability Challenge\nWhy Traditional Monitoring Falls Short\nTraditional application monitoring tools weren't built for AI applications. They can tell you if your API is responding, but they can't answer the questions that matter most for AI systems:\n- Why did my RAG pipeline retrieve irrelevant documents?\n- Which LLM calls are driving my costs?\n- How are my agents communicating with each other?\n- What's causing hallucinations in my responses?\n- Why is my embedding generation so slow?\nW...\n\nSource 5 (https://noveum.ai/en/docs/advanced/rag-observability):\n\ud83d\udd27 Best Practices for RAG Observability\n1. Comprehensive Pipeline Tracing\n2. Quality Monitoring\n3. A/B Testing for RAG Components\n\ud83c\udfaf Advanced RAG Patterns\nMulti-Modal RAG\nConversational RAG\nRAG observability with Noveum.ai provides the deep insights needed to build, optimize, and scale retrieval-augmented generation systems. By implementing comprehensive tracing across retrieval, context preparation, and generation stages, you can ensure your RAG pipeline delivers accurate, relevant, and cost-effe...\n\nUser Question: What are the current trends in observability tools?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", + "prompt.prompt_length": 3490, + "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/blog/from-logs-to-intelligent-choices-inside-noveum-ais-evaluation-process):\nFrom Development to Production - Inside Noveum.ai's AI Observability Platform\nShashank Agarw...", + "prompt.instruction_following": "rag_optimized", + "response.answer_length": 997, + "response.answer_completeness": "complete", + "response.response_quality": "high", + "response.source_citation": 5, + "response.context_utilization": 0.33876996262317366, + "output_response": "RAG Answer: The provided context does not specifically outline current trends in observability tools in general. However, it highlights the unique challenges faced by AI applications and how traditional monitorin...", + "evaluation.retrieval_effectiveness": 1, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 5, + "evaluation.context_relevance": "high", + "evaluation.overall_quality": "high", + "evaluation.ready_for_production": true, + "rag.retrieval_strategy": "semantic_similarity", + "rag.vector_search_results": 5, + "rag.context_synthesis": "multi_source", + "rag.document_coverage": 1, + "rag.information_density": 0.33876996262317366 + }, + "events": [], + "links": [], + "trace_trace_id": "945b2757-8a5a-4aab-a869-8f44c02166a5", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-14 12:53:27.771000000", + "trace_end_time": "2025-10-14 12:53:33.575000000", + "trace_duration_ms": 5804.15, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-14 12:53:27.771000000", + "updated_at": "2025-10-14 12:53:27.771000000" + }, + { + "span_id": "11c07411-df30-42a4-9062-90a032ef75ed", + "trace_id": "4bdd5814-acbb-4de7-a56b-543b87a1cfa5", + "parent_span_id": "ec530838-a7cf-477b-9186-a70114c8a12f", + "name": "agent.web_search_generation", + "start_time": "2025-10-14 12:53:20.120000000", + "end_time": "2025-10-14 12:53:26.750000000", + "duration_ms": 6629.273999999999, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "web_search_agent", + "agent.operation": "web_search_generation", + "agent.capabilities": [ + "web_search", + "content_synthesis", + "response_generation" + ], + "agent.id": "noveum_web_search_agent", + "input_query": "Tell me about recent developments in machine learning", + "query_length": 53, + "query_type": "web_search_query", + "output_response": "Web Search Answer: The web search did not yield specific results regarding recent developments in machine learning. However, I can provide a general overview based on my knowledge up to October 2023.\n\nRecent development...", + "web_search.results_count": 1, + "web_search.context_length": 155, + "web_search.response_length": 1849, + "web_search.mode": "external_web_search" + }, + "events": [], + "links": [], + "trace_trace_id": "4bdd5814-acbb-4de7-a56b-543b87a1cfa5", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-14 12:53:20.120000000", + "trace_end_time": "2025-10-14 12:53:26.750000000", + "trace_duration_ms": 6630.38, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-14 12:53:20.120000000", + "updated_at": "2025-10-14 12:53:20.120000000" + }, + { + "span_id": "282b121f-61dd-4dc5-9e55-cecc3f02f688", + "trace_id": "4bdd5814-acbb-4de7-a56b-543b87a1cfa5", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-14 12:53:20.120000000", + "end_time": "2025-10-14 12:53:26.750000000", + "duration_ms": 6630.264, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "Tell me about recent developments in machine learning", + "query.length": 53, + "processing.mode": "Web Search", + "processing.time_seconds": 6.629686117172241, + "processing.response_length": 1849, + "processing.sources_count": 1, + "output_response": "Final Answer: The web search did not yield specific results regarding recent developments in machine learning. However, I can provide a general overview based on my knowledge up to October 2023.\n\nRecent development...", + "final_answer_mode": "Web Search", + "query_processed.input_query": "Tell me about recent developments in machine learning", + "query_processed.output_response": "Successfully processed query using Web Search, generated 1849 character response", + "query_processed.mode": "Web Search", + "query_processed.processing_time": 6.629686117172241, + "query_processed.response_length": 1849 + }, + "events": [], + "links": [], + "trace_trace_id": "4bdd5814-acbb-4de7-a56b-543b87a1cfa5", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-14 12:53:20.120000000", + "trace_end_time": "2025-10-14 12:53:26.750000000", + "trace_duration_ms": 6630.38, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-14 12:53:20.120000000", + "updated_at": "2025-10-14 12:53:20.120000000" + }, + { + "span_id": "ec530838-a7cf-477b-9186-a70114c8a12f", + "trace_id": "4bdd5814-acbb-4de7-a56b-543b87a1cfa5", + "parent_span_id": "282b121f-61dd-4dc5-9e55-cecc3f02f688", + "name": "agent.query_routing", + "start_time": "2025-10-14 12:53:20.120000000", + "end_time": "2025-10-14 12:53:26.750000000", + "duration_ms": 6629.59, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "Tell me about recent developments in machine learning", + "query_length": 53, + "query_type": "routing_query", + "output_response": "Routed to Web Search for query processing", + "router.classification": "Web Search", + "router.confidence_score": 1, + "router.rag_keyword_score": 0, + "router.web_keyword_score": 1, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [], + "trace_trace_id": "4bdd5814-acbb-4de7-a56b-543b87a1cfa5", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-14 12:53:20.120000000", + "trace_end_time": "2025-10-14 12:53:26.750000000", + "trace_duration_ms": 6630.38, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-14 12:53:20.120000000", + "updated_at": "2025-10-14 12:53:20.120000000" + }, + { + "span_id": "df882f90-cf1a-42a6-a2ae-78b2a13f27a1", + "trace_id": "4bdd5814-acbb-4de7-a56b-543b87a1cfa5", + "parent_span_id": "ec530838-a7cf-477b-9186-a70114c8a12f", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-14 12:53:20.120000000", + "end_time": "2025-10-14 12:53:20.120000000", + "duration_ms": 0.082, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: Tell me about recent developments in machine learning", + "query_length": 53, + "query_type": "routing_evaluation_query", + "classification.mode": "Web Search", + "classification.rag_keyword_score": 0, + "classification.web_keyword_score": 1, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "complex", + "query.intent": "general_knowledge", + "query.keyword_density": 0.125, + "query.domain_affinity": "general", + "routing.decision": "Routed to Web Search based on analysis", + "routing.rationale": "RAG score: 0, Web score: 1, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "RAG", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.019230769230769232, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: Web Search (Confidence: 1.00)" + }, + "events": [], + "links": [], + "trace_trace_id": "4bdd5814-acbb-4de7-a56b-543b87a1cfa5", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-14 12:53:20.120000000", + "trace_end_time": "2025-10-14 12:53:26.750000000", + "trace_duration_ms": 6630.38, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-14 12:53:20.120000000", + "updated_at": "2025-10-14 12:53:20.120000000" + }, + { + "span_id": "bea819fd-3673-480b-9f7b-a3a2efef4218", + "trace_id": "4bdd5814-acbb-4de7-a56b-543b87a1cfa5", + "parent_span_id": "11c07411-df30-42a4-9062-90a032ef75ed", + "name": "agent.llm_model_execution", + "start_time": "2025-10-14 12:53:20.455000000", + "end_time": "2025-10-14 12:53:26.746000000", + "duration_ms": 6291.401, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: Tell me about recent developments in machine learning", + "query_length": 53, + "query_type": "web_search_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 6.290812253952026, + "model.latency_ms": 6290.812253952026, + "model.start_time": 1760446400.4555368, + "model.end_time": 1760446406.746349, + "model.performance_tier": "slow", + "model.prompt_tokens": 163, + "model.completion_tokens": 462, + "model.total_tokens": 625, + "model.tokens_per_second": 99.35124031199013, + "model.estimated_cost": 0.01875, + "model.efficiency_score": 2.9584, + "model.response_length": 1849, + "model.response_quality": "high", + "model.output_response": "Model Response: The web search did not yield specific results regarding recent developments in machine learning. However, I can provide a general overview based on my knowledge up to October 2023.\n\nRecent development...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [], + "trace_trace_id": "4bdd5814-acbb-4de7-a56b-543b87a1cfa5", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-14 12:53:20.120000000", + "trace_end_time": "2025-10-14 12:53:26.750000000", + "trace_duration_ms": 6630.38, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-14 12:53:20.120000000", + "updated_at": "2025-10-14 12:53:20.120000000" + }, + { + "span_id": "6e62af18-569c-4783-b2ae-177c58542bd0", + "trace_id": "4bdd5814-acbb-4de7-a56b-543b87a1cfa5", + "parent_span_id": "11c07411-df30-42a4-9062-90a032ef75ed", + "name": "agent.web_search_evaluation_metrics", + "start_time": "2025-10-14 12:53:26.749000000", + "end_time": "2025-10-14 12:53:26.750000000", + "duration_ms": 0.217, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "web_search_evaluation_metrics", + "agent.capabilities": [ + "web_search_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: Tell me about recent developments in machine learning", + "query_length": 53, + "query_type": "web_search_evaluation_query", + "web_search.results_count": 1, + "web_search.context_length": 155, + "web_search.context_synthesized": "Context from 1 web sources", + "web_search.search_effectiveness": 0.2, + "web_search.context_quality": "low", + "web_search.source_diversity": 1, + "web_search.information_synthesis": "low", + "web_search.external_knowledge_utilization": 0.155, + "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=Tell+me+about+recent+developments+in+machine+learning\n\nUser Question: Tell me about recent developments in machine learning\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", + "prompt.prompt_length": 653, + "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=Tell+me+about+recent+developments+in+machine+learning", + "prompt.instruction_following": "web_search_optimized", + "response.answer_length": 1849, + "response.answer_completeness": "complete", + "response.response_quality": "medium", + "response.source_citation": 1, + "response.context_utilization": 11.929032258064517, + "output_response": "Web Search Answer: The web search did not yield specific results regarding recent developments in machine learning. However, I can provide a general overview based on my knowledge up to October 2023.\n\nRecent development...", + "evaluation.search_effectiveness": 0.2, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 1, + "evaluation.context_relevance": "low", + "evaluation.overall_quality": "low", + "evaluation.ready_for_production": true, + "web_search.search_strategy": "duckduckgo_api", + "web_search.real_time_data": true, + "web_search.external_sources": 1, + "web_search.information_freshness": "current", + "web_search.knowledge_synthesis": "single_source" + }, + "events": [], + "links": [], + "trace_trace_id": "4bdd5814-acbb-4de7-a56b-543b87a1cfa5", + "trace_name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "trace_status": "ok", + "trace_status_message": "", + "trace_start_time": "2025-10-14 12:53:20.120000000", + "trace_end_time": "2025-10-14 12:53:26.750000000", + "trace_duration_ms": 6630.38, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "trace_attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "created_at": "2025-10-14 12:53:20.120000000", + "updated_at": "2025-10-14 12:53:20.120000000" + } +] \ No newline at end of file diff --git a/noveum_customer_support_bt/traces/traces/traces_batch_001.json b/noveum_customer_support_bt/traces/traces/traces_batch_001.json new file mode 100644 index 0000000..c0b74d2 --- /dev/null +++ b/noveum_customer_support_bt/traces/traces/traces_batch_001.json @@ -0,0 +1,13915 @@ +{ + "success": true, + "traces": [ + { + "trace_id": "6fe93abf-1413-49e8-8464-58558e25ab7b", + "name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "status": "ok", + "status_message": "", + "start_time": "2025-10-22 16:58:42.084000000", + "end_time": "2025-10-22 16:58:44.766000000", + "duration_ms": 2681.9289999999996, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "spans": [ + { + "span_id": "4149abdc-acdc-448c-b9b0-b09e166e461a", + "trace_id": "6fe93abf-1413-49e8-8464-58558e25ab7b", + "parent_span_id": "817c9703-0062-4bd6-b3ee-b50a1121c2d2", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-22 16:58:42.084000000", + "end_time": "2025-10-22 16:58:42.084000000", + "duration_ms": 0.015000000000000001, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What are the latest breakthroughs in artificial intelligence?", + "query_length": 61, + "query_type": "routing_evaluation_query", + "classification.mode": "Web Search", + "classification.rag_keyword_score": 0, + "classification.web_keyword_score": 1, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "complex", + "query.intent": "general_knowledge", + "query.keyword_density": 0.125, + "query.domain_affinity": "general", + "routing.decision": "Routed to Web Search based on analysis", + "routing.rationale": "RAG score: 0, Web score: 1, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "RAG", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.019230769230769232, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: Web Search (Confidence: 1.00)" + }, + "events": [], + "links": [] + }, + { + "span_id": "817c9703-0062-4bd6-b3ee-b50a1121c2d2", + "trace_id": "6fe93abf-1413-49e8-8464-58558e25ab7b", + "parent_span_id": "ff171ffc-dc6b-4cb4-9833-64c5622d9941", + "name": "agent.query_routing", + "start_time": "2025-10-22 16:58:42.084000000", + "end_time": "2025-10-22 16:58:44.766000000", + "duration_ms": 2681.799, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What are the latest breakthroughs in artificial intelligence?", + "query_length": 61, + "query_type": "routing_query", + "output_response": "Routed to Web Search for query processing", + "router.classification": "Web Search", + "router.confidence_score": 1, + "router.rag_keyword_score": 0, + "router.web_keyword_score": 1, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [] + }, + { + "span_id": "ad476d62-d6b5-4142-8771-29855db3f39e", + "trace_id": "6fe93abf-1413-49e8-8464-58558e25ab7b", + "parent_span_id": "817c9703-0062-4bd6-b3ee-b50a1121c2d2", + "name": "agent.web_search_generation", + "start_time": "2025-10-22 16:58:42.084000000", + "end_time": "2025-10-22 16:58:44.766000000", + "duration_ms": 2681.732, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "web_search_agent", + "agent.operation": "web_search_generation", + "agent.capabilities": [ + "web_search", + "content_synthesis", + "response_generation" + ], + "agent.id": "noveum_web_search_agent", + "input_query": "What are the latest breakthroughs in artificial intelligence?", + "query_length": 61, + "query_type": "web_search_query", + "output_response": "Web Search Answer: The web search did not yield specific results regarding the latest breakthroughs in artificial intelligence. However, I can provide a general overview based on knowledge up to October 2023.\n\nRecent br...", + "web_search.results_count": 1, + "web_search.context_length": 163, + "web_search.response_length": 1040, + "web_search.mode": "external_web_search" + }, + "events": [], + "links": [] + }, + { + "span_id": "ff171ffc-dc6b-4cb4-9833-64c5622d9941", + "trace_id": "6fe93abf-1413-49e8-8464-58558e25ab7b", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-22 16:58:42.084000000", + "end_time": "2025-10-22 16:58:44.766000000", + "duration_ms": 2681.902, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What are the latest breakthroughs in artificial intelligence?", + "query.length": 61, + "processing.mode": "Web Search", + "processing.time_seconds": 2.6818161010742188, + "processing.response_length": 1040, + "processing.sources_count": 1, + "output_response": "Final Answer: The web search did not yield specific results regarding the latest breakthroughs in artificial intelligence. However, I can provide a general overview based on knowledge up to October 2023.\n\nRecent br...", + "final_answer_mode": "Web Search", + "query_processed.input_query": "What are the latest breakthroughs in artificial intelligence?", + "query_processed.output_response": "Successfully processed query using Web Search, generated 1040 character response", + "query_processed.mode": "Web Search", + "query_processed.processing_time": 2.6818161010742188, + "query_processed.response_length": 1040 + }, + "events": [], + "links": [] + }, + { + "span_id": "467cb806-a6ef-43da-9739-570730dd244b", + "trace_id": "6fe93abf-1413-49e8-8464-58558e25ab7b", + "parent_span_id": "ad476d62-d6b5-4142-8771-29855db3f39e", + "name": "agent.llm_model_execution", + "start_time": "2025-10-22 16:58:42.438000000", + "end_time": "2025-10-22 16:58:44.766000000", + "duration_ms": 2327.706, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What are the latest breakthroughs in artificial intelligence?", + "query_length": 61, + "query_type": "web_search_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 2.3276820182800293, + "model.latency_ms": 2327.6820182800293, + "model.start_time": 1761152322.438459, + "model.end_time": 1761152324.766141, + "model.performance_tier": "medium", + "model.prompt_tokens": 167, + "model.completion_tokens": 260, + "model.total_tokens": 427, + "model.tokens_per_second": 183.44430065903882, + "model.estimated_cost": 0.01281, + "model.efficiency_score": 2.4355971896955504, + "model.response_length": 1040, + "model.response_quality": "high", + "model.output_response": "Model Response: The web search did not yield specific results regarding the latest breakthroughs in artificial intelligence. However, I can provide a general overview based on knowledge up to October 2023.\n\nRecent br...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [] + }, + { + "span_id": "a80e4596-01ea-44e0-a590-13e347857f2a", + "trace_id": "6fe93abf-1413-49e8-8464-58558e25ab7b", + "parent_span_id": "ad476d62-d6b5-4142-8771-29855db3f39e", + "name": "agent.web_search_evaluation_metrics", + "start_time": "2025-10-22 16:58:44.766000000", + "end_time": "2025-10-22 16:58:44.766000000", + "duration_ms": 0.008, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "web_search_evaluation_metrics", + "agent.capabilities": [ + "web_search_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What are the latest breakthroughs in artificial intelligence?", + "query_length": 61, + "query_type": "web_search_evaluation_query", + "web_search.results_count": 1, + "web_search.context_length": 163, + "web_search.context_synthesized": "Context from 1 web sources", + "web_search.search_effectiveness": 0.2, + "web_search.context_quality": "low", + "web_search.source_diversity": 1, + "web_search.information_synthesis": "low", + "web_search.external_knowledge_utilization": 0.163, + "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What+are+the+latest+breakthroughs+in+artificial+intelligence?\n\nUser Question: What are the latest breakthroughs in artificial intelligence?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", + "prompt.prompt_length": 669, + "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What+are+the+latest+breakthroughs+in+artificial+intelligence?", + "prompt.instruction_following": "web_search_optimized", + "response.answer_length": 1040, + "response.answer_completeness": "complete", + "response.response_quality": "medium", + "response.source_citation": 1, + "response.context_utilization": 6.380368098159509, + "output_response": "Web Search Answer: The web search did not yield specific results regarding the latest breakthroughs in artificial intelligence. However, I can provide a general overview based on knowledge up to October 2023.\n\nRecent br...", + "evaluation.search_effectiveness": 0.2, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 1, + "evaluation.context_relevance": "low", + "evaluation.overall_quality": "low", + "evaluation.ready_for_production": true, + "web_search.search_strategy": "duckduckgo_api", + "web_search.real_time_data": true, + "web_search.external_sources": 1, + "web_search.information_freshness": "current", + "web_search.knowledge_synthesis": "single_source" + }, + "events": [], + "links": [] + } + ], + "created_at": "2025-10-22 16:58:42.084000000", + "updated_at": "2025-10-22 16:58:42.084000000" + }, + { + "trace_id": "ec7c6cae-9bbd-47f4-a8ae-2d6d9b5011ba", + "name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "status": "ok", + "status_message": "", + "start_time": "2025-10-22 16:58:32.800000000", + "end_time": "2025-10-22 16:58:41.077000000", + "duration_ms": 8276.788, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "spans": [ + { + "span_id": "409f29f6-21c7-400a-a9e0-11c87c4313cb", + "trace_id": "ec7c6cae-9bbd-47f4-a8ae-2d6d9b5011ba", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-22 16:58:32.801000000", + "end_time": "2025-10-22 16:58:41.077000000", + "duration_ms": 8276.701, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What's happening in the software development world today?", + "query.length": 57, + "processing.mode": "Web Search", + "processing.time_seconds": 8.276506900787354, + "processing.response_length": 1424, + "processing.sources_count": 1, + "output_response": "Final Answer: The web search did not yield any specific results regarding current events in the software development world. Therefore, I cannot provide detailed information on the latest trends, technologies, or ne...", + "final_answer_mode": "Web Search", + "query_processed.input_query": "What's happening in the software development world today?", + "query_processed.output_response": "Successfully processed query using Web Search, generated 1424 character response", + "query_processed.mode": "Web Search", + "query_processed.processing_time": 8.276506900787354, + "query_processed.response_length": 1424 + }, + "events": [], + "links": [] + }, + { + "span_id": "43389a18-a566-41e7-8db5-f688f651a106", + "trace_id": "ec7c6cae-9bbd-47f4-a8ae-2d6d9b5011ba", + "parent_span_id": "af400e17-8b20-4218-834b-9330ccaa2635", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-22 16:58:32.801000000", + "end_time": "2025-10-22 16:58:32.801000000", + "duration_ms": 0.047, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What's happening in the software development world today?", + "query_length": 57, + "query_type": "routing_evaluation_query", + "classification.mode": "Web Search", + "classification.rag_keyword_score": 0, + "classification.web_keyword_score": 1, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "complex", + "query.intent": "general_knowledge", + "query.keyword_density": 0.125, + "query.domain_affinity": "general", + "routing.decision": "Routed to Web Search based on analysis", + "routing.rationale": "RAG score: 0, Web score: 1, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "RAG", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.019230769230769232, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: Web Search (Confidence: 1.00)" + }, + "events": [], + "links": [] + }, + { + "span_id": "6b431bca-af7e-4f7e-aedb-b103d7f47cae", + "trace_id": "ec7c6cae-9bbd-47f4-a8ae-2d6d9b5011ba", + "parent_span_id": "af400e17-8b20-4218-834b-9330ccaa2635", + "name": "agent.web_search_generation", + "start_time": "2025-10-22 16:58:32.801000000", + "end_time": "2025-10-22 16:58:41.077000000", + "duration_ms": 8276.225999999999, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "web_search_agent", + "agent.operation": "web_search_generation", + "agent.capabilities": [ + "web_search", + "content_synthesis", + "response_generation" + ], + "agent.id": "noveum_web_search_agent", + "input_query": "What's happening in the software development world today?", + "query_length": 57, + "query_type": "web_search_query", + "output_response": "Web Search Answer: The web search did not yield any specific results regarding current events in the software development world. Therefore, I cannot provide detailed information on the latest trends, technologies, or ne...", + "web_search.results_count": 1, + "web_search.context_length": 159, + "web_search.response_length": 1424, + "web_search.mode": "external_web_search" + }, + "events": [], + "links": [] + }, + { + "span_id": "af400e17-8b20-4218-834b-9330ccaa2635", + "trace_id": "ec7c6cae-9bbd-47f4-a8ae-2d6d9b5011ba", + "parent_span_id": "409f29f6-21c7-400a-a9e0-11c87c4313cb", + "name": "agent.query_routing", + "start_time": "2025-10-22 16:58:32.801000000", + "end_time": "2025-10-22 16:58:41.077000000", + "duration_ms": 8276.465999999999, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What's happening in the software development world today?", + "query_length": 57, + "query_type": "routing_query", + "output_response": "Routed to Web Search for query processing", + "router.classification": "Web Search", + "router.confidence_score": 1, + "router.rag_keyword_score": 0, + "router.web_keyword_score": 1, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [] + }, + { + "span_id": "37b153b2-d02b-4b7e-9229-1ba3eaae44ee", + "trace_id": "ec7c6cae-9bbd-47f4-a8ae-2d6d9b5011ba", + "parent_span_id": "6b431bca-af7e-4f7e-aedb-b103d7f47cae", + "name": "agent.llm_model_execution", + "start_time": "2025-10-22 16:58:33.172000000", + "end_time": "2025-10-22 16:58:41.077000000", + "duration_ms": 7904.722, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What's happening in the software development world today?", + "query_length": 57, + "query_type": "web_search_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 7.904651880264282, + "model.latency_ms": 7904.651880264282, + "model.start_time": 1761152313.172655, + "model.end_time": 1761152321.077307, + "model.performance_tier": "slow", + "model.prompt_tokens": 165, + "model.completion_tokens": 356, + "model.total_tokens": 521, + "model.tokens_per_second": 65.91055594754174, + "model.estimated_cost": 0.01563, + "model.efficiency_score": 2.7332053742802302, + "model.response_length": 1424, + "model.response_quality": "high", + "model.output_response": "Model Response: The web search did not yield any specific results regarding current events in the software development world. Therefore, I cannot provide detailed information on the latest trends, technologies, or ne...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [] + }, + { + "span_id": "cc941b8e-6625-43e9-a0a9-7328af24f12f", + "trace_id": "ec7c6cae-9bbd-47f4-a8ae-2d6d9b5011ba", + "parent_span_id": "6b431bca-af7e-4f7e-aedb-b103d7f47cae", + "name": "agent.web_search_evaluation_metrics", + "start_time": "2025-10-22 16:58:41.077000000", + "end_time": "2025-10-22 16:58:41.077000000", + "duration_ms": 0.026, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "web_search_evaluation_metrics", + "agent.capabilities": [ + "web_search_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What's happening in the software development world today?", + "query_length": 57, + "query_type": "web_search_evaluation_query", + "web_search.results_count": 1, + "web_search.context_length": 159, + "web_search.context_synthesized": "Context from 1 web sources", + "web_search.search_effectiveness": 0.2, + "web_search.context_quality": "low", + "web_search.source_diversity": 1, + "web_search.information_synthesis": "low", + "web_search.external_knowledge_utilization": 0.159, + "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What's+happening+in+the+software+development+world+today?\n\nUser Question: What's happening in the software development world today?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", + "prompt.prompt_length": 661, + "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What's+happening+in+the+software+development+world+today?", + "prompt.instruction_following": "web_search_optimized", + "response.answer_length": 1424, + "response.answer_completeness": "complete", + "response.response_quality": "medium", + "response.source_citation": 1, + "response.context_utilization": 8.955974842767295, + "output_response": "Web Search Answer: The web search did not yield any specific results regarding current events in the software development world. Therefore, I cannot provide detailed information on the latest trends, technologies, or ne...", + "evaluation.search_effectiveness": 0.2, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 1, + "evaluation.context_relevance": "low", + "evaluation.overall_quality": "low", + "evaluation.ready_for_production": true, + "web_search.search_strategy": "duckduckgo_api", + "web_search.real_time_data": true, + "web_search.external_sources": 1, + "web_search.information_freshness": "current", + "web_search.knowledge_synthesis": "single_source" + }, + "events": [], + "links": [] + } + ], + "created_at": "2025-10-22 16:58:32.800000000", + "updated_at": "2025-10-22 16:58:32.800000000" + }, + { + "trace_id": "343764b8-7ac9-4549-93cf-8abc11ceba6f", + "name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "status": "ok", + "status_message": "", + "start_time": "2025-10-22 16:58:26.542000000", + "end_time": "2025-10-22 16:58:31.796000000", + "duration_ms": 5253.973, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "spans": [ + { + "span_id": "3b06d098-6eb7-4bb6-885e-443ddb28cc41", + "trace_id": "343764b8-7ac9-4549-93cf-8abc11ceba6f", + "parent_span_id": "5c2aa1cf-56a8-4fd5-af90-90a94e723bb5", + "name": "agent.query_routing", + "start_time": "2025-10-22 16:58:26.542000000", + "end_time": "2025-10-22 16:58:31.796000000", + "duration_ms": 5253.835, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What are the newest features in cloud computing?", + "query_length": 48, + "query_type": "routing_query", + "output_response": "Routed to RAG for query processing", + "router.classification": "RAG", + "router.confidence_score": 1, + "router.rag_keyword_score": 1, + "router.web_keyword_score": 0, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [] + }, + { + "span_id": "5c2aa1cf-56a8-4fd5-af90-90a94e723bb5", + "trace_id": "343764b8-7ac9-4549-93cf-8abc11ceba6f", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-22 16:58:26.542000000", + "end_time": "2025-10-22 16:58:31.796000000", + "duration_ms": 5253.913, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What are the newest features in cloud computing?", + "query.length": 48, + "processing.mode": "RAG", + "processing.time_seconds": 5.253868818283081, + "processing.response_length": 837, + "processing.sources_count": 5, + "output_response": "Final Answer: The provided context does not specifically mention any new features in cloud computing as a general topic. However, it does highlight several recent enhancements related to the Noveum.ai platform, whi...", + "final_answer_mode": "RAG", + "query_processed.input_query": "What are the newest features in cloud computing?", + "query_processed.output_response": "Successfully processed query using RAG, generated 837 character response", + "query_processed.mode": "RAG", + "query_processed.processing_time": 5.253868818283081, + "query_processed.response_length": 837 + }, + "events": [], + "links": [] + }, + { + "span_id": "6b838ed6-f5bb-4bcf-a76b-bac283bf7571", + "trace_id": "343764b8-7ac9-4549-93cf-8abc11ceba6f", + "parent_span_id": "3b06d098-6eb7-4bb6-885e-443ddb28cc41", + "name": "agent.llm-rag", + "start_time": "2025-10-22 16:58:26.542000000", + "end_time": "2025-10-22 16:58:31.796000000", + "duration_ms": 5253.519, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "rag_agent", + "agent.operation": "llm-rag", + "agent.capabilities": [ + "document_retrieval", + "context_generation", + "response_generation" + ], + "agent.id": "noveum_rag_agent", + "input_query": "What are the newest features in cloud computing?", + "query_length": 48, + "query_type": "rag_query", + "output_response": "RAG Answer: The provided context does not specifically mention any new features in cloud computing as a general topic. However, it does highlight several recent enhancements related to the Noveum.ai platform, whi...", + "rag.context_length": 2579, + "rag.sources_count": 5, + "rag.answer_length": 837, + "rag.mode": "retrieval_augmented_generation" + }, + "events": [], + "links": [] + }, + { + "span_id": "f51d8f1e-eae3-488d-9a96-49ab97377851", + "trace_id": "343764b8-7ac9-4549-93cf-8abc11ceba6f", + "parent_span_id": "3b06d098-6eb7-4bb6-885e-443ddb28cc41", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-22 16:58:26.542000000", + "end_time": "2025-10-22 16:58:26.542000000", + "duration_ms": 0.082, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What are the newest features in cloud computing?", + "query_length": 48, + "query_type": "routing_evaluation_query", + "classification.mode": "RAG", + "classification.rag_keyword_score": 1, + "classification.web_keyword_score": 0, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "documentation", + "query.keyword_density": 0.125, + "query.domain_affinity": "noveum", + "routing.decision": "Routed to RAG based on analysis", + "routing.rationale": "RAG score: 1, Web score: 0, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "Web Search", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.019230769230769232, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: RAG (Confidence: 1.00)" + }, + "events": [], + "links": [] + }, + { + "span_id": "93a0fd69-47b6-4e7c-b8ab-8eaa5aeac240", + "trace_id": "343764b8-7ac9-4549-93cf-8abc11ceba6f", + "parent_span_id": "6b838ed6-f5bb-4bcf-a76b-bac283bf7571", + "name": "agent.llm_model_execution", + "start_time": "2025-10-22 16:58:27.411000000", + "end_time": "2025-10-22 16:58:31.373000000", + "duration_ms": 3961.481, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What are the newest features in cloud computing?", + "query_length": 48, + "query_type": "rag_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 3.9613850116729736, + "model.latency_ms": 3961.3850116729736, + "model.start_time": 1761152307.411936, + "model.end_time": 1761152311.373321, + "model.performance_tier": "medium", + "model.prompt_tokens": 780, + "model.completion_tokens": 209, + "model.total_tokens": 989, + "model.tokens_per_second": 249.66015600243944, + "model.estimated_cost": 0.029670000000000002, + "model.efficiency_score": 0.846309403437816, + "model.response_length": 837, + "model.response_quality": "high", + "model.output_response": "Model Response: The provided context does not specifically mention any new features in cloud computing as a general topic. However, it does highlight several recent enhancements related to the Noveum.ai platform, whi...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [] + }, + { + "span_id": "48cbc1df-f03a-4c49-90e5-19a2047c1c7b", + "trace_id": "343764b8-7ac9-4549-93cf-8abc11ceba6f", + "parent_span_id": "6b838ed6-f5bb-4bcf-a76b-bac283bf7571", + "name": "agent.rag_evaluation_metrics", + "start_time": "2025-10-22 16:58:31.373000000", + "end_time": "2025-10-22 16:58:31.373000000", + "duration_ms": 0.061, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "rag_evaluation_metrics", + "agent.capabilities": [ + "retrieval_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What are the newest features in cloud computing?", + "query_length": 48, + "query_type": "rag_evaluation_query", + "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/changelog):\n- \ud83d\udcc8 Enhanced Dashboard Analytics: Real-time request tracking, latency monitoring, and cost analysis\n- \ud83d\udd0d Improved Logs Interface: Better search functionality, detailed trace views, and enhanced debugging capabilities\n- \ud83d\udc65 Team Management: Advanced member mana...", + "retrieval.context_length": 2579, + "retrieval.sources_count": 5, + "retrieval.context_quality": "high", + "retrieval.effectiveness": 1, + "retrieval.context_utilization": 2.579, + "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/changelog):\n- \ud83d\udcc8 Enhanced Dashboard Analytics: Real-time request tracking, latency monitoring, and cost analysis\n- \ud83d\udd0d Improved Logs Interface: Better search functionality, detailed trace views, and enhanced debugging capabilities\n- \ud83d\udc65 Team Management: Advanced member management with role-based access control and invitation system\n- \ud83d\udee1\ufe0f Security Improvements: Enhanced API key management and secure credential storage\n4 months ago\n- \ud83d\udcca Advanced Metrics Collection: Comprehensive telemetry system with custom metrics ...\n\nSource 2 (https://noveum.ai/en/changelog):\n- \ud83d\udcdd Documentation Updates: Comprehensive guides for integration and best practices\n5 months ago\n- \ud83c\udf89 Platform Foundation: Initial release of Noveum.ai AI observability platform\n- \ud83d\ude80 Core Features: AI Gateway integration, basic evaluation metrics, and monitoring capabilities\n- \ud83d\udcc8 Evaluation Jobs: Automated model evaluation with configurable metrics and reporting\n- \ud83d\udd0c Provider Integrations: Support for major AI providers and custom model deployments...\n\nSource 3 (https://noveum.ai/docs/platform/dashboard):\n- Bulk Export: Download multiple traces simultaneously\n- Comparative Analysis: Compare performance across multiple traces\nReal-Time Updates\n- Live Refresh: Automatic updates as new traces arrive\n- Connection Monitoring: Real-time status of your trace ingestion pipeline\n- Performance Indicators: Live metrics for system health monitoring\nReady to dive deeper? Explore Projects & Environments to organize your AI applications, or check out Team Collaboration to share insights with your team.\nGet Earl...\n\nSource 4 (https://noveum.ai/en/changelog):\nChangelog\nStay up to date with the latest changes in our product.\n2 months ago\n- \ud83c\udfaf Major Platform Evolution: NovaEval Framework - Launched comprehensive AI model evaluation framework with 20+ built-in scorers including accuracy, RAG metrics, conversational scoring, and specialized agent evaluation tools. Features production-grade deployment on Docker and Kubernetes. (GitHub | PyPI Package)\n- \ud83d\ude80 Next-Generation Traces UI - Complete redesign featuring three-pane layout with directory tree navigatio...\n\nSource 5 (https://noveum.ai/en/blog):\n8/12/2025\nDiscover how Noveum.ai provides comprehensive tracing and observability for AI applications, from development debugging to production optimization.\nShashank Agarwal\n3/3/2025\nDiscover how Noveum.ai provides comprehensive tracing and observability for LLM applications, RAG systems, and multi-agent workflows with our powerful Python and TypeScript SDKs.\nShashank Agarwal\n3/2/2025...\n\nUser Question: What are the newest features in cloud computing?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", + "prompt.prompt_length": 3123, + "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/changelog):\n- \ud83d\udcc8 Enhanced Dashboard Analytics: Real-time request tracking, latency monitoring, and cost analysis\n- \ud83d\udd0d Improved Logs Interface: Better search functionality,...", + "prompt.instruction_following": "rag_optimized", + "response.answer_length": 837, + "response.answer_completeness": "complete", + "response.response_quality": "high", + "response.source_citation": 5, + "response.context_utilization": 0.3245443970531214, + "output_response": "RAG Answer: The provided context does not specifically mention any new features in cloud computing as a general topic. However, it does highlight several recent enhancements related to the Noveum.ai platform, whi...", + "evaluation.retrieval_effectiveness": 1, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 5, + "evaluation.context_relevance": "high", + "evaluation.overall_quality": "high", + "evaluation.ready_for_production": true, + "rag.retrieval_strategy": "semantic_similarity", + "rag.vector_search_results": 5, + "rag.context_synthesis": "multi_source", + "rag.document_coverage": 1, + "rag.information_density": 0.3245443970531214 + }, + "events": [], + "links": [] + } + ], + "created_at": "2025-10-22 16:58:26.542000000", + "updated_at": "2025-10-22 16:58:26.542000000" + }, + { + "trace_id": "3f7b6a62-8888-4a5d-b12c-37a3829aea49", + "name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "status": "ok", + "status_message": "", + "start_time": "2025-10-22 16:58:19.885000000", + "end_time": "2025-10-22 16:58:25.536000000", + "duration_ms": 5651.67, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "spans": [ + { + "span_id": "7384d697-d7bf-4709-8377-ec26b094b550", + "trace_id": "3f7b6a62-8888-4a5d-b12c-37a3829aea49", + "parent_span_id": "e631b22a-a621-4ecc-bb01-f731ca66f3d0", + "name": "agent.query_routing", + "start_time": "2025-10-22 16:58:19.885000000", + "end_time": "2025-10-22 16:58:25.536000000", + "duration_ms": 5651.335999999999, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What's the current status of cryptocurrency markets?", + "query_length": 52, + "query_type": "routing_query", + "output_response": "Routed to Web Search for query processing", + "router.classification": "Web Search", + "router.confidence_score": 1, + "router.rag_keyword_score": 0, + "router.web_keyword_score": 3, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [] + }, + { + "span_id": "7abf372d-d9f4-41d1-9e3a-d4f01173fa5e", + "trace_id": "3f7b6a62-8888-4a5d-b12c-37a3829aea49", + "parent_span_id": "7384d697-d7bf-4709-8377-ec26b094b550", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-22 16:58:19.885000000", + "end_time": "2025-10-22 16:58:19.885000000", + "duration_ms": 0.037, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What's the current status of cryptocurrency markets?", + "query_length": 52, + "query_type": "routing_evaluation_query", + "classification.mode": "Web Search", + "classification.rag_keyword_score": 0, + "classification.web_keyword_score": 3, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "complex", + "query.intent": "general_knowledge", + "query.keyword_density": 0.42857142857142855, + "query.domain_affinity": "general", + "routing.decision": "Routed to Web Search based on analysis", + "routing.rationale": "RAG score: 0, Web score: 3, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "RAG", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.057692307692307696, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: Web Search (Confidence: 1.00)" + }, + "events": [], + "links": [] + }, + { + "span_id": "e5d9c3c3-6c18-45d2-a6b6-58bde4d4e461", + "trace_id": "3f7b6a62-8888-4a5d-b12c-37a3829aea49", + "parent_span_id": "7384d697-d7bf-4709-8377-ec26b094b550", + "name": "agent.web_search_generation", + "start_time": "2025-10-22 16:58:19.885000000", + "end_time": "2025-10-22 16:58:25.536000000", + "duration_ms": 5651.161, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "web_search_agent", + "agent.operation": "web_search_generation", + "agent.capabilities": [ + "web_search", + "content_synthesis", + "response_generation" + ], + "agent.id": "noveum_web_search_agent", + "input_query": "What's the current status of cryptocurrency markets?", + "query_length": 52, + "query_type": "web_search_query", + "output_response": "Web Search Answer: The provided web search results do not contain specific information regarding the current status of cryptocurrency markets. To get the latest updates on cryptocurrency prices, trends, and market analy...", + "web_search.results_count": 1, + "web_search.context_length": 251, + "web_search.response_length": 416, + "web_search.mode": "external_web_search" + }, + "events": [], + "links": [] + }, + { + "span_id": "e631b22a-a621-4ecc-bb01-f731ca66f3d0", + "trace_id": "3f7b6a62-8888-4a5d-b12c-37a3829aea49", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-22 16:58:19.885000000", + "end_time": "2025-10-22 16:58:25.536000000", + "duration_ms": 5651.588000000001, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What's the current status of cryptocurrency markets?", + "query.length": 52, + "processing.mode": "Web Search", + "processing.time_seconds": 5.651373863220215, + "processing.response_length": 416, + "processing.sources_count": 1, + "output_response": "Final Answer: The provided web search results do not contain specific information regarding the current status of cryptocurrency markets. To get the latest updates on cryptocurrency prices, trends, and market analy...", + "final_answer_mode": "Web Search", + "query_processed.input_query": "What's the current status of cryptocurrency markets?", + "query_processed.output_response": "Successfully processed query using Web Search, generated 416 character response", + "query_processed.mode": "Web Search", + "query_processed.processing_time": 5.651373863220215, + "query_processed.response_length": 416 + }, + "events": [], + "links": [] + }, + { + "span_id": "34e2f906-724e-4116-b61f-82eaa6b75959", + "trace_id": "3f7b6a62-8888-4a5d-b12c-37a3829aea49", + "parent_span_id": "e5d9c3c3-6c18-45d2-a6b6-58bde4d4e461", + "name": "agent.llm_model_execution", + "start_time": "2025-10-22 16:58:24.327000000", + "end_time": "2025-10-22 16:58:25.536000000", + "duration_ms": 1208.66, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What's the current status of cryptocurrency markets?", + "query_length": 52, + "query_type": "web_search_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 1.208608865737915, + "model.latency_ms": 1208.608865737915, + "model.start_time": 1761152304.327749, + "model.end_time": 1761152305.5363579, + "model.performance_tier": "fast", + "model.prompt_tokens": 187, + "model.completion_tokens": 104, + "model.total_tokens": 291, + "model.tokens_per_second": 240.77268357809888, + "model.estimated_cost": 0.00873, + "model.efficiency_score": 1.429553264604811, + "model.response_length": 416, + "model.response_quality": "high", + "model.output_response": "Model Response: The provided web search results do not contain specific information regarding the current status of cryptocurrency markets. To get the latest updates on cryptocurrency prices, trends, and market analy...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [] + }, + { + "span_id": "5548e02e-2ea8-472c-9c42-f811d6e4d30b", + "trace_id": "3f7b6a62-8888-4a5d-b12c-37a3829aea49", + "parent_span_id": "e5d9c3c3-6c18-45d2-a6b6-58bde4d4e461", + "name": "agent.web_search_evaluation_metrics", + "start_time": "2025-10-22 16:58:25.536000000", + "end_time": "2025-10-22 16:58:25.536000000", + "duration_ms": 0.044, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "web_search_evaluation_metrics", + "agent.capabilities": [ + "web_search_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What's the current status of cryptocurrency markets?", + "query_length": 52, + "query_type": "web_search_evaluation_query", + "web_search.results_count": 1, + "web_search.context_length": 251, + "web_search.context_synthesized": "Context from 1 web sources", + "web_search.search_effectiveness": 0.2, + "web_search.context_quality": "low", + "web_search.source_diversity": 1, + "web_search.information_synthesis": "low", + "web_search.external_knowledge_utilization": 0.251, + "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nLog in to WhatsApp Web for simple, reliable and private messaging on your desktop. Send and receive messages and files with ease, all for \u2026\nURL: https://duckduckgo.com/?q=What's+the+current+status+of+cryptocurrency+markets?\n\nUser Question: What's the current status of cryptocurrency markets?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", + "prompt.prompt_length": 748, + "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nLog in to WhatsApp Web for simple, reliable and private messaging on your desktop. Send and receive messages and files with ease, all for \u2026\nURL: https://duckduckgo.com/?q=W...", + "prompt.instruction_following": "web_search_optimized", + "response.answer_length": 416, + "response.answer_completeness": "complete", + "response.response_quality": "medium", + "response.source_citation": 1, + "response.context_utilization": 1.657370517928287, + "output_response": "Web Search Answer: The provided web search results do not contain specific information regarding the current status of cryptocurrency markets. To get the latest updates on cryptocurrency prices, trends, and market analy...", + "evaluation.search_effectiveness": 0.2, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 1, + "evaluation.context_relevance": "low", + "evaluation.overall_quality": "low", + "evaluation.ready_for_production": true, + "web_search.search_strategy": "duckduckgo_api", + "web_search.real_time_data": true, + "web_search.external_sources": 1, + "web_search.information_freshness": "current", + "web_search.knowledge_synthesis": "single_source" + }, + "events": [], + "links": [] + } + ], + "created_at": "2025-10-22 16:58:19.885000000", + "updated_at": "2025-10-22 16:58:19.885000000" + }, + { + "trace_id": "51f4534f-23e4-4be7-8d6e-e93391d6e0e2", + "name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "status": "ok", + "status_message": "", + "start_time": "2025-10-22 16:58:15.254000000", + "end_time": "2025-10-22 16:58:18.877000000", + "duration_ms": 3622.355, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "spans": [ + { + "span_id": "a9e34cea-d4bd-478b-abf2-7a3a38dcd40e", + "trace_id": "51f4534f-23e4-4be7-8d6e-e93391d6e0e2", + "parent_span_id": "8e9b2d06-3e11-4d17-9a32-b1529a2c10a4", + "name": "agent.web_search_generation", + "start_time": "2025-10-22 16:58:15.254000000", + "end_time": "2025-10-22 16:58:18.876000000", + "duration_ms": 3621.674, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "web_search_agent", + "agent.operation": "web_search_generation", + "agent.capabilities": [ + "web_search", + "content_synthesis", + "response_generation" + ], + "agent.id": "noveum_web_search_agent", + "input_query": "What are the latest updates in Python programming?", + "query_length": 50, + "query_type": "web_search_query", + "output_response": "Web Search Answer: The provided web search results do not contain specific information about the latest updates in Python programming. They primarily focus on general news coverage and updates from India and around the ...", + "web_search.results_count": 1, + "web_search.context_length": 753, + "web_search.response_length": 580, + "web_search.mode": "external_web_search" + }, + "events": [], + "links": [] + }, + { + "span_id": "8e9b2d06-3e11-4d17-9a32-b1529a2c10a4", + "trace_id": "51f4534f-23e4-4be7-8d6e-e93391d6e0e2", + "parent_span_id": "bda936a0-24f3-4841-af4f-227197e10b9d", + "name": "agent.query_routing", + "start_time": "2025-10-22 16:58:15.254000000", + "end_time": "2025-10-22 16:58:18.876000000", + "duration_ms": 3621.863, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What are the latest updates in Python programming?", + "query_length": 50, + "query_type": "routing_query", + "output_response": "Routed to Web Search for query processing", + "router.classification": "Web Search", + "router.confidence_score": 1, + "router.rag_keyword_score": 0, + "router.web_keyword_score": 2, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [] + }, + { + "span_id": "bda936a0-24f3-4841-af4f-227197e10b9d", + "trace_id": "51f4534f-23e4-4be7-8d6e-e93391d6e0e2", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-22 16:58:15.254000000", + "end_time": "2025-10-22 16:58:18.877000000", + "duration_ms": 3622.271, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What are the latest updates in Python programming?", + "query.length": 50, + "processing.mode": "Web Search", + "processing.time_seconds": 3.6219100952148438, + "processing.response_length": 580, + "processing.sources_count": 1, + "output_response": "Final Answer: The provided web search results do not contain specific information about the latest updates in Python programming. They primarily focus on general news coverage and updates from India and around the ...", + "final_answer_mode": "Web Search", + "query_processed.input_query": "What are the latest updates in Python programming?", + "query_processed.output_response": "Successfully processed query using Web Search, generated 580 character response", + "query_processed.mode": "Web Search", + "query_processed.processing_time": 3.6219100952148438, + "query_processed.response_length": 580 + }, + "events": [], + "links": [] + }, + { + "span_id": "e27e3812-d49d-4f4f-98b0-144cf0ceffc3", + "trace_id": "51f4534f-23e4-4be7-8d6e-e93391d6e0e2", + "parent_span_id": "8e9b2d06-3e11-4d17-9a32-b1529a2c10a4", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-22 16:58:15.254000000", + "end_time": "2025-10-22 16:58:15.254000000", + "duration_ms": 0.047, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What are the latest updates in Python programming?", + "query_length": 50, + "query_type": "routing_evaluation_query", + "classification.mode": "Web Search", + "classification.rag_keyword_score": 0, + "classification.web_keyword_score": 2, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "general_knowledge", + "query.keyword_density": 0.25, + "query.domain_affinity": "general", + "routing.decision": "Routed to Web Search based on analysis", + "routing.rationale": "RAG score: 0, Web score: 2, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "RAG", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.038461538461538464, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: Web Search (Confidence: 1.00)" + }, + "events": [], + "links": [] + }, + { + "span_id": "b0eea8ec-5cff-4667-bea7-aeb6d0f0daf3", + "trace_id": "51f4534f-23e4-4be7-8d6e-e93391d6e0e2", + "parent_span_id": "a9e34cea-d4bd-478b-abf2-7a3a38dcd40e", + "name": "agent.llm_model_execution", + "start_time": "2025-10-22 16:58:15.568000000", + "end_time": "2025-10-22 16:58:18.876000000", + "duration_ms": 3307.5739999999996, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What are the latest updates in Python programming?", + "query_length": 50, + "query_type": "web_search_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 3.307521104812622, + "model.latency_ms": 3307.521104812622, + "model.start_time": 1761152295.568949, + "model.end_time": 1761152298.87647, + "model.performance_tier": "medium", + "model.prompt_tokens": 312, + "model.completion_tokens": 145, + "model.total_tokens": 457, + "model.tokens_per_second": 138.16994223711538, + "model.estimated_cost": 0.01371, + "model.efficiency_score": 1.2691466083150984, + "model.response_length": 580, + "model.response_quality": "high", + "model.output_response": "Model Response: The provided web search results do not contain specific information about the latest updates in Python programming. They primarily focus on general news coverage and updates from India and around the ...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [] + }, + { + "span_id": "d0bfcbe5-cb6c-4837-bdf3-f2c65be145cd", + "trace_id": "51f4534f-23e4-4be7-8d6e-e93391d6e0e2", + "parent_span_id": "a9e34cea-d4bd-478b-abf2-7a3a38dcd40e", + "name": "agent.web_search_evaluation_metrics", + "start_time": "2025-10-22 16:58:18.876000000", + "end_time": "2025-10-22 16:58:18.876000000", + "duration_ms": 0.04, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "web_search_evaluation_metrics", + "agent.capabilities": [ + "web_search_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What are the latest updates in Python programming?", + "query_length": 50, + "query_type": "web_search_evaluation_query", + "web_search.results_count": 1, + "web_search.context_length": 753, + "web_search.context_synthesized": "Context from 1 web sources", + "web_search.search_effectiveness": 0.2, + "web_search.context_quality": "medium", + "web_search.source_diversity": 1, + "web_search.information_synthesis": "low", + "web_search.external_knowledge_utilization": 0.753, + "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\n3 days ago \u00b7 Latest news headlines from India & around the world. Check out today\u2019s news coverage live with videos & photos on \u2026 6 days ago \u00b7 India News | Latest India News | Read latest and breaking news from India. Today's top India news headlines, news on \u2026 2 days ago \u00b7 Top News Stories of the day, Latest News Headlines, News Specials, Breaking News and Latest India News, World current affairs & \u2026 NDTV.com provides latest news from India and around the world. Get breaking news alerts from India and follow today\u2019s live news updates in \u2026 6 days ago \u00b7 Get Latest entertainment news from Bollywood, Hollywood and regional film industries.\nURL: https://duckduckgo.com/?q=What+are+the+latest+updates+in+Python+programming?\n\nUser Question: What are the latest updates in Python programming?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", + "prompt.prompt_length": 1248, + "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\n3 days ago \u00b7 Latest news headlines from India & around the world. Check out today\u2019s news coverage live with videos & photos on \u2026 6 days ago \u00b7 India News | Latest India News...", + "prompt.instruction_following": "web_search_optimized", + "response.answer_length": 580, + "response.answer_completeness": "complete", + "response.response_quality": "medium", + "response.source_citation": 1, + "response.context_utilization": 0.7702523240371846, + "output_response": "Web Search Answer: The provided web search results do not contain specific information about the latest updates in Python programming. They primarily focus on general news coverage and updates from India and around the ...", + "evaluation.search_effectiveness": 0.2, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 1, + "evaluation.context_relevance": "medium", + "evaluation.overall_quality": "low", + "evaluation.ready_for_production": true, + "web_search.search_strategy": "duckduckgo_api", + "web_search.real_time_data": true, + "web_search.external_sources": 1, + "web_search.information_freshness": "current", + "web_search.knowledge_synthesis": "single_source" + }, + "events": [], + "links": [] + } + ], + "created_at": "2025-10-22 16:58:15.254000000", + "updated_at": "2025-10-22 16:58:15.254000000" + }, + { + "trace_id": "e30e9ba6-ffd1-4eb9-b7ae-6d19e04343d6", + "name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "status": "ok", + "status_message": "", + "start_time": "2025-10-22 16:58:11.033000000", + "end_time": "2025-10-22 16:58:14.250000000", + "duration_ms": 3216.801, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "spans": [ + { + "span_id": "5f5916a7-b9ea-4348-861d-db69b25d6b62", + "trace_id": "e30e9ba6-ffd1-4eb9-b7ae-6d19e04343d6", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-22 16:58:11.033000000", + "end_time": "2025-10-22 16:58:14.250000000", + "duration_ms": 3216.77, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What happened in tech news this week?", + "query.length": 37, + "processing.mode": "Web Search", + "processing.time_seconds": 3.2166178226470947, + "processing.response_length": 510, + "processing.sources_count": 1, + "output_response": "Final Answer: The web search results do not provide specific information about recent events in tech news for this week. They primarily discuss the grammatical usage of the word \"happened\" and its variations, such ...", + "final_answer_mode": "Web Search", + "query_processed.input_query": "What happened in tech news this week?", + "query_processed.output_response": "Successfully processed query using Web Search, generated 510 character response", + "query_processed.mode": "Web Search", + "query_processed.processing_time": 3.2166178226470947, + "query_processed.response_length": 510 + }, + "events": [], + "links": [] + }, + { + "span_id": "804613d0-a701-4c22-add1-d315d7a29813", + "trace_id": "e30e9ba6-ffd1-4eb9-b7ae-6d19e04343d6", + "parent_span_id": "5f5916a7-b9ea-4348-861d-db69b25d6b62", + "name": "agent.query_routing", + "start_time": "2025-10-22 16:58:11.033000000", + "end_time": "2025-10-22 16:58:14.250000000", + "duration_ms": 3216.594, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What happened in tech news this week?", + "query_length": 37, + "query_type": "routing_query", + "output_response": "Routed to Web Search for query processing", + "router.classification": "Web Search", + "router.confidence_score": 1, + "router.rag_keyword_score": 0, + "router.web_keyword_score": 2, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [] + }, + { + "span_id": "f6c8d34f-185f-49ba-a5a4-a9d25ccb4fc6", + "trace_id": "e30e9ba6-ffd1-4eb9-b7ae-6d19e04343d6", + "parent_span_id": "804613d0-a701-4c22-add1-d315d7a29813", + "name": "agent.web_search_generation", + "start_time": "2025-10-22 16:58:11.034000000", + "end_time": "2025-10-22 16:58:14.250000000", + "duration_ms": 3216.4959999999996, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "web_search_agent", + "agent.operation": "web_search_generation", + "agent.capabilities": [ + "web_search", + "content_synthesis", + "response_generation" + ], + "agent.id": "noveum_web_search_agent", + "input_query": "What happened in tech news this week?", + "query_length": 37, + "query_type": "web_search_query", + "output_response": "Web Search Answer: The web search results do not provide specific information about recent events in tech news for this week. They primarily discuss the grammatical usage of the word \"happened\" and its variations, such ...", + "web_search.results_count": 1, + "web_search.context_length": 724, + "web_search.response_length": 510, + "web_search.mode": "external_web_search" + }, + "events": [], + "links": [] + }, + { + "span_id": "11d68082-e7b8-4bbf-833a-79e296c31eda", + "trace_id": "e30e9ba6-ffd1-4eb9-b7ae-6d19e04343d6", + "parent_span_id": "804613d0-a701-4c22-add1-d315d7a29813", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-22 16:58:11.034000000", + "end_time": "2025-10-22 16:58:11.034000000", + "duration_ms": 0.017, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What happened in tech news this week?", + "query_length": 37, + "query_type": "routing_evaluation_query", + "classification.mode": "Web Search", + "classification.rag_keyword_score": 0, + "classification.web_keyword_score": 2, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "general_knowledge", + "query.keyword_density": 0.2857142857142857, + "query.domain_affinity": "general", + "routing.decision": "Routed to Web Search based on analysis", + "routing.rationale": "RAG score: 0, Web score: 2, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "RAG", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.038461538461538464, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: Web Search (Confidence: 1.00)" + }, + "events": [], + "links": [] + }, + { + "span_id": "420b4ca1-c9f1-488f-ac2c-c53766717634", + "trace_id": "e30e9ba6-ffd1-4eb9-b7ae-6d19e04343d6", + "parent_span_id": "f6c8d34f-185f-49ba-a5a4-a9d25ccb4fc6", + "name": "agent.llm_model_execution", + "start_time": "2025-10-22 16:58:11.468000000", + "end_time": "2025-10-22 16:58:14.250000000", + "duration_ms": 2781.777, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What happened in tech news this week?", + "query_length": 37, + "query_type": "web_search_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 2.781723976135254, + "model.latency_ms": 2781.723976135254, + "model.start_time": 1761152291.468626, + "model.end_time": 1761152294.25035, + "model.performance_tier": "medium", + "model.prompt_tokens": 301, + "model.completion_tokens": 127, + "model.total_tokens": 428, + "model.tokens_per_second": 153.8614196346811, + "model.estimated_cost": 0.01284, + "model.efficiency_score": 1.191588785046729, + "model.response_length": 510, + "model.response_quality": "high", + "model.output_response": "Model Response: The web search results do not provide specific information about recent events in tech news for this week. They primarily discuss the grammatical usage of the word \"happened\" and its variations, such ...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [] + }, + { + "span_id": "56b8bf09-8ebf-44e1-b01f-daff23d3c42c", + "trace_id": "e30e9ba6-ffd1-4eb9-b7ae-6d19e04343d6", + "parent_span_id": "f6c8d34f-185f-49ba-a5a4-a9d25ccb4fc6", + "name": "agent.web_search_evaluation_metrics", + "start_time": "2025-10-22 16:58:14.250000000", + "end_time": "2025-10-22 16:58:14.250000000", + "duration_ms": 0.060000000000000005, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "web_search_evaluation_metrics", + "agent.capabilities": [ + "web_search_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What happened in tech news this week?", + "query_length": 37, + "query_type": "web_search_evaluation_query", + "web_search.results_count": 1, + "web_search.context_length": 724, + "web_search.context_synthesized": "Context from 1 web sources", + "web_search.search_effectiveness": 0.2, + "web_search.context_quality": "medium", + "web_search.source_diversity": 1, + "web_search.information_synthesis": "low", + "web_search.external_knowledge_utilization": 0.724, + "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nhappened\u7684\u7528\u6cd5\u5982\u4e0b\uff1a 1.happened\u662f \u4e0d\u53ca\u7269\u52a8\u8bcd,\u4f5c\u201c\u53d1\u751f\u201d\u89e3\u65f6,\u53ef\u6307\u5076\u7136\u7684\u4e8b\u6545,\u4e5f\u53ef\u6307\u6709\u8ba1\u5212\u6216\u65e0\u8ba1\u5212\u53d1\u751f\u7684\u4e8b\u3002\u4e00\u822c\u4ee5\u4e8b\u4ef6\u3001\u73af\u5883\u3001\u5f62\u52bf\u7b49\u4f5c\u4e3b\u8bed,\u8868\u793a\u201c\u67d0\u4eba\u53d1\u751f\u67d0\u4e8b\u201d\u65f6,\u987b\u4ee5\u4ecb\u8bcdto\u5f15\u51fa\u201c\u67d0\u4eba\u201d\u3002 2. \u2026 what happened\u8fd8\u662fwhat's happened\"What happened\"\u548c\"What's happened\"\u90fd\u7528\u4e8e\u8be2\u95ee\u8fc7\u53bb\u53d1\u751f\u4e86\u4ec0\u4e48\u4e8b\u60c5\uff0c\u4e3b\u8981\u7684\u533a\u522b\u5728\u4e8e\u8bed\u6cd5\u548c\u4f7f\u7528\u65b9\u5f0f\u3002\"What happened\"\u662f\u4e00\u4e2a\u8fc7\u53bb\u65f6\u6001\u7684\u7b80\u5355 \u63a5\u4e0b\u6765\u8ba9\u6211\u4eec\u770b\u4e0bwhat happen\u548cwhat happened\u7684\u7528\u6cd5\u533a\u522b\uff1a 1.\u8bcd\u6027\uff1awhat happen\u662f\u7528\u539f\u5f62\u52a8\u8bcdhappen\u6784\u6210\u7684\u73b0\u5728\u65f6\u77ed\u8bed\uff0c\u800cwhat happened\u662f\u7528\u52a8\u8bcdhappen\u7684\u8fc7\u53bb\u5f0fhappened\u6784\u6210\u7684\u8fc7\u53bb\u65f6\u77ed \u2026 What happened \u548c What's happened\u4e24\u8005\u7684\u533a\u522b\uff1a \u662f\u95ee What happened\uff1f\u53ea\u662f\u60f3\u77e5\u9053\u8fc7\u53bb\u6240\u53d1\u751f\u7684\u4e8b\u60c5\uff0c \u800c\u5982\u679c\u7528What's happened\uff1f\u5219\u5f3a\u8c03\u5bf9\u73b0\u5728\u7684\u5f71\u54cd\u3002\uff0c\u5373\u60f3\u77e5\u9053\u8fc7\u53bb\u6240\u53d1\u751f\u7684\u4ec0\u4e48\u4e8b\u5bfc\u81f4\u4e86\u8bf4 \u2026 it so happened that\u7684\u53e5\u5f0fhappen\u8868\u793a\u201c\u78b0\u5de7\u6216\u6070\u5de7\u53d1\u751f\u67d0\u4e8b\u201d\u65f6,\u53ef\u7528\u201cIt happens / happened that...\u201d\u8fd9\u4e00\u7ed3\u6784\u6765\u8868\u8fbe.\u4f8b\u5982\uff1a It happened that Brian and Peter were at home that day.\u78b0\u5de7\u90a3\u5929\u5e03\u83b1\u6069\u548c \u2026\nURL: https://duckduckgo.com/?q=What+happened+in+tech+news+this+week?\n\nUser Question: What happened in tech news this week?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", + "prompt.prompt_length": 1206, + "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nhappened\u7684\u7528\u6cd5\u5982\u4e0b\uff1a 1.happened\u662f \u4e0d\u53ca\u7269\u52a8\u8bcd,\u4f5c\u201c\u53d1\u751f\u201d\u89e3\u65f6,\u53ef\u6307\u5076\u7136\u7684\u4e8b\u6545,\u4e5f\u53ef\u6307\u6709\u8ba1\u5212\u6216\u65e0\u8ba1\u5212\u53d1\u751f\u7684\u4e8b\u3002\u4e00\u822c\u4ee5\u4e8b\u4ef6\u3001\u73af\u5883\u3001\u5f62\u52bf\u7b49\u4f5c\u4e3b\u8bed,\u8868\u793a\u201c\u67d0\u4eba\u53d1\u751f\u67d0\u4e8b\u201d\u65f6,\u987b\u4ee5\u4ecb\u8bcdto\u5f15\u51fa\u201c\u67d0\u4eba\u201d\u3002 2. \u2026 what happened\u8fd8\u662fwhat's happened\"What happened\"\u548c\"What's happene...", + "prompt.instruction_following": "web_search_optimized", + "response.answer_length": 510, + "response.answer_completeness": "complete", + "response.response_quality": "medium", + "response.source_citation": 1, + "response.context_utilization": 0.7044198895027625, + "output_response": "Web Search Answer: The web search results do not provide specific information about recent events in tech news for this week. They primarily discuss the grammatical usage of the word \"happened\" and its variations, such ...", + "evaluation.search_effectiveness": 0.2, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 1, + "evaluation.context_relevance": "medium", + "evaluation.overall_quality": "low", + "evaluation.ready_for_production": true, + "web_search.search_strategy": "duckduckgo_api", + "web_search.real_time_data": true, + "web_search.external_sources": 1, + "web_search.information_freshness": "current", + "web_search.knowledge_synthesis": "single_source" + }, + "events": [], + "links": [] + } + ], + "created_at": "2025-10-22 16:58:11.033000000", + "updated_at": "2025-10-22 16:58:11.033000000" + }, + { + "trace_id": "45efae74-ee25-4004-a509-6e0ff628a0cd", + "name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "status": "ok", + "status_message": "", + "start_time": "2025-10-22 16:58:04.883000000", + "end_time": "2025-10-22 16:58:10.029000000", + "duration_ms": 5146.244000000001, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "spans": [ + { + "span_id": "82eb19e8-f6cf-43fd-ba56-640a6fd3ed0e", + "trace_id": "45efae74-ee25-4004-a509-6e0ff628a0cd", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-22 16:58:04.883000000", + "end_time": "2025-10-22 16:58:10.029000000", + "duration_ms": 5146.146, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What are the current trends in observability tools?", + "query.length": 51, + "processing.mode": "RAG", + "processing.time_seconds": 5.146108150482178, + "processing.response_length": 815, + "processing.sources_count": 5, + "output_response": "Final Answer: The provided context does not explicitly outline current trends in observability tools. However, it highlights the unique challenges faced by AI applications that traditional monitoring tools cannot a...", + "final_answer_mode": "RAG", + "query_processed.input_query": "What are the current trends in observability tools?", + "query_processed.output_response": "Successfully processed query using RAG, generated 815 character response", + "query_processed.mode": "RAG", + "query_processed.processing_time": 5.146108150482178, + "query_processed.response_length": 815 + }, + "events": [], + "links": [] + }, + { + "span_id": "e81617c6-c2a1-4b08-a5fc-dde3439fe2f4", + "trace_id": "45efae74-ee25-4004-a509-6e0ff628a0cd", + "parent_span_id": "8001ba0e-95c7-42da-9087-603a38e1a500", + "name": "agent.llm-rag", + "start_time": "2025-10-22 16:58:04.883000000", + "end_time": "2025-10-22 16:58:10.029000000", + "duration_ms": 5145.655, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "rag_agent", + "agent.operation": "llm-rag", + "agent.capabilities": [ + "document_retrieval", + "context_generation", + "response_generation" + ], + "agent.id": "noveum_rag_agent", + "input_query": "What are the current trends in observability tools?", + "query_length": 51, + "query_type": "rag_query", + "output_response": "RAG Answer: The provided context does not explicitly outline current trends in observability tools. However, it highlights the unique challenges faced by AI applications that traditional monitoring tools cannot a...", + "rag.context_length": 2943, + "rag.sources_count": 5, + "rag.answer_length": 815, + "rag.mode": "retrieval_augmented_generation" + }, + "events": [], + "links": [] + }, + { + "span_id": "8001ba0e-95c7-42da-9087-603a38e1a500", + "trace_id": "45efae74-ee25-4004-a509-6e0ff628a0cd", + "parent_span_id": "82eb19e8-f6cf-43fd-ba56-640a6fd3ed0e", + "name": "agent.query_routing", + "start_time": "2025-10-22 16:58:04.883000000", + "end_time": "2025-10-22 16:58:10.029000000", + "duration_ms": 5146.036999999999, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What are the current trends in observability tools?", + "query_length": 51, + "query_type": "routing_query", + "output_response": "Routed to RAG for query processing", + "router.classification": "RAG", + "router.confidence_score": 0.3333333333333333, + "router.rag_keyword_score": 2, + "router.web_keyword_score": 1, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [] + }, + { + "span_id": "79b7d138-05e1-4f18-8b63-dcfe9732180b", + "trace_id": "45efae74-ee25-4004-a509-6e0ff628a0cd", + "parent_span_id": "8001ba0e-95c7-42da-9087-603a38e1a500", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-22 16:58:04.883000000", + "end_time": "2025-10-22 16:58:04.883000000", + "duration_ms": 0.051, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What are the current trends in observability tools?", + "query_length": 51, + "query_type": "routing_evaluation_query", + "classification.mode": "RAG", + "classification.rag_keyword_score": 2, + "classification.web_keyword_score": 1, + "classification.confidence_score": 0.3333333333333333, + "classification.confidence_level": "medium", + "classification.method": "keyword_based", + "query.complexity": "complex", + "query.intent": "documentation", + "query.keyword_density": 0.375, + "query.domain_affinity": "noveum", + "routing.decision": "Routed to RAG based on analysis", + "routing.rationale": "RAG score: 2, Web score: 1, Confidence: 0.33", + "routing.expected_performance": "medium", + "routing.alternative_mode": "Web Search", + "routing.decision_confidence": 0.3333333333333333, + "evaluation.routing_accuracy": "medium", + "evaluation.keyword_coverage": 0.057692307692307696, + "evaluation.query_understanding": "ambiguous", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: RAG (Confidence: 0.33)" + }, + "events": [], + "links": [] + }, + { + "span_id": "28c233a0-d42a-4200-86b0-dfbd2d5be06e", + "trace_id": "45efae74-ee25-4004-a509-6e0ff628a0cd", + "parent_span_id": "e81617c6-c2a1-4b08-a5fc-dde3439fe2f4", + "name": "agent.llm_model_execution", + "start_time": "2025-10-22 16:58:05.635000000", + "end_time": "2025-10-22 16:58:09.222000000", + "duration_ms": 3586.924, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What are the current trends in observability tools?", + "query_length": 51, + "query_type": "rag_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 3.5868873596191406, + "model.latency_ms": 3586.8873596191406, + "model.start_time": 1761152285.6352398, + "model.end_time": 1761152289.2221272, + "model.performance_tier": "medium", + "model.prompt_tokens": 872, + "model.completion_tokens": 203, + "model.total_tokens": 1075, + "model.tokens_per_second": 299.7027484337129, + "model.estimated_cost": 0.03225, + "model.efficiency_score": 0.7581395348837209, + "model.response_length": 815, + "model.response_quality": "high", + "model.output_response": "Model Response: The provided context does not explicitly outline current trends in observability tools. However, it highlights the unique challenges faced by AI applications that traditional monitoring tools cannot a...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [] + }, + { + "span_id": "024c05b6-edf9-40ef-931b-9660d48b4948", + "trace_id": "45efae74-ee25-4004-a509-6e0ff628a0cd", + "parent_span_id": "e81617c6-c2a1-4b08-a5fc-dde3439fe2f4", + "name": "agent.rag_evaluation_metrics", + "start_time": "2025-10-22 16:58:09.222000000", + "end_time": "2025-10-22 16:58:09.222000000", + "duration_ms": 0.020999999999999998, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "rag_evaluation_metrics", + "agent.capabilities": [ + "retrieval_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What are the current trends in observability tools?", + "query_length": 51, + "query_type": "rag_evaluation_query", + "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/blog/from-logs-to-intelligent-choices-inside-noveum-ais-evaluation-process):\nFrom Development to Production - Inside Noveum.ai's AI Observability Platform\nShashank Agarwal\n3/3/2025\nIntroduction\nEvery AI application tells a story\u2014but most developers never get to hear it...", + "retrieval.context_length": 2943, + "retrieval.sources_count": 5, + "retrieval.context_quality": "high", + "retrieval.effectiveness": 1, + "retrieval.context_utilization": 2.943, + "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/blog/from-logs-to-intelligent-choices-inside-noveum-ais-evaluation-process):\nFrom Development to Production - Inside Noveum.ai's AI Observability Platform\nShashank Agarwal\n3/3/2025\nIntroduction\nEvery AI application tells a story\u2014but most developers never get to hear it. When your RAG pipeline returns irrelevant results, when your multi-agent system gets stuck in loops, or when your LLM costs suddenly spike, you're left guessing what went wrong and where.\nThat's exactly the challenge Noveum.ai solves. Rather than flying blind, our platform provides comprehensive tracing a...\n\nSource 2 (https://noveum.ai/docs/getting-started/tracing-concepts):\n- Explore Framework Integrations - Framework-specific guidance\n- Learn Advanced Patterns - Custom instrumentation techniques\n- Master the Dashboard - Analyze your traces effectively\nRemember: Good observability is not about collecting all possible data, but about collecting the right data that helps you understand, debug, and optimize your AI applications.\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to O...\n\nSource 3 (https://noveum.ai/en/docs/getting-started/tracing-concepts):\n- Explore Framework Integrations - Framework-specific guidance\n- Learn Advanced Patterns - Custom instrumentation techniques\n- Master the Dashboard - Analyze your traces effectively\nRemember: Good observability is not about collecting all possible data, but about collecting the right data that helps you understand, debug, and optimize your AI applications.\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to O...\n\nSource 4 (https://noveum.ai/en/blog/from-logs-to-intelligent-choices-inside-noveum-ais-evaluation-process):\nThe AI Observability Challenge\nWhy Traditional Monitoring Falls Short\nTraditional application monitoring tools weren't built for AI applications. They can tell you if your API is responding, but they can't answer the questions that matter most for AI systems:\n- Why did my RAG pipeline retrieve irrelevant documents?\n- Which LLM calls are driving my costs?\n- How are my agents communicating with each other?\n- What's causing hallucinations in my responses?\n- Why is my embedding generation so slow?\nW...\n\nSource 5 (https://noveum.ai/en/docs/advanced/rag-observability):\n\ud83d\udd27 Best Practices for RAG Observability\n1. Comprehensive Pipeline Tracing\n2. Quality Monitoring\n3. A/B Testing for RAG Components\n\ud83c\udfaf Advanced RAG Patterns\nMulti-Modal RAG\nConversational RAG\nRAG observability with Noveum.ai provides the deep insights needed to build, optimize, and scale retrieval-augmented generation systems. By implementing comprehensive tracing across retrieval, context preparation, and generation stages, you can ensure your RAG pipeline delivers accurate, relevant, and cost-effe...\n\nUser Question: What are the current trends in observability tools?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", + "prompt.prompt_length": 3490, + "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/blog/from-logs-to-intelligent-choices-inside-noveum-ais-evaluation-process):\nFrom Development to Production - Inside Noveum.ai's AI Observability Platform\nShashank Agarw...", + "prompt.instruction_following": "rag_optimized", + "response.answer_length": 815, + "response.answer_completeness": "complete", + "response.response_quality": "high", + "response.source_citation": 5, + "response.context_utilization": 0.2769283044512402, + "output_response": "RAG Answer: The provided context does not explicitly outline current trends in observability tools. However, it highlights the unique challenges faced by AI applications that traditional monitoring tools cannot a...", + "evaluation.retrieval_effectiveness": 1, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 5, + "evaluation.context_relevance": "high", + "evaluation.overall_quality": "high", + "evaluation.ready_for_production": true, + "rag.retrieval_strategy": "semantic_similarity", + "rag.vector_search_results": 5, + "rag.context_synthesis": "multi_source", + "rag.document_coverage": 1, + "rag.information_density": 0.2769283044512402 + }, + "events": [], + "links": [] + } + ], + "created_at": "2025-10-22 16:58:04.883000000", + "updated_at": "2025-10-22 16:58:04.883000000" + }, + { + "trace_id": "c5a64d08-d00f-46cf-ab3c-d8d473c10eca", + "name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "status": "ok", + "status_message": "", + "start_time": "2025-10-22 16:57:57.908000000", + "end_time": "2025-10-22 16:58:03.875000000", + "duration_ms": 5967.0779999999995, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "spans": [ + { + "span_id": "99a5e8f3-a9f5-4953-94ea-ad0fa0cf6ec2", + "trace_id": "c5a64d08-d00f-46cf-ab3c-d8d473c10eca", + "parent_span_id": "2a351fd9-a7ee-42da-9e2e-86a643ceed0e", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-22 16:57:57.908000000", + "end_time": "2025-10-22 16:57:57.908000000", + "duration_ms": 0.026, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: Tell me about recent developments in machine learning", + "query_length": 53, + "query_type": "routing_evaluation_query", + "classification.mode": "Web Search", + "classification.rag_keyword_score": 0, + "classification.web_keyword_score": 1, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "complex", + "query.intent": "general_knowledge", + "query.keyword_density": 0.125, + "query.domain_affinity": "general", + "routing.decision": "Routed to Web Search based on analysis", + "routing.rationale": "RAG score: 0, Web score: 1, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "RAG", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.019230769230769232, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: Web Search (Confidence: 1.00)" + }, + "events": [], + "links": [] + }, + { + "span_id": "6e891e51-ca6d-4e6c-80d0-0791ee22d5b9", + "trace_id": "c5a64d08-d00f-46cf-ab3c-d8d473c10eca", + "parent_span_id": "2a351fd9-a7ee-42da-9e2e-86a643ceed0e", + "name": "agent.web_search_generation", + "start_time": "2025-10-22 16:57:57.908000000", + "end_time": "2025-10-22 16:58:03.874000000", + "duration_ms": 5966.728, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "web_search_agent", + "agent.operation": "web_search_generation", + "agent.capabilities": [ + "web_search", + "content_synthesis", + "response_generation" + ], + "agent.id": "noveum_web_search_agent", + "input_query": "Tell me about recent developments in machine learning", + "query_length": 53, + "query_type": "web_search_query", + "output_response": "Web Search Answer: The web search results did not provide specific information about recent developments in machine learning. Therefore, I cannot summarize any recent advancements based on the provided data.\n\nHowever, i...", + "web_search.results_count": 1, + "web_search.context_length": 768, + "web_search.response_length": 687, + "web_search.mode": "external_web_search" + }, + "events": [], + "links": [] + }, + { + "span_id": "6d622376-d48b-4a64-b0a2-1402117febaf", + "trace_id": "c5a64d08-d00f-46cf-ab3c-d8d473c10eca", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-22 16:57:57.908000000", + "end_time": "2025-10-22 16:58:03.875000000", + "duration_ms": 5967.035, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "Tell me about recent developments in machine learning", + "query.length": 53, + "processing.mode": "Web Search", + "processing.time_seconds": 5.966889142990112, + "processing.response_length": 687, + "processing.sources_count": 1, + "output_response": "Final Answer: The web search results did not provide specific information about recent developments in machine learning. Therefore, I cannot summarize any recent advancements based on the provided data.\n\nHowever, i...", + "final_answer_mode": "Web Search", + "query_processed.input_query": "Tell me about recent developments in machine learning", + "query_processed.output_response": "Successfully processed query using Web Search, generated 687 character response", + "query_processed.mode": "Web Search", + "query_processed.processing_time": 5.966889142990112, + "query_processed.response_length": 687 + }, + "events": [], + "links": [] + }, + { + "span_id": "2a351fd9-a7ee-42da-9e2e-86a643ceed0e", + "trace_id": "c5a64d08-d00f-46cf-ab3c-d8d473c10eca", + "parent_span_id": "6d622376-d48b-4a64-b0a2-1402117febaf", + "name": "agent.query_routing", + "start_time": "2025-10-22 16:57:57.908000000", + "end_time": "2025-10-22 16:58:03.875000000", + "duration_ms": 5966.8640000000005, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "Tell me about recent developments in machine learning", + "query_length": 53, + "query_type": "routing_query", + "output_response": "Routed to Web Search for query processing", + "router.classification": "Web Search", + "router.confidence_score": 1, + "router.rag_keyword_score": 0, + "router.web_keyword_score": 1, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [] + }, + { + "span_id": "ba457b3b-9be8-4e24-af63-4118833711e6", + "trace_id": "c5a64d08-d00f-46cf-ab3c-d8d473c10eca", + "parent_span_id": "6e891e51-ca6d-4e6c-80d0-0791ee22d5b9", + "name": "agent.llm_model_execution", + "start_time": "2025-10-22 16:57:58.324000000", + "end_time": "2025-10-22 16:58:03.874000000", + "duration_ms": 5550.059, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: Tell me about recent developments in machine learning", + "query_length": 53, + "query_type": "web_search_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 5.550010919570923, + "model.latency_ms": 5550.010919570923, + "model.start_time": 1761152278.3248482, + "model.end_time": 1761152283.874859, + "model.performance_tier": "slow", + "model.prompt_tokens": 316, + "model.completion_tokens": 171, + "model.total_tokens": 487, + "model.tokens_per_second": 87.74757510524871, + "model.estimated_cost": 0.01461, + "model.efficiency_score": 1.4106776180698153, + "model.response_length": 687, + "model.response_quality": "high", + "model.output_response": "Model Response: The web search results did not provide specific information about recent developments in machine learning. Therefore, I cannot summarize any recent advancements based on the provided data.\n\nHowever, i...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [] + }, + { + "span_id": "7b70a916-1c2b-4898-a052-cfd98a979167", + "trace_id": "c5a64d08-d00f-46cf-ab3c-d8d473c10eca", + "parent_span_id": "6e891e51-ca6d-4e6c-80d0-0791ee22d5b9", + "name": "agent.web_search_evaluation_metrics", + "start_time": "2025-10-22 16:58:03.874000000", + "end_time": "2025-10-22 16:58:03.874000000", + "duration_ms": 0.02, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "web_search_evaluation_metrics", + "agent.capabilities": [ + "web_search_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: Tell me about recent developments in machine learning", + "query_length": 53, + "query_type": "web_search_evaluation_query", + "web_search.results_count": 1, + "web_search.context_length": 768, + "web_search.context_synthesized": "Context from 1 web sources", + "web_search.search_effectiveness": 0.2, + "web_search.context_quality": "medium", + "web_search.source_diversity": 1, + "web_search.information_synthesis": "low", + "web_search.external_knowledge_utilization": 0.768, + "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nFeb 3, 2024 \u00b7 tell sb sth\u4e2d\u7684tell\u66f4\u591a\u662f\u544a\u77e5\u7684\u610f\u601d\uff0c\u5c31\u662f\u544a\u8bc9\u522b\u4eba\u6709\u8fd9\u4e48\u4e00\u4e2a\u4e8b\u7684\u5b58\u5728 tell sb about sth \u7684\u610f\u601d\u5305\u542b\u66f4\u591a\u4e00\u4e9b\uff0c\u9664\u4e86\u544a\u77e5\u6709\u8fd9\u4e48\u4e00\u4e2a\u4e8b\u60c5\u7684\u5b58\u5728\uff0c\u8fd8\u4f5c\u4e86\u4e00\u4e9b\u8bf4\u660e \u6bd4\u5982\uff1a tell me he \u2026 \u4e00\u3001Say\u548cTell\u7684\u7528\u6cd5 \u6211\u4eec\u5148\u5c06say\u3001tell\u3001speak\u3001talk\u8fd9\u56db\u4e2a\u8bcd\u5206\u6210\u4e24\u7ec4\uff0c\u5206\u522b\u4e3a\uff1a \u7b2c\u2460\u7ec4\uff1asay & tell\uff0c\u8fd9\u4e24\u4e2a\u8bcd\u4e4b\u540e\u53ef\u4ee5\u8868\u793a\u8c08\u8bdd\u7684 \u201c\u7ec6\u8282\u201d\u3002 \u7b2c\u2461\u7ec4\uff1aspeak & talk\uff0c\u4e0d\u80fd\u8868\u793a\u8c08\u8bdd\u7684\u201c\u7ec6\u8282\u201d\u3002 \u2026 tell of 1\u3001Tell of a turning point in your career and what you learned. \u8bb2\u8ff0\u4f60 \u804c\u4e1a\u751f\u6daf \u4e00\u4e2a\u8f6c\u6298\u70b9\uff0c\u4ee5\u53ca\u4f60\u5b66\u5230\u4e86\u4ec0\u4e48\u3002 2\u3001I have made the Sovereign Lord my refuge; I will tell of all your \u2026 tell sb to do sth \u548c tell sb do sth \u90fd\u8868\u793a\u544a\u8bc9\u67d0\u4eba\u505a\u67d0\u4e8b\uff0ctell sb to do sth \u5f3a\u8c03\u544a\u8bc9\u67d0\u4eba\u5e94\u8be5\u505a\u4ec0\u4e48\u5177\u4f53\u52a8\u4f5c\u6216\u884c\u4e3a\uff0c\u800c tell sb do sth \u5f3a\u8c03\u544a\u8bc9\u67d0\u4eba\u6b63\u5728\u8fdb\u884c\u7684\u5177\u4f53\u884c\u52a8\u6216\u72b6\u6001\u3002 \u5b83\u4eec\u5728\u91ca\u4e49\u3001 \u2026 Aug 23, 2023 \u00b7 tell\u7684\u8fc7\u53bb\u5f0f\u548c\u8fc7\u53bb\u5206\u8bcd\u90fd\u662ftold\u3002 \u8bfb\u97f3\uff1a\u82f1[tel]\uff0c\u7f8e[tel]\u3002 \u91ca\u4e49\uff1a vt.\u544a\u8bc9\uff0c\u8bf4\uff1b\u8fa8\u522b\uff1b\u5429\u5490\uff1b\u65ad\u5b9a\u3002 vi.\u8bb2\u8ff0\uff1b\u544a\u53d1\uff0c\u6cc4\u5bc6\uff1b\u8bc6\u522b\u3002 \u53d8\u5f62\uff1a\u8fc7\u53bb\u5f0ftold\uff0c\u8fc7\u53bb\u5206\u8bcdtold\uff0c\u73b0\u5728\u5206 \u2026\nURL: https://duckduckgo.com/?q=Tell+me+about+recent+developments+in+machine+learning\n\nUser Question: Tell me about recent developments in machine learning\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", + "prompt.prompt_length": 1266, + "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nFeb 3, 2024 \u00b7 tell sb sth\u4e2d\u7684tell\u66f4\u591a\u662f\u544a\u77e5\u7684\u610f\u601d\uff0c\u5c31\u662f\u544a\u8bc9\u522b\u4eba\u6709\u8fd9\u4e48\u4e00\u4e2a\u4e8b\u7684\u5b58\u5728 tell sb about sth \u7684\u610f\u601d\u5305\u542b\u66f4\u591a\u4e00\u4e9b\uff0c\u9664\u4e86\u544a\u77e5\u6709\u8fd9\u4e48\u4e00\u4e2a\u4e8b\u60c5\u7684\u5b58\u5728\uff0c\u8fd8\u4f5c\u4e86\u4e00\u4e9b\u8bf4\u660e \u6bd4\u5982\uff1a tell me he \u2026 \u4e00\u3001Say\u548cTell\u7684\u7528\u6cd5 \u6211\u4eec\u5148\u5c06say\u3001tell\u3001speak\u3001talk\u8fd9\u56db\u4e2a\u8bcd\u5206\u6210\u4e24\u7ec4\uff0c\u5206\u522b...", + "prompt.instruction_following": "web_search_optimized", + "response.answer_length": 687, + "response.answer_completeness": "complete", + "response.response_quality": "medium", + "response.source_citation": 1, + "response.context_utilization": 0.89453125, + "output_response": "Web Search Answer: The web search results did not provide specific information about recent developments in machine learning. Therefore, I cannot summarize any recent advancements based on the provided data.\n\nHowever, i...", + "evaluation.search_effectiveness": 0.2, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 1, + "evaluation.context_relevance": "medium", + "evaluation.overall_quality": "low", + "evaluation.ready_for_production": true, + "web_search.search_strategy": "duckduckgo_api", + "web_search.real_time_data": true, + "web_search.external_sources": 1, + "web_search.information_freshness": "current", + "web_search.knowledge_synthesis": "single_source" + }, + "events": [], + "links": [] + } + ], + "created_at": "2025-10-22 16:57:57.908000000", + "updated_at": "2025-10-22 16:57:57.908000000" + }, + { + "trace_id": "b1b13e60-5b32-44a0-9769-197a944989d0", + "name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "status": "ok", + "status_message": "", + "start_time": "2025-10-22 16:57:51.967000000", + "end_time": "2025-10-22 16:57:56.899000000", + "duration_ms": 4932.393, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "spans": [ + { + "span_id": "1e1a341f-66c1-48b0-95eb-fb092928c473", + "trace_id": "b1b13e60-5b32-44a0-9769-197a944989d0", + "parent_span_id": "1c5c4291-bbb5-4f85-b6d9-b6e8df66f742", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-22 16:57:51.967000000", + "end_time": "2025-10-22 16:57:51.967000000", + "duration_ms": 0.033, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What's the weather like today?", + "query_length": 30, + "query_type": "routing_evaluation_query", + "classification.mode": "Web Search", + "classification.rag_keyword_score": 0, + "classification.web_keyword_score": 2, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "general_knowledge", + "query.keyword_density": 0.4, + "query.domain_affinity": "general", + "routing.decision": "Routed to Web Search based on analysis", + "routing.rationale": "RAG score: 0, Web score: 2, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "RAG", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.038461538461538464, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: Web Search (Confidence: 1.00)" + }, + "events": [], + "links": [] + }, + { + "span_id": "1c5c4291-bbb5-4f85-b6d9-b6e8df66f742", + "trace_id": "b1b13e60-5b32-44a0-9769-197a944989d0", + "parent_span_id": "49e7cace-ba21-4807-a238-9a3880487583", + "name": "agent.query_routing", + "start_time": "2025-10-22 16:57:51.967000000", + "end_time": "2025-10-22 16:57:56.899000000", + "duration_ms": 4931.972, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What's the weather like today?", + "query_length": 30, + "query_type": "routing_query", + "output_response": "Routed to Web Search for query processing", + "router.classification": "Web Search", + "router.confidence_score": 1, + "router.rag_keyword_score": 0, + "router.web_keyword_score": 2, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [] + }, + { + "span_id": "49e7cace-ba21-4807-a238-9a3880487583", + "trace_id": "b1b13e60-5b32-44a0-9769-197a944989d0", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-22 16:57:51.967000000", + "end_time": "2025-10-22 16:57:56.899000000", + "duration_ms": 4932.316, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What's the weather like today?", + "query.length": 30, + "processing.mode": "Web Search", + "processing.time_seconds": 4.932007789611816, + "processing.response_length": 260, + "processing.sources_count": 1, + "output_response": "Final Answer: The web search results do not provide specific information about today's weather. To find out the current weather conditions, I recommend checking a reliable weather website or app for detailed update...", + "final_answer_mode": "Web Search", + "query_processed.input_query": "What's the weather like today?", + "query_processed.output_response": "Successfully processed query using Web Search, generated 260 character response", + "query_processed.mode": "Web Search", + "query_processed.processing_time": 4.932007789611816, + "query_processed.response_length": 260 + }, + "events": [], + "links": [] + }, + { + "span_id": "58d217aa-4350-43e8-8813-468149dc4107", + "trace_id": "b1b13e60-5b32-44a0-9769-197a944989d0", + "parent_span_id": "1c5c4291-bbb5-4f85-b6d9-b6e8df66f742", + "name": "agent.web_search_generation", + "start_time": "2025-10-22 16:57:51.967000000", + "end_time": "2025-10-22 16:57:56.899000000", + "duration_ms": 4931.796, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "web_search_agent", + "agent.operation": "web_search_generation", + "agent.capabilities": [ + "web_search", + "content_synthesis", + "response_generation" + ], + "agent.id": "noveum_web_search_agent", + "input_query": "What's the weather like today?", + "query_length": 30, + "query_type": "web_search_query", + "output_response": "Web Search Answer: The web search results do not provide specific information about today's weather. To find out the current weather conditions, I recommend checking a reliable weather website or app for detailed update...", + "web_search.results_count": 1, + "web_search.context_length": 995, + "web_search.response_length": 260, + "web_search.mode": "external_web_search" + }, + "events": [], + "links": [] + }, + { + "span_id": "d89dd84e-0b5e-4c92-942b-18c4735713d4", + "trace_id": "b1b13e60-5b32-44a0-9769-197a944989d0", + "parent_span_id": "58d217aa-4350-43e8-8813-468149dc4107", + "name": "agent.llm_model_execution", + "start_time": "2025-10-22 16:57:55.269000000", + "end_time": "2025-10-22 16:57:56.899000000", + "duration_ms": 1629.7140000000002, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What's the weather like today?", + "query_length": 30, + "query_type": "web_search_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 1.629619836807251, + "model.latency_ms": 1629.619836807251, + "model.start_time": 1761152275.269459, + "model.end_time": 1761152276.8990788, + "model.performance_tier": "fast", + "model.prompt_tokens": 367, + "model.completion_tokens": 65, + "model.total_tokens": 432, + "model.tokens_per_second": 265.0925020932329, + "model.estimated_cost": 0.012960000000000001, + "model.efficiency_score": 0.6018518518518519, + "model.response_length": 260, + "model.response_quality": "high", + "model.output_response": "Model Response: The web search results do not provide specific information about today's weather. To find out the current weather conditions, I recommend checking a reliable weather website or app for detailed update...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [] + }, + { + "span_id": "d076719d-da79-43fb-92ea-6015f856146b", + "trace_id": "b1b13e60-5b32-44a0-9769-197a944989d0", + "parent_span_id": "58d217aa-4350-43e8-8813-468149dc4107", + "name": "agent.web_search_evaluation_metrics", + "start_time": "2025-10-22 16:57:56.899000000", + "end_time": "2025-10-22 16:57:56.899000000", + "duration_ms": 0.049, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "web_search_evaluation_metrics", + "agent.capabilities": [ + "web_search_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What's the weather like today?", + "query_length": 30, + "query_type": "web_search_evaluation_query", + "web_search.results_count": 1, + "web_search.context_length": 995, + "web_search.context_synthesized": "Context from 1 web sources", + "web_search.search_effectiveness": 0.2, + "web_search.context_quality": "high", + "web_search.source_diversity": 1, + "web_search.information_synthesis": "low", + "web_search.external_knowledge_utilization": 0.995, + "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nLog in to WhatsApp Web for simple, reliable and private messaging on your desktop. Send and receive messages and files with ease, all for free. 2 days ago \u00b7 More videos you may like 00:13 Wanna be yours #kimpolin #colinbridgerton #benedictbri... 2 hours ago \u00b7 101 views 00:14 Viscount & Vicountess's Marital bliss #bridgerton \u2026 Nov 15, 2024 \u00b7 Learn the correct usage of \"what's the weather like today\" and \"how's the weather today\" in English. Discover differences, examples, alternatives and tips for choosing the right \u2026 Nov 15, 2024 \u00b7 Learn the correct usage of \"how's the weather today\" and \"what's the weather today\" in English. Discover differences, examples, alternatives and tips for choosing the right \u2026 2 days ago \u00b7 Get the latest hourly weather updates for Thane today. Detailed forecast including temperature, wind, rain, snow, and UV index. Stay informed about today's weather conditions in \u2026\nURL: https://duckduckgo.com/?q=What's+the+weather+like+today?\n\nUser Question: What's the weather like today?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", + "prompt.prompt_length": 1470, + "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nLog in to WhatsApp Web for simple, reliable and private messaging on your desktop. Send and receive messages and files with ease, all for free. 2 days ago \u00b7 More videos you...", + "prompt.instruction_following": "web_search_optimized", + "response.answer_length": 260, + "response.answer_completeness": "complete", + "response.response_quality": "medium", + "response.source_citation": 1, + "response.context_utilization": 0.2613065326633166, + "output_response": "Web Search Answer: The web search results do not provide specific information about today's weather. To find out the current weather conditions, I recommend checking a reliable weather website or app for detailed update...", + "evaluation.search_effectiveness": 0.2, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 1, + "evaluation.context_relevance": "high", + "evaluation.overall_quality": "low", + "evaluation.ready_for_production": true, + "web_search.search_strategy": "duckduckgo_api", + "web_search.real_time_data": true, + "web_search.external_sources": 1, + "web_search.information_freshness": "current", + "web_search.knowledge_synthesis": "single_source" + }, + "events": [], + "links": [] + } + ], + "created_at": "2025-10-22 16:57:51.967000000", + "updated_at": "2025-10-22 16:57:51.967000000" + }, + { + "trace_id": "e4f3cc20-c137-4905-80fe-2d29b03c2231", + "name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "status": "ok", + "status_message": "", + "start_time": "2025-10-22 16:57:47.477000000", + "end_time": "2025-10-22 16:57:50.959000000", + "duration_ms": 3481.856, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "spans": [ + { + "span_id": "41167ca4-e1ad-48a5-b5eb-aa272ad88a85", + "trace_id": "e4f3cc20-c137-4905-80fe-2d29b03c2231", + "parent_span_id": "56959f47-5219-4674-8c0d-e5fa928dcd45", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-22 16:57:47.477000000", + "end_time": "2025-10-22 16:57:47.477000000", + "duration_ms": 0.061, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What are the latest AI news today?", + "query_length": 34, + "query_type": "routing_evaluation_query", + "classification.mode": "Web Search", + "classification.rag_keyword_score": 0, + "classification.web_keyword_score": 3, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "general_knowledge", + "query.keyword_density": 0.42857142857142855, + "query.domain_affinity": "general", + "routing.decision": "Routed to Web Search based on analysis", + "routing.rationale": "RAG score: 0, Web score: 3, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "RAG", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.057692307692307696, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: Web Search (Confidence: 1.00)" + }, + "events": [], + "links": [] + }, + { + "span_id": "73005751-9eb1-4540-aa49-2ac580cd7418", + "trace_id": "e4f3cc20-c137-4905-80fe-2d29b03c2231", + "parent_span_id": "56959f47-5219-4674-8c0d-e5fa928dcd45", + "name": "agent.web_search_generation", + "start_time": "2025-10-22 16:57:47.477000000", + "end_time": "2025-10-22 16:57:50.958000000", + "duration_ms": 3480.8469999999998, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "web_search_agent", + "agent.operation": "web_search_generation", + "agent.capabilities": [ + "web_search", + "content_synthesis", + "response_generation" + ], + "agent.id": "noveum_web_search_agent", + "input_query": "What are the latest AI news today?", + "query_length": 34, + "query_type": "web_search_query", + "output_response": "Web Search Answer: The provided web search results do not contain specific information about the latest AI news today. They primarily focus on general news coverage from India and around the world, including politics, b...", + "web_search.results_count": 1, + "web_search.context_length": 887, + "web_search.response_length": 503, + "web_search.mode": "external_web_search" + }, + "events": [], + "links": [] + }, + { + "span_id": "9e1b87be-bc29-4084-b37c-6e1220dde8eb", + "trace_id": "e4f3cc20-c137-4905-80fe-2d29b03c2231", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-22 16:57:47.477000000", + "end_time": "2025-10-22 16:57:50.959000000", + "duration_ms": 3481.7580000000003, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What are the latest AI news today?", + "query.length": 34, + "processing.mode": "Web Search", + "processing.time_seconds": 3.4813451766967773, + "processing.response_length": 503, + "processing.sources_count": 1, + "output_response": "Final Answer: The provided web search results do not contain specific information about the latest AI news today. They primarily focus on general news coverage from India and around the world, including politics, b...", + "final_answer_mode": "Web Search", + "query_processed.input_query": "What are the latest AI news today?", + "query_processed.output_response": "Successfully processed query using Web Search, generated 503 character response", + "query_processed.mode": "Web Search", + "query_processed.processing_time": 3.4813451766967773, + "query_processed.response_length": 503 + }, + "events": [], + "links": [] + }, + { + "span_id": "56959f47-5219-4674-8c0d-e5fa928dcd45", + "trace_id": "e4f3cc20-c137-4905-80fe-2d29b03c2231", + "parent_span_id": "9e1b87be-bc29-4084-b37c-6e1220dde8eb", + "name": "agent.query_routing", + "start_time": "2025-10-22 16:57:47.477000000", + "end_time": "2025-10-22 16:57:50.958000000", + "duration_ms": 3481.254, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What are the latest AI news today?", + "query_length": 34, + "query_type": "routing_query", + "output_response": "Routed to Web Search for query processing", + "router.classification": "Web Search", + "router.confidence_score": 1, + "router.rag_keyword_score": 0, + "router.web_keyword_score": 3, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [] + }, + { + "span_id": "3dbf95ba-3ef1-4294-93d4-217f4ada5ffe", + "trace_id": "e4f3cc20-c137-4905-80fe-2d29b03c2231", + "parent_span_id": "73005751-9eb1-4540-aa49-2ac580cd7418", + "name": "agent.llm_model_execution", + "start_time": "2025-10-22 16:57:47.864000000", + "end_time": "2025-10-22 16:57:50.958000000", + "duration_ms": 3093.442, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What are the latest AI news today?", + "query_length": 34, + "query_type": "web_search_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 3.0933361053466797, + "model.latency_ms": 3093.3361053466797, + "model.start_time": 1761152267.864976, + "model.end_time": 1761152270.958312, + "model.performance_tier": "medium", + "model.prompt_tokens": 341, + "model.completion_tokens": 125, + "model.total_tokens": 466, + "model.tokens_per_second": 150.6464167261171, + "model.estimated_cost": 0.013980000000000001, + "model.efficiency_score": 1.0793991416309012, + "model.response_length": 503, + "model.response_quality": "high", + "model.output_response": "Model Response: The provided web search results do not contain specific information about the latest AI news today. They primarily focus on general news coverage from India and around the world, including politics, b...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [] + }, + { + "span_id": "f7fe795b-ce1b-4fcc-83f5-06a5f074f38d", + "trace_id": "e4f3cc20-c137-4905-80fe-2d29b03c2231", + "parent_span_id": "73005751-9eb1-4540-aa49-2ac580cd7418", + "name": "agent.web_search_evaluation_metrics", + "start_time": "2025-10-22 16:57:50.958000000", + "end_time": "2025-10-22 16:57:50.958000000", + "duration_ms": 0.026, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "web_search_evaluation_metrics", + "agent.capabilities": [ + "web_search_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What are the latest AI news today?", + "query_length": 34, + "query_type": "web_search_evaluation_query", + "web_search.results_count": 1, + "web_search.context_length": 887, + "web_search.context_synthesized": "Context from 1 web sources", + "web_search.search_effectiveness": 0.2, + "web_search.context_quality": "high", + "web_search.source_diversity": 1, + "web_search.information_synthesis": "low", + "web_search.external_knowledge_utilization": 0.887, + "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\n3 days ago \u00b7 Latest news headlines from India & around the world. Check out today\u2019s news coverage live with videos & photos on NDTV.com. 6 days ago \u00b7 India News | Latest India News | Read latest and breaking news from India. Today's top India news headlines, news on Indian politics, elections, government, business, \u2026 2 days ago \u00b7 Top News Stories of the day, Latest News Headlines, News Specials, Breaking News and Latest India News, World current affairs & Political News all around the clock at \u2026 NDTV.com provides latest news from India and around the world. Get breaking news alerts from India and follow today\u2019s live news updates in field of politics, business, technology, Bollywood ... 6 days ago \u00b7 Get Latest entertainment news from Bollywood, Hollywood and regional film industries.\nURL: https://duckduckgo.com/?q=What+are+the+latest+AI+news+today?\n\nUser Question: What are the latest AI news today?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", + "prompt.prompt_length": 1366, + "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\n3 days ago \u00b7 Latest news headlines from India & around the world. Check out today\u2019s news coverage live with videos & photos on NDTV.com. 6 days ago \u00b7 India News | Latest In...", + "prompt.instruction_following": "web_search_optimized", + "response.answer_length": 503, + "response.answer_completeness": "complete", + "response.response_quality": "medium", + "response.source_citation": 1, + "response.context_utilization": 0.5670800450958287, + "output_response": "Web Search Answer: The provided web search results do not contain specific information about the latest AI news today. They primarily focus on general news coverage from India and around the world, including politics, b...", + "evaluation.search_effectiveness": 0.2, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 1, + "evaluation.context_relevance": "high", + "evaluation.overall_quality": "low", + "evaluation.ready_for_production": true, + "web_search.search_strategy": "duckduckgo_api", + "web_search.real_time_data": true, + "web_search.external_sources": 1, + "web_search.information_freshness": "current", + "web_search.knowledge_synthesis": "single_source" + }, + "events": [], + "links": [] + } + ], + "created_at": "2025-10-22 16:57:47.477000000", + "updated_at": "2025-10-22 16:57:47.477000000" + }, + { + "trace_id": "823b7927-10bb-49ef-a94e-1b914c298bfb", + "name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "status": "ok", + "status_message": "", + "start_time": "2025-10-22 16:57:38.501000000", + "end_time": "2025-10-22 16:57:46.469000000", + "duration_ms": 7968.149, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "spans": [ + { + "span_id": "b9c9d21e-5f76-4e5a-b99e-2b4ab821caee", + "trace_id": "823b7927-10bb-49ef-a94e-1b914c298bfb", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-22 16:57:38.501000000", + "end_time": "2025-10-22 16:57:46.469000000", + "duration_ms": 7968.089, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What are the benefits of using Noveum Trace?", + "query.length": 44, + "processing.mode": "RAG", + "processing.time_seconds": 7.968040943145752, + "processing.response_length": 1396, + "processing.sources_count": 5, + "output_response": "Final Answer: Using Noveum Trace offers several benefits, including:\n\n1. **Real-Time Monitoring**: The Noveum platform provides a real-time dashboard for analyzing traces and performance, with live updates as new t...", + "final_answer_mode": "RAG", + "query_processed.input_query": "What are the benefits of using Noveum Trace?", + "query_processed.output_response": "Successfully processed query using RAG, generated 1396 character response", + "query_processed.mode": "RAG", + "query_processed.processing_time": 7.968040943145752, + "query_processed.response_length": 1396 + }, + "events": [], + "links": [] + }, + { + "span_id": "bc85c745-4c27-4aa4-bc00-4c9f10a2f049", + "trace_id": "823b7927-10bb-49ef-a94e-1b914c298bfb", + "parent_span_id": "b9c9d21e-5f76-4e5a-b99e-2b4ab821caee", + "name": "agent.query_routing", + "start_time": "2025-10-22 16:57:38.501000000", + "end_time": "2025-10-22 16:57:46.469000000", + "duration_ms": 7968.001, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What are the benefits of using Noveum Trace?", + "query_length": 44, + "query_type": "routing_query", + "output_response": "Routed to RAG for query processing", + "router.classification": "RAG", + "router.confidence_score": 1, + "router.rag_keyword_score": 2, + "router.web_keyword_score": 0, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [] + }, + { + "span_id": "3ab8cc4c-f5ed-410a-b4ee-48d0954b756d", + "trace_id": "823b7927-10bb-49ef-a94e-1b914c298bfb", + "parent_span_id": "bc85c745-4c27-4aa4-bc00-4c9f10a2f049", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-22 16:57:38.501000000", + "end_time": "2025-10-22 16:57:38.501000000", + "duration_ms": 0.09899999999999999, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What are the benefits of using Noveum Trace?", + "query_length": 44, + "query_type": "routing_evaluation_query", + "classification.mode": "RAG", + "classification.rag_keyword_score": 2, + "classification.web_keyword_score": 0, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "noveum_specific", + "query.keyword_density": 0.25, + "query.domain_affinity": "noveum", + "routing.decision": "Routed to RAG based on analysis", + "routing.rationale": "RAG score: 2, Web score: 0, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "Web Search", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.038461538461538464, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: RAG (Confidence: 1.00)" + }, + "events": [], + "links": [] + }, + { + "span_id": "67292076-ec8a-4856-9c5b-726868552157", + "trace_id": "823b7927-10bb-49ef-a94e-1b914c298bfb", + "parent_span_id": "bc85c745-4c27-4aa4-bc00-4c9f10a2f049", + "name": "agent.llm-rag", + "start_time": "2025-10-22 16:57:38.501000000", + "end_time": "2025-10-22 16:57:46.469000000", + "duration_ms": 7967.679, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "rag_agent", + "agent.operation": "llm-rag", + "agent.capabilities": [ + "document_retrieval", + "context_generation", + "response_generation" + ], + "agent.id": "noveum_rag_agent", + "input_query": "What are the benefits of using Noveum Trace?", + "query_length": 44, + "query_type": "rag_query", + "output_response": "RAG Answer: Using Noveum Trace offers several benefits, including:\n\n1. **Real-Time Monitoring**: The Noveum platform provides a real-time dashboard for analyzing traces and performance, with live updates as new t...", + "rag.context_length": 2773, + "rag.sources_count": 5, + "rag.answer_length": 1396, + "rag.mode": "retrieval_augmented_generation" + }, + "events": [], + "links": [] + }, + { + "span_id": "d044ae4a-c463-4c1b-abce-a9a18f045e75", + "trace_id": "823b7927-10bb-49ef-a94e-1b914c298bfb", + "parent_span_id": "67292076-ec8a-4856-9c5b-726868552157", + "name": "agent.llm_model_execution", + "start_time": "2025-10-22 16:57:38.901000000", + "end_time": "2025-10-22 16:57:45.885000000", + "duration_ms": 6984.242, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What are the benefits of using Noveum Trace?", + "query_length": 44, + "query_type": "rag_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 6.984165906906128, + "model.latency_ms": 6984.165906906128, + "model.start_time": 1761152258.9012191, + "model.end_time": 1761152265.885385, + "model.performance_tier": "slow", + "model.prompt_tokens": 828, + "model.completion_tokens": 349, + "model.total_tokens": 1177, + "model.tokens_per_second": 168.524060809631, + "model.estimated_cost": 0.03531, + "model.efficiency_score": 1.1860662701784197, + "model.response_length": 1396, + "model.response_quality": "high", + "model.output_response": "Model Response: Using Noveum Trace offers several benefits, including:\n\n1. **Real-Time Monitoring**: The Noveum platform provides a real-time dashboard for analyzing traces and performance, with live updates as new t...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [] + }, + { + "span_id": "5b1548cf-0de7-49d4-b9b8-9ad1f9f2abc9", + "trace_id": "823b7927-10bb-49ef-a94e-1b914c298bfb", + "parent_span_id": "67292076-ec8a-4856-9c5b-726868552157", + "name": "agent.rag_evaluation_metrics", + "start_time": "2025-10-22 16:57:45.885000000", + "end_time": "2025-10-22 16:57:45.885000000", + "duration_ms": 0.061, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "rag_evaluation_metrics", + "agent.capabilities": [ + "retrieval_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What are the benefits of using Noveum Trace?", + "query_length": 44, + "query_type": "rag_evaluation_query", + "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/docs/platform/dashboard):\n- Bulk Export: Download multiple traces simultaneously\n- Comparative Analysis: Compare performance across multiple traces\nReal-Time Updates\n- Live Refresh: Automatic updates as new traces arrive\n- Connection Monitoring: Real-time status of your t...", + "retrieval.context_length": 2773, + "retrieval.sources_count": 5, + "retrieval.context_quality": "high", + "retrieval.effectiveness": 1, + "retrieval.context_utilization": 2.773, + "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/docs/platform/dashboard):\n- Bulk Export: Download multiple traces simultaneously\n- Comparative Analysis: Compare performance across multiple traces\nReal-Time Updates\n- Live Refresh: Automatic updates as new traces arrive\n- Connection Monitoring: Real-time status of your trace ingestion pipeline\n- Performance Indicators: Live metrics for system health monitoring\nReady to dive deeper? Explore Projects & Environments to organize your AI applications, or check out Team Collaboration to share insights with your team.\nGet Earl...\n\nSource 2 (https://noveum.ai/en/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\n\ud83d\udcd8 TypeScript SDK (\n@noveum/trace\n)- Framework integrations for Next.js, Express.js, and Hono\n- TypeScript-first design with full type safety\n- Universal compatibility (Node.js, Edge Runtime, browsers)\n-\n\ud83d\udcca Noveum Platform\n- Real-time dashboard for analyzing traces and performance\n- Ad...\n\nSource 3 (https://noveum.ai/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\n\ud83d\udcd8 TypeScript SDK (\n@noveum/trace\n)- Framework integrations for Next.js, Express.js, and Hono\n- TypeScript-first design with full type safety\n- Universal compatibility (Node.js, Edge Runtime, browsers)\n-\n\ud83d\udcca Noveum Platform\n- Real-time dashboard for analyzing traces and performance\n- Ad...\n\nSource 4 (https://noveum.ai/en/docs/getting-started/overview):\n- Advanced search & filtering across traces and spans\n- Cost analysis and optimization recommendations\n- Team collaboration with shared insights and alerts\n\ud83d\udd0d What Noveum Traces\nLLM Operations\n- Model calls across all providers (OpenAI, Anthropic, Google, etc.)\n- Token usage and cost calculation\n- Prompt engineering effectiveness\n- Response quality metrics\nRAG Pipelines\n- Document retrieval performance and relevance\n- Embedding generation costs and latency\n- Context assembly and prompt constructi...\n\nSource 5 (https://noveum.ai/docs/getting-started/overview):\n- Advanced search & filtering across traces and spans\n- Cost analysis and optimization recommendations\n- Team collaboration with shared insights and alerts\n\ud83d\udd0d What Noveum Traces\nLLM Operations\n- Model calls across all providers (OpenAI, Anthropic, Google, etc.)\n- Token usage and cost calculation\n- Prompt engineering effectiveness\n- Response quality metrics\nRAG Pipelines\n- Document retrieval performance and relevance\n- Embedding generation costs and latency\n- Context assembly and prompt constructi...\n\nUser Question: What are the benefits of using Noveum Trace?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", + "prompt.prompt_length": 3313, + "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/docs/platform/dashboard):\n- Bulk Export: Download multiple traces simultaneously\n- Comparative Analysis: Compare performance across multiple traces\nReal-Time Updates\n- Live...", + "prompt.instruction_following": "rag_optimized", + "response.answer_length": 1396, + "response.answer_completeness": "complete", + "response.response_quality": "high", + "response.source_citation": 5, + "response.context_utilization": 0.5034258925351605, + "output_response": "RAG Answer: Using Noveum Trace offers several benefits, including:\n\n1. **Real-Time Monitoring**: The Noveum platform provides a real-time dashboard for analyzing traces and performance, with live updates as new t...", + "evaluation.retrieval_effectiveness": 1, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 5, + "evaluation.context_relevance": "high", + "evaluation.overall_quality": "high", + "evaluation.ready_for_production": true, + "rag.retrieval_strategy": "semantic_similarity", + "rag.vector_search_results": 5, + "rag.context_synthesis": "multi_source", + "rag.document_coverage": 1, + "rag.information_density": 0.5034258925351605 + }, + "events": [], + "links": [] + } + ], + "created_at": "2025-10-22 16:57:38.501000000", + "updated_at": "2025-10-22 16:57:38.501000000" + }, + { + "trace_id": "1ae0d6c4-d1fb-4239-b536-3333187d44aa", + "name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "status": "ok", + "status_message": "", + "start_time": "2025-10-22 16:57:32.800000000", + "end_time": "2025-10-22 16:57:37.494000000", + "duration_ms": 4694.468, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "spans": [ + { + "span_id": "aaba719b-36ec-46ab-af2e-3ea8c18e9872", + "trace_id": "1ae0d6c4-d1fb-4239-b536-3333187d44aa", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-22 16:57:32.800000000", + "end_time": "2025-10-22 16:57:37.494000000", + "duration_ms": 4694.408, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "How do I configure Noveum for my system?", + "query.length": 40, + "processing.mode": "RAG", + "processing.time_seconds": 4.694374084472656, + "processing.response_length": 553, + "processing.sources_count": 5, + "output_response": "Final Answer: To configure Noveum for your system, you can refer to the SDK Integration Guide available in the Noveum documentation. This guide will help you trace your AI applications with minimal code changes. Th...", + "final_answer_mode": "RAG", + "query_processed.input_query": "How do I configure Noveum for my system?", + "query_processed.output_response": "Successfully processed query using RAG, generated 553 character response", + "query_processed.mode": "RAG", + "query_processed.processing_time": 4.694374084472656, + "query_processed.response_length": 553 + }, + "events": [], + "links": [] + }, + { + "span_id": "4254e6f9-10b8-42ba-939d-d7c124f80f9a", + "trace_id": "1ae0d6c4-d1fb-4239-b536-3333187d44aa", + "parent_span_id": "6d5d074e-8fd3-4d33-88fb-7f54b0959030", + "name": "agent.llm-rag", + "start_time": "2025-10-22 16:57:32.800000000", + "end_time": "2025-10-22 16:57:37.494000000", + "duration_ms": 4694.155000000001, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "rag_agent", + "agent.operation": "llm-rag", + "agent.capabilities": [ + "document_retrieval", + "context_generation", + "response_generation" + ], + "agent.id": "noveum_rag_agent", + "input_query": "How do I configure Noveum for my system?", + "query_length": 40, + "query_type": "rag_query", + "output_response": "RAG Answer: To configure Noveum for your system, you can refer to the SDK Integration Guide available in the Noveum documentation. This guide will help you trace your AI applications with minimal code changes. Th...", + "rag.context_length": 2237, + "rag.sources_count": 5, + "rag.answer_length": 553, + "rag.mode": "retrieval_augmented_generation" + }, + "events": [], + "links": [] + }, + { + "span_id": "6d5d074e-8fd3-4d33-88fb-7f54b0959030", + "trace_id": "1ae0d6c4-d1fb-4239-b536-3333187d44aa", + "parent_span_id": "aaba719b-36ec-46ab-af2e-3ea8c18e9872", + "name": "agent.query_routing", + "start_time": "2025-10-22 16:57:32.800000000", + "end_time": "2025-10-22 16:57:37.494000000", + "duration_ms": 4694.318, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "How do I configure Noveum for my system?", + "query_length": 40, + "query_type": "routing_query", + "output_response": "Routed to RAG for query processing", + "router.classification": "RAG", + "router.confidence_score": 1, + "router.rag_keyword_score": 2, + "router.web_keyword_score": 0, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [] + }, + { + "span_id": "2cc2ab0b-2572-4a75-88a3-18b3a1eaca78", + "trace_id": "1ae0d6c4-d1fb-4239-b536-3333187d44aa", + "parent_span_id": "6d5d074e-8fd3-4d33-88fb-7f54b0959030", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-22 16:57:32.800000000", + "end_time": "2025-10-22 16:57:32.800000000", + "duration_ms": 0.04, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: How do I configure Noveum for my system?", + "query_length": 40, + "query_type": "routing_evaluation_query", + "classification.mode": "RAG", + "classification.rag_keyword_score": 2, + "classification.web_keyword_score": 0, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "noveum_specific", + "query.keyword_density": 0.25, + "query.domain_affinity": "noveum", + "routing.decision": "Routed to RAG based on analysis", + "routing.rationale": "RAG score: 2, Web score: 0, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "Web Search", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.038461538461538464, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: RAG (Confidence: 1.00)" + }, + "events": [], + "links": [] + }, + { + "span_id": "f1d9f779-3312-43c1-b549-00fc874429f0", + "trace_id": "1ae0d6c4-d1fb-4239-b536-3333187d44aa", + "parent_span_id": "4254e6f9-10b8-42ba-939d-d7c124f80f9a", + "name": "agent.llm_model_execution", + "start_time": "2025-10-22 16:57:33.281000000", + "end_time": "2025-10-22 16:57:37.060000000", + "duration_ms": 3778.743, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: How do I configure Noveum for my system?", + "query_length": 40, + "query_type": "rag_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 3.7786097526550293, + "model.latency_ms": 3778.6097526550293, + "model.start_time": 1761152253.28163, + "model.end_time": 1761152257.0602398, + "model.performance_tier": "medium", + "model.prompt_tokens": 693, + "model.completion_tokens": 138, + "model.total_tokens": 831, + "model.tokens_per_second": 219.92215507730066, + "model.estimated_cost": 0.02493, + "model.efficiency_score": 0.6654632972322503, + "model.response_length": 553, + "model.response_quality": "high", + "model.output_response": "Model Response: To configure Noveum for your system, you can refer to the SDK Integration Guide available in the Noveum documentation. This guide will help you trace your AI applications with minimal code changes. Th...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [] + }, + { + "span_id": "5500be90-0cfa-44dd-a6e3-e72ac9f0a79e", + "trace_id": "1ae0d6c4-d1fb-4239-b536-3333187d44aa", + "parent_span_id": "4254e6f9-10b8-42ba-939d-d7c124f80f9a", + "name": "agent.rag_evaluation_metrics", + "start_time": "2025-10-22 16:57:37.060000000", + "end_time": "2025-10-22 16:57:37.060000000", + "duration_ms": 0.076, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "rag_evaluation_metrics", + "agent.capabilities": [ + "retrieval_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: How do I configure Noveum for my system?", + "query_length": 40, + "query_type": "rag_evaluation_query", + "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/docs):\nWelcome to Noveum.ai Docs\nComprehensive AI tracing and observability for LLM applications, RAG systems, and AI agents\nWelcome to the Noveum.ai documentation! Here you'll find everything you need to integrate, configure, and optimize your AI applications using ou...", + "retrieval.context_length": 2237, + "retrieval.sources_count": 5, + "retrieval.context_quality": "high", + "retrieval.effectiveness": 1, + "retrieval.context_utilization": 2.237, + "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/docs):\nWelcome to Noveum.ai Docs\nComprehensive AI tracing and observability for LLM applications, RAG systems, and AI agents\nWelcome to the Noveum.ai documentation! Here you'll find everything you need to integrate, configure, and optimize your AI applications using our comprehensive tracing and observability platform.\nWe provide powerful SDKs for Python and TypeScript that enable you to trace LLM calls, RAG pipelines, and multi-agent workflows with minimal code changes. Our documentation is organized ...\n\nSource 2 (https://noveum.ai/docs):\nWelcome to Noveum.ai Docs\nComprehensive AI tracing and observability for LLM applications, RAG systems, and AI agents\nWelcome to the Noveum.ai documentation! Here you'll find everything you need to integrate, configure, and optimize your AI applications using our comprehensive tracing and observability platform.\nWe provide powerful SDKs for Python and TypeScript that enable you to trace LLM calls, RAG pipelines, and multi-agent workflows with minimal code changes. Our documentation is organized ...\n\nSource 3 (https://noveum.ai/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nSource 4 (https://noveum.ai/en/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nSource 5 (https://noveum.ai/en/docs):\n- \ud83c\udf0d Universal: Works across frameworks, clouds, and deployment models\n\ud83e\udd1d Community & Support\n- \ud83d\udcac Discord Community: Join our Discord\n- \ud83d\udce7 Email Support: [email protected]\n- \ud83d\udc1b Bug Reports: GitHub Issues\n- \ud83d\udcd6 Knowledge Base: Help Center\nReady to get started? Head to our SDK Integration Guide to begin tracing your AI applications in under 5 minutes!\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability ...\n\nUser Question: How do I configure Noveum for my system?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", + "prompt.prompt_length": 2773, + "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/docs):\nWelcome to Noveum.ai Docs\nComprehensive AI tracing and observability for LLM applications, RAG systems, and AI agents\nWelcome to the Noveum.ai documentation! Here...", + "prompt.instruction_following": "rag_optimized", + "response.answer_length": 553, + "response.answer_completeness": "complete", + "response.response_quality": "high", + "response.source_citation": 5, + "response.context_utilization": 0.24720607957085383, + "output_response": "RAG Answer: To configure Noveum for your system, you can refer to the SDK Integration Guide available in the Noveum documentation. This guide will help you trace your AI applications with minimal code changes. Th...", + "evaluation.retrieval_effectiveness": 1, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 5, + "evaluation.context_relevance": "high", + "evaluation.overall_quality": "high", + "evaluation.ready_for_production": true, + "rag.retrieval_strategy": "semantic_similarity", + "rag.vector_search_results": 5, + "rag.context_synthesis": "multi_source", + "rag.document_coverage": 1, + "rag.information_density": 0.24720607957085383 + }, + "events": [], + "links": [] + } + ], + "created_at": "2025-10-22 16:57:32.800000000", + "updated_at": "2025-10-22 16:57:32.800000000" + }, + { + "trace_id": "fb0d31fc-2465-4ecd-973c-4cd443d9befa", + "name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "status": "ok", + "status_message": "", + "start_time": "2025-10-22 16:57:24.828000000", + "end_time": "2025-10-22 16:57:31.793000000", + "duration_ms": 6965.302000000001, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "spans": [ + { + "span_id": "3068013a-6844-4f3c-8ec2-854a7353591e", + "trace_id": "fb0d31fc-2465-4ecd-973c-4cd443d9befa", + "parent_span_id": "a80d8fab-d5dc-4417-a5e5-9a2a76ed7419", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-22 16:57:24.828000000", + "end_time": "2025-10-22 16:57:24.828000000", + "duration_ms": 0.038, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What monitoring capabilities does Noveum provide?", + "query_length": 49, + "query_type": "routing_evaluation_query", + "classification.mode": "RAG", + "classification.rag_keyword_score": 2, + "classification.web_keyword_score": 0, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "noveum_specific", + "query.keyword_density": 0.3333333333333333, + "query.domain_affinity": "noveum", + "routing.decision": "Routed to RAG based on analysis", + "routing.rationale": "RAG score: 2, Web score: 0, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "Web Search", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.038461538461538464, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: RAG (Confidence: 1.00)" + }, + "events": [], + "links": [] + }, + { + "span_id": "189b89e3-fbbd-41de-8521-efb6a68f62f2", + "trace_id": "fb0d31fc-2465-4ecd-973c-4cd443d9befa", + "parent_span_id": "a80d8fab-d5dc-4417-a5e5-9a2a76ed7419", + "name": "agent.llm-rag", + "start_time": "2025-10-22 16:57:24.828000000", + "end_time": "2025-10-22 16:57:31.793000000", + "duration_ms": 6964.955, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "rag_agent", + "agent.operation": "llm-rag", + "agent.capabilities": [ + "document_retrieval", + "context_generation", + "response_generation" + ], + "agent.id": "noveum_rag_agent", + "input_query": "What monitoring capabilities does Noveum provide?", + "query_length": 49, + "query_type": "rag_query", + "output_response": "RAG Answer: Noveum provides comprehensive monitoring capabilities for AI applications, including the ability to monitor, trace, and optimize AI agents across various frameworks such as LangChain, CrewAI, AutoGen,...", + "rag.context_length": 2740, + "rag.sources_count": 5, + "rag.answer_length": 582, + "rag.mode": "retrieval_augmented_generation" + }, + "events": [], + "links": [] + }, + { + "span_id": "f5cffb7d-341b-4cde-b162-02208042463d", + "trace_id": "fb0d31fc-2465-4ecd-973c-4cd443d9befa", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-22 16:57:24.828000000", + "end_time": "2025-10-22 16:57:31.793000000", + "duration_ms": 6965.236, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What monitoring capabilities does Noveum provide?", + "query.length": 49, + "processing.mode": "RAG", + "processing.time_seconds": 6.965206146240234, + "processing.response_length": 582, + "processing.sources_count": 5, + "output_response": "Final Answer: Noveum provides comprehensive monitoring capabilities for AI applications, including the ability to monitor, trace, and optimize AI agents across various frameworks such as LangChain, CrewAI, AutoGen,...", + "final_answer_mode": "RAG", + "query_processed.input_query": "What monitoring capabilities does Noveum provide?", + "query_processed.output_response": "Successfully processed query using RAG, generated 582 character response", + "query_processed.mode": "RAG", + "query_processed.processing_time": 6.965206146240234, + "query_processed.response_length": 582 + }, + "events": [], + "links": [] + }, + { + "span_id": "a80d8fab-d5dc-4417-a5e5-9a2a76ed7419", + "trace_id": "fb0d31fc-2465-4ecd-973c-4cd443d9befa", + "parent_span_id": "f5cffb7d-341b-4cde-b162-02208042463d", + "name": "agent.query_routing", + "start_time": "2025-10-22 16:57:24.828000000", + "end_time": "2025-10-22 16:57:31.793000000", + "duration_ms": 6965.17, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What monitoring capabilities does Noveum provide?", + "query_length": 49, + "query_type": "routing_query", + "output_response": "Routed to RAG for query processing", + "router.classification": "RAG", + "router.confidence_score": 1, + "router.rag_keyword_score": 2, + "router.web_keyword_score": 0, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [] + }, + { + "span_id": "a2b632b8-fbcb-480f-9292-9f513c64df38", + "trace_id": "fb0d31fc-2465-4ecd-973c-4cd443d9befa", + "parent_span_id": "189b89e3-fbbd-41de-8521-efb6a68f62f2", + "name": "agent.llm_model_execution", + "start_time": "2025-10-22 16:57:27.824000000", + "end_time": "2025-10-22 16:57:31.155000000", + "duration_ms": 3330.989, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What monitoring capabilities does Noveum provide?", + "query_length": 49, + "query_type": "rag_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 3.3309478759765625, + "model.latency_ms": 3330.9478759765625, + "model.start_time": 1761152247.824474, + "model.end_time": 1761152251.155422, + "model.performance_tier": "medium", + "model.prompt_tokens": 821, + "model.completion_tokens": 145, + "model.total_tokens": 966, + "model.tokens_per_second": 290.00754018607677, + "model.estimated_cost": 0.028980000000000002, + "model.efficiency_score": 0.6024844720496895, + "model.response_length": 582, + "model.response_quality": "high", + "model.output_response": "Model Response: Noveum provides comprehensive monitoring capabilities for AI applications, including the ability to monitor, trace, and optimize AI agents across various frameworks such as LangChain, CrewAI, AutoGen,...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [] + }, + { + "span_id": "40ff54c4-07ca-4b2d-a9b4-7e3ae7adcd81", + "trace_id": "fb0d31fc-2465-4ecd-973c-4cd443d9befa", + "parent_span_id": "189b89e3-fbbd-41de-8521-efb6a68f62f2", + "name": "agent.rag_evaluation_metrics", + "start_time": "2025-10-22 16:57:31.155000000", + "end_time": "2025-10-22 16:57:31.155000000", + "duration_ms": 0.018000000000000002, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "rag_evaluation_metrics", + "agent.capabilities": [ + "retrieval_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What monitoring capabilities does Noveum provide?", + "query_length": 49, + "query_type": "rag_evaluation_query", + "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en):\nMonitor all your AI Agents\nimprove AI Agents today\nNoveum.ai helps you monitor, trace, and optimize your AI applications.\nNoveum.ai works with any AI framework \u2013 LangChain, CrewAI, AutoGen, custom implementations, or direct LLM calls. One dashboard shows everything.\n...", + "retrieval.context_length": 2740, + "retrieval.sources_count": 5, + "retrieval.context_quality": "high", + "retrieval.effectiveness": 1, + "retrieval.context_utilization": 2.74, + "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en):\nMonitor all your AI Agents\nimprove AI Agents today\nNoveum.ai helps you monitor, trace, and optimize your AI applications.\nNoveum.ai works with any AI framework \u2013 LangChain, CrewAI, AutoGen, custom implementations, or direct LLM calls. One dashboard shows everything.\nMonitor, Evaluate, Improve Your AI Agents\nThe control plane for AI agents.\nMonitor Everything, Miss Nothing\nOur lightweight SDKs capture every trace and span across your AI agent ecosystem\u2014from simple LLM calls to complex multi-agent...\n\nSource 2 (https://noveum.ai/):\nMonitor all your AI Agents\nimprove AI Agents today\nNoveum.ai helps you monitor, trace, and optimize your AI applications.\nNoveum.ai works with any AI framework \u2013 LangChain, CrewAI, AutoGen, custom implementations, or direct LLM calls. One dashboard shows everything.\nMonitor, Evaluate, Improve Your AI Agents\nThe control plane for AI agents.\nMonitor Everything, Miss Nothing\nOur lightweight SDKs capture every trace and span across your AI agent ecosystem\u2014from simple LLM calls to complex multi-agent...\n\nSource 3 (https://noveum.ai):\nMonitor all your AI Agents\nimprove AI Agents today\nNoveum.ai helps you monitor, trace, and optimize your AI applications.\nNoveum.ai works with any AI framework \u2013 LangChain, CrewAI, AutoGen, custom implementations, or direct LLM calls. One dashboard shows everything.\nMonitor, Evaluate, Improve Your AI Agents\nThe control plane for AI agents.\nMonitor Everything, Miss Nothing\nOur lightweight SDKs capture every trace and span across your AI agent ecosystem\u2014from simple LLM calls to complex multi-agent...\n\nSource 4 (https://noveum.ai/en/docs/getting-started/overview):\nNoveum.ai Overview\nComprehensive AI tracing and observability platform for LLM applications, RAG systems, and AI agents\nWelcome to Noveum.ai\u2014the comprehensive tracing and observability platform built specifically for AI applications. Whether you're building LLM-powered chatbots, RAG systems, multi-agent workflows, or any AI-driven application, Noveum provides the insights you need to understand, debug, and optimize your systems.\n\ud83c\udfaf Why AI Applications Need Specialized Observability\nTraditional mo...\n\nSource 5 (https://noveum.ai/docs/getting-started/overview):\nNoveum.ai Overview\nComprehensive AI tracing and observability platform for LLM applications, RAG systems, and AI agents\nWelcome to Noveum.ai\u2014the comprehensive tracing and observability platform built specifically for AI applications. Whether you're building LLM-powered chatbots, RAG systems, multi-agent workflows, or any AI-driven application, Noveum provides the insights you need to understand, debug, and optimize your systems.\n\ud83c\udfaf Why AI Applications Need Specialized Observability\nTraditional mo...\n\nUser Question: What monitoring capabilities does Noveum provide?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", + "prompt.prompt_length": 3285, + "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en):\nMonitor all your AI Agents\nimprove AI Agents today\nNoveum.ai helps you monitor, trace, and optimize your AI applications.\nNoveum.ai works with any AI framework \u2013 LangC...", + "prompt.instruction_following": "rag_optimized", + "response.answer_length": 582, + "response.answer_completeness": "complete", + "response.response_quality": "high", + "response.source_citation": 5, + "response.context_utilization": 0.2124087591240876, + "output_response": "RAG Answer: Noveum provides comprehensive monitoring capabilities for AI applications, including the ability to monitor, trace, and optimize AI agents across various frameworks such as LangChain, CrewAI, AutoGen,...", + "evaluation.retrieval_effectiveness": 1, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 5, + "evaluation.context_relevance": "high", + "evaluation.overall_quality": "high", + "evaluation.ready_for_production": true, + "rag.retrieval_strategy": "semantic_similarity", + "rag.vector_search_results": 5, + "rag.context_synthesis": "multi_source", + "rag.document_coverage": 1, + "rag.information_density": 0.2124087591240876 + }, + "events": [], + "links": [] + } + ], + "created_at": "2025-10-22 16:57:24.828000000", + "updated_at": "2025-10-22 16:57:24.828000000" + }, + { + "trace_id": "3bbd0b36-c30f-4126-b62a-dd0fae1c701d", + "name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "status": "ok", + "status_message": "", + "start_time": "2025-10-22 16:57:19.339000000", + "end_time": "2025-10-22 16:57:23.822000000", + "duration_ms": 4483.158, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "spans": [ + { + "span_id": "b0f0a2e0-c1f7-4ba4-8b75-7090e931a738", + "trace_id": "3bbd0b36-c30f-4126-b62a-dd0fae1c701d", + "parent_span_id": "4db80813-e62f-48ec-9091-62d569527caf", + "name": "agent.query_routing", + "start_time": "2025-10-22 16:57:19.339000000", + "end_time": "2025-10-22 16:57:23.822000000", + "duration_ms": 4482.909000000001, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "How does Noveum handle agent tracing?", + "query_length": 37, + "query_type": "routing_query", + "output_response": "Routed to RAG for query processing", + "router.classification": "RAG", + "router.confidence_score": 1, + "router.rag_keyword_score": 2, + "router.web_keyword_score": 0, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [] + }, + { + "span_id": "24da2221-b0ab-4161-98ad-e7b5775fcb27", + "trace_id": "3bbd0b36-c30f-4126-b62a-dd0fae1c701d", + "parent_span_id": "b0f0a2e0-c1f7-4ba4-8b75-7090e931a738", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-22 16:57:19.339000000", + "end_time": "2025-10-22 16:57:19.339000000", + "duration_ms": 0.026, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: How does Noveum handle agent tracing?", + "query_length": 37, + "query_type": "routing_evaluation_query", + "classification.mode": "RAG", + "classification.rag_keyword_score": 2, + "classification.web_keyword_score": 0, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "noveum_specific", + "query.keyword_density": 0.3333333333333333, + "query.domain_affinity": "noveum", + "routing.decision": "Routed to RAG based on analysis", + "routing.rationale": "RAG score: 2, Web score: 0, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "Web Search", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.038461538461538464, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: RAG (Confidence: 1.00)" + }, + "events": [], + "links": [] + }, + { + "span_id": "d2b495eb-7e07-48a3-aa6d-1835aab18c11", + "trace_id": "3bbd0b36-c30f-4126-b62a-dd0fae1c701d", + "parent_span_id": "b0f0a2e0-c1f7-4ba4-8b75-7090e931a738", + "name": "agent.llm-rag", + "start_time": "2025-10-22 16:57:19.339000000", + "end_time": "2025-10-22 16:57:23.822000000", + "duration_ms": 4482.564, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "rag_agent", + "agent.operation": "llm-rag", + "agent.capabilities": [ + "document_retrieval", + "context_generation", + "response_generation" + ], + "agent.id": "noveum_rag_agent", + "input_query": "How does Noveum handle agent tracing?", + "query_length": 37, + "query_type": "rag_query", + "output_response": "RAG Answer: Noveum.ai handles agent tracing through its specialized Multi-Agent Tracing capabilities, which are designed to observe complex workflows and inter-agent communications within multi-agent systems. The...", + "rag.context_length": 2736, + "rag.sources_count": 5, + "rag.answer_length": 663, + "rag.mode": "retrieval_augmented_generation" + }, + "events": [], + "links": [] + }, + { + "span_id": "4db80813-e62f-48ec-9091-62d569527caf", + "trace_id": "3bbd0b36-c30f-4126-b62a-dd0fae1c701d", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-22 16:57:19.339000000", + "end_time": "2025-10-22 16:57:23.822000000", + "duration_ms": 4483.031000000001, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "How does Noveum handle agent tracing?", + "query.length": 37, + "processing.mode": "RAG", + "processing.time_seconds": 4.482964038848877, + "processing.response_length": 663, + "processing.sources_count": 5, + "output_response": "Final Answer: Noveum.ai handles agent tracing through its specialized Multi-Agent Tracing capabilities, which are designed to observe complex workflows and inter-agent communications within multi-agent systems. The...", + "final_answer_mode": "RAG", + "query_processed.input_query": "How does Noveum handle agent tracing?", + "query_processed.output_response": "Successfully processed query using RAG, generated 663 character response", + "query_processed.mode": "RAG", + "query_processed.processing_time": 4.482964038848877, + "query_processed.response_length": 663 + }, + "events": [], + "links": [] + }, + { + "span_id": "b80cb5cd-24f2-484d-bbff-9febe2aa3ffe", + "trace_id": "3bbd0b36-c30f-4126-b62a-dd0fae1c701d", + "parent_span_id": "d2b495eb-7e07-48a3-aa6d-1835aab18c11", + "name": "agent.llm_model_execution", + "start_time": "2025-10-22 16:57:19.958000000", + "end_time": "2025-10-22 16:57:22.709000000", + "duration_ms": 2750.9480000000003, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: How does Noveum handle agent tracing?", + "query_length": 37, + "query_type": "rag_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 2.750890016555786, + "model.latency_ms": 2750.890016555786, + "model.start_time": 1761152239.958973, + "model.end_time": 1761152242.709863, + "model.performance_tier": "medium", + "model.prompt_tokens": 817, + "model.completion_tokens": 165, + "model.total_tokens": 982, + "model.tokens_per_second": 356.975376728983, + "model.estimated_cost": 0.02946, + "model.efficiency_score": 0.675152749490835, + "model.response_length": 663, + "model.response_quality": "high", + "model.output_response": "Model Response: Noveum.ai handles agent tracing through its specialized Multi-Agent Tracing capabilities, which are designed to observe complex workflows and inter-agent communications within multi-agent systems. The...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [] + }, + { + "span_id": "d85fcf7d-7eaa-4299-92fc-c969fab63a6e", + "trace_id": "3bbd0b36-c30f-4126-b62a-dd0fae1c701d", + "parent_span_id": "d2b495eb-7e07-48a3-aa6d-1835aab18c11", + "name": "agent.rag_evaluation_metrics", + "start_time": "2025-10-22 16:57:22.710000000", + "end_time": "2025-10-22 16:57:22.710000000", + "duration_ms": 0.036000000000000004, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "rag_evaluation_metrics", + "agent.capabilities": [ + "retrieval_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: How does Noveum handle agent tracing?", + "query_length": 37, + "query_type": "rag_evaluation_query", + "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, vector searches, data retrieval, agent reasoning, and more. Without proper observability, debugging ...", + "retrieval.context_length": 2736, + "retrieval.sources_count": 5, + "retrieval.context_quality": "high", + "retrieval.effectiveness": 1, + "retrieval.context_utilization": 2.736, + "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, vector searches, data retrieval, agent reasoning, and more. Without proper observability, debugging becomes a nightmare, optimization is guesswork, and understanding user interactions is nearly impossible.\nNoveum.ai solves this by providing comprehensive tracing and observability specifically designed for AI applications.\nKey benefits of using Noveum.ai:\n- Complete Visibility: Trace...\n\nSource 2 (https://noveum.ai/docs/advanced/multi-agent-tracing):\nMulti-Agent Tracing\nObserve complex agent workflows and inter-agent communications with comprehensive tracing\nMulti-agent systems represent some of the most complex AI applications, involving multiple agents that coordinate, communicate, and collaborate to achieve shared goals. Noveum.ai provides specialized tracing capabilities to help you understand and optimize these intricate workflows.\n\ud83c\udfaf Why Multi-Agent Tracing Matters\nMulti-agent systems introduce unique observability challenges:\n- Complex...\n\nSource 3 (https://noveum.ai/en/docs/advanced/multi-agent-tracing):\nMulti-Agent Tracing\nObserve complex agent workflows and inter-agent communications with comprehensive tracing\nMulti-agent systems represent some of the most complex AI applications, involving multiple agents that coordinate, communicate, and collaborate to achieve shared goals. Noveum.ai provides specialized tracing capabilities to help you understand and optimize these intricate workflows.\n\ud83c\udfaf Why Multi-Agent Tracing Matters\nMulti-agent systems introduce unique observability challenges:\n- Complex...\n\nSource 4 (https://noveum.ai/en/docs/advanced/performance-optimization):\n\ud83d\udd17 Next Steps\n- Multi-Agent Tracing - Observe agent workflows\n- RAG Pipeline Observability - Monitor retrieval and generation systems\n- Custom Instrumentation - Add domain-specific tracing\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nSource 5 (https://noveum.ai/docs/advanced/performance-optimization):\n\ud83d\udd17 Next Steps\n- Multi-Agent Tracing - Observe agent workflows\n- RAG Pipeline Observability - Monitor retrieval and generation systems\n- Custom Instrumentation - Add domain-specific tracing\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nUser Question: How does Noveum handle agent tracing?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", + "prompt.prompt_length": 3269, + "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, ...", + "prompt.instruction_following": "rag_optimized", + "response.answer_length": 663, + "response.answer_completeness": "complete", + "response.response_quality": "high", + "response.source_citation": 5, + "response.context_utilization": 0.24232456140350878, + "output_response": "RAG Answer: Noveum.ai handles agent tracing through its specialized Multi-Agent Tracing capabilities, which are designed to observe complex workflows and inter-agent communications within multi-agent systems. The...", + "evaluation.retrieval_effectiveness": 1, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 5, + "evaluation.context_relevance": "high", + "evaluation.overall_quality": "high", + "evaluation.ready_for_production": true, + "rag.retrieval_strategy": "semantic_similarity", + "rag.vector_search_results": 5, + "rag.context_synthesis": "multi_source", + "rag.document_coverage": 1, + "rag.information_density": 0.24232456140350878 + }, + "events": [], + "links": [] + } + ], + "created_at": "2025-10-22 16:57:19.339000000", + "updated_at": "2025-10-22 16:57:19.339000000" + }, + { + "trace_id": "fee3e6d3-8da1-4ef3-8214-a7d7323d3a49", + "name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "status": "ok", + "status_message": "", + "start_time": "2025-10-22 16:57:13.334000000", + "end_time": "2025-10-22 16:57:18.331000000", + "duration_ms": 4996.502, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "spans": [ + { + "span_id": "93d82f68-6c62-40de-a675-c4d7c40c1d01", + "trace_id": "fee3e6d3-8da1-4ef3-8214-a7d7323d3a49", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-22 16:57:13.334000000", + "end_time": "2025-10-22 16:57:18.330000000", + "duration_ms": 4996.436, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What APIs are available in Noveum platform?", + "query.length": 43, + "processing.mode": "RAG", + "processing.time_seconds": 4.996389865875244, + "processing.response_length": 270, + "processing.sources_count": 5, + "output_response": "Final Answer: The provided context does not specify the APIs available in the Noveum platform. For detailed information about the APIs, I recommend checking the official Noveum documentation or the SDK Integration ...", + "final_answer_mode": "RAG", + "query_processed.input_query": "What APIs are available in Noveum platform?", + "query_processed.output_response": "Successfully processed query using RAG, generated 270 character response", + "query_processed.mode": "RAG", + "query_processed.processing_time": 4.996389865875244, + "query_processed.response_length": 270 + }, + "events": [], + "links": [] + }, + { + "span_id": "1be727c6-14e8-4b2e-b8df-a1cab902454f", + "trace_id": "fee3e6d3-8da1-4ef3-8214-a7d7323d3a49", + "parent_span_id": "93d82f68-6c62-40de-a675-c4d7c40c1d01", + "name": "agent.query_routing", + "start_time": "2025-10-22 16:57:13.334000000", + "end_time": "2025-10-22 16:57:18.330000000", + "duration_ms": 4996.354, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What APIs are available in Noveum platform?", + "query_length": 43, + "query_type": "routing_query", + "output_response": "Routed to RAG for query processing", + "router.classification": "RAG", + "router.confidence_score": 1, + "router.rag_keyword_score": 3, + "router.web_keyword_score": 0, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [] + }, + { + "span_id": "58ab1d39-d349-4164-8ff9-26ece4b1e1aa", + "trace_id": "fee3e6d3-8da1-4ef3-8214-a7d7323d3a49", + "parent_span_id": "1be727c6-14e8-4b2e-b8df-a1cab902454f", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-22 16:57:13.334000000", + "end_time": "2025-10-22 16:57:13.334000000", + "duration_ms": 0.041999999999999996, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What APIs are available in Noveum platform?", + "query_length": 43, + "query_type": "routing_evaluation_query", + "classification.mode": "RAG", + "classification.rag_keyword_score": 3, + "classification.web_keyword_score": 0, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "noveum_specific", + "query.keyword_density": 0.42857142857142855, + "query.domain_affinity": "noveum", + "routing.decision": "Routed to RAG based on analysis", + "routing.rationale": "RAG score: 3, Web score: 0, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "Web Search", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.057692307692307696, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: RAG (Confidence: 1.00)" + }, + "events": [], + "links": [] + }, + { + "span_id": "f11ebf68-a416-484e-8e55-f77a11b9ea1c", + "trace_id": "fee3e6d3-8da1-4ef3-8214-a7d7323d3a49", + "parent_span_id": "1be727c6-14e8-4b2e-b8df-a1cab902454f", + "name": "agent.llm-rag", + "start_time": "2025-10-22 16:57:13.334000000", + "end_time": "2025-10-22 16:57:18.330000000", + "duration_ms": 4996.146000000001, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "rag_agent", + "agent.operation": "llm-rag", + "agent.capabilities": [ + "document_retrieval", + "context_generation", + "response_generation" + ], + "agent.id": "noveum_rag_agent", + "input_query": "What APIs are available in Noveum platform?", + "query_length": 43, + "query_type": "rag_query", + "output_response": "RAG Answer: The provided context does not specify the APIs available in the Noveum platform. For detailed information about the APIs, I recommend checking the official Noveum documentation or the SDK Integration ...", + "rag.context_length": 2284, + "rag.sources_count": 5, + "rag.answer_length": 270, + "rag.mode": "retrieval_augmented_generation" + }, + "events": [], + "links": [] + }, + { + "span_id": "11394488-8802-403d-a926-7fda3a510b61", + "trace_id": "fee3e6d3-8da1-4ef3-8214-a7d7323d3a49", + "parent_span_id": "f11ebf68-a416-484e-8e55-f77a11b9ea1c", + "name": "agent.llm_model_execution", + "start_time": "2025-10-22 16:57:13.862000000", + "end_time": "2025-10-22 16:57:16.049000000", + "duration_ms": 2187.29, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What APIs are available in Noveum platform?", + "query_length": 43, + "query_type": "rag_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 2.187239170074463, + "model.latency_ms": 2187.239170074463, + "model.start_time": 1761152233.862141, + "model.end_time": 1761152236.04938, + "model.performance_tier": "medium", + "model.prompt_tokens": 705, + "model.completion_tokens": 67, + "model.total_tokens": 772, + "model.tokens_per_second": 352.9563710098141, + "model.estimated_cost": 0.02316, + "model.efficiency_score": 0.34974093264248707, + "model.response_length": 270, + "model.response_quality": "high", + "model.output_response": "Model Response: The provided context does not specify the APIs available in the Noveum platform. For detailed information about the APIs, I recommend checking the official Noveum documentation or the SDK Integration ...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [] + }, + { + "span_id": "341cf8ab-cc33-4d57-952f-ba1065f0432e", + "trace_id": "fee3e6d3-8da1-4ef3-8214-a7d7323d3a49", + "parent_span_id": "f11ebf68-a416-484e-8e55-f77a11b9ea1c", + "name": "agent.rag_evaluation_metrics", + "start_time": "2025-10-22 16:57:16.049000000", + "end_time": "2025-10-22 16:57:16.049000000", + "duration_ms": 0.041, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "rag_evaluation_metrics", + "agent.capabilities": [ + "retrieval_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What APIs are available in Noveum platform?", + "query_length": 43, + "query_type": "rag_evaluation_query", + "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/docs):\n- \ud83c\udf0d Universal: Works across frameworks, clouds, and deployment models\n\ud83e\udd1d Community & Support\n- \ud83d\udcac Discord Community: Join our Discord\n- \ud83d\udce7 Email Support: [email protected]\n- \ud83d\udc1b Bug Reports: GitHub Issues\n- \ud83d\udcd6 Knowledge Base: Help Center\nReady to get started? Head to ...", + "retrieval.context_length": 2284, + "retrieval.sources_count": 5, + "retrieval.context_quality": "high", + "retrieval.effectiveness": 1, + "retrieval.context_utilization": 2.284, + "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/docs):\n- \ud83c\udf0d Universal: Works across frameworks, clouds, and deployment models\n\ud83e\udd1d Community & Support\n- \ud83d\udcac Discord Community: Join our Discord\n- \ud83d\udce7 Email Support: [email protected]\n- \ud83d\udc1b Bug Reports: GitHub Issues\n- \ud83d\udcd6 Knowledge Base: Help Center\nReady to get started? Head to our SDK Integration Guide to begin tracing your AI applications in under 5 minutes!\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability ...\n\nSource 2 (https://noveum.ai/docs):\n- \ud83c\udf0d Universal: Works across frameworks, clouds, and deployment models\n\ud83e\udd1d Community & Support\n- \ud83d\udcac Discord Community: Join our Discord\n- \ud83d\udce7 Email Support: [email protected]\n- \ud83d\udc1b Bug Reports: GitHub Issues\n- \ud83d\udcd6 Knowledge Base: Help Center\nReady to get started? Head to our SDK Integration Guide to begin tracing your AI applications in under 5 minutes!\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability ...\n\nSource 3 (https://noveum.ai/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nSource 4 (https://noveum.ai/en/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nSource 5 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, vector searches, data retrieval, agent reasoning, and more. Without proper observability, debugging becomes a nightmare, optimization is guesswork, and understanding user interactions is nearly impossible.\nNoveum.ai solves this by providing comprehensive tracing and observability specifically designed for AI applications.\nKey benefits of using Noveum.ai:\n- Complete Visibility: Trace...\n\nUser Question: What APIs are available in Noveum platform?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", + "prompt.prompt_length": 2823, + "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/docs):\n- \ud83c\udf0d Universal: Works across frameworks, clouds, and deployment models\n\ud83e\udd1d Community & Support\n- \ud83d\udcac Discord Community: Join our Discord\n- \ud83d\udce7 Email Support: [email prot...", + "prompt.instruction_following": "rag_optimized", + "response.answer_length": 270, + "response.answer_completeness": "complete", + "response.response_quality": "high", + "response.source_citation": 5, + "response.context_utilization": 0.11821366024518389, + "output_response": "RAG Answer: The provided context does not specify the APIs available in the Noveum platform. For detailed information about the APIs, I recommend checking the official Noveum documentation or the SDK Integration ...", + "evaluation.retrieval_effectiveness": 1, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 5, + "evaluation.context_relevance": "high", + "evaluation.overall_quality": "high", + "evaluation.ready_for_production": true, + "rag.retrieval_strategy": "semantic_similarity", + "rag.vector_search_results": 5, + "rag.context_synthesis": "multi_source", + "rag.document_coverage": 1, + "rag.information_density": 0.11821366024518389 + }, + "events": [], + "links": [] + } + ], + "created_at": "2025-10-22 16:57:13.334000000", + "updated_at": "2025-10-22 16:57:13.334000000" + }, + { + "trace_id": "93bd37c4-4289-43c9-b34e-19eaaef610c2", + "name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "status": "ok", + "status_message": "", + "start_time": "2025-10-22 16:57:06.605000000", + "end_time": "2025-10-22 16:57:12.327000000", + "duration_ms": 5721.707, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "spans": [ + { + "span_id": "24930c28-7f93-426c-8914-bd0e0085589f", + "trace_id": "93bd37c4-4289-43c9-b34e-19eaaef610c2", + "parent_span_id": "bfc91518-f818-4f35-9113-70b04eb55edc", + "name": "agent.query_routing", + "start_time": "2025-10-22 16:57:06.605000000", + "end_time": "2025-10-22 16:57:12.327000000", + "duration_ms": 5721.561000000001, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "How do I set up observability with Noveum?", + "query_length": 42, + "query_type": "routing_query", + "output_response": "Routed to RAG for query processing", + "router.classification": "RAG", + "router.confidence_score": 1, + "router.rag_keyword_score": 2, + "router.web_keyword_score": 0, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [] + }, + { + "span_id": "11fc18a8-0380-48ca-894e-cc26f648d290", + "trace_id": "93bd37c4-4289-43c9-b34e-19eaaef610c2", + "parent_span_id": "24930c28-7f93-426c-8914-bd0e0085589f", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-22 16:57:06.605000000", + "end_time": "2025-10-22 16:57:06.606000000", + "duration_ms": 0.047, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: How do I set up observability with Noveum?", + "query_length": 42, + "query_type": "routing_evaluation_query", + "classification.mode": "RAG", + "classification.rag_keyword_score": 2, + "classification.web_keyword_score": 0, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "noveum_specific", + "query.keyword_density": 0.25, + "query.domain_affinity": "noveum", + "routing.decision": "Routed to RAG based on analysis", + "routing.rationale": "RAG score: 2, Web score: 0, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "Web Search", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.038461538461538464, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: RAG (Confidence: 1.00)" + }, + "events": [], + "links": [] + }, + { + "span_id": "bfc91518-f818-4f35-9113-70b04eb55edc", + "trace_id": "93bd37c4-4289-43c9-b34e-19eaaef610c2", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-22 16:57:06.605000000", + "end_time": "2025-10-22 16:57:12.327000000", + "duration_ms": 5721.635, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "How do I set up observability with Noveum?", + "query.length": 42, + "processing.mode": "RAG", + "processing.time_seconds": 5.721601963043213, + "processing.response_length": 815, + "processing.sources_count": 5, + "output_response": "Final Answer: To set up observability with Noveum, follow these steps:\n\n1. **Sign Up**: Go to [noveum.ai](https://noveum.ai) and create an account.\n2. **Create a Project**: After signing up, create a project to obt...", + "final_answer_mode": "RAG", + "query_processed.input_query": "How do I set up observability with Noveum?", + "query_processed.output_response": "Successfully processed query using RAG, generated 815 character response", + "query_processed.mode": "RAG", + "query_processed.processing_time": 5.721601963043213, + "query_processed.response_length": 815 + }, + "events": [], + "links": [] + }, + { + "span_id": "2a82e4b1-c93d-498f-9dd5-e971f228d642", + "trace_id": "93bd37c4-4289-43c9-b34e-19eaaef610c2", + "parent_span_id": "24930c28-7f93-426c-8914-bd0e0085589f", + "name": "agent.llm-rag", + "start_time": "2025-10-22 16:57:06.606000000", + "end_time": "2025-10-22 16:57:12.327000000", + "duration_ms": 5721.352000000001, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "rag_agent", + "agent.operation": "llm-rag", + "agent.capabilities": [ + "document_retrieval", + "context_generation", + "response_generation" + ], + "agent.id": "noveum_rag_agent", + "input_query": "How do I set up observability with Noveum?", + "query_length": 42, + "query_type": "rag_query", + "output_response": "RAG Answer: To set up observability with Noveum, follow these steps:\n\n1. **Sign Up**: Go to [noveum.ai](https://noveum.ai) and create an account.\n2. **Create a Project**: After signing up, create a project to obt...", + "rag.context_length": 2870, + "rag.sources_count": 6, + "rag.answer_length": 815, + "rag.mode": "retrieval_augmented_generation" + }, + "events": [], + "links": [] + }, + { + "span_id": "520b4e59-da14-4db2-af02-0973cda9bd86", + "trace_id": "93bd37c4-4289-43c9-b34e-19eaaef610c2", + "parent_span_id": "2a82e4b1-c93d-498f-9dd5-e971f228d642", + "name": "agent.llm_model_execution", + "start_time": "2025-10-22 16:57:07.109000000", + "end_time": "2025-10-22 16:57:11.837000000", + "duration_ms": 4728.018999999999, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: How do I set up observability with Noveum?", + "query_length": 42, + "query_type": "rag_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 4.727963924407959, + "model.latency_ms": 4727.963924407959, + "model.start_time": 1761152227.109188, + "model.end_time": 1761152231.837152, + "model.performance_tier": "medium", + "model.prompt_tokens": 852, + "model.completion_tokens": 203, + "model.total_tokens": 1055, + "model.tokens_per_second": 223.1404504915101, + "model.estimated_cost": 0.03165, + "model.efficiency_score": 0.7725118483412322, + "model.response_length": 815, + "model.response_quality": "high", + "model.output_response": "Model Response: To set up observability with Noveum, follow these steps:\n\n1. **Sign Up**: Go to [noveum.ai](https://noveum.ai) and create an account.\n2. **Create a Project**: After signing up, create a project to obt...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [] + }, + { + "span_id": "dff080ab-238c-4cca-a910-cf530f910bda", + "trace_id": "93bd37c4-4289-43c9-b34e-19eaaef610c2", + "parent_span_id": "2a82e4b1-c93d-498f-9dd5-e971f228d642", + "name": "agent.rag_evaluation_metrics", + "start_time": "2025-10-22 16:57:11.837000000", + "end_time": "2025-10-22 16:57:11.838000000", + "duration_ms": 0.259, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "rag_evaluation_metrics", + "agent.capabilities": [ + "retrieval_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: How do I set up observability with Noveum?", + "query_length": 42, + "query_type": "rag_evaluation_query", + "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\n- Production Ready: Built for scale with intelligent sampling and batching\n- Privacy Focused: You control what data is captured and how it's stored\n- Open Source SDKs: Transparent, auditable, and extensible\nGetting ...", + "retrieval.context_length": 2870, + "retrieval.sources_count": 6, + "retrieval.context_quality": "high", + "retrieval.effectiveness": 1.2, + "retrieval.context_utilization": 2.87, + "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\n- Production Ready: Built for scale with intelligent sampling and batching\n- Privacy Focused: You control what data is captured and how it's stored\n- Open Source SDKs: Transparent, auditable, and extensible\nGetting Started\nReady to add comprehensive observability to your AI applications?\n- Sign up at noveum.ai\n- Create a project and get your API key\n- Install the SDK for your preferred language\n- Add tracing to your AI workflows\n- Explore insights in the Noveum.ai dashboard\nCheck out our integra...\n\nSource 2 (https://noveum.ai/en/docs/getting-started/tracing-concepts):\n- Explore Framework Integrations - Framework-specific guidance\n- Learn Advanced Patterns - Custom instrumentation techniques\n- Master the Dashboard - Analyze your traces effectively\nRemember: Good observability is not about collecting all possible data, but about collecting the right data that helps you understand, debug, and optimize your AI applications.\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to O...\n\nSource 3 (https://noveum.ai/docs/getting-started/tracing-concepts):\n- Explore Framework Integrations - Framework-specific guidance\n- Learn Advanced Patterns - Custom instrumentation techniques\n- Master the Dashboard - Analyze your traces effectively\nRemember: Good observability is not about collecting all possible data, but about collecting the right data that helps you understand, debug, and optimize your AI applications.\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to O...\n\nSource 4 (https://noveum.ai/en/docs/getting-started/overview):\nNoveum.ai Overview\nComprehensive AI tracing and observability platform for LLM applications, RAG systems, and AI agents\nWelcome to Noveum.ai\u2014the comprehensive tracing and observability platform built specifically for AI applications. Whether you're building LLM-powered chatbots, RAG systems, multi-agent workflows, or any AI-driven application, Noveum provides the insights you need to understand, debug, and optimize your systems.\n\ud83c\udfaf Why AI Applications Need Specialized Observability\nTraditional mo...\n\nSource 5 (https://noveum.ai/docs/getting-started/overview):\nNoveum.ai Overview\nComprehensive AI tracing and observability platform for LLM applications, RAG systems, and AI agents\nWelcome to Noveum.ai\u2014the comprehensive tracing and observability platform built specifically for AI applications. Whether you're building LLM-powered chatbots, RAG systems, multi-agent workflows, or any AI-driven application, Noveum provides the insights you need to understand, debug, and optimize your systems.\n\ud83c\udfaf Why AI Applications Need Specialized Observability\nTraditional mo...\n\nUser Question: How do I set up observability with Noveum?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", + "prompt.prompt_length": 3408, + "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\n- Production Ready: Built for scale with intelligent sampling and batching\n- Privacy Focused: You control what data...", + "prompt.instruction_following": "rag_optimized", + "response.answer_length": 815, + "response.answer_completeness": "complete", + "response.response_quality": "high", + "response.source_citation": 6, + "response.context_utilization": 0.28397212543554007, + "output_response": "RAG Answer: To set up observability with Noveum, follow these steps:\n\n1. **Sign Up**: Go to [noveum.ai](https://noveum.ai) and create an account.\n2. **Create a Project**: After signing up, create a project to obt...", + "evaluation.retrieval_effectiveness": 1.2, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 6, + "evaluation.context_relevance": "high", + "evaluation.overall_quality": "high", + "evaluation.ready_for_production": true, + "rag.retrieval_strategy": "semantic_similarity", + "rag.vector_search_results": 6, + "rag.context_synthesis": "multi_source", + "rag.document_coverage": 1.2, + "rag.information_density": 0.28397212543554007 + }, + "events": [], + "links": [] + } + ], + "created_at": "2025-10-22 16:57:06.605000000", + "updated_at": "2025-10-22 16:57:06.605000000" + }, + { + "trace_id": "df0c6344-1a2e-4c53-bb7d-4e43b3bce1b8", + "name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "status": "ok", + "status_message": "", + "start_time": "2025-10-22 16:56:59.213000000", + "end_time": "2025-10-22 16:57:05.598000000", + "duration_ms": 6384.386, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "spans": [ + { + "span_id": "dad7df65-be49-4245-8721-c91da4962b9a", + "trace_id": "df0c6344-1a2e-4c53-bb7d-4e43b3bce1b8", + "parent_span_id": "9024b0d9-2b75-428f-af8f-75a921a7c69c", + "name": "agent.query_routing", + "start_time": "2025-10-22 16:56:59.213000000", + "end_time": "2025-10-22 16:57:05.598000000", + "duration_ms": 6384.249, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What features does Noveum Trace offer?", + "query_length": 38, + "query_type": "routing_query", + "output_response": "Routed to RAG for query processing", + "router.classification": "RAG", + "router.confidence_score": 1, + "router.rag_keyword_score": 3, + "router.web_keyword_score": 0, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [] + }, + { + "span_id": "9024b0d9-2b75-428f-af8f-75a921a7c69c", + "trace_id": "df0c6344-1a2e-4c53-bb7d-4e43b3bce1b8", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-22 16:56:59.213000000", + "end_time": "2025-10-22 16:57:05.598000000", + "duration_ms": 6384.32, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What features does Noveum Trace offer?", + "query.length": 38, + "processing.mode": "RAG", + "processing.time_seconds": 6.384284973144531, + "processing.response_length": 1017, + "processing.sources_count": 5, + "output_response": "Final Answer: Noveum Trace offers several key features through its Python SDK (`noveum-trace`) and TypeScript SDK (`@noveum/trace`):\n\n1. **Decorator-based Tracing**: It provides a simple way to trace LLM calls, age...", + "final_answer_mode": "RAG", + "query_processed.input_query": "What features does Noveum Trace offer?", + "query_processed.output_response": "Successfully processed query using RAG, generated 1017 character response", + "query_processed.mode": "RAG", + "query_processed.processing_time": 6.384284973144531, + "query_processed.response_length": 1017 + }, + "events": [], + "links": [] + }, + { + "span_id": "d07fdbdc-2b8f-4de9-ac32-988389701771", + "trace_id": "df0c6344-1a2e-4c53-bb7d-4e43b3bce1b8", + "parent_span_id": "dad7df65-be49-4245-8721-c91da4962b9a", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-22 16:56:59.214000000", + "end_time": "2025-10-22 16:56:59.214000000", + "duration_ms": 0.060000000000000005, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What features does Noveum Trace offer?", + "query_length": 38, + "query_type": "routing_evaluation_query", + "classification.mode": "RAG", + "classification.rag_keyword_score": 3, + "classification.web_keyword_score": 0, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "noveum_specific", + "query.keyword_density": 0.5, + "query.domain_affinity": "noveum", + "routing.decision": "Routed to RAG based on analysis", + "routing.rationale": "RAG score: 3, Web score: 0, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "Web Search", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.057692307692307696, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: RAG (Confidence: 1.00)" + }, + "events": [], + "links": [] + }, + { + "span_id": "83f86157-5af4-4c99-aad1-647866cd804e", + "trace_id": "df0c6344-1a2e-4c53-bb7d-4e43b3bce1b8", + "parent_span_id": "dad7df65-be49-4245-8721-c91da4962b9a", + "name": "agent.llm-rag", + "start_time": "2025-10-22 16:56:59.214000000", + "end_time": "2025-10-22 16:57:05.598000000", + "duration_ms": 6384.048, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "rag_agent", + "agent.operation": "llm-rag", + "agent.capabilities": [ + "document_retrieval", + "context_generation", + "response_generation" + ], + "agent.id": "noveum_rag_agent", + "input_query": "What features does Noveum Trace offer?", + "query_length": 38, + "query_type": "rag_query", + "output_response": "RAG Answer: Noveum Trace offers several key features through its Python SDK (`noveum-trace`) and TypeScript SDK (`@noveum/trace`):\n\n1. **Decorator-based Tracing**: It provides a simple way to trace LLM calls, age...", + "rag.context_length": 2787, + "rag.sources_count": 5, + "rag.answer_length": 1017, + "rag.mode": "retrieval_augmented_generation" + }, + "events": [], + "links": [] + }, + { + "span_id": "3b4fa9a7-88bf-4ee1-aeb7-bfa9573f28f2", + "trace_id": "df0c6344-1a2e-4c53-bb7d-4e43b3bce1b8", + "parent_span_id": "83f86157-5af4-4c99-aad1-647866cd804e", + "name": "agent.llm_model_execution", + "start_time": "2025-10-22 16:56:59.591000000", + "end_time": "2025-10-22 16:57:04.869000000", + "duration_ms": 5278.925, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What features does Noveum Trace offer?", + "query_length": 38, + "query_type": "rag_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 5.278868913650513, + "model.latency_ms": 5278.868913650513, + "model.start_time": 1761152219.5910342, + "model.end_time": 1761152224.869903, + "model.performance_tier": "slow", + "model.prompt_tokens": 830, + "model.completion_tokens": 254, + "model.total_tokens": 1084, + "model.tokens_per_second": 205.3470199263535, + "model.estimated_cost": 0.03252, + "model.efficiency_score": 0.9381918819188192, + "model.response_length": 1017, + "model.response_quality": "high", + "model.output_response": "Model Response: Noveum Trace offers several key features through its Python SDK (`noveum-trace`) and TypeScript SDK (`@noveum/trace`):\n\n1. **Decorator-based Tracing**: It provides a simple way to trace LLM calls, age...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [] + }, + { + "span_id": "f9214dbe-adb9-44ff-8361-70c872cef66d", + "trace_id": "df0c6344-1a2e-4c53-bb7d-4e43b3bce1b8", + "parent_span_id": "83f86157-5af4-4c99-aad1-647866cd804e", + "name": "agent.rag_evaluation_metrics", + "start_time": "2025-10-22 16:57:04.870000000", + "end_time": "2025-10-22 16:57:04.870000000", + "duration_ms": 0.043000000000000003, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "rag_evaluation_metrics", + "agent.capabilities": [ + "retrieval_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What features does Noveum Trace offer?", + "query_length": 38, + "query_type": "rag_evaluation_query", + "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\n\ud83d\udcd8 TypeScript SDK (\n@noveum/trace\n)- Framework ...", + "retrieval.context_length": 2787, + "retrieval.sources_count": 5, + "retrieval.context_quality": "high", + "retrieval.effectiveness": 1, + "retrieval.context_utilization": 2.787, + "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\n\ud83d\udcd8 TypeScript SDK (\n@noveum/trace\n)- Framework integrations for Next.js, Express.js, and Hono\n- TypeScript-first design with full type safety\n- Universal compatibility (Node.js, Edge Runtime, browsers)\n-\n\ud83d\udcca Noveum Platform\n- Real-time dashboard for analyzing traces and performance\n- Ad...\n\nSource 2 (https://noveum.ai/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\n\ud83d\udcd8 TypeScript SDK (\n@noveum/trace\n)- Framework integrations for Next.js, Express.js, and Hono\n- TypeScript-first design with full type safety\n- Universal compatibility (Node.js, Edge Runtime, browsers)\n-\n\ud83d\udcca Noveum Platform\n- Real-time dashboard for analyzing traces and performance\n- Ad...\n\nSource 3 (https://noveum.ai/en/docs/getting-started/sdk-integration):\n- \ud83d\udc65 Collaborate with your team on insights\n\ud83d\udd12 Security & Privacy\n- \ud83d\udd10 Encryption: All data encrypted in transit and at rest\n- \ud83c\udf9b\ufe0f Configurable Capture: Control what data is collected\n- \ud83c\udfe0 Data Residency: Choose your data storage region\n- \u23f0 Retention Control: Set custom data retention policies\nNext Steps\n- Tracing Concepts - Learn about traces, spans, and observability best practices\n- Framework Integrations - Deep dive into specific framework setups\n- Multi-Agent Tracing - Observe complex agent work...\n\nSource 4 (https://noveum.ai/docs/getting-started/sdk-integration):\n- \ud83d\udc65 Collaborate with your team on insights\n\ud83d\udd12 Security & Privacy\n- \ud83d\udd10 Encryption: All data encrypted in transit and at rest\n- \ud83c\udf9b\ufe0f Configurable Capture: Control what data is collected\n- \ud83c\udfe0 Data Residency: Choose your data storage region\n- \u23f0 Retention Control: Set custom data retention policies\nNext Steps\n- Tracing Concepts - Learn about traces, spans, and observability best practices\n- Framework Integrations - Deep dive into specific framework setups\n- Multi-Agent Tracing - Observe complex agent work...\n\nSource 5 (https://noveum.ai/docs/platform/dashboard):\n- Bulk Export: Download multiple traces simultaneously\n- Comparative Analysis: Compare performance across multiple traces\nReal-Time Updates\n- Live Refresh: Automatic updates as new traces arrive\n- Connection Monitoring: Real-time status of your trace ingestion pipeline\n- Performance Indicators: Live metrics for system health monitoring\nReady to dive deeper? Explore Projects & Environments to organize your AI applications, or check out Team Collaboration to share insights with your team.\nGet Earl...\n\nUser Question: What features does Noveum Trace offer?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", + "prompt.prompt_length": 3321, + "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frame...", + "prompt.instruction_following": "rag_optimized", + "response.answer_length": 1017, + "response.answer_completeness": "complete", + "response.response_quality": "high", + "response.source_citation": 5, + "response.context_utilization": 0.36490850376749195, + "output_response": "RAG Answer: Noveum Trace offers several key features through its Python SDK (`noveum-trace`) and TypeScript SDK (`@noveum/trace`):\n\n1. **Decorator-based Tracing**: It provides a simple way to trace LLM calls, age...", + "evaluation.retrieval_effectiveness": 1, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 5, + "evaluation.context_relevance": "high", + "evaluation.overall_quality": "high", + "evaluation.ready_for_production": true, + "rag.retrieval_strategy": "semantic_similarity", + "rag.vector_search_results": 5, + "rag.context_synthesis": "multi_source", + "rag.document_coverage": 1, + "rag.information_density": 0.36490850376749195 + }, + "events": [], + "links": [] + } + ], + "created_at": "2025-10-22 16:56:59.213000000", + "updated_at": "2025-10-22 16:56:59.213000000" + }, + { + "trace_id": "29a862bf-d7aa-4cf9-963c-0a4244343bbe", + "name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "status": "ok", + "status_message": "", + "start_time": "2025-10-22 16:56:54.718000000", + "end_time": "2025-10-22 16:56:58.207000000", + "duration_ms": 3488.068, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "spans": [ + { + "span_id": "138a5089-5268-447a-afb5-4f14fb504281", + "trace_id": "29a862bf-d7aa-4cf9-963c-0a4244343bbe", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-22 16:56:54.719000000", + "end_time": "2025-10-22 16:56:58.207000000", + "duration_ms": 3487.959, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What are Noveum's pricing plans?", + "query.length": 32, + "processing.mode": "RAG", + "processing.time_seconds": 3.487928867340088, + "processing.response_length": 441, + "processing.sources_count": 5, + "output_response": "Final Answer: The provided context does not include specific information about Noveum's pricing plans. It mentions that all users will have access to the Observability suite for free, and early users will receive f...", + "final_answer_mode": "RAG", + "query_processed.input_query": "What are Noveum's pricing plans?", + "query_processed.output_response": "Successfully processed query using RAG, generated 441 character response", + "query_processed.mode": "RAG", + "query_processed.processing_time": 3.487928867340088, + "query_processed.response_length": 441 + }, + "events": [], + "links": [] + }, + { + "span_id": "7cda4dab-0fc5-4cae-8d8c-9445f661c67b", + "trace_id": "29a862bf-d7aa-4cf9-963c-0a4244343bbe", + "parent_span_id": "138a5089-5268-447a-afb5-4f14fb504281", + "name": "agent.query_routing", + "start_time": "2025-10-22 16:56:54.719000000", + "end_time": "2025-10-22 16:56:58.207000000", + "duration_ms": 3487.848, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What are Noveum's pricing plans?", + "query_length": 32, + "query_type": "routing_query", + "output_response": "Routed to RAG for query processing", + "router.classification": "RAG", + "router.confidence_score": 1, + "router.rag_keyword_score": 3, + "router.web_keyword_score": 0, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [] + }, + { + "span_id": "922c84a5-b14a-4b2f-8759-427d6e089812", + "trace_id": "29a862bf-d7aa-4cf9-963c-0a4244343bbe", + "parent_span_id": "7cda4dab-0fc5-4cae-8d8c-9445f661c67b", + "name": "agent.llm-rag", + "start_time": "2025-10-22 16:56:54.719000000", + "end_time": "2025-10-22 16:56:58.207000000", + "duration_ms": 3487.546, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "rag_agent", + "agent.operation": "llm-rag", + "agent.capabilities": [ + "document_retrieval", + "context_generation", + "response_generation" + ], + "agent.id": "noveum_rag_agent", + "input_query": "What are Noveum's pricing plans?", + "query_length": 32, + "query_type": "rag_query", + "output_response": "RAG Answer: The provided context does not include specific information about Noveum's pricing plans. It mentions that all users will have access to the Observability suite for free, and early users will receive f...", + "rag.context_length": 2105, + "rag.sources_count": 5, + "rag.answer_length": 441, + "rag.mode": "retrieval_augmented_generation" + }, + "events": [], + "links": [] + }, + { + "span_id": "c8d7bb28-e763-47c9-b3e9-77f6a91a22be", + "trace_id": "29a862bf-d7aa-4cf9-963c-0a4244343bbe", + "parent_span_id": "7cda4dab-0fc5-4cae-8d8c-9445f661c67b", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-22 16:56:54.719000000", + "end_time": "2025-10-22 16:56:54.719000000", + "duration_ms": 0.068, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What are Noveum's pricing plans?", + "query_length": 32, + "query_type": "routing_evaluation_query", + "classification.mode": "RAG", + "classification.rag_keyword_score": 3, + "classification.web_keyword_score": 0, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "noveum_specific", + "query.keyword_density": 0.6, + "query.domain_affinity": "noveum", + "routing.decision": "Routed to RAG based on analysis", + "routing.rationale": "RAG score: 3, Web score: 0, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "Web Search", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.057692307692307696, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: RAG (Confidence: 1.00)" + }, + "events": [], + "links": [] + }, + { + "span_id": "a378123a-8a79-4bc5-8e20-8ec54c2f9783", + "trace_id": "29a862bf-d7aa-4cf9-963c-0a4244343bbe", + "parent_span_id": "922c84a5-b14a-4b2f-8759-427d6e089812", + "name": "agent.llm_model_execution", + "start_time": "2025-10-22 16:56:55.154000000", + "end_time": "2025-10-22 16:56:57.166000000", + "duration_ms": 2012.174, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What are Noveum's pricing plans?", + "query_length": 32, + "query_type": "rag_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 2.012089252471924, + "model.latency_ms": 2012.0892524719238, + "model.start_time": 1761152215.154625, + "model.end_time": 1761152217.1667142, + "model.performance_tier": "medium", + "model.prompt_tokens": 658, + "model.completion_tokens": 110, + "model.total_tokens": 768, + "model.tokens_per_second": 381.6928096288395, + "model.estimated_cost": 0.02304, + "model.efficiency_score": 0.57421875, + "model.response_length": 441, + "model.response_quality": "high", + "model.output_response": "Model Response: The provided context does not include specific information about Noveum's pricing plans. It mentions that all users will have access to the Observability suite for free, and early users will receive f...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [] + }, + { + "span_id": "5813673b-a1dc-46c7-9bea-92ce6aca2b5e", + "trace_id": "29a862bf-d7aa-4cf9-963c-0a4244343bbe", + "parent_span_id": "922c84a5-b14a-4b2f-8759-427d6e089812", + "name": "agent.rag_evaluation_metrics", + "start_time": "2025-10-22 16:56:57.166000000", + "end_time": "2025-10-22 16:56:57.167000000", + "duration_ms": 0.059, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "rag_evaluation_metrics", + "agent.capabilities": [ + "retrieval_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What are Noveum's pricing plans?", + "query_length": 32, + "query_type": "rag_evaluation_query", + "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....", + "retrieval.context_length": 2105, + "retrieval.sources_count": 5, + "retrieval.context_quality": "high", + "retrieval.effectiveness": 1, + "retrieval.context_utilization": 2.105, + "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nSource 2 (https://noveum.ai/en/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nSource 3 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nThe AI landscape is complex and moving fast. With Noveum.ai, you'll have the insights you need to build, debug, and optimize AI applications with confidence.\nReady to see what's really happening inside your AI applications? Try Noveum.ai today and transform how you build and monitor AI systems.\nLet's build more reliable, observable AI\u2014together.\nGet Early Access to Noveum.ai Platform\nJoin the select group of AI teams optimizing their models with our data-driven platform. We're onboarding users in...\n\nSource 4 (https://noveum.ai/en/blog/from-logs-to-intelligent-choices-inside-noveum-ais-evaluation-process):\nGet Early Access to Noveum.ai Platform\nJoin the select group of AI teams optimizing their models with our data-driven platform. We're onboarding users in limited batches to ensure a premium experience....\n\nSource 5 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, vector searches, data retrieval, agent reasoning, and more. Without proper observability, debugging becomes a nightmare, optimization is guesswork, and understanding user interactions is nearly impossible.\nNoveum.ai solves this by providing comprehensive tracing and observability specifically designed for AI applications.\nKey benefits of using Noveum.ai:\n- Complete Visibility: Trace...\n\nUser Question: What are Noveum's pricing plans?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", + "prompt.prompt_length": 2633, + "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to ...", + "prompt.instruction_following": "rag_optimized", + "response.answer_length": 441, + "response.answer_completeness": "complete", + "response.response_quality": "high", + "response.source_citation": 5, + "response.context_utilization": 0.20950118764845604, + "output_response": "RAG Answer: The provided context does not include specific information about Noveum's pricing plans. It mentions that all users will have access to the Observability suite for free, and early users will receive f...", + "evaluation.retrieval_effectiveness": 1, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 5, + "evaluation.context_relevance": "high", + "evaluation.overall_quality": "high", + "evaluation.ready_for_production": true, + "rag.retrieval_strategy": "semantic_similarity", + "rag.vector_search_results": 5, + "rag.context_synthesis": "multi_source", + "rag.document_coverage": 1, + "rag.information_density": 0.20950118764845604 + }, + "events": [], + "links": [] + } + ], + "created_at": "2025-10-22 16:56:54.718000000", + "updated_at": "2025-10-22 16:56:54.718000000" + }, + { + "trace_id": "bc60c122-f30b-4114-9f78-ee40a024b525", + "name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "status": "ok", + "status_message": "", + "start_time": "2025-10-22 16:56:44.873000000", + "end_time": "2025-10-22 16:56:53.701000000", + "duration_ms": 8828.055, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "spans": [ + { + "span_id": "c5baf5c2-d18a-458a-b922-a19b96db53d9", + "trace_id": "bc60c122-f30b-4114-9f78-ee40a024b525", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-22 16:56:44.873000000", + "end_time": "2025-10-22 16:56:53.701000000", + "duration_ms": 8827.979, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "How do I integrate Noveum Trace in my application?", + "query.length": 50, + "processing.mode": "RAG", + "processing.time_seconds": 8.827935218811035, + "processing.response_length": 1145, + "processing.sources_count": 5, + "output_response": "Final Answer: To integrate Noveum Trace into your application, you can follow these steps based on the SDK Integration Guide:\n\n1. **Create Your Account & Get API Key**:\n - Sign up at [noveum.ai](https://noveum.ai...", + "final_answer_mode": "RAG", + "query_processed.input_query": "How do I integrate Noveum Trace in my application?", + "query_processed.output_response": "Successfully processed query using RAG, generated 1145 character response", + "query_processed.mode": "RAG", + "query_processed.processing_time": 8.827935218811035, + "query_processed.response_length": 1145 + }, + "events": [], + "links": [] + }, + { + "span_id": "8819dde5-9cbe-4cdb-b919-9c5140582359", + "trace_id": "bc60c122-f30b-4114-9f78-ee40a024b525", + "parent_span_id": "c5baf5c2-d18a-458a-b922-a19b96db53d9", + "name": "agent.query_routing", + "start_time": "2025-10-22 16:56:44.873000000", + "end_time": "2025-10-22 16:56:53.701000000", + "duration_ms": 8827.883, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "How do I integrate Noveum Trace in my application?", + "query_length": 50, + "query_type": "routing_query", + "output_response": "Routed to RAG for query processing", + "router.classification": "RAG", + "router.confidence_score": 1, + "router.rag_keyword_score": 2, + "router.web_keyword_score": 0, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [] + }, + { + "span_id": "932c4b0e-0f33-40c0-bb19-78b9f81e48a6", + "trace_id": "bc60c122-f30b-4114-9f78-ee40a024b525", + "parent_span_id": "8819dde5-9cbe-4cdb-b919-9c5140582359", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-22 16:56:44.873000000", + "end_time": "2025-10-22 16:56:44.873000000", + "duration_ms": 0.043000000000000003, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: How do I integrate Noveum Trace in my application?", + "query_length": 50, + "query_type": "routing_evaluation_query", + "classification.mode": "RAG", + "classification.rag_keyword_score": 2, + "classification.web_keyword_score": 0, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "noveum_specific", + "query.keyword_density": 0.2222222222222222, + "query.domain_affinity": "noveum", + "routing.decision": "Routed to RAG based on analysis", + "routing.rationale": "RAG score: 2, Web score: 0, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "Web Search", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.038461538461538464, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: RAG (Confidence: 1.00)" + }, + "events": [], + "links": [] + }, + { + "span_id": "2ece2bd6-e4c8-47bc-b3c8-26f82e9910f1", + "trace_id": "bc60c122-f30b-4114-9f78-ee40a024b525", + "parent_span_id": "8819dde5-9cbe-4cdb-b919-9c5140582359", + "name": "agent.llm-rag", + "start_time": "2025-10-22 16:56:44.873000000", + "end_time": "2025-10-22 16:56:53.701000000", + "duration_ms": 8827.666000000001, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "rag_agent", + "agent.operation": "llm-rag", + "agent.capabilities": [ + "document_retrieval", + "context_generation", + "response_generation" + ], + "agent.id": "noveum_rag_agent", + "input_query": "How do I integrate Noveum Trace in my application?", + "query_length": 50, + "query_type": "rag_query", + "output_response": "RAG Answer: To integrate Noveum Trace into your application, you can follow these steps based on the SDK Integration Guide:\n\n1. **Create Your Account & Get API Key**:\n - Sign up at [noveum.ai](https://noveum.ai...", + "rag.context_length": 2803, + "rag.sources_count": 5, + "rag.answer_length": 1145, + "rag.mode": "retrieval_augmented_generation" + }, + "events": [], + "links": [] + }, + { + "span_id": "e6315110-36b1-4503-beaf-12d972e25a7d", + "trace_id": "bc60c122-f30b-4114-9f78-ee40a024b525", + "parent_span_id": "2ece2bd6-e4c8-47bc-b3c8-26f82e9910f1", + "name": "agent.llm_model_execution", + "start_time": "2025-10-22 16:56:45.576000000", + "end_time": "2025-10-22 16:56:53.143000000", + "duration_ms": 7566.311, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: How do I integrate Noveum Trace in my application?", + "query_length": 50, + "query_type": "rag_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 7.566190004348755, + "model.latency_ms": 7566.190004348755, + "model.start_time": 1761152205.576988, + "model.end_time": 1761152213.143178, + "model.performance_tier": "slow", + "model.prompt_tokens": 837, + "model.completion_tokens": 286, + "model.total_tokens": 1123, + "model.tokens_per_second": 148.4234468543009, + "model.estimated_cost": 0.03369, + "model.efficiency_score": 1.0195903829029385, + "model.response_length": 1145, + "model.response_quality": "high", + "model.output_response": "Model Response: To integrate Noveum Trace into your application, you can follow these steps based on the SDK Integration Guide:\n\n1. **Create Your Account & Get API Key**:\n - Sign up at [noveum.ai](https://noveum.ai...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [] + }, + { + "span_id": "ae91d520-8442-48e5-aabf-fd58aed23694", + "trace_id": "bc60c122-f30b-4114-9f78-ee40a024b525", + "parent_span_id": "2ece2bd6-e4c8-47bc-b3c8-26f82e9910f1", + "name": "agent.rag_evaluation_metrics", + "start_time": "2025-10-22 16:56:53.143000000", + "end_time": "2025-10-22 16:56:53.143000000", + "duration_ms": 0.091, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "rag_evaluation_metrics", + "agent.capabilities": [ + "retrieval_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: How do I integrate Noveum Trace in my application?", + "query_length": 50, + "query_type": "rag_evaluation_query", + "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\n\ud83d\udcd8 TypeScript SDK (\n@noveum/trace\n)- Framework ...", + "retrieval.context_length": 2803, + "retrieval.sources_count": 5, + "retrieval.context_quality": "high", + "retrieval.effectiveness": 1, + "retrieval.context_utilization": 2.803, + "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\n\ud83d\udcd8 TypeScript SDK (\n@noveum/trace\n)- Framework integrations for Next.js, Express.js, and Hono\n- TypeScript-first design with full type safety\n- Universal compatibility (Node.js, Edge Runtime, browsers)\n-\n\ud83d\udcca Noveum Platform\n- Real-time dashboard for analyzing traces and performance\n- Ad...\n\nSource 2 (https://noveum.ai/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\n\ud83d\udcd8 TypeScript SDK (\n@noveum/trace\n)- Framework integrations for Next.js, Express.js, and Hono\n- TypeScript-first design with full type safety\n- Universal compatibility (Node.js, Edge Runtime, browsers)\n-\n\ud83d\udcca Noveum Platform\n- Real-time dashboard for analyzing traces and performance\n- Ad...\n\nSource 3 (https://noveum.ai/docs/getting-started/sdk-integration):\nSDK Integration Guide\nIntegrate Noveum.ai tracing into your AI applications with Python or TypeScript SDKs\nThe Noveum.ai SDKs provide comprehensive tracing and observability for your AI applications with minimal code changes. Whether you're building LLM applications, RAG systems, or multi-agent workflows, our SDKs automatically capture essential metrics and traces.\n\ud83d\ude80 Quick Start\n1. Create Your Account & Get API Key\n- Sign up at noveum.ai\n- Create a project in your dashboard\n- Generate an API key...\n\nSource 4 (https://noveum.ai/en/docs/getting-started/sdk-integration):\nSDK Integration Guide\nIntegrate Noveum.ai tracing into your AI applications with Python or TypeScript SDKs\nThe Noveum.ai SDKs provide comprehensive tracing and observability for your AI applications with minimal code changes. Whether you're building LLM applications, RAG systems, or multi-agent workflows, our SDKs automatically capture essential metrics and traces.\n\ud83d\ude80 Quick Start\n1. Create Your Account & Get API Key\n- Sign up at noveum.ai\n- Create a project in your dashboard\n- Generate an API key...\n\nSource 5 (https://noveum.ai/en/docs/getting-started/sdk-integration):\n\ud83d\udd27 Framework Integrations\nNext.js Integration\nExpress.js Integration\nFastAPI Integration (Python)\n\ud83d\udcca Advanced Features\nCustom Attributes & Events\nSampling Configuration\n\ud83d\udd17 What's Captured Automatically\n- \ud83d\udcca Performance Metrics: Latency, throughput, error rates\n- \ud83d\udcb0 Cost Tracking: Token usage, API costs across providers\n- \ud83d\udd0d Request/Response: Configurable capture of inputs/outputs\n- \ud83c\udff7\ufe0f Metadata: Model names, parameters, user context\n- \ud83c\udf0a Context Flow: Trace relationships across services\n- \ud83d\udc1b Error Detail...\n\nUser Question: How do I integrate Noveum Trace in my application?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", + "prompt.prompt_length": 3349, + "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frame...", + "prompt.instruction_following": "rag_optimized", + "response.answer_length": 1145, + "response.answer_completeness": "complete", + "response.response_quality": "high", + "response.source_citation": 5, + "response.context_utilization": 0.4084909026043525, + "output_response": "RAG Answer: To integrate Noveum Trace into your application, you can follow these steps based on the SDK Integration Guide:\n\n1. **Create Your Account & Get API Key**:\n - Sign up at [noveum.ai](https://noveum.ai...", + "evaluation.retrieval_effectiveness": 1, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 5, + "evaluation.context_relevance": "high", + "evaluation.overall_quality": "high", + "evaluation.ready_for_production": true, + "rag.retrieval_strategy": "semantic_similarity", + "rag.vector_search_results": 5, + "rag.context_synthesis": "multi_source", + "rag.document_coverage": 1, + "rag.information_density": 0.4084909026043525 + }, + "events": [], + "links": [] + } + ], + "created_at": "2025-10-22 16:56:44.873000000", + "updated_at": "2025-10-22 16:56:44.873000000" + }, + { + "trace_id": "3ad192e5-bbd4-4236-8144-019d53512769", + "name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "status": "ok", + "status_message": "", + "start_time": "2025-10-22 16:56:35.188000000", + "end_time": "2025-10-22 16:56:43.863000000", + "duration_ms": 8675.356000000002, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "spans": [ + { + "span_id": "9a9c611d-f552-4b06-a764-4463edb70e2c", + "trace_id": "3ad192e5-bbd4-4236-8144-019d53512769", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-22 16:56:35.188000000", + "end_time": "2025-10-22 16:56:43.863000000", + "duration_ms": 8675.25, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What is Noveum and what does it do?", + "query.length": 35, + "processing.mode": "RAG", + "processing.time_seconds": 8.67501425743103, + "processing.response_length": 527, + "processing.sources_count": 5, + "output_response": "Final Answer: Noveum.ai is a comprehensive tracing and observability platform specifically designed for AI applications, including LLM-powered chatbots, RAG systems, and multi-agent workflows. It provides the neces...", + "final_answer_mode": "RAG", + "query_processed.input_query": "What is Noveum and what does it do?", + "query_processed.output_response": "Successfully processed query using RAG, generated 527 character response", + "query_processed.mode": "RAG", + "query_processed.processing_time": 8.67501425743103, + "query_processed.response_length": 527 + }, + "events": [], + "links": [] + }, + { + "span_id": "99eadeae-0733-4bf3-982e-e9f3014f5913", + "trace_id": "3ad192e5-bbd4-4236-8144-019d53512769", + "parent_span_id": "7e2c959c-157a-43d5-9870-32994cddd388", + "name": "agent.llm-rag", + "start_time": "2025-10-22 16:56:35.188000000", + "end_time": "2025-10-22 16:56:43.863000000", + "duration_ms": 8674.699999999999, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "rag_agent", + "agent.operation": "llm-rag", + "agent.capabilities": [ + "document_retrieval", + "context_generation", + "response_generation" + ], + "agent.id": "noveum_rag_agent", + "input_query": "What is Noveum and what does it do?", + "query_length": 35, + "query_type": "rag_query", + "output_response": "RAG Answer: Noveum.ai is a comprehensive tracing and observability platform specifically designed for AI applications, including LLM-powered chatbots, RAG systems, and multi-agent workflows. It provides the neces...", + "rag.context_length": 2334, + "rag.sources_count": 5, + "rag.answer_length": 527, + "rag.mode": "retrieval_augmented_generation" + }, + "events": [], + "links": [] + }, + { + "span_id": "7e2c959c-157a-43d5-9870-32994cddd388", + "trace_id": "3ad192e5-bbd4-4236-8144-019d53512769", + "parent_span_id": "9a9c611d-f552-4b06-a764-4463edb70e2c", + "name": "agent.query_routing", + "start_time": "2025-10-22 16:56:35.188000000", + "end_time": "2025-10-22 16:56:43.863000000", + "duration_ms": 8674.961, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What is Noveum and what does it do?", + "query_length": 35, + "query_type": "routing_query", + "output_response": "Routed to RAG for query processing", + "router.classification": "RAG", + "router.confidence_score": 1, + "router.rag_keyword_score": 1, + "router.web_keyword_score": 0, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [] + }, + { + "span_id": "6f81284c-a9aa-4d62-a402-1b474f2eba51", + "trace_id": "3ad192e5-bbd4-4236-8144-019d53512769", + "parent_span_id": "7e2c959c-157a-43d5-9870-32994cddd388", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-22 16:56:35.188000000", + "end_time": "2025-10-22 16:56:35.188000000", + "duration_ms": 0.054, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What is Noveum and what does it do?", + "query_length": 35, + "query_type": "routing_evaluation_query", + "classification.mode": "RAG", + "classification.rag_keyword_score": 1, + "classification.web_keyword_score": 0, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "noveum_specific", + "query.keyword_density": 0.125, + "query.domain_affinity": "noveum", + "routing.decision": "Routed to RAG based on analysis", + "routing.rationale": "RAG score: 1, Web score: 0, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "Web Search", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.019230769230769232, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: RAG (Confidence: 1.00)" + }, + "events": [], + "links": [] + }, + { + "span_id": "9f42d18a-0235-4ae2-838c-19a88ddc95e1", + "trace_id": "3ad192e5-bbd4-4236-8144-019d53512769", + "parent_span_id": "99eadeae-0733-4bf3-982e-e9f3014f5913", + "name": "agent.llm_model_execution", + "start_time": "2025-10-22 16:56:36.031000000", + "end_time": "2025-10-22 16:56:42.012000000", + "duration_ms": 5980.947999999999, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What is Noveum and what does it do?", + "query_length": 35, + "query_type": "rag_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 5.980915784835815, + "model.latency_ms": 5980.915784835815, + "model.start_time": 1761152196.031816, + "model.end_time": 1761152202.0127318, + "model.performance_tier": "slow", + "model.prompt_tokens": 716, + "model.completion_tokens": 131, + "model.total_tokens": 847, + "model.tokens_per_second": 141.61710856178715, + "model.estimated_cost": 0.025410000000000002, + "model.efficiency_score": 0.6221959858323495, + "model.response_length": 527, + "model.response_quality": "high", + "model.output_response": "Model Response: Noveum.ai is a comprehensive tracing and observability platform specifically designed for AI applications, including LLM-powered chatbots, RAG systems, and multi-agent workflows. It provides the neces...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [] + }, + { + "span_id": "7bcbe4b7-585a-4c9d-add8-16e8aa62254f", + "trace_id": "3ad192e5-bbd4-4236-8144-019d53512769", + "parent_span_id": "99eadeae-0733-4bf3-982e-e9f3014f5913", + "name": "agent.rag_evaluation_metrics", + "start_time": "2025-10-22 16:56:42.012000000", + "end_time": "2025-10-22 16:56:42.012000000", + "duration_ms": 0.018000000000000002, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "rag_evaluation_metrics", + "agent.capabilities": [ + "retrieval_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What is Noveum and what does it do?", + "query_length": 35, + "query_type": "rag_evaluation_query", + "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, vector searches, data retrieval, agent reasoning, and more. Without proper observability, debugging ...", + "retrieval.context_length": 2334, + "retrieval.sources_count": 5, + "retrieval.context_quality": "high", + "retrieval.effectiveness": 1, + "retrieval.context_utilization": 2.334, + "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, vector searches, data retrieval, agent reasoning, and more. Without proper observability, debugging becomes a nightmare, optimization is guesswork, and understanding user interactions is nearly impossible.\nNoveum.ai solves this by providing comprehensive tracing and observability specifically designed for AI applications.\nKey benefits of using Noveum.ai:\n- Complete Visibility: Trace...\n\nSource 2 (https://noveum.ai/en/docs/getting-started/overview):\nNoveum.ai Overview\nComprehensive AI tracing and observability platform for LLM applications, RAG systems, and AI agents\nWelcome to Noveum.ai\u2014the comprehensive tracing and observability platform built specifically for AI applications. Whether you're building LLM-powered chatbots, RAG systems, multi-agent workflows, or any AI-driven application, Noveum provides the insights you need to understand, debug, and optimize your systems.\n\ud83c\udfaf Why AI Applications Need Specialized Observability\nTraditional mo...\n\nSource 3 (https://noveum.ai/docs/getting-started/overview):\nNoveum.ai Overview\nComprehensive AI tracing and observability platform for LLM applications, RAG systems, and AI agents\nWelcome to Noveum.ai\u2014the comprehensive tracing and observability platform built specifically for AI applications. Whether you're building LLM-powered chatbots, RAG systems, multi-agent workflows, or any AI-driven application, Noveum provides the insights you need to understand, debug, and optimize your systems.\n\ud83c\udfaf Why AI Applications Need Specialized Observability\nTraditional mo...\n\nSource 4 (https://noveum.ai/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nSource 5 (https://noveum.ai/en/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nUser Question: What is Noveum and what does it do?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", + "prompt.prompt_length": 2865, + "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, ...", + "prompt.instruction_following": "rag_optimized", + "response.answer_length": 527, + "response.answer_completeness": "complete", + "response.response_quality": "high", + "response.source_citation": 5, + "response.context_utilization": 0.22579263067694944, + "output_response": "RAG Answer: Noveum.ai is a comprehensive tracing and observability platform specifically designed for AI applications, including LLM-powered chatbots, RAG systems, and multi-agent workflows. It provides the neces...", + "evaluation.retrieval_effectiveness": 1, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 5, + "evaluation.context_relevance": "high", + "evaluation.overall_quality": "high", + "evaluation.ready_for_production": true, + "rag.retrieval_strategy": "semantic_similarity", + "rag.vector_search_results": 5, + "rag.context_synthesis": "multi_source", + "rag.document_coverage": 1, + "rag.information_density": 0.22579263067694944 + }, + "events": [], + "links": [] + } + ], + "created_at": "2025-10-22 16:56:35.188000000", + "updated_at": "2025-10-22 16:56:35.188000000" + }, + { + "trace_id": "9af2b4b0-0c2f-4d2e-ae99-30df0d2f0fa6", + "name": "auto_trace_system_initialization", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "status": "ok", + "status_message": "", + "start_time": "2025-10-22 16:56:34.565000000", + "end_time": "2025-10-22 16:56:35.186000000", + "duration_ms": 621.177, + "span_count": 1, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "spans": [ + { + "span_id": "d39297f4-b4fe-4f9c-b45b-854411ba770a", + "trace_id": "9af2b4b0-0c2f-4d2e-ae99-30df0d2f0fa6", + "parent_span_id": "", + "name": "system_initialization", + "start_time": "2025-10-22 16:56:34.565000000", + "end_time": "2025-10-22 16:56:35.186000000", + "duration_ms": 621.02, + "status": "ok", + "status_message": "", + "attributes": { + "system.force_scrape": false, + "system.config": { + "noveum_base_url": "https://noveum.ai", + "max_pages_to_scrape": 50, + "chunk_size": 1000, + "chunk_overlap": 200, + "max_search_results": 5, + "rag_threshold": 0.7, + "noveum_docs_file": "noveum_docs.json", + "vector_store_path": "noveum_vectorstore" + }, + "input_query": "Initialize system with force_scrape=False", + "output_response": "System initialization: RAG system loaded, vector store ready, agent operational", + "system.initialized": true, + "system.documents_loaded": 38, + "system.vectorstore_ready": true + }, + "events": [], + "links": [] + } + ], + "created_at": "2025-10-22 16:56:34.565000000", + "updated_at": "2025-10-22 16:56:34.565000000" + }, + { + "trace_id": "bae13042-cf2e-4813-a673-7dc2a611725e", + "name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "status": "ok", + "status_message": "", + "start_time": "2025-10-16 08:50:56.153000000", + "end_time": "2025-10-16 08:51:01.908000000", + "duration_ms": 5754.578, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "spans": [ + { + "span_id": "8c144ad8-a439-4986-b741-3bc4b7fc6110", + "trace_id": "bae13042-cf2e-4813-a673-7dc2a611725e", + "parent_span_id": "d37fe1a1-a126-4e4e-a914-93dd402fed3f", + "name": "agent.query_routing", + "start_time": "2025-10-16 08:50:56.153000000", + "end_time": "2025-10-16 08:51:01.908000000", + "duration_ms": 5754.43, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What are the latest breakthroughs in artificial intelligence?", + "query_length": 61, + "query_type": "routing_query", + "output_response": "Routed to Web Search for query processing", + "router.classification": "Web Search", + "router.confidence_score": 1, + "router.rag_keyword_score": 0, + "router.web_keyword_score": 1, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [] + }, + { + "span_id": "b294f9b7-7762-47a9-b49d-00801a984af2", + "trace_id": "bae13042-cf2e-4813-a673-7dc2a611725e", + "parent_span_id": "8c144ad8-a439-4986-b741-3bc4b7fc6110", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-16 08:50:56.153000000", + "end_time": "2025-10-16 08:50:56.153000000", + "duration_ms": 0.024, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What are the latest breakthroughs in artificial intelligence?", + "query_length": 61, + "query_type": "routing_evaluation_query", + "classification.mode": "Web Search", + "classification.rag_keyword_score": 0, + "classification.web_keyword_score": 1, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "complex", + "query.intent": "general_knowledge", + "query.keyword_density": 0.125, + "query.domain_affinity": "general", + "routing.decision": "Routed to Web Search based on analysis", + "routing.rationale": "RAG score: 0, Web score: 1, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "RAG", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.019230769230769232, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: Web Search (Confidence: 1.00)" + }, + "events": [], + "links": [] + }, + { + "span_id": "ab6a281a-4989-40c8-9011-d0ec7dde7ea9", + "trace_id": "bae13042-cf2e-4813-a673-7dc2a611725e", + "parent_span_id": "8c144ad8-a439-4986-b741-3bc4b7fc6110", + "name": "agent.web_search_generation", + "start_time": "2025-10-16 08:50:56.153000000", + "end_time": "2025-10-16 08:51:01.908000000", + "duration_ms": 5754.317999999999, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "web_search_agent", + "agent.operation": "web_search_generation", + "agent.capabilities": [ + "web_search", + "content_synthesis", + "response_generation" + ], + "agent.id": "noveum_web_search_agent", + "input_query": "What are the latest breakthroughs in artificial intelligence?", + "query_length": 61, + "query_type": "web_search_query", + "output_response": "Web Search Answer: The web search did not yield specific results regarding the latest breakthroughs in artificial intelligence. However, I can provide a general overview based on knowledge up to October 2023.\n\nRecent br...", + "web_search.results_count": 1, + "web_search.context_length": 163, + "web_search.response_length": 1489, + "web_search.mode": "external_web_search" + }, + "events": [], + "links": [] + }, + { + "span_id": "d37fe1a1-a126-4e4e-a914-93dd402fed3f", + "trace_id": "bae13042-cf2e-4813-a673-7dc2a611725e", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-16 08:50:56.153000000", + "end_time": "2025-10-16 08:51:01.908000000", + "duration_ms": 5754.527, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What are the latest breakthroughs in artificial intelligence?", + "query.length": 61, + "processing.mode": "Web Search", + "processing.time_seconds": 5.754457473754883, + "processing.response_length": 1489, + "processing.sources_count": 1, + "output_response": "Final Answer: The web search did not yield specific results regarding the latest breakthroughs in artificial intelligence. However, I can provide a general overview based on knowledge up to October 2023.\n\nRecent br...", + "final_answer_mode": "Web Search", + "query_processed.input_query": "What are the latest breakthroughs in artificial intelligence?", + "query_processed.output_response": "Successfully processed query using Web Search, generated 1489 character response", + "query_processed.mode": "Web Search", + "query_processed.processing_time": 5.754457473754883, + "query_processed.response_length": 1489 + }, + "events": [], + "links": [] + }, + { + "span_id": "f61ef76d-4887-42da-9fd6-e2cdb25a9e09", + "trace_id": "bae13042-cf2e-4813-a673-7dc2a611725e", + "parent_span_id": "ab6a281a-4989-40c8-9011-d0ec7dde7ea9", + "name": "agent.llm_model_execution", + "start_time": "2025-10-16 08:50:56.502000000", + "end_time": "2025-10-16 08:51:01.908000000", + "duration_ms": 5405.44, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What are the latest breakthroughs in artificial intelligence?", + "query_length": 61, + "query_type": "web_search_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 5.405392408370972, + "model.latency_ms": 5405.392408370972, + "model.start_time": 1760604656.5026655, + "model.end_time": 1760604661.908058, + "model.performance_tier": "slow", + "model.prompt_tokens": 167, + "model.completion_tokens": 372, + "model.total_tokens": 539, + "model.tokens_per_second": 99.71523976044487, + "model.estimated_cost": 0.01617, + "model.efficiency_score": 2.7625231910946195, + "model.response_length": 1489, + "model.response_quality": "high", + "model.output_response": "Model Response: The web search did not yield specific results regarding the latest breakthroughs in artificial intelligence. However, I can provide a general overview based on knowledge up to October 2023.\n\nRecent br...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [] + }, + { + "span_id": "550bf5ae-c816-4ef0-a6c1-1553b1c258b8", + "trace_id": "bae13042-cf2e-4813-a673-7dc2a611725e", + "parent_span_id": "ab6a281a-4989-40c8-9011-d0ec7dde7ea9", + "name": "agent.web_search_evaluation_metrics", + "start_time": "2025-10-16 08:51:01.908000000", + "end_time": "2025-10-16 08:51:01.908000000", + "duration_ms": 0.018000000000000002, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "web_search_evaluation_metrics", + "agent.capabilities": [ + "web_search_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What are the latest breakthroughs in artificial intelligence?", + "query_length": 61, + "query_type": "web_search_evaluation_query", + "web_search.results_count": 1, + "web_search.context_length": 163, + "web_search.context_synthesized": "Context from 1 web sources", + "web_search.search_effectiveness": 0.2, + "web_search.context_quality": "low", + "web_search.source_diversity": 1, + "web_search.information_synthesis": "low", + "web_search.external_knowledge_utilization": 0.163, + "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What+are+the+latest+breakthroughs+in+artificial+intelligence?\n\nUser Question: What are the latest breakthroughs in artificial intelligence?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", + "prompt.prompt_length": 669, + "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What+are+the+latest+breakthroughs+in+artificial+intelligence?", + "prompt.instruction_following": "web_search_optimized", + "response.answer_length": 1489, + "response.answer_completeness": "complete", + "response.response_quality": "medium", + "response.source_citation": 1, + "response.context_utilization": 9.134969325153374, + "output_response": "Web Search Answer: The web search did not yield specific results regarding the latest breakthroughs in artificial intelligence. However, I can provide a general overview based on knowledge up to October 2023.\n\nRecent br...", + "evaluation.search_effectiveness": 0.2, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 1, + "evaluation.context_relevance": "low", + "evaluation.overall_quality": "low", + "evaluation.ready_for_production": true, + "web_search.search_strategy": "duckduckgo_api", + "web_search.real_time_data": true, + "web_search.external_sources": 1, + "web_search.information_freshness": "current", + "web_search.knowledge_synthesis": "single_source" + }, + "events": [], + "links": [] + } + ], + "created_at": "2025-10-16 08:50:56.153000000", + "updated_at": "2025-10-16 08:50:56.153000000" + }, + { + "trace_id": "79f9d235-d46e-4740-be35-ef0006071f6c", + "name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "status": "ok", + "status_message": "", + "start_time": "2025-10-16 08:50:51.539000000", + "end_time": "2025-10-16 08:50:55.150000000", + "duration_ms": 3611.084, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "spans": [ + { + "span_id": "04cb7d29-9ea8-4ae0-9834-030cae944eca", + "trace_id": "79f9d235-d46e-4740-be35-ef0006071f6c", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-16 08:50:51.539000000", + "end_time": "2025-10-16 08:50:55.150000000", + "duration_ms": 3611.008, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What's happening in the software development world today?", + "query.length": 57, + "processing.mode": "Web Search", + "processing.time_seconds": 3.6109254360198975, + "processing.response_length": 526, + "processing.sources_count": 1, + "output_response": "Final Answer: The provided web search results do not contain specific information about current events or trends in the software development world. They primarily focus on unrelated topics, such as the meaning of \"...", + "final_answer_mode": "Web Search", + "query_processed.input_query": "What's happening in the software development world today?", + "query_processed.output_response": "Successfully processed query using Web Search, generated 526 character response", + "query_processed.mode": "Web Search", + "query_processed.processing_time": 3.6109254360198975, + "query_processed.response_length": 526 + }, + "events": [], + "links": [] + }, + { + "span_id": "e5e3c927-783e-48c5-aec7-b32bae98c0a2", + "trace_id": "79f9d235-d46e-4740-be35-ef0006071f6c", + "parent_span_id": "f630c3e2-091e-4bd9-9556-e0ebc1c50291", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-16 08:50:51.539000000", + "end_time": "2025-10-16 08:50:51.540000000", + "duration_ms": 0.041, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What's happening in the software development world today?", + "query_length": 57, + "query_type": "routing_evaluation_query", + "classification.mode": "Web Search", + "classification.rag_keyword_score": 0, + "classification.web_keyword_score": 1, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "complex", + "query.intent": "general_knowledge", + "query.keyword_density": 0.125, + "query.domain_affinity": "general", + "routing.decision": "Routed to Web Search based on analysis", + "routing.rationale": "RAG score: 0, Web score: 1, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "RAG", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.019230769230769232, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: Web Search (Confidence: 1.00)" + }, + "events": [], + "links": [] + }, + { + "span_id": "f630c3e2-091e-4bd9-9556-e0ebc1c50291", + "trace_id": "79f9d235-d46e-4740-be35-ef0006071f6c", + "parent_span_id": "04cb7d29-9ea8-4ae0-9834-030cae944eca", + "name": "agent.query_routing", + "start_time": "2025-10-16 08:50:51.539000000", + "end_time": "2025-10-16 08:50:55.150000000", + "duration_ms": 3610.876, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What's happening in the software development world today?", + "query_length": 57, + "query_type": "routing_query", + "output_response": "Routed to Web Search for query processing", + "router.classification": "Web Search", + "router.confidence_score": 1, + "router.rag_keyword_score": 0, + "router.web_keyword_score": 1, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [] + }, + { + "span_id": "d7b0f399-3b43-43bf-bac1-b2101268841c", + "trace_id": "79f9d235-d46e-4740-be35-ef0006071f6c", + "parent_span_id": "f630c3e2-091e-4bd9-9556-e0ebc1c50291", + "name": "agent.web_search_generation", + "start_time": "2025-10-16 08:50:51.540000000", + "end_time": "2025-10-16 08:50:55.150000000", + "duration_ms": 3610.679, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "web_search_agent", + "agent.operation": "web_search_generation", + "agent.capabilities": [ + "web_search", + "content_synthesis", + "response_generation" + ], + "agent.id": "noveum_web_search_agent", + "input_query": "What's happening in the software development world today?", + "query_length": 57, + "query_type": "web_search_query", + "output_response": "Web Search Answer: The provided web search results do not contain specific information about current events or trends in the software development world. They primarily focus on unrelated topics, such as the meaning of \"...", + "web_search.results_count": 1, + "web_search.context_length": 589, + "web_search.response_length": 526, + "web_search.mode": "external_web_search" + }, + "events": [], + "links": [] + }, + { + "span_id": "4ed5dae4-f850-4644-aa55-2f1c00d11c22", + "trace_id": "79f9d235-d46e-4740-be35-ef0006071f6c", + "parent_span_id": "d7b0f399-3b43-43bf-bac1-b2101268841c", + "name": "agent.llm_model_execution", + "start_time": "2025-10-16 08:50:51.976000000", + "end_time": "2025-10-16 08:50:55.150000000", + "duration_ms": 3174.51, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What's happening in the software development world today?", + "query_length": 57, + "query_type": "web_search_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 3.174463987350464, + "model.latency_ms": 3174.463987350464, + "model.start_time": 1760604651.9761698, + "model.end_time": 1760604655.1506338, + "model.performance_tier": "medium", + "model.prompt_tokens": 272, + "model.completion_tokens": 131, + "model.total_tokens": 403, + "model.tokens_per_second": 126.95056601866197, + "model.estimated_cost": 0.01209, + "model.efficiency_score": 1.305210918114144, + "model.response_length": 526, + "model.response_quality": "high", + "model.output_response": "Model Response: The provided web search results do not contain specific information about current events or trends in the software development world. They primarily focus on unrelated topics, such as the meaning of \"...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [] + }, + { + "span_id": "eb151834-9322-4f63-add0-dec9a2966367", + "trace_id": "79f9d235-d46e-4740-be35-ef0006071f6c", + "parent_span_id": "d7b0f399-3b43-43bf-bac1-b2101268841c", + "name": "agent.web_search_evaluation_metrics", + "start_time": "2025-10-16 08:50:55.150000000", + "end_time": "2025-10-16 08:50:55.150000000", + "duration_ms": 0.02, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "web_search_evaluation_metrics", + "agent.capabilities": [ + "web_search_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What's happening in the software development world today?", + "query_length": 57, + "query_type": "web_search_evaluation_query", + "web_search.results_count": 1, + "web_search.context_length": 589, + "web_search.context_synthesized": "Context from 1 web sources", + "web_search.search_effectiveness": 0.2, + "web_search.context_quality": "medium", + "web_search.source_diversity": 1, + "web_search.information_synthesis": "low", + "web_search.external_knowledge_utilization": 0.589, + "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\n\u66fc\u5df4\u7684\u610f\u601d\u662f\u4e00\u79cd\u6bd2\u86c7\uff0c\u8c61\u5f81\u7740\u79d1\u6bd4\u5728\u7403\u573a\u4e0a\u7684\u653b\u51fb\u6027\u548c\u575a\u97e7\u4e0d\u62d4 \u201cWhat can i say\uff0c Mamba out\uff01\u201d \u4f46\u662f\u7ecf\u8fc7\u4e92\u8054\u7f51\u7684\u89e3\u6784\uff0c\u8fd9\u53e5\u8bdd\u6210\u4e3a\u4e86\u4e00\u53e5\u4e07\u7528\u573a\u666f\u7684\u8bdd \u2026 Aprende ingl\u00e9s gratis con nuestros cursos. \u00a1Mas de 100 lecciones con ejemplos y audio! Distintas y divertidas formas de estudiar ingl\u00e9s: \u2026 \u77e5\u4e4e\uff0c\u4e2d\u6587\u4e92\u8054\u7f51\u9ad8\u8d28\u91cf\u7684\u95ee\u7b54\u793e\u533a\u548c\u521b\u4f5c\u8005\u805a\u96c6\u7684\u539f\u521b\u5185\u5bb9\u5e73\u53f0\uff0c\u4e8e 2011 \u5e74 1 \u6708\u6b63\u5f0f\u4e0a\u7ebf\uff0c\u4ee5\u300c\u8ba9\u4eba\u4eec\u66f4\u597d\u7684\u5206\u4eab\u77e5\u8bc6\u3001\u7ecf\u9a8c\u548c\u89c1\u89e3\uff0c\u627e\u5230\u81ea\u5df1\u7684\u89e3\u7b54\u300d\u4e3a\u54c1 \u2026 \u8f6f\u4ef6\u7279\u70b9 1\u3001Windows Desktop Runtime\u662f\u5fae\u8f6fWindows\u684c\u9762\u7a0b\u5e8f\u8fd0\u884c\u5e93\uff08\u542b\u5e38\u89c4\u8fd0\u884c\u5e93\uff09 2\u3001\u80fd\u5b8c\u7f8e\u517c\u5bb9\u5fae\u8f6f\u4e0d\u540c\u7248\u672c\u7684Windows\u7cfb\u7edf\uff0c\u89e3\u51b3\u5176\u7a0b\u5e8f\u7f3a \u2026 \u77e5\u4e4e\uff0c\u4e2d\u6587\u4e92\u8054\u7f51\u9ad8\u8d28\u91cf\u7684\u95ee\u7b54\u793e\u533a\u548c\u521b\u4f5c\u8005\u805a\u96c6\u7684\u539f\u521b\u5185\u5bb9\u5e73\u53f0\uff0c\u4e8e 2011 \u5e74 1 \u6708\u6b63\u5f0f\u4e0a\u7ebf\uff0c\u4ee5\u300c\u8ba9\u4eba\u4eec\u66f4\u597d\u7684\u5206\u4eab\u77e5\u8bc6\u3001\u7ecf\u9a8c\u548c\u89c1\u89e3\uff0c\u627e\u5230\u81ea\u5df1\u7684\u89e3\u7b54\u300d\u4e3a\u54c1 \u2026\nURL: https://duckduckgo.com/?q=What's+happening+in+the+software+development+world+today?\n\nUser Question: What's happening in the software development world today?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", + "prompt.prompt_length": 1091, + "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\n\u66fc\u5df4\u7684\u610f\u601d\u662f\u4e00\u79cd\u6bd2\u86c7\uff0c\u8c61\u5f81\u7740\u79d1\u6bd4\u5728\u7403\u573a\u4e0a\u7684\u653b\u51fb\u6027\u548c\u575a\u97e7\u4e0d\u62d4 \u201cWhat can i say\uff0c Mamba out\uff01\u201d \u4f46\u662f\u7ecf\u8fc7\u4e92\u8054\u7f51\u7684\u89e3\u6784\uff0c\u8fd9\u53e5\u8bdd\u6210\u4e3a\u4e86\u4e00\u53e5\u4e07\u7528\u573a\u666f\u7684\u8bdd \u2026 Aprende ingl\u00e9s gratis con nuestros cursos. \u00a1Mas de 100 lecciones con ejemplos y audio...", + "prompt.instruction_following": "web_search_optimized", + "response.answer_length": 526, + "response.answer_completeness": "complete", + "response.response_quality": "medium", + "response.source_citation": 1, + "response.context_utilization": 0.8930390492359932, + "output_response": "Web Search Answer: The provided web search results do not contain specific information about current events or trends in the software development world. They primarily focus on unrelated topics, such as the meaning of \"...", + "evaluation.search_effectiveness": 0.2, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 1, + "evaluation.context_relevance": "medium", + "evaluation.overall_quality": "low", + "evaluation.ready_for_production": true, + "web_search.search_strategy": "duckduckgo_api", + "web_search.real_time_data": true, + "web_search.external_sources": 1, + "web_search.information_freshness": "current", + "web_search.knowledge_synthesis": "single_source" + }, + "events": [], + "links": [] + } + ], + "created_at": "2025-10-16 08:50:51.539000000", + "updated_at": "2025-10-16 08:50:51.539000000" + }, + { + "trace_id": "77b81bd5-5a04-4fc0-9fc6-571f7907d6b1", + "name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "status": "ok", + "status_message": "", + "start_time": "2025-10-16 08:50:44.888000000", + "end_time": "2025-10-16 08:50:50.529000000", + "duration_ms": 5641.226, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "spans": [ + { + "span_id": "888b4441-3678-42a5-a752-8daae2433482", + "trace_id": "77b81bd5-5a04-4fc0-9fc6-571f7907d6b1", + "parent_span_id": "ba6ffeeb-78b5-44dd-b460-7933ca6e7739", + "name": "agent.query_routing", + "start_time": "2025-10-16 08:50:44.888000000", + "end_time": "2025-10-16 08:50:50.529000000", + "duration_ms": 5641.062, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What are the newest features in cloud computing?", + "query_length": 48, + "query_type": "routing_query", + "output_response": "Routed to RAG for query processing", + "router.classification": "RAG", + "router.confidence_score": 1, + "router.rag_keyword_score": 1, + "router.web_keyword_score": 0, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [] + }, + { + "span_id": "b6bae768-93a6-47d9-8369-61897f9d7828", + "trace_id": "77b81bd5-5a04-4fc0-9fc6-571f7907d6b1", + "parent_span_id": "888b4441-3678-42a5-a752-8daae2433482", + "name": "agent.llm-rag", + "start_time": "2025-10-16 08:50:44.888000000", + "end_time": "2025-10-16 08:50:50.529000000", + "duration_ms": 5640.875999999999, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "rag_agent", + "agent.operation": "llm-rag", + "agent.capabilities": [ + "document_retrieval", + "context_generation", + "response_generation" + ], + "agent.id": "noveum_rag_agent", + "input_query": "What are the newest features in cloud computing?", + "query_length": 48, + "query_type": "rag_query", + "output_response": "RAG Answer: The provided context does not specifically mention any new features in cloud computing as a general topic. However, it does highlight several recent enhancements and features related to the Noveum.ai ...", + "rag.context_length": 2579, + "rag.sources_count": 5, + "rag.answer_length": 983, + "rag.mode": "retrieval_augmented_generation" + }, + "events": [], + "links": [] + }, + { + "span_id": "ba6ffeeb-78b5-44dd-b460-7933ca6e7739", + "trace_id": "77b81bd5-5a04-4fc0-9fc6-571f7907d6b1", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-16 08:50:44.888000000", + "end_time": "2025-10-16 08:50:50.529000000", + "duration_ms": 5641.156, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What are the newest features in cloud computing?", + "query.length": 48, + "processing.mode": "RAG", + "processing.time_seconds": 5.64110541343689, + "processing.response_length": 983, + "processing.sources_count": 5, + "output_response": "Final Answer: The provided context does not specifically mention any new features in cloud computing as a general topic. However, it does highlight several recent enhancements and features related to the Noveum.ai ...", + "final_answer_mode": "RAG", + "query_processed.input_query": "What are the newest features in cloud computing?", + "query_processed.output_response": "Successfully processed query using RAG, generated 983 character response", + "query_processed.mode": "RAG", + "query_processed.processing_time": 5.64110541343689, + "query_processed.response_length": 983 + }, + "events": [], + "links": [] + }, + { + "span_id": "f5538bff-8908-4a56-a622-4ba2eeedfd29", + "trace_id": "77b81bd5-5a04-4fc0-9fc6-571f7907d6b1", + "parent_span_id": "888b4441-3678-42a5-a752-8daae2433482", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-16 08:50:44.888000000", + "end_time": "2025-10-16 08:50:44.888000000", + "duration_ms": 0.032, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What are the newest features in cloud computing?", + "query_length": 48, + "query_type": "routing_evaluation_query", + "classification.mode": "RAG", + "classification.rag_keyword_score": 1, + "classification.web_keyword_score": 0, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "documentation", + "query.keyword_density": 0.125, + "query.domain_affinity": "noveum", + "routing.decision": "Routed to RAG based on analysis", + "routing.rationale": "RAG score: 1, Web score: 0, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "Web Search", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.019230769230769232, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: RAG (Confidence: 1.00)" + }, + "events": [], + "links": [] + }, + { + "span_id": "856e51cf-fa4d-459b-a046-1ea7316b903e", + "trace_id": "77b81bd5-5a04-4fc0-9fc6-571f7907d6b1", + "parent_span_id": "b6bae768-93a6-47d9-8369-61897f9d7828", + "name": "agent.llm_model_execution", + "start_time": "2025-10-16 08:50:45.423000000", + "end_time": "2025-10-16 08:50:49.997000000", + "duration_ms": 4573.782999999999, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What are the newest features in cloud computing?", + "query_length": 48, + "query_type": "rag_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 4.573714733123779, + "model.latency_ms": 4573.714733123779, + "model.start_time": 1760604645.423261, + "model.end_time": 1760604649.9969757, + "model.performance_tier": "medium", + "model.prompt_tokens": 780, + "model.completion_tokens": 245, + "model.total_tokens": 1025, + "model.tokens_per_second": 224.1066747291299, + "model.estimated_cost": 0.03075, + "model.efficiency_score": 0.9590243902439024, + "model.response_length": 983, + "model.response_quality": "high", + "model.output_response": "Model Response: The provided context does not specifically mention any new features in cloud computing as a general topic. However, it does highlight several recent enhancements and features related to the Noveum.ai ...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [] + }, + { + "span_id": "a2fa7400-74c4-4718-a562-899b91503c95", + "trace_id": "77b81bd5-5a04-4fc0-9fc6-571f7907d6b1", + "parent_span_id": "b6bae768-93a6-47d9-8369-61897f9d7828", + "name": "agent.rag_evaluation_metrics", + "start_time": "2025-10-16 08:50:49.997000000", + "end_time": "2025-10-16 08:50:49.997000000", + "duration_ms": 0.048, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "rag_evaluation_metrics", + "agent.capabilities": [ + "retrieval_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What are the newest features in cloud computing?", + "query_length": 48, + "query_type": "rag_evaluation_query", + "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/changelog):\n- \ud83d\udcc8 Enhanced Dashboard Analytics: Real-time request tracking, latency monitoring, and cost analysis\n- \ud83d\udd0d Improved Logs Interface: Better search functionality, detailed trace views, and enhanced debugging capabilities\n- \ud83d\udc65 Team Management: Advanced member mana...", + "retrieval.context_length": 2579, + "retrieval.sources_count": 5, + "retrieval.context_quality": "high", + "retrieval.effectiveness": 1, + "retrieval.context_utilization": 2.579, + "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/changelog):\n- \ud83d\udcc8 Enhanced Dashboard Analytics: Real-time request tracking, latency monitoring, and cost analysis\n- \ud83d\udd0d Improved Logs Interface: Better search functionality, detailed trace views, and enhanced debugging capabilities\n- \ud83d\udc65 Team Management: Advanced member management with role-based access control and invitation system\n- \ud83d\udee1\ufe0f Security Improvements: Enhanced API key management and secure credential storage\n4 months ago\n- \ud83d\udcca Advanced Metrics Collection: Comprehensive telemetry system with custom metrics ...\n\nSource 2 (https://noveum.ai/en/changelog):\n- \ud83d\udcdd Documentation Updates: Comprehensive guides for integration and best practices\n5 months ago\n- \ud83c\udf89 Platform Foundation: Initial release of Noveum.ai AI observability platform\n- \ud83d\ude80 Core Features: AI Gateway integration, basic evaluation metrics, and monitoring capabilities\n- \ud83d\udcc8 Evaluation Jobs: Automated model evaluation with configurable metrics and reporting\n- \ud83d\udd0c Provider Integrations: Support for major AI providers and custom model deployments...\n\nSource 3 (https://noveum.ai/docs/platform/dashboard):\n- Bulk Export: Download multiple traces simultaneously\n- Comparative Analysis: Compare performance across multiple traces\nReal-Time Updates\n- Live Refresh: Automatic updates as new traces arrive\n- Connection Monitoring: Real-time status of your trace ingestion pipeline\n- Performance Indicators: Live metrics for system health monitoring\nReady to dive deeper? Explore Projects & Environments to organize your AI applications, or check out Team Collaboration to share insights with your team.\nGet Earl...\n\nSource 4 (https://noveum.ai/en/changelog):\nChangelog\nStay up to date with the latest changes in our product.\n2 months ago\n- \ud83c\udfaf Major Platform Evolution: NovaEval Framework - Launched comprehensive AI model evaluation framework with 20+ built-in scorers including accuracy, RAG metrics, conversational scoring, and specialized agent evaluation tools. Features production-grade deployment on Docker and Kubernetes. (GitHub | PyPI Package)\n- \ud83d\ude80 Next-Generation Traces UI - Complete redesign featuring three-pane layout with directory tree navigatio...\n\nSource 5 (https://noveum.ai/en/blog):\n8/12/2025\nDiscover how Noveum.ai provides comprehensive tracing and observability for AI applications, from development debugging to production optimization.\nShashank Agarwal\n3/3/2025\nDiscover how Noveum.ai provides comprehensive tracing and observability for LLM applications, RAG systems, and multi-agent workflows with our powerful Python and TypeScript SDKs.\nShashank Agarwal\n3/2/2025...\n\nUser Question: What are the newest features in cloud computing?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", + "prompt.prompt_length": 3123, + "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/changelog):\n- \ud83d\udcc8 Enhanced Dashboard Analytics: Real-time request tracking, latency monitoring, and cost analysis\n- \ud83d\udd0d Improved Logs Interface: Better search functionality,...", + "prompt.instruction_following": "rag_optimized", + "response.answer_length": 983, + "response.answer_completeness": "complete", + "response.response_quality": "high", + "response.source_citation": 5, + "response.context_utilization": 0.381155486622722, + "output_response": "RAG Answer: The provided context does not specifically mention any new features in cloud computing as a general topic. However, it does highlight several recent enhancements and features related to the Noveum.ai ...", + "evaluation.retrieval_effectiveness": 1, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 5, + "evaluation.context_relevance": "high", + "evaluation.overall_quality": "high", + "evaluation.ready_for_production": true, + "rag.retrieval_strategy": "semantic_similarity", + "rag.vector_search_results": 5, + "rag.context_synthesis": "multi_source", + "rag.document_coverage": 1, + "rag.information_density": 0.381155486622722 + }, + "events": [], + "links": [] + } + ], + "created_at": "2025-10-16 08:50:44.888000000", + "updated_at": "2025-10-16 08:50:44.888000000" + }, + { + "trace_id": "32170d1a-d211-46d8-9cdc-5a4265f0c443", + "name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "status": "ok", + "status_message": "", + "start_time": "2025-10-16 08:50:42.124000000", + "end_time": "2025-10-16 08:50:43.885000000", + "duration_ms": 1761.047, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "spans": [ + { + "span_id": "4c391bb1-1a62-4e2a-913b-96f83feff494", + "trace_id": "32170d1a-d211-46d8-9cdc-5a4265f0c443", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-16 08:50:42.124000000", + "end_time": "2025-10-16 08:50:43.885000000", + "duration_ms": 1760.974, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What's the current status of cryptocurrency markets?", + "query.length": 52, + "processing.mode": "Web Search", + "processing.time_seconds": 1.7608659267425537, + "processing.response_length": 363, + "processing.sources_count": 1, + "output_response": "Final Answer: The web search did not yield any specific information regarding the current status of cryptocurrency markets. Therefore, I cannot provide an update on market trends, prices, or significant events affe...", + "final_answer_mode": "Web Search", + "query_processed.input_query": "What's the current status of cryptocurrency markets?", + "query_processed.output_response": "Successfully processed query using Web Search, generated 363 character response", + "query_processed.mode": "Web Search", + "query_processed.processing_time": 1.7608659267425537, + "query_processed.response_length": 363 + }, + "events": [], + "links": [] + }, + { + "span_id": "7ab2ec7b-a617-4ebf-9bb1-82dc6b18e160", + "trace_id": "32170d1a-d211-46d8-9cdc-5a4265f0c443", + "parent_span_id": "4c391bb1-1a62-4e2a-913b-96f83feff494", + "name": "agent.query_routing", + "start_time": "2025-10-16 08:50:42.124000000", + "end_time": "2025-10-16 08:50:43.885000000", + "duration_ms": 1760.833, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What's the current status of cryptocurrency markets?", + "query_length": 52, + "query_type": "routing_query", + "output_response": "Routed to Web Search for query processing", + "router.classification": "Web Search", + "router.confidence_score": 1, + "router.rag_keyword_score": 0, + "router.web_keyword_score": 3, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [] + }, + { + "span_id": "db01d228-7c63-4814-b2ea-bdf3898f9c0d", + "trace_id": "32170d1a-d211-46d8-9cdc-5a4265f0c443", + "parent_span_id": "7ab2ec7b-a617-4ebf-9bb1-82dc6b18e160", + "name": "agent.web_search_generation", + "start_time": "2025-10-16 08:50:42.124000000", + "end_time": "2025-10-16 08:50:43.885000000", + "duration_ms": 1760.701, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "web_search_agent", + "agent.operation": "web_search_generation", + "agent.capabilities": [ + "web_search", + "content_synthesis", + "response_generation" + ], + "agent.id": "noveum_web_search_agent", + "input_query": "What's the current status of cryptocurrency markets?", + "query_length": 52, + "query_type": "web_search_query", + "output_response": "Web Search Answer: The web search did not yield any specific information regarding the current status of cryptocurrency markets. Therefore, I cannot provide an update on market trends, prices, or significant events affe...", + "web_search.results_count": 1, + "web_search.context_length": 154, + "web_search.response_length": 363, + "web_search.mode": "external_web_search" + }, + "events": [], + "links": [] + }, + { + "span_id": "da7b0550-b13c-4518-ad3b-050c666d198c", + "trace_id": "32170d1a-d211-46d8-9cdc-5a4265f0c443", + "parent_span_id": "7ab2ec7b-a617-4ebf-9bb1-82dc6b18e160", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-16 08:50:42.124000000", + "end_time": "2025-10-16 08:50:42.124000000", + "duration_ms": 0.029, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What's the current status of cryptocurrency markets?", + "query_length": 52, + "query_type": "routing_evaluation_query", + "classification.mode": "Web Search", + "classification.rag_keyword_score": 0, + "classification.web_keyword_score": 3, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "complex", + "query.intent": "general_knowledge", + "query.keyword_density": 0.42857142857142855, + "query.domain_affinity": "general", + "routing.decision": "Routed to Web Search based on analysis", + "routing.rationale": "RAG score: 0, Web score: 3, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "RAG", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.057692307692307696, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: Web Search (Confidence: 1.00)" + }, + "events": [], + "links": [] + }, + { + "span_id": "950e18b6-9f62-4ac4-8788-3cac08fb7899", + "trace_id": "32170d1a-d211-46d8-9cdc-5a4265f0c443", + "parent_span_id": "db01d228-7c63-4814-b2ea-bdf3898f9c0d", + "name": "agent.llm_model_execution", + "start_time": "2025-10-16 08:50:42.465000000", + "end_time": "2025-10-16 08:50:43.885000000", + "duration_ms": 1419.5690000000002, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What's the current status of cryptocurrency markets?", + "query_length": 52, + "query_type": "web_search_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 1.4195220470428467, + "model.latency_ms": 1419.5220470428467, + "model.start_time": 1760604642.4658613, + "model.end_time": 1760604643.8853834, + "model.performance_tier": "fast", + "model.prompt_tokens": 162, + "model.completion_tokens": 90, + "model.total_tokens": 252, + "model.tokens_per_second": 177.52454111224782, + "model.estimated_cost": 0.00756, + "model.efficiency_score": 1.4404761904761905, + "model.response_length": 363, + "model.response_quality": "high", + "model.output_response": "Model Response: The web search did not yield any specific information regarding the current status of cryptocurrency markets. Therefore, I cannot provide an update on market trends, prices, or significant events affe...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [] + }, + { + "span_id": "e87846c9-ce82-42c2-a517-e0f67e9d14c1", + "trace_id": "32170d1a-d211-46d8-9cdc-5a4265f0c443", + "parent_span_id": "db01d228-7c63-4814-b2ea-bdf3898f9c0d", + "name": "agent.web_search_evaluation_metrics", + "start_time": "2025-10-16 08:50:43.885000000", + "end_time": "2025-10-16 08:50:43.885000000", + "duration_ms": 0.02, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "web_search_evaluation_metrics", + "agent.capabilities": [ + "web_search_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What's the current status of cryptocurrency markets?", + "query_length": 52, + "query_type": "web_search_evaluation_query", + "web_search.results_count": 1, + "web_search.context_length": 154, + "web_search.context_synthesized": "Context from 1 web sources", + "web_search.search_effectiveness": 0.2, + "web_search.context_quality": "low", + "web_search.source_diversity": 1, + "web_search.information_synthesis": "low", + "web_search.external_knowledge_utilization": 0.154, + "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What's+the+current+status+of+cryptocurrency+markets?\n\nUser Question: What's the current status of cryptocurrency markets?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", + "prompt.prompt_length": 651, + "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What's+the+current+status+of+cryptocurrency+markets?", + "prompt.instruction_following": "web_search_optimized", + "response.answer_length": 363, + "response.answer_completeness": "complete", + "response.response_quality": "medium", + "response.source_citation": 1, + "response.context_utilization": 2.357142857142857, + "output_response": "Web Search Answer: The web search did not yield any specific information regarding the current status of cryptocurrency markets. Therefore, I cannot provide an update on market trends, prices, or significant events affe...", + "evaluation.search_effectiveness": 0.2, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 1, + "evaluation.context_relevance": "low", + "evaluation.overall_quality": "low", + "evaluation.ready_for_production": true, + "web_search.search_strategy": "duckduckgo_api", + "web_search.real_time_data": true, + "web_search.external_sources": 1, + "web_search.information_freshness": "current", + "web_search.knowledge_synthesis": "single_source" + }, + "events": [], + "links": [] + } + ], + "created_at": "2025-10-16 08:50:42.124000000", + "updated_at": "2025-10-16 08:50:42.124000000" + }, + { + "trace_id": "9424d420-deee-478b-b727-9dc791a7457c", + "name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "status": "ok", + "status_message": "", + "start_time": "2025-10-16 08:50:31.476000000", + "end_time": "2025-10-16 08:50:41.121000000", + "duration_ms": 9644.828, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "spans": [ + { + "span_id": "305daacf-990b-4e23-8720-48900f75d9be", + "trace_id": "9424d420-deee-478b-b727-9dc791a7457c", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-16 08:50:31.476000000", + "end_time": "2025-10-16 08:50:41.121000000", + "duration_ms": 9644.752999999999, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What are the latest updates in Python programming?", + "query.length": 50, + "processing.mode": "Web Search", + "processing.time_seconds": 9.644662618637085, + "processing.response_length": 1408, + "processing.sources_count": 1, + "output_response": "Final Answer: The web search did not yield specific results regarding the latest updates in Python programming. However, I can provide some general information based on my knowledge up to October 2023.\n\nAs of late ...", + "final_answer_mode": "Web Search", + "query_processed.input_query": "What are the latest updates in Python programming?", + "query_processed.output_response": "Successfully processed query using Web Search, generated 1408 character response", + "query_processed.mode": "Web Search", + "query_processed.processing_time": 9.644662618637085, + "query_processed.response_length": 1408 + }, + "events": [], + "links": [] + }, + { + "span_id": "56b4cc1b-0646-4dee-acb3-db98f3d44477", + "trace_id": "9424d420-deee-478b-b727-9dc791a7457c", + "parent_span_id": "fb234e30-69be-4d06-9248-c05bcc22f063", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-16 08:50:31.476000000", + "end_time": "2025-10-16 08:50:31.476000000", + "duration_ms": 0.036000000000000004, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What are the latest updates in Python programming?", + "query_length": 50, + "query_type": "routing_evaluation_query", + "classification.mode": "Web Search", + "classification.rag_keyword_score": 0, + "classification.web_keyword_score": 2, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "general_knowledge", + "query.keyword_density": 0.25, + "query.domain_affinity": "general", + "routing.decision": "Routed to Web Search based on analysis", + "routing.rationale": "RAG score: 0, Web score: 2, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "RAG", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.038461538461538464, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: Web Search (Confidence: 1.00)" + }, + "events": [], + "links": [] + }, + { + "span_id": "fb234e30-69be-4d06-9248-c05bcc22f063", + "trace_id": "9424d420-deee-478b-b727-9dc791a7457c", + "parent_span_id": "305daacf-990b-4e23-8720-48900f75d9be", + "name": "agent.query_routing", + "start_time": "2025-10-16 08:50:31.476000000", + "end_time": "2025-10-16 08:50:41.121000000", + "duration_ms": 9644.627, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What are the latest updates in Python programming?", + "query_length": 50, + "query_type": "routing_query", + "output_response": "Routed to Web Search for query processing", + "router.classification": "Web Search", + "router.confidence_score": 1, + "router.rag_keyword_score": 0, + "router.web_keyword_score": 2, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [] + }, + { + "span_id": "f8a793a9-63ee-49b7-9931-d7dcc2bb0ca7", + "trace_id": "9424d420-deee-478b-b727-9dc791a7457c", + "parent_span_id": "fb234e30-69be-4d06-9248-c05bcc22f063", + "name": "agent.web_search_generation", + "start_time": "2025-10-16 08:50:31.477000000", + "end_time": "2025-10-16 08:50:41.121000000", + "duration_ms": 9644.478, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "web_search_agent", + "agent.operation": "web_search_generation", + "agent.capabilities": [ + "web_search", + "content_synthesis", + "response_generation" + ], + "agent.id": "noveum_web_search_agent", + "input_query": "What are the latest updates in Python programming?", + "query_length": 50, + "query_type": "web_search_query", + "output_response": "Web Search Answer: The web search did not yield specific results regarding the latest updates in Python programming. However, I can provide some general information based on my knowledge up to October 2023.\n\nAs of late ...", + "web_search.results_count": 1, + "web_search.context_length": 152, + "web_search.response_length": 1408, + "web_search.mode": "external_web_search" + }, + "events": [], + "links": [] + }, + { + "span_id": "b968e3d3-d4aa-4357-85d8-34a612748fbd", + "trace_id": "9424d420-deee-478b-b727-9dc791a7457c", + "parent_span_id": "f8a793a9-63ee-49b7-9931-d7dcc2bb0ca7", + "name": "agent.llm_model_execution", + "start_time": "2025-10-16 08:50:32.016000000", + "end_time": "2025-10-16 08:50:41.121000000", + "duration_ms": 9105.344000000001, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What are the latest updates in Python programming?", + "query_length": 50, + "query_type": "web_search_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 9.105276584625244, + "model.latency_ms": 9105.276584625244, + "model.start_time": 1760604632.0161095, + "model.end_time": 1760604641.121386, + "model.performance_tier": "slow", + "model.prompt_tokens": 161, + "model.completion_tokens": 352, + "model.total_tokens": 513, + "model.tokens_per_second": 56.340957381374714, + "model.estimated_cost": 0.015390000000000001, + "model.efficiency_score": 2.7446393762183234, + "model.response_length": 1408, + "model.response_quality": "high", + "model.output_response": "Model Response: The web search did not yield specific results regarding the latest updates in Python programming. However, I can provide some general information based on my knowledge up to October 2023.\n\nAs of late ...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [] + }, + { + "span_id": "80817711-e64c-4a9f-802e-2a9e36ff093d", + "trace_id": "9424d420-deee-478b-b727-9dc791a7457c", + "parent_span_id": "f8a793a9-63ee-49b7-9931-d7dcc2bb0ca7", + "name": "agent.web_search_evaluation_metrics", + "start_time": "2025-10-16 08:50:41.121000000", + "end_time": "2025-10-16 08:50:41.121000000", + "duration_ms": 0.02, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "web_search_evaluation_metrics", + "agent.capabilities": [ + "web_search_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What are the latest updates in Python programming?", + "query_length": 50, + "query_type": "web_search_evaluation_query", + "web_search.results_count": 1, + "web_search.context_length": 152, + "web_search.context_synthesized": "Context from 1 web sources", + "web_search.search_effectiveness": 0.2, + "web_search.context_quality": "low", + "web_search.source_diversity": 1, + "web_search.information_synthesis": "low", + "web_search.external_knowledge_utilization": 0.152, + "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What+are+the+latest+updates+in+Python+programming?\n\nUser Question: What are the latest updates in Python programming?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", + "prompt.prompt_length": 647, + "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What+are+the+latest+updates+in+Python+programming?", + "prompt.instruction_following": "web_search_optimized", + "response.answer_length": 1408, + "response.answer_completeness": "complete", + "response.response_quality": "medium", + "response.source_citation": 1, + "response.context_utilization": 9.263157894736842, + "output_response": "Web Search Answer: The web search did not yield specific results regarding the latest updates in Python programming. However, I can provide some general information based on my knowledge up to October 2023.\n\nAs of late ...", + "evaluation.search_effectiveness": 0.2, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 1, + "evaluation.context_relevance": "low", + "evaluation.overall_quality": "low", + "evaluation.ready_for_production": true, + "web_search.search_strategy": "duckduckgo_api", + "web_search.real_time_data": true, + "web_search.external_sources": 1, + "web_search.information_freshness": "current", + "web_search.knowledge_synthesis": "single_source" + }, + "events": [], + "links": [] + } + ], + "created_at": "2025-10-16 08:50:31.476000000", + "updated_at": "2025-10-16 08:50:31.476000000" + }, + { + "trace_id": "a3735d64-89bb-4fdf-a851-7a1f3515c52d", + "name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "status": "ok", + "status_message": "", + "start_time": "2025-10-16 08:50:27.687000000", + "end_time": "2025-10-16 08:50:30.473000000", + "duration_ms": 2786.145, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "spans": [ + { + "span_id": "13af0f3c-bd5c-4024-8fa3-c9e347a1ef20", + "trace_id": "a3735d64-89bb-4fdf-a851-7a1f3515c52d", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-16 08:50:27.687000000", + "end_time": "2025-10-16 08:50:30.473000000", + "duration_ms": 2786.078, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What happened in tech news this week?", + "query.length": 37, + "processing.mode": "Web Search", + "processing.time_seconds": 2.7859997749328613, + "processing.response_length": 396, + "processing.sources_count": 1, + "output_response": "Final Answer: It appears that the web search did not yield any specific results regarding recent tech news. Unfortunately, I cannot provide a summary of this week's tech news based on the available information. For...", + "final_answer_mode": "Web Search", + "query_processed.input_query": "What happened in tech news this week?", + "query_processed.output_response": "Successfully processed query using Web Search, generated 396 character response", + "query_processed.mode": "Web Search", + "query_processed.processing_time": 2.7859997749328613, + "query_processed.response_length": 396 + }, + "events": [], + "links": [] + }, + { + "span_id": "47e8f375-35e6-4097-86ff-df73e8181c23", + "trace_id": "a3735d64-89bb-4fdf-a851-7a1f3515c52d", + "parent_span_id": "a4f1a7b3-dc35-4f13-96d3-b6dc8fb21a37", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-16 08:50:27.687000000", + "end_time": "2025-10-16 08:50:27.687000000", + "duration_ms": 0.038, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What happened in tech news this week?", + "query_length": 37, + "query_type": "routing_evaluation_query", + "classification.mode": "Web Search", + "classification.rag_keyword_score": 0, + "classification.web_keyword_score": 2, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "general_knowledge", + "query.keyword_density": 0.2857142857142857, + "query.domain_affinity": "general", + "routing.decision": "Routed to Web Search based on analysis", + "routing.rationale": "RAG score: 0, Web score: 2, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "RAG", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.038461538461538464, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: Web Search (Confidence: 1.00)" + }, + "events": [], + "links": [] + }, + { + "span_id": "a4f1a7b3-dc35-4f13-96d3-b6dc8fb21a37", + "trace_id": "a3735d64-89bb-4fdf-a851-7a1f3515c52d", + "parent_span_id": "13af0f3c-bd5c-4024-8fa3-c9e347a1ef20", + "name": "agent.query_routing", + "start_time": "2025-10-16 08:50:27.687000000", + "end_time": "2025-10-16 08:50:30.473000000", + "duration_ms": 2785.964, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What happened in tech news this week?", + "query_length": 37, + "query_type": "routing_query", + "output_response": "Routed to Web Search for query processing", + "router.classification": "Web Search", + "router.confidence_score": 1, + "router.rag_keyword_score": 0, + "router.web_keyword_score": 2, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [] + }, + { + "span_id": "ebfff280-5eb6-4838-8de7-35c85e2e3eaa", + "trace_id": "a3735d64-89bb-4fdf-a851-7a1f3515c52d", + "parent_span_id": "a4f1a7b3-dc35-4f13-96d3-b6dc8fb21a37", + "name": "agent.web_search_generation", + "start_time": "2025-10-16 08:50:27.687000000", + "end_time": "2025-10-16 08:50:30.473000000", + "duration_ms": 2785.8230000000003, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "web_search_agent", + "agent.operation": "web_search_generation", + "agent.capabilities": [ + "web_search", + "content_synthesis", + "response_generation" + ], + "agent.id": "noveum_web_search_agent", + "input_query": "What happened in tech news this week?", + "query_length": 37, + "query_type": "web_search_query", + "output_response": "Web Search Answer: It appears that the web search did not yield any specific results regarding recent tech news. Unfortunately, I cannot provide a summary of this week's tech news based on the available information. For...", + "web_search.results_count": 1, + "web_search.context_length": 139, + "web_search.response_length": 396, + "web_search.mode": "external_web_search" + }, + "events": [], + "links": [] + }, + { + "span_id": "835b9351-d381-4785-8538-fbe1064fc8cd", + "trace_id": "a3735d64-89bb-4fdf-a851-7a1f3515c52d", + "parent_span_id": "ebfff280-5eb6-4838-8de7-35c85e2e3eaa", + "name": "agent.llm_model_execution", + "start_time": "2025-10-16 08:50:28.026000000", + "end_time": "2025-10-16 08:50:30.473000000", + "duration_ms": 2447.269, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What happened in tech news this week?", + "query_length": 37, + "query_type": "web_search_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 2.447227716445923, + "model.latency_ms": 2447.227716445923, + "model.start_time": 1760604628.0260897, + "model.end_time": 1760604630.4733174, + "model.performance_tier": "medium", + "model.prompt_tokens": 155, + "model.completion_tokens": 99, + "model.total_tokens": 254, + "model.tokens_per_second": 103.79091340501853, + "model.estimated_cost": 0.00762, + "model.efficiency_score": 1.5590551181102361, + "model.response_length": 396, + "model.response_quality": "high", + "model.output_response": "Model Response: It appears that the web search did not yield any specific results regarding recent tech news. Unfortunately, I cannot provide a summary of this week's tech news based on the available information. For...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [] + }, + { + "span_id": "93c6f427-6fcd-4318-aec8-095553fed469", + "trace_id": "a3735d64-89bb-4fdf-a851-7a1f3515c52d", + "parent_span_id": "ebfff280-5eb6-4838-8de7-35c85e2e3eaa", + "name": "agent.web_search_evaluation_metrics", + "start_time": "2025-10-16 08:50:30.473000000", + "end_time": "2025-10-16 08:50:30.473000000", + "duration_ms": 0.017, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "web_search_evaluation_metrics", + "agent.capabilities": [ + "web_search_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What happened in tech news this week?", + "query_length": 37, + "query_type": "web_search_evaluation_query", + "web_search.results_count": 1, + "web_search.context_length": 139, + "web_search.context_synthesized": "Context from 1 web sources", + "web_search.search_effectiveness": 0.2, + "web_search.context_quality": "low", + "web_search.source_diversity": 1, + "web_search.information_synthesis": "low", + "web_search.external_knowledge_utilization": 0.139, + "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What+happened+in+tech+news+this+week?\n\nUser Question: What happened in tech news this week?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", + "prompt.prompt_length": 621, + "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What+happened+in+tech+news+this+week?", + "prompt.instruction_following": "web_search_optimized", + "response.answer_length": 396, + "response.answer_completeness": "complete", + "response.response_quality": "medium", + "response.source_citation": 1, + "response.context_utilization": 2.8489208633093526, + "output_response": "Web Search Answer: It appears that the web search did not yield any specific results regarding recent tech news. Unfortunately, I cannot provide a summary of this week's tech news based on the available information. For...", + "evaluation.search_effectiveness": 0.2, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 1, + "evaluation.context_relevance": "low", + "evaluation.overall_quality": "low", + "evaluation.ready_for_production": true, + "web_search.search_strategy": "duckduckgo_api", + "web_search.real_time_data": true, + "web_search.external_sources": 1, + "web_search.information_freshness": "current", + "web_search.knowledge_synthesis": "single_source" + }, + "events": [], + "links": [] + } + ], + "created_at": "2025-10-16 08:50:27.687000000", + "updated_at": "2025-10-16 08:50:27.687000000" + }, + { + "trace_id": "fa5bb641-f4b7-4741-b06c-392a5f2a9aea", + "name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "status": "ok", + "status_message": "", + "start_time": "2025-10-16 08:50:21.645000000", + "end_time": "2025-10-16 08:50:26.683000000", + "duration_ms": 5038.112, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "spans": [ + { + "span_id": "622e7353-4eeb-47d5-8558-4ecec28cdcfd", + "trace_id": "fa5bb641-f4b7-4741-b06c-392a5f2a9aea", + "parent_span_id": "c0af6629-48b1-45fc-9dd2-60f098ea140e", + "name": "agent.query_routing", + "start_time": "2025-10-16 08:50:21.645000000", + "end_time": "2025-10-16 08:50:26.683000000", + "duration_ms": 5037.965, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What are the current trends in observability tools?", + "query_length": 51, + "query_type": "routing_query", + "output_response": "Routed to RAG for query processing", + "router.classification": "RAG", + "router.confidence_score": 0.3333333333333333, + "router.rag_keyword_score": 2, + "router.web_keyword_score": 1, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [] + }, + { + "span_id": "c0af6629-48b1-45fc-9dd2-60f098ea140e", + "trace_id": "fa5bb641-f4b7-4741-b06c-392a5f2a9aea", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-16 08:50:21.645000000", + "end_time": "2025-10-16 08:50:26.683000000", + "duration_ms": 5038.0419999999995, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What are the current trends in observability tools?", + "query.length": 51, + "processing.mode": "RAG", + "processing.time_seconds": 5.0380027294158936, + "processing.response_length": 897, + "processing.sources_count": 5, + "output_response": "Final Answer: The provided context does not specifically outline current trends in observability tools beyond the focus on AI applications. However, it highlights the limitations of traditional monitoring tools for...", + "final_answer_mode": "RAG", + "query_processed.input_query": "What are the current trends in observability tools?", + "query_processed.output_response": "Successfully processed query using RAG, generated 897 character response", + "query_processed.mode": "RAG", + "query_processed.processing_time": 5.0380027294158936, + "query_processed.response_length": 897 + }, + "events": [], + "links": [] + }, + { + "span_id": "ee55906d-ffe3-42fd-9899-a476d4c33ff5", + "trace_id": "fa5bb641-f4b7-4741-b06c-392a5f2a9aea", + "parent_span_id": "622e7353-4eeb-47d5-8558-4ecec28cdcfd", + "name": "agent.llm-rag", + "start_time": "2025-10-16 08:50:21.645000000", + "end_time": "2025-10-16 08:50:26.683000000", + "duration_ms": 5037.803, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "rag_agent", + "agent.operation": "llm-rag", + "agent.capabilities": [ + "document_retrieval", + "context_generation", + "response_generation" + ], + "agent.id": "noveum_rag_agent", + "input_query": "What are the current trends in observability tools?", + "query_length": 51, + "query_type": "rag_query", + "output_response": "RAG Answer: The provided context does not specifically outline current trends in observability tools beyond the focus on AI applications. However, it highlights the limitations of traditional monitoring tools for...", + "rag.context_length": 2943, + "rag.sources_count": 5, + "rag.answer_length": 897, + "rag.mode": "retrieval_augmented_generation" + }, + "events": [], + "links": [] + }, + { + "span_id": "fe610ecb-cc8e-4466-b719-34c6e0d45d87", + "trace_id": "fa5bb641-f4b7-4741-b06c-392a5f2a9aea", + "parent_span_id": "622e7353-4eeb-47d5-8558-4ecec28cdcfd", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-16 08:50:21.645000000", + "end_time": "2025-10-16 08:50:21.645000000", + "duration_ms": 0.032, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What are the current trends in observability tools?", + "query_length": 51, + "query_type": "routing_evaluation_query", + "classification.mode": "RAG", + "classification.rag_keyword_score": 2, + "classification.web_keyword_score": 1, + "classification.confidence_score": 0.3333333333333333, + "classification.confidence_level": "medium", + "classification.method": "keyword_based", + "query.complexity": "complex", + "query.intent": "documentation", + "query.keyword_density": 0.375, + "query.domain_affinity": "noveum", + "routing.decision": "Routed to RAG based on analysis", + "routing.rationale": "RAG score: 2, Web score: 1, Confidence: 0.33", + "routing.expected_performance": "medium", + "routing.alternative_mode": "Web Search", + "routing.decision_confidence": 0.3333333333333333, + "evaluation.routing_accuracy": "medium", + "evaluation.keyword_coverage": 0.057692307692307696, + "evaluation.query_understanding": "ambiguous", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: RAG (Confidence: 0.33)" + }, + "events": [], + "links": [] + }, + { + "span_id": "69a4773d-c738-489e-87ed-999583e02187", + "trace_id": "fa5bb641-f4b7-4741-b06c-392a5f2a9aea", + "parent_span_id": "ee55906d-ffe3-42fd-9899-a476d4c33ff5", + "name": "agent.llm_model_execution", + "start_time": "2025-10-16 08:50:22.600000000", + "end_time": "2025-10-16 08:50:25.786000000", + "duration_ms": 3185.257, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What are the current trends in observability tools?", + "query_length": 51, + "query_type": "rag_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 3.185206174850464, + "model.latency_ms": 3185.206174850464, + "model.start_time": 1760604622.6008554, + "model.end_time": 1760604625.7860615, + "model.performance_tier": "medium", + "model.prompt_tokens": 872, + "model.completion_tokens": 224, + "model.total_tokens": 1096, + "model.tokens_per_second": 344.09075577390337, + "model.estimated_cost": 0.03288, + "model.efficiency_score": 0.8184306569343066, + "model.response_length": 897, + "model.response_quality": "high", + "model.output_response": "Model Response: The provided context does not specifically outline current trends in observability tools beyond the focus on AI applications. However, it highlights the limitations of traditional monitoring tools for...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [] + }, + { + "span_id": "5e5a9d30-4cd0-45cd-9494-192c280727ac", + "trace_id": "fa5bb641-f4b7-4741-b06c-392a5f2a9aea", + "parent_span_id": "ee55906d-ffe3-42fd-9899-a476d4c33ff5", + "name": "agent.rag_evaluation_metrics", + "start_time": "2025-10-16 08:50:25.786000000", + "end_time": "2025-10-16 08:50:25.786000000", + "duration_ms": 0.066, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "rag_evaluation_metrics", + "agent.capabilities": [ + "retrieval_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What are the current trends in observability tools?", + "query_length": 51, + "query_type": "rag_evaluation_query", + "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/blog/from-logs-to-intelligent-choices-inside-noveum-ais-evaluation-process):\nFrom Development to Production - Inside Noveum.ai's AI Observability Platform\nShashank Agarwal\n3/3/2025\nIntroduction\nEvery AI application tells a story\u2014but most developers never get to hear it...", + "retrieval.context_length": 2943, + "retrieval.sources_count": 5, + "retrieval.context_quality": "high", + "retrieval.effectiveness": 1, + "retrieval.context_utilization": 2.943, + "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/blog/from-logs-to-intelligent-choices-inside-noveum-ais-evaluation-process):\nFrom Development to Production - Inside Noveum.ai's AI Observability Platform\nShashank Agarwal\n3/3/2025\nIntroduction\nEvery AI application tells a story\u2014but most developers never get to hear it. When your RAG pipeline returns irrelevant results, when your multi-agent system gets stuck in loops, or when your LLM costs suddenly spike, you're left guessing what went wrong and where.\nThat's exactly the challenge Noveum.ai solves. Rather than flying blind, our platform provides comprehensive tracing a...\n\nSource 2 (https://noveum.ai/docs/getting-started/tracing-concepts):\n- Explore Framework Integrations - Framework-specific guidance\n- Learn Advanced Patterns - Custom instrumentation techniques\n- Master the Dashboard - Analyze your traces effectively\nRemember: Good observability is not about collecting all possible data, but about collecting the right data that helps you understand, debug, and optimize your AI applications.\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to O...\n\nSource 3 (https://noveum.ai/en/docs/getting-started/tracing-concepts):\n- Explore Framework Integrations - Framework-specific guidance\n- Learn Advanced Patterns - Custom instrumentation techniques\n- Master the Dashboard - Analyze your traces effectively\nRemember: Good observability is not about collecting all possible data, but about collecting the right data that helps you understand, debug, and optimize your AI applications.\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to O...\n\nSource 4 (https://noveum.ai/en/blog/from-logs-to-intelligent-choices-inside-noveum-ais-evaluation-process):\nThe AI Observability Challenge\nWhy Traditional Monitoring Falls Short\nTraditional application monitoring tools weren't built for AI applications. They can tell you if your API is responding, but they can't answer the questions that matter most for AI systems:\n- Why did my RAG pipeline retrieve irrelevant documents?\n- Which LLM calls are driving my costs?\n- How are my agents communicating with each other?\n- What's causing hallucinations in my responses?\n- Why is my embedding generation so slow?\nW...\n\nSource 5 (https://noveum.ai/en/docs/advanced/rag-observability):\n\ud83d\udd27 Best Practices for RAG Observability\n1. Comprehensive Pipeline Tracing\n2. Quality Monitoring\n3. A/B Testing for RAG Components\n\ud83c\udfaf Advanced RAG Patterns\nMulti-Modal RAG\nConversational RAG\nRAG observability with Noveum.ai provides the deep insights needed to build, optimize, and scale retrieval-augmented generation systems. By implementing comprehensive tracing across retrieval, context preparation, and generation stages, you can ensure your RAG pipeline delivers accurate, relevant, and cost-effe...\n\nUser Question: What are the current trends in observability tools?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", + "prompt.prompt_length": 3490, + "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/blog/from-logs-to-intelligent-choices-inside-noveum-ais-evaluation-process):\nFrom Development to Production - Inside Noveum.ai's AI Observability Platform\nShashank Agarw...", + "prompt.instruction_following": "rag_optimized", + "response.answer_length": 897, + "response.answer_completeness": "complete", + "response.response_quality": "high", + "response.source_citation": 5, + "response.context_utilization": 0.30479102956167176, + "output_response": "RAG Answer: The provided context does not specifically outline current trends in observability tools beyond the focus on AI applications. However, it highlights the limitations of traditional monitoring tools for...", + "evaluation.retrieval_effectiveness": 1, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 5, + "evaluation.context_relevance": "high", + "evaluation.overall_quality": "high", + "evaluation.ready_for_production": true, + "rag.retrieval_strategy": "semantic_similarity", + "rag.vector_search_results": 5, + "rag.context_synthesis": "multi_source", + "rag.document_coverage": 1, + "rag.information_density": 0.30479102956167176 + }, + "events": [], + "links": [] + } + ], + "created_at": "2025-10-16 08:50:21.645000000", + "updated_at": "2025-10-16 08:50:21.645000000" + }, + { + "trace_id": "c0a95e8e-513c-47eb-b746-4e6016aeaf7d", + "name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "status": "ok", + "status_message": "", + "start_time": "2025-10-16 08:50:08.944000000", + "end_time": "2025-10-16 08:50:20.641000000", + "duration_ms": 11696.768, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "spans": [ + { + "span_id": "2e06953f-5cec-4873-92b5-91f7281f6d7c", + "trace_id": "c0a95e8e-513c-47eb-b746-4e6016aeaf7d", + "parent_span_id": "9e41eab5-cd8e-4e46-a15b-e5f52c9a4073", + "name": "agent.query_routing", + "start_time": "2025-10-16 08:50:08.944000000", + "end_time": "2025-10-16 08:50:20.641000000", + "duration_ms": 11696.524, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "Tell me about recent developments in machine learning", + "query_length": 53, + "query_type": "routing_query", + "output_response": "Routed to Web Search for query processing", + "router.classification": "Web Search", + "router.confidence_score": 1, + "router.rag_keyword_score": 0, + "router.web_keyword_score": 1, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [] + }, + { + "span_id": "9e41eab5-cd8e-4e46-a15b-e5f52c9a4073", + "trace_id": "c0a95e8e-513c-47eb-b746-4e6016aeaf7d", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-16 08:50:08.944000000", + "end_time": "2025-10-16 08:50:20.641000000", + "duration_ms": 11696.671999999999, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "Tell me about recent developments in machine learning", + "query.length": 53, + "processing.mode": "Web Search", + "processing.time_seconds": 11.696574449539185, + "processing.response_length": 2001, + "processing.sources_count": 1, + "output_response": "Final Answer: The web search did not yield specific results regarding recent developments in machine learning. However, I can provide a general overview based on my knowledge up to October 2023.\n\nRecent development...", + "final_answer_mode": "Web Search", + "query_processed.input_query": "Tell me about recent developments in machine learning", + "query_processed.output_response": "Successfully processed query using Web Search, generated 2001 character response", + "query_processed.mode": "Web Search", + "query_processed.processing_time": 11.696574449539185, + "query_processed.response_length": 2001 + }, + "events": [], + "links": [] + }, + { + "span_id": "7ddf938b-183a-427f-849b-b31b18a98d87", + "trace_id": "c0a95e8e-513c-47eb-b746-4e6016aeaf7d", + "parent_span_id": "2e06953f-5cec-4873-92b5-91f7281f6d7c", + "name": "agent.web_search_generation", + "start_time": "2025-10-16 08:50:08.945000000", + "end_time": "2025-10-16 08:50:20.641000000", + "duration_ms": 11696.328, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "web_search_agent", + "agent.operation": "web_search_generation", + "agent.capabilities": [ + "web_search", + "content_synthesis", + "response_generation" + ], + "agent.id": "noveum_web_search_agent", + "input_query": "Tell me about recent developments in machine learning", + "query_length": 53, + "query_type": "web_search_query", + "output_response": "Web Search Answer: The web search did not yield specific results regarding recent developments in machine learning. However, I can provide a general overview based on my knowledge up to October 2023.\n\nRecent development...", + "web_search.results_count": 1, + "web_search.context_length": 155, + "web_search.response_length": 2001, + "web_search.mode": "external_web_search" + }, + "events": [], + "links": [] + }, + { + "span_id": "f3e4c594-f822-4f04-ab43-98fac161b9a3", + "trace_id": "c0a95e8e-513c-47eb-b746-4e6016aeaf7d", + "parent_span_id": "2e06953f-5cec-4873-92b5-91f7281f6d7c", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-16 08:50:08.945000000", + "end_time": "2025-10-16 08:50:08.945000000", + "duration_ms": 0.046, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: Tell me about recent developments in machine learning", + "query_length": 53, + "query_type": "routing_evaluation_query", + "classification.mode": "Web Search", + "classification.rag_keyword_score": 0, + "classification.web_keyword_score": 1, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "complex", + "query.intent": "general_knowledge", + "query.keyword_density": 0.125, + "query.domain_affinity": "general", + "routing.decision": "Routed to Web Search based on analysis", + "routing.rationale": "RAG score: 0, Web score: 1, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "RAG", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.019230769230769232, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: Web Search (Confidence: 1.00)" + }, + "events": [], + "links": [] + }, + { + "span_id": "b47d09be-a43b-4b0a-b7d3-d98bdf1773a3", + "trace_id": "c0a95e8e-513c-47eb-b746-4e6016aeaf7d", + "parent_span_id": "7ddf938b-183a-427f-849b-b31b18a98d87", + "name": "agent.llm_model_execution", + "start_time": "2025-10-16 08:50:09.490000000", + "end_time": "2025-10-16 08:50:20.641000000", + "duration_ms": 11151.141000000001, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: Tell me about recent developments in machine learning", + "query_length": 53, + "query_type": "web_search_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 11.151098012924194, + "model.latency_ms": 11151.098012924194, + "model.start_time": 1760604609.4902723, + "model.end_time": 1760604620.6413703, + "model.performance_tier": "slow", + "model.prompt_tokens": 163, + "model.completion_tokens": 500, + "model.total_tokens": 663, + "model.tokens_per_second": 59.456028386763236, + "model.estimated_cost": 0.01989, + "model.efficiency_score": 3.018099547511312, + "model.response_length": 2001, + "model.response_quality": "high", + "model.output_response": "Model Response: The web search did not yield specific results regarding recent developments in machine learning. However, I can provide a general overview based on my knowledge up to October 2023.\n\nRecent development...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [] + }, + { + "span_id": "bfea8500-0e5c-4556-a014-5b2ec49b7ad5", + "trace_id": "c0a95e8e-513c-47eb-b746-4e6016aeaf7d", + "parent_span_id": "7ddf938b-183a-427f-849b-b31b18a98d87", + "name": "agent.web_search_evaluation_metrics", + "start_time": "2025-10-16 08:50:20.641000000", + "end_time": "2025-10-16 08:50:20.641000000", + "duration_ms": 0.018000000000000002, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "web_search_evaluation_metrics", + "agent.capabilities": [ + "web_search_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: Tell me about recent developments in machine learning", + "query_length": 53, + "query_type": "web_search_evaluation_query", + "web_search.results_count": 1, + "web_search.context_length": 155, + "web_search.context_synthesized": "Context from 1 web sources", + "web_search.search_effectiveness": 0.2, + "web_search.context_quality": "low", + "web_search.source_diversity": 1, + "web_search.information_synthesis": "low", + "web_search.external_knowledge_utilization": 0.155, + "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=Tell+me+about+recent+developments+in+machine+learning\n\nUser Question: Tell me about recent developments in machine learning\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", + "prompt.prompt_length": 653, + "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=Tell+me+about+recent+developments+in+machine+learning", + "prompt.instruction_following": "web_search_optimized", + "response.answer_length": 2001, + "response.answer_completeness": "complete", + "response.response_quality": "medium", + "response.source_citation": 1, + "response.context_utilization": 12.90967741935484, + "output_response": "Web Search Answer: The web search did not yield specific results regarding recent developments in machine learning. However, I can provide a general overview based on my knowledge up to October 2023.\n\nRecent development...", + "evaluation.search_effectiveness": 0.2, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 1, + "evaluation.context_relevance": "low", + "evaluation.overall_quality": "low", + "evaluation.ready_for_production": true, + "web_search.search_strategy": "duckduckgo_api", + "web_search.real_time_data": true, + "web_search.external_sources": 1, + "web_search.information_freshness": "current", + "web_search.knowledge_synthesis": "single_source" + }, + "events": [], + "links": [] + } + ], + "created_at": "2025-10-16 08:50:08.944000000", + "updated_at": "2025-10-16 08:50:08.944000000" + }, + { + "trace_id": "3af8c6e8-62d9-4893-92ba-cc265934ef03", + "name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "status": "ok", + "status_message": "", + "start_time": "2025-10-16 08:50:04.544000000", + "end_time": "2025-10-16 08:50:07.941000000", + "duration_ms": 3397.228, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "spans": [ + { + "span_id": "33297286-d440-446a-ba73-7cabd9d4b2e6", + "trace_id": "3af8c6e8-62d9-4893-92ba-cc265934ef03", + "parent_span_id": "4a3c230e-0dd5-4ba2-8e87-2a301f654fba", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-16 08:50:04.544000000", + "end_time": "2025-10-16 08:50:04.544000000", + "duration_ms": 0.038, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What's the weather like today?", + "query_length": 30, + "query_type": "routing_evaluation_query", + "classification.mode": "Web Search", + "classification.rag_keyword_score": 0, + "classification.web_keyword_score": 2, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "general_knowledge", + "query.keyword_density": 0.4, + "query.domain_affinity": "general", + "routing.decision": "Routed to Web Search based on analysis", + "routing.rationale": "RAG score: 0, Web score: 2, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "RAG", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.038461538461538464, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: Web Search (Confidence: 1.00)" + }, + "events": [], + "links": [] + }, + { + "span_id": "4a3c230e-0dd5-4ba2-8e87-2a301f654fba", + "trace_id": "3af8c6e8-62d9-4893-92ba-cc265934ef03", + "parent_span_id": "eefcbdc5-99fc-4f06-a9f4-f51ba50564db", + "name": "agent.query_routing", + "start_time": "2025-10-16 08:50:04.544000000", + "end_time": "2025-10-16 08:50:07.941000000", + "duration_ms": 3397.056, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What's the weather like today?", + "query_length": 30, + "query_type": "routing_query", + "output_response": "Routed to Web Search for query processing", + "router.classification": "Web Search", + "router.confidence_score": 1, + "router.rag_keyword_score": 0, + "router.web_keyword_score": 2, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [] + }, + { + "span_id": "eefcbdc5-99fc-4f06-a9f4-f51ba50564db", + "trace_id": "3af8c6e8-62d9-4893-92ba-cc265934ef03", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-16 08:50:04.544000000", + "end_time": "2025-10-16 08:50:07.941000000", + "duration_ms": 3397.169, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What's the weather like today?", + "query.length": 30, + "processing.mode": "Web Search", + "processing.time_seconds": 3.3970956802368164, + "processing.response_length": 518, + "processing.sources_count": 1, + "output_response": "Final Answer: The web search results do not provide specific details about today's weather conditions in your location. However, they mention that you can get real-time updates, including temperature, humidity, and...", + "final_answer_mode": "Web Search", + "query_processed.input_query": "What's the weather like today?", + "query_processed.output_response": "Successfully processed query using Web Search, generated 518 character response", + "query_processed.mode": "Web Search", + "query_processed.processing_time": 3.3970956802368164, + "query_processed.response_length": 518 + }, + "events": [], + "links": [] + }, + { + "span_id": "9683465c-e851-4004-a858-c8fdd1a2b3da", + "trace_id": "3af8c6e8-62d9-4893-92ba-cc265934ef03", + "parent_span_id": "4a3c230e-0dd5-4ba2-8e87-2a301f654fba", + "name": "agent.web_search_generation", + "start_time": "2025-10-16 08:50:04.545000000", + "end_time": "2025-10-16 08:50:07.941000000", + "duration_ms": 3396.871, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "web_search_agent", + "agent.operation": "web_search_generation", + "agent.capabilities": [ + "web_search", + "content_synthesis", + "response_generation" + ], + "agent.id": "noveum_web_search_agent", + "input_query": "What's the weather like today?", + "query_length": 30, + "query_type": "web_search_query", + "output_response": "Web Search Answer: The web search results do not provide specific details about today's weather conditions in your location. However, they mention that you can get real-time updates, including temperature, humidity, and...", + "web_search.results_count": 1, + "web_search.context_length": 1137, + "web_search.response_length": 518, + "web_search.mode": "external_web_search" + }, + "events": [], + "links": [] + }, + { + "span_id": "6413eb7e-e5cd-4c46-9651-de14c72c260e", + "trace_id": "3af8c6e8-62d9-4893-92ba-cc265934ef03", + "parent_span_id": "9683465c-e851-4004-a858-c8fdd1a2b3da", + "name": "agent.llm_model_execution", + "start_time": "2025-10-16 08:50:04.986000000", + "end_time": "2025-10-16 08:50:07.941000000", + "duration_ms": 2955.55, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What's the weather like today?", + "query_length": 30, + "query_type": "web_search_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 2.955509662628174, + "model.latency_ms": 2955.509662628174, + "model.start_time": 1760604604.9862766, + "model.end_time": 1760604607.9417863, + "model.performance_tier": "medium", + "model.prompt_tokens": 403, + "model.completion_tokens": 129, + "model.total_tokens": 532, + "model.tokens_per_second": 180.0027950261957, + "model.estimated_cost": 0.015960000000000002, + "model.efficiency_score": 0.9736842105263158, + "model.response_length": 518, + "model.response_quality": "high", + "model.output_response": "Model Response: The web search results do not provide specific details about today's weather conditions in your location. However, they mention that you can get real-time updates, including temperature, humidity, and...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [] + }, + { + "span_id": "64df69fb-0bcc-4845-8bfe-5a721c1ed912", + "trace_id": "3af8c6e8-62d9-4893-92ba-cc265934ef03", + "parent_span_id": "9683465c-e851-4004-a858-c8fdd1a2b3da", + "name": "agent.web_search_evaluation_metrics", + "start_time": "2025-10-16 08:50:07.941000000", + "end_time": "2025-10-16 08:50:07.941000000", + "duration_ms": 0.017, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "web_search_evaluation_metrics", + "agent.capabilities": [ + "web_search_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What's the weather like today?", + "query_length": 30, + "query_type": "web_search_evaluation_query", + "web_search.results_count": 1, + "web_search.context_length": 1137, + "web_search.context_synthesized": "Context from 1 web sources", + "web_search.search_effectiveness": 0.2, + "web_search.context_quality": "high", + "web_search.source_diversity": 1, + "web_search.information_synthesis": "low", + "web_search.external_knowledge_utilization": 1.137, + "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nStay updated with the latest weather conditions in Komna. Get real-time updates, temperature, humidity, AQI, and detailed forecast for today. Dec 31, 2024 \u00b7 Komna Weather Forecast: Check the latest temperature in Komna today along with details such as humidity, visibility, air quality, wind speed, pressure, and weekly weather report at Shuru. 2 days ago \u00b7 Latest weather forecast for India for today's, hourly weather forecast, including today's temperatures in India, wind, rain and more. 2 days ago \u00b7 Get the latest hourly weather updates for India today. Detailed forecast including temperature, wind, rain, snow, and UV index. Stay informed about today's weather conditions in India. 2 days ago \u00b7 Today's Weather Report ... Forecasts Urban Meteorological Services Short to Medium Range Model Guidance Extended Range Model Guidance Seasonal Forecast Quantitative Precipitation Forecast Latest CAP Alerts Specialised products & forecasts Cyclones | Monsoon | Satellite Radar | Rainfall | Agromet Advisories MAUSAM : | | Video Meghdoot Agro ...\nURL: https://duckduckgo.com/?q=What's+the+weather+like+today?\n\nUser Question: What's the weather like today?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", + "prompt.prompt_length": 1612, + "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nStay updated with the latest weather conditions in Komna. Get real-time updates, temperature, humidity, AQI, and detailed forecast for today. Dec 31, 2024 \u00b7 Komna Weather F...", + "prompt.instruction_following": "web_search_optimized", + "response.answer_length": 518, + "response.answer_completeness": "complete", + "response.response_quality": "medium", + "response.source_citation": 1, + "response.context_utilization": 0.455584872471416, + "output_response": "Web Search Answer: The web search results do not provide specific details about today's weather conditions in your location. However, they mention that you can get real-time updates, including temperature, humidity, and...", + "evaluation.search_effectiveness": 0.2, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 1, + "evaluation.context_relevance": "high", + "evaluation.overall_quality": "low", + "evaluation.ready_for_production": true, + "web_search.search_strategy": "duckduckgo_api", + "web_search.real_time_data": true, + "web_search.external_sources": 1, + "web_search.information_freshness": "current", + "web_search.knowledge_synthesis": "single_source" + }, + "events": [], + "links": [] + } + ], + "created_at": "2025-10-16 08:50:04.544000000", + "updated_at": "2025-10-16 08:50:04.544000000" + }, + { + "trace_id": "d7b8b811-ad01-48f1-917f-ace7e8d289e6", + "name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "status": "ok", + "status_message": "", + "start_time": "2025-10-16 08:50:01.375000000", + "end_time": "2025-10-16 08:50:03.540000000", + "duration_ms": 2164.576, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "spans": [ + { + "span_id": "719a1a8f-fc50-439c-899b-436146accd23", + "trace_id": "d7b8b811-ad01-48f1-917f-ace7e8d289e6", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-16 08:50:01.375000000", + "end_time": "2025-10-16 08:50:03.540000000", + "duration_ms": 2164.4809999999998, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What are the latest AI news today?", + "query.length": 34, + "processing.mode": "Web Search", + "processing.time_seconds": 2.1643810272216797, + "processing.response_length": 298, + "processing.sources_count": 1, + "output_response": "Final Answer: The web search did not yield any specific results regarding the latest AI news today. Unfortunately, I cannot provide current updates or news articles on AI developments at this moment. For the latest...", + "final_answer_mode": "Web Search", + "query_processed.input_query": "What are the latest AI news today?", + "query_processed.output_response": "Successfully processed query using Web Search, generated 298 character response", + "query_processed.mode": "Web Search", + "query_processed.processing_time": 2.1643810272216797, + "query_processed.response_length": 298 + }, + "events": [], + "links": [] + }, + { + "span_id": "644a712f-1c9e-4e9a-9073-74621b754aa2", + "trace_id": "d7b8b811-ad01-48f1-917f-ace7e8d289e6", + "parent_span_id": "719a1a8f-fc50-439c-899b-436146accd23", + "name": "agent.query_routing", + "start_time": "2025-10-16 08:50:01.376000000", + "end_time": "2025-10-16 08:50:03.540000000", + "duration_ms": 2164.3219999999997, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What are the latest AI news today?", + "query_length": 34, + "query_type": "routing_query", + "output_response": "Routed to Web Search for query processing", + "router.classification": "Web Search", + "router.confidence_score": 1, + "router.rag_keyword_score": 0, + "router.web_keyword_score": 3, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [] + }, + { + "span_id": "8a4abdbe-6ad7-4d43-b234-81667c4abbc2", + "trace_id": "d7b8b811-ad01-48f1-917f-ace7e8d289e6", + "parent_span_id": "644a712f-1c9e-4e9a-9073-74621b754aa2", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-16 08:50:01.376000000", + "end_time": "2025-10-16 08:50:01.376000000", + "duration_ms": 0.051, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What are the latest AI news today?", + "query_length": 34, + "query_type": "routing_evaluation_query", + "classification.mode": "Web Search", + "classification.rag_keyword_score": 0, + "classification.web_keyword_score": 3, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "general_knowledge", + "query.keyword_density": 0.42857142857142855, + "query.domain_affinity": "general", + "routing.decision": "Routed to Web Search based on analysis", + "routing.rationale": "RAG score: 0, Web score: 3, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "RAG", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.057692307692307696, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: Web Search (Confidence: 1.00)" + }, + "events": [], + "links": [] + }, + { + "span_id": "904621e8-c5fa-4b2d-a628-8f10eb9da51d", + "trace_id": "d7b8b811-ad01-48f1-917f-ace7e8d289e6", + "parent_span_id": "644a712f-1c9e-4e9a-9073-74621b754aa2", + "name": "agent.web_search_generation", + "start_time": "2025-10-16 08:50:01.376000000", + "end_time": "2025-10-16 08:50:03.540000000", + "duration_ms": 2164.076, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "web_search_agent", + "agent.operation": "web_search_generation", + "agent.capabilities": [ + "web_search", + "content_synthesis", + "response_generation" + ], + "agent.id": "noveum_web_search_agent", + "input_query": "What are the latest AI news today?", + "query_length": 34, + "query_type": "web_search_query", + "output_response": "Web Search Answer: The web search did not yield any specific results regarding the latest AI news today. Unfortunately, I cannot provide current updates or news articles on AI developments at this moment. For the latest...", + "web_search.results_count": 1, + "web_search.context_length": 136, + "web_search.response_length": 298, + "web_search.mode": "external_web_search" + }, + "events": [], + "links": [] + }, + { + "span_id": "421dd1af-2dbc-477f-bd59-b3194b14ae3a", + "trace_id": "d7b8b811-ad01-48f1-917f-ace7e8d289e6", + "parent_span_id": "904621e8-c5fa-4b2d-a628-8f10eb9da51d", + "name": "agent.llm_model_execution", + "start_time": "2025-10-16 08:50:01.708000000", + "end_time": "2025-10-16 08:50:03.540000000", + "duration_ms": 1831.237, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What are the latest AI news today?", + "query_length": 34, + "query_type": "web_search_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 1.8311803340911865, + "model.latency_ms": 1831.1803340911865, + "model.start_time": 1760604601.7090185, + "model.end_time": 1760604603.5401988, + "model.performance_tier": "fast", + "model.prompt_tokens": 153, + "model.completion_tokens": 74, + "model.total_tokens": 227, + "model.tokens_per_second": 123.96376029926071, + "model.estimated_cost": 0.00681, + "model.efficiency_score": 1.3127753303964758, + "model.response_length": 298, + "model.response_quality": "high", + "model.output_response": "Model Response: The web search did not yield any specific results regarding the latest AI news today. Unfortunately, I cannot provide current updates or news articles on AI developments at this moment. For the latest...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [] + }, + { + "span_id": "159aad6f-7174-406d-b7d9-eed9360d993c", + "trace_id": "d7b8b811-ad01-48f1-917f-ace7e8d289e6", + "parent_span_id": "904621e8-c5fa-4b2d-a628-8f10eb9da51d", + "name": "agent.web_search_evaluation_metrics", + "start_time": "2025-10-16 08:50:03.540000000", + "end_time": "2025-10-16 08:50:03.540000000", + "duration_ms": 0.028, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "web_search_evaluation_metrics", + "agent.capabilities": [ + "web_search_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What are the latest AI news today?", + "query_length": 34, + "query_type": "web_search_evaluation_query", + "web_search.results_count": 1, + "web_search.context_length": 136, + "web_search.context_synthesized": "Context from 1 web sources", + "web_search.search_effectiveness": 0.2, + "web_search.context_quality": "low", + "web_search.source_diversity": 1, + "web_search.information_synthesis": "low", + "web_search.external_knowledge_utilization": 0.136, + "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What+are+the+latest+AI+news+today?\n\nUser Question: What are the latest AI news today?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", + "prompt.prompt_length": 615, + "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What+are+the+latest+AI+news+today?", + "prompt.instruction_following": "web_search_optimized", + "response.answer_length": 298, + "response.answer_completeness": "complete", + "response.response_quality": "medium", + "response.source_citation": 1, + "response.context_utilization": 2.1911764705882355, + "output_response": "Web Search Answer: The web search did not yield any specific results regarding the latest AI news today. Unfortunately, I cannot provide current updates or news articles on AI developments at this moment. For the latest...", + "evaluation.search_effectiveness": 0.2, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 1, + "evaluation.context_relevance": "low", + "evaluation.overall_quality": "low", + "evaluation.ready_for_production": true, + "web_search.search_strategy": "duckduckgo_api", + "web_search.real_time_data": true, + "web_search.external_sources": 1, + "web_search.information_freshness": "current", + "web_search.knowledge_synthesis": "single_source" + }, + "events": [], + "links": [] + } + ], + "created_at": "2025-10-16 08:50:01.375000000", + "updated_at": "2025-10-16 08:50:01.375000000" + }, + { + "trace_id": "df8340b4-b75a-48e9-a2f3-99725b46a00d", + "name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "status": "ok", + "status_message": "", + "start_time": "2025-10-16 08:49:17.236000000", + "end_time": "2025-10-16 08:50:00.368000000", + "duration_ms": 43131.554000000004, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "spans": [ + { + "span_id": "dcf2b38e-ab90-45ee-9a65-e6fea47a3fd7", + "trace_id": "df8340b4-b75a-48e9-a2f3-99725b46a00d", + "parent_span_id": "d0e2b320-c18d-4ec6-98a1-74da5252107e", + "name": "agent.query_routing", + "start_time": "2025-10-16 08:49:17.236000000", + "end_time": "2025-10-16 08:50:00.368000000", + "duration_ms": 43131.384, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What are the benefits of using Noveum Trace?", + "query_length": 44, + "query_type": "routing_query", + "output_response": "Routed to RAG for query processing", + "router.classification": "RAG", + "router.confidence_score": 1, + "router.rag_keyword_score": 2, + "router.web_keyword_score": 0, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [] + }, + { + "span_id": "d0e2b320-c18d-4ec6-98a1-74da5252107e", + "trace_id": "df8340b4-b75a-48e9-a2f3-99725b46a00d", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-16 08:49:17.236000000", + "end_time": "2025-10-16 08:50:00.368000000", + "duration_ms": 43131.481, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What are the benefits of using Noveum Trace?", + "query.length": 44, + "processing.mode": "RAG", + "processing.time_seconds": 43.131428718566895, + "processing.response_length": 1415, + "processing.sources_count": 5, + "output_response": "Final Answer: The benefits of using Noveum Trace include:\n\n1. **Comprehensive Tracing**: Noveum Trace offers decorator-based tracing for LLM calls, agents, and RAG pipelines, allowing for detailed monitoring of AI ...", + "final_answer_mode": "RAG", + "query_processed.input_query": "What are the benefits of using Noveum Trace?", + "query_processed.output_response": "Successfully processed query using RAG, generated 1415 character response", + "query_processed.mode": "RAG", + "query_processed.processing_time": 43.131428718566895, + "query_processed.response_length": 1415 + }, + "events": [], + "links": [] + }, + { + "span_id": "fa93821f-c809-4071-87e1-a6c55014d37c", + "trace_id": "df8340b4-b75a-48e9-a2f3-99725b46a00d", + "parent_span_id": "dcf2b38e-ab90-45ee-9a65-e6fea47a3fd7", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-16 08:49:17.236000000", + "end_time": "2025-10-16 08:49:17.236000000", + "duration_ms": 0.034, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What are the benefits of using Noveum Trace?", + "query_length": 44, + "query_type": "routing_evaluation_query", + "classification.mode": "RAG", + "classification.rag_keyword_score": 2, + "classification.web_keyword_score": 0, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "noveum_specific", + "query.keyword_density": 0.25, + "query.domain_affinity": "noveum", + "routing.decision": "Routed to RAG based on analysis", + "routing.rationale": "RAG score: 2, Web score: 0, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "Web Search", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.038461538461538464, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: RAG (Confidence: 1.00)" + }, + "events": [], + "links": [] + }, + { + "span_id": "024ad5aa-f394-4c26-af51-564f3f51c6f2", + "trace_id": "df8340b4-b75a-48e9-a2f3-99725b46a00d", + "parent_span_id": "dcf2b38e-ab90-45ee-9a65-e6fea47a3fd7", + "name": "agent.llm-rag", + "start_time": "2025-10-16 08:49:17.237000000", + "end_time": "2025-10-16 08:50:00.368000000", + "duration_ms": 43131.19300000001, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "rag_agent", + "agent.operation": "llm-rag", + "agent.capabilities": [ + "document_retrieval", + "context_generation", + "response_generation" + ], + "agent.id": "noveum_rag_agent", + "input_query": "What are the benefits of using Noveum Trace?", + "query_length": 44, + "query_type": "rag_query", + "output_response": "RAG Answer: The benefits of using Noveum Trace include:\n\n1. **Comprehensive Tracing**: Noveum Trace offers decorator-based tracing for LLM calls, agents, and RAG pipelines, allowing for detailed monitoring of AI ...", + "rag.context_length": 2773, + "rag.sources_count": 5, + "rag.answer_length": 1415, + "rag.mode": "retrieval_augmented_generation" + }, + "events": [], + "links": [] + }, + { + "span_id": "3d9db951-d8b0-4990-820d-61e4b1011709", + "trace_id": "df8340b4-b75a-48e9-a2f3-99725b46a00d", + "parent_span_id": "024ad5aa-f394-4c26-af51-564f3f51c6f2", + "name": "agent.llm_model_execution", + "start_time": "2025-10-16 08:49:20.224000000", + "end_time": "2025-10-16 08:49:58.420000000", + "duration_ms": 38196.111000000004, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What are the benefits of using Noveum Trace?", + "query_length": 44, + "query_type": "rag_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 38.19604015350342, + "model.latency_ms": 38196.04015350342, + "model.start_time": 1760604560.2242246, + "model.end_time": 1760604598.4202647, + "model.performance_tier": "slow", + "model.prompt_tokens": 828, + "model.completion_tokens": 353, + "model.total_tokens": 1181, + "model.tokens_per_second": 30.919435503098253, + "model.estimated_cost": 0.03543, + "model.efficiency_score": 1.1981371718882303, + "model.response_length": 1415, + "model.response_quality": "high", + "model.output_response": "Model Response: The benefits of using Noveum Trace include:\n\n1. **Comprehensive Tracing**: Noveum Trace offers decorator-based tracing for LLM calls, agents, and RAG pipelines, allowing for detailed monitoring of AI ...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [] + }, + { + "span_id": "8c33ffac-4d76-42ba-9737-89596e136b3d", + "trace_id": "df8340b4-b75a-48e9-a2f3-99725b46a00d", + "parent_span_id": "024ad5aa-f394-4c26-af51-564f3f51c6f2", + "name": "agent.rag_evaluation_metrics", + "start_time": "2025-10-16 08:49:58.420000000", + "end_time": "2025-10-16 08:49:58.420000000", + "duration_ms": 0.037, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "rag_evaluation_metrics", + "agent.capabilities": [ + "retrieval_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What are the benefits of using Noveum Trace?", + "query_length": 44, + "query_type": "rag_evaluation_query", + "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/docs/platform/dashboard):\n- Bulk Export: Download multiple traces simultaneously\n- Comparative Analysis: Compare performance across multiple traces\nReal-Time Updates\n- Live Refresh: Automatic updates as new traces arrive\n- Connection Monitoring: Real-time status of your t...", + "retrieval.context_length": 2773, + "retrieval.sources_count": 5, + "retrieval.context_quality": "high", + "retrieval.effectiveness": 1, + "retrieval.context_utilization": 2.773, + "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/docs/platform/dashboard):\n- Bulk Export: Download multiple traces simultaneously\n- Comparative Analysis: Compare performance across multiple traces\nReal-Time Updates\n- Live Refresh: Automatic updates as new traces arrive\n- Connection Monitoring: Real-time status of your trace ingestion pipeline\n- Performance Indicators: Live metrics for system health monitoring\nReady to dive deeper? Explore Projects & Environments to organize your AI applications, or check out Team Collaboration to share insights with your team.\nGet Earl...\n\nSource 2 (https://noveum.ai/en/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\n\ud83d\udcd8 TypeScript SDK (\n@noveum/trace\n)- Framework integrations for Next.js, Express.js, and Hono\n- TypeScript-first design with full type safety\n- Universal compatibility (Node.js, Edge Runtime, browsers)\n-\n\ud83d\udcca Noveum Platform\n- Real-time dashboard for analyzing traces and performance\n- Ad...\n\nSource 3 (https://noveum.ai/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\n\ud83d\udcd8 TypeScript SDK (\n@noveum/trace\n)- Framework integrations for Next.js, Express.js, and Hono\n- TypeScript-first design with full type safety\n- Universal compatibility (Node.js, Edge Runtime, browsers)\n-\n\ud83d\udcca Noveum Platform\n- Real-time dashboard for analyzing traces and performance\n- Ad...\n\nSource 4 (https://noveum.ai/en/docs/getting-started/overview):\n- Advanced search & filtering across traces and spans\n- Cost analysis and optimization recommendations\n- Team collaboration with shared insights and alerts\n\ud83d\udd0d What Noveum Traces\nLLM Operations\n- Model calls across all providers (OpenAI, Anthropic, Google, etc.)\n- Token usage and cost calculation\n- Prompt engineering effectiveness\n- Response quality metrics\nRAG Pipelines\n- Document retrieval performance and relevance\n- Embedding generation costs and latency\n- Context assembly and prompt constructi...\n\nSource 5 (https://noveum.ai/docs/getting-started/overview):\n- Advanced search & filtering across traces and spans\n- Cost analysis and optimization recommendations\n- Team collaboration with shared insights and alerts\n\ud83d\udd0d What Noveum Traces\nLLM Operations\n- Model calls across all providers (OpenAI, Anthropic, Google, etc.)\n- Token usage and cost calculation\n- Prompt engineering effectiveness\n- Response quality metrics\nRAG Pipelines\n- Document retrieval performance and relevance\n- Embedding generation costs and latency\n- Context assembly and prompt constructi...\n\nUser Question: What are the benefits of using Noveum Trace?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", + "prompt.prompt_length": 3313, + "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/docs/platform/dashboard):\n- Bulk Export: Download multiple traces simultaneously\n- Comparative Analysis: Compare performance across multiple traces\nReal-Time Updates\n- Live...", + "prompt.instruction_following": "rag_optimized", + "response.answer_length": 1415, + "response.answer_completeness": "complete", + "response.response_quality": "high", + "response.source_citation": 5, + "response.context_utilization": 0.5102776776054815, + "output_response": "RAG Answer: The benefits of using Noveum Trace include:\n\n1. **Comprehensive Tracing**: Noveum Trace offers decorator-based tracing for LLM calls, agents, and RAG pipelines, allowing for detailed monitoring of AI ...", + "evaluation.retrieval_effectiveness": 1, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 5, + "evaluation.context_relevance": "high", + "evaluation.overall_quality": "high", + "evaluation.ready_for_production": true, + "rag.retrieval_strategy": "semantic_similarity", + "rag.vector_search_results": 5, + "rag.context_synthesis": "multi_source", + "rag.document_coverage": 1, + "rag.information_density": 0.5102776776054815 + }, + "events": [], + "links": [] + } + ], + "created_at": "2025-10-16 08:49:17.236000000", + "updated_at": "2025-10-16 08:49:17.236000000" + }, + { + "trace_id": "325ec189-2876-4ebe-b33c-649491a5053e", + "name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "status": "ok", + "status_message": "", + "start_time": "2025-10-16 08:49:11.192000000", + "end_time": "2025-10-16 08:49:16.232000000", + "duration_ms": 5039.841, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "spans": [ + { + "span_id": "5b1a7121-d0bd-4525-821d-7721c6fbbd70", + "trace_id": "325ec189-2876-4ebe-b33c-649491a5053e", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-16 08:49:11.192000000", + "end_time": "2025-10-16 08:49:16.232000000", + "duration_ms": 5039.753, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "How do I configure Noveum for my system?", + "query.length": 40, + "processing.mode": "RAG", + "processing.time_seconds": 5.039698600769043, + "processing.response_length": 712, + "processing.sources_count": 5, + "output_response": "Final Answer: To configure Noveum for your system, you can refer to the SDK Integration Guide available in the Noveum documentation. This guide will help you trace your AI applications with minimal code changes. Th...", + "final_answer_mode": "RAG", + "query_processed.input_query": "How do I configure Noveum for my system?", + "query_processed.output_response": "Successfully processed query using RAG, generated 712 character response", + "query_processed.mode": "RAG", + "query_processed.processing_time": 5.039698600769043, + "query_processed.response_length": 712 + }, + "events": [], + "links": [] + }, + { + "span_id": "761c92c6-d15a-4a8e-a16a-a365922f8d7c", + "trace_id": "325ec189-2876-4ebe-b33c-649491a5053e", + "parent_span_id": "5b1a7121-d0bd-4525-821d-7721c6fbbd70", + "name": "agent.query_routing", + "start_time": "2025-10-16 08:49:11.192000000", + "end_time": "2025-10-16 08:49:16.232000000", + "duration_ms": 5039.642, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "How do I configure Noveum for my system?", + "query_length": 40, + "query_type": "routing_query", + "output_response": "Routed to RAG for query processing", + "router.classification": "RAG", + "router.confidence_score": 1, + "router.rag_keyword_score": 2, + "router.web_keyword_score": 0, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [] + }, + { + "span_id": "8fcf969e-a577-45aa-9fe6-1870d9665189", + "trace_id": "325ec189-2876-4ebe-b33c-649491a5053e", + "parent_span_id": "761c92c6-d15a-4a8e-a16a-a365922f8d7c", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-16 08:49:11.192000000", + "end_time": "2025-10-16 08:49:11.192000000", + "duration_ms": 0.045000000000000005, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: How do I configure Noveum for my system?", + "query_length": 40, + "query_type": "routing_evaluation_query", + "classification.mode": "RAG", + "classification.rag_keyword_score": 2, + "classification.web_keyword_score": 0, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "noveum_specific", + "query.keyword_density": 0.25, + "query.domain_affinity": "noveum", + "routing.decision": "Routed to RAG based on analysis", + "routing.rationale": "RAG score: 2, Web score: 0, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "Web Search", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.038461538461538464, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: RAG (Confidence: 1.00)" + }, + "events": [], + "links": [] + }, + { + "span_id": "9e5bbea4-cc68-4fe6-b3d0-c87852635ec9", + "trace_id": "325ec189-2876-4ebe-b33c-649491a5053e", + "parent_span_id": "761c92c6-d15a-4a8e-a16a-a365922f8d7c", + "name": "agent.llm-rag", + "start_time": "2025-10-16 08:49:11.192000000", + "end_time": "2025-10-16 08:49:16.232000000", + "duration_ms": 5039.39, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "rag_agent", + "agent.operation": "llm-rag", + "agent.capabilities": [ + "document_retrieval", + "context_generation", + "response_generation" + ], + "agent.id": "noveum_rag_agent", + "input_query": "How do I configure Noveum for my system?", + "query_length": 40, + "query_type": "rag_query", + "output_response": "RAG Answer: To configure Noveum for your system, you can refer to the SDK Integration Guide available in the Noveum documentation. This guide will help you trace your AI applications with minimal code changes. Th...", + "rag.context_length": 2237, + "rag.sources_count": 5, + "rag.answer_length": 712, + "rag.mode": "retrieval_augmented_generation" + }, + "events": [], + "links": [] + }, + { + "span_id": "5ee706ad-c4ae-4c0b-94d1-6b9c43d173b4", + "trace_id": "325ec189-2876-4ebe-b33c-649491a5053e", + "parent_span_id": "9e5bbea4-cc68-4fe6-b3d0-c87852635ec9", + "name": "agent.llm_model_execution", + "start_time": "2025-10-16 08:49:11.607000000", + "end_time": "2025-10-16 08:49:15.718000000", + "duration_ms": 4110.915, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: How do I configure Noveum for my system?", + "query_length": 40, + "query_type": "rag_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 4.110846042633057, + "model.latency_ms": 4110.846042633057, + "model.start_time": 1760604551.6074958, + "model.end_time": 1760604555.7183418, + "model.performance_tier": "medium", + "model.prompt_tokens": 693, + "model.completion_tokens": 178, + "model.total_tokens": 871, + "model.tokens_per_second": 211.87852596934326, + "model.estimated_cost": 0.02613, + "model.efficiency_score": 0.817451205510907, + "model.response_length": 712, + "model.response_quality": "high", + "model.output_response": "Model Response: To configure Noveum for your system, you can refer to the SDK Integration Guide available in the Noveum documentation. This guide will help you trace your AI applications with minimal code changes. Th...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [] + }, + { + "span_id": "d13bdeda-81af-400a-82ca-a5ef7e1e8a6d", + "trace_id": "325ec189-2876-4ebe-b33c-649491a5053e", + "parent_span_id": "9e5bbea4-cc68-4fe6-b3d0-c87852635ec9", + "name": "agent.rag_evaluation_metrics", + "start_time": "2025-10-16 08:49:15.718000000", + "end_time": "2025-10-16 08:49:15.718000000", + "duration_ms": 0.118, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "rag_evaluation_metrics", + "agent.capabilities": [ + "retrieval_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: How do I configure Noveum for my system?", + "query_length": 40, + "query_type": "rag_evaluation_query", + "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/docs):\nWelcome to Noveum.ai Docs\nComprehensive AI tracing and observability for LLM applications, RAG systems, and AI agents\nWelcome to the Noveum.ai documentation! Here you'll find everything you need to integrate, configure, and optimize your AI applications using ou...", + "retrieval.context_length": 2237, + "retrieval.sources_count": 5, + "retrieval.context_quality": "high", + "retrieval.effectiveness": 1, + "retrieval.context_utilization": 2.237, + "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/docs):\nWelcome to Noveum.ai Docs\nComprehensive AI tracing and observability for LLM applications, RAG systems, and AI agents\nWelcome to the Noveum.ai documentation! Here you'll find everything you need to integrate, configure, and optimize your AI applications using our comprehensive tracing and observability platform.\nWe provide powerful SDKs for Python and TypeScript that enable you to trace LLM calls, RAG pipelines, and multi-agent workflows with minimal code changes. Our documentation is organized ...\n\nSource 2 (https://noveum.ai/docs):\nWelcome to Noveum.ai Docs\nComprehensive AI tracing and observability for LLM applications, RAG systems, and AI agents\nWelcome to the Noveum.ai documentation! Here you'll find everything you need to integrate, configure, and optimize your AI applications using our comprehensive tracing and observability platform.\nWe provide powerful SDKs for Python and TypeScript that enable you to trace LLM calls, RAG pipelines, and multi-agent workflows with minimal code changes. Our documentation is organized ...\n\nSource 3 (https://noveum.ai/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nSource 4 (https://noveum.ai/en/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nSource 5 (https://noveum.ai/en/docs):\n- \ud83c\udf0d Universal: Works across frameworks, clouds, and deployment models\n\ud83e\udd1d Community & Support\n- \ud83d\udcac Discord Community: Join our Discord\n- \ud83d\udce7 Email Support: [email protected]\n- \ud83d\udc1b Bug Reports: GitHub Issues\n- \ud83d\udcd6 Knowledge Base: Help Center\nReady to get started? Head to our SDK Integration Guide to begin tracing your AI applications in under 5 minutes!\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability ...\n\nUser Question: How do I configure Noveum for my system?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", + "prompt.prompt_length": 2773, + "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/docs):\nWelcome to Noveum.ai Docs\nComprehensive AI tracing and observability for LLM applications, RAG systems, and AI agents\nWelcome to the Noveum.ai documentation! Here...", + "prompt.instruction_following": "rag_optimized", + "response.answer_length": 712, + "response.answer_completeness": "complete", + "response.response_quality": "high", + "response.source_citation": 5, + "response.context_utilization": 0.3182834152883326, + "output_response": "RAG Answer: To configure Noveum for your system, you can refer to the SDK Integration Guide available in the Noveum documentation. This guide will help you trace your AI applications with minimal code changes. Th...", + "evaluation.retrieval_effectiveness": 1, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 5, + "evaluation.context_relevance": "high", + "evaluation.overall_quality": "high", + "evaluation.ready_for_production": true, + "rag.retrieval_strategy": "semantic_similarity", + "rag.vector_search_results": 5, + "rag.context_synthesis": "multi_source", + "rag.document_coverage": 1, + "rag.information_density": 0.3182834152883326 + }, + "events": [], + "links": [] + } + ], + "created_at": "2025-10-16 08:49:11.192000000", + "updated_at": "2025-10-16 08:49:11.192000000" + }, + { + "trace_id": "d5528586-c741-4703-8791-5fccf5c3e123", + "name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "status": "ok", + "status_message": "", + "start_time": "2025-10-16 08:49:05.152000000", + "end_time": "2025-10-16 08:49:10.187000000", + "duration_ms": 5034.604, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "spans": [ + { + "span_id": "7d6564fc-41b2-4d74-acb1-e77b575c0006", + "trace_id": "d5528586-c741-4703-8791-5fccf5c3e123", + "parent_span_id": "e79a037e-82b1-41e9-bdc7-42353e98ead7", + "name": "agent.query_routing", + "start_time": "2025-10-16 08:49:05.152000000", + "end_time": "2025-10-16 08:49:10.187000000", + "duration_ms": 5034.446, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What monitoring capabilities does Noveum provide?", + "query_length": 49, + "query_type": "routing_query", + "output_response": "Routed to RAG for query processing", + "router.classification": "RAG", + "router.confidence_score": 1, + "router.rag_keyword_score": 2, + "router.web_keyword_score": 0, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [] + }, + { + "span_id": "e79a037e-82b1-41e9-bdc7-42353e98ead7", + "trace_id": "d5528586-c741-4703-8791-5fccf5c3e123", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-16 08:49:05.152000000", + "end_time": "2025-10-16 08:49:10.187000000", + "duration_ms": 5034.531, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What monitoring capabilities does Noveum provide?", + "query.length": 49, + "processing.mode": "RAG", + "processing.time_seconds": 5.0344908237457275, + "processing.response_length": 552, + "processing.sources_count": 5, + "output_response": "Final Answer: Noveum provides comprehensive monitoring capabilities for AI applications, including the ability to monitor, trace, and optimize AI agents across various frameworks such as LangChain, CrewAI, AutoGen,...", + "final_answer_mode": "RAG", + "query_processed.input_query": "What monitoring capabilities does Noveum provide?", + "query_processed.output_response": "Successfully processed query using RAG, generated 552 character response", + "query_processed.mode": "RAG", + "query_processed.processing_time": 5.0344908237457275, + "query_processed.response_length": 552 + }, + "events": [], + "links": [] + }, + { + "span_id": "78297793-d73b-4311-b5ba-73509ae9f96e", + "trace_id": "d5528586-c741-4703-8791-5fccf5c3e123", + "parent_span_id": "7d6564fc-41b2-4d74-acb1-e77b575c0006", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-16 08:49:05.153000000", + "end_time": "2025-10-16 08:49:05.153000000", + "duration_ms": 0.029, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What monitoring capabilities does Noveum provide?", + "query_length": 49, + "query_type": "routing_evaluation_query", + "classification.mode": "RAG", + "classification.rag_keyword_score": 2, + "classification.web_keyword_score": 0, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "noveum_specific", + "query.keyword_density": 0.3333333333333333, + "query.domain_affinity": "noveum", + "routing.decision": "Routed to RAG based on analysis", + "routing.rationale": "RAG score: 2, Web score: 0, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "Web Search", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.038461538461538464, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: RAG (Confidence: 1.00)" + }, + "events": [], + "links": [] + }, + { + "span_id": "6a101d9d-de83-415a-8fb0-01a9fae5ffb9", + "trace_id": "d5528586-c741-4703-8791-5fccf5c3e123", + "parent_span_id": "7d6564fc-41b2-4d74-acb1-e77b575c0006", + "name": "agent.llm-rag", + "start_time": "2025-10-16 08:49:05.153000000", + "end_time": "2025-10-16 08:49:10.187000000", + "duration_ms": 5034.282, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "rag_agent", + "agent.operation": "llm-rag", + "agent.capabilities": [ + "document_retrieval", + "context_generation", + "response_generation" + ], + "agent.id": "noveum_rag_agent", + "input_query": "What monitoring capabilities does Noveum provide?", + "query_length": 49, + "query_type": "rag_query", + "output_response": "RAG Answer: Noveum provides comprehensive monitoring capabilities for AI applications, including the ability to monitor, trace, and optimize AI agents across various frameworks such as LangChain, CrewAI, AutoGen,...", + "rag.context_length": 2740, + "rag.sources_count": 5, + "rag.answer_length": 552, + "rag.mode": "retrieval_augmented_generation" + }, + "events": [], + "links": [] + }, + { + "span_id": "26e510c0-441a-4f9f-9dce-31180f47d060", + "trace_id": "d5528586-c741-4703-8791-5fccf5c3e123", + "parent_span_id": "6a101d9d-de83-415a-8fb0-01a9fae5ffb9", + "name": "agent.llm_model_execution", + "start_time": "2025-10-16 08:49:05.787000000", + "end_time": "2025-10-16 08:49:09.274000000", + "duration_ms": 3486.699, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What monitoring capabilities does Noveum provide?", + "query_length": 49, + "query_type": "rag_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 3.4866299629211426, + "model.latency_ms": 3486.6299629211426, + "model.start_time": 1760604545.787763, + "model.end_time": 1760604549.274393, + "model.performance_tier": "medium", + "model.prompt_tokens": 821, + "model.completion_tokens": 138, + "model.total_tokens": 959, + "model.tokens_per_second": 275.0506965747916, + "model.estimated_cost": 0.02877, + "model.efficiency_score": 0.5755995828988529, + "model.response_length": 552, + "model.response_quality": "high", + "model.output_response": "Model Response: Noveum provides comprehensive monitoring capabilities for AI applications, including the ability to monitor, trace, and optimize AI agents across various frameworks such as LangChain, CrewAI, AutoGen,...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [] + }, + { + "span_id": "fcd89e38-3d21-468b-860a-ae2de0bbd119", + "trace_id": "d5528586-c741-4703-8791-5fccf5c3e123", + "parent_span_id": "6a101d9d-de83-415a-8fb0-01a9fae5ffb9", + "name": "agent.rag_evaluation_metrics", + "start_time": "2025-10-16 08:49:09.274000000", + "end_time": "2025-10-16 08:49:09.274000000", + "duration_ms": 0.054, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "rag_evaluation_metrics", + "agent.capabilities": [ + "retrieval_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What monitoring capabilities does Noveum provide?", + "query_length": 49, + "query_type": "rag_evaluation_query", + "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en):\nMonitor all your AI Agents\nimprove AI Agents today\nNoveum.ai helps you monitor, trace, and optimize your AI applications.\nNoveum.ai works with any AI framework \u2013 LangChain, CrewAI, AutoGen, custom implementations, or direct LLM calls. One dashboard shows everything.\n...", + "retrieval.context_length": 2740, + "retrieval.sources_count": 5, + "retrieval.context_quality": "high", + "retrieval.effectiveness": 1, + "retrieval.context_utilization": 2.74, + "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en):\nMonitor all your AI Agents\nimprove AI Agents today\nNoveum.ai helps you monitor, trace, and optimize your AI applications.\nNoveum.ai works with any AI framework \u2013 LangChain, CrewAI, AutoGen, custom implementations, or direct LLM calls. One dashboard shows everything.\nMonitor, Evaluate, Improve Your AI Agents\nThe control plane for AI agents.\nMonitor Everything, Miss Nothing\nOur lightweight SDKs capture every trace and span across your AI agent ecosystem\u2014from simple LLM calls to complex multi-agent...\n\nSource 2 (https://noveum.ai/):\nMonitor all your AI Agents\nimprove AI Agents today\nNoveum.ai helps you monitor, trace, and optimize your AI applications.\nNoveum.ai works with any AI framework \u2013 LangChain, CrewAI, AutoGen, custom implementations, or direct LLM calls. One dashboard shows everything.\nMonitor, Evaluate, Improve Your AI Agents\nThe control plane for AI agents.\nMonitor Everything, Miss Nothing\nOur lightweight SDKs capture every trace and span across your AI agent ecosystem\u2014from simple LLM calls to complex multi-agent...\n\nSource 3 (https://noveum.ai):\nMonitor all your AI Agents\nimprove AI Agents today\nNoveum.ai helps you monitor, trace, and optimize your AI applications.\nNoveum.ai works with any AI framework \u2013 LangChain, CrewAI, AutoGen, custom implementations, or direct LLM calls. One dashboard shows everything.\nMonitor, Evaluate, Improve Your AI Agents\nThe control plane for AI agents.\nMonitor Everything, Miss Nothing\nOur lightweight SDKs capture every trace and span across your AI agent ecosystem\u2014from simple LLM calls to complex multi-agent...\n\nSource 4 (https://noveum.ai/en/docs/getting-started/overview):\nNoveum.ai Overview\nComprehensive AI tracing and observability platform for LLM applications, RAG systems, and AI agents\nWelcome to Noveum.ai\u2014the comprehensive tracing and observability platform built specifically for AI applications. Whether you're building LLM-powered chatbots, RAG systems, multi-agent workflows, or any AI-driven application, Noveum provides the insights you need to understand, debug, and optimize your systems.\n\ud83c\udfaf Why AI Applications Need Specialized Observability\nTraditional mo...\n\nSource 5 (https://noveum.ai/docs/getting-started/overview):\nNoveum.ai Overview\nComprehensive AI tracing and observability platform for LLM applications, RAG systems, and AI agents\nWelcome to Noveum.ai\u2014the comprehensive tracing and observability platform built specifically for AI applications. Whether you're building LLM-powered chatbots, RAG systems, multi-agent workflows, or any AI-driven application, Noveum provides the insights you need to understand, debug, and optimize your systems.\n\ud83c\udfaf Why AI Applications Need Specialized Observability\nTraditional mo...\n\nUser Question: What monitoring capabilities does Noveum provide?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", + "prompt.prompt_length": 3285, + "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en):\nMonitor all your AI Agents\nimprove AI Agents today\nNoveum.ai helps you monitor, trace, and optimize your AI applications.\nNoveum.ai works with any AI framework \u2013 LangC...", + "prompt.instruction_following": "rag_optimized", + "response.answer_length": 552, + "response.answer_completeness": "complete", + "response.response_quality": "high", + "response.source_citation": 5, + "response.context_utilization": 0.20145985401459854, + "output_response": "RAG Answer: Noveum provides comprehensive monitoring capabilities for AI applications, including the ability to monitor, trace, and optimize AI agents across various frameworks such as LangChain, CrewAI, AutoGen,...", + "evaluation.retrieval_effectiveness": 1, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 5, + "evaluation.context_relevance": "high", + "evaluation.overall_quality": "high", + "evaluation.ready_for_production": true, + "rag.retrieval_strategy": "semantic_similarity", + "rag.vector_search_results": 5, + "rag.context_synthesis": "multi_source", + "rag.document_coverage": 1, + "rag.information_density": 0.20145985401459854 + }, + "events": [], + "links": [] + } + ], + "created_at": "2025-10-16 08:49:05.152000000", + "updated_at": "2025-10-16 08:49:05.152000000" + }, + { + "trace_id": "8100b869-07c0-4e37-b51e-092c9d20b4a7", + "name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "status": "ok", + "status_message": "", + "start_time": "2025-10-16 08:48:57.574000000", + "end_time": "2025-10-16 08:49:04.149000000", + "duration_ms": 6574.500999999999, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "spans": [ + { + "span_id": "bd5f205f-0329-4811-b883-129d69f12382", + "trace_id": "8100b869-07c0-4e37-b51e-092c9d20b4a7", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-16 08:48:57.574000000", + "end_time": "2025-10-16 08:49:04.149000000", + "duration_ms": 6574.424, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "How does Noveum handle agent tracing?", + "query.length": 37, + "processing.mode": "RAG", + "processing.time_seconds": 6.5743818283081055, + "processing.response_length": 643, + "processing.sources_count": 5, + "output_response": "Final Answer: Noveum.ai handles agent tracing through its specialized Multi-Agent Tracing capabilities, which are designed to observe complex workflows and inter-agent communications within multi-agent systems. The...", + "final_answer_mode": "RAG", + "query_processed.input_query": "How does Noveum handle agent tracing?", + "query_processed.output_response": "Successfully processed query using RAG, generated 643 character response", + "query_processed.mode": "RAG", + "query_processed.processing_time": 6.5743818283081055, + "query_processed.response_length": 643 + }, + "events": [], + "links": [] + }, + { + "span_id": "0801bcfe-8c09-4ea6-a754-5564395d6ded", + "trace_id": "8100b869-07c0-4e37-b51e-092c9d20b4a7", + "parent_span_id": "3c35d86a-0df9-42dc-bfe3-bbe978cf62b5", + "name": "agent.llm-rag", + "start_time": "2025-10-16 08:48:57.575000000", + "end_time": "2025-10-16 08:49:04.149000000", + "duration_ms": 6574.135, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "rag_agent", + "agent.operation": "llm-rag", + "agent.capabilities": [ + "document_retrieval", + "context_generation", + "response_generation" + ], + "agent.id": "noveum_rag_agent", + "input_query": "How does Noveum handle agent tracing?", + "query_length": 37, + "query_type": "rag_query", + "output_response": "RAG Answer: Noveum.ai handles agent tracing through its specialized Multi-Agent Tracing capabilities, which are designed to observe complex workflows and inter-agent communications within multi-agent systems. The...", + "rag.context_length": 2736, + "rag.sources_count": 5, + "rag.answer_length": 643, + "rag.mode": "retrieval_augmented_generation" + }, + "events": [], + "links": [] + }, + { + "span_id": "3c35d86a-0df9-42dc-bfe3-bbe978cf62b5", + "trace_id": "8100b869-07c0-4e37-b51e-092c9d20b4a7", + "parent_span_id": "bd5f205f-0329-4811-b883-129d69f12382", + "name": "agent.query_routing", + "start_time": "2025-10-16 08:48:57.575000000", + "end_time": "2025-10-16 08:49:04.149000000", + "duration_ms": 6574.342, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "How does Noveum handle agent tracing?", + "query_length": 37, + "query_type": "routing_query", + "output_response": "Routed to RAG for query processing", + "router.classification": "RAG", + "router.confidence_score": 1, + "router.rag_keyword_score": 2, + "router.web_keyword_score": 0, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [] + }, + { + "span_id": "b1914316-f545-4bc9-abf4-3494cf42c4a5", + "trace_id": "8100b869-07c0-4e37-b51e-092c9d20b4a7", + "parent_span_id": "3c35d86a-0df9-42dc-bfe3-bbe978cf62b5", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-16 08:48:57.575000000", + "end_time": "2025-10-16 08:48:57.575000000", + "duration_ms": 0.032, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: How does Noveum handle agent tracing?", + "query_length": 37, + "query_type": "routing_evaluation_query", + "classification.mode": "RAG", + "classification.rag_keyword_score": 2, + "classification.web_keyword_score": 0, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "noveum_specific", + "query.keyword_density": 0.3333333333333333, + "query.domain_affinity": "noveum", + "routing.decision": "Routed to RAG based on analysis", + "routing.rationale": "RAG score: 2, Web score: 0, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "Web Search", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.038461538461538464, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: RAG (Confidence: 1.00)" + }, + "events": [], + "links": [] + }, + { + "span_id": "0906814b-6a80-40e1-9f00-5376b809703a", + "trace_id": "8100b869-07c0-4e37-b51e-092c9d20b4a7", + "parent_span_id": "0801bcfe-8c09-4ea6-a754-5564395d6ded", + "name": "agent.llm_model_execution", + "start_time": "2025-10-16 08:48:58.009000000", + "end_time": "2025-10-16 08:49:03.349000000", + "duration_ms": 5340.01, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: How does Noveum handle agent tracing?", + "query_length": 37, + "query_type": "rag_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 5.339954614639282, + "model.latency_ms": 5339.954614639282, + "model.start_time": 1760604538.0093303, + "model.end_time": 1760604543.349285, + "model.performance_tier": "slow", + "model.prompt_tokens": 817, + "model.completion_tokens": 160, + "model.total_tokens": 977, + "model.tokens_per_second": 182.9603565022054, + "model.estimated_cost": 0.02931, + "model.efficiency_score": 0.6581371545547595, + "model.response_length": 643, + "model.response_quality": "high", + "model.output_response": "Model Response: Noveum.ai handles agent tracing through its specialized Multi-Agent Tracing capabilities, which are designed to observe complex workflows and inter-agent communications within multi-agent systems. The...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [] + }, + { + "span_id": "29822edb-c15f-4baa-b6ec-84d85fcc926d", + "trace_id": "8100b869-07c0-4e37-b51e-092c9d20b4a7", + "parent_span_id": "0801bcfe-8c09-4ea6-a754-5564395d6ded", + "name": "agent.rag_evaluation_metrics", + "start_time": "2025-10-16 08:49:03.349000000", + "end_time": "2025-10-16 08:49:03.349000000", + "duration_ms": 0.034999999999999996, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "rag_evaluation_metrics", + "agent.capabilities": [ + "retrieval_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: How does Noveum handle agent tracing?", + "query_length": 37, + "query_type": "rag_evaluation_query", + "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, vector searches, data retrieval, agent reasoning, and more. Without proper observability, debugging ...", + "retrieval.context_length": 2736, + "retrieval.sources_count": 5, + "retrieval.context_quality": "high", + "retrieval.effectiveness": 1, + "retrieval.context_utilization": 2.736, + "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, vector searches, data retrieval, agent reasoning, and more. Without proper observability, debugging becomes a nightmare, optimization is guesswork, and understanding user interactions is nearly impossible.\nNoveum.ai solves this by providing comprehensive tracing and observability specifically designed for AI applications.\nKey benefits of using Noveum.ai:\n- Complete Visibility: Trace...\n\nSource 2 (https://noveum.ai/docs/advanced/multi-agent-tracing):\nMulti-Agent Tracing\nObserve complex agent workflows and inter-agent communications with comprehensive tracing\nMulti-agent systems represent some of the most complex AI applications, involving multiple agents that coordinate, communicate, and collaborate to achieve shared goals. Noveum.ai provides specialized tracing capabilities to help you understand and optimize these intricate workflows.\n\ud83c\udfaf Why Multi-Agent Tracing Matters\nMulti-agent systems introduce unique observability challenges:\n- Complex...\n\nSource 3 (https://noveum.ai/en/docs/advanced/multi-agent-tracing):\nMulti-Agent Tracing\nObserve complex agent workflows and inter-agent communications with comprehensive tracing\nMulti-agent systems represent some of the most complex AI applications, involving multiple agents that coordinate, communicate, and collaborate to achieve shared goals. Noveum.ai provides specialized tracing capabilities to help you understand and optimize these intricate workflows.\n\ud83c\udfaf Why Multi-Agent Tracing Matters\nMulti-agent systems introduce unique observability challenges:\n- Complex...\n\nSource 4 (https://noveum.ai/en/docs/advanced/performance-optimization):\n\ud83d\udd17 Next Steps\n- Multi-Agent Tracing - Observe agent workflows\n- RAG Pipeline Observability - Monitor retrieval and generation systems\n- Custom Instrumentation - Add domain-specific tracing\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nSource 5 (https://noveum.ai/docs/advanced/performance-optimization):\n\ud83d\udd17 Next Steps\n- Multi-Agent Tracing - Observe agent workflows\n- RAG Pipeline Observability - Monitor retrieval and generation systems\n- Custom Instrumentation - Add domain-specific tracing\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nUser Question: How does Noveum handle agent tracing?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", + "prompt.prompt_length": 3269, + "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, ...", + "prompt.instruction_following": "rag_optimized", + "response.answer_length": 643, + "response.answer_completeness": "complete", + "response.response_quality": "high", + "response.source_citation": 5, + "response.context_utilization": 0.23501461988304093, + "output_response": "RAG Answer: Noveum.ai handles agent tracing through its specialized Multi-Agent Tracing capabilities, which are designed to observe complex workflows and inter-agent communications within multi-agent systems. The...", + "evaluation.retrieval_effectiveness": 1, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 5, + "evaluation.context_relevance": "high", + "evaluation.overall_quality": "high", + "evaluation.ready_for_production": true, + "rag.retrieval_strategy": "semantic_similarity", + "rag.vector_search_results": 5, + "rag.context_synthesis": "multi_source", + "rag.document_coverage": 1, + "rag.information_density": 0.23501461988304093 + }, + "events": [], + "links": [] + } + ], + "created_at": "2025-10-16 08:48:57.574000000", + "updated_at": "2025-10-16 08:48:57.574000000" + }, + { + "trace_id": "c28bd31c-d200-4467-91fd-a3b80187c885", + "name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "status": "ok", + "status_message": "", + "start_time": "2025-10-16 08:48:17.977000000", + "end_time": "2025-10-16 08:48:56.570000000", + "duration_ms": 38592.996, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "spans": [ + { + "span_id": "5f8ea007-edef-477c-b8f1-57c2da9e9258", + "trace_id": "c28bd31c-d200-4467-91fd-a3b80187c885", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-16 08:48:17.977000000", + "end_time": "2025-10-16 08:48:56.570000000", + "duration_ms": 38592.912, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What APIs are available in Noveum platform?", + "query.length": 43, + "processing.mode": "RAG", + "processing.time_seconds": 38.59272003173828, + "processing.response_length": 270, + "processing.sources_count": 5, + "output_response": "Final Answer: The provided context does not specify the APIs available in the Noveum platform. For detailed information about the APIs, I recommend checking the official Noveum documentation or the SDK Integration ...", + "final_answer_mode": "RAG", + "query_processed.input_query": "What APIs are available in Noveum platform?", + "query_processed.output_response": "Successfully processed query using RAG, generated 270 character response", + "query_processed.mode": "RAG", + "query_processed.processing_time": 38.59272003173828, + "query_processed.response_length": 270 + }, + "events": [], + "links": [] + }, + { + "span_id": "db385f4d-beef-465b-9831-a65080537519", + "trace_id": "c28bd31c-d200-4467-91fd-a3b80187c885", + "parent_span_id": "5f8ea007-edef-477c-b8f1-57c2da9e9258", + "name": "agent.query_routing", + "start_time": "2025-10-16 08:48:17.977000000", + "end_time": "2025-10-16 08:48:56.570000000", + "duration_ms": 38592.66499999999, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What APIs are available in Noveum platform?", + "query_length": 43, + "query_type": "routing_query", + "output_response": "Routed to RAG for query processing", + "router.classification": "RAG", + "router.confidence_score": 1, + "router.rag_keyword_score": 3, + "router.web_keyword_score": 0, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [] + }, + { + "span_id": "3ccebe1c-1f2a-433b-a36f-453a0b803511", + "trace_id": "c28bd31c-d200-4467-91fd-a3b80187c885", + "parent_span_id": "db385f4d-beef-465b-9831-a65080537519", + "name": "agent.llm-rag", + "start_time": "2025-10-16 08:48:17.978000000", + "end_time": "2025-10-16 08:48:56.570000000", + "duration_ms": 38592.416999999994, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "rag_agent", + "agent.operation": "llm-rag", + "agent.capabilities": [ + "document_retrieval", + "context_generation", + "response_generation" + ], + "agent.id": "noveum_rag_agent", + "input_query": "What APIs are available in Noveum platform?", + "query_length": 43, + "query_type": "rag_query", + "output_response": "RAG Answer: The provided context does not specify the APIs available in the Noveum platform. For detailed information about the APIs, I recommend checking the official Noveum documentation or the SDK Integration ...", + "rag.context_length": 2284, + "rag.sources_count": 5, + "rag.answer_length": 270, + "rag.mode": "retrieval_augmented_generation" + }, + "events": [], + "links": [] + }, + { + "span_id": "6d1bb8e9-315d-433a-8df5-7f5fb2eeee16", + "trace_id": "c28bd31c-d200-4467-91fd-a3b80187c885", + "parent_span_id": "db385f4d-beef-465b-9831-a65080537519", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-16 08:48:17.978000000", + "end_time": "2025-10-16 08:48:17.978000000", + "duration_ms": 0.043000000000000003, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What APIs are available in Noveum platform?", + "query_length": 43, + "query_type": "routing_evaluation_query", + "classification.mode": "RAG", + "classification.rag_keyword_score": 3, + "classification.web_keyword_score": 0, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "noveum_specific", + "query.keyword_density": 0.42857142857142855, + "query.domain_affinity": "noveum", + "routing.decision": "Routed to RAG based on analysis", + "routing.rationale": "RAG score: 3, Web score: 0, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "Web Search", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.057692307692307696, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: RAG (Confidence: 1.00)" + }, + "events": [], + "links": [] + }, + { + "span_id": "2f11df8a-8f45-425d-a0ce-1eec7887b508", + "trace_id": "c28bd31c-d200-4467-91fd-a3b80187c885", + "parent_span_id": "3ccebe1c-1f2a-433b-a36f-453a0b803511", + "name": "agent.llm_model_execution", + "start_time": "2025-10-16 08:48:18.392000000", + "end_time": "2025-10-16 08:48:55.545000000", + "duration_ms": 37152.496999999996, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What APIs are available in Noveum platform?", + "query_length": 43, + "query_type": "rag_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 37.15243363380432, + "model.latency_ms": 37152.43363380432, + "model.start_time": 1760604498.3926067, + "model.end_time": 1760604535.5450404, + "model.performance_tier": "slow", + "model.prompt_tokens": 705, + "model.completion_tokens": 67, + "model.total_tokens": 772, + "model.tokens_per_second": 20.779257897592238, + "model.estimated_cost": 0.02316, + "model.efficiency_score": 0.34974093264248707, + "model.response_length": 270, + "model.response_quality": "high", + "model.output_response": "Model Response: The provided context does not specify the APIs available in the Noveum platform. For detailed information about the APIs, I recommend checking the official Noveum documentation or the SDK Integration ...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [] + }, + { + "span_id": "d13e608c-06e4-47c4-b602-c40ac9560be2", + "trace_id": "c28bd31c-d200-4467-91fd-a3b80187c885", + "parent_span_id": "3ccebe1c-1f2a-433b-a36f-453a0b803511", + "name": "agent.rag_evaluation_metrics", + "start_time": "2025-10-16 08:48:55.545000000", + "end_time": "2025-10-16 08:48:55.545000000", + "duration_ms": 0.031, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "rag_evaluation_metrics", + "agent.capabilities": [ + "retrieval_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What APIs are available in Noveum platform?", + "query_length": 43, + "query_type": "rag_evaluation_query", + "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/docs):\n- \ud83c\udf0d Universal: Works across frameworks, clouds, and deployment models\n\ud83e\udd1d Community & Support\n- \ud83d\udcac Discord Community: Join our Discord\n- \ud83d\udce7 Email Support: [email protected]\n- \ud83d\udc1b Bug Reports: GitHub Issues\n- \ud83d\udcd6 Knowledge Base: Help Center\nReady to get started? Head to ...", + "retrieval.context_length": 2284, + "retrieval.sources_count": 5, + "retrieval.context_quality": "high", + "retrieval.effectiveness": 1, + "retrieval.context_utilization": 2.284, + "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/docs):\n- \ud83c\udf0d Universal: Works across frameworks, clouds, and deployment models\n\ud83e\udd1d Community & Support\n- \ud83d\udcac Discord Community: Join our Discord\n- \ud83d\udce7 Email Support: [email protected]\n- \ud83d\udc1b Bug Reports: GitHub Issues\n- \ud83d\udcd6 Knowledge Base: Help Center\nReady to get started? Head to our SDK Integration Guide to begin tracing your AI applications in under 5 minutes!\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability ...\n\nSource 2 (https://noveum.ai/docs):\n- \ud83c\udf0d Universal: Works across frameworks, clouds, and deployment models\n\ud83e\udd1d Community & Support\n- \ud83d\udcac Discord Community: Join our Discord\n- \ud83d\udce7 Email Support: [email protected]\n- \ud83d\udc1b Bug Reports: GitHub Issues\n- \ud83d\udcd6 Knowledge Base: Help Center\nReady to get started? Head to our SDK Integration Guide to begin tracing your AI applications in under 5 minutes!\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability ...\n\nSource 3 (https://noveum.ai/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nSource 4 (https://noveum.ai/en/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nSource 5 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, vector searches, data retrieval, agent reasoning, and more. Without proper observability, debugging becomes a nightmare, optimization is guesswork, and understanding user interactions is nearly impossible.\nNoveum.ai solves this by providing comprehensive tracing and observability specifically designed for AI applications.\nKey benefits of using Noveum.ai:\n- Complete Visibility: Trace...\n\nUser Question: What APIs are available in Noveum platform?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", + "prompt.prompt_length": 2823, + "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/docs):\n- \ud83c\udf0d Universal: Works across frameworks, clouds, and deployment models\n\ud83e\udd1d Community & Support\n- \ud83d\udcac Discord Community: Join our Discord\n- \ud83d\udce7 Email Support: [email prot...", + "prompt.instruction_following": "rag_optimized", + "response.answer_length": 270, + "response.answer_completeness": "complete", + "response.response_quality": "high", + "response.source_citation": 5, + "response.context_utilization": 0.11821366024518389, + "output_response": "RAG Answer: The provided context does not specify the APIs available in the Noveum platform. For detailed information about the APIs, I recommend checking the official Noveum documentation or the SDK Integration ...", + "evaluation.retrieval_effectiveness": 1, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 5, + "evaluation.context_relevance": "high", + "evaluation.overall_quality": "high", + "evaluation.ready_for_production": true, + "rag.retrieval_strategy": "semantic_similarity", + "rag.vector_search_results": 5, + "rag.context_synthesis": "multi_source", + "rag.document_coverage": 1, + "rag.information_density": 0.11821366024518389 + }, + "events": [], + "links": [] + } + ], + "created_at": "2025-10-16 08:48:17.977000000", + "updated_at": "2025-10-16 08:48:17.977000000" + }, + { + "trace_id": "b90ef3fc-11ea-4230-a59b-e2f2db8f2de0", + "name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "status": "ok", + "status_message": "", + "start_time": "2025-10-16 08:48:08.007000000", + "end_time": "2025-10-16 08:48:16.975000000", + "duration_ms": 8967.264, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "spans": [ + { + "span_id": "3f3babbd-abf5-4f5a-b31c-fa5dcec9c360", + "trace_id": "b90ef3fc-11ea-4230-a59b-e2f2db8f2de0", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-16 08:48:08.007000000", + "end_time": "2025-10-16 08:48:16.975000000", + "duration_ms": 8967.201, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "How do I set up observability with Noveum?", + "query.length": 42, + "processing.mode": "RAG", + "processing.time_seconds": 8.96716833114624, + "processing.response_length": 838, + "processing.sources_count": 5, + "output_response": "Final Answer: To set up observability with Noveum, follow these steps:\n\n1. **Sign Up**: Go to [noveum.ai](https://noveum.ai) and create an account.\n2. **Create a Project**: Once signed in, create a new project to o...", + "final_answer_mode": "RAG", + "query_processed.input_query": "How do I set up observability with Noveum?", + "query_processed.output_response": "Successfully processed query using RAG, generated 838 character response", + "query_processed.mode": "RAG", + "query_processed.processing_time": 8.96716833114624, + "query_processed.response_length": 838 + }, + "events": [], + "links": [] + }, + { + "span_id": "41f6453c-4be6-4b66-8f9b-8e795f7b41b0", + "trace_id": "b90ef3fc-11ea-4230-a59b-e2f2db8f2de0", + "parent_span_id": "b0a76345-49e5-4c1b-915f-cec90ea2b1f7", + "name": "agent.llm-rag", + "start_time": "2025-10-16 08:48:08.007000000", + "end_time": "2025-10-16 08:48:16.974000000", + "duration_ms": 8966.964, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "rag_agent", + "agent.operation": "llm-rag", + "agent.capabilities": [ + "document_retrieval", + "context_generation", + "response_generation" + ], + "agent.id": "noveum_rag_agent", + "input_query": "How do I set up observability with Noveum?", + "query_length": 42, + "query_type": "rag_query", + "output_response": "RAG Answer: To set up observability with Noveum, follow these steps:\n\n1. **Sign Up**: Go to [noveum.ai](https://noveum.ai) and create an account.\n2. **Create a Project**: Once signed in, create a new project to o...", + "rag.context_length": 2870, + "rag.sources_count": 6, + "rag.answer_length": 838, + "rag.mode": "retrieval_augmented_generation" + }, + "events": [], + "links": [] + }, + { + "span_id": "b0a76345-49e5-4c1b-915f-cec90ea2b1f7", + "trace_id": "b90ef3fc-11ea-4230-a59b-e2f2db8f2de0", + "parent_span_id": "3f3babbd-abf5-4f5a-b31c-fa5dcec9c360", + "name": "agent.query_routing", + "start_time": "2025-10-16 08:48:08.007000000", + "end_time": "2025-10-16 08:48:16.974000000", + "duration_ms": 8967.129, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "How do I set up observability with Noveum?", + "query_length": 42, + "query_type": "routing_query", + "output_response": "Routed to RAG for query processing", + "router.classification": "RAG", + "router.confidence_score": 1, + "router.rag_keyword_score": 2, + "router.web_keyword_score": 0, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [] + }, + { + "span_id": "fe3074a7-f4c0-442e-9a5b-e2cc11d93785", + "trace_id": "b90ef3fc-11ea-4230-a59b-e2f2db8f2de0", + "parent_span_id": "b0a76345-49e5-4c1b-915f-cec90ea2b1f7", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-16 08:48:08.007000000", + "end_time": "2025-10-16 08:48:08.007000000", + "duration_ms": 0.032, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: How do I set up observability with Noveum?", + "query_length": 42, + "query_type": "routing_evaluation_query", + "classification.mode": "RAG", + "classification.rag_keyword_score": 2, + "classification.web_keyword_score": 0, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "noveum_specific", + "query.keyword_density": 0.25, + "query.domain_affinity": "noveum", + "routing.decision": "Routed to RAG based on analysis", + "routing.rationale": "RAG score: 2, Web score: 0, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "Web Search", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.038461538461538464, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: RAG (Confidence: 1.00)" + }, + "events": [], + "links": [] + }, + { + "span_id": "e16139ee-048e-4c2f-87f8-90442f41a0ae", + "trace_id": "b90ef3fc-11ea-4230-a59b-e2f2db8f2de0", + "parent_span_id": "41f6453c-4be6-4b66-8f9b-8e795f7b41b0", + "name": "agent.llm_model_execution", + "start_time": "2025-10-16 08:48:08.644000000", + "end_time": "2025-10-16 08:48:15.709000000", + "duration_ms": 7064.898, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: How do I set up observability with Noveum?", + "query_length": 42, + "query_type": "rag_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 7.0648510456085205, + "model.latency_ms": 7064.8510456085205, + "model.start_time": 1760604488.6448367, + "model.end_time": 1760604495.7096877, + "model.performance_tier": "slow", + "model.prompt_tokens": 852, + "model.completion_tokens": 209, + "model.total_tokens": 1061, + "model.tokens_per_second": 150.1800948315128, + "model.estimated_cost": 0.031830000000000004, + "model.efficiency_score": 0.7898209236569275, + "model.response_length": 838, + "model.response_quality": "high", + "model.output_response": "Model Response: To set up observability with Noveum, follow these steps:\n\n1. **Sign Up**: Go to [noveum.ai](https://noveum.ai) and create an account.\n2. **Create a Project**: Once signed in, create a new project to o...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [] + }, + { + "span_id": "59e85f8e-c19b-4794-9a69-7b5bc90208f0", + "trace_id": "b90ef3fc-11ea-4230-a59b-e2f2db8f2de0", + "parent_span_id": "41f6453c-4be6-4b66-8f9b-8e795f7b41b0", + "name": "agent.rag_evaluation_metrics", + "start_time": "2025-10-16 08:48:15.709000000", + "end_time": "2025-10-16 08:48:15.709000000", + "duration_ms": 0.039, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "rag_evaluation_metrics", + "agent.capabilities": [ + "retrieval_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: How do I set up observability with Noveum?", + "query_length": 42, + "query_type": "rag_evaluation_query", + "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\n- Production Ready: Built for scale with intelligent sampling and batching\n- Privacy Focused: You control what data is captured and how it's stored\n- Open Source SDKs: Transparent, auditable, and extensible\nGetting ...", + "retrieval.context_length": 2870, + "retrieval.sources_count": 6, + "retrieval.context_quality": "high", + "retrieval.effectiveness": 1.2, + "retrieval.context_utilization": 2.87, + "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\n- Production Ready: Built for scale with intelligent sampling and batching\n- Privacy Focused: You control what data is captured and how it's stored\n- Open Source SDKs: Transparent, auditable, and extensible\nGetting Started\nReady to add comprehensive observability to your AI applications?\n- Sign up at noveum.ai\n- Create a project and get your API key\n- Install the SDK for your preferred language\n- Add tracing to your AI workflows\n- Explore insights in the Noveum.ai dashboard\nCheck out our integra...\n\nSource 2 (https://noveum.ai/en/docs/getting-started/tracing-concepts):\n- Explore Framework Integrations - Framework-specific guidance\n- Learn Advanced Patterns - Custom instrumentation techniques\n- Master the Dashboard - Analyze your traces effectively\nRemember: Good observability is not about collecting all possible data, but about collecting the right data that helps you understand, debug, and optimize your AI applications.\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to O...\n\nSource 3 (https://noveum.ai/docs/getting-started/tracing-concepts):\n- Explore Framework Integrations - Framework-specific guidance\n- Learn Advanced Patterns - Custom instrumentation techniques\n- Master the Dashboard - Analyze your traces effectively\nRemember: Good observability is not about collecting all possible data, but about collecting the right data that helps you understand, debug, and optimize your AI applications.\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to O...\n\nSource 4 (https://noveum.ai/en/docs/getting-started/overview):\nNoveum.ai Overview\nComprehensive AI tracing and observability platform for LLM applications, RAG systems, and AI agents\nWelcome to Noveum.ai\u2014the comprehensive tracing and observability platform built specifically for AI applications. Whether you're building LLM-powered chatbots, RAG systems, multi-agent workflows, or any AI-driven application, Noveum provides the insights you need to understand, debug, and optimize your systems.\n\ud83c\udfaf Why AI Applications Need Specialized Observability\nTraditional mo...\n\nSource 5 (https://noveum.ai/docs/getting-started/overview):\nNoveum.ai Overview\nComprehensive AI tracing and observability platform for LLM applications, RAG systems, and AI agents\nWelcome to Noveum.ai\u2014the comprehensive tracing and observability platform built specifically for AI applications. Whether you're building LLM-powered chatbots, RAG systems, multi-agent workflows, or any AI-driven application, Noveum provides the insights you need to understand, debug, and optimize your systems.\n\ud83c\udfaf Why AI Applications Need Specialized Observability\nTraditional mo...\n\nUser Question: How do I set up observability with Noveum?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", + "prompt.prompt_length": 3408, + "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\n- Production Ready: Built for scale with intelligent sampling and batching\n- Privacy Focused: You control what data...", + "prompt.instruction_following": "rag_optimized", + "response.answer_length": 838, + "response.answer_completeness": "complete", + "response.response_quality": "high", + "response.source_citation": 6, + "response.context_utilization": 0.29198606271777006, + "output_response": "RAG Answer: To set up observability with Noveum, follow these steps:\n\n1. **Sign Up**: Go to [noveum.ai](https://noveum.ai) and create an account.\n2. **Create a Project**: Once signed in, create a new project to o...", + "evaluation.retrieval_effectiveness": 1.2, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 6, + "evaluation.context_relevance": "high", + "evaluation.overall_quality": "high", + "evaluation.ready_for_production": true, + "rag.retrieval_strategy": "semantic_similarity", + "rag.vector_search_results": 6, + "rag.context_synthesis": "multi_source", + "rag.document_coverage": 1.2, + "rag.information_density": 0.29198606271777006 + }, + "events": [], + "links": [] + } + ], + "created_at": "2025-10-16 08:48:08.007000000", + "updated_at": "2025-10-16 08:48:08.007000000" + }, + { + "trace_id": "b8abebac-a7dc-494c-a1eb-6470181f32d2", + "name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "status": "ok", + "status_message": "", + "start_time": "2025-10-16 08:47:57.667000000", + "end_time": "2025-10-16 08:48:07.005000000", + "duration_ms": 9337.405, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "spans": [ + { + "span_id": "f049ed2b-ff2f-4092-9974-acc66f81bd8d", + "trace_id": "b8abebac-a7dc-494c-a1eb-6470181f32d2", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-16 08:47:57.667000000", + "end_time": "2025-10-16 08:48:07.005000000", + "duration_ms": 9337.298, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What features does Noveum Trace offer?", + "query.length": 38, + "processing.mode": "RAG", + "processing.time_seconds": 9.33726453781128, + "processing.response_length": 1018, + "processing.sources_count": 5, + "output_response": "Final Answer: Noveum Trace offers several key features through its Python SDK (`noveum-trace`) and TypeScript SDK (`@noveum/trace`):\n\n1. **Decorator-Based Tracing**: It provides a decorator-based approach for traci...", + "final_answer_mode": "RAG", + "query_processed.input_query": "What features does Noveum Trace offer?", + "query_processed.output_response": "Successfully processed query using RAG, generated 1018 character response", + "query_processed.mode": "RAG", + "query_processed.processing_time": 9.33726453781128, + "query_processed.response_length": 1018 + }, + "events": [], + "links": [] + }, + { + "span_id": "211f1fe7-4cfb-4c1c-9c72-62ae2825dda3", + "trace_id": "b8abebac-a7dc-494c-a1eb-6470181f32d2", + "parent_span_id": "f049ed2b-ff2f-4092-9974-acc66f81bd8d", + "name": "agent.query_routing", + "start_time": "2025-10-16 08:47:57.668000000", + "end_time": "2025-10-16 08:48:07.005000000", + "duration_ms": 9337.223, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What features does Noveum Trace offer?", + "query_length": 38, + "query_type": "routing_query", + "output_response": "Routed to RAG for query processing", + "router.classification": "RAG", + "router.confidence_score": 1, + "router.rag_keyword_score": 3, + "router.web_keyword_score": 0, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [] + }, + { + "span_id": "976c0c4b-efc1-4b7d-bfad-350c7fd0f71e", + "trace_id": "b8abebac-a7dc-494c-a1eb-6470181f32d2", + "parent_span_id": "211f1fe7-4cfb-4c1c-9c72-62ae2825dda3", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-16 08:47:57.668000000", + "end_time": "2025-10-16 08:47:57.668000000", + "duration_ms": 0.028, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What features does Noveum Trace offer?", + "query_length": 38, + "query_type": "routing_evaluation_query", + "classification.mode": "RAG", + "classification.rag_keyword_score": 3, + "classification.web_keyword_score": 0, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "noveum_specific", + "query.keyword_density": 0.5, + "query.domain_affinity": "noveum", + "routing.decision": "Routed to RAG based on analysis", + "routing.rationale": "RAG score: 3, Web score: 0, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "Web Search", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.057692307692307696, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: RAG (Confidence: 1.00)" + }, + "events": [], + "links": [] + }, + { + "span_id": "d2afaf74-f63b-4141-9fa3-662162f51cc0", + "trace_id": "b8abebac-a7dc-494c-a1eb-6470181f32d2", + "parent_span_id": "211f1fe7-4cfb-4c1c-9c72-62ae2825dda3", + "name": "agent.llm-rag", + "start_time": "2025-10-16 08:47:57.668000000", + "end_time": "2025-10-16 08:48:07.005000000", + "duration_ms": 9337.056, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "rag_agent", + "agent.operation": "llm-rag", + "agent.capabilities": [ + "document_retrieval", + "context_generation", + "response_generation" + ], + "agent.id": "noveum_rag_agent", + "input_query": "What features does Noveum Trace offer?", + "query_length": 38, + "query_type": "rag_query", + "output_response": "RAG Answer: Noveum Trace offers several key features through its Python SDK (`noveum-trace`) and TypeScript SDK (`@noveum/trace`):\n\n1. **Decorator-Based Tracing**: It provides a decorator-based approach for traci...", + "rag.context_length": 2787, + "rag.sources_count": 5, + "rag.answer_length": 1018, + "rag.mode": "retrieval_augmented_generation" + }, + "events": [], + "links": [] + }, + { + "span_id": "09811a73-405d-4c0d-ba5d-0472b3708062", + "trace_id": "b8abebac-a7dc-494c-a1eb-6470181f32d2", + "parent_span_id": "d2afaf74-f63b-4141-9fa3-662162f51cc0", + "name": "agent.llm_model_execution", + "start_time": "2025-10-16 08:47:58.659000000", + "end_time": "2025-10-16 08:48:05.878000000", + "duration_ms": 7218.618, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What features does Noveum Trace offer?", + "query_length": 38, + "query_type": "rag_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 7.218573331832886, + "model.latency_ms": 7218.573331832886, + "model.start_time": 1760604478.659586, + "model.end_time": 1760604485.8781593, + "model.performance_tier": "slow", + "model.prompt_tokens": 830, + "model.completion_tokens": 254, + "model.total_tokens": 1084, + "model.tokens_per_second": 150.16817730724068, + "model.estimated_cost": 0.03252, + "model.efficiency_score": 0.9391143911439115, + "model.response_length": 1018, + "model.response_quality": "high", + "model.output_response": "Model Response: Noveum Trace offers several key features through its Python SDK (`noveum-trace`) and TypeScript SDK (`@noveum/trace`):\n\n1. **Decorator-Based Tracing**: It provides a decorator-based approach for traci...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [] + }, + { + "span_id": "015688fe-d287-48e8-ac59-63781ffd6f89", + "trace_id": "b8abebac-a7dc-494c-a1eb-6470181f32d2", + "parent_span_id": "d2afaf74-f63b-4141-9fa3-662162f51cc0", + "name": "agent.rag_evaluation_metrics", + "start_time": "2025-10-16 08:48:05.878000000", + "end_time": "2025-10-16 08:48:05.878000000", + "duration_ms": 0.024, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "rag_evaluation_metrics", + "agent.capabilities": [ + "retrieval_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What features does Noveum Trace offer?", + "query_length": 38, + "query_type": "rag_evaluation_query", + "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\n\ud83d\udcd8 TypeScript SDK (\n@noveum/trace\n)- Framework ...", + "retrieval.context_length": 2787, + "retrieval.sources_count": 5, + "retrieval.context_quality": "high", + "retrieval.effectiveness": 1, + "retrieval.context_utilization": 2.787, + "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\n\ud83d\udcd8 TypeScript SDK (\n@noveum/trace\n)- Framework integrations for Next.js, Express.js, and Hono\n- TypeScript-first design with full type safety\n- Universal compatibility (Node.js, Edge Runtime, browsers)\n-\n\ud83d\udcca Noveum Platform\n- Real-time dashboard for analyzing traces and performance\n- Ad...\n\nSource 2 (https://noveum.ai/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\n\ud83d\udcd8 TypeScript SDK (\n@noveum/trace\n)- Framework integrations for Next.js, Express.js, and Hono\n- TypeScript-first design with full type safety\n- Universal compatibility (Node.js, Edge Runtime, browsers)\n-\n\ud83d\udcca Noveum Platform\n- Real-time dashboard for analyzing traces and performance\n- Ad...\n\nSource 3 (https://noveum.ai/en/docs/getting-started/sdk-integration):\n- \ud83d\udc65 Collaborate with your team on insights\n\ud83d\udd12 Security & Privacy\n- \ud83d\udd10 Encryption: All data encrypted in transit and at rest\n- \ud83c\udf9b\ufe0f Configurable Capture: Control what data is collected\n- \ud83c\udfe0 Data Residency: Choose your data storage region\n- \u23f0 Retention Control: Set custom data retention policies\nNext Steps\n- Tracing Concepts - Learn about traces, spans, and observability best practices\n- Framework Integrations - Deep dive into specific framework setups\n- Multi-Agent Tracing - Observe complex agent work...\n\nSource 4 (https://noveum.ai/docs/getting-started/sdk-integration):\n- \ud83d\udc65 Collaborate with your team on insights\n\ud83d\udd12 Security & Privacy\n- \ud83d\udd10 Encryption: All data encrypted in transit and at rest\n- \ud83c\udf9b\ufe0f Configurable Capture: Control what data is collected\n- \ud83c\udfe0 Data Residency: Choose your data storage region\n- \u23f0 Retention Control: Set custom data retention policies\nNext Steps\n- Tracing Concepts - Learn about traces, spans, and observability best practices\n- Framework Integrations - Deep dive into specific framework setups\n- Multi-Agent Tracing - Observe complex agent work...\n\nSource 5 (https://noveum.ai/docs/platform/dashboard):\n- Bulk Export: Download multiple traces simultaneously\n- Comparative Analysis: Compare performance across multiple traces\nReal-Time Updates\n- Live Refresh: Automatic updates as new traces arrive\n- Connection Monitoring: Real-time status of your trace ingestion pipeline\n- Performance Indicators: Live metrics for system health monitoring\nReady to dive deeper? Explore Projects & Environments to organize your AI applications, or check out Team Collaboration to share insights with your team.\nGet Earl...\n\nUser Question: What features does Noveum Trace offer?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", + "prompt.prompt_length": 3321, + "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frame...", + "prompt.instruction_following": "rag_optimized", + "response.answer_length": 1018, + "response.answer_completeness": "complete", + "response.response_quality": "high", + "response.source_citation": 5, + "response.context_utilization": 0.36526731252242556, + "output_response": "RAG Answer: Noveum Trace offers several key features through its Python SDK (`noveum-trace`) and TypeScript SDK (`@noveum/trace`):\n\n1. **Decorator-Based Tracing**: It provides a decorator-based approach for traci...", + "evaluation.retrieval_effectiveness": 1, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 5, + "evaluation.context_relevance": "high", + "evaluation.overall_quality": "high", + "evaluation.ready_for_production": true, + "rag.retrieval_strategy": "semantic_similarity", + "rag.vector_search_results": 5, + "rag.context_synthesis": "multi_source", + "rag.document_coverage": 1, + "rag.information_density": 0.36526731252242556 + }, + "events": [], + "links": [] + } + ], + "created_at": "2025-10-16 08:47:57.667000000", + "updated_at": "2025-10-16 08:47:57.667000000" + }, + { + "trace_id": "be9ccbaf-ccb4-4ebd-a380-4247f4d01868", + "name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "status": "ok", + "status_message": "", + "start_time": "2025-10-16 08:47:52.037000000", + "end_time": "2025-10-16 08:47:56.662000000", + "duration_ms": 4625.492, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "spans": [ + { + "span_id": "46ee4d00-f914-41d8-9f91-c2db85f1c613", + "trace_id": "be9ccbaf-ccb4-4ebd-a380-4247f4d01868", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-16 08:47:52.037000000", + "end_time": "2025-10-16 08:47:56.662000000", + "duration_ms": 4625.395, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What are Noveum's pricing plans?", + "query.length": 32, + "processing.mode": "RAG", + "processing.time_seconds": 4.6253437995910645, + "processing.response_length": 471, + "processing.sources_count": 5, + "output_response": "Final Answer: The provided context does not contain specific information about Noveum's pricing plans. It mentions that all users will have access to the Observability suite for free, and early users will receive f...", + "final_answer_mode": "RAG", + "query_processed.input_query": "What are Noveum's pricing plans?", + "query_processed.output_response": "Successfully processed query using RAG, generated 471 character response", + "query_processed.mode": "RAG", + "query_processed.processing_time": 4.6253437995910645, + "query_processed.response_length": 471 + }, + "events": [], + "links": [] + }, + { + "span_id": "6d15369a-ae35-45c7-84dd-bdf484f73f2d", + "trace_id": "be9ccbaf-ccb4-4ebd-a380-4247f4d01868", + "parent_span_id": "46ee4d00-f914-41d8-9f91-c2db85f1c613", + "name": "agent.query_routing", + "start_time": "2025-10-16 08:47:52.037000000", + "end_time": "2025-10-16 08:47:56.662000000", + "duration_ms": 4625.286, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What are Noveum's pricing plans?", + "query_length": 32, + "query_type": "routing_query", + "output_response": "Routed to RAG for query processing", + "router.classification": "RAG", + "router.confidence_score": 1, + "router.rag_keyword_score": 3, + "router.web_keyword_score": 0, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [] + }, + { + "span_id": "bf201676-a3a9-433a-a8c6-d394b7aead3a", + "trace_id": "be9ccbaf-ccb4-4ebd-a380-4247f4d01868", + "parent_span_id": "6d15369a-ae35-45c7-84dd-bdf484f73f2d", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-16 08:47:52.037000000", + "end_time": "2025-10-16 08:47:52.037000000", + "duration_ms": 0.048, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What are Noveum's pricing plans?", + "query_length": 32, + "query_type": "routing_evaluation_query", + "classification.mode": "RAG", + "classification.rag_keyword_score": 3, + "classification.web_keyword_score": 0, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "noveum_specific", + "query.keyword_density": 0.6, + "query.domain_affinity": "noveum", + "routing.decision": "Routed to RAG based on analysis", + "routing.rationale": "RAG score: 3, Web score: 0, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "Web Search", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.057692307692307696, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: RAG (Confidence: 1.00)" + }, + "events": [], + "links": [] + }, + { + "span_id": "f8346f60-131b-4007-ba3e-7b0ff10cdfde", + "trace_id": "be9ccbaf-ccb4-4ebd-a380-4247f4d01868", + "parent_span_id": "6d15369a-ae35-45c7-84dd-bdf484f73f2d", + "name": "agent.llm-rag", + "start_time": "2025-10-16 08:47:52.037000000", + "end_time": "2025-10-16 08:47:56.662000000", + "duration_ms": 4624.998, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "rag_agent", + "agent.operation": "llm-rag", + "agent.capabilities": [ + "document_retrieval", + "context_generation", + "response_generation" + ], + "agent.id": "noveum_rag_agent", + "input_query": "What are Noveum's pricing plans?", + "query_length": 32, + "query_type": "rag_query", + "output_response": "RAG Answer: The provided context does not contain specific information about Noveum's pricing plans. It mentions that all users will have access to the Observability suite for free, and early users will receive f...", + "rag.context_length": 2105, + "rag.sources_count": 5, + "rag.answer_length": 471, + "rag.mode": "retrieval_augmented_generation" + }, + "events": [], + "links": [] + }, + { + "span_id": "0186c28c-7596-4279-a4eb-7733cd665609", + "trace_id": "be9ccbaf-ccb4-4ebd-a380-4247f4d01868", + "parent_span_id": "f8346f60-131b-4007-ba3e-7b0ff10cdfde", + "name": "agent.llm_model_execution", + "start_time": "2025-10-16 08:47:52.579000000", + "end_time": "2025-10-16 08:47:55.842000000", + "duration_ms": 3262.5829999999996, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What are Noveum's pricing plans?", + "query_length": 32, + "query_type": "rag_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 3.2625203132629395, + "model.latency_ms": 3262.5203132629395, + "model.start_time": 1760604472.580008, + "model.end_time": 1760604475.8425283, + "model.performance_tier": "medium", + "model.prompt_tokens": 658, + "model.completion_tokens": 117, + "model.total_tokens": 775, + "model.tokens_per_second": 237.54641368804243, + "model.estimated_cost": 0.02325, + "model.efficiency_score": 0.607741935483871, + "model.response_length": 471, + "model.response_quality": "high", + "model.output_response": "Model Response: The provided context does not contain specific information about Noveum's pricing plans. It mentions that all users will have access to the Observability suite for free, and early users will receive f...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [] + }, + { + "span_id": "3bdf8d5b-c10d-44f3-8514-8facc7f63db8", + "trace_id": "be9ccbaf-ccb4-4ebd-a380-4247f4d01868", + "parent_span_id": "f8346f60-131b-4007-ba3e-7b0ff10cdfde", + "name": "agent.rag_evaluation_metrics", + "start_time": "2025-10-16 08:47:55.842000000", + "end_time": "2025-10-16 08:47:55.842000000", + "duration_ms": 0.032, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "rag_evaluation_metrics", + "agent.capabilities": [ + "retrieval_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What are Noveum's pricing plans?", + "query_length": 32, + "query_type": "rag_evaluation_query", + "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....", + "retrieval.context_length": 2105, + "retrieval.sources_count": 5, + "retrieval.context_quality": "high", + "retrieval.effectiveness": 1, + "retrieval.context_utilization": 2.105, + "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nSource 2 (https://noveum.ai/en/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nSource 3 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nThe AI landscape is complex and moving fast. With Noveum.ai, you'll have the insights you need to build, debug, and optimize AI applications with confidence.\nReady to see what's really happening inside your AI applications? Try Noveum.ai today and transform how you build and monitor AI systems.\nLet's build more reliable, observable AI\u2014together.\nGet Early Access to Noveum.ai Platform\nJoin the select group of AI teams optimizing their models with our data-driven platform. We're onboarding users in...\n\nSource 4 (https://noveum.ai/en/blog/from-logs-to-intelligent-choices-inside-noveum-ais-evaluation-process):\nGet Early Access to Noveum.ai Platform\nJoin the select group of AI teams optimizing their models with our data-driven platform. We're onboarding users in limited batches to ensure a premium experience....\n\nSource 5 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, vector searches, data retrieval, agent reasoning, and more. Without proper observability, debugging becomes a nightmare, optimization is guesswork, and understanding user interactions is nearly impossible.\nNoveum.ai solves this by providing comprehensive tracing and observability specifically designed for AI applications.\nKey benefits of using Noveum.ai:\n- Complete Visibility: Trace...\n\nUser Question: What are Noveum's pricing plans?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", + "prompt.prompt_length": 2633, + "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to ...", + "prompt.instruction_following": "rag_optimized", + "response.answer_length": 471, + "response.answer_completeness": "complete", + "response.response_quality": "high", + "response.source_citation": 5, + "response.context_utilization": 0.22375296912114015, + "output_response": "RAG Answer: The provided context does not contain specific information about Noveum's pricing plans. It mentions that all users will have access to the Observability suite for free, and early users will receive f...", + "evaluation.retrieval_effectiveness": 1, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 5, + "evaluation.context_relevance": "high", + "evaluation.overall_quality": "high", + "evaluation.ready_for_production": true, + "rag.retrieval_strategy": "semantic_similarity", + "rag.vector_search_results": 5, + "rag.context_synthesis": "multi_source", + "rag.document_coverage": 1, + "rag.information_density": 0.22375296912114015 + }, + "events": [], + "links": [] + } + ], + "created_at": "2025-10-16 08:47:52.037000000", + "updated_at": "2025-10-16 08:47:52.037000000" + }, + { + "trace_id": "8d33a529-d9b1-4208-b528-762222d5ef8c", + "name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "status": "ok", + "status_message": "", + "start_time": "2025-10-16 08:47:42.103000000", + "end_time": "2025-10-16 08:47:51.032000000", + "duration_ms": 8928.948, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "spans": [ + { + "span_id": "770ce93d-07ae-4b9f-ae15-c6e450634b9d", + "trace_id": "8d33a529-d9b1-4208-b528-762222d5ef8c", + "parent_span_id": "9977d5d6-a7ff-469e-bec7-5e4dfc38c149", + "name": "agent.llm-rag", + "start_time": "2025-10-16 08:47:42.103000000", + "end_time": "2025-10-16 08:47:51.032000000", + "duration_ms": 8928.644999999999, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "rag_agent", + "agent.operation": "llm-rag", + "agent.capabilities": [ + "document_retrieval", + "context_generation", + "response_generation" + ], + "agent.id": "noveum_rag_agent", + "input_query": "How do I integrate Noveum Trace in my application?", + "query_length": 50, + "query_type": "rag_query", + "output_response": "RAG Answer: To integrate Noveum Trace into your application, you can use either the Python SDK (`noveum-trace`) or the TypeScript SDK (`@noveum/trace`). Here\u2019s a quick start guide based on the documentation:\n\n1. ...", + "rag.context_length": 2803, + "rag.sources_count": 5, + "rag.answer_length": 1134, + "rag.mode": "retrieval_augmented_generation" + }, + "events": [], + "links": [] + }, + { + "span_id": "9977d5d6-a7ff-469e-bec7-5e4dfc38c149", + "trace_id": "8d33a529-d9b1-4208-b528-762222d5ef8c", + "parent_span_id": "f3a8fa53-5641-4a23-b4d3-762577a89151", + "name": "agent.query_routing", + "start_time": "2025-10-16 08:47:42.103000000", + "end_time": "2025-10-16 08:47:51.032000000", + "duration_ms": 8928.793000000001, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "How do I integrate Noveum Trace in my application?", + "query_length": 50, + "query_type": "routing_query", + "output_response": "Routed to RAG for query processing", + "router.classification": "RAG", + "router.confidence_score": 1, + "router.rag_keyword_score": 2, + "router.web_keyword_score": 0, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [] + }, + { + "span_id": "c678e222-ef74-48ee-8628-16461ab0f48f", + "trace_id": "8d33a529-d9b1-4208-b528-762222d5ef8c", + "parent_span_id": "9977d5d6-a7ff-469e-bec7-5e4dfc38c149", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-16 08:47:42.103000000", + "end_time": "2025-10-16 08:47:42.103000000", + "duration_ms": 0.027, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: How do I integrate Noveum Trace in my application?", + "query_length": 50, + "query_type": "routing_evaluation_query", + "classification.mode": "RAG", + "classification.rag_keyword_score": 2, + "classification.web_keyword_score": 0, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "noveum_specific", + "query.keyword_density": 0.2222222222222222, + "query.domain_affinity": "noveum", + "routing.decision": "Routed to RAG based on analysis", + "routing.rationale": "RAG score: 2, Web score: 0, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "Web Search", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.038461538461538464, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: RAG (Confidence: 1.00)" + }, + "events": [], + "links": [] + }, + { + "span_id": "f3a8fa53-5641-4a23-b4d3-762577a89151", + "trace_id": "8d33a529-d9b1-4208-b528-762222d5ef8c", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-16 08:47:42.103000000", + "end_time": "2025-10-16 08:47:51.032000000", + "duration_ms": 8928.868, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "How do I integrate Noveum Trace in my application?", + "query.length": 50, + "processing.mode": "RAG", + "processing.time_seconds": 8.928829431533813, + "processing.response_length": 1134, + "processing.sources_count": 5, + "output_response": "Final Answer: To integrate Noveum Trace into your application, you can use either the Python SDK (`noveum-trace`) or the TypeScript SDK (`@noveum/trace`). Here\u2019s a quick start guide based on the documentation:\n\n1. ...", + "final_answer_mode": "RAG", + "query_processed.input_query": "How do I integrate Noveum Trace in my application?", + "query_processed.output_response": "Successfully processed query using RAG, generated 1134 character response", + "query_processed.mode": "RAG", + "query_processed.processing_time": 8.928829431533813, + "query_processed.response_length": 1134 + }, + "events": [], + "links": [] + }, + { + "span_id": "f459b71f-1d43-41b0-b9bf-d94a908b7afa", + "trace_id": "8d33a529-d9b1-4208-b528-762222d5ef8c", + "parent_span_id": "770ce93d-07ae-4b9f-ae15-c6e450634b9d", + "name": "agent.llm_model_execution", + "start_time": "2025-10-16 08:47:43.044000000", + "end_time": "2025-10-16 08:47:50.008000000", + "duration_ms": 6963.959, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: How do I integrate Noveum Trace in my application?", + "query_length": 50, + "query_type": "rag_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 6.9639081954956055, + "model.latency_ms": 6963.9081954956055, + "model.start_time": 1760604463.044064, + "model.end_time": 1760604470.0079722, + "model.performance_tier": "slow", + "model.prompt_tokens": 837, + "model.completion_tokens": 283, + "model.total_tokens": 1120, + "model.tokens_per_second": 160.8292310235276, + "model.estimated_cost": 0.0336, + "model.efficiency_score": 1.0125, + "model.response_length": 1134, + "model.response_quality": "high", + "model.output_response": "Model Response: To integrate Noveum Trace into your application, you can use either the Python SDK (`noveum-trace`) or the TypeScript SDK (`@noveum/trace`). Here\u2019s a quick start guide based on the documentation:\n\n1. ...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [] + }, + { + "span_id": "b8627171-b081-4b08-b900-c24375b73c3f", + "trace_id": "8d33a529-d9b1-4208-b528-762222d5ef8c", + "parent_span_id": "770ce93d-07ae-4b9f-ae15-c6e450634b9d", + "name": "agent.rag_evaluation_metrics", + "start_time": "2025-10-16 08:47:50.008000000", + "end_time": "2025-10-16 08:47:50.008000000", + "duration_ms": 0.041, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "rag_evaluation_metrics", + "agent.capabilities": [ + "retrieval_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: How do I integrate Noveum Trace in my application?", + "query_length": 50, + "query_type": "rag_evaluation_query", + "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\n\ud83d\udcd8 TypeScript SDK (\n@noveum/trace\n)- Framework ...", + "retrieval.context_length": 2803, + "retrieval.sources_count": 5, + "retrieval.context_quality": "high", + "retrieval.effectiveness": 1, + "retrieval.context_utilization": 2.803, + "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\n\ud83d\udcd8 TypeScript SDK (\n@noveum/trace\n)- Framework integrations for Next.js, Express.js, and Hono\n- TypeScript-first design with full type safety\n- Universal compatibility (Node.js, Edge Runtime, browsers)\n-\n\ud83d\udcca Noveum Platform\n- Real-time dashboard for analyzing traces and performance\n- Ad...\n\nSource 2 (https://noveum.ai/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frameworks\n- Context propagation across async operations\n-\n\ud83d\udcd8 TypeScript SDK (\n@noveum/trace\n)- Framework integrations for Next.js, Express.js, and Hono\n- TypeScript-first design with full type safety\n- Universal compatibility (Node.js, Edge Runtime, browsers)\n-\n\ud83d\udcca Noveum Platform\n- Real-time dashboard for analyzing traces and performance\n- Ad...\n\nSource 3 (https://noveum.ai/docs/getting-started/sdk-integration):\nSDK Integration Guide\nIntegrate Noveum.ai tracing into your AI applications with Python or TypeScript SDKs\nThe Noveum.ai SDKs provide comprehensive tracing and observability for your AI applications with minimal code changes. Whether you're building LLM applications, RAG systems, or multi-agent workflows, our SDKs automatically capture essential metrics and traces.\n\ud83d\ude80 Quick Start\n1. Create Your Account & Get API Key\n- Sign up at noveum.ai\n- Create a project in your dashboard\n- Generate an API key...\n\nSource 4 (https://noveum.ai/en/docs/getting-started/sdk-integration):\nSDK Integration Guide\nIntegrate Noveum.ai tracing into your AI applications with Python or TypeScript SDKs\nThe Noveum.ai SDKs provide comprehensive tracing and observability for your AI applications with minimal code changes. Whether you're building LLM applications, RAG systems, or multi-agent workflows, our SDKs automatically capture essential metrics and traces.\n\ud83d\ude80 Quick Start\n1. Create Your Account & Get API Key\n- Sign up at noveum.ai\n- Create a project in your dashboard\n- Generate an API key...\n\nSource 5 (https://noveum.ai/en/docs/getting-started/sdk-integration):\n\ud83d\udd27 Framework Integrations\nNext.js Integration\nExpress.js Integration\nFastAPI Integration (Python)\n\ud83d\udcca Advanced Features\nCustom Attributes & Events\nSampling Configuration\n\ud83d\udd17 What's Captured Automatically\n- \ud83d\udcca Performance Metrics: Latency, throughput, error rates\n- \ud83d\udcb0 Cost Tracking: Token usage, API costs across providers\n- \ud83d\udd0d Request/Response: Configurable capture of inputs/outputs\n- \ud83c\udff7\ufe0f Metadata: Model names, parameters, user context\n- \ud83c\udf0a Context Flow: Trace relationships across services\n- \ud83d\udc1b Error Detail...\n\nUser Question: How do I integrate Noveum Trace in my application?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", + "prompt.prompt_length": 3349, + "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/docs):\nCore Components\n-\n\ud83d\udc0d Python SDK (\nnoveum-trace\n)- Decorator-based tracing for LLM calls, agents, and RAG pipelines\n- Automatic instrumentation for popular AI frame...", + "prompt.instruction_following": "rag_optimized", + "response.answer_length": 1134, + "response.answer_completeness": "complete", + "response.response_quality": "high", + "response.source_citation": 5, + "response.context_utilization": 0.4045665358544417, + "output_response": "RAG Answer: To integrate Noveum Trace into your application, you can use either the Python SDK (`noveum-trace`) or the TypeScript SDK (`@noveum/trace`). Here\u2019s a quick start guide based on the documentation:\n\n1. ...", + "evaluation.retrieval_effectiveness": 1, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 5, + "evaluation.context_relevance": "high", + "evaluation.overall_quality": "high", + "evaluation.ready_for_production": true, + "rag.retrieval_strategy": "semantic_similarity", + "rag.vector_search_results": 5, + "rag.context_synthesis": "multi_source", + "rag.document_coverage": 1, + "rag.information_density": 0.4045665358544417 + }, + "events": [], + "links": [] + } + ], + "created_at": "2025-10-16 08:47:42.103000000", + "updated_at": "2025-10-16 08:47:42.103000000" + }, + { + "trace_id": "06192a09-b6bb-4548-8a94-e123d6caaffe", + "name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "status": "ok", + "status_message": "", + "start_time": "2025-10-16 08:47:03.776000000", + "end_time": "2025-10-16 08:47:41.100000000", + "duration_ms": 37324.601, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "spans": [ + { + "span_id": "0b3d6893-776e-4118-b8b5-b1c5ea51def1", + "trace_id": "06192a09-b6bb-4548-8a94-e123d6caaffe", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-16 08:47:03.776000000", + "end_time": "2025-10-16 08:47:41.100000000", + "duration_ms": 37324.516, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What is Noveum and what does it do?", + "query.length": 35, + "processing.mode": "RAG", + "processing.time_seconds": 37.324477195739746, + "processing.response_length": 648, + "processing.sources_count": 5, + "output_response": "Final Answer: Noveum.ai is a comprehensive tracing and observability platform specifically designed for AI applications, including those powered by large language models (LLMs), retrieval-augmented generation (RAG)...", + "final_answer_mode": "RAG", + "query_processed.input_query": "What is Noveum and what does it do?", + "query_processed.output_response": "Successfully processed query using RAG, generated 648 character response", + "query_processed.mode": "RAG", + "query_processed.processing_time": 37.324477195739746, + "query_processed.response_length": 648 + }, + "events": [], + "links": [] + }, + { + "span_id": "137ad05b-bada-42be-abe5-be0c277f6d0e", + "trace_id": "06192a09-b6bb-4548-8a94-e123d6caaffe", + "parent_span_id": "c3664d20-1708-45f5-a051-b626b0ec77cd", + "name": "agent.llm-rag", + "start_time": "2025-10-16 08:47:03.776000000", + "end_time": "2025-10-16 08:47:41.100000000", + "duration_ms": 37324.042, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "rag_agent", + "agent.operation": "llm-rag", + "agent.capabilities": [ + "document_retrieval", + "context_generation", + "response_generation" + ], + "agent.id": "noveum_rag_agent", + "input_query": "What is Noveum and what does it do?", + "query_length": 35, + "query_type": "rag_query", + "output_response": "RAG Answer: Noveum.ai is a comprehensive tracing and observability platform specifically designed for AI applications, including those powered by large language models (LLMs), retrieval-augmented generation (RAG)...", + "rag.context_length": 2334, + "rag.sources_count": 5, + "rag.answer_length": 648, + "rag.mode": "retrieval_augmented_generation" + }, + "events": [], + "links": [] + }, + { + "span_id": "c3664d20-1708-45f5-a051-b626b0ec77cd", + "trace_id": "06192a09-b6bb-4548-8a94-e123d6caaffe", + "parent_span_id": "0b3d6893-776e-4118-b8b5-b1c5ea51def1", + "name": "agent.query_routing", + "start_time": "2025-10-16 08:47:03.776000000", + "end_time": "2025-10-16 08:47:41.100000000", + "duration_ms": 37324.382999999994, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What is Noveum and what does it do?", + "query_length": 35, + "query_type": "routing_query", + "output_response": "Routed to RAG for query processing", + "router.classification": "RAG", + "router.confidence_score": 1, + "router.rag_keyword_score": 1, + "router.web_keyword_score": 0, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [] + }, + { + "span_id": "ce9a94bf-1a72-4599-8561-11e66802b92d", + "trace_id": "06192a09-b6bb-4548-8a94-e123d6caaffe", + "parent_span_id": "c3664d20-1708-45f5-a051-b626b0ec77cd", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-16 08:47:03.776000000", + "end_time": "2025-10-16 08:47:03.776000000", + "duration_ms": 0.067, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What is Noveum and what does it do?", + "query_length": 35, + "query_type": "routing_evaluation_query", + "classification.mode": "RAG", + "classification.rag_keyword_score": 1, + "classification.web_keyword_score": 0, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "noveum_specific", + "query.keyword_density": 0.125, + "query.domain_affinity": "noveum", + "routing.decision": "Routed to RAG based on analysis", + "routing.rationale": "RAG score: 1, Web score: 0, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "Web Search", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.019230769230769232, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: RAG (Confidence: 1.00)" + }, + "events": [], + "links": [] + }, + { + "span_id": "38414416-907a-4188-95fe-c0a5f900f72c", + "trace_id": "06192a09-b6bb-4548-8a94-e123d6caaffe", + "parent_span_id": "137ad05b-bada-42be-abe5-be0c277f6d0e", + "name": "agent.llm_model_execution", + "start_time": "2025-10-16 08:47:10.392000000", + "end_time": "2025-10-16 08:47:40.392000000", + "duration_ms": 30000.104, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What is Noveum and what does it do?", + "query_length": 35, + "query_type": "rag_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 30.000043392181396, + "model.latency_ms": 30000.043392181396, + "model.start_time": 1760604430.3921378, + "model.end_time": 1760604460.3921812, + "model.performance_tier": "slow", + "model.prompt_tokens": 716, + "model.completion_tokens": 162, + "model.total_tokens": 878, + "model.tokens_per_second": 29.266624335244266, + "model.estimated_cost": 0.026340000000000002, + "model.efficiency_score": 0.7380410022779044, + "model.response_length": 648, + "model.response_quality": "high", + "model.output_response": "Model Response: Noveum.ai is a comprehensive tracing and observability platform specifically designed for AI applications, including those powered by large language models (LLMs), retrieval-augmented generation (RAG)...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [] + }, + { + "span_id": "67e23649-b00b-4eb8-9a69-865e80f833b3", + "trace_id": "06192a09-b6bb-4548-8a94-e123d6caaffe", + "parent_span_id": "137ad05b-bada-42be-abe5-be0c277f6d0e", + "name": "agent.rag_evaluation_metrics", + "start_time": "2025-10-16 08:47:40.392000000", + "end_time": "2025-10-16 08:47:40.392000000", + "duration_ms": 0.029, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "rag_evaluation_metrics", + "agent.capabilities": [ + "retrieval_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What is Noveum and what does it do?", + "query_length": 35, + "query_type": "rag_evaluation_query", + "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, vector searches, data retrieval, agent reasoning, and more. Without proper observability, debugging ...", + "retrieval.context_length": 2334, + "retrieval.sources_count": 5, + "retrieval.context_quality": "high", + "retrieval.effectiveness": 1, + "retrieval.context_utilization": 2.334, + "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, vector searches, data retrieval, agent reasoning, and more. Without proper observability, debugging becomes a nightmare, optimization is guesswork, and understanding user interactions is nearly impossible.\nNoveum.ai solves this by providing comprehensive tracing and observability specifically designed for AI applications.\nKey benefits of using Noveum.ai:\n- Complete Visibility: Trace...\n\nSource 2 (https://noveum.ai/en/docs/getting-started/overview):\nNoveum.ai Overview\nComprehensive AI tracing and observability platform for LLM applications, RAG systems, and AI agents\nWelcome to Noveum.ai\u2014the comprehensive tracing and observability platform built specifically for AI applications. Whether you're building LLM-powered chatbots, RAG systems, multi-agent workflows, or any AI-driven application, Noveum provides the insights you need to understand, debug, and optimize your systems.\n\ud83c\udfaf Why AI Applications Need Specialized Observability\nTraditional mo...\n\nSource 3 (https://noveum.ai/docs/getting-started/overview):\nNoveum.ai Overview\nComprehensive AI tracing and observability platform for LLM applications, RAG systems, and AI agents\nWelcome to Noveum.ai\u2014the comprehensive tracing and observability platform built specifically for AI applications. Whether you're building LLM-powered chatbots, RAG systems, multi-agent workflows, or any AI-driven application, Noveum provides the insights you need to understand, debug, and optimize your systems.\n\ud83c\udfaf Why AI Applications Need Specialized Observability\nTraditional mo...\n\nSource 4 (https://noveum.ai/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nSource 5 (https://noveum.ai/en/docs/getting-started/overview):\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to Observability suite for free, early users get free eval jobs and premium support for the first year....\n\nUser Question: What is Noveum and what does it do?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", + "prompt.prompt_length": 2865, + "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform):\nWhy Noveum.ai?\nBuilding production AI applications involves complex workflows with multiple components: LLM calls, ...", + "prompt.instruction_following": "rag_optimized", + "response.answer_length": 648, + "response.answer_completeness": "complete", + "response.response_quality": "high", + "response.source_citation": 5, + "response.context_utilization": 0.2776349614395887, + "output_response": "RAG Answer: Noveum.ai is a comprehensive tracing and observability platform specifically designed for AI applications, including those powered by large language models (LLMs), retrieval-augmented generation (RAG)...", + "evaluation.retrieval_effectiveness": 1, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 5, + "evaluation.context_relevance": "high", + "evaluation.overall_quality": "high", + "evaluation.ready_for_production": true, + "rag.retrieval_strategy": "semantic_similarity", + "rag.vector_search_results": 5, + "rag.context_synthesis": "multi_source", + "rag.document_coverage": 1, + "rag.information_density": 0.2776349614395887 + }, + "events": [], + "links": [] + } + ], + "created_at": "2025-10-16 08:47:03.776000000", + "updated_at": "2025-10-16 08:47:03.776000000" + }, + { + "trace_id": "3552ee8e-eafb-499a-b109-a9fb851a41d2", + "name": "auto_trace_system_initialization", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "status": "ok", + "status_message": "", + "start_time": "2025-10-16 08:47:03.695000000", + "end_time": "2025-10-16 08:47:03.770000000", + "duration_ms": 74.912, + "span_count": 1, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.9" + }, + "attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.9", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "spans": [ + { + "span_id": "050851c9-b2d3-4c82-accb-4085dc69f785", + "trace_id": "3552ee8e-eafb-499a-b109-a9fb851a41d2", + "parent_span_id": "", + "name": "system_initialization", + "start_time": "2025-10-16 08:47:03.695000000", + "end_time": "2025-10-16 08:47:03.770000000", + "duration_ms": 74.58, + "status": "ok", + "status_message": "", + "attributes": { + "system.force_scrape": false, + "system.config": { + "noveum_base_url": "https://noveum.ai", + "max_pages_to_scrape": 50, + "chunk_size": 1000, + "chunk_overlap": 200, + "max_search_results": 5, + "rag_threshold": 0.7, + "noveum_docs_file": "noveum_docs.json", + "vector_store_path": "noveum_vectorstore" + }, + "input_query": "Initialize system with force_scrape=False", + "output_response": "System initialization: RAG system loaded, vector store ready, agent operational", + "system.initialized": true, + "system.documents_loaded": 38, + "system.vectorstore_ready": true + }, + "events": [], + "links": [] + } + ], + "created_at": "2025-10-16 08:47:03.695000000", + "updated_at": "2025-10-16 08:47:03.695000000" + }, + { + "trace_id": "e6ef2449-7429-465b-9bff-f3def8de5bb0", + "name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "status": "ok", + "status_message": "", + "start_time": "2025-10-14 12:54:06.700000000", + "end_time": "2025-10-14 12:54:10.637000000", + "duration_ms": 3937.729, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "spans": [ + { + "span_id": "6b53235d-8b91-4d9e-8437-915828e84600", + "trace_id": "e6ef2449-7429-465b-9bff-f3def8de5bb0", + "parent_span_id": "e73ab184-50c1-4209-9276-be11e69e9513", + "name": "agent.query_routing", + "start_time": "2025-10-14 12:54:06.700000000", + "end_time": "2025-10-14 12:54:10.637000000", + "duration_ms": 3937.492, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What are the latest breakthroughs in artificial intelligence?", + "query_length": 61, + "query_type": "routing_query", + "output_response": "Routed to Web Search for query processing", + "router.classification": "Web Search", + "router.confidence_score": 1, + "router.rag_keyword_score": 0, + "router.web_keyword_score": 1, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [] + }, + { + "span_id": "a7d1e33a-95a1-41a9-a117-62b4c373bd60", + "trace_id": "e6ef2449-7429-465b-9bff-f3def8de5bb0", + "parent_span_id": "6b53235d-8b91-4d9e-8437-915828e84600", + "name": "agent.web_search_generation", + "start_time": "2025-10-14 12:54:06.700000000", + "end_time": "2025-10-14 12:54:10.637000000", + "duration_ms": 3936.983, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "web_search_agent", + "agent.operation": "web_search_generation", + "agent.capabilities": [ + "web_search", + "content_synthesis", + "response_generation" + ], + "agent.id": "noveum_web_search_agent", + "input_query": "What are the latest breakthroughs in artificial intelligence?", + "query_length": 61, + "query_type": "web_search_query", + "output_response": "Web Search Answer: The web search did not yield specific results regarding the latest breakthroughs in artificial intelligence. However, I can provide a general overview based on knowledge up to October 2023.\n\nRecent br...", + "web_search.results_count": 1, + "web_search.context_length": 163, + "web_search.response_length": 1090, + "web_search.mode": "external_web_search" + }, + "events": [], + "links": [] + }, + { + "span_id": "da630cab-2ba6-4f12-98b9-6ab1c0899c17", + "trace_id": "e6ef2449-7429-465b-9bff-f3def8de5bb0", + "parent_span_id": "6b53235d-8b91-4d9e-8437-915828e84600", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-14 12:54:06.700000000", + "end_time": "2025-10-14 12:54:06.700000000", + "duration_ms": 0.27999999999999997, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What are the latest breakthroughs in artificial intelligence?", + "query_length": 61, + "query_type": "routing_evaluation_query", + "classification.mode": "Web Search", + "classification.rag_keyword_score": 0, + "classification.web_keyword_score": 1, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "complex", + "query.intent": "general_knowledge", + "query.keyword_density": 0.125, + "query.domain_affinity": "general", + "routing.decision": "Routed to Web Search based on analysis", + "routing.rationale": "RAG score: 0, Web score: 1, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "RAG", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.019230769230769232, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: Web Search (Confidence: 1.00)" + }, + "events": [], + "links": [] + }, + { + "span_id": "e73ab184-50c1-4209-9276-be11e69e9513", + "trace_id": "e6ef2449-7429-465b-9bff-f3def8de5bb0", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-14 12:54:06.700000000", + "end_time": "2025-10-14 12:54:10.637000000", + "duration_ms": 3937.663, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What are the latest breakthroughs in artificial intelligence?", + "query.length": 61, + "processing.mode": "Web Search", + "processing.time_seconds": 3.93752384185791, + "processing.response_length": 1090, + "processing.sources_count": 1, + "output_response": "Final Answer: The web search did not yield specific results regarding the latest breakthroughs in artificial intelligence. However, I can provide a general overview based on knowledge up to October 2023.\n\nRecent br...", + "final_answer_mode": "Web Search", + "query_processed.input_query": "What are the latest breakthroughs in artificial intelligence?", + "query_processed.output_response": "Successfully processed query using Web Search, generated 1090 character response", + "query_processed.mode": "Web Search", + "query_processed.processing_time": 3.93752384185791, + "query_processed.response_length": 1090 + }, + "events": [], + "links": [] + }, + { + "span_id": "bb47766c-afda-44fe-8838-7936c80cd4b6", + "trace_id": "e6ef2449-7429-465b-9bff-f3def8de5bb0", + "parent_span_id": "a7d1e33a-95a1-41a9-a117-62b4c373bd60", + "name": "agent.llm_model_execution", + "start_time": "2025-10-14 12:54:07.104000000", + "end_time": "2025-10-14 12:54:10.637000000", + "duration_ms": 3533.321, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What are the latest breakthroughs in artificial intelligence?", + "query_length": 61, + "query_type": "web_search_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 3.533268928527832, + "model.latency_ms": 3533.268928527832, + "model.start_time": 1760446447.104266, + "model.end_time": 1760446450.6375349, + "model.performance_tier": "medium", + "model.prompt_tokens": 167, + "model.completion_tokens": 272, + "model.total_tokens": 439, + "model.tokens_per_second": 124.2475477752307, + "model.estimated_cost": 0.013170000000000001, + "model.efficiency_score": 2.4829157175398633, + "model.response_length": 1090, + "model.response_quality": "high", + "model.output_response": "Model Response: The web search did not yield specific results regarding the latest breakthroughs in artificial intelligence. However, I can provide a general overview based on knowledge up to October 2023.\n\nRecent br...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [] + }, + { + "span_id": "1d914a9d-b770-45dc-8308-6a36e9e0db89", + "trace_id": "e6ef2449-7429-465b-9bff-f3def8de5bb0", + "parent_span_id": "a7d1e33a-95a1-41a9-a117-62b4c373bd60", + "name": "agent.web_search_evaluation_metrics", + "start_time": "2025-10-14 12:54:10.637000000", + "end_time": "2025-10-14 12:54:10.637000000", + "duration_ms": 0.022, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "web_search_evaluation_metrics", + "agent.capabilities": [ + "web_search_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What are the latest breakthroughs in artificial intelligence?", + "query_length": 61, + "query_type": "web_search_evaluation_query", + "web_search.results_count": 1, + "web_search.context_length": 163, + "web_search.context_synthesized": "Context from 1 web sources", + "web_search.search_effectiveness": 0.2, + "web_search.context_quality": "low", + "web_search.source_diversity": 1, + "web_search.information_synthesis": "low", + "web_search.external_knowledge_utilization": 0.163, + "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What+are+the+latest+breakthroughs+in+artificial+intelligence?\n\nUser Question: What are the latest breakthroughs in artificial intelligence?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", + "prompt.prompt_length": 669, + "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What+are+the+latest+breakthroughs+in+artificial+intelligence?", + "prompt.instruction_following": "web_search_optimized", + "response.answer_length": 1090, + "response.answer_completeness": "complete", + "response.response_quality": "medium", + "response.source_citation": 1, + "response.context_utilization": 6.6871165644171775, + "output_response": "Web Search Answer: The web search did not yield specific results regarding the latest breakthroughs in artificial intelligence. However, I can provide a general overview based on knowledge up to October 2023.\n\nRecent br...", + "evaluation.search_effectiveness": 0.2, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 1, + "evaluation.context_relevance": "low", + "evaluation.overall_quality": "low", + "evaluation.ready_for_production": true, + "web_search.search_strategy": "duckduckgo_api", + "web_search.real_time_data": true, + "web_search.external_sources": 1, + "web_search.information_freshness": "current", + "web_search.knowledge_synthesis": "single_source" + }, + "events": [], + "links": [] + } + ], + "created_at": "2025-10-14 12:54:06.700000000", + "updated_at": "2025-10-14 12:54:06.700000000" + }, + { + "trace_id": "2ba8cab0-2366-4add-8ac3-b490a05e9a60", + "name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "status": "ok", + "status_message": "", + "start_time": "2025-10-14 12:53:56.480000000", + "end_time": "2025-10-14 12:54:05.687000000", + "duration_ms": 9207.67, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "spans": [ + { + "span_id": "91f3999c-2edf-4c56-8f4d-ab43b6ebbaee", + "trace_id": "2ba8cab0-2366-4add-8ac3-b490a05e9a60", + "parent_span_id": "cbe51252-a323-4852-a90e-c907a0480fa3", + "name": "agent.web_search_generation", + "start_time": "2025-10-14 12:53:56.480000000", + "end_time": "2025-10-14 12:54:05.685000000", + "duration_ms": 9205.212, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "web_search_agent", + "agent.operation": "web_search_generation", + "agent.capabilities": [ + "web_search", + "content_synthesis", + "response_generation" + ], + "agent.id": "noveum_web_search_agent", + "input_query": "What's happening in the software development world today?", + "query_length": 57, + "query_type": "web_search_query", + "output_response": "Web Search Answer: The software development world is currently experiencing rapid changes driven by advancements in artificial intelligence (AI) and evolving programming practices. Key trends include:\n\n1. **AI Integrati...", + "web_search.results_count": 1, + "web_search.context_length": 975, + "web_search.response_length": 1636, + "web_search.mode": "external_web_search" + }, + "events": [], + "links": [] + }, + { + "span_id": "ea2aee75-4537-46ea-954b-2002fc0026c5", + "trace_id": "2ba8cab0-2366-4add-8ac3-b490a05e9a60", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-14 12:53:56.480000000", + "end_time": "2025-10-14 12:54:05.687000000", + "duration_ms": 9207.528999999999, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What's happening in the software development world today?", + "query.length": 57, + "processing.mode": "Web Search", + "processing.time_seconds": 9.206756830215454, + "processing.response_length": 1636, + "processing.sources_count": 1, + "output_response": "Final Answer: The software development world is currently experiencing rapid changes driven by advancements in artificial intelligence (AI) and evolving programming practices. Key trends include:\n\n1. **AI Integrati...", + "final_answer_mode": "Web Search", + "query_processed.input_query": "What's happening in the software development world today?", + "query_processed.output_response": "Successfully processed query using Web Search, generated 1636 character response", + "query_processed.mode": "Web Search", + "query_processed.processing_time": 9.206756830215454, + "query_processed.response_length": 1636 + }, + "events": [], + "links": [] + }, + { + "span_id": "cbe51252-a323-4852-a90e-c907a0480fa3", + "trace_id": "2ba8cab0-2366-4add-8ac3-b490a05e9a60", + "parent_span_id": "ea2aee75-4537-46ea-954b-2002fc0026c5", + "name": "agent.query_routing", + "start_time": "2025-10-14 12:53:56.480000000", + "end_time": "2025-10-14 12:54:05.686000000", + "duration_ms": 9205.569000000001, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What's happening in the software development world today?", + "query_length": 57, + "query_type": "routing_query", + "output_response": "Routed to Web Search for query processing", + "router.classification": "Web Search", + "router.confidence_score": 1, + "router.rag_keyword_score": 0, + "router.web_keyword_score": 1, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [] + }, + { + "span_id": "d850ecee-86ef-4818-9cec-638bfee26293", + "trace_id": "2ba8cab0-2366-4add-8ac3-b490a05e9a60", + "parent_span_id": "cbe51252-a323-4852-a90e-c907a0480fa3", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-14 12:53:56.480000000", + "end_time": "2025-10-14 12:53:56.480000000", + "duration_ms": 0.041999999999999996, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What's happening in the software development world today?", + "query_length": 57, + "query_type": "routing_evaluation_query", + "classification.mode": "Web Search", + "classification.rag_keyword_score": 0, + "classification.web_keyword_score": 1, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "complex", + "query.intent": "general_knowledge", + "query.keyword_density": 0.125, + "query.domain_affinity": "general", + "routing.decision": "Routed to Web Search based on analysis", + "routing.rationale": "RAG score: 0, Web score: 1, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "RAG", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.019230769230769232, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: Web Search (Confidence: 1.00)" + }, + "events": [], + "links": [] + }, + { + "span_id": "0d4387a4-adf3-42c1-aca7-51453a90dd36", + "trace_id": "2ba8cab0-2366-4add-8ac3-b490a05e9a60", + "parent_span_id": "91f3999c-2edf-4c56-8f4d-ab43b6ebbaee", + "name": "agent.llm_model_execution", + "start_time": "2025-10-14 12:53:59.426000000", + "end_time": "2025-10-14 12:54:05.684000000", + "duration_ms": 6258.643, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What's happening in the software development world today?", + "query_length": 57, + "query_type": "web_search_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 6.25848388671875, + "model.latency_ms": 6258.48388671875, + "model.start_time": 1760446439.426367, + "model.end_time": 1760446445.684851, + "model.performance_tier": "slow", + "model.prompt_tokens": 369, + "model.completion_tokens": 409, + "model.total_tokens": 778, + "model.tokens_per_second": 124.31125718019486, + "model.estimated_cost": 0.02334, + "model.efficiency_score": 2.102827763496144, + "model.response_length": 1636, + "model.response_quality": "high", + "model.output_response": "Model Response: The software development world is currently experiencing rapid changes driven by advancements in artificial intelligence (AI) and evolving programming practices. Key trends include:\n\n1. **AI Integrati...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [] + }, + { + "span_id": "717e3c4a-6300-45e5-9456-caa3e773d758", + "trace_id": "2ba8cab0-2366-4add-8ac3-b490a05e9a60", + "parent_span_id": "91f3999c-2edf-4c56-8f4d-ab43b6ebbaee", + "name": "agent.web_search_evaluation_metrics", + "start_time": "2025-10-14 12:54:05.685000000", + "end_time": "2025-10-14 12:54:05.685000000", + "duration_ms": 0.044, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "web_search_evaluation_metrics", + "agent.capabilities": [ + "web_search_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What's happening in the software development world today?", + "query_length": 57, + "query_type": "web_search_evaluation_query", + "web_search.results_count": 1, + "web_search.context_length": 975, + "web_search.context_synthesized": "Context from 1 web sources", + "web_search.search_effectiveness": 0.2, + "web_search.context_quality": "high", + "web_search.source_diversity": 1, + "web_search.information_synthesis": "low", + "web_search.external_knowledge_utilization": 0.975, + "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nLog in to WhatsApp Web for simple, reliable and private messaging on your desktop. Send and receive messages and files with ease, all for free. Sep 23, 2025 \u00b7 Chief Economists' Outlook 2025 states that these under-the-radar trends are shaping the global economy, from the impact of AI on research productivity to a tightened labor \u2026 Sep 1, 2025 \u00b7 THE AI DEVELOPMENT REVOLUTION: WHAT'S REALLY HAPPENING? Let's face it - the software development world is changing faster than a React component re-rendering on \u2026 Oct 7, 2025 \u00b7 Computer Programming Research. Read current computer science articles on everything from computer programs to detect cancer genes and control vehicle maintenance to \u2026 Aug 24, 2025 \u00b7 Artificial Intelligence News. Everything on AI including futuristic robots with artificial intelligence, computer models of human intelligence and more.\nURL: https://duckduckgo.com/?q=What's+happening+in+the+software+development+world+today?\n\nUser Question: What's happening in the software development world today?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", + "prompt.prompt_length": 1477, + "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nLog in to WhatsApp Web for simple, reliable and private messaging on your desktop. Send and receive messages and files with ease, all for free. Sep 23, 2025 \u00b7 Chief Economi...", + "prompt.instruction_following": "web_search_optimized", + "response.answer_length": 1636, + "response.answer_completeness": "complete", + "response.response_quality": "medium", + "response.source_citation": 1, + "response.context_utilization": 1.677948717948718, + "output_response": "Web Search Answer: The software development world is currently experiencing rapid changes driven by advancements in artificial intelligence (AI) and evolving programming practices. Key trends include:\n\n1. **AI Integrati...", + "evaluation.search_effectiveness": 0.2, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 1, + "evaluation.context_relevance": "high", + "evaluation.overall_quality": "low", + "evaluation.ready_for_production": true, + "web_search.search_strategy": "duckduckgo_api", + "web_search.real_time_data": true, + "web_search.external_sources": 1, + "web_search.information_freshness": "current", + "web_search.knowledge_synthesis": "single_source" + }, + "events": [], + "links": [] + } + ], + "created_at": "2025-10-14 12:53:56.480000000", + "updated_at": "2025-10-14 12:53:56.480000000" + }, + { + "trace_id": "a7c3399e-372f-4aae-baeb-05215cc4b85f", + "name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "status": "ok", + "status_message": "", + "start_time": "2025-10-14 12:53:50.963000000", + "end_time": "2025-10-14 12:53:55.471000000", + "duration_ms": 4507.555, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "spans": [ + { + "span_id": "3d39f522-8ae7-4988-aafd-58b09734eff0", + "trace_id": "a7c3399e-372f-4aae-baeb-05215cc4b85f", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-14 12:53:50.963000000", + "end_time": "2025-10-14 12:53:55.471000000", + "duration_ms": 4507.466, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What are the newest features in cloud computing?", + "query.length": 48, + "processing.mode": "RAG", + "processing.time_seconds": 4.507421255111694, + "processing.response_length": 870, + "processing.sources_count": 5, + "output_response": "Final Answer: The provided context does not specifically mention any new features in cloud computing as a general topic. However, it does highlight several recent enhancements related to the Noveum.ai platform, whi...", + "final_answer_mode": "RAG", + "query_processed.input_query": "What are the newest features in cloud computing?", + "query_processed.output_response": "Successfully processed query using RAG, generated 870 character response", + "query_processed.mode": "RAG", + "query_processed.processing_time": 4.507421255111694, + "query_processed.response_length": 870 + }, + "events": [], + "links": [] + }, + { + "span_id": "54aa6700-f2de-4cf0-b1ad-a3d24ed692ed", + "trace_id": "a7c3399e-372f-4aae-baeb-05215cc4b85f", + "parent_span_id": "d17f31cf-c570-4604-8df3-532b2e530fce", + "name": "agent.llm-rag", + "start_time": "2025-10-14 12:53:50.963000000", + "end_time": "2025-10-14 12:53:55.471000000", + "duration_ms": 4507.083, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "rag_agent", + "agent.operation": "llm-rag", + "agent.capabilities": [ + "document_retrieval", + "context_generation", + "response_generation" + ], + "agent.id": "noveum_rag_agent", + "input_query": "What are the newest features in cloud computing?", + "query_length": 48, + "query_type": "rag_query", + "output_response": "RAG Answer: The provided context does not specifically mention any new features in cloud computing as a general topic. However, it does highlight several recent enhancements related to the Noveum.ai platform, whi...", + "rag.context_length": 2579, + "rag.sources_count": 5, + "rag.answer_length": 870, + "rag.mode": "retrieval_augmented_generation" + }, + "events": [], + "links": [] + }, + { + "span_id": "8878096f-6cfa-4f70-9844-d7723a0875f0", + "trace_id": "a7c3399e-372f-4aae-baeb-05215cc4b85f", + "parent_span_id": "d17f31cf-c570-4604-8df3-532b2e530fce", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-14 12:53:50.963000000", + "end_time": "2025-10-14 12:53:50.963000000", + "duration_ms": 0.036000000000000004, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What are the newest features in cloud computing?", + "query_length": 48, + "query_type": "routing_evaluation_query", + "classification.mode": "RAG", + "classification.rag_keyword_score": 1, + "classification.web_keyword_score": 0, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "documentation", + "query.keyword_density": 0.125, + "query.domain_affinity": "noveum", + "routing.decision": "Routed to RAG based on analysis", + "routing.rationale": "RAG score: 1, Web score: 0, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "Web Search", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.019230769230769232, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: RAG (Confidence: 1.00)" + }, + "events": [], + "links": [] + }, + { + "span_id": "d17f31cf-c570-4604-8df3-532b2e530fce", + "trace_id": "a7c3399e-372f-4aae-baeb-05215cc4b85f", + "parent_span_id": "3d39f522-8ae7-4988-aafd-58b09734eff0", + "name": "agent.query_routing", + "start_time": "2025-10-14 12:53:50.963000000", + "end_time": "2025-10-14 12:53:55.471000000", + "duration_ms": 4507.3730000000005, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What are the newest features in cloud computing?", + "query_length": 48, + "query_type": "routing_query", + "output_response": "Routed to RAG for query processing", + "router.classification": "RAG", + "router.confidence_score": 1, + "router.rag_keyword_score": 1, + "router.web_keyword_score": 0, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [] + }, + { + "span_id": "97ef01a8-5930-4902-909a-245c516a5871", + "trace_id": "a7c3399e-372f-4aae-baeb-05215cc4b85f", + "parent_span_id": "54aa6700-f2de-4cf0-b1ad-a3d24ed692ed", + "name": "agent.llm_model_execution", + "start_time": "2025-10-14 12:53:51.439000000", + "end_time": "2025-10-14 12:53:55.077000000", + "duration_ms": 3637.337, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What are the newest features in cloud computing?", + "query_length": 48, + "query_type": "rag_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 3.6372909545898438, + "model.latency_ms": 3637.2909545898438, + "model.start_time": 1760446431.439748, + "model.end_time": 1760446435.077039, + "model.performance_tier": "medium", + "model.prompt_tokens": 780, + "model.completion_tokens": 217, + "model.total_tokens": 997, + "model.tokens_per_second": 274.1050997698989, + "model.estimated_cost": 0.02991, + "model.efficiency_score": 0.872617853560682, + "model.response_length": 870, + "model.response_quality": "high", + "model.output_response": "Model Response: The provided context does not specifically mention any new features in cloud computing as a general topic. However, it does highlight several recent enhancements related to the Noveum.ai platform, whi...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [] + }, + { + "span_id": "ea792ec6-64b4-4ed5-b418-ba902e90c57a", + "trace_id": "a7c3399e-372f-4aae-baeb-05215cc4b85f", + "parent_span_id": "54aa6700-f2de-4cf0-b1ad-a3d24ed692ed", + "name": "agent.rag_evaluation_metrics", + "start_time": "2025-10-14 12:53:55.077000000", + "end_time": "2025-10-14 12:53:55.077000000", + "duration_ms": 0.034, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "rag_evaluation_metrics", + "agent.capabilities": [ + "retrieval_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What are the newest features in cloud computing?", + "query_length": 48, + "query_type": "rag_evaluation_query", + "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/changelog):\n- \ud83d\udcc8 Enhanced Dashboard Analytics: Real-time request tracking, latency monitoring, and cost analysis\n- \ud83d\udd0d Improved Logs Interface: Better search functionality, detailed trace views, and enhanced debugging capabilities\n- \ud83d\udc65 Team Management: Advanced member mana...", + "retrieval.context_length": 2579, + "retrieval.sources_count": 5, + "retrieval.context_quality": "high", + "retrieval.effectiveness": 1, + "retrieval.context_utilization": 2.579, + "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/changelog):\n- \ud83d\udcc8 Enhanced Dashboard Analytics: Real-time request tracking, latency monitoring, and cost analysis\n- \ud83d\udd0d Improved Logs Interface: Better search functionality, detailed trace views, and enhanced debugging capabilities\n- \ud83d\udc65 Team Management: Advanced member management with role-based access control and invitation system\n- \ud83d\udee1\ufe0f Security Improvements: Enhanced API key management and secure credential storage\n4 months ago\n- \ud83d\udcca Advanced Metrics Collection: Comprehensive telemetry system with custom metrics ...\n\nSource 2 (https://noveum.ai/en/changelog):\n- \ud83d\udcdd Documentation Updates: Comprehensive guides for integration and best practices\n5 months ago\n- \ud83c\udf89 Platform Foundation: Initial release of Noveum.ai AI observability platform\n- \ud83d\ude80 Core Features: AI Gateway integration, basic evaluation metrics, and monitoring capabilities\n- \ud83d\udcc8 Evaluation Jobs: Automated model evaluation with configurable metrics and reporting\n- \ud83d\udd0c Provider Integrations: Support for major AI providers and custom model deployments...\n\nSource 3 (https://noveum.ai/docs/platform/dashboard):\n- Bulk Export: Download multiple traces simultaneously\n- Comparative Analysis: Compare performance across multiple traces\nReal-Time Updates\n- Live Refresh: Automatic updates as new traces arrive\n- Connection Monitoring: Real-time status of your trace ingestion pipeline\n- Performance Indicators: Live metrics for system health monitoring\nReady to dive deeper? Explore Projects & Environments to organize your AI applications, or check out Team Collaboration to share insights with your team.\nGet Earl...\n\nSource 4 (https://noveum.ai/en/changelog):\nChangelog\nStay up to date with the latest changes in our product.\n2 months ago\n- \ud83c\udfaf Major Platform Evolution: NovaEval Framework - Launched comprehensive AI model evaluation framework with 20+ built-in scorers including accuracy, RAG metrics, conversational scoring, and specialized agent evaluation tools. Features production-grade deployment on Docker and Kubernetes. (GitHub | PyPI Package)\n- \ud83d\ude80 Next-Generation Traces UI - Complete redesign featuring three-pane layout with directory tree navigatio...\n\nSource 5 (https://noveum.ai/en/blog):\n8/12/2025\nDiscover how Noveum.ai provides comprehensive tracing and observability for AI applications, from development debugging to production optimization.\nShashank Agarwal\n3/3/2025\nDiscover how Noveum.ai provides comprehensive tracing and observability for LLM applications, RAG systems, and multi-agent workflows with our powerful Python and TypeScript SDKs.\nShashank Agarwal\n3/2/2025...\n\nUser Question: What are the newest features in cloud computing?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", + "prompt.prompt_length": 3123, + "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/changelog):\n- \ud83d\udcc8 Enhanced Dashboard Analytics: Real-time request tracking, latency monitoring, and cost analysis\n- \ud83d\udd0d Improved Logs Interface: Better search functionality,...", + "prompt.instruction_following": "rag_optimized", + "response.answer_length": 870, + "response.answer_completeness": "complete", + "response.response_quality": "high", + "response.source_citation": 5, + "response.context_utilization": 0.33734005428460645, + "output_response": "RAG Answer: The provided context does not specifically mention any new features in cloud computing as a general topic. However, it does highlight several recent enhancements related to the Noveum.ai platform, whi...", + "evaluation.retrieval_effectiveness": 1, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 5, + "evaluation.context_relevance": "high", + "evaluation.overall_quality": "high", + "evaluation.ready_for_production": true, + "rag.retrieval_strategy": "semantic_similarity", + "rag.vector_search_results": 5, + "rag.context_synthesis": "multi_source", + "rag.document_coverage": 1, + "rag.information_density": 0.33734005428460645 + }, + "events": [], + "links": [] + } + ], + "created_at": "2025-10-14 12:53:50.963000000", + "updated_at": "2025-10-14 12:53:50.963000000" + }, + { + "trace_id": "d682797f-2ce0-448e-8f0d-891541430988", + "name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "status": "ok", + "status_message": "", + "start_time": "2025-10-14 12:53:44.917000000", + "end_time": "2025-10-14 12:53:49.957000000", + "duration_ms": 5039.64, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "spans": [ + { + "span_id": "e2d314c8-687f-4b8d-869c-8d7f2a22a4f3", + "trace_id": "d682797f-2ce0-448e-8f0d-891541430988", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-14 12:53:44.917000000", + "end_time": "2025-10-14 12:53:49.957000000", + "duration_ms": 5039.565, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What's the current status of cryptocurrency markets?", + "query.length": 52, + "processing.mode": "Web Search", + "processing.time_seconds": 5.0394370555877686, + "processing.response_length": 338, + "processing.sources_count": 1, + "output_response": "Final Answer: The web search results did not provide any specific information regarding the current status of cryptocurrency markets. Therefore, I cannot provide an update on market conditions, prices, or trends at...", + "final_answer_mode": "Web Search", + "query_processed.input_query": "What's the current status of cryptocurrency markets?", + "query_processed.output_response": "Successfully processed query using Web Search, generated 338 character response", + "query_processed.mode": "Web Search", + "query_processed.processing_time": 5.0394370555877686, + "query_processed.response_length": 338 + }, + "events": [], + "links": [] + }, + { + "span_id": "d367a6bd-86af-41c5-96ca-5284425775ce", + "trace_id": "d682797f-2ce0-448e-8f0d-891541430988", + "parent_span_id": "f137a412-6ddf-4ace-908e-9edf73534aa7", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-14 12:53:44.917000000", + "end_time": "2025-10-14 12:53:44.918000000", + "duration_ms": 0.046, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What's the current status of cryptocurrency markets?", + "query_length": 52, + "query_type": "routing_evaluation_query", + "classification.mode": "Web Search", + "classification.rag_keyword_score": 0, + "classification.web_keyword_score": 3, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "complex", + "query.intent": "general_knowledge", + "query.keyword_density": 0.42857142857142855, + "query.domain_affinity": "general", + "routing.decision": "Routed to Web Search based on analysis", + "routing.rationale": "RAG score: 0, Web score: 3, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "RAG", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.057692307692307696, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: Web Search (Confidence: 1.00)" + }, + "events": [], + "links": [] + }, + { + "span_id": "f137a412-6ddf-4ace-908e-9edf73534aa7", + "trace_id": "d682797f-2ce0-448e-8f0d-891541430988", + "parent_span_id": "e2d314c8-687f-4b8d-869c-8d7f2a22a4f3", + "name": "agent.query_routing", + "start_time": "2025-10-14 12:53:44.917000000", + "end_time": "2025-10-14 12:53:49.957000000", + "duration_ms": 5039.404, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What's the current status of cryptocurrency markets?", + "query_length": 52, + "query_type": "routing_query", + "output_response": "Routed to Web Search for query processing", + "router.classification": "Web Search", + "router.confidence_score": 1, + "router.rag_keyword_score": 0, + "router.web_keyword_score": 3, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [] + }, + { + "span_id": "987b6910-9b6d-4207-b0f3-8807a23e3ddb", + "trace_id": "d682797f-2ce0-448e-8f0d-891541430988", + "parent_span_id": "f137a412-6ddf-4ace-908e-9edf73534aa7", + "name": "agent.web_search_generation", + "start_time": "2025-10-14 12:53:44.918000000", + "end_time": "2025-10-14 12:53:49.957000000", + "duration_ms": 5039.248, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "web_search_agent", + "agent.operation": "web_search_generation", + "agent.capabilities": [ + "web_search", + "content_synthesis", + "response_generation" + ], + "agent.id": "noveum_web_search_agent", + "input_query": "What's the current status of cryptocurrency markets?", + "query_length": 52, + "query_type": "web_search_query", + "output_response": "Web Search Answer: The web search results did not provide any specific information regarding the current status of cryptocurrency markets. Therefore, I cannot provide an update on market conditions, prices, or trends at...", + "web_search.results_count": 1, + "web_search.context_length": 154, + "web_search.response_length": 338, + "web_search.mode": "external_web_search" + }, + "events": [], + "links": [] + }, + { + "span_id": "b2614d0e-5fcd-46ec-8d25-f1b04eb142af", + "trace_id": "d682797f-2ce0-448e-8f0d-891541430988", + "parent_span_id": "987b6910-9b6d-4207-b0f3-8807a23e3ddb", + "name": "agent.llm_model_execution", + "start_time": "2025-10-14 12:53:45.255000000", + "end_time": "2025-10-14 12:53:49.957000000", + "duration_ms": 4701.744, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What's the current status of cryptocurrency markets?", + "query_length": 52, + "query_type": "web_search_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 4.701629161834717, + "model.latency_ms": 4701.629161834717, + "model.start_time": 1760446425.2554948, + "model.end_time": 1760446429.957124, + "model.performance_tier": "medium", + "model.prompt_tokens": 162, + "model.completion_tokens": 84, + "model.total_tokens": 246, + "model.tokens_per_second": 52.32228904756993, + "model.estimated_cost": 0.00738, + "model.efficiency_score": 1.3739837398373984, + "model.response_length": 338, + "model.response_quality": "high", + "model.output_response": "Model Response: The web search results did not provide any specific information regarding the current status of cryptocurrency markets. Therefore, I cannot provide an update on market conditions, prices, or trends at...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [] + }, + { + "span_id": "663e8789-75d0-4c4d-b14d-4088022436ac", + "trace_id": "d682797f-2ce0-448e-8f0d-891541430988", + "parent_span_id": "987b6910-9b6d-4207-b0f3-8807a23e3ddb", + "name": "agent.web_search_evaluation_metrics", + "start_time": "2025-10-14 12:53:49.957000000", + "end_time": "2025-10-14 12:53:49.957000000", + "duration_ms": 0.032, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "web_search_evaluation_metrics", + "agent.capabilities": [ + "web_search_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What's the current status of cryptocurrency markets?", + "query_length": 52, + "query_type": "web_search_evaluation_query", + "web_search.results_count": 1, + "web_search.context_length": 154, + "web_search.context_synthesized": "Context from 1 web sources", + "web_search.search_effectiveness": 0.2, + "web_search.context_quality": "low", + "web_search.source_diversity": 1, + "web_search.information_synthesis": "low", + "web_search.external_knowledge_utilization": 0.154, + "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What's+the+current+status+of+cryptocurrency+markets?\n\nUser Question: What's the current status of cryptocurrency markets?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", + "prompt.prompt_length": 651, + "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What's+the+current+status+of+cryptocurrency+markets?", + "prompt.instruction_following": "web_search_optimized", + "response.answer_length": 338, + "response.answer_completeness": "complete", + "response.response_quality": "medium", + "response.source_citation": 1, + "response.context_utilization": 2.1948051948051948, + "output_response": "Web Search Answer: The web search results did not provide any specific information regarding the current status of cryptocurrency markets. Therefore, I cannot provide an update on market conditions, prices, or trends at...", + "evaluation.search_effectiveness": 0.2, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 1, + "evaluation.context_relevance": "low", + "evaluation.overall_quality": "low", + "evaluation.ready_for_production": true, + "web_search.search_strategy": "duckduckgo_api", + "web_search.real_time_data": true, + "web_search.external_sources": 1, + "web_search.information_freshness": "current", + "web_search.knowledge_synthesis": "single_source" + }, + "events": [], + "links": [] + } + ], + "created_at": "2025-10-14 12:53:44.917000000", + "updated_at": "2025-10-14 12:53:44.917000000" + }, + { + "trace_id": "b0858dca-4173-493b-a6c5-4e993f606477", + "name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "status": "ok", + "status_message": "", + "start_time": "2025-10-14 12:53:37.818000000", + "end_time": "2025-10-14 12:53:43.908000000", + "duration_ms": 6089.715, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "spans": [ + { + "span_id": "3382fe7c-aaca-46e0-a8d2-dbee75ff3bed", + "trace_id": "b0858dca-4173-493b-a6c5-4e993f606477", + "parent_span_id": "6cf94049-87b0-4fc5-b37e-32d261543647", + "name": "agent.query_routing", + "start_time": "2025-10-14 12:53:37.818000000", + "end_time": "2025-10-14 12:53:43.908000000", + "duration_ms": 6089.22, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What are the latest updates in Python programming?", + "query_length": 50, + "query_type": "routing_query", + "output_response": "Routed to Web Search for query processing", + "router.classification": "Web Search", + "router.confidence_score": 1, + "router.rag_keyword_score": 0, + "router.web_keyword_score": 2, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [] + }, + { + "span_id": "6cf94049-87b0-4fc5-b37e-32d261543647", + "trace_id": "b0858dca-4173-493b-a6c5-4e993f606477", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-14 12:53:37.818000000", + "end_time": "2025-10-14 12:53:43.908000000", + "duration_ms": 6089.607, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What are the latest updates in Python programming?", + "query.length": 50, + "processing.mode": "Web Search", + "processing.time_seconds": 6.089300870895386, + "processing.response_length": 1324, + "processing.sources_count": 1, + "output_response": "Final Answer: The web search did not yield specific results regarding the latest updates in Python programming. However, I can provide a general overview based on my knowledge up to October 2023.\n\nAs of late 2023, ...", + "final_answer_mode": "Web Search", + "query_processed.input_query": "What are the latest updates in Python programming?", + "query_processed.output_response": "Successfully processed query using Web Search, generated 1324 character response", + "query_processed.mode": "Web Search", + "query_processed.processing_time": 6.089300870895386, + "query_processed.response_length": 1324 + }, + "events": [], + "links": [] + }, + { + "span_id": "21c8befa-356a-4034-8d62-792238cb9e6e", + "trace_id": "b0858dca-4173-493b-a6c5-4e993f606477", + "parent_span_id": "3382fe7c-aaca-46e0-a8d2-dbee75ff3bed", + "name": "agent.web_search_generation", + "start_time": "2025-10-14 12:53:37.819000000", + "end_time": "2025-10-14 12:53:43.908000000", + "duration_ms": 6088.807, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "web_search_agent", + "agent.operation": "web_search_generation", + "agent.capabilities": [ + "web_search", + "content_synthesis", + "response_generation" + ], + "agent.id": "noveum_web_search_agent", + "input_query": "What are the latest updates in Python programming?", + "query_length": 50, + "query_type": "web_search_query", + "output_response": "Web Search Answer: The web search did not yield specific results regarding the latest updates in Python programming. However, I can provide a general overview based on my knowledge up to October 2023.\n\nAs of late 2023, ...", + "web_search.results_count": 1, + "web_search.context_length": 152, + "web_search.response_length": 1324, + "web_search.mode": "external_web_search" + }, + "events": [], + "links": [] + }, + { + "span_id": "482feedf-52f4-4c6a-9186-d59898820caa", + "trace_id": "b0858dca-4173-493b-a6c5-4e993f606477", + "parent_span_id": "3382fe7c-aaca-46e0-a8d2-dbee75ff3bed", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-14 12:53:37.819000000", + "end_time": "2025-10-14 12:53:37.819000000", + "duration_ms": 0.039, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What are the latest updates in Python programming?", + "query_length": 50, + "query_type": "routing_evaluation_query", + "classification.mode": "Web Search", + "classification.rag_keyword_score": 0, + "classification.web_keyword_score": 2, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "general_knowledge", + "query.keyword_density": 0.25, + "query.domain_affinity": "general", + "routing.decision": "Routed to Web Search based on analysis", + "routing.rationale": "RAG score: 0, Web score: 2, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "RAG", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.038461538461538464, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: Web Search (Confidence: 1.00)" + }, + "events": [], + "links": [] + }, + { + "span_id": "84f1f63f-ee05-411e-a7dd-4ee9e9826f5a", + "trace_id": "b0858dca-4173-493b-a6c5-4e993f606477", + "parent_span_id": "21c8befa-356a-4034-8d62-792238cb9e6e", + "name": "agent.llm_model_execution", + "start_time": "2025-10-14 12:53:38.191000000", + "end_time": "2025-10-14 12:53:43.907000000", + "duration_ms": 5716.078, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What are the latest updates in Python programming?", + "query_length": 50, + "query_type": "web_search_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 5.715963125228882, + "model.latency_ms": 5715.963125228882, + "model.start_time": 1760446418.191805, + "model.end_time": 1760446423.907768, + "model.performance_tier": "slow", + "model.prompt_tokens": 161, + "model.completion_tokens": 331, + "model.total_tokens": 492, + "model.tokens_per_second": 86.07473302765561, + "model.estimated_cost": 0.01476, + "model.efficiency_score": 2.6910569105691056, + "model.response_length": 1324, + "model.response_quality": "high", + "model.output_response": "Model Response: The web search did not yield specific results regarding the latest updates in Python programming. However, I can provide a general overview based on my knowledge up to October 2023.\n\nAs of late 2023, ...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [] + }, + { + "span_id": "2fb9ecf3-a497-4285-a694-1c95080d9e6a", + "trace_id": "b0858dca-4173-493b-a6c5-4e993f606477", + "parent_span_id": "21c8befa-356a-4034-8d62-792238cb9e6e", + "name": "agent.web_search_evaluation_metrics", + "start_time": "2025-10-14 12:53:43.908000000", + "end_time": "2025-10-14 12:53:43.908000000", + "duration_ms": 0.037, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "web_search_evaluation_metrics", + "agent.capabilities": [ + "web_search_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What are the latest updates in Python programming?", + "query_length": 50, + "query_type": "web_search_evaluation_query", + "web_search.results_count": 1, + "web_search.context_length": 152, + "web_search.context_synthesized": "Context from 1 web sources", + "web_search.search_effectiveness": 0.2, + "web_search.context_quality": "low", + "web_search.source_diversity": 1, + "web_search.information_synthesis": "low", + "web_search.external_knowledge_utilization": 0.152, + "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What+are+the+latest+updates+in+Python+programming?\n\nUser Question: What are the latest updates in Python programming?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", + "prompt.prompt_length": 647, + "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What+are+the+latest+updates+in+Python+programming?", + "prompt.instruction_following": "web_search_optimized", + "response.answer_length": 1324, + "response.answer_completeness": "complete", + "response.response_quality": "medium", + "response.source_citation": 1, + "response.context_utilization": 8.710526315789474, + "output_response": "Web Search Answer: The web search did not yield specific results regarding the latest updates in Python programming. However, I can provide a general overview based on my knowledge up to October 2023.\n\nAs of late 2023, ...", + "evaluation.search_effectiveness": 0.2, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 1, + "evaluation.context_relevance": "low", + "evaluation.overall_quality": "low", + "evaluation.ready_for_production": true, + "web_search.search_strategy": "duckduckgo_api", + "web_search.real_time_data": true, + "web_search.external_sources": 1, + "web_search.information_freshness": "current", + "web_search.knowledge_synthesis": "single_source" + }, + "events": [], + "links": [] + } + ], + "created_at": "2025-10-14 12:53:37.818000000", + "updated_at": "2025-10-14 12:53:37.818000000" + }, + { + "trace_id": "dc81be1a-e93d-4f60-90f3-ce1acaa8cb3e", + "name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "status": "ok", + "status_message": "", + "start_time": "2025-10-14 12:53:34.584000000", + "end_time": "2025-10-14 12:53:36.780000000", + "duration_ms": 2196.4579999999996, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "spans": [ + { + "span_id": "9dd5ac2c-0deb-48e3-ac8b-0e6e31f7ff30", + "trace_id": "dc81be1a-e93d-4f60-90f3-ce1acaa8cb3e", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-14 12:53:34.584000000", + "end_time": "2025-10-14 12:53:36.780000000", + "duration_ms": 2196.351, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What happened in tech news this week?", + "query.length": 37, + "processing.mode": "Web Search", + "processing.time_seconds": 2.1948978900909424, + "processing.response_length": 324, + "processing.sources_count": 1, + "output_response": "Final Answer: I'm sorry, but I couldn't find any specific information regarding tech news from this week based on the search results. It appears that there were no relevant articles or updates available. If you're ...", + "final_answer_mode": "Web Search", + "query_processed.input_query": "What happened in tech news this week?", + "query_processed.output_response": "Successfully processed query using Web Search, generated 324 character response", + "query_processed.mode": "Web Search", + "query_processed.processing_time": 2.1948978900909424, + "query_processed.response_length": 324 + }, + "events": [], + "links": [] + }, + { + "span_id": "b1cbafce-51fb-48fd-bc4a-d49081a2f374", + "trace_id": "dc81be1a-e93d-4f60-90f3-ce1acaa8cb3e", + "parent_span_id": "9dd5ac2c-0deb-48e3-ac8b-0e6e31f7ff30", + "name": "agent.query_routing", + "start_time": "2025-10-14 12:53:34.584000000", + "end_time": "2025-10-14 12:53:36.778000000", + "duration_ms": 2194.2200000000003, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What happened in tech news this week?", + "query_length": 37, + "query_type": "routing_query", + "output_response": "Routed to Web Search for query processing", + "router.classification": "Web Search", + "router.confidence_score": 1, + "router.rag_keyword_score": 0, + "router.web_keyword_score": 2, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [] + }, + { + "span_id": "e3064687-1b18-478b-9e9d-4d081a6dac23", + "trace_id": "dc81be1a-e93d-4f60-90f3-ce1acaa8cb3e", + "parent_span_id": "b1cbafce-51fb-48fd-bc4a-d49081a2f374", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-14 12:53:34.584000000", + "end_time": "2025-10-14 12:53:34.584000000", + "duration_ms": 0.028, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What happened in tech news this week?", + "query_length": 37, + "query_type": "routing_evaluation_query", + "classification.mode": "Web Search", + "classification.rag_keyword_score": 0, + "classification.web_keyword_score": 2, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "medium", + "query.intent": "general_knowledge", + "query.keyword_density": 0.2857142857142857, + "query.domain_affinity": "general", + "routing.decision": "Routed to Web Search based on analysis", + "routing.rationale": "RAG score: 0, Web score: 2, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "RAG", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.038461538461538464, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: Web Search (Confidence: 1.00)" + }, + "events": [], + "links": [] + }, + { + "span_id": "eb9dbc11-44b3-4041-8c83-0f33b8b761bb", + "trace_id": "dc81be1a-e93d-4f60-90f3-ce1acaa8cb3e", + "parent_span_id": "b1cbafce-51fb-48fd-bc4a-d49081a2f374", + "name": "agent.web_search_generation", + "start_time": "2025-10-14 12:53:34.584000000", + "end_time": "2025-10-14 12:53:36.774000000", + "duration_ms": 2189.866, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "web_search_agent", + "agent.operation": "web_search_generation", + "agent.capabilities": [ + "web_search", + "content_synthesis", + "response_generation" + ], + "agent.id": "noveum_web_search_agent", + "input_query": "What happened in tech news this week?", + "query_length": 37, + "query_type": "web_search_query", + "output_response": "Web Search Answer: I'm sorry, but I couldn't find any specific information regarding tech news from this week based on the search results. It appears that there were no relevant articles or updates available. If you're ...", + "web_search.results_count": 1, + "web_search.context_length": 139, + "web_search.response_length": 324, + "web_search.mode": "external_web_search" + }, + "events": [], + "links": [] + }, + { + "span_id": "c8ee237b-ab5e-4b36-83f0-2f8478971898", + "trace_id": "dc81be1a-e93d-4f60-90f3-ce1acaa8cb3e", + "parent_span_id": "eb9dbc11-44b3-4041-8c83-0f33b8b761bb", + "name": "agent.llm_model_execution", + "start_time": "2025-10-14 12:53:34.981000000", + "end_time": "2025-10-14 12:53:36.768000000", + "duration_ms": 1786.467, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What happened in tech news this week?", + "query_length": 37, + "query_type": "web_search_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 1.785179853439331, + "model.latency_ms": 1785.179853439331, + "model.start_time": 1760446414.981964, + "model.end_time": 1760446416.767144, + "model.performance_tier": "fast", + "model.prompt_tokens": 155, + "model.completion_tokens": 81, + "model.total_tokens": 236, + "model.tokens_per_second": 132.19956495997977, + "model.estimated_cost": 0.00708, + "model.efficiency_score": 1.3728813559322033, + "model.response_length": 324, + "model.response_quality": "high", + "model.output_response": "Model Response: I'm sorry, but I couldn't find any specific information regarding tech news from this week based on the search results. It appears that there were no relevant articles or updates available. If you're ...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [] + }, + { + "span_id": "9c8e1a08-61c8-409e-a79d-41c74a8d1505", + "trace_id": "dc81be1a-e93d-4f60-90f3-ce1acaa8cb3e", + "parent_span_id": "eb9dbc11-44b3-4041-8c83-0f33b8b761bb", + "name": "agent.web_search_evaluation_metrics", + "start_time": "2025-10-14 12:53:36.772000000", + "end_time": "2025-10-14 12:53:36.774000000", + "duration_ms": 1.5939999999999999, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "web_search_evaluation_metrics", + "agent.capabilities": [ + "web_search_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What happened in tech news this week?", + "query_length": 37, + "query_type": "web_search_evaluation_query", + "web_search.results_count": 1, + "web_search.context_length": 139, + "web_search.context_synthesized": "Context from 1 web sources", + "web_search.search_effectiveness": 0.2, + "web_search.context_quality": "low", + "web_search.source_diversity": 1, + "web_search.information_synthesis": "low", + "web_search.external_knowledge_utilization": 0.139, + "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What+happened+in+tech+news+this+week?\n\nUser Question: What happened in tech news this week?\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", + "prompt.prompt_length": 621, + "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=What+happened+in+tech+news+this+week?", + "prompt.instruction_following": "web_search_optimized", + "response.answer_length": 324, + "response.answer_completeness": "complete", + "response.response_quality": "medium", + "response.source_citation": 1, + "response.context_utilization": 2.3309352517985613, + "output_response": "Web Search Answer: I'm sorry, but I couldn't find any specific information regarding tech news from this week based on the search results. It appears that there were no relevant articles or updates available. If you're ...", + "evaluation.search_effectiveness": 0.2, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 1, + "evaluation.context_relevance": "low", + "evaluation.overall_quality": "low", + "evaluation.ready_for_production": true, + "web_search.search_strategy": "duckduckgo_api", + "web_search.real_time_data": true, + "web_search.external_sources": 1, + "web_search.information_freshness": "current", + "web_search.knowledge_synthesis": "single_source" + }, + "events": [], + "links": [] + } + ], + "created_at": "2025-10-14 12:53:34.584000000", + "updated_at": "2025-10-14 12:53:34.584000000" + }, + { + "trace_id": "945b2757-8a5a-4aab-a869-8f44c02166a5", + "name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "status": "ok", + "status_message": "", + "start_time": "2025-10-14 12:53:27.771000000", + "end_time": "2025-10-14 12:53:33.575000000", + "duration_ms": 5804.15, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "spans": [ + { + "span_id": "59595232-8873-419c-9b6a-0727445a2afa", + "trace_id": "945b2757-8a5a-4aab-a869-8f44c02166a5", + "parent_span_id": "dfde2f79-d0c1-481e-819d-60730cb60fe5", + "name": "agent.query_routing", + "start_time": "2025-10-14 12:53:27.771000000", + "end_time": "2025-10-14 12:53:33.575000000", + "duration_ms": 5803.9800000000005, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "What are the current trends in observability tools?", + "query_length": 51, + "query_type": "routing_query", + "output_response": "Routed to RAG for query processing", + "router.classification": "RAG", + "router.confidence_score": 0.3333333333333333, + "router.rag_keyword_score": 2, + "router.web_keyword_score": 1, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [] + }, + { + "span_id": "a59b3296-aaba-4642-b5a0-a271787a5607", + "trace_id": "945b2757-8a5a-4aab-a869-8f44c02166a5", + "parent_span_id": "59595232-8873-419c-9b6a-0727445a2afa", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-14 12:53:27.771000000", + "end_time": "2025-10-14 12:53:27.771000000", + "duration_ms": 0.031, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: What are the current trends in observability tools?", + "query_length": 51, + "query_type": "routing_evaluation_query", + "classification.mode": "RAG", + "classification.rag_keyword_score": 2, + "classification.web_keyword_score": 1, + "classification.confidence_score": 0.3333333333333333, + "classification.confidence_level": "medium", + "classification.method": "keyword_based", + "query.complexity": "complex", + "query.intent": "documentation", + "query.keyword_density": 0.375, + "query.domain_affinity": "noveum", + "routing.decision": "Routed to RAG based on analysis", + "routing.rationale": "RAG score: 2, Web score: 1, Confidence: 0.33", + "routing.expected_performance": "medium", + "routing.alternative_mode": "Web Search", + "routing.decision_confidence": 0.3333333333333333, + "evaluation.routing_accuracy": "medium", + "evaluation.keyword_coverage": 0.057692307692307696, + "evaluation.query_understanding": "ambiguous", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: RAG (Confidence: 0.33)" + }, + "events": [], + "links": [] + }, + { + "span_id": "dfde2f79-d0c1-481e-819d-60730cb60fe5", + "trace_id": "945b2757-8a5a-4aab-a869-8f44c02166a5", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-14 12:53:27.771000000", + "end_time": "2025-10-14 12:53:33.575000000", + "duration_ms": 5804.074, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "What are the current trends in observability tools?", + "query.length": 51, + "processing.mode": "RAG", + "processing.time_seconds": 5.804032802581787, + "processing.response_length": 997, + "processing.sources_count": 5, + "output_response": "Final Answer: The provided context does not specifically outline current trends in observability tools in general. However, it highlights the unique challenges faced by AI applications and how traditional monitorin...", + "final_answer_mode": "RAG", + "query_processed.input_query": "What are the current trends in observability tools?", + "query_processed.output_response": "Successfully processed query using RAG, generated 997 character response", + "query_processed.mode": "RAG", + "query_processed.processing_time": 5.804032802581787, + "query_processed.response_length": 997 + }, + "events": [], + "links": [] + }, + { + "span_id": "82489444-deba-46d2-b08b-c09754189566", + "trace_id": "945b2757-8a5a-4aab-a869-8f44c02166a5", + "parent_span_id": "59595232-8873-419c-9b6a-0727445a2afa", + "name": "agent.llm-rag", + "start_time": "2025-10-14 12:53:27.772000000", + "end_time": "2025-10-14 12:53:33.575000000", + "duration_ms": 5803.534, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "rag_agent", + "agent.operation": "llm-rag", + "agent.capabilities": [ + "document_retrieval", + "context_generation", + "response_generation" + ], + "agent.id": "noveum_rag_agent", + "input_query": "What are the current trends in observability tools?", + "query_length": 51, + "query_type": "rag_query", + "output_response": "RAG Answer: The provided context does not specifically outline current trends in observability tools in general. However, it highlights the unique challenges faced by AI applications and how traditional monitorin...", + "rag.context_length": 2943, + "rag.sources_count": 5, + "rag.answer_length": 997, + "rag.mode": "retrieval_augmented_generation" + }, + "events": [], + "links": [] + }, + { + "span_id": "34b1f14c-3fd3-4f89-b4dd-6150a52befd7", + "trace_id": "945b2757-8a5a-4aab-a869-8f44c02166a5", + "parent_span_id": "82489444-deba-46d2-b08b-c09754189566", + "name": "agent.llm_model_execution", + "start_time": "2025-10-14 12:53:28.257000000", + "end_time": "2025-10-14 12:53:33.162000000", + "duration_ms": 4905.2429999999995, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: What are the current trends in observability tools?", + "query_length": 51, + "query_type": "rag_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 4.9052040576934814, + "model.latency_ms": 4905.204057693481, + "model.start_time": 1760446408.2575018, + "model.end_time": 1760446413.162706, + "model.performance_tier": "medium", + "model.prompt_tokens": 872, + "model.completion_tokens": 249, + "model.total_tokens": 1121, + "model.tokens_per_second": 228.53279635569638, + "model.estimated_cost": 0.03363, + "model.efficiency_score": 0.8893844781445138, + "model.response_length": 997, + "model.response_quality": "high", + "model.output_response": "Model Response: The provided context does not specifically outline current trends in observability tools in general. However, it highlights the unique challenges faced by AI applications and how traditional monitorin...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [] + }, + { + "span_id": "d5d2b966-3c37-4d6d-b7b8-c58c9f22aad9", + "trace_id": "945b2757-8a5a-4aab-a869-8f44c02166a5", + "parent_span_id": "82489444-deba-46d2-b08b-c09754189566", + "name": "agent.rag_evaluation_metrics", + "start_time": "2025-10-14 12:53:33.162000000", + "end_time": "2025-10-14 12:53:33.162000000", + "duration_ms": 0.036000000000000004, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "rag_evaluation_metrics", + "agent.capabilities": [ + "retrieval_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: What are the current trends in observability tools?", + "query_length": 51, + "query_type": "rag_evaluation_query", + "retrieval.context_retrieved": "Context: Source 1 (https://noveum.ai/en/blog/from-logs-to-intelligent-choices-inside-noveum-ais-evaluation-process):\nFrom Development to Production - Inside Noveum.ai's AI Observability Platform\nShashank Agarwal\n3/3/2025\nIntroduction\nEvery AI application tells a story\u2014but most developers never get to hear it...", + "retrieval.context_length": 2943, + "retrieval.sources_count": 5, + "retrieval.context_quality": "high", + "retrieval.effectiveness": 1, + "retrieval.context_utilization": 2.943, + "prompt.complete_prompt": "You are a helpful assistant for Noveum.ai. Answer the user's question based on the provided context from Noveum's documentation.\n\nContext from Noveum documentation:\nSource 1 (https://noveum.ai/en/blog/from-logs-to-intelligent-choices-inside-noveum-ais-evaluation-process):\nFrom Development to Production - Inside Noveum.ai's AI Observability Platform\nShashank Agarwal\n3/3/2025\nIntroduction\nEvery AI application tells a story\u2014but most developers never get to hear it. When your RAG pipeline returns irrelevant results, when your multi-agent system gets stuck in loops, or when your LLM costs suddenly spike, you're left guessing what went wrong and where.\nThat's exactly the challenge Noveum.ai solves. Rather than flying blind, our platform provides comprehensive tracing a...\n\nSource 2 (https://noveum.ai/docs/getting-started/tracing-concepts):\n- Explore Framework Integrations - Framework-specific guidance\n- Learn Advanced Patterns - Custom instrumentation techniques\n- Master the Dashboard - Analyze your traces effectively\nRemember: Good observability is not about collecting all possible data, but about collecting the right data that helps you understand, debug, and optimize your AI applications.\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to O...\n\nSource 3 (https://noveum.ai/en/docs/getting-started/tracing-concepts):\n- Explore Framework Integrations - Framework-specific guidance\n- Learn Advanced Patterns - Custom instrumentation techniques\n- Master the Dashboard - Analyze your traces effectively\nRemember: Good observability is not about collecting all possible data, but about collecting the right data that helps you understand, debug, and optimize your AI applications.\nGet Early Access to Noveum.ai Platform\nBe the first one to get notified when we open Noveum Platform to more users. All users get access to O...\n\nSource 4 (https://noveum.ai/en/blog/from-logs-to-intelligent-choices-inside-noveum-ais-evaluation-process):\nThe AI Observability Challenge\nWhy Traditional Monitoring Falls Short\nTraditional application monitoring tools weren't built for AI applications. They can tell you if your API is responding, but they can't answer the questions that matter most for AI systems:\n- Why did my RAG pipeline retrieve irrelevant documents?\n- Which LLM calls are driving my costs?\n- How are my agents communicating with each other?\n- What's causing hallucinations in my responses?\n- Why is my embedding generation so slow?\nW...\n\nSource 5 (https://noveum.ai/en/docs/advanced/rag-observability):\n\ud83d\udd27 Best Practices for RAG Observability\n1. Comprehensive Pipeline Tracing\n2. Quality Monitoring\n3. A/B Testing for RAG Components\n\ud83c\udfaf Advanced RAG Patterns\nMulti-Modal RAG\nConversational RAG\nRAG observability with Noveum.ai provides the deep insights needed to build, optimize, and scale retrieval-augmented generation systems. By implementing comprehensive tracing across retrieval, context preparation, and generation stages, you can ensure your RAG pipeline delivers accurate, relevant, and cost-effe...\n\nUser Question: What are the current trends in observability tools?\n\nInstructions:\n1. Answer based primarily on the provided context\n2. If the context doesn't contain enough information, say so clearly\n3. Be specific and cite sources when possible\n4. Keep responses concise but informative\n5. If the question is not related to Noveum, politely redirect to ask about Noveum\n\nAnswer:", + "prompt.prompt_length": 3490, + "prompt.context_injection": "Context injected: Source 1 (https://noveum.ai/en/blog/from-logs-to-intelligent-choices-inside-noveum-ais-evaluation-process):\nFrom Development to Production - Inside Noveum.ai's AI Observability Platform\nShashank Agarw...", + "prompt.instruction_following": "rag_optimized", + "response.answer_length": 997, + "response.answer_completeness": "complete", + "response.response_quality": "high", + "response.source_citation": 5, + "response.context_utilization": 0.33876996262317366, + "output_response": "RAG Answer: The provided context does not specifically outline current trends in observability tools in general. However, it highlights the unique challenges faced by AI applications and how traditional monitorin...", + "evaluation.retrieval_effectiveness": 1, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 5, + "evaluation.context_relevance": "high", + "evaluation.overall_quality": "high", + "evaluation.ready_for_production": true, + "rag.retrieval_strategy": "semantic_similarity", + "rag.vector_search_results": 5, + "rag.context_synthesis": "multi_source", + "rag.document_coverage": 1, + "rag.information_density": 0.33876996262317366 + }, + "events": [], + "links": [] + } + ], + "created_at": "2025-10-14 12:53:27.771000000", + "updated_at": "2025-10-14 12:53:27.771000000" + }, + { + "trace_id": "4bdd5814-acbb-4de7-a56b-543b87a1cfa5", + "name": "auto_trace_tool-orchestator", + "project": "noveum-ai-agent-rag-websearch", + "environment": "development", + "status": "ok", + "status_message": "", + "start_time": "2025-10-14 12:53:20.120000000", + "end_time": "2025-10-14 12:53:26.750000000", + "duration_ms": 6630.38, + "span_count": 6, + "error_count": 0, + "sdk": { + "name": "noveum-trace-python", + "version": "0.3.8" + }, + "attributes": { + "noveum.project": "noveum-ai-agent-rag-websearch", + "noveum.environment": "development", + "noveum.sdk.version": "0.3.8", + "noveum.sampling.decision": "record" + }, + "metadata": { + "user_id": null, + "session_id": null, + "request_id": null, + "tags": {}, + "custom_attributes": {} + }, + "spans": [ + { + "span_id": "11c07411-df30-42a4-9062-90a032ef75ed", + "trace_id": "4bdd5814-acbb-4de7-a56b-543b87a1cfa5", + "parent_span_id": "ec530838-a7cf-477b-9186-a70114c8a12f", + "name": "agent.web_search_generation", + "start_time": "2025-10-14 12:53:20.120000000", + "end_time": "2025-10-14 12:53:26.750000000", + "duration_ms": 6629.273999999999, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "web_search_agent", + "agent.operation": "web_search_generation", + "agent.capabilities": [ + "web_search", + "content_synthesis", + "response_generation" + ], + "agent.id": "noveum_web_search_agent", + "input_query": "Tell me about recent developments in machine learning", + "query_length": 53, + "query_type": "web_search_query", + "output_response": "Web Search Answer: The web search did not yield specific results regarding recent developments in machine learning. However, I can provide a general overview based on my knowledge up to October 2023.\n\nRecent development...", + "web_search.results_count": 1, + "web_search.context_length": 155, + "web_search.response_length": 1849, + "web_search.mode": "external_web_search" + }, + "events": [], + "links": [] + }, + { + "span_id": "282b121f-61dd-4dc5-9e55-cecc3f02f688", + "trace_id": "4bdd5814-acbb-4de7-a56b-543b87a1cfa5", + "parent_span_id": "", + "name": "tool-orchestator", + "start_time": "2025-10-14 12:53:20.120000000", + "end_time": "2025-10-14 12:53:26.750000000", + "duration_ms": 6630.264, + "status": "ok", + "status_message": "", + "attributes": { + "input_query": "Tell me about recent developments in machine learning", + "query.length": 53, + "processing.mode": "Web Search", + "processing.time_seconds": 6.629686117172241, + "processing.response_length": 1849, + "processing.sources_count": 1, + "output_response": "Final Answer: The web search did not yield specific results regarding recent developments in machine learning. However, I can provide a general overview based on my knowledge up to October 2023.\n\nRecent development...", + "final_answer_mode": "Web Search", + "query_processed.input_query": "Tell me about recent developments in machine learning", + "query_processed.output_response": "Successfully processed query using Web Search, generated 1849 character response", + "query_processed.mode": "Web Search", + "query_processed.processing_time": 6.629686117172241, + "query_processed.response_length": 1849 + }, + "events": [], + "links": [] + }, + { + "span_id": "ec530838-a7cf-477b-9186-a70114c8a12f", + "trace_id": "4bdd5814-acbb-4de7-a56b-543b87a1cfa5", + "parent_span_id": "282b121f-61dd-4dc5-9e55-cecc3f02f688", + "name": "agent.query_routing", + "start_time": "2025-10-14 12:53:20.120000000", + "end_time": "2025-10-14 12:53:26.750000000", + "duration_ms": 6629.59, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "query_router", + "agent.operation": "query_routing", + "agent.capabilities": [ + "query_classification", + "routing_decision" + ], + "agent.id": "noveum_query_router", + "input_query": "Tell me about recent developments in machine learning", + "query_length": 53, + "query_type": "routing_query", + "output_response": "Routed to Web Search for query processing", + "router.classification": "Web Search", + "router.confidence_score": 1, + "router.rag_keyword_score": 0, + "router.web_keyword_score": 1, + "router.mode": "intelligent_routing" + }, + "events": [], + "links": [] + }, + { + "span_id": "df882f90-cf1a-42a6-a2ae-78b2a13f27a1", + "trace_id": "4bdd5814-acbb-4de7-a56b-543b87a1cfa5", + "parent_span_id": "ec530838-a7cf-477b-9186-a70114c8a12f", + "name": "agent.routing_evaluation_metrics", + "start_time": "2025-10-14 12:53:20.120000000", + "end_time": "2025-10-14 12:53:20.120000000", + "duration_ms": 0.082, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "routing_evaluation_metrics", + "agent.capabilities": [ + "routing_analysis", + "decision_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Routing evaluation for query: Tell me about recent developments in machine learning", + "query_length": 53, + "query_type": "routing_evaluation_query", + "classification.mode": "Web Search", + "classification.rag_keyword_score": 0, + "classification.web_keyword_score": 1, + "classification.confidence_score": 1, + "classification.confidence_level": "high", + "classification.method": "keyword_based", + "query.complexity": "complex", + "query.intent": "general_knowledge", + "query.keyword_density": 0.125, + "query.domain_affinity": "general", + "routing.decision": "Routed to Web Search based on analysis", + "routing.rationale": "RAG score: 0, Web score: 1, Confidence: 1.00", + "routing.expected_performance": "high", + "routing.alternative_mode": "RAG", + "routing.decision_confidence": 1, + "evaluation.routing_accuracy": "high", + "evaluation.keyword_coverage": 0.019230769230769232, + "evaluation.query_understanding": "clear", + "evaluation.ready_for_production": true, + "router.classification_strategy": "hybrid_keyword_llm", + "router.keyword_matching": "used", + "router.llm_fallback": "not_needed", + "router.decision_time": "instant", + "output_response": "Routing Decision: Web Search (Confidence: 1.00)" + }, + "events": [], + "links": [] + }, + { + "span_id": "bea819fd-3673-480b-9f7b-a3a2efef4218", + "trace_id": "4bdd5814-acbb-4de7-a56b-543b87a1cfa5", + "parent_span_id": "11c07411-df30-42a4-9062-90a032ef75ed", + "name": "agent.llm_model_execution", + "start_time": "2025-10-14 12:53:20.455000000", + "end_time": "2025-10-14 12:53:26.746000000", + "duration_ms": 6291.401, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "model_details", + "agent.operation": "llm_model_execution", + "agent.capabilities": [ + "model_invocation", + "parameter_tracking", + "latency_measurement" + ], + "agent.id": "noveum_model_details", + "input_query": "Model execution for query: Tell me about recent developments in machine learning", + "query_length": 53, + "query_type": "web_search_model_query", + "model.name": "gpt-4o-mini", + "model.temperature": 0.1, + "model.max_tokens": null, + "model.top_p": null, + "model.frequency_penalty": null, + "model.presence_penalty": null, + "model.provider": "openai", + "model.type": "chat_completion", + "model.version": "gpt-4o-mini", + "model.latency_seconds": 6.290812253952026, + "model.latency_ms": 6290.812253952026, + "model.start_time": 1760446400.4555368, + "model.end_time": 1760446406.746349, + "model.performance_tier": "slow", + "model.prompt_tokens": 163, + "model.completion_tokens": 462, + "model.total_tokens": 625, + "model.tokens_per_second": 99.35124031199013, + "model.estimated_cost": 0.01875, + "model.efficiency_score": 2.9584, + "model.response_length": 1849, + "model.response_quality": "high", + "model.output_response": "Model Response: The web search did not yield specific results regarding recent developments in machine learning. However, I can provide a general overview based on my knowledge up to October 2023.\n\nRecent development...", + "model.config": { + "name": "gpt-4o-mini", + "temperature": 0.1, + "max_tokens": null, + "top_p": null, + "frequency_penalty": null, + "presence_penalty": null, + "provider": "openai", + "type": "chat_completion" + } + }, + "events": [], + "links": [] + }, + { + "span_id": "6e62af18-569c-4783-b2ae-177c58542bd0", + "trace_id": "4bdd5814-acbb-4de7-a56b-543b87a1cfa5", + "parent_span_id": "11c07411-df30-42a4-9062-90a032ef75ed", + "name": "agent.web_search_evaluation_metrics", + "start_time": "2025-10-14 12:53:26.749000000", + "end_time": "2025-10-14 12:53:26.750000000", + "duration_ms": 0.217, + "status": "ok", + "status_message": "", + "attributes": { + "agent.type": "other_details", + "agent.operation": "web_search_evaluation_metrics", + "agent.capabilities": [ + "web_search_analysis", + "response_evaluation", + "quality_assessment" + ], + "agent.id": "noveum_other_details", + "input_query": "Evaluation for query: Tell me about recent developments in machine learning", + "query_length": 53, + "query_type": "web_search_evaluation_query", + "web_search.results_count": 1, + "web_search.context_length": 155, + "web_search.context_synthesized": "Context from 1 web sources", + "web_search.search_effectiveness": 0.2, + "web_search.context_quality": "low", + "web_search.source_diversity": 1, + "web_search.information_synthesis": "low", + "web_search.external_knowledge_utilization": 0.155, + "prompt.complete_prompt": "You are a helpful assistant. Answer the user's question based on the provided web search results.\n\nWeb Search Results:\nSource 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=Tell+me+about+recent+developments+in+machine+learning\n\nUser Question: Tell me about recent developments in machine learning\n\nInstructions:\n1. Answer based on the provided web search results\n2. Synthesize information from multiple sources when relevant\n3. Be informative and accurate\n4. If the results don't contain enough information, say so clearly\n5. Keep responses concise but comprehensive\n6. Cite sources when possible\n\nAnswer:", + "prompt.prompt_length": 653, + "prompt.context_injection": "Web context injected: Source 1 - Search Result 1:\nNo good DuckDuckGo Search Result was found\nURL: https://duckduckgo.com/?q=Tell+me+about+recent+developments+in+machine+learning", + "prompt.instruction_following": "web_search_optimized", + "response.answer_length": 1849, + "response.answer_completeness": "complete", + "response.response_quality": "medium", + "response.source_citation": 1, + "response.context_utilization": 11.929032258064517, + "output_response": "Web Search Answer: The web search did not yield specific results regarding recent developments in machine learning. However, I can provide a general overview based on my knowledge up to October 2023.\n\nRecent development...", + "evaluation.search_effectiveness": 0.2, + "evaluation.response_completeness": "complete", + "evaluation.source_diversity": 1, + "evaluation.context_relevance": "low", + "evaluation.overall_quality": "low", + "evaluation.ready_for_production": true, + "web_search.search_strategy": "duckduckgo_api", + "web_search.real_time_data": true, + "web_search.external_sources": 1, + "web_search.information_freshness": "current", + "web_search.knowledge_synthesis": "single_source" + }, + "events": [], + "links": [] + } + ], + "created_at": "2025-10-14 12:53:20.120000000", + "updated_at": "2025-10-14 12:53:20.120000000" + } + ], + "pagination": { + "total": 321, + "limit": 50, + "offset": 0, + "has_more": true + }, + "timestamp": "2025-10-22T17:00:36.145Z", + "processing_time_ms": 137 +} \ No newline at end of file diff --git a/noveum_customer_support_bt/upload_dataset.py b/noveum_customer_support_bt/upload_dataset.py new file mode 100644 index 0000000..33ec2c0 --- /dev/null +++ b/noveum_customer_support_bt/upload_dataset.py @@ -0,0 +1,213 @@ +#!/usr/bin/env python3 +""" +Script to upload dataset items to Noveum API. +Reads a JSON file containing a list of dataset items and uploads them via POST request. +""" + +import os +import json +import requests +import argparse +from dotenv import load_dotenv +from typing import List, Dict, Any + +# Load environment variables +load_dotenv() + +# Default dataset JSON path +dataset_json = 'processed_agent_dataset.json' + +# Get API credentials from environment +api_key = os.getenv('NOVEUM_API_KEY') +org_slug = os.getenv('NOVEUM_ORG_SLUG') +dataset_slug = os.getenv('NOVEUM_DATASET_SLUG') +latest_version = os.getenv('LATEST_VERSION') +beta_env = os.getenv('BETA', 'false').lower() == 'true' + +def validate_environment(): + """Validate that all required environment variables are set""" + required_vars = { + 'NOVEUM_API_KEY': api_key, + 'NOVEUM_ORG_SLUG': org_slug, + 'NOVEUM_DATASET_SLUG': dataset_slug, + 'LATEST_VERSION': latest_version + } + + missing_vars = [var for var, value in required_vars.items() if not value] + + if missing_vars: + print(f"Error: Missing required environment variables: {', '.join(missing_vars)}") + print("Please set these variables in your .env file or environment") + return False + + return True + +def load_dataset_items(file_path: str) -> List[Dict[str, Any]]: + """Load dataset items from JSON file""" + try: + with open(file_path, 'r', encoding='utf-8') as f: + data = json.load(f) + + if not isinstance(data, list): + print(f"Error: JSON file should contain a list of objects, got {type(data)}") + return [] + + print(f"Loaded {len(data)} items from {file_path}") + return data + + except FileNotFoundError: + print(f"Error: File not found: {file_path}") + return [] + except json.JSONDecodeError as e: + print(f"Error: Invalid JSON in file {file_path}: {e}") + return [] + except (OSError, IOError) as e: + print(f"Error loading dataset: {e}") + return [] + +def upload_dataset_items(items: List[Dict[str, Any]], version: str, item_type: str = "conversation") -> bool: + """Upload dataset items to Noveum API""" + if not items: + print("No items to upload") + return False + + # Schema keys from schema.tsx - these will be surfaced at the same level as content + schema_keys = { + "item_id", "dataset_id", "item_key", "item_hash", "organization_id", + "organization_slug", "dataset_slug", "item_version", "deleted_at_version", + "deleted_at_date", "item_type", "schema_version", "source_trace_id", + "source_span_id", "content", "metadata", "agent_name", "agent_role", + "agent_task", "agent_response", "system_prompt", "user_id", "session_id", + "turn_id", "ground_truth", "expected_tool_call", "tools_available", + "tool_calls", "tool_call_results", "parameters_passed", "retrieval_query", + "retrieved_context", "exit_status", "agent_exit", "trace_data", + "conversation_id", "speaker", "message", "conversation_context", + "input_text", "output_text", "expected_output", "evaluation_context", + "criteria", "quality_score", "validation_status", "validation_errors", + "tags", "custom_attributes", "created_at", "updated_at" + } + + # Transform items to the required format + transformed_items = [] + for item in items: + # Create a copy of the item to avoid modifying the original + item['item_type'] = item_type + item_copy = item.copy() + + # Start with base structure + transformed_item = { + "item_key": item.get("turn_id", ""), + "item_type": item_type, # Use the provided item_type + "metadata": {} # Empty metadata as specified + } + + # Surface schema keys at the same level as content + for key in schema_keys: + if key in item_copy: + value = item_copy.pop(key) + # Handle special cases for required fields + if key == "item_type" and (not value or value == ""): + # Keep our default value if the item's value is empty + continue + elif key == "metadata": + # Ensure metadata is always an object + if isinstance(value, dict): + transformed_item[key] = value + elif isinstance(value, str) and value.strip(): + try: + transformed_item[key] = json.loads(value) + except json.JSONDecodeError: + transformed_item[key] = {} + else: + transformed_item[key] = {} + elif key == "content": + # Handle content field - always ensure it's an object + if isinstance(value, str) and value.strip(): + try: + transformed_item[key] = json.loads(value) + except json.JSONDecodeError: + transformed_item[key] = {} + elif isinstance(value, dict): + transformed_item[key] = value + else: + transformed_item[key] = {} + else: + transformed_item[key] = value + + # Always put the entire original item in content field + transformed_item["content"] = item + transformed_items.append(transformed_item) + + # Construct API URL based on BETA environment variable + if beta_env: + api_url = f"https://noveum.ai/api/v1/datasets/{dataset_slug}/items?organizationSlug={org_slug}" + else: + api_url = f"https://noveum.ai/api/v1/organizations/{org_slug}/datasets/{dataset_slug}/items" + + # Prepare headers + headers = { + 'Content-Type': 'application/json', + 'Authorization': f'Bearer {api_key}', + 'Cookie': f'apiKeyCookie={api_key}' + } + + # Prepare request data + request_data = { + "version": version, + "items": transformed_items + } + + print(f"Uploading {len(items)} items to: {api_url}") + print(f"Organization: {org_slug}") + print(f"Dataset: {dataset_slug}") + print(f"Version: {version}") + + try: + response = requests.post(api_url, headers=headers, json=request_data, timeout=30) + response.raise_for_status() + + print(f"Successfully uploaded {len(items)} items") + print(f"Response status: {response.status_code}") + + # Print response content if available + try: + response_data = response.json() + print(f"Response data: {json.dumps(response_data, indent=2)}") + except json.JSONDecodeError: + print(f"Response text: {response.text}") + + return True + + except requests.exceptions.RequestException as e: + print(f"Error uploading dataset items: {e}") + if hasattr(e, 'response') and e.response is not None: + print(f"Response status: {e.response.status_code}") + print(f"Response text: {e.response.text}") + return False + +def main(): + default_item_type = "conversation" + + parser = argparse.ArgumentParser(description='Upload dataset items to Noveum API') + parser.add_argument('--dataset-json', type=str, default=dataset_json, + help=f'Path to JSON file containing dataset items (default: {dataset_json})') + parser.add_argument('--item-type', type=str, default=default_item_type, + help=f'Item type for the dataset items (default: {default_item_type})') + + args = parser.parse_args() + + # Validate environment variables + if not validate_environment(): + return 1 + + # Load dataset items + items = load_dataset_items(args.dataset_json) + if not items: + return 1 + + # Upload items + success = upload_dataset_items(items, latest_version, args.item_type) + return 0 if success else 1 + +if __name__ == "__main__": + exit(main()) diff --git a/noveum_customer_support_bt/upload_scores.py b/noveum_customer_support_bt/upload_scores.py new file mode 100644 index 0000000..0b8c198 --- /dev/null +++ b/noveum_customer_support_bt/upload_scores.py @@ -0,0 +1,337 @@ +#!/usr/bin/env python3 +""" +Script to upload scorer results to Noveum API. +Reads scores and reasonings from a CSV file and uploads them via API. +""" + +import os +import json +import argparse +import csv +from typing import Dict, List, Optional +import requests +from dotenv import load_dotenv + + +def load_api_data(api_data_path: str) -> Dict[str, str]: + """ + Load api_data.json and create a mapping from item_key to item_id. + + Args: + api_data_path: Path to the api_data.json file + + Returns: + Dictionary mapping item_key to item_id + """ + print(f"Loading API data from {api_data_path}...") + with open(api_data_path, 'r') as f: + data = json.load(f) + + # Create mapping from item_key to item_id + key_to_id = {} + items = data.get('items', []) + + for item in items: + item_key = item.get('item_key') + item_id = item.get('item_id') + if item_key and item_id: + key_to_id[item_key] = item_id + + print(f"Loaded {len(key_to_id)} item mappings") + return key_to_id + + +def read_csv_data( + csv_path: str, + item_key_col: str, + score_col: str, + reasoning_col: str +) -> List[Dict]: + """ + Read CSV file and extract relevant columns. + + Args: + csv_path: Path to the CSV file + item_key_col: Column name for item keys + score_col: Column name for scores + reasoning_col: Column name for reasonings + + Returns: + List of dictionaries with item_key, score, and reasoning + """ + print(f"Reading CSV from {csv_path}...") + results = [] + + with open(csv_path, 'r', encoding='utf-8') as f: + reader = csv.DictReader(f) + + # Verify columns exist + if item_key_col not in reader.fieldnames: + raise ValueError(f"Column '{item_key_col}' not found in CSV. Available columns: {reader.fieldnames}") + if score_col not in reader.fieldnames: + raise ValueError(f"Column '{score_col}' not found in CSV. Available columns: {reader.fieldnames}") + if reasoning_col not in reader.fieldnames: + raise ValueError(f"Column '{reasoning_col}' not found in CSV. Available columns: {reader.fieldnames}") + + for row in reader: + item_key = row[item_key_col] + score = row[score_col] + reasoning = row[reasoning_col] + + # Skip empty rows + if not item_key or not score: + continue + + results.append({ + 'item_key': item_key, + 'score': float(score), + 'reasoning': reasoning + }) + + print(f"Read {len(results)} rows from CSV") + return results + + +def create_batch_payload( + csv_data: List[Dict], + key_to_id: Dict[str, str], + org_slug: str, + project: str, + environment: str, + dataset_slug: str, + dataset_version: str, + scorer_id: str = "custom_scorer", + scorer_version: str = "1.0.0" +) -> List[Dict]: + """ + Create the batch payload for API submission. + + Args: + csv_data: List of dictionaries with item_key, score, and reasoning + key_to_id: Mapping from item_key to item_id + org_slug: Organization slug + project: Project name + environment: Environment name + dataset_slug: Dataset slug + dataset_version: Dataset version + scorer_id: Scorer ID (default: "custom_scorer") + scorer_version: Scorer version (default: "1.0.0") + + Returns: + List of result objects ready for API submission + """ + results = [] + skipped = [] + + for row in csv_data: + item_key = row['item_key'] + + # Find corresponding item_id + item_id = key_to_id.get(item_key) + + if not item_id: + skipped.append(item_key) + continue + + result = { + "organizationSlug": org_slug, + "project": project, + "environment": environment, + "datasetSlug": dataset_slug, + "datasetVersion": dataset_version, + "itemId": item_id, + "scorerId": scorer_id, + "scorerVersion": scorer_version, + "score": row['score'], + "passed": row['score'] > 0.5, # Default threshold, can be adjusted + "metadata": { + "details": row['reasoning'] + }, + "executionTimeMs": 0.0 + } + + results.append(result) + + if skipped: + print(f"Warning: Skipped {len(skipped)} rows with missing item_id mappings") + print(f"First few skipped keys: {skipped[:5]}") + + print(f"Created {len(results)} results for upload") + return results + + +def upload_results( + results: List[Dict], + api_key: str, + org_slug: str, + batch_size: int = 100 +) -> None: + """ + Upload results to the API in batches. + + Args: + results: List of result objects + api_key: API key for authentication + org_slug: Organization slug + batch_size: Number of results per batch (default: 100) + """ + api_url = f"https://beta.noveum.ai/api/v1/scorers/results/batch?organizationSlug={org_slug}" + + headers = { + "Content-Type": "application/json", + "Authorization": f"Bearer {api_key}" + } + + # Split results into batches + total = len(results) + batches = [results[i:i + batch_size] for i in range(0, total, batch_size)] + + print(f"\nUploading {total} results in {len(batches)} batches...") + + for i, batch in enumerate(batches, 1): + payload = {"results": batch} + + try: + response = requests.post( + api_url, + headers=headers, + json=payload, + timeout=60 + ) + + if response.status_code == 200: + print(f"āœ“ Batch {i}/{len(batches)} uploaded successfully ({len(batch)} results)") + else: + print(f"āœ— Batch {i}/{len(batches)} failed: {response.status_code}") + print(f" Response: {response.text}") + + except Exception as e: + print(f"āœ— Batch {i}/{len(batches)} error: {str(e)}") + + print("\nUpload complete!") + + +def main(): + parser = argparse.ArgumentParser( + description="Upload scorer results from CSV to Noveum API" + ) + parser.add_argument( + "csv_file", + help="Path to the CSV file containing scores and reasonings" + ) + parser.add_argument( + "--item-key-col", + required=True, + help="Column name for item keys" + ) + parser.add_argument( + "--score-col", + required=True, + help="Column name for scores" + ) + parser.add_argument( + "--reasoning-col", + required=True, + help="Column name for reasonings" + ) + parser.add_argument( + "--api-data", + default="api_data.json", + help="Path to api_data.json file (default: api_data.json)" + ) + parser.add_argument( + "--scorer-id", + default="custom_scorer", + help="Scorer ID (default: custom_scorer)" + ) + parser.add_argument( + "--scorer-version", + default="1.0.0", + help="Scorer version (default: 1.0.0)" + ) + parser.add_argument( + "--batch-size", + type=int, + default=100, + help="Number of results per batch (default: 100)" + ) + parser.add_argument( + "--dry-run", + action="store_true", + help="Prepare data but don't upload to API" + ) + + args = parser.parse_args() + + # Load environment variables + load_dotenv() + print("Loaded environment variables") + + # Get required environment variables + required_vars = { + 'NOVEUM_PROJECT': os.getenv('NOVEUM_PROJECT'), + 'NOVEUM_ENVIRONMENT': os.getenv('NOVEUM_ENVIRONMENT'), + 'NOVEUM_API_KEY': os.getenv('NOVEUM_API_KEY'), + 'NOVEUM_ORG_SLUG': os.getenv('NOVEUM_ORG_SLUG'), + 'NOVEUM_DATASET_SLUG': os.getenv('NOVEUM_DATASET_SLUG'), + 'LATEST_VERSION': os.getenv('LATEST_VERSION') + } + + # Check for missing variables + missing_vars = [var for var, value in required_vars.items() if not value] + if missing_vars: + print(f"Error: Missing required environment variables: {', '.join(missing_vars)}") + print("\nPlease set them in your .env file or environment:") + for var in missing_vars: + print(f" {var}=") + return 1 + + # Load API data + key_to_id = load_api_data(args.api_data) + + # Read CSV data + csv_data = read_csv_data( + args.csv_file, + args.item_key_col, + args.score_col, + args.reasoning_col + ) + + # Create batch payload + results = create_batch_payload( + csv_data=csv_data, + key_to_id=key_to_id, + org_slug=required_vars['NOVEUM_ORG_SLUG'], + project=required_vars['NOVEUM_PROJECT'], + environment=required_vars['NOVEUM_ENVIRONMENT'], + dataset_slug=required_vars['NOVEUM_DATASET_SLUG'], + dataset_version=required_vars['LATEST_VERSION'], + scorer_id=args.scorer_id, + scorer_version=args.scorer_version + ) + + if not results: + print("Error: No valid results to upload") + return 1 + + if args.dry_run: + print("\n--- DRY RUN MODE ---") + print(f"Would upload {len(results)} results") + print("\nSample result:") + print(json.dumps(results[0], indent=2)) + return 0 + + # Upload results + upload_results( + results=results, + api_key=required_vars['NOVEUM_API_KEY'], + org_slug=required_vars['NOVEUM_ORG_SLUG'], + batch_size=args.batch_size + ) + + return 0 + + +if __name__ == "__main__": + exit(main()) + From 1bd3b3da6b89a0bc2d247fd739c814e3101a775b Mon Sep 17 00:00:00 2001 From: Aman India Date: Thu, 23 Oct 2025 12:00:44 +0530 Subject: [PATCH 2/4] Add requirements.txt for core and dataset dependencies; update dataset upload command in Jupyter notebook --- .../noveum_user_bot.ipynb | 6390 +---------------- noveum_customer_support_bt/requirements.txt | 32 + 2 files changed, 273 insertions(+), 6149 deletions(-) create mode 100644 noveum_customer_support_bt/requirements.txt diff --git a/noveum_customer_support_bt/noveum_user_bot.ipynb b/noveum_customer_support_bt/noveum_user_bot.ipynb index 85fc754..f4a20ba 100644 --- a/noveum_customer_support_bt/noveum_user_bot.ipynb +++ b/noveum_customer_support_bt/noveum_user_bot.ipynb @@ -7,8 +7,8 @@ "metadata": {}, "outputs": [], "source": [ - "# Fix: Upload Dataset with Proper Environment Loading\n", - "!cd /Users/mramanindia/work/NovaEval/noveum_customer_support_bt && source .env && python upload_dataset.py --dataset-json split_datasets/agent.rag_evaluation_metrics_dataset.json --item-type conversation\n" + "\n", + "!cd noveum_customer_support_bt && source .env && python upload_dataset.py \n" ] }, { @@ -210,396 +210,10 @@ }, { "cell_type": "code", - "execution_count": 20, - "id": "88c54bec", + "execution_count": null, + "id": "563ea15c", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Collecting requests==2.32.3 (from -r ./noveum_agent_requirements.txt (line 3))\n", - " Using cached requests-2.32.3-py3-none-any.whl.metadata (4.6 kB)\n", - "Collecting beautifulsoup4==4.12.3 (from -r ./noveum_agent_requirements.txt (line 4))\n", - " Using cached beautifulsoup4-4.12.3-py3-none-any.whl.metadata (3.8 kB)\n", - "Collecting python-dotenv==1.0.1 (from -r ./noveum_agent_requirements.txt (line 5))\n", - " Using cached python_dotenv-1.0.1-py3-none-any.whl.metadata (23 kB)\n", - "Collecting langchain==0.3.26 (from -r ./noveum_agent_requirements.txt (line 8))\n", - " Using cached langchain-0.3.26-py3-none-any.whl.metadata (7.8 kB)\n", - "Collecting langchain-community==0.3.18 (from -r ./noveum_agent_requirements.txt (line 9))\n", - " Using cached langchain_community-0.3.18-py3-none-any.whl.metadata (2.4 kB)\n", - "Collecting langchain-core==0.3.66 (from -r ./noveum_agent_requirements.txt (line 10))\n", - " Using cached langchain_core-0.3.66-py3-none-any.whl.metadata (5.8 kB)\n", - "Collecting langchain-openai==0.3.25 (from -r ./noveum_agent_requirements.txt (line 11))\n", - " Using cached langchain_openai-0.3.25-py3-none-any.whl.metadata (2.3 kB)\n", - "Collecting trafilatura>=1.6.4 (from -r ./noveum_agent_requirements.txt (line 14))\n", - " Using cached trafilatura-2.0.0-py3-none-any.whl.metadata (12 kB)\n", - "Collecting lxml>=5.3.0 (from -r ./noveum_agent_requirements.txt (line 15))\n", - " Using cached lxml-6.0.2-cp312-cp312-macosx_10_13_universal2.whl.metadata (3.6 kB)\n", - "Collecting faiss-cpu==1.12.0 (from -r ./noveum_agent_requirements.txt (line 18))\n", - " Using cached faiss_cpu-1.12.0-cp312-cp312-macosx_14_0_arm64.whl.metadata (5.1 kB)\n", - "Collecting duckduckgo-search>=6.1.12 (from -r ./noveum_agent_requirements.txt (line 22))\n", - " Using cached duckduckgo_search-8.1.1-py3-none-any.whl.metadata (16 kB)\n", - "Requirement already satisfied: pandas>=2.2.3 in /Users/mramanindia/work/NovaEval/.venv/lib/python3.12/site-packages (from -r ./noveum_agent_requirements.txt (line 25)) (2.3.3)\n", - "Requirement already satisfied: tqdm>=4.67.1 in /Users/mramanindia/work/NovaEval/.venv/lib/python3.12/site-packages (from -r ./noveum_agent_requirements.txt (line 26)) (4.67.1)\n", - "Requirement already satisfied: noveum_trace>=0.3.5 in /Users/mramanindia/work/NovaEval/.venv/lib/python3.12/site-packages (from -r ./noveum_agent_requirements.txt (line 29)) (0.3.9)\n", - "Collecting jupyter==1.0.0 (from -r ./noveum_agent_requirements.txt (line 32))\n", - " Using cached jupyter-1.0.0-py2.py3-none-any.whl.metadata (995 bytes)\n", - "Collecting ipykernel==6.29.4 (from -r ./noveum_agent_requirements.txt (line 33))\n", - " Using cached ipykernel-6.29.4-py3-none-any.whl.metadata (6.3 kB)\n", - "Requirement already satisfied: charset-normalizer<4,>=2 in /Users/mramanindia/work/NovaEval/.venv/lib/python3.12/site-packages (from requests==2.32.3->-r ./noveum_agent_requirements.txt (line 3)) (3.4.4)\n", - "Requirement already satisfied: idna<4,>=2.5 in /Users/mramanindia/work/NovaEval/.venv/lib/python3.12/site-packages (from requests==2.32.3->-r ./noveum_agent_requirements.txt (line 3)) (3.11)\n", - "Requirement already satisfied: urllib3<3,>=1.21.1 in /Users/mramanindia/work/NovaEval/.venv/lib/python3.12/site-packages (from requests==2.32.3->-r ./noveum_agent_requirements.txt (line 3)) (2.5.0)\n", - "Requirement already satisfied: certifi>=2017.4.17 in /Users/mramanindia/work/NovaEval/.venv/lib/python3.12/site-packages (from requests==2.32.3->-r ./noveum_agent_requirements.txt (line 3)) (2025.10.5)\n", - "Collecting soupsieve>1.2 (from beautifulsoup4==4.12.3->-r ./noveum_agent_requirements.txt (line 4))\n", - " Using cached soupsieve-2.8-py3-none-any.whl.metadata (4.6 kB)\n", - "Collecting langchain-text-splitters<1.0.0,>=0.3.8 (from langchain==0.3.26->-r ./noveum_agent_requirements.txt (line 8))\n", - " Using cached langchain_text_splitters-0.3.11-py3-none-any.whl.metadata (1.8 kB)\n", - "Collecting langsmith>=0.1.17 (from langchain==0.3.26->-r ./noveum_agent_requirements.txt (line 8))\n", - " Downloading langsmith-0.4.37-py3-none-any.whl.metadata (14 kB)\n", - "Requirement already satisfied: pydantic<3.0.0,>=2.7.4 in /Users/mramanindia/work/NovaEval/.venv/lib/python3.12/site-packages (from langchain==0.3.26->-r ./noveum_agent_requirements.txt (line 8)) (2.12.3)\n", - "Collecting SQLAlchemy<3,>=1.4 (from langchain==0.3.26->-r ./noveum_agent_requirements.txt (line 8))\n", - " Using cached sqlalchemy-2.0.44-cp312-cp312-macosx_11_0_arm64.whl.metadata (9.5 kB)\n", - "Requirement already satisfied: PyYAML>=5.3 in /Users/mramanindia/work/NovaEval/.venv/lib/python3.12/site-packages (from langchain==0.3.26->-r ./noveum_agent_requirements.txt (line 8)) (6.0.3)\n", - "Collecting aiohttp<4.0.0,>=3.8.3 (from langchain-community==0.3.18->-r ./noveum_agent_requirements.txt (line 9))\n", - " Using cached aiohttp-3.13.1-cp312-cp312-macosx_11_0_arm64.whl.metadata (8.1 kB)\n", - "Requirement already satisfied: tenacity!=8.4.0,<10,>=8.1.0 in /Users/mramanindia/work/NovaEval/.venv/lib/python3.12/site-packages (from langchain-community==0.3.18->-r ./noveum_agent_requirements.txt (line 9)) (9.1.2)\n", - "Collecting dataclasses-json<0.7,>=0.5.7 (from langchain-community==0.3.18->-r ./noveum_agent_requirements.txt (line 9))\n", - " Using cached dataclasses_json-0.6.7-py3-none-any.whl.metadata (25 kB)\n", - "Requirement already satisfied: pydantic-settings<3.0.0,>=2.4.0 in /Users/mramanindia/work/NovaEval/.venv/lib/python3.12/site-packages (from langchain-community==0.3.18->-r ./noveum_agent_requirements.txt (line 9)) (2.11.0)\n", - "Collecting langsmith>=0.1.17 (from langchain==0.3.26->-r ./noveum_agent_requirements.txt (line 8))\n", - " Using cached langsmith-0.3.45-py3-none-any.whl.metadata (15 kB)\n", - "Collecting httpx-sse<1.0.0,>=0.4.0 (from langchain-community==0.3.18->-r ./noveum_agent_requirements.txt (line 9))\n", - " Using cached httpx_sse-0.4.3-py3-none-any.whl.metadata (9.7 kB)\n", - "Requirement already satisfied: numpy<3,>=1.26.2 in /Users/mramanindia/work/NovaEval/.venv/lib/python3.12/site-packages (from langchain-community==0.3.18->-r ./noveum_agent_requirements.txt (line 9)) (2.3.4)\n", - "Collecting jsonpatch<2.0,>=1.33 (from langchain-core==0.3.66->-r ./noveum_agent_requirements.txt (line 10))\n", - " Using cached jsonpatch-1.33-py2.py3-none-any.whl.metadata (3.0 kB)\n", - "Collecting packaging<25,>=23.2 (from langchain-core==0.3.66->-r ./noveum_agent_requirements.txt (line 10))\n", - " Using cached packaging-24.2-py3-none-any.whl.metadata (3.2 kB)\n", - "Requirement already satisfied: typing-extensions>=4.7 in /Users/mramanindia/work/NovaEval/.venv/lib/python3.12/site-packages (from langchain-core==0.3.66->-r ./noveum_agent_requirements.txt (line 10)) (4.15.0)\n", - "Collecting openai<2.0.0,>=1.86.0 (from langchain-openai==0.3.25->-r ./noveum_agent_requirements.txt (line 11))\n", - " Using cached openai-1.109.1-py3-none-any.whl.metadata (29 kB)\n", - "Collecting tiktoken<1,>=0.7 (from langchain-openai==0.3.25->-r ./noveum_agent_requirements.txt (line 11))\n", - " Using cached tiktoken-0.12.0-cp312-cp312-macosx_11_0_arm64.whl.metadata (6.7 kB)\n", - "Collecting notebook (from jupyter==1.0.0->-r ./noveum_agent_requirements.txt (line 32))\n", - " Using cached notebook-7.4.7-py3-none-any.whl.metadata (10 kB)\n", - "Collecting qtconsole (from jupyter==1.0.0->-r ./noveum_agent_requirements.txt (line 32))\n", - " Using cached qtconsole-5.7.0-py3-none-any.whl.metadata (5.4 kB)\n", - "Collecting jupyter-console (from jupyter==1.0.0->-r ./noveum_agent_requirements.txt (line 32))\n", - " Using cached jupyter_console-6.6.3-py3-none-any.whl.metadata (5.8 kB)\n", - "Collecting nbconvert (from jupyter==1.0.0->-r ./noveum_agent_requirements.txt (line 32))\n", - " Using cached nbconvert-7.16.6-py3-none-any.whl.metadata (8.5 kB)\n", - "Collecting ipywidgets (from jupyter==1.0.0->-r ./noveum_agent_requirements.txt (line 32))\n", - " Using cached ipywidgets-8.1.7-py3-none-any.whl.metadata (2.4 kB)\n", - "Requirement already satisfied: appnope in /Users/mramanindia/work/NovaEval/.venv/lib/python3.12/site-packages (from ipykernel==6.29.4->-r ./noveum_agent_requirements.txt (line 33)) (0.1.4)\n", - "Requirement already satisfied: comm>=0.1.1 in /Users/mramanindia/work/NovaEval/.venv/lib/python3.12/site-packages (from ipykernel==6.29.4->-r ./noveum_agent_requirements.txt (line 33)) (0.2.3)\n", - "Requirement already satisfied: debugpy>=1.6.5 in /Users/mramanindia/work/NovaEval/.venv/lib/python3.12/site-packages (from ipykernel==6.29.4->-r ./noveum_agent_requirements.txt (line 33)) (1.8.17)\n", - "Requirement already satisfied: ipython>=7.23.1 in /Users/mramanindia/work/NovaEval/.venv/lib/python3.12/site-packages (from ipykernel==6.29.4->-r ./noveum_agent_requirements.txt (line 33)) (9.6.0)\n", - "Requirement already satisfied: jupyter-client>=6.1.12 in /Users/mramanindia/work/NovaEval/.venv/lib/python3.12/site-packages (from ipykernel==6.29.4->-r ./noveum_agent_requirements.txt (line 33)) (8.6.3)\n", - "Requirement already satisfied: jupyter-core!=5.0.*,>=4.12 in /Users/mramanindia/work/NovaEval/.venv/lib/python3.12/site-packages (from ipykernel==6.29.4->-r ./noveum_agent_requirements.txt (line 33)) (5.9.1)\n", - "Requirement already satisfied: matplotlib-inline>=0.1 in /Users/mramanindia/work/NovaEval/.venv/lib/python3.12/site-packages (from ipykernel==6.29.4->-r ./noveum_agent_requirements.txt (line 33)) (0.1.7)\n", - "Requirement already satisfied: nest-asyncio in /Users/mramanindia/work/NovaEval/.venv/lib/python3.12/site-packages (from ipykernel==6.29.4->-r ./noveum_agent_requirements.txt (line 33)) (1.6.0)\n", - "Requirement already satisfied: psutil in /Users/mramanindia/work/NovaEval/.venv/lib/python3.12/site-packages (from ipykernel==6.29.4->-r ./noveum_agent_requirements.txt (line 33)) (7.1.1)\n", - "Requirement already satisfied: pyzmq>=24 in /Users/mramanindia/work/NovaEval/.venv/lib/python3.12/site-packages (from ipykernel==6.29.4->-r ./noveum_agent_requirements.txt (line 33)) (27.1.0)\n", - "Requirement already satisfied: tornado>=6.1 in /Users/mramanindia/work/NovaEval/.venv/lib/python3.12/site-packages (from ipykernel==6.29.4->-r ./noveum_agent_requirements.txt (line 33)) (6.5.2)\n", - "Requirement already satisfied: traitlets>=5.4.0 in /Users/mramanindia/work/NovaEval/.venv/lib/python3.12/site-packages (from ipykernel==6.29.4->-r ./noveum_agent_requirements.txt (line 33)) (5.14.3)\n", - "Collecting courlan>=1.3.2 (from trafilatura>=1.6.4->-r ./noveum_agent_requirements.txt (line 14))\n", - " Using cached courlan-1.3.2-py3-none-any.whl.metadata (17 kB)\n", - "Collecting htmldate>=1.9.2 (from trafilatura>=1.6.4->-r ./noveum_agent_requirements.txt (line 14))\n", - " Using cached htmldate-1.9.3-py3-none-any.whl.metadata (10 kB)\n", - "Collecting justext>=3.0.1 (from trafilatura>=1.6.4->-r ./noveum_agent_requirements.txt (line 14))\n", - " Using cached justext-3.0.2-py2.py3-none-any.whl.metadata (7.3 kB)\n", - "Requirement already satisfied: click>=8.1.8 in /Users/mramanindia/work/NovaEval/.venv/lib/python3.12/site-packages (from duckduckgo-search>=6.1.12->-r ./noveum_agent_requirements.txt (line 22)) (8.3.0)\n", - "Collecting primp>=0.15.0 (from duckduckgo-search>=6.1.12->-r ./noveum_agent_requirements.txt (line 22))\n", - " Using cached primp-0.15.0-cp38-abi3-macosx_11_0_arm64.whl.metadata (13 kB)\n", - "Requirement already satisfied: python-dateutil>=2.8.2 in /Users/mramanindia/work/NovaEval/.venv/lib/python3.12/site-packages (from pandas>=2.2.3->-r ./noveum_agent_requirements.txt (line 25)) (2.9.0.post0)\n", - "Requirement already satisfied: pytz>=2020.1 in /Users/mramanindia/work/NovaEval/.venv/lib/python3.12/site-packages (from pandas>=2.2.3->-r ./noveum_agent_requirements.txt (line 25)) (2025.2)\n", - "Requirement already satisfied: tzdata>=2022.7 in /Users/mramanindia/work/NovaEval/.venv/lib/python3.12/site-packages (from pandas>=2.2.3->-r ./noveum_agent_requirements.txt (line 25)) (2025.2)\n", - "Collecting aiohappyeyeballs>=2.5.0 (from aiohttp<4.0.0,>=3.8.3->langchain-community==0.3.18->-r ./noveum_agent_requirements.txt (line 9))\n", - " Using cached aiohappyeyeballs-2.6.1-py3-none-any.whl.metadata (5.9 kB)\n", - "Collecting aiosignal>=1.4.0 (from aiohttp<4.0.0,>=3.8.3->langchain-community==0.3.18->-r ./noveum_agent_requirements.txt (line 9))\n", - " Using cached aiosignal-1.4.0-py3-none-any.whl.metadata (3.7 kB)\n", - "Collecting attrs>=17.3.0 (from aiohttp<4.0.0,>=3.8.3->langchain-community==0.3.18->-r ./noveum_agent_requirements.txt (line 9))\n", - " Using cached attrs-25.4.0-py3-none-any.whl.metadata (10 kB)\n", - "Collecting frozenlist>=1.1.1 (from aiohttp<4.0.0,>=3.8.3->langchain-community==0.3.18->-r ./noveum_agent_requirements.txt (line 9))\n", - " Using cached frozenlist-1.8.0-cp312-cp312-macosx_11_0_arm64.whl.metadata (20 kB)\n", - "Collecting multidict<7.0,>=4.5 (from aiohttp<4.0.0,>=3.8.3->langchain-community==0.3.18->-r ./noveum_agent_requirements.txt (line 9))\n", - " Using cached multidict-6.7.0-cp312-cp312-macosx_11_0_arm64.whl.metadata (5.3 kB)\n", - "Collecting propcache>=0.2.0 (from aiohttp<4.0.0,>=3.8.3->langchain-community==0.3.18->-r ./noveum_agent_requirements.txt (line 9))\n", - " Using cached propcache-0.4.1-cp312-cp312-macosx_11_0_arm64.whl.metadata (13 kB)\n", - "Collecting yarl<2.0,>=1.17.0 (from aiohttp<4.0.0,>=3.8.3->langchain-community==0.3.18->-r ./noveum_agent_requirements.txt (line 9))\n", - " Using cached yarl-1.22.0-cp312-cp312-macosx_11_0_arm64.whl.metadata (75 kB)\n", - "Collecting babel>=2.16.0 (from courlan>=1.3.2->trafilatura>=1.6.4->-r ./noveum_agent_requirements.txt (line 14))\n", - " Using cached babel-2.17.0-py3-none-any.whl.metadata (2.0 kB)\n", - "Collecting tld>=0.13 (from courlan>=1.3.2->trafilatura>=1.6.4->-r ./noveum_agent_requirements.txt (line 14))\n", - " Using cached tld-0.13.1-py2.py3-none-any.whl.metadata (10 kB)\n", - "Collecting marshmallow<4.0.0,>=3.18.0 (from dataclasses-json<0.7,>=0.5.7->langchain-community==0.3.18->-r ./noveum_agent_requirements.txt (line 9))\n", - " Using cached marshmallow-3.26.1-py3-none-any.whl.metadata (7.3 kB)\n", - "Collecting typing-inspect<1,>=0.4.0 (from dataclasses-json<0.7,>=0.5.7->langchain-community==0.3.18->-r ./noveum_agent_requirements.txt (line 9))\n", - " Using cached typing_inspect-0.9.0-py3-none-any.whl.metadata (1.5 kB)\n", - "Collecting dateparser>=1.1.2 (from htmldate>=1.9.2->trafilatura>=1.6.4->-r ./noveum_agent_requirements.txt (line 14))\n", - " Using cached dateparser-1.2.2-py3-none-any.whl.metadata (29 kB)\n", - "Collecting lxml>=5.3.0 (from -r ./noveum_agent_requirements.txt (line 15))\n", - " Using cached lxml-5.4.0-cp312-cp312-macosx_10_9_universal2.whl.metadata (3.5 kB)\n", - "Requirement already satisfied: decorator in /Users/mramanindia/work/NovaEval/.venv/lib/python3.12/site-packages (from ipython>=7.23.1->ipykernel==6.29.4->-r ./noveum_agent_requirements.txt (line 33)) (5.2.1)\n", - "Requirement already satisfied: ipython-pygments-lexers in /Users/mramanindia/work/NovaEval/.venv/lib/python3.12/site-packages (from ipython>=7.23.1->ipykernel==6.29.4->-r ./noveum_agent_requirements.txt (line 33)) (1.1.1)\n", - "Requirement already satisfied: jedi>=0.16 in /Users/mramanindia/work/NovaEval/.venv/lib/python3.12/site-packages (from ipython>=7.23.1->ipykernel==6.29.4->-r ./noveum_agent_requirements.txt (line 33)) (0.19.2)\n", - "Requirement already satisfied: pexpect>4.3 in /Users/mramanindia/work/NovaEval/.venv/lib/python3.12/site-packages (from ipython>=7.23.1->ipykernel==6.29.4->-r ./noveum_agent_requirements.txt (line 33)) (4.9.0)\n", - "Requirement already satisfied: prompt_toolkit<3.1.0,>=3.0.41 in /Users/mramanindia/work/NovaEval/.venv/lib/python3.12/site-packages (from ipython>=7.23.1->ipykernel==6.29.4->-r ./noveum_agent_requirements.txt (line 33)) (3.0.51)\n", - "Requirement already satisfied: pygments>=2.4.0 in /Users/mramanindia/work/NovaEval/.venv/lib/python3.12/site-packages (from ipython>=7.23.1->ipykernel==6.29.4->-r ./noveum_agent_requirements.txt (line 33)) (2.19.2)\n", - "Requirement already satisfied: stack_data in /Users/mramanindia/work/NovaEval/.venv/lib/python3.12/site-packages (from ipython>=7.23.1->ipykernel==6.29.4->-r ./noveum_agent_requirements.txt (line 33)) (0.6.3)\n", - "Collecting jsonpointer>=1.9 (from jsonpatch<2.0,>=1.33->langchain-core==0.3.66->-r ./noveum_agent_requirements.txt (line 10))\n", - " Using cached jsonpointer-3.0.0-py2.py3-none-any.whl.metadata (2.3 kB)\n", - "Requirement already satisfied: platformdirs>=2.5 in /Users/mramanindia/work/NovaEval/.venv/lib/python3.12/site-packages (from jupyter-core!=5.0.*,>=4.12->ipykernel==6.29.4->-r ./noveum_agent_requirements.txt (line 33)) (4.5.0)\n", - "INFO: pip is looking at multiple versions of langchain-text-splitters to determine which version is compatible with other requirements. This could take a while.\n", - "Collecting langchain-text-splitters<1.0.0,>=0.3.8 (from langchain==0.3.26->-r ./noveum_agent_requirements.txt (line 8))\n", - " Using cached langchain_text_splitters-0.3.10-py3-none-any.whl.metadata (1.9 kB)\n", - " Using cached langchain_text_splitters-0.3.9-py3-none-any.whl.metadata (1.9 kB)\n", - " Using cached langchain_text_splitters-0.3.8-py3-none-any.whl.metadata (1.9 kB)\n", - "Requirement already satisfied: httpx<1,>=0.23.0 in /Users/mramanindia/work/NovaEval/.venv/lib/python3.12/site-packages (from langsmith>=0.1.17->langchain==0.3.26->-r ./noveum_agent_requirements.txt (line 8)) (0.28.1)\n", - "Collecting orjson<4.0.0,>=3.9.14 (from langsmith>=0.1.17->langchain==0.3.26->-r ./noveum_agent_requirements.txt (line 8))\n", - " Using cached orjson-3.11.3-cp312-cp312-macosx_15_0_arm64.whl.metadata (41 kB)\n", - "Collecting requests-toolbelt<2.0.0,>=1.0.0 (from langsmith>=0.1.17->langchain==0.3.26->-r ./noveum_agent_requirements.txt (line 8))\n", - " Using cached requests_toolbelt-1.0.0-py2.py3-none-any.whl.metadata (14 kB)\n", - "Collecting zstandard<0.24.0,>=0.23.0 (from langsmith>=0.1.17->langchain==0.3.26->-r ./noveum_agent_requirements.txt (line 8))\n", - " Using cached zstandard-0.23.0-cp312-cp312-macosx_11_0_arm64.whl.metadata (3.0 kB)\n", - "Requirement already satisfied: anyio<5,>=3.5.0 in /Users/mramanindia/work/NovaEval/.venv/lib/python3.12/site-packages (from openai<2.0.0,>=1.86.0->langchain-openai==0.3.25->-r ./noveum_agent_requirements.txt (line 11)) (4.11.0)\n", - "Collecting distro<2,>=1.7.0 (from openai<2.0.0,>=1.86.0->langchain-openai==0.3.25->-r ./noveum_agent_requirements.txt (line 11))\n", - " Using cached distro-1.9.0-py3-none-any.whl.metadata (6.8 kB)\n", - "Collecting jiter<1,>=0.4.0 (from openai<2.0.0,>=1.86.0->langchain-openai==0.3.25->-r ./noveum_agent_requirements.txt (line 11))\n", - " Using cached jiter-0.11.1-cp312-cp312-macosx_11_0_arm64.whl.metadata (5.2 kB)\n", - "Requirement already satisfied: sniffio in /Users/mramanindia/work/NovaEval/.venv/lib/python3.12/site-packages (from openai<2.0.0,>=1.86.0->langchain-openai==0.3.25->-r ./noveum_agent_requirements.txt (line 11)) (1.3.1)\n", - "Requirement already satisfied: annotated-types>=0.6.0 in /Users/mramanindia/work/NovaEval/.venv/lib/python3.12/site-packages (from pydantic<3.0.0,>=2.7.4->langchain==0.3.26->-r ./noveum_agent_requirements.txt (line 8)) (0.7.0)\n", - "Requirement already satisfied: pydantic-core==2.41.4 in /Users/mramanindia/work/NovaEval/.venv/lib/python3.12/site-packages (from pydantic<3.0.0,>=2.7.4->langchain==0.3.26->-r ./noveum_agent_requirements.txt (line 8)) (2.41.4)\n", - "Requirement already satisfied: typing-inspection>=0.4.2 in /Users/mramanindia/work/NovaEval/.venv/lib/python3.12/site-packages (from pydantic<3.0.0,>=2.7.4->langchain==0.3.26->-r ./noveum_agent_requirements.txt (line 8)) (0.4.2)\n", - "Requirement already satisfied: six>=1.5 in /Users/mramanindia/work/NovaEval/.venv/lib/python3.12/site-packages (from python-dateutil>=2.8.2->pandas>=2.2.3->-r ./noveum_agent_requirements.txt (line 25)) (1.17.0)\n", - "Requirement already satisfied: regex>=2022.1.18 in /Users/mramanindia/work/NovaEval/.venv/lib/python3.12/site-packages (from tiktoken<1,>=0.7->langchain-openai==0.3.25->-r ./noveum_agent_requirements.txt (line 11)) (2025.10.23)\n", - "Collecting widgetsnbextension~=4.0.14 (from ipywidgets->jupyter==1.0.0->-r ./noveum_agent_requirements.txt (line 32))\n", - " Using cached widgetsnbextension-4.0.14-py3-none-any.whl.metadata (1.6 kB)\n", - "Collecting jupyterlab_widgets~=3.0.15 (from ipywidgets->jupyter==1.0.0->-r ./noveum_agent_requirements.txt (line 32))\n", - " Using cached jupyterlab_widgets-3.0.15-py3-none-any.whl.metadata (20 kB)\n", - "Collecting bleach!=5.0.0 (from bleach[css]!=5.0.0->nbconvert->jupyter==1.0.0->-r ./noveum_agent_requirements.txt (line 32))\n", - " Using cached bleach-6.2.0-py3-none-any.whl.metadata (30 kB)\n", - "Collecting defusedxml (from nbconvert->jupyter==1.0.0->-r ./noveum_agent_requirements.txt (line 32))\n", - " Using cached defusedxml-0.7.1-py2.py3-none-any.whl.metadata (32 kB)\n", - "Requirement already satisfied: jinja2>=3.0 in /Users/mramanindia/work/NovaEval/.venv/lib/python3.12/site-packages (from nbconvert->jupyter==1.0.0->-r ./noveum_agent_requirements.txt (line 32)) (3.1.6)\n", - "Collecting jupyterlab-pygments (from nbconvert->jupyter==1.0.0->-r ./noveum_agent_requirements.txt (line 32))\n", - " Using cached jupyterlab_pygments-0.3.0-py3-none-any.whl.metadata (4.4 kB)\n", - "Requirement already satisfied: markupsafe>=2.0 in /Users/mramanindia/work/NovaEval/.venv/lib/python3.12/site-packages (from nbconvert->jupyter==1.0.0->-r ./noveum_agent_requirements.txt (line 32)) (3.0.3)\n", - "Collecting mistune<4,>=2.0.3 (from nbconvert->jupyter==1.0.0->-r ./noveum_agent_requirements.txt (line 32))\n", - " Using cached mistune-3.1.4-py3-none-any.whl.metadata (1.8 kB)\n", - "Collecting nbclient>=0.5.0 (from nbconvert->jupyter==1.0.0->-r ./noveum_agent_requirements.txt (line 32))\n", - " Using cached nbclient-0.10.2-py3-none-any.whl.metadata (8.3 kB)\n", - "Collecting nbformat>=5.7 (from nbconvert->jupyter==1.0.0->-r ./noveum_agent_requirements.txt (line 32))\n", - " Using cached nbformat-5.10.4-py3-none-any.whl.metadata (3.6 kB)\n", - "Collecting pandocfilters>=1.4.1 (from nbconvert->jupyter==1.0.0->-r ./noveum_agent_requirements.txt (line 32))\n", - " Using cached pandocfilters-1.5.1-py2.py3-none-any.whl.metadata (9.0 kB)\n", - "Collecting jupyter-server<3,>=2.4.0 (from notebook->jupyter==1.0.0->-r ./noveum_agent_requirements.txt (line 32))\n", - " Using cached jupyter_server-2.17.0-py3-none-any.whl.metadata (8.5 kB)\n", - "Collecting jupyterlab-server<3,>=2.27.1 (from notebook->jupyter==1.0.0->-r ./noveum_agent_requirements.txt (line 32))\n", - " Downloading jupyterlab_server-2.28.0-py3-none-any.whl.metadata (5.9 kB)\n", - "Collecting jupyterlab<4.5,>=4.4.9 (from notebook->jupyter==1.0.0->-r ./noveum_agent_requirements.txt (line 32))\n", - " Downloading jupyterlab-4.4.10-py3-none-any.whl.metadata (16 kB)\n", - "Collecting notebook-shim<0.3,>=0.2 (from notebook->jupyter==1.0.0->-r ./noveum_agent_requirements.txt (line 32))\n", - " Using cached notebook_shim-0.2.4-py3-none-any.whl.metadata (4.0 kB)\n", - "Collecting qtpy>=2.4.0 (from qtconsole->jupyter==1.0.0->-r ./noveum_agent_requirements.txt (line 32))\n", - " Using cached QtPy-2.4.3-py3-none-any.whl.metadata (12 kB)\n", - "Collecting webencodings (from bleach!=5.0.0->bleach[css]!=5.0.0->nbconvert->jupyter==1.0.0->-r ./noveum_agent_requirements.txt (line 32))\n", - " Using cached webencodings-0.5.1-py2.py3-none-any.whl.metadata (2.1 kB)\n", - "Collecting tinycss2<1.5,>=1.1.0 (from bleach[css]!=5.0.0->nbconvert->jupyter==1.0.0->-r ./noveum_agent_requirements.txt (line 32))\n", - " Using cached tinycss2-1.4.0-py3-none-any.whl.metadata (3.0 kB)\n", - "Collecting tzlocal>=0.2 (from dateparser>=1.1.2->htmldate>=1.9.2->trafilatura>=1.6.4->-r ./noveum_agent_requirements.txt (line 14))\n", - " Using cached tzlocal-5.3.1-py3-none-any.whl.metadata (7.6 kB)\n", - "Requirement already satisfied: httpcore==1.* in /Users/mramanindia/work/NovaEval/.venv/lib/python3.12/site-packages (from httpx<1,>=0.23.0->langsmith>=0.1.17->langchain==0.3.26->-r ./noveum_agent_requirements.txt (line 8)) (1.0.9)\n", - "Requirement already satisfied: h11>=0.16 in /Users/mramanindia/work/NovaEval/.venv/lib/python3.12/site-packages (from httpcore==1.*->httpx<1,>=0.23.0->langsmith>=0.1.17->langchain==0.3.26->-r ./noveum_agent_requirements.txt (line 8)) (0.16.0)\n", - "Requirement already satisfied: parso<0.9.0,>=0.8.4 in /Users/mramanindia/work/NovaEval/.venv/lib/python3.12/site-packages (from jedi>=0.16->ipython>=7.23.1->ipykernel==6.29.4->-r ./noveum_agent_requirements.txt (line 33)) (0.8.5)\n", - "Collecting argon2-cffi>=21.1 (from jupyter-server<3,>=2.4.0->notebook->jupyter==1.0.0->-r ./noveum_agent_requirements.txt (line 32))\n", - " Using cached argon2_cffi-25.1.0-py3-none-any.whl.metadata (4.1 kB)\n", - "Collecting jupyter-events>=0.11.0 (from jupyter-server<3,>=2.4.0->notebook->jupyter==1.0.0->-r ./noveum_agent_requirements.txt (line 32))\n", - " Using cached jupyter_events-0.12.0-py3-none-any.whl.metadata (5.8 kB)\n", - "Collecting jupyter-server-terminals>=0.4.4 (from jupyter-server<3,>=2.4.0->notebook->jupyter==1.0.0->-r ./noveum_agent_requirements.txt (line 32))\n", - " Using cached jupyter_server_terminals-0.5.3-py3-none-any.whl.metadata (5.6 kB)\n", - "Collecting prometheus-client>=0.9 (from jupyter-server<3,>=2.4.0->notebook->jupyter==1.0.0->-r ./noveum_agent_requirements.txt (line 32))\n", - " Using cached prometheus_client-0.23.1-py3-none-any.whl.metadata (1.9 kB)\n", - "Collecting send2trash>=1.8.2 (from jupyter-server<3,>=2.4.0->notebook->jupyter==1.0.0->-r ./noveum_agent_requirements.txt (line 32))\n", - " Using cached Send2Trash-1.8.3-py3-none-any.whl.metadata (4.0 kB)\n", - "Collecting terminado>=0.8.3 (from jupyter-server<3,>=2.4.0->notebook->jupyter==1.0.0->-r ./noveum_agent_requirements.txt (line 32))\n", - " Using cached terminado-0.18.1-py3-none-any.whl.metadata (5.8 kB)\n", - "Collecting websocket-client>=1.7 (from jupyter-server<3,>=2.4.0->notebook->jupyter==1.0.0->-r ./noveum_agent_requirements.txt (line 32))\n", - " Using cached websocket_client-1.9.0-py3-none-any.whl.metadata (8.3 kB)\n", - "Collecting async-lru>=1.0.0 (from jupyterlab<4.5,>=4.4.9->notebook->jupyter==1.0.0->-r ./noveum_agent_requirements.txt (line 32))\n", - " Using cached async_lru-2.0.5-py3-none-any.whl.metadata (4.5 kB)\n", - "Collecting jupyter-lsp>=2.0.0 (from jupyterlab<4.5,>=4.4.9->notebook->jupyter==1.0.0->-r ./noveum_agent_requirements.txt (line 32))\n", - " Using cached jupyter_lsp-2.3.0-py3-none-any.whl.metadata (1.8 kB)\n", - "Requirement already satisfied: setuptools>=41.1.0 in /Users/mramanindia/work/NovaEval/.venv/lib/python3.12/site-packages (from jupyterlab<4.5,>=4.4.9->notebook->jupyter==1.0.0->-r ./noveum_agent_requirements.txt (line 32)) (80.9.0)\n", - "Collecting json5>=0.9.0 (from jupyterlab-server<3,>=2.27.1->notebook->jupyter==1.0.0->-r ./noveum_agent_requirements.txt (line 32))\n", - " Using cached json5-0.12.1-py3-none-any.whl.metadata (36 kB)\n", - "Collecting jsonschema>=4.18.0 (from jupyterlab-server<3,>=2.27.1->notebook->jupyter==1.0.0->-r ./noveum_agent_requirements.txt (line 32))\n", - " Using cached jsonschema-4.25.1-py3-none-any.whl.metadata (7.6 kB)\n", - "Collecting lxml_html_clean (from lxml[html_clean]>=4.4.2->justext>=3.0.1->trafilatura>=1.6.4->-r ./noveum_agent_requirements.txt (line 14))\n", - " Using cached lxml_html_clean-0.4.3-py3-none-any.whl.metadata (2.3 kB)\n", - "Collecting fastjsonschema>=2.15 (from nbformat>=5.7->nbconvert->jupyter==1.0.0->-r ./noveum_agent_requirements.txt (line 32))\n", - " Using cached fastjsonschema-2.21.2-py3-none-any.whl.metadata (2.3 kB)\n", - "Requirement already satisfied: ptyprocess>=0.5 in /Users/mramanindia/work/NovaEval/.venv/lib/python3.12/site-packages (from pexpect>4.3->ipython>=7.23.1->ipykernel==6.29.4->-r ./noveum_agent_requirements.txt (line 33)) (0.7.0)\n", - "Requirement already satisfied: wcwidth in /Users/mramanindia/work/NovaEval/.venv/lib/python3.12/site-packages (from prompt_toolkit<3.1.0,>=3.0.41->ipython>=7.23.1->ipykernel==6.29.4->-r ./noveum_agent_requirements.txt (line 33)) (0.2.14)\n", - "Requirement already satisfied: mypy-extensions>=0.3.0 in /Users/mramanindia/work/NovaEval/.venv/lib/python3.12/site-packages (from typing-inspect<1,>=0.4.0->dataclasses-json<0.7,>=0.5.7->langchain-community==0.3.18->-r ./noveum_agent_requirements.txt (line 9)) (1.1.0)\n", - "Requirement already satisfied: executing>=1.2.0 in /Users/mramanindia/work/NovaEval/.venv/lib/python3.12/site-packages (from stack_data->ipython>=7.23.1->ipykernel==6.29.4->-r ./noveum_agent_requirements.txt (line 33)) (2.2.1)\n", - "Requirement already satisfied: asttokens>=2.1.0 in /Users/mramanindia/work/NovaEval/.venv/lib/python3.12/site-packages (from stack_data->ipython>=7.23.1->ipykernel==6.29.4->-r ./noveum_agent_requirements.txt (line 33)) (3.0.0)\n", - "Requirement already satisfied: pure-eval in /Users/mramanindia/work/NovaEval/.venv/lib/python3.12/site-packages (from stack_data->ipython>=7.23.1->ipykernel==6.29.4->-r ./noveum_agent_requirements.txt (line 33)) (0.2.3)\n", - "Collecting argon2-cffi-bindings (from argon2-cffi>=21.1->jupyter-server<3,>=2.4.0->notebook->jupyter==1.0.0->-r ./noveum_agent_requirements.txt (line 32))\n", - " Using cached argon2_cffi_bindings-25.1.0-cp39-abi3-macosx_11_0_arm64.whl.metadata (7.4 kB)\n", - "Collecting jsonschema-specifications>=2023.03.6 (from jsonschema>=4.18.0->jupyterlab-server<3,>=2.27.1->notebook->jupyter==1.0.0->-r ./noveum_agent_requirements.txt (line 32))\n", - " Using cached jsonschema_specifications-2025.9.1-py3-none-any.whl.metadata (2.9 kB)\n", - "Collecting referencing>=0.28.4 (from jsonschema>=4.18.0->jupyterlab-server<3,>=2.27.1->notebook->jupyter==1.0.0->-r ./noveum_agent_requirements.txt (line 32))\n", - " Using cached referencing-0.37.0-py3-none-any.whl.metadata (2.8 kB)\n", - "Collecting rpds-py>=0.7.1 (from jsonschema>=4.18.0->jupyterlab-server<3,>=2.27.1->notebook->jupyter==1.0.0->-r ./noveum_agent_requirements.txt (line 32))\n", - " Using cached rpds_py-0.27.1-cp312-cp312-macosx_11_0_arm64.whl.metadata (4.2 kB)\n", - "Collecting python-json-logger>=2.0.4 (from jupyter-events>=0.11.0->jupyter-server<3,>=2.4.0->notebook->jupyter==1.0.0->-r ./noveum_agent_requirements.txt (line 32))\n", - " Using cached python_json_logger-4.0.0-py3-none-any.whl.metadata (4.0 kB)\n", - "Collecting rfc3339-validator (from jupyter-events>=0.11.0->jupyter-server<3,>=2.4.0->notebook->jupyter==1.0.0->-r ./noveum_agent_requirements.txt (line 32))\n", - " Using cached rfc3339_validator-0.1.4-py2.py3-none-any.whl.metadata (1.5 kB)\n", - "Collecting rfc3986-validator>=0.1.1 (from jupyter-events>=0.11.0->jupyter-server<3,>=2.4.0->notebook->jupyter==1.0.0->-r ./noveum_agent_requirements.txt (line 32))\n", - " Using cached rfc3986_validator-0.1.1-py2.py3-none-any.whl.metadata (1.7 kB)\n", - "Collecting fqdn (from jsonschema[format-nongpl]>=4.18.0->jupyter-events>=0.11.0->jupyter-server<3,>=2.4.0->notebook->jupyter==1.0.0->-r ./noveum_agent_requirements.txt (line 32))\n", - " Using cached fqdn-1.5.1-py3-none-any.whl.metadata (1.4 kB)\n", - "Collecting isoduration (from jsonschema[format-nongpl]>=4.18.0->jupyter-events>=0.11.0->jupyter-server<3,>=2.4.0->notebook->jupyter==1.0.0->-r ./noveum_agent_requirements.txt (line 32))\n", - " Using cached isoduration-20.11.0-py3-none-any.whl.metadata (5.7 kB)\n", - "Collecting rfc3987-syntax>=1.1.0 (from jsonschema[format-nongpl]>=4.18.0->jupyter-events>=0.11.0->jupyter-server<3,>=2.4.0->notebook->jupyter==1.0.0->-r ./noveum_agent_requirements.txt (line 32))\n", - " Using cached rfc3987_syntax-1.1.0-py3-none-any.whl.metadata (7.7 kB)\n", - "Collecting uri-template (from jsonschema[format-nongpl]>=4.18.0->jupyter-events>=0.11.0->jupyter-server<3,>=2.4.0->notebook->jupyter==1.0.0->-r ./noveum_agent_requirements.txt (line 32))\n", - " Using cached uri_template-1.3.0-py3-none-any.whl.metadata (8.8 kB)\n", - "Collecting webcolors>=24.6.0 (from jsonschema[format-nongpl]>=4.18.0->jupyter-events>=0.11.0->jupyter-server<3,>=2.4.0->notebook->jupyter==1.0.0->-r ./noveum_agent_requirements.txt (line 32))\n", - " Using cached webcolors-24.11.1-py3-none-any.whl.metadata (2.2 kB)\n", - "Requirement already satisfied: cffi>=1.0.1 in /Users/mramanindia/work/NovaEval/.venv/lib/python3.12/site-packages (from argon2-cffi-bindings->argon2-cffi>=21.1->jupyter-server<3,>=2.4.0->notebook->jupyter==1.0.0->-r ./noveum_agent_requirements.txt (line 32)) (2.0.0)\n", - "Requirement already satisfied: pycparser in /Users/mramanindia/work/NovaEval/.venv/lib/python3.12/site-packages (from cffi>=1.0.1->argon2-cffi-bindings->argon2-cffi>=21.1->jupyter-server<3,>=2.4.0->notebook->jupyter==1.0.0->-r ./noveum_agent_requirements.txt (line 32)) (2.23)\n", - "Collecting lark>=1.2.2 (from rfc3987-syntax>=1.1.0->jsonschema[format-nongpl]>=4.18.0->jupyter-events>=0.11.0->jupyter-server<3,>=2.4.0->notebook->jupyter==1.0.0->-r ./noveum_agent_requirements.txt (line 32))\n", - " Using cached lark-1.3.0-py3-none-any.whl.metadata (1.8 kB)\n", - "Collecting arrow>=0.15.0 (from isoduration->jsonschema[format-nongpl]>=4.18.0->jupyter-events>=0.11.0->jupyter-server<3,>=2.4.0->notebook->jupyter==1.0.0->-r ./noveum_agent_requirements.txt (line 32))\n", - " Using cached arrow-1.4.0-py3-none-any.whl.metadata (7.7 kB)\n", - "Using cached requests-2.32.3-py3-none-any.whl (64 kB)\n", - "Using cached beautifulsoup4-4.12.3-py3-none-any.whl (147 kB)\n", - "Using cached python_dotenv-1.0.1-py3-none-any.whl (19 kB)\n", - "Using cached langchain-0.3.26-py3-none-any.whl (1.0 MB)\n", - "Using cached langchain_community-0.3.18-py3-none-any.whl (2.5 MB)\n", - "Using cached langchain_core-0.3.66-py3-none-any.whl (438 kB)\n", - "Using cached langchain_openai-0.3.25-py3-none-any.whl (69 kB)\n", - "Using cached faiss_cpu-1.12.0-cp312-cp312-macosx_14_0_arm64.whl (3.4 MB)\n", - "Using cached jupyter-1.0.0-py2.py3-none-any.whl (2.7 kB)\n", - "Using cached ipykernel-6.29.4-py3-none-any.whl (117 kB)\n", - "Using cached trafilatura-2.0.0-py3-none-any.whl (132 kB)\n", - "Using cached duckduckgo_search-8.1.1-py3-none-any.whl (18 kB)\n", - "Using cached aiohttp-3.13.1-cp312-cp312-macosx_11_0_arm64.whl (490 kB)\n", - "Using cached courlan-1.3.2-py3-none-any.whl (33 kB)\n", - "Using cached dataclasses_json-0.6.7-py3-none-any.whl (28 kB)\n", - "Using cached htmldate-1.9.3-py3-none-any.whl (31 kB)\n", - "Using cached lxml-5.4.0-cp312-cp312-macosx_10_9_universal2.whl (8.1 MB)\n", - "Using cached httpx_sse-0.4.3-py3-none-any.whl (9.0 kB)\n", - "Using cached jsonpatch-1.33-py2.py3-none-any.whl (12 kB)\n", - "Using cached justext-3.0.2-py2.py3-none-any.whl (837 kB)\n", - "Using cached langchain_text_splitters-0.3.8-py3-none-any.whl (32 kB)\n", - "Using cached langsmith-0.3.45-py3-none-any.whl (363 kB)\n", - "Using cached openai-1.109.1-py3-none-any.whl (948 kB)\n", - "Using cached packaging-24.2-py3-none-any.whl (65 kB)\n", - "Using cached primp-0.15.0-cp38-abi3-macosx_11_0_arm64.whl (3.0 MB)\n", - "Using cached soupsieve-2.8-py3-none-any.whl (36 kB)\n", - "Using cached sqlalchemy-2.0.44-cp312-cp312-macosx_11_0_arm64.whl (2.1 MB)\n", - "Using cached tiktoken-0.12.0-cp312-cp312-macosx_11_0_arm64.whl (994 kB)\n", - "Using cached ipywidgets-8.1.7-py3-none-any.whl (139 kB)\n", - "Using cached jupyter_console-6.6.3-py3-none-any.whl (24 kB)\n", - "Using cached nbconvert-7.16.6-py3-none-any.whl (258 kB)\n", - "Using cached notebook-7.4.7-py3-none-any.whl (14.3 MB)\n", - "Using cached qtconsole-5.7.0-py3-none-any.whl (125 kB)\n", - "Using cached aiohappyeyeballs-2.6.1-py3-none-any.whl (15 kB)\n", - "Using cached aiosignal-1.4.0-py3-none-any.whl (7.5 kB)\n", - "Using cached attrs-25.4.0-py3-none-any.whl (67 kB)\n", - "Using cached babel-2.17.0-py3-none-any.whl (10.2 MB)\n", - "Using cached bleach-6.2.0-py3-none-any.whl (163 kB)\n", - "Using cached dateparser-1.2.2-py3-none-any.whl (315 kB)\n", - "Using cached distro-1.9.0-py3-none-any.whl (20 kB)\n", - "Using cached frozenlist-1.8.0-cp312-cp312-macosx_11_0_arm64.whl (50 kB)\n", - "Using cached jiter-0.11.1-cp312-cp312-macosx_11_0_arm64.whl (315 kB)\n", - "Using cached jsonpointer-3.0.0-py2.py3-none-any.whl (7.6 kB)\n", - "Using cached jupyter_server-2.17.0-py3-none-any.whl (388 kB)\n", - "Downloading jupyterlab-4.4.10-py3-none-any.whl (12.3 MB)\n", - "\u001b[2K \u001b[90mā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”\u001b[0m \u001b[32m12.3/12.3 MB\u001b[0m \u001b[31m135.7 kB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:02\u001b[0m00:04\u001b[0m\n", - "\u001b[?25hDownloading jupyterlab_server-2.28.0-py3-none-any.whl (59 kB)\n", - "Using cached jupyterlab_widgets-3.0.15-py3-none-any.whl (216 kB)\n", - "Using cached marshmallow-3.26.1-py3-none-any.whl (50 kB)\n", - "Using cached mistune-3.1.4-py3-none-any.whl (53 kB)\n", - "Using cached multidict-6.7.0-cp312-cp312-macosx_11_0_arm64.whl (43 kB)\n", - "Using cached nbclient-0.10.2-py3-none-any.whl (25 kB)\n", - "Using cached nbformat-5.10.4-py3-none-any.whl (78 kB)\n", - "Using cached notebook_shim-0.2.4-py3-none-any.whl (13 kB)\n", - "Using cached orjson-3.11.3-cp312-cp312-macosx_15_0_arm64.whl (127 kB)\n", - "Using cached pandocfilters-1.5.1-py2.py3-none-any.whl (8.7 kB)\n", - "Using cached propcache-0.4.1-cp312-cp312-macosx_11_0_arm64.whl (47 kB)\n", - "Using cached QtPy-2.4.3-py3-none-any.whl (95 kB)\n", - "Using cached requests_toolbelt-1.0.0-py2.py3-none-any.whl (54 kB)\n", - "Using cached tld-0.13.1-py2.py3-none-any.whl (274 kB)\n", - "Using cached typing_inspect-0.9.0-py3-none-any.whl (8.8 kB)\n", - "Using cached widgetsnbextension-4.0.14-py3-none-any.whl (2.2 MB)\n", - "Using cached yarl-1.22.0-cp312-cp312-macosx_11_0_arm64.whl (94 kB)\n", - "Using cached zstandard-0.23.0-cp312-cp312-macosx_11_0_arm64.whl (633 kB)\n", - "Using cached defusedxml-0.7.1-py2.py3-none-any.whl (25 kB)\n", - "Using cached jupyterlab_pygments-0.3.0-py3-none-any.whl (15 kB)\n", - "Using cached argon2_cffi-25.1.0-py3-none-any.whl (14 kB)\n", - "Using cached async_lru-2.0.5-py3-none-any.whl (6.1 kB)\n", - "Using cached fastjsonschema-2.21.2-py3-none-any.whl (24 kB)\n", - "Using cached json5-0.12.1-py3-none-any.whl (36 kB)\n", - "Using cached jsonschema-4.25.1-py3-none-any.whl (90 kB)\n", - "Using cached jupyter_events-0.12.0-py3-none-any.whl (19 kB)\n", - "Using cached jupyter_lsp-2.3.0-py3-none-any.whl (76 kB)\n", - "Using cached jupyter_server_terminals-0.5.3-py3-none-any.whl (13 kB)\n", - "Using cached prometheus_client-0.23.1-py3-none-any.whl (61 kB)\n", - "Using cached Send2Trash-1.8.3-py3-none-any.whl (18 kB)\n", - "Using cached terminado-0.18.1-py3-none-any.whl (14 kB)\n", - "Using cached tinycss2-1.4.0-py3-none-any.whl (26 kB)\n", - "Using cached tzlocal-5.3.1-py3-none-any.whl (18 kB)\n", - "Using cached webencodings-0.5.1-py2.py3-none-any.whl (11 kB)\n", - "Using cached websocket_client-1.9.0-py3-none-any.whl (82 kB)\n", - "Using cached lxml_html_clean-0.4.3-py3-none-any.whl (14 kB)\n", - "Using cached jsonschema_specifications-2025.9.1-py3-none-any.whl (18 kB)\n", - "Using cached python_json_logger-4.0.0-py3-none-any.whl (15 kB)\n", - "Using cached referencing-0.37.0-py3-none-any.whl (26 kB)\n", - "Using cached rfc3986_validator-0.1.1-py2.py3-none-any.whl (4.2 kB)\n", - "Using cached rpds_py-0.27.1-cp312-cp312-macosx_11_0_arm64.whl (345 kB)\n", - "Using cached argon2_cffi_bindings-25.1.0-cp39-abi3-macosx_11_0_arm64.whl (31 kB)\n", - "Using cached rfc3339_validator-0.1.4-py2.py3-none-any.whl (3.5 kB)\n", - "Using cached rfc3987_syntax-1.1.0-py3-none-any.whl (8.0 kB)\n", - "Using cached webcolors-24.11.1-py3-none-any.whl (14 kB)\n", - "Using cached fqdn-1.5.1-py3-none-any.whl (9.1 kB)\n", - "Using cached isoduration-20.11.0-py3-none-any.whl (11 kB)\n", - "Using cached uri_template-1.3.0-py3-none-any.whl (11 kB)\n", - "Using cached arrow-1.4.0-py3-none-any.whl (68 kB)\n", - "Using cached lark-1.3.0-py3-none-any.whl (113 kB)\n", - "Installing collected packages: webencodings, fastjsonschema, zstandard, widgetsnbextension, websocket-client, webcolors, uri-template, tzlocal, typing-inspect, tld, tinycss2, terminado, SQLAlchemy, soupsieve, send2trash, rpds-py, rfc3986-validator, rfc3339-validator, requests, python-json-logger, python-dotenv, propcache, prometheus-client, primp, pandocfilters, packaging, orjson, multidict, mistune, lxml, lark, jupyterlab_widgets, jupyterlab-pygments, jsonpointer, json5, jiter, httpx-sse, frozenlist, fqdn, distro, defusedxml, bleach, babel, attrs, async-lru, aiohappyeyeballs, yarl, tiktoken, rfc3987-syntax, requests-toolbelt, referencing, qtpy, marshmallow, lxml_html_clean, jupyter-server-terminals, jsonpatch, faiss-cpu, duckduckgo-search, dateparser, courlan, beautifulsoup4, arrow, argon2-cffi-bindings, aiosignal, openai, langsmith, jsonschema-specifications, isoduration, ipywidgets, ipykernel, htmldate, dataclasses-json, argon2-cffi, aiohttp, qtconsole, langchain-core, justext, jupyter-console, jsonschema, trafilatura, nbformat, langchain-text-splitters, langchain-openai, nbclient, langchain, jupyter-events, nbconvert, langchain-community, jupyter-server, notebook-shim, jupyterlab-server, jupyter-lsp, jupyterlab, notebook, jupyter\n", - " Attempting uninstall: requests\n", - " Found existing installation: requests 2.32.5\n", - " Uninstalling requests-2.32.5:\n", - " Successfully uninstalled requests-2.32.5\n", - " Attempting uninstall: python-dotenv\n", - " Found existing installation: python-dotenv 1.1.1\n", - " Uninstalling python-dotenv-1.1.1:\n", - " Successfully uninstalled python-dotenv-1.1.1\n", - " Attempting uninstall: packaging\n", - " Found existing installation: packaging 25.0\n", - " Uninstalling packaging-25.0:\n", - " Successfully uninstalled packaging-25.0\n", - " Attempting uninstall: marshmallow\n", - " Found existing installation: marshmallow 4.0.1\n", - " Uninstalling marshmallow-4.0.1:\n", - " Successfully uninstalled marshmallow-4.0.1\n", - " Attempting uninstall: ipykernel\n", - " Found existing installation: ipykernel 7.0.1\n", - " Uninstalling ipykernel-7.0.1:\n", - " Successfully uninstalled ipykernel-7.0.1\n", - "Successfully installed SQLAlchemy-2.0.44 aiohappyeyeballs-2.6.1 aiohttp-3.13.1 aiosignal-1.4.0 argon2-cffi-25.1.0 argon2-cffi-bindings-25.1.0 arrow-1.4.0 async-lru-2.0.5 attrs-25.4.0 babel-2.17.0 beautifulsoup4-4.12.3 bleach-6.2.0 courlan-1.3.2 dataclasses-json-0.6.7 dateparser-1.2.2 defusedxml-0.7.1 distro-1.9.0 duckduckgo-search-8.1.1 faiss-cpu-1.12.0 fastjsonschema-2.21.2 fqdn-1.5.1 frozenlist-1.8.0 htmldate-1.9.3 httpx-sse-0.4.3 ipykernel-6.29.4 ipywidgets-8.1.7 isoduration-20.11.0 jiter-0.11.1 json5-0.12.1 jsonpatch-1.33 jsonpointer-3.0.0 jsonschema-4.25.1 jsonschema-specifications-2025.9.1 jupyter-1.0.0 jupyter-console-6.6.3 jupyter-events-0.12.0 jupyter-lsp-2.3.0 jupyter-server-2.17.0 jupyter-server-terminals-0.5.3 jupyterlab-4.4.10 jupyterlab-pygments-0.3.0 jupyterlab-server-2.28.0 jupyterlab_widgets-3.0.15 justext-3.0.2 langchain-0.3.26 langchain-community-0.3.18 langchain-core-0.3.66 langchain-openai-0.3.25 langchain-text-splitters-0.3.8 langsmith-0.3.45 lark-1.3.0 lxml-5.4.0 lxml_html_clean-0.4.3 marshmallow-3.26.1 mistune-3.1.4 multidict-6.7.0 nbclient-0.10.2 nbconvert-7.16.6 nbformat-5.10.4 notebook-7.4.7 notebook-shim-0.2.4 openai-1.109.1 orjson-3.11.3 packaging-24.2 pandocfilters-1.5.1 primp-0.15.0 prometheus-client-0.23.1 propcache-0.4.1 python-dotenv-1.0.1 python-json-logger-4.0.0 qtconsole-5.7.0 qtpy-2.4.3 referencing-0.37.0 requests-2.32.3 requests-toolbelt-1.0.0 rfc3339-validator-0.1.4 rfc3986-validator-0.1.1 rfc3987-syntax-1.1.0 rpds-py-0.27.1 send2trash-1.8.3 soupsieve-2.8 terminado-0.18.1 tiktoken-0.12.0 tinycss2-1.4.0 tld-0.13.1 trafilatura-2.0.0 typing-inspect-0.9.0 tzlocal-5.3.1 uri-template-1.3.0 webcolors-24.11.1 webencodings-0.5.1 websocket-client-1.9.0 widgetsnbextension-4.0.14 yarl-1.22.0 zstandard-0.23.0\n", - "\n", - "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m24.3.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m25.2\u001b[0m\n", - "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n" - ] - } - ], + "outputs": [], "source": [ "!pip3 install -r ./noveum_agent_requirements.txt\n" ] @@ -651,21 +265,19 @@ ] }, { - "cell_type": "code", - "execution_count": null, - "id": "2209736e", + "cell_type": "markdown", + "id": "a7677fe0", "metadata": {}, - "outputs": [], "source": [ + "## These are required for the project\n", "\n", + "- set openai api key\n", + "- set gemini api key\n", + "- set noveum api key\n", + "- set environment\n", + "- set project'\n", "\n", - "## set openai api key\n", - "## set gemini api key\n", - "## set noveum api key\n", - "## set environment\n", - "## set project'\n", - "\n", - "# These are required for the project\n" + "\n" ] }, { @@ -2196,5778 +1808,258 @@ }, { "cell_type": "code", - "execution_count": 58, - "id": "f727a446", + "execution_count": null, + "id": "5645fc74", + "metadata": {}, + "outputs": [], + "source": [ + "demo_noveum_agent()" + ] + }, + { + "cell_type": "markdown", + "id": "d5b29876", + "metadata": {}, + "source": [ + "## Downloading the data set" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bfadf5f1", + "metadata": {}, + "outputs": [], + "source": [ + "!python noveum_customer_support_bt/traces/fetch_traces_api.py 50\n", + "\n", + "#. This script fetches traces for our project and saves them locally." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "eb4628eb", + "metadata": {}, + "outputs": [], + "source": [ + "!python NovaEval/noveum_customer_support_bt/traces/combine_spans_api_compat.py" + ] + }, + { + "cell_type": "markdown", + "id": "0bddd552", + "metadata": {}, + "source": [ + "## Data Filteration and mapping" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "437d8751", + "metadata": {}, + "outputs": [], + "source": [ + "!python preprocess_filter.py ./traces/traces/dataset.json\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "30f02b6c", + "metadata": {}, + "outputs": [], + "source": [ + "!python preprocess_map.py ./traces/dataset_filtered.json" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d10b392f", + "metadata": {}, + "outputs": [], + "source": [ + "!python preprocess_map.py NovaEval/noveum_customer_support_bt/traces/traces/dataset_filtered.json\n", + "\n" + ] + }, + { + "cell_type": "markdown", + "id": "008fd0e1", + "metadata": {}, + "source": [ + "## Running eval on the dataset" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0b22eb74", + "metadata": {}, + "outputs": [], + "source": [ + "# 1. Setup\n", + "!source .venv/bin/activate\n", + "!cd noveum_customer_support_bt\n", + "\n", + "# 2. Create Dataset\n", + "!python create_dataset.py --dataset-type agent --description \"Customer Support Agent Evaluation Dataset\" --pretty\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "9f7feb8d", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "šŸŽ¬ NOVEUM AI AGENT DEMO\n", - "==================================================\n", - "\n", - "1ļøāƒ£ Initializing the system...\n", - "šŸš€ Initializing Noveum AI Agent...\n", - "šŸ“ Using existing scraped data...\n", - "āœ… Loaded 38 documents from noveum_docs.json\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-10-22 22:26:35 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_system_initialization (ID: 9af2b4b0-0c2f-4d2e-ae99-30df0d2f0fa6) - 1 spans\n", - "2025-10-22 22:26:35 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_system_initialization (ID: 9af2b4b0-0c2f-4d2e-ae99-30df0d2f0fa6) - 1 spans\n", - "2025-10-22 22:26:35 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 9af2b4b0-0c2f-4d2e-ae99-30df0d2f0fa6\n", - "2025-10-22 22:26:35 - noveum_trace.transport.http_transport - INFO - āœ… Trace 9af2b4b0-0c2f-4d2e-ae99-30df0d2f0fa6 successfully queued for export\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "āœ… Loaded existing vector store from noveum_vectorstore\n", - "āœ… Noveum AI Agent initialized successfully!\n", - "\n", - "2ļøāƒ£ Running 20 demo queries...\n", + "Creating dataset version at: https://noveum.ai/api/v1/datasets/customersupportagentdemo-new/versions?organizationSlug=magic-api\n", + "Organization: magic-api\n", + "Dataset: customersupportagentdemo-new\n", + "Version: 0.0.2\n", + "Successfully created dataset version\n", + "Response status: 201\n", "\n", - "--- Demo Query 1 ---\n", + "Response saved to: dataset_version_response.json\n", "\n", - "šŸŽÆ Processing query: 'What is Noveum and what does it do?'\n", - "šŸ§  Routing to RAG system for: 'What is Noveum and what does it do?'\n", - "šŸ” Found 5 relevant documents for query: 'What is Noveum and what does it do?'\n" + "Response data:\n", + "{\n", + " \"success\": true,\n", + " \"version\": \"0.0.2\"\n", + "}\n" ] - }, + } + ], + "source": [ + "# 3. Create Version\n", + "!python create_dataset_version.py --pretty\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b3eeb124", + "metadata": {}, + "outputs": [], + "source": [ + "from demo_utils import run_complete_agent_evaluation\n", + "import os\n", + "\n", + "# Process all JSON files in split_datasets directory\n", + "for file in os.listdir('split_datasets'):\n", + " if file.endswith('.json'):\n", + " print(f'Processing {file}...')\n", + " run_complete_agent_evaluation(\n", + " f'split_datasets/{file}', \n", + " sample_size=25, \n", + " evaluation_name=file.replace('.json', ''),\n", + " output_dir='./demo_results'\n", + " )\n", + " print(f'Completed {file}\\n')" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "1f5ca2d3", + "metadata": {}, + "outputs": [ { - "name": "stderr", + "name": "stdout", "output_type": "stream", "text": [ - "2025-10-22 22:26:43 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_tool-orchestator (ID: 3ad192e5-bbd4-4236-8144-019d53512769) - 6 spans\n", - "2025-10-22 22:26:43 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_tool-orchestator (ID: 3ad192e5-bbd4-4236-8144-019d53512769) - 6 spans\n", - "2025-10-22 22:26:43 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 3ad192e5-bbd4-4236-8144-019d53512769\n", - "2025-10-22 22:26:43 - noveum_trace.transport.http_transport - INFO - āœ… Trace 3ad192e5-bbd4-4236-8144-019d53512769 successfully queued for export\n" + "Created api_data.json with 20 items\n", + "Sample items:\n", + " {'item_key': 'eda4fe22-9a2b-4b73-856b-f4f3309bf719', 'item_id': 'item_1'}\n", + " {'item_key': '0ffffba1-8a37-443c-8866-d53ffbfa7718', 'item_id': 'item_2'}\n", + " {'item_key': 'f1f37bd7-0851-4659-b493-b80d3800d920', 'item_id': 'item_3'}\n" ] - }, + } + ], + "source": [ + "\n", + "import pandas as pd\n", + "import json\n", + "\n", + "# Read the CSV file\n", + "df = pd.read_csv('demo_results/agent.query_routing_dataset/agent_evaluation_results.csv')\n", + "\n", + "# Create API data structure with all task_ids\n", + "api_data = {\n", + " 'items': [\n", + " {\n", + " 'item_key': str(row['task_id']),\n", + " 'item_id': f'item_{i+1}' # Generate unique item IDs\n", + " }\n", + " for i, row in df.iterrows()\n", + " ]\n", + "}\n", + "\n", + "# Save to JSON\n", + "with open('api_data.json', 'w') as f:\n", + " json.dump(api_data, f, indent=2)\n", + "\n", + "print('Created api_data.json with', len(api_data['items']), 'items')\n", + "print('Sample items:')\n", + "for item in api_data['items'][:3]:\n", + " print(f' {item}')\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "eb3eb1ba", + "metadata": {}, + "outputs": [], + "source": [ + "!python upload_scores.py demo_results/agent.query_routing_dataset/agent_evaluation_results.csv --item-key-col task_id --score-col context_relevancy --reasoning-col context_relevancy_reasoning --api-data api_data.json --scorer-id context_relevancy_scorer" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d511e96b", + "metadata": {}, + "outputs": [], + "source": [ + "!python upload_scores.py demo_results/agent.query_routing_dataset/agent_evaluation_results.csv --item-key-col task_id --score-col role_adherence --reasoning-col role_adherence_reasoning --api-data api_data.json --scorer-id role_adherence_scorer" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "08f948b3", + "metadata": {}, + "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "šŸ” Found 5 relevant documents for query: 'What is Noveum and what does it do?'\n", - "āœ… Query processed in 8.68s using RAG\n", - "\n", - "================================================================================\n", - "šŸ¤– NOVEUM AI AGENT RESPONSE\n", - "================================================================================\n", - "šŸ“Š Mode: RAG\n", - "ā±ļø Processing Time: 8.68s\n", - "šŸ“… Timestamp: 2025-10-22 22:26:43\n", - "šŸ“š Sources (5):\n", - " 1. https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform\n", - " 2. https://noveum.ai/en/docs/getting-started/overview\n", - " 3. https://noveum.ai/docs/getting-started/overview\n", - " ... and 2 more\n", - "\n", - "šŸ’¬ Answer:\n", - "----------------------------------------\n", - "Noveum.ai is a comprehensive tracing and observability platform specifically designed for AI applications, including LLM-powered chatbots, RAG systems, and multi-agent workflows. It provides the necessary insights to understand, debug, and optimize these systems, addressing the complexities involved in building production AI applications, such as LLM calls, vector searches, and agent reasoning. Noveum.ai enhances visibility into these workflows, making debugging easier and optimization more effective (Source 1, Source 2).\n", - "================================================================================\n", + "Loaded environment variables\n", + "Loading API data from api_data.json...\n", + "Loaded 20 item mappings\n", + "Reading CSV from demo_results/agent.query_routing_dataset/agent_evaluation_results.csv...\n", + "Read 20 rows from CSV\n", + "Created 20 results for upload\n", "\n", - "--- Demo Query 2 ---\n", + "Uploading 20 results in 1 batches...\n", + "āœ— Batch 1/1 failed: 201\n", + " Response: {\"success\":true,\"created\":20,\"failed\":0,\"results\":[{\"result_id\":\"35b42464-7989-4caa-90cb-5636319d4fd1\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_1\",\"scorer_id\":\"parameter_correctness_scorer\",\"score\":0,\"passed\":0,\"metadata\":\"{\\\"details\\\":\\\"Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:03:01.353000000\",\"updated_at\":\"2025-10-22 18:03:01.353000000\",\"scorer_name\":\"Parameter Correctness Scorer\"},{\"result_id\":\"28380f39-f4e4-41a3-a5c7-01241174fb78\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_10\",\"scorer_id\":\"parameter_correctness_scorer\",\"score\":0,\"passed\":0,\"metadata\":\"{\\\"details\\\":\\\"Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:03:01.353000000\",\"updated_at\":\"2025-10-22 18:03:01.353000000\",\"scorer_name\":\"Parameter Correctness Scorer\"},{\"result_id\":\"f1ea8545-c1ab-4791-958b-04d3c75cf92a\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_11\",\"scorer_id\":\"parameter_correctness_scorer\",\"score\":0,\"passed\":0,\"metadata\":\"{\\\"details\\\":\\\"Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:03:01.353000000\",\"updated_at\":\"2025-10-22 18:03:01.353000000\",\"scorer_name\":\"Parameter Correctness Scorer\"},{\"result_id\":\"db8c7724-19ca-46c6-8c6b-796f229279e4\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_12\",\"scorer_id\":\"parameter_correctness_scorer\",\"score\":0,\"passed\":0,\"metadata\":\"{\\\"details\\\":\\\"Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:03:01.353000000\",\"updated_at\":\"2025-10-22 18:03:01.353000000\",\"scorer_name\":\"Parameter Correctness Scorer\"},{\"result_id\":\"86929a3d-38c7-4bcf-8c40-820f3d9e69ef\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_13\",\"scorer_id\":\"parameter_correctness_scorer\",\"score\":0,\"passed\":0,\"metadata\":\"{\\\"details\\\":\\\"Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:03:01.353000000\",\"updated_at\":\"2025-10-22 18:03:01.353000000\",\"scorer_name\":\"Parameter Correctness Scorer\"},{\"result_id\":\"068aaa68-b454-47db-854d-c09b87d33ec9\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_14\",\"scorer_id\":\"parameter_correctness_scorer\",\"score\":0,\"passed\":0,\"metadata\":\"{\\\"details\\\":\\\"Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:03:01.353000000\",\"updated_at\":\"2025-10-22 18:03:01.353000000\",\"scorer_name\":\"Parameter Correctness Scorer\"},{\"result_id\":\"b80aac89-5007-412c-a372-d7a80703ec0f\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_15\",\"scorer_id\":\"parameter_correctness_scorer\",\"score\":0,\"passed\":0,\"metadata\":\"{\\\"details\\\":\\\"Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:03:01.353000000\",\"updated_at\":\"2025-10-22 18:03:01.353000000\",\"scorer_name\":\"Parameter Correctness Scorer\"},{\"result_id\":\"07585067-3e96-4491-8f4b-a4fa89b80493\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_16\",\"scorer_id\":\"parameter_correctness_scorer\",\"score\":0,\"passed\":0,\"metadata\":\"{\\\"details\\\":\\\"Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:03:01.353000000\",\"updated_at\":\"2025-10-22 18:03:01.353000000\",\"scorer_name\":\"Parameter Correctness Scorer\"},{\"result_id\":\"172395b1-bd1d-43dd-9359-51854355376b\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_17\",\"scorer_id\":\"parameter_correctness_scorer\",\"score\":0,\"passed\":0,\"metadata\":\"{\\\"details\\\":\\\"Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:03:01.353000000\",\"updated_at\":\"2025-10-22 18:03:01.353000000\",\"scorer_name\":\"Parameter Correctness Scorer\"},{\"result_id\":\"a54ae11d-5232-4434-b704-1a5f8b327ee7\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_18\",\"scorer_id\":\"parameter_correctness_scorer\",\"score\":0,\"passed\":0,\"metadata\":\"{\\\"details\\\":\\\"Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:03:01.353000000\",\"updated_at\":\"2025-10-22 18:03:01.353000000\",\"scorer_name\":\"Parameter Correctness Scorer\"},{\"result_id\":\"61847a0c-de20-4e36-abd0-2067bb210207\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_19\",\"scorer_id\":\"parameter_correctness_scorer\",\"score\":0,\"passed\":0,\"metadata\":\"{\\\"details\\\":\\\"Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:03:01.353000000\",\"updated_at\":\"2025-10-22 18:03:01.353000000\",\"scorer_name\":\"Parameter Correctness Scorer\"},{\"result_id\":\"70f241a7-05cc-47df-8bc3-2eb3b428d064\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_2\",\"scorer_id\":\"parameter_correctness_scorer\",\"score\":0,\"passed\":0,\"metadata\":\"{\\\"details\\\":\\\"Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:03:01.353000000\",\"updated_at\":\"2025-10-22 18:03:01.353000000\",\"scorer_name\":\"Parameter Correctness Scorer\"},{\"result_id\":\"1b5c2018-e801-4ac1-98e9-8dab56fa19ed\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_20\",\"scorer_id\":\"parameter_correctness_scorer\",\"score\":0,\"passed\":0,\"metadata\":\"{\\\"details\\\":\\\"Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:03:01.353000000\",\"updated_at\":\"2025-10-22 18:03:01.353000000\",\"scorer_name\":\"Parameter Correctness Scorer\"},{\"result_id\":\"65aa0719-a555-4fae-8203-d3f06bcd0f5e\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_3\",\"scorer_id\":\"parameter_correctness_scorer\",\"score\":0,\"passed\":0,\"metadata\":\"{\\\"details\\\":\\\"Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:03:01.353000000\",\"updated_at\":\"2025-10-22 18:03:01.353000000\",\"scorer_name\":\"Parameter Correctness Scorer\"},{\"result_id\":\"5e710451-f3d1-4d35-8ed2-8c6b54a46ccb\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_4\",\"scorer_id\":\"parameter_correctness_scorer\",\"score\":0,\"passed\":0,\"metadata\":\"{\\\"details\\\":\\\"Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:03:01.353000000\",\"updated_at\":\"2025-10-22 18:03:01.353000000\",\"scorer_name\":\"Parameter Correctness Scorer\"},{\"result_id\":\"1db2a945-e30a-43ee-b591-c98dbe004f47\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_5\",\"scorer_id\":\"parameter_correctness_scorer\",\"score\":0,\"passed\":0,\"metadata\":\"{\\\"details\\\":\\\"Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:03:01.353000000\",\"updated_at\":\"2025-10-22 18:03:01.353000000\",\"scorer_name\":\"Parameter Correctness Scorer\"},{\"result_id\":\"297fadf1-51fa-43e7-91cc-58b783550d3b\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_6\",\"scorer_id\":\"parameter_correctness_scorer\",\"score\":0,\"passed\":0,\"metadata\":\"{\\\"details\\\":\\\"Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:03:01.353000000\",\"updated_at\":\"2025-10-22 18:03:01.353000000\",\"scorer_name\":\"Parameter Correctness Scorer\"},{\"result_id\":\"f00da0b9-f3df-4055-9507-0b4b9ae31501\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_7\",\"scorer_id\":\"parameter_correctness_scorer\",\"score\":0,\"passed\":0,\"metadata\":\"{\\\"details\\\":\\\"Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:03:01.353000000\",\"updated_at\":\"2025-10-22 18:03:01.353000000\",\"scorer_name\":\"Parameter Correctness Scorer\"},{\"result_id\":\"3f9733b7-23b3-4396-84a6-5f87146cb0a2\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_8\",\"scorer_id\":\"parameter_correctness_scorer\",\"score\":0,\"passed\":0,\"metadata\":\"{\\\"details\\\":\\\"Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:03:01.353000000\",\"updated_at\":\"2025-10-22 18:03:01.353000000\",\"scorer_name\":\"Parameter Correctness Scorer\"},{\"result_id\":\"c6629883-f239-4b64-ba54-2fe517d25551\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_9\",\"scorer_id\":\"parameter_correctness_scorer\",\"score\":0,\"passed\":0,\"metadata\":\"{\\\"details\\\":\\\"Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:03:01.353000000\",\"updated_at\":\"2025-10-22 18:03:01.353000000\",\"scorer_name\":\"Parameter Correctness Scorer\"}]}\n", "\n", - "šŸŽÆ Processing query: 'How do I integrate Noveum Trace in my application?'\n", - "šŸ§  Routing to RAG system for: 'How do I integrate Noveum Trace in my application?'\n", - "šŸ” Found 5 relevant documents for query: 'How do I integrate Noveum Trace in my application?'\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-10-22 22:26:53 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_tool-orchestator (ID: bc60c122-f30b-4114-9f78-ee40a024b525) - 6 spans\n", - "2025-10-22 22:26:53 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_tool-orchestator (ID: bc60c122-f30b-4114-9f78-ee40a024b525) - 6 spans\n", - "2025-10-22 22:26:53 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace bc60c122-f30b-4114-9f78-ee40a024b525\n", - "2025-10-22 22:26:53 - noveum_trace.transport.http_transport - INFO - āœ… Trace bc60c122-f30b-4114-9f78-ee40a024b525 successfully queued for export\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "šŸ” Found 5 relevant documents for query: 'How do I integrate Noveum Trace in my application?'\n", - "āœ… Query processed in 8.83s using RAG\n", - "\n", - "================================================================================\n", - "šŸ¤– NOVEUM AI AGENT RESPONSE\n", - "================================================================================\n", - "šŸ“Š Mode: RAG\n", - "ā±ļø Processing Time: 8.83s\n", - "šŸ“… Timestamp: 2025-10-22 22:26:53\n", - "šŸ“š Sources (5):\n", - " 1. https://noveum.ai/en/docs\n", - " 2. https://noveum.ai/docs\n", - " 3. https://noveum.ai/docs/getting-started/sdk-integration\n", - " ... and 2 more\n", - "\n", - "šŸ’¬ Answer:\n", - "----------------------------------------\n", - "To integrate Noveum Trace into your application, you can follow these steps based on the SDK Integration Guide:\n", - "\n", - "1. **Create Your Account & Get API Key**:\n", - " - Sign up at [noveum.ai](https://noveum.ai).\n", - " - Create a project in your dashboard.\n", - " - Generate an API key.\n", - "\n", - "2. **Choose Your SDK**:\n", - " - For Python applications, use the **Python SDK** (`noveum-trace`), which offers decorator-based tracing for LLM calls, agents, and RAG pipelines.\n", - " - For TypeScript applications, use the **TypeScript SDK** (`@noveum/trace`), designed for frameworks like Next.js, Express.js, and Hono, with full type safety.\n", - "\n", - "3. **Integrate the SDK**:\n", - " - Follow the specific integration instructions for your chosen SDK. The SDKs are designed to provide comprehensive tracing and observability with minimal code changes.\n", - "\n", - "4. **Capture Metrics**:\n", - " - The SDKs automatically capture essential metrics and traces, including performance metrics, cost tracking, request/response data, and metadata.\n", - "\n", - "For detailed integration steps, refer to the SDK Integration Guide in the Noveum documentation ([Source 3](https://noveum.ai/docs/getting-started/sdk-integration)).\n", - "================================================================================\n", - "\n", - "--- Demo Query 3 ---\n", - "\n", - "šŸŽÆ Processing query: 'What are Noveum's pricing plans?'\n", - "šŸ§  Routing to RAG system for: 'What are Noveum's pricing plans?'\n", - "šŸ” Found 5 relevant documents for query: 'What are Noveum's pricing plans?'\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-10-22 22:26:58 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_tool-orchestator (ID: 29a862bf-d7aa-4cf9-963c-0a4244343bbe) - 6 spans\n", - "2025-10-22 22:26:58 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_tool-orchestator (ID: 29a862bf-d7aa-4cf9-963c-0a4244343bbe) - 6 spans\n", - "2025-10-22 22:26:58 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 29a862bf-d7aa-4cf9-963c-0a4244343bbe\n", - "2025-10-22 22:26:58 - noveum_trace.transport.http_transport - INFO - āœ… Trace 29a862bf-d7aa-4cf9-963c-0a4244343bbe successfully queued for export\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "šŸ” Found 5 relevant documents for query: 'What are Noveum's pricing plans?'\n", - "āœ… Query processed in 3.49s using RAG\n", - "\n", - "================================================================================\n", - "šŸ¤– NOVEUM AI AGENT RESPONSE\n", - "================================================================================\n", - "šŸ“Š Mode: RAG\n", - "ā±ļø Processing Time: 3.49s\n", - "šŸ“… Timestamp: 2025-10-22 22:26:58\n", - "šŸ“š Sources (5):\n", - " 1. https://noveum.ai/docs/getting-started/overview\n", - " 2. https://noveum.ai/en/docs/getting-started/overview\n", - " 3. https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform\n", - " ... and 2 more\n", - "\n", - "šŸ’¬ Answer:\n", - "----------------------------------------\n", - "The provided context does not include specific information about Noveum's pricing plans. It mentions that all users will have access to the Observability suite for free, and early users will receive free evaluation jobs and premium support for the first year (Source 1 and Source 2). However, details on ongoing pricing or additional plans are not available in the documentation. If you have further questions about Noveum, feel free to ask!\n", - "================================================================================\n", - "\n", - "--- Demo Query 4 ---\n", - "\n", - "šŸŽÆ Processing query: 'What features does Noveum Trace offer?'\n", - "šŸ§  Routing to RAG system for: 'What features does Noveum Trace offer?'\n", - "šŸ” Found 5 relevant documents for query: 'What features does Noveum Trace offer?'\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-10-22 22:27:05 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_tool-orchestator (ID: df0c6344-1a2e-4c53-bb7d-4e43b3bce1b8) - 6 spans\n", - "2025-10-22 22:27:05 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_tool-orchestator (ID: df0c6344-1a2e-4c53-bb7d-4e43b3bce1b8) - 6 spans\n", - "2025-10-22 22:27:05 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace df0c6344-1a2e-4c53-bb7d-4e43b3bce1b8\n", - "2025-10-22 22:27:05 - noveum_trace.transport.http_transport - INFO - āœ… Trace df0c6344-1a2e-4c53-bb7d-4e43b3bce1b8 successfully queued for export\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "šŸ” Found 5 relevant documents for query: 'What features does Noveum Trace offer?'\n", - "āœ… Query processed in 6.38s using RAG\n", - "\n", - "================================================================================\n", - "šŸ¤– NOVEUM AI AGENT RESPONSE\n", - "================================================================================\n", - "šŸ“Š Mode: RAG\n", - "ā±ļø Processing Time: 6.38s\n", - "šŸ“… Timestamp: 2025-10-22 22:27:05\n", - "šŸ“š Sources (5):\n", - " 1. https://noveum.ai/en/docs\n", - " 2. https://noveum.ai/docs\n", - " 3. https://noveum.ai/en/docs/getting-started/sdk-integration\n", - " ... and 2 more\n", - "\n", - "šŸ’¬ Answer:\n", - "----------------------------------------\n", - "Noveum Trace offers several key features through its Python SDK (`noveum-trace`) and TypeScript SDK (`@noveum/trace`):\n", - "\n", - "1. **Decorator-based Tracing**: It provides a simple way to trace LLM calls, agents, and RAG pipelines using decorators.\n", - "2. **Automatic Instrumentation**: The SDK automatically instruments popular AI frameworks, making it easier to integrate tracing without extensive manual setup.\n", - "3. **Context Propagation**: It supports context propagation across asynchronous operations, ensuring that trace data remains consistent throughout the execution flow.\n", - "4. **TypeScript-first Design**: The TypeScript SDK is designed with full type safety and universal compatibility across Node.js, Edge Runtime, and browsers.\n", - "5. **Framework Integrations**: It includes integrations for popular frameworks like Next.js, Express.js, and Hono.\n", - "\n", - "Additionally, the Noveum Platform features a real-time dashboard for analyzing traces and performance, allowing for collaborative insights and monitoring (Sources 1, 2, 3, 4).\n", - "================================================================================\n", - "\n", - "--- Demo Query 5 ---\n", - "\n", - "šŸŽÆ Processing query: 'How do I set up observability with Noveum?'\n", - "šŸ§  Routing to RAG system for: 'How do I set up observability with Noveum?'\n", - "šŸ” Found 5 relevant documents for query: 'How do I set up observability with Noveum?'\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-10-22 22:27:12 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_tool-orchestator (ID: 93bd37c4-4289-43c9-b34e-19eaaef610c2) - 6 spans\n", - "2025-10-22 22:27:12 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_tool-orchestator (ID: 93bd37c4-4289-43c9-b34e-19eaaef610c2) - 6 spans\n", - "2025-10-22 22:27:12 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 93bd37c4-4289-43c9-b34e-19eaaef610c2\n", - "2025-10-22 22:27:12 - noveum_trace.transport.http_transport - INFO - āœ… Trace 93bd37c4-4289-43c9-b34e-19eaaef610c2 successfully queued for export\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "šŸ” Found 5 relevant documents for query: 'How do I set up observability with Noveum?'\n", - "āœ… Query processed in 5.72s using RAG\n", - "\n", - "================================================================================\n", - "šŸ¤– NOVEUM AI AGENT RESPONSE\n", - "================================================================================\n", - "šŸ“Š Mode: RAG\n", - "ā±ļø Processing Time: 5.72s\n", - "šŸ“… Timestamp: 2025-10-22 22:27:12\n", - "šŸ“š Sources (5):\n", - " 1. https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform\n", - " 2. https://noveum.ai/en/docs/getting-started/tracing-concepts\n", - " 3. https://noveum.ai/docs/getting-started/tracing-concepts\n", - " ... and 2 more\n", - "\n", - "šŸ’¬ Answer:\n", - "----------------------------------------\n", - "To set up observability with Noveum, follow these steps:\n", - "\n", - "1. **Sign Up**: Go to [noveum.ai](https://noveum.ai) and create an account.\n", - "2. **Create a Project**: After signing up, create a project to obtain your API key.\n", - "3. **Install the SDK**: Choose your preferred programming language and install the corresponding SDK.\n", - "4. **Add Tracing**: Integrate tracing into your AI workflows using the SDK.\n", - "5. **Explore Insights**: Use the Noveum.ai dashboard to analyze the insights gathered from your traces.\n", - "\n", - "For more detailed guidance, you can explore framework-specific integrations and advanced instrumentation techniques in the Noveum documentation (Source 2 and Source 3). Remember, effective observability is about collecting the right data to help you understand, debug, and optimize your AI applications (Source 2).\n", - "================================================================================\n", - "\n", - "--- Demo Query 6 ---\n", - "\n", - "šŸŽÆ Processing query: 'What APIs are available in Noveum platform?'\n", - "šŸ§  Routing to RAG system for: 'What APIs are available in Noveum platform?'\n", - "šŸ” Found 5 relevant documents for query: 'What APIs are available in Noveum platform?'\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-10-22 22:27:18 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_tool-orchestator (ID: fee3e6d3-8da1-4ef3-8214-a7d7323d3a49) - 6 spans\n", - "2025-10-22 22:27:18 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_tool-orchestator (ID: fee3e6d3-8da1-4ef3-8214-a7d7323d3a49) - 6 spans\n", - "2025-10-22 22:27:18 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace fee3e6d3-8da1-4ef3-8214-a7d7323d3a49\n", - "2025-10-22 22:27:18 - noveum_trace.transport.http_transport - INFO - āœ… Trace fee3e6d3-8da1-4ef3-8214-a7d7323d3a49 successfully queued for export\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "šŸ” Found 5 relevant documents for query: 'What APIs are available in Noveum platform?'\n", - "āœ… Query processed in 5.00s using RAG\n", - "\n", - "================================================================================\n", - "šŸ¤– NOVEUM AI AGENT RESPONSE\n", - "================================================================================\n", - "šŸ“Š Mode: RAG\n", - "ā±ļø Processing Time: 5.00s\n", - "šŸ“… Timestamp: 2025-10-22 22:27:18\n", - "šŸ“š Sources (5):\n", - " 1. https://noveum.ai/en/docs\n", - " 2. https://noveum.ai/docs\n", - " 3. https://noveum.ai/docs/getting-started/overview\n", - " ... and 2 more\n", - "\n", - "šŸ’¬ Answer:\n", - "----------------------------------------\n", - "The provided context does not specify the APIs available in the Noveum platform. For detailed information about the APIs, I recommend checking the official Noveum documentation or the SDK Integration Guide. If you have any other questions about Noveum, feel free to ask!\n", - "================================================================================\n", - "\n", - "--- Demo Query 7 ---\n", - "\n", - "šŸŽÆ Processing query: 'How does Noveum handle agent tracing?'\n", - "šŸ§  Routing to RAG system for: 'How does Noveum handle agent tracing?'\n", - "šŸ” Found 5 relevant documents for query: 'How does Noveum handle agent tracing?'\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-10-22 22:27:23 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_tool-orchestator (ID: 3bbd0b36-c30f-4126-b62a-dd0fae1c701d) - 6 spans\n", - "2025-10-22 22:27:23 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_tool-orchestator (ID: 3bbd0b36-c30f-4126-b62a-dd0fae1c701d) - 6 spans\n", - "2025-10-22 22:27:23 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 3bbd0b36-c30f-4126-b62a-dd0fae1c701d\n", - "2025-10-22 22:27:23 - noveum_trace.transport.http_transport - INFO - āœ… Trace 3bbd0b36-c30f-4126-b62a-dd0fae1c701d successfully queued for export\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "šŸ” Found 5 relevant documents for query: 'How does Noveum handle agent tracing?'\n", - "āœ… Query processed in 4.48s using RAG\n", - "\n", - "================================================================================\n", - "šŸ¤– NOVEUM AI AGENT RESPONSE\n", - "================================================================================\n", - "šŸ“Š Mode: RAG\n", - "ā±ļø Processing Time: 4.48s\n", - "šŸ“… Timestamp: 2025-10-22 22:27:23\n", - "šŸ“š Sources (5):\n", - " 1. https://noveum.ai/en/blog/noveum-ai-your-one-stop-ai-evaluation-platform\n", - " 2. https://noveum.ai/docs/advanced/multi-agent-tracing\n", - " 3. https://noveum.ai/en/docs/advanced/multi-agent-tracing\n", - " ... and 2 more\n", - "\n", - "šŸ’¬ Answer:\n", - "----------------------------------------\n", - "Noveum.ai handles agent tracing through its specialized Multi-Agent Tracing capabilities, which are designed to observe complex workflows and inter-agent communications within multi-agent systems. These systems often involve multiple agents that coordinate, communicate, and collaborate to achieve shared goals, presenting unique observability challenges. Noveum.ai provides comprehensive tracing to help users understand and optimize these intricate workflows, ensuring better visibility into agent interactions and performance (Source 2 and Source 3). \n", - "\n", - "For more detailed insights or specific use cases, you may want to explore the Noveum documentation further.\n", - "================================================================================\n", - "\n", - "--- Demo Query 8 ---\n", - "\n", - "šŸŽÆ Processing query: 'What monitoring capabilities does Noveum provide?'\n", - "šŸ§  Routing to RAG system for: 'What monitoring capabilities does Noveum provide?'\n", - "šŸ” Found 5 relevant documents for query: 'What monitoring capabilities does Noveum provide?'\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-10-22 22:27:31 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_tool-orchestator (ID: fb0d31fc-2465-4ecd-973c-4cd443d9befa) - 6 spans\n", - "2025-10-22 22:27:31 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_tool-orchestator (ID: fb0d31fc-2465-4ecd-973c-4cd443d9befa) - 6 spans\n", - "2025-10-22 22:27:31 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace fb0d31fc-2465-4ecd-973c-4cd443d9befa\n", - "2025-10-22 22:27:31 - noveum_trace.transport.http_transport - INFO - āœ… Trace fb0d31fc-2465-4ecd-973c-4cd443d9befa successfully queued for export\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "šŸ” Found 5 relevant documents for query: 'What monitoring capabilities does Noveum provide?'\n", - "āœ… Query processed in 6.97s using RAG\n", - "\n", - "================================================================================\n", - "šŸ¤– NOVEUM AI AGENT RESPONSE\n", - "================================================================================\n", - "šŸ“Š Mode: RAG\n", - "ā±ļø Processing Time: 6.97s\n", - "šŸ“… Timestamp: 2025-10-22 22:27:31\n", - "šŸ“š Sources (5):\n", - " 1. https://noveum.ai/en\n", - " 2. https://noveum.ai/\n", - " 3. https://noveum.ai\n", - " ... and 2 more\n", - "\n", - "šŸ’¬ Answer:\n", - "----------------------------------------\n", - "Noveum provides comprehensive monitoring capabilities for AI applications, including the ability to monitor, trace, and optimize AI agents across various frameworks such as LangChain, CrewAI, AutoGen, and custom implementations. The platform features a unified dashboard that allows users to monitor everything in their AI ecosystem, capturing every trace and span from simple LLM calls to complex multi-agent interactions. This is facilitated by lightweight SDKs that ensure no detail is missed, enabling users to evaluate and improve their AI agents effectively (Sources 1, 2, 4).\n", - "================================================================================\n", - "\n", - "--- Demo Query 9 ---\n", - "\n", - "šŸŽÆ Processing query: 'How do I configure Noveum for my system?'\n", - "šŸ§  Routing to RAG system for: 'How do I configure Noveum for my system?'\n", - "šŸ” Found 5 relevant documents for query: 'How do I configure Noveum for my system?'\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-10-22 22:27:37 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_tool-orchestator (ID: 1ae0d6c4-d1fb-4239-b536-3333187d44aa) - 6 spans\n", - "2025-10-22 22:27:37 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_tool-orchestator (ID: 1ae0d6c4-d1fb-4239-b536-3333187d44aa) - 6 spans\n", - "2025-10-22 22:27:37 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 1ae0d6c4-d1fb-4239-b536-3333187d44aa\n", - "2025-10-22 22:27:37 - noveum_trace.transport.http_transport - INFO - āœ… Trace 1ae0d6c4-d1fb-4239-b536-3333187d44aa successfully queued for export\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "šŸ” Found 5 relevant documents for query: 'How do I configure Noveum for my system?'\n", - "āœ… Query processed in 4.69s using RAG\n", - "\n", - "================================================================================\n", - "šŸ¤– NOVEUM AI AGENT RESPONSE\n", - "================================================================================\n", - "šŸ“Š Mode: RAG\n", - "ā±ļø Processing Time: 4.69s\n", - "šŸ“… Timestamp: 2025-10-22 22:27:37\n", - "šŸ“š Sources (5):\n", - " 1. https://noveum.ai/en/docs\n", - " 2. https://noveum.ai/docs\n", - " 3. https://noveum.ai/docs/getting-started/overview\n", - " ... and 2 more\n", - "\n", - "šŸ’¬ Answer:\n", - "----------------------------------------\n", - "To configure Noveum for your system, you can refer to the SDK Integration Guide available in the Noveum documentation. This guide will help you trace your AI applications with minimal code changes. The SDKs provided for Python and TypeScript are designed to facilitate this integration. \n", - "\n", - "For detailed steps and specific configurations, please check the relevant sections in the documentation. If you need further assistance, you can also reach out to the Noveum community via Discord or email support at [email protected] (Sources: Source 1, Source 5).\n", - "================================================================================\n", - "\n", - "--- Demo Query 10 ---\n", - "\n", - "šŸŽÆ Processing query: 'What are the benefits of using Noveum Trace?'\n", - "šŸ§  Routing to RAG system for: 'What are the benefits of using Noveum Trace?'\n", - "šŸ” Found 5 relevant documents for query: 'What are the benefits of using Noveum Trace?'\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-10-22 22:27:46 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_tool-orchestator (ID: 823b7927-10bb-49ef-a94e-1b914c298bfb) - 6 spans\n", - "2025-10-22 22:27:46 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_tool-orchestator (ID: 823b7927-10bb-49ef-a94e-1b914c298bfb) - 6 spans\n", - "2025-10-22 22:27:46 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 823b7927-10bb-49ef-a94e-1b914c298bfb\n", - "2025-10-22 22:27:46 - noveum_trace.transport.http_transport - INFO - āœ… Trace 823b7927-10bb-49ef-a94e-1b914c298bfb successfully queued for export\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "šŸ” Found 5 relevant documents for query: 'What are the benefits of using Noveum Trace?'\n", - "āœ… Query processed in 7.97s using RAG\n", - "\n", - "================================================================================\n", - "šŸ¤– NOVEUM AI AGENT RESPONSE\n", - "================================================================================\n", - "šŸ“Š Mode: RAG\n", - "ā±ļø Processing Time: 7.97s\n", - "šŸ“… Timestamp: 2025-10-22 22:27:46\n", - "šŸ“š Sources (5):\n", - " 1. https://noveum.ai/docs/platform/dashboard\n", - " 2. https://noveum.ai/en/docs\n", - " 3. https://noveum.ai/docs\n", - " ... and 2 more\n", - "\n", - "šŸ’¬ Answer:\n", - "----------------------------------------\n", - "Using Noveum Trace offers several benefits, including:\n", - "\n", - "1. **Real-Time Monitoring**: The Noveum platform provides a real-time dashboard for analyzing traces and performance, with live updates as new traces arrive and real-time status monitoring of your trace ingestion pipeline (Source 1).\n", - "\n", - "2. **Advanced Analysis**: Users can perform comparative analysis across multiple traces, allowing for performance comparisons and insights into system health through live metrics (Source 1).\n", - "\n", - "3. **Cost Analysis and Optimization**: Noveum Trace includes features for cost analysis and optimization recommendations, helping users manage and reduce expenses associated with AI operations (Source 4).\n", - "\n", - "4. **Collaboration Features**: The platform supports team collaboration by enabling shared insights and alerts, facilitating better communication and teamwork (Source 4).\n", - "\n", - "5. **Comprehensive Tracing**: With the Python and TypeScript SDKs, Noveum Trace offers decorator-based tracing for LLM calls, agents, and RAG pipelines, along with automatic instrumentation for popular AI frameworks (Source 2).\n", - "\n", - "6. **Detailed Metrics**: It provides detailed metrics on model calls, token usage, prompt engineering effectiveness, and response quality, which are crucial for optimizing AI applications (Source 4).\n", - "\n", - "Overall, Noveum Trace enhances the ability to monitor, analyze, and optimize AI applications effectively.\n", - "================================================================================\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/Users/mramanindia/work/NovaEval/.venv/lib/python3.12/site-packages/langchain_community/utilities/duckduckgo_search.py:63: RuntimeWarning: This package (`duckduckgo_search`) has been renamed to `ddgs`! Use `pip install ddgs` instead.\n", - " with DDGS() as ddgs:\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "--- Demo Query 11 ---\n", - "\n", - "šŸŽÆ Processing query: 'What are the latest AI news today?'\n", - "šŸŒ Routing to Web Search for: 'What are the latest AI news today?'\n", - "šŸ” Found 1 web search results for: 'What are the latest AI news today?'\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-10-22 22:27:50 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_tool-orchestator (ID: e4f3cc20-c137-4905-80fe-2d29b03c2231) - 6 spans\n", - "2025-10-22 22:27:50 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_tool-orchestator (ID: e4f3cc20-c137-4905-80fe-2d29b03c2231) - 6 spans\n", - "2025-10-22 22:27:50 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace e4f3cc20-c137-4905-80fe-2d29b03c2231\n", - "2025-10-22 22:27:50 - noveum_trace.transport.http_transport - INFO - āœ… Trace e4f3cc20-c137-4905-80fe-2d29b03c2231 successfully queued for export\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "āœ… Query processed in 3.48s using Web Search\n", - "\n", - "================================================================================\n", - "šŸ¤– NOVEUM AI AGENT RESPONSE\n", - "================================================================================\n", - "šŸ“Š Mode: Web Search\n", - "ā±ļø Processing Time: 3.48s\n", - "šŸ“… Timestamp: 2025-10-22 22:27:50\n", - "šŸ“š Sources (1):\n", - " 1. https://duckduckgo.com/?q=What+are+the+latest+AI+news+today?\n", - "\n", - "šŸ’¬ Answer:\n", - "----------------------------------------\n", - "The provided web search results do not contain specific information about the latest AI news today. They primarily focus on general news coverage from India and around the world, including politics, business, and entertainment, but do not mention any recent developments or updates in artificial intelligence.\n", - "\n", - "For the latest AI news, I recommend checking dedicated technology news websites or platforms that specialize in AI developments, as they would provide more relevant and up-to-date information.\n", - "================================================================================\n", - "\n", - "--- Demo Query 12 ---\n", - "\n", - "šŸŽÆ Processing query: 'What's the weather like today?'\n", - "šŸŒ Routing to Web Search for: 'What's the weather like today?'\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/Users/mramanindia/work/NovaEval/.venv/lib/python3.12/site-packages/langchain_community/utilities/duckduckgo_search.py:63: RuntimeWarning: This package (`duckduckgo_search`) has been renamed to `ddgs`! Use `pip install ddgs` instead.\n", - " with DDGS() as ddgs:\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "šŸ” Found 1 web search results for: 'What's the weather like today?'\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-10-22 22:27:56 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_tool-orchestator (ID: b1b13e60-5b32-44a0-9769-197a944989d0) - 6 spans\n", - "2025-10-22 22:27:56 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_tool-orchestator (ID: b1b13e60-5b32-44a0-9769-197a944989d0) - 6 spans\n", - "2025-10-22 22:27:56 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace b1b13e60-5b32-44a0-9769-197a944989d0\n", - "2025-10-22 22:27:56 - noveum_trace.transport.http_transport - INFO - āœ… Trace b1b13e60-5b32-44a0-9769-197a944989d0 successfully queued for export\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "āœ… Query processed in 4.93s using Web Search\n", - "\n", - "================================================================================\n", - "šŸ¤– NOVEUM AI AGENT RESPONSE\n", - "================================================================================\n", - "šŸ“Š Mode: Web Search\n", - "ā±ļø Processing Time: 4.93s\n", - "šŸ“… Timestamp: 2025-10-22 22:27:56\n", - "šŸ“š Sources (1):\n", - " 1. https://duckduckgo.com/?q=What's+the+weather+like+today?\n", - "\n", - "šŸ’¬ Answer:\n", - "----------------------------------------\n", - "The web search results do not provide specific information about today's weather. To find out the current weather conditions, I recommend checking a reliable weather website or app for detailed updates, including temperature, wind, and precipitation forecasts.\n", - "================================================================================\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/Users/mramanindia/work/NovaEval/.venv/lib/python3.12/site-packages/langchain_community/utilities/duckduckgo_search.py:63: RuntimeWarning: This package (`duckduckgo_search`) has been renamed to `ddgs`! Use `pip install ddgs` instead.\n", - " with DDGS() as ddgs:\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "--- Demo Query 13 ---\n", - "\n", - "šŸŽÆ Processing query: 'Tell me about recent developments in machine learning'\n", - "šŸŒ Routing to Web Search for: 'Tell me about recent developments in machine learning'\n", - "šŸ” Found 1 web search results for: 'Tell me about recent developments in machine learning'\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-10-22 22:28:03 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_tool-orchestator (ID: c5a64d08-d00f-46cf-ab3c-d8d473c10eca) - 6 spans\n", - "2025-10-22 22:28:03 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_tool-orchestator (ID: c5a64d08-d00f-46cf-ab3c-d8d473c10eca) - 6 spans\n", - "2025-10-22 22:28:03 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace c5a64d08-d00f-46cf-ab3c-d8d473c10eca\n", - "2025-10-22 22:28:03 - noveum_trace.transport.http_transport - INFO - āœ… Trace c5a64d08-d00f-46cf-ab3c-d8d473c10eca successfully queued for export\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "āœ… Query processed in 5.97s using Web Search\n", - "\n", - "================================================================================\n", - "šŸ¤– NOVEUM AI AGENT RESPONSE\n", - "================================================================================\n", - "šŸ“Š Mode: Web Search\n", - "ā±ļø Processing Time: 5.97s\n", - "šŸ“… Timestamp: 2025-10-22 22:28:03\n", - "šŸ“š Sources (1):\n", - " 1. https://duckduckgo.com/?q=Tell+me+about+recent+developments+in+machine+learning\n", - "\n", - "šŸ’¬ Answer:\n", - "----------------------------------------\n", - "The web search results did not provide specific information about recent developments in machine learning. Therefore, I cannot summarize any recent advancements based on the provided data.\n", - "\n", - "However, if you're interested in general trends in machine learning, I can mention that recent developments often include advancements in deep learning architectures, improvements in natural language processing (NLP) models, and increased applications of machine learning in various industries such as healthcare, finance, and autonomous systems. For the latest updates, I recommend checking reputable tech news sources or academic journals focused on artificial intelligence and machine learning.\n", - "================================================================================\n", - "\n", - "--- Demo Query 14 ---\n", - "\n", - "šŸŽÆ Processing query: 'What are the current trends in observability tools?'\n", - "šŸ§  Routing to RAG system for: 'What are the current trends in observability tools?'\n", - "šŸ” Found 5 relevant documents for query: 'What are the current trends in observability tools?'\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-10-22 22:28:10 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_tool-orchestator (ID: 45efae74-ee25-4004-a509-6e0ff628a0cd) - 6 spans\n", - "2025-10-22 22:28:10 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_tool-orchestator (ID: 45efae74-ee25-4004-a509-6e0ff628a0cd) - 6 spans\n", - "2025-10-22 22:28:10 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 45efae74-ee25-4004-a509-6e0ff628a0cd\n", - "2025-10-22 22:28:10 - noveum_trace.transport.http_transport - INFO - āœ… Trace 45efae74-ee25-4004-a509-6e0ff628a0cd successfully queued for export\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "šŸ” Found 5 relevant documents for query: 'What are the current trends in observability tools?'\n", - "āœ… Query processed in 5.15s using RAG\n", - "\n", - "================================================================================\n", - "šŸ¤– NOVEUM AI AGENT RESPONSE\n", - "================================================================================\n", - "šŸ“Š Mode: RAG\n", - "ā±ļø Processing Time: 5.15s\n", - "šŸ“… Timestamp: 2025-10-22 22:28:10\n", - "šŸ“š Sources (5):\n", - " 1. https://noveum.ai/en/blog/from-logs-to-intelligent-choices-inside-noveum-ais-evaluation-process\n", - " 2. https://noveum.ai/docs/getting-started/tracing-concepts\n", - " 3. https://noveum.ai/en/docs/getting-started/tracing-concepts\n", - " ... and 2 more\n", - "\n", - "šŸ’¬ Answer:\n", - "----------------------------------------\n", - "The provided context does not explicitly outline current trends in observability tools. However, it highlights the unique challenges faced by AI applications that traditional monitoring tools cannot adequately address. For instance, Noveum.ai emphasizes the importance of comprehensive tracing and collecting the right data to understand, debug, and optimize AI systems, rather than just gathering all possible data (Source 2 and Source 4).\n", - "\n", - "Additionally, the context mentions advanced practices such as comprehensive pipeline tracing, quality monitoring, and A/B testing for RAG components, which suggest a trend towards more specialized and tailored observability solutions for AI applications (Source 5).\n", - "\n", - "If you have specific questions about Noveum.ai's observability platform or its features, feel free to ask!\n", - "================================================================================\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/Users/mramanindia/work/NovaEval/.venv/lib/python3.12/site-packages/langchain_community/utilities/duckduckgo_search.py:63: RuntimeWarning: This package (`duckduckgo_search`) has been renamed to `ddgs`! Use `pip install ddgs` instead.\n", - " with DDGS() as ddgs:\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "--- Demo Query 15 ---\n", - "\n", - "šŸŽÆ Processing query: 'What happened in tech news this week?'\n", - "šŸŒ Routing to Web Search for: 'What happened in tech news this week?'\n", - "šŸ” Found 1 web search results for: 'What happened in tech news this week?'\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-10-22 22:28:14 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_tool-orchestator (ID: e30e9ba6-ffd1-4eb9-b7ae-6d19e04343d6) - 6 spans\n", - "2025-10-22 22:28:14 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_tool-orchestator (ID: e30e9ba6-ffd1-4eb9-b7ae-6d19e04343d6) - 6 spans\n", - "2025-10-22 22:28:14 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace e30e9ba6-ffd1-4eb9-b7ae-6d19e04343d6\n", - "2025-10-22 22:28:14 - noveum_trace.transport.http_transport - INFO - āœ… Trace e30e9ba6-ffd1-4eb9-b7ae-6d19e04343d6 successfully queued for export\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "āœ… Query processed in 3.22s using Web Search\n", - "\n", - "================================================================================\n", - "šŸ¤– NOVEUM AI AGENT RESPONSE\n", - "================================================================================\n", - "šŸ“Š Mode: Web Search\n", - "ā±ļø Processing Time: 3.22s\n", - "šŸ“… Timestamp: 2025-10-22 22:28:14\n", - "šŸ“š Sources (1):\n", - " 1. https://duckduckgo.com/?q=What+happened+in+tech+news+this+week?\n", - "\n", - "šŸ’¬ Answer:\n", - "----------------------------------------\n", - "The web search results do not provide specific information about recent events in tech news for this week. They primarily discuss the grammatical usage of the word \"happened\" and its variations, such as \"what happened\" and \"what's happened.\" \n", - "\n", - "To find out what happened in tech news this week, I recommend checking reliable tech news websites or platforms that aggregate current events in technology. If you have specific topics or companies in mind, I can help guide you on where to look for that information.\n", - "================================================================================\n", - "\n", - "--- Demo Query 16 ---\n", - "\n", - "šŸŽÆ Processing query: 'What are the latest updates in Python programming?'\n", - "šŸŒ Routing to Web Search for: 'What are the latest updates in Python programming?'\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/Users/mramanindia/work/NovaEval/.venv/lib/python3.12/site-packages/langchain_community/utilities/duckduckgo_search.py:63: RuntimeWarning: This package (`duckduckgo_search`) has been renamed to `ddgs`! Use `pip install ddgs` instead.\n", - " with DDGS() as ddgs:\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "šŸ” Found 1 web search results for: 'What are the latest updates in Python programming?'\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-10-22 22:28:18 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_tool-orchestator (ID: 51f4534f-23e4-4be7-8d6e-e93391d6e0e2) - 6 spans\n", - "2025-10-22 22:28:18 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_tool-orchestator (ID: 51f4534f-23e4-4be7-8d6e-e93391d6e0e2) - 6 spans\n", - "2025-10-22 22:28:18 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 51f4534f-23e4-4be7-8d6e-e93391d6e0e2\n", - "2025-10-22 22:28:18 - noveum_trace.transport.http_transport - INFO - āœ… Trace 51f4534f-23e4-4be7-8d6e-e93391d6e0e2 successfully queued for export\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "āœ… Query processed in 3.62s using Web Search\n", - "\n", - "================================================================================\n", - "šŸ¤– NOVEUM AI AGENT RESPONSE\n", - "================================================================================\n", - "šŸ“Š Mode: Web Search\n", - "ā±ļø Processing Time: 3.62s\n", - "šŸ“… Timestamp: 2025-10-22 22:28:18\n", - "šŸ“š Sources (1):\n", - " 1. https://duckduckgo.com/?q=What+are+the+latest+updates+in+Python+programming?\n", - "\n", - "šŸ’¬ Answer:\n", - "----------------------------------------\n", - "The provided web search results do not contain specific information about the latest updates in Python programming. They primarily focus on general news coverage and updates from India and around the world, without addressing programming or Python specifically.\n", - "\n", - "For the latest updates in Python programming, I recommend checking official sources such as the Python Software Foundation's website or popular programming news platforms like Real Python or Python Weekly. These sources typically provide information on new releases, features, and enhancements in the Python language.\n", - "================================================================================\n", - "\n", - "--- Demo Query 17 ---\n", - "\n", - "šŸŽÆ Processing query: 'What's the current status of cryptocurrency markets?'\n", - "šŸŒ Routing to Web Search for: 'What's the current status of cryptocurrency markets?'\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/Users/mramanindia/work/NovaEval/.venv/lib/python3.12/site-packages/langchain_community/utilities/duckduckgo_search.py:63: RuntimeWarning: This package (`duckduckgo_search`) has been renamed to `ddgs`! Use `pip install ddgs` instead.\n", - " with DDGS() as ddgs:\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "šŸ” Found 1 web search results for: 'What's the current status of cryptocurrency markets?'\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-10-22 22:28:25 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_tool-orchestator (ID: 3f7b6a62-8888-4a5d-b12c-37a3829aea49) - 6 spans\n", - "2025-10-22 22:28:25 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_tool-orchestator (ID: 3f7b6a62-8888-4a5d-b12c-37a3829aea49) - 6 spans\n", - "2025-10-22 22:28:25 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 3f7b6a62-8888-4a5d-b12c-37a3829aea49\n", - "2025-10-22 22:28:25 - noveum_trace.transport.http_transport - INFO - āœ… Trace 3f7b6a62-8888-4a5d-b12c-37a3829aea49 successfully queued for export\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "āœ… Query processed in 5.65s using Web Search\n", - "\n", - "================================================================================\n", - "šŸ¤– NOVEUM AI AGENT RESPONSE\n", - "================================================================================\n", - "šŸ“Š Mode: Web Search\n", - "ā±ļø Processing Time: 5.65s\n", - "šŸ“… Timestamp: 2025-10-22 22:28:25\n", - "šŸ“š Sources (1):\n", - " 1. https://duckduckgo.com/?q=What's+the+current+status+of+cryptocurrency+markets?\n", - "\n", - "šŸ’¬ Answer:\n", - "----------------------------------------\n", - "The provided web search results do not contain specific information regarding the current status of cryptocurrency markets. To get the latest updates on cryptocurrency prices, trends, and market analysis, I recommend checking financial news websites, cryptocurrency exchanges, or market tracking platforms like CoinMarketCap or CoinGecko. If you have any other questions or need further assistance, feel free to ask!\n", - "================================================================================\n", - "\n", - "--- Demo Query 18 ---\n", - "\n", - "šŸŽÆ Processing query: 'What are the newest features in cloud computing?'\n", - "šŸ§  Routing to RAG system for: 'What are the newest features in cloud computing?'\n", - "šŸ” Found 5 relevant documents for query: 'What are the newest features in cloud computing?'\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-10-22 22:28:31 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_tool-orchestator (ID: 343764b8-7ac9-4549-93cf-8abc11ceba6f) - 6 spans\n", - "2025-10-22 22:28:31 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_tool-orchestator (ID: 343764b8-7ac9-4549-93cf-8abc11ceba6f) - 6 spans\n", - "2025-10-22 22:28:31 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 343764b8-7ac9-4549-93cf-8abc11ceba6f\n", - "2025-10-22 22:28:31 - noveum_trace.transport.http_transport - INFO - āœ… Trace 343764b8-7ac9-4549-93cf-8abc11ceba6f successfully queued for export\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "šŸ” Found 5 relevant documents for query: 'What are the newest features in cloud computing?'\n", - "āœ… Query processed in 5.25s using RAG\n", - "\n", - "================================================================================\n", - "šŸ¤– NOVEUM AI AGENT RESPONSE\n", - "================================================================================\n", - "šŸ“Š Mode: RAG\n", - "ā±ļø Processing Time: 5.25s\n", - "šŸ“… Timestamp: 2025-10-22 22:28:31\n", - "šŸ“š Sources (5):\n", - " 1. https://noveum.ai/en/changelog\n", - " 2. https://noveum.ai/en/changelog\n", - " 3. https://noveum.ai/docs/platform/dashboard\n", - " ... and 2 more\n", - "\n", - "šŸ’¬ Answer:\n", - "----------------------------------------\n", - "The provided context does not specifically mention any new features in cloud computing as a general topic. However, it does highlight several recent enhancements related to the Noveum.ai platform, which may involve cloud computing aspects. These include:\n", - "\n", - "1. **Enhanced Dashboard Analytics**: Real-time request tracking, latency monitoring, and cost analysis (Source 1).\n", - "2. **Improved Logs Interface**: Better search functionality and enhanced debugging capabilities (Source 1).\n", - "3. **Advanced Metrics Collection**: A comprehensive telemetry system with custom metrics (Source 1).\n", - "4. **Major Platform Evolution**: The launch of the NovaEval Framework for AI model evaluation, which includes production-grade deployment on Docker and Kubernetes (Source 4).\n", - "\n", - "If you have specific questions about Noveum.ai or its features, feel free to ask!\n", - "================================================================================\n", - "\n", - "--- Demo Query 19 ---\n", - "\n", - "šŸŽÆ Processing query: 'What's happening in the software development world today?'\n", - "šŸŒ Routing to Web Search for: 'What's happening in the software development world today?'\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/Users/mramanindia/work/NovaEval/.venv/lib/python3.12/site-packages/langchain_community/utilities/duckduckgo_search.py:63: RuntimeWarning: This package (`duckduckgo_search`) has been renamed to `ddgs`! Use `pip install ddgs` instead.\n", - " with DDGS() as ddgs:\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "šŸ” Found 1 web search results for: 'What's happening in the software development world today?'\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-10-22 22:28:41 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_tool-orchestator (ID: ec7c6cae-9bbd-47f4-a8ae-2d6d9b5011ba) - 6 spans\n", - "2025-10-22 22:28:41 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_tool-orchestator (ID: ec7c6cae-9bbd-47f4-a8ae-2d6d9b5011ba) - 6 spans\n", - "2025-10-22 22:28:41 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace ec7c6cae-9bbd-47f4-a8ae-2d6d9b5011ba\n", - "2025-10-22 22:28:41 - noveum_trace.transport.http_transport - INFO - āœ… Trace ec7c6cae-9bbd-47f4-a8ae-2d6d9b5011ba successfully queued for export\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "āœ… Query processed in 8.28s using Web Search\n", - "\n", - "================================================================================\n", - "šŸ¤– NOVEUM AI AGENT RESPONSE\n", - "================================================================================\n", - "šŸ“Š Mode: Web Search\n", - "ā±ļø Processing Time: 8.28s\n", - "šŸ“… Timestamp: 2025-10-22 22:28:41\n", - "šŸ“š Sources (1):\n", - " 1. https://duckduckgo.com/?q=What's+happening+in+the+software+development+world+today?\n", - "\n", - "šŸ’¬ Answer:\n", - "----------------------------------------\n", - "The web search did not yield any specific results regarding current events in the software development world. Therefore, I cannot provide detailed information on the latest trends, technologies, or news in this field.\n", - "\n", - "However, generally speaking, the software development landscape is often influenced by several ongoing trends, such as:\n", - "\n", - "1. **Increased Adoption of AI and Machine Learning**: Many companies are integrating AI into their software solutions to enhance functionality and user experience.\n", - "\n", - "2. **Remote Work and Collaboration Tools**: The shift to remote work has led to a surge in the development and use of collaboration tools and platforms.\n", - "\n", - "3. **DevOps and Continuous Integration/Continuous Deployment (CI/CD)**: There is a growing emphasis on DevOps practices to improve collaboration between development and operations teams, leading to faster and more reliable software delivery.\n", - "\n", - "4. **Focus on Cybersecurity**: With the rise in cyber threats, there is an increasing focus on building secure software and implementing robust security practices throughout the development lifecycle.\n", - "\n", - "5. **Low-Code and No-Code Development**: These platforms are gaining popularity as they allow users to create applications with minimal coding, making software development more accessible.\n", - "\n", - "For the most accurate and up-to-date information, I recommend checking reputable tech news websites or software development blogs.\n", - "================================================================================\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/Users/mramanindia/work/NovaEval/.venv/lib/python3.12/site-packages/langchain_community/utilities/duckduckgo_search.py:63: RuntimeWarning: This package (`duckduckgo_search`) has been renamed to `ddgs`! Use `pip install ddgs` instead.\n", - " with DDGS() as ddgs:\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "--- Demo Query 20 ---\n", - "\n", - "šŸŽÆ Processing query: 'What are the latest breakthroughs in artificial intelligence?'\n", - "šŸŒ Routing to Web Search for: 'What are the latest breakthroughs in artificial intelligence?'\n", - "šŸ” Found 1 web search results for: 'What are the latest breakthroughs in artificial intelligence?'\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-10-22 22:28:44 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_tool-orchestator (ID: 6fe93abf-1413-49e8-8464-58558e25ab7b) - 6 spans\n", - "2025-10-22 22:28:44 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_tool-orchestator (ID: 6fe93abf-1413-49e8-8464-58558e25ab7b) - 6 spans\n", - "2025-10-22 22:28:44 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 6fe93abf-1413-49e8-8464-58558e25ab7b\n", - "2025-10-22 22:28:44 - noveum_trace.transport.http_transport - INFO - āœ… Trace 6fe93abf-1413-49e8-8464-58558e25ab7b successfully queued for export\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "āœ… Query processed in 2.68s using Web Search\n", - "\n", - "================================================================================\n", - "šŸ¤– NOVEUM AI AGENT RESPONSE\n", - "================================================================================\n", - "šŸ“Š Mode: Web Search\n", - "ā±ļø Processing Time: 2.68s\n", - "šŸ“… Timestamp: 2025-10-22 22:28:44\n", - "šŸ“š Sources (1):\n", - " 1. https://duckduckgo.com/?q=What+are+the+latest+breakthroughs+in+artificial+intelligence?\n", - "\n", - "šŸ’¬ Answer:\n", - "----------------------------------------\n", - "The web search did not yield specific results regarding the latest breakthroughs in artificial intelligence. However, I can provide a general overview based on knowledge up to October 2023.\n", - "\n", - "Recent breakthroughs in artificial intelligence include advancements in natural language processing (NLP), particularly with models like GPT-4, which have improved understanding and generation of human-like text. Additionally, there have been significant developments in computer vision, enabling AI systems to better interpret and analyze visual data.\n", - "\n", - "Another notable area of progress is in reinforcement learning, where AI systems are increasingly capable of learning complex tasks through trial and error, leading to applications in robotics and game playing. Furthermore, AI ethics and safety have gained attention, with researchers focusing on creating more transparent and fair AI systems.\n", - "\n", - "For the most current and specific breakthroughs, I recommend checking reputable technology news sources or academic journals that focus on AI research.\n", - "================================================================================\n", - "\n", - "šŸŽ‰ Demo completed! Check Noveum Trace dashboard for detailed observability data.\n", - "šŸ’” You can now use noveum_agent.process_query('your question') for your own queries!\n" - ] - } - ], - "source": [ - "demo_noveum_agent()" - ] - }, - { - "cell_type": "markdown", - "id": "d5b29876", - "metadata": {}, - "source": [ - "## Downloading the data set" - ] - }, - { - "cell_type": "markdown", - "id": "bcf1f2b8", - "metadata": {}, - "source": [ - "## šŸ“‹ Complete End-to-End Workflow Overview\n", - "\n", - "This notebook demonstrates a **complete end-to-end workflow** for AI agent evaluation:\n", - "\n", - "### šŸ”„ Workflow Steps:\n", - "\n", - "1. **šŸ¤– Agent Creation & Demo** - Build and test the Noveum AI agent with RAG + Web Search\n", - "2. **šŸ“Š Trace Collection** - Download traces from Noveum platform \n", - "3. **šŸ”„ Data Processing** - Combine, filter, and map trace data\n", - "4. **šŸ“ˆ Dataset Management** - Create dataset and upload items to Noveum\n", - "5. **šŸ“Š Score Upload** - Upload all evaluation metrics to the platform\n", - "\n", - "### šŸŽÆ What You'll Learn:\n", - "\n", - "- How to build a sophisticated AI agent with dual knowledge sources\n", - "- Complete observability and tracing implementation\n", - "- End-to-end evaluation pipeline from traces to scores\n", - "- Noveum platform integration for dataset and score management\n", - "\n", - "### šŸ“Š Expected Results:\n", - "\n", - "- **50 traces** with **279 spans** collected\n", - "- **12 conversation items** uploaded to dataset\n", - "- **5 evaluation metrics** successfully uploaded\n", - "- Complete observability pipeline operational\n", - "\n", - "Let's start! šŸš€\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "0f918640", - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/Library/Frameworks/Python.framework/Versions/3.12/lib/python3.12/pty.py:95: DeprecationWarning: This process (pid=52016) is multi-threaded, use of forkpty() may lead to deadlocks in the child.\n", - " pid, fd = os.forkpty()\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Fetching 50 traces for project: noveum-ai-agent-rag-websearch\n", - "Cleaning existing traces directory: /Users/mramanindia/work/NovaEval/noveum_customer_support_bt/traces/traces\n", - "Created traces directory: /Users/mramanindia/work/NovaEval/noveum_customer_support_bt/traces/traces\n", - "Will fetch in 1 batch(es) of up to 100 traces each\n", - "\n", - "--- Batch 1/1 ---\n", - "Fetching traces: size=50, from=0\n", - "Successfully fetched 50 traces\n", - "Saved batch 1 to: /Users/mramanindia/work/NovaEval/noveum_customer_support_bt/traces/traces/traces_batch_001.json\n", - "Batch 1 complete: 50 traces\n", - "Total fetched so far: 50/50\n", - "Reached target of 50 traces\n", - "\n", - "=== Summary ===\n", - "Total traces fetched: 50\n", - "Batches created: 1\n", - "Traces directory: /Users/mramanindia/work/NovaEval/noveum_customer_support_bt/traces/traces\n", - "Created files: traces_batch_001.json\n" - ] - } - ], - "source": [ - "!python noveum_customer_support_bt/traces/fetch_traces_api.py 50\n", - "\n", - "#. This script fetches traces for our project and saves them locally." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "641efa41", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Processing traces from: /Users/mramanindia/work/NovaEval/noveum_customer_support_bt/traces/traces\n", - "Found 1 trace files: ['traces_batch_001.json']\n", - "Processing traces_batch_001.json...\n", - "Combined 290 spans total\n", - "Saved combined spans to: /Users/mramanindia/work/NovaEval/noveum_customer_support_bt/traces/traces/dataset.json\n", - "\n", - "Sample of first span keys: ['span_id', 'trace_id', 'parent_span_id', 'name', 'start_time', 'end_time', 'duration_ms', 'status', 'status_message', 'attributes', 'events', 'links', 'trace_trace_id', 'trace_name', 'project', 'environment', 'trace_status', 'trace_status_message', 'trace_start_time', 'trace_end_time', 'trace_duration_ms', 'span_count', 'error_count', 'sdk', 'trace_attributes', 'metadata', 'created_at', 'updated_at']\n", - "Total spans: 290\n", - "\n", - "Span types distribution:\n", - " agent.llm-rag: 26\n", - " agent.llm_model_execution: 48\n", - " agent.query_routing: 48\n", - " agent.rag_evaluation_metrics: 26\n", - " agent.routing_evaluation_metrics: 48\n", - " agent.web_search_evaluation_metrics: 22\n", - " agent.web_search_generation: 22\n", - " system_initialization: 2\n", - " tool-orchestator: 48\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/Library/Frameworks/Python.framework/Versions/3.12/lib/python3.12/pty.py:95: DeprecationWarning: This process (pid=52016) is multi-threaded, use of forkpty() may lead to deadlocks in the child.\n", - " pid, fd = os.forkpty()\n" - ] - } - ], - "source": [ - "!python NovaEval/noveum_customer_support_bt/traces/combine_spans_api_compat.py" - ] - }, - { - "cell_type": "markdown", - "id": "0bddd552", - "metadata": {}, - "source": [ - "## Data Filteration and mapping" - ] - }, - { - "cell_type": "code", - "execution_count": 62, - "id": "3dfb8588", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Reading ./traces/traces/dataset.json...\n", - "Original dataset: 290 records\n", - "Filtering spans...\n", - "After filtering: 290 records\n", - "Converting tool output format...\n", - "Writing ./traces/traces/dataset_filtered.json...\n", - "Filtering complete! Output: ./traces/traces/dataset_filtered.json\n", - "\n", - "Success! Created ./traces/traces/dataset_filtered.json\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/Library/Frameworks/Python.framework/Versions/3.12/lib/python3.12/pty.py:95: DeprecationWarning: This process (pid=52016) is multi-threaded, use of forkpty() may lead to deadlocks in the child.\n", - " pid, fd = os.forkpty()\n" - ] - } - ], - "source": [ - "!python preprocess_filter.py ./traces/traces/dataset.json\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": 63, - "id": "a7484b26", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Error: File ./traces/dataset_filtered.json not found\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/Library/Frameworks/Python.framework/Versions/3.12/lib/python3.12/pty.py:95: DeprecationWarning: This process (pid=52016) is multi-threaded, use of forkpty() may lead to deadlocks in the child.\n", - " pid, fd = os.forkpty()\n" - ] - } - ], - "source": [ - "!python preprocess_map.py ./traces/dataset_filtered.json" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "9bcd53eb", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Reading /Users/mramanindia/work/NovaEval/noveum_customer_support_bt/traces/traces/dataset_filtered.json...\n", - "Input dataset: 290 records\n", - "Mapping spans...\n", - "Writing /Users/mramanindia/work/NovaEval/noveum_customer_support_bt/traces/traces/dataset_filtered_mapped.json...\n", - "Mapping complete! Output: /Users/mramanindia/work/NovaEval/noveum_customer_support_bt/traces/traces/dataset_filtered_mapped.json\n", - "\n", - "Success! Created /Users/mramanindia/work/NovaEval/noveum_customer_support_bt/traces/traces/dataset_filtered_mapped.json\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/Library/Frameworks/Python.framework/Versions/3.12/lib/python3.12/pty.py:95: DeprecationWarning: This process (pid=52016) is multi-threaded, use of forkpty() may lead to deadlocks in the child.\n", - " pid, fd = os.forkpty()\n" - ] - } - ], - "source": [ - "!python preprocess_map.py NovaEval/noveum_customer_support_bt/traces/traces/dataset_filtered.json\n", - "\n" - ] - }, - { - "cell_type": "markdown", - "id": "008fd0e1", - "metadata": {}, - "source": [ - "## Running eval on the dataset" - ] - }, - { - "cell_type": "code", - "execution_count": 72, - "id": "430d60b0", - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/Library/Frameworks/Python.framework/Versions/3.12/lib/python3.12/pty.py:95: DeprecationWarning: This process (pid=52016) is multi-threaded, use of forkpty() may lead to deadlocks in the child.\n", - " pid, fd = os.forkpty()\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "zsh:source:1: no such file or directory: .venv/bin/activate\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/Library/Frameworks/Python.framework/Versions/3.12/lib/python3.12/pty.py:95: DeprecationWarning: This process (pid=52016) is multi-threaded, use of forkpty() may lead to deadlocks in the child.\n", - " pid, fd = os.forkpty()\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "zsh:cd:1: no such file or directory: noveum_customer_support_bt\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/Library/Frameworks/Python.framework/Versions/3.12/lib/python3.12/pty.py:95: DeprecationWarning: This process (pid=52016) is multi-threaded, use of forkpty() may lead to deadlocks in the child.\n", - " pid, fd = os.forkpty()\n", - "/Library/Frameworks/Python.framework/Versions/3.12/lib/python3.12/pty.py:95: DeprecationWarning: This process (pid=52016) is multi-threaded, use of forkpty() may lead to deadlocks in the child.\n", - " pid, fd = os.forkpty()\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "āš ļø WARNING: Please update the dataset slug in your .env file after creating the dataset!\n", - " The API will return a slug that you should set as NOVEUM_DATASET_SLUG in your .env file.\n", - "\n", - "Creating dataset at: https://noveum.ai/api/v1/datasets\n", - "Organization: magic-api\n", - "Dataset name: customersupportagentdemo_new\n", - "Dataset type: agent\n", - "Description: Customer Support Agent Evaluation Dataset\n", - "Visibility: org\n", - "Environment: \n", - "Error creating dataset: 409 Client Error: Conflict for url: https://noveum.ai/api/v1/datasets\n", - "Response status: 409\n", - "Response text: DATASET_SLUG_EXISTS\n" - ] - } - ], - "source": [ - "# 1. Setup\n", - "!cd /Users/mramanindia/work/NovaEval\n", - "!source .venv/bin/activate\n", - "!cd noveum_customer_support_bt\n", - "\n", - "# 2. Create Dataset\n", - "!python create_dataset.py --dataset-type agent --description \"Customer Support Agent Evaluation Dataset\" --pretty\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "id": "9f7feb8d", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Creating dataset version at: https://noveum.ai/api/v1/datasets/customersupportagentdemo-new/versions?organizationSlug=magic-api\n", - "Organization: magic-api\n", - "Dataset: customersupportagentdemo-new\n", - "Version: 0.0.2\n", - "Successfully created dataset version\n", - "Response status: 201\n", - "\n", - "Response saved to: dataset_version_response.json\n", - "\n", - "Response data:\n", - "{\n", - " \"success\": true,\n", - " \"version\": \"0.0.2\"\n", - "}\n" - ] - } - ], - "source": [ - "# 3. Create Version\n", - "!python create_dataset_version.py --pretty\n" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "id": "d36f1da5", - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-10-22 22:59:42 - noveum_trace.transport.batch_processor - INFO - šŸ”„ Batch processor background thread started (batch_size=100, timeout=5.0s)\n", - "2025-10-22 22:59:42 - noveum_trace.transport.batch_processor - INFO - Batch processor started with batch_size=100\n", - "2025-10-22 22:59:42 - noveum_trace.transport.http_transport - INFO - HTTP transport initialized for endpoint: https://api.noveum.ai/api\n", - "2025-10-22 22:59:42 - noveum_trace.core.client - INFO - Noveum Trace client initialized\n", - "2025-10-22 22:59:42,946 - INFO - novaeval.models.base - Noveum tracing initialized successfully\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "āœ… All imports successful!\n", - "āœ… list_dataset_files function defined!\n", - "āœ… load_and_analyze_dataset function defined!\n", - "āœ… parse_tools_from_prompt function defined!\n", - "āœ… parse_params function defined!\n", - "āœ… identify_span_type function defined!\n", - "āœ… map_span_to_agent_data function defined!\n", - "āœ… convert_spans_to_agent_dataset function defined!\n", - "āœ… analyze_dataset_statistics function defined!\n", - "āœ… setup_gemini_model function defined!\n", - "āœ… setup_agent_evaluator function defined!\n", - "āœ… run_evaluation function defined!\n", - "āœ… analyze_agent_behavior_patterns function defined!\n", - "āœ… export_processed_dataset function defined!\n", - "āœ… setup_logging function defined!\n", - "āœ… validate_environment function defined!\n", - "āœ… print_demo_summary function defined!\n", - "āœ… run_complete_agent_evaluation function defined!\n", - "Processing agent.llm_model_execution_dataset.json...\n", - "šŸš€ Starting Complete Agent Evaluation Pipeline\n", - "šŸ“ Processing file: split_datasets/agent.llm_model_execution_dataset.json\n", - "============================================================\n", - "\n", - "šŸ“‹ Step 1: Environment Setup\n", - "āœ… Logging configured at INFO level\n", - "šŸ” Environment validation:\n", - " āœ… gemini_api_key: True\n", - " āœ… pandas_available: True\n", - " āœ… novaeval_available: True\n", - "āœ… Environment ready for evaluation!\n", - "\n", - "šŸ“‹ Step 2: Loading Dataset\n", - "šŸ“Š Loaded 20 spans from split_datasets/agent.llm_model_execution_dataset.json\n", - "\n", - "šŸ” Available span types:\n", - " - agent.llm_model_execution: 20\n", - "āœ… Dataset loaded: 20 spans\n", - "\n", - "šŸ“‹ Step 3: Converting to AgentDataset Format\n", - "šŸ”„ Converting spans to AgentData objects...\n", - "agent_task not found\n", - "agent_response is not available ce8556f4-bbeb-46fc-8bfd-dfaedb61cb70\n", - "agent_task not found\n", - "agent_response is not available cc3073ec-c970-4821-8335-8d0fc8f37930\n", - "agent_task not found\n", - "agent_response is not available c24fae6d-6314-4342-8214-7e9905ae83db\n", - "agent_task not found\n", - "agent_response is not available 93879d8f-77c7-4fd4-881b-be500e2dcbde\n", - "agent_task not found\n", - "agent_response is not available f92643c0-44a3-4fae-9d29-45321342d12f\n", - "agent_task not found\n", - "agent_response is not available 44f379aa-10fe-4a51-847c-5288444d7843\n", - "agent_task not found\n", - "agent_response is not available e8e79f2a-2ed0-4981-943c-adcf6f7d2b15\n", - "agent_task not found\n", - "agent_response is not available e6d0eb41-5f64-44c5-8a42-a80c336de03e\n", - "agent_task not found\n", - "agent_response is not available c545cb62-5a51-4f2e-b020-b3f1d1dfe217\n", - "agent_task not found\n", - "agent_response is not available c2f21e4d-a5e6-4134-aeaf-96e2f566cf96\n", - "agent_task not found\n", - "agent_response is not available 39d99d2b-518f-439c-8758-7a01b970fd9e\n", - "agent_task not found\n", - "agent_response is not available 9e5ade74-3815-4dad-8abe-d4281d7e62da\n", - "agent_task not found\n", - "agent_response is not available 4f38a85b-0721-47a5-8a51-d0808e460f96\n", - "agent_task not found\n", - "agent_response is not available f4c03a8d-b90d-4c1a-a259-826d09e3e57a\n", - "agent_task not found\n", - "agent_response is not available 48fb40ba-a39f-4e13-9d94-18794c7e5e2b\n", - "agent_task not found\n", - "agent_response is not available d4d544a0-f052-4889-82eb-4346f2e447a7\n", - "agent_task not found\n", - "agent_response is not available 304f9e3d-9063-4661-9777-618efc49f5c1\n", - "agent_task not found\n", - "agent_response is not available e0b4ccfb-18e7-4d71-b2dd-5b807eb27513\n", - "agent_task not found\n", - "agent_response is not available f1425131-f8a2-43f7-aa17-d009f745ed06\n", - "agent_task not found\n", - "agent_response is not available f41e836c-1a82-46e0-9d51-011713874e94\n", - "\n", - "āœ… Successfully converted 20 spans to AgentData\n", - "šŸ“Š AgentDataset created with 20 records\n", - "āœ… AgentDataset created: 20 records\n", - "\n", - "šŸ“‹ Step 4: Dataset Analysis\n", - "šŸ“ˆ Dataset Statistics:\n", - "\n", - "Agent Types: {'agent': 20}\n", - "Records with responses: 0\n", - "Records with tool calls: 0\n", - "Records with retrieval: 0\n", - "Tool usage: {}\n", - "šŸ” Dataset Analysis:\n", - "\n", - "=== Agent Behavior Patterns ===\n", - "\n", - "šŸ“ˆ Tool Usage:\n", - "\n", - "šŸ“‹ Task Types:\n", - "\n", - "šŸ“‹ Step 5: Setting up Evaluation\n", - "āœ… GEMINI_API_KEY found in environment\n", - "āœ… Gemini model initialized\n", - "āœ… Initialized 5 scoring functions:\n", - " - task_progression_scorer\n", - " - context_relevancy_scorer\n", - " - role_adherence_scorer\n", - " - tool_relevancy_scorer\n", - " - parameter_correctness_scorer\n", - "\n", - "āœ… AgentEvaluator created with Gemini model and scoring functions\n", - "āœ… Evaluation components ready!\n", - "\n", - "šŸ“‹ Step 6: Running Evaluation\n", - "šŸŽÆ Evaluating 25 samples...\n", - "šŸš€ Running evaluation on sample data...\n", - "\n", - "šŸ“Š Evaluating 0 sample records...\n", - "2025-10-22 22:59:42 - INFO - novaeval.evaluators.agent_evaluator - Starting agent evaluation process\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Evaluating samples: 0it [00:00, ?it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 22:59:43 - INFO - novaeval.evaluators.agent_evaluator - Saving final results\n", - "2025-10-22 22:59:43 - INFO - novaeval.evaluators.agent_evaluator - Agent evaluation completed\n", - "\n", - "āœ… Evaluation completed!\n", - "āŒ Results file not found\n", - "āŒ Evaluation failed\n", - "\n", - "šŸ“‹ Step 7: Exporting Dataset\n", - "šŸ’¾ Exporting processed dataset...\n", - "āœ… Exported to ./processed_datasets/agent.llm_model_execution_dataset_processed_dataset.json\n", - "āœ… Exported to ./processed_datasets/agent.llm_model_execution_dataset_processed_dataset.csv\n", - "āœ… Export completed successfully!\n", - "\n", - "============================================================\n", - "šŸŽ‰ EVALUATION PIPELINE COMPLETED!\n", - "šŸ“Š Final Results:\n", - " - File processed: split_datasets/agent.llm_model_execution_dataset.json\n", - " - Spans loaded: 20\n", - " - Dataset size: 20\n", - " - Evaluation completed: False\n", - " - Export successful: True\n", - " - Errors encountered: 1\n", - "Completed agent.llm_model_execution_dataset.json\n", - "\n", - "Processing agent.query_routing_dataset.json...\n", - "šŸš€ Starting Complete Agent Evaluation Pipeline\n", - "šŸ“ Processing file: split_datasets/agent.query_routing_dataset.json\n", - "============================================================\n", - "\n", - "šŸ“‹ Step 1: Environment Setup\n", - "āœ… Logging configured at INFO level\n", - "šŸ” Environment validation:\n", - " āœ… gemini_api_key: True\n", - " āœ… pandas_available: True\n", - " āœ… novaeval_available: True\n", - "āœ… Environment ready for evaluation!\n", - "\n", - "šŸ“‹ Step 2: Loading Dataset\n", - "šŸ“Š Loaded 20 spans from split_datasets/agent.query_routing_dataset.json\n", - "\n", - "šŸ” Available span types:\n", - " - agent.query_routing: 20\n", - "āœ… Dataset loaded: 20 spans\n", - "\n", - "šŸ“‹ Step 3: Converting to AgentDataset Format\n", - "šŸ”„ Converting spans to AgentData objects...\n", - "\n", - "āœ… Successfully converted 20 spans to AgentData\n", - "šŸ“Š AgentDataset created with 20 records\n", - "āœ… AgentDataset created: 20 records\n", - "\n", - "šŸ“‹ Step 4: Dataset Analysis\n", - "šŸ“ˆ Dataset Statistics:\n", - "\n", - "Agent Types: {'agent': 20}\n", - "Records with responses: 0\n", - "Records with tool calls: 0\n", - "Records with retrieval: 0\n", - "Tool usage: {}\n", - "šŸ” Dataset Analysis:\n", - "\n", - "=== Agent Behavior Patterns ===\n", - "\n", - "šŸ“ˆ Tool Usage:\n", - "\n", - "šŸ“‹ Task Types:\n", - " - other: 20\n", - "\n", - "šŸ“‹ Step 5: Setting up Evaluation\n", - "āœ… GEMINI_API_KEY found in environment\n", - "2025-10-22 22:59:43 - INFO - novaeval.models.base - Noveum tracing initialized successfully\n", - "āœ… Gemini model initialized\n", - "āœ… Initialized 5 scoring functions:\n", - " - task_progression_scorer\n", - " - context_relevancy_scorer\n", - " - role_adherence_scorer\n", - " - tool_relevancy_scorer\n", - " - parameter_correctness_scorer\n", - "\n", - "āœ… AgentEvaluator created with Gemini model and scoring functions\n", - "āœ… Evaluation components ready!\n", - "\n", - "šŸ“‹ Step 6: Running Evaluation\n", - "šŸŽÆ Evaluating 25 samples...\n", - "šŸš€ Running evaluation on sample data...\n", - "\n", - "šŸ“Š Evaluating 20 sample records...\n", - "2025-10-22 22:59:43 - INFO - novaeval.evaluators.agent_evaluator - Starting agent evaluation process\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\n", - "Evaluating samples: 0it [00:00, ?it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 22:59:43 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-10-22 22:59:44 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 62c88c1e-760c-437e-a9d7-84a84e258789) - 1 spans\n", - "2025-10-22 22:59:44 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 62c88c1e-760c-437e-a9d7-84a84e258789) - 1 spans\n", - "2025-10-22 22:59:44 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 62c88c1e-760c-437e-a9d7-84a84e258789\n", - "2025-10-22 22:59:44 - noveum_trace.transport.http_transport - INFO - āœ… Trace 62c88c1e-760c-437e-a9d7-84a84e258789 successfully queued for export\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 22:59:44 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-10-22 22:59:46 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: cf8d91be-c525-4e98-b2ab-b6dead66b872) - 1 spans\n", - "2025-10-22 22:59:46 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: cf8d91be-c525-4e98-b2ab-b6dead66b872) - 1 spans\n", - "2025-10-22 22:59:46 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace cf8d91be-c525-4e98-b2ab-b6dead66b872\n", - "2025-10-22 22:59:46 - noveum_trace.transport.http_transport - INFO - āœ… Trace cf8d91be-c525-4e98-b2ab-b6dead66b872 successfully queued for export\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 22:59:46 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-10-22 22:59:47 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 087e523c-cf91-49e6-8a3f-63d7e6ea817a) - 1 spans\n", - "2025-10-22 22:59:47 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 087e523c-cf91-49e6-8a3f-63d7e6ea817a) - 1 spans\n", - "2025-10-22 22:59:47 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 087e523c-cf91-49e6-8a3f-63d7e6ea817a\n", - "2025-10-22 22:59:47 - noveum_trace.transport.http_transport - INFO - āœ… Trace 087e523c-cf91-49e6-8a3f-63d7e6ea817a successfully queued for export\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 22:59:47 - INFO - novaeval.evaluators.agent_evaluator - Saving intermediate results after 1 samples\n", - "2025-10-22 22:59:47 - INFO - novaeval.evaluators.agent_evaluator - Intermediate results saved to demo_results/agent.query_routing_dataset/agent_evaluation_results.csv\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Evaluating samples: 1it [00:04, 4.56s/it]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 22:59:47 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-10-22 22:59:48 - noveum_trace.transport.batch_processor - INFO - ā° TIMEOUT TRIGGER: Sending batch due to timeout (5.1s >= 5.0s)\n", - "2025-10-22 22:59:48 - noveum_trace.transport.batch_processor - INFO - šŸ“¤ SENDING BATCH: 3 traces via send_callback\n", - "2025-10-22 22:59:48 - noveum_trace.transport.http_transport - INFO - šŸš€ SENDING BATCH: 3 traces to https://api.noveum.ai/api/v1/traces\n", - "2025-10-22 22:59:48 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: f5d0b1e3-779c-4804-8cb6-c37743638b63) - 1 spans\n", - "2025-10-22 22:59:48 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: f5d0b1e3-779c-4804-8cb6-c37743638b63) - 1 spans\n", - "2025-10-22 22:59:48 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace f5d0b1e3-779c-4804-8cb6-c37743638b63\n", - "2025-10-22 22:59:48 - noveum_trace.transport.http_transport - INFO - āœ… Trace f5d0b1e3-779c-4804-8cb6-c37743638b63 successfully queued for export\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 22:59:48 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-10-22 22:59:49 - noveum_trace.transport.http_transport - INFO - šŸ“” HTTP RESPONSE: Status 200 from https://api.noveum.ai/api/v1/traces\n", - "2025-10-22 22:59:49 - noveum_trace.transport.http_transport - INFO - āœ… Successfully sent batch of 3 traces\n", - "2025-10-22 22:59:49 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully sent batch of 3 traces via callback\n", - "2025-10-22 22:59:50 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 9c192a1f-994d-4dba-a03d-79b70c731c6f) - 1 spans\n", - "2025-10-22 22:59:50 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 9c192a1f-994d-4dba-a03d-79b70c731c6f) - 1 spans\n", - "2025-10-22 22:59:50 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 9c192a1f-994d-4dba-a03d-79b70c731c6f\n", - "2025-10-22 22:59:50 - noveum_trace.transport.http_transport - INFO - āœ… Trace 9c192a1f-994d-4dba-a03d-79b70c731c6f successfully queued for export\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 22:59:50 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-10-22 22:59:52 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 4d902ae0-f8b9-4d47-b1d8-cfcc59d71ef9) - 1 spans\n", - "2025-10-22 22:59:52 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 4d902ae0-f8b9-4d47-b1d8-cfcc59d71ef9) - 1 spans\n", - "2025-10-22 22:59:52 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 4d902ae0-f8b9-4d47-b1d8-cfcc59d71ef9\n", - "2025-10-22 22:59:52 - noveum_trace.transport.http_transport - INFO - āœ… Trace 4d902ae0-f8b9-4d47-b1d8-cfcc59d71ef9 successfully queued for export\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 22:59:52 - INFO - novaeval.evaluators.agent_evaluator - Saving intermediate results after 2 samples\n", - "2025-10-22 22:59:52 - INFO - novaeval.evaluators.agent_evaluator - Intermediate results saved to demo_results/agent.query_routing_dataset/agent_evaluation_results.csv\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Evaluating samples: 2it [00:09, 5.04s/it]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 22:59:52 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-10-22 22:59:53 - noveum_trace.transport.batch_processor - INFO - ā° TIMEOUT TRIGGER: Sending batch due to timeout (5.4s >= 5.0s)\n", - "2025-10-22 22:59:53 - noveum_trace.transport.batch_processor - INFO - šŸ“¤ SENDING BATCH: 3 traces via send_callback\n", - "2025-10-22 22:59:53 - noveum_trace.transport.http_transport - INFO - šŸš€ SENDING BATCH: 3 traces to https://api.noveum.ai/api/v1/traces\n", - "2025-10-22 22:59:53 - noveum_trace.transport.http_transport - INFO - šŸ“” HTTP RESPONSE: Status 200 from https://api.noveum.ai/api/v1/traces\n", - "2025-10-22 22:59:53 - noveum_trace.transport.http_transport - INFO - āœ… Successfully sent batch of 3 traces\n", - "2025-10-22 22:59:53 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully sent batch of 3 traces via callback\n", - "2025-10-22 22:59:55 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: d6f4e5b6-7b0c-4cc5-a6ab-0d7f6ee82997) - 1 spans\n", - "2025-10-22 22:59:55 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: d6f4e5b6-7b0c-4cc5-a6ab-0d7f6ee82997) - 1 spans\n", - "2025-10-22 22:59:55 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace d6f4e5b6-7b0c-4cc5-a6ab-0d7f6ee82997\n", - "2025-10-22 22:59:55 - noveum_trace.transport.http_transport - INFO - āœ… Trace d6f4e5b6-7b0c-4cc5-a6ab-0d7f6ee82997 successfully queued for export\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 22:59:55 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-10-22 22:59:58 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 1791b898-a25e-406e-9946-13403e6ea2a5) - 1 spans\n", - "2025-10-22 22:59:58 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 1791b898-a25e-406e-9946-13403e6ea2a5) - 1 spans\n", - "2025-10-22 22:59:58 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 1791b898-a25e-406e-9946-13403e6ea2a5\n", - "2025-10-22 22:59:58 - noveum_trace.transport.http_transport - INFO - āœ… Trace 1791b898-a25e-406e-9946-13403e6ea2a5 successfully queued for export\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 22:59:58 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-10-22 22:59:58 - noveum_trace.transport.batch_processor - INFO - ā° TIMEOUT TRIGGER: Sending batch due to timeout (5.1s >= 5.0s)\n", - "2025-10-22 22:59:58 - noveum_trace.transport.batch_processor - INFO - šŸ“¤ SENDING BATCH: 2 traces via send_callback\n", - "2025-10-22 22:59:58 - noveum_trace.transport.http_transport - INFO - šŸš€ SENDING BATCH: 2 traces to https://api.noveum.ai/api/v1/traces\n", - "2025-10-22 22:59:58 - noveum_trace.transport.http_transport - INFO - šŸ“” HTTP RESPONSE: Status 200 from https://api.noveum.ai/api/v1/traces\n", - "2025-10-22 22:59:58 - noveum_trace.transport.http_transport - INFO - āœ… Successfully sent batch of 2 traces\n", - "2025-10-22 22:59:58 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully sent batch of 2 traces via callback\n", - "2025-10-22 22:59:59 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 93cd53ab-4aee-474b-80a1-b2e056179d42) - 1 spans\n", - "2025-10-22 22:59:59 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 93cd53ab-4aee-474b-80a1-b2e056179d42) - 1 spans\n", - "2025-10-22 22:59:59 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 93cd53ab-4aee-474b-80a1-b2e056179d42\n", - "2025-10-22 22:59:59 - noveum_trace.transport.http_transport - INFO - āœ… Trace 93cd53ab-4aee-474b-80a1-b2e056179d42 successfully queued for export\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 22:59:59 - INFO - novaeval.evaluators.agent_evaluator - Saving intermediate results after 3 samples\n", - "2025-10-22 22:59:59 - INFO - novaeval.evaluators.agent_evaluator - Intermediate results saved to demo_results/agent.query_routing_dataset/agent_evaluation_results.csv\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Evaluating samples: 3it [00:16, 5.53s/it]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 22:59:59 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-10-22 23:00:01 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: bdd95aaf-99e9-4926-ae57-f3846a34dfe2) - 1 spans\n", - "2025-10-22 23:00:01 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: bdd95aaf-99e9-4926-ae57-f3846a34dfe2) - 1 spans\n", - "2025-10-22 23:00:01 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace bdd95aaf-99e9-4926-ae57-f3846a34dfe2\n", - "2025-10-22 23:00:01 - noveum_trace.transport.http_transport - INFO - āœ… Trace bdd95aaf-99e9-4926-ae57-f3846a34dfe2 successfully queued for export\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:00:01 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-10-22 23:00:03 - noveum_trace.transport.batch_processor - INFO - ā° TIMEOUT TRIGGER: Sending batch due to timeout (5.2s >= 5.0s)\n", - "2025-10-22 23:00:03 - noveum_trace.transport.batch_processor - INFO - šŸ“¤ SENDING BATCH: 2 traces via send_callback\n", - "2025-10-22 23:00:03 - noveum_trace.transport.http_transport - INFO - šŸš€ SENDING BATCH: 2 traces to https://api.noveum.ai/api/v1/traces\n", - "2025-10-22 23:00:03 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 14a0cc75-5abe-4057-8b8e-474c095520bd) - 1 spans\n", - "2025-10-22 23:00:03 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 14a0cc75-5abe-4057-8b8e-474c095520bd) - 1 spans\n", - "2025-10-22 23:00:03 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 14a0cc75-5abe-4057-8b8e-474c095520bd\n", - "2025-10-22 23:00:03 - noveum_trace.transport.http_transport - INFO - āœ… Trace 14a0cc75-5abe-4057-8b8e-474c095520bd successfully queued for export\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:00:03 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-10-22 23:00:04 - noveum_trace.transport.http_transport - INFO - šŸ“” HTTP RESPONSE: Status 200 from https://api.noveum.ai/api/v1/traces\n", - "2025-10-22 23:00:04 - noveum_trace.transport.http_transport - INFO - āœ… Successfully sent batch of 2 traces\n", - "2025-10-22 23:00:04 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully sent batch of 2 traces via callback\n", - "2025-10-22 23:00:05 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 3cdcd252-7600-4247-8557-f67f1a66beae) - 1 spans\n", - "2025-10-22 23:00:05 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 3cdcd252-7600-4247-8557-f67f1a66beae) - 1 spans\n", - "2025-10-22 23:00:05 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 3cdcd252-7600-4247-8557-f67f1a66beae\n", - "2025-10-22 23:00:05 - noveum_trace.transport.http_transport - INFO - āœ… Trace 3cdcd252-7600-4247-8557-f67f1a66beae successfully queued for export\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:00:05 - INFO - novaeval.evaluators.agent_evaluator - Saving intermediate results after 4 samples\n", - "2025-10-22 23:00:05 - INFO - novaeval.evaluators.agent_evaluator - Intermediate results saved to demo_results/agent.query_routing_dataset/agent_evaluation_results.csv\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Evaluating samples: 4it [00:21, 5.69s/it]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:00:05 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-10-22 23:00:06 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 57555bab-ecd2-4006-a353-7d27bef7df0b) - 1 spans\n", - "2025-10-22 23:00:06 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 57555bab-ecd2-4006-a353-7d27bef7df0b) - 1 spans\n", - "2025-10-22 23:00:06 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 57555bab-ecd2-4006-a353-7d27bef7df0b\n", - "2025-10-22 23:00:06 - noveum_trace.transport.http_transport - INFO - āœ… Trace 57555bab-ecd2-4006-a353-7d27bef7df0b successfully queued for export\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:00:06 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-10-22 23:00:07 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 6b673f69-57ca-406b-8795-a74e96080081) - 1 spans\n", - "2025-10-22 23:00:07 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 6b673f69-57ca-406b-8795-a74e96080081) - 1 spans\n", - "2025-10-22 23:00:07 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 6b673f69-57ca-406b-8795-a74e96080081\n", - "2025-10-22 23:00:07 - noveum_trace.transport.http_transport - INFO - āœ… Trace 6b673f69-57ca-406b-8795-a74e96080081 successfully queued for export\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:00:07 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-10-22 23:00:08 - noveum_trace.transport.batch_processor - INFO - ā° TIMEOUT TRIGGER: Sending batch due to timeout (5.0s >= 5.0s)\n", - "2025-10-22 23:00:08 - noveum_trace.transport.batch_processor - INFO - šŸ“¤ SENDING BATCH: 4 traces via send_callback\n", - "2025-10-22 23:00:08 - noveum_trace.transport.http_transport - INFO - šŸš€ SENDING BATCH: 4 traces to https://api.noveum.ai/api/v1/traces\n", - "2025-10-22 23:00:08 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 47efa03a-15f4-4818-b062-db186df9847c) - 1 spans\n", - "2025-10-22 23:00:08 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 47efa03a-15f4-4818-b062-db186df9847c) - 1 spans\n", - "2025-10-22 23:00:08 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 47efa03a-15f4-4818-b062-db186df9847c\n", - "2025-10-22 23:00:08 - noveum_trace.transport.http_transport - INFO - āœ… Trace 47efa03a-15f4-4818-b062-db186df9847c successfully queued for export\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:00:08 - INFO - novaeval.evaluators.agent_evaluator - Saving intermediate results after 5 samples\n", - "2025-10-22 23:00:08 - INFO - novaeval.evaluators.agent_evaluator - Intermediate results saved to demo_results/agent.query_routing_dataset/agent_evaluation_results.csv\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Evaluating samples: 5it [00:25, 5.03s/it]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:00:08 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-10-22 23:00:09 - noveum_trace.transport.http_transport - INFO - šŸ“” HTTP RESPONSE: Status 200 from https://api.noveum.ai/api/v1/traces\n", - "2025-10-22 23:00:09 - noveum_trace.transport.http_transport - INFO - āœ… Successfully sent batch of 4 traces\n", - "2025-10-22 23:00:09 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully sent batch of 4 traces via callback\n", - "2025-10-22 23:00:10 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: a1e9ac36-8c30-4784-9f33-93ed2497ee97) - 1 spans\n", - "2025-10-22 23:00:10 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: a1e9ac36-8c30-4784-9f33-93ed2497ee97) - 1 spans\n", - "2025-10-22 23:00:10 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace a1e9ac36-8c30-4784-9f33-93ed2497ee97\n", - "2025-10-22 23:00:10 - noveum_trace.transport.http_transport - INFO - āœ… Trace a1e9ac36-8c30-4784-9f33-93ed2497ee97 successfully queued for export\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:00:10 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-10-22 23:00:11 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 62a690ab-2806-4619-92df-eec54d03b859) - 1 spans\n", - "2025-10-22 23:00:11 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 62a690ab-2806-4619-92df-eec54d03b859) - 1 spans\n", - "2025-10-22 23:00:11 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 62a690ab-2806-4619-92df-eec54d03b859\n", - "2025-10-22 23:00:11 - noveum_trace.transport.http_transport - INFO - āœ… Trace 62a690ab-2806-4619-92df-eec54d03b859 successfully queued for export\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:00:11 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-10-22 23:00:12 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 239c63de-f804-41f5-98af-463f42dd4089) - 1 spans\n", - "2025-10-22 23:00:12 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 239c63de-f804-41f5-98af-463f42dd4089) - 1 spans\n", - "2025-10-22 23:00:12 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 239c63de-f804-41f5-98af-463f42dd4089\n", - "2025-10-22 23:00:12 - noveum_trace.transport.http_transport - INFO - āœ… Trace 239c63de-f804-41f5-98af-463f42dd4089 successfully queued for export\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:00:12 - INFO - novaeval.evaluators.agent_evaluator - Saving intermediate results after 6 samples\n", - "2025-10-22 23:00:12 - INFO - novaeval.evaluators.agent_evaluator - Intermediate results saved to demo_results/agent.query_routing_dataset/agent_evaluation_results.csv\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Evaluating samples: 6it [00:29, 4.54s/it]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:00:12 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-10-22 23:00:13 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 4b6d4dfb-7c55-4296-9ad6-0490385e162d) - 1 spans\n", - "2025-10-22 23:00:13 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 4b6d4dfb-7c55-4296-9ad6-0490385e162d) - 1 spans\n", - "2025-10-22 23:00:13 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 4b6d4dfb-7c55-4296-9ad6-0490385e162d\n", - "2025-10-22 23:00:13 - noveum_trace.transport.http_transport - INFO - āœ… Trace 4b6d4dfb-7c55-4296-9ad6-0490385e162d successfully queued for export\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:00:13 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-10-22 23:00:14 - noveum_trace.transport.batch_processor - INFO - ā° TIMEOUT TRIGGER: Sending batch due to timeout (5.4s >= 5.0s)\n", - "2025-10-22 23:00:14 - noveum_trace.transport.batch_processor - INFO - šŸ“¤ SENDING BATCH: 5 traces via send_callback\n", - "2025-10-22 23:00:14 - noveum_trace.transport.http_transport - INFO - šŸš€ SENDING BATCH: 5 traces to https://api.noveum.ai/api/v1/traces\n", - "2025-10-22 23:00:14 - noveum_trace.transport.http_transport - INFO - šŸ“” HTTP RESPONSE: Status 200 from https://api.noveum.ai/api/v1/traces\n", - "2025-10-22 23:00:14 - noveum_trace.transport.http_transport - INFO - āœ… Successfully sent batch of 5 traces\n", - "2025-10-22 23:00:14 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully sent batch of 5 traces via callback\n", - "2025-10-22 23:00:14 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: b6ec3f3c-f9ec-4714-b99e-034d4fa7655b) - 1 spans\n", - "2025-10-22 23:00:14 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: b6ec3f3c-f9ec-4714-b99e-034d4fa7655b) - 1 spans\n", - "2025-10-22 23:00:14 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace b6ec3f3c-f9ec-4714-b99e-034d4fa7655b\n", - "2025-10-22 23:00:14 - noveum_trace.transport.http_transport - INFO - āœ… Trace b6ec3f3c-f9ec-4714-b99e-034d4fa7655b successfully queued for export\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:00:14 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-10-22 23:00:16 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: ed49c38e-e2d4-4cb2-8049-ec501c7407d7) - 1 spans\n", - "2025-10-22 23:00:16 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: ed49c38e-e2d4-4cb2-8049-ec501c7407d7) - 1 spans\n", - "2025-10-22 23:00:16 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace ed49c38e-e2d4-4cb2-8049-ec501c7407d7\n", - "2025-10-22 23:00:16 - noveum_trace.transport.http_transport - INFO - āœ… Trace ed49c38e-e2d4-4cb2-8049-ec501c7407d7 successfully queued for export\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:00:16 - INFO - novaeval.evaluators.agent_evaluator - Saving intermediate results after 7 samples\n", - "2025-10-22 23:00:16 - INFO - novaeval.evaluators.agent_evaluator - Intermediate results saved to demo_results/agent.query_routing_dataset/agent_evaluation_results.csv\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Evaluating samples: 7it [00:33, 4.32s/it]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:00:16 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-10-22 23:00:17 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 1df4cc5a-2027-401e-8a59-c0bc394f0463) - 1 spans\n", - "2025-10-22 23:00:17 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 1df4cc5a-2027-401e-8a59-c0bc394f0463) - 1 spans\n", - "2025-10-22 23:00:17 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 1df4cc5a-2027-401e-8a59-c0bc394f0463\n", - "2025-10-22 23:00:17 - noveum_trace.transport.http_transport - INFO - āœ… Trace 1df4cc5a-2027-401e-8a59-c0bc394f0463 successfully queued for export\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:00:17 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-10-22 23:00:18 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 75755b07-45c9-4ad8-a59e-f6b04685a2be) - 1 spans\n", - "2025-10-22 23:00:18 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 75755b07-45c9-4ad8-a59e-f6b04685a2be) - 1 spans\n", - "2025-10-22 23:00:18 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 75755b07-45c9-4ad8-a59e-f6b04685a2be\n", - "2025-10-22 23:00:18 - noveum_trace.transport.http_transport - INFO - āœ… Trace 75755b07-45c9-4ad8-a59e-f6b04685a2be successfully queued for export\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:00:18 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-10-22 23:00:19 - noveum_trace.transport.batch_processor - INFO - ā° TIMEOUT TRIGGER: Sending batch due to timeout (5.3s >= 5.0s)\n", - "2025-10-22 23:00:19 - noveum_trace.transport.batch_processor - INFO - šŸ“¤ SENDING BATCH: 4 traces via send_callback\n", - "2025-10-22 23:00:19 - noveum_trace.transport.http_transport - INFO - šŸš€ SENDING BATCH: 4 traces to https://api.noveum.ai/api/v1/traces\n", - "2025-10-22 23:00:19 - noveum_trace.transport.http_transport - INFO - šŸ“” HTTP RESPONSE: Status 200 from https://api.noveum.ai/api/v1/traces\n", - "2025-10-22 23:00:19 - noveum_trace.transport.http_transport - INFO - āœ… Successfully sent batch of 4 traces\n", - "2025-10-22 23:00:19 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully sent batch of 4 traces via callback\n", - "2025-10-22 23:00:20 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: dea54b30-9b78-45a1-861c-acef72c97a3f) - 1 spans\n", - "2025-10-22 23:00:20 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: dea54b30-9b78-45a1-861c-acef72c97a3f) - 1 spans\n", - "2025-10-22 23:00:20 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace dea54b30-9b78-45a1-861c-acef72c97a3f\n", - "2025-10-22 23:00:20 - noveum_trace.transport.http_transport - INFO - āœ… Trace dea54b30-9b78-45a1-861c-acef72c97a3f successfully queued for export\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:00:20 - INFO - novaeval.evaluators.agent_evaluator - Saving intermediate results after 8 samples\n", - "2025-10-22 23:00:20 - INFO - novaeval.evaluators.agent_evaluator - Intermediate results saved to demo_results/agent.query_routing_dataset/agent_evaluation_results.csv\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Evaluating samples: 8it [00:37, 4.16s/it]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:00:20 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-10-22 23:00:21 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 9cb16557-f687-4b6d-aa25-798f27e9615c) - 1 spans\n", - "2025-10-22 23:00:21 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 9cb16557-f687-4b6d-aa25-798f27e9615c) - 1 spans\n", - "2025-10-22 23:00:21 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 9cb16557-f687-4b6d-aa25-798f27e9615c\n", - "2025-10-22 23:00:21 - noveum_trace.transport.http_transport - INFO - āœ… Trace 9cb16557-f687-4b6d-aa25-798f27e9615c successfully queued for export\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:00:21 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-10-22 23:00:22 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 39999cf9-16cd-4b1f-8167-39953b328da6) - 1 spans\n", - "2025-10-22 23:00:22 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 39999cf9-16cd-4b1f-8167-39953b328da6) - 1 spans\n", - "2025-10-22 23:00:22 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 39999cf9-16cd-4b1f-8167-39953b328da6\n", - "2025-10-22 23:00:22 - noveum_trace.transport.http_transport - INFO - āœ… Trace 39999cf9-16cd-4b1f-8167-39953b328da6 successfully queued for export\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:00:22 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-10-22 23:00:24 - noveum_trace.transport.batch_processor - INFO - ā° TIMEOUT TRIGGER: Sending batch due to timeout (5.1s >= 5.0s)\n", - "2025-10-22 23:00:24 - noveum_trace.transport.batch_processor - INFO - šŸ“¤ SENDING BATCH: 3 traces via send_callback\n", - "2025-10-22 23:00:24 - noveum_trace.transport.http_transport - INFO - šŸš€ SENDING BATCH: 3 traces to https://api.noveum.ai/api/v1/traces\n", - "2025-10-22 23:00:24 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 0829f590-d837-4dee-9e65-feee419011b2) - 1 spans\n", - "2025-10-22 23:00:24 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 0829f590-d837-4dee-9e65-feee419011b2) - 1 spans\n", - "2025-10-22 23:00:24 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 0829f590-d837-4dee-9e65-feee419011b2\n", - "2025-10-22 23:00:24 - noveum_trace.transport.http_transport - INFO - āœ… Trace 0829f590-d837-4dee-9e65-feee419011b2 successfully queued for export\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:00:24 - INFO - novaeval.evaluators.agent_evaluator - Saving intermediate results after 9 samples\n", - "2025-10-22 23:00:24 - INFO - novaeval.evaluators.agent_evaluator - Intermediate results saved to demo_results/agent.query_routing_dataset/agent_evaluation_results.csv\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Evaluating samples: 9it [00:41, 4.21s/it]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:00:24 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-10-22 23:00:24 - noveum_trace.transport.http_transport - INFO - šŸ“” HTTP RESPONSE: Status 200 from https://api.noveum.ai/api/v1/traces\n", - "2025-10-22 23:00:24 - noveum_trace.transport.http_transport - INFO - āœ… Successfully sent batch of 3 traces\n", - "2025-10-22 23:00:24 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully sent batch of 3 traces via callback\n", - "2025-10-22 23:00:26 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 4e2bb2ea-fcec-476a-8505-3f15ad8243ca) - 1 spans\n", - "2025-10-22 23:00:26 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 4e2bb2ea-fcec-476a-8505-3f15ad8243ca) - 1 spans\n", - "2025-10-22 23:00:26 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 4e2bb2ea-fcec-476a-8505-3f15ad8243ca\n", - "2025-10-22 23:00:26 - noveum_trace.transport.http_transport - INFO - āœ… Trace 4e2bb2ea-fcec-476a-8505-3f15ad8243ca successfully queued for export\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:00:26 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-10-22 23:00:28 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: a091f50f-c6f6-4e99-a812-c427266b63f3) - 1 spans\n", - "2025-10-22 23:00:28 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: a091f50f-c6f6-4e99-a812-c427266b63f3) - 1 spans\n", - "2025-10-22 23:00:28 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace a091f50f-c6f6-4e99-a812-c427266b63f3\n", - "2025-10-22 23:00:28 - noveum_trace.transport.http_transport - INFO - āœ… Trace a091f50f-c6f6-4e99-a812-c427266b63f3 successfully queued for export\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:00:28 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-10-22 23:00:29 - noveum_trace.transport.batch_processor - INFO - ā° TIMEOUT TRIGGER: Sending batch due to timeout (5.4s >= 5.0s)\n", - "2025-10-22 23:00:29 - noveum_trace.transport.batch_processor - INFO - šŸ“¤ SENDING BATCH: 3 traces via send_callback\n", - "2025-10-22 23:00:29 - noveum_trace.transport.http_transport - INFO - šŸš€ SENDING BATCH: 3 traces to https://api.noveum.ai/api/v1/traces\n", - "2025-10-22 23:00:30 - noveum_trace.transport.http_transport - INFO - šŸ“” HTTP RESPONSE: Status 200 from https://api.noveum.ai/api/v1/traces\n", - "2025-10-22 23:00:30 - noveum_trace.transport.http_transport - INFO - āœ… Successfully sent batch of 3 traces\n", - "2025-10-22 23:00:30 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully sent batch of 3 traces via callback\n", - "2025-10-22 23:00:30 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 236d1ec0-daf8-46f7-be50-ea50e4a5e1c6) - 1 spans\n", - "2025-10-22 23:00:30 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 236d1ec0-daf8-46f7-be50-ea50e4a5e1c6) - 1 spans\n", - "2025-10-22 23:00:30 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 236d1ec0-daf8-46f7-be50-ea50e4a5e1c6\n", - "2025-10-22 23:00:30 - noveum_trace.transport.http_transport - INFO - āœ… Trace 236d1ec0-daf8-46f7-be50-ea50e4a5e1c6 successfully queued for export\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:00:30 - INFO - novaeval.evaluators.agent_evaluator - Saving intermediate results after 10 samples\n", - "2025-10-22 23:00:30 - INFO - novaeval.evaluators.agent_evaluator - Intermediate results saved to demo_results/agent.query_routing_dataset/agent_evaluation_results.csv\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Evaluating samples: 10it [00:47, 4.74s/it]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:00:30 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-10-22 23:00:31 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 609077f6-7021-4b16-83e5-d3ed68573359) - 1 spans\n", - "2025-10-22 23:00:31 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 609077f6-7021-4b16-83e5-d3ed68573359) - 1 spans\n", - "2025-10-22 23:00:31 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 609077f6-7021-4b16-83e5-d3ed68573359\n", - "2025-10-22 23:00:31 - noveum_trace.transport.http_transport - INFO - āœ… Trace 609077f6-7021-4b16-83e5-d3ed68573359 successfully queued for export\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:00:31 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-10-22 23:00:32 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: c2bcfc15-818d-4d23-91ef-c38392c4ce9a) - 1 spans\n", - "2025-10-22 23:00:32 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: c2bcfc15-818d-4d23-91ef-c38392c4ce9a) - 1 spans\n", - "2025-10-22 23:00:32 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace c2bcfc15-818d-4d23-91ef-c38392c4ce9a\n", - "2025-10-22 23:00:32 - noveum_trace.transport.http_transport - INFO - āœ… Trace c2bcfc15-818d-4d23-91ef-c38392c4ce9a successfully queued for export\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:00:32 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-10-22 23:00:34 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 027b030d-d0d7-49f7-a9e8-8933f7829573) - 1 spans\n", - "2025-10-22 23:00:34 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 027b030d-d0d7-49f7-a9e8-8933f7829573) - 1 spans\n", - "2025-10-22 23:00:34 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 027b030d-d0d7-49f7-a9e8-8933f7829573\n", - "2025-10-22 23:00:34 - noveum_trace.transport.http_transport - INFO - āœ… Trace 027b030d-d0d7-49f7-a9e8-8933f7829573 successfully queued for export\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:00:34 - INFO - novaeval.evaluators.agent_evaluator - Saving intermediate results after 11 samples\n", - "2025-10-22 23:00:34 - INFO - novaeval.evaluators.agent_evaluator - Intermediate results saved to demo_results/agent.query_routing_dataset/agent_evaluation_results.csv\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Evaluating samples: 11it [00:51, 4.41s/it]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:00:34 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-10-22 23:00:35 - noveum_trace.transport.batch_processor - INFO - ā° TIMEOUT TRIGGER: Sending batch due to timeout (5.2s >= 5.0s)\n", - "2025-10-22 23:00:35 - noveum_trace.transport.batch_processor - INFO - šŸ“¤ SENDING BATCH: 4 traces via send_callback\n", - "2025-10-22 23:00:35 - noveum_trace.transport.http_transport - INFO - šŸš€ SENDING BATCH: 4 traces to https://api.noveum.ai/api/v1/traces\n", - "2025-10-22 23:00:35 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 3b7a2d72-a669-4745-939e-e3eba5fb166b) - 1 spans\n", - "2025-10-22 23:00:35 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 3b7a2d72-a669-4745-939e-e3eba5fb166b) - 1 spans\n", - "2025-10-22 23:00:35 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 3b7a2d72-a669-4745-939e-e3eba5fb166b\n", - "2025-10-22 23:00:35 - noveum_trace.transport.http_transport - INFO - āœ… Trace 3b7a2d72-a669-4745-939e-e3eba5fb166b successfully queued for export\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:00:35 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-10-22 23:00:36 - noveum_trace.transport.http_transport - INFO - šŸ“” HTTP RESPONSE: Status 200 from https://api.noveum.ai/api/v1/traces\n", - "2025-10-22 23:00:36 - noveum_trace.transport.http_transport - INFO - āœ… Successfully sent batch of 4 traces\n", - "2025-10-22 23:00:36 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully sent batch of 4 traces via callback\n", - "2025-10-22 23:00:36 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 3155450e-76b4-4a55-b15b-5c369a6cea5c) - 1 spans\n", - "2025-10-22 23:00:36 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 3155450e-76b4-4a55-b15b-5c369a6cea5c) - 1 spans\n", - "2025-10-22 23:00:36 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 3155450e-76b4-4a55-b15b-5c369a6cea5c\n", - "2025-10-22 23:00:36 - noveum_trace.transport.http_transport - INFO - āœ… Trace 3155450e-76b4-4a55-b15b-5c369a6cea5c successfully queued for export\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:00:36 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-10-22 23:00:38 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 21337429-011a-4aba-a4e5-83ac5be321ce) - 1 spans\n", - "2025-10-22 23:00:38 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 21337429-011a-4aba-a4e5-83ac5be321ce) - 1 spans\n", - "2025-10-22 23:00:38 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 21337429-011a-4aba-a4e5-83ac5be321ce\n", - "2025-10-22 23:00:38 - noveum_trace.transport.http_transport - INFO - āœ… Trace 21337429-011a-4aba-a4e5-83ac5be321ce successfully queued for export\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:00:38 - INFO - novaeval.evaluators.agent_evaluator - Saving intermediate results after 12 samples\n", - "2025-10-22 23:00:38 - INFO - novaeval.evaluators.agent_evaluator - Intermediate results saved to demo_results/agent.query_routing_dataset/agent_evaluation_results.csv\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Evaluating samples: 12it [00:55, 4.40s/it]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:00:38 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-10-22 23:00:40 - noveum_trace.transport.batch_processor - INFO - ā° TIMEOUT TRIGGER: Sending batch due to timeout (5.4s >= 5.0s)\n", - "2025-10-22 23:00:40 - noveum_trace.transport.batch_processor - INFO - šŸ“¤ SENDING BATCH: 3 traces via send_callback\n", - "2025-10-22 23:00:40 - noveum_trace.transport.http_transport - INFO - šŸš€ SENDING BATCH: 3 traces to https://api.noveum.ai/api/v1/traces\n", - "2025-10-22 23:00:40 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 4ebfd611-8645-4781-a3fa-adc5413e7ef9) - 1 spans\n", - "2025-10-22 23:00:40 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 4ebfd611-8645-4781-a3fa-adc5413e7ef9) - 1 spans\n", - "2025-10-22 23:00:40 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 4ebfd611-8645-4781-a3fa-adc5413e7ef9\n", - "2025-10-22 23:00:40 - noveum_trace.transport.http_transport - INFO - āœ… Trace 4ebfd611-8645-4781-a3fa-adc5413e7ef9 successfully queued for export\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:00:40 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-10-22 23:00:41 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 622d34b2-775e-4533-9b86-f3a269f7a87d) - 1 spans\n", - "2025-10-22 23:00:41 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 622d34b2-775e-4533-9b86-f3a269f7a87d) - 1 spans\n", - "2025-10-22 23:00:41 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 622d34b2-775e-4533-9b86-f3a269f7a87d\n", - "2025-10-22 23:00:41 - noveum_trace.transport.http_transport - INFO - āœ… Trace 622d34b2-775e-4533-9b86-f3a269f7a87d successfully queued for export\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:00:41 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-10-22 23:00:43 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 094c79f8-e0c9-4ce7-b493-3ead460412da) - 1 spans\n", - "2025-10-22 23:00:43 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 094c79f8-e0c9-4ce7-b493-3ead460412da) - 1 spans\n", - "2025-10-22 23:00:43 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 094c79f8-e0c9-4ce7-b493-3ead460412da\n", - "2025-10-22 23:00:43 - noveum_trace.transport.http_transport - INFO - āœ… Trace 094c79f8-e0c9-4ce7-b493-3ead460412da successfully queued for export\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:00:43 - INFO - novaeval.evaluators.agent_evaluator - Saving intermediate results after 13 samples\n", - "2025-10-22 23:00:43 - INFO - novaeval.evaluators.agent_evaluator - Intermediate results saved to demo_results/agent.query_routing_dataset/agent_evaluation_results.csv\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Evaluating samples: 13it [01:00, 4.59s/it]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:00:43 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-10-22 23:00:43 - noveum_trace.transport.http_transport - INFO - šŸ“” HTTP RESPONSE: Status 200 from https://api.noveum.ai/api/v1/traces\n", - "2025-10-22 23:00:43 - noveum_trace.transport.http_transport - INFO - āœ… Successfully sent batch of 3 traces\n", - "2025-10-22 23:00:43 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully sent batch of 3 traces via callback\n", - "2025-10-22 23:00:45 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: ba787a5a-c805-4920-bcb6-54f73a11e9be) - 1 spans\n", - "2025-10-22 23:00:45 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: ba787a5a-c805-4920-bcb6-54f73a11e9be) - 1 spans\n", - "2025-10-22 23:00:45 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace ba787a5a-c805-4920-bcb6-54f73a11e9be\n", - "2025-10-22 23:00:45 - noveum_trace.transport.http_transport - INFO - āœ… Trace ba787a5a-c805-4920-bcb6-54f73a11e9be successfully queued for export\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:00:45 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-10-22 23:00:45 - noveum_trace.transport.batch_processor - INFO - ā° TIMEOUT TRIGGER: Sending batch due to timeout (5.5s >= 5.0s)\n", - "2025-10-22 23:00:45 - noveum_trace.transport.batch_processor - INFO - šŸ“¤ SENDING BATCH: 4 traces via send_callback\n", - "2025-10-22 23:00:45 - noveum_trace.transport.http_transport - INFO - šŸš€ SENDING BATCH: 4 traces to https://api.noveum.ai/api/v1/traces\n", - "2025-10-22 23:00:46 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: f7978f02-3999-4b3e-adc7-86d57c952fff) - 1 spans\n", - "2025-10-22 23:00:46 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: f7978f02-3999-4b3e-adc7-86d57c952fff) - 1 spans\n", - "2025-10-22 23:00:46 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace f7978f02-3999-4b3e-adc7-86d57c952fff\n", - "2025-10-22 23:00:46 - noveum_trace.transport.http_transport - INFO - āœ… Trace f7978f02-3999-4b3e-adc7-86d57c952fff successfully queued for export\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:00:46 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-10-22 23:00:47 - noveum_trace.transport.http_transport - INFO - šŸ“” HTTP RESPONSE: Status 200 from https://api.noveum.ai/api/v1/traces\n", - "2025-10-22 23:00:47 - noveum_trace.transport.http_transport - INFO - āœ… Successfully sent batch of 4 traces\n", - "2025-10-22 23:00:47 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully sent batch of 4 traces via callback\n", - "2025-10-22 23:00:47 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: f7c102f2-ae59-4acf-8a11-c8763070bc2d) - 1 spans\n", - "2025-10-22 23:00:47 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: f7c102f2-ae59-4acf-8a11-c8763070bc2d) - 1 spans\n", - "2025-10-22 23:00:47 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace f7c102f2-ae59-4acf-8a11-c8763070bc2d\n", - "2025-10-22 23:00:47 - noveum_trace.transport.http_transport - INFO - āœ… Trace f7c102f2-ae59-4acf-8a11-c8763070bc2d successfully queued for export\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:00:47 - INFO - novaeval.evaluators.agent_evaluator - Saving intermediate results after 14 samples\n", - "2025-10-22 23:00:47 - INFO - novaeval.evaluators.agent_evaluator - Intermediate results saved to demo_results/agent.query_routing_dataset/agent_evaluation_results.csv\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Evaluating samples: 14it [01:04, 4.50s/it]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:00:47 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-10-22 23:00:48 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 499bdbdf-4ddd-4197-a8e4-8fa056cf2c81) - 1 spans\n", - "2025-10-22 23:00:48 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 499bdbdf-4ddd-4197-a8e4-8fa056cf2c81) - 1 spans\n", - "2025-10-22 23:00:48 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 499bdbdf-4ddd-4197-a8e4-8fa056cf2c81\n", - "2025-10-22 23:00:48 - noveum_trace.transport.http_transport - INFO - āœ… Trace 499bdbdf-4ddd-4197-a8e4-8fa056cf2c81 successfully queued for export\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:00:48 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-10-22 23:00:50 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 9941d150-0874-4f64-8348-a5e8a777128e) - 1 spans\n", - "2025-10-22 23:00:50 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 9941d150-0874-4f64-8348-a5e8a777128e) - 1 spans\n", - "2025-10-22 23:00:50 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 9941d150-0874-4f64-8348-a5e8a777128e\n", - "2025-10-22 23:00:50 - noveum_trace.transport.http_transport - INFO - āœ… Trace 9941d150-0874-4f64-8348-a5e8a777128e successfully queued for export\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:00:50 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-10-22 23:00:51 - noveum_trace.transport.batch_processor - INFO - ā° TIMEOUT TRIGGER: Sending batch due to timeout (5.2s >= 5.0s)\n", - "2025-10-22 23:00:51 - noveum_trace.transport.batch_processor - INFO - šŸ“¤ SENDING BATCH: 4 traces via send_callback\n", - "2025-10-22 23:00:51 - noveum_trace.transport.http_transport - INFO - šŸš€ SENDING BATCH: 4 traces to https://api.noveum.ai/api/v1/traces\n", - "2025-10-22 23:00:51 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: b4820be8-ae51-49bc-97d5-6aaf3f26ccbf) - 1 spans\n", - "2025-10-22 23:00:51 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: b4820be8-ae51-49bc-97d5-6aaf3f26ccbf) - 1 spans\n", - "2025-10-22 23:00:51 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace b4820be8-ae51-49bc-97d5-6aaf3f26ccbf\n", - "2025-10-22 23:00:51 - noveum_trace.transport.http_transport - INFO - āœ… Trace b4820be8-ae51-49bc-97d5-6aaf3f26ccbf successfully queued for export\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:00:51 - INFO - novaeval.evaluators.agent_evaluator - Saving intermediate results after 15 samples\n", - "2025-10-22 23:00:51 - INFO - novaeval.evaluators.agent_evaluator - Intermediate results saved to demo_results/agent.query_routing_dataset/agent_evaluation_results.csv\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Evaluating samples: 15it [01:08, 4.18s/it]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:00:51 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-10-22 23:00:52 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 58a8930f-83c2-4717-9f71-5a34660d8fa2) - 1 spans\n", - "2025-10-22 23:00:52 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 58a8930f-83c2-4717-9f71-5a34660d8fa2) - 1 spans\n", - "2025-10-22 23:00:52 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 58a8930f-83c2-4717-9f71-5a34660d8fa2\n", - "2025-10-22 23:00:52 - noveum_trace.transport.http_transport - INFO - āœ… Trace 58a8930f-83c2-4717-9f71-5a34660d8fa2 successfully queued for export\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:00:52 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-10-22 23:00:52 - noveum_trace.transport.http_transport - INFO - šŸ“” HTTP RESPONSE: Status 200 from https://api.noveum.ai/api/v1/traces\n", - "2025-10-22 23:00:52 - noveum_trace.transport.http_transport - INFO - āœ… Successfully sent batch of 4 traces\n", - "2025-10-22 23:00:52 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully sent batch of 4 traces via callback\n", - "2025-10-22 23:00:53 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 9572f06a-e0ba-4e67-81eb-eaad1774cc15) - 1 spans\n", - "2025-10-22 23:00:53 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 9572f06a-e0ba-4e67-81eb-eaad1774cc15) - 1 spans\n", - "2025-10-22 23:00:53 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 9572f06a-e0ba-4e67-81eb-eaad1774cc15\n", - "2025-10-22 23:00:53 - noveum_trace.transport.http_transport - INFO - āœ… Trace 9572f06a-e0ba-4e67-81eb-eaad1774cc15 successfully queued for export\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:00:53 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-10-22 23:00:54 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 11aa6f5b-1e71-440f-b4d7-cfa0b1a3ae56) - 1 spans\n", - "2025-10-22 23:00:54 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 11aa6f5b-1e71-440f-b4d7-cfa0b1a3ae56) - 1 spans\n", - "2025-10-22 23:00:54 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 11aa6f5b-1e71-440f-b4d7-cfa0b1a3ae56\n", - "2025-10-22 23:00:54 - noveum_trace.transport.http_transport - INFO - āœ… Trace 11aa6f5b-1e71-440f-b4d7-cfa0b1a3ae56 successfully queued for export\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:00:54 - INFO - novaeval.evaluators.agent_evaluator - Saving intermediate results after 16 samples\n", - "2025-10-22 23:00:54 - INFO - novaeval.evaluators.agent_evaluator - Intermediate results saved to demo_results/agent.query_routing_dataset/agent_evaluation_results.csv\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Evaluating samples: 16it [01:11, 3.92s/it]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:00:54 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-10-22 23:00:56 - noveum_trace.transport.batch_processor - INFO - ā° TIMEOUT TRIGGER: Sending batch due to timeout (5.4s >= 5.0s)\n", - "2025-10-22 23:00:56 - noveum_trace.transport.batch_processor - INFO - šŸ“¤ SENDING BATCH: 4 traces via send_callback\n", - "2025-10-22 23:00:56 - noveum_trace.transport.http_transport - INFO - šŸš€ SENDING BATCH: 4 traces to https://api.noveum.ai/api/v1/traces\n", - "2025-10-22 23:00:56 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 1700f8dd-b141-4677-9778-db8440931e33) - 1 spans\n", - "2025-10-22 23:00:56 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 1700f8dd-b141-4677-9778-db8440931e33) - 1 spans\n", - "2025-10-22 23:00:56 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 1700f8dd-b141-4677-9778-db8440931e33\n", - "2025-10-22 23:00:56 - noveum_trace.transport.http_transport - INFO - āœ… Trace 1700f8dd-b141-4677-9778-db8440931e33 successfully queued for export\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:00:56 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-10-22 23:00:57 - noveum_trace.transport.http_transport - INFO - šŸ“” HTTP RESPONSE: Status 200 from https://api.noveum.ai/api/v1/traces\n", - "2025-10-22 23:00:57 - noveum_trace.transport.http_transport - INFO - āœ… Successfully sent batch of 4 traces\n", - "2025-10-22 23:00:57 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully sent batch of 4 traces via callback\n", - "2025-10-22 23:00:58 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 4ba7a1c6-56f2-40bb-bb89-bbcd9bf864b9) - 1 spans\n", - "2025-10-22 23:00:58 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 4ba7a1c6-56f2-40bb-bb89-bbcd9bf864b9) - 1 spans\n", - "2025-10-22 23:00:58 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 4ba7a1c6-56f2-40bb-bb89-bbcd9bf864b9\n", - "2025-10-22 23:00:58 - noveum_trace.transport.http_transport - INFO - āœ… Trace 4ba7a1c6-56f2-40bb-bb89-bbcd9bf864b9 successfully queued for export\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:00:58 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-10-22 23:00:59 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: c25eaffa-a215-4fb8-af0d-0d0fcf0acde9) - 1 spans\n", - "2025-10-22 23:00:59 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: c25eaffa-a215-4fb8-af0d-0d0fcf0acde9) - 1 spans\n", - "2025-10-22 23:00:59 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace c25eaffa-a215-4fb8-af0d-0d0fcf0acde9\n", - "2025-10-22 23:00:59 - noveum_trace.transport.http_transport - INFO - āœ… Trace c25eaffa-a215-4fb8-af0d-0d0fcf0acde9 successfully queued for export\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:00:59 - INFO - novaeval.evaluators.agent_evaluator - Saving intermediate results after 17 samples\n", - "2025-10-22 23:00:59 - INFO - novaeval.evaluators.agent_evaluator - Intermediate results saved to demo_results/agent.query_routing_dataset/agent_evaluation_results.csv\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Evaluating samples: 17it [01:16, 4.19s/it]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:00:59 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-10-22 23:01:00 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: b925a4e7-7af5-4039-a7ab-0fdb05e03c71) - 1 spans\n", - "2025-10-22 23:01:00 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: b925a4e7-7af5-4039-a7ab-0fdb05e03c71) - 1 spans\n", - "2025-10-22 23:01:00 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace b925a4e7-7af5-4039-a7ab-0fdb05e03c71\n", - "2025-10-22 23:01:00 - noveum_trace.transport.http_transport - INFO - āœ… Trace b925a4e7-7af5-4039-a7ab-0fdb05e03c71 successfully queued for export\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:01:00 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-10-22 23:01:01 - noveum_trace.transport.batch_processor - INFO - ā° TIMEOUT TRIGGER: Sending batch due to timeout (5.0s >= 5.0s)\n", - "2025-10-22 23:01:01 - noveum_trace.transport.batch_processor - INFO - šŸ“¤ SENDING BATCH: 4 traces via send_callback\n", - "2025-10-22 23:01:01 - noveum_trace.transport.http_transport - INFO - šŸš€ SENDING BATCH: 4 traces to https://api.noveum.ai/api/v1/traces\n", - "2025-10-22 23:01:01 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 9639d4eb-e562-4372-a019-e6eb2f867d0f) - 1 spans\n", - "2025-10-22 23:01:01 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 9639d4eb-e562-4372-a019-e6eb2f867d0f) - 1 spans\n", - "2025-10-22 23:01:01 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 9639d4eb-e562-4372-a019-e6eb2f867d0f\n", - "2025-10-22 23:01:01 - noveum_trace.transport.http_transport - INFO - āœ… Trace 9639d4eb-e562-4372-a019-e6eb2f867d0f successfully queued for export\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:01:01 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-10-22 23:01:01 - noveum_trace.transport.http_transport - INFO - šŸ“” HTTP RESPONSE: Status 200 from https://api.noveum.ai/api/v1/traces\n", - "2025-10-22 23:01:01 - noveum_trace.transport.http_transport - INFO - āœ… Successfully sent batch of 4 traces\n", - "2025-10-22 23:01:01 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully sent batch of 4 traces via callback\n", - "2025-10-22 23:01:03 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 02b5f22f-eb99-4514-9a0d-fb74709ade16) - 1 spans\n", - "2025-10-22 23:01:03 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 02b5f22f-eb99-4514-9a0d-fb74709ade16) - 1 spans\n", - "2025-10-22 23:01:03 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 02b5f22f-eb99-4514-9a0d-fb74709ade16\n", - "2025-10-22 23:01:03 - noveum_trace.transport.http_transport - INFO - āœ… Trace 02b5f22f-eb99-4514-9a0d-fb74709ade16 successfully queued for export\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:01:03 - INFO - novaeval.evaluators.agent_evaluator - Saving intermediate results after 18 samples\n", - "2025-10-22 23:01:03 - INFO - novaeval.evaluators.agent_evaluator - Intermediate results saved to demo_results/agent.query_routing_dataset/agent_evaluation_results.csv\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Evaluating samples: 18it [01:20, 4.06s/it]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:01:03 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-10-22 23:01:04 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: ed8a64df-f92b-4a38-b236-02cd5720da91) - 1 spans\n", - "2025-10-22 23:01:04 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: ed8a64df-f92b-4a38-b236-02cd5720da91) - 1 spans\n", - "2025-10-22 23:01:04 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace ed8a64df-f92b-4a38-b236-02cd5720da91\n", - "2025-10-22 23:01:04 - noveum_trace.transport.http_transport - INFO - āœ… Trace ed8a64df-f92b-4a38-b236-02cd5720da91 successfully queued for export\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:01:04 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-10-22 23:01:06 - noveum_trace.transport.batch_processor - INFO - ā° TIMEOUT TRIGGER: Sending batch due to timeout (5.3s >= 5.0s)\n", - "2025-10-22 23:01:06 - noveum_trace.transport.batch_processor - INFO - šŸ“¤ SENDING BATCH: 3 traces via send_callback\n", - "2025-10-22 23:01:06 - noveum_trace.transport.http_transport - INFO - šŸš€ SENDING BATCH: 3 traces to https://api.noveum.ai/api/v1/traces\n", - "2025-10-22 23:01:06 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 65462cbf-b85d-40b9-b50d-a71850336ebe) - 1 spans\n", - "2025-10-22 23:01:06 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 65462cbf-b85d-40b9-b50d-a71850336ebe) - 1 spans\n", - "2025-10-22 23:01:06 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 65462cbf-b85d-40b9-b50d-a71850336ebe\n", - "2025-10-22 23:01:06 - noveum_trace.transport.http_transport - INFO - āœ… Trace 65462cbf-b85d-40b9-b50d-a71850336ebe successfully queued for export\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:01:06 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-10-22 23:01:07 - noveum_trace.transport.http_transport - INFO - šŸ“” HTTP RESPONSE: Status 200 from https://api.noveum.ai/api/v1/traces\n", - "2025-10-22 23:01:07 - noveum_trace.transport.http_transport - INFO - āœ… Successfully sent batch of 3 traces\n", - "2025-10-22 23:01:07 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully sent batch of 3 traces via callback\n", - "2025-10-22 23:01:08 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: a174d9f5-fa2d-4c6d-a674-341e8382a11d) - 1 spans\n", - "2025-10-22 23:01:08 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: a174d9f5-fa2d-4c6d-a674-341e8382a11d) - 1 spans\n", - "2025-10-22 23:01:08 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace a174d9f5-fa2d-4c6d-a674-341e8382a11d\n", - "2025-10-22 23:01:08 - noveum_trace.transport.http_transport - INFO - āœ… Trace a174d9f5-fa2d-4c6d-a674-341e8382a11d successfully queued for export\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:01:08 - INFO - novaeval.evaluators.agent_evaluator - Saving intermediate results after 19 samples\n", - "2025-10-22 23:01:08 - INFO - novaeval.evaluators.agent_evaluator - Intermediate results saved to demo_results/agent.query_routing_dataset/agent_evaluation_results.csv\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Evaluating samples: 19it [01:25, 4.36s/it]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:01:08 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-10-22 23:01:09 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 672a16dd-589f-4d48-a8e4-c9e6bb5d871b) - 1 spans\n", - "2025-10-22 23:01:09 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 672a16dd-589f-4d48-a8e4-c9e6bb5d871b) - 1 spans\n", - "2025-10-22 23:01:09 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 672a16dd-589f-4d48-a8e4-c9e6bb5d871b\n", - "2025-10-22 23:01:09 - noveum_trace.transport.http_transport - INFO - āœ… Trace 672a16dd-589f-4d48-a8e4-c9e6bb5d871b successfully queued for export\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:01:09 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-10-22 23:01:11 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 89085ca6-0aa1-44f9-9d93-804b9725c2c0) - 1 spans\n", - "2025-10-22 23:01:11 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 89085ca6-0aa1-44f9-9d93-804b9725c2c0) - 1 spans\n", - "2025-10-22 23:01:11 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 89085ca6-0aa1-44f9-9d93-804b9725c2c0\n", - "2025-10-22 23:01:11 - noveum_trace.transport.http_transport - INFO - āœ… Trace 89085ca6-0aa1-44f9-9d93-804b9725c2c0 successfully queued for export\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:01:11 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-10-22 23:01:12 - noveum_trace.transport.batch_processor - INFO - ā° TIMEOUT TRIGGER: Sending batch due to timeout (5.3s >= 5.0s)\n", - "2025-10-22 23:01:12 - noveum_trace.transport.batch_processor - INFO - šŸ“¤ SENDING BATCH: 4 traces via send_callback\n", - "2025-10-22 23:01:12 - noveum_trace.transport.http_transport - INFO - šŸš€ SENDING BATCH: 4 traces to https://api.noveum.ai/api/v1/traces\n", - "2025-10-22 23:01:12 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: b7a046e6-46d4-41f7-a47b-35dae73bb923) - 1 spans\n", - "2025-10-22 23:01:12 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: b7a046e6-46d4-41f7-a47b-35dae73bb923) - 1 spans\n", - "2025-10-22 23:01:12 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace b7a046e6-46d4-41f7-a47b-35dae73bb923\n", - "2025-10-22 23:01:12 - noveum_trace.transport.http_transport - INFO - āœ… Trace b7a046e6-46d4-41f7-a47b-35dae73bb923 successfully queued for export\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:01:12 - INFO - novaeval.evaluators.agent_evaluator - Saving intermediate results after 20 samples\n", - "2025-10-22 23:01:12 - INFO - novaeval.evaluators.agent_evaluator - Intermediate results saved to demo_results/agent.query_routing_dataset/agent_evaluation_results.csv\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Evaluating samples: 20it [01:29, 4.46s/it]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:01:12 - INFO - novaeval.evaluators.agent_evaluator - Saving final results\n", - "2025-10-22 23:01:12 - INFO - novaeval.evaluators.agent_evaluator - Reloaded 20 results from CSV\n", - "2025-10-22 23:01:12 - INFO - novaeval.evaluators.agent_evaluator - Agent evaluation completed\n", - "\n", - "āœ… Evaluation completed!\n", - "\n", - "šŸ“Š Results Summary:\n", - " - task_progression: 1.27\n", - " - context_relevancy: 7.36\n", - " - role_adherence: 5.45\n", - " - tool_relevancy: 0.00\n", - " - parameter_correctness: 0.00\n", - "\n", - "šŸ” Individual Scores:\n", - "\n", - " Record 1 (Task: eda4fe22-9a2b-4b73-856b-f4f3309bf719):\n", - " - task_progression: 1.0\n", - " - context_relevancy: 7.5\n", - " - role_adherence: 1.0\n", - " - tool_relevancy: 0.0\n", - " - parameter_correctness: 0.0\n", - "\n", - " Record 2 (Task: 0ffffba1-8a37-443c-8866-d53ffbfa7718):\n", - " - task_progression: 2.8\n", - " - context_relevancy: 7.8\n", - " - role_adherence: 10.0\n", - " - tool_relevancy: 0.0\n", - " - parameter_correctness: 0.0\n", - "\n", - " Record 3 (Task: f1f37bd7-0851-4659-b493-b80d3800d920):\n", - " - task_progression: 1.0\n", - " - context_relevancy: 7.5\n", - " - role_adherence: 10.0\n", - " - tool_relevancy: 0.0\n", - " - parameter_correctness: 0.0\n", - "\n", - " Record 4 (Task: 43cdf081-4f01-49cd-b566-dbd1619e6cd2):\n", - " - task_progression: 1.0\n", - " - context_relevancy: 7.5\n", - " - role_adherence: 10.0\n", - " - tool_relevancy: 0.0\n", - " - parameter_correctness: 0.0\n", - "\n", - " Record 5 (Task: 9a1983f4-09da-4b53-80e6-38de6878e0e7):\n", - " - task_progression: 2.8\n", - " - context_relevancy: 7.8\n", - " - role_adherence: 1.0\n", - " - tool_relevancy: 0.0\n", - " - parameter_correctness: 0.0\n", - "\n", - " Record 6 (Task: 5d2517e1-220a-429d-9d59-f701bda25eed):\n", - " - task_progression: 1.0\n", - " - context_relevancy: 1.0\n", - " - role_adherence: 1.0\n", - " - tool_relevancy: 0.0\n", - " - parameter_correctness: 0.0\n", - "\n", - " Record 7 (Task: 52aacb67-c361-4445-9b72-c157f79f47d6):\n", - " - task_progression: 1.0\n", - " - context_relevancy: 7.5\n", - " - role_adherence: 1.0\n", - " - tool_relevancy: 0.0\n", - " - parameter_correctness: 0.0\n", - "\n", - " Record 8 (Task: a81ca3a8-80aa-4c39-876e-8d40ea7a0aef):\n", - " - task_progression: 1.0\n", - " - context_relevancy: 7.5\n", - " - role_adherence: 10.0\n", - " - tool_relevancy: 0.0\n", - " - parameter_correctness: 0.0\n", - "\n", - " Record 9 (Task: 230aad27-f3dd-4968-a45a-3c2f07ac28ed):\n", - " - task_progression: 1.0\n", - " - context_relevancy: 7.8\n", - " - role_adherence: 10.0\n", - " - tool_relevancy: 0.0\n", - " - parameter_correctness: 0.0\n", - "\n", - " Record 10 (Task: 83c7dcce-3d89-4da1-8b3f-d419885d4cbc):\n", - " - task_progression: 1.0\n", - " - context_relevancy: 7.8\n", - " - role_adherence: 1.0\n", - " - tool_relevancy: 0.0\n", - " - parameter_correctness: 0.0\n", - "\n", - " Record 11 (Task: 2218f641-604c-491a-9710-b51a9941b982):\n", - " - task_progression: 1.0\n", - " - context_relevancy: 7.8\n", - " - role_adherence: 9.0\n", - " - tool_relevancy: 0.0\n", - " - parameter_correctness: 0.0\n", - "\n", - " Record 12 (Task: 255fd49c-84b4-4b18-887e-6308a412d535):\n", - " - task_progression: 1.0\n", - " - context_relevancy: 7.8\n", - " - role_adherence: 1.0\n", - " - tool_relevancy: 0.0\n", - " - parameter_correctness: 0.0\n", - "\n", - " Record 13 (Task: dc511122-c0b6-415c-9a49-c7b45132dd87):\n", - " - task_progression: 1.0\n", - " - context_relevancy: 7.8\n", - " - role_adherence: 10.0\n", - " - tool_relevancy: 0.0\n", - " - parameter_correctness: 0.0\n", - "\n", - " Record 14 (Task: 04bebf38-a343-4563-80db-0154bef8d927):\n", - " - task_progression: 1.0\n", - " - context_relevancy: 7.8\n", - " - role_adherence: 1.0\n", - " - tool_relevancy: 0.0\n", - " - parameter_correctness: 0.0\n", - "\n", - " Record 15 (Task: 5e043630-6493-42b5-beb8-79faa19bfa37):\n", - " - task_progression: 1.0\n", - " - context_relevancy: 7.8\n", - " - role_adherence: 10.0\n", - " - tool_relevancy: 0.0\n", - " - parameter_correctness: 0.0\n", - "\n", - " Record 16 (Task: 7da9814d-a2e8-4c4e-b750-68b26bd5fd22):\n", - " - task_progression: 1.0\n", - " - context_relevancy: 7.5\n", - " - role_adherence: 1.0\n", - " - tool_relevancy: 0.0\n", - " - parameter_correctness: 0.0\n", - "\n", - " Record 17 (Task: 16143f74-2831-4753-b33d-ce4b645093c5):\n", - " - task_progression: 1.0\n", - " - context_relevancy: 7.8\n", - " - role_adherence: 10.0\n", - " - tool_relevancy: 0.0\n", - " - parameter_correctness: 0.0\n", - "\n", - " Record 18 (Task: fc64e6cc-6739-4256-ac4a-7b80c3028233):\n", - " - task_progression: 1.0\n", - " - context_relevancy: 7.8\n", - " - role_adherence: 1.0\n", - " - tool_relevancy: 0.0\n", - " - parameter_correctness: 0.0\n", - "\n", - " Record 19 (Task: b7945c49-f584-4c70-972d-536a805d8a31):\n", - " - task_progression: 1.0\n", - " - context_relevancy: 7.5\n", - " - role_adherence: 1.0\n", - " - tool_relevancy: 0.0\n", - " - parameter_correctness: 0.0\n", - "\n", - " Record 20 (Task: f5c40ecf-36c0-45ba-9cc9-dc0329b0324b):\n", - " - task_progression: 2.8\n", - " - context_relevancy: 7.8\n", - " - role_adherence: 10.0\n", - " - tool_relevancy: 0.0\n", - " - parameter_correctness: 0.0\n", - "āœ… Evaluation completed successfully!\n", - "\n", - "šŸ“‹ Step 7: Exporting Dataset\n", - "šŸ’¾ Exporting processed dataset...\n", - "āœ… Exported to ./processed_datasets/agent.query_routing_dataset_processed_dataset.json\n", - "āœ… Exported to ./processed_datasets/agent.query_routing_dataset_processed_dataset.csv\n", - "āœ… Export completed successfully!\n", - "\n", - "============================================================\n", - "šŸŽ‰ EVALUATION PIPELINE COMPLETED!\n", - "šŸ“Š Final Results:\n", - " - File processed: split_datasets/agent.query_routing_dataset.json\n", - " - Spans loaded: 20\n", - " - Dataset size: 20\n", - " - Evaluation completed: True\n", - " - Export successful: True\n", - " - Results saved to: ./demo_results/agent.query_routing_dataset/\n", - "Completed agent.query_routing_dataset.json\n", - "\n", - "Processing tool-orchestator_dataset.json...\n", - "šŸš€ Starting Complete Agent Evaluation Pipeline\n", - "šŸ“ Processing file: split_datasets/tool-orchestator_dataset.json\n", - "============================================================\n", - "\n", - "šŸ“‹ Step 1: Environment Setup\n", - "āœ… Logging configured at INFO level\n", - "šŸ” Environment validation:\n", - " āœ… gemini_api_key: True\n", - " āœ… pandas_available: True\n", - " āœ… novaeval_available: True\n", - "āœ… Environment ready for evaluation!\n", - "\n", - "šŸ“‹ Step 2: Loading Dataset\n", - "šŸ“Š Loaded 20 spans from split_datasets/tool-orchestator_dataset.json\n", - "\n", - "šŸ” Available span types:\n", - " - tool-orchestator: 20\n", - "āœ… Dataset loaded: 20 spans\n", - "\n", - "šŸ“‹ Step 3: Converting to AgentDataset Format\n", - "šŸ”„ Converting spans to AgentData objects...\n", - "returning unknown type for span\n", - "{'span_id': 'b6b7c1c0-3b7b-4e19-805d-a16beb7ea871', 'trace_id': 'eda4fe22-9a2b-4b73-856b-f4f3309bf719', 'parent_span_id': '', 'name': 'tool-orchestator', 'start_time': '2025-10-09 08:26:46.384000000', 'end_time': '2025-10-09 08:26:48.685000000', 'duration_ms': 2301.412, 'status': 'ok', 'status_message': '', 'attributes': {'input_query': 'What are the latest breakthroughs in artificial intelligence?', 'query.length': 61, 'processing.mode': 'Web Search', 'processing.time_seconds': 2.301326274871826, 'processing.response_length': 423, 'processing.sources_count': 1, 'output_response': 'Final Answer: The provided web search results do not contain specific information about the latest breakthroughs in artificial intelligence. To find the most recent advancements in AI, I recommend checking reputabl...', 'final_answer_mode': 'Web Search', 'query_processed.input_query': 'What are the latest breakthroughs in artificial intelligence?', 'query_processed.output_response': 'Successfully processed query using Web Search, generated 423 character response', 'query_processed.mode': 'Web Search', 'query_processed.processing_time': 2.301326274871826, 'query_processed.response_length': 423}, 'events': [], 'links': [], 'trace_trace_id': 'eda4fe22-9a2b-4b73-856b-f4f3309bf719', 'trace_name': 'auto_trace_tool-orchestator', 'project': 'noveum-ai-agent-rag-websearch', 'environment': 'development', 'trace_status': 'ok', 'trace_status_message': '', 'trace_start_time': '2025-10-09 08:26:46.384000000', 'trace_end_time': '2025-10-09 08:26:48.685000000', 'trace_duration_ms': 2301.488, 'span_count': 6, 'error_count': 0, 'sdk': {'name': 'noveum-trace-python', 'version': '0.3.8'}, 'trace_attributes': {'noveum.project': 'noveum-ai-agent-rag-websearch', 'noveum.environment': 'development', 'noveum.sdk.version': '0.3.8', 'noveum.sampling.decision': 'record'}, 'metadata': {'user_id': None, 'session_id': None, 'request_id': None, 'tags': {}, 'custom_attributes': {}}, 'created_at': '2025-10-09 08:26:49.422000000', 'updated_at': '2025-10-09 08:26:49.422000000'}\n", - "Spans with unknown type: 1\n", - "returning unknown type for span\n", - "{'span_id': '3045d971-b7ff-46cc-be11-6db7d65dfefd', 'trace_id': '0ffffba1-8a37-443c-8866-d53ffbfa7718', 'parent_span_id': '', 'name': 'tool-orchestator', 'start_time': '2025-10-09 08:26:41.850000000', 'end_time': '2025-10-09 08:26:45.377000000', 'duration_ms': 3527.75, 'status': 'ok', 'status_message': '', 'attributes': {'input_query': \"What's happening in the software development world today?\", 'query.length': 57, 'processing.mode': 'Web Search', 'processing.time_seconds': 3.527592897415161, 'processing.response_length': 795, 'processing.sources_count': 1, 'output_response': \"Final Answer: The web search did not yield any specific results regarding current events in the software development world. Therefore, I cannot provide detailed information on what's happening today in that field.\\n...\", 'final_answer_mode': 'Web Search', 'query_processed.input_query': \"What's happening in the software development world today?\", 'query_processed.output_response': 'Successfully processed query using Web Search, generated 795 character response', 'query_processed.mode': 'Web Search', 'query_processed.processing_time': 3.527592897415161, 'query_processed.response_length': 795}, 'events': [], 'links': [], 'trace_trace_id': '0ffffba1-8a37-443c-8866-d53ffbfa7718', 'trace_name': 'auto_trace_tool-orchestator', 'project': 'noveum-ai-agent-rag-websearch', 'environment': 'development', 'trace_status': 'ok', 'trace_status_message': '', 'trace_start_time': '2025-10-09 08:26:41.850000000', 'trace_end_time': '2025-10-09 08:26:45.377000000', 'trace_duration_ms': 3527.831, 'span_count': 6, 'error_count': 0, 'sdk': {'name': 'noveum-trace-python', 'version': '0.3.8'}, 'trace_attributes': {'noveum.project': 'noveum-ai-agent-rag-websearch', 'noveum.environment': 'development', 'noveum.sdk.version': '0.3.8', 'noveum.sampling.decision': 'record'}, 'metadata': {'user_id': None, 'session_id': None, 'request_id': None, 'tags': {}, 'custom_attributes': {}}, 'created_at': '2025-10-09 08:26:49.422000000', 'updated_at': '2025-10-09 08:26:49.422000000'}\n", - "Spans with unknown type: 2\n", - "returning unknown type for span\n", - "{'span_id': '42d4cb1f-17ec-433a-8087-90292c2fa9d9', 'trace_id': 'f1f37bd7-0851-4659-b493-b80d3800d920', 'parent_span_id': '', 'name': 'tool-orchestator', 'start_time': '2025-10-09 08:26:35.915000000', 'end_time': '2025-10-09 08:26:40.841000000', 'duration_ms': 4926.2519999999995, 'status': 'ok', 'status_message': '', 'attributes': {'input_query': 'What are the newest features in cloud computing?', 'query.length': 48, 'processing.mode': 'RAG', 'processing.time_seconds': 4.926196813583374, 'processing.response_length': 883, 'processing.sources_count': 5, 'output_response': 'Final Answer: The provided context does not specifically mention any new features in cloud computing as a general topic. However, it does highlight several recent enhancements and features related to the Noveum.ai ...', 'final_answer_mode': 'RAG', 'query_processed.input_query': 'What are the newest features in cloud computing?', 'query_processed.output_response': 'Successfully processed query using RAG, generated 883 character response', 'query_processed.mode': 'RAG', 'query_processed.processing_time': 4.926196813583374, 'query_processed.response_length': 883}, 'events': [], 'links': [], 'trace_trace_id': 'f1f37bd7-0851-4659-b493-b80d3800d920', 'trace_name': 'auto_trace_tool-orchestator', 'project': 'noveum-ai-agent-rag-websearch', 'environment': 'development', 'trace_status': 'ok', 'trace_status_message': '', 'trace_start_time': '2025-10-09 08:26:35.915000000', 'trace_end_time': '2025-10-09 08:26:40.841000000', 'trace_duration_ms': 4926.342, 'span_count': 6, 'error_count': 0, 'sdk': {'name': 'noveum-trace-python', 'version': '0.3.8'}, 'trace_attributes': {'noveum.project': 'noveum-ai-agent-rag-websearch', 'noveum.environment': 'development', 'noveum.sdk.version': '0.3.8', 'noveum.sampling.decision': 'record'}, 'metadata': {'user_id': None, 'session_id': None, 'request_id': None, 'tags': {}, 'custom_attributes': {}}, 'created_at': '2025-10-09 08:26:44.053000000', 'updated_at': '2025-10-09 08:26:44.053000000'}\n", - "Spans with unknown type: 3\n", - "returning unknown type for span\n", - "{'span_id': 'e22b809d-5320-40b7-b78a-e3e125538a6b', 'trace_id': '43cdf081-4f01-49cd-b566-dbd1619e6cd2', 'parent_span_id': '', 'name': 'tool-orchestator', 'start_time': '2025-10-09 08:26:32.239000000', 'end_time': '2025-10-09 08:26:34.906000000', 'duration_ms': 2667.1330000000003, 'status': 'ok', 'status_message': '', 'attributes': {'input_query': \"What's the current status of cryptocurrency markets?\", 'query.length': 52, 'processing.mode': 'Web Search', 'processing.time_seconds': 2.6669082641601562, 'processing.response_length': 371, 'processing.sources_count': 1, 'output_response': 'Final Answer: The web search results did not provide any specific information regarding the current status of cryptocurrency markets. Therefore, I cannot provide an accurate update on market conditions, trends, or ...', 'final_answer_mode': 'Web Search', 'query_processed.input_query': \"What's the current status of cryptocurrency markets?\", 'query_processed.output_response': 'Successfully processed query using Web Search, generated 371 character response', 'query_processed.mode': 'Web Search', 'query_processed.processing_time': 2.6669082641601562, 'query_processed.response_length': 371}, 'events': [], 'links': [], 'trace_trace_id': '43cdf081-4f01-49cd-b566-dbd1619e6cd2', 'trace_name': 'auto_trace_tool-orchestator', 'project': 'noveum-ai-agent-rag-websearch', 'environment': 'development', 'trace_status': 'ok', 'trace_status_message': '', 'trace_start_time': '2025-10-09 08:26:32.237000000', 'trace_end_time': '2025-10-09 08:26:34.906000000', 'trace_duration_ms': 2668.841, 'span_count': 6, 'error_count': 0, 'sdk': {'name': 'noveum-trace-python', 'version': '0.3.8'}, 'trace_attributes': {'noveum.project': 'noveum-ai-agent-rag-websearch', 'noveum.environment': 'development', 'noveum.sdk.version': '0.3.8', 'noveum.sampling.decision': 'record'}, 'metadata': {'user_id': None, 'session_id': None, 'request_id': None, 'tags': {}, 'custom_attributes': {}}, 'created_at': '2025-10-09 08:26:38.618000000', 'updated_at': '2025-10-09 08:26:38.618000000'}\n", - "Spans with unknown type: 4\n", - "returning unknown type for span\n", - "{'span_id': '813a4450-1cf8-4d43-8493-eabfa200919b', 'trace_id': '9a1983f4-09da-4b53-80e6-38de6878e0e7', 'parent_span_id': '', 'name': 'tool-orchestator', 'start_time': '2025-10-09 08:26:27.992000000', 'end_time': '2025-10-09 08:26:31.231000000', 'duration_ms': 3239.601, 'status': 'ok', 'status_message': '', 'attributes': {'input_query': 'What are the latest updates in Python programming?', 'query.length': 50, 'processing.mode': 'Web Search', 'processing.time_seconds': 3.2394051551818848, 'processing.response_length': 583, 'processing.sources_count': 1, 'output_response': 'Final Answer: The web search results do not provide specific information about the latest updates in Python programming. They focus primarily on news related to China, including geopolitics, economy, and lifestyle,...', 'final_answer_mode': 'Web Search', 'query_processed.input_query': 'What are the latest updates in Python programming?', 'query_processed.output_response': 'Successfully processed query using Web Search, generated 583 character response', 'query_processed.mode': 'Web Search', 'query_processed.processing_time': 3.2394051551818848, 'query_processed.response_length': 583}, 'events': [], 'links': [], 'trace_trace_id': '9a1983f4-09da-4b53-80e6-38de6878e0e7', 'trace_name': 'auto_trace_tool-orchestator', 'project': 'noveum-ai-agent-rag-websearch', 'environment': 'development', 'trace_status': 'ok', 'trace_status_message': '', 'trace_start_time': '2025-10-09 08:26:27.992000000', 'trace_end_time': '2025-10-09 08:26:31.231000000', 'trace_duration_ms': 3239.706, 'span_count': 6, 'error_count': 0, 'sdk': {'name': 'noveum-trace-python', 'version': '0.3.8'}, 'trace_attributes': {'noveum.project': 'noveum-ai-agent-rag-websearch', 'noveum.environment': 'development', 'noveum.sdk.version': '0.3.8', 'noveum.sampling.decision': 'record'}, 'metadata': {'user_id': None, 'session_id': None, 'request_id': None, 'tags': {}, 'custom_attributes': {}}, 'created_at': '2025-10-09 08:26:33.463000000', 'updated_at': '2025-10-09 08:26:33.463000000'}\n", - "Spans with unknown type: 5\n", - "returning unknown type for span\n", - "{'span_id': '6102352a-833e-493d-adbb-887e2af17918', 'trace_id': '5d2517e1-220a-429d-9d59-f701bda25eed', 'parent_span_id': '', 'name': 'tool-orchestator', 'start_time': '2025-10-09 08:26:23.210000000', 'end_time': '2025-10-09 08:26:26.985000000', 'duration_ms': 3775.0409999999997, 'status': 'ok', 'status_message': '', 'attributes': {'input_query': 'What happened in tech news this week?', 'query.length': 37, 'processing.mode': 'Web Search', 'processing.time_seconds': 3.774940013885498, 'processing.response_length': 501, 'processing.sources_count': 1, 'output_response': 'Final Answer: The web search results do not provide specific information about recent events in tech news for this week. They primarily discuss the grammatical usage of the phrase \"what happened\" and its variations...', 'final_answer_mode': 'Web Search', 'query_processed.input_query': 'What happened in tech news this week?', 'query_processed.output_response': 'Successfully processed query using Web Search, generated 501 character response', 'query_processed.mode': 'Web Search', 'query_processed.processing_time': 3.774940013885498, 'query_processed.response_length': 501}, 'events': [], 'links': [], 'trace_trace_id': '5d2517e1-220a-429d-9d59-f701bda25eed', 'trace_name': 'auto_trace_tool-orchestator', 'project': 'noveum-ai-agent-rag-websearch', 'environment': 'development', 'trace_status': 'ok', 'trace_status_message': '', 'trace_start_time': '2025-10-09 08:26:23.210000000', 'trace_end_time': '2025-10-09 08:26:26.985000000', 'trace_duration_ms': 3775.124, 'span_count': 6, 'error_count': 0, 'sdk': {'name': 'noveum-trace-python', 'version': '0.3.8'}, 'trace_attributes': {'noveum.project': 'noveum-ai-agent-rag-websearch', 'noveum.environment': 'development', 'noveum.sdk.version': '0.3.8', 'noveum.sampling.decision': 'record'}, 'metadata': {'user_id': None, 'session_id': None, 'request_id': None, 'tags': {}, 'custom_attributes': {}}, 'created_at': '2025-10-09 08:26:28.174000000', 'updated_at': '2025-10-09 08:26:28.174000000'}\n", - "Spans with unknown type: 6\n", - "returning unknown type for span\n", - "{'span_id': '2926418a-fe09-4df0-bf19-a527bc24a478', 'trace_id': '52aacb67-c361-4445-9b72-c157f79f47d6', 'parent_span_id': '', 'name': 'tool-orchestator', 'start_time': '2025-10-09 08:26:16.544000000', 'end_time': '2025-10-09 08:26:22.201000000', 'duration_ms': 5656.852, 'status': 'ok', 'status_message': '', 'attributes': {'input_query': 'What are the current trends in observability tools?', 'query.length': 51, 'processing.mode': 'RAG', 'processing.time_seconds': 5.656759023666382, 'processing.response_length': 1074, 'processing.sources_count': 5, 'output_response': 'Final Answer: The provided context does not explicitly outline current trends in observability tools. However, it highlights some key aspects of observability that are particularly relevant to AI applications. \\n\\n1....', 'final_answer_mode': 'RAG', 'query_processed.input_query': 'What are the current trends in observability tools?', 'query_processed.output_response': 'Successfully processed query using RAG, generated 1074 character response', 'query_processed.mode': 'RAG', 'query_processed.processing_time': 5.656759023666382, 'query_processed.response_length': 1074}, 'events': [], 'links': [], 'trace_trace_id': '52aacb67-c361-4445-9b72-c157f79f47d6', 'trace_name': 'auto_trace_tool-orchestator', 'project': 'noveum-ai-agent-rag-websearch', 'environment': 'development', 'trace_status': 'ok', 'trace_status_message': '', 'trace_start_time': '2025-10-09 08:26:16.544000000', 'trace_end_time': '2025-10-09 08:26:22.201000000', 'trace_duration_ms': 5657.012, 'span_count': 6, 'error_count': 0, 'sdk': {'name': 'noveum-trace-python', 'version': '0.3.8'}, 'trace_attributes': {'noveum.project': 'noveum-ai-agent-rag-websearch', 'noveum.environment': 'development', 'noveum.sdk.version': '0.3.8', 'noveum.sampling.decision': 'record'}, 'metadata': {'user_id': None, 'session_id': None, 'request_id': None, 'tags': {}, 'custom_attributes': {}}, 'created_at': '2025-10-09 08:26:22.904000000', 'updated_at': '2025-10-09 08:26:22.904000000'}\n", - "Spans with unknown type: 7\n", - "returning unknown type for span\n", - "{'span_id': 'afe85a1d-13c8-4476-8154-52d58ba83d52', 'trace_id': 'a81ca3a8-80aa-4c39-876e-8d40ea7a0aef', 'parent_span_id': '', 'name': 'tool-orchestator', 'start_time': '2025-10-09 08:26:08.903000000', 'end_time': '2025-10-09 08:26:15.537000000', 'duration_ms': 6633.559, 'status': 'ok', 'status_message': '', 'attributes': {'input_query': 'Tell me about recent developments in machine learning', 'query.length': 53, 'processing.mode': 'Web Search', 'processing.time_seconds': 6.63348126411438, 'processing.response_length': 1639, 'processing.sources_count': 1, 'output_response': 'Final Answer: The provided web search results do not contain specific information about recent developments in machine learning. Therefore, I cannot provide a detailed answer based on those results.\\n\\nHowever, I can...', 'final_answer_mode': 'Web Search', 'query_processed.input_query': 'Tell me about recent developments in machine learning', 'query_processed.output_response': 'Successfully processed query using Web Search, generated 1639 character response', 'query_processed.mode': 'Web Search', 'query_processed.processing_time': 6.63348126411438, 'query_processed.response_length': 1639}, 'events': [], 'links': [], 'trace_trace_id': 'a81ca3a8-80aa-4c39-876e-8d40ea7a0aef', 'trace_name': 'auto_trace_tool-orchestator', 'project': 'noveum-ai-agent-rag-websearch', 'environment': 'development', 'trace_status': 'ok', 'trace_status_message': '', 'trace_start_time': '2025-10-09 08:26:08.903000000', 'trace_end_time': '2025-10-09 08:26:15.537000000', 'trace_duration_ms': 6633.662, 'span_count': 6, 'error_count': 0, 'sdk': {'name': 'noveum-trace-python', 'version': '0.3.8'}, 'trace_attributes': {'noveum.project': 'noveum-ai-agent-rag-websearch', 'noveum.environment': 'development', 'noveum.sdk.version': '0.3.8', 'noveum.sampling.decision': 'record'}, 'metadata': {'user_id': None, 'session_id': None, 'request_id': None, 'tags': {}, 'custom_attributes': {}}, 'created_at': '2025-10-09 08:26:16.227000000', 'updated_at': '2025-10-09 08:26:16.227000000'}\n", - "Spans with unknown type: 8\n", - "returning unknown type for span\n", - "{'span_id': 'db8da393-6d72-421e-87dc-96f56b6576e5', 'trace_id': '230aad27-f3dd-4968-a45a-3c2f07ac28ed', 'parent_span_id': '', 'name': 'tool-orchestator', 'start_time': '2025-10-09 08:26:02.424000000', 'end_time': '2025-10-09 08:26:07.894000000', 'duration_ms': 5470.179, 'status': 'ok', 'status_message': '', 'attributes': {'input_query': \"What's the weather like today?\", 'query.length': 30, 'processing.mode': 'Web Search', 'processing.time_seconds': 5.469927072525024, 'processing.response_length': 376, 'processing.sources_count': 1, 'output_response': \"Final Answer: The provided web search results do not contain specific information about today's weather in any particular location. They mention weather forecasts for various places, including France and Singapore,...\", 'final_answer_mode': 'Web Search', 'query_processed.input_query': \"What's the weather like today?\", 'query_processed.output_response': 'Successfully processed query using Web Search, generated 376 character response', 'query_processed.mode': 'Web Search', 'query_processed.processing_time': 5.469927072525024, 'query_processed.response_length': 376}, 'events': [], 'links': [], 'trace_trace_id': '230aad27-f3dd-4968-a45a-3c2f07ac28ed', 'trace_name': 'auto_trace_tool-orchestator', 'project': 'noveum-ai-agent-rag-websearch', 'environment': 'development', 'trace_status': 'ok', 'trace_status_message': '', 'trace_start_time': '2025-10-09 08:26:02.424000000', 'trace_end_time': '2025-10-09 08:26:07.894000000', 'trace_duration_ms': 5470.232, 'span_count': 6, 'error_count': 0, 'sdk': {'name': 'noveum-trace-python', 'version': '0.3.8'}, 'trace_attributes': {'noveum.project': 'noveum-ai-agent-rag-websearch', 'noveum.environment': 'development', 'noveum.sdk.version': '0.3.8', 'noveum.sampling.decision': 'record'}, 'metadata': {'user_id': None, 'session_id': None, 'request_id': None, 'tags': {}, 'custom_attributes': {}}, 'created_at': '2025-10-09 08:26:08.586000000', 'updated_at': '2025-10-09 08:26:08.586000000'}\n", - "Spans with unknown type: 9\n", - "returning unknown type for span\n", - "{'span_id': 'a290202d-1b3f-431f-8728-fcb937380f8c', 'trace_id': '83c7dcce-3d89-4da1-8b3f-d419885d4cbc', 'parent_span_id': '', 'name': 'tool-orchestator', 'start_time': '2025-10-09 08:25:56.886000000', 'end_time': '2025-10-09 08:26:01.416000000', 'duration_ms': 4530.039000000001, 'status': 'ok', 'status_message': '', 'attributes': {'input_query': 'What are the latest AI news today?', 'query.length': 34, 'processing.mode': 'Web Search', 'processing.time_seconds': 4.529800176620483, 'processing.response_length': 1003, 'processing.sources_count': 1, 'output_response': 'Final Answer: The latest news in artificial intelligence includes several key updates:\\n\\n1. **AI Spending Concerns**: Jeff Bezos has expressed concerns that the current level of investment in AI resembles an \"indust...', 'final_answer_mode': 'Web Search', 'query_processed.input_query': 'What are the latest AI news today?', 'query_processed.output_response': 'Successfully processed query using Web Search, generated 1003 character response', 'query_processed.mode': 'Web Search', 'query_processed.processing_time': 4.529800176620483, 'query_processed.response_length': 1003}, 'events': [], 'links': [], 'trace_trace_id': '83c7dcce-3d89-4da1-8b3f-d419885d4cbc', 'trace_name': 'auto_trace_tool-orchestator', 'project': 'noveum-ai-agent-rag-websearch', 'environment': 'development', 'trace_status': 'ok', 'trace_status_message': '', 'trace_start_time': '2025-10-09 08:25:56.886000000', 'trace_end_time': '2025-10-09 08:26:01.416000000', 'trace_duration_ms': 4530.08, 'span_count': 6, 'error_count': 0, 'sdk': {'name': 'noveum-trace-python', 'version': '0.3.8'}, 'trace_attributes': {'noveum.project': 'noveum-ai-agent-rag-websearch', 'noveum.environment': 'development', 'noveum.sdk.version': '0.3.8', 'noveum.sampling.decision': 'record'}, 'metadata': {'user_id': None, 'session_id': None, 'request_id': None, 'tags': {}, 'custom_attributes': {}}, 'created_at': '2025-10-09 08:26:02.122000000', 'updated_at': '2025-10-09 08:26:02.122000000'}\n", - "Spans with unknown type: 10\n", - "returning unknown type for span\n", - "{'span_id': '5fbaef83-95fc-4bed-9695-9d69c37c35e4', 'trace_id': '2218f641-604c-491a-9710-b51a9941b982', 'parent_span_id': '', 'name': 'tool-orchestator', 'start_time': '2025-10-09 08:25:48.107000000', 'end_time': '2025-10-09 08:25:55.878000000', 'duration_ms': 7770.846, 'status': 'ok', 'status_message': '', 'attributes': {'input_query': 'What are the benefits of using Noveum Trace?', 'query.length': 44, 'processing.mode': 'RAG', 'processing.time_seconds': 7.770798206329346, 'processing.response_length': 1460, 'processing.sources_count': 5, 'output_response': 'Final Answer: Using Noveum Trace offers several benefits, including:\\n\\n1. **Real-Time Monitoring**: The Noveum platform provides a real-time dashboard for analyzing traces and performance, with live updates as new t...', 'final_answer_mode': 'RAG', 'query_processed.input_query': 'What are the benefits of using Noveum Trace?', 'query_processed.output_response': 'Successfully processed query using RAG, generated 1460 character response', 'query_processed.mode': 'RAG', 'query_processed.processing_time': 7.770798206329346, 'query_processed.response_length': 1460}, 'events': [], 'links': [], 'trace_trace_id': '2218f641-604c-491a-9710-b51a9941b982', 'trace_name': 'auto_trace_tool-orchestator', 'project': 'noveum-ai-agent-rag-websearch', 'environment': 'development', 'trace_status': 'ok', 'trace_status_message': '', 'trace_start_time': '2025-10-09 08:25:48.107000000', 'trace_end_time': '2025-10-09 08:25:55.878000000', 'trace_duration_ms': 7770.953, 'span_count': 6, 'error_count': 0, 'sdk': {'name': 'noveum-trace-python', 'version': '0.3.8'}, 'trace_attributes': {'noveum.project': 'noveum-ai-agent-rag-websearch', 'noveum.environment': 'development', 'noveum.sdk.version': '0.3.8', 'noveum.sampling.decision': 'record'}, 'metadata': {'user_id': None, 'session_id': None, 'request_id': None, 'tags': {}, 'custom_attributes': {}}, 'created_at': '2025-10-09 08:25:56.565000000', 'updated_at': '2025-10-09 08:25:56.565000000'}\n", - "Spans with unknown type: 11\n", - "returning unknown type for span\n", - "{'span_id': 'b3ac3924-928b-4d48-a2a7-aa749802fef9', 'trace_id': '255fd49c-84b4-4b18-887e-6308a412d535', 'parent_span_id': '', 'name': 'tool-orchestator', 'start_time': '2025-10-09 08:25:43.830000000', 'end_time': '2025-10-09 08:25:47.099000000', 'duration_ms': 3268.8160000000003, 'status': 'ok', 'status_message': '', 'attributes': {'input_query': 'How do I configure Noveum for my system?', 'query.length': 40, 'processing.mode': 'RAG', 'processing.time_seconds': 3.268723249435425, 'processing.response_length': 613, 'processing.sources_count': 5, 'output_response': 'Final Answer: To configure Noveum for your system, you can refer to the SDK Integration Guide available in the Noveum documentation. This guide will help you trace your AI applications with minimal code changes. Th...', 'final_answer_mode': 'RAG', 'query_processed.input_query': 'How do I configure Noveum for my system?', 'query_processed.output_response': 'Successfully processed query using RAG, generated 613 character response', 'query_processed.mode': 'RAG', 'query_processed.processing_time': 3.268723249435425, 'query_processed.response_length': 613}, 'events': [], 'links': [], 'trace_trace_id': '255fd49c-84b4-4b18-887e-6308a412d535', 'trace_name': 'auto_trace_tool-orchestator', 'project': 'noveum-ai-agent-rag-websearch', 'environment': 'development', 'trace_status': 'ok', 'trace_status_message': '', 'trace_start_time': '2025-10-09 08:25:43.830000000', 'trace_end_time': '2025-10-09 08:25:47.099000000', 'trace_duration_ms': 3268.862, 'span_count': 6, 'error_count': 0, 'sdk': {'name': 'noveum-trace-python', 'version': '0.3.8'}, 'trace_attributes': {'noveum.project': 'noveum-ai-agent-rag-websearch', 'noveum.environment': 'development', 'noveum.sdk.version': '0.3.8', 'noveum.sampling.decision': 'record'}, 'metadata': {'user_id': None, 'session_id': None, 'request_id': None, 'tags': {}, 'custom_attributes': {}}, 'created_at': '2025-10-09 08:25:50.304000000', 'updated_at': '2025-10-09 08:25:50.304000000'}\n", - "Spans with unknown type: 12\n", - "returning unknown type for span\n", - "{'span_id': 'b8055ffc-ee7d-46ad-b6a0-84e80f9399e2', 'trace_id': 'dc511122-c0b6-415c-9a49-c7b45132dd87', 'parent_span_id': '', 'name': 'tool-orchestator', 'start_time': '2025-10-09 08:25:38.322000000', 'end_time': '2025-10-09 08:25:42.822000000', 'duration_ms': 4500.335, 'status': 'ok', 'status_message': '', 'attributes': {'input_query': 'What monitoring capabilities does Noveum provide?', 'query.length': 49, 'processing.mode': 'RAG', 'processing.time_seconds': 4.5002899169921875, 'processing.response_length': 839, 'processing.sources_count': 5, 'output_response': 'Final Answer: Noveum provides comprehensive monitoring capabilities for AI applications, including the ability to monitor, trace, and optimize AI agents across various frameworks such as LangChain, CrewAI, AutoGen,...', 'final_answer_mode': 'RAG', 'query_processed.input_query': 'What monitoring capabilities does Noveum provide?', 'query_processed.output_response': 'Successfully processed query using RAG, generated 839 character response', 'query_processed.mode': 'RAG', 'query_processed.processing_time': 4.5002899169921875, 'query_processed.response_length': 839}, 'events': [], 'links': [], 'trace_trace_id': 'dc511122-c0b6-415c-9a49-c7b45132dd87', 'trace_name': 'auto_trace_tool-orchestator', 'project': 'noveum-ai-agent-rag-websearch', 'environment': 'development', 'trace_status': 'ok', 'trace_status_message': '', 'trace_start_time': '2025-10-09 08:25:38.322000000', 'trace_end_time': '2025-10-09 08:25:42.822000000', 'trace_duration_ms': 4500.425, 'span_count': 6, 'error_count': 0, 'sdk': {'name': 'noveum-trace-python', 'version': '0.3.8'}, 'trace_attributes': {'noveum.project': 'noveum-ai-agent-rag-websearch', 'noveum.environment': 'development', 'noveum.sdk.version': '0.3.8', 'noveum.sampling.decision': 'record'}, 'metadata': {'user_id': None, 'session_id': None, 'request_id': None, 'tags': {}, 'custom_attributes': {}}, 'created_at': '2025-10-09 08:25:45.028000000', 'updated_at': '2025-10-09 08:25:45.028000000'}\n", - "Spans with unknown type: 13\n", - "returning unknown type for span\n", - "{'span_id': '532b9e1f-5c4f-4b41-98bb-e9a1510f0f1c', 'trace_id': '04bebf38-a343-4563-80db-0154bef8d927', 'parent_span_id': '', 'name': 'tool-orchestator', 'start_time': '2025-10-09 08:25:33.693000000', 'end_time': '2025-10-09 08:25:37.313000000', 'duration_ms': 3619.5789999999997, 'status': 'ok', 'status_message': '', 'attributes': {'input_query': 'How does Noveum handle agent tracing?', 'query.length': 37, 'processing.mode': 'RAG', 'processing.time_seconds': 3.6195271015167236, 'processing.response_length': 691, 'processing.sources_count': 5, 'output_response': 'Final Answer: Noveum.ai handles agent tracing through its specialized Multi-Agent Tracing capabilities, which are designed to observe complex workflows and inter-agent communications within multi-agent systems. The...', 'final_answer_mode': 'RAG', 'query_processed.input_query': 'How does Noveum handle agent tracing?', 'query_processed.output_response': 'Successfully processed query using RAG, generated 691 character response', 'query_processed.mode': 'RAG', 'query_processed.processing_time': 3.6195271015167236, 'query_processed.response_length': 691}, 'events': [], 'links': [], 'trace_trace_id': '04bebf38-a343-4563-80db-0154bef8d927', 'trace_name': 'auto_trace_tool-orchestator', 'project': 'noveum-ai-agent-rag-websearch', 'environment': 'development', 'trace_status': 'ok', 'trace_status_message': '', 'trace_start_time': '2025-10-09 08:25:33.693000000', 'trace_end_time': '2025-10-09 08:25:37.313000000', 'trace_duration_ms': 3619.672, 'span_count': 6, 'error_count': 0, 'sdk': {'name': 'noveum-trace-python', 'version': '0.3.8'}, 'trace_attributes': {'noveum.project': 'noveum-ai-agent-rag-websearch', 'noveum.environment': 'development', 'noveum.sdk.version': '0.3.8', 'noveum.sampling.decision': 'record'}, 'metadata': {'user_id': None, 'session_id': None, 'request_id': None, 'tags': {}, 'custom_attributes': {}}, 'created_at': '2025-10-09 08:25:40.339000000', 'updated_at': '2025-10-09 08:25:40.339000000'}\n", - "Spans with unknown type: 14\n", - "returning unknown type for span\n", - "{'span_id': '14064e3f-f9c9-4bb7-b8cb-1ad52938b075', 'trace_id': '5e043630-6493-42b5-beb8-79faa19bfa37', 'parent_span_id': '', 'name': 'tool-orchestator', 'start_time': '2025-10-09 08:25:29.901000000', 'end_time': '2025-10-09 08:25:32.685000000', 'duration_ms': 2783.257, 'status': 'ok', 'status_message': '', 'attributes': {'input_query': 'What APIs are available in Noveum platform?', 'query.length': 43, 'processing.mode': 'RAG', 'processing.time_seconds': 2.783203125, 'processing.response_length': 270, 'processing.sources_count': 5, 'output_response': 'Final Answer: The provided context does not specify the APIs available in the Noveum platform. For detailed information about the APIs, I recommend checking the official Noveum documentation or the SDK Integration ...', 'final_answer_mode': 'RAG', 'query_processed.input_query': 'What APIs are available in Noveum platform?', 'query_processed.output_response': 'Successfully processed query using RAG, generated 270 character response', 'query_processed.mode': 'RAG', 'query_processed.processing_time': 2.783203125, 'query_processed.response_length': 270}, 'events': [], 'links': [], 'trace_trace_id': '5e043630-6493-42b5-beb8-79faa19bfa37', 'trace_name': 'auto_trace_tool-orchestator', 'project': 'noveum-ai-agent-rag-websearch', 'environment': 'development', 'trace_status': 'ok', 'trace_status_message': '', 'trace_start_time': '2025-10-09 08:25:29.901000000', 'trace_end_time': '2025-10-09 08:25:32.685000000', 'trace_duration_ms': 2783.326, 'span_count': 6, 'error_count': 0, 'sdk': {'name': 'noveum-trace-python', 'version': '0.3.8'}, 'trace_attributes': {'noveum.project': 'noveum-ai-agent-rag-websearch', 'noveum.environment': 'development', 'noveum.sdk.version': '0.3.8', 'noveum.sampling.decision': 'record'}, 'metadata': {'user_id': None, 'session_id': None, 'request_id': None, 'tags': {}, 'custom_attributes': {}}, 'created_at': '2025-10-09 08:25:34.918000000', 'updated_at': '2025-10-09 08:25:34.918000000'}\n", - "Spans with unknown type: 15\n", - "returning unknown type for span\n", - "{'span_id': 'a74fe8a9-2654-4b9d-ba87-dc2f86b6466d', 'trace_id': '7da9814d-a2e8-4c4e-b750-68b26bd5fd22', 'parent_span_id': '', 'name': 'tool-orchestator', 'start_time': '2025-10-09 08:25:20.586000000', 'end_time': '2025-10-09 08:25:28.897000000', 'duration_ms': 8310.113, 'status': 'ok', 'status_message': '', 'attributes': {'input_query': 'How do I set up observability with Noveum?', 'query.length': 42, 'processing.mode': 'RAG', 'processing.time_seconds': 8.31006908416748, 'processing.response_length': 951, 'processing.sources_count': 5, 'output_response': 'Final Answer: To set up observability with Noveum, follow these steps:\\n\\n1. **Sign Up**: Go to [noveum.ai](https://noveum.ai) and create an account.\\n2. **Create a Project**: Once signed in, create a new project to o...', 'final_answer_mode': 'RAG', 'query_processed.input_query': 'How do I set up observability with Noveum?', 'query_processed.output_response': 'Successfully processed query using RAG, generated 951 character response', 'query_processed.mode': 'RAG', 'query_processed.processing_time': 8.31006908416748, 'query_processed.response_length': 951}, 'events': [], 'links': [], 'trace_trace_id': '7da9814d-a2e8-4c4e-b750-68b26bd5fd22', 'trace_name': 'auto_trace_tool-orchestator', 'project': 'noveum-ai-agent-rag-websearch', 'environment': 'development', 'trace_status': 'ok', 'trace_status_message': '', 'trace_start_time': '2025-10-09 08:25:20.586000000', 'trace_end_time': '2025-10-09 08:25:28.897000000', 'trace_duration_ms': 8310.205, 'span_count': 6, 'error_count': 0, 'sdk': {'name': 'noveum-trace-python', 'version': '0.3.8'}, 'trace_attributes': {'noveum.project': 'noveum-ai-agent-rag-websearch', 'noveum.environment': 'development', 'noveum.sdk.version': '0.3.8', 'noveum.sampling.decision': 'record'}, 'metadata': {'user_id': None, 'session_id': None, 'request_id': None, 'tags': {}, 'custom_attributes': {}}, 'created_at': '2025-10-09 08:25:29.588000000', 'updated_at': '2025-10-09 08:25:29.588000000'}\n", - "Spans with unknown type: 16\n", - "returning unknown type for span\n", - "{'span_id': 'c700d0f4-f1f8-4339-aa42-a3060ae2ae11', 'trace_id': '16143f74-2831-4753-b33d-ce4b645093c5', 'parent_span_id': '', 'name': 'tool-orchestator', 'start_time': '2025-10-09 08:25:12.442000000', 'end_time': '2025-10-09 08:25:19.579000000', 'duration_ms': 7137.314, 'status': 'ok', 'status_message': '', 'attributes': {'input_query': 'What features does Noveum Trace offer?', 'query.length': 38, 'processing.mode': 'RAG', 'processing.time_seconds': 7.1372880935668945, 'processing.response_length': 1014, 'processing.sources_count': 5, 'output_response': 'Final Answer: Noveum Trace offers several key features through its Python SDK (`noveum-trace`) and TypeScript SDK (`@noveum/trace`):\\n\\n1. **Decorator-based Tracing**: It provides a simple way to trace LLM calls, age...', 'final_answer_mode': 'RAG', 'query_processed.input_query': 'What features does Noveum Trace offer?', 'query_processed.output_response': 'Successfully processed query using RAG, generated 1014 character response', 'query_processed.mode': 'RAG', 'query_processed.processing_time': 7.1372880935668945, 'query_processed.response_length': 1014}, 'events': [], 'links': [], 'trace_trace_id': '16143f74-2831-4753-b33d-ce4b645093c5', 'trace_name': 'auto_trace_tool-orchestator', 'project': 'noveum-ai-agent-rag-websearch', 'environment': 'development', 'trace_status': 'ok', 'trace_status_message': '', 'trace_start_time': '2025-10-09 08:25:12.442000000', 'trace_end_time': '2025-10-09 08:25:19.579000000', 'trace_duration_ms': 7137.402, 'span_count': 6, 'error_count': 0, 'sdk': {'name': 'noveum-trace-python', 'version': '0.3.8'}, 'trace_attributes': {'noveum.project': 'noveum-ai-agent-rag-websearch', 'noveum.environment': 'development', 'noveum.sdk.version': '0.3.8', 'noveum.sampling.decision': 'record'}, 'metadata': {'user_id': None, 'session_id': None, 'request_id': None, 'tags': {}, 'custom_attributes': {}}, 'created_at': '2025-10-09 08:25:20.290000000', 'updated_at': '2025-10-09 08:25:20.290000000'}\n", - "Spans with unknown type: 17\n", - "returning unknown type for span\n", - "{'span_id': 'f7f5981f-d840-4265-8970-d319b876366d', 'trace_id': 'fc64e6cc-6739-4256-ac4a-7b80c3028233', 'parent_span_id': '', 'name': 'tool-orchestator', 'start_time': '2025-10-09 08:25:07.932000000', 'end_time': '2025-10-09 08:25:11.438000000', 'duration_ms': 3506.0159999999996, 'status': 'ok', 'status_message': '', 'attributes': {'input_query': \"What are Noveum's pricing plans?\", 'query.length': 32, 'processing.mode': 'RAG', 'processing.time_seconds': 3.5059969425201416, 'processing.response_length': 399, 'processing.sources_count': 5, 'output_response': \"Final Answer: The provided context does not include specific information about Noveum's pricing plans. However, it mentions that all users will have access to the Observability suite for free, and early users will ...\", 'final_answer_mode': 'RAG', 'query_processed.input_query': \"What are Noveum's pricing plans?\", 'query_processed.output_response': 'Successfully processed query using RAG, generated 399 character response', 'query_processed.mode': 'RAG', 'query_processed.processing_time': 3.5059969425201416, 'query_processed.response_length': 399}, 'events': [], 'links': [], 'trace_trace_id': 'fc64e6cc-6739-4256-ac4a-7b80c3028233', 'trace_name': 'auto_trace_tool-orchestator', 'project': 'noveum-ai-agent-rag-websearch', 'environment': 'development', 'trace_status': 'ok', 'trace_status_message': '', 'trace_start_time': '2025-10-09 08:25:07.932000000', 'trace_end_time': '2025-10-09 08:25:11.438000000', 'trace_duration_ms': 3506.0989999999997, 'span_count': 6, 'error_count': 0, 'sdk': {'name': 'noveum-trace-python', 'version': '0.3.8'}, 'trace_attributes': {'noveum.project': 'noveum-ai-agent-rag-websearch', 'noveum.environment': 'development', 'noveum.sdk.version': '0.3.8', 'noveum.sampling.decision': 'record'}, 'metadata': {'user_id': None, 'session_id': None, 'request_id': None, 'tags': {}, 'custom_attributes': {}}, 'created_at': '2025-10-09 08:25:12.651000000', 'updated_at': '2025-10-09 08:25:12.651000000'}\n", - "Spans with unknown type: 18\n", - "returning unknown type for span\n", - "{'span_id': '0c7bbe2b-18af-4fdb-a5bf-4a91daa3bf50', 'trace_id': 'b7945c49-f584-4c70-972d-536a805d8a31', 'parent_span_id': '', 'name': 'tool-orchestator', 'start_time': '2025-10-09 08:24:58.518000000', 'end_time': '2025-10-09 08:25:06.920000000', 'duration_ms': 8402.378, 'status': 'ok', 'status_message': '', 'attributes': {'input_query': 'How do I integrate Noveum Trace in my application?', 'query.length': 50, 'processing.mode': 'RAG', 'processing.time_seconds': 8.402259826660156, 'processing.response_length': 1151, 'processing.sources_count': 5, 'output_response': 'Final Answer: To integrate Noveum Trace into your application, you can use either the Python SDK (`noveum-trace`) or the TypeScript SDK (`@noveum/trace`). Hereā€™s a quick start guide based on the provided documentat...', 'final_answer_mode': 'RAG', 'query_processed.input_query': 'How do I integrate Noveum Trace in my application?', 'query_processed.output_response': 'Successfully processed query using RAG, generated 1151 character response', 'query_processed.mode': 'RAG', 'query_processed.processing_time': 8.402259826660156, 'query_processed.response_length': 1151}, 'events': [], 'links': [], 'trace_trace_id': 'b7945c49-f584-4c70-972d-536a805d8a31', 'trace_name': 'auto_trace_tool-orchestator', 'project': 'noveum-ai-agent-rag-websearch', 'environment': 'development', 'trace_status': 'ok', 'trace_status_message': '', 'trace_start_time': '2025-10-09 08:24:58.518000000', 'trace_end_time': '2025-10-09 08:25:06.920000000', 'trace_duration_ms': 8402.516, 'span_count': 6, 'error_count': 0, 'sdk': {'name': 'noveum-trace-python', 'version': '0.3.8'}, 'trace_attributes': {'noveum.project': 'noveum-ai-agent-rag-websearch', 'noveum.environment': 'development', 'noveum.sdk.version': '0.3.8', 'noveum.sampling.decision': 'record'}, 'metadata': {'user_id': None, 'session_id': None, 'request_id': None, 'tags': {}, 'custom_attributes': {}}, 'created_at': '2025-10-09 08:25:07.620000000', 'updated_at': '2025-10-09 08:25:07.620000000'}\n", - "Spans with unknown type: 19\n", - "returning unknown type for span\n", - "{'span_id': 'dc869742-6797-4016-8061-37530fc7d433', 'trace_id': 'f5c40ecf-36c0-45ba-9cc9-dc0329b0324b', 'parent_span_id': '', 'name': 'tool-orchestator', 'start_time': '2025-10-09 08:24:54.009000000', 'end_time': '2025-10-09 08:24:57.511000000', 'duration_ms': 3502.0229999999997, 'status': 'ok', 'status_message': '', 'attributes': {'input_query': 'What is Noveum and what does it do?', 'query.length': 35, 'processing.mode': 'RAG', 'processing.time_seconds': 3.5019419193267822, 'processing.response_length': 611, 'processing.sources_count': 5, 'output_response': 'Final Answer: Noveum.ai is a comprehensive tracing and observability platform specifically designed for AI applications, including those powered by large language models (LLMs), retrieval-augmented generation (RAG)...', 'final_answer_mode': 'RAG', 'query_processed.input_query': 'What is Noveum and what does it do?', 'query_processed.output_response': 'Successfully processed query using RAG, generated 611 character response', 'query_processed.mode': 'RAG', 'query_processed.processing_time': 3.5019419193267822, 'query_processed.response_length': 611}, 'events': [], 'links': [], 'trace_trace_id': 'f5c40ecf-36c0-45ba-9cc9-dc0329b0324b', 'trace_name': 'auto_trace_tool-orchestator', 'project': 'noveum-ai-agent-rag-websearch', 'environment': 'development', 'trace_status': 'ok', 'trace_status_message': '', 'trace_start_time': '2025-10-09 08:24:54.009000000', 'trace_end_time': '2025-10-09 08:24:57.511000000', 'trace_duration_ms': 3502.062, 'span_count': 6, 'error_count': 0, 'sdk': {'name': 'noveum-trace-python', 'version': '0.3.8'}, 'trace_attributes': {'noveum.project': 'noveum-ai-agent-rag-websearch', 'noveum.environment': 'development', 'noveum.sdk.version': '0.3.8', 'noveum.sampling.decision': 'record'}, 'metadata': {'user_id': None, 'session_id': None, 'request_id': None, 'tags': {}, 'custom_attributes': {}}, 'created_at': '2025-10-09 08:24:59.716000000', 'updated_at': '2025-10-09 08:24:59.716000000'}\n", - "Spans with unknown type: 20\n", - "\n", - "āœ… Successfully converted 20 spans to AgentData\n", - "šŸ“Š AgentDataset created with 20 records\n", - "āœ… AgentDataset created: 20 records\n", - "\n", - "šŸ“‹ Step 4: Dataset Analysis\n", - "šŸ“ˆ Dataset Statistics:\n", - "\n", - "Agent Types: {'unknown': 20}\n", - "Records with responses: 0\n", - "Records with tool calls: 0\n", - "Records with retrieval: 0\n", - "Tool usage: {}\n", - "šŸ” Dataset Analysis:\n", - "\n", - "=== Agent Behavior Patterns ===\n", - "\n", - "šŸ“ˆ Tool Usage:\n", - "\n", - "šŸ“‹ Task Types:\n", - "\n", - "šŸ“‹ Step 5: Setting up Evaluation\n", - "āœ… GEMINI_API_KEY found in environment\n", - "2025-10-22 23:01:12 - INFO - novaeval.models.base - Noveum tracing initialized successfully\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "āœ… Gemini model initialized\n", - "āœ… Initialized 5 scoring functions:\n", - " - task_progression_scorer\n", - " - context_relevancy_scorer\n", - " - role_adherence_scorer\n", - " - tool_relevancy_scorer\n", - " - parameter_correctness_scorer\n", - "\n", - "āœ… AgentEvaluator created with Gemini model and scoring functions\n", - "āœ… Evaluation components ready!\n", - "\n", - "šŸ“‹ Step 6: Running Evaluation\n", - "šŸŽÆ Evaluating 25 samples...\n", - "šŸš€ Running evaluation on sample data...\n", - "\n", - "šŸ“Š Evaluating 0 sample records...\n", - "2025-10-22 23:01:12 - INFO - novaeval.evaluators.agent_evaluator - Starting agent evaluation process\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Evaluating samples: 0it [00:00, ?it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:01:12 - INFO - novaeval.evaluators.agent_evaluator - Saving final results\n", - "2025-10-22 23:01:12 - INFO - novaeval.evaluators.agent_evaluator - Agent evaluation completed\n", - "\n", - "āœ… Evaluation completed!\n", - "āŒ Results file not found\n", - "āŒ Evaluation failed\n", - "\n", - "šŸ“‹ Step 7: Exporting Dataset\n", - "šŸ’¾ Exporting processed dataset...\n", - "āœ… Exported to ./processed_datasets/tool-orchestator_dataset_processed_dataset.json\n", - "āœ… Exported to ./processed_datasets/tool-orchestator_dataset_processed_dataset.csv\n", - "āœ… Export completed successfully!\n", - "\n", - "============================================================\n", - "šŸŽ‰ EVALUATION PIPELINE COMPLETED!\n", - "šŸ“Š Final Results:\n", - " - File processed: split_datasets/tool-orchestator_dataset.json\n", - " - Spans loaded: 20\n", - " - Dataset size: 20\n", - " - Evaluation completed: False\n", - " - Export successful: True\n", - " - Errors encountered: 1\n", - "Completed tool-orchestator_dataset.json\n", - "\n", - "Processing agent.web_search_generation_dataset.json...\n", - "šŸš€ Starting Complete Agent Evaluation Pipeline\n", - "šŸ“ Processing file: split_datasets/agent.web_search_generation_dataset.json\n", - "============================================================\n", - "\n", - "šŸ“‹ Step 1: Environment Setup\n", - "āœ… Logging configured at INFO level\n", - "šŸ” Environment validation:\n", - " āœ… gemini_api_key: True\n", - " āœ… pandas_available: True\n", - " āœ… novaeval_available: True\n", - "āœ… Environment ready for evaluation!\n", - "\n", - "šŸ“‹ Step 2: Loading Dataset\n", - "šŸ“Š Loaded 8 spans from split_datasets/agent.web_search_generation_dataset.json\n", - "\n", - "šŸ” Available span types:\n", - " - agent.web_search_generation: 8\n", - "āœ… Dataset loaded: 8 spans\n", - "\n", - "šŸ“‹ Step 3: Converting to AgentDataset Format\n", - "šŸ”„ Converting spans to AgentData objects...\n", - "agent_task not found\n", - "agent_response is not available 2aca614b-b016-4d56-abe2-43a144dc681d\n", - "agent_task not found\n", - "agent_response is not available 03e3ae4d-0107-401d-8ca0-05fc0f28fb24\n", - "agent_task not found\n", - "agent_response is not available dfdb9aaf-9925-4a5c-a0c6-2cb00f70d4d3\n", - "agent_task not found\n", - "agent_response is not available eefcfdfc-6433-459e-a134-e2647b8418e4\n", - "agent_task not found\n", - "agent_response is not available c57b4b54-b9b0-4a4c-b5cd-57a4cfe5301b\n", - "agent_task not found\n", - "agent_response is not available a062bef6-d973-489b-91df-5d40a2aef077\n", - "agent_task not found\n", - "agent_response is not available 41f82a95-620d-440c-84e6-36d6d1b9e079\n", - "agent_task not found\n", - "agent_response is not available 63f2a0d5-9016-4483-8ec0-e6aa6b27689f\n", - "\n", - "āœ… Successfully converted 8 spans to AgentData\n", - "šŸ“Š AgentDataset created with 8 records\n", - "āœ… AgentDataset created: 8 records\n", - "\n", - "šŸ“‹ Step 4: Dataset Analysis\n", - "šŸ“ˆ Dataset Statistics:\n", - "\n", - "Agent Types: {'agent': 8}\n", - "Records with responses: 0\n", - "Records with tool calls: 0\n", - "Records with retrieval: 0\n", - "Tool usage: {}\n", - "šŸ” Dataset Analysis:\n", - "\n", - "=== Agent Behavior Patterns ===\n", - "\n", - "šŸ“ˆ Tool Usage:\n", - "\n", - "šŸ“‹ Task Types:\n", - "\n", - "šŸ“‹ Step 5: Setting up Evaluation\n", - "āœ… GEMINI_API_KEY found in environment\n", - "2025-10-22 23:01:12 - INFO - novaeval.models.base - Noveum tracing initialized successfully\n", - "āœ… Gemini model initialized\n", - "āœ… Initialized 5 scoring functions:\n", - " - task_progression_scorer\n", - " - context_relevancy_scorer\n", - " - role_adherence_scorer\n", - " - tool_relevancy_scorer\n", - " - parameter_correctness_scorer\n", - "\n", - "āœ… AgentEvaluator created with Gemini model and scoring functions\n", - "āœ… Evaluation components ready!\n", - "\n", - "šŸ“‹ Step 6: Running Evaluation\n", - "šŸŽÆ Evaluating 25 samples...\n", - "šŸš€ Running evaluation on sample data...\n", - "\n", - "šŸ“Š Evaluating 0 sample records...\n", - "2025-10-22 23:01:12 - INFO - novaeval.evaluators.agent_evaluator - Starting agent evaluation process\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\n", - "Evaluating samples: 0it [00:00, ?it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:01:12 - INFO - novaeval.evaluators.agent_evaluator - Saving final results\n", - "2025-10-22 23:01:12 - INFO - novaeval.evaluators.agent_evaluator - Agent evaluation completed\n", - "\n", - "āœ… Evaluation completed!\n", - "āŒ Results file not found\n", - "āŒ Evaluation failed\n", - "\n", - "šŸ“‹ Step 7: Exporting Dataset\n", - "šŸ’¾ Exporting processed dataset...\n", - "āœ… Exported to ./processed_datasets/agent.web_search_generation_dataset_processed_dataset.json\n", - "āœ… Exported to ./processed_datasets/agent.web_search_generation_dataset_processed_dataset.csv\n", - "āœ… Export completed successfully!\n", - "\n", - "============================================================\n", - "šŸŽ‰ EVALUATION PIPELINE COMPLETED!\n", - "šŸ“Š Final Results:\n", - " - File processed: split_datasets/agent.web_search_generation_dataset.json\n", - " - Spans loaded: 8\n", - " - Dataset size: 8\n", - " - Evaluation completed: False\n", - " - Export successful: True\n", - " - Errors encountered: 1\n", - "Completed agent.web_search_generation_dataset.json\n", - "\n", - "Processing agent.rag_evaluation_metrics_dataset.json...\n", - "šŸš€ Starting Complete Agent Evaluation Pipeline\n", - "šŸ“ Processing file: split_datasets/agent.rag_evaluation_metrics_dataset.json\n", - "============================================================\n", - "\n", - "šŸ“‹ Step 1: Environment Setup\n", - "āœ… Logging configured at INFO level\n", - "šŸ” Environment validation:\n", - " āœ… gemini_api_key: True\n", - " āœ… pandas_available: True\n", - " āœ… novaeval_available: True\n", - "āœ… Environment ready for evaluation!\n", - "\n", - "šŸ“‹ Step 2: Loading Dataset\n", - "šŸ“Š Loaded 12 spans from split_datasets/agent.rag_evaluation_metrics_dataset.json\n", - "\n", - "šŸ” Available span types:\n", - " - agent.rag_evaluation_metrics: 12\n", - "āœ… Dataset loaded: 12 spans\n", - "\n", - "šŸ“‹ Step 3: Converting to AgentDataset Format\n", - "šŸ”„ Converting spans to AgentData objects...\n", - "\n", - "āœ… Successfully converted 12 spans to AgentData\n", - "šŸ“Š AgentDataset created with 12 records\n", - "āœ… AgentDataset created: 12 records\n", - "\n", - "šŸ“‹ Step 4: Dataset Analysis\n", - "šŸ“ˆ Dataset Statistics:\n", - "\n", - "Agent Types: {'agent': 12}\n", - "Records with responses: 12\n", - "Records with tool calls: 0\n", - "Records with retrieval: 12\n", - "Tool usage: {}\n", - "šŸ” Dataset Analysis:\n", - "\n", - "=== Agent Behavior Patterns ===\n", - "\n", - "šŸ“ˆ Tool Usage:\n", - "\n", - "šŸ“‹ Task Types:\n", - " - other: 12\n", - "\n", - "šŸ“ Response Statistics:\n", - " - Average response length: 215.0 characters\n", - " - Min response length: 215\n", - " - Max response length: 215\n", - "\n", - "šŸ“‹ Step 5: Setting up Evaluation\n", - "āœ… GEMINI_API_KEY found in environment\n", - "2025-10-22 23:01:12 - INFO - novaeval.models.base - Noveum tracing initialized successfully\n", - "āœ… Gemini model initialized\n", - "āœ… Initialized 5 scoring functions:\n", - " - task_progression_scorer\n", - " - context_relevancy_scorer\n", - " - role_adherence_scorer\n", - " - tool_relevancy_scorer\n", - " - parameter_correctness_scorer\n", - "\n", - "āœ… AgentEvaluator created with Gemini model and scoring functions\n", - "āœ… Evaluation components ready!\n", - "\n", - "šŸ“‹ Step 6: Running Evaluation\n", - "šŸŽÆ Evaluating 25 samples...\n", - "šŸš€ Running evaluation on sample data...\n", - "\n", - "šŸ“Š Evaluating 12 sample records...\n", - "2025-10-22 23:01:12 - INFO - novaeval.evaluators.agent_evaluator - Starting agent evaluation process\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\n", - "Evaluating samples: 0it [00:00, ?it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:01:12 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-10-22 23:01:12 - noveum_trace.transport.http_transport - INFO - šŸ“” HTTP RESPONSE: Status 200 from https://api.noveum.ai/api/v1/traces\n", - "2025-10-22 23:01:12 - noveum_trace.transport.http_transport - INFO - āœ… Successfully sent batch of 4 traces\n", - "2025-10-22 23:01:12 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully sent batch of 4 traces via callback\n", - "2025-10-22 23:01:13 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: eb775ca5-3cc0-47f9-b26b-0e3e770dadd5) - 1 spans\n", - "2025-10-22 23:01:13 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: eb775ca5-3cc0-47f9-b26b-0e3e770dadd5) - 1 spans\n", - "2025-10-22 23:01:13 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace eb775ca5-3cc0-47f9-b26b-0e3e770dadd5\n", - "2025-10-22 23:01:13 - noveum_trace.transport.http_transport - INFO - āœ… Trace eb775ca5-3cc0-47f9-b26b-0e3e770dadd5 successfully queued for export\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:01:13 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-10-22 23:01:16 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: cce316e9-4f3f-4d2b-bdba-b2856b9c4181) - 1 spans\n", - "2025-10-22 23:01:16 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: cce316e9-4f3f-4d2b-bdba-b2856b9c4181) - 1 spans\n", - "2025-10-22 23:01:16 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace cce316e9-4f3f-4d2b-bdba-b2856b9c4181\n", - "2025-10-22 23:01:16 - noveum_trace.transport.http_transport - INFO - āœ… Trace cce316e9-4f3f-4d2b-bdba-b2856b9c4181 successfully queued for export\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:01:16 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-10-22 23:01:17 - noveum_trace.transport.batch_processor - INFO - ā° TIMEOUT TRIGGER: Sending batch due to timeout (5.3s >= 5.0s)\n", - "2025-10-22 23:01:17 - noveum_trace.transport.batch_processor - INFO - šŸ“¤ SENDING BATCH: 3 traces via send_callback\n", - "2025-10-22 23:01:17 - noveum_trace.transport.http_transport - INFO - šŸš€ SENDING BATCH: 3 traces to https://api.noveum.ai/api/v1/traces\n", - "2025-10-22 23:01:17 - noveum_trace.transport.http_transport - INFO - šŸ“” HTTP RESPONSE: Status 200 from https://api.noveum.ai/api/v1/traces\n", - "2025-10-22 23:01:17 - noveum_trace.transport.http_transport - INFO - āœ… Successfully sent batch of 3 traces\n", - "2025-10-22 23:01:17 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully sent batch of 3 traces via callback\n", - "2025-10-22 23:01:18 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: ef7447dd-1534-4496-9aff-02d51839cd0c) - 1 spans\n", - "2025-10-22 23:01:18 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: ef7447dd-1534-4496-9aff-02d51839cd0c) - 1 spans\n", - "2025-10-22 23:01:18 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace ef7447dd-1534-4496-9aff-02d51839cd0c\n", - "2025-10-22 23:01:18 - noveum_trace.transport.http_transport - INFO - āœ… Trace ef7447dd-1534-4496-9aff-02d51839cd0c successfully queued for export\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:01:18 - INFO - novaeval.evaluators.agent_evaluator - Saving intermediate results after 1 samples\n", - "2025-10-22 23:01:18 - INFO - novaeval.evaluators.agent_evaluator - Intermediate results saved to demo_results/agent.rag_evaluation_metrics_dataset/agent_evaluation_results.csv\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Evaluating samples: 1it [00:06, 6.46s/it]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:01:18 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-10-22 23:01:20 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: d32ecc4b-d331-4e62-9ef2-ec61e6fc52bf) - 1 spans\n", - "2025-10-22 23:01:20 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: d32ecc4b-d331-4e62-9ef2-ec61e6fc52bf) - 1 spans\n", - "2025-10-22 23:01:20 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace d32ecc4b-d331-4e62-9ef2-ec61e6fc52bf\n", - "2025-10-22 23:01:20 - noveum_trace.transport.http_transport - INFO - āœ… Trace d32ecc4b-d331-4e62-9ef2-ec61e6fc52bf successfully queued for export\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:01:20 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-10-22 23:01:22 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 37646e1d-3bea-43ae-af16-71bf5ba46b9c) - 1 spans\n", - "2025-10-22 23:01:22 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 37646e1d-3bea-43ae-af16-71bf5ba46b9c) - 1 spans\n", - "2025-10-22 23:01:22 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 37646e1d-3bea-43ae-af16-71bf5ba46b9c\n", - "2025-10-22 23:01:22 - noveum_trace.transport.http_transport - INFO - āœ… Trace 37646e1d-3bea-43ae-af16-71bf5ba46b9c successfully queued for export\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:01:22 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-10-22 23:01:22 - noveum_trace.transport.batch_processor - INFO - ā° TIMEOUT TRIGGER: Sending batch due to timeout (5.2s >= 5.0s)\n", - "2025-10-22 23:01:22 - noveum_trace.transport.batch_processor - INFO - šŸ“¤ SENDING BATCH: 3 traces via send_callback\n", - "2025-10-22 23:01:22 - noveum_trace.transport.http_transport - INFO - šŸš€ SENDING BATCH: 3 traces to https://api.noveum.ai/api/v1/traces\n", - "2025-10-22 23:01:22 - noveum_trace.transport.http_transport - INFO - šŸ“” HTTP RESPONSE: Status 200 from https://api.noveum.ai/api/v1/traces\n", - "2025-10-22 23:01:22 - noveum_trace.transport.http_transport - INFO - āœ… Successfully sent batch of 3 traces\n", - "2025-10-22 23:01:22 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully sent batch of 3 traces via callback\n", - "2025-10-22 23:01:23 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 309e9e0e-8ee6-46e4-86c2-60922d034adb) - 1 spans\n", - "2025-10-22 23:01:23 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 309e9e0e-8ee6-46e4-86c2-60922d034adb) - 1 spans\n", - "2025-10-22 23:01:23 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 309e9e0e-8ee6-46e4-86c2-60922d034adb\n", - "2025-10-22 23:01:23 - noveum_trace.transport.http_transport - INFO - āœ… Trace 309e9e0e-8ee6-46e4-86c2-60922d034adb successfully queued for export\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:01:23 - INFO - novaeval.evaluators.agent_evaluator - Saving intermediate results after 2 samples\n", - "2025-10-22 23:01:23 - INFO - novaeval.evaluators.agent_evaluator - Intermediate results saved to demo_results/agent.rag_evaluation_metrics_dataset/agent_evaluation_results.csv\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Evaluating samples: 2it [00:10, 5.24s/it]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:01:23 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-10-22 23:01:24 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 7876eb8f-cae3-442a-b676-588d0e06a56b) - 1 spans\n", - "2025-10-22 23:01:24 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 7876eb8f-cae3-442a-b676-588d0e06a56b) - 1 spans\n", - "2025-10-22 23:01:24 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 7876eb8f-cae3-442a-b676-588d0e06a56b\n", - "2025-10-22 23:01:24 - noveum_trace.transport.http_transport - INFO - āœ… Trace 7876eb8f-cae3-442a-b676-588d0e06a56b successfully queued for export\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:01:24 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-10-22 23:01:26 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: a1a2f4ae-eaf9-4742-94d2-b2b650faf225) - 1 spans\n", - "2025-10-22 23:01:26 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: a1a2f4ae-eaf9-4742-94d2-b2b650faf225) - 1 spans\n", - "2025-10-22 23:01:26 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace a1a2f4ae-eaf9-4742-94d2-b2b650faf225\n", - "2025-10-22 23:01:26 - noveum_trace.transport.http_transport - INFO - āœ… Trace a1a2f4ae-eaf9-4742-94d2-b2b650faf225 successfully queued for export\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:01:26 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-10-22 23:01:27 - noveum_trace.transport.batch_processor - INFO - ā° TIMEOUT TRIGGER: Sending batch due to timeout (5.1s >= 5.0s)\n", - "2025-10-22 23:01:27 - noveum_trace.transport.batch_processor - INFO - šŸ“¤ SENDING BATCH: 3 traces via send_callback\n", - "2025-10-22 23:01:27 - noveum_trace.transport.http_transport - INFO - šŸš€ SENDING BATCH: 3 traces to https://api.noveum.ai/api/v1/traces\n", - "2025-10-22 23:01:27 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 7fd55fac-bb86-4db1-9e20-e5ce69a772fb) - 1 spans\n", - "2025-10-22 23:01:27 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 7fd55fac-bb86-4db1-9e20-e5ce69a772fb) - 1 spans\n", - "2025-10-22 23:01:27 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 7fd55fac-bb86-4db1-9e20-e5ce69a772fb\n", - "2025-10-22 23:01:27 - noveum_trace.transport.http_transport - INFO - āœ… Trace 7fd55fac-bb86-4db1-9e20-e5ce69a772fb successfully queued for export\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:01:27 - INFO - novaeval.evaluators.agent_evaluator - Saving intermediate results after 3 samples\n", - "2025-10-22 23:01:27 - INFO - novaeval.evaluators.agent_evaluator - Intermediate results saved to demo_results/agent.rag_evaluation_metrics_dataset/agent_evaluation_results.csv\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Evaluating samples: 3it [00:15, 4.93s/it]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:01:27 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-10-22 23:01:28 - noveum_trace.transport.http_transport - INFO - šŸ“” HTTP RESPONSE: Status 200 from https://api.noveum.ai/api/v1/traces\n", - "2025-10-22 23:01:28 - noveum_trace.transport.http_transport - INFO - āœ… Successfully sent batch of 3 traces\n", - "2025-10-22 23:01:28 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully sent batch of 3 traces via callback\n", - "2025-10-22 23:01:28 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 3e3b5c0c-04e1-4bf5-8ea2-c70387c8e327) - 1 spans\n", - "2025-10-22 23:01:28 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 3e3b5c0c-04e1-4bf5-8ea2-c70387c8e327) - 1 spans\n", - "2025-10-22 23:01:28 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 3e3b5c0c-04e1-4bf5-8ea2-c70387c8e327\n", - "2025-10-22 23:01:28 - noveum_trace.transport.http_transport - INFO - āœ… Trace 3e3b5c0c-04e1-4bf5-8ea2-c70387c8e327 successfully queued for export\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:01:28 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-10-22 23:01:30 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 3e9b223d-42c3-4f70-8541-e8b7c27201fc) - 1 spans\n", - "2025-10-22 23:01:30 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 3e9b223d-42c3-4f70-8541-e8b7c27201fc) - 1 spans\n", - "2025-10-22 23:01:30 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 3e9b223d-42c3-4f70-8541-e8b7c27201fc\n", - "2025-10-22 23:01:30 - noveum_trace.transport.http_transport - INFO - āœ… Trace 3e9b223d-42c3-4f70-8541-e8b7c27201fc successfully queued for export\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:01:30 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-10-22 23:01:31 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: f822ee4c-33fa-4a46-8a1c-a9fd7a749983) - 1 spans\n", - "2025-10-22 23:01:31 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: f822ee4c-33fa-4a46-8a1c-a9fd7a749983) - 1 spans\n", - "2025-10-22 23:01:31 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace f822ee4c-33fa-4a46-8a1c-a9fd7a749983\n", - "2025-10-22 23:01:31 - noveum_trace.transport.http_transport - INFO - āœ… Trace f822ee4c-33fa-4a46-8a1c-a9fd7a749983 successfully queued for export\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:01:31 - INFO - novaeval.evaluators.agent_evaluator - Saving intermediate results after 4 samples\n", - "2025-10-22 23:01:31 - INFO - novaeval.evaluators.agent_evaluator - Intermediate results saved to demo_results/agent.rag_evaluation_metrics_dataset/agent_evaluation_results.csv\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Evaluating samples: 4it [00:18, 4.36s/it]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:01:31 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-10-22 23:01:32 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: b9ec0566-acc2-4c49-8fa4-fa58362b7056) - 1 spans\n", - "2025-10-22 23:01:32 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: b9ec0566-acc2-4c49-8fa4-fa58362b7056) - 1 spans\n", - "2025-10-22 23:01:32 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace b9ec0566-acc2-4c49-8fa4-fa58362b7056\n", - "2025-10-22 23:01:32 - noveum_trace.transport.http_transport - INFO - āœ… Trace b9ec0566-acc2-4c49-8fa4-fa58362b7056 successfully queued for export\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:01:32 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-10-22 23:01:32 - noveum_trace.transport.batch_processor - INFO - ā° TIMEOUT TRIGGER: Sending batch due to timeout (5.3s >= 5.0s)\n", - "2025-10-22 23:01:32 - noveum_trace.transport.batch_processor - INFO - šŸ“¤ SENDING BATCH: 5 traces via send_callback\n", - "2025-10-22 23:01:32 - noveum_trace.transport.http_transport - INFO - šŸš€ SENDING BATCH: 5 traces to https://api.noveum.ai/api/v1/traces\n", - "2025-10-22 23:01:33 - noveum_trace.transport.http_transport - INFO - šŸ“” HTTP RESPONSE: Status 200 from https://api.noveum.ai/api/v1/traces\n", - "2025-10-22 23:01:33 - noveum_trace.transport.http_transport - INFO - āœ… Successfully sent batch of 5 traces\n", - "2025-10-22 23:01:33 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully sent batch of 5 traces via callback\n", - "2025-10-22 23:01:34 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 4a7712cd-d5db-43d4-b9f9-3bf2854fd36d) - 1 spans\n", - "2025-10-22 23:01:34 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 4a7712cd-d5db-43d4-b9f9-3bf2854fd36d) - 1 spans\n", - "2025-10-22 23:01:34 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 4a7712cd-d5db-43d4-b9f9-3bf2854fd36d\n", - "2025-10-22 23:01:34 - noveum_trace.transport.http_transport - INFO - āœ… Trace 4a7712cd-d5db-43d4-b9f9-3bf2854fd36d successfully queued for export\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:01:34 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-10-22 23:01:35 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: d3a4f8d6-9ac8-4312-982f-ed0ba66a9792) - 1 spans\n", - "2025-10-22 23:01:35 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: d3a4f8d6-9ac8-4312-982f-ed0ba66a9792) - 1 spans\n", - "2025-10-22 23:01:35 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace d3a4f8d6-9ac8-4312-982f-ed0ba66a9792\n", - "2025-10-22 23:01:35 - noveum_trace.transport.http_transport - INFO - āœ… Trace d3a4f8d6-9ac8-4312-982f-ed0ba66a9792 successfully queued for export\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:01:35 - INFO - novaeval.evaluators.agent_evaluator - Saving intermediate results after 5 samples\n", - "2025-10-22 23:01:35 - INFO - novaeval.evaluators.agent_evaluator - Intermediate results saved to demo_results/agent.rag_evaluation_metrics_dataset/agent_evaluation_results.csv\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Evaluating samples: 5it [00:22, 4.25s/it]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:01:35 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-10-22 23:01:36 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 089525d7-5531-478e-b4f3-fec4d1ac533d) - 1 spans\n", - "2025-10-22 23:01:36 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 089525d7-5531-478e-b4f3-fec4d1ac533d) - 1 spans\n", - "2025-10-22 23:01:36 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 089525d7-5531-478e-b4f3-fec4d1ac533d\n", - "2025-10-22 23:01:36 - noveum_trace.transport.http_transport - INFO - āœ… Trace 089525d7-5531-478e-b4f3-fec4d1ac533d successfully queued for export\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:01:36 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-10-22 23:01:37 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 2214e8fb-a31c-4111-8735-0c6c8057a66c) - 1 spans\n", - "2025-10-22 23:01:37 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 2214e8fb-a31c-4111-8735-0c6c8057a66c) - 1 spans\n", - "2025-10-22 23:01:37 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 2214e8fb-a31c-4111-8735-0c6c8057a66c\n", - "2025-10-22 23:01:37 - noveum_trace.transport.http_transport - INFO - āœ… Trace 2214e8fb-a31c-4111-8735-0c6c8057a66c successfully queued for export\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:01:37 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-10-22 23:01:38 - noveum_trace.transport.batch_processor - INFO - ā° TIMEOUT TRIGGER: Sending batch due to timeout (5.3s >= 5.0s)\n", - "2025-10-22 23:01:38 - noveum_trace.transport.batch_processor - INFO - šŸ“¤ SENDING BATCH: 4 traces via send_callback\n", - "2025-10-22 23:01:38 - noveum_trace.transport.http_transport - INFO - šŸš€ SENDING BATCH: 4 traces to https://api.noveum.ai/api/v1/traces\n", - "2025-10-22 23:01:38 - noveum_trace.transport.http_transport - INFO - šŸ“” HTTP RESPONSE: Status 200 from https://api.noveum.ai/api/v1/traces\n", - "2025-10-22 23:01:38 - noveum_trace.transport.http_transport - INFO - āœ… Successfully sent batch of 4 traces\n", - "2025-10-22 23:01:38 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully sent batch of 4 traces via callback\n", - "2025-10-22 23:01:38 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 4d917e77-4f61-46e5-8aed-f705e23cf42b) - 1 spans\n", - "2025-10-22 23:01:38 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 4d917e77-4f61-46e5-8aed-f705e23cf42b) - 1 spans\n", - "2025-10-22 23:01:38 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 4d917e77-4f61-46e5-8aed-f705e23cf42b\n", - "2025-10-22 23:01:38 - noveum_trace.transport.http_transport - INFO - āœ… Trace 4d917e77-4f61-46e5-8aed-f705e23cf42b successfully queued for export\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:01:38 - INFO - novaeval.evaluators.agent_evaluator - Saving intermediate results after 6 samples\n", - "2025-10-22 23:01:38 - INFO - novaeval.evaluators.agent_evaluator - Intermediate results saved to demo_results/agent.rag_evaluation_metrics_dataset/agent_evaluation_results.csv\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Evaluating samples: 6it [00:26, 4.03s/it]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:01:38 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-10-22 23:01:40 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: f08a5cf5-f699-499a-bf6f-7c5ab49bee74) - 1 spans\n", - "2025-10-22 23:01:40 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: f08a5cf5-f699-499a-bf6f-7c5ab49bee74) - 1 spans\n", - "2025-10-22 23:01:40 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace f08a5cf5-f699-499a-bf6f-7c5ab49bee74\n", - "2025-10-22 23:01:40 - noveum_trace.transport.http_transport - INFO - āœ… Trace f08a5cf5-f699-499a-bf6f-7c5ab49bee74 successfully queued for export\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:01:40 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-10-22 23:01:41 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 22fad5c7-5c05-46dc-be41-410da97ae840) - 1 spans\n", - "2025-10-22 23:01:41 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 22fad5c7-5c05-46dc-be41-410da97ae840) - 1 spans\n", - "2025-10-22 23:01:41 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 22fad5c7-5c05-46dc-be41-410da97ae840\n", - "2025-10-22 23:01:41 - noveum_trace.transport.http_transport - INFO - āœ… Trace 22fad5c7-5c05-46dc-be41-410da97ae840 successfully queued for export\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:01:41 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-10-22 23:01:42 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 81742ed5-9524-4203-aaf6-837045f5d1ce) - 1 spans\n", - "2025-10-22 23:01:42 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 81742ed5-9524-4203-aaf6-837045f5d1ce) - 1 spans\n", - "2025-10-22 23:01:42 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 81742ed5-9524-4203-aaf6-837045f5d1ce\n", - "2025-10-22 23:01:42 - noveum_trace.transport.http_transport - INFO - āœ… Trace 81742ed5-9524-4203-aaf6-837045f5d1ce successfully queued for export\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:01:42 - INFO - novaeval.evaluators.agent_evaluator - Saving intermediate results after 7 samples\n", - "2025-10-22 23:01:42 - INFO - novaeval.evaluators.agent_evaluator - Intermediate results saved to demo_results/agent.rag_evaluation_metrics_dataset/agent_evaluation_results.csv\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Evaluating samples: 7it [00:30, 3.93s/it]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:01:42 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-10-22 23:01:43 - noveum_trace.transport.batch_processor - INFO - ā° TIMEOUT TRIGGER: Sending batch due to timeout (5.4s >= 5.0s)\n", - "2025-10-22 23:01:43 - noveum_trace.transport.batch_processor - INFO - šŸ“¤ SENDING BATCH: 4 traces via send_callback\n", - "2025-10-22 23:01:43 - noveum_trace.transport.http_transport - INFO - šŸš€ SENDING BATCH: 4 traces to https://api.noveum.ai/api/v1/traces\n", - "2025-10-22 23:01:43 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 8dd29a67-8d57-4f95-b0bd-9c1fa22f55da) - 1 spans\n", - "2025-10-22 23:01:43 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 8dd29a67-8d57-4f95-b0bd-9c1fa22f55da) - 1 spans\n", - "2025-10-22 23:01:43 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 8dd29a67-8d57-4f95-b0bd-9c1fa22f55da\n", - "2025-10-22 23:01:43 - noveum_trace.transport.http_transport - INFO - āœ… Trace 8dd29a67-8d57-4f95-b0bd-9c1fa22f55da successfully queued for export\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:01:43 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-10-22 23:01:44 - noveum_trace.transport.http_transport - INFO - šŸ“” HTTP RESPONSE: Status 200 from https://api.noveum.ai/api/v1/traces\n", - "2025-10-22 23:01:44 - noveum_trace.transport.http_transport - INFO - āœ… Successfully sent batch of 4 traces\n", - "2025-10-22 23:01:44 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully sent batch of 4 traces via callback\n", - "2025-10-22 23:01:45 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: ae9c03e4-9a18-4c40-a616-34ba6df7216e) - 1 spans\n", - "2025-10-22 23:01:45 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: ae9c03e4-9a18-4c40-a616-34ba6df7216e) - 1 spans\n", - "2025-10-22 23:01:45 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace ae9c03e4-9a18-4c40-a616-34ba6df7216e\n", - "2025-10-22 23:01:45 - noveum_trace.transport.http_transport - INFO - āœ… Trace ae9c03e4-9a18-4c40-a616-34ba6df7216e successfully queued for export\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:01:45 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-10-22 23:01:46 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: aefaba52-0c8a-428c-af02-72f1145c5c99) - 1 spans\n", - "2025-10-22 23:01:46 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: aefaba52-0c8a-428c-af02-72f1145c5c99) - 1 spans\n", - "2025-10-22 23:01:46 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace aefaba52-0c8a-428c-af02-72f1145c5c99\n", - "2025-10-22 23:01:46 - noveum_trace.transport.http_transport - INFO - āœ… Trace aefaba52-0c8a-428c-af02-72f1145c5c99 successfully queued for export\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:01:46 - INFO - novaeval.evaluators.agent_evaluator - Saving intermediate results after 8 samples\n", - "2025-10-22 23:01:46 - INFO - novaeval.evaluators.agent_evaluator - Intermediate results saved to demo_results/agent.rag_evaluation_metrics_dataset/agent_evaluation_results.csv\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Evaluating samples: 8it [00:34, 3.87s/it]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:01:46 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-10-22 23:01:47 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 6d0b6d2c-d570-40cb-8d9d-c0a1187d7ec5) - 1 spans\n", - "2025-10-22 23:01:47 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 6d0b6d2c-d570-40cb-8d9d-c0a1187d7ec5) - 1 spans\n", - "2025-10-22 23:01:47 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 6d0b6d2c-d570-40cb-8d9d-c0a1187d7ec5\n", - "2025-10-22 23:01:47 - noveum_trace.transport.http_transport - INFO - āœ… Trace 6d0b6d2c-d570-40cb-8d9d-c0a1187d7ec5 successfully queued for export\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:01:47 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-10-22 23:01:49 - noveum_trace.transport.batch_processor - INFO - ā° TIMEOUT TRIGGER: Sending batch due to timeout (5.4s >= 5.0s)\n", - "2025-10-22 23:01:49 - noveum_trace.transport.batch_processor - INFO - šŸ“¤ SENDING BATCH: 4 traces via send_callback\n", - "2025-10-22 23:01:49 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: af1cb192-c53d-4c68-bcf1-63ae80a319ab) - 1 spans\n", - "2025-10-22 23:01:49 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: af1cb192-c53d-4c68-bcf1-63ae80a319ab) - 1 spans\n", - "2025-10-22 23:01:49 - noveum_trace.transport.http_transport - INFO - šŸš€ SENDING BATCH: 4 traces to https://api.noveum.ai/api/v1/traces\n", - "2025-10-22 23:01:49 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace af1cb192-c53d-4c68-bcf1-63ae80a319ab\n", - "2025-10-22 23:01:49 - noveum_trace.transport.http_transport - INFO - āœ… Trace af1cb192-c53d-4c68-bcf1-63ae80a319ab successfully queued for export\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:01:49 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-10-22 23:01:49 - noveum_trace.transport.http_transport - INFO - šŸ“” HTTP RESPONSE: Status 200 from https://api.noveum.ai/api/v1/traces\n", - "2025-10-22 23:01:49 - noveum_trace.transport.http_transport - INFO - āœ… Successfully sent batch of 4 traces\n", - "2025-10-22 23:01:49 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully sent batch of 4 traces via callback\n", - "2025-10-22 23:01:50 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 6dc7d782-6f29-481b-9738-d22d793c9d53) - 1 spans\n", - "2025-10-22 23:01:50 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 6dc7d782-6f29-481b-9738-d22d793c9d53) - 1 spans\n", - "2025-10-22 23:01:50 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 6dc7d782-6f29-481b-9738-d22d793c9d53\n", - "2025-10-22 23:01:50 - noveum_trace.transport.http_transport - INFO - āœ… Trace 6dc7d782-6f29-481b-9738-d22d793c9d53 successfully queued for export\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:01:50 - INFO - novaeval.evaluators.agent_evaluator - Saving intermediate results after 9 samples\n", - "2025-10-22 23:01:50 - INFO - novaeval.evaluators.agent_evaluator - Intermediate results saved to demo_results/agent.rag_evaluation_metrics_dataset/agent_evaluation_results.csv\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Evaluating samples: 9it [00:38, 3.94s/it]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:01:50 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-10-22 23:01:51 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 79ae16c8-3b0e-4e3c-bc10-9eae81586163) - 1 spans\n", - "2025-10-22 23:01:51 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 79ae16c8-3b0e-4e3c-bc10-9eae81586163) - 1 spans\n", - "2025-10-22 23:01:51 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 79ae16c8-3b0e-4e3c-bc10-9eae81586163\n", - "2025-10-22 23:01:51 - noveum_trace.transport.http_transport - INFO - āœ… Trace 79ae16c8-3b0e-4e3c-bc10-9eae81586163 successfully queued for export\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:01:51 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-10-22 23:01:52 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: e69630b6-5e34-4ae7-9195-929d04ef31a5) - 1 spans\n", - "2025-10-22 23:01:52 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: e69630b6-5e34-4ae7-9195-929d04ef31a5) - 1 spans\n", - "2025-10-22 23:01:52 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace e69630b6-5e34-4ae7-9195-929d04ef31a5\n", - "2025-10-22 23:01:52 - noveum_trace.transport.http_transport - INFO - āœ… Trace e69630b6-5e34-4ae7-9195-929d04ef31a5 successfully queued for export\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:01:52 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-10-22 23:01:53 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: ef939d49-86d5-4ab4-b8a3-918e10f86639) - 1 spans\n", - "2025-10-22 23:01:53 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: ef939d49-86d5-4ab4-b8a3-918e10f86639) - 1 spans\n", - "2025-10-22 23:01:53 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace ef939d49-86d5-4ab4-b8a3-918e10f86639\n", - "2025-10-22 23:01:53 - noveum_trace.transport.http_transport - INFO - āœ… Trace ef939d49-86d5-4ab4-b8a3-918e10f86639 successfully queued for export\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:01:53 - INFO - novaeval.evaluators.agent_evaluator - Saving intermediate results after 10 samples\n", - "2025-10-22 23:01:53 - INFO - novaeval.evaluators.agent_evaluator - Intermediate results saved to demo_results/agent.rag_evaluation_metrics_dataset/agent_evaluation_results.csv\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Evaluating samples: 10it [00:41, 3.77s/it]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:01:53 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-10-22 23:01:54 - noveum_trace.transport.batch_processor - INFO - ā° TIMEOUT TRIGGER: Sending batch due to timeout (5.4s >= 5.0s)\n", - "2025-10-22 23:01:54 - noveum_trace.transport.batch_processor - INFO - šŸ“¤ SENDING BATCH: 5 traces via send_callback\n", - "2025-10-22 23:01:54 - noveum_trace.transport.http_transport - INFO - šŸš€ SENDING BATCH: 5 traces to https://api.noveum.ai/api/v1/traces\n", - "2025-10-22 23:01:54 - noveum_trace.transport.http_transport - INFO - šŸ“” HTTP RESPONSE: Status 200 from https://api.noveum.ai/api/v1/traces\n", - "2025-10-22 23:01:54 - noveum_trace.transport.http_transport - INFO - āœ… Successfully sent batch of 5 traces\n", - "2025-10-22 23:01:54 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully sent batch of 5 traces via callback\n", - "2025-10-22 23:01:57 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 926cf84f-fcb5-4e44-b36b-dfa5f9c28c55) - 1 spans\n", - "2025-10-22 23:01:57 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 926cf84f-fcb5-4e44-b36b-dfa5f9c28c55) - 1 spans\n", - "2025-10-22 23:01:57 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 926cf84f-fcb5-4e44-b36b-dfa5f9c28c55\n", - "2025-10-22 23:01:57 - noveum_trace.transport.http_transport - INFO - āœ… Trace 926cf84f-fcb5-4e44-b36b-dfa5f9c28c55 successfully queued for export\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:01:57 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-10-22 23:01:58 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 5bb8255c-0f41-4735-a818-52f6115b7de1) - 1 spans\n", - "2025-10-22 23:01:58 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 5bb8255c-0f41-4735-a818-52f6115b7de1) - 1 spans\n", - "2025-10-22 23:01:58 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 5bb8255c-0f41-4735-a818-52f6115b7de1\n", - "2025-10-22 23:01:58 - noveum_trace.transport.http_transport - INFO - āœ… Trace 5bb8255c-0f41-4735-a818-52f6115b7de1 successfully queued for export\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:01:58 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-10-22 23:01:59 - noveum_trace.transport.batch_processor - INFO - ā° TIMEOUT TRIGGER: Sending batch due to timeout (5.5s >= 5.0s)\n", - "2025-10-22 23:01:59 - noveum_trace.transport.batch_processor - INFO - šŸ“¤ SENDING BATCH: 2 traces via send_callback\n", - "2025-10-22 23:01:59 - noveum_trace.transport.http_transport - INFO - šŸš€ SENDING BATCH: 2 traces to https://api.noveum.ai/api/v1/traces\n", - "2025-10-22 23:02:00 - noveum_trace.transport.http_transport - INFO - šŸ“” HTTP RESPONSE: Status 200 from https://api.noveum.ai/api/v1/traces\n", - "2025-10-22 23:02:00 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 96472f81-b98d-4453-97e8-b0000f4ac7b6) - 1 spans\n", - "2025-10-22 23:02:00 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 96472f81-b98d-4453-97e8-b0000f4ac7b6) - 1 spans\n", - "2025-10-22 23:02:00 - noveum_trace.transport.http_transport - INFO - āœ… Successfully sent batch of 2 traces\n", - "2025-10-22 23:02:00 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 96472f81-b98d-4453-97e8-b0000f4ac7b6\n", - "2025-10-22 23:02:00 - noveum_trace.transport.http_transport - INFO - āœ… Trace 96472f81-b98d-4453-97e8-b0000f4ac7b6 successfully queued for export\n", - "2025-10-22 23:02:00 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully sent batch of 2 traces via callback\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:02:00 - INFO - novaeval.evaluators.agent_evaluator - Saving intermediate results after 11 samples\n", - "2025-10-22 23:02:00 - INFO - novaeval.evaluators.agent_evaluator - Intermediate results saved to demo_results/agent.rag_evaluation_metrics_dataset/agent_evaluation_results.csv\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Evaluating samples: 11it [00:47, 4.60s/it]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:02:00 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-10-22 23:02:02 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 3ec56361-a2aa-45bd-ad9e-12139cd5b25b) - 1 spans\n", - "2025-10-22 23:02:02 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 3ec56361-a2aa-45bd-ad9e-12139cd5b25b) - 1 spans\n", - "2025-10-22 23:02:02 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 3ec56361-a2aa-45bd-ad9e-12139cd5b25b\n", - "2025-10-22 23:02:02 - noveum_trace.transport.http_transport - INFO - āœ… Trace 3ec56361-a2aa-45bd-ad9e-12139cd5b25b successfully queued for export\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:02:02 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-10-22 23:02:03 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: bd7c6734-9502-43af-9227-7e3256eac18c) - 1 spans\n", - "2025-10-22 23:02:03 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: bd7c6734-9502-43af-9227-7e3256eac18c) - 1 spans\n", - "2025-10-22 23:02:03 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace bd7c6734-9502-43af-9227-7e3256eac18c\n", - "2025-10-22 23:02:03 - noveum_trace.transport.http_transport - INFO - āœ… Trace bd7c6734-9502-43af-9227-7e3256eac18c successfully queued for export\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:02:03 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-10-22 23:02:05 - noveum_trace.transport.batch_processor - INFO - ā° TIMEOUT TRIGGER: Sending batch due to timeout (5.5s >= 5.0s)\n", - "2025-10-22 23:02:05 - noveum_trace.transport.batch_processor - INFO - šŸ“¤ SENDING BATCH: 3 traces via send_callback\n", - "2025-10-22 23:02:05 - noveum_trace.transport.http_transport - INFO - šŸš€ SENDING BATCH: 3 traces to https://api.noveum.ai/api/v1/traces\n", - "2025-10-22 23:02:05 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 8c77b5d2-4e09-4e86-89db-9c4432c6a8f9) - 1 spans\n", - "2025-10-22 23:02:05 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 8c77b5d2-4e09-4e86-89db-9c4432c6a8f9) - 1 spans\n", - "2025-10-22 23:02:05 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 8c77b5d2-4e09-4e86-89db-9c4432c6a8f9\n", - "2025-10-22 23:02:05 - noveum_trace.transport.http_transport - INFO - āœ… Trace 8c77b5d2-4e09-4e86-89db-9c4432c6a8f9 successfully queued for export\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:02:05 - INFO - novaeval.evaluators.agent_evaluator - Saving intermediate results after 12 samples\n", - "2025-10-22 23:02:05 - INFO - novaeval.evaluators.agent_evaluator - Intermediate results saved to demo_results/agent.rag_evaluation_metrics_dataset/agent_evaluation_results.csv\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Evaluating samples: 12it [00:52, 4.42s/it]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:02:05 - INFO - novaeval.evaluators.agent_evaluator - Saving final results\n", - "2025-10-22 23:02:05 - INFO - novaeval.evaluators.agent_evaluator - Reloaded 12 results from CSV\n", - "2025-10-22 23:02:05 - INFO - novaeval.evaluators.agent_evaluator - Agent evaluation completed\n", - "\n", - "āœ… Evaluation completed!\n", - "\n", - "šŸ“Š Results Summary:\n", - " - task_progression: 4.27\n", - " - context_relevancy: 7.90\n", - " - role_adherence: 9.08\n", - " - tool_relevancy: 0.00\n", - " - parameter_correctness: 0.00\n", - "\n", - "šŸ” Individual Scores:\n", - "\n", - " Record 1 (Task: f1f37bd7-0851-4659-b493-b80d3800d920):\n", - " - task_progression: 3.8\n", - " - context_relevancy: 7.8\n", - " - role_adherence: 9.0\n", - " - tool_relevancy: 0.0\n", - " - parameter_correctness: 0.0\n", - "\n", - " Record 2 (Task: 52aacb67-c361-4445-9b72-c157f79f47d6):\n", - " - task_progression: 2.8\n", - " - context_relevancy: 7.8\n", - " - role_adherence: 9.0\n", - " - tool_relevancy: 0.0\n", - " - parameter_correctness: 0.0\n", - "\n", - " Record 3 (Task: 2218f641-604c-491a-9710-b51a9941b982):\n", - " - task_progression: 4.5\n", - " - context_relevancy: 7.9\n", - " - role_adherence: 9.0\n", - " - tool_relevancy: 0.0\n", - " - parameter_correctness: 0.0\n", - "\n", - " Record 4 (Task: 255fd49c-84b4-4b18-887e-6308a412d535):\n", - " - task_progression: 4.5\n", - " - context_relevancy: 7.8\n", - " - role_adherence: 9.0\n", - " - tool_relevancy: 0.0\n", - " - parameter_correctness: 0.0\n", - "\n", - " Record 5 (Task: dc511122-c0b6-415c-9a49-c7b45132dd87):\n", - " - task_progression: 4.5\n", - " - context_relevancy: 8.1\n", - " - role_adherence: 9.0\n", - " - tool_relevancy: 0.0\n", - " - parameter_correctness: 0.0\n", - "\n", - " Record 6 (Task: 04bebf38-a343-4563-80db-0154bef8d927):\n", - " - task_progression: 4.5\n", - " - context_relevancy: 7.8\n", - " - role_adherence: 9.0\n", - " - tool_relevancy: 0.0\n", - " - parameter_correctness: 0.0\n", - "\n", - " Record 7 (Task: 5e043630-6493-42b5-beb8-79faa19bfa37):\n", - " - task_progression: 4.5\n", - " - context_relevancy: 7.8\n", - " - role_adherence: 9.0\n", - " - tool_relevancy: 0.0\n", - " - parameter_correctness: 0.0\n", - "\n", - " Record 8 (Task: 7da9814d-a2e8-4c4e-b750-68b26bd5fd22):\n", - " - task_progression: 4.5\n", - " - context_relevancy: 7.8\n", - " - role_adherence: 9.0\n", - " - tool_relevancy: 0.0\n", - " - parameter_correctness: 0.0\n", - "\n", - " Record 9 (Task: 16143f74-2831-4753-b33d-ce4b645093c5):\n", - " - task_progression: 4.5\n", - " - context_relevancy: 8.1\n", - " - role_adherence: 9.0\n", - " - tool_relevancy: 0.0\n", - " - parameter_correctness: 0.0\n", - "\n", - " Record 10 (Task: fc64e6cc-6739-4256-ac4a-7b80c3028233):\n", - " - task_progression: 4.1\n", - " - context_relevancy: 7.8\n", - " - role_adherence: 9.0\n", - " - tool_relevancy: 0.0\n", - " - parameter_correctness: 0.0\n", - "\n", - " Record 11 (Task: b7945c49-f584-4c70-972d-536a805d8a31):\n", - " - task_progression: 4.5\n", - " - context_relevancy: 8.0\n", - " - role_adherence: 9.0\n", - " - tool_relevancy: 0.0\n", - " - parameter_correctness: 0.0\n", - "\n", - " Record 12 (Task: f5c40ecf-36c0-45ba-9cc9-dc0329b0324b):\n", - " - task_progression: 4.5\n", - " - context_relevancy: 8.1\n", - " - role_adherence: 10.0\n", - " - tool_relevancy: 0.0\n", - " - parameter_correctness: 0.0\n", - "āœ… Evaluation completed successfully!\n", - "\n", - "šŸ“‹ Step 7: Exporting Dataset\n", - "šŸ’¾ Exporting processed dataset...\n", - "āœ… Exported to ./processed_datasets/agent.rag_evaluation_metrics_dataset_processed_dataset.json\n", - "āœ… Exported to ./processed_datasets/agent.rag_evaluation_metrics_dataset_processed_dataset.csv\n", - "āœ… Export completed successfully!\n", - "\n", - "============================================================\n", - "šŸŽ‰ EVALUATION PIPELINE COMPLETED!\n", - "šŸ“Š Final Results:\n", - " - File processed: split_datasets/agent.rag_evaluation_metrics_dataset.json\n", - " - Spans loaded: 12\n", - " - Dataset size: 12\n", - " - Evaluation completed: True\n", - " - Export successful: True\n", - " - Results saved to: ./demo_results/agent.rag_evaluation_metrics_dataset/\n", - "Completed agent.rag_evaluation_metrics_dataset.json\n", - "\n", - "Processing agent.routing_evaluation_metrics_dataset.json...\n", - "šŸš€ Starting Complete Agent Evaluation Pipeline\n", - "šŸ“ Processing file: split_datasets/agent.routing_evaluation_metrics_dataset.json\n", - "============================================================\n", - "\n", - "šŸ“‹ Step 1: Environment Setup\n", - "āœ… Logging configured at INFO level\n", - "šŸ” Environment validation:\n", - " āœ… gemini_api_key: True\n", - " āœ… pandas_available: True\n", - " āœ… novaeval_available: True\n", - "āœ… Environment ready for evaluation!\n", - "\n", - "šŸ“‹ Step 2: Loading Dataset\n", - "šŸ“Š Loaded 20 spans from split_datasets/agent.routing_evaluation_metrics_dataset.json\n", - "\n", - "šŸ” Available span types:\n", - " - agent.routing_evaluation_metrics: 20\n", - "āœ… Dataset loaded: 20 spans\n", - "\n", - "šŸ“‹ Step 3: Converting to AgentDataset Format\n", - "šŸ”„ Converting spans to AgentData objects...\n", - "agent_task not found\n", - "agent_response is not available 95dac0c3-83fd-459c-af4e-c6a1734fd874\n", - "agent_task not found\n", - "agent_response is not available 516060a3-0f58-46ba-8a6a-4b502a376492\n", - "agent_task not found\n", - "agent_response is not available 8f8d2dc4-150f-4e43-aad3-96da4d06f586\n", - "agent_task not found\n", - "agent_response is not available cf0a915c-668b-412e-9c14-69119d059dc2\n", - "agent_task not found\n", - "agent_response is not available f75b51b8-4882-4170-bc08-2adcab250f50\n", - "agent_task not found\n", - "agent_response is not available 4180abcf-924c-45ac-8dfa-17c0bc603eff\n", - "agent_task not found\n", - "agent_response is not available 8b0022cc-91dd-4065-91ed-b77217f92bd7\n", - "agent_task not found\n", - "agent_response is not available 780df186-fe5a-4742-b937-2a142ad528c6\n", - "agent_task not found\n", - "agent_response is not available f80ae3e8-b4dd-42c6-b17b-de7805cedbb9\n", - "agent_task not found\n", - "agent_response is not available 209259cf-411d-4b97-8f7b-40b2530eec81\n", - "agent_task not found\n", - "agent_response is not available c3f3bd32-9954-443f-93e5-52aa6cf412a0\n", - "agent_task not found\n", - "agent_response is not available 940ce621-0015-45ed-adc6-c4caa2a95e81\n", - "agent_task not found\n", - "agent_response is not available ea28f049-6bc9-48b4-8386-279abe087ca8\n", - "agent_task not found\n", - "agent_response is not available 01a03da5-64df-41e3-b9c9-556e12a8edd7\n", - "agent_task not found\n", - "agent_response is not available 8d9c632b-b2e8-496d-b1b0-a89e780ef810\n", - "agent_task not found\n", - "agent_response is not available 72dfbba4-0672-4adc-a1bf-308157d831e6\n", - "agent_task not found\n", - "agent_response is not available 92c81ee2-728d-4ea6-bc52-b81295d2aea2\n", - "agent_task not found\n", - "agent_response is not available 807627b0-41d9-478f-8a8b-26540d70af68\n", - "agent_task not found\n", - "agent_response is not available bf860672-af22-44c1-b13b-0eabe8f72503\n", - "agent_task not found\n", - "agent_response is not available 5fa714f1-eeb9-4617-bc7e-67062e6d1eab\n", - "\n", - "āœ… Successfully converted 20 spans to AgentData\n", - "šŸ“Š AgentDataset created with 20 records\n", - "āœ… AgentDataset created: 20 records\n", - "\n", - "šŸ“‹ Step 4: Dataset Analysis\n", - "šŸ“ˆ Dataset Statistics:\n", - "\n", - "Agent Types: {'agent': 20}\n", - "Records with responses: 0\n", - "Records with tool calls: 0\n", - "Records with retrieval: 0\n", - "Tool usage: {}\n", - "šŸ” Dataset Analysis:\n", - "\n", - "=== Agent Behavior Patterns ===\n", - "\n", - "šŸ“ˆ Tool Usage:\n", - "\n", - "šŸ“‹ Task Types:\n", - "\n", - "šŸ“‹ Step 5: Setting up Evaluation\n", - "āœ… GEMINI_API_KEY found in environment\n", - "2025-10-22 23:02:05 - INFO - novaeval.models.base - Noveum tracing initialized successfully\n", - "āœ… Gemini model initialized\n", - "āœ… Initialized 5 scoring functions:\n", - " - task_progression_scorer\n", - " - context_relevancy_scorer\n", - " - role_adherence_scorer\n", - " - tool_relevancy_scorer\n", - " - parameter_correctness_scorer\n", - "\n", - "āœ… AgentEvaluator created with Gemini model and scoring functions\n", - "āœ… Evaluation components ready!\n", - "\n", - "šŸ“‹ Step 6: Running Evaluation\n", - "šŸŽÆ Evaluating 25 samples...\n", - "šŸš€ Running evaluation on sample data...\n", - "\n", - "šŸ“Š Evaluating 0 sample records...\n", - "2025-10-22 23:02:05 - INFO - novaeval.evaluators.agent_evaluator - Starting agent evaluation process\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\n", - "Evaluating samples: 0it [00:00, ?it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:02:05 - INFO - novaeval.evaluators.agent_evaluator - Saving final results\n", - "2025-10-22 23:02:05 - INFO - novaeval.evaluators.agent_evaluator - Agent evaluation completed\n", - "\n", - "āœ… Evaluation completed!\n", - "āŒ Results file not found\n", - "āŒ Evaluation failed\n", - "\n", - "šŸ“‹ Step 7: Exporting Dataset\n", - "šŸ’¾ Exporting processed dataset...\n", - "āœ… Exported to ./processed_datasets/agent.routing_evaluation_metrics_dataset_processed_dataset.json\n", - "āœ… Exported to ./processed_datasets/agent.routing_evaluation_metrics_dataset_processed_dataset.csv\n", - "āœ… Export completed successfully!\n", - "\n", - "============================================================\n", - "šŸŽ‰ EVALUATION PIPELINE COMPLETED!\n", - "šŸ“Š Final Results:\n", - " - File processed: split_datasets/agent.routing_evaluation_metrics_dataset.json\n", - " - Spans loaded: 20\n", - " - Dataset size: 20\n", - " - Evaluation completed: False\n", - " - Export successful: True\n", - " - Errors encountered: 1\n", - "Completed agent.routing_evaluation_metrics_dataset.json\n", - "\n", - "Processing agent.llm-rag_dataset.json...\n", - "šŸš€ Starting Complete Agent Evaluation Pipeline\n", - "šŸ“ Processing file: split_datasets/agent.llm-rag_dataset.json\n", - "============================================================\n", - "\n", - "šŸ“‹ Step 1: Environment Setup\n", - "āœ… Logging configured at INFO level\n", - "šŸ” Environment validation:\n", - " āœ… gemini_api_key: True\n", - " āœ… pandas_available: True\n", - " āœ… novaeval_available: True\n", - "āœ… Environment ready for evaluation!\n", - "\n", - "šŸ“‹ Step 2: Loading Dataset\n", - "šŸ“Š Loaded 12 spans from split_datasets/agent.llm-rag_dataset.json\n", - "\n", - "šŸ” Available span types:\n", - " - agent.llm-rag: 12\n", - "āœ… Dataset loaded: 12 spans\n", - "\n", - "šŸ“‹ Step 3: Converting to AgentDataset Format\n", - "šŸ”„ Converting spans to AgentData objects...\n", - "agent_task not found\n", - "agent_response is not available ea4d61c5-b8dd-4677-b21b-2b17170e2163\n", - "agent_task not found\n", - "agent_response is not available 7505f97a-cbc0-42fe-8a72-95387d723daa\n", - "agent_task not found\n", - "agent_response is not available 8a6e18a1-babc-4166-926f-188f344d514e\n", - "agent_task not found\n", - "agent_response is not available 5bec4165-6596-43ac-930b-acfcde1cc02a\n", - "agent_task not found\n", - "agent_response is not available f9f79a07-4eda-41b8-9634-d4460114caad\n", - "agent_task not found\n", - "agent_response is not available edc52a5d-8450-4d0f-ad14-958ec4d147f4\n", - "agent_task not found\n", - "agent_response is not available 6c362c73-e063-4813-8180-d26b6a52b798\n", - "agent_task not found\n", - "agent_response is not available 1393d065-a9a4-4475-808f-2d00b3c68939\n", - "agent_task not found\n", - "agent_response is not available f419377a-fc24-4abf-8a1c-03857824e374\n", - "agent_task not found\n", - "agent_response is not available 0e58e31f-a8ab-4aee-8fc2-abcae1e21d9d\n", - "agent_task not found\n", - "agent_response is not available 48155aad-6204-4a52-8624-8fa5d6fe407f\n", - "agent_task not found\n", - "agent_response is not available 57ded2eb-0f3f-474e-8704-5ad9974ab4bb\n", - "\n", - "āœ… Successfully converted 12 spans to AgentData\n", - "šŸ“Š AgentDataset created with 12 records\n", - "āœ… AgentDataset created: 12 records\n", - "\n", - "šŸ“‹ Step 4: Dataset Analysis\n", - "šŸ“ˆ Dataset Statistics:\n", - "\n", - "Agent Types: {'agent': 12}\n", - "Records with responses: 0\n", - "Records with tool calls: 0\n", - "Records with retrieval: 0\n", - "Tool usage: {}\n", - "šŸ” Dataset Analysis:\n", - "\n", - "=== Agent Behavior Patterns ===\n", - "\n", - "šŸ“ˆ Tool Usage:\n", - "\n", - "šŸ“‹ Task Types:\n", - "\n", - "šŸ“‹ Step 5: Setting up Evaluation\n", - "āœ… GEMINI_API_KEY found in environment\n", - "2025-10-22 23:02:05 - INFO - novaeval.models.base - Noveum tracing initialized successfully\n", - "āœ… Gemini model initialized\n", - "āœ… Initialized 5 scoring functions:\n", - " - task_progression_scorer\n", - " - context_relevancy_scorer\n", - " - role_adherence_scorer\n", - " - tool_relevancy_scorer\n", - " - parameter_correctness_scorer\n", - "\n", - "āœ… AgentEvaluator created with Gemini model and scoring functions\n", - "āœ… Evaluation components ready!\n", - "\n", - "šŸ“‹ Step 6: Running Evaluation\n", - "šŸŽÆ Evaluating 25 samples...\n", - "šŸš€ Running evaluation on sample data...\n", - "\n", - "šŸ“Š Evaluating 0 sample records...\n", - "2025-10-22 23:02:05 - INFO - novaeval.evaluators.agent_evaluator - Starting agent evaluation process\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\n", - "Evaluating samples: 0it [00:00, ?it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:02:05 - INFO - novaeval.evaluators.agent_evaluator - Saving final results\n", - "2025-10-22 23:02:05 - INFO - novaeval.evaluators.agent_evaluator - Agent evaluation completed\n", - "\n", - "āœ… Evaluation completed!\n", - "āŒ Results file not found\n", - "āŒ Evaluation failed\n", - "\n", - "šŸ“‹ Step 7: Exporting Dataset\n", - "šŸ’¾ Exporting processed dataset...\n", - "āœ… Exported to ./processed_datasets/agent.llm-rag_dataset_processed_dataset.json\n", - "āœ… Exported to ./processed_datasets/agent.llm-rag_dataset_processed_dataset.csv\n", - "āœ… Export completed successfully!\n", - "\n", - "============================================================\n", - "šŸŽ‰ EVALUATION PIPELINE COMPLETED!\n", - "šŸ“Š Final Results:\n", - " - File processed: split_datasets/agent.llm-rag_dataset.json\n", - " - Spans loaded: 12\n", - " - Dataset size: 12\n", - " - Evaluation completed: False\n", - " - Export successful: True\n", - " - Errors encountered: 1\n", - "Completed agent.llm-rag_dataset.json\n", - "\n", - "Processing agent.web_search_evaluation_metrics_dataset.json...\n", - "šŸš€ Starting Complete Agent Evaluation Pipeline\n", - "šŸ“ Processing file: split_datasets/agent.web_search_evaluation_metrics_dataset.json\n", - "============================================================\n", - "\n", - "šŸ“‹ Step 1: Environment Setup\n", - "āœ… Logging configured at INFO level\n", - "šŸ” Environment validation:\n", - " āœ… gemini_api_key: True\n", - " āœ… pandas_available: True\n", - " āœ… novaeval_available: True\n", - "āœ… Environment ready for evaluation!\n", - "\n", - "šŸ“‹ Step 2: Loading Dataset\n", - "šŸ“Š Loaded 8 spans from split_datasets/agent.web_search_evaluation_metrics_dataset.json\n", - "\n", - "šŸ” Available span types:\n", - " - agent.web_search_evaluation_metrics: 8\n", - "āœ… Dataset loaded: 8 spans\n", - "\n", - "šŸ“‹ Step 3: Converting to AgentDataset Format\n", - "šŸ”„ Converting spans to AgentData objects...\n", - "\n", - "āœ… Successfully converted 8 spans to AgentData\n", - "šŸ“Š AgentDataset created with 8 records\n", - "āœ… AgentDataset created: 8 records\n", - "\n", - "šŸ“‹ Step 4: Dataset Analysis\n", - "šŸ“ˆ Dataset Statistics:\n", - "\n", - "Agent Types: {'agent': 8}\n", - "Records with responses: 8\n", - "Records with tool calls: 0\n", - "Records with retrieval: 8\n", - "Tool usage: {}\n", - "šŸ” Dataset Analysis:\n", - "\n", - "=== Agent Behavior Patterns ===\n", - "\n", - "šŸ“ˆ Tool Usage:\n", - "\n", - "šŸ“‹ Task Types:\n", - " - other: 8\n", - "\n", - "šŸ“ Response Statistics:\n", - " - Average response length: 222.0 characters\n", - " - Min response length: 222\n", - " - Max response length: 222\n", - "\n", - "šŸ“‹ Step 5: Setting up Evaluation\n", - "āœ… GEMINI_API_KEY found in environment\n", - "2025-10-22 23:02:05 - INFO - novaeval.models.base - Noveum tracing initialized successfully\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "āœ… Gemini model initialized\n", - "āœ… Initialized 5 scoring functions:\n", - " - task_progression_scorer\n", - " - context_relevancy_scorer\n", - " - role_adherence_scorer\n", - " - tool_relevancy_scorer\n", - " - parameter_correctness_scorer\n", - "\n", - "āœ… AgentEvaluator created with Gemini model and scoring functions\n", - "āœ… Evaluation components ready!\n", - "\n", - "šŸ“‹ Step 6: Running Evaluation\n", - "šŸŽÆ Evaluating 25 samples...\n", - "šŸš€ Running evaluation on sample data...\n", - "\n", - "šŸ“Š Evaluating 8 sample records...\n", - "2025-10-22 23:02:05 - INFO - novaeval.evaluators.agent_evaluator - Starting agent evaluation process\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Evaluating samples: 0it [00:00, ?it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:02:05 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-10-22 23:02:05 - noveum_trace.transport.http_transport - INFO - šŸ“” HTTP RESPONSE: Status 200 from https://api.noveum.ai/api/v1/traces\n", - "2025-10-22 23:02:05 - noveum_trace.transport.http_transport - INFO - āœ… Successfully sent batch of 3 traces\n", - "2025-10-22 23:02:05 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully sent batch of 3 traces via callback\n", - "2025-10-22 23:02:07 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: e7588539-fcf8-44a3-9f31-efe82ba97700) - 1 spans\n", - "2025-10-22 23:02:07 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: e7588539-fcf8-44a3-9f31-efe82ba97700) - 1 spans\n", - "2025-10-22 23:02:07 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace e7588539-fcf8-44a3-9f31-efe82ba97700\n", - "2025-10-22 23:02:07 - noveum_trace.transport.http_transport - INFO - āœ… Trace e7588539-fcf8-44a3-9f31-efe82ba97700 successfully queued for export\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:02:07 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-10-22 23:02:08 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: a47d3d33-371b-4045-9fb1-378424928ada) - 1 spans\n", - "2025-10-22 23:02:08 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: a47d3d33-371b-4045-9fb1-378424928ada) - 1 spans\n", - "2025-10-22 23:02:08 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace a47d3d33-371b-4045-9fb1-378424928ada\n", - "2025-10-22 23:02:08 - noveum_trace.transport.http_transport - INFO - āœ… Trace a47d3d33-371b-4045-9fb1-378424928ada successfully queued for export\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:02:08 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-10-22 23:02:09 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 69314585-f591-4d20-ba7c-81dcd5fa7aee) - 1 spans\n", - "2025-10-22 23:02:09 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 69314585-f591-4d20-ba7c-81dcd5fa7aee) - 1 spans\n", - "2025-10-22 23:02:09 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 69314585-f591-4d20-ba7c-81dcd5fa7aee\n", - "2025-10-22 23:02:09 - noveum_trace.transport.http_transport - INFO - āœ… Trace 69314585-f591-4d20-ba7c-81dcd5fa7aee successfully queued for export\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:02:09 - INFO - novaeval.evaluators.agent_evaluator - Saving intermediate results after 1 samples\n", - "2025-10-22 23:02:09 - INFO - novaeval.evaluators.agent_evaluator - Intermediate results saved to demo_results/agent.web_search_evaluation_metrics_dataset/agent_evaluation_results.csv\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Evaluating samples: 1it [00:04, 4.12s/it]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:02:09 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-10-22 23:02:10 - noveum_trace.transport.batch_processor - INFO - ā° TIMEOUT TRIGGER: Sending batch due to timeout (5.4s >= 5.0s)\n", - "2025-10-22 23:02:10 - noveum_trace.transport.batch_processor - INFO - šŸ“¤ SENDING BATCH: 4 traces via send_callback\n", - "2025-10-22 23:02:10 - noveum_trace.transport.http_transport - INFO - šŸš€ SENDING BATCH: 4 traces to https://api.noveum.ai/api/v1/traces\n", - "2025-10-22 23:02:10 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: a05b5d92-06f6-4fec-9673-f33c267ef809) - 1 spans\n", - "2025-10-22 23:02:10 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: a05b5d92-06f6-4fec-9673-f33c267ef809) - 1 spans\n", - "2025-10-22 23:02:10 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace a05b5d92-06f6-4fec-9673-f33c267ef809\n", - "2025-10-22 23:02:10 - noveum_trace.transport.http_transport - INFO - āœ… Trace a05b5d92-06f6-4fec-9673-f33c267ef809 successfully queued for export\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:02:10 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-10-22 23:02:11 - noveum_trace.transport.http_transport - INFO - šŸ“” HTTP RESPONSE: Status 200 from https://api.noveum.ai/api/v1/traces\n", - "2025-10-22 23:02:11 - noveum_trace.transport.http_transport - INFO - āœ… Successfully sent batch of 4 traces\n", - "2025-10-22 23:02:11 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully sent batch of 4 traces via callback\n", - "2025-10-22 23:02:12 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 032c73be-2c3f-4980-9cdf-762de61c437e) - 1 spans\n", - "2025-10-22 23:02:12 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 032c73be-2c3f-4980-9cdf-762de61c437e) - 1 spans\n", - "2025-10-22 23:02:12 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 032c73be-2c3f-4980-9cdf-762de61c437e\n", - "2025-10-22 23:02:12 - noveum_trace.transport.http_transport - INFO - āœ… Trace 032c73be-2c3f-4980-9cdf-762de61c437e successfully queued for export\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:02:12 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-10-22 23:02:13 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 4ccea784-7905-4081-b536-1a6fade9180f) - 1 spans\n", - "2025-10-22 23:02:13 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 4ccea784-7905-4081-b536-1a6fade9180f) - 1 spans\n", - "2025-10-22 23:02:13 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 4ccea784-7905-4081-b536-1a6fade9180f\n", - "2025-10-22 23:02:13 - noveum_trace.transport.http_transport - INFO - āœ… Trace 4ccea784-7905-4081-b536-1a6fade9180f successfully queued for export\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:02:13 - INFO - novaeval.evaluators.agent_evaluator - Saving intermediate results after 2 samples\n", - "2025-10-22 23:02:13 - INFO - novaeval.evaluators.agent_evaluator - Intermediate results saved to demo_results/agent.web_search_evaluation_metrics_dataset/agent_evaluation_results.csv\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Evaluating samples: 2it [00:07, 3.79s/it]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:02:13 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-10-22 23:02:14 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 4a768f59-41a4-4da7-b4fb-1530bd85f1bc) - 1 spans\n", - "2025-10-22 23:02:14 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 4a768f59-41a4-4da7-b4fb-1530bd85f1bc) - 1 spans\n", - "2025-10-22 23:02:14 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 4a768f59-41a4-4da7-b4fb-1530bd85f1bc\n", - "2025-10-22 23:02:14 - noveum_trace.transport.http_transport - INFO - āœ… Trace 4a768f59-41a4-4da7-b4fb-1530bd85f1bc successfully queued for export\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:02:14 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-10-22 23:02:15 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 490c1659-4aca-401f-bf9d-34891b18717a) - 1 spans\n", - "2025-10-22 23:02:15 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 490c1659-4aca-401f-bf9d-34891b18717a) - 1 spans\n", - "2025-10-22 23:02:15 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 490c1659-4aca-401f-bf9d-34891b18717a\n", - "2025-10-22 23:02:15 - noveum_trace.transport.http_transport - INFO - āœ… Trace 490c1659-4aca-401f-bf9d-34891b18717a successfully queued for export\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:02:15 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-10-22 23:02:16 - noveum_trace.transport.batch_processor - INFO - ā° TIMEOUT TRIGGER: Sending batch due to timeout (5.5s >= 5.0s)\n", - "2025-10-22 23:02:16 - noveum_trace.transport.batch_processor - INFO - šŸ“¤ SENDING BATCH: 5 traces via send_callback\n", - "2025-10-22 23:02:16 - noveum_trace.transport.http_transport - INFO - šŸš€ SENDING BATCH: 5 traces to https://api.noveum.ai/api/v1/traces\n", - "2025-10-22 23:02:16 - noveum_trace.transport.http_transport - INFO - šŸ“” HTTP RESPONSE: Status 200 from https://api.noveum.ai/api/v1/traces\n", - "2025-10-22 23:02:16 - noveum_trace.transport.http_transport - INFO - āœ… Successfully sent batch of 5 traces\n", - "2025-10-22 23:02:16 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully sent batch of 5 traces via callback\n", - "2025-10-22 23:02:17 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: c83bf958-6430-47bc-9407-5933e25f4892) - 1 spans\n", - "2025-10-22 23:02:17 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: c83bf958-6430-47bc-9407-5933e25f4892) - 1 spans\n", - "2025-10-22 23:02:17 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace c83bf958-6430-47bc-9407-5933e25f4892\n", - "2025-10-22 23:02:17 - noveum_trace.transport.http_transport - INFO - āœ… Trace c83bf958-6430-47bc-9407-5933e25f4892 successfully queued for export\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:02:17 - INFO - novaeval.evaluators.agent_evaluator - Saving intermediate results after 3 samples\n", - "2025-10-22 23:02:17 - INFO - novaeval.evaluators.agent_evaluator - Intermediate results saved to demo_results/agent.web_search_evaluation_metrics_dataset/agent_evaluation_results.csv\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Evaluating samples: 3it [00:11, 3.78s/it]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:02:17 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-10-22 23:02:18 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: eacc2b6e-8d42-4c12-82df-b0298723c50e) - 1 spans\n", - "2025-10-22 23:02:18 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: eacc2b6e-8d42-4c12-82df-b0298723c50e) - 1 spans\n", - "2025-10-22 23:02:18 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace eacc2b6e-8d42-4c12-82df-b0298723c50e\n", - "2025-10-22 23:02:18 - noveum_trace.transport.http_transport - INFO - āœ… Trace eacc2b6e-8d42-4c12-82df-b0298723c50e successfully queued for export\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:02:18 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-10-22 23:02:19 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 1f3f745f-2735-4197-b27d-0e54cd4fdd9b) - 1 spans\n", - "2025-10-22 23:02:19 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 1f3f745f-2735-4197-b27d-0e54cd4fdd9b) - 1 spans\n", - "2025-10-22 23:02:19 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 1f3f745f-2735-4197-b27d-0e54cd4fdd9b\n", - "2025-10-22 23:02:19 - noveum_trace.transport.http_transport - INFO - āœ… Trace 1f3f745f-2735-4197-b27d-0e54cd4fdd9b successfully queued for export\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:02:19 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-10-22 23:02:20 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 04df8130-9aa6-4c7c-b184-df6b8d827620) - 1 spans\n", - "2025-10-22 23:02:20 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 04df8130-9aa6-4c7c-b184-df6b8d827620) - 1 spans\n", - "2025-10-22 23:02:21 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 04df8130-9aa6-4c7c-b184-df6b8d827620\n", - "2025-10-22 23:02:21 - noveum_trace.transport.http_transport - INFO - āœ… Trace 04df8130-9aa6-4c7c-b184-df6b8d827620 successfully queued for export\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:02:21 - INFO - novaeval.evaluators.agent_evaluator - Saving intermediate results after 4 samples\n", - "2025-10-22 23:02:21 - INFO - novaeval.evaluators.agent_evaluator - Intermediate results saved to demo_results/agent.web_search_evaluation_metrics_dataset/agent_evaluation_results.csv\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Evaluating samples: 4it [00:15, 3.84s/it]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:02:21 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-10-22 23:02:21 - noveum_trace.transport.batch_processor - INFO - ā° TIMEOUT TRIGGER: Sending batch due to timeout (5.2s >= 5.0s)\n", - "2025-10-22 23:02:21 - noveum_trace.transport.batch_processor - INFO - šŸ“¤ SENDING BATCH: 4 traces via send_callback\n", - "2025-10-22 23:02:21 - noveum_trace.transport.http_transport - INFO - šŸš€ SENDING BATCH: 4 traces to https://api.noveum.ai/api/v1/traces\n", - "2025-10-22 23:02:22 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 25522bcf-a4e6-4f72-b6b1-be6e12c6b628) - 1 spans\n", - "2025-10-22 23:02:22 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 25522bcf-a4e6-4f72-b6b1-be6e12c6b628) - 1 spans\n", - "2025-10-22 23:02:22 - noveum_trace.transport.http_transport - INFO - šŸ“” HTTP RESPONSE: Status 200 from https://api.noveum.ai/api/v1/traces\n", - "2025-10-22 23:02:22 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 25522bcf-a4e6-4f72-b6b1-be6e12c6b628\n", - "2025-10-22 23:02:22 - noveum_trace.transport.http_transport - INFO - āœ… Successfully sent batch of 4 traces\n", - "2025-10-22 23:02:22 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully sent batch of 4 traces via callback\n", - "2025-10-22 23:02:22 - noveum_trace.transport.http_transport - INFO - āœ… Trace 25522bcf-a4e6-4f72-b6b1-be6e12c6b628 successfully queued for export\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:02:22 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-10-22 23:02:23 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 8d0257b7-8b4a-43d2-b3f8-c331f9442e9a) - 1 spans\n", - "2025-10-22 23:02:23 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 8d0257b7-8b4a-43d2-b3f8-c331f9442e9a) - 1 spans\n", - "2025-10-22 23:02:23 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 8d0257b7-8b4a-43d2-b3f8-c331f9442e9a\n", - "2025-10-22 23:02:23 - noveum_trace.transport.http_transport - INFO - āœ… Trace 8d0257b7-8b4a-43d2-b3f8-c331f9442e9a successfully queued for export\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:02:23 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-10-22 23:02:26 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: df6eff5c-8fa9-4c0a-9f8b-4fb5eb71694a) - 1 spans\n", - "2025-10-22 23:02:26 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: df6eff5c-8fa9-4c0a-9f8b-4fb5eb71694a) - 1 spans\n", - "2025-10-22 23:02:26 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace df6eff5c-8fa9-4c0a-9f8b-4fb5eb71694a\n", - "2025-10-22 23:02:26 - noveum_trace.transport.http_transport - INFO - āœ… Trace df6eff5c-8fa9-4c0a-9f8b-4fb5eb71694a successfully queued for export\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:02:26 - INFO - novaeval.evaluators.agent_evaluator - Saving intermediate results after 5 samples\n", - "2025-10-22 23:02:26 - INFO - novaeval.evaluators.agent_evaluator - Intermediate results saved to demo_results/agent.web_search_evaluation_metrics_dataset/agent_evaluation_results.csv\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Evaluating samples: 5it [00:21, 4.57s/it]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:02:26 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-10-22 23:02:27 - noveum_trace.transport.batch_processor - INFO - ā° TIMEOUT TRIGGER: Sending batch due to timeout (5.9s >= 5.0s)\n", - "2025-10-22 23:02:27 - noveum_trace.transport.batch_processor - INFO - šŸ“¤ SENDING BATCH: 3 traces via send_callback\n", - "2025-10-22 23:02:27 - noveum_trace.transport.http_transport - INFO - šŸš€ SENDING BATCH: 3 traces to https://api.noveum.ai/api/v1/traces\n", - "2025-10-22 23:02:27 - noveum_trace.transport.http_transport - INFO - šŸ“” HTTP RESPONSE: Status 200 from https://api.noveum.ai/api/v1/traces\n", - "2025-10-22 23:02:27 - noveum_trace.transport.http_transport - INFO - āœ… Successfully sent batch of 3 traces\n", - "2025-10-22 23:02:27 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully sent batch of 3 traces via callback\n", - "2025-10-22 23:02:28 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 9fca5686-3b49-4bba-961f-452fba26f5e6) - 1 spans\n", - "2025-10-22 23:02:28 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 9fca5686-3b49-4bba-961f-452fba26f5e6) - 1 spans\n", - "2025-10-22 23:02:28 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 9fca5686-3b49-4bba-961f-452fba26f5e6\n", - "2025-10-22 23:02:28 - noveum_trace.transport.http_transport - INFO - āœ… Trace 9fca5686-3b49-4bba-961f-452fba26f5e6 successfully queued for export\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:02:28 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-10-22 23:02:29 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: d7502a23-efaa-437e-ab40-138c0a1c3cc9) - 1 spans\n", - "2025-10-22 23:02:29 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: d7502a23-efaa-437e-ab40-138c0a1c3cc9) - 1 spans\n", - "2025-10-22 23:02:29 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace d7502a23-efaa-437e-ab40-138c0a1c3cc9\n", - "2025-10-22 23:02:29 - noveum_trace.transport.http_transport - INFO - āœ… Trace d7502a23-efaa-437e-ab40-138c0a1c3cc9 successfully queued for export\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:02:29 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-10-22 23:02:30 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 539d7f80-b556-441a-b288-08da522ef2ed) - 1 spans\n", - "2025-10-22 23:02:30 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 539d7f80-b556-441a-b288-08da522ef2ed) - 1 spans\n", - "2025-10-22 23:02:30 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 539d7f80-b556-441a-b288-08da522ef2ed\n", - "2025-10-22 23:02:30 - noveum_trace.transport.http_transport - INFO - āœ… Trace 539d7f80-b556-441a-b288-08da522ef2ed successfully queued for export\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:02:30 - INFO - novaeval.evaluators.agent_evaluator - Saving intermediate results after 6 samples\n", - "2025-10-22 23:02:30 - INFO - novaeval.evaluators.agent_evaluator - Intermediate results saved to demo_results/agent.web_search_evaluation_metrics_dataset/agent_evaluation_results.csv\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Evaluating samples: 6it [00:25, 4.33s/it]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:02:30 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-10-22 23:02:31 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: f4853141-1b6b-493e-94c2-ebd82a327a63) - 1 spans\n", - "2025-10-22 23:02:31 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: f4853141-1b6b-493e-94c2-ebd82a327a63) - 1 spans\n", - "2025-10-22 23:02:31 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace f4853141-1b6b-493e-94c2-ebd82a327a63\n", - "2025-10-22 23:02:31 - noveum_trace.transport.http_transport - INFO - āœ… Trace f4853141-1b6b-493e-94c2-ebd82a327a63 successfully queued for export\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:02:31 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-10-22 23:02:32 - noveum_trace.transport.batch_processor - INFO - ā° TIMEOUT TRIGGER: Sending batch due to timeout (5.5s >= 5.0s)\n", - "2025-10-22 23:02:32 - noveum_trace.transport.batch_processor - INFO - šŸ“¤ SENDING BATCH: 4 traces via send_callback\n", - "2025-10-22 23:02:32 - noveum_trace.transport.http_transport - INFO - šŸš€ SENDING BATCH: 4 traces to https://api.noveum.ai/api/v1/traces\n", - "2025-10-22 23:02:33 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: dc0ec2ed-2ab0-4e6f-9005-08c47f6be881) - 1 spans\n", - "2025-10-22 23:02:33 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: dc0ec2ed-2ab0-4e6f-9005-08c47f6be881) - 1 spans\n", - "2025-10-22 23:02:33 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace dc0ec2ed-2ab0-4e6f-9005-08c47f6be881\n", - "2025-10-22 23:02:33 - noveum_trace.transport.http_transport - INFO - āœ… Trace dc0ec2ed-2ab0-4e6f-9005-08c47f6be881 successfully queued for export\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:02:33 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-10-22 23:02:33 - noveum_trace.transport.http_transport - INFO - šŸ“” HTTP RESPONSE: Status 200 from https://api.noveum.ai/api/v1/traces\n", - "2025-10-22 23:02:33 - noveum_trace.transport.http_transport - INFO - āœ… Successfully sent batch of 4 traces\n", - "2025-10-22 23:02:33 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully sent batch of 4 traces via callback\n", - "2025-10-22 23:02:34 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 6be3cc26-2f44-4534-8c17-af00776df15d) - 1 spans\n", - "2025-10-22 23:02:34 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 6be3cc26-2f44-4534-8c17-af00776df15d) - 1 spans\n", - "2025-10-22 23:02:34 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 6be3cc26-2f44-4534-8c17-af00776df15d\n", - "2025-10-22 23:02:34 - noveum_trace.transport.http_transport - INFO - āœ… Trace 6be3cc26-2f44-4534-8c17-af00776df15d successfully queued for export\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:02:34 - INFO - novaeval.evaluators.agent_evaluator - Saving intermediate results after 7 samples\n", - "2025-10-22 23:02:34 - INFO - novaeval.evaluators.agent_evaluator - Intermediate results saved to demo_results/agent.web_search_evaluation_metrics_dataset/agent_evaluation_results.csv\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Evaluating samples: 7it [00:28, 4.12s/it]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:02:34 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-10-22 23:02:36 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: d900cab2-2ff3-4c05-b0c0-7d5ed8303c40) - 1 spans\n", - "2025-10-22 23:02:36 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: d900cab2-2ff3-4c05-b0c0-7d5ed8303c40) - 1 spans\n", - "2025-10-22 23:02:36 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace d900cab2-2ff3-4c05-b0c0-7d5ed8303c40\n", - "2025-10-22 23:02:36 - noveum_trace.transport.http_transport - INFO - āœ… Trace d900cab2-2ff3-4c05-b0c0-7d5ed8303c40 successfully queued for export\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:02:36 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-10-22 23:02:37 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: 2337a5e3-f182-40a2-8172-7b973fd6f67f) - 1 spans\n", - "2025-10-22 23:02:37 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: 2337a5e3-f182-40a2-8172-7b973fd6f67f) - 1 spans\n", - "2025-10-22 23:02:37 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace 2337a5e3-f182-40a2-8172-7b973fd6f67f\n", - "2025-10-22 23:02:37 - noveum_trace.transport.http_transport - INFO - āœ… Trace 2337a5e3-f182-40a2-8172-7b973fd6f67f successfully queued for export\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:02:37 - INFO - google_genai.models - AFC is enabled with max remote calls: 10.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-10-22 23:02:38 - noveum_trace.transport.batch_processor - INFO - ā° TIMEOUT TRIGGER: Sending batch due to timeout (5.4s >= 5.0s)\n", - "2025-10-22 23:02:38 - noveum_trace.transport.batch_processor - INFO - šŸ“¤ SENDING BATCH: 4 traces via send_callback\n", - "2025-10-22 23:02:38 - noveum_trace.transport.http_transport - INFO - šŸš€ SENDING BATCH: 4 traces to https://api.noveum.ai/api/v1/traces\n", - "2025-10-22 23:02:38 - noveum_trace.transport.http_transport - INFO - šŸ“” HTTP RESPONSE: Status 200 from https://api.noveum.ai/api/v1/traces\n", - "2025-10-22 23:02:38 - noveum_trace.transport.http_transport - INFO - āœ… Successfully sent batch of 4 traces\n", - "2025-10-22 23:02:38 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully sent batch of 4 traces via callback\n", - "2025-10-22 23:02:38 - noveum_trace.transport.http_transport - INFO - šŸ“¤ EXPORTING TRACE: auto_trace_generate (ID: f49c469d-893d-4a5a-8cf0-b5112321d143) - 1 spans\n", - "2025-10-22 23:02:38 - noveum_trace.transport.batch_processor - INFO - šŸ“„ ADDING TRACE TO QUEUE: auto_trace_generate (ID: f49c469d-893d-4a5a-8cf0-b5112321d143) - 1 spans\n", - "2025-10-22 23:02:38 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully queued trace f49c469d-893d-4a5a-8cf0-b5112321d143\n", - "2025-10-22 23:02:38 - noveum_trace.transport.http_transport - INFO - āœ… Trace f49c469d-893d-4a5a-8cf0-b5112321d143 successfully queued for export\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:02:38 - INFO - novaeval.evaluators.agent_evaluator - Saving intermediate results after 8 samples\n", - "2025-10-22 23:02:38 - INFO - novaeval.evaluators.agent_evaluator - Intermediate results saved to demo_results/agent.web_search_evaluation_metrics_dataset/agent_evaluation_results.csv\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Evaluating samples: 8it [00:33, 4.15s/it]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-10-22 23:02:38 - INFO - novaeval.evaluators.agent_evaluator - Saving final results\n", - "2025-10-22 23:02:38 - INFO - novaeval.evaluators.agent_evaluator - Reloaded 8 results from CSV\n", - "2025-10-22 23:02:38 - INFO - novaeval.evaluators.agent_evaluator - Agent evaluation completed\n", - "\n", - "āœ… Evaluation completed!\n", - "\n", - "šŸ“Š Results Summary:\n", - " - task_progression: 3.75\n", - " - context_relevancy: 7.84\n", - " - role_adherence: 8.97\n", - " - tool_relevancy: 0.00\n", - " - parameter_correctness: 0.00\n", - "\n", - "šŸ” Individual Scores:\n", - "\n", - " Record 1 (Task: eda4fe22-9a2b-4b73-856b-f4f3309bf719):\n", - " - task_progression: 4.2\n", - " - context_relevancy: 7.8\n", - " - role_adherence: 9.0\n", - " - tool_relevancy: 0.0\n", - " - parameter_correctness: 0.0\n", - "\n", - " Record 2 (Task: 0ffffba1-8a37-443c-8866-d53ffbfa7718):\n", - " - task_progression: 3.8\n", - " - context_relevancy: 7.8\n", - " - role_adherence: 9.0\n", - " - tool_relevancy: 0.0\n", - " - parameter_correctness: 0.0\n", - "\n", - " Record 3 (Task: 43cdf081-4f01-49cd-b566-dbd1619e6cd2):\n", - " - task_progression: 4.2\n", - " - context_relevancy: 7.8\n", - " - role_adherence: 9.0\n", - " - tool_relevancy: 0.0\n", - " - parameter_correctness: 0.0\n", - "\n", - " Record 4 (Task: 9a1983f4-09da-4b53-80e6-38de6878e0e7):\n", - " - task_progression: 2.5\n", - " - context_relevancy: 7.8\n", - " - role_adherence: 9.0\n", - " - tool_relevancy: 0.0\n", - " - parameter_correctness: 0.0\n", - "\n", - " Record 5 (Task: 5d2517e1-220a-429d-9d59-f701bda25eed):\n", - " - task_progression: 2.8\n", - " - context_relevancy: 7.8\n", - " - role_adherence: 9.0\n", - " - tool_relevancy: 0.0\n", - " - parameter_correctness: 0.0\n", - "\n", - " Record 6 (Task: a81ca3a8-80aa-4c39-876e-8d40ea7a0aef):\n", - " - task_progression: 3.8\n", - " - context_relevancy: 7.8\n", - " - role_adherence: 9.0\n", - " - tool_relevancy: 0.0\n", - " - parameter_correctness: 0.0\n", - "\n", - " Record 7 (Task: 230aad27-f3dd-4968-a45a-3c2f07ac28ed):\n", - " - task_progression: 4.2\n", - " - context_relevancy: 7.8\n", - " - role_adherence: 9.0\n", - " - tool_relevancy: 0.0\n", - " - parameter_correctness: 0.0\n", - "\n", - " Record 8 (Task: 83c7dcce-3d89-4da1-8b3f-d419885d4cbc):\n", - " - task_progression: 4.5\n", - " - context_relevancy: 8.1\n", - " - role_adherence: 8.8\n", - " - tool_relevancy: 0.0\n", - " - parameter_correctness: 0.0\n", - "āœ… Evaluation completed successfully!\n", - "\n", - "šŸ“‹ Step 7: Exporting Dataset\n", - "šŸ’¾ Exporting processed dataset...\n", - "āœ… Exported to ./processed_datasets/agent.web_search_evaluation_metrics_dataset_processed_dataset.json\n", - "āœ… Exported to ./processed_datasets/agent.web_search_evaluation_metrics_dataset_processed_dataset.csv\n", - "āœ… Export completed successfully!\n", - "\n", - "============================================================\n", - "šŸŽ‰ EVALUATION PIPELINE COMPLETED!\n", - "šŸ“Š Final Results:\n", - " - File processed: split_datasets/agent.web_search_evaluation_metrics_dataset.json\n", - " - Spans loaded: 8\n", - " - Dataset size: 8\n", - " - Evaluation completed: True\n", - " - Export successful: True\n", - " - Results saved to: ./demo_results/agent.web_search_evaluation_metrics_dataset/\n", - "Completed agent.web_search_evaluation_metrics_dataset.json\n", - "\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-10-22 23:02:43 - noveum_trace.transport.batch_processor - INFO - ā° TIMEOUT TRIGGER: Sending batch due to timeout (5.1s >= 5.0s)\n", - "2025-10-22 23:02:43 - noveum_trace.transport.batch_processor - INFO - šŸ“¤ SENDING BATCH: 1 traces via send_callback\n", - "2025-10-22 23:02:43 - noveum_trace.transport.http_transport - INFO - šŸš€ SENDING BATCH: 1 traces to https://api.noveum.ai/api/v1/traces\n", - "2025-10-22 23:02:44 - noveum_trace.transport.http_transport - INFO - šŸ“” HTTP RESPONSE: Status 200 from https://api.noveum.ai/api/v1/traces\n", - "2025-10-22 23:02:44 - noveum_trace.transport.http_transport - INFO - āœ… Successfully sent batch of 1 traces\n", - "2025-10-22 23:02:44 - noveum_trace.transport.batch_processor - INFO - āœ… Successfully sent batch of 1 traces via callback\n" - ] - } - ], - "source": [ - "from demo_utils import run_complete_agent_evaluation\n", - "import os\n", - "\n", - "# Process all JSON files in split_datasets directory\n", - "for file in os.listdir('split_datasets'):\n", - " if file.endswith('.json'):\n", - " print(f'Processing {file}...')\n", - " run_complete_agent_evaluation(\n", - " f'split_datasets/{file}', \n", - " sample_size=25, \n", - " evaluation_name=file.replace('.json', ''),\n", - " output_dir='./demo_results'\n", - " )\n", - " print(f'Completed {file}\\n')" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "id": "1f5ca2d3", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Created api_data.json with 20 items\n", - "Sample items:\n", - " {'item_key': 'eda4fe22-9a2b-4b73-856b-f4f3309bf719', 'item_id': 'item_1'}\n", - " {'item_key': '0ffffba1-8a37-443c-8866-d53ffbfa7718', 'item_id': 'item_2'}\n", - " {'item_key': 'f1f37bd7-0851-4659-b493-b80d3800d920', 'item_id': 'item_3'}\n" - ] - } - ], - "source": [ - "\n", - "import pandas as pd\n", - "import json\n", - "\n", - "# Read the CSV file\n", - "df = pd.read_csv('demo_results/agent.query_routing_dataset/agent_evaluation_results.csv')\n", - "\n", - "# Create API data structure with all task_ids\n", - "api_data = {\n", - " 'items': [\n", - " {\n", - " 'item_key': str(row['task_id']),\n", - " 'item_id': f'item_{i+1}' # Generate unique item IDs\n", - " }\n", - " for i, row in df.iterrows()\n", - " ]\n", - "}\n", - "\n", - "# Save to JSON\n", - "with open('api_data.json', 'w') as f:\n", - " json.dump(api_data, f, indent=2)\n", - "\n", - "print('Created api_data.json with', len(api_data['items']), 'items')\n", - "print('Sample items:')\n", - "for item in api_data['items'][:3]:\n", - " print(f' {item}')\n" - ] - }, - { - "cell_type": "code", - "execution_count": 16, - "id": "06289a7d", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Loaded environment variables\n", - "Loading API data from api_data.json...\n", - "Loaded 20 item mappings\n", - "Reading CSV from demo_results/agent.query_routing_dataset/agent_evaluation_results.csv...\n", - "Read 20 rows from CSV\n", - "Created 20 results for upload\n", - "\n", - "Uploading 20 results in 1 batches...\n", - "āœ— Batch 1/1 failed: 201\n", - " Response: {\"success\":true,\"created\":20,\"failed\":0,\"results\":[{\"result_id\":\"c92dbfca-8820-45d4-be82-a970e7e45b14\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_1\",\"scorer_id\":\"context_relevancy_scorer\",\"score\":7.5,\"passed\":1,\"metadata\":\"{\\\"details\\\":\\\"The agent's response should provide information about the latest breakthroughs in AI. Without seeing the actual response, it's impossible to confirm alignment with role and task. However, the premise indicates that the content should be factually correct and presented in an informative manner, suggesting an appropriate tone for the given role.\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:01:06.219000000\",\"updated_at\":\"2025-10-22 18:01:06.219000000\",\"scorer_name\":\"Context Relevancy Scorer\"},{\"result_id\":\"0075462f-01a1-45b4-8113-007ae6fcde5d\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_10\",\"scorer_id\":\"context_relevancy_scorer\",\"score\":7.8,\"passed\":1,\"metadata\":\"{\\\"details\\\":\\\"The agent response's direct relevance is confirmed, matching the specified task. The alignment with the designated role is evident, and the tone and content are appropriate. There's a slight lack of demonstrating deeper understanding, which slightly reduces the score, but overall, it's well-executed and helpful.\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:01:06.219000000\",\"updated_at\":\"2025-10-22 18:01:06.219000000\",\"scorer_name\":\"Context Relevancy Scorer\"},{\"result_id\":\"12ce4442-7cff-4e80-ba5b-7c9caa1cf2b0\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_11\",\"scorer_id\":\"context_relevancy_scorer\",\"score\":7.8,\"passed\":1,\"metadata\":\"{\\\"details\\\":\\\"The agent response's direct relevance to the task is good, and it generally aligns with the agent's role. The tone and content are appropriate. However, without seeing the actual response, a slightly higher score is withheld as it's impossible to confirm the response's comprehensive understanding and perfect alignment with the task's requirements.\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:01:06.219000000\",\"updated_at\":\"2025-10-22 18:01:06.219000000\",\"scorer_name\":\"Context Relevancy Scorer\"},{\"result_id\":\"f084daf2-094b-4a0d-98ee-8b67fcba5eec\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_12\",\"scorer_id\":\"context_relevancy_scorer\",\"score\":7.8,\"passed\":1,\"metadata\":\"{\\\"details\\\":\\\"The agent response appears helpful and relevant to the assigned task of configuring Noveum. It's aligned with the agent's role and expertise. The tone and content are appropriate. However, without the actual response, a definitive assessment of its quality and level of detail is impossible; hence, a slightly lower score.\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:01:06.219000000\",\"updated_at\":\"2025-10-22 18:01:06.219000000\",\"scorer_name\":\"Context Relevancy Scorer\"},{\"result_id\":\"ccdd6d5e-d882-4ea1-b957-5a99a62fbaab\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_13\",\"scorer_id\":\"context_relevancy_scorer\",\"score\":7.8,\"passed\":1,\"metadata\":\"{\\\"details\\\":\\\"The agent response's relevance is good given the prompt, and the response is appropriate for the agent's designated role. The content's tone is also aligned with the task, and the response demonstrates good understanding of the task requirements. Further details could enhance the response.\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:01:06.219000000\",\"updated_at\":\"2025-10-22 18:01:06.219000000\",\"scorer_name\":\"Context Relevancy Scorer\"},{\"result_id\":\"f530766b-4b3d-4186-a175-e9220af2bf74\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_14\",\"scorer_id\":\"context_relevancy_scorer\",\"score\":7.8,\"passed\":1,\"metadata\":\"{\\\"details\\\":\\\"The agent response's relevance to the assigned task is clear, and it aligns well with the agent's role. The tone and content are appropriate. There's a solid understanding of the task requirements. The response is helpful. It could gain a higher score if it gives an answer.\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:01:06.219000000\",\"updated_at\":\"2025-10-22 18:01:06.219000000\",\"scorer_name\":\"Context Relevancy Scorer\"},{\"result_id\":\"5cb29b73-a4f1-48b7-b215-f300fb850b51\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_15\",\"scorer_id\":\"context_relevancy_scorer\",\"score\":7.8,\"passed\":1,\"metadata\":\"{\\\"details\\\":\\\"The agent response's relevance to the assigned task is clear, and it aligns well with the agent's specified role. The content and tone are suitable for the role. Demonstrating a more extensive understanding of available APIs in Noveum would improve the score.\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:01:06.219000000\",\"updated_at\":\"2025-10-22 18:01:06.219000000\",\"scorer_name\":\"Context Relevancy Scorer\"},{\"result_id\":\"df6a1c18-72cd-4ecf-a796-2970e989358b\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_16\",\"scorer_id\":\"context_relevancy_scorer\",\"score\":7.5,\"passed\":1,\"metadata\":\"{\\\"details\\\":\\\"The response should directly address setting up observability with Noveum, which is highly relevant to the provided task. The response is likely aligned with the agent's role and expertise. Improvement could include a more helpful response, leading to a higher score.\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:01:06.219000000\",\"updated_at\":\"2025-10-22 18:01:06.219000000\",\"scorer_name\":\"Context Relevancy Scorer\"},{\"result_id\":\"eb68976c-4108-4800-af06-83e31f0ce74c\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_17\",\"scorer_id\":\"context_relevancy_scorer\",\"score\":7.8,\"passed\":1,\"metadata\":\"{\\\"details\\\":\\\"The agent response's direct relevance is strong, fitting the task and agent role well. The content is suitable. While it efficiently addresses the query, there's room to further showcase the agent's expertise and provide a more comprehensive overview of Noveum Trace's features. A more elaborate answer could have improved the score.\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:01:06.219000000\",\"updated_at\":\"2025-10-22 18:01:06.219000000\",\"scorer_name\":\"Context Relevancy Scorer\"},{\"result_id\":\"5d535892-430c-4920-8a08-3a93431b032e\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_18\",\"scorer_id\":\"context_relevancy_scorer\",\"score\":7.8,\"passed\":1,\"metadata\":\"{\\\"details\\\":\\\"The agent response's relevance to the assigned task is clear and aligns well with the agent's role. The tone and content are appropriate. However, without the actual response, it's impossible to gauge the depth of understanding or how helpful it is in completing the task, so the score is slightly adjusted.\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:01:06.219000000\",\"updated_at\":\"2025-10-22 18:01:06.219000000\",\"scorer_name\":\"Context Relevancy Scorer\"},{\"result_id\":\"55439906-553e-4155-87d2-848d453689e2\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_19\",\"scorer_id\":\"context_relevancy_scorer\",\"score\":7.5,\"passed\":1,\"metadata\":\"{\\\"details\\\":\\\"The agent response would likely be directly relevant to the task if there was an actual response to evaluate. The prompt itself is designed to elicit a response about integrating Noveum Trace, aligning with the agent's expected role. Without a response, evaluating tone, content, and task understanding is impossible, but the prompt itself is well-suited.\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:01:06.219000000\",\"updated_at\":\"2025-10-22 18:01:06.219000000\",\"scorer_name\":\"Context Relevancy Scorer\"},{\"result_id\":\"eb624892-309d-4359-bfa2-251303cc7351\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_2\",\"scorer_id\":\"context_relevancy_scorer\",\"score\":7.8,\"passed\":1,\"metadata\":\"{\\\"details\\\":\\\"The agent response's direct relevance and its alignment with the agent's role are satisfactory. The tone and content are appropriate, showing understanding of the task. However, achieving a higher score requires a more detailed and insightful response on current trends, implying a need for better task completion progression.\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:01:06.219000000\",\"updated_at\":\"2025-10-22 18:01:06.219000000\",\"scorer_name\":\"Context Relevancy Scorer\"},{\"result_id\":\"41801712-7f2a-40be-9d29-f614143c0989\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_20\",\"scorer_id\":\"context_relevancy_scorer\",\"score\":7.8,\"passed\":1,\"metadata\":\"{\\\"details\\\":\\\"The response is relevant to the task of defining Noveum. It's well-aligned with the agent's role of providing information. The tone is appropriate. Demonstrating a slightly more detailed understanding of Noveum's function or purpose would elevate the score further, though the foundational elements of task completion are present.\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:01:06.219000000\",\"updated_at\":\"2025-10-22 18:01:06.219000000\",\"scorer_name\":\"Context Relevancy Scorer\"},{\"result_id\":\"f507b5b0-399c-4eb6-befa-af29ff56a48d\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_3\",\"scorer_id\":\"context_relevancy_scorer\",\"score\":7.5,\"passed\":1,\"metadata\":\"{\\\"details\\\":\\\"The response should list the newest features in cloud computing based on the agent's task. It should be aligned with the agent's role as well as the expertise. The tone and content are appropriate for the role. Demonstrating a deeper understanding of the task requirements would be key to increase the score.\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:01:06.219000000\",\"updated_at\":\"2025-10-22 18:01:06.219000000\",\"scorer_name\":\"Context Relevancy Scorer\"},{\"result_id\":\"ff03a298-77c5-4785-8bd1-d5a19038e745\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_4\",\"scorer_id\":\"context_relevancy_scorer\",\"score\":7.5,\"passed\":1,\"metadata\":\"{\\\"details\\\":\\\"The agent's response should provide real-time or recent information regarding the current status of the cryptocurrency markets. Given the task of providing market updates, the agent's response needs to be relevant and aligned with the role of an information provider. The score reflects a solid starting point but can improve through richer details.\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:01:06.219000000\",\"updated_at\":\"2025-10-22 18:01:06.219000000\",\"scorer_name\":\"Context Relevancy Scorer\"},{\"result_id\":\"95f84560-319e-4a2c-ad3c-f0631a1bae56\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_5\",\"scorer_id\":\"context_relevancy_scorer\",\"score\":7.8,\"passed\":1,\"metadata\":\"{\\\"details\\\":\\\"The agent response's direct relevance and role alignment is good. The tone and content are appropriate. However, the score could be higher if the response also demonstrated a helpful level of task understanding by providing specific examples or sources for the latest Python programming updates to aid task completion.\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:01:06.219000000\",\"updated_at\":\"2025-10-22 18:01:06.219000000\",\"scorer_name\":\"Context Relevancy Scorer\"},{\"result_id\":\"3e2efda3-20a2-4f71-a6f8-430f50a454a6\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_6\",\"scorer_id\":\"context_relevancy_scorer\",\"score\":1,\"passed\":1,\"metadata\":\"{\\\"details\\\":\\\"The agent provided no response, which is completely irrelevant and inappropriate. It fails to address the task of summarizing tech news and demonstrates no understanding of the requirements. Therefore, the score is significantly low, reflecting the complete lack of a meaningful answer.\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:01:06.219000000\",\"updated_at\":\"2025-10-22 18:01:06.219000000\",\"scorer_name\":\"Context Relevancy Scorer\"},{\"result_id\":\"99f927fb-447b-4eaa-8413-76932289cc37\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_7\",\"scorer_id\":\"context_relevancy_scorer\",\"score\":7.5,\"passed\":1,\"metadata\":\"{\\\"details\\\":\\\"The agent's response should provide a list of current trends in observability tools. This shows a good understanding of the task. The information should be factual and relevant. Its appropriateness hinges on the quality of the response, specifically the trends mentioned and their relevance in current times. A detailed response will increase the score.\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:01:06.219000000\",\"updated_at\":\"2025-10-22 18:01:06.219000000\",\"scorer_name\":\"Context Relevancy Scorer\"},{\"result_id\":\"a0c16e26-c4d0-4765-afe0-c9a007fd1481\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_8\",\"scorer_id\":\"context_relevancy_scorer\",\"score\":7.5,\"passed\":1,\"metadata\":\"{\\\"details\\\":\\\"The response is relevant to the task of describing recent developments in machine learning, fitting the agent's role. The tone and content are appropriate. However, the depth of the response could be increased to demonstrate a stronger understanding of the current state of the field to get a better score.\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:01:06.219000000\",\"updated_at\":\"2025-10-22 18:01:06.219000000\",\"scorer_name\":\"Context Relevancy Scorer\"},{\"result_id\":\"696c0bee-158d-4134-8785-19503ec01da3\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_9\",\"scorer_id\":\"context_relevancy_scorer\",\"score\":7.8,\"passed\":1,\"metadata\":\"{\\\"details\\\":\\\"The agent response's relevance is good given the weather query. It's appropriately aligned with the role, providing information about the weather. The tone is suitable, indicating task understanding. To improve, it could offer a bit more detail, but overall it is a helpful start.\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:01:06.219000000\",\"updated_at\":\"2025-10-22 18:01:06.219000000\",\"scorer_name\":\"Context Relevancy Scorer\"}]}\n", - "\n", - "Upload complete!\n" - ] - } - ], - "source": [ - "!python upload_scores.py demo_results/agent.query_routing_dataset/agent_evaluation_results.csv --item-key-col task_id --score-col context_relevancy --reasoning-col context_relevancy_reasoning --api-data api_data.json --scorer-id context_relevancy_scorer" - ] - }, - { - "cell_type": "code", - "execution_count": 17, - "id": "cbf96327", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Loaded environment variables\n", - "Loading API data from api_data.json...\n", - "Loaded 20 item mappings\n", - "Reading CSV from demo_results/agent.query_routing_dataset/agent_evaluation_results.csv...\n", - "Read 20 rows from CSV\n", - "Created 20 results for upload\n", - "\n", - "Uploading 20 results in 1 batches...\n", - "āœ— Batch 1/1 failed: 201\n", - " Response: {\"success\":true,\"created\":20,\"failed\":0,\"results\":[{\"result_id\":\"bcc832a3-fda1-4129-8988-5ba5a30ed2f0\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_1\",\"scorer_id\":\"role_adherence_scorer\",\"score\":1,\"passed\":1,\"metadata\":\"{\\\"details\\\":\\\"The agent completely fails to adhere to its role as it provided no response and made no tool calls. There is no evidence of an attempt to answer the question about AI breakthroughs. The agent's lack of any output indicates a total disregard for the task and role instructions.\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:01:48.477000000\",\"updated_at\":\"2025-10-22 18:01:48.477000000\",\"scorer_name\":\"Role Adherence Scorer\"},{\"result_id\":\"7f5cdc48-40bb-4944-ae1d-c4e9d3ace92d\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_10\",\"scorer_id\":\"role_adherence_scorer\",\"score\":1,\"passed\":1,\"metadata\":\"{\\\"details\\\":\\\"The agent completely fails to adhere to its role. It provides no response or tool calls, indicating a complete inability to fulfill its assigned task of providing AI news. It acts inappropriately by failing to respond, showing no understanding or attempt to interact with any available tools.\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:01:48.477000000\",\"updated_at\":\"2025-10-22 18:01:48.477000000\",\"scorer_name\":\"Role Adherence Scorer\"},{\"result_id\":\"a862d4fb-240a-4682-b8f1-2f68eb386142\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_11\",\"scorer_id\":\"role_adherence_scorer\",\"score\":9,\"passed\":1,\"metadata\":\"{\\\"details\\\":\\\"The agent perfectly adheres to its role, providing a concise response about the benefits of using Noveum Trace. The absence of tool calls indicates no deviation from the provided context. The response is consistent, focused, and free of any role contradictions or inappropriate behaviors, demonstrating a strong understanding of its assigned task.\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:01:48.477000000\",\"updated_at\":\"2025-10-22 18:01:48.477000000\",\"scorer_name\":\"Role Adherence Scorer\"},{\"result_id\":\"e8604ecb-b171-45b0-aab3-1fa128c734e4\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_12\",\"scorer_id\":\"role_adherence_scorer\",\"score\":1,\"passed\":1,\"metadata\":\"{\\\"details\\\":\\\"The agent provided no response, and therefore failed to adhere to the role or address the task. It did not provide any information or utilize any tools. The lack of response demonstrates complete failure in understanding and executing the assigned instructions.\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:01:48.477000000\",\"updated_at\":\"2025-10-22 18:01:48.477000000\",\"scorer_name\":\"Role Adherence Scorer\"},{\"result_id\":\"b0ef13bb-bf66-47b9-9458-b5d52d1488fb\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_13\",\"scorer_id\":\"role_adherence_scorer\",\"score\":10,\"passed\":1,\"metadata\":\"{\\\"details\\\":\\\"The agent perfectly adheres to its role and task. The agent's response is relevant to its role and the tool calls are appropriate for this role and its task. The agent doesn't contradict its assigned task. No deviations or inconsistencies are observed.\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:01:48.477000000\",\"updated_at\":\"2025-10-22 18:01:48.477000000\",\"scorer_name\":\"Role Adherence Scorer\"},{\"result_id\":\"10dcd5c4-9fb7-4670-98a6-b858caacd938\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_14\",\"scorer_id\":\"role_adherence_scorer\",\"score\":1,\"passed\":1,\"metadata\":\"{\\\"details\\\":\\\"The agent completely fails to fulfill its role. The agent provides no response and makes no tool calls, indicating a complete disregard for the assigned task. There's no evidence of role adherence, task completion, or relevant information retrieval. The agent is inactive, rendering the evaluation impossible.\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:01:48.477000000\",\"updated_at\":\"2025-10-22 18:01:48.477000000\",\"scorer_name\":\"Role Adherence Scorer\"},{\"result_id\":\"fb9c6145-3e19-493a-808e-884484daf019\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_15\",\"scorer_id\":\"role_adherence_scorer\",\"score\":10,\"passed\":1,\"metadata\":\"{\\\"details\\\":\\\"The agent perfectly adheres to its role and task. The agent correctly identified that there were no tools calls, as it was a prompt and not a request for a specific action. The response is consistent with the expectations of the assigned role.\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:01:48.477000000\",\"updated_at\":\"2025-10-22 18:01:48.477000000\",\"scorer_name\":\"Role Adherence Scorer\"},{\"result_id\":\"0a8d4083-d2fe-4083-b161-e8fddd8f231f\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_16\",\"scorer_id\":\"role_adherence_scorer\",\"score\":1,\"passed\":1,\"metadata\":\"{\\\"details\\\":\\\"The agent provided no response or tool calls, therefore failing to adhere to any aspect of its role and task. It did not attempt to answer the question or make any progress towards the objective. The lack of any action completely contradicts the expectation for a helpful response.\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:01:48.477000000\",\"updated_at\":\"2025-10-22 18:01:48.477000000\",\"scorer_name\":\"Role Adherence Scorer\"},{\"result_id\":\"5dc8fa70-5bab-4697-81ea-745e24680c3d\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_17\",\"scorer_id\":\"role_adherence_scorer\",\"score\":10,\"passed\":1,\"metadata\":\"{\\\"details\\\":\\\"The agent perfectly adheres to its role and task. The agent has not provided a response nor made any tool calls, which is the expected behavior given the prompt which instructed it to evaluate an agent's response. No deviations or contradictions were observed, maintaining complete consistency.\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:01:48.477000000\",\"updated_at\":\"2025-10-22 18:01:48.477000000\",\"scorer_name\":\"Role Adherence Scorer\"},{\"result_id\":\"7c645c45-f5a4-41a3-ab8f-51168c02856d\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_18\",\"scorer_id\":\"role_adherence_scorer\",\"score\":1,\"passed\":1,\"metadata\":\"{\\\"details\\\":\\\"The agent provided no response, and made no tool calls, meaning the agent completely failed to fulfill its role or task. It did not try to answer the question about pricing plans or attempt any data analysis or retrieval. This constitutes a complete failure to adhere to the role.\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:01:48.477000000\",\"updated_at\":\"2025-10-22 18:01:48.477000000\",\"scorer_name\":\"Role Adherence Scorer\"},{\"result_id\":\"ca5a3ea3-7023-4a11-b12e-07bf43d91e4d\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_19\",\"scorer_id\":\"role_adherence_scorer\",\"score\":1,\"passed\":1,\"metadata\":\"{\\\"details\\\":\\\"The agent provided no response, meaning the agent failed at every aspect of its role. It did not provide a response, tool calls, or do anything to accomplish its assigned task. The agent completely ignored all instructions and failed to operate as requested. Therefore, the adherence score is very low.\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:01:48.477000000\",\"updated_at\":\"2025-10-22 18:01:48.477000000\",\"scorer_name\":\"Role Adherence Scorer\"},{\"result_id\":\"497d5631-bb51-415d-84b8-ea1d0669768c\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_2\",\"scorer_id\":\"role_adherence_scorer\",\"score\":10,\"passed\":1,\"metadata\":\"{\\\"details\\\":\\\"The agent perfectly adheres to its role and task. The response is consistent with expectations for the assigned role. The absence of any tool calls is acceptable as there is no requirement to make a tool call in this scenario, therefore the agent has no need to deviate from its role.\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:01:48.477000000\",\"updated_at\":\"2025-10-22 18:01:48.477000000\",\"scorer_name\":\"Role Adherence Scorer\"},{\"result_id\":\"2510458b-3a84-4f67-b3c6-8d55dfb2ba92\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_20\",\"scorer_id\":\"role_adherence_scorer\",\"score\":10,\"passed\":1,\"metadata\":\"{\\\"details\\\":\\\"The agent perfectly adheres to its role and task, maintaining consistency throughout. The absence of any tool calls implies that the agent may have responded directly with the answer to the question. This aligns well with the prompt, suggesting an exact response is expected given the lack of tool usage. No deviations from the role or inappropriate behaviors are observed.\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:01:48.477000000\",\"updated_at\":\"2025-10-22 18:01:48.477000000\",\"scorer_name\":\"Role Adherence Scorer\"},{\"result_id\":\"37bb1928-0b4e-4eff-9c9b-99c374df758a\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_3\",\"scorer_id\":\"role_adherence_scorer\",\"score\":10,\"passed\":1,\"metadata\":\"{\\\"details\\\":\\\"The agent perfectly adheres to its role and task. The absence of tool calls indicates that the agent knows to respond without them. The agent does not stray from the assigned task and provides an expected answer while acting appropriately as an agent.\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:01:48.477000000\",\"updated_at\":\"2025-10-22 18:01:48.477000000\",\"scorer_name\":\"Role Adherence Scorer\"},{\"result_id\":\"56b5f6b9-aa35-4c1e-9380-4f6e42d37ef6\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_4\",\"scorer_id\":\"role_adherence_scorer\",\"score\":10,\"passed\":1,\"metadata\":\"{\\\"details\\\":\\\"The agent perfectly adheres to its role and task. The absence of tool calls indicates no actions were taken, which aligns with the prompt requesting an evaluation without any tool usage. The response, while missing, doesn't deviate from the intended role of reporting current crypto market status. The consistency is flawless.\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:01:48.477000000\",\"updated_at\":\"2025-10-22 18:01:48.477000000\",\"scorer_name\":\"Role Adherence Scorer\"},{\"result_id\":\"00ffa56b-dee2-4798-adac-c9b67c04409b\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_5\",\"scorer_id\":\"role_adherence_scorer\",\"score\":1,\"passed\":1,\"metadata\":\"{\\\"details\\\":\\\"The agent provided no response and made no tool calls. This is a complete failure to adhere to the role and task. The agent should have provided some information, even if it had to use a tool to gather it. The lack of any output demonstrates a fundamental misunderstanding of the prompt's requirements.\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:01:48.477000000\",\"updated_at\":\"2025-10-22 18:01:48.477000000\",\"scorer_name\":\"Role Adherence Scorer\"},{\"result_id\":\"19e7022b-a731-4bb7-ba91-ea1f15acea19\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_6\",\"scorer_id\":\"role_adherence_scorer\",\"score\":1,\"passed\":1,\"metadata\":\"{\\\"details\\\":\\\"The agent completely fails to adhere to its role. It's supposed to summarize tech news but hasn't provided any information or made any tool calls. Its response is blank, and thus it cannot fulfill its assigned task. There are significant role contradictions and inappropriate behaviors due to this complete lack of output.\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:01:48.477000000\",\"updated_at\":\"2025-10-22 18:01:48.477000000\",\"scorer_name\":\"Role Adherence Scorer\"},{\"result_id\":\"1855df2f-4341-4e34-b02b-70b4f0844690\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_7\",\"scorer_id\":\"role_adherence_scorer\",\"score\":1,\"passed\":1,\"metadata\":\"{\\\"details\\\":\\\"The agent completely fails to adhere to its assigned role or perform the task. No tool calls were made, indicating a complete lack of engagement with the prompt. There is no information provided about the current trends in observability tools, directly contradicting the instructions. The agent fails at all aspects of the evaluation.\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:01:48.477000000\",\"updated_at\":\"2025-10-22 18:01:48.477000000\",\"scorer_name\":\"Role Adherence Scorer\"},{\"result_id\":\"13771286-0139-4c43-9287-c3290f402da1\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_8\",\"scorer_id\":\"role_adherence_scorer\",\"score\":10,\"passed\":1,\"metadata\":\"{\\\"details\\\":\\\"The agent perfectly adheres to its role and task. The response is consistent with the role of machine learning expert. The agent appropriately answers the prompt, delivering a relevant overview. No tool calls were made, suggesting that the agent could answer the query without external assistance, which aligns with perfect role adherence.\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:01:48.477000000\",\"updated_at\":\"2025-10-22 18:01:48.477000000\",\"scorer_name\":\"Role Adherence Scorer\"},{\"result_id\":\"a66d4346-5a43-4b06-a269-216bcc40f03e\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_9\",\"scorer_id\":\"role_adherence_scorer\",\"score\":10,\"passed\":1,\"metadata\":\"{\\\"details\\\":\\\"The agent perfectly adheres to its role and task. The agent correctly executes its role and the tool calls are appropriate for retrieving information about the weather, as intended. The response is consistent with the role and task requirements, showing no contradictions or inappropriate behaviors. The agent demonstrates flawless adherence.\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:01:48.477000000\",\"updated_at\":\"2025-10-22 18:01:48.477000000\",\"scorer_name\":\"Role Adherence Scorer\"}]}\n", - "\n", - "Upload complete!\n" - ] - } - ], - "source": [ - "!python upload_scores.py demo_results/agent.query_routing_dataset/agent_evaluation_results.csv --item-key-col task_id --score-col role_adherence --reasoning-col role_adherence_reasoning --api-data api_data.json --scorer-id role_adherence_scorer" - ] - }, - { - "cell_type": "code", - "execution_count": 20, - "id": "08f948b3", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Loaded environment variables\n", - "Loading API data from api_data.json...\n", - "Loaded 20 item mappings\n", - "Reading CSV from demo_results/agent.query_routing_dataset/agent_evaluation_results.csv...\n", - "Read 20 rows from CSV\n", - "Created 20 results for upload\n", - "\n", - "Uploading 20 results in 1 batches...\n", - "āœ— Batch 1/1 failed: 201\n", - " Response: {\"success\":true,\"created\":20,\"failed\":0,\"results\":[{\"result_id\":\"35b42464-7989-4caa-90cb-5636319d4fd1\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_1\",\"scorer_id\":\"parameter_correctness_scorer\",\"score\":0,\"passed\":0,\"metadata\":\"{\\\"details\\\":\\\"Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:03:01.353000000\",\"updated_at\":\"2025-10-22 18:03:01.353000000\",\"scorer_name\":\"Parameter Correctness Scorer\"},{\"result_id\":\"28380f39-f4e4-41a3-a5c7-01241174fb78\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_10\",\"scorer_id\":\"parameter_correctness_scorer\",\"score\":0,\"passed\":0,\"metadata\":\"{\\\"details\\\":\\\"Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:03:01.353000000\",\"updated_at\":\"2025-10-22 18:03:01.353000000\",\"scorer_name\":\"Parameter Correctness Scorer\"},{\"result_id\":\"f1ea8545-c1ab-4791-958b-04d3c75cf92a\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_11\",\"scorer_id\":\"parameter_correctness_scorer\",\"score\":0,\"passed\":0,\"metadata\":\"{\\\"details\\\":\\\"Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:03:01.353000000\",\"updated_at\":\"2025-10-22 18:03:01.353000000\",\"scorer_name\":\"Parameter Correctness Scorer\"},{\"result_id\":\"db8c7724-19ca-46c6-8c6b-796f229279e4\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_12\",\"scorer_id\":\"parameter_correctness_scorer\",\"score\":0,\"passed\":0,\"metadata\":\"{\\\"details\\\":\\\"Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:03:01.353000000\",\"updated_at\":\"2025-10-22 18:03:01.353000000\",\"scorer_name\":\"Parameter Correctness Scorer\"},{\"result_id\":\"86929a3d-38c7-4bcf-8c40-820f3d9e69ef\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_13\",\"scorer_id\":\"parameter_correctness_scorer\",\"score\":0,\"passed\":0,\"metadata\":\"{\\\"details\\\":\\\"Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:03:01.353000000\",\"updated_at\":\"2025-10-22 18:03:01.353000000\",\"scorer_name\":\"Parameter Correctness Scorer\"},{\"result_id\":\"068aaa68-b454-47db-854d-c09b87d33ec9\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_14\",\"scorer_id\":\"parameter_correctness_scorer\",\"score\":0,\"passed\":0,\"metadata\":\"{\\\"details\\\":\\\"Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:03:01.353000000\",\"updated_at\":\"2025-10-22 18:03:01.353000000\",\"scorer_name\":\"Parameter Correctness Scorer\"},{\"result_id\":\"b80aac89-5007-412c-a372-d7a80703ec0f\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_15\",\"scorer_id\":\"parameter_correctness_scorer\",\"score\":0,\"passed\":0,\"metadata\":\"{\\\"details\\\":\\\"Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:03:01.353000000\",\"updated_at\":\"2025-10-22 18:03:01.353000000\",\"scorer_name\":\"Parameter Correctness Scorer\"},{\"result_id\":\"07585067-3e96-4491-8f4b-a4fa89b80493\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_16\",\"scorer_id\":\"parameter_correctness_scorer\",\"score\":0,\"passed\":0,\"metadata\":\"{\\\"details\\\":\\\"Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:03:01.353000000\",\"updated_at\":\"2025-10-22 18:03:01.353000000\",\"scorer_name\":\"Parameter Correctness Scorer\"},{\"result_id\":\"172395b1-bd1d-43dd-9359-51854355376b\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_17\",\"scorer_id\":\"parameter_correctness_scorer\",\"score\":0,\"passed\":0,\"metadata\":\"{\\\"details\\\":\\\"Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:03:01.353000000\",\"updated_at\":\"2025-10-22 18:03:01.353000000\",\"scorer_name\":\"Parameter Correctness Scorer\"},{\"result_id\":\"a54ae11d-5232-4434-b704-1a5f8b327ee7\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_18\",\"scorer_id\":\"parameter_correctness_scorer\",\"score\":0,\"passed\":0,\"metadata\":\"{\\\"details\\\":\\\"Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:03:01.353000000\",\"updated_at\":\"2025-10-22 18:03:01.353000000\",\"scorer_name\":\"Parameter Correctness Scorer\"},{\"result_id\":\"61847a0c-de20-4e36-abd0-2067bb210207\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_19\",\"scorer_id\":\"parameter_correctness_scorer\",\"score\":0,\"passed\":0,\"metadata\":\"{\\\"details\\\":\\\"Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:03:01.353000000\",\"updated_at\":\"2025-10-22 18:03:01.353000000\",\"scorer_name\":\"Parameter Correctness Scorer\"},{\"result_id\":\"70f241a7-05cc-47df-8bc3-2eb3b428d064\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_2\",\"scorer_id\":\"parameter_correctness_scorer\",\"score\":0,\"passed\":0,\"metadata\":\"{\\\"details\\\":\\\"Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:03:01.353000000\",\"updated_at\":\"2025-10-22 18:03:01.353000000\",\"scorer_name\":\"Parameter Correctness Scorer\"},{\"result_id\":\"1b5c2018-e801-4ac1-98e9-8dab56fa19ed\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_20\",\"scorer_id\":\"parameter_correctness_scorer\",\"score\":0,\"passed\":0,\"metadata\":\"{\\\"details\\\":\\\"Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:03:01.353000000\",\"updated_at\":\"2025-10-22 18:03:01.353000000\",\"scorer_name\":\"Parameter Correctness Scorer\"},{\"result_id\":\"65aa0719-a555-4fae-8203-d3f06bcd0f5e\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_3\",\"scorer_id\":\"parameter_correctness_scorer\",\"score\":0,\"passed\":0,\"metadata\":\"{\\\"details\\\":\\\"Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:03:01.353000000\",\"updated_at\":\"2025-10-22 18:03:01.353000000\",\"scorer_name\":\"Parameter Correctness Scorer\"},{\"result_id\":\"5e710451-f3d1-4d35-8ed2-8c6b54a46ccb\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_4\",\"scorer_id\":\"parameter_correctness_scorer\",\"score\":0,\"passed\":0,\"metadata\":\"{\\\"details\\\":\\\"Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:03:01.353000000\",\"updated_at\":\"2025-10-22 18:03:01.353000000\",\"scorer_name\":\"Parameter Correctness Scorer\"},{\"result_id\":\"1db2a945-e30a-43ee-b591-c98dbe004f47\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_5\",\"scorer_id\":\"parameter_correctness_scorer\",\"score\":0,\"passed\":0,\"metadata\":\"{\\\"details\\\":\\\"Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:03:01.353000000\",\"updated_at\":\"2025-10-22 18:03:01.353000000\",\"scorer_name\":\"Parameter Correctness Scorer\"},{\"result_id\":\"297fadf1-51fa-43e7-91cc-58b783550d3b\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_6\",\"scorer_id\":\"parameter_correctness_scorer\",\"score\":0,\"passed\":0,\"metadata\":\"{\\\"details\\\":\\\"Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:03:01.353000000\",\"updated_at\":\"2025-10-22 18:03:01.353000000\",\"scorer_name\":\"Parameter Correctness Scorer\"},{\"result_id\":\"f00da0b9-f3df-4055-9507-0b4b9ae31501\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_7\",\"scorer_id\":\"parameter_correctness_scorer\",\"score\":0,\"passed\":0,\"metadata\":\"{\\\"details\\\":\\\"Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:03:01.353000000\",\"updated_at\":\"2025-10-22 18:03:01.353000000\",\"scorer_name\":\"Parameter Correctness Scorer\"},{\"result_id\":\"3f9733b7-23b3-4396-84a6-5f87146cb0a2\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_8\",\"scorer_id\":\"parameter_correctness_scorer\",\"score\":0,\"passed\":0,\"metadata\":\"{\\\"details\\\":\\\"Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:03:01.353000000\",\"updated_at\":\"2025-10-22 18:03:01.353000000\",\"scorer_name\":\"Parameter Correctness Scorer\"},{\"result_id\":\"c6629883-f239-4b64-ba54-2fe517d25551\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_9\",\"scorer_id\":\"parameter_correctness_scorer\",\"score\":0,\"passed\":0,\"metadata\":\"{\\\"details\\\":\\\"Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:03:01.353000000\",\"updated_at\":\"2025-10-22 18:03:01.353000000\",\"scorer_name\":\"Parameter Correctness Scorer\"}]}\n", - "\n", - "Upload complete!\n" + "Upload complete!\n" ] } ], "source": [ "!python upload_scores.py demo_results/agent.query_routing_dataset/agent_evaluation_results.csv --item-key-col task_id --score-col parameter_correctness --reasoning-col parameter_correctness_reasoning --api-data api_data.json --scorer-id parameter_correctness_scorer" ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "83466df7", - "metadata": {}, - "outputs": [], - "source": [] } ], "metadata": { diff --git a/noveum_customer_support_bt/requirements.txt b/noveum_customer_support_bt/requirements.txt new file mode 100644 index 0000000..0961865 --- /dev/null +++ b/noveum_customer_support_bt/requirements.txt @@ -0,0 +1,32 @@ +# Core dependencies +pydantic>=2.0.0 +pyyaml>=6.0 +requests>=2.28.0 +numpy>=1.21.0 +pandas>=1.3.0 +tqdm>=4.64.0 +click>=8.0.0 +rich>=12.0.0 +jinja2>=3.0.0 +plotly>=5.0.0 +scikit-learn>=1.0.0 +ijson>=3.2.0 +python-dotenv>=1.0.0 +noveum-trace>=0.3.5 +typing_extensions>=4.7.0 + +# Dataset and model dependencies +datasets>=2.0.0 +transformers>=4.20.0 +openai>=1.0.0 +anthropic>=0.3.0 +boto3>=1.26.0 +sentence-transformers>=2.2.0 +ollama==0.5.3 + +# Optional dependencies for specific features +# tiktoken>=0.4.0 # For accurate OpenAI token counting +# evaluate>=0.4.0 # For additional metrics +# rouge-score>=0.1.2 # For ROUGE metrics +# bert-score>=0.3.13 # For BERTScore +# sacrebleu>=2.3.0 # For BLEU metrics From 6e5981e87a87b464706526de5278637883993708 Mon Sep 17 00:00:00 2001 From: Aman India Date: Thu, 23 Oct 2025 12:12:35 +0530 Subject: [PATCH 3/4] Add Jupyter notebook for Noveum AI Agent with RAG and Web Search capabilities; remove requirements.txt; update README with additional instructions. --- noveum_customer_support_bt/README_workflow.md | 1 + ...er_bot.ipynb => e2e_noveum_user_bot.ipynb} | 118 ++++++------------ noveum_customer_support_bt/requirements.txt | 32 ----- 3 files changed, 40 insertions(+), 111 deletions(-) rename noveum_customer_support_bt/{noveum_user_bot.ipynb => e2e_noveum_user_bot.ipynb} (88%) delete mode 100644 noveum_customer_support_bt/requirements.txt diff --git a/noveum_customer_support_bt/README_workflow.md b/noveum_customer_support_bt/README_workflow.md index 054f833..db7a8b8 100644 --- a/noveum_customer_support_bt/README_workflow.md +++ b/noveum_customer_support_bt/README_workflow.md @@ -78,3 +78,4 @@ Make sure your .env file contains: - If upload fails, verify the JSON format matches expected schema - Use --pretty flag for better formatted output - Check API responses for specific error messages + diff --git a/noveum_customer_support_bt/noveum_user_bot.ipynb b/noveum_customer_support_bt/e2e_noveum_user_bot.ipynb similarity index 88% rename from noveum_customer_support_bt/noveum_user_bot.ipynb rename to noveum_customer_support_bt/e2e_noveum_user_bot.ipynb index f4a20ba..59871f6 100644 --- a/noveum_customer_support_bt/noveum_user_bot.ipynb +++ b/noveum_customer_support_bt/e2e_noveum_user_bot.ipynb @@ -7,8 +7,8 @@ "metadata": {}, "outputs": [], "source": [ - "\n", - "!cd noveum_customer_support_bt && source .env && python upload_dataset.py \n" + "# Fix: Upload Dataset with Proper Environment Loading\n", + "!cd /Users/mramanindia/work/NovaEval/noveum_customer_support_bt && source .env && python upload_dataset.py --dataset-json split_datasets/agent.rag_evaluation_metrics_dataset.json --item-type conversation\n" ] }, { @@ -211,7 +211,7 @@ { "cell_type": "code", "execution_count": null, - "id": "563ea15c", + "id": "e27db4ec", "metadata": {}, "outputs": [], "source": [ @@ -265,19 +265,21 @@ ] }, { - "cell_type": "markdown", - "id": "a7677fe0", + "cell_type": "code", + "execution_count": null, + "id": "2209736e", "metadata": {}, + "outputs": [], "source": [ - "## These are required for the project\n", "\n", - "- set openai api key\n", - "- set gemini api key\n", - "- set noveum api key\n", - "- set environment\n", - "- set project'\n", "\n", - "\n" + "## set openai api key\n", + "## set gemini api key\n", + "## set noveum api key\n", + "## set environment\n", + "## set project'\n", + "\n", + "# These are required for the project\n" ] }, { @@ -1534,7 +1536,7 @@ }, { "cell_type": "code", - "execution_count": 54, + "execution_count": null, "id": "a865a57f", "metadata": {}, "outputs": [ @@ -1809,7 +1811,7 @@ { "cell_type": "code", "execution_count": null, - "id": "5645fc74", + "id": "eed0fd46", "metadata": {}, "outputs": [], "source": [ @@ -1827,7 +1829,7 @@ { "cell_type": "code", "execution_count": null, - "id": "bfadf5f1", + "id": "244c76d1", "metadata": {}, "outputs": [], "source": [ @@ -1839,7 +1841,7 @@ { "cell_type": "code", "execution_count": null, - "id": "eb4628eb", + "id": "d9fe279c", "metadata": {}, "outputs": [], "source": [ @@ -1857,18 +1859,7 @@ { "cell_type": "code", "execution_count": null, - "id": "437d8751", - "metadata": {}, - "outputs": [], - "source": [ - "!python preprocess_filter.py ./traces/traces/dataset.json\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "30f02b6c", + "id": "4b68d30e", "metadata": {}, "outputs": [], "source": [ @@ -1878,7 +1869,7 @@ { "cell_type": "code", "execution_count": null, - "id": "d10b392f", + "id": "9e242af4", "metadata": {}, "outputs": [], "source": [ @@ -1897,7 +1888,7 @@ { "cell_type": "code", "execution_count": null, - "id": "0b22eb74", + "id": "770777e3", "metadata": {}, "outputs": [], "source": [ @@ -1912,31 +1903,10 @@ }, { "cell_type": "code", - "execution_count": 1, - "id": "9f7feb8d", + "execution_count": null, + "id": "9358c590", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Creating dataset version at: https://noveum.ai/api/v1/datasets/customersupportagentdemo-new/versions?organizationSlug=magic-api\n", - "Organization: magic-api\n", - "Dataset: customersupportagentdemo-new\n", - "Version: 0.0.2\n", - "Successfully created dataset version\n", - "Response status: 201\n", - "\n", - "Response saved to: dataset_version_response.json\n", - "\n", - "Response data:\n", - "{\n", - " \"success\": true,\n", - " \"version\": \"0.0.2\"\n", - "}\n" - ] - } - ], + "outputs": [], "source": [ "# 3. Create Version\n", "!python create_dataset_version.py --pretty\n" @@ -1945,10 +1915,11 @@ { "cell_type": "code", "execution_count": null, - "id": "b3eeb124", + "id": "224276f1", "metadata": {}, "outputs": [], "source": [ + "# Getting scores\n", "from demo_utils import run_complete_agent_evaluation\n", "import os\n", "\n", @@ -2015,7 +1986,7 @@ { "cell_type": "code", "execution_count": null, - "id": "eb3eb1ba", + "id": "529a4eed", "metadata": {}, "outputs": [], "source": [ @@ -2025,7 +1996,7 @@ { "cell_type": "code", "execution_count": null, - "id": "d511e96b", + "id": "c67e5df7", "metadata": {}, "outputs": [], "source": [ @@ -2034,32 +2005,21 @@ }, { "cell_type": "code", - "execution_count": 20, - "id": "08f948b3", + "execution_count": null, + "id": "d84c0615", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Loaded environment variables\n", - "Loading API data from api_data.json...\n", - "Loaded 20 item mappings\n", - "Reading CSV from demo_results/agent.query_routing_dataset/agent_evaluation_results.csv...\n", - "Read 20 rows from CSV\n", - "Created 20 results for upload\n", - "\n", - "Uploading 20 results in 1 batches...\n", - "āœ— Batch 1/1 failed: 201\n", - " Response: {\"success\":true,\"created\":20,\"failed\":0,\"results\":[{\"result_id\":\"35b42464-7989-4caa-90cb-5636319d4fd1\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_1\",\"scorer_id\":\"parameter_correctness_scorer\",\"score\":0,\"passed\":0,\"metadata\":\"{\\\"details\\\":\\\"Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:03:01.353000000\",\"updated_at\":\"2025-10-22 18:03:01.353000000\",\"scorer_name\":\"Parameter Correctness Scorer\"},{\"result_id\":\"28380f39-f4e4-41a3-a5c7-01241174fb78\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_10\",\"scorer_id\":\"parameter_correctness_scorer\",\"score\":0,\"passed\":0,\"metadata\":\"{\\\"details\\\":\\\"Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:03:01.353000000\",\"updated_at\":\"2025-10-22 18:03:01.353000000\",\"scorer_name\":\"Parameter Correctness Scorer\"},{\"result_id\":\"f1ea8545-c1ab-4791-958b-04d3c75cf92a\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_11\",\"scorer_id\":\"parameter_correctness_scorer\",\"score\":0,\"passed\":0,\"metadata\":\"{\\\"details\\\":\\\"Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:03:01.353000000\",\"updated_at\":\"2025-10-22 18:03:01.353000000\",\"scorer_name\":\"Parameter Correctness Scorer\"},{\"result_id\":\"db8c7724-19ca-46c6-8c6b-796f229279e4\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_12\",\"scorer_id\":\"parameter_correctness_scorer\",\"score\":0,\"passed\":0,\"metadata\":\"{\\\"details\\\":\\\"Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:03:01.353000000\",\"updated_at\":\"2025-10-22 18:03:01.353000000\",\"scorer_name\":\"Parameter Correctness Scorer\"},{\"result_id\":\"86929a3d-38c7-4bcf-8c40-820f3d9e69ef\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_13\",\"scorer_id\":\"parameter_correctness_scorer\",\"score\":0,\"passed\":0,\"metadata\":\"{\\\"details\\\":\\\"Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:03:01.353000000\",\"updated_at\":\"2025-10-22 18:03:01.353000000\",\"scorer_name\":\"Parameter Correctness Scorer\"},{\"result_id\":\"068aaa68-b454-47db-854d-c09b87d33ec9\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_14\",\"scorer_id\":\"parameter_correctness_scorer\",\"score\":0,\"passed\":0,\"metadata\":\"{\\\"details\\\":\\\"Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:03:01.353000000\",\"updated_at\":\"2025-10-22 18:03:01.353000000\",\"scorer_name\":\"Parameter Correctness Scorer\"},{\"result_id\":\"b80aac89-5007-412c-a372-d7a80703ec0f\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_15\",\"scorer_id\":\"parameter_correctness_scorer\",\"score\":0,\"passed\":0,\"metadata\":\"{\\\"details\\\":\\\"Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:03:01.353000000\",\"updated_at\":\"2025-10-22 18:03:01.353000000\",\"scorer_name\":\"Parameter Correctness Scorer\"},{\"result_id\":\"07585067-3e96-4491-8f4b-a4fa89b80493\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_16\",\"scorer_id\":\"parameter_correctness_scorer\",\"score\":0,\"passed\":0,\"metadata\":\"{\\\"details\\\":\\\"Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:03:01.353000000\",\"updated_at\":\"2025-10-22 18:03:01.353000000\",\"scorer_name\":\"Parameter Correctness Scorer\"},{\"result_id\":\"172395b1-bd1d-43dd-9359-51854355376b\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_17\",\"scorer_id\":\"parameter_correctness_scorer\",\"score\":0,\"passed\":0,\"metadata\":\"{\\\"details\\\":\\\"Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:03:01.353000000\",\"updated_at\":\"2025-10-22 18:03:01.353000000\",\"scorer_name\":\"Parameter Correctness Scorer\"},{\"result_id\":\"a54ae11d-5232-4434-b704-1a5f8b327ee7\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_18\",\"scorer_id\":\"parameter_correctness_scorer\",\"score\":0,\"passed\":0,\"metadata\":\"{\\\"details\\\":\\\"Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:03:01.353000000\",\"updated_at\":\"2025-10-22 18:03:01.353000000\",\"scorer_name\":\"Parameter Correctness Scorer\"},{\"result_id\":\"61847a0c-de20-4e36-abd0-2067bb210207\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_19\",\"scorer_id\":\"parameter_correctness_scorer\",\"score\":0,\"passed\":0,\"metadata\":\"{\\\"details\\\":\\\"Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:03:01.353000000\",\"updated_at\":\"2025-10-22 18:03:01.353000000\",\"scorer_name\":\"Parameter Correctness Scorer\"},{\"result_id\":\"70f241a7-05cc-47df-8bc3-2eb3b428d064\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_2\",\"scorer_id\":\"parameter_correctness_scorer\",\"score\":0,\"passed\":0,\"metadata\":\"{\\\"details\\\":\\\"Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:03:01.353000000\",\"updated_at\":\"2025-10-22 18:03:01.353000000\",\"scorer_name\":\"Parameter Correctness Scorer\"},{\"result_id\":\"1b5c2018-e801-4ac1-98e9-8dab56fa19ed\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_20\",\"scorer_id\":\"parameter_correctness_scorer\",\"score\":0,\"passed\":0,\"metadata\":\"{\\\"details\\\":\\\"Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:03:01.353000000\",\"updated_at\":\"2025-10-22 18:03:01.353000000\",\"scorer_name\":\"Parameter Correctness Scorer\"},{\"result_id\":\"65aa0719-a555-4fae-8203-d3f06bcd0f5e\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_3\",\"scorer_id\":\"parameter_correctness_scorer\",\"score\":0,\"passed\":0,\"metadata\":\"{\\\"details\\\":\\\"Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:03:01.353000000\",\"updated_at\":\"2025-10-22 18:03:01.353000000\",\"scorer_name\":\"Parameter Correctness Scorer\"},{\"result_id\":\"5e710451-f3d1-4d35-8ed2-8c6b54a46ccb\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_4\",\"scorer_id\":\"parameter_correctness_scorer\",\"score\":0,\"passed\":0,\"metadata\":\"{\\\"details\\\":\\\"Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:03:01.353000000\",\"updated_at\":\"2025-10-22 18:03:01.353000000\",\"scorer_name\":\"Parameter Correctness Scorer\"},{\"result_id\":\"1db2a945-e30a-43ee-b591-c98dbe004f47\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_5\",\"scorer_id\":\"parameter_correctness_scorer\",\"score\":0,\"passed\":0,\"metadata\":\"{\\\"details\\\":\\\"Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:03:01.353000000\",\"updated_at\":\"2025-10-22 18:03:01.353000000\",\"scorer_name\":\"Parameter Correctness Scorer\"},{\"result_id\":\"297fadf1-51fa-43e7-91cc-58b783550d3b\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_6\",\"scorer_id\":\"parameter_correctness_scorer\",\"score\":0,\"passed\":0,\"metadata\":\"{\\\"details\\\":\\\"Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:03:01.353000000\",\"updated_at\":\"2025-10-22 18:03:01.353000000\",\"scorer_name\":\"Parameter Correctness Scorer\"},{\"result_id\":\"f00da0b9-f3df-4055-9507-0b4b9ae31501\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_7\",\"scorer_id\":\"parameter_correctness_scorer\",\"score\":0,\"passed\":0,\"metadata\":\"{\\\"details\\\":\\\"Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:03:01.353000000\",\"updated_at\":\"2025-10-22 18:03:01.353000000\",\"scorer_name\":\"Parameter Correctness Scorer\"},{\"result_id\":\"3f9733b7-23b3-4396-84a6-5f87146cb0a2\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_8\",\"scorer_id\":\"parameter_correctness_scorer\",\"score\":0,\"passed\":0,\"metadata\":\"{\\\"details\\\":\\\"Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:03:01.353000000\",\"updated_at\":\"2025-10-22 18:03:01.353000000\",\"scorer_name\":\"Parameter Correctness Scorer\"},{\"result_id\":\"c6629883-f239-4b64-ba54-2fe517d25551\",\"organization_slug\":\"magic-api\",\"dataset_slug\":\"customersupportagentdemo-new\",\"item_id\":\"item_9\",\"scorer_id\":\"parameter_correctness_scorer\",\"score\":0,\"passed\":0,\"metadata\":\"{\\\"details\\\":\\\"Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']\\\"}\",\"error\":\"\",\"execution_time_ms\":0,\"created_at\":\"2025-10-22 18:03:01.353000000\",\"updated_at\":\"2025-10-22 18:03:01.353000000\",\"scorer_name\":\"Parameter Correctness Scorer\"}]}\n", - "\n", - "Upload complete!\n" - ] - } - ], + "outputs": [], "source": [ "!python upload_scores.py demo_results/agent.query_routing_dataset/agent_evaluation_results.csv --item-key-col task_id --score-col parameter_correctness --reasoning-col parameter_correctness_reasoning --api-data api_data.json --scorer-id parameter_correctness_scorer" ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "83466df7", + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { diff --git a/noveum_customer_support_bt/requirements.txt b/noveum_customer_support_bt/requirements.txt deleted file mode 100644 index 0961865..0000000 --- a/noveum_customer_support_bt/requirements.txt +++ /dev/null @@ -1,32 +0,0 @@ -# Core dependencies -pydantic>=2.0.0 -pyyaml>=6.0 -requests>=2.28.0 -numpy>=1.21.0 -pandas>=1.3.0 -tqdm>=4.64.0 -click>=8.0.0 -rich>=12.0.0 -jinja2>=3.0.0 -plotly>=5.0.0 -scikit-learn>=1.0.0 -ijson>=3.2.0 -python-dotenv>=1.0.0 -noveum-trace>=0.3.5 -typing_extensions>=4.7.0 - -# Dataset and model dependencies -datasets>=2.0.0 -transformers>=4.20.0 -openai>=1.0.0 -anthropic>=0.3.0 -boto3>=1.26.0 -sentence-transformers>=2.2.0 -ollama==0.5.3 - -# Optional dependencies for specific features -# tiktoken>=0.4.0 # For accurate OpenAI token counting -# evaluate>=0.4.0 # For additional metrics -# rouge-score>=0.1.2 # For ROUGE metrics -# bert-score>=0.3.13 # For BERTScore -# sacrebleu>=2.3.0 # For BLEU metrics From 17aceda75ea9cbaf616ce888e1b5336deebeccb9 Mon Sep 17 00:00:00 2001 From: Aman India Date: Thu, 23 Oct 2025 12:14:13 +0530 Subject: [PATCH 4/4] Remove deprecated evaluation result datasets and related files for agent performance metrics, including CSV and JSON formats across various evaluation types. --- .../agent_evaluation_results.csv | 21 - .../agent_evaluation_results.csv | 13 - .../agent_evaluation_results.csv | 9 - .../noveum_vectorstore/index.faiss | Bin 1130541 -> 0 bytes .../noveum_vectorstore/index.pkl | Bin 188134 -> 0 bytes ...gent.llm-rag_dataset_processed_dataset.csv | 13 - ...ent.llm-rag_dataset_processed_dataset.json | 266 - ...el_execution_dataset_processed_dataset.csv | 21 - ...l_execution_dataset_processed_dataset.json | 442 - ...uery_routing_dataset_processed_dataset.csv | 21 - ...ery_routing_dataset_processed_dataset.json | 622 - ...tion_metrics_dataset_processed_dataset.csv | 22 - ...ion_metrics_dataset_processed_dataset.json | 446 - ...tion_metrics_dataset_processed_dataset.csv | 21 - ...ion_metrics_dataset_processed_dataset.json | 442 - ...tion_metrics_dataset_processed_dataset.csv | 14 - ...ion_metrics_dataset_processed_dataset.json | 298 - ...h_generation_dataset_processed_dataset.csv | 9 - ..._generation_dataset_processed_dataset.json | 178 - ...-orchestator_dataset_processed_dataset.csv | 21 - ...orchestator_dataset_processed_dataset.json | 442 - .../split_datasets/agent.llm-rag_dataset.json | 734 - .../agent.llm_model_execution_dataset.json | 1782 -- .../agent.query_routing_dataset.json | 1222 - .../agent.rag_evaluation_metrics_dataset.json | 998 - ...nt.routing_evaluation_metrics_dataset.json | 1602 -- ...web_search_evaluation_metrics_dataset.json | 682 - .../agent.web_search_generation_dataset.json | 490 - .../tool-orchestator_dataset.json | 1162 - .../traces/traces/dataset.json | 20904 ---------------- .../traces/traces/dataset_filtered.json | 20904 ---------------- .../traces/dataset_filtered_mapped.json | 20904 ---------------- .../traces/traces/traces_batch_001.json | 13915 ---------- 33 files changed, 88620 deletions(-) delete mode 100644 noveum_customer_support_bt/demo_results/agent.query_routing_dataset/agent_evaluation_results.csv delete mode 100644 noveum_customer_support_bt/demo_results/agent.rag_evaluation_metrics_dataset/agent_evaluation_results.csv delete mode 100644 noveum_customer_support_bt/demo_results/agent.web_search_evaluation_metrics_dataset/agent_evaluation_results.csv delete mode 100644 noveum_customer_support_bt/noveum_vectorstore/index.faiss delete mode 100644 noveum_customer_support_bt/noveum_vectorstore/index.pkl delete mode 100644 noveum_customer_support_bt/processed_datasets/agent.llm-rag_dataset_processed_dataset.csv delete mode 100644 noveum_customer_support_bt/processed_datasets/agent.llm-rag_dataset_processed_dataset.json delete mode 100644 noveum_customer_support_bt/processed_datasets/agent.llm_model_execution_dataset_processed_dataset.csv delete mode 100644 noveum_customer_support_bt/processed_datasets/agent.llm_model_execution_dataset_processed_dataset.json delete mode 100644 noveum_customer_support_bt/processed_datasets/agent.query_routing_dataset_processed_dataset.csv delete mode 100644 noveum_customer_support_bt/processed_datasets/agent.query_routing_dataset_processed_dataset.json delete mode 100644 noveum_customer_support_bt/processed_datasets/agent.rag_evaluation_metrics_dataset_processed_dataset.csv delete mode 100644 noveum_customer_support_bt/processed_datasets/agent.rag_evaluation_metrics_dataset_processed_dataset.json delete mode 100644 noveum_customer_support_bt/processed_datasets/agent.routing_evaluation_metrics_dataset_processed_dataset.csv delete mode 100644 noveum_customer_support_bt/processed_datasets/agent.routing_evaluation_metrics_dataset_processed_dataset.json delete mode 100644 noveum_customer_support_bt/processed_datasets/agent.web_search_evaluation_metrics_dataset_processed_dataset.csv delete mode 100644 noveum_customer_support_bt/processed_datasets/agent.web_search_evaluation_metrics_dataset_processed_dataset.json delete mode 100644 noveum_customer_support_bt/processed_datasets/agent.web_search_generation_dataset_processed_dataset.csv delete mode 100644 noveum_customer_support_bt/processed_datasets/agent.web_search_generation_dataset_processed_dataset.json delete mode 100644 noveum_customer_support_bt/processed_datasets/tool-orchestator_dataset_processed_dataset.csv delete mode 100644 noveum_customer_support_bt/processed_datasets/tool-orchestator_dataset_processed_dataset.json delete mode 100644 noveum_customer_support_bt/split_datasets/agent.llm-rag_dataset.json delete mode 100644 noveum_customer_support_bt/split_datasets/agent.llm_model_execution_dataset.json delete mode 100644 noveum_customer_support_bt/split_datasets/agent.query_routing_dataset.json delete mode 100644 noveum_customer_support_bt/split_datasets/agent.rag_evaluation_metrics_dataset.json delete mode 100644 noveum_customer_support_bt/split_datasets/agent.routing_evaluation_metrics_dataset.json delete mode 100644 noveum_customer_support_bt/split_datasets/agent.web_search_evaluation_metrics_dataset.json delete mode 100644 noveum_customer_support_bt/split_datasets/agent.web_search_generation_dataset.json delete mode 100644 noveum_customer_support_bt/split_datasets/tool-orchestator_dataset.json delete mode 100644 noveum_customer_support_bt/traces/traces/dataset.json delete mode 100644 noveum_customer_support_bt/traces/traces/dataset_filtered.json delete mode 100644 noveum_customer_support_bt/traces/traces/dataset_filtered_mapped.json delete mode 100644 noveum_customer_support_bt/traces/traces/traces_batch_001.json diff --git a/noveum_customer_support_bt/demo_results/agent.query_routing_dataset/agent_evaluation_results.csv b/noveum_customer_support_bt/demo_results/agent.query_routing_dataset/agent_evaluation_results.csv deleted file mode 100644 index 0bca67e..0000000 --- a/noveum_customer_support_bt/demo_results/agent.query_routing_dataset/agent_evaluation_results.csv +++ /dev/null @@ -1,21 +0,0 @@ -user_id,task_id,turn_id,agent_name,task_progression,context_relevancy,role_adherence,tool_relevancy,parameter_correctness,task_progression_reasoning,context_relevancy_reasoning,role_adherence_reasoning,tool_relevancy_reasoning,parameter_correctness_reasoning -,eda4fe22-9a2b-4b73-856b-f4f3309bf719,0b19befc-5dc0-4b00-9a8b-5f2f2c914f17,agent,1.0,7.5,1.0,0.0,0.0,"The agent's response is missing. It's impossible to evaluate its understanding, alignment, progress, efficiency, or proximity to completion without any content provided. Therefore, the task progression is at its minimal level.","The agent's response should provide information about the latest breakthroughs in AI. Without seeing the actual response, it's impossible to confirm alignment with role and task. However, the premise indicates that the content should be factually correct and presented in an informative manner, suggesting an appropriate tone for the given role.",The agent completely fails to adhere to its role as it provided no response and made no tool calls. There is no evidence of an attempt to answer the question about AI breakthroughs. The agent's lack of any output indicates a total disregard for the task and role instructions.,"Error: Missing required fields: ['tool_calls'], out of all required fields: ['tools_available', 'tool_calls']","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']" -,0ffffba1-8a37-443c-8866-d53ffbfa7718,30644d2a-5ab1-4a73-b4ff-cf9fa478e36b,agent,2.8,7.8,10.0,0.0,0.0,"The agent identifies its role and task, but the response is missing. A response is not provided, hindering concrete progress. The agent is beginning to understand, but further actions need to be performed for better completion.","The agent response's direct relevance and its alignment with the agent's role are satisfactory. The tone and content are appropriate, showing understanding of the task. However, achieving a higher score requires a more detailed and insightful response on current trends, implying a need for better task completion progression.","The agent perfectly adheres to its role and task. The response is consistent with expectations for the assigned role. The absence of any tool calls is acceptable as there is no requirement to make a tool call in this scenario, therefore the agent has no need to deviate from its role.","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tools_available', 'tool_calls']","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']" -,f1f37bd7-0851-4659-b493-b80d3800d920,89eac2de-b7da-4db3-bf7c-8e87c429481e,agent,1.0,7.5,10.0,0.0,0.0,"The agent's response is missing. There is no information provided regarding the assigned task. Therefore, no progress has been made, and the response is unhelpful. A score of 1 reflects this lack of response and failure to address the prompt.",The response should list the newest features in cloud computing based on the agent's task. It should be aligned with the agent's role as well as the expertise. The tone and content are appropriate for the role. Demonstrating a deeper understanding of the task requirements would be key to increase the score.,The agent perfectly adheres to its role and task. The absence of tool calls indicates that the agent knows to respond without them. The agent does not stray from the assigned task and provides an expected answer while acting appropriately as an agent.,"Error: Missing required fields: ['tool_calls'], out of all required fields: ['tools_available', 'tool_calls']","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']" -,43cdf081-4f01-49cd-b566-dbd1619e6cd2,962afebb-26d1-45ff-acf1-218163cd6bb8,agent,1.0,7.5,10.0,0.0,0.0,"The agent's response is missing. It's impossible to evaluate its understanding, alignment, progress, efficiency, or proximity to completion without any content provided. Therefore, it made no progress and is off-topic. ","The agent's response should provide real-time or recent information regarding the current status of the cryptocurrency markets. Given the task of providing market updates, the agent's response needs to be relevant and aligned with the role of an information provider. The score reflects a solid starting point but can improve through richer details.","The agent perfectly adheres to its role and task. The absence of tool calls indicates no actions were taken, which aligns with the prompt requesting an evaluation without any tool usage. The response, while missing, doesn't deviate from the intended role of reporting current crypto market status. The consistency is flawless.","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tools_available', 'tool_calls']","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']" -,9a1983f4-09da-4b53-80e6-38de6878e0e7,246cf822-4562-4d3a-8a5a-e38e0a0b02a0,agent,2.8,7.8,1.0,0.0,0.0,"The agent identifies the task correctly. However, the agent's response is missing. There is no actual response regarding the latest updates in Python programming. Therefore, the progress is minimal as it shows understanding of the task, but provides no concrete information.","The agent response's direct relevance and role alignment is good. The tone and content are appropriate. However, the score could be higher if the response also demonstrated a helpful level of task understanding by providing specific examples or sources for the latest Python programming updates to aid task completion.","The agent provided no response and made no tool calls. This is a complete failure to adhere to the role and task. The agent should have provided some information, even if it had to use a tool to gather it. The lack of any output demonstrates a fundamental misunderstanding of the prompt's requirements.","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tools_available', 'tool_calls']","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']" -,5d2517e1-220a-429d-9d59-f701bda25eed,20544f91-8828-4a0e-b253-ed1f8b95b205,agent,1.0,1.0,1.0,0.0,0.0,"The agent's response is missing. It's impossible to determine if the agent understands its role or has made any progress. Therefore, the task progression is rated as minimal, with no concrete advancement toward the goal, as there is no response to evaluate.","The agent provided no response, which is completely irrelevant and inappropriate. It fails to address the task of summarizing tech news and demonstrates no understanding of the requirements. Therefore, the score is significantly low, reflecting the complete lack of a meaningful answer.","The agent completely fails to adhere to its role. It's supposed to summarize tech news but hasn't provided any information or made any tool calls. Its response is blank, and thus it cannot fulfill its assigned task. There are significant role contradictions and inappropriate behaviors due to this complete lack of output.","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tools_available', 'tool_calls']","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']" -,52aacb67-c361-4445-9b72-c157f79f47d6,5359c861-c51b-446d-9ad2-b3d8c21ea70f,agent,1.0,7.5,1.0,0.0,0.0,"The agent's response is missing. Without a response, it's impossible to determine if the agent understands its role or the assigned task. No progress has been made, as there's nothing to evaluate for alignment, concrete advancement, efficiency, or completeness.","The agent's response should provide a list of current trends in observability tools. This shows a good understanding of the task. The information should be factual and relevant. Its appropriateness hinges on the quality of the response, specifically the trends mentioned and their relevance in current times. A detailed response will increase the score.","The agent completely fails to adhere to its assigned role or perform the task. No tool calls were made, indicating a complete lack of engagement with the prompt. There is no information provided about the current trends in observability tools, directly contradicting the instructions. The agent fails at all aspects of the evaluation.","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tools_available', 'tool_calls']","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']" -,a81ca3a8-80aa-4c39-876e-8d40ea7a0aef,acf6a99a-39e0-4f87-a07e-3a61213ab032,agent,1.0,7.5,10.0,0.0,0.0,"The agent provided no response, meaning it did not address the task. It's difficult to assess understanding or progress based on an empty reply. The agent failed to engage with the prompt. Therefore, the score is a 1.0.","The response is relevant to the task of describing recent developments in machine learning, fitting the agent's role. The tone and content are appropriate. However, the depth of the response could be increased to demonstrate a stronger understanding of the current state of the field to get a better score.","The agent perfectly adheres to its role and task. The response is consistent with the role of machine learning expert. The agent appropriately answers the prompt, delivering a relevant overview. No tool calls were made, suggesting that the agent could answer the query without external assistance, which aligns with perfect role adherence.","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tools_available', 'tool_calls']","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']" -,230aad27-f3dd-4968-a45a-3c2f07ac28ed,0b47ea07-1eda-4b6a-a183-ea9ef047e431,agent,1.0,7.8,10.0,0.0,0.0,"The agent's response is missing. There is no information to assess. Therefore, it's impossible to determine if the agent understands its role, the task, or has made any progress. The response is completely unhelpful, failing to engage with the prompt.","The agent response's relevance is good given the weather query. It's appropriately aligned with the role, providing information about the weather. The tone is suitable, indicating task understanding. To improve, it could offer a bit more detail, but overall it is a helpful start.","The agent perfectly adheres to its role and task. The agent correctly executes its role and the tool calls are appropriate for retrieving information about the weather, as intended. The response is consistent with the role and task requirements, showing no contradictions or inappropriate behaviors. The agent demonstrates flawless adherence.","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tools_available', 'tool_calls']","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']" -,83c7dcce-3d89-4da1-8b3f-d419885d4cbc,3866ddb3-b17e-47a1-8b3f-bca1e32a2ae4,agent,1.0,7.8,1.0,0.0,0.0,"The agent's response is missing. It's impossible to evaluate its understanding, alignment, progress, efficiency, or proximity to completion without any content provided. Therefore, the task progression is at the lowest possible level.","The agent response's direct relevance is confirmed, matching the specified task. The alignment with the designated role is evident, and the tone and content are appropriate. There's a slight lack of demonstrating deeper understanding, which slightly reduces the score, but overall, it's well-executed and helpful.","The agent completely fails to adhere to its role. It provides no response or tool calls, indicating a complete inability to fulfill its assigned task of providing AI news. It acts inappropriately by failing to respond, showing no understanding or attempt to interact with any available tools.","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tools_available', 'tool_calls']","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']" -,2218f641-604c-491a-9710-b51a9941b982,3ce2e35e-4518-43b4-8a76-4413da6832ea,agent,1.0,7.8,9.0,0.0,0.0,"The agent's response is missing. There is no information provided regarding the agent's understanding, alignment to the task, progress, efficiency or approach. Thus, it hasn't made any progress, and is, therefore, off-topic.","The agent response's direct relevance to the task is good, and it generally aligns with the agent's role. The tone and content are appropriate. However, without seeing the actual response, a slightly higher score is withheld as it's impossible to confirm the response's comprehensive understanding and perfect alignment with the task's requirements.","The agent perfectly adheres to its role, providing a concise response about the benefits of using Noveum Trace. The absence of tool calls indicates no deviation from the provided context. The response is consistent, focused, and free of any role contradictions or inappropriate behaviors, demonstrating a strong understanding of its assigned task.","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tools_available', 'tool_calls']","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']" -,255fd49c-84b4-4b18-887e-6308a412d535,c0c39e8b-9a42-4c18-8737-9224cd8a45c0,agent,1.0,7.8,1.0,0.0,0.0,"The provided response from the agent is missing. Since the agent has not provided any answer, the agent has made no progress on the task. The response is neither aligned with the task nor does it demonstrate any understanding of the assigned instructions.","The agent response appears helpful and relevant to the assigned task of configuring Noveum. It's aligned with the agent's role and expertise. The tone and content are appropriate. However, without the actual response, a definitive assessment of its quality and level of detail is impossible; hence, a slightly lower score.","The agent provided no response, and therefore failed to adhere to the role or address the task. It did not provide any information or utilize any tools. The lack of response demonstrates complete failure in understanding and executing the assigned instructions.","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tools_available', 'tool_calls']","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']" -,dc511122-c0b6-415c-9a49-c7b45132dd87,80987c51-ea82-4a41-97c7-ed591082b9bf,agent,1.0,7.8,10.0,0.0,0.0,"The agent's response is missing. There is no information provided regarding Noveum's monitoring capabilities. The agent has not provided any answer and has not made any progress, therefore it is off-topic and unhelpful. The agent completely failed to fulfill the task.","The agent response's relevance is good given the prompt, and the response is appropriate for the agent's designated role. The content's tone is also aligned with the task, and the response demonstrates good understanding of the task requirements. Further details could enhance the response.",The agent perfectly adheres to its role and task. The agent's response is relevant to its role and the tool calls are appropriate for this role and its task. The agent doesn't contradict its assigned task. No deviations or inconsistencies are observed.,"Error: Missing required fields: ['tool_calls'], out of all required fields: ['tools_available', 'tool_calls']","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']" -,04bebf38-a343-4563-80db-0154bef8d927,2f89c1e0-9b62-4651-914c-b71cf561c33c,agent,1.0,7.8,1.0,0.0,0.0,"The agent's response is missing. There is no attempt to answer the question, and therefore, no progress has been made. The agent has not provided any information about Noveum or agent tracing. This indicates a complete failure to engage with the task.","The agent response's relevance to the assigned task is clear, and it aligns well with the agent's role. The tone and content are appropriate. There's a solid understanding of the task requirements. The response is helpful. It could gain a higher score if it gives an answer.","The agent completely fails to fulfill its role. The agent provides no response and makes no tool calls, indicating a complete disregard for the assigned task. There's no evidence of role adherence, task completion, or relevant information retrieval. The agent is inactive, rendering the evaluation impossible.","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tools_available', 'tool_calls']","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']" -,5e043630-6493-42b5-beb8-79faa19bfa37,e30e3e61-6edd-4b62-ab77-627faa2bf741,agent,1.0,7.8,10.0,0.0,0.0,"The agent's response is missing. There is no information provided regarding which APIs are available. The agent has not addressed the question at all, failing to make any progress toward the goal. The response is unhelpful and off-topic. No attempt was made.","The agent response's relevance to the assigned task is clear, and it aligns well with the agent's specified role. The content and tone are suitable for the role. Demonstrating a more extensive understanding of available APIs in Noveum would improve the score.","The agent perfectly adheres to its role and task. The agent correctly identified that there were no tools calls, as it was a prompt and not a request for a specific action. The response is consistent with the expectations of the assigned role.","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tools_available', 'tool_calls']","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']" -,7da9814d-a2e8-4c4e-b750-68b26bd5fd22,2f513795-764f-4eb6-8dd2-108d4cdf8432,agent,1.0,7.5,1.0,0.0,0.0,"The agent's response is missing. There is no information provided about how the agent attempted to answer the question. Therefore, no progress was made toward task completion, and the response is unhelpful. The agent did not respond with any information.","The response should directly address setting up observability with Noveum, which is highly relevant to the provided task. The response is likely aligned with the agent's role and expertise. Improvement could include a more helpful response, leading to a higher score.","The agent provided no response or tool calls, therefore failing to adhere to any aspect of its role and task. It did not attempt to answer the question or make any progress towards the objective. The lack of any action completely contradicts the expectation for a helpful response.","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tools_available', 'tool_calls']","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']" -,16143f74-2831-4753-b33d-ce4b645093c5,29c7a515-2383-4b7d-8fc7-beac3f403641,agent,1.0,7.8,10.0,0.0,0.0,"The agent's response is missing. There is no information to evaluate. It fails to answer the question, and provides no helpful information or any indication of understanding the task. Therefore, the agent has made no progress.","The agent response's direct relevance is strong, fitting the task and agent role well. The content is suitable. While it efficiently addresses the query, there's room to further showcase the agent's expertise and provide a more comprehensive overview of Noveum Trace's features. A more elaborate answer could have improved the score.","The agent perfectly adheres to its role and task. The agent has not provided a response nor made any tool calls, which is the expected behavior given the prompt which instructed it to evaluate an agent's response. No deviations or contradictions were observed, maintaining complete consistency.","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tools_available', 'tool_calls']","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']" -,fc64e6cc-6739-4256-ac4a-7b80c3028233,12fee687-2a2e-401f-83c2-9c8dc7ecd853,agent,1.0,7.8,1.0,0.0,0.0,"The agent's response, or lack thereof, does not provide any information regarding Noveum's pricing plans. It's a placeholder. There is no progress made, the response is off-topic, and fails to address the core task. The agent has not understood the task at hand.","The agent response's relevance to the assigned task is clear and aligns well with the agent's role. The tone and content are appropriate. However, without the actual response, it's impossible to gauge the depth of understanding or how helpful it is in completing the task, so the score is slightly adjusted.","The agent provided no response, and made no tool calls, meaning the agent completely failed to fulfill its role or task. It did not try to answer the question about pricing plans or attempt any data analysis or retrieval. This constitutes a complete failure to adhere to the role.","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tools_available', 'tool_calls']","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']" -,b7945c49-f584-4c70-972d-536a805d8a31,9c5aec32-b6eb-4c42-93d5-3970fb118c00,agent,1.0,7.5,1.0,0.0,0.0,"The agent has not provided a response, meaning no progress has been made. The task is to provide instructions on how to integrate Noveum Trace. The lack of content indicates a complete failure to address the prompt.","The agent response would likely be directly relevant to the task if there was an actual response to evaluate. The prompt itself is designed to elicit a response about integrating Noveum Trace, aligning with the agent's expected role. Without a response, evaluating tone, content, and task understanding is impossible, but the prompt itself is well-suited.","The agent provided no response, meaning the agent failed at every aspect of its role. It did not provide a response, tool calls, or do anything to accomplish its assigned task. The agent completely ignored all instructions and failed to operate as requested. Therefore, the adherence score is very low.","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tools_available', 'tool_calls']","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']" -,f5c40ecf-36c0-45ba-9cc9-dc0329b0324b,f132d053-f82c-4e66-adf4-bd7ce68b660d,agent,2.8,7.8,10.0,0.0,0.0,"The agent seems to understand the task to define Noveum. However, without the agent's response, it's impossible to evaluate the content's accuracy or the degree of progress. Therefore, only the basic understanding of the task can be inferred, resulting in a score reflecting minimal progress with potential for improvement.","The response is relevant to the task of defining Noveum. It's well-aligned with the agent's role of providing information. The tone is appropriate. Demonstrating a slightly more detailed understanding of Noveum's function or purpose would elevate the score further, though the foundational elements of task completion are present.","The agent perfectly adheres to its role and task, maintaining consistency throughout. The absence of any tool calls implies that the agent may have responded directly with the answer to the question. This aligns well with the prompt, suggesting an exact response is expected given the lack of tool usage. No deviations from the role or inappropriate behaviors are observed.","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tools_available', 'tool_calls']","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']" diff --git a/noveum_customer_support_bt/demo_results/agent.rag_evaluation_metrics_dataset/agent_evaluation_results.csv b/noveum_customer_support_bt/demo_results/agent.rag_evaluation_metrics_dataset/agent_evaluation_results.csv deleted file mode 100644 index bf749b0..0000000 --- a/noveum_customer_support_bt/demo_results/agent.rag_evaluation_metrics_dataset/agent_evaluation_results.csv +++ /dev/null @@ -1,13 +0,0 @@ -user_id,task_id,turn_id,agent_name,task_progression,context_relevancy,role_adherence,tool_relevancy,parameter_correctness,task_progression_reasoning,context_relevancy_reasoning,role_adherence_reasoning,tool_relevancy_reasoning,parameter_correctness_reasoning -,f1f37bd7-0851-4659-b493-b80d3800d920,ac166548-ffa5-47b9-8cf7-4787f270cb5a,agent,3.8,7.8,9.0,0.0,0.0,"The agent understands its role and the task. The response is aligned with the query and attempts to answer it using the provided context. While the answer mentions 'recent enhancements' it's focused on a specific platform, which limits its scope, hence some gaps. Progress is good, but not fully comprehensive.","The agent acknowledges the limitations of the provided context, which is a good start. It correctly identifies the lack of general cloud computing features while focusing on the specific information available. The response is helpful and progresses the task, even if it is a negative answer, indicating understanding.","The agent perfectly adheres to its role and task. The response is relevant, concise and accurately answers the question based on the provided RAG context. The agent correctly interprets the information and offers a focused response, without any role contradictions. The agent demonstrates excellent comprehension of its role.","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tools_available', 'tool_calls']","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']" -,52aacb67-c361-4445-9b72-c157f79f47d6,e193cd4f-cca3-4b94-95d3-a1f2446b9fc6,agent,2.8,7.8,9.0,0.0,0.0,"The agent understands the task, but the response is somewhat limited. While it acknowledges the context doesn't directly answer the question, it identifies some relevant aspects. More direct information about trends would improve the score. The agent shows minimal concrete advancement.","The agent accurately acknowledges the limitations of the provided context while still attempting to address the query. The response is relevant, aligning with the agent's task by identifying the absence of direct information and pivoting to related aspects. The tone and approach are suitable for the role.","The agent accurately fulfills its role as an evaluator, focusing on the provided agent's response and task. The response is direct and doesn't introduce irrelevant information or act outside its assigned scope. The analysis is well-structured and relevant to the provided criteria, reflecting a strong adherence to the role.","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tools_available', 'tool_calls']","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']" -,2218f641-604c-491a-9710-b51a9941b982,432e74f3-3291-411b-8d7e-85493868f8cf,agent,4.5,7.9,9.0,0.0,0.0,"The agent correctly identifies and begins to list benefits, showing a solid understanding of the task. The response is directly aligned with the prompt's requirements. The approach is efficient, quickly addressing the query's core. While not fully complete (truncated sentence), it shows substantial advancement.","The response directly answers the question about the benefits of using Noveum Trace. It's aligned with the agent's task of providing information. The answer is appropriate in tone and content. The inclusion of a numbered list indicates a solid understanding of the task. However, the response is cut off, which prevents a higher score.","The agent perfectly adheres to its role by providing a relevant answer. The response is focused on the benefits of Noveum Trace, directly answering the query. No tool calls were made, which is acceptable in this case. There are no contradictions or inappropriate behaviors, and the content is appropriate for the assigned task.","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tools_available', 'tool_calls']","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']" -,255fd49c-84b4-4b18-887e-6308a412d535,a0475dd5-ef9f-4eb2-bca4-9be72bf6bfd6,agent,4.5,7.8,9.0,0.0,0.0,"The agent understands the task and provides a relevant answer, pointing to the SDK Integration Guide. This demonstrates a strong understanding of how to configure Noveum. The response efficiently directs the user to a valuable resource, making substantial progress toward task completion.","The response is relevant and helpful, providing a clear starting point for the user. It aligns well with the agent's task by suggesting a specific resource (SDK Integration Guide). While informative, the response could be slightly improved by giving more specific details or examples to enhance user assistance.","The agent perfectly adheres to its role, providing a concise answer related to the prompt's request. The response focuses on the Noveum configuration and points to relevant documentation, which is aligned with the task. There are no deviations or contradictions observed, showcasing consistency throughout.","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tools_available', 'tool_calls']","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']" -,dc511122-c0b6-415c-9a49-c7b45132dd87,ed8ad539-f008-4f1a-beb1-12ad5793950e,agent,4.5,8.1,9.0,0.0,0.0,"The agent correctly answers the query and demonstrates a strong understanding of its task. The response is directly relevant and provides specific details about Noveum's monitoring capabilities. The information is concise, and the inclusion of frameworks shows comprehensive understanding. Excellent progress with very minor room for improvement.","The response directly answers the query about Noveum's monitoring capabilities. It correctly identifies key aspects like monitoring, tracing, and optimization. The agent's expertise is evident. The response is helpful and appropriately worded, making it well-suited to the assigned task and agent role. A slightly more detailed explanation could potentially improve the score.","The agent perfectly adheres to its role by providing a concise and relevant answer to the prompt. It directly addresses the query about Noveum's monitoring capabilities. The response is appropriate for the context, and there are no deviations or inconsistencies. The absence of tool calls reinforces the agent's role here.","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tools_available', 'tool_calls']","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']" -,04bebf38-a343-4563-80db-0154bef8d927,c7358391-9ac1-4312-a4b3-79edfa8bc0d3,agent,4.5,7.8,9.0,0.0,0.0,"The agent clearly understands the query and provides a relevant response directly answering the question. The provided information regarding Noveum's agent tracing capabilities suggests strong progress. The response is concise and appears well-informed, leading to a high progression score.","The response directly answers the question about agent tracing by providing information related to Noveum. The content aligns well with the agent's task and role as an information provider. The answer is appropriate in tone and content, offering a concise and helpful overview, thus achieving a solid score.","The agent perfectly adheres to its role by providing a concise and relevant answer to the query. The response directly addresses the question about how Noveum.ai handles agent tracing. There are no deviations, contradictions or inappropriate behaviors. The content is focused and aligns perfectly with the expected output.","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tools_available', 'tool_calls']","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']" -,5e043630-6493-42b5-beb8-79faa19bfa37,6b21de5d-456f-4501-953f-0ce5bc4fa5a5,agent,4.5,7.8,9.0,0.0,0.0,"The agent correctly identifies the limitations of the provided context and suggests appropriate next steps. The response is directly aligned with the task, displaying a strong understanding of how to address the user's query. This demonstrates significant progress with no apparent inefficiency.",The agent response is relevant and acknowledges the lack of information in the context. It offers a helpful suggestion to find the desired information. The response is well-aligned with the agent's role and tone is appropriate. A slightly higher score is awarded as the response provides a clear next step.,"The agent perfectly adheres to its role by providing a relevant answer based on the context. The agent correctly states it lacks the necessary information to provide the requested data, offering a helpful suggestion to get the information. No role contradictions or inappropriate behaviors are present. The response is consistent with the role.","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tools_available', 'tool_calls']","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']" -,7da9814d-a2e8-4c4e-b750-68b26bd5fd22,2ad4e155-7b76-4040-923d-d6b65a97a45b,agent,4.5,7.8,9.0,0.0,0.0,"The agent clearly understands the task and provides a relevant response. It directly addresses the user's query by offering actionable steps to set up observability with Noveum. The provided steps indicate substantial progress, although the response is cut off, hinting at incomplete task completion. The approach is efficient.","The agent provides a relevant answer that is well-aligned with the task of explaining how to set up observability with Noveum. The response clearly outlines the initial steps, demonstrating an understanding of the task requirements and providing helpful information. The tone is appropriate for the role of providing instructions.","The agent perfectly adheres to its role by providing a clear and concise answer to the user's query. The response is directly relevant and helpful, offering a step-by-step guide to setting up observability with Noveum. The answer is presented in a straightforward manner, consistent with the task's requirements.","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tools_available', 'tool_calls']","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']" -,16143f74-2831-4753-b33d-ce4b645093c5,6b822089-be40-457d-9fa6-0979bafdfb85,agent,4.5,8.1,9.0,0.0,0.0,"The agent correctly identifies its role and provides a relevant response, directly addressing the query about Noveum Trace's features. The response includes concrete information about the Python and TypeScript SDKs and a key feature. It is well-structured and advances substantially towards task completion.","The response is highly relevant, providing a clear and concise answer to the query about Noveum Trace features. It aligns perfectly with the agent's task and role as an information provider. The content is well-structured and demonstrates a good understanding of the task requirements, making it helpful for task completion.","The agent perfectly adheres to its role by providing a concise and relevant response to the query about Noveum Trace features. It stays focused on the task and accurately presents the information, using technical terms appropriately. The absence of tool calls further emphasizes the straightforward and task-oriented response, demonstrating excellent role adherence.","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tools_available', 'tool_calls']","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']" -,fc64e6cc-6739-4256-ac4a-7b80c3028233,08db68f6-3623-44b5-840b-41400fac7333,agent,4.1,7.8,9.0,0.0,0.0,"The agent correctly identifies the task and responds with relevant information extracted from the given context. It acknowledges the lack of explicit pricing plan details while providing related context. This shows a good understanding and efficient approach, making it almost complete.","The agent correctly identifies the lack of pricing plan information while providing a relevant piece of context, the Observability suite's free access. The response is appropriate for the task and role, although it could be improved by acknowledging the limitation more explicitly to get a higher score.",The agent perfectly adheres to its role and task. The response is consistent with the lack of information regarding pricing plans. The agent acknowledges the limitations of the provided context without exhibiting any role contradictions or inappropriate behaviors. There are no deviations from the assigned role.,"Error: Missing required fields: ['tool_calls'], out of all required fields: ['tools_available', 'tool_calls']","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']" -,b7945c49-f584-4c70-972d-536a805d8a31,6a57c92e-e473-425a-978b-19dbe976a46e,agent,4.5,8.0,9.0,0.0,0.0,"The agent correctly identifies the task and provides a relevant response, indicating a good understanding of its role. The agent offers specific information about using the Python or TypeScript SDKs, showing concrete progress toward the goal of integration. The answer is concise and points to a quick start, suggesting an efficient approach. ","The response is relevant and helpful, providing a clear starting point for integrating Noveum Trace. It correctly identifies the available SDKs. The response aligns well with the agent's role. It adequately demonstrates an understanding of the task requirements. A slightly more detailed response could have improved the score.","The agent accurately addresses the query about integrating Noveum Trace. The provided response is relevant and directly answers the question, demonstrating a clear understanding of the task. The agent remains focused and doesn't exhibit any role contradictions or inappropriate behavior. No tool calls were made, suggesting that the initial information retrieved by the RAG system was sufficient.","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tools_available', 'tool_calls']","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']" -,f5c40ecf-36c0-45ba-9cc9-dc0329b0324b,efb19490-bc9e-48dc-a365-2f839100e41a,agent,4.5,8.1,10.0,0.0,0.0,"The agent correctly identifies the task and provides a relevant answer about Noveum. The response directly addresses the query, indicating strong understanding and concrete progress. The answer is concise and informative. The approach is efficient and directly addresses the user's need. The agent has provided a complete and helpful answer.","The response is highly relevant, providing a concise and accurate definition of Noveum.ai and its function, directly addressing the query. The answer aligns well with the agent's role as an information provider. The content is suitable and helpful, showcasing a good understanding of the task requirements.","The agent perfectly adheres to its role and task. The response provides a clear and concise answer regarding Noveum.ai's function, aligning completely with the role's objective to answer a question about a technology. There are no deviations, contradictions, or inappropriate behaviors observed.","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tools_available', 'tool_calls']","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']" diff --git a/noveum_customer_support_bt/demo_results/agent.web_search_evaluation_metrics_dataset/agent_evaluation_results.csv b/noveum_customer_support_bt/demo_results/agent.web_search_evaluation_metrics_dataset/agent_evaluation_results.csv deleted file mode 100644 index eb15ebe..0000000 --- a/noveum_customer_support_bt/demo_results/agent.web_search_evaluation_metrics_dataset/agent_evaluation_results.csv +++ /dev/null @@ -1,9 +0,0 @@ -user_id,task_id,turn_id,agent_name,task_progression,context_relevancy,role_adherence,tool_relevancy,parameter_correctness,task_progression_reasoning,context_relevancy_reasoning,role_adherence_reasoning,tool_relevancy_reasoning,parameter_correctness_reasoning -,eda4fe22-9a2b-4b73-856b-f4f3309bf719,3b7b6838-22d4-4edf-9b00-7cb011f18a52,agent,4.2,7.8,9.0,0.0,0.0,"The agent correctly identifies that the web search results did not provide the needed information. It understands its task and provides a logical next step (checking reputable sources). The response is concise and helpful, showing good progress. Some improvement in its search query design might be possible.","The agent provides a relevant response acknowledging the limitations of the web search results. It correctly states the lack of specific information and offers a suggestion for further research. The response is appropriate in tone and content, demonstrating a good understanding of the task requirements and offering a helpful next step.","The agent perfectly adheres to its role and task. The response is relevant, and the agent accurately reports the inability to find the requested information, providing a suggestion. No inappropriate behavior or role contradictions are present, and the response stays focused on the assigned task throughout the interaction.","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tools_available', 'tool_calls']","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']" -,0ffffba1-8a37-443c-8866-d53ffbfa7718,a5d480f0-ea5b-4190-a3c3-f94e26b48573,agent,3.8,7.8,9.0,0.0,0.0,"The agent understands the task and demonstrates a basic approach by using web search. The response indicates an attempt to find information, but it also clearly shows a limitation of the search. While it did not fully succeed, it tried the expected method. There is a clear understanding of the task and a logical, albeit unsuccessful, approach. ","The agent acknowledges its inability to find specific information, which is appropriate. The response directly addresses the task and is consistent with the agent's role. It is helpful, as it provides a clear answer based on the search results. A slightly higher score could be earned with a more proactive attempt to search or suggest alternative approaches.","The agent perfectly adheres to its role by attempting to answer the user's query by searching the web. The response acknowledges the lack of results, demonstrating appropriate task execution. The agent correctly uses the tools and stays focused on gathering and presenting the requested information. There are no deviations or inappropriate behaviors.","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tools_available', 'tool_calls']","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']" -,43cdf081-4f01-49cd-b566-dbd1619e6cd2,2fd285de-68a1-4b7a-a4ed-05f539afa737,agent,4.2,7.8,9.0,0.0,0.0,"The agent correctly interprets the task and attempts to fulfill it by utilizing web search. The agent acknowledges the limitations of the search results, demonstrating understanding. Though it could provide a more informative answer based on the search, it still delivers a useful response, advancing the goal.","The response directly addresses the query about cryptocurrency market status. It acknowledges the limitations of the search results, which aligns with the agent's role. The answer is appropriate and helpful by providing a clear assessment. While it could elaborate slightly, the response is good.","The agent perfectly adheres to its role by attempting to answer the user's question about cryptocurrency markets. The response acknowledges the inability to provide a direct answer due to search result limitations. There are no role contradictions, and the focus remains on delivering information relevant to the query.","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tools_available', 'tool_calls']","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']" -,9a1983f4-09da-4b53-80e6-38de6878e0e7,48deb050-478d-486c-be3a-b8c0069d7abe,agent,2.5,7.8,9.0,0.0,0.0,"The agent understands its task to find information about Python updates. However, the response is not useful. It acknowledges a web search but then provides unrelated information. While it identifies the limitation of the search, it doesn't try to find alternative solutions or provide any actual updates.","The response accurately reflects the search results, acknowledging their lack of relevance to the query about Python updates. This aligns well with the agent's task and role as an evaluator. The response is helpful in assessing the information provided by the search. However, it could be slightly improved by suggesting future steps.","The agent perfectly adheres to its role by providing a relevant answer based on the query. It correctly identifies the lack of information on recent updates in Python programming from the search results, and doesn't stray from its task. No role contradictions or inappropriate behaviors are observed, thus maintaining a high score.","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tools_available', 'tool_calls']","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']" -,5d2517e1-220a-429d-9d59-f701bda25eed,4ecede78-e7fc-48ed-9fba-1241d6890a1d,agent,2.8,7.8,9.0,0.0,0.0,"The agent understands the task, which is to provide a summary of tech news. However, the agent's response indicates that the search results were not helpful. While this response is relevant, it did not provide the information requested. Therefore, the response made minimal progress towards completing the task.","The agent correctly identifies the lack of relevant information in the search results, directly addressing the query's core. The response aligns well with the agent's task to summarize search results. The evaluation is appropriate. Improvement could involve briefly mentioning what *was* found.",The agent perfectly adheres to its role by providing a relevant answer regarding tech news based on web search results. The agent's response is consistent with the task and demonstrates a clear understanding of the information request. There are no deviations or contradictions observed in the agent's actions.,"Error: Missing required fields: ['tool_calls'], out of all required fields: ['tools_available', 'tool_calls']","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']" -,a81ca3a8-80aa-4c39-876e-8d40ea7a0aef,91fdf43c-7170-498e-9035-1e2b11b32b48,agent,3.8,7.8,9.0,0.0,0.0,"The agent acknowledges the limitations of the provided search results and proactively offers alternative actions. The response is aligned with the task, showing a moderate understanding of the objective. However, it lacks concrete details on machine learning developments, limiting the overall progress.","The agent acknowledges the lack of information in the search results and states that they cannot answer the query. This is a relevant response. Furthermore, it hints at potentially providing an answer based on other sources, which progresses towards task completion and shows an understanding of the task requirements.",The agent perfectly adheres to its role by acknowledging the lack of direct information and explaining its limitations. The response stays focused on the task of answering a question. No role contradictions or inappropriate behaviors are present. The agent provides a clear and concise response based on the available information.,"Error: Missing required fields: ['tool_calls'], out of all required fields: ['tools_available', 'tool_calls']","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']" -,230aad27-f3dd-4968-a45a-3c2f07ac28ed,096e6592-23fa-432a-931d-8f39656c7fcc,agent,4.2,7.8,9.0,0.0,0.0,"The agent correctly interprets the prompt and its task, and provides a relevant, albeit incomplete, response. It acknowledges the web search results, demonstrating understanding. It is close to achieving the goal, but hasn't fully delivered on it yet. There are minor improvements to make, preventing a perfect score.",The response is relevant as it addresses the weather query. It explains that the search results lack specific weather information. The agent's role is well-defined as it summarizes web search findings. The tone is appropriate and indicates task comprehension. A score of 7.8 recognizes the utility while suggesting possible elaboration.,"The agent perfectly adheres to its role by providing a response that's relevant to the given query. The absence of tool calls indicates that the agent assessed the question could be answered without them, which is appropriate. The response is consistent with the task and demonstrates a good understanding of its role, providing a concise and accurate answer. ","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tools_available', 'tool_calls']","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']" -,83c7dcce-3d89-4da1-8b3f-d419885d4cbc,b56248e6-e526-4000-8c10-f5d13cf2e12d,agent,4.5,8.1,8.8,0.0,0.0,"The agent correctly identifies its role and the task. It provides a response directly addressing the query about the latest AI news. The response includes a summary of current events. The agent makes substantial progress and the approach is efficient, demonstrating a strong understanding of the prompt.","The agent's response directly addresses the query about the latest AI news. It provides a concise summary with relevant details, indicating a good understanding of the task. The format is clear and aligns well with the agent's role. Some further details or source citation could slightly improve the score.","The agent adheres perfectly to the role, demonstrating a comprehensive understanding of the task. The response is relevant, concise and informative. The agent correctly identifies key information about AI news. No role contradictions or inappropriate behaviors are observed. The provided response is well structured and relevant.","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tools_available', 'tool_calls']","Error: Missing required fields: ['tool_calls'], out of all required fields: ['tool_calls', 'parameters_passed', 'tool_call_results']" diff --git a/noveum_customer_support_bt/noveum_vectorstore/index.faiss b/noveum_customer_support_bt/noveum_vectorstore/index.faiss deleted file mode 100644 index 83cb625da4eb1c8330cf8e5b1637edc3b48f3738..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1130541 zcmXtA30RF=)GjGeh@_~9NSP(n*=v)CWQYtQM9Dl|^Gu0ER1`{vh)5zD&tBV*v5bjG zBr?xLX8yJN|NlOZ=U&}&zHjff-}Mgbe5Zbk`#9t_q8na@<)%aA%il?MWg; zjjqA{U9Q2*!s)O*zYDE#AB&ihihjCXv z?gl)b0rKLofzrCh0t~$@(5>A%j4v?IzBv;QS0DA2p5LdzjJCG?da5p8n%$WD4{L;< zhQ6*^Pb_Z6Hg$W+?QPDensK*zy`gpM2zF~<91LGH3^Or|j$d?5o z_VKkb<~`eq$A*l6;umJ}*om^L8U{Dshjn^Q!rJlHFtBzxX3lIX6PFCoEQQ|OKf@N@ zTNY{;y&8}Di-Y9wqr=hKeY`AUzfix}h=t5t3!w+%HJ{A>xwOnZDVJ~mDZ12}fx~Zp z0k38Cv1ES{+a8{eC9Av1!%es1*t%o+wBplZL+)gZebi3olsChKRW=eQd_$uPkaQvg(x08PX?cT%>s{kWfNB6`kaQcxTS8@+7kBlkZ&djLc6_D7&zzxbi8*62^WwM zT7ll58t}yUHzGkAakJn-lE3W7b1n}@ze9hS!uxi^1pd9wVX$}Zid|eoAZ*kIc>XPu z_NFUMJ4}b7Th_AV_a^>ori;upj)6??LVP>};qSugQl0<#9}Rg(kw5=?ceAjq(}x?) z7=@`uJLL90#`3V6BdswAN?|e1WtX7T=>X=1)RXSUlelx!r<&+tosk<&<*uhkaJsJ0 z>FkbSmU}eQH2Gqnz6p0eHA-5H@&(TmZQ*LbCsdfx88HznnkT}HD?40f?`sJ-Dp9;? z)+%T6`AaONP)CNo8Or6PUD~C@u=dIt`viRX0q0N_tB4o!m=GgfWPWxiSi~ci( z$q9!r;_clkod23&43n#!1dsB$jQS84UF<2^^}SDPJjqfcdh&|1u`FfR6dd@vmw4s< zj&+RVa$4qI#TCMAS7UjxQ!4no41|O+x-j_ZY+kT-vq*n1m@pPD3}Xgx!kjp~{-Y*J zL}A*bLE!wmH+bIZFD}k*#EYU&h(*Nn2b&$|Fg&UmYZ-DGD|>tZ(?C5u*DY9d z3E0ot`P*@y3Fk!1+|EK_zo_#eC~+IW?bE~2&nj5_%H76HU!~yW0M0h-@!-L`uRw6) z=9p$|6TK9`d;MUH76K=$`Qx!-(1RWPO0lKH8v&7{QNtl|v2|SvWvORsngnz?S zZ1>zN=hQ4KKDNdN*pT~zB~5S1Mh~@uvrl(}l9WUAkI@zp#*eZk+)uf?hMEPm@wI2<^H`oc_f%m}N( zcIbw|P+TiRl*V+RzV5*DIFAFlr%l6@T(X zAq9T*afQxb=Yj1}0eUYSt2D->zKx{)$b*MX6nCSAqf2L*H1E57Rq$OJRLV>th+ znSA`Rz6`(n2px}QK;K5hQ6+16M2$qGeg)zP?6NKwW*#4aq#0;?gA2`3=Ugo2dgB5F1VcOOpd^Ix%Eo=9aCHqG}=;So0{LxXq>Tn%&qHhSjfSSB@ zYBtQ@oD9@YboO8H%1=|Su-L`J2h_RbY?!90ZF7nFwYbBGU-|QSwPi|73@i%l21$cQ zarZ!LY1`=;sx$xl^C+e-yQnGr6$xED?h+U8fr1txVA6P=EL^446dXDVq|-oak+J6| zpw|N{NZ%bOD$X`yRC7!yYYp+4TFva_?`Yb}g1EC7sE1+N?{Q%3P+uzDQ(Wl(YXbR` z#(eh6avc40qUvqp)bA|MwgyBmx}`B)zYl_WPf3^p>Ti+Iq#=(@$i?VI-l)z(v&mEz zTp7T;BWoemgYnSQV*9&ckY^}BX;5HzM>)CWE%wQ56_78HsxR{fErG}F3y|+K;;HSs z;iAASa5Qm+!k%9I`>+?($Lkn*6r{eDIl<<#JaMRYc4#pez3d<=e;j9ZdOe54XbWD% zmWhJC`*HTm0OGBa5I(D#a31T0dI4cDZRaqiw2?Ft|GK^v)GthFm%SNXuU9{qQ8NUt ztmNFWvmgJy#T$R=RFhp^3_$W(EFp9ranLjNeA##;{>b!Nwgv5fdJ+$}mj%x)Wbn6Y zK>Y-XS=|tq_k!UEs^PR#^VGUX=c@DQxc}tUga@!HNQZ|85X_H*_WCILL z)Cb3WT`=mWMe<^rih~2;{rAqCxJ~JXAPt0z89w0IwFuv9J-GY+^-%t&76yL{)}%dY zA#0Cw%u)ABTyhoDyR65ttsP{hrw+XN8w$2{F2ZPBg5-yosyAT_JwKW&kA#QaPN32_ zQ_n0d>80pf(iAJEJOScomsR@7qVS?s)=f zr`Mv%?R4hY##s0~n*mXKFJQsiIwJbs1B~jyz{ctwb8qD+sc!`JK39FLd`H@8eV&+= zCB6@c)>x*T0`jWlWw$UJtC{Rvw`%CV-w@v~e~2Du3gP+4vpENAS;~a6WHhWCA<4rb z@qn0XaY$(i{PoS&oFCnc?z>r|b4$yA9M@ zT5QIadm3QQnQGGh(Mo#0F%A27}Qh#)T}&C2gVN{kuy&DLR2R#q%9-;%C-3?3&Zq2qt~F!w%yWlDD$+ItH) zn@7Rn)w{sYsu~nbb3?P>>locBT5*w3e#g7WLxxW7#JxM{$UQZ80<>@++t+Rz?~FtE)*g-h|?bT#vCkdAL0e!5UT;iK=mtMg{cQ`74qeygQ+ zv{{;_VAEqHPGY~NK7x<6Q*cJj>YVhJ{K7(QVEAzj`C+h+J%b^UEzrmIp2)m@7|yyo zAYGpythEzOx|~)%6_sy4|6?1TTKEB?Go;cG@$yw&PCTS}p3@)b{Lt~;Ud44tb*_57 z(j#>)5qZvndR&fCIzE-$r`aA_OhNnsUkls4!k` zze1yQ;OxNP!t(WDER-#|f1D4}`AYhk79U;{l&_ROQ(Q{xx`t-W&N~O*nuRKlSU;#H z;|tcITbE+2TyY;wXO4igQ#@QcHkk}BUqxux}0`Bl@Dv=#db8Zwy~r;zGHkOc18Az-ysYJzqX9Mz}s%Abe~`a>Kdp z9*3@1xQ3F|SyQ(@}5D+udNWaH}7L=zconO36CRu6^}qc+q=}ihD!4^!TfAhwzKB7L9gX*#mh|P z1j=tnei6evnnQTHtI#pAgZa}pGR1`{ek+Iz|1sAmU%`H1Gj3b4M5K>w#a&rD@UPfR zvyT33@{8w;{37-KWo&8qS$PgrnoTnenDM@w9NYRZJRb5<3?Fa>(|11;2B&&Rg} zw=_l*4J2U#{qMQKs<5Fdmx1N1LL{Gzj)RWitH^s;7`IM%>~ez93k7)XPE&t}$^|tP z@1wq^Ay|H1u71XFR12V74{BCG*-?33%**%ER4xdGyrkn=;vJmf<%X>Xw#U?KDO$=1 z%yo$cGkm{BJm(7-*W0F17$yAYMCslH+u1i6X(lJHgi0@+n@)swH9rV7=b5e75x8DM zX??aUI^Mbs9*~UQ3r>MS^}h_e>MMT^N>iK{TSB{Y9jkKe;q`NnaL>sj@auE7sUF6q zJ|>*<6>;TKpd2H}KPlhK2S5FwCGDqdx?1~w{stib3XX%?fb-+Ug7lTU4L-osxw;4L zg3`H>NbAmV`w)jT&k?6?*kSDX$BeM6CBG+BHdC2?QLC%yc_KxmH0lB5f2%wlboKLcdES4%%FO7x zWQS%$$Z{kPiNXJdvx$Z^xaY_1=(oa{u<#n5*Vhw?Uzql4(q+H;FGYj7Ues2V%XPU~&PW%kdvQ z#r!0s+#}9TxlDPbwKQp5kxl+hTs#`0G>!fJ*$$MCyuP&y`s7}tzDkE*6T&fe<$ghP zAdz@~1QL(q*)b>*#Xk1mu70l7 zIcACElU>wIbL>D5sbjGO2~!%H%V4QbZxL}Y7DGO@M>=;PUjkG!K|X*H=Mip`$-C|m z?bcpU9Kh6>63=1Ll1Z8?Xb($Eawz*B)~J~%aXhNI33*0T^Dojqmeac}7&P@H-*!w; z?j>GIN5y;O?FHppB<{z<`w`Q>S%LCJjishj+3D2(lQ$hC(eRX zpWRvcq?brJf^{tW#jX$cq}kUiM)MhIFgC-5&IP9}?gDzXyy5vgnpynJN19oRN~^)# zKGu~b^z6c|$EVQTc@)GizM=MyZRT7fZLSTZp~5kI8&kfG@+c5zsCg>Y`8*?kseGYk z!=B4(&c%|hb`a!0*!3@`!KSD^&oStU+%AKWKLFA$RygrJ^RZmZlpcJ4Q~tB)=*(++QtG?G)kWNIFCHqX0? z%g7I?Suq;6=`HPB9E8a~jd;b^TjE|?F>`EsSd(;jFYJHQk{>U=1rw)q59%qHQpi=fmEAt$DH066hAx4E;vgZY~e5AU{>3y4I)uQ&zI=0Ic$^C4XM?kbQ?W z=C)Jnz-hjs>biPX!0*hRh5Z(_Voe=vp;ladX>vRdZ=X5_+mhGIIh#Jf=eB`z+20Q= z?`L;tn;8UujMJfHTT2Mv+6vp3Nr>F3gT>dnNzEB!-gcTRPd(O98t58|^#g0mmn-&* z4f#2Y-jA_Cx1p?j7ECA_CiXeJz=Gm(5tdrQMi04>lhf)VWU~IaiZc7FK2dmZinr(= z=fsL_4q^H7AFOzug{1q-7q->q&D7=DMlX@}1;4j-WU~0ZS z@7E$rB!wJ@6TkiV>dT>+RC)vdoQTHLku&+uZ$qGAkO|%!_X<3$4Ta90y5Kvr9Xz?L zl^a_WV?;|m8JPDMrwnT&&yFh5*ypchz7o!eb5`B+b8XD@=X{&k@7rPeSb z;S1~tti&h}6Yx8EN%)+u&>EYc#>;(HL&nwRSkkKtFTXKWe!cKP&^_?um2E(4L*FyI z(ARV`RKNKX{CAkb;-FqQ)b15F(w`v-L->1FAky_^!`nJJL+z%5Jabb#?Vc?R+nks0 z4~5~YioLkQ#(=X+45FQk_|%Lg_y+6977J&vW}i*?{iTVdVsbdvF=9hs?HOZjA$ zrL>CNAw7Kip=;4ORO_ly_c_eb{1a^hqQJiHA{>2W7KFzYA-zWu9$C+apK+m24RBrF zLXJ-C0qR_RP3!P46PwAHm||!hXDch))#XntkKor+XRv;|g`l%xJb!sN7c}~R#MSUP z=vHzDZ{Bi}dmywb;(nt<_nFRm+mZ$7GXKH1-Vm5o-8MyU+3t>MC z9-PpV)OUROrH2~dkt>sf9!sg*)xHoPpUaw}M{_w3A z5XRp)AgbkB$(c2yCG`_O|0Yvfmapg5qt=PGCJ8wGT?VfD7mL(8IRSgzWYJ^9P8Y&x zFVkEy|AeuKwkH_flYJj-C8(FgeRNz{=%@BEa|#Dk$NE%&n=$pbKDYM))lMF<7BGG5HeBQG1lOLsK;`}20{yD7 zZ~MdH#gzH{PSd~WbGx~mnYkbK7gegd%VUeH!%W%2V5Yh^bY~A-H5ccrQKdS$8 z1FrVrvs8~~&#NIL*A)O^8m)I%vZl+og0p7?Y2SNsz1j}wsCNa59)rxEx=EcInALaV zh6OF8>XV3;Yh1?6e}MLj8VcXWoH)c44SZYSo&!eU(`u{yqp#0e$I)jDaYZn(PL%ZO z+yk0@j%439kAumyhi>I3SY$>%HyZ|0KtL}WNwA!$DM|LiF1l%*i) z9G%;2*~-_MMKAd!?DGwHwv9P-Uj2>D`C5lxoKlV*yo{yi$|82&eFXdhb6Ixas90v& zl>ZCvgv9sosH-39JYB`2ufAdRU+VJ8nu<4C8uqb;t zTM*;HY7Hs?(iq}X9sF^z5fVq@`>ppG-CNL_UDWzRH(nQGyY3`!v;xOBZzzc~b6Oc* zCf#kQ_(R$K*f@qQTPxfe_AgT(U2gYZI)7}!*^r6j+_=skQ>-{*?k z(E8IP_IA!S^d0$ESnpmadb-z_zKtc4ZKBdx@@Yc(2;u>ZEc*zyxkp`ypXG)07LwKs#BtcQRW)@Dg-cmH zZwFfOLrCYw=~@u#JPyqQPcUQi9@?RWi(%BR$SR&!K4jUDGMHJj0sMMpC&~LU#f4E{ zo67gD)8O3lmx{MEif72ja>6n0dA&>wls$!2q=jtxOjG5Xj`sZq8IO;+><()usdoio zTdMoF`Sl+z=~YdV7bkyhfjbi?h&h^7!ruRdi0u3r9(!}3`zc<*Q(@hqw^b?2xqS!h zTi8L<8+ww~El9U6yo(`Ort~_1@z^JcS>d z)F4jzi{Fz4F26JuNuTgbtzaaLmP-?FsvcCDs3kuL|9Pe2^2XQkfXHQqUXQW5?Hg4$ zrt-qkzKf82usT0f+R@}>OU0#dxwK3uZ=v+)bI)kDA>T!Ii2e<6y|$94_LT;PSy12Z zQWZ}eYuSJ>)tHU@XNXqcDpx&7U@(Ya*-={RMBKd`aIDC}JL zLU}aAoCK(q&{Goc@ps{Ea^ATxco%*I$g=?Pt6V<}#q46z?6%L8?*sBtg7gVCEb0df z$7LdUEVlc>F1#@&3vJDI17RoUamoX9UOyPqo4y03neQBp;ZD<)FxBuiBM-s4-HyWK z)EUy=e==rO&tW|~mJ<)?apjXIKO2MHKYi1L`A&x!eeYna)I(G|H<){&4JZ%VXyh0^ zZu?aHwAfDWG}Z<0tNl1-4IC4;5$b3LfYG5jNcjWG7kC2otcLbSdu+zZ2jpC_9zfXo z1=;gLk!m6WCIqodX7g~(olxZH#kbELEZoHGNco0r=)Z0i}!bc~b z#@6&XAoUoJpJW5QM1&wM$~ouoT~HQ)2Y#k<@V-shOVq`uQTBX3I}DU}c<`D9K>OkD zc{6b=_XMZ-;Xr;ERZd9TSx>wW6FHqTl9yHdD~JyTYvzl}uhZVN%3mohQW+!cTrECf z%|z0(nr!{RO-Q;7{MuPLj0N2RX$dE){jj>QQ@s?t6-{? zz90`n>pw<4(G?7I*JH?p`n-HWzOdr>8OVrr}(mn4TD6JV`ltFs_5H`$d zj1%kB<5BxM@LN0gu{XM_808k)Ya?*mlLn;iNZAXimKf|%1m^2|z^MoA;nS@P7-;nY z$qN%7wvqE+W(%d4nvw62Jd5Z(dytkqAm|Pq1d5+(-YeAZk1|kt4m(<9AY~08O~Emj zuCiu(Y_LV%10?;I6Q>*p(m~CLz+pfhkdx;?3 zh}WaVzdH$4`zFs$p6?gUt|B3A-4i&!<01|jaTU6cG8gaP{ZQJ7Q!{>`;i9$Lik+RI zZuA?ZnFHGO7|p0YESct;GZN-k@!^1ka-3FhSNke9NZZQL1FSzihlyscI5}x2M6`88 zpH`-%L#Nq|G2K+oVdNz=$J?HRZ9Y4gTCeJdALD)s$^<}uLFHW0qI4rF4@g|fyf%$f z8chDR9`SVyL>8Y_8o`J^U}Q-FTxqzAFfyMXzqKEa#!asBE#%3t;|KxnrB>X%Q*H8y zoz+}`6Q|-5y}6nzR5PoMR0GP=diV5^m&__a;je$+-c^3C za_&E**$M9Iuo+L6KNN%=?fB-~)ZSFzPON=j}7|WhP3xVIKkn9^65xDjRuC@pwj7-nkzB74p2Wa$`J5(Sun3t z)>i85i2$Pj9scy`4jA{km~b=*f{UIZ=@nWGX~(MDPRHsSYiUReUGDyA%*t=Pf;szk z(+tW`-W_a2^Pz9H7c9XEa|q&(MWwQXdWWS3#0N!JB@=RPD%sa zgQtY%OgJ8>a}rx_@x*MK?Ny$a=4+@j+tAI1yym^3$~()0X8yGIGoZJy1O`qkz{x*1 zvc{K^1@#_MHpje*2-1QIng!UxtXn}yvo)czTlrLYRq1hVGO1q3Z0vF=* z(G9D-HR(3e3`c1pBW{*K%llA%jYXBO2n$f*aSn+Cph@iuYRzh=I z<}hwR*^E4Cuuz_h@-Y6hug~LlIm&FKDVnX*cd@R4y^wqyi+P#H+L|;|@56u}kHvt1 z=3-D=6O4A+DCWO(7UNQORCyViKY&NZON9Y+Z0f-8`DkHgrY-+b-IwNgSA<34Ee-3%Y-UdNn@d5}jxrult`i4aN5SKiF_fY1 zuyMVYU}siCQnn`FD;aqduHT~{SJxR?G6ZP$fi>S%<1}Le;vVuy`HW_+#5J3MJeQgY zWA{7T$7yaN$bZpXFBZt3F!dU{VKt$V{#)XV zX^duRTBYCn-yBd{seCcz=h;ZP56L5enq?A(uyJEc+_&5ZR3_Lo#tc5S?27N7=L6M6 z<#HHW_#OMdqtEfz-=uk1B(DEy#s3&Pv7qKjNV7jJc{!!ONZFJ0@~==DGCVCt%}sDp z=}~q!xwWi;^gxI47Et+;{G;kCIrw)9s!Xjk&a&K4=`SA4{X}{7A3&soRGdz89`Ye` zq0z_((0JxXL3PzWeRD|+>F^F$k8pr5)t^H8+=j4mn<1Ree2Ig{uBYc+^VyG$1K|0D zjqvAiHSRZlgY5LjR66;DV0~TSGvYe(vGKiS``w$+d&6pWH|8kQi?ina6YKHu?TT5W zjCg4Cv_iAgX*J9K8_nLm7z{hVN8;K(=h3R}ChnJqct7F`?hZQuququ}(?2?YHGwGaf>Xk;zqS7#n{cvhcse}`e|~t0kMHlp-44t7f@L*j z%Z&$d$^%<@k7mbz#OZ!{bl4V`X$XXz!E1?>$7bvXz#rZg8z=4df$=v=;QOgnk( zO*={V6}nw@`I3E?ab3^$vc#v6{dx_~=5Ob;4)F5d4bAqfMZr*$yT`uCZCxE~iI^=0Xa zGw|`=W*8oFNz#2~*j^|3B5)=9)~cg?m{r7n3_AqVSTYRtxdGF=HjoZkzi^n%5%^H| zWKQEJ(Xe9cI`;YQ8+<(i+3!>fxztHVj;~Bdqk+CKz_2~uI#8cy6gkT6`!k)JfWG`t zw=?(}#Np&*Gx@h>RPC^+L*hTMd3M~9EE;ZbmP@B6!tGldR9#`&jWep|=zH(4h!_@wL$MCL z+){~DTh{7BF3uS{lm)*Xia*zNg>8oh;Ob2S;d|~JEO^kBN6Jgsc}yG}>Ne12TfJ4> zGJOemY;7!)+avCJ4q0BaLcp9yit=IIKjRb`*S!LlYiCcoiAL) z!+F(Zo8G%1(rzhVUHMP*wbK~h;DWB0bnJ*UjsJ}gS3Amp(0*{jeYf~>tp}aqT3oc0 z{8PU^d}7XBL2GBroGT!3g*E@Yx33&`{T8k9qsSoNeeaTh`!Yg)s$~t94faFRyBDy3 z?cUt|O=DX1{`L__^YA9Q0S07rG9``*ykL+54sT{fcGK zzlfg8HT8ud9cDsm^IL@P2K?EiW^$id3*nP?nk_sOqv;~Ccw`WKnOIx;EVbl&LL%@;6qmMFE7`^=?`S{s*x3|- zk3G-A>(jH~h~F`wpK;&W1DH<9J8j+KQCOVY5~G*hCeCOle_DsYsJ_z})eb(-a^bIh zZ)s?6+IK6P$gc0&bE6ZD`E`YD{hdJnr6`T%`z z8=+gf%B*>?tEZ3b1^mM)9&k4TPq_dsKbWwJAs4A|H7p{kDd1$B7@Y zufZ_+RrYSJ>XS8t z=g5pbb#VGjXK88s2zCp>nKS*59zgi;1je#ow-aBFXNqm)%eN7T3Gejm?>?MHr=t| zdCF!Z98ep5n%$X_0r{tG@!E^E(4f5&s@i?LS4R%|8-!OrrGe9Cd*WyVKH;>9e1Bw{ zrc2ls(y8A-Si{Mt9$5J^i$649jjLPigzmFSaoc}~kk*IGcf{jdD?K1i$LRXaWs?Ub z=;!zWXrI}12C~Og4{-W+U-W&w2-$pdPWr^NuC4>Z3chO5p1)q7kB(bUiA^zqkTN=! zk#6F@9!Ht;fDBN&c6MnhD9!D7MW3%dq9+&CbCP2NZK&4PNH}A)dmV!bev_GB9>7&w zSJeNpU-dOpT)#Qe%Y`s5LypHnWz<@wzo6br+6iszy%kvDj_2l71G*+0OlVHHGUvzA z!yt0qex;R=b=3f~WAA`>#R)KtUyiEhlotEmTM2J7+=$mouprr<-#m5-AFRDBww9S< z-`C$jr)Etpai>OUraB+uLL@H4)KLqGBYHq6SU^xyeXhgIkC6`loW*&qT)bU9Rx>iiK^{x5ss0;>L^R}dR?2^9 z3+7elR0}?6_dQm+EFLdT%hPsra)x+rpq)|$Lu4>evLBZ*_CDIR6? zI@}XdSCX%R`a8m~VdV_SjvdbRt{%bNpYI}RIja7b?>ivrRgP^yZ*gNqN8C`o0JqPm zFL$*JL*f{m@Tm!pn^X+OkLp*|d`7{rD&IrCLR;us2*=jVl6k(NN)O=o@>*iv+6;VP zwM}7=_7jGLd8Yi-Q>U#|vq~J1Ia~&<&tlW6SK$VpI+;2LgyvzNb-w64pbPlhw&N`~ z+R4)cyK;*mtF^D`^Y|URZRCp!NiKu`l&b6U0soB$@+(3=_NTbu%sBLXjK&>0y-w)|X7FVb zlIK$xh4r)SW0BS`4WgeapFmuI&!TtQaT|xWJPRW-?}!x_8c2&FI^bL$ zEXe~%(oZq5k*-`3_=bf|H4(|}n#tV0E0A8tG;U$=wbKI_Wzq+0Wp%{Y>pg*Zl+C(y z2*`_wW5ddjyf@;8t@uOUSN$mV4e-F=H3^XV@f`f^-HT5g+*0u}u!sHN!B7qLb}DMe z?o_x|K3Y((3hEVXUB50C*6bh^pFdh?ARQ-k#GMm*(X-nemF~K%>?K^U(f1Ii+RERl zJHV^jR~V>gp;6wA&Vl*;@|69`uCN6SJsIg4&h~#*#Xa{T&A}*Sm5g}#EbmaqHn^m;x&F7cbh!jbDB4| z;_Yjk5lZ(1t>=T)u>eN?QHFJ^rX^m$GilzKb0vVEwmgJA{o;{$hwa=k8EP*r#OpcJ za8bQl(s9y)VrIMXO`<{9aK7i3-lQ|;cgNL~=IdL?G0unBjTOg*+s_A-JEo)d z;y}%ola_ML3yHKZ;<*TDT|WX3a06W1Ad@`R4BWru43t+}i#cs-i&J59R1bjkd7||j zUkiwT=vrf7%}SaVlb+xEk;lG|>JEy3hqmm=J59Q%u0h|+7{X0%pTqWd(tz+U%(e`c zv;GdiH*3;CdHi`cm7p?Pk6S-ML>$J`miB1eu$$~Qq(7##{Y#lK9M&}P$4l`(GT~(# zKJdX!Mr7ZFjOm)Kg{m9pN0QS zbfN6W$wMg}k@HU+h4(9d0cE=?oGU(+?~f73^pV5D9*WJacF{G>@K2c?asONAvZX&D zz4tE|r!#@6Y#i`*ebpZ8>!P?6e^+E0p2f`nDm0YEq1lxCKzUm93U9#gbZw;~3o#%8zx_A5NJb*FLO=v^TiTr0PGFT^X&tN_)wFGlf|WH|g2Mk?^;E z1}mI90Y}!=B6;h>33TczJtIIL`~PD4;1``mLk#KTdw^g5)2xgd1ouJ{Jn; z+2-^6A~jEMPX){LAB-{;>gf#ztHr0)IV+D&_`0SttV~aAfi^phB)u>Dzh<#O{6T$K zM%m>sUVUrHFLiFo_V+y@BAVCWeV$imXUof2qMH?x*1^%Q#kkby7iG5-gb@#H5s4bp z-v@y1g)}G8kpBRy$DahP1?wjDm*h)~=<^|)2t+KM}+>c6p_>kGwu#EqzWJYu5@s@V&9{I0|4 zGlzAHZ^7)`Eokx9RYP7<@e+2^{sHo_Djze=5r*cUMu6vBJxP40sa>yGl@?GA6Jzxk zaq>$z!S6+tPo}(qHO@Xz{ii&&7&EP_$gk`RT{CM*@x7#q)0G!jSXUn4U+q2YdjEYu zy#jGgS5RSsbR6#aFVE?rvyJ?BH~QYokgCi>8B5UogT2}`o=1MTBRmhcLLBp5kfv~B z4|h=chPXof3ffE9UM41dqO-i(rfNR4@X-NH-R`j%oNidt-?(C4E`bbcw1#Ed6j@TPPt7ZjE&9=_X39jNbzb!j}G5{_-q$}y% zwL!MuHUDZi<>dWE>$26#=K$$EzjQqiRYq2xxa+%AW}Q5cbUX`IZ@PwrVINHaA-$}vDa!HYKrf%5b8UP1mJ zl;*j-SRm;P(5HMLkVZ<%zrx6Jpz?Cq!R0#qdVLFhXJ19~HG*`BT^_$n++5rkLK+R= zmFKqNwZYCOoYRRewiKeb4gi)aGsG~OBO%R3kutu*BliAy97)5(iuRYVqf<8QYFQI$ zPK)ee9wV&)AXXUMtI7Q8Rh&wXyTckhSnbL*1wVh5HWX5N9`41m8e8`kW zlfFa6)cv4l3$4p^IL&g@3=kJD7!2dKr)r6(M9R4DpnQ~ar+=Wrrr@ZZ3H7lV# zv3ABqjLuL{_QWk#^nF|H38-aL4m7)~;(-&(b8%abI1Hf2A|Ce|F6@Q{qw&$|?C#%3 zRe6MXO%tTOz;0J>B+O4=hJNNBMc(-wMqW|+ZLyF~?u3J0mvPwmNds7aAfINjCTiX! zDt4wJ%~UvXhw>^&Gf>!ee>76>@kPCI)eMMJPlB2ek^dKj1D0j5R{R%i3CA;r@#7h{ zMOKsnKNXYtifPrXB-J*yPHbyb-9?EH56Mo1jAXK0r295kL(zA zPi&Zd4L5h0$}(U5V=;3YLVvp`*ql5Wd%p>Q=`ZKv+RulDRkM7}$&c?bvi(ntz1x)s zB^b-78-KBB$8oSbZzRmSp9H_IJ_WtQ4RKS=a{MlD!ofZf;=_l!SaW3`%%|s+dcP^h z$M#1s-?KKV`+Rrwg4Sb?f@@uO=^p+P%ISB?%o$+@r5)nHX3SGq)o(St3tG){vkFDC z9-pzo^(f{$Z9sGFVOTIu%ght)(EmVZQQUPQI~P)_`P4Tay7|M#w&2`&4{o#mHMaYfi^8Wl_paAlMpUEc zf&DJwpJSgzMMfK5>D3c&c%OiEPA8#po)qEUB{1c~1lD)$5+3twJH(zn2|ZbVJ~eU@ zW;+^5daYPc=*$mx49B$EkHI~B3SSdvE!+I<1U7SDI?=SoT?uv z1Jf@7{Ts8a*5k8?{=8ABlZ+^O2ak6-usJh!Xgt>%qrya^UEM_SkuU6}!((Cd=OunT zXbwNyjF%nm-^4#H{*#UEU1X!sbY`^ZE<89=18+{rU|SZgm4{5@v~&)5uppA@UNnU5 zr3W-U>2ufa30-8f9&_p5&saHqH+7+z8(hn_l6i*J`Qz@X*mzhENxuq6=55uN2bP4e zX=6XBn!|@d8^z=9?Sbob!o+|e81T>nQa)tkl4Ey8{S(#YjEp}JQEfg#XFIGPugm_l z_{82HHIR1Gjkxd4H9&t?wWe!U=i`QL13k~3-1+-Pp5gc%qeq+KJ)3#-JFKqZ&PHdj zn&B4I-Ao1q_5H1 z8vh~8>d5dZ<5=p;JkVjGviEp@`0ybDtaGo6bxxhtdqF3z8t>)w0FPhs!Odr0Q@;dM z)uK=KdloSFl!pF4xWt;kxM2%`=lX1tYWc(W3 zRh)2{4DtV66&Akjafa(E>VZ)C(sUdeTa1C$FH^NtSDrS5p7o{Ks?pm!c(9`}*JfJF z z?)Jmk4Z^`TXc-J{pNY-ODsyZrm+<%7^`J-jDR|_&7gKNbftY6vWcRk+WUVt^3O8W2 z?J~37z7p4MP1UAM%)+s|Hlx{D3tpbW(QO5NW>Pwp5nl-Xb`Cu3`epd#GYOv`7z8_3 zwdP)%I>FQ2GkD*BxvbUnimEkCqddH|S`G2vw?=&7UV5hJ@;*W9r+TC#^%qVw?ZwAE zGMBp_m1v*6?<4Jk>d3Tj58=w4Io3TUWv2-nno;quq>pPd;+DhE35EvAdgA z@eOp#tR{)e#of!#+3%UJ;Lf_S@@wsEyw(4l=#=D(_uu`&mktv|++xPpo6|G2+b+B4 z4yh%g?Fl zfw{1B@EA66eNWZ5GVimF_TSDy8m+i!ak*`NRwzFqf{cJ?sFZh zkUdIPM)vyJdrMIvDP=?%B`G5+b)V}HWhPr>OZMKI=Q=&F=YM#9=!d>vFYa+Z=Umr& zb>G*A9gMgLS&QBa9eXQD_=hpUAy^iWDP|tni5C7wa@HbOZQk%&@~Ma7wwiK^%VN2F zgsz-QpTnOtYz7?OTn53T_tBbNWzbW77F?}48&Tb!e|+vJV|T9+2fCi(Ib)aO*TVMb zJF-kX=)4^p444DNjq;622>Te3g8N%^k{^aS$g)XU#LqXi#8K?K-&I^c?<3WeJ2Jc7 zVtQbl_u<0ltluRKgne`-T-23Rt87hX2@sEgP1J7mKidNzoof!0XU&#vFNEQEuiN7D z6i;s4y0`4uueU7D-Gm-b9>%_a1e}(}kf^b6VeQoM93x4~}5;V;l3Dob{BQu7p zvpFz~G+6EmTp4Q1rz9`<}zbUTg`>F)=2!u38%P6z1!k^ z_jQcUf`2^#s>k7Xjkgv3*tm>Qh{b*CC5vnz|FJ_SL z)k2VVf_fP}_|1v0AhNtjG_!0iX{|U^TLngDG>|E_M(BOY8VhSa6wSSR{x1$ieAAFN z!o+|foO%~Y69}DH3vOw=0iV1*jX^{8I6aRO?n|}L)C2sJiwkM6TXc6CR`D;iHRmVUxd;oL&?wh~KC``sQ`m z+6BU2X>oLrU{-B5NjR2@`|GyXcss>t_gtY_1M@zr*XNNxdn>(!hdPvKa#JJlecgDC zZ`JW4?#V{Ho4y)5cfa4KV1pk)#@Dq*K{jFGH69 z@+?cJ(*a7;XA@U9@h-nrAl8^&#T9FeAo;D1JeYKa-SnRzPYxP{dTwcqv;p=qN&?~; zrh8)>%xvh$FHdMKqxL;xq%XMb5lw}bzE=ApDt$kx>suW0Wh*M5GA+rJE4>vy7qQ~?UIr==LI7aCoQQvtQ z1~1sp6b{XuEyeK|knWqi0y??z_t6{i(e^aq*Z8l{?yzN}JhDZA@eLe1-~x+s7$@i~ zxoc7z=y>-M-kTr8u9g(SO+Hq;{AH;4aCJMYmr*2&2k*plAD6>~f;)_QNs?}WaSwZO z(n_p)Z~%9?+DtT<{2i&@h#Skbs&_8mJEAQb84WJD7mwV2Rly6SL!q3u=XzRkz&LN`dJ3Brm{Dk+V%>6>?1Xce|&N{@SCvb+T z9WNh!iuwB&vudv#=sGPyVd3NId88Fqqtbp&7bcNU%ON~`i0z97={Hx>)t}+dsX34< zO?blOIglP=&w>K+*ti@V{$LZ71++lYU*dvWUHZ=st=iAN zwLKM=F(3O<7!X&Yd>!j*_6x?I@6M|n(!;&Z+mW;g%#qnhI){_4V)OsI$VlsIi4SyTY80L-{S?-(k?Ko9 z8dW`4?T2_m-+>_CiuUVENqdcE7YgnNx|2WL1d0nX7HUE14D$8j+Pa>Sum$a_4HD$3@Snjl zR`p{I+3Dj|=yov_tG^xqH>Tfa#81-VtO0KD{lmVEEd=u6;1+lc36FtrhLb-*rD>e^ zq(I(3D;c|vejiRaS}^qvuI}DOlE1E~Uv)jCiJ88SE1Z~fUU_%9e@{MB{@$xbP5iTE z8V_H4Tx->HE6@)H%`oKVheNSh#1SMeV?Txuz~PhXh_45ba^N5&jVVU zg{0Lx@KuKUh2mz*&+D|*mkhTw)7&v$$;fvQk1obW^jTi_hTR}b|D!PaB#odgF5d-&!$>&IoSAdse`daDi*7yXp~LDD8lUkbuWE#)9u!hh1FZ75gJ!=f8e zLe=rlf2DY3qri7#qAD^mm58B)fgydaSF!i!<4pmcC})ngjcl9YRO$|GJC%DXDe8NcZUef}#& z`8M`;(@G>yfz%6vbcIIY8Q}veJg}bc$mu<_r1=zH0i8RLKT%jJCS9E-$P2N6KTg#q}pZej^;9@%j=luh`Vi!k9IcT9wC`KA=lme0r^16FXP$XEPKi+#{l^|#mVf)>_JM;D_?^i@2?^08`9#} zfv%gAzmR*K?MR!P5hvz!_a>iTkta?``U&I}xwj?`)4n!Ag$1O!1o5wU*Sk7X{v(sh0c;_P?_q4f}zqmx`DiYX@F2U3GJsOUy1kYY+_M8UTeq`Bb^7R zzPUIpnKDQ-PM9x>UeyBqau3SfKC!~B!FgXc=K(580Zxfk zxXzsjCrYLIzei_|;pPF%GL9wd#-@9SFfhF^a`*@Kos zWfbJIaX?%k8lGu~vleBDpUobs>{tZe907+9j78V%Ji?A7&4}gOG?XcFx5hh15r zubH0dg)JHnh3fJ2`C0CP#`BZWX3!*lFncs=HTC4`?+tj@J}vO(dJ9OOlnKXom!b7# za|!7dyvAS)-hE9eyzS7DpIaHgcP`2n2~WR>raP`+c>n4$;f$F~Ix~jXyPqTt7kq(s zw_3=LTmLYx^T`l2-+=4!Vi;Xx3OsAsO2`wt*q+G;@KSMW+1e)&238H?J-5W*fe&qQ zRiZ}TvEGV3BMn8vofi?W{DHOGCAhX31^qY2Vd#u{BIEH=i2ObmmyBKywf`JK_oM&} zUEUa*|3F-B^h@|P7>}=SUWBTtAoX}JTF)G;6(_KrE_;cN=^-(^VQwnhe?9 z8*7#0eijUr)4VGj0NSXFXQ`W@j7J2~RtIzB<;Er-;MgLH>$ zNTc*XY^)}`bZCOpe_Qk8yGL={9VJp~b)kA4jK4l^lJmxx$SRv#%E2F&!;UBOarlrk zP}DXBhpnz9*O!jv4Z1$ne0TK3;Wa06(}F;n!FpAgCwE4dfL)rH^z*{(N=-ih-eBsv zq0nr!D|Bn|0kh+(a+4urFeNA%R{Tivj+@#D_FvmA9ICg=OR065t=CqO7mSDUu}4~B zZNHjQcajbNU9|zjK_04h#j|bSSd+F7kj|Y`e}UVby*TRqP1tfa7?$6u4&NP*K+^%U zSl2$8=pWQk6vp1skQmNGvX#-%jG`8-|-DwyPeCVVLO@LH6w8~$yVNN zbpSTg=OyS&VEyxHs+NVVOCRR#v`(02_2e5Ho)J&iz0ov2wgK0B_NIAiAtr|&yeyJ(xaqN0Bt$2R7v5hsQ3%~qn(K?6CZ z?p|2fr6=#0uonGvLxfY8I5s#t1F0TZ?8&E5>&H2-YvP#Xz4FD=P(H|PoSe9E zCx7IW$_6{U$1KwIsvaNrJ0)hl87VhBvsZNu2`vI}SPzc>cz03enF;UHJi$HSD=2Iz zX`PPrIu$jtHJ9eCnarefe8H%8C7p%paru36E%vHf9Z1h!uc1Dd$w%*LOZTV3?&kV@ zu~8EtPJTk`ZX@CBA$xGUU@txV8zKEWw%ET8mfpR~j@4Zwx}Dii^ZX{{IRw_@JC4$4 zB+Z_(s#P1Y0S@)(d5I8^a_}nw)W!(+)U*D z@XliGw?;B!0fSEatofA7=LGcyA2rXIx9-1}oy<&t5fl57kG2wQ8-3nuGJOx@S6`@6 zBU_BfYR)I0XwUadcIScaje#&wF4~a=Ar}VV>jwLwYuXm=zMgh+c|d7yc zM}Y&y=_nm+{bX$4S_-?dLEv<_XcQ^0tXL&p52t>b z_!j&3I;v_|WM@4E!WMpFpr2)268{y~XgLw3a zl~i1}^3DTn^&%SvSEejsepRLwcf>@#mL>SaEDuBkkFBo4Zlv<1R8uFkAw z>nHd`&stE=<9eq;&G|YT;8Ms<%=b=$rcFXYVfCs+t)}jr{g9Jf9SGm#=zqaT&xKRF z#)|Vz>$2kya&V??GsUwS+G{-JT)Tqnt3Qf|CT&N+05fw*^<7bO^jx65Bt_cR?kN<6%o_-eE`2-e&r#$aAme+AX8~}Ap1o$?HMaS75a^S}+Y>qAc{>^W+1gBty*$~(A3ihM#wTD}`#iJ9B zR@B$B&VBfu_0NEKo&B9;iNjMGDV&u$r}}CS>?s90YvDNi5F>ntL7QrG@BWKLcK6*# zxGfal5HE^;pL*kiZ{e_euRkMwf^QOjqFd+JV#Br$jJOCjo;I>{R5vK7PM_Dz<(&3| zFDdR%7^fpA4(f$}h8i<*au)5y6Gj+PJ{ zzY}y=tAo8gW~+7Kx=z*j-NYGE)vrrZ6<%$@PR#ySh)Z9M(?ov%imJCCTGNDE?DAsrLBKg4W6VZ?=-Oe~`XMk9RLQDn_l^sOkLl z7>u#|igYfF^ba%F=-{CgL%hBG9=2O|4K&|Q=20KWy^r@HSPhh<;fN0&iYsmQV$sg$ zK=r0EdGAP?YMq99n)P0E6_$0r#Ek4wC~VH0*b4Zx;~K(fZWz9dkw!tnOf)Xgk!2Z6 zx%q?{c;)9cR5cy{^*2*`cfIFDu=6s6#mD@xY-vrUkLmheL#>B(WRiISmObxJx???# zZ$|U~r?r>8h8;uI193~51N9DXG3uab&~yc^iSmTG$4{Zc*U>dzYsL)j4&~2Q;H2>b zA;fT>_%(T~3@^F`q`k2GSye{-#nwKri}ZX*r|*Yv=#g_@6NFRfV-bO`^0f5(o7X^^0-tExqSDM%cQE_+B-}Bj z8J)#V40PXvZqePv)nm)pN}DX)Q~t|))@%!Y=XPf?*~FebikhU>23f3*mb8mTf9NB8 zRO-jH$qyjdH3OSAsm5EqXboOI^jWukhcrt2Z`hWPQ)8W!-V(W2(jec@856!NJ$a9I-~d%)@6#c1+=2RIpbMAk38;_>0Z zaY#KWex-T~>T&rk$Ur#%%)^qGJz=AzRM!O4Og<8~tjV(+ze5nFD=h-Vcc?s;)roMP znPm>7>sZqPp9Gz$=E}i2aHUOCNq&&<^S=s>LDvZ6m-xC)q3ATgfuDNggAdOi18K5C zWAS%CdiVb== z?rxZNUq}ACwhHTdmttb>ZG1325IRlmC@%k$@75zjtjbrTQ%9}q*khn!;UYjSr z3>(Dj&T*68+iwGTHjJRpa|)W{_;JlFRC-mZWOk~wyzUOGK>G2u)!;dTh0XY9nO-x$?lMQu)a zHw4LBj8)A^|Ujq*z;Vpk(^QHAbM*oEUY@O+6mOG(vXXUBGs_52<~+DG!A-@mfN z-kZR9wY&0){9q49PMTdqz6nq4s)bELB+Fu3pj(GuNax2|>^D`INj|>?(7CGT3AHBY zHc_yob|$X&A^q}ou~2oO({m-Ny)TR%3e+30)8l3Zewqv^*Z2pWsPJ})?M=y_Z5RK& zy~hZ@iPvcU{Mka~$zW$>dqMsj-u$XjF^Dqw-s=G$79)-t{TMUz48*dJn1U zPic@TcnfzoUnP3>ZUuBca2qof-vRj;Ri_ne z){R)K9z*$s6R0yHOoD~R^Fd`3bW9|W-vzz!W}y0zc#Ie1r!XH2BcyYr znSaqp*$44r0$w@zMV${4A2Z61w3*kwiJ|rm)T5tp@8$~_y);7*e^4*&Q2LsU{cFYv zPssV<2`BjDgky_wQbgtM@EzpwIs zj5H_t;`({*8=CQ8S65DcQqWn5ku_>dbv-Iy>$KHpl-S__HzZ^f_e-cz2lzaL-r7SyjmgH;}jd+gJ3@hhf#kgumr9(|47Zh|I6?{W^M}=1{9^j2s59FV0={^5w$~rN{k&?1K)c>#@Lj1#(FT>tP_dwC> z(@f!p(kYJ4{fQGEW90eODl-zaUa+9gXzi$944=1T;G}vBfbuEGj$6TUPbTG(x99CT z%^?l_2#Z4KyTZoFpt6==JDLHzs)r;!h-*q~LH^;_u=z_f*l!=AFqe^r;^f`cJ_2zn z>(U{We0g^f+WIUU8sZ||Ti41X{U!n_^A9m}VauZ3DsZ3jJp6sCm@OAXbTSUJjEv;Bz^DjqG`Ie`)PvXmevZqQ>Dj=Wkx`ua0-?!1{%PiTk5 z*HymKTV)5Naiym}-LtC3CC!r2HF3~TUqQHt+dsP_WhE6l{afT*%5jddZXFgO`7?2_ z#|~jz_msi_^6;iG*(9+-pKtxIDPeq`@^BiJiRK@kLTA3V;u^nC-hjV?hBB2GQEowf zaZt3$Fq6+(W)TN|g%c;fikk<%lWu#A+twg1`dJT@E*jQDGRompPQqPRbdaXqBj9<* zW1>&{rG)E=m~vxAUfk4_igR4oFADE{AIv-wSHk?JUO-t3)l6ea`3m88RiV5R?T_-M zlp7{#<~PknlZjKo@4wDWor8Ph)y!hvFQ(S_f0;>;Tbj2)4}Bz@fP&C|kl215MAG*+ zs6IuCV_&X(5sYljn~lzZ1A7K5FROGWZr)c++-wSz1BfSj@l5G9uVpm<|Lh}`@6tXO zQr;1R#9JD|boe@D6l*c6xuCORt-l`!?^1V8ITupxF!BVdCK-8m=6EQSDL$YaTxnHd zIK+%ojUs7k&C97ll%2(5`N%%_$f*h8^LHejr*cTjFrEmyUTt2p7|Q1etG50D!ej`3 zRLrg>ZKd4%rOMd_oevU@C_TrNZzBz+A)#+U!QrZ1JR_;BV7oBjJG5=U~%VA-0| z4e)YAe<1$F(_LK|WmJ{(|10PJSI+;BtepQ}IsdiDFimuws! zfp?zx@rfZX@xPBJ@w9&f9`N2r8nyn6ePhpH;hIn4XK^z=s8O7w~qwH^G0UN1$h$9=vVAIP`B`UB3BsNVvT6;>>L=p1brA^p8*Aa~Iml z>pxo1eM?^E@qwjgg<8odNnkj;j-nbf)}pNGo~r@*DV^?aU^w zkJa2;VS=hAXg#7VF;`sr*PIMqPUPRa5 zcqH16naug65IEiC2jmAF#08I+$s>X1aCrZxqMAzr6npFQQSsMcZSi8NJ!h)p(L4_P zFd?8j%A&4Z|3WQ(wO_dQyWSs&8&yU6eH_M4K0ZhLxLO_`G72^>S`X%u=KtSm$?we` z4OilIc)#&?(X8$uRJG?CI|^@CUjY|7zh*P~9M`ya(t%PZGoaewUsqP=Ip5pk+vufO zf4Tu4{Lxc-Eqx8s7wtp!Lp>vg_a6s4rd$yJp19$es>>?gcWI5`c+fmZ)vT8Gl^;vG zi^Jw8vg=Q8pk6H#Ui*@roMP|JovZn=3!SGhwFbIAUVr*(*!Q`yESfP8Op|EtSf@pz zZjZa*@iLwD9F>CxJN97J1iJU-l%812s*$*SaD#X6`=3SPJ(^W=<1&9;W1Z^BJi1Oc z3afLord9dPIq$L7Z<>|jmdfb|hP=yx2iCXo{S1WdM%(!^!*u9s-%a(s=yojuwl#f_ z7qsaiYPuu9c z)rax7JHWi&hiPA)!qXSQxYM6Q!21Hxq}xewc3G~aJrQjSo-#TcL3K`h_77j#egeWx znX$5?938zw8U~L*swNQ1UI&wxg6Y;UmQ`Td`3z)WIm^8U^l-5)rN_Gy#v}b>i>Ofo_$hgL{ zy=ZOt$h)gBJNn7|w0z9`#hCy#2fk?9#oJ zq&3JX-{;6P^+Pf2ZIoQ_zNxI{Qnv!%9?vR)M^{G3Zu#x;_TPbEHKMchUK1yi3U|S@ zs8z^5)7&gN`?tG?Fv2N3`M8^0USgu1eRwK-wD82F%+t8+9etlP^dk5icrPyWr~9Qh z%f>OoLvX^0Owl81CpHc@Qtzq#)~=Sk*6cF%bWPdCX|e1Xq${Z}WpS06;2K*7#Ff1E zb60t_pCKchV_^pN{L)!RNjxO(cRIxtma28uSnx=67``2w^qEt^S%g3AMOX^bS;?A1 z9OUA0S@iwDQtgx$k!;cCtGH*zN6b9qj_Ul0&%GBV5r4@wN(dDeq@x!>Qm$7&fU@57mK%M0XPbc1_&lec^If(GH zOiQoLugqP7#LE!05Af}|82+%Ku1r367G7*#fybBI@_9`cg3YgRc=p&`VT$(o`6Q9% zXfD4*w9~9RVU3Dom$tIz27Bn9WAE2d&2-{7r*YNe(A?;)!ag`#qbX2LVEyiw*sflY z*kQPx^h$P;vHSPq?e9)Fzn-N`Kg(dvr&gkH%`C{9+JoDUc?I--;zZ5X^2fFNg!NS+ zsjz{x`DKXeeG87>7cmcdRNzpR{4bh}Iix~&#Px9!Ni^qUjs6C#{_-?FVR}5& z(};&GZ6nn_lg8t^J}xkz@D?iF)Hn)pSCjsX>H_X@0!sYKqT(@^MdeXhj6b8^wL*ELL~x>hVN5x!yz zu0D8%YO9%S)#`#+lzspR8|A2Ye>n8?Jp`u2Q!VSknumkn?3c|*I*W94919ur7oS@7 zg08i#<(T0*#FGZXaJ7w8J-M^PBP=LfO*76v4(AHZOEqXKg&q#LRrdeNqppeIlE9)xpoDIB^y;>J^h)v z^`0OX`wzmBguM)}EX4CP$8K5KN7mxXHppj={Cl(3vSzDitn9EIFF3D}9)+K9)3;*W zy6QO?E_{m@O%ssx948OJ6`v~(Yh9P)hVQrGM1xxVeB?MST@N@L{}M`9k%rTd9>WE? zVc4-wLrL5~n6A&MzfoaUuZGhIGxoE-bWiF-52c{mlZwMa9o=}pbsI5!Nt(Da_mB9p z&6W|SiiUbO@Mic0rfN*}I(Zow_vsSm?vG*cwh%gajn(=*3l(!0ZfAK@i-hIOojCQ{ za-j9G+^uaS=>`bC(~A#1x0G4V4^X`)EXRJw)hq8p#`GuJBSF8lbbea1j?rM6yO;Xu z6&xQ@R^gALYrVuz&${yWgZ69p4=iD%f4TB$N&|#UzAaXrY>3peu+Lzn=<~{s+g~s6 z4t3lwyf&OwK8&3-(%~I?oIv7XZZ+Z*dumn8?4LQ%by|Y3CM-0_gS)3!;hnr;*m-Ue z`LrCmhvGv_KPzBPtSgWnfGYWOD*A5fkzZic$&S|@qz5mjSgV>Bi|c%21j&w!)B)J%dTR=ps!lu1KxPiLyTSh1&MPYzD*qH{cSFZdvHv@ zQgqnW2&zU6WVPN+Q6945+H`~Rlva`B&y2yYEgDr|4{21;nHI-})NLSi=9~rcdPv+u z{&_T0JWBoxslSzOhbJry&Xxg;N^8!rJdpTuB@*5u)d-N5QMJO#k~(5m-+0Bns5Bh) z8Ia%S)UQDM35)%2Da?@#YiB~KO99$hRIlJP^1!I{=ex)vAg>0>*C<~*!gLB0oYxYL zYha|UmADz+Qxb;~js^+82H?(db%gR6F1Lb| z7Z(rBQkl|jkFGpG^2zu+s0(?KA#gLK4&?zx-2V1lwPz65Cz;ME1xY)=b(0!6Z0Qy4 z(z-OSu*n-OWe=dV1l1Iu6PqDUG}wT{N37Ro#M?3R%PyK(uR@q_TriM6q%&9rFK2cG z;yXb(2XDD-frfr?!Ph(DwEr%wu3G|1M-(Pr(!8H+gruW2q~nz*1^Z`7*q}uRXfkKN zcvO_Z1COPMVH?~;*rTyP8XKp4|H4-GFH&p54xMXg6;BWjBjpev8rpNkKje4CgXR8UwV4@>G_ zByY=UpOoLg!L2f}Z$=={nJ}tN;(3M#n!OUI!tw?2yd(`mTCFXTrluOs!$zq+u+yiP zjMj{M+h12YL`zwRFt$F-p8l4gFqZNecHlKy+B_8mND#WMHidmy3XTgsJ=5`XB)Xs6S}x8_J0 zkHUNMe6vLJdfh=kFB>U4!d`ZOA%m7m;x4wkz5{=;<`xu9c4E)I8bk6qKS}R}#2cy) zMdyiM^C$<=dc15V6^DdJ>tUa(QQ}shJEr|C#dgtx&LpZL$0Gb9|LLRf4_tRt&r^D) zjgEW8>k|e`@=euhZ|RIn(Ag$gI7Y;4yp~SEk^`YQ)`5}#b>y|) zz1FH6i|~qiAs-f{yDKaelqU(& zC$)CvIU&t72O}F@V`Ciz`?7upuQ?=%oeJxKlsiyOT8tCBncVS5PsZr)f@ z&VjGoE&=Hmru+=$m|Eq%pT2m+h}Y=sw+Y%u&^q^n-qqZ&;9fjy5@b*3J4U@GlFwr* z&!OI^!0j0gujAkvKk)w0>p<5{S(UZC*2s>u$r({Tq^q~uui-t9&^+RwVE$O2E4|xD z+XNRLi^LC|uZU_kc;{`Hy*%9W`1pJ*uC6184O&XQo(0Rb~qdw@BoQ#y|a>8#xYlFe9TAd8aMC55_EK8mWs@+8Plx`q z!`RGk&qdm^B`SMCsyRV*q+zy)Fb>YLnA1SMh35a;BxAt6XZZ7>K^}1rGbySDJ7PkW zN8{w@$T$83W9ycJvQyBjwN~LCk{(idv`{*r&#e=J@`s9SWMGVz<+B|c(y$upTdus# z^P>eoSSNL+S;^^(T)=ycm6ozomANBj!RYteh4wp5;W@-j@RWh+-Qe6^BjQN|@6TuT zxXR&N{#tU%mht`Yhe-JZxGwYuU#n>Rbg&Ot)_(~7+}&jRhYoy9>~~?Bx18Df>hZ%v z&%n%DDOlt3G02&%FHhDAlsla#}fH763@eRYLuW$C!4-D}qFjw|;+kqa(AE##-owfM)BiMX#yK6b3?%jb67#h%zc zVOwvVgRZ?k;yuFFdwU&t0AB@{n~%evBTf0MqFuZ&##*FKH~~}cX0iDF?PVLI9x^G^ zlJ`4$6&s(o<~^s|$kyxr2xR-2NSP+ZSaG$!eViRG%`ejyILNKU|2Ln+%T*KUUe`^k z${fo{xN&V=`Sr#)cxq@%*Z+d~TG7n902_2my9{v~qWFKGW=eKuBt9++0rj|;B`1N- z1Z!u|{mSmoz%G9OnC`gy%);iU3>@wv>u=qHq0et(q2ocEX>~Q)H11FeCdJY(Ts3uM(O=WfZ-NU=PU9si(r)<~0X8hcl zRkB6jrtH@IUHGTjFANTBD4p6?msAI$KF!Pf)Oa7xIbtLak zeg1JAMp%<*D0kZIz~&t<35Q)vq0j!; z=o%A4?-h%CdYZ{ObPw0s1-?@E)~SkGRyD12!a)A`wM2xR@x;B${y`qyvt?uAICg$W zjMm$36MOfc0r&Tc<#((s`6S;q(yO);H@2xKy9c*p`=V|M$J7s+;^k@ZwP#1(*Qu-Q zdUyrD(rU19YxxzHI0nfMFT+s(?_HMgc_FV|P{`DKt36raU02ql`w6xjkRVI-1>`kt z0|m*!ymo6>Z54JMQZAo_c{HoOD@zrnORV_Brh0t%v1I(uza}sG_Z;(@((g{HZo(5k zH_7dDgxGZWFg$ws3^xRC;G;%H;0rv54+h@Fny=rnPy2#^YAx^VNeeJe$*Mr(rEEzb$s z2i)+o4g5Dqhd0$(fK#7tg=Yzkuvx9EU>It~SNg1AN5<~2SWnL%2-;*_e&LOs)Oh=e zLa;7(I2ONXZL@DR2M5Q3Lae$F-Vqf$GT! zjBU0-=e>2&$80+;UUwO)I7euV>(!Lgmu|tk+DY)U$_$q28^LnpT%=)WG}ODRD`U5O z0;*Zzy$7N3qX3x}`Vo_}YH?bVyd4q_%W{W+j~U%>``;=|GSwjApqEcbHCb-;5!SY_ zr2TCzPi|ereG{|AnC}L#{#*=ey1*ZYd#t6$%#^BzsD}uX;-PlNDyD7!S>#y0g#SK$ zgjR3r@SQt4gH=r{*=b2F9&S+v(-!OzRG;EtMkKyE-5eg>+QyPXH;OK8TFHQ-M1?zS z=MK8B#eH9v5U`2;7-}nP-pB;ixApF>RQQ4~JdqFACs0W_-LieT7 zD+k(lZ0OdAf3WEXJ?>S9-iGm@aDL&>Lio>rJzUMM!?bSBNarP~w?X~=`io)eIkMfG zZ<;nS@3jk8PU1GHvABLtl=Ma z%!MB>4g=LA?3=e$Jx?CekWY`bmsFn+)!z$W@kpS)<)h#3q`C>=@YpH2Oq}A;1=A>2t(u$n_jYc$3E~Ph3;c<;TrY!GSN^-5mo2_YqGoW z8C%c8TAvY+_F@H^|2m7*cd~E7PTYKPw2b{x6VG(%BJF2~;Nu_pa3$SSTGo359e(%4 zK6`)A-q!=d47u&uE@}2)6TeU`h~=8-@w=}5MYC{Mcw#$NIxp=aA9k;-O`*$Tn z^V~GN9b%}7^j_dib&4DR>afOx4${5Es(|90H`5nLpDibGnM)_wbhst1e|VV<%W4Pf z+897}{HXRBx4*Vx#Ch=L@+@xfAru!c(&KHnmEhN|y+L#G4c&KjKMwW$OOKz45;KrRp zaJc+6GuSXeOz!3^-z~ce^lz$$__8CF_MB46e(8h*_i&PkX5;H*K!mXB=QFC;6ZG)t>aHxH}m_zq$r~LrO)EUe-v9+9;#F4ZSTpREQ-H){4 zb531nb#F#P-0o7PYxuzoWBz)uz?WW|nEQbOrD0%by?oq#Fp+(z5{u27ha%Mu`+DOk zq~6}Ep~s0`p8cS}x|_D~pyr@>j&K4WPu|N&+f>x%qVQo@G~=>Z_;{5VIpHlX`qvic z4H`-No&xVfJM&KZZBgOVrO29UZ#BXHJ;ItjO?b(}-MI3~mWs3Y>%300;$#OT9e~td zptxm-O|kZpMe7QTo?q$+&v$HQNA4N`X({=m{5U+(ZNl##){}&-%tLNdd?b_kQTC$q zG+{P@%df|p;`)hnPk==&<&?2QAf@Ye*jrwM>#zNShev9m)r_8~|9S)aW644F=Hnk7 zU`>+);?_$eM!1cHIil5@ElSr_{5aMb%!0f=poLy@{8~_#Dg35up_(gWXD&_FK6*J` z-rjPWkv@Z5lQ7K?y%0tkiu$LM(7L73*#**jNx|9)4e21LUSEfFYZ83%Fq3N%!%=CD zH^bfWVbCSDKAh2TI%&wKf_hv|e_sWYi~2zKcO`=A4JYN;3I9G<@W-z|ILS8$ch%SD z#MNj?pWT?Yps`k)b5T>}@BvsZi_zNBP}b;Lhh6w+#!n`_Mbe`RTSeLDUVL_1Lwu34 zhV~^&@t^Fk;66Q1E%gRzjvOGI5`m-bczk?S_T0H0?-^3Yy5HK2g`12Zf5FYXO@)8N%kHC~ z>|RZwH+U6x3n~(6zb;@fWjX44cX?jHlmCph3M)I?>hdk!TVRX6dqLq4X)Dapage03 zX>Y>u+K}ncC*EDvk#@R4dr36|*?*d0(er)c+JGf&+Uz5YbPPT-UoL|-n*zO#*yL;~ z3GbEmh4Q~gAl9t6*2Uxj)b9BbPan~6>K93zA`}-)XlTyq-|$PPp+Fkh+tI!!{P(B{ zCryLx?qrCA1!tk+Y{;Xm^?3lKOQHCiF3&i)SoJt4{kHOPxv<`~L&R3U1FANxYQ_og zRyVxpTr?3I8$fLJpTdJ$GW9-}4y1BRwKnqs3%NIZ+A9mK=#p>XOz zS6E%+9jLzg{k^)pZMIW9S-64|9twXi3#fZ_J188Wa|Jxrgip@b1?9bZt@Q({6NO7i znhpq4#U+1~$j8C~oHy zHGRn!CNW3*3j!|qgQ{urAEZP3LXJ+7=HBsp=r`~d#yh0q=d4ah+E9?DV~y?x!=9ee zP;K&+yko=mAmIjiSUo6A3&16ta^Un4Ey9)43a|0+t{X6*EC@+|0^x(8en9FWAa9PP z`b?7ctHAR`bIrM1+lkt}4-&LGLGu~$6U-a*P@9~!fjJvYQMyE=FE&NOX_3ERpqTo7 ztioLQy~>Gn?ls}p*_11OchISeBMLUL4ztpMc$!x|TA#1_=Alvgzl`ot&}!~+AT9y_ z)6KBiKYjQgj;=eN>Mo2MQA(jj)5u7n63RW#Y0%W(X=`e0@1%?nB{a}7+CwzB=Q&0t zZ7R|pZ@aWLG~eg=qff=Xzk9~>eAoSrMg5njJ&LBFt4@w)p5APuVR^B__k3%6_uVyoze?-%-bBK4y9 z{UefXyY>`SKTvNgt^@LWc%=0uOz+%K=9a7>{Ts>hHHTJ4fqBm*f@)0=KLBYE ztk@B(uo_;}uFWay5NTcqfwT}%T?^9voUl>*-aoD?e;MDGV|j4{AzYJ0Z7+KWk4we_ z^Bt5{D^klWLz&xDWx&R!-v$>~9LHS8*(-lwGUim901C2YAH|K+@CHV}F!x z<(cmXi+S;7KpY{HW|pctRM?4i=ch2jC(0vs0j)c}D?bmKbQ}K9{fSuGXFXDe0ukHe zncB;7UL!~s+cNT1gh7EgGB_V8KcX6&zzzS7QLlraQ|LL{+3h%u1yj%87s?}2{(!`l znii*g1aY0JTQT&(YfrVu?_S;{9tab^zF0`o;N(G%h~eW7YWwS_ROMGz)$8y+Gi`ao z09&9MCLgk!JZNL`vvYB2pBg-=@*O&Nc#SzlQR39XP;D?}jFcULh5KRZ>0RVmv@r8V zf2wD5C~UKybj4bjx6YU=Tp|30B8&YAf3po68!cV-l{>vyjMpDnH!w+!a(_ zvg^|3RJZ$}!8Q#)y3ZU&)_%!oERy`XApglp>i}goY5;T6H zGBuimRJl`Rr&(f9S{P|xUzPiy;ziQfVt8paPWl4>wtX)Gv+fGzwf{9w0pen)-*mPl z>_d}rBP3-fKpH~nCrD1~D2tmVqQX7GJfQqRPJDfoeEJ*0gSSB1n1AbeO^^;n;uJ9| zdLL?4?H@Oz6oRd{JTn}iOpuWWu#$Y+3F?AFKcr~X43i?O4=h;mtuNnZ#H|z zRmvsCh^fif@yoUt*2;LicHQ+xT;W&I#-)WBrGq1}*Di=^eyYEyfpngQBxQ7Cs~ z^}uG5v=mxK<_W?cG3fMQ@^)5|vO4)@z%a^IUgK8BC{T4u_`?aCq{3CIA3>TE4+ZH+ z;&ry5Nn};69XauVyp1`K=7&C0kHc2m#zOs!yg2oe6QkS|$rr-KLPO=hVaMOcsIn|} zvH_e)_=WF3d4kG2$wwjaKcjWSCKEiMq?aAidQ0`0kH-uk4hqAOH7bP5QEyHflTn_+ zDP!foO)gb=Sd~Ndyx18CcVXp<5Z0)3U67X&Q0W-*3_$rP&b)CA+RyUE>uz@hr*D|}P?%=VU^1mbN}I7xa1$aArfcbBxJyBP5(r+iE8 zxA^hs3j9}bmArCI;&Bh5cXM!6_DlIGWnXRBx19Y-{}I3K%(5$Jfl6De+=*~R)3e`v zP#Gxc86@sS@~`l>?HtkujWmSY*y?92E}!MZ{m*{I;gfc-dIrZdy4~r_xDK!B8CCkq ztAo-$Tl5ZK+daS7J(on7YDM3p&mi4eeiFBM&47|a!BB6`Qtd~xjd&rjCLfkzEzPub z<;yRvWJ9>BIaf2nGqNl~Zu35mPD5;AP{4i|>os2!dSM$}u)7QOqs&FJr?Z`;N@Q9 zq_*I<5Khxzr_)USY0yA;91sV$D@Wsc;BsQ~2h8=N4!2k}1EsS!24&N8BQnxp(}uo+ zK1Z4k+0Sy~j=&i>3AaK{Ff(}or!`|E?0ay_ zs4To3Wd-+Kn(#TincRARv-tsC zaGZL6G7ng-!AC|1wHEvU{wS2tv#f5c@T!}st z?t;%dGv0sVW^MCK12*A7XKtq-50-hIrS0?&=hn=3Y15(-1yhFIb zyszwjXq#;M`3*ZY@V7Q5uZ^Ty6grJ8#H}Cy;Et66Hrp_UA4?b~4}ZR;`C^%d9>?k2 z>6{Owt(Kx^qfM|eu&&g#AFTEbwob2ys(xwCP`mqPJn8Ml>-{@`3X9Kp0*$pcocG_s^&@Cf7Hb1gz zeh=q%q!W5Oc${p|h)WBI9h zZ@RXlCieT~AgJGPW8X-8oL)-L2kM7c-TLxw-CV@ZY#VuK@3^z9)y;!yWdXNIyhdZ*qRqQt%45Eq!kKe>F}1Hj+4ey5XMb1uaHk{<$BE|%^lx+YuVs3`u;CI1gXdP zQh$NHvX6pQeiNql%21EX>)xHfAd#MB)m;yoj$FoRta!L@m^fqP0==!a!~DJ*I}Qs> zKw1N;`Pwqx|1>&lqvgn`w)|P-Wb8k&E?-;u5`WWq!h~_sqf-zT`1FC{ee20r4$;tS z{A#eex=entwBxG&PX|8|iNQ}XGIbqHIM`8)@a-qdy&l3xGZS8Ca9^2m@0XyxfEGpV z84-dy2%^xJZ6;1mlvZtTs z!_Lji;Pk_dSiWcq>x$o&S=6w)x zYU~5guU0`YJHQs}488&*zvM8t04KD6P$>S?T+7nH6WU&|g+@M;aB;#5aq!b}&8Y|U z%ChU&p051;02u5cp_vcYzG3bu90#18{2*z-U`D9oA5JMcx7?umpG)T=d-SM0B z|6`&!sV=8}kzI+fyLE}4!*9u-jdv}=?KDXe*0vm%_SR~RKvioz!K>dl0<-YW> zxI*7ZI!}10`bAcA^Jk0lGG&}@KT)gODbntJ;CP}|JkKrEt~fA~SA?4Im~?~W%8bxgdMZc$!P`aXyL|x{Mv|rp5Dgp4|l0~gTuit^3|vYICx$)IKP^n z!X|t`c zP2Ygud@~)ldTe0L8pSc{Jv44UUFid1652*#raG?d9{d3bU&IRiaHJaHJbx>W%C85V zMjNr`cEjbrXD3uW0O7vEJfu3)Q2jH)7*O~b^TmUn2ii~)j=~J~9M`=w;s&E*G=sP` zCp_UzKhFTKVk4e5w+sC4Jq^QZjAFzIg!6XV-4m{{^S0EZ`Au+|bCHO38wzt%!!$N6 zL$wAiBOsyKGPdMeZ+ZKC310M-pyoqb1AQHA&^#a=+OgR{TmUTxl?(3&-Yk-Gbw9Hf z+D{LwQx84Y>^JWPCY9gme4@HCtN9@Q#HUbXHNOtsmbin$t_E{UAn@&V)aknm6?ZMs z9i;tgr7P)YU{q{|R~$A&?Ygx%@g`(vc?pBW!LTJM0iX71LFcv5v%}g4up0W?!N>Wg zW?w*(AkM|B*SL0QpAK5v`!ku(yBMH45WOqsibYX}H6bO7xmvfx=A}5c$g!#~s8_-J zAn;hX5^dr0ZP|_HZNSf28nLftG!pgzaS6t>|52sicAXystIc-dob>0K0`qfV?>!vW zY4v1}?h&9cPIrwSg!s8Yo498Nrg$YPqITFMtUfbt~1)k+Oc=I;W(FD_WwY4pGPop zX>BB&5Y$haHG{$PT-jP24}>$hxU9$yA2*yVUrt@XsTUaafFQ2ZjMJ};gayziP9kAk6<7J4 z{tSdOGTp5hNt-g&*HLt?Vc|$WB);Z;r|09UB}cSMGyF2zn@wDUN2mT0DHq}dX&W4z zT1%!}sL%Jr9f#XaPqBOOT)26#r98DHjWBj9oVjH$ZwV&? z-Y=m?)(zGx)PYaa8p=Bpdcf@=8*pvqG^Bdh5H_IVJk5*sI3=_N=zY1QT|MYA`{B2f zeSVt{gsB{K{!o3!fbZ3L%qFovTWEhqkapvNhfF2;091N7y!UrOJ+39Narn?|iA9beTBnuXQknO-mO z7F{#2k2l~snpH4r+Ed6(+9H$b`~%WW_`}6qj`8ozWZ#AdnV-${Lq2#P6tPsXr4p z*8OAM%pBm^^EmP}8J>Gr{v=MS#fRqn6W`;~NQcy6Q$kaq_M>%Zw4){uE(GkG7%y$aZw*d|Kk0*HM9My;-MRe^L#cE=IM~a@`6f*M z6y%*2VsyrKjcM#_;(>RNGUu6=dQ&sPm!5wa(jJtiX!5+R@+Xk{%N#sEzaT9#o0A^o z>G@%T#>J|+y<+5vG(~!Qgz7!lxdUYl4=aAIOpgsSMH03la5q9##L>(ubK)p z4{$m^3rH_<`W$ik^i-waVb_rQa(r=!NGy&0ZJ_@&`zI5YnfWWlwz<#i(wD(y)XZ+D_4?e9+SXv>shsZgxn3Pr)NV`I-6` z_CuGXy~1IMC-yHL#mVF0jdg*VOWRk&^j|IHo}ZnOcoi}z@1s3}b~UETthc>^)>e`Z z7le^jn!1i<0iu-=|4}j(6P7s2U&p?Jk?9adb*k!#k^Wbjk_S6qhGPy}L}2qfu&SHA z==SChD&IR~btg`>p?U=~r<;iRfejUg@EZD!VB(->kfCR%G#>HCAXK`MJRMsTau&>f z4nfklnz#2BAn7X1-7^haTjs;^>E`@(MJWBfRparjSSYO7nVpO`9vs8DJ|p0|cMcLy zpiZN0YW%!!$zSGpC<&-nQ1#o)HetBcWD*ibO4>Ul9?v2?ReMFX8pyH+jRaS?s8L>a zc+`b@X_XBy*QX3qYusX_)xcVJZ1#7}SJwWP77ecX0(YB`tvr&# zS>iPwI>#T#%drL*gK?SO5!Tn`7T9Y!q#k5PeDh&mWCdxck)&D2D=iLgJPnkl?ld|a6h7Q+ z_9^>zXsq~ndYm}?c{r%OCvC4h6p$9@%k=2{-KUR*!ndJ0fodxy!vhqD*^k@S~o(nc#eJ#e_e@gp2 z5J*>G{djAn9tOL`50p37&N9-1r`;UTr)z5JluM->#i)F`KF#YPwhAc{yWfR~q^ut7 zcgj}!-scr7vvgFw1L`$|%bfBML3s#LCLxqop9T41(Wo@^(l>(xt23(Bo3t1^WcLsv z%W9(HGJ3suQq!=?_YmHamuN>Ay$;FyfU3)>Lz2Kf)dY!4fxG~czE-+KOSlDvk^Vqj zr&Yd5>8F{c!#U}8M!m<9mKtE;$S2I-FC&Y*B5_w^(RI3|T;Y9*UEFYvMdd7E)stKp z&wDcrlhxIoZ2m3Nkel`8)L>R&X;_P~N?$Y?EO*8`6QX&F@er}}E_^d?AITSt{q zoI2=4diEr4KSklMD5@co}{7q@oQb>=26x#V`3e+xx^Y3E~xBj@2!JW zU*=Mismn?GAbCOf?|8ec#NbJEK2i!J55WIDbCZpaUlku7w3b_^$Fe^)_29D3bVhY6 z$)`wzmNnJbl)pv75+q+Plt=yDZ~_b)IS=yZA}a4k>%{8%z^XTkkk%R98m?6S1SWMiU3JM(c&&!zRk}vA6DYzIXE} zKx+W%eRFCym+_SmdZeR#}kxP<26Z#TsqZ!daU^$N z^@4iZj4OX*>-AH+Ah(S2qk2gBhuUvWSr6Nrl0qJB3e{aTjCW6Gq*tNe1uM`a>Wz&K z)_Mju7o>Yc)>2(=G-F*APmxzak50OrG>xEsfY%APNYk0A+=naQb@0)#S!nT5RH;b*PtFjHz$b1Z1a7R(L`4bXfYg*N<&LaY>wEN3KVQ0A`kIkJxxAg~J2vR)M`#r?fHJ}mwH9nnSshTB7xg8>^S247zO&Zl z+3=rv7q0RJ%1hX>Q9I!Nocrh#>nA9$Qn@CkN9#~FY>%$Lt8;u8#9k*1XS((`h{KWv zeTJaCxT@!f-&ux#rs)3Al9Bdhl!0njnx0jD4eFG0@bVr^J}C*+I=6h70+bV~9FmcD z*Ss1PPaJa#rk;OBdF~&=&sXFTt&lzg>`jZre)9`3HpvEP?vgSCm5Tsr8`4wPKxq~7 z1%fygmG3Dz{U78_)RPuvja6n1lsh3$n8Oubw6^TZUifus= zM#ezbuLg2L^VV{^tYGE;P;_<8V?9TQflsI%^j`lPDlBsG-@fW{)7f#jcvOU(yeI^k zfB2~#Fk+#+ot%K*-8^7-;Q(&nG=iVIugi79o4_Tnb$GB9J%f3UrL-Nk62Cf3<>8SZ zG4IS1j9$Nj-x_Hut#eJmsa-ej_pcMiCO2WvvmE&4i-kB~U0eFTuMMY1tV2Y_3%pmJ z3Im7jfcM6=<*SnBoW@K)vqh%_Eq2i+gYo3Pmxn|P58cloaco7ts0;23&a;~yYoQ~ z5nQ-j#Us0IWa`OGT$UOK_cjmX{XFe~_J-4#rRiWtc=LG{gxf5aJGK9?hNlB8-g*wJ zPt$6CM{a`D&$FTY%vzX|Kb8Nw;3IACRhQ1Q_F>(mOsMVQ%Gbn<0*jf4w0);}!on{z zAiL9M%y8>~rayeSTIWmAlli@2O=P9`iSK(CirDwRk=9SXyM92F<%dF@dGYkRsW_*> z723mJ_{*lTG~umFxLE&VZ$CDWyRCPjpKmm`x7!SdbEENeMRj?`F-TawYayRp@|IrJoMrFzE8)Do zF%<2uChv@xk0%GZurb>uq*o4+i(UGI-5VEKajzQAoOl@>qHD;3j(*TSH6CYT0=xO& zMLg#45@@~Q;K;ccx3GqM(Y64Jp6B3){K-6gWp7BdjpMZDRD(JAeB5o=wV@mnyw`}L ziYe;2F#hvdiO?=6WYt=jazO;oNAsylDpZ`Orj5|7R6J?l^RffKS7> zg8uqdvUOxVxyfmmtYhAT7mpnSz0+d&+jDn>V^<^CFxrscZJfpIHa*HNJ^KJgB)Rg% zpX$lF2CY1=x9QF5cZtFv*^tw^$+*VZnElL)t2G(y(T*E$(pA06@<-Iiytr$E`V_xz zx-WtQbKqR=EL>39l;3h1DmE7%#pP!v@{hp>VblRV?%QPp9}y$@e%9T@gUN8+0(LEoIEl> zS_U5xJ!c%n@XjXE?c-vk9sr&2M5wf#$Egn3z+e5u$3E9?vu-Wy)!c2;Yk z7&n@)`8iN)8SE^}$ycoy(SR?jT?=VWaw%)hAI_`GP3+&HqgW-TPRYY#Z=B?{mA&|p z4oXa7T2vvSqKszu~Zr6Zx`C0jJuXp?Vp^KP*~@8#aZe3aVdr?tYMFeBT^F_yp5)=3!pz+iDN^8RDW%PNm?xs)t93?(!#7{?=&<1#$+vizucJzdj$)^LZp2{`ng!5 z!}!>e-B22ug41(8iC#BTnI`u=Qq7`SlrayTa06+tnBm{EQWG;8jTT1G{dl(UIk(MZ z=fGUtSr*3#hv12AJ^0w_9!vaDrm$U%j%X+e8|6`>M$qj!-5V(C74YiGGGVX-66UGb zVr`EMjr)7L-ppH<_tfpkdw=>&b1#(#QnvD`u|wGXt64DH&<<36Q}0SqFkC(@tH+yX zbiocq?U|mgr=&H+Nxd(^MO=b3R=9S{T+*BcahyDUxr;nB+(HsJ0Bo$L_F9b_3Bz!> zZM-iP#;j|vIo6Rp6YG00OOf-Ky9#|#8MtGjs~Am(~;^N_iia9ej5fo!V}<$Uq3lD_9k&jsYux92lFoU1-Hd|V2@2; z;j^>omI-tXMUEPa=!BKx_{2+^tJPZLT-ybBTE7ipNeRxn+!gj~tZ~?nk-XNhooe54 zMe94*<@S_Wtg07Y=ZrYkFkTnt*LM8Ny2=s(hoU4 z=U~^@|Jb09&>&5r_MzKjL7I6OO09c*g z0}o_$5~{`xKQsl>3);5pcj8RD7%?a(7Rri`Xx(aW06D!akWP@}O@gsnA8#aFVB`2! zQBo(AF8GNCs(l!`hN0^9u3tm2B4@6={<#j2MxlPbhrVSE#DNb>LG?r0=x6Xdp#v&y zvNEq0AGP5fbi!r)(d%O(>DwA49>x2U&)^B$&)W1MDZ<|R~%j@Ye#2k?rR4r*&S&F1{P^@p7mAhDn(SGyC zKZf(8$9m)BCa2J3<0MS<>kp$NwulM>NhN8yZ{?>nW___v9@;fG-h_+^s-+A)un9>9b7&%q$n z9@}j#&@^$riA!%d$QDLTptIdamb%xN&$Khc9R~X~<4?M=E3TcGcFY9T4|2e|bZy0a zx@Vc)W+Z-9^8tmGG)~^7#dbKdJDE{U!_Su zFE%161DBj#!&;9o6PEeYvh0S~$vt0cVOTXMsr1~y&1G0eE=HxV9_+H?g#VIo8HvAv zxF?%9hN-bOY}*U}=(|Amd?R7GD3sNkVlh!*qBknkVUH;qN&T-fRm?&)Y-0i-n~tpfzuB(Y^-fwZ=dSjnf4 z*xF_XZXddu_$nK=3~P!sUr}>iJ4QMdJVtEc>A0Qr@FCdvX&!6%C5qN)Ee77#gJzl( ztn6wiN!uf7VC{JsuOZyR4cod%rHg{^)JD<>@L*ROoQ(3v($E&1I01i+FTwqqJrKU~ zBL4g0uW*~@xLb6#3y?$2y94nE8Z=8+{RaWrX<+N{8n)jxlGI}w)!!#pca;TeT|xEL zl?}Cd8&h4%OVVNPz+ zYAm!_rA_ePsiRcq`*C4EEhCMruns@94CW7q{$U?uucPuL)c3N^-AK52htBQH-H7XF z+i>C)&z6RJap2|=U~u-axOvkChrK?|X#JS`nMsb~^9gUxF9X_ptGiza#DCqb2xx*nOJuJSvor0Eg zlNA5)L_yc2j`)JwFR$5vg+{#9x5wDjccaqgK)k9vgJx37HdyXE3(gJyhM#hyfP9CW zs-4b(o%sWgW2`||D_UzS*fQChxak(s>qO&oV_*fH+eZdeN82CN@Um;;J=SmdRj9 zjh!0hS2k?RgZ}UTK#Tb?#P=h3-~9D#(W!Jn8Vxo!J1U6tvi@5C7CT!kmJ6)^(*1z) zpi%g;Dy>F5CF{T7q)jVu$@F&GZk;!g|9OThi(DCLSD<|t)O%8`NBDjNd~?zjTH7?^ z?=)6?OHN(h3lgDLP8{KVV^KQlC1H0YRyMdqb05gb6X3VX8%RBjpN5+&yy8^X%*2~` zcj6V6YNF58jn+B^SYTkw$uQ=NF;9xBDVwKkXS5gcb@*aIT#NmalQiQ$&!KZvtT>bA zpu8gKN2zo)`5i&p3yHT#$4pi0Eog0_@kw_<{NPD7$wD(;B4&{aI-WY^x_5Tjmfy%+LA?vZ^zA^c2!!IbQ9kCupQ|0k+dw37GkRx zYyhe=Y<;00Ub34b-Z~eP9`Qx;3fdOOE^8gThU1k)H#CgzC&cJx*z0_q^zLq`6wBGvDP2@gDlLuVHMM;{Btppd zd(h}2jZ;N9?ETmf zuh~?J@6jQ6eH(qJ)iqSm7%|~>Tk^G?Q1R+N=zR4k+gt5*WKoYiepHR3G%zRZI?YLucCL;4hY-WlIh1GXrn&c zVbGG3|6pp2)-9{k9yO}!d-5!rDer8A%2;M>8V5tYgS6x)8Tn~&sxTtlYbq#5VaF`G zfcR9G)-g$tZee3*wt|ZJbriOOT5HO5w4bJQ(yCf9^&2QE8~Av7jI^7G@e0vVke$xF|ih;DdXZ+?{5a!yMwO!ww#pH(}eGdGk zdsHh9DM>SdTO;XQ!8+BIh6RP2pK@z+HgpeMY*+}|ex52DVaj*=b}?quGt4mlD%v_s zM>eS!+*uf=Rk~$fMGbi{e~JjqrF$v0Xaw{(5TEf?9UTSbMapxaN1icdX9m#J-viuy zdWf=Vr&-<^KS@2rDc8ZG$Pa`w8OoExnU4Fldwpwjr89_2C&V%SX0@A69Q^)AJsjLMEOOy}7>`rw#ww!W_ z417MI9q+L%Ul=uJivO{Ct2JV%{VY_PgYq}gp(v9n&G+-$1C8?H)Wf*&nYY;Rd85K1 zP35;#rCx^W9m?x|;<`b&27)vxqz`Gql_n#f4bgq* z+%xlYjP#FqGy18*JxM*x=ZN1L(*H=>TKNWXe$siM{Q&Z|3iB1tYQGp!e$wip_;ukD zZ2FJ@<4vk@J>6`|YG07X@WW$49+X?Y6r>%QsvXrMp6P{n_}WQE`H6PQyJXh%UoQS` zmyfdz(->vMuq?H{xb0~RpC9)cs|sb?j&EgGQo509%v-~y&FPwdCbDWhqMC*9iUJOg|8$aee_E;W zLv*hNT&LPfc>VPO#3xq6y+dQ!nw=%kRrZoIk4!YgfH~Gc>5237Nrl<@@b)AB8k82(nlRhs*2o>$7*GPQ-nnx8wi@;yq@X@+maI@M6FdcAMQ+JQCOwnsE3j4;e7CUNbX4eXXX$!)@etM*;C0YAV z9~|yAg12r`3;uLiDK^YGq2?%7wCOHgGi%C8`@&?sCUm~u#q_w7bhs)+uT8lNd?r8YdNqQDILi4wgxO?_QtbgSvb{^}<@3@$<)Z_Y?o%#`n z)oRHT+Xu^#TW);m>oie6%$Pq~XUWxEnZGAnw>Vx5Z8Qy6oJ_-hLDR(Y-UV=FX*zEk zk$@Gy%h_DIm(SuJ4S9S6TX~jSj}%~m>cARIm6_CWpqx^ z2P^(PMnag!P+l-KMm%%31fSI%d3<6p{&nQg`Dr*`Za9p~S7x%db~GGqunv1~Si-Kw zKLSm=n)3X{Aoj4NKV+2Vqeailpf|CZr1fX+&{!VgUTCtpsf;^t*0ag>)_iq!Lq6@D zJwG+Dt~9h6BU5s{_~_#=VE_Jk{0-f=Ie12icyZf>*1QaM{nM9vX;FCG!j=2h8KG)P zjCNel_uTQ4<=>)NWQP#Q{CgZ7SXUw~>yzBS#bDe<(D5$oOL z$gwr>&*vyOmbDquCf3?j}$sUZ>j}7s+fR24?;(&u?82|V)P%TP>d9`^|GXrU1+7zgcB-I)3U2Qw| ztKJJHPd4Xtf`{1sscz4{|sXmV~>n7i9o6SONKV#oUMdOax6n2&F*QNUnWE;PO(BSI~k^exKCocUg z-f!P2rbJZ-!W{Xf&rUY`)-ZP9K{8%&H52CBeuBSm7#jUdl2_cj$S3y4n8Li+`Eh*2 z^QodubB#1_eh*LUJ^`v})^N75q&-8Kwg+t9Yanvv0`{?zD}1XOXRbk4h|Y_X2PWxL z&A%2c?A}0F`T|z>*kx7!R9kdDWrsqs&Gn|{Ucp1~IOPce$-{8tyh-rfah#-{gI{s8 z*qUi+@cUx3Q&cD1+^-LBJM1-1slJaNT9bh`hc1bG1>;~w_Fhqv+1b<7m`=%ix`WlR z2VoJx(6;|5RloAmfIPHbJBn(L?)|a<6Hsk)sxQ2?y)z8)--hnzekk0;XG={8BPXG* zr!((Wm(H&u?5B0X&yP4){kU#%1FkT%qJ|5nV_{r*d+MJ$@<4(s46$p;d-{dLwrSPm zB?~io=;t7azBWXDek4WY@l;Tsq1MmdK2dhn$;MgaORl)*@rcW*EGxYop8J`ObN+RQ zH3u719jn8X2!1VoJU+dcBdCAm*em9+$?Aj{a;OFvopzG8F+Q|MZCGi*09mqf6AbHb z!8;utDF{~~G{is_r*D!yR)q7G#!a~0l1o6C%*%@Bfc3&eDF0SRDjd$){#YC|tf%@3 zU%zaQ%bB6#29}m~24?);f?I;Bq5gf4zv9Y~@IgC!;8*r8zy%^M+wzIb0q>u-1jQw^ zxA3y&9n{XMfnFebxE()BzfWsl#!x z!b#p?N=@nU&R=_B$v>pNg}_D6Q0>8y7G9uuI(1Gx_M!MH1|*k4+yQUJmD%kSj_UoSORg}N3|~5T(+-Qmb|dK< zFx(G3rv8E~!aKCOUQ1q_-bFTS)fDL8EcvoN-WomtuH7zTOPm@?ngdX;vg+5JIE|4{ z*k%rdyS$IH5#AVLCF*xe@|;)a6skULc4`zXGJB4`gL;a&M;n5f&M%<4Lx;wrFuT zh#Tg((Y1GLs4wpV&5s?K>I%ERcws=9q1rFZg8mh;j*gr- z6;>vu!M9O$2n$DZDA`W?x|EaFVZQag{BvSWEY0Dc>w?OSG_lrhznGcNkh;Jcy2=8n|mygnwUL5yfBD zz_e*Sz#>9a)dOima2l+U4OaA#wC=LW|scmR%6ot+?jkV4z zTJeI%ZSlsv)$;fg`tHg|2QTiPtGE!9UX0GG4{H8%KN-q#`aOC3?Gf0t-esh*;&aD` zTID@g7a}y@`}fmQ zAM>)vHCQe1ibip^!X5*cW=h||`BBB9@O)db^1x)-yzNgm#BM*ko?65rE9!INbV0{q z*{K0Ydm@w`xlYgTAw5RAwjIno@?5k2d8#m|d<*j~)4iA46tI&H$*^c^0q*;51EhIX z9RSq`FY(+h$WuVO)xKhvM|&<-9*0#UPcp(aO~u-)NSLXu*&+%yYA8?9YMI&6BrKR} zCKj1ZfrGvExzEX$_^^b|>uA2cYF!A6#excVLH$F%$X$|Vf>P&4xOI`YFgB{qLt?uz z*MJ1g(FXdO4QWrsXPcw=&xGUZ>Y;Gm?-A}9pTbBJajGk5V!45QTo8sW>jqcEZs;-h z0^Brh%j-39;^eQeb(2U|lDkIn8&}$rxEKgKmF~)3R=1(jV*K!dWc=oTRn-;dKl-O3 zoz8Dpj)RZYSK;3YbpAhn1KI~cx?4KyMncQDWuo2cJJ@G(mgwGNr1)kxmY#w2SW7*J z>yzD7Zwk_dEXbll+?kS#e~vX1^!cFYTrO63a@VN-P&}u!)bCJdNT2RU_kb1$%`yleTt@v+uf4-~Rd~o^N8y7aS=MmkzVfu%BW-u=bUtXClNw=ul^dzsU`3OhJ z*S3=6U)jQDdr3DGpeEa&Fnk@7))cDNZ`@e}vGeQ8R_>9E^qk^vPCAXhNj(Z(bv${! z+XBc(DE|q+b^1Zl$5=@5r0Z*!ttFjr#yW0Viax8eu>JO$QhW6=G{|A1{*~oG^@uAE zJOiptObBj|HzSIwFh2f9d!;==dG4`8zX`&9HplCf^4R35_#yijQsrL$|9Us3+A1O1LYig}2n#l-#-u1xjZxu0z@X-1}o z{C2j&$Ybq#L5^$@3+1k^HfScu_+-Lg5TonxjU+Ob~i?rSf7^v%YGk_dBn=33-jW zTGDTLXMY>1)|ogE`s~{;j&8I;M{gh1BjhW8v$TnWC1D{F_Th>j@t(Ktx2fv?ukD)S z_}Ax|$hC%j2LFLt7dkc)Z;?MJ0Huv5KlD)AgzEVKD!oypo2PU=x;QO^FE>|`J{yFz zW*X&z7U#Lcz=l~Yv8y+b_wuAXfsXMI&s@(Fjo*yIvNOjNW@!|kD<45ToPF>5Eb zr0Hqx-YX7(H9Ln^VJ+b}8rs~43EQ^9xUXS^)z!FXjRdyobv32?F#l~Apc)XTYTID) z`dFpoku(kDL|WrNx~Br+n5V+5Md6{4+0}--{#XisU-VU4S@|!~WVm2TFF&8vAJorc z`BT`uAr5z8J^sM|y&w)l(%_2kNWWZX=Qrw#+jK1k=?PeuZNTqEAA_VLEmS{i4{eVoJL-|gB!I*utF*6C9(*n7E0gg&wJA638mL4V}PMC>-qZR6EOS2 z7Rq{lBjFfk7=L0@&yioJeF?2S8595B5TqS(Zp%~DJ2zpawV~2#xVXm_PZ%9os2s5Av%+jTO}`yM22Y%!&^tx<*fI=NBVD#Pd+x?Oo&Mp}+h-645$ruu_; zKtr|2&+Vcz7SMyb>hfcE4r+8{J~m>qv8h|Hh81 z^8TONzJs4*Ia0h;p*IvZem`54Jw*5VqG>TIQF&+Ov2kUh9pB}VDabo(NT;!pW7h*|F75e} z51=yA=bv5Fc*Q{9{hGmx%kc8W@hl-YQ$xJQNPF`I9rg;!8>HraBTgQZa<$j!xz-bB z1TTWN5shI-qiQl_Ogy8pG0OdLVM-_?oh3KD4#sJ1&4GLaQjUTrpB-mCzP99U#o08b zG|EvQpz@IlkE$6s#;w5b7=L@V^yt~ACXOnvg7Hor_~xQuL3uDVJClgy9hC+^>(h~-dV%-`W)EBhimz%E zyj9!@8WwlSArv*8I`*FFteIbNb((qr$OI{|44LApds<3qwN zWdDr>hsJlQ?t3Y|gomTraPm}~_>_3O8dNTk@JwfpAY8|xy9$AFRruO&Cs5rp+Y6QC zt%sq?s8tSt0Uz-;3}BSq!n*c$xLAL$(#NPg&wxkkku)4}&JBfqocdAq1-Lr50;(HO zHAY%f<;YdJ?LJK`8`;?6w930EUtpwX*qCO8gn3EWe99qhMS)a(gHPPv6VDyPljE)v z*0e;*7Maqzq@9tlNcfzbhW|b7#Hn|%`;{yt{RX7*m7b?;F@`;8*O_|DkutIIjIwAU zExY3W-!0@Jv!*;P`6PsmY%ja}>v5;4H`%4<5s=#Z1sn-~2RiW$;Ar|9m^7*c@6_CY z`F>8a{p-4XV)08B(O`w#b>Kfi?}LwJoEwJ4F01(1z28Mp z>MfBRCE3o&FBQ6D3rS+Cz)6-KR!DxJT#B0TdHOXatrxsq9E9*X0s7UsgC+f1u-VO%(ZQxW zpMCuoifltppN*roPU2_vyYqx`Hj?JVf^HaK6VDWEe66YcePbfnJ+Z^+qv?Ev?Q$H_ zWDnFUU4YH#Jl}n;hO*k8QGGn?+$?!u@4d5Eg^j^$3}tuJ}-HpBm5t@qK8e|QV>6MC$Zo;UyZ zBM;}VKFtgU)aACZg?Lu~EYlgDkHh}kNFFB!cN%01$_qL7>tSI&uO9zoc2Rh4rac+y zeT5_bV!&3ap%0}h_qpx2E%0whH9_M>m%KKd<|#K_-X%H=Xs^a4W?a>>l{H8372W82 zo#G3bCJzLJ%}EiXC!SY%!FEL+0jga=bC>BqyNVv;{H49mHbHfRJ==_u^d2_5&POPY zxa|C3O*QTk)(v%PO#{;pC*V@qcgSgcTD!oYx%SYRqwqmE@pbhjtB?1PumKh)=`+fI zHo|W{hOF;|Wzrv)F5Q3{T)?jvC#&^w9o^H`T<i3vswF6Xbtk|?)jLCx}AoQ^=V*Y z@E`WA_yr4QRj}QOx^n86k(#9Pnv!Y+OR61XIYYjPc3JToXZL&3GMD?haG4Ai1MX+~S##MgSDYDV6T zIjE&LFonmlOY|Wwe6Rf5x>VF%(u3kLm~UK5_a;3(O#Isb)tYR5Ru^(ZYV+^q-TBeJ zOL6@A2GX_gq@;X7i}{`KQt}!>^N_UO(!sYYuY0H*SI0Mk*b+M&99bJY`?r<< z*d;l{cq;GqHwh`yVr_#yp2Va;&1x~(qE zDZGt?9<9?57R%@P?}_XBV*k|^TFOnl+mfF7&~9h-U30{!oJvN#A&C#+wr3sw*dvg4 zJ;-qCoZ9?lb|P;3>PBZ`b&{>3>!VdU^4Zgy;;e#nF>32`vG>V8!nFg=;U{|Hj1jHD z-2A2X(^!V1ERNxny=~~;n5isu!%+Ef_9e<+57d5c1(fT2Vy?ZyY)$Nvb=ag%H$LB` zr#xKQ8%Hj`DpYLgho0SUh|il%hQ}T?r0vWEX#V{eHeX{!9$!bk^QV1u#*Gq1tL8#0 zk3C2+1l>=~==B(An{rhc=JcU8alp~-^dYeQdHhw!Nlw}m4*JD`+r8;oz6tBZymPg9udCMlzyIvu zR%kn|nb~<*;eHs$9B>g@M{8MZby8zqbP%gQF@|QT&Oq~mrY$zhjFbq>w`tBYkE9CH zGeYqgaV})hc|vUCNLXoJ3kjRhxqgtO`O@$1X$BbR!O<7i+LQbmow>IYX?>vbdq;eb zYRwf-hyC0EHHMvq5h=y0Ca`;%Ok=_ZnIReG3|L${Qry zl?`8qvdvdAfbtKH_Wg+qQWvs(n*=DDNc*5vu2lG{A^yVacR%9u{F!3X%Rol@i%~tp zhO|IVngr9*&x(IPJgRa+U2j?&e<-%<3Le`$l(xazi>nL8p~S^_tCy3yj?xQ)xK{kW zQFtcor#ZB^5Fsr7?15_z)n#a|E9Z0T@aWzDLCqa^*#zw^!lhqGwTX$RYf~*n;IJ;+ zgu>Ug?`|N@_yvTSS`~w}6`??5&`fbH&uGxFMU|!_U4>MC{Pu1=S+Db3_+i>e8f;mo z*$A6qQ($v`ee*JS(E1jmIOC>adYp8E3vmSZc+yYtC-yNtp>aEAE|0%N7SOH*Z}oTs zwhYi>zun!X!ePZ_X#s)Qnac@<(fMl=ij#;US+GrwCQ&*T+dn@ruNabFU%kVkFJQL%gH$Z`xzhdPR`_^CWv_ za*g_mR{UJYE#i8c{jg$rF*D!vowZ8;fhxYFxnSTLXQt|bcs!G8LKaOlmP49Y3y-0X zvE$13TKyHrnB~;~rQeYBG>MpUAOiK4(!SJ9s&S?1&jhc= zA(v*W^@A&J7ZtW5{YV-IUjFZ;&^f$GVF+wr)fE*U5Vv7!8l7`Bsy;i!U1SHBN#H9^%XbYTmcP!X3NiV%~Br^ zRGjo+R4F6<1of*A#1G;O%($wdb6*+?!WprpRtMR0{t%(J+>|d(z9{6@jZEbrX<_2c zjqu{gVA4Hrn7qH1kxqe`(*of4x*1?`DoN9x?pdf6`PzBZgafJ`s{F#Xua~hEyke;= zybAX!Tnr>>YBtm-T|+s}NH4Ih!7s#7?=*bZv8@d8UkV-XS;CF^W~7nxf$*5IhB*-Z z#STVhWnpsYRzZ3a2~)`rr3%t>q)VcJbhXA{`ZqXVat3!B76M%pdbW846kksG##ht+ zhV5 z)#=&HkNfKJ<>6~l@y@>?0YKPNrKNr59##6RO4qs0uqB-uk5=uL0qJABo}5jg8Zj#MYT(vfp|;&iRrDa*DszhT89(AXR7Px7_JmbE4eubv+!T1!Q3I%<;9AQ z++d47aQofj`sUxzq|h7*>mXvp1ynxu>62QrZAyRWzj~KgGa(1YWp72sa5#Ken11NOx7r%WvWBft zd2Hn?Nb}33Po}8<7lZ|9@MRKf^Q@RX+wRC|&0yif5O|QfM-c8}Y|;U3Q18Z!I2@d` zT`;y}H`{1GQj|8(=he1t!|QQf$gj4-Eq@k6ZqG)Nv__RD+W0yjNrRz^8R=PA9$p<% ze<(kd3XMmGV`j4pDrfQg=_{F2(}Q6goxR?y)eYGAdOZ2I9zfbz{hKiCKR7XWnw&f1 zte`PuUfLLowB}SmE68KgZeaa-F!FDzj$ro?Pd@F6l_Z`ZT$_os?o7R(@SN_&DAqnU zG7*IDic|5%DHCjM>KxuB_8=eHk@(ZN zikmJhIe_oAjo{wQ^_VbjKk1<)g~uB5Mq2XeQt54VeX0ZadMX)?jT*{GKY-11MaXwPTTni|D9RNtC2R~IwgPw_xG;NoL{23|Y=r}PUu z*U?l%8Uenac2{17eRgPqwGK8BQ#O7Eqx-RfJOU7Zg3<((oA@K74SWBuHd0)*t$xnd zOb@=m$V01GllSZk?~@9U^iq{~CvOSsUp8e*6TkS}r%Ia?-QSDSc`Ws)`~=NG)fnt6 z8^taDSW_=a3*?=J@)@1)yMXG8ke&kXEjij*1)Xt%b|~sC--6_oA=+u5*c~-c>2glI zz>mhZ2J#7tm-(oiRZz653|5bCM*V@aBEa%J)^2>85&q!e%5zwA#|W`x=n^2G3Qqr$ zMbWv}NL(WRUZ`J%OXTUQdOGCIkoZaUL{QZ@@g37mIfCRhfc!a-Cz03N*ziZpUKBsz zc-wO;+QjtK+|GE%EPFKIgy)RgTf)a0vw<*AqqGb4eBjS>XPj7nKT|rC^kY>We;m_T z5|%4YL*>boRwSGjZ9eYDuJ6+nzN`KZ`2EfR@+L})VZldTQ2L#Efz+EQp;|o4ltxxP zlh7B{xaw17QtvFeO39JR1*##p2{0`-J|YFE(fTK1}@cBMJ>V+w}}>aS_$ zI0b6RPr~9U5kg@`YvW7sJWHPsEo(x(1XDpC52o(BhW*c*@{yZ@Q1zgebdEyLnvQ~W z1JzeQ>W!4+mm{eJRM>Xm=X)RNM>?zSpNU+>eSAe)nFaecxAC7LHgg ze$_bx^qJy{TR%y$);RWATh*T+56;rp#-Zwms(u*Ny2e1)O69SjKCHrhrK7NwQ4*ec zc8rmRR%2&RhQ{FoO_(Odt6o*l)Qz6uB>jxa-z{@9#`~?(2xso$w4j5k?zFTfl6W1- zn+k=csyA4+G1K|Mo~fvEf7hCYK)uRL>NkSY;|K2EfklSrknkRk<-S7VP@wv(sza&) zpfy6m6wQj|eyRt+jp&Ylgo(j}l+p#bfoKP#+9w;sich~GW2ZMgxBg3`G%{fmQ+`b4 zT&;&s#LWCCn7`VQlUL`Y%|&y~L7{YBQhBiEz4by=8mrr1BTl?ZIm(f|2^84&uIi6g zYFcx39pX(!+6SiWO;NlhcWElL#B;>&4oLlG?eUkOu%8E5&aCR$_HAy0b(dtRJ~}9k zT5IrJmE(k=zZm%xiqjwJRYWs-FC(mG)GI^vL)^@$r;ZeF(uHND=WkKYO-7}Iy1a|T zZkY{8qg2C;oey#5PHRrS62tbrQT=n}6{>n14mM}OdRL(8iD?=yoTusp?0p()EB3V% zF0WgGackNiy!lFuc<>4Q>AYFv4oSlMTr5PHHGurdA0X&WPc%M*n4dORRxIr<*!CQS2v;z0adjkKquQG@a*9p2hPH?*y;mbf3n(J7_ugAZ8D2kEsowvurDK?rQu_ zILKY-(m0sM>0UsWZ;jFUUPrE8Gg?!~?ENpn&4u&PYP*Td?=^tiwfl;JZ~W0={C576*P2uL0KcZ}XZ5jA;7M4zL#;rnWj|ZnTw&>R%H0tmS_MiF=uEsMUeoltg zV(u8Qdaw&hK9z`D+Ra!ntfx#HnZa`I*57+Wj>qvJocCiI0*d5;%RP+e-4Os^)bmh1<=%t&qep!FE&T_h@Y zIPgV>ej>#JBXhGc&X-GvqG~`f!>sc+wO*Elx4(OF6MbWT{Jw-~r?rA%vKD`+mPU^L^4o$s=Iy6++F0K=R%wV3PJ67cGMM&^xaC|$$YvaG(7J>>dLiY72$UAoyy>@rQte|XWd^|_QNr7IG zS`p-N3bOm0U?D%(^74s;C5<0SvhS;Tu&5=Q(64S?Oz&IECD>^T4?cbjqxAMd#ltf! z(u>Z6{(mk=OS2?&OUMHM{q-O+@v+G1Vj$@AWR}iTk(=(WQR`8eNB2BzHH4t0w^{ke z@v_|GA&YF41)j@S3(6g^JZmQ@?&!NrBcqnok~wWQ$(*RQu)pCvbhgk1OC^!$HQ~e<4hI>|t~bnI3D5!FiU_{(A~@Xt)z|@4N%!eb>?6^C0-U zH;{`A7Vu})zhLPnCvLI&7Tl`0i7#^h3wDDiVtl_?cD8v1ruw|X;2uUY{Xi(~EoUOU zXYS>mXaUAw_jTX^vg9hWo81x?Vp%Md)z7P8A7g#AG~?tG*_FWEms9)G?BL(52t-9wzJYYEjYyxs7{f_Li23QsLhg#MetI+Kb7gB%)8?1%Zj}TBXcD+6p^XtA9sy_Gozty^o zivWw-lbO{IU7i!P5!BjS7F-n-v=@M1PAzo}t*g~Unwu>zsCxo}y;CvRyCYPtsVV*G z9v8oyP2hY%gZ`63kZMe8yvBhWFFlTydl4#48$x7g2H3YdtBqd09-JF=NB>7HsxZeS zsV?3+XDB~BUqaV^fo@~ZK=gx92u?b|3x4Nl{dby6KZ|flb;8}7x0JWeZQyw?9ni!h z3d%Ze#aosJlKu|UjyR$npMiNZd&1d8|3uz$M>=2YI_9tS=I$f(rEAJZ;TCg2q%}xE znnz}-`)Q5)*IF2S#$J})BJFNcg@uwZPTGf{Ho-bTq>bwWAs_XyV6vn1ahU>n@zy-M zPZ#DiZiRFXw*#kV&%tG+nOx+)S`_x$hDN5J2upmK*Yc}SIBcV&_+eD{RG_i3)YzWj zeXcPx`ZxqCl5B*reotZIwv(kF(Bq}FuT`MaZ>_(f+H5=PS|k$x`$@JgEx9 z;{8uRR&zJZ>hl)t6uHQ(=v#sGn}jN`o&OI(W9+A8$|oae(3^!F73eNX9G^_ z#XRY`@q*;;RL`~0Dt9kBpS%FRhSv$_!m)M21xDyg z=ew6??`M{-XK8oHNc3Lb0Q_cbk@Wc(>|_d2u5CDBHBwz5;hD>#>F-(m(XEhLavt)& z*l=%c9J*T_#<=^FWn9rik(Dr6I&2Ems9ZSvstmL1H)e#ZiaW4;-xHvG=FSy^xZ%Iw z#8EZRCY;*H%X>@~1&{ZGT92r`_BpOag zAgsXP))vz3gB!TtwdD#!3Pua?{4-TUc>)T%Ehlt6Plpwdt<#D}T>67KE9%I_cKK*o`-q4iyaOXHEeE#| zi1fE?@uA*eoa8ErTd~61n#KEfku;7f%qnwVD+r??=<+@Aae2v{2i;=EP1ESU9@=}- zvmSU3md0J3@qKl6SdWVI<@xDKW8yiU2 zb_esq+Oqgn$KVz{M`&WIClz*7WL;8t#tM@+RPjMhRCTUixAZ!R`+dJK|E;vw=4y`a z)Td3jxE6yuM5BrRMkHPaXP+EYd`tC#K9Ao*RP9~h^`|{oHSFJ!&f;jTM|0Md#N`+r z8ibw$-Z0-8eMP+P76@{>0QtSHDg7jZ*F~aB&HA!zXH6dX=DWx~-w4xg&>kA=A1G{) zg>jqFyT)M*+95HmA`^4J+{g4I)mh%37_b;pMi?=PJC!`pI=Ag0gQkr|i|+pk!gGzP zU8^NF7;k+F?b)N06lHb|755<3KhQO*c$u!RjXKs8EW4jaF9%m{ zvcrnIT+judt6wlJdMh~OQ!cFarv2p1Ar&qot+R$WPnI|JmkPK2ZzY4p+$RuqwjpFi zeGn=y-DVx7bq(fJAJT6@7?kdG!Qd^maB;O9g&#b15C`LVUAcV-s9MqdFe%4rZsNG<2@t6=kauma!modJulHO&K9lq>PwRyF%aE~FzN9LD9yG3yUMYI z>8G*0a2ii&s4G3kj7QQt#Gl7CDldag4-r>&hu}BHSWveM^R8#3u}Z8VRc#Qzf#>1p zqWoh%dTCI2z0bhXU+I`$QXAc#-p0tp(Rj1oQlNTNYeXD!LaVsBu(|>F>tzRV2M5UX zn$@am!g9iK<~+_7y%$;tH+mM_$G!yp49&R8g~%Vzgui<~mwQ{rFyb3#w`z$X z91+iYt--kOH{sc^?^t>z2E7KPyCl>KL&8v2Vw0!limn5HFw)Rk8aqFG_dntHqP?VC z=IMhBq;uoUDz5a&tRU@sSm`EVS(b%<{hLdr(L(m=VCA&>GCsr}^4=c=(#BA*wwW~E zVgtFoN27P=aOvIIgnK*G@%g%S5c$=0tJw9XAm>O4lmOGvP;CcUri zpyw7IDJ??za}tQRS(-^IdIk0sq|12WuKLnjvkvk*jRd2n#h5o>AJ46E1Jf?F){^f4 z!gcAM5~AMEB3&mT_BAs`cFco<&WFW$|<$DB&Ke z`58_8%82JU)eKq{M8o3sURcpuWIE0M%pB&J@mtMwq-Ek6h;r-(IYGuqJjkg=py2jF zEKCco(zArD1Ow)nf{;bEnX~(ZiZZhgBwM;$6Fni%4Uu`181J_kdn!1##!0 za$+M%{sFR9^wTOG5%+yCC#}dy=V9UGwIbDLD)SCBW#n@(?~fG*xoMTYsq%q@``rK0 z0pYr06wf+8xC-}4UxLzIPKtgy>OS7xK9u^QBA&uiZz+a zQv&ICksw3S^7K)r_$;b>N1!=#sz>4bdt`>&iEg~$v5lnIf?J#WNPH@z{>BR$AM<+2 zfwVm*Um>ThCBL_G5Rxxt#3O1g(dS(OD|>DN4mYo;5_DSX$7PqIPo2l#>M#Yn+R+FbIFY#j^w$-&2wLsW<`s##prw687P-v z(Uqy-emxONM;(FqO$X3<+$AV;uMXu+zq8}ff0%Q{4Q=r8_M{zNIg_6Q^4UOfM5;qb zm=h$um(%$`V{2&B8Yj{mXK|OwH=r#l2x|gv0-(3vam!WD9Z=PEOlvyBu^sktXHA?iUgswvwMsw62@tRFYklAX3i|iD-H_m4-lR|u8O^}hr1x%42=rt z?8Y4nS!&lCmO3zoGF(dz8pZUebLB*SP zn1B0;PV2L7r2&xVI`kt zqnl19P+ha^McWwl47A=en{bQ2Gm*3kVdxBu+S~<{$0M(S6fftTS9D(Nw$7ZePGm

Y00>0s9~p7Tm3d9^%K!LpX7L1F2$2=TrOx znkG+}tC3)jDpujrnZMzEoU?4d;E1@iORL5Yd&k+qEmw`C^FZ$5oviX-H$FAEF`rv= z2Bv@CkHqIiIafAo6(&00+JlN)Xio4;gHr8{;Z8vK4-uoSWJ=^OY_!z_I$d^@1Fmso z)%2y}1BL0267I3t+&H<}$?} zjZ4-*Xte-ez88r%Llg#4e)PmB0~`MB#S_vDC7|MO;ocq;7tq|&x*3zioKBr^sn-cO z6{>?0zC#>05?`ZLGd=O$@T^1c#P2f|suj zGWskJnHi0Qr;>h$^fD*RiMEji*PJEsvZPufjn}wJM|nNhhiG>zDYkIk>J z?M(+r!cRGCp(o~edWyWAzfoyL(vTQ3ZydM=){>+f#Es#*h(|e_)bOjq3I6%3G3rOA zfUT2`P`SGA&00ZxQ5Dn6@h7qSp2>5%08xL4&Mu(4dV7U3TI8(rudC>yA*~n83a@ zkJ--8*MKx7Z`1pw*5VR*TlyPuIH+7XYFP<&4F3okCw3f@h2zPmjQV8?PYUQ=(?5&k zOxv_0g=Muwzl;6Erx_dZ-lKMSu5CXkD_x1f4{xK+@-h~3b0ap)FJYC9vc>uNiQ?YT z$4Gnv^ga2#Sws1I!xHFeQUlK(ej*63B;^W-z>m1^d}|qUm>#o?bLJZc6hiau+(+dW zdu1~x?S!3^ zt^sKQ_;S}0#|F{;kBp%r@u9xx*0`>$o=+Z+#(c43CZum(1f;29UQPg;J9dYLcpfQ6 zcqK0m2FV=o@idm5-$je#BR*>=*NOKvg1#rtct(KoD)c=((OBS$-D`yYuN|Q3QQ_=@ z-pLR)yB6LYyBgTe|B8zKhQO{lo!Gsj!%=bEHDt&AM8#9;T8UHPw{Hk1eB{T|&!EyQ z9jhnNep+{kOP1rD8m0<|s(4#*F#VoUT#78#-U9L@NFEMIn}}`OpNX`KO9hQZ5SQ{` zSgAM)cev2r5WUBu+uM#voDS2bj;hkO`Fo$@iKCgAYlVz3O0>1I^d(Ip0$06JISj6^ z!ZqU)yzyo8H$w5{t#>a3)e4we??Te1aO%xv!6)cRm8WMu90uPZhZ*@$PW)2k6$brS z!uRJEBYB7_j!oZSCHIUtf(rMQrawBpq42EnNck7iRr-?n7HJGf`Gy5k>i}t4#gnM~ z(sa7F@4ZHgEtA$m>SJ?$V3Rv6Ol{30`7-5|A?5B~(z+3#v=wm=pXEQ46aI-_Uq7RD zqg1Bf_$8X@S#bkhW0+ss2*`UecS|GQByOp z)QYxkH=RxsbSxmAXXJCW#s`{_*8YLy!x;HN)LXX;j`XdsG%CA)Do8`x0GzBx;y=C7 zFcLbWcFCzKzE-$(e!dw-oN0k^XUf%BVS-mIsNB-}uVTtGkRJqv!{pHsD!#A<`PsC7 zjERigx*3$7f8~CSo^9R-amQmYD*F|vIwp-#6*I*t!w*g3iO(uv*t?_T=T->fEF{i` z*~5WfOZzT1H9I7Np(c{wWBg(&^WU%^iIW)V4Mutzrn$RH<=ftO3nf2lMRTR2JR-!` zUIk_yuK;;c(u0voQv>;YAkC*OUTwkOce_D)GoQ5n5yIpiNI5Qa&zaL2x(tb<6~>X4 z?S(%hBqq;jsq_F(x$8>DDG^Inns6FBI?$TVn#Wn}ZL~kATFMAKjCr)@&XLx|no;3R zq4u$RV0pa`K8}b2!U^TUapfv&Jf7ZJD!lqpaSo0HJulMGf!R7m0rfxVntOqNYbzvA zFSdk+h0Cs>(G}IpOf$VgN0?p)d34}NGBtncJ+0U!kap= zE334u@@Hxem0qB0Aq;6on&rBdc!5!TSojG`+}ya=pMf8skIOJ%6Eb^2$GJW zy6Zt)SjJ-i+|*J(h+_L0C`ODlwT5&7e5&_Kc$dzgen%alJYwThpV{f`_2dWZ3+kBw zaT&ik))WX&;ZcQ&(sEV(o#6T{U_rhMy17T9$Y{cMS+bod!9T{1yFh|uVWEJV4@_0oWN51NM zTO77}52HH7o0C2%?9oskLaUB5W@J1}-FF0j>^0*9*TzWlqiA@_N0RO$UEWhDEt+;w zkGw$vKC`1|Z@J=>6;0H>I1Tu*5#xNWtaX@ zN`EpMH!kyafq>(;C>IK$-k^y}N3w29vx$=~Y8*dq6;JOy)hPW!T1VVXO;^jal*~M@P1DWB<-;UujjNj(t>zU_?9sI2Meus0!Z((W%Hj>Z>kf{Zlp(EcR4(> zOUDnTods!Disfb?o&wUt@^!CVBtKOZ?=`GI_1>6e*&Nk}1LY%U)b-YqpBLd7g-F@} zly)ax(@+mVX*2vfWCg9M+=8B!U#uZqgL@5Q&7&nP03G~BQ4in=%#;%;W@j0B6X>p=E~u9X3ZSrtz zi(oKIq&-1GJT+(lXjL{A1@*koy2oS5|tz#J#k) zfDtV{)AGI^CcJ&DO?W$8_R%$y8qfOjspnG&pVbaG6u-umj3Vd~$-}R+TFAV0 zt6Akq;ETufkT!qzL*V{VD0{b-+h2G=htM|a_pmack^EY+6Sp)Sf!!_cv*K4xvDw~F zbZ^uiXAX8ha&vSnnuMo|vedVlvv0ESWansp&2uH5ZuJp|`y|2}XAg*OTtoKQa|xr9 za>2aCF6?DvBB$K4hwMI`rA<*U_?+C4V^pQNT32b-M|*J4@szdi{YBrG z8PH=-ciH#kB`no%4)um*!0;@MY}a`b_c*m&Mh^9o6JozJLz6kIxK@9j-RBIOZC{DE zyVsQGN4{lK3*)f(JiX&u>4G`mDlsSa1bM^%<{(G?q>0euL&iYs&!*7mBdeT1*MO zqoHf&o)S#rrqkNG#_AgQtx1ig{$GEnzORLRbUR;rZAU)7*?3p18SnuP zB>#YF@D6D1<=Xi5|d>u{XrZW*U}hwc{PGW^9j zF#YeTi0NC2HT&twi`VxF$~7E6&=oXw-_w2LwLG+298i4W)qz_Y&5lzzbGI{o7&nr4 z-E0Lqh6a30pWi_LP;a=m%xrmgt^+xrjUYlk%A!{`~thYihT^ig_-H1#dCdHe~| z|K>uWbwmCxY(J)rJIbP(d=OtV=Ht_2g{-oVxttxBh&R4;E=vFF&gq)C!?b(Ce)m%F zIAwsEMeEq#Y4v2M<({(HIXfQlKm&dIIIvNHr?H{=Km2po5&oq=1;og5N_Q=MGrvb{EGSj#0TKGQ(Gk+1#2NoA1KUQ_P`dkCuE}uM@BAG zml)F1M6$E-)W01C`VkYk(JWid>6#^&wl58u&Ygpc&UKImy+T2^b_UX%!PRpwkjBo< zTR6$1UR&Tr+a&h*QBNMwvbu~9W_)_hG|bY?2f_eu-)@+o+ODcuikr~v2o%H} z0!vHM3gu2{`l*(*Y7hu&%sJmC!*tpU<8Mlkq!S=3~nRzpCo$NV`%z zcj*sId)`;7*ey8jEW7sF3SDR|HSaqEP``_>su^(TnvI!sA9&NwCE9?kIq3AtTNc!b zmN}L-{Lq7C@N-l*cy4hKKg8COlvlpv<{pQ#YUZ-bqhWH_T6+KI+DCZZXagVy;6OY(kHEHC^y zO&Ddo#H9GixbFnr&-XroA6GZWBKtz|*D0Oy%mTKTEa6mdxUY39E~a&bK2JxGUBuOM&4>$dr@e;MdbJEDql_d$*7K4w?!3D z)~q8>{V*D7{)D=I{jxQhKZ*OX;It8DR9=SZ3r=gkY#yq8qTPbOZ>)sree3c=Jr7Wg zHIOwrq-zWu3?MSwmG>?Ghjv--SWI7<@74!^u7|bWZ;XVa;8w`t()fn>{Y(i?ZSDof zzTOcF-_2w-I@pOni7kYDY$WG&^pdu-W}r#d25msg7%{zXG9&zxqbICVc_zPl(0Z5O zDt(D(koguAP0qV5UVd&3FGoKVIm7CpQN~!`S)|kJ)=EH?m+4)a$#s4!V9}05ICdoq zLK^L+UX=k(d{>{3_>>1!>#%cW7*M{+7nP|>f8dBqQH=g8l57LSuG~g2Z^LP9H9r9< zM$o(aXZTdMop{U;>3ZQ~2M@SlRg<4SdIn!!jt2L>T~r;3Ta&iKr!L(z53i+S;D=+X zZm6D)g2LDJ-TjDTw&TRT?Ko);cKM$vfB(KArk>mg|HT(7+>_O)xlHqnqq{w(Ij=8R z_9K-e%<{Yk-?z9KC#?kUcJ5R-%UatNAmJvAh&BYmO8in(lYdS=2@#XcXm0Evyn1p` zX|TSEn~%cR+uc{f=HZ!|bt`pMz3@>5?{VDgNtoTj5`)Iomvl`)`hb_5Ta7>e`>Gvk z?5^^Vjy(~H8~D@N{$f+TeG2Ea6mLz7nUC0*X}g806E*I$?)uX7zmbG9Pgx`DmSSOK zFgu;P9!Ptzum)y|%g{3R1`bZ{i%EN|tLEF#r3qL1YN6>iAgsdt=Zz$B29o|^l*>?1 zr<&4hOpocwj=Ro@#a|pGX)#9FiK>XQ*XU;dE>j$f{mn;yp^ln%w)lQgHxkvgydw4+|RC#9IJ|3ss+6sFNOGR<5TY~wv zVHPLq$!)y?q)~}EQ0(x~gBqM-4=Sf#|F;DmHcS$?i}RszPC8TZcFrurmZ?q|ju9KmtD6LIH%*O;1PGu=S8t5H{YvvCkOAE&ho%m!l0)@#tNv4(q( z&w;yJw`kSa-VYCgu`SLc)tXQm{fEsBOfWH(w$-~x{~6uUznTNixgG~gI{RSo?5he> zHU7`ORbeyXAI$!tWw+h`U)!WhnQ_|zIHQga{x$rBI^LmzG$8ml>swVPgk^X!p(o9G zW70>K^4#tV3I{0PmV@WV8?;vB8Yo}u52l~i0DV`|^V2Y@uBO#7A$q1Ru0AJCpw(kbh3lx&|%gq>qq15*{iEV~)K8C2pw0Z|VI7w+HXwz}S6A`~Zv2 z9j7&q29o|Q5o4w~Yjt-;qS35P;={P7cpsWm4OmL6y|#R&b#JPhsYrUM3iEu{*VfYK z<=`nhh=0Bc;v^tU=5W5DbmXNSTkSCONrs=U#w?T@>;Jdjm zQoV7HB`$J$#e2nLjOu`?ye=6w0q&Zeg~E~R)wtv(M}I*z%dA%|l_^hik$eR%{<1@m zHb*rU#phOElVH^&S4p_YTg|U6$p;BKCQcl7n$6qLi75`+8#Y4WBV08-36$gXK2|wE z=`MwZ)W7WrU#jbIr3dPD_(poRx+L!4{i3Flw$Vqz6}-G*0!I0#ftt5Oi!9K)_7F*{(~?*c2Xa*1?;mg0^{qZvPOa4nF3WSX}Uy7i|%U5$5ijG;ij zixJlY=_cqiT`Nc@bIL`!mQ*x&*j(`g_Gr69aW3^)HnK)HuD~-B4?%nYm3@-5DaZGJ*b)sFuRT9`I!ABZ!9}v!wG|MRKZkK`Pp-6eYs)dks`&n*-z4N z(X!FngMS)GiWS)ToC5ODLgC-C`CdSI0TE3$va2uGA{<b}2!HXaN`EPSP@J#g zRVT$8noKyx_Oq^9@>b-B?&Au(JLGS6GvX-j?3KgLFR|gHCzyyOg^$Uj8A-}x=8&)) zC=VI&Y!$vA|1%3H|9~_jz4y?Fmrq=yatq0qqTA$CN+$z(2OPg8j{K(~*hNbuo&zh| zk7{*V4-rIbF)PhyI;x2DB(0@W*UIr(h-ocuqh}FTX*N1s_!WL z@)7NU`T+3k>~-yMo99?_;bD0Es8mcG)`?N9k-WUp?~q!1tw@TmBwf*fQ#|>l@mpZE zYg6T6wRFw$#JVw})3t2ae|{VJmQ~av*ao=`@_}?Q3@Is4o&a|T)&c80Z~5|bzEE+a zIg;&e9#i#)r0W@JL5<3%)^=@3D?by|S81!bv3>C=(PPMeuzXitNx8r$?kyHeZkpgQ z^I&$W`7JT`jsrZYQIFQP34?AgYHOn6>vA>Tv^CR!YL`>(A?YO*Q$f8I&3jf)Qz}}4 z!jB&nv4Z>@Dy~;OmJXo>%C!?{)-c z$C>dJ^mf9W!}qKF0OgG6oWB@UT?K1%81+u5mdb%}f@(%8?ZfH&z-_m}VcJrPYG3JF zs;$+G_#X)OwePP@$KC}gK)eCeKjPG9Vf8x9QCdw9rz;&x`x#skKKfcg{8^>B{qECq z;bL@QMr99SUi%l-`Fe5wVjW3biHinoB!AXKDlSsFMY$prHWJ_PCU@Tec>|RflJY^2 z<|i$?1*q@BHl{!9Jl zwfRcNFGzhU+2N@%{Ay=Q-?Q?i9vStvqUm|_MEqLfD9FQ#v)fv+=@nCD@}!$$RZ}BQ zwL|*yV%0Hwem;ey&oINoP1Rr#)dDO0NzaYh){?rr{#N;G>J^Auxd$=UEEA&+c2ao+ zT_Or_O9xX%HLq|E9BaK2og3}|@@>NJmjUh{zKHIY*Q@GD)HF7hzuIkLZk;Vy?8mv1 zuoHbF{|V~TD9`IlURCqP`58hiiba+nBs+4dQ=K`8&`Xt-hM8eud0) znV|W$Fo^ig7Kz{J92HcXGeoHLlvLluhf9UhH6Z==#GW6@q% zybcnozvi)fFj8+q@e>eNBlRrcN!oNg61@*d-%@^;&|D=j(se-itfkyX&yS(Lq}h-# z3aNJpq;a{@1a`CsGSw?r{Xo)#l6;rSWf{`QnJev0<3XiwRPSx*rs`C47pbQ-59{|l zR)t-ZKUL>_<<|nKj;k>7Md=i=;MG7*Isw~v-;DDf7chDa|fg-d~D?@^s|w;@0@_YAg(lwMXTfB7JqCG&mL)%*oZR zUz?^;d9He$gUURsJUw9mox z0o7wo?v$zY604Q#fl7Z3a@fnLmVk6O9{F@rVV&~jKz&EUcD(+t{B^_|w6x4-igyY} zTJrlN1}QwpoU7C$TX33Df0~c4bx2_cu6E57QNP=1sJ{T!!fYYiZHCNUY{GR1YVhyT zp3>4g6FOgLBqz4I%X-dgz<1jgz^Ue15G)X{1ydh*T|X=v)Luq4r~?lc+OdNtmdTNK zd!gT$V4+<*f|nS0@cG6%ylr5mxNze!>wfQ`_TP9c4^3+ z9;`y$fm-~$>M*bVc%SHapVq;e`wfN`JIGb7wPo^I8U+j&zk>_O!y~4P4||gJ3@Q_fY)fSOkmOEn)Q@>htn} z_2EO?_Zks30Shz2q#BP|@gAVDWAgNQ*m0vhT5tIabZmThwYL0~X3JZQn@aoloYkJ~ z7J*^CQ)y$Hwb039rEK_cD__@RBZ{bUJf2ue&vuj|#R|FS7VWklWxnQ*hse6|$M9fy z1KHnxBbMaR{y=w(`3R3G*nim)`J;FxVv4eGXJ zgEu6x$cC~gy8H$Ut*ECRawlHSvZH5sx5jA_hXH>a){@sMZ47-b{w?Ys`T?!xZNYz~ z^+lhHkq~>P5dMAojnxLM;qfhBqRU>|^J2q&tP!7}o!DxS#E6gL_0j60`pN-;iuHkTjU-kHyI55g>?cCuF8&XAY48q;q&$R3ScW&KAN zR1UJPS=HsaNz2*1H!s0&c0KMA$g1Z0;^cR@WuA*1U@I z@4SQGZ9K4SqABkB&kWlJPKGVh#-ORAAJl#1iEi`8@wW#S@yX>z(#D+D%V%zw)b65o zaBL@xIGYOnYj;D74=!Ad@BE+yxN*G)t{CRPSDVa{8Fu01odUA9(!jsbZP85CNY^=^VZqJ~Fy$QOzFsNZPI!x-p7dr^ z7w~v%AM|`U0ehWa%?JFwj+Bo;V}y!hUqmNy1J4|93h%qTqVJv(qwi#D=bE_5HEs!# z#)G=)4^e$5X6aE{SK%^H{-bT1*N{l}*bP68N5967_@!iomg-#0jG^ZhkAlJ&*d5_4 z_TB6&A06n*Cj8!yDL>z^mo@s~FuP?~>2wGwM%oXbf>oT+W3L3Wc`-#hy4!KP71iXw zPYXf2y``MED-m|QY|Rhss1$P=l74S8ftxHCtX1(^-{78D@0JBLM*QlyjVWv}PxIx; zEkcpb$%p?b5mX-{cgR^xOUg#dBl4#Xaz-zcBSn*SrM%%Tk9T$9C3$|Bb<&#u*Os28 zyxCWyAGR1gAC}|%_2(f!af?RP`=X^z_~^@2PMCrZ^vA1umi=1My}{LIVOXCbysTcD zZ$Lmx+4J8Rcv0^%(A+}c#cq=38O|D<7nB=#_`eQtWJ(LFt&Vca?K+%(uN8>}a{BP< zvee2QVyB;h%2f?;c;RM+!9wN6C+F{Y@{0*?%_)q1yp}BIN%JZ-%(w| z=QLfYzuAe0)H2{kTe>L>=3ArmdJK9#=i%#>9Gu^4 zCs5tPx6)`Z(klRkNi)lvaH|0WdDqJ3Xsc_6JuVbs{Oe;Hix1g&^Y~3Xe?SMR?s>r9 zIH*2xj^#BNYyH?PEc8S_*aJ+gIh>?3QVmP%scK=Y4>e3*t3x=^gIs zoi}L0wKFhs#0a@Ic|7(TJs!~Ik~nRB93sn7?QNQ<%u=Z@-cC09Nj~{ z4570hgL@Nue$dcWTu4CawxTuf)1fuIomeJj^jZbYn#97f!A;tp`x+vz)>{g69&HTy z0fm`Q7d_Q3ni0m=C)Hte&02+xQPZmNUVFO8=5^M{$uZkdVc*G6RQ=HS;OU}~yzLkdemC+f&Kus4KW*Diu3DE#m>nsftQ ze7p&@#~0#^^&Xt?L(;jbYPaBIFbu4NI zvVhb6$U!*o%~+b>0PeZzlyF;-uid%UO6&1{grxcx*J)3Kzw=#Ce6^R2(%!&?kgZ5u z%|q#-z!it5G(&}-gomIX_K)f!SeB=k0M#S>c6v^8Gn~6_+Q)|M zS)lMvD9vHh`7tA`!4*G?MAYy$*zx{Jme2NSsJ01TcH>WI*=IdPL}7Ql z6J`lJU&Mi`Yt6sV<97q zgOMBdVti&A4)2|(&c|b0G{MHY&Be-^v2b-zRuwN7zWk?s=eS%;pM@J;s&nJ3^Ekw5 zk$4`Wq%lYerv2Ej3@c7wx?Z2G<_$J6fkk-IaZyMmONkuq)c#O(RxTYC}rW2mA zBh3w2(}#sXIt^#|`@^yAXE6K?t*?9DiP!QruEM$SR?WFOK7B`wDS8B(b~LH_|J2u? z@aXlWxIe0pkxpWb$C`7h4|E7@0?UV&3gQrIs9%COV^4Wf<0)mN*yfTCWwKAMZk%KW+U-U2$tvEagQO zy$|hKrKxN`1aNos#`~-5301Si1)OkArVpD!m=!FE#~}~OaMi2zT8x;^W{vzJG_AjY zMW+TVW$Jdho!x(EEdVA1Igh;5$npd@gQ6 z(pg}JmDBz`XS=;CIC*j*B<>up>FQZZ z$Jwel0|(zdf~0ML^eQ-b_XE=T^22x?_)%O_Zfo}!X--H7JMxZ|iE#Pp5{PWE0!V{_ zvkYRSNAby;K}sXa&Cw6&nH?u-Ker}v`ZbmJ;%VDrsJ@E9oQ9c--(i+rFW!9f08DpX zFNlZL*pXt(D;8eSuJC(^UjL#Mm&od?1l7h^XcGU4S?pUz+}asCiBOu$J^23U77V<2 zkxi-fhLNtt!R;Hv#5af8_}vJX7g@8}re&D0L%BMdMSZLf=Lwf#zS-@{Y&VXXx4Il=p%-1Sf>p^K#nfny}rsd*fTM z^YC289RA?t$=z6iCJNL!ovTez9)pK#Ke44flfYtM4e4C%EgRwS7Pf#XBTV6>{W$p| zVR`f|n_RvEX3ngH;ijvx?v)o+Ii)nftdX;b`!f_r0V7)KVS&+9BwSXQ%!te6 zgd>Jkc$@vQyJqq99Fb*IEZU3>$Lr5J3DO!cD|s0skD#=k7~{SHNsqGaMvgqNL$q+d zRG0f-Fn~t=J5ekHfX=1G=evRCN< zY=B0z=D_!Xmjul>=GyGgk`91sLC4X`B47Kk-eH{o>k!PPb&r)M^|@%vNngOg%0UX} z=(%|tPM#TPZrIg9&3O2mrONNfq|C#(!T1^0Zfd~^8|Zg8LFtA1{g$ZtMDjoCoY3Jd zg3<+~4{%tmBGSpeXw$!ihO|1>ZV->jT&kftMTIj10)8m(L~FvkDXpwgJ|*RFC+yp( zBYxV_f_L1wl{opOco+6aVG@k4{g-@QEvn_eqLFk*uglX>)fo9rP<7a{|6Fz<*ou=L z!hw}{VcxFe!e!+nVbLiKNatxByl(>eMEtowiGIEf{JSnon^#t*^_My`fPCf@|!w4a7F|114)MUE|zrF9sBTdneRws-*g}$RnBc<;K#fy}O8t=LPY1Qv2H&~XvM~HG zGm*6jb=GwDY*$T{&qmSoO-1@3=Ki`8Yh%JPsb{zs95=yTqB}nMx-x;th7D z*&0R|i-eWTb(0$;6xO=96`9Piw2BoxPhU2$t zsV|^>kRZPFCEY4E7>{B!zhF4t9!6B0g~wqzK=n+Vb%)vRvqSPkg2teAn(Kgy$7bDj z!C`$KROv+09!&Xky@m81`K+Zt@kP=|jCe>Ir!~Uzft%sh!6AyH(0>0@2=E%g3tujV z`i>u}Vnkf3dTtlM$$ig zLH87tx2E}Kb+3$pO+5}O?Iwu36$g>V@h3kuQuPI7<9$AiJR_c5+#d&Dn26NNDk?mY z4lirmgrVowX?ISgb$0LXg$3nIJ81Vx%!%*^>Y>S3mQAX9VhVdH7diFGq=zs<@(P*~ zgM+BBQ+Z^+h0BPcCG-j*FTybpJBW2q1F0vG&Tu6!MPxxkpwSdn@yBEahj)%O=% zVw@^=yHq1x+Ds0<@J6HJNj(@rH7H11s~%aE#w6c`6JqBA`BHp3uo_QVI!#_W@mhI$ zOdZ->+d9>TN2dFk=(@pEpNUx1n zcf3CJ-Re|nJHm84*u57J_Th)R2|&FX^84kiOX?^{%Jf!x1e6{(PTPiSYuW>8NJg4c zD4rpXrQFJ9yIrDzaukl9IKmzsxB&JJsnq+pj*m*L_|K0%*#G=4^2xi9v=>Xwyevuw zhw=qK4RBsTEs?q80xmWQ1l98+kB-#)R2)>LqZCi5UZm1|DfJJ4S#fiDs9UD+>GvEe zADg1;ClC)Xw|5P(huKi-8}-D&UB>#SdL45!suK^JunUVf4p>^_F^h$ zXmz<39<^?c&w^@JdFO+Mdtus!aJch%FGQDz3aTwewMp7M5=l?vq8WFAiy+-840AdP zswqC8C`Uv3i#>Oy`yg=@m~h&WYSV(-wi%E6nT4$5vyd&leNW*NF1%O8HsiY+^{XR^Yf?~oe#(DVed0GQ^~AX9d6F(u+6f#jUMSB73il6ozXB#7 z){+*FqB%v1(){yBs5^KDzPRuT)p3()Z}~*CRO~=&0l818#{aJI6r09urgJ~QGYjY) zD5I`Cu;B^T{I3qS*v>U49}eRd%|4;^?zTAKp0RYJKE$0+7a3F;z>T~P4F1f_ z6XT{`V4jWiNl((*Usxb)h5`scOy_C*e?+S^B-C~nH5 zu3usE#+EYotRFygCudq-h8y`=Xmhs^&)jo_as4k~WnK;h`kdf&?K1fDKbY6OC!4;% zJ8wNjPX>hk#S^slP=(I~O=Vt7_ITY`ma@SX>@Lod9@nD9^jhXJ;I}@H_3O<$O*{eT zyN=Kd30m#DzV0Tju5ta#|6t*$WFC1j8NL~M;EV73gqruU4qaia(FLGmNHx~5j;qkN zHucV9-b43EMl$(PDq!y&_%rh$-W&9S_D=7vZ9a7__xe-}Mqb|he3FxV)2gLR8e+re zYc@ zyk{M)!Uq@YCFp0aFSpQ+JvWv=1L^lrBp>RFy4De}cd$MGWOo3*_*IAzHO|A_e?2JI zE8t#+3*N(QO=iXxZE#wgm^!@<@9-p6)blHZW3?aA{kR_NywwP#IfBqronYjcTC_&D z9jDw7()c0XK02CDd7p>|2lH|K&I@$kVkazNS1~xvP|SW{E*I-q^56nPl^?uS|6(;i z_{!@w{yUzB!39)ry$`^$NgE)7)?=A9@-jAU`%Pn+I0GpURZin?6HlO=7ONj#N6K9h za`zN_VZI-!{$Y+eJrD8t5JVTOm(8C=$tg|iP=1c#cQh`1X}KkQ)wW{y-)Cc!wAsFt zAFS8g6;*gLDxaQ@!f6olC6LZd53Sk}CNFA@*=NG=WvBgeYfKvSnR6PPT1G+hBcLJt zfwr#x(zWb6BW#w|!#?9x&n!@NA;_QaaK25~zgB8K$3R~vV>Wnk5U&+d0G$W?gLgXa zyz7pW_<>F2U9ZH6WxY2PxtaU`=Mhmb{t~LiM!2_D<%*x13|~bFLHq9bmjmdx*YbDTe5tgU&jJ z+-%i!WVLF@#Cz6~Y94;=trSJ$ny}&4Yox+Kg;hJ82g;r^PO;XHenPVYI&HIdrGu&+ zs~Lqdd2wG(bHSI-xCVo|25B$;+y%rJ&_9cw;XTof^H;xNpxG)mdr=R$rt=MKv&WAg zUzrKTT8FkC@x=-k*}LoI7-wiD6=!^^et;eJ35Qnqet_;!F9E;&1>ra&K9D}&juw^w z-mgi_4dpN0_6xscRM@cs8SI*@d_N9+7zAZqnMSL#3tyQ;qbqvP%)|{ahr|eElIhI1zW?VM`1LY z#C<@*P?cMF%3?UXee}6@VQ^jf_C-rZT#J4&w0~qr6G@y31HQ&%#Qko(%eXkCJb>^^ zL9iic6LarX4ND*sXC<0L+U5EX_&!qnJMJk}+$k@$Z!hKm;W=T}TP@AKc1)WDM*PEQ zKEx5*CRp0@DTK{6k<&`nvLQjHFg7MZ&84Js$!qqb!1H!Ht$*Pnw%NWv)?U_z&$)0} z{f>Qaa!^oCVOjiWc`$h!tUGOkr5kqRfnnQVk#cSkDO{62&3Ws zICqgWBo&FTVX@A7LB|ju(sJQRt$j#*FWN0?4`*{9LxZ(NSXQ_P12%U+b*)LR z4gh#)e-a^_%j*(yDnwAl{mb?g)pW&m*`kLqEGIS=g7%I!03 zj~{@PXQ)|M0(DBJLzi)ek~AD{uSmlu&GzAq%UP&$V(!1o^nQ|1dWOd80ed^x^M|f4 z8EG;N@phFa>o?PwKb!3a?RxzwI-TVRE>}Uk@8OO$8%=}oma8%9dM{)hYRE5>T1mmxb))r|RxG~lo`n57pAp_S=^jVt&T#VK zBdI&|5L>s;1_L7vq^f(;osu*jrfk^6mfxr+NBYwGM57Gwn*B5E;TNcM6V3l&-{YB0 zah_LCq?%E6!Y78BA?aU9ni(jrobXgSrZODStrlR%Mp%t|O8&>uRR%<%1YrwA!9)~9F~I;7 z!GPOYyA``zvAer501FTmMNu(OQ85tYcGke|E^K_Z*q!fNe(8H&Y|MOhce}BM*E#-M zSM$E|?fRj7v0nv^+E7OOdD=+PWA$lO^L?;dguA`tK6TMcJe-Y=BFsgD`?&V1qekzt zdA)|sac>;tx~o9={&iZ|6_4sal;zdtOjRd%dakis4(R|C*(Yv2m$vUxlx2 zP6f>(z^z8dDvzew7`mA2oU5mytp)sn85=)6zSGnga43!*luy8Yf|VGxB!QjWwo@3+ z2wG0yPtoCHUN!b}A!zvnVtDuSw4`pdLJwu}A+)>o(+7oXrbmSwWXW=zA z@PU102ppmgje7{KHxPV39$MjqwdnV8)>`FrN#H#@TsojmE_EmLBi~>4OrQ?+nO+{I z=K09o-bOnS@Y<}gf=5zK<~JatySf!SE#My{a7UKQ^^MV!mYk>j;G-7cwc^>$(IU{7 z+RI@Ix%LbotUuP>ANw6l3l^+o@TMh?%KKTN%0$TIVZeZ{)|| zG~V33E43+`msdN-h{kP?8GWt4+3gj1-i%hAr=0@E4hNRv&c31xsa4lTG_-U+cl)i7zybht9OqpZ$FVJdU<4fxBQjtx{MQ%3uDYfM3-m zd?wKL-sx;K@amLK(r9)(UczHir*4a~_!0bN4gVTVp`($br)`lJU4snQGxUZ3V0n%M z+vPDBD?&Tr4#s^^!ssuX7w@=f@euxbsG2xhppSTcX^5$Bxi_jbC3-#KK)opSBIXuYJ7?htV@3zspkto=2Z`(M?;d?30E zG~a6lf5HXUCo{AFop5|CfZZ0aq&ezM&R0e=nmx~|A?jg6i$AUs6;{r+LYG@nOIGX? z5v8pnN53(&?}6VtFJubo>u$^EQtLA275R8Qjiaek^xRTSN~%2gz)O z{3!U<3a(%JFI}h-FH*9_5j?MGG;^WY(D^p>;~O@A0&6L)!6V~=)x&SkECz#a4Xp%E zQiliDEWqG%g~w6UBNt)ztNO@bGMd9^e2)#2$Y?Wo4~CW#z<()A{87uMU57?1!O+Pn zc+(Z~Y&k~&L#!R~J5J0)9Dch=G+jK4`V2Y44Sp{W@c&l4FY0eD@BY4?uusH;&FN6aU>6Mz_wp)t2lIjz$I$ zq8eOEgZ~~t=VtXI;{gr+Za?BinbB3mWS7<=q5gUWZ!A!c%HSZoZK98_64_|#`Z&85 znTO(t zCCO-pwF^62t`6L$;LEc-3wp!?zsgU8UieoGW_~ksx5-S%O z*OI|pr{kYcm;Kb3DHnv93+jc9=H9+MJ?rnEHVvZ@%g!4;3*I|Q6|PSr|1P<#mh@b@ zLceShyb0dgqG}(X&&ZYt+Dg3~^pa{eou+H;`K3mdk1#&MYQ{6ZMaP6-$xnnbD65-q-@)#y4wLf@eywz#EzzLk0yjhwh9I zSn{yF@w41}`V#7#X0M^UglEg_$YSnq`jxLsHngWDMzmv)A)=baNvvGGY* zU*%o*2Tk{hVi|OlUuAfk#|z|$`u@!wgv^1rykBQQ1_m#C(s&jHt`d*aJvsY-BTb%? zg=vqrCh+dU8#wXYU1Vet;zZpoj9g3^oeNB2yoaiF|1dxPoWRK5=*#duzyo(W(QONU zo7A4_pI%_JBx64n=BmLkCY(CnpI9s!xAO8Avd zZr)jSL+wU~5&XOGUZ^?a)N?v=_aS4SQa{~#t-yK&oT9L>RTMKj#%NJx{4IKxD@@O3 zbfo%^CQcHAb2^-;9x-KJb#zt!52{(QhXIK&A|QOXHBTt7vws z6>4PO2twY)KMG{hZ-;%>;9Nx|c*y8d1{IRH(@PBmGM)^=rd#u#>cDnx5GrM4SA$Tm2A?z$ZSI92n_#+Uhb`+8G&yZ zG`)G0Fj=#YJY<0`pjFe?kmu~|5?+D&rZnUDF0o{4Anx#OOVi9_Cbv{oYR)w|d6r*- z<{@AK>EWB`Oy#X&V2NcSPX|*h`%gCy&F{N&ap?A)un(PKz zjg4OG-~OzEA5{(^W4O!{92xM+PR?&v6yEl{Fj#FoHE^1cYl&&U9T|Bc_WK;X)M6S~ zq5y+K6g#ckf-Sd^O`()Su|U4c>Li0qiJ(nd>7v#SnP^)lK1XarDDzM zOP}=t#2!CY=Sn_QrqN2(ut^b0hn5Z+VP^t9!`p zxTp1C(h|H+QT@A94ZYMkTyD3v=HO0$RFhUWIPaqB(h{?TDixf~*ER1`cmz-eKtR=os0H#Xsr*0wBo+*xu{g0XXhL|yz zG#)1hUMsDaPkbmQq+AuRr=_blXFl+V5%^T{K~LVCJ(C26qD6%vo4G>|AJKVaimG(PSHJmiPCZdI73M*IKc3}Zj~ZC7{5!?Hj$H(vTZ|Yb+D1c*>o9J$3l@-n3=X19qQLPmg?3NI!2-mEwcy z(X*0^uIj)vv5Vos5zvqmgr`H`8402;bawm+;?~?bb{5XW{yTAKo3%a_|zh zs+GI;>T#Mz{T(F-y`3q$f30fme5W6uyF650$qUX5UPQlQNcJevLH5WfjB^jR)Addl z)JLauVQMwr&7wyiufogilW6eaoP52|Nt*a6hu(53f%;F6ROnURzsx_L^79xsopqRO zzfYhgJ&Vc7%bYnH=O?uu;vn-j-bI%jd&q-7qNsw)EzY-X4u9*?LNd;(N3H8QC%m)z z$JBIPuPF4-esA(y%hI)AN@{CPuJJAosZ!0wsb1zFnPXl4?>;s zhj%q(5w8iDbGLvg?`DYc_4?AFW`((BZhxKozb-$QH2#0U3HZK|I>5o%k*2aJ8D^}SZA`b8_#)B*44Y3G(y?8j#s7~GzQT?x~l zaE`^omr6xcE+)~-G;vc6x$;6LmDKPz_Zz zX->|!kD}hRQJFoyiRo>xa^Tn}BIf2gF4nWjJ$ePD_a# z?y_R@dI~+j-F(I{`dWP5=8Vt(T)1_48)~pP27OSSu1?rvYRzDh>>M=4)SG4M=mTtR zvxy!C+tZ%jD=46NC;Ig|k3?_rh?3X1Qpemnv&SSZ_c(!Lai`dsIx%|h4O_i7cL-gS z^ZCe-*Jj-%dIMPhi}s$atHqfAw2rSPb1cXs`+QzaiB%G?AKAI9bru)-=qFB;Z6#wO zii$zG?A5oR2w+8`c^#tX9Zd}i?3pt2-8Vl~oq2bGyR^Q}>&EWqZ$X8~ae94l-d5gt zV6!#PWg#+m9_QgyQv&;G?eBUL_!)4w`(#q86?g5nhQPbrd{~@1GJda&Y$Nzqxj-sX zKf9hgp^vKg&`HNctW*b9)s_C6{{ue{=f+PeiBmaW)9nr^G$5`fQ>{&=o<%_Mn^yF< zeA+KkgsdKE$$xPR zAWt%1a*N}B;fZR)vf2E&NOuOF6V{)DzpmoSH8=C?Y27sDEl>l(^Vv-`VqZKrJ${nL zMO0Cd-LuOte1MnvjTFEq?(fxz3>NLSTgatnbtd20H>u;?>lVZBWBW9edk;?qcBIkE z^5?1DG)Eq^)P21OomE*z?OOF!Qsb`mYw6nx!F}VMG+7+0e zJU;o#9yn74^+;wN40d*0vWBNU^WcBua`4@RM&d`}9hx0F!(zC7R4YjtVJ&6R2P>ho zob~)jZ*9}`7`>_ILMw0Y7wuP$A=EhmTY2IORx3*?k@4jlqKJ{+#!q$ZT(E+Z|us-1P>nYVeStF=YvQrZ5H#oU&}rK0=pU zyP0|h)|Q{{Mq5lheRjyHhf}gzKCw5gtT=~Ty|~V0zMQ7xzbi?@F^{%aHCV{tc?z;E zL!0iNR#zwZ@b(ibqW;S;KH#cCXwhTqW^d`Qqx!t~FLTe)~- z$V|S#ff`t8>1zF_fYCIx`)b<%Ylj$FakWZ%o{xcRV%?xjLVeaj1|H(gJ+jGNw@UC*{ad+~u@e{OI%Mf~ zv)2Z9rfn=`G`tlaAYfi!>=$%)ZRfgzkn=)QT2Sw*M2;<1)HfZk`5GL5l~;70PVOPO zsr%PNhTf;Kdm>OvgXl_Y2bryTSBAa`fLGz&AAeGT&&_}pOGHH2Z?bf<6_uyqEXfu} z2pZkr)B*a_lCMj3Hk|dtT1qyKEo;5Msv@(;RCe06Q$btH>*o(CQwQy8I_N?-@ST^- zi|FO6FgE%JxMumV{zYm~@-+qgXYe;oFIEWq(vb#U+rV|2?XZq~5<#tdofPm*z>e*F zwzNO)x_eCUEd-2|n;rJk1AX3l)n~hS*EC%<4YR@7Nb!swvCML7r`5z$hATvL>SY?A zdxPcXpNd9h>#-9KmRcL`e8uKADHt-ok&vVxkC)g5xijB7P z-07r&ixU1wLDxZ3EwDg0@pF8J0WX3&?I_Zx-nHWW1=hg2;8-#0;P6E3w#&^OuWnSX zCqEl5kkGP(KBEg2ewcN&_WCp0_zoHJIlJK@%(E{yv-oS^m>Qg<33`g&&kKWA_111r zjs`Ts@4zr09_y;JxD48!;8)4`CjX)3bQHys`-P<{&xtPtFCh(=+P!m=&n`%6wk1el zJ<05C<{-(Yw-N^rm(Ux$vZT=Pi2OCP6z6d|MjMu8qk8Xc2s)d=JyO^PXlMs+)Nd@6 z9cEP+6Ic5A3HV$^2e_|g)Hqka)gQeuP^P?XMg40nvcL}s(^~_bD^jg0<5c?(^Z3k; z#_aBIMQ?H6^;NR4t zlI7%uxr=GQs?pF0ohW=%EQ4S8e!GnVkD=)DziC~HC9D3?bE$^kpcgd$9Uh^G0xxUm zb~z&cF~x3LCwi`1M1?!KVXvFW+#{b8u#~&}S5cpQ;Xv>TG^}W3YEqd?0v_g|w|r_f z`xtxw0dLQ{PGx(zOF>WbpZd{87Xm{Y(6Bj@&1+<=h=dQx5SqV0hW3=d~{d-VlnFX)W3Mkcez?gTbo z8`*;hOxq)F)c4Vb$G&E+H+WBNmhCY*n@}r^_k&*DpzPU=Mjzj;4F4KTI59RxK*KTS z%snz91KQfeF??x&$w7emG^KfjFkFZ=CwOF$(WsH^*8;zj+XIw@lSu>Z6)vK|Dhos})nMu9Gf`e`0MK9CH+vljDHH1nu@lxb&40x3}@p%Q5zOu-a&8x7h7E*D_b!*S2)H!OSe4qJH)f^ zH0Pmh+RLxL6$lv?8E-wq_nj5|FBjkIZLlq?U-q6o!oR{N@*3AI7I%Dp2VHMGhQTi~ znO3Xa%lUJ5t>P z^(;d-Y*6sUx_7`vRuh|8)@<3&XS+=SC*C#nrZ)GOO~|!m`)7x;yfS#-WHSoch_3ay z8kH*ICE*{qSb^-)bLR^5v%T(RS4-x(n_UI(EhtdSx@Tk~2`(2K=2-=LhtZGL`H{N_ zy&{diI`b}&i`0i!+r3<1T*x&7sm+-&Z18PE^-QC)1=fv#g92E|UZ+0Mvf9<5RVVOz zmjc@DZSE|rxO}I$G8tP$`kActXjA8ja;dRJoxFdNP-CgUJJM^1WxamQV@;IN4(LtMwm5^b{{?iMayphhW+)fYQCexj>+t}>C(VrcgE~lX6_F36f zs}~*kinW$Z*y>Bj2-x&(p3-^b!-|vnMd}8F*Oa~EdOmbJhK^R+?!y0Uiq}s0{)+jr^(kPiDut9Yeg+vn-nRc4>xGS1sbjbZ~RB5Mj}eJ z&GNp;9Jzd2Q$ohX8DZxwD@NEcuu7m`2^eYmP`=omZZbQX(=7)N=$R;hv&I8LYuIaW z53)a;ardlt05F?ye6({3+9blh8$KtuSgQrOq_E%H#ng!cwu!@k z57U62RTZ?8sbRwN=*s8G##5-GOPd*-C3t9tx60CxCg1BeyLMLg27HsXs(eyM@tIe} zWlaTe#5%9S+*J5=S~#vb*IO}GAn#>htTLQY^g+>7U@o$>+w3?!jq}?Lg7ztc?54a1 zZVF&(*4jsXD{4u3dl4D<8mid*D4!j0n7=tT1a~D{AHJdxaP$?Og>6wx9;Uq~t~cYD>>BY-VaZZyF;_Ip#m-W(+t_fHj1!<&m@?{Ox> zyC6Nrw+1<1%dJOtX`qJbo73L1 z;@iV)mvWEF4>)7Z*D084CcLHdHYFJI;%r}EaYeO>dyS44&7+4{Eqg9d>ckx!T)3<( zAG}TNDmI^cOztGd9!uv>#e4FbxLSDbS{kwakZ{W^r^|kvD|^XB^rBX2o!`bze&B1o zao`8B{!?YH>gFJ~7Qew4ee!6p#h-Y^Mn|gVSzMRyGgQ>=UyWb>>Z__nT%{VnhsdZ! zPeit!MP=u4-z))d?CDbB%Ce+=PL7{5TVLy)K`S;cr%lb~OS#RBA4V;q$fwy@pP$DS zLl#=j2Tsthb`0b#e9qdX>~E|0CpUTS-7vBn6|Dbk2o`Hy0_60J8$1KQKe#qH7mqK~ zAHSiplbLOaI}UuwbA9H}n>gHQo>YRvFSSwEe^ionTnDIydu~v(m;H6?TH$;(`yI~S z;j=Ym?*zUYvq8n@Qf}iyYk`&C!qh>%M=tVhVk7+40%kE- zJ5ru(+qtj|)-PK2zx<<`Q?5)(fRuvT6}hldgQ*}vfyHM4(#bBu{Wyts%CP@ zqm5i7dm~vjo3|X+;sAHdwwH3-b>R`)Ypd_oe{dwu<#)^6q&{$ax$ak2UR27lgITIowzMVHvhyZ=u+LOJTlcxzG;S7Es^y?j<7#%`2u^9C#qG0GENRa+aM=;xDd|gws&&*& z58FFJX4_pz4$VxUe=oZ@ke2RT{^ zJth5O`!MQ*{YRIRs5d^|u(Ist@5KcR7nOP3La?8Qc>B@H+Bct&pXP*cEWSULbKMcX z^Vg2L6~DlEUw&0n$UBztpuz2#BY6FAVJoJ{W#X$`DXN0&&esYADI#r+ti zWs`fIw82W#`{*U^wbYhNY{vOIjXTLvxldDI<+-Yqb0Iw~+W}qDKA4_Xj^w89Qbq=*2=b_9Jjh$D}l8V15 zwrda-+g4omTDpMf$7-?O;}p3+-pijS50rmi^x$2QQ`Mo}?u>q-zz)CZ?V1qk+G!DI z1QYj(i30BB(;m17bZyHU6!iWV@4dE)Mkk-wcN|X&!!y`p`ECAqHRHPt&i!)a&Yo^^ zL{xV6YKSvBV~S9FyGPV$<^_K8ue|$cp3TSh&(o_b=f&rVC+J--oO}Ia zok)<&Y3SNLS?lb!wj#Gu@!J33Z)(>tR{gqr5m;&?^Co&z;Z2Ii?k=f&-RvzhqJQ`w z?DLZafyoXl=H z`f}8g1}hS=|!_aD}ig;H(^SZcs@AHwo{!O{8*P z7PXGXa^d;+RA#n%vhR!U^PJxw40fB|qqFga<-f+ysC=8n{J_~=mP>A^ObvnmX~2O! z^buz1_FJ&)*jc~`OK&Qfk3 z{nC2qSO)hz8YgCjdde_cAL~10+2A>YTX_GRGFQtJs(1C5#BEAs_0NN!jkxQu(*hX9 z;00OL?+kUk5Jhc&A^*lXt#}>X-mZ_@-?$gIX!2H}H%ZK>MiZ-i6u$9uX^pBX&HjSh zxcYN1wWs`Y+TZvAeV+V`$5qDfuYYxwz+%Q)sh?hFfnS@r)tv=AsoxyY{!>{_-F{MR zj$cai+qIN4)3>~=#ISRgfOZY}`;$ME_N=s?9hgavUw$@PPA;60N=+K%kSp8Ztp93-H0n&= z*tVNj{VSnDylQL1H&tH!CNqa3PoC0epL|%4W|mTWpQ@k<3#`yChA;Tp+iME@#P+>> zX<_q{x{cRzHqHP-?`A|nyQZLQwC0m zCA;R+GY3C*w%yFJU7wi!<753o&0N)xF%1ZuO@nsLq43PAdd~Ce^gU)Xfp2MC&o~7R zBgazZmCK9coLb-?VN4vJnW*0;p5UMfwxaj^{}?(+k15iLlhRJ|hPF75=*oA_<7S6? zJ1%B4!(Dz6{qXtl2g_Ff>N43{4EVC#0^Q4%pM?w5BD?$UQqW`?bChn@+}hNFU#Hv> z`^(=`b22a+IQf@=2~_Cl8vhzA_X@M#Mz=hzbIIT>amj2NH7ULx^<}5#g}Br7)ynK2 zFdSnEp`>M%+4ODk{@!N%1nnlkcYNsaQHGWAFYb0U*aN03Ck(;*(XN;4}w{ehl z=7y_^cjof-{DY|3#zkbf@XxhU^s?U)=+2e&>C9vGeBEt=4x!L(9y&)zPI0c&DzV<~ zg(^I{6hUiI?>Ec1*!u6vXjI^&DA>qXI(hAbW;&s6&)vXLAM@*@A*)308%q6XzCBB) zjvL#Xi{4G`kAX%nr_p<=-`(2KKQExEL+P#4Ulp-7$vQf@rSTV8x;_lwJ1ALuJzsg_ z%8UC4$j097X#d|_)VxhWTGwwr8*K&s$EaO17Jkw4z7^Oi64Q`HOM zSvov5D%5Ip^}e|!^yc)+SsKb~@i^7L{33O`|52ll)dBnE>|8#lZ1K2;G zJ@1bcyPB8bbDfH5OH8U2t+MK05RuH0~f%CjxU(zXQ6MzF>?^-W>kk0{tYSr`cqA*CP&zJiO0G zg7#E_m2=3CHl=B9!dJp`rCCexJwXeIJbowm@2bcA@ZK|W$d@9nyosTD6~}4d55W(Z zy`bSg15|~vw+Mch7R`00$qnNKyoIhbYXD(gjaMV|k3^r5^<$*u;DeN}Q%>~fI{F+` z$>>ovUU^aj``kv)QMTO?%4n8r&E56&lS>HrDzIN_?6mB($2JL?{|SS4>Ae3zLj6eN zztIy}-o;=g{1``dIYsNtXS~zhv;6Uv&_q6V|F7}P{BHVo6*IItJ$P!TUFJGS_&d~H zXPWrTS;Oyw*N>9BuQ&W0ur0mFRSs5 z@8gRap0U*U{gU8$rNJuq$JOMz(^{qM{lcza+VPsVE%elR=LLGlcx!EZ4)lUFUg3yC zRpB@6xB$mX^qd3-=|2aKs3+&`G`CqRTyVxZ{4zcMxRP#NbEN?<&KvAD9)MGZlu&Jd zwdLBwAF&0$0W)~|V{iy{{+`{`20VQ;3Y`#U$+>R^w6dpq{IRIE*>RCCZ@MH>3(Vp! z>1DO?g4i#HjuA8C^I8XQ?oHqmb?X3gb^nTdyik6EMz-QPgmo0CS4-5QAg)`@NjSHf zp-|Hb9-C(7J0_Oi`X}IP1?q}>zd`;y@CP_2kxf1TZEEzoD(k-9)PVjrANSe;=~54ab(I}Q4hPVfnG(3fs%T%88O_(*7TwQm1x@OM7}zc1jk zDA%uM$}PuZ<(p4YwYr@xp*o0wttzHgG=o$4bmCEk{T1mw3Y%<1d?{QdD|><8;-L4_ zjIY$6=Gf7L7Dov^FR$&nkALsRNt>U;6Mw)NIV}qYuWvPe4SLnm=2;OP*fW;FM}%A@6`GlyynG3~fZx71lD%#|6g?VQvHvTPd;CYM z{;g$X0I4h6M3eEh;2!=`;kAJO2k%$skdyPX^mD^YO{KxMUlWR|G82nS+bWyxdr3N_6wiZ3K>pepl@Q*GOA z1Wth;S;KMD%V=I;w%yT;$)@ zcGj<>Ye5g^l+)ab$hJFf($XGb++<=bdUyxboidV;333bHz2e~0c?6wH(5wtjfxr6+ zUtS126hPl(4E_7Nn2k?r^%nW;nq1%_QseLB)zH1E4Ho5<9EqsMFhi7sOG_5z=OO5{@_kdBPlLXt8GDLl))@yhiapn zGl4I&coTePMa>epfn?`$W{i|pWh!Ts%q5`;0RkX}Wil#yp+A}mZoeSMh&|3=p!{}+AI%*kj>Xnw2L5BP& z!{8noQ{<+>GeY*MW^ofXe~xsIw}#|@$iNETU2hh(Xfl#tA1Ml7(VR`q#3f3CW>mHw zDe&yw6*!clcb74Fi85LUYatFz3nh>6n!3*1f)W{uh93~d$NgQ^mwn&eC1eI0SXjMFow0;WM6H%1km}=ald`%5Cj7W%5t=e+B9L#UmyUrr|HGO1XxUUCRbCWYu9lQfYzQy#nW4 z&Alid+)q*|2}@{P-#oJ4np~9kRZa3LkV;F&6xT1knd;md#D!Lt*6r?8l8*yZ`08TZ zGn0@{PF%Y_AbeMLs<|`=H3^L)*Jksx9(y{^BD&*$lAm|q#btYRkef4}sgP`^c-EX5 z*7YOo<^6*HEMK3;@Y9aLYI*BF8fF% zqHblLT3(+^l>LW0>H`xl<4mw`yk*)P@-CQHZ+TKnZs(&saNN?A^cye8?mq`~${TCa z=E8h-GIPtb{lt-;WyFVxr>Wo25BPs1^L?Dlzww*?rTL|FaXd|5@twv9r-RfOcLk5? zGTahXGMB!;?KDkqJcuHnec_f>BIuNzm30{lW8NHBu!{^1Tf@(Aw(0OW^Mv#EcWSI> zOP%U6QshRzJ#5U@z*W(FZpsq5vCbNCu*eFm&2wRMA9vf-nnKspR^vN~opfxjdDM4+ zvuqMtRwnk#PHqR2&G_ZMt2y;Fd_3OlM6 z25q*C?v`IZEL=tp$h**bBzzdRtM`Zp-CIF;J>GD$FeTYfR8@K*b zW>3q0UT1BS+f~+W^v1bZDU^k{G@t* zEQkl>NhaIIE*R%5Yt>>lGXK&Jv}pYzsyGZ|*j`ej)(Le(KNsiaHnwL|Tb|9JdX4MS zkHgpHsP_j1#wZ>o9ur&t3z2D6_b~QVzy8-;KMt&ge4~gq>-y-)1J2uSg#OgDGq>HW z#g%ud6m;!7GOK#B(~MK}e!?W$+<1lDb1sC-o%GV@oq9{tPknwD)K5QLr^^d`b->?T z!qnrB&Tcx^ANNq74QH&ip7*JYnS;u|^g30%)>fBnoyg^@R_CB$X(}VRHtHo#3>>TJ z-RM|0y?~mPy{mSY4FiSj_O`fQ@Uw(Qo$#E4_A;eOZ*93aChK{fnt93VX)fwg$pi`x zE6!#Ql5Ts;uO&j5Y_idV`_<+7j3}%B?+o<(2HigYN_A#zVG2G}Ll*dU*VKp(pX|iG z3u=<-39m_`7<+*=Y>wZF{1DFd!z6A@g$&a>0)lV*J1ifjSY*!%Om_ zQQztPoh?-6Q%gP;okATPN2!~qkEvcKuW^-Te=WL<73*$CrcajEzfCirCh)g;waI4J zOlj6(tk*i8a;t{v0n>Nn^P)amE`A`4+w?ted*!QFe_bLyF2wVq-1VhzJ6A?clG(2t zQQJhL*SmT6fl_=sXEz%0Y7xdWnfhOE#^^sb>vH4NSpIRi1DnT{tr(!Mz7M0!tChKQ z*#ca%atDqoIZpw<#K5gx^h}o~8koU)=J0lnVn1FNqTj!<5afC zb*_CYKm7?Bt891U_laAU%40s+E&GMNSwDSsaSvU-Y7#lGR%RsGTIjcqo33r>r{x=XLK@ zRHJKA={@;~WxLzf;%wjw){a z0#n=iN9S09brOrI3N?5-PqiJ@mT%#Vw+@p^N}EEvIQJ7p&%1le;6n$pxWM=A8$LgC zAi2$RMXfsN(e7n+`i;Tbxqe<*WXNn_Dz0d{+}BchOCSC5;UzK{`S-Vj-raW({j7M8 zG7gMj)1$x%n$^k=wUri-??W{SPNGs}YOA9^#%jX{;7_&ZoSnRPD=(SaG;@l^*Ha{Z zJGPj1i(i#?S>|-;sb>eTrw4`FaC$&7vOX(EQEe_B?D@q>{qE(0b*pD^-td+J=E@EY z=F;{K_S&rTwEMl~;3@umBK9$La*ANX)d}4Wu;JyGE5?%HoPuYD^0D*nxceeau1{~O zd53Lv(?j{CS=0YQ0{P&cfi!Q~C*p(q#8=l^Wca&oqfr{XEDFa=QD*V~ZxfR)#?_V?>^SDksKEAR9ir+zF7VC7nnz2bAof@fYS%#B=Ulos7mtLTff zLpXWjM%3|Z22b&Y=b2Pzd^K(zyIdaWxskvvl>b;=tPk_6DoYeFLfz=iVqd@z0&W1W zpKzhGiUy9$Z}bACy1XUBPhVz6sjW$0EJbnm`r_OF{O5-s=O$WHuDhccxJqAxGr`Z# z$#CUkm$f2n-4G5ux8Ka0T@ngu&-{aBLfSByJ8^>rJ;#_EzdlzL=bRrU^b|vL@R_Z< z_-gsBV(q|!{O!$n8dmxe(Sdz^rGdW;+=*!+ZX1B7=>wxpGObyCn&0 zLU~@|Jg(QjY0)(gG8_TD!6DJ%+^U9yEOb`rvklq^n?l|aT1RcVpO4RH6w$yz?Krrq zcEI;7(}G%%!Go!FHK8Y@U+2%JhPi5s4dTh`%8cIRdX0}L_f@YbeDVqgw{X>2!m=^` zGTptI%72#I$cIhxp%%j^yvQJ$S3!ciqj}@F0|uLE$(93rYhVi5m)b;U3ml+Eud8s` zN@3LedwbsH)exBHDN(O{xm~eTk0Dd^{EGP*cnIB6+{~ZMbJtyL%wxaLG1!MQ?T;E< z=Ql&j(bzr*8T%(HZ+XNG=X6phYI^dLEj6gA#|gT5rU>1wJz6KEy*8XDzqTnSQHR># zw3BT)v1d>^d|U)=n$jC}0=?4fxCQgoarbZ~=KX_|9Q9T3frp7f$oT%6E9Sz7A)*CcB}Zx?)Sh`YX=TkDF$V({E1962kT zj#RSc#iP&gfeCHYt19{Q={u9SSFd&8#{&e;vqmqm)lX8EF>Sm;;9tGpVx>kY%Pqho zZM0R!+;ZTrk~IJO&(w#7H?nn~9UmA!iyu6&(UV73;hc5c^tRW*3fj@=7rn5Gzxq)w zCnZ0)#bdtt=xwFeljqo#Tlz+{7TY=E`y81Z zt2W5V)OOSrHe7IKY^1?<2`-h`7X==`TD9e_2YXu9t?|^*k-%14r8C_y?_loQCZ3E( zDE|xhw0C(!d;IEf7gxng>vxeayk66gsi6X#EXJ)UZ?r8P{_JA*iPz=GFuZC7HiuQ{_zgZDuROASR0CvN_H~=hSCkv0u0&||Xi+j{Ko$o4`K1h|NvylL?pX$XGS>PM#0ne?U{a^5qmp25L-$(=T`LXIe^ z-z0*3-LlDcTk)=a%@1)4z zJum-?cp^@12v)}98IJ-DM33=V8nl_hdWzYt=cO9o0sU{)Wf!S1+YgL?SLjPiyLx>% zt=~@!e7XkiTTZU`RZTxuvYr^)7~Hdgus1Z>;z;Nh&J!3#ljEB4R_F3M!K1iNY4VW3 ziwb%Wwb$0*H#I+gFw4u~NOw;BdzJQ&^2p+5@QpGW-{2FxgLrwuM&fm(!PWA=oaO8D zv$00W0r$u6BC`e!T{aN>jObaZnk?7TT?Tu+5!Sh170Dlx(3^o%5;e#>yW?E@+Y8izdI!|JW#j0-(yLT5zKZq4Z2~-*V(g*9bBv~@ zwAXB@d?dUiP-|0i(1O4H_ zUtZOvxL)=7wOaLg82NP`uW$aTD%WrHkYhd13t%QSdeu`HKa4(>joUvp*l2LsXcuKX zJh)WAgNb*e-RMd=oT=2Tq6OZa-TQAq-)`m$ElTK!qIslSn`{EuL**iluzR$S6a1WI z(x#*8mhakB=o9?*!(aFQby(D&wo{l)0s39T%W`u41>00TcqR*jejahB2M^}+jozf-_fo$Ue?~n%jWfDmUTCp4HCMw)e74{M1}`gkHKYCI z2b>#vdr$~N>uKvg+;erWF2xSFGCYm(u(DK{L*mK8-^y_On99ExI*<)UL7(D|%+2Cc z(_8e?n`Fx>h4|b2W!C%KnrgES21ANZn@RA2s(t>V@<~boef!cO>-#NX)N1Vq__CbR zyWtjr%!c7{_f__GD|)}#qMb^L7#t=`Y< z4GnAQ%!4NnlEyDp&a;djw|qwaQ9Bu$pEW+W{_K=bf1Kok+HjT+3SDNbH35?e+y_6N z$LilFf>QUUu<^s-InrhA;gimA`yGR*`1DJ{-DiFlpA5LM27CNR7=KmkTPUwASA`dx z*ds=`#Vceh%J66FAr&m(JZt)heai4^2gi59ebrDayp_t^E|cF+_(s=z4U;-ER+UO! zi1j_k^&aHW7z;t85!Rle(Yef#91M?}x-8)#{LOL&d{J-3KHgNQ5^x}=(dU*KiEk}U zmrm0$r`K|y{!RG%+m%M!i!zIf>hPU=sBvjm`SbG;f+wTe;~o*XLSU`}7)O>tiy32P z=s1&i@D%$ij5SjA9)z;Vr+S@iuf|4Qv${63p^5%DWBtx$i_u8Nt9^A%5eM?E;ko)h zD>M&}eD|M#cF~&~JIU5V;!V9U>WX%G9j7VwZFR`GbgYvP!-vy{iA@E1TEbf!>@*&Z z40Zw!@tyQ|F>7usE|xD=K{x3ZKgx>}Lr=1)U3dgx^une_$0_@Ut`_6F|6MrCdFCVv zcwL_AfOBH+=Od5Ri@}+<37UwIg{amxQwZ3lbVqzoXmVYaab+cP6~1h}z|g|dtcB4- zX;pm;7Yk@;y4JNYf$R915h`6^F%x3?2=l}r&@Jww5- z(D%1R^|@UYc~{g_f)6s96#nJ{yo(2Pc0o1%Z$eg&816SUVmu`L7LQ!Go0e_`_9b29 zo;_R2FZ;7;U?v%@Sijgzqr(Mw7QffNskXEjNn2YU=0OFknYtzL0I~sl4Lk#Xmu8dY zoz7z~f@)R3-Kai#`hz`|wSRE7>3x5WnDJIcIQiqbdlhm$z0IwaGF&h=sy*)=?5X!% z4JPy6;4h4AvM^c9o9eZw#6AZZ=r>fEH7{_y5PbDt@##@%S+zxJs*(}S@Ero<6<>-E zB;+Gp?nHM1kHD*YH4K<==_S_Vrv*8I6&{r@rB1gDo9%~8;5_b7c9ih(@JzFe_fpm7 zKjr+#ju5;dO>c4#cj+?C4{u04d;DU9(|xD8SWZ6Uw__n80iBrbZ1#%EN8y z@9)I{r{z`)^j1K}N1Ihxti^&oR`B2Ic#Uq_W3>Y#N1;N#9|igknqj@o z{%{WTc}Z#N0Qv$tBah5AKCj7Dxy$qQWb~EO)!fwY;(B<1HNgHvtMMhs3Kia;@b>^@ zGdy!|xIk9L$gPaW6y;9brCbfaSxpXxb+jOZ%+gljnfh0pEm{69(89hdX>#fvo1P-$qIQ5hxCbI(OWMr0G&+4>40dz4X0L^498 zG$dO+_uMq>P0HSTm%aVY?e~Y*EA>2|=X38l=RMAIKlhSmZzq%vAx^<$kKKa25mJ8u zw@=kb)tjUKp&*}$j?SS#eORi8F{rp-dF`XK>njdcJ`kuE#GAI&k$IMNdAwgU(af-b zdh*Lq`D7nYXGqJnhiPq^VPS$VKi7E_#QR;NbF+f@qMwHHm+2bqH*0#{>Eu56d36g5 zir5QX7CzOCf7*yI*m)KW?m4p{O+MZ&v6ZLv9k^5QPYfN>l+Wp=g^y0V0VbEil3_h@ zXV;6OcW7^Xu+T1VRkRN8e7{_bn>b$dXc^0wwA+m%Oz7;8YXDDzD|nJ zoZkjre$YCV#aA^KmioOB^qgU2n#*-pqK48P0aYE zNY`V_*R_^kHHF~U;jE}h`yM;^MDo4i{rH<~XZ1R^$F2lo$p4P-(m|HZcm@wTG{xxu z?&6}Kt2oV-5IzTKKa_9i=#ve;C1Zrs97pLB_yB7ZyQB3k9Xa=fKXm^r#cgcky>jc`jy(^u8af8k8xDuRiK7{rq324%9JP@G5Gg38C38SjP7;g`pJA>NwU9sR5FzB8 zUOemj8*#<99J*U}kluBYRZa*GxP&w=98};6-){ASbxT8K=(jjF=Yh0y&F^&HRM zgl=u0!4+JxMNZA%!Y8h`!|>q*=^euD4`9EtO)S#std1>@2WAmux^ z3W;MKQ!Mb=j~{S$89gUJF{GHu_j-o;DN&oC!%w)Ph@n>#{Zy`?a4 zxS7Vab2Pke`4~dK8ABhJ3=Wg~!>dE{(PwgXZdTy|9duW)TkbE!`6r>^=;J0`Vsb_E zzcsmjdkYnNHnBk=OlmL@_qVt%Anqp){_Pge>-}HgS$YIB`7<^{w$W8k;;y*m& z#ILZXa(3okFc>`s<(!L9cw{0j@81zB#`TbdL2|(ZOF7oHC%>=ThB<6;;Jwz)58);Hch*EdM*|R*r8kf7>F^REI$?)!JKXy%JXj8xW}e)vsqnU zeC00QU;Y($@9^R?4?LiALmF_a?vr50x#w(WR7csaG(t9eGo7uUYtGHHYH{aJdK&sW zpBQ-?=sv*EZUghKV^481=E>)*z-_`1{^#msws7oUd{x^5dS!2h$wgmaL-J_G-oLGy zr^G=Pva9{aDqhL$JD*}M(Jpuy+qSx=RJC;Zt3D_G;T@Kr#-t4`(DCM4?E0ezFBAUedvoan+uvdUr50Ogxr@do9|@`}d9I&A1?-`;{~oc*p+^bXe{$ z>(6-)%MRC+ofdAuuEbNv8--mrt)*+ z?FEv?%qo{`1i2+#GxlsN_U(5#FuwUWo4-U`?Ew*rOP!AJeoKN;TV9WHpVfv4Sb!4qe6dAGLXLBDu_ zpwC09KTx%)c!uT`2M^CG`JQcM>-wKZ0$BHPsvoop<*M zhu^`h5A!5p5LP@g#2Pzl@aZ#8&~7-*f$E&rJj_Qm{z4BINq7u%ri_*wH>_Y+U?Dy! zuO=Jkw2+DH15g~%`sG*lphJen#KMF4?4js&sgQ*oh)2SD%nTXm7npAlXl-2-^f@aFjsXW-vhd+AlG!`phj!9bTKuz0f-&$#yu8h*MChnnof zxEgIGU0=l3h-U}8)3YyYzu>^(YkdYSc>vAnJg;NB24ih)rMA?o2b)AsxZEETsI(5y z&-~9@6R!UMhWZ_T9lB%uaWl+!Y+p708|NFbOMf3>p^1*@aVS8TmW{)ZF4kb2)Pvjq ziicmOCXiE+$4o1t(NFiU2##0)qyw1st|h3udMdo@QBIiRDJ%7Evh{!Z^U-HlDeWK) zcI2?agd*I0tvgS&IfGOq3QxgqR|IZ6R)*dB)s!yH5^?5*>XJBGEMIp>ah#aS?<$-G zsvjJ=zCvu-un3Os%0;Im>maw*azQ*q`fDC`@Lzz&LH~Tx?d*8TwAbjr=_bv$J8v97 zd+*h;2f`(GD&{`%zbEc^R@3Ld`QLoHA8G_?JA&2gLFs~>4V&)1umc}0^rM=yR5h-&l!h>u zzp2g8QN9)ZJK4(_=KYYknFYQ&Ojv6H`!23i`UaYv=|OdWPD`94Pd9tZtWLVX8b4P) z^Mfl8S0lX-(igPR42g)6gjHzS;)_fm|3i#;6A3qb|Ip|<)27A)?9kdD?Xobt-Ds3|Cv$;HWa*=3Je`vd&l^uw4^s#!>I?V0+OEdH+ zoteAi4>mFBt|1=AfqFl*q{l(kh{e=GmM0tXowQG_lTUw@pP=G|R<-2ceI@Yf(he=@ z4`JGOH$ELY6}q;yBAsZUp_ppxCzx~6>x2!jMP9HG9Gv<=)f?L|SIfM-wgP>g(x2#j zbCd|5vQB9Z-eHV0A5@Sf{KaDV*#0W%=;}z^%X8zq;)=`Zxb9Cj1|L0*jjl8Y(*80t zsTrOtbi~$)#!_K7;iA-C{Z>PGj5ULD#g~F0HrDk#5dKQSX;i$}Whf_J{00fh`6^HN zE&p14+oavXy1JbtZq{0^TwSG+NZ(4*mPovVbzi1qX`NI-`WB3W4Ai`Vu|>4PSfufC z6>q|2RJhp9(waHmG^_d@)eR%xAhVO^l4fd#dA{Qm_T_)x8bjv{o+Mlu0CQF>SNVab z^tV;j!ph6NL5=y>`pXI{313eGc@P$N`VAZM`y@7=CV+G%Qao`udCa%jXMpm7u)HR| zy5s~eE&l_GGuqM4An!u|K%PuhJcQmej^gw3YqWMx`V4jo#U(%Y%6CrstSXjnPl#*g^unBqcJTEUJx5lV#6s^*LRHU7 z8>S&&4|KK-;Ut8-UCb!2k$6+-9^R%j18%ubAZCFn-il7CNcRLv zn~-j&y0oKxN#;S*fTyskXB?yPilBmdKsp}=Jjz6Q#D>neXoV+Zye0W+H80xCvSfvY z8q!y+biW~Zl`g?GyE{wrHn?lFt$f{bjM!>6PLMw2JI;-V-@SAh|W5Bpf!gp(5RD4MO zh3bTQ6CQhD?pS@ks;4n&r0R0OxHmA~;4>)hQ=j`c>^qR=lnM)TFXpo!)~}HCtGqTN z5vlgH10P*gy$2A#3Si5-y6XM-ZQyD#P;V}KC`^RHPie7%gfnEn= zdwtCB_efV^gW@AO>aBsy3}40+x0BWe@}>N)t}$XDMk;G34z(q#g$5 z1xJ(j?4^7=Y3{>9+)PsVAyjRsIwxO+q%)NM(2!1GXVb0$VSzTSeJ98)v*%i$7*txt zt+R(De1=B`-XI*_k81$MCQT*r8}%tpt1)8tN^4HF zK)sB|*eGWOC@n=eD&xoh@FD)99ub|D_PY_+8*NUSYz{OD0His9^|$v5(q5pv7HKP} zeDH(@*pbbs&#cFjS_<* zI4F-eg_HinnAe?fPy5%XynFSiI#oJs_|^4DEba44hXlo>LWb*LFa1pqh~2~+d#;~ z0;oS=2daJ+zFGuc$D zLh**undIT2PDKy~RK5VxmF*thKceoJFP)tPyLAGk(lJ&c#!O)V`87tG0w#`X4a7;z zZr69X)2KEU9(f6>&qMVtGPGAP%fd{qS?WN#bu=SPhNo@2h?_<>*lONpP@3u6z}j-K zW*zBFS_j~sq%=Jk>V$#Ma-kyi# zRnkD!pz>ejciH^LW8vV*6|A0PJQ7C(^)0Bs(h-jM)&=U-;O~7~h4=oYjCxJfFA7(> zQB2?XM)l^9dPG3ngZs10rRr7C=Yd(@3rIML^BX@={SK%|m7?b*8|sH##!an}(VVGT z75z`$%P0RKDPD4HdNg5gwx-p{FXBN;vA8ighQ1@xN7D6d?TGElcL<8P#9!lqbd!eg zlqo!+YY6H?aN{yAY!~aI!RW`>Z=9bRhak@)loreXwGmW5fbdchjxo}EVuMWzl5b>X z?;9fZ@KE(JmHw^u9i;NG<$>)P=@V4`UiE4^HX4XuBVr|K3G6&H5s51aTbq!ED}{M0 z$^>C9@v^QQsu?XvXH@m_sMn{N_2H*zUf)yk8dLq$4*tIxc^*3Bdp}DLT7%Y2Izja7 zEm}3Nbbs(YVwl1bEqP|v^6x>7cj9nVJTb;K6$m?F;~)XVA!^=5+-`du(CLKG*Kvk` z+N-PhlRP9U{X#sA;cN1MUPt}GT-aYL2c=bO>f^+zK9Hv1Q!OAI7sPK!xWcISMOrBx z-}cx?{i6+_anDme1l>9xfD7B4;7ACt7i;y9G#w|8i_|w_)YIdc;nmo?(1nEe-{D?L zJ@TBLCH2}gst-c?lNpDnf~skH9jAJ1JMH=!R$DurQ+=VT(I;YrBn>B`J~n2g6I3qX z>ZbRRdb2RXlyu?L9dOfV6H;H48R}Gksv`%V>e?cYKs52bDc-#j%9jA)3e4_i&Q-tV zcmh3(KHHk~@iZV0Ey$BlZ^BkmzX(rvHdFnGeAR1iN9%d12i0zF<^tBD@Hm#o9l%FJ zliA1AXF%9n)i*lPESY+$ldJfgOP87$D3i?FWJ z!O8;vsJGWm(- z$II?e(WVh5YubVBk2PY%_@gj-i8tTy@DmvJoQNB~o`Oq9U1h@P-q7><3G7fwYkbPg zxjTJ_*L$%U_s$v0TYFiHb|Vb=pqzcQZ*G*h7dH}J&y5DxDURIszsBK_*pA2Udde0ZpCX>UNM&h@GO^ngYu=#FQ1u>oy&@X}p0?pH`+SAv zMjc>YKn$C0e@y5kj6(O=CH%!BUwPLt8-h;n!>d`(q2RKfJg3`-hyT7#=N%eju3IQK zT|7}dk_R!u?2|Y?U^SEu^c0u-*1-NPqO~qPK~6kpM6Vx!33KYhfk!(szj{iZ%_ z&aMw}dq1(*OEH*cw4TkZ)|SunJ_fHyEA;)|A42;V;H=uG#7NybGN-s68k|!5gx8kx z_6ygMm%=z?+Ts>Ths#R-qg|=%mPOttVss>f?)V8(`p}wJ>ny zWQ=hb&e)Bf{M+X^96r@X-Zy^%&7N;$6KsRwXXI`+;?4yepXCD8xVsFUv<@4mgu$BO zJGJ`0x7g#0snBhAB^t->74~Bu;Y$O3sBx_kx_(`WM{Yk8IvIy(uYextlC>CZMjVEd z0blT4N;ZsdQ%7FS8V{vY=$WMLv^OcuO~$5XeBd@Njc45w=aaM8u78mjzO9b@v~L1S zjjE;XaiYIG<3Q)b>A3P|!CGb$Zos=gG?bm|-l2P{z~SF?v2K13IeMd!7`0*({ROlD;w!6Mf81Esm-S6KNDoq zPIDY`j5A&9moU8JN6n7Qt$3??)|gYg1vf0+4+f_mifosTP*m!{Z!AvbgV+7Q#=gz@ z&&WnHrh6nr{TITWo104Okc}dF^L$S8CY=n}#SP_@jE|!0mN78@>N-q#dyJio z*vhD0<@h%1aDvtb?Cu4D>%jB)FzG$~H@6zR4jaPedAH@O`FY80g_*2f(HH-` zovpoXnaqUc89{R@Sy2R|+yRdr+ab}u7U1T~Fy*=pxIXj}GYmKL&M7Bg!tRs8r92I? z@?wEgI#qrX z|B^Zhg&&gw>(IIaTQ0q_pz&ZGIn;Zs?0LO2Z{%+(>m1nz>OOT69>THZ9N1ZvM0 zTB=#u?1~>yp25?1R9iE~$s_Z3RpF>hVLyDEZj63w`{TX*<(#mXcRp$@Prmb@xGj@3 zCcb0n798w(6nfJdRloFfzTikOtY}#s4-dQ#%Rfy)tFufKc(W4dzDRFJvbCWJ^19C- z7+Ms7yGESWZuYLl{M&H4PgD8rzD5QI?*Z%HB{b$pZa-!;%x_f&DFgBd``1ciD=Ua; zy9Ak=1KO*em3Uo-GIVd7Mw7V|A}iY4-Xu{3A6CT z>32YQg1XI_q*~|ja79lTg5lWkB7U9G$RWbRC|W}SNoRFnwK@D zI1+`Lew{UIA2-Dp=ikF@&zk(aizV+lavu_J$ZIX%i%Y(iU}3UYZce;}^F;xt{KuCI zlG&IV1uQcB6x95B2njRu2}eYw){RB|_Y_lq)Ym4IKLv%^FTPnyXKMtb--%i^W?k!v zu+{Q2t87_J_a2Jy$CSUA>LxF=U5+*U1=Rta1+}b%G(aA4&VQ|DAyjFJ6L)nHP+n zO#2zNABU#dCM^BkJ#2QRgelIanr9!+CCgTGhr+3=zrp0^L%0*!M>dqfF%-;QkY&{KpXLi~CA75Z*A!zk08 zKpId|EZK+!MsiZaH;{a?K5;^974{_eiN$uFV~Fc=z%w@cwRZ)kz-dJP4e-4~Mv4m#hB#Wb+c|L631}BKXBxSzA91tsa;Vo*!lQYgQuF zj(RU%qJ`0IzUNPGN#g~=ZJ8Ps3sk@O$nz|UBYLvgupH3zJAn#E9`9TT zYX=`!8j?l(BtV5zU(#HSNPk(u)JwD{Rryoq-_uNzKII?S3TRjAC`T-a7FP>>NOQSk zSmZjGt^X7uxW8ELWX?`qU4YjE`;eYF1vcR;`I*Z55U95a=LOWPx-RJ*RQKeS@5pa_ z_94Bt0)xeC%&9*D#+hEEJ(k`0;%B))Tm%Z6G95gCcv9s(e7|{9M znnk|4%L1nD5TteJeJz<&hJpMzP+yR4!%J>Ota|Yw=%4DOatcR(xd1NZjiurjHFv2$ zcH;N3#c=p*J?voMBnjI^!rQiLPSL@_6!)b+6AF_qdZdyzwm_rld&TlkZ;dd^xZNK-4{fvM{S92ZS+jD}Dop zp1DqV87X>pAI8XIAscf98y2s`;{$37x(6VxL+jptc%Ww>#uV7fS8uN1pSVLX>4XdK zx9|@6r8iI~dJbMKUrqd7T^Q?>(&UDST3X0KWCVs^&-G6PP-u!6W;xLU}6E z$o$7&d)B6DDQWgvyt&ybq}ov!Bwv;N(I}5ZUInp{k33^=FrT!J(!sF!*D_dP+e+RE zT?p%kO;!4e|4G%C>lRiQjtzCl8(xCTjUS+h9w8k!Sy8{rSnR_*7Tf$1yEXQ`82)W3 z`Ns2v|9XUj{xI#@6XNx;XwbKZ()Wb(he3G?9XXQWbG+p<%!FO~fMDF1ANO}P*8eL4@t36`NC*^ zRjqQ;{i2Sm7c4k(7Xqd^3B_#}Hl9JkEW(K9?7-VvNSc^Yz2c3UaX|b{>pt7V2d2-- zC-OBfW0Y1y(se+(1&FgnNXjOr{C8ZhR-EozX)pXn=baIURAE5Sfjd>4+WT`8^4_?POLF}4bDnxNzb_GNz&bl*Ku*!WHg1czBy9`6F1n=OTgoJz zLA-qVa$1+0O`0l1)a~pA$+?#V=~IrH+LH7R>1PLNU)>ScecTMBj~L~U8V4tRuO*Bj z&C77jwYK0gIvxq1@L@YQ?i*IAQTR@up&j#KGI&N#0d@W-F9?+c$o0d6Z)F$f!ooZ%F}7cZ;|(H2owWG zyp36TwNP<0<+1qO`U8^wf`Chpk>ZM^BlyrU(U^R4r_#|FZr+qrU23dvxyut1T{-zO zPC9~eZ!28w*${1BImoitS(rAl9IG8HMvABMH9RS{11J6!`}`X)!gzW4l$|8cCARm- z)R0!zE*g~qpIcAEA)Q=sbIoyZYSn5~T6jwLT#>OU6W`8zfs_~QPqiKplhGB4Q$&Xa zNAd_qmHt7hMNXVAm6kA|eUs1s3gvTl7Qo3H27z@Z7H2g8VHUe(28#1LVH+ zaaz(+xMp~LPWMilE>86xa8ko5H0}raI;|amv>=^}(;HsbsmpIiA0+P(!${vKZwOVf zU09umecyvwx(k5f%?od>XKKEdeJIHP8gxjcUYw7!J!7bM5v8Tx0wWDcx_G@$Ye!mp zJm{nkc`P=fa);6s%HMFiPwqV8^fI4oSF%LE%%$YZtYyT3T&VYiv7e78AbB%fclQ*& z4!bEX?A=<$C8Q~^e3~ax-m^KeZ+yr{Ak8%k>bI$?ein|8VQa1(6XgB0^*a5-w|2Yf zeMX#k7wCGLtL3%%R@c*lbeyovyv6Dbxu!G|s(YVF-?!2{Na9k;lUS&^bX!$^W$kK* z6-IxxeJrE3)W2ZW+x7s;W&XFR2@u~P;i|U7_{XXj#h&zY1Nxbf2W85eIvMl?($<2! zx28qSbn1y@Y02jS`Nk^lzOZB#X@XL*V)tg|XqQ9XNwNDI|P8!Nl zMp{8sr?ute4cO443+jG2&AZ|Osd$9O2k{=iU{jMV*e7}ikUxRVHJ@peW{JCLOMF_7 zdMe+<_1pO(Vb0E~K2G-z7r>9sR-s-UqyCZVUyvpU5xqaBLw-G1csag1M!pKJ(k1^I z?1gk&JtSNagaM#|N_Kn4a@6%Rkv(VW;ybrQM)xA9kH%N5IYl)!NK|%7gr#4cRZU@0 zw=-~fsU8OWG*r4=8{$??izd%$CH-m^_T7b(zrqQ&cd3?WU*iAr1of8qg7djr`b_F! zhAFH@l~YRR5I&;vq{Lmq$-%Ko&r?mX1&@{~&7yEzmUMb9#@MZ^(m3>agqztgG9pAFLQWJ>#IK!-`dydr0$zJs_mw}_F4#hPDR1NEmcaotWW;klrG z5{)5BL;uHCX-kN=qw(!pEjVSYWGy~l6F*H%__s-oVA{*0K>Agj+qIfm9hyk}!yC%W z$)N#9;iBCY;4jD*VB-8sP{;MK;vQT&`Iy2v;r_Y=LhtVZ-(+j{>9#SB z-WUxog|w#``Iw0vQqbfltpgfuiaR>{Ye;K}q7il)=hfF#4~y1)Hp4xCE|IP|gyg%K zZ=DuCbvqySpRqvyfS z59aXKoPpv8!f^0zcLK6@<$=jo2S(f~NpmVq3e8JYe?5?AmB<+LetQ#WrO-1LF&Or4qCT<@qjeggad5ODlRu1if*r7Ex@&MI4 z(JmZcQ|kJyp?NldbrV`C%uun%J~IPEw=Vm{iu@pK9R8J^owgEOUT(wDyN*J_PczJ0 zYlx|L#`6PH-+}d%El{xg5_sSHigVXD!ch&!;K;V)I2O5Z$4>^b!M+T*)1?(GGl=J} z&qu?bs9Vfo@hFXU;xh5@ODY>S^FP^tt)3jwb0YuLXa4PSKh6fy)!RyD4{>4HkDJN|vjgSk>aN;V z=8G|?p%XeTG~!_?<8b#CA31Bxa&WVi(63G_ezJ`@Pdw{|2WFLMieGHzhdWPT2Zv0S zb&be7H2$f^N^7KkiZ8ZZxLX6#k;jSz-7{p(S|TsBq5b{oIjDPk+~kai2iPNhD_pWl z6KRJYV$Oj8zQehz%+oiJy61k0vGew8i(ZBCNjBfbA8QY3bd}Sy9y3MzcP6r+&Nm!a zIuHgIrJ}pZTJ91MCcplE!{+H+Wah0GV2^Dlg^$-s%wd;s;Hzf5*JlfPdrUOY?{UC_ zrSkDb2l>g{5gY7ls;jYlN1A1Bl4uSMOZC)zt$o! z4Q)Z=*WsF-@6gk2FHU{G1=Dxj!#Lm5(4buwp6H$kL0iM|VtrZ@L30-0ytAlw{1VRU z6GVFgmg2!TMewyoCu!-h3lgHt*~MK8xp9mEr|Z)7|FRp^#^KIoMF?|Why_72Si;Y> zxRlP|8f$DQRo*|{L;JXfIB>nBZfv9Xa9Qy(4Oga>gM~+9d`;&Wn$dHkE6y*MuuS6a zKoeO={_jczosImjBY(DjirAO#gfz$e>->)5OS~T+x_KMy#ku^!;xue%ca?4J|3fVM z@>r9%@*q54)n5Mlt^>zSoyFDn+w-!X{$rk5OYnU6A<*{hCzeuovuN@1A&$?nXyCj{rWSX1}z8hz+U;78C{*e zx1G+cX8Uu?T6A90{C_wrs3oZQdk3G#`o(ULe5WBg2jmEdwUGZlxDp23!KLfW_?8I^ zXy1w)a0#fOcsz&xi-y43_zwJQ)JE9+vp-P1Gp9)_;BX1Uj$OTA^6DM<=#ew$|7mqm@5>nZi*^Hl}7!Z%8!lQvmBu!q?BVUidyd7z*i$5w^6 z;plEU|K`OI>>1rjwx~Rc#?|iO`*|rflVFp{U_qEF8?E|*bC*W&;9r}-@U{j1-g(fc*W(FoE6)d@Sx`16RBGT_ zze+eW@35$=)14oy<0YOY7s*+dn*+s5yjWNVUKp&0?_m}^b4iYHSXUc^a-2Eg3o5Mo zowbU)#|xbQwmW`K?JC=Z_0A6u3D!JPp@uOprP1`EOw(WB;naGgW@nVl7UpTJr5?DLw~m-TtQ{lGIew1PopO~A~5u}nV@U4?g| zEDH!LZ@3j#HOiKY^R0ME=|++HZz;%} zMik2-=oNKUP=3kE>kT;Nt+wCR1F-w_Si;OHI4AQj{0Lr)wwm^E*CR(`^~H|&u&;+d zg3V;qsKbJAL=Z;fh9097?h)VBl+NWX;Mb8ha!%$Vey6uSC(Kjzs5lSPGw8XrC@Z?R zO8n7ZgZ1`|=Z!!0z+ua`2#SpeykIPsU%X3oS*Qtr(uyz6-w5ZgF2gAi_Gr`Fm!CLF zOZJOD;<>xIxOP)rS?i!NZtXu8&JBxVr#6QOiV+sSu;ok50TW3t6LvuDrnS+{?1fO* zrJTXDKUN`e7Ga_dIZbF7}-6X!ZK!H{Wm*1eNAC|+`}p^?Pp zta!K${^wsPMzvXk+p|u?zq#&$at!j;u3~fzR#Iw(-=D98k^!CM%h&nZ6~21>qH9x5 z*M=8=K@J*p4TuXt&()d6eGcw5(UGR1`!H}&yuttx9_pzndR2iV1{8q)4?h_^(H9bC zJf|3#@hR1|%T6v)q#xp8M5#IEbl#o6xweihb1DGJ5x6$&w1`OjC20O}-?r{D?c{3M z=zWjYLw+Yr9fcohzvdGC~6`;7J-);j=SVrqbB7{X#I^*V6Ji0cTjE6hFU=(vk z7>mRg@bCK)q3TQL?j9I0Gn+UjK@esOo14XWS5uuQH5kgy_uGflS3d^gI~Z%c3Lg)4 zqBv|8s-{SDp@TuJ+%h)*^*?= z6fR6?(-*fqzXA#KQQapZpp?VD?JVQ&%3zDuU-xRmTa?3jM=C@<` zd2ce7Ikjhv!)JryGNlo_d4>z^!@-hpRjX?0#?pMaQSF(yGCmyF+7{7!*Fmqz=GdoG zZAto0@dKy#%A2!S!-iUOn6yWJK$DpL>wOHf51#JB`fa1%DJuaLyG#dHSqYqX) z0p?DC&2esYUbhujoHV1>F)YjMz*Jq4UKBKk=vUKCzG)c@3Uf{zw}9PO@&)NXywKiF zguh*Z;cxXxllsEb{4GM`+)2ClQe$v!UWlZxsxbWEWGm?(*b9@~_5<%mX-bOR_o-tKg=k72(_vU@8BNU8n~|J_?8jv3xZ zniulAO*q9KN=&<>bf#?DM1fmvg(3#vP%6=Bu7QsTP%nn_6)1nF`(_}Q9~CIHgCoN8VSDw)St z?%NB63(mli+&qZiy9t$MC#|oz3$g>RY6tgB!;U9cDU48BnG?1G=@0z7teR|k{zO$R z*(Li(@mGu4*f3XA9z{VDiDQ#O9x+4Ue^BI8q%(bkUz5}miDQ~{>0@%=l}lr z>`$=gzudfl>H%pSReAHX@G;PJxw=l5sp;5F>&R(Nr2nOz8sZ2f%$2D_8IVo}!ZR4( z@h>a-*j2LH$632>XOJ|j!e8wC-$I;}(twl3g}o;Ok+@sxmR=>k)WfFdJHw5oK0Nel z5k@^QL|0mC*LN4RI@=0dQ`@8RH>A%b=|mBxmbf$eBQXDh7zJ_}Wd0Y;)};KpX+2$AGkp=vPl)zTMYK+|@)f z;zjx{&WYEQ8q)N$m9K}IV9LoG0bvR%pW3Z+isC$_H5I=dqxs>cPxe-EMaRGx5wZO$ z(Dv(c!@Xl{T5HLI>Yb$=9>o=(7|hRd8Po^RI{9)uNMUq=9zU*&1?6=iJOS}xKZi2Rc zW?yR*i*2V^NsM|vK-dkZjcZfAyw~==^pvo+81$>v6XfBTSDcIL)5yM!TO;A1@MQ*g z+%!d91D>|D1=1E8!f?gGOzG=&D<;u?pBk1*U*o?w7{pI6e}rQr^&v9+7?Kyn4Z*8P zt3dw#C1F6Z1G*nMxMyRf`;~9PG(3vbSHRg_-0|h7cpx1k=$^>CM}c8TB7klJ`m&WU z{!9k(cea*12l=yM82rnI+dSHYGa^Qcn>uSyvoncf1mSE%Q^+UC}>QPc3iK(BdXU3Ek-{Z1M`6IESnED$_J@QoDP`se^rt7FUZS}x&OmTq9 zJ<{)3Q17BBE(<08JVrXx7dzD*A_!v`@hS1bU+PJe;hKHZwA3G^JgCc6{u%iLS9zv- zmEZ3%srZ7tl2m-&O}ibImwV#iqK5L(<#0j$JfXB4c|q;am6epMakTgGHy~fco~?fa z)Td-E!eSLaD(?tO?Gi}G*W%yr2_)ao$h)!CBb;&m?DP2Eb{YKM>B&i(!rZ0bsaO1* zIB^w{hthsEu~9fBR;CR?*VF=_ek$!X+gEWv;Y10#?zJQhWx#d&rK=vKHhah(>T{UF zqKmfp#rC8it30D5wCm&(hNYpAE*}w&kOqpLnJBh7}?^Q@|uHkCg>w_g*u4oR@e{2y`kc1Rs6#Q0%4g6**4)Dz1O zo}L6VzqJP9ZBG4A?cIYnnaAE6u-&UW&NE6ytHs-hTh}th=O>QSnSs=&P`xYtdDE1) zr#U|e)WcO?3F@EM<)x==xbh>^Q%2$tAm4+^BOAx8)PSzJ^0L~Rr`A#~{G#!9Q9O>M zjoBU10;b9KY96&plfDQSsCsskAF9`bR3lUi)39S;O-cEWPNViC;g`~LlI8%P=GWt0 zAMX}pe+}c_&o4mTm>#0|ehHF3pjYb`pAzen%*{xsLnK4adYRNMGZ=Qq9 zd(%N2a+|~dcpd~#*@1uR6_24MdNO&|6^sjhgu8krKx#=fUikYbey)rXPYdmN?{}^E zWQPVk#laap(oW&ofGu#VvIb1rb3lA|pMtf#GWZFb-qJF4C_cYkjgJUP!mJS&VbGQF ze6FJnTKs#8Z)Vb-z#D4gqiVi_K9f%@^M+Rzzaer?dzMq~3V-x+G#%FX!8vaexuy9y zyn1zrY`FUh?98;6Enf`=%;@zlwi>A-uP|REZoxk87uDmoHebei_Hw<`L$Pl@qBR%_FK48E-UTE zi`_%Dr@9ql`0Q>xcx|q>Z)HunR9wbqS!?moj-Qxl7%K;+d=_CtXUSyUi&*E_Tx^JE z(PYDFAIp|EXiuvzcr$yYG%+;ema(J3riMEokyRIZAJ5LGr9M9I{JU}dKx+qz%w^+< zJe+r1pTB>@`P>EZGU0tAPV)>)tcQxgu{(XXUL7h!cD@!z{@sU#JASjCEeGOi_Yc_p z-($wF4FEO1Q`bTId9T;%&6t7tWrv_R?-isSU5N2?x;(`MUtgZAHT@VV+s{6yowcS2 zwhq66wF7oSPtjN2%r=nf{#O}QLtp#m>_+AmIDG6Ad%i)Bd)-KrE`<$2XT6m+a9tA? z@Wzj4Uh9MEzC#0-Vi5X^xwK~eY@vg=_xHJ)7wy}BchRq61KN#S3;Pmm!KC*Zs5jqU z-hCb_?TuCd-Jhr!-9q;1m59a*Xq`@(CoZp%fweN+WT%^3H3lO}ao;>=S?`0Bwz3B8 z!AW^3ZaCwh&adgct>sDdLxXGO&?#XX_Od$)rD+}I+z=~x=4~wdwT%Y-T~F}(E(C!>=bm*m^ac7 zcC9y)4@dOme={0r>c1QfWp*BNXHg>B2S#ZrUvT)mhPc7ojF;DF%)1Q>m3v~w$r7$(k?Hxt+652N3LI&y5?_h2z|BL5KITEzp( zY2C@zF86V8@DpfU;0@zv_2aoA8~FIRp1iLUoi{h*Et|Jtm88$6{J#VrKP0hQW192d zSj0v;uMkuN{8y`!@Vq#hEqACdA9U@9Lq~D`VUCvkTs@UDklFP*jcXT9`Cu2-q&%1T zbXvp%9U8(w_v!M!)kADKd>H@G{Yblt(fe>krUMR(I12x~oS@lweR;FjTKr}?19sRQ z!XY;Ocu=eJIH%lQHpK2g_aJsBgF|L=M*5tQhrN$k2L?FeYY15S^Q<^H!O#Ugst+_brdv* zg5m=Eog;m=`tytNvh^w^8G4ryRMY#Lb2>4Y1{Q;#BK zGc@(K!g;Asc{=oZM|u|aA)Joz zW-jSoLBT#5!= zZzJI%@2ecMG7_}Ox`&H!exq+Fv`x0w=L0^l{I#2x7l{Zr-jL&duOS#Kk%57E^CqFMX z!uO?}#r_3xviIEPa=T>m()7=`V^2#}H<;^fDrvrsSjP*B-4!vVjh#H5-j+Wp&k&)3 zx-zo6kI(1q72urrfvxM&5~e!EbIL&+cl9oWj|gOxc z=br@e6sMe4aTkhn6(=cdp!{V2=4~Tf{U1kH8CKO2MNw1`5ye0R1G@`B<9$AndtzqqRrj2kk`(r4J6B99!oW)+ zm#18^&}7yn9W};@ntaTrTVM5&#cs`{Yk1Eh&La%D*IP>w+k^Z#rU>?c4Hr9a=D--_ zQwKiaoz6?Q)$%_dMQNK`^e{(td1`V34boD_9hpysubidOtKJ-U#8E@XP`5sn#r^VMOWya!}tGoft-c(hZ+5OU}O|MA_GZX3o1~>RBK9 zJ(;RL`NKgiYgr=RxdR{VO|3KLN;ug^3-F7|HqAi-FU0B0B~&N25bh_08|7Nd+x=J2 zP`f2Y*XThD>++hut9eihs~UQEsl{)xgN}H&LbP}`Sb>v8bn_$N={|hbJteKRt(1dW zSonOGDFj}niC*hfNVBtiYu_*NH@Xt79N(7WJaWiGqo&IV#j0rBujS#!I`rEujXoh` z_ZyH?Ufq<;58SL`!rO9u|79~apCA2q2zgIgLBJ-ep7unnUbxRP>gi|&ETP;{$JB>K zYv`l5i$*UA@R;f!nFV4Cx<8j!#|U^W>Te{5R)bC&56zHAMy7}IyftGqIGwIes)$-orlMQ02PnAhcZP*e zwoE3Bg~Yh>nZ-Hf?keA~mvbxX|Kh6J-|Vy~tlHBjyG3-eWi{!2csZ*@iTXm}Ax85W z%?aGL*AHKN>5o$i(NCN96w(ZP>A4J&-mRCg(E>)hK@&5uie|?yV)I#GL275$%T)Pe zcD-+25H@Cbs$w>mC!e@%8u*Ow`MMKy8JU`0Jk?1;yK=v@M}&T$W>uz;<@`^EhNMti zCDwhIofdpxw<=laC!Uciqx)5RmZdXLHw@fTHa8Yi!GHZy!NWZ4!yEzaO}HL8a^FAI zV`_eLeFAu;45!RbeMPY`@eF;!+a~Td$F@$0-mbb&UMHZxWX%QV_F3IMWk!pAs(<$` z1l+W|{@0C$#TL-ubD0<$&HoOqV7Jyq7@CYwr*h`Hjy%~TTz4q+U6>v>TKI6}H;!9u z$4SA*)sxIPt~4Nm2W~h@M)!5L>0)q$17l;e^zfORp&ZN@o7d^<`TNqYeQ+DEzUn1Uyxwc@O@LpKue!;G zeX7ve0W}l4cGuuNrkm-80?j`a4%ywpV?f)T1?fw|3DDJbklt z*v6;th^pVSlJ%Cr{Hx6Vx$$0qeSzaD%zGNfjytBlp;Ix-vgQd`DS^uxW5MBz%F^Dk z4;f=ajSiL83)lM-JcwH4S%*4}yKTk)gL9&Ij7vW`(Z4vE`uetJgwf`{&|vKSZZ(}w zayI&hd;&s6F6$k0Obv}gCp`C1&zp9#@1JozuE&Kek9OUADSe)nO;-IA%#Zq%)8_Yo z&JQqpSoHd6qZ0e)!ROZt@RjN_`a7*$aEgET+OIad^r6}P9m#k*qpe&w?GcyN;l_ly_e@08bRht>}P6Hv@Ft18P7Lu{}+6ha6ZeI zW%i%LIlBJH!p@!c4xDKhBW!%;qZb^c&5c4b{CXliJg|d3{9YTpvYNV_-G9EQm;D5R z2PF86;OiOwlJ-7(!ae#G(cE{Hf)3$UHLg;-fb4SZlpuod2&kRhnJ z#j^?ADvI{S9yczJ1vEFuT&p7Ct7&ufTMS+yT({BAWa&DAc4RJ9;3=Meu^(4zvp@mE zsC%w8Z1fB?1_Rq9=8%DXeBsY1`WwB44R5SDxLM_Nc`n{>+QtDnD|3OMcUhc_T2yUn z@8Ez|71{M*vhhkP_;sopxWSLW2Uhq-c024QQ?DFnaFPtlQCmUNptf5G)Tj)e=uPEv zcvH^5Z^_%cI2k?LaPtDK5g>1Iw(;v4KH8?2Ho5J{=I{ot@;}|JQR{aD%iB<5T z_<2JHzEhtrGmL*Rm_@w?(P~=@@(Ge}dmg5(Zfp%KnO}Of+Jb)UXmT>yvrQh$jr5tm-~IZL z$>xBG7L&bXOSk8RHBSM9XEU%Lm@{6RoUozSM$RlbPqn_fpYa(1EsIU$`!ME?!3E@) zxr&jiafO%?S(rBQ&>Cxcw|gq^!(2A~kD4TSBnd6!!+nc0G$H&~Xcn%8r~nR|v=#a_ zg`is*JVO`P4iLt#;6Heq9Rj>gTeCOjS+5QYqo=Z^M-nnRE9SxQKYwbSPl~xOxcuAA zG%s=z!&C9?t{)WcMI4xBt8=?GlI`MV@`7<)3A)zkiLCtaUoJNS-_qU0Zv1uBRB5eU zg(rHIC37725A#Ljhe&+i@>@U?i83|M8Sbz`pHZZb9TiKT&G7fg=$sfePeyMI?p>Rn zPMyalV}rjW_%Y*2ti~t817a`mzY4y^7uRb=|10<{;NKpB`=Z3HxlK-J?L7ag$x|&` ztk@H3%m|9kI6x70h4l$fFQW$;UWTBEtmlpAztZ(BZZgWI6CssL5=gjn5c=N6;Jlca&J7db}?v*B&m6O!_?Z-z*W>-$|mD30{=8W1T+uHi`TP zetIDpZZjUgYxAQtexsExk3Y{wHzGqaSuEc^|CyiKMyJ6S$L+30dF+v+{6tumZ-aCE4_MrW!AY3szpJmcnA`Q7V3Jz{s1YSk@PnxD5F z^M=}7{G}?D_SCXr7p>aa>4evL>Fk3o^wvF(wmI2MJ$(C!%J(fVbAS3n4T{fTN1xte z-H;75DX<1V>{v<+@K~Yttw@uP#tI(N|2O9t|CDNcEy`shr_;x&kuqP6b-Zat2v;w+ zmTu4T<;3ONR7{1kGAUmK)i@DDY4?f|u9IerPNf=Ax0Rn~5Esq} z3;wtzy!-fR?-@PliRTGExI2eTY@LT~9k;6c{vWC2TC5MW=`v-e{pDuAunrflwF%8t zyCUx3+I;!Qwxb;F>&y|)oVm-jE^=zM4jftUz9=zhydF>OWtl5$`S@WEo$S+E2Bl=9 z7qLfqbJ||kWv6o?&lq+{nWwju9!>3Mx6p^))YViL>sN>=8smz0aj|Yvs5>8cm`4vh z;wU{MR^z@3$keV)7e9>er8c5gMHr8h+4Kw4Ac4O{0%h*~Q*?DX(J?hO7y0^5j@4D=R$iHQI-LRuK z;ht!8$GEb5$UmdPp@9m_3wV$o13vSh|pu9ij2QNzW*F`q| zRF?{RP~m`WqW_9?vGwRfGuEO^o724N;av;HO^#YyiuUIiua+L^pzBRusGX8p(dXIT z>c+=CJTYe!j&qj3-pQ#`F4^Gw{!aROs*SZV)^rb!DWemo#)#mi*=1pma*{``RY{)) z)8`>8IHaZ@kAI&+1$!6equJwVN?ZXQ{&OmgEOwgxt2xSrHTtm4)bczfE{3%;o`Wv^ zL0#(EP^NAiz<0M~m$(*LzDT$(Q1KD<{Q8iKId3B0=0#~)Y#X)b2liELguSLGp5fbe z&T`z)uIizKjeOQ6zs@xyj{cmVZR#~``Pm$NGab*dV2#mc!S(dlJK@^av5Kzd{*iue zIzY2p_mJzaV9lSnTlB7OIf)))&&vntL`+e6=93HFHOo=l(348 z+A}*~(oNiN0h#-L0qz^^B6Hj+Bg4j3kO%*5;F^!CQq+P``cvy{`beSd634Lg{Mv^) zr`6{YgT4@rx=+iRB~!q^*8aBH)WS@fS2-&;B;0+fI zN#ymj57L}`UDR`@H!2|aE*1#`dEYQk;PGtem~3)-xuw+X-XsQ=l3)BX)7ONtwPOAy z>Xpu4ASv26m2NNZ@Q`!9a$d@5PTlyFfytC_%29!OP(Oo;YhbEG4^sU>rFFH}FGZ`* zSGmu@RoLIdn@}gR%xoJ`;-aVQdwaj-PT8#ld2jBS)2bY>9Coacm%e52O*b(vFK#3_}46%?}9UEWGLL~XaO z^R+#cn?#*$9BmV%J8X|py=-E5L!}h8{A`%%8I7L7ICsU^)>oI8f3fb*@r4@qen_(} z??oN2*Ofjc;k)l_^4R+da&Gg5RPL}B>Z>;o?|wip`EWZ8<3YWPSCxbAmFNC0!_<_c zMP<7uojBLZ9iqjH&77-9XM@MO;Ey2fy&dZlZ%rof3D!e73XW<<|898cLxXxVa9z*H z{fjGosw{U!be97q);qzz=xc7Sp&J_u%3T)h;fHxTwmt$J;fXpbA|_pn&}IyAOw_?H z9$wvtp9fdggMT*w|D^KMVZW@yCuefSRbBPI+P^6vYK{t=x{~|ksc+_%hEMKHp`8}c zwgLZ1*XFM|_^uzJ2i1ZvSc7a+J%a;0B*fL=BZKSt^2y%{bBB4@%_qI)@uTPt2BQTy zUhhY4g#NaGLnxwlZ7xu;FN3Er?zb46guPVj=&(c01vrTf27N1HrLZO*q`P}@sa`(R zB;za_4hEmmq-(i=4JqK%0@|xdk~;XWoWQ@+60(sPck>zjc3@Td{(Kib{}p01f$~Q- z+_CRS>#CM3scPsB`c-v_=>==~rt=MF)Aj|cxX@`li-T*6&y_~8rwWTR^;50GomYuY zUeEb<9aoL9lP0KN%_L!i|Rb+T%non5T*-5WeU?FU-tU?K# z%AfbK$T(C_S3X*Xo(`KL9a8d8$g&U>cIYIb{^;l2^<1ZIC-NwdHQ(P?pbMj`Yt%U> zo~%tPC*iyQZ|9*a7O4{d(tPdfl~QZUH-g3*Lj|JOFmS~(VTaV;u=h#%rg$$6y^nD= zpOaH}Ri&5S`xv-_-ifDtHOi1%nL={AeN*VVZl-T&M~@|3v*vjUJgL>WmQT3U!g{7g zb^n0UWN-_g;T)BYsVfld$IMHqc%bXI05;^;lTI_2>zPWtph zp+1DI@YEM{01uRV6>yzacFU~;){fIx?{wz8H`)<;gFBwtrp%aC3``RkKLH(VxCh$g zD7h=VdyIGYzW2Sw@BWP-bIf@!ZN%{|cy8%_0pUKb@8cn2(EXq0)ATOy_}f_ z-6#^5S5+Pk1ECALDw}U@DQ<4}NHk<3UvQOg%@kY-oaEvPET!E_VzSQ_s zN!jRodD(KFo33;%n)Z1(GwvDl`vrRQi)dHENr9K3S!(EULvIprlEZ)gCX6A055%=T zJE>9h3v>T^{Q77C{YGcwA{jNL{v^y~T#u;J_Bt1`ohM2^amt$4epsgzdPt3@D++&F z4;&v!&3ArOIo1!CrKbhbzS>i1U5Oml^PyO)wc}RxXsqIyCn9Mya+vg5#jMcDwCGZM zZL~(q>x!VG$RTc|Xt%tM(eG4w;w-haa;#|k&5xocW6kx(Pub|-lfN>otKAE8ixD%a z`PnFQOx{`W6B*1b`(B7C`4ty`SW+(F2?Xy&ODh+YQ~#a=zpg~KR*h%Rv6T%gRF{dp zg1O93RYEjcW*1^Jb+p zdYjvBEkm(>_4xbBMGT!Qf!*qv{H%b90)9@cm>11^el#Q8H}r*_x|5lkptaGH7gXBs zeMXy$7x$jfswI+arfyUpw&s)2Uc&veql6}9;Epa4=B&TGe{VEBK{NQ)O+U;=8-}ji z&R-n!lDYrqm2-3BQwd}|jL}_l?_c7hZT6`->H8J9l)$}Gt*$7cljO<Z4e3<3wT zdxPq

efR#xlQ$fB`%mZ5v;8CN^A5qPdp}s(?zp%f-vXki#bHj* zMsoDV4%}$05te?HS*v>PhfSZ4ahF3`SQs1tPkPcl;;!3e(c{P1Aj@2Rt{ib-51muh zLDJvBX{^8a-pNvIwC@hj-^cMzPt$Psb3J}`lc8)HU?2^T4pBeDdl@9+uSPDo=4}DK zuN(tE^CIZ(YaMA8eHF6$naYej3pJO{dwzF>yL1=dg>!4=+l7<(#~nwpq0>|u9%;q8 zeNF{y-FWTdS`P9=(KMD@;{-0BWI%U8ZfDL(OXdAjFJR3!XYSY|8#bIBq<#i0qfGd0 ze+_@w$CdY_EwaI`4fvo_bNEZW8)}>~p{t(kv?&tqRSbdv+n21S;|l)1@*%GLPz~J2 z-G$E+x5>=cjpQTCSf<9?qnWXcw26chMX%w4p9%EY&`W*3+-tZ6`_<`+&Hp5WdWLDN zE)1KOheIu!L5PeLQ%*LjTKl1Ag5b*VU0AYdCck{PuRQmtTy*-NFP-$S;Ny9n@zcl4 zIL6l+Z5}xte6Zj<-mc>g-T^bD z?xhv-YrSnUC2|E+-#!7KU5sJ&Mmli+6usZwtPXEKtTP(FSqBHxhsviD4&qp+r!=Mn z5izJ0tnVKv)L2s@f8qE&Nl-l`2s84cv-`XQ>E^Lf``c}zEM%vk%h@|{?e1OxXVFoQO7$JM&vxbGfLo!^G11Y66&TaWPUDY?w&gdUIi`xWPB^pIWo zA4coKr`~OXI`t9uZFFHtUxH+(sCm+>{|0$ghQhkVbUw!G-F)He{WvcC7ETx~U|>Zt z8`apGrIeZSX)axO=bi`gf?pVPPwh%`nIMuXpEBPT9VN{Lmn_(&y*_vf9CHEwI=nZY zZba`FLmRqFGD$d;HNd34%`ngQ2{isv8}D{FtKu*2?RJ%_Zgy|_$W*?WzPu`YukL`X z^5y8&U!NaLzm48kjuMY|<9>cCv=ndbx<$e&-vD?K=ZD-chW)qB9Ex9h;IBo6aC^o* zq1Nb|&0M@=6N(3%#vsK@_|;9(^e(C?os+(kzRhDbgG|}RD+htrQoCK}n7C{$9-N#yWY9UqbEO0b&r#S-Ug&e+V5r5XY z79Mt82_q+@fbQ-7oMI*^ulT-_LY#TYgnNDYsA4N->-PlfBWvMKP&K&Y_>L_)bP8hr zF6I{gG!8qNo>YbLd0xtXHi5qI0!T#>&F!MoApfS3?8Qo5f|JqaD zFlZ)8RB$*39LHZ~#^82YB1CZk6j{?dg*+(rgYsSX00# z|M98WWbAJA9MkI6mr)(g)RS{;IWTwuA0j`g{lnndr&VuN?F3z4)V=-KmxxS$~_I^5%cE zH~6s*5SIYesNC`_hOM}D51NI%WP~>?@VgyXvCubga_?M^$U%b*&-SOc(TL@4*^H;VdD%5L;*T0k_SASf9tGXz+M8 z@^f1t??R4n-Wv^k|Yl(~b*Rf*|;`I5gA2q~hk;;yn-$uekIqG6lPCo-DR`>|3xcB&YbuCU94n970 z<>^KpScgfqz-!la;^_)VtG5GO`ZeGei`~duwBUrZf@*D-jqO{o zFk6WAFFNqRfu2CQ0fafKmI!-lW6Nb`*xcubW@uVl__j=sm0!{maToUz*7QY(ve>E` zAUz^?U8eU$O!UBP({fbUXZI=_3%C0MaXt{fiAkQfK(tHJ@^i-0xq2uhlzz}gI(|aJ z2E15mz_+l@_-|)@e!EV#Cdc>_9tqqSUx0J4{S8LC1+C+7yFK6pt#6u2y zVN~lKIOoQ3$oh9xX(>4OXaP!#F5=0ML4;RfY{&?@=a;V%G#8rBXnwi2i7X7R6xV%% zvE7tDI6rR}Zp)0L+KbDc8a19lPIAvFpPlnvU-w*PDeoy5k4y+gr zgsu3}ZyxP|j;q3fpJVn4%55Or(=_l~1Iv1qu*%hQ*vpycQFAk{3g0rl1RSxd$%DEi z!;*q*F|&=XATGwn2m3<3I~q_}Xif(**yOrNszEt+X;XOL*A|J#;J|hIp3Urw=v}lK zBYqgO8{^%1!|OfdrNa%O-s(np?UFy<^^U?*o%X<<-=~!pBksxv>piWtcAg{AVvZr7 z=+IPb^{K^|P(SD4-%r%XG+;9(?v^p-J4uf;Bs|%H^lT(-lsCp_BVjTe&G%tCMx~v1TGD#ZS)S`~$-UbA^x7n*{@rHLSQxf?2hQq| z4l6$dvdZOaV#b)J@`y3*x3SL$&(ga1uso9T;DirO)+#4MECb%=BWoV9|Z{`jIj5iyb=eZNQX1t4Ov{ya7rdZ0g+y ze0~i;f4ZaWRD=+f3+kw`L!?bTPIV4VWB}a_wF*O0J*2`piV^p{T3=}rP51hlaA{W$ zNtnvjJSfM2G`J+qp)`-Un(8jeo4`PwR3vTBIv6h1tUc{2e|vQVmDi5R$v}DzPp@^X znq#lomYRg{0h&v@{=*)Z7SlQ9v5evi9VTt%G(TvbP#YUP+6CIxU&PN0U}rTKaphBI zEWflKJAK#yOE+JEvr$v|nOlR=ecW(PHBUOU13!P{zR=#=grtAj{7@H(#2ER>yMKmhHniX{qfEJqi^?lSly?j^D*XXp$36s8w{Rdo2c}z30O>dpTQi>#Pczc^?nPUklCzK#cw|p)C&Oe;Y@k8$(65^R@0DH0@5~66lq49tph4uL)eW9-|`pg$k!c@(fyO*(DRuM zkVk~BG20;g@g(f++!Bf3*w(=_aq^c<#3!Bk3*Sh>UOhfFzaLVJB=t|81Ux*r~&Sf3~A z?j+w*os+NEw0bj{&k$N{>qhnPHwFWbjK%w-x@#A@y;nFcg1R^Z^>;YMj!}II(%N`y z?`IKG-xa9-WY(b*)RQ|ThWz~n7rQu+Rw*Ftap7fF8Sgp2MvEjV_fTPST^}O`)u$>L-iqcKVJ{zoirVr+VTENI!f}>eCO6ghz!~Q z)F*&XLpi7Ms(EAdsW+n7V=VhLRFW1M2|GO|0nJ}i?~ax^1hcE~>_)SxcjJZ9S;W6`*gU!ea>IF4I%8Z{BdJ|rAr3j% zsJsL6*f@`P4)Ei%AFbknjt4-AM)8g$u-s?q|?-fgS1co1nRA%NzLp6Xc_W z@xv3?qK$#V2{wFF1r~ce7UW&D#NniCoT&~i$q!E^uC^Ac&)_q7E9_a`m?Pi`BV+9DH^!b zdmCfdBlQV!1f3;7emW+r(@nYR>-hXCBrG!4CUjj7Zu`ldrAIDGkfeS<#Tamls_u`Q@vpNJCH{rZL}9l_r!tn z_oOWt@wRs2wRNN??rLfMQPb)SsB!-3SCcf(T*$xD80)6B2kKKZsyDt;)0a;14)!P|?Kl!PF08?n-qP64 zWDi<6OV!h)-V}M{zM^$TfpF;;3&n1M5TCUfNjs9q?5TVvRDaeKbH`oPsNUGO^My$M zfF1T6K>FW}Qw>R_&1>m&;A`IY5u`DANd1#4UqE^9w}G8-QfRbh(Z8G8M#;nZjMEda z+=I@(Jm{`H`(q_;w%LNMbdTb+{$t?W;nQ?phZROmvf}{{&tUa$2iViuPBPhd17sUC z;lGUPz{AY&qWCSBMljb8r+tB3)-=i1jm|K?Up z)?(ky2?$dIS>{h``LL@E`lh#q>(SNu;XRj_-JG%VM^Y`#z`@RZ@br^-yIwd>duk$` zum56mHqMfZx*h`>JGQZUDA=6FS-cP z^AoaS=v;A!QS;ILLND68nUCiUHN0c>u5$06lQL^mV?5;DlTWtWB)9H+3;_>4WRr>J zFm_@uz{?JD%C7rjLi6j4;wXEZ+5xlD8puC0&7`~kB|&S(=yx(He)ziEY1A9uf?p}? zjYqStYjh`lqS!~v?8heHNxQuGBmBf6D==S<9&kS8GQ!(cfTP|f3X8cJQk?dY+qe>dM1bO z6(JBl;iVXC;U;@Gje|m_i!uFvXnzh36>ZL^qRQPyoeeQ>e|u^8Uu*4`m}_vf+gfqa z#*6n_ycZUAZAH)B3llPRrBnPNXdW|@<}jM8n5tatx9gem%p8AY zMg8Dl=>-To{!GP2e0A5C*V+fd+$Uw&)%Y>qnH2$*H?7#JYg*Be-Y4^1xe;o=$wckC zayW9dFZA!81k-g(*uzhI<-pl1IMpv2-2X}a&=;5xm;(*s2FqpR+>q)J?OW|dniG5W z^anKZy#rg8ZiV9`gE4J!xUgEH10IatX9@IT8*&RlJ!{S^TR!ro1V2L`%A4A<&WN*& z#;PGa;wrzwUPU2ck+APOTd48BSeL>+x%A?FT@Gjnm*nuf*_hR^OsG1ma1BDLYi$43 zgHLN8hG{!W@a8~639So(utTaj6u)UC$LIHlCU+*mw^2jk-6^R(`}jCS8O5Oo8_eVU zPD{!acI|1PY#u}X_+j6%@#siycWE5&=`a-^444KNCJOY79wNsKxqzQsF1z0vTT9Y& zRBfQDEkD`^@~mM?PV+#$`4yOaP)~HK*`4qqP`3A>bH0D>;Y$+RNUvUZF|b=V2;a2> zZ;joGgoi+}!5q&t@Mxkh>|5E;`95F8)_r5S$U8{;TZ>grV8>y3Y|!!Jc>Uvg`Sq2) zEPk^ax8I$^KP>LXRc=k*Iuv3LFXy#)WWnEp9q^^RP^~9xOy}4a4jqVl2DnPfDY}bQ zg73-zmfFY!S|&N-K(pnb>Sq3j`TXvR?O-$h0kmGfTDD3#DtzYb2Cs)knqSu+17VWX ziUhRTUmFb0w!+=JQxIAYko{vExEj;Zv4!w?b8oPIPkY@$S5Yn80O#u~VEw5+eCEs- znArXt&=_Upn7s-^(W&BKwrojvO!W$Y#;Ya@|MM?Z-blg*=^1^C&01Cp)dx10^t+nn zQ#(LHk*BEk9%Y-%Z!LwK_%gxokpOyoJP56D6 zAyatQ?8P+5JY*v%hG1*{2(I?*%jat5;?iR-Nb8DQ-i@X`9($_llQaRQHVVVZ3k>A_ zuTvyp5DR&9m(?AY%Ob>7EG%6IRBx=|k0D~SjWO;ykj4VLoyULsXQ2ItIMlcQ4qC_g ztkFxus@#ii_X!Ep+0q67;AKdfHgA7AyD@198ixK+7(sKWjzJ@PLwe<8thv?|?R4rg z!g0JDKzlSs>+vEzV=?b)d=)qCw-W5sW()cF!l$Y_o8#?W+5>2k$+ zgrU&paRAf`KTSMeLq?x(D$C~NvDWMVGPe(i*Nuv*@ImG24LMZmS$=^v+*^Bhg*hi& zK<|iJ?9m{4?{{-PI~J$QNE6W*^I`77{mjC#H%wj-4A0mvd~co$iZ3cks?)nbt8o0h zesGNTrro<`frR_eb6Nu#a?S`()ZQ#idJEBH;&zyQa|Y#lB_kdb_rJarq&=W@oi<|1 z@{sHnhvvW#>kjaH{Zp}P`*C<4z0y57xkeRd&}Yh#ksh*PyBboh-6Q{xK)57E|2hY^ z?284}yq3l$SGPR_1NJ%b=FJ<)80RFUHKTKmlfd;!XC67mM)bHknY4_nss;5dexQXX z8%}4Z_WeE&h@Zrui@!VHy&45N(feo~(LmS&YvV46C$$cV-f!N+B&!;7!KOn%>yEU> zc>Am= zTAh?K(W1l-jW@Mt0UdML!Lal0gpXKl=zC${wo+?1^5AMuKv{qb8vSnob9$q!0SHs_W0F`8pP zf|$ycb2?c7IXa@OQCIj~Cz5@wc1yeSW(%PIMlrb#x7Ivl{kuCdqb~p0?5sved`_|6 zBv#)>^!#`m?zok(FZXCqdP*6a(xN8V_A}v)N7s|pe$NzVeOF7WLG12ph8uN)*zffd zfcO)7b$FY7xz%|za%d+z9_a#)5*zcG-^`)^xFy)U`EI4>h022iSET#KA?vg)hdQx@ z)?K8++?X%l(Ds)J*ZsI39d|W{;KIQ$y);644==|4`it@Lh-UIs?`KGw83!IZhJ=|i zHuM;M555&B2Z8WVQ#Ajc!Zmu|X_X*-!wGAp--#YvW7kL9@W(CDZAyuFfqO z`Nllim)>zi@?GHB&TZw#I~Rd;sVH#21o~+XTB>oVXLi2R z-}x5Ub6TuB<+!}GG#3cBncH>-an&=}601s442{Ge%k5QsvG<4@C(VR;{W@Sp-y1^V z64f^n$IMLJ&R#y5>VU+>f-qdF*c1PVMLUkt9mJP0c~+57-YVo=2!3g$ z!%o+F1Apg7(=+OD@*23KATj%X8%t(gW2Dm85MCAy#BJ!6a8D!#=VklkHe^(LReDO* zHZE}#Z~kdW*V291Z9{Nn6D<&5f?9+05mN5b6Wp#WZ*T*8S%yr7`Z)h`q6mmtxd0 z&*pQS-!Z~o#ce+AJqZt{unVTB>Vy%G zfWqgt%iQ?%=(>Eym?whx16IuLgd2VZLf5zmB>ok_g+(ymA%QhEG-IzK+;EcX5FYMj zA@5!_W8}qwIFEJC?2aKpcaZWNre;!XnT4PQ%gFlDX2I60!@n{c* zb(0JKET%lDtu!}gr^jQP$CVg4v=I-B+RoM;yhxfmPGJEMu1MvZZFstiu3PHwU{LQEMj6?HeYw9NJ*PA$j*9hyC$(BAJZA>=TH%hD zZ&+in=B+adHq-wGq*CXH_nr zxqA#tP9_3rY4|N8wS>o0<_1sbEFSys`IsWbZ0*McyOGr&g+D3cI#T*ufJL29^W=};TkyuWji{a8oG@_&C+r2^38_$X?MX&H9~ADBH-pE=n)2CWcQINQ2sfCF zLoavW86H;PxGOQc=B+f+4zEFd){p5Ux#B+4MC?Qw1UESiPi(yEneva+ zZ)=J>V;2KyLJTe0rs@l=-)|OwJ5Ls~vv%Xs$`rAA+bC?Yb$|BheYmoK=mmU&Qe{0!K`G^VYWzmD0@2X=~q6^GUQLEOOuotX7HzRp8B=3NfZ=$&ROr|{A&7&bm9vw>T%C*!xA^r%X-o#a`VeG_- zgPHq<%c_Sarv`4}fuFlc;xzWH%s_k6sh%WkL*h9}8U%40bx((6Zw- z(AX7d>z?#dbxQs>A4uCFX?2*f$4q&Ap&Y@S0vs;#WCjBd<5xmx;t|)QvJD8-Vm;2pz6cT zsb?d`9PTD(-+ZGndcB&B#onlT8ub4mX+r2+_af*QEG7>%7^$u(PhU{%Pa^S&c)s$u zyTS(LtEit+rMWY6-mxw3is7gAL*m9fRMkGEodY?^m86C1 z0rhPNtLjsa)C(Wg{UE64t8|#s``YgnM*q*Hfpp)&m?xCKJ5?O9dxMEceMu~h>%>SG z09)Rdbvye6cD;7yXFG2kh zrPs5_E0M0Yl;kmiVu`Kg2}XGfwC2)iMXEcP>7eS{D4k21U+j$iN@GrDgh_Hbt0M@j z#KeFbm=o#3D@uY;X+PDcRX#HGX9*GxC||F%P8CK{pPzc#I^s#|CFpkj2 zI^7=yV*f?zj=YuEH`Jxx}!A0Micgd(LR>!6ZpWt$O?{XvlE+2zq)=ZUAr_H#n+XfcX%@x+Q zn84GLnz8HvFI=L(oCQAdlCOUL#B(DJF}!zuKB1s7FKKa99<_LYBfcDjHD~N(n8QZl zH$g}Cy~#y|U8r0g+m0KJ9>R9d42Pyqmg2}xr5XeC3;29n78s4b1@lMLkn~*k{N!7l zzHglHO1n+*i-!O5vhc}_)i@fW#9u3|im$X=)=^eA3KrFZ?&H@<1ElXntsFG|A;oJQ z_nfm1TxuB02iX$s9hb1ksOmDj_YQ#R)8#sYP+8!z9fu~@=Pvqtn|HK_r}@=WdbB&*{L*=&!hYs|#U-o}3=Ig)K9VVhdIr*7gday%}|G0$vz^ zw+!~9exVts=i-0+E#=p)TeWYy4#QCmf}n4aGX(6~58FxK&wTX(Gu#%#!C)ccVV(@m z_@J5V7K4RiikxlIQaTRE!?UKzm^fuuwt+=^KJ2)@OqpQLtxaoi-HnI9#`nI+>tH1E zh6$KxvPc@SZFqbP-R0VF1JKyShtxb~8Glr~-~1A19B}3oSL}1WgyLF=o$kG32RyE6 z+g3jVr^;@zLj|jaaeM{)I&nOt9vsG3?Ho+;7%uKbjhE5ZOJSYEKp8Qs6UH>F#qU8{H;%Lb4EodVK{4|0g zM|#P3J?p_8XA`*HGF7|I;X3%O?Zz+H>?A|9bWYsQ5YSXROz-D&qas3);zS^A1%dSws~@#)jBXq)q`vtTZ(qWqT$e{mbhW=HAds*PQIqn zcCe2cKl-gZB`&vmBliC|F7IupbDrru-RK9A_}}xlxc^6MIACim2ih3H)KMqInvxsh z&(|YpZ+Hqm-h2+G@2-Qzs7BoT?n%gTMOd)HLHdo4gcs>P@P5lJFke2Gw_I!o>7U~9 z?f_flGlnzsk;yo1^j?1P++p_l&ORLE)d2X+|KQtHYhGbj8!K6FKD#z!G*_|YRidD= z;^AW&`EAN^?ZPuN&?|B+FaMb$25-#9cs@=Vxu1g0#aW`i)mZ4?;3}>;vkxejShb0V zSw!dzw!&pCP+SDXn~!FRa@dCztVfe@$#nH;Jsj9^Zow;TY0tuiBbZaC5&wM0PGz*(}TM5s%byhim&FH?<>PPKidH+|O>OezzBL2G1 zg7Yh#WRty#Ea_UlPW&*3|4 zZ)1}^8F+nfAf5FR1;pQMTX;{xvN^b6fH6xR{Y#4nx61$GYhdiu!E8W7fF^r7Xo#~J z@d3X6(*wp`oQyO^=(r;o&(fZk7pF!_T2mJH^$G0!SB+B)sn)jRnRm|2aO4u<+V_|s zyoN5P-0(Wy0;*YT=hO&nTz5dVi4MZ#O}XfIdox7co+dvQ-K5|7#uBGA#6>whaZi|u z?Cx2iZI%5LGdu4GS~nz&hV+lFvS;@&G`Zx(=ufaZ5Gx+U7<0vQiVui;Fl$sUQapgP zLw0aVMDX?xeAsfWju?DQvaVOVYZ^X3x9v^>m(e)~GIcWx)?Y z^_e|=UkR;|0Y4bbVECsIoOl(77akHfpL+9SRu02n$7FCUUL3SXXX0TfS){{}%n%f_sE1)1)qKwKh6x%ZG(=e1DJ^@-Aj ze9OPyJhaCspqzqDr~Sa*@f&N=sw>#EJ1llKIe;q9S1vjyF1^ZRKep85Z>j~!!^uvJ zVuHl$F!Y!MX`NMqG#J2j&+zRFeKi z!U&wczdw?m!S1%+`0-?8OuE)vzUz?yzrPfaj(VbWJG(nDl(5%|=Nli=dgL85*k76L;I8$lYhQhNm>_4JPRv*4-gk7D0~F+A9Uz+E$Lt!2>zEsVE<2R(lw#v zIZDxD)NZ=tJP%Y&Y|3IHcW7U!v!RnTeK!!iwngG`Xpyy0{=yHb-f+O9KcBYR1LYf!7bK z%Tx3oVM*=gs<$de94N&@2 zLpY=D_SaM@-A!@enCuZWwtwr!soOY*m1K0>mUL+B3IU;)ivHU%0Zbp7ScF6?vU6bq(wKK;Dd zbK-`k+Pb$w<;J+fO_20EqdelIQLFk3Em0xvd5*7Cw3qhO(B4}^zvuY< z@qN8~>Z3L1JshT_Tps*(2S&+Vkfl*t8y5@7P zL&c7x#+r-sZbI?Xspw13rV|%4rT<8uXo(y1=k92Vr01C47jxB{qLoe=F55hwFmX>W4&dq2IgF8GSB1rLTTL5 zCC@N;{vV*VXUaQoom51c?2Qn+TB%$E$iK7T!TW&xF_8D9V`9d`s$i~amoI!-T?Q|h zhDuL|I(=rvPnJ^NGM0Q=y{a5#(_M2(7|uw;i02iHfjq0&cR3%3Q@N^ls#ElNSW|g4 zn36{K9frNG;?O2LHsVavTKu|xVun_Jf&$!Bxrft0@>oeJa|DR;@l9bacF?ZpV!047oXvu!zhJGBXW{%bN6 zwWE9d%hR+|cN+@QT1XfQo|_U0pSojvzi=dw!6`=1!NW;GP2t|q?{ z#wf>N5X$ge36Qsu{#-lK8i%7;%F;hkJI-vbWTKZ~`)>s0X}X;6IeH;4K& zM9{d|vz+5v!dU!dHJ30T9E0kN6U6J3b4=i*CBefVc>8HNg60FPO7uDTJ9fjRkyiOm z|I5LeX(iS$=4@wBx|w(a6PuXHyb*31|H~hNI2lMc(E1v2m2q`Cb_{O>^b^FHY+w|9 zPp~7H*2DwUJQ9cc81Ko_wtf>98H0Yz*mYF1bJ>%gMxApaPRq$eLNP=T0JTN$`ZKF_*HnKtu^fpy%86;Y%6xJ z8p%&BON6*?cY!d6v|&1s9;?dA)c=nRJ%wp@F?jF(Mc6jkT`1kv#~KB#3)Rd8pzH;d zCf^%fOX+J^-0KIDPDa1j?Lr%}L*!~yLG!RjOT8)L9P^Q`ODayJOpo%_ zCu&_d)wXm=Y%M8E(C&?nVubq|S_jqdBH@k$=6V+4+U{qCS}Wo!&7cExU)>A~I5BV} zJGt#IKCJEn(GT_``CRNfdIt5|03aDJyalFM&LHq>=Ez^-S38VQ2@^Gj;3}J)Pqd<76%^1}kEl+8DmM0c5ue-F5 z(TlxE*)MUdJ!QfRp=b1eaB)y|Zu;I9rr z^4Pq>63QfYVE-Tu{{G_6(l64p($|xq-JB@CrID|^R%1LPuctgUP_0v4(Vp;sih(@6 z(lDyeambZTf_Q-S?-T`u`&!CAFe&B)Y*_Dv%&{q_e3Gy#i4n$-uG}M3ohu#S(`YG@ zj$tZ~9bNoQLwbkK+G>Q9A^4~aT=~ykosz(A(B6E7#i_qyfc&#mbE$Ii_wlA&>66^L zsgwa4aOK5*(=#2!UwB!&fKhIYDl;FPu?nvg(0QIV`iwB1^u-+vxF$iIqI1s{e_&aA zPqX;gCH!3NdO+G*W#Y;sviB_vrPG{++>!Be6woFED+Z1Y3LySg)FnGUQVjr>7~v z-m9@3V_S=ty(`596F#x$^PO>-XKlXI>IOPDFpvhrzTow17BI)zTCVRuoeezfj?Jfy z5niLNfyM7B{OYgAtXp7xo;RUFv;LY9RIg~kC(NvYHMWJwtaVS|(wmJ?*<%%t3baF~ z;yAc9RtGK0BQV)8QnNi^zPz{|^ ztVUm5UG&Kv4|u?^iv<50Kz}Q@^jU|e ztKGrv0pGOrS!nN-BCajb5tg3S_;H_L9C&#$tzAE#r74j<6+hNM-DY2~t6LnzkRQ1l z+y@@7JpjF%-qUWjY$xxZ8LJs%8>O{$X~mBJ2!*I(EB?@>8$9?2@?v(LnqPkE`ekOk zq8wh-*O410n*z?$lP+D%<>b$HvL%kdiXR5Do!JIS>w=5Swn3vu#{8MBrTndTTZBeN z;{F!{!TaAn42E!7yMG`aNop%MANIkE{~5B9L!rcLvqjRsY)#U?wn+2IE2F=N#+!Tc z(~nZo>4h&JXxWmlqjS@vv$Np0-cStP*It}#Q-PNo*z(%$d&CV@{)`Q&G;o7@pY z^e6MVgLhG1E~VaiA(or01gbUJpz(bX{Bk-TEs?CW|2)m_@)WG_`3b9rJ{2?WP!9WK z2Je?(#t%jvgbQ}rNOggG-%k_QzxaWgC#p3%Q}G~cIQtefxwsp{u6O5d9e<#IS_kpV z!GhPg|DLrPc@!I0JI(eK1;O?UburJk2(7~l>FmoB+8@r&8cUbMn6YKCG+r@5bN$OX zDDbSy#~!QA$L!O>;BtDdZv8J<9v;9qeCZ6WzIfu(*|T6nWditnpJFoitLRkitQi0P zsVECdWmIe6>7=qx;XPyUalDUvmT$I8->eG6yW*K&a$f$;~kDg;rYO(yl!$K z`i0g4>LH}L!;3dQC|p7{X0>id-1FH;$3}8x-$U5JR+rbgxsdu#m%fc{AXm?fl%DCE zVR31?!YcHRzK7cvtzgM1MpB#JRKAO9BRxHO!l4#n@FORX7md)9G`BKu!eXHDLb1_7 z(QauRc6Pl+J)xueL6$-tPHT(Nc84&+$brX(uG5Zb-$8cR=P7qOrZa`j{N7+`(17m8 zGMo5d&{Ir$kzK1n$-0Y!Q_HJnP@)_%UU;+4?@RSRDHF&te zR@(3AA<$U4quUznS>_I1hTg*sE9T2rw@qaK!StQX&6O;y=O!5Y?H8=0b3cZpTXUSZ z8uH5%+4dnN(6j6a5_a*oF9I>Ho<`LSg7EOH}^UUr22E^o|7^=*z3 zsp<0T%Rk^%x>J0teOb*lANAZ78eOOROM`4>&l{upj-ajHs=n2nFAu6Gx0>1Up{0~< z>D|tc|8r5`9OSi@5EP) zYbPgm`>xS?48xF5)wyml?UhXYPJImz9@dm<{;6jWGJEmq8Jn~@gAZc2`wPJDO}GdQ zG83&UYw>BJv0{qdK0$Q_8E@*yO>Yn4hiQwn#0B`VfxXP@lfZ~uu*v&-iW3R*`+&l3 znnUz&`5C7U3P-C&@1b70Ie*giIpJF@oq_AfkB;mtR1GO?33;9c&!^s|_2>hgjFOOg zO=JBt4vCi;)wCo|2htQU+{F_}7s%$*%D_fz!yX#a`M6ErXh~z>+B<11MeD+sZfZxI zbQqg;xTQJkbROqB)Zp!sPYL2jr4_3B=W9tBnfCnvPR_ z@od&#pgNc8_k_Wqbk-rid`KNpy^2>Sx|zwJZ@u{EJBM+jvzW8ne5q>@{ zhpukMe8ZaVns*k@MGd!AT2(hcjXxmiAwfKfZRkFVQ$1~^Z^U>Ap!)#zbsob{<5_|@ zh^-&lUS1#B4Ty(X-!B@zVZ}Gpt@RVFhIk1F>vG(%ri6W2vI;EjdLZw+8;KvV#geJA&u}AgQv{L*fX9xRxMA`-BpkzOp~So8>jdEqp13y= zw&cga5VQKc|KLWnZgH%iGwsK7=CQb!d|UA!&O1xtP*73tz{d z!^swdrC)7^#v>0i`g~sb&_E`;6tF89schJkee89-A0J#E3r*gS;2+*JfSRApVcM42 zIJDG3Ds4dchjdLyd_%lbDTphy)DNI{;}#HC!+f8~FqH2^)3-Kw$j^rlInz)135X3f zg`Zg_@AumJFRocW2qRqYYJ-e+f`e`~I>UPpV)Ppzy`W9oSA;Hi>PymSvZr4L z4tVRp+nXQN5Ox813F=85)o-9UdYo5t298OLv>p;xLh0UUB&RNP15Nm=nHw2#vX*$v z=fr6pNVoo`)=aCg?faTit@1C_H*jfC9r70sh&$FH`6OJ>Css>iQ9n!4Dl{jxdAs#< zMA4^cq5FcqcgZRjYP}UsO)YE39lO03BiCAUg?%L#y|tu)nesKsjo;#yd?O&;2C2*P zK@()mi7(iE2lAT7>hky5v`2LEP?*uhn)_$0XT-5Q=kWt*U6=L->uHaw)&qmqvnPG? zL`%GvA3v*^B+LY*5vUH7j#L^CSB=Zj$Q0VIvEx4=uc2y3oQ*H0>ow%-`@a>WIc1Y^ zEnu@lq=v8%$pb;Fk_@yvJB2u41U&fq8kYyDvrT)T@*<>-q|#jvrsP%m!w$!4F&Yci z=uk{tx)n*2@X$7&G}FD$Fr|AGf2*1&?tszTCyTwEt8>ze(D%T1&@DFLYA#PdvWKAg z)~ue#X{36mUU*JgCYKr2Os3wRD__0;0;<2x%&Y^eD|`fT0g&H?%b}K_c!KyzzO$GJ z2|PP)bMA(pi?0v@G%Z3Fz(KQfWBU ziMU^-`)JHK%iUcZo17s2j(SZV&&vPC zd7+~V{k~9W=$ZE4eUe$1i~GoPJCpX@Lf+a2=Hy&Z*e%rfA6;lDdj0B$ZG6hGQ?;H| zc-nVz6lv+|_|hv6l&&To&>X9~0_nPidtrT9bN?keL;VViy1xPw3Y$oUMfC6ZvU4I`T2zD~4x7>(>zbKBZdkVVZt!LJV|IYu-t|GVBzhdxV?9RiD*|`_6Y19T zVX)pQHfhd3Hg(!$(!t*`FTW0XoeB`8nW*vs(x#X->!2V_4OHv$OvG1EctXDi!g3&= zAV|Z=fG+jr`wkzpW6pjO-q9m~UaK&dQy++K{e0P^;(C%i1o24-`N8o@udtKmML^gn z%rBRTzF)Mkc+_xua%3EkZWNC^6OlL-x1T5y5 zrOPN+(b}il2+}ahyD-OYGc^SlEEUFR7nYl1_@k|2o-y4=kiHqo(<)5Pr}yD9!cH(1MfK zgqIgVt2`%ZCk^Rjxwgv={5J9&kUq;NF2bpP?r4x*+YE1d3a) zU)z}f_h<+z4Q;k-2zircTxmwN&PpS7y#AjcZAJ5HAjyyVtofYAG(mAl-dJg6=IG>w z#7!!bW6CR2PNe)9Q<^IAH=RL7{+4tycHI`m3^LZR2VbYcsM(Liem5&lT&lGQ_Nwv_ zl=boc3vUAT6_VzJ^PdkW-TnAm<^#l+ln2Br z?4v1Y)Kv{ylP?%MItf9Eu#}d}Lw?pNDdhEZc zb%;M&r5PxTRP_XDXa7Hb`k9gERbGlT>;R#1$fH1K{#YInl(DJnBp+6j=4}aS5-rTnpz72E+4H~EkPN; z5V6N7T}!o(ezgS#?p~?Z1%Kr?P&pSqpE?wamT~BJ^Pkqy={OJu3G#J7ewb4}Clto1 z@mmG00pEB}N!p3iSR`eBO!>GMo5xo9Hp&V?X=loYmFLCHyC#9BT#9FP6WC6xLE`aR zXRzH~CP)X9AK!>eqMvG(|FnYsh1RI`xTAc4xN_+>Sf23Uv^IoW8$o|Tt18VBYOcZb znK4!PWq0-!>A`y%^5&WkWnKBEg`*&2kP#noA(r%~yZDM zReE4#*PDWJCNxTFOrF6IZ_#%MbRB%vMK@4hP|YoAL4KldTkaPc1(b_I>hrchwIPTT zw6@#Fu%0&_vDiV8K)MPG9S?$fKhgut?@bL(+CXHEzk`&uYROBhH6d<@N7DECqz6#x ziuENvRbHCbpOdz)(poC(BaZ`L_pU|CEb^(g1!)rYBj-NM3v-n-?(C?-u}1 zNVlmu6!g#*W~~d7Ru>`!U6+P(G$4(oO&HuvWr>p31(jEz+>_TzJ5G7pIM!ucMpb6g z=UNR;Izqe+JRm5yNAh-HaH$abPOb&hUulJ~SqHTOy6~ti|6yT^)%@_L^SCnX5_|X{ zTO2EG_dKps_Jjg7{tIG?_zv0A_JQQ4&=L^JQq>bN^oW9a_!C;M*M;4 zKh5_SNmxE8Tx{&~7re)q@nsga{8-vYXtZXYRIeTRZ4mw?pYd2&&(UjJx{&(emsE_=dL||_r^=8^{h5_$G6(--R5|15n{l( zAXYfio_$(y27;!3qJIN!{5cH&jM|E8O*pr*I0N7N?!xu;@}S#5Pd;ntbWMV-u6(fX z3@qz52{t_ch5@bCLrE7p&$3Gs(Ob!}X%AFtpELxkERC&Xu&}-PbhXnL#>W zmzS%qpC7$eLl&+_=6`yPOmZs~{caDHV|Avo-qjlO7tPGcGc1=I{2cLezi#~Bgu1fc z%sbfl^KUlgup|5&^as}7wU=9qf8nLf7*+ zT|cNf0lmY0bU%X_Sm1<{?P`M+k^ZD6iE=bKtpXPpjzhmhkMIs0Ti z`K2zh6-P`7AN#}n%Twg#SMfNaiKqMk;dpCXf6Y6+Zn*J9v3PNE98NIE1-))obe$`; zttm$}SvwG7cE&*SX><+yZ6(z@P~E_BR+!N^3ts_+Ycb(7rn8WBASy|>u zw)a^SFKV5Iw65~aKR@}nmls&Htt-{MmOXulhW%&AFFl4}zul5u{g4miN`H!v#egS?wWxDs`Ev$BH3#a));#oOf;= z(p=M?hW2Ux-p4JFkHhUOXZI-I9NA7==)? z8a#Z^mS1?1p!NRqgnB+j4lvm)&dxE1@Pc-{&X0SlM&!Ch_0VIrC#arqT{sJAEoct? zz&xpfS$2-*AIK|N*WHRE93o+|S#`d;!k^8X_=HiNFu!%S7&Gf9e7v&hL_zu3gR zL*)vq?vgMA|CXe%_D0V9M??W`iOo^h!+ym^v!p-A;ra0Unw0iu@!i!sjMhoU-d@Rm zeeH;k=y~^I{V7QO2Yq%}NQGh1JG5|;o<*=SUnsYZIt5JvuVZ_|iucESB~f z?NS~g`pxQxw<<$$;6hto(<}q^p9**}q9qS6$f@E2kN0o!!itGt7hyxST2BT$HIa93 zOqP~(KULi+A%v*_N3M+3IPV!FKR>z*3L`83&^>_kjPbAL_u)y-3{*WzH~}krx1ri9 z@-hD8&WV$tk$rn{qGFCzd_epuvd-7y_iq`3s!cVf#BU(^%t(lZ!|Zp93{B|3kuoHF zCpor+;Cky(arMFBIjDH~)uuV1ILGwtR6Kd1KaPlzU{g-}3IC@jdlZ;(HK)&_=)B;p zi4YP01u*X@`)=a}ioZK{oh6BvnR(J;q1JC_>fWk4bZhvKHP~HW`u46(Sb4o_eZSMW zGw-er!$U4}MMaDc3L8^&O{aS@&b}5jHn3UVm<`WwB`<|UVeW!ds^d3EJcSouyn_z& z+_BG&jl6JVD&6-{1|}yv()=L4ZWSj8BgBo-$d39<0m2Q!>!)aTjxwjYCQ2_rKAqdB z`03k<1hF^!o2LG*(-?dEpZBVDr?9(}pnGo?60XAc@OQ+|FF@bo9{-$Cd+)1!x|9)O-%^c?hXyFzeqgP}sObSC_lqvjv(HFnCoC;>{n3DQyahBS>#x zRN^}9-L5t_JICa?E`8}9ww9cP~`)W}LV?5mV>pC~StfJvn%5;&Cu)v&ZjUUu&w_oapbAE;@>_atwgzNZriUZ6UlZUi^%y~~4`i2e` zr#)RTZcjVt(eyOjJWFTFv=5dCi{2>CP8aJrtJ1IwCp7pb;k*W%GsC!|+K zozl>FW!im&jt}zSuFn=!9KG1Dxtf6n2e@i-#b(|wI?{koppd)Q< z67b~shMe>g27WyyNn4@+*n8Tu_qHniCsj?idR&4vjKYQDplHts!oE>Bc0wYSJsm~9 z)*7gPsyuyP#~w$Q`a;Io)Bzpr@RiuRScvk*s( z+D`nK1H^gS#5g_q^msf%>Qx~AmB%z%=z4skhVV!#ZS^F~f%wZ7R{i{y|HwQQwMT1V z&b*cQkiL6cG_w`f=wK)J(RXd69hH8Ok*&N%g+mRH#sV83;%t!QtO1>f?X#G``=N(j2woL- zNOi^5=p-`R;}fbd;O}ob?@7;()4a3mF80(XVOr8rg0PzImph5XG2;5DH=^B%08H<- z4!0IBp`R~6+rp)A_HGY4huNj7o{58H+?~F>S@i_y+q=DdZc{NGe zOwfF?kUkOl%6E|d*UBa_B3kbg((eWJrKW$UF<2Dzi!OYbB;3~&@AMEoKDW@Gp4yGL zCk1G2V3EcGs9(^aR{|_b>xp|U9p%BM*7A7wM?kzNNDor&^;7r_ey8%Pyj;R64Q5te zgJZLrR&g`&jZhk2;S+fW_Ud#^PVd7Nt~&m<ZFZJeZdOY7Ts@Qh@vn8`;90 zJB)PV;Z5H%ZP+&?Jf_Y3d!dWTOCbIh1u^D87|CB(F2-ZC_G(X8H|6y-?Rk9`Pv_u< zBH@C69ZP;^)G%##==00gzAa=94EANNXtQHA+)MZf89{Eg(^tw%jU;#+_GdCfr|H zNSa3^J%jq0jrsnly-;V&I90o>UvL_%JLCq$2l5l`8?3-ZVWYxjrCo&b@WiEzJQ({BYzJ2zZot=`%{AoR z(P?}N_3duF=-*JTuW!KZ`c`9vJ>amu5S8Zh0iI@1u` zvYUy<(Om^$A5i|Hv?$CwXDat*&&MV6Zo`_kL27MaT7V@^UNng-zvOHX503+$gLlFq zB+ZX9zXpD-XDmN2w4&Ou%;p)>_Y=&LBpln4b{_HcQnT9w|dw0fB zp?Gzm#RulJWvZ6El`!o84nNNS0oUnlKKX8kXp|pG^LrjUQN~Tz0;JJ^<{n9-qkUm@ zB#)aPfAA6cn~j3-MZ9N6ahF*O!hsq}pKF$9zSs0vyHqBfNyahbyW`LIo0Ya__Azwk z_@ON5*4T#s`dSF&$)Nq*G9a#CG*(6!2bxK-NY{*{<5b>(i;b=#&5`Kza1E+_YEtHaQL*K2R&=6$H`kO>{K2O6m}9G0`V_f8P*#c81E3Io1}k* zBRf5{7**|(M_@`X?CN?7bW(%oeLRjvdct8>w@)l)p&vL59s@7S8QPXzfD`1!uRJfG1VGZK~n`5>i9$=_Tg z@6wcXwyBs`d9bQS6!)tdQ67@~7S4=33~Toi_GMqfks|~6w`0{LVJ0Z8*tx+9rNbHV zEDr5+Q|$4d33~(n!%3#@s&0XJfU*H2Nq9#5-56DtcgDbojq_?Iw{3Kj%N}NFlK)0w z#|PfHe))T`zMeO|_K2WdPwunpEfg2bNf>}fCcDU^*TO-4Ht`o-+ajj2n73`cpyAQl zyjPp4Ld|)dQ)c9=E7_MadM3-iF|>M;h~zsMT`&7)JrO7$L5I^L7)2)IU|35n?w!y0=)dH zj>g#OBJO@NQRNNjS+t$0JdScRjjHF`QO=yaDEZ4*V%eim(&x5Z)dA@X(isL^cb=ij zRdMKxouKqpy=(dqbZIAffNg~RnOfybC@U27`9MGWP&R`rj;v#pRUze8%40Hz)AvBv z_lH*HU^I^!${?$>RnVnIoU|@{sQnQrD;Jd0;fLD6K9pU6;w`#PmA&BkLqkZjdkdvQ zh*L1xeYYTQgp?29j_DeyvN_5R1^G<0Ydixehoyc9LdE^cYv(SoR~)Q-AW#;>n|oE0 zd3EdY_%1C(i&_Pg$%mox$vz&9VNYg5nANTY?H|;MpYt69@m;Rr4wqqk#V`7fzRp_h zj|NsU`}AS>b#)sXw&)OaUH(in@tGrEmU(H^9_V2pQpzsogG$-ZOgRqvbYvQERGk--kEFgvk?yUu)XO%X>GDuG3oV8na|%+pOK%yOyYX;26Zztizu)>?gvqqh-pJuNZYG9eT_^0zR&$IL4zA z>Ktv)95!gVUbPgVIlmpcR-{40l~*+v!hWM}`WRllR;d`A6pklmZ3Cx(y0ZU-lNwq7 z7#6yxV_T~qn&^q4Nbkp1t#2d0YYL%DhqJ=a28FFpI6pM6KYw$uvHClfZgUc%$^TC5 z+Cko!Qx1#JK zx{?3+^ZI9van`OeV1{>stQQhEr=`FUj zVZh$L8QlVs$mkd5q2d% zdav9Xet62{Cq~q}2<-f6VcE$9W}6?t9kq^PQtvEqgT3f0TFXI?`?0qBSA89(CK z!WWHr;^3zut*8wI9Ud>wk9L)Z`nky8!E^(8+!7fS!)0>jXN_ZqUconwrSxlU1XeXp z!tv9UxNY_j`KMPX@a~5+TV_YXv6naS%$dco?r*p>y=X}LlPtg!?w4WX-%@+1z9YQ>Z@7}U*SND!Xy&u{hznp=%s{Rq4fNB1YSEXD@o+sVhn8}gL$qnb9~o5|_<+t}qLJ!HG@eMH_$ zH{QzrF3wqNDxZ?K+UFTBtGoQdIqepsYnUT{^J*jh`q~MeRO}ZS7G)Y*d!Ers7hZnq z#+R>1gSi*NK`Z@vb(bZ;rn>O#@dCB=ZbZ%R-eH^9^fLF(sqR-SC-1MV6bkJGN?zp}X z=bz31J0BO>EIL!PtTg0$?Jd;Wv&l^gVM>$9nAPgKfVf{ctIwa^XcN%$ba<26Tjv6Z|A$kX-htt{ms=&mZcv zW47CDd9QWzg-@I_+@5-rdN~l&ZMsOhCb4$+DN#N@53zm>Ok0{RE?Vh9lY!kuoA2q^ zEaN6BJmCgowY0t>wo7$Ey@k6XCS%8xjX-sTA9UR$)qtFL;UG9}2!eOTb$R>~FBx-a zA%y-KiVuS>i=lVy*p)nngm04YOZ#fwK-r~Xnt1U1rD#Qa2@@{j@<#|QCb^+@nt6MH61X=M>m^?m0`1=qSBQ7s(cHX0wfp z&AIsjBi{Inu7-ZkCx_n!x(=vilfXR98qzu$^W<~&!DZ4={`cxswtQSAzN&2jy$)`L zsVBZcLh@L~-oLGyr%i(`WVeQ&t9T`|-%?t0i8g^@YS?6c;QlT9A$xpX+;ltx4_SE2hYyYA?YLn; z*DGl*@LuAb&dU=p__}BQ*qBW)>4X!g+R1G?8^)fk#Y;;j%{we9wbs zc~XJ}q=$jCgFcr6uHj@YH$qorKvXnVfKKL?ZFNlZw7 z4BKlNsyP1Z670}@85$4!=d;(whM%1E z8oMXnrulZ|P5}w(wLrMUilZMA|9jxhazme)OMm$EJmv^{b_S}y2c-*A9h&movNX)F z7=z7P)Z;UUUJ`vo}uz!VY}AybJYQ zUDe}COKAvm`5U@#+)lm~13EX9bIkiAaWm`lDx0v@0uEnXuk;PHIO9ise@;uBBTHJm zW%W-tgSGVh*W3@zKwORVI@r6con|PVvqxBkbz6PYsGqM>kJ~sO&HXUy%J(irn?q2O>Z1Yj1U2S^3l50xVx@f2@G9x(8c5vBGvmAA>dSj^{ojKa zm|KF5S6TvTf4P543p`h7hix_)ONHHpi&7`{t%mRz4ToilZw15HIOp>~_$vvgQSoBe z;hcE!2W&{rS9QYgbT{JLr|cFDYS>8PW^LU_dIpFz66sq>+7gL(u-;4BqsMHQAbkr4 z!}Qg>fw9FJg|SHEzjkGc(*-EGsVf1|!(P$IJxH1qHu3n|;1B>;xSM@^V<=&vi{CneNg_VS_B|sj8#g)8aL;svcr&$6>XCkd9 zjwFxy_TU+ydLS$}#8;OZ!ArU~lGYh*WJ7p2<1ggNU5ba$b51Tkzj;j%r}+$NGy_9^ zW#PZG@ksoO-$zbjVOuAn>e>82cW|ixr>bw1hKYV%gjDaG^jXzfx;!PWSx9Gne)NX# zcbuT-r!8#8{VAyWS!tsib0%QKdL!O{*IMe`>vZ09XjKhT&kC9&Zew?x>5knX?$Gxb zq&Lus&SN3olCS+%Bk?gKogmtuG?%&&KG3@Gg&G?K&rODBH=Q+#H}CGdh(~&+aD!oy zKse356db|DPE5qmeo@5pXkFtC3t915t2D^@%Jun#->mkHdidfK-8))XtTnwD4GNP; zgEGQvdA)Q9e0&@Q9UNN%>2)B@kKqeCFf~8tM(gs*x4Cd_6C&{t+Ao|bNiS&#_n=y> z8nm9}nrgMemHvk2XPSsDv-QEctP6RC(%eBYlKridi-f_XyU&PgLuRvcg*#xwsA3=rl8QRC_@Gg?JpzzlkRiPnoW8Y2x22kDGhmr1ul>8wnnnvyiVs#WbN zlSSl}!X#-?-bLthr45X>q(OJD0vKHzHvLfM3SI-TdL$D{m>Nh8&e11G$JiTYnbai99!%HW?sno}w) z%)FS-DjK{((y#K`oJ~l*rycb8s>&Wf{4RiP@9L@7J8QB#X(_@{89%YYhxm&!qA2|O$C2xfH78BB5Sj)6 z(j36XyN3j6FHl~Kv=!Lr!kQyQ2i6unM;$S^@WkmGePeK2m6I9&w<2I^c_$f5N>mo zBRsAM1zJnmUZNe0pIiX;19zgzv#8$CVU~r=ZTd;+8*rGlu}X^*P9k}0Mmk!MUQyW? zP%gt--;04t=P*INDim)hok<=Z%svjofKM-gbftIM!yI+JJaoYp*j=rUR63^qcw?q8 zfczRGO#zd~v;pEIW|Q_4?m5=R!km|&avth;u}^!2?qfHVYeQ{Gw~l3m$?(jptGI1o ziEc}_g3?Ss2Gy2BH0w!c(tQA~Ta>1UTGfKVXKFZ-cLbV8)lJ*fN_`CYcuX&f9qy_#9u#UpVvP;No_N=L})R1YYt!9RzS zglASLqpXSYqIpU;irJgrsB8`?BLdertpN`Lr@OEjc;&awNeN5$3DUS6S}Bz2=XjKX|ep@bS?(v0fd*5 zaEy`O6A6|nNWPKXc<+Fe;i1YgmHsvQ2~v63n|VF9w6yn7yv9_X+M)X&MxF<1 zduFk{!`5Phrk!EU>up*!uXKIzbI}NeC0g>#taas4jpwG3sCXjCc^5qgQ3E#9zDmR) zYTiZM?uIz9^C_WMtuX}Dj;-QP@{p+X3-L70Tbl>;ca#riLY7em%KB|6$B9!pkfz{^ z5fF|G;x{B*VU&H5R@#ei{SH%plmHsnJmo{s#rFtY*xm?o#shn?P8Uhjaq_rGxe=pG zkMEyXoxPi}obdi9JV>!7&*>{EYtyJ4g!Csfp0^8BPt)IV>PN3x*VnSz+S#1?3ssLk z6{93+I1%yLiIGlFwSckBA0lP5Fscse!s$EVwm~9NF3M_E`v|I!Y<+5IPq_C%6VKb? z-7BGd2@tNpf*$5vpC-d1yhNgt7_Rzj&K1miU^gJR)CZ&E9lBFJ(~P zi}x>MtqT9cn{h|*@$h8!c~?0Q_EzOar&=UaraHBXzx{50p$uUUw6$6YD8oyWmfyOvg(9)N2WB5!Ln9N{#$~z zUwd+Irw-h5{9{c{a!(jh!x8;=dqZr%I#Dqx7ryoKY%)GLN{3C^o}@7}Q?Jo#xSIpEp>xSwzs%gUa^yVtrh&9*NeSa}_7 zI~wD~u`{?s&SVk1=P0gT@L7D+kA+S9+X(&CnpmD6qrE;6WYQ4>K5pV5G=6LkH-~KH z!yf3%4$btT`bK-YU;7Iy%!x+B6C0VDFYErtq3~T_q_KeAtO7K5JT3NCHCG#@_vf;eEZvsJ{P9!{e;_z#~UR+#`*?&u&C4#18G0Q?0w~8)Ba=d5&Ut? zqLrx4$cFd5zagy+WO$m%tBWSWPw!Qru2s#ALr4p*o6qH)y|+bh+Cg^ac{mp4nUT+& z$P%|0Xld@`H@j8X>QW27^m{(@o3GE8l+=Vpd`ZfPgEf5<~N4-dlQ zv<>p^OmnQy+OxX7UIKlFrsY*P9@VY^TDm4-)9qRC;N&AQvPDPQ3)r3a2;Ie_H&$R~ zU`sxGr=vXT7Y>oZ!#ZqZ;0$o$j;vDcb$HJ_`L7i>$$tnHNN9|?)#km6XdhX z|1hllthlkfKmMinnsuhd;DB50(PrTY7!?qX%YMb{DOLouyDq@Ht+a~!DrV$$5(+;K^fQIW*_OQPvf`I)1}Cabg=W10LXNBfT2gzQT6smH&6MUC5Xu(YsJyv>XPEm zsNbObXcJk+EoJH8{<7j|v}V!swJ;^M2meCPfjz@|q}z;S(WlXZ?w6k9VCSD}`Of$UBp(k>{Aub$y3-#89H zR@Vewe-5PkkKr|M_vTx!G?1MJwgswJNxjRzb>4^y!wJhl?A*aaUyp=?zSWWH6bNgP z`cpIW&Ifo9Qwxg^Mln^lb>I&)PV9r-lg?-HRC4NYjEm!xV>W-5BQUYDY1=s>Vim| z^DP_>&)K2!ikY8V;*$~&*89eLm=@EN?f7SiKT{r2FBTCF%#xR_^my>tvq1MO!VR9X z4uOMYj*B}|Ex^2c-n_L=ImS;Y72GpdIBx5}b*}vugdeP)b07Zv*%7wQT;Ph)TBLYG z(_MY!mTQe}wj!oqJ{}O?&iPWP=*u_e1YbiYCimzh){H0%7eLk|~S%o*aMgJxk z6`OIY6}~#uO%l#>pUyQ=t#_&JI?U_IB=IIM)70a}&-W?J#({c4K)IC1l3L;0CcVYC z4rVaVWHx>Ff%mCW+Ryk^GzO3Aq#>-V>bC=aM%=QwIlt07jWqOzxV&fVxX$ zP|zhLY~n{Z_+V$>&8*g2$$WFqK#t2{`p)2H!Mxk;gttu#7B}-L_B-OBB|e3}`=1NX zALdfKUx3CeO>5Jui8S|{2zd)Wv7K#C(7jK?Wq<4NNz+{hn^Zyzx4kH7>+pmw2pgHiEm|t1VPderZ%gyROt< z&R&~{kF)Ak;a&M=JxTq@+R(cQ^iSSi6a2I(fsq+Xu0J9dmhiBcf5E zF^+NTzPHj4 zmh#09bdh?&d*beYVt%zD*w*2jswqZ#1V$Jh6p9xqZ@k};ZQ6k9+j0KobTGI`vvc3i zf;bVPe5 zX5|**p97%QbjiUI$J2MIc;hDCIyhH*@_IwQ-SjZh@6evGm@D7DttHQVwBr}Yy#?2E z);y+OBwK9OnROg_24lNwMe}!GA!TV@{L~^7i07!c>ae6BLm4&vb(MxHwOt9sNl4la zDUZ0_WDh9Jr`q7eS(5Y};piQ0Y}jnEw5Fq0X(5jq2XMoe*IL2{>9lc|R(~fw!*tf2 z-dX>_!f&`RcN3bWJ<>?EKCfnH1mPDO;vBAe3~nsHqA6G$l|OZ5JGixAFM77VChwO1 zVk!r-*VYkPbVh9)%f_+hb>tVnqs*ss9O3R$^;sk>i8@)Qh!^A0FOhV~zrJ$k@5m~i zDW~@Z-}~G~;#5>RbKl54@MYjBR5(%x7D0Q*A~gNgn3ZXg;cVmn5L(a_68sy$j|rZ{ z=}%eG_`34tWj)?_Xe?w_Hj?YtEf*_BYcM&XCFO7p;l)$R-yrd8m>K)AHG=A}AL*Gh z;2Rppi#|Pob?NIc*V0H~rP4d7xNmJnBR(_78?78-aqfp#NHq*@rWbMOn-*Mm!#>3a z(6_KXZ*SKY0;dM4x(B8Ah!a@zcVS#{C23yLi1%pk)s&92^abfQ zjOkI#l>R$B(~dQrmx`n#VWV_Zu>-%vIMU6TnA`gVn0~iH>J#br-!;*`NqbayM4XJ- zot_DWNu*z8vo+?J(`=8(Z2krVbkbpK$#RGqK7|wi!Ooxsf;37MFDo6{JM#e6*xyX{ zESLmzFY>{$E=amltVt_o_3K#!aSEJvTS%NXnqAO|=Bma%4sH>XOVOtS2apK zi_3$1D*cUA_hSCVwL(94qwsRM2=D4LaOYoaR&@L0FCgSf)*UjbJ9r9p2t zBW#&ywt5vaYjg~^{wanWM<=AdP+F5kTd!j+v-QEFS}yx@G!(-2#Dmywhf13$J=DHQ zK5V}4##?5`3c^)5<2(kGz9fGF#JhsDIir5YYe7eF`29GAZ(^HOFHqcmGpY*`7vZxa zOL_e5Wn7bZ2%esH=AD<^Wz-wce&{?jxs`zDDyq|G&KK6;G5f>6Bjblu%yRoai#I;-o-M{A@Y^iZ_U3M3GgK{6iG`e{4WS< zyOuQhGV0+-sH@vX8W=2s!XHzWzM^&M%TpJslU~x1BV8}SrGSUZhsZ_GE!4HeD5nBO zab_PwD#hXl%h2dk2~-@_gRt@*swV{1B$f{Fm8Z%~fiOq;J<{DH;r8wdO=PQLB%h$& zKYWMsVVclG7NY5{3rKoYZYz&e`im1D3Z-L~_{_%A!B-%{Zj|a{*080uhOh&5FID2d z*(td2t2g{wRmRrd3MP*Efd~9ns2pm$t-Z$Rz7!6Mp%pf?Cr7r@?qGPfv3wApjN0-! zNO9Kmdmg|;MtJi@PwBjuX$^t+ol$&+;{2$ly9MPKLSNsboHr7R+mhCw!|JQd6#p~I zDJCo{Wxlg@kaQ>1saaU{I~otw=h?JBxbl~;{vA=Z3#41%avwSp;QD&@XBqAGYM2Q%+dW119?D!pAnQlDLakkBb5FdO#kmDi+fZTq2J& zkbh^!lJpL7m#vIz-w0pa-T*JJu4IIllmj|bF6*Y2JR}fLB4LK&1ti~yqwQU|-IjM6 zh419`v~xd9hE6e)K=Cc{J^op|4~`}6p&7Sxyt0-4 z`MJ2vlg?w?Z~`fASS^Y6mzmR%XB%u4q}7L4A-q(6TgkjW z(X%q=r*Y+Hm%AFu{<}@ce`I5a7AAsn4l%wx8TF|oAFU-VMI7FkclLAOEpt}kohd7E z#i}W&|L}IfDdwd&`3TBGIy*3MJx-dk9Z!|6P&h8g`)S99|HDA9?ZiQboN5)e=nvJ97F4>Q z_Ed2av~Ko!cp1h|9)ji7Bd~jl9|pSHO7f%hOw>xC`>m=k<-;#b_@$+_X;nYn$?%2d zS})!!+ypjiw^VVGr+c~5+$>~}3(&ta@}P|RM}D8#8@`1N7bP8rX>uHQ;HOqOTIHe0 zHv;)3AY4Y%7FWfyq75u<$8Ox?-4MthY37_L(5Qa(7YESzX)~C4XBMdcM>@J{7Ug^I zuOR$_ACsl>x9zXGN5h-1_V{VcSn`8T`s*TBNZz>d-@^ z*BU4vB}xB4y9RejYc!@ALITY(jx%FBdZy0Q9n@@zbTj$=dxCYl2)Pzf8q;5yNIpwx zBjsO7`@~jp4S9YZ=+zXceife!;(j$(1gd`|&0VF(a{FI~$r}fw(ke6d_7Dvw&%i0` zE|CT~q__tG^4Zsj355SA;rYXzB42h0yRYiV7T48yQ$2b+-daSS&_I%37E13LeI0;QGvtSB zshK46=}LPhMe1?FaJKkyJX8KQ+#pyeyl%Shj!-_E=D5tW^&r?6$k@oACfv=`uWHuN zdrJ!*+-#Ykc`18e(^%3R2o>I{d!kwgt6`^c*pvFap=%bXHLpmabC_waM|=!aD?pe6 ze-^vqaAR+*xjliC=K}isbkgd5rhG8Xhs2xr$v`*(T{o8l@f{=H6vton0?M(N9+|G8 zIU^5f_y5?E9?^aayIobEMqkTN%}PFacNH0~3rI_t0nG&9^$%xWRO|%br=21Vo)6O_ zZAnY&%KNJ`lqLn5BNY(d!;_LDNVvdy?YStXo(xyo3duhL%}Lc9o)H&H*VE=mIl%JG z(SmZKuvk#v(2S72>+F?sD=UaH-zY)T=T>|HwUop6O zQ#4u`h!K-R__sUGyvN6ya_i$9xVEPaII~1Ph_IK_GgHwAhG|l#M~R>PcCe3krpslC zda|8yymtBK?2jfO?q=$70*LN1Y#4p<$@LyB*YI3(&$(oZ)pfbH4*X?@|D^m_*xzJ+sYSUoc zty9_ z)Ras8E#&4YEts=Kb2%`;Up{EzuC3>^6xUpBRCNytS3^)=Pa`W9#sFOxpl%y}-<_V9 zz2}DEe@|(;AKbt{dxWrKO(sjjt&K3=_6IYaH-?8F{w{Rfy79mIX0pYEVo~qdMR+iG zsr>lw8ai0agbiO@r0KkeSZC!Xz$V*7=Yx-M&wqV*;~L%NzCktRw(5Vx-NQ$;U0%%O z`|f@f8Nca$aeCJAbep-NL79nEd95~SFx>mR4SUv3;(5L^r9qoFEPdPsmb@_nO*fyW z-+vl&2VKIRFPig4x6NeNh~*GscLw7YE|V`V*s4ExHMtqyvh2?c#=CPk6Xf9 zdiKS{sRzaGBrK_uBUTz4;N{MBas3gF_biQY>fkz{>ZDcvZOmV@S#xaaQ5@AH2G{JR zGn3j(mD>-d%e_W|>FqkH?LVWI44AVRqB2_3K3zJz*R{9UZFV01YrF}2Y`ITo?3{&R z{d3VeC=IS}n2QF3%OIAS$+rW$3$46_w*JGBa)k#s6~jqAAGz{*CYa1LV=QwKU%0I% z?-<)!vdlkh0PE?&DMLD z?)k>ly0*^#if0yd!do4V;n?P{*(@hLIly`x?a#ag@{d-N5tEIJ3&d^XDd-wV=kdOM4k2c$?L#vcIZ0(31obF#F?4|FCce=unH@ETgh!R|$;6mrZ zG(n%=yTyPNX7YVy3$Fk3E)EN;%T+Dj*{}!1q4(%Gs{-Db4TEv3JMunT)t^r7GPxC-;h!ih)pJ2$^5uN zJZBdr3WpuSJ9B%8@%1N)?5N>FCvZ8A{&WX!Z>uHgzwzocir4j{xKHOkPS~X*scz{S zCu#nR_@OCX#S7fVdRr*R5U zJb8-YK5^!hG3CXc7YzG}3PUHJisuW43*6?{3#nJ+9sh9!gjL!fE}gv1pZtTbNq15G z>9)?4>h~2`wu*tfhdp@Prav*H>uwmVQ%`1o93Uq4{w}T zX!S{4xGj!_-ySU~*WBo#u^eK(5xTE((^5ZRt)J$2AYh=v1l1pCXBmm!Nds}OYbLGL zQNBu9h_)?0L3pd>7_3(agEmR&Ze}PcPQ1tbZ;;ub02QViL|Xt|gRKiK)s*Ts;qUeW zA6ec8UOsn(h;B=yS?73)r-_{JkOyl=oRu5v9!6RxEL_}Jman}=`@4rSj`E=)EzB*E$zqpXj zSEOIZx8b|-Qn8y_7naz3K;K3hlqM7S(+|G1-i?BCBfkBsEuAmWK9f&RX_8XfRP}JSG3qEmwvT;&4B7^bHwFruel{2F+DDNN3F(2iD!X&MYNsX9X5r=vlZd**mo-n9ITTJy}f-T^`(~h znxAY=XA3Uc0B+}j>Z%N;t+WIE=JtH5Qwa>|q$8s;XwQC@NSIt9#?JB5sJ_Z+RH*n_ z(wY?xfceO}{D}1yNj-(_w-@%3LYkJf9d zc!Tr}5GM-NUk8uxgo``!pxe4NV(PucVoI%(Raon{Vg#G*wHG6Yl!MY+rB3nqwa5+? zmhXD^1BO)kekD-h_v8-0C7N@7{?we)N!a0rFng zzYs>9V1yy)>SL}j5nG-(2J}ZXe7hFTu6$LnW^X>T=nw{vEWQKj7nK{hwD}^uKi`#6 z?ZLm_?)=bQSE!)93Z=<&ZW!G-apO( zI7Jf&q=zu*C7nkeO0$abQuVc5Qo|QsI_&c{0AJ>=U+djGAty|$?9H+XvCsZe@~xKT8SKY?NA4ui!s z4MeT&holp|N$V09&cp_Lazxjm4t(alV@S1t=ie7r<=rmkrBFEgD=8Jyp#lybcjn6v ztiilgdi9}xCQ$!_?l2dm9^}-|+I~jI(Eb%?7w0_}?Kd6}v<95-;35~F4uFx;x2mRA zx7Ug}BhAs=tA%j3r?Wd=>ysw!45YzC_0>#UyNx6Ka5;{e%bk%pA5TrPkh*>SFezm} zP_DF3Zk)iuN4rT;a}y8;p5*i4fBTy3Fw zwYO;gWQn+3cAsYCPvOi~Cq8Na1)=cY@liuqyWLFIv^$|q3ZBZJ(K%*Hv)^=ZMbd8c zJ=P+w(@=`ci&GJMEXOPLE+9Me1V{t({`6f9^)B`F4`x<&U9-yeAX;0VNAfHMN}GTD zHwc&755O5EtKjtD5&TBs2>3KT4*f#r0_jh!>d!}PG8C|}%)Ft47j)Shk0NmnWk2kc?U&lmp$;t0=3dXFr!Z3N-^HR`h) zig!dKI`^G4ClE$4(+X4N$>4XyFd)w*jK=>4{l5A_bGewU`L2hAS@>e77eD;<7%DyO ze1V=h`t@4S_3%3F;bF5a3h7?)L5;puys+opK@csMkzTx}Y6Um%)uE$3;T5T!<*FgNh%>yH$C$ zy33^U)?#RYf&AFol9Shfjj#QH`b`if5k@S;d!HTf;p8mk1qmNJAbAn!kYr4r<&02% zcFp%_Mx4sYn~9&@&uQLNe!!-u>q^BJ0cUg6`suqecTkwpHhl#sOigK>#)foS3Y(Ma z0@X3aCsHi7)R#{!+KL>%NTz%~eV%)EWKh&~p7QmuRlg2bTzka2k@EKF+a&{*+Lxii zeCpx$r0Kb;*=`drLj2n0;_#I#Fz*S-W!>@_;R)O3cOHo|Iq!ZP6qYW!-GIDHApBpoB`Iq8Vl=M$l9p1rr>DLpoDsfU&{KHS+rXZBUTBt7d4X(xeqM932INwF8d9*EDiE|x33gR`O{uI=+q+=#B z@>`l^lN({RIfsPbiaTOKop(xG6EDmG`ujk=3WQaVp0}R8h`b`c*KIGUH_^MO0jD_u zC*1@YjT=D0qDLaxtTtD_`&Irps^LooN;^`mX$yue0McV9TIyog->HM!b)o``EK6N$V-ssgMO~*nvM~d6}M{&uB<>cZ!t>iBnXGuy_gST zMfeOj7M7~nQQI5|3;Dg6S)>KExNvc%NDOZ#2>UR_(3KIt(;Q_6{`~!mk@pcxTeRWi zqsdFP)G8mS`i3x=1!SEN)emI~<(&uL%@ZH*RBA^Dt`dZwVpNl@FzQcx^1q#d^o9tT zl#I0p*AnDAWpAUujQW$EY@P#FJy(e_i|TXox16*SY(*dJ=64#FB}QY&qvx_F`Q{D>?`{|>uA-R^em&B;%45f`HRR@Axj^%pD&0z03^cz0 z~te+S|SLiP;+Oi;xUSmrP2+W8Yb(hCLI`gXr&2dW&>}gW;|wPgLz=|W70-7A+_l- z^6j^%ClX+Mv{gZg%`ABBY!0NU7mO%O1iGG3`s4JjX!7!D zQ*`};a79^ix>^nPSdbckuf1G0R;uGB+XIXKg^q#iuFMwIF&u#1_5wsVW{W>0z_ z)a#4Vqxs}HCG{%281-5ZFABwxgsDRD8F>#*m@XC1K3;f%aLEX5hI;||PM}$omN=W{ zZ5kB|K{`?CH6$O5R;3YO_jwcCy}gO>OGnb|2B;rcti>Ki8dUCn5bw>bRs-p6Epd<1 ztKM{9?C-&eK(i+nR+bENdk%vuBiFM(uZ_sBu3-;a92WLuvT$|lk%W`amDd9D z@QnPJ;x3^1lpM1tjXd}#rOjZ_ae7D3yE~G`kXte}ocNhAXBHgroCY*k( z(SEh(nBo9cd!5eMpwcsxGgRD3nhH1R4^eX(Mtn-V^hYR9NL;Q}eo@VI67F{ahq1s_ zPSvckM-G$Z(-`STsr0An%VJLGsunLBje*+!9jj4NB z{MnB6pRp2XCZ~QsH1wfByG(}tJU0(*58e00xOX6$JCs2 z*0+wLOYNarm&o&|W;BTdeWdbqO8ZccC|`)=8D!Ip7WnPUKII*io|B|;knoGs{Rpai z{^98bARi(U?w?fJ2q_Pu-ClQWI<`m#cN>d(6<^r7KqtBFWF-H$z*gqJ?kEnq%;SIB z9fWqWBmd-=h|^B$$<3jcan;DjnB|uY+fG*F$AACC&mW@2)8jV0@7p$fvTYNdVQY_V zv(Mo9J{#fohZ-?_T;jlhaK)%fUOdX8!IMHq5Mrn~r-|IyDg>`w873XJ zUxv(GHq!OwP^f*vRHoh{9HhOgJLfd#KQ2G={!o;yb+_FPoqXHzfWpJjsnA1?IkB3~ zud?HPUZ2F_+xz0T_)y%~>KQxHwSqN!Umu$rB=WMC{juc43LLN~PDWns$xpPLraja1 zIL;00$wwyb)As*hB$tU&e3qMphqnI2G=r6LaK>jbb3~}zta}knkIhF1JdY-+3Et+e zH)&6+FL*02PMR3h<>o8LLBkp?`RLsG(6=bBfQ~fse%qrb$9I=5>!Hgt6{Y#MbFjr(Lm zZ_!`g%Bv~W{l^p zzNh&w!{OLN%nzph;g8#j`+qA`T(ocg-NP>LQ_(sk3HByif=S=iV71Uj-m91AuUJr?>4088l0Pb>*Yc zz4_mqCYr`C#=&*#wlcFg4Q>3RwNx)SYJmf$de-GPYB=(q0n_Bp6(RET2R-arIuo7s zoABFSdc0njFf5pdl#5BcU(E*6zjqE?E*Jw?(~MJ2S%B{Wc-F`qG!=6o@^xpfa@BDD zKGCARE=-!g8#lka3g>c~%AD2#;@pkq;#%Ti>=I!rgX&j;*_?^|U80-H2b~M!fSc(2 zouMP2fMcO2j1L{a_XVf&@vD0C{&qCCn*D|?NR5~DyQ%&!!N+&&SgpWT{5KY}G4`>7 zdVv3Ga~dj6EN3ycjpf7c18@Y%`*-uSkonF^)eP+FevQ_Zg;UQ8f$_R@7J4Mo{p09HmfT`YFM2P^68g^Hqk0i^KCHrce;wH~;4Pc zeiROOd5=`H*iII~iI#7*Kj#kNJ4SqhsKjG%qiBa%TOM<|v5$HCEav{U z4luaoEcw9V5xS0|GX#4bY5#uQUX0ykivi(B;h(!5G#{@oZ}}zRH}l!B)%p+)YdC-p zZ&QNvZnTsR*bC?$L{{E%_TT$NT=MS`BU}?xLu$*Jm%>HcwkHL}29k4j0QDHN@z;aO zF`ebJMyU{6ycVZ<#fxDHgTemt3y5}WrF9H%FJ($1dZwhpnX%sdbekT$#9%qH+3#_G zYAiIjkB6}EBe-hEcHww5pAmWWu7jdE_a)7baz zer$H4E+>3K*DDW@@`O?I3XyUtzhzV+#UHe9cf%pGzpNxR2Evm!$urkbPz>oj;8(EU zKFWJz!8o9L;v1IFK*D$a*TzYD8P<@5Z{Rk+H6-ouMr-2J*;!+`s&|7|rQ*Nf6i~hW zRNtCer=-E*(y7!JMIs>FlltWx-nyLtPkeWyrENCU3zz{G^L%ja_-e9Q@CIn+Y0iHp zlxlA;swNG>`e^^WIV5MTcnuWP6i0NLn9$UbGBrcE5v!tE^c> zJ#C79BNZQ*W*8;d?3wcJ(SkD`*dh<`WYtj2?m!Zzb>X`fLyxiUHh!|`h&i=G( z$8Rr+RGfs-*6!Tvq`v%6W4kupvO7LKz8Q4yE|X=tB}hH2Dd^?J=e{a|jhPc+o1rfA zv@?~WXs0GFZ-ty!nT>B<{W;YRr#MQtqLzGdmyhB>);%E3lrLREt2nvhgdtX5?JD+1 ztdf1_w~|{VJzGBOGj84Ks`>`^dDf8>uOkf-1?BFtnBrkA&+hQzPj2LhY5ux0s+YI- z=e$_3&wt0(_`1SWyF^Ykh#^<#45HEgjB1>o=kDB4=sal@n5i2 z{r5Q9Ga0Er;kH+_rpDjttmzvDOIO}k^(>G7UH~nh{lNg=wwl;?)`X7+s@9Qm#Y_y6 z81V~ze`>@DFW7&#)`Dk>Ijv_I`siok-aeaPO3k%O*T{*9?J*;03r_5%6;q1WX}YEw z$=G+BSg+@k1aUG;>Un~AIuNg0A1LsvPOoS6(%_}x3xRkQ!rixvWj!w7-J`$Q-=yZS zIkX?7+3WG+GZ*uDHZGFZt0~{%3BRoh;M47u*mZmjetqX2d~Bs<^C}(j(W@S)K0oX4 zWbjzH2?(2@b-^>XCGn_cX2mQ)SOSKtPm2$0GT@_!g`{3$#ACeA)&B^W4A@im3<$Y0 z5|RURaO90v#1m6-;rQx6`T$8g3F3BeUXZOc2bjAKMT!v+9;$UKEymXF*1}>*83C z|5$x75I)sh1Lu5OagX8+DAr`j()yE?=2e=LaN9tZzj2oz7n;LQoq@2dM<1+VF_C-t z(R0qE1(bFpO^k$95SOwB)n^F{@`p7ngBBmF$)hU+=wOBlk(}cMUTM0L@EL!M90jDy zK-Fw&1f2zI_8Wr>o&fa&^l(`SnoB>CG$ce-7i`;yxPq|4L&DDdCnG&2gIhhX(iv1Y zNVp|*Zl{9Tzu;I@u=3i9lzG=s0bM~90)fxZ~l#hsVVf`?K>-;m6C-2j&DLM zzXnK}45&`|(rrUkpIQ|=^rPjpUuruP>tu$;tJ{Birp8}=(hUpAf zxPkpsQmXWD$(<++^W87fEzjV8!Dgbyiq|0hOfhtIXE}84NH9t|h)QdV-c5l#hvFzk zIt4y^Rp&`#nNS*;bcs;;b-uM3cT~*4-#rV_xlaZZn|}tQaBtG<<1zngcQ!Jo87L0= zM|)ON9m`)0FR+Tf`uJw$a5>T1gDb9Xu{#>Q{Hx0`_WF`~9td;UtI}6c>&AYl)4m$6 zZg^RF4Wzo2UW*L8SN#6XtUnkd=?U2Sb`Q2%{s~A=QD5v4f5P|UpB*0XW8{57`W11P zwbXC@7^uGCmR&6o+-d?JS-Ar!PS9se52)=~p9K`Uqr%YQjQ0v>VNalQzEL}&A%2yY zTRl;H4wP3|aAFluuNM%G;pp5n;-UA<;FUike89=JSvYHUrP8m2nH>fB47T&{3r6uo z%CUxO0Yi6am|M|q+;ZKWpM7&g;Tt1<gag{^L ztE4)EQMat%=DUWx>DZ1kceEdR_*uwK?YygW*p7;K0IeTtL3@``{>7~jLwc{jBctmI z%6kFD4u|Bwh4abltMWux$qAPw#R6m2G=(E`%8_CNK6FNF;`Tm39z?8Z=LtjS-qX^* z6X&eP*%rY(ypJuY`udhJU1@V~(qQQEZVUXEYpV1Qc=<;$eeHd9O-ULD&bB)YWA4=C zL4W4r+|i{~9_^;bI{3Uum$&>AhED<=rTY6n6a1APW@A6zsi^03R#YNBYsq2=Y9htyj3%q>3AhjFBoy1 z+vYsxW)_qe?nC>oZxmi>Rb9q~B(M(E&I0iOCq4u6^+^5_jy!*cqk|hr3`!TIL$J5& zbr|4Zjb|=wfb~j0;?0HianFrh?ABg~N6&Y`YK?VdLru;ljyRLr&|v zNW21c-%2}!X2d+$cWa#>p2CF7!Pva-DnS?qqx3US=^4@(NZ7_Hhe+6mrGI9^-=tlr zcq5}IN7S%*!QSuOjs7)SV7-8MRh&$$jm|go z2hy1I?m{nAze8aHc|?e|J`69nepf!B%7@;ZJxQ()87(SK2b=)70ERdvx0%gsea$9}R)e7781$l|bCf)OAP~aq^M{ z`D0ENv~Kkhnl-Z48YgcR6i0k^tG>D}(A-G7;6!cZcOjr%d)6hF_M7YSnHlcrfIC+_ z(@>qU@F&_Okb=xgOet4#JR^gT3d|-V2KZUL$EX^%<%9 z*>%+2s=RHvepsVsb)zGas`%wvqC3!>3l9gp1oBVf_JVk>*iv0X^9kUU zV~Qcq3bp<1>T>tqyQm+Bs5u!Q)3=u9_Q|E*-@68bn#~a=YSipS_oM+PwmDB29EXJc zggK#7%?UfZ@4#DjD}~?nV@RK2q-BMQIpq$C3&8kRI?`MXos%0?VOsd{4DHF04@LhE zD^T@6)g+Kd;-qD~Fvu236O#Xms=~Ep!XKmO=aGIr0Hj-ycm^(Kj%UiR&>!+P`xx;$ zAR>sdq@>FTGD3_?^P3QPR1emdz#UiAk{o5y)|i^8&pKBL^We0e+lHrlqb{Uy6$7p_qihHO(y}OQ&QGS|cx>nLA ztr{qgzPNO{>M^D`lJJLShmBF!zlP!trgR};B%>JsM2~BSgv(WR{x@(t?zr?7Ztk+; zt96C~&E~ZCM$AHr3y)p69!l%KqTGBXZ#n~%9(od(ElQWa1)4Vr(gam=Kf*;#MsG*b z^?T?U=Ck5gn{MPmq6tf1LY>7+k>)@^n5gE+@UJohbkm~L%!agaiSlMr+0B z_hvk^xIWFKFOmL>Wc@=-IMp(c7lpmYw*dJzPV*n~(}|$CO?mtgJx@XC4lR_0UP7fC zX@;a`v3T#&XRI)Yr`eglngtBn)dPUBXfAQkedtX7bA9tBYU{~p}WHdZ$cNU8d>G3ST zT3FpUPdw^F-(zOx!SJ1BaI4@i_V`8naM8VXgB9Xn?0vd7Z#=R46ef9_V(fEM9DZXs zk7zX*<2yWJjV6Z5P#DM^u4dwyVtcvA%Z~>f(1n*Nr!c4B2+F35(Y#$U8XZ_EcQu&> z1LJzpd%Eoeki+%&7U9C)-Jzc#cwA6_R3w3hwhX1^}-!u+W!4h6SY zeACQny8}(<=*yXJPr$=<^}V-Mn+Sb;on+@B9zbh?SwsH9^g|}R>|BJlt!@v=Uai2l zqyC6-llWH}lMNs42mj2^TB-}|J<^OnTKEIkWc85^cKj4q%E#bJ6gNIIvoRdg3l-~543-@hBubOq-taljL)`v&7{hC@mSeRB% z#{7(c88-i6pH{{^(KQftBAjqxS~5ygGrp_I2XSRgXP&=fJU-Y}jnlnwrv@>yUb81K z=4(0HnC=Aco|dparLQ>rgU(g-p|hsLOYmMzQ$Baf2vKgN!=DeYEA^M8!JkWUs$L5= zT+qWiCtLAM+GDgwSO@v_evGVc>>^uPe}tbq3n9{PG~a&pk;qG-J)aBZ$FJkO^0CwE06I_RKF+Gf$cb`-07B^AfGTB7^?H*n%^2^{wx zp<+g{dqCgmH^!X)-{D6=9@e+dh66{QGKw$nd^bU(YTx!yL##J+GIzf4SnD|a8^+}u zK~uYv5LVnmD-nr_}%@Q zHl(_)_-I~2*L#KPT6Et~wjosW=XEC3Y#ayF!?Ln<7>F%7*tznX*zcO?t#j@ycwKLe z*Nn%?Gqq139g(YzZWaqGBfC(~ycMRgA8?~~9j5M?4|V!T!rZ|kH(;k`{>u-V53_Hh zufu&%|2&kGfIoM3kd#Zd?8qjCH)y#m3%AD=!SXu8#S4?S!r$;9vaJ*F=+t(E$8n@% zbopY(bJxA)g4Z3bzulgD&wpz+Rr+S4;rNG(1?%e9teVlyJo3A|ZALj3@9+#ybfHM-o62i-fSx!*Jo zNZzY8V}VPdrr78zs(xtc`vWRle+B9VJbi2y{=9J-Y|FPw>NCRN0eJ4hMK8i}ENs08 zE3UPdR3}In4(Yx&+|u%yNb`SLtvh}F-a#t4&IwA#_A+vcJl*b!-Z(o zGm?6S;yi+4+fkHl_@X_g_l4fyDTm0iBUH!RW%EzD^xd~EKmFdBujrWwj>Ya&U*mAv zs6x5+!@UBE2aL0A$tON^!Vrt8V&N%EKH%9<)Ze_1^?I3u`VEFDJeFoZ0;I=Y+MhUY z4-lWg=F_K$qk6!<+wSuC#BoTtE|(bo!see^@Po0V_yq1D6X;y@j5`@{dxsf6sG&3b zC{L%i#}Y@hqdIE9=HG~wDu#4Ts)K_#t+f}v2x}oH{q98klaCcse`%*JxP{HqN64dY zzrla?a?w9xGX~NgXOvr*wqO`U4P6Dh#}DESd%nT22VH@BP^|jWic|e59KgxT8Y+B5 z;(9DQ_ghfzC=ZA5ocl_Ak~COhG$W3e$7pUu{jDJmfmk;;tk*0EiKi&;cab=W&QkT1 z(Z_o-;v`fU^sRvwqPy9H9OaIU$9h4xYZp*)F!348zoAdqaDX_qo^74G#&Lp-uPQ3@xMf!LOEURWE3rx+f^khP|uO z(fq$bLfzX&{Q^iiU!Rq9%-2pcO=m;hU*J8@hLU2(EqF)i^L+=BCcz0K{5c(@CG$<1 zVx#CaXi{%E_1|rn;$XxnS1^7mii-Gkm~uW1ySGU94oX^!#xvJpIXz#eu<+{SO4ivZ z6&~BZMvo;A3LboM5L9zjc)9effw&VelYO3S%}Ik`Y{?h4oizfB54L#y{w+M;-;w*@ zX#p4io)jB`&jRTw?zgswY~kl6a78n3AD7?g>=UQ%#!~e@#aVrhM;vSkuRM+-;R^N6MyTWJ09Fp>e6K-w(sd(M z-@v}nYq3q6OVIzE6c>G;VdKOOszzmqe-lu+MV~>v=Ct==tPwR0XxItZY&xD-r?UV_U&(c0hXv&VUhD5;NBtYhzfr+5wj>n) zSnXm;pDCSnt1)2GVQ+ch>1)jW^ioiLu#%vGq4dw3E3e-lhP7`G z0O}1KQnFj9m^JHPz$ku3^OwT>m$L}xmx|E$Z6#?(P1kmJH55P5T0X(oH+SMq zy(WUN5eOTB_8mg!rN@!#7v5cd4zFIOf=-(lP#jOaq{&SD13eqI!4Kc|D{drg%w;p8u^5HFEt zag}qY+yTN#jQRN+D270Mz-|QY2cM*uYW*^FdlDo41{c!ek!nc%*}4?zewb&!n`mBr zC2R7`q$;n$i+3qKBtqe;pnqx$&&+`Tadh4RJ#S$c7a>A~2$_*RBIX=LJfG+7J6gtlQMOx0$nw)W z(&4((XiI5Z>%|Z}tF`NH^>nP_;M4Od2y>VdbxK*m$+YxJXKh%+_m(1X6t4XpB|5FC zZ}^?6PMWP&Rb4JR{Afyxrr??DJ}=nt@0s7%ts6=fybp0g6 z)Pjm#o|KUhJdu#SXjRq1a@yYu@Yi*it<~UpVL4=Dx0*7#-vq93t(*jJOVk&~j?Mw^ zZ$qd}Ij~teJ74f&llSWVvXh2W!EOTQWmq21)gT2nRdsAmbDcxm!1@DtjmNPpzD9OY zJtwsmv!0cg=Wb2mCVO(~`?;5@6~l5vPqzy2H;y4hh0i|0=xy$>y8E z3GG%dohYg#H*AJi5`jwKhP1~+M z?{<)2FH!PqJ_#me=#DNur;z^k>62l40yB7do;c2i4MVo<P9jd z#&FkyM_2e{yTdB%M1q2s61-Qc4V5K0Nv`vLPesmNCwL&2^sY(Ux*wr616#-uF@5E- zd)fKH#%o3!5)aKNqx|+d%60x>9JDx|8pZp7T?Z1fr7BjyQ}>B*6Zp9-PN=l31~mxF zFQL`cxa%P*^r0qqTUSvUCYdvChZQxght&o)gsfOnZZWZzB$ z<=yK4@tn$d-V1WH$;;3iEBcDZ)~Un=p4#Y-|2|;7w-EMFO$mQ$!P=R1rGhSp37*LpI02#t_QHEXCoyBZU?89#G7EBoZQ_LY|AU67R*F$dr!Lsu)*yZC~4 z86t0SL&vM)t0*EdpHRJ8ofOyyZDs*jb7`_Jd`Y2aBs9ycfoL%y3E8I~8=W&60gvQ9 z72l9Uwsn-Jmtc4lAt%zd@mH-SIz@?lgUaX*V@Al_BPtX6iL(bLQlDMNRq?nOaKZw5 zk~f&+ve{WbS2`l#u?z+j`}#N;pFz#GW_*^!jDSmDKEO?fEmiUNV~x(ypqB#$ye}&! z+)kXq3tBqJrCqD(xWZ%U!N$(0*9f(8{${i0oCNB@KPh5qaaKFxlk#maX{gnzW->Q@Y#hlj#8kZSjHz68PR?W(LqI>GpdlfATwu z{kmZt^W0Ybc|3+tFY+O-b^hen2y=!X7WdH7VC384zqm3~uUS=ka661pr}QVYfX4iW zR+b&eK|afAfwP@1&}Wy)x>o-K{}|pY=GpGCzLet>w3xv61ihff_wvd3{yC*Xwo;<& zvI#6>oQz*sK5c%4y!(l#`rj98idWJe15R2zCvH;bcc!!9QsjP-JF!((KE zfQJw=smYmO?b`}#t4(b-&2gRQk9ev6MHHsw#Z@J~o54#Wa>f^A&em$9U#c=3WM*D& z^)}LvbE~cQGW#>QO(Juc`388rieVyi{M7e!FTFc+HJf<_G!QJino!SNA9G!3G>&1l z74s!Fz1j28d@>pUy_L{3!n}bU+DP-+q`!R^YhG_K`%L@W#L$+*&AC(HKAQIUpnAU6 z*663ng$i>zv2G{Wbg+$V{nd|V-c47Sy>a1KE6uy%N$}6?4Z02O(Wv{9y@m%ZG*TuGc#l+auq5QSoOuk)?X^_)9 zGauq=k0qm?>7ipoVfYYU$=kN}pr$#xSdewq#&;0}=2nG20M$H=d|(NF2F?k=ek08cX4TIc&avIR>#nAQSi_t zF>l=m>-8iwE;IF-KmbmB?He|Z{ythS;Q^&TA|a$FlH zT3I(U@`w2E&oLTxc_$ghF#PuUM{UBqR6_@Z;enY!p0Y)Y1n`R;m|-yWDxpQ)ar?8l z6MWCeIM%!?@XQr3cdv^jjYi3`V_a$O0E?+R!`)=G*zlNHXOoM;To%vmZiclLv<-fI zT0z%n^A9Uyc7Qdz!JXO`HQB;yb-tpY!2)|t*Sm$F4jEdQ&9I&7l^IPX7q*wC#>{?W zkJXQ5Pek3y1qrjH#8IVuC2EbVvkFMm497OOK;RyWVRLw<;b?0OyWE5t8+5p32YC3tJVN{5ZTott3B4-CEsQ;^N|a_gSgzD?u;3y z6*ZvnbMU<~0$Pn3>vY1bO`e(`C?*9T7MKeYd{LXM3w=|rTi05_TwG;GSQal})~+z^ zBlr~N7*VRwhrI$>)y&e|dgGB({;CN?_81>$>3ziWAPP_Zc4 z=T@d@yC97NM`hE|elYH~`b^bQ;nsV6Zs^6U~uFL)n@akP5 zoKxrK(PwJsqVdCade!i~!?%Kk>~pn+Y-EWctf}UdAn`Dgqq zxu1&jo-G4UhG^GLLA2s;LHYRlA2sCcKEk=_o460~Y`s(D{j8QbA3ha+NIki9O0 zr;fZK>RrKJU!E)RF8Fu+VeAcYVqrDe*X+o5x#|2l zH;7$dk8%Bvy&3yKQwvp)*k9eg#^MG`yOq_A7FSn|y$&XxomWNp&AqQ&wrwJ(ZjDrW zAf7GJ>=xH;XhR=HZsc3nO6#cuLwRNABD(#&mioWbYk1F^rFv6_Ck2iQ=Ihss=ySCS zlEU~LPkM|O5mQ$w(y%snxvGw-=zH%e4>~{a3YSefpQu(}T9ZUQiJH~323jfoWE;!3|6JtFYM$~y(_}tfcO6{{Sh6`cPi`vm&`c zW%7DmM;?7MTGw4!LS{=!Q+I*VO)9p49;usOC9c? zqqN{roPFqX>Qfc>=ydeg_20P3Cua*%!-$1)*V^3jsO%!p&%CwWDY|~gSKpalQ8{(P zHPefV(cxyp`R<(=^5~KVJS*=kGChDgC)ACuzb=DLrL7hhZ)TJ2XFGCEhpx2u6XtNo zv+E|0*QncC zGl_M!KAKd4M~yivhL80&^+MnNThF^%Y)w3|a5s6rT&mXnT7-Ug=l(NB(olHll^U)b z-?%#e9W2!MLbh^tqLgDtCRtB3{A*cwFoj*Zjg=*1hqFs3ZE@RJnR5)nbB=N^Q1cGe zVf?widAkeQ=MSX+PCLs1laKJU#5~&c%NY+=BhNnN>QSZWv0FbbxMvZ6I{T9wd)4Lb zo9cQ#o7|Sm{5iq%2DIbCaThtTl#9N4YZsxGm05$M;j^Tv3Ew(VeCa_~z3Z#DcC;I+ ze(7DZP0gHgO@=LhJ?f+nFJHrz{`?T#ZsA%pr;)<+=$wG2bQI4-#raX0KR(#+VExJE zzW6!RjYcQ0(uH!J;a6u|*|pPW3QSF8Xd|IsI4qx^F5LZ)o|V^zi|*?#6;+kheR4=z zl%25NIMQSEhp(?lpRj@S6@22@FAXA5Uf2TYZ{-W{u7InbK&Ypti;X;vxSpiwm%I%Xx0{ck>@9#oDa2gGZaom^~BGQ+P)E-9~* z8|>s6fiF~~T_zn|lfuS3;v4S~) zg8B0(0k0zHtvdO5FV&afa+0I7o;DcoKCQl&Jl725Jo^h8J(bQe)fIG|XAQK_z|vt< zvhzV`j|${Y z!m~kw2|6L3PK;&KCu{a}!_V323R~*QutfasbjM~j{K8~14CR~FK>37alM%hDOMD(V zW#*J=7fb0sP9Mk|kEP)u`f$oqTzK?Ot;&9d;14P!exAO6bEp`f(x00}9OJ%jt@WU? zRrG|S`}n={I)Z2Mo((NzKx$4l-dOc>Q99TtA0;oaSADO)1glmRrY7N&G`UD3AAfn5 zPv4tOuf{hf@FCva`C7zodTCgdy3TGcqq?TiktI2@>clY1-K&+4oLqE{*LF!#pHB~< zR`u#|c)(Aq>$d9@mKem5G5K}##gFK$*B@}`KAIk|gtq)jree9KLH9hk%J__JK~ldP5n=DXC2?q?Z2OMkeYL|@`1!8S^=AhaFc|qkm zkc(65A3zQmXxLhX6gUnJ59dQGYr#8B=BjA?P=SRQY@@R!R_5_zW{3|ZS%B{pbWDSR z2wtp@MU>OwhwoVp`=SQ$e!Ujf1E2fQvVWoUjk8&5eSOF^-{(nuI3Wa?VIA%dsLtPt zl#t*~d0^Ex^vfPvyQ3W4s+3)WTNS+A?4L&8P_VUb7AxgW^anRjP)9O#3_j$9SL;Nb=53i?;XN+c!y7l_$<)5_#gk&F@V}T_f+vpD{+#OB zpwVus-ZIfzerKRov`fJ^2*;PzSDo+qk?FGn{U>q_-yXEJ)e<`JInArp;|mnmqC3N1 zgm@oBg?AUH65E{!?8=6@{{?KLwz{d~wdIz|nf@iEm)3 zPRMEnWlo2i9JSTnFtUOkEBBV~mFuPr481e{#XWC!Cgc*){my^rxd-66uU>K9*=2mx zb2h#dP_B%eov$1viFjy50Xg@`X=~+QkFhV;Rj6IjtQ;_{un(I|1!m^ZgG)tF5VBR> z4yc#BIKLl!xB7r1bsNr7?@}*XcW=pLWHO$1tgjq2b|G(T9mnuc+4@m81_S9Wo2$^H zbGh`Kp&8WtK?JY;PzGv)xm1|zhIxobY)&|h4tqB zD{@Nver*~4M({pywbn_7R}r$6gl>s>4sWSdSiD-;BOf;nFDsop;@JSb_Nd?iYuRK3 z?7KSr)}5Y}ix9&?|ExT1#2?GY(YTik?--*+v#qv{?|6N-K~{K^ zZtk(e>NnS4_MU!HG+2iB$4~W=_iw%;FrVQ#>07@eZQGIxJ?|<}@7&?m3N|&rz8&6g zd1hi3*Y~N4`xTFURmf13(=Ho0&!0z}IUx-1-MzY#A64`wa4Yq_P>A#9ipGbs&`LG{1ng&99NCM5_CCOM#x{WWhfD z`OK5s(3c1+Sc(1Ie{qCeE4e7|Yh|S0Ny+mfd*^)dwditB6`EYQd5)7eR4%<$5< z?z}ZwUsggd_JLQs8J?niaci@Bc5(NWR^)lY_gYLp`qX#H%K6w!5xp^wL_gu>DS1># ze^)u9U@WcaRN7iSYMrS+lX(Srg(i$R!>Cuw?aJjj=YT|ktU$e&#UN-->OD4JdfnpcbYJC8GkGJ zNp!st!!sQ(pts&wKOf5{z-J11f{+_%nDabQXH6WL`NX|Tjxr+uOl4*|$gh+)R}pDE zCHhBs?lL16_Nt0y={_%!e)2r!%3GF^wHSOPZUrkm(=ofr_g>+LduhiU2awG-7^Y^_ zDD|1Ni00bwCX?50+^i>eu3s-g`|TCSBY&vJ7b;=?vGs`m)U96NPrg2DzcTq1`b`s# z?WWE*H`2?p4f)=*eFDBGOuo21HK&}G7-nVw%FGWow>WGD?BUVl0>RGf7_3C528#9Q%V19#)o~{`_i*F#zuR)^f}_?i<7Se{J(w-}M7@6E#d2>q(W=FVYF+!iOKbSF#pckb# z@cH^Lgwgd@kK59fyT=GRYaQZJ7z~r0F>g^Z&F0XC4(_~hn};-WgJ~nCF=lksHpGt8 z1DBhhr!yWzQwQdv4O@z8u%CQ=DT9WU#%$TAfx!0|M!|DJ@=Ihp4V~m3ZpE~XI-|~o zwGw6y2d1PMx|QK=g*q3=0%RC|b95QPUP$ngf(L+kYichlD$G2jp#2OnHR&k0Bsy!} zgP9^Bqh@JZ=MbXhUGmD!-)0kf(}H;l@ALh@_-5 z!~I|f4UUo}vvpuIj~iLMkVO6CcZSupe~pP^L7qk;TT&l|Or$#0+DCu!9GS5t?!Nq7V!i?iY9FGZJl;oMbmo8}bR>rtZlUSvRI zsx;vr+*5A$4W+;eV=d4-`kPo9YXuMKVECWTJn1gMs@Ah!(+K)*I4z6ajoumdH0O!DN%?b@ zHf$vVDt{BFA9)hAkio%*6=hFX-1DF4PqmxI3gliIc{V_dI1wSt`F|_2R-so+UsBPX z*(K(q#z%$GeSTuAv7K8Zj7bu@DzGlx^2?^>vySJq=4R`Z*b0&KF| zkK%U;3~cnx3k*WwBh1Gla9?_L@~?axv$UHGE@0F!GU5&#LkUa3)Z54oAr|PmfOawF z>R?%6F|)OD+kMH*o{`(>`t?Z6ZX060>!)L*>hMlq3uQ0ZhF5R1!MwXB=I8SbhY;qQ z%IFGaObXwtC+ZdKiFYhK+FC&O4Q(nP+|D5#BNyXdzk%GZNmc1T@D8~|x2FO{XVbx1 z#bwp&QIz}48BQyasRGkavsXeZv1#l&RuejCxiEoOSGJ`)8SW~1M2s998ceT0R>J*` zfqdM$o!@?c#Yv01>mtiOP~Xs#d^tVdTG4M2zB7dDuSl~Fc{i8dos6WHRq;;Bwl5QB zOm^URYCq$1WaV6=rPaA6m9e{_&JAlS+gz+ng&yaVNgqn;JPmr&4`kBr* z-$c-@^#yhDKELSo&pg`h?jqc0HJ_^c%%KApi+T;~(U#u(f50(S(r0`8rS|hj$ir>7 zi0|hd)ex@`{{0o#KJPxjZEl5gxQ~bYv}B{md1MPkUrtJFwMSEG`cHxL)+ufusKDWd zi6tKH;11#wHRyPf*LSSLo9rj>zOHA<*~3o1cxa<-3(V2@Gu@(nNxgUNGgZFa1NxmS z)N;6MF4;6>i9Atu4?ldh6X)i~*;jhfSBDzXtbeC2KY884q1x%^Sc!F@(-C9&?&0FH zt9vVU4Bx{Q{R+_ZQw#OU`%8Jz$Q-K1_fT$WKTD+MJZdRcDjRR_UW*0|7>8?qi|DB| zmHXBnNPiPv()Z!*rRVRb)MnBdg>~V!3-Df=TBp?Ho8wiRQG-N|UN-#j!apG@9pJ`+ z^EubBN*s0PxjF>@Zn@7uJ(o=MiG!rMlW{?+Y>du{Qh9 z!4D(VqRe01=HgB+d(z5Sck71y#q{l0@5H;4MHqXbLq2+Gr+y`*S&xSe!*pb4CtlxZ zm_|+L^*wPPzvo>p7mRnBU#x8HUF^PE+5aU z;rT9e<&i68gO>gEqpgd$UacN9wcT;ic2+g6pJ}URjNZeQ?3U{2JLUBG(x*8`r_yrI z@@TFxce>o#asi=7Sgt4`@AW&v>%JD?)n#z~)wyM|m(@$Z*}ahBU#?dfopXrTmyfg7 zC3AQNoMz$dw3y;eCrmpmO(o1F9 z;#z>hie^?{Ca{h&{-p~o8Q)9~>v2J;372W$zR#kv!?mpMk7}HcU2F==O=C;RuygLAV^VJr~y*A89$-x}h3BW~sQ>&y&+;!(&Sp)na|lm3+s_riqo~DN4-kCT0rj|K4J|2ocP)Le*8pP|7$rt_{?NE zzwQz)aI_r91U1k{INW?T_g$S=w_UJ^_op1-(;E}T*nA85b#x5bc&()3mu4&f31y_& z^Y1fOaJJGT6xK(LICF@~f&K2rdeHs&9~@$XXOu_dGG*LTGEhkRC#xHJ`rW3_l>FKBPvC%Npu`sn=wAp8Ia(X+S z)-Sg-`&}|BznqqLheDqb&R<8wSkQCdsNcI(x;`eEm(B~PRpV#rN_K_l^%8Uc_COld zpfnY279=}-_(CnSZ%}!?Bk9V%wftc6F;2)=!rH*sX)~uT9xh+|YtD@K06g z;hSBq-%*GQ#NIROMk_KsbiN+z)W`7?=xRR6Rf->0&K~o1WBU~4nZK$`E-+YrZZnrh zRlIHav?e=sz`N?l#h$Sm4GYJ!AP%`_aKCr0bkc_?PCu}KkL*ez?}VIc)$k1(eW~C3 zkCZl7K63EQ{nj^rd#sBxC&SC;Nc=nZUA#k|{Ra^vz4}+v zha5Ghq@EL>T_kPZz`I^%=jUkXM!c6P_8u2)cb#5_mE$yz?FtMa?zS5v z{nGNl-)8ZGIv+(`wl{2ZWfcD#+g3*186-v&txJD;rihR=Jv4ky)bd}bpeg*|Qdx~P z60<2mffESzf*PNUHCk!?08skLD;v&j0Kw!=SGFCQWkGDxYd_Zm4Gj{`mXCv?X zim12MsPMyKqTkaOV#DtaYG330gw^T1OCSBhfLu zIy~mBGByACePh+`MxZ>_dk)=hctO2-@JJZHMx8Qh-RM4LpEE{4=hg3Axy0^AW#kU~ z1IlQQ47twGE_z#N8Li5HPZit!op9WgQoEyQA^oh6o1LRSiFW2Qc=s(_hWMr|ZK;1! z9e(}?$1_YGTD!@Dy<~8hm{4jzRUYXitmO+47>MBie7i$o)y4U^TDvHW^YtjDcVurb zqMn?g!VUKc_zjKS`rP{JK|coXaIsTm=-q*K`cL9~GW`m6Fj{B)fsXI&$DJqH>YfEl z&>{CJ{9@ug9-EQErY7OVw6S|St>3Yfd>v1Sc4-$>r-OpQ1B_#lFRS(ef9#_{d)uf*7?*))Q@Ig%{h*-l-GEhqCkRoa8@(X_;k_+*M;en?Gw^vXVTz{WsH{ z_$;r<@ZH1}-w#m(cGl9+G<>cER zB^ur_j00{^XN%=DH7~%U+VIQF5^cEhpo5s@o=vRfpHGTP?t1vJ|6H<98tlD|5{ za?4M*2wW<_T?GHJ8V|DlqeTVWcLdFotA>}hj61xWolnky-ri#9AuqdkRvAV`9q9fm z|Kka#E*r0*JacLi^i4u<39KUatjme>DNkh;o-gujEfu=d+zPGY?d{Jq^vC!N;X8C` zzpH9-+#m8ge}eFNvZn7T0)LCP_4XP@;whguaPHDE1Rc`Lp4BjV06*An0h3VuubbI; zDp*kDXr<`Ds7_+WkW>P@gU>A1fIlC+(mS`%LF+3>vmV0b5k{*PC<!%chfT zj;W=+jl%KZ{Tx4N;lH&!_g@+t<^qpcz-jXR$a;7_X?uF|vn|*r6q+6;47-6?6iDUoC;M<@gAGoSsriL5Bs9PVEnd_ zqZjy!9zJYFemAy*d!Gu_3BeDAIj8pyLE8AtsdtCb-x(C;Sq0C|Izm_FV>zqS=uA@HpUAg;@rG#V4%3CLjr9vOyP&-+O zaUOU_**r#0v_Mm>+t#(>nw@bUS&p9USp9VtD+ClcMA;jBGt9sa{rYI{=F^nX*xj$Q zlZP+fdzrtb)$VQq@WT@ZOB3{&P_v?F>?(6!Y|b-J#8OJ0uuwr8W%tc>z`KrF=e5DM zo)!9SNvnRsFs$zX*Fm4C;vh{;jK1AdvODfci_dO2h}MklM<&CTue&Ad+_Byka7E%< zyM5X<+Ftnton`PcMd7;m!VPQ5Ayfa5I$xWxe>|aL5%AM|LT=Sx7R?aoUoyREa+ArD z&~G-G75+lVJnG@A%HX_I)u82L-k5NhrWejDtG}-xZF82>?v+=WEU3Zr+$m+2(GyGR z!0d9mt|EI^DNi?QR#xAN7u1QP3ZO=q=9fE5!8hyZqg^(mU(2bdyE|kt=t4Y0+T^K* zkG>QB$v^KkV6abfA;t-U`F&@OtmXKmxBYK9k4v$iqE*uM&NxKjuGi72$bnIpghpF}2&) ztlCC~;c!bQ1^uMWmCEqkAvxvza=sS-nX?J~u8_BkmkVTb{*at5ifpZC*oW?)l6

3?7`jYchD;CAU-26Y|N=Az)Ia6x!*f`S(t1nDSlx1Z<&e} zlAqA3J_dPzmj2C9mc7-Ngh_&Yk960m$)buoapfEHmY>#A9jX&@B1y?mm}%Pj^n53$2C(S78QeD!P~Q6q1f-)OXO@ zTVG*RN2E=x!2j)TAk82u{_O?l;G>Y+`4#%@O4K^k+Jo*N>M_zxs^_D;CUf}~%N$zP z=6Ua1O3FjRuY1HDd%-b(tzC0a65|#Ek%d>VN~-vES1hj@S9;O zt%^-~cKKmZwsiw}OkJQHAg+l5^6ikFa0J!$=^B!>gRt0K&Zs{G%4aXiC`GD0ks6x~ zq(?M_LFkYlA}n(6s=ko6z%WDUJ0wj_eKi9XWIl>|h8EJ|*c+tYgf?nOA|t$KEPSNzLiJ}0YCgp68s*7|e_7DdVUQP@1i?!iGt!0{!aD7K zy2qTn3VIDc%7WfxlQvy0NV8DiauUz4J01RHt-#|W=P>F)srTbzyYW>#7IkSWkj94W z6w(iSrqKD28LA&xrQ!Wv9McdM3hKcy%413T5~$8uR{h(`%MorbQTQi~=FI?_3wYPt zk7?oCG$s=bna{fkq=T|Bdjw%y`9gd)%%5Vn9;x4;`Ylx7bvbz=X0dq%SSC6F^|6?{ zPB0SQy7*?g;qlj%;Fiz>s2(#F{*%wE(oexX!UVlvlcz~$k#pV9w^L^zuE&bYy})VZ zG|U@r$ty31Bk58oG}Ni$EhE$UgfUmD__uuGO-5rv(j1U|q8anukjf}-GB2_<8s)u2 z)n`@tuegnT3sS#J^%7L=QqO|+@Ta~g#h5VGmD|r5D5(F8cH{3O`3#MUh4L&>Wo==< z?(eETg`vkPOsHkVEz5eS+~fYE!npSsJJs`J)TdBuA_zmPJfP18U)5JeW2a_3ZSetx ziCFnCP)mJ5bo+Xakv^^R1HKl$ka{*?S8F&YKIU0d?^StN@7{q>d1kvHexw@N&d5_} zR1cB*9_ZB8hZ9za;E5?fo*l@es2Hhn1NCu`Uc(@dZeX!@0qvubry<@Hfo>m}t0)4K zkGmkk>@N_vBGns@E~pOA;(ihLH$(DpBDnP~^s`+De@y?OL$_d5IaR*JhqUHAD4lN& z&RrdpS4R6)y;Q#w?f10PQV)ZXrX}pWjSjw6(%6aiz|X27y`J>u)N9A|ikedOW{BGr zPI7ub>4#Xz+Z`_~k9DTL|3RQ0lhcf#b;5X6BOX#Y98=rq$p@B=;CR+z_?R;SWP?;_ zv?vLFyS;^>r(#7TzjN5HS%TQ_Swk*awjDgY=7WE04>|18dz`$$3;Jwa1`|GQ;+C&$ z;c~!oI2u$N_4gzQ$C<-fdQ5*O0a@3Xx;>amqc;8!NdGf5- zOv~CDKDb&oZ5K9-t%!dB`U^M8QDf=cfD{+VFf)*bU+?0Mp{H3zK?JKb+oEF6@sF?6 ztx;DRPaOg@M!uY$kLxt?BKQouE^4=|Bj38-5NB?;glpypw6spNm-c>a6ztFMx*5oX z+E>u1$!s(@wg6qHQ9mGY4$GoFtlBz;@E@Dw#VVI-Y96>-w?1zFTLx+L+$`mR=Ip^m`d_`KXArtSg(hxQFKkc*>aObhcENrrbJtkaV1R2R(1r<`!4IVWQneX-Lm6 z=`S>vV_7HpeC#!LE5ub&JVa{Ehp1llh%j#@1lAGPuLjR|4abNopa;-xj`P!4t9i42=DSo(f zW&n5TSSdc~IWo^|d)_(C1g)%^vQhqee8JoeyqDD)>6x9(E@(Z(riXfRu~~Jx#t+=@ zzUNHogzj8BI|F9~>GSr7-(rGYJ)AV6xeS+xTlb_Ptuus}TeJE@ZlRh>>$=CVo?#ho zJ7>TtR~XeJ4|uc!c8s@@b1&7E#TRw>xdHdF;`R{~X_h=|+b@;>Ja5)YuKT@>lr2}Y zr=5-X1IwMbug-q9_~mS#{U(dK?W@5nDlTZ>x()T@QNs}eztrTIi*zME; zXiVop-Tb*z;fHXWH5WeF1PGS_mtpSa2J(V7k6k&rnZ-^&#HyYD2)`TO!ZAewDrO=v zc?>(?G70bc3ZWCZ9bbgLM8YNB*k>lz{yPcopQ#~x1eCzY$(8uj_$iFOep=A@z+?D2 zpyMdz-XeXO+QgPGx{%2ngJUz*-@In@eN@+zNHu{r6YOQ=*U9{j=|=2dVWEUOo7CR*(OgYc8+s?klHvKPcf}f>z^S2p5Z{ap!N_q21&?*rB2W?#fPu zoa$kaN6)DCNQx2qzKKBBlN1y2H`GviCQg*Yjc>xMI!SD^CRWDqtr{=0F=C8PSN{IA zD}HNY2M2xrV|Uw~g^?>R3pM`+QJwkmtZ$I=bUTjoy9zrx2gx_p-$R?%oYvo`sG0;oMGieIs7TcVNMj>N1PYhp>DVhhB7!UvS|*S@6LFJI!_g?QCZ}oM6Fd zPPlRKIsBNL0W**7V=;ym5IbZ7Z#lz}g?d=R%j*|}s^9(0nO*jrf~&o|5dQyxJKwW` zYN(1QcFuhVIl9!J>}L-OS3QQ)UYofU8iV`$@pNJ>OetJLJTsjaKe(;%4aY5NPv<}H zW62Yo}2+S2+Z=?djg-YPm3IYd?r-PUjS!?gTvxcjDAmw7*%vAN*9` z3R8Yx1t@w9bB|nxb#n&GmO<69dWW@=Y7pynw3baRE<=xi+2U*ucbe-kg<%+!)C@<^ ze%IP%dw|Bom&{HBRR^{1y7P@8S>QUDz5_OlMdBdYecoQ;%??OB2-3Epob~g)xV$og z5hsg1%YMM%Vt-U~ooJOa>hino?H9jHE8Scfh6^~Q4?eB~T*{~2h1 zcskbTG80tph8Q`^Yuz5IoCl3hgqZiYHl5)ZPXErpFLCY+zMK_~5!Ly3uY*8bLY&!x zudE(~q&Y4MY%8Q|pA77JG-QuXRAl3Ccd>o)^jaj6$ zfS}zR60D8_@qDI3=0P^=v6D;Z9p~`EyB-~zPYyh)?5&)nUVQylr(nUiqv{Up59 zw*fAGS;9Kp-XW%UqJ1gG*szEz{jqi3_H0a@T}U~O=`qz*{j*`EyI@-VO@hWN2ER1H zp{E*SsK;)695V@fd(>C?qhiV}YilVlnyMOO(+7rON5gH{{I>x0d0?_Z^t{tW5_YNl zQ=bQvSCDDBK;;hmJgyi|&)tethurMl6dWY-Ve-cPSYz->APz>-XN=YsNF!42Bq{CY zqUyp~*B9{J4#@a8icen6P|d;NNH*LZ_!+~x{1R@Fd*EeJZH3XeWSN=5ReajAB@ZsC ziNl_sMA9)z_whYP!_eaDL!}WUjb9v#eny-cs&uO${3Q%I13DpTnb((R%714=X+5vN znZrxu(6E~FL}nLR@T;rraN8X8gGS58=S+FEpE}|@jK!0Gw?PY!*@D&*9hW-E z*SpF!TRQT*p0APg6Rnj6Gip)-N$(jN366;6Gh8YcSvG?I08Eds(q?3;cF2@^zIz-}ZR!w3sB#4$*5$Xqw)ChjhH z1@(C#@o6=*K5r(wochj43y4JpI^1;bSh%Z6z^vovkaU+U)U{+eF6)qZ0b6*Kz>_~F zoU{<1-YFZJ6)u1WwN5b7B1~x|w|#+BFP^f_ifj#WCn{aPfAj|+EXE@(USi+;Y~kNJ zkK(%zoS)o)uKFB_lN9&hmvw)!#N?;abuOfpDId;dQasUkeis>VFA}bP`X;XRMqGb> z70{e;vDsQUy!|9*KN$lhJ&kc_%y5~!Aup47PSpLN;e9s#hGWs=FfL<<_<7{M=xVDk z%eD*A@KLm!aykQUWX0eHgXcnxgY+i*64a14{a2yg{)hI$zIF(kTDWnSw$b?Q=P#go zhpQzU@nOG?yq@7)%Z>l>sq+ux@m1d^I4{?;czcsPb?WAu)JqVn4P-z-X>-T>J=@KMO$P&-D z;Dll9{{3$JwR;QfX!(jMt+f4*F8i^Tj#_q*=(G~HaN!qBy7z^Czf|=9o}Rh5;x~Ja4iB_$k$nPQiOS9#Jwq6u59oz7haVIoQ zTOH9R|5u&-;$5uX&=ar z4o7+dafgaK_PjVmD$ddQ?*;@sDrbi*?!#NVT^i%5xr}&F#2BuGnQ>a^Xh{3x&fANM zmnPQk1%6Mr0BL;1%lx(Nck*kUnZmAtzDHGD1g$}po?q3}QMNUG0923cx1%L_fQE9a zK>%#lG{!?cg9XJJZ@xc)V}9gnmh8C7#;3Gl8w@5%;#8>Ktg%p@u+q?hw>s8{a|hsW zM|5I-xemB6F%$@wtMKr2;wC5xu;tzz51b}lEfqgHTna<-v{iaeac{ek8NxQfm}h@$ zA;~)^Pe^rLgZm|wOR(W% zehiVzI*${?>0mo=snWK5`S@>8yHSerMnLO{qyf}_`99ja^T%4L@=^>g{timNkZ#l# zUkqZ|SHs2BRwpRmsQG_*+qliBqAmJ#eTJZ0+3&`JA z=|*}#l3o^sL-_Me-C`p!BJ@WAj)j4r`TF%w)^IE>x$a zH+sVmdJd+=KpmiKLdUM{P`xj=!*(dR84JWM;*!GvD2kYe^Bqkd$ zW18TM`LH71OEx?^0H(aS1!)~VGSW^Ib6uF5*;dlJW3}`1ka`cG`T?XFWbcJ*F&8VBcD# z#jX+e?^isHS1yi2@;p_%wWH-;I8$JRl)w1Yago4x=E}QL9=MZ^C>9@|T37kz$1%^T zz6Y>+9rwWP)Cp`=+ZgKod;`-yeUSVZSbm$w6qb^|gbo$Q@lfAVR+bt9=N1GjuOrUU zS*aeYnsV}2&cqGEcY8aS96J}w&rE^G=eFSQ#zUFzTUY80=-|R#aYFT18U=r6q;tj7 zmIoRA&gI_L5gOIIptWc8S%kS3(x_hyBQ1r5i#XQZ41WD>tgwPjH~+y%M`cpKL}m`r zm7RC=rhbGjd1B(ZWp6bL+{W;h(YfH-!2pI>hGB8WUO~DWXpIP`qe%}Ypl4zso;mzO z+x3sP7`ZJ1HjYVRfqgf@x1aw3eI~DSg5%4g&204kCj7VKZWuUlYE|z+X@2tQGA<<@ zNMmZEH!sE}_g*7;dpOd)EhF#5s82yXhrUSjm4p-IK^vWDxy?bH7#Uh`~xJn^lYw%|h^9QF8v z)T>~pM!F*TdQ5q0CgGkjlypfH>$4h2(sk(FJd54D^$sh0wZ`yBy1~h|A(AdZ>Q_M{ zzuwSew0;hOQcd#l8CyYVJB4%W)|q3o+~!r>`**m%{C5D6`W%w<9Q#-@i#T^K^#vM9!e|`0 zFp}v+nh{4H5Tp~4a9ynSX~Z+TG6+hFSA873T9~LIt*vR=r7oE1v{QXFXwkii>f0$i z1JaL39+t*^4z`_YuKHUrtt1~?*Sv}fQ|P^_{sgT>RXr1K3#Cb9gN{6QNC7 zCpJLEab???hVoGLTSfY2D?AZ*F1x%zK-gj4sn)1zDj9Trs_MS zMGexppMmm#Q5*#Ii7+~BBU9_$C9j9lL?ZZ(hI(+xDz7;CH>}-qtClp3(jrwox2vBP zQ;+2r+g&hKt%WAxTO|R!_T_BqvcfUpVOWh z<@FiqMCB#8Zv8^Y8*>b(N1=3*%C*eFFT*h__!*sb8w?Boo`GjeJt0@y1Fq9E^3wzC z(R<}I-hDZTrKe4xU4317!}k~(y{IY27wgCd6Qkgj?8Z~7Z;~gb)}6C6%-A$AJdeGOZR*taBFq_L#_k#4X(A zr~^#$Zi@?@(!s>h3~4N~UPM>!=(idlzTbv-taZ?!ek-2zu#YtETQ0)&nKZSbGZ8P> zr1J!vK#l9W*#`DK{Wlhd=<{%W!Or`)fjsx?(BSAm{_@9hyk>X-p1$jkm6tN`u8SqS zPn-z3d8;(9AKyTmlxlKtsttx}4&jk2^~A!9oZC4q#*Fxl+D{Lw5jMAi!fnDaxJ~v=u!34P|{53b((^;r(hq5DAaA;DL5=Se%psDH?iS(APuF7tVJZg_~%?=OUqV`c-1<2cp~(oTAN3Ed4Blhz z+Kc#CFC6nfHRJskSg_;i>v8U2kUC?BL44DXLQI|whkWy3)$NB6{pmGc81@nC-(Huw z`cW#aMWI-}D;Ybr{z-EVVm0*W`<#IF@}16h`PsJ?%n5abuiK~KpP9Luy9=XX@1)wC z@WWXDQ6fVzoB5M%|^;Aup$q%!oJU`^xA{^wZ;T|EZ>u zw>$!xtGy)64+?F6!S1)=+{SGIzc6f`d>iHtYv*jk@XQ*Re6X23`{R}pSvr zbTSq8 ztfZ-n z)9SLp&RT-v0%`h|pjlxknr*huoZ2fMd;6SVK||9)t&?3$kgVL)9o&d7M_m7aCFg#y zzQ1<}pS*T-2K5qHSN#}PEJ)Hkx!D<7-#kFNbvNbqNEfTM)Q=f^8{URp0m1~#-oJxuJ zKY}~fLHKDe?W-w&3Zv;~FgLFf9nS2=u!RrQ`)ED2&?G+&=Uc_l_vsqSownfE=n@jX z;k!onM4!wHFyb`b$EWQ?`=6W=db-~<@mtQAK)ttYcv<4FPZBrN_X)J=ULsQ$L z3vaPN10h{J__o!bKyOA6t%W7$6B6;%ni%|iq95-+IA65V+m~rut)6`NzM&-ShQK-# zh2NfV@x|aFmMrehB7Pb2QQuE$96m)?tzGrcbRJevPp)c}un@O9MX)#BoFrka*lnxN zpS+w8v?eMC#rCu3sip#D+q<3kg86OPvDeAa%(FStO%3u)>U`paXx6077;bZarFd0o zAYb;K&LVCu##?3%a_aKN7&$nG{yrgNM(tBzQW+x^)~vj?no)gH{sj`=?PVz%eLk&C zBic956kb0b#NQkX=GJ?L;KK!1aCWDIlE#OMe`?bC@m-SkNE^4$m|asxZpbr{#`Egn z-)K*?{mVNXqVbEAp4dpT_ zUYP%KKRut)ko#B{3*tqfn$QprFyc!bPg`1yK3`v&T(D&gPqttkWou5H0!7mvvD_}T zpefx8Y91H_hyR>_!=?+lsSR;+r9J&#E0Z~`4c2-*5w|$BksFtF;A7jpf`vlOsaqM}_lIq&i~b%X3&S(|ER=?zN$LV1MU*g7O-#Zv7#M|EjQki$e<5YxD*` zwbbB-S(V!6jp^*5p%KI%kF;&a_z;%;AkLU8Lp64&^3kFDO;lX}?ph2Ke}6^$+)f3; zQ1F~IR6~3tR9-pi7!cPqh5uGX!rYi|KpYE0`{~LSze=Ef;zC%YTbK9Gx(zkf)RI9% zJL8;Rhw+c+R+!&$G}ajTOVoTf4i0x|FQ{e(@dRvmeGPWou9JjO*pPVxVH|q5@4!AB z%}{I3hng7kz;lU=@ zu*r+gW22(q&}+|8Tou$5ORwfZL-r6lbZ9JJ1q){GT_(naS;(cQ!!a~x4C+U%Vb*(c zMK^jTf^dRQ&wYWT=GKseAE-EZPF5=1t7gNG4_U^3PyNm)$Fa=&srE$kG*#E&emF{y zz9S6LL;H*WTnLAdbOfd>s*X`pMss?vOg{oV*pjbiljC6cnzdjc9>nccF;5i#hUi`eJR2Xh{^(BqL3RaqPDy*N@W+%<1qvl_- zC+XgDcw4s|9UHjmi?WZ%J$RE#JD-9 z#rWbNQ25Y)@OVs~edvE2U1eBROBWUsK}A3eLa`MSQR2)R6Gg=C7Q4GpFDi~C!>|gCTuU^zm=#T%fLHYA&XlV zebWRjKq9Y;F4{eafiWCEg++-PK9pgH@gO(Q-ZB=s!x& zu|KP)pobW|f$!*d6UMLqEnEWq6(gWKIXYV}5tT95lIL3i4V=#6>xBQxYSV|k_AKV( z7Z<78C%>p-uUlK)58G+fu-f^!qJ|Dck4F*oOd9HglbZivqY0pqxbvD0`cjR<{H)e~ zb1zisY)55qsn#5e=(;HxT;`&G+^MgoMOD_&pM;v^&>=6yvG^kd4CA2GFq*e=4YW@J zI}8h?d-*IfcGq3L=Dr`j)q%{qnz0-0XmIjm_E)?%zPBFsFNb^-a@g>v0#=D%`Xrmo- z;2C>h`1I9-EcRcISh&qBHDyf-hlM-IU)xG%c{JlEyl)<$4>o^AACG>zRnL`1p9%1{ zUz6k84A*F^fyD;74s;|mPDh^MlVLII{(Tnq91OWazaF`Pw-1qb^h`CkZY{d@h-4?6 z_3Bf#CHHlRfY$w(^?Oab#4>)yf5Niqq4@`@*X3pkV5n^RupS-sEhC*T=Fz}Y0`{rZ z&+2B?_@;uj>Gd0XY4B~p)UE7vdMA0OXX8CDQy4W%=p9qz)?+$2{)E1hisG3BdgwYUR3g0BhQ zFI{L+Mz0<=n$fR3bH_^Y*y#j~s}Mo(O*k|64NqVFlAv|0Ve>DLXFxuQ>(k0JTdDcn zKIGn5Tj1{qnk~!Y07K=*&*fyuxg2czWaj8~3Opyu?n1_RZ65_rJ*}cVj#=Rm3B00B z4;Wwc{OA$N_~|4XZ*!*H`|&;^?iu<}RSR#c9Ov9dfBGrRJ(vA=Osp$0Tn}G&j^X7v z$lcZoZ?2%b`F@2~3fGX(zk2JZTLO4sJdr?MaO(=|ti5bnkkS6&BsQJ{{!2`DtVBj@ zqmN~k3P;7bBp<3{Eyx|e)D-SNb8^lV8+h=CRz~O0Bi|gvb8RGiy3sZ?wqymJx58sf z@`M6h>P|b}Q8+}Jc_{j10IakiHgxI!skmK&l73duLQ-n0;JrL z3IFX(1)n+ciKlJ2CKfP6}vBSvJgBBHu6`S{u!}tg{VrmutYf+|X=`c}03lywfzFJI`6h&=Le4 zN5E`@pL8gCA5|P5#1;HL)3fRC%-jp(QO8Gr!}m{bgz+n7w$&t)hg7T>D~bl^<=}s1 zDBG@`mf(@g7}&(nnZhrAfrf^(u4@uW#*@X>dBO9qoTF9upK*`OOw{Te%YR=M2xy!v zti3dKf;QY$@OKM_=3(#>a-IYpdEl`^-bM2dI&fs%3_>nMlM<#;WTuA3eJIq7Emm+7 z?JxC&&-=t6wLOX4t(pEYZw_P5xvhO2y)V^;T_7F|zLEG{B>k-z$A169OI+udQQ+Zj%S_{U7ox>()=5X;1G+g>mTA z+qtC4J&^sNw#zE`QBDd_7UTbbui|0gbGr5Lz2bM@jJ{-eLqZoolY8_Q23uXRnG)^`>tUmf z_H_NUT_BTDCTBa+sgOdRD3QY`gDd!l>vg+D$S_RKBH#}Ne48*H3m9YiiID51;hyPR zPF`xeLSJ z$s?Pa$~UK4WZ`36Y6`DBQ;|&u^S5oF;YN$O2WSO`PL${~HOQp^8Q;2~&{~0T<~{>!u=@Bkti@f8~y9Din<*MqP!LQ7+mBc9fs*O{vLr?y$TnCsxIoh$U+24{N*V1W!LIxwSs;dm3%^^(MF%D zx?g?|?tO6&EiLOtt(-UVi+BIj^L<~%v@wsk&z5@9E3?1s)GL=|bN6f%wR;DLW?U8_ z6%#mTD?~WneBR{qpB@?4!{@ z#GlAQraZ7HrdbWJdGO~W@zJi8Fe zJnIjSo?MB?-+HL>J{wI-mNcUQ#qqt)?vXsg$6i<5kjecYd&&H=D9!7&g-^?6{Iq0Y z)YwcNv8tUOFS~0~59$9}a2?S_Z~SkVjt%gYCG*uL+nAL+_R&l_RAf7+xK^W+PA3>O zqxTG1Ak5nLa{bJhUoPKd7O(AGSG;KbO9kV3vv*ZaQB3r5Q#-sA&&!abZMv*)&^KeeX{m8Jf|CCk-kn zGef7-i@b^=vwO=S-%9F%J0hsXn)x(dX7bp|C#d-5hn(4N3jcB|VCr9__H7~ki!>AX zJn?+rIQlK_n3}huW~u5h7mf~XECzcPl9L9-35)o}^BV zPm5hGzf;ZIkNBqZbmcqHTNW=hhGiwRQTx3}yDR zRLSr7ai@a{^@_Th##36?s?~YrJkE2iD>?RPM3+`nSDWlO;Lfk|=<%IYBJde+o4Qlh zO&ZG2hxk&Vh|jA0f*`qaTcBP)!%2RA{Yy2OyMjD?T3X`!*kYcSa>F`qB1e&Jrq?M? zKu0-HrPA!M(z@o_TO3+?sS4hQbN8N%)WZhXl%HJQip~Mf8hE5IAE?vqqH==aJj#))bpMK)$%a6*qL}b7gCj^^?0? z{PQHw^}EKcU+z{{VLEm zJld_gT7Ef&7mUTJUFAJE>8qzSbGO`mJYRe~Li|p3QiBJ!Rpr0dm#I_c2!nTbx~~_J zGbSpWF`|Lt5_n6ud@d@~mkzKk%rzd*lICZt9TG&H)$P+dO#P&APx$O;dfQ&`pr>}y zdD~O}JsH>|od2=srJ8a{&DLyi!C*z971boLf%b)OLoMB=DRU3dBRqe1d+04Pc$mA` z93HxG6XCOTXx_V)*nkvry4+D>e@hY4yP`I8fqvn3m##B-LSf9bWS6Zr{jhsd6TTAM zRKE*~q{JtADWheac3jp%SKG3hhVA`q{d42Kc$PW9a`7>=SlzlJcG@^yC}NBralwJ6 zybBP?dp^;#%;&r_*A2QgYmvFObwaLM`uv3Ktiu|~w!ebq!lW(AuWlWKaWZk*JZidX zwFivP%} z<%vY#2DmyJ1dnGZg{T8V2`^D=UrL6oTr3UX@*Ok^hO(bBh{C;l(`xJK+ zCrbY18gWOt*L^(CAKKVb%CDgQ>2)l?EAyG<^Xo};`!LQ*u5*Aq-Nv$ea&B3I? z@Ux5!Mj16g74zH@I|sBeHA~P1ddKHOs{Pi!eD+mMHKenLs{AuMAE{8^A2_G5Ce-24 zTMDmNoKD`*Y?Iqr*8H-XJ%--oX&;uQHJngJ4n3bFe2T}@Qv3VpC-}e}Hd(r%WWEFR z>^t;owJoA#Xm&m3qh`CvFg{w$o(xaeSB~IzQAOnJ+pFoW+c=JJ&8MNOM0Ana{9@BV zj@TcEd!J0;AH#_nn#ph~;WI?c+eK`2!u#S*(sHjfA3b&)?=QEMmii?y&b7j(9Vm zy`g?z;lFUYihTaXa2CEFPG#@|$Cj8RHf+34KBt>%^c~-I`^xtkH73j<8_Y_=o{5G- zr(Mq!bN`&RKu7Az@BWfgO9#4qJiA2wQm5_5jFzG1j(Cos>F6x2hkjvjzQW&Sad~aG z{IbNaa%4EI^(;REpUS6QN}wJc4MwWM4o8fZvj(m!Y;csG9&qKSdAiWd{`vUX_x1V$ zeE+zFPKI;2@NpZ)-|)fpOZiD;JTIMH*orzNXi_n4UU3bcz_U+PRNG&tR7BfF>=Ths z*E+j{-#n~fc$m}L7uByW#tVFow7Ws{=SHYCZ zIPRjtc}A7#h9ju%sV!EcuTaC5ujgZQTJ|dp9=1Bn_Gf%1UFkgwTBb>hcev9V`A&e_YwIC_{V>fhKXhtKV9deD{3~Bp&xboqn;9)Otr1m zMxxdv=93oOa7ArbGdy5whidm}CR(g=hv6xkqE$kS>Psocs%WL!x zp{KRSg)5>=ufUq#HoVS;6aOZ?WsI4V?mQ8#lD87{1e?z>+5q=Ss8!|Szt_6l zw}wu0cgWIY8NOGIM_}+FiW>#l6t||5^8*WDT&%=uSu^d%{^8G%~5=N zb8p~YAYp!0$c}*;Yb{YzSsK~s=}P;)Fz`nKZwR#|lFwD;+jc$pLiyFya$ExN{Ut+# zsE!`rY2uyw!gh2|4jd4}MOMY~*5OMSxGxPBgEOe=#W*tfi+LCI5>j|TL^b`@wus)@ z$(`c+%qE+G1!VZ7%?z(24K|Fp5R1>uVV&^m2dAYuTa1P}`l%=l8`X7Vim4;@#io0)<1rQ1P<&!z>HX;dUsiv`_zug+FKUXx zP1-iqc)?X1R&}$wv$V2=worX~*vOEIo|e>;r!Bs@eYNSc5`q2UX`8Uoj3*+02q)hw z^yFMCZnUQg1-k$+on+UOicU!3l8xwE%D(&Gl<1Xt`3~+ekw4cxMDR59_taJ~ zarR{JfP*r+4mAr;wHTgnDjVGVR^OAiZcQ-yTEK4$@GoDA_+qpHLx&me65t4H^Gewy zxI;q=kWbEQ!ryx~L32}uL07r|wzY&YGk96S2dK?~4cXi$xL#hm8bH=@{fus7%&E~! zroI%eul6=5$No7D57ql^|8h5l~i+{Ns4rHo{TZs0+|GnMD1 zy%cW~6ELV+2!EM)NC5v)9EQ5^oj?)E@U}I)_@`bd$q3YA$`HkTU&}^X-1|vD#MasulUUciTUv zu0{1JmD%Jz@Tdy7MDTtDo+2|>eSXi#$_}2@do1o>S!7KfYFMl``o}?8g zK8dnmY&I#p`FV{h433b4f^*9#mo0?+M7?#d!Qu5* zle=?v!rxMY=YF0z;e>*BwZ2?p3(fM2Om=f}>=CNgurz@eC2|VRHb0wuIL(_5xP%Ze z)Oa5%$7#U-ODrEuw?Myu8-|D(HxHP8XUwArE*gcJ?Wk`xLpEJ}zJ?bRy@#bzp~vs2 z_T<^hXj3y5WK8^NU=8S*?)0nAd1NoU$atBF6M{@_GBlg&^e>9LCYB}C48i{yJ*kit zkn!+Fi>?XY%2k%kqP9!VlgUp?{h7)08o#mbdh=VU{?+u+do~O{<~H3MYVauolPw=Q zx@pHA#}zm`wWe}G?s1zY3`jP3uAzT<%DXf|HfCLsHi>ax7G!>s-dnWmjJhPUSk^s{Vdy{Ofsk)p&eFKXL&5Lq|28Zm z)UR4PdoeWEAVv?f(OU4_%6M4hj+DOg3LTnQPU3+&h5A#b|5N*BSWM;=I@1}Rs|tf_ zX>jhF+=7-U_;#~y#&fF389%M4G4uCyb{F=E1u}Zj_+7*EBkhXG28B+s$=ZPHn2YAJc=uAWxLv04M{Imjw`V)E zG;+RvcF-1M3D>6c6Q*T()m}A|$m9a32TmTeN5zcnK@%5D6U%;AFnN^fa%~Ck*q+Vk z8Jg}m0A4+TFmGh?Oq01}4!3a9g0eJn=tA>*e0Xn?pZVQ!Ij&Ogw&84YFXT~lAa@~I zX8Sru?jhmP0)Ro#+~b&Y)@0+^UR`vQ$W|;LZ$Az&+I7(UM*OF8S?W@-r19BoI2?Ws z`Oq4bIo!&RzaFK#lRt=z%bn;ETNyd4bSsr6%+JhqpFe%je`d) z$hhcv2ENRH8D?@klNG{)k0E4jS(*r52OMO>$e0wgf`qo!(9sN@CSZ}pt43vlZXn|| z(Axraqk1;zM;%+&p(7=uo znbRR{&JQEUEs^`ng+7Z_hsW(&jjqhT*B;CgZEWgJ6W3FDEU zc3BfJx#$j2yU#hX+QCyn*9veH4W0N3e4S~y8<@Gm=mO(I()#*3pr)P*csnwB$#694 zl!h)WXU(~{s%m~Rr_oy^QoI#39GkoznV87^uQ_9Fh0%Ur7Dh6%JoM30OLr$53EVT@ zmP3wjAm|0)cMKUUH9Dhldb+Sbm5V)II>^D(WBL4r9cf4QAF;Ndb(wk|?jXN+nnfi8 z%jgMFrReRIS#o)^uiWAg-uq3wNlqQg>%>x?dS{0(+-PG(IySZdRXCHx>1Y11Rm|Z( z`Ld}>3x|t{T(_y_<6@$Jn-p3U8%4Vk`mo2$jykm~_UfKJRwT6bR_*4Olk#6IH?9Ah zPgR&D`T#jpU-(yt-S11vq~d zdjZ}p$sx^JaJ3Ul$i85p4(-q!KCYYia{CblUR=xv0!N7>c)$J3{5b0Tv$Vc{@v*f> zueD;Mx4quz9gXu+YRD_S>^U#Ki*%KE@g{eBc5jtK=c=u_T+ec{k?TzInOIP-vOhv! z3Kr$e8Z+_TaSZ0FjXe6Gw0<0$tgnx2glDs3Xkp(i);agLQ2Lp0NezIl|c z>l~tUyqrjRR((p_b~Bmg9-Nm}vzm*V-U-iNuky6M{(|S+uB@VB3;g7`x<1lUFP(q) z8m>Q7c!hT%kI;d@k5(~wqY7wRM854&NuRzOEpkSsVju1sI@b9kx3FKqFMIl>^&Qhv zZ(31^f|g$sPG@uI>#s`5Qp09wtRYug_>PKQzDE|Dl^l9@8@Gra%x6#ZrSMsW_2A>V zb=}LQ^ot>Oa@#UnN~*V0VE%dZv}fu^fqZNm6v;13#&W(S8y#HrFn#u}Aulx_sl0l) z>ipF%@s-qxdPeGH4(m}*=W)8gi$etU0{{(b$7NNLke?j4vzVI3@}5B}`EoAGx{o?g1;moa)s^@7@F>Id$y ztESFA$V(Sk)&=`+ucta&5A&YS<#<4N5goL=y>2slDc}1zSz|55t(s0+jp(7VzItcV z2yxxFxC-iA73YTB+0?`BiU)C)cmx%;^OW5R+~cj5Ppb98-PE(*EP8fh z4bMKjoU(gm*SN0SA6}cQwia?~mx?UDo)mu$=j4sQ+w1YQ?yJuuJImt90h}?zTkNrW zYwgtKscPNhFE@^K(5Loz$s^wevGd{TGPV2&?fR##L?4k$J2&~y6OVM%bdp^+w37!r z&ecmF4Us;%_HuERPdD|m6{a3%&+W&kLpl1}QFZ2$zlL21 z`79ezM{ZoA>|%xbx=Y-+-+~$k zEaV&u?Pc=U-PX{DA9%?RC-EoWRC-mI>Ery*mK~{!+1)*Z9|!JJ>-IdOsIT}gduy? zvUOeu*|+8vg*9PIy+AoY*P#W!ile{#C|-Vudcw0_{%#{NCTZ#|V2KkyZego^&yQB9 zRf?FnUsTF_7Ux%laF;$+bf+%;_)C`sSkFlAGJGm04qXwjYH?rrBrTrG)GtbPYWCGZ z4nFds7^Ztydcbo+C$QV7RRXxF(BE8qgd^vS@&`8L$M}Bm=S}mpYh%()I+ilML*t^W zQs~_5Vno$QaeR=X9&gfV@77BE@4l0sdSM2C%F~L9kBHX;M+I{JKDp`QhJ$HS+-%g9 zDjmprJW-d)|5(M1nk9i7GHe^kd;M=y{g263)SG%1Q-(Zp+j4jRy7VOSuFCayp_=>8 zm-{=<)9r({b3y-N)U7}di(hJGVACJ;qn#d9K3F;qeJk>KVBcc&&BqfJl>P%D$YPnypY zxwjlQ^JTC^%y^SWPMq9=fVBdj!PULjV$ajv{5WTRnJvVe@3R8y_t)alD49#&n8*38 zU#RJk?c{!E6R#V65VcrX1N(sq`L)N)8|2%(H2r#fKCe5&fSfMd2kCtxXO{<$dHZF$3A z3N9sJsoJlfa-Y-1Ws=QId49!rPWJ7>vuzs4`u$@VJV@weIjqHMvgW>~N~V8SJ5mdi zxtD7Z#q|8H54hRdc%I@G!L8QdIg1D5$T?!RN|B4%&;JiUOm4#V7u@uOD;`<3`WJgU zEC~?o>>GnGH+oRs5Bs!3jwgiArQI%vMPz|eod4ZcUeg2TVl2k_CvA6u53BQ=m(AsY z3FX+olrKFky+W-G?#`;ZA1w}?hw(Y#*$wOmvfqu9=2q0+Ps_^F&;QYym*sVzc1Kk7 z)pnFGDS`(yenf_I&IIAT=;g~eF8r4YZ1qgVg-;Sr*S}Gbe@*2%@8NuYMHnp*Cn)l0 zIc+$&k6jSw`c#jCGUjOST_bg=(sr`Kp{{(fn7bIBbG^WI2tB8VEiY#3TQ7FFrNB4z z{qZ(csFFpS&vzU;O#Q8Qm4I{9(rc^5a7_93ZRFV#cz?mREDd%aM{~z4AYeJ!ZL*4= zN8`laG2>}YLN+Oz*wc-{qq)sr#iu=LlEDRN5izRSeD1e3QZ1jhjSIf<01nM3qdnq2 zUg!B;dzv1?IdjhX)M)Hm7+saOCS@@Chfe<8NVRS@qW!ubk1Cp}0*@c1Se;!1LpA!8 zOYN*8*X`+#`fe<8eLBqO8Co)-w_I7+MSh}Ovc&rbs`#(HG?@o$)C#r3dqUu_)WkQh z#IA(L7I3(j`!5B!R)lZweaH)|k4ZD1vp@U*z8mR4a?TzSgUiyD)SnD4q0KFe1q`gQ zl7P+Jz}<<5J}E@$hoUUGeO_W6f@Hp9Cl#&>&f8=?QRodH?Ky@3N6eT*;5J^gu_bwVJ+%ZLZ$Lkf7SvlB9w)3Vy*g7D zXB;|f%pc7O{UPGr0|dB%W>!gN^aVJ;S+6~S_n-bbDAW|*8HpzJKh3`=b@K;lTx(fz zQ?~^E){k7hkoB z!n5-$HtU}6SYv&$&l36cZ9iHw=)2nVG}cmdwVg&?YS|=OfDf&v9yZ6-qk*H+X<5;= ze8VM@mw(GmHQatFU_0-2sV4#*4~h-*n_<1-y${x6)VN-CrWk&9n6iI4hxHsry2UZR z?7vL~6u~>QIZHFR88}plu@3Cfs5(7R9@=p0oz*Lh=GFTy&s9Sv*^AwCR`KewXLxXY z>ojN;Q)?<^cDzX5?#Ec)wCy>7HFq=V-Km=T`MIY2w(Si%)~vdAxqpCstp1Gp;z3^PvDGAuk5l$+;u9^_W(_- zU0zoH8%74311}b))7`pq=-g@K?%sks#kpp20kp2}+;9!Gclu^AI^%Dp0&>UnUIH9J z;=YaL&Df8;@mKT2t2ZV)#U?yR5PWXRNM% zOSuQECTN4K-iZ6yfd<$rsxrDAC2!xa5=$j;-hNK9M$2kkFZ_xKu3Ar@em;d--AlxG zrf$6RdKuIADy+mZ`g&|7Z=bi1&(aZUy8j0C&+NqIc9bOGiWS&u0r$(dg_5B=PO#CJ z&>Q;gy&+AxxrouXc@?IAHQ0iGErtjE0;AS*9V zz++25kw*f!A#HG>YLNsyw{n{R0Ox0wtbXtbZMg?+xJl>1`pmkf8E>sK1Os)m9mW22q2e`yAsm?-Y7 zcGJ-5a(#t#qjeZ{D^ErRvsnjddgH}}!}Nv9RWG%a{IP{z*$=dAZyv+B_AE14NrhfH z;yL5<3{2+Z+a9IuuW>Yss|}B%7pxyv1u5u5b>n`9GCIe5*N3cYho1FPyS6$^Ro2*|YT8`Zy7R)>%{_D0+^v}Gh;CFb2 z|8(Oe_{;6ys-YS|vpU4nBG*J79vDJ7gX6IG?I7sZH&pFJ5j;Dk7=3QAkdD>8#of-Q zFgTS96tmaycj7qW*lsrX@oggZE_50%41bO+?JasXog_|8{z~9zv8kLtYQM3bOm4dB z-R_LKH2y=xWV*|aw-+%q83R|?X#dGL>AC0O>twtzbcfMS23HjP6?wgsylKTk=z*h# z!?2&KRRI;J)R1+s>&~@L{;E`!`-DXghj{?@KbDPf4^ZHvUpQS9r z2U-vA{6#rW-On0>Q`0M?M%<&8Up{lY`J0r{I6qUh6_{^4EVNTe39TS{ya^0I-w0?3 z_HA&%@HEwR#$K+=ZkrybLreEk?Y)HrxS61Z3Eq^(ydJ@G$2jY2XA5y!`~-zMuo}OE z>r$6$6X;T>K;gE#A#dcW5}uB`BvmllfsKD2w6d(!=_U2UvgJ*$rk?YDCvp!cE;r=y z($#{VlhG9SpeyibcZgKu$ZBB=-x^PdHSxsv~T%Z8UDEVxW9ONB@f*Vx6uu{S_wLc zfVDinR;V%>8Qz~-2m8@SrwwfIrgwrS)RfA3We@l=guf?^Bg0XCVUeQMpo_d>hrKp^ z4g68-M%VFQQMs!g6=UTAwn776vwRBHP9Z1J(9l+|9?PxPNxRj*!71GN*l-1ZD+ZN6 zrHseu*FCR1@3w{DJtLcHf{pOn*qZK=%+335y#b;aK5$Mg!^Hi*jgG3+F z5Vc1@i%Vb<_6+MoNoOw#;IPqjmKM>kfTJ%s_K#QAn9%Q{MaWULrbnc?2V^8U3EZS^ zwyUh==`NyZW)0mv>^bs-63{yhBzhCxz>&s(*>CXDWE2Y7mAKcx3!9t>xX*Oa0xp+Nr~40Y0GuW$-Rd;!LC6q^SjH2{sv7#0)!%usCR= zSI4#g3H{Mj0~ZC>78<$=&aquCfQ5R?g%>8*A>>r}{a8j#GH{k`rjFzzmDgZ>m*T%a zFmM(g#ZBkmP*ofM(s#@S0Ss2-zxZ+Uk`EbJP4H%%tG1+>4{NgXxLO8xCFa%WNQQ1l z&a~W$9D0En?F*K1_Gbl`%yePEDChlW*LiBo-3heLej|@WnX<9)-JE~s~S1aHy zKiydyTE&*(d3n<4|17s^&L`ui9^I-Wn~qQBc=t4Z+Pi@z{aQsIh_4K0J2gUGw_hn_F{2{H~RDhcinK+%paGo z`qz51-42smnOb1z+0=nG+&NQk703lSB>!fDJ`8xY@f)M=H8K?oIE^p|gsew|^*Ag{ zHWNN;aq5z{b*Sp96ha@W#5aRgnRCMEQeiUM2JZ6(`a6r?%G6)VSJ&;y@?s`?2$);P zFuP3>b*eDL|9|P#mJQZW8^X>_h95yDyZyFlY^%orskT=?)|MN??Q63CjQmLCusI-+$$-7*0F@8`XWwae~SBCdi zwzP}Jow=qi`+HfzzZk~@Re$$vYevH;TfhFKOC`+ zQoC+o`)OCxT3+kGSAoa(rZ$$(_CBP>MIW){t0R|cG*9mB@2ana_*;S-CXubZgQMYe4EC$(sOUs~*xCK8)2;aY=2 zIBxl7k-oaH?0fnW)w=(l|Ab`l<=&mNKKq&-;4k(aD=stJ{HJYdZLmHZxQgvsdB{-> z8|i<2`q7JX6F6{$gN&L|M)zr3MK+5Yqgyr&=Zekl;Mt9AGJpPZmV8U5vizJcEbM4iaNrBFT03 zvVi^e*=3oK_t+nF3AxOyLVcFGa&ui*Kg~ab{mWgUjhADsvp%=sYb72k`)TcT|K)M~ zfLd|U)Z2Pd!W8zf1}R)yoz6a5Pdi*qZGE$n>kM1Z!eJ||+0a4H@IS+i{)JHD5vCn) z=ac`+AH1!2J3X~`CE;GAy?&i?o*m2Y;j=MEsbuUk)xxcz8rkicz}yhcIBzL&xjV*l z-CFIijh^=Lnecr5h5DXu&1q$p>6y3jnpyBH?Kb8y_gJ!%Q3H(m;&Wq5$q<`VVea#I z=QaAp&xYKW^6`(?SE%q~Z#jB>6S>-^pDM*Zddbs=>iNogG8xZ43>aq7d+#1pJKx!A zaa!>_hwJ>YYN!rwSWRDe4gEbX0%y>#=1zXKWmLMOJih+BifHjw%<#l>AASYd_V`(T zxynzYhQ(x@&#`9o8okl|1%FOUR>9sA1FGz}Mou&Qbo)uQjc;Fmw!+B2QvmaB-*S(_Q&P$d5 z$_eJ@Jng^wGUtwi+^20K$K#!g&ts0#;szrLb)|mg8p4G~h4J~GLv_$IFRD{9g-~PW zXT0uGF)9~-S$#;gsLfY$G3t{_%*jQeQ(|<=aa}TkRIx z@s}d|BMx(KJWSO~%;c3-*zzJWfw~-@ zsN3CWAhX%tASdU^RJBE8u3G9hzmI-PP7d?s%kuNH#^Y0F09|&?rk^@~P`MTsrvl|P zPi$O4$Dg^!LpnIfEx8T|;1_lBJHXay9z&Fv@4!#nEQwb`ob%~Epf8+ST);1|ADe!*=w zM|0kRwPno{o%Ho1ZB)0(MJQoz3SWJ@l}z2(`p?v{RGNX&a=^?kbTYLx{dc_&p$^r! z4eAm^|<54M^pTC_{3}6r~Csku16bA{dSfx&oVgM5ps|D zMW*KG&G=9JQ0{tHQn;MH=w?8Th_AeKRg@Li;w=zm2FJc%*5cf&6Uy7Z6&+mZB@UF! zWu107iFUr5O{f8#z8ZU^w)xKr?htVm-qPkPGdN~QU+NwCj)wl6%Dvj`r=8nYs4n5y z*QxGz#<&T%K*#1hu%--%?Yw7BY1M0zUCYw-?Zo?A8wSWX7- z9NOg7UHofuTaR6;XlfTrvaP4cQ~I{0^p?N;yu%gJKT>(+sI82iqK>&|%He?*cw_KB zeqdirR=v57OS`7hjlRbS%TL)$4%b*a^$qXFwEu96AsckRM=#jD$TeR0V=W)Aa+&{aW4|nZoDhOz*fHCjeSm0vjb#ydRQiz zP8HswiapuPvnKE4@Yk{2;bt1G9pfUa{a1){q=(9@Z!5ClNA#VkH+Fc?!sv>`h!DoxC2Cm(rALFX&-FXY^9wqZ~sqx3z zaF|EnI+_r(h~m%m;OZrE%lX6LbEclO0$-`j)h23SMLgp^eSx?%ejZ02*+`ui)fa=k zj*{UOtOfd`tDbvombJ#t`qZnyX)@d!-R8W(WKp%nNoDTI@CzO;C6??!d`NYNqO|;tMfVN*)}>o_Z)ch z1f>ipDD$68RiFN@vH&Z^A)F^BC~uKj7YMUXkI)XQ~~fDj$wodrh5AgP&Zb1?{V6-8XuWPrkTq zO(lyJ^UOvApl0cG_D`IV^Pb3aD?*@u=v?)E0vyGGY0riA#2mdW#Z%TFzD!Q6K8s(D zc%|?+T;yy)4ecgIKi^LypVc#Xg>!Do>MFk8a&xb4d%&t>=eo)Svp&wUp@vdY#f-*?<2@j;n51+V)zl{E~N4jfipV`nR^& z=su9a58U&?A7wMqk&o4zO&71Y)1^bt7+7xEFyR~3*;9~m?x>^DA6fb_$D~<=9#KzL zoH0GGb~n!@7Vl}nW=3r$xK6^vD z*{lp7eKcLA4gJUlZvsvo6TQDy;I(5G5q^){)+T51+2|@8O|KaYv951?kn`8>PUu-N zyQC+9zs0?eHozi3y}XkXs|K}uk6~J8J zWl_F-^Do&<{$qu97SP{x;6V{tsGzN^yx^c1a&E8S_9p^9zkE)yyHE11$#>Ms=jTO4 zh#lkefoCN(dV*T_tRRENj^t${_oyl#8o-DAQ-uo7W^gB?=b^VwQoCFQ`JY>iz-Jp? z(!-;)>0y4ex{A)(zcr)JX!o69@+tG2TTM76=2-mX;szf$D*s?rE6E?%oI=u9G zNSS+C_HP39d=PGGU97fwihsvJ(==mn0(%`?q)w);7Y}#NA@4=oxl^Ax+-qqWiJFwC zD^+^SWc6_OID!ULyGr^Q&g0X6bE{J08$pL88=RoWCra|b_QfoySsK^lvJj^QzplGg zZQW8$*BW$&FkXrA1LHfJ8r55C%%Ngk&RNh?ly6BkxhK5>!(XVyP63SH(Q)?{npzMY zdKUt(%~iqb3MF|S7SMhA+^Zm?^Ho^bAmdp?=Mo+g8eBs^;rwl!d-wgPN)dsw*U}Bp zPG?2-8!poHjb(_R1=vII3^cLkCuKPEY?0!!?e_76wXhmJGqz{)sVQ-{|1N=bRW4^-)uVRLx#o$R7I;P;wlnzC*C==R%!&q8E`5Z9m1(@|@uMhp;|u>J@rlV4MnLA#)AS=TR8z39&xMC0@C| zu%5B=Jzwo>H^UKJpqwUyLZ!Tn>(p~7`PCE@v%&mKFp~ZD_>D{g0vC*vXUDD{z!fVnQJfv5Ssz@Wd z_2;gAvzb0r+j<|N=N;NokFsAdXLUHNTD*d`Ro83A0Q+MZSS?U%wB^PJ)pq+g1zjrY zb{g8hQLzx}}-gDT-6wdpp zs6=gW(c#(o@|NRj%Ao*R9eWC34mJFoEUUNjhOzc~SyWqExgY0sMbyUrD@10l;ZbzP zya3&6Y7Y1rdui_Lh;t(apTjoWgC%^M#Z$>FzX)H-WkN=z?b;qsz*quK)7a6r%G99o zZoMP%e$DNd6!vmGH{DQ?Mm0R1g`?2eJoN7n^Z)6J?$~SRqLqPZ@T7UP>+E9e^}d;s zL#q(#+LC7x_8i8&9r?8ZUj7L~>k>S#sP_loyIi}j2JiKd;A(KsIl_Dryn=9#iseS< zo!RJ&lP|i+SXyscS0sZCEke^!w;xyuNWAVeoK8p^fy_x=9&7j((6|*2Z$%Ud3_dVZ&kmCPJL zms+2!2;nU^MjF3E@FE27%w7EVa^YcfO^uMzn(x-E&+=p@!)fc`O|MQ*r~1Q-Y4EZ% zS|9f&Yxu9?@7e713{Nkvb~wU{@iXv-tNCrVLdzKp#5sRaY&c*do@brZ^Pu`3{D>Ud zL@{(X6(0573a?__ez=jw_1Sm{qYI$t)xHAX)$uBA*yzCbv$As$mq8Lb&1e|-q$Wmp zikA<|(6ElVkRe=B;Af-DxN$}lba8t&dJo*9PvCteWHvf?;yyZmd6U3(x!kq7+UTNA ziI2$4yYbD?G2FJ+B`Y!w5m;#*8=Yiwoi;gL>1$wwnqFeBFlzyhQ2})ZQ@`E2*m-vj zjd`-5uPufLEA`4F>+Q{L=EvZ?yj}7<#?n*Rf1XFkR}7~aZ(xD0(9j52^Ij6VmY^RL zw1d&O(7)3N{i3>mCdRdZ!}GZFtt68LNR#1=%TcNmY-f0 zb%p2Ox6_aE#nR)XY})j#cgxxg|6n{7K_~Gqp3e_j&Q!jwx6zqub%2*4R%Au!&Eax? z=01jBf>vM0vp!!|>3EIJ=yvdpHXaWd6raMLL9*RkzP(_U$$%^-6RVhWI-wsVI8npz z1Wb;NP~c^Ov0H(sykl!wdHu*`o_p$(z(2I)Db5RNwF>)}Z)T(0n+Lg3;;n9~{DBhm zDd?Nhwackr>;#RHU1q2mqFm?-ew;LwTD;7s3?H`LY%dqi8lZyKX{+b@m25p>r&~65 zq{egH+29=fCByforR*9g(dXp6dpaXCQD6TpAfru7_b993p&4}pE&2jF-itmKK1fN} zliXl3)`Gwb@E$X1*5sP0vVjpXNMTfNW&Uvj9>h&k*a_@WOuK(M*b?GAy99=6lzHz_$iae(dy{Bj5XFp zTwaRV{bC4Si3;MmX6SzS3LD)$V4E_yJ{sq=f$NmfQO460ul`2lyctc-~&3UxnJubgS)R#fBdOIG{O7xmzZ1FC-4xJ)!6qHeg*y%9$2%vMv2Rrba3_; zn%a9i-?8)+#h*Fzm_|e)S6->nRmw`^$&B`{)bA6r*&oO#uan8Yet6fnV*V98G?mF- z1a*E;q0g1c@xoq~mPU8NGfHSP39oNzL&Q&;Nq&P5o2;39ZqFl=$ADuQwMWML94I$~ zz;(!#PJkn}iv5|jB(gbcfx+A0VUrl1MsnF$qYq3TBA_Xx=j*Wy3^blmEg2!@7?+%e zqg2S1!DgW!&UGMvz1C(o5@vP~S7fAKTDzr|R_^Yl|I+VP> z)~N@4Cg~}=jOHpaoU>re5F{>{uZ0La4ST})2zoB3ts*t8pc#V&OCLW<04uEwZO^{uhwvpw=a*w4d3g_ zjIHDF`M)=CEiwbHS8oH8Iy&=aAG_n6RIe@|vmOx8XQm zJ#WbG|JH?x7pKdSn`etSiyipur@4HQa*yezjF5c|ZeeNsZxKC2h#X&6exnWT%NAn%Q?C7%4KEf>yiCq*Ya z?X;ZZ4Cc0wrlo$ks9lM;MA~@$u>CUc{7)>7ZopEf+=6$5iZFk|6mS`@C!_9q$|*Z@@z&YR zvdaryY561``b-=zE2SroPJWCFD=Tr`#Bp>lE8Qp8DIG7=+KkP^JJFoy!i|0QGS2@j z&Kma{>}S=~eB&?MbJ~@A_hFpBAGqzIeN;xr!m?{aolsE0hK=>l)QxU=_V?IG&#M@%_e4-z()(X(Mq)OU*~O?pXPOKVyDd%i|v zX(j7aM&P_ZU%>YF4{amOd2~)#3a6)Dh34UBgjGo+PUDVoEjEg_`uAD0Hq$W1tDkIi z*GqJFa))v2z3@)whPcYFKALnlkq6>7p_P3;+!CE7T1!he(O}4 za5TDr#}em(S49&(J*W)T|0O>+fEcfw>P+`hcqOz}tCn;2*6w(tZ&3ElH77Ptd*PPw=qT=QC3W@E*I{@aO*~^0EjmKD2q-G$8MT9&wJVh z85>!r#0s#OIuPu(yc43Kt_O?E_h8oS<6t+ZjWkL8j&?1g1dSP5 z8ujMe-G8&Y?n_a9_nY#XVyj+pNT*!mLs*!%LQsr}@GrJ*o8&H{2K zK0}&U2s{4^3jnZ_Sr(4EPJ3={Xh&n>2IWs`@I-akFZGzvb&%hO?UTAOOAn6=wudhTVi)Z2M!8rn-#89m- zhMUuh<*~$aXzEr^j;Zbe2Q6!J^N1YqyA}XnF9nOGW-Zx<@#U=F`K9uCTn*VX?3~cc z4CU$ibZ1)0hn@<@ch;uM+$p$zX|3~Z#%n&i7G8inY-RLLTpSd`u&3{NZ&WYn-W>pY$)6M6X{mSQlnz-~ z$Cr4O#+rCnx;ne!mq3kK<@Ge!dhOj%c}~jXZ)~|PZL5s71s7_+MX|U zP1cOK*#qBJ{={+4`$br8rfi%%M!K5hFya%Wxs?aKd*a;ky*SF_IS}u%#<5nC_z=IB zY@r&>WasDYVN_#m~ zA}*5fAOGuasCb3A-e2_!+qZBF9BKE6QEkhD?Y%Ioc?|Aw&|=M&OR&f8qv&>aE8!a5 zD@WX@Fp=IX9vEKE{^@g+Z8;Z(YW`EsCJDrp=rH>vTe6YXh`&AnNUt!n={>L{?HsJGMtM)_HAwUrOFcD0{QG7rA3RQ=o_RvY zxd~_-cxVY-*Pf*T;%Ih8?;-Wi4@jQY3zSw;SibDq9mP@nNt27{ZLPDlrZT| z_d#C@MxP9Gpju-=;T>UaxpS3<>XpA2nvbmxH@n2KhP7j4R`Xiu@cAT=w&KyX>Y@Ih za>&ex#EGHB;J1hLdP3@`D1Gx888HxrO_oZ z#wCs3Gg^ii_2g=-iMyoZr(EI#N9pWhC*L28=l<~U0OpyOHbZtNQ4U+zX53XB)xDX;mN4axkGPsY9 zN6V(Cl@>+9C3$g9CjNS8FWWshz}`kRk%TSic6kEMyF)o~;#Z`4MRUJoAUz=RmQ<(z zdxbVu;c)HxH6%V3#I>~ECarg;>6&fUY!9k>ta# z#Cb4wUsG^d?=6*UAdrGQhPMjcK{h5Uho1EdGcd-aOYywIrUjFqL zdrp2J;%RTmW3D#D;UBPl>J3I5i?^JekzOP1(ujKKB-G#D7Z2A=!PqTtF#XYX;!RrH z?MoYY9=B0@s&y&rW1mOZJP@d!I32Ut;h<4k3`fr>0K#@b`adnDdfeh3QVO(0>G zH~Lim7LEVTlxl1g=bgUt7)d+gOG``D7x?8_7^9ldDy}5_kZUIgGNn-$-{_;b1~R|2 zIvcsNSjnb z8ng)J%-;p>7fVt7KJ^I@_Eqsfx2Xd#@_q}&wXE?x7f_m8@oD3~pV*-9TVdvzML0R53*@*Vr@Z+hV*Ea*X$Bc_vlU2;ub0>X3AZCt3RKkOiq_J-oyK3ezmQ$X5HWJcL)&n&9L zlqdJSppO^4u7JVdPB?!qtqn&wllFTJG;8w#lwRNNzD)frs2-)_dKxo1u1kW3^ncZy zHI7YSZ`$9W-=q7qa*yEXZjpSi?o+J2Y#k^McJxMXHZim(w7&2igP(4Kyb;@B_~#S& zC!U_k@M{1Q9FGF|QzQ=7jE>CIknZ32x_*CjbIAX50yK~*xw68P`IkIRW!eS=(vlq+c+as-+AqNf1x;xbcP@u3S}QI zAo)oVlJ5j}&!mZiDfh*cogHB9lUPB0q4WjS^D9s|)S-E8Aib!L54$wm4TS$V_}ea# zHQPh0vIJGbiWA&gebW$j2pV_gfyI)UK5)k|i;k~8+ItRzGY=i$*i5PQ#2g%~DZ)KX zF0psbYqF=QSCP&O``cbncmoFmJk|Fg)e^HR*+yPFTo`6pNHO`IATA{?64=V{8Ys=7 zdP;E)WicA6Av(^hpyqFt-xg5W4CNU-@_q)8Z@|1AXBcTo_NAZ!jA=6!;wr7=P&vL&}2aIr@D}{|?vaS|&p98GG_3qsZ5%RMlkj z&fDqQ(IG*(3?qGj0rQ#x%uUf6%aG|&Wi!7e*;lXF{; zCk}^#Vf#dOHD3r>wib-d_ra@7Jxr`|0K=BhK4JTJP>yh#-rt$KFJFasjs1{%0~D^P z+=(N$z$Cgi z)~NDN`z0G)Rc$N(Cr=gD){ z!+I?y+_j8LS>Ua8Jkq^YG?m@r}8Gc zUiTE}-y*Z}7m`+@UTnZVB+f;>^8Lg|Qz4+`I@oz(KKXlVZ^|iAc~j~cB%cIh$FIi7 zcvGOPN;Iiu!(M35Vp)U+w^TnyV>$p8Uk&%XPxY=x#}Xq{#*lfdGmxi-LwB@n#LWwe z3t?NhFA%OHaX9%%x;L17>N!oJcy-4cF8X)olqV49Uq$cv>7v=b-MBIWfU*{#`k{S? zCgaxeI)XGNWhj$n{Y9I#ggY#2(0TE1>{3<~Ybh1)x{t1-bd5If_d;#bj5AuDe^YVr z(6-X2av3U}Pc^GF3cGx>2y=7YRes4-u5x-@2&md4&xNJa3q}3ySFnburYb9?3`==7 z_UKV#@(!yghbk0mE>tc`x=t*deS)4To((GJqGLqznLs?x#!c(3FjCTY0lklWZzW|2 z*OGBQH>k*hZCm^ zm2%z!W`1QX^*Ui3ye!|X@-U&eR$d>@4{jYL21E?Rdgf;_Y-n59(oa{KhaVFG zHU5Be=ZE-`)}M(x(FeTK^TBa5?SY-QOT4uD2<b2(t)I{131gSS%`_+M+Tx%%^@7%!sGp6& zo9~MS-+P1RS~MOQmZ?4BVTm)lpW~$|Wq7OlQONt-2Yp_|-S;IFAQTC#qR>+;)XyV=jhhp^MX`QmBZKHdoLf^Qu&dG-8$as5I= z@gQmh(lPSyHQagj&&OE1qOQp7_K)4(u|?+pi^g469W|bmtBn88P5yqp4YN{Rfwt_C zG{#`ws2QacJ8@n`AY`7e&09X!mBpXDW$ul>==UL9zDO_!ntxe>jiv3o`n>(3FW4(N ziXZxH0%?2FoW{_PdnJvPCf)itYWn@gL!TqidVB=- zxIYv->%7BWS>s@TND_8@Z;yNH*~n!vI?Qz40~S1OIi6R~a%o(6Y1lSQVUOWRk5^0` zbEj3Nd{*O;(6sqZ_A@Cvh1k#y*X2J}qL$i{d%%ox4APl`>CO0<8RE}wRp>59jC4&_vHkkES{KAJEW zyXsv<`V3BW0H+Vz4UQjjG1klVD`^Za^mzd zVcO*&^bMN^g@;>0^H0>L>;u^fk(1PCWr{c}@ z>fmpehE!8>NpubVq3jG?8x;d~lNa%hE~5p_ANLBf1gc&5*f18F#61(ad*|@a7mi|w zLEYuC>J53F@9Uw#>pWg;lZY89XSENOM2OVb2&@^g2g=TO@J^|8=kzx-(obaj4(rNZ z4=K}|xEV@fItl7ScJGBd|1T?!FF0LO9vp7Z%^s~2xi_weV@-auR~>=Vc}N;dKDXC) zsp?hDDgB>#B4Z$ANFN+{!j4LR=&nJ43Es~2vo29!4eaB z=v)iZW$nN$v=4v#aWZ72_|X`Qk?RZ7(b9C0u&H?fjFL?8mGu}l_&e=&I8m2ZEIcQ? zygg+!t&b4ddY2H6?wUSFZnNoaRtbKhA}9@O#C?_@!6g$~Ga3W= zcVG&J99k!fGSbl@QL9lsN@E0s9oQ|&6YE~7!|B+Oun(=;YbAXSeCzcct;QvytIY)H znfVFC)o-vQeY%)aJ{KBcfD9PlPMUWQVVZu+Fz&=<$nLk9QO}FO_I7?&l#kVs zan(aH$>As)&e|Z2Ay9ovf6uQ#=YdPFA++ChMfC>~KC=t;9kd?`Uwa#`EeFB@I5ICE z3s%mQms?-OMZ12nA>+=l7Dp*-ZEz3!92v}_j+W!*jm2O|-YGDs8Gq+z#T^nge4#@M z%$*3^OIm+~TkfGd=EJc|K)gu%YYc>8+&sK3+good4)yL0zkmJF()YpA z%6f8?C9+eCHo}?%O*rAS{Lr9@Y`s1c$0y7M>NlLUB^PME)Oo?Om{mBhx24l<)d7DahZ2ocRLnNJxrTp5iwTxt5E z{Nk`e4EV91aLR;VD_Mp`0kpPqQ-KqcE}_Ekpdszix$|%wUUP_6v)hBO`{02!j;7#C z-wT2;94>!uf-Zhn1aS!?{18q_X@YQ3Q<^f1mju6tg4=u5`?Uj_XcV`A&8t2*&ME3U)0^@1tV%*(5$g?ljCw9RdIyk_qKMX zT)m&>oOOJ^lhu7O5hqTc1y|d2!X7PSpp9)^ndWnf&d-X*ahiPl@tyWheISp$7bLn?r)$hu;?5#1Yr)^C>g$XzQ8#5q9OUzH}eY zvv~5p@485(H4YYP@ZQD>5plM)v@&~)^j(T$_$%FwR3lsAkyCpx>sE`+dQ59oG@3$t zjyGp#S3Spm;iEM)R_H(WD*LZXxG){|S8UYSAZ6dMqtgvWXiiGK$FcA2F zabnPz5zwL82T?L04%i-Nuxxh=$Gj_~xhg|{J6pxIRlUFP@(ZBxz)i-QY*iacY0ZVs};i~Uft>}+7HC^HA$W##+v`<5Gg;aCe zzXKa`su6UU`dGxR{|vctmMpTh1GvqO#{BqoaIt}jnlpA_U3KX?tC76rxe2vedoaov zDgEyJ6RUje&~%nDeoh^Nr2Y6?T4RZ7MZT?f4wr+_%xYYsZ4|Rq5=U|Bci}Xh)_(3q z`w-uCkXPH#+WwzvQ2o%poTCkR@a8}<;`KBzofSqscNzaU^^tB}k3)k!wOGMzXVqU2 zrneNRe%ZEK-_-HR8^so!Gyyu#I0ZFF^~7&GGDVkwI)Zd8?mF0v-)z@_KYDE}cjwQL zs=l43`^idLli``qc+w(ksy_FfUjl2JHXnMn3h}&sTb z-ZRgzfYz%Zo`5kqld*T`PaKo;4xTjIMm28EzZCC=?_Q>Yv;rGT>&6zX>CTSyn81kN z1Ze@*K)hsA40Dh)9U4t{r{{HjdG^cdQrC44)zOU8L)yRgj@cdt#1l|i(-=r2uumh8 z!{nCE^7Gs!FwLkn67Rw+6I0sTsxI~ILGXyQ!V7^n*wk4)V8)!5@YgwywQSrP^>ie? zK2X-Z63_S5-w0uAE$}ZM2bv$^vbl2Y;9=1C?H-Jt>jk7gf%p(R`ZIa^9uG89-^-MKDp>gm$X8(Wv5Cl*USuKGjluKsDM9+56E-uf^pm6;A^lE> z#;MaJr1RF6U#!O`tO=FWC$e=<9sW@3MLuag)EM!SeLQ!c5y#Q90{g(d`&RrEUK2@w zLuf!AcF`Rn)cDeSsJ2Im$Yg<~Uz%|`COYQ}z?SxcMsM1J(zw)jjC={VdmqU{R`16z z-*h?gGp58?lb&eENfT)mR??V&`N#mJEm?;_=kY&!_O##JNbvj-2iHw{lLlIk@=hz# zr2c$QCw-|f+VId6P+XD_c~|j`R5%mv-V=Hn?qi+Z3gP!sOLg2}7w?ZWcWP{rdIS3& zJugaP&fq_}myy25N$$t-z`70MQfM9MHR~9wW?%xU-&5Eq*pp$-6>cjneB;S;Ak7a7 zOX#z?j=zPp^>^f?Q}8LD1;qI*B_>f^Y!IO^hH9}p9I#mhq<^%prq$D^^K6pCiBAW> z#DZqLY}8&xb1&z93ubTbSwq_mJAr=2kFm%Mz`bi7WxE}dQ2863n@i|EgMo5D&qPK# zM(bv@4oLemy!KYq2+O0om?Q}I@v-v(X!qR~4G+E7P9CPuNsEd6$@^LB6V+wz-Y<~9 z|DMR2dqpS>tHzG%7gWC(&)9){uMPE44jXJ=o0I1yy=aF0M=ii}btPnWTM6BgPAbnY z%#TIkgQ!Ou>ly=at?OUqX$A2EyR$PA&K~pu<$=kI;oawCODO-i-&aeR^o9)&Hs_R^ zXq5I<+C^ba zjiLMq(C6ZtiWkc$GjP2j$e+ktPj53if2cnE zv&Q|)Gokc0VY(ncOnJs3*kI_-PxU#XG7j!ie-2yXxCW??F)05G(%6W`vbf4qsB?YZ zVF){WFb6W{JOmmiRc!J zl_|ZYd?9H+w(Ykakp5sveK%=n4&|vuH8|lY)nC6Xb0r)t+|nsW7F@KzV3N&`XOz6lU% zR0N~n4%8UwTPwbY#>*QZ^*azRX%$W`Te?tE#)Xa@=ZkjV(}28%Al<-Jh7cOyD>~HP z0+dDImVT=l9SLs#WZ`vMwkuOoG)Q2CwEaXH2)PXin6XEDWZ z2{x|1R&k~5-(wdROtny638%~)!iqAQLh^(nq3Yk{K@+CNlWGw6htwmDR;+EftQq@A z>tWtrz>%~*`!j%^_o@FyTcBg0q1r(5AlUuzZ&Y50v@1-T4Kl6vaD4x%AEoo?4abP1@3- zs*Z{81mPyG;O?YHg0NeGr6f+$oF6(Gt9NZo^FCgXMr1clJK&%(kCacBf0nJ|YgQIe z4){oAp={aG?`UvmkoI|Gh4c za(M~W&O7E66e}pdQamG9>^-MZe52=9U;fvnn$m<2w7NEVgjg^*_?#Q@GsWEHW_y|&7%@q$*E=OA9 z3@Tovo&ddsBM^VmSCS@CzL~P&H9+TCm3^q1Q+%On!ey(Gq&meKj!aaSJs+N-rXDw9<-{8y^Fe=N<3W3qB?{D6dDk`CelgT`&q43|lXQ=C6aspMGik12jTj22k8-EgoRxP=x)~wvr{DEqOp&g!SdG@!^Mb=s$QHl$w!Oyw#Y~G1F^JvH4al z`f7KASx^_5z9$lo&D@H!Mi1ewKgZ+C_N(lN-^o4Y{%A3tf>{4U-}=*g3t)}d=qPeiHNMx5Wo+563i)sUaJ zUM?+d%(ItXLLFO2Ie)}6+6U4Hlf^de2)*Oza?cWa|Lr7OHQ9$=J8b3sBl(y%Di99O ziGp7nCyI5OCdlAi3%>Us=Q&}2tBx=5HR9`5UHAa!^<21L#*8$3nSSCJE>4e!hp~fs zA8#k1x#4uovgSY+c>iS@MA$EpsoH;7&D$9kY(9&|6SbP(8=@io%XIi~W(`cp59XEU zd&;H{jb)2zd(krG7}W4=!&k)&gL+dBYI{%ghIwD7K(;!sWFwU^LL~~e)mG(B$%(*(O9^XyUIEkK( zT8f0VnIdWX9vbhJtayN{wEFhad&W#!Uq1g06c0GSPHjC6?sos!e`QuO&2Br64cx?? z9An{7?j}4{ZYjBA4<2j8v)nE?38g&o0M}EMqomYvgr(?Q{mjZ&w?qF`0zMDf4Eo{ArQ3$uGP?O-X5ooajmbb--@zE>G&^Hot8@A?;>vqQBMc0^NqgFWACl2Wx`Gk-|n73}PXjM40s{gh+ zO@*BFIM)4UQ>0@@`doP6@`QPL9Y8u>_Tp|F1`VAhn}lSDZc`3pL`xyY%97~yM|14sK`!#aSjcRdvnzjgGDN<`QtuQ_|(HZ9M)Q& z*Oa*q$DpGf*ud?30d=$(P_bJ6TR%sJ+}{ZIyI*8hqlWWUzxr#NgtU+!Cbfm70ako* zjp|6}Bo{GSyKJ^4w{ZH1E@HU|o{)!`@0-gj%ewQ7_RVSD)=0~WHP~|fH~bJ7$q0{C z-HM*aPqP^-&SZ4NH)oqe=g zGrD(v=EY8Gp4G~=OTZXnGSGx}Q~H8I1{WZrrx-Id5^+^Z!!tjxvKyYY;02%a^q4P~wm zS@Ms&3fsl-_12QGQ64s_4}P!m@WjTqz>RmxtplBrFi*V~Yj_^jv@31G{X6UOZn|E) z$LB9}?ziRsU7LAu+#vSo@@bfEjnH}eE*kGAnvd6AgTVv(NVij|FeoMlVTR81vN7$#a!@)m>^6G_CVc^=)lQn6Fj(A*bhA*#7M~8x`?GSe-r~ zh@<$fTidbgmuQx~w;CT)R=27rR4>qY@jZbX+4x=c;M}i%up+A~?my}yRE-;bY6zqk zw5`HZajIjC7?2YOcdun=J!(XPoZJdXC&xf{%jE;PIwkv{{38iAg4OUs0eLbKh&q6(*;D`N&kwKUw9iyD&c6jOLp^{V{|e&g_9f4Njs( z)ObuD+ZTqfPZZU5ZiAi$qjCHG)p%r+5n2}wz~?DhNVP0`CG-}=3s5=QAD@a8S|hC) zoSnM?sK-$09)&X=fghDl;}zp*T~i|msI=b-+7VBc9>4?nufQt#x3=r#jZcZH9Dh;DiUe)W(NtM~qSZAp5P&(w5KJgv9Ac{Ho3e6jsu4 za$konkdd~NQBA|IH=dv~ax^?^tKkUz`IVe0ZO*yY?Ugmx#0zublY15=zy>jII^ zkyNiTd;M3K{LlzPIvGgS$Ljd~?3dzUlP#dMitm^~?A-XSVnEPYSmvU~X7+E%JOixh z{*rc#Fi+m|PRCo}zi2TId#UtidT%ZVM(aq@!Kdl_B%KRC_`5p`NI8lN3sy0=(RW3Y z{E4R>2RX_eU#nx7L363}T>sd+Xik}r(pM$h9Xa8@BwR+~Zy@f;CXQk1Sgl)i$3Ob+ zV4QCvn#>PnwP)DMquVm@iJZ$^?!}6BXI|inA>TD-mzku$k@`Zsow^T0BR4_qyNSYK zPAw$-hpgz4NSq6sog=(mnwA4;4N2nwv+PsY;;@x8wollt&t>RlzYVweuOPn4hQz@Q zkbPz4vw>= zf88!XJc5RevQ+=UtnA&;)cGB3d0--`$26+HkFV$`3)i#()mN7yYw(sebz!q-7R>CQ zsd^APMn>VrDJ{@uc(9D?-kTE-%Qgk8MS{yypx3BgZMb(sQ*0O*rL;K^uPV==8NVwDmIO|NvqLKIbM8hU-{GNZr^Ct)yu>q;S)F#J zv9^W8pw7ficaYvEY|f5=rSbicaJEWwzkID>_E$~;X?%I-ZyuxHgBH`A;a2z(Ug%HH z8n!=-Pk-DI7M<6?wTOB^nig+NIfQhclCT{r94$yQ==1H7)09Wy^VZKp&3~?7{j~;n zzukxtZ));`1yfi<`VxcY8_Ef;CE&Nu2xr+hlz)9{DeZy}2PX?P1~f-F+%8RH5-=YN zn~lf3(nfN#GhG*;8dDhlAFW+<_gV`<^}(j14H6eujmhF;3eJRb;PkJcK^8>fy>} zK``R08Bf`0DjV$bf;|iuLIUucMhVqzEI#WCy9om#r?UPSFqa`e3MeW*ZlxGXQ zaGrW22-EDEuniGgFxK0#O3RXN!aJX~0DV7_mIcy6Yz19MqdG&k^L?<`afbM>#Wm6+ zfk<9K>yUX->(Vg-FC}}RQ9>Uf!c%~zNcAM?1X_o^K!+3diSmabqQ1pC_@jRqj=wnt zGvAyB@_&qKTqcjbq9M-*XRfx^WIkL+oHt7oWJdewkiHyJ+y&_SgzEVSJi%A>q|(wzUQ^raQ!=dk{t)WF=+3Cu;nKIYqI~rU ztQdE}n=ldy7a8FktoqM|v;kc|qzp%%%V>QjoBVbRNHD0m*DZX!7hu4!j5N?MG zI!4_3t`+%OA5^@0#4%d=Avk_7T8#nkWRO8U(^&aQNRRlAZ${`=jS1;vAiiM4Q6h-e z?k?`6&vlbJq2d_QV}krHCT({DK(SDxbqgAz1b8LU{)2gSpXI><6VTtk!k~}ulw=e{S9}jb4fVhi?eKX+3 zt~$K#mK>z{M4Bfc?e3irdk4bWbYQK*8?%`FFr@E+zdkWl9CB;71>6}L4N>OCO2dM} z&Cj_tIP>2@YjGBVwvV^UMws&5fxc#pdWIP#T*ju(laY<@4)^DUX_ao7U0zKd$ftFC zb3dZ51Ks)TmMh=?vl$%E5w(|4kq)`2;E*_it|)5(fyvUr8JRq(>w?P0o<=>zCPQcA-`h6iOL~ysc7hGI4xlnBbzZ5<3*qNI3x&7ZRWT0~etDM*;yii&HLx9R@T9rBfBBj?nW^Sql)BC?AB`K9-zvF%i}#7pM=#`jI76+YOL(2`4=$ zLmX!+zaVX0M-kpSYe{QpDHl{8TvE2dDzhCp;S@V&bx~;~JbtexZ{>cKvZ0ko9uM0T z=z`9<+pNIU3MdW7k!} z-)R~u%|ZE_Xn*w>Q=0ErMTtgvaq3~5_p-By{1T;b2vi<=?c5Dibw++w(;%@2e*Rsg zd80>rNuDvFyzUoHY-Yp=BY=E?T$6f$4eLiHU~^}zzlW~>^*<~IdL9w)8@mAeHGs8u z(U*dgroP{2Rua?wQme{)dB$H;yv{elRwPUn7iL+jj7?k{`3RMNJQz1nWlW&I zI~LO;7IM>)Rm!&mVJ_AEUWKH4b7#a$%%kBkI4&x+ zT~4)7UKE>;T*dAr(6e-(7FJ=O8aLAZjCxoQwu5l*gB}grR-NPi9_R7kM>|f(m94O| zFy=n{y`iS?ex$E*O<~;ENRHoJ26w|;a7CNia^968{Q8AIth7-W>WuHmKYp(ZM{*Cq znuF^h7Im?7&OG)hX}nB%R|4&_CJS$uu`+MNN3p0|UpyMvo4?J8;>rC`>^w0NR@nRy zO@=12a-VrnvA?dI|FHo-H?x|w+Ej^SJnQg*flo2{VT9)7s2CnLAQ|Gnmx>$nYVu!+ z`8f942^-|7;~cDw`|My|(He_cNAPhWXA zVL7I)wvqLX=koCl>&PEhL&fTK4bY+Y6n=`oIKkaxz5S(dB0z&zE1e?SoUF!(~LR z<|1N+2hM)hTo#$Q!mvLZ(5A&W^jQB3A55opO}x!nzyW=vT1UNx&3O0bA#&v<4^DF` zdag9%h6zo0OroWH{?nWJ*(QjUz7w&>x!o9>9!$ql2)YqjeCvs=_$2HD^Y`)udmn4Q z+^ng5pks%Bszo8yKkV3R0s|NAhS7}|i@p;g<(9<3l6nbCUfc3zbXo7Lt3RjO74?2L zh8ml^c&9NTxZyeg^#I>KZZJ=)V=13z42K}{s2h$hWI^%?)wiik*twcby3iNOi}UfJ z@kQvov60L_@`&x}VT}!3+FV~(sair=G2oo ztr_hjne|Qt4qOb+ZWcq2%UAJ0poyG$yf$>NGg)T)#b~~^vXyI(ddR$8)zI2|6Ri6i z3%`flfY(WBn!7f3TGe|8Z%u&9J?J`u>n%K9zXW=YDS{QXuRwNnYh3w_fWVTxSSgiA5|08k%@$TD5-HuZ~eUYIJLY&V9T8eK^^ z!Io?t!nT+0#H*WX3iq5}sJ_<1YtL&w^MyV);;S%3&4B~mqd|DU3gtzJ zFx8W%x2%8>XU~bdn{H~>Whj#V$@`C82Xg2ABSpIR8!ZrarhkB>$g7mORWd|-Zk+p^4zx~6IYFsuMorkr| zhe40PZQyhDGpKr1^Vzbb1H9~#gwsy`K;OM|FGh|95Z+->j~2Y2(NOqhIuB@Ekj{&% z*Zpj*cx9^>*zaEp-n+v#O#0mx^>3R?szLbIt{LAuHv&41F`zN1DSr zt?X6bog1~J=T5?|gXf)lxcuW|>^tqG>Q6isqC>UtRpA7F@HxW#l0V}JnF7xLUDNJ% zzm4g?YT>L0dOVe~DQjOZ>3?Vv&*%`Lec9p%&l2VzUmyiHi#S-xM9_CHNm9adG-#8DI} z#YVsa3$Tm0JFDocfQbdz3D_+b22v^}VS-91wuAxPofWVXuoJM0FtI@WX7BTSfAEN0 z?mfG+Grt(l+10*mO`sNC4HE7q;Ldq(VDgLy3LmwXyh{bGgY0DdNVHt=h?(>rEftRZ znr_Re{zL=Iethe^#LS>Z^{~~x1ylzcA#d|myz1~D(rYjy=(JjE7_-+~4%jr2+xHm7 ze!v!3I`}6Lf3wU~3zctd(Czx{p-XKbyvHexqv`phKZ3AQEVeu?>UO)zj1M_U;$}Ry z=?3d$JPKT&wZc-LI4qp>2O{=>(kdC3=B~qWFRAC=)ko-o13ylm)mg~BPYB8iVG*c#Sg2RiJ}X_~&_MY((dc3Qy5tauSjrvUxMYCFjg=6Tmc9Bk8%1P_6gA0vt_nv=Z z%2wK|>3Ftgc~m=ofSy}>v+M~@IN~Y^M>Jz)W8UROh{A0-e^E=mDPj;Oz7dyRYC-YC zlHHV9 zmqqCH(-o|W_QIzEbHzjad&4-5(qdHa;QjBDR^j&BwmZ=xDx22cTu?0VY35os?)p63 zUj=dLQ1;6)!tXrsrOJHqj`Zcx2Ri?CTn$ zY7S}b(aDOQcV5;=E9Tqt@=u*8Cs)aNkAqMS(Zj_RbD{RT8Khm!(9NI$sQLfQGmtcA zzHUi4-euRR{%0X^kyh0d#Y1FfoX3CN4`H^09;ckaV_^xPIA(j3LSZIuDcvhm3@!+^ z;yv+fF?6K!pD1tG@~C6tMzRKLS$D!Q_O0+oN-s&Z0XDa4Vs1qv0Wl|l_zMLE62Kpyi7 zsn4rTGf?%xR$QJ-*E=9A+rMB-tm#aL=VJ(8JHyAFFEzeT_lwWP#jx~oU%8>jMV8St z0Rplw;&x~Oq=L-+LMgK{;;_l_0f_NYDV_%5@VGe?HEROroU6N)3 z#RWGS|0h_Lnw)eDTN4_i+0p-p^Sc`_#Q=*FxN|ySP0KJK{X^qRWNlux<{=MT5RPqx zm^o{ZG!MK?i-Wd4SAb&7Nk8lS*u<+F*ooVLir@H?-Mx`CA~a1JC0AcG&TLEPTX=nL z%#&v&;HHS1pqGDD)vG4xs4=Hl!K?NwRlX9Zdun^F`8);J4USkS=88p+v7VyT&@UqyG|A zT8eZ9w2PRe(~=G9H6w4b7h3*Jg01sfh?&WQxs(0^_@&>Up0jMtN%P=yzjLhP^i70c zv!&ul`yP*+Nmpq44mtrD8|)=v6?1)d0R6vRMB+#szc`pQY%M`~07pv#Vffzqo-jZMF8cP#gL$+iqF zMAE5(d@EBm_rNIqn8YyDkgpbra z^s}}V^x!z@m)5Fpfpl>@FDi8e{-b=pkVZH zUi;k#<qcG#@^G5DwwIJQ0rKry;%YR%(^0NlVk{}{P;%>#AS}nPj{PyO>VKeb zU&+3Q50iwG=J`YTvE7N6(-~{tU9SA5K@e6dfM^nQ#h7F30UW6BioKt|*UP}og6k}&8A z^-p#H@dROYHJ+Kei*>B5M!qeLc*F(LE{zZwO>I zq)pj{#(ILZCs!W0;f?CNd1x|_4^|peG+wZs5k4X52j^BTw=7zJxJwe(N+6C8y zC#|=#ewSL2@8}2eLKp0Fa-*m=q*yyYAXmh#*-aYoJW`y&AoeOc+dUWie@6iE9n55n zVVhYvJlH&(^20__?*u~^Ty&1?)J=?EtdI0rke#@WuU~Zv%pPx7Yfg0;h{U<9_gx22 z<9(s$gD+d(5q=+j;s~cR>U*fIXP~qi^~tw``JH6YJlUgpu`!kSvNnI4v`A?RPTGL4 zHx3e{H7T}+I?SKvwjpzRzy?(Mh59h6_Eg_T)gM#!L0UmW`k5G~t zi5z(Bfb!Nl4Mo}u3MZ|kK2i`gUY3od9h8Q{txE<$MW-o@`fTjq;m6|MB(3OKK2e-@ zO3?MMNXKd4UChLYRZBEO3-#rK-L$s~`TU!`yXg8}+8Kd}_7O<_8*{jQCgmFadxVw! z%^`feCX{b(wet!nFMOj#sYYq;<-6}Q>f2zm;!V1|SDKx)JS>|{_u1WjEmS{3U1vmR zQ=mMMbPe(sMmnAG?D!L;8iVDn_lr+wZv$x}ofdii_!|qNG9o2Nx6x1U8P|JX&7chX9TUU zbBvKL1=}yn;f(8d=smRqs=RyjIF;oKJ0MNWl)kCGD1)gyK3Q^J*L!%I^i30x86yft z-+}%m(MaRqhnAld#4Q@qdyM)ZKweHu^{M`qs%Mgvb3;IfB#r0o2S{tdNPlRE*O(eN z^(_SHRhj(Hf?r;omg!!f&Yc|RjO6Lza92yxY5H>PkZ4AFkR|kLOFl7-saPnTUN^fT zD4(tH@@aM|dsE$xtF={nG^(`&vwT-6&x(Ox=$^0iP0-nHuTIA+9w00iwxvUKJ(M!h2CZ8X%YRJ{iHm~KmWVxas7#p}6d(4!vma@9ceyzR}1M}T@j zs^>}f0Rv8rJq^?!(RMRys>Vs!Y)d}1F?8{AtK}9OAH+RoO(| z1NyYdQZ=LX-p~WLB~(H3_Qbi5Dev7CUjlIeC*0!1r$D$0)I%fuog)-)D?XyWsxHoy z_l$bRaG--HY2c6I=vRBL`iKGT2eZxT2qXKnBE6A>>n6-q8lL*F>v8nbcTCj*=>q=k zeHhHyydN&KpnHSXPXN++8p0MNZAch293r+yGSXWas^1m;V~&oOhP1n^i%Z~yW7_E- zB^h+^EmOIBef@P&v%&=rRkn~v>on#E5>5l{{UiHMtIFH?-C@^Whr|AUZ{Ya) zQqbGh08Y?;P*X`yjE}Jg&YR=-#f`Ew8W> zGmYi`%}+(u?cHdv_kGy&mlePDrwG4}`-MfHTJpZvpP3*!C50RAUM1Bb60;P{)y ze7N5ye3rEq^E<5M>~xij;z4g>@@sRd> ziK9FkW2m{DUswJZu^C_QUxV6!dYt9=mz}qsM}1#&d9U6QY@F^3yNxDfroCE@y+RIR zMXQdynzakBNRAQPraxgLK8?j0HpaZKr#tU5f&NB2r@!p4Ifr$_8}Vn$BY3UGLveEc zRGwR5jVteT2d%>%Ski2i+|si+bJvAwIRE@HJkY2g24!>v{bA$qUB{ZZHDnU6+hv-> z6JJ!mPJxbJR`LNxt=P646KTA)HLBRoA7mk)**q79U!H-Da~Rgm+YJ4_wUhc+ z=3&l?Z)n-^P-@E!nIVbjD}%9(>mI zpD418IDIyb+&Yn;t9gPZPw zx~22c#@T@Hb2XBk?(W7N5u0%K7ZXwBmm!?3)KK`^yuq|BVEkrSzZv?6OPk8%iv zJFO!5WY+-L>obe)*Y3)v{eFO|_D-os|q{s?a(E<4>{ zt;e0iyQ6;1sbE%o5-ydMLr&u}+WCepw1?7+$2IuqZZw{;VfV~)tr=M)mKH*cb`y?NJf6q(pfw#0bezi}q!R{`0 zHh6&J{U5{Wp)KU1CRXa_m}RvCRBZ%oiW9^2&E=a2H-#zq?X550cH$Q5U-`sTjVzN* z@$=mJIJHMIZrKqdVjkv*frG8E)x{J>^+@9>2Zak~hV{ms_k%UJYFkP*$C)A>IMc+sV5)*oG100$IIc%>|x>H3|KKXnC4)IUr*b~p-W9<`|f8TbX_W#8ZN~? zl|Nzr%u2R9sk)ppdW7a+MKwt^f+czZP}0e${u3#R>xJYKhngKtwY4qgs=#PwPN#Xf0k|(SslF zvjoSjt1n%PPD#oaw3yc!FCAVjXdaT*TiW|}<8=;I;Hrd15L;r&e|rtYA0ujkXTNsx zAG;(6n@r)||AqsuNP`=vHv;7f9Gb8aBWw<$;t#??*3kVQJ4An*r}!a#PgpaIt!h^t zW*6PUfe+Ve2#e*jf)B)XeX!pu3oYd)-f2b8d}z0`dagNQWX>N(ydjAX;g)A@{>USU zcRRpv%IsSFMRpQy`|1YAy*ta+(e==(0{N_IbmmTBni#qDnb`aIAK}`5r;R6FaQg5z zU~c|G`)Lfrk<w((Ot$}i#Psp`Xn5~IjycV0*?#|~q zyT~JddgF-YSA~i#ec`gI4e*)GBzWXeRa(zTgcju|u*GUCzQ3)$eD7~Vct29SUO5L^ zd+b4qAyoflL$Aj`yQ5cyQO@H zEZye0r}kUO#>_f;$MCWonHfHMK7@MLhx(&;!_;}jc&j%(%Qt?ln0u}!?{&2aU%J#5 z^26F|8#X)-0q#d|^nPcdb!Z~rSe??CzdnFf9-DwoiWAVhpjpe!GUI3j7Bp|cGLNMQ z(lbKw7;!FSO{Dv~HjIE3<~5P937zT%OPVkJ?5?K2p#dC!-b8zfUxTFiJCW80{*==h zz9~((;_2`oJD}>&voQSV8&wn7f5uX*z5EEYnG16a* z>KWFj2656Pn3{H0{QK^q%L#S8scrq?ja4`B*yf?M4c1yzMJNs>F2?*`j_Nu}F9_mV z@#{uWdN|$B(egrsu=u?PuGv?SVY#lH&#BF$cQ1u%JMOUY+I+&LpGdWdNoQ(NEk)qa zuG@sd*ER2N(1X%HfiP36Vz8z%3}_6R$*vU{^&7O*X*$wXNcG2W(KE7jyS#(%W{srb zmbIDw-e?v`317T_APnq zN9(awpceb??jaQpD=te748ks!DR^?uTb<@Se=rB*8`5(P#j};RWMQ%4pf_Y6WZiF$ zkyD%Cjo($d72SKC21n(|pTotgt%rctQ7ZoXaB#eQ9*}}1;g%R~+g-XjRpXXd^)hXn zHIrlhtYefvTEpWrfMPBmn;aFY&aaP^aMUXjH_W+)q>GU>7Z0U(-v{PZ5#wFf5Xf-m9y4q-%-VvG#3n5?Zi|)5RYe4O~}_1OyuCEO@zmg zN7!k_2W^dj6U>s%JR$vtq^GfC^$ggx`V08XX^(Q{Ei}Jf%q?*VIqI z`uXiK@>mp(y5|K-GimDEphNKxaLP61%j90AF=5tmI!in|M@tWgK;0#U*rur-SDHRO zWEBp+G)t`?TyeXouodY`(m3$)(icMi2<@j@)dRM#?1l;th}$qFwVB*K_#W&SxC5uo zn@QKMhm@;=#w34D4|DRqn~u$<>o|=O4+>S+uM7}L@3FJzE~C=hgFLhlKKe5}NCh=M zK|BERUKJpIVx@Uo36uKaDc+yBa|48GO9g2R&6|NkDQ~y3w+EU-wWn7V{^HN;*+|-q zo6n6z(oN9GaS?Q%^a^PmxqnlwnA)}+4=r%Sb!{3-!U1SDJp?js`iPrvuYd+Wv*l;G zW{HmnDo(mTvXqg2f_ha3V6jMt=~p%I+OmNloDo}Uc9bsj1`C7bW_-!vi$ZSQz*G*B z7AD@@0MCyNBHcsh_}yE>NTeNfY(+Agda`O0bJ`2DILbbevm*UQ)% zUa}OnQHOh#&W4gSH5=lSrlA~Xq!-xMkmusKcPhT`)J}%_FM&>XE#bzzhNO`Tfbf{H z200K-&+?DR%EH58TLtM!Buphglp;vakuHe>($yNnY2V;{NjmN}DgwGDxU_u;6kksG z##fC?5L6?AI0#qty9#Z|<6e!6)-2FWP@fVq9F z^YegC+;B?`;C8#k_07MaX^}Y+)yQBS}->O|=_4phZo7q{8?vMlv zE?h^-6`^!a*A7O~C3qcN?7LQBDh#9hF{lRFw{~Ok-pqr_ckx3L{=kh!etd7U@1Xd5 z-{HxOd?Jgh+66ZSB?4g?pS^nwyWGG?1}@l*hZo*Q(sDpN?)+euE7%`7BFu`r@~zf% z-g$%8s64jv6{Pv)lE;(P_kyqh4ZloeZJ)kjPq#a8S~FPiAQbMW>=A^!7<+KPHn?|V zMjQ@~+O8N|vYTzN8zD;T*Wh~Fw&C^ouH;u+pceAZajEF(W+- z%Qseml<&$N?HhvT_l)IYr4w8mRJ?K86x*1%0{Ia9Y`+%?qxm+2s#J&9VPI5*S|=c%#Hh~UeNs>Ip`D07 zO?2FJVex+aplt+qXRO1-v2mn_4k|p>kT=qjPnSw>tLsx8z}M4<;l#)xjPwIEf93$> zS+(Tz;q-P3{^!MFq@DzlrU&x8u(eh$8-6wp>2r{J2M|a1^uE5=R6PHDUZkzDmJcsB z(5+34&%NaGvy#P3K0#-R>{Hy!-cGcKw%yF7#L-wH!Q#%>S#IcN z>^#1P9Dg%h?2q5BvCV z;kKtm>qka7d-oF{{*%NlW0wxEoR%v6 z!p?Ov(~w4h@-yzrtFX`ZO|j;IregAj&tQBnR***k;!jYTfN~SRhqh%O{?$T?tG4xz zS(<4f7Z`bH6>IXIec;2vLL|MU^X}v=Vcm;nOljihboRW`Ca>@9Md>t#dQ^Ub=Adef z`ivvF#qTE6OVR>)XQ6yXmwV2j`XZ#Kz2y6E@@7cJ4&-J+KYhjM9RGhH1YZ{|o0%^!P@^9)W{^as-4QU71)p(cE5fHIct6~A<8>n|u zq&$qI+SP8gUMCh!t^?f{?Zt_EOl2ChMZ$gf+Exlx*UAg7n`NrJKB##0F#o6ejO_UH zJrI8@I`il57BcYNah&*~DwtYz!RHQI*fxC(QXFaXd`F<#74*86y{x8PVNU&+qM?HN zYns`PK^pRtuxK)!mrj_`#^e$_%c{YLlr^PZf|($X2UGT4!+z(@_=ruxsCrO~yF{U9 zH3vbuf$FO-^+qc2a@P`}UiUlI8&v&}z7*tbv^x!56gR{D@_Mv}ai}=qSMP<|4}Gf3 zqT#E=&)VrgpDC`m^_3KBjl+Lybo~kP;4E!TJgR=E>W5LSYYeMfsXP|chgG<*bQHEW zK8Wd0PcYKZYV7Rska)bW3D+F;s;ldny3sS7q@PjwyJc=Bc&~LT;mlo}8ch4&-;34K znn>bxByTDdma5)h*@jG~`+KIK%Kcrd7XbAtGpXMQN{{cqa~l>KokPNVIFU>HP!*X2 z)u*lwsRn@72nka(0n7bV4}hI$O@Gt4W-Xrtjl;gz$A~*It(dxo>bqIFbR{pnS*?x2 z1TAqSR3ux8yvl5#_2;Cyb-gIFPDeBazPq$Drs`21S2F7JkazM0^Vh8e=|D-j&pTPv zQGCYRtQp09>>O2}f_faaNypjXPtW7T7kBVPy~or?T8nmvO(5XYFUZ*GP0y|W)F_Qi z*u<0{Q#n`j!DBI_APVNKa^U3EIcamzLUTYUop-PzMDwA^0#q8S`(I;Dyh=GrM>hCn zK%sSSU4QhCrVUruA>L%9ePHt5ql&lWE={GDc#inp9;x4~O@09i`+1<{3|-H*PYYA5 zvp7rj(Lrg{nuF%3948F@$;hWroPJZUBAU^A8DTY}UKy${;$}uYb)b7=qurph-GnLFyb? zxul1TAJ-fFZEab^8$W5{eIAW_4uhbB(|Ez8nLO{vPVgGm9#Zb!M$0(|Fnd4;OsW5j zWm}nZSCjX`UhYEY#vweu`UQ0U))<}ccH-(az=E z)*tQ1#bM+b5AL_Dx+v`23~o;TEy~u_l0iRaV(By+ZWTs*JUFJZg+G6zamS|+ce)&0 zO{PP_>CZ0GuU{f733LY^i^X7^{9SAM&rDYId=9jgzGm?>J!!Q#4*W7BwP^#_ zVUYK0QMtpO?%(-=6bp>Z&Bl0NF703I0mTfn&fnB}SrXp<>cvfKnDFHNpPc=jn)0Wg zD}?8VQds1EmDBHH(ApI;wWYP7xv;`fj4yJ!!wULcgw*|X)KuG=%ymo&c=b+~d7~Rh z=dpEU=sv=wp-s`L{#kZ2Z6(lkb?*y)&|kMU{_CviETfB%eLV{D&YeZC-Q6)OIGdRy z=ZJVI=o~n$2=<_JGXA48LVm2_6%z(Y8b6d|-&6BoQHwXBU!6Lb)~BX(h~pL>l6(TA z4E93hgLD?@<@Ntukd_S(qFZ7X_{Y_S$fQRir>mi$&y!jDPeg8-yGE_YpS&7Sw$%uN zm)v3%AIHfGj|VKWbryIoUnMAa!1AoEq`0H+GL4K{TvO(>-6V6O)<9eX+8_7E44EC1 z3fU&s=v;q2*f(e-O+2oEcU5~CfA6_4?YW21HDp?>35L+K7k1@GnSFzun0xy@nC!cb zcAf{o-@U$EXgHrgt?~s+KRI#>+MhDN?k2v_{V&)Knt%y?W7*jjm6+o55<_|#%e4Jr z;IC&Yyl3p?`7=yp{lgrKJ}y?zEgyE@*<+UWSml){tFcFYFH3a;IbccuCxJnfz)cSib3l zX(Q;2>ymf~-qKrUpWg`;bk8;Agmg~~hxpSj(qC_*`ahaZSPzu%BDLENIz#9*$-`>g zG%JXw+#QCgr#j1$dBbR*itVhlc1un%1gcY{vCupl^Z35E@XzWHT$nMNt9dPK^cMa! z+W^^LJA$eC8sYuIkQb(AYD_;`F#DQKG*)5TAwIMa3)eRhR^LNG#n|=w7%ZssPN@2f z^M9vx9vcA`wI(sE@6~xu@CHz8Z&`R%RMK7memOPOHMFi)6KHPMys*wm2=PwA5bsX# zXLU8{zjFuu&L(iWph5qMp-45RHCb)XO_n611VM~Qk`)37OiCdx%E8n zg*}>jL_t}nt(b3VDCzew^_T2|U#pR~KF4htn=oU{u?ZDNx^kvg_3gnl%j`;Hx?kMm^6 zOK8Hg|Le*e#|B8Jjke(U^cgshXebxDuM$PQwxO}vC&Ch6=C%AP6b;=VDSjB$BL!$| zEG5CmI zS=djj*0W^D8wi=2go{!*w=2-1XZSafI?IkQW;pTNVU{&zgtTuG1u2V0LAihp@_??c~{DfAsp&75tpr zgXs@?uBkok!Aj4K7as0G^;{FJa`&RssSDt1be(X1BlA6civ{=W=|s5Cy?a*`gaOXk z`bo@hzA^e54UnbTam>>7ETlptdM~dJe$%%|`g{y=G=nJDww$mUsV2j;eMx@$?2>(Q}e0d=v+2J)-v7@rbb5m}@f|OEX&W${j~EY8=1k{zT_W zb4l3D?7BOn+sV_ASJfQsH&x|?@iO=$lfL~%V$>N+mKAW3a;J!uPCSTl2mXoN$fqLV zqJvBgu~e~uiq&(t!gN}1pgPl<%smCZzwV;X^ksyBAGpcoE|}`mf!Wr)joy=6Fm-KO z2OakMjv6coD=?&ug>)-+1NS@DTwzGzC;^_or)VfoKw-D#_>Pb*nt*fVjtn4G2UofYE`OKtQ zDi#?9V}fU0@a#VojEhDxlN#ATuOaa@cYnEqDg4W}+QDf3FqqB^q5MQ=A5-)W4~OJF zKZF|_K-hL0^TOM)gq0^Ce{yYZYGxo6c2s6vQh3IS4zJhoK~7W^u3oqFI)!n*Uzq>a zSd3i7(VhCVi5J&kNXKY2twHBYeXal}pBz+tOZ9<1kKRF4tzF>ty8~A>?BA({{L{vO z=3HG8mt%BTFnacX%Y3Kz5ee0|K(ONlDCl)f=_e7gHWHny)stmAtMQ<>2fBuim+AW2s1wb= zvd4M!vUlaCJFK|#h3epQ^$Vs(Zw31T%7r!Fw4a|yC&{3)#rrLCUkbqO$;fjjd_nds#we1$gz-eVXV@r zj-&;7;Z=KH9Jo?x7^ZL_DrvtUUehYAQR$IDy6wA&s%62gnkI0W8+aahCMrG_pqB=P*M|%&{h5YoCAHA)$t{db8ihCOE&-}XwMN7tC$);3i>erM zzh1Twf1tlitEQ)`3Cr=x%xSDEdM~gLZuBg;k6j7+88zf87b3qu75?tgK=}X^U+!%g z!-#K~?aIZ1a6~-qwHo8gZ^F}|HIVaW(N5W86(mYSi6XQZLEG4sI&;>&nY0@W~rtr=oQpQkS^mzyXr}A%~~kvO!v?>dxLrX_wn4S zH!$@=8!h<`AY7O3M?=;7StRY7Ot}aN1#Qsw<}Xq9Y$MQVIl}kwP;@`3#}jUL)|e#s z!t9sM$`b+czt&{!0W7dCWTd|#p;afeyp%}T2jFj5Lpa3_S8Gl@^BnT}H`99V{=lNf z$Fc+-4}^QD=4U+NDDh%pmn)3aY1n+@x=i79?&OO=`bOHXG(}VIS~@NwjjiI(J!7!r-PodskEk}P zpyS2@rZk0*-7Hr5qlGLBjADh$+K_kY!Qu|AmK1-ac!5=qS`hWUjqqNuOH14*2%D%T z-$CW-Oywzo^t(uuVQ6{gI8%HU)uR*8oH^B_@clI+!|h~uUihfFq}YO6+j~fSDx>}; z2pS*rdclFTJttowr>-Htw{swpFJ;6dYAw;{eIYA*W(xK2kV)E=I}eP3%8!STbV-KtFBEeiTw#@OkBg8ko4E2Yeuo?Dv?+}b z$*Uo)KP&3h0Bl!&7bzim%-hlmNJDVqJ0y*Z3F9^q)-H9i?T)N zd&>zZmtf(QDd2uR2}(yEgM>}{(P``@D08m@70t?7a`bQJRCz-ilH7r`!%HXfb3i^D zD2_;V2#K?UrT6k^=02vXHnnjQ&2c7oj%~`RW*K=GtXN&9AzTIjA!gihYz0PrKZxY} zgyV|0%(Zw0*h>o$>Y=B#=<$;g<`K57L)C+D8{-K6fvt%*o}rbi5fqIIgFhYDfbuJz zqYGG>i-GcNVA=}4YL+BR8X zZiYsMs~~8{0+!P42Kk{PbdC9i)OP{9R_8JDQeQ|sO+Cv)y*c$*gwpeW(i@V0ugCo2 z+T0bC$0M(S6fdWom({r4wl18o zPGmKHYL|@kd(>wtBi-gIsPCY)Jdwss(g9tID~0ly z(GGgj{&=F&XvBZ!tTf|*F5ijAx#C~)JNSq09rNn#C~4e~-Bk+55-$jHn~P4}j$owU z4J19xLedWli>BTh>X!)8px}f(xm&%CTE!j2XXJ%VIQ0Zrsk;OBdv7O;4jV~|9~k{x zrecNUC$y@MLEfLGeKV3}?`lZGBtgDMy6abCQExhP zw$@V5gN6L-Onz#bj9z8P%bYgoFwL=X8B(0kPUe#D>A^{Jq1E6J9cGZGqI)S%Bk6cX zeFv>wlPiqsh_s0n_`ll?q!~ozzrElTavXBIyhOiUNm~1wd(geOE+fsPdOpf)GUsox z%)V7Ep7)`Zq&y`2x=Y-#7uDx(}Cs! z-gVlXT=>#L@^L{+(pe)QDPS{ql0G|%^r`WAS>Nlu<3)OcWPM(NaYz_d+BuAh= z7IW7RLBd;S-%K}5e)R|35_FapQ*lZqQ%oQE~R!q3bXiP|&1F}!rFyHkljN&HqBHN&G z-V0QHR+ayX+sL;d^}AFrLDeqxEOJ5hMJdLFv98>1_5eZsXS5x656Nd}R4kNdi7KNr zU#tJp^(l-zR$^jJV{Td2Q{^7_9~sWQN875NAEQ2nS`$GSs`G$8>wQ&U8BH8*crrgzeSnu9J@F#t{AbzA8+0MvQ zXjBi8`X1=m&W96Lh>!_Kfjm2qM^Q0S;|A*EAiajc9^JuW?|fMFE>A}AKG^hL6uV#TYN}s&V|x> zO~9#}z4FRvx3ZV&ccR^%_FC#;Fw(Syowv~5*Gif=(jNF(Ri)SC-kf^vm{wU$s@@E7 zyTVCM?x!)qS+xABleFMYsC^S$7|4a;Esr%l}Q zr8Qg*Tn@*BYhjH&iNayVFqRe*DBsWbfiT-*{8&Sc&`QvB3;cwV_^zUVX37_h75nhaK|E^Wn!rzm+Kv_z4$@_#s z?En|K>D^uM?$lBiWIOVMhHLTIfC!BI`aH8mJp*3(VmXu6+t{`{_cMpJGM9Vbe}c{J z9r-;mlch9`7pAn=rC0Th;MMRkPG6jWPa3)KhhBeiW!+n%epDKpw4phhXLk@5rjL+E zfA!;4I#$7VwifbOy=ZZLZBu!y$1$Y&h@PhcVO>_V{Bfrm-g>fFx?i)D>4o89bG=e{ z9(RllYqbN%4qOf2njYdwXUOODS^$5SwZv=Pz4_NSN4Sc!_4XvpsN~YTLti?3eGET5 zGd9z*mWB_~%ckwZhO&T!`%q)S203yJof~k}88R9gN~5oLaL14{tgVq@6)MhoP5d;PtTvoZiQ-M+eJwX&rIFM&LIN z+`==f3mN@f23|fcA}#C47A^1Mx&EFqro|;V)U_FJa(JM0m~k6DZ`R@#SG{3^?FMP2 zmr1cOp>st$%V%S*vHVb1N%0UV)gGWh^9wAz>r%YC^^R!Xr-;4Y6o$oywIs!zw|TQo zoE=q{pZ704Q+r)2J~GOg{~cD2TYOKknKy2u*xy~QKK@d4eY&4nWEf%f@}sD25XMwrh;1ERs21kk8KR+V`EwEZN_r5+ZyQi4Q6g^LlZv#?bnZtJR&Q`Vs=2hOa{}uc zmEpE?hMaPRQ9bg&hXJr-oRyq&sg8VeQJIX0G7X%iBuX zY889Z#hBl>+==^Y$FW5(X7TK|S&x08`yC-K{68?ay1oqS_t zOYGb?Kr^dOcRBNG6-ihmI;_qY59hgKH@a_<)<*W2oXM`-&S44@ryAI5BAYG;T1TYM z72E&L0m^HjSYhDZy72F;0slS6TwdATM^5W;K*GO7t;W9yF20`1oxW{{_LKHt$I6bl zD?0^ps)R${%u4KeFhcW$==xnGb+i|SlRoKxbSiY_D0ouOev_8+_ zL0dnFII)_$6gA>Uay!%Yjd>B@frXE&$SgV^!t!N2dKKr;{wn)qVX+5xo@Ec(Sx$H) z(Sp&OaKoT;_%SyFW}Mi^VvH&wcJO%KYPtgp^RR>$*DnZFzj4foUG|)etGv4s{{M#C z<=H?rq~nR5bKXNvb?Q&{wF8B#9>Zv_&D=_j;k`IKlT;Ir7Of_pna1DTzoqaE$1d!E z1>5$q!{Z%g9Gz9V*(`*3VIVFtiWTD`CdrHcQgP|~+C0>J4?k_!0Xe6IK(pX3C}&QM~JE+Be1V5Y)1xdzbZcVc^!j5Yxg{HaXK7T#9z$l-6AZ%@;q_ zv%;glt^&M%1appEhPAT?$yUL7Sf%3{Ni~RdJ2jEbEG|ROz**vKPj{N@P=#R_e9#7m z4@pMtvOPd!;)`dcf~td>cij1g&@6BrM1KP|ibdie*<l{gSfmR zf)OW+J%n4F$q_+?enA$FKf4vR?fq)hv!}5hEN=> ze@Z)GL0vwjVmG^5)SB)|FhakazViLV?btfY4=4r{dk2}eZ#uKG4MD|)REPNAf&bv? zj2d7s16mKnYI#7MG@Qz#X>y2kP=*l_b{&eVY zWExiOIs;Vhh8jD`Yuz8HoCl3hgqZuc7JvMEBmF-Ezr?#U_;OY_L{#DBUI&1Cgbss(Gk)*w}J8v$7R$Olb;WPf`n?E>J~?i0jaLn@ne5gYs}(k zO9Mgp+}(wD2Fyph&6_j|`^??VIK`2loiPyy-8+bv z`qaloFG^U)TRX(G&MxRWx;cxu(hu9z>A*(U-i4Iom=>d_>YoiQ-33$YZ4xwIG3W)I ze|WkvhI#D9M==wzw?{pdKPsl&vX++eqM52OHf=ySb~4(AEq)15p9iMvh0E=(lCVqV zpZYwYyn;;2`6_qV=do|_%$%)Ab;ujOpNs=V0ZiHuhgAoi0^(pKea2{QfixoJ&OxQ! zoK;;oRrdvazXLKpj^vY8F;sJ~Kb8%527Jcwu0MraI0<_B#mF3h<-|(8m4rsAp9i^NeBJV)XeM4Gv&XtVYHrC zApOW57XIueJKi#f8o{IFqjP3l?}xr9hcS5S?>1=ZF-y=|qQeqLnf#^& z<$5={dP^ss=kf|kKhauQFyp2rp!6eQ5mMcPV{$i`AwpQSYxTHe@~KSCmc!8Cda{Q2 zOpFX0L3_}HR5(>WC0z9VVJvI!S_p)N*eC4}5+;cFz}-kXh7lHMh+~lAkhymDP264h z66*0F;!{0ra=xMLdb*sE77z;y^|{%cF>ptdh*`as6P2!y8&wR1#dxgc3+z*nE&SW$ zQGEA-)8iY^tp-QpB*i`WW$j-qG5w)*oik}=%7=5A6i+mn*Hs4IjfAV8zKLtS5!anx z2{b2M)Nl?HS`1NB6_Emg{lHS`UxyD)+=to>l5dydIFnMkMQ3PZy5brk2fEBLF@E3l?6H4%9r1tLhz=Cq#5@x zgOR;B@tNXtEbr%n+e_m>=_=9-*x>9nbWh3030AG)wci8ymuk%iZoMdCI<@7k;!kRt zwLYdjytE4W#XDG~fg{#mHjQa~>hPW08WEn4l}^=PGr}5Gg9 zKjs3&9V+hVa&fR!oTI<=1_VB=V23R3!8_Ys8j~rxjCfGQ7_ERA@mlC)MEm2;-HVEs zCe-Q$eowXlX?(@Y{FQY%`L!-gVb=iP<0>wK)hzrA-u5Fhgmt0`&;Hg@ zl6O#^km|ZB_d8gDtD&3rKuCaO52?}{l(E5Sm^E=W3<)6bA<4@mdpbb7wq$TAqBhHb{O#O8eT z_rY>mm$8C49jxarQQDR-ANLJvH9D%i5zu-fX#n*v-`AwN{Juu2ycC1pl!MYQq#Lzw zE(Wvgs~g3X)+Z_7sQ<8~;#84JWM;*x!TcpWhp=Q)^y@*;!}jOrMDW;T$d z$21`s^B^F>OEx&$A0|J~ht!TA8EGeqd3BhR*-p~Bqu%+sNWBNB{_YZ^+z?cInI)}r z80j!9NGipVVRyBMT&hXJFCgro+^|-j(veopi(-@$pF$oo{{=^Ni}d zKdak`&U8*0&sMgJq2A9oF#GR6BtHh0-{vxfrQ|Q6V`VZP>Ql!}Ol4Majz$O@HJDKhGJo}(hl$PDGRN3ydw4Us0v%xs~|h`Q%FLb8<+ zGP7lGk@Y^OKi-cwe%*V|InVQ5zkAQ6JfbprV&b`lrMj73!?kRD9N!D%IUJpH@Yp?s+DUB4EhhV5pdo-y#XYyr?Qx#567G!c<&E=kwnQ0rWF!{!Cn72NxG;9U7H1(9+N zNqUZbcsqqScRJ+)^(A36_MR2b4CCt(N2UnUiAcCEmW{8^b2=~x+nuCx9K2AHrX#Jb zvvRHtbqyUaVB*^c*s{_ERG31)tNKsST2%Bi z;kHnkM4CJCt$iQDqq*jaYr&)8KW+VuAzWQ=*RE|r<(s4Izp`V?M~W)CnmjDV4~Vx| z?+FgXr~crX8-sSAl33~{6Wt!qX-wq|YJ93blpaxdMD;AXty;s#w^ir{m%Du^D>P?^ zDh(Bcdq`dh#*aLN1s0bvZ|*cW>zRP0o!O*GQd9XranO0fky3oHm|8Q@c;y<8zf}&YF;eeqoQQcu>*DRo1 zke*%WgxgQ}spANh!QLOy9E;K$W6|<(_NmJeq#706^fR(^Y0xJ?$^5?|si;=Ka zr@R#TQi#~}ykf1CfBKbXj*8>nZ(usALzTCR%-Du_EV>(4JxUs>BD1AzP}H%`W-8ZF z`DtGl3FNDkR%NQc!)**m<9-6F14i>8C?~>%$S9`P+quw9X(F-Wx{flqy{fJ_`8TX- zyGA1oqqImx=62!Z9LiXJuq_YAskP80fBgf5w`}mWGC@9;uBQg&v-wE*oAUdb+m>V4 z)N_>5Bo+A()h<%5c$~(_pA@%-O3BY)!jh-rQQ$#f+9{^A_3wxD{xsUZjkrrk*{(*| zh_>?*b(En(?%G(z$WDJ(g<2&%;&CBq9;|+Luk|7$Hx8McE-DIPK z^e1kaX2jcH6D+un9oQ_n1YQSv^EYJ&vE)-8 zlxFwDkL$D1U`GRBdq$~i(?N|Z_~neAY`du`j;Xc>Z>5@xHD@{Za-NIMEuyp|#rimG zZ435y?lxAb$wC>F7tW{FGM7h+LuBa8D|l`+?UDTQ1+s`_>{fmUmN&nEwms7DMnGFU zM)#~;$bH5hWRBn;Q-^@_kO^|bs}HbyS}VFHN64OM%-4GwW1^QHUnc*fymcp5DZL>! zxy+F?=lCI!zViy{1%B--@wT+L)tui=L1(rJ_EkSDJrXw-F}7k1^Ki?OB6mcw&osDmMrda1n#3(NyUxn5BUkhj)yrqopZ@yCMh^Sz~myMqL~T z9MrlSv>6qy(Y#`}3Gr}g;V#f(_2u^nQ!)KjG*Asd+~umG@v|B^eqEDrnM)>{YL!XXTSm1RCie_=gT^xq zCY|+0B!fU$&K>R%Nr3Y~5w=`YD2F}n`FBM9rZc*3f>XCRHXkMZ6?DG|U zGh*&6MwkFI=y z>(a-WH*nX{#{T`d!k!1AVem1$0q;C24R-`5VzsT^IUQFV-kg~8N3Ra`TwVEuo)frU zdz3hEGfH$azD>2}&bpkd#yeyj)fsRQS>Os~yYzyBC6PSYxNx4G0fs;IdW;^zH|LezNLP zvGaLV>E0ud5eDMbdTnIcpN8n)C=qC0vlrFM0_t_yAS)l!&n{<~>#jnnbtonk?qKSi zI|9~F0FQx9h#n$#24IIT!)jhJHm}$bKzF6MLKEIPP2XNBA@F1z=509bzO#E z!^dwH$gxj%LikR4x3l(K5HCW<)K|Ktd1;KW8v`1F?D@b<`gLv2tO{(H{fU-*X%*D+no%_7{%y@?jNF7(rv0hl4L7h54I3QcW2&`6tAZ4m zf3*Nb&R=TrK<8ssTKGxi-%1vU%_W>c*S*3IQJHk@5};t zXUbQcyauEjx zGJPmq+cy*cm^v}D^Vzt!vmtkGQJoVuvBzRN*i9;9`k?{d+rgh{#jcz2vXX^p1xRB;k0{Lft1AM|_muDGUyxy?lb zcTEsZ0xx^2_+M_Xlb2B8}CrDdhdb0-nQ@O2F7#{ok z2~O$Q5C)}K%A!l8Ij=2+>O&0pw{9gsbEfmzm5f8)4dkR>boMNXGKyR5-<^>w`buqK<(+(-`pcwVh99`fwNQr9`+D66$dT%)*@{dk#4 zJRJcex^IGRbDeZe77m6>dw;=LvtQUCt2?-!@J3oUP1WP1DJR*&$m-B;YG2Y(W;o(e zIwX`dk}aEB$pgLWseThZELQ;OU6|8o1pb({Tem$e36&1N?{E}o&4hlNIGoRnh?|;e zq@!Wzz?0x$Z6Q5Jd0|J>=2+=3vOQ0nVcddJEwJuPFmkBOsUP@4s~d`6mBs^wr5A?H zN5z?E-yB2L*QAG#IE=k4$OPhD`E>n8!pCSv+68&pI7Zy1Gz4MaQux(kGiGeCm8a>u zfaaka6b=a+{cqw?KtJu>DPu`%CiXvXt?HF}JQ7@8cjV0LZHr2y6VCx@B;2ucsQl)9 z3hDeX*m?o1sx?I9b`@^?m8u_@VJu6gkj`aPtHWTqX$yYkp;d=SaNF!v^JLIZJw^Jt|YvNy) z+ay5pv0H(#iLJBf>QZ@JIhe~$PYN+leUInp$d%to zX=1}4T{DuM%CiZBc8I49JSu8Dse~&|r2F|Q5uZ(ly1~z9<3m%~jiFl6w_O3GAHdk?` z;y?INcJ+AY8E5dou#Rx9zl}UweKx+o_Y3-n}F4NY1t1McVZ^WDV{l%6i^ga9yy00PX9wWVggyYQhK)TXL8gU~i z4Eydd50&R2zG2gUwgK|Kuz9I1w|aQG0yCb^FvXl-&P@4w(iyn>^8l#RtrpO^V0nx_ zC!bC}{tb{0k#9de(~a}4Nxoq)s|rsvp>DeH+YP*@`fslaMxt8EGw`e-p%QNO;1$ z&2m_`*8=v;UyP@(4;8JAJK^pT#!!1<2UK-oaxa7RKG%jgeH;459%6kH{h->`e8Rax z5pqAAIIJ(MToenc&Pc;E)vH&2(7P!gIzo+`k66tHb!44E`>{jf2Ke;aNLOd}B&G%B zg31|2otekf-zyxi71tQ4zvTXJUSM6e3(hnfE`91ARN5Ej{g(&C$AZSdC@Vmv=ZFW8 zG{06Yx+UIpZwcgG$UoEfR`&J9cZcstx)?@k)`D;qs2;(+ZY-#E{op=Hc?{4-?-E7# zT>16dCFIl1fwVe#kU>a(Q>SocV9{*ynDl)BJv&L+h9Et~YhT- zLeIO6sqV9Mr1gkz_Y0*f&#o{YBI(Jt@~&SizObFA=^W z)h-6zNXBbR>H+CWBrnNGQ*z={zT{3EoO7=tYvq=S;s5DV?s5yP=-w>K5)_Uj`3Q8H zwVC>TF_6~*!fZy@KpH;_?w#*}V)9p&SEw;F<#XsbP%z>hqg(?ilLD28betQtz1Q3!36^@;Lm1`zxe=h1&;TqQa-_d67EGPZ;rZ1=iBo zRO?OkG!`hQe|KmZtEsVtKx&q2Xr^W9wd*a>we}aOt+lqN}7>0WjT2#1F3Q=nnzAp zsQOUSe7h>&F#f?{`SL^pP;RG@55QL8(a?4MTXp@gXvIk^b#BV%zBn$H$Al;?pi?;+ z@hgn;j^(7CWxY>X8s$LpeEvaP-4|59g2=Cl{Rdk}%6%$)cFzTAg7PnjyPqc18dzRB z0K@?-pb=se?MbygtvPw(BVztZC#iVr%QFBPD^BXYLL+S^0yfrRDg$`p?#Sm4Dr*#^vxRFEp=#0^jWjl#I=@2YCNkuHWv+4$ z<((U*wiLwgxO~6VDgQ_Mfw}i6B`x|%wEt8_eBQi59}p)A;vCd_K7x}b5jTE))7|&^ z2u-^=;kkR8ut)18)z2FFA0Q8hxlLX$(z}fOHf zB^OmjLm3K^Pbd8u%OcwbyAa_RET2`MEV*t0eqWStRQj7{LgGnDRKW^Ieg6g8c5c z9F+s~=sB`tzA0mXhU-_0^I5f321`D%IroR5Kw7XO{|mR+g_Qg1enO}?-p*g;a9IBM zrP6RZ(mUGe>$jO7?VtIjXS(iIuW|@p)k%a~G*dW5`k)i(Fxu;;_e+({ss081!mm2= z$EdvWt`GmIIpg2=^x=k^`@t(dn|}WoI(KSKy6+5pGWNvv@^ITnsV#d)L!!LDcxqdZd(-wDbx={>_1(0a#G z)=_T-qx@2%aR|aKxX@-BKE8yExRck4+om)rdA>F9+v#nE?(8*?_PdAm8f7eylp9i! zLyw6afGWQtO^&M){-IB&H=^=vZx@wYC$3t6rP&vh{}QADMY1RDk4czH$A-u}FY?Xf z6)vLkC2=0KM|h|qHu?LPsqF8{kL8r})xoh(n`-2{2uH4B!aqMz{Ix#Mtl1JtPjHpF zsO8r?C&*+~ZE4Z&f=yyQr z!(NY>quo)>BDB49(d`gNHn-#~AHB84Wp407sIKHJj-gj@w z^DdalviP-_cVQhIxwcT3I57h^*U|^83(w$y|68o)Q=Q)$U?rC>h{VC(rZU)o-Z?bU zPTsO=hvkjhNb@^Ad2jpCSiegSqz?>cGfS%Qd%qgMB%cquC(S2gpX6aujVJWPUZAn# z2fao3+t>;trvHWHcE*^b)skl^A8T}P25wt)LGv>X!d_oDi6@QL(!WQ_Gc|Ye^1hpJ z{h`+wd9n}_4GVEdOd}jUd9y~JcYS6$N)8)-3YYe1Bu`jv!hzpUu#k-E+-=NsJP;lt zvIbIe-$~a%~amZCxr(=8B2;?gnt`zT)ONjgfDX9bC*cGy}^z< zZ}=@nEJ?}fb-E3|HFSa8JAbc8ebigfHOt+4j_g-{Jj-o=EXTTS5!2q*)#TkM*>7zj zI#r3%-Dn3i$8EXgOY#w4f9I6@e!{18KXUTFh9YmZ zd_eD4iPio&{U=8UeHMG~RTlK{CbuCz?>y?f?$ll<{MW`nUe`QurM0D`YsYzU_t?!x zOEL6&5ai!)2M=G>k{7cZYVJFm;gOC*z&3N82sAL}dM25;a_}m7Z{rEfH($uTlf3vZ zM{j(4m&^6$Uhw8;3{KCrmZzKA$w##=!=$Gr6P55n#OcUJIbt2ChZ1sC15k?i*}`0DNsn;+ES zbbLv_3x3ltL+5XGB+wZ!u5D{#+YH9G$6E8Ezmv>>`gr}A6prp021Xn9gLdrd8-X%`qw4kl+;$PoxcVemv*SYn-NQF zFzw4EPV=dmwW-Q|t342pX72&x@PXKK>2{{D!FQS~-xBSMlMeUhQ+J#c)W2d@?0M{U zIvwq2IG|`}P4!bV+jP3V>~PajcI?xJdmM4YTKV;P2Uio)&jWQ=s|SO3mp6oY7opP3 zT{_k8=DdZfc}|!@W1pn@Sz21I0XjBlHX}KIw#BufeOtLbW+If%EdaTvF%(|zBjdKZ z0v$&bnV8|J`CTD)sDm7^+*4k?Q=1Q1en$H>=9qNoRY?wubp-FEb3mAa7qe*}@C+^j zDM#GW>IYu$ZvsZ6>hf7OW~w$_OGZZmT?_TmIr#U(O!YN%ZmAFN!&>nDHI4YXZG9wR zFE0Mx2?7lwb!j4r=6jRwxg^#mwf}z%%#p*AZ{&+3C&H|LftQEN5@d?O;KjwzH6@19`1KCHQ%T zD-u^oJEuiB{DwfU5e%03YtntXvm9_ZQ(W$P6sd=R>J7%YLtA9xE4Y-HG-$s`0Xb-}rQ0bxwT9Zk(%vySiC`>NzjFE3kCgIR0jR zJ07z9JK=LMpPvQ7PHb6bC09&)1Ed3xxSAjK8Ut~9b=iStwz#V# z8w&qwl+(3B8Y56Y!TZ(!fUsRsj{@O#1^3hM5H|0l`x6(V;v&KZanJb)JKpJ)szEWL zS0{`)aE4J$=$`&L3H5_^6CeG@TzxpCH;oq`&bJpGOMUQIjs=V_i~!Zw#DDT&hdhjo zs4593fz|}uS?#S%dp+Ly(CT*V|rh}*GEGzO*EHT`>sNdFO?z7`zzJbPAKUx7pBK# z(Q%&w=^0jOZz=3gPuCi`T}Cl(t!RBDjoyQEN^=P9jVA`I#C7_6Ma-l2(#qi!ZvWJc zJCE#$gm?5@bp#L>@TkNVlC%|JRxjCX&{Ck6w_u+s>ofx~n-S-Uk`CYC$L+@K;qhJ4 z%gzTa2MiVJGnu|Eq&lwBgs`I~Q(8l55gTU7WStwBHK!VeET1QAJU;7aUZm;+|DMU% zy}l!F+^4owx^ZuJfj#0{bJ9>a&+ea?V7?wsuP}mb^epj@YS9(?GW7dL_}%q|Yv*5Y zv^kaY(DLOjJX~uFn51U0C)pijk>_2=+SgwW9F~a1-QS7OVOKC`cO*1iW6by8ug?fK z#IaSoNuL+O%3Ora7Q+-*NW%>m*>&xdXm;Gcg4e62|H6(dKB1hDi%o@PeEJ5KB2I)hCo}x(1{>u)mI;S*Q0}t-4GP?%y7)v@pyXqR&h3q@$_n zG8s9f0oGcVF8Y466U2Y;V$~h-Fx(lc9NVvRnHobq=gD54HIelH$DKDUp=UnE>8K`f zOYnhd?|l>Gz0-uLJ2ZG2ZnARBy|mb6yHlI*hysFT3c6ZvU+p z?!|#>>_~IViE}ljQ{KM}Ra_#QrU+16Of|{cEe!{g*;YQ(P&60!@&75a}3qM&odO*z7h0j9SE$j;A?gCM*~}54?o^ zQ>H=3LsdBSF!Oeahr)!)GWy0l<~HWNs!v9`gkLL}%#AjV5RWoGFydh7pYjfrPji{| zqN1jh29O8lLE{F=V3ikz!)J_w2~S#JtXN&q*W+)F5!7#Dp51bdYLqw;| zUWDkmO#f6{`7)t2^zHMOc*$`CM*TBD<*^8taZ#iNo0C{Wo;_YwE}4=K*Dk_;Fk;etdFZLaZ-qgx?;{r_ksWO9(cE@C)hca zYnT4)hRk9AG{yfk-*RJzHlnO;N64GdSt!lQ>sI3yFJke$$1WhwR=p=wpQpc1*24ES z0n!_|*6gU}UoQ&>KRW{CS7FiDUfdw70ViEa`v43_)9E3gI5kiI9)vu-&bke6#TQ{1 zt=$kwJ+9UmN#F5vzZ1|d;TBLoQGbPi@{Ux$Lg{tFQkb6q8_64?7NLnnEnJbbpl->O z?$o>9oIbDfo8AZiU0#NIon6AAfoVf~5EpqW4a;kX>_E~Z;&kLA zg-Kv)`WIWau8HjTUtu-M5zYS2Lh=WoyneI8deFW9Jl3vvJ-%d{GroU!56F{<-u92g z(jHrZ{FLrU$6FA1wIMcJ9uMh$cOdh`J+ajMfWmQlPh2*p(!D2hCRvfjGUlZ3p@^zl#qsLwgHOT8!B~(`UIG>JV?x_oCNT1Ua6kRN=(wyt+qHN&HXQ#A315iEBb4q# z@|-YeRFSk!>SxxAauMnWNN^2;MUm?j#(+89gG-pmMi&(T%|Fgu z+(tCI_W?-D>!^Rx9rAJAgv#>svE4xPTA@=(vuWhJGQ@?<~77joC)`{>hPu<+!5#M>7I>2gjwS^Pa>#9eN66~Cc|B<>_kD8YipQ9!s; z;d5!-#fTUFtkKO4u<}zIdj4QLVdyR8?Z{iEVD-_hL9JCQ<1S#^w=7EUi6DkdJcB z+n6eLPae-stoDK0Ayz;b2Mz5HC_f}NTij(C6Pn<_Rt{oSpslukfi)I+4y(|L-v>Q} zTLX5W$!%-)yK4aY8GmG?g@EQ4r|k=8q;n~QX^OF{6Dz!?${UEoUHGAVCv=#3gM41Hs!Mje;4@SGOMDM%Zt2<}ar5R1ynL3{8VKv* zqOTht7+nn;j@*jm_XK4+g7gwDw49@|BJ%a|Kpd{EFg%9qoHruzq&)n9_OOo$164!E ztS_;HR|CLj*2pS&TWJ&0c2n@l${sn2KNnQFrD{|Q z+IfRT+Ux`BbA@H-6y&aWMXEf9bOnwMnkA?fIpt*XV{$d5bLptpD8m_nm($h&WhZ!J z{ice!q0g{2E)QVirj?X~yg=8dt3c_)Ext7vWg?2BM7?oOsK39V4cjctcNsvcXDgZZ z<&}=~29Q@0pG((^OA|+MuX91F&f&_UUr3o0P&SBcm;vv&VkYUT*Ep-qTb0RaiJL8% z$_KuTTBGJbVX=1nvNN~-9xV(5Ac)4XhjCpJ(DJLS`&{-Qksfr-2RiW+3 zV-TPAhf=-@2Ux_x+&Wc}&cWz-u)aq}ZN3DAQ5gNx5M^i24^Q z48rzd*~Q|V{ZD2p4ZyOxHYAKb$S4ET^{;vnN$U!#4W&u(FMYbK-R}?8LGbSlYAFe1*QTUeNTCKoE7UWmVJ8wqz_rfl=)Eq?tR^^ zxnU4g=@2MdlJ=z5ho3+^r)sdku(w;zpVj~$I(>wq;%ed9$}39 z5!IKeq+Cp>bJN;a^oRTPINTmX-*yM1ga;A0wXw3WvMI7%E+i*WVrl+h-xL?Ar)F^T z45bT^bPXq8S5b%Li9l&+%F59C;(YQ~dv%mS!OMj0;P7I%E7dcc9kh=!;Tc@zPUKy6 zDr=>jutMWmTAFf?dRqm|UMQ&31oadr{L_&(V`)RrR^-s^yDjCJX8d@kNoWdYk~{#r zH1PqW{>S_2!^Qslo4{pi4W2`LAdpS~(hoxE2hul4`2q1}?+Pv^kEHTbAe@xMH(G%I zE>L(!J&xDl0)(=%yL>sqE2`<-F*p&HV){t|E~)n`48 zJo&7e8Na zh`YXRr1!gSyqa?sPXA~MZuV8AS57zRvhnc?t{b*3Ro1<54V*zlzmsOhNg>k{Gp8<-)7JVw5zn=P(&MOld^#K$fW&U3_|6SSN&w~ z*Ae)&`c$N!olJc_Vv#f+YGkCq9=6EMdE!ZDGOi@?hB>Gnc4D8#%Ri$%;nW*yRqiSnXq$v zBf0;~4qiR0qtuQiiPSiMsO~!x?E>_%_tmQCv1lcYr!wE*-9Y|ZJe}(udM}c`6tmOa zM#BQjLUf5<0akkRbbL)X{F##}0%u-g?VB3PXX*B=S>Sh3p3+VPMc3xzkJjY2Q|Duv zO%pkBMJ?&|VioV0WGLMZd&nEXduV?MUknMgkjr=Umq(m-$;WpyaM$&%=%p7S8=U{j z?tXa=Uz#|<-`GrinD+ww+nC6#&JS2|)N^?GFHjzPu?w^NHdkYWxo6^O{*z(r(L@Z~ zUW?EAx)$HRH~`TZgJj+b8yVL_p z?)-&t)VU+S7~ev^acV2$M_TfQ2h8~E#7W%pTQ-=jK90VhBeC7gF8q(#botvo6SsdD z%Bfb_@&*>H!TC9SUA(27;=4u*uYMi2ytyU*>wR8OpYsvsHP9|NgkA4L&&Ygu25Z|` z%8iG%;L-8T@WRtEvZ%C`>mld+&~a{V1wOQ?AA)Xms>p5PwkTTl45V9$NH{|G3Dutg z8N;pkmnMhco7-D4w(><-@UJh``dhf4)CTWkx-KPYo7QV|(vKNNWVX`8{Al3pH>i4EWZa&Rthx6T;-B9Wede zFnrl#zud86D-4=X_vf_@hStYHNB9FScJ9)y_y;3wmi0$}#cS>Nfa(kJIth|n-l6MX zD|KIK&&U?l+3@Awyyk#o;N|xZKIl2}-pOb16Pv<&UyTsq12^Q@oBV{9V}oJj%kyko zqejy2eiRS}s9MAU_qxOWasSETo4rw8$EC9^c(4)ed1>aw`|s|Kf6U&qt`kqAntPX7 zEAU9+eZq80*&2R8>!2U(S^7ROo6%Xu#%_X*hZ@MZJ@vu82W5I^H$&o}+pJ-?9He=} zANQu_C{Cc`0O1Ix<@rnLHVOxq^pL~qc~gB~ft3I55Pl|#LdurgTtjsBHPn7(Oygs= zU0_l3ZxH?+V!WpZgkH4}BlRzUmtIwF8a)eH&B}7k{rZx69)4xK6FGs+*_isVQsJP& zs^pgbvfu1{*5Oe(v^u2cvL|gjsNShJJ6k3!AIxbj_^R30;lJMA+GWy*#1}AhPZUni z>%)2JZ}2ybX7iTzrMHbg9ALT6Im%AVwXT%4xb6IZA>)-ozYYzJIm-hRG;@kzJ*J&RbI0E$^;Z+JIPv6Ez zHtdOyqV2RZEo`OVK`-e|dulfM=?ZDa+o5a7X)*kfiKH51L&uzh3!Z5e_()g@1Hjnz z{f^i8uuBHJy?rm#^LJs?1G-7q^Dh;szhgg|9~M+oSR6G$9!}T^>&{wY;fCFKX!K54 z+L^Mu#(8M+s-waw(mM1!K-GGjdKw6$;bWkqh#yJMl^KM>a=rC}jv+p6JqS-~?nB~x z(PgOzoIm&&8m-O2;_SWHEwL-AV=hcsN55Z{I;5Ree&R5D5EpipR}d+5^tPtMsr zwm^?&iI069(YWbM7|}KcgKzXl*0r+yHq}W|U2v*@?Sf|u=$I6SSO2!8T1jC_2ZqiV zz$3fezzq$DK>7T4*m8p&pRlW!>`8k}DC{CV0G*60f#TEK))V9i^BAt?^y;&ge83Q6 z9y6>y(i*Z6UOOm!yb=!VHIVe5!Zpvg-+@SD2Z~kaqRB=7N z&+#4(cCXHJ(I z!;KrZuvJBMtYR#F2Ox^!!WnVnmOwXVR79A zLA4IE?$(y2&Xcg$3{O1kH3n<>&4Nd1c0hU&eAo8CenqQsZh9STpLzmloGdb>o;SLfyLcTC2Du`MOlhEUqit}y@%jq4o*@Ak7 zyDoGG!cLr;WXj?fZ04$eOy6~a9-CHa3KNv4AuJN)bt>@xkn?cqyssQ8J9dB|yDLIz zueY6FBl#8>zlGib`=tw7>3wJIs~8H>ER48S=~&UiTMtPWquV79N!pr`KTy|(&(G{r zHAXm;1P|%G#>LlLBh?Zo?8E^>B4OcybwK>dd_NnDSzl|Dmd|2S|GNw^?wbVlPz4{7 zcGu3%v4z>cu0Y|T-CFIFpOJc&yv9z<%4`P2A>zdB64H7B#P@SZE97f+f6Uf8+f?F& z_h?d>CCWv zp^i8Y$#aQCiG>PVk#r+zo$sLE-5H6~Fy%jg=-=EJH`h;OJqF|{eJ#tX9%j|v28p4I z&k@E>B`h_N-RiD_zW&`|a#K@gU1>feT_zT1^%EE4)8P1x5rVWf7%X&VHw{`#y+hOC zZqsz)h^+@`{C6Hf2pMljB#hWcn2~6p)%B3IByFmn&pJ>NU z)f&y-6i!n0%?~Fug@g7x(Pp_|KZ6d7mPKXY;e48r7DTFfrgRzMufj}h(>?^$bsxX+ zfc1;%E`8UYt-xg(SH=1&%zfq-0N$e&=X9mkNg0OXrJW`x~3d;0F=t>J5> z8P%Fs)DR4>4AGxAvX&cyHPYo8)sjZ*1jasDN?&57qch+`2S-V`ie5YNMEl$m_}@Mg z{_lQ!ZsRgvJO4cu?zIdMPtHvQ)vM&cM8BAPq&@)hwmR$H3~Ds)1n0lovUTyqp>uvh z|M%4-@d51_-4+yX*R+|1Lr>E?PwXagz1q$Y_If>bZTc6kwB5~)>m>nsUe0#OH!O2MLcXnOd(7)5Ab%4yEy1 z?3@ouo00cG(sGROpYuk4MBL&c(rAs4bg~Fbz6N%ELKtC)Hn>HUpmDI=_sPsVdO3`q zbOF2lUdhP+Yjj?!zfe5zV+YVWY5(nu#!JhehzX@NdGzOcIN#okleW~kkFLUrhmkS? zl{@G*jBOxkys$I7k@z{}Fp!=lKD`A#w)t>&Pbzvicjf(?M~d04>c~59_fij~GMX2m zJOS05An$~PX^{D3Cq8p51*Jh}w~7b5%2Kq@r+s!wzpkz|Nc9<~IuOUEjDYb4H5ERv zm;XwI;%WLl?6Jv+AN+TZDc!o__5i4w9Yh&Ggy>dyn*8?!xU_sMC=aOcckjt2^uC|A zY=LDzmeF;GAaBg59)-d|Y?2J0U(?<}q^Y|esKP0;;nYvW-D?^75kb5TpF$nst}a~! z+4+dkVLg$25j-08lDL%Ki+nxQwd%j4N{8f-meK`vzaq9CZKya{w}0F{Hmc)=3P0Pu zaW^zt5yU%07($;)tAVl};adN9j&A=+(%}8z{pGgGZy;${K|TgcM?2v5{*B=ES4D?AJJ zh>rX!J|9==(kcw`?Xm+ZGXa&~5H2y&$ZSiWM5JrOv$3hj8!g4yIh&wyNsh)#m*Ccd zL{MH{aSizpLDyMP&%`Y{mF>iO`7+X=EX`&+aZq#Vv?3IZeL7;RFo3IJPgULFIIeN-n;H@#s7avo&wY2E-O7tymty(7R?72ze<|!ZY9}rtR5%cB;HCDUCx9fWlKQX zO1zr*3Oac&mYom%64PzgDj%V#>zOk=9>~`q=>ug-o_t(+ zM2$44h&HK@)Q^I^v5tHj-lF@&A4O;A$Q!C0gO79VMY?mhYfQ-?F_} zCGk=Pp8pqbA_>EAa&B*=YvGg;Fy*zMZ!E=e!Lbz{fig{z@#`)$aj2*p7#I`@;j-ryw$NBP0eVDlICMzonkVmP_=M?i8fck$Bp5`-j^? z)y(*5z36!vw~A|odGx*RnJ@7Cp*<(vs^*X>{igB{%9IJ;PGdy#jS$>-y_jyi7AWt+ zpS9k>Z<9?=RL^Gf4)=x9 z1P7qGWNqgx1f|a?YfwI33@VOfeRo#lE!ygFTXWh2(9D-G{2%r5C_$Q$@a+jK>6O4# z*6hSj=tvj9#wA<8E?A$FSAa3cd*W`t?VxHPDYsbX{Vq@CmZEK^P?eKc_$AUjKv)2B zYBD`9K<_hb6vra&U1wAes)tFlb*#|$bJOWX1MA$8vT7~S>NcBv=TeT!Zb+-4(re=f zT@>U;g-OIrZ1z-BnJ*6?SeLx*Wu~xNd1}IGpj?YB{%wYo7t(rPA}oMkm*jRATsEdFL#p``|V-zt{v>z<6+Wc{bx4*CFe8z!=SmK zJ#a>S!v-03VQA|^nA^7oCIxNd$uDAcvoC$X&l6h0)=!IJ_w-dT`m3d^S@k6>zV`@^ z+OL-uNiV>xc0HNd@hwILOvgTb+;UdjK8o*-8_TES7|!rFlqT;YzB#kax?amwzIg`11_hspaHixX*#_ zN*{^!bDdd0*bQ*(5G$MP4aSl#mEe$XKY1deFOIRT1Xk5vh*OSj9@*<(3|)Bpp!@3^372Y%nP$<^ayIR36uN^`uFEe!=ppW^{Hn$o*>OW54w z*V&uJcSZjTpK#i=rt;v!5tyR?4oq71k~xKMpiW{ce(JzwJL3sB&&61NZrYUh85*vO zzFz{_{^g?PGCz#AcaX_Zzrf2p0Xh}Lz`oiK+4G^7>0NXS1^tPs=j_;xH_dpp{JEIQ zw;|1sj^+Vdk!G(EKUWShjF#H^bS6Xr7Hpd=Y3*3@i<;bVp#}f7<~zp#*w{EoOj}%6 z{U`+H8e&oHvoP04*(VV~sr_SmxK8oO$Yp`w3OZKxVtm_>Uw{j?Wn9n9(d zjtLO%lP0JZfv#KT_KgwG9^b|M<^?#QS8GZ2u50CkzH0s zWbf@~RZ1FYpb!Xt%F159=k$k9=-&6f=bZ2J9nXE=^P>KNL`E3H zSsQxre>W$?Jm0@Cz@iKHxao%5`WeHqwLigOR~gp(XN>gug5CoMn*TzJvSE1ejk~1z z;p2w>!|OfHf??)e$f-U<5dX<})1#`!ba%^fVRLtdLr z<}^lldf!44m*RglYcq4>5Y1IyQyN!Zz_>DFKH|EeX8p|wDEZX@3&Y*yHqQ&tdd@_} zqcUc_j{F|kk!L)5sIV!1I>(89Jsu&gGo*}35Z%mL!_tZGp=?+`F#k~rlR}5U^r9&F zIUx?#yb>a=wK=C+hDEzN@`Nkzz(!KOX;hCly_SZJOFc2ai94q?74@5EyBq2_$Rl)Z zvO4Z5-~JGq8B4sr3W=+5LBAHfg~c$qn@0P+_B5mUe~C}_XQ=a(H=mT?=Y;yQ)KFLM z*+X0K{wE6UMWkuy&cVm>hs1MD)GhL8a&f-J*Z>cUu}!;=GD+x zS8vW+dV3100UYT(7pbmkFP291{htL6u&)W-*CtUNbbuQRk0`7uPU6ER%vE^PES+wiJ!#jcXgeJe&D(=J zj-_c0t~2)T&}p0joj7qnZ2RrWi9g*-MlAs$=w73dTY+>Jy{GRKCl+M#_<;=X*!h5I zHyycZ;W*LqG2QnRmmrQMcI9qq&q<$WpjT-tcCN-7xH~i#ChqKn(0P}_Gjq4Rt)=?r zW#i*n#Fkl_x>t82y`M*nZ7WrMjSN06f@Y<_; zFv!7cjd$Ys*bI>x6O2zyMxX_UUt8`pzp7Az(1I0dil zFT>zvCOCP$1(fZz5Q=ALZIo`}x5iOyPEZS4({NDOe(XxmO?$FY9^0v>Ar4_m=ar{f zaU&9kBzy!KFCUlD0BB8ErPpa3YBWLBHqbc=lQj_z}K z>iD*@X756H>uCWAcaO3m33GvP4#cgvs2oLtjbP8dd_#IaDxRis(K@WuCbvDLFv_XU za8jdvnD@gJu1#nqBP!{chzG{-^Zz{%UZwO*t{z@;)A(*8HB3WV&;=hn$iNS+7DL|t zb70!$i}-VSkM^b8D>gswF!cNV5&UP^O6Bp$pF!wXbIruW4~%M&(|NHEVYQi=T{3x# zYiN7K6N3kUEc)jmlWyDarH+R&u1OI%kId34k5{}VH!sq14<0=@x=Pdfd~t^0jfM0K z>gtNqCGotdH+CUBu+4=(mmMI!S_SO8*Is&C6+8oiX$a|Su0c?%%}l4>X1OcjB&J^Q#JR5rOV6?btg+)Il0TG%R_k#2xk40Y zP5F|r!GxC@a#EdVK(A%@yIRtExq{#KGk89%nN(U&o=a@m`VPrwK*htuw1!s7uZgqk ze!#O$4WQYgV6a(Y#@9F%;*OsOfv_lEG&m!?TDqXmDQCI0-eVvS#Z>(i-_gZc5i9Wu z()B-sZfHhm{|Vw~m^Nw)9Q}2G&1$(tWWTI~#A|H($gpNM9@zgj|2ro>;xnCNw zz&2;ti0jb`i!%GcIohAr8Uk&D=;t;~QusjSnU#m-RKuccb}`hO#$b)zAGL-!ac2%x z??hiWIeMY0sbg!Gx|3#V30HJYn~5ZTg5u7v+3)&J^mCd}eA&G10^(yU;!=A_-dxn& zTu;m!cN)JJeih#%k7>U?yNAt0Ej1?G>1d|>KAgYM3g=&%P4{44!x)e3Djr$t=p+|y zo(;sCV#@3sFx1&261$IO%ExX9*^TtsQt4&#!!h!d?^VTdX!NWbP_1* zy*AkQyMwM@&k@HB<01blF#XOpaXWn!8|`x()BLOP^gGtLUj7zVw)*VKDx)gjuGU)d z&axd%l!w<+25=7xp!Fi3?a`PMg?07aTEaJ+?(~K<{swLuwj7#A)Phl8 zGm!i~%sUcCSegt)1B`ez?-5A;#+|YX9{%(bj4S&PSFB@kQNKasr^i|^a|BJ_6F?r_ zP2qs*Q&VSn2UME%L)(OVB{$PPOl*nhuoN4n9}?uP`K`;xg~Gw$PDZkRFE5}^+8fUM+56u2MenGZTyZwlh$g7^W54((zD3 zNxaA@9|`hx}W}Bpwq}vL%q8f)8O)WpA(-k`kgp# zW5=(oqI<~_=a5J6gV@>0Ncw^g2k&6?8rW?A8@A_as_;)Wm#_4V{DP$4Q${cqq|1_g zJD<7tDJ1XO&PH#Kh6a9(X{~$lTC-jPaVZXe-cp_|Hv-BEFx@OZKil>iqYRVRmX~SO zIwn7Sg!^l67u6o83-Z(Wr`u-b3yDL!!}Ji2>KM;9#ZwM>#{7tt7%&0Vfh9YG+O~s9J(5cr5$eW?V;e)8M zHuWBuDqevbj|C7tw*z+ZcwVKcg}3QD_uMFa*7J;3@B0_HRo@F&9(5o;WXo+@ z_Mv{rIP4x$3^f1Bzkp}QbvX6@Q?`5Y7#5y4h_WbMdwQ3i*Q1R9>t1aHWgfhzJ55l} z2q;Se`3X(yIb|Ytc4J8%1?p@UxMpQ6s{BdV&=BY4kp`go7UQbkisHEjNzJQ#%VXE$ z7<|tgrwts!zqW4y)WZUmtyi>vuOPn)oQn_EP1|3qMV`dvj~$$<eW_;jFJ;mTiKc<+Oz-N>sQ-oWU*sITDz124=)#c!z{-YK8WXx@N$kP%m6n{%Hb z#-jsy)rFX1VJsV0)T-i&>-Q{$>aoS$zfd(F1G|(8i;lFn*2A+v7|SQV1G#N_)$uzO z_7ciFP@QUD*PN-Wc$xl^$4LZIlc>Hb>oNRYMjD2-e zkXCV;OVK=H4bE8*z{qb=-j9L;13hWh=&tf)TFM`)-zF4isoYHb$ti22@^~KS=OOiv zs{F0Vic2nn?caEzzS8*h5D^3 zgMi8}=6y~HolEsN=_6h$aR%}@e6ht|?f0tyq&N~gIYCR1Og zKUeG6KH8cGT%5y67ldoTe5KFmaWfD9js6O%r?q|ZOrhp>hRYnNzO0aRYbT@pA&CF- zp9Jn>UuKk3zwa~@TGfG_O|O%WSOWuZbq3W3A^m04|Kqm;O!8lxS`4)AO3$HjvQ|TV zNoJTiT;W6zx6$6k1u$`RJ3-z_VHoIrP*k}@WeQ=ozK`NrpbU+at*Shv>hFyV_NeN; z5x?=yPM^g|`p&CmOm9X!q77@YDxdN?teNkPo}1<|>h~gXtWcU^_Q))sIG1{9chO|| zDXgnMnY@oJ^=|AX@g^fqt(yCd-^?_*6<4VTz7<+}pTvt33-Fss57Mq2?c<70T=AI7 zMK0IvBx#&by4a)oI&^<4@#l|12>ZPacZ9COytO0Q^kF7=tHUaBtz8`SY?}v(S0Av} zi6_Lk!gf+*)sf#?HIg6d*~1#wW5T6RZ#iUaQ(4DlI~YH5sL!&_;Q~OZ>n|wo7fx1&+!7oy;d@ATpRAXV<+x@ z*iOE_Jri;scNbsHr(m$9r+gJ#2wNR;#Z$2d+cu!<<^Hsk$CeC(4GY@A+RFZL`pQtb ztW!MPe%grJ9!`aJH5&6YttESYtrho-Im&&SPm`W8t}>GD4XJVBD%8H$jc@+=k&``%+i-QsMcAu8_}`KgIJV>tw$7^#{jaCPUGL$zp-v&qqbKiLGYkJswB_389i+SS zCOq_Y7kU{ja=%7>Q+mC$Z&goY{dYPK+-=Bc4$y7S27GbCTz**|!sZ%K{>{x-$K}KF zeQ3NHqJo~aaO%M%89vb;+FY)QmtOSN41s8PJ1+&YR-57Sh67~|eV3GdU?ulnR81Rc zzY8~9m@eDLd~(md?jqBShsjWSmYq-fW(=Zh^YW8x%fE@|MZ;LZre28ysOBkU_7k9W zle6?z;GO0Fgu`tI`8CB1QjgI!Sc{E-)(;yQ9)$7P+p(jgJ%)^XgVR1ZKwjr$obu0& z(^`WbJI-4=h43Y=3y|gj*0nOl!EwDX*ECX2*f0vR0&ihv;{*KJBPL&#w&C4JuLU|k zPREg{KZ0@o&j>+y1eZSNalYX!kl#DQlxi>I!wMgI-Eb<9LGF99_HsWoMFu8 z8Z1JmE-xG52!EU(VDa%9+&5r8w^^(&>xyJe$Nu?Z){W}uz)iUIyKowJ6xO9Z%YU7- zmWwGb67KLz z0hfrK(8t&m<3c@geQY(DsoNU0siS4W>Hi}@q7FPeI`F+5(I!# z#i*oKFsPOnFFA9L{r#KG+Sqo3UXAHHyAz3k#mz81c(;gdSRba+cM2s2DR4EwP5hg< z1E(FjhH>XM^V1(*VU6XB!DU?&Z1+ST4)z_#Dw=fSrg~bu+~zY-o$&=>QDX7tb-1O# z8|WP2Cr?sy1B6TNyzIRoOyQ|Dt)&{@_&9x;6SanC(Y2YKH%?$t(Z+I3#79wMdILEG z3}x+2(}`2Y!k@e-Tsp;Fjh*mtj%wvD62`&v`D|F-t(P1h7lBWQYS^1JFYKUijD%Gz z{Pzn_&|dJOudO^*EnKvTIPAVc?>Hn?8p~Q|y2u7al~8B10XHvi2;a)O%JJDYJk>e@ z7W^(jQTU0)9BnB3)}Ji0+qdPz7k*{K{@BYJ)8mowp*a|nORu9RT~|h9O3ks z24jy+fs!-5_}$DNijTmpPBpo|^Jr$g)gKji5-+=d@7I=}Oh7nwDFjG6fbcFVN*>{M zizVRDe+kA|R_BfVyfn5&$8g)q`dI()V>ai-YHl8~AKaZA%7;JKve+M8Fy{6FTpHaU zsg5DhFB&a29inTZljP+#^KtspeTsXl>LOt0I~?-e6`q6#fZ5UO+IGW7!pHXk(6Y-5 z;kD6T_V2t^(wv~G+qk;On>{DvpuXGT(5!`g(_d%eqhR!D z{#yFJ+M#$i-(J5z#2lq-yuVm+r~jQ-^Tt-~i=9o|@aq~F1dg-7AL{%veKc?Ms&w~2baTv6H**R|S67>dyl zr!kL?dT^jm9XT@c2M{mH>QC$O>LbR>o5<-6=p#A#0P9@q!tY}G*Y<0 zKE@lxl1H7{v1)qYf80adA2SNACe-9*hhGYvVKG!^M>O=4Q;p&`y&7`foLYoId)g;< zkJkK5FiiWf2bcExmTPe@R6FcXnp{+OE$RhrV%3-0leefY?N052lb*q1Z00`^)yoai z;i~AC)f*QN&4F{hPvh)czJ%NAoUj0=0vqy0b&aq)cyrPeBs~FDN8P@r;0o`CP!jTi z#oTVq$~q-5@(AcVurBdXFe6-}`rWH<8F1*Uoov0}AQJwVD4d6DTsw*-i!Zsis#Bmi z1E}s*Em5{vuHC<^gM85<42F$&{6R%)j%A!=`jq+11WCCv_*wTxP+UNT8ZI-aX@(IJgS>^K(d}3=U<2^ z@f#U&ks$q7^+2`vKx{0r!teTiXc(p^W8-h)d9&?e)Z<{tn7aWbW+z3b&NRJ3xHFi(H5FAKyPk(mL73_7o6r zu)CS3f!T)QHjS38%+z7M|CBPNb6t~rN*ae|llC5SInn_){XK#Y=Z#T%2o)3Ra9!_# zK-!|Edc;2j4nD8Hx#mOniWX349XXFjISqE?O4Cl z1JV&$GZW;V$u6=lT{Ai2fWRg?wdIOYKf!fn9d0#YHIiO4H~I$NRXb8WSrFBn6aHhsr(O(<;OhIY3W^2gpu}k_}`&=k~A649zP|lN^Yz385qJXppzsKrI$8KSG_URqWUqR1jA^oM_;qBfvp-9#%2a7E(1CSrnR4e5i&q_@>{X?62 z+mQU0$_Q|4m^r_8-kwvQ5d{MqaFr9Op2$NlVGfT0!-IMVkk5!V8RAvJvt}x?<(DB3wA(Jli$ousS!*-yw6k?)nx;ngBI)9k^QW zg5CXT-}rIviX+yh#EYiAvSH?l!HjaO!iju&Y8ueGYn2D-`RWNhKjjHhMuRltJ3yL? zv|dtWCdx6|)V9m$8K7TDS9TExc2@X@FO`F_;VtS(C*%=BeN^)WOYO(=iRYWrKA_uC zb7>jYUEHh6hZ}ay$fbNNit=d>CYnd`?C4?I16t>;(q8NO6_gKIyt$58H6UC%^#ATo z8p6tNS+X(HeR;Jh3wTJm1^JUNilgO*yl~uECk%-@HLoqtpy{dcj53_&WCHMZPa<&A z;GOK&utY}n2r0EEvCG-EIP%JMG$~EPfY#fpe2nr)RI@-H2t?L8Hf8f=`X2QwBh6$s ziz5X2Wc5=^+zI4gMBn-^6>f#f4P)0&gDE0FQXbbF8CWEgr=$F<>R0(XxZ<%((8Jw@ zN5^oWEX$ANN~KLK`7}fFsz7U_{Is?_b2StgF+4GUFSLBUA9JVn#E`UZg7ib>A5_^n z*>PVLmWdCr%Q9y=XMgf6blr{Jee`a>pOFv3<;%%F|6Atv(t8VwF3K0nPi)XA&NTbB zSJb0@EhsmMv&}OEaf_yDv?1lK&w{dm@>l$3v%%2OF%1YikU3rlh&Pbx4h<)qhSME9 z8R;l&nz$Xvm$B5gznJO;z(HN4&%j~#jis^91=hc_B~T_I4VX|hUoE>#0?H#W`GH+$ z^170|q|#+kH1t2J|8pv*Q9t3VAioE8EBhh&WBJ*>1@wyw#e4qx%)V7Dke(|%3+kh& zYzc&Krr+ie{Ox8$d)N-Z2Fu;h^JWzBewcR5=TM~V3*C!HYsm}1-5cRZ+6SdQ_hHLW z&h=uKR`DQZN1^;1k2(e9?c~f8HwYt3SVhSg$}Y=Ty=ZUBw?=~U6cBcpT37NC?Cm+q zCw+I&dCy??UqlkNZIJ%()MNWQ&+Hu{BK$788(lo2u#c{qPHgU(?&OIdvGG2cR68$N zpQitbZ>J7w31e`3#5A_^!XZ$-4DyhwAE4?VZHqd>*{O6L`y6jZ{U5RP+U`8c0D`zP zZ&JHD)EnH5x1Vl?ot}TOmVvpX{s^0;_lNcEupEhV>G_vEc$b1&JgG7U=)LN^k-RKd z-adT4D~D<)1!*_7s?!C@!^5%>dugr5VD(Nupxc58%58jr*2>NL&G|E3;wRx)6iCsIN#%^>smATh5MXz76 zNIe8~-kP`PMya}GuW!Gjd+e;a#ilk=<%P$4cL8|{q#gogTz6DsraC6y=nu=616aH+ zCy&IS7Tup;k#|950erWKo<)>jBvkHDTK+P{7n`N5XQUVW@m@PQAUPcMn1>|37kamf zSYyAlu;bTFdj7&=TrB>^?!lh)rlaB((#xfy`O!Eny`L~}8Vg?0J*<=&sHVufcCE@v zCq1j-mKx0{&#VTI>$FGv2@};Pw>21Th$|qS)By+N2m|CLlD=Ie|Ab4D@-k7*?PJ?_GjUk((zoVSQ8KQF0_ zhe|_U0`H`0aeEGfrMiDjt8j_?njLQ$u}KHD0kl z`2tX0S3O1yTTr1PPt107ISAqJqEM|hX)BJb-;%s_1QNzk`9|f}%gg==)f*vx<1o1` zu52{~Nc%bUjcJ}+DehMuRj6JiWgC?pw5n%E{ZdqZmaxQ$cYykk++ctwCr<$PceV%f zkXEYZw5F$?(N6m#Q1v}Yqwz%xUH;2@1oeb(YBT1VVCfWzT?*C+m31gXfy!%y1(^Ei ztMXbv{U>eH8V|*WJU!wLW1)JVq#KO-MogRfkcIC*iqvD{1D_wk{RM^)Uax7DA3s}t zD_Z}3qv>A!5ck-P#JwN0v`>XY)n}-hqxGrk^Q!ZsTqtPG#jdv}p>MTS)C2jBbzE$a z^0=y9@l7ujKDxTAekWz{Tg{{h!>w?kxuGwk=DDD*D#g~}s?v1iUx9MLNT zMuwQmrcLeS)hEIH%j+^+8oCv?{qW+}j@{h6&UD9y1L>NpoM+fz=RS;kKaa2WZ3R&o zS1~i72AY_L!0m$HA|>0De;iKF{@J!g+uEiSn~v92`86qopxOOwi{KZ`h*!((~v;d&_hwZ-`Aox1flu z#Imr)d~Vbtx%{%ROb{Q?*K(BXX>nJC#{lg36N!`7<>2#|rcf^|kX!kE2IKTzyp@+1 z>tSA-tMUFYH^PB-ZKUtuRAFMO!)5IZpyRUo8@#wf^!NNl`LDsRm4v?sd&rHwyfJ0@ zH7u&OL6~&OVs;OYL+35V=sUP6A9~D;(|M_R7Ta8mnAeau@N39SzFWH+KOYXmb+sJO zYu!pbzBdm?zM%VJ3nkdLtBDSqjrd}E255MoAujT()V40YhW~nB#nWYV#g$q6pna|8 zy#M_i%=~MFNv{L(@7zp;>WT3Au&%tiPoVCRHLzfX0Uss5z{_dd(Q()!dFWarSZMu} z^|TmHbF=_lzIWq0&h&sC>w;u(_5;DrhjOd@S8QN-Gf~`hxB5Ib>jB~yAAgzL4P~d( zCq!+lhoYKrkoBEbVb4J}>~e=qV40gFizY;Bipw9P<UDq1u_cDYy&e*#MI|q1+ zzXwm_x9O>18FU*DFAHPQb_=yvX0_x4GHIWJ^Sa#l@{`w|UBx{Y)}YShc=);e zI@=R>3ifA&W1Gq#=^x)ihR2Yedk^7Vj$8--kwLud)p+>RxiK4%IfR!~%)vpM!thMk zB>w7X1AfBlF|Id@0Ky9xmhOG8*7OaVX!QsTdUS=AU30+m#~Jjr|AsVPezdq1-^ZPJ?BJZLaGP(W9cdk8qf$HTO|rxx@LFB9|Lr7=)exry^l5;d)_(V zA`V?K2or`c1?_;RkTbkJbb3D*U9Q9thRm@vT?g|#GGXI^uHuPJ8PeS2o)Ig!`kdWO zrop;t+eH2mI;R6C)W3P^khiFppa%hCy77two=E4Gcj>1N|Lavv%radCjb`nk@BVc7 z)4;Wun6d+@&M}VmVoKUrUFygMeC5L~IQd*3-r*5F3;M$%!v0$Py>+7?9I(Ho2AI55 zM~>Fr4smOjBk=%4zTCj3tuja5*IUKbBZK5=@0;Lk>i~P5E~B<#fB1IQjHz?;%?*?j zYCFmOrhS>>LRwE4*=K-YVz!)d$p|)uJfxal09Os$%kXm{&}r>2cIamx%xvYMX}bC) zs&yr-!NQ~^#HQt?KQ$33Y3T%S{&LBA2d*?)a%G2%pCS#Azjd-fCtPcMKb#9ym0 zwDm}lh&?t|8f{Bt4I^66wG{W@Lq6?WofE(ne+;xcgoJzUxiJKP<&>i5MjakjIuP_6 zmJ4XxTaGw%2A8d#0$+R{pkdw%s)(B*&Ia~>D9py5mSbOA$xVKT zv{oOT_(ZEQl4=-*{So24YAl`8I#68obNN8Nv~RKYeXmPMTqNJTpD)Zim`I(Ge;L)D zJl430oYOoDGFA+h_p?VcL*Eae)^@0f>&8JAQ7y-l~ zGWl^&PPL9dE$$Ql4#z$RPhqh<%mSZYWO@$2p~&wGR@8ChA40~!2l270=Kt1ZvPs`t zoVFFg?2v{K=^cfCZneOnPkL#JSHEHh`agmUOMl{?e|T=gc?=#ji8UWomVbP&n+(^^ zg8mR(&@ zTRI=l0OAac3bWx4HrJJN22Q|`Up1g(RJ!6DY}#xtam7WPa{HiYG|Pbn4cIKJ>uwX@ zt|qYE+UByje5|~4xgJj$)EG~v_u&r;3q+6X4Wt`kKv=~N1GnOgOKZjJf!5qHMnH$7 zv5;1(FZcJW$89gy1=2{mHzo{9(?gWLu)g?-4cXH{GbdsnENk`&TRqhgq=osh@9f2b zo;^U}EZ-zps`;x?lBapy=nk&E(f}=1MBuNMz#n_{=80`Duq?M!2r|E^xJMFJz{%SH zcdr`9h^s+)go|(JIYWAO+$?G#?se+NFAaPSQLmnY@*J0<7Ndn_JUX5W#xpM;pwg64 zav_WieTJlIbgnwQ`_gK>mTNtY(ooaPC`Mez+~ypr;&_@DpmX92EH|L4dwL!9NDd(3 z5UD<()k;HndBp@g7D;eE-j37x;91)_;#0~_w7HY6S?#+Yi9a-3%gul=E@pLcMZZ>D zlQJk-J7|-xtZ9E(s2bf<>4tUOcK~TRkXCAoS5HNr(v)knhRJ?Tmw>zxU*uQ_#EqzT zG8g6bBy_+uozh~ves?s1%k?1iHEC9L+T$w*FPPTA zz#b>j#jX;Fiytk#59y+LQlckh}~M-@t-rKDc2{I>rTD z(vttdXUQo@S||E8d4+}_4gz^8^wy@ikM=kMd*Zsu22(b%&EUvCo?nf_&U#5*pZef- zvl%_lcQqr=0bYOfB~M8uOz6mf$oX*1awE&RSX&Y|gVCoS@ODcacl?wI(~mEKuCFJ+ z)b#Tj(tGR~?W}ZC{@!Y;YJ}0cYiS$xH zCSHgI@}pdNZl#A*b3i^CD^2>4rai!IE^ATwX|-;7JN6>^M$Aew?0o zkR}6pe2wC%{1Oe%{J2CEhbD>gG-rO0?h8LO zx+@I6mdM`zUPJ!5Kc*e*j&B1lL(=Q#I4d(6h?`;J5IaHsfm7`&z6IqM`;@Ms{g9sl zWed_RADWv)B;F;z9sq<}sk`Kq;!|mKt~V%ewaBqMr#vEvt9j+30ug_67fiREtx>t< zZ6_;y^2t@>z3MR-=1pQ|@21Non%$tK`zZHLio=nj1qN5V7uwg>7!v&-(oad}3YVI@ zBI$#6XrpyFs%hL**C(yn4%?@v2=Y!WdX!cSeR2+-B{zZUJ9Y!Duh5Z|7}ak$ z%$wB$t`BVi9gmIS9`{_U^#2)aRh558vl!J9uJ?48Dz8z7Rlb{^%f13iBHKvv$>8^K zF;I=6!mG+^aV^|{YJl18a>Ra4ePv{NHNMrSgxTC_%U1{Auj2K~iP2nnuG=H?s{9q5 zyXe|%KfDN^0Vk}+a;iTKc^6d2r5aNCk{xOv1AYOk$j8O8QEh|y6K#dcuBwLk+@IAr z@vq7^KyxUkkLZ9Neb+MbXiC%hT|Im78(N`NS=7*XjwCOoyfKn@5xa+FAbB8mJ%2h< zE(ZO*ozUoC9a_^R5V-Od`&8XR(zw~g4{b!k@SmvilymrNcs+0%P{stp8_@i!^TMXG zji9kn=4p>QU)(#B*J7R4E`!snHdlFV!XONMGl|AD6h6jwm6uLozW>EY;ro8S;#%UF z|9G{g(_y3a7D1dK3M}gLrR8Jz-bwZ4^^eXtw&4|_J};@@C=5N8pfWfptx+D7>Ie&R zwralm9~WzMHetOHN73s;CG@_v8-{d0t}+7>=W?oFsDFGhBY!NY-odV+E^$;6QoRbQ zL3cV|)StNv<8RtwvUxNRZ{=T#`X;(fPsT~D$)8)60C6a57iNuW-W9I?JA8?)J@J=5 zw;`azU$HtbMKrE|5?;S+4(1&;u*$x5k@QFGXelVW-2`LrL8Nvlc<1ytDES#K**WzuBSDCos&)j5z1FhO~?+KMuqHgP(&^81)jM*VYCKYqF)$ zQKs~_Eo;F^U#fD$!k{~-GUdRCIY9ja42b-!`Xl61Es=UNNd89ghRVP|IRSs4 zIm4n}1>l?apS4uuf_{SH6Ur`}GC%o&Jh6DY6*o&Xkb7`GP)=5!TZ~)NT={Dtd?+r5 zDBBi5xI@Aa9D6Z>{Gc9?KB{cQt=8v>uNy<@HG5Tl5#$SSj>C@ZO64Fo>*M(P8=i{qJsxP~o0-w%uX-e&94GEn(eRhQ(~@Nr(MsF_uaYQ2i3 z6MS9sUik-iI#-Lfe|0X{>3LDt+x;LP(G2Lzrjc=v9;AE z)k{$8P}NhoXf_DRTcYxdi$1TY!baikHYyjfzjXy%eNv56mQomoJ5w#mZ|TDOpn6c; zl&+y8UZB6rk$tX#S4#~N9&u_}9aPyw<%RHJ-^A25vMH2 ziu?Mpq%{?|`Cc~lqg|Ndg+tAkLCpbUq4aKhX>Wc5sNRKXW^Ji*K6#)#;u_YulLZ)9 z#Ay2T?gs}qtp@AQ%YiUU8q$ToUHgZ1Y~;%*o6E`WJK6N(-+?qxE}EbZW>H_*dNX}# z*t&rB(`<;;YZFR;nuJG8|rixITzKwb(e5~ zr3qm%5f!&wJbE9!(f#=}wKedsvrt&J{ z-5e*W`iL}UE#*X^ju+q>hqZ?^W5gFgcmT5&|Afv(ebrlID;ut6qngg3>r;b-^7*x` zM$>C|vOdA}DO=Rz#$`t0$73(m`%--`g(psWA{4&|WvwTk@)6XBfV%ZO_>rNrRj*78 z8eO0vUkbUpjV1Xt4P_gx;=0r!GgJ;zJvHf2Q2~^ts`_iQ%?64RhfY9V2S^h!^yw-_ zoF}MnfUcf-?mnq@uyE)el@-K@=e6mcQhI*;$dycS?7C^z7%=7(^+Q}Fio$*2#gyrz6HO5msJTR>$>2rOcz9krKGv~zV7`LS!S2Z*;bUbn3ZX~b82J|!Ln*1GA`HIHF zwj2&+RctJJDuI^*xoo8xPUQjhfdIJ0SVWnM2yg=iLrGEB(P`;e{6d3z1jgj|;U(xOP z*Nrtuha!P8Dcro1!4BG`>{#dopxzqv$TmU3EvWtwX*A7iH~!-LX{7bi8l5{2 zW{HugdQr+teT+Q;$_J5lLX!Owpk6O0%yCQWC!jOA0a9*OK8h(%tGMjuopy9wBjwNB zRlO>ot@!(_?FZ^Pt`a>f_F&1N$@CcpNIqYZK1u3ru>gk^g1j}52UFe@yInO^y*5=h zVD|0}ew-MCs&An(BF!nE6VVciKK}*kD+1vMsfPhh3$-HfV<6V~uNU|CtdB7t zH^RYLjnu@ zVzX5iGPH9w-uh`nx@YnL+;r&%-4=dk3FB#RltznqdPy$3VKE(BA9##+mRs>Q{da1k zuZKhWj&$re-&E7Q$0!`p+@JqD>gB%pu_1K%Z!g-+z3ASed^h9Ur+^;-T>BP*(c|hKz89sl%qixq4yjm!mQ3 zyLBQy>Y&3roxcG8-d)EIwyu19h%WxQ(Oj;k=N35M9w99=^kFF0kOM~Cgt-CESi1cl zTE1!^8}%G016wBH1HWuE&kw{!=Q?0{jWO_MW&-!|{0*ngjo96y>)_?Gj`nUz9lULmPsL6=^n?k?Q@~$zAtck{Q}ni%4EF%YA?FAug<>D{vzfi zX)%BGy8J2NAr5w4$Va{@LKByCwRW^$l8<~J;x6{Di^AN&(eQEnNd8`QM;fy`%{AEU zt0qggt_C_UwDxPFG5xs+4h`rB`uE&mSCXX+5bGs<2Akni2Mls+;$6CTH)6Z4Iycx; zG)D%#UkYpLZ{e015x6TL78V{dzZjABys5